Allow registering Snowflake datasources from UI + enhancements

README.md

@@ -323,4 +323,3 @@ Unicons by IconScout: [https://github.com/Iconscout/unicons](https://github.com/
 - Mailing List: [https://dev.eclipse.org/mailman/listinfo/dirigible-dev](https://dev.eclipse.org/mailman/listinfo/dirigible-dev)
 - Issues: [https://github.com/eclipse/dirigible/issues](https://github.com/eclipse/dirigible/issues)
 - Eclipse Foundation Help Desk: https://gitlab.eclipse.org/eclipsefdn/helpdesk
-

components/data/data-source-snowpark/src/main/java/org/eclipse/dirigible/components/data/source/snowpark/SnowflakeDatabaseConfigurator.java

@@ -21,6 +21,7 @@
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Paths;
+import java.util.Objects;
 import java.util.concurrent.TimeUnit;
 
 @Component
@@ -37,51 +38,67 @@ public boolean isApplicable(DatabaseSystem databaseSystem) {
 
     @Override
     public void apply(HikariConfig config) {
+        setCommonConfigurations(config);
+
+        boolean registeredUsernameAndPass = StringUtils.isNotBlank(config.getUsername()) && StringUtils.isNotBlank(config.getPassword());
+
+        if (registeredUsernameAndPass && userAndPassAreNotDummyValues(config)) {
+            logger.info("There ARE registered username and pass for config [{}] and they  will be used.", config);
+            config.addDataSourceProperty("user", config.getUsername());
+            config.addDataSourceProperty("password", config.getPassword());
+
+        } else {
+            configureOAuth(config);
+        }
+    }
+
+    private void setCommonConfigurations(HikariConfig config) {
         config.setConnectionTestQuery("SELECT 1"); // connection validation query
         config.setKeepaliveTime(TimeUnit.MINUTES.toMillis(5)); // validation execution interval, must be bigger than idle timeout
         config.setMaxLifetime(TimeUnit.MINUTES.toMillis(9)); // recreate connections after specified time
         config.setLeakDetectionThreshold(TimeUnit.MINUTES.toMillis(5));
 
         config.addDataSourceProperty("CLIENT_SESSION_KEEP_ALIVE", true);
         config.addDataSourceProperty("CLIENT_SESSION_KEEP_ALIVE_HEARTBEAT_FREQUENCY", 900);
+    }
+
+    private void configureOAuth(HikariConfig config) {
+        if (!hasTokenFile()) {
+            throw new IllegalStateException("There in no username and/or password (or both are dummy values) for provided config [" + config
+                    + "]. Assuming it should use oauth token but there is no token file at " + TOKEN_FILE_PATH);
+        }
+
+        logger.info("Missing username and/or password for config [{}]. OAuth token will be used.", config);
+
+        config.setUsername(null);
+        config.setPassword(null);
+        config.addDataSourceProperty("authenticator", "OAUTH");
+        config.addDataSourceProperty("token", loadTokenFile());
 
         addDataSourcePropertyIfConfigAvailable("SNOWFLAKE_WAREHOUSE", "warehouse", config);
+
+        // automatically populated by Snowflake unless explicitly set
+        // https://docs.snowflake.com/en/developer-guide/snowpark-container-services/additional-considerations-services-jobs
         addDataSourcePropertyIfConfigAvailable("SNOWFLAKE_ACCOUNT", "account", config);
         addDataSourcePropertyIfConfigAvailable("SNOWFLAKE_DATABASE", "db", config);
         addDataSourcePropertyIfConfigAvailable("SNOWFLAKE_SCHEMA", "schema", config);
 
-        String url;
-        if (hasTokenFile()) {
-            logger.info("There IS token file. OAuth will be added to [{}]", config);
-
-            config.setUsername(null);
-            config.setPassword(null);
-            config.addDataSourceProperty("authenticator", "OAUTH");
-            config.addDataSourceProperty("token", loadTokenFile());
-            url = "jdbc:snowflake://" + Configuration.get("SNOWFLAKE_HOST") + ":" + Configuration.get("SNOWFLAKE_PORT");
-        } else {
-            logger.info("There is NO token file. User/password will be added to [{}]", config);
-
-            addDataSourcePropertyIfConfigAvailable("SNOWFLAKE_ROLE", "role", config);
-            addDataSourcePropertyIfConfigAvailable("SNOWFLAKE_USERNAME", "user", config);
-            addDataSourcePropertyIfConfigAvailable("SNOWFLAKE_PASSWORD", "password", config);
+        String url = "jdbc:snowflake://" + Configuration.get("SNOWFLAKE_HOST") + ":" + Configuration.get("SNOWFLAKE_PORT");
 
-            url = Configuration.get("SNOWFLAKE_URL", config.getJdbcUrl());
-        }
-        logger.info("Built url [{}]", url);
+        logger.info("Will be used url [{}] for config [{}]", url, config);
         config.addDataSourceProperty("url", url);
         config.setJdbcUrl(url);
     }
 
-    private static String loadTokenFile() {
+    private String loadTokenFile() {
         try {
             return new String(Files.readAllBytes(Paths.get(TOKEN_FILE_PATH)));
         } catch (IOException ex) {
             throw new IllegalStateException("Failed to load token file from path " + TOKEN_FILE_PATH, ex);
         }
     }
 
-    private static boolean hasTokenFile() {
+    private boolean hasTokenFile() {
         return Files.exists(Paths.get(TOKEN_FILE_PATH));
     }
 
@@ -95,4 +112,18 @@ private void addDataSourcePropertyIfConfigAvailable(String configName, String pr
         }
     }
 
+    private boolean userAndPassAreNotDummyValues(HikariConfig config) {
+        return isNotDummyValue(config.getUsername()) && isNotDummyValue(config.getPassword());
+    }
+
+    /**
+     * Note: needed for backward compatibility with Snowflake native applications until they are updated
+     *
+     * @param value
+     * @return
+     */
+    private boolean isNotDummyValue(String value) {
+        return !Objects.equals(value, "not-used-in-snowpark-scenario");
+    }
+
 }

components/data/data-sources/src/main/java/org/eclipse/dirigible/components/data/sources/domain/DataSource.java

@@ -9,22 +9,15 @@
  */
 package org.eclipse.dirigible.components.data.sources.domain;
 
-import java.util.ArrayList;
-import java.util.List;
+import com.google.gson.annotations.Expose;
+import jakarta.persistence.*;
 import org.eclipse.dirigible.components.base.artefact.Artefact;
 import org.eclipse.dirigible.components.base.encryption.Encrypted;
 import org.hibernate.annotations.LazyCollection;
 import org.hibernate.annotations.LazyCollectionOption;
-import com.google.gson.annotations.Expose;
-import jakarta.persistence.CascadeType;
-import jakarta.persistence.Column;
-import jakarta.persistence.Entity;
-import jakarta.persistence.FetchType;
-import jakarta.persistence.GeneratedValue;
-import jakarta.persistence.GenerationType;
-import jakarta.persistence.Id;
-import jakarta.persistence.OneToMany;
-import jakarta.persistence.Table;
+
+import java.util.ArrayList;
+import java.util.List;
 
 /**
  * The Class DataSource.
@@ -53,12 +46,12 @@ public class DataSource extends Artefact {
     private String url;
 
     /** The username. */
-    @Column(name = "DS_USERNAME", columnDefinition = "VARCHAR", nullable = false, length = 255)
+    @Column(name = "DS_USERNAME", columnDefinition = "VARCHAR", nullable = true, length = 255)
     @Expose
     private String username;
 
     /** The password. */
-    @Column(name = "DS_PASSWORD", columnDefinition = "VARCHAR", nullable = false, length = 255)
+    @Column(name = "DS_PASSWORD", columnDefinition = "VARCHAR", nullable = true, length = 255)
     @Expose
     @Encrypted
     private String password;

components/data/data-sources/src/main/java/org/eclipse/dirigible/components/data/sources/service/CustomDataSourcesService.java

@@ -38,57 +38,68 @@ public class CustomDataSourcesService {
     public void initialize() {
         String customDataSourcesList = Configuration.get("DIRIGIBLE_DATABASE_CUSTOM_DATASOURCES");
         if ((customDataSourcesList != null) && !"".equals(customDataSourcesList)) {
-            if (logger.isTraceEnabled()) {
-                logger.trace("Custom datasources list: " + customDataSourcesList);
-            }
+            logger.trace("Custom datasources list: [{}]", customDataSourcesList);
             StringTokenizer tokens = new StringTokenizer(customDataSourcesList, ",");
             while (tokens.hasMoreTokens()) {
                 String name = tokens.nextToken();
-                if (logger.isInfoEnabled()) {
-                    logger.info("Initializing a custom datasource with name: " + name);
-                }
+                logger.info("Initializing a custom datasource with name [{}]", name);
                 saveDataSource(name);
             }
         } else {
-            if (logger.isTraceEnabled()) {
-                logger.trace("No custom datasources configured");
-            }
-        }
-        if (logger.isDebugEnabled()) {
-            logger.debug(this.getClass()
-                             .getCanonicalName()
-                    + " module initialized.");
+            logger.trace("No custom datasources configured");
         }
+        logger.debug("[{}] module initialized.", this.getClass()
+                                                     .getCanonicalName());
     }
 
     /**
      * Save data source model.
      *
      * @param name the name
-     * @return the data source
      */
     private void saveDataSource(String name) {
-        String databaseDriver = Configuration.get(name + "_DRIVER");
-        String databaseUrl = Configuration.get(name + "_URL");
-        String databaseUsername = Configuration.get(name + "_USERNAME");
-        String databasePassword = Configuration.get(name + "_PASSWORD");
-        String databaseSchema = Configuration.get(name + "_SCHEMA");
+        String databaseDriver = getRequiredParameter(name, "DRIVER");
+        String databaseUrl = getRequiredParameter(name, "URL");
+        String databaseUsername = getOptionalParameter(name, "USERNAME");
+        String databasePassword = getOptionalParameter(name, "PASSWORD");
+        String databaseSchema = getOptionalParameter(name, "SCHEMA");
 
-        if ((databaseDriver != null) && (databaseUrl != null) && (databaseUsername != null) && (databasePassword != null)) {
-            org.eclipse.dirigible.components.data.sources.domain.DataSource ds =
-                    new org.eclipse.dirigible.components.data.sources.domain.DataSource("ENV_" + name, name, null, databaseDriver,
-                            databaseUrl, databaseUsername, databasePassword);
-            ds.setSchema(databaseSchema);
-            ds.updateKey();
-            ds.setLifecycle(ArtefactLifecycle.NEW);
-            DataSource maybe = dataSourceService.findByKey(ds.getKey());
-            if (maybe != null) {
-                dataSourceService.delete(maybe);
-            }
-            dataSourceService.save(ds);
-        } else {
-            throw new IllegalArgumentException("Invalid configuration for the custom datasource: " + name);
+        org.eclipse.dirigible.components.data.sources.domain.DataSource ds =
+                new org.eclipse.dirigible.components.data.sources.domain.DataSource("ENV_" + name, name, null, databaseDriver, databaseUrl,
+                        databaseUsername, databasePassword);
+        ds.setSchema(databaseSchema);
+        ds.updateKey();
+        ds.setLifecycle(ArtefactLifecycle.NEW);
+        DataSource maybe = dataSourceService.findByKey(ds.getKey());
+        if (maybe != null) {
+            dataSourceService.delete(maybe);
+        }
+        dataSourceService.save(ds);
+    }
+
+    private String getRequiredParameter(String dataSourceName, String suffix) {
+        String configName = createConfigName(dataSourceName, suffix);
+        String value = Configuration.get(configName);
+        if (null == value || value.trim()
+                                  .isEmpty()) {
+            throw new IllegalArgumentException("Missing required configuration parameter [" + configName + "] for data source ["
+                    + dataSourceName + "]. The value is: " + value);
+        }
+        return value;
+    }
+
+    private String createConfigName(String dataSourceName, String suffix) {
+        return dataSourceName + "_" + suffix;
+    }
+
+    private String getOptionalParameter(String dataSourceName, String suffix) {
+        String configName = createConfigName(dataSourceName, suffix);
+        String value = Configuration.get(configName);
+        if (null == value || value.trim()
+                                  .isEmpty()) {
+            logger.info("Optional parameter [{}] for data source [{}] is missing. The value is: [{}]", configName, dataSourceName, value);
         }
+        return value;
     }
 
 }

components/ide/ide-ui-databases/src/main/resources/META-INF/dirigible/ide-databases/dialogs/database-dialog.html

@@ -59,10 +59,9 @@
                 </fd-form-group>
                 <fd-form-group>
                     <fd-form-item>
-                        <fd-form-label for="username" dg-colon="true" dg-required="true">Username</fd-form-label>
+                        <fd-form-label for="username" dg-colon="true">Username</fd-form-label>
                         <fd-input id="username" name="username" type="text" placeholder="Enter username"
-                            ng-model="database.username" dg-input-rules="inputRules" ng-trim="false"
-                            state="{{ forms.dbForm['username'].$valid ? '' : 'error' }}" ng-required="true"></fd-input>
+                            ng-model="database.username"></fd-input>
                     </fd-form-item>
                 </fd-form-group>
                 <fd-form-group>
@@ -77,7 +76,7 @@
                         <fd-form-label for="parameters" dg-colon="true">Parameters</fd-form-label>
                         <fd-input id="parameters" name="parameters" type="text"
                             state="{{ forms.dbForm['parameters'].$valid ? '' : 'error' }}"
-                            dg-input-rules="{patterns: ['^([A-Za-z0-9=,]+)=([A-Za-z0-9]+?)(?=,[^,]+=|$)$']}"
+                            dg-input-rules="{patterns: ['^([A-Za-z0-9._-]+=[A-Za-z0-9._-]+)(,[A-Za-z0-9._-]+=[A-Za-z0-9._-]+)*$']}"
                             placeholder="Enter parameters, e.g. name1=value1,name2=value2 ..."
                             ng-model="database.parameters" ng-minlength="3"></fd-input>
                     </fd-form-item>

components/ide/ide-ui-databases/src/main/resources/META-INF/dirigible/ide-databases/dialogs/database-dialog.js

@@ -29,13 +29,23 @@ dbdialog.controller('DBDialogController', ['$scope', 'messageHub', 'ViewParamete
     $scope.editMode = false;
 
     $scope.urls = {
-        "org.h2.Driver": "jdbc:h2:path/name",
-        "org.postgresql.Driver": "jdbc:postgresql://host:port/database",
-        "com.mysql.cj.jdbc.Driver": "jdbc:mysql://host:port/database",
-        "org.mariadb.jdbc.Driver": "jdbc:mariadb://host:port/database",
-        "com.sap.db.jdbc.Driver": "jdbc:sap://host:port/?encrypt=true&validateCertificate=false",
-        "net.snowflake.client.jdbc.SnowflakeDriver": "jdbc:snowflake://account_identifier.snowflakecomputing.com/?db=SNOWFLAKE_SAMPLE_DATA&schema=TPCH_SF1000",
-        "org.eclipse.dirigible.mongodb.jdbc.Driver": "jdbc:mongodb://host:port/database",
+        "org.h2.Driver": "jdbc:h2:<path>/<name>",
+        "org.postgresql.Driver": "jdbc:postgresql://<host>:<port>/<database>",
+        "com.mysql.cj.jdbc.Driver": "jdbc:mysql://<host>:<port>/<database>",
+        "org.mariadb.jdbc.Driver": "jdbc:mariadb://<host>:<port>/<database>",
+        "com.sap.db.jdbc.Driver": "jdbc:sap://<host>:<port>/?encrypt=true&validateCertificate=false",
+        "net.snowflake.client.jdbc.SnowflakeDriver": "jdbc:snowflake://<account_identifier>.snowflakecomputing.com",
+        "org.eclipse.dirigible.mongodb.jdbc.Driver": "jdbc:mongodb://<host>:<port>/<database>",
+    };
+
+    $scope.parameters = {
+        'org.h2.Driver': '',
+        'org.postgresql.Driver': '',
+        'com.mysql.cj.jdbc.Driver': '',
+        'org.mariadb.jdbc.Driver': '',
+        'com.sap.db.jdbc.Driver': '',
+        'net.snowflake.client.jdbc.SnowflakeDriver': 'db=<database>,schema=<schema>',
+        'org.eclipse.dirigible.mongodb.jdbc.Driver': '',
     };
 
     $scope.drivers = [
@@ -61,6 +71,7 @@ dbdialog.controller('DBDialogController', ['$scope', 'messageHub', 'ViewParamete
         $scope.database.url = $scope.urls[$scope.database.driver];
         $scope.database.username = "";
         $scope.database.password = "";
+        $scope.database.parameters = $scope.parameters[$scope.database.driver];
     };
 
     function getTopic() {

components/platform-ide/view-databases/src/main/resources/META-INF/dirigible/view-databases/dialogs/database-dialog.html

@@ -51,9 +51,8 @@
                 </bk-form-group>
                 <bk-form-group>
                     <bk-form-item>
-                        <bk-form-label for="username" colon="true" required>Username</bk-form-label>
-                        <bk-input id="username" name="username" type="text" placeholder="Enter username" ng-model="database.username" input-rules="inputRules" ng-trim="false" state="{{ forms.dbForm['username'].$valid ? '' : 'error' }}"
-                            ng-required="true"></bk-input>
+                        <bk-form-label for="username" colon="true">Username</bk-form-label>
+                        <bk-input id="username" name="username" type="text" placeholder="Enter username" ng-model="database.username"></bk-input>
                     </bk-form-item>
                 </bk-form-group>
                 <bk-form-group>
@@ -65,7 +64,7 @@
                 <bk-form-group>
                     <bk-form-item>
                         <bk-form-label for="parameters" colon="true">Parameters</bk-form-label>
-                        <bk-input id="parameters" name="parameters" type="text" state="{{ forms.dbForm['parameters'].$valid ? '' : 'error' }}" input-rules="{patterns: ['^([A-Za-z0-9=,]+)=([A-Za-z0-9]+?)(?=,[^,]+=|$)$']}"
+                        <bk-input id="parameters" name="parameters" type="text" state="{{ forms.dbForm['parameters'].$valid ? '' : 'error' }}" input-rules="{patterns: ['^([A-Za-z0-9._-]+=[A-Za-z0-9._-]+)(,[A-Za-z0-9._-]+=[A-Za-z0-9._-]+)*$']}"
                             placeholder="Enter parameters, e.g. name1=value1,name2=value2 ..." ng-model="database.parameters" ng-minlength="3"></bk-input>
                     </bk-form-item>
                 </bk-form-group>

components/platform-ide/view-databases/src/main/resources/META-INF/dirigible/view-databases/dialogs/database-dialog.js

@@ -27,13 +27,23 @@ dbdialog.controller('DBDialogController', ($scope, ViewParameters, Dialogs) => {
     };
 
     $scope.urls = {
-        'org.h2.Driver': 'jdbc:h2:path/name',
-        'org.postgresql.Driver': 'jdbc:postgresql://host:port/database',
-        'com.mysql.cj.jdbc.Driver': 'jdbc:mysql://host:port/database',
-        'org.mariadb.jdbc.Driver': 'jdbc:mariadb://host:port/database',
-        'com.sap.db.jdbc.Driver': 'jdbc:sap://host:port/?encrypt=true&validateCertificate=false',
-        'net.snowflake.client.jdbc.SnowflakeDriver': 'jdbc:snowflake://account_identifier.snowflakecomputing.com/?db=SNOWFLAKE_SAMPLE_DATA&schema=TPCH_SF1000',
-        'org.eclipse.dirigible.mongodb.jdbc.Driver': 'jdbc:mongodb://host:port/database',
+        "org.h2.Driver": "jdbc:h2:<path>/<name>",
+        "org.postgresql.Driver": "jdbc:postgresql://<host>:<port>/<database>",
+        "com.mysql.cj.jdbc.Driver": "jdbc:mysql://<host>:<port>/<database>",
+        "org.mariadb.jdbc.Driver": "jdbc:mariadb://<host>:<port>/<database>",
+        "com.sap.db.jdbc.Driver": "jdbc:sap://<host>:<port>/?encrypt=true&validateCertificate=false",
+        "net.snowflake.client.jdbc.SnowflakeDriver": "jdbc:snowflake://<account_identifier>.snowflakecomputing.com",
+        "org.eclipse.dirigible.mongodb.jdbc.Driver": "jdbc:mongodb://<host>:<port>/<database>",
+    };
+
+    $scope.parameters = {
+        'org.h2.Driver': '',
+        'org.postgresql.Driver': '',
+        'com.mysql.cj.jdbc.Driver': '',
+        'org.mariadb.jdbc.Driver': '',
+        'com.sap.db.jdbc.Driver': '',
+        'net.snowflake.client.jdbc.SnowflakeDriver': 'db=<database>,schema=<schema>',
+        'org.eclipse.dirigible.mongodb.jdbc.Driver': '',
     };
 
     $scope.drivers = [
@@ -59,6 +69,7 @@ dbdialog.controller('DBDialogController', ($scope, ViewParameters, Dialogs) => {
         $scope.database.url = $scope.urls[$scope.database.driver];
         $scope.database.username = '';
         $scope.database.password = '';
+        $scope.database.parameters = $scope.parameters[$scope.database.driver];
     };
 
     $scope.save = () => {