Skip to content

Commit

Permalink
Allow registering Snowflake datasources from UI + enhancements (#4585)
Browse files Browse the repository at this point in the history 
* support more advanced datasource properties

 Like the one in snowflake - https://docs.snowflake.com/en/developer-guide/jdbc/jdbc-parameters

Signed-off-by: Iliyan Velichkov <velichkov.iliyan@gmail.com>

* update Snowflake database registation template

Signed-off-by: Iliyan Velichkov <velichkov.iliyan@gmail.com>

* make user and password optional

There are scenarios where user and password are optional. For example in Snowflake where token file could be used instead.

Signed-off-by: Iliyan Velichkov <velichkov.iliyan@gmail.com>

* user/pass no longer required param for env data sources

Signed-off-by: Iliyan Velichkov <velichkov.iliyan@gmail.com>

* refactoring

Signed-off-by: Iliyan Velichkov <velichkov.iliyan@gmail.com>

* use env variables for oauth only

Signed-off-by: Iliyan Velichkov <velichkov.iliyan@gmail.com>

* formatting

Signed-off-by: Iliyan Velichkov <velichkov.iliyan@gmail.com>

---------

Signed-off-by: Iliyan Velichkov <velichkov.iliyan@gmail.com>
  • Loading branch information
@iliyan-velichkov
iliyan-velichkov authored Jan 23, 2025
1 parent cdaaa99 commit 44261be
Show file tree
Hide file tree
Showing 8 changed files with 145 additions and 91 deletions.
1 change: 0 additions & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -323,4 +323,3 @@ Unicons by IconScout: [https://github.com/Iconscout/unicons](https://github.com/
- Mailing List: [https://dev.eclipse.org/mailman/listinfo/dirigible-dev](https://dev.eclipse.org/mailman/listinfo/dirigible-dev)
- Issues: [https://github.com/eclipse/dirigible/issues](https://github.com/eclipse/dirigible/issues)
- Eclipse Foundation Help Desk: https://gitlab.eclipse.org/eclipsefdn/helpdesk
71 changes: 51 additions & 20 deletions .../org/eclipse/dirigible/components/data/source/snowpark/SnowflakeDatabaseConfigurator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.Objects;
import java.util.concurrent.TimeUnit;

@Component
Expand All @@ -37,51 +38,67 @@ public boolean isApplicable(DatabaseSystem databaseSystem) {

@Override
public void apply(HikariConfig config) {
setCommonConfigurations(config);

boolean registeredUsernameAndPass = StringUtils.isNotBlank(config.getUsername()) && StringUtils.isNotBlank(config.getPassword());

if (registeredUsernameAndPass && userAndPassAreNotDummyValues(config)) {
logger.info("There ARE registered username and pass for config [{}] and they will be used.", config);
config.addDataSourceProperty("user", config.getUsername());
config.addDataSourceProperty("password", config.getPassword());

} else {
configureOAuth(config);
}
}

private void setCommonConfigurations(HikariConfig config) {
config.setConnectionTestQuery("SELECT 1"); // connection validation query
config.setKeepaliveTime(TimeUnit.MINUTES.toMillis(5)); // validation execution interval, must be bigger than idle timeout
config.setMaxLifetime(TimeUnit.MINUTES.toMillis(9)); // recreate connections after specified time
config.setLeakDetectionThreshold(TimeUnit.MINUTES.toMillis(5));

config.addDataSourceProperty("CLIENT_SESSION_KEEP_ALIVE", true);
config.addDataSourceProperty("CLIENT_SESSION_KEEP_ALIVE_HEARTBEAT_FREQUENCY", 900);
}

private void configureOAuth(HikariConfig config) {
if (!hasTokenFile()) {
throw new IllegalStateException("There in no username and/or password (or both are dummy values) for provided config [" + config
+ "]. Assuming it should use oauth token but there is no token file at " + TOKEN_FILE_PATH);
}

logger.info("Missing username and/or password for config [{}]. OAuth token will be used.", config);

config.setUsername(null);
config.setPassword(null);
config.addDataSourceProperty("authenticator", "OAUTH");
config.addDataSourceProperty("token", loadTokenFile());

addDataSourcePropertyIfConfigAvailable("SNOWFLAKE_WAREHOUSE", "warehouse", config);

// automatically populated by Snowflake unless explicitly set
// https://docs.snowflake.com/en/developer-guide/snowpark-container-services/additional-considerations-services-jobs
addDataSourcePropertyIfConfigAvailable("SNOWFLAKE_ACCOUNT", "account", config);
addDataSourcePropertyIfConfigAvailable("SNOWFLAKE_DATABASE", "db", config);
addDataSourcePropertyIfConfigAvailable("SNOWFLAKE_SCHEMA", "schema", config);

String url;
if (hasTokenFile()) {
logger.info("There IS token file. OAuth will be added to [{}]", config);

config.setUsername(null);
config.setPassword(null);
config.addDataSourceProperty("authenticator", "OAUTH");
config.addDataSourceProperty("token", loadTokenFile());
url = "jdbc:snowflake://" + Configuration.get("SNOWFLAKE_HOST") + ":" + Configuration.get("SNOWFLAKE_PORT");
} else {
logger.info("There is NO token file. User/password will be added to [{}]", config);

addDataSourcePropertyIfConfigAvailable("SNOWFLAKE_ROLE", "role", config);
addDataSourcePropertyIfConfigAvailable("SNOWFLAKE_USERNAME", "user", config);
addDataSourcePropertyIfConfigAvailable("SNOWFLAKE_PASSWORD", "password", config);
String url = "jdbc:snowflake://" + Configuration.get("SNOWFLAKE_HOST") + ":" + Configuration.get("SNOWFLAKE_PORT");

url = Configuration.get("SNOWFLAKE_URL", config.getJdbcUrl());
}
logger.info("Built url [{}]", url);
logger.info("Will be used url [{}] for config [{}]", url, config);
config.addDataSourceProperty("url", url);
config.setJdbcUrl(url);
}

private static String loadTokenFile() {
private String loadTokenFile() {
try {
return new String(Files.readAllBytes(Paths.get(TOKEN_FILE_PATH)));
} catch (IOException ex) {
throw new IllegalStateException("Failed to load token file from path " + TOKEN_FILE_PATH, ex);
}
}

private static boolean hasTokenFile() {
private boolean hasTokenFile() {
return Files.exists(Paths.get(TOKEN_FILE_PATH));
}

Expand All @@ -95,4 +112,18 @@ private void addDataSourcePropertyIfConfigAvailable(String configName, String pr
}
}

private boolean userAndPassAreNotDummyValues(HikariConfig config) {
return isNotDummyValue(config.getUsername()) && isNotDummyValue(config.getPassword());
}

/**
* Note: needed for backward compatibility with Snowflake native applications until they are updated
*
* @param value
* @return
*/
private boolean isNotDummyValue(String value) {
return !Objects.equals(value, "not-used-in-snowpark-scenario");
}

}
21 changes: 7 additions & 14 deletions ...ources/src/main/java/org/eclipse/dirigible/components/data/sources/domain/DataSource.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,22 +9,15 @@
*/
package org.eclipse.dirigible.components.data.sources.domain;

import java.util.ArrayList;
import java.util.List;
import com.google.gson.annotations.Expose;
import jakarta.persistence.*;
import org.eclipse.dirigible.components.base.artefact.Artefact;
import org.eclipse.dirigible.components.base.encryption.Encrypted;
import org.hibernate.annotations.LazyCollection;
import org.hibernate.annotations.LazyCollectionOption;
import com.google.gson.annotations.Expose;
import jakarta.persistence.CascadeType;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.FetchType;
import jakarta.persistence.GeneratedValue;
import jakarta.persistence.GenerationType;
import jakarta.persistence.Id;
import jakarta.persistence.OneToMany;
import jakarta.persistence.Table;

import java.util.ArrayList;
import java.util.List;

/**
* The Class DataSource.
Expand Down Expand Up @@ -53,12 +46,12 @@ public class DataSource extends Artefact {
private String url;

/** The username. */
@Column(name = "DS_USERNAME", columnDefinition = "VARCHAR", nullable = false, length = 255)
@Column(name = "DS_USERNAME", columnDefinition = "VARCHAR", nullable = true, length = 255)
@Expose
private String username;

/** The password. */
@Column(name = "DS_PASSWORD", columnDefinition = "VARCHAR", nullable = false, length = 255)
@Column(name = "DS_PASSWORD", columnDefinition = "VARCHAR", nullable = true, length = 255)
@Expose
@Encrypted
private String password;
Expand Down
79 changes: 45 additions & 34 deletions .../java/org/eclipse/dirigible/components/data/sources/service/CustomDataSourcesService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,57 +38,68 @@ public class CustomDataSourcesService {
public void initialize() {
String customDataSourcesList = Configuration.get("DIRIGIBLE_DATABASE_CUSTOM_DATASOURCES");
if ((customDataSourcesList != null) && !"".equals(customDataSourcesList)) {
if (logger.isTraceEnabled()) {
logger.trace("Custom datasources list: " + customDataSourcesList);
}
logger.trace("Custom datasources list: [{}]", customDataSourcesList);
StringTokenizer tokens = new StringTokenizer(customDataSourcesList, ",");
while (tokens.hasMoreTokens()) {
String name = tokens.nextToken();
if (logger.isInfoEnabled()) {
logger.info("Initializing a custom datasource with name: " + name);
}
logger.info("Initializing a custom datasource with name [{}]", name);
saveDataSource(name);
}
} else {
if (logger.isTraceEnabled()) {
logger.trace("No custom datasources configured");
}
}
if (logger.isDebugEnabled()) {
logger.debug(this.getClass()
.getCanonicalName()
+ " module initialized.");
logger.trace("No custom datasources configured");
}
logger.debug("[{}] module initialized.", this.getClass()
.getCanonicalName());
}

/**
* Save data source model.
*
* @param name the name
* @return the data source
*/
private void saveDataSource(String name) {
String databaseDriver = Configuration.get(name + "_DRIVER");
String databaseUrl = Configuration.get(name + "_URL");
String databaseUsername = Configuration.get(name + "_USERNAME");
String databasePassword = Configuration.get(name + "_PASSWORD");
String databaseSchema = Configuration.get(name + "_SCHEMA");
String databaseDriver = getRequiredParameter(name, "DRIVER");
String databaseUrl = getRequiredParameter(name, "URL");
String databaseUsername = getOptionalParameter(name, "USERNAME");
String databasePassword = getOptionalParameter(name, "PASSWORD");
String databaseSchema = getOptionalParameter(name, "SCHEMA");

if ((databaseDriver != null) && (databaseUrl != null) && (databaseUsername != null) && (databasePassword != null)) {
org.eclipse.dirigible.components.data.sources.domain.DataSource ds =
new org.eclipse.dirigible.components.data.sources.domain.DataSource("ENV_" + name, name, null, databaseDriver,
databaseUrl, databaseUsername, databasePassword);
ds.setSchema(databaseSchema);
ds.updateKey();
ds.setLifecycle(ArtefactLifecycle.NEW);
DataSource maybe = dataSourceService.findByKey(ds.getKey());
if (maybe != null) {
dataSourceService.delete(maybe);
}
dataSourceService.save(ds);
} else {
throw new IllegalArgumentException("Invalid configuration for the custom datasource: " + name);
org.eclipse.dirigible.components.data.sources.domain.DataSource ds =
new org.eclipse.dirigible.components.data.sources.domain.DataSource("ENV_" + name, name, null, databaseDriver, databaseUrl,
databaseUsername, databasePassword);
ds.setSchema(databaseSchema);
ds.updateKey();
ds.setLifecycle(ArtefactLifecycle.NEW);
DataSource maybe = dataSourceService.findByKey(ds.getKey());
if (maybe != null) {
dataSourceService.delete(maybe);
}
dataSourceService.save(ds);
}

private String getRequiredParameter(String dataSourceName, String suffix) {
String configName = createConfigName(dataSourceName, suffix);
String value = Configuration.get(configName);
if (null == value || value.trim()
.isEmpty()) {
throw new IllegalArgumentException("Missing required configuration parameter [" + configName + "] for data source ["
+ dataSourceName + "]. The value is: " + value);
}
return value;
}

private String createConfigName(String dataSourceName, String suffix) {
return dataSourceName + "_" + suffix;
}

private String getOptionalParameter(String dataSourceName, String suffix) {
String configName = createConfigName(dataSourceName, suffix);
String value = Configuration.get(configName);
if (null == value || value.trim()
.isEmpty()) {
logger.info("Optional parameter [{}] for data source [{}] is missing. The value is: [{}]", configName, dataSourceName, value);
}
return value;
}

}
7 changes: 3 additions & 4 deletions ...atabases/src/main/resources/META-INF/dirigible/ide-databases/dialogs/database-dialog.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,10 +59,9 @@
</fd-form-group>
<fd-form-group>
<fd-form-item>
<fd-form-label for="username" dg-colon="true" dg-required="true">Username</fd-form-label>
<fd-form-label for="username" dg-colon="true">Username</fd-form-label>
<fd-input id="username" name="username" type="text" placeholder="Enter username"
ng-model="database.username" dg-input-rules="inputRules" ng-trim="false"
state="{{ forms.dbForm['username'].$valid ? '' : 'error' }}" ng-required="true"></fd-input>
ng-model="database.username"></fd-input>
</fd-form-item>
</fd-form-group>
<fd-form-group>
Expand All @@ -77,7 +76,7 @@
<fd-form-label for="parameters" dg-colon="true">Parameters</fd-form-label>
<fd-input id="parameters" name="parameters" type="text"
state="{{ forms.dbForm['parameters'].$valid ? '' : 'error' }}"
dg-input-rules="{patterns: ['^([A-Za-z0-9=,]+)=([A-Za-z0-9]+?)(?=,[^,]+=|$)$']}"
dg-input-rules="{patterns: ['^([A-Za-z0-9._-]+=[A-Za-z0-9._-]+)(,[A-Za-z0-9._-]+=[A-Za-z0-9._-]+)*$']}"
placeholder="Enter parameters, e.g. name1=value1,name2=value2 ..."
ng-model="database.parameters" ng-minlength="3"></fd-input>
</fd-form-item>
Expand Down
25 changes: 18 additions & 7 deletions ...-databases/src/main/resources/META-INF/dirigible/ide-databases/dialogs/database-dialog.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,23 @@ dbdialog.controller('DBDialogController', ['$scope', 'messageHub', 'ViewParamete
$scope.editMode = false;

$scope.urls = {
"org.h2.Driver": "jdbc:h2:path/name",
"org.postgresql.Driver": "jdbc:postgresql://host:port/database",
"com.mysql.cj.jdbc.Driver": "jdbc:mysql://host:port/database",
"org.mariadb.jdbc.Driver": "jdbc:mariadb://host:port/database",
"com.sap.db.jdbc.Driver": "jdbc:sap://host:port/?encrypt=true&validateCertificate=false",
"net.snowflake.client.jdbc.SnowflakeDriver": "jdbc:snowflake://account_identifier.snowflakecomputing.com/?db=SNOWFLAKE_SAMPLE_DATA&schema=TPCH_SF1000",
"org.eclipse.dirigible.mongodb.jdbc.Driver": "jdbc:mongodb://host:port/database",
"org.h2.Driver": "jdbc:h2:<path>/<name>",
"org.postgresql.Driver": "jdbc:postgresql://<host>:<port>/<database>",
"com.mysql.cj.jdbc.Driver": "jdbc:mysql://<host>:<port>/<database>",
"org.mariadb.jdbc.Driver": "jdbc:mariadb://<host>:<port>/<database>",
"com.sap.db.jdbc.Driver": "jdbc:sap://<host>:<port>/?encrypt=true&validateCertificate=false",
"net.snowflake.client.jdbc.SnowflakeDriver": "jdbc:snowflake://<account_identifier>.snowflakecomputing.com",
"org.eclipse.dirigible.mongodb.jdbc.Driver": "jdbc:mongodb://<host>:<port>/<database>",
};

$scope.parameters = {
'org.h2.Driver': '',
'org.postgresql.Driver': '',
'com.mysql.cj.jdbc.Driver': '',
'org.mariadb.jdbc.Driver': '',
'com.sap.db.jdbc.Driver': '',
'net.snowflake.client.jdbc.SnowflakeDriver': 'db=<database>,schema=<schema>',
'org.eclipse.dirigible.mongodb.jdbc.Driver': '',
};

$scope.drivers = [
Expand All @@ -61,6 +71,7 @@ dbdialog.controller('DBDialogController', ['$scope', 'messageHub', 'ViewParamete
$scope.database.url = $scope.urls[$scope.database.driver];
$scope.database.username = "";
$scope.database.password = "";
$scope.database.parameters = $scope.parameters[$scope.database.driver];
};

function getTopic() {
Expand Down
7 changes: 3 additions & 4 deletions ...tabases/src/main/resources/META-INF/dirigible/view-databases/dialogs/database-dialog.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,8 @@
</bk-form-group>
<bk-form-group>
<bk-form-item>
<bk-form-label for="username" colon="true" required>Username</bk-form-label>
<bk-input id="username" name="username" type="text" placeholder="Enter username" ng-model="database.username" input-rules="inputRules" ng-trim="false" state="{{ forms.dbForm['username'].$valid ? '' : 'error' }}"
ng-required="true"></bk-input>
<bk-form-label for="username" colon="true">Username</bk-form-label>
<bk-input id="username" name="username" type="text" placeholder="Enter username" ng-model="database.username"></bk-input>
</bk-form-item>
</bk-form-group>
<bk-form-group>
Expand All @@ -65,7 +64,7 @@
<bk-form-group>
<bk-form-item>
<bk-form-label for="parameters" colon="true">Parameters</bk-form-label>
<bk-input id="parameters" name="parameters" type="text" state="{{ forms.dbForm['parameters'].$valid ? '' : 'error' }}" input-rules="{patterns: ['^([A-Za-z0-9=,]+)=([A-Za-z0-9]+?)(?=,[^,]+=|$)$']}"
<bk-input id="parameters" name="parameters" type="text" state="{{ forms.dbForm['parameters'].$valid ? '' : 'error' }}" input-rules="{patterns: ['^([A-Za-z0-9._-]+=[A-Za-z0-9._-]+)(,[A-Za-z0-9._-]+=[A-Za-z0-9._-]+)*$']}"
placeholder="Enter parameters, e.g. name1=value1,name2=value2 ..." ng-model="database.parameters" ng-minlength="3"></bk-input>
</bk-form-item>
</bk-form-group>
Expand Down
25 changes: 18 additions & 7 deletions ...databases/src/main/resources/META-INF/dirigible/view-databases/dialogs/database-dialog.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,23 @@ dbdialog.controller('DBDialogController', ($scope, ViewParameters, Dialogs) => {
};

$scope.urls = {
'org.h2.Driver': 'jdbc:h2:path/name',
'org.postgresql.Driver': 'jdbc:postgresql://host:port/database',
'com.mysql.cj.jdbc.Driver': 'jdbc:mysql://host:port/database',
'org.mariadb.jdbc.Driver': 'jdbc:mariadb://host:port/database',
'com.sap.db.jdbc.Driver': 'jdbc:sap://host:port/?encrypt=true&validateCertificate=false',
'net.snowflake.client.jdbc.SnowflakeDriver': 'jdbc:snowflake://account_identifier.snowflakecomputing.com/?db=SNOWFLAKE_SAMPLE_DATA&schema=TPCH_SF1000',
'org.eclipse.dirigible.mongodb.jdbc.Driver': 'jdbc:mongodb://host:port/database',
"org.h2.Driver": "jdbc:h2:<path>/<name>",
"org.postgresql.Driver": "jdbc:postgresql://<host>:<port>/<database>",
"com.mysql.cj.jdbc.Driver": "jdbc:mysql://<host>:<port>/<database>",
"org.mariadb.jdbc.Driver": "jdbc:mariadb://<host>:<port>/<database>",
"com.sap.db.jdbc.Driver": "jdbc:sap://<host>:<port>/?encrypt=true&validateCertificate=false",
"net.snowflake.client.jdbc.SnowflakeDriver": "jdbc:snowflake://<account_identifier>.snowflakecomputing.com",
"org.eclipse.dirigible.mongodb.jdbc.Driver": "jdbc:mongodb://<host>:<port>/<database>",
};

$scope.parameters = {
'org.h2.Driver': '',
'org.postgresql.Driver': '',
'com.mysql.cj.jdbc.Driver': '',
'org.mariadb.jdbc.Driver': '',
'com.sap.db.jdbc.Driver': '',
'net.snowflake.client.jdbc.SnowflakeDriver': 'db=<database>,schema=<schema>',
'org.eclipse.dirigible.mongodb.jdbc.Driver': '',
};

$scope.drivers = [
Expand All @@ -59,6 +69,7 @@ dbdialog.controller('DBDialogController', ($scope, ViewParameters, Dialogs) => {
$scope.database.url = $scope.urls[$scope.database.driver];
$scope.database.username = '';
$scope.database.password = '';
$scope.database.parameters = $scope.parameters[$scope.database.driver];
};

$scope.save = () => {
Expand Down

0 comments on commit 44261be

Please sign in to comment.