From 543b65a3cca1e712d61c694b39652b77b841a39a Mon Sep 17 00:00:00 2001 From: Bruce Irschick Date: Mon, 12 Dec 2022 11:07:19 -0800 Subject: [PATCH 01/18] [AD-1014] Developer Guide. (#451) * [AD-1014] Developer Guide. * Commit Code Coverage Badge * [AD-1014] Updates to use existing GETTING_STARTED.md and added schema-caching.md * Commit Code Coverage Badge Co-authored-by: birschick-bq --- GETTING_STARTED.md | 102 +++++++++++++++++------ README.md | 7 +- src/markdown/index.md | 6 ++ src/markdown/schema/schema-caching.md | 111 ++++++++++++++++++++++++++ 4 files changed, 202 insertions(+), 24 deletions(-) create mode 100644 src/markdown/schema/schema-caching.md diff --git a/GETTING_STARTED.md b/GETTING_STARTED.md index a13bf849..34c4aa24 100644 --- a/GETTING_STARTED.md +++ b/GETTING_STARTED.md @@ -156,8 +156,45 @@ rather than the cluster endpoint since we have set up the SSH tunnel. ~~~ mongo --host 127.0.0.1:27017 --username --password ~~~ + +## Database User Account Definitions + +The integration tests assume the following two user accounts are created +in the target database server. + +### Administrative User + +User: `documentdb` + +#### Definition: + +```json +{ + "user" : "documentdb", + "roles" : [ { + "db" : "admin", + "role" : "root" + } ] +} +``` + +### Restricted Access User + +User: `docDbRestricted` + +#### Definition + +```json +{ + "user" : "docDbRestricted", + "roles" : [ { + "db" : "admin", + "role" : "readAnyDatabase" + } ] +} +``` -##### Connect with TLS +## Connect with TLS When connecting to a TLS-enabled cluster you can follow the same steps to set up an SSH tunnel but will need to also download the Amazon DocumentDB Certificate Authority (CA) file before trying to connect. 1. Download the CA file. @@ -178,8 +215,8 @@ access the cluster from localhost, the server certificate does not match the hos mongo --host 127.0.0.1:27017 --username --password --tls --tlsCAFile rds-combined-ca-bundle.pem --tlsAllowInvalidHostnames ~~~ -##### Connect Programmatically -###### Without TLS +### Connect Programmatically +#### Without TLS Connecting without TLS is very straightforward. We essentially follow the same steps as when connecting using the `mongo` shell. 1. Setup the SSH tunnel. See step 3 in section [Setting Up Environment Variables](#setting-up-environment-variables) for @@ -201,7 +238,7 @@ Make sure to set the hostname, username, password and target database. The targe } ~~~ -###### With TLS +#### With TLS Connecting with TLS programmatically is slightly different from how we did it with the `mongo` shell. 1. Create a test or simple main to run. 2. Use either the Driver Manager, Data Source class or Connection class to establish a connection to `localhost:27017`. @@ -224,36 +261,57 @@ class: } ~~~ -#### Setting Up Environment Variables -1. Create and set the Environment Variables: +## Integration Testing - ~~~ - DOC_DB_USER_NAME= - DOC_DB_PASSWORD= - DOC_DB_LOCAL_PORT=27019 - DOC_DB_USER=@ - DOC_DB_HOST= - DOC_DB_PRIV_KEY_FILE=~/.ssh/.pem - ~~~ +By default, integration testing is disabled for local development. To enable +integration testing, follow the directions below. + +### Setting Up Environment Variables -2. Ensure the private key file .pem is in the location set by the environment variable +To enable integration testing the following environment variables allow +you to customize the credentials and DocumentDB cluster settings. + +1. Create and set the following environment variables: + +| Variable | Description | Example | +|------------------------|--------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------| +| `DOC_DB_USER_NAME` | This is the DocumentDB user. | `documentdb` | +| `DOC_DB_PASSWORD` | This is the DocumentDB password. | `aSecret` | +| `DOC_DB_LOCAL_PORT` | This is the port number used locally via an SSH Tunnel. It is recommend to use a different value than the default 27017. | `27019` | +| `DOC_DB_USER` | This is the user and host of SSH Tunnel EC2 instance. | `ec2-user@254.254.254.254` | +| `DOC_DB_HOST` | This is the host of the DocumentDB cluster server. | `docdb-jdbc-literal-test.cluster-abcdefghijk.us-east-2.docdb.amazonaws.com` | +| `DOC_DB_PRIV_KEY_FILE` | This is the path to the SSH Tunnel private key-pair file. | `~/.ssh/ec2-literal.pem` | + +### SSH Tunnel + +1. Ensure the private key file .pem is in the location set by the environment variable `DOC_DB_PRIV_KEY_FILE`. -3. Start an SSH port-forwarding tunnel: +2. Assuming you have the environment variables setup above, starting an SSH tunnel from the command line should look like this: + ~~~shell + ssh [-f] -N -i $DOC_DB_PRIV_KEY_FILE -L $DOC_DB_LOCAL_PORT:$DOC_DB_HOST:27017 $DOC_DB_USER ~~~ - ssh [-f] -N -i ~/.ssh/.pem -L $DOC_DB_LOCAL_PORT:$DOC_DB_HOST:27017 $DOC_DB_USER - ~~~ - + - The `-L` flag defines the port forwarded to the remote host and remote port. Adding the `-N` flag means do not execute a remote command, you will not get a shell in this case. The `-f` switch instructs SSH to run in the background. -#### Bypass Testing DocumentDB +### Enable Integration Testing of Amazon DocumentDB + +To enable integration testing in the IDE, update the grade property, as intructed below. + 1. Modify the */gradle.properties* file in the source code and uncomment the following line: -`runRemoteIntegrationTests=false` +`runRemoteIntegrationTests=true` + +### Project Secrets + +For the purposes of automated integration testing in **GitHub**, this project maintains the value for the environment variables above +as project secrets. See the workflow file [gradle.yml](https://github.com/aws/amazon-documentdb-jdbc-driver/blob/1edd9e21fdcccfe62d366580702f2904136298e5/.github/workflows/gradle.yml) ## Troubleshooting + ### Issues with JDK + 1. Confirm project SDK is Java Version 1.8 via the IntelliJ top menu toolbar under *File → Project Structure → Platform Settings -> SDK* and reload the JDK home path by browsing to the path and click *apply* and *ok*. Restart IntelliJ IDEA. @@ -277,5 +335,3 @@ class: below. Go to EC2 Dashboard → **Network & Security** Group in the left menu → **Security** Group. ![Security Policy for EC2 Instance](src/markdown/images/getting-started/security-policy-ec2-instance.png) - - \ No newline at end of file diff --git a/README.md b/README.md index 9799a5b3..467c8c14 100644 --- a/README.md +++ b/README.md @@ -67,4 +67,9 @@ your issue. ## Security Notice -If you discover a potential security issue in this project, please consult our [security guidance page](SECURITY.md). \ No newline at end of file +If you discover a potential security issue in this project, please consult our [security guidance page](SECURITY.md). + +## Contributor's Getting Started Guide + +If you're a developer and want to contribute to this project, ensure to read and follow the +[Getting Started as a Developer](GETTING_STARTED.md) guide. diff --git a/src/markdown/index.md b/src/markdown/index.md index fc4c9a1d..d1b6d7d6 100644 --- a/src/markdown/index.md +++ b/src/markdown/index.md @@ -51,6 +51,12 @@ The Amazon DocumentDB JDBC driver can perform automatic schema discovery and gen DocumentDB schema mapping. See the [schema discovery documentation](schema/schema-discovery.md) for more details of this process. +## Schema Caching + +Once schema is discovered, it is cached in the database to improve performance for subsequent access. +See the [schema caching documentation](schema/schema-caching.md) to learn +more about schema caching behaviour and access requirements. + ## Schema Management The SQL to DocumentDB schema mapping can be managed in the following ways: diff --git a/src/markdown/schema/schema-caching.md b/src/markdown/schema/schema-caching.md new file mode 100644 index 00000000..7547af7d --- /dev/null +++ b/src/markdown/schema/schema-caching.md @@ -0,0 +1,111 @@ +# Schema Caching + +## Schema Caching Behaviour + +When a connection is made to an Amazon DocumentDB database, the Amazon DocumentDB JDBC driver +checks for a previously cached version of the mapped schema. If a previous version exists, +the latest version of the cached schema is read and used for all further interaction with the database. + +If a previously cached version does not exist, the process of [schema discover](schema-discovery.md) is automatically +started on all the accessible collections in the database. The discovery process uses the properties +`scanMethod` (default `random`), and `scanLimit` (default `1000`) when sampling documents from the database. +At the end of the discovery process, the resulting schema mapping is written to the cache using the name +associated with the property `schemaName` (default `_default`). + +If some reason the resulting schema cannot be saved to the cache, the resulting schema will still be used +in-memory for the life of the connection. The implication of not having access to a cached version of the +schema is that the schema discovery will have to be performed for each connection - which could have a seriously +negative impact on performance. + +## Cache Location + +The SQL schema mapping cache is stored in two collections on the same database as +the sampled collections. The collection `_sqlSchemas` stores the names and versions of +all the sampled schemas for the given database. The collection `_sqlTableSchemas` stores the +column to field mappings for all the cached SQL schema mappings. The two cache collections +have a strong parent/child relationship and must be maintained in a consistent way. Always use +the [schema management CLI](manage-schema-cli.md) to ensure consistency in the cache collections. + +## User Permissions for Creating and Updating the Schema Cache + +To be able to store or update the SQL schema mappings to the cache collections, the connected +Amazon DocumentDB user account must have write permissions to create and update the +cache collections. Once the schema is cached, users need only read permission on the +cache collections. + +To allow access for an Amazon DocumentDB user, ensure to set or add the appropriate roles as +described below. + +### Enable Access per Database + +To allow read and write access to specific databases in your server, add +a `readWrite` [built-in role](https://www.mongodb.com/docs/manual/reference/built-in-roles/#mongodb-authrole-readWrite) +for each database the user should have access to be able to create and update the cached schema for specific +databases. + +```json +roles: [ + {role: "readWrite", db: "yourDatabase1"}, + {role: "readWrite", db: "yourDatabase2"} ... +] +``` + +### Enable Access for Any Database + +To allow read and write access to any databases in your server, add +a `readWriteAnyDatabase` [built-in role](https://www.mongodb.com/docs/manual/reference/built-in-roles/#mongodb-authrole-readWriteAnyDatabase) +on the `admin` database to be able to create and update the cached schema in any database. + +```json +roles: [ + {role: "readWriteAnyDatabase", db: "admin"} +] +``` + +### Collection-Level Access Control + +If [collection-level access control](https://www.mongodb.com/docs/manual/core/collection-level-access-control/) +is implemented, then ensure `find`, `insert`, and `update` actions are +allowed on the cache collections (`_sqlSchemas` and `_sqlTableSchemas`) + +## User Permissions for Reading an Existing Schema Cache + +To be able to read the SQL schema mappings to the cache collections, the connected +Amazon DocumentDB user account must have read permissions to read the +cache collections. + +To allow access for an Amazon DocumentDB user, ensure to set or add the appropriate roles as +described below. + +### Enable Access per Database + +To allow read access to specific databases in your server, add +a `read` [built-in role](https://www.mongodb.com/docs/manual/reference/built-in-roles/#mongodb-authrole-read) +for each database the user should have access to be able to read the cached schema for specific +databases. + +```json +roles: [ + {role: "read", db: "yourDatabase1"}, + {role: "read", db: "yourDatabase2"} ... +] +``` + +### Enable Access for Any Database + +To allow read access to any databases in your server, add +a `readAnyDatabase` [built-in role](https://www.mongodb.com/docs/manual/reference/built-in-roles/#mongodb-authrole-readAnyDatabase) +on the `admin` database to be able to read the cached schema in any database. + +```json +roles: [ + {role: "readAnyDatabase", db: "admin"} +] +``` + +### Collection-Level Access Control + +If [collection-level access control](https://www.mongodb.com/docs/manual/core/collection-level-access-control/) +is implemented, then ensure `find` actions are +allowed on the cache collections (`_sqlSchemas` and `_sqlTableSchemas`) + From c1281a57680562336c1bfcfa8e4fe32a62611ea5 Mon Sep 17 00:00:00 2001 From: affonsoBQ <67347924+affonsoBQ@users.noreply.github.com> Date: Mon, 12 Dec 2022 14:34:18 -0800 Subject: [PATCH 02/18] Bump JDBC version from 1.4.1 to 1.4.2 (#457) * Bump JDBC version from 1.4.1 to 1.4.2 * Commit Code Coverage Badge Co-authored-by: affonsoBQ --- gradle.properties | 2 +- src/main/resources/documentdb-jdbc.properties | 2 +- src/main/resources/project.properties | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/gradle.properties b/gradle.properties index 5a2cdb8e..e3d26f00 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,7 +1,7 @@ org.gradle.warning.mode=all MAJOR_VERSION=1 MINOR_VERSION=4 -PATCH_VERSION=1 +PATCH_VERSION=2 APPLICATION_NAME=Amazon DocumentDB JDBC Driver # Comment out if not creating a beta version #BETA_VERSION=1 diff --git a/src/main/resources/documentdb-jdbc.properties b/src/main/resources/documentdb-jdbc.properties index 65100c3c..4d073f42 100644 --- a/src/main/resources/documentdb-jdbc.properties +++ b/src/main/resources/documentdb-jdbc.properties @@ -1 +1 @@ -default.application.name=Amazon DocumentDB JDBC Driver v1.4.1 +default.application.name=Amazon DocumentDB JDBC Driver v1.4.2 diff --git a/src/main/resources/project.properties b/src/main/resources/project.properties index 59615ed8..a382f8e5 100644 --- a/src/main/resources/project.properties +++ b/src/main/resources/project.properties @@ -1,3 +1,3 @@ -driver.full.version=1.4.1 +driver.full.version=1.4.2 driver.major.version=1 driver.minor.version=4 From 94c65d9092d538afa01085d2e12eccc9f217dd9a Mon Sep 17 00:00:00 2001 From: affonsoBQ <67347924+affonsoBQ@users.noreply.github.com> Date: Wed, 14 Dec 2022 15:54:08 -0800 Subject: [PATCH 03/18] Add Mongo Driver Information (#433) * Bump commons-text from 1.9 to 1.10.0 Bumps commons-text from 1.9 to 1.10.0. --- updated-dependencies: - dependency-name: org.apache.commons:commons-text dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Bump com.github.spotbugs from 5.0.10 to 5.0.13 Bumps com.github.spotbugs from 5.0.10 to 5.0.13. --- updated-dependencies: - dependency-name: com.github.spotbugs dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Bump mockito-core from 4.8.0 to 4.8.1 Bumps [mockito-core](https://github.com/mockito/mockito) from 4.8.0 to 4.8.1. - [Release notes](https://github.com/mockito/mockito/releases) - [Commits](https://github.com/mockito/mockito/compare/v4.8.0...v4.8.1) --- updated-dependencies: - dependency-name: org.mockito:mockito-core dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Commit Code Coverage Badge * Bump sslcontext-kickstart from 7.4.7 to 7.4.8 Bumps [sslcontext-kickstart](https://github.com/Hakky54/sslcontext-kickstart) from 7.4.7 to 7.4.8. - [Release notes](https://github.com/Hakky54/sslcontext-kickstart/releases) - [Changelog](https://github.com/Hakky54/sslcontext-kickstart/blob/master/CHANGELOG.md) - [Commits](https://github.com/Hakky54/sslcontext-kickstart/compare/v7.4.7...v7.4.8) --- updated-dependencies: - dependency-name: io.github.hakky54:sslcontext-kickstart dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Bump checkstyle from 10.3.4 to 10.4 Bumps [checkstyle](https://github.com/checkstyle/checkstyle) from 10.3.4 to 10.4. - [Release notes](https://github.com/checkstyle/checkstyle/releases) - [Commits](https://github.com/checkstyle/checkstyle/compare/checkstyle-10.3.4...checkstyle-10.4) --- updated-dependencies: - dependency-name: com.puppycrawl.tools:checkstyle dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Revert "Bump checkstyle from 10.3.4 to 10.4" This reverts commit b4b048dc870e23654f2e71f5303ed1b178f7e742. * Adding mongo driver information * fixing build.gradle * adding missing class * Revert "adding missing class" This reverts commit 5a53d21f7654be97b259e36dbeae5ded19b46702. * adding missing class * adding license * fixing check style * adding final test fixture * Commit Code Coverage Badge * Commit Code Coverage Badge * Commit Code Coverage Badge * [AD-1032] Driver / Application information for CloudWatch. Simplify application and version handling. * Commit Code Coverage Badge * [AD-1032] Refactored creating the MongoClient for cleaner use. * [AD-1032] Fixed style issues. * [AD-1032] Fixed style issues #2. * [AD-1032] Fixed style issues #3. * Commit Code Coverage Badge Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Bruce Irschick Co-authored-by: birschick-bq Co-authored-by: affonsoBQ --- .github/badges/jacoco.svg | 2 +- .gitignore | 2 +- build.gradle | 10 +--- .../documentdb/jdbc/DocumentDbConnection.java | 6 +-- .../jdbc/DocumentDbConnectionProperties.java | 54 ++++++++++++++----- .../documentdb/jdbc/DocumentDbDriver.java | 5 ++ .../documentdb/jdbc/DocumentDbMain.java | 3 +- .../jdbc/DocumentDbQueryExecutor.java | 5 +- .../metadata/DocumentDbMetadataService.java | 5 +- .../jdbc/persist/DocumentDbSchemaReader.java | 3 +- .../jdbc/persist/DocumentDbSchemaWriter.java | 3 +- src/main/resources/documentdb-jdbc.properties | 1 - src/main/resources/project.properties | 3 -- .../jdbc/DocumentDbQueryExecutorTest.java | 5 +- .../DocumentDbAbstractTestEnvironment.java | 7 ++- 15 files changed, 60 insertions(+), 54 deletions(-) delete mode 100644 src/main/resources/documentdb-jdbc.properties delete mode 100644 src/main/resources/project.properties diff --git a/.github/badges/jacoco.svg b/.github/badges/jacoco.svg index 3f1a2f9e..735cecc8 100644 --- a/.github/badges/jacoco.svg +++ b/.github/badges/jacoco.svg @@ -1 +1 @@ -coverage91.1% \ No newline at end of file +coverage91.2% \ No newline at end of file diff --git a/.gitignore b/.gitignore index ec79dfae..bb9b2b1d 100644 --- a/.gitignore +++ b/.gitignore @@ -27,5 +27,5 @@ build/ /src/main/resources/LICENSE /src/main/resources/NOTICE # Generated -/common/src/main/resources/common.properties +/src/main/resources/project.properties diff --git a/build.gradle b/build.gradle index 57296949..a32bf01d 100644 --- a/build.gradle +++ b/build.gradle @@ -278,14 +278,7 @@ task writeProperties(type: WriteProperties) { property("driver.major.version", MAJOR_VERSION) property("driver.minor.version", MINOR_VERSION) property("driver.full.version", project.version) -} - -/** - * Write default application name to properties file under common. - */ -task writeCommonProperties(type: WriteProperties) { - outputFile("$projectDir/common/src/main/resources/common.properties") - property("default.application.name", "${APPLICATION_NAME} v${project.version}") + property("default.application.name", APPLICATION_NAME) } import io.franzbecker.gradle.lombok.task.DelombokTask @@ -305,7 +298,6 @@ javadoc { processResources { from(writeProperties) - from(writeCommonProperties) duplicatesStrategy(DuplicatesStrategy.INCLUDE) } diff --git a/src/main/java/software/amazon/documentdb/jdbc/DocumentDbConnection.java b/src/main/java/software/amazon/documentdb/jdbc/DocumentDbConnection.java index 38fb8ed4..c7170256 100644 --- a/src/main/java/software/amazon/documentdb/jdbc/DocumentDbConnection.java +++ b/src/main/java/software/amazon/documentdb/jdbc/DocumentDbConnection.java @@ -21,11 +21,9 @@ import com.jcraft.jsch.JSch; import com.jcraft.jsch.JSchException; import com.jcraft.jsch.Session; -import com.mongodb.MongoClientSettings; import com.mongodb.MongoCommandException; import com.mongodb.MongoSecurityException; import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoClients; import com.mongodb.client.MongoDatabase; import lombok.AllArgsConstructor; import lombok.Getter; @@ -302,9 +300,7 @@ DocumentDbConnectionProperties getConnectionProperties() { private void initializeClients(final DocumentDbConnectionProperties connectionProperties) throws SQLException { // Create the mongo client. - final MongoClientSettings settings = connectionProperties - .buildMongoClientSettings(getSshLocalPort()); - mongoClient = MongoClients.create(settings); + mongoClient = connectionProperties.createMongoClient(getSshLocalPort()); mongoDatabase = mongoClient.getDatabase(connectionProperties.getDatabase()); pingDatabase(); } diff --git a/src/main/java/software/amazon/documentdb/jdbc/DocumentDbConnectionProperties.java b/src/main/java/software/amazon/documentdb/jdbc/DocumentDbConnectionProperties.java index ce4e3170..ba66099b 100644 --- a/src/main/java/software/amazon/documentdb/jdbc/DocumentDbConnectionProperties.java +++ b/src/main/java/software/amazon/documentdb/jdbc/DocumentDbConnectionProperties.java @@ -20,8 +20,11 @@ import com.google.common.base.Strings; import com.mongodb.MongoClientSettings; import com.mongodb.MongoCredential; +import com.mongodb.MongoDriverInformation; import com.mongodb.ReadPreference; import com.mongodb.ServerAddress; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; import com.mongodb.connection.SslSettings; import com.mongodb.event.ServerMonitorListener; import lombok.SneakyThrows; @@ -71,20 +74,10 @@ public class DocumentDbConnectionProperties extends Properties { public static final String DOCUMENTDB_CUSTOM_OPTIONS = "DOCUMENTDB_CUSTOM_OPTIONS"; private static String classPathLocationName = null; private static String[] documentDbSearchPaths = null; - private static final String DEFAULT_APPLICATION_NAME_KEY = "default.application.name"; - private static final String PROPERTIES_FILE_PATH = "/documentdb-jdbc.properties"; static final String DEFAULT_APPLICATION_NAME; static { - String defaultAppName = ""; - try (InputStream is = DocumentDbConnectionProperties.class.getResourceAsStream(PROPERTIES_FILE_PATH)) { - final Properties p = new Properties(); - p.load(is); - defaultAppName = p.getProperty(DEFAULT_APPLICATION_NAME_KEY); - } catch (Exception e) { - LOGGER.error("Error loading default application name: " + e.getMessage()); - } - DEFAULT_APPLICATION_NAME = defaultAppName; + DEFAULT_APPLICATION_NAME = DocumentDbDriver.DEFAULT_APPLICATION_NAME; } /** @@ -178,6 +171,20 @@ private static String getClassPathLocation() { return classPathLocation; } + /** + * Return MongoDriverInformation object. It will initialize the Object with application name + * and driver version. + * + * @return MongoDriverInformation + */ + private MongoDriverInformation getMongoDriverInformation() { + final MongoDriverInformation mongoDriverInformation = MongoDriverInformation.builder() + .driverName(getApplicationName()) + .driverVersion(DocumentDbDriver.DRIVER_VERSION) + .build(); + return mongoDriverInformation; + } + /** * Gets the hostname. * @@ -258,7 +265,7 @@ public void setDatabase(final String database) { public String getApplicationName() { return getProperty( DocumentDbConnectionProperty.APPLICATION_NAME.getName(), - DocumentDbConnectionProperty.APPLICATION_NAME.getDefaultValue() ); + DocumentDbConnectionProperty.APPLICATION_NAME.getDefaultValue()); } /** @@ -680,6 +687,29 @@ public DocumentDbAllowDiskUseOption getAllowDiskUseOption() { return getPropertyAsAllowDiskUseOption(DocumentDbConnectionProperty.ALLOW_DISK_USE.getName()); } + /** + * Creates a {@link MongoClient} instance from the connection properties. + * + * @return a new instance of a {@link MongoClient}. + */ + public MongoClient createMongoClient() { + return MongoClients.create( + buildMongoClientSettings(), + getMongoDriverInformation()); + } + + /** + * Creates a {@link MongoClient} instance from the connection properties using + * the SSH tunnel port on the local host. + * + * @return a new instance of a {@link MongoClient}. + */ + public MongoClient createMongoClient(final int sshLocalPort) { + return MongoClients.create( + buildMongoClientSettings(sshLocalPort), + getMongoDriverInformation()); + } + /** * Builds the MongoClientSettings from properties. * diff --git a/src/main/java/software/amazon/documentdb/jdbc/DocumentDbDriver.java b/src/main/java/software/amazon/documentdb/jdbc/DocumentDbDriver.java index f01b2d08..c4e8259b 100644 --- a/src/main/java/software/amazon/documentdb/jdbc/DocumentDbDriver.java +++ b/src/main/java/software/amazon/documentdb/jdbc/DocumentDbDriver.java @@ -37,10 +37,12 @@ public class DocumentDbDriver extends software.amazon.documentdb.jdbc.common.Dri private static final String DRIVER_MAJOR_VERSION_KEY = "driver.major.version"; private static final String DRIVER_MINOR_VERSION_KEY = "driver.minor.version"; private static final String DRIVER_FULL_VERSION_KEY = "driver.full.version"; + private static final String DEFAULT_APPLICATION_NAME_KEY = "default.application.name"; private static final String PROPERTIES_FILE_PATH = "/project.properties"; static final int DRIVER_MAJOR_VERSION; static final int DRIVER_MINOR_VERSION; static final String DRIVER_VERSION; + static final String DEFAULT_APPLICATION_NAME; // Registers the JDBC driver. static { @@ -48,18 +50,21 @@ public class DocumentDbDriver extends software.amazon.documentdb.jdbc.common.Dri int majorVersion = 0; int minorVersion = 0; String fullVersion = ""; + String defaultApplicationName = ""; try (InputStream is = DocumentDbDatabaseMetaData.class.getResourceAsStream(PROPERTIES_FILE_PATH)) { final Properties p = new Properties(); p.load(is); majorVersion = Integer.parseInt(p.getProperty(DRIVER_MAJOR_VERSION_KEY)); minorVersion = Integer.parseInt(p.getProperty(DRIVER_MINOR_VERSION_KEY)); fullVersion = p.getProperty(DRIVER_FULL_VERSION_KEY); + defaultApplicationName = p.getProperty(DEFAULT_APPLICATION_NAME_KEY); } catch (Exception e) { LOGGER.error("Error loading driver version: " + e.getMessage()); } DRIVER_MAJOR_VERSION = majorVersion; DRIVER_MINOR_VERSION = minorVersion; DRIVER_VERSION = fullVersion; + DEFAULT_APPLICATION_NAME = defaultApplicationName; new DocumentDbDriver().register(); } diff --git a/src/main/java/software/amazon/documentdb/jdbc/DocumentDbMain.java b/src/main/java/software/amazon/documentdb/jdbc/DocumentDbMain.java index 5d2d33a1..ba6d4e40 100644 --- a/src/main/java/software/amazon/documentdb/jdbc/DocumentDbMain.java +++ b/src/main/java/software/amazon/documentdb/jdbc/DocumentDbMain.java @@ -36,7 +36,6 @@ import com.google.common.collect.ImmutableList; import com.mongodb.DuplicateKeyException; import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoClients; import lombok.Getter; import lombok.NonNull; import org.apache.commons.cli.CommandLine; @@ -393,7 +392,7 @@ private static void performCommand( private static MongoClient getMongoClient(final DocumentDbConnectionProperties properties) { if (client == null) { - client = MongoClients.create(properties.buildMongoClientSettings()); + client = properties.createMongoClient(); } return client; } diff --git a/src/main/java/software/amazon/documentdb/jdbc/DocumentDbQueryExecutor.java b/src/main/java/software/amazon/documentdb/jdbc/DocumentDbQueryExecutor.java index d2f2754a..946fbfa5 100644 --- a/src/main/java/software/amazon/documentdb/jdbc/DocumentDbQueryExecutor.java +++ b/src/main/java/software/amazon/documentdb/jdbc/DocumentDbQueryExecutor.java @@ -18,11 +18,9 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; -import com.mongodb.MongoClientSettings; import com.mongodb.MongoException; import com.mongodb.client.AggregateIterable; import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoClients; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoCursor; import com.mongodb.client.MongoDatabase; @@ -221,8 +219,7 @@ private void resetQueryState() { } private void performCancel() throws SQLException { - final MongoClientSettings settings = connectionProperties.buildMongoClientSettings(); - try (MongoClient client = MongoClients.create(settings)) { + try (MongoClient client = connectionProperties.createMongoClient()) { final MongoDatabase database = client.getDatabase("admin"); // Find the opId to kill using the queryId. diff --git a/src/main/java/software/amazon/documentdb/jdbc/metadata/DocumentDbMetadataService.java b/src/main/java/software/amazon/documentdb/jdbc/metadata/DocumentDbMetadataService.java index 51469c9d..81ad876b 100644 --- a/src/main/java/software/amazon/documentdb/jdbc/metadata/DocumentDbMetadataService.java +++ b/src/main/java/software/amazon/documentdb/jdbc/metadata/DocumentDbMetadataService.java @@ -16,9 +16,7 @@ package software.amazon.documentdb.jdbc.metadata; -import com.mongodb.MongoClientSettings; import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoClients; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoDatabase; import com.mongodb.client.model.EstimatedDocumentCountOptions; @@ -387,10 +385,9 @@ private static DocumentDbSchema getCollectionMetadataDirect( final Map tableMap, final MongoClient client) throws SQLException { - final MongoClientSettings settings = properties.buildMongoClientSettings(); final MongoClient mongoClient = client != null ? client - : MongoClients.create(settings); + : properties.createMongoClient(); try { final MongoDatabase database = mongoClient.getDatabase(databaseName); for (String collectionName : getFilteredCollectionNames(database)) { diff --git a/src/main/java/software/amazon/documentdb/jdbc/persist/DocumentDbSchemaReader.java b/src/main/java/software/amazon/documentdb/jdbc/persist/DocumentDbSchemaReader.java index c4bfb0d2..15035230 100644 --- a/src/main/java/software/amazon/documentdb/jdbc/persist/DocumentDbSchemaReader.java +++ b/src/main/java/software/amazon/documentdb/jdbc/persist/DocumentDbSchemaReader.java @@ -18,7 +18,6 @@ import com.mongodb.MongoException; import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoClients; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoDatabase; import lombok.NonNull; @@ -87,7 +86,7 @@ public DocumentDbSchemaReader(final @NonNull DocumentDbConnectionProperties prop this.properties = properties; this.client = client != null ? client - : MongoClients.create(properties.buildMongoClientSettings()); + : properties.createMongoClient(); this.closeClient = client == null; } diff --git a/src/main/java/software/amazon/documentdb/jdbc/persist/DocumentDbSchemaWriter.java b/src/main/java/software/amazon/documentdb/jdbc/persist/DocumentDbSchemaWriter.java index cc188a8a..0bd0af98 100644 --- a/src/main/java/software/amazon/documentdb/jdbc/persist/DocumentDbSchemaWriter.java +++ b/src/main/java/software/amazon/documentdb/jdbc/persist/DocumentDbSchemaWriter.java @@ -20,7 +20,6 @@ import com.mongodb.MongoException; import com.mongodb.client.ClientSession; import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoClients; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoDatabase; import com.mongodb.client.model.UpdateOptions; @@ -87,7 +86,7 @@ public DocumentDbSchemaWriter(final @NonNull DocumentDbConnectionProperties prop this.properties = properties; this.client = client != null ? client - : MongoClients.create(properties.buildMongoClientSettings()); + : properties.createMongoClient(); this.closeClient = client == null; } diff --git a/src/main/resources/documentdb-jdbc.properties b/src/main/resources/documentdb-jdbc.properties deleted file mode 100644 index 4d073f42..00000000 --- a/src/main/resources/documentdb-jdbc.properties +++ /dev/null @@ -1 +0,0 @@ -default.application.name=Amazon DocumentDB JDBC Driver v1.4.2 diff --git a/src/main/resources/project.properties b/src/main/resources/project.properties deleted file mode 100644 index a382f8e5..00000000 --- a/src/main/resources/project.properties +++ /dev/null @@ -1,3 +0,0 @@ -driver.full.version=1.4.2 -driver.major.version=1 -driver.minor.version=4 diff --git a/src/test/java/software/amazon/documentdb/jdbc/DocumentDbQueryExecutorTest.java b/src/test/java/software/amazon/documentdb/jdbc/DocumentDbQueryExecutorTest.java index 30e1642e..acc1c5d1 100644 --- a/src/test/java/software/amazon/documentdb/jdbc/DocumentDbQueryExecutorTest.java +++ b/src/test/java/software/amazon/documentdb/jdbc/DocumentDbQueryExecutorTest.java @@ -18,10 +18,8 @@ import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.ThreadFactoryBuilder; -import com.mongodb.MongoClientSettings; import com.mongodb.client.FindIterable; import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoClients; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoCursor; import com.mongodb.client.MongoDatabase; @@ -321,8 +319,7 @@ private static class MockQueryExecutor extends DocumentDbQueryExecutor { @Override protected java.sql.ResultSet runQuery(final String sql) throws SQLException { - final MongoClientSettings settings = VALID_CONNECTION_PROPERTIES.buildMongoClientSettings(); - try (MongoClient client = MongoClients.create(settings)) { + try (MongoClient client = VALID_CONNECTION_PROPERTIES.createMongoClient()) { final MongoDatabase database = client.getDatabase(VALID_CONNECTION_PROPERTIES.getDatabase()); final MongoCollection collection = database.getCollection( diff --git a/src/testFixtures/java/software/amazon/documentdb/jdbc/common/test/DocumentDbAbstractTestEnvironment.java b/src/testFixtures/java/software/amazon/documentdb/jdbc/common/test/DocumentDbAbstractTestEnvironment.java index 740fc8a9..ba25da47 100644 --- a/src/testFixtures/java/software/amazon/documentdb/jdbc/common/test/DocumentDbAbstractTestEnvironment.java +++ b/src/testFixtures/java/software/amazon/documentdb/jdbc/common/test/DocumentDbAbstractTestEnvironment.java @@ -18,7 +18,6 @@ import com.mongodb.MongoException; import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoClients; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoDatabase; import com.mongodb.client.result.InsertOneResult; @@ -214,9 +213,9 @@ public String newCollectionName(final boolean isTemporary) { @Override public MongoClient createMongoClient() throws SQLException { - return MongoClients.create(DocumentDbConnectionProperties - .getPropertiesFromConnectionString(getJdbcConnectionString()) - .buildMongoClientSettings()); + final DocumentDbConnectionProperties properties = DocumentDbConnectionProperties + .getPropertiesFromConnectionString(getJdbcConnectionString()); + return properties.createMongoClient(); } @Override From ad1d4e5547d9be9afa0d3361c68c5484facc9c17 Mon Sep 17 00:00:00 2001 From: Bruce Irschick Date: Tue, 20 Dec 2022 10:12:25 -0800 Subject: [PATCH 04/18] [AD-1017] Implement single-instance SSH tunnel. (#453) * [AD-1017] Implement single-instance SSH tunnel. * Handle late-bound invocation (DbVisualizer) * Ensure support for multiple hash algorithms. * Properly embed new SSH tunnel component. * [AD-1017] Attempt to debug multi-process test. * [AD-1017] Attempt to debug multi-process test. * [AD-1017] Attempt to debug multi-process test - #3. * Commit Code Coverage Badge * [AD-1017] Attempt to find correct location for test source file. * [AD-1017] Attempt to find correct location for test source file - #2. * [AD-1017] Fix spotBugs error; code review improvements. * [AD-1017] Ignore IO exceptions when reading spawned processes. * Commit Code Coverage Badge * [AD-1017] Update dependency for mongo to 4.8.1 * Commit Code Coverage Badge * [AD-1017] Attempt improve stability of SSH Tunnel client tests. * [AD-1017] Attempt improve stability of multiprocess connection test. * Commit Code Coverage Badge * Update src/main/java/software/amazon/documentdb/jdbc/DocumentDbConnectionProperties.java Co-authored-by: Alexey Temnikov * [AD-1017] Properly rename parameter. * Commit Code Coverage Badge * [AD-1017] Fix issue with output not being sent to the console (i.e., change level from INFO to ERROR). * Commit Code Coverage Badge * [AD-1017] Attempt to increase code coverage. * [AD-1017] Fix style issue. * Commit Code Coverage Badge Co-authored-by: birschick-bq Co-authored-by: Alexey Temnikov --- .github/badges/branches.svg | 2 +- .github/badges/jacoco.svg | 2 +- NOTICE | 12 +- build.gradle | 31 +- config/spotbugs/spotbugs-exclude.xml | 8 +- .../documentdb/jdbc/DocumentDbConnection.java | 246 +----- .../jdbc/DocumentDbConnectionProperties.java | 416 +++++++--- .../jdbc/DocumentDbConnectionProperty.java | 11 +- .../documentdb/jdbc/DocumentDbMain.java | 56 +- .../jdbc/common/DatabaseMetaData.java | 4 +- .../jdbc/common/utilities/SqlError.java | 12 +- .../DocumentDbMultiThreadFileChannel.java | 143 ++++ .../sshtunnel/DocumentDbSshTunnelClient.java | 221 ++++++ .../sshtunnel/DocumentDbSshTunnelLock.java | 207 +++++ .../sshtunnel/DocumentDbSshTunnelServer.java | 738 ++++++++++++++++++ .../sshtunnel/DocumentDbSshTunnelService.java | 560 +++++++++++++ .../DocumentDbSshTunnelTestClientRunner.java | 151 ++++ src/main/resources/jdbc.properties | 9 +- .../scalardatatypepromotion-opaque.png | Bin 50358 -> 48344 bytes .../scalardatatypepromotion-transparent.png | Bin 53611 -> 51303 bytes src/markdown/support/troubleshooting-guide.md | 15 + .../DocumentDbConnectionPropertiesTest.java | 143 ++-- .../jdbc/DocumentDbConnectionTest.java | 123 ++- .../documentdb/jdbc/DocumentDbMainTest.java | 64 +- .../jdbc/common/DatabaseMetaDataTest.java | 2 +- ...ocumentDbQueryMappingServiceBasicTest.java | 4 +- ...mentDbQueryMappingServiceDateTimeTest.java | 6 +- .../DocumentDbSshTunnelClientTest.java | 147 ++++ .../DocumentDbSshTunnelServerTest.java | 59 ++ .../DocumentDbSshTunnelServiceTest.java | 246 ++++++ .../DocumentDbAbstractTestEnvironment.java | 22 +- .../jdbc/common/test/DocumentDbTest.java | 13 +- 32 files changed, 3204 insertions(+), 469 deletions(-) create mode 100644 src/main/java/software/amazon/documentdb/jdbc/sshtunnel/DocumentDbMultiThreadFileChannel.java create mode 100644 src/main/java/software/amazon/documentdb/jdbc/sshtunnel/DocumentDbSshTunnelClient.java create mode 100644 src/main/java/software/amazon/documentdb/jdbc/sshtunnel/DocumentDbSshTunnelLock.java create mode 100644 src/main/java/software/amazon/documentdb/jdbc/sshtunnel/DocumentDbSshTunnelServer.java create mode 100644 src/main/java/software/amazon/documentdb/jdbc/sshtunnel/DocumentDbSshTunnelService.java create mode 100644 src/main/java/software/amazon/documentdb/jdbc/sshtunnel/DocumentDbSshTunnelTestClientRunner.java create mode 100644 src/test/java/software/amazon/documentdb/jdbc/sshtunnel/DocumentDbSshTunnelClientTest.java create mode 100644 src/test/java/software/amazon/documentdb/jdbc/sshtunnel/DocumentDbSshTunnelServerTest.java create mode 100644 src/test/java/software/amazon/documentdb/jdbc/sshtunnel/DocumentDbSshTunnelServiceTest.java diff --git a/.github/badges/branches.svg b/.github/badges/branches.svg index ca9aebc6..1a82651a 100644 --- a/.github/badges/branches.svg +++ b/.github/badges/branches.svg @@ -1 +1 @@ -branches80.5% \ No newline at end of file +branches77.7% \ No newline at end of file diff --git a/.github/badges/jacoco.svg b/.github/badges/jacoco.svg index 735cecc8..59cb6f0d 100644 --- a/.github/badges/jacoco.svg +++ b/.github/badges/jacoco.svg @@ -1 +1 @@ -coverage91.2% \ No newline at end of file +coverage90.1% \ No newline at end of file diff --git a/NOTICE b/NOTICE index 5b173ffb..cda25688 100644 --- a/NOTICE +++ b/NOTICE @@ -348,12 +348,12 @@ Package Artifact: https://repo.maven.apache.org/maven2/com/google/protobuf/proto Package Artifact: https://repo.maven.apache.org/maven2/com/google/protobuf/protobuf-java/3.6.1/protobuf-java-3.6.1-sources.jar Package Artifact: https://github.com/google/protobuf.git ----------------------- -Package ID: com.jcraft:jsch:0.1.55 -Package Homepage: http://www.jcraft.com/jsch/ +Package ID: com.github.mwiede:jsch:0.2.4 +Package Homepage: https://github.com/mwiede/jsch Package SPDX-License-Identifier: BSD-3-Clause -Package Artifact: https://repo.maven.apache.org/maven2/com/jcraft/jsch/0.1.55/jsch-0.1.55.jar -Package Artifact: https://repo.maven.apache.org/maven2/com/jcraft/jsch/0.1.55/jsch-0.1.55-sources.jar -Package Artifact: http://git.jcraft.com/jsch.git +Package Artifact: https://repo.maven.org/maven2/com/github/mwiede/jsch/0.2.4/jsch-0.2.4.jar +Package Artifact: https://repo.maven.org/maven2/com/github/mwiede/jsch/0.2.4/jsch-0.2.4-sources.jar +Package Artifact: https://github.com/mwiede/jsch.git ----------------------- Package ID: org.codehaus.janino:commons-compiler:3.1.6 Package Homepage: http://janino-compiler.github.io/commons-compiler/ @@ -16570,7 +16570,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ----------------------- -com.jcraft/jsch/0.1.55 LICENSE +github.com/mwiede/jsch/0.2.4 LICENSE JSch 0.0.* was released under the GNU LGPL license. Later, we have switched over to a BSD-style license. diff --git a/build.gradle b/build.gradle index a32bf01d..12a8c815 100644 --- a/build.gradle +++ b/build.gradle @@ -1,6 +1,6 @@ plugins { id 'java' - id 'com.github.spotbugs' version '5.0.10' + id 'com.github.spotbugs' version '5.0.13' id 'checkstyle' id 'jacoco' id 'com.github.hierynomus.license' version '0.16.1' @@ -92,7 +92,7 @@ shadowJar { minimize { exclude(dependency('org.apache.calcite::')) exclude(dependency('org.slf4j.*::')) - exclude(dependency('com.jcraft::')) + exclude(dependency('com.github.mwiede::')) } } @@ -145,24 +145,32 @@ spotbugsMain { // Configure HTML report reports { xml { - enabled = true + required.set(true) destination = file("$buildDir/reports/spotbugs/main.xml") } + html { + required.set(true) + destination = file("$buildDir/reports/spotbugs/main.html") } + } } spotbugsTest { // Configure HTML report reports { xml { - enabled = true + required.set(true) destination = file("$buildDir/reports/spotbugs/test.xml") } + html { + required.set(true) + destination = file("$buildDir/reports/spotbugs/test.html") } + } } task checkSpotBugsMainReport { doLast { - def xmlReport = spotbugsMain.reports.getByName("XML") + def xmlReport = spotbugsMain.reports.getByName("xml") def slurped = new groovy.xml.XmlSlurper().parse(xmlReport.destination) def bugsFound = slurped.BugInstance.size() slurped.BugInstance.each { @@ -179,7 +187,7 @@ task checkSpotBugsMainReport { task checkSpotBugsTestReport { doLast { - def xmlReport = spotbugsTest.reports.getByName("XML") + def xmlReport = spotbugsTest.reports.getByName("xml") def slurped = new XmlSlurper().parse(xmlReport.destination) def bugsFound = slurped.BugInstance.size() slurped.BugInstance.each { @@ -312,13 +320,14 @@ dependencies { implementation group: 'com.google.guava', name: 'guava', version: '31.1-jre' implementation group: 'org.slf4j', name: 'slf4j-log4j12', version: '2.0.5' implementation group: 'org.apache.commons', name: 'commons-text', version: '1.10.0' + implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.12.0' implementation group: 'org.apache.logging.log4j', name: 'log4j-api', version: '2.19.0' implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.19.0' - implementation group: 'org.mongodb', name: 'mongodb-driver-sync', version: '4.7.2' - implementation group: 'com.jcraft', name: 'jsch', version: '0.1.55' + implementation group: 'org.mongodb', name: 'mongodb-driver-sync', version: '4.8.1' + implementation group: 'com.github.mwiede', name: 'jsch', version: '0.2.4' implementation group: 'org.apache.calcite', name: 'calcite-core', version: '1.32.0' implementation group: 'commons-beanutils', name: 'commons-beanutils', version: '1.9.4' - implementation 'io.github.hakky54:sslcontext-kickstart:7.4.7' + implementation 'io.github.hakky54:sslcontext-kickstart:7.4.8' compileOnly group: 'org.immutables', name: 'value', version: '2.9.2' compileOnly group: 'com.puppycrawl.tools', name: 'checkstyle', version: '10.5.0' @@ -331,7 +340,7 @@ dependencies { testAnnotationProcessor group: 'org.projectlombok', name: 'lombok', version: '1.18.24' testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-api', version: '5.9.1' testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-params', version: '5.9.1' - testImplementation group: 'org.mockito', name: 'mockito-core', version: '4.8.0' + testImplementation group: 'org.mockito', name: 'mockito-core', version: '4.9.0' testRuntimeOnly group: 'de.flapdoodle.embed', name: 'de.flapdoodle.embed.mongo', version: '3.5.3' testRuntimeOnly 'org.junit.jupiter:junit-jupiter-api:5.9.1' testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine' @@ -342,7 +351,7 @@ dependencies { testFixturesCompileOnly group: 'de.flapdoodle.embed', name: 'de.flapdoodle.embed.mongo', version: '3.5.3' testFixturesCompileOnly 'org.junit.jupiter:junit-jupiter-api:5.9.1' testFixturesImplementation group: 'com.google.guava', name: 'guava', version: '29.0-jre' - testFixturesImplementation group: 'org.mongodb', name: 'mongodb-driver-sync', version: '4.7.2' + testFixturesImplementation group: 'org.mongodb', name: 'mongodb-driver-sync', version: '4.8.1' testFixturesImplementation group: 'org.apache.logging.log4j', name: 'log4j-api', version: '2.17.2' testFixturesImplementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.17.2' } diff --git a/config/spotbugs/spotbugs-exclude.xml b/config/spotbugs/spotbugs-exclude.xml index 8b08357e..a618e626 100644 --- a/config/spotbugs/spotbugs-exclude.xml +++ b/config/spotbugs/spotbugs-exclude.xml @@ -90,7 +90,10 @@ - + + + + @@ -98,6 +101,9 @@ + + + diff --git a/src/main/java/software/amazon/documentdb/jdbc/DocumentDbConnection.java b/src/main/java/software/amazon/documentdb/jdbc/DocumentDbConnection.java index c7170256..514dfa15 100644 --- a/src/main/java/software/amazon/documentdb/jdbc/DocumentDbConnection.java +++ b/src/main/java/software/amazon/documentdb/jdbc/DocumentDbConnection.java @@ -16,20 +16,11 @@ package software.amazon.documentdb.jdbc; -import com.jcraft.jsch.HostKey; -import com.jcraft.jsch.HostKeyRepository; -import com.jcraft.jsch.JSch; -import com.jcraft.jsch.JSchException; -import com.jcraft.jsch.Session; import com.mongodb.MongoCommandException; import com.mongodb.MongoSecurityException; import com.mongodb.client.MongoClient; import com.mongodb.client.MongoDatabase; -import lombok.AllArgsConstructor; -import lombok.Getter; import lombok.SneakyThrows; -import org.apache.commons.lang3.tuple.ImmutablePair; -import org.apache.commons.lang3.tuple.Pair; import org.bson.Document; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,20 +28,20 @@ import software.amazon.documentdb.jdbc.common.utilities.SqlError; import software.amazon.documentdb.jdbc.common.utilities.SqlState; import software.amazon.documentdb.jdbc.metadata.DocumentDbDatabaseSchemaMetadata; +import software.amazon.documentdb.jdbc.sshtunnel.DocumentDbSshTunnelClient; -import java.nio.file.Files; -import java.nio.file.Path; import java.sql.DatabaseMetaData; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.SQLFeatureNotSupportedException; import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; -import static software.amazon.documentdb.jdbc.DocumentDbConnectionProperties.getDocumentDbSearchPaths; -import static software.amazon.documentdb.jdbc.DocumentDbConnectionProperties.getPath; -import static software.amazon.documentdb.jdbc.DocumentDbConnectionProperties.isNullOrWhitespace; import static software.amazon.documentdb.jdbc.DocumentDbConnectionProperty.REFRESH_SCHEMA; import static software.amazon.documentdb.jdbc.metadata.DocumentDbDatabaseSchemaMetadata.VERSION_LATEST_OR_NEW; import static software.amazon.documentdb.jdbc.metadata.DocumentDbDatabaseSchemaMetadata.VERSION_NEW; @@ -63,22 +54,17 @@ public class DocumentDbConnection extends Connection private static final Logger LOGGER = LoggerFactory.getLogger(DocumentDbConnection.class.getName()); - public static final String SSH_KNOWN_HOSTS_FILE = "~/.ssh/known_hosts"; - public static final String STRICT_HOST_KEY_CHECKING = "StrictHostKeyChecking"; - public static final String HASH_KNOWN_HOSTS = "HashKnownHosts"; - public static final String SERVER_HOST_KEY = "server_host_key"; - public static final String YES = "yes"; - public static final String NO = "no"; - public static final String LOCALHOST = "localhost"; - public static final int DEFAULT_DOCUMENTDB_PORT = 27017; - public static final int DEFAULT_SSH_PORT = 22; + private static final Set SECRET_PROPERTIES = + Collections.unmodifiableSet(new HashSet<>(Arrays.asList( + DocumentDbConnectionProperty.PASSWORD.getName(), + DocumentDbConnectionProperty.PASSWORD.getName()))); private final DocumentDbConnectionProperties connectionProperties; private DocumentDbDatabaseMetaData metadata; private DocumentDbDatabaseSchemaMetadata databaseMetadata; private MongoClient mongoClient = null; private MongoDatabase mongoDatabase = null; - private SshPortForwardingSession session; + private DocumentDbSshTunnelClient sshTunnelClient; /** * DocumentDbConnection constructor, initializes super class. @@ -91,53 +77,37 @@ public class DocumentDbConnection extends Connection final StringBuilder sb = new StringBuilder(); sb.append("Creating connection with following properties:"); for (String propertyName : connectionProperties.stringPropertyNames()) { - if (!DocumentDbConnectionProperty.PASSWORD.getName().equals(propertyName)) { + if (!SECRET_PROPERTIES.contains(propertyName)) { sb.append(String.format("%n Connection property %s=%s", propertyName, connectionProperties.get(propertyName).toString())); } } LOGGER.debug(sb.toString()); } - this.session = createSshTunnel(connectionProperties); + if (connectionProperties.enableSshTunnel()) { + ensureSshTunnel(connectionProperties); + } else { + LOGGER.debug("Internal SSH tunnel not used."); + } initializeClients(connectionProperties); } /** - * Initializes the SSH session and creates a port forwarding tunnel. + * Ensures an SSH Tunnel service is started for this set of SSH Tunnel properties, or confirms + * an SSH Tunnel is already running. It ensures an SSH Tunnel client session is active and also ensures the + * SSH Tunnel's listening port is valid. * - * @param connectionProperties the {@link DocumentDbConnectionProperties} connection properties. - * @return a {@link Session} session. This session must be closed by calling the - * {@link Session#disconnect()} method. - * @throws SQLException if unable to create SSH session or create the port forwarding tunnel. + * @param connectionProperties the connection properties to use for the SSH Tunnel. + * @throws SQLException when unable to ensure an SSH Tunnel is started. */ - public static SshPortForwardingSession createSshTunnel( - final DocumentDbConnectionProperties connectionProperties) throws SQLException { - if (!connectionProperties.enableSshTunnel()) { - LOGGER.info("Internal SSH tunnel not started."); - return null; - } else if (!connectionProperties.isSshPrivateKeyFileExists()) { - throw SqlError.createSQLException( - LOGGER, - SqlState.CONNECTION_EXCEPTION, - SqlError.SSH_PRIVATE_KEY_FILE_NOT_FOUND, - connectionProperties.getSshPrivateKeyFile()); - } - - LOGGER.info("Internal SSH tunnel starting."); + private void ensureSshTunnel(final DocumentDbConnectionProperties connectionProperties) throws SQLException { try { - final JSch jSch = new JSch(); - addIdentity(connectionProperties, jSch); - final Session session = createSession(connectionProperties, jSch); - connectSession(connectionProperties, jSch, session); - final SshPortForwardingSession portForwardingSession = getPortForwardingSession( - connectionProperties, session); - LOGGER.info("Internal SSH tunnel started on local port '{}'.", - portForwardingSession.localPort); - return portForwardingSession; + this.sshTunnelClient = new DocumentDbSshTunnelClient(connectionProperties); } catch (SQLException e) { throw e; } catch (Exception e) { - throw new SQLException(e.getMessage(), e); + throw SqlError.createSQLException(LOGGER, SqlState.CONNECTION_EXCEPTION, e, + SqlError.SSH_TUNNEL_ERROR, e.getMessage()); } } @@ -149,7 +119,7 @@ public static SshPortForwardingSession createSshTunnel( public int getSshLocalPort() { // Get the port from the SSH tunnel session, if it exists. if (isSshTunnelActive()) { - return session.localPort; + return sshTunnelClient.getServiceListeningPort(); } return 0; } @@ -159,9 +129,10 @@ public int getSshLocalPort() { * * @return returns {@code true} if the SSH tunnel is active, {@code false}, otherwise. */ + @SneakyThrows public boolean isSshTunnelActive() { // indicate whether the SSH tunnel is enabled - return session != null; + return sshTunnelClient != null && sshTunnelClient.getServiceListeningPort() > 0; } @Override @@ -175,7 +146,7 @@ public boolean isValid(final int timeout) throws SQLException { if (mongoDatabase != null) { try { // Convert to milliseconds - final int maxTimeMS = timeout + 1000; + final long maxTimeMS = TimeUnit.SECONDS.toMillis(timeout); pingDatabase(maxTimeMS); return true; } catch (Exception e) { @@ -186,7 +157,7 @@ public boolean isValid(final int timeout) throws SQLException { } @Override - public void doClose() { + public void doClose() throws SQLException { if (mongoDatabase != null) { mongoDatabase = null; } @@ -194,9 +165,17 @@ public void doClose() { mongoClient.close(); mongoClient = null; } - if (session != null) { - session.session.disconnect(); - session = null; + if (sshTunnelClient != null) { + try { + sshTunnelClient.close(); + } catch (SQLException e) { + throw e; + } catch (Exception e) { + throw SqlError.createSQLException(LOGGER, SqlState.CONNECTION_EXCEPTION, e, + SqlError.SSH_TUNNEL_ERROR, e.getMessage()); + } finally { + sshTunnelClient = null; + } } } @@ -309,7 +288,7 @@ private void pingDatabase() throws SQLException { pingDatabase(0); } - private void pingDatabase(final int maxTimeMS) throws SQLException { + private void pingDatabase(final long maxTimeMS) throws SQLException { try { final String maxTimeMSOption = (maxTimeMS > 0) ? String.format(", \"maxTimeMS\" : %d", maxTimeMS) @@ -341,145 +320,4 @@ private void pingDatabase(final int maxTimeMS) throws SQLException { throw new SQLException(e.getMessage(), e); } } - - private static SshPortForwardingSession getPortForwardingSession( - final DocumentDbConnectionProperties connectionProperties, - final Session session) throws JSchException { - final Pair clusterHostAndPort = getHostAndPort( - connectionProperties.getHostname(), DEFAULT_DOCUMENTDB_PORT); - final int localPort = session.setPortForwardingL( - LOCALHOST, 0, clusterHostAndPort.getLeft(), clusterHostAndPort.getRight()); - return new SshPortForwardingSession(session, localPort); - } - - private static Pair getHostAndPort( - final String hostname, - final int defaultPort) { - final String clusterHost; - final int clusterPort; - final int portSeparatorIndex = hostname.indexOf(':'); - if (portSeparatorIndex >= 0) { - clusterHost = hostname.substring(0, portSeparatorIndex); - clusterPort = Integer.parseInt( - hostname.substring(portSeparatorIndex + 1)); - } else { - clusterHost = hostname; - clusterPort = defaultPort; - } - return new ImmutablePair<>(clusterHost, clusterPort); - } - - private static void connectSession( - final DocumentDbConnectionProperties connectionProperties, - final JSch jSch, - final Session session) throws SQLException { - setSecurityConfig(connectionProperties, jSch, session); - try { - session.connect(); - } catch (JSchException e) { - throw new SQLException(e.getMessage(), e); - } - } - - private static void addIdentity( - final DocumentDbConnectionProperties connectionProperties, - final JSch jSch) throws JSchException { - final String privateKeyFileName = getPath(connectionProperties.getSshPrivateKeyFile(), - getDocumentDbSearchPaths()).toString(); - LOGGER.debug("SSH private key file resolved to '{}'.", privateKeyFileName); - // If passPhrase protected, will need to provide this, too. - final String passPhrase = !isNullOrWhitespace(connectionProperties.getSshPrivateKeyPassphrase()) - ? connectionProperties.getSshPrivateKeyPassphrase() - : null; - jSch.addIdentity(privateKeyFileName, passPhrase); - } - - private static Session createSession( - final DocumentDbConnectionProperties connectionProperties, - final JSch jSch) throws SQLException { - final String sshUsername = connectionProperties.getSshUser(); - final Pair sshHostAndPort = getHostAndPort( - connectionProperties.getSshHostname(), DEFAULT_SSH_PORT); - setKnownHostsFile(connectionProperties, jSch); - try { - return jSch.getSession(sshUsername, sshHostAndPort.getLeft(), sshHostAndPort.getRight()); - } catch (JSchException e) { - throw new SQLException(e.getMessage(), e); - } - } - - private static void setSecurityConfig( - final DocumentDbConnectionProperties connectionProperties, - final JSch jSch, - final Session session) { - if (!connectionProperties.getSshStrictHostKeyChecking()) { - session.setConfig(STRICT_HOST_KEY_CHECKING, NO); - return; - } - setHostKeyType(connectionProperties, jSch, session); - } - - private static void setHostKeyType( - final DocumentDbConnectionProperties connectionProperties, - final JSch jSch, final Session session) { - final HostKeyRepository keyRepository = jSch.getHostKeyRepository(); - final HostKey[] hostKeys = keyRepository.getHostKey(); - final Pair sshHostAndPort = getHostAndPort( - connectionProperties.getSshHostname(), DEFAULT_SSH_PORT); - final HostKey hostKey = Arrays.stream(hostKeys) - .filter(hk -> hk.getHost().equals(sshHostAndPort.getLeft())) - .findFirst().orElse(null); - // This will ensure a match between how the host key was hashed in the known_hosts file. - final String hostKeyType = (hostKey != null) ? hostKey.getType() : null; - // Set the hash algorithm - if (hostKeyType != null) { - session.setConfig(SERVER_HOST_KEY, hostKeyType); - } - // The default behaviour of `ssh-keygen` is to hash known hosts keys - session.setConfig(HASH_KNOWN_HOSTS, YES); - } - - private static void setKnownHostsFile( - final DocumentDbConnectionProperties connectionProperties, - final JSch jSch) throws SQLException { - if (!connectionProperties.getSshStrictHostKeyChecking()) { - return; - } - final String knowHostsFilename; - if (!isNullOrWhitespace(connectionProperties.getSshKnownHostsFile())) { - final Path knownHostsPath = getPath(connectionProperties.getSshKnownHostsFile()); - if (Files.exists(knownHostsPath)) { - knowHostsFilename = knownHostsPath.toString(); - } else { - throw SqlError.createSQLException( - LOGGER, - SqlState.INVALID_PARAMETER_VALUE, - SqlError.KNOWN_HOSTS_FILE_NOT_FOUND, - connectionProperties.getSshKnownHostsFile()); - } - } else { - knowHostsFilename = getPath(SSH_KNOWN_HOSTS_FILE).toString(); - } - try { - jSch.setKnownHosts(knowHostsFilename); - } catch (JSchException e) { - throw new SQLException(e.getMessage(), e); - } - } - - /** - * Container for the SSH port forwarding tunnel session. - */ - @Getter - @AllArgsConstructor - public static class SshPortForwardingSession { - /** - * Gets the SSH session. - */ - private final Session session; - /** - * Gets the local port for the port forwarding tunnel. - */ - private final int localPort; - } } diff --git a/src/main/java/software/amazon/documentdb/jdbc/DocumentDbConnectionProperties.java b/src/main/java/software/amazon/documentdb/jdbc/DocumentDbConnectionProperties.java index ba66099b..be8df8f9 100644 --- a/src/main/java/software/amazon/documentdb/jdbc/DocumentDbConnectionProperties.java +++ b/src/main/java/software/amazon/documentdb/jdbc/DocumentDbConnectionProperties.java @@ -44,6 +44,8 @@ import java.net.URI; import java.net.URISyntaxException; import java.net.URL; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.InvalidPathException; import java.nio.file.Path; @@ -53,6 +55,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.Properties; import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; @@ -64,22 +67,41 @@ public class DocumentDbConnectionProperties extends Properties { public static final String USER_HOME_PROPERTY = "user.home"; private static final Logger LOGGER = LoggerFactory.getLogger(DocumentDbConnectionProperties.class.getName()); - private static final String USER_HOME_PATH_NAME = System.getProperty(USER_HOME_PROPERTY); - private static final String DOCUMENTDB_HOME_PATH_NAME = Paths.get( - USER_HOME_PATH_NAME, ".documentdb").toString(); private static final Pattern WHITE_SPACE_PATTERN = Pattern.compile("^\\s*$"); private static final String ROOT_PEM_RESOURCE_FILE_NAME = "/rds-ca-2019-root.pem"; public static final String HOME_PATH_PREFIX_REG_EXPR = "^~[/\\\\].*$"; public static final int FETCH_SIZE_DEFAULT = 2000; public static final String DOCUMENTDB_CUSTOM_OPTIONS = "DOCUMENTDB_CUSTOM_OPTIONS"; - private static String classPathLocationName = null; private static String[] documentDbSearchPaths = null; static final String DEFAULT_APPLICATION_NAME; + public static final String USER_HOME_PATH_NAME = System.getProperty(USER_HOME_PROPERTY); + public static final String DOCUMENTDB_HOME_PATH_NAME = Paths.get( + USER_HOME_PATH_NAME, ".documentdb").toString(); + public static final String CONNECTION_STRING_TEMPLATE = "//%s%s/%s%s"; + static { DEFAULT_APPLICATION_NAME = DocumentDbDriver.DEFAULT_APPLICATION_NAME; } + /** + * Enumeration of type of validation. + */ + public enum ValidationType { + /** + * No validation. + */ + NONE, + /** + * Validate client connection required properties. + */ + CLIENT, + /** + * Validate SSH tunnel required properties. + */ + SSH_TUNNEL, + } + /** * Constructor for DocumentDbConnectionProperties, initializes with given properties. * @@ -114,36 +136,11 @@ public static String[] getDocumentDbSearchPaths() { } /** - * Gets the user's home path name. - * - * @return the user's home path name. - */ - static String getUserHomePathName() { - return USER_HOME_PATH_NAME; - } - - /** - * Gets the ~/.documentdb path name. + * Gets the parent folder location of the current class. * - * @return the ~/.documentdb path name. + * @return a string representing the parent folder location of the current class. */ - static String getDocumentdbHomePathName() { - return DOCUMENTDB_HOME_PATH_NAME; - } - - /** - * Gets the class path's location name. - * - * @return the class path's location name. - */ - static String getClassPathLocationName() { - if (classPathLocationName == null) { - classPathLocationName = getClassPathLocation(); - } - return classPathLocationName; - } - - private static String getClassPathLocation() { + public static String getClassPathLocation() { String classPathLocation = null; final URL classPathLocationUrl = DocumentDbConnectionProperties.class .getProtectionDomain() @@ -801,46 +798,150 @@ public MongoClientSettings buildMongoClientSettings( * * @return a {@link String} with the sanitized connection properties. */ - public String buildSanitizedConnectionString() { - final String connectionStringTemplate = "//%s%s/%s%s"; - final String loginInfo = isNullOrWhitespace(getUser()) ? "" : getUser() + "@"; - final String hostInfo = isNullOrWhitespace(getHostname()) ? "" : getHostname(); - final String databaseInfo = isNullOrWhitespace(getDatabase()) ? "" : getDatabase(); + public @NonNull String buildSanitizedConnectionString() { + final String loginInfo = buildLoginInfo(getUser(), null); + final String hostInfo = buildHostInfo(getHostname()); + final String databaseInfo = buildDatabaseInfo(getDatabase()); final StringBuilder optionalInfo = new StringBuilder(); - if (!getApplicationName() - .equals(DocumentDbConnectionProperty.APPLICATION_NAME.getDefaultValue())) { - appendOption(optionalInfo, DocumentDbConnectionProperty.APPLICATION_NAME, getApplicationName()); - } - if (getLoginTimeout() != Integer.parseInt(DocumentDbConnectionProperty.LOGIN_TIMEOUT_SEC.getDefaultValue())) { - appendOption(optionalInfo, DocumentDbConnectionProperty.LOGIN_TIMEOUT_SEC, getLoginTimeout()); - } - if (getMetadataScanLimit() != Integer.parseInt(DocumentDbConnectionProperty.METADATA_SCAN_LIMIT.getDefaultValue())) { - appendOption(optionalInfo, DocumentDbConnectionProperty.METADATA_SCAN_LIMIT, getMetadataScanLimit()); - } - if (getMetadataScanMethod() != DocumentDbMetadataScanMethod.fromString(DocumentDbConnectionProperty.METADATA_SCAN_METHOD.getDefaultValue())) { - appendOption(optionalInfo, DocumentDbConnectionProperty.METADATA_SCAN_METHOD, getMetadataScanMethod().getName()); - } - if (getRetryReadsEnabled() != Boolean.parseBoolean(DocumentDbConnectionProperty.RETRY_READS_ENABLED.getDefaultValue())) { - appendOption(optionalInfo, DocumentDbConnectionProperty.RETRY_READS_ENABLED, getRetryReadsEnabled()); + buildSanitizedOptionalInfo(optionalInfo, this); + return buildConnectionString(loginInfo, hostInfo, databaseInfo, optionalInfo.toString()); + } + + @NonNull static String buildDatabaseInfo(final @Nullable String database) { + return isNullOrWhitespace(database) ? "" : encodeValue(database); + } + + @NonNull static String buildHostInfo(final @Nullable String hostname) { + return isNullOrWhitespace(hostname) ? "" : hostname; + } + + @NonNull static String buildLoginInfo(final @Nullable String user, final @Nullable String password) { + final String userString = isNullOrWhitespace(user) + ? "" + : encodeValue(user); + final String passwordString = isNullOrWhitespace(password) + ? "" + : ":" + encodeValue(password); + final String userInfo = isNullOrWhitespace(userString) && isNullOrWhitespace(passwordString) + ? "" + : "@"; + return userString + passwordString + userInfo; + } + + static @NonNull String buildConnectionString( + final String loginInfo, + final String hostInfo, + final String databaseInfo, + final String optionalInfo) { + return String.format(CONNECTION_STRING_TEMPLATE, + loginInfo, + hostInfo, + databaseInfo, + optionalInfo); + } + + /** + * Builds the sanitized optional info connection string. I does not include + * sensitive options like SSH_PRIVATE_KEY_PASSPHRASE. + * + * @param optionalInfo the connection string to build. + */ + static void buildSanitizedOptionalInfo( + final StringBuilder optionalInfo, + final DocumentDbConnectionProperties properties) { + maybeAppendOptionalValue(optionalInfo, DocumentDbConnectionProperty.APPLICATION_NAME, properties.getApplicationName()); + maybeAppendOptionalValue(optionalInfo, DocumentDbConnectionProperty.LOGIN_TIMEOUT_SEC, properties.getLoginTimeout()); + maybeAppendOptionalValue(optionalInfo, DocumentDbConnectionProperty.METADATA_SCAN_LIMIT, properties.getMetadataScanLimit()); + maybeAppendOptionalValue(optionalInfo, properties.getMetadataScanMethod()); + maybeAppendOptionalValue(optionalInfo, DocumentDbConnectionProperty.RETRY_READS_ENABLED, properties.getRetryReadsEnabled()); + maybeAppendOptionalValue(optionalInfo, properties.getReadPreference()); + maybeAppendOptionalValue(optionalInfo, DocumentDbConnectionProperty.REPLICA_SET, properties.getReplicaSet(), null); + maybeAppendOptionalValue(optionalInfo, DocumentDbConnectionProperty.TLS_ENABLED, properties.getTlsEnabled()); + maybeAppendOptionalValue(optionalInfo, DocumentDbConnectionProperty.TLS_ALLOW_INVALID_HOSTNAMES, properties.getTlsAllowInvalidHostnames()); + maybeAppendOptionalValue(optionalInfo, DocumentDbConnectionProperty.TLS_CA_FILE, properties.getTlsCAFilePath(), null); + maybeAppendOptionalValue(optionalInfo, DocumentDbConnectionProperty.SCHEMA_NAME, properties.getSchemaName()); + maybeAppendOptionalValue(optionalInfo, DocumentDbConnectionProperty.SSH_USER, properties.getSshUser(), null); + maybeAppendOptionalValue(optionalInfo, DocumentDbConnectionProperty.SSH_HOSTNAME, properties.getSshHostname(), null); + maybeAppendOptionalValue(optionalInfo, DocumentDbConnectionProperty.SSH_PRIVATE_KEY_FILE, properties.getSshPrivateKeyFile(), null); + maybeAppendOptionalValue(optionalInfo, DocumentDbConnectionProperty.SSH_STRICT_HOST_KEY_CHECKING, properties.getSshStrictHostKeyChecking()); + maybeAppendOptionalValue(optionalInfo, DocumentDbConnectionProperty.SSH_KNOWN_HOSTS_FILE, properties.getSshKnownHostsFile(), null); + maybeAppendOptionalValue(optionalInfo, DocumentDbConnectionProperty.DEFAULT_FETCH_SIZE, properties.getDefaultFetchSize()); + maybeAppendOptionalValue(optionalInfo, DocumentDbConnectionProperty.REFRESH_SCHEMA, properties.getRefreshSchema()); + maybeAppendOptionalValue(optionalInfo, DocumentDbConnectionProperty.DEFAULT_AUTH_DB, properties.getDefaultAuthenticationDatabase()); + maybeAppendOptionalValue(optionalInfo, properties.getAllowDiskUseOption()); + } + + static void maybeAppendOptionalValue(final StringBuilder optionalInfo, + final DocumentDbConnectionProperty property, + final String value) { + if (!property.getDefaultValue().equals(value)) { + appendOption(optionalInfo, property, value); } - if (getReadPreference() != null) { - appendOption(optionalInfo, DocumentDbConnectionProperty.READ_PREFERENCE, getReadPreference().getName()); + } + + static void maybeAppendOptionalValue(final StringBuilder optionalInfo, + final DocumentDbConnectionProperty property, + final String value, + final String defaultValue) { + if (!Objects.equals(defaultValue, value)) { + appendOption(optionalInfo, property, value); } - if (getReplicaSet() != null) { - appendOption(optionalInfo, DocumentDbConnectionProperty.REPLICA_SET, getReplicaSet()); + } + + static void maybeAppendOptionalValue(final StringBuilder optionalInfo, + final DocumentDbConnectionProperty property, + final int value) { + if (value != Integer.parseInt(property.getDefaultValue())) { + appendOption(optionalInfo, property, value); } - if (getTlsEnabled() != Boolean.parseBoolean(DocumentDbConnectionProperty.TLS_ENABLED.getDefaultValue())) { - appendOption(optionalInfo, DocumentDbConnectionProperty.TLS_ENABLED, getTlsEnabled()); + } + + static void maybeAppendOptionalValue(final StringBuilder optionalInfo, + final DocumentDbConnectionProperty property, + final boolean value) { + if (value != Boolean.parseBoolean(property.getDefaultValue())) { + appendOption(optionalInfo, property, value); } - if (getTlsAllowInvalidHostnames() != Boolean.parseBoolean(DocumentDbConnectionProperty.TLS_ALLOW_INVALID_HOSTNAMES.getDefaultValue())) { - appendOption(optionalInfo, DocumentDbConnectionProperty.TLS_ALLOW_INVALID_HOSTNAMES, getTlsAllowInvalidHostnames()); + } + + static void maybeAppendOptionalValue(final StringBuilder optionalInfo, + final DocumentDbMetadataScanMethod value) { + if (value != DocumentDbMetadataScanMethod.fromString( + DocumentDbConnectionProperty.METADATA_SCAN_METHOD.getDefaultValue())) { + appendOption(optionalInfo, DocumentDbConnectionProperty.METADATA_SCAN_METHOD, value.getName()); } - if (getTlsCAFilePath() != null) { - appendOption(optionalInfo, DocumentDbConnectionProperty.TLS_CA_FILE, getTlsCAFilePath()); + } + + static void maybeAppendOptionalValue(final StringBuilder optionalInfo, + final DocumentDbReadPreference value) { + if (value != null) { + appendOption(optionalInfo, DocumentDbConnectionProperty.READ_PREFERENCE, value.getName()); } - if (!DocumentDbConnectionProperty.SCHEMA_NAME.getDefaultValue().equals(getSchemaName())) { - appendOption(optionalInfo, DocumentDbConnectionProperty.SCHEMA_NAME, getSchemaName()); + } + + static void maybeAppendOptionalValue(final StringBuilder optionalInfo, + final DocumentDbAllowDiskUseOption value) { + if (value != DocumentDbAllowDiskUseOption.fromString( + DocumentDbConnectionProperty.ALLOW_DISK_USE.getDefaultValue())) { + appendOption(optionalInfo, DocumentDbConnectionProperty.ALLOW_DISK_USE, value.getName()); } + } + + /** + * Builds the connection string for SSH properties. + * + * @return a connection string with SSH properties. + */ + public String buildSshConnectionString() { + final String loginInfo = ""; + final String hostInfo = buildHostInfo(getHostname()); + final String databaseInfo = ""; + final StringBuilder optionalInfo = new StringBuilder(); + buildSshOptionalInfo(optionalInfo); + return buildConnectionString(loginInfo, hostInfo, databaseInfo, optionalInfo.toString()); + } + + private void buildSshOptionalInfo(final StringBuilder optionalInfo) { if (getSshUser() != null) { appendOption(optionalInfo, DocumentDbConnectionProperty.SSH_USER, getSshUser()); } @@ -850,36 +951,45 @@ public String buildSanitizedConnectionString() { if (getSshPrivateKeyFile() != null) { appendOption(optionalInfo, DocumentDbConnectionProperty.SSH_PRIVATE_KEY_FILE, getSshPrivateKeyFile()); } + if (getSshPrivateKeyPassphrase() != null && !DocumentDbConnectionProperty.SSH_PRIVATE_KEY_PASSPHRASE.getDefaultValue().equals(getSshPrivateKeyPassphrase())) { + appendOption(optionalInfo, DocumentDbConnectionProperty.SSH_PRIVATE_KEY_PASSPHRASE, getSshPrivateKeyPassphrase()); + } if (getSshStrictHostKeyChecking() != Boolean.parseBoolean(DocumentDbConnectionProperty.SSH_STRICT_HOST_KEY_CHECKING.getDefaultValue())) { appendOption(optionalInfo, DocumentDbConnectionProperty.SSH_STRICT_HOST_KEY_CHECKING, getSshStrictHostKeyChecking()); } if (getSshKnownHostsFile() != null && !DocumentDbConnectionProperty.SSH_KNOWN_HOSTS_FILE.getDefaultValue().equals(getSshKnownHostsFile())) { appendOption(optionalInfo, DocumentDbConnectionProperty.SSH_KNOWN_HOSTS_FILE, getSshKnownHostsFile()); } - if (getDefaultFetchSize() != Integer.parseInt(DocumentDbConnectionProperty.DEFAULT_FETCH_SIZE.getDefaultValue())) { - appendOption(optionalInfo, DocumentDbConnectionProperty.DEFAULT_FETCH_SIZE, getDefaultFetchSize()); - } - if (getRefreshSchema() != Boolean.parseBoolean(DocumentDbConnectionProperty.REFRESH_SCHEMA.getDefaultValue())) { - appendOption(optionalInfo, DocumentDbConnectionProperty.REFRESH_SCHEMA, getRefreshSchema()); - } - if (getDefaultAuthenticationDatabase() != null && !DocumentDbConnectionProperty.DEFAULT_AUTH_DB.getDefaultValue().equals(getDefaultAuthenticationDatabase())) { - appendOption(optionalInfo, DocumentDbConnectionProperty.DEFAULT_AUTH_DB, getDefaultAuthenticationDatabase()); - } - if (getAllowDiskUseOption() != DocumentDbAllowDiskUseOption.fromString(DocumentDbConnectionProperty.ALLOW_DISK_USE.getDefaultValue())) { - appendOption(optionalInfo, DocumentDbConnectionProperty.ALLOW_DISK_USE, getAllowDiskUseOption().getName()); - } - return String.format(connectionStringTemplate, - loginInfo, - hostInfo, - databaseInfo, - optionalInfo); } - private void appendOption(final StringBuilder optionInfo, + /** + * Appends an option and value to the string. + * + * @param optionInfo the connection string to build. + * @param option the option to add. + * @param optionValue the option value to set. + */ + public static void appendOption(final StringBuilder optionInfo, final DocumentDbConnectionProperty option, final Object optionValue) { optionInfo.append(optionInfo.length() == 0 ? "?" : "&"); - optionInfo.append(option.getName()).append("=").append(optionValue); + optionInfo.append(option.getName()) + .append("=") + .append(optionValue == null ? "" : encodeValue(optionValue.toString())); + } + + /** + * Encodes a value into URL encoded value. + * + * @param value the value to encode. + * @return the encoded value. + */ + public static String encodeValue(final String value) { + try { + return URLEncoder.encode(value, StandardCharsets.UTF_8.toString()); + } catch (UnsupportedEncodingException e) { + return value; + } } /** @@ -887,28 +997,60 @@ private void appendOption(final StringBuilder optionInfo, * @throws SQLException if the required properties are not correctly set. */ public void validateRequiredProperties() throws SQLException { - if (isNullOrWhitespace(getUser()) - || isNullOrWhitespace(getPassword())) { + validateRequiredProperties(ValidationType.CLIENT); + } + + /** + * Validates the existing properties. + * @param validationType Which validation type to perform. + * @throws SQLException if the required properties are not correctly set. + */ + public void validateRequiredProperties(final ValidationType validationType) throws SQLException { + if ((isNullOrWhitespace(getUser()) + || isNullOrWhitespace(getPassword())) && validationType == ValidationType.CLIENT) { throw SqlError.createSQLException( LOGGER, SqlState.INVALID_PARAMETER_VALUE, SqlError.MISSING_USER_PASSWORD ); } - if (isNullOrWhitespace(getDatabase())) { + if (isNullOrWhitespace(getDatabase()) && validationType == ValidationType.CLIENT) { throw SqlError.createSQLException( LOGGER, SqlState.INVALID_PARAMETER_VALUE, SqlError.MISSING_DATABASE ); } - if (isNullOrWhitespace(getHostname())) { + if (isNullOrWhitespace(getHostname()) + && (validationType == ValidationType.CLIENT || validationType == ValidationType.SSH_TUNNEL)) { throw SqlError.createSQLException( LOGGER, SqlState.INVALID_PARAMETER_VALUE, SqlError.MISSING_HOSTNAME ); } + + if (isNullOrWhitespace(getSshUser()) && validationType == ValidationType.SSH_TUNNEL) { + throw SqlError.createSQLException( + LOGGER, + SqlState.INVALID_PARAMETER_VALUE, + SqlError.MISSING_SSH_USER + ); + } + if (isNullOrWhitespace(getSshHostname()) && validationType == ValidationType.SSH_TUNNEL) { + throw SqlError.createSQLException( + LOGGER, + SqlState.INVALID_PARAMETER_VALUE, + SqlError.MISSING_SSH_HOSTNAME + ); + } + if (isNullOrWhitespace(getSshPrivateKeyFile()) && validationType == ValidationType.SSH_TUNNEL) { + throw SqlError.createSQLException( + LOGGER, + SqlState.INVALID_PARAMETER_VALUE, + SqlError.MISSING_SSH_PRIVATE_KEY_FILE + ); + } } /** @@ -923,6 +1065,19 @@ public static DocumentDbConnectionProperties getPropertiesFromConnectionString(f return getPropertiesFromConnectionString(new Properties(), documentDbUrl, DOCUMENT_DB_SCHEME); } + /** + * Gets the connection properties from the connection string. + * + * @param documentDbUrl the given properties. + * @param validationType Which properties to validate. + * @return a {@link DocumentDbConnectionProperties} with the properties set. + * @throws SQLException if connection string is invalid. + */ + public static DocumentDbConnectionProperties getPropertiesFromConnectionString( + final String documentDbUrl, + final ValidationType validationType) throws SQLException { + return getPropertiesFromConnectionString(new Properties(), documentDbUrl, DOCUMENT_DB_SCHEME, validationType); + } /** * Gets the connection properties from the connection string. @@ -936,6 +1091,22 @@ public static DocumentDbConnectionProperties getPropertiesFromConnectionString(f public static DocumentDbConnectionProperties getPropertiesFromConnectionString( final Properties info, final String documentDbUrl, final String connectionStringPrefix) throws SQLException { + return getPropertiesFromConnectionString(info, documentDbUrl, connectionStringPrefix, ValidationType.CLIENT); + } + + /** + * Gets the connection properties from the connection string. + * + * @param info the given properties. + * @param documentDbUrl the connection string. + * @param connectionStringPrefix the connection string prefix. + * @param validationType Which validation to perform. + * @return a {@link DocumentDbConnectionProperties} with the properties set. + * @throws SQLException if connection string is invalid. + */ + public static DocumentDbConnectionProperties getPropertiesFromConnectionString( + final Properties info, final String documentDbUrl, final String connectionStringPrefix, + final ValidationType validationType) throws SQLException { final DocumentDbConnectionProperties properties = new DocumentDbConnectionProperties(info); final String postSchemeSuffix = documentDbUrl.substring(connectionStringPrefix.length()); @@ -943,11 +1114,11 @@ public static DocumentDbConnectionProperties getPropertiesFromConnectionString( try { final URI uri = new URI(postSchemeSuffix); - setHostName(properties, uri); + setHostName(properties, uri, validationType); - setUserPassword(properties, uri); + setUserPassword(properties, uri, validationType); - setDatabase(properties, uri); + setDatabase(properties, uri, validationType); setOptionalProperties(properties, uri); @@ -957,15 +1128,17 @@ public static DocumentDbConnectionProperties getPropertiesFromConnectionString( throw SqlError.createSQLException( LOGGER, SqlState.CONNECTION_FAILURE, + e, SqlError.INVALID_CONNECTION_PROPERTIES, - documentDbUrl + documentDbUrl + " : '" + e.getMessage() + "'" ); } catch (UnsupportedEncodingException e) { throw new SQLException(e.getMessage(), e); } } - properties.validateRequiredProperties(); + properties.validateRequiredProperties(validationType); + return properties; } @@ -984,11 +1157,13 @@ static void setCustomOptions(final DocumentDbConnectionProperties properties) { } } - private static void setDatabase(final Properties properties, final URI mongoUri) - throws SQLException { + private static void setDatabase( + final Properties properties, + final URI mongoUri, + final ValidationType validationType) throws SQLException { if (isNullOrWhitespace(mongoUri.getPath())) { - if (properties.getProperty( - DocumentDbConnectionProperty.DATABASE.getName(), null) == null) { + if (properties.getProperty(DocumentDbConnectionProperty.DATABASE.getName(), null) == null + && validationType == ValidationType.CLIENT) { throw SqlError.createSQLException( LOGGER, SqlState.CONNECTION_FAILURE, @@ -1018,13 +1193,15 @@ private static void setOptionalProperties(final Properties properties, final URI } } - private static void setUserPassword(final Properties properties, final URI mongoUri) - throws UnsupportedEncodingException, SQLException { + private static void setUserPassword( + final Properties properties, + final URI mongoUri, + final ValidationType validationType) throws UnsupportedEncodingException, SQLException { if (mongoUri.getUserInfo() == null) { - if (properties.getProperty( - DocumentDbConnectionProperty.USER.getName(), null) == null - || properties.getProperty( - DocumentDbConnectionProperty.PASSWORD.getName(), null) == null) { + if ((properties.getProperty(DocumentDbConnectionProperty.USER.getName(), null) == null + && validationType == ValidationType.CLIENT) + || (properties.getProperty(DocumentDbConnectionProperty.PASSWORD.getName(), null) == null + && validationType == ValidationType.CLIENT)) { throw SqlError.createSQLException( LOGGER, SqlState.CONNECTION_FAILURE, @@ -1048,11 +1225,14 @@ private static void setUserPassword(final Properties properties, final URI mongo } } - private static void setHostName(final Properties properties, final URI mongoUri) throws SQLException { - String hostName = mongoUri.getHost(); + private static void setHostName( + final Properties properties, + final URI uri, + final ValidationType validationType) throws SQLException { + String hostName = uri.getHost(); if (hostName == null) { - if (properties.getProperty( - DocumentDbConnectionProperty.HOSTNAME.getName(), null) == null) { + if (properties.getProperty(DocumentDbConnectionProperty.HOSTNAME.getName(), null) == null + && (validationType == ValidationType.CLIENT || validationType == ValidationType.SSH_TUNNEL)) { throw SqlError.createSQLException( LOGGER, SqlState.CONNECTION_FAILURE, @@ -1061,8 +1241,8 @@ private static void setHostName(final Properties properties, final URI mongoUri) return; } - if (mongoUri.getPort() > 0) { - hostName += ":" + mongoUri.getPort(); + if (uri.getPort() > 0) { + hostName += ":" + uri.getPort(); } addPropertyIfNotSet(properties, DocumentDbConnectionProperty.HOSTNAME.getName(), hostName); @@ -1259,31 +1439,31 @@ void appendEmbeddedAndOptionalCaCertificates(final List caCertifica * Gets an absolute path from the given file path. It performs the substitution for a leading * '~' to be replaced by the user's home directory. * - * @param filePath the given file path to process. + * @param filePathString the given file path to process. * @param searchFolders list of folders * @return a {@link Path} for the absolution path for the given file path. */ - public static Path getPath(final String filePath, final String... searchFolders) { - if (filePath.matches(HOME_PATH_PREFIX_REG_EXPR)) { - final String fromHomePath = filePath.replaceFirst("~", + public static Path getPath(final String filePathString, final String... searchFolders) { + final Path filePath = Paths.get(filePathString); + if (filePathString.matches(HOME_PATH_PREFIX_REG_EXPR)) { + final String fromHomePath = filePathString.replaceFirst("~", Matcher.quoteReplacement(USER_HOME_PATH_NAME)); return Paths.get(fromHomePath).toAbsolutePath(); } else { - final Path origFilePath = Paths.get(filePath); - if (origFilePath.isAbsolute()) { - return origFilePath; + if (filePath.isAbsolute()) { + return filePath; } for (String searchFolder : searchFolders) { if (searchFolder == null) { continue; } - final Path testPath = Paths.get(searchFolder, filePath); + final Path testPath = Paths.get(searchFolder, filePathString); if (testPath.toAbsolutePath().toFile().exists()) { return testPath; } } } - return Paths.get(filePath).toAbsolutePath(); + return filePath.toAbsolutePath(); } /** @@ -1318,7 +1498,7 @@ private DocumentDbMetadataScanMethod getPropertyAsScanMethod(@NonNull final Stri if (getProperty(key) != null) { property = DocumentDbMetadataScanMethod.fromString(getProperty(key)); } else if (DocumentDbConnectionProperty.getPropertyFromKey(key) != null) { - property = DocumentDbMetadataScanMethod.fromString( + property = DocumentDbMetadataScanMethod.fromString( DocumentDbConnectionProperty.getPropertyFromKey(key).getDefaultValue()); } } catch (IllegalArgumentException e) { diff --git a/src/main/java/software/amazon/documentdb/jdbc/DocumentDbConnectionProperty.java b/src/main/java/software/amazon/documentdb/jdbc/DocumentDbConnectionProperty.java index 9c674f6d..849e31ed 100644 --- a/src/main/java/software/amazon/documentdb/jdbc/DocumentDbConnectionProperty.java +++ b/src/main/java/software/amazon/documentdb/jdbc/DocumentDbConnectionProperty.java @@ -16,6 +16,7 @@ package software.amazon.documentdb.jdbc; +import org.checkerframework.checker.nullness.qual.NonNull; import software.amazon.documentdb.jdbc.common.utilities.ConnectionProperty; import java.util.Arrays; @@ -130,9 +131,9 @@ public enum DocumentDbConnectionProperty implements ConnectionProperty { * @param description Description of the property. */ DocumentDbConnectionProperty( - final String connectionProperty, - final String defaultValue, - final String description) { + final @NonNull String connectionProperty, + final @NonNull String defaultValue, + final @NonNull String description) { this.connectionProperty = connectionProperty; this.defaultValue = defaultValue; this.description = description; @@ -143,7 +144,7 @@ public enum DocumentDbConnectionProperty implements ConnectionProperty { * * @return the connection property. */ - public String getName() { + public @NonNull String getName() { return connectionProperty; } @@ -152,7 +153,7 @@ public String getName() { * * @return the default value of the connection property. */ - public String getDefaultValue() { + public @NonNull String getDefaultValue() { return defaultValue; } diff --git a/src/main/java/software/amazon/documentdb/jdbc/DocumentDbMain.java b/src/main/java/software/amazon/documentdb/jdbc/DocumentDbMain.java index ba6d4e40..dfbe5a5c 100644 --- a/src/main/java/software/amazon/documentdb/jdbc/DocumentDbMain.java +++ b/src/main/java/software/amazon/documentdb/jdbc/DocumentDbMain.java @@ -57,10 +57,10 @@ import software.amazon.documentdb.jdbc.metadata.DocumentDbSchemaColumn; import software.amazon.documentdb.jdbc.metadata.DocumentDbSchemaTable; import software.amazon.documentdb.jdbc.persist.DocumentDbSchemaSecurityException; +import software.amazon.documentdb.jdbc.sshtunnel.DocumentDbSshTunnelService; import java.io.Console; import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStreamWriter; import java.io.PrintWriter; @@ -68,6 +68,7 @@ import java.io.Writer; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.sql.SQLException; @@ -117,6 +118,7 @@ public class DocumentDbMain { private static final Options HELP_VERSION_OPTIONS; private static final Option HELP_OPTION; private static final Option VERSION_OPTION; + private static final Options SSH_TUNNEL_SERVICE_OPTIONS; private static final OptionGroup COMMAND_OPTIONS; private static final List