Remove hard-coded collation in JDBC adapter (#1518)

scalar-labs · Feb 22, 2024 · 19aee9d · 19aee9d
1 parent 6c343ce
commit 19aee9d
Show file tree

Hide file tree

Showing 9 changed files with 385 additions and 76 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -227,15 +227,11 @@ jobs:
     name: MySQL 5.7 integration test
     runs-on: ubuntu-latest
 
-    services:
-      mysql:
-        image: mysql:5.7
-        env:
-          MYSQL_ROOT_PASSWORD: mysql
-        ports:
-          - 3306:3306
-
     steps:
+      - name: Run MySQL 5.7
+        run: |
+          docker run -e MYSQL_ROOT_PASSWORD=mysql -p 3306:3306 -d mysql:5.7 --character-set-server=utf8mb4 --collation-server=utf8mb4_bin
+
       - uses: actions/checkout@v4
 
       - name: Set up JDK 8
@@ -260,15 +256,11 @@ jobs:
     name: MySQL 8.0 integration test
     runs-on: ubuntu-latest
 
-    services:
-      mysql:
-        image: mysql:8.0
-        env:
-          MYSQL_ROOT_PASSWORD: mysql
-        ports:
-          - 3306:3306
-
     steps:
+      - name: Run MySQL 8.0
+        run: |
+          docker run -e MYSQL_ROOT_PASSWORD=mysql -p 3306:3306 -d mysql:8.0 --character-set-server=utf8mb4 --collation-server=utf8mb4_bin
+
       - uses: actions/checkout@v4
 
       - name: Set up JDK 8
@@ -293,15 +285,11 @@ jobs:
     name: MySQL 8.1 integration test
     runs-on: ubuntu-latest
 
-    services:
-      mysql:
-        image: mysql:8.1
-        env:
-          MYSQL_ROOT_PASSWORD: mysql
-        ports:
-          - 3306:3306
-
     steps:
+      - name: Run MySQL 8.1
+        run: |
+          docker run -e MYSQL_ROOT_PASSWORD=mysql -p 3306:3306 -d mysql:8.1 --character-set-server=utf8mb4 --collation-server=utf8mb4_bin
+
       - uses: actions/checkout@v4
 
       - name: Set up JDK 8
@@ -607,6 +595,7 @@ jobs:
           MSSQL_PID: "Express"
           SA_PASSWORD: "SqlServer17"
           ACCEPT_EULA: "Y"
+          MSSQL_COLLATION: "Japanese_BIN2"
         ports:
           - 1433:1433
 
@@ -642,6 +631,7 @@ jobs:
           MSSQL_PID: "Express"
           SA_PASSWORD: "SqlServer19"
           ACCEPT_EULA: "Y"
+          MSSQL_COLLATION: "Japanese_BIN2"
         ports:
           - 1433:1433
 
@@ -677,6 +667,7 @@ jobs:
           MSSQL_PID: "Express"
           SA_PASSWORD: "SqlServer22"
           ACCEPT_EULA: "Y"
+          MSSQL_COLLATION: "Japanese_BIN2"
         ports:
           - 1433:1433
 
@@ -733,15 +724,11 @@ jobs:
     name: MariaDB 10 integration test
     runs-on: ubuntu-latest
 
-    services:
-      mariadb:
-        image: mariadb:10.11
-        env:
-          MYSQL_ROOT_PASSWORD: mysql
-        ports:
-          - 3306:3306
-
     steps:
+      - name: Run MariaDB 10.11
+        run: |
+          docker run -e MYSQL_ROOT_PASSWORD=mysql -p 3306:3306 -d mariadb:10.11 --character-set-server=utf8mb4 --collation-server=utf8mb4_bin
+
       - uses: actions/checkout@v4
 
       - name: Set up JDK 8
@@ -774,14 +761,12 @@ jobs:
           HEAP_NEWSIZE: 512m
         ports:
           - 9042:9042
-      mysql:
-        image: mysql:8
-        env:
-          MYSQL_ROOT_PASSWORD: mysql
-        ports:
-          - 3306:3306
 
     steps:
+      - name: Run MySQL 8
+        run: |
+          docker run -e MYSQL_ROOT_PASSWORD=mysql -p 3306:3306 -d mysql:8 --character-set-server=utf8mb4 --collation-server=utf8mb4_bin
+
       - uses: actions/checkout@v4
 
       - name: Set up JDK 8
@@ -806,15 +791,11 @@ jobs:
     name: ScalarDB Server integration test
     runs-on: ubuntu-latest
 
-    services:
-      mysql:
-        image: mysql:8
-        env:
-          MYSQL_ROOT_PASSWORD: mysql
-        ports:
-          - 3306:3306
-
     steps:
+      - name: Run MySQL 8
+        run: |
+          docker run -e MYSQL_ROOT_PASSWORD=mysql -p 3306:3306 -d mysql:8 --character-set-server=utf8mb4 --collation-server=utf8mb4_bin
+
       - uses: actions/checkout@v4
 
       - name: Set up JDK 8

diff --git a/...tegration-test/java/com/scalar/db/storage/cassandra/CassandraJapaneseIntegrationTest.java b/...tegration-test/java/com/scalar/db/storage/cassandra/CassandraJapaneseIntegrationTest.java
@@ -0,0 +1,19 @@
+package com.scalar.db.storage.cassandra;
+
+import com.scalar.db.api.DistributedStorageJapaneseIntegrationTestBase;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Properties;
+
+public class CassandraJapaneseIntegrationTest
+    extends DistributedStorageJapaneseIntegrationTestBase {
+  @Override
+  protected Properties getProperties(String testName) {
+    return CassandraEnv.getProperties(testName);
+  }
+
+  @Override
+  protected Map<String, String> getCreationOptions() {
+    return Collections.singletonMap(CassandraAdmin.REPLICATION_FACTOR, "1");
+  }
+}
diff --git a/...src/integration-test/java/com/scalar/db/storage/cosmos/CosmosJapaneseIntegrationTest.java b/...src/integration-test/java/com/scalar/db/storage/cosmos/CosmosJapaneseIntegrationTest.java
@@ -0,0 +1,26 @@
+package com.scalar.db.storage.cosmos;
+
+import com.scalar.db.api.DistributedStorageJapaneseIntegrationTestBase;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Properties;
+
+public class CosmosJapaneseIntegrationTest extends DistributedStorageJapaneseIntegrationTestBase {
+
+  @Override
+  protected Properties getProperties(String testName) {
+    return CosmosEnv.getProperties(testName);
+  }
+
+  @Override
+  protected String getNamespace() {
+    String namespace = super.getNamespace();
+    Optional<String> databasePrefix = CosmosEnv.getDatabasePrefix();
+    return databasePrefix.map(prefix -> prefix + namespace).orElse(namespace);
+  }
+
+  @Override
+  protected Map<String, String> getCreationOptions() {
+    return CosmosEnv.getCreationOptions();
+  }
+}
diff --git a/...src/integration-test/java/com/scalar/db/storage/dynamo/DynamoJapaneseIntegrationTest.java b/...src/integration-test/java/com/scalar/db/storage/dynamo/DynamoJapaneseIntegrationTest.java
@@ -0,0 +1,18 @@
+package com.scalar.db.storage.dynamo;
+
+import com.scalar.db.api.DistributedStorageJapaneseIntegrationTestBase;
+import java.util.Map;
+import java.util.Properties;
+
+public class DynamoJapaneseIntegrationTest extends DistributedStorageJapaneseIntegrationTestBase {
+
+  @Override
+  protected Properties getProperties(String testName) {
+    return DynamoEnv.getProperties(testName);
+  }
+
+  @Override
+  protected Map<String, String> getCreationOptions() {
+    return DynamoEnv.getCreationOptions();
+  }
+}
diff --git a/...integration-test/java/com/scalar/db/storage/jdbc/JdbcDatabaseJapaneseIntegrationTest.java b/...integration-test/java/com/scalar/db/storage/jdbc/JdbcDatabaseJapaneseIntegrationTest.java
@@ -0,0 +1,13 @@
+package com.scalar.db.storage.jdbc;
+
+import com.scalar.db.api.DistributedStorageJapaneseIntegrationTestBase;
+import java.util.Properties;
+
+public class JdbcDatabaseJapaneseIntegrationTest
+    extends DistributedStorageJapaneseIntegrationTestBase {
+
+  @Override
+  protected Properties getProperties(String testName) {
+    return JdbcEnv.getProperties(testName);
+  }
+}
diff --git a/core/src/main/java/com/scalar/db/storage/jdbc/RdbEngineMysql.java b/core/src/main/java/com/scalar/db/storage/jdbc/RdbEngineMysql.java
@@ -23,16 +23,12 @@ class RdbEngineMysql implements RdbEngineStrategy {
 
   @Override
   public String[] createSchemaSqls(String fullSchema) {
-    return new String[] {
-      "CREATE SCHEMA " + enclose(fullSchema) + " character set utf8 COLLATE utf8_bin"
-    };
+    return new String[] {"CREATE SCHEMA " + enclose(fullSchema)};
   }
 
   @Override
   public String[] createSchemaIfNotExistsSqls(String schema) {
-    return new String[] {
-      "CREATE SCHEMA IF NOT EXISTS " + enclose(schema) + " character set utf8 COLLATE utf8_bin"
-    };
+    return new String[] {"CREATE SCHEMA IF NOT EXISTS " + enclose(schema)};
   }
 
   @Override

diff --git a/core/src/main/java/com/scalar/db/storage/jdbc/RdbEngineSqlServer.java b/core/src/main/java/com/scalar/db/storage/jdbc/RdbEngineSqlServer.java
@@ -176,7 +176,7 @@ public String getDataTypeForEngine(DataType scalarDbDataType) {
       case INT:
         return "INT";
       case TEXT:
-        return "VARCHAR(8000) COLLATE Latin1_General_BIN";
+        return "VARCHAR(8000)";
       default:
         throw new AssertionError();
     }

diff --git a/core/src/test/java/com/scalar/db/storage/jdbc/JdbcAdminTest.java b/core/src/test/java/com/scalar/db/storage/jdbc/JdbcAdminTest.java
@@ -288,12 +288,9 @@ public void createNamespace_forMysql_shouldExecuteCreateNamespaceStatement()
       throws ExecutionException, SQLException {
     createNamespace_forX_shouldExecuteCreateNamespaceStatement(
         RdbEngine.MYSQL,
-        Collections.singletonList("CREATE SCHEMA `my_ns` character set utf8 COLLATE utf8_bin"),
+        Collections.singletonList("CREATE SCHEMA `my_ns`"),
         "SELECT 1 FROM `" + METADATA_SCHEMA + "`.`namespaces` LIMIT 1",
-        Collections.singletonList(
-            "CREATE SCHEMA IF NOT EXISTS `"
-                + METADATA_SCHEMA
-                + "` character set utf8 COLLATE utf8_bin"),
+        Collections.singletonList("CREATE SCHEMA IF NOT EXISTS `" + METADATA_SCHEMA + "`"),
         "CREATE TABLE IF NOT EXISTS `"
             + METADATA_SCHEMA
             + "`.`namespaces`(`namespace_name` VARCHAR(128), PRIMARY KEY (`namespace_name`))",
@@ -472,7 +469,7 @@ public void createTableInternal_ForSqlServer_ShouldCreateTableAndIndexes()
       throws ExecutionException, SQLException {
     createTableInternal_ForX_CreateTableAndIndexes(
         RdbEngine.SQL_SERVER,
-        "CREATE TABLE [my_ns].[foo_table]([c3] BIT,[c1] VARCHAR(8000) COLLATE Latin1_General_BIN,"
+        "CREATE TABLE [my_ns].[foo_table]([c3] BIT,[c1] VARCHAR(8000),"
             + "[c4] VARBINARY(8000),[c2] BIGINT,[c5] INT,[c6] FLOAT,[c7] FLOAT(24), PRIMARY KEY ([c3] ASC,[c1] DESC,[c4] ASC))",
         "CREATE INDEX [index_my_ns_foo_table_c4] ON [my_ns].[foo_table] ([c4])",
         "CREATE INDEX [index_my_ns_foo_table_c1] ON [my_ns].[foo_table] ([c1])");
@@ -569,7 +566,7 @@ public void createTableInternal_IfNotExistsForSqlServer_ShouldCreateTableAndInde
       throws ExecutionException, SQLException {
     createTableInternal_IfNotExistsForX_createTableAndIndexesIfNotExists(
         RdbEngine.SQL_SERVER,
-        "CREATE TABLE [my_ns].[foo_table]([c3] BIT,[c1] VARCHAR(8000) COLLATE Latin1_General_BIN,"
+        "CREATE TABLE [my_ns].[foo_table]([c3] BIT,[c1] VARCHAR(8000),"
             + "[c4] VARBINARY(8000),[c2] BIGINT,[c5] INT,[c6] FLOAT,[c7] FLOAT(24), PRIMARY KEY ([c3] ASC,[c1] DESC,[c4] ASC))",
         "CREATE INDEX [index_my_ns_foo_table_c4] ON [my_ns].[foo_table] ([c4])",
         "CREATE INDEX [index_my_ns_foo_table_c1] ON [my_ns].[foo_table] ([c1])");
@@ -754,7 +751,7 @@ public void addTableMetadata_ifNotExistsAndOverwriteMetadataForMysql_ShouldWorkP
       throws Exception {
     addTableMetadata_createMetadataTableIfNotExistsForXAndOverwriteMetadata_ShouldWorkProperly(
         RdbEngine.MYSQL,
-        "CREATE SCHEMA IF NOT EXISTS `" + METADATA_SCHEMA + "` character set utf8 COLLATE utf8_bin",
+        "CREATE SCHEMA IF NOT EXISTS `" + METADATA_SCHEMA + "`",
         "CREATE TABLE IF NOT EXISTS `"
             + METADATA_SCHEMA
             + "`.`metadata`("
@@ -920,7 +917,7 @@ public void addTableMetadata_ifNotExistsAndDoNotOverwriteMetadataForMysql_Should
       throws Exception {
     addTableMetadata_createMetadataTableIfNotExistsForX_ShouldWorkProperly(
         RdbEngine.MYSQL,
-        "CREATE SCHEMA IF NOT EXISTS `" + METADATA_SCHEMA + "` character set utf8 COLLATE utf8_bin",
+        "CREATE SCHEMA IF NOT EXISTS `" + METADATA_SCHEMA + "`",
         "CREATE TABLE IF NOT EXISTS `"
             + METADATA_SCHEMA
             + "`.`metadata`("
@@ -2883,13 +2880,9 @@ public void repairNamespace_forMysql_shouldCreateNamespaceIfNotExistsAndUpsertMe
       throws ExecutionException, SQLException {
     repairNamespace_forX_shouldWorkProperly(
         RdbEngine.MYSQL,
-        Collections.singletonList(
-            "CREATE SCHEMA IF NOT EXISTS `my_ns` character set utf8 COLLATE utf8_bin"),
+        Collections.singletonList("CREATE SCHEMA IF NOT EXISTS `my_ns`"),
         "SELECT 1 FROM `scalardb`.`namespaces` LIMIT 1",
-        Collections.singletonList(
-            "CREATE SCHEMA IF NOT EXISTS `"
-                + METADATA_SCHEMA
-                + "` character set utf8 COLLATE utf8_bin"),
+        Collections.singletonList("CREATE SCHEMA IF NOT EXISTS `" + METADATA_SCHEMA + "`"),
         "CREATE TABLE IF NOT EXISTS `"
             + METADATA_SCHEMA
             + "`.`namespaces`(`namespace_name` VARCHAR(128), PRIMARY KEY (`namespace_name`))",
@@ -3031,10 +3024,7 @@ public void upgrade_ForMysql_ShouldInsertAllNamespacesFromMetadataTable()
         RdbEngine.MYSQL,
         "SELECT 1 FROM `" + METADATA_SCHEMA + "`.`metadata` LIMIT 1",
         "SELECT 1 FROM `" + METADATA_SCHEMA + "`.`namespaces` LIMIT 1",
-        ImmutableList.of(
-            "CREATE SCHEMA IF NOT EXISTS `"
-                + METADATA_SCHEMA
-                + "` character set utf8 COLLATE utf8_bin"),
+        ImmutableList.of("CREATE SCHEMA IF NOT EXISTS `" + METADATA_SCHEMA + "`"),
         "CREATE TABLE IF NOT EXISTS `"
             + METADATA_SCHEMA
             + "`.`namespaces`(`namespace_name` VARCHAR(128), PRIMARY KEY (`namespace_name`))",
@@ -3244,10 +3234,7 @@ public void upsertIntoNamespacesTable_ForExistingNamespace_ShouldNotThrowExcepti
     createNamespaceTableIfNotExists_forX_shouldCreateMetadataSchemaAndNamespacesTableIfNotExists(
         RdbEngine.MYSQL,
         "SELECT 1 FROM `" + METADATA_SCHEMA + "`.`namespaces` LIMIT 1",
-        Collections.singletonList(
-            "CREATE SCHEMA IF NOT EXISTS `"
-                + METADATA_SCHEMA
-                + "` character set utf8 COLLATE utf8_bin"),
+        Collections.singletonList("CREATE SCHEMA IF NOT EXISTS `" + METADATA_SCHEMA + "`"),
         "CREATE TABLE IF NOT EXISTS `"
             + METADATA_SCHEMA
             + "`.`namespaces`(`namespace_name` VARCHAR(128), PRIMARY KEY (`namespace_name`))",