From 78519b44d5f34072c0b9ddfbcc3a2698979a28b5 Mon Sep 17 00:00:00 2001 From: panbingkun Date: Mon, 20 May 2024 11:22:01 +0800 Subject: [PATCH 1/6] [ONLY TEST] Test mariadb 3.x --- .../org/apache/spark/sql/jdbc/MariaDBKrbIntegrationSuite.scala | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MariaDBKrbIntegrationSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MariaDBKrbIntegrationSuite.scala index 6825c001f7670..2f1af506bcb20 100644 --- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MariaDBKrbIntegrationSuite.scala +++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MariaDBKrbIntegrationSuite.scala @@ -38,7 +38,7 @@ class MariaDBKrbIntegrationSuite extends DockerKrbJDBCIntegrationSuite { override protected val keytabFileName = "mariadb.keytab" override val db = new DatabaseOnDocker { - override val imageName = sys.env.getOrElse("MARIADB_DOCKER_IMAGE_NAME", "mariadb:10.5.12") + override val imageName = sys.env.getOrElse("MARIADB_DOCKER_IMAGE_NAME", "mariadb:lts") override val env = Map( "MYSQL_ROOT_PASSWORD" -> "rootpass" ) diff --git a/pom.xml b/pom.xml index 611e82f343d8c..ea2637c1278be 100644 --- a/pom.xml +++ b/pom.xml @@ -322,7 +322,7 @@ -Djdk.reflect.useDirectMethodHandle=false -Dio.netty.tryReflectionSetAccessible=true - 2.7.12 + 3.4.0 8.3.0 42.7.3 11.5.9.0 From e32a4edded7cab7c5278ebfab0ea210914718ad7 Mon Sep 17 00:00:00 2001 From: panbingkun Date: Tue, 21 May 2024 14:45:49 +0800 Subject: [PATCH 2/6] add permitMysqlScheme --- .../org/apache/spark/sql/jdbc/MariaDBKrbIntegrationSuite.scala | 2 +- .../jdbc/connection/MariaDBConnectionProviderSuite.scala | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MariaDBKrbIntegrationSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MariaDBKrbIntegrationSuite.scala index 6825c001f7670..68d828b59450a 100644 --- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MariaDBKrbIntegrationSuite.scala +++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MariaDBKrbIntegrationSuite.scala @@ -46,7 +46,7 @@ class MariaDBKrbIntegrationSuite extends DockerKrbJDBCIntegrationSuite { override val jdbcPort = 3306 override def getJdbcUrl(ip: String, port: Int): String = - s"jdbc:mysql://$ip:$port/mysql?user=$principal" + s"jdbc:mysql://$ip:$port/mysql?permitMysqlScheme&user=$principal" override def getEntryPoint: Option[String] = Some("/docker-entrypoint/mariadb_docker_entrypoint.sh") diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/MariaDBConnectionProviderSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/MariaDBConnectionProviderSuite.scala index d8bdf26b35c7d..8792c49624c34 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/MariaDBConnectionProviderSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/MariaDBConnectionProviderSuite.scala @@ -22,6 +22,7 @@ class MariaDBConnectionProviderSuite extends ConnectionProviderSuiteBase { val provider = new MariaDBConnectionProvider() val driver = registerDriver(provider.driverClass) - testSecureConnectionProvider(provider, driver, options("jdbc:mysql://localhost/mysql")) + testSecureConnectionProvider(provider, driver, + options("jdbc:mysql://localhost/mysql?permitMysqlScheme")) } } From 68ade95ca6f613d386345f8495c8f6faffd44c7a Mon Sep 17 00:00:00 2001 From: panbingkun Date: Tue, 21 May 2024 15:56:41 +0800 Subject: [PATCH 3/6] fix --- .../sql/jdbc/MySQLIntegrationSuite.scala | 57 ++++++++++++------- .../sql/jdbc/v2/MySQLIntegrationSuite.scala | 4 +- 2 files changed, 37 insertions(+), 24 deletions(-) diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala index e6cca2ac9cd0a..f9bb547cac854 100644 --- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala +++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala @@ -154,30 +154,43 @@ class MySQLIntegrationSuite extends DockerJDBCIntegrationSuite { test("SPARK-47462: Unsigned numeric types") { val df = sqlContext.read.jdbc(jdbcUrl, "unsigned_numbers", new Properties) val rows = df.head() - assert(rows.get(0).isInstanceOf[Short]) - assert(rows.get(1).isInstanceOf[Integer]) - assert(rows.get(2).isInstanceOf[Integer]) - assert(rows.get(3).isInstanceOf[Long]) - assert(rows.get(4).isInstanceOf[BigDecimal]) - assert(rows.get(5).isInstanceOf[BigDecimal]) - assert(rows.get(6).isInstanceOf[Double]) - // Unlike MySQL, MariaDB seems not to distinguish signed and unsigned tinyint(1). val isMaria = jdbcUrl.indexOf("disableMariaDbDriver") == -1 if (isMaria) { + assert(rows.get(0).isInstanceOf[Integer]) + assert(rows.get(1).isInstanceOf[Long]) + assert(rows.get(2).isInstanceOf[Integer]) + assert(rows.get(3).isInstanceOf[BigDecimal]) + assert(rows.get(4).isInstanceOf[BigDecimal]) + assert(rows.get(5).isInstanceOf[BigDecimal]) + assert(rows.get(6).isInstanceOf[Double]) + // Unlike MySQL, MariaDB seems not to distinguish signed and unsigned tinyint(1). assert(rows.get(7).isInstanceOf[Boolean]) - } else { - assert(rows.get(7).isInstanceOf[Short]) - } - assert(rows.getShort(0) === 255) - assert(rows.getInt(1) === 65535) - assert(rows.getInt(2) === 16777215) - assert(rows.getLong(3) === 4294967295L) - assert(rows.getAs[BigDecimal](4).equals(new BigDecimal("9223372036854775808"))) - assert(rows.getAs[BigDecimal](5).equals(new BigDecimal("123456789012345.123456789012345000"))) - assert(rows.getDouble(6) === 1.0000000000000002) - if (isMaria) { + + assert(rows.getInt(0) === 255) + assert(rows.getLong(1) === 65535L) + assert(rows.getInt(2) === 16777215) + assert(rows.getAs[BigDecimal](3).equals(new BigDecimal("4294967295"))) + assert(rows.getAs[BigDecimal](4).equals(new BigDecimal("9223372036854775808"))) + assert(rows.getAs[BigDecimal](5).equals(new BigDecimal("123456789012345.123456789012345000"))) + assert(rows.getDouble(6) === 1.0000000000000002) assert(rows.getBoolean(7) === false) } else { + assert(rows.get(0).isInstanceOf[Short]) + assert(rows.get(1).isInstanceOf[Integer]) + assert(rows.get(2).isInstanceOf[Integer]) + assert(rows.get(3).isInstanceOf[Long]) + assert(rows.get(4).isInstanceOf[BigDecimal]) + assert(rows.get(5).isInstanceOf[BigDecimal]) + assert(rows.get(6).isInstanceOf[Double]) + assert(rows.get(7).isInstanceOf[Short]) + + assert(rows.getShort(0) === 255) + assert(rows.getInt(1) === 65535) + assert(rows.getInt(2) === 16777215) + assert(rows.getLong(3) === 4294967295L) + assert(rows.getAs[BigDecimal](4).equals(new BigDecimal("9223372036854775808"))) + assert(rows.getAs[BigDecimal](5).equals(new BigDecimal("123456789012345.123456789012345000"))) + assert(rows.getDouble(6) === 1.0000000000000002) assert(rows.getShort(7) === 0) } } @@ -367,13 +380,13 @@ class MySQLOverMariaConnectorIntegrationSuite extends MySQLIntegrationSuite { override val db = new MySQLDatabaseOnDocker { override def getJdbcUrl(ip: String, port: Int): String = - s"jdbc:mysql://$ip:$port/mysql?user=root&password=rootpass&allowPublicKeyRetrieval=true" + - s"&useSSL=false" + s"jdbc:mysql://$ip:$port/mysql?permitMysqlScheme&user=root&password=rootpass" + + s"&allowPublicKeyRetrieval=true&useSSL=false" } override def testConnection(): Unit = { Using.resource(getConnection()) { conn => - assert(conn.getClass.getName === "org.mariadb.jdbc.MariaDbConnection") + assert(conn.getClass.getName === "org.mariadb.jdbc.Connection") } } } diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/MySQLIntegrationSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/MySQLIntegrationSuite.scala index d5478e664221d..bc14486f2fbb9 100644 --- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/MySQLIntegrationSuite.scala +++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/MySQLIntegrationSuite.scala @@ -177,7 +177,7 @@ class MySQLOverMariaConnectorIntegrationSuite extends MySQLIntegrationSuite { override val db = new MySQLDatabaseOnDocker { override def getJdbcUrl(ip: String, port: Int): String = - s"jdbc:mysql://$ip:$port/mysql?user=root&password=rootpass&allowPublicKeyRetrieval=true" + - s"&useSSL=false" + s"jdbc:mysql://$ip:$port/mysql?permitMysqlScheme&user=root&password=rootpass" + + s"&allowPublicKeyRetrieval=true&useSSL=false" } } From 94334aadd7750b457d9876ed63ff7bcfa7a4e678 Mon Sep 17 00:00:00 2001 From: panbingkun Date: Tue, 21 May 2024 16:14:16 +0800 Subject: [PATCH 4/6] fix JDBCSuite --- .../src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala index 9c117c50cc2df..74cd52040d92b 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala @@ -1400,7 +1400,7 @@ class JDBCSuite extends QueryTest with SharedSparkSession { test("SPARK-18419: Fix `asConnectionProperties` to filter case-insensitively") { val parameters = Map( - "url" -> "jdbc:mysql://localhost:3306/temp", + "url" -> "jdbc:mysql://localhost:3306/temp?permitMysqlScheme", "dbtable" -> "t1", "numPartitions" -> "10") assert(new JDBCOptions(parameters).asConnectionProperties.isEmpty) From e30e5ed7841d1ad7aa803eba32fc7cee7ee337bd Mon Sep 17 00:00:00 2001 From: panbingkun Date: Wed, 22 May 2024 17:22:34 +0800 Subject: [PATCH 5/6] update MariaDB image to 10.5.24 --- .../apache/spark/sql/jdbc/MariaDBKrbIntegrationSuite.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MariaDBKrbIntegrationSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MariaDBKrbIntegrationSuite.scala index 68d828b59450a..2a2b65de8b434 100644 --- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MariaDBKrbIntegrationSuite.scala +++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MariaDBKrbIntegrationSuite.scala @@ -25,9 +25,9 @@ import org.apache.spark.sql.execution.datasources.jdbc.connection.SecureConnecti import org.apache.spark.tags.DockerTest /** - * To run this test suite for a specific version (e.g., mariadb:10.5.12): + * To run this test suite for a specific version (e.g., mariadb:10.5.24): * {{{ - * ENABLE_DOCKER_INTEGRATION_TESTS=1 MARIADB_DOCKER_IMAGE_NAME=mariadb:10.5.12 + * ENABLE_DOCKER_INTEGRATION_TESTS=1 MARIADB_DOCKER_IMAGE_NAME=mariadb:10.5.24 * ./build/sbt -Pdocker-integration-tests * "docker-integration-tests/testOnly org.apache.spark.sql.jdbc.MariaDBKrbIntegrationSuite" * }}} @@ -38,7 +38,7 @@ class MariaDBKrbIntegrationSuite extends DockerKrbJDBCIntegrationSuite { override protected val keytabFileName = "mariadb.keytab" override val db = new DatabaseOnDocker { - override val imageName = sys.env.getOrElse("MARIADB_DOCKER_IMAGE_NAME", "mariadb:10.5.12") + override val imageName = sys.env.getOrElse("MARIADB_DOCKER_IMAGE_NAME", "mariadb:10.5.24") override val env = Map( "MYSQL_ROOT_PASSWORD" -> "rootpass" ) From 91a3d08438ee434798dd476cb4de9d8e46961ac9 Mon Sep 17 00:00:00 2001 From: panbingkun Date: Thu, 23 May 2024 17:34:02 +0800 Subject: [PATCH 6/6] update --- .../sql/jdbc/MySQLIntegrationSuite.scala | 51 +++++++------------ .../apache/spark/sql/jdbc/MySQLDialect.scala | 8 ++- 2 files changed, 26 insertions(+), 33 deletions(-) diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala index f9bb547cac854..bc22aa05efa52 100644 --- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala +++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala @@ -154,43 +154,30 @@ class MySQLIntegrationSuite extends DockerJDBCIntegrationSuite { test("SPARK-47462: Unsigned numeric types") { val df = sqlContext.read.jdbc(jdbcUrl, "unsigned_numbers", new Properties) val rows = df.head() + assert(rows.get(0).isInstanceOf[Short]) + assert(rows.get(1).isInstanceOf[Integer]) + assert(rows.get(2).isInstanceOf[Integer]) + assert(rows.get(3).isInstanceOf[Long]) + assert(rows.get(4).isInstanceOf[BigDecimal]) + assert(rows.get(5).isInstanceOf[BigDecimal]) + assert(rows.get(6).isInstanceOf[Double]) + // Unlike MySQL, MariaDB seems not to distinguish signed and unsigned tinyint(1). val isMaria = jdbcUrl.indexOf("disableMariaDbDriver") == -1 if (isMaria) { - assert(rows.get(0).isInstanceOf[Integer]) - assert(rows.get(1).isInstanceOf[Long]) - assert(rows.get(2).isInstanceOf[Integer]) - assert(rows.get(3).isInstanceOf[BigDecimal]) - assert(rows.get(4).isInstanceOf[BigDecimal]) - assert(rows.get(5).isInstanceOf[BigDecimal]) - assert(rows.get(6).isInstanceOf[Double]) - // Unlike MySQL, MariaDB seems not to distinguish signed and unsigned tinyint(1). assert(rows.get(7).isInstanceOf[Boolean]) - - assert(rows.getInt(0) === 255) - assert(rows.getLong(1) === 65535L) - assert(rows.getInt(2) === 16777215) - assert(rows.getAs[BigDecimal](3).equals(new BigDecimal("4294967295"))) - assert(rows.getAs[BigDecimal](4).equals(new BigDecimal("9223372036854775808"))) - assert(rows.getAs[BigDecimal](5).equals(new BigDecimal("123456789012345.123456789012345000"))) - assert(rows.getDouble(6) === 1.0000000000000002) - assert(rows.getBoolean(7) === false) } else { - assert(rows.get(0).isInstanceOf[Short]) - assert(rows.get(1).isInstanceOf[Integer]) - assert(rows.get(2).isInstanceOf[Integer]) - assert(rows.get(3).isInstanceOf[Long]) - assert(rows.get(4).isInstanceOf[BigDecimal]) - assert(rows.get(5).isInstanceOf[BigDecimal]) - assert(rows.get(6).isInstanceOf[Double]) assert(rows.get(7).isInstanceOf[Short]) - - assert(rows.getShort(0) === 255) - assert(rows.getInt(1) === 65535) - assert(rows.getInt(2) === 16777215) - assert(rows.getLong(3) === 4294967295L) - assert(rows.getAs[BigDecimal](4).equals(new BigDecimal("9223372036854775808"))) - assert(rows.getAs[BigDecimal](5).equals(new BigDecimal("123456789012345.123456789012345000"))) - assert(rows.getDouble(6) === 1.0000000000000002) + } + assert(rows.getShort(0) === 255) + assert(rows.getInt(1) === 65535) + assert(rows.getInt(2) === 16777215) + assert(rows.getLong(3) === 4294967295L) + assert(rows.getAs[BigDecimal](4).equals(new BigDecimal("9223372036854775808"))) + assert(rows.getAs[BigDecimal](5).equals(new BigDecimal("123456789012345.123456789012345000"))) + assert(rows.getDouble(6) === 1.0000000000000002) + if (isMaria) { + assert(rows.getBoolean(7) === false) + } else { assert(rows.getShort(7) === 0) } } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MySQLDialect.scala b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MySQLDialect.scala index 50951042737a2..8e7063a187835 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MySQLDialect.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MySQLDialect.scala @@ -142,10 +142,14 @@ private case class MySQLDialect() extends JdbcDialect with SQLConfHelper { case Types.SMALLINT => if (md.build().getBoolean("isSigned")) { Some(ShortType) + } else if ("TINYINT UNSIGNED".equalsIgnoreCase(typeName)) { + Some(ShortType) } else { Some(IntegerType) } - case Types.INTEGER if "MEDIUMINT UNSIGNED".equalsIgnoreCase(typeName) => + case Types.INTEGER + if "MEDIUMINT UNSIGNED".equalsIgnoreCase(typeName) || + "SMALLINT UNSIGNED".equalsIgnoreCase(typeName) => // Signed values in [-8388608, 8388607] and unsigned values in [0, 16777215], // both of them fit IntegerType Some(IntegerType) @@ -158,6 +162,8 @@ private case class MySQLDialect() extends JdbcDialect with SQLConfHelper { // scalastyle:on line.size.limit Some(getTimestampType(md.build())) case Types.TIMESTAMP => Some(TimestampType) + case Types.BIGINT if "INTEGER UNSIGNED".equalsIgnoreCase(typeName) => + Some(LongType) case _ => None } }