From 962c3820776057fd48df39ce4f9f889118def4ba Mon Sep 17 00:00:00 2001 From: zy-kkk Date: Fri, 6 Sep 2024 19:30:32 +0800 Subject: [PATCH] [fix](jdbc catalog) Fix type recognition error when using `query` tvf to query doris (#40481) pick (#40122) Using string to match Doris type will not work with query tvf, so use field matching instead --- .../jdbc/client/JdbcMySQLClient.java | 25 +++++++++++-------- .../datasource/jdbc/util/JdbcFieldSchema.java | 13 ++++++++++ .../jdbc/test_doris_jdbc_catalog.out | 17 +++++++++++++ .../jdbc/test_doris_jdbc_catalog.groovy | 3 +++ 4 files changed, 47 insertions(+), 11 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java index f58753bccee03d..5624392de14c39 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java @@ -35,6 +35,7 @@ import java.sql.Statement; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.function.Consumer; @@ -174,7 +175,7 @@ protected Set getFilterInternalDatabases() { protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) { // For Doris type if (isDoris) { - return dorisTypeToDoris(fieldSchema.getDataTypeName().orElse("unknown").toUpperCase()); + return dorisTypeToDoris(fieldSchema); } // For mysql type: "INT UNSIGNED": // fieldSchema.getDataTypeName().orElse("unknown").split(" ")[0] == "INT" @@ -319,7 +320,8 @@ private Map getColumnsDataTypeUseQuery(String remoteDbName, Stri return fieldtoType; } - private Type dorisTypeToDoris(String type) { + private Type dorisTypeToDoris(JdbcFieldSchema fieldSchema) { + String type = fieldSchema.getDataTypeName().orElse("unknown").toUpperCase(); if (type == null || type.isEmpty()) { return Type.UNSUPPORTED; } @@ -329,7 +331,9 @@ private Type dorisTypeToDoris(String type) { // For ARRAY type if (upperType.startsWith("ARRAY")) { String innerType = upperType.substring(6, upperType.length() - 1).trim(); - Type arrayInnerType = dorisTypeToDoris(innerType); + JdbcFieldSchema innerFieldSchema = new JdbcFieldSchema(fieldSchema); + innerFieldSchema.setDataTypeName(Optional.of(innerType)); + Type arrayInnerType = dorisTypeToDoris(innerFieldSchema); return ArrayType.create(arrayInnerType, true); } @@ -356,9 +360,8 @@ private Type dorisTypeToDoris(String type) { return Type.DOUBLE; case "DECIMAL": case "DECIMALV3": { - String[] params = upperType.substring(openParen + 1, upperType.length() - 1).split(","); - int precision = Integer.parseInt(params[0].trim()); - int scale = Integer.parseInt(params[1].trim()); + int precision = fieldSchema.getColumnSize().orElse(0); + int scale = fieldSchema.getDecimalDigits().orElse(0); return createDecimalOrStringType(precision, scale); } case "DATE": @@ -374,11 +377,11 @@ private Type dorisTypeToDoris(String type) { return ScalarType.createDatetimeV2Type(scale); } case "CHAR": - case "VARCHAR": { - int length = Integer.parseInt(upperType.substring(openParen + 1, upperType.length() - 1)); - return baseType.equals("CHAR") - ? ScalarType.createCharType(length) : ScalarType.createVarcharType(length); - } + ScalarType charType = ScalarType.createType(PrimitiveType.CHAR); + charType.setLength(fieldSchema.getColumnSize().orElse(0)); + return charType; + case "VARCHAR": + return ScalarType.createVarcharType(fieldSchema.getColumnSize().orElse(0)); case "STRING": case "TEXT": case "JSON": diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/util/JdbcFieldSchema.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/util/JdbcFieldSchema.java index 735de93e9eb2ca..90850ffca21607 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/util/JdbcFieldSchema.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/util/JdbcFieldSchema.java @@ -47,6 +47,19 @@ public class JdbcFieldSchema { protected int charOctetLength; protected boolean isAllowNull; + + public JdbcFieldSchema(JdbcFieldSchema other) { + this.columnName = other.columnName; + this.dataType = other.dataType; + this.dataTypeName = other.dataTypeName; + this.columnSize = other.columnSize; + this.decimalDigits = other.decimalDigits; + this.numPrecRadix = other.numPrecRadix; + this.remarks = other.remarks; + this.charOctetLength = other.charOctetLength; + this.isAllowNull = other.isAllowNull; + } + public JdbcFieldSchema(ResultSet rs) throws SQLException { this.columnName = rs.getString("COLUMN_NAME"); this.dataType = getInteger(rs, "DATA_TYPE").orElseThrow(() -> new IllegalStateException("DATA_TYPE is null")); diff --git a/regression-test/data/external_table_p0/jdbc/test_doris_jdbc_catalog.out b/regression-test/data/external_table_p0/jdbc/test_doris_jdbc_catalog.out index aabc7c64d178a9..3fec3f8a5748f2 100644 --- a/regression-test/data/external_table_p0/jdbc/test_doris_jdbc_catalog.out +++ b/regression-test/data/external_table_p0/jdbc/test_doris_jdbc_catalog.out @@ -124,6 +124,23 @@ true 1 1 1 1 1 1.0 1.0 1.00000 1.0000000000 2021-01-01 2021-01-01T00:00 a a {"a" g1 1 2 3 4 p1 g2 1 2 3 4 p2 +-- !sql -- +bool_col tinyint Yes true \N NONE +tinyint_col tinyint Yes true \N NONE +smallint_col smallint Yes true \N NONE +int_col int Yes true \N NONE +bigint_col bigint Yes true \N NONE +largeint_col char(85) Yes true \N NONE +float_col float Yes true \N NONE +double_col double Yes true \N NONE +decimal_col decimal(10,5) Yes true \N NONE +decimal_col2 decimal(30,10) Yes true \N NONE +date_col date Yes true \N NONE +datetime_col datetime(6) Yes true \N NONE +char_col char(85) Yes true \N NONE +varchar_col char(85) Yes true \N NONE +json_col text Yes true \N NONE + -- !sql -- doris_jdbc_catalog diff --git a/regression-test/suites/external_table_p0/jdbc/test_doris_jdbc_catalog.groovy b/regression-test/suites/external_table_p0/jdbc/test_doris_jdbc_catalog.groovy index bdccd9fa5e1265..2051a4ad54d6da 100644 --- a/regression-test/suites/external_table_p0/jdbc/test_doris_jdbc_catalog.groovy +++ b/regression-test/suites/external_table_p0/jdbc/test_doris_jdbc_catalog.groovy @@ -228,6 +228,9 @@ suite("test_doris_jdbc_catalog", "p0,external,doris,external_docker,external_doc sql """insert into test_insert_order(gameid,did,cid,bid,aid,pname) select 'g2',4,3,2,1,'p2'"""; qt_sql """select * from test_insert_order order by gameid, aid, bid, cid, did;""" + // test query tvf + qt_sql """desc function query("catalog" = "doris_jdbc_catalog", "query" = "select * from regression_test_jdbc_catalog_p0.base");""" + //clean qt_sql """select current_catalog()""" sql "switch internal"