diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergGlueExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergGlueExternalCatalog.java index 9f3e05757c9ec6..ffe48e68a49af2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergGlueExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergGlueExternalCatalog.java @@ -58,7 +58,7 @@ protected void initCatalog() { catalogProperties.get(S3Properties.Env.ENDPOINT)); catalogProperties.putIfAbsent(S3FileIOProperties.ENDPOINT, endpoint); - glueCatalog.initialize(icebergCatalogType, catalogProperties); + glueCatalog.initialize(getName(), catalogProperties); catalog = glueCatalog; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHMSExternalCatalog.java index 34e6f0c187e5ba..c1475064934232 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHMSExternalCatalog.java @@ -44,7 +44,7 @@ protected void initCatalog() { Map catalogProperties = catalogProperty.getProperties(); String metastoreUris = catalogProperty.getOrDefault(HMSProperties.HIVE_METASTORE_URIS, ""); catalogProperties.put(CatalogProperties.URI, metastoreUris); - hiveCatalog.initialize(icebergCatalogType, catalogProperties); + hiveCatalog.initialize(getName(), catalogProperties); catalog = hiveCatalog; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHadoopExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHadoopExternalCatalog.java index 01eee31d3b08d3..bf9e8c2b3f06fb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHadoopExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHadoopExternalCatalog.java @@ -60,7 +60,7 @@ protected void initCatalog() { String warehouse = catalogProperty.getHadoopProperties().get(CatalogProperties.WAREHOUSE_LOCATION); hadoopCatalog.setConf(conf); catalogProperties.put(CatalogProperties.WAREHOUSE_LOCATION, warehouse); - hadoopCatalog.initialize(icebergCatalogType, catalogProperties); + hadoopCatalog.initialize(getName(), catalogProperties); catalog = hadoopCatalog; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index edb83ac5bc8970..66c57e3730707f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -107,6 +107,11 @@ public void createDb(CreateDbStmt stmt) throws DdlException { ErrorReport.reportDdlException(ErrorCode.ERR_DB_CREATE_EXISTS, dbName); } } + String icebergCatalogType = dorisCatalog.getIcebergCatalogType(); + if (!properties.isEmpty() && !IcebergExternalCatalog.ICEBERG_HMS.equals(icebergCatalogType)) { + throw new DdlException( + "Not supported: create database with properties for iceberg catalog type: " + icebergCatalogType); + } nsCatalog.createNamespace(Namespace.of(dbName), properties); dorisCatalog.onRefreshCache(true); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergRestExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergRestExternalCatalog.java index 77a6a7404ef444..908a4fa9e3f271 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergRestExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergRestExternalCatalog.java @@ -47,7 +47,7 @@ protected void initCatalog() { Configuration conf = replaceS3Properties(getConfiguration()); - catalog = CatalogUtil.buildIcebergCatalog(icebergCatalogType, + catalog = CatalogUtil.buildIcebergCatalog(getName(), convertToRestCatalogProperties(), conf); } diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_show_create.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_show_create.groovy index 8065998fa71199..23705c5494ddf0 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_show_create.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_show_create.groovy @@ -22,21 +22,23 @@ suite("test_iceberg_show_create", "p0,external,doris,external_docker,external_do return } - String rest_port = context.config.otherConfigs.get("iceberg_rest_uri_port") - String minio_port = context.config.otherConfigs.get("iceberg_minio_port") - String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") String catalog_name = "test_iceberg_show_create" + String hivePrefix = "hive2"; + String hms_port = context.config.otherConfigs.get(hivePrefix + "HmsPort") + String hdfs_port = context.config.otherConfigs.get(hivePrefix + "HdfsPort") + String iceberg_catalog_name = "test_iceberg_write_partitions_iceberg_${hivePrefix}" + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String default_fs = "hdfs://${externalEnvIp}:${hdfs_port}" + String warehouse = "${default_fs}/warehouse" sql """drop catalog if exists ${catalog_name}""" - sql """ - CREATE CATALOG ${catalog_name} PROPERTIES ( + sql """create catalog if not exists ${catalog_name} properties ( 'type'='iceberg', - 'iceberg.catalog.type'='rest', - 'uri' = 'http://${externalEnvIp}:${rest_port}', - "s3.access_key" = "admin", - "s3.secret_key" = "password", - "s3.endpoint" = "http://${externalEnvIp}:${minio_port}", - "s3.region" = "us-east-1" + 'iceberg.catalog.type'='hms', + 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}', + 'fs.defaultFS' = '${default_fs}', + 'warehouse' = '${warehouse}', + 'use_meta_cache' = 'true' );""" sql """ switch ${catalog_name} """ @@ -49,25 +51,49 @@ suite("test_iceberg_show_create", "p0,external,doris,external_docker,external_do sql """ drop database if exists ${db1} """ sql """ drop database if exists ${db2} """ - sql """ create database ${db1} properties ('location'='s3a://warehouse/wh/${db1}') """ + sql """ create database ${db1} properties ('location'='${warehouse}/other_location') """ sql """ create database ${db2} """ String result = "" result = sql "show create database ${db1}" logger.info("${result}") - assertTrue(result.toString().containsIgnoreCase("s3a://warehouse/wh/${db1}")) + assertTrue(result.toString().containsIgnoreCase("${warehouse}/other_location")) result = sql "show create database ${db2}" logger.info("${result}") - assertTrue(result.toString().containsIgnoreCase("s3a://warehouse/wh/${db2}")) + assertTrue(result.toString().containsIgnoreCase("${warehouse}/${db2}")) sql """ create table ${db1}.${tb1} (id int) """ result = sql "show create table ${db1}.${tb1}" logger.info("${result}") - assertTrue(result.toString().containsIgnoreCase("s3a://warehouse/wh/${db1}/${tb1}")) + assertTrue(result.toString().containsIgnoreCase("${warehouse}/other_location/${tb1}")) sql """ drop table ${db1}.${tb1} """ sql """ drop database ${db1} """ sql """ drop database ${db2} """ + String rest_port = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minio_port = context.config.otherConfigs.get("iceberg_minio_port") + sql """drop catalog if exists ${catalog_name}""" + sql """ + CREATE CATALOG ${catalog_name} PROPERTIES ( + 'type'='iceberg', + 'iceberg.catalog.type'='rest', + 'uri' = 'http://${externalEnvIp}:${rest_port}', + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "http://${externalEnvIp}:${minio_port}", + "s3.region" = "us-east-1" + );""" + + sql """ switch ${catalog_name} """ + sql """ drop database if exists ${db1} """ + + test { + sql """ create database ${db1} properties ('location'='${warehouse}/other_location') """ + exception "Not supported: create database with properties for iceberg catalog type" + } + + sql """ drop database if exists ${db1} """ + sql """drop catalog if exists ${catalog_name}""" }