[bugfix](iceberg)Restrictions on creating a database (apache#39641)

## Proposed changes 1. Restrictions on creating a database. Currently, only attributes of the `hms` type of database can be added. Like: ``` create database db properties ('a'='b'); ``` 2. Unify the catalog name of iceberg and doris to be the same.
seawinde · Aug 26, 2024 · 2a5b30d · 2a5b30d
1 parent ce9cd66
commit 2a5b30d
Show file tree

Hide file tree

Showing 6 changed files with 50 additions and 19 deletions.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergGlueExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergGlueExternalCatalog.java
@@ -58,7 +58,7 @@ protected void initCatalog() {
                 catalogProperties.get(S3Properties.Env.ENDPOINT));
         catalogProperties.putIfAbsent(S3FileIOProperties.ENDPOINT, endpoint);
 
-        glueCatalog.initialize(icebergCatalogType, catalogProperties);
+        glueCatalog.initialize(getName(), catalogProperties);
         catalog = glueCatalog;
     }
 

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHMSExternalCatalog.java
@@ -44,7 +44,7 @@ protected void initCatalog() {
         Map<String, String> catalogProperties = catalogProperty.getProperties();
         String metastoreUris = catalogProperty.getOrDefault(HMSProperties.HIVE_METASTORE_URIS, "");
         catalogProperties.put(CatalogProperties.URI, metastoreUris);
-        hiveCatalog.initialize(icebergCatalogType, catalogProperties);
+        hiveCatalog.initialize(getName(), catalogProperties);
         catalog = hiveCatalog;
     }
 }

diff --git a/...-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHadoopExternalCatalog.java b/...-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHadoopExternalCatalog.java
@@ -60,7 +60,7 @@ protected void initCatalog() {
         String warehouse = catalogProperty.getHadoopProperties().get(CatalogProperties.WAREHOUSE_LOCATION);
         hadoopCatalog.setConf(conf);
         catalogProperties.put(CatalogProperties.WAREHOUSE_LOCATION, warehouse);
-        hadoopCatalog.initialize(icebergCatalogType, catalogProperties);
+        hadoopCatalog.initialize(getName(), catalogProperties);
         catalog = hadoopCatalog;
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
@@ -107,6 +107,11 @@ public void createDb(CreateDbStmt stmt) throws DdlException {
                 ErrorReport.reportDdlException(ErrorCode.ERR_DB_CREATE_EXISTS, dbName);
             }
         }
+        String icebergCatalogType = dorisCatalog.getIcebergCatalogType();
+        if (!properties.isEmpty() && !IcebergExternalCatalog.ICEBERG_HMS.equals(icebergCatalogType)) {
+            throw new DdlException(
+                "Not supported: create database with properties for iceberg catalog type: " + icebergCatalogType);
+        }
         nsCatalog.createNamespace(Namespace.of(dbName), properties);
         dorisCatalog.onRefreshCache(true);
     }

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergRestExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergRestExternalCatalog.java
@@ -47,7 +47,7 @@ protected void initCatalog() {
 
         Configuration conf = replaceS3Properties(getConfiguration());
 
-        catalog = CatalogUtil.buildIcebergCatalog(icebergCatalogType,
+        catalog = CatalogUtil.buildIcebergCatalog(getName(),
                 convertToRestCatalogProperties(),
                 conf);
     }

diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_show_create.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_show_create.groovy
@@ -22,21 +22,23 @@ suite("test_iceberg_show_create", "p0,external,doris,external_docker,external_do
         return
     }
 
-    String rest_port = context.config.otherConfigs.get("iceberg_rest_uri_port")
-    String minio_port = context.config.otherConfigs.get("iceberg_minio_port")
-    String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
     String catalog_name = "test_iceberg_show_create"
+    String hivePrefix = "hive2";
+    String hms_port = context.config.otherConfigs.get(hivePrefix + "HmsPort")
+    String hdfs_port = context.config.otherConfigs.get(hivePrefix + "HdfsPort")
+    String iceberg_catalog_name = "test_iceberg_write_partitions_iceberg_${hivePrefix}"
+    String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+    String default_fs = "hdfs://${externalEnvIp}:${hdfs_port}"
+    String warehouse = "${default_fs}/warehouse"
 
     sql """drop catalog if exists ${catalog_name}"""
-    sql """
-    CREATE CATALOG ${catalog_name} PROPERTIES (
+    sql """create catalog if not exists ${catalog_name} properties (
         'type'='iceberg',
-        'iceberg.catalog.type'='rest',
-        'uri' = 'http://${externalEnvIp}:${rest_port}',
-        "s3.access_key" = "admin",
-        "s3.secret_key" = "password",
-        "s3.endpoint" = "http://${externalEnvIp}:${minio_port}",
-        "s3.region" = "us-east-1"
+        'iceberg.catalog.type'='hms',
+        'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}',
+        'fs.defaultFS' = '${default_fs}',
+        'warehouse' = '${warehouse}',
+        'use_meta_cache' = 'true'
     );"""
 
     sql """ switch ${catalog_name} """
@@ -49,25 +51,49 @@ suite("test_iceberg_show_create", "p0,external,doris,external_docker,external_do
     sql """ drop database if exists ${db1} """
     sql """ drop database if exists ${db2} """
 
-    sql """ create database ${db1} properties ('location'='s3a://warehouse/wh/${db1}') """
+    sql """ create database ${db1} properties ('location'='${warehouse}/other_location') """
     sql """ create database ${db2} """
 
     String result = ""
     result = sql "show create database ${db1}"
     logger.info("${result}")
-    assertTrue(result.toString().containsIgnoreCase("s3a://warehouse/wh/${db1}"))
+    assertTrue(result.toString().containsIgnoreCase("${warehouse}/other_location"))
 
     result = sql "show create database ${db2}"
     logger.info("${result}")
-    assertTrue(result.toString().containsIgnoreCase("s3a://warehouse/wh/${db2}"))
+    assertTrue(result.toString().containsIgnoreCase("${warehouse}/${db2}"))
 
     sql """ create table ${db1}.${tb1} (id int) """
     result = sql "show create table ${db1}.${tb1}"
     logger.info("${result}")
-    assertTrue(result.toString().containsIgnoreCase("s3a://warehouse/wh/${db1}/${tb1}"))
+    assertTrue(result.toString().containsIgnoreCase("${warehouse}/other_location/${tb1}"))
 
     sql """ drop table ${db1}.${tb1} """
     sql """ drop database ${db1} """
     sql """ drop database ${db2} """
 
+    String rest_port = context.config.otherConfigs.get("iceberg_rest_uri_port")
+    String minio_port = context.config.otherConfigs.get("iceberg_minio_port")
+    sql """drop catalog if exists ${catalog_name}"""
+    sql """
+    CREATE CATALOG ${catalog_name} PROPERTIES (
+        'type'='iceberg',
+        'iceberg.catalog.type'='rest',
+        'uri' = 'http://${externalEnvIp}:${rest_port}',
+        "s3.access_key" = "admin",
+        "s3.secret_key" = "password",
+        "s3.endpoint" = "http://${externalEnvIp}:${minio_port}",
+        "s3.region" = "us-east-1"
+    );"""
+
+    sql """ switch ${catalog_name} """
+    sql """ drop database if exists ${db1} """
+
+    test {
+        sql """ create database ${db1} properties ('location'='${warehouse}/other_location') """
+        exception "Not supported: create database with properties for iceberg catalog type"
+    }
+
+    sql """ drop database if exists ${db1} """
+    sql """drop catalog if exists ${catalog_name}"""
 }