Skip to content

Commit

Permalink
1
Browse files Browse the repository at this point in the history
  • Loading branch information
morningman committed Jan 15, 2025
1 parent 9f850e5 commit 1c1f31b
Showing 6 changed files with 98 additions and 0 deletions.
8 changes: 8 additions & 0 deletions fe/fe-core/pom.xml
Original file line number Diff line number Diff line change
@@ -934,6 +934,14 @@ under the License.
<artifactId>azure-storage-blob-batch</artifactId>
<version>${azure.sdk.batch.version}</version>
</dependency>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>s3tables</artifactId>
</dependency>
<dependency>
<groupId>software.amazon.s3tables</groupId>
<artifactId>s3-tables-catalog-for-iceberg</artifactId>
</dependency>
</dependencies>
<repositories>
<!-- for huawei obs sdk -->
Original file line number Diff line number Diff line change
@@ -40,6 +40,7 @@ public abstract class IcebergExternalCatalog extends ExternalCatalog {
public static final String ICEBERG_HADOOP = "hadoop";
public static final String ICEBERG_GLUE = "glue";
public static final String ICEBERG_DLF = "dlf";
public static final String ICEBERG_S3_TABLES = "s3tables";
public static final String EXTERNAL_CATALOG_NAME = "external_catalog.name";
protected String icebergCatalogType;
protected Catalog catalog;
Original file line number Diff line number Diff line change
@@ -41,6 +41,8 @@ public static ExternalCatalog createCatalog(long catalogId, String name, String
return new IcebergDLFExternalCatalog(catalogId, name, resource, props, comment);
case IcebergExternalCatalog.ICEBERG_HADOOP:
return new IcebergHadoopExternalCatalog(catalogId, name, resource, props, comment);
case IcebergExternalCatalog.ICEBERG_S3_TABLES:
return new IcebergS3TablesExternalCatalog(catalogId, name, resource, props, comment);
default:
throw new DdlException("Unknown " + IcebergExternalCatalog.ICEBERG_CATALOG_TYPE
+ " value: " + catalogType);
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

package org.apache.doris.datasource.iceberg;

import org.apache.doris.datasource.CatalogProperty;
import org.apache.doris.datasource.property.PropertyConverter;
import org.apache.doris.datasource.property.constants.S3Properties;

import org.apache.iceberg.CatalogProperties;
import org.apache.iceberg.aws.AwsClientProperties;
import org.apache.iceberg.aws.s3.S3FileIOProperties;
import software.amazon.s3tables.iceberg.S3TablesCatalog;

import java.util.HashMap;
import java.util.Map;

public class IcebergS3TablesExternalCatalog extends IcebergExternalCatalog {

public IcebergS3TablesExternalCatalog(long catalogId, String name, String resource, Map<String, String> props,
String comment) {
super(catalogId, name, comment);
props = PropertyConverter.convertToMetaProperties(props);
catalogProperty = new CatalogProperty(resource, props);
}

@Override
protected void initCatalog() {
icebergCatalogType = ICEBERG_S3_TABLES;
S3TablesCatalog s3TablesCatalog = new S3TablesCatalog();
Map<String, String> s3TablesCatalogProperties = convertToS3TablesCatalogProperties();
String warehouse = catalogProperty.getHadoopProperties().get(CatalogProperties.WAREHOUSE_LOCATION);
s3TablesCatalogProperties.put(CatalogProperties.WAREHOUSE_LOCATION, warehouse);
s3TablesCatalog.initialize(getName(), s3TablesCatalogProperties);
catalog = s3TablesCatalog;
}

private Map<String, String> convertToS3TablesCatalogProperties() {
Map<String, String> props = catalogProperty.getProperties();
Map<String, String> s3Properties = new HashMap<>(props);
if (props.containsKey(S3Properties.ENDPOINT)) {
s3Properties.put(S3FileIOProperties.ENDPOINT, props.get(S3Properties.ENDPOINT));
}
if (props.containsKey(S3Properties.ACCESS_KEY)) {
s3Properties.put(S3FileIOProperties.ACCESS_KEY_ID, props.get(S3Properties.ACCESS_KEY));
}
if (props.containsKey(S3Properties.SECRET_KEY)) {
s3Properties.put(S3FileIOProperties.SECRET_ACCESS_KEY, props.get(S3Properties.SECRET_KEY));
}
if (props.containsKey(S3Properties.REGION)) {
s3Properties.put(AwsClientProperties.CLIENT_REGION, props.get(S3Properties.REGION));
}
if (props.containsKey(PropertyConverter.USE_PATH_STYLE)) {
s3Properties.put(S3FileIOProperties.PATH_STYLE_ACCESS, props.get(PropertyConverter.USE_PATH_STYLE));
}
return s3Properties;
}
}
Original file line number Diff line number Diff line change
@@ -146,6 +146,7 @@
import org.apache.doris.datasource.iceberg.IcebergHMSExternalCatalog;
import org.apache.doris.datasource.iceberg.IcebergHadoopExternalCatalog;
import org.apache.doris.datasource.iceberg.IcebergRestExternalCatalog;
import org.apache.doris.datasource.iceberg.IcebergS3TablesExternalCatalog;
import org.apache.doris.datasource.infoschema.ExternalInfoSchemaDatabase;
import org.apache.doris.datasource.infoschema.ExternalInfoSchemaTable;
import org.apache.doris.datasource.infoschema.ExternalMysqlDatabase;
@@ -417,6 +418,8 @@ public class GsonUtils {
.registerSubtype(IcebergRestExternalCatalog.class, IcebergRestExternalCatalog.class.getSimpleName())
.registerSubtype(IcebergDLFExternalCatalog.class, IcebergDLFExternalCatalog.class.getSimpleName())
.registerSubtype(IcebergHadoopExternalCatalog.class, IcebergHadoopExternalCatalog.class.getSimpleName())
.registerSubtype(IcebergS3TablesExternalCatalog.class,
IcebergS3TablesExternalCatalog.class.getSimpleName())
.registerSubtype(PaimonExternalCatalog.class, PaimonExternalCatalog.class.getSimpleName())
.registerSubtype(PaimonHMSExternalCatalog.class, PaimonHMSExternalCatalog.class.getSimpleName())
.registerSubtype(PaimonFileExternalCatalog.class, PaimonFileExternalCatalog.class.getSimpleName())
12 changes: 12 additions & 0 deletions fe/pom.xml
Original file line number Diff line number Diff line change
@@ -379,6 +379,8 @@ under the License.
<azure.sdk.batch.version>12.22.0</azure.sdk.batch.version>
<semver4j.version>5.3.0</semver4j.version>
<aliyun-sdk-oss.version>3.15.0</aliyun-sdk-oss.version>
<s3tables.version>2.29.26</s3tables.version>
<s3tables.catalog.version>0.1.3</s3tables.catalog.version>
</properties>
<profiles>
<profile>
@@ -1675,6 +1677,16 @@ under the License.
<artifactId>semver4j</artifactId>
<version>${semver4j.version}</version>
</dependency>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>s3tables</artifactId>
<version>${s3tables.version}</version>
</dependency>
<dependency>
<groupId>software.amazon.s3tables</groupId>
<artifactId>s3-tables-catalog-for-iceberg</artifactId>
<version>${s3tables.catalog.version}</version>
</dependency>
</dependencies>
</dependencyManagement>
<dependencies>

0 comments on commit 1c1f31b

Please sign in to comment.