[feat](cloud) S3 storage vault support property use_path_style

be/src/io/fs/s3_file_system.cpp

@@ -86,7 +86,8 @@ Status ObjClientHolder::reset(const S3ClientConf& conf) {
     S3ClientConf reset_conf;
     {
         std::shared_lock lock(_mtx);
-        if (conf.ak == _conf.ak && conf.sk == _conf.sk && conf.token == _conf.token) {
+        if (conf.ak == _conf.ak && conf.sk == _conf.sk && conf.token == _conf.token &&
+            conf.use_virtual_addressing == _conf.use_virtual_addressing) {
             return Status::OK(); // Same conf
         }
 
@@ -95,6 +96,7 @@ Status ObjClientHolder::reset(const S3ClientConf& conf) {
         reset_conf.sk = conf.sk;
         reset_conf.token = conf.token;
         reset_conf.bucket = conf.bucket;
+        reset_conf.use_virtual_addressing = conf.use_virtual_addressing;
         // Should check endpoint here?
     }
 

be/src/util/s3_util.cpp

@@ -401,15 +401,15 @@ S3Conf S3Conf::get_s3_conf(const cloud::ObjectStoreInfoPB& info) {
     S3Conf ret {
             .bucket = info.bucket(),
             .prefix = info.prefix(),
-            .client_conf {
-                    .endpoint = info.endpoint(),
-                    .region = info.region(),
-                    .ak = info.ak(),
-                    .sk = info.sk(),
-                    .token {},
-                    .bucket = info.bucket(),
-                    .provider = io::ObjStorageType::AWS,
-            },
+            .client_conf {.endpoint = info.endpoint(),
+                          .region = info.region(),
+                          .ak = info.ak(),
+                          .sk = info.sk(),
+                          .token {},
+                          .bucket = info.bucket(),
+                          .provider = io::ObjStorageType::AWS,
+                          .use_virtual_addressing =
+                                  info.has_use_path_style() ? !info.use_path_style() : true},
             .sse_enabled = info.sse_enabled(),
     };
 

cloud/src/meta-service/meta_service_resource.cpp

@@ -701,6 +701,9 @@ static int alter_s3_storage_vault(InstanceInfoPB& instance, std::unique_ptr<Tran
     new_vault.mutable_obj_info()->set_ak(cipher_ak_sk_pair.first);
     new_vault.mutable_obj_info()->set_sk(cipher_ak_sk_pair.second);
     new_vault.mutable_obj_info()->mutable_encryption_info()->CopyFrom(encryption_info);
+    if (obj_info.has_use_path_style()) {
+        new_vault.mutable_obj_info()->set_use_path_style(obj_info.use_path_style());
+    }
 
     auto new_vault_info = new_vault.DebugString();
     val = new_vault.SerializeAsString();

fe/fe-core/src/main/java/org/apache/doris/catalog/S3StorageVault.java

@@ -19,6 +19,7 @@
 
 import org.apache.doris.analysis.CreateResourceStmt;
 import org.apache.doris.common.DdlException;
+import org.apache.doris.datasource.property.PropertyConverter;
 import org.apache.doris.datasource.property.constants.S3Properties;
 
 import com.google.gson.annotations.SerializedName;
@@ -62,7 +63,8 @@ public class S3StorageVault extends StorageVault {
             TYPE,
             S3Properties.ACCESS_KEY,
             S3Properties.SECRET_KEY,
-            VAULT_NAME
+            VAULT_NAME,
+            PropertyConverter.USE_PATH_STYLE
     ));
 
     @SerializedName(value = "properties")

fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java

@@ -32,6 +32,7 @@
 import com.amazonaws.auth.SystemPropertiesCredentialsProvider;
 import com.amazonaws.auth.WebIdentityTokenCredentialsProvider;
 import com.amazonaws.auth.profile.ProfileCredentialsProvider;
+import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
 import com.google.common.collect.Maps;
 import org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider;
@@ -334,6 +335,15 @@ public static Cloud.ObjectStoreInfoPB.Builder getObjStoreInfoPB(Map<String, Stri
             // S3 Provider properties should be case insensitive.
             builder.setProvider(Provider.valueOf(properties.get(S3Properties.PROVIDER).toUpperCase()));
         }
+
+        if (properties.containsKey(PropertyConverter.USE_PATH_STYLE)) {
+            String value = properties.get(PropertyConverter.USE_PATH_STYLE);
+            Preconditions.checkArgument(!Strings.isNullOrEmpty(value), "use_path_style cannot be empty");
+            Preconditions.checkArgument(value.equalsIgnoreCase("true")
+                    || value.equalsIgnoreCase("false"),
+                    "Invalid use_path_style value: %s only 'true' or 'false' is acceptable", value);
+            builder.setUsePathStyle(value.equalsIgnoreCase("true"));
+        }
         return builder;
     }
 }

gensrc/proto/cloud.proto

@@ -197,6 +197,7 @@ message ObjectStoreInfoPB {
     optional string user_id = 13;
     optional EncryptionInfoPB encryption_info = 14;
     optional bool sse_enabled = 15;
+    optional bool use_path_style = 16;
 }
 
 // The legacy ObjectStoreInfoPB is stored in InstanceInfoPB

regression-test/suites/vault_p0/alter/test_alter_use_path_style.groovy

@@ -0,0 +1,108 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_alter_use_path_style", "nonConcurrent") {
+    def suiteName = name;
+    if (!isCloudMode()) {
+        logger.info("skip ${suiteName} case, because not cloud mode")
+        return
+    }
+
+    if (!enableStoragevault()) {
+        logger.info("skip ${suiteName} case, because storage vault not enabled")
+        return
+    }
+
+    sql """
+        CREATE STORAGE VAULT IF NOT EXISTS ${suiteName}
+        PROPERTIES (
+            "type"="S3",
+            "s3.endpoint"="${getS3Endpoint()}",
+            "s3.region" = "${getS3Region()}",
+            "s3.access_key" = "${getS3AK()}",
+            "s3.secret_key" = "${getS3SK()}",
+            "s3.root.path" = "${suiteName}",
+            "s3.bucket" = "${getS3BucketName()}",
+            "s3.external_endpoint" = "",
+            "provider" = "${getS3Provider()}",
+            "use_path_style" = "false"
+        );
+    """
+
+    sql """
+        ALTER STORAGE VAULT ${suiteName}
+        PROPERTIES (
+            "type"="S3",
+            "use_path_style" = "true"
+        );
+    """
+
+    def vaultInfos = sql """ SHOW STORAGE VAULT; """
+    boolean exist = false
+
+    for (int i = 0; i < vaultInfos.size(); i++) {
+        def name = vaultInfos[i][0]
+        logger.info("name is ${name}, info ${vaultInfos[i]}")
+        if (name.equals(suiteName)) {
+            assertTrue(vaultInfos[i][2].contains("""use_path_style: true"""))
+            exist = true
+        }
+    }
+    assertTrue(exist)
+
+
+    sql """
+        ALTER STORAGE VAULT ${suiteName}
+        PROPERTIES (
+            "type"="S3",
+            "use_path_style" = "false"
+        );
+    """
+
+    vaultInfos = sql """ SHOW STORAGE VAULT; """
+    exist = false
+
+    for (int i = 0; i < vaultInfos.size(); i++) {
+        def name = vaultInfos[i][0]
+        logger.info("name is ${name}, info ${vaultInfos[i]}")
+        if (name.equals(suiteName)) {
+            assertTrue(vaultInfos[i][2].contains("""use_path_style: false"""))
+            exist = true
+        }
+    }
+    assertTrue(exist)
+
+    expectExceptionLike({
+        sql """
+            ALTER STORAGE VAULT ${suiteName}
+            PROPERTIES (
+            "type"="S3",
+            "use_path_style" = ""
+            );
+        """
+    }, "use_path_style cannot be empty")
+
+    expectExceptionLike({
+        sql """
+            ALTER STORAGE VAULT ${suiteName}
+            PROPERTIES (
+            "type"="S3",
+            "use_path_style" = "abc"
+            );
+        """
+    }, "Invalid use_path_style value")
+}

regression-test/suites/vault_p0/default/test_default_vault.groovy

@@ -92,9 +92,11 @@ suite("test_default_vault", "nonConcurrent") {
         """
 
         sql """ insert into ${tableName} values(1, 1); """
-        result """ select * from ${tableName}; """
-        assertEqual(result.size(), 1)
-        assertEqual(result[0][0], 1)
+        sql """ sync;"""
+        def result = sql """ select * from ${tableName}; """
+        logger.info("result:${result}");
+        assertTrue(result.size() == 1)
+        assertTrue(result[0][0].toInteger() == 1)
 
         def create_table_stmt = sql """ show create table ${tableName} """
         assertTrue(create_table_stmt[0][1].contains("built_in_storage_vault"))