enhance(controller): support virtual host path configuration

client/starwhale/core/dataset/store.py

@@ -2,6 +2,7 @@
 
 import io
 import os
+import json
 import shutil
 import typing as t
 from abc import ABCMeta, abstractmethod
@@ -189,6 +190,10 @@ def __init__(
         self.total_max_attempts = int(
             os.environ.get("SW_S3_TOTAL_MAX_ATTEMPTS", total_max_attempts)
         )
+        # configs like {"addressing_style": "auto"}
+        # more info in: botocore.Config._validate_s3_configuration
+        # https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html
+        self.extra_s3_configs = json.loads(os.environ.get("SW_S3_EXTRA_CONFIGS", "{}"))
 
     def __str__(self) -> str:
         return f"endpoint[{self.endpoint}]-region[{self.region}]"
@@ -360,6 +365,7 @@ def __init__(
             aws_access_key_id=conn.access_key,
             aws_secret_access_key=conn.secret_key,
             config=S3Config(
+                s3=conn.extra_s3_configs,
                 connect_timeout=conn.connect_timeout,
                 read_timeout=conn.read_timeout,
                 signature_version="s3v4",

server/storage-access-layer/src/main/java/ai/starwhale/mlops/storage/configuration/StorageProperties.java

@@ -16,6 +16,8 @@
 
 package ai.starwhale.mlops.storage.configuration;
 
+import ai.starwhale.mlops.storage.fs.AliyunEnv;
+import ai.starwhale.mlops.storage.fs.BotoS3Config;
 import ai.starwhale.mlops.storage.fs.FileStorageEnv;
 import ai.starwhale.mlops.storage.fs.S3Env;
 import ai.starwhale.mlops.storage.s3.S3Config;
@@ -34,16 +36,35 @@ public class StorageProperties {
 
     public Map<String, FileStorageEnv> toFileStorageEnvs() {
         Map<String, FileStorageEnv> ret = new HashMap<>();
-        if (null != s3Config) {
-            S3Env s3Env = new S3Env();
-            s3Env.add(S3Env.ENV_BUCKET, s3Config.getBucket());
-            s3Env.add(S3Env.ENV_ENDPOINT, s3Config.getEndpoint());
-            s3Env.add(S3Env.ENV_SECRET_ID, s3Config.getAccessKey());
-            s3Env.add(S3Env.ENV_SECRET_KEY, s3Config.getSecretKey());
-            s3Env.add(S3Env.ENV_REGION, s3Config.getRegion());
-            s3Env.add(S3Env.ENV_KEY_PREFIX, pathPrefix);
+        if (s3Config == null) {
+            return ret;
+        }
+
+        String t = type;
+        if (t.isEmpty()) {
+            t = FileStorageEnv.FileSystemEnvType.S3.name();
+        }
+        if (t.equalsIgnoreCase(FileStorageEnv.FileSystemEnvType.S3.name())) {
+            var s3Env = new S3Env();
+            updateS3Config(s3Env);
             ret.put(s3Env.getEnvType().name(), s3Env);
+        } else if (t.equalsIgnoreCase(FileStorageEnv.FileSystemEnvType.ALIYUN.name())) {
+            var aliyunEnv = new AliyunEnv();
+            updateS3Config(aliyunEnv);
+            // force using virtual host path
+            aliyunEnv.setExtraS3Configs(new BotoS3Config(BotoS3Config.AddressingStyleType.VIRTUAL).toEnvStr());
+            ret.put(aliyunEnv.getEnvType().name(), aliyunEnv);
         }
+
         return ret;
     }
+
+    private void updateS3Config(S3Env s3Env) {
+        s3Env.setEndPoint(s3Config.getEndpoint())
+                .setBucket(s3Config.getBucket())
+                .setAccessKey(s3Config.getAccessKey())
+                .setSecret(s3Config.getSecretKey())
+                .setRegion(s3Config.getRegion())
+                .setKeyPrefix(pathPrefix);
+    }
 }

server/storage-access-layer/src/main/java/ai/starwhale/mlops/storage/fs/AliyunEnv.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright 2022 Starwhale, Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ai.starwhale.mlops.storage.fs;
+
+public class AliyunEnv extends S3Env {
+    public static final String ENV_EXTRA_S3_CONFIGS = "SW_S3_EXTRA_CONFIGS";
+
+    public AliyunEnv() {
+        super(FileSystemEnvType.ALIYUN);
+    }
+
+    public void setExtraS3Configs(String extraS3Configs) {
+        this.add(ENV_EXTRA_S3_CONFIGS, extraS3Configs);
+    }
+}

server/storage-access-layer/src/main/java/ai/starwhale/mlops/storage/fs/BotoS3Config.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright 2022 Starwhale, Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ai.starwhale.mlops.storage.fs;
+
+/**
+ * BotoS3Config is for boto3 configuration communicate with starwhale python sdk
+ * see <a href="https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html">doc of config</a>
+ */
+public class BotoS3Config {
+    public final AddressingStyleType addressingStyle;
+
+    public enum AddressingStyleType {
+        VIRTUAL, AUTO, PATH
+    }
+
+    public BotoS3Config(AddressingStyleType addressingStyle) {
+        this.addressingStyle = addressingStyle;
+    }
+
+    public String toEnvStr() {
+        return "{\"addressing_style\": \"" + this.addressingStyle.name().toLowerCase() + "\"}";
+    }
+}

server/storage-access-layer/src/main/java/ai/starwhale/mlops/storage/fs/FileStorageEnv.java

@@ -49,4 +49,7 @@ public FileStorageEnv(FileSystemEnvType envType) {
         envs.put(ENV_TYPE, envType.name());
     }
 
+    public void setKeyPrefix(String keyPrefix) {
+        this.add(ENV_KEY_PREFIX, keyPrefix);
+    }
 }

server/storage-access-layer/src/main/java/ai/starwhale/mlops/storage/fs/S3Env.java

@@ -34,4 +34,33 @@ public class S3Env extends FileStorageEnv {
     public S3Env() {
         super(FileSystemEnvType.S3);
     }
+
+    public S3Env(FileSystemEnvType t) {
+        super(t);
+    }
+
+    public S3Env setEndPoint(String endPoint) {
+        this.add(ENV_ENDPOINT, endPoint);
+        return this;
+    }
+
+    public S3Env setBucket(String bucket) {
+        this.add(ENV_BUCKET, bucket);
+        return this;
+    }
+
+    public S3Env setAccessKey(String accessKey) {
+        this.add(ENV_SECRET_ID, accessKey);
+        return this;
+    }
+
+    public S3Env setSecret(String secret) {
+        this.add(ENV_SECRET_KEY, secret);
+        return this;
+    }
+
+    public S3Env setRegion(String region) {
+        this.add(ENV_REGION, region);
+        return this;
+    }
 }

server/storage-access-layer/src/test/java/ai/starwhale/mlops/storage/fs/AliyunEnvTest.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2022 Starwhale, Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ai.starwhale.mlops.storage.fs;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.is;
+
+import org.junit.jupiter.api.Test;
+
+public class AliyunEnvTest extends S3EnvTest {
+    @Test
+    public void testSet() {
+        var aliyunEnv = new AliyunEnv();
+        assertThat(aliyunEnv.getEnvType(), is(FileStorageEnv.FileSystemEnvType.ALIYUN));
+        var conf = randomString();
+        aliyunEnv.setExtraS3Configs(conf);
+        assertThat(mapContains(aliyunEnv.getEnvs(), AliyunEnv.ENV_EXTRA_S3_CONFIGS, conf), is(true));
+    }
+}

server/storage-access-layer/src/test/java/ai/starwhale/mlops/storage/fs/BotoS3ConfigTest.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2022 Starwhale, Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ai.starwhale.mlops.storage.fs;
+
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.is;
+
+import java.util.Map;
+import org.junit.jupiter.api.Test;
+
+public class BotoS3ConfigTest {
+    @Test
+    public void testEnvStr() {
+        var tests = Map.of(
+                BotoS3Config.AddressingStyleType.AUTO, "{\"addressing_style\": \"auto\"}",
+                BotoS3Config.AddressingStyleType.VIRTUAL, "{\"addressing_style\": \"virtual\"}",
+                BotoS3Config.AddressingStyleType.PATH, "{\"addressing_style\": \"path\"}"
+        );
+
+        tests.forEach((BotoS3Config.AddressingStyleType t, String expect) -> {
+            var botoS3Config = new BotoS3Config(t);
+            assertThat(botoS3Config.toEnvStr(), is(expect));
+        });
+    }
+}

server/storage-access-layer/src/test/java/ai/starwhale/mlops/storage/fs/S3EnvTest.java

@@ -0,0 +1,59 @@
+/*
+ * Copyright 2022 Starwhale, Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ai.starwhale.mlops.storage.fs;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.is;
+
+import java.util.Map;
+import java.util.UUID;
+import org.junit.jupiter.api.Test;
+
+public class S3EnvTest {
+    @Test
+    public void testSet() {
+        var s3Env = new S3Env();
+        assertThat(s3Env.getEnvType(), is(FileStorageEnv.FileSystemEnvType.S3));
+        var envValue = randomString();
+        s3Env.setEndPoint(envValue);
+        assertThat(mapContains(s3Env.getEnvs(), S3Env.ENV_ENDPOINT, envValue), is(true));
+
+        envValue = randomString();
+        s3Env.setBucket(envValue);
+        assertThat(mapContains(s3Env.getEnvs(), S3Env.ENV_BUCKET, envValue), is(true));
+
+        envValue = randomString();
+        s3Env.setAccessKey(envValue);
+        assertThat(mapContains(s3Env.getEnvs(), S3Env.ENV_SECRET_ID, envValue), is(true));
+
+        envValue = randomString();
+        s3Env.setSecret(envValue);
+        assertThat(mapContains(s3Env.getEnvs(), S3Env.ENV_SECRET_KEY, envValue), is(true));
+
+        envValue = randomString();
+        s3Env.setRegion(envValue);
+        assertThat(mapContains(s3Env.getEnvs(), S3Env.ENV_REGION, envValue), is(true));
+    }
+
+    public String randomString() {
+        return UUID.randomUUID().toString();
+    }
+
+    public boolean mapContains(Map<String, String> map, String key, String val) {
+        return map.containsKey(key) && map.get(key).equals(val);
+    }
+}