enhance(controller): make init container compatible with both minio a…

…nd aliyun oss (star-whale#890) make init container compatible with both minio and aliyun oss
anda-ren · Aug 10, 2022 · 0d2c783 · 0d2c783
1 parent 60c2955
commit 0d2c783
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 21 deletions.
diff --git a/server/controller/src/main/java/ai/starwhale/mlops/schedule/k8s/K8sTaskScheduler.java b/server/controller/src/main/java/ai/starwhale/mlops/schedule/k8s/K8sTaskScheduler.java
@@ -89,15 +89,16 @@ private void deployTaskToK8s(K8sClient client, String image, TaskTrigger task) {
         log.debug("deploying task to k8s {} {} {}", task.getId(), task.getResultPath(), task.getTaskType());
         Map<String, String> envs = new HashMap<>();
         List<String> downloads = new ArrayList<>();
-        String prefix = "minio/starwhale/";
+        String prefix = "s3://"+storageProperties.getS3Config().getBucket()+"/";
         downloads.add(prefix + task.getSwModelPackage().getPath()+";/opt/starwhale/swmp/");
         downloads.add(prefix + task.getSwrt().getPath()+";/opt/starwhale/swrt/");
         envs.put("DOWNLOADS", Strings.join(downloads, ' '));
         String input = generateConfigFile(task);
         envs.put("INPUT", input);
-        envs.put("MINIO_SERVICE",storageProperties.getS3Config().getEndpoint());
-        envs.put("MINIO_ACCESS_KEY",storageProperties.getS3Config().getAccessKey());
-        envs.put("MINIO_SECRET_KEY",storageProperties.getS3Config().getSecretKey());
+        envs.put("ENDPOINT_URL",storageProperties.getS3Config().getEndpoint());
+        envs.put("AWS_ACCESS_KEY_ID",storageProperties.getS3Config().getAccessKey());
+        envs.put("AWS_SECRET_ACCESS_KEY",storageProperties.getS3Config().getSecretKey());
+        envs.put("AWS_S3_REGION",storageProperties.getS3Config().getRegion());
         try {
             String cmd = "ppl";
             if (task.getTaskType() == TaskType.CMP) {
@@ -108,9 +109,10 @@ private void deployTaskToK8s(K8sClient client, String image, TaskTrigger task) {
             V1Job job = client.renderJob(getJobTemplate(), task.getId().toString(), "worker", image, List.of(cmd), envs,resourceRequirements);
             // set result upload path
             job.getSpec().getTemplate().getSpec().getContainers().get(0).env(List.of(new V1EnvVar().name("DST").value(prefix+ task.getResultPath().resultDir())
-                ,new V1EnvVar().name("MINIO_SERVICE").value(storageProperties.getS3Config().getEndpoint())
-                ,new V1EnvVar().name("MINIO_ACCESS_KEY").value(storageProperties.getS3Config().getAccessKey())
-                ,new V1EnvVar().name("MINIO_SECRET_KEY").value(storageProperties.getS3Config().getSecretKey())
+                ,new V1EnvVar().name("ENDPOINT_URL").value(storageProperties.getS3Config().getEndpoint())
+                ,new V1EnvVar().name("AWS_ACCESS_KEY_ID").value(storageProperties.getS3Config().getAccessKey())
+                ,new V1EnvVar().name("AWS_S3_REGION").value(storageProperties.getS3Config().getRegion())
+                ,new V1EnvVar().name("AWS_SECRET_ACCESS_KEY").value(storageProperties.getS3Config().getSecretKey())
                 )
             );
             client.deploy(job);
@@ -135,7 +137,7 @@ private String generateConfigFile(TaskTrigger task) {
 
                 task.getSwdsBlocks().forEach(swdsBlock -> {
                     JSONObject ds = JSONUtil.createObj();
-                    ds.set("bucket", "starwhale");
+                    ds.set("bucket", storageProperties.getS3Config().getBucket());
                     ds.set("key", JSONUtil.createObj()
                         .set("data", String.format(dataFormat, swdsBlock.getLocationInput().getFile(), swdsBlock.getLocationInput().getOffset(), swdsBlock.getLocationInput().getOffset() + swdsBlock.getLocationInput().getSize() - 1))
                         .set("label", String.format(dataFormat, swdsBlock.getLocationLabel().getFile(), swdsBlock.getLocationLabel().getOffset(), swdsBlock.getLocationLabel().getOffset() + swdsBlock.getLocationLabel().getSize() - 1))
@@ -153,7 +155,7 @@ private String generateConfigFile(TaskTrigger task) {
                 JSONArray cmp = JSONUtil.createArray();
                 task.getCmpInputFilePaths().forEach(inputFilePath -> {
                     JSONObject ds = JSONUtil.createObj();
-                    ds.set("bucket", "starwhale");
+                    ds.set("bucket", storageProperties.getS3Config().getBucket());
                     ds.set("key", JSONUtil.createObj()
                         .set("data", inputFilePath)
                     );

diff --git a/server/controller/src/main/resources/template/job.yaml b/server/controller/src/main/resources/template/job.yaml
@@ -13,18 +13,12 @@ spec:
       initContainers:
         - name: data-provider
           imagePullPolicy: IfNotPresent
-          image: minio/mc:latest
+          image: anigeo/awscli:latest
           volumeMounts:
             - mountPath: /opt/starwhale
               name: data
-          command:
-            #https://github.com/minio/mc/issues/3599
-            - sh
-            - -c
-            - >-
-              sleep 10;
-              mc alias set minio $MINIO_SERVICE $MINIO_ACCESS_KEY $MINIO_SECRET_KEY &&
-              for item in $DOWNLOADS; do SRC=$(cut -d';' -f1 <<<$item) && DST=$(cut -d';' -f2 <<<$item) && echo $SRC,$DST && mkdir -p $DST && mc cp -r $SRC $DST; done
+          command: ["/bin/sh"]
+          args: ["-c", "for item in $DOWNLOADS; do SRC=$(echo $item|cut -d';' -f1) && DST=$(echo $item|cut -d';' -f2) && echo $SRC,$DST && mkdir -p $DST && aws --endpoint-url=$ENDPOINT_URL --region=$AWS_S3_REGION s3 cp --recursive $SRC $DST; done"]
         - name: untar
           imagePullPolicy: IfNotPresent
           image: ghcr.io/star-whale/starwhale:latest
@@ -62,16 +56,15 @@ spec:
       containers:
         - name: result-uploader
           imagePullPolicy: IfNotPresent
-          image: minio/mc:latest
+          image: anigeo/awscli:latest
           volumeMounts:
             - mountPath: /opt/starwhale
               name: data
           command:
             - sh
             - -c
             - >-
-              mc alias set minio $MINIO_SERVICE $MINIO_ACCESS_KEY $MINIO_SECRET_KEY &&
-              mc cp -r /opt/starwhale/result/current $DST
+              aws --endpoint-url=$ENDPOINT_URL --region=$AWS_S3_REGION s3 cp /opt/starwhale/result/current $DST
       volumes:
         - name: pip-cache
           hostPath: