From ab108d4c2ca1b31df7dda924f1300ffce0dfeb39 Mon Sep 17 00:00:00 2001 From: Tommy Li Date: Mon, 10 Aug 2020 11:02:20 -0700 Subject: [PATCH] fix(sdk): Update SDK to pull artifact related information from annotations (#259) * pull artifact related information from annotations * update test cases * update dict.get syntax for the merging --- .../kfp_tekton/compiler/_op_to_template.py | 26 ++--- sdk/python/kfp_tekton/compiler/compiler.py | 9 ++ .../tests/compiler/testdata/affinity.yaml | 3 + .../compiler/testdata/basic_no_decorator.yaml | 3 + .../compiler/testdata/big_data_passing.yaml | 110 +++++++++++++----- .../tests/compiler/testdata/compose.yaml | 46 +++++--- .../tests/compiler/testdata/condition.yaml | 44 +++++-- .../tests/compiler/testdata/exit_handler.yaml | 25 ++-- .../compiler/testdata/hidden_output_file.yaml | 25 ++-- .../compiler/testdata/imagepullsecrets.yaml | 25 ++-- .../compiler/testdata/init_container.yaml | 3 + .../testdata/input_artifact_raw_value.yaml | 3 + sdk/python/tests/compiler/testdata/katib.yaml | 25 ++-- .../compiler/testdata/load_from_yaml.yaml | 25 ++-- .../tests/compiler/testdata/loop_static.yaml | 3 + .../compiler/testdata/node_selector.yaml | 3 + .../compiler/testdata/parallel_join.yaml | 3 + .../parallel_join_with_argo_vars.yaml | 44 +++++-- .../parallel_join_with_artifacts.yaml | 44 +++++-- .../testdata/parallel_join_with_logging.yaml | 65 ++++++++--- .../testdata/pipeline_transformers.yaml | 3 + .../compiler/testdata/pipelineparams.yaml | 25 ++-- .../compiler/testdata/resourceop_basic.yaml | 3 + sdk/python/tests/compiler/testdata/retry.yaml | 3 + .../tests/compiler/testdata/sequential.yaml | 3 + .../tests/compiler/testdata/sidecar.yaml | 25 ++-- .../tests/compiler/testdata/timeout.yaml | 3 + .../tests/compiler/testdata/tolerations.yaml | 25 ++-- .../tests/compiler/testdata/volume.yaml | 25 ++-- .../tests/compiler/testdata/volume_op.yaml | 3 + .../compiler/testdata/volume_snapshot_op.yaml | 3 + .../compiler/testdata/withitem_nested.yaml | 3 + 32 files changed, 482 insertions(+), 176 deletions(-) diff --git a/sdk/python/kfp_tekton/compiler/_op_to_template.py b/sdk/python/kfp_tekton/compiler/_op_to_template.py index 00cdce1260..57b80e0bb5 100644 --- a/sdk/python/kfp_tekton/compiler/_op_to_template.py +++ b/sdk/python/kfp_tekton/compiler/_op_to_template.py @@ -291,16 +291,10 @@ def _process_output_artifacts(outputs_dict: Dict[Text, Any], Dict[Text, Any] """ if outputs_dict.get('artifacts'): - # TODO: Pull default values from KFP configmap when integrated with KFP. - storage_location = outputs_dict['artifacts'][0].get('s3', {}) - insecure = storage_location.get("insecure", True) - endpoint = storage_location.get("endpoint", "minio-service.$NAMESPACE:9000") - # We want to use the insecure flag to figure out whether to use http or https scheme - endpoint = re.sub(r"https?://", "", endpoint) - endpoint = 'http://' + endpoint if insecure else 'https://' + endpoint - access_key = storage_location.get("accessKeySecret", {"name": "mlpipeline-minio-artifact", "key": "accesskey"}) - secret_access_key = storage_location.get("secretKeySecret", {"name": "mlpipeline-minio-artifact", "key": "secretkey"}) - bucket = storage_location.get("bucket", "mlpipeline") + endpoint = '${ARTIFACT_ENDPOINT_SCHEME}${ARTIFACT_ENDPOINT}' + access_key = {"name": "mlpipeline-minio-artifact", "key": "accesskey"} + secret_access_key = {"name": "mlpipeline-minio-artifact", "key": "secretkey"} + bucket = '$ARTIFACT_BUCKET' copy_artifacts_step = { 'image': 'minio/mc', 'name': 'copy-artifacts', @@ -311,7 +305,11 @@ def _process_output_artifacts(outputs_dict: Dict[Text, Any], 'env': [ {'name': 'PIPELINERUN', 'valueFrom': {'fieldRef': {'fieldPath': "metadata.labels['tekton.dev/pipelineRun']"}}}, {'name': 'PIPELINETASK', 'valueFrom': {'fieldRef': {'fieldPath': "metadata.labels['tekton.dev/pipelineTask']"}}}, - {'name': 'NAMESPACE', 'valueFrom': {'fieldRef': {'fieldPath': "metadata.namespace"}}}, + {'name': 'ARTIFACT_ENDPOINT_SCHEME', 'valueFrom': + {'fieldRef': {'fieldPath': "metadata.annotations['tekton.dev/artifact_endpoint_scheme']"}}}, + {'name': 'ARTIFACT_ENDPOINT', 'valueFrom': + {'fieldRef': {'fieldPath': "metadata.annotations['tekton.dev/artifact_endpoint']"}}}, + {'name': 'ARTIFACT_BUCKET', 'valueFrom': {'fieldRef': {'fieldPath': "metadata.annotations['tekton.dev/artifact_bucket']"}}}, {'name': 'AWS_ACCESS_KEY_ID', 'valueFrom': {'secretKeyRef': {'name': access_key['name'], 'key': access_key['key']}}}, {'name': 'AWS_SECRET_ACCESS_KEY', 'valueFrom': {'secretKeyRef': {'name': secret_access_key['name'], 'key': secret_access_key['key']}}} @@ -467,8 +465,10 @@ def _op_to_template(op: BaseOp, pipelinerun_output_artifacts={}, enable_artifact output_annotation = pipelinerun_output_artifacts.get(processed_op.name, []) output_annotation.append( { - 'name': output_artifact.get('name'), - 'path': output_artifact.get('path') + 'name': output_artifact.get('name', ''), + 'path': output_artifact.get('path', ''), + 'key': "artifacts/$PIPELINERUN/%s/%s.tgz" % + (processed_op.name, output_artifact.get('name', '').replace(processed_op.name + '-', '')) } ) pipelinerun_output_artifacts[processed_op.name] = output_annotation diff --git a/sdk/python/kfp_tekton/compiler/compiler.py b/sdk/python/kfp_tekton/compiler/compiler.py index e65f0b4dbb..74f6aae39b 100644 --- a/sdk/python/kfp_tekton/compiler/compiler.py +++ b/sdk/python/kfp_tekton/compiler/compiler.py @@ -23,6 +23,7 @@ import textwrap from typing import Callable, List, Text, Dict, Any +from os import environ as env # Kubeflow Pipeline imports from kfp import dsl @@ -40,6 +41,11 @@ from kfp_tekton.compiler._op_to_template import _op_to_template +DEFAULT_ARTIFACT_BUCKET = env.get('DEFAULT_ARTIFACT_BUCKET', 'mlpipeline') +DEFAULT_ARTIFACT_ENDPOINT = env.get('DEFAULT_ARTIFACT_ENDPOINT', 'minio-service.kubeflow:9000') +DEFAULT_ARTIFACT_ENDPOINT_SCHEME = env.get('DEFAULT_ARTIFACT_ENDPOINT_SCHEME', 'http://') + + def _get_super_condition_template(): python_script = textwrap.dedent('''\ @@ -484,6 +490,9 @@ def _create_pipeline_workflow(self, args, pipeline, op_transformers=None, pipeli 'annotations': { 'tekton.dev/output_artifacts': json.dumps(self.output_artifacts, sort_keys=True), 'tekton.dev/input_artifacts': json.dumps(self.input_artifacts, sort_keys=True), + 'tekton.dev/artifact_bucket': DEFAULT_ARTIFACT_BUCKET, + 'tekton.dev/artifact_endpoint': DEFAULT_ARTIFACT_ENDPOINT, + 'tekton.dev/artifact_endpoint_scheme': DEFAULT_ARTIFACT_ENDPOINT_SCHEME, 'sidecar.istio.io/inject': 'false' # disable Istio inject since Tekton cannot run with Istio sidecar } }, diff --git a/sdk/python/tests/compiler/testdata/affinity.yaml b/sdk/python/tests/compiler/testdata/affinity.yaml index eb1aeae9bf..cb18df8d83 100644 --- a/sdk/python/tests/compiler/testdata/affinity.yaml +++ b/sdk/python/tests/compiler/testdata/affinity.yaml @@ -19,6 +19,9 @@ metadata: pipelines.kubeflow.org/pipeline_spec: '{"description": "A pipeline with affinity", "name": "affinity"}' sidecar.istio.io/inject: 'false' + tekton.dev/artifact_bucket: mlpipeline + tekton.dev/artifact_endpoint: minio-service.kubeflow:9000 + tekton.dev/artifact_endpoint_scheme: http:// tekton.dev/input_artifacts: '{}' tekton.dev/output_artifacts: '{}' name: affinity diff --git a/sdk/python/tests/compiler/testdata/basic_no_decorator.yaml b/sdk/python/tests/compiler/testdata/basic_no_decorator.yaml index ab3803e76c..b21953ecff 100644 --- a/sdk/python/tests/compiler/testdata/basic_no_decorator.yaml +++ b/sdk/python/tests/compiler/testdata/basic_no_decorator.yaml @@ -21,6 +21,9 @@ metadata: "message"}, {"default": "default_output", "name": "outputpath"}], "name": "Save Most Frequent Word"}' sidecar.istio.io/inject: 'false' + tekton.dev/artifact_bucket: mlpipeline + tekton.dev/artifact_endpoint: minio-service.kubeflow:9000 + tekton.dev/artifact_endpoint_scheme: http:// tekton.dev/input_artifacts: '{}' tekton.dev/output_artifacts: '{}' name: save-most-frequent-word diff --git a/sdk/python/tests/compiler/testdata/big_data_passing.yaml b/sdk/python/tests/compiler/testdata/big_data_passing.yaml index 3e6b6658ad..0e0163ba01 100644 --- a/sdk/python/tests/compiler/testdata/big_data_passing.yaml +++ b/sdk/python/tests/compiler/testdata/big_data_passing.yaml @@ -18,6 +18,9 @@ metadata: annotations: pipelines.kubeflow.org/pipeline_spec: '{"name": "File passing pipelines"}' sidecar.istio.io/inject: 'false' + tekton.dev/artifact_bucket: mlpipeline + tekton.dev/artifact_endpoint: minio-service.kubeflow:9000 + tekton.dev/artifact_endpoint_scheme: http:// tekton.dev/input_artifacts: '{"print-params": [{"name": "gen-params-Output", "parent_task": "gen-params"}], "print-text": [{"name": "repeat-line-output_text", "parent_task": "repeat-line"}], "print-text-2": [{"name": "split-text-lines-odd_lines", "parent_task": @@ -26,12 +29,15 @@ metadata: "parent_task": "write-numbers"}], "print-text-5": [{"name": "sum-numbers-Output", "parent_task": "sum-numbers"}], "sum-numbers": [{"name": "write-numbers-numbers", "parent_task": "write-numbers"}]}' - tekton.dev/output_artifacts: '{"gen-params": [{"name": "gen-params-Output", "path": - "/tmp/outputs/Output/data"}], "repeat-line": [{"name": "repeat-line-output_text", - "path": "/tmp/outputs/output_text/data"}], "split-text-lines": [{"name": "split-text-lines-even_lines", - "path": "/tmp/outputs/even_lines/data"}, {"name": "split-text-lines-odd_lines", - "path": "/tmp/outputs/odd_lines/data"}], "sum-numbers": [{"name": "sum-numbers-Output", - "path": "/tmp/outputs/Output/data"}], "write-numbers": [{"name": "write-numbers-numbers", + tekton.dev/output_artifacts: '{"gen-params": [{"key": "artifacts/$PIPELINERUN/gen-params/Output.tgz", + "name": "gen-params-Output", "path": "/tmp/outputs/Output/data"}], "repeat-line": + [{"key": "artifacts/$PIPELINERUN/repeat-line/output_text.tgz", "name": "repeat-line-output_text", + "path": "/tmp/outputs/output_text/data"}], "split-text-lines": [{"key": "artifacts/$PIPELINERUN/split-text-lines/even_lines.tgz", + "name": "split-text-lines-even_lines", "path": "/tmp/outputs/even_lines/data"}, + {"key": "artifacts/$PIPELINERUN/split-text-lines/odd_lines.tgz", "name": "split-text-lines-odd_lines", + "path": "/tmp/outputs/odd_lines/data"}], "sum-numbers": [{"key": "artifacts/$PIPELINERUN/sum-numbers/Output.tgz", + "name": "sum-numbers-Output", "path": "/tmp/outputs/Output/data"}], "write-numbers": + [{"key": "artifacts/$PIPELINERUN/write-numbers/numbers.tgz", "name": "write-numbers-numbers", "path": "/tmp/outputs/numbers/data"}]}' name: file-passing-pipelines spec: @@ -75,10 +81,18 @@ spec: valueFrom: fieldRef: fieldPath: metadata.labels['tekton.dev/pipelineTask'] - - name: NAMESPACE + - name: ARTIFACT_ENDPOINT_SCHEME valueFrom: fieldRef: - fieldPath: metadata.namespace + fieldPath: metadata.annotations['tekton.dev/artifact_endpoint_scheme'] + - name: ARTIFACT_ENDPOINT + valueFrom: + fieldRef: + fieldPath: metadata.annotations['tekton.dev/artifact_endpoint'] + - name: ARTIFACT_BUCKET + valueFrom: + fieldRef: + fieldPath: metadata.annotations['tekton.dev/artifact_bucket'] - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: @@ -93,12 +107,12 @@ spec: name: copy-artifacts script: '#!/usr/bin/env sh - mc config host add storage http://minio-service.$NAMESPACE:9000 $AWS_ACCESS_KEY_ID - $AWS_SECRET_ACCESS_KEY + mc config host add storage ${ARTIFACT_ENDPOINT_SCHEME}${ARTIFACT_ENDPOINT} + $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY tar -cvzf output_text.tgz $(workspaces.repeat-line.path)/repeat-line-output_text - mc cp output_text.tgz storage/mlpipeline/artifacts/$PIPELINERUN/$PIPELINETASK/output_text.tgz + mc cp output_text.tgz storage/$ARTIFACT_BUCKET/artifacts/$PIPELINERUN/$PIPELINETASK/output_text.tgz ' workspaces: @@ -207,10 +221,18 @@ spec: valueFrom: fieldRef: fieldPath: metadata.labels['tekton.dev/pipelineTask'] - - name: NAMESPACE + - name: ARTIFACT_ENDPOINT_SCHEME + valueFrom: + fieldRef: + fieldPath: metadata.annotations['tekton.dev/artifact_endpoint_scheme'] + - name: ARTIFACT_ENDPOINT + valueFrom: + fieldRef: + fieldPath: metadata.annotations['tekton.dev/artifact_endpoint'] + - name: ARTIFACT_BUCKET valueFrom: fieldRef: - fieldPath: metadata.namespace + fieldPath: metadata.annotations['tekton.dev/artifact_bucket'] - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: @@ -225,16 +247,16 @@ spec: name: copy-artifacts script: '#!/usr/bin/env sh - mc config host add storage http://minio-service.$NAMESPACE:9000 $AWS_ACCESS_KEY_ID - $AWS_SECRET_ACCESS_KEY + mc config host add storage ${ARTIFACT_ENDPOINT_SCHEME}${ARTIFACT_ENDPOINT} + $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY tar -cvzf even_lines.tgz $(workspaces.split-text-lines.path)/split-text-lines-even_lines - mc cp even_lines.tgz storage/mlpipeline/artifacts/$PIPELINERUN/$PIPELINETASK/even_lines.tgz + mc cp even_lines.tgz storage/$ARTIFACT_BUCKET/artifacts/$PIPELINERUN/$PIPELINETASK/even_lines.tgz tar -cvzf odd_lines.tgz $(workspaces.split-text-lines.path)/split-text-lines-odd_lines - mc cp odd_lines.tgz storage/mlpipeline/artifacts/$PIPELINERUN/$PIPELINETASK/odd_lines.tgz + mc cp odd_lines.tgz storage/$ARTIFACT_BUCKET/artifacts/$PIPELINERUN/$PIPELINETASK/odd_lines.tgz ' volumes: @@ -332,10 +354,18 @@ spec: valueFrom: fieldRef: fieldPath: metadata.labels['tekton.dev/pipelineTask'] - - name: NAMESPACE + - name: ARTIFACT_ENDPOINT_SCHEME + valueFrom: + fieldRef: + fieldPath: metadata.annotations['tekton.dev/artifact_endpoint_scheme'] + - name: ARTIFACT_ENDPOINT valueFrom: fieldRef: - fieldPath: metadata.namespace + fieldPath: metadata.annotations['tekton.dev/artifact_endpoint'] + - name: ARTIFACT_BUCKET + valueFrom: + fieldRef: + fieldPath: metadata.annotations['tekton.dev/artifact_bucket'] - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: @@ -350,12 +380,12 @@ spec: name: copy-artifacts script: '#!/usr/bin/env sh - mc config host add storage http://minio-service.$NAMESPACE:9000 $AWS_ACCESS_KEY_ID - $AWS_SECRET_ACCESS_KEY + mc config host add storage ${ARTIFACT_ENDPOINT_SCHEME}${ARTIFACT_ENDPOINT} + $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY tar -cvzf numbers.tgz $(workspaces.write-numbers.path)/write-numbers-numbers - mc cp numbers.tgz storage/mlpipeline/artifacts/$PIPELINERUN/$PIPELINETASK/numbers.tgz + mc cp numbers.tgz storage/$ARTIFACT_BUCKET/artifacts/$PIPELINERUN/$PIPELINETASK/numbers.tgz ' workspaces: @@ -431,10 +461,18 @@ spec: valueFrom: fieldRef: fieldPath: metadata.labels['tekton.dev/pipelineTask'] - - name: NAMESPACE + - name: ARTIFACT_ENDPOINT_SCHEME + valueFrom: + fieldRef: + fieldPath: metadata.annotations['tekton.dev/artifact_endpoint_scheme'] + - name: ARTIFACT_ENDPOINT + valueFrom: + fieldRef: + fieldPath: metadata.annotations['tekton.dev/artifact_endpoint'] + - name: ARTIFACT_BUCKET valueFrom: fieldRef: - fieldPath: metadata.namespace + fieldPath: metadata.annotations['tekton.dev/artifact_bucket'] - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: @@ -449,12 +487,12 @@ spec: name: copy-artifacts script: '#!/usr/bin/env sh - mc config host add storage http://minio-service.$NAMESPACE:9000 $AWS_ACCESS_KEY_ID - $AWS_SECRET_ACCESS_KEY + mc config host add storage ${ARTIFACT_ENDPOINT_SCHEME}${ARTIFACT_ENDPOINT} + $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY tar -cvzf Output.tgz $(workspaces.sum-numbers.path)/sum-numbers-Output - mc cp Output.tgz storage/mlpipeline/artifacts/$PIPELINERUN/$PIPELINETASK/Output.tgz + mc cp Output.tgz storage/$ARTIFACT_BUCKET/artifacts/$PIPELINERUN/$PIPELINETASK/Output.tgz ' workspaces: @@ -526,10 +564,18 @@ spec: valueFrom: fieldRef: fieldPath: metadata.labels['tekton.dev/pipelineTask'] - - name: NAMESPACE + - name: ARTIFACT_ENDPOINT_SCHEME + valueFrom: + fieldRef: + fieldPath: metadata.annotations['tekton.dev/artifact_endpoint_scheme'] + - name: ARTIFACT_ENDPOINT + valueFrom: + fieldRef: + fieldPath: metadata.annotations['tekton.dev/artifact_endpoint'] + - name: ARTIFACT_BUCKET valueFrom: fieldRef: - fieldPath: metadata.namespace + fieldPath: metadata.annotations['tekton.dev/artifact_bucket'] - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: @@ -544,12 +590,12 @@ spec: name: copy-artifacts script: '#!/usr/bin/env sh - mc config host add storage http://minio-service.$NAMESPACE:9000 $AWS_ACCESS_KEY_ID - $AWS_SECRET_ACCESS_KEY + mc config host add storage ${ARTIFACT_ENDPOINT_SCHEME}${ARTIFACT_ENDPOINT} + $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY tar -cvzf Output.tgz $(results.output.path) - mc cp Output.tgz storage/mlpipeline/artifacts/$PIPELINERUN/$PIPELINETASK/Output.tgz + mc cp Output.tgz storage/$ARTIFACT_BUCKET/artifacts/$PIPELINERUN/$PIPELINETASK/Output.tgz ' - name: print-params diff --git a/sdk/python/tests/compiler/testdata/compose.yaml b/sdk/python/tests/compiler/testdata/compose.yaml index bcf023726e..9fea409401 100644 --- a/sdk/python/tests/compiler/testdata/compose.yaml +++ b/sdk/python/tests/compiler/testdata/compose.yaml @@ -22,12 +22,16 @@ metadata: "name": "outputpath", "optional": true, "type": "String"}], "name": "Download and Save Most Frequent"}' sidecar.istio.io/inject: 'false' + tekton.dev/artifact_bucket: mlpipeline + tekton.dev/artifact_endpoint: minio-service.kubeflow:9000 + tekton.dev/artifact_endpoint_scheme: http:// tekton.dev/input_artifacts: '{"get-frequent": [{"name": "download-downloaded", "parent_task": "download"}], "save": [{"name": "get-frequent-word", "parent_task": "get-frequent"}]}' - tekton.dev/output_artifacts: '{"download": [{"name": "download-downloaded", "path": - "/tmp/results.txt"}], "get-frequent": [{"name": "get-frequent-word", "path": - "/tmp/message.txt"}]}' + tekton.dev/output_artifacts: '{"download": [{"key": "artifacts/$PIPELINERUN/download/downloaded.tgz", + "name": "download-downloaded", "path": "/tmp/results.txt"}], "get-frequent": + [{"key": "artifacts/$PIPELINERUN/get-frequent/word.tgz", "name": "get-frequent-word", + "path": "/tmp/message.txt"}]}' name: download-and-save-most-frequent spec: params: @@ -69,10 +73,18 @@ spec: valueFrom: fieldRef: fieldPath: metadata.labels['tekton.dev/pipelineTask'] - - name: NAMESPACE + - name: ARTIFACT_ENDPOINT_SCHEME valueFrom: fieldRef: - fieldPath: metadata.namespace + fieldPath: metadata.annotations['tekton.dev/artifact_endpoint_scheme'] + - name: ARTIFACT_ENDPOINT + valueFrom: + fieldRef: + fieldPath: metadata.annotations['tekton.dev/artifact_endpoint'] + - name: ARTIFACT_BUCKET + valueFrom: + fieldRef: + fieldPath: metadata.annotations['tekton.dev/artifact_bucket'] - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: @@ -87,12 +99,12 @@ spec: name: copy-artifacts script: '#!/usr/bin/env sh - mc config host add storage http://minio-service.$NAMESPACE:9000 $AWS_ACCESS_KEY_ID - $AWS_SECRET_ACCESS_KEY + mc config host add storage ${ARTIFACT_ENDPOINT_SCHEME}${ARTIFACT_ENDPOINT} + $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY tar -cvzf downloaded.tgz $(results.downloaded.path) - mc cp downloaded.tgz storage/mlpipeline/artifacts/$PIPELINERUN/$PIPELINETASK/downloaded.tgz + mc cp downloaded.tgz storage/$ARTIFACT_BUCKET/artifacts/$PIPELINERUN/$PIPELINETASK/downloaded.tgz ' - name: get-frequent @@ -124,10 +136,18 @@ spec: valueFrom: fieldRef: fieldPath: metadata.labels['tekton.dev/pipelineTask'] - - name: NAMESPACE + - name: ARTIFACT_ENDPOINT_SCHEME + valueFrom: + fieldRef: + fieldPath: metadata.annotations['tekton.dev/artifact_endpoint_scheme'] + - name: ARTIFACT_ENDPOINT + valueFrom: + fieldRef: + fieldPath: metadata.annotations['tekton.dev/artifact_endpoint'] + - name: ARTIFACT_BUCKET valueFrom: fieldRef: - fieldPath: metadata.namespace + fieldPath: metadata.annotations['tekton.dev/artifact_bucket'] - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: @@ -142,12 +162,12 @@ spec: name: copy-artifacts script: '#!/usr/bin/env sh - mc config host add storage http://minio-service.$NAMESPACE:9000 $AWS_ACCESS_KEY_ID - $AWS_SECRET_ACCESS_KEY + mc config host add storage ${ARTIFACT_ENDPOINT_SCHEME}${ARTIFACT_ENDPOINT} + $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY tar -cvzf word.tgz $(results.word.path) - mc cp word.tgz storage/mlpipeline/artifacts/$PIPELINERUN/$PIPELINETASK/word.tgz + mc cp word.tgz storage/$ARTIFACT_BUCKET/artifacts/$PIPELINERUN/$PIPELINETASK/word.tgz ' - name: save diff --git a/sdk/python/tests/compiler/testdata/condition.yaml b/sdk/python/tests/compiler/testdata/condition.yaml index 916168b4f9..848fdbe350 100644 --- a/sdk/python/tests/compiler/testdata/condition.yaml +++ b/sdk/python/tests/compiler/testdata/condition.yaml @@ -21,10 +21,14 @@ metadata: "type": "String"}, {"default": "tails", "name": "forced_result2", "optional": true, "type": "String"}], "name": "Flip Coin Example Pipeline"}' sidecar.istio.io/inject: 'false' + tekton.dev/artifact_bucket: mlpipeline + tekton.dev/artifact_endpoint: minio-service.kubeflow:9000 + tekton.dev/artifact_endpoint_scheme: http:// tekton.dev/input_artifacts: '{"print1": [{"name": "flip-again-output", "parent_task": "flip-again"}], "print2": [{"name": "flip-output", "parent_task": "flip"}]}' - tekton.dev/output_artifacts: '{"flip": [{"name": "flip-output", "path": "/tmp/output"}], - "flip-again": [{"name": "flip-again-output", "path": "/tmp/output"}]}' + tekton.dev/output_artifacts: '{"flip": [{"key": "artifacts/$PIPELINERUN/flip/output.tgz", + "name": "flip-output", "path": "/tmp/output"}], "flip-again": [{"key": "artifacts/$PIPELINERUN/flip-again/output.tgz", + "name": "flip-again-output", "path": "/tmp/output"}]}' name: flip-coin-example-pipeline spec: params: @@ -69,10 +73,18 @@ spec: valueFrom: fieldRef: fieldPath: metadata.labels['tekton.dev/pipelineTask'] - - name: NAMESPACE + - name: ARTIFACT_ENDPOINT_SCHEME valueFrom: fieldRef: - fieldPath: metadata.namespace + fieldPath: metadata.annotations['tekton.dev/artifact_endpoint_scheme'] + - name: ARTIFACT_ENDPOINT + valueFrom: + fieldRef: + fieldPath: metadata.annotations['tekton.dev/artifact_endpoint'] + - name: ARTIFACT_BUCKET + valueFrom: + fieldRef: + fieldPath: metadata.annotations['tekton.dev/artifact_bucket'] - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: @@ -87,12 +99,12 @@ spec: name: copy-artifacts script: '#!/usr/bin/env sh - mc config host add storage http://minio-service.$NAMESPACE:9000 $AWS_ACCESS_KEY_ID - $AWS_SECRET_ACCESS_KEY + mc config host add storage ${ARTIFACT_ENDPOINT_SCHEME}${ARTIFACT_ENDPOINT} + $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY tar -cvzf output.tgz $(results.output.path) - mc cp output.tgz storage/mlpipeline/artifacts/$PIPELINERUN/$PIPELINETASK/output.tgz + mc cp output.tgz storage/$ARTIFACT_BUCKET/artifacts/$PIPELINERUN/$PIPELINETASK/output.tgz ' - conditions: @@ -134,10 +146,18 @@ spec: valueFrom: fieldRef: fieldPath: metadata.labels['tekton.dev/pipelineTask'] - - name: NAMESPACE + - name: ARTIFACT_ENDPOINT_SCHEME + valueFrom: + fieldRef: + fieldPath: metadata.annotations['tekton.dev/artifact_endpoint_scheme'] + - name: ARTIFACT_ENDPOINT + valueFrom: + fieldRef: + fieldPath: metadata.annotations['tekton.dev/artifact_endpoint'] + - name: ARTIFACT_BUCKET valueFrom: fieldRef: - fieldPath: metadata.namespace + fieldPath: metadata.annotations['tekton.dev/artifact_bucket'] - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: @@ -152,12 +172,12 @@ spec: name: copy-artifacts script: '#!/usr/bin/env sh - mc config host add storage http://minio-service.$NAMESPACE:9000 $AWS_ACCESS_KEY_ID - $AWS_SECRET_ACCESS_KEY + mc config host add storage ${ARTIFACT_ENDPOINT_SCHEME}${ARTIFACT_ENDPOINT} + $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY tar -cvzf output.tgz $(results.output.path) - mc cp output.tgz storage/mlpipeline/artifacts/$PIPELINERUN/$PIPELINETASK/output.tgz + mc cp output.tgz storage/$ARTIFACT_BUCKET/artifacts/$PIPELINERUN/$PIPELINETASK/output.tgz ' - conditions: diff --git a/sdk/python/tests/compiler/testdata/exit_handler.yaml b/sdk/python/tests/compiler/testdata/exit_handler.yaml index fffc8a8856..cd6cf98915 100644 --- a/sdk/python/tests/compiler/testdata/exit_handler.yaml +++ b/sdk/python/tests/compiler/testdata/exit_handler.yaml @@ -21,10 +21,13 @@ metadata: or not).", "inputs": [{"default": "gs://ml-pipeline-playground/shakespeare1.txt", "name": "url", "optional": true}], "name": "Exit Handler"}' sidecar.istio.io/inject: 'false' + tekton.dev/artifact_bucket: mlpipeline + tekton.dev/artifact_endpoint: minio-service.kubeflow:9000 + tekton.dev/artifact_endpoint_scheme: http:// tekton.dev/input_artifacts: '{"echo-2": [{"name": "gcs-download-data", "parent_task": "gcs-download"}]}' - tekton.dev/output_artifacts: '{"gcs-download": [{"name": "gcs-download-data", - "path": "/tmp/results.txt"}]}' + tekton.dev/output_artifacts: '{"gcs-download": [{"key": "artifacts/$PIPELINERUN/gcs-download/data.tgz", + "name": "gcs-download-data", "path": "/tmp/results.txt"}]}' name: exit-handler spec: params: @@ -76,10 +79,18 @@ spec: valueFrom: fieldRef: fieldPath: metadata.labels['tekton.dev/pipelineTask'] - - name: NAMESPACE + - name: ARTIFACT_ENDPOINT_SCHEME valueFrom: fieldRef: - fieldPath: metadata.namespace + fieldPath: metadata.annotations['tekton.dev/artifact_endpoint_scheme'] + - name: ARTIFACT_ENDPOINT + valueFrom: + fieldRef: + fieldPath: metadata.annotations['tekton.dev/artifact_endpoint'] + - name: ARTIFACT_BUCKET + valueFrom: + fieldRef: + fieldPath: metadata.annotations['tekton.dev/artifact_bucket'] - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: @@ -94,12 +105,12 @@ spec: name: copy-artifacts script: '#!/usr/bin/env sh - mc config host add storage http://minio-service.$NAMESPACE:9000 $AWS_ACCESS_KEY_ID - $AWS_SECRET_ACCESS_KEY + mc config host add storage ${ARTIFACT_ENDPOINT_SCHEME}${ARTIFACT_ENDPOINT} + $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY tar -cvzf data.tgz $(results.data.path) - mc cp data.tgz storage/mlpipeline/artifacts/$PIPELINERUN/$PIPELINETASK/data.tgz + mc cp data.tgz storage/$ARTIFACT_BUCKET/artifacts/$PIPELINERUN/$PIPELINETASK/data.tgz ' - name: echo-2 diff --git a/sdk/python/tests/compiler/testdata/hidden_output_file.yaml b/sdk/python/tests/compiler/testdata/hidden_output_file.yaml index a374eee653..f994de2b63 100644 --- a/sdk/python/tests/compiler/testdata/hidden_output_file.yaml +++ b/sdk/python/tests/compiler/testdata/hidden_output_file.yaml @@ -19,10 +19,13 @@ metadata: pipelines.kubeflow.org/pipeline_spec: '{"description": "Run a script that passes file to a non configurable path", "name": "Hidden output file pipeline"}' sidecar.istio.io/inject: 'false' + tekton.dev/artifact_bucket: mlpipeline + tekton.dev/artifact_endpoint: minio-service.kubeflow:9000 + tekton.dev/artifact_endpoint_scheme: http:// tekton.dev/input_artifacts: '{"echo": [{"name": "download-file-data", "parent_task": "download-file"}]}' - tekton.dev/output_artifacts: '{"download-file": [{"name": "download-file-data", - "path": "/tmp/results.txt"}]}' + tekton.dev/output_artifacts: '{"download-file": [{"key": "artifacts/$PIPELINERUN/download-file/data.tgz", + "name": "download-file-data", "path": "/tmp/results.txt"}]}' name: hidden-output-file-pipeline spec: pipelineSpec: @@ -62,10 +65,18 @@ spec: valueFrom: fieldRef: fieldPath: metadata.labels['tekton.dev/pipelineTask'] - - name: NAMESPACE + - name: ARTIFACT_ENDPOINT_SCHEME valueFrom: fieldRef: - fieldPath: metadata.namespace + fieldPath: metadata.annotations['tekton.dev/artifact_endpoint_scheme'] + - name: ARTIFACT_ENDPOINT + valueFrom: + fieldRef: + fieldPath: metadata.annotations['tekton.dev/artifact_endpoint'] + - name: ARTIFACT_BUCKET + valueFrom: + fieldRef: + fieldPath: metadata.annotations['tekton.dev/artifact_bucket'] - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: @@ -80,12 +91,12 @@ spec: name: copy-artifacts script: '#!/usr/bin/env sh - mc config host add storage http://minio-service.$NAMESPACE:9000 $AWS_ACCESS_KEY_ID - $AWS_SECRET_ACCESS_KEY + mc config host add storage ${ARTIFACT_ENDPOINT_SCHEME}${ARTIFACT_ENDPOINT} + $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY tar -cvzf data.tgz $(results.data.path) - mc cp data.tgz storage/mlpipeline/artifacts/$PIPELINERUN/$PIPELINETASK/data.tgz + mc cp data.tgz storage/$ARTIFACT_BUCKET/artifacts/$PIPELINERUN/$PIPELINETASK/data.tgz ' volumes: diff --git a/sdk/python/tests/compiler/testdata/imagepullsecrets.yaml b/sdk/python/tests/compiler/testdata/imagepullsecrets.yaml index 234b4eb08d..8af407e2a1 100644 --- a/sdk/python/tests/compiler/testdata/imagepullsecrets.yaml +++ b/sdk/python/tests/compiler/testdata/imagepullsecrets.yaml @@ -21,9 +21,12 @@ metadata: flies are following flies.", "name": "message", "optional": true}], "name": "Save Most Frequent"}' sidecar.istio.io/inject: 'false' + tekton.dev/artifact_bucket: mlpipeline + tekton.dev/artifact_endpoint: minio-service.kubeflow:9000 + tekton.dev/artifact_endpoint_scheme: http:// tekton.dev/input_artifacts: '{}' - tekton.dev/output_artifacts: '{"get-frequent": [{"name": "get-frequent-word", - "path": "/tmp/message.txt"}]}' + tekton.dev/output_artifacts: '{"get-frequent": [{"key": "artifacts/$PIPELINERUN/get-frequent/word.tgz", + "name": "get-frequent-word", "path": "/tmp/message.txt"}]}' name: save-most-frequent spec: params: @@ -63,10 +66,18 @@ spec: valueFrom: fieldRef: fieldPath: metadata.labels['tekton.dev/pipelineTask'] - - name: NAMESPACE + - name: ARTIFACT_ENDPOINT_SCHEME valueFrom: fieldRef: - fieldPath: metadata.namespace + fieldPath: metadata.annotations['tekton.dev/artifact_endpoint_scheme'] + - name: ARTIFACT_ENDPOINT + valueFrom: + fieldRef: + fieldPath: metadata.annotations['tekton.dev/artifact_endpoint'] + - name: ARTIFACT_BUCKET + valueFrom: + fieldRef: + fieldPath: metadata.annotations['tekton.dev/artifact_bucket'] - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: @@ -81,12 +92,12 @@ spec: name: copy-artifacts script: '#!/usr/bin/env sh - mc config host add storage http://minio-service.$NAMESPACE:9000 $AWS_ACCESS_KEY_ID - $AWS_SECRET_ACCESS_KEY + mc config host add storage ${ARTIFACT_ENDPOINT_SCHEME}${ARTIFACT_ENDPOINT} + $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY tar -cvzf word.tgz $(results.word.path) - mc cp word.tgz storage/mlpipeline/artifacts/$PIPELINERUN/$PIPELINETASK/word.tgz + mc cp word.tgz storage/$ARTIFACT_BUCKET/artifacts/$PIPELINERUN/$PIPELINETASK/word.tgz ' podTemplate: diff --git a/sdk/python/tests/compiler/testdata/init_container.yaml b/sdk/python/tests/compiler/testdata/init_container.yaml index 6833d38775..f8766bd612 100644 --- a/sdk/python/tests/compiler/testdata/init_container.yaml +++ b/sdk/python/tests/compiler/testdata/init_container.yaml @@ -19,6 +19,9 @@ metadata: pipelines.kubeflow.org/pipeline_spec: '{"description": "A pipeline with init container.", "name": "InitContainer"}' sidecar.istio.io/inject: 'false' + tekton.dev/artifact_bucket: mlpipeline + tekton.dev/artifact_endpoint: minio-service.kubeflow:9000 + tekton.dev/artifact_endpoint_scheme: http:// tekton.dev/input_artifacts: '{}' tekton.dev/output_artifacts: '{}' name: initcontainer diff --git a/sdk/python/tests/compiler/testdata/input_artifact_raw_value.yaml b/sdk/python/tests/compiler/testdata/input_artifact_raw_value.yaml index f097251072..804def3bad 100644 --- a/sdk/python/tests/compiler/testdata/input_artifact_raw_value.yaml +++ b/sdk/python/tests/compiler/testdata/input_artifact_raw_value.yaml @@ -20,6 +20,9 @@ metadata: define artifact inputs and pass raw artifacts to them.", "name": "Pipeline with artifact input raw argument value."}' sidecar.istio.io/inject: 'false' + tekton.dev/artifact_bucket: mlpipeline + tekton.dev/artifact_endpoint: minio-service.kubeflow:9000 + tekton.dev/artifact_endpoint_scheme: http:// tekton.dev/input_artifacts: '{}' tekton.dev/output_artifacts: '{}' name: pipeline-with-artifact-input-raw-argument-value diff --git a/sdk/python/tests/compiler/testdata/katib.yaml b/sdk/python/tests/compiler/testdata/katib.yaml index 944c837932..88822b6fba 100644 --- a/sdk/python/tests/compiler/testdata/katib.yaml +++ b/sdk/python/tests/compiler/testdata/katib.yaml @@ -26,10 +26,13 @@ metadata: {"default": "True", "name": "deleteAfterDone", "optional": true, "type": "Boolean"}], "name": "Launch katib experiment"}' sidecar.istio.io/inject: 'false' + tekton.dev/artifact_bucket: mlpipeline + tekton.dev/artifact_endpoint: minio-service.kubeflow:9000 + tekton.dev/artifact_endpoint_scheme: http:// tekton.dev/input_artifacts: '{"my-out-cop": [{"name": "mnist-hpo-bestHyperParameter", "parent_task": "mnist-hpo"}]}' - tekton.dev/output_artifacts: '{"mnist-hpo": [{"name": "mnist-hpo-bestHyperParameter", - "path": "/output.txt"}]}' + tekton.dev/output_artifacts: '{"mnist-hpo": [{"key": "artifacts/$PIPELINERUN/mnist-hpo/bestHyperParameter.tgz", + "name": "mnist-hpo-bestHyperParameter", "path": "/output.txt"}]}' name: launch-katib-experiment spec: params: @@ -142,10 +145,18 @@ spec: valueFrom: fieldRef: fieldPath: metadata.labels['tekton.dev/pipelineTask'] - - name: NAMESPACE + - name: ARTIFACT_ENDPOINT_SCHEME valueFrom: fieldRef: - fieldPath: metadata.namespace + fieldPath: metadata.annotations['tekton.dev/artifact_endpoint_scheme'] + - name: ARTIFACT_ENDPOINT + valueFrom: + fieldRef: + fieldPath: metadata.annotations['tekton.dev/artifact_endpoint'] + - name: ARTIFACT_BUCKET + valueFrom: + fieldRef: + fieldPath: metadata.annotations['tekton.dev/artifact_bucket'] - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: @@ -160,12 +171,12 @@ spec: name: copy-artifacts script: '#!/usr/bin/env sh - mc config host add storage http://minio-service.$NAMESPACE:9000 $AWS_ACCESS_KEY_ID - $AWS_SECRET_ACCESS_KEY + mc config host add storage ${ARTIFACT_ENDPOINT_SCHEME}${ARTIFACT_ENDPOINT} + $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY tar -cvzf bestHyperParameter.tgz $(results.besthyperparameter.path) - mc cp bestHyperParameter.tgz storage/mlpipeline/artifacts/$PIPELINERUN/$PIPELINETASK/bestHyperParameter.tgz + mc cp bestHyperParameter.tgz storage/$ARTIFACT_BUCKET/artifacts/$PIPELINERUN/$PIPELINETASK/bestHyperParameter.tgz ' - name: my-out-cop diff --git a/sdk/python/tests/compiler/testdata/load_from_yaml.yaml b/sdk/python/tests/compiler/testdata/load_from_yaml.yaml index 5adda46978..454f610972 100644 --- a/sdk/python/tests/compiler/testdata/load_from_yaml.yaml +++ b/sdk/python/tests/compiler/testdata/load_from_yaml.yaml @@ -19,9 +19,12 @@ metadata: pipelines.kubeflow.org/pipeline_spec: '{"description": "A pipeline with components loaded from yaml", "name": "component-yaml-pipeline"}' sidecar.istio.io/inject: 'false' + tekton.dev/artifact_bucket: mlpipeline + tekton.dev/artifact_endpoint: minio-service.kubeflow:9000 + tekton.dev/artifact_endpoint_scheme: http:// tekton.dev/input_artifacts: '{}' - tekton.dev/output_artifacts: '{"busybox": [{"name": "busybox-dummy_output_path", - "path": "/tmp/outputs/dummy_output_path/data"}]}' + tekton.dev/output_artifacts: '{"busybox": [{"key": "artifacts/$PIPELINERUN/busybox/dummy_output_path.tgz", + "name": "busybox-dummy_output_path", "path": "/tmp/outputs/dummy_output_path/data"}]}' name: component-yaml-pipeline spec: pipelineSpec: @@ -50,10 +53,18 @@ spec: valueFrom: fieldRef: fieldPath: metadata.labels['tekton.dev/pipelineTask'] - - name: NAMESPACE + - name: ARTIFACT_ENDPOINT_SCHEME valueFrom: fieldRef: - fieldPath: metadata.namespace + fieldPath: metadata.annotations['tekton.dev/artifact_endpoint_scheme'] + - name: ARTIFACT_ENDPOINT + valueFrom: + fieldRef: + fieldPath: metadata.annotations['tekton.dev/artifact_endpoint'] + - name: ARTIFACT_BUCKET + valueFrom: + fieldRef: + fieldPath: metadata.annotations['tekton.dev/artifact_bucket'] - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: @@ -68,11 +79,11 @@ spec: name: copy-artifacts script: '#!/usr/bin/env sh - mc config host add storage http://minio-service.$NAMESPACE:9000 $AWS_ACCESS_KEY_ID - $AWS_SECRET_ACCESS_KEY + mc config host add storage ${ARTIFACT_ENDPOINT_SCHEME}${ARTIFACT_ENDPOINT} + $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY tar -cvzf dummy_output_path.tgz $(results.dummy-output-path.path) - mc cp dummy_output_path.tgz storage/mlpipeline/artifacts/$PIPELINERUN/$PIPELINETASK/dummy_output_path.tgz + mc cp dummy_output_path.tgz storage/$ARTIFACT_BUCKET/artifacts/$PIPELINERUN/$PIPELINETASK/dummy_output_path.tgz ' diff --git a/sdk/python/tests/compiler/testdata/loop_static.yaml b/sdk/python/tests/compiler/testdata/loop_static.yaml index b9de8235f7..81a692c572 100644 --- a/sdk/python/tests/compiler/testdata/loop_static.yaml +++ b/sdk/python/tests/compiler/testdata/loop_static.yaml @@ -19,6 +19,9 @@ metadata: pipelines.kubeflow.org/pipeline_spec: '{"inputs": [{"default": "10", "name": "my_pipe_param", "optional": true}], "name": "my-loop-pipeline"}' sidecar.istio.io/inject: 'false' + tekton.dev/artifact_bucket: mlpipeline + tekton.dev/artifact_endpoint: minio-service.kubeflow:9000 + tekton.dev/artifact_endpoint_scheme: http:// tekton.dev/input_artifacts: '{"my-in-coop1": [{"name": "loop-item-param-subvar-A_a", "parent_task": null}], "my-in-coop2": [{"name": "loop-item-param-subvar-B_b", "parent_task": null}]}' diff --git a/sdk/python/tests/compiler/testdata/node_selector.yaml b/sdk/python/tests/compiler/testdata/node_selector.yaml index 48346a9b13..0b5fa646af 100644 --- a/sdk/python/tests/compiler/testdata/node_selector.yaml +++ b/sdk/python/tests/compiler/testdata/node_selector.yaml @@ -19,6 +19,9 @@ metadata: pipelines.kubeflow.org/pipeline_spec: '{"description": "A pipeline with Node Selector", "name": "node_selector"}' sidecar.istio.io/inject: 'false' + tekton.dev/artifact_bucket: mlpipeline + tekton.dev/artifact_endpoint: minio-service.kubeflow:9000 + tekton.dev/artifact_endpoint_scheme: http:// tekton.dev/input_artifacts: '{}' tekton.dev/output_artifacts: '{}' name: node-selector diff --git a/sdk/python/tests/compiler/testdata/parallel_join.yaml b/sdk/python/tests/compiler/testdata/parallel_join.yaml index 1151b0a080..f80a7c537d 100644 --- a/sdk/python/tests/compiler/testdata/parallel_join.yaml +++ b/sdk/python/tests/compiler/testdata/parallel_join.yaml @@ -21,6 +21,9 @@ metadata: "name": "url1", "optional": true}, {"default": "gs://ml-pipeline-playground/shakespeare2.txt", "name": "url2", "optional": true}], "name": "Parallel pipeline"}' sidecar.istio.io/inject: 'false' + tekton.dev/artifact_bucket: mlpipeline + tekton.dev/artifact_endpoint: minio-service.kubeflow:9000 + tekton.dev/artifact_endpoint_scheme: http:// tekton.dev/input_artifacts: '{}' tekton.dev/output_artifacts: '{}' name: parallel-pipeline diff --git a/sdk/python/tests/compiler/testdata/parallel_join_with_argo_vars.yaml b/sdk/python/tests/compiler/testdata/parallel_join_with_argo_vars.yaml index 71d2411eff..de34af9ad1 100644 --- a/sdk/python/tests/compiler/testdata/parallel_join_with_argo_vars.yaml +++ b/sdk/python/tests/compiler/testdata/parallel_join_with_argo_vars.yaml @@ -22,10 +22,14 @@ metadata: "optional": true}, {"default": "gs://ml-pipeline-playground/shakespeare2.txt", "name": "url2", "optional": true}], "name": "Parallel pipeline with argo vars"}' sidecar.istio.io/inject: 'false' + tekton.dev/artifact_bucket: mlpipeline + tekton.dev/artifact_endpoint: minio-service.kubeflow:9000 + tekton.dev/artifact_endpoint_scheme: http:// tekton.dev/input_artifacts: '{"echo": [{"name": "gcs-download-2-data", "parent_task": "gcs-download-2"}, {"name": "gcs-download-data", "parent_task": "gcs-download"}]}' - tekton.dev/output_artifacts: '{"gcs-download": [{"name": "gcs-download-data", - "path": "/tmp/results.txt"}], "gcs-download-2": [{"name": "gcs-download-2-data", + tekton.dev/output_artifacts: '{"gcs-download": [{"key": "artifacts/$PIPELINERUN/gcs-download/data.tgz", + "name": "gcs-download-data", "path": "/tmp/results.txt"}], "gcs-download-2": + [{"key": "artifacts/$PIPELINERUN/gcs-download-2/data.tgz", "name": "gcs-download-2-data", "path": "/tmp/results.txt"}]}' name: parallel-pipeline-with-argo-vars spec: @@ -70,10 +74,18 @@ spec: valueFrom: fieldRef: fieldPath: metadata.labels['tekton.dev/pipelineTask'] - - name: NAMESPACE + - name: ARTIFACT_ENDPOINT_SCHEME valueFrom: fieldRef: - fieldPath: metadata.namespace + fieldPath: metadata.annotations['tekton.dev/artifact_endpoint_scheme'] + - name: ARTIFACT_ENDPOINT + valueFrom: + fieldRef: + fieldPath: metadata.annotations['tekton.dev/artifact_endpoint'] + - name: ARTIFACT_BUCKET + valueFrom: + fieldRef: + fieldPath: metadata.annotations['tekton.dev/artifact_bucket'] - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: @@ -88,12 +100,12 @@ spec: name: copy-artifacts script: '#!/usr/bin/env sh - mc config host add storage http://minio-service.$NAMESPACE:9000 $AWS_ACCESS_KEY_ID - $AWS_SECRET_ACCESS_KEY + mc config host add storage ${ARTIFACT_ENDPOINT_SCHEME}${ARTIFACT_ENDPOINT} + $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY tar -cvzf data.tgz $(results.data.path) - mc cp data.tgz storage/mlpipeline/artifacts/$PIPELINERUN/$PIPELINETASK/data.tgz + mc cp data.tgz storage/$ARTIFACT_BUCKET/artifacts/$PIPELINERUN/$PIPELINETASK/data.tgz ' - name: gcs-download-2 @@ -125,10 +137,18 @@ spec: valueFrom: fieldRef: fieldPath: metadata.labels['tekton.dev/pipelineTask'] - - name: NAMESPACE + - name: ARTIFACT_ENDPOINT_SCHEME + valueFrom: + fieldRef: + fieldPath: metadata.annotations['tekton.dev/artifact_endpoint_scheme'] + - name: ARTIFACT_ENDPOINT + valueFrom: + fieldRef: + fieldPath: metadata.annotations['tekton.dev/artifact_endpoint'] + - name: ARTIFACT_BUCKET valueFrom: fieldRef: - fieldPath: metadata.namespace + fieldPath: metadata.annotations['tekton.dev/artifact_bucket'] - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: @@ -143,12 +163,12 @@ spec: name: copy-artifacts script: '#!/usr/bin/env sh - mc config host add storage http://minio-service.$NAMESPACE:9000 $AWS_ACCESS_KEY_ID - $AWS_SECRET_ACCESS_KEY + mc config host add storage ${ARTIFACT_ENDPOINT_SCHEME}${ARTIFACT_ENDPOINT} + $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY tar -cvzf data.tgz $(results.data.path) - mc cp data.tgz storage/mlpipeline/artifacts/$PIPELINERUN/$PIPELINETASK/data.tgz + mc cp data.tgz storage/$ARTIFACT_BUCKET/artifacts/$PIPELINERUN/$PIPELINETASK/data.tgz ' - name: echo diff --git a/sdk/python/tests/compiler/testdata/parallel_join_with_artifacts.yaml b/sdk/python/tests/compiler/testdata/parallel_join_with_artifacts.yaml index a996b0c7f5..91c94cdd2b 100644 --- a/sdk/python/tests/compiler/testdata/parallel_join_with_artifacts.yaml +++ b/sdk/python/tests/compiler/testdata/parallel_join_with_artifacts.yaml @@ -21,10 +21,14 @@ metadata: "name": "url1", "optional": true}, {"default": "gs://ml-pipeline-playground/shakespeare2.txt", "name": "url2", "optional": true}], "name": "Parallel pipeline"}' sidecar.istio.io/inject: 'false' + tekton.dev/artifact_bucket: mlpipeline + tekton.dev/artifact_endpoint: minio-service.kubeflow:9000 + tekton.dev/artifact_endpoint_scheme: http:// tekton.dev/input_artifacts: '{"echo": [{"name": "gcs-download-2-data", "parent_task": "gcs-download-2"}, {"name": "gcs-download-data", "parent_task": "gcs-download"}]}' - tekton.dev/output_artifacts: '{"gcs-download": [{"name": "gcs-download-data", - "path": "/tmp/results.txt"}], "gcs-download-2": [{"name": "gcs-download-2-data", + tekton.dev/output_artifacts: '{"gcs-download": [{"key": "artifacts/$PIPELINERUN/gcs-download/data.tgz", + "name": "gcs-download-data", "path": "/tmp/results.txt"}], "gcs-download-2": + [{"key": "artifacts/$PIPELINERUN/gcs-download-2/data.tgz", "name": "gcs-download-2-data", "path": "/tmp/results.txt"}]}' name: parallel-pipeline spec: @@ -69,10 +73,18 @@ spec: valueFrom: fieldRef: fieldPath: metadata.labels['tekton.dev/pipelineTask'] - - name: NAMESPACE + - name: ARTIFACT_ENDPOINT_SCHEME valueFrom: fieldRef: - fieldPath: metadata.namespace + fieldPath: metadata.annotations['tekton.dev/artifact_endpoint_scheme'] + - name: ARTIFACT_ENDPOINT + valueFrom: + fieldRef: + fieldPath: metadata.annotations['tekton.dev/artifact_endpoint'] + - name: ARTIFACT_BUCKET + valueFrom: + fieldRef: + fieldPath: metadata.annotations['tekton.dev/artifact_bucket'] - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: @@ -87,12 +99,12 @@ spec: name: copy-artifacts script: '#!/usr/bin/env sh - mc config host add storage http://minio-service.$NAMESPACE:9000 $AWS_ACCESS_KEY_ID - $AWS_SECRET_ACCESS_KEY + mc config host add storage ${ARTIFACT_ENDPOINT_SCHEME}${ARTIFACT_ENDPOINT} + $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY tar -cvzf data.tgz $(results.data.path) - mc cp data.tgz storage/mlpipeline/artifacts/$PIPELINERUN/$PIPELINETASK/data.tgz + mc cp data.tgz storage/$ARTIFACT_BUCKET/artifacts/$PIPELINERUN/$PIPELINETASK/data.tgz ' - name: gcs-download-2 @@ -124,10 +136,18 @@ spec: valueFrom: fieldRef: fieldPath: metadata.labels['tekton.dev/pipelineTask'] - - name: NAMESPACE + - name: ARTIFACT_ENDPOINT_SCHEME + valueFrom: + fieldRef: + fieldPath: metadata.annotations['tekton.dev/artifact_endpoint_scheme'] + - name: ARTIFACT_ENDPOINT + valueFrom: + fieldRef: + fieldPath: metadata.annotations['tekton.dev/artifact_endpoint'] + - name: ARTIFACT_BUCKET valueFrom: fieldRef: - fieldPath: metadata.namespace + fieldPath: metadata.annotations['tekton.dev/artifact_bucket'] - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: @@ -142,12 +162,12 @@ spec: name: copy-artifacts script: '#!/usr/bin/env sh - mc config host add storage http://minio-service.$NAMESPACE:9000 $AWS_ACCESS_KEY_ID - $AWS_SECRET_ACCESS_KEY + mc config host add storage ${ARTIFACT_ENDPOINT_SCHEME}${ARTIFACT_ENDPOINT} + $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY tar -cvzf data.tgz $(results.data.path) - mc cp data.tgz storage/mlpipeline/artifacts/$PIPELINERUN/$PIPELINETASK/data.tgz + mc cp data.tgz storage/$ARTIFACT_BUCKET/artifacts/$PIPELINERUN/$PIPELINETASK/data.tgz ' - name: echo diff --git a/sdk/python/tests/compiler/testdata/parallel_join_with_logging.yaml b/sdk/python/tests/compiler/testdata/parallel_join_with_logging.yaml index 88191fedf0..283c493434 100644 --- a/sdk/python/tests/compiler/testdata/parallel_join_with_logging.yaml +++ b/sdk/python/tests/compiler/testdata/parallel_join_with_logging.yaml @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - apiVersion: tekton.dev/v1beta1 kind: PipelineRun metadata: @@ -22,10 +21,14 @@ metadata: "name": "url1", "optional": true}, {"default": "gs://ml-pipeline-playground/shakespeare2.txt", "name": "url2", "optional": true}], "name": "Parallel pipeline"}' sidecar.istio.io/inject: 'false' + tekton.dev/artifact_bucket: mlpipeline + tekton.dev/artifact_endpoint: minio-service.kubeflow:9000 + tekton.dev/artifact_endpoint_scheme: http:// tekton.dev/input_artifacts: '{}' - tekton.dev/output_artifacts: '{"echo": [{"name": "echo", "path": "logs_to_S3"}], - "gcs-download": [{"name": "gcs-download", "path": "logs_to_S3"}], "gcs-download-2": - [{"name": "gcs-download-2", "path": "logs_to_S3"}]}' + tekton.dev/output_artifacts: '{"echo": [{"key": "artifacts/$PIPELINERUN/echo/echo.tgz", + "name": "echo", "path": "logs_to_S3"}], "gcs-download": [{"key": "artifacts/$PIPELINERUN/gcs-download/gcs-download.tgz", + "name": "gcs-download", "path": "logs_to_S3"}], "gcs-download-2": [{"key": "artifacts/$PIPELINERUN/gcs-download-2/gcs-download-2.tgz", + "name": "gcs-download-2", "path": "logs_to_S3"}]}' name: parallel-pipeline spec: params: @@ -82,10 +85,18 @@ spec: valueFrom: fieldRef: fieldPath: metadata.labels['tekton.dev/pipelineTask'] - - name: NAMESPACE + - name: ARTIFACT_ENDPOINT_SCHEME + valueFrom: + fieldRef: + fieldPath: metadata.annotations['tekton.dev/artifact_endpoint_scheme'] + - name: ARTIFACT_ENDPOINT valueFrom: fieldRef: - fieldPath: metadata.namespace + fieldPath: metadata.annotations['tekton.dev/artifact_endpoint'] + - name: ARTIFACT_BUCKET + valueFrom: + fieldRef: + fieldPath: metadata.annotations['tekton.dev/artifact_bucket'] - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: @@ -104,13 +115,13 @@ spec: name: copy-artifacts script: '#!/usr/bin/env sh - mc config host add storage http://minio-service.$NAMESPACE:9000 $AWS_ACCESS_KEY_ID - $AWS_SECRET_ACCESS_KEY + mc config host add storage ${ARTIFACT_ENDPOINT_SCHEME}${ARTIFACT_ENDPOINT} + $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY cat /var/log/containers/$TASKRUN*$NAMESPACE*step-main*.log > step-main.log && tar -czf $TASKRUN-$NAMESPACE-step-main_log.tgz step-main.log - mc cp $TASKRUN-$NAMESPACE-step-main_log.tgz storage/mlpipeline/artifacts/$PIPELINERUN/$PIPELINETASK/ + mc cp $TASKRUN-$NAMESPACE-step-main_log.tgz storage/$ARTIFACT_BUCKET/artifacts/$PIPELINERUN/$PIPELINETASK/ ' volumes: @@ -168,10 +179,18 @@ spec: valueFrom: fieldRef: fieldPath: metadata.labels['tekton.dev/pipelineTask'] - - name: NAMESPACE + - name: ARTIFACT_ENDPOINT_SCHEME + valueFrom: + fieldRef: + fieldPath: metadata.annotations['tekton.dev/artifact_endpoint_scheme'] + - name: ARTIFACT_ENDPOINT valueFrom: fieldRef: - fieldPath: metadata.namespace + fieldPath: metadata.annotations['tekton.dev/artifact_endpoint'] + - name: ARTIFACT_BUCKET + valueFrom: + fieldRef: + fieldPath: metadata.annotations['tekton.dev/artifact_bucket'] - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: @@ -190,13 +209,13 @@ spec: name: copy-artifacts script: '#!/usr/bin/env sh - mc config host add storage http://minio-service.$NAMESPACE:9000 $AWS_ACCESS_KEY_ID - $AWS_SECRET_ACCESS_KEY + mc config host add storage ${ARTIFACT_ENDPOINT_SCHEME}${ARTIFACT_ENDPOINT} + $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY cat /var/log/containers/$TASKRUN*$NAMESPACE*step-main*.log > step-main.log && tar -czf $TASKRUN-$NAMESPACE-step-main_log.tgz step-main.log - mc cp $TASKRUN-$NAMESPACE-step-main_log.tgz storage/mlpipeline/artifacts/$PIPELINERUN/$PIPELINETASK/ + mc cp $TASKRUN-$NAMESPACE-step-main_log.tgz storage/$ARTIFACT_BUCKET/artifacts/$PIPELINERUN/$PIPELINETASK/ ' volumes: @@ -254,10 +273,18 @@ spec: valueFrom: fieldRef: fieldPath: metadata.labels['tekton.dev/pipelineTask'] - - name: NAMESPACE + - name: ARTIFACT_ENDPOINT_SCHEME + valueFrom: + fieldRef: + fieldPath: metadata.annotations['tekton.dev/artifact_endpoint_scheme'] + - name: ARTIFACT_ENDPOINT + valueFrom: + fieldRef: + fieldPath: metadata.annotations['tekton.dev/artifact_endpoint'] + - name: ARTIFACT_BUCKET valueFrom: fieldRef: - fieldPath: metadata.namespace + fieldPath: metadata.annotations['tekton.dev/artifact_bucket'] - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: @@ -276,13 +303,13 @@ spec: name: copy-artifacts script: '#!/usr/bin/env sh - mc config host add storage http://minio-service.$NAMESPACE:9000 $AWS_ACCESS_KEY_ID - $AWS_SECRET_ACCESS_KEY + mc config host add storage ${ARTIFACT_ENDPOINT_SCHEME}${ARTIFACT_ENDPOINT} + $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY cat /var/log/containers/$TASKRUN*$NAMESPACE*step-main*.log > step-main.log && tar -czf $TASKRUN-$NAMESPACE-step-main_log.tgz step-main.log - mc cp $TASKRUN-$NAMESPACE-step-main_log.tgz storage/mlpipeline/artifacts/$PIPELINERUN/$PIPELINETASK/ + mc cp $TASKRUN-$NAMESPACE-step-main_log.tgz storage/$ARTIFACT_BUCKET/artifacts/$PIPELINERUN/$PIPELINETASK/ ' volumes: diff --git a/sdk/python/tests/compiler/testdata/pipeline_transformers.yaml b/sdk/python/tests/compiler/testdata/pipeline_transformers.yaml index cd0f8d0e3c..3e6b1557e8 100644 --- a/sdk/python/tests/compiler/testdata/pipeline_transformers.yaml +++ b/sdk/python/tests/compiler/testdata/pipeline_transformers.yaml @@ -20,6 +20,9 @@ metadata: to apply functions to all ops in the pipeline by pipeline transformers", "name": "Pipeline transformer"}' sidecar.istio.io/inject: 'false' + tekton.dev/artifact_bucket: mlpipeline + tekton.dev/artifact_endpoint: minio-service.kubeflow:9000 + tekton.dev/artifact_endpoint_scheme: http:// tekton.dev/input_artifacts: '{}' tekton.dev/output_artifacts: '{}' name: pipeline-transformer diff --git a/sdk/python/tests/compiler/testdata/pipelineparams.yaml b/sdk/python/tests/compiler/testdata/pipelineparams.yaml index 59423ed5dc..45c464e54e 100644 --- a/sdk/python/tests/compiler/testdata/pipelineparams.yaml +++ b/sdk/python/tests/compiler/testdata/pipelineparams.yaml @@ -21,10 +21,13 @@ metadata: true, "type": "String"}, {"default": "10", "name": "sleep_ms", "optional": true, "type": "Integer"}], "name": "PipelineParams"}' sidecar.istio.io/inject: 'false' + tekton.dev/artifact_bucket: mlpipeline + tekton.dev/artifact_endpoint: minio-service.kubeflow:9000 + tekton.dev/artifact_endpoint_scheme: http:// tekton.dev/input_artifacts: '{"echo": [{"name": "download-downloaded_resultOutput", "parent_task": "download"}]}' - tekton.dev/output_artifacts: '{"download": [{"name": "download-downloaded_resultOutput", - "path": "/tmp/results.txt"}]}' + tekton.dev/output_artifacts: '{"download": [{"key": "artifacts/$PIPELINERUN/download/downloaded_resultOutput.tgz", + "name": "download-downloaded_resultOutput", "path": "/tmp/results.txt"}]}' name: pipelineparams spec: params: @@ -74,10 +77,18 @@ spec: valueFrom: fieldRef: fieldPath: metadata.labels['tekton.dev/pipelineTask'] - - name: NAMESPACE + - name: ARTIFACT_ENDPOINT_SCHEME valueFrom: fieldRef: - fieldPath: metadata.namespace + fieldPath: metadata.annotations['tekton.dev/artifact_endpoint_scheme'] + - name: ARTIFACT_ENDPOINT + valueFrom: + fieldRef: + fieldPath: metadata.annotations['tekton.dev/artifact_endpoint'] + - name: ARTIFACT_BUCKET + valueFrom: + fieldRef: + fieldPath: metadata.annotations['tekton.dev/artifact_bucket'] - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: @@ -92,12 +103,12 @@ spec: name: copy-artifacts script: '#!/usr/bin/env sh - mc config host add storage http://minio-service.$NAMESPACE:9000 $AWS_ACCESS_KEY_ID - $AWS_SECRET_ACCESS_KEY + mc config host add storage ${ARTIFACT_ENDPOINT_SCHEME}${ARTIFACT_ENDPOINT} + $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY tar -cvzf downloaded_resultOutput.tgz $(results.downloaded-resultoutput.path) - mc cp downloaded_resultOutput.tgz storage/mlpipeline/artifacts/$PIPELINERUN/$PIPELINETASK/downloaded_resultOutput.tgz + mc cp downloaded_resultOutput.tgz storage/$ARTIFACT_BUCKET/artifacts/$PIPELINERUN/$PIPELINETASK/downloaded_resultOutput.tgz ' - name: echo diff --git a/sdk/python/tests/compiler/testdata/resourceop_basic.yaml b/sdk/python/tests/compiler/testdata/resourceop_basic.yaml index ccb8e55c9a..cc46c292f4 100644 --- a/sdk/python/tests/compiler/testdata/resourceop_basic.yaml +++ b/sdk/python/tests/compiler/testdata/resourceop_basic.yaml @@ -19,6 +19,9 @@ metadata: pipelines.kubeflow.org/pipeline_spec: '{"description": "A Basic Example on ResourceOp Usage.", "name": "ResourceOp Basic"}' sidecar.istio.io/inject: 'false' + tekton.dev/artifact_bucket: mlpipeline + tekton.dev/artifact_endpoint: minio-service.kubeflow:9000 + tekton.dev/artifact_endpoint_scheme: http:// tekton.dev/input_artifacts: '{}' tekton.dev/output_artifacts: '{}' name: resourceop-basic diff --git a/sdk/python/tests/compiler/testdata/retry.yaml b/sdk/python/tests/compiler/testdata/retry.yaml index c1982134dc..c8227d42d8 100644 --- a/sdk/python/tests/compiler/testdata/retry.yaml +++ b/sdk/python/tests/compiler/testdata/retry.yaml @@ -20,6 +20,9 @@ metadata: two steps which fail randomly. It shows how to use ContainerOp(...).set_retry(...).", "name": "Retry random failures"}' sidecar.istio.io/inject: 'false' + tekton.dev/artifact_bucket: mlpipeline + tekton.dev/artifact_endpoint: minio-service.kubeflow:9000 + tekton.dev/artifact_endpoint_scheme: http:// tekton.dev/input_artifacts: '{}' tekton.dev/output_artifacts: '{}' name: retry-random-failures diff --git a/sdk/python/tests/compiler/testdata/sequential.yaml b/sdk/python/tests/compiler/testdata/sequential.yaml index d3b8346427..6d7b4098f5 100644 --- a/sdk/python/tests/compiler/testdata/sequential.yaml +++ b/sdk/python/tests/compiler/testdata/sequential.yaml @@ -21,6 +21,9 @@ metadata: "name": "url", "optional": true}, {"default": "/tmp/results.txt", "name": "path", "optional": true}], "name": "Sequential pipeline"}' sidecar.istio.io/inject: 'false' + tekton.dev/artifact_bucket: mlpipeline + tekton.dev/artifact_endpoint: minio-service.kubeflow:9000 + tekton.dev/artifact_endpoint_scheme: http:// tekton.dev/input_artifacts: '{}' tekton.dev/output_artifacts: '{}' name: sequential-pipeline diff --git a/sdk/python/tests/compiler/testdata/sidecar.yaml b/sdk/python/tests/compiler/testdata/sidecar.yaml index 940957d649..1d9d3fa153 100644 --- a/sdk/python/tests/compiler/testdata/sidecar.yaml +++ b/sdk/python/tests/compiler/testdata/sidecar.yaml @@ -19,10 +19,13 @@ metadata: pipelines.kubeflow.org/pipeline_spec: '{"description": "A pipeline with sidecars.", "name": "Sidecar"}' sidecar.istio.io/inject: 'false' + tekton.dev/artifact_bucket: mlpipeline + tekton.dev/artifact_endpoint: minio-service.kubeflow:9000 + tekton.dev/artifact_endpoint_scheme: http:// tekton.dev/input_artifacts: '{"echo": [{"name": "download-downloaded", "parent_task": "download"}]}' - tekton.dev/output_artifacts: '{"download": [{"name": "download-downloaded", "path": - "/tmp/results.txt"}]}' + tekton.dev/output_artifacts: '{"download": [{"key": "artifacts/$PIPELINERUN/download/downloaded.tgz", + "name": "download-downloaded", "path": "/tmp/results.txt"}]}' name: sidecar spec: pipelineSpec: @@ -54,10 +57,18 @@ spec: valueFrom: fieldRef: fieldPath: metadata.labels['tekton.dev/pipelineTask'] - - name: NAMESPACE + - name: ARTIFACT_ENDPOINT_SCHEME valueFrom: fieldRef: - fieldPath: metadata.namespace + fieldPath: metadata.annotations['tekton.dev/artifact_endpoint_scheme'] + - name: ARTIFACT_ENDPOINT + valueFrom: + fieldRef: + fieldPath: metadata.annotations['tekton.dev/artifact_endpoint'] + - name: ARTIFACT_BUCKET + valueFrom: + fieldRef: + fieldPath: metadata.annotations['tekton.dev/artifact_bucket'] - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: @@ -72,12 +83,12 @@ spec: name: copy-artifacts script: '#!/usr/bin/env sh - mc config host add storage http://minio-service.$NAMESPACE:9000 $AWS_ACCESS_KEY_ID - $AWS_SECRET_ACCESS_KEY + mc config host add storage ${ARTIFACT_ENDPOINT_SCHEME}${ARTIFACT_ENDPOINT} + $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY tar -cvzf downloaded.tgz $(results.downloaded.path) - mc cp downloaded.tgz storage/mlpipeline/artifacts/$PIPELINERUN/$PIPELINETASK/downloaded.tgz + mc cp downloaded.tgz storage/$ARTIFACT_BUCKET/artifacts/$PIPELINERUN/$PIPELINETASK/downloaded.tgz ' - name: echo diff --git a/sdk/python/tests/compiler/testdata/timeout.yaml b/sdk/python/tests/compiler/testdata/timeout.yaml index 3ce16c5c54..17d92523e6 100644 --- a/sdk/python/tests/compiler/testdata/timeout.yaml +++ b/sdk/python/tests/compiler/testdata/timeout.yaml @@ -19,6 +19,9 @@ metadata: pipelines.kubeflow.org/pipeline_spec: '{"description": "shows how to use ContainerOp set_timeout().", "name": "pipeline includes two steps which fail randomly."}' sidecar.istio.io/inject: 'false' + tekton.dev/artifact_bucket: mlpipeline + tekton.dev/artifact_endpoint: minio-service.kubeflow:9000 + tekton.dev/artifact_endpoint_scheme: http:// tekton.dev/input_artifacts: '{}' tekton.dev/output_artifacts: '{}' name: pipeline-includes-two-steps-which-fail-randomly diff --git a/sdk/python/tests/compiler/testdata/tolerations.yaml b/sdk/python/tests/compiler/testdata/tolerations.yaml index e7ec4ff9d3..853dddd305 100644 --- a/sdk/python/tests/compiler/testdata/tolerations.yaml +++ b/sdk/python/tests/compiler/testdata/tolerations.yaml @@ -19,9 +19,12 @@ metadata: pipelines.kubeflow.org/pipeline_spec: '{"description": "A pipeline with tolerations", "name": "tolerations"}' sidecar.istio.io/inject: 'false' + tekton.dev/artifact_bucket: mlpipeline + tekton.dev/artifact_endpoint: minio-service.kubeflow:9000 + tekton.dev/artifact_endpoint_scheme: http:// tekton.dev/input_artifacts: '{}' - tekton.dev/output_artifacts: '{"download": [{"name": "download-downloaded", "path": - "/tmp/results.txt"}]}' + tekton.dev/output_artifacts: '{"download": [{"key": "artifacts/$PIPELINERUN/download/downloaded.tgz", + "name": "download-downloaded", "path": "/tmp/results.txt"}]}' name: tolerations spec: pipelineSpec: @@ -48,10 +51,18 @@ spec: valueFrom: fieldRef: fieldPath: metadata.labels['tekton.dev/pipelineTask'] - - name: NAMESPACE + - name: ARTIFACT_ENDPOINT_SCHEME valueFrom: fieldRef: - fieldPath: metadata.namespace + fieldPath: metadata.annotations['tekton.dev/artifact_endpoint_scheme'] + - name: ARTIFACT_ENDPOINT + valueFrom: + fieldRef: + fieldPath: metadata.annotations['tekton.dev/artifact_endpoint'] + - name: ARTIFACT_BUCKET + valueFrom: + fieldRef: + fieldPath: metadata.annotations['tekton.dev/artifact_bucket'] - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: @@ -66,12 +77,12 @@ spec: name: copy-artifacts script: '#!/usr/bin/env sh - mc config host add storage http://minio-service.$NAMESPACE:9000 $AWS_ACCESS_KEY_ID - $AWS_SECRET_ACCESS_KEY + mc config host add storage ${ARTIFACT_ENDPOINT_SCHEME}${ARTIFACT_ENDPOINT} + $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY tar -cvzf downloaded.tgz $(results.downloaded.path) - mc cp downloaded.tgz storage/mlpipeline/artifacts/$PIPELINERUN/$PIPELINETASK/downloaded.tgz + mc cp downloaded.tgz storage/$ARTIFACT_BUCKET/artifacts/$PIPELINERUN/$PIPELINETASK/downloaded.tgz ' taskRunSpecs: diff --git a/sdk/python/tests/compiler/testdata/volume.yaml b/sdk/python/tests/compiler/testdata/volume.yaml index b2dc0f2008..df2001bc91 100644 --- a/sdk/python/tests/compiler/testdata/volume.yaml +++ b/sdk/python/tests/compiler/testdata/volume.yaml @@ -19,10 +19,13 @@ metadata: pipelines.kubeflow.org/pipeline_spec: '{"description": "A pipeline with volume.", "name": "Volume"}' sidecar.istio.io/inject: 'false' + tekton.dev/artifact_bucket: mlpipeline + tekton.dev/artifact_endpoint: minio-service.kubeflow:9000 + tekton.dev/artifact_endpoint_scheme: http:// tekton.dev/input_artifacts: '{"echo": [{"name": "download-downloaded", "parent_task": "download"}]}' - tekton.dev/output_artifacts: '{"download": [{"name": "download-downloaded", "path": - "/tmp/results.txt"}]}' + tekton.dev/output_artifacts: '{"download": [{"key": "artifacts/$PIPELINERUN/download/downloaded.tgz", + "name": "download-downloaded", "path": "/tmp/results.txt"}]}' name: volume spec: pipelineSpec: @@ -57,10 +60,18 @@ spec: valueFrom: fieldRef: fieldPath: metadata.labels['tekton.dev/pipelineTask'] - - name: NAMESPACE + - name: ARTIFACT_ENDPOINT_SCHEME valueFrom: fieldRef: - fieldPath: metadata.namespace + fieldPath: metadata.annotations['tekton.dev/artifact_endpoint_scheme'] + - name: ARTIFACT_ENDPOINT + valueFrom: + fieldRef: + fieldPath: metadata.annotations['tekton.dev/artifact_endpoint'] + - name: ARTIFACT_BUCKET + valueFrom: + fieldRef: + fieldPath: metadata.annotations['tekton.dev/artifact_bucket'] - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: @@ -75,12 +86,12 @@ spec: name: copy-artifacts script: '#!/usr/bin/env sh - mc config host add storage http://minio-service.$NAMESPACE:9000 $AWS_ACCESS_KEY_ID - $AWS_SECRET_ACCESS_KEY + mc config host add storage ${ARTIFACT_ENDPOINT_SCHEME}${ARTIFACT_ENDPOINT} + $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY tar -cvzf downloaded.tgz $(results.downloaded.path) - mc cp downloaded.tgz storage/mlpipeline/artifacts/$PIPELINERUN/$PIPELINETASK/downloaded.tgz + mc cp downloaded.tgz storage/$ARTIFACT_BUCKET/artifacts/$PIPELINERUN/$PIPELINETASK/downloaded.tgz ' volumes: diff --git a/sdk/python/tests/compiler/testdata/volume_op.yaml b/sdk/python/tests/compiler/testdata/volume_op.yaml index 6e0ed34df4..69c80d44ab 100644 --- a/sdk/python/tests/compiler/testdata/volume_op.yaml +++ b/sdk/python/tests/compiler/testdata/volume_op.yaml @@ -20,6 +20,9 @@ metadata: Usage.", "inputs": [{"default": "10M", "name": "size", "optional": true}], "name": "VolumeOp Basic"}' sidecar.istio.io/inject: 'false' + tekton.dev/artifact_bucket: mlpipeline + tekton.dev/artifact_endpoint: minio-service.kubeflow:9000 + tekton.dev/artifact_endpoint_scheme: http:// tekton.dev/input_artifacts: '{"cop": [{"name": "create-pvc-name", "parent_task": "create-pvc"}]}' tekton.dev/output_artifacts: '{}' diff --git a/sdk/python/tests/compiler/testdata/volume_snapshot_op.yaml b/sdk/python/tests/compiler/testdata/volume_snapshot_op.yaml index b1c3e135a2..9b9d140a36 100644 --- a/sdk/python/tests/compiler/testdata/volume_snapshot_op.yaml +++ b/sdk/python/tests/compiler/testdata/volume_snapshot_op.yaml @@ -21,6 +21,9 @@ metadata: to run this pipeline.", "inputs": [{"name": "url"}], "name": "VolumeSnapshotOp Sequential"}' sidecar.istio.io/inject: 'false' + tekton.dev/artifact_bucket: mlpipeline + tekton.dev/artifact_endpoint: minio-service.kubeflow:9000 + tekton.dev/artifact_endpoint_scheme: http:// tekton.dev/input_artifacts: '{"step1-ingest": [{"name": "create-volume-name", "parent_task": "create-volume"}], "step1-snap": [{"name": "create-volume-name", "parent_task": "create-volume"}], "step2-gunzip": [{"name": "create-volume-name", diff --git a/sdk/python/tests/compiler/testdata/withitem_nested.yaml b/sdk/python/tests/compiler/testdata/withitem_nested.yaml index 568b5e64af..dd994bc2ed 100644 --- a/sdk/python/tests/compiler/testdata/withitem_nested.yaml +++ b/sdk/python/tests/compiler/testdata/withitem_nested.yaml @@ -19,6 +19,9 @@ metadata: pipelines.kubeflow.org/pipeline_spec: '{"inputs": [{"default": "10", "name": "my_pipe_param", "optional": true, "type": "Integer"}], "name": "my-pipeline"}' sidecar.istio.io/inject: 'false' + tekton.dev/artifact_bucket: mlpipeline + tekton.dev/artifact_endpoint: minio-service.kubeflow:9000 + tekton.dev/artifact_endpoint_scheme: http:// tekton.dev/input_artifacts: '{"my-in-coop1": [{"name": "loop-item-param-00000001-subvar-a", "parent_task": null}], "my-in-coop2": [{"name": "loop-item-param-00000001-subvar-b", "parent_task": null}], "my-inner-inner-coop": [{"name": "loop-item-param-00000001-subvar-a",