diff --git a/sdk/python/tests/compiler/testdata/input_artifact_raw_value.py b/sdk/python/tests/compiler/testdata/input_artifact_raw_value.py index 0ede6cd9430..5eaa16bb3f5 100644 --- a/sdk/python/tests/compiler/testdata/input_artifact_raw_value.py +++ b/sdk/python/tests/compiler/testdata/input_artifact_raw_value.py @@ -14,31 +14,45 @@ import sys -from kfp import dsl +from kfp import dsl, components from pathlib import Path sys.path.insert(0, __file__ + '/../../../../') -def component_with_inline_input_artifact(text: str): - return dsl.ContainerOp( - name='component_with_inline_input_artifact', - image='alpine', - command=['cat', dsl.InputArgumentPath(text, path='/tmp/inputs/text/data', input='text')], # path and input are optional - ) - - -def component_with_input_artifact(text): - """A component that passes text as input artifact""" - - return dsl.ContainerOp( - name='component_with_input_artifact', - artifact_argument_paths=[ - dsl.InputArgumentPath(argument=text, path='/tmp/inputs/text/data', input='text'), # path and input are optional - ], - image='alpine', - command=['cat', '/tmp/inputs/text/data'], - ) +component_with_inline_input_artifact = components.load_component_from_text(""" +name: component_with_inline_input_artifact +description: component_with_inline_input_artifact +inputs: + - {name: text, type: String} +implementation: + container: + image: alpine + command: + - sh + - -c + args: + - | + cat $0 + - {inputPath: text} +""") + +component_with_input_artifact = components.load_component_from_text(""" +name: component_with_input_artifact +description: component_with_input_artifact +inputs: + - {name: text, type: String} +implementation: + container: + image: alpine + command: + - sh + - -c + args: + - | + cat $0 + - {inputPath: text} +""") def component_with_hardcoded_input_artifact_value(): @@ -52,7 +66,7 @@ def component_with_input_artifact_value_from_file(file_path): @dsl.pipeline( - name='Pipeline with artifact input raw argument value.', + name='pipeline-with-artifact-input-raw-argument-value', description='Pipeline shows how to define artifact inputs and pass raw artifacts to them.' ) def input_artifact_pipeline(): diff --git a/sdk/python/tests/compiler/testdata/input_artifact_raw_value.yaml b/sdk/python/tests/compiler/testdata/input_artifact_raw_value.yaml index 0c01a1b6c2b..e06005ecd14 100644 --- a/sdk/python/tests/compiler/testdata/input_artifact_raw_value.yaml +++ b/sdk/python/tests/compiler/testdata/input_artifact_raw_value.yaml @@ -27,8 +27,7 @@ metadata: []}' sidecar.istio.io/inject: "false" pipelines.kubeflow.org/pipeline_spec: '{"description": "Pipeline shows how to - define artifact inputs and pass raw artifacts to them.", "name": "Pipeline with - artifact input raw argument value."}' + define artifact inputs and pass raw artifacts to them.", "name": "pipeline-with-artifact-input-raw-argument-value"}' spec: pipelineSpec: tasks: @@ -42,9 +41,13 @@ spec: set -exo pipefail echo -n "Constant artifact value" > /tmp/inputs/text/data - name: main - command: - - cat + args: + - | + cat $0 - /tmp/inputs/text/data + command: + - sh + - -c image: alpine metadata: labels: @@ -52,6 +55,10 @@ spec: pipelines.kubeflow.org/generation: '' pipelines.kubeflow.org/cache_enabled: "true" annotations: + pipelines.kubeflow.org/component_spec: '{"description": "component_with_inline_input_artifact", + "implementation": {"container": {"args": ["cat $0\n", {"inputPath": + "text"}], "command": ["sh", "-c"], "image": "alpine"}}, "inputs": [{"name": + "text", "type": "String"}], "name": "component_with_inline_input_artifact"}' tekton.dev/template: '' stepTemplate: volumeMounts: @@ -71,9 +78,13 @@ spec: set -exo pipefail echo -n "Constant artifact value" > /tmp/inputs/text/data - name: main - command: - - cat + args: + - | + cat $0 - /tmp/inputs/text/data + command: + - sh + - -c image: alpine metadata: labels: @@ -81,6 +92,10 @@ spec: pipelines.kubeflow.org/generation: '' pipelines.kubeflow.org/cache_enabled: "true" annotations: + pipelines.kubeflow.org/component_spec: '{"description": "component_with_input_artifact", + "implementation": {"container": {"args": ["cat $0\n", {"inputPath": + "text"}], "command": ["sh", "-c"], "image": "alpine"}}, "inputs": [{"name": + "text", "type": "String"}], "name": "component_with_input_artifact"}' tekton.dev/template: '' stepTemplate: volumeMounts: @@ -100,9 +115,13 @@ spec: set -exo pipefail echo -n "hard-coded artifact value" > /tmp/inputs/text/data - name: main - command: - - cat + args: + - | + cat $0 - /tmp/inputs/text/data + command: + - sh + - -c image: alpine metadata: labels: @@ -110,6 +129,10 @@ spec: pipelines.kubeflow.org/generation: '' pipelines.kubeflow.org/cache_enabled: "true" annotations: + pipelines.kubeflow.org/component_spec: '{"description": "component_with_input_artifact", + "implementation": {"container": {"args": ["cat $0\n", {"inputPath": + "text"}], "command": ["sh", "-c"], "image": "alpine"}}, "inputs": [{"name": + "text", "type": "String"}], "name": "component_with_input_artifact"}' tekton.dev/template: '' stepTemplate: volumeMounts: @@ -130,9 +153,13 @@ spec: echo -n "Text from a file with hard-coded artifact value " > /tmp/inputs/text/data - name: main - command: - - cat + args: + - | + cat $0 - /tmp/inputs/text/data + command: + - sh + - -c image: alpine metadata: labels: @@ -140,6 +167,10 @@ spec: pipelines.kubeflow.org/generation: '' pipelines.kubeflow.org/cache_enabled: "true" annotations: + pipelines.kubeflow.org/component_spec: '{"description": "component_with_input_artifact", + "implementation": {"container": {"args": ["cat $0\n", {"inputPath": + "text"}], "command": ["sh", "-c"], "image": "alpine"}}, "inputs": [{"name": + "text", "type": "String"}], "name": "component_with_input_artifact"}' tekton.dev/template: '' stepTemplate: volumeMounts: diff --git a/sdk/python/tests/compiler/testdata/input_artifact_raw_value_noninlined.yaml b/sdk/python/tests/compiler/testdata/input_artifact_raw_value_noninlined.yaml index 0c01a1b6c2b..e06005ecd14 100644 --- a/sdk/python/tests/compiler/testdata/input_artifact_raw_value_noninlined.yaml +++ b/sdk/python/tests/compiler/testdata/input_artifact_raw_value_noninlined.yaml @@ -27,8 +27,7 @@ metadata: []}' sidecar.istio.io/inject: "false" pipelines.kubeflow.org/pipeline_spec: '{"description": "Pipeline shows how to - define artifact inputs and pass raw artifacts to them.", "name": "Pipeline with - artifact input raw argument value."}' + define artifact inputs and pass raw artifacts to them.", "name": "pipeline-with-artifact-input-raw-argument-value"}' spec: pipelineSpec: tasks: @@ -42,9 +41,13 @@ spec: set -exo pipefail echo -n "Constant artifact value" > /tmp/inputs/text/data - name: main - command: - - cat + args: + - | + cat $0 - /tmp/inputs/text/data + command: + - sh + - -c image: alpine metadata: labels: @@ -52,6 +55,10 @@ spec: pipelines.kubeflow.org/generation: '' pipelines.kubeflow.org/cache_enabled: "true" annotations: + pipelines.kubeflow.org/component_spec: '{"description": "component_with_inline_input_artifact", + "implementation": {"container": {"args": ["cat $0\n", {"inputPath": + "text"}], "command": ["sh", "-c"], "image": "alpine"}}, "inputs": [{"name": + "text", "type": "String"}], "name": "component_with_inline_input_artifact"}' tekton.dev/template: '' stepTemplate: volumeMounts: @@ -71,9 +78,13 @@ spec: set -exo pipefail echo -n "Constant artifact value" > /tmp/inputs/text/data - name: main - command: - - cat + args: + - | + cat $0 - /tmp/inputs/text/data + command: + - sh + - -c image: alpine metadata: labels: @@ -81,6 +92,10 @@ spec: pipelines.kubeflow.org/generation: '' pipelines.kubeflow.org/cache_enabled: "true" annotations: + pipelines.kubeflow.org/component_spec: '{"description": "component_with_input_artifact", + "implementation": {"container": {"args": ["cat $0\n", {"inputPath": + "text"}], "command": ["sh", "-c"], "image": "alpine"}}, "inputs": [{"name": + "text", "type": "String"}], "name": "component_with_input_artifact"}' tekton.dev/template: '' stepTemplate: volumeMounts: @@ -100,9 +115,13 @@ spec: set -exo pipefail echo -n "hard-coded artifact value" > /tmp/inputs/text/data - name: main - command: - - cat + args: + - | + cat $0 - /tmp/inputs/text/data + command: + - sh + - -c image: alpine metadata: labels: @@ -110,6 +129,10 @@ spec: pipelines.kubeflow.org/generation: '' pipelines.kubeflow.org/cache_enabled: "true" annotations: + pipelines.kubeflow.org/component_spec: '{"description": "component_with_input_artifact", + "implementation": {"container": {"args": ["cat $0\n", {"inputPath": + "text"}], "command": ["sh", "-c"], "image": "alpine"}}, "inputs": [{"name": + "text", "type": "String"}], "name": "component_with_input_artifact"}' tekton.dev/template: '' stepTemplate: volumeMounts: @@ -130,9 +153,13 @@ spec: echo -n "Text from a file with hard-coded artifact value " > /tmp/inputs/text/data - name: main - command: - - cat + args: + - | + cat $0 - /tmp/inputs/text/data + command: + - sh + - -c image: alpine metadata: labels: @@ -140,6 +167,10 @@ spec: pipelines.kubeflow.org/generation: '' pipelines.kubeflow.org/cache_enabled: "true" annotations: + pipelines.kubeflow.org/component_spec: '{"description": "component_with_input_artifact", + "implementation": {"container": {"args": ["cat $0\n", {"inputPath": + "text"}], "command": ["sh", "-c"], "image": "alpine"}}, "inputs": [{"name": + "text", "type": "String"}], "name": "component_with_input_artifact"}' tekton.dev/template: '' stepTemplate: volumeMounts: diff --git a/sdk/python/tests/compiler/testdata/pipelineparams.py b/sdk/python/tests/compiler/testdata/pipelineparams.py index b9c4f18dd9c..34ab12ae68b 100644 --- a/sdk/python/tests/compiler/testdata/pipelineparams.py +++ b/sdk/python/tests/compiler/testdata/pipelineparams.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from kfp import dsl +from kfp import dsl, components from kubernetes.client.models import V1EnvVar @@ -24,21 +24,43 @@ def pipelineparams_pipeline(tag: str = 'latest', sleep_ms: int = 10): image='hashicorp/http-echo:%s' % tag, args=['-text="hello world"'], ) + op1 = components.load_component_from_text(""" + name: download + description: download + inputs: + - {name: sleep_ms, type: Integer} + outputs: + - {name: data, type: String} + implementation: + container: + image: busy:placeholder + command: + - sh + - -c + args: + - | + sleep $0; wget localhost:5678 -O $1 + - {inputValue: sleep_ms} + - {outputPath: data} + """)(sleep_ms) + op1.container.image = "busy:%s" % tag - op1 = dsl.ContainerOp( - name='download', - image='busybox:%s' % tag, - command=['sh', '-c'], - arguments=['sleep %s; wget localhost:5678 -O /tmp/results.txt' % sleep_ms], - sidecars=[echo], - file_outputs={'downloaded_resultOutput': '/tmp/results.txt'}) - - op2 = dsl.ContainerOp( - name='echo', - image='library/bash', - command=['sh', '-c'], - arguments=['echo $MSG %s' % op1.output]) - + op2 = components.load_component_from_text(""" + name: echo + description: echo + inputs: + - {name: message, type: String} + implementation: + container: + image: library/bash + command: + - sh + - -c + args: + - | + echo $MSG $0 + - {inputValue: message} + """)(op1.output) op2.container.add_env_variable(V1EnvVar(name='MSG', value='pipelineParams: ')) diff --git a/sdk/python/tests/compiler/testdata/pipelineparams.yaml b/sdk/python/tests/compiler/testdata/pipelineparams.yaml index 8a782c4813b..bf65de9015e 100644 --- a/sdk/python/tests/compiler/testdata/pipelineparams.yaml +++ b/sdk/python/tests/compiler/testdata/pipelineparams.yaml @@ -17,15 +17,15 @@ kind: PipelineRun metadata: name: pipeline-params annotations: - tekton.dev/output_artifacts: '{"download": [{"key": "artifacts/$PIPELINERUN/download/downloaded_resultOutput.tgz", - "name": "download-downloaded_resultOutput", "path": "/tmp/results.txt"}]}' - tekton.dev/input_artifacts: '{"echo": [{"name": "download-downloaded_resultOutput", - "parent_task": "download"}]}' + tekton.dev/output_artifacts: '{"download": [{"key": "artifacts/$PIPELINERUN/download/data.tgz", + "name": "download-data", "path": "/tmp/outputs/data/data"}]}' + tekton.dev/input_artifacts: '{"echo": [{"name": "download-data", "parent_task": + "download"}]}' tekton.dev/artifact_bucket: mlpipeline tekton.dev/artifact_endpoint: minio-service.kubeflow:9000 tekton.dev/artifact_endpoint_scheme: http:// - tekton.dev/artifact_items: '{"download": [["downloaded_resultOutput", "$(results.downloaded-resultoutput.path)"]], - "echo": []}' + tekton.dev/artifact_items: '{"download": [["data", "$(results.data.path)"]], "echo": + []}' sidecar.istio.io/inject: "false" pipelines.kubeflow.org/pipeline_spec: '{"description": "A pipeline with multiple pipeline params.", "inputs": [{"default": "latest", "name": "tag", "optional": @@ -54,39 +54,44 @@ spec: steps: - name: main args: - - sleep $(inputs.params.sleep_ms); wget localhost:5678 -O $(results.downloaded-resultoutput.path) + - | + sleep $0; wget localhost:5678 -O $1 + - $(inputs.params.sleep_ms) + - $(results.data.path) command: - sh - -c - image: busybox:$(inputs.params.tag) + image: busy:$(inputs.params.tag) params: - name: sleep_ms - name: tag results: - - name: downloaded-resultoutput - description: /tmp/results.txt - sidecars: - - args: - - -text="hello world" - image: hashicorp/http-echo:$(inputs.params.tag) - name: echo + - name: data + description: /tmp/outputs/data/data metadata: labels: pipelines.kubeflow.org/pipelinename: '' pipelines.kubeflow.org/generation: '' pipelines.kubeflow.org/cache_enabled: "true" annotations: + pipelines.kubeflow.org/component_spec: '{"description": "download", "implementation": + {"container": {"args": ["sleep $0; wget localhost:5678 -O $1\n", {"inputValue": + "sleep_ms"}, {"outputPath": "data"}], "command": ["sh", "-c"], "image": + "busy:placeholder"}}, "inputs": [{"name": "sleep_ms", "type": "Integer"}], + "name": "download", "outputs": [{"name": "data", "type": "String"}]}' tekton.dev/template: '' timeout: 0s - name: echo params: - - name: download-downloaded_resultOutput - value: $(tasks.download.results.downloaded-resultoutput) + - name: download-data + value: $(tasks.download.results.data) taskSpec: steps: - name: main args: - - echo $MSG $(inputs.params.download-downloaded_resultOutput) + - | + echo $MSG $0 + - $(inputs.params.download-data) command: - sh - -c @@ -95,13 +100,17 @@ spec: value: 'pipelineParams: ' image: library/bash params: - - name: download-downloaded_resultOutput + - name: download-data metadata: labels: pipelines.kubeflow.org/pipelinename: '' pipelines.kubeflow.org/generation: '' pipelines.kubeflow.org/cache_enabled: "true" annotations: + pipelines.kubeflow.org/component_spec: '{"description": "echo", "implementation": + {"container": {"args": ["echo $MSG $0\n", {"inputValue": "message"}], + "command": ["sh", "-c"], "image": "library/bash"}}, "inputs": [{"name": + "message", "type": "String"}], "name": "echo"}' tekton.dev/template: '' timeout: 0s timeout: 0s diff --git a/sdk/python/tests/compiler/testdata/pipelineparams_noninlined.yaml b/sdk/python/tests/compiler/testdata/pipelineparams_noninlined.yaml index 8a782c4813b..bf65de9015e 100644 --- a/sdk/python/tests/compiler/testdata/pipelineparams_noninlined.yaml +++ b/sdk/python/tests/compiler/testdata/pipelineparams_noninlined.yaml @@ -17,15 +17,15 @@ kind: PipelineRun metadata: name: pipeline-params annotations: - tekton.dev/output_artifacts: '{"download": [{"key": "artifacts/$PIPELINERUN/download/downloaded_resultOutput.tgz", - "name": "download-downloaded_resultOutput", "path": "/tmp/results.txt"}]}' - tekton.dev/input_artifacts: '{"echo": [{"name": "download-downloaded_resultOutput", - "parent_task": "download"}]}' + tekton.dev/output_artifacts: '{"download": [{"key": "artifacts/$PIPELINERUN/download/data.tgz", + "name": "download-data", "path": "/tmp/outputs/data/data"}]}' + tekton.dev/input_artifacts: '{"echo": [{"name": "download-data", "parent_task": + "download"}]}' tekton.dev/artifact_bucket: mlpipeline tekton.dev/artifact_endpoint: minio-service.kubeflow:9000 tekton.dev/artifact_endpoint_scheme: http:// - tekton.dev/artifact_items: '{"download": [["downloaded_resultOutput", "$(results.downloaded-resultoutput.path)"]], - "echo": []}' + tekton.dev/artifact_items: '{"download": [["data", "$(results.data.path)"]], "echo": + []}' sidecar.istio.io/inject: "false" pipelines.kubeflow.org/pipeline_spec: '{"description": "A pipeline with multiple pipeline params.", "inputs": [{"default": "latest", "name": "tag", "optional": @@ -54,39 +54,44 @@ spec: steps: - name: main args: - - sleep $(inputs.params.sleep_ms); wget localhost:5678 -O $(results.downloaded-resultoutput.path) + - | + sleep $0; wget localhost:5678 -O $1 + - $(inputs.params.sleep_ms) + - $(results.data.path) command: - sh - -c - image: busybox:$(inputs.params.tag) + image: busy:$(inputs.params.tag) params: - name: sleep_ms - name: tag results: - - name: downloaded-resultoutput - description: /tmp/results.txt - sidecars: - - args: - - -text="hello world" - image: hashicorp/http-echo:$(inputs.params.tag) - name: echo + - name: data + description: /tmp/outputs/data/data metadata: labels: pipelines.kubeflow.org/pipelinename: '' pipelines.kubeflow.org/generation: '' pipelines.kubeflow.org/cache_enabled: "true" annotations: + pipelines.kubeflow.org/component_spec: '{"description": "download", "implementation": + {"container": {"args": ["sleep $0; wget localhost:5678 -O $1\n", {"inputValue": + "sleep_ms"}, {"outputPath": "data"}], "command": ["sh", "-c"], "image": + "busy:placeholder"}}, "inputs": [{"name": "sleep_ms", "type": "Integer"}], + "name": "download", "outputs": [{"name": "data", "type": "String"}]}' tekton.dev/template: '' timeout: 0s - name: echo params: - - name: download-downloaded_resultOutput - value: $(tasks.download.results.downloaded-resultoutput) + - name: download-data + value: $(tasks.download.results.data) taskSpec: steps: - name: main args: - - echo $MSG $(inputs.params.download-downloaded_resultOutput) + - | + echo $MSG $0 + - $(inputs.params.download-data) command: - sh - -c @@ -95,13 +100,17 @@ spec: value: 'pipelineParams: ' image: library/bash params: - - name: download-downloaded_resultOutput + - name: download-data metadata: labels: pipelines.kubeflow.org/pipelinename: '' pipelines.kubeflow.org/generation: '' pipelines.kubeflow.org/cache_enabled: "true" annotations: + pipelines.kubeflow.org/component_spec: '{"description": "echo", "implementation": + {"container": {"args": ["echo $MSG $0\n", {"inputValue": "message"}], + "command": ["sh", "-c"], "image": "library/bash"}}, "inputs": [{"name": + "message", "type": "String"}], "name": "echo"}' tekton.dev/template: '' timeout: 0s timeout: 0s