forked from opendatahub-io/data-science-pipelines
-
Notifications
You must be signed in to change notification settings - Fork 3
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* support input artifacts * fix conflicts * fix merging bug
- Loading branch information
Showing
6 changed files
with
256 additions
and
13 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
70 changes: 70 additions & 0 deletions
70
sdk/python/tests/compiler/testdata/input_artifact_raw_value.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,70 @@ | ||
| # Copyright 2020 kubeflow.org | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
|
|
||
|
|
||
| import sys | ||
| from pathlib import Path | ||
|
|
||
| sys.path.insert(0, __file__ + '/../../../../') | ||
|
|
||
| import kfp | ||
| from kfp import dsl | ||
|
|
||
|
|
||
| def component_with_inline_input_artifact(text: str): | ||
| return dsl.ContainerOp( | ||
| name='component_with_inline_input_artifact', | ||
| image='alpine', | ||
| command=['cat', dsl.InputArgumentPath(text, path='/tmp/inputs/text/data', input='text')], # path and input are optional | ||
| ) | ||
|
|
||
|
|
||
| def component_with_input_artifact(text): | ||
| '''A component that passes text as input artifact''' | ||
|
|
||
| return dsl.ContainerOp( | ||
| name='component_with_input_artifact', | ||
| artifact_argument_paths=[ | ||
| dsl.InputArgumentPath(argument=text, path='/tmp/inputs/text/data', input='text'), # path and input are optional | ||
| ], | ||
| image='alpine', | ||
| command=['cat', '/tmp/inputs/text/data'], | ||
| ) | ||
|
|
||
| def component_with_hardcoded_input_artifact_value(): | ||
| '''A component that passes hard-coded text as input artifact''' | ||
| return component_with_input_artifact('hard-coded artifact value') | ||
|
|
||
|
|
||
| def component_with_input_artifact_value_from_file(file_path): | ||
| '''A component that passes contents of a file as input artifact''' | ||
| return component_with_input_artifact(Path(file_path).read_text()) | ||
|
|
||
|
|
||
| @dsl.pipeline( | ||
| name='Pipeline with artifact input raw argument value.', | ||
| description='Pipeline shows how to define artifact inputs and pass raw artifacts to them.' | ||
| ) | ||
| def input_artifact_pipeline(): | ||
| component_with_inline_input_artifact('Constant artifact value') | ||
| component_with_input_artifact('Constant artifact value') | ||
| component_with_hardcoded_input_artifact_value() | ||
|
|
||
| file_path = str(Path(__file__).parent.joinpath('input_artifact_raw_value.txt')) | ||
| component_with_input_artifact_value_from_file(file_path) | ||
|
|
||
| if __name__ == '__main__': | ||
| # don't use top-level import of TektonCompiler to prevent monkey-patching KFP compiler when using KFP's dsl-compile | ||
| from kfp_tekton.compiler import TektonCompiler | ||
| TektonCompiler().compile(input_artifact_pipeline, __file__.replace('.py', '.yaml')) |
1 change: 1 addition & 0 deletions
1
sdk/python/tests/compiler/testdata/input_artifact_raw_value.txt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| Text from a file with hard-coded artifact value |
142 changes: 142 additions & 0 deletions
142
sdk/python/tests/compiler/testdata/input_artifact_raw_value.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,142 @@ | ||
| # Copyright 2020 kubeflow.org | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
|
|
||
| apiVersion: tekton.dev/v1beta1 | ||
| kind: Task | ||
| metadata: | ||
| name: component-with-inline-input-artifact | ||
| spec: | ||
| stepTemplate: | ||
| volumeMounts: | ||
| - mountPath: /tmp/inputs/text | ||
| name: text | ||
| steps: | ||
| - image: busybox | ||
| name: copy-inputs | ||
| script: | | ||
| #!/bin/sh | ||
| set -exo pipefail | ||
| echo -n "Constant artifact value" > /tmp/inputs/text/data | ||
| - command: | ||
| - cat | ||
| - /tmp/inputs/text/data | ||
| image: alpine | ||
| name: component-with-inline-input-artifact | ||
| volumes: | ||
| - emptyDir: {} | ||
| name: text | ||
| --- | ||
| apiVersion: tekton.dev/v1beta1 | ||
| kind: Task | ||
| metadata: | ||
| name: component-with-input-artifact | ||
| spec: | ||
| stepTemplate: | ||
| volumeMounts: | ||
| - mountPath: /tmp/inputs/text | ||
| name: text | ||
| steps: | ||
| - image: busybox | ||
| name: copy-inputs | ||
| script: | | ||
| #!/bin/sh | ||
| set -exo pipefail | ||
| echo -n "Constant artifact value" > /tmp/inputs/text/data | ||
| - command: | ||
| - cat | ||
| - /tmp/inputs/text/data | ||
| image: alpine | ||
| name: component-with-input-artifact | ||
| volumes: | ||
| - emptyDir: {} | ||
| name: text | ||
| --- | ||
| apiVersion: tekton.dev/v1beta1 | ||
| kind: Task | ||
| metadata: | ||
| name: component-with-input-artifact-2 | ||
| spec: | ||
| stepTemplate: | ||
| volumeMounts: | ||
| - mountPath: /tmp/inputs/text | ||
| name: text | ||
| steps: | ||
| - image: busybox | ||
| name: copy-inputs | ||
| script: | | ||
| #!/bin/sh | ||
| set -exo pipefail | ||
| echo -n "hard-coded artifact value" > /tmp/inputs/text/data | ||
| - command: | ||
| - cat | ||
| - /tmp/inputs/text/data | ||
| image: alpine | ||
| name: component-with-input-artifact-2 | ||
| volumes: | ||
| - emptyDir: {} | ||
| name: text | ||
| --- | ||
| apiVersion: tekton.dev/v1beta1 | ||
| kind: Task | ||
| metadata: | ||
| name: component-with-input-artifact-3 | ||
| spec: | ||
| stepTemplate: | ||
| volumeMounts: | ||
| - mountPath: /tmp/inputs/text | ||
| name: text | ||
| steps: | ||
| - image: busybox | ||
| name: copy-inputs | ||
| script: | | ||
| #!/bin/sh | ||
| set -exo pipefail | ||
| echo -n "Text from a file with hard-coded artifact value | ||
| " > /tmp/inputs/text/data | ||
| - command: | ||
| - cat | ||
| - /tmp/inputs/text/data | ||
| image: alpine | ||
| name: component-with-input-artifact-3 | ||
| volumes: | ||
| - emptyDir: {} | ||
| name: text | ||
| --- | ||
| apiVersion: tekton.dev/v1beta1 | ||
| kind: Pipeline | ||
| metadata: | ||
| annotations: | ||
| pipelines.kubeflow.org/pipeline_spec: '{"description": "Pipeline shows how to | ||
| define artifact inputs and pass raw artifacts to them.", "name": "Pipeline with | ||
| artifact input raw argument value."}' | ||
| name: pipeline-with-artifact-input-raw-argument-value | ||
| spec: | ||
| params: [] | ||
| tasks: | ||
| - name: component-with-inline-input-artifact | ||
| params: [] | ||
| taskRef: | ||
| name: component-with-inline-input-artifact | ||
| - name: component-with-input-artifact | ||
| params: [] | ||
| taskRef: | ||
| name: component-with-input-artifact | ||
| - name: component-with-input-artifact-2 | ||
| params: [] | ||
| taskRef: | ||
| name: component-with-input-artifact-2 | ||
| - name: component-with-input-artifact-3 | ||
| params: [] | ||
| taskRef: | ||
| name: component-with-input-artifact-3 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters