Skip to content

Commit

Permalink
Remove containerOp for pipelineparam and raw input samples (kubeflow#708
Browse files Browse the repository at this point in the history 
)

* remove containerOp for pipelineparam and raw input samples

* fix-typo
  • Loading branch information
@Tomcli
Tomcli authored Aug 23, 2021
1 parent 9fbccd6 commit 386ad23
Show file tree
Hide file tree
Showing 6 changed files with 210 additions and 94 deletions.
56 changes: 35 additions & 21 deletions sdk/python/tests/compiler/testdata/input_artifact_raw_value.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,31 +14,45 @@

import sys

from kfp import dsl
from kfp import dsl, components
from pathlib import Path

sys.path.insert(0, __file__ + '/../../../../')


def component_with_inline_input_artifact(text: str):
return dsl.ContainerOp(
name='component_with_inline_input_artifact',
image='alpine',
command=['cat', dsl.InputArgumentPath(text, path='/tmp/inputs/text/data', input='text')], # path and input are optional
)


def component_with_input_artifact(text):
"""A component that passes text as input artifact"""

return dsl.ContainerOp(
name='component_with_input_artifact',
artifact_argument_paths=[
dsl.InputArgumentPath(argument=text, path='/tmp/inputs/text/data', input='text'), # path and input are optional
],
image='alpine',
command=['cat', '/tmp/inputs/text/data'],
)
component_with_inline_input_artifact = components.load_component_from_text("""
name: component_with_inline_input_artifact
description: component_with_inline_input_artifact
inputs:
- {name: text, type: String}
implementation:
container:
image: alpine
command:
- sh
- -c
args:
- |
cat $0
- {inputPath: text}
""")

component_with_input_artifact = components.load_component_from_text("""
name: component_with_input_artifact
description: component_with_input_artifact
inputs:
- {name: text, type: String}
implementation:
container:
image: alpine
command:
- sh
- -c
args:
- |
cat $0
- {inputPath: text}
""")


def component_with_hardcoded_input_artifact_value():
Expand All @@ -52,7 +66,7 @@ def component_with_input_artifact_value_from_file(file_path):


@dsl.pipeline(
name='Pipeline with artifact input raw argument value.',
name='pipeline-with-artifact-input-raw-argument-value',
description='Pipeline shows how to define artifact inputs and pass raw artifacts to them.'
)
def input_artifact_pipeline():
Expand Down
51 changes: 41 additions & 10 deletions sdk/python/tests/compiler/testdata/input_artifact_raw_value.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,7 @@ metadata:
[]}'
sidecar.istio.io/inject: "false"
pipelines.kubeflow.org/pipeline_spec: '{"description": "Pipeline shows how to
define artifact inputs and pass raw artifacts to them.", "name": "Pipeline with
artifact input raw argument value."}'
define artifact inputs and pass raw artifacts to them.", "name": "pipeline-with-artifact-input-raw-argument-value"}'
spec:
pipelineSpec:
tasks:
Expand All @@ -42,16 +41,24 @@ spec:
set -exo pipefail
echo -n "Constant artifact value" > /tmp/inputs/text/data
- name: main
command:
- cat
args:
- |
cat $0
- /tmp/inputs/text/data
command:
- sh
- -c
image: alpine
metadata:
labels:
pipelines.kubeflow.org/pipelinename: ''
pipelines.kubeflow.org/generation: ''
pipelines.kubeflow.org/cache_enabled: "true"
annotations:
pipelines.kubeflow.org/component_spec: '{"description": "component_with_inline_input_artifact",
"implementation": {"container": {"args": ["cat $0\n", {"inputPath":
"text"}], "command": ["sh", "-c"], "image": "alpine"}}, "inputs": [{"name":
"text", "type": "String"}], "name": "component_with_inline_input_artifact"}'
tekton.dev/template: ''
stepTemplate:
volumeMounts:
Expand All @@ -71,16 +78,24 @@ spec:
set -exo pipefail
echo -n "Constant artifact value" > /tmp/inputs/text/data
- name: main
command:
- cat
args:
- |
cat $0
- /tmp/inputs/text/data
command:
- sh
- -c
image: alpine
metadata:
labels:
pipelines.kubeflow.org/pipelinename: ''
pipelines.kubeflow.org/generation: ''
pipelines.kubeflow.org/cache_enabled: "true"
annotations:
pipelines.kubeflow.org/component_spec: '{"description": "component_with_input_artifact",
"implementation": {"container": {"args": ["cat $0\n", {"inputPath":
"text"}], "command": ["sh", "-c"], "image": "alpine"}}, "inputs": [{"name":
"text", "type": "String"}], "name": "component_with_input_artifact"}'
tekton.dev/template: ''
stepTemplate:
volumeMounts:
Expand All @@ -100,16 +115,24 @@ spec:
set -exo pipefail
echo -n "hard-coded artifact value" > /tmp/inputs/text/data
- name: main
command:
- cat
args:
- |
cat $0
- /tmp/inputs/text/data
command:
- sh
- -c
image: alpine
metadata:
labels:
pipelines.kubeflow.org/pipelinename: ''
pipelines.kubeflow.org/generation: ''
pipelines.kubeflow.org/cache_enabled: "true"
annotations:
pipelines.kubeflow.org/component_spec: '{"description": "component_with_input_artifact",
"implementation": {"container": {"args": ["cat $0\n", {"inputPath":
"text"}], "command": ["sh", "-c"], "image": "alpine"}}, "inputs": [{"name":
"text", "type": "String"}], "name": "component_with_input_artifact"}'
tekton.dev/template: ''
stepTemplate:
volumeMounts:
Expand All @@ -130,16 +153,24 @@ spec:
echo -n "Text from a file with hard-coded artifact value
" > /tmp/inputs/text/data
- name: main
command:
- cat
args:
- |
cat $0
- /tmp/inputs/text/data
command:
- sh
- -c
image: alpine
metadata:
labels:
pipelines.kubeflow.org/pipelinename: ''
pipelines.kubeflow.org/generation: ''
pipelines.kubeflow.org/cache_enabled: "true"
annotations:
pipelines.kubeflow.org/component_spec: '{"description": "component_with_input_artifact",
"implementation": {"container": {"args": ["cat $0\n", {"inputPath":
"text"}], "command": ["sh", "-c"], "image": "alpine"}}, "inputs": [{"name":
"text", "type": "String"}], "name": "component_with_input_artifact"}'
tekton.dev/template: ''
stepTemplate:
volumeMounts:
Expand Down
51 changes: 41 additions & 10 deletions sdk/python/tests/compiler/testdata/input_artifact_raw_value_noninlined.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,7 @@ metadata:
[]}'
sidecar.istio.io/inject: "false"
pipelines.kubeflow.org/pipeline_spec: '{"description": "Pipeline shows how to
define artifact inputs and pass raw artifacts to them.", "name": "Pipeline with
artifact input raw argument value."}'
define artifact inputs and pass raw artifacts to them.", "name": "pipeline-with-artifact-input-raw-argument-value"}'
spec:
pipelineSpec:
tasks:
Expand All @@ -42,16 +41,24 @@ spec:
set -exo pipefail
echo -n "Constant artifact value" > /tmp/inputs/text/data
- name: main
command:
- cat
args:
- |
cat $0
- /tmp/inputs/text/data
command:
- sh
- -c
image: alpine
metadata:
labels:
pipelines.kubeflow.org/pipelinename: ''
pipelines.kubeflow.org/generation: ''
pipelines.kubeflow.org/cache_enabled: "true"
annotations:
pipelines.kubeflow.org/component_spec: '{"description": "component_with_inline_input_artifact",
"implementation": {"container": {"args": ["cat $0\n", {"inputPath":
"text"}], "command": ["sh", "-c"], "image": "alpine"}}, "inputs": [{"name":
"text", "type": "String"}], "name": "component_with_inline_input_artifact"}'
tekton.dev/template: ''
stepTemplate:
volumeMounts:
Expand All @@ -71,16 +78,24 @@ spec:
set -exo pipefail
echo -n "Constant artifact value" > /tmp/inputs/text/data
- name: main
command:
- cat
args:
- |
cat $0
- /tmp/inputs/text/data
command:
- sh
- -c
image: alpine
metadata:
labels:
pipelines.kubeflow.org/pipelinename: ''
pipelines.kubeflow.org/generation: ''
pipelines.kubeflow.org/cache_enabled: "true"
annotations:
pipelines.kubeflow.org/component_spec: '{"description": "component_with_input_artifact",
"implementation": {"container": {"args": ["cat $0\n", {"inputPath":
"text"}], "command": ["sh", "-c"], "image": "alpine"}}, "inputs": [{"name":
"text", "type": "String"}], "name": "component_with_input_artifact"}'
tekton.dev/template: ''
stepTemplate:
volumeMounts:
Expand All @@ -100,16 +115,24 @@ spec:
set -exo pipefail
echo -n "hard-coded artifact value" > /tmp/inputs/text/data
- name: main
command:
- cat
args:
- |
cat $0
- /tmp/inputs/text/data
command:
- sh
- -c
image: alpine
metadata:
labels:
pipelines.kubeflow.org/pipelinename: ''
pipelines.kubeflow.org/generation: ''
pipelines.kubeflow.org/cache_enabled: "true"
annotations:
pipelines.kubeflow.org/component_spec: '{"description": "component_with_input_artifact",
"implementation": {"container": {"args": ["cat $0\n", {"inputPath":
"text"}], "command": ["sh", "-c"], "image": "alpine"}}, "inputs": [{"name":
"text", "type": "String"}], "name": "component_with_input_artifact"}'
tekton.dev/template: ''
stepTemplate:
volumeMounts:
Expand All @@ -130,16 +153,24 @@ spec:
echo -n "Text from a file with hard-coded artifact value
" > /tmp/inputs/text/data
- name: main
command:
- cat
args:
- |
cat $0
- /tmp/inputs/text/data
command:
- sh
- -c
image: alpine
metadata:
labels:
pipelines.kubeflow.org/pipelinename: ''
pipelines.kubeflow.org/generation: ''
pipelines.kubeflow.org/cache_enabled: "true"
annotations:
pipelines.kubeflow.org/component_spec: '{"description": "component_with_input_artifact",
"implementation": {"container": {"args": ["cat $0\n", {"inputPath":
"text"}], "command": ["sh", "-c"], "image": "alpine"}}, "inputs": [{"name":
"text", "type": "String"}], "name": "component_with_input_artifact"}'
tekton.dev/template: ''
stepTemplate:
volumeMounts:
Expand Down
52 changes: 37 additions & 15 deletions sdk/python/tests/compiler/testdata/pipelineparams.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

from kfp import dsl
from kfp import dsl, components
from kubernetes.client.models import V1EnvVar


Expand All @@ -24,21 +24,43 @@ def pipelineparams_pipeline(tag: str = 'latest', sleep_ms: int = 10):
image='hashicorp/http-echo:%s' % tag,
args=['-text="hello world"'],
)
op1 = components.load_component_from_text("""
name: download
description: download
inputs:
- {name: sleep_ms, type: Integer}
outputs:
- {name: data, type: String}
implementation:
container:
image: busy:placeholder
command:
- sh
- -c
args:
- |
sleep $0; wget localhost:5678 -O $1
- {inputValue: sleep_ms}
- {outputPath: data}
""")(sleep_ms)
op1.container.image = "busy:%s" % tag

op1 = dsl.ContainerOp(
name='download',
image='busybox:%s' % tag,
command=['sh', '-c'],
arguments=['sleep %s; wget localhost:5678 -O /tmp/results.txt' % sleep_ms],
sidecars=[echo],
file_outputs={'downloaded_resultOutput': '/tmp/results.txt'})

op2 = dsl.ContainerOp(
name='echo',
image='library/bash',
command=['sh', '-c'],
arguments=['echo $MSG %s' % op1.output])

op2 = components.load_component_from_text("""
name: echo
description: echo
inputs:
- {name: message, type: String}
implementation:
container:
image: library/bash
command:
- sh
- -c
args:
- |
echo $MSG $0
- {inputValue: message}
""")(op1.output)
op2.container.add_env_variable(V1EnvVar(name='MSG', value='pipelineParams: '))


Expand Down
Loading

0 comments on commit 386ad23

Please sign in to comment.