From 22b7b99a8bff9278df4f7d051ffaca85837688e2 Mon Sep 17 00:00:00 2001 From: Victor Date: Thu, 27 Aug 2020 09:03:53 -0700 Subject: [PATCH] fix(sdk): Fix opsgroups dependency resolution (#4370) --- sdk/python/kfp/compiler/compiler.py | 4 +- sdk/python/tests/compiler/compiler_tests.py | 4 + .../tests/compiler/testdata/opsgroups.py | 42 +++++++++++ .../tests/compiler/testdata/opsgroups.yaml | 73 +++++++++++++++++++ 4 files changed, 121 insertions(+), 2 deletions(-) create mode 100644 sdk/python/tests/compiler/testdata/opsgroups.py create mode 100644 sdk/python/tests/compiler/testdata/opsgroups.yaml diff --git a/sdk/python/kfp/compiler/compiler.py b/sdk/python/kfp/compiler/compiler.py index b7ca2c53684..d81b4ab8bc1 100644 --- a/sdk/python/kfp/compiler/compiler.py +++ b/sdk/python/kfp/compiler/compiler.py @@ -367,8 +367,8 @@ def _get_dependency_opsgroup(group, dependencies): for op_name in upstream_op_names: if op_name in pipeline.ops: upstream_op = pipeline.ops[op_name] - elif op_name in opsgroups_groups: - upstream_op = opsgroups_groups[op_name] + elif op_name in opsgroups: + upstream_op = opsgroups[op_name] else: raise ValueError('compiler cannot find the ' + op_name) upstream_groups, downstream_groups = \ diff --git a/sdk/python/tests/compiler/compiler_tests.py b/sdk/python/tests/compiler/compiler_tests.py index ca0b541edfe..2c21f8c6a0e 100644 --- a/sdk/python/tests/compiler/compiler_tests.py +++ b/sdk/python/tests/compiler/compiler_tests.py @@ -355,6 +355,10 @@ def test_py_compile_with_pipelineparams(self): """Test pipeline with multiple pipeline params.""" self._test_py_compile_yaml('pipelineparams') + def test_py_compile_with_opsgroups(self): + """Test pipeline with multiple opsgroups.""" + self._test_py_compile_yaml('opsgroups') + def test_py_compile_condition(self): """Test a pipeline with conditions.""" self._test_py_compile_zip('coin') diff --git a/sdk/python/tests/compiler/testdata/opsgroups.py b/sdk/python/tests/compiler/testdata/opsgroups.py new file mode 100644 index 00000000000..9af194e14dd --- /dev/null +++ b/sdk/python/tests/compiler/testdata/opsgroups.py @@ -0,0 +1,42 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import kfp +import kfp.dsl as dsl + + +@dsl.graph_component +def echo1_graph_component(text1): + dsl.ContainerOp( + name='echo1-task1', + image='library/bash:4.4.23', + command=['sh', '-c'], + arguments=['echo "$0"', text1]) + + +@dsl.graph_component +def echo2_graph_component(text2): + dsl.ContainerOp( + name='echo2-task1', + image='library/bash:4.4.23', + command=['sh', '-c'], + arguments=['echo "$0"', text2]) + + +@dsl.pipeline() +def opsgroups_pipeline(text1='message 1', text2='message 2'): + step1_graph_component = echo1_graph_component(text1) + step2_graph_component = echo2_graph_component(text2) + step2_graph_component.after(step1_graph_component) diff --git a/sdk/python/tests/compiler/testdata/opsgroups.yaml b/sdk/python/tests/compiler/testdata/opsgroups.yaml new file mode 100644 index 00000000000..8156934eaf9 --- /dev/null +++ b/sdk/python/tests/compiler/testdata/opsgroups.yaml @@ -0,0 +1,73 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: opsgroups-pipeline- + annotations: {pipelines.kubeflow.org/kfp_sdk_version: 1.0.0, pipelines.kubeflow.org/pipeline_compilation_time: '2020-08-13T11:25:18.232372', + pipelines.kubeflow.org/pipeline_spec: '{"inputs": [{"default": "message 1", "name": + "text1", "optional": true}, {"default": "message 2", "name": "text2", "optional": + true}], "name": "Execution order pipeline"}'} + labels: {pipelines.kubeflow.org/kfp_sdk_version: 1.0.0} +spec: + entrypoint: opsgroups-pipeline + templates: + - name: echo1-task1 + container: + args: [echo "$0", '{{inputs.parameters.text1}}'] + command: [sh, -c] + image: library/bash:4.4.23 + inputs: + parameters: + - {name: text1} + - name: echo2-task1 + container: + args: [echo "$0", '{{inputs.parameters.text2}}'] + command: [sh, -c] + image: library/bash:4.4.23 + inputs: + parameters: + - {name: text2} + - name: graph-echo1-graph-component-1 + inputs: + parameters: + - {name: text1} + dag: + tasks: + - name: echo1-task1 + template: echo1-task1 + arguments: + parameters: + - {name: text1, value: '{{inputs.parameters.text1}}'} + - name: graph-echo2-graph-component-2 + inputs: + parameters: + - {name: text2} + dag: + tasks: + - name: echo2-task1 + template: echo2-task1 + arguments: + parameters: + - {name: text2, value: '{{inputs.parameters.text2}}'} + - name: opsgroups-pipeline + inputs: + parameters: + - {name: text1} + - {name: text2} + dag: + tasks: + - name: graph-echo1-graph-component-1 + template: graph-echo1-graph-component-1 + arguments: + parameters: + - {name: text1, value: '{{inputs.parameters.text1}}'} + - name: graph-echo2-graph-component-2 + template: graph-echo2-graph-component-2 + dependencies: [graph-echo1-graph-component-1] + arguments: + parameters: + - {name: text2, value: '{{inputs.parameters.text2}}'} + arguments: + parameters: + - {name: text1, value: message 1} + - {name: text2, value: message 2} + serviceAccountName: pipeline-runner