From 7e4e7cd9b0fd13add1fefbdcbee1f1456aaa7eda Mon Sep 17 00:00:00 2001 From: numerology Date: Wed, 20 Nov 2019 13:30:47 -0800 Subject: [PATCH 1/8] update --- samples/core/loop_static/loop_static.py | 6 +++--- sdk/python/kfp/compiler/_k8s_helper.py | 2 +- sdk/python/kfp/dsl/_pipeline_param.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/samples/core/loop_static/loop_static.py b/samples/core/loop_static/loop_static.py index ed7dd915ca2..3691a872dbe 100644 --- a/samples/core/loop_static/loop_static.py +++ b/samples/core/loop_static/loop_static.py @@ -18,20 +18,20 @@ @dsl.pipeline(name='my-pipeline') def pipeline(my_pipe_param=10): - loop_args = [{'a': 1, 'b': 2}, {'a': 10, 'b': 20}] + loop_args = [{'A_a': 1, 'B_b': 2}, {'A_a': 10, 'B_b': 20}] with dsl.ParallelFor(loop_args) as item: op1 = dsl.ContainerOp( name="my-in-coop1", image="library/bash:4.4.23", command=["sh", "-c"], - arguments=["echo op1 %s %s" % (item.a, my_pipe_param)], + arguments=["echo op1 %s %s" % (item.A_a, my_pipe_param)], ) op2 = dsl.ContainerOp( name="my-in-coop2", image="library/bash:4.4.23", command=["sh", "-c"], - arguments=["echo op2 %s" % item.b], + arguments=["echo op2 %s" % item.B_b], ) op_out = dsl.ContainerOp( diff --git a/sdk/python/kfp/compiler/_k8s_helper.py b/sdk/python/kfp/compiler/_k8s_helper.py index 5a917fc522f..1938247d174 100644 --- a/sdk/python/kfp/compiler/_k8s_helper.py +++ b/sdk/python/kfp/compiler/_k8s_helper.py @@ -21,7 +21,7 @@ def sanitize_k8s_name(name): """From _make_kubernetes_name sanitize_k8s_name cleans and converts the names in the workflow. """ - return re.sub('-+', '-', re.sub('[^-0-9a-z]+', '-', name.lower())).lstrip('-').rstrip('-') + return re.sub('-+', '-', re.sub('[^-_0-9A-Za-z]+', '-', name)).lstrip('-').rstrip('-') def convert_k8s_obj_to_json(k8s_obj): diff --git a/sdk/python/kfp/dsl/_pipeline_param.py b/sdk/python/kfp/dsl/_pipeline_param.py index 380e080fe95..1f72791e72b 100644 --- a/sdk/python/kfp/dsl/_pipeline_param.py +++ b/sdk/python/kfp/dsl/_pipeline_param.py @@ -26,7 +26,7 @@ def sanitize_k8s_name(name): """From _make_kubernetes_name sanitize_k8s_name cleans and converts the names in the workflow. """ - return re.sub('-+', '-', re.sub('[^-0-9a-z]+', '-', name.lower())).lstrip('-').rstrip('-') + return re.sub('-+', '-', re.sub('[^-_0-9A-Za-z]+', '-', name)).lstrip('-').rstrip('-') def match_serialized_pipelineparam(payload: str): From 10346c29cabfca3561df6765e7fa93f942f00400 Mon Sep 17 00:00:00 2001 From: numerology Date: Wed, 20 Nov 2019 14:51:19 -0800 Subject: [PATCH 2/8] add allow_capital --- sdk/python/kfp/compiler/_k8s_helper.py | 7 +++++-- sdk/python/kfp/compiler/compiler.py | 6 +++--- sdk/python/kfp/dsl/_pipeline_param.py | 7 +++++-- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/sdk/python/kfp/compiler/_k8s_helper.py b/sdk/python/kfp/compiler/_k8s_helper.py index 1938247d174..e8d72e99b6c 100644 --- a/sdk/python/kfp/compiler/_k8s_helper.py +++ b/sdk/python/kfp/compiler/_k8s_helper.py @@ -17,11 +17,14 @@ from .. import dsl -def sanitize_k8s_name(name): +def sanitize_k8s_name(name, allow_capital=False): """From _make_kubernetes_name sanitize_k8s_name cleans and converts the names in the workflow. """ - return re.sub('-+', '-', re.sub('[^-_0-9A-Za-z]+', '-', name)).lstrip('-').rstrip('-') + if allow_capital: + return re.sub('-+', '-', re.sub('[^-_0-9A-Za-z]+', '-', name)).lstrip('-').rstrip('-') + else: + return re.sub('-+', '-', re.sub('[^-_0-9a-z]+', '-', name.lower())).lstrip('-').rstrip('-') def convert_k8s_obj_to_json(k8s_obj): diff --git a/sdk/python/kfp/compiler/compiler.py b/sdk/python/kfp/compiler/compiler.py index f15c2ff3387..ff876838e92 100644 --- a/sdk/python/kfp/compiler/compiler.py +++ b/sdk/python/kfp/compiler/compiler.py @@ -494,7 +494,7 @@ def _group_to_dag_template(self, group, inputs, outputs, dependencies): for argument_set in loop_tasks: c_dict = {} for k, v in argument_set.items(): - c_dict[sanitize_k8s_name(k)] = v + c_dict[sanitize_k8s_name(k, True)] = v sanitized_tasks.append(c_dict) else: sanitized_tasks = loop_tasks @@ -757,7 +757,7 @@ def _create_workflow(self, if arg_name == input.name: arg_type = input.type break - args_list.append(dsl.PipelineParam(sanitize_k8s_name(arg_name), param_type=arg_type)) + args_list.append(dsl.PipelineParam(sanitize_k8s_name(arg_name, True), param_type=arg_type)) with dsl.Pipeline(pipeline_name) as dsl_pipeline: pipeline_func(*args_list) @@ -770,7 +770,7 @@ def _create_workflow(self, # Fill in the default values. args_list_with_defaults = [] if pipeline_meta.inputs: - args_list_with_defaults = [dsl.PipelineParam(sanitize_k8s_name(arg_name)) + args_list_with_defaults = [dsl.PipelineParam(sanitize_k8s_name(arg_name, True)) for arg_name in argspec.args] if argspec.defaults: for arg, default in zip(reversed(args_list_with_defaults), reversed(argspec.defaults)): diff --git a/sdk/python/kfp/dsl/_pipeline_param.py b/sdk/python/kfp/dsl/_pipeline_param.py index 1f72791e72b..982d95a4429 100644 --- a/sdk/python/kfp/dsl/_pipeline_param.py +++ b/sdk/python/kfp/dsl/_pipeline_param.py @@ -22,11 +22,14 @@ PipelineParamTuple = namedtuple('PipelineParamTuple', 'name op pattern') -def sanitize_k8s_name(name): +def sanitize_k8s_name(name, allow_capital=True): """From _make_kubernetes_name sanitize_k8s_name cleans and converts the names in the workflow. """ - return re.sub('-+', '-', re.sub('[^-_0-9A-Za-z]+', '-', name)).lstrip('-').rstrip('-') + if allow_capital: + return re.sub('-+', '-', re.sub('[^-_0-9A-Za-z]+', '-', name)).lstrip('-').rstrip('-') + else: + return re.sub('-+', '-', re.sub('[^-_0-9a-z]+', '-', name.lower())).lstrip('-').rstrip('-') def match_serialized_pipelineparam(payload: str): From 2571300e8977b06084cdd0223934ac96c36b96d4 Mon Sep 17 00:00:00 2001 From: numerology Date: Wed, 20 Nov 2019 14:53:01 -0800 Subject: [PATCH 3/8] fix --- sdk/python/kfp/dsl/_pipeline_param.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/python/kfp/dsl/_pipeline_param.py b/sdk/python/kfp/dsl/_pipeline_param.py index 982d95a4429..129b7954a86 100644 --- a/sdk/python/kfp/dsl/_pipeline_param.py +++ b/sdk/python/kfp/dsl/_pipeline_param.py @@ -22,7 +22,7 @@ PipelineParamTuple = namedtuple('PipelineParamTuple', 'name op pattern') -def sanitize_k8s_name(name, allow_capital=True): +def sanitize_k8s_name(name, allow_capital=False): """From _make_kubernetes_name sanitize_k8s_name cleans and converts the names in the workflow. """ @@ -45,7 +45,7 @@ def match_serialized_pipelineparam(payload: str): for match in matches: pattern = '{{pipelineparam:op=%s;name=%s}}' % (match[0], match[1]) param_tuples.append(PipelineParamTuple( - name=sanitize_k8s_name(match[1]), + name=sanitize_k8s_name(match[1], True), op=sanitize_k8s_name(match[0]), pattern=pattern)) return param_tuples From 8141bb993a41b39c257625c543022d4e70d22866 Mon Sep 17 00:00:00 2001 From: numerology Date: Wed, 20 Nov 2019 15:25:23 -0800 Subject: [PATCH 4/8] fix volume_ops sample --- samples/core/volume_ops/volume_ops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/core/volume_ops/volume_ops.py b/samples/core/volume_ops/volume_ops.py index bd900282f01..d533ae7d898 100644 --- a/samples/core/volume_ops/volume_ops.py +++ b/samples/core/volume_ops/volume_ops.py @@ -22,7 +22,7 @@ ) def volumeop_basic(size): vop = dsl.VolumeOp( - name="create_pvc", + name="create-pvc", resource_name="my-pvc", modes=dsl.VOLUME_MODE_RWO, size=size From a129eb41afdc6efe2bb92b6ae39cd6630dd74afe Mon Sep 17 00:00:00 2001 From: numerology Date: Wed, 20 Nov 2019 16:44:32 -0800 Subject: [PATCH 5/8] fix pipeline name sanitization --- sdk/python/kfp/compiler/_k8s_helper.py | 14 +++++++++++--- sdk/python/kfp/dsl/_pipeline_param.py | 24 ++++++++++++++++-------- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/sdk/python/kfp/compiler/_k8s_helper.py b/sdk/python/kfp/compiler/_k8s_helper.py index e8d72e99b6c..8992047d8e4 100644 --- a/sdk/python/kfp/compiler/_k8s_helper.py +++ b/sdk/python/kfp/compiler/_k8s_helper.py @@ -17,14 +17,22 @@ from .. import dsl -def sanitize_k8s_name(name, allow_capital=False): +def sanitize_k8s_name(name, allow_capital_underscore=False): """From _make_kubernetes_name sanitize_k8s_name cleans and converts the names in the workflow. + + Args: + name: original name, + allow_capital_underscore: whether to allow capital letter and underscore + in this name. + + Returns: + sanitized name. """ - if allow_capital: + if allow_capital_underscore: return re.sub('-+', '-', re.sub('[^-_0-9A-Za-z]+', '-', name)).lstrip('-').rstrip('-') else: - return re.sub('-+', '-', re.sub('[^-_0-9a-z]+', '-', name.lower())).lstrip('-').rstrip('-') + return re.sub('-+', '-', re.sub('[^-0-9a-z]+', '-', name.lower())).lstrip('-').rstrip('-') def convert_k8s_obj_to_json(k8s_obj): diff --git a/sdk/python/kfp/dsl/_pipeline_param.py b/sdk/python/kfp/dsl/_pipeline_param.py index 129b7954a86..0103fad47fd 100644 --- a/sdk/python/kfp/dsl/_pipeline_param.py +++ b/sdk/python/kfp/dsl/_pipeline_param.py @@ -22,14 +22,22 @@ PipelineParamTuple = namedtuple('PipelineParamTuple', 'name op pattern') -def sanitize_k8s_name(name, allow_capital=False): - """From _make_kubernetes_name - sanitize_k8s_name cleans and converts the names in the workflow. - """ - if allow_capital: - return re.sub('-+', '-', re.sub('[^-_0-9A-Za-z]+', '-', name)).lstrip('-').rstrip('-') - else: - return re.sub('-+', '-', re.sub('[^-_0-9a-z]+', '-', name.lower())).lstrip('-').rstrip('-') +def sanitize_k8s_name(name, allow_capital_underscore=False): + """From _make_kubernetes_name + sanitize_k8s_name cleans and converts the names in the workflow. + + Args: + name: original name, + allow_capital_underscore: whether to allow capital letter and underscore + in this name. + + Returns: + sanitized name. + """ + if allow_capital_underscore: + return re.sub('-+', '-', re.sub('[^-_0-9A-Za-z]+', '-', name)).lstrip('-').rstrip('-') + else: + return re.sub('-+', '-', re.sub('[^-0-9a-z]+', '-', name.lower())).lstrip('-').rstrip('-') def match_serialized_pipelineparam(payload: str): From 3b992adf527c70d669498f58761eab6b60642985 Mon Sep 17 00:00:00 2001 From: numerology Date: Wed, 20 Nov 2019 17:28:49 -0800 Subject: [PATCH 6/8] fix unittests --- .../compiler/testdata/pipelineparams.yaml | 12 +++--- .../compiler/testdata/recursive_while.yaml | 2 +- .../testdata/volume_snapshotop_rokurl.yaml | 12 +++--- .../compiler/testdata/withitem_basic.yaml | 26 ++++++------ .../compiler/testdata/withitem_nested.yaml | 40 +++++++++---------- .../compiler/testdata/withparam_global.yaml | 16 ++++---- .../testdata/withparam_global_dict.yaml | 16 ++++---- 7 files changed, 62 insertions(+), 62 deletions(-) diff --git a/sdk/python/tests/compiler/testdata/pipelineparams.yaml b/sdk/python/tests/compiler/testdata/pipelineparams.yaml index 859d228b01c..174fdeb1493 100644 --- a/sdk/python/tests/compiler/testdata/pipelineparams.yaml +++ b/sdk/python/tests/compiler/testdata/pipelineparams.yaml @@ -25,18 +25,18 @@ spec: parameters: - name: tag value: latest - - name: sleep-ms + - name: sleep_ms value: '10' templates: - name: download inputs: parameters: - - name: sleep-ms + - name: sleep_ms - name: tag container: image: busybox:{{inputs.parameters.tag}} args: - - sleep {{inputs.parameters.sleep-ms}}; wget localhost:5678 -O /tmp/results.txt + - sleep {{inputs.parameters.sleep_ms}}; wget localhost:5678 -O /tmp/results.txt command: - sh - "-c" @@ -70,15 +70,15 @@ spec: - name: pipelineparams inputs: parameters: - - name: sleep-ms + - name: sleep_ms - name: tag dag: tasks: - name: download arguments: parameters: - - name: sleep-ms - value: "{{inputs.parameters.sleep-ms}}" + - name: sleep_ms + value: "{{inputs.parameters.sleep_ms}}" - name: tag value: "{{inputs.parameters.tag}}" template: download diff --git a/sdk/python/tests/compiler/testdata/recursive_while.yaml b/sdk/python/tests/compiler/testdata/recursive_while.yaml index 7338ef8ae9c..eeed7c8ee74 100644 --- a/sdk/python/tests/compiler/testdata/recursive_while.yaml +++ b/sdk/python/tests/compiler/testdata/recursive_while.yaml @@ -8,7 +8,7 @@ metadata: spec: arguments: parameters: - - name: maxval + - name: maxVal value: '12' entrypoint: pipeline-flip-coin serviceAccountName: pipeline-runner diff --git a/sdk/python/tests/compiler/testdata/volume_snapshotop_rokurl.yaml b/sdk/python/tests/compiler/testdata/volume_snapshotop_rokurl.yaml index deb31c4341c..2d9e6570671 100644 --- a/sdk/python/tests/compiler/testdata/volume_snapshotop_rokurl.yaml +++ b/sdk/python/tests/compiler/testdata/volume_snapshotop_rokurl.yaml @@ -8,7 +8,7 @@ metadata: spec: arguments: parameters: - - name: rok-url + - name: rok_url entrypoint: volumesnapshotop-rokurl serviceAccountName: pipeline-runner templates: @@ -56,7 +56,7 @@ spec: successCondition: status.readyToUse == true - inputs: parameters: - - name: rok-url + - name: rok_url name: create-volume-1 outputs: parameters: @@ -72,7 +72,7 @@ spec: resource: action: create manifest: "apiVersion: v1\nkind: PersistentVolumeClaim\nmetadata:\n annotations:\n\ - \ rok/origin: '{{inputs.parameters.rok-url}}'\n name: '{{workflow.name}}-vol1'\n\ + \ rok/origin: '{{inputs.parameters.rok_url}}'\n name: '{{workflow.name}}-vol1'\n\ spec:\n accessModes:\n - ReadWriteMany\n resources:\n requests:\n \ \ storage: 1Gi\n" - inputs: @@ -192,8 +192,8 @@ spec: template: create-snapshot-2 - arguments: parameters: - - name: rok-url - value: '{{inputs.parameters.rok-url}}' + - name: rok_url + value: '{{inputs.parameters.rok_url}}' name: create-volume-1 template: create-volume-1 - arguments: @@ -242,5 +242,5 @@ spec: template: step3-output inputs: parameters: - - name: rok-url + - name: rok_url name: volumesnapshotop-rokurl diff --git a/sdk/python/tests/compiler/testdata/withitem_basic.yaml b/sdk/python/tests/compiler/testdata/withitem_basic.yaml index 01a8178aab3..f19512bbd2e 100644 --- a/sdk/python/tests/compiler/testdata/withitem_basic.yaml +++ b/sdk/python/tests/compiler/testdata/withitem_basic.yaml @@ -8,7 +8,7 @@ metadata: spec: arguments: parameters: - - name: my-pipe-param + - name: my_pipe_param value: '10' entrypoint: my-pipeline serviceAccountName: pipeline-runner @@ -19,8 +19,8 @@ spec: parameters: - name: loop-item-param-00000001-subvar-a value: '{{inputs.parameters.loop-item-param-00000001-subvar-a}}' - - name: my-pipe-param - value: '{{inputs.parameters.my-pipe-param}}' + - name: my_pipe_param + value: '{{inputs.parameters.my_pipe_param}}' name: my-in-coop1 template: my-in-coop1 - arguments: @@ -33,11 +33,11 @@ spec: parameters: - name: loop-item-param-00000001-subvar-a - name: loop-item-param-00000001-subvar-b - - name: my-pipe-param + - name: my_pipe_param name: for-loop-for-loop-00000001-1 - container: args: - - echo op1 {{inputs.parameters.loop-item-param-00000001-subvar-a}} {{inputs.parameters.my-pipe-param}} + - echo op1 {{inputs.parameters.loop-item-param-00000001-subvar-a}} {{inputs.parameters.my_pipe_param}} command: - sh - -c @@ -45,7 +45,7 @@ spec: inputs: parameters: - name: loop-item-param-00000001-subvar-a - - name: my-pipe-param + - name: my_pipe_param name: my-in-coop1 - container: args: @@ -60,14 +60,14 @@ spec: name: my-in-coop2 - container: args: - - echo {{inputs.parameters.my-pipe-param}} + - echo {{inputs.parameters.my_pipe_param}} command: - sh - -c image: library/bash:4.4.23 inputs: parameters: - - name: my-pipe-param + - name: my_pipe_param name: my-out-cop - dag: tasks: @@ -77,8 +77,8 @@ spec: value: '{{item.a}}' - name: loop-item-param-00000001-subvar-b value: '{{item.b}}' - - name: my-pipe-param - value: '{{inputs.parameters.my-pipe-param}}' + - name: my_pipe_param + value: '{{inputs.parameters.my_pipe_param}}' name: for-loop-for-loop-00000001-1 template: for-loop-for-loop-00000001-1 withItems: @@ -88,11 +88,11 @@ spec: b: 20 - arguments: parameters: - - name: my-pipe-param - value: '{{inputs.parameters.my-pipe-param}}' + - name: my_pipe_param + value: '{{inputs.parameters.my_pipe_param}}' name: my-out-cop template: my-out-cop inputs: parameters: - - name: my-pipe-param + - name: my_pipe_param name: my-pipeline diff --git a/sdk/python/tests/compiler/testdata/withitem_nested.yaml b/sdk/python/tests/compiler/testdata/withitem_nested.yaml index fbfba671d95..b1f8bed1be8 100644 --- a/sdk/python/tests/compiler/testdata/withitem_nested.yaml +++ b/sdk/python/tests/compiler/testdata/withitem_nested.yaml @@ -8,7 +8,7 @@ metadata: spec: arguments: parameters: - - name: my-pipe-param + - name: my_pipe_param value: '10' entrypoint: my-pipeline serviceAccountName: pipeline-runner @@ -21,8 +21,8 @@ spec: value: '{{inputs.parameters.loop-item-param-00000001-subvar-a}}' - name: loop-item-param-00000002 value: '{{item}}' - - name: my-pipe-param - value: '{{inputs.parameters.my-pipe-param}}' + - name: my_pipe_param + value: '{{inputs.parameters.my_pipe_param}}' name: for-loop-for-loop-00000002-2 template: for-loop-for-loop-00000002-2 withItems: @@ -33,8 +33,8 @@ spec: parameters: - name: loop-item-param-00000001-subvar-a value: '{{inputs.parameters.loop-item-param-00000001-subvar-a}}' - - name: my-pipe-param - value: '{{inputs.parameters.my-pipe-param}}' + - name: my_pipe_param + value: '{{inputs.parameters.my_pipe_param}}' name: my-in-coop1 template: my-in-coop1 - arguments: @@ -47,7 +47,7 @@ spec: parameters: - name: loop-item-param-00000001-subvar-a - name: loop-item-param-00000001-subvar-b - - name: my-pipe-param + - name: my_pipe_param name: for-loop-for-loop-00000001-1 - dag: tasks: @@ -57,19 +57,19 @@ spec: value: '{{inputs.parameters.loop-item-param-00000001-subvar-a}}' - name: loop-item-param-00000002 value: '{{inputs.parameters.loop-item-param-00000002}}' - - name: my-pipe-param - value: '{{inputs.parameters.my-pipe-param}}' + - name: my_pipe_param + value: '{{inputs.parameters.my_pipe_param}}' name: my-inner-inner-coop template: my-inner-inner-coop inputs: parameters: - name: loop-item-param-00000001-subvar-a - name: loop-item-param-00000002 - - name: my-pipe-param + - name: my_pipe_param name: for-loop-for-loop-00000002-2 - container: args: - - echo op1 {{inputs.parameters.loop-item-param-00000001-subvar-a}} {{inputs.parameters.my-pipe-param}} + - echo op1 {{inputs.parameters.loop-item-param-00000001-subvar-a}} {{inputs.parameters.my_pipe_param}} command: - sh - -c @@ -77,7 +77,7 @@ spec: inputs: parameters: - name: loop-item-param-00000001-subvar-a - - name: my-pipe-param + - name: my_pipe_param name: my-in-coop1 - container: args: @@ -93,7 +93,7 @@ spec: - container: args: - echo op1 {{inputs.parameters.loop-item-param-00000001-subvar-a}} {{inputs.parameters.loop-item-param-00000002}} - {{inputs.parameters.my-pipe-param}} + {{inputs.parameters.my_pipe_param}} command: - sh - -c @@ -102,18 +102,18 @@ spec: parameters: - name: loop-item-param-00000001-subvar-a - name: loop-item-param-00000002 - - name: my-pipe-param + - name: my_pipe_param name: my-inner-inner-coop - container: args: - - echo {{inputs.parameters.my-pipe-param}} + - echo {{inputs.parameters.my_pipe_param}} command: - sh - -c image: library/bash:4.4.23 inputs: parameters: - - name: my-pipe-param + - name: my_pipe_param name: my-out-cop - dag: tasks: @@ -123,8 +123,8 @@ spec: value: '{{item.a}}' - name: loop-item-param-00000001-subvar-b value: '{{item.b}}' - - name: my-pipe-param - value: '{{inputs.parameters.my-pipe-param}}' + - name: my_pipe_param + value: '{{inputs.parameters.my_pipe_param}}' name: for-loop-for-loop-00000001-1 template: for-loop-for-loop-00000001-1 withItems: @@ -134,11 +134,11 @@ spec: b: 20 - arguments: parameters: - - name: my-pipe-param - value: '{{inputs.parameters.my-pipe-param}}' + - name: my_pipe_param + value: '{{inputs.parameters.my_pipe_param}}' name: my-out-cop template: my-out-cop inputs: parameters: - - name: my-pipe-param + - name: my_pipe_param name: my-pipeline \ No newline at end of file diff --git a/sdk/python/tests/compiler/testdata/withparam_global.yaml b/sdk/python/tests/compiler/testdata/withparam_global.yaml index 3911119d18e..d45d9527317 100644 --- a/sdk/python/tests/compiler/testdata/withparam_global.yaml +++ b/sdk/python/tests/compiler/testdata/withparam_global.yaml @@ -8,7 +8,7 @@ metadata: spec: arguments: parameters: - - name: loopidy-doop + - name: loopidy_doop value: '[3, 5, 7, 9]' entrypoint: my-pipeline serviceAccountName: pipeline-runner @@ -17,24 +17,24 @@ spec: tasks: - arguments: parameters: - - name: loopidy-doop - value: '{{inputs.parameters.loopidy-doop}}' + - name: loopidy_doop + value: '{{inputs.parameters.loopidy_doop}}' name: my-in-cop1 template: my-in-cop1 inputs: parameters: - - name: loopidy-doop + - name: loopidy_doop name: for-loop-for-loop-00000001-1 - container: args: - - 'echo no output global op1, item: {{inputs.parameters.loopidy-doop}}' + - 'echo no output global op1, item: {{inputs.parameters.loopidy_doop}}' command: - sh - -c image: library/bash:4.4.23 inputs: parameters: - - name: loopidy-doop + - name: loopidy_doop name: my-in-cop1 - container: args: @@ -68,13 +68,13 @@ spec: tasks: - arguments: parameters: - - name: loopidy-doop + - name: loopidy_doop value: '{{item}}' dependencies: - my-out-cop0 name: for-loop-for-loop-00000001-1 template: for-loop-for-loop-00000001-1 - withParam: '{{workflow.parameters.loopidy-doop}}' + withParam: '{{workflow.parameters.loopidy_doop}}' - name: my-out-cop0 template: my-out-cop0 - arguments: diff --git a/sdk/python/tests/compiler/testdata/withparam_global_dict.yaml b/sdk/python/tests/compiler/testdata/withparam_global_dict.yaml index dd5f4bc6282..53da6e7f66d 100644 --- a/sdk/python/tests/compiler/testdata/withparam_global_dict.yaml +++ b/sdk/python/tests/compiler/testdata/withparam_global_dict.yaml @@ -8,7 +8,7 @@ metadata: spec: arguments: parameters: - - name: loopidy-doop + - name: loopidy_doop value: '[{"a": 1, "b": 2}, {"a": 10, "b": 20}]' entrypoint: my-pipeline serviceAccountName: pipeline-runner @@ -17,24 +17,24 @@ spec: tasks: - arguments: parameters: - - name: loopidy-doop-subvar-a - value: '{{inputs.parameters.loopidy-doop-subvar-a}}' + - name: loopidy_doop-subvar-a + value: '{{inputs.parameters.loopidy_doop-subvar-a}}' name: my-in-cop1 template: my-in-cop1 inputs: parameters: - - name: loopidy-doop-subvar-a + - name: loopidy_doop-subvar-a name: for-loop-for-loop-00000001-1 - container: args: - - 'echo no output global op1, item.a: {{inputs.parameters.loopidy-doop-subvar-a}}' + - 'echo no output global op1, item.a: {{inputs.parameters.loopidy_doop-subvar-a}}' command: - sh - -c image: library/bash:4.4.23 inputs: parameters: - - name: loopidy-doop-subvar-a + - name: loopidy_doop-subvar-a name: my-in-cop1 - container: args: @@ -68,13 +68,13 @@ spec: tasks: - arguments: parameters: - - name: loopidy-doop-subvar-a + - name: loopidy_doop-subvar-a value: '{{item.a}}' dependencies: - my-out-cop0 name: for-loop-for-loop-00000001-1 template: for-loop-for-loop-00000001-1 - withParam: '{{workflow.parameters.loopidy-doop}}' + withParam: '{{workflow.parameters.loopidy_doop}}' - name: my-out-cop0 template: my-out-cop0 - arguments: From b0b0cacea7e8ad3afb6c8b1f155ad61a1e3e1565 Mon Sep 17 00:00:00 2001 From: numerology Date: Thu, 21 Nov 2019 09:31:19 -0800 Subject: [PATCH 7/8] fix sanitization in _client.py --- sdk/python/kfp/_client.py | 5 +++-- test/sample-test/configs/xgboost_training_cm.config.yaml | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/sdk/python/kfp/_client.py b/sdk/python/kfp/_client.py index 3d5dd2053fc..3bb4277e66a 100644 --- a/sdk/python/kfp/_client.py +++ b/sdk/python/kfp/_client.py @@ -308,8 +308,9 @@ def run_pipeline(self, experiment_id, job_name, pipeline_package_path=None, para if pipeline_package_path: pipeline_obj = self._extract_pipeline_yaml(pipeline_package_path) pipeline_json_string = json.dumps(pipeline_obj) - api_params = [kfp_server_api.ApiParameter(name=sanitize_k8s_name(k), value=str(v)) - for k,v in params.items()] + api_params = [kfp_server_api.ApiParameter( + name=sanitize_k8s_name(name=k, allow_capital_underscore=True), + value=str(v)) for k,v in params.items()] key = kfp_server_api.models.ApiResourceKey(id=experiment_id, type=kfp_server_api.models.ApiResourceType.EXPERIMENT) reference = kfp_server_api.models.ApiResourceReference(key, kfp_server_api.models.ApiRelationship.OWNER) diff --git a/test/sample-test/configs/xgboost_training_cm.config.yaml b/test/sample-test/configs/xgboost_training_cm.config.yaml index 79c9b23f87e..49dac8b3600 100644 --- a/test/sample-test/configs/xgboost_training_cm.config.yaml +++ b/test/sample-test/configs/xgboost_training_cm.config.yaml @@ -16,8 +16,8 @@ test_name: xgboost_training_cm arguments: output: project: ml-pipeline-test - train-data: gs://ml-pipeline-dataset/sample-test/sfpd/train_20.csv - eval-data: gs://ml-pipeline-dataset/sample-test/sfpd/eval_5.csv + train_data: gs://ml-pipeline-dataset/sample-test/sfpd/train_20.csv + eval_data: gs://ml-pipeline-dataset/sample-test/sfpd/eval_5.csv schema: gs://ml-pipeline-dataset/sample-test/sfpd/schema.json rounds: 5 workers: 2 From a1c544862acfe6fca9701d09b543f7907f5f4a30 Mon Sep 17 00:00:00 2001 From: numerology Date: Thu, 21 Nov 2019 11:11:59 -0800 Subject: [PATCH 8/8] fix component output sanitization --- sdk/python/kfp/compiler/compiler.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sdk/python/kfp/compiler/compiler.py b/sdk/python/kfp/compiler/compiler.py index ff876838e92..2bec9cc0d6a 100644 --- a/sdk/python/kfp/compiler/compiler.py +++ b/sdk/python/kfp/compiler/compiler.py @@ -698,23 +698,23 @@ def _sanitize_and_inject_artifact(self, pipeline: dsl.Pipeline, pipeline_conf=No sanitized_name = sanitize_k8s_name(op.name) op.name = sanitized_name for param in op.outputs.values(): - param.name = sanitize_k8s_name(param.name) + param.name = sanitize_k8s_name(param.name, True) if param.op_name: param.op_name = sanitize_k8s_name(param.op_name) if op.output is not None and not isinstance(op.output, dsl._container_op._MultipleOutputsError): - op.output.name = sanitize_k8s_name(op.output.name) + op.output.name = sanitize_k8s_name(op.output.name, True) op.output.op_name = sanitize_k8s_name(op.output.op_name) if op.dependent_names: op.dependent_names = [sanitize_k8s_name(name) for name in op.dependent_names] if isinstance(op, dsl.ContainerOp) and op.file_outputs is not None: sanitized_file_outputs = {} for key in op.file_outputs.keys(): - sanitized_file_outputs[sanitize_k8s_name(key)] = op.file_outputs[key] + sanitized_file_outputs[sanitize_k8s_name(key, True)] = op.file_outputs[key] op.file_outputs = sanitized_file_outputs elif isinstance(op, dsl.ResourceOp) and op.attribute_outputs is not None: sanitized_attribute_outputs = {} for key in op.attribute_outputs.keys(): - sanitized_attribute_outputs[sanitize_k8s_name(key)] = \ + sanitized_attribute_outputs[sanitize_k8s_name(key, True)] = \ op.attribute_outputs[key] op.attribute_outputs = sanitized_attribute_outputs sanitized_ops[sanitized_name] = op