From 5050f584cae63e001fad531b65f89b8e1a1606c0 Mon Sep 17 00:00:00 2001 From: numerology Date: Tue, 21 Apr 2020 14:49:29 -0700 Subject: [PATCH 1/6] add telemetry pod labels --- .../kfp/compiler/_default_transformers.py | 39 +++++++++++++++++-- sdk/python/kfp/compiler/compiler.py | 32 ++++++++++++--- 2 files changed, 61 insertions(+), 10 deletions(-) diff --git a/sdk/python/kfp/compiler/_default_transformers.py b/sdk/python/kfp/compiler/_default_transformers.py index 1072fe49589..eaeb94497c2 100644 --- a/sdk/python/kfp/compiler/_default_transformers.py +++ b/sdk/python/kfp/compiler/_default_transformers.py @@ -12,16 +12,36 @@ # See the License for the specific language governing permissions and # limitations under the License. +import uuid +from typing import Dict, Optional, Text from ..dsl._container_op import BaseOp, ContainerOp +# Pod label indicating the SDK type from which the pipeline is +# generated. By default it's set to kfp. +_SDK_ENV_LABEL = 'pipelines.kubeflow.org/pipeline-sdk-type' +_SDK_ENV_DEFAULT = 'kfp' + +# Pod label representing the random pipeline ID. If two pods have the same +# pipeline ID then they belong to the same KFP pipeline. +_PIPELINE_UUID_LABEL = 'pipelines.kubeflow.org/pipeline-uuid' + +def get_default_telemetry_labels() -> Dict[Text, Text]: + """Returns the default pod labels for telemetry purpose.""" + result = { + _SDK_ENV_LABEL: _SDK_ENV_DEFAULT, + _PIPELINE_UUID_LABEL: str(uuid.uuid4()) + } + return result + + def add_pod_env(op: BaseOp) -> BaseOp: """Adds pod environment info to ContainerOp. """ - if isinstance(op, ContainerOp) and op.pod_labels and 'add-pod-env' in op.pod_labels and op.pod_labels['add-pod-env'] == 'true': + if isinstance(op, ContainerOp) and op.pod_labels and op.pod_labels.get('add-pod-env', None) == 'true': from kubernetes import client as k8s_client op.container.add_env_variable( k8s_client.V1EnvVar( - name='KFP_POD_NAME', + name='KFP_POD_NAME', value_from=k8s_client.V1EnvVarSource( field_ref=k8s_client.V1ObjectFieldSelector( field_path='metadata.name' @@ -30,7 +50,7 @@ def add_pod_env(op: BaseOp) -> BaseOp: ) ).add_env_variable( k8s_client.V1EnvVar( - name='KFP_NAMESPACE', + name='KFP_NAMESPACE', value_from=k8s_client.V1EnvVarSource( field_ref=k8s_client.V1ObjectFieldSelector( field_path='metadata.namespace' @@ -38,4 +58,15 @@ def add_pod_env(op: BaseOp) -> BaseOp: ) ) ) - return op \ No newline at end of file + return op + + +def add_pod_labels(labels: Optional[Dict[Text, Text]] = None) -> BaseOp: + """Adds random pipeline uuid to each pod.""" + + def _add_pod_labels(task): + for k, v in labels.items(): + task.add_pod_label(k, v) + return task + + return _add_pod_labels diff --git a/sdk/python/kfp/compiler/compiler.py b/sdk/python/kfp/compiler/compiler.py index 784b36a8d2d..61f383633f6 100644 --- a/sdk/python/kfp/compiler/compiler.py +++ b/sdk/python/kfp/compiler/compiler.py @@ -25,7 +25,7 @@ from .. import dsl from ._k8s_helper import convert_k8s_obj_to_json, sanitize_k8s_name from ._op_to_template import _op_to_template -from ._default_transformers import add_pod_env +from ._default_transformers import add_pod_env, add_pod_labels, get_default_telemetry_labels from ..components.structures import InputSpec from ..components._yaml_utils import dump_yaml @@ -754,7 +754,8 @@ def _create_workflow(self, pipeline_description: Text=None, params_list: List[dsl.PipelineParam]=None, pipeline_conf: dsl.PipelineConf = None, - ) -> Dict[Text, Any]: + allow_telemetry: bool = True, + ) -> Dict[Text, Any]: """ Internal implementation of create_workflow.""" params_list = params_list or [] argspec = inspect.getfullargspec(pipeline_func) @@ -816,6 +817,12 @@ def _create_workflow(self, default=param.value) for param in params_list] op_transformers = [add_pod_env] + # By default adds telemetry instruments. Users can opt out toggling + # allow_telemetry. + if allow_telemetry: + pod_labels = get_default_telemetry_labels() + op_transformers.append(add_pod_labels(pod_labels)) + op_transformers.extend(pipeline_conf.op_transformers) workflow = self._create_pipeline_workflow( @@ -866,7 +873,14 @@ def _compile(self, pipeline_func, pipeline_conf: dsl.PipelineConf = None): """Compile the given pipeline function into workflow.""" return self._create_workflow(pipeline_func=pipeline_func, pipeline_conf=pipeline_conf) - def compile(self, pipeline_func, package_path, type_check=True, pipeline_conf: dsl.PipelineConf = None): + def compile( + self, + pipeline_func, + package_path, + type_check=True, + pipeline_conf: dsl.PipelineConf = None, + allow_telemetry: bool = True, + ): """Compile the given pipeline function into workflow yaml. Args: @@ -874,6 +888,9 @@ def compile(self, pipeline_func, package_path, type_check=True, pipeline_conf: d package_path: the output workflow tar.gz file path. for example, "~/a.tar.gz" type_check: whether to enable the type check or not, default: False. pipeline_conf: PipelineConf instance. Can specify op transforms, image pull secrets and other pipeline-level configuration options. Overrides any configuration that may be set by the pipeline. + allow_telemetry: If set to true, two pod labels will be attached to k8s + pods spawned by this pipeline: 1) pipeline SDK style, 2) pipeline random + ID. """ import kfp type_check_old_value = kfp.TYPE_CHECK @@ -882,7 +899,8 @@ def compile(self, pipeline_func, package_path, type_check=True, pipeline_conf: d self._create_and_write_workflow( pipeline_func=pipeline_func, pipeline_conf=pipeline_conf, - package_path=package_path) + package_path=package_path, + allow_telemetry=allow_telemetry) finally: kfp.TYPE_CHECK = type_check_old_value @@ -929,7 +947,8 @@ def _create_and_write_workflow( pipeline_description: Text=None, params_list: List[dsl.PipelineParam]=None, pipeline_conf: dsl.PipelineConf=None, - package_path: Text=None + package_path: Text=None, + allow_telemetry: bool=True ) -> None: """Compile the given pipeline function and dump it to specified file format.""" workflow = self._create_workflow( @@ -937,7 +956,8 @@ def _create_and_write_workflow( pipeline_name, pipeline_description, params_list, - pipeline_conf) + pipeline_conf, + allow_telemetry) self._write_workflow(workflow, package_path) _validate_workflow(workflow) From ef2365ddbafa74c9329bcda10ac82a68d5266463 Mon Sep 17 00:00:00 2001 From: numerology Date: Wed, 22 Apr 2020 09:34:09 -0700 Subject: [PATCH 2/6] revert the id label --- .../kfp/compiler/_default_transformers.py | 9 +--- sdk/python/kfp/compiler/main.py | 52 ++++++++++++++++--- 2 files changed, 46 insertions(+), 15 deletions(-) diff --git a/sdk/python/kfp/compiler/_default_transformers.py b/sdk/python/kfp/compiler/_default_transformers.py index eaeb94497c2..0400732b3de 100644 --- a/sdk/python/kfp/compiler/_default_transformers.py +++ b/sdk/python/kfp/compiler/_default_transformers.py @@ -12,8 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import uuid -from typing import Dict, Optional, Text +from typing import Callable, Dict, Optional, Text from ..dsl._container_op import BaseOp, ContainerOp # Pod label indicating the SDK type from which the pipeline is @@ -21,15 +20,11 @@ _SDK_ENV_LABEL = 'pipelines.kubeflow.org/pipeline-sdk-type' _SDK_ENV_DEFAULT = 'kfp' -# Pod label representing the random pipeline ID. If two pods have the same -# pipeline ID then they belong to the same KFP pipeline. -_PIPELINE_UUID_LABEL = 'pipelines.kubeflow.org/pipeline-uuid' def get_default_telemetry_labels() -> Dict[Text, Text]: """Returns the default pod labels for telemetry purpose.""" result = { _SDK_ENV_LABEL: _SDK_ENV_DEFAULT, - _PIPELINE_UUID_LABEL: str(uuid.uuid4()) } return result @@ -61,7 +56,7 @@ def add_pod_env(op: BaseOp) -> BaseOp: return op -def add_pod_labels(labels: Optional[Dict[Text, Text]] = None) -> BaseOp: +def add_pod_labels(labels: Optional[Dict[Text, Text]] = None) -> Callable: """Adds random pipeline uuid to each pod.""" def _add_pod_labels(task): diff --git a/sdk/python/kfp/compiler/main.py b/sdk/python/kfp/compiler/main.py index db20502ecf4..75827777076 100644 --- a/sdk/python/kfp/compiler/main.py +++ b/sdk/python/kfp/compiler/main.py @@ -23,6 +23,16 @@ import tempfile from deprecated.sphinx import deprecated +def _str2bool(v): + if isinstance(v, bool): + return v + if v.lower() in ('yes', 'true', 't', 'y', '1'): + return True + elif v.lower() in ('no', 'false', 'f', 'n', '0'): + return False + else: + raise argparse.ArgumentTypeError('Boolean value expected.') + def parse_arguments(): """Parse command line arguments.""" @@ -47,12 +57,19 @@ def parse_arguments(): parser.add_argument('--disable-type-check', action='store_true', help='disable the type check, default is enabled.') + parser.add_argument('--enable-telemetry', + type=_str2bool, + nargs='?', + const=True, + default=True, + help='allow adding pod labels, default is enabled.') args = parser.parse_args() return args -def _compile_pipeline_function(pipeline_funcs, function_name, output_path, type_check): +def _compile_pipeline_function( + pipeline_funcs, function_name, output_path, type_check, allow_telemetry): if len(pipeline_funcs) == 0: raise ValueError('A function with @dsl.pipeline decorator is required in the py file.') @@ -68,7 +85,8 @@ def _compile_pipeline_function(pipeline_funcs, function_name, output_path, type_ else: pipeline_func = pipeline_funcs[0] - kfp.compiler.Compiler().compile(pipeline_func, output_path, type_check) + kfp.compiler.Compiler().compile( + pipeline_func, output_path, type_check, allow_telemetry=allow_telemetry) class PipelineCollectorContext(): @@ -90,26 +108,31 @@ def __exit__(self, *args): Please switch to compiling pipeline files or functions. If you use this feature please create an issue in https://github.com/kubeflow/pipelines/issues .''' ) -def compile_package(package_path, namespace, function_name, output_path, type_check): +def compile_package( + package_path, namespace, function_name, output_path, type_check, allow_telemetry): tmpdir = tempfile.mkdtemp() sys.path.insert(0, tmpdir) try: subprocess.check_call(['python3', '-m', 'pip', 'install', package_path, '-t', tmpdir]) with PipelineCollectorContext() as pipeline_funcs: __import__(namespace) - _compile_pipeline_function(pipeline_funcs, function_name, output_path, type_check) + _compile_pipeline_function( + pipeline_funcs, function_name, output_path, type_check, + allow_telemetry=allow_telemetry) finally: del sys.path[0] shutil.rmtree(tmpdir) -def compile_pyfile(pyfile, function_name, output_path, type_check): +def compile_pyfile(pyfile, function_name, output_path, type_check, allow_telemetry): sys.path.insert(0, os.path.dirname(pyfile)) try: filename = os.path.basename(pyfile) with PipelineCollectorContext() as pipeline_funcs: __import__(os.path.splitext(filename)[0]) - _compile_pipeline_function(pipeline_funcs, function_name, output_path, type_check) + _compile_pipeline_function( + pipeline_funcs, function_name, output_path, type_check, + allow_telemetry=allow_telemetry) finally: del sys.path[0] @@ -120,9 +143,22 @@ def main(): (args.py is not None and args.package is not None)): raise ValueError('Either --py or --package is needed but not both.') if args.py: - compile_pyfile(args.py, args.function, args.output, not args.disable_type_check) + compile_pyfile( + args.py, + args.function, + args.output, + not args.disable_type_check, + args.enable_telemetry + ) else: if args.namespace is None: raise ValueError('--namespace is required for compiling packages.') - compile_package(args.package, args.namespace, args.function, args.output, not args.disable_type_check) + compile_package( + args.package, + args.namespace, + args.function, + args.output, + not args.disable_type_check, + args.enable_telemetry + ) From 8062d98def9ea95fa69e06018f186f77cc2110e7 Mon Sep 17 00:00:00 2001 From: numerology Date: Wed, 22 Apr 2020 10:19:58 -0700 Subject: [PATCH 3/6] update compiler tests --- .../tests/compiler/testdata/add_pod_env.yaml | 1 + .../compiler/testdata/artifact_location.yaml | 3 ++ sdk/python/tests/compiler/testdata/basic.yaml | 6 +++ .../compiler/testdata/basic_no_decorator.yaml | 8 ++++ sdk/python/tests/compiler/testdata/coin.yaml | 12 +++++ .../tests/compiler/testdata/compose.yaml | 9 ++++ .../compiler/testdata/default_value.yaml | 6 +++ .../compiler/testdata/imagepullsecrets.yaml | 3 ++ .../testdata/input_artifact_raw_value.yaml | 12 +++++ .../loop_over_lightweight_output.yaml | 4 ++ .../parallelfor_item_argument_resolving.yaml | 44 ++++++++++++++----- .../compiler/testdata/param_op_transform.yaml | 1 + .../testdata/param_substitutions.yaml | 6 +++ .../compiler/testdata/pipelineparams.yaml | 6 +++ .../testdata/preemptible_tpu_gpu.yaml | 3 ++ .../compiler/testdata/recursive_do_while.yaml | 15 +++++++ .../compiler/testdata/recursive_while.yaml | 15 +++++++ .../compiler/testdata/resourceop_basic.yaml | 6 +++ .../tests/compiler/testdata/sidecar.yaml | 6 +++ .../tests/compiler/testdata/timeout.yaml | 6 +++ .../tests/compiler/testdata/volume.yaml | 6 +++ .../testdata/volume_snapshotop_rokurl.yaml | 24 ++++++++++ .../volume_snapshotop_sequential.yaml | 24 ++++++++++ .../compiler/testdata/volumeop_basic.yaml | 6 +++ .../tests/compiler/testdata/volumeop_dag.yaml | 12 +++++ .../compiler/testdata/volumeop_parallel.yaml | 12 +++++ .../testdata/volumeop_sequential.yaml | 12 +++++ .../compiler/testdata/withitem_basic.yaml | 9 ++++ .../compiler/testdata/withitem_nested.yaml | 12 +++++ .../compiler/testdata/withparam_global.yaml | 9 ++++ .../testdata/withparam_global_dict.yaml | 9 ++++ .../compiler/testdata/withparam_output.yaml | 9 ++++ .../testdata/withparam_output_dict.yaml | 9 ++++ 33 files changed, 314 insertions(+), 11 deletions(-) diff --git a/sdk/python/tests/compiler/testdata/add_pod_env.yaml b/sdk/python/tests/compiler/testdata/add_pod_env.yaml index a45e831445a..d2c9ef6dbcd 100644 --- a/sdk/python/tests/compiler/testdata/add_pod_env.yaml +++ b/sdk/python/tests/compiler/testdata/add_pod_env.yaml @@ -28,6 +28,7 @@ spec: image: library/bash metadata: labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp add-pod-env: 'true' name: echo - dag: diff --git a/sdk/python/tests/compiler/testdata/artifact_location.yaml b/sdk/python/tests/compiler/testdata/artifact_location.yaml index a96f25c24cd..ddaaf3175fe 100644 --- a/sdk/python/tests/compiler/testdata/artifact_location.yaml +++ b/sdk/python/tests/compiler/testdata/artifact_location.yaml @@ -54,6 +54,9 @@ spec: - name: namespace - name: tag name: foo + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp outputs: artifacts: - name: out_art diff --git a/sdk/python/tests/compiler/testdata/basic.yaml b/sdk/python/tests/compiler/testdata/basic.yaml index 4b025b23288..fd1ae511186 100644 --- a/sdk/python/tests/compiler/testdata/basic.yaml +++ b/sdk/python/tests/compiler/testdata/basic.yaml @@ -57,6 +57,7 @@ spec: - -c image: python:3.5-jessie name: exiting + metadata: {'labels': {'pipelines.kubeflow.org/pipeline-sdk-type': 'kfp'}} - container: args: - python -c "from collections import Counter; words = Counter('{{inputs.parameters.message}}'.split()); @@ -72,6 +73,9 @@ spec: parameters: - name: message name: get-frequent + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp outputs: artifacts: - name: get-frequent-word @@ -98,6 +102,8 @@ spec: - name: get-frequent-word - name: outputpath metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp annotations: tf-version.cloud-tpus.google.com: "1.12" name: save diff --git a/sdk/python/tests/compiler/testdata/basic_no_decorator.yaml b/sdk/python/tests/compiler/testdata/basic_no_decorator.yaml index ed27c418f0b..3db83fe038c 100644 --- a/sdk/python/tests/compiler/testdata/basic_no_decorator.yaml +++ b/sdk/python/tests/compiler/testdata/basic_no_decorator.yaml @@ -58,6 +58,9 @@ spec: - -c image: python:3.5-jessie name: exiting + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp - container: args: - python -c "from collections import Counter; words = Counter('{{inputs.parameters.message}}'.split()); @@ -73,6 +76,9 @@ spec: parameters: - name: message name: get-frequent + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp outputs: artifacts: - name: get-frequent-word @@ -99,6 +105,8 @@ spec: - name: get-frequent-word - name: outputpath metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp annotations: tf-version.cloud-tpus.google.com: "1.12" name: save diff --git a/sdk/python/tests/compiler/testdata/coin.yaml b/sdk/python/tests/compiler/testdata/coin.yaml index db0fb7cab48..e0461d44997 100644 --- a/sdk/python/tests/compiler/testdata/coin.yaml +++ b/sdk/python/tests/compiler/testdata/coin.yaml @@ -75,6 +75,9 @@ spec: - -c image: python:alpine3.6 name: flip + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp outputs: artifacts: - name: flip-output @@ -92,6 +95,9 @@ spec: - -c image: python:alpine3.6 name: flip-again + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp outputs: artifacts: - name: flip-again-output @@ -129,6 +135,9 @@ spec: parameters: - name: flip-again-output name: print1 + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp - container: command: - echo @@ -138,3 +147,6 @@ spec: parameters: - name: flip-again-output name: print2 + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp diff --git a/sdk/python/tests/compiler/testdata/compose.yaml b/sdk/python/tests/compiler/testdata/compose.yaml index 3402b71526b..52a1a7d13a2 100644 --- a/sdk/python/tests/compiler/testdata/compose.yaml +++ b/sdk/python/tests/compiler/testdata/compose.yaml @@ -36,6 +36,9 @@ spec: parameters: - name: url name: download + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp outputs: artifacts: - name: download-downloaded @@ -87,6 +90,9 @@ spec: parameters: - name: download-downloaded name: get-frequent + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp outputs: artifacts: - name: get-frequent-word @@ -108,3 +114,6 @@ spec: - name: get-frequent-word - name: outputpath name: save + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp diff --git a/sdk/python/tests/compiler/testdata/default_value.yaml b/sdk/python/tests/compiler/testdata/default_value.yaml index b728c3b9def..3485d912d86 100644 --- a/sdk/python/tests/compiler/testdata/default_value.yaml +++ b/sdk/python/tests/compiler/testdata/default_value.yaml @@ -56,6 +56,9 @@ spec: parameters: - name: url name: download + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp outputs: artifacts: - name: download-downloaded @@ -75,3 +78,6 @@ spec: parameters: - name: download-downloaded name: echo + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp diff --git a/sdk/python/tests/compiler/testdata/imagepullsecrets.yaml b/sdk/python/tests/compiler/testdata/imagepullsecrets.yaml index 111c70eaddc..18635a0a2a4 100644 --- a/sdk/python/tests/compiler/testdata/imagepullsecrets.yaml +++ b/sdk/python/tests/compiler/testdata/imagepullsecrets.yaml @@ -25,6 +25,9 @@ spec: parameters: - name: message name: get-frequent + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp outputs: artifacts: - name: get-frequent-word diff --git a/sdk/python/tests/compiler/testdata/input_artifact_raw_value.yaml b/sdk/python/tests/compiler/testdata/input_artifact_raw_value.yaml index 52c3df06063..be6638c8181 100644 --- a/sdk/python/tests/compiler/testdata/input_artifact_raw_value.yaml +++ b/sdk/python/tests/compiler/testdata/input_artifact_raw_value.yaml @@ -22,6 +22,9 @@ spec: raw: data: Constant artifact value name: component-with-inline-input-artifact + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp - container: command: - cat @@ -34,6 +37,9 @@ spec: raw: data: Constant artifact value name: component-with-input-artifact + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp - container: command: - cat @@ -46,6 +52,9 @@ spec: raw: data: hard-coded artifact value name: component-with-input-artifact-2 + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp - container: command: - cat @@ -58,6 +67,9 @@ spec: raw: data: Text from a file with hard-coded artifact value name: component-with-input-artifact-3 + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp - dag: tasks: - name: component-with-inline-input-artifact diff --git a/sdk/python/tests/compiler/testdata/loop_over_lightweight_output.yaml b/sdk/python/tests/compiler/testdata/loop_over_lightweight_output.yaml index 6139c6503df..d231b4e6566 100644 --- a/sdk/python/tests/compiler/testdata/loop_over_lightweight_output.yaml +++ b/sdk/python/tests/compiler/testdata/loop_over_lightweight_output.yaml @@ -44,6 +44,8 @@ - "name": |- produce-list-data_list-loop-item "metadata": + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp "annotations": "pipelines.kubeflow.org/component_spec": |- {"inputs": [{"name": "data"}], "name": "Consume data"} @@ -104,6 +106,8 @@ "image": |- busybox "metadata": + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp "annotations": "pipelines.kubeflow.org/component_spec": |- {"name": "Produce list", "outputs": [{"name": "data_list"}]} diff --git a/sdk/python/tests/compiler/testdata/parallelfor_item_argument_resolving.yaml b/sdk/python/tests/compiler/testdata/parallelfor_item_argument_resolving.yaml index 47e49a54654..c589797249e 100644 --- a/sdk/python/tests/compiler/testdata/parallelfor_item_argument_resolving.yaml +++ b/sdk/python/tests/compiler/testdata/parallelfor_item_argument_resolving.yaml @@ -48,7 +48,9 @@ spec: parameters: - name: produce-list-of-strings-output - metadata: + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp annotations: pipelines.kubeflow.org/component_spec: "{\"inputs\": [{\"name\": \"param1\"}], \"name\": \"Consume\"}" name: consume @@ -90,7 +92,9 @@ spec: parameters: - name: produce-list-of-strings-output-loop-item - metadata: + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp annotations: pipelines.kubeflow.org/component_spec: "{\"inputs\": [{\"name\": \"param1\"}], \"name\": \"Consume\"}" name: consume-2 @@ -132,7 +136,9 @@ spec: parameters: - name: produce-str-output - metadata: + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp annotations: pipelines.kubeflow.org/component_spec: "{\"inputs\": [{\"name\": \"param1\"}], \"name\": \"Consume\"}" name: consume-3 @@ -174,7 +180,9 @@ spec: parameters: - name: produce-list-of-ints-output - metadata: + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp annotations: pipelines.kubeflow.org/component_spec: "{\"inputs\": [{\"name\": \"param1\"}], \"name\": \"Consume\"}" name: consume-4 @@ -216,7 +224,9 @@ spec: parameters: - name: produce-list-of-ints-output-loop-item - metadata: + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp annotations: pipelines.kubeflow.org/component_spec: "{\"inputs\": [{\"name\": \"param1\"}], \"name\": \"Consume\"}" name: consume-5 @@ -258,7 +268,9 @@ spec: parameters: - name: produce-list-of-dicts-output - metadata: + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp annotations: pipelines.kubeflow.org/component_spec: "{\"inputs\": [{\"name\": \"param1\"}], \"name\": \"Consume\"}" name: consume-6 @@ -300,7 +312,9 @@ spec: parameters: - name: produce-list-of-dicts-output-loop-item-subvar-aaa - metadata: + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp annotations: pipelines.kubeflow.org/component_spec: "{\"inputs\": [{\"name\": \"param1\"}], \"name\": \"Consume\"}" name: consume-7 @@ -502,7 +516,9 @@ spec: with open(output_file, 'w') as f: f.write(_output_serializers[idx](_outputs[idx])) image: "tensorflow/tensorflow:1.13.2-py3" - metadata: + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp annotations: pipelines.kubeflow.org/component_spec: "{\"name\": \"Produce list of dicts\", \"outputs\": [{\"name\": \"Output\", \"type\": \"JsonArray\"}]}" name: produce-list-of-dicts @@ -564,7 +580,9 @@ spec: with open(output_file, 'w') as f: f.write(_output_serializers[idx](_outputs[idx])) image: "tensorflow/tensorflow:1.13.2-py3" - metadata: + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp annotations: pipelines.kubeflow.org/component_spec: "{\"name\": \"Produce list of ints\", \"outputs\": [{\"name\": \"Output\", \"type\": \"JsonArray\"}]}" name: produce-list-of-ints @@ -626,7 +644,9 @@ spec: with open(output_file, 'w') as f: f.write(_output_serializers[idx](_outputs[idx])) image: "tensorflow/tensorflow:1.13.2-py3" - metadata: + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp annotations: pipelines.kubeflow.org/component_spec: "{\"name\": \"Produce list of strings\", \"outputs\": [{\"name\": \"Output\", \"type\": \"JsonArray\"}]}" name: produce-list-of-strings @@ -682,7 +702,9 @@ spec: with open(output_file, 'w') as f: f.write(_output_serializers[idx](_outputs[idx])) image: "tensorflow/tensorflow:1.13.2-py3" - metadata: + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp annotations: pipelines.kubeflow.org/component_spec: "{\"name\": \"Produce str\", \"outputs\": [{\"name\": \"Output\", \"type\": \"String\"}]}" name: produce-str diff --git a/sdk/python/tests/compiler/testdata/param_op_transform.yaml b/sdk/python/tests/compiler/testdata/param_op_transform.yaml index b0da3751f66..09880f9e013 100644 --- a/sdk/python/tests/compiler/testdata/param_op_transform.yaml +++ b/sdk/python/tests/compiler/testdata/param_op_transform.yaml @@ -21,6 +21,7 @@ spec: - name: param metadata: labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp param: '{{inputs.parameters.param}}' name: cop - dag: diff --git a/sdk/python/tests/compiler/testdata/param_substitutions.yaml b/sdk/python/tests/compiler/testdata/param_substitutions.yaml index 76f1f1a9f8b..cd74997e64c 100644 --- a/sdk/python/tests/compiler/testdata/param_substitutions.yaml +++ b/sdk/python/tests/compiler/testdata/param_substitutions.yaml @@ -23,11 +23,17 @@ spec: parameters: - name: create-volume-name name: cop + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp volumes: - name: create-volume persistentVolumeClaim: claimName: '{{inputs.parameters.create-volume-name}}' - name: create-volume + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp outputs: parameters: - name: create-volume-manifest diff --git a/sdk/python/tests/compiler/testdata/pipelineparams.yaml b/sdk/python/tests/compiler/testdata/pipelineparams.yaml index 174fdeb1493..a07dfb8495a 100644 --- a/sdk/python/tests/compiler/testdata/pipelineparams.yaml +++ b/sdk/python/tests/compiler/testdata/pipelineparams.yaml @@ -29,6 +29,9 @@ spec: value: '10' templates: - name: download + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp inputs: parameters: - name: sleep_ms @@ -54,6 +57,9 @@ spec: args: - -text="hello world" - name: echo + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp inputs: parameters: - name: download-downloaded diff --git a/sdk/python/tests/compiler/testdata/preemptible_tpu_gpu.yaml b/sdk/python/tests/compiler/testdata/preemptible_tpu_gpu.yaml index 85fb22644c2..d2a1c68bc57 100644 --- a/sdk/python/tests/compiler/testdata/preemptible_tpu_gpu.yaml +++ b/sdk/python/tests/compiler/testdata/preemptible_tpu_gpu.yaml @@ -32,6 +32,9 @@ spec: limits: nvidia.com/gpu: 1 name: flip + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp outputs: artifacts: - name: flip-output diff --git a/sdk/python/tests/compiler/testdata/recursive_do_while.yaml b/sdk/python/tests/compiler/testdata/recursive_do_while.yaml index 3ed3bbf88f3..80e58dd586e 100644 --- a/sdk/python/tests/compiler/testdata/recursive_do_while.yaml +++ b/sdk/python/tests/compiler/testdata/recursive_do_while.yaml @@ -32,6 +32,9 @@ spec: - -c image: python:alpine3.6 name: flip + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp outputs: artifacts: - name: flip-output @@ -49,6 +52,9 @@ spec: - -c image: python:alpine3.6 name: flip-2 + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp outputs: artifacts: - name: flip-2-output @@ -62,6 +68,9 @@ spec: - -c image: python:alpine3.6 name: flip-3 + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp outputs: artifacts: - name: flip-3-output @@ -129,6 +138,9 @@ spec: parameters: - name: flip-output name: print + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp - container: command: - echo @@ -138,3 +150,6 @@ spec: parameters: - name: flip-output name: print-2 + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp diff --git a/sdk/python/tests/compiler/testdata/recursive_while.yaml b/sdk/python/tests/compiler/testdata/recursive_while.yaml index eeed7c8ee74..e75b75b3939 100644 --- a/sdk/python/tests/compiler/testdata/recursive_while.yaml +++ b/sdk/python/tests/compiler/testdata/recursive_while.yaml @@ -46,6 +46,9 @@ spec: - -c image: python:alpine3.6 name: flip + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp outputs: artifacts: - name: flip-output @@ -63,6 +66,9 @@ spec: - -c image: python:alpine3.6 name: flip-2 + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp outputs: artifacts: - name: flip-2-output @@ -76,6 +82,9 @@ spec: - -c image: python:alpine3.6 name: flip-3 + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp outputs: artifacts: - name: flip-3-output @@ -131,6 +140,9 @@ spec: parameters: - name: flip-output name: print + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp - container: command: - echo @@ -140,3 +152,6 @@ spec: parameters: - name: flip-output name: print-2 + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp diff --git a/sdk/python/tests/compiler/testdata/resourceop_basic.yaml b/sdk/python/tests/compiler/testdata/resourceop_basic.yaml index 666606a62ab..080f07916c9 100644 --- a/sdk/python/tests/compiler/testdata/resourceop_basic.yaml +++ b/sdk/python/tests/compiler/testdata/resourceop_basic.yaml @@ -27,6 +27,9 @@ spec: parameters: - name: create-my-secret-name name: cop + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp volumes: - name: my-secret secret: @@ -36,6 +39,9 @@ spec: - name: password - name: username name: create-my-secret + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp outputs: parameters: - name: create-my-secret-manifest diff --git a/sdk/python/tests/compiler/testdata/sidecar.yaml b/sdk/python/tests/compiler/testdata/sidecar.yaml index cab18f2784f..a6803534882 100644 --- a/sdk/python/tests/compiler/testdata/sidecar.yaml +++ b/sdk/python/tests/compiler/testdata/sidecar.yaml @@ -43,7 +43,13 @@ spec: command: - sh - "-c" + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp - name: echo + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp inputs: parameters: - name: download-downloaded diff --git a/sdk/python/tests/compiler/testdata/timeout.yaml b/sdk/python/tests/compiler/testdata/timeout.yaml index 95895f1671d..26d8bea0420 100644 --- a/sdk/python/tests/compiler/testdata/timeout.yaml +++ b/sdk/python/tests/compiler/testdata/timeout.yaml @@ -30,6 +30,9 @@ spec: - -c image: python:alpine3.6 name: random-failure + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp - container: args: - import random; import sys; exit_code = random.choice([0,1]); print(exit_code); @@ -39,3 +42,6 @@ spec: - -c image: python:alpine3.6 name: random-failure-2 + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp diff --git a/sdk/python/tests/compiler/testdata/volume.yaml b/sdk/python/tests/compiler/testdata/volume.yaml index 6e7026b31f8..870309114bf 100644 --- a/sdk/python/tests/compiler/testdata/volume.yaml +++ b/sdk/python/tests/compiler/testdata/volume.yaml @@ -40,6 +40,9 @@ spec: - mountPath: /secret/gcp-credentials name: gcp-credentials name: download + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp outputs: artifacts: - name: download-downloaded @@ -63,6 +66,9 @@ spec: parameters: - name: download-downloaded name: echo + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp - dag: tasks: - name: download diff --git a/sdk/python/tests/compiler/testdata/volume_snapshotop_rokurl.yaml b/sdk/python/tests/compiler/testdata/volume_snapshotop_rokurl.yaml index 2d9e6570671..be305de17f3 100644 --- a/sdk/python/tests/compiler/testdata/volume_snapshotop_rokurl.yaml +++ b/sdk/python/tests/compiler/testdata/volume_snapshotop_rokurl.yaml @@ -16,6 +16,9 @@ spec: parameters: - name: create-volume-1-name name: create-snapshot-1 + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp outputs: parameters: - name: create-snapshot-1-manifest @@ -37,6 +40,9 @@ spec: parameters: - name: create-volume-2-name name: create-snapshot-2 + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp outputs: parameters: - name: create-snapshot-2-manifest @@ -58,6 +64,9 @@ spec: parameters: - name: rok_url name: create-volume-1 + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp outputs: parameters: - name: create-volume-1-manifest @@ -80,6 +89,9 @@ spec: - name: create-snapshot-1-name - name: create-snapshot-1-size name: create-volume-2 + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp outputs: parameters: - name: create-volume-2-manifest @@ -102,6 +114,9 @@ spec: - name: create-snapshot-2-name - name: create-snapshot-2-size name: create-volume-3 + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp outputs: parameters: - name: create-volume-3-manifest @@ -133,6 +148,9 @@ spec: parameters: - name: create-volume-1-name name: step1-concat + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp volumes: - name: create-volume-1 persistentVolumeClaim: @@ -150,6 +168,9 @@ spec: parameters: - name: create-volume-2-name name: step2-gunzip + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp volumes: - name: create-volume-2 persistentVolumeClaim: @@ -166,6 +187,9 @@ spec: parameters: - name: create-volume-3-name name: step3-output + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp volumes: - name: create-volume-3 persistentVolumeClaim: diff --git a/sdk/python/tests/compiler/testdata/volume_snapshotop_sequential.yaml b/sdk/python/tests/compiler/testdata/volume_snapshotop_sequential.yaml index a0d9ac8f2ab..c34153b2789 100644 --- a/sdk/python/tests/compiler/testdata/volume_snapshotop_sequential.yaml +++ b/sdk/python/tests/compiler/testdata/volume_snapshotop_sequential.yaml @@ -13,6 +13,9 @@ spec: serviceAccountName: pipeline-runner templates: - name: create-volume + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp outputs: parameters: - name: create-volume-manifest @@ -44,6 +47,9 @@ spec: - name: create-volume-name - name: url name: step1-ingest + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp volumes: - name: create-volume persistentVolumeClaim: @@ -52,6 +58,9 @@ spec: parameters: - name: create-volume-name name: step1-snap + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp outputs: parameters: - name: step1-snap-manifest @@ -83,6 +92,9 @@ spec: parameters: - name: create-volume-name name: step2-gunzip + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp volumes: - name: create-volume persistentVolumeClaim: @@ -91,6 +103,9 @@ spec: parameters: - name: create-volume-name name: step2-snap + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp outputs: parameters: - name: step2-snap-manifest @@ -122,6 +137,9 @@ spec: parameters: - name: create-volume-name name: step3-copy + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp volumes: - name: create-volume persistentVolumeClaim: @@ -130,6 +148,9 @@ spec: parameters: - name: create-volume-name name: step3-snap + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp outputs: parameters: - name: step3-snap-manifest @@ -160,6 +181,9 @@ spec: parameters: - name: create-volume-name name: step4-output + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp volumes: - name: create-volume persistentVolumeClaim: diff --git a/sdk/python/tests/compiler/testdata/volumeop_basic.yaml b/sdk/python/tests/compiler/testdata/volumeop_basic.yaml index e2ab8103572..efb62868898 100644 --- a/sdk/python/tests/compiler/testdata/volumeop_basic.yaml +++ b/sdk/python/tests/compiler/testdata/volumeop_basic.yaml @@ -26,6 +26,9 @@ spec: parameters: - name: create-pvc-name name: cop + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp volumes: - name: create-pvc persistentVolumeClaim: @@ -34,6 +37,9 @@ spec: parameters: - name: size name: create-pvc + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp outputs: parameters: - name: create-pvc-manifest diff --git a/sdk/python/tests/compiler/testdata/volumeop_dag.yaml b/sdk/python/tests/compiler/testdata/volumeop_dag.yaml index 420deff0e56..f13bd96ae78 100644 --- a/sdk/python/tests/compiler/testdata/volumeop_dag.yaml +++ b/sdk/python/tests/compiler/testdata/volumeop_dag.yaml @@ -12,6 +12,9 @@ spec: serviceAccountName: pipeline-runner templates: - name: create-pvc + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp outputs: parameters: - name: create-pvc-manifest @@ -42,6 +45,9 @@ spec: parameters: - name: create-pvc-name name: step1 + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp volumes: - name: create-pvc persistentVolumeClaim: @@ -60,6 +66,9 @@ spec: parameters: - name: create-pvc-name name: step2 + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp volumes: - name: create-pvc persistentVolumeClaim: @@ -78,6 +87,9 @@ spec: parameters: - name: create-pvc-name name: step3 + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp volumes: - name: create-pvc persistentVolumeClaim: diff --git a/sdk/python/tests/compiler/testdata/volumeop_parallel.yaml b/sdk/python/tests/compiler/testdata/volumeop_parallel.yaml index cf278e50692..c7b07029fdd 100644 --- a/sdk/python/tests/compiler/testdata/volumeop_parallel.yaml +++ b/sdk/python/tests/compiler/testdata/volumeop_parallel.yaml @@ -12,6 +12,9 @@ spec: serviceAccountName: pipeline-runner templates: - name: create-pvc + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp outputs: parameters: - name: create-pvc-manifest @@ -42,6 +45,9 @@ spec: parameters: - name: create-pvc-name name: step1 + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp volumes: - name: create-pvc persistentVolumeClaim: @@ -60,6 +66,9 @@ spec: parameters: - name: create-pvc-name name: step2 + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp volumes: - name: create-pvc persistentVolumeClaim: @@ -78,6 +87,9 @@ spec: parameters: - name: create-pvc-name name: step3 + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp volumes: - name: create-pvc persistentVolumeClaim: diff --git a/sdk/python/tests/compiler/testdata/volumeop_sequential.yaml b/sdk/python/tests/compiler/testdata/volumeop_sequential.yaml index 0d9fe372799..30293db89c6 100644 --- a/sdk/python/tests/compiler/testdata/volumeop_sequential.yaml +++ b/sdk/python/tests/compiler/testdata/volumeop_sequential.yaml @@ -12,6 +12,9 @@ spec: serviceAccountName: pipeline-runner templates: - name: mypvc + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp outputs: parameters: - name: mypvc-manifest @@ -42,6 +45,9 @@ spec: parameters: - name: mypvc-name name: step1 + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp volumes: - name: mypvc persistentVolumeClaim: @@ -60,6 +66,9 @@ spec: parameters: - name: mypvc-name name: step2 + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp volumes: - name: mypvc persistentVolumeClaim: @@ -77,6 +86,9 @@ spec: parameters: - name: mypvc-name name: step3 + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp volumes: - name: mypvc persistentVolumeClaim: diff --git a/sdk/python/tests/compiler/testdata/withitem_basic.yaml b/sdk/python/tests/compiler/testdata/withitem_basic.yaml index f19512bbd2e..26e1a2b5bc4 100644 --- a/sdk/python/tests/compiler/testdata/withitem_basic.yaml +++ b/sdk/python/tests/compiler/testdata/withitem_basic.yaml @@ -47,6 +47,9 @@ spec: - name: loop-item-param-00000001-subvar-a - name: my_pipe_param name: my-in-coop1 + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp - container: args: - echo op2 {{inputs.parameters.loop-item-param-00000001-subvar-b}} @@ -58,6 +61,9 @@ spec: parameters: - name: loop-item-param-00000001-subvar-b name: my-in-coop2 + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp - container: args: - echo {{inputs.parameters.my_pipe_param}} @@ -69,6 +75,9 @@ spec: parameters: - name: my_pipe_param name: my-out-cop + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp - dag: tasks: - arguments: diff --git a/sdk/python/tests/compiler/testdata/withitem_nested.yaml b/sdk/python/tests/compiler/testdata/withitem_nested.yaml index b1f8bed1be8..7f12f304d66 100644 --- a/sdk/python/tests/compiler/testdata/withitem_nested.yaml +++ b/sdk/python/tests/compiler/testdata/withitem_nested.yaml @@ -79,6 +79,9 @@ spec: - name: loop-item-param-00000001-subvar-a - name: my_pipe_param name: my-in-coop1 + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp - container: args: - echo op2 {{inputs.parameters.loop-item-param-00000001-subvar-b}} @@ -90,6 +93,9 @@ spec: parameters: - name: loop-item-param-00000001-subvar-b name: my-in-coop2 + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp - container: args: - echo op1 {{inputs.parameters.loop-item-param-00000001-subvar-a}} {{inputs.parameters.loop-item-param-00000002}} @@ -104,6 +110,9 @@ spec: - name: loop-item-param-00000002 - name: my_pipe_param name: my-inner-inner-coop + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp - container: args: - echo {{inputs.parameters.my_pipe_param}} @@ -115,6 +124,9 @@ spec: parameters: - name: my_pipe_param name: my-out-cop + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp - dag: tasks: - arguments: diff --git a/sdk/python/tests/compiler/testdata/withparam_global.yaml b/sdk/python/tests/compiler/testdata/withparam_global.yaml index a60c4eaf0db..6ad2c5939d8 100644 --- a/sdk/python/tests/compiler/testdata/withparam_global.yaml +++ b/sdk/python/tests/compiler/testdata/withparam_global.yaml @@ -36,6 +36,9 @@ spec: parameters: - name: loopidy_doop-loop-item name: my-in-cop1 + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp - container: args: - python -c "import json; import sys; json.dump([i for i in range(20, 31)], @@ -45,6 +48,9 @@ spec: - -c image: python:alpine3.6 name: my-out-cop0 + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp outputs: artifacts: - name: my-out-cop0-out @@ -64,6 +70,9 @@ spec: parameters: - name: my-out-cop0-out name: my-out-cop2 + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp - dag: tasks: - arguments: diff --git a/sdk/python/tests/compiler/testdata/withparam_global_dict.yaml b/sdk/python/tests/compiler/testdata/withparam_global_dict.yaml index f78f3c69b5b..d2ac3f5a3d2 100644 --- a/sdk/python/tests/compiler/testdata/withparam_global_dict.yaml +++ b/sdk/python/tests/compiler/testdata/withparam_global_dict.yaml @@ -36,6 +36,9 @@ spec: parameters: - name: loopidy_doop-loop-item-subvar-a name: my-in-cop1 + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp - container: args: - python -c "import json; import sys; json.dump([i for i in range(20, 31)], @@ -45,6 +48,9 @@ spec: - -c image: python:alpine3.6 name: my-out-cop0 + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp outputs: artifacts: - name: my-out-cop0-out @@ -64,6 +70,9 @@ spec: parameters: - name: my-out-cop0-out name: my-out-cop2 + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp - dag: tasks: - arguments: diff --git a/sdk/python/tests/compiler/testdata/withparam_output.yaml b/sdk/python/tests/compiler/testdata/withparam_output.yaml index 8978d1d055b..35e3887c591 100644 --- a/sdk/python/tests/compiler/testdata/withparam_output.yaml +++ b/sdk/python/tests/compiler/testdata/withparam_output.yaml @@ -33,6 +33,9 @@ spec: parameters: - name: my-out-cop0-out-loop-item name: my-in-cop1 + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp - container: args: - python -c "import json; import sys; json.dump([i for i in range(20, 31)], @@ -42,6 +45,9 @@ spec: - -c image: python:alpine3.6 name: my-out-cop0 + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp outputs: artifacts: - name: my-out-cop0-out @@ -61,6 +67,9 @@ spec: parameters: - name: my-out-cop0-out name: my-out-cop2 + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp - dag: tasks: - arguments: diff --git a/sdk/python/tests/compiler/testdata/withparam_output_dict.yaml b/sdk/python/tests/compiler/testdata/withparam_output_dict.yaml index 89aa8bb4b2a..4613eb6ae65 100644 --- a/sdk/python/tests/compiler/testdata/withparam_output_dict.yaml +++ b/sdk/python/tests/compiler/testdata/withparam_output_dict.yaml @@ -33,6 +33,9 @@ spec: parameters: - name: my-out-cop0-out-loop-item-subvar-a name: my-in-cop1 + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp - container: args: - 'python -c "import json; import sys; json.dump([{''a'': 1, ''b'': 2}, {''a'': @@ -42,6 +45,9 @@ spec: - -c image: python:alpine3.6 name: my-out-cop0 + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp outputs: artifacts: - name: my-out-cop0-out @@ -61,6 +67,9 @@ spec: parameters: - name: my-out-cop0-out name: my-out-cop2 + metadata: + labels: + pipelines.kubeflow.org/pipeline-sdk-type: kfp - dag: tasks: - arguments: From a08c0611b6988e23cfb54ab9de591640b454d01f Mon Sep 17 00:00:00 2001 From: numerology Date: Wed, 22 Apr 2020 10:27:30 -0700 Subject: [PATCH 4/6] update cli arg --- sdk/python/kfp/compiler/main.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/sdk/python/kfp/compiler/main.py b/sdk/python/kfp/compiler/main.py index 75827777076..557e64370ea 100644 --- a/sdk/python/kfp/compiler/main.py +++ b/sdk/python/kfp/compiler/main.py @@ -57,12 +57,9 @@ def parse_arguments(): parser.add_argument('--disable-type-check', action='store_true', help='disable the type check, default is enabled.') - parser.add_argument('--enable-telemetry', - type=_str2bool, - nargs='?', - const=True, - default=True, - help='allow adding pod labels, default is enabled.') + parser.add_argument('--disable-telemetry', + action='store_true', + help='disable adding telemetry labels, default is enabled.') args = parser.parse_args() return args @@ -148,7 +145,7 @@ def main(): args.function, args.output, not args.disable_type_check, - args.enable_telemetry + not args.disable_telemetry ) else: if args.namespace is None: @@ -159,6 +156,6 @@ def main(): args.function, args.output, not args.disable_type_check, - args.enable_telemetry + not args.disable_telemetry ) From a2273eee63299607d3c3c2714f575cd55e46dc99 Mon Sep 17 00:00:00 2001 From: numerology Date: Wed, 22 Apr 2020 14:16:23 -0700 Subject: [PATCH 5/6] bypass tfx --- sdk/python/kfp/compiler/_default_transformers.py | 5 ++++- sdk/python/kfp/compiler/compiler.py | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/sdk/python/kfp/compiler/_default_transformers.py b/sdk/python/kfp/compiler/_default_transformers.py index 0400732b3de..2dd67ae7282 100644 --- a/sdk/python/kfp/compiler/_default_transformers.py +++ b/sdk/python/kfp/compiler/_default_transformers.py @@ -61,7 +61,10 @@ def add_pod_labels(labels: Optional[Dict[Text, Text]] = None) -> Callable: def _add_pod_labels(task): for k, v in labels.items(): - task.add_pod_label(k, v) + # Only append but not update. + # This is needed to bypass TFX pipelines/components. + if k not in task.pod_labels: + task.add_pod_label(k, v) return task return _add_pod_labels diff --git a/sdk/python/kfp/compiler/compiler.py b/sdk/python/kfp/compiler/compiler.py index 61f383633f6..44cc1996aa5 100644 --- a/sdk/python/kfp/compiler/compiler.py +++ b/sdk/python/kfp/compiler/compiler.py @@ -819,6 +819,7 @@ def _create_workflow(self, op_transformers = [add_pod_env] # By default adds telemetry instruments. Users can opt out toggling # allow_telemetry. + # Also, TFX pipelines will be bypassed for pipeline compiled by tfx>0.21.4. if allow_telemetry: pod_labels = get_default_telemetry_labels() op_transformers.append(add_pod_labels(pod_labels)) From 9f08e5ec45ceeb0804e5f852a3788244e127626d Mon Sep 17 00:00:00 2001 From: numerology Date: Wed, 22 Apr 2020 14:44:39 -0700 Subject: [PATCH 6/6] update docstring --- sdk/python/kfp/compiler/_default_transformers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/python/kfp/compiler/_default_transformers.py b/sdk/python/kfp/compiler/_default_transformers.py index 2dd67ae7282..bb33d212668 100644 --- a/sdk/python/kfp/compiler/_default_transformers.py +++ b/sdk/python/kfp/compiler/_default_transformers.py @@ -57,7 +57,7 @@ def add_pod_env(op: BaseOp) -> BaseOp: def add_pod_labels(labels: Optional[Dict[Text, Text]] = None) -> Callable: - """Adds random pipeline uuid to each pod.""" + """Adds provided pod labels to each pod.""" def _add_pod_labels(task): for k, v in labels.items():