From d8eaeaad95288709fecd3b4ef6aac34ce7e0c324 Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Thu, 15 Aug 2019 17:25:59 -0700 Subject: [PATCH] SDK - Preserving the pipeline input information in the compiled Workflow (#1381) * SDK - Preserving the pipeline metadata in the compiled Workflow * Stabilizing the DSL compiler tests --- sdk/python/kfp/compiler/compiler.py | 4 ++++ sdk/python/tests/compiler/testdata/add_pod_env.yaml | 3 +++ sdk/python/tests/compiler/testdata/artifact_location.yaml | 5 +++++ sdk/python/tests/compiler/testdata/basic.yaml | 2 ++ sdk/python/tests/compiler/testdata/coin.yaml | 2 ++ sdk/python/tests/compiler/testdata/compose.yaml | 2 ++ sdk/python/tests/compiler/testdata/default_value.yaml | 2 ++ sdk/python/tests/compiler/testdata/imagepullsecrets.yaml | 2 ++ sdk/python/tests/compiler/testdata/param_op_transform.yaml | 6 ++++++ sdk/python/tests/compiler/testdata/param_substitutions.yaml | 3 +++ sdk/python/tests/compiler/testdata/pipelineparams.yaml | 5 +++++ sdk/python/tests/compiler/testdata/preemptible_tpu_gpu.yaml | 2 ++ sdk/python/tests/compiler/testdata/recursive_do_while.yaml | 3 +++ sdk/python/tests/compiler/testdata/recursive_while.yaml | 4 ++++ sdk/python/tests/compiler/testdata/resourceop_basic.yaml | 5 +++++ sdk/python/tests/compiler/testdata/retry.yaml | 4 ++++ sdk/python/tests/compiler/testdata/sidecar.yaml | 3 +++ sdk/python/tests/compiler/testdata/timeout.yaml | 4 ++++ sdk/python/tests/compiler/testdata/volume.yaml | 3 +++ .../tests/compiler/testdata/volume_snapshotop_rokurl.yaml | 4 ++++ .../compiler/testdata/volume_snapshotop_sequential.yaml | 4 ++++ sdk/python/tests/compiler/testdata/volumeop_basic.yaml | 4 ++++ sdk/python/tests/compiler/testdata/volumeop_dag.yaml | 3 +++ sdk/python/tests/compiler/testdata/volumeop_parallel.yaml | 3 +++ sdk/python/tests/compiler/testdata/volumeop_sequential.yaml | 3 +++ 25 files changed, 85 insertions(+) diff --git a/sdk/python/kfp/compiler/compiler.py b/sdk/python/kfp/compiler/compiler.py index 30d71870424..92d8658dcd6 100644 --- a/sdk/python/kfp/compiler/compiler.py +++ b/sdk/python/kfp/compiler/compiler.py @@ -655,6 +655,10 @@ def _compile(self, pipeline_func): op_transformers = [add_pod_env] op_transformers.extend(p.conf.op_transformers) workflow = self._create_pipeline_workflow(args_list_with_defaults, p, op_transformers) + + import json + workflow.setdefault('metadata', {}).setdefault('annotations', {})['pipelines.kubeflow.org/pipeline_spec'] = json.dumps(pipeline_meta.to_dict(), sort_keys=True) + return workflow def compile(self, pipeline_func, package_path, type_check=True): diff --git a/sdk/python/tests/compiler/testdata/add_pod_env.yaml b/sdk/python/tests/compiler/testdata/add_pod_env.yaml index fb4b114587e..bfcdc1a41a5 100644 --- a/sdk/python/tests/compiler/testdata/add_pod_env.yaml +++ b/sdk/python/tests/compiler/testdata/add_pod_env.yaml @@ -1,6 +1,9 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: + annotations: + pipelines.kubeflow.org/pipeline_spec: '{"description": "Test adding pod env", + "inputs": [], "name": "Test adding pod env"}' generateName: test-adding-pod-env- spec: arguments: diff --git a/sdk/python/tests/compiler/testdata/artifact_location.yaml b/sdk/python/tests/compiler/testdata/artifact_location.yaml index 0bd8432ff8c..c6c369b10de 100644 --- a/sdk/python/tests/compiler/testdata/artifact_location.yaml +++ b/sdk/python/tests/compiler/testdata/artifact_location.yaml @@ -14,6 +14,11 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: + annotations: + pipelines.kubeflow.org/pipeline_spec: '{"description": "hello world", "inputs": + [{"default": null, "description": "", "name": "tag", "type": ""}, {"default": + "kubeflow", "description": "", "name": "namespace", "type": ""}, {"default": + "foobar", "description": "", "name": "bucket", "type": ""}], "name": "foo"}' generateName: foo- spec: arguments: diff --git a/sdk/python/tests/compiler/testdata/basic.yaml b/sdk/python/tests/compiler/testdata/basic.yaml index 3a88a734f5a..03b8e27880a 100644 --- a/sdk/python/tests/compiler/testdata/basic.yaml +++ b/sdk/python/tests/compiler/testdata/basic.yaml @@ -14,6 +14,8 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: + annotations: + pipelines.kubeflow.org/pipeline_spec: '{"description": "Get Most Frequent Word and Save to GCS", "inputs": [{"default": null, "description": "", "name": "message", "type": ""}, {"default": null, "description": "", "name": "outputpath", "type": ""}], "name": "Save Most Frequent"}' generateName: save-most-frequent- spec: arguments: diff --git a/sdk/python/tests/compiler/testdata/coin.yaml b/sdk/python/tests/compiler/testdata/coin.yaml index 5a044b747ff..c70e31294eb 100644 --- a/sdk/python/tests/compiler/testdata/coin.yaml +++ b/sdk/python/tests/compiler/testdata/coin.yaml @@ -14,6 +14,8 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: + annotations: + pipelines.kubeflow.org/pipeline_spec: '{"description": "shows how to use dsl.Condition.", "inputs": [], "name": "pipeline flip coin"}' generateName: pipeline-flip-coin- spec: arguments: diff --git a/sdk/python/tests/compiler/testdata/compose.yaml b/sdk/python/tests/compiler/testdata/compose.yaml index 645eb5d2018..59dc3513dba 100644 --- a/sdk/python/tests/compiler/testdata/compose.yaml +++ b/sdk/python/tests/compiler/testdata/compose.yaml @@ -14,6 +14,8 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: + annotations: + pipelines.kubeflow.org/pipeline_spec: '{"description": "Download and Get Most Frequent Word and Save to GCS", "inputs": [{"default": null, "description": "", "name": "url", "type": ""}, {"default": null, "description": "", "name": "outputpath", "type": ""}], "name": "Download and Save Most Frequent"}' generateName: download-and-save-most-frequent- spec: arguments: diff --git a/sdk/python/tests/compiler/testdata/default_value.yaml b/sdk/python/tests/compiler/testdata/default_value.yaml index daa6140c978..a03a0803463 100644 --- a/sdk/python/tests/compiler/testdata/default_value.yaml +++ b/sdk/python/tests/compiler/testdata/default_value.yaml @@ -14,6 +14,8 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: + annotations: + pipelines.kubeflow.org/pipeline_spec: '{"description": "A pipeline with parameter and default value.", "inputs": [{"default": "gs://ml-pipeline/shakespeare1.txt", "description": "", "name": "url", "type": ""}], "name": "Default Value"}' generateName: default-value- spec: arguments: diff --git a/sdk/python/tests/compiler/testdata/imagepullsecrets.yaml b/sdk/python/tests/compiler/testdata/imagepullsecrets.yaml index b7a5a91c8d3..3ea58057fac 100644 --- a/sdk/python/tests/compiler/testdata/imagepullsecrets.yaml +++ b/sdk/python/tests/compiler/testdata/imagepullsecrets.yaml @@ -1,6 +1,8 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: + annotations: + pipelines.kubeflow.org/pipeline_spec: '{"description": "Get Most Frequent Word and Save to GCS", "inputs": [{"default": null, "description": "", "name": "message", "type": ""}], "name": "Save Most Frequent"}' generateName: save-most-frequent- spec: arguments: diff --git a/sdk/python/tests/compiler/testdata/param_op_transform.yaml b/sdk/python/tests/compiler/testdata/param_op_transform.yaml index a9f800c2acd..7a394c755e2 100644 --- a/sdk/python/tests/compiler/testdata/param_op_transform.yaml +++ b/sdk/python/tests/compiler/testdata/param_op_transform.yaml @@ -1,6 +1,12 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: + annotations: + pipelines.kubeflow.org/pipeline_spec: '{"description": "Test that parameters used + in Op transformation functions as pod labels would be correcly identified and + set as arguments in he generated yaml", "inputs": [{"default": null, "description": + "", "name": "param", "type": ""}], "name": "Parameters in Op transformation + functions"}' generateName: parameters-in-op-transformation-functions- spec: arguments: diff --git a/sdk/python/tests/compiler/testdata/param_substitutions.yaml b/sdk/python/tests/compiler/testdata/param_substitutions.yaml index b29d212be15..14ef651d68f 100644 --- a/sdk/python/tests/compiler/testdata/param_substitutions.yaml +++ b/sdk/python/tests/compiler/testdata/param_substitutions.yaml @@ -1,6 +1,9 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: + annotations: + pipelines.kubeflow.org/pipeline_spec: '{"description": "Test the same PipelineParam + getting substituted in multiple places", "inputs": [], "name": "Param Substitutions"}' generateName: param-substitutions- spec: arguments: diff --git a/sdk/python/tests/compiler/testdata/pipelineparams.yaml b/sdk/python/tests/compiler/testdata/pipelineparams.yaml index f5c5c30222a..2f145d3ba2e 100644 --- a/sdk/python/tests/compiler/testdata/pipelineparams.yaml +++ b/sdk/python/tests/compiler/testdata/pipelineparams.yaml @@ -14,6 +14,11 @@ --- apiVersion: argoproj.io/v1alpha1 metadata: + annotations: + pipelines.kubeflow.org/pipeline_spec: '{"description": "A pipeline with multiple + pipeline params.", "inputs": [{"default": "latest", "description": "", "name": + "tag", "type": ""}, {"default": 10, "description": "", "name": "sleep_ms", "type": + ""}], "name": "PipelineParams"}' generateName: pipelineparams- spec: entrypoint: pipelineparams diff --git a/sdk/python/tests/compiler/testdata/preemptible_tpu_gpu.yaml b/sdk/python/tests/compiler/testdata/preemptible_tpu_gpu.yaml index 172adfff15a..48d426336d6 100644 --- a/sdk/python/tests/compiler/testdata/preemptible_tpu_gpu.yaml +++ b/sdk/python/tests/compiler/testdata/preemptible_tpu_gpu.yaml @@ -1,6 +1,8 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: + annotations: + pipelines.kubeflow.org/pipeline_spec: '{"description": "shows how to use dsl.Condition.", "inputs": [], "name": "pipeline flip coin"}' generateName: pipeline-flip-coin- spec: arguments: diff --git a/sdk/python/tests/compiler/testdata/recursive_do_while.yaml b/sdk/python/tests/compiler/testdata/recursive_do_while.yaml index f3f18980b57..9da2c3c65c0 100644 --- a/sdk/python/tests/compiler/testdata/recursive_do_while.yaml +++ b/sdk/python/tests/compiler/testdata/recursive_do_while.yaml @@ -1,6 +1,9 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: + annotations: + pipelines.kubeflow.org/pipeline_spec: '{"description": "shows how to use graph_component.", + "inputs": [], "name": "pipeline flip coin"}' generateName: pipeline-flip-coin- spec: arguments: diff --git a/sdk/python/tests/compiler/testdata/recursive_while.yaml b/sdk/python/tests/compiler/testdata/recursive_while.yaml index d07fa97722b..3ec59455a18 100644 --- a/sdk/python/tests/compiler/testdata/recursive_while.yaml +++ b/sdk/python/tests/compiler/testdata/recursive_while.yaml @@ -1,6 +1,10 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: + annotations: + pipelines.kubeflow.org/pipeline_spec: '{"description": "shows how to use dsl.Condition.", + "inputs": [{"default": 12, "description": "", "name": "maxVal", "type": ""}], + "name": "pipeline flip coin"}' generateName: pipeline-flip-coin- spec: arguments: diff --git a/sdk/python/tests/compiler/testdata/resourceop_basic.yaml b/sdk/python/tests/compiler/testdata/resourceop_basic.yaml index 2da6dfd2188..a12332fff6f 100644 --- a/sdk/python/tests/compiler/testdata/resourceop_basic.yaml +++ b/sdk/python/tests/compiler/testdata/resourceop_basic.yaml @@ -1,6 +1,11 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: + annotations: + pipelines.kubeflow.org/pipeline_spec: '{"description": "A Basic Example on ResourceOp + Usage.", "inputs": [{"default": null, "description": "", "name": "username", + "type": ""}, {"default": null, "description": "", "name": "password", "type": + ""}], "name": "ResourceOp Basic"}' generateName: resourceop-basic- spec: arguments: diff --git a/sdk/python/tests/compiler/testdata/retry.yaml b/sdk/python/tests/compiler/testdata/retry.yaml index 8e9c927a1a0..5c40177f48d 100644 --- a/sdk/python/tests/compiler/testdata/retry.yaml +++ b/sdk/python/tests/compiler/testdata/retry.yaml @@ -1,6 +1,10 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: + annotations: + pipelines.kubeflow.org/pipeline_spec: '{"description": "shows how to use ContainerOp + set_retry().", "inputs": [], "name": "pipeline includes two steps which fail + randomly."}' generateName: pipeline-includes-two-steps-which-fail-randomly- spec: arguments: diff --git a/sdk/python/tests/compiler/testdata/sidecar.yaml b/sdk/python/tests/compiler/testdata/sidecar.yaml index be267bfb038..924a382244d 100644 --- a/sdk/python/tests/compiler/testdata/sidecar.yaml +++ b/sdk/python/tests/compiler/testdata/sidecar.yaml @@ -13,6 +13,9 @@ # limitations under the License. kind: Workflow metadata: + annotations: + pipelines.kubeflow.org/pipeline_spec: '{"description": "A pipeline with sidecars.", + "inputs": [], "name": "Sidecar"}' generateName: sidecar- apiVersion: argoproj.io/v1alpha1 spec: diff --git a/sdk/python/tests/compiler/testdata/timeout.yaml b/sdk/python/tests/compiler/testdata/timeout.yaml index 606eb2bbc89..335668ef5eb 100644 --- a/sdk/python/tests/compiler/testdata/timeout.yaml +++ b/sdk/python/tests/compiler/testdata/timeout.yaml @@ -1,6 +1,10 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: + annotations: + pipelines.kubeflow.org/pipeline_spec: '{"description": "shows how to use ContainerOp + set_retry().", "inputs": [], "name": "pipeline includes two steps which fail + randomly."}' generateName: pipeline-includes-two-steps-which-fail-randomly- spec: activeDeadlineSeconds: 50 diff --git a/sdk/python/tests/compiler/testdata/volume.yaml b/sdk/python/tests/compiler/testdata/volume.yaml index 73175df6e91..470fc321288 100644 --- a/sdk/python/tests/compiler/testdata/volume.yaml +++ b/sdk/python/tests/compiler/testdata/volume.yaml @@ -14,6 +14,9 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: + annotations: + pipelines.kubeflow.org/pipeline_spec: '{"description": "A pipeline with volume.", + "inputs": [], "name": "Volume"}' generateName: volume- spec: arguments: diff --git a/sdk/python/tests/compiler/testdata/volume_snapshotop_rokurl.yaml b/sdk/python/tests/compiler/testdata/volume_snapshotop_rokurl.yaml index 20106baef00..4b6c5bb9344 100644 --- a/sdk/python/tests/compiler/testdata/volume_snapshotop_rokurl.yaml +++ b/sdk/python/tests/compiler/testdata/volume_snapshotop_rokurl.yaml @@ -1,6 +1,10 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: + annotations: + pipelines.kubeflow.org/pipeline_spec: '{"description": "The fifth example of the + design doc.", "inputs": [{"default": null, "description": "", "name": "rok_url", + "type": ""}], "name": "VolumeSnapshotOp RokURL"}' generateName: volumesnapshotop-rokurl- spec: arguments: diff --git a/sdk/python/tests/compiler/testdata/volume_snapshotop_sequential.yaml b/sdk/python/tests/compiler/testdata/volume_snapshotop_sequential.yaml index d6a772ad94a..8d03485c2f0 100644 --- a/sdk/python/tests/compiler/testdata/volume_snapshotop_sequential.yaml +++ b/sdk/python/tests/compiler/testdata/volume_snapshotop_sequential.yaml @@ -1,6 +1,10 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: + annotations: + pipelines.kubeflow.org/pipeline_spec: '{"description": "The fourth example of + the design doc.", "inputs": [{"default": null, "description": "", "name": "url", + "type": ""}], "name": "VolumeSnapshotOp Sequential"}' generateName: volumesnapshotop-sequential- spec: arguments: diff --git a/sdk/python/tests/compiler/testdata/volumeop_basic.yaml b/sdk/python/tests/compiler/testdata/volumeop_basic.yaml index 4172075e989..eda060bc615 100644 --- a/sdk/python/tests/compiler/testdata/volumeop_basic.yaml +++ b/sdk/python/tests/compiler/testdata/volumeop_basic.yaml @@ -1,6 +1,10 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: + annotations: + pipelines.kubeflow.org/pipeline_spec: '{"description": "A Basic Example on VolumeOp + Usage.", "inputs": [{"default": null, "description": "", "name": "size", "type": + ""}], "name": "VolumeOp Basic"}' generateName: volumeop-basic- spec: arguments: diff --git a/sdk/python/tests/compiler/testdata/volumeop_dag.yaml b/sdk/python/tests/compiler/testdata/volumeop_dag.yaml index 3f8836927a8..31158a458cc 100644 --- a/sdk/python/tests/compiler/testdata/volumeop_dag.yaml +++ b/sdk/python/tests/compiler/testdata/volumeop_dag.yaml @@ -1,6 +1,9 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: + annotations: + pipelines.kubeflow.org/pipeline_spec: '{"description": "The second example of + the design doc.", "inputs": [], "name": "Volume Op DAG"}' generateName: volume-op-dag- spec: arguments: diff --git a/sdk/python/tests/compiler/testdata/volumeop_parallel.yaml b/sdk/python/tests/compiler/testdata/volumeop_parallel.yaml index 83e11f021b9..064b16a1432 100644 --- a/sdk/python/tests/compiler/testdata/volumeop_parallel.yaml +++ b/sdk/python/tests/compiler/testdata/volumeop_parallel.yaml @@ -1,6 +1,9 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: + annotations: + pipelines.kubeflow.org/pipeline_spec: '{"description": "The first example of the + design doc.", "inputs": [], "name": "VolumeOp Parallel"}' generateName: volumeop-parallel- spec: arguments: diff --git a/sdk/python/tests/compiler/testdata/volumeop_sequential.yaml b/sdk/python/tests/compiler/testdata/volumeop_sequential.yaml index 8fd38be9a00..827d4b2445f 100644 --- a/sdk/python/tests/compiler/testdata/volumeop_sequential.yaml +++ b/sdk/python/tests/compiler/testdata/volumeop_sequential.yaml @@ -1,6 +1,9 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: + annotations: + pipelines.kubeflow.org/pipeline_spec: '{"description": "The third example of the + design doc.", "inputs": [], "name": "VolumeOp Sequential"}' generateName: volumeop-sequential- spec: arguments: