SDK - Preserving the pipeline metadata in the compiled Workflow

sdk/python/kfp/compiler/compiler.py

@@ -646,6 +646,10 @@ def _compile(self, pipeline_func):
     p.ops = sanitized_ops
 
     workflow = self._create_pipeline_workflow(args_list_with_defaults, p, p.conf.op_transformers)
+
+    import json
+    workflow.setdefault('metadata', {}).setdefault('annotations', {})['kubeflow.org/pipelines/pipeline_spec'] = json.dumps(pipeline_meta.to_dict())
+
     return workflow
 
   def compile(self, pipeline_func, package_path, type_check=True):

sdk/python/tests/compiler/testdata/artifact_location.yaml

@@ -14,6 +14,11 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
+  annotations:
+    kubeflow.org/pipelines/pipeline_spec: '{"name": "foo", "description": "hello world",
+      "inputs": [{"name": "tag", "description": "", "type": "", "default": null},
+      {"name": "namespace", "description": "", "type": "", "default": "kubeflow"},
+      {"name": "bucket", "description": "", "type": "", "default": "foobar"}]}'
   generateName: foo-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/basic.yaml

@@ -14,6 +14,11 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
+  annotations:
+    kubeflow.org/pipelines/pipeline_spec: '{"name": "Save Most Frequent", "description":
+      "Get Most Frequent Word and Save to GCS", "inputs": [{"name": "message", "description":
+      "", "type": "", "default": null}, {"name": "outputpath", "description": "",
+      "type": "", "default": null}]}'
   generateName: save-most-frequent-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/coin.yaml

@@ -14,6 +14,9 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
+  annotations:
+    kubeflow.org/pipelines/pipeline_spec: '{"name": "pipeline flip coin", "description":
+      "shows how to use dsl.Condition.", "inputs": []}'
   generateName: pipeline-flip-coin-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/compose.yaml

@@ -14,6 +14,11 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
+  annotations:
+    kubeflow.org/pipelines/pipeline_spec: '{"name": "Download and Save Most Frequent",
+      "description": "Download and Get Most Frequent Word and Save to GCS", "inputs":
+      [{"name": "url", "description": "", "type": "", "default": null}, {"name": "outputpath",
+      "description": "", "type": "", "default": null}]}'
   generateName: download-and-save-most-frequent-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/default_value.yaml

@@ -14,6 +14,10 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
+  annotations:
+    kubeflow.org/pipelines/pipeline_spec: '{"name": "Default Value", "description":
+      "A pipeline with parameter and default value.", "inputs": [{"name": "url", "description":
+      "", "type": "", "default": "gs://ml-pipeline/shakespeare1.txt"}]}'
   generateName: default-value-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/imagepullsecret.yaml

@@ -1,6 +1,10 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
+  annotations:
+    kubeflow.org/pipelines/pipeline_spec: '{"name": "Save Most Frequent", "description":
+      "Get Most Frequent Word and Save to GCS", "inputs": [{"name": "message", "description":
+      "", "type": "", "default": null}]}'
   generateName: save-most-frequent-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/immediate_value.yaml

@@ -14,6 +14,9 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
+  annotations:
+    kubeflow.org/pipelines/pipeline_spec: '{"name": "Immediate Value", "description":
+      "A pipeline with parameter values hard coded", "inputs": []}'
   generateName: immediate-value-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/param_substitutions.yaml

@@ -1,6 +1,10 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
+  annotations:
+    kubeflow.org/pipelines/pipeline_spec: '{"name": "Param Substitutions", "description":
+      "Test the same PipelineParam getting substituted in multiple places", "inputs":
+      []}'
   generateName: param-substitutions-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/pipelineparams.yaml

@@ -14,6 +14,11 @@
 ---
 apiVersion: argoproj.io/v1alpha1
 metadata:
+  annotations:
+    kubeflow.org/pipelines/pipeline_spec: '{"name": "PipelineParams", "description":
+      "A pipeline with multiple pipeline params.", "inputs": [{"name": "tag", "description":
+      "", "type": "", "default": "latest"}, {"name": "sleep_ms", "description": "",
+      "type": "", "default": 10}]}'
   generateName: pipelineparams-
 spec:
   entrypoint: pipelineparams

sdk/python/tests/compiler/testdata/recursive_do_while.yaml

@@ -1,6 +1,9 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
+  annotations:
+    kubeflow.org/pipelines/pipeline_spec: '{"name": "pipeline flip coin", "description":
+      "shows how to use graph_component.", "inputs": []}'
   generateName: pipeline-flip-coin-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/recursive_while.yaml

@@ -1,6 +1,9 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
+  annotations:
+    kubeflow.org/pipelines/pipeline_spec: '{"name": "pipeline flip coin", "description":
+      "shows how to use dsl.Condition.", "inputs": []}'
   generateName: pipeline-flip-coin-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/resourceop_basic.yaml

@@ -1,6 +1,11 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
+  annotations:
+    kubeflow.org/pipelines/pipeline_spec: '{"name": "ResourceOp Basic", "description":
+      "A Basic Example on ResourceOp Usage.", "inputs": [{"name": "username", "description":
+      "", "type": "", "default": null}, {"name": "password", "description": "", "type":
+      "", "default": null}]}'
   generateName: resourceop-basic-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/retry.yaml

@@ -1,6 +1,10 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
+  annotations:
+    kubeflow.org/pipelines/pipeline_spec: '{"name": "pipeline includes two steps which
+      fail randomly.", "description": "shows how to use ContainerOp set_retry().",
+      "inputs": []}'
   generateName: pipeline-includes-two-steps-which-fail-randomly-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/sidecar.yaml

@@ -13,6 +13,9 @@
 # limitations under the License.
 kind: Workflow
 metadata:
+  annotations:
+    kubeflow.org/pipelines/pipeline_spec: '{"name": "Sidecar", "description": "A pipeline
+      with sidecars.", "inputs": []}'
   generateName: sidecar-
 apiVersion: argoproj.io/v1alpha1
 spec:

sdk/python/tests/compiler/testdata/volume.yaml

@@ -14,6 +14,9 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
+  annotations:
+    kubeflow.org/pipelines/pipeline_spec: '{"name": "Volume", "description": "A pipeline
+      with volume.", "inputs": []}'
   generateName: volume-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/volume_snapshotop_rokurl.yaml

@@ -1,6 +1,10 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
+  annotations:
+    kubeflow.org/pipelines/pipeline_spec: '{"name": "VolumeSnapshotOp RokURL", "description":
+      "The fifth example of the design doc.", "inputs": [{"name": "rok_url", "description":
+      "", "type": "", "default": null}]}'
   generateName: volumesnapshotop-rokurl-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/volume_snapshotop_sequential.yaml

@@ -1,6 +1,10 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
+  annotations:
+    kubeflow.org/pipelines/pipeline_spec: '{"name": "VolumeSnapshotOp Sequential",
+      "description": "The fourth example of the design doc.", "inputs": [{"name":
+      "url", "description": "", "type": "", "default": null}]}'
   generateName: volumesnapshotop-sequential-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/volumeop_basic.yaml

@@ -1,6 +1,10 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
+  annotations:
+    kubeflow.org/pipelines/pipeline_spec: '{"name": "VolumeOp Basic", "description":
+      "A Basic Example on VolumeOp Usage.", "inputs": [{"name": "size", "description":
+      "", "type": "", "default": null}]}'
   generateName: volumeop-basic-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/volumeop_dag.yaml

@@ -1,6 +1,9 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
+  annotations:
+    kubeflow.org/pipelines/pipeline_spec: '{"name": "Volume Op DAG", "description":
+      "The second example of the design doc.", "inputs": []}'
   generateName: volume-op-dag-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/volumeop_parallel.yaml

@@ -1,6 +1,9 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
+  annotations:
+    kubeflow.org/pipelines/pipeline_spec: '{"name": "VolumeOp Parallel", "description":
+      "The first example of the design doc.", "inputs": []}'
   generateName: volumeop-parallel-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/volumeop_sequential.yaml

@@ -1,6 +1,9 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
+  annotations:
+    kubeflow.org/pipelines/pipeline_spec: '{"name": "VolumeOp Sequential", "description":
+      "The third example of the design doc.", "inputs": []}'
   generateName: volumeop-sequential-
 spec:
   arguments: