SDK - Persisting all output values

Currently, the parameter output values are not saved to storage and their values are lost as soon as garbage collector removes the workflow object.
This change makes is so the parameter output values are persisted.

sdk/python/kfp/compiler/_op_to_template.py

@@ -190,6 +190,8 @@ def _op_to_template(op: BaseOp):
     if isinstance(op, dsl.ContainerOp):
         # default output artifacts
         output_artifact_paths = OrderedDict(op.output_artifact_paths)
+        # This should have been as easy as output_artifact_paths.update(op.file_outputs), but the _outputs_to_json function changes the output names and we must do the same here, so that the names are the same
+        output_artifact_paths.update(sorted(((param.full_name, processed_op.file_outputs[param.name]) for param in processed_op.outputs.values()), key=lambda x: x[0]))
 
         output_artifacts = [
              K8sHelper.convert_k8s_obj_to_json(

sdk/python/tests/compiler/compiler_tests.py

@@ -98,6 +98,12 @@ def my_pipeline(msg1, json, kind, msg2='value2'):
           ]},
         'name': 'echo',
         'outputs': {
+          'artifacts': [
+            {
+              'name': 'echo-merged',
+              'path': '/tmp/message.txt',
+            },
+          ],
           'parameters': [
             {'name': 'echo-merged',
             'valueFrom': {'path': '/tmp/message.txt'}
@@ -571,7 +577,8 @@ def _test_op_to_template_yaml(self, ops, file_base_name):
     compiled_template = compiler._op_to_template._op_to_template(ops)
 
     del compiled_template['name'], expected['name']
-    del compiled_template['outputs']['parameters'][0]['name'], expected['outputs']['parameters'][0]['name']
+    for output in compiled_template['outputs'].get('parameters', []) + compiled_template['outputs'].get('artifacts', []) + expected['outputs'].get('parameters', []) + expected['outputs'].get('artifacts', []):
+      del output['name']
     assert compiled_template == expected
 
   def test_tolerations(self):

sdk/python/tests/compiler/testdata/basic.yaml

@@ -73,6 +73,9 @@ spec:
       - name: message
     name: get-frequent
     outputs:
+      artifacts:
+      - name: get-frequent-word
+        path: /tmp/message.txt
       parameters:
       - name: get-frequent-word
         valueFrom:

sdk/python/tests/compiler/testdata/basic_no_decorator.yaml

@@ -74,6 +74,9 @@ spec:
           - name: message
       name: get-frequent
       outputs:
+        artifacts:
+          - name: get-frequent-word
+            path: /tmp/message.txt
         parameters:
           - name: get-frequent-word
             valueFrom:

sdk/python/tests/compiler/testdata/coin.yaml

@@ -91,6 +91,9 @@ spec:
       image: python:alpine3.6
     name: flip
     outputs:
+      artifacts:
+      - name: flip-output
+        path: /tmp/output
       parameters:
       - name: flip-output
         valueFrom:
@@ -105,6 +108,9 @@ spec:
       image: python:alpine3.6
     name: flip-again
     outputs:
+      artifacts:
+      - name: flip-again-output
+        path: /tmp/output
       parameters:
       - name: flip-again-output
         valueFrom:

sdk/python/tests/compiler/testdata/compose.yaml

@@ -37,6 +37,9 @@ spec:
       - name: url
     name: download
     outputs:
+      artifacts:
+      - name: download-downloaded
+        path: /tmp/results.txt
       parameters:
       - name: download-downloaded
         valueFrom:
@@ -85,6 +88,9 @@ spec:
       - name: download-downloaded
     name: get-frequent
     outputs:
+      artifacts:
+      - name: get-frequent-word
+        path: /tmp/message.txt
       parameters:
       - name: get-frequent-word
         valueFrom:

sdk/python/tests/compiler/testdata/default_value.yaml

@@ -57,6 +57,9 @@ spec:
       - name: url
     name: download
     outputs:
+      artifacts:
+      - name: download-downloaded
+        path: /tmp/results.txt
       parameters:
       - name: download-downloaded
         valueFrom:

sdk/python/tests/compiler/testdata/imagepullsecrets.yaml

@@ -26,6 +26,9 @@ spec:
       - name: message
     name: get-frequent
     outputs:
+      artifacts:
+      - name: get-frequent-word
+        path: /tmp/message.txt
       parameters:
       - name: get-frequent-word
         valueFrom:

sdk/python/tests/compiler/testdata/pipelineparams.yaml

@@ -41,6 +41,9 @@ spec:
       - sh
       - "-c"
     outputs:
+      artifacts:
+      - name: download-downloaded
+        path: "/tmp/results.txt"
       parameters:
       - name: download-downloaded
         valueFrom:

sdk/python/tests/compiler/testdata/preemptible_tpu_gpu.yaml

@@ -25,6 +25,9 @@ spec:
       nodeSelector:
         cloud.google.com/gke-preemptible: 'true'
       outputs:
+        artifacts:
+          - name: flip-output
+            path: /tmp/output
         parameters:
           - name: flip-output
             valueFrom:

sdk/python/tests/compiler/testdata/recursive_do_while.yaml

@@ -33,6 +33,9 @@ spec:
       image: python:alpine3.6
     name: flip
     outputs:
+      artifacts:
+      - name: flip-output
+        path: /tmp/output
       parameters:
       - name: flip-output
         valueFrom:
@@ -47,6 +50,9 @@ spec:
       image: python:alpine3.6
     name: flip-2
     outputs:
+      artifacts:
+      - name: flip-2-output
+        path: /tmp/output
       parameters:
       - name: flip-2-output
         valueFrom:
@@ -61,6 +67,9 @@ spec:
       image: python:alpine3.6
     name: flip-3
     outputs:
+      artifacts:
+      - name: flip-3-output
+        path: /tmp/output
       parameters:
       - name: flip-3-output
         valueFrom:

sdk/python/tests/compiler/testdata/recursive_while.yaml

@@ -55,6 +55,9 @@ spec:
       image: python:alpine3.6
     name: flip
     outputs:
+      artifacts:
+      - name: flip-output
+        path: /tmp/output
       parameters:
       - name: flip-output
         valueFrom:
@@ -69,6 +72,9 @@ spec:
       image: python:alpine3.6
     name: flip-2
     outputs:
+      artifacts:
+      - name: flip-2-output
+        path: /tmp/output
       parameters:
       - name: flip-2-output
         valueFrom:
@@ -83,6 +89,9 @@ spec:
       image: python:alpine3.6
     name: flip-3
     outputs:
+      artifacts:
+      - name: flip-3-output
+        path: /tmp/output
       parameters:
       - name: flip-3-output
         valueFrom:

sdk/python/tests/compiler/testdata/sidecar.yaml

@@ -23,6 +23,9 @@ spec:
     parameters: []
   templates:
   - outputs:
+      artifacts:
+      - name: download-downloaded
+        path: "/tmp/results.txt"
       parameters:
       - name: download-downloaded
         valueFrom:

sdk/python/tests/compiler/testdata/tolerations.yaml

@@ -20,6 +20,9 @@ spec:
     parameters: []
   templates:
   - outputs:
+      artifacts:
+      - name: download-downloaded
+        path: "/tmp/results.txt"
       parameters:
       - name: download-downloaded
         valueFrom:

sdk/python/tests/compiler/testdata/volume.yaml

@@ -41,6 +41,9 @@ spec:
         name: gcp-credentials
     name: download
     outputs:
+      artifacts:
+      - name: download-downloaded
+        path: /tmp/results.txt
       parameters:
       - name: download-downloaded
         valueFrom: