Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

SDK - Compiler - Include the SDK version information in the compiled workflows #3583

Merged
merged 3 commits into from
Apr 25, 2020
Merged

SDK - Compiler - Include the SDK version information in the compiled workflows #3583

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c5f9937
SDK - Compiler - Include the SDK version information in the compiled …
Ark-kun Apr 22, 2020
68e5850
Fixed the unit tests
Ark-kun Apr 22, 2020
b901baf
Removed the sdk_version annotation.
Ark-kun Apr 25, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 12 additions & 2 deletions sdk/python/kfp/compiler/compiler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import datetime
import json
from collections import defaultdict
from deprecated import deprecated
Expand All @@ -20,6 +21,7 @@
import zipfile
from typing import Callable, Set, List, Text, Dict, Tuple, Any, Union, Optional

import kfp
from kfp.dsl import _for_loop

from .. import dsl
Expand Down Expand Up @@ -828,8 +830,16 @@ def _create_workflow(self,
from ._data_passing_rewriter import fix_big_data_passing
workflow = fix_big_data_passing(workflow)

import json
workflow.setdefault('metadata', {}).setdefault('annotations', {})['pipelines.kubeflow.org/pipeline_spec'] = json.dumps(pipeline_meta.to_dict(), sort_keys=True)
metadata = workflow.setdefault('metadata', {})
annotations = metadata.setdefault('annotations', {})

annotations['pipelines.kubeflow.org/kfp_sdk_version'] = kfp.__version__
annotations['pipelines.kubeflow.org/pipeline_compilation_time'] = datetime.datetime.now().isoformat()
annotations['pipelines.kubeflow.org/pipeline_spec'] = json.dumps(pipeline_meta.to_dict(), sort_keys=True)
numerology marked this conversation as resolved.
Show resolved Hide resolved

# Labels might be logged better than annotations so adding some information here as well
labels = metadata.setdefault('labels', {})
labels['pipelines.kubeflow.org/kfp_sdk_version'] = kfp.__version__

return workflow

Expand Down
24 changes: 12 additions & 12 deletions sdk/python/tests/compiler/compiler_tests.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,9 @@ def test_basic_workflow(self):
golden = yaml.safe_load(f)
compiled = self._get_yaml_from_zip(package_path)

for workflow in golden, compiled:
del workflow['metadata']

self.maxDiff = None
# Comment next line for generating golden yaml.
self.assertEqual(golden, compiled)
Expand Down Expand Up @@ -205,8 +208,7 @@ def test_basic_workflow_without_decorator(self):
golden = yaml.safe_load(f)

for workflow in golden, compiled_workflow:
annotations = workflow['metadata']['annotations']
del annotations['pipelines.kubeflow.org/pipeline_spec']
del workflow['metadata']

self.assertEqual(golden, compiled_workflow)
finally:
Expand All @@ -232,8 +234,7 @@ def test_composing_workflow(self):
compiled = self._get_yaml_from_zip(compose_package_path)

for workflow in golden, compiled:
annotations = workflow['metadata']['annotations']
del annotations['pipelines.kubeflow.org/pipeline_spec']
del workflow['metadata']

self.maxDiff = None
# Comment next line for generating golden yaml.
Expand Down Expand Up @@ -263,8 +264,7 @@ def test_package_compile(self):
compiled = self._get_yaml_from_zip(target_zip)

for workflow in golden, compiled:
annotations = workflow['metadata']['annotations']
del annotations['pipelines.kubeflow.org/pipeline_spec']
del workflow['metadata']

self.maxDiff = None
self.assertEqual(golden, compiled)
Expand All @@ -285,8 +285,7 @@ def _test_py_compile_zip(self, file_base_name):
compiled = self._get_yaml_from_zip(target_zip)

for workflow in golden, compiled:
annotations = workflow['metadata']['annotations']
del annotations['pipelines.kubeflow.org/pipeline_spec']
del workflow['metadata']

self.maxDiff = None
self.assertEqual(golden, compiled)
Expand All @@ -306,8 +305,7 @@ def _test_py_compile_targz(self, file_base_name):
compiled = self._get_yaml_from_tar(target_tar)

for workflow in golden, compiled:
annotations = workflow['metadata']['annotations']
del annotations['pipelines.kubeflow.org/pipeline_spec']
del workflow['metadata']

self.maxDiff = None
self.assertEqual(golden, compiled)
Expand All @@ -329,8 +327,7 @@ def _test_py_compile_yaml(self, file_base_name):
compiled = yaml.safe_load(f)

for workflow in golden, compiled:
annotations = workflow['metadata']['annotations']
del annotations['pipelines.kubeflow.org/pipeline_spec']
del workflow['metadata']

self.maxDiff = None
self.assertEqual(golden, compiled)
Expand All @@ -354,6 +351,9 @@ def _test_sample_py_compile_yaml(self, file_base_name):
with open(os.path.join(test_data_dir, target_yaml), 'r') as f:
compiled = yaml.safe_load(f)

for workflow in golden, compiled:
del workflow['metadata']

self.maxDiff = None
self.assertEqual(golden, compiled)
finally:
Expand Down