From 592ca36f22a58aeffbad5902b999678cc43e1c96 Mon Sep 17 00:00:00 2001 From: Ning Gao Date: Thu, 28 Feb 2019 17:27:32 -0800 Subject: [PATCH 1/6] add core types and type checking function --- sdk/python/kfp/dsl/_types.py | 206 +++++++++++++++++++++++++++++ sdk/python/tests/dsl/main.py | 3 +- sdk/python/tests/dsl/type_tests.py | 92 +++++++++++++ 3 files changed, 300 insertions(+), 1 deletion(-) create mode 100644 sdk/python/kfp/dsl/_types.py create mode 100644 sdk/python/tests/dsl/type_tests.py diff --git a/sdk/python/kfp/dsl/_types.py b/sdk/python/kfp/dsl/_types.py new file mode 100644 index 00000000000..e5c467e212a --- /dev/null +++ b/sdk/python/kfp/dsl/_types.py @@ -0,0 +1,206 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class MetaType: + '''MetaType is a base type for all scalar and artifact types. + ''' + pass + +# Primitive Types +class MetaInt(MetaType): + openapi_schema_validator = '''{ + "type": "integer" + }''' + +def Integer(attr={}): + return type('Integer', (MetaInt, ), attr) + +class MetaString(MetaType): + openapi_schema_validator = '''{ + "type": "string" + }''' + +def String(attr={}): + return type('String', (MetaString, ), attr) + +class MetaFloat(MetaType): + openapi_schema_validator = '''{ + "type": "number" + }''' + +def Float(attr={}): + return type('Float', (MetaFloat, ), attr) + +class MetaBool(MetaType): + openapi_schema_validator = '''{ + "type": "boolean" + }''' + +def Bool(attr={}): + return type('Bool', (MetaBool, ), attr) + +class MetaList(MetaType): + openapi_schema_validator = '''{ + "type": "array" + }''' + +def List(attr={}): + return type('List', (MetaList, ), attr) + +class MetaDict(MetaType): + openapi_schema_validator = '''{ + "type": "object", + }''' + +def Dict(attr={}): + return type('Dict', (MetaDict, ), attr) + +# GCP Types +class MetaGCSPath(MetaType): + openapi_schema_validator = '''{ + "type": "string", + "pattern": "^gs://$" + } + }''' + # path_type describes the paths, for example, bucket, directory, file, etc. + path_type = '' + # file_type describes the files, for example, JSON, CSV, etc. + file_type = '' + +def GCSPath(attr={}): + return type('GCSPath', (MetaGCSPath, ), attr) + +class MetaGCRPath(MetaType): + openapi_schema_validator = '''{ + "type": "string", + "pattern": "^(us.|eu.|asia.)?gcr\\.io/.*$" + } + }''' + +def GCRPath(attr={}): + return type('GCRPath', (MetaGCRPath, ), attr) + +class MetaGCPRegion(MetaType): + openapi_schema_validator = '''{ + "type": "string", + "enum": ["asia-east1","asia-east2","asia-northeast1", + "asia-south1","asia-southeast1","australia-southeast1", + "europe-north1","europe-west1","europe-west2", + "europe-west3","europe-west4","northamerica-northeast1", + "southamerica-east1","us-central1","us-east1", + "us-east4","us-west1", "us-west4" ] + }''' + +def GCPRegion(attr={}): + return type('GCPRegion', (MetaGCPRegion, ), attr) + +class MetaGCPProjectID(MetaType): + '''MetaGCPProjectID: GCP project id''' + openapi_schema_validator = '''{ + "type": "string" + }''' + +def GCPProjectID(attr={}): + return type('GCPProjectID', (MetaGCPProjectID, ), attr) + +# General Types +class MetaLocalPath(MetaType): + #TODO: add restriction to path + openapi_schema_validator = '''{ + "type": "string" + }''' + +def LocalPath(attr={}): + return type('LocalPath', (MetaLocalPath, ), attr) + +class InconsistentTypeException(Exception): + '''InconsistencyTypeException is raised when two types are not consistent''' + pass + +def _check_valid_dict(payload): + '''_check_valid_dict_type checks whether a dict is a correct serialization of a type + Args: + payload(dict) + ''' + if not isinstance(payload, dict) or len(payload) != 1: + return False + for type_name in payload: + if not isinstance(payload[type_name], dict): + return False + property_types = (int, str, float, bool) + for property_name in payload[type_name]: + if not isinstance(property_name, property_types) or not isinstance(payload[type_name][property_name], property_types): + return False + return True + +def _class_to_dict(payload): + '''serialize_type serializes the type instance into a json string + Args: + payload(type): A class that describes a type + + Return: + dict + ''' + attrs = set([i for i in dir(payload) if not i.startswith('__')]) + json_dict = {payload.__name__: {}} + for attr in attrs: + json_dict[payload.__name__][attr] = getattr(payload, attr) + return json_dict + +def _str_to_dict(payload): + import json + json_dict = json.loads(payload) + if not _check_valid_dict(json_dict): + raise ValueError(payload + ' is not a valid type string') + return json_dict + +def _check_dict_types(typeA, typeB): + '''_check_type_types checks the type consistency. + Args: + typeA (dict): A dict that describes a type from the upstream component output + typeB (dict): A dict that describes a type from the downstream component input + ''' + typeA_name,_ = list(typeA.items())[0] + typeB_name,_ = list(typeB.items())[0] + if typeA_name != typeB_name: + return False + type_name = typeA_name + for type_property in typeA[type_name]: + if type_property not in typeB[type_name]: + print(type_name + ' has a property ' + str(type_property) + ' that the latter does not.') + return False + if typeA[type_name][type_property] != typeB[type_name][type_property]: + print(type_name + ' has a property ' + str(type_property) + ' with value: ' + + str(typeA[type_name][type_property]) + ' and ' + + str(typeB[type_name][type_property])) + return False + return True + +def check_types(typeA, typeB): + '''check_types checks the type consistency. + For each of the attribute in typeA, there is the same attribute in typeB with the same value. + However, typeB could contain more attributes that typeA does not contain. + Args: + typeA (type/str/dict): it describes a type from the upstream component output + typeB (type/str/dict): it describes a type from the downstream component input + ''' + if isinstance(typeA, type): + typeA = _class_to_dict(typeA) + elif isinstance(typeA, str): + typeA = _str_to_dict(typeA) + if isinstance(typeB, type): + typeB = _class_to_dict(typeB) + elif isinstance(typeB, str): + typeB = _str_to_dict(typeB) + return _check_dict_types(typeA, typeB) \ No newline at end of file diff --git a/sdk/python/tests/dsl/main.py b/sdk/python/tests/dsl/main.py index 7e194cf386f..6192a944d88 100644 --- a/sdk/python/tests/dsl/main.py +++ b/sdk/python/tests/dsl/main.py @@ -20,7 +20,7 @@ import pipeline_param_tests import container_op_tests import ops_group_tests - +import type_tests if __name__ == '__main__': suite = unittest.TestSuite() @@ -28,6 +28,7 @@ suite.addTests(unittest.defaultTestLoader.loadTestsFromModule(pipeline_tests)) suite.addTests(unittest.defaultTestLoader.loadTestsFromModule(container_op_tests)) suite.addTests(unittest.defaultTestLoader.loadTestsFromModule(ops_group_tests)) + suite.addTests(unittest.defaultTestLoader.loadTestsFromModule(type_tests)) runner = unittest.TextTestRunner() if not runner.run(suite).wasSuccessful(): sys.exit(1) diff --git a/sdk/python/tests/dsl/type_tests.py b/sdk/python/tests/dsl/type_tests.py new file mode 100644 index 00000000000..1fed6d20a8d --- /dev/null +++ b/sdk/python/tests/dsl/type_tests.py @@ -0,0 +1,92 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from kfp.dsl._types import _class_to_dict, _str_to_dict, check_types, GCSPath +import unittest + +class TestTypes(unittest.TestCase): + + def test_class_to_dict(self): + """Test _class_to_dict function.""" + gcspath_dict = _class_to_dict(GCSPath({'path_type': 'file', 'file_type': 'csv'})) + golden_dict = { + 'GCSPath': { + 'path_type': 'file', + 'file_type': 'csv', + 'openapi_schema_validator': '''{ + "type": "object", + "properties": { + "path": { + "type": "string", + "pattern": "^gs://$" + } + } + + }''' + } + } + self.assertEqual(golden_dict, gcspath_dict) + + def test_str_to_dict(self): + gcspath_str = '{"GCSPath": {"file_type": "csv", "path_type": "file"}}' + gcspath_dict = _str_to_dict(gcspath_str) + golden_dict = { + 'GCSPath': { + 'path_type': 'file', + 'file_type': 'csv' + } + } + self.assertEqual(golden_dict, gcspath_dict) + gcspath_str = '{"file_type": "csv", "path_type": "file"}' + with self.assertRaises(ValueError): + gcspath_dict = _str_to_dict(gcspath_str) + + def test_check_types(self): + #Core types + typeA = GCSPath({'path_type': 'file', 'file_type': 'csv'}) + typeB = GCSPath({'path_type': 'file', 'file_type': 'csv'}) + self.assertTrue(check_types(typeA, typeB)) + typeA = GCSPath({'path_type': 'file', 'file_type': 'tsv'}) + typeB = GCSPath({'path_type': 'file', 'file_type': 'csv'}) + self.assertFalse(check_types(typeA, typeB)) + + # Custom types + typeA = { + 'A':{ + 'X': 'value1', + 'Y': 'value2' + } + } + typeB = { + 'B':{ + 'X': 'value1', + 'Y': 'value2' + } + } + typeC = { + 'A':{ + 'X': 'value1' + } + } + typeD = { + 'A':{ + 'X': 'value1', + 'Y': 'value3' + } + } + self.assertFalse(check_types(typeA, typeB)) + self.assertFalse(check_types(typeA, typeC)) + self.assertTrue(check_types(typeC, typeA)) + self.assertFalse(check_types(typeA, typeD)) \ No newline at end of file From 3a3fc179692d99e27ac517d47612e5f1c0b76cdc Mon Sep 17 00:00:00 2001 From: Ning Gao Date: Thu, 28 Feb 2019 17:52:28 -0800 Subject: [PATCH 2/6] fix unit test bug --- sdk/python/kfp/dsl/_types.py | 1 - sdk/python/tests/dsl/type_tests.py | 10 ++-------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/sdk/python/kfp/dsl/_types.py b/sdk/python/kfp/dsl/_types.py index e5c467e212a..7d2baaff7b9 100644 --- a/sdk/python/kfp/dsl/_types.py +++ b/sdk/python/kfp/dsl/_types.py @@ -71,7 +71,6 @@ class MetaGCSPath(MetaType): openapi_schema_validator = '''{ "type": "string", "pattern": "^gs://$" - } }''' # path_type describes the paths, for example, bucket, directory, file, etc. path_type = '' diff --git a/sdk/python/tests/dsl/type_tests.py b/sdk/python/tests/dsl/type_tests.py index 1fed6d20a8d..d7791264b6b 100644 --- a/sdk/python/tests/dsl/type_tests.py +++ b/sdk/python/tests/dsl/type_tests.py @@ -26,14 +26,8 @@ def test_class_to_dict(self): 'path_type': 'file', 'file_type': 'csv', 'openapi_schema_validator': '''{ - "type": "object", - "properties": { - "path": { - "type": "string", - "pattern": "^gs://$" - } - } - + "type": "string", + "pattern": "^gs://$" }''' } } From db45632c5f165e89aeb30b1361cb062e6286dcb3 Mon Sep 17 00:00:00 2001 From: Ning Gao Date: Fri, 1 Mar 2019 10:23:44 -0800 Subject: [PATCH 3/6] avoid defining dynamic classes --- sdk/python/kfp/dsl/_types.py | 86 ++++++++++-------------------- sdk/python/tests/dsl/type_tests.py | 19 +++---- 2 files changed, 34 insertions(+), 71 deletions(-) diff --git a/sdk/python/kfp/dsl/_types.py b/sdk/python/kfp/dsl/_types.py index 7d2baaff7b9..71993d06512 100644 --- a/sdk/python/kfp/dsl/_types.py +++ b/sdk/python/kfp/dsl/_types.py @@ -12,85 +12,66 @@ # See the License for the specific language governing permissions and # limitations under the License. -class MetaType: +class BaseType: '''MetaType is a base type for all scalar and artifact types. ''' pass # Primitive Types -class MetaInt(MetaType): +class Integer(BaseType): openapi_schema_validator = '''{ "type": "integer" }''' -def Integer(attr={}): - return type('Integer', (MetaInt, ), attr) - -class MetaString(MetaType): +class String(BaseType): openapi_schema_validator = '''{ "type": "string" }''' -def String(attr={}): - return type('String', (MetaString, ), attr) - -class MetaFloat(MetaType): +class Float(BaseType): openapi_schema_validator = '''{ "type": "number" }''' -def Float(attr={}): - return type('Float', (MetaFloat, ), attr) - -class MetaBool(MetaType): +class Bool(BaseType): openapi_schema_validator = '''{ "type": "boolean" }''' -def Bool(attr={}): - return type('Bool', (MetaBool, ), attr) - -class MetaList(MetaType): +class List(BaseType): openapi_schema_validator = '''{ "type": "array" }''' -def List(attr={}): - return type('List', (MetaList, ), attr) - -class MetaDict(MetaType): +class Dict(BaseType): openapi_schema_validator = '''{ "type": "object", }''' -def Dict(attr={}): - return type('Dict', (MetaDict, ), attr) - # GCP Types -class MetaGCSPath(MetaType): +class GCSPath(BaseType): openapi_schema_validator = '''{ "type": "string", "pattern": "^gs://$" }''' - # path_type describes the paths, for example, bucket, directory, file, etc. - path_type = '' - # file_type describes the files, for example, JSON, CSV, etc. - file_type = '' -def GCSPath(attr={}): - return type('GCSPath', (MetaGCSPath, ), attr) + def __init__(self, path_type='', file_type=''): + ''' + Args + :param path_type: describes the paths, for example, bucket, directory, file, etc + :param file_type: describes the files, for example, JSON, CSV, etc. + ''' + self.path_type = path_type + self.file_type = file_type -class MetaGCRPath(MetaType): +class GCRPath(BaseType): openapi_schema_validator = '''{ "type": "string", "pattern": "^(us.|eu.|asia.)?gcr\\.io/.*$" } }''' -def GCRPath(attr={}): - return type('GCRPath', (MetaGCRPath, ), attr) - -class MetaGCPRegion(MetaType): +class GCPRegion(BaseType): openapi_schema_validator = '''{ "type": "string", "enum": ["asia-east1","asia-east2","asia-northeast1", @@ -101,28 +82,19 @@ class MetaGCPRegion(MetaType): "us-east4","us-west1", "us-west4" ] }''' -def GCPRegion(attr={}): - return type('GCPRegion', (MetaGCPRegion, ), attr) - -class MetaGCPProjectID(MetaType): +class GCPProjectID(BaseType): '''MetaGCPProjectID: GCP project id''' openapi_schema_validator = '''{ "type": "string" }''' -def GCPProjectID(attr={}): - return type('GCPProjectID', (MetaGCPProjectID, ), attr) - # General Types -class MetaLocalPath(MetaType): +class LocalPath(BaseType): #TODO: add restriction to path openapi_schema_validator = '''{ "type": "string" }''' -def LocalPath(attr={}): - return type('LocalPath', (MetaLocalPath, ), attr) - class InconsistentTypeException(Exception): '''InconsistencyTypeException is raised when two types are not consistent''' pass @@ -143,19 +115,15 @@ def _check_valid_dict(payload): return False return True -def _class_to_dict(payload): +def _instance_to_dict(instance): '''serialize_type serializes the type instance into a json string Args: - payload(type): A class that describes a type + instance(BaseType): An instance that describes a type Return: dict ''' - attrs = set([i for i in dir(payload) if not i.startswith('__')]) - json_dict = {payload.__name__: {}} - for attr in attrs: - json_dict[payload.__name__][attr] = getattr(payload, attr) - return json_dict + return {type(instance).__name__: instance.__dict__} def _str_to_dict(payload): import json @@ -194,12 +162,12 @@ def check_types(typeA, typeB): typeA (type/str/dict): it describes a type from the upstream component output typeB (type/str/dict): it describes a type from the downstream component input ''' - if isinstance(typeA, type): - typeA = _class_to_dict(typeA) + if isinstance(typeA, BaseType): + typeA = _instance_to_dict(typeA) elif isinstance(typeA, str): typeA = _str_to_dict(typeA) - if isinstance(typeB, type): - typeB = _class_to_dict(typeB) + if isinstance(typeB, BaseType): + typeB = _instance_to_dict(typeB) elif isinstance(typeB, str): typeB = _str_to_dict(typeB) return _check_dict_types(typeA, typeB) \ No newline at end of file diff --git a/sdk/python/tests/dsl/type_tests.py b/sdk/python/tests/dsl/type_tests.py index d7791264b6b..525b5b5f49e 100644 --- a/sdk/python/tests/dsl/type_tests.py +++ b/sdk/python/tests/dsl/type_tests.py @@ -13,22 +13,18 @@ # limitations under the License. -from kfp.dsl._types import _class_to_dict, _str_to_dict, check_types, GCSPath +from kfp.dsl._types import _instance_to_dict, _str_to_dict, check_types, GCSPath import unittest class TestTypes(unittest.TestCase): def test_class_to_dict(self): """Test _class_to_dict function.""" - gcspath_dict = _class_to_dict(GCSPath({'path_type': 'file', 'file_type': 'csv'})) + gcspath_dict = _instance_to_dict(GCSPath(path_type='file', file_type='csv')) golden_dict = { 'GCSPath': { 'path_type': 'file', 'file_type': 'csv', - 'openapi_schema_validator': '''{ - "type": "string", - "pattern": "^gs://$" - }''' } } self.assertEqual(golden_dict, gcspath_dict) @@ -45,16 +41,15 @@ def test_str_to_dict(self): self.assertEqual(golden_dict, gcspath_dict) gcspath_str = '{"file_type": "csv", "path_type": "file"}' with self.assertRaises(ValueError): - gcspath_dict = _str_to_dict(gcspath_str) + _str_to_dict(gcspath_str) def test_check_types(self): #Core types - typeA = GCSPath({'path_type': 'file', 'file_type': 'csv'}) - typeB = GCSPath({'path_type': 'file', 'file_type': 'csv'}) + typeA = GCSPath(path_type='file', file_type='csv') + typeB = GCSPath(path_type='file', file_type='csv') self.assertTrue(check_types(typeA, typeB)) - typeA = GCSPath({'path_type': 'file', 'file_type': 'tsv'}) - typeB = GCSPath({'path_type': 'file', 'file_type': 'csv'}) - self.assertFalse(check_types(typeA, typeB)) + typeC = GCSPath(path_type='file', file_type='tsv') + self.assertFalse(check_types(typeA, typeC)) # Custom types typeA = { From dcc3baf3e1eaad78835dd4459ceb2a2124eb4e5d Mon Sep 17 00:00:00 2001 From: Ning Gao Date: Fri, 1 Mar 2019 11:12:09 -0800 Subject: [PATCH 4/6] typo fix --- sdk/python/kfp/dsl/_types.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/python/kfp/dsl/_types.py b/sdk/python/kfp/dsl/_types.py index 71993d06512..4516cd70edb 100644 --- a/sdk/python/kfp/dsl/_types.py +++ b/sdk/python/kfp/dsl/_types.py @@ -159,8 +159,8 @@ def check_types(typeA, typeB): For each of the attribute in typeA, there is the same attribute in typeB with the same value. However, typeB could contain more attributes that typeA does not contain. Args: - typeA (type/str/dict): it describes a type from the upstream component output - typeB (type/str/dict): it describes a type from the downstream component input + typeA (BaseType/str/dict): it describes a type from the upstream component output + typeB (BaseType/str/dict): it describes a type from the downstream component input ''' if isinstance(typeA, BaseType): typeA = _instance_to_dict(typeA) From 3081d47eea1943d42b7649a7689b7e4f4f9dfe7b Mon Sep 17 00:00:00 2001 From: Ning Gao Date: Fri, 1 Mar 2019 13:56:12 -0800 Subject: [PATCH 5/6] use python struct for the openapi schema --- sdk/python/kfp/dsl/_types.py | 45 ++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/sdk/python/kfp/dsl/_types.py b/sdk/python/kfp/dsl/_types.py index 4516cd70edb..c560d74387e 100644 --- a/sdk/python/kfp/dsl/_types.py +++ b/sdk/python/kfp/dsl/_types.py @@ -19,41 +19,41 @@ class BaseType: # Primitive Types class Integer(BaseType): - openapi_schema_validator = '''{ + openapi_schema_validator = { "type": "integer" - }''' + } class String(BaseType): - openapi_schema_validator = '''{ + openapi_schema_validator = { "type": "string" - }''' + } class Float(BaseType): - openapi_schema_validator = '''{ + openapi_schema_validator = { "type": "number" - }''' + } class Bool(BaseType): - openapi_schema_validator = '''{ + openapi_schema_validator = { "type": "boolean" - }''' + } class List(BaseType): - openapi_schema_validator = '''{ + openapi_schema_validator = { "type": "array" - }''' + } class Dict(BaseType): - openapi_schema_validator = '''{ + openapi_schema_validator = { "type": "object", - }''' + } # GCP Types class GCSPath(BaseType): - openapi_schema_validator = '''{ + openapi_schema_validator = { "type": "string", "pattern": "^gs://$" - }''' + } def __init__(self, path_type='', file_type=''): ''' @@ -65,14 +65,13 @@ def __init__(self, path_type='', file_type=''): self.file_type = file_type class GCRPath(BaseType): - openapi_schema_validator = '''{ + openapi_schema_validator = { "type": "string", "pattern": "^(us.|eu.|asia.)?gcr\\.io/.*$" - } - }''' + } class GCPRegion(BaseType): - openapi_schema_validator = '''{ + openapi_schema_validator = { "type": "string", "enum": ["asia-east1","asia-east2","asia-northeast1", "asia-south1","asia-southeast1","australia-southeast1", @@ -80,20 +79,20 @@ class GCPRegion(BaseType): "europe-west3","europe-west4","northamerica-northeast1", "southamerica-east1","us-central1","us-east1", "us-east4","us-west1", "us-west4" ] - }''' + } class GCPProjectID(BaseType): '''MetaGCPProjectID: GCP project id''' - openapi_schema_validator = '''{ + openapi_schema_validator = { "type": "string" - }''' + } # General Types class LocalPath(BaseType): #TODO: add restriction to path - openapi_schema_validator = '''{ + openapi_schema_validator = { "type": "string" - }''' + } class InconsistentTypeException(Exception): '''InconsistencyTypeException is raised when two types are not consistent''' From 10ceb0994d5923c638385064979599a1e71d1d50 Mon Sep 17 00:00:00 2001 From: Ning Gao Date: Mon, 4 Mar 2019 13:47:49 -0800 Subject: [PATCH 6/6] update param name in the check_type functions remove schema validators for GCRPath, and adjust for GCRPath, GCSPath change _check_valid_dict to _check_valid_type_dict to avoid confusion fix typo in the comments adjust function order for readability --- sdk/python/kfp/dsl/_types.py | 82 +++++++++++++++++------------------- 1 file changed, 38 insertions(+), 44 deletions(-) diff --git a/sdk/python/kfp/dsl/_types.py b/sdk/python/kfp/dsl/_types.py index c560d74387e..c0dacca71dd 100644 --- a/sdk/python/kfp/dsl/_types.py +++ b/sdk/python/kfp/dsl/_types.py @@ -52,7 +52,7 @@ class Dict(BaseType): class GCSPath(BaseType): openapi_schema_validator = { "type": "string", - "pattern": "^gs://$" + "pattern": "^gs://.*$" } def __init__(self, path_type='', file_type=''): @@ -67,18 +67,12 @@ def __init__(self, path_type='', file_type=''): class GCRPath(BaseType): openapi_schema_validator = { "type": "string", - "pattern": "^(us.|eu.|asia.)?gcr\\.io/.*$" + "pattern": "^.*gcr\\.io/.*$" } class GCPRegion(BaseType): openapi_schema_validator = { - "type": "string", - "enum": ["asia-east1","asia-east2","asia-northeast1", - "asia-south1","asia-southeast1","australia-southeast1", - "europe-north1","europe-west1","europe-west2", - "europe-west3","europe-west4","northamerica-northeast1", - "southamerica-east1","us-central1","us-east1", - "us-east4","us-west1", "us-west4" ] + "type": "string" } class GCPProjectID(BaseType): @@ -98,8 +92,26 @@ class InconsistentTypeException(Exception): '''InconsistencyTypeException is raised when two types are not consistent''' pass -def _check_valid_dict(payload): - '''_check_valid_dict_type checks whether a dict is a correct serialization of a type +def check_types(checked_type, expected_type): + '''check_types checks the type consistency. + For each of the attribute in checked_type, there is the same attribute in expected_type with the same value. + However, expected_type could contain more attributes that checked_type does not contain. + Args: + checked_type (BaseType/str/dict): it describes a type from the upstream component output + expected_type (BaseType/str/dict): it describes a type from the downstream component input + ''' + if isinstance(checked_type, BaseType): + checked_type = _instance_to_dict(checked_type) + elif isinstance(checked_type, str): + checked_type = _str_to_dict(checked_type) + if isinstance(expected_type, BaseType): + expected_type = _instance_to_dict(expected_type) + elif isinstance(expected_type, str): + expected_type = _str_to_dict(expected_type) + return _check_dict_types(checked_type, expected_type) + +def _check_valid_type_dict(payload): + '''_check_valid_type_dict checks whether a dict is a correct serialization of a type Args: payload(dict) ''' @@ -115,7 +127,7 @@ def _check_valid_dict(payload): return True def _instance_to_dict(instance): - '''serialize_type serializes the type instance into a json string + '''_instance_to_dict serializes the type instance into a python dictionary Args: instance(BaseType): An instance that describes a type @@ -127,46 +139,28 @@ def _instance_to_dict(instance): def _str_to_dict(payload): import json json_dict = json.loads(payload) - if not _check_valid_dict(json_dict): + if not _check_valid_type_dict(json_dict): raise ValueError(payload + ' is not a valid type string') return json_dict -def _check_dict_types(typeA, typeB): +def _check_dict_types(checked_type, expected_type): '''_check_type_types checks the type consistency. Args: - typeA (dict): A dict that describes a type from the upstream component output - typeB (dict): A dict that describes a type from the downstream component input + checked_type (dict): A dict that describes a type from the upstream component output + expected_type (dict): A dict that describes a type from the downstream component input ''' - typeA_name,_ = list(typeA.items())[0] - typeB_name,_ = list(typeB.items())[0] - if typeA_name != typeB_name: + checked_type_name,_ = list(checked_type.items())[0] + expected_type_name,_ = list(expected_type.items())[0] + if checked_type_name != expected_type_name: return False - type_name = typeA_name - for type_property in typeA[type_name]: - if type_property not in typeB[type_name]: + type_name = checked_type_name + for type_property in checked_type[type_name]: + if type_property not in expected_type[type_name]: print(type_name + ' has a property ' + str(type_property) + ' that the latter does not.') return False - if typeA[type_name][type_property] != typeB[type_name][type_property]: + if checked_type[type_name][type_property] != expected_type[type_name][type_property]: print(type_name + ' has a property ' + str(type_property) + ' with value: ' + - str(typeA[type_name][type_property]) + ' and ' + - str(typeB[type_name][type_property])) + str(checked_type[type_name][type_property]) + ' and ' + + str(expected_type[type_name][type_property])) return False - return True - -def check_types(typeA, typeB): - '''check_types checks the type consistency. - For each of the attribute in typeA, there is the same attribute in typeB with the same value. - However, typeB could contain more attributes that typeA does not contain. - Args: - typeA (BaseType/str/dict): it describes a type from the upstream component output - typeB (BaseType/str/dict): it describes a type from the downstream component input - ''' - if isinstance(typeA, BaseType): - typeA = _instance_to_dict(typeA) - elif isinstance(typeA, str): - typeA = _str_to_dict(typeA) - if isinstance(typeB, BaseType): - typeB = _instance_to_dict(typeB) - elif isinstance(typeB, str): - typeB = _str_to_dict(typeB) - return _check_dict_types(typeA, typeB) \ No newline at end of file + return True \ No newline at end of file