From 883f8b6b67100c6ed77fb794f3e29f1c0d60ec4f Mon Sep 17 00:00:00 2001 From: Yonggang Liu Date: Fri, 12 Apr 2024 12:12:22 -0400 Subject: [PATCH] HARMONY-1727: Add support of extraArgs to message for data operation v0.19.0 --- example/example_message.json | 9 +++++--- harmony/message.py | 43 +++++++++++++++++++++++++++++++++++- tests/example_messages.py | 25 +++++++++++++++------ tests/test_message.py | 18 ++++++++++++--- 4 files changed, 81 insertions(+), 14 deletions(-) diff --git a/example/example_message.json b/example/example_message.json index 774b8d6..0c76f87 100644 --- a/example/example_message.json +++ b/example/example_message.json @@ -1,6 +1,6 @@ { - "$schema": "../../harmony/app/schemas/data-operation/0.18.0/data-operation-v0.18.0.json", - "version": "0.18.0", + "$schema": "../../harmony/app/schemas/data-operation/0.19.0/data-operation-v0.19.0.json", + "version": "0.19.0", "callback": "http://localhost/some-path", "stagingLocation": "s3://example-bucket/public/some-org/some-service/some-uuid/", "user": "jdoe", @@ -68,5 +68,8 @@ "extendDimensions": [ "lat", "lon" - ] + ], + "extraArgs": { + "cut": false + } } diff --git a/harmony/message.py b/harmony/message.py index 74d69fb..bcc77fe 100644 --- a/harmony/message.py +++ b/harmony/message.py @@ -54,6 +54,22 @@ def __init__(self, data, properties=[], list_properties={}): value = [Class(item) for item in items] setattr(self, prop, value) + def __getitem__(self, key): + """ + Retrieve the value corresponding to a key in data + + Parameters + ---------- + key : str + The key to retrieve the value for + + Returns + ------- + value : object or None + The value corresponding to the key if it exists, otherwise None + """ + return self.data.get(key) + def process(self, *prop): """ Marks the given property as having been processed and returns its value. @@ -532,6 +548,25 @@ def __init__(self, message_data=None, start=None, end=None): self.end = end +class ExtraArgs(JsonObject): + """ + Extra Args parameters as found in a Harmony message's "extraArgs" object + The value of extra args parameter can be retrieved via [''], + e.g. message.extraArgs['cut'] will return the value of 'cut' parameter in extraArgs. + """ + + def __init__(self, message_data): + """ + Constructor + + Parameters + ---------- + message_data : dictionary + The Harmony message "extraArgs" object to deserialize + """ + super().__init__(message_data) + + class Message(JsonObject): """ Top-level object corresponding to an incoming Harmony message. Constructing @@ -584,6 +619,9 @@ class Message(JsonObject): file and false otherwise. extendDimensions: list A list of dimensions to extend. + extraArgs: object + A map of key (string type) and value (any type) pairs indicating the extra arguments + that should be passed to the worker command """ def __init__(self, json_str_or_dict, decrypter=lambda x: x): @@ -619,7 +657,8 @@ def __init__(self, json_str_or_dict, decrypter=lambda x: x): 'subset', 'temporal', 'concatenate', - 'extendDimensions' + 'extendDimensions', + 'extraArgs' ], list_properties={'sources': Source} ) @@ -634,6 +673,8 @@ def __init__(self, json_str_or_dict, decrypter=lambda x: x): self.temporal = Temporal(json_obj['temporal']) if self.accessToken is not None: self.accessToken = self.decrypter(self.accessToken) + if self.extraArgs is not None: + self.extraArgs = ExtraArgs(json_obj['extraArgs']) @property def json(self): diff --git a/tests/example_messages.py b/tests/example_messages.py index 3b74800..3fc888e 100644 --- a/tests/example_messages.py +++ b/tests/example_messages.py @@ -1,7 +1,7 @@ minimal_message = """ { - "$schema": "../../harmony/app/schemas/data-operation/0.18.0/data-operation-v0.18.0.json", - "version": "0.18.0", + "$schema": "../../harmony/app/schemas/data-operation/0.19.0/data-operation-v0.19.0.json", + "version": "0.19.0", "callback": "http://localhost/some-path", "stagingLocation": "s3://example-bucket/public/some-org/some-service/some-uuid/", "user": "jdoe", @@ -19,8 +19,8 @@ minimal_source_message = """ { - "$schema": "../../harmony/app/schemas/data-operation/0.18.0/data-operation-v0.18.0.json", - "version": "0.18.0", + "$schema": "../../harmony/app/schemas/data-operation/0.19.0/data-operation-v0.19.0.json", + "version": "0.19.0", "callback": "http://localhost/some-path", "stagingLocation": "s3://example-bucket/public/some-org/some-service/some-uuid/", "user": "jdoe", @@ -46,8 +46,8 @@ full_message = """ { - "$schema": "../../harmony/app/schemas/data-operation/0.18.0/data-operation-v0.18.0.json", - "version": "0.18.0", + "$schema": "../../harmony/app/schemas/data-operation/0.19.0/data-operation-v0.19.0.json", + "version": "0.19.0", "callback": "http://localhost/some-path", "stagingLocation": "s3://example-bucket/public/some-org/some-service/some-uuid/", "user": "jdoe", @@ -180,6 +180,17 @@ }] }, "concatenate": true, - "extendDimensions": ["lat", "lon"] + "extendDimensions": ["lat", "lon"], + "extraArgs": { + "cut": false, + "intParam": 100, + "floatParam": 123.456, + "stringParam": "value", + "arrayParam": [1, 2, 3], + "objectParam": { + "name": "obj1", + "attributes": ["x", "y"] + } + } } """ diff --git a/tests/test_message.py b/tests/test_message.py index b8afac0..b7c3673 100644 --- a/tests/test_message.py +++ b/tests/test_message.py @@ -13,7 +13,7 @@ def setUp(self): def test_when_provided_a_full_message_it_parses_it_into_objects(self): message = Message(full_message) - self.assertEqual(message.version, '0.18.0') + self.assertEqual(message.version, '0.19.0') self.assertEqual(message.callback, 'http://localhost/some-path') self.assertEqual(message.stagingLocation, 's3://example-bucket/public/some-org/some-service/some-uuid/') self.assertEqual(message.user, 'jdoe') @@ -79,12 +79,13 @@ def test_when_provided_a_full_message_it_parses_it_into_objects(self): self.assertEqual(message.subset.dimensions[1].min, None) self.assertEqual(message.subset.dimensions[1].max, 10) self.assertEqual(message.extendDimensions, ["lat", "lon"]) + self.assertEqual(message.extraArgs['cut'], False) def test_when_provided_a_minimal_message_it_parses_it_into_objects(self): message = Message(minimal_message) - self.assertEqual(message.version, '0.18.0') + self.assertEqual(message.version, '0.19.0') self.assertEqual(message.callback, 'http://localhost/some-path') self.assertEqual(message.stagingLocation, 's3://example-bucket/public/some-org/some-service/some-uuid/') self.assertEqual(message.user, 'jdoe') @@ -104,7 +105,7 @@ def test_when_provided_a_minimal_message_it_parses_it_into_objects(self): def test_when_provided_a_message_with_minimal_source_it_parses_it_into_objects(self): message = Message(minimal_source_message) - self.assertEqual(message.version, '0.18.0') + self.assertEqual(message.version, '0.19.0') self.assertEqual(message.callback, 'http://localhost/some-path') self.assertEqual(message.user, 'jdoe') self.assertEqual(message.accessToken, 'ABCD1234567890') @@ -174,3 +175,14 @@ def test_processing_a_property_removes_it_from_json_output(self): # Point property is generated self.assertEqual(message.subset.point, [-160.2, 80.2]) + + def test_extra_args(self): + message = Message(full_message) + + self.assertEqual(message.extraArgs['cut'], False) + self.assertEqual(message.extraArgs['intParam'], 100) + self.assertEqual(message.extraArgs['floatParam'], 123.456) + self.assertEqual(message.extraArgs['stringParam'], 'value') + self.assertEqual(message.extraArgs['arrayParam'], [1, 2, 3]) + self.assertEqual(message.extraArgs['objectParam'], {'name': 'obj1', 'attributes': ['x', 'y']}) + self.assertIsNone(message.extraArgs['nonExistent']) \ No newline at end of file