From 990afcda5e3cc0834f83a809778d22750df146f6 Mon Sep 17 00:00:00 2001 From: James Saryerwinnie Date: Tue, 25 Nov 2014 13:07:09 -0800 Subject: [PATCH] Fix regression for supported timestamp formats We now accept: * datetime objects * ISO8601 * epoch seconds (both as an int or a string) anytime a timestamp type is needed. This puts back the previous behavior that botocore had. To accomodate this, I've added a new date parsing utility method in the utils module. It accepts any of the types listed above and returns a datetime object with tzinfo. This makes it easy to write protocol specific timestamp serializers as they now can assume they'll always be working with a datetime object with time zone info. --- botocore/serialize.py | 61 ++++++++++++++++++++------------ botocore/utils.py | 52 ++++++++++++++++++++++++++- botocore/validate.py | 6 ++-- tests/integration/test_client.py | 31 ++++++++++++++++ tests/unit/test_serialize.py | 57 +++++++++++++++++++++++++++++ tests/unit/test_utils.py | 47 +++++++++++++++++++++++- 6 files changed, 225 insertions(+), 29 deletions(-) diff --git a/botocore/serialize.py b/botocore/serialize.py index 58fe5e99dc..bac25a0628 100644 --- a/botocore/serialize.py +++ b/botocore/serialize.py @@ -41,13 +41,14 @@ import time import base64 from xml.etree import ElementTree +import calendar import datetime from dateutil.tz import tzutc import six from botocore.compat import json, formatdate -from botocore.utils import parse_timestamp +from botocore.utils import parse_timestamp, parse_to_aware_datetime from botocore.utils import percent_encode from botocore import validate @@ -132,34 +133,16 @@ def _timestamp_iso8601(self, value): timestamp_format = ISO8601_MICRO else: timestamp_format = ISO8601 - if value.tzinfo is None: - # I think a case would be made that if no time zone is provided, - # we should use the local time. However, to restore backwards - # compat, the previous behavior was to assume UTC, which is - # what we're going to do here. - datetime_obj = value.replace(tzinfo=tzutc()) - else: - datetime_obj = value.astimezone(tzutc()) - return datetime_obj.strftime(timestamp_format) + return value.strftime(timestamp_format) def _timestamp_unixtimestamp(self, value): - return int(time.mktime(value.timetuple())) + return int(calendar.timegm(value.timetuple())) def _timestamp_rfc822(self, value): return formatdate(value) def _convert_timestamp_to_str(self, value): - # This is a general purpose method that handles several cases of - # converting the provided value to a string timestamp suitable to be - # serialized to an http request. It can handle: - # 1) A datetime.datetime object. - if isinstance(value, datetime.datetime): - datetime_obj = value - else: - # 2) A string object that's formatted as a timestamp. - # We document this as being an iso8601 timestamp, although - # parse_timestamp is a bit more flexible. - datetime_obj = parse_timestamp(value) + datetime_obj = parse_to_aware_datetime(value) converter = getattr( self, '_timestamp_%s' % self.TIMESTAMP_FORMAT.lower()) final_value = converter(datetime_obj) @@ -304,6 +287,8 @@ def _serialize_type_list(self, serialized, value, shape, prefix=''): class JSONSerializer(Serializer): + TIMESTAMP_FORMAT = 'unixtimestamp' + def serialize_to_request(self, parameters, operation_model): target = '%s.%s' % (operation_model.metadata['targetPrefix'], operation_model.name) @@ -315,9 +300,39 @@ def serialize_to_request(self, parameters, operation_model): 'X-Amz-Target': target, 'Content-Type': 'application/x-amz-json-%s' % json_version, } - serialized['body'] = json.dumps(parameters) + body = {} + input_shape = operation_model.input_shape + if input_shape is not None: + self._serialize(body, parameters, input_shape) + serialized['body'] = json.dumps(body) return serialized + def _serialize(self, serialized, value, shape, key=None): + method = getattr(self, '_serialize_type_%s' % shape.type_name, + self._default_serialize) + method(serialized, value, shape, key) + + def _serialize_type_structure(self, serialized, value, shape, key): + if key is not None: + # If a key is provided, this is a result of a recursive + # call so we need to add a new child dict as the value + # of the passed in serialized dict. We'll then add + # all the structure members as key/vals in the new serialized + # dictionary we just created. + new_serialized = {} + serialized[key] = new_serialized + serialized = new_serialized + members = shape.members + for member_key, member_value in value.items(): + member_shape = members[member_key] + self._serialize(serialized, member_value, member_shape, member_key) + + def _default_serialize(self, serialized, value, shape, key): + serialized[key] = value + + def _serialize_type_timestamp(self, serialized, value, shape, key): + serialized[key] = self._convert_timestamp_to_str(value) + class BaseRestSerializer(Serializer): """Base class for rest protocols. diff --git a/botocore/utils.py b/botocore/utils.py index 8296e179fd..e58201788b 100644 --- a/botocore/utils.py +++ b/botocore/utils.py @@ -15,7 +15,7 @@ from six import string_types, text_type import dateutil.parser -from dateutil.tz import tzlocal +from dateutil.tz import tzlocal, tzutc from botocore.exceptions import InvalidExpressionError, ConfigNotFound from botocore.compat import json, quote @@ -288,12 +288,62 @@ def parse_timestamp(value): if isinstance(value, (int, float)): # Possibly an epoch time. return datetime.datetime.fromtimestamp(value, tzlocal()) + else: + try: + return datetime.datetime.fromtimestamp(float(value), tzlocal()) + except (TypeError, ValueError): + pass try: return dateutil.parser.parse(value) except (TypeError, ValueError) as e: raise ValueError('Invalid timestamp "%s": %s' % (value, e)) +def parse_to_aware_datetime(value): + """Converted the passed in value to a datetime object with tzinfo. + + This function can be used to normalize all timestamp inputs. This + function accepts a number of different types of inputs, but + will always return a datetime.datetime object with time zone + information. + + The input param ``value`` can be one of several types: + + * A datetime object (both naive and aware) + * An integer representing the epoch time (can also be a string + of the integer, i.e '0', instead of 0). The epoch time is + considered to be UTC. + * An iso8601 formatted timestamp. This does not need to be + a complete timestamp, it can contain just the date portion + without the time component. + + The returned value will be a datetime object that will have tzinfo. + If no timezone info was provided in the input value, then UTC is + assumed, not local time. + + """ + # This is a general purpose method that handles several cases of + # converting the provided value to a string timestamp suitable to be + # serialized to an http request. It can handle: + # 1) A datetime.datetime object. + if isinstance(value, datetime.datetime): + datetime_obj = value + else: + # 2) A string object that's formatted as a timestamp. + # We document this as being an iso8601 timestamp, although + # parse_timestamp is a bit more flexible. + datetime_obj = parse_timestamp(value) + if datetime_obj.tzinfo is None: + # I think a case would be made that if no time zone is provided, + # we should use the local time. However, to restore backwards + # compat, the previous behavior was to assume UTC, which is + # what we're going to do here. + datetime_obj = datetime_obj.replace(tzinfo=tzutc()) + else: + datetime_obj = datetime_obj.astimezone(tzutc()) + return datetime_obj + + class CachedProperty(object): """A read only property that caches the initially computed value. diff --git a/botocore/validate.py b/botocore/validate.py index 13a6729c39..b5c1effdb7 100644 --- a/botocore/validate.py +++ b/botocore/validate.py @@ -17,7 +17,7 @@ import decimal from datetime import datetime -from botocore.utils import parse_timestamp +from botocore.utils import parse_to_aware_datetime from botocore.exceptions import ParamValidationError @@ -250,10 +250,8 @@ def _validate_timestamp(self, param, shape, errors, name): valid_types=valid_type_names) def _type_check_datetime(self, value): - if isinstance(value, datetime): - return True try: - parse_timestamp(value) + parse_to_aware_datetime(value) return True except (TypeError, ValueError): return False diff --git a/tests/integration/test_client.py b/tests/integration/test_client.py index 9c57865fe5..77bdb9dbbe 100644 --- a/tests/integration/test_client.py +++ b/tests/integration/test_client.py @@ -13,6 +13,7 @@ import time import random import logging +import datetime from tests import unittest from six import StringIO @@ -93,3 +94,33 @@ def test_debug_log_contains_headers_and_body(self): debug_log_contents = debug_log.getvalue() self.assertIn('Response headers', debug_log_contents) self.assertIn('Response body', debug_log_contents) + + +class TestAcceptedDateTimeFormats(unittest.TestCase): + def setUp(self): + self.session = botocore.session.get_session() + self.client = self.session.create_client('emr') + + def test_accepts_datetime_object(self): + response = self.client.list_clusters( + CreatedAfter=datetime.datetime.now()) + self.assertIn('Clusters', response) + + def test_accepts_epoch_format(self): + response = self.client.list_clusters(CreatedAfter=0) + self.assertIn('Clusters', response) + + def test_accepts_iso_8601_unaware(self): + response = self.client.list_clusters( + CreatedAfter='2014-01-01T00:00:00') + self.assertIn('Clusters', response) + + def test_accepts_iso_8601_utc(self): + response = self.client.list_clusters( + CreatedAfter='2014-01-01T00:00:00Z') + self.assertIn('Clusters', response) + + def test_accepts_iso_8701_local(self): + response = self.client.list_clusters( + CreatedAfter='2014-01-01T00:00:00-08:00') + self.assertIn('Clusters', response) diff --git a/tests/unit/test_serialize.py b/tests/unit/test_serialize.py index 99f413c0e2..beeb570ccd 100644 --- a/tests/unit/test_serialize.py +++ b/tests/unit/test_serialize.py @@ -12,6 +12,7 @@ """ import base64 +import json import datetime import dateutil.tz from tests import unittest @@ -144,3 +145,59 @@ def test_microsecond_timestamp_without_tz_info(self): {'Timestamp': '2014-01-01T12:12:12.123456'}) self.assertEqual(request['body']['Timestamp'], '2014-01-01T12:12:12.123456Z') + + +class TestJSONTimestampSerialization(unittest.TestCase): + def setUp(self): + self.model = { + 'metadata': {'protocol': 'json', 'apiVersion': '2014-01-01', + 'jsonVersion': '1.1', 'targetPrefix': 'foo'}, + 'documentation': '', + 'operations': { + 'TestOperation': { + 'name': 'TestOperation', + 'http': { + 'method': 'POST', + 'requestUri': '/', + }, + 'input': {'shape': 'InputShape'}, + } + }, + 'shapes': { + 'InputShape': { + 'type': 'structure', + 'members': { + 'Timestamp': {'shape': 'TimestampType'}, + } + }, + 'TimestampType': { + 'type': 'timestamp', + } + } + } + self.service_model = ServiceModel(self.model) + + def serialize_to_request(self, input_params): + request_serializer = serialize.create_serializer( + self.service_model.metadata['protocol']) + return request_serializer.serialize_to_request( + input_params, self.service_model.operation_model('TestOperation')) + + def test_accepts_iso_8601_format(self): + body = json.loads(self.serialize_to_request( + {'Timestamp': '1970-01-01T00:00:00'})['body']) + self.assertEqual(body['Timestamp'], 0) + + def test_accepts_epoch(self): + body = json.loads(self.serialize_to_request( + {'Timestamp': '0'})['body']) + self.assertEqual(body['Timestamp'], 0) + # Can also be an integer 0. + body = json.loads(self.serialize_to_request( + {'Timestamp': 0})['body']) + self.assertEqual(body['Timestamp'], 0) + + def test_accepts_partial_iso_format(self): + body = json.loads(self.serialize_to_request( + {'Timestamp': '1970-01-01'})['body']) + self.assertEqual(body['Timestamp'], 0) diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index a736bd7cf8..6906721d58 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -12,7 +12,7 @@ # language governing permissions and limitations under the License. from tests import unittest -from dateutil.tz import tzutc +from dateutil.tz import tzutc, tzoffset import datetime import mock @@ -26,6 +26,7 @@ from botocore.utils import parse_key_val_file_contents from botocore.utils import parse_key_val_file from botocore.utils import parse_timestamp +from botocore.utils import parse_to_aware_datetime from botocore.utils import CachedProperty from botocore.utils import ArgumentGenerator from botocore.model import DenormalizedStructureBuilder @@ -196,6 +197,16 @@ def test_parse_epoch(self): parse_timestamp(1222172800), datetime.datetime(2008, 9, 23, 12, 26, 40, tzinfo=tzutc())) + def test_parse_epoch_zero_time(self): + self.assertEqual( + parse_timestamp(0), + datetime.datetime(1970, 1, 1, 0, 0, 0, tzinfo=tzutc())) + + def test_parse_epoch_as_string(self): + self.assertEqual( + parse_timestamp('1222172800'), + datetime.datetime(2008, 9, 23, 12, 26, 40, tzinfo=tzutc())) + def test_parse_rfc822(self): self.assertEqual( parse_timestamp('Wed, 02 Oct 2002 13:00:00 GMT'), @@ -206,6 +217,40 @@ def test_parse_invalid_timestamp(self): parse_timestamp('invalid date') +class TestParseToUTCDatetime(unittest.TestCase): + def test_handles_utc_time(self): + original = datetime.datetime(1970, 1, 1, 0, 0, 0, tzinfo=tzutc()) + self.assertEqual(parse_to_aware_datetime(original), original) + + def test_handles_other_timezone(self): + tzinfo = tzoffset("BRST", -10800) + original = datetime.datetime(2014, 1, 1, 0, 0, 0, tzinfo=tzinfo) + self.assertEqual(parse_to_aware_datetime(original), original) + + def test_handles_naive_datetime(self): + original = datetime.datetime(1970, 1, 1, 0, 0, 0) + expected = datetime.datetime(1970, 1, 1, 0, 0, 0, tzinfo=tzutc()) + self.assertEqual(parse_to_aware_datetime(original), expected) + + def test_handles_string_epoch(self): + expected = datetime.datetime(1970, 1, 1, 0, 0, 0, tzinfo=tzutc()) + self.assertEqual(parse_to_aware_datetime('0'), expected) + + def test_handles_int_epoch(self): + expected = datetime.datetime(1970, 1, 1, 0, 0, 0, tzinfo=tzutc()) + self.assertEqual(parse_to_aware_datetime(0), expected) + + def test_handles_full_iso_8601(self): + expected = datetime.datetime(1970, 1, 1, 0, 0, 0, tzinfo=tzutc()) + self.assertEqual( + parse_to_aware_datetime('1970-01-01T00:00:00Z'), + expected) + + def test_year_only_iso_8601(self): + expected = datetime.datetime(1970, 1, 1, 0, 0, 0, tzinfo=tzutc()) + self.assertEqual(parse_to_aware_datetime('1970-01-01'), expected) + + class TestCachedProperty(unittest.TestCase): def test_cached_property_same_value(self): class CacheMe(object):