diff --git a/awscli/customizations/s3/utils.py b/awscli/customizations/s3/utils.py index 15c91a72d83d..a85427e0b26c 100644 --- a/awscli/customizations/s3/utils.py +++ b/awscli/customizations/s3/utils.py @@ -23,6 +23,7 @@ from six.moves import queue from dateutil.parser import parse from dateutil.tz import tzlocal +from botocore.compat import unquote_str from awscli.customizations.s3.constants import MAX_PARTS from awscli.customizations.s3.constants import MAX_SINGLE_UPLOAD_SIZE @@ -298,14 +299,14 @@ def __init__(self, operation, endpoint, date_parser=_date_parser): self._date_parser = date_parser def list_objects(self, bucket, prefix=None): - kwargs = {'bucket': bucket} + kwargs = {'bucket': bucket, 'encoding_type': 'url'} if prefix is not None: kwargs['prefix'] = prefix pages = self._operation.paginate(self._endpoint, **kwargs) for response, page in pages: contents = page['Contents'] for content in contents: - source_path = bucket + '/' + content['Key'] + source_path = bucket + '/' + unquote_str(content['Key']) size = content['Size'] last_update = self._date_parser(content['LastModified']) yield source_path, size, last_update diff --git a/tests/unit/customizations/s3/test_utils.py b/tests/unit/customizations/s3/test_utils.py index 0d409972a0d5..1a6bb3bf4a8f 100644 --- a/tests/unit/customizations/s3/test_utils.py +++ b/tests/unit/customizations/s3/test_utils.py @@ -218,6 +218,33 @@ def test_list_objects(self): self.assertEqual(objects, [('foo/a', 1, now), ('foo/b', 2, now), ('foo/c', 3, now)]) + def test_urlencoded_keys(self): + # In order to workaround control chars being in key names, + # we force the urlencoding of the key names and we decode + # them before yielding them. For example, note the %0D + # in foo.txt: + now = mock.sentinel.now + self.operation.paginate.return_value = [ + (None, {'Contents': [ + {'LastModified': '2014-02-27T04:20:38.000Z', + 'Key': 'bar%0D.txt', 'Size': 1}]}), + ] + lister = BucketLister(self.operation, self.endpoint, self.date_parser) + objects = list(lister.list_objects(bucket='foo')) + # And note how it's been converted to '\r'. + self.assertEqual(objects, [('foo/bar\r.txt', 1, now)]) + + def test_urlencoded_with_unicode_keys(self): + now = mock.sentinel.now + self.operation.paginate.return_value = [ + (None, {'Contents': [ + {'LastModified': '2014-02-27T04:20:38.000Z', + 'Key': '%E2%9C%93', 'Size': 1}]}), + ] + lister = BucketLister(self.operation, self.endpoint, self.date_parser) + objects = list(lister.list_objects(bucket='foo')) + # And note how it's been converted to '\r'. + self.assertEqual(objects, [(u'foo/\u2713', 1, now)]) if __name__ == "__main__": unittest.main()