Merge branch 'release-1.2.11'

* release-1.2.11: (21 commits) Bumping version to 1.2.11 Update S3 model to latest version Update KMS model to latest version Log traceback on ConnectionErrors Update ec2 pagination models Add Glacier paginator model Update IAM paginators Update contributing guide with latest model format Add route53 check for Throttling code as well Add the retry for Route53 when PriorRequestNotComplete happens Add test for us-gov-west-1 and virtual style Hoist region check to fix_s3_host Avoid slow io.open() by handling encoding by ourselves Update a docstring about s3 addressing_style Fix code for cli integration tests Revert "Merge pull request #671 from boto/revert-655-dns-style-host" Update comment in test about invalid xml 500 error Add debug log for when parsing exception is caught Use utf8 encoding to open model file Make error parsing more robust ...
boto · Oct 15, 2015 · 0dd0c05 · 0dd0c05
2 parents 53db8b8 + 62585a1
commit 0dd0c05
Show file tree

Hide file tree

Showing 32 changed files with 2,018 additions and 266 deletions.
diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
@@ -28,19 +28,19 @@ contributions as well:
   In practice, this means that every bug fix and feature addition should
   include unit tests.
 * We may choose not to accept pull requests that change the JSON service descriptions,
-  such as ``botocore/data/aws/s3/2006-03-01.normal.json``.  We generate these
+  such as ``botocore/data/aws/s3/2006-03-01/service-2.json``.  We generate these
   files upstream based on our internal knowledge of the AWS services. If there
   is something incorrect with or missing from a service description, it may be
   more appropriate to submit an
   `issue <https://github.com/boto/botocore/issues>`__ so we can get the issue
-  fixed upstream.  This constraint only applies to the ``*.normal.json`` files.
+  fixed upstream.  This constraint only applies to the ``*/service-2.json`` files.
   We do accept, and encourage, changes to any of the following files
   in ``botocore/data/aws/``:
 
   * ``_endpoints.json``
   * ``_retry.json``
-  * ``*.paginators.json``
-  * ``*.waiters.json``
+  * ``*.paginators-1.json``
+  * ``*.waiters-2.json``
 
 * Code should follow `pep 8 <https://www.python.org/dev/peps/pep-0008/>`__,
   although if you are modifying an existing module, it is more important

diff --git a/botocore/__init__.py b/botocore/__init__.py
@@ -16,7 +16,7 @@
 import re
 import logging
 
-__version__ = '1.2.10'
+__version__ = '1.2.11'
 
 
 class NullHandler(logging.Handler):

diff --git a/botocore/client.py b/botocore/client.py
@@ -21,12 +21,15 @@
 from botocore.endpoint import EndpointCreator, DEFAULT_TIMEOUT
 from botocore.exceptions import ClientError, DataNotFoundError
 from botocore.exceptions import OperationNotPageableError
+from botocore.exceptions import InvalidS3AddressingStyleError
 from botocore.hooks import first_non_none_response
 from botocore.model import ServiceModel
 from botocore.paginate import Paginator
 from botocore.signers import RequestSigner
 from botocore.utils import CachedProperty
 from botocore.utils import get_service_module_name
+from botocore.utils import fix_s3_host
+from botocore.utils import switch_to_virtual_host_style
 from botocore.docs.docstring import ClientMethodDocstring
 from botocore.docs.docstring import PaginatorDocstring
 
@@ -164,6 +167,30 @@ def _determine_region_name(self, endpoint_config, region_name=None,
 
         return region_name
 
+    def _inject_s3_configuration(self, config_kwargs, scoped_config,
+                                 client_config):
+        s3_configuration = None
+
+        # Check the scoped config first
+        if scoped_config is not None:
+            s3_configuration = scoped_config.get('s3')
+
+        # Next specfic client config values takes precedence over
+        # specific values in the scoped config.
+        if client_config is not None:
+            if client_config.s3 is not None:
+                if s3_configuration is None:
+                    s3_configuration = client_config.s3
+                else:
+                    # The current s3_configuration dictionary may be
+                    # from a source that only should be read from so
+                    # we want to be safe and just make a copy of it to modify
+                    # before it actually gets updated.
+                    s3_configuration = s3_configuration.copy()
+                    s3_configuration.update(client_config.s3)
+
+        config_kwargs['s3'] = s3_configuration
+
     def _get_client_args(self, service_model, region_name, is_secure,
                          endpoint_url, verify, credentials,
                          scoped_config, client_config):
@@ -218,6 +245,11 @@ def _get_client_args(self, service_model, region_name, is_secure,
             config_kwargs.update(
                 connect_timeout=client_config.connect_timeout,
                 read_timeout=client_config.read_timeout)
+
+        # Add any additional s3 configuration for client
+        self._inject_s3_configuration(
+            config_kwargs, scoped_config, client_config)
+
         new_config = Config(**config_kwargs)
 
         endpoint_creator = EndpointCreator(self._endpoint_resolver,
@@ -307,10 +339,30 @@ def __init__(self, serializer, endpoint, response_parser,
         self.meta = ClientMeta(event_emitter, self._client_config,
                                endpoint.host, service_model,
                                self._PY_TO_OP_NAME)
+        self._register_handlers()
+
+    def _register_handlers(self):
+        # Register the handler required to sign requests.
         self.meta.events.register('request-created.%s' %
-                                  service_model.endpoint_prefix,
+                                  self.meta.service_model.endpoint_prefix,
                                   self._sign_request)
 
+        # If the virtual host addressing style is being forced,
+        # switch the default fix_s3_host handler for the more general
+        # switch_to_virtual_host_style handler that does not have opt out
+        # cases (other than throwing an error if the name is DNS incompatible)
+        if self.meta.config.s3 is None:
+            s3_addressing_style = None
+        else:
+            s3_addressing_style = self.meta.config.s3.get('addressing_style')
+
+        if s3_addressing_style == 'path':
+            self.meta.events.unregister('before-sign.s3', fix_s3_host)
+        elif s3_addressing_style == 'virtual':
+            self.meta.events.unregister('before-sign.s3', fix_s3_host)
+            self.meta.events.register(
+                'before-sign.s3', switch_to_virtual_host_style)
+
     @property
     def _service_model(self):
         return self.meta.service_model
@@ -541,23 +593,61 @@ def method_to_api_mapping(self):
 class Config(object):
     """Advanced configuration for Botocore clients.
 
-    This class allows you to configure:
-
-        * Region name
-        * Signature version
-        * User agent
-        * User agent extra
-        * Connect timeout
-        * Read timeout
-
+    :type region_name: str
+    :param region_name: The region to use in instantiating the client
+
+    :type signature_version: str
+    :param signature_version: The signature version when signing requests.
+
+    :type user_agent: str
+    :param user_agent: The value to use in the User-Agent header.
+
+    :type user_agent_extra: str
+    :param user_agent_extra: The value to append to the current User-Agent
+        header value.
+
+    :type connect_timeout: int
+    :param connect_timeout: The time in seconds till a timeout exception is
+        thrown when attempting to make a connection.
+
+    :type read_timeout: int
+    :param read_timeout: The time in seconds till a timeout exception is
+        thrown when attempting to read from a connection.
+
+    :type s3: dict
+    :param s3: A dictionary of s3 specific configurations.
+        Valid keys are:
+            * 'addressing_style' -- Refers to the style in which to address
+              s3 endpoints. Values must be a string that equals:
+                  * auto -- Addressing style is chosen for user. Depending
+                            on the configuration of client, the endpoint
+                            may be addressed in the virtual or the path
+                            style. Note that this is the default behavior if
+                            no style is specified.
+                  * virtual -- Addressing style is always virtual. The name of
+                               the bucket must be DNS compatible or an
+                               exception will be thrown. Endpoints will be
+                               addressed as such: mybucket.s3.amazonaws.com
+                  * path -- Addressing style is always by path. Endpoints will
+                            be addressed as such: s3.amazonaws.com/mybucket
     """
     def __init__(self, region_name=None, signature_version=None,
                  user_agent=None, user_agent_extra=None,
                  connect_timeout=DEFAULT_TIMEOUT,
-                 read_timeout=DEFAULT_TIMEOUT):
+                 read_timeout=DEFAULT_TIMEOUT,
+                 s3=None):
         self.region_name = region_name
         self.signature_version = signature_version
         self.user_agent = user_agent
         self.user_agent_extra = user_agent_extra
         self.connect_timeout = connect_timeout
         self.read_timeout = read_timeout
+        self._validate_s3_configuration(s3)
+        self.s3 = s3
+
+    def _validate_s3_configuration(self, s3):
+        if s3 is not None:
+            addressing_style = s3.get('addressing_style')
+            if addressing_style not in ['virtual', 'auto', 'path', None]:
+                raise InvalidS3AddressingStyleError(
+                    s3_addressing_style=addressing_style)
diff --git a/botocore/data/_retry.json b/botocore/data/_retry.json
@@ -150,6 +150,28 @@
           }
         }
       }
+    },
+    "route53": {
+      "__default__": {
+        "policies": {
+          "request_limit_exceeded": {
+            "applies_when": {
+              "response": {
+                "service_error_code": "Throttling",
+                "http_status_code": 400
+              }
+            }
+          },
+          "still_processing": {
+            "applies_when": {
+              "response": {
+                "service_error_code": "PriorRequestNotComplete",
+                "http_status_code": 400
+              }
+            }
+          }
+        }
+      }
     }
   }
 }
diff --git a/botocore/data/ec2/2015-10-01/paginators-1.json b/botocore/data/ec2/2015-10-01/paginators-1.json
@@ -12,16 +12,22 @@
       "limit_key": "MaxResults",
       "result_key": "Reservations"
     },
+    "DescribeReservedInstancesOfferings": {
+      "input_token": "NextToken",
+      "output_token": "NextToken",
+      "limit_key": "MaxResults",
+      "result_key": "ReservedInstancesOfferings"
+    },
     "DescribeReservedInstancesModifications": {
       "input_token": "NextToken",
       "output_token": "NextToken",
       "result_key": "ReservedInstancesModifications"
     },
-    "DescribeReservedInstancesOfferings": {
+    "DescribeSnapshots": {
       "input_token": "NextToken",
       "output_token": "NextToken",
       "limit_key": "MaxResults",
-      "result_key": "ReservedInstancesOfferings"
+      "result_key": "Snapshots"
     },
     "DescribeSpotPriceHistory": {
       "input_token": "NextToken",
@@ -41,11 +47,11 @@
       "limit_key": "MaxResults",
       "result_key": "VolumeStatuses"
     },
-    "DescribeSnapshots": {
+    "DescribeVolumes": {
       "input_token": "NextToken",
       "output_token": "NextToken",
       "limit_key": "MaxResults",
-      "result_key": "Snapshots"
+      "result_key": "Volumes"
     }
   }
 }
diff --git a/botocore/data/glacier/2012-06-01/paginators-1.json b/botocore/data/glacier/2012-06-01/paginators-1.json
@@ -0,0 +1,28 @@
+{
+  "pagination": {
+    "ListJobs": {
+      "input_token": "marker",
+      "output_token": "Marker",
+      "limit_key": "limit",
+      "result_key": "JobList"
+    },
+    "ListMultipartUploads": {
+      "input_token": "marker",
+      "output_token": "Marker",
+      "limit_key": "limit",
+      "result_key": "UploadsList"
+    },
+    "ListParts": {
+      "input_token": "marker",
+      "output_token": "Marker",
+      "limit_key": "limit",
+      "result_key": "Parts"
+    },
+    "ListVaults": {
+      "input_token": "marker",
+      "output_token": "Marker",
+      "limit_key": "limit",
+      "result_key": "VaultList"
+    }
+  }
+}
diff --git a/botocore/data/iam/2010-05-08/paginators-1.json b/botocore/data/iam/2010-05-08/paginators-1.json
@@ -24,6 +24,38 @@
       "limit_key": "MaxItems",
       "result_key": "AccountAliases"
     },
+    "ListAttachedGroupPolicies": {
+      "input_token": "Marker",
+      "output_token": "Marker",
+      "more_results": "IsTruncated",
+      "limit_key": "MaxItems",
+      "result_key": "AttachedPolicies"
+    },
+    "ListAttachedRolePolicies": {
+      "input_token": "Marker",
+      "output_token": "Marker",
+      "more_results": "IsTruncated",
+      "limit_key": "MaxItems",
+      "result_key": "AttachedPolicies"
+    },
+    "ListAttachedUserPolicies": {
+      "input_token": "Marker",
+      "output_token": "Marker",
+      "more_results": "IsTruncated",
+      "limit_key": "MaxItems",
+      "result_key": "AttachedPolicies"
+    },
+    "ListEntitiesForPolicy": {
+      "input_token": "Marker",
+      "output_token": "Marker",
+      "more_results": "IsTruncated",
+      "limit_key": "MaxItems",
+      "result_key": [
+        "PolicyGroups",
+        "PolicyUsers",
+        "PolicyRoles"
+      ]
+    },
     "ListGroupPolicies": {
       "input_token": "Marker",
       "output_token": "Marker",
@@ -66,26 +98,33 @@
       "limit_key": "MaxItems",
       "result_key": "MFADevices"
     },
-    "ListRolePolicies": {
+    "ListPolicies": {
       "input_token": "Marker",
       "output_token": "Marker",
       "more_results": "IsTruncated",
       "limit_key": "MaxItems",
-      "result_key": "PolicyNames"
+      "result_key": "Policies"
     },
-    "ListRoles": {
+    "ListPolicyVersions": {
       "input_token": "Marker",
       "output_token": "Marker",
       "more_results": "IsTruncated",
       "limit_key": "MaxItems",
-      "result_key": "Roles"
+      "result_key": "Versions"
     },
-    "ListPolicies": {
+    "ListRolePolicies": {
       "input_token": "Marker",
       "output_token": "Marker",
       "more_results": "IsTruncated",
       "limit_key": "MaxItems",
-      "result_key": "Policies"
+      "result_key": "PolicyNames"
+    },
+    "ListRoles": {
+      "input_token": "Marker",
+      "output_token": "Marker",
+      "more_results": "IsTruncated",
+      "limit_key": "MaxItems",
+      "result_key": "Roles"
     },
     "ListServerCertificates": {
       "input_token": "Marker",