Release 2.6.0

CHANGELOG.md

@@ -1,5 +1,13 @@
 # CHANGELOG
 
+# 2.6.0
+## Content
+### Features
+* Support for On-Premise IBM Cloud Object Storage (3.14.8+)
+### Defect Fixes
+* IBM Python SDK aligned with AWS Python SDK - Boto3(v1.10.36), Botocore(v1.13.36) and S3transfer(v0.2.1)
+* Update psutil to use more recent package version (psutil>=5.6.6,<5.6.8)
+
 # 2.5.5
 ## Content
 ### Defect Fixes

error]

@@ -0,0 +1,53 @@
+nose.plugins.cover: ERROR: Coverage not available: unable to import coverage module
+Traceback (most recent call last):
+  File "scripts/ci/run-tests", line 19, in <module>
+    run('nosetests --with-coverage --cover-erase --cover-package ibm_botocore '
+  File "scripts/ci/run-tests", line 16, in run
+    return check_call(command, shell=True)
+  File "/root/.pyenv/versions/2.7.8/lib/python2.7/subprocess.py", line 535, in check_call
+    retcode = call(*popenargs, **kwargs)
+  File "/root/.pyenv/versions/2.7.8/lib/python2.7/subprocess.py", line 522, in call
+    return Popen(*popenargs, **kwargs).wait()
+  File "/root/.pyenv/versions/2.7.8/lib/python2.7/subprocess.py", line 1376, in wait
+    pid, sts = _eintr_retry_call(os.waitpid, self.pid, 0)
+  File "/root/.pyenv/versions/2.7.8/lib/python2.7/subprocess.py", line 476, in _eintr_retry_call
+    return func(*args)
+KeyboardInterrupt
+Traceback (most recent call last):
+  File "/root/.pyenv/versions/2.7.8/bin/nosetests", line 11, in <module>
+    sys.exit(run_exit())
+  File "/root/.pyenv/versions/2.7.8/lib/python2.7/site-packages/nose/core.py", line 121, in __init__
+    **extra_args)
+  File "/root/.pyenv/versions/2.7.8/lib/python2.7/unittest/main.py", line 94, in __init__
+    self.parseArgs(argv)
+  File "/root/.pyenv/versions/2.7.8/lib/python2.7/site-packages/nose/core.py", line 179, in parseArgs
+    self.createTests()
+  File "/root/.pyenv/versions/2.7.8/lib/python2.7/site-packages/nose/core.py", line 193, in createTests
+    self.test = self.testLoader.loadTestsFromNames(self.testNames)
+  File "/root/.pyenv/versions/2.7.8/lib/python2.7/site-packages/nose/loader.py", line 481, in loadTestsFromNames
+    return unittest.TestLoader.loadTestsFromNames(self, names, module)
+  File "/root/.pyenv/versions/2.7.8/lib/python2.7/unittest/loader.py", line 130, in loadTestsFromNames
+    suites = [self.loadTestsFromName(name, module) for name in names]
+  File "/root/.pyenv/versions/2.7.8/lib/python2.7/site-packages/nose/loader.py", line 433, in loadTestsFromName
+    discovered=discovered)
+  File "/root/.pyenv/versions/2.7.8/lib/python2.7/site-packages/nose/loader.py", line 354, in loadTestsFromModule
+    tests.extend(self.loadTestsFromDir(module_path))
+  File "/root/.pyenv/versions/2.7.8/lib/python2.7/site-packages/nose/loader.py", line 183, in loadTestsFromDir
+    entry_path, discovered=True)
+  File "/root/.pyenv/versions/2.7.8/lib/python2.7/site-packages/nose/loader.py", line 418, in loadTestsFromName
+    addr.filename, addr.module)
+  File "/root/.pyenv/versions/2.7.8/lib/python2.7/site-packages/nose/importer.py", line 47, in importFromPath
+    return self.importFromDir(dir_path, fqname)
+  File "/root/.pyenv/versions/2.7.8/lib/python2.7/site-packages/nose/importer.py", line 94, in importFromDir
+    mod = load_module(part_fqname, fh, filename, desc)
+  File "/git/cleversafe/sdk/python/ibm-cos-sdk-python-core/tests/functional/test_waiter_config.py", line 14, in <module>
+    from jsonschema import Draft4Validator
+  File "/root/.pyenv/versions/2.7.8/lib/python2.7/site-packages/jsonschema/__init__.py", line 12, in <module>
+    from jsonschema.exceptions import (
+  File "/root/.pyenv/versions/2.7.8/lib/python2.7/site-packages/jsonschema/exceptions.py", line 6, in <module>
+    from jsonschema import _utils
+  File "/root/.pyenv/versions/2.7.8/lib/python2.7/site-packages/jsonschema/_utils.py", line 6, in <module>
+    from jsonschema.compat import str_types, MutableMapping, urlsplit
+  File "/root/.pyenv/versions/2.7.8/lib/python2.7/site-packages/jsonschema/compat.py", line 1, in <module>
+    import operator
+KeyboardInterrupt

ibm_botocore/__init__.py

@@ -17,7 +17,7 @@
 import logging
 
 __author__ = 'IBM'
-__version__ = '2.5.5'
+__version__ = '2.6.0'
 
 
 class NullHandler(logging.Handler):

ibm_botocore/args.py

@@ -20,6 +20,7 @@
 import logging
 import socket
 
+import ibm_botocore.exceptions
 import ibm_botocore.serialize
 import ibm_botocore.utils
 from ibm_botocore.signers import RequestSigner
@@ -30,14 +31,39 @@
 logger = logging.getLogger(__name__)
 
 
+VALID_STS_REGIONAL_ENDPOINTS_CONFIG = [
+    'legacy',
+    'regional',
+]
+LEGACY_GLOBAL_STS_REGIONS = [
+    'ap-northeast-1',
+    'ap-south-1',
+    'ap-southeast-1',
+    'ap-southeast-2',
+    'aws-global',
+    'ca-central-1',
+    'eu-central-1',
+    'eu-north-1',
+    'eu-west-1',
+    'eu-west-2',
+    'eu-west-3',
+    'sa-east-1',
+    'us-east-1',
+    'us-east-2',
+    'us-west-1',
+    'us-west-2',
+]
+
+
 class ClientArgsCreator(object):
     def __init__(self, event_emitter, user_agent, response_parser_factory,
-                 loader, exceptions_factory):
+                 loader, exceptions_factory, config_store):
         self._event_emitter = event_emitter
         self._user_agent = user_agent
         self._response_parser_factory = response_parser_factory
         self._loader = loader
         self._exceptions_factory = exceptions_factory
+        self._config_store = config_store
 
     def get_client_args(self, service_model, region_name, is_secure,
                         endpoint_url, verify, credentials, scoped_config,
@@ -113,9 +139,6 @@ def compute_client_args(self, service_model, client_config,
             if raw_value is not None:
                 parameter_validation = ibm_botocore.utils.ensure_boolean(raw_value)
 
-        endpoint_config = endpoint_bridge.resolve(
-            service_name, region_name, endpoint_url, is_secure)
-
         # Override the user agent if specified in the client config.
         user_agent = self._user_agent
         if client_config is not None:
@@ -124,6 +147,13 @@ def compute_client_args(self, service_model, client_config,
             if client_config.user_agent_extra is not None:
                 user_agent += ' %s' % client_config.user_agent_extra
 
+        endpoint_config = self._compute_endpoint_config(
+            service_name=service_name,
+            region_name=region_name,
+            endpoint_url=endpoint_url,
+            is_secure=is_secure,
+            endpoint_bridge=endpoint_bridge,
+        )
         # Create a new client config to be passed to the client based
         # on the final values. We do not want the user to be able
         # to try to modify an existing client with a client config.
@@ -141,8 +171,7 @@ def compute_client_args(self, service_model, client_config,
                 client_cert=client_config.client_cert,
                 inject_host_prefix=client_config.inject_host_prefix,
             )
-        s3_config = self.compute_s3_config(scoped_config,
-                                           client_config)
+        s3_config = self.compute_s3_config(client_config)
         return {
             'service_name': service_name,
             'parameter_validation': parameter_validation,
@@ -154,29 +183,8 @@ def compute_client_args(self, service_model, client_config,
             'socket_options': self._compute_socket_options(scoped_config)
         }
 
-    def compute_s3_config(self, scoped_config, client_config):
-        s3_configuration = None
-
-        # Check the scoped config first.
-        if scoped_config is not None:
-            s3_configuration = scoped_config.get('s3')
-            # Until we have proper validation of the config file (including
-            # nested types), we have to account for the fact that the s3
-            # key could be parsed as a string, e.g 's3 = foo'.
-            # In the case we'll ignore the key for now.
-            if not isinstance(s3_configuration, dict):
-                logger.debug("The s3 config key is not a dictionary type, "
-                             "ignoring its value of: %s", s3_configuration)
-                s3_configuration = None
-
-            # Convert logic for several s3 keys in the scoped config
-            # so that the various strings map to the appropriate boolean value.
-            if s3_configuration:
-                boolean_keys = ['use_accelerate_endpoint',
-                                'use_dualstack_endpoint',
-                                'payload_signing_enabled']
-                s3_configuration = self._convert_config_to_bool(
-                    s3_configuration, boolean_keys)
+    def compute_s3_config(self, client_config):
+        s3_configuration = self._config_store.get_config_variable('s3')
 
         # Next specific client config values takes precedence over
         # specific values in the scoped config.
@@ -194,14 +202,41 @@ def compute_s3_config(self, scoped_config, client_config):
 
         return s3_configuration
 
-    def _convert_config_to_bool(self, config_dict, keys):
-        # Make sure any further modifications to this section of the config
-        # will not affect the scoped config by making a copy of it.
-        config_copy = config_dict.copy()
-        present_keys = [k for k in keys if k in config_copy]
-        for key in present_keys:
-            config_copy[key] = ibm_botocore.utils.ensure_boolean(config_copy[key])
-        return config_copy
+    def _compute_endpoint_config(self, service_name, region_name, endpoint_url,
+                                 is_secure, endpoint_bridge):
+        endpoint_config = endpoint_bridge.resolve(
+            service_name, region_name, endpoint_url, is_secure)
+        if self._should_set_global_sts_endpoint(
+                service_name, region_name, endpoint_url):
+            self._set_global_sts_endpoint(endpoint_config, is_secure)
+        return endpoint_config
+
+    def _should_set_global_sts_endpoint(self, service_name, region_name,
+                                        endpoint_url):
+        if service_name != 'sts':
+            return False
+        if endpoint_url:
+            return False
+        return (
+            self._get_sts_regional_endpoints_config() == 'legacy' and
+            region_name in LEGACY_GLOBAL_STS_REGIONS
+        )
+
+    def _get_sts_regional_endpoints_config(self):
+        sts_regional_endpoints_config = self._config_store.get_config_variable(
+            'sts_regional_endpoints')
+        if not sts_regional_endpoints_config:
+            sts_regional_endpoints_config = 'legacy'
+        if sts_regional_endpoints_config not in \
+                VALID_STS_REGIONAL_ENDPOINTS_CONFIG:
+            raise ibm_botocore.exceptions.InvalidSTSRegionalEndpointsConfigError(
+                sts_regional_endpoints_config=sts_regional_endpoints_config)
+        return sts_regional_endpoints_config
+
+    def _set_global_sts_endpoint(self, endpoint_config, is_secure):
+        scheme = 'https' if is_secure else 'http'
+        endpoint_config['endpoint_url'] = '%s://sts.amazonaws.com' % scheme
+        endpoint_config['signing_region'] = 'us-east-1'
 
     def _get_default_s3_region(self, service_name, endpoint_bridge):
         # If a user is providing a custom URL, the endpoint resolver will

ibm_botocore/auth.py

@@ -411,19 +411,6 @@ def _set_necessary_date_headers(self, request):
 
 
 class S3SigV4Auth(SigV4Auth):
-    def __init__(self, credentials, service_name, region_name):
-        super(S3SigV4Auth, self).__init__(
-            credentials, service_name, region_name)
-        self._default_region_name = region_name
-
-    def add_auth(self, request):
-        # If we ever decide to share auth sessions, this could potentially be
-        # a source of concurrency bugs.
-        signing_context = request.context.get('signing', {})
-        self._region_name = signing_context.get(
-            'region', self._default_region_name)
-        super(S3SigV4Auth, self).add_auth(request)
-
     def _modify_request_before_signing(self, request):
         super(S3SigV4Auth, self)._modify_request_before_signing(request)
         if 'X-Amz-Content-SHA256' in request.headers:

ibm_botocore/client.py

@@ -26,13 +26,10 @@
 from ibm_botocore.paginate import Paginator
 from ibm_botocore.utils import CachedProperty
 from ibm_botocore.utils import get_service_module_name
-from ibm_botocore.utils import switch_host_s3_accelerate
 from ibm_botocore.utils import S3RegionRedirector
-from ibm_botocore.utils import fix_s3_host
-from ibm_botocore.utils import switch_to_virtual_host_style
-from ibm_botocore.utils import S3_ACCELERATE_WHITELIST
+from ibm_botocore.utils import S3ArnParamHandler
+from ibm_botocore.utils import S3EndpointSetter
 from ibm_botocore.args import ClientArgsCreator
-from ibm_botocore.compat import urlsplit
 from ibm_botocore import UNSIGNED
 # Keep this imported.  There's pre-existing code that uses
 # "from ibm_botocore.client import Config".
@@ -168,104 +165,22 @@ def _register_endpoint_discovery(self, client, endpoint_url, config):
             events.register('before-parameter-build',
                             block_endpoint_discovery_required_operations)
 
-
     def _register_s3_events(self, client, endpoint_bridge, endpoint_url,
                             client_config, scoped_config):
         if client.meta.service_model.service_name != 's3':
             return
         S3RegionRedirector(endpoint_bridge, client).register()
-        self._set_s3_addressing_style(
-            endpoint_url, client.meta.config.s3, client.meta.events,
-            client.meta.partition
-        )
-        # Enable accelerate if the configuration is set to to true or the
-        # endpoint being used matches one of the accelerate endpoints.
-        if self._is_s3_accelerate(endpoint_url, client.meta.config.s3):
-            # Also make sure that the hostname gets switched to
-            # s3-accelerate.amazonaws.com
-            client.meta.events.register_first(
-                'before-sign.s3', switch_host_s3_accelerate)
-
+        S3ArnParamHandler().register(client.meta.events)
+        S3EndpointSetter(
+            endpoint_resolver=self._endpoint_resolver,
+            region=client.meta.region_name,
+            s3_config=client.meta.config.s3,
+            endpoint_url=endpoint_url,
+            partition=client.meta.partition
+        ).register(client.meta.events)
         self._set_s3_presign_signature_version(
             client.meta, client_config, scoped_config)
 
-    def _set_s3_addressing_style(self, endpoint_url, s3_config, event_emitter,
-                                 partition):
-        if s3_config is None:
-            s3_config = {}
-
-        addressing_style = self._get_s3_addressing_style(
-            endpoint_url, s3_config)
-        handler = self._get_s3_addressing_handler(
-            endpoint_url, s3_config, addressing_style, partition)
-        if handler is not None:
-            event_emitter.register('before-sign.s3', handler)
-
-    def _get_s3_addressing_style(self, endpoint_url, s3_config):
-        # Use virtual host style addressing if accelerate is enabled or if
-        # the given endpoint url is an accelerate endpoint.
-        accelerate = s3_config.get('use_accelerate_endpoint', False)
-        if accelerate or self._is_s3_accelerate(endpoint_url, s3_config):
-            return 'virtual'
-
-        # If a particular addressing style is configured, use it.
-        configured_addressing_style = s3_config.get('addressing_style')
-        if configured_addressing_style:
-            return configured_addressing_style
-
-    def _get_s3_addressing_handler(self, endpoint_url, s3_config,
-                                   addressing_style, partition):
-        # If virtual host style was configured, use it regardless of whether
-        # or not the bucket looks dns compatible.
-        if addressing_style == 'virtual':
-            logger.debug("Using S3 virtual host style addressing.")
-            return switch_to_virtual_host_style
-
-        # If path style is configured, no additional steps are needed. If
-        # endpoint_url was specified, don't default to virtual. We could
-        # potentially default provided endpoint urls to virtual hosted
-        # style, but for now it is avoided.
-        if addressing_style == 'path' or endpoint_url is not None:
-            logger.debug("Using S3 path style addressing.")
-            return None
-
-        logger.debug("Defaulting to S3 virtual host style addressing with "
-                     "path style addressing fallback.")
-
-        # By default, try to use virtual style with path fallback.
-        return fix_s3_host
-
-    def _is_s3_accelerate(self, endpoint_url, s3_config):
-        # Accelerate has been explicitly configured.
-        if s3_config is not None and s3_config.get('use_accelerate_endpoint'):
-            return True
-
-        # Accelerate mode is turned on automatically if an endpoint url is
-        # provided that matches the accelerate scheme.
-        if endpoint_url is None:
-            return False
-
-        # Accelerate is only valid for Amazon endpoints.
-        netloc = urlsplit(endpoint_url).netloc
-        if not netloc.endswith('amazonaws.com'):
-            return False
-
-        # The first part of the url should always be s3-accelerate.
-        parts = netloc.split('.')
-        if parts[0] != 's3-accelerate':
-            return False
-
-        # Url parts between 's3-accelerate' and 'amazonaws.com' which
-        # represent different url features.
-        feature_parts = parts[1:-2]
-
-        # There should be no duplicate url parts.
-        if len(feature_parts) != len(set(feature_parts)):
-            return False
-
-        # Remaining parts must all be in the whitelist.
-        return all(p in S3_ACCELERATE_WHITELIST for p in feature_parts)
-
     def _set_s3_presign_signature_version(self, client_meta,
                                           client_config, scoped_config):
         # This will return the manually configured signature version, or None
@@ -322,7 +237,7 @@ def _get_client_args(self, service_model, region_name, is_secure,
         args_creator = ClientArgsCreator(
             self._event_emitter, self._user_agent,
             self._response_parser_factory, self._loader,
-            self._exceptions_factory)
+            self._exceptions_factory, config_store=self._config_store)
         return args_creator.get_client_args(
             service_model, region_name, is_secure, endpoint_url,
             verify, credentials, scoped_config, client_config, endpoint_bridge)