Bug/region endpoints

S3/ConnMan.py

@@ -83,7 +83,7 @@ def __init__(self, id, hostname, ssl, cfg):
                 debug(u'non-proxied HTTPSConnection(%s)' % hostname)
 
             # S3's wildcart certificate doesn't work with DNS-style named buckets.
-            if 's3.amazonaws.com' in hostname and http_connection.context:
+            if 'amazonaws.com' in hostname and http_connection.context:
                 http_connection.context.check_hostname = False
                 debug(u'Disabling SSL certificate hostname verification for S3 wildcard cert')
 

S3/RegionEndpoints.py

@@ -0,0 +1,17 @@
+region_endpoints = {'us-east-1'      : 's3.amazonaws.com',
+                    'US'             : 's3.amazonaws.com',
+                    'us-west-1'      : 's3-us-west-1.amazonaws.com',
+                    'us-west-2'      : 's3-us-west-2.amazonaws.com',
+                    'eu-west-1'      : 's3-eu-west-1.amazonaws.com',
+                    'EU'             : 's3-eu-west-1.amazonaws.com',
+                    'eu-central-1'   : 's3-eu-central-1.amazonaws.com',
+                    'ap-southeast-1' : 's3-ap-southeast-1.amazonaws.com',
+                    'ap-southeast-2' : 's3-ap-southeast-2.amazonaws.com',
+                    'ap-northeast-1' : 's3-ap-northeast-1.amazonaws.com',
+                    'sa-east-1'      : 's3-sa-east-1.amazonaws.com',
+                    }
+
+def region_endpoint(region):
+    if region in region_endpoints:
+        return region_endpoints[region]
+    return None

S3/S3.py

@@ -34,6 +34,7 @@
 from S3Uri import S3Uri
 from ConnMan import ConnMan
 from Crypto import sign_string_v2, sign_string_v4, checksum_sha256
+from RegionEndpoints import region_endpoint
 
 try:
     import magic
@@ -963,6 +964,7 @@ def send_request(self, request, retries = _max_retries):
             if region is not None:
                 S3Request.region_map[request.resource['bucket']] = region
                 warning('Forwarding request to %s' % region)
+                set_host_bucket(region)
                 return self.send_request(request)
             else:
                 warning('Could not determine bucket location. Please consider using --region parameter.')
@@ -1107,6 +1109,7 @@ def send_file(self, request, file, labels, buffer = '', throttle = 0, retries =
             if region is not None:
                 S3Request.region_map[request.resource['bucket']] = region
                 warning('Forwarding request to %s' % region)
+                set_host_bucket(region)
                 return self.send_file(request, file, labels, buffer, offset = offset, chunk_size = chunk_size)
 
         # S3 from time to time doesn't send ETag back in a response :-(
@@ -1221,6 +1224,7 @@ def recv_file(self, request, stream, labels, start_position = 0, retries = _max_
             if region is not None:
                 S3Request.region_map[request.resource['bucket']] = region
                 warning('Forwarding request to %s' % region)
+                set_host_bucket(region)
                 return self.recv_file(request, stream, labels)
 
         if response["status"] < 200 or response["status"] > 299:
@@ -1328,4 +1332,13 @@ def compute_content_md5(body):
     if base64md5[-1] == '\n':
         base64md5 = base64md5[0:-1]
     return base64md5
+
+def set_host_bucket(region):
+    cfg = Config()
+    if region:
+        endpoint = region_endpoint(region)
+        if endpoint:
+            cfg.host_base = endpoint
+            cfg.host_bucket = "%(bucket)s." + endpoint
+__all__.append('set_host_bucket')
 # vim:et:ts=4:sts=4:ai

s3cmd

@@ -2411,6 +2411,10 @@ def main():
     ## Set socket read()/write() timeout
     socket.setdefaulttimeout(cfg.socket_timeout)
 
+    ## Use region-specific endpoints when region is known
+    if options.bucket_location:
+        set_host_bucket(options.bucket_location)
+
     if cfg.encrypt and cfg.gpg_passphrase == "":
         error(u"Encryption requested but no passphrase set in config file.")
         error(u"Please re-run 's3cmd --configure' and supply it.")
@@ -2516,7 +2520,7 @@ if __name__ == '__main__':
         from S3.ExitCodes import *
         from S3.Exceptions import *
         from S3 import PkgInfo
-        from S3.S3 import S3
+        from S3.S3 import S3, set_host_bucket
         from S3.Config import Config
         from S3.SortedDict import SortedDict
         from S3.FileDict import FileDict