feat: add opt-in support for S3 us-east-1 regional endpoint (#2960)

* generalize resolveRegionalEndpointsFlag function; update STS customization

.changes/next-release/feature-Region-e1805cab.json

@@ -0,0 +1,5 @@
+{
+  "type": "feature",
+  "category": "Region",
+  "description": "s3 client now support sending request to us-east-1 regional endpoint with `s3UsEast1RegionalEndpoint` client configuration set to `regional`"
+}

lib/config.js

@@ -82,6 +82,12 @@ var PromisesDependency;
  *   @return [Boolean] whether to disable S3 body signing when using signature version `v4`.
  *     Body signing can only be disabled when using https. Defaults to `true`.
  *
+ * @!attribute s3UsEast1RegionalEndpoint
+ *   @return ['legacy'|'regional'] when region is set to 'us-east-1', whether to send s3
+ *     request to global endpoints or 'us-east-1' regional endpoints. This config is only
+ *     applicable to S3 client;
+ *     Defaults to 'legacy'
+ *
  * @!attribute useAccelerateEndpoint
  *   @note This configuration option is only compatible with S3 while accessing
  *     dns-compatible buckets.
@@ -238,6 +244,10 @@ AWS.Config = AWS.util.inherit({
    * @option options s3DisableBodySigning [Boolean] whether S3 body signing
    *   should be disabled when using signature version `v4`. Body signing
    *   can only be disabled when using https. Defaults to `true`.
+   * @option options s3UsEast1RegionalEndpoint ['legacy'|'regional'] when region
+   *   is set to 'us-east-1', whether to send s3 request to global endpoints or
+   *   'us-east-1' regional endpoints. This config is only applicable to S3 client.
+   *   Defaults to `legacy`
    *
    * @option options retryDelayOptions [map] A set of options to configure
    *   the retry delay on retryable errors. Currently supported options are:
@@ -523,6 +533,7 @@ AWS.Config = AWS.util.inherit({
     s3ForcePathStyle: false,
     s3BucketEndpoint: false,
     s3DisableBodySigning: true,
+    s3UsEast1RegionalEndpoint: 'legacy',
     computeChecksums: true,
     convertResponseTypes: true,
     correctClockSkew: false,
@@ -537,7 +548,7 @@ AWS.Config = AWS.util.inherit({
     endpointDiscoveryEnabled: false,
     endpointCacheSize: 1000,
     hostPrefixEnabled: true,
-    stsRegionalEndpoints: null
+    stsRegionalEndpoints: 'legacy'
   },
 
   /**

lib/config_regional_endpoint.js

@@ -0,0 +1,67 @@
+var AWS = require('./core');
+/**
+ * @api private
+ */
+function validateRegionalEndpointsFlagValue(configValue, errorOptions) {
+  if (typeof configValue !== 'string') return undefined;
+  else if (['legacy', 'regional'].indexOf(configValue.toLowerCase()) >= 0) {
+    return configValue.toLowerCase();
+  } else {
+    throw AWS.util.error(new Error(), errorOptions);
+  }
+}
+
+/**
+ * Resolve the configuration value for regional endpoint from difference sources: client
+ * config, environmental variable, shared config file. Value can be case-insensitive
+ * 'legacy' or 'reginal'.
+ * @param originalConfig user-supplied config object to resolve
+ * @param options a map of config property names from individual configuration source
+ *  - env: name of environmental variable that refers to the config
+ *  - sharedConfig: name of shared configuration file property that refers to the config
+ *  - clientConfig: name of client configuration property that refers to the config
+ *
+ * @api private
+ */
+function resolveRegionalEndpointsFlag(originalConfig, options) {
+  originalConfig = originalConfig || {};
+  //validate config value
+  var resolved;
+  if (originalConfig[options.clientConfig]) {
+    resolved = validateRegionalEndpointsFlagValue(originalConfig[options.clientConfig], {
+      code: 'InvalidConfiguration',
+      message: 'invalid "' + options.clientConfig + '" configuration. Expect "legacy" ' +
+      ' or "regional". Got "' + originalConfig[options.clientConfig] + '".'
+    });
+    if (resolved) return resolved;
+  }
+  if (!AWS.util.isNode()) return resolved;
+  //validate environmental variable
+  if (Object.prototype.hasOwnProperty.call(process.env, options.env)) {
+    var envFlag = process.env[options.env];
+    resolved = validateRegionalEndpointsFlagValue(envFlag, {
+      code: 'InvalidEnvironmentalVariable',
+      message: 'invalid ' + options.env + ' environmental variable. Expect "legacy" ' +
+      ' or "regional". Got "' + process.env[options.env] + '".'
+    });
+    if (resolved) return resolved;
+  }
+  //validate shared config file
+  var profile = {};
+  try {
+    var profiles = AWS.util.getProfilesFromSharedConfig(AWS.util.iniLoader);
+    profile = profiles[process.env.AWS_PROFILE || AWS.util.defaultProfile];
+  } catch (e) {};
+  if (profile && Object.prototype.hasOwnProperty.call(profile, options.sharedConfig)) {
+    var fileFlag = profile[options.sharedConfig];
+    resolved = validateRegionalEndpointsFlagValue(fileFlag, {
+      code: 'InvalidConfiguration',
+      message: 'invalid ' + options.sharedConfig + ' profile config. Expect "legacy" ' +
+      ' or "regional". Got "' + profile[options.sharedConfig] + '".'
+    });
+    if (resolved) return resolved;
+  }
+  return resolved;
+}
+
+module.exports = resolveRegionalEndpointsFlag;

lib/services/s3.js

@@ -1,5 +1,6 @@
 var AWS = require('../core');
 var v4Credentials = require('../signers/v4_credentials');
+var resolveRegionalEndpointsFlag = require('../config_regional_endpoint');
 
 // Pull in managed upload extension
 require('../s3/managed_upload');
@@ -105,6 +106,7 @@ AWS.util.update(AWS.S3.prototype, {
     request.addListener('validate', this.validateBucketEndpoint);
     request.addListener('validate', this.correctBucketRegionFromCache);
     request.addListener('validate', this.validateBucketName, prependListener);
+    request.addListener('validate', this.optInUsEast1RegionalEndpoint, prependListener);
 
     request.addListener('build', this.addContentType);
     request.addListener('build', this.populateURI);
@@ -189,6 +191,34 @@ AWS.util.update(AWS.S3.prototype, {
     return invalidOperations.indexOf(operation) === -1;
   },
 
+  /**
+   * When us-east-1 region endpoint configuration is set, in stead of sending request to
+   * global endpoint(e.g. 's3.amazonaws.com'), we will send request to
+   * 's3.us-east-1.amazonaws.com'.
+   * @api private
+   */
+  optInUsEast1RegionalEndpoint: function optInUsEast1RegionalEndpoint(req) {
+    var service = req.service;
+    var config = service.config;
+    config.s3UsEast1RegionalEndpoint = resolveRegionalEndpointsFlag(service._originalConfig, {
+      env: 'AWS_S3_US_EAST_1_REGIONAL_ENDPOINT',
+      sharedConfig: 's3_us_east_1_regional_endpoint',
+      clientConfig: 's3UsEast1RegionalEndpoint'
+    });
+    if (
+      !(service._originalConfig || {}).endpoint &&
+      req.httpRequest.region === 'us-east-1' &&
+      config.s3UsEast1RegionalEndpoint === 'regional' &&
+      req.httpRequest.endpoint.hostname.indexOf('s3.amazonaws.com') >= 0
+    ) {
+      var insertPoint = config.endpoint.indexOf('.amazonaws.com');
+      regionalEndpoint = config.endpoint.substring(0, insertPoint) +
+        '.us-east-1' + config.endpoint.substring(insertPoint);
+      var endpoint = req.httpRequest.endpoint;
+      endpoint.hostname = regionalEndpoint;
+      endpoint.host = regionalEndpoint;
+    }
+  },
 
   /**
    * S3 prefers dns-compatible bucket names to be moved from the uri path

lib/services/sts.js

@@ -1,5 +1,5 @@
 var AWS = require('../core');
-var regionConfig = require('../region_config');
+var resolveRegionalEndpointsFlag = require('../config_regional_endpoint');
 var ENV_REGIONAL_ENDPOINT_ENABLED = 'AWS_STS_REGIONAL_ENDPOINTS';
 var CONFIG_REGIONAL_ENDPOINT_ENABLED = 'sts_regional_endpoints';
 
@@ -51,77 +51,38 @@ AWS.util.update(AWS.STS.prototype, {
   /**
    * @api private
    */
-  validateRegionalEndpointsFlagValue: function validateRegionalEndpointsFlagValue(configValue, errorOptions) {
-    if (typeof configValue === 'string' && ['legacy', 'regional'].indexOf(configValue.toLowerCase()) >= 0) {
-      this.config.stsRegionalEndpoints = configValue.toLowerCase();
-      return;
-    } else {
-      throw AWS.util.error(new Error(), errorOptions);
-    }
+  setupRequestListeners: function setupRequestListeners(request) {
+    request.addListener('validate', this.optInRegionalEndpoint, true);
   },
 
   /**
    * @api private
    */
-  validateRegionalEndpointsFlag: function validateRegionalEndpointsFlag() {
-    //validate config value
-    var config = this.config;
-    if (config.stsRegionalEndpoints) {
-      this.validateRegionalEndpointsFlagValue(config.stsRegionalEndpoints, {
-        code: 'InvalidConfiguration',
-        message: 'invalid "stsRegionalEndpoints" configuration. Expect "legacy" ' +
-        ' or "regional". Got "' + config.stsRegionalEndpoints + '".'
-      });
-    }
-    if (!AWS.util.isNode()) return;
-    //validate environmental variable
-    if (Object.prototype.hasOwnProperty.call(process.env, ENV_REGIONAL_ENDPOINT_ENABLED)) {
-      var envFlag = process.env[ENV_REGIONAL_ENDPOINT_ENABLED];
-      this.validateRegionalEndpointsFlagValue(envFlag, {
-        code: 'InvalidEnvironmentalVariable',
-        message: 'invalid ' + ENV_REGIONAL_ENDPOINT_ENABLED + ' environmental variable. Expect "legacy" ' +
-        ' or "regional". Got "' + process.env[ENV_REGIONAL_ENDPOINT_ENABLED] + '".'
-      });
-    }
-    //validate shared config file
-    var profile = {};
-    try {
-      var profiles = AWS.util.getProfilesFromSharedConfig(AWS.util.iniLoader);
-      profile = profiles[process.env.AWS_PROFILE || AWS.util.defaultProfile];
-    } catch (e) {};
-    if (profile && Object.prototype.hasOwnProperty.call(profile, CONFIG_REGIONAL_ENDPOINT_ENABLED)) {
-      var fileFlag = profile[CONFIG_REGIONAL_ENDPOINT_ENABLED];
-      this.validateRegionalEndpointsFlagValue(fileFlag, {
-        code: 'InvalidConfiguration',
-        message: 'invalid '+CONFIG_REGIONAL_ENDPOINT_ENABLED+' profile config. Expect "legacy" ' +
-        ' or "regional". Got "' + profile[CONFIG_REGIONAL_ENDPOINT_ENABLED] + '".'
-      });
-    }
-  },
-
-  /**
-   * @api private
-   */
-  optInRegionalEndpoint: function optInRegionalEndpoint() {
-    this.validateRegionalEndpointsFlag();
-    var config = this.config;
-    if (config.stsRegionalEndpoints === 'regional') {
-      regionConfig(this);
-      if (!this.isGlobalEndpoint) return;
-      this.isGlobalEndpoint = false;
+  optInRegionalEndpoint: function optInRegionalEndpoint(req) {
+    var service = req.service;
+    var config = service.config;
+    config.stsRegionalEndpoints = resolveRegionalEndpointsFlag(service._originalConfig, {
+      env: ENV_REGIONAL_ENDPOINT_ENABLED,
+      sharedConfig: CONFIG_REGIONAL_ENDPOINT_ENABLED,
+      clientConfig: 'stsRegionalEndpoints'
+    });
+    if (
+      config.stsRegionalEndpoints === 'regional' &&
+      service.isGlobalEndpoint
+    ) {
       //client will throw if region is not supplied; request will be signed with specified region
       if (!config.region) {
         throw AWS.util.error(new Error(),
           {code: 'ConfigError', message: 'Missing region in config'});
       }
       var insertPoint = config.endpoint.indexOf('.amazonaws.com');
-      config.endpoint = config.endpoint.substring(0, insertPoint) +
+      regionalEndpoint = config.endpoint.substring(0, insertPoint) +
         '.' + config.region + config.endpoint.substring(insertPoint);
+      var endpoint = req.httpRequest.endpoint;
+      endpoint.hostname = regionalEndpoint;
+      endpoint.host = regionalEndpoint;
+      req.httpRequest.region = config.region;
     }
-  },
-
-  validateService: function validateService() {
-    this.optInRegionalEndpoint();
   }
 
 });

scripts/region-checker/whitelist.js

@@ -2,7 +2,11 @@ var whitelist = {
     '/config.js': [
         24,
         25,
-        187
+        85,
+        86,
+        193,
+        247,
+        248
     ],
     '/credentials/cognito_identity_credentials.js': [
         78,
@@ -25,16 +29,20 @@ var whitelist = {
         316
     ],
     '/services/s3.js': [
-        68,
         69,
-        515,
-        517,
-        516,
-        636,
-        647,
-        648,
-        649,
-        654
+        70,
+        194,
+        196,
+        209,
+        215,
+        545,
+        546,
+        547,
+        666,
+        677,
+        678,
+        684,
+        679
     ]
 };