feat: Configured endpoint resolution (aka service-specific endpoint c…

…onfiguration) (#1861)

* Add the ignoreConfiguredEndpointURLs config field in runtime side.

* Add ignoreConfiguredEndpointURLs config field to codegen side.

* Change default value for endpoint in AWS SDK codegen to use AWSClientConfigDefaultsProvider.configuredEndpoint that resolves configured endpoint if needed.

* Add configuredEndpoint static func to AWSClientConfigDefaultsProvider; it uses AWSEndpointConfig to resolve and return configured endpoint if needed & possible.

* Create AWSEndpointConfig that resolves configured endpoint if needed & possible. Add unit tests for it too.

* Add shared config's services section parsing logic to wrapper. Add a new test case for multiple services in services section case. Confirmed tests pass by depending on WIP branch of aws-crt-swift.

* Bump CRT version to 0.43.0

* Update codegen test

* Fix typos

* Typos

* Add trace-level logging for configured endpoint resolution.

* Address swiftlint warning

* Relocate configured endpoint resolution location from client config initialization to operation call stack.

* Update codegen test

* Add trace logging for configured endpoint, add error throw for when services section referenced by resolved profile doesn't exist, and add check for empty values for reoslved endpoint (empty values = proceed to next possible source). Implementation is 100% matching to SEP now.

* Update docc comment on client config for disable flag.

* Update runtime tests

---------

Co-authored-by: Sichan Yoo <chanyoo@amazon.com>

Package.swift

@@ -16,7 +16,7 @@ import PackageDescription
 // MARK: - Dynamic Content
 
 let clientRuntimeVersion: Version = "0.108.0"
-let crtVersion: Version = "0.42.0"
+let crtVersion: Version = "0.43.0"
 
 let excludeRuntimeUnitTests = false
 

Sources/Core/AWSClientRuntime/Sources/AWSClientRuntime/AWSClientConfigDefaultsProvider.swift

@@ -135,4 +135,16 @@ public class AWSClientConfigDefaultsProvider {
             rateLimitingMode: resolvedRateLimitingMode
         )
     }
+
+    public static func configuredEndpoint(
+        _ sdkID: String,
+        _ ignoreConfiguredEndpointURLs: Bool? = nil
+    ) throws -> String? {
+        let fileBasedConfig = try CRTFileBasedConfiguration.make()
+        return try AWSEndpointConfig.configuredEndpoint(
+            sdkID: sdkID,
+            ignoreConfiguredEndpointURLs: ignoreConfiguredEndpointURLs,
+            fileBasedConfig: fileBasedConfig
+        )
+    }
 }

Sources/Core/AWSClientRuntime/Sources/AWSClientRuntime/Config/AWSDefaultClientConfiguration.swift

@@ -45,4 +45,31 @@ public protocol AWSDefaultClientConfiguration {
     ///
     /// If set, this value gets used when resolving max attempts value from the standard progression of potential sources. If no value could be resolved, the SDK uses max attempts value of 3 by default.
     var maxAttempts: Int? { get set }
+
+    /// Specifies whether the endpoint configured via environment variables or shared config file should be used by the service client.
+    ///
+    /// If `false`, the endpoint for the service client is resolved in the following order:
+    ///     1. The `endpoint` value provided at service client initialization via the config.
+    ///     2. The value of `AWS_ENDPOINT_URL_<SERVICE>` environment variable. `<SERVICE>` is the transformed value of the given service client's `serviceName` property, where spaces are replaced with underscores and letters are uppercased. E.g., `"API Gateway"` => `"API_GATEWAY"`.
+    ///     3. The value of `AWS_ENDPOINT_URL` environment variable; this is used to configure the endpoint for all services.
+    ///     4. In the shared config file, the `endpoint_url` property under selected profile's `services` section's `<SERVICE> =` subsection. `<SERVICE>` is the transformed value of the given service client's `serviceName` property, where spaces are replaced with underscores and letters are lowercased. E.g., `"API Gateway` => `"api_gateway`.
+    ///     5. In the shared config file, the `endpoint_url` property under the selected profile; this is used to configure the endpoint for all services.
+    ///     6. If the endpoint wasn't configured anywhere, then AWS Swift SDK determines the endpoint to use for each operation call by using the built-in endpoint resolver for the service.
+    /// Note: For the profile name used in resolution steps 4 & 5, the value of `AWS_PROFILE` environment variable is used if set. Otherwise, `default` is used.
+    ///
+    /// If `true`, the endpoint for the service client is resolved in the following order:
+    ///     1. The `endpoint` value provided at service client initialization via the config.
+    ///     2. If `endpoint` wasn't configured by the user via service client config, AWS Swift SDK determines the endpoint to use for each operation call by using the built-in endpoint resolver for the service.
+    ///
+    /// If this config value isn't set, AWS Swift SDK looks at the relevant environment variable and shared config file property when endpoint for an operation is resolved. The flag value gets determined by looking at the possible sources in the following order:
+    ///     1. The value of `ignoreConfiguredEndpointURLs` in the client config.
+    ///     2. The value of `AWS_IGNORE_CONFIGURED_ENDPOINT_URLS` environment variable.
+    ///     3. In the shared config file, the value of `ignore_configured_endpoint_urls` property under the selected profile.
+    ///     4. If this flag was not set anywhere, the AWS Swift SDK defaults to the `false` case and attempts to resolve configured endpoint.
+    /// Note: For the profile name used in resolution step 3, the value of`AWS_PROFILE` environment variable is used if set. Otherwise, `default` is used.
+    ///
+    /// For more information on endpoint configuration via environment variables and the shared config file, see the [official AWS documentation](https://docs.aws.amazon.com/sdkref/latest/guide/feature-ss-endpoints.html).
+    ///
+    /// For the list of valid `<SERVICE>` values for the services, refer to [the official list](https://docs.aws.amazon.com/sdkref/latest/guide/ss-endpoints-table.html).
+    var ignoreConfiguredEndpointURLs: Bool? { get set }
 }

Sources/Core/AWSClientRuntime/Sources/AWSClientRuntime/Endpoints/AWSEndpointConfig.swift

@@ -0,0 +1,95 @@
+//
+// Copyright Amazon.com Inc. or its affiliates.
+// All Rights Reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+import class Foundation.ProcessInfo
+import enum Smithy.ClientError
+import struct Foundation.Locale
+import struct Smithy.SwiftLogger
+@_spi(FileBasedConfig) import AWSSDKCommon
+
+public enum AWSEndpointConfig {
+    static func configuredEndpoint(
+        sdkID: String,
+        ignoreConfiguredEndpointURLs: Bool?,
+        fileBasedConfig: FileBasedConfiguration
+    ) throws -> String? {
+        // First, resolve disable flag
+        let disableFlag = FieldResolver(
+            configValue: ignoreConfiguredEndpointURLs,
+            envVarName: "AWS_IGNORE_CONFIGURED_ENDPOINT_URLS",
+            configFieldName: "ignore_configured_endpoint_urls",
+            fileBasedConfig: fileBasedConfig,
+            profileName: nil,
+            converter: { Bool($0) }
+        ).value ?? false
+
+        // Resolve configured endpoint only if disableFlag is false
+        if disableFlag {
+            return nil
+        } else {
+            let logger = SwiftLogger(label: "ConfiguredEndpointResolver")
+            let removedSpaceID = sdkID.replacingOccurrences(of: " ", with: "_")
+            // 1. Environment variable
+            let env = ProcessInfo.processInfo.environment
+            //  a. Service-specific configured endpoint from `AWS_ENDPOINT_URL_<SERVICE>`
+            let uppercasedID = removedSpaceID.uppercased(with: Locale(identifier: "en_US"))
+            if let val = env["AWS_ENDPOINT_URL_\(uppercasedID)"], val != "" {
+                logger.trace("Resolved configured endpoint from AWS_ENDPOINT_URL_\(uppercasedID): \(val)")
+                return val
+            }
+            //  b. Global configured endpoint from `AWS_ENDPOINT_URL`
+            if let val = env["AWS_ENDPOINT_URL"], val != "" {
+                logger.trace("Resolved configured endpoint from AWS_ENDPOINT_URL: \(val)")
+                return val
+            }
+            // 2. Shared config property
+            let configuredEndpointKey = FileBasedConfigurationKey(rawValue: "endpoint_url")
+            let profileName = env["AWS_PROFILE"] ?? "default"
+            //  a. For service-specific configured endpoint.
+            //     Profile section => referenced services section => nested subsection with service name
+            /* E.g.,
+                 [profile dev]
+                 services = devServices
+
+                 [services devServices]
+                 s3 =
+                   endpoint_url = https://abcde:9000
+             */
+            if let servicesSectionName = fileBasedConfig.section(for: profileName)?.string(for: "services") {
+                guard let servicesSection = fileBasedConfig.section(for: servicesSectionName, type: .services) else {
+                    throw ClientError.dataNotFound(
+                        "The [services \(servicesSectionName)] section doesn't exist!"
+                    )
+                }
+                let serviceSubsection = servicesSection.subproperties(
+                    for: FileBasedConfigurationKey(
+                        rawValue: removedSpaceID.lowercased(with: Locale(identifier: "en_US"))
+                    )
+                )
+                if let val = serviceSubsection?.value(for: configuredEndpointKey), val != "" {
+                    logger.trace(
+                        "Resolved configured endpoint from service-specific field in shared config file: \(val)"
+                    )
+                    return val
+                }
+            }
+            //  b. For global configured endpoint. Profile section => property
+            /* E.g.,
+                  [profile dev]
+                  services = devServices
+                  endpoint_url = http://localhost:5567
+             */
+            if let val = fileBasedConfig.section(for: profileName)?.string(for: configuredEndpointKey), val != "" {
+                logger.trace("Resolved configured endpoint from global field in shared config file: \(val)")
+                return val
+            }
+            // 3. Configured endpoint not found anywhere; return nil.
+            logger.trace("No configured endpoint found. Defaulting to SDK logic for resolving endpoint...")
+            return nil
+        }
+    }
+}

Sources/Core/AWSClientRuntime/Tests/AWSClientRuntimeTests/Endpoints/ConfiguredEndpointTests.swift

@@ -0,0 +1,141 @@
+//
+// Copyright Amazon.com Inc. or its affiliates.
+// All Rights Reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+import XCTest
+@testable @_spi(FileBasedConfig) import AWSClientRuntime
+@_spi(FileBasedConfig) @testable import AWSSDKCommon
+
+class ConfiguredEndpointTests: XCTestCase {
+    // Shared values
+    let fileBasedConfig = try! CRTFileBasedConfiguration(
+        configFilePath: Bundle.module.path(forResource: "configured_endpoint_tests", ofType: nil)!
+    )
+    let sdkID = "Test Service"
+
+    // After each test, unset all environment variables that could've been set
+    override func tearDown() {
+        unsetenv("AWS_PROFILE")
+        unsetenv("AWS_ENDPOINT_URL")
+        unsetenv("AWS_ENDPOINT_URL_TEST_SERVICE")
+        unsetenv("AWS_IGNORE_CONFIGURED_ENDPOINT_URLS")
+    }
+
+    /*
+        DISABLE FLAG TESTS
+
+        Needs to resolve furthest left option for flag:
+            client-config | env-var | shared-config-file
+     */
+
+    func testClientConfigDisableFlag() throws {
+        let resolvedEndpoint = try AWSEndpointConfig.configuredEndpoint(
+            sdkID: sdkID,
+            ignoreConfiguredEndpointURLs: true,
+            fileBasedConfig: fileBasedConfig
+        )
+        XCTAssertNil(resolvedEndpoint)
+    }
+
+    func testEnvVarDisableFlag() throws {
+        setenv("AWS_IGNORE_CONFIGURED_ENDPOINT_URLS", "true", 1)
+        let resolvedEndpoint = try AWSEndpointConfig.configuredEndpoint(
+            sdkID: sdkID,
+            ignoreConfiguredEndpointURLs: nil,
+            fileBasedConfig: fileBasedConfig
+        )
+        XCTAssertNil(resolvedEndpoint)
+    }
+
+    func testSharedConfigFileDisableFlag() throws {
+        setenv("AWS_PROFILE", "ignoreConfiguredEndpointProfile", 1)
+        let resolvedEndpoint = try AWSEndpointConfig.configuredEndpoint(
+            sdkID: sdkID,
+            ignoreConfiguredEndpointURLs: nil,
+            fileBasedConfig: fileBasedConfig
+        )
+        XCTAssertNil(resolvedEndpoint)
+    }
+
+    /*
+        CONFIGURED ENDPOINT RESOLUTION TESTS
+
+        Needs to resolve furthest left option for configured endpoint:
+            service-specific-env-var | global-env-var | service-specific-config-file | global-config-file
+        *client-config is handled implicitly by executing configured endpoint resolution only if user didn't provide it
+     */
+
+    func testResolveServiceSpecificEnvVar() throws {
+        setenv("AWS_ENDPOINT_URL", "https://env-global.com:9000", 1)
+        setenv("AWS_ENDPOINT_URL_TEST_SERVICE", "https://env-specific.com:9000", 1)
+        let resolvedEndpoint = try AWSEndpointConfig.configuredEndpoint(
+            sdkID: sdkID,
+            ignoreConfiguredEndpointURLs: nil,
+            fileBasedConfig: fileBasedConfig
+        )
+        XCTAssertEqual(resolvedEndpoint, "https://env-specific.com:9000")
+    }
+
+    func testResolveGlobalEnvVar() throws {
+        setenv("AWS_ENDPOINT_URL", "https://env-global.com:9000", 1)
+        let resolvedEndpoint = try AWSEndpointConfig.configuredEndpoint(
+            sdkID: sdkID,
+            ignoreConfiguredEndpointURLs: nil,
+            fileBasedConfig: fileBasedConfig
+        )
+        XCTAssertEqual(resolvedEndpoint, "https://env-global.com:9000")
+    }
+
+    func testResolveServiceSpecificConfig() throws {
+        setenv("AWS_PROFILE", "serviceSpecificConfiguredEndpointProfile", 1)
+        let resolvedEndpoint = try AWSEndpointConfig.configuredEndpoint(
+            sdkID: sdkID,
+            ignoreConfiguredEndpointURLs: nil,
+            fileBasedConfig: fileBasedConfig
+        )
+        XCTAssertEqual(resolvedEndpoint, "https://config-specific.com:1000")
+    }
+
+    func testResolveServiceSpecificConfig2() throws {
+        setenv("AWS_PROFILE", "serviceSpecificConfiguredEndpointProfile", 1)
+        let resolvedEndpoint = try AWSEndpointConfig.configuredEndpoint(
+            sdkID: "Test Service 2",
+            ignoreConfiguredEndpointURLs: nil,
+            fileBasedConfig: fileBasedConfig
+        )
+        XCTAssertEqual(resolvedEndpoint, "https://config-specific.com:2000")
+    }
+
+    func testResolveGlobalConfigWhenMissingServicesSection() throws {
+        setenv("AWS_PROFILE", "serviceSpecificConfiguredEndpointProfile", 1)
+        let resolvedEndpoint = try AWSEndpointConfig.configuredEndpoint(
+            sdkID: "absent-service-name",
+            ignoreConfiguredEndpointURLs: nil,
+            fileBasedConfig: fileBasedConfig
+        )
+        XCTAssertEqual(resolvedEndpoint, "https://config-global.com:2000")
+    }
+
+    func testResolveGlobalConfigwhenMissingMatchingServiceSubsection() throws {
+        setenv("AWS_PROFILE", "globalConfiguredEndpointProfile", 1)
+        let resolvedEndpoint = try AWSEndpointConfig.configuredEndpoint(
+            sdkID: sdkID,
+            ignoreConfiguredEndpointURLs: nil,
+            fileBasedConfig: fileBasedConfig
+        )
+        XCTAssertEqual(resolvedEndpoint, "https://config-global.com:1000")
+    }
+
+    func testResolveNilWhenNothingIsConfigured() throws {
+        setenv("AWS_PROFILE", "noConfiguredEndpointProfile", 1)
+        let resolvedEndpoint = try AWSEndpointConfig.configuredEndpoint(
+            sdkID: sdkID,
+            ignoreConfiguredEndpointURLs: nil,
+            fileBasedConfig: fileBasedConfig
+        )
+        XCTAssertNil(resolvedEndpoint)
+    }
+}

Sources/Core/AWSClientRuntime/Tests/AWSClientRuntimeTests/Resources/configured_endpoint_tests

@@ -0,0 +1,19 @@
+[profile noConfiguredEndpointProfile]
+region = dummyValue
+
+[profile globalConfiguredEndpointProfile]
+endpoint_url = https://config-global.com:1000
+
+[profile serviceSpecificConfiguredEndpointProfile]
+endpoint_url = https://config-global.com:2000
+services = serviceSpecificEndpoints
+
+[services serviceSpecificEndpoints]
+test_service =
+    endpoint_url = https://config-specific.com:1000
+test_service_2 =
+    endpoint_url = https://config-specific.com:2000
+
+[profile ignoreConfiguredEndpointProfile]
+ignore_configured_endpoint_urls = true
+endpoint_url = http://this-should-be-ignored

Sources/Core/AWSSDKCommon/Sources/AWSSDKCommon/FileBasedConfiguration/CRTFileBasedConfiguration.swift

@@ -24,6 +24,8 @@ extension CRTFileBasedConfigurationSectionType {
             self = .profile
         case .ssoSession:
             self = .ssoSession
+        case .services:
+            self = .services
         }
     }
 }

Sources/Core/AWSSDKCommon/Sources/AWSSDKCommon/FileBasedConfiguration/FileBasedConfiguration.swift

@@ -13,6 +13,7 @@
 public enum FileBasedConfigurationSectionType {
     case profile
     case ssoSession
+    case services
 }
 
 @_spi(FileBasedConfig)

codegen/smithy-aws-swift-codegen/src/main/kotlin/software/amazon/smithy/aws/swift/codegen/config/AWSDefaultClientConfiguration.kt

@@ -39,6 +39,7 @@ class AWSDefaultClientConfiguration : ClientConfiguration {
             { it.format("\$N.retryMode()", AWSClientRuntimeTypes.Core.AWSClientConfigDefaultsProvider) },
             true
         ),
-        ConfigProperty("maxAttempts", SwiftTypes.Int.toOptional())
+        ConfigProperty("maxAttempts", SwiftTypes.Int.toOptional()),
+        ConfigProperty("ignoreConfiguredEndpointURLs", SwiftTypes.Bool.toOptional())
     )
 }

codegen/smithy-aws-swift-codegen/src/main/kotlin/software/amazon/smithy/aws/swift/codegen/middleware/OperationEndpointResolverMiddleware.kt

@@ -5,6 +5,7 @@
 
 package software.amazon.smithy.aws.swift.codegen.middleware
 
+import software.amazon.smithy.aws.swift.codegen.swiftmodules.AWSClientRuntimeTypes
 import software.amazon.smithy.codegen.core.Symbol
 import software.amazon.smithy.jmespath.JmespathExpression
 import software.amazon.smithy.model.node.Node
@@ -196,6 +197,14 @@ class OperationEndpointResolverMiddleware(
                     param.default.isPresent -> {
                         "config.${getBuiltInName(param)} ?? ${param.defaultValueLiteral}"
                     }
+                    getBuiltInName(param).equals("endpoint") -> {
+                        writer.write(
+                            "let configuredEndpoint = try config.${getBuiltInName(param)} ?? \$N.configuredEndpoint(\$S, config.ignoreConfiguredEndpointURLs)",
+                            AWSClientRuntimeTypes.Core.AWSClientConfigDefaultsProvider,
+                            ctx.settings.sdkId
+                        )
+                        "configuredEndpoint"
+                    }
                     else -> {
                         "config.${getBuiltInName(param)}"
                     }

codegen/smithy-aws-swift-codegen/src/test/kotlin/software/amazon/smithy/aws/swift/codegen/OperationEndpointResolverMiddlewareTests.kt

@@ -25,6 +25,7 @@ class OperationEndpointResolverMiddlewareTests {
         middleware.render(context.ctx, writer, operation, "operationStack")
         var contents = writer.toString()
         val expected = """
+let configuredEndpoint = try config.endpoint ?? AWSClientRuntime.AWSClientConfigDefaultsProvider.configuredEndpoint("Json Protocol", config.ignoreConfiguredEndpointURLs)
 // OperationContextParam - JMESPath expression: "bar.objects[].content"
 let bar = input.bar
 let objects = bar?.objects
@@ -50,7 +51,7 @@ let projection2: [Swift.String]? = objects2?.compactMap { original in
     let id = original.id
     return id
 }
-let endpointParams = EndpointParams(boolBar: true, boolBaz: input.fuzz, boolFoo: config.boolFoo, endpoint: config.endpoint, flattenedArray: projection, keysFunctionArray: keys, region: region, stringArrayBar: ["five", "six", "seven"], stringBar: "some value", stringBaz: input.buzz, stringFoo: config.stringFoo, subfield: subfield2, wildcardProjectionArray: projection2)
+let endpointParams = EndpointParams(boolBar: true, boolBaz: input.fuzz, boolFoo: config.boolFoo, endpoint: configuredEndpoint, flattenedArray: projection, keysFunctionArray: keys, region: region, stringArrayBar: ["five", "six", "seven"], stringBar: "some value", stringBaz: input.buzz, stringFoo: config.stringFoo, subfield: subfield2, wildcardProjectionArray: projection2)
 builder.applyEndpoint(AWSClientRuntime.EndpointResolverMiddleware<GetThingOutput, EndpointParams>(endpointResolverBlock: { [config] in try config.endpointResolver.resolve(params: ${'$'}0) }, endpointParams: endpointParams))
 """
         contents.shouldContainOnlyOnce(expected)