Feature: configurable connectors for AWS clients

CHANGELOG.next.toml

@@ -86,6 +86,99 @@ references = ["smithy-rs#1923"]
 meta = { "breaking" = false, "tada" = false, "bug" = false }
 author = "ysaito1001"
 
+[[aws-sdk-rust]]
+message = """
+<details>
+<summary>The HTTP connector used when making requests is now configurable through `SdkConfig`.</summary>
+
+```rust
+use std::time::Duration;
+use aws_smithy_client::{Client, hyper_ext};
+use aws_smithy_client::erase::DynConnector;
+use aws_smithy_client::http_connector::ConnectorSettings;
+use aws_types::SdkConfig;
+
+let https_connector = hyper_rustls::HttpsConnectorBuilder::new()
+    .with_webpki_roots()
+    .https_only()
+    .enable_http1()
+    .enable_http2()
+    .build();
+
+let smithy_connector = hyper_ext::Adapter::builder()
+    // Optionally set things like timeouts as well
+    .connector_settings(
+        ConnectorSettings::builder()
+            .connect_timeout(Duration::from_secs(5))
+            .build()
+    )
+    .build(https_connector);
+
+let sdk_config = aws_config::from_env()
+    .http_connector(smithy_connector)
+    .load()
+    .await;
+
+let client = Client::new(&sdk_config);
+
+// When sent, this operation will go through the custom smithy connector instead of
+// the default HTTP connector.
+let op = client
+    .get_object()
+    .bucket("some-test-bucket")
+    .key("test.txt")
+    .send()
+    .await
+    .unwrap();
+```
+
+</details>
+"""
+references = ["smithy-rs#1225", "smithy-rs#1918"]
+meta = { "breaking" = false, "tada" = true, "bug" = false }
+author = "Velfi"
+
+[[aws-sdk-rust]]
+message = """
+`<service>::Client::from_conf_conn` has been removed since it's now possible to configure the connection from the
+shared and service configs. To update your code, pass connections to the `http_connector` method during config creation.
+
+<details>
+<summary>Example</summary>
+
+before:
+
+```rust
+    let conf = aws_sdk_sts::Config::builder()
+        // The builder has no defaults but setting other fields is omitted for brevity...
+        .build();
+    let (server, request) = capture_request(None);
+    let client = aws_sdk_sts::Client::from_conf_conn(conf, server);
+```
+
+after:
+
+```rust
+    let (server, request) = capture_request(None);
+    let conf = aws_sdk_sts::Config::builder()
+        // The builder has no defaults but setting other fields is omitted for brevity...
+        .http_connector(server)
+        .build();
+    let client = aws_sdk_sts::Client::from_conf(conf);
+```
+
+</details>
+"""
+references = ["smithy-rs#1225", "smithy-rs#1918"]
+meta = { "breaking" = true, "tada" = false, "bug" = false }
+author = "Velfi"
+
+[[aws-sdk-rust]]
+message = "Add `to_vec` method to `aws_smithy_http::byte_stream::AggregatedBytes`."
+references = ["smithy-rs#1918"]
+meta = { "breaking" = false, "tada" = false, "bug" = false }
+author = "Velfi"
+
 [[aws-sdk-rust]]
 message = "Ability to add an inline policy or a list of policy ARNs to the `AssumeRoleProvider` builder."
 references = ["aws-sdk-rust#641", "smithy-rs#1892"]

aws/SDK_CHANGELOG.next.json

@@ -341,4 +341,4 @@
     }
   ],
   "aws-sdk-model": []
-}
+}

aws/rust-runtime/aws-config/Cargo.toml

@@ -17,21 +17,20 @@ rt-tokio = ["aws-smithy-async/rt-tokio", "tokio/rt"]
 default = ["client-hyper", "rustls", "rt-tokio"]
 
 [dependencies]
-aws-sdk-sts = { path = "../../sdk/build/aws-sdk/sdk/sts", default-features = false }
+aws-http = { path = "../../sdk/build/aws-sdk/sdk/aws-http" }
 aws-sdk-sso = { path = "../../sdk/build/aws-sdk/sdk/sso", default-features = false }
+aws-sdk-sts = { path = "../../sdk/build/aws-sdk/sdk/sts", default-features = false }
 aws-smithy-async = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-async" }
 aws-smithy-client = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-client", default-features = false }
+aws-smithy-http = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-http" }
+aws-smithy-http-tower = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-http-tower" }
+aws-smithy-json = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-json" }
 aws-smithy-types = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-types" }
 aws-types = { path = "../../sdk/build/aws-sdk/sdk/aws-types" }
+hyper = { version = "0.14.12", default-features = false }
 time = { version = "0.3.4", features = ["parsing"] }
 tokio = { version = "1.8.4", features = ["sync"] }
 tracing = { version = "0.1" }
-hyper = { version = "0.14.12", default-features = false }
-
-aws-http = { path = "../../sdk/build/aws-sdk/sdk/aws-http" }
-aws-smithy-http = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-http" }
-aws-smithy-http-tower = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-http-tower" }
-aws-smithy-json = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-json" }
 
 # implementation detail of SSO credential caching
 ring = "0.16"

aws/rust-runtime/aws-config/external-types.toml

@@ -12,6 +12,7 @@ allowed_external_types = [
    "aws_smithy_client::http_connector::ConnectorSettings",
    "aws_smithy_http::body::SdkBody",
    "aws_smithy_http::result::SdkError",
+   "aws_smithy_http::endpoint",
    "aws_smithy_types::retry",
    "aws_smithy_types::retry::*",
    "aws_smithy_types::timeout",

aws/rust-runtime/aws-config/src/lib.rs

@@ -91,6 +91,18 @@
 //! # }
 //! ```
 
+pub use aws_smithy_http::endpoint;
+// Re-export types from smithy-types
+pub use aws_smithy_types::retry;
+pub use aws_smithy_types::timeout;
+// Re-export types from aws-types
+pub use aws_types::{
+    app_name::{AppName, InvalidAppName},
+    SdkConfig,
+};
+/// Load default sources for all configuration with override support
+pub use loader::ConfigLoader;
+
 #[allow(dead_code)]
 const PKG_VERSION: &str = env!("CARGO_PKG_VERSION");
 
@@ -132,16 +144,6 @@ pub mod connector;
 
 pub mod credential_process;
 
-// Re-export types from smithy-types
-pub use aws_smithy_types::retry;
-pub use aws_smithy_types::timeout;
-
-// Re-export types from aws-types
-pub use aws_types::{
-    app_name::{AppName, InvalidAppName},
-    SdkConfig,
-};
-
 /// Create an environment loader for AWS Configuration
 ///
 /// # Examples
@@ -162,13 +164,9 @@ pub async fn load_from_env() -> aws_types::SdkConfig {
     from_env().load().await
 }
 
-/// Load default sources for all configuration with override support
-pub use loader::ConfigLoader;
-
 mod loader {
     use std::sync::Arc;
 
-    use crate::connector::default_connector;
     use aws_smithy_async::rt::sleep::{default_async_sleep, AsyncSleep};
     use aws_smithy_client::http_connector::{ConnectorSettings, HttpConnector};
     use aws_smithy_types::retry::RetryConfig;
@@ -178,6 +176,7 @@ mod loader {
     use aws_types::endpoint::ResolveAwsEndpoint;
     use aws_types::SdkConfig;
 
+    use crate::connector::default_connector;
     use crate::default_provider::{app_name, credentials, region, retry_config, timeout_config};
     use crate::meta::region::ProvideRegion;
     use crate::provider_config::ProviderConfig;
@@ -222,11 +221,13 @@ mod loader {
         ///
         /// # Examples
         /// ```no_run
-        /// # use aws_smithy_types::retry::RetryConfig;
         /// # async fn create_config() {
-        ///     let config = aws_config::from_env()
-        ///         .retry_config(RetryConfig::standard().with_max_attempts(2))
-        ///         .load().await;
+        /// use aws_config::retry::RetryConfig;
+        ///
+        /// let config = aws_config::from_env()
+        ///     .retry_config(RetryConfig::standard().with_max_attempts(2))
+        ///     .load()
+        ///     .await;
         /// # }
         /// ```
         pub fn retry_config(mut self, retry_config: RetryConfig) -> Self {
@@ -242,16 +243,16 @@ mod loader {
         /// ```no_run
         /// # use std::time::Duration;
         /// # async fn create_config() {
-        ///  use aws_smithy_types::timeout::TimeoutConfig;
+        /// use aws_config::timeout::TimeoutConfig;
         ///
-        ///  let config = aws_config::from_env()
-        ///     .timeout_config(
-        ///         TimeoutConfig::builder()
-        ///             .operation_timeout(Duration::from_secs(5))
-        ///             .build()
-        ///     )
-        ///     .load()
-        ///     .await;
+        /// let config = aws_config::from_env()
+        ///    .timeout_config(
+        ///        TimeoutConfig::builder()
+        ///            .operation_timeout(Duration::from_secs(5))
+        ///            .build()
+        ///    )
+        ///    .load()
+        ///    .await;
         /// # }
         /// ```
         pub fn timeout_config(mut self, timeout_config: TimeoutConfig) -> Self {
@@ -267,9 +268,41 @@ mod loader {
             self
         }
 
-        /// Override the [`HttpConnector`] used to build [`SdkConfig`](aws_types::SdkConfig).
-        pub fn http_connector(mut self, http_connector: HttpConnector) -> Self {
-            self.http_connector = Some(http_connector);
+        /// Override the [`HttpConnector`] for this [`ConfigLoader`]. The connector will be used when
+        /// sending operations. This **does not set** the HTTP connector used by config providers.
+        /// To change that connector, use [ConfigLoader::configure].
+        ///
+        /// ## Examples
+        /// ```no_run
+        /// # #[cfg(feature = "client-hyper")]
+        /// # async fn create_config() {
+        /// use std::time::Duration;
+        /// use aws_smithy_client::{Client, hyper_ext};
+        /// use aws_smithy_client::erase::DynConnector;
+        /// use aws_smithy_client::http_connector::ConnectorSettings;
+        ///
+        /// let https_connector = hyper_rustls::HttpsConnectorBuilder::new()
+        ///     .with_webpki_roots()
+        ///     .https_only()
+        ///     .enable_http1()
+        ///     .enable_http2()
+        ///     .build();
+        /// let smithy_connector = hyper_ext::Adapter::builder()
+        ///     // Optionally set things like timeouts as well
+        ///     .connector_settings(
+        ///         ConnectorSettings::builder()
+        ///             .connect_timeout(Duration::from_secs(5))
+        ///             .build()
+        ///     )
+        ///     .build(https_connector);
+        /// let sdk_config = aws_config::from_env()
+        ///     .http_connector(smithy_connector)
+        ///     .load()
+        ///     .await;
+        /// # }
+        /// ```
+        pub fn http_connector(mut self, http_connector: impl Into<HttpConnector>) -> Self {
+            self.http_connector = Some(http_connector.into());
             self
         }
 
@@ -308,11 +341,13 @@ mod loader {
         ///
         /// Use a static endpoint for all services
         /// ```no_run
-        /// # async fn doc() {
-        /// use aws_smithy_http::endpoint::Endpoint;
+        /// # async fn create_config() {
+        /// use aws_config::endpoint::Endpoint;
+        ///
         /// let sdk_config = aws_config::from_env()
-        ///   .endpoint_resolver(Endpoint::immutable("http://localhost:1234".parse().expect("valid URI")))
-        ///   .load().await;
+        ///     .endpoint_resolver(Endpoint::immutable("http://localhost:1234".parse().expect("valid URI")))
+        ///     .load()
+        ///     .await;
         /// # }
         pub fn endpoint_resolver(
             mut self,
@@ -325,12 +360,14 @@ mod loader {
         /// Set configuration for all sub-loaders (credentials, region etc.)
         ///
         /// Update the `ProviderConfig` used for all nested loaders. This can be used to override
-        /// the HTTPs connector used or to stub in an in memory `Env` or `Fs` for testing.
+        /// the HTTPs connector used by providers or to stub in an in memory `Env` or `Fs` for testing.
+        /// This **does not set** the HTTP connector used when sending operations. To change that
+        /// connector, use [ConfigLoader::http_connector].
         ///
         /// # Examples
         /// ```no_run
         /// # #[cfg(feature = "hyper-client")]
-        /// # async fn docs() {
+        /// # async fn create_config() {
         /// use aws_config::provider_config::ProviderConfig;
         /// let custom_https_connector = hyper_rustls::HttpsConnectorBuilder::new().
         ///     with_webpki_roots()
@@ -444,14 +481,15 @@ mod loader {
 
     #[cfg(test)]
     mod test {
-        use crate::from_env;
-        use crate::provider_config::ProviderConfig;
         use aws_smithy_async::rt::sleep::TokioSleep;
         use aws_smithy_client::erase::DynConnector;
         use aws_smithy_client::never::NeverConnector;
         use aws_types::credentials::ProvideCredentials;
         use aws_types::os_shim_internal::Env;
 
+        use crate::from_env;
+        use crate::provider_config::ProviderConfig;
+
         #[tokio::test]
         async fn provider_config_used() {
             let env = Env::from_slice(&[

aws/rust-runtime/aws-types/Cargo.toml

@@ -9,6 +9,8 @@ repository = "https://github.com/awslabs/smithy-rs"
 
 [features]
 hardcoded-credentials = []
+# This feature is to be used only for doc comments
+examples = ["aws-smithy-client/client-hyper", "aws-smithy-client/rustls", "hyper-rustls"]
 
 [dependencies]
 aws-smithy-async = { path = "../../../rust-runtime/aws-smithy-async" }
@@ -18,6 +20,10 @@ aws-smithy-http = { path = "../../../rust-runtime/aws-smithy-http" }
 tracing = "0.1"
 zeroize = "1"
 http = "0.2.6"
+# cargo does not support optional test dependencies, so to completely disable rustls when
+# the native-tls feature is enabled, we need to add the webpki-roots feature here.
+# https://github.com/rust-lang/cargo/issues/1596
+hyper-rustls = { version = "0.23.0", optional = true, features = ["rustls-native-certs", "http2", "webpki-roots"] }
 
 [dev-dependencies]
 futures-util = "0.3.16"