Update to GA release of Rust SDK

Signed-off-by: James Bornholt <bornholt@amazon.com>

mountpoint-s3-client/Cargo.toml

@@ -39,11 +39,12 @@ rand_chacha = { version = "0.3.1", optional = true }
 
 [dev-dependencies]
 anyhow = { version = "1.0.64", features = ["backtrace"] }
-aws-config = "0.56.0"
-aws-credential-types = "0.56.0"
-aws-sdk-s3 = "0.30.0"
-aws-sdk-sts = "0.30.0"
-aws-smithy-runtime-api = "0.56.1"
+aws-config = "1.1.4"
+aws-credential-types = "1.1.4"
+aws-sdk-s3 = "1.14.0"
+aws-sdk-sts = "1.12.0"
+aws-smithy-runtime-api = "1.1.4"
+aws-smithy-types = "1.1.4"
 bytes = "1.2.1"
 clap = { version = "4.1.9", features = ["derive"] }
 ctor = "0.2.6"

mountpoint-s3-client/tests/auth.rs

@@ -6,6 +6,8 @@ use std::io::Write;
 use std::option::Option::None;
 
 use aws_config::default_provider::credentials::DefaultCredentialsChain;
+#[cfg(not(feature = "s3express_tests"))]
+use aws_config::BehaviorVersion;
 use aws_credential_types::provider::ProvideCredentials;
 use aws_sdk_s3::config::Region;
 use aws_sdk_s3::primitives::ByteStream;
@@ -31,11 +33,9 @@ async fn test_static_provider() {
 
     let key = format!("{prefix}/hello");
     let body = b"hello world!";
-    let mut request = sdk_client.put_object();
-    if cfg!(not(feature = "s3express_tests")) {
-        request = request.bucket(&bucket);
-    }
-    request
+    sdk_client
+        .put_object()
+        .bucket(&bucket)
         .key(&key)
         .body(ByteStream::from(Bytes::from_static(body)))
         .send()
@@ -108,11 +108,9 @@ async fn test_profile_provider_async() {
 
     let key = format!("{prefix}/hello");
     let body = b"hello world!";
-    let mut request = sdk_client.put_object();
-    if cfg!(not(feature = "s3express_tests")) {
-        request = request.bucket(&bucket);
-    }
-    request
+    sdk_client
+        .put_object()
+        .bucket(&bucket)
         .key(&key)
         .body(ByteStream::from(Bytes::from_static(body)))
         .send()
@@ -220,7 +218,7 @@ async fn test_scoped_credentials() {
             .unwrap();
     }
 
-    let config = aws_config::from_env()
+    let config = aws_config::defaults(BehaviorVersion::latest())
         .region(Region::new(get_test_region()))
         .load()
         .await;
@@ -246,9 +244,9 @@ async fn test_scoped_credentials() {
 
     // Build a S3CrtClient that uses a static credentials provider with the creds we just got
     let config = CredentialsProviderStaticOptions {
-        access_key_id: credentials.access_key_id().unwrap(),
-        secret_access_key: credentials.secret_access_key().unwrap(),
-        session_token: credentials.session_token(),
+        access_key_id: credentials.access_key_id(),
+        secret_access_key: credentials.secret_access_key(),
+        session_token: Some(credentials.session_token()),
     };
     let provider = CredentialsProvider::new_static(&Allocator::default(), config).unwrap();
     let config = S3ClientConfig::new()

mountpoint-s3-client/tests/common/mod.rs

@@ -1,5 +1,6 @@
 #![cfg(feature = "s3_tests")]
 
+use aws_config::BehaviorVersion;
 use aws_sdk_s3 as s3;
 use aws_sdk_s3::config::Region;
 use aws_sdk_s3::error::SdkError;
@@ -114,23 +115,54 @@ pub fn get_s3express_endpoint() -> String {
 }
 
 pub async fn get_test_sdk_client() -> s3::Client {
-    let mut config = aws_config::from_env().region(Region::new(get_test_region()));
+    let sdk_config = aws_config::defaults(BehaviorVersion::latest())
+        .region(Region::new(get_test_region()))
+        .load()
+        .await;
+    let mut s3_config = s3::config::Builder::from(&sdk_config);
+
+    // TODO: remove when the Rust SDK supports S3 Express One Zone. For now, we force the SDK to
+    // always use SigV4 and point it at the manually specified Express One Zone endpoint.
     if cfg!(feature = "s3express_tests") {
-        config = config.endpoint_url(get_s3express_endpoint());
+        let endpoint = s3::config::endpoint::Endpoint::builder()
+            .url(get_s3express_endpoint())
+            .property(
+                "authSchemes",
+                vec![aws_smithy_types::Document::from({
+                    let mut out = std::collections::HashMap::<String, aws_smithy_types::Document>::new();
+                    out.insert("disableDoubleEncoding".to_string(), true.into());
+                    out.insert("name".to_string(), "sigv4".to_string().into());
+                    out.insert("signingName".to_string(), "s3".to_string().into());
+                    out.insert("signingRegion".to_string(), get_test_region().into());
+                    out
+                })],
+            )
+            .build();
+
+        #[derive(Debug)]
+        struct StaticEndpointResolver(aws_sdk_s3::config::endpoint::Endpoint);
+
+        impl s3::config::endpoint::ResolveEndpoint for StaticEndpointResolver {
+            fn resolve_endpoint(
+                &self,
+                _params: &s3::config::endpoint::Params,
+            ) -> s3::config::endpoint::EndpointFuture<'_> {
+                s3::config::endpoint::EndpointFuture::ready(Ok(self.0.clone()))
+            }
+        }
+
+        s3_config = s3_config.endpoint_resolver(StaticEndpointResolver(endpoint));
     }
-    let config = config.load().await;
-    s3::Client::new(&config)
+
+    s3::Client::from_conf(s3_config.build())
 }
 
 /// Create some objects in a prefix for testing.
 pub async fn create_objects_for_test(client: &s3::Client, bucket: &str, prefix: &str, names: &[impl AsRef<str>]) {
     for name in names {
-        let mut request = client.put_object();
-        if cfg!(not(feature = "s3express_tests")) {
-            request = request.bucket(bucket);
-        }
-
-        request
+        client
+            .put_object()
+            .bucket(bucket)
             .key(format!("{}{}", prefix, name.as_ref()))
             .body(ByteStream::from(Bytes::from_static(b".")))
             .send()
@@ -145,20 +177,18 @@ pub async fn get_mpu_count_for_key(
     prefix: &str,
     key: &str,
 ) -> Result<usize, SdkError<ListMultipartUploadsError, HttpResponse>> {
-    let mut request = client.list_multipart_uploads();
-    if cfg!(not(feature = "s3express_tests")) {
-        request = request.bucket(bucket);
-    }
-
     // This could be broken if we have initiated more than one multipart upload using the same key
     // since ListMultipartUploads returns all multipart uploads for that key.
-    let upload_count = request
+    let upload_count = client
+        .list_multipart_uploads()
+        .bucket(bucket)
         .prefix(prefix)
         .send()
         .await?
         .uploads()
-        .map(|upload| upload.iter().filter(|&u| u.key() == Some(key)).collect::<Vec<_>>())
-        .map_or(0, |u| u.len());
+        .iter()
+        .filter(|&u| u.key() == Some(key))
+        .count();
 
     Ok(upload_count)
 }

mountpoint-s3-client/tests/delete_object.rs

@@ -15,11 +15,9 @@ async fn test_delete_object() {
 
     let key = format!("{prefix}/hello");
     let body = b"hello world!";
-    let mut request = sdk_client.put_object();
-    if cfg!(not(feature = "s3express_tests")) {
-        request = request.bucket(&bucket);
-    }
-    request
+    sdk_client
+        .put_object()
+        .bucket(&bucket)
         .key(&key)
         .body(ByteStream::from(Bytes::from_static(body)))
         .send()
@@ -32,11 +30,13 @@ async fn test_delete_object() {
         .await
         .expect("delete_object should succeed");
 
-    let mut request = sdk_client.head_object();
-    if cfg!(not(feature = "s3express_tests")) {
-        request = request.bucket(&bucket);
-    }
-    let head_obj_err = request.key(&key).send().await.expect_err("object should not exist");
+    let head_obj_err = sdk_client
+        .head_object()
+        .bucket(&bucket)
+        .key(&key)
+        .send()
+        .await
+        .expect_err("object should not exist");
 
     assert!(head_obj_err.into_service_error().is_not_found());
 }
@@ -48,11 +48,9 @@ async fn test_delete_object_no_obj() {
 
     let key = format!("{prefix}/nonexistent_key");
 
-    let mut request = sdk_client.head_object();
-    if cfg!(not(feature = "s3express_tests")) {
-        request = request.bucket(&bucket);
-    }
-    let head_obj_err = request
+    let head_obj_err = sdk_client
+        .head_object()
+        .bucket(&bucket)
         .bucket(&bucket)
         .key(&key)
         .send()

mountpoint-s3-client/tests/endpoint_config.rs

@@ -15,11 +15,9 @@ async fn run_test<F: FnOnce(&str) -> EndpointConfig>(f: F, prefix: &str, bucket:
     // Create one object named "hello"
     let key = format!("{prefix}hello");
     let body = b"hello world!";
-    let mut request = sdk_client.put_object();
-    if cfg!(not(feature = "s3express_tests")) {
-        request = request.bucket(&bucket);
-    }
-    request
+    sdk_client
+        .put_object()
+        .bucket(&bucket)
         .key(&key)
         .body(ByteStream::from(Bytes::from_static(body)))
         .send()

mountpoint-s3-client/tests/get_object.rs

@@ -30,11 +30,9 @@ async fn test_get_object(size: usize, range: Option<Range<u64>>) {
 
     let key = format!("{prefix}/test");
     let body = vec![0x42; size];
-    let mut request = sdk_client.put_object();
-    if cfg!(not(feature = "s3express_tests")) {
-        request = request.bucket(&bucket);
-    }
-    request
+    sdk_client
+        .put_object()
+        .bucket(&bucket)
         .key(&key)
         .body(ByteStream::from(body.clone()))
         .send()
@@ -102,11 +100,10 @@ async fn test_get_object_success_if_match() {
     // Create one object named "hello"
     let key = format!("{prefix}/hello");
     let body = b"hello world!";
-    let mut request = sdk_client.put_object();
-    if cfg!(not(feature = "s3express_tests")) {
-        request = request.bucket(&bucket);
-    }
-    let response = request
+
+    let response = sdk_client
+        .put_object()
+        .bucket(&bucket)
         .key(&key)
         .body(ByteStream::from(Bytes::from_static(body)))
         .send()
@@ -131,11 +128,9 @@ async fn test_get_object_412_if_match() {
     // Create one object named "hello"
     let key = format!("{prefix}/hello");
     let body = b"hello world!";
-    let mut request = sdk_client.put_object();
-    if cfg!(not(feature = "s3express_tests")) {
-        request = request.bucket(&bucket);
-    }
-    request
+    sdk_client
+        .put_object()
+        .bucket(&bucket)
         .key(&key)
         .body(ByteStream::from(Bytes::from_static(body)))
         .send()

mountpoint-s3-client/tests/get_object_attributes.rs

@@ -28,11 +28,7 @@ async fn create_mpu_object(
 ) -> (Vec<CompletedPart>, CompleteMultipartUploadOutput) {
     let sdk_client = get_test_sdk_client().await;
 
-    let mut request = sdk_client.create_multipart_upload();
-    if cfg!(not(feature = "s3express_tests")) {
-        request = request.bucket(bucket);
-    }
-    let mut create_mpu = request.key(key);
+    let mut create_mpu = sdk_client.create_multipart_upload().bucket(bucket).key(key);
     if let Some(algorithm) = &checksum_algorithm {
         create_mpu = create_mpu.checksum_algorithm(algorithm.to_owned());
     }
@@ -46,11 +42,9 @@ async fn create_mpu_object(
         let part_body = vec![0; part_size.to_owned()];
         let part_num = (part_index + 1) as i32;
 
-        let mut request = sdk_client.upload_part();
-        if cfg!(not(feature = "s3express_tests")) {
-            request = request.bucket(bucket);
-        }
-        let mut upload_part = request
+        let mut upload_part = sdk_client
+            .upload_part()
+            .bucket(bucket)
             .key(key)
             .part_number(part_num)
             .upload_id(upload_id)
@@ -92,11 +86,9 @@ async fn create_mpu_object(
         .set_parts(Some(completed_parts.clone()))
         .build();
 
-    let mut request = sdk_client.complete_multipart_upload();
-    if cfg!(not(feature = "s3express_tests")) {
-        request = request.bucket(bucket);
-    }
-    let complete_mpu_output = request
+    let complete_mpu_output = sdk_client
+        .complete_multipart_upload()
+        .bucket(bucket)
         .key(key)
         .upload_id(upload_id)
         .multipart_upload(completed_mpu)
@@ -115,11 +107,9 @@ async fn test_with_checksum(checksum_algorithm: ChecksumAlgorithm) {
     let key = format!("{prefix}/hello");
     let body = b"hello world!";
 
-    let mut request = sdk_client.put_object();
-    if cfg!(not(feature = "s3express_tests")) {
-        request = request.bucket(&bucket);
-    }
-    let put_object_output = request
+    let put_object_output = sdk_client
+        .put_object()
+        .bucket(&bucket)
         .key(&key)
         .checksum_algorithm(checksum_algorithm.clone())
         .body(ByteStream::from(Bytes::from_static(body)))
@@ -181,11 +171,9 @@ async fn test_get_attributes() {
     // Create one object named "hello"
     let key = format!("{prefix}/hello");
     let body = b"hello world!";
-    let mut request = sdk_client.put_object();
-    if cfg!(not(feature = "s3express_tests")) {
-        request = request.bucket(&bucket);
-    }
-    let put_object_output = request
+    let put_object_output = sdk_client
+        .put_object()
+        .bucket(&bucket)
         .key(&key)
         .body(ByteStream::from(Bytes::from_static(body)))
         .send()
@@ -234,11 +222,9 @@ async fn test_get_attributes_all_none() {
     // Create one object named "hello"
     let key = format!("{prefix}/hello");
     let body = b"hello world!";
-    let mut request = sdk_client.put_object();
-    if cfg!(not(feature = "s3express_tests")) {
-        request = request.bucket(&bucket);
-    }
-    request
+    sdk_client
+        .put_object()
+        .bucket(&bucket)
         .key(&key)
         .checksum_algorithm(ChecksumAlgorithm::Crc32C)
         .body(ByteStream::from(Bytes::from_static(body)))