feature: support flexible checksums (#1561)

feature: support flexible checksums
update: s3 model
update: CHANGELOG.next.toml

CHANGELOG.next.toml

@@ -143,6 +143,63 @@ references = ["smithy-rs#1157"]
 meta = { "breaking" = true, "tada" = false, "bug" = false }
 author = "82marbag"
 
+[[aws-sdk-rust]]
+message = """
+The AWS SDK for Rust now supports [additional checksum algorithms for Amazon S3](https://aws.amazon.com/blogs/aws/new-additional-checksum-algorithms-for-amazon-s3/).
+When getting and putting objects, you may now request that the request body be validated with a checksum. The supported
+algorithms are SHA-1, SHA-256, CRC-32, and CRC-32C.
+
+```rust
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let sdk_config = aws_config::load_from_env().await;
+    let s3_client = aws_sdk_s3::Client::new(&sdk_config);
+    let body = aws_sdk_s3::types::ByteStream::read_from()
+        .path(std::path::Path::new("./path/to/your/file.txt"))
+        .build()
+        .await
+        .unwrap();
+
+    let _ = s3_client
+        .put_object()
+        .bucket("your-bucket")
+        .key("file.txt")
+        .body(body)
+        // When using this field, the checksum will be calculated for you
+        .checksum_algorithm(aws_sdk_s3::model::ChecksumAlgorithm::Crc32C)
+        .send()
+        .await?;
+
+    let body = aws_sdk_s3::types::ByteStream::read_from()
+        .path(std::path::Path::new("./path/to/your/other-file.txt"))
+        .build()
+        .await
+        .unwrap();
+
+    let _ = s3_client
+        .put_object()
+        .bucket("your-bucket")
+        .key("other-file.txt")
+        .body(body)
+        // Alternatively, you can pass a checksum that you've calculated yourself. It must be base64
+        // encoded. Also, make sure that you're base64 encoding the bytes of the checksum, not its
+        // string representation.
+        .checksum_crc32_c(aws_smithy_types::base64::encode(&A_PRECALCULATED_CRC_32_C_CHECKSUM[..]))
+        .send()
+        .await?;
+}
+```
+"""
+references = ["smithy-rs#1482"]
+meta = { "breaking" = false, "tada" = true, "bug" = false }
+author = "Velfi"
+
+[[smithy-rs]]
+message = "Update codegen to generate support for flexible checksums."
+references = ["smithy-rs#1482"]
+meta = { "breaking" = false, "tada" = true, "bug" = false }
+author = "Velfi"
+
 [[aws-sdk-rust]]
 message = "SDK crate READMEs now include an example of creating a client"
 references = ["smithy-rs#1571", "smithy-rs#1385"]

aws/rust-runtime/aws-http/src/content_encoding.rs

@@ -44,7 +44,9 @@ impl AwsChunkedBodyOptions {
     }
 
     fn total_trailer_length(&self) -> u64 {
-        self.trailer_lengths.iter().sum()
+        self.trailer_lengths.iter().sum::<u64>()
+            // We need to account for a CRLF after each trailer name/value pair
+            + (self.trailer_lengths.len() * CRLF.len()) as u64
     }
 
     /// Set a trailer len
@@ -519,10 +521,12 @@ mod tests {
     }
 
     #[tokio::test]
-    #[should_panic = "called `Result::unwrap()` on an `Err` value: ReportedTrailerLengthMismatch { actual: 42, expected: 0 }"]
+    #[should_panic = "called `Result::unwrap()` on an `Err` value: ReportedTrailerLengthMismatch { actual: 44, expected: 0 }"]
     async fn test_aws_chunked_encoding_incorrect_trailer_length_panic() {
         let input_str = "Hello world";
         // Test body has no trailers, so this length is incorrect and will trigger an assert panic
+        // When the panic occurs, it will actually expect a length of 44. This is because, when using
+        // aws-chunked encoding, each trailer will end with a CRLF which is 2 bytes long.
         let wrong_trailer_len = 42;
         let opts = AwsChunkedBodyOptions::new(input_str.len() as u64, vec![wrong_trailer_len]);
         let mut body = AwsChunkedBody::new(SdkBody::from(input_str), opts);

aws/rust-runtime/aws-inlineable/Cargo.toml

@@ -12,29 +12,32 @@ publish = false
 repository = "https://github.com/awslabs/smithy-rs"
 
 [dependencies]
-aws-smithy-http = { path = "../../../rust-runtime/aws-smithy-http" }
-aws-smithy-client = { path = "../../../rust-runtime/aws-smithy-client" }
-aws-http = { path = "../aws-http" }
 aws-endpoint = { path = "../aws-endpoint" }
-aws-smithy-types = { path = "../../../rust-runtime/aws-smithy-types" }
+aws-http = { path = "../aws-http" }
+aws-sig-auth = { path = "../../rust-runtime/aws-sig-auth" }
+aws-smithy-checksums = { path = "../../../rust-runtime/aws-smithy-checksums" }
+aws-smithy-client = { path = "../../../rust-runtime/aws-smithy-client" }
+aws-smithy-http = { path = "../../../rust-runtime/aws-smithy-http" }
 aws-smithy-http-tower= { path = "../../../rust-runtime/aws-smithy-http-tower" }
+aws-smithy-types = { path = "../../../rust-runtime/aws-smithy-types" }
 aws-types = { path = "../../rust-runtime/aws-types" }
-aws-sig-auth = { path = "../../rust-runtime/aws-sig-auth" }
+bytes = "1"
+bytes-utils = "0.1.1"
+hex = "0.4.3"
 http = "0.2.4"
-tower = { version = "0.4", default-features = false }
-
-# Checksum dependencies:
+http-body = "0.4.5"
+md-5 = "0.10.1"
 ring = "0.16"
-bytes-utils = "0.1.1"
-tokio-stream = "0.1.7"
-bytes = "1"
 tokio = { version = "1", features = ["full"] }
-hex = "0.4.3"
+tokio-stream = "0.1.7"
+tower = { version = "0.4", default-features = false }
+tracing = "0.1"
 
 [dev-dependencies]
-temp-file = "0.1.6"
-aws-smithy-http = { path = "../../../rust-runtime/aws-smithy-http", features = ["rt-tokio"] }
 aws-smithy-client = { path = "../../../rust-runtime/aws-smithy-client", features = ["test-util"] }
+aws-smithy-http = { path = "../../../rust-runtime/aws-smithy-http", features = ["rt-tokio"] }
+tempfile = "3.2.0"
+tracing-subscriber = { version = "0.3.5", features = ["env-filter"] }
 
 [package.metadata.docs.rs]
 all-features = true

aws/rust-runtime/aws-inlineable/src/glacier_checksums.rs

@@ -200,7 +200,7 @@ mod test {
         while test_data.len() < total_size {
             test_data.extend_from_slice(base_seq)
         }
-        let target = temp_file::empty();
+        let target = tempfile::NamedTempFile::new().unwrap();
         tokio::fs::write(target.path(), test_data).await.unwrap();
         let body = ByteStream::from_path(target.path())
             .await