diff --git a/Cargo.lock b/Cargo.lock index bb89c3376..4a670f05f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -435,9 +435,9 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.4.4" +version = "1.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5ac934720fbb46206292d2c75b57e67acfc56fe7dfd34fb9a02334af08409ea" +checksum = "bee7643696e7fdd74c10f9eb42848a87fe469d35eae9c3323f80aa98f350baac" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -461,9 +461,9 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "1.65.0" +version = "1.69.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3ba2c5c0f2618937ce3d4a5ad574b86775576fa24006bcb3128c6e2cbf3c34e" +checksum = "0a88f1c30e4ffa2464f910297c24736ff68cca9e8d2b7d52596b54efd99b9c1e" dependencies = [ "aws-credential-types", "aws-runtime", @@ -472,7 +472,7 @@ dependencies = [ "aws-smithy-checksums", "aws-smithy-eventstream", "aws-smithy-http", - "aws-smithy-json 0.61.1", + "aws-smithy-json 0.61.2", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -503,7 +503,7 @@ dependencies = [ "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json 0.61.1", + "aws-smithy-json 0.61.2", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -525,7 +525,7 @@ dependencies = [ "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json 0.61.1", + "aws-smithy-json 0.61.2", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -547,7 +547,7 @@ dependencies = [ "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json 0.61.1", + "aws-smithy-json 0.61.2", "aws-smithy-query", "aws-smithy-runtime", "aws-smithy-runtime-api", @@ -562,9 +562,9 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.2.6" +version = "1.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d3820e0c08d0737872ff3c7c1f21ebbb6693d832312d6152bf18ef50a5471c2" +checksum = "690118821e46967b3c4501d67d7d52dd75106a9c54cf36cefa1985cedbe94e05" dependencies = [ "aws-credential-types", "aws-smithy-eventstream", @@ -591,9 +591,9 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "1.2.1" +version = "1.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62220bc6e97f946ddd51b5f1361f78996e704677afc518a4ff66b7a72ea1378c" +checksum = "fa59d1327d8b5053c54bf2eaae63bf629ba9e904434d0835a28ed3c0ed0a614e" dependencies = [ "futures-util", "pin-project-lite", @@ -602,15 +602,16 @@ dependencies = [ [[package]] name = "aws-smithy-checksums" -version = "0.60.13" +version = "0.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba1a71073fca26775c8b5189175ea8863afb1c9ea2cceb02a5de5ad9dfbaa795" +checksum = "f2f45a1c384d7a393026bc5f5c177105aa9fa68e4749653b985707ac27d77295" dependencies = [ "aws-smithy-http", "aws-smithy-types", "bytes", "crc32c", "crc32fast", + "crc64fast-nvme", "hex", "http 0.2.12", "http-body 0.4.6", @@ -623,9 +624,9 @@ dependencies = [ [[package]] name = "aws-smithy-eventstream" -version = "0.60.5" +version = "0.60.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cef7d0a272725f87e51ba2bf89f8c21e4df61b9e49ae1ac367a6d69916ef7c90" +checksum = "8b18559a41e0c909b77625adf2b8c50de480a8041e5e4a3f5f7d177db70abc5a" dependencies = [ "aws-smithy-types", "bytes", @@ -634,9 +635,9 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.60.11" +version = "0.60.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c8bc3e8fdc6b8d07d976e301c02fe553f72a39b7a9fea820e023268467d7ab6" +checksum = "7809c27ad8da6a6a68c454e651d4962479e81472aa19ae99e59f9aba1f9713cc" dependencies = [ "aws-smithy-eventstream", "aws-smithy-runtime-api", @@ -664,9 +665,9 @@ dependencies = [ [[package]] name = "aws-smithy-json" -version = "0.61.1" +version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee4e69cc50921eb913c6b662f8d909131bb3e6ad6cb6090d3a39b66fc5c52095" +checksum = "623a51127f24c30776c8b374295f2df78d92517386f77ba30773f15a30ce1422" dependencies = [ "aws-smithy-types", ] @@ -683,9 +684,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.7.4" +version = "1.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f20685047ca9d6f17b994a07f629c813f08b5bce65523e47124879e60103d45" +checksum = "865f7050bbc7107a6c98a397a9fcd9413690c27fa718446967cf03b2d3ac517e" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -727,9 +728,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.2.9" +version = "1.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fbd94a32b3a7d55d3806fe27d98d3ad393050439dd05eb53ece36ec5e3d3510" +checksum = "a28f6feb647fb5e0d5b50f0472c19a7db9462b74e2fec01bb0b44eedcc834e97" dependencies = [ "base64-simd", "bytes", @@ -762,9 +763,9 @@ dependencies = [ [[package]] name = "aws-types" -version = "1.3.3" +version = "1.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5221b91b3e441e6675310829fd8984801b772cb1546ef6c0e54dec9f1ac13fef" +checksum = "b0df5a18c4f951c645300d365fec53a61418bcf4650f604f85fe2a665bfaa0c2" dependencies = [ "aws-credential-types", "aws-smithy-async", @@ -967,6 +968,25 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" +[[package]] +name = "cbindgen" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fce8dd7fcfcbf3a0a87d8f515194b49d6135acab73e18bd380d1d93bb1a15eb" +dependencies = [ + "clap", + "heck 0.4.1", + "indexmap", + "log", + "proc-macro2", + "quote", + "serde", + "serde_json", + "syn 2.0.90", + "tempfile", + "toml", +] + [[package]] name = "cc" version = "1.2.3" @@ -1065,7 +1085,7 @@ version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "quote", "syn 2.0.90", @@ -1152,6 +1172,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + [[package]] name = "crc32c" version = "0.6.8" @@ -1170,6 +1205,16 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crc64fast-nvme" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5e2ee08013e3f228d6d2394116c4549a6df77708442c62d887d83f68ef2ee37" +dependencies = [ + "cbindgen", + "crc", +] + [[package]] name = "criterion" version = "0.5.1" @@ -1844,6 +1889,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + [[package]] name = "heck" version = "0.5.0" @@ -3543,6 +3594,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" +dependencies = [ + "serde", +] + [[package]] name = "serial_test" version = "3.2.0" @@ -4053,6 +4113,40 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.8.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + [[package]] name = "tower-service" version = "0.3.3" @@ -4600,6 +4694,15 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "0.6.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d71a593cc5c42ad7876e2c1fda56f314f3754c084128833e64f1345ff8a03a" +dependencies = [ + "memchr", +] + [[package]] name = "winsafe" version = "0.0.19" diff --git a/mountpoint-s3-client/Cargo.toml b/mountpoint-s3-client/Cargo.toml index 5ce03b6bf..3e53efb53 100644 --- a/mountpoint-s3-client/Cargo.toml +++ b/mountpoint-s3-client/Cargo.toml @@ -38,7 +38,7 @@ rand_chacha = { version = "0.3.1", optional = true } [dev-dependencies] aws-config = "1.5.10" aws-credential-types = "1.2.1" -aws-sdk-s3 = "1.65.0" +aws-sdk-s3 = "1.69.0" aws-smithy-runtime-api = "1.7.3" bytes = "1.9.0" clap = { version = "4.5.23", features = ["derive"] } diff --git a/mountpoint-s3-client/src/checksums.rs b/mountpoint-s3-client/src/checksums.rs index 412d8ba4a..b03794441 100644 --- a/mountpoint-s3-client/src/checksums.rs +++ b/mountpoint-s3-client/src/checksums.rs @@ -1,6 +1,7 @@ //! Provides base64 encoding/decoding for various checksums. pub use mountpoint_s3_crt::checksums::crc32::{self, Crc32}; pub use mountpoint_s3_crt::checksums::crc32c::{self, Crc32c}; +pub use mountpoint_s3_crt::checksums::crc64nvme::{self, Crc64nvme}; pub use mountpoint_s3_crt::checksums::sha1::{self, Sha1}; pub use mountpoint_s3_crt::checksums::sha256::{self, Sha256}; @@ -8,6 +9,11 @@ use base64ct::Base64; use base64ct::Encoding; use thiserror::Error; +/// The base64 encoding for this CRC64-NVME checksum value. +pub fn crc64nvme_to_base64(checksum: &Crc64nvme) -> String { + Base64::encode_string(&checksum.value().to_be_bytes()) +} + /// The base64 encoding for this CRC32C checksum value. pub fn crc32c_to_base64(checksum: &Crc32c) -> String { Base64::encode_string(&checksum.value().to_be_bytes()) diff --git a/mountpoint-s3-client/src/mock_client.rs b/mountpoint-s3-client/src/mock_client.rs index 30bfb1abd..679088e9f 100644 --- a/mountpoint-s3-client/src/mock_client.rs +++ b/mountpoint-s3-client/src/mock_client.rs @@ -22,7 +22,8 @@ use time::OffsetDateTime; use tracing::trace; use crate::checksums::{ - crc32, crc32_to_base64, crc32c, crc32c_to_base64, sha1, sha1_to_base64, sha256, sha256_to_base64, + crc32, crc32_to_base64, crc32c, crc32c_to_base64, crc64nvme, crc64nvme_to_base64, sha1, sha1_to_base64, sha256, + sha256_to_base64, }; use crate::error_metadata::{ClientErrorMetadata, ProvideErrorMetadata}; use crate::object_client::{ @@ -631,6 +632,10 @@ fn compute_checksum(content: &[u8], algorithms: &[ChecksumAlgorithm]) -> Checksu let mut checksum = Checksum::empty(); for algorithm in algorithms { match algorithm { + ChecksumAlgorithm::Crc64nvme => { + let crc64nvme = crc64nvme::checksum(content); + checksum.checksum_crc64nvme = Some(crc64nvme_to_base64(&crc64nvme)); + } ChecksumAlgorithm::Crc32 => { let crc32 = crc32::checksum(content); checksum.checksum_crc32 = Some(crc32_to_base64(&crc32)); @@ -1031,6 +1036,7 @@ impl ObjectClient for MockClient { .enumerate() .map(|(i, part)| ObjectPart { checksum: Some(Checksum { + checksum_crc64nvme: None, checksum_crc32: None, checksum_crc32c: part.checksum.clone(), checksum_sha1: None, @@ -1848,6 +1854,7 @@ mod tests { client.add_object("a.txt", MockObject::constant(0u8, 5, ETag::for_tests())); let mut object_b = MockObject::constant(1u8, 5, ETag::for_tests()); object_b.set_checksum(Checksum { + checksum_crc64nvme: None, checksum_crc32: None, checksum_crc32c: None, checksum_sha1: Some(String::from("QwzjTQIHJO11oZbfwq1nx3dy0Wk=")), @@ -2201,11 +2208,13 @@ mod tests { // We trust that other tests will cover checksum correctness, // so let's just check the right checksums are set. let Checksum { + checksum_crc64nvme, checksum_crc32, checksum_crc32c, checksum_sha1, checksum_sha256, } = attrs.checksum.expect("object checksum should be present"); + assert!(checksum_crc64nvme.is_none(), "CRC64NVME should not be set"); assert!(checksum_crc32.is_none(), "CRC32 should not be set"); assert!(checksum_crc32c.is_some(), "CRC32C should be set"); assert!(checksum_sha1.is_none(), "SHA1 should not be set"); diff --git a/mountpoint-s3-client/src/object_client.rs b/mountpoint-s3-client/src/object_client.rs index f23042043..9108361ca 100644 --- a/mountpoint-s3-client/src/object_client.rs +++ b/mountpoint-s3-client/src/object_client.rs @@ -10,7 +10,9 @@ use std::collections::HashMap; use thiserror::Error; use time::OffsetDateTime; -use crate::checksums::{self, crc32_to_base64, crc32c_to_base64, sha1_to_base64, sha256_to_base64}; +use crate::checksums::{ + self, crc32_to_base64, crc32c_to_base64, crc64nvme_to_base64, sha1_to_base64, sha256_to_base64, +}; use crate::error_metadata::{ClientErrorMetadata, ProvideErrorMetadata}; mod etag; @@ -567,6 +569,7 @@ impl PutObjectSingleParams { #[derive(Debug, Clone)] #[non_exhaustive] pub enum UploadChecksum { + Crc64nvme(checksums::Crc64nvme), Crc32c(checksums::Crc32c), Crc32(checksums::Crc32), Sha1(checksums::Sha1), @@ -577,6 +580,7 @@ impl UploadChecksum { /// The checksum algorithm used to compute this checksum. pub fn checksum_algorithm(&self) -> ChecksumAlgorithm { match self { + UploadChecksum::Crc64nvme(_) => ChecksumAlgorithm::Crc64nvme, UploadChecksum::Crc32c(_) => ChecksumAlgorithm::Crc32c, UploadChecksum::Crc32(_) => ChecksumAlgorithm::Crc32, UploadChecksum::Sha1(_) => ChecksumAlgorithm::Sha1, @@ -807,6 +811,9 @@ impl fmt::Display for ObjectAttribute { /// S3 API Reference* for more details. #[derive(Clone, Debug, PartialEq, Eq)] pub struct Checksum { + /// Base64-encoded, 64-bit CRC64NVME checksum of the object + pub checksum_crc64nvme: Option, + /// Base64-encoded, 32-bit CRC32 checksum of the object pub checksum_crc32: Option, @@ -824,6 +831,7 @@ impl Checksum { /// Construct an empty [Checksum] pub fn empty() -> Self { Self { + checksum_crc64nvme: None, checksum_crc32: None, checksum_crc32c: None, checksum_sha1: None, @@ -838,12 +846,16 @@ impl Checksum { // Pattern match forces us to accomodate any new fields when added. let Self { + checksum_crc64nvme, checksum_crc32, checksum_crc32c, checksum_sha1, checksum_sha256, } = &self; + if checksum_crc64nvme.is_some() { + algorithms.push(ChecksumAlgorithm::Crc64nvme); + } if checksum_crc32.is_some() { algorithms.push(ChecksumAlgorithm::Crc32); } @@ -865,6 +877,7 @@ impl From> for Checksum { fn from(value: Option) -> Self { let mut checksum = Checksum::empty(); match value.as_ref() { + Some(UploadChecksum::Crc64nvme(crc64)) => checksum.checksum_crc64nvme = Some(crc64nvme_to_base64(crc64)), Some(UploadChecksum::Crc32c(crc32c)) => checksum.checksum_crc32c = Some(crc32c_to_base64(crc32c)), Some(UploadChecksum::Crc32(crc32)) => checksum.checksum_crc32 = Some(crc32_to_base64(crc32)), Some(UploadChecksum::Sha1(sha1)) => checksum.checksum_sha1 = Some(sha1_to_base64(sha1)), @@ -923,6 +936,7 @@ mod tests { #[test] fn test_checksum_algorithm_one_set() { let checksum = Checksum { + checksum_crc64nvme: None, checksum_crc32: None, checksum_crc32c: None, checksum_sha1: Some("checksum_sha1".to_string()), @@ -934,6 +948,7 @@ mod tests { #[test] fn test_checksum_algorithm_none_set() { let checksum = Checksum { + checksum_crc64nvme: None, checksum_crc32: None, checksum_crc32c: None, checksum_sha1: None, @@ -946,6 +961,7 @@ mod tests { fn test_checksum_algorithm_many_set() { // Amazon S3 doesn't support more than one algorithm today, but just in case... let's show we don't panic. let checksum = Checksum { + checksum_crc64nvme: None, checksum_crc32: None, checksum_crc32c: Some("checksum_crc32c".to_string()), checksum_sha1: Some("checksum_sha1".to_string()), diff --git a/mountpoint-s3-client/src/s3_crt_client.rs b/mountpoint-s3-client/src/s3_crt_client.rs index 43a720b44..3c67e982f 100644 --- a/mountpoint-s3-client/src/s3_crt_client.rs +++ b/mountpoint-s3-client/src/s3_crt_client.rs @@ -38,7 +38,7 @@ use pin_project::{pin_project, pinned_drop}; use thiserror::Error; use tracing::{debug, error, trace, Span}; -use crate::checksums::{crc32_to_base64, crc32c_to_base64, sha1_to_base64, sha256_to_base64}; +use crate::checksums::{crc32_to_base64, crc32c_to_base64, crc64nvme_to_base64, sha1_to_base64, sha256_to_base64}; use crate::endpoint_config::EndpointError; use crate::endpoint_config::{self, EndpointConfig}; use crate::error_metadata::{ClientErrorMetadata, ProvideErrorMetadata}; @@ -956,6 +956,7 @@ impl<'a> S3Message<'a> { checksum: &UploadChecksum, ) -> Result<(), mountpoint_s3_crt::common::error::Error> { let header = match checksum { + UploadChecksum::Crc64nvme(crc64) => Header::new("x-amz-checksum-crc64nvme", crc64nvme_to_base64(crc64)), UploadChecksum::Crc32c(crc32c) => Header::new("x-amz-checksum-crc32c", crc32c_to_base64(crc32c)), UploadChecksum::Crc32(crc32) => Header::new("x-amz-checksum-crc32", crc32_to_base64(crc32)), UploadChecksum::Sha1(sha1) => Header::new("x-amz-checksum-sha1", sha1_to_base64(sha1)), @@ -1157,8 +1158,10 @@ fn parse_checksum(headers: &Headers) -> Result { let checksum_crc32c = headers.get_as_optional_string("x-amz-checksum-crc32c")?; let checksum_sha1 = headers.get_as_optional_string("x-amz-checksum-sha1")?; let checksum_sha256 = headers.get_as_optional_string("x-amz-checksum-sha256")?; + let checksum_crc64nvme = headers.get_as_optional_string("x-amz-checksum-crc64nvme")?; Ok(Checksum { + checksum_crc64nvme, checksum_crc32, checksum_crc32c, checksum_sha1, diff --git a/mountpoint-s3-client/src/s3_crt_client/get_object_attributes.rs b/mountpoint-s3-client/src/s3_crt_client/get_object_attributes.rs index 3a1fb4fc1..31d285ebb 100644 --- a/mountpoint-s3-client/src/s3_crt_client/get_object_attributes.rs +++ b/mountpoint-s3-client/src/s3_crt_client/get_object_attributes.rs @@ -86,8 +86,10 @@ impl GetObjectAttributesResult { let checksum_crc32c = get_field_or_none(element, "ChecksumCRC32C")?; let checksum_sha1 = get_field_or_none(element, "ChecksumSHA1")?; let checksum_sha256 = get_field_or_none(element, "ChecksumSHA256")?; + let checksum_crc64nvme = get_field_or_none(element, "ChecksumCRC64NVME")?; Ok(Checksum { + checksum_crc64nvme, checksum_crc32, checksum_crc32c, checksum_sha1, diff --git a/mountpoint-s3-client/tests/put_object_single.rs b/mountpoint-s3-client/tests/put_object_single.rs index 65263b5c4..a822b01ae 100644 --- a/mountpoint-s3-client/tests/put_object_single.rs +++ b/mountpoint-s3-client/tests/put_object_single.rs @@ -6,14 +6,13 @@ use std::collections::HashMap; use aws_sdk_s3::primitives::ByteStream; use common::*; -use mountpoint_s3_client::checksums::crc32c; +use mountpoint_s3_client::checksums::{crc32, crc32c, crc64nvme, sha1, sha256}; use mountpoint_s3_client::config::S3ClientConfig; use mountpoint_s3_client::error::{ObjectClientError, PutObjectError}; use mountpoint_s3_client::types::{ Checksum, ChecksumAlgorithm, GetObjectParams, PutObjectResult, PutObjectSingleParams, UploadChecksum, }; use mountpoint_s3_client::{ObjectClient, S3CrtClient}; -use mountpoint_s3_crt::checksums::{crc32, sha1, sha256}; use rand::Rng; use test_case::test_case; @@ -79,6 +78,7 @@ async fn test_put_object_single_empty( object_client_test!(test_put_object_single_empty); #[test_case(None; "no checksum")] +#[test_case(Some(ChecksumAlgorithm::Crc64nvme))] #[test_case(Some(ChecksumAlgorithm::Crc32c))] #[test_case(Some(ChecksumAlgorithm::Crc32))] #[test_case(Some(ChecksumAlgorithm::Sha1))] @@ -98,6 +98,7 @@ async fn test_put_checksums(checksum_algorithm: Option) { rng.fill(&mut contents[..]); let upload_checksum = match checksum_algorithm { + Some(ChecksumAlgorithm::Crc64nvme) => Some(UploadChecksum::Crc64nvme(crc64nvme::checksum(&contents))), Some(ChecksumAlgorithm::Crc32c) => Some(UploadChecksum::Crc32c(crc32c::checksum(&contents))), Some(ChecksumAlgorithm::Crc32) => Some(UploadChecksum::Crc32(crc32::checksum(&contents))), Some(ChecksumAlgorithm::Sha1) => Some(UploadChecksum::Sha1( @@ -117,7 +118,7 @@ async fn test_put_checksums(checksum_algorithm: Option) { .expect("put_object should succeed"); let sdk_client = get_test_sdk_client().await; - let output = sdk_client + let head_output = sdk_client .head_object() .bucket(&bucket) .key(key) @@ -126,14 +127,23 @@ async fn test_put_checksums(checksum_algorithm: Option) { .await .unwrap(); - let output_checksum = Checksum { - checksum_crc32: output.checksum_crc32, - checksum_crc32c: output.checksum_crc32_c, - checksum_sha1: output.checksum_sha1, - checksum_sha256: output.checksum_sha256, + let head_checksum = Checksum { + checksum_crc64nvme: if upload_checksum.is_none() { + // If no checksum was sent with the PutObject request, S3 automatically uses CRC-64NVME. + // We'll ignore it for the test below. + // See https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html. + None + } else { + head_output.checksum_crc64_nvme + }, + checksum_crc32: head_output.checksum_crc32, + checksum_crc32c: head_output.checksum_crc32_c, + checksum_sha1: head_output.checksum_sha1, + checksum_sha256: head_output.checksum_sha256, }; + let checksum: Checksum = upload_checksum.into(); - assert_eq!(output_checksum, checksum); + assert_eq!(head_checksum, checksum); } #[tokio::test] @@ -355,6 +365,7 @@ async fn test_append_fails_if_not_supported() { .put_object() .bucket(&bucket) .key(&key) + .checksum_algorithm(aws_sdk_s3::types::ChecksumAlgorithm::Crc64Nvme) .body(ByteStream::from_static(b"data")) .send() .await @@ -446,6 +457,7 @@ async fn test_append_existing_object() { .put_object() .bucket(&bucket) .key(&key) + .checksum_algorithm(aws_sdk_s3::types::ChecksumAlgorithm::Crc64Nvme) .body(ByteStream::from_static(contents)) .send() .await @@ -457,7 +469,8 @@ async fn test_append_existing_object() { rng.fill(&mut contents[..]); expected_contents.extend_from_slice(&contents); - let params = PutObjectSingleParams::new_for_append(object_size as u64); + let params = PutObjectSingleParams::new_for_append(object_size as u64) + .checksum(Some(UploadChecksum::Crc64nvme(crc64nvme::checksum(&contents)))); let put_object_result = client .put_object_single(&bucket, &key, ¶ms, &contents) .await @@ -493,6 +506,7 @@ async fn test_append_existing_object_if_match() { .put_object() .bucket(&bucket) .key(&key) + .checksum_algorithm(aws_sdk_s3::types::ChecksumAlgorithm::Crc64Nvme) .body(ByteStream::from_static(contents)) .send() .await @@ -505,7 +519,9 @@ async fn test_append_existing_object_if_match() { rng.fill(&mut contents[..]); expected_contents.extend_from_slice(&contents); - let params = PutObjectSingleParams::new_for_append(object_size as u64).if_match(Some(etag.to_owned().into())); + let params = PutObjectSingleParams::new_for_append(object_size as u64) + .checksum(Some(UploadChecksum::Crc64nvme(crc64nvme::checksum(&contents)))) + .if_match(Some(etag.to_owned().into())); let put_object_result = client .put_object_single(&bucket, &key, ¶ms, &contents) .await @@ -538,6 +554,7 @@ async fn test_append_existing_object_if_match_fails() { .put_object() .bucket(&bucket) .key(&key) + .checksum_algorithm(aws_sdk_s3::types::ChecksumAlgorithm::Crc64Nvme) .body(ByteStream::from_static(INITIAL_CONTENT)) .send() .await @@ -549,14 +566,18 @@ async fn test_append_existing_object_if_match_fails() { .put_object() .bucket(&bucket) .key(&key) + .checksum_algorithm(aws_sdk_s3::types::ChecksumAlgorithm::Crc64Nvme) .body(ByteStream::from_static(b"replace")) .send() .await .expect("sdk put_object should succeed"); assert_ne!(etag, replace_result.e_tag().expect("sdk put_object should return etag")); - let params = PutObjectSingleParams::new_for_append(object_size as u64).if_match(Some(etag.to_owned().into())); - let put_object_result = client.put_object_single(&bucket, &key, ¶ms, b"append").await; + let contents = b"append"; + let params = PutObjectSingleParams::new_for_append(object_size as u64) + .checksum(Some(UploadChecksum::Crc64nvme(crc64nvme::checksum(contents)))) + .if_match(Some(etag.to_owned().into())); + let put_object_result = client.put_object_single(&bucket, &key, ¶ms, contents).await; let error = put_object_result.expect_err("put_object_single with the old etag should fail"); assert!(matches!( @@ -581,13 +602,15 @@ async fn test_append_existing_object_with_empty() { .put_object() .bucket(&bucket) .key(&key) + .checksum_algorithm(aws_sdk_s3::types::ChecksumAlgorithm::Crc64Nvme) .body(ByteStream::from_static(contents)) .send() .await .expect("sdk put_object should succeed"); let contents = vec![0u8; 0]; - let params = PutObjectSingleParams::new_for_append(object_size as u64); + let params = PutObjectSingleParams::new_for_append(object_size as u64) + .checksum(Some(UploadChecksum::Crc64nvme(crc64nvme::checksum(&contents)))); let result = client.put_object_single(&bucket, &key, ¶ms, &contents).await; assert!(matches!( result, @@ -611,13 +634,15 @@ async fn test_append_empty_object_with_empty() { .put_object() .bucket(&bucket) .key(&key) + .checksum_algorithm(aws_sdk_s3::types::ChecksumAlgorithm::Crc64Nvme) .body(ByteStream::from_static(contents)) .send() .await .expect("sdk put_object should succeed"); let contents = vec![0u8; 0]; - let params = PutObjectSingleParams::new_for_append(object_size as u64); + let params = PutObjectSingleParams::new_for_append(object_size as u64) + .checksum(Some(UploadChecksum::Crc64nvme(crc64nvme::checksum(&contents)))); let result = client.put_object_single(&bucket, &key, ¶ms, &contents).await; assert!(matches!( result, @@ -661,6 +686,7 @@ async fn test_append_invalid_offset() { .put_object() .bucket(&bucket) .key(&key) + .checksum_algorithm(aws_sdk_s3::types::ChecksumAlgorithm::Crc64Nvme) .body(ByteStream::from_static(contents)) .send() .await @@ -671,7 +697,8 @@ async fn test_append_invalid_offset() { let mut contents = vec![0u8; 32]; rng.fill(&mut contents[..]); - let params = PutObjectSingleParams::new_for_append(object_size as u64 + 1); + let params = PutObjectSingleParams::new_for_append(object_size as u64 + 1) + .checksum(Some(UploadChecksum::Crc64nvme(crc64nvme::checksum(&contents)))); let result = client.put_object_single(&bucket, &key, ¶ms, &contents).await; assert!(matches!( result, @@ -758,6 +785,7 @@ async fn test_append_with_invalid_checksum() { )); } +#[test_case(aws_sdk_s3::types::ChecksumAlgorithm::Crc64Nvme)] #[test_case(aws_sdk_s3::types::ChecksumAlgorithm::Crc32C)] #[test_case(aws_sdk_s3::types::ChecksumAlgorithm::Crc32)] #[test_case(aws_sdk_s3::types::ChecksumAlgorithm::Sha1)] @@ -765,8 +793,6 @@ async fn test_append_with_invalid_checksum() { #[tokio::test] #[cfg(feature = "s3express_tests")] async fn test_append_with_matching_checksum(checksum_algorithm: aws_sdk_s3::types::ChecksumAlgorithm) { - use mountpoint_s3_client::checksums::{crc32, crc32c, sha1, sha256}; - let (bucket, prefix) = get_test_bucket_and_prefix("test_append_with_matching_checksum"); let client = get_test_client(); let key = format!("{prefix}hello"); @@ -791,6 +817,7 @@ async fn test_append_with_matching_checksum(checksum_algorithm: aws_sdk_s3::type let mut contents = vec![0u8; 32]; rng.fill(&mut contents[..]); let checksum = match checksum_algorithm { + aws_sdk_s3::types::ChecksumAlgorithm::Crc64Nvme => UploadChecksum::Crc64nvme(crc64nvme::checksum(&contents)), aws_sdk_s3::types::ChecksumAlgorithm::Crc32C => UploadChecksum::Crc32c(crc32c::checksum(&contents)), aws_sdk_s3::types::ChecksumAlgorithm::Crc32 => UploadChecksum::Crc32(crc32::checksum(&contents)), aws_sdk_s3::types::ChecksumAlgorithm::Sha1 => UploadChecksum::Sha1(sha1::checksum(&contents).unwrap()), @@ -805,3 +832,54 @@ async fn test_append_with_matching_checksum(checksum_algorithm: aws_sdk_s3::type .await .expect("append should succeed"); } + +#[cfg(feature = "s3express_tests")] +#[tokio::test] +async fn test_append_crc64() { + use mountpoint_s3_client::types::{ChecksumMode, HeadObjectParams}; + + const PART_SIZE: usize = 5 * 1024 * 1024; + let (bucket, prefix) = get_test_bucket_and_prefix("test_append_crc64"); + let client_config = S3ClientConfig::new() + .part_size(PART_SIZE) + .endpoint_config(get_test_endpoint_config()); + let client = S3CrtClient::new(client_config).expect("could not create test client"); + let key = format!("{prefix}hello"); + + let initial_contents = vec![0u8; 1024]; + client + .put_object_single( + &bucket, + &key, + &PutObjectSingleParams::new().checksum(None), + &initial_contents, + ) + .await + .expect("initial put_object with no checksum should succeed"); + + let head_object = client + .head_object( + &bucket, + &key, + &HeadObjectParams::new().checksum_mode(Some(ChecksumMode::Enabled)), + ) + .await + .unwrap(); + + assert!( + head_object.checksum.checksum_crc64nvme.is_some(), + "service should have computed crc64nvme checksum" + ); + + let contents = vec![1u8; 1024]; + let append_checksum = UploadChecksum::Crc64nvme(crc64nvme::checksum(&contents)); + client + .put_object_single( + &bucket, + &key, + &PutObjectSingleParams::new_for_append(initial_contents.len() as u64).checksum(Some(append_checksum)), + &contents, + ) + .await + .expect("append put_object with crc64nvme checksum should succeed"); +} diff --git a/mountpoint-s3-crt/benches/checksums.rs b/mountpoint-s3-crt/benches/checksums.rs index f46ad1031..98684a89f 100644 --- a/mountpoint-s3-crt/benches/checksums.rs +++ b/mountpoint-s3-crt/benches/checksums.rs @@ -1,7 +1,7 @@ //! Benchmarks for the CRT checksums library use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; -use mountpoint_s3_crt::checksums::{crc32, crc32c, crc64, sha1, sha256}; +use mountpoint_s3_crt::checksums::{crc32, crc32c, crc64nvme, sha1, sha256}; use rand::rngs::SmallRng; use rand::{Rng, SeedableRng}; @@ -41,8 +41,8 @@ fn crc32c(c: &mut Criterion) { benchmark_hasher(c, crc32c::checksum, "crc32c"); } -fn crc64(c: &mut Criterion) { - benchmark_hasher(c, crc64::checksum, "crc64"); +fn crc64nvme(c: &mut Criterion) { + benchmark_hasher(c, crc64nvme::checksum, "crc64"); } fn sha1(c: &mut Criterion) { @@ -53,5 +53,5 @@ fn sha256(c: &mut Criterion) { benchmark_hasher(c, sha256::checksum, "sha256"); } -criterion_group!(checksum_benches, crc32, crc32c, crc64, sha1, sha256); +criterion_group!(checksum_benches, crc32, crc32c, crc64nvme, sha1, sha256); criterion_main!(checksum_benches); diff --git a/mountpoint-s3-crt/src/checksums/crc64.rs b/mountpoint-s3-crt/src/checksums/crc64nvme.rs similarity index 55% rename from mountpoint-s3-crt/src/checksums/crc64.rs rename to mountpoint-s3-crt/src/checksums/crc64nvme.rs index 82c5629ec..bac0be472 100644 --- a/mountpoint-s3-crt/src/checksums/crc64.rs +++ b/mountpoint-s3-crt/src/checksums/crc64nvme.rs @@ -2,52 +2,52 @@ use mountpoint_s3_crt_sys::aws_checksums_crc64nvme; /// CRC64-NVME (aka. CRC64-Rocksoft) checksum #[derive(Debug, Clone, PartialEq, Eq, Copy)] -pub struct Crc64(u64); +pub struct Crc64nvme(u64); -impl Crc64 { - /// Create a new CRC64 checksum with the given value. +impl Crc64nvme { + /// Create a new CRC64-NVME checksum with the given value. pub fn new(value: u64) -> Self { Self(value) } - /// The CRC64 checksum value. + /// The CRC64-NVME checksum value. pub fn value(&self) -> u64 { self.0 } } -/// Computes the CRC64 checksum of a byte slice. +/// Computes the CRC64-NVME checksum of a byte slice. /// -/// Use [Crc64Hasher] for more advanced use-cases. -pub fn checksum(buf: &[u8]) -> Crc64 { - let mut hasher = Crc64Hasher::new(); +/// Use [Crc64nvmeHasher] for more advanced use-cases. +pub fn checksum(buf: &[u8]) -> Crc64nvme { + let mut hasher = Crc64nvmeHasher::new(); hasher.update(buf); hasher.finalize() } -/// CRC64 Hasher +/// CRC64-NVME Hasher #[derive(Debug, Clone)] -pub struct Crc64Hasher { - state: Crc64, +pub struct Crc64nvmeHasher { + state: Crc64nvme, } -impl Crc64Hasher { - /// Create a new CRC64 Hasher. +impl Crc64nvmeHasher { + /// Create a new CRC64-NVME Hasher. pub fn new() -> Self { - Self { state: Crc64(0) } + Self { state: Crc64nvme(0) } } /// Update the hash state with the given bytes slice. pub fn update(&mut self, buf: &[u8]) { - self.state = Crc64(Self::crc64(buf, self.state.0)); + self.state = Crc64nvme(Self::crc64(buf, self.state.0)); } - /// Finalize the hash state and return the computed CRC64 checksum value. - pub fn finalize(self) -> Crc64 { + /// Finalize the hash state and return the computed CRC64-NVME checksum value. + pub fn finalize(self) -> Crc64nvme { self.state } - /// Compute CRC64 checksum of the data in the given bytes slice, append to the previous checksum. + /// Compute CRC64-NVME checksum of the data in the given bytes slice, append to the previous checksum. /// /// The underlying CRT function requires the buffer's length to be type [::libc::c_int], so this function cannot take /// any buffer that is bigger than [::libc::c_int::MAX] as an input. @@ -64,7 +64,7 @@ impl Crc64Hasher { } } -impl Default for Crc64Hasher { +impl Default for Crc64nvmeHasher { fn default() -> Self { Self::new() } @@ -72,21 +72,21 @@ impl Default for Crc64Hasher { #[cfg(test)] mod tests { - use crate::checksums::crc64::{self, Crc64}; + use crate::checksums::crc64nvme::{self, Crc64nvme}; #[test] - fn crc64_simple() { + fn crc64nvme_simple() { let buf: &[u8] = b"123456789"; - let crc = crc64::checksum(buf); - assert_eq!(crc, Crc64(0xAE8B14860A799888)); + let crc = crc64nvme::checksum(buf); + assert_eq!(crc, Crc64nvme(0xAE8B14860A799888)); } #[test] - fn crc64_append() { - let mut hasher = crc64::Crc64Hasher::new(); + fn crc64nvme_append() { + let mut hasher = crc64nvme::Crc64nvmeHasher::new(); hasher.update(b"1234"); hasher.update(b"56789"); let crc = hasher.finalize(); - assert_eq!(crc, Crc64(0xAE8B14860A799888)); + assert_eq!(crc, Crc64nvme(0xAE8B14860A799888)); } } diff --git a/mountpoint-s3-crt/src/checksums/mod.rs b/mountpoint-s3-crt/src/checksums/mod.rs index 3babeed60..ae3f65820 100644 --- a/mountpoint-s3-crt/src/checksums/mod.rs +++ b/mountpoint-s3-crt/src/checksums/mod.rs @@ -6,8 +6,8 @@ pub mod crc32; /// CRC32C checksums pub mod crc32c; -/// CRC64 checksums -pub mod crc64; +/// CRC64NVME checksums +pub mod crc64nvme; /// SHA1 checksums pub mod sha1; diff --git a/mountpoint-s3-crt/src/s3/client.rs b/mountpoint-s3-crt/src/s3/client.rs index be745d396..3f58e187d 100644 --- a/mountpoint-s3-crt/src/s3/client.rs +++ b/mountpoint-s3-crt/src/s3/client.rs @@ -1463,6 +1463,8 @@ impl ChecksumConfig { #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] #[non_exhaustive] pub enum ChecksumAlgorithm { + /// Crc64nvme checksum. + Crc64nvme, /// Crc32c checksum. Crc32c, /// Crc32 checksum. @@ -1482,6 +1484,7 @@ impl ChecksumAlgorithm { fn from_aws_s3_checksum_algorithm(algorithm: aws_s3_checksum_algorithm) -> Option { match algorithm { aws_s3_checksum_algorithm::AWS_SCA_NONE => None, + aws_s3_checksum_algorithm::AWS_SCA_CRC64NVME => Some(ChecksumAlgorithm::Crc64nvme), aws_s3_checksum_algorithm::AWS_SCA_CRC32C => Some(ChecksumAlgorithm::Crc32c), aws_s3_checksum_algorithm::AWS_SCA_CRC32 => Some(ChecksumAlgorithm::Crc32), aws_s3_checksum_algorithm::AWS_SCA_SHA1 => Some(ChecksumAlgorithm::Sha1), @@ -1494,6 +1497,7 @@ impl ChecksumAlgorithm { impl Display for ChecksumAlgorithm { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { + ChecksumAlgorithm::Crc64nvme => f.write_str("CRC64NVME"), ChecksumAlgorithm::Crc32c => f.write_str("CRC32C"), ChecksumAlgorithm::Crc32 => f.write_str("CRC32"), ChecksumAlgorithm::Sha1 => f.write_str("SHA1"), diff --git a/mountpoint-s3/Cargo.toml b/mountpoint-s3/Cargo.toml index e5b6b9138..cce66cef7 100644 --- a/mountpoint-s3/Cargo.toml +++ b/mountpoint-s3/Cargo.toml @@ -58,7 +58,7 @@ assert_cmd = "2.0.16" assert_fs = "1.1.2" aws-config = "1.5.10" aws-credential-types = "1.2.1" -aws-sdk-s3 = "1.65.0" +aws-sdk-s3 = "1.69.0" ctor = "0.2.9" filetime = "0.2.25" futures = { version = "*", features = ["thread-pool"] } diff --git a/mountpoint-s3/src/upload/hasher.rs b/mountpoint-s3/src/upload/hasher.rs index 7addcf289..a74107790 100644 --- a/mountpoint-s3/src/upload/hasher.rs +++ b/mountpoint-s3/src/upload/hasher.rs @@ -1,4 +1,4 @@ -use mountpoint_s3_client::checksums::{crc32, crc32c, sha1, sha256}; +use mountpoint_s3_client::checksums::{crc32, crc32c, crc64nvme, sha1, sha256}; use mountpoint_s3_client::types::{ChecksumAlgorithm, UploadChecksum}; use mountpoint_s3_crt::common::allocator::Allocator; use thiserror::Error; @@ -7,6 +7,7 @@ use thiserror::Error; pub enum ChecksumHasher { #[default] None, + Crc64nvme(crc64nvme::Crc64nvmeHasher), Crc32(crc32::Hasher), Crc32c(crc32c::Hasher), Sha1(sha1::Sha1Hasher), @@ -16,6 +17,7 @@ pub enum ChecksumHasher { impl ChecksumHasher { pub fn new(checksum_algorithm: &Option) -> Result { match checksum_algorithm { + Some(ChecksumAlgorithm::Crc64nvme) => Ok(Self::Crc64nvme(crc64nvme::Crc64nvmeHasher::new())), Some(ChecksumAlgorithm::Crc32) => Ok(Self::Crc32(crc32::Hasher::new())), Some(ChecksumAlgorithm::Crc32c) => Ok(Self::Crc32c(crc32c::Hasher::new())), Some(ChecksumAlgorithm::Sha1) => Ok(Self::Sha1(sha1::Sha1Hasher::new(&Allocator::default())?)), @@ -28,6 +30,7 @@ impl ChecksumHasher { pub fn update(&mut self, data: &[u8]) -> Result<(), ChecksumHasherError> { match self { ChecksumHasher::None => {} + ChecksumHasher::Crc64nvme(hasher) => hasher.update(data), ChecksumHasher::Crc32(hasher) => hasher.update(data), ChecksumHasher::Crc32c(hasher) => hasher.update(data), ChecksumHasher::Sha1(hasher) => hasher.update(data)?, @@ -39,6 +42,7 @@ impl ChecksumHasher { pub fn finalize(self) -> Result, ChecksumHasherError> { match self { ChecksumHasher::None => Ok(None), + ChecksumHasher::Crc64nvme(hasher) => Ok(Some(UploadChecksum::Crc64nvme(hasher.finalize()))), ChecksumHasher::Crc32(hasher) => Ok(Some(UploadChecksum::Crc32(hasher.finalize()))), ChecksumHasher::Crc32c(hasher) => Ok(Some(UploadChecksum::Crc32c(hasher.finalize()))), ChecksumHasher::Sha1(hasher) => Ok(Some(UploadChecksum::Sha1(hasher.finalize(&Allocator::default())?))), diff --git a/mountpoint-s3/src/upload/incremental.rs b/mountpoint-s3/src/upload/incremental.rs index 581963892..29a7abf70 100644 --- a/mountpoint-s3/src/upload/incremental.rs +++ b/mountpoint-s3/src/upload/incremental.rs @@ -488,6 +488,7 @@ mod tests { #[test_case(Some(MockObject::constant(0xab, 20, ETag::for_tests()).with_computed_checksums(&[ChecksumAlgorithm::Crc32c])))] #[test_case(Some(MockObject::from([]).with_computed_checksums(&[ChecksumAlgorithm::Crc32c])))] #[test_case(Some(MockObject::from([0xbb; 128])))] + #[test_case(Some(MockObject::from([0xbb; 128]).with_computed_checksums(&[ChecksumAlgorithm::Crc64nvme])))] #[test_case(Some(MockObject::from([0xbb; 128]).with_computed_checksums(&[ChecksumAlgorithm::Crc32c])))] #[test_case(Some(MockObject::from([0xbb; 128]).with_computed_checksums(&[ChecksumAlgorithm::Crc32])))] #[test_case(Some(MockObject::from([0xbb; 128]).with_computed_checksums(&[ChecksumAlgorithm::Sha1])))] @@ -562,6 +563,7 @@ mod tests { #[test_case(Some(MockObject::constant(0xab, 20, ETag::for_tests()).with_computed_checksums(&[ChecksumAlgorithm::Crc32c])))] #[test_case(Some(MockObject::from([]).with_computed_checksums(&[ChecksumAlgorithm::Crc32c])))] #[test_case(Some(MockObject::from([0xbb; 128])))] + #[test_case(Some(MockObject::from([0xbb; 128]).with_computed_checksums(&[ChecksumAlgorithm::Crc64nvme])))] #[test_case(Some(MockObject::from([0xbb; 128]).with_computed_checksums(&[ChecksumAlgorithm::Crc32c])))] #[test_case(Some(MockObject::from([0xbb; 128]).with_computed_checksums(&[ChecksumAlgorithm::Crc32])))] #[test_case(Some(MockObject::from([0xbb; 128]).with_computed_checksums(&[ChecksumAlgorithm::Sha1])))] @@ -627,11 +629,13 @@ mod tests { } #[test_case(None, None)] + #[test_case(None, Some(ChecksumAlgorithm::Crc64nvme))] #[test_case(None, Some(ChecksumAlgorithm::Crc32c))] #[test_case(None, Some(ChecksumAlgorithm::Crc32))] #[test_case(None, Some(ChecksumAlgorithm::Sha1))] #[test_case(None, Some(ChecksumAlgorithm::Sha256))] #[test_case(Some(&[]), Some(ChecksumAlgorithm::Crc32c))] + #[test_case(Some(&[ChecksumAlgorithm::Crc64nvme]), None)] #[test_case(Some(&[ChecksumAlgorithm::Crc32c]), None)] #[test_case(Some(&[ChecksumAlgorithm::Crc32]), None)] #[test_case(Some(&[ChecksumAlgorithm::Sha1]), None)] diff --git a/mountpoint-s3/tests/common/fuse.rs b/mountpoint-s3/tests/common/fuse.rs index fa779968d..bb33801da 100644 --- a/mountpoint-s3/tests/common/fuse.rs +++ b/mountpoint-s3/tests/common/fuse.rs @@ -506,6 +506,7 @@ pub mod s3_session { params: PutObjectSingleParams, ) -> Result<(), Box> { let checksum_algorithm = params.checksum.map(|c| match c.checksum_algorithm() { + mountpoint_s3_crt::s3::client::ChecksumAlgorithm::Crc64nvme => ChecksumAlgorithm::Crc64Nvme, mountpoint_s3_crt::s3::client::ChecksumAlgorithm::Crc32c => ChecksumAlgorithm::Crc32C, mountpoint_s3_crt::s3::client::ChecksumAlgorithm::Crc32 => ChecksumAlgorithm::Crc32, mountpoint_s3_crt::s3::client::ChecksumAlgorithm::Sha1 => ChecksumAlgorithm::Sha1, @@ -600,6 +601,7 @@ pub mod s3_session { .send(), )?; let object_checksum = attrs.checksum.map(|checksum| Checksum { + checksum_crc64nvme: checksum.checksum_crc64_nvme, checksum_crc32: checksum.checksum_crc32, checksum_crc32c: checksum.checksum_crc32_c, checksum_sha1: checksum.checksum_sha1, @@ -615,6 +617,7 @@ pub mod s3_session { .into_iter() .map(|part| { Some(Checksum { + checksum_crc64nvme: part.checksum_crc64_nvme, checksum_crc32: part.checksum_crc32, checksum_crc32c: part.checksum_crc32_c, checksum_sha1: part.checksum_sha1, diff --git a/mountpoint-s3/tests/fuse_tests/write_test.rs b/mountpoint-s3/tests/fuse_tests/write_test.rs index 96c3be185..7f4a05219 100644 --- a/mountpoint-s3/tests/fuse_tests/write_test.rs +++ b/mountpoint-s3/tests/fuse_tests/write_test.rs @@ -1416,13 +1416,19 @@ fn write_checksums_test( assert!(object_crc32c || parts_crc32c, "crc32c is used for trailing checksums"); } UploadChecksumsMode::Disabled => { - // For S3 Standard, the list of parts is only present if checksums were used, but for S3 - // Express One Zone the list of parts is always present. The important thing is just that - // the *checksums* aren't present, because we disabled those. + // If no checksum was sent with the PutObject request, S3 automatically uses CRC-64NVME. + // We'll ignore it for the test below. + // See https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html. assert!( - object_checksum.is_none_or(|c| c == Checksum::empty()), + object_checksum.is_none_or(|c| Checksum { + checksum_crc64nvme: None, + ..c + } == Checksum::empty()), "checksums should not be present when upload checksums are disabled" ); + + // For S3 Standard, the list of parts is only present if checksums were used, but for S3 + // Express One Zone the list of parts is always present. for part_checksum in part_checksums { assert!( part_checksum.is_none_or(|c| c == Checksum::empty()), @@ -1562,6 +1568,9 @@ fn append_with_checksums(creator_fn: impl TestSessionCreator, checksum_algorithm const INITIAL_CONTENT: &[u8] = b"initial"; let checksum = match checksum_algorithm { None => None, + Some(ChecksumAlgorithm::Crc64nvme) => Some(UploadChecksum::Crc64nvme( + mountpoint_s3_client::checksums::crc64nvme::checksum(INITIAL_CONTENT), + )), Some(ChecksumAlgorithm::Crc32c) => Some(UploadChecksum::Crc32c( mountpoint_s3_client::checksums::crc32c::checksum(INITIAL_CONTENT), )), @@ -1596,6 +1605,7 @@ fn append_with_checksums(creator_fn: impl TestSessionCreator, checksum_algorithm #[cfg(feature = "s3express_tests")] #[test_case(None)] +#[test_case(Some(ChecksumAlgorithm::Crc64nvme))] #[test_case(Some(ChecksumAlgorithm::Crc32c))] #[test_case(Some(ChecksumAlgorithm::Crc32))] #[test_case(Some(ChecksumAlgorithm::Sha1))] @@ -1605,6 +1615,7 @@ fn append_with_checksums_s3(checksum_algorithm: Option) { } #[test_case(None)] +#[test_case(Some(ChecksumAlgorithm::Crc64nvme))] #[test_case(Some(ChecksumAlgorithm::Crc32c))] #[test_case(Some(ChecksumAlgorithm::Crc32))] #[test_case(Some(ChecksumAlgorithm::Sha1))]