diff --git a/mountpoint-s3-client/src/checksums.rs b/mountpoint-s3-client/src/checksums.rs index 412d8ba4a..b03794441 100644 --- a/mountpoint-s3-client/src/checksums.rs +++ b/mountpoint-s3-client/src/checksums.rs @@ -1,6 +1,7 @@ //! Provides base64 encoding/decoding for various checksums. pub use mountpoint_s3_crt::checksums::crc32::{self, Crc32}; pub use mountpoint_s3_crt::checksums::crc32c::{self, Crc32c}; +pub use mountpoint_s3_crt::checksums::crc64nvme::{self, Crc64nvme}; pub use mountpoint_s3_crt::checksums::sha1::{self, Sha1}; pub use mountpoint_s3_crt::checksums::sha256::{self, Sha256}; @@ -8,6 +9,11 @@ use base64ct::Base64; use base64ct::Encoding; use thiserror::Error; +/// The base64 encoding for this CRC64-NVME checksum value. +pub fn crc64nvme_to_base64(checksum: &Crc64nvme) -> String { + Base64::encode_string(&checksum.value().to_be_bytes()) +} + /// The base64 encoding for this CRC32C checksum value. pub fn crc32c_to_base64(checksum: &Crc32c) -> String { Base64::encode_string(&checksum.value().to_be_bytes()) diff --git a/mountpoint-s3-client/src/mock_client.rs b/mountpoint-s3-client/src/mock_client.rs index 30bfb1abd..679088e9f 100644 --- a/mountpoint-s3-client/src/mock_client.rs +++ b/mountpoint-s3-client/src/mock_client.rs @@ -22,7 +22,8 @@ use time::OffsetDateTime; use tracing::trace; use crate::checksums::{ - crc32, crc32_to_base64, crc32c, crc32c_to_base64, sha1, sha1_to_base64, sha256, sha256_to_base64, + crc32, crc32_to_base64, crc32c, crc32c_to_base64, crc64nvme, crc64nvme_to_base64, sha1, sha1_to_base64, sha256, + sha256_to_base64, }; use crate::error_metadata::{ClientErrorMetadata, ProvideErrorMetadata}; use crate::object_client::{ @@ -631,6 +632,10 @@ fn compute_checksum(content: &[u8], algorithms: &[ChecksumAlgorithm]) -> Checksu let mut checksum = Checksum::empty(); for algorithm in algorithms { match algorithm { + ChecksumAlgorithm::Crc64nvme => { + let crc64nvme = crc64nvme::checksum(content); + checksum.checksum_crc64nvme = Some(crc64nvme_to_base64(&crc64nvme)); + } ChecksumAlgorithm::Crc32 => { let crc32 = crc32::checksum(content); checksum.checksum_crc32 = Some(crc32_to_base64(&crc32)); @@ -1031,6 +1036,7 @@ impl ObjectClient for MockClient { .enumerate() .map(|(i, part)| ObjectPart { checksum: Some(Checksum { + checksum_crc64nvme: None, checksum_crc32: None, checksum_crc32c: part.checksum.clone(), checksum_sha1: None, @@ -1848,6 +1854,7 @@ mod tests { client.add_object("a.txt", MockObject::constant(0u8, 5, ETag::for_tests())); let mut object_b = MockObject::constant(1u8, 5, ETag::for_tests()); object_b.set_checksum(Checksum { + checksum_crc64nvme: None, checksum_crc32: None, checksum_crc32c: None, checksum_sha1: Some(String::from("QwzjTQIHJO11oZbfwq1nx3dy0Wk=")), @@ -2201,11 +2208,13 @@ mod tests { // We trust that other tests will cover checksum correctness, // so let's just check the right checksums are set. let Checksum { + checksum_crc64nvme, checksum_crc32, checksum_crc32c, checksum_sha1, checksum_sha256, } = attrs.checksum.expect("object checksum should be present"); + assert!(checksum_crc64nvme.is_none(), "CRC64NVME should not be set"); assert!(checksum_crc32.is_none(), "CRC32 should not be set"); assert!(checksum_crc32c.is_some(), "CRC32C should be set"); assert!(checksum_sha1.is_none(), "SHA1 should not be set"); diff --git a/mountpoint-s3-client/src/object_client.rs b/mountpoint-s3-client/src/object_client.rs index f23042043..9108361ca 100644 --- a/mountpoint-s3-client/src/object_client.rs +++ b/mountpoint-s3-client/src/object_client.rs @@ -10,7 +10,9 @@ use std::collections::HashMap; use thiserror::Error; use time::OffsetDateTime; -use crate::checksums::{self, crc32_to_base64, crc32c_to_base64, sha1_to_base64, sha256_to_base64}; +use crate::checksums::{ + self, crc32_to_base64, crc32c_to_base64, crc64nvme_to_base64, sha1_to_base64, sha256_to_base64, +}; use crate::error_metadata::{ClientErrorMetadata, ProvideErrorMetadata}; mod etag; @@ -567,6 +569,7 @@ impl PutObjectSingleParams { #[derive(Debug, Clone)] #[non_exhaustive] pub enum UploadChecksum { + Crc64nvme(checksums::Crc64nvme), Crc32c(checksums::Crc32c), Crc32(checksums::Crc32), Sha1(checksums::Sha1), @@ -577,6 +580,7 @@ impl UploadChecksum { /// The checksum algorithm used to compute this checksum. pub fn checksum_algorithm(&self) -> ChecksumAlgorithm { match self { + UploadChecksum::Crc64nvme(_) => ChecksumAlgorithm::Crc64nvme, UploadChecksum::Crc32c(_) => ChecksumAlgorithm::Crc32c, UploadChecksum::Crc32(_) => ChecksumAlgorithm::Crc32, UploadChecksum::Sha1(_) => ChecksumAlgorithm::Sha1, @@ -807,6 +811,9 @@ impl fmt::Display for ObjectAttribute { /// S3 API Reference* for more details. #[derive(Clone, Debug, PartialEq, Eq)] pub struct Checksum { + /// Base64-encoded, 64-bit CRC64NVME checksum of the object + pub checksum_crc64nvme: Option, + /// Base64-encoded, 32-bit CRC32 checksum of the object pub checksum_crc32: Option, @@ -824,6 +831,7 @@ impl Checksum { /// Construct an empty [Checksum] pub fn empty() -> Self { Self { + checksum_crc64nvme: None, checksum_crc32: None, checksum_crc32c: None, checksum_sha1: None, @@ -838,12 +846,16 @@ impl Checksum { // Pattern match forces us to accomodate any new fields when added. let Self { + checksum_crc64nvme, checksum_crc32, checksum_crc32c, checksum_sha1, checksum_sha256, } = &self; + if checksum_crc64nvme.is_some() { + algorithms.push(ChecksumAlgorithm::Crc64nvme); + } if checksum_crc32.is_some() { algorithms.push(ChecksumAlgorithm::Crc32); } @@ -865,6 +877,7 @@ impl From> for Checksum { fn from(value: Option) -> Self { let mut checksum = Checksum::empty(); match value.as_ref() { + Some(UploadChecksum::Crc64nvme(crc64)) => checksum.checksum_crc64nvme = Some(crc64nvme_to_base64(crc64)), Some(UploadChecksum::Crc32c(crc32c)) => checksum.checksum_crc32c = Some(crc32c_to_base64(crc32c)), Some(UploadChecksum::Crc32(crc32)) => checksum.checksum_crc32 = Some(crc32_to_base64(crc32)), Some(UploadChecksum::Sha1(sha1)) => checksum.checksum_sha1 = Some(sha1_to_base64(sha1)), @@ -923,6 +936,7 @@ mod tests { #[test] fn test_checksum_algorithm_one_set() { let checksum = Checksum { + checksum_crc64nvme: None, checksum_crc32: None, checksum_crc32c: None, checksum_sha1: Some("checksum_sha1".to_string()), @@ -934,6 +948,7 @@ mod tests { #[test] fn test_checksum_algorithm_none_set() { let checksum = Checksum { + checksum_crc64nvme: None, checksum_crc32: None, checksum_crc32c: None, checksum_sha1: None, @@ -946,6 +961,7 @@ mod tests { fn test_checksum_algorithm_many_set() { // Amazon S3 doesn't support more than one algorithm today, but just in case... let's show we don't panic. let checksum = Checksum { + checksum_crc64nvme: None, checksum_crc32: None, checksum_crc32c: Some("checksum_crc32c".to_string()), checksum_sha1: Some("checksum_sha1".to_string()), diff --git a/mountpoint-s3-client/src/s3_crt_client.rs b/mountpoint-s3-client/src/s3_crt_client.rs index 43a720b44..3c67e982f 100644 --- a/mountpoint-s3-client/src/s3_crt_client.rs +++ b/mountpoint-s3-client/src/s3_crt_client.rs @@ -38,7 +38,7 @@ use pin_project::{pin_project, pinned_drop}; use thiserror::Error; use tracing::{debug, error, trace, Span}; -use crate::checksums::{crc32_to_base64, crc32c_to_base64, sha1_to_base64, sha256_to_base64}; +use crate::checksums::{crc32_to_base64, crc32c_to_base64, crc64nvme_to_base64, sha1_to_base64, sha256_to_base64}; use crate::endpoint_config::EndpointError; use crate::endpoint_config::{self, EndpointConfig}; use crate::error_metadata::{ClientErrorMetadata, ProvideErrorMetadata}; @@ -956,6 +956,7 @@ impl<'a> S3Message<'a> { checksum: &UploadChecksum, ) -> Result<(), mountpoint_s3_crt::common::error::Error> { let header = match checksum { + UploadChecksum::Crc64nvme(crc64) => Header::new("x-amz-checksum-crc64nvme", crc64nvme_to_base64(crc64)), UploadChecksum::Crc32c(crc32c) => Header::new("x-amz-checksum-crc32c", crc32c_to_base64(crc32c)), UploadChecksum::Crc32(crc32) => Header::new("x-amz-checksum-crc32", crc32_to_base64(crc32)), UploadChecksum::Sha1(sha1) => Header::new("x-amz-checksum-sha1", sha1_to_base64(sha1)), @@ -1157,8 +1158,10 @@ fn parse_checksum(headers: &Headers) -> Result { let checksum_crc32c = headers.get_as_optional_string("x-amz-checksum-crc32c")?; let checksum_sha1 = headers.get_as_optional_string("x-amz-checksum-sha1")?; let checksum_sha256 = headers.get_as_optional_string("x-amz-checksum-sha256")?; + let checksum_crc64nvme = headers.get_as_optional_string("x-amz-checksum-crc64nvme")?; Ok(Checksum { + checksum_crc64nvme, checksum_crc32, checksum_crc32c, checksum_sha1, diff --git a/mountpoint-s3-client/src/s3_crt_client/get_object_attributes.rs b/mountpoint-s3-client/src/s3_crt_client/get_object_attributes.rs index 3a1fb4fc1..31d285ebb 100644 --- a/mountpoint-s3-client/src/s3_crt_client/get_object_attributes.rs +++ b/mountpoint-s3-client/src/s3_crt_client/get_object_attributes.rs @@ -86,8 +86,10 @@ impl GetObjectAttributesResult { let checksum_crc32c = get_field_or_none(element, "ChecksumCRC32C")?; let checksum_sha1 = get_field_or_none(element, "ChecksumSHA1")?; let checksum_sha256 = get_field_or_none(element, "ChecksumSHA256")?; + let checksum_crc64nvme = get_field_or_none(element, "ChecksumCRC64NVME")?; Ok(Checksum { + checksum_crc64nvme, checksum_crc32, checksum_crc32c, checksum_sha1, diff --git a/mountpoint-s3-client/tests/put_object_single.rs b/mountpoint-s3-client/tests/put_object_single.rs index 65263b5c4..683aa1684 100644 --- a/mountpoint-s3-client/tests/put_object_single.rs +++ b/mountpoint-s3-client/tests/put_object_single.rs @@ -6,14 +6,13 @@ use std::collections::HashMap; use aws_sdk_s3::primitives::ByteStream; use common::*; -use mountpoint_s3_client::checksums::crc32c; +use mountpoint_s3_client::checksums::{crc32, crc32c, crc64nvme, sha1, sha256}; use mountpoint_s3_client::config::S3ClientConfig; use mountpoint_s3_client::error::{ObjectClientError, PutObjectError}; use mountpoint_s3_client::types::{ Checksum, ChecksumAlgorithm, GetObjectParams, PutObjectResult, PutObjectSingleParams, UploadChecksum, }; use mountpoint_s3_client::{ObjectClient, S3CrtClient}; -use mountpoint_s3_crt::checksums::{crc32, sha1, sha256}; use rand::Rng; use test_case::test_case; @@ -79,6 +78,7 @@ async fn test_put_object_single_empty( object_client_test!(test_put_object_single_empty); #[test_case(None; "no checksum")] +#[test_case(Some(ChecksumAlgorithm::Crc64nvme))] #[test_case(Some(ChecksumAlgorithm::Crc32c))] #[test_case(Some(ChecksumAlgorithm::Crc32))] #[test_case(Some(ChecksumAlgorithm::Sha1))] @@ -98,6 +98,7 @@ async fn test_put_checksums(checksum_algorithm: Option) { rng.fill(&mut contents[..]); let upload_checksum = match checksum_algorithm { + Some(ChecksumAlgorithm::Crc64nvme) => Some(UploadChecksum::Crc64nvme(crc64nvme::checksum(&contents))), Some(ChecksumAlgorithm::Crc32c) => Some(UploadChecksum::Crc32c(crc32c::checksum(&contents))), Some(ChecksumAlgorithm::Crc32) => Some(UploadChecksum::Crc32(crc32::checksum(&contents))), Some(ChecksumAlgorithm::Sha1) => Some(UploadChecksum::Sha1( @@ -117,7 +118,7 @@ async fn test_put_checksums(checksum_algorithm: Option) { .expect("put_object should succeed"); let sdk_client = get_test_sdk_client().await; - let output = sdk_client + let head_output = sdk_client .head_object() .bucket(&bucket) .key(key) @@ -126,14 +127,23 @@ async fn test_put_checksums(checksum_algorithm: Option) { .await .unwrap(); - let output_checksum = Checksum { - checksum_crc32: output.checksum_crc32, - checksum_crc32c: output.checksum_crc32_c, - checksum_sha1: output.checksum_sha1, - checksum_sha256: output.checksum_sha256, + let head_checksum = Checksum { + checksum_crc64nvme: if upload_checksum.is_none() { + // If no checksum was sent with the PutObject request, S3 automatically uses CRC-64NVME. + // We'll ignore it for the test below. + // See https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html. + None + } else { + head_output.checksum_crc64_nvme + }, + checksum_crc32: head_output.checksum_crc32, + checksum_crc32c: head_output.checksum_crc32_c, + checksum_sha1: head_output.checksum_sha1, + checksum_sha256: head_output.checksum_sha256, }; + let checksum: Checksum = upload_checksum.into(); - assert_eq!(output_checksum, checksum); + assert_eq!(head_checksum, checksum); } #[tokio::test] @@ -355,6 +365,7 @@ async fn test_append_fails_if_not_supported() { .put_object() .bucket(&bucket) .key(&key) + .checksum_algorithm(aws_sdk_s3::types::ChecksumAlgorithm::Crc64Nvme) .body(ByteStream::from_static(b"data")) .send() .await @@ -446,6 +457,7 @@ async fn test_append_existing_object() { .put_object() .bucket(&bucket) .key(&key) + .checksum_algorithm(aws_sdk_s3::types::ChecksumAlgorithm::Crc64Nvme) .body(ByteStream::from_static(contents)) .send() .await @@ -457,7 +469,8 @@ async fn test_append_existing_object() { rng.fill(&mut contents[..]); expected_contents.extend_from_slice(&contents); - let params = PutObjectSingleParams::new_for_append(object_size as u64); + let params = PutObjectSingleParams::new_for_append(object_size as u64) + .checksum(Some(UploadChecksum::Crc64nvme(crc64nvme::checksum(&contents)))); let put_object_result = client .put_object_single(&bucket, &key, ¶ms, &contents) .await @@ -493,6 +506,7 @@ async fn test_append_existing_object_if_match() { .put_object() .bucket(&bucket) .key(&key) + .checksum_algorithm(aws_sdk_s3::types::ChecksumAlgorithm::Crc64Nvme) .body(ByteStream::from_static(contents)) .send() .await @@ -505,7 +519,9 @@ async fn test_append_existing_object_if_match() { rng.fill(&mut contents[..]); expected_contents.extend_from_slice(&contents); - let params = PutObjectSingleParams::new_for_append(object_size as u64).if_match(Some(etag.to_owned().into())); + let params = PutObjectSingleParams::new_for_append(object_size as u64) + .checksum(Some(UploadChecksum::Crc64nvme(crc64nvme::checksum(&contents)))) + .if_match(Some(etag.to_owned().into())); let put_object_result = client .put_object_single(&bucket, &key, ¶ms, &contents) .await @@ -538,6 +554,7 @@ async fn test_append_existing_object_if_match_fails() { .put_object() .bucket(&bucket) .key(&key) + .checksum_algorithm(aws_sdk_s3::types::ChecksumAlgorithm::Crc64Nvme) .body(ByteStream::from_static(INITIAL_CONTENT)) .send() .await @@ -549,14 +566,18 @@ async fn test_append_existing_object_if_match_fails() { .put_object() .bucket(&bucket) .key(&key) + .checksum_algorithm(aws_sdk_s3::types::ChecksumAlgorithm::Crc64Nvme) .body(ByteStream::from_static(b"replace")) .send() .await .expect("sdk put_object should succeed"); assert_ne!(etag, replace_result.e_tag().expect("sdk put_object should return etag")); - let params = PutObjectSingleParams::new_for_append(object_size as u64).if_match(Some(etag.to_owned().into())); - let put_object_result = client.put_object_single(&bucket, &key, ¶ms, b"append").await; + let contents = b"append"; + let params = PutObjectSingleParams::new_for_append(object_size as u64) + .checksum(Some(UploadChecksum::Crc64nvme(crc64nvme::checksum(contents)))) + .if_match(Some(etag.to_owned().into())); + let put_object_result = client.put_object_single(&bucket, &key, ¶ms, contents).await; let error = put_object_result.expect_err("put_object_single with the old etag should fail"); assert!(matches!( @@ -581,13 +602,15 @@ async fn test_append_existing_object_with_empty() { .put_object() .bucket(&bucket) .key(&key) + .checksum_algorithm(aws_sdk_s3::types::ChecksumAlgorithm::Crc64Nvme) .body(ByteStream::from_static(contents)) .send() .await .expect("sdk put_object should succeed"); let contents = vec![0u8; 0]; - let params = PutObjectSingleParams::new_for_append(object_size as u64); + let params = PutObjectSingleParams::new_for_append(object_size as u64) + .checksum(Some(UploadChecksum::Crc64nvme(crc64nvme::checksum(&contents)))); let result = client.put_object_single(&bucket, &key, ¶ms, &contents).await; assert!(matches!( result, @@ -611,13 +634,15 @@ async fn test_append_empty_object_with_empty() { .put_object() .bucket(&bucket) .key(&key) + .checksum_algorithm(aws_sdk_s3::types::ChecksumAlgorithm::Crc64Nvme) .body(ByteStream::from_static(contents)) .send() .await .expect("sdk put_object should succeed"); let contents = vec![0u8; 0]; - let params = PutObjectSingleParams::new_for_append(object_size as u64); + let params = PutObjectSingleParams::new_for_append(object_size as u64) + .checksum(Some(UploadChecksum::Crc64nvme(crc64nvme::checksum(&contents)))); let result = client.put_object_single(&bucket, &key, ¶ms, &contents).await; assert!(matches!( result, @@ -661,6 +686,7 @@ async fn test_append_invalid_offset() { .put_object() .bucket(&bucket) .key(&key) + .checksum_algorithm(aws_sdk_s3::types::ChecksumAlgorithm::Crc64Nvme) .body(ByteStream::from_static(contents)) .send() .await @@ -671,7 +697,8 @@ async fn test_append_invalid_offset() { let mut contents = vec![0u8; 32]; rng.fill(&mut contents[..]); - let params = PutObjectSingleParams::new_for_append(object_size as u64 + 1); + let params = PutObjectSingleParams::new_for_append(object_size as u64 + 1) + .checksum(Some(UploadChecksum::Crc64nvme(crc64nvme::checksum(&contents)))); let result = client.put_object_single(&bucket, &key, ¶ms, &contents).await; assert!(matches!( result, @@ -758,6 +785,7 @@ async fn test_append_with_invalid_checksum() { )); } +#[test_case(aws_sdk_s3::types::ChecksumAlgorithm::Crc64Nvme)] #[test_case(aws_sdk_s3::types::ChecksumAlgorithm::Crc32C)] #[test_case(aws_sdk_s3::types::ChecksumAlgorithm::Crc32)] #[test_case(aws_sdk_s3::types::ChecksumAlgorithm::Sha1)] @@ -765,8 +793,6 @@ async fn test_append_with_invalid_checksum() { #[tokio::test] #[cfg(feature = "s3express_tests")] async fn test_append_with_matching_checksum(checksum_algorithm: aws_sdk_s3::types::ChecksumAlgorithm) { - use mountpoint_s3_client::checksums::{crc32, crc32c, sha1, sha256}; - let (bucket, prefix) = get_test_bucket_and_prefix("test_append_with_matching_checksum"); let client = get_test_client(); let key = format!("{prefix}hello"); @@ -791,6 +817,7 @@ async fn test_append_with_matching_checksum(checksum_algorithm: aws_sdk_s3::type let mut contents = vec![0u8; 32]; rng.fill(&mut contents[..]); let checksum = match checksum_algorithm { + aws_sdk_s3::types::ChecksumAlgorithm::Crc64Nvme => UploadChecksum::Crc64nvme(crc64nvme::checksum(&contents)), aws_sdk_s3::types::ChecksumAlgorithm::Crc32C => UploadChecksum::Crc32c(crc32c::checksum(&contents)), aws_sdk_s3::types::ChecksumAlgorithm::Crc32 => UploadChecksum::Crc32(crc32::checksum(&contents)), aws_sdk_s3::types::ChecksumAlgorithm::Sha1 => UploadChecksum::Sha1(sha1::checksum(&contents).unwrap()), diff --git a/mountpoint-s3-crt/benches/checksums.rs b/mountpoint-s3-crt/benches/checksums.rs index f46ad1031..98684a89f 100644 --- a/mountpoint-s3-crt/benches/checksums.rs +++ b/mountpoint-s3-crt/benches/checksums.rs @@ -1,7 +1,7 @@ //! Benchmarks for the CRT checksums library use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; -use mountpoint_s3_crt::checksums::{crc32, crc32c, crc64, sha1, sha256}; +use mountpoint_s3_crt::checksums::{crc32, crc32c, crc64nvme, sha1, sha256}; use rand::rngs::SmallRng; use rand::{Rng, SeedableRng}; @@ -41,8 +41,8 @@ fn crc32c(c: &mut Criterion) { benchmark_hasher(c, crc32c::checksum, "crc32c"); } -fn crc64(c: &mut Criterion) { - benchmark_hasher(c, crc64::checksum, "crc64"); +fn crc64nvme(c: &mut Criterion) { + benchmark_hasher(c, crc64nvme::checksum, "crc64"); } fn sha1(c: &mut Criterion) { @@ -53,5 +53,5 @@ fn sha256(c: &mut Criterion) { benchmark_hasher(c, sha256::checksum, "sha256"); } -criterion_group!(checksum_benches, crc32, crc32c, crc64, sha1, sha256); +criterion_group!(checksum_benches, crc32, crc32c, crc64nvme, sha1, sha256); criterion_main!(checksum_benches); diff --git a/mountpoint-s3-crt/src/checksums/crc64.rs b/mountpoint-s3-crt/src/checksums/crc64nvme.rs similarity index 55% rename from mountpoint-s3-crt/src/checksums/crc64.rs rename to mountpoint-s3-crt/src/checksums/crc64nvme.rs index 82c5629ec..bac0be472 100644 --- a/mountpoint-s3-crt/src/checksums/crc64.rs +++ b/mountpoint-s3-crt/src/checksums/crc64nvme.rs @@ -2,52 +2,52 @@ use mountpoint_s3_crt_sys::aws_checksums_crc64nvme; /// CRC64-NVME (aka. CRC64-Rocksoft) checksum #[derive(Debug, Clone, PartialEq, Eq, Copy)] -pub struct Crc64(u64); +pub struct Crc64nvme(u64); -impl Crc64 { - /// Create a new CRC64 checksum with the given value. +impl Crc64nvme { + /// Create a new CRC64-NVME checksum with the given value. pub fn new(value: u64) -> Self { Self(value) } - /// The CRC64 checksum value. + /// The CRC64-NVME checksum value. pub fn value(&self) -> u64 { self.0 } } -/// Computes the CRC64 checksum of a byte slice. +/// Computes the CRC64-NVME checksum of a byte slice. /// -/// Use [Crc64Hasher] for more advanced use-cases. -pub fn checksum(buf: &[u8]) -> Crc64 { - let mut hasher = Crc64Hasher::new(); +/// Use [Crc64nvmeHasher] for more advanced use-cases. +pub fn checksum(buf: &[u8]) -> Crc64nvme { + let mut hasher = Crc64nvmeHasher::new(); hasher.update(buf); hasher.finalize() } -/// CRC64 Hasher +/// CRC64-NVME Hasher #[derive(Debug, Clone)] -pub struct Crc64Hasher { - state: Crc64, +pub struct Crc64nvmeHasher { + state: Crc64nvme, } -impl Crc64Hasher { - /// Create a new CRC64 Hasher. +impl Crc64nvmeHasher { + /// Create a new CRC64-NVME Hasher. pub fn new() -> Self { - Self { state: Crc64(0) } + Self { state: Crc64nvme(0) } } /// Update the hash state with the given bytes slice. pub fn update(&mut self, buf: &[u8]) { - self.state = Crc64(Self::crc64(buf, self.state.0)); + self.state = Crc64nvme(Self::crc64(buf, self.state.0)); } - /// Finalize the hash state and return the computed CRC64 checksum value. - pub fn finalize(self) -> Crc64 { + /// Finalize the hash state and return the computed CRC64-NVME checksum value. + pub fn finalize(self) -> Crc64nvme { self.state } - /// Compute CRC64 checksum of the data in the given bytes slice, append to the previous checksum. + /// Compute CRC64-NVME checksum of the data in the given bytes slice, append to the previous checksum. /// /// The underlying CRT function requires the buffer's length to be type [::libc::c_int], so this function cannot take /// any buffer that is bigger than [::libc::c_int::MAX] as an input. @@ -64,7 +64,7 @@ impl Crc64Hasher { } } -impl Default for Crc64Hasher { +impl Default for Crc64nvmeHasher { fn default() -> Self { Self::new() } @@ -72,21 +72,21 @@ impl Default for Crc64Hasher { #[cfg(test)] mod tests { - use crate::checksums::crc64::{self, Crc64}; + use crate::checksums::crc64nvme::{self, Crc64nvme}; #[test] - fn crc64_simple() { + fn crc64nvme_simple() { let buf: &[u8] = b"123456789"; - let crc = crc64::checksum(buf); - assert_eq!(crc, Crc64(0xAE8B14860A799888)); + let crc = crc64nvme::checksum(buf); + assert_eq!(crc, Crc64nvme(0xAE8B14860A799888)); } #[test] - fn crc64_append() { - let mut hasher = crc64::Crc64Hasher::new(); + fn crc64nvme_append() { + let mut hasher = crc64nvme::Crc64nvmeHasher::new(); hasher.update(b"1234"); hasher.update(b"56789"); let crc = hasher.finalize(); - assert_eq!(crc, Crc64(0xAE8B14860A799888)); + assert_eq!(crc, Crc64nvme(0xAE8B14860A799888)); } } diff --git a/mountpoint-s3-crt/src/checksums/mod.rs b/mountpoint-s3-crt/src/checksums/mod.rs index 3babeed60..ae3f65820 100644 --- a/mountpoint-s3-crt/src/checksums/mod.rs +++ b/mountpoint-s3-crt/src/checksums/mod.rs @@ -6,8 +6,8 @@ pub mod crc32; /// CRC32C checksums pub mod crc32c; -/// CRC64 checksums -pub mod crc64; +/// CRC64NVME checksums +pub mod crc64nvme; /// SHA1 checksums pub mod sha1; diff --git a/mountpoint-s3-crt/src/s3/client.rs b/mountpoint-s3-crt/src/s3/client.rs index be745d396..3f58e187d 100644 --- a/mountpoint-s3-crt/src/s3/client.rs +++ b/mountpoint-s3-crt/src/s3/client.rs @@ -1463,6 +1463,8 @@ impl ChecksumConfig { #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] #[non_exhaustive] pub enum ChecksumAlgorithm { + /// Crc64nvme checksum. + Crc64nvme, /// Crc32c checksum. Crc32c, /// Crc32 checksum. @@ -1482,6 +1484,7 @@ impl ChecksumAlgorithm { fn from_aws_s3_checksum_algorithm(algorithm: aws_s3_checksum_algorithm) -> Option { match algorithm { aws_s3_checksum_algorithm::AWS_SCA_NONE => None, + aws_s3_checksum_algorithm::AWS_SCA_CRC64NVME => Some(ChecksumAlgorithm::Crc64nvme), aws_s3_checksum_algorithm::AWS_SCA_CRC32C => Some(ChecksumAlgorithm::Crc32c), aws_s3_checksum_algorithm::AWS_SCA_CRC32 => Some(ChecksumAlgorithm::Crc32), aws_s3_checksum_algorithm::AWS_SCA_SHA1 => Some(ChecksumAlgorithm::Sha1), @@ -1494,6 +1497,7 @@ impl ChecksumAlgorithm { impl Display for ChecksumAlgorithm { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { + ChecksumAlgorithm::Crc64nvme => f.write_str("CRC64NVME"), ChecksumAlgorithm::Crc32c => f.write_str("CRC32C"), ChecksumAlgorithm::Crc32 => f.write_str("CRC32"), ChecksumAlgorithm::Sha1 => f.write_str("SHA1"), diff --git a/mountpoint-s3/src/upload/hasher.rs b/mountpoint-s3/src/upload/hasher.rs index 7addcf289..a74107790 100644 --- a/mountpoint-s3/src/upload/hasher.rs +++ b/mountpoint-s3/src/upload/hasher.rs @@ -1,4 +1,4 @@ -use mountpoint_s3_client::checksums::{crc32, crc32c, sha1, sha256}; +use mountpoint_s3_client::checksums::{crc32, crc32c, crc64nvme, sha1, sha256}; use mountpoint_s3_client::types::{ChecksumAlgorithm, UploadChecksum}; use mountpoint_s3_crt::common::allocator::Allocator; use thiserror::Error; @@ -7,6 +7,7 @@ use thiserror::Error; pub enum ChecksumHasher { #[default] None, + Crc64nvme(crc64nvme::Crc64nvmeHasher), Crc32(crc32::Hasher), Crc32c(crc32c::Hasher), Sha1(sha1::Sha1Hasher), @@ -16,6 +17,7 @@ pub enum ChecksumHasher { impl ChecksumHasher { pub fn new(checksum_algorithm: &Option) -> Result { match checksum_algorithm { + Some(ChecksumAlgorithm::Crc64nvme) => Ok(Self::Crc64nvme(crc64nvme::Crc64nvmeHasher::new())), Some(ChecksumAlgorithm::Crc32) => Ok(Self::Crc32(crc32::Hasher::new())), Some(ChecksumAlgorithm::Crc32c) => Ok(Self::Crc32c(crc32c::Hasher::new())), Some(ChecksumAlgorithm::Sha1) => Ok(Self::Sha1(sha1::Sha1Hasher::new(&Allocator::default())?)), @@ -28,6 +30,7 @@ impl ChecksumHasher { pub fn update(&mut self, data: &[u8]) -> Result<(), ChecksumHasherError> { match self { ChecksumHasher::None => {} + ChecksumHasher::Crc64nvme(hasher) => hasher.update(data), ChecksumHasher::Crc32(hasher) => hasher.update(data), ChecksumHasher::Crc32c(hasher) => hasher.update(data), ChecksumHasher::Sha1(hasher) => hasher.update(data)?, @@ -39,6 +42,7 @@ impl ChecksumHasher { pub fn finalize(self) -> Result, ChecksumHasherError> { match self { ChecksumHasher::None => Ok(None), + ChecksumHasher::Crc64nvme(hasher) => Ok(Some(UploadChecksum::Crc64nvme(hasher.finalize()))), ChecksumHasher::Crc32(hasher) => Ok(Some(UploadChecksum::Crc32(hasher.finalize()))), ChecksumHasher::Crc32c(hasher) => Ok(Some(UploadChecksum::Crc32c(hasher.finalize()))), ChecksumHasher::Sha1(hasher) => Ok(Some(UploadChecksum::Sha1(hasher.finalize(&Allocator::default())?))), diff --git a/mountpoint-s3/src/upload/incremental.rs b/mountpoint-s3/src/upload/incremental.rs index 581963892..29a7abf70 100644 --- a/mountpoint-s3/src/upload/incremental.rs +++ b/mountpoint-s3/src/upload/incremental.rs @@ -488,6 +488,7 @@ mod tests { #[test_case(Some(MockObject::constant(0xab, 20, ETag::for_tests()).with_computed_checksums(&[ChecksumAlgorithm::Crc32c])))] #[test_case(Some(MockObject::from([]).with_computed_checksums(&[ChecksumAlgorithm::Crc32c])))] #[test_case(Some(MockObject::from([0xbb; 128])))] + #[test_case(Some(MockObject::from([0xbb; 128]).with_computed_checksums(&[ChecksumAlgorithm::Crc64nvme])))] #[test_case(Some(MockObject::from([0xbb; 128]).with_computed_checksums(&[ChecksumAlgorithm::Crc32c])))] #[test_case(Some(MockObject::from([0xbb; 128]).with_computed_checksums(&[ChecksumAlgorithm::Crc32])))] #[test_case(Some(MockObject::from([0xbb; 128]).with_computed_checksums(&[ChecksumAlgorithm::Sha1])))] @@ -562,6 +563,7 @@ mod tests { #[test_case(Some(MockObject::constant(0xab, 20, ETag::for_tests()).with_computed_checksums(&[ChecksumAlgorithm::Crc32c])))] #[test_case(Some(MockObject::from([]).with_computed_checksums(&[ChecksumAlgorithm::Crc32c])))] #[test_case(Some(MockObject::from([0xbb; 128])))] + #[test_case(Some(MockObject::from([0xbb; 128]).with_computed_checksums(&[ChecksumAlgorithm::Crc64nvme])))] #[test_case(Some(MockObject::from([0xbb; 128]).with_computed_checksums(&[ChecksumAlgorithm::Crc32c])))] #[test_case(Some(MockObject::from([0xbb; 128]).with_computed_checksums(&[ChecksumAlgorithm::Crc32])))] #[test_case(Some(MockObject::from([0xbb; 128]).with_computed_checksums(&[ChecksumAlgorithm::Sha1])))] @@ -627,11 +629,13 @@ mod tests { } #[test_case(None, None)] + #[test_case(None, Some(ChecksumAlgorithm::Crc64nvme))] #[test_case(None, Some(ChecksumAlgorithm::Crc32c))] #[test_case(None, Some(ChecksumAlgorithm::Crc32))] #[test_case(None, Some(ChecksumAlgorithm::Sha1))] #[test_case(None, Some(ChecksumAlgorithm::Sha256))] #[test_case(Some(&[]), Some(ChecksumAlgorithm::Crc32c))] + #[test_case(Some(&[ChecksumAlgorithm::Crc64nvme]), None)] #[test_case(Some(&[ChecksumAlgorithm::Crc32c]), None)] #[test_case(Some(&[ChecksumAlgorithm::Crc32]), None)] #[test_case(Some(&[ChecksumAlgorithm::Sha1]), None)] diff --git a/mountpoint-s3/tests/common/fuse.rs b/mountpoint-s3/tests/common/fuse.rs index fa779968d..9424c3af2 100644 --- a/mountpoint-s3/tests/common/fuse.rs +++ b/mountpoint-s3/tests/common/fuse.rs @@ -600,6 +600,7 @@ pub mod s3_session { .send(), )?; let object_checksum = attrs.checksum.map(|checksum| Checksum { + checksum_crc64nvme: checksum.checksum_crc64_nvme, checksum_crc32: checksum.checksum_crc32, checksum_crc32c: checksum.checksum_crc32_c, checksum_sha1: checksum.checksum_sha1, @@ -615,6 +616,7 @@ pub mod s3_session { .into_iter() .map(|part| { Some(Checksum { + checksum_crc64nvme: part.checksum_crc64_nvme, checksum_crc32: part.checksum_crc32, checksum_crc32c: part.checksum_crc32_c, checksum_sha1: part.checksum_sha1,