Add support for CRC64NVME

Signed-off-by: Alessandro Passaro <alexpax@amazon.co.uk>

mountpoint-s3-client/src/checksums.rs

@@ -1,13 +1,19 @@
 //! Provides base64 encoding/decoding for various checksums.
 pub use mountpoint_s3_crt::checksums::crc32::{self, Crc32};
 pub use mountpoint_s3_crt::checksums::crc32c::{self, Crc32c};
+pub use mountpoint_s3_crt::checksums::crc64nvme::{self, Crc64nvme};
 pub use mountpoint_s3_crt::checksums::sha1::{self, Sha1};
 pub use mountpoint_s3_crt::checksums::sha256::{self, Sha256};
 
 use base64ct::Base64;
 use base64ct::Encoding;
 use thiserror::Error;
 
+/// The base64 encoding for this CRC64-NVME checksum value.
+pub fn crc64nvme_to_base64(checksum: &Crc64nvme) -> String {
+    Base64::encode_string(&checksum.value().to_be_bytes())
+}
+
 /// The base64 encoding for this CRC32C checksum value.
 pub fn crc32c_to_base64(checksum: &Crc32c) -> String {
     Base64::encode_string(&checksum.value().to_be_bytes())

mountpoint-s3-client/src/mock_client.rs

@@ -22,7 +22,7 @@ use time::OffsetDateTime;
 use tracing::trace;
 
 use crate::checksums::{
-    crc32, crc32_to_base64, crc32c, crc32c_to_base64, sha1, sha1_to_base64, sha256, sha256_to_base64,
+    crc32, crc32_to_base64, crc32c, crc32c_to_base64, crc64nvme, crc64nvme_to_base64, sha1, sha1_to_base64, sha256, sha256_to_base64
 };
 use crate::error_metadata::{ClientErrorMetadata, ProvideErrorMetadata};
 use crate::object_client::{
@@ -631,6 +631,10 @@ fn compute_checksum(content: &[u8], algorithms: &[ChecksumAlgorithm]) -> Checksu
     let mut checksum = Checksum::empty();
     for algorithm in algorithms {
         match algorithm {
+            ChecksumAlgorithm::Crc64nvme => {
+                let crc64nvme = crc64nvme::checksum(content);
+                checksum.checksum_crc64nvme = Some(crc64nvme_to_base64(&crc64nvme));
+            }
             ChecksumAlgorithm::Crc32 => {
                 let crc32 = crc32::checksum(content);
                 checksum.checksum_crc32 = Some(crc32_to_base64(&crc32));
@@ -1031,6 +1035,7 @@ impl ObjectClient for MockClient {
                                         .enumerate()
                                         .map(|(i, part)| ObjectPart {
                                             checksum: Some(Checksum {
+                                                checksum_crc64nvme: None,
                                                 checksum_crc32: None,
                                                 checksum_crc32c: part.checksum.clone(),
                                                 checksum_sha1: None,
@@ -1848,6 +1853,7 @@ mod tests {
         client.add_object("a.txt", MockObject::constant(0u8, 5, ETag::for_tests()));
         let mut object_b = MockObject::constant(1u8, 5, ETag::for_tests());
         object_b.set_checksum(Checksum {
+            checksum_crc64nvme: None,
             checksum_crc32: None,
             checksum_crc32c: None,
             checksum_sha1: Some(String::from("QwzjTQIHJO11oZbfwq1nx3dy0Wk=")),
@@ -2201,11 +2207,13 @@ mod tests {
             // We trust that other tests will cover checksum correctness,
             // so let's just check the right checksums are set.
             let Checksum {
+                checksum_crc64nvme,
                 checksum_crc32,
                 checksum_crc32c,
                 checksum_sha1,
                 checksum_sha256,
             } = attrs.checksum.expect("object checksum should be present");
+            assert!(checksum_crc64nvme.is_none(), "CRC64NVME should not be set");
             assert!(checksum_crc32.is_none(), "CRC32 should not be set");
             assert!(checksum_crc32c.is_some(), "CRC32C should be set");
             assert!(checksum_sha1.is_none(), "SHA1 should not be set");

mountpoint-s3-client/src/object_client.rs

@@ -10,7 +10,9 @@ use std::collections::HashMap;
 use thiserror::Error;
 use time::OffsetDateTime;
 
-use crate::checksums::{self, crc32_to_base64, crc32c_to_base64, sha1_to_base64, sha256_to_base64};
+use crate::checksums::{
+    self, crc32_to_base64, crc32c_to_base64, crc64nvme_to_base64, sha1_to_base64, sha256_to_base64,
+};
 use crate::error_metadata::{ClientErrorMetadata, ProvideErrorMetadata};
 
 mod etag;
@@ -567,6 +569,7 @@ impl PutObjectSingleParams {
 #[derive(Debug, Clone)]
 #[non_exhaustive]
 pub enum UploadChecksum {
+    Crc64nvme(checksums::Crc64nvme),
     Crc32c(checksums::Crc32c),
     Crc32(checksums::Crc32),
     Sha1(checksums::Sha1),
@@ -577,6 +580,7 @@ impl UploadChecksum {
     /// The checksum algorithm used to compute this checksum.
     pub fn checksum_algorithm(&self) -> ChecksumAlgorithm {
         match self {
+            UploadChecksum::Crc64nvme(_) => ChecksumAlgorithm::Crc64nvme,
             UploadChecksum::Crc32c(_) => ChecksumAlgorithm::Crc32c,
             UploadChecksum::Crc32(_) => ChecksumAlgorithm::Crc32,
             UploadChecksum::Sha1(_) => ChecksumAlgorithm::Sha1,
@@ -807,6 +811,9 @@ impl fmt::Display for ObjectAttribute {
 /// S3 API Reference* for more details.
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub struct Checksum {
+    /// Base64-encoded, 64-bit CRC64NVME checksum of the object
+    pub checksum_crc64nvme: Option<String>,
+
     /// Base64-encoded, 32-bit CRC32 checksum of the object
     pub checksum_crc32: Option<String>,
 
@@ -824,6 +831,7 @@ impl Checksum {
     /// Construct an empty [Checksum]
     pub fn empty() -> Self {
         Self {
+            checksum_crc64nvme: None,
             checksum_crc32: None,
             checksum_crc32c: None,
             checksum_sha1: None,
@@ -838,12 +846,16 @@ impl Checksum {
 
         // Pattern match forces us to accomodate any new fields when added.
         let Self {
+            checksum_crc64nvme,
             checksum_crc32,
             checksum_crc32c,
             checksum_sha1,
             checksum_sha256,
         } = &self;
 
+        if checksum_crc64nvme.is_some() {
+            algorithms.push(ChecksumAlgorithm::Crc64nvme);
+        }
         if checksum_crc32.is_some() {
             algorithms.push(ChecksumAlgorithm::Crc32);
         }
@@ -865,6 +877,7 @@ impl From<Option<UploadChecksum>> for Checksum {
     fn from(value: Option<UploadChecksum>) -> Self {
         let mut checksum = Checksum::empty();
         match value.as_ref() {
+            Some(UploadChecksum::Crc64nvme(crc64)) => checksum.checksum_crc64nvme = Some(crc64nvme_to_base64(crc64)),
             Some(UploadChecksum::Crc32c(crc32c)) => checksum.checksum_crc32c = Some(crc32c_to_base64(crc32c)),
             Some(UploadChecksum::Crc32(crc32)) => checksum.checksum_crc32 = Some(crc32_to_base64(crc32)),
             Some(UploadChecksum::Sha1(sha1)) => checksum.checksum_sha1 = Some(sha1_to_base64(sha1)),
@@ -923,6 +936,7 @@ mod tests {
     #[test]
     fn test_checksum_algorithm_one_set() {
         let checksum = Checksum {
+            checksum_crc64nvme: None,
             checksum_crc32: None,
             checksum_crc32c: None,
             checksum_sha1: Some("checksum_sha1".to_string()),
@@ -934,6 +948,7 @@ mod tests {
     #[test]
     fn test_checksum_algorithm_none_set() {
         let checksum = Checksum {
+            checksum_crc64nvme: None,
             checksum_crc32: None,
             checksum_crc32c: None,
             checksum_sha1: None,
@@ -946,6 +961,7 @@ mod tests {
     fn test_checksum_algorithm_many_set() {
         // Amazon S3 doesn't support more than one algorithm today, but just in case... let's show we don't panic.
         let checksum = Checksum {
+            checksum_crc64nvme: None,
             checksum_crc32: None,
             checksum_crc32c: Some("checksum_crc32c".to_string()),
             checksum_sha1: Some("checksum_sha1".to_string()),

mountpoint-s3-client/src/s3_crt_client.rs

@@ -38,7 +38,7 @@ use pin_project::{pin_project, pinned_drop};
 use thiserror::Error;
 use tracing::{debug, error, trace, Span};
 
-use crate::checksums::{crc32_to_base64, crc32c_to_base64, sha1_to_base64, sha256_to_base64};
+use crate::checksums::{crc32_to_base64, crc32c_to_base64, crc64nvme_to_base64, sha1_to_base64, sha256_to_base64};
 use crate::endpoint_config::EndpointError;
 use crate::endpoint_config::{self, EndpointConfig};
 use crate::error_metadata::{ClientErrorMetadata, ProvideErrorMetadata};
@@ -956,6 +956,7 @@ impl<'a> S3Message<'a> {
         checksum: &UploadChecksum,
     ) -> Result<(), mountpoint_s3_crt::common::error::Error> {
         let header = match checksum {
+            UploadChecksum::Crc64nvme(crc64) => Header::new("x-amz-checksum-crc64nvme", crc64nvme_to_base64(crc64)),
             UploadChecksum::Crc32c(crc32c) => Header::new("x-amz-checksum-crc32c", crc32c_to_base64(crc32c)),
             UploadChecksum::Crc32(crc32) => Header::new("x-amz-checksum-crc32", crc32_to_base64(crc32)),
             UploadChecksum::Sha1(sha1) => Header::new("x-amz-checksum-sha1", sha1_to_base64(sha1)),
@@ -1157,8 +1158,10 @@ fn parse_checksum(headers: &Headers) -> Result<Checksum, HeadersError> {
     let checksum_crc32c = headers.get_as_optional_string("x-amz-checksum-crc32c")?;
     let checksum_sha1 = headers.get_as_optional_string("x-amz-checksum-sha1")?;
     let checksum_sha256 = headers.get_as_optional_string("x-amz-checksum-sha256")?;
+    let checksum_crc64nvme = headers.get_as_optional_string("x-amz-checksum-crc64nvme")?;
 
     Ok(Checksum {
+        checksum_crc64nvme,
         checksum_crc32,
         checksum_crc32c,
         checksum_sha1,

mountpoint-s3-client/src/s3_crt_client/get_object_attributes.rs

@@ -86,8 +86,10 @@ impl GetObjectAttributesResult {
         let checksum_crc32c = get_field_or_none(element, "ChecksumCRC32C")?;
         let checksum_sha1 = get_field_or_none(element, "ChecksumSHA1")?;
         let checksum_sha256 = get_field_or_none(element, "ChecksumSHA256")?;
+        let checksum_crc64nvme = get_field_or_none(element, "ChecksumCRC64NVME")?;
 
         Ok(Checksum {
+            checksum_crc64nvme,
             checksum_crc32,
             checksum_crc32c,
             checksum_sha1,