Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Change blake2 hash to use blake2b instead of blake2s #192

Merged
merged 8 commits into from
May 22, 2023
68 changes: 48 additions & 20 deletions crates/rattler_repodata_gateway/src/fetch/cache/mod.rs
Original file line number Diff line number Diff line change
@@ -1,10 +1,15 @@
mod cache_headers;

use blake2::digest::consts::U32;
use blake2::Blake2b;
pub use cache_headers::CacheHeaders;
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use std::{fs::File, io::Read, path::Path, str::FromStr, time::SystemTime};
use url::Url;

/// Custom blake2b type
pub type Blake2b256 = Blake2b<U32>;

/// Representation of the `.state.json` file alongside a `repodata.json` file.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RepoDataState {
Expand Down Expand Up @@ -36,7 +41,7 @@ pub struct RepoDataState {
deserialize_with = "deserialize_blake2_hash",
serialize_with = "serialize_blake2_hash"
)]
pub blake2_hash: Option<blake2::digest::Output<blake2::Blake2s256>>,
pub blake2_hash: Option<blake2::digest::Output<Blake2b256>>,

/// Whether or not zst is available for the subdirectory
pub has_zst: Option<Expiring<bool>>,
Expand Down Expand Up @@ -119,22 +124,22 @@ fn duration_to_nanos<S: Serializer>(time: &SystemTime, s: S) -> Result<S::Ok, S:

fn deserialize_blake2_hash<'de, D>(
deserializer: D,
) -> Result<Option<blake2::digest::Output<blake2::Blake2s256>>, D::Error>
) -> Result<Option<blake2::digest::Output<Blake2b256>>, D::Error>
where
D: Deserializer<'de>,
{
use serde::de::Error;
match Option::<&'de str>::deserialize(deserializer)? {
Some(str) => Ok(Some(
rattler_digest::parse_digest_from_hex::<blake2::Blake2s256>(str)
rattler_digest::parse_digest_from_hex::<Blake2b256>(str)
.ok_or_else(|| D::Error::custom("failed to parse blake2 hash"))?,
)),
None => Ok(None),
}
}

fn serialize_blake2_hash<S: Serializer>(
time: &Option<blake2::digest::Output<blake2::Blake2s256>>,
time: &Option<blake2::digest::Output<Blake2b256>>,
s: S,
) -> Result<S::Ok, S::Error> {
match time.as_ref() {
Expand All @@ -148,22 +153,45 @@ mod test {
use super::RepoDataState;
use std::str::FromStr;

const JSON_STATE_ONE: &str = r#"{
"cache_control": "public, max-age=1200",
"etag": "\"bec332621e00fc4ad87ba185171bcf46\"",
"has_zst": {
"last_checked": "2023-02-13T14:08:50Z",
"value": true
},
"mod": "Mon, 13 Feb 2023 13:49:56 GMT",
"mtime_ns": 1676297333020928000,
"size": 156627374,
"url": "https://conda.anaconda.org/conda-forge/win-64/repodata.json.zst"
}"#;

const JSON_STATE_TWO: &str = r#"{
"url": "https://repo.anaconda.com/pkgs/main/osx-64/repodata.json.zst",
"etag": "W/\"2f8b1ff101d75e40adf28c3fcbcd330b\"",
"mod": "Thu, 18 May 2023 13:28:44 GMT",
"cache_control": "public, max-age=30",
"mtime_ns": 1684418349941482000,
"size": 38001429,
"blake2_hash": "a1bb42ccd11d5610189380b8b0a71ca0fa7e3273ff6235ae1d543606041eb3bd",
"has_zst": {
"value": true,
"last_checked": "2023-05-18T13:59:07.112638Z"
},
"has_bz2": {
"value": true,
"last_checked": "2023-05-18T13:59:07.112638Z"
},
"has_jlap": null
}"#;

#[test]
pub fn test_parse_repo_data_state_one() {
insta::assert_yaml_snapshot!(RepoDataState::from_str(JSON_STATE_ONE).unwrap());
}

#[test]
pub fn test_parse_repo_data_state() {
insta::assert_yaml_snapshot!(RepoDataState::from_str(
r#"{
"cache_control": "public, max-age=1200",
"etag": "\"bec332621e00fc4ad87ba185171bcf46\"",
"has_zst": {
"last_checked": "2023-02-13T14:08:50Z",
"value": true
},
"mod": "Mon, 13 Feb 2023 13:49:56 GMT",
"mtime_ns": 1676297333020928000,
"size": 156627374,
"url": "https://conda.anaconda.org/conda-forge/win-64/repodata.json.zst"
}"#,
)
.unwrap());
pub fn test_parse_repo_data_state_two() {
insta::assert_yaml_snapshot!(RepoDataState::from_str(JSON_STATE_TWO).unwrap())
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
---
source: crates/rattler_repodata_gateway/src/fetch/cache/mod.rs
expression: "RepoDataState::from_str(JSON_STATE_ONE).unwrap()"
---
url: "https://conda.anaconda.org/conda-forge/win-64/repodata.json.zst"
etag: "\"bec332621e00fc4ad87ba185171bcf46\""
mod: "Mon, 13 Feb 2023 13:49:56 GMT"
cache_control: "public, max-age=1200"
mtime_ns: 1676297333020928000
size: 156627374
has_zst:
value: true
last_checked: "2023-02-13T14:08:50Z"
has_bz2: ~
has_jlap: ~

Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
---
source: crates/rattler_repodata_gateway/src/fetch/cache/mod.rs
expression: "RepoDataState::from_str(JSON_STATE_TWO).unwrap()"
---
url: "https://repo.anaconda.com/pkgs/main/osx-64/repodata.json.zst"
etag: "W/\"2f8b1ff101d75e40adf28c3fcbcd330b\""
mod: "Thu, 18 May 2023 13:28:44 GMT"
cache_control: "public, max-age=30"
mtime_ns: 1684418349941482000
size: 38001429
blake2_hash: a1bb42ccd11d5610189380b8b0a71ca0fa7e3273ff6235ae1d543606041eb3bd
has_zst:
value: true
last_checked: "2023-05-18T13:59:07.112638Z"
has_bz2:
value: true
last_checked: "2023-05-18T13:59:07.112638Z"
has_jlap: ~

6 changes: 3 additions & 3 deletions crates/rattler_repodata_gateway/src/fetch/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -510,7 +510,7 @@ async fn stream_and_decode_to_file(
content_encoding: Encoding,
temp_dir: &Path,
mut progress: Option<Box<dyn FnMut(DownloadProgress) + Send>>,
) -> Result<(NamedTempFile, blake2::digest::Output<blake2::Blake2s256>), FetchRepoDataError> {
) -> Result<(NamedTempFile, blake2::digest::Output<cache::Blake2b256>), FetchRepoDataError> {
// Determine the length of the response in bytes and notify the listener that a download is
// starting. The response may be compressed. Decompression happens below.
let content_size = response.content_length();
Expand Down Expand Up @@ -566,7 +566,7 @@ async fn stream_and_decode_to_file(
// Clone the file handle and create a hashing writer so we can compute a hash while the content
// is being written to disk.
let file = tokio::fs::File::from_std(temp_file.as_file().try_clone().unwrap());
let mut hashing_file_writer = HashingWriter::<_, blake2::Blake2s256>::new(file);
let mut hashing_file_writer = HashingWriter::<_, cache::Blake2b256>::new(file);

// Decode, hash and write the data to the file.
let bytes = tokio::io::copy(&mut decoded_repo_data_json_bytes, &mut hashing_file_writer)
Expand Down Expand Up @@ -1006,7 +1006,7 @@ mod test {

assert_eq!(
result.cache_state.blake2_hash.unwrap()[..],
hex!("791749939c9d6e26801bbcd525b908da15d42d3249f01efaca1ed1133f38bb87")[..]
hex!("a1861e448e4a62b88dce47c95351bfbe7fc22451a73f89a09d782492540e0675")[..]
);
assert_eq!(
std::fs::read_to_string(result.repo_data_json_path).unwrap(),
Expand Down