From 300891fd734f0da2bf5e3b50d67c213d3daeb885 Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Tue, 14 Jan 2025 19:06:04 +0100 Subject: [PATCH 01/24] refactor: Improve AuthenticationStorage --- crates/rattler-bin/src/commands/create.rs | 7 +- crates/rattler_networking/Cargo.toml | 1 - .../src/authentication_middleware.rs | 22 ++++-- .../authentication_storage/backends/file.rs | 68 +++++-------------- .../src/authentication_storage/storage.rs | 56 +++++++-------- .../src/fetch/jlap/mod.rs | 2 +- .../rattler_repodata_gateway/src/fetch/mod.rs | 2 +- 7 files changed, 64 insertions(+), 94 deletions(-) diff --git a/crates/rattler-bin/src/commands/create.rs b/crates/rattler-bin/src/commands/create.rs index 73bad476a..ca42863b9 100644 --- a/crates/rattler-bin/src/commands/create.rs +++ b/crates/rattler-bin/src/commands/create.rs @@ -21,7 +21,7 @@ use rattler_conda_types::{ Channel, ChannelConfig, GenericVirtualPackage, MatchSpec, ParseStrictness, Platform, PrefixRecord, RepoDataRecord, Version, }; -use rattler_networking::{AuthenticationMiddleware, AuthenticationStorage}; +use rattler_networking::AuthenticationMiddleware; use rattler_repodata_gateway::{Gateway, RepoData, SourceConfig}; use rattler_solve::{ libsolv_c::{self}, @@ -147,11 +147,8 @@ pub async fn create(opt: Opt) -> anyhow::Result<()> { .build() .expect("failed to create client"); - let authentication_storage = AuthenticationStorage::default(); let download_client = reqwest_middleware::ClientBuilder::new(download_client) - .with_arc(Arc::new(AuthenticationMiddleware::new( - authentication_storage, - ))) + .with_arc(Arc::new(AuthenticationMiddleware::default()?)) .with(rattler_networking::OciMiddleware) .with(rattler_networking::GCSMiddleware) .build(); diff --git a/crates/rattler_networking/Cargo.toml b/crates/rattler_networking/Cargo.toml index 2c61af948..4e4dca6bf 100644 --- a/crates/rattler_networking/Cargo.toml +++ b/crates/rattler_networking/Cargo.toml @@ -22,7 +22,6 @@ async-trait = { workspace = true } base64 = { workspace = true } chrono = { workspace = true } dirs = { workspace = true } -fslock = { workspace = true } google-cloud-auth = { workspace = true, optional = true } google-cloud-token = { workspace = true, optional = true } http = { workspace = true } diff --git a/crates/rattler_networking/src/authentication_middleware.rs b/crates/rattler_networking/src/authentication_middleware.rs index 25e2a3af4..0f6fe7eb7 100644 --- a/crates/rattler_networking/src/authentication_middleware.rs +++ b/crates/rattler_networking/src/authentication_middleware.rs @@ -8,9 +8,10 @@ use reqwest_middleware::{Middleware, Next}; use std::path::{Path, PathBuf}; use std::sync::OnceLock; use url::Url; +use anyhow::Result; /// `reqwest` middleware to authenticate requests -#[derive(Clone, Default)] +#[derive(Clone)] pub struct AuthenticationMiddleware { auth_storage: AuthenticationStorage, } @@ -53,6 +54,13 @@ impl AuthenticationMiddleware { Self { auth_storage } } + /// Create a new authentication middleware with the default authentication storage + pub fn default() -> Result { + Ok(Self { + auth_storage: AuthenticationStorage::default()?, + }) + } + /// Authenticate the given URL with the given authentication information fn authenticate_url(url: Url, auth: &Option) -> Url { if let Some(credentials) = auth { @@ -176,7 +184,7 @@ mod tests { #[test] fn test_store_fallback() -> anyhow::Result<()> { let tdir = tempdir()?; - let mut storage = AuthenticationStorage::new(); + let mut storage = AuthenticationStorage::empty(); storage.add_backend(Arc::from(FileStorage::new( tdir.path().to_path_buf().join("auth.json"), )?)); @@ -191,7 +199,7 @@ mod tests { #[tokio::test] async fn test_conda_token_storage() -> anyhow::Result<()> { let tdir = tempdir()?; - let mut storage = AuthenticationStorage::new(); + let mut storage = AuthenticationStorage::empty(); storage.add_backend(Arc::from(FileStorage::new( tdir.path().to_path_buf().join("auth.json"), )?)); @@ -245,7 +253,7 @@ mod tests { #[tokio::test] async fn test_bearer_storage() -> anyhow::Result<()> { let tdir = tempdir()?; - let mut storage = AuthenticationStorage::new(); + let mut storage = AuthenticationStorage::empty(); storage.add_backend(Arc::from(FileStorage::new( tdir.path().to_path_buf().join("auth.json"), )?)); @@ -305,7 +313,7 @@ mod tests { #[tokio::test] async fn test_basic_auth_storage() -> anyhow::Result<()> { let tdir = tempdir()?; - let mut storage = AuthenticationStorage::new(); + let mut storage = AuthenticationStorage::empty(); storage.add_backend(Arc::from(FileStorage::new( tdir.path().to_path_buf().join("auth.json"), )?)); @@ -383,7 +391,7 @@ mod tests { ("*.com", false), ] { let tdir = tempdir()?; - let mut storage = AuthenticationStorage::new(); + let mut storage = AuthenticationStorage::empty(); storage.add_backend(Arc::from(FileStorage::new( tdir.path().to_path_buf().join("auth.json"), )?)); @@ -418,7 +426,7 @@ mod tests { .to_str() .unwrap(), ), - || AuthenticationStorage::from_env().unwrap(), + || AuthenticationStorage::default().unwrap(), ); let host = "test.example.com"; diff --git a/crates/rattler_networking/src/authentication_storage/backends/file.rs b/crates/rattler_networking/src/authentication_storage/backends/file.rs index 1a661e966..9bdf69bcb 100644 --- a/crates/rattler_networking/src/authentication_storage/backends/file.rs +++ b/crates/rattler_networking/src/authentication_storage/backends/file.rs @@ -1,17 +1,16 @@ //! file storage for passwords. use anyhow::Result; -use fslock::LockFile; use std::collections::BTreeMap; use std::path::Path; -use std::sync::Arc; -use std::{path::PathBuf, sync::Mutex}; +use std::sync::{Arc, RwLock}; +use std::path::PathBuf; use crate::authentication_storage::StorageBackend; use crate::Authentication; #[derive(Clone, Debug)] struct FileStorageCache { - cache: BTreeMap, + content: BTreeMap, file_exists: bool, } @@ -25,7 +24,7 @@ pub struct FileStorage { /// The cache of the file storage /// This is used to avoid reading the file from disk every time /// a credential is accessed - cache: Arc>, + cache: Arc>, } /// An error that can occur when accessing the file storage @@ -35,45 +34,15 @@ pub enum FileStorageError { #[error("IO error: {0}")] IOError(#[from] std::io::Error), - /// Failed to lock the file storage file - #[error("failed to lock file storage file {0}")] - FailedToLock(String, #[source] std::io::Error), - /// An error occurred when (de)serializing the credentials #[error("JSON error: {0}")] JSONError(#[from] serde_json::Error), } -/// Lock the file storage file for reading and writing. This will block until the lock is -/// acquired. -fn lock_file_storage(path: &Path, write: bool) -> Result, FileStorageError> { - if !write && !path.exists() { - return Ok(None); - } - - std::fs::create_dir_all(path.parent().unwrap())?; - let path = path.with_extension("lock"); - let mut lock = fslock::LockFile::open(&path) - .map_err(|e| FileStorageError::FailedToLock(path.to_string_lossy().into_owned(), e))?; - - // First try to lock the file without block. If we can't immediately get the lock we block and issue a debug message. - if !lock - .try_lock_with_pid() - .map_err(|e| FileStorageError::FailedToLock(path.to_string_lossy().into_owned(), e))? - { - tracing::debug!("waiting for lock on {}", path.to_string_lossy()); - lock.lock_with_pid() - .map_err(|e| FileStorageError::FailedToLock(path.to_string_lossy().into_owned(), e))?; - } - - Ok(Some(lock)) -} - impl FileStorageCache { pub fn from_path(path: &Path) -> Result { let file_exists = path.exists(); - let cache = if file_exists { - lock_file_storage(path, false)?; + let content = if file_exists { let file = std::fs::File::open(path)?; let reader = std::io::BufReader::new(file); serde_json::from_reader(reader)? @@ -81,7 +50,7 @@ impl FileStorageCache { BTreeMap::new() }; - Ok(Self { cache, file_exists }) + Ok(Self { content, file_exists }) } } @@ -89,33 +58,30 @@ impl FileStorage { /// Create a new file storage with the given path pub fn new(path: PathBuf) -> Result { // read the JSON file if it exists, and store it in the cache - let cache = Arc::new(Mutex::new(FileStorageCache::from_path(&path)?)); + let cache = Arc::new(RwLock::new(FileStorageCache::from_path(&path)?)); Ok(Self { path, cache }) } - /// Read the JSON file and deserialize it into a `BTreeMap`, or return an empty `BTreeMap` if the + /// Updates the cache by reading the JSON file and deserializing it into a `BTreeMap`, or return an empty `BTreeMap` if the /// file does not exist fn read_json(&self) -> Result, FileStorageError> { let new_cache = FileStorageCache::from_path(&self.path)?; - let mut cache = self.cache.lock().unwrap(); - cache.cache = new_cache.cache; + let mut cache = self.cache.write().unwrap(); + cache.content = new_cache.content; cache.file_exists = new_cache.file_exists; - - Ok(cache.cache.clone()) + Ok(cache.content.clone()) } /// Serialize the given `BTreeMap` and write it to the JSON file fn write_json(&self, dict: &BTreeMap) -> Result<(), FileStorageError> { - let _lock = lock_file_storage(&self.path, true)?; - let file = std::fs::File::create(&self.path)?; let writer = std::io::BufWriter::new(file); serde_json::to_writer(writer, dict)?; // Store the new data in the cache - let mut cache = self.cache.lock().unwrap(); - cache.cache = dict.clone(); + let mut cache = self.cache.write().unwrap(); + cache.content = dict.clone(); cache.file_exists = true; Ok(()) @@ -130,8 +96,8 @@ impl StorageBackend for FileStorage { } fn get(&self, host: &str) -> Result> { - let cache = self.cache.lock().unwrap(); - Ok(cache.cache.get(host).cloned()) + let cache = self.cache.read().unwrap(); + Ok(cache.content.get(host).cloned()) } fn delete(&self, host: &str) -> Result<()> { @@ -151,8 +117,8 @@ impl Default for FileStorage { path.push("credentials.json"); Self::new(path.clone()).unwrap_or(Self { path, - cache: Arc::new(Mutex::new(FileStorageCache { - cache: BTreeMap::new(), + cache: Arc::new(RwLock::new(FileStorageCache { + content: BTreeMap::new(), file_exists: false, })), }) diff --git a/crates/rattler_networking/src/authentication_storage/storage.rs b/crates/rattler_networking/src/authentication_storage/storage.rs index 12a983ced..8be95ba07 100644 --- a/crates/rattler_networking/src/authentication_storage/storage.rs +++ b/crates/rattler_networking/src/authentication_storage/storage.rs @@ -25,26 +25,9 @@ pub struct AuthenticationStorage { cache: Arc>>>, } -impl Default for AuthenticationStorage { - fn default() -> Self { - let mut storage = Self::new(); - - storage.add_backend(Arc::from(KeyringAuthenticationStorage::default())); - storage.add_backend(Arc::from(FileStorage::default())); - storage.add_backend(Arc::from(NetRcStorage::from_env().unwrap_or_else( - |(path, err)| { - tracing::warn!("error reading netrc file from {}: {}", path.display(), err); - NetRcStorage::default() - }, - ))); - - storage - } -} - impl AuthenticationStorage { /// Create a new authentication storage with no backends - pub fn new() -> Self { + pub fn empty() -> Self { Self { backends: vec![], cache: Arc::new(Mutex::new(HashMap::new())), @@ -52,27 +35,38 @@ impl AuthenticationStorage { } /// Create a new authentication storage with the default backends - /// respecting the `RATTLER_AUTH_FILE` environment variable. - /// If the variable is set, the file storage backend will be used - /// with the path specified in the variable - pub fn from_env() -> Result { + /// Following order: + /// - file storage from $RATTLER_AUTH_FILE (if set) + /// - file storage from the default location + /// - keyring storage + /// - netrc storage + pub fn default() -> Result { + let mut storage = Self::empty(); + if let Ok(auth_file) = std::env::var("RATTLER_AUTH_FILE") { let path = std::path::Path::new(&auth_file); - tracing::info!( "\"RATTLER_AUTH_FILE\" environment variable set, using file storage at {}", auth_file ); - - Ok(Self::from_file(path)?) - } else { - Ok(Self::default()) + storage.add_backend(Arc::from(FileStorage::new(path.into())?)); } + storage.add_backend(Arc::from(FileStorage::default())); + storage.add_backend(Arc::from(KeyringAuthenticationStorage::default())); + storage.add_backend(Arc::from(NetRcStorage::from_env().unwrap_or_else( + |(path, err)| { + tracing::warn!("error reading netrc file from {}: {}", path.display(), err); + NetRcStorage::default() + }, + ))); + + Ok(storage) } /// Create a new authentication storage with just a file storage backend + /// Returns an error if the file is not found. pub fn from_file(path: &std::path::Path) -> Result { - let mut storage = Self::new(); + let mut storage = Self::empty(); let backend = FileStorage::new(path.to_path_buf()).map_err(|e| { anyhow!( "Error creating file storage backend from file ({}): {}", @@ -91,6 +85,12 @@ impl AuthenticationStorage { self.backends.push(backend); } + /// Add a new storage backend to the authentication storage to the front + /// (backends are tried in the order they are added) + pub fn add_backend_front(&mut self, backend: Arc) { + self.backends.insert(0, backend); + } + /// Store the given authentication information for the given host pub fn store(&self, host: &str, authentication: &Authentication) -> Result<()> { { diff --git a/crates/rattler_repodata_gateway/src/fetch/jlap/mod.rs b/crates/rattler_repodata_gateway/src/fetch/jlap/mod.rs index 3ff546ef2..a143f0bf7 100644 --- a/crates/rattler_repodata_gateway/src/fetch/jlap/mod.rs +++ b/crates/rattler_repodata_gateway/src/fetch/jlap/mod.rs @@ -30,7 +30,7 @@ //! pub async fn main() { //! let subdir_url = Url::parse("https://conda.anaconda.org/conda-forge/osx-64/").unwrap(); //! let client = reqwest_middleware::ClientBuilder::new(reqwest::Client::new()) -//! .with_arc(Arc::new(AuthenticationMiddleware::default())) +//! .with_arc(Arc::new(AuthenticationMiddleware::default().unwrap())) //! .build(); //! let cache = Path::new("./cache"); //! let current_repo_data = cache.join("c93ef9c9.json"); diff --git a/crates/rattler_repodata_gateway/src/fetch/mod.rs b/crates/rattler_repodata_gateway/src/fetch/mod.rs index 1b31fff3f..f73df748e 100644 --- a/crates/rattler_repodata_gateway/src/fetch/mod.rs +++ b/crates/rattler_repodata_gateway/src/fetch/mod.rs @@ -1472,7 +1472,7 @@ mod test { let client = Client::builder().no_gzip().build().unwrap(); let authenticated_client = reqwest_middleware::ClientBuilder::new(client) - .with_arc(Arc::new(AuthenticationMiddleware::default())) + .with_arc(Arc::new(AuthenticationMiddleware::default().unwrap())) .build(); let result = fetch_repo_data( From fa8c910cd72dff9ffece7346f403fbf7f3e93954 Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Tue, 14 Jan 2025 19:13:14 +0100 Subject: [PATCH 02/24] fix --- .../src/authentication_storage/storage.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/rattler_networking/src/authentication_storage/storage.rs b/crates/rattler_networking/src/authentication_storage/storage.rs index 8be95ba07..0f07d3e2b 100644 --- a/crates/rattler_networking/src/authentication_storage/storage.rs +++ b/crates/rattler_networking/src/authentication_storage/storage.rs @@ -85,10 +85,10 @@ impl AuthenticationStorage { self.backends.push(backend); } - /// Add a new storage backend to the authentication storage to the front + /// Add a new storage backend to the authentication storage at the given index /// (backends are tried in the order they are added) - pub fn add_backend_front(&mut self, backend: Arc) { - self.backends.insert(0, backend); + pub fn insert_backend(&mut self, index: usize, backend: Arc) { + self.backends.insert(index, backend); } /// Store the given authentication information for the given host From ad3917d6cb1593c31c20044624d2487eaa8937cc Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Tue, 14 Jan 2025 19:32:08 +0100 Subject: [PATCH 03/24] . --- .../src/authentication_storage/storage.rs | 20 ++----------------- 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/crates/rattler_networking/src/authentication_storage/storage.rs b/crates/rattler_networking/src/authentication_storage/storage.rs index 0f07d3e2b..6c67f72f5 100644 --- a/crates/rattler_networking/src/authentication_storage/storage.rs +++ b/crates/rattler_networking/src/authentication_storage/storage.rs @@ -37,8 +37,8 @@ impl AuthenticationStorage { /// Create a new authentication storage with the default backends /// Following order: /// - file storage from $RATTLER_AUTH_FILE (if set) - /// - file storage from the default location /// - keyring storage + /// - file storage from the default location /// - netrc storage pub fn default() -> Result { let mut storage = Self::empty(); @@ -51,8 +51,8 @@ impl AuthenticationStorage { ); storage.add_backend(Arc::from(FileStorage::new(path.into())?)); } - storage.add_backend(Arc::from(FileStorage::default())); storage.add_backend(Arc::from(KeyringAuthenticationStorage::default())); + storage.add_backend(Arc::from(FileStorage::default())); storage.add_backend(Arc::from(NetRcStorage::from_env().unwrap_or_else( |(path, err)| { tracing::warn!("error reading netrc file from {}: {}", path.display(), err); @@ -63,22 +63,6 @@ impl AuthenticationStorage { Ok(storage) } - /// Create a new authentication storage with just a file storage backend - /// Returns an error if the file is not found. - pub fn from_file(path: &std::path::Path) -> Result { - let mut storage = Self::empty(); - let backend = FileStorage::new(path.to_path_buf()).map_err(|e| { - anyhow!( - "Error creating file storage backend from file ({}): {}", - path.display(), - e - ) - })?; - - storage.add_backend(Arc::from(backend)); - Ok(storage) - } - /// Add a new storage backend to the authentication storage /// (backends are tried in the order they are added) pub fn add_backend(&mut self, backend: Arc) { From e3bee3b64c6e14714312eb48394f8addf5942585 Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Tue, 14 Jan 2025 19:40:05 +0100 Subject: [PATCH 04/24] . --- .../authentication_storage/backends/file.rs | 21 ++++++------------- .../src/authentication_storage/storage.rs | 2 +- 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/crates/rattler_networking/src/authentication_storage/backends/file.rs b/crates/rattler_networking/src/authentication_storage/backends/file.rs index 9bdf69bcb..0e3ba8a7f 100644 --- a/crates/rattler_networking/src/authentication_storage/backends/file.rs +++ b/crates/rattler_networking/src/authentication_storage/backends/file.rs @@ -63,6 +63,12 @@ impl FileStorage { Ok(Self { path, cache }) } + /// Create a new file storage with the default path + pub fn default() -> Result { + let path = dirs::home_dir().unwrap().join(".rattler").join("credentials.json"); + Self::new(path) + } + /// Updates the cache by reading the JSON file and deserializing it into a `BTreeMap`, or return an empty `BTreeMap` if the /// file does not exist fn read_json(&self) -> Result, FileStorageError> { @@ -110,21 +116,6 @@ impl StorageBackend for FileStorage { } } -impl Default for FileStorage { - fn default() -> Self { - let mut path = dirs::home_dir().unwrap(); - path.push(".rattler"); - path.push("credentials.json"); - Self::new(path.clone()).unwrap_or(Self { - path, - cache: Arc::new(RwLock::new(FileStorageCache { - content: BTreeMap::new(), - file_exists: false, - })), - }) - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/rattler_networking/src/authentication_storage/storage.rs b/crates/rattler_networking/src/authentication_storage/storage.rs index 6c67f72f5..60cbf77b4 100644 --- a/crates/rattler_networking/src/authentication_storage/storage.rs +++ b/crates/rattler_networking/src/authentication_storage/storage.rs @@ -52,7 +52,7 @@ impl AuthenticationStorage { storage.add_backend(Arc::from(FileStorage::new(path.into())?)); } storage.add_backend(Arc::from(KeyringAuthenticationStorage::default())); - storage.add_backend(Arc::from(FileStorage::default())); + storage.add_backend(Arc::from(FileStorage::default()?)); storage.add_backend(Arc::from(NetRcStorage::from_env().unwrap_or_else( |(path, err)| { tracing::warn!("error reading netrc file from {}: {}", path.display(), err); From 0f63dc28a205a34ba15ba4ee20804b5d44139853 Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Tue, 14 Jan 2025 19:43:51 +0100 Subject: [PATCH 05/24] fix --- .../src/authentication_middleware.rs | 2 +- .../src/authentication_storage/backends/file.rs | 12 +++++++++--- .../src/authentication_storage/storage.rs | 15 +++++++++++++-- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/crates/rattler_networking/src/authentication_middleware.rs b/crates/rattler_networking/src/authentication_middleware.rs index 0f6fe7eb7..059dafd6d 100644 --- a/crates/rattler_networking/src/authentication_middleware.rs +++ b/crates/rattler_networking/src/authentication_middleware.rs @@ -1,5 +1,6 @@ //! `reqwest` middleware that authenticates requests with data from the `AuthenticationStorage` use crate::{Authentication, AuthenticationStorage}; +use anyhow::Result; use async_trait::async_trait; use base64::prelude::BASE64_STANDARD; use base64::Engine; @@ -8,7 +9,6 @@ use reqwest_middleware::{Middleware, Next}; use std::path::{Path, PathBuf}; use std::sync::OnceLock; use url::Url; -use anyhow::Result; /// `reqwest` middleware to authenticate requests #[derive(Clone)] diff --git a/crates/rattler_networking/src/authentication_storage/backends/file.rs b/crates/rattler_networking/src/authentication_storage/backends/file.rs index 0e3ba8a7f..2404bb40b 100644 --- a/crates/rattler_networking/src/authentication_storage/backends/file.rs +++ b/crates/rattler_networking/src/authentication_storage/backends/file.rs @@ -2,8 +2,8 @@ use anyhow::Result; use std::collections::BTreeMap; use std::path::Path; -use std::sync::{Arc, RwLock}; use std::path::PathBuf; +use std::sync::{Arc, RwLock}; use crate::authentication_storage::StorageBackend; use crate::Authentication; @@ -50,7 +50,10 @@ impl FileStorageCache { BTreeMap::new() }; - Ok(Self { content, file_exists }) + Ok(Self { + content, + file_exists, + }) } } @@ -65,7 +68,10 @@ impl FileStorage { /// Create a new file storage with the default path pub fn default() -> Result { - let path = dirs::home_dir().unwrap().join(".rattler").join("credentials.json"); + let path = dirs::home_dir() + .unwrap() + .join(".rattler") + .join("credentials.json"); Self::new(path) } diff --git a/crates/rattler_networking/src/authentication_storage/storage.rs b/crates/rattler_networking/src/authentication_storage/storage.rs index 60cbf77b4..56aad487f 100644 --- a/crates/rattler_networking/src/authentication_storage/storage.rs +++ b/crates/rattler_networking/src/authentication_storage/storage.rs @@ -49,10 +49,21 @@ impl AuthenticationStorage { "\"RATTLER_AUTH_FILE\" environment variable set, using file storage at {}", auth_file ); - storage.add_backend(Arc::from(FileStorage::new(path.into())?)); + storage.add_backend(Arc::from(FileStorage::new(path.into()).map_err(|e| { + anyhow!( + "Error creating file storage backend from file ({}): {}", + path.display(), + e + ) + })?)); } storage.add_backend(Arc::from(KeyringAuthenticationStorage::default())); - storage.add_backend(Arc::from(FileStorage::default()?)); + storage.add_backend(Arc::from(FileStorage::default().map_err(|e| { + anyhow!( + "Error creating file storage backend from default path: {}", + e + ) + })?)); storage.add_backend(Arc::from(NetRcStorage::from_env().unwrap_or_else( |(path, err)| { tracing::warn!("error reading netrc file from {}: {}", path.display(), err); From 58c83f405e92fe07270f9433073b26981dcd5417 Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Tue, 14 Jan 2025 19:55:51 +0100 Subject: [PATCH 06/24] fix --- crates/rattler-bin/src/commands/create.rs | 2 +- crates/rattler/src/cli/auth.rs | 2 +- .../src/authentication_middleware.rs | 6 ++--- .../authentication_storage/backends/file.rs | 6 ++--- .../src/authentication_storage/storage.rs | 22 ++++++++++--------- 5 files changed, 20 insertions(+), 18 deletions(-) diff --git a/crates/rattler-bin/src/commands/create.rs b/crates/rattler-bin/src/commands/create.rs index ca42863b9..ea452bb64 100644 --- a/crates/rattler-bin/src/commands/create.rs +++ b/crates/rattler-bin/src/commands/create.rs @@ -148,7 +148,7 @@ pub async fn create(opt: Opt) -> anyhow::Result<()> { .expect("failed to create client"); let download_client = reqwest_middleware::ClientBuilder::new(download_client) - .with_arc(Arc::new(AuthenticationMiddleware::default()?)) + .with_arc(Arc::new(AuthenticationMiddleware::new()?)) .with(rattler_networking::OciMiddleware) .with(rattler_networking::GCSMiddleware) .build(); diff --git a/crates/rattler/src/cli/auth.rs b/crates/rattler/src/cli/auth.rs index 0873e8741..3f88bbaa3 100644 --- a/crates/rattler/src/cli/auth.rs +++ b/crates/rattler/src/cli/auth.rs @@ -141,7 +141,7 @@ fn logout(args: LogoutArgs, storage: AuthenticationStorage) -> Result<(), Authen /// CLI entrypoint for authentication pub async fn execute(args: Args) -> Result<(), AuthenticationCLIError> { - let storage = AuthenticationStorage::default(); + let storage = AuthenticationStorage::default().map_err(AuthenticationCLIError::StorageError)?; match args.subcommand { Subcommand::Login(args) => login(args, storage), diff --git a/crates/rattler_networking/src/authentication_middleware.rs b/crates/rattler_networking/src/authentication_middleware.rs index 059dafd6d..71a746303 100644 --- a/crates/rattler_networking/src/authentication_middleware.rs +++ b/crates/rattler_networking/src/authentication_middleware.rs @@ -50,14 +50,14 @@ impl Middleware for AuthenticationMiddleware { impl AuthenticationMiddleware { /// Create a new authentication middleware with the given authentication storage - pub fn new(auth_storage: AuthenticationStorage) -> Self { + pub fn from_auth_storage(auth_storage: AuthenticationStorage) -> Self { Self { auth_storage } } /// Create a new authentication middleware with the default authentication storage - pub fn default() -> Result { + pub fn new() -> Result { Ok(Self { - auth_storage: AuthenticationStorage::default()?, + auth_storage: AuthenticationStorage::new()?, }) } diff --git a/crates/rattler_networking/src/authentication_storage/backends/file.rs b/crates/rattler_networking/src/authentication_storage/backends/file.rs index 2404bb40b..bc71d504a 100644 --- a/crates/rattler_networking/src/authentication_storage/backends/file.rs +++ b/crates/rattler_networking/src/authentication_storage/backends/file.rs @@ -59,7 +59,7 @@ impl FileStorageCache { impl FileStorage { /// Create a new file storage with the given path - pub fn new(path: PathBuf) -> Result { + pub fn from_path(path: PathBuf) -> Result { // read the JSON file if it exists, and store it in the cache let cache = Arc::new(RwLock::new(FileStorageCache::from_path(&path)?)); @@ -67,12 +67,12 @@ impl FileStorage { } /// Create a new file storage with the default path - pub fn default() -> Result { + pub fn new() -> Result { let path = dirs::home_dir() .unwrap() .join(".rattler") .join("credentials.json"); - Self::new(path) + Self::from_path(path) } /// Updates the cache by reading the JSON file and deserializing it into a `BTreeMap`, or return an empty `BTreeMap` if the diff --git a/crates/rattler_networking/src/authentication_storage/storage.rs b/crates/rattler_networking/src/authentication_storage/storage.rs index 56aad487f..68a605af6 100644 --- a/crates/rattler_networking/src/authentication_storage/storage.rs +++ b/crates/rattler_networking/src/authentication_storage/storage.rs @@ -36,11 +36,11 @@ impl AuthenticationStorage { /// Create a new authentication storage with the default backends /// Following order: - /// - file storage from $RATTLER_AUTH_FILE (if set) + /// - file storage from `RATTLER_AUTH_FILE` (if set) /// - keyring storage /// - file storage from the default location /// - netrc storage - pub fn default() -> Result { + pub fn new() -> Result { let mut storage = Self::empty(); if let Ok(auth_file) = std::env::var("RATTLER_AUTH_FILE") { @@ -49,16 +49,18 @@ impl AuthenticationStorage { "\"RATTLER_AUTH_FILE\" environment variable set, using file storage at {}", auth_file ); - storage.add_backend(Arc::from(FileStorage::new(path.into()).map_err(|e| { - anyhow!( - "Error creating file storage backend from file ({}): {}", - path.display(), - e - ) - })?)); + storage.add_backend(Arc::from(FileStorage::from_path(path.into()).map_err( + |e| { + anyhow!( + "Error creating file storage backend from file ({}): {}", + path.display(), + e + ) + }, + )?)); } storage.add_backend(Arc::from(KeyringAuthenticationStorage::default())); - storage.add_backend(Arc::from(FileStorage::default().map_err(|e| { + storage.add_backend(Arc::from(FileStorage::new().map_err(|e| { anyhow!( "Error creating file storage backend from default path: {}", e From 65f329c4f067b5e962e49b876a025d430e2408f0 Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Tue, 14 Jan 2025 19:57:43 +0100 Subject: [PATCH 07/24] fix --- .../src/authentication_middleware.rs | 14 +++++++------- .../src/authentication_storage/backends/file.rs | 4 ++-- crates/rattler_repodata_gateway/src/fetch/mod.rs | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/crates/rattler_networking/src/authentication_middleware.rs b/crates/rattler_networking/src/authentication_middleware.rs index 71a746303..8a4d0f55c 100644 --- a/crates/rattler_networking/src/authentication_middleware.rs +++ b/crates/rattler_networking/src/authentication_middleware.rs @@ -174,7 +174,7 @@ mod tests { ) { let (captured_tx, captured_rx) = tokio::sync::mpsc::channel(1); let client = reqwest_middleware::ClientBuilder::new(reqwest::Client::default()) - .with_arc(Arc::new(AuthenticationMiddleware::new(storage.clone()))) + .with_arc(Arc::new(AuthenticationMiddleware::from_auth_storage(storage.clone()))) .with_arc(Arc::new(CaptureAbortMiddleware { captured_tx })) .build(); @@ -185,7 +185,7 @@ mod tests { fn test_store_fallback() -> anyhow::Result<()> { let tdir = tempdir()?; let mut storage = AuthenticationStorage::empty(); - storage.add_backend(Arc::from(FileStorage::new( + storage.add_backend(Arc::from(FileStorage::from_path( tdir.path().to_path_buf().join("auth.json"), )?)); @@ -200,7 +200,7 @@ mod tests { async fn test_conda_token_storage() -> anyhow::Result<()> { let tdir = tempdir()?; let mut storage = AuthenticationStorage::empty(); - storage.add_backend(Arc::from(FileStorage::new( + storage.add_backend(Arc::from(FileStorage::from_path( tdir.path().to_path_buf().join("auth.json"), )?)); @@ -254,7 +254,7 @@ mod tests { async fn test_bearer_storage() -> anyhow::Result<()> { let tdir = tempdir()?; let mut storage = AuthenticationStorage::empty(); - storage.add_backend(Arc::from(FileStorage::new( + storage.add_backend(Arc::from(FileStorage::from_path( tdir.path().to_path_buf().join("auth.json"), )?)); let host = "bearer.example.com"; @@ -314,7 +314,7 @@ mod tests { async fn test_basic_auth_storage() -> anyhow::Result<()> { let tdir = tempdir()?; let mut storage = AuthenticationStorage::empty(); - storage.add_backend(Arc::from(FileStorage::new( + storage.add_backend(Arc::from(FileStorage::from_path( tdir.path().to_path_buf().join("auth.json"), )?)); let host = "basic.example.com"; @@ -392,7 +392,7 @@ mod tests { ] { let tdir = tempdir()?; let mut storage = AuthenticationStorage::empty(); - storage.add_backend(Arc::from(FileStorage::new( + storage.add_backend(Arc::from(FileStorage::from_path( tdir.path().to_path_buf().join("auth.json"), )?)); @@ -426,7 +426,7 @@ mod tests { .to_str() .unwrap(), ), - || AuthenticationStorage::default().unwrap(), + || AuthenticationStorage::new().unwrap(), ); let host = "test.example.com"; diff --git a/crates/rattler_networking/src/authentication_storage/backends/file.rs b/crates/rattler_networking/src/authentication_storage/backends/file.rs index bc71d504a..fb359bf82 100644 --- a/crates/rattler_networking/src/authentication_storage/backends/file.rs +++ b/crates/rattler_networking/src/authentication_storage/backends/file.rs @@ -134,7 +134,7 @@ mod tests { let file = tempdir().unwrap(); let path = file.path().join("test.json"); - let storage = FileStorage::new(path.clone()).unwrap(); + let storage = FileStorage::from_path(path.clone()).unwrap(); assert_eq!(storage.get("test").unwrap(), None); @@ -170,6 +170,6 @@ mod tests { let mut file = std::fs::File::create(&path).unwrap(); file.write_all(b"invalid json").unwrap(); - assert!(FileStorage::new(path.clone()).is_err()); + assert!(FileStorage::from_path(path.clone()).is_err()); } } diff --git a/crates/rattler_repodata_gateway/src/fetch/mod.rs b/crates/rattler_repodata_gateway/src/fetch/mod.rs index f73df748e..bef708dde 100644 --- a/crates/rattler_repodata_gateway/src/fetch/mod.rs +++ b/crates/rattler_repodata_gateway/src/fetch/mod.rs @@ -1472,7 +1472,7 @@ mod test { let client = Client::builder().no_gzip().build().unwrap(); let authenticated_client = reqwest_middleware::ClientBuilder::new(client) - .with_arc(Arc::new(AuthenticationMiddleware::default().unwrap())) + .with_arc(Arc::new(AuthenticationMiddleware::new().unwrap())) .build(); let result = fetch_repo_data( From feab1cdfb7d08d712d43a78126b704e0ade8fd60 Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Tue, 14 Jan 2025 19:59:30 +0100 Subject: [PATCH 08/24] fix --- crates/rattler/src/cli/auth.rs | 2 +- crates/rattler_networking/src/authentication_middleware.rs | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/rattler/src/cli/auth.rs b/crates/rattler/src/cli/auth.rs index 3f88bbaa3..78635fa2b 100644 --- a/crates/rattler/src/cli/auth.rs +++ b/crates/rattler/src/cli/auth.rs @@ -141,7 +141,7 @@ fn logout(args: LogoutArgs, storage: AuthenticationStorage) -> Result<(), Authen /// CLI entrypoint for authentication pub async fn execute(args: Args) -> Result<(), AuthenticationCLIError> { - let storage = AuthenticationStorage::default().map_err(AuthenticationCLIError::StorageError)?; + let storage = AuthenticationStorage::new().map_err(AuthenticationCLIError::StorageError)?; match args.subcommand { Subcommand::Login(args) => login(args, storage), diff --git a/crates/rattler_networking/src/authentication_middleware.rs b/crates/rattler_networking/src/authentication_middleware.rs index 8a4d0f55c..322a2355d 100644 --- a/crates/rattler_networking/src/authentication_middleware.rs +++ b/crates/rattler_networking/src/authentication_middleware.rs @@ -174,7 +174,9 @@ mod tests { ) { let (captured_tx, captured_rx) = tokio::sync::mpsc::channel(1); let client = reqwest_middleware::ClientBuilder::new(reqwest::Client::default()) - .with_arc(Arc::new(AuthenticationMiddleware::from_auth_storage(storage.clone()))) + .with_arc(Arc::new(AuthenticationMiddleware::from_auth_storage( + storage.clone(), + ))) .with_arc(Arc::new(CaptureAbortMiddleware { captured_tx })) .build(); From 241b6cb39dd1139384e82341dddaf709cf2ef5aa Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Tue, 14 Jan 2025 20:00:07 +0100 Subject: [PATCH 09/24] fix --- py-rattler/pixi.toml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/py-rattler/pixi.toml b/py-rattler/pixi.toml index 290d248e9..f580c49bb 100644 --- a/py-rattler/pixi.toml +++ b/py-rattler/pixi.toml @@ -41,19 +41,19 @@ typer = "*" types-networkx = "*" [feature.test.tasks] -test = { cmd = "pytest --doctest-modules", depends_on = ["build"] } +test = { cmd = "pytest --doctest-modules", depends-on = ["build"] } fmt-python = "ruff format rattler examples tests" fmt-rust = "cargo fmt --all" lint-python = "ruff check ." lint-rust = "cargo clippy --all" -fmt = { depends_on = ["fmt-python", "fmt-rust"] } -lint = { depends_on = ["type-check", "lint-python", "lint-rust"] } -type-check = { cmd = "mypy", depends_on = ["build"] } +fmt = { depends-on = ["fmt-python", "fmt-rust"] } +lint = { depends-on = ["type-check", "lint-python", "lint-rust"] } +type-check = { cmd = "mypy", depends-on = ["build"] } # checks for the CI fmt-rust-check = "cargo fmt --all --check" fmt-python-check = "ruff format rattler examples tests --diff" -fmt-check = { depends_on = ["fmt-python-check", "fmt-rust-check"] } +fmt-check = { depends-on = ["fmt-python-check", "fmt-rust-check"] } [feature.docs.dependencies] mkdocs = ">=1.5.3,<2" From 69c7aace25b2a2e4120c2153143d28ef8382d6c2 Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Tue, 14 Jan 2025 20:19:32 +0100 Subject: [PATCH 10/24] py-rattler fixes --- py-rattler/src/networking/client.rs | 35 +++++++++++++++---------- py-rattler/src/networking/middleware.rs | 9 +------ py-rattler/src/networking/mod.rs | 2 +- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/py-rattler/src/networking/client.rs b/py-rattler/src/networking/client.rs index 2ff7fce32..48d105839 100644 --- a/py-rattler/src/networking/client.rs +++ b/py-rattler/src/networking/client.rs @@ -1,5 +1,5 @@ -use crate::networking::middleware::PyMiddleware; -use pyo3::{pyclass, pymethods}; +use crate::{error::PyRattlerError, networking::middleware::PyMiddleware}; +use pyo3::{pyclass, pymethods, PyResult}; use rattler_networking::{ AuthenticationMiddleware, GCSMiddleware, MirrorMiddleware, OciMiddleware, }; @@ -16,22 +16,29 @@ pub struct PyClientWithMiddleware { impl PyClientWithMiddleware { #[new] #[pyo3(signature = (middlewares=None))] - pub fn new(middlewares: Option>) -> Self { + pub fn new(middlewares: Option>) -> PyResult { let middlewares = middlewares.unwrap_or_default(); - let client = reqwest_middleware::ClientBuilder::new(reqwest::Client::new()); - let client = middlewares - .into_iter() - .fold(client, |client, middleware| match middleware { - PyMiddleware::Mirror(middleware) => client.with(MirrorMiddleware::from(middleware)), - PyMiddleware::Authentication(middleware) => { - client.with(AuthenticationMiddleware::from(middleware)) + let mut client = reqwest_middleware::ClientBuilder::new(reqwest::Client::new()); + for middleware in middlewares { + match middleware { + PyMiddleware::Mirror(middleware) => { + client = client.with(MirrorMiddleware::from(middleware)); } - PyMiddleware::Oci(middleware) => client.with(OciMiddleware::from(middleware)), - PyMiddleware::Gcs(middleware) => client.with(GCSMiddleware::from(middleware)), - }); + PyMiddleware::Authentication(_) => { + client = + client.with(AuthenticationMiddleware::new().map_err(PyRattlerError::from)?); + } + PyMiddleware::Oci(middleware) => { + client = client.with(OciMiddleware::from(middleware)); + } + PyMiddleware::Gcs(middleware) => { + client = client.with(GCSMiddleware::from(middleware)); + } + } + } let client = client.build(); - Self { inner: client } + Ok(Self { inner: client }) } } diff --git a/py-rattler/src/networking/middleware.rs b/py-rattler/src/networking/middleware.rs index cae3510c9..3e01bbf9a 100644 --- a/py-rattler/src/networking/middleware.rs +++ b/py-rattler/src/networking/middleware.rs @@ -1,7 +1,6 @@ use pyo3::{pyclass, pymethods, FromPyObject, PyResult}; use rattler_networking::{ - mirror_middleware::Mirror, AuthenticationMiddleware, AuthenticationStorage, GCSMiddleware, - MirrorMiddleware, OciMiddleware, + mirror_middleware::Mirror, GCSMiddleware, MirrorMiddleware, OciMiddleware, }; use std::collections::HashMap; use url::Url; @@ -70,12 +69,6 @@ impl PyAuthenticationMiddleware { } } -impl From for AuthenticationMiddleware { - fn from(_value: PyAuthenticationMiddleware) -> Self { - AuthenticationMiddleware::new(AuthenticationStorage::default()) - } -} - #[pyclass] #[repr(transparent)] #[derive(Clone)] diff --git a/py-rattler/src/networking/mod.rs b/py-rattler/src/networking/mod.rs index 28f8cd05c..692eb8bc8 100644 --- a/py-rattler/src/networking/mod.rs +++ b/py-rattler/src/networking/mod.rs @@ -33,7 +33,7 @@ pub fn py_fetch_repo_data<'a>( client: Option, ) -> PyResult> { let mut meta_futures = Vec::new(); - let client = client.unwrap_or(PyClientWithMiddleware::new(None)); + let client = client.unwrap_or(PyClientWithMiddleware::new(None)?); for (subdir, chan, platform) in get_subdir_urls(channels, platforms)? { let callback = callback.as_ref().map(|callback| { From 28f00fe7ae78234ec0297c83052c99ba7d4cae04 Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Tue, 14 Jan 2025 20:20:36 +0100 Subject: [PATCH 11/24] fix --- crates/rattler_repodata_gateway/src/fetch/jlap/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/rattler_repodata_gateway/src/fetch/jlap/mod.rs b/crates/rattler_repodata_gateway/src/fetch/jlap/mod.rs index a143f0bf7..de9c80242 100644 --- a/crates/rattler_repodata_gateway/src/fetch/jlap/mod.rs +++ b/crates/rattler_repodata_gateway/src/fetch/jlap/mod.rs @@ -30,7 +30,7 @@ //! pub async fn main() { //! let subdir_url = Url::parse("https://conda.anaconda.org/conda-forge/osx-64/").unwrap(); //! let client = reqwest_middleware::ClientBuilder::new(reqwest::Client::new()) -//! .with_arc(Arc::new(AuthenticationMiddleware::default().unwrap())) +//! .with_arc(Arc::new(AuthenticationMiddleware::new().unwrap())) //! .build(); //! let cache = Path::new("./cache"); //! let current_repo_data = cache.join("c93ef9c9.json"); From 9ab0f2bf05cfcd096c2902491577f9c52c930ced Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Tue, 14 Jan 2025 21:03:55 +0100 Subject: [PATCH 12/24] re-add lock_file_storage, rename method --- crates/rattler/src/cli/auth.rs | 2 +- crates/rattler_networking/Cargo.toml | 1 + .../src/authentication_middleware.rs | 4 +-- .../authentication_storage/backends/file.rs | 32 ++++++++++++++++++- .../src/authentication_storage/storage.rs | 2 +- 5 files changed, 36 insertions(+), 5 deletions(-) diff --git a/crates/rattler/src/cli/auth.rs b/crates/rattler/src/cli/auth.rs index 78635fa2b..48442ebf1 100644 --- a/crates/rattler/src/cli/auth.rs +++ b/crates/rattler/src/cli/auth.rs @@ -141,7 +141,7 @@ fn logout(args: LogoutArgs, storage: AuthenticationStorage) -> Result<(), Authen /// CLI entrypoint for authentication pub async fn execute(args: Args) -> Result<(), AuthenticationCLIError> { - let storage = AuthenticationStorage::new().map_err(AuthenticationCLIError::StorageError)?; + let storage = AuthenticationStorage::from_env_and_defaults().map_err(AuthenticationCLIError::StorageError)?; match args.subcommand { Subcommand::Login(args) => login(args, storage), diff --git a/crates/rattler_networking/Cargo.toml b/crates/rattler_networking/Cargo.toml index 4e4dca6bf..2c61af948 100644 --- a/crates/rattler_networking/Cargo.toml +++ b/crates/rattler_networking/Cargo.toml @@ -22,6 +22,7 @@ async-trait = { workspace = true } base64 = { workspace = true } chrono = { workspace = true } dirs = { workspace = true } +fslock = { workspace = true } google-cloud-auth = { workspace = true, optional = true } google-cloud-token = { workspace = true, optional = true } http = { workspace = true } diff --git a/crates/rattler_networking/src/authentication_middleware.rs b/crates/rattler_networking/src/authentication_middleware.rs index 322a2355d..1c63fec3a 100644 --- a/crates/rattler_networking/src/authentication_middleware.rs +++ b/crates/rattler_networking/src/authentication_middleware.rs @@ -57,7 +57,7 @@ impl AuthenticationMiddleware { /// Create a new authentication middleware with the default authentication storage pub fn new() -> Result { Ok(Self { - auth_storage: AuthenticationStorage::new()?, + auth_storage: AuthenticationStorage::from_env_and_defaults()?, }) } @@ -428,7 +428,7 @@ mod tests { .to_str() .unwrap(), ), - || AuthenticationStorage::new().unwrap(), + || AuthenticationStorage::from_env_and_defaults().unwrap(), ); let host = "test.example.com"; diff --git a/crates/rattler_networking/src/authentication_storage/backends/file.rs b/crates/rattler_networking/src/authentication_storage/backends/file.rs index fb359bf82..484338d3e 100644 --- a/crates/rattler_networking/src/authentication_storage/backends/file.rs +++ b/crates/rattler_networking/src/authentication_storage/backends/file.rs @@ -1,5 +1,6 @@ //! file storage for passwords. use anyhow::Result; +use fslock::LockFile; use std::collections::BTreeMap; use std::path::Path; use std::path::PathBuf; @@ -34,18 +35,45 @@ pub enum FileStorageError { #[error("IO error: {0}")] IOError(#[from] std::io::Error), + /// Failed to lock the file storage file + #[error("failed to lock file storage file {0}")] + FailedToLock(String, #[source] std::io::Error), + /// An error occurred when (de)serializing the credentials #[error("JSON error: {0}")] JSONError(#[from] serde_json::Error), } +/// Lock the file storage file for reading and writing. This will block until the lock is +/// acquired. +fn lock_file_storage(path: &Path) -> Result { + let path = path.with_extension("lock"); + std::fs::create_dir_all(path.parent().unwrap())?; + let mut lock = fslock::LockFile::open(&path) + .map_err(|e| FileStorageError::FailedToLock(path.to_string_lossy().into_owned(), e))?; + + // First try to lock the file without block. If we can't immediately get the lock we block and issue a debug message. + if !lock + .try_lock_with_pid() + .map_err(|e| FileStorageError::FailedToLock(path.to_string_lossy().into_owned(), e))? + { + tracing::debug!("waiting for lock on {}", path.to_string_lossy()); + lock.lock_with_pid() + .map_err(|e| FileStorageError::FailedToLock(path.to_string_lossy().into_owned(), e))?; + } + + Ok(lock) +} + impl FileStorageCache { pub fn from_path(path: &Path) -> Result { let file_exists = path.exists(); let content = if file_exists { + let _lock = lock_file_storage(path)?; let file = std::fs::File::open(path)?; let reader = std::io::BufReader::new(file); - serde_json::from_reader(reader)? + let content = serde_json::from_reader(reader)?; + content } else { BTreeMap::new() }; @@ -87,6 +115,8 @@ impl FileStorage { /// Serialize the given `BTreeMap` and write it to the JSON file fn write_json(&self, dict: &BTreeMap) -> Result<(), FileStorageError> { + let _lock = lock_file_storage(&self.path)?; + let file = std::fs::File::create(&self.path)?; let writer = std::io::BufWriter::new(file); serde_json::to_writer(writer, dict)?; diff --git a/crates/rattler_networking/src/authentication_storage/storage.rs b/crates/rattler_networking/src/authentication_storage/storage.rs index 68a605af6..4577b4e54 100644 --- a/crates/rattler_networking/src/authentication_storage/storage.rs +++ b/crates/rattler_networking/src/authentication_storage/storage.rs @@ -40,7 +40,7 @@ impl AuthenticationStorage { /// - keyring storage /// - file storage from the default location /// - netrc storage - pub fn new() -> Result { + pub fn from_env_and_defaults() -> Result { let mut storage = Self::empty(); if let Ok(auth_file) = std::env::var("RATTLER_AUTH_FILE") { From 1089a913c7184dce28e375fc59fb7550edd1f928 Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Tue, 14 Jan 2025 21:08:31 +0100 Subject: [PATCH 13/24] fmt --- crates/rattler/src/cli/auth.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/rattler/src/cli/auth.rs b/crates/rattler/src/cli/auth.rs index 48442ebf1..a557539e4 100644 --- a/crates/rattler/src/cli/auth.rs +++ b/crates/rattler/src/cli/auth.rs @@ -141,7 +141,8 @@ fn logout(args: LogoutArgs, storage: AuthenticationStorage) -> Result<(), Authen /// CLI entrypoint for authentication pub async fn execute(args: Args) -> Result<(), AuthenticationCLIError> { - let storage = AuthenticationStorage::from_env_and_defaults().map_err(AuthenticationCLIError::StorageError)?; + let storage = AuthenticationStorage::from_env_and_defaults() + .map_err(AuthenticationCLIError::StorageError)?; match args.subcommand { Subcommand::Login(args) => login(args, storage), From 2efceeed6083ae199a84b47f89bb84c09e8b7c66 Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Tue, 14 Jan 2025 21:14:56 +0100 Subject: [PATCH 14/24] clippy --- .../src/authentication_storage/backends/file.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/rattler_networking/src/authentication_storage/backends/file.rs b/crates/rattler_networking/src/authentication_storage/backends/file.rs index 484338d3e..af50b1d01 100644 --- a/crates/rattler_networking/src/authentication_storage/backends/file.rs +++ b/crates/rattler_networking/src/authentication_storage/backends/file.rs @@ -72,8 +72,7 @@ impl FileStorageCache { let _lock = lock_file_storage(path)?; let file = std::fs::File::open(path)?; let reader = std::io::BufReader::new(file); - let content = serde_json::from_reader(reader)?; - content + serde_json::from_reader(reader)? } else { BTreeMap::new() }; From 1963a80eeaef3dbce0fff450d2623cc8ab8b7c13 Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Tue, 14 Jan 2025 21:38:22 +0100 Subject: [PATCH 15/24] no anyhow result --- crates/rattler/src/cli/auth.rs | 9 ++++++-- .../src/authentication_middleware.rs | 4 ++-- .../src/authentication_storage/storage.rs | 21 ++++--------------- 3 files changed, 13 insertions(+), 21 deletions(-) diff --git a/crates/rattler/src/cli/auth.rs b/crates/rattler/src/cli/auth.rs index a557539e4..80a473427 100644 --- a/crates/rattler/src/cli/auth.rs +++ b/crates/rattler/src/cli/auth.rs @@ -1,6 +1,6 @@ //! This module contains CLI common entrypoint for authentication. use clap::Parser; -use rattler_networking::{Authentication, AuthenticationStorage}; +use rattler_networking::{authentication_storage::backends::file::FileStorageError, Authentication, AuthenticationStorage}; use thiserror; /// Command line arguments that contain authentication data @@ -71,6 +71,11 @@ pub enum AuthenticationCLIError { #[error("Authentication with anaconda.org requires a conda token. Use `--conda-token` to provide one")] AnacondaOrgBadMethod, + /// Wrapper for errors that are generated from the underlying storage system + /// (keyring or file system) + #[error("Failed to initialize the authentication storage system")] + InitializeStorageError(#[source] FileStorageError), + /// Wrapper for errors that are generated from the underlying storage system /// (keyring or file system) #[error("Failed to interact with the authentication storage system")] @@ -142,7 +147,7 @@ fn logout(args: LogoutArgs, storage: AuthenticationStorage) -> Result<(), Authen /// CLI entrypoint for authentication pub async fn execute(args: Args) -> Result<(), AuthenticationCLIError> { let storage = AuthenticationStorage::from_env_and_defaults() - .map_err(AuthenticationCLIError::StorageError)?; + .map_err(AuthenticationCLIError::InitializeStorageError)?; match args.subcommand { Subcommand::Login(args) => login(args, storage), diff --git a/crates/rattler_networking/src/authentication_middleware.rs b/crates/rattler_networking/src/authentication_middleware.rs index 1c63fec3a..dcc8ceeb0 100644 --- a/crates/rattler_networking/src/authentication_middleware.rs +++ b/crates/rattler_networking/src/authentication_middleware.rs @@ -1,6 +1,6 @@ //! `reqwest` middleware that authenticates requests with data from the `AuthenticationStorage` +use crate::authentication_storage::backends::file::FileStorageError; use crate::{Authentication, AuthenticationStorage}; -use anyhow::Result; use async_trait::async_trait; use base64::prelude::BASE64_STANDARD; use base64::Engine; @@ -55,7 +55,7 @@ impl AuthenticationMiddleware { } /// Create a new authentication middleware with the default authentication storage - pub fn new() -> Result { + pub fn new() -> Result { Ok(Self { auth_storage: AuthenticationStorage::from_env_and_defaults()?, }) diff --git a/crates/rattler_networking/src/authentication_storage/storage.rs b/crates/rattler_networking/src/authentication_storage/storage.rs index 4577b4e54..50bd27d7a 100644 --- a/crates/rattler_networking/src/authentication_storage/storage.rs +++ b/crates/rattler_networking/src/authentication_storage/storage.rs @@ -10,7 +10,7 @@ use url::Url; use super::{ authentication::Authentication, - backends::{file::FileStorage, keyring::KeyringAuthenticationStorage, netrc::NetRcStorage}, + backends::{file::{FileStorage, FileStorageError}, keyring::KeyringAuthenticationStorage, netrc::NetRcStorage}, StorageBackend, }; @@ -40,7 +40,7 @@ impl AuthenticationStorage { /// - keyring storage /// - file storage from the default location /// - netrc storage - pub fn from_env_and_defaults() -> Result { + pub fn from_env_and_defaults() -> Result { let mut storage = Self::empty(); if let Ok(auth_file) = std::env::var("RATTLER_AUTH_FILE") { @@ -49,23 +49,10 @@ impl AuthenticationStorage { "\"RATTLER_AUTH_FILE\" environment variable set, using file storage at {}", auth_file ); - storage.add_backend(Arc::from(FileStorage::from_path(path.into()).map_err( - |e| { - anyhow!( - "Error creating file storage backend from file ({}): {}", - path.display(), - e - ) - }, - )?)); + storage.add_backend(Arc::from(FileStorage::from_path(path.into())?)); } storage.add_backend(Arc::from(KeyringAuthenticationStorage::default())); - storage.add_backend(Arc::from(FileStorage::new().map_err(|e| { - anyhow!( - "Error creating file storage backend from default path: {}", - e - ) - })?)); + storage.add_backend(Arc::from(FileStorage::new()?)); storage.add_backend(Arc::from(NetRcStorage::from_env().unwrap_or_else( |(path, err)| { tracing::warn!("error reading netrc file from {}: {}", path.display(), err); From 9a625f3c7544d9b095b7a6c64c1bd6a66a12c10c Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Tue, 14 Jan 2025 21:50:59 +0100 Subject: [PATCH 16/24] fmt --- crates/rattler/src/cli/auth.rs | 4 +++- .../src/authentication_storage/storage.rs | 6 +++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/crates/rattler/src/cli/auth.rs b/crates/rattler/src/cli/auth.rs index 80a473427..da7830211 100644 --- a/crates/rattler/src/cli/auth.rs +++ b/crates/rattler/src/cli/auth.rs @@ -1,6 +1,8 @@ //! This module contains CLI common entrypoint for authentication. use clap::Parser; -use rattler_networking::{authentication_storage::backends::file::FileStorageError, Authentication, AuthenticationStorage}; +use rattler_networking::{ + authentication_storage::backends::file::FileStorageError, Authentication, AuthenticationStorage, +}; use thiserror; /// Command line arguments that contain authentication data diff --git a/crates/rattler_networking/src/authentication_storage/storage.rs b/crates/rattler_networking/src/authentication_storage/storage.rs index 50bd27d7a..c7fea094a 100644 --- a/crates/rattler_networking/src/authentication_storage/storage.rs +++ b/crates/rattler_networking/src/authentication_storage/storage.rs @@ -10,7 +10,11 @@ use url::Url; use super::{ authentication::Authentication, - backends::{file::{FileStorage, FileStorageError}, keyring::KeyringAuthenticationStorage, netrc::NetRcStorage}, + backends::{ + file::{FileStorage, FileStorageError}, + keyring::KeyringAuthenticationStorage, + netrc::NetRcStorage, + }, StorageBackend, }; From 585bf96ee970e698ae4dd7688896203f14a58ee7 Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Tue, 14 Jan 2025 21:56:09 +0100 Subject: [PATCH 17/24] add errors --- py-rattler/src/error.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/py-rattler/src/error.rs b/py-rattler/src/error.rs index 0676219bc..37a44915d 100644 --- a/py-rattler/src/error.rs +++ b/py-rattler/src/error.rs @@ -8,6 +8,7 @@ use rattler_conda_types::{ VersionBumpError, VersionExtendError, }; use rattler_lock::{ConversionError, ParseCondaLockError}; +use rattler_networking::authentication_storage::backends::file::FileStorageError; use rattler_package_streaming::ExtractError; use rattler_repodata_gateway::{fetch::FetchRepoDataError, GatewayError}; use rattler_shell::activation::ActivationError; @@ -76,6 +77,8 @@ pub enum PyRattlerError { ), #[error(transparent)] ValidatePackageRecordsError(#[from] ValidatePackageRecordsError), + #[error(transparent)] + FileStorageError(#[from] FileStorageError), } fn pretty_print_error(mut err: &dyn Error) -> String { @@ -166,6 +169,9 @@ impl From for PyErr { PyRattlerError::ValidatePackageRecordsError(err) => { ValidatePackageRecordsException::new_err(pretty_print_error(&err)) } + PyRattlerError::FileStorageError(err) => { + FileStorageException::new_err(pretty_print_error(&err)) + } } } } @@ -202,3 +208,4 @@ create_exception!( PyException ); create_exception!(exceptions, ValidatePackageRecordsException, PyException); +create_exception!(exceptions, FileStorageException, PyException); From 26d9e723b2c4d6067a38ba6b9d9efee23906d6d1 Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Tue, 14 Jan 2025 22:48:30 +0100 Subject: [PATCH 18/24] make backends public --- .../rattler_networking/src/authentication_storage/storage.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/rattler_networking/src/authentication_storage/storage.rs b/crates/rattler_networking/src/authentication_storage/storage.rs index c7fea094a..218603937 100644 --- a/crates/rattler_networking/src/authentication_storage/storage.rs +++ b/crates/rattler_networking/src/authentication_storage/storage.rs @@ -25,7 +25,8 @@ use super::{ /// Credentials are stored and retrieved from the backends in the /// order they are added to the storage pub struct AuthenticationStorage { - backends: Vec>, + /// Authentication backends + pub backends: Vec>, cache: Arc>>>, } From 6f93e41c4b809b5ec33d8f941eb37eb9e3d06d52 Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Wed, 15 Jan 2025 09:59:14 +0100 Subject: [PATCH 19/24] Use async_fd_lock instead --- crates/rattler_networking/Cargo.toml | 2 +- .../authentication_storage/backends/file.rs | 50 +++++++------------ 2 files changed, 18 insertions(+), 34 deletions(-) diff --git a/crates/rattler_networking/Cargo.toml b/crates/rattler_networking/Cargo.toml index 2c61af948..1e36bd80e 100644 --- a/crates/rattler_networking/Cargo.toml +++ b/crates/rattler_networking/Cargo.toml @@ -18,11 +18,11 @@ gcs = ["google-cloud-auth", "google-cloud-token"] [dependencies] anyhow = { workspace = true } +async-fd-lock = { workspace = true } async-trait = { workspace = true } base64 = { workspace = true } chrono = { workspace = true } dirs = { workspace = true } -fslock = { workspace = true } google-cloud-auth = { workspace = true, optional = true } google-cloud-token = { workspace = true, optional = true } http = { workspace = true } diff --git a/crates/rattler_networking/src/authentication_storage/backends/file.rs b/crates/rattler_networking/src/authentication_storage/backends/file.rs index af50b1d01..6622cc512 100644 --- a/crates/rattler_networking/src/authentication_storage/backends/file.rs +++ b/crates/rattler_networking/src/authentication_storage/backends/file.rs @@ -1,7 +1,8 @@ //! file storage for passwords. use anyhow::Result; -use fslock::LockFile; +use async_fd_lock::{RwLockWriteGuard, blocking::{LockRead, LockWrite}}; use std::collections::BTreeMap; +use std::fs::File; use std::path::Path; use std::path::PathBuf; use std::sync::{Arc, RwLock}; @@ -36,43 +37,23 @@ pub enum FileStorageError { IOError(#[from] std::io::Error), /// Failed to lock the file storage file - #[error("failed to lock file storage file {0}")] - FailedToLock(String, #[source] std::io::Error), + #[error("failed to lock file storage file: {0:?}")] + FailedToLock(async_fd_lock::LockError), /// An error occurred when (de)serializing the credentials #[error("JSON error: {0}")] JSONError(#[from] serde_json::Error), } -/// Lock the file storage file for reading and writing. This will block until the lock is -/// acquired. -fn lock_file_storage(path: &Path) -> Result { - let path = path.with_extension("lock"); - std::fs::create_dir_all(path.parent().unwrap())?; - let mut lock = fslock::LockFile::open(&path) - .map_err(|e| FileStorageError::FailedToLock(path.to_string_lossy().into_owned(), e))?; - - // First try to lock the file without block. If we can't immediately get the lock we block and issue a debug message. - if !lock - .try_lock_with_pid() - .map_err(|e| FileStorageError::FailedToLock(path.to_string_lossy().into_owned(), e))? - { - tracing::debug!("waiting for lock on {}", path.to_string_lossy()); - lock.lock_with_pid() - .map_err(|e| FileStorageError::FailedToLock(path.to_string_lossy().into_owned(), e))?; - } - - Ok(lock) -} - impl FileStorageCache { pub fn from_path(path: &Path) -> Result { let file_exists = path.exists(); let content = if file_exists { - let _lock = lock_file_storage(path)?; - let file = std::fs::File::open(path)?; - let reader = std::io::BufReader::new(file); - serde_json::from_reader(reader)? + let read_guard = File::options() + .read(true) + .open(&path)? + .lock_read().map_err(FileStorageError::FailedToLock)?; + serde_json::from_reader(read_guard)? } else { BTreeMap::new() }; @@ -114,11 +95,14 @@ impl FileStorage { /// Serialize the given `BTreeMap` and write it to the JSON file fn write_json(&self, dict: &BTreeMap) -> Result<(), FileStorageError> { - let _lock = lock_file_storage(&self.path)?; - - let file = std::fs::File::create(&self.path)?; - let writer = std::io::BufWriter::new(file); - serde_json::to_writer(writer, dict)?; + let write_guard: std::result::Result, async_fd_lock::LockError> = File::options() + .create(true) + .write(true) + .truncate(true) + .open(&self.path)? + .lock_write(); + let write_guard = write_guard.map_err(FileStorageError::FailedToLock)?; + serde_json::to_writer(write_guard, dict)?; // Store the new data in the cache let mut cache = self.cache.write().unwrap(); From 4cb24a14dbfb0a9adaeea4c2f7573cf8720954dc Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Wed, 15 Jan 2025 10:03:14 +0100 Subject: [PATCH 20/24] fmt --- .../src/authentication_storage/backends/file.rs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/crates/rattler_networking/src/authentication_storage/backends/file.rs b/crates/rattler_networking/src/authentication_storage/backends/file.rs index 6622cc512..1bad8d385 100644 --- a/crates/rattler_networking/src/authentication_storage/backends/file.rs +++ b/crates/rattler_networking/src/authentication_storage/backends/file.rs @@ -1,6 +1,9 @@ //! file storage for passwords. use anyhow::Result; -use async_fd_lock::{RwLockWriteGuard, blocking::{LockRead, LockWrite}}; +use async_fd_lock::{ + blocking::{LockRead, LockWrite}, + RwLockWriteGuard, +}; use std::collections::BTreeMap; use std::fs::File; use std::path::Path; @@ -52,7 +55,8 @@ impl FileStorageCache { let read_guard = File::options() .read(true) .open(&path)? - .lock_read().map_err(FileStorageError::FailedToLock)?; + .lock_read() + .map_err(FileStorageError::FailedToLock)?; serde_json::from_reader(read_guard)? } else { BTreeMap::new() @@ -95,7 +99,10 @@ impl FileStorage { /// Serialize the given `BTreeMap` and write it to the JSON file fn write_json(&self, dict: &BTreeMap) -> Result<(), FileStorageError> { - let write_guard: std::result::Result, async_fd_lock::LockError> = File::options() + let write_guard: std::result::Result< + RwLockWriteGuard, + async_fd_lock::LockError, + > = File::options() .create(true) .write(true) .truncate(true) From 912c184f285838296be15a895306b84854907f2d Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Wed, 15 Jan 2025 10:06:57 +0100 Subject: [PATCH 21/24] fix --- .../src/authentication_storage/backends/file.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/rattler_networking/src/authentication_storage/backends/file.rs b/crates/rattler_networking/src/authentication_storage/backends/file.rs index 1bad8d385..c213945f5 100644 --- a/crates/rattler_networking/src/authentication_storage/backends/file.rs +++ b/crates/rattler_networking/src/authentication_storage/backends/file.rs @@ -54,7 +54,7 @@ impl FileStorageCache { let content = if file_exists { let read_guard = File::options() .read(true) - .open(&path)? + .open(path)? .lock_read() .map_err(FileStorageError::FailedToLock)?; serde_json::from_reader(read_guard)? From 4c41a088f86d4214d19acfe2d37fb6439405316d Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Fri, 17 Jan 2025 11:18:37 +0100 Subject: [PATCH 22/24] review --- crates/rattler-bin/src/commands/create.rs | 2 +- crates/rattler_networking/src/authentication_middleware.rs | 2 +- .../src/authentication_storage/storage.rs | 6 ------ crates/rattler_repodata_gateway/src/fetch/jlap/mod.rs | 2 +- crates/rattler_repodata_gateway/src/fetch/mod.rs | 2 +- py-rattler/src/networking/client.rs | 2 +- 6 files changed, 5 insertions(+), 11 deletions(-) diff --git a/crates/rattler-bin/src/commands/create.rs b/crates/rattler-bin/src/commands/create.rs index ea452bb64..7ab8e7fd0 100644 --- a/crates/rattler-bin/src/commands/create.rs +++ b/crates/rattler-bin/src/commands/create.rs @@ -148,7 +148,7 @@ pub async fn create(opt: Opt) -> anyhow::Result<()> { .expect("failed to create client"); let download_client = reqwest_middleware::ClientBuilder::new(download_client) - .with_arc(Arc::new(AuthenticationMiddleware::new()?)) + .with_arc(Arc::new(AuthenticationMiddleware::from_env_and_defaults()?)) .with(rattler_networking::OciMiddleware) .with(rattler_networking::GCSMiddleware) .build(); diff --git a/crates/rattler_networking/src/authentication_middleware.rs b/crates/rattler_networking/src/authentication_middleware.rs index dcc8ceeb0..6fdf4ad7d 100644 --- a/crates/rattler_networking/src/authentication_middleware.rs +++ b/crates/rattler_networking/src/authentication_middleware.rs @@ -55,7 +55,7 @@ impl AuthenticationMiddleware { } /// Create a new authentication middleware with the default authentication storage - pub fn new() -> Result { + pub fn from_env_and_defaults() -> Result { Ok(Self { auth_storage: AuthenticationStorage::from_env_and_defaults()?, }) diff --git a/crates/rattler_networking/src/authentication_storage/storage.rs b/crates/rattler_networking/src/authentication_storage/storage.rs index 218603937..92054b50c 100644 --- a/crates/rattler_networking/src/authentication_storage/storage.rs +++ b/crates/rattler_networking/src/authentication_storage/storage.rs @@ -74,12 +74,6 @@ impl AuthenticationStorage { self.backends.push(backend); } - /// Add a new storage backend to the authentication storage at the given index - /// (backends are tried in the order they are added) - pub fn insert_backend(&mut self, index: usize, backend: Arc) { - self.backends.insert(index, backend); - } - /// Store the given authentication information for the given host pub fn store(&self, host: &str, authentication: &Authentication) -> Result<()> { { diff --git a/crates/rattler_repodata_gateway/src/fetch/jlap/mod.rs b/crates/rattler_repodata_gateway/src/fetch/jlap/mod.rs index de9c80242..f89aeb3cc 100644 --- a/crates/rattler_repodata_gateway/src/fetch/jlap/mod.rs +++ b/crates/rattler_repodata_gateway/src/fetch/jlap/mod.rs @@ -30,7 +30,7 @@ //! pub async fn main() { //! let subdir_url = Url::parse("https://conda.anaconda.org/conda-forge/osx-64/").unwrap(); //! let client = reqwest_middleware::ClientBuilder::new(reqwest::Client::new()) -//! .with_arc(Arc::new(AuthenticationMiddleware::new().unwrap())) +//! .with_arc(Arc::new(AuthenticationMiddleware::from_env_and_defaults().unwrap())) //! .build(); //! let cache = Path::new("./cache"); //! let current_repo_data = cache.join("c93ef9c9.json"); diff --git a/crates/rattler_repodata_gateway/src/fetch/mod.rs b/crates/rattler_repodata_gateway/src/fetch/mod.rs index bef708dde..a199685b5 100644 --- a/crates/rattler_repodata_gateway/src/fetch/mod.rs +++ b/crates/rattler_repodata_gateway/src/fetch/mod.rs @@ -1472,7 +1472,7 @@ mod test { let client = Client::builder().no_gzip().build().unwrap(); let authenticated_client = reqwest_middleware::ClientBuilder::new(client) - .with_arc(Arc::new(AuthenticationMiddleware::new().unwrap())) + .with_arc(Arc::new(AuthenticationMiddleware::from_env_and_defaults().unwrap())) .build(); let result = fetch_repo_data( diff --git a/py-rattler/src/networking/client.rs b/py-rattler/src/networking/client.rs index 48d105839..f8e74ecbb 100644 --- a/py-rattler/src/networking/client.rs +++ b/py-rattler/src/networking/client.rs @@ -26,7 +26,7 @@ impl PyClientWithMiddleware { } PyMiddleware::Authentication(_) => { client = - client.with(AuthenticationMiddleware::new().map_err(PyRattlerError::from)?); + client.with(AuthenticationMiddleware::from_env_and_defaults().map_err(PyRattlerError::from)?); } PyMiddleware::Oci(middleware) => { client = client.with(OciMiddleware::from(middleware)); From 64b5eeec1482e379032d57a007a9b38cc5f3ba5c Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Fri, 17 Jan 2025 11:25:36 +0100 Subject: [PATCH 23/24] fmt --- crates/rattler_repodata_gateway/src/fetch/mod.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/rattler_repodata_gateway/src/fetch/mod.rs b/crates/rattler_repodata_gateway/src/fetch/mod.rs index a199685b5..59e42bf95 100644 --- a/crates/rattler_repodata_gateway/src/fetch/mod.rs +++ b/crates/rattler_repodata_gateway/src/fetch/mod.rs @@ -1472,7 +1472,9 @@ mod test { let client = Client::builder().no_gzip().build().unwrap(); let authenticated_client = reqwest_middleware::ClientBuilder::new(client) - .with_arc(Arc::new(AuthenticationMiddleware::from_env_and_defaults().unwrap())) + .with_arc(Arc::new( + AuthenticationMiddleware::from_env_and_defaults().unwrap(), + )) .build(); let result = fetch_repo_data( From 87bc6177f7a55108b837f6420641b8249268d808 Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Fri, 17 Jan 2025 11:31:16 +0100 Subject: [PATCH 24/24] fix --- py-rattler/src/networking/client.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/py-rattler/src/networking/client.rs b/py-rattler/src/networking/client.rs index f8e74ecbb..b6cc5a231 100644 --- a/py-rattler/src/networking/client.rs +++ b/py-rattler/src/networking/client.rs @@ -25,8 +25,10 @@ impl PyClientWithMiddleware { client = client.with(MirrorMiddleware::from(middleware)); } PyMiddleware::Authentication(_) => { - client = - client.with(AuthenticationMiddleware::from_env_and_defaults().map_err(PyRattlerError::from)?); + client = client.with( + AuthenticationMiddleware::from_env_and_defaults() + .map_err(PyRattlerError::from)?, + ); } PyMiddleware::Oci(middleware) => { client = client.with(OciMiddleware::from(middleware));