From 4ad8efb85cb4ffa65a997fb4284e4ea5affbcf88 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Wed, 23 Aug 2023 13:12:25 +0200 Subject: [PATCH] make rattler-package-streaming compile with wasm --- crates/rattler_conda_types/src/channel/mod.rs | 21 ++-- .../src/match_spec/parse.rs | 3 + crates/rattler_conda_types/src/platform.rs | 4 +- crates/rattler_networking/Cargo.toml | 7 +- crates/rattler_networking/src/lib.rs | 4 + crates/rattler_package_streaming/Cargo.toml | 14 +-- .../src/reqwest/blocking.rs | 98 +++++++++++++++++ .../src/reqwest/mod.rs | 100 +----------------- .../tests/extract.rs | 2 +- 9 files changed, 140 insertions(+), 113 deletions(-) create mode 100644 crates/rattler_package_streaming/src/reqwest/blocking.rs diff --git a/crates/rattler_conda_types/src/channel/mod.rs b/crates/rattler_conda_types/src/channel/mod.rs index f19dd1a3a..84c7697d6 100644 --- a/crates/rattler_conda_types/src/channel/mod.rs +++ b/crates/rattler_conda_types/src/channel/mod.rs @@ -66,13 +66,20 @@ impl Channel { Channel::from_url(url, platforms, config) } else if is_path(channel) { let path = PathBuf::from(channel); - let absolute_path = absolute_path(&path); - let url = Url::from_directory_path(absolute_path) - .map_err(|_| ParseChannelError::InvalidPath(path))?; - Self { - platforms, - base_url: url, - name: Some(channel.to_owned()), + + #[cfg(target_arch = "wasm32")] + return Err(ParseChannelError::InvalidPath(path)); + + #[cfg(not(target_arch = "wasm32"))] + { + let absolute_path = absolute_path(&path); + let url = Url::from_directory_path(absolute_path) + .map_err(|_| ParseChannelError::InvalidPath(path))?; + Self { + platforms, + base_url: url, + name: Some(channel.to_owned()), + } } } else { Channel::from_name(channel, platforms, config) diff --git a/crates/rattler_conda_types/src/match_spec/parse.rs b/crates/rattler_conda_types/src/match_spec/parse.rs index c4759e3f2..798d9be5f 100644 --- a/crates/rattler_conda_types/src/match_spec/parse.rs +++ b/crates/rattler_conda_types/src/match_spec/parse.rs @@ -348,6 +348,9 @@ fn parse(input: &str) -> Result { if is_package_file(input) { let _url = match Url::parse(input) { Ok(url) => url, + #[cfg(target_arch = "wasm32")] + Err(_) => return Err(ParseMatchSpecError::InvalidPackagePathOrUrl), + #[cfg(not(target_arch = "wasm32"))] Err(_) => match PathBuf::from_str(input) { Ok(path) => Url::from_file_path(path) .map_err(|_| ParseMatchSpecError::InvalidPackagePathOrUrl)?, diff --git a/crates/rattler_conda_types/src/platform.rs b/crates/rattler_conda_types/src/platform.rs index 0939809a3..269807fc1 100644 --- a/crates/rattler_conda_types/src/platform.rs +++ b/crates/rattler_conda_types/src/platform.rs @@ -146,7 +146,9 @@ impl Platform { target_os = "emscripten", windows )))] - compile_error!("unsupported target os"); + { + return Platform::Linux64; + } } /// Returns a string representation of the platform. diff --git a/crates/rattler_networking/Cargo.toml b/crates/rattler_networking/Cargo.toml index 9d43358a3..f165a6fdf 100644 --- a/crates/rattler_networking/Cargo.toml +++ b/crates/rattler_networking/Cargo.toml @@ -11,8 +11,10 @@ license.workspace = true readme.workspace = true [features] +default = ['blocking'] native-tls = ['reqwest/native-tls'] rustls-tls = ['reqwest/rustls-tls'] +blocking = ['reqwest/blocking'] [dependencies] anyhow = "1.0.71" @@ -20,13 +22,16 @@ dirs = "5.0.1" keyring = "2.0.4" lazy_static = "1.4.0" libc = "0.2.147" -reqwest = { version = "0.11.18", features = ["blocking"], default-features = false} +reqwest = { version = "0.11.18", default-features = false} retry-policies = { version = "0.2.0", default-features = false } serde = "1.0.171" serde_json = "1.0.102" thiserror = "1.0.43" tracing = "0.1.37" +[target.'cfg( target_arch = "wasm32" )'.dependencies] +getrandom = { version = "0.2.10", features = ["js"] } + [dev-dependencies] anyhow = "1.0.71" insta = { version = "1.30.0", features = ["json"] } diff --git a/crates/rattler_networking/src/lib.rs b/crates/rattler_networking/src/lib.rs index 6da7abfb3..db63bf945 100644 --- a/crates/rattler_networking/src/lib.rs +++ b/crates/rattler_networking/src/lib.rs @@ -111,6 +111,7 @@ impl AuthenticatedClient { } } +#[cfg(feature = "blocking")] /// A blocking client that can be used to make authenticated requests, based on the [`reqwest::blocking::Client`] pub struct AuthenticatedClientBlocking { /// The underlying client @@ -120,6 +121,7 @@ pub struct AuthenticatedClientBlocking { auth_storage: AuthenticationStorage, } +#[cfg(feature = "blocking")] impl AuthenticatedClientBlocking { /// Create a new authenticated client from the given client and authentication storage pub fn from_client( @@ -133,6 +135,7 @@ impl AuthenticatedClientBlocking { } } +#[cfg(feature = "blocking")] impl Default for AuthenticatedClientBlocking { fn default() -> Self { AuthenticatedClientBlocking { @@ -142,6 +145,7 @@ impl Default for AuthenticatedClientBlocking { } } +#[cfg(feature = "blocking")] impl AuthenticatedClientBlocking { /// Create a GET request builder for the given URL (see also [`reqwest::blocking::Client::get`]) pub fn get(&self, url: U) -> reqwest::blocking::RequestBuilder { diff --git a/crates/rattler_package_streaming/Cargo.toml b/crates/rattler_package_streaming/Cargo.toml index 9f86bc7f4..b84bc43b1 100644 --- a/crates/rattler_package_streaming/Cargo.toml +++ b/crates/rattler_package_streaming/Cargo.toml @@ -11,7 +11,7 @@ license.workspace = true readme.workspace = true [dependencies] -bzip2 = { version = "0.4" } +bzip2 = "0.4.4" chrono = "0.4.26" futures-util = { version = "0.3.28", optional = true } itertools = "0.11.0" @@ -24,16 +24,16 @@ tokio = { version = "1", optional = true } tokio-util = { version = "0.7", optional = true } reqwest = { version = "0.11.18", optional = true, default-features = false } url = "2.4.0" -zip = { version = "0.6.6" } -zstd = "0.12.3" +zip = { version = "0.6.6", default-features = false, features = ["deflate", "time"] } +zstd = { version = "0.12.3", features = ["wasm"], default-features = false } rattler_networking = { version = "0.8.0", path = "../rattler_networking", default-features = false } [features] -default = ['native-tls'] +default = ["native-tls", "blocking"] tokio = ["dep:tokio", "bzip2/tokio", "tokio/fs", "tokio-util/io", "tokio-util/io-util", "reqwest?/stream", "futures-util"] -native-tls = ['reqwest?/native-tls'] -rustls-tls = ['reqwest?/rustls-tls'] -reqwest = ["reqwest/blocking"] +native-tls = ["rattler_networking/native-tls"] +rustls-tls = ["rattler_networking/rustls-tls"] +blocking = ["rattler_networking/blocking"] [dev-dependencies] tempfile = "3.6.0" diff --git a/crates/rattler_package_streaming/src/reqwest/blocking.rs b/crates/rattler_package_streaming/src/reqwest/blocking.rs new file mode 100644 index 000000000..61ff52a6c --- /dev/null +++ b/crates/rattler_package_streaming/src/reqwest/blocking.rs @@ -0,0 +1,98 @@ +//! Functionality to stream and extract packages directly from a [`reqwest::Url`] with a blocking client + +use crate::{ExtractError, ExtractResult}; +use rattler_conda_types::package::ArchiveType; +use rattler_networking::AuthenticatedClientBlocking; +use reqwest::blocking::Response; +use reqwest::IntoUrl; +use std::path::Path; + +/// Extracts the contents a `.tar.bz2` package archive from the specified remote location. +/// +/// ```rust,no_run +/// # use std::path::Path; +/// use rattler_package_streaming::reqwest::extract_tar_bz2; +/// use rattler_networking::AuthenticatedClientBlocking; +/// # use reqwest::blocking::Client; +/// let _ = extract_tar_bz2( +/// AuthenticatedClientBlocking::default(), +/// "https://conda.anaconda.org/conda-forge/win-64/python-3.11.0-hcf16a7b_0_cpython.tar.bz2", +/// Path::new("/tmp")) +/// .unwrap(); +/// ``` +pub fn extract_tar_bz2( + client: AuthenticatedClientBlocking, + url: impl IntoUrl, + destination: &Path, +) -> Result { + // Send the request for the file + let response = client + .get(url) + .send() + .and_then(Response::error_for_status) + .map_err(ExtractError::ReqwestError)?; + + // The `response` is used to stream in the package data + crate::read::extract_tar_bz2(response, destination) +} + +/// Extracts the contents a `.conda` package archive from the specified remote location. +/// +/// ```rust,no_run +/// # use std::path::Path; +/// use rattler_package_streaming::reqwest::extract_conda; +/// use rattler_networking::AuthenticatedClientBlocking; +/// # use reqwest::blocking::Client; +/// let _ = extract_conda( +/// AuthenticatedClientBlocking::default(), +/// "https://conda.anaconda.org/conda-forge/linux-64/python-3.10.8-h4a9ceb5_0_cpython.conda", +/// Path::new("/tmp")) +/// .unwrap(); +/// ``` +pub fn extract_conda( + client: AuthenticatedClientBlocking, + url: impl IntoUrl, + destination: &Path, +) -> Result { + // Send the request for the file + let response = client + .get(url) + .send() + .and_then(Response::error_for_status) + .map_err(ExtractError::ReqwestError)?; + + // The `response` is used to stream in the package data + crate::read::extract_conda(response, destination) +} + +/// Extracts the contents a package archive from the specified remote location. The type of package +/// is determined based on the path of the url. +/// +/// ```rust,no_run +/// # use std::path::Path; +/// use rattler_package_streaming::reqwest::extract; +/// use rattler_networking::AuthenticatedClientBlocking; +/// # use reqwest::blocking::Client; +/// let _ = extract( +/// AuthenticatedClientBlocking::default(), +/// "https://conda.anaconda.org/conda-forge/linux-64/python-3.10.8-h4a9ceb5_0_cpython.conda", +/// Path::new("/tmp")) +/// .unwrap(); +/// ``` +pub fn extract( + client: AuthenticatedClientBlocking, + url: impl IntoUrl, + destination: &Path, +) -> Result { + let url = url + .into_url() + .map_err(reqwest::Error::from) + .map_err(ExtractError::ReqwestError)?; + + match ArchiveType::try_from(Path::new(url.path())) + .ok_or(ExtractError::UnsupportedArchiveType)? + { + ArchiveType::TarBz2 => extract_tar_bz2(client, url, destination), + ArchiveType::Conda => extract_conda(client, url, destination), + } +} diff --git a/crates/rattler_package_streaming/src/reqwest/mod.rs b/crates/rattler_package_streaming/src/reqwest/mod.rs index f9b563f8b..0d9a33641 100644 --- a/crates/rattler_package_streaming/src/reqwest/mod.rs +++ b/crates/rattler_package_streaming/src/reqwest/mod.rs @@ -3,99 +3,7 @@ #[cfg(feature = "tokio")] pub mod tokio; -use crate::{ExtractError, ExtractResult}; -use rattler_conda_types::package::ArchiveType; -use rattler_networking::AuthenticatedClientBlocking; -use reqwest::blocking::Response; -use reqwest::IntoUrl; -use std::path::Path; - -/// Extracts the contents a `.tar.bz2` package archive from the specified remote location. -/// -/// ```rust,no_run -/// # use std::path::Path; -/// use rattler_package_streaming::reqwest::extract_tar_bz2; -/// use rattler_networking::AuthenticatedClientBlocking; -/// # use reqwest::blocking::Client; -/// let _ = extract_tar_bz2( -/// AuthenticatedClientBlocking::default(), -/// "https://conda.anaconda.org/conda-forge/win-64/python-3.11.0-hcf16a7b_0_cpython.tar.bz2", -/// Path::new("/tmp")) -/// .unwrap(); -/// ``` -pub fn extract_tar_bz2( - client: AuthenticatedClientBlocking, - url: impl IntoUrl, - destination: &Path, -) -> Result { - // Send the request for the file - let response = client - .get(url) - .send() - .and_then(Response::error_for_status) - .map_err(ExtractError::ReqwestError)?; - - // The `response` is used to stream in the package data - crate::read::extract_tar_bz2(response, destination) -} - -/// Extracts the contents a `.conda` package archive from the specified remote location. -/// -/// ```rust,no_run -/// # use std::path::Path; -/// use rattler_package_streaming::reqwest::extract_conda; -/// use rattler_networking::AuthenticatedClientBlocking; -/// # use reqwest::blocking::Client; -/// let _ = extract_conda( -/// AuthenticatedClientBlocking::default(), -/// "https://conda.anaconda.org/conda-forge/linux-64/python-3.10.8-h4a9ceb5_0_cpython.conda", -/// Path::new("/tmp")) -/// .unwrap(); -/// ``` -pub fn extract_conda( - client: AuthenticatedClientBlocking, - url: impl IntoUrl, - destination: &Path, -) -> Result { - // Send the request for the file - let response = client - .get(url) - .send() - .and_then(Response::error_for_status) - .map_err(ExtractError::ReqwestError)?; - - // The `response` is used to stream in the package data - crate::read::extract_conda(response, destination) -} - -/// Extracts the contents a package archive from the specified remote location. The type of package -/// is determined based on the path of the url. -/// -/// ```rust,no_run -/// # use std::path::Path; -/// use rattler_package_streaming::reqwest::extract; -/// use rattler_networking::AuthenticatedClientBlocking; -/// # use reqwest::blocking::Client; -/// let _ = extract( -/// AuthenticatedClientBlocking::default(), -/// "https://conda.anaconda.org/conda-forge/linux-64/python-3.10.8-h4a9ceb5_0_cpython.conda", -/// Path::new("/tmp")) -/// .unwrap(); -/// ``` -pub fn extract( - client: AuthenticatedClientBlocking, - url: impl IntoUrl, - destination: &Path, -) -> Result { - let url = url - .into_url() - .map_err(reqwest::Error::from) - .map_err(ExtractError::ReqwestError)?; - - match ArchiveType::try_from(Path::new(url.path())) - .ok_or(ExtractError::UnsupportedArchiveType)? - { - ArchiveType::TarBz2 => extract_tar_bz2(client, url, destination), - ArchiveType::Conda => extract_conda(client, url, destination), - } -} +#[cfg(feature = "blocking")] +pub mod blocking; +#[cfg(feature = "blocking")] +pub use blocking::{extract, extract_conda, extract_tar_bz2}; diff --git a/crates/rattler_package_streaming/tests/extract.rs b/crates/rattler_package_streaming/tests/extract.rs index b5d436c11..b07f30adc 100644 --- a/crates/rattler_package_streaming/tests/extract.rs +++ b/crates/rattler_package_streaming/tests/extract.rs @@ -194,7 +194,7 @@ async fn test_extract_conda_async(#[case] input: &str, #[case] sha256: &str, #[c assert_eq!(&format!("{:x}", result.md5), md5); } -#[cfg(feature = "reqwest")] +#[cfg(all(feature = "reqwest", feature = "blocking"))] #[apply(url_archives)] fn test_extract_url(#[case] url: &str, #[case] sha256: &str, #[case] md5: &str) { use rattler_networking::AuthenticatedClientBlocking;