Refactor: Extract new crate binstalk-{signal, downloader} (rust-lang#518

)

* Refactor: Extract new crate binstalk-downloader
* Re-export `PkgFmt` from `binstalk_manifests`
* Update release-pr.yml
* Update dependabot

Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com>

.github/dependabot.yml

@@ -24,6 +24,10 @@ updates:
     directory: "/crates/binstalk"
     schedule:
       interval: "daily"
+  - package-ecosystem: "cargo"
+    directory: "/crates/binstalk-downloader"
+    schedule:
+      interval: "daily"
   - package-ecosystem: "cargo"
     directory: "/crates/detect-wasi"
     schedule:

.github/workflows/release-pr.yml

@@ -10,6 +10,7 @@ on:
           - bin
           - binstalk
           - binstalk-manifests
+          - binstalk-downloader
           - detect-targets
           - detect-wasi
           - fs-lock

Cargo.toml

@@ -3,6 +3,7 @@ members = [
     "crates/bin",
     "crates/binstalk",
     "crates/binstalk-manifests",
+    "crates/binstalk-downloader",
     "crates/detect-wasi",
     "crates/fs-lock",
     "crates/normalize-path",

crates/bin/src/entry.rs

@@ -35,7 +35,8 @@ pub async fn install_crates(mut args: Args, jobserver_client: LazyJobserverClien
         args.min_tls_version.map(|v| v.into()),
         Duration::from_millis(rate_limit.duration.get()),
         rate_limit.request_count,
-    )?;
+    )
+    .map_err(BinstallError::from)?;
 
     // Build crates.io api client
     let crates_io_api_client = crates_io_api::AsyncClient::with_http_client(

crates/binstalk-downloader/Cargo.toml

@@ -0,0 +1,70 @@
+[package]
+name = "binstalk-downloader"
+description = "The binstall toolkit for downloading and extracting file"
+repository = "https://github.com/cargo-bins/cargo-binstall"
+documentation = "https://docs.rs/binstalk-downloader"
+version = "0.1.0"
+rust-version = "1.61.0"
+authors = ["ryan <ryan@kurte.nz>"]
+edition = "2021"
+license = "GPL-3.0"
+
+[dependencies]
+binstalk-manifests = { version = "0.1.0", path = "../binstalk-manifests" }
+bytes = "1.2.1"
+bzip2 = "0.4.3"
+digest = "0.10.5"
+flate2 = { version = "1.0.24", default-features = false }
+futures-util = { version = "0.3.25", default-features = false, features = ["std"] }
+generic-array = "0.14.6"
+httpdate = "1.0.2"
+log = { version = "0.4.17", features = ["std"] }
+reqwest = { version = "0.11.12", features = ["stream", "gzip", "brotli", "deflate"], default-features = false }
+scopeguard = "1.1.0"
+# Use a fork here since we need PAX support, but the upstream
+# does not hav the PR merged yet.
+#
+#tar = "0.4.38"
+tar = { package = "binstall-tar", version = "0.4.39" }
+tempfile = "3.3.0"
+thiserror = "1.0.37"
+tokio = { version = "1.21.2", features = ["macros", "rt-multi-thread", "sync", "time"], default-features = false }
+tower = { version = "0.4.13", features = ["limit", "util"] }
+trust-dns-resolver = { version = "0.21.2", optional = true, default-features = false, features = ["dnssec-ring"] }
+url = "2.3.1"
+
+xz2 = "0.1.7"
+
+# Disable all features of zip except for features of compression algorithms:
+# Disabled features include:
+#  - aes-crypto: Enables decryption of files which were encrypted with AES, absolutely zero use for
+#    this crate.
+#  - time: Enables features using the [time](https://github.com/time-rs/time) crate,
+#    which is not used by this crate.
+zip = { version = "0.6.3", default-features = false, features = ["deflate", "bzip2", "zstd"] }
+
+# zstd is also depended by zip.
+# Since zip 0.6.3 depends on zstd 0.11, we also have to use 0.11 here,
+# otherwise there will be a link conflict.
+zstd = { version = "0.11.2", default-features = false }
+
+[features]
+default = ["static", "rustls"]
+
+static = ["bzip2/static", "xz2/static"]
+pkg-config = ["zstd/pkg-config"]
+
+zlib-ng = ["flate2/zlib-ng"]
+
+rustls = [
+    "reqwest/rustls-tls",
+
+    # Enable the following features only if trust-dns-resolver is enabled.
+    "trust-dns-resolver?/dns-over-rustls",
+    # trust-dns-resolver currently supports https with rustls
+    "trust-dns-resolver?/dns-over-https-rustls",
+]
+native-tls = ["reqwest/native-tls", "trust-dns-resolver?/dns-over-native-tls"]
+
+# Enable trust-dns-resolver so that features on it will also be enabled.
+trust-dns = ["trust-dns-resolver", "reqwest/trust-dns"]

crates/binstalk-downloader/src/download.rs

@@ -0,0 +1,170 @@
+use std::{fmt::Debug, future::Future, io, marker::PhantomData, path::Path, pin::Pin};
+
+use binstalk_manifests::cargo_toml_binstall::{PkgFmtDecomposed, TarBasedFmt};
+use digest::{Digest, FixedOutput, HashMarker, Output, OutputSizeUser, Update};
+use log::debug;
+use thiserror::Error as ThisError;
+
+pub use binstalk_manifests::cargo_toml_binstall::PkgFmt;
+pub use tar::Entries;
+pub use zip::result::ZipError;
+
+use crate::remote::{Client, Error as RemoteError, Url};
+
+mod async_extracter;
+pub use async_extracter::TarEntriesVisitor;
+use async_extracter::*;
+
+mod extracter;
+mod stream_readable;
+
+pub type CancellationFuture = Option<Pin<Box<dyn Future<Output = Result<(), io::Error>> + Send>>>;
+
+#[derive(Debug, ThisError)]
+pub enum DownloadError {
+    #[error(transparent)]
+    Unzip(#[from] ZipError),
+
+    #[error(transparent)]
+    Remote(#[from] RemoteError),
+
+    /// A generic I/O error.
+    ///
+    /// - Code: `binstall::io`
+    /// - Exit: 74
+    #[error(transparent)]
+    Io(io::Error),
+
+    #[error("installation cancelled by user")]
+    UserAbort,
+}
+
+impl From<io::Error> for DownloadError {
+    fn from(err: io::Error) -> Self {
+        if err.get_ref().is_some() {
+            let kind = err.kind();
+
+            let inner = err
+                .into_inner()
+                .expect("err.get_ref() returns Some, so err.into_inner() should also return Some");
+
+            inner
+                .downcast()
+                .map(|b| *b)
+                .unwrap_or_else(|err| DownloadError::Io(io::Error::new(kind, err)))
+        } else {
+            DownloadError::Io(err)
+        }
+    }
+}
+
+impl From<DownloadError> for io::Error {
+    fn from(e: DownloadError) -> io::Error {
+        match e {
+            DownloadError::Io(io_error) => io_error,
+            e => io::Error::new(io::ErrorKind::Other, e),
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct Download<D: Digest = NoDigest> {
+    client: Client,
+    url: Url,
+    _digest: PhantomData<D>,
+    _checksum: Vec<u8>,
+}
+
+impl Download {
+    pub fn new(client: Client, url: Url) -> Self {
+        Self {
+            client,
+            url,
+            _digest: PhantomData::default(),
+            _checksum: Vec::new(),
+        }
+    }
+
+    /// Download a file from the provided URL and process them in memory.
+    ///
+    /// This does not support verifying a checksum due to the partial extraction
+    /// and will ignore one if specified.
+    ///
+    /// `cancellation_future` can be used to cancel the extraction and return
+    /// [`DownloadError::UserAbort`] error.
+    pub async fn and_visit_tar<V: TarEntriesVisitor + Debug + Send + 'static>(
+        self,
+        fmt: TarBasedFmt,
+        visitor: V,
+        cancellation_future: CancellationFuture,
+    ) -> Result<V::Target, DownloadError> {
+        let stream = self.client.get_stream(self.url).await?;
+
+        debug!("Downloading and extracting then in-memory processing");
+
+        let ret =
+            extract_tar_based_stream_and_visit(stream, fmt, visitor, cancellation_future).await?;
+
+        debug!("Download, extraction and in-memory procession OK");
+
+        Ok(ret)
+    }
+
+    /// Download a file from the provided URL and extract it to the provided path.
+    ///
+    /// `cancellation_future` can be used to cancel the extraction and return
+    /// [`DownloadError::UserAbort`] error.
+    pub async fn and_extract(
+        self,
+        fmt: PkgFmt,
+        path: impl AsRef<Path>,
+        cancellation_future: CancellationFuture,
+    ) -> Result<(), DownloadError> {
+        let stream = self.client.get_stream(self.url).await?;
+
+        let path = path.as_ref();
+        debug!("Downloading and extracting to: '{}'", path.display());
+
+        match fmt.decompose() {
+            PkgFmtDecomposed::Tar(fmt) => {
+                extract_tar_based_stream(stream, path, fmt, cancellation_future).await?
+            }
+            PkgFmtDecomposed::Bin => extract_bin(stream, path, cancellation_future).await?,
+            PkgFmtDecomposed::Zip => extract_zip(stream, path, cancellation_future).await?,
+        }
+
+        debug!("Download OK, extracted to: '{}'", path.display());
+
+        Ok(())
+    }
+}
+
+impl<D: Digest> Download<D> {
+    pub fn new_with_checksum(client: Client, url: Url, checksum: Vec<u8>) -> Self {
+        Self {
+            client,
+            url,
+            _digest: PhantomData::default(),
+            _checksum: checksum,
+        }
+    }
+
+    // TODO: implement checking the sum, may involve bringing (parts of) and_extract() back in here
+}
+
+#[derive(Clone, Copy, Debug, Default)]
+pub struct NoDigest;
+
+impl FixedOutput for NoDigest {
+    fn finalize_into(self, _out: &mut Output<Self>) {}
+}
+
+impl OutputSizeUser for NoDigest {
+    type OutputSize = generic_array::typenum::U0;
+}
+
+impl Update for NoDigest {
+    fn update(&mut self, _data: &[u8]) {}
+}
+
+impl HashMarker for NoDigest {}