0.5 - support for udeb/debian-installer, also changed defaults for co…

…nfigs. added support for changing number of download tasks, some refactoring

Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "aptmirs"
 description = "A simple tool for mirroring apt/deb repositories"
-version = "0.4.0"
+version = "0.5.0"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

src/config.rs

@@ -78,7 +78,6 @@ pub struct MirrorOpts {
     pub source: bool,
 }
 
-
 impl Ord for MirrorOpts {
     fn cmp(&self, other: &Self) -> Ordering {
         match self.url.cmp(&other.url) {

src/main.rs

@@ -1,8 +1,9 @@
 use std::path::PathBuf;
 
-use clap::{command, arg, Parser, ArgAction};
+use clap::{command, arg, Parser};
 use config::read_config;
 use error::MirsError;
+use mirror::downloader::Downloader;
 
 use crate::error::Result;
 
@@ -13,25 +14,22 @@ mod config;
 
 #[tokio::main]
 async fn main() -> Result<()> {
-    let cli = Cli::parse();
-
-    let output = match cli.output {
-        Some(output) => output,
-        None => std::env::current_dir()?
-    };
+    let cli_opts = CliOpts::parse();
 
     let opts = read_config(
-        &cli.config.expect("config should have a default value")
+        cli_opts.config.as_ref().expect("config should have a default value")
     ).await?;
 
     if opts.is_empty() {
-        return Err(MirsError::Config { msg: String::from("config file did not contain any valid repositories") })
+        return Err(MirsError::Config { msg: format!("no valid repositories in: {}", cli_opts.config.unwrap().to_string_lossy()) })
     }
 
+    let mut downloader = Downloader::build(cli_opts.dl_threads);
+
     for opt in opts {
         println!("{} Mirroring {}", now(), &opt);
 
-        match mirror::mirror(&opt, &output).await {
+        match mirror::mirror(&opt, &cli_opts, &mut downloader).await {
             Ok(result) => println!("{} Mirroring done: {result}", now()),
             Err(e) => println!("{} Mirroring failed: {e}", now())
         }
@@ -42,15 +40,22 @@ async fn main() -> Result<()> {
 
 #[derive(Parser)]
 #[command(author, version, about)]
-struct Cli {
-    #[arg(short, long, env, value_name = "CONFIG_FILE", default_value = "./mirror.list")]
+struct CliOpts {
+    #[arg(short, long, env, value_name = "CONFIG_FILE", default_value = "/etc/apt/mirror.list", 
+        help = "The path to the config file containing the mirror options")]
     config: Option<PathBuf>,
 
-    #[arg(short, long, env, value_name = "OUTPUT")]
-    output: Option<PathBuf>,
+    #[arg(short, long, env, value_name = "OUTPUT",
+        help = "The directory where the mirrors will be downloaded into")]
+    output: PathBuf,
+
+    #[arg(short, long, env, value_name = "UDEB", default_value_t = false,
+        help = "Download packages for debian-installer")]
+    pub udeb: bool,
 
-    #[arg(short, long, action = ArgAction::Count)]
-    verbose: u8,
+    #[arg(short, long, env, value_name = "DL_THREADS", default_value_t = 8_u8,
+        help = "The maximum number of concurrent downloading tasks")]
+    dl_threads: u8,
 }
 
 fn now() -> String {

src/metadata/release.rs

@@ -2,7 +2,7 @@ use std::{path::{Path, Component}, collections::{BTreeMap, BTreeSet}};
 
 use tokio::{fs::File, io::{BufReader, AsyncBufReadExt}};
 
-use crate::{error::{Result, MirsError}, config::MirrorOpts};
+use crate::{error::{Result, MirsError}, config::MirrorOpts, CliOpts};
 
 use super::checksum::Checksum;
 
@@ -121,8 +121,8 @@ impl Release {
         self.map.get("Components")
     }
 
-    pub fn into_filtered_files(self, opts: &MirrorOpts) -> ReleaseFileIterator {
-        ReleaseFileIterator::new(self, opts)
+    pub fn into_filtered_files<'a>(self, opts: &'a MirrorOpts, cli_opts: &'a CliOpts) -> ReleaseFileIterator<'a> {
+        ReleaseFileIterator::new(self, opts, cli_opts)
     }
 
     pub fn deduplicate(&mut self, mut old_release: Release) {
@@ -148,7 +148,7 @@ pub struct ReleaseFileIterator<'a> {
 }
 
 impl<'a> ReleaseFileIterator<'a> {
-    pub fn new(release: Release, opts: &'a MirrorOpts) -> Self {
+    pub fn new(release: Release, opts: &'a MirrorOpts, cli_opts: &'a CliOpts) -> Self {
         let (file_prefix_filter, dir_filter) = if opts.source {
             let file_prefix_filter = Vec::from([
                 String::from("Release"),
@@ -180,14 +180,24 @@ impl<'a> ReleaseFileIterator<'a> {
                 String::from("Contents-all.diff"),
                 String::from("Packages.diff"),
             ]);
-
+
+            if cli_opts.udeb {
+                file_prefix_filter.push(String::from("Contents-udeb-all"));
+                dir_filter.insert(String::from("debian-installer"));
+            }
+
             for arch in &opts.arch {
                 dir_filter.insert(format!("binary-{arch}"));
                 dir_filter.insert(format!("Contents-{arch}.diff"));
 
+
                 file_prefix_filter.push(format!("Components-{arch}"));
                 file_prefix_filter.push(format!("Contents-{arch}"));
                 file_prefix_filter.push(format!("Commands-{arch}"));
+
+                if cli_opts.udeb {
+                    file_prefix_filter.push(format!("Contents-udeb-{arch}"));
+                }
             }
 
             (file_prefix_filter, dir_filter)
@@ -228,9 +238,12 @@ impl<'a> Iterator for ReleaseFileIterator<'a> {
                         let part_name = part.to_str()
                             .expect("path should be utf8");
 
-                        if parts.peek().is_none() && 
-                            self.file_prefix_filter.iter().any(|v| part_name.starts_with(v)) {
-                            return Some((path, file_entry))
+                        if parts.peek().is_none() {
+                            if self.file_prefix_filter.iter().any(|v| part_name.starts_with(v)) {
+                                return Some((path, file_entry))
+                            }
+
+                            break
                         } 
 
                         if !self.dir_filter.contains(part_name) {

src/mirror.rs

@@ -2,11 +2,12 @@ use std::collections::BTreeMap;
 use std::ffi::OsStr;
 use std::fmt::Display;
 use std::os::unix::ffi::OsStrExt;
-use std::path::{Path, PathBuf};
+use std::path::PathBuf;
 use std::sync::atomic::Ordering;
 
 use indicatif::{MultiProgress, HumanBytes};
 
+use crate::CliOpts;
 use crate::config::MirrorOpts;
 use crate::error::{Result, MirsError};
 use crate::metadata::IndexSource;
@@ -41,17 +42,17 @@ impl Display for MirrorResult {
     }
 }
 
-pub async fn mirror(opts: &MirrorOpts, output_dir: &Path) -> Result<MirrorResult> {
-    let repo = Repository::build(&opts.url, &opts.suite, output_dir)?;
+pub async fn mirror(opts: &MirrorOpts, cli_opts: &CliOpts, downloader: &mut Downloader) -> Result<MirrorResult> {
+    let repo = Repository::build(&opts.url, &opts.suite, &cli_opts.output)?;
 
-    let mut downloader = Downloader::build(8);
     let mut progress = downloader.progress();
+    progress.reset();
 
     let mut total_download_size = 0_u64;
 
     progress.next_step("Downloading release").await;
 
-    let release = match download_release(&repo, &mut downloader).await {
+    let release = match download_release(&repo, downloader).await {
         Ok(Some(release)) => release,
         Ok(None) => {
             _ = repo.delete_tmp();
@@ -77,7 +78,7 @@ pub async fn mirror(opts: &MirrorOpts, output_dir: &Path) -> Result<MirrorResult
 
     progress.next_step("Downloading indices").await;
 
-    let indices = match download_indices(release, opts, &mut progress, &repo, &mut downloader).await {
+    let indices = match download_indices(release, opts, cli_opts, &mut progress, &repo, downloader).await {
         Ok(indices) if indices.is_empty() => {
             repo.finalize().await?;
             return Ok(MirrorResult::IrrelevantChanges)
@@ -93,7 +94,7 @@ pub async fn mirror(opts: &MirrorOpts, output_dir: &Path) -> Result<MirrorResult
 
     progress.next_step("Downloading packages").await;
 
-    if let Err(e) = download_from_indices(&repo, &mut downloader, indices).await {
+    if let Err(e) = download_from_indices(&repo, downloader, indices).await {
         _ = repo.delete_tmp();
         return Err(MirsError::DownloadPackages { inner: Box::new(e) })
     }
@@ -114,12 +115,12 @@ pub async fn mirror(opts: &MirrorOpts, output_dir: &Path) -> Result<MirrorResult
     })
 }
 
-async fn download_indices(release: Release, opts: &MirrorOpts, progress: &mut Progress, repo: &Repository, downloader: &mut Downloader) -> Result<Vec<PathBuf>> {
+async fn download_indices(release: Release, opts: &MirrorOpts, cli_opts: &CliOpts, progress: &mut Progress, repo: &Repository, downloader: &mut Downloader) -> Result<Vec<PathBuf>> {
     let mut indices = Vec::new();
 
     let by_hash = release.acquire_by_hash();
 
-    for (path, file_entry) in release.into_filtered_files(opts) {
+    for (path, file_entry) in release.into_filtered_files(opts, cli_opts) {
         let url = repo.to_url_in_dist(&path);
         let file_path_in_tmp = repo.to_path_in_tmp(&url);
 

src/mirror/progress.rs

@@ -106,6 +106,12 @@ impl Progress {
         progress_bar.set_message(HumanBytes(self.bytes.success()).to_string());
     }
 
+    pub fn reset(&mut self) {
+        self.bytes.reset();
+        self.files.reset();
+        self.step = Arc::new(AtomicU8::new(0));
+    }
+
     pub async fn next_step(&mut self, step_name: &str) {
         *self.step_name.lock().await = step_name.to_string();