0.4, support for deb-src, some refactoring and bugfixes

Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "aptmirs"
 description = "A simple tool for mirroring apt/deb repositories"
-version = "0.3.3"
+version = "0.4.0"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

src/config.rs

@@ -75,6 +75,7 @@ pub struct MirrorOpts {
     pub suite: String,
     pub components: Vec<String>,
     pub arch: Vec<String>,
+    pub source: bool,
 }
 
 
@@ -124,8 +125,10 @@ impl MirrorOpts {
     pub fn try_from(mut line: &str) -> Result<MirrorOpts> {
         let mut arch = Vec::new();
 
+        let mut source = false;
+
         line = if let Some(line) = line.strip_prefix("deb-src") {
-            arch.push("source".to_string());
+            source = true;
             line
         } else if let Some(line) = line.strip_prefix("deb") {
             line
@@ -187,7 +190,8 @@ impl MirrorOpts {
             url: url.to_owned(),
             suite: suite.to_owned(),
             components,
-            arch
+            arch,
+            source
         })
     }
 }

src/error.rs

@@ -36,8 +36,11 @@ pub enum MirsError {
     #[error("url does not point to a valid repository, no release file found")]
     NoReleaseFile,
 
-    #[error("unable to parse package file {path}")]
-    ParsingPackage { path: PathBuf },
+    #[error("unable to parse packages file {path}")]
+    ParsingPackages { path: PathBuf },
+
+    #[error("unable to parse sources file {path}")]
+    ParsingSources { path: PathBuf },
 
     #[error("unable to parse url {url}")]
     UrlParsing { url: String },
@@ -51,6 +54,9 @@ pub enum MirsError {
     #[error(transparent)]
     Hex(#[from]FromHexError),
 
+    #[error("{value} is not a recognized checksum")]
+    IntoChecksum { value: String },
+
     #[error("checksum failed for: {url}, expected hash: {expected}, calculated hash: {hash}")]
     Checksum { url: String, expected: String, hash: String },
 

src/metadata.rs

@@ -1,3 +1,95 @@
+use std::{io::{Read, BufRead, BufReader}, sync::{Arc, atomic::{AtomicU64, Ordering}}, path::{Path, PathBuf}};
+
+use crate::error::{Result, MirsError};
+
+use self::{checksum::Checksum, packages_file::PackagesFile, sources_file::SourcesFile};
+
 pub mod release;
-pub mod package;
-pub mod checksum;
+pub mod packages_file;
+pub mod sources_file;
+pub mod checksum;
+
+pub enum IndexSource {
+    Packages(PathBuf),
+    Sources(PathBuf)
+}
+
+impl IndexSource {
+    pub fn into_reader(self) -> Result<Box<dyn IndexFileEntryIterator>> {
+        match self {
+            IndexSource::Packages(path) => PackagesFile::build(&path),
+            IndexSource::Sources(path) => SourcesFile::build(&path),
+        }
+    }
+}
+
+impl From<PathBuf> for IndexSource {
+    fn from(value: PathBuf) -> Self {
+        match value.file_name().expect("indices should have names")
+            .to_str().expect("the file name of indices should be valid utf8") {
+            v if v.starts_with("Packages") => IndexSource::Packages(value),
+            v if v.starts_with("Sources") => IndexSource::Sources(value),
+            _ => unreachable!("implementation error; non-index file as IndexSource")
+        }
+    }
+}
+
+pub trait IndexFileEntryIterator : Iterator<Item = Result<IndexFileEntry>> {
+    fn size(&self) -> u64;
+    fn counter(&self) -> Arc<AtomicU64>;
+}
+
+pub struct IndexFileEntry {
+    pub path: String,
+    pub size: u64,
+    pub checksum: Option<Checksum>
+}
+pub struct TrackingReader<R: Read> {
+    inner: R,
+    read: Arc<AtomicU64>
+}
+
+impl<R: Read> Read for TrackingReader<R> {
+    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
+        let result = self.inner.read(buf);
+
+        if let Ok(read) = result {
+            self.read.fetch_add(read as u64, Ordering::SeqCst);
+        }
+
+        result
+    }
+}
+
+pub fn create_reader<R: Read + 'static>(file: R, path: &Path) -> Result<(Box<dyn BufRead>, Arc<AtomicU64>)> {
+    let counter = Arc::new(AtomicU64::from(0));
+
+    let file_reader = TrackingReader {
+        inner: file,
+        read: counter.clone(),
+    };
+
+    let reader: Box<dyn BufRead> = match path.extension()
+        .map(|v|
+            v.to_str().expect("extension must be valid ascii")
+        ) {
+        Some("xz") => {
+            let xz_decoder = xz2::read::XzDecoder::new_multi_decoder(file_reader);
+            Box::new(BufReader::with_capacity(1024*1024, xz_decoder))
+        }
+        Some("gz") => {
+            let gz_decoder = flate2::read::GzDecoder::new(file_reader);
+            Box::new(BufReader::with_capacity(1024*1024, gz_decoder))
+        },
+        Some("bz2") => {
+            let bz2_decoder = bzip2::read::BzDecoder::new(file_reader);
+            Box::new(BufReader::with_capacity(1024*1024, bz2_decoder))
+        },
+        None => {
+            Box::new(BufReader::with_capacity(1024*1024, file_reader))
+        },
+        _ => return Err(MirsError::ParsingPackages { path: path.to_owned() })
+    };
+
+    Ok((reader, counter))
+}

src/metadata/checksum.rs

@@ -5,7 +5,7 @@ use sha1::{Sha1, Digest, digest::{FixedOutput, Update}};
 use sha2::{Sha256, Sha512};
 use tokio::io::AsyncReadExt;
 
-use crate::error::Result;
+use crate::error::{Result, MirsError};
 
 #[derive(Debug, PartialEq)]
 pub enum Checksum {
@@ -15,6 +15,36 @@ pub enum Checksum {
     Sha512([u8; 64])
 }
 
+impl TryFrom<&str> for Checksum {
+    type Error = MirsError;
+
+    fn try_from(value: &str) -> std::prelude::v1::Result<Self, Self::Error> {
+        match value.len() {
+            32 => {
+                let mut bytes = [0_u8; 16];
+                hex::decode_to_slice(value, &mut bytes)?;
+                Ok(bytes.into())
+            },
+            40 => {
+                let mut bytes = [0_u8; 20];
+                hex::decode_to_slice(value, &mut bytes)?;
+                Ok(bytes.into())
+            },
+            64 => {
+                let mut bytes = [0_u8; 32];
+                hex::decode_to_slice(value, &mut bytes)?;
+                Ok(bytes.into())
+            },
+            128 => {
+                let mut bytes = [0_u8; 64];
+                hex::decode_to_slice(value, &mut bytes)?;
+                Ok(bytes.into())
+            }
+            _ => Err(MirsError::IntoChecksum { value: value.to_string() })
+        }
+    }
+}
+
 impl From<[u8; 16]> for Checksum {
     fn from(value: [u8; 16]) -> Self {
         Self::Md5(value)
@@ -77,6 +107,23 @@ impl Checksum {
 
         Ok(hasher.compute())
     }
+
+    pub fn bits(&self) -> usize {
+        match self {
+            Checksum::Md5(v) => v.len(),
+            Checksum::Sha1(v) => v.len(),
+            Checksum::Sha256(v) => v.len(),
+            Checksum::Sha512(v) => v.len(),
+        }
+    }
+
+    pub fn replace_if_stronger(&mut self, other: Checksum) {
+        if self.bits() >= other.bits() {
+            return
+        }
+
+        *self = other
+    }
 }
 
 impl ChecksumType {

src/metadata/package.rs → src/metadata/packages_file.rs

@@ -1,88 +1,46 @@
-use std::{path::{Path, PathBuf}, fs::File, io::{BufReader, BufRead, Read}, sync::{atomic::{AtomicU64, Ordering}, Arc}};
+use std::{path::{Path, PathBuf}, fs::File, io::BufRead, sync::{atomic::AtomicU64, Arc}};
 
 use crate::error::{Result, MirsError};
 
-use super::checksum::{Checksum, ChecksumType};
+use super::{checksum::{Checksum, ChecksumType}, create_reader, IndexFileEntry, IndexFileEntryIterator};
 
-pub struct TrackingReader<R: Read> {
-    inner: R,
-    read: Arc<AtomicU64>
-}
-
-impl<R: Read> Read for TrackingReader<R> {
-    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
-        let result = self.inner.read(buf);
-
-        if let Ok(read) = result {
-            self.read.fetch_add(read as u64, Ordering::SeqCst);
-        }
-
-        result
-    }
-}
-
-pub struct Package {
+pub struct PackagesFile {
     reader: Box<dyn BufRead>,
     path: PathBuf,
     buf: String,
     size: u64,
     read: Arc<AtomicU64>
 }
 
-impl Package {
-    pub fn build(path: &Path) -> Result<Self> {
+impl PackagesFile {
+    pub fn build(path: &Path) -> Result<Box<dyn IndexFileEntryIterator>> {
         let file = File::open(path)?;
         let size = file.metadata()?.len();
 
-        let counter = Arc::new(AtomicU64::from(0));
+        let (reader, counter) = create_reader(file, path)?;
 
-        let file_reader = TrackingReader {
-            inner: file,
-            read: counter.clone(),
-        };
-
-        let reader: Box<dyn BufRead> = match path.extension()
-            .map(|v|
-                v.to_str().expect("extension must be valid ascii")
-            ) {
-            Some("xz") => {
-                let xz_decoder = xz2::read::XzDecoder::new_multi_decoder(file_reader);
-                Box::new(BufReader::with_capacity(1024*1024, xz_decoder))
-            }
-            Some("gz") => {
-                let gz_decoder = flate2::read::GzDecoder::new(file_reader);
-                Box::new(BufReader::with_capacity(1024*1024, gz_decoder))
-            },
-            Some("bz2") => {
-                let bz2_decoder = bzip2::read::BzDecoder::new(file_reader);
-                Box::new(BufReader::with_capacity(1024*1024, bz2_decoder))
-            },
-            None => {
-                Box::new(BufReader::with_capacity(1024*1024, file_reader))
-            },
-            _ => return Err(MirsError::ParsingPackage { path: path.to_owned() })
-        };
-
-        Ok(Self {
+        Ok(Box::new(Self {
             reader,
             path: path.to_path_buf(),
             buf: String::with_capacity(1024*8),
             size,
             read: counter,
-        })
+        }))
     }
+}
 
-    pub fn size(&self) -> u64 {
+impl IndexFileEntryIterator for PackagesFile {
+    fn size(&self) -> u64 {
         self.size
     }
 
-    pub fn counter(&self) -> Arc<AtomicU64> {
+    fn counter(&self) -> Arc<AtomicU64> {
         self.read.clone()
     }
 }
 
-impl Iterator for Package {
-    type Item = Result<(String, u64, Option<Checksum>)>;
+impl Iterator for PackagesFile {
+    type Item = Result<IndexFileEntry>;
 
     fn next(&mut self) -> Option<Self::Item> {
         loop {
@@ -148,10 +106,14 @@ impl Iterator for Package {
 
         self.buf.clear();
 
-        if let (Some(p), Some(s), h) = (path, size, hash) {
-            Some(Ok((p, s, h)))
+        if let (Some(path), Some(size), checksum) = (path, size, hash) {
+            Some(Ok(IndexFileEntry {
+                path,
+                size, 
+                checksum
+            }))
         } else {
             None
         }
     }
-}
+}