lazy transducer PR

Cargo.toml

@@ -18,6 +18,8 @@ env_logger = "0.4.3"
 
 [dependencies]
 plain = "0.2.3"
+rayon = "0.9"
+lazy_transducer = "0.1"
 log = { version = "0.3.8", optional = true }
 
 [dependencies.scroll]

src/elf/mod.rs

@@ -39,6 +39,8 @@
 #[macro_use]
 mod gnu_hash;
 
+use lazy_transducer::{Builder, ScrollTransducer};
+
 // These are shareable values for the 32/64 bit implementations.
 //
 // They are publicly re-exported by the pub-using module
@@ -62,6 +64,7 @@ macro_rules! if_sylvan {
 
 if_sylvan! {
     use scroll::{self, ctx, Pread, Endian};
+    use scroll::ctx::SizeWith;
     use strtab::Strtab;
     use error;
     use container::{Container, Ctx};
@@ -78,6 +81,7 @@ if_sylvan! {
     pub type ProgramHeaders = Vec<ProgramHeader>;
     pub type SectionHeaders = Vec<SectionHeader>;
     pub type ShdrIdx = usize;
+    pub type Relocs<'a> = ScrollTransducer<'a, Reloc, (bool, Ctx)>;
 
     #[derive(Debug)]
     /// An ELF binary. The underlying data structures are read according to the headers byte order and container size (32 or 64).
@@ -105,11 +109,11 @@ if_sylvan! {
         /// Contains dynamic linking information, with the _DYNAMIC array + a preprocessed DynamicInfo for that array
         pub dynamic: Option<Dynamic>,
         /// The dynamic relocation entries (strings, copy-data, etc.) with an addend
-        pub dynrelas: Vec<Reloc>,
+        pub dynrelas: Relocs<'a>,
         /// The dynamic relocation entries without an addend
-        pub dynrels: Vec<Reloc>,
+        pub dynrels: Relocs<'a>,
         /// The plt relocation entries (procedure linkage table). For 32-bit binaries these are usually Rel (no addend)
-        pub pltrelocs: Vec<Reloc>,
+        pub pltrelocs: Relocs<'a>,
         /// Section relocations by section index (only present if this is a relocatable object file)
         pub shdr_relocs: Vec<(ShdrIdx, Vec<Reloc>)>,
         /// The binary's soname, if it has one
@@ -231,9 +235,9 @@ if_sylvan! {
             let mut soname = None;
             let mut libraries = vec![];
             let mut dynsyms = Symtab::default();
-            let mut dynrelas = vec![];
-            let mut dynrels = vec![];
-            let mut pltrelocs = vec![];
+            let (mut dynrelas, dynrela_ctx) = (Builder::new(bytes), (true, ctx));;
+            let (mut dynrels, dynrel_ctx) = (Builder::new(bytes), (false, ctx));
+            let (mut pltrelocs, mut pltrela_ctx) = (Builder::new(bytes), (false, ctx));
             let mut dynstrtab = Strtab::default();
             let dynamic = Dynamic::parse(bytes, &program_headers, bias, ctx)?;
             if let Some(ref dynamic) = dynamic {
@@ -253,10 +257,11 @@ if_sylvan! {
                 let num_syms = if dyn_info.syment == 0 { 0 } else { if dyn_info.strtab <= dyn_info.symtab { 0 } else { (dyn_info.strtab - dyn_info.symtab) / dyn_info.syment }};
                 dynsyms = Symtab::parse(bytes, dyn_info.symtab, num_syms, ctx)?;
                 // parse the dynamic relocations
-                dynrelas = Reloc::parse(bytes, dyn_info.rela, dyn_info.relasz, true, ctx)?;
-                dynrels = Reloc::parse(bytes, dyn_info.rel, dyn_info.relsz, false, ctx)?;
-                let is_rela = dyn_info.pltrel as u64 == dyn::DT_RELA;
-                pltrelocs = Reloc::parse(bytes, dyn_info.jmprel, dyn_info.pltrelsz, is_rela, ctx)?;
+                dynrelas = dynrelas.input(&bytes[dyn_info.rela..]).count(dyn_info.relasz / Reloc::size_with(&dynrela_ctx));
+                //dynrelas = Reloc::parse(bytes, dyn_info.rela, dyn_info.relasz, true, ctx)?;
+                dynrels = dynrels.input(&bytes[dyn_info.rel..]).count(dyn_info.relsz / Reloc::size_with(&dynrel_ctx));
+                pltrela_ctx.0 = dyn_info.pltrel as u64 == dyn::DT_RELA;
+                pltrelocs = pltrelocs.input(&bytes[dyn_info.jmprel..]).count(dyn_info.pltrelsz / Reloc::size_with(&pltrela_ctx));
             }
 
             // iterate through shdrs again iff we're an ET_REL
@@ -288,9 +293,9 @@ if_sylvan! {
                 dynstrtab: dynstrtab,
                 syms: syms,
                 strtab: strtab,
-                dynrelas: dynrelas,
-                dynrels: dynrels,
-                pltrelocs: pltrelocs,
+                dynrelas: dynrelas.parse_with(dynrela_ctx).unwrap(),
+                dynrels: dynrels.parse_with(dynrel_ctx).unwrap(),
+                pltrelocs: pltrelocs.parse_with(pltrela_ctx).unwrap(),
                 shdr_relocs: shdr_relocs,
                 soname: soname,
                 interpreter: interpreter,

src/elf/sym.rs

@@ -262,6 +262,8 @@ pub mod sym64 {
 }
 
 if_std! {
+    use lazy_transducer::{ScrollTransducer, IntoIter, IntoParIter};
+    use rayon::prelude::*;
     use scroll::{ctx, Pread};
     use scroll::ctx::SizeWith;
     use core::fmt::{self, Debug};
@@ -402,14 +404,24 @@ if_std! {
         }
     }
 
-    #[derive(Default)]
     /// An ELF symbol table, allowing lazy iteration over symbols
     pub struct Symtab<'a> {
         bytes: &'a [u8],
-        count: usize,
-        ctx: Ctx,
         start: usize,
         end: usize,
+        lt: ScrollTransducer<'a, Sym, Ctx>,
+    }
+
+    impl<'a> Default for Symtab<'a> {
+        fn default() -> Self {
+            let bytes = &[];
+            Symtab {
+                bytes,
+                start: 0,
+                end: 0,
+                lt: ScrollTransducer::parse_with(bytes, 0, Ctx::default()).unwrap(),
+            }
+        }
     }
 
     impl<'a> Debug for Symtab<'a> {
@@ -418,62 +430,78 @@ if_std! {
             fmt.debug_struct("Symtab")
                 .field("bytes", &len)
                 .field("range", &format!("{:#x}..{:#x}", self.start, self.end))
-                .field("count", &self.count)
+                .field("count", &self.lt.len())
                 .field("Symbols", &self.to_vec())
                 .finish()
         }
     }
 
+    pub type SymIter<'a> = IntoIter<'a, (&'a [u8], Ctx), Sym>;
+    pub type SymParIter<'a> = IntoParIter<'a, (&'a [u8], Ctx), Sym>;
+
     impl<'a> Symtab<'a> {
         /// Parse a table of `count` ELF symbols from `offset`.
         pub fn parse(bytes: &'a [u8], offset: usize, count: usize, ctx: Ctx) -> Result<Symtab<'a>> {
+            // scrolltransducer does all this for us, but i don't feel like re-mapping the error
             let size = count * Sym::size_with(&ctx);
             // TODO: make this a better error message when too large
             let bytes = bytes.pread_with(offset, size)?;
-            Ok(Symtab { bytes, count, ctx, start: offset, end: offset+size })
+            let lt = ScrollTransducer::parse_with(bytes, count, ctx).unwrap();
+            Ok(Symtab { bytes, start: offset, end: offset+size, lt })
         }
 
         /// Try to parse a single symbol from the binary, at `index`.
         pub fn get(&self, index: usize) -> Option<Sym> {
-            if index >= self.count {
-                None
-            } else {
-                Some(self.bytes.pread_with(index * Sym::size_with(&self.ctx), self.ctx).unwrap())
-            }
+            self.lt.get(index)
         }
 
         /// The number of symbols in the table.
         #[inline]
         pub fn len(&self) -> usize {
-            self.count
+            self.lt.len()
         }
 
         /// Iterate over all symbols.
-        pub fn iter(&self) -> SymIterator<'a> {
+        pub fn iter(&self) -> SymIter<'a> {
             self.into_iter()
         }
 
+        /// Iterate over all symbols in parallel
+        pub fn par_iter(&self) -> SymParIter<'a> {
+            self.lt.clone().into_par_iter()
+        }
+
         /// Parse all symbols into a vector.
         pub fn to_vec(&self) -> Vec<Sym> {
-            self.iter().collect()
+            self.lt.clone().into_par_iter().collect()
         }
     }
 
     impl<'a, 'b> IntoIterator for &'b Symtab<'a> {
-        type Item = <SymIterator<'a> as Iterator>::Item;
-        type IntoIter = SymIterator<'a>;
+        //type Item = <SymIterator<'a> as Iterator>::Item;
+        type Item = <SymIter<'a> as Iterator>::Item;
+        type IntoIter = SymIter<'a>;
 
         fn into_iter(self) -> Self::IntoIter {
-            SymIterator {
-                bytes: self.bytes,
-                offset: 0,
-                index: 0,
-                count: self.count,
-                ctx: self.ctx,
-            }
+            self.lt.clone().into_iter()
         }
     }
 
+    // impl<'a, 'b> IntoIterator for &'b Symtab<'a> {
+    //     type Item = <SymIterator<'a> as Iterator>::Item;
+    //     type IntoIter = SymIterator<'a>;
+
+    //     fn into_iter(self) -> Self::IntoIter {
+    //         SymIterator {
+    //             bytes: self.bytes,
+    //             offset: 0,
+    //             index: 0,
+    //             count: self.count,
+    //             ctx: self.ctx,
+    //         }
+    //     }
+    // }
+
     /// An iterator over symbols in an ELF symbol table
     pub struct SymIterator<'a> {
         bytes: &'a [u8],

src/lib.rs

@@ -93,6 +93,9 @@ extern crate core;
 #[cfg(feature = "std")]
 pub mod error;
 
+extern crate lazy_transducer;
+extern crate rayon;
+
 pub mod strtab;
 
 /////////////////////////