Prevent huge memory allocations from bad data

If we get bad width/height data for the image bail out instead of doing
a huge allocation that will crash the program.

This was caught using afl-fuzz

src/decoders/arw.rs

@@ -141,7 +141,7 @@ impl<'a> ArwDecoder<'a> {
 
   fn decode_arw1(buf: &[u8], width: usize, height: usize) -> Vec<u16> {
     let mut pump = BitPumpMSB::new(buf);
-    let mut out: Vec<u16> = vec![0; width*height];
+    let mut out: Vec<u16> = alloc_image!(width, height);
 
     let mut sum: i32 = 0;
     for x in 0..width {

src/decoders/basics.rs

@@ -83,7 +83,7 @@ pub static LITTLE_ENDIAN: Endian = Endian{big: false};
 pub fn decode_threaded<F>(width: usize, height: usize, closure: &F) -> Vec<u16>
   where F : Fn(&mut [u16], usize)+Sync {
 
-  let mut out: Vec<u16> = vec![0; width*height];
+  let mut out: Vec<u16> = alloc_image!(width, height);
   out.par_chunks_mut(width).enumerate().for_each(|(row, line)| {
     closure(line, row);
   });
@@ -93,7 +93,7 @@ pub fn decode_threaded<F>(width: usize, height: usize, closure: &F) -> Vec<u16>
 pub fn decode_threaded_multiline<F>(width: usize, height: usize, lines: usize, closure: &F) -> Vec<u16>
   where F : Fn(&mut [u16], usize)+Sync {
 
-  let mut out: Vec<u16> = vec![0; width*height];
+  let mut out: Vec<u16> = alloc_image!(width, height);
   out.par_chunks_mut(width*lines).enumerate().for_each(|(row, line)| {
     closure(line, row*lines);
   });

src/decoders/cr2.rs

@@ -44,7 +44,7 @@ impl<'a> Decoder for Cr2Decoder<'a> {
       let mut width = ljpegwidth;
       let mut height = decompressor.height();
       let cpp = if decompressor.super_h() == 2 {3} else {1};
-      let mut ljpegout = vec![0 as u16; width*height];
+      let mut ljpegout = alloc_image!(width, height);
       try!(decompressor.decode(&mut ljpegout, 0, width, width, height));
 
       // Linearize the output (applies only to D2000 as far as I can tell)
@@ -87,7 +87,7 @@ impl<'a> Decoder for Cr2Decoder<'a> {
         if canoncol.get_usize(0) == 0 {
           (width, height, cpp, ljpegout)
         } else {
-          let mut out = vec![0 as u16; width*height];
+          let mut out = alloc_image!(width, height);
           let mut fieldwidths = Vec::new();
           for _ in 0..canoncol.get_usize(0) {
             fieldwidths.push(canoncol.get_usize(1));

src/decoders/crw.rs

@@ -189,7 +189,7 @@ impl<'a> CrwDecoder<'a> {
   }
 
   fn decode_compressed(&self, cam: &Camera, width: usize, height: usize) -> Result<Vec<u16>,String> {
-    let mut out = vec![0 as u16; width*height];
+    let mut out = alloc_image!(width, height);
 
     let dectable = fetch_tag!(self.ciff, CiffTag::DecoderTable).get_usize(0);
     if dectable > 2 {

src/decoders/dcr.rs

@@ -61,7 +61,7 @@ impl<'a> DcrDecoder<'a> {
   }
 
   fn decode_kodak65000(buf: &[u8], curve: &LookupTable, width: usize, height: usize) -> Vec<u16> {
-    let mut out: Vec<u16> = vec![0; width*height];
+    let mut out: Vec<u16> = alloc_image!(width, height);
     let mut input = ByteStream::new(buf, LITTLE_ENDIAN);
 
     let mut random: u32 = 0;

src/decoders/dng.rs

@@ -200,7 +200,7 @@ impl<'a> DngDecoder<'a> {
       }
       let offset = offsets.get_usize(0);
       let src = &self.buffer[offset..];
-      let mut out = vec![0 as u16; width*height];
+      let mut out = alloc_image!(width, height);
       let decompressor = try!(LjpegDecompressor::new(src));
       try!(decompressor.decode(&mut out, 0, width, width, height));
       Ok(out)

src/decoders/kdc.rs

@@ -82,7 +82,7 @@ impl<'a> KdcDecoder<'a> {
   }
 
   fn decode_dc120(src: &[u8], width: usize, height: usize) -> Vec<u16> {
-    let mut out = vec![0u16; width*height];
+    let mut out = alloc_image!(width, height);
 
     let mul: [usize;4] = [162, 192, 187,  92];
     let add: [usize;4] = [  0, 636, 424, 212];

src/decoders/mod.rs

@@ -25,6 +25,18 @@ macro_rules! fetch_ifd {
   );
 }
 
+macro_rules! alloc_image {
+  ($width:expr, $height:expr) => (
+    {
+      if $width * $height > 500000000 {
+        panic!("rawloader: surely there's no such thing as a >500MP image!");
+      }
+      let mut out: Vec<u16> = vec![0; $width * $height];
+      out
+    }
+  );
+}
+
 extern crate toml;
 use self::toml::Value;
 mod image;

src/decoders/mos.rs

@@ -89,7 +89,7 @@ impl<'a> MosDecoder<'a> {
     let decompressor = try!(LjpegDecompressor::new_full(src, true, true));
     let ljpegout = try!(decompressor.decode_leaf(width, height));
     if cam.find_hint("interlaced") {
-      let mut out = vec![0 as u16; width*height];
+      let mut out = alloc_image!(width, height);
       for (row,line) in ljpegout.chunks_exact(width).enumerate() {
         let orow = if row & 1 == 1 {height-1-row/2} else {row/2};
         out[orow*width .. (orow+1)*width].copy_from_slice(line);

src/decoders/nef.rs

@@ -285,7 +285,7 @@ impl<'a> NefDecoder<'a> {
     }
     let curve = LookupTable::new(&points[0..max]);
 
-    let mut out = vec![0 as u16; width * height];
+    let mut out = alloc_image!(width, height);
     let mut pump = BitPumpMSB::new(src);
     let mut random = pump.peek_bits(24);
 

src/decoders/orf.rs

@@ -77,7 +77,7 @@ impl<'a> OrfDecoder<'a> {
    */
 
   pub fn decode_compressed(buf: &'a [u8], width: usize, height: usize) -> Vec<u16> {
-    let mut out: Vec<u16> = vec![0; width*height];
+    let mut out: Vec<u16> = alloc_image!(width, height);
 
     /* Build a table to quickly look up "high" value */
     let mut bittable: [u8; 4096] = [0; 4096];

src/decoders/packed.rs

@@ -74,7 +74,7 @@ pub fn decode_12be(buf: &[u8], width: usize, height: usize) -> Vec<u16> {
 }
 
 pub fn decode_12be_msb16(buf: &[u8], width: usize, height: usize) -> Vec<u16> {
-  let mut out: Vec<u16> = vec![0; width*height];
+  let mut out: Vec<u16> = alloc_image!(width, height);
 
   for (o, i) in out.chunks_exact_mut(4).zip(buf.chunks_exact(6)) {
     let g1:  u16 = i[ 0] as u16;
@@ -109,7 +109,7 @@ pub fn decode_12le_16bitaligned(buf: &[u8], width: usize, height: usize) -> Vec<
 }
 
 pub fn decode_12be_msb32(buf: &[u8], width: usize, height: usize) -> Vec<u16> {
-  let mut out: Vec<u16> = vec![0; width*height];
+  let mut out: Vec<u16> = alloc_image!(width, height);
 
   for (o, i) in out.chunks_exact_mut(8).zip(buf.chunks_exact(12)) {
     let g1:  u16 = i[ 0] as u16;

src/decoders/pef.rs

@@ -121,7 +121,7 @@ impl<'a> PefDecoder<'a> {
     let mut pred_left1: i32;
     let mut pred_left2: i32;
 
-    let mut out = vec![0 as u16; width*height];
+    let mut out = alloc_image!(width, height);
     for row in 0..height {
       pred_up1[row & 1] += try!(htable.huff_decode(&mut pump));
       pred_up2[row & 1] += try!(htable.huff_decode(&mut pump));

src/decoders/raf.rs

@@ -110,7 +110,7 @@ impl<'a> RafDecoder<'a> {
       let rotatedwidth = cropheight + cropwidth/2;
       let rotatedheight = rotatedwidth-1;
 
-      let mut out: Vec<u16> = vec![0; rotatedwidth * rotatedheight];
+      let mut out: Vec<u16> = alloc_image!(rotatedwidth, rotatedheight);
       for row in 0..cropheight {
         let inb = &src[(row+y)*width+x..];
         for col in 0..cropwidth {
@@ -125,7 +125,7 @@ impl<'a> RafDecoder<'a> {
       let rotatedwidth = cropwidth + cropheight/2;
       let rotatedheight = rotatedwidth-1;
 
-      let mut out: Vec<u16> = vec![0; rotatedwidth * rotatedheight];
+      let mut out: Vec<u16> = alloc_image!(rotatedwidth, rotatedheight);
       for row in 0..cropheight {
         let inb = &src[(row+y)*width+x..];
         for col in 0..cropwidth {

src/decoders/srw.rs

@@ -74,7 +74,7 @@ impl<'a> Decoder for SrwDecoder<'a> {
 
 impl<'a> SrwDecoder<'a> {
   pub fn decode_srw1(buf: &[u8], loffsets: &[u8], width: usize, height: usize) -> Vec<u16> {
-    let mut out: Vec<u16> = vec![0; width*height];
+    let mut out: Vec<u16> = alloc_image!(width, height);
 
     for row in 0..height {
       let mut len: [u32; 4] = [if row < 2 {7} else {4}; 4];
@@ -141,7 +141,7 @@ impl<'a> SrwDecoder<'a> {
   }
 
   pub fn decode_srw2(buf: &[u8], width: usize, height: usize) -> Vec<u16> {
-    let mut out: Vec<u16> = vec![0; width*height];
+    let mut out: Vec<u16> = alloc_image!(width, height);
 
     // This format has a variable length encoding of how many bits are needed
     // to encode the difference between pixels, we use a table to process it
@@ -213,7 +213,7 @@ impl<'a> SrwDecoder<'a> {
     // and Loring von Palleske (Samsung) for pointing to the open-source code of
     // Samsung's DNG converter at http://opensource.samsung.com/
 
-    let mut out: Vec<u16> = vec![0; width*height];
+    let mut out: Vec<u16> = alloc_image!(width, height);
     let mut pump = BitPumpMSB32::new(buf);
 
     // Process the initial metadata bits, we only really use initVal, width and

src/decoders/tfr.rs

@@ -47,7 +47,7 @@ impl<'a> TfrDecoder<'a> {
   }
 
   fn decode_compressed(&self, src: &[u8], width: usize, height: usize) -> Result<Vec<u16>,String> {
-    let mut out = vec![0 as u16; width*height];
+    let mut out = alloc_image!(width, height);
     let decompressor = try!(LjpegDecompressor::new_full(src, true, false));
     try!(decompressor.decode(&mut out, 0, width, width, height));
     Ok(out)