Merge branch 'valldrac-deflate'

Adds the option to compress memory images at the kernel.
This saves space on disk and an transfer over TCP

README.md

@@ -20,19 +20,22 @@ LiME utilizes the insmod command to load the module, passing required arguments
 ```
 insmod ./lime.ko "path=<outfile | tcp:<port>> format=<raw|padded|lime> [digest=<digest>] [dio=<0|1>]"
 
-path (required):   outfile ~ name of file to write to on local system (SD Card)
-                   tcp:port ~ network port to communicate over
+path (required):     outfile ~ name of file to write to on local system (SD Card)
+                     tcp:port ~ network port to communicate over
         
-format (required): padded ~ pads all non-System RAM ranges with 0s
-                   lime ~ each range prepended with fixed-size header containing address space info
-                   raw ~ concatenates all System RAM ranges (warning : original position of dumped memory is likely to be lost)
+format (required):   padded ~ pads all non-System RAM ranges with 0s
+                     lime ~ each range prepended with fixed-size header containing address space info
+                     raw ~ concatenates all System RAM ranges (warning : original position of dumped memory is likely to be lost)
 
-digest (optional): Hash the RAM and provide a .digest file with the sum.
-                   Supports kernel version 2.6.11 and up. See below for
-                   available digest options.
+digest (optional):   Hash the RAM and provide a .digest file with the sum.
+                     Supports kernel version 2.6.11 and up. See below for
+                     available digest options.
 
-dio (optional):    1 ~ attempt to enable Direct IO
-                   0 ~ default, do not attempt Direct IO
+compress (optional): 1 ~ compress output with zlib
+                     0 ~ do not compress (default)
+
+dio (optional):      1 ~ attempt to enable Direct IO
+                     0 ~ do not attempt Direct IO (default)
         
 localhostonly (optional):  1 ~ restricts the tcp to only listen on localhost,
                            0 ~ binds on all interfaces (default)
@@ -89,6 +92,19 @@ sha3-224, sha3-256, sha3-384, sha3-512
 rmd128, rmd160, rmd256, rmd320
 ```
 
+## Compression
+
+Compression can reduce significantly the time required to acquire a memory capture. It can achieve the speedup of 4x over uncompressed transfers with a few memory overhead (~ 24 KB).
+
+The RAM file will be in the zlib format, which is different from the gzip or zip formats. The reason is that the deflate library embedded in the kernel do not support them.
+
+To decompress it you can use [pigz](https://zlib.net/pigz/) or any zlib-compatible library.
+
+```
+$ nc localhost 4444 | unpigz > ram.lime
+```
+
+Note that only the RAM file is compressed. The digest file is not compressed, and the hash value will match the uncompressed data.
 
 ## Presentation <a name="present"/>
 LiME was first presented at Shmoocon 2012 by Joe Sylve.  

src/Makefile

@@ -21,7 +21,7 @@
 
 
 obj-m := lime.o
-lime-objs := tcp.o disk.o main.o hash.o
+lime-objs := tcp.o disk.o main.o hash.o deflate.o
 
 KVER ?= $(shell uname -r)
 

src/deflate.c

@@ -0,0 +1,99 @@
+/*
+ * LiME - Linux Memory Extractor
+ * Copyright (c) 2011-2014 Joe Sylve - 504ENSICS Labs
+ *
+ *
+ * Author:
+ * Joe Sylve       - joe.sylve@gmail.com, @jtsylve
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or (at
+ * your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+
+#ifdef CONFIG_ZLIB_DEFLATE
+#include <linux/zlib.h>
+
+#include "lime.h"
+
+/* Balance high compression level and memory footprint. */
+#define DEFLATE_WBITS       11  /* 8KB */
+#define DEFLATE_MEMLEVEL    5   /* 12KB */
+
+static struct z_stream_s zstream;
+
+static void *next_out;
+static size_t avail_out;
+
+extern int deflate_begin_stream(void *out, size_t outlen)
+{
+    int size;
+
+    size = zlib_deflate_workspacesize(DEFLATE_WBITS, DEFLATE_MEMLEVEL);
+    zstream.workspace = kzalloc(size, GFP_NOIO);
+    if (!zstream.workspace) {
+        return -ENOMEM;
+    }
+
+    if (zlib_deflateInit2(&zstream, Z_DEFAULT_COMPRESSION, Z_DEFLATED,
+                          DEFLATE_WBITS,
+                          DEFLATE_MEMLEVEL,
+                          Z_DEFAULT_STRATEGY) != Z_OK) {
+        kfree(zstream.workspace);
+        return -EINVAL;
+    }
+
+    next_out = out;
+    avail_out = outlen;
+
+    zstream.next_out = next_out;
+    zstream.avail_out = avail_out;
+
+    return 0;
+}
+
+int deflate_end_stream(void)
+{
+    zlib_deflateEnd(&zstream);
+    kfree(zstream.workspace);
+    return 0;
+}
+
+ssize_t deflate(const void *in, size_t inlen)
+{
+    int flush, ret;
+
+    if (in && inlen > 0)
+        flush = Z_NO_FLUSH;
+    else
+        flush = Z_FINISH;
+
+    if (zstream.avail_out != 0) {
+        zstream.next_in = in;
+        zstream.avail_in = inlen;
+    }
+
+    zstream.next_out = next_out;
+    zstream.avail_out = avail_out;
+
+    ret = zlib_deflate(&zstream, flush);
+
+    if (ret != Z_OK && !(flush == Z_FINISH && ret == Z_STREAM_END)) {
+        DBG("Deflate error: %d", ret);
+        return -EIO;
+    }
+
+    return avail_out - zstream.avail_out;
+}
+
+#endif

src/disk.c

@@ -23,45 +23,43 @@
 
 #include "lime.h"
 
-
 ssize_t write_vaddr_disk(void *, size_t);
-int setup_disk(void);
-void cleanup_disk(void);
-
-static void disable_dio(void);
 
 static struct file * f = NULL;
-extern char * path;
-extern int dio;
-static int reopen = 0;
-
-static void disable_dio() {
-    DBG("Direct IO may not be supported on this file system. Retrying.");
-    dio = 0;
-    reopen = 1;
-    cleanup_disk();
-    setup_disk();
+
+static int dio_write_test(char *path, int oflags)
+{
+    int ok;
+
+    f = filp_open(path, oflags | O_DIRECT, 0444);
+    if (f && !IS_ERR(f)) {
+        ok = write_vaddr_disk("DIO", 3) == 3;
+        filp_close(f, NULL);
+    } else {
+        ok = 0;
+    }
+
+    return ok;
 }
 
-int setup_disk() {
+int setup_disk(char *path, int dio) {
     mm_segment_t fs;
+    int oflags;
     int err;
 
     fs = get_fs();
     set_fs(KERNEL_DS);
 
-    if (dio && reopen) {
-        f = filp_open(path, O_WRONLY | O_CREAT | O_LARGEFILE | O_SYNC | O_DIRECT, 0444);
-    } else if (dio) {
-        f = filp_open(path, O_WRONLY | O_CREAT | O_LARGEFILE | O_TRUNC | O_SYNC | O_DIRECT, 0444);
-    }
+    oflags = O_WRONLY | O_CREAT | O_LARGEFILE | O_TRUNC | O_SYNC;
 
-    if(!dio || (f == ERR_PTR(-EINVAL))) {
+    if (dio && dio_write_test(path, oflags)) {
+        oflags |= O_DIRECT;
+    } else {
         DBG("Direct IO Disabled");
-        f = filp_open(path, O_WRONLY | O_CREAT | O_LARGEFILE | O_TRUNC, 0444);
-        dio = 0;
     }
 
+    f = filp_open(path, oflags, 0444);
+
     if (!f || IS_ERR(f)) {
         DBG("Error opening file %ld", PTR_ERR(f));
         set_fs(fs);
@@ -75,7 +73,7 @@ int setup_disk() {
     return 0;
 }
 
-void cleanup_disk() {
+void cleanup_disk(void) {
     mm_segment_t fs;
 
     fs = get_fs();
@@ -107,11 +105,5 @@ ssize_t write_vaddr_disk(void * v, size_t is) {
 
     set_fs(fs);
 
-    if (s != is && dio) {
-        disable_dio();
-        f->f_pos = pos;
-        return write_vaddr_disk(v, is);
-    }
-
     return s;
 }