Hyperscan MPM integration v6

configure.ac

@@ -585,6 +585,44 @@
         AC_MSG_RESULT(no)
     fi
 
+  # libhs
+    enable_hyperscan="no"
+
+    # Try pkg-config first:
+    PKG_CHECK_MODULES([libhs], libhs,, [with_pkgconfig_libhs=no])
+    if test "$with_pkgconfig_libhs" != "no"; then
+        CPPFLAGS="${CPPFLAGS} ${libhs_CFLAGS}"
+        LIBS="${LIBS} ${libhs_LIBS}"
+    fi
+
+    AC_ARG_WITH(libhs_includes,
+            [  --with-libhs-includes=DIR  libhs include directory],
+            [with_libhs_includes="$withval"],[with_libhs_includes=no])
+    AC_ARG_WITH(libhs_libraries,
+            [  --with-libhs-libraries=DIR    libhs library directory],
+            [with_libhs_libraries="$withval"],[with_libhs_libraries="no"])
+
+    if test "$with_libhs_includes" != "no"; then
+        CPPFLAGS="${CPPFLAGS} -I${with_libhs_includes}"
+    fi
+    AC_CHECK_HEADER(hs.h,HYPERSCAN="yes",HYPERSCAN="no")
+    if test "$HYPERSCAN" = "yes"; then
+        if test "$with_libhs_libraries" != "no"; then
+            LDFLAGS="${LDFLAGS}  -L${with_libhs_libraries}"
+        fi
+
+        AC_CHECK_LIB(hs,hs_compile,,HYPERSCAN="no")
+        enable_hyperscan="yes"
+        if test "$HYPERSCAN" = "no"; then
+            echo
+            echo "   Hyperscan headers are present, but link test failed."
+            echo "   Check that you have a shared library and C++ linkage available."
+            echo
+            enable_hyperscan="no"
+        fi
+    fi
+    AS_IF([test "x$enable_hyperscan" = "xyes"], [AC_DEFINE([BUILD_HYPERSCAN], [1], [Intel Hyperscan support enabled])])
+
   # libyaml
     AC_ARG_WITH(libyaml_includes,
             [  --with-libyaml-includes=DIR  libyaml include directory],
@@ -1896,6 +1934,7 @@ SURICATA_BUILD_CONF="Suricata Configuration:
   Non-bundled htp:                         ${enable_non_bundled_htp}
   Old barnyard2 support:                   ${enable_old_barnyard2}
   CUDA enabled:                            ${enable_cuda}
+  Hyperscan support:                       ${enable_hyperscan}
 
   Suricatasc install:                      ${enable_python}
 

src/Makefile.am

@@ -372,6 +372,7 @@ util-mpm-ac-tile.c util-mpm-ac-tile.h \
 util-mpm-ac-tile-small.c \
 util-mpm-b2g.c util-mpm-b2g.h \
 util-mpm-b3g.c util-mpm-b3g.h \
+util-mpm-hs.c util-mpm-hs.h \
 util-mpm.c util-mpm.h \
 util-mpm-wumanber.c util-mpm-wumanber.h \
 util-optimize.h \

src/detect-engine-mpm.c

@@ -1211,28 +1211,40 @@ static void PopulateMpmHelperAddPatternToPktCtx(MpmCtx *mpm_ctx,
                                                 Signature *s, uint8_t flags,
                                                 int chop)
 {
+    uint16_t pat_offset = cd->offset;
+    uint16_t pat_depth = cd->depth;
+
+    /* recompute offset/depth to cope with chop */
+    if (chop && (pat_depth || pat_offset)) {
+        pat_offset += cd->fp_chop_offset;
+        if (pat_depth) {
+            pat_depth -= cd->content_len;
+            pat_depth += cd->fp_chop_offset + cd->fp_chop_len;
+        }
+    }
+
     if (cd->flags & DETECT_CONTENT_NOCASE) {
         if (chop) {
             MpmAddPatternCI(mpm_ctx,
                             cd->content + cd->fp_chop_offset, cd->fp_chop_len,
-                            0, 0,
+                            pat_offset, pat_depth,
                             cd->id, s->num, flags);
         } else {
             MpmAddPatternCI(mpm_ctx,
                             cd->content, cd->content_len,
-                            0, 0,
+                            pat_offset, pat_depth,
                             cd->id, s->num, flags);
         }
     } else {
         if (chop) {
             MpmAddPatternCS(mpm_ctx,
                             cd->content + cd->fp_chop_offset, cd->fp_chop_len,
-                            0, 0,
+                            pat_offset, pat_depth,
                             cd->id, s->num, flags);
         } else {
             MpmAddPatternCS(mpm_ctx,
                             cd->content, cd->content_len,
-                            0, 0,
+                            pat_offset, pat_depth,
                             cd->id, s->num, flags);
         }
     }

src/detect-engine.c

@@ -1020,6 +1020,9 @@ static uint8_t DetectEngineCtxLoadConf(DetectEngineCtx *de_ctx)
         /* for now, since we still haven't implemented any intelligence into
          * understanding the patterns and distributing mpm_ctx across sgh */
         if (de_ctx->mpm_matcher == DEFAULT_MPM || de_ctx->mpm_matcher == MPM_AC_GFBS ||
+#ifdef BUILD_HYPERSCAN
+            de_ctx->mpm_matcher == MPM_HS ||
+#endif
 #ifdef __SC_CUDA_SUPPORT__
             de_ctx->mpm_matcher == MPM_AC_BS || de_ctx->mpm_matcher == MPM_AC_CUDA) {
 #else

src/runmode-unittests.c

@@ -108,6 +108,7 @@
 #include "util-memrchr.h"
 
 #include "util-mpm-ac.h"
+#include "util-mpm-hs.h"
 #include "detect-engine-mpm.h"
 
 #include "util-decode-asn1.h"
@@ -288,6 +289,9 @@ void RunUnittests(int list_unittests, char *regex_arg)
         uint32_t failed = UtRunTests(regex_arg);
         PacketPoolDestroy();
         UtCleanup();
+#ifdef BUILD_HYPERSCAN
+        MpmHSGlobalCleanup();
+#endif
 #ifdef __SC_CUDA_SUPPORT__
         if (PatternMatchDefaultMatcher() == MPM_AC_CUDA)
             MpmCudaBufferDeSetup();

src/suricata.c

@@ -194,6 +194,7 @@
 #include "util-cuda-buffer.h"
 #include "util-mpm-ac.h"
 #endif
+#include "util-mpm-hs.h"
 #include "util-storage.h"
 #include "host-storage.h"
 
@@ -2581,6 +2582,10 @@ int main(int argc, char **argv)
 
     SC_ATOMIC_DESTROY(engine_stage);
 
+#ifdef BUILD_HYPERSCAN
+    MpmHSGlobalCleanup();
+#endif
+
 #ifdef __SC_CUDA_SUPPORT__
     if (PatternMatchDefaultMatcher() == MPM_AC_CUDA)
         MpmCudaBufferDeSetup();

src/util-hash-lookup3.c

@@ -449,6 +449,180 @@ uint32_t hashlittle( const void *key, size_t length, uint32_t initval)
 }
 
 
+/*
+-------------------------------------------------------------------------------
+hashlittle_safe() -- hash a variable-length key into a 32-bit value
+  k       : the key (the unaligned variable-length array of bytes)
+  length  : the length of the key, counting by bytes
+  initval : can be any 4-byte value
+Returns a 32-bit value.  Every bit of the key affects every bit of
+the return value.  Two keys differing by one or two bits will have
+totally different hash values.
+
+The best hash table sizes are powers of 2.  There is no need to do
+mod a prime (mod is sooo slow!).  If you need less than 32 bits,
+use a bitmask.  For example, if you need only 10 bits, do
+  h = (h & hashmask(10));
+In which case, the hash table should have hashsize(10) elements.
+
+If you are hashing n strings (uint8_t **)k, do it like this:
+  for (i=0, h=0; i<n; ++i) h = hashlittle( k[i], len[i], h);
+
+By Bob Jenkins, 2006.  bob_jenkins@burtleburtle.net.  You may use this
+code any way you wish, private, educational, or commercial.  It's free.
+
+This version has been modified from hashlittle() above to avoid accesses beyond
+the last byte of the key, which causes warnings from Valgrind and Address
+Sanitizer.
+
+Use for hash table lookup, or anything where one collision in 2^^32 is
+acceptable.  Do NOT use for cryptographic purposes.
+-------------------------------------------------------------------------------
+*/
+
+uint32_t hashlittle_safe(const void *key, size_t length, uint32_t initval)
+{
+  uint32_t a,b,c;                                          /* internal state */
+  union { const void *ptr; size_t i; } u;     /* needed for Mac Powerbook G4 */
+
+  /* Set up the internal state */
+  a = b = c = 0xdeadbeef + ((uint32_t)length) + initval;
+
+  u.ptr = key;
+  if (HASH_LITTLE_ENDIAN && ((u.i & 0x3) == 0)) {
+    const uint32_t *k = (const uint32_t *)key;         /* read 32-bit chunks */
+
+    /*------ all but last block: aligned reads and affect 32 bits of (a,b,c) */
+    while (length > 12)
+    {
+      a += k[0];
+      b += k[1];
+      c += k[2];
+      mix(a,b,c);
+      length -= 12;
+      k += 3;
+    }
+
+    /*----------------------------- handle the last (probably partial) block */
+    /*
+     * Note that unlike hashlittle() above, we use the "safe" version of this
+     * block that is #ifdef VALGRIND above, in order to avoid warnings from
+     * Valgrind or Address Sanitizer.
+     */
+
+    const uint8_t  *k8 = (const uint8_t *)k;
+
+    switch(length)
+    {
+    case 12: c+=k[2]; b+=k[1]; a+=k[0]; break;
+    case 11: c+=((uint32_t)k8[10])<<16;  /* fall through */
+    case 10: c+=((uint32_t)k8[9])<<8;    /* fall through */
+    case 9 : c+=k8[8];                   /* fall through */
+    case 8 : b+=k[1]; a+=k[0]; break;
+    case 7 : b+=((uint32_t)k8[6])<<16;   /* fall through */
+    case 6 : b+=((uint32_t)k8[5])<<8;    /* fall through */
+    case 5 : b+=k8[4];                   /* fall through */
+    case 4 : a+=k[0]; break;
+    case 3 : a+=((uint32_t)k8[2])<<16;   /* fall through */
+    case 2 : a+=((uint32_t)k8[1])<<8;    /* fall through */
+    case 1 : a+=k8[0]; break;
+    case 0 : return c;
+    }
+  } else if (HASH_LITTLE_ENDIAN && ((u.i & 0x1) == 0)) {
+    const uint16_t *k = (const uint16_t *)key;         /* read 16-bit chunks */
+    const uint8_t  *k8;
+
+    /*--------------- all but last block: aligned reads and different mixing */
+    while (length > 12)
+    {
+      a += k[0] + (((uint32_t)k[1])<<16);
+      b += k[2] + (((uint32_t)k[3])<<16);
+      c += k[4] + (((uint32_t)k[5])<<16);
+      mix(a,b,c);
+      length -= 12;
+      k += 6;
+    }
+
+    /*----------------------------- handle the last (probably partial) block */
+    k8 = (const uint8_t *)k;
+    switch(length)
+    {
+    case 12: c+=k[4]+(((uint32_t)k[5])<<16);
+             b+=k[2]+(((uint32_t)k[3])<<16);
+             a+=k[0]+(((uint32_t)k[1])<<16);
+             break;
+    case 11: c+=((uint32_t)k8[10])<<16;     /* fall through */
+    case 10: c+=k[4];
+             b+=k[2]+(((uint32_t)k[3])<<16);
+             a+=k[0]+(((uint32_t)k[1])<<16);
+             break;
+    case 9 : c+=k8[8];                      /* fall through */
+    case 8 : b+=k[2]+(((uint32_t)k[3])<<16);
+             a+=k[0]+(((uint32_t)k[1])<<16);
+             break;
+    case 7 : b+=((uint32_t)k8[6])<<16;      /* fall through */
+    case 6 : b+=k[2];
+             a+=k[0]+(((uint32_t)k[1])<<16);
+             break;
+    case 5 : b+=k8[4];                      /* fall through */
+    case 4 : a+=k[0]+(((uint32_t)k[1])<<16);
+             break;
+    case 3 : a+=((uint32_t)k8[2])<<16;      /* fall through */
+    case 2 : a+=k[0];
+             break;
+    case 1 : a+=k8[0];
+             break;
+    case 0 : return c;                     /* zero length requires no mixing */
+    }
+
+  } else {                        /* need to read the key one byte at a time */
+    const uint8_t *k = (const uint8_t *)key;
+
+    /*--------------- all but the last block: affect some 32 bits of (a,b,c) */
+    while (length > 12)
+    {
+      a += k[0];
+      a += ((uint32_t)k[1])<<8;
+      a += ((uint32_t)k[2])<<16;
+      a += ((uint32_t)k[3])<<24;
+      b += k[4];
+      b += ((uint32_t)k[5])<<8;
+      b += ((uint32_t)k[6])<<16;
+      b += ((uint32_t)k[7])<<24;
+      c += k[8];
+      c += ((uint32_t)k[9])<<8;
+      c += ((uint32_t)k[10])<<16;
+      c += ((uint32_t)k[11])<<24;
+      mix(a,b,c);
+      length -= 12;
+      k += 12;
+    }
+
+    /*-------------------------------- last block: affect all 32 bits of (c) */
+    switch(length)                   /* all the case statements fall through */
+    {
+    case 12: c+=((uint32_t)k[11])<<24;
+    case 11: c+=((uint32_t)k[10])<<16;
+    case 10: c+=((uint32_t)k[9])<<8;
+    case 9 : c+=k[8];
+    case 8 : b+=((uint32_t)k[7])<<24;
+    case 7 : b+=((uint32_t)k[6])<<16;
+    case 6 : b+=((uint32_t)k[5])<<8;
+    case 5 : b+=k[4];
+    case 4 : a+=((uint32_t)k[3])<<24;
+    case 3 : a+=((uint32_t)k[2])<<16;
+    case 2 : a+=((uint32_t)k[1])<<8;
+    case 1 : a+=k[0];
+             break;
+    case 0 : return c;
+    }
+  }
+
+  final(a,b,c);
+  return c;
+}
+
+
 /*
  * hashlittle2: return 2 32-bit hash values
  *

src/util-hash-lookup3.h

@@ -52,6 +52,11 @@ void hashword2 (const uint32_t *k,          /* the key, an array of uint32_t val
 
 uint32_t hashlittle( const void *key, size_t length, uint32_t initval);
 
+/* A variant of hashlittle() that ensures avoids accesses beyond the last byte
+ * of the string, which will cause warnings from tools like Valgrind or Address
+ * Sanitizer. */
+uint32_t hashlittle_safe(const void *key, size_t length, uint32_t initval);
+
 void hashlittle2(const void *key,       /* the key to hash */
                  size_t      length,    /* length of the key */
                  uint32_t   *pc,        /* IN: primary initval, OUT: primary hash */