Thresholds/v20 backoff/v1

doc/userguide/configuration/global-thresholds.rst

@@ -6,6 +6,9 @@ Thresholds can be configured in the rules themselves, see
 their intelligence for creating a rule combined with a judgement on how often
 a rule will alert.
 
+Thresholds are tracked in a hash table that is sized according to configuration, see:
+:ref:`suricata-yaml-thresholds`.
+
 Threshold Config
 ----------------
 
@@ -20,7 +23,7 @@ Syntax:
 ::
 
   threshold gen_id <gid>, sig_id <sid>, type <threshold|limit|both>, \
-    track <by_src|by_dst|by_rule|by_both>, count <N>, seconds <T>
+    track <by_src|by_dst|by_rule|by_both|by_flow>, count <N>, seconds <T>
 
 rate_filter
 ~~~~~~~~~~~
@@ -55,6 +58,7 @@ done per IP-address. The Host table is used for storage. When using by_rule
 it's done globally for the rule.
 Option by_both used to track per IP pair of source and destination. Packets
 going to opposite directions between same addresses tracked as the same pair.
+The by_flow option tracks the rule matches in the flow.
 
 count
 ^^^^^

doc/userguide/configuration/suricata-yaml.rst

@@ -739,6 +739,22 @@ To let Suricata make these decisions set default to 'auto':
     prefilter:
       default: auto
 
+.. _suricata-yaml-thresholds:
+
+Thresholding Settings
+~~~~~~~~~~~~~~~~~~~~~
+
+Thresholding uses a central hash table for tracking thresholds of the types: by_src, by_dst, by_both.
+
+::
+
+  detect:
+    thresholds:
+      hash-size: 16384
+      memcap: 16mb
+
+``detect.thresholds.hash-size`` controls the number of hash rows in the hash table.
+``detect.thresholds.memcap`` controls how much memory can be used for the hash table and the data stored in it.
 
 Pattern matcher settings
 ~~~~~~~~~~~~~~~~~~~~~~~~

doc/userguide/rules/thresholding.rst

@@ -4,6 +4,9 @@ Thresholding Keywords
 Thresholding can be configured per rule and also globally, see
 :doc:`../configuration/global-thresholds`.
 
+Thresholds are tracked in a hash table that is sized according to configuration, see:
+:ref:`suricata-yaml-thresholds`.
+
 *Note: mixing rule and global thresholds is not supported in 1.3 and
 before. See bug #425.* For the state of the support in 1.4 see
 :ref:`global-thresholds-vs-rule-thresholds`
@@ -16,7 +19,7 @@ frequency. It has 3 modes: threshold, limit and both.
 
 Syntax::
 
-  threshold: type <threshold|limit|both>, track <by_src|by_dst|by_rule|by_both>, count <N>, seconds <T>
+  threshold: type <threshold|limit|both>, track <by_src|by_dst|by_rule|by_both|by_flow>, count <N>, seconds <T>
 
 type "threshold"
 ~~~~~~~~~~~~~~~~
@@ -85,6 +88,27 @@ performed for each of the matches.
 
   *Rule actions drop (IPS mode) and reject are applied to each packet.*
 
+
+track
+~~~~~
+
+.. table::
+
+   +------------------+--------------------------+
+   |Option            |Tracks By                 |
+   +==================+==========================+
+   |by_src            |source IP                 |
+   +------------------+--------------------------+
+   |by_dst            |destination IP            |
+   +------------------+--------------------------+
+   |by_both           |pair of src IP and dst IP |
+   +------------------+--------------------------+
+   |by_rule           |signature id              |
+   +------------------+--------------------------+
+   |by_flow           |flow                      |
+   +------------------+--------------------------+
+
+
 detection_filter
 ----------------
 
@@ -97,7 +121,7 @@ again.
 
 Syntax::
 
-  detection_filter: track <by_src|by_dst|by_rule|by_both>, count <N>, seconds <T>
+  detection_filter: track <by_src|by_dst|by_rule|by_both|by_flow>, count <N>, seconds <T>
 
 Example::
 

src/app-layer-htp-range.c

@@ -127,14 +127,13 @@ static void ContainerUrlRangeFree(void *s)
     }
 }
 
-static inline bool ContainerValueRangeTimeout(HttpRangeContainerFile *cu, const SCTime_t ts)
+static inline bool ContainerValueRangeTimeout(void *data, const SCTime_t ts)
 {
+    HttpRangeContainerFile *cu = data;
     // we only timeout if we have no flow referencing us
     if (SCTIME_CMP_GT(ts, cu->expire) || cu->error) {
-        if (SC_ATOMIC_GET(cu->hdata->use_cnt) == 0) {
-            DEBUG_VALIDATE_BUG_ON(cu->files == NULL);
-            return true;
-        }
+        DEBUG_VALIDATE_BUG_ON(cu->files == NULL);
+        return true;
     }
     return false;
 }
@@ -171,10 +170,10 @@ void HttpRangeContainersInit(void)
         }
     }
 
-    ContainerUrlRangeList.ht =
-            THashInit("app-layer.protocols.http.byterange", sizeof(HttpRangeContainerFile),
-                    ContainerUrlRangeSet, ContainerUrlRangeFree, ContainerUrlRangeHash,
-                    ContainerUrlRangeCompare, false, memcap, CONTAINER_URLRANGE_HASH_SIZE);
+    ContainerUrlRangeList.ht = THashInit("app-layer.protocols.http.byterange",
+            sizeof(HttpRangeContainerFile), ContainerUrlRangeSet, ContainerUrlRangeFree,
+            ContainerUrlRangeHash, ContainerUrlRangeCompare, ContainerValueRangeTimeout, false,
+            memcap, CONTAINER_URLRANGE_HASH_SIZE);
     ContainerUrlRangeList.timeout = timeout;
 
     SCLogDebug("containers started");
@@ -187,49 +186,7 @@ void HttpRangeContainersDestroy(void)
 
 uint32_t HttpRangeContainersTimeoutHash(const SCTime_t ts)
 {
-    SCLogDebug("timeout: starting");
-    uint32_t cnt = 0;
-
-    for (uint32_t i = 0; i < ContainerUrlRangeList.ht->config.hash_size; i++) {
-        THashHashRow *hb = &ContainerUrlRangeList.ht->array[i];
-
-        if (HRLOCK_TRYLOCK(hb) != 0)
-            continue;
-        /* hash bucket is now locked */
-        THashData *h = hb->head;
-        while (h) {
-            DEBUG_VALIDATE_BUG_ON(SC_ATOMIC_GET(h->use_cnt) > (uint32_t)INT_MAX);
-            THashData *n = h->next;
-            THashDataLock(h);
-            if (ContainerValueRangeTimeout(h->data, ts)) {
-                /* remove from the hash */
-                if (h->prev != NULL)
-                    h->prev->next = h->next;
-                if (h->next != NULL)
-                    h->next->prev = h->prev;
-                if (hb->head == h)
-                    hb->head = h->next;
-                if (hb->tail == h)
-                    hb->tail = h->prev;
-                h->next = NULL;
-                h->prev = NULL;
-                // we should log the timed out file somehow...
-                // but it does not belong to any flow...
-                SCLogDebug("timeout: removing range %p", h);
-                ContainerUrlRangeFree(h->data); // TODO do we need a "RECYCLE" func?
-                DEBUG_VALIDATE_BUG_ON(SC_ATOMIC_GET(h->use_cnt) > (uint32_t)INT_MAX);
-                THashDataUnlock(h);
-                THashDataMoveToSpare(ContainerUrlRangeList.ht, h);
-            } else {
-                THashDataUnlock(h);
-            }
-            h = n;
-        }
-        HRLOCK_UNLOCK(hb);
-    }
-
-    SCLogDebug("timeout: ending");
-    return cnt;
+    return THashExpire(ContainerUrlRangeList.ht, ts);
 }
 
 /**

src/datasets.c

@@ -701,7 +701,7 @@ Dataset *DatasetGet(const char *name, enum DatasetTypes type, const char *save,
     switch (type) {
         case DATASET_TYPE_MD5:
             set->hash = THashInit(cnf_name, sizeof(Md5Type), Md5StrSet, Md5StrFree, Md5StrHash,
-                    Md5StrCompare, load != NULL ? 1 : 0, memcap > 0 ? memcap : default_memcap,
+                    Md5StrCompare, NULL, load != NULL ? 1 : 0, memcap > 0 ? memcap : default_memcap,
                     hashsize > 0 ? hashsize : default_hashsize);
             if (set->hash == NULL)
                 goto out_err;
@@ -710,7 +710,7 @@ Dataset *DatasetGet(const char *name, enum DatasetTypes type, const char *save,
             break;
         case DATASET_TYPE_STRING:
             set->hash = THashInit(cnf_name, sizeof(StringType), StringSet, StringFree, StringHash,
-                    StringCompare, load != NULL ? 1 : 0, memcap > 0 ? memcap : default_memcap,
+                    StringCompare, NULL, load != NULL ? 1 : 0, memcap > 0 ? memcap : default_memcap,
                     hashsize > 0 ? hashsize : default_hashsize);
             if (set->hash == NULL)
                 goto out_err;
@@ -719,7 +719,7 @@ Dataset *DatasetGet(const char *name, enum DatasetTypes type, const char *save,
             break;
         case DATASET_TYPE_SHA256:
             set->hash = THashInit(cnf_name, sizeof(Sha256Type), Sha256StrSet, Sha256StrFree,
-                    Sha256StrHash, Sha256StrCompare, load != NULL ? 1 : 0,
+                    Sha256StrHash, Sha256StrCompare, NULL, load != NULL ? 1 : 0,
                     memcap > 0 ? memcap : default_memcap,
                     hashsize > 0 ? hashsize : default_hashsize);
             if (set->hash == NULL)
@@ -729,7 +729,7 @@ Dataset *DatasetGet(const char *name, enum DatasetTypes type, const char *save,
             break;
         case DATASET_TYPE_IPV4:
             set->hash = THashInit(cnf_name, sizeof(IPv4Type), IPv4Set, IPv4Free, IPv4Hash,
-                    IPv4Compare, load != NULL ? 1 : 0, memcap > 0 ? memcap : default_memcap,
+                    IPv4Compare, NULL, load != NULL ? 1 : 0, memcap > 0 ? memcap : default_memcap,
                     hashsize > 0 ? hashsize : default_hashsize);
             if (set->hash == NULL)
                 goto out_err;
@@ -738,7 +738,7 @@ Dataset *DatasetGet(const char *name, enum DatasetTypes type, const char *save,
             break;
         case DATASET_TYPE_IPV6:
             set->hash = THashInit(cnf_name, sizeof(IPv6Type), IPv6Set, IPv6Free, IPv6Hash,
-                    IPv6Compare, load != NULL ? 1 : 0, memcap > 0 ? memcap : default_memcap,
+                    IPv6Compare, NULL, load != NULL ? 1 : 0, memcap > 0 ? memcap : default_memcap,
                     hashsize > 0 ? hashsize : default_hashsize);
             if (set->hash == NULL)
                 goto out_err;

src/detect-content.c

@@ -556,7 +556,7 @@ static void PropagateLimits(Signature *s, SigMatch *sm_head)
                 SCLogDebug("stored: offset %u depth %u offset_plus_pat %u", offset, depth,
                         offset_plus_pat);
 
-                if ((cd->flags & (DETECT_DEPTH | DETECT_CONTENT_WITHIN)) == 0) {
+                if ((cd->flags & (DETECT_CONTENT_DEPTH | DETECT_CONTENT_WITHIN)) == 0) {
                     if (depth)
                         SCLogDebug("no within, reset depth");
                     depth = 0;

src/detect-detection-filter.c

@@ -47,8 +47,9 @@
  *\brief Regex for parsing our detection_filter options
  */
 #define PARSE_REGEX                                                                                \
-    "^\\s*(track|count|seconds)\\s+(by_src|by_dst|\\d+)\\s*,\\s*(track|count|seconds)\\s+(by_src|" \
-    "by_dst|\\d+)\\s*,\\s*(track|count|seconds)\\s+(by_src|by_dst|\\d+)\\s*$"
+    "^\\s*(track|count|seconds)\\s+(by_src|by_dst|by_flow|\\d+)\\s*,\\s*(track|count|seconds)\\s+" \
+    "(by_src|"                                                                                     \
+    "by_dst|by_flow|\\d+)\\s*,\\s*(track|count|seconds)\\s+(by_src|by_dst|by_flow|\\d+)\\s*$"
 
 static DetectParseRegex parse_regex;
 
@@ -158,6 +159,8 @@ static DetectThresholdData *DetectDetectionFilterParse(const char *rawstr)
             df->track = TRACK_DST;
         if (strncasecmp(args[i], "by_src", strlen("by_src")) == 0)
             df->track = TRACK_SRC;
+        if (strncasecmp(args[i], "by_flow", strlen("by_flow")) == 0)
+            df->track = TRACK_FLOW;
         if (strncasecmp(args[i], "count", strlen("count")) == 0)
             count_pos = i + 1;
         if (strncasecmp(args[i], "seconds", strlen("seconds")) == 0)
@@ -375,7 +378,7 @@ static int DetectDetectionFilterTestSig1(void)
     ThreadVars th_v;
     DetectEngineThreadCtx *det_ctx;
 
-    HostInitConfig(HOST_QUIET);
+    ThresholdInit();
 
     memset(&th_v, 0, sizeof(th_v));
 
@@ -415,7 +418,7 @@ static int DetectDetectionFilterTestSig1(void)
     DetectEngineCtxFree(de_ctx);
 
     UTHFreePackets(&p, 1);
-    HostShutdown();
+    ThresholdDestroy();
 
     PASS;
 }
@@ -432,7 +435,7 @@ static int DetectDetectionFilterTestSig2(void)
     ThreadVars th_v;
     DetectEngineThreadCtx *det_ctx;
 
-    HostInitConfig(HOST_QUIET);
+    ThresholdInit();
 
     memset(&th_v, 0, sizeof(th_v));
 
@@ -477,7 +480,7 @@ static int DetectDetectionFilterTestSig2(void)
     DetectEngineCtxFree(de_ctx);
 
     UTHFreePackets(&p, 1);
-    HostShutdown();
+    ThresholdDestroy();
 
     PASS;
 }
@@ -490,7 +493,7 @@ static int DetectDetectionFilterTestSig3(void)
     ThreadVars th_v;
     DetectEngineThreadCtx *det_ctx;
 
-    HostInitConfig(HOST_QUIET);
+    ThresholdInit();
 
     memset(&th_v, 0, sizeof(th_v));
 
@@ -553,7 +556,7 @@ static int DetectDetectionFilterTestSig3(void)
     DetectEngineCtxFree(de_ctx);
 
     UTHFreePackets(&p, 1);
-    HostShutdown();
+    ThresholdDestroy();
 
     PASS;
 }

src/detect-engine-address-ipv6.c

@@ -49,7 +49,7 @@
  * \retval 1 If a < b.
  * \retval 0 Otherwise, i.e. a >= b.
  */
-int AddressIPv6Lt(Address *a, Address *b)
+int AddressIPv6Lt(const Address *a, const Address *b)
 {
     int i = 0;
 
@@ -87,7 +87,7 @@ int AddressIPv6LtU32(uint32_t *a, uint32_t *b)
  * \retval 1 If a > b.
  * \retval 0 Otherwise, i.e. a <= b.
  */
-int AddressIPv6Gt(Address *a, Address *b)
+int AddressIPv6Gt(const Address *a, const Address *b)
 {
     int i = 0;
 
@@ -125,7 +125,7 @@ int AddressIPv6GtU32(uint32_t *a, uint32_t *b)
  * \retval 1 If a == b.
  * \retval 0 Otherwise.
  */
-int AddressIPv6Eq(Address *a, Address *b)
+int AddressIPv6Eq(const Address *a, const Address *b)
 {
     int i = 0;
 
@@ -159,7 +159,7 @@ int AddressIPv6EqU32(uint32_t *a, uint32_t *b)
  * \retval 1 If a <= b.
  * \retval 0 Otherwise, i.e. a > b.
  */
-int AddressIPv6Le(Address *a, Address *b)
+int AddressIPv6Le(const Address *a, const Address *b)
 {
 
     if (AddressIPv6Eq(a, b) == 1)
@@ -191,7 +191,7 @@ int AddressIPv6LeU32(uint32_t *a, uint32_t *b)
  * \retval 1 If a >= b.
  * \retval 0 Otherwise, i.e. a < b.
  */
-int AddressIPv6Ge(Address *a, Address *b)
+int AddressIPv6Ge(const Address *a, const Address *b)
 {
 
     if (AddressIPv6Eq(a, b) == 1)

src/detect-engine-address-ipv6.h

@@ -24,11 +24,11 @@
 #ifndef SURICATA_DETECT_ENGINE_ADDRESS_IPV6_H
 #define SURICATA_DETECT_ENGINE_ADDRESS_IPV6_H
 
-int AddressIPv6Lt(Address *, Address *);
-int AddressIPv6Gt(Address *, Address *);
-int AddressIPv6Eq(Address *, Address *);
-int AddressIPv6Le(Address *, Address *);
-int AddressIPv6Ge(Address *, Address *);
+int AddressIPv6Lt(const Address *, const Address *);
+int AddressIPv6Gt(const Address *, const Address *);
+int AddressIPv6Eq(const Address *, const Address *);
+int AddressIPv6Le(const Address *, const Address *);
+int AddressIPv6Ge(const Address *, const Address *);
 
 int AddressIPv6LeU32(uint32_t *a, uint32_t *b);
 int AddressIPv6LtU32(uint32_t *a, uint32_t *b);

src/detect-engine-build.c

@@ -2219,8 +2219,6 @@ int SigGroupBuild(DetectEngineCtx *de_ctx)
     SCProfilingRuleInitCounters(de_ctx);
 #endif
 
-    ThresholdHashAllocate(de_ctx);
-
     if (!DetectEngineMultiTenantEnabled()) {
         VarNameStoreActivate();
     }