detect: absent keyword to test absence of sticky buffer

Ticket: 2224

It takes an argument to match only if the buffer is absent,
or it can still match if the buffer is present, but we test
the absence of some content

doc/userguide/rules/payload-keywords.rst

@@ -274,6 +274,28 @@ You can also use the negation (!) before isdataat.
 
 .. image:: payload-keywords/isdataat1.png
 
+absent
+------
+
+The keyword ``absent`` checks that a sticky buffer does not exist.
+It can be used without any argument to match only on absent buffer :
+
+Example of ``absent`` in a rule:
+
+.. container:: example-rule
+
+   alert http any any -> any any (msg:"HTTP request without referer";  http.referer; absent; sid:1; rev:1;)
+
+
+It can take an argument "or_else" to match on absent buffer or on what comes next such as negated content, for instance :
+
+.. container:: example-rule
+
+   alert http any any -> any any (msg:"HTTP request without referer";  http.referer; absent: or_else; content: !"abc"; sid:1; rev:1;)
+
+An absent keyword cannot work with positive (non negated) content or pcre.
+For files (ie ``file.data``), absent means there are no files in the transaction.
+
 bsize
 -----
 

src/detect-engine-content-inspection.c

@@ -382,6 +382,14 @@ static int DetectEngineContentInspectionInternal(DetectEngineThreadCtx *det_ctx,
                     prev_offset);
         } while(1);
 
+    } else if (smd->type == DETECT_ABSENT) {
+        const DetectAbsentData *id = (DetectAbsentData *)smd->ctx;
+        if (!id->or_else) {
+            // we match only on absent buffer
+            goto no_match;
+        } else {
+            goto match;
+        }
     } else if (smd->type == DETECT_ISDATAAT) {
         SCLogDebug("inspecting isdataat");
 
@@ -648,8 +656,7 @@ static int DetectEngineContentInspectionInternal(DetectEngineThreadCtx *det_ctx,
             goto match;
         }
         goto no_match_discontinue;
-    }
-    else if (smd->type == DETECT_LUA) {
+    } else if (smd->type == DETECT_LUA) {
         SCLogDebug("lua starting");
 
         if (DetectLuaMatchBuffer(det_ctx, s, smd, buffer, buffer_len,
@@ -760,6 +767,24 @@ bool DetectEngineContentInspectionBuffer(DetectEngineCtx *de_ctx, DetectEngineTh
         return false;
 }
 
+bool DetectContentInspectionMatchOnAbsentBuffer(const SigMatchData *smd)
+{
+    // we will match on NULL buffers there is one absent
+    bool absent_data = false;
+    while (1) {
+        if (smd->type == DETECT_ABSENT) {
+            absent_data = true;
+            break;
+        }
+        if (smd->is_last) {
+            break;
+        }
+        // smd does not get reused after this loop
+        smd++;
+    }
+    return absent_data;
+}
+
 #ifdef UNITTESTS
 #include "tests/detect-engine-content-inspection.c"
 #endif

src/detect-engine-content-inspection.h

@@ -68,6 +68,12 @@ bool DetectEngineContentInspectionBuffer(DetectEngineCtx *de_ctx, DetectEngineTh
         const Signature *s, const SigMatchData *smd, Packet *p, Flow *f, const InspectionBuffer *b,
         const enum DetectContentInspectionType inspection_mode);
 
+/** \brief tells if we should match on absent buffer, because
+ *  there is an absent keyword being used
+ *  \param smd array of content inspection matches
+ *  \retval bool true to match on absent buffer, false otherwise */
+bool DetectContentInspectionMatchOnAbsentBuffer(const SigMatchData *smd);
+
 void DetectEngineContentInspectionRegisterTests(void);
 
 #endif /* SURICATA_DETECT_ENGINE_CONTENT_INSPECTION_H */

src/detect-engine-register.h

@@ -80,6 +80,7 @@ enum DetectKeywordId {
     DETECT_IPPROTO,
     DETECT_FTPBOUNCE,
     DETECT_ISDATAAT,
+    DETECT_ABSENT,
     DETECT_ID,
     DETECT_RPC,
     DETECT_FLOWVAR,

src/detect-engine.c

@@ -659,6 +659,7 @@ static void AppendAppInspectEngine(DetectEngineCtx *de_ctx,
     new_engine->sm_list = t->sm_list;
     new_engine->sm_list_base = t->sm_list_base;
     new_engine->smd = smd;
+    new_engine->match_on_null = DetectContentInspectionMatchOnAbsentBuffer(smd);
     new_engine->progress = t->progress;
     new_engine->v2 = t->v2;
     SCLogDebug("sm_list %d new_engine->v2 %p/%p/%p", new_engine->sm_list, new_engine->v2.Callback,
@@ -2153,6 +2154,9 @@ uint8_t DetectEngineInspectBufferGeneric(DetectEngineCtx *de_ctx, DetectEngineTh
     const InspectionBuffer *buffer = engine->v2.GetData(det_ctx, transforms,
             f, flags, txv, list_id);
     if (unlikely(buffer == NULL)) {
+        if (eof && engine->match_on_null) {
+            return DETECT_ENGINE_INSPECT_SIG_MATCH;
+        }
         return eof ? DETECT_ENGINE_INSPECT_SIG_CANT_MATCH :
                      DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
     }
@@ -2214,6 +2218,13 @@ uint8_t DetectEngineInspectMultiBufferGeneric(DetectEngineCtx *de_ctx,
         }
         local_id++;
     } while (1);
+    if (local_id == 0) {
+        const bool eof = (AppLayerParserGetStateProgress(f->proto, f->alproto, txv, flags) >
+                          engine->progress);
+        if (eof && engine->match_on_null) {
+            return DETECT_ENGINE_INSPECT_SIG_MATCH;
+        }
+    }
     return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
 }
 

src/detect-file-data.c

@@ -396,7 +396,15 @@ uint8_t DetectEngineInspectFiledata(DetectEngineCtx *de_ctx, DetectEngineThreadC
 
     AppLayerGetFileState files = AppLayerParserGetTxFiles(f, txv, flags);
     FileContainer *ffc = files.fc;
-    if (ffc == NULL) {
+    if (ffc == NULL || ffc->head == NULL) {
+        const bool eof = (AppLayerParserGetStateProgress(f->proto, f->alproto, txv, flags) >
+                          engine->progress);
+        if (eof && engine->match_on_null) {
+            return DETECT_ENGINE_INSPECT_SIG_MATCH;
+        }
+        if (ffc != NULL) {
+            return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
+        }
         return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES;
     }
 

src/detect-filemagic.c

@@ -307,7 +307,15 @@ static uint8_t DetectEngineInspectFilemagic(DetectEngineCtx *de_ctx, DetectEngin
 
     AppLayerGetFileState files = AppLayerParserGetTxFiles(f, txv, flags);
     FileContainer *ffc = files.fc;
-    if (ffc == NULL) {
+    if (ffc == NULL || ffc->head == NULL) {
+        const bool eof = (AppLayerParserGetStateProgress(f->proto, f->alproto, txv, flags) >
+                          engine->progress);
+        if (eof && engine->match_on_null) {
+            return DETECT_ENGINE_INSPECT_SIG_MATCH;
+        }
+        if (ffc != NULL) {
+            return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
+        }
         return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES;
     }
 

src/detect-filename.c

@@ -244,7 +244,15 @@ static uint8_t DetectEngineInspectFilename(DetectEngineCtx *de_ctx, DetectEngine
 
     AppLayerGetFileState files = AppLayerParserGetTxFiles(f, txv, flags);
     FileContainer *ffc = files.fc;
-    if (ffc == NULL) {
+    if (ffc == NULL || ffc->head == NULL) {
+        const bool eof = (AppLayerParserGetStateProgress(f->proto, f->alproto, txv, flags) >
+                          engine->progress);
+        if (eof && engine->match_on_null) {
+            return DETECT_ENGINE_INSPECT_SIG_MATCH;
+        }
+        if (ffc != NULL) {
+            return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
+        }
         return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES;
     }
 

src/detect-http-client-body.c

@@ -312,6 +312,9 @@ static uint8_t DetectEngineInspectBufferHttpBody(DetectEngineCtx *de_ctx,
     const InspectionBuffer *buffer = HttpRequestBodyGetDataCallback(
             det_ctx, engine->v2.transforms, f, flags, txv, engine->sm_list, engine->sm_list_base);
     if (buffer == NULL || buffer->inspect == NULL) {
+        if (eof && engine->match_on_null) {
+            return DETECT_ENGINE_INSPECT_SIG_MATCH;
+        }
         return eof ? DETECT_ENGINE_INSPECT_SIG_CANT_MATCH : DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
     }
 

src/detect-http-header.c

@@ -176,6 +176,8 @@ static uint8_t DetectEngineInspectBufferHttpHeader(DetectEngineCtx *de_ctx,
 
     const int list_id = engine->sm_list;
     InspectionBuffer *buffer = InspectionBufferGet(det_ctx, list_id);
+    bool eof =
+            (AppLayerParserGetStateProgress(f->proto, f->alproto, txv, flags) > engine->progress);
     if (buffer->inspect == NULL) {
         SCLogDebug("setting up inspect buffer %d", list_id);
 
@@ -189,6 +191,9 @@ static uint8_t DetectEngineInspectBufferHttpHeader(DetectEngineCtx *de_ctx,
         uint8_t *rawdata = GetBufferForTX(txv, det_ctx, f, flags, &rawdata_len);
         if (rawdata_len == 0) {
             SCLogDebug("no data");
+            if (engine->match_on_null && eof) {
+                return DETECT_ENGINE_INSPECT_SIG_MATCH;
+            }
             goto end;
         }
         /* setup buffer and apply transforms */
@@ -209,14 +214,8 @@ static uint8_t DetectEngineInspectBufferHttpHeader(DetectEngineCtx *de_ctx,
         return DETECT_ENGINE_INSPECT_SIG_MATCH;
     }
 end:
-    if (flags & STREAM_TOSERVER) {
-        if (AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP1, txv, flags) >
-                HTP_REQUEST_HEADERS)
-            return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH;
-    } else {
-        if (AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP1, txv, flags) >
-                HTP_RESPONSE_HEADERS)
-            return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH;
+    if (eof) {
+        return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH;
     }
     return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
 }