File data/v3

rust/src/filecontainer.rs

@@ -24,8 +24,6 @@ use crate::core::*;
 extern {
     pub fn FileFlowFlagsToFlags(flow_file_flags: u16, flags: u8) -> u16;
 }
-pub const FILE_USE_DETECT:    u16 = BIT_U16!(13);
-
 
 #[repr(C)]
 #[derive(Debug)]

rust/src/http2/http2.rs

@@ -211,8 +211,8 @@ impl HTTP2Transaction {
     }
 
     pub fn update_file_flags(&mut self, flow_file_flags: u16) {
-        self.ft_ts.file_flags = unsafe { FileFlowFlagsToFlags(flow_file_flags, STREAM_TOSERVER) | FILE_USE_DETECT };
-        self.ft_tc.file_flags = unsafe { FileFlowFlagsToFlags(flow_file_flags, STREAM_TOCLIENT) | FILE_USE_DETECT };
+        self.ft_ts.file_flags = unsafe { FileFlowFlagsToFlags(flow_file_flags, STREAM_TOSERVER) };
+        self.ft_tc.file_flags = unsafe { FileFlowFlagsToFlags(flow_file_flags, STREAM_TOCLIENT) };
     }
 
     fn decompress<'a>(

rust/src/nfs/nfs.rs

@@ -154,7 +154,7 @@ impl NFSTransactionFile {
     }
     pub fn update_file_flags(&mut self, flow_file_flags: u16) {
         let dir_flag = if self.direction == Direction::ToServer { STREAM_TOSERVER } else { STREAM_TOCLIENT };
-        self.file_tracker.file_flags = unsafe { FileFlowFlagsToFlags(flow_file_flags, dir_flag) | FILE_USE_DETECT };
+        self.file_tracker.file_flags = unsafe { FileFlowFlagsToFlags(flow_file_flags, dir_flag) };
     }
 }
 

rust/src/smb/files.rs

@@ -46,7 +46,7 @@ impl SMBTransactionFile {
 
     pub fn update_file_flags(&mut self, flow_file_flags: u16) {
         let dir_flag = if self.direction == Direction::ToServer { STREAM_TOSERVER } else { STREAM_TOCLIENT };
-        self.file_tracker.file_flags = unsafe { FileFlowFlagsToFlags(flow_file_flags, dir_flag) | FILE_USE_DETECT };
+        self.file_tracker.file_flags = unsafe { FileFlowFlagsToFlags(flow_file_flags, dir_flag) };
     }
 }
 

src/app-layer-ftp.c

@@ -1002,8 +1002,7 @@ static AppLayerResult FTPDataParse(Flow *f, FtpDataState *ftpdata_state,
         ftpdata_state->tx_data.file_tx = direction & (STREAM_TOSERVER | STREAM_TOCLIENT);
 
     /* we depend on detection engine for file pruning */
-    const uint16_t flags =
-            FileFlowFlagsToFlags(ftpdata_state->tx_data.file_flags, direction) | FILE_USE_DETECT;
+    const uint16_t flags = FileFlowFlagsToFlags(ftpdata_state->tx_data.file_flags, direction);
     int ret = 0;
 
     SCLogDebug("FTP-DATA input_len %u flags %04x dir %d/%s EOF %s", input_len, flags, direction,

src/app-layer-htp-file.c

@@ -70,6 +70,14 @@ int HTPFileOpen(HtpState *s, HtpTxUserData *tx, const uint8_t *filename, uint16_
     if (FileOpenFileWithId(files, &htp_sbcfg, s->file_track_id++, filename, filename_len, data,
                 data_len, flags) != 0) {
         retval = -1;
+    } else {
+        const HTPCfgDir *cfg;
+        if (direction & STREAM_TOCLIENT) {
+            cfg = &s->cfg->response;
+        } else {
+            cfg = &s->cfg->request;
+        }
+        FileSetInspectSizes(files->tail, cfg->inspect_window, cfg->inspect_min_size);
     }
 
     tx->tx_data.files_opened++;
@@ -160,6 +168,9 @@ int HTPFileOpenWithRange(HtpState *s, HtpTxUserData *txud, const uint8_t *filena
     if (FileOpenFileWithId(files, &htp_sbcfg, s->file_track_id++, filename, filename_len, data,
                 data_len, flags) != 0) {
         SCReturnInt(-1);
+    } else {
+        const HTPCfgDir *cfg = &s->cfg->response;
+        FileSetInspectSizes(files->tail, cfg->inspect_window, cfg->inspect_min_size);
     }
     txud->tx_data.files_opened++;
 

src/app-layer-smtp.c

@@ -496,8 +496,6 @@ int SMTPProcessDataChunk(const uint8_t *chunk, uint32_t len,
     DEBUG_VALIDATE_BUG_ON(tx == NULL);
 
     uint16_t flags = FileFlowToFlags(flow, STREAM_TOSERVER);
-    /* we depend on detection engine for file pruning */
-    flags |= FILE_USE_DETECT;
 
     /* Find file */
     if (entity->ctnt_flags & CTNT_IS_ATTACHMENT) {
@@ -1214,7 +1212,7 @@ static int SMTPProcessRequest(SMTPState *state, Flow *f, AppLayerParserState *ps
                 }
                 if (FileOpenFileWithId(&tx->files_ts, &smtp_config.sbcfg, state->file_track_id++,
                             (uint8_t *)rawmsgname, strlen(rawmsgname), NULL, 0,
-                            FILE_NOMD5 | FILE_NOMAGIC | FILE_USE_DETECT) == 0) {
+                            FILE_NOMD5 | FILE_NOMAGIC) == 0) {
                     SMTPNewFile(tx, tx->files_ts.tail);
                 }
             } else if (smtp_config.decode_mime) {

src/detect-engine-file.c

@@ -49,30 +49,6 @@
 #include "util-profiling.h"
 #include "util-validate.h"
 
-FileAppProto file_protos_ts_static[] = {
-    { ALPROTO_HTTP1, HTP_REQUEST_BODY },
-    { ALPROTO_SMTP, 0 },
-    { ALPROTO_FTP, 0 },
-    { ALPROTO_FTPDATA, 0 },
-    { ALPROTO_SMB, 0 },
-    { ALPROTO_NFS, 0 },
-    { ALPROTO_HTTP2, HTTP2StateDataClient },
-    { ALPROTO_UNKNOWN, 0 },
-};
-
-FileAppProto file_protos_tc_static[] = {
-    { ALPROTO_HTTP1, HTP_RESPONSE_BODY },
-    { ALPROTO_FTP, 0 },
-    { ALPROTO_FTPDATA, 0 },
-    { ALPROTO_SMB, 0 },
-    { ALPROTO_NFS, 0 },
-    { ALPROTO_HTTP2, HTTP2StateDataServer },
-    { ALPROTO_UNKNOWN, 0 },
-};
-
-FileAppProto *file_protos_ts = file_protos_ts_static;
-FileAppProto *file_protos_tc = file_protos_tc_static;
-
 /**
  *  \brief Inspect the file inspecting keywords.
  *

src/detect-engine-file.h

@@ -28,12 +28,4 @@ uint8_t DetectFileInspectGeneric(DetectEngineCtx *de_ctx, DetectEngineThreadCtx
         const struct DetectEngineAppInspectionEngine_ *engine, const Signature *s, Flow *f,
         uint8_t flags, void *_alstate, void *tx, uint64_t tx_id);
 
-typedef struct FileAppProto {
-    AppProto alproto;
-    int progress;
-} FileAppProto;
-
-extern FileAppProto *file_protos_ts;
-extern FileAppProto *file_protos_tc;
-
 #endif /* __DETECT_ENGINE_FILE_H__ */

src/detect-engine-mpm.c

@@ -87,9 +87,8 @@ static int g_mpm_list_cnt[DETECT_BUFFER_MPM_TYPE_SIZE] = { 0, 0, 0 };
  *  \note to be used at start up / registration only. Errors are fatal.
  */
 void DetectAppLayerMpmRegister2(const char *name, int direction, int priority,
-        int (*PrefilterRegister)(DetectEngineCtx *de_ctx, SigGroupHead *sgh, MpmCtx *mpm_ctx,
-                const DetectBufferMpmRegistry *mpm_reg, int list_id),
-        InspectionBufferGetDataPtr GetData, AppProto alproto, int tx_min_progress)
+        PrefilterRegisterFunc PrefilterRegister, InspectionBufferGetDataPtr GetData,
+        AppProto alproto, int tx_min_progress)
 {
     SCLogDebug("registering %s/%d/%d/%p/%p/%u/%d", name, direction, priority,
             PrefilterRegister, GetData, alproto, tx_min_progress);
@@ -1571,19 +1570,9 @@ static void MpmStoreSetup(const DetectEngineCtx *de_ctx, MpmStore *ms)
     for (sig = 0; sig < (ms->sid_array_size * 8); sig++) {
         if (ms->sid_array[sig / 8] & (1 << (sig % 8))) {
             s = de_ctx->sig_array[sig];
+            DEBUG_VALIDATE_BUG_ON(s == NULL);
             if (s == NULL)
                 continue;
-            if ((s->flags & ms->direction) == 0) {
-                SCLogDebug("s->flags %x ms->direction %x", s->flags, ms->direction);
-                continue;
-            }
-            if (s->init_data->mpm_sm == NULL)
-                continue;
-            int list = s->init_data->mpm_sm_list;
-            if (list < 0)
-                continue;
-            if (list != ms->sm_list)
-                continue;
 
             SCLogDebug("%p: direction %d adding %u", ms, ms->direction, s->id);
 

src/detect-engine-mpm.h

@@ -75,6 +75,9 @@ MpmStore *MpmStorePrepareBuffer(DetectEngineCtx *de_ctx, SigGroupHead *sgh, enum
  */
 int DetectSetFastPatternAndItsId(DetectEngineCtx *de_ctx);
 
+typedef int (*PrefilterRegisterFunc)(DetectEngineCtx *de_ctx, SigGroupHead *sgh, MpmCtx *mpm_ctx,
+        const DetectBufferMpmRegistry *mpm_reg, int list_id);
+
 /** \brief register an app layer keyword for mpm
  *  \param name buffer name
  *  \param direction SIG_FLAG_TOSERVER or SIG_FLAG_TOCLIENT
@@ -88,17 +91,14 @@ int DetectSetFastPatternAndItsId(DetectEngineCtx *de_ctx);
  *        If both are needed, register the keyword twice.
  */
 void DetectAppLayerMpmRegister2(const char *name, int direction, int priority,
-        int (*PrefilterRegister)(DetectEngineCtx *de_ctx, SigGroupHead *sgh, MpmCtx *mpm_ctx,
-                const DetectBufferMpmRegistry *mpm_reg, int list_id),
-        InspectionBufferGetDataPtr GetData, AppProto alproto, int tx_min_progress);
+        PrefilterRegisterFunc PrefilterRegister, InspectionBufferGetDataPtr GetData,
+        AppProto alproto, int tx_min_progress);
 void DetectAppLayerMpmRegisterByParentId(
         DetectEngineCtx *de_ctx,
         const int id, const int parent_id,
         DetectEngineTransforms *transforms);
 
-void DetectPktMpmRegister(const char *name, int priority,
-        int (*PrefilterRegister)(DetectEngineCtx *de_ctx, SigGroupHead *sgh, MpmCtx *mpm_ctx,
-                const DetectBufferMpmRegistry *mpm_reg, int list_id),
+void DetectPktMpmRegister(const char *name, int priority, PrefilterRegisterFunc PrefilterRegister,
         InspectionBufferGetPktDataPtr GetData);
 void DetectPktMpmRegisterByParentId(DetectEngineCtx *de_ctx,
         const int id, const int parent_id,

src/detect-engine-register.c

@@ -447,6 +447,14 @@ int SigTableList(const char *keyword)
     return TM_ECODE_DONE;
 }
 
+static void DetectFileHandlerRegister(void)
+{
+    for (int i = 0; i < DETECT_TBLSIZE; i++) {
+        if (filehandler_table[i].name)
+            DetectFileRegisterFileProtocols(&filehandler_table[i]);
+    }
+}
+
 void SigTableSetup(void)
 {
     memset(sigmatch_table, 0, sizeof(sigmatch_table));
@@ -689,6 +697,8 @@ void SigTableSetup(void)
     DetectTransformUrlDecodeRegister();
     DetectTransformXorRegister();
 
+    DetectFileHandlerRegister();
+
     /* close keyword registration */
     DetectBufferTypeCloseRegistration();
 }

src/detect-engine.c

@@ -708,7 +708,7 @@ static void AppendAppInspectEngine(DetectEngineCtx *de_ctx,
         } else {
             new_engine->id = DE_STATE_FLAG_BASE; /* id is used as flag in stateful detect */
             SCLogDebug("sid %u: engine %p/%u %s", s->id, new_engine, new_engine->id,
-                    DetectEngineBufferTypeGetNameById(de_ctx, t->sm_list));
+                    DetectEngineBufferTypeGetNameById(de_ctx, new_engine->sm_list));
         }
 
         /* prepend engine if forced or if our engine has a lower progress. */
@@ -721,7 +721,7 @@ static void AppendAppInspectEngine(DetectEngineCtx *de_ctx,
         } else {
             new_engine->id = ++(*last_id);
             SCLogDebug("sid %u: engine %p/%u %s", s->id, new_engine, new_engine->id,
-                    DetectEngineBufferTypeGetNameById(de_ctx, t->sm_list));
+                    DetectEngineBufferTypeGetNameById(de_ctx, new_engine->sm_list));
         }
 
     } else {
@@ -741,7 +741,7 @@ static void AppendAppInspectEngine(DetectEngineCtx *de_ctx,
         } else {
             new_engine->id = ++(*last_id);
             SCLogDebug("sid %u: engine %p/%u %s", s->id, new_engine, new_engine->id,
-                    DetectEngineBufferTypeGetNameById(de_ctx, t->sm_list));
+                    DetectEngineBufferTypeGetNameById(de_ctx, new_engine->sm_list));
         }
     }