Ips delayed v1.4

src/app-layer-htp.c

@@ -95,16 +95,6 @@ static uint64_t htp_state_memuse = 0;
 static uint64_t htp_state_memcnt = 0;
 #endif
 
-/** part of the engine needs the request body (e.g. http_client_body keyword) */
-uint8_t need_htp_request_body = 0;
-/** part of the engine needs the request body multipart header (e.g. filename
- *  and / or fileext keywords) */
-uint8_t need_htp_request_multipart_hdr = 0;
-/** part of the engine needs the request file (e.g. log-file module) */
-uint8_t need_htp_request_file = 0;
-/** part of the engine needs the request body (e.g. file_data keyword) */
-uint8_t need_htp_response_body = 0;
-
 SCEnumCharMap http_decoder_event_table[ ] = {
     { "UNKNOWN_ERROR",
         HTTP_DECODER_EVENT_UNKNOWN_ERROR},
@@ -356,7 +346,8 @@ void HTPStateTransactionFree(void *state, uint16_t id) {
 void AppLayerHtpEnableRequestBodyCallback(void)
 {
     SCEnter();
-    need_htp_request_body = 1;
+
+    SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_REQUEST_BODY);
     SCReturn;
 }
 
@@ -368,7 +359,8 @@ void AppLayerHtpEnableRequestBodyCallback(void)
 void AppLayerHtpEnableResponseBodyCallback(void)
 {
     SCEnter();
-    need_htp_response_body = 1;
+
+    SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_RESPONSE_BODY);
     SCReturn;
 }
 
@@ -382,7 +374,7 @@ void AppLayerHtpNeedMultipartHeader(void) {
     SCEnter();
     AppLayerHtpEnableRequestBodyCallback();
 
-    need_htp_request_multipart_hdr = 1;
+    SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_REQUEST_MULTIPART);
     SCReturn;
 }
 
@@ -399,7 +391,7 @@ void AppLayerHtpNeedFileInspection(void)
     AppLayerHtpEnableRequestBodyCallback();
     AppLayerHtpEnableResponseBodyCallback();
 
-    need_htp_request_file = 1;
+    SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_REQUEST_FILE);
     SCReturn;
 }
 
@@ -1779,6 +1771,9 @@ int HTPCallbackRequestBodyData(htp_tx_data_t *d)
 {
     SCEnter();
 
+    if (!(SC_ATOMIC_GET(htp_config_flags) & HTP_REQUIRE_REQUEST_BODY))
+        SCReturnInt(HOOK_OK);
+
 #ifdef PRINT
     printf("HTPBODY START: \n");
     PrintRawDataFp(stdout, (uint8_t *)d->data, d->len);
@@ -1899,6 +1894,9 @@ int HTPCallbackResponseBodyData(htp_tx_data_t *d)
 {
     SCEnter();
 
+    if (!(SC_ATOMIC_GET(htp_config_flags) & HTP_REQUIRE_RESPONSE_BODY))
+        SCReturnInt(HOOK_OK);
+
     HtpState *hstate = (HtpState *)d->tx->connp->user_data;
     if (hstate == NULL) {
         SCReturnInt(HOOK_ERROR);
@@ -2135,6 +2133,10 @@ static void HTPConfigure(void)
             HTPCallbackRequestUriNormalizeQuery);
 #endif
     htp_config_set_generate_request_uri_normalized(cfglist.cfg, 1);
+    htp_config_register_request_body_data(cfglist.cfg,
+                                          HTPCallbackRequestBodyData);
+    htp_config_register_response_body_data(cfglist.cfg,
+                                           HTPCallbackResponseBodyData);
 
     default_config = ConfGetNode("libhtp.default-config");
     if (NULL != default_config) {
@@ -2275,6 +2277,10 @@ static void HTPConfigure(void)
             htprec->response_body_limit = HTP_CONFIG_DEFAULT_REQUEST_BODY_LIMIT;
             htp_config_register_request(htp, HTPCallbackRequest);
             htp_config_register_response(htp, HTPCallbackResponse);
+            htp_config_register_request_body_data(htp,
+                                                  HTPCallbackRequestBodyData);
+            htp_config_register_response_body_data(htp,
+                                                   HTPCallbackResponseBodyData);
 #ifdef HAVE_HTP_URI_NORMALIZE_HOOK
             htp_config_register_request_uri_normalize(htp,
                     HTPCallbackRequestUriNormalizeQuery);
@@ -2471,38 +2477,11 @@ void RegisterHTPParsers(void)
     AppLayerRegisterProto(proto_name, ALPROTO_HTTP, STREAM_TOCLIENT,
                           HTPHandleResponseData);
 
+    SC_ATOMIC_INIT(htp_config_flags);
     HTPConfigure();
     SCReturn;
 }
 
-/**
- * \brief This function is called at the end of SigLoadSignatures.  This function
- *        enables the htp layer to register a callback for the http request body.
- *        need_htp_request_body is a flag that informs the htp app layer that
- *        a module in the engine needs the http request body.
- */
-void AppLayerHtpRegisterExtraCallbacks(void) {
-    SCEnter();
-    SCLogDebug("Registering extra htp callbacks");
-
-    HTPCfgRec *p_cfglist = &cfglist;
-    while (p_cfglist != NULL) {
-        if (need_htp_request_body == 1) {
-            SCLogDebug("Registering callback htp_config_register_request_body_data on htp");
-            htp_config_register_request_body_data(p_cfglist->cfg,
-                                                  HTPCallbackRequestBodyData);
-        }
-        if (need_htp_response_body == 1) {
-            SCLogDebug("Registering callback htp_config_register_response_body_data on htp");
-            htp_config_register_response_body_data(p_cfglist->cfg,
-                                                   HTPCallbackResponseBodyData);
-        }
-        p_cfglist = p_cfglist->next;
-    }
-    SCReturn;
-}
-
-
 #ifdef UNITTESTS
 static HTPCfgRec cfglist_backup;
 

src/app-layer-htp.h

@@ -204,14 +204,16 @@ typedef struct HtpState_ {
 } HtpState;
 
 /** part of the engine needs the request body (e.g. http_client_body keyword) */
-extern uint8_t need_htp_request_body;
+#define HTP_REQUIRE_REQUEST_BODY        (1 << 0)
 /** part of the engine needs the request body multipart header (e.g. filename
  *  and / or fileext keywords) */
-extern uint8_t need_htp_request_multipart_hdr;
+#define HTP_REQUIRE_REQUEST_MULTIPART   (1 << 1)
 /** part of the engine needs the request file (e.g. log-file module) */
-extern uint8_t need_htp_request_file;
+#define HTP_REQUIRE_REQUEST_FILE        (1 << 2)
 /** part of the engine needs the request body (e.g. file_data keyword) */
-extern uint8_t need_htp_response_body;
+#define HTP_REQUIRE_RESPONSE_BODY       (1 << 3)
+
+SC_ATOMIC_DECLARE(uint32_t, htp_config_flags);
 
 void RegisterHTPParsers(void);
 void HTPParserRegisterTests(void);
@@ -224,7 +226,6 @@ int HTPCallbackRequestBodyData(htp_tx_data_t *);
 int HtpTransactionGetLoggableId(Flow *);
 void HtpBodyPrint(HtpBody *);
 void HtpBodyFree(HtpBody *);
-void AppLayerHtpRegisterExtraCallbacks(void);
 /* To free the state from unittests using app-layer-htp */
 void HTPStateFree(void *);
 void AppLayerHtpEnableRequestBodyCallback(void);

src/detect-engine.c

@@ -126,32 +126,12 @@ static void *DetectEngineLiveRuleSwap(void *arg)
         exit(EXIT_FAILURE);
     }
 
-    uint8_t local_need_htp_request_body = need_htp_request_body;
-    uint8_t local_need_htp_request_multipart_hdr = need_htp_request_multipart_hdr;
-    uint8_t local_need_htp_request_file = need_htp_request_file;
-    uint8_t local_need_htp_response_body = need_htp_response_body;
-
     if (SigLoadSignatures(de_ctx, NULL, FALSE) < 0) {
         SCLogError(SC_ERR_NO_RULES_LOADED, "Loading signatures failed.");
         if (de_ctx->failure_fatal)
             exit(EXIT_FAILURE);
     }
 
-    if (local_need_htp_request_body != need_htp_request_body ||
-        local_need_htp_request_multipart_hdr != need_htp_request_multipart_hdr ||
-        local_need_htp_request_file != need_htp_request_file ||
-        local_need_htp_response_body != need_htp_response_body) {
-        SCLogInfo("===== New ruleset requires enabling htp features that "
-                  "can't be enabled at runtime.  You will have to restart "
-                  "engine to load the new ruleset =====");
-        DetectEngineCtxFree(de_ctx);
-        UtilSignalHandlerSetup(SIGUSR2, SignalHandlerSigusr2);
-
-        TmThreadsSetFlag(tv_local, THV_CLOSED);
-
-        pthread_exit(NULL);
-    }
-
     SCThresholdConfInitContext(de_ctx, NULL);
 
     /* start the process of swapping detect threads ctxs */

src/detect.h

@@ -671,6 +671,9 @@ typedef struct DetectEngineCtx_ {
     /** Store rule file and line so that parsers can use them in errors. */
     char *rule_file;
     int rule_line;
+
+    /** Is detect engine using a delayed init */
+    int delayed_detect;
 } DetectEngineCtx;
 
 /* Engine groups profiles (low, medium, high, custom) */

src/source-pcap.c

@@ -287,6 +287,9 @@ TmEcode ReceivePcapLoop(ThreadVars *tv, void *data, void *slot)
             int dbreak = 0;
             SCLogError(SC_ERR_PCAP_DISPATCH, "error code %" PRId32 " %s",
                        r, pcap_geterr(ptv->pcap_handle));
+            if (r == PCAP_ERROR_BREAK) {
+                SCReturnInt(ptv->cb_result);
+            }
             do {
                 usleep(PCAP_RECONNECT_TIMEOUT);
                 if (suricata_ctl_flags != 0) {

src/suricata.c

@@ -686,6 +686,7 @@ int main(int argc, char **argv)
 #endif /* OS_WIN32 */
     int build_info = 0;
     int rule_reload = 0;
+    int delayed_detect = 1;
 
     char *log_dir;
 #ifdef OS_WIN32
@@ -1541,7 +1542,6 @@ int main(int argc, char **argv)
 
         AppLayerHtpEnableRequestBodyCallback();
         AppLayerHtpNeedFileInspection();
-        AppLayerHtpRegisterExtraCallbacks();
 
         UtInitialize();
         UTHRegisterTests();
@@ -1734,18 +1734,42 @@ int main(int argc, char **argv)
     if (MagicInit() != 0)
         exit(EXIT_FAILURE);
 
-    if (SigLoadSignatures(de_ctx, sig_file, sig_file_exclusive) < 0) {
-        if (sig_file == NULL) {
-            SCLogError(SC_ERR_OPENING_FILE, "Signature file has not been provided");
-        } else {
-            SCLogError(SC_ERR_NO_RULES_LOADED, "Loading signatures failed.");
+    /* In offline mode delayed init of detect is a bad idea */
+    if ((run_mode == RUNMODE_PCAP_FILE) ||
+        (run_mode == RUNMODE_ERF_FILE) ||
+        engine_analysis) {
+        delayed_detect = 0;
+    } else {
+        ConfNode *denode = NULL;
+        ConfNode *decnf = ConfGetNode("detect-engine");
+        if (decnf != NULL) {
+            TAILQ_FOREACH(denode, &decnf->head, next) {
+                if (strcmp(denode->val, "delayed-detect") == 0) {
+                    (void)ConfGetChildValueBool(denode, "delayed-detect", &delayed_detect);
+                }
+            }
         }
-        if (de_ctx->failure_fatal)
-            exit(EXIT_FAILURE);
     }
+    de_ctx->delayed_detect = delayed_detect;
 
-    if (engine_analysis) {
-        exit(EXIT_SUCCESS);
+    SCLogInfo("Delayed detect %s", delayed_detect ? "enabled" : "disabled");
+    if (delayed_detect) {
+        SCLogInfo("Packets will start being processed before signatures are active.");
+    }
+
+    if (!delayed_detect) {
+        if (SigLoadSignatures(de_ctx, sig_file, sig_file_exclusive) < 0) {
+            if (sig_file == NULL) {
+                SCLogError(SC_ERR_OPENING_FILE, "Signature file has not been provided");
+            } else {
+                SCLogError(SC_ERR_NO_RULES_LOADED, "Loading signatures failed.");
+            }
+            if (de_ctx->failure_fatal)
+                exit(EXIT_FAILURE);
+        }
+        if (engine_analysis) {
+            exit(EXIT_SUCCESS);
+        }
     }
 
     /* registering singal handlers we use.  We register usr2 here, so that one
@@ -1761,7 +1785,6 @@ int main(int argc, char **argv)
     SCCudaPBSetUpQueuesAndBuffers();
 #endif /* __SC_CUDA_SUPPORT__ */
 
-    AppLayerHtpRegisterExtraCallbacks();
     SCThresholdConfInitContext(de_ctx,NULL);
     SCAsn1LoadConfig();
 
@@ -1855,6 +1878,21 @@ int main(int argc, char **argv)
     /* Un-pause all the paused threads */
     TmThreadContinueThreads();
 
+    if (delayed_detect) {
+        if (SigLoadSignatures(de_ctx, sig_file, sig_file_exclusive) < 0) {
+            if (sig_file == NULL) {
+                SCLogError(SC_ERR_OPENING_FILE, "Signature file has not been provided");
+            } else {
+                SCLogError(SC_ERR_NO_RULES_LOADED, "Loading signatures failed.");
+            }
+            if (de_ctx->failure_fatal)
+                exit(EXIT_FAILURE);
+        }
+        TmThreadActivateDummySlot();
+        SCLogInfo("Signature(s) loaded, Detect thread(s) activated.");
+    }
+
+
 #ifdef DBG_MEM_ALLOC
     SCLogInfo("Memory used at startup: %"PRIdMAX, (intmax_t)global_mem);
 #ifdef DBG_MEM_ALLOC_SKIP_STARTUP