app-layer/pd: review bailout conditions

To take TCP window into account
And to actually bail out if we received too much data
where the limit is configured by stream.reassembly.depth

src/app-layer.c

@@ -209,57 +209,52 @@ static void TCPProtoDetectCheckBailConditions(ThreadVars *tv,
     const uint32_t size_tc = StreamDataAvailableForProtoDetect(&ssn->server);
     SCLogDebug("size_ts %" PRIu32 ", size_tc %" PRIu32, size_ts, size_tc);
 
-    DEBUG_VALIDATE_BUG_ON(size_ts > 1000000UL);
-    DEBUG_VALIDATE_BUG_ON(size_tc > 1000000UL);
+    /* at least 100000 whatever the conditions
+     * and can be more if window is bigger and if configuration allows it */
+    const uint32_t size_tc_limit =
+            MAX(100000, MIN(ssn->client.window, stream_config.reassembly_depth));
+    const uint32_t size_ts_limit =
+            MAX(100000, MIN(ssn->server.window, stream_config.reassembly_depth));
 
     if (ProtoDetectDone(f, ssn, STREAM_TOSERVER) &&
         ProtoDetectDone(f, ssn, STREAM_TOCLIENT))
     {
         goto failure;
 
+        /* we bail out whatever the pp and pm states if
+         * we received too much data */
+    } else if (size_tc > 2 * size_tc_limit || size_ts > 2 * size_ts_limit) {
+        AppLayerDecoderEventsSetEventRaw(&p->app_layer_events, APPLAYER_PROTO_DETECTION_SKIPPED);
+        goto failure;
+
     } else if (FLOW_IS_PM_DONE(f, STREAM_TOSERVER) && FLOW_IS_PP_DONE(f, STREAM_TOSERVER) &&
-            size_ts > 100000 && size_tc == 0)
-    {
+               size_ts > size_ts_limit && size_tc == 0) {
         AppLayerDecoderEventsSetEventRaw(&p->app_layer_events,
                 APPLAYER_PROTO_DETECTION_SKIPPED);
         goto failure;
 
     } else if (FLOW_IS_PM_DONE(f, STREAM_TOCLIENT) && FLOW_IS_PP_DONE(f, STREAM_TOCLIENT) &&
-            size_tc > 100000 && size_ts == 0)
-    {
+               size_tc > size_tc_limit && size_ts == 0) {
         AppLayerDecoderEventsSetEventRaw(&p->app_layer_events,
                 APPLAYER_PROTO_DETECTION_SKIPPED);
         goto failure;
 
     /* little data in ts direction, pp done, pm not done (max
      * depth not reached), ts direction done, lots of data in
      * tc direction. */
-    } else if (size_tc > 100000 &&
-            FLOW_IS_PP_DONE(f, STREAM_TOSERVER) && !(FLOW_IS_PM_DONE(f, STREAM_TOSERVER)) &&
-            FLOW_IS_PM_DONE(f, STREAM_TOCLIENT) && FLOW_IS_PP_DONE(f, STREAM_TOCLIENT))
-    {
+    } else if (size_tc > size_tc_limit && FLOW_IS_PP_DONE(f, STREAM_TOSERVER) &&
+               !(FLOW_IS_PM_DONE(f, STREAM_TOSERVER)) && FLOW_IS_PM_DONE(f, STREAM_TOCLIENT) &&
+               FLOW_IS_PP_DONE(f, STREAM_TOCLIENT)) {
         AppLayerDecoderEventsSetEventRaw(&p->app_layer_events,
                 APPLAYER_PROTO_DETECTION_SKIPPED);
         goto failure;
 
     /* little data in tc direction, pp done, pm not done (max
      * depth not reached), tc direction done, lots of data in
      * ts direction. */
-    } else if (size_ts > 100000 &&
-            FLOW_IS_PP_DONE(f, STREAM_TOCLIENT) && !(FLOW_IS_PM_DONE(f, STREAM_TOCLIENT)) &&
-            FLOW_IS_PM_DONE(f, STREAM_TOSERVER) && FLOW_IS_PP_DONE(f, STREAM_TOSERVER))
-    {
-        AppLayerDecoderEventsSetEventRaw(&p->app_layer_events,
-                APPLAYER_PROTO_DETECTION_SKIPPED);
-        goto failure;
-
-    /* in case of really low TS data (e.g. 4 bytes) we can have
-     * the PP complete, PM not complete (depth not reached) and
-     * the TC side also not recognized (proto unknown) */
-    } else if (size_tc > 100000 &&
-            FLOW_IS_PP_DONE(f, STREAM_TOSERVER) && !(FLOW_IS_PM_DONE(f, STREAM_TOSERVER)) &&
-            (!FLOW_IS_PM_DONE(f, STREAM_TOCLIENT) && !FLOW_IS_PP_DONE(f, STREAM_TOCLIENT)))
-    {
+    } else if (size_ts > size_ts_limit && FLOW_IS_PP_DONE(f, STREAM_TOCLIENT) &&
+               !(FLOW_IS_PM_DONE(f, STREAM_TOCLIENT)) && FLOW_IS_PM_DONE(f, STREAM_TOSERVER) &&
+               FLOW_IS_PP_DONE(f, STREAM_TOSERVER)) {
         AppLayerDecoderEventsSetEventRaw(&p->app_layer_events,
                 APPLAYER_PROTO_DETECTION_SKIPPED);
         goto failure;