Next/20220727/v7

.github/workflows/builds.yml

@@ -39,7 +39,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Cache ~/.cargo
-        uses: actions/cache@c3f1317a9e7b1ef106c153ac8c0f00fed3ddbc0d
+        uses: actions/cache@0865c47f36e68161719c5b124609996bb5c40129
         with:
           path: ~/.cargo
           key: cargo
@@ -123,7 +123,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Cache ~/.cargo
-        uses: actions/cache@c3f1317a9e7b1ef106c153ac8c0f00fed3ddbc0d
+        uses: actions/cache@0865c47f36e68161719c5b124609996bb5c40129
         with:
           path: ~/.cargo
           key: cbindgen
@@ -150,7 +150,7 @@ jobs:
     steps:
       # Cache Rust stuff.
       - name: Cache cargo registry
-        uses: actions/cache@c3f1317a9e7b1ef106c153ac8c0f00fed3ddbc0d
+        uses: actions/cache@0865c47f36e68161719c5b124609996bb5c40129
         with:
           path: ~/.cargo/registry
           key: cargo-registry
@@ -234,6 +234,11 @@ jobs:
                 texlive-upquote \
                 texlive-capt-of \
                 texlive-needspace \
+      - name: Setup cppclean
+        run: |
+          git clone --depth 1 --branch suricata https://github.com/catenacyber/cppclean
+          cd cppclean
+          python3 setup.py install
       - name: Configuring
         run: |
           ./autogen.sh
@@ -242,6 +247,9 @@ jobs:
         env:
           DISTCHECK_CONFIGURE_FLAGS: "--enable-unittests --enable-debug --enable-lua --enable-geoip --enable-profiling --enable-profiling-locks --enable-dpdk"
       - run: test -e doc/userguide/suricata.1
+      - name: Checking includes
+        run: |
+          cppclean src/*.h | grep "does not need to be #included" | python3 scripts/cppclean_check.py
       - name: Building Rust documentation
         run: make doc
         working-directory: rust
@@ -328,7 +336,7 @@ jobs:
 
       # Cache Rust stuff.
       - name: Cache cargo registry
-        uses: actions/cache@c3f1317a9e7b1ef106c153ac8c0f00fed3ddbc0d
+        uses: actions/cache@0865c47f36e68161719c5b124609996bb5c40129
         with:
           path: ~/.cargo/registry
           key: cargo-registry
@@ -415,7 +423,7 @@ jobs:
 
       # Cache Rust stuff.
       - name: Cache cargo registry
-        uses: actions/cache@c3f1317a9e7b1ef106c153ac8c0f00fed3ddbc0d
+        uses: actions/cache@0865c47f36e68161719c5b124609996bb5c40129
         with:
           path: ~/.cargo/registry
           key: cargo-registry
@@ -502,7 +510,7 @@ jobs:
 
       # Cache Rust stuff.
       - name: Cache cargo registry
-        uses: actions/cache@c3f1317a9e7b1ef106c153ac8c0f00fed3ddbc0d
+        uses: actions/cache@0865c47f36e68161719c5b124609996bb5c40129
         with:
           path: ~/.cargo/registry
           key: cargo-registry
@@ -940,7 +948,7 @@ jobs:
 
       # Cache Rust stuff.
       - name: Cache cargo registry
-        uses: actions/cache@c3f1317a9e7b1ef106c153ac8c0f00fed3ddbc0d
+        uses: actions/cache@0865c47f36e68161719c5b124609996bb5c40129
         with:
           path: ~/.cargo/registry
           key: cargo-registry
@@ -1012,7 +1020,7 @@ jobs:
 
       # Cache Rust stuff.
       - name: Cache cargo registry
-        uses: actions/cache@c3f1317a9e7b1ef106c153ac8c0f00fed3ddbc0d
+        uses: actions/cache@0865c47f36e68161719c5b124609996bb5c40129
         with:
           path: ~/.cargo/registry
           key: cargo-registry
@@ -1105,7 +1113,7 @@ jobs:
 
       # Cache Rust stuff.
       - name: Cache cargo registry
-        uses: actions/cache@c3f1317a9e7b1ef106c153ac8c0f00fed3ddbc0d
+        uses: actions/cache@0865c47f36e68161719c5b124609996bb5c40129
         with:
           path: ~/.cargo/registry
           key: cargo-registry
@@ -1168,7 +1176,7 @@ jobs:
     steps:
       # Cache Rust stuff.
       - name: Cache cargo registry
-        uses: actions/cache@c3f1317a9e7b1ef106c153ac8c0f00fed3ddbc0d
+        uses: actions/cache@0865c47f36e68161719c5b124609996bb5c40129
         with:
           path: ~/.cargo/registry
           key: cargo-registry
@@ -1300,7 +1308,7 @@ jobs:
     steps:
       # Cache Rust stuff.
       - name: Cache cargo registry
-        uses: actions/cache@c3f1317a9e7b1ef106c153ac8c0f00fed3ddbc0d
+        uses: actions/cache@0865c47f36e68161719c5b124609996bb5c40129
         with:
           path: ~/.cargo/registry
           key: cargo-registry

.github/workflows/commits.yml

@@ -16,7 +16,7 @@ jobs:
     container: ubuntu:20.04
     steps:
       - name: Caching ~/.cargo
-        uses: actions/cache@c3f1317a9e7b1ef106c153ac8c0f00fed3ddbc0d
+        uses: actions/cache@0865c47f36e68161719c5b124609996bb5c40129
         with:
           path: ~/.cargo
           key: commit-check-cargo

.github/workflows/formatting.yml

@@ -25,7 +25,7 @@ jobs:
 
       # Cache Rust stuff.
       - name: Cache cargo registry
-        uses: actions/cache@c3f1317a9e7b1ef106c153ac8c0f00fed3ddbc0d
+        uses: actions/cache@0865c47f36e68161719c5b124609996bb5c40129
         with:
           path: ~/.cargo/registry
           key: cargo-registry

rust/src/ike/logger.rs

@@ -214,12 +214,14 @@ fn log_ikev2(tx: &IKETransaction, jb: &mut JsonBuilder) -> Result<(), JsonError>
     jb.open_object("ikev2")?;
 
     jb.set_uint("errors", tx.errors as u64)?;
-    jb.open_array("notify")?;
-    for notify in tx.notify_types.iter() {
-        jb.append_string(&format!("{:?}", notify))?;
+    if !tx.notify_types.is_empty() {
+        jb.open_array("notify")?;
+        for notify in tx.notify_types.iter() {
+            jb.append_string(&format!("{:?}", notify))?;
+        }
+        jb.close()?;
     }
     jb.close()?;
-    jb.close()?;
     Ok(())
 }
 

rust/src/mqtt/logger.rs

@@ -233,11 +233,13 @@ fn log_mqtt(tx: &MQTTTransaction, flags: u32, js: &mut JsonBuilder) -> Result<()
                 log_mqtt_header(js, &msg.header)?;
                 js.set_uint("message_id", unsuback.message_id as u64)?;
                 if let Some(codes) = &unsuback.reason_codes {
-                    js.open_array("reason_codes")?;
-                    for t in codes {
-                        js.append_uint(*t as u64)?;
+                    if codes.len() > 0 {
+                        js.open_array("reason_codes")?;
+                        for t in codes {
+                            js.append_uint(*t as u64)?;
+                        }
+                        js.close()?; // reason_codes
                     }
-                    js.close()?; // reason_codes
                 }
                 js.close()?; // unsuback
             }

rust/src/quic/logger.rs

@@ -30,14 +30,16 @@ fn log_template(tx: &QuicTransaction, js: &mut JsonBuilder) -> Result<(), JsonEr
             js.set_string("ua", &String::from_utf8_lossy(&ua))?;
         }
     }
-    js.open_array("cyu")?;
-    for cyu in &tx.cyu {
-        js.start_object()?;
-        js.set_string("hash", &cyu.hash)?;
-        js.set_string("string", &cyu.string)?;
+    if tx.cyu.len() > 0 {
+        js.open_array("cyu")?;
+        for cyu in &tx.cyu {
+            js.start_object()?;
+            js.set_string("hash", &cyu.hash)?;
+            js.set_string("string", &cyu.string)?;
+            js.close()?;
+        }
         js.close()?;
     }
-    js.close()?;
 
     js.close()?;
     Ok(())

scripts/cppclean_check.py

@@ -0,0 +1,30 @@
+import sys
+
+#cppclean src/*.h | grep "does not need to be #included"
+retcode = 0
+for l in sys.stdin:
+    includer = l.split(':')[0]
+    included = l.split("'")[1]
+
+    if included == "rust.h" or included == "suricata-common.h":
+        continue
+    if includer == "src/suricata-common.h" or includer == "src/rust-context.h" or includer == "src/rust.h" or includer == "src/threads.h":
+        continue
+
+    if included == "util-file.h" and includer == "src/detect.h":
+        # SigTableElmt structure field FileMatch being a function pointer using a parameter File defined in util-file.h
+        continue
+    if included == "conf.h" and includer == "src/suricata-plugin.h":
+        # SCEveFileType structure field Init being a function pointer using a parameter ConfNode defined in conf.h
+        continue
+    if included == "util-debug-filters.h" and includer == "src/util-debug.h":
+        # Macro SCEnter using SCLogCheckFDFilterEntry defined in util-debug-filters.h
+        continue
+    if included == "util-spm-bs.h" and includer == "src/util-spm.h":
+        # Macro SpmSearch using BasicSearch defined in util-spm-bs.h
+        continue
+
+    print("Unnecessary include from %s for %s" % (includer, included))
+    retcode = 1
+
+sys.exit(retcode)

src/Makefile.am

@@ -38,7 +38,6 @@ noinst_HEADERS = \
 	app-layer-modbus.h \
 	app-layer-quic.h \
 	app-layer-mqtt.h \
-	app-layer-nbss.h \
 	app-layer-nfs-tcp.h \
 	app-layer-nfs-udp.h \
 	app-layer-ntp.h \
@@ -81,7 +80,6 @@ noinst_HEADERS = \
 	decode-ipv6.h \
 	decode-mpls.h \
 	decode-nsh.h \
-	decode-null.h \
 	decode-ppp.h \
 	decode-pppoe.h \
 	decode-raw.h \
@@ -607,7 +605,6 @@ noinst_HEADERS = \
 	util-validate.h \
 	util-var.h \
 	util-var-name.h \
-	util-vector.h \
 	win32-misc.h \
 	win32-service.h \
 	win32-syscall.h \

src/alert-fastlog.c

@@ -40,6 +40,7 @@
 
 #include "detect-parse.h"
 #include "detect-engine.h"
+#include "detect-engine-build.h"
 #include "detect-engine-mpm.h"
 #include "detect-reference.h"
 #include "util-classification-config.h"

src/app-layer-dcerpc-common.h

@@ -18,11 +18,6 @@
 #ifndef __APP_LAYER_DCERPC_COMMON_H__
 #define __APP_LAYER_DCERPC_COMMON_H__
 
-#include "app-layer-protos.h"
-#include "app-layer-parser.h"
-#include "flow.h"
-#include "queue.h"
-#include "util-byte.h"
 
 void RegisterDCERPCParsers(void);
 void DCERPCParserTests(void);

src/app-layer-detect-proto.c

@@ -31,6 +31,7 @@
 
 #include "detect.h"
 #include "detect-engine-port.h"
+#include "detect-engine-build.h"
 #include "detect-parse.h"
 #include "detect-engine.h"
 #include "detect-content.h"

src/app-layer-dnp3.c

@@ -16,12 +16,15 @@
  */
 
 #include "suricata-common.h"
+#include "suricata.h"
 #include "stream.h"
 #include "util-byte.h"
 #include "util-unittest.h"
 #include "util-hashlist.h"
 
 #include "util-print.h"
+#include "util-spm-bs.h"
+#include "util-enum.h"
 
 #include "app-layer.h"
 #include "app-layer-protos.h"

src/app-layer-dnp3.h

@@ -18,9 +18,6 @@
 #ifndef __APP_LAYER_DNP3_H__
 #define __APP_LAYER_DNP3_H__
 
-#include "detect-engine-state.h"
-#include "util-hashlist.h"
-#include "util-byte.h"
 #include "rust.h"
 
 /**

src/app-layer-enip-common.h

@@ -24,10 +24,7 @@
 #ifndef __APP_LAYER_ENIP_COMMON_H__
 #define __APP_LAYER_ENIP_COMMON_H__
 
-#include "app-layer-protos.h"
-#include "app-layer-parser.h"
-#include "flow.h"
-#include "queue.h"
+#include "rust.h"
 
 // EtherNet/IP commands
 #define NOP                0x0000

src/app-layer-enip.h

@@ -24,9 +24,6 @@
 #ifndef __APP_LAYER_ENIP_H__
 #define __APP_LAYER_ENIP_H__
 
-#include "decode.h"
-#include "detect-engine-state.h"
-#include "queue.h"
 
 void RegisterENIPUDPParsers(void);
 void RegisterENIPTCPParsers(void);

src/app-layer-frames.c

@@ -28,6 +28,7 @@
 
 #include "stream-tcp.h"
 #include "app-layer-frames.h"
+#include "app-layer-parser.h"
 
 static void FrameDebug(const char *prefix, const Frames *frames, const Frame *frame)
 {

src/app-layer-frames.h

@@ -24,12 +24,7 @@
 #ifndef __APP_LAYER_FRAMES_H__
 #define __APP_LAYER_FRAMES_H__
 
-#include "app-layer-events.h"
-#include "detect-engine-state.h"
-#include "util-file.h"
-#include "stream-tcp-private.h"
 #include "rust.h"
-#include "app-layer-parser.h"
 
 typedef int64_t FrameId;
 

src/app-layer-ftp.c

@@ -48,6 +48,7 @@
 #include "app-layer-parser.h"
 #include "app-layer-ftp.h"
 #include "app-layer-expectation.h"
+#include "app-layer-detect-proto.h"
 
 #include "util-spm.h"
 #include "util-mpm.h"

src/app-layer-htp-body.c

@@ -45,6 +45,7 @@
 #include "app-layer-parser.h"
 #include "app-layer-htp.h"
 #include "app-layer-htp-body.h"
+#include "app-layer-htp-mem.h"
 
 #include "util-spm.h"
 #include "util-debug.h"

src/app-layer-htp-file.c

@@ -45,6 +45,7 @@
 #include "app-layer-parser.h"
 #include "app-layer-htp.h"
 #include "app-layer-htp-file.h"
+#include "app-layer-htp-range.h"
 
 #include "util-spm.h"
 #include "util-debug.h"

src/app-layer-htp-libhtp.h

@@ -42,7 +42,6 @@
 #ifndef __APP_LAYER_HTP_LIBHTP__H__
 #define __APP_LAYER_HTP_LIBHTP__H__
 
-#include "suricata.h"
 #include "suricata-common.h"
 
 bstr *SCHTPGenerateNormalizedUri(htp_tx_t *tx, htp_uri_t *uri, int uri_include_all);

src/app-layer-htp-mem.h

@@ -15,7 +15,6 @@
  * 02110-1301, USA.
  */
 
-#include "stream-tcp-reassemble.h"
 
 void HTPParseMemcap(void);
 void *HTPMalloc(size_t size);