ARM-support updates to docker integrations

.github/dependabot.yml

@@ -7,8 +7,6 @@ updates:
       day: "tuesday"
     target-branch: "develop"
     ignore:
-      # ignore all patch updates since we are using ~=
-      # this does not work for security updates
       - dependency-name: "*"
         update-types: ["version-update:semver-patch"]
       - dependency-name: "boto3"
@@ -23,8 +21,6 @@ updates:
       day: "tuesday"
     target-branch: "develop"
     ignore:
-      # ignore all patch updates since we are using ~=
-      # this does not work for security updates
       - dependency-name: "*"
         update-types: ["version-update:semver-patch"]
 
@@ -35,8 +31,6 @@ updates:
       day: "tuesday"
     target-branch: "develop"
     ignore:
-      # ignore all patch updates since we are using ~=
-      # this does not work for security updates
       - dependency-name: "*"
         update-types: [ "version-update:semver-patch" ]
 
@@ -47,8 +41,6 @@ updates:
       day: "tuesday"
     target-branch: "develop"
     ignore:
-      # ignore all patch updates since we are using ~=
-      # this does not work for security updates
       - dependency-name: "*"
         update-types: [ "version-update:semver-patch" ]
 
@@ -71,8 +63,6 @@ updates:
       day: "tuesday"
     target-branch: "develop"
     ignore:
-      # ignore all patch updates since we are using ~=
-      # this does not work for security updates
       - dependency-name: "*"
         update-types: [ "version-update:semver-patch" ]
 
@@ -92,8 +82,6 @@ updates:
       day: "tuesday"
     target-branch: "develop"
     ignore:
-      # ignore all patch updates since we are using ~=
-      # this does not work for security updates
       - dependency-name: "*"
         update-types: ["version-update:semver-patch"]
 
@@ -104,8 +92,6 @@ updates:
       day: "tuesday"
     target-branch: "develop"
     ignore:
-      # ignore all patch updates since we are using ~=
-      # this does not work for security updates
       - dependency-name: "*"
         update-types: ["version-update:semver-patch"]
 
@@ -116,8 +102,6 @@ updates:
       day: "tuesday"
     target-branch: "develop"
     ignore:
-      # ignore all patch updates since we are using ~=
-      # this does not work for security updates
       - dependency-name: "*"
         update-types: ["version-update:semver-patch"]
 
@@ -128,8 +112,16 @@ updates:
       day: "tuesday"
     target-branch: "develop"
     ignore:
-      # ignore all patch updates since we are using ~=
-      # this does not work for security updates
+      - dependency-name: "*"
+        update-types: ["version-update:semver-patch"]
+
+  - package-ecosystem: "docker"
+    directory: "/integrations/cyberchef"
+    schedule:
+      interval: "weekly"
+      day: "tuesday"
+    target-branch: "develop"
+    ignore:
       - dependency-name: "*"
         update-types: ["version-update:semver-patch"]
 
@@ -152,20 +144,26 @@ updates:
       day: "tuesday"
     target-branch: "develop"
     ignore:
-      # ignore all patch updates since we are using ~=
-      # this does not work for security updates
       - dependency-name: "*"
         update-types: ["version-update:semver-patch"]
 
+  - package-ecosystem: "docker"
+    directory: "/integrations/thug"
+    schedule:
+      interval: "weekly"
+      day: "tuesday"
+    target-branch: "develop"
+    ignore:
+      - dependency-name: "*"
+        update-types: [ "version-update:semver-patch" ]
+
   - package-ecosystem: "docker"
     directory: "/integrations/phishing_analyzers"
     schedule:
       interval: "weekly"
       day: "tuesday"
     target-branch: "develop"
     ignore:
-      # ignore all patch updates since we are using ~=
-      # this does not work for security updates
       - dependency-name: "*"
         update-types: ["version-update:semver-patch"]
 
@@ -176,7 +174,5 @@ updates:
       day: "tuesday"
     target-branch: "develop"
     ignore:
-      # ignore all patch updates since we are using ~=
-      # this does not work for security updates
       - dependency-name: "*"
         update-types: ["version-update:semver-patch"]

api_app/analyzers_manager/file_analyzers/detectiteasy.py

@@ -1,60 +1,61 @@
+import json
 import logging
 
-from api_app.analyzers_manager.classes import DockerBasedAnalyzer, FileAnalyzer
+import die
+
+from api_app.analyzers_manager.classes import FileAnalyzer
 from tests.mock_utils import MockUpResponse
 
 logger = logging.getLogger(__name__)
 
 
-class DetectItEasy(FileAnalyzer, DockerBasedAnalyzer):
-    name: str = "executable_analyzer"
-    url: str = "http://malware_tools_analyzers:4002/die"
-    # http request polling max number of tries
-    max_tries: int = 10
-    # interval between http request polling (in secs)
-    poll_distance: int = 1
+class DetectItEasy(FileAnalyzer):
 
     def update(self):
         pass
 
     def run(self):
-        fname = str(self.filename).replace("/", "_").replace(" ", "_")
-        # get the file to send
-        binary = self.read_file_bytes()
-        args = [f"@{fname}", "--json"]
-        req_data = {
-            "args": args,
-        }
-        req_files = {fname: binary}
-        logger.info(
-            f"Running {self.analyzer_name} on {self.filename} with args: {args}"
+        logger.info(f"Running DIE on {self.filepath} for {self.md5}")
+
+        json_report = die.scan_file(
+            self.filepath, die.ScanFlags.RESULT_AS_JSON, str(die.database_path / "db")
         )
-        report = self._docker_run(req_data, req_files, analyzer_name=self.analyzer_name)
-        if not report:
-            self.report.errors.append("DIE did not detect the file type")
-            return {}
-        return report
+
+        return json.loads(json_report)
 
     @staticmethod
     def mocked_docker_analyzer_get(*args, **kwargs):
         return MockUpResponse(
             {
-                "report": {
-                    "arch": "NOEXEC",
-                    "mode": "Unknown",
-                    "type": "Unknown",
-                    "detects": [
-                        {
-                            "name": "Zip",
-                            "type": "archive",
-                            "string": "archive: Zip(2.0)[38.5%,1 file]",
-                            "options": "38.5%,1 file",
-                            "version": "2.0",
-                        }
-                    ],
-                    "filetype": "Binary",
-                    "endianess": "LE",
-                }
+                "detects": [
+                    {
+                        "filetype": "PE64",
+                        "parentfilepart": "Header",
+                        "values": [
+                            {
+                                "info": "Console64,console",
+                                "name": "GNU linker ld (GNU Binutils)",
+                                "string": "Linker: GNU linker ld (GNU Binutils)(2.28)[Console64,console]",
+                                "type": "Linker",
+                                "version": "2.28",
+                            },
+                            {
+                                "info": "",
+                                "name": "MinGW",
+                                "string": "Compiler: MinGW",
+                                "type": "Compiler",
+                                "version": "",
+                            },
+                            {
+                                "info": "NRV,brute",
+                                "name": "UPX",
+                                "string": "Packer: UPX(4.24)[NRV,brute]",
+                                "type": "Packer",
+                                "version": "4.24",
+                            },
+                        ],
+                    }
+                ]
             },
             200,
         )

api_app/analyzers_manager/file_analyzers/thug_file.py

@@ -8,7 +8,7 @@
 
 class ThugFile(FileAnalyzer, DockerBasedAnalyzer):
     name: str = "Thug"
-    url: str = "http://malware_tools_analyzers:4002/thug"
+    url: str = "http://thug:4002/thug"
     # http request polling max number of tries
     max_tries: int = 15
     # interval between http request polling (in secs)

api_app/analyzers_manager/migrations/0149_alter_die_analyzer.py

@@ -0,0 +1,35 @@
+from django.db import migrations
+
+
+def migrate(apps, schema_editor):
+    PythonModule = apps.get_model("api_app", "PythonModule")
+
+    pm = PythonModule.objects.get(
+        module="detectiteasy.DetectItEasy",
+        base_path="api_app.analyzers_manager.file_analyzers",
+    )
+    pm.parameters.all().delete()
+
+    AnalyzerConfig = apps.get_model("analyzers_manager", "AnalyzerConfig")
+
+    ac = AnalyzerConfig.objects.get(
+        name="DetectItEasy",
+    )
+    ac.docker_based = False
+    ac.save()
+
+
+def reverse_migrate(apps, schema_editor): ...
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("api_app", "0065_job_mpnodesearch"),
+        (
+            "analyzers_manager",
+            "0148_analyzer_config_nuclei",
+        ),
+    ]
+    operations = [
+        migrations.RunPython(migrate, reverse_migrate),
+    ]

api_app/analyzers_manager/observable_analyzers/thug_url.py

@@ -8,7 +8,7 @@
 
 class ThugUrl(ObservableAnalyzer, DockerBasedAnalyzer):
     name: str = "Thug"
-    url: str = "http://malware_tools_analyzers:4002/thug"
+    url: str = "http://thug:4002/thug"
     # http request polling max number of tries
     max_tries: int = 15
     # interval between http request polling (in seconds)
@@ -23,6 +23,11 @@ class ThugUrl(ObservableAnalyzer, DockerBasedAnalyzer):
 
     def _thug_args_builder(self):
         user_agent = self.user_agent
+        if not user_agent:
+            user_agent = (
+                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+                "(KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36 Edg/131.0.2903.86"
+            )
         dom_events = self.dom_events
         use_proxy = self.use_proxy
         proxy = self.proxy
@@ -48,6 +53,7 @@ def run(self):
         tmp_dir = secrets.token_hex(4)
         tmp_dir_full_path = "/opt/deploy/thug" + tmp_dir
         # make request data
+        # the option -n is bugged and does not work https://github.com/intelowlproject/IntelOwl/issues/2656
         args.extend(["-n", tmp_dir_full_path, self.observable_name])
 
         req_data = {

docker/Dockerfile

@@ -35,18 +35,15 @@ RUN mkdir -p ${LOG_PATH} \
 # python3-psycopg2 is required to use PostgresSQL with Django
 # apache2-utils is required to execute htpasswd
 # tshark is required for Hfinger file analyzer
+# libemail-outlook-message-perl and libemail-address-perl are required for msgconvert
 RUN apt-get update \
     && apt-get install -y --no-install-recommends apt-utils libsasl2-dev libssl-dev netcat-traditional \
-    vim libldap2-dev libfuzzy-dev net-tools python3-psycopg2 git apache2-utils tshark \
-    && apt-get clean \
+    vim libldap2-dev libfuzzy-dev net-tools python3-psycopg2 git apache2-utils tshark  \
+    libemail-outlook-message-perl libemail-address-perl \
+    && apt-get clean && apt-get autoclean && apt-get autoremove -y \
     && rm -rf /var/lib/apt/lists/* \
     && pip3 install --no-cache-dir --upgrade pip
 
-# perl not interactive
-ENV PERL_MM_USE_DEFAULT=1
-# msgconvert
-RUN cpan -T Email::Outlook::Message
-
 COPY requirements/project-requirements.txt $PYTHONPATH/project-requirements.txt
 COPY requirements/certego-requirements.txt $PYTHONPATH/certego-requirements.txt
 WORKDIR $PYTHONPATH

docker/hooks/build

@@ -5,8 +5,6 @@ echo "$(pwd)"
 echo "display dockerfile path"
 echo $DOCKERFILE_PATH
 cd ..
-echo "display path"
-echo "$(pwd)"
 echo "current branch"
 echo "$SOURCE_BRANCH"
 

integrations/cyberchef/Dockerfile

@@ -1,5 +1,5 @@
 # Adapted from https://github.com/gchq/CyberChef-server/blob/master/Dockerfile
-FROM node:alpine3.10
+FROM node:lts-alpine3.21
 RUN apk update && apk --no-cache add git
 LABEL author = "Wes Lambert, wlambertts@gmail.com"
 LABEL description="Dockerised version of Cyberchef server (https://github.com/gchq/CyberChef-server)"

integrations/cyberchef/hooks/build

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+echo "display path"
+echo "$(pwd)"
+echo "display dockerfile path"
+echo $DOCKERFILE_PATH
+echo "current branch"
+echo "$SOURCE_BRANCH"
+
+version_regex='^v[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$'
+if [[ "$SOURCE_BRANCH" == "master" || "$SOURCE_BRANCH" =~ $version_regex ]]; then
+  echo "The branch is master, proceeding with multi-arch build"
+  docker buildx create --name multiarch --use
+  docker buildx build -f "$DOCKERFILE_PATH" -t "$IMAGE_NAME" --platform linux/arm64,linux/amd64 --push .
+else
+  echo "The branch is not master, proceeding with classic build"
+  docker buildx build -f "$DOCKERFILE_PATH" -t "$IMAGE_NAME" --push .
+fi