Goresym analyzer, fixes#1451 and fixes executable file support

api_app/analyzers_manager/file_analyzers/goresym.py

@@ -0,0 +1,149 @@
+import logging
+
+from api_app.analyzers_manager.classes import DockerBasedAnalyzer, FileAnalyzer
+from api_app.analyzers_manager.exceptions import AnalyzerRunException
+from tests.mock_utils import MockUpResponse
+
+logger = logging.getLogger(__name__)
+
+
+class GoReSym(FileAnalyzer, DockerBasedAnalyzer):
+    name: str = "executable_analyzer"
+    url: str = "http://malware_tools_analyzers:4002/goresym"
+    # interval between http request polling
+    poll_distance: int = 5
+    # http request polling max number of tries
+    max_tries: int = 5
+    default: bool = False
+    paths: bool = False
+    types: bool = False
+    manual: str = ""
+    version: str = ""
+
+    def update(self) -> bool:
+        pass
+
+    def getArgs(self):
+        args = []
+        if self.default:
+            args.append("-d")
+        if self.paths:
+            args.append("-p")
+        if self.types:
+            args.append("-t")
+        if self.manual:
+            args.append("-m " + self.manual)
+        if self.version:
+            args.append("-v " + self.version)
+        return args
+
+    def run(self):
+        binary = self.read_file_bytes()
+        fname = str(self.filename).replace("/", "_").replace(" ", "_")
+        args = self.getArgs()
+        args.append(f"@{fname}")
+        req_data = {"args": args}
+        req_files = {fname: binary}
+        logger.info(
+            f"Running {self.analyzer_name} on {self.filename} with args: {args}"
+        )
+        result = self._docker_run(req_data, req_files, analyzer_name=self.analyzer_name)
+        if "error" in result:
+            er = (
+                "Failed to parse file: failed to read pclntab: failed to locate pclntab"
+            )
+            if result["error"] == er:
+                logger.warning(f"Not a GO-compiled file: {result['error']}")
+                return f"Not a Go-compiled file: {result['error']}"
+            raise AnalyzerRunException(result["error"])
+        return result
+
+    @staticmethod
+    def mocked_docker_analyzer_get(*args, **kwargs):
+        return MockUpResponse(
+            {
+                "report": {
+                    "Version": "1.22.3",
+                    "BuildId": """nwVuB9ChiwRxUV3uqogj
+                    /gqLuN_Lt0hiTuOBT4YDw
+                /7ArmhPs-KKm8m0kwm8Ud/RNgWKMZv5-p8k3r8fJCp""",
+                    "Arch": "amd64",
+                    "OS": "linux",
+                    "TabMeta": {
+                        "VA": 8261824,
+                        "Version": "1.20",
+                        "Endianess": "LittleEndian",
+                        "CpuQuantum": 1,
+                        "CpuQuantumStr": "x86/x64/wasm",
+                        "PointerSize": 8,
+                    },
+                    "ModuleMeta": {
+                        "VA": 10005792,
+                        "TextVA": 4198400,
+                        "Types": 7045120,
+                        "ETypes": 8248647,
+                        "Typelinks": {"Data": 8251936, "Len": 1791, "Capacity": 1791},
+                        "ITablinks": {"Data": 8259104, "Len": 339, "Capacity": 339},
+                        "LegacyTypes": {"Data": 0, "Len": 0, "Capacity": 0},
+                    },
+                    "Types": None,
+                    "Interfaces": None,
+                    "BuildInfo": {
+                        "GoVersion": "go1.22.3",
+                        "Path": "github.com/g4ze/byoc/reverse-proxy",
+                        "Main": {
+                            "Path": "github.com/g4ze/byoc",
+                            "Version": "(devel)",
+                            "Sum": "",
+                            "Replace": None,
+                        },
+                        "Deps": [
+                            {
+                                "Path": "github.com/joho/godotenv",
+                                "Version": "v1.5.1",
+                                "Sum": """h1:7eLL/
+                                +HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=""",
+                                "Replace": None,
+                            },
+                            {
+                                "Path": "github.com/lib/pq",
+                                "Version": "v1.10.9",
+                                "Sum": """h1:YXG7RB+JIjhP29X
+                                +OtkiDnYaXQwpS4JEWq7dtCCRUEw=""",
+                                "Replace": None,
+                            },
+                        ],
+                        "Settings": [
+                            {"Key": "-buildmode", "Value": "exe"},
+                            {"Key": "-compiler", "Value": "gc"},
+                            {"Key": "CGO_ENABLED", "Value": "1"},
+                            {"Key": "CGO_CFLAGS", "Value": ""},
+                            {"Key": "CGO_CPPFLAGS", "Value": ""},
+                            {"Key": "CGO_CXXFLAGS", "Value": ""},
+                            {"Key": "CGO_LDFLAGS", "Value": ""},
+                            {"Key": "GOARCH", "Value": "amd64"},
+                            {"Key": "GOOS", "Value": "linux"},
+                            {"Key": "GOAMD64", "Value": "v1"},
+                            {"Key": "vcs", "Value": "git"},
+                            {
+                                "Key": "vcs.revision",
+                                "Value": "34e6cafd47a85a15e9aeedd63786a2ba72e5b301",
+                            },
+                            {"Key": "vcs.time", "Value": "2024-06-24T07:44:25Z"},
+                            {"Key": "vcs.modified", "Value": "true"},
+                        ],
+                    },
+                    "Files": None,
+                    "UserFunctions": [
+                        {
+                            "Start": 7043712,
+                            "End": 7043758,
+                            "PackageName": "main",
+                            "FullName": "main.main.NewSingleHostReverseProxy.func1",
+                        },
+                    ],
+                    "StdFunctions": None,
+                }
+            },
+            200,
+        )

api_app/analyzers_manager/migrations/0103_add_x_executable.py

@@ -0,0 +1,208 @@
+from django.db import migrations, models
+
+import api_app.fields
+
+
+def migrate(apps, schema_editor):
+    AnalyzerConfig = apps.get_model("analyzers_manager", "AnalyzerConfig")
+    prev_analyzers_names = [
+        "ELF_Info",
+        "Capa_Info",
+        "Intezer_Scan",
+        "Malpedia_Scan",
+        "Qiling_Linux",
+    ]
+
+    for name in prev_analyzers_names:
+        config = AnalyzerConfig.objects.get(name=name)
+        try:
+            # we are not operating on the not_supported_filetypes field
+            # as no analyzer has "application/x-executable" in it
+            config.supported_filetypes.append("application/x-executable")
+        except ValueError:
+            pass
+        config.save()
+
+
+def reverse_migrate(apps, schema_editor):
+    AnalyzerConfig = apps.get_model("analyzers_manager", "AnalyzerConfig")
+    prev_analyzers_names = [
+        "ELF_Info",
+        "Capa_Info",
+        "Intezer_Scan",
+        "Malpedia_Scan",
+        "Qiling_Linux",
+    ]
+
+    for name in prev_analyzers_names:
+        config = AnalyzerConfig.objects.get(name=name)
+        try:
+            # we are not operating on the not_supported_filetypes field
+            # as no analyzer has "application/x-executable" in it
+            config.supported_filetypes.remove("application/x-executable")
+        except ValueError:
+            pass
+        config.save()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("analyzers_manager", "0102_analyzer_config_ja4_db"),
+        ("api_app", "0062_alter_parameter_python_module"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="analyzerconfig",
+            name="not_supported_filetypes",
+            field=api_app.fields.ChoiceArrayField(
+                base_field=models.CharField(
+                    choices=[
+                        ("application/w-script-file", "Wscript"),
+                        ("application/javascript", "Javascript1"),
+                        ("application/x-javascript", "Javascript2"),
+                        ("text/javascript", "Javascript3"),
+                        ("application/x-vbscript", "Vb Script"),
+                        ("text/x-ms-iqy", "Iqy"),
+                        ("application/vnd.android.package-archive", "Apk"),
+                        ("application/x-dex", "Dex"),
+                        ("application/onenote", "One Note"),
+                        ("application/zip", "Zip1"),
+                        ("multipart/x-zip", "Zip2"),
+                        ("application/java-archive", "Java"),
+                        ("text/rtf", "Rtf1"),
+                        ("application/rtf", "Rtf2"),
+                        ("application/x-sharedlib", "Shared Lib"),
+                        ("application/vnd.microsoft.portable-executable", "Exe"),
+                        ("application/x-elf", "Elf"),
+                        ("application/octet-stream", "Octet"),
+                        ("application/vnd.tcpdump.pcap", "Pcap"),
+                        ("application/pdf", "Pdf"),
+                        ("text/html", "Html"),
+                        ("application/x-mspublisher", "Pub"),
+                        ("application/vnd.ms-excel.addin.macroEnabled", "Excel Macro1"),
+                        (
+                            "application/vnd.ms-excel.sheet.macroEnabled.12",
+                            "Excel Macro2",
+                        ),
+                        ("application/vnd.ms-excel", "Excel1"),
+                        ("application/excel", "Excel2"),
+                        (
+                            "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+                            "Doc",
+                        ),
+                        ("application/xml", "Xml1"),
+                        ("text/xml", "Xml2"),
+                        ("application/encrypted", "Encrypted"),
+                        ("text/plain", "Plain"),
+                        ("text/csv", "Csv"),
+                        (
+                            "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+                            "Pptx",
+                        ),
+                        ("application/msword", "Word1"),
+                        (
+                            "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+                            "Word2",
+                        ),
+                        ("application/vnd.ms-powerpoint", "Powerpoint"),
+                        ("application/vnd.ms-office", "Office"),
+                        ("application/x-binary", "Binary"),
+                        ("application/x-macbinary", "Mac1"),
+                        ("application/mac-binary", "Mac2"),
+                        ("application/x-mach-binary", "Mac3"),
+                        ("application/x-zip-compressed", "Compress1"),
+                        ("application/x-compressed", "Compress2"),
+                        ("application/vnd.ms-outlook", "Outlook"),
+                        ("message/rfc822", "Eml"),
+                        ("application/pkcs7-signature", "Pkcs7"),
+                        ("application/x-pkcs7-signature", "Xpkcs7"),
+                        ("multipart/mixed", "Mixed"),
+                        ("text/x-shellscript", "X Shellscript"),
+                        ("application/x-executable", "Executable"),
+                    ],
+                    max_length=90,
+                ),
+                blank=True,
+                default=list,
+                size=None,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="analyzerconfig",
+            name="supported_filetypes",
+            field=api_app.fields.ChoiceArrayField(
+                base_field=models.CharField(
+                    choices=[
+                        ("application/w-script-file", "Wscript"),
+                        ("application/javascript", "Javascript1"),
+                        ("application/x-javascript", "Javascript2"),
+                        ("text/javascript", "Javascript3"),
+                        ("application/x-vbscript", "Vb Script"),
+                        ("text/x-ms-iqy", "Iqy"),
+                        ("application/vnd.android.package-archive", "Apk"),
+                        ("application/x-dex", "Dex"),
+                        ("application/onenote", "One Note"),
+                        ("application/zip", "Zip1"),
+                        ("multipart/x-zip", "Zip2"),
+                        ("application/java-archive", "Java"),
+                        ("text/rtf", "Rtf1"),
+                        ("application/rtf", "Rtf2"),
+                        ("application/x-sharedlib", "Shared Lib"),
+                        ("application/vnd.microsoft.portable-executable", "Exe"),
+                        ("application/x-elf", "Elf"),
+                        ("application/octet-stream", "Octet"),
+                        ("application/vnd.tcpdump.pcap", "Pcap"),
+                        ("application/pdf", "Pdf"),
+                        ("text/html", "Html"),
+                        ("application/x-mspublisher", "Pub"),
+                        ("application/vnd.ms-excel.addin.macroEnabled", "Excel Macro1"),
+                        (
+                            "application/vnd.ms-excel.sheet.macroEnabled.12",
+                            "Excel Macro2",
+                        ),
+                        ("application/vnd.ms-excel", "Excel1"),
+                        ("application/excel", "Excel2"),
+                        (
+                            "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+                            "Doc",
+                        ),
+                        ("application/xml", "Xml1"),
+                        ("text/xml", "Xml2"),
+                        ("application/encrypted", "Encrypted"),
+                        ("text/plain", "Plain"),
+                        ("text/csv", "Csv"),
+                        (
+                            "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+                            "Pptx",
+                        ),
+                        ("application/msword", "Word1"),
+                        (
+                            "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+                            "Word2",
+                        ),
+                        ("application/vnd.ms-powerpoint", "Powerpoint"),
+                        ("application/vnd.ms-office", "Office"),
+                        ("application/x-binary", "Binary"),
+                        ("application/x-macbinary", "Mac1"),
+                        ("application/mac-binary", "Mac2"),
+                        ("application/x-mach-binary", "Mac3"),
+                        ("application/x-zip-compressed", "Compress1"),
+                        ("application/x-compressed", "Compress2"),
+                        ("application/vnd.ms-outlook", "Outlook"),
+                        ("message/rfc822", "Eml"),
+                        ("application/pkcs7-signature", "Pkcs7"),
+                        ("application/x-pkcs7-signature", "Xpkcs7"),
+                        ("multipart/mixed", "Mixed"),
+                        ("text/x-shellscript", "X Shellscript"),
+                        ("application/x-executable", "Executable"),
+                    ],
+                    max_length=90,
+                ),
+                blank=True,
+                default=list,
+                size=None,
+            ),
+        ),
+        migrations.RunPython(migrate, reverse_migrate),
+    ]