DataModel Plugin

api_app/analyzers_manager/classes.py

@@ -35,6 +35,48 @@ class BaseAnalyzerMixin(Plugin, metaclass=ABCMeta):
     ObservableTypes = ObservableTypes
     TypeChoices = TypeChoices
 
+    MALICIOUS_EVALUATION = 75
+    SUSPICIOUS_EVALUATION = 35
+    FALSE_POSITIVE = -50
+
+    def threat_to_evaluation(self, threat_level):
+        # MAGIC NUMBERS HERE!!!
+        # I know, it should be 25-50-75-100. We raised it a bit because too many false positives were generated
+        self.report: AnalyzerReport
+        if threat_level >= self.MALICIOUS_EVALUATION:
+            evaluation = self.report.data_model_class.EVALUATIONS.MALICIOUS.value
+        elif threat_level >= self.SUSPICIOUS_EVALUATION:
+            evaluation = self.report.data_model_class.EVALUATIONS.SUSPICIOUS.value
+        elif threat_level <= self.FALSE_POSITIVE:
+            evaluation = self.report.data_model_class.EVALUATIONS.TRUSTED.value
+        else:
+            evaluation = self.report.data_model_class.EVALUATIONS.CLEAN.value
+        return evaluation
+
+    def _do_create_data_model(self) -> bool:
+        if self.report.job.observable_classification == ObservableTypes.GENERIC:
+            return False
+        return True
+
+    def _create_data_model_mtm(self):
+        return {}
+
+    def _update_data_model(self, data_model) -> None:
+        mtm = self._create_data_model_mtm()
+        for field_name, value in mtm.items():
+            field = getattr(data_model, field_name)
+            field.add(*value)
+
+    def create_data_model(self):
+        self.report: AnalyzerReport
+        if self._do_create_data_model():
+            data_model = self.report.create_data_model()
+            if data_model:
+                self._update_data_model(data_model)
+                data_model.save()
+            return data_model
+        return None
+
     @classmethod
     @property
     def config_exception(cls):
@@ -108,7 +150,11 @@ def after_run_success(self, content):
         Args:
             content (any): The content to process after a successful run.
         """
-        super().after_run_success(self._validate_result(content, max_recursion=15))
+        result = super().after_run_success(
+            self._validate_result(content, max_recursion=15)
+        )
+        self.create_data_model()
+        return result
 
 
 class ObservableAnalyzer(BaseAnalyzerMixin, metaclass=ABCMeta):
@@ -326,7 +372,7 @@ def __polling(self, req_key: str, chance: int, re_poll_try: int = 0):
             return self.__polling(req_key, chance, re_poll_try=re_poll_try + 1)
         else:
             status = json_data.get("status", None)
-            if status and status == self._job.Status.RUNNING.value:
+            if status and status == self._job.STATUSES.RUNNING.value:
                 logger.info(
                     f"Poll number #{chance + 1}, "
                     f"status: 'running' <-- {self.__repr__()}"

api_app/analyzers_manager/file_analyzers/elf_info.py

@@ -46,7 +46,7 @@ def run(self):
             )
             logger.warning(warning_message)
             self.report.errors.append(warning_message)
-            self.report.status = self.report.Status.FAILED
+            self.report.status = self.report.STATUSES.FAILED
             self.report.save()
 
         return results

api_app/analyzers_manager/file_analyzers/pe_info.py

@@ -165,7 +165,7 @@ def run(self):
             )
             logger.warning(warning_message)
             self.report.errors.append(warning_message)
-            self.report.status = self.report.Status.FAILED
+            self.report.status = self.report.STATUSES.FAILED
             self.report.save()
 
         return results

api_app/analyzers_manager/file_analyzers/yara_scan.py

@@ -438,3 +438,33 @@ def update(cls):
         logger.info("Finished updating yara rules")
         set_permissions(settings.YARA_RULES_PATH)
         return True
+
+    def _create_data_model_mtm(self):
+        from api_app.data_model_manager.models import Signature
+
+        signatures = []
+        for yara_signatures in self.report.report.values():
+            for yara_signature in yara_signatures:
+                url = yara_signature.pop("rule_url", None)
+                sign = Signature.objects.create(
+                    provider=Signature.PROVIDERS.YARA.value,
+                    signature=yara_signature,
+                    url=url,
+                    score=1,
+                )
+                signatures.append(sign)
+
+        return {"signatures": signatures}
+
+    def _update_data_model(self, data_model):
+        from api_app.data_model_manager.models import FileDataModel
+
+        super()._update_data_model(data_model)
+        data_model: FileDataModel
+        signatures = data_model.signatures.count()
+
+        if signatures:
+            self.MALICIOUS_EVALUATION = 20
+            self.SUSPICIOUS_EVALUATION = 10
+
+            data_model.evaluation = self.threat_to_evaluation(signatures)

api_app/analyzers_manager/migrations/0134_analyzerconfig_mapping_data_model.py

@@ -0,0 +1,20 @@
+# Generated by Django 4.2.15 on 2024-10-14 07:24
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("analyzers_manager", "0133_analyzer_config_urldna_search"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="analyzerconfig",
+            name="mapping_data_model",
+            field=models.JSONField(
+                default=dict, help_text="Mapping data_model_key: analyzer_report_key. "
+            ),
+        ),
+    ]

api_app/analyzers_manager/migrations/0135_data_mapping.py

@@ -0,0 +1,58 @@
+# Generated by Django 4.2.15 on 2024-10-14 07:24
+
+from django.db import migrations
+
+
+def migrate_urlhaus(apps, schema_editor):
+    AnalyzerConfig = apps.get_model("analyzers_manager", "AnalyzerConfig")
+    ac = AnalyzerConfig.objects.filter(name="URLhaus").first()
+    if not ac:
+        return
+    ac.mapping_data_model = {
+        "urlhaus_reference": "external_references",
+        "$malicious": "evaluation",
+        "urls.url": "related_threats",
+    }
+    ac.save()
+
+
+def migrate_maxmind(apps, schema_editor):
+    AnalyzerConfig = apps.get_model("analyzers_manager", "AnalyzerConfig")
+    ac = AnalyzerConfig.objects.filter(name="MaxMindGeoIP").first()
+    if not ac:
+        return
+    ac.mapping_data_model = {
+        "country.iso_code": "country_code",
+        "registered_country_code.iso_code": "registered_country_code",
+        "autonomous_system_number": "asn",
+        "autonomous_system_organization": "isp",
+    }
+    ac.save()
+
+
+def migrate_abuse_ipdb(apps, schema_editor):
+    AnalyzerConfig = apps.get_model("analyzers_manager", "AnalyzerConfig")
+    ac = AnalyzerConfig.objects.filter(name="AbuseIPDB").first()
+    if not ac:
+        return
+    ac.mapping_data_model = {
+        "data.countryCode": "country_code",
+        "permalink": "external_references",
+        "data.hostnames": "resolutions",
+        "data.isp": "isp",
+        "categories_found": "tags",
+    }
+    ac.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("analyzers_manager", "0134_analyzerconfig_mapping_data_model"),
+    ]
+
+    operations = [
+        migrations.RunPython(migrate_maxmind, migrations.RunPython.noop),
+        migrations.RunPython(migrate_abuse_ipdb, migrations.RunPython.noop),
+        migrations.RunPython(migrate_urlhaus, migrations.RunPython.noop),
+    ]

api_app/analyzers_manager/migrations/0136_alter_analyzerconfig_mapping_data_model_and_more.py

@@ -0,0 +1,189 @@
+# Generated by Django 4.2.16 on 2024-11-08 09:21
+
+from django.db import migrations, models
+
+import api_app.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("analyzers_manager", "0135_data_mapping"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="analyzerconfig",
+            name="mapping_data_model",
+            field=models.JSONField(
+                default=dict,
+                help_text="Mapping analyzer_report_key: data_model_key. Keys preceded by the symbol $ will be considered as constants.",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="analyzerconfig",
+            name="not_supported_filetypes",
+            field=api_app.fields.ChoiceArrayField(
+                base_field=models.CharField(
+                    choices=[
+                        ("application/w-script-file", "Wscript"),
+                        ("application/javascript", "Javascript1"),
+                        ("application/x-javascript", "Javascript2"),
+                        ("text/javascript", "Javascript3"),
+                        ("application/x-vbscript", "Vb Script"),
+                        ("text/x-ms-iqy", "Iqy"),
+                        ("application/vnd.android.package-archive", "Apk"),
+                        ("application/x-dex", "Dex"),
+                        ("application/onenote", "One Note"),
+                        ("application/zip", "Zip1"),
+                        ("multipart/x-zip", "Zip2"),
+                        ("application/java-archive", "Java"),
+                        ("text/rtf", "Rtf1"),
+                        ("application/rtf", "Rtf2"),
+                        ("application/x-sharedlib", "Shared Lib"),
+                        ("application/vnd.microsoft.portable-executable", "Exe"),
+                        ("application/x-elf", "Elf"),
+                        ("application/octet-stream", "Octet"),
+                        ("application/vnd.tcpdump.pcap", "Pcap"),
+                        ("application/pdf", "Pdf"),
+                        ("text/html", "Html"),
+                        ("application/x-mspublisher", "Pub"),
+                        ("application/vnd.ms-excel.addin.macroEnabled", "Excel Macro1"),
+                        (
+                            "application/vnd.ms-excel.sheet.macroEnabled.12",
+                            "Excel Macro2",
+                        ),
+                        ("application/vnd.ms-excel", "Excel1"),
+                        ("application/excel", "Excel2"),
+                        (
+                            "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+                            "Doc",
+                        ),
+                        ("application/xml", "Xml1"),
+                        ("text/xml", "Xml2"),
+                        ("application/encrypted", "Encrypted"),
+                        ("text/plain", "Plain"),
+                        ("text/csv", "Csv"),
+                        (
+                            "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+                            "Pptx",
+                        ),
+                        ("application/msword", "Word1"),
+                        (
+                            "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+                            "Word2",
+                        ),
+                        ("application/vnd.ms-powerpoint", "Powerpoint"),
+                        ("application/vnd.ms-office", "Office"),
+                        ("application/x-binary", "Binary"),
+                        ("application/x-macbinary", "Mac1"),
+                        ("application/mac-binary", "Mac2"),
+                        ("application/x-mach-binary", "Mac3"),
+                        ("application/x-zip-compressed", "Compress1"),
+                        ("application/x-compressed", "Compress2"),
+                        ("application/vnd.ms-outlook", "Outlook"),
+                        ("message/rfc822", "Eml"),
+                        ("application/pkcs7-signature", "Pkcs7"),
+                        ("application/x-pkcs7-signature", "Xpkcs7"),
+                        ("multipart/mixed", "Mixed"),
+                        ("text/x-shellscript", "X Shellscript"),
+                        ("application/x-chrome-extension", "Crx"),
+                        ("application/json", "Json"),
+                        ("application/x-executable", "Executable"),
+                        ("text/x-java", "Java2"),
+                        ("text/x-kotlin", "Kotlin"),
+                        ("text/x-swift", "Swift"),
+                        ("text/x-objective-c", "Objective C Code"),
+                        ("application/x-ms-shortcut", "Lnk"),
+                    ],
+                    max_length=90,
+                ),
+                blank=True,
+                default=list,
+                size=None,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="analyzerconfig",
+            name="supported_filetypes",
+            field=api_app.fields.ChoiceArrayField(
+                base_field=models.CharField(
+                    choices=[
+                        ("application/w-script-file", "Wscript"),
+                        ("application/javascript", "Javascript1"),
+                        ("application/x-javascript", "Javascript2"),
+                        ("text/javascript", "Javascript3"),
+                        ("application/x-vbscript", "Vb Script"),
+                        ("text/x-ms-iqy", "Iqy"),
+                        ("application/vnd.android.package-archive", "Apk"),
+                        ("application/x-dex", "Dex"),
+                        ("application/onenote", "One Note"),
+                        ("application/zip", "Zip1"),
+                        ("multipart/x-zip", "Zip2"),
+                        ("application/java-archive", "Java"),
+                        ("text/rtf", "Rtf1"),
+                        ("application/rtf", "Rtf2"),
+                        ("application/x-sharedlib", "Shared Lib"),
+                        ("application/vnd.microsoft.portable-executable", "Exe"),
+                        ("application/x-elf", "Elf"),
+                        ("application/octet-stream", "Octet"),
+                        ("application/vnd.tcpdump.pcap", "Pcap"),
+                        ("application/pdf", "Pdf"),
+                        ("text/html", "Html"),
+                        ("application/x-mspublisher", "Pub"),
+                        ("application/vnd.ms-excel.addin.macroEnabled", "Excel Macro1"),
+                        (
+                            "application/vnd.ms-excel.sheet.macroEnabled.12",
+                            "Excel Macro2",
+                        ),
+                        ("application/vnd.ms-excel", "Excel1"),
+                        ("application/excel", "Excel2"),
+                        (
+                            "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+                            "Doc",
+                        ),
+                        ("application/xml", "Xml1"),
+                        ("text/xml", "Xml2"),
+                        ("application/encrypted", "Encrypted"),
+                        ("text/plain", "Plain"),
+                        ("text/csv", "Csv"),
+                        (
+                            "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+                            "Pptx",
+                        ),
+                        ("application/msword", "Word1"),
+                        (
+                            "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+                            "Word2",
+                        ),
+                        ("application/vnd.ms-powerpoint", "Powerpoint"),
+                        ("application/vnd.ms-office", "Office"),
+                        ("application/x-binary", "Binary"),
+                        ("application/x-macbinary", "Mac1"),
+                        ("application/mac-binary", "Mac2"),
+                        ("application/x-mach-binary", "Mac3"),
+                        ("application/x-zip-compressed", "Compress1"),
+                        ("application/x-compressed", "Compress2"),
+                        ("application/vnd.ms-outlook", "Outlook"),
+                        ("message/rfc822", "Eml"),
+                        ("application/pkcs7-signature", "Pkcs7"),
+                        ("application/x-pkcs7-signature", "Xpkcs7"),
+                        ("multipart/mixed", "Mixed"),
+                        ("text/x-shellscript", "X Shellscript"),
+                        ("application/x-chrome-extension", "Crx"),
+                        ("application/json", "Json"),
+                        ("application/x-executable", "Executable"),
+                        ("text/x-java", "Java2"),
+                        ("text/x-kotlin", "Kotlin"),
+                        ("text/x-swift", "Swift"),
+                        ("text/x-objective-c", "Objective C Code"),
+                        ("application/x-ms-shortcut", "Lnk"),
+                    ],
+                    max_length=90,
+                ),
+                blank=True,
+                default=list,
+                size=None,
+            ),
+        ),
+    ]