Playbook choice for pivot and ingestor (intelowlproject#2411)

* As title Signed-off-by: 0ssigeno <s.berni@certego.net> * Blake Signed-off-by: 0ssigeno <s.berni@certego.net> * Fixes Signed-off-by: 0ssigeno <s.berni@certego.net> * Fix tests Signed-off-by: 0ssigeno <s.berni@certego.net> * Fixes Signed-off-by: 0ssigeno <s.berni@certego.net> * Fixes Signed-off-by: 0ssigeno <s.berni@certego.net> * Fixes Signed-off-by: 0ssigeno <s.berni@certego.net> * Fix migrations after rebase Signed-off-by: 0ssigeno <s.berni@certego.net> --------- Signed-off-by: 0ssigeno <s.berni@certego.net>
standa4 · Oct 11, 2024 · 1e3756b · 1e3756b
1 parent 9a2f19e
commit 1e3756b
Show file tree

Hide file tree

Showing 25 changed files with 412 additions and 97 deletions.
diff --git a/...nalyzers_manager/migrations/0105_alter_analyzerconfig_not_supported_filetypes_and_more.py b/...nalyzers_manager/migrations/0105_alter_analyzerconfig_not_supported_filetypes_and_more.py
@@ -0,0 +1,170 @@
+# Generated by Django 4.2.11 on 2024-07-09 07:50
+
+from django.db import migrations, models
+
+import api_app.fields
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("analyzers_manager", "0104_analyzer_config_goresym"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="analyzerconfig",
+            name="not_supported_filetypes",
+            field=api_app.fields.ChoiceArrayField(
+                base_field=models.CharField(
+                    choices=[
+                        ("application/w-script-file", "Wscript"),
+                        ("application/javascript", "Javascript1"),
+                        ("application/x-javascript", "Javascript2"),
+                        ("text/javascript", "Javascript3"),
+                        ("application/x-vbscript", "Vb Script"),
+                        ("text/x-ms-iqy", "Iqy"),
+                        ("application/vnd.android.package-archive", "Apk"),
+                        ("application/x-dex", "Dex"),
+                        ("application/onenote", "One Note"),
+                        ("application/zip", "Zip1"),
+                        ("multipart/x-zip", "Zip2"),
+                        ("application/java-archive", "Java"),
+                        ("text/rtf", "Rtf1"),
+                        ("application/rtf", "Rtf2"),
+                        ("application/x-sharedlib", "Shared Lib"),
+                        ("application/vnd.microsoft.portable-executable", "Exe"),
+                        ("application/x-elf", "Elf"),
+                        ("application/octet-stream", "Octet"),
+                        ("application/vnd.tcpdump.pcap", "Pcap"),
+                        ("application/pdf", "Pdf"),
+                        ("text/html", "Html"),
+                        ("application/x-mspublisher", "Pub"),
+                        ("application/vnd.ms-excel.addin.macroEnabled", "Excel Macro1"),
+                        (
+                            "application/vnd.ms-excel.sheet.macroEnabled.12",
+                            "Excel Macro2",
+                        ),
+                        ("application/vnd.ms-excel", "Excel1"),
+                        ("application/excel", "Excel2"),
+                        (
+                            "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+                            "Doc",
+                        ),
+                        ("application/xml", "Xml1"),
+                        ("text/xml", "Xml2"),
+                        ("application/encrypted", "Encrypted"),
+                        ("text/plain", "Plain"),
+                        ("text/csv", "Csv"),
+                        (
+                            "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+                            "Pptx",
+                        ),
+                        ("application/msword", "Word1"),
+                        (
+                            "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+                            "Word2",
+                        ),
+                        ("application/vnd.ms-powerpoint", "Powerpoint"),
+                        ("application/vnd.ms-office", "Office"),
+                        ("application/x-binary", "Binary"),
+                        ("application/x-macbinary", "Mac1"),
+                        ("application/mac-binary", "Mac2"),
+                        ("application/x-mach-binary", "Mac3"),
+                        ("application/x-zip-compressed", "Compress1"),
+                        ("application/x-compressed", "Compress2"),
+                        ("application/vnd.ms-outlook", "Outlook"),
+                        ("message/rfc822", "Eml"),
+                        ("application/pkcs7-signature", "Pkcs7"),
+                        ("application/x-pkcs7-signature", "Xpkcs7"),
+                        ("multipart/mixed", "Mixed"),
+                        ("text/x-shellscript", "X Shellscript"),
+                        ("application/x-chrome-extension", "Crx"),
+                        ("application/json", "Json"),
+                        ("application/x-executable", "Executable"),
+                    ],
+                    max_length=90,
+                ),
+                blank=True,
+                default=list,
+                size=None,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="analyzerconfig",
+            name="supported_filetypes",
+            field=api_app.fields.ChoiceArrayField(
+                base_field=models.CharField(
+                    choices=[
+                        ("application/w-script-file", "Wscript"),
+                        ("application/javascript", "Javascript1"),
+                        ("application/x-javascript", "Javascript2"),
+                        ("text/javascript", "Javascript3"),
+                        ("application/x-vbscript", "Vb Script"),
+                        ("text/x-ms-iqy", "Iqy"),
+                        ("application/vnd.android.package-archive", "Apk"),
+                        ("application/x-dex", "Dex"),
+                        ("application/onenote", "One Note"),
+                        ("application/zip", "Zip1"),
+                        ("multipart/x-zip", "Zip2"),
+                        ("application/java-archive", "Java"),
+                        ("text/rtf", "Rtf1"),
+                        ("application/rtf", "Rtf2"),
+                        ("application/x-sharedlib", "Shared Lib"),
+                        ("application/vnd.microsoft.portable-executable", "Exe"),
+                        ("application/x-elf", "Elf"),
+                        ("application/octet-stream", "Octet"),
+                        ("application/vnd.tcpdump.pcap", "Pcap"),
+                        ("application/pdf", "Pdf"),
+                        ("text/html", "Html"),
+                        ("application/x-mspublisher", "Pub"),
+                        ("application/vnd.ms-excel.addin.macroEnabled", "Excel Macro1"),
+                        (
+                            "application/vnd.ms-excel.sheet.macroEnabled.12",
+                            "Excel Macro2",
+                        ),
+                        ("application/vnd.ms-excel", "Excel1"),
+                        ("application/excel", "Excel2"),
+                        (
+                            "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+                            "Doc",
+                        ),
+                        ("application/xml", "Xml1"),
+                        ("text/xml", "Xml2"),
+                        ("application/encrypted", "Encrypted"),
+                        ("text/plain", "Plain"),
+                        ("text/csv", "Csv"),
+                        (
+                            "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+                            "Pptx",
+                        ),
+                        ("application/msword", "Word1"),
+                        (
+                            "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+                            "Word2",
+                        ),
+                        ("application/vnd.ms-powerpoint", "Powerpoint"),
+                        ("application/vnd.ms-office", "Office"),
+                        ("application/x-binary", "Binary"),
+                        ("application/x-macbinary", "Mac1"),
+                        ("application/mac-binary", "Mac2"),
+                        ("application/x-mach-binary", "Mac3"),
+                        ("application/x-zip-compressed", "Compress1"),
+                        ("application/x-compressed", "Compress2"),
+                        ("application/vnd.ms-outlook", "Outlook"),
+                        ("message/rfc822", "Eml"),
+                        ("application/pkcs7-signature", "Pkcs7"),
+                        ("application/x-pkcs7-signature", "Xpkcs7"),
+                        ("multipart/mixed", "Mixed"),
+                        ("text/x-shellscript", "X Shellscript"),
+                        ("application/x-chrome-extension", "Crx"),
+                        ("application/json", "Json"),
+                        ("application/x-executable", "Executable"),
+                    ],
+                    max_length=90,
+                ),
+                blank=True,
+                default=list,
+                size=None,
+            ),
+        ),
+    ]
diff --git a/api_app/ingestors_manager/admin.py b/api_app/ingestors_manager/admin.py
@@ -18,7 +18,11 @@ class IngestorConfigAdminView(PythonConfigAdminView):
         "name",
         "python_module",
         "disabled",
-        "playbook_to_execute",
+        "get_playbooks_choice",
         "schedule",
     )
     exclude = ["user", "periodic_task"]
+
+    @admin.display(description="Playbooks choice")
+    def get_playbooks_choice(self, instance: IngestorConfig):
+        return instance.playbooks_names
diff --git a/api_app/ingestors_manager/classes.py b/api_app/ingestors_manager/classes.py
@@ -52,7 +52,11 @@ def _user(self):
 
     def before_run(self):
         self._config: IngestorConfig
-        self._config.validate_playbook_to_execute(self._user)
+        self._config.validate_playbooks(self._user)
+
+    def get_playbook_to_execute(self):
+        self._config: IngestorConfig
+        return self._config.playbooks_choice.first()
 
     def after_run_success(self, content):
         # exhaust generator
@@ -67,7 +71,8 @@ def after_run_success(self, content):
                 content,
                 TLP.CLEAR.value,
                 self._user,
-                self._config.delay,
+                delay=self._config.delay,
+                playbook_to_execute=self.get_playbook_to_execute(),
             ),
             maxlen=0,
         )

diff --git a/api_app/ingestors_manager/migrations/0020_ingestor_config_threatfox.py b/api_app/ingestors_manager/migrations/0020_ingestor_config_threatfox.py
@@ -47,13 +47,8 @@ def reverse_migrate(apps, schema_editor):
 class Migration(migrations.Migration):
     atomic = False
     dependencies = [
-        ('api_app', '0062_alter_parameter_python_module'),
-        ('ingestors_manager', '0019_ingestor_config_malwarebazaar'),
-    ]
-
-    operations = [
-        migrations.RunPython(
-            migrate, reverse_migrate
-        )
+        ("api_app", "0062_alter_parameter_python_module"),
+        ("ingestors_manager", "0019_ingestor_config_malwarebazaar"),
     ]
 
+    operations = [migrations.RunPython(migrate, reverse_migrate)]
diff --git a/...p/ingestors_manager/migrations/0023_remove_ingestorconfig_playbook_to_execute_and_more.py b/...p/ingestors_manager/migrations/0023_remove_ingestorconfig_playbook_to_execute_and_more.py
@@ -0,0 +1,28 @@
+# Generated by Django 4.2.11 on 2024-07-09 07:50
+
+from django.db import migrations, models
+
+
+def migrate(apps, schema_editor):
+    IngestorConfig = apps.get_model("ingestors_manager", "IngestorConfig")
+    for ingestor in IngestorConfig.objects.all():
+        ingestor.playbooks_choice.set([ingestor.playbook_to_execute])
+        ingestor.save()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("playbooks_manager", "0050_add_goresym_to_sample_static_abalysis"),
+        ("ingestors_manager", "0022_ingestor_fix_duplicated_users"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="ingestorconfig",
+            name="playbooks_choice",
+            field=models.ManyToManyField(
+                related_name="ingestors", to="playbooks_manager.playbookconfig"
+            ),
+        ),
+        migrations.RunPython(migrate, reverse_code=migrations.RunPython.noop),
+    ]
diff --git a/api_app/ingestors_manager/migrations/0024_remove_ingestorconfig_playbook_to_execute.py b/api_app/ingestors_manager/migrations/0024_remove_ingestorconfig_playbook_to_execute.py
@@ -0,0 +1,19 @@
+# Generated by Django 4.2.11 on 2024-07-09 08:22
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        (
+            "ingestors_manager",
+            "0023_remove_ingestorconfig_playbook_to_execute_and_more",
+        ),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="ingestorconfig",
+            name="playbook_to_execute",
+        ),
+    ]
diff --git a/api_app/ingestors_manager/models.py b/api_app/ingestors_manager/models.py
@@ -69,10 +69,9 @@ class IngestorConfig(PythonConfig, CreateJobsFromPlaybookInterface):
         related_name="%(class)ss",
         limit_choices_to={"base_path": PythonModuleBasePaths.Ingestor.value},
     )
-    playbook_to_execute = models.ForeignKey(
+    playbooks_choice = models.ManyToManyField(
         PlaybookConfig,
         related_name="ingestors",
-        on_delete=models.CASCADE,
     )
     user = models.ForeignKey(
         settings.AUTH_USER_MODEL,

diff --git a/api_app/ingestors_manager/serializers.py b/api_app/ingestors_manager/serializers.py
@@ -16,8 +16,8 @@
 
 class IngestorConfigSerializer(PythonConfigSerializer):
     schedule = CrontabScheduleSerializer(read_only=True)
-    playbook_to_execute = rfs.SlugRelatedField(
-        queryset=PlaybookConfig.objects.all(), slug_field="name", many=False
+    playbooks_choice = rfs.SlugRelatedField(
+        queryset=PlaybookConfig.objects.all(), slug_field="name", many=True
     )
 
     class Meta:
@@ -33,7 +33,9 @@ class IngestorConfigSerializerForMigration(PythonConfigSerializerForMigration):
     schedule = CrontabScheduleSerializer(read_only=True)
     periodic_task = PeriodicTaskSerializer(read_only=True)
     user = UserProfileSerializer(read_only=True)
-    playbook_to_execute = rfs.SlugRelatedField(read_only=True, slug_field="name")
+    playbooks_choice = rfs.SlugRelatedField(
+        read_only=True, slug_field="name", many=True
+    )
 
     class Meta:
         model = IngestorConfig