Update abuse.ch services (#2683)

* Added service_api_key parameter and migrations

* Added mb_get and mb_google

* Added logging for api key

* Fix deepsource warning

* WAD Analyzer, Closes #814 (#2655)

* WAD Analyzer, Closes #814

* Remove WAD from FREE_TO_USE_ANALYZERS playbook

* Update WAD maximum_tlp to CLEAR

* Fix WAD monkeypatch

* Update WAD error message to a more generic one

* Update migration number and dependencies

* Added service_api_key parameter and migrations

* Added mb_get and mb_google

* Added logging for api key

* Fix deepsource warning

* Fixed migration number

* Removed wrongly duplicated migration

* Added other analyzers to reverse_migrate

* Added common mixin and updated code accordingly

* Solved MRO

* Deepsource

* Made mixin compatible with ingestors

* Removed inheritance from analyzers

* Added missing return statement

* Removed old configs and used a property

* Left behind values

* Added explainatory comment

---------

Co-authored-by: Pragati Raj <basedBaba@proton.me>

api_app/analyzers_manager/file_analyzers/yaraify_file_scan.py

@@ -4,7 +4,6 @@
 import json
 import logging
 import time
-from typing import Dict
 
 import requests
 
@@ -27,7 +26,10 @@ class YARAifyFileScan(FileAnalyzer, YARAify):
     skip_noisy: bool
     skip_known: bool
 
-    def config(self, runtime_configuration: Dict):
+    def update(self) -> bool:
+        pass
+
+    def config(self, runtime_configuration: dict):
         FileAnalyzer.config(self, runtime_configuration)
         self.query = "lookup_hash"
         YARAify.config(self, runtime_configuration)
@@ -73,7 +75,9 @@ def run(self):
                 "file": (name_to_send, file),
             }
             logger.info(f"yara file scan md5 {self.md5} sending sample for analysis")
-            response = requests.post(self.url, files=files_)
+            response = requests.post(
+                self.url, files=files_, headers=self.authentication_header
+            )
             response.raise_for_status()
             scan_response = response.json()
             scan_query_status = scan_response.get("query_status")
@@ -92,7 +96,9 @@ def run(self):
                             f"task_id: {task_id}"
                         )
                         data = {"query": "get_results", "task_id": task_id}
-                        response = requests.post(self.url, json=data)
+                        response = requests.post(
+                            self.url, json=data, headers=self.authentication_header
+                        )
                         response.raise_for_status()
                         task_response = response.json()
                         logger.debug(task_response)

api_app/analyzers_manager/migrations/0147_alter_analyzer_config_feodo_yaraify_urlhaus_yaraify_scan.py

@@ -0,0 +1,98 @@
+from django.db import migrations
+
+
+def migrate(apps, schema_editor):
+    Parameter = apps.get_model("api_app", "Parameter")
+    PythonModule = apps.get_model("api_app", "PythonModule")
+
+    # observables
+    observable_analyzers = [
+        "urlhaus.URLHaus",
+        "yaraify.YARAify",
+        "feodo_tracker.Feodo_Tracker",
+        "threatfox.ThreatFox",
+        "mb_get.MB_GET",
+        "mb_google.MB_GOOGLE",
+    ]
+    for observable_analyzer in observable_analyzers:
+        module = PythonModule.objects.get(
+            module=observable_analyzer,
+            base_path="api_app.analyzers_manager.observable_analyzers",
+        )
+        Parameter.objects.create(
+            name="service_api_key",
+            type="str",
+            description="Optional API key to connect to abuse.ch services.",
+            is_secret=True,
+            required=False,
+            python_module=module,
+        )
+
+    # files
+    yaraify_scan_module = PythonModule.objects.get(
+        module="yaraify_file_scan.YARAifyFileScan",
+        base_path="api_app.analyzers_manager.file_analyzers",
+    )
+    Parameter.objects.create(
+        name="service_api_key",
+        type="str",
+        description="Optional API key to connect to abuse.ch services.",
+        is_secret=True,
+        required=False,
+        python_module=yaraify_scan_module,
+    )
+
+
+def reverse_migrate(apps, schema_editor):
+    Parameter = apps.get_model("api_app", "Parameter")
+    PythonModule = apps.get_model("api_app", "PythonModule")
+
+    # observables
+    observable_analyzers = [
+        "urlhaus.URLHaus",
+        "yaraify.YARAify",
+        "feodo_tracker.Feodo_Tracker",
+        "threatfox.ThreatFox",
+        "mb_get.MB_GET",
+        "mb_google.MB_GOOGLE",
+    ]
+    for observable_analyzer in observable_analyzers:
+        module = PythonModule.objects.get(
+            module=observable_analyzer,
+            base_path="api_app.analyzers_manager.observable_analyzers",
+        )
+        Parameter.objects.get(
+            name="service_api_key",
+            type="str",
+            description="Optional API key to connect to abuse.ch services.",
+            is_secret=True,
+            required=False,
+            python_module=module,
+        ).delete()
+
+    # files
+    yaraify_scan_module = PythonModule.objects.get(
+        module="yaraify_file_scan.YARAifyFileScan",
+        base_path="api_app.analyzers_manager.file_analyzers",
+    )
+    Parameter.objects.get(
+        name="service_api_key",
+        type="str",
+        description="Optional API key to connect to abuse.ch services.",
+        is_secret=True,
+        required=False,
+        python_module=yaraify_scan_module,
+    ).delete()
+
+
+class Migration(migrations.Migration):
+    atomic = False
+    dependencies = [
+        ("api_app", "0065_job_mpnodesearch"),
+        (
+            "analyzers_manager",
+            "0146_analyzer_config_wad",
+        ),
+    ]
+
+    operations = [migrations.RunPython(migrate, reverse_migrate)]

api_app/analyzers_manager/observable_analyzers/feodo_tracker.py

@@ -11,12 +11,14 @@
 
 from api_app.analyzers_manager import classes
 from api_app.analyzers_manager.exceptions import AnalyzerRunException
+from api_app.mixins import AbuseCHMixin
+from api_app.models import PluginConfig
 from tests.mock_utils import MockUpResponse, if_mock_connections, patch
 
 logger = logging.getLogger(__name__)
 
 
-class Feodo_Tracker(classes.ObservableAnalyzer):
+class Feodo_Tracker(AbuseCHMixin, classes.ObservableAnalyzer):
     """
     Feodo Tracker offers various blocklists,
     helping network owners to protect their
@@ -65,6 +67,22 @@ def run(self):
             raise AnalyzerRunException(f"Key error in run: {e}")
         return result
 
+    # this is necessary because during the "update()" flow the config()
+    # method is not called and the attributes would not be accessible by "cls"
+    @classmethod
+    def get_service_auth_headers(cls) -> {}:
+        for plugin in PluginConfig.objects.filter(
+            parameter__python_module=cls.python_module,
+            parameter__is_secret=True,
+            parameter__name="service_api_key",
+        ):
+            if plugin.value:
+                logger.debug("Found auth key for feodo tracker update")
+                return {"Auth-Key": plugin.value}
+
+        logger.debug("Not found auth key for feodo tracker update")
+        return {}
+
     @classmethod
     def update(cls) -> bool:
         """
@@ -74,7 +92,7 @@ def update(cls) -> bool:
             logger.info(f"starting download of db from {db_url}")
 
             try:
-                r = requests.get(db_url)
+                r = requests.get(db_url, headers=cls.get_service_auth_headers())
                 r.raise_for_status()
             except requests.RequestException:
                 return False

api_app/analyzers_manager/observable_analyzers/mb_get.py

@@ -1,27 +1,40 @@
 # This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl
 # See the file 'LICENSE' for copying permission.
+import logging
 
 import requests
 
 from api_app.analyzers_manager import classes
+from api_app.mixins import AbuseCHMixin
 from tests.mock_utils import MockUpResponse, if_mock_connections, patch
 
+logger = logging.getLogger(__name__)
 
-class MB_GET(classes.ObservableAnalyzer):
+
+class MB_GET(AbuseCHMixin, classes.ObservableAnalyzer):
     url: str = "https://mb-api.abuse.ch/api/v1/"
     sample_url: str = "https://bazaar.abuse.ch/sample/"
 
+    def update(self) -> bool:
+        pass
+
     def run(self):
-        return self.query_mb_api(observable_name=self.observable_name)
+        return self.query_mb_api(
+            observable_name=self.observable_name,
+            headers=self.authentication_header,
+        )
 
     @classmethod
-    def query_mb_api(cls, observable_name: str) -> dict:
+    def query_mb_api(cls, observable_name: str, headers: dict = None) -> dict:
         """
         This is in a ``classmethod`` so it can be reused in ``MB_GOOGLE``.
         """
         post_data = {"query": "get_info", "hash": observable_name}
 
-        response = requests.post(cls.url, data=post_data)
+        if headers is None:
+            headers = {}
+
+        response = requests.post(cls.url, data=post_data, headers=headers)
         response.raise_for_status()
 
         result = response.json()

api_app/analyzers_manager/observable_analyzers/mb_google.py

@@ -11,13 +11,19 @@ class MB_GOOGLE(MB_GET):
     This is a modified version of MB_GET.
     """
 
+    def update(self) -> bool:
+        pass
+
     def run(self):
         results = {}
 
         query = f"{self.observable_name} site:bazaar.abuse.ch"
         for url in googlesearch.search(query, stop=20):
             mb_hash = url.split("/")[-2]
-            res = super().query_mb_api(observable_name=mb_hash)
+            res = super().query_mb_api(
+                observable_name=mb_hash,
+                headers=self.authentication_header,
+            )
             results[mb_hash] = res
 
         return results

api_app/analyzers_manager/observable_analyzers/threatfox.py

@@ -2,14 +2,18 @@
 # See the file 'LICENSE' for copying permission.
 
 import json
+import logging
 
 import requests
 
 from api_app.analyzers_manager import classes
+from api_app.mixins import AbuseCHMixin
 from tests.mock_utils import MockUpResponse, if_mock_connections, patch
 
+logger = logging.getLogger(__name__)
 
-class ThreatFox(classes.ObservableAnalyzer):
+
+class ThreatFox(AbuseCHMixin, classes.ObservableAnalyzer):
     url: str = "https://threatfox-api.abuse.ch/api/v1/"
     disable: bool = False  # optional
 
@@ -22,7 +26,11 @@ def run(self):
 
         payload = {"query": "search_ioc", "search_term": self.observable_name}
 
-        response = requests.post(self.url, data=json.dumps(payload))
+        response = requests.post(
+            self.url,
+            data=json.dumps(payload),
+            headers=self.authentication_header,
+        )
         response.raise_for_status()
 
         result = response.json()

api_app/analyzers_manager/observable_analyzers/urlhaus.py

@@ -1,14 +1,18 @@
 # This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl
 # See the file 'LICENSE' for copying permission.
+import logging
 
 import requests
 
-from api_app.analyzers_manager import classes
+from api_app.analyzers_manager.classes import ObservableAnalyzer
 from api_app.analyzers_manager.exceptions import AnalyzerRunException
+from api_app.mixins import AbuseCHMixin
 from tests.mock_utils import MockUpResponse, if_mock_connections, patch
 
+logger = logging.getLogger(__name__)
 
-class URLHaus(classes.ObservableAnalyzer):
+
+class URLHaus(AbuseCHMixin, ObservableAnalyzer):
     url = "https://urlhaus-api.abuse.ch/v1/"
     disable: bool = False  # optional
 
@@ -34,7 +38,11 @@ def run(self):
                 f"not supported observable type {self.observable_classification}."
             )
 
-        response = requests.post(self.url + uri, data=post_data, headers=headers)
+        response = requests.post(
+            self.url + uri,
+            data=post_data,
+            headers=self.authentication_header | headers,
+        )
         response.raise_for_status()
 
         return response.json()

api_app/analyzers_manager/observable_analyzers/yaraify.py

@@ -1,19 +1,26 @@
 # This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl
 # See the file 'LICENSE' for copying permission.
+import logging
 
 import requests
 
 from api_app.analyzers_manager.classes import ObservableAnalyzer
+from api_app.mixins import AbuseCHMixin
 from tests.mock_utils import MockUpResponse, if_mock_connections, patch
 
+logger = logging.getLogger(__name__)
 
-class YARAify(ObservableAnalyzer):
+
+class YARAify(AbuseCHMixin, ObservableAnalyzer):
     url: str = "https://yaraify-api.abuse.ch/api/v1/"
 
     query: str
     result_max: int
     _api_key_name: str
 
+    def update(self) -> bool:
+        pass
+
     def run(self):
         data = {"search_term": self.observable_name, "query": self.query}
 
@@ -23,7 +30,9 @@ def run(self):
         if getattr(self, "_api_key_name", None):
             data["malpedia-token"] = self._api_key_name
 
-        response = requests.post(self.url, json=data)
+        response = requests.post(
+            self.url, json=data, headers=self.authentication_header
+        )
         response.raise_for_status()
 
         result = response.json()

api_app/ingestors_manager/ingestors/malware_bazaar.py

@@ -10,12 +10,13 @@
 
 from api_app.ingestors_manager.classes import Ingestor
 from api_app.ingestors_manager.exceptions import IngestorRunException
+from api_app.mixins import AbuseCHMixin
 from tests.mock_utils import MockUpResponse, if_mock_connections
 
 logger = logging.getLogger(__name__)
 
 
-class MalwareBazaar(Ingestor):
+class MalwareBazaar(AbuseCHMixin, Ingestor):
     # API endpoint
     url: str
     # Download samples that are up to X hours old
@@ -32,6 +33,7 @@ def get_signature_information(self, signature):
         result = requests.post(
             self.url,
             data={"query": "get_siginfo", "signature": signature, "limit": 100},
+            headers=self.authentication_header,
             timeout=30,
         )
         result.raise_for_status()
@@ -75,6 +77,7 @@ def download_sample(self, h):
                 "query": "get_file",
                 "sha256_hash": h,
             },
+            headers=self.authentication_header,
             timeout=60,
         )
         sample_archive.raise_for_status()