Closes #1886 observable analyzer Tor_Nodes_DanMeUk (#2217)

* fixes 1886 observable analyzer Tor_Nodes_DanMeUk

added analyzer code for Tor_Node_DanMeUk

migrations for Tor_Nodes_DanMeUk analyzer

added Tor_Nodes_DanMeUk analyzer to FREE_TO_USE_ANALYZERS

updated docs

checking pre-commit

* Updated Usage.md

* Update api_app/analyzers_manager/migrations/0071_analyzer_config_tor_nodes_danmeuk.py

---------

Co-authored-by: Matteo Lodi <30625432+mlodic@users.noreply.github.com>

api_app/analyzers_manager/migrations/0071_analyzer_config_tor_nodes_danmeuk.py

@@ -0,0 +1,101 @@
+from django.db import migrations
+from django.db.models.fields.related_descriptors import (
+    ForwardManyToOneDescriptor,
+    ForwardOneToOneDescriptor,
+    ManyToManyDescriptor,
+)
+
+plugin = {'python_module': {'health_check_schedule': None, 'update_schedule': {'minute': '0', 'hour': '*/6', 'day_of_week': '*', 'day_of_month': '*', 'month_of_year': '*'}, 'module': 'tor_nodes_danmeuk.TorNodesDanMeUK', 'base_path': 'api_app.analyzers_manager.observable_analyzers'}, 'name': 'Tor_Nodes_DanMeUk', 'description': 'check if an IP is a Tor Node', 'disabled': False, 'soft_time_limit': 30, 'routing_key': 'default', 'health_check_status': True, 'type': 'observable', 'docker_based': False, 'maximum_tlp': 'RED', 'observable_supported': ['ip'], 'supported_filetypes': [], 'run_hash': False, 'run_hash_type': '', 'not_supported_filetypes': [], 'model': 'analyzers_manager.AnalyzerConfig'}
+
+params = []
+
+values = []
+
+
+def _get_real_obj(Model, field, value):
+    def _get_obj(Model, other_model, value):
+        if isinstance(value, dict):
+            real_vals = {}
+            for key, real_val in value.items():
+                real_vals[key] = _get_real_obj(other_model, key, real_val)
+            value = other_model.objects.get_or_create(**real_vals)[0]
+        # it is just the primary key serialized
+        else:
+            if isinstance(value, int):
+                if Model.__name__ == "PluginConfig":
+                    value = other_model.objects.get(name=plugin["name"])
+                else:
+                    value = other_model.objects.get(pk=value)
+            else:
+                value = other_model.objects.get(name=value)
+        return value
+
+    if (
+        type(getattr(Model, field))
+        in [ForwardManyToOneDescriptor, ForwardOneToOneDescriptor]
+        and value
+    ):
+        other_model = getattr(Model, field).get_queryset().model
+        value = _get_obj(Model, other_model, value)
+    elif type(getattr(Model, field)) in [ManyToManyDescriptor] and value:
+        other_model = getattr(Model, field).rel.model
+        value = [_get_obj(Model, other_model, val) for val in value]
+    return value
+
+def _create_object(Model, data):
+    mtm, no_mtm = {}, {}
+    for field, value in data.items():
+        value = _get_real_obj(Model, field, value)
+        if type(getattr(Model, field)) is ManyToManyDescriptor:
+            mtm[field] = value
+        else:
+            no_mtm[field] = value
+    try:
+        o = Model.objects.get(**no_mtm)
+    except Model.DoesNotExist:
+        o = Model(**no_mtm)
+        o.full_clean()
+        o.save()
+        for field, value in mtm.items():
+            attribute = getattr(o, field)
+            if value is not None:
+                attribute.set(value)
+        return False
+    return True
+
+def migrate(apps, schema_editor):
+    Parameter = apps.get_model("api_app", "Parameter")
+    PluginConfig = apps.get_model("api_app", "PluginConfig")    
+    python_path = plugin.pop("model")
+    Model = apps.get_model(*python_path.split("."))
+    if not Model.objects.filter(name=plugin["name"]).exists():
+        exists = _create_object(Model, plugin)
+        if not exists:
+            for param in params:
+                _create_object(Parameter, param)
+            for value in values:
+                _create_object(PluginConfig, value)
+
+
+
+def reverse_migrate(apps, schema_editor):
+    python_path = plugin.pop("model")
+    Model = apps.get_model(*python_path.split("."))
+    Model.objects.get(name=plugin["name"]).delete()
+
+
+
+class Migration(migrations.Migration):
+    atomic = False
+    dependencies = [
+        ('api_app', '0061_job_depth_analysis'),
+        ('analyzers_manager', '0070_urlhaus_threatfox_disable_param'),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            migrate, reverse_migrate
+        )
+    ]
+
+

api_app/analyzers_manager/observable_analyzers/tor_nodes_danmeuk.py

@@ -0,0 +1,87 @@
+# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl
+# See the file 'LICENSE' for copying permission.
+
+import logging
+import os
+
+import requests
+from django.conf import settings
+
+from api_app.analyzers_manager import classes
+from api_app.analyzers_manager.exceptions import AnalyzerRunException
+from tests.mock_utils import MockUpResponse, if_mock_connections, patch
+
+logger = logging.getLogger(__name__)
+
+db_name = "tor_nodes_addresses.txt"
+database_location = f"{settings.MEDIA_ROOT}/{db_name}"
+
+
+class TorNodesDanMeUK(classes.ObservableAnalyzer):
+    def run(self):
+        result = {"found": False}
+        if not os.path.isfile(database_location) and not self.update():
+            raise AnalyzerRunException("Failed extraction of tor db")
+
+        if not os.path.exists(database_location):
+            raise AnalyzerRunException(
+                f"database location {database_location} does not exist"
+            )
+
+        with open(database_location, "r", encoding="utf-8") as f:
+            db = f.read()
+
+        db_list = db.split("\n")
+        if self.observable_name in db_list:
+            result["found"] = True
+
+        return result
+
+    @classmethod
+    def update(cls):
+        try:
+            logger.info("starting download of tor nodes from https://dan.me.uk")
+            url = "https://www.dan.me.uk/torlist/?full"
+            r = requests.get(url)
+            r.raise_for_status()
+
+            data_extracted = r.content.decode()
+            tor_nodes_list = data_extracted.split("\n")
+
+            with open(database_location, "w", encoding="utf-8") as f:
+                for ip in tor_nodes_list:
+                    if ip:
+                        f.write(f"{ip}\n")
+
+            if not os.path.exists(database_location):
+                return False
+
+            logger.info("ended download of tor nodes from https://dan.me.uk")
+            return True
+        except Exception as e:
+            logger.exception(e)
+
+        return False
+
+    @classmethod
+    def _monkeypatch(cls):
+        patches = [
+            if_mock_connections(
+                patch(
+                    "requests.get",
+                    return_value=MockUpResponse(
+                        {},
+                        200,
+                        content=b"""100.10.37.131
+100.14.156.183
+100.16.153.149
+100.4.55.171
+100.8.8.137
+101.100.141.137
+101.55.125.10
+102.119.243.196""",
+                    ),
+                ),
+            )
+        ]
+        return super()._monkeypatch(patches=patches)

api_app/playbooks_manager/migrations/0029_add_tor_nodes_danmeuk_analyzer_free_to_use.py

@@ -0,0 +1,34 @@
+# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl
+# See the file 'LICENSE' for copying permission.
+
+
+from django.db import migrations
+
+
+def migrate(apps, schema_editor):
+    playbook_config = apps.get_model("playbooks_manager", "PlaybookConfig")
+    AnalyzerConfig = apps.get_model("analyzers_manager", "AnalyzerConfig")
+    pc = playbook_config.objects.get(name="FREE_TO_USE_ANALYZERS")
+    pc.analyzers.add(AnalyzerConfig.objects.get(name="Tor_Nodes_DanMeUk").id)
+    pc.full_clean()
+    pc.save()
+
+
+def reverse_migrate(apps, schema_editor):
+    playbook_config = apps.get_model("playbooks_manager", "PlaybookConfig")
+    AnalyzerConfig = apps.get_model("analyzers_manager", "AnalyzerConfig")
+    pc = playbook_config.objects.get(name="FREE_TO_USE_ANALYZERS")
+    pc.analyzers.remove(AnalyzerConfig.objects.get(name="Tor_Nodes_DanMeUk").id)
+    pc.full_clean()
+    pc.save()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("playbooks_manager", "0028_add_bgp_ranking_to_free_to_use"),
+        ("analyzers_manager", "0071_analyzer_config_tor_nodes_danmeuk"),
+    ]
+
+    operations = [
+        migrations.RunPython(migrate, reverse_migrate),
+    ]

docs/source/Usage.md

@@ -233,6 +233,7 @@ The following is the list of the available analyzers you can run out-of-the-box.
 * `TalosReputation`: check an IP reputation from [Talos](https://talosintelligence.com/reputation_center/)
 * `ThreatFox`: search for an IOC in [ThreatFox](https://threatfox.abuse.ch/api/)'s database
 * `Threatminer`: retrieve data from [Threatminer](https://www.threatminer.org/) API
+* `TorNodesDanMeUk`: check if an IP is a Tor Node using a list of all Tor nodes provided by [dan.me.uk](https://www.dan.me.uk/tornodes)
 * `TorProject`: check if an IP is a Tor Exit Node
 * `Triage_Search`: Search for reports of observables or upload from URL on triage cloud
 * `Tranco`: Check if a domain is in the latest [Tranco](https://tranco-list.eu/) ranking top sites list