apk artifacts analyzer closes#2444 + upgraded stringsifter (intelowlp…

…roject#2469)

* init

* init

* works

* docker

* comments

* apk_artifacts

* apk_artifacts

* file supp

api_app/analyzers_manager/file_analyzers/artifacts.py

@@ -0,0 +1,83 @@
+import logging
+
+from api_app.analyzers_manager.classes import DockerBasedAnalyzer, FileAnalyzer
+from api_app.analyzers_manager.exceptions import AnalyzerRunException
+from tests.mock_utils import MockUpResponse
+
+logger = logging.getLogger(__name__)
+
+
+class Artifacts(FileAnalyzer, DockerBasedAnalyzer):
+    name: str = "apk_analyzer"
+    url: str = "http://malware_tools_analyzers:4002/artifacts"
+    # interval between http request polling
+    poll_distance: int = 2
+    # http request polling max number of tries
+    max_tries: int = 10
+    artifacts_report: bool = False
+    artifacts_analysis: bool = True
+
+    def update(self) -> bool:
+        pass
+
+    def run(self):
+        if self.artifacts_report and self.artifacts_analysis:
+            raise AnalyzerRunException(
+                "You can't run both report and analysis at the same time"
+            )
+        binary = self.read_file_bytes()
+        fname = str(self.filename).replace("/", "_").replace(" ", "_")
+        args = [f"@{fname}"]
+        if self.artifacts_report:
+            args.append("--report")
+        req_data = {"args": args}
+        req_files = {fname: binary}
+        logger.info(
+            f"Running {self.analyzer_name} on {self.filename} with args: {args}"
+        )
+        result = self._docker_run(req_data, req_files, analyzer_name=self.analyzer_name)
+        return result
+
+    # flake8: noqa
+    @staticmethod
+    def mocked_docker_analyzer_get(*args, **kwargs):
+        return MockUpResponse(
+            {
+                "report": {
+                    "name": "APK_Artifacts",
+                    "process_time": 5.07,
+                    "status": "SUCCESS",
+                    "end_time": "2024-08-27T10:03:15.563886Z",
+                    "parameters": {},
+                    "type": "analyzer",
+                    "id": 72,
+                    "report": {
+                        "dex": ["classes.dex"],
+                        "md5": "8a05a189e58ccd7275f7ffdf88c2c191",
+                        "root": [],
+                        "family": {
+                            "name": "CryCrypto",
+                            "match": 11.11,
+                            "value": {
+                                "intent": 33.33,
+                                "permission": 0.0,
+                                "application": 0.0,
+                            },
+                        },
+                        "string": {"known": [], "base64": [], "telegram_id": []},
+                        "library": [],
+                        "network": {"ip": [], "url": [], "param": []},
+                        "sandbox": [
+                            "https://tria.ge/s?q=8a05a189e58ccd7275f7ffdf88c2c191",
+                            "https://www.joesandbox.com/analysis/search?q=8a05a189e58ccd7275f7ffdf88c2c191",
+                            "https://www.virustotal.com/gui/search/8a05a189e58ccd7275f7ffdf88c2c191",
+                            "https://bazaar.abuse.ch/browse.php?search=md5:8a05a189e58ccd7275f7ffdf88c2c191",
+                            "https://koodous.com/apks?search=8a05a189e58ccd7275f7ffdf88c2c191",
+                        ],
+                        "version": "1.1.1",
+                        "elapsed_time": 0.02,
+                    },
+                }
+            },
+            200,
+        )

api_app/analyzers_manager/migrations/0119_analyzer_config_apk_artifacts.py

@@ -0,0 +1,122 @@
+from django.db import migrations
+from django.db.models.fields.related_descriptors import (
+    ForwardManyToOneDescriptor,
+    ForwardOneToOneDescriptor,
+    ManyToManyDescriptor,
+)
+
+plugin = {
+    "python_module": {
+        "health_check_schedule": None,
+        "update_schedule": None,
+        "module": "artifacts.Artifacts",
+        "base_path": "api_app.analyzers_manager.file_analyzers",
+    },
+    "name": "APK_Artifacts",
+    "description": "[Artifacts](https://github.com/guelfoweb/artifacts) is a tool that does APK strings analysis. Useful for first analysis.",
+    "disabled": False,
+    "soft_time_limit": 60,
+    "routing_key": "default",
+    "health_check_status": True,
+    "type": "file",
+    "docker_based": False,
+    "maximum_tlp": "RED",
+    "observable_supported": [],
+    "supported_filetypes": [
+        "application/zip",
+        "application/java-archive",
+        "application/vnd.android.package-archive",
+        "application/x-dex",
+    ],
+    "run_hash": False,
+    "run_hash_type": "",
+    "not_supported_filetypes": [],
+    "model": "analyzers_manager.AnalyzerConfig",
+}
+
+params = []
+
+values = []
+
+
+def _get_real_obj(Model, field, value):
+    def _get_obj(Model, other_model, value):
+        if isinstance(value, dict):
+            real_vals = {}
+            for key, real_val in value.items():
+                real_vals[key] = _get_real_obj(other_model, key, real_val)
+            value = other_model.objects.get_or_create(**real_vals)[0]
+        # it is just the primary key serialized
+        else:
+            if isinstance(value, int):
+                if Model.__name__ == "PluginConfig":
+                    value = other_model.objects.get(name=plugin["name"])
+                else:
+                    value = other_model.objects.get(pk=value)
+            else:
+                value = other_model.objects.get(name=value)
+        return value
+
+    if (
+        type(getattr(Model, field))
+        in [ForwardManyToOneDescriptor, ForwardOneToOneDescriptor]
+        and value
+    ):
+        other_model = getattr(Model, field).get_queryset().model
+        value = _get_obj(Model, other_model, value)
+    elif type(getattr(Model, field)) in [ManyToManyDescriptor] and value:
+        other_model = getattr(Model, field).rel.model
+        value = [_get_obj(Model, other_model, val) for val in value]
+    return value
+
+
+def _create_object(Model, data):
+    mtm, no_mtm = {}, {}
+    for field, value in data.items():
+        value = _get_real_obj(Model, field, value)
+        if type(getattr(Model, field)) is ManyToManyDescriptor:
+            mtm[field] = value
+        else:
+            no_mtm[field] = value
+    try:
+        o = Model.objects.get(**no_mtm)
+    except Model.DoesNotExist:
+        o = Model(**no_mtm)
+        o.full_clean()
+        o.save()
+        for field, value in mtm.items():
+            attribute = getattr(o, field)
+            if value is not None:
+                attribute.set(value)
+        return False
+    return True
+
+
+def migrate(apps, schema_editor):
+    Parameter = apps.get_model("api_app", "Parameter")
+    PluginConfig = apps.get_model("api_app", "PluginConfig")
+    python_path = plugin.pop("model")
+    Model = apps.get_model(*python_path.split("."))
+    if not Model.objects.filter(name=plugin["name"]).exists():
+        exists = _create_object(Model, plugin)
+        if not exists:
+            for param in params:
+                _create_object(Parameter, param)
+            for value in values:
+                _create_object(PluginConfig, value)
+
+
+def reverse_migrate(apps, schema_editor):
+    python_path = plugin.pop("model")
+    Model = apps.get_model(*python_path.split("."))
+    Model.objects.get(name=plugin["name"]).delete()
+
+
+class Migration(migrations.Migration):
+    atomic = False
+    dependencies = [
+        ("api_app", "0062_alter_parameter_python_module"),
+        ("analyzers_manager", "0118_analyzer_config_droidlysis"),
+    ]
+
+    operations = [migrations.RunPython(migrate, reverse_migrate)]

integrations/malware_tools_analyzers/Dockerfile

@@ -1,7 +1,6 @@
 # this base was derived from Thug requirements:
 # https://github.com/REMnux/docker/blob/master/thug/Dockerfile
-FROM python:3.8-slim
-# CARE! Stringsifter (numpy) does work only with python3.8
+FROM python:3.9-slim
 
 ENV PROJECT_PATH /opt/deploy
 ENV LOG_PATH /var/log/intel_owl/malware_tools_analyzers
@@ -127,6 +126,16 @@ RUN python3 -m venv venv \
  && rm -f "${PROJECT_PATH}/droidlysis/conf/general.conf"
 COPY ./droidlysis/general.conf ${PROJECT_PATH}/droidlysis/conf/general.conf
 
+# Install artifacts
+WORKDIR ${PROJECT_PATH}/artifacts
+RUN python3 -m venv venv \
+    && . venv/bin/activate \
+    && pip3 install --no-cache-dir --upgrade pip \
+    && git clone https://github.com/guelfoweb/artifacts.git \
+    && cd artifacts \
+    && pip install --no-cache-dir -r requirements.txt \
+    && chmod +x artifacts.py 
+
 # Install Detect-it-Easy
 WORKDIR ${PROJECT_PATH}/die
 RUN apt-get install --no-install-recommends -y wget tar libglib2.0-0 && \

integrations/malware_tools_analyzers/app.py

@@ -40,6 +40,11 @@
 executor = Executor(app)
 shell2http = Shell2HTTP(app, executor)
 
+# we are changeing the directory for execution of
+#  artifacts script as it requires us to be in the
+#  same directory
+os.chdir("/opt/deploy/artifacts/artifacts")
+
 
 # Box-JS report
 def read_files_and_make_report(dir_loc):
@@ -221,6 +226,11 @@ def intercept_thug_result(context, future: Future) -> None:
     callback_fn=intercept_droidlysis_result,
 )
 
+# flake8: noqa
+shell2http.register_command(
+    endpoint="artifacts",
+    command_name="/opt/deploy/artifacts/venv/bin/python3 /opt/deploy/artifacts/artifacts/artifacts.py",
+)
 # goresym is the command for GoReSym
 shell2http.register_command(
     endpoint="goresym",

integrations/malware_tools_analyzers/requirements/stringsifter-requirements.txt

@@ -1,2 +1,2 @@
 wheel==0.40.0
-stringsifter==2.20201202
+stringsifter==3.20230711