Allow updater to fetch python packages for service

configextractor_/configextractor_.py

@@ -11,6 +11,7 @@
 import hashlib
 import os
 import regex
+import sys
 import tempfile
 
 from configextractor.main import ConfigExtractor as CX
@@ -47,6 +48,10 @@ def _load_rules(self) -> None:
             blocklist = []
             blocklist_location = os.path.join(self.rules_directory, 'blocked_parsers')
             self.source_map = json.loads(open(os.path.join(self.rules_directory, 'source_mapping.json')).read())
+            python_packages_dir = os.path.join(self.rules_directory, 'python_packages')
+            if python_packages_dir not in sys.path:
+                sys.path.append(python_packages_dir)
+
             if os.path.exists(blocklist_location):
                 for line in open(blocklist_location, 'r').readlines():
                     _, source, _, parser_name = line.split('_', 3)

configextractor_/update_server.py

@@ -1,8 +1,10 @@
 import os
 import json
 import shutil
+import subprocess
 import tempfile
 
+
 from assemblyline.common import forge
 from assemblyline.common.isotime import epoch_to_iso
 from assemblyline.odm.models.signature import Signature
@@ -52,6 +54,17 @@ def import_parsers(cx: ConfigExtractor):
                 self.log.debug(resp)
                 self.log.debug(source_map)
 
+                # Find any requirement files and pip install to a specific directory that will get transferred to services
+                for root, _, files in os.walk(dir):
+                    for file in files:
+                        if file == "requirements.txt":
+                            err = subprocess.run(['pip', 'install',
+                            '-r', os.path.join(root, file),
+                            '-t', os.path.join(self.latest_updates_dir, 'python_packages')]).stderr
+                            if err:
+                                self.log.error(err)
+
+
                 # Save a local copy of the directory that may potentially contain dependency libraries for the parsers
                 try:
                     destination = os.path.join(self.latest_updates_dir, source_name)