Issues/42

Dockerfile

@@ -1,74 +1,47 @@
 ARG branch=latest
 FROM cccs/assemblyline-v4-service-base:$branch AS base
 
-ENV SERVICE_PATH configextractor_.ConfigExtractor
-ENV YARA_VERSION=4.2.0
+ENV SERVICE_PATH configextractor_.configextractor_.ConfigExtractor
+ENV YARA_VERSION=4.2.3
 
 USER assemblyline
 RUN pip uninstall -y yara-python
 
 USER root
-RUN apt-get update && apt-get install -y git libssl1.1 libmagic1 upx-ucl && rm -rf /var/lib/apt/lists/*
+RUN apt-get update && apt-get install -y git libssl1.1 libmagic1 upx-ucl mono-complete && rm -rf /var/lib/apt/lists/*
 # Create a temporary image to do our compiling in
 FROM base AS build
 
 RUN apt-get update && apt-get install -y git libssl-dev libmagic-dev automake libtool make gcc wget libjansson-dev pkg-config && rm -rf /var/lib/apt/lists/*
 
 # Compile and install YARA
-RUN wget -O /tmp/yara.tar.gz https://github.com/VirusTotal/yara/archive/v$YARA_VERSION.tar.gz
+RUN wget -O /tmp/yara.tar.gz https://github.com/VirusTotal/yara/archive/v${YARA_VERSION}.tar.gz
 RUN tar -zxf /tmp/yara.tar.gz -C /tmp
-WORKDIR /tmp/yara-$YARA_VERSION
+WORKDIR /tmp/yara-${YARA_VERSION}
 RUN ./bootstrap.sh
-RUN ./configure --enable-cuckoo --enable-magic --enable-dotnet --with-crypto --prefix /tmp/yara_install
+RUN ./configure --enable-magic --enable-dotnet --with-crypto --prefix /tmp/yara_install
 RUN make
 RUN make install
 
+
 # Build the yara python plugins, install other dependencies
 USER assemblyline
 RUN touch /tmp/before-pip
-# Get ConfigExtractor library
-RUN git clone --recurse-submodules https://github.com/CybercentreCanada/configextractor-py.git /tmp/configextractor-py
-RUN pip install --global-option="build" --global-option="--enable-dotnet" --global-option="--enable-magic" yara-python==$YARA_VERSION
-RUN pip install --no-cache-dir --user --use-deprecated=legacy-resolver \
- gitpython plyara /tmp/configextractor-py/RATDecoders/ /tmp/configextractor-py/
-
-RUN git clone https://github.com/kevoreilly/CAPEv2.git /tmp/CAPEv2
-
-# Install packages required for CAPE's cuckoo library
-RUN pip install chardet pyzipper \
- && rm -rf ~/.cache/pip
-
-# Remove disabled/test parsers
-RUN rm -f /tmp/CAPEv2/modules/processing/parsers/CAPE/*.py_disabled
-RUN rm -f /tmp/CAPEv2/modules/processing/parsers/CAPE/test_cape.py
 
-# Remove 'bad' parsers
-RUN rm -f /tmp/CAPEv2/modules/processing/parsers/CAPE/LokiBot.py
-RUN rm -f /tmp/CAPEv2/modules/processing/parsers/CAPE/GuLoader.py
-RUN rm -f /tmp/CAPEv2/modules/processing/parsers/CAPE/SquirrelWaffle.py
-RUN rm -f /tmp/CAPEv2/modules/processing/parsers/CAPE/BuerLoader.py
-
-RUN mkdir -p /tmp/al_service/CAPEv2/modules/processing/parsers/CAPE/
-RUN cp -r /tmp/CAPEv2/modules/processing/parsers/CAPE/* /tmp/al_service/CAPEv2/modules/processing/parsers/CAPE/
-RUN mkdir -p /tmp/al_service/CAPEv2/lib
-RUN cp -r /tmp/CAPEv2/lib/* /tmp/al_service/CAPEv2/lib/
-
-RUN rm -rf /tmp/CAPEv2
+# Get ConfigExtractor library
+RUN pip install -U git+https://github.com/CybercentreCanada/configextractor-py@revamp
+RUN pip install -U git+https://github.com/CybercentreCanada/maco
 
-# # Remove files that existed before the pip install so that our copy command below doesn't take a snapshot of
-# # files that already exist in the base image
-# RUN find /var/lib/assemblyline/.local -type f ! -newer /tmp/before-pip -delete
+RUN pip install --no-cache-dir --user --global-option="build" --global-option="--enable-dotnet" --global-option="--enable-magic" git+https://github.com/VirusTotal/yara-python.git@d29ca083f4cb25ea52988314b844bb7cf8594b5b
+RUN pip install --no-cache-dir --user gitpython plyara markupsafe==2.0.1
 
-# # Switch back to root and change the ownership of the files to be copied due to bitbucket pipeline uid nonsense
-# USER root
-# RUN chown root:root -R /var/lib/assemblyline/.local
+# Public libraries that can be used by parsers
+RUN pip install --no-cache-dir --user netstruct beautifulsoup4 pyOpenSSL
 
 # Revert back to before the compile
 FROM base
 
 COPY --from=build /tmp/yara_install /usr/local
-COPY --from=build /tmp/configextractor-py/dependencies /opt/al_service/dependencies
-COPY --from=build /tmp/al_service/CAPEv2/ /opt/al_service/CAPEv2
 COPY --chown=assemblyline:assemblyline --from=build /var/lib/assemblyline/.local /var/lib/assemblyline/.local
 
 # Create directories
@@ -79,10 +52,6 @@ RUN mkdir -p /opt/al_service
 WORKDIR /opt/al_service
 COPY . .
 
-# Make sure we actually have the right version of pyparsing by uninstalling it as root
-# then later reinstalling an exact version as the user account
-RUN pip uninstall --yes pyparsing flask
-
 # Cleanup
 RUN rm ./Dockerfile
 
@@ -91,11 +60,7 @@ RUN chown -R assemblyline /opt/al_service
 
 # Patch version in manifest
 ARG version=4.0.0.dev1
-ENV PUBLIC_SERVICE_VERSION=$version
-ENV CAPE_PARSERS_DIR=/opt/al_service/CAPEv2/modules/processing/parsers/CAPE/
-ENV PYTHONPATH=$PYTHONPATH:/opt/al_service/CAPEv2/
 RUN sed -i -e "s/\$SERVICE_TAG/$version/g" service_manifest.yml
 
 # Switch to assemblyline user
 USER assemblyline
-RUN pip install --user pyparsing==2.3.0 flask~=1.1.0

README.md

@@ -1,4 +1,25 @@
 # ConfigExtractor Service
-**NOTE** : All malware parsers, yara rules etc used in this service are open-source and available in the [ConfigExtractor](https://github.com/CybercentreCanada/configextractor-py) Python library.
+This Assemblyline service extracts malware configurations (such as IP, URL and domain) for various malware family by leveraging the [ConfigExtractor Python library](https://github.com/CybercentreCanada/configextractor-py) for analysis.
 
-This Assemblyline service extracts malware configurations (such as IP, URL and domain) for various malware family by leveraging the ConfigExtractor Python library for analysis.
+## Updater
+
+### Sources
+The updater for this service requires matches on directories containing parsers.
+
+
+For example, the CAPE source will have a match pattern of `.*/modules/processing/parsers/CAPE/$` in which we're trying to target the parsers in this directory only.
+
+### Persistence
+The updater assumes that you have attached a storage volume to store your collection of sources. Contrary to other services, this updater relies on a storage volume to maintain persistence rather than Assemblyline's datastore.
+
+### Python Packages
+The updater is able to scan through the directory containing parsers and look for `requirements.txt` files and install Python packages to a directory that should get passed onto service instances.
+
+## [ConfigExtractor Python Library](https://github.com/CybercentreCanada/configextractor-py)
+
+All parser directories that are able to work with this library should also be compatible with the service.
+
+At the time of writing, we officially support the following frameworks:
+ - [MWCP](https://github.com/dod-cyber-crime-center/DC3-MWCP)
+ - [CAPE w/ MACO output](https://github.com/kevoreilly/CAPEv2)
+ - [MACO](https://github.com/CybercentreCanada/Maco)