Merge pull request #2 from quirinziessler/dev

solve conflicts

.github/workflows/gh-pages.yml

@@ -23,7 +23,7 @@ jobs:
       - name: Setup Node
         uses: actions/setup-node@v4
         with:
-          node-version: '16.x'
+          node-version: '20.x'
 
       - name: Cache dependencies
         uses: actions/cache@v4

.github/workflows/ruff.yml

@@ -0,0 +1,36 @@
+name: Ruff Linter
+
+on:
+  workflow_dispatch:
+  pull_request_target:
+  push:
+
+jobs:
+  ruff-linting:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target'
+        uses: actions/checkout@v4
+        # by default the pull_requst_target event checks out the base branch, i.e. dev
+        # so we need to explicitly checkout the head of the PR
+        # we use fetch-depth 0 to make sure the full history is checked out and we can compare against
+        # the base commit (branch) of the PR
+        # more info https://github.community/t/github-actions-are-severely-limited-on-prs/18179/16
+        # we checkout merge_commit here as this contains all new code from dev also. we don't need to compare against base_commit
+        with:
+          persist-credentials: false
+          fetch-depth: 0
+          ref: refs/pull/${{ github.event.pull_request.number }}/merge
+          # repository: ${{github.event.pull_request.head.repo.full_name}}
+
+      - name: Checkout
+        # for non PR runs we just checkout the default, which is a sha on a branch probably
+        if: github.event_name != 'pull_request' && github.event_name != 'pull_request_target'
+        uses: actions/checkout@v4
+
+      - name: Install Ruff Linter
+        run: pip install -r requirements-lint.txt
+
+      - name: Run Ruff Linter
+        run: ruff dojo

.gitignore

@@ -38,6 +38,7 @@ pip-delete-this-directory.txt
 .tox/
 .coverage
 .cache
+.ruff_cache
 nosetests.xml
 coverage.xml
 

Dockerfile.django-alpine

@@ -75,6 +75,7 @@ COPY \
   docker/entrypoint-unit-tests.sh \
   docker/entrypoint-unit-tests-devDocker.sh \
   docker/wait-for-it.sh \
+  docker/secret-file-loader.sh \
   docker/certs/* \
   /
 COPY wsgi.py manage.py docker/unit-tests.sh ./

Dockerfile.django-debian

@@ -80,6 +80,7 @@ COPY \
   docker/entrypoint-unit-tests.sh \
   docker/entrypoint-unit-tests-devDocker.sh \
   docker/wait-for-it.sh \
+  docker/secret-file-loader.sh \
   docker/certs/* \
   /
 COPY wsgi.py manage.py docker/unit-tests.sh ./

Dockerfile.integration-tests-debian

@@ -61,6 +61,7 @@ WORKDIR /app
 COPY --from=openapitools /opt/openapi-generator/modules/openapi-generator-cli/target/openapi-generator-cli.jar /usr/local/bin/openapi-generator-cli.jar
 
 COPY docker/wait-for-it.sh \
+  docker/secret-file-loader.sh \
   docker/entrypoint-integration-tests.sh \
   /
 

Dockerfile.nginx-alpine

@@ -34,7 +34,7 @@ RUN CPUCOUNT=1 pip3 wheel --wheel-dir=/tmp/wheels -r ./requirements.txt
 FROM build AS collectstatic
 
 # Node installation from https://github.com/nodejs/docker-node
-ENV NODE_VERSION 14.21.2
+ENV NODE_VERSION 20.11.0
 
 RUN addgroup -g 1000 node \
     && adduser -u 1000 -G node -s /bin/sh -D node \

Dockerfile.nginx-debian

@@ -44,8 +44,8 @@ RUN \
   apt-get -y update && \
   apt-get -y install --no-install-recommends apt-transport-https ca-certificates curl wget gnupg && \
   curl -sSL https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add --no-tty - && \
-  echo 'deb https://deb.nodesource.com/node_14.x bullseye main' > /etc/apt/sources.list.d/nodesource.list && \
-  echo 'deb-src https://deb.nodesource.com/node_14.x bullseye main' >> /etc/apt/sources.list.d/nodesource.list && \
+  echo 'deb https://deb.nodesource.com/node_20.x bullseye main' > /etc/apt/sources.list.d/nodesource.list && \
+  echo 'deb-src https://deb.nodesource.com/node_20.x bullseye main' >> /etc/apt/sources.list.d/nodesource.list && \
   apt-get update -y -o Dir::Etc::sourcelist="sources.list.d/nodesource.list" \
   -o Dir::Etc::sourceparts="-" -o APT::Get::List-Cleanup="0" && \
   curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - && \

docker/entrypoint-celery-worker.sh

@@ -3,6 +3,8 @@ umask 0002
 
 id
 
+. /secret-file-loader.sh
+
 # Allow for bind-mount multiple settings.py overrides
 FILES=$(ls /app/docker/extra_settings/* 2>/dev/null)
 NUM_FILES=$(echo "$FILES" | wc -w)

docker/entrypoint-initializer.sh

@@ -1,5 +1,7 @@
 #!/bin/sh
 
+. /secret-file-loader.sh
+
 initialize_data()
 {
     # Test types shall be initialized every time by the initializer, to make sure test types are complete

docker/entrypoint-integration-tests.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+. /secret-file-loader.sh
+
 echo "Testing DefectDojo Service"
 
 echo "Waiting max 60s for services to start"

docker/entrypoint-unit-tests-devDocker.sh

@@ -6,6 +6,8 @@ set -x
 set -e
 set -v
 
+. /secret-file-loader.sh
+
 cd /app
 # Unset the database URL so that we can force the DD_TEST_DATABASE_NAME (see django "DATABASES" configuration in settings.dist.py)
 unset DD_DATABASE_URL

docker/entrypoint-unit-tests.sh

@@ -6,6 +6,8 @@
 # set -e
 # set -v
 
+. /secret-file-loader.sh
+
 cd /app
 # Unset the database URL so that we can force the DD_TEST_DATABASE_NAME (see django "DATABASES" configuration in settings.dist.py)
 unset DD_DATABASE_URL

docker/entrypoint-uwsgi-dev.sh

@@ -1,5 +1,7 @@
 #!/bin/sh
 
+. /secret-file-loader.sh
+
 
 cd /app
 

docker/entrypoint-uwsgi.sh

@@ -1,5 +1,7 @@
 #!/bin/sh
 
+. /secret-file-loader.sh
+
 # Allow for bind-mount multiple settings.py overrides
 FILES=$(ls /app/docker/extra_settings/* 2>/dev/null)
 NUM_FILES=$(echo "$FILES" | wc -w)

docker/entrypoint.sh

@@ -1,5 +1,7 @@
 #!/bin/sh
 
+. /secret-file-loader.sh
+
 # Waits for the database to come up.
 ./docker/wait-for-it.sh $DD_DATABASE_HOST:$DD_DATABASE_PORT
 

docker/secret-file-loader.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+
+# Convert all environment variables with names ending in _FILE into the content of
+# the file that they point at and use the name without the trailing _FILE.
+# This can be used to carry in Docker secrets.
+# Inspired by https://github.com/grafana/grafana-docker/pull/166
+# But rewrote for /bin/sh
+for VAR_NAME in $(env | grep '^DD_[^=]\+_FILE=.\+' | sed -r "s/([^=]*)_FILE=.*/\1/g"); do
+    VAR_NAME_FILE="$VAR_NAME"_FILE
+    if [ -n "$(eval echo "\$$VAR_NAME")" ]; then
+        echo >&2 "WARNING: Both $VAR_NAME and $VAR_NAME_FILE are set. Content of $VAR_NAME will be overridden."
+    fi
+    echo "Getting secret $VAR_NAME from $(eval echo "\$$VAR_NAME_FILE")"
+    export "$VAR_NAME"="$(cat "$(eval echo "\$$VAR_NAME_FILE")")"
+    unset "$VAR_NAME_FILE"
+done

docs/content/en/integrations/parsers/file/acunetix.md

@@ -3,3 +3,6 @@ title: "Acunetix Scanner"
 toc_hide: true
 ---
 XML format
+
+### Sample Scan Data
+Sample Acunetix Scanner scans can be found [here](https://github.com/DefectDojo/django-DefectDojo/tree/master/unittests/scans/acunetix).

docs/content/en/integrations/parsers/file/acunetix360.md

@@ -3,3 +3,6 @@ title: "Acunetix 360 Scanner"
 toc_hide: true
 ---
 Vulnerabilities List - JSON report
+
+### Sample Scan Data
+Sample Acunetix 360 Scanner scans can be found [here](https://github.com/DefectDojo/django-DefectDojo/tree/master/unittests/scans/acunetix360).

docs/content/en/integrations/parsers/file/anchore_enterprise.md

@@ -3,3 +3,6 @@ title: "Anchore Enterprise Policy Check"
 toc_hide: true
 ---
 Anchore-CLI JSON policy check report format.
+
+### Sample Scan Data
+Sample Anchore Enterprise Policy Check scans can be found [here](https://github.com/DefectDojo/django-DefectDojo/tree/master/unittests/scans/anchore_enterprise).

docs/content/en/integrations/parsers/file/anchore_grype.md

@@ -12,7 +12,6 @@ Anchore Grype JSON files are created using the Grype CLI, using the '-o json' op
 grype yourApp/example-page -o json > example_vulns.json
 {{< /highlight >}}
 
-
 ### Acceptable JSON Format
 All properties are expected as strings and are required by the parser.
 

docs/content/en/integrations/parsers/file/anchorectl_policies.md

@@ -2,4 +2,7 @@
 title: "AnchoreCTL Policies Report"
 toc_hide: true
 ---
-AnchoreCTLs JSON policies report format
+AnchoreCTLs JSON policies report format
+
+### Sample Scan Data
+Sample AnchoreCTL Policies Report scans can be found [here](https://github.com/DefectDojo/django-DefectDojo/tree/master/unittests/scans/anchorectl_policies).

docs/content/en/integrations/parsers/file/anchorectl_vulns.md

@@ -2,4 +2,7 @@
 title: "AnchoreCTL Vuln Report"
 toc_hide: true
 ---
-AnchoreCTLs JSON vulnerability report format
+AnchoreCTLs JSON vulnerability report format
+
+### Sample Scan Data
+Sample AnchoreCTL Vuln Report scans can be found [here](https://github.com/DefectDojo/django-DefectDojo/tree/master/unittests/scans/anchorectl_vulns).

docs/content/en/integrations/parsers/file/appspider.md

@@ -4,3 +4,6 @@ toc_hide: true
 ---
 Use the VulnerabilitiesSummary.xml file found in the zipped report
 download.
+
+### Sample Scan Data
+Sample AppSpider (Rapid7) scans can be found [here](https://github.com/DefectDojo/django-DefectDojo/tree/master/unittests/scans/appspider).

docs/content/en/integrations/parsers/file/aqua.md

@@ -3,3 +3,6 @@ title: "Aqua"
 toc_hide: true
 ---
 JSON report format.
+
+### Sample Scan Data
+Sample Aqua scans can be found [here](https://github.com/DefectDojo/django-DefectDojo/tree/master/unittests/scans/aqua).

docs/content/en/integrations/parsers/file/arachni.md

@@ -9,3 +9,6 @@ Reports are generated with `arachni_reporter` tool this way:
 {{< highlight bash >}}
 arachni_reporter --reporter 'json' js.com.afr
 {{< /highlight >}}
+
+### Sample Scan Data
+Sample Arachni Scanner scans can be found [here](https://github.com/DefectDojo/django-DefectDojo/tree/master/unittests/scans/arachni).

docs/content/en/integrations/parsers/file/asff.md

@@ -8,3 +8,6 @@ AWS Security Hub consumes, aggregates, organizes, and prioritizes findings from
 Reference: https://docs.aws.amazon.com/securityhub/latest/userguide/securityhub-findings-format.html
 
 Prowler tool can generate this format with option `-M json-asff`.
+
+### Sample Scan Data
+Sample AWS Security Finding Format (ASFF) scans can be found [here](https://github.com/DefectDojo/django-DefectDojo/tree/master/unittests/scans/asff).

docs/content/en/integrations/parsers/file/auditjs.md

@@ -7,3 +7,6 @@ AuditJS scanning tool using OSSIndex database and generated with `--json` or `-j
 {{< highlight bash >}}
 auditjs ossi --json > auditjs_report.json
 {{< /highlight >}}
+
+### Sample Scan Data
+Sample AuditJS (OSSIndex) scans can be found [here](https://github.com/DefectDojo/django-DefectDojo/tree/master/unittests/scans/auditjs).

docs/content/en/integrations/parsers/file/aws_prowler.md

@@ -3,3 +3,6 @@ title: "AWS Prowler Scanner"
 toc_hide: true
 ---
 Prowler file can be imported as a CSV (`-M csv`) or JSON (`-M json`) file.
+
+### Sample Scan Data
+Sample AWS Prowler Scanner scans can be found [here](https://github.com/DefectDojo/django-DefectDojo/tree/master/unittests/scans/aws_prowler).

docs/content/en/integrations/parsers/file/aws_prowler_v3.md

@@ -8,7 +8,6 @@ DefectDojo parser accepts a .json file.  Please note: earlier versions of AWS Pr
 
 JSON reports can be created from the [AWS Prowler V3 CLI](https://docs.prowler.cloud/en/latest/tutorials/reporting/#json) using the following command: `prowler <provider> -M json`
 
-
 ### Acceptable JSON Format
 Parser expects an array of assessments.  All properties are strings and are required by the parser.
 

docs/content/en/integrations/parsers/file/aws_scout2.md

@@ -12,3 +12,6 @@ Please switch to the new parser for ScoutSuite.
 {{% alert title="Warning" color="warning" %}}
 This parser is disactivated by default in releases >= 2.3.1 and will be removed in release >= 3.x.x.
 {{% /alert %}}
+
+### Sample Scan Data
+Sample AWS Scout2 Scanner (deprecated) scans can be found [here](https://github.com/DefectDojo/django-DefectDojo/tree/master/unittests/scans/aws_scout2).

docs/content/en/integrations/parsers/file/awssecurityhub.md

@@ -7,7 +7,6 @@ DefectDojo parser accepts a .json file.
 
 JSON reports can be created from the [AWS Security Hub CLI](https://docs.aws.amazon.com/cli/latest/reference/securityhub/get-findings.html) using the following command: `aws securityhub get-findings`.
 
-
 ### Acceptable JSON Format
 Parser expects a .json file, with an array of Findings contained within a single JSON object.  All properties are strings and are required by the parser.
 

docs/content/en/integrations/parsers/file/azure_security_center_recommendations.md

@@ -3,3 +3,6 @@ title: "Azure Security Center Recommendations Scan"
 toc_hide: true
 ---
 Azure Security Center recommendations can be exported from the user interface in CSV format.
+
+### Sample Scan Data
+Sample Azure Security Center Recommendations Scan scans can be found [here](https://github.com/DefectDojo/django-DefectDojo/tree/master/unittests/scans/azure_security_center_recommendations).

docs/content/en/integrations/parsers/file/bandit.md

@@ -9,7 +9,6 @@ DefectDojo parser accepts a .json file.
 To export a .json file from Bandit, you will need to install and run the .json report formatter from your Bandit instance.  
 See Bandit documentation: https://bandit.readthedocs.io/en/latest/formatters/index.html
 
-
 ### Acceptable JSON Format
 All properties are expected as strings, except "metrics" properties, which are expected as numbers.  All properties are required by the parser.
 

docs/content/en/integrations/parsers/file/blackduck.md

@@ -8,4 +8,7 @@ toc_hide: true
 The zip file must contain the security.csv and files.csv in order to
 produce findings that bear file locations information.
 * Import a single security.csv file. Findings will not have any file location
-information.
+information.
+
+### Sample Scan Data
+Sample Blackduck Hub scans can be found [here](https://github.com/DefectDojo/django-DefectDojo/tree/master/unittests/scans/blackduck).

docs/content/en/integrations/parsers/file/blackduck_binary_analysis.md

@@ -18,3 +18,6 @@ Black Duck Binary Analysis can also detect if sensitive information like email a
 #### **How** ####
 * Initiate Black Duck Binary Analysis scans using the UI, REST API, or drivers such as [pwn_bdba_scan](https://github.com/0dayinc/pwn/blob/master/bin/pwn_bdba_scan) found within the security automation framework, [PWN](https://github.com/0dayinc/pwn)
 * Import a single BDBA vulnerabilty csv results file into DefectDojo leveraging the UI, REST API, or drivers such as [pwn_defectdojo_importscan](https://github.com/0dayInc/pwn/blob/master/bin/pwn_defectdojo_importscan) or [pwn_defectdojo_reimportscan](https://github.com/0dayInc/pwn/blob/master/bin/pwn_defectdojo_reimportscan).
+
+### Sample Scan Data
+Sample Blackduck Binary Analysis scans can be found [here](https://github.com/DefectDojo/django-DefectDojo/tree/master/unittests/scans/blackduck_binary_analysis).

docs/content/en/integrations/parsers/file/blackduck_component_risk.md

@@ -2,4 +2,7 @@
 title: "Blackduck Component Risk"
 toc_hide: true
 ---
-Upload the zip file containing the security.csv and files.csv.
+Upload the zip file containing the security.csv and files.csv.
+
+### Sample Scan Data
+Sample Blackduck Component Risk scans can be found [here](https://github.com/DefectDojo/django-DefectDojo/tree/master/unittests/scans/blackduck_component_risk).

docs/content/en/integrations/parsers/file/brakeman.md

@@ -3,3 +3,6 @@ title: "Brakeman Scan"
 toc_hide: true
 ---
 Import Brakeman Scanner findings in JSON format.
+
+### Sample Scan Data
+Sample Brakeman Scan scans can be found [here](https://github.com/DefectDojo/django-DefectDojo/tree/master/unittests/scans/brakeman).

docs/content/en/integrations/parsers/file/bugcrowd.md

@@ -3,3 +3,6 @@ title: "Bugcrowd"
 toc_hide: true
 ---
 Import Bugcrowd results in CSV format.
+
+### Sample Scan Data
+Sample Bugcrowd scans can be found [here](https://github.com/DefectDojo/django-DefectDojo/tree/master/unittests/scans/bugcrowd).

docs/content/en/integrations/parsers/file/bundler_audit.md

@@ -3,3 +3,6 @@ title: "Bundler-Audit"
 toc_hide: true
 ---
 Import the text output generated with bundle-audit check
+
+### Sample Scan Data
+Sample Bundler-Audit scans can be found [here](https://github.com/DefectDojo/django-DefectDojo/tree/master/unittests/scans/bundler_audit).

docs/content/en/integrations/parsers/file/burp_api.md

@@ -2,4 +2,7 @@
 title: "Burp REST API"
 toc_hide: true
 ---
-Import Burp REST API scan data in JSON format (/scan/[task_id] endpoint).
+Import Burp REST API scan data in JSON format (/scan/[task_id] endpoint).
+
+### Sample Scan Data
+Sample Burp REST API scans can be found [here](https://github.com/DefectDojo/django-DefectDojo/tree/master/unittests/scans/burp_api).