ENG-4766:Add trivy to SDK CI build (#77)

* ENG-4766:Add trivy to SDK CI build

* Adding Nightly build scan

* Adding document related to cve audit

.circleci/config.yml

@@ -9,21 +9,30 @@ workflows:
           context: org-sightmachine
       - run-unit-tests:
           context: org-sightmachine
+      - cve-repo-scan:
+          context: org-sightmachine
+  nightly-cve:
+    triggers:
+      - schedule:
+          # Should run around 6/7pm Pacific time
+          cron: "00 2 * * *"
+          filters:
+            branches:
+              only:
+                - master
+    jobs:
+      - cve-image-scan:
+          context: org-sightmachine
 
 orbs:
   gcp-gke: circleci/gcp-gke@0.2.0
   python: circleci/python@2.1.1
-  codecov: codecov/codecov@3.3.0
+  gcp-cli: circleci/gcp-cli@3.1.1
+  slack: circleci/slack@4.13.1
+  codecov: codecov/codecov@4.0.1
+  sm-build: sightmachine/ci-orb-sm-build@0.1.0
 
 commands:
-  disable_docker_hub:
-    description: "Disable Docker Hub access for running container/VM."
-    steps:
-      - run:
-          name: Disable Docker Hub
-          command: |
-            mkdir -p ~/.docker/
-            echo '{"auths":{"https://index.docker.io/v1/":{"auth":"YTpiYWQK"}}}' > ~/.docker/config.json
   disable_build_if_not_pr:
     description: "Stop the current build if not in a PR context."
     steps:
@@ -34,6 +43,62 @@ commands:
               circleci-agent step halt
             fi
 
+  merge-pr-branch:
+    steps:
+      # Make sure that PR merges can never be considered releases.
+      - run:
+          name: PR Merges should never be considered releases
+          command: echo 'unset CIRCLE_TAG' >> $BASH_ENV
+      - sm-build/jq_install
+      - run:
+          name: Get PR base branch.
+          command: |
+            echo $"export CIRCLE_TARGET_BRANCH=\"$(curl -s -u ops:${GITHUB_PERSONAL_ACCESS_TOKEN} https://api.github.com/repos/sightmachine/${CIRCLE_PROJECT_REPONAME}/pulls/${CIRCLE_PULL_REQUEST##*/} | jq --raw-output '.base.ref')\"" >> $BASH_ENV
+            source $BASH_ENV
+            echo "Github target branch: $CIRCLE_TARGET_BRANCH"
+      - run:
+          name: Merge with the base branch.
+          command: |
+            git --version
+            git config --global user.email 'ops@sightmachine.com'
+            git config --global user.name 'circleci'
+            git merge -m "Prospective PR merge" origin/$CIRCLE_TARGET_BRANCH
+
+  send_slack_msg_on_fail:
+    description: "If the job fails for the master branch, send a message to the Slack channel."
+    steps:
+      - slack/notify:
+          event: fail
+          channel: monitoring-sdk
+          # branch_pattern: master
+          custom: |
+            {
+              "text": "Uh-oh! *$CIRCLE_PROJECT_REPONAME* (master) - Job Failed <$CIRCLE_BUILD_URL|#$CIRCLE_BUILD_NUM>",
+              "blocks": [
+                {
+                  "text": {
+                    "text": "Uh-oh! *${CIRCLE_PROJECT_REPONAME}* (master) - Job Failed [#${CIRCLE_BUILD_NUM}]($CIRCLE_BUILD_URL)",
+                    "type": "mrkdwn"
+                  },
+                  "type": "header"
+                },
+                {
+                  "elements": [
+                    {
+                      "action_id": "basic_fail_view",
+                      "text": {
+                        "text": "View Job",
+                        "type": "plain_text"
+                      },
+                      "type": "button",
+                      "url": "${CIRCLE_BUILD_URL}"
+                    }
+                  ],
+                  "type": "actions"
+                }
+              ]
+            }
+
 
 jobs:
   ## ------------------ Run Unit Tests ------------------
@@ -45,7 +110,8 @@ jobs:
     steps:
       # Make sure that the build doesn't accidentally pull images from Docker Hub.
       # Pulls from Docker Hub may fail due to rate-limiting after Nov 1 2020.
-      - disable_docker_hub
+      - sm-build/disable_docker_hub_access
+      - sm-build/configure_gcr_docker_access
       - attach_workspace:
           at: /tmp/workspace
 
@@ -75,6 +141,8 @@ jobs:
           name: Save Unit Test Results
           path: htmlcov
 
+      - send_slack_msg_on_fail    
+
   ## ------------------ Check Black python code formate ------------------
   black-python-code-formatter:
     parameters:
@@ -101,6 +169,54 @@ jobs:
               exit 1
             fi
 
+  ## ------------------ Check python code against known security issues ------------------
+  cve-repo-scan:
+    parameters:
+      with_merge: # Part of the PR-merge workflow
+        type: boolean
+        default: false
+    docker:
+      # https://circleci.com/developer/images/image/cimg/python
+      - image: cimg/base:current
+    steps:
+      - when:
+          condition: << parameters.with_merge >>
+          steps:
+            - disable_build_if_not_pr
+      - checkout
+      - when:
+          condition: << parameters.with_merge >>
+          steps:
+            - merge-pr-branch
+      - sm-build/trivy_scan_filesystem:
+          report_json_name: fs_report.json
+          report_html_name: fs_report.html
+      - store_artifacts:
+          name: Save Filesystem Report
+          path: fs_report.html
+
+  ## ------------- Check python code and container image against known security issues -----------
+  cve-image-scan:
+    docker:
+      # https://circleci.com/developer/images/image/cimg/python
+      - image: cimg/base:current
+    steps:
+      # Checkout the code from the master branch
+      - checkout
+      # Perform the security scan on the filesystem
+      - sm-build/trivy_scan_filesystem:
+          report_json_name: fs_report_master.json
+          report_html_name: fs_report_master.html
+      # Store the artifacts for the master branch scan
+      - store_artifacts:
+          name: Save Master Branch Filesystem Report
+          path: fs_report_master.html
+      # The following will mark the build as failed if there are issues in the report.
+      - sm-build/trivy_notify_on_issues:
+          slack_channel: monitoring-sdk
+          slack_branch_pattern: master
+          report_list: Repo|fs_report_master.json|fs_report_master.html
+
   ## ------------------ Mypy for Static Type Checking python code ------------------
   mypy-type-checks:
     parameters:

docs/cve_audit.md

@@ -0,0 +1,40 @@
+# CVE Audit Task
+
+There are 2 CVE audit tasks for the SDK build.
+
+1. **Lightweight Repository Audit Report:**
+   - This report provides information based on the repository build.
+   - It is triggered on each commit and pull request (PR).
+   - Focuses on essential information related to the repository's build status.
+
+2. **Nightly Master Branch Audit Report:**
+   - This report provides daily build-based information specifically for the master branch.
+   - It runs nightly on the latest master branch.
+   - Excludes container-based audit reporting as there is no associated container with the SDK.
+
+The CVE audit report is conducted using a tool called [Trivy](https://trivy.dev/). The primary
+configuration for the tool is [trivy.yaml](../trivy.yaml). Any unique global parameters for a
+specific run are passed at execution.
+
+The reporting tool currently only reports on CRITICAL & HIGH severity issues.
+Vulnerability issues can further be filtered by status:
+
+1. affected
+2. fixed
+3. will_not_fix
+4. fix_deferred
+5. end_of_life
+
+Or by ID via a `.trivyignore` or `.trivyignore.yaml` file. Instructions for this can be found in the
+[Trivy documentation](https://aquasecurity.github.io/trivy/v0.49/docs/configuration/filtering/).
+
+## Trivy Template Information
+
+The following links provide information on the Go-Template Engine within Trivy.
+
+| Description                                 | Link                                                                   |
+|---------------------------------------------|------------------------------------------------------------------------|
+| Usage of Go-Templates in Trivy Command line | https://aquasecurity.github.io/trivy/v0.17.2/examples/report/#template |
+| Go-Template Function list from Sprig        | https://masterminds.github.io/sprig/ |
+| Trivy Code: wiring up function map          | https://github.com/aquasecurity/trivy/blob/v0.49.1/pkg/report/template.go#L31 |
+| Trivy Code: passing data state to template  | https://github.com/aquasecurity/trivy/blob/v0.49.1/pkg/report/template.go#L79 |

trivy.yaml

@@ -0,0 +1,9 @@
+# Default settings to run trivy with
+# https://aquasecurity.github.io/trivy/v0.33/docs/references/customization/config-file/
+severity:
+  - CRITICAL
+  - HIGH
+
+scan:
+  file-patterns:
+    - 'pip:requirements.*.txt'