create image for dss (Yelp#245)

use correct docker setting (Yelp#246)

Use escape sequence to replace clear (Yelp#247)

Build docker images for DSS client (Yelp#248)

Build on tag push (Yelp#249)

Publish to Artifactory (Yelp#250)

.dockerignore

@@ -0,0 +1,19 @@
+*.egg-info
+*.py[co]
+*.sw[op]
+
+/.git
+/.coverage
+/.github
+/.pytest_cache
+/.python-version
+/.tox
+/.vscode
+/tmp
+/user-config
+/venv
+/venv*
+
+.*ignore
+!.gitignore
+!.dockerignore

.gitignore

@@ -10,5 +10,6 @@
 
 .*ignore
 !.gitignore
+!.dockerignore
 .python-version
 .vscode

.secrets.baseline

@@ -3,7 +3,7 @@
     "files": "test_data/.*|tests/.*|^.secrets.baseline$",
     "lines": null
   },
-  "generated_at": "2020-01-13T18:12:22Z",
+  "generated_at": "2020-02-17T15:57:42Z",
   "plugins_used": [
     {
       "name": "AWSKeyDetector"
@@ -204,15 +204,6 @@
         "type": "Hex High Entropy String"
       }
     ],
-    "test_data/short_files/first_line.py": [
-      {
-        "hashed_secret": "0de9a11b3f37872868ca49ecd726c955e25b6e21",
-        "is_secret": false,
-        "is_verified": false,
-        "line_number": 1,
-        "type": "Hex High Entropy String"
-      }
-    ],
     "test_data/short_files/last_line.ini": [
       {
         "hashed_secret": "0de9a11b3f37872868ca49ecd726c955e25b6e21",

.travis.yml

@@ -3,27 +3,39 @@ dist: trusty
 env:
     global:
         - TOXENV=py27
-        - DOCKER_USER=travisrw@us.ibm.com
-        - DOCKER_IMAGE_HOST=txo-whitewater-public-docker-local.artifactory.swg-devops.com
-        - DOCKER_IMAGE=txo-whitewater-public-docker-local.artifactory.swg-devops.com/whitewater-detect-secrets
+
+        - DOCKER_REGISTRY_ICR=us.icr.io
+        - DOCKER_USER_ICR=iamapikey
+        - DOCKER_DOMAIN_ICR=$DOCKER_REGISTRY_ICR/git-defenders
+
+        - DOCKER_REGISTRY_ART=txo-toolbox-team-docker-local.artifactory.swg-devops.com
+        - DOCKER_USER_ART=dssadmin@us.ibm.com
+        - DOCKER_DOMAIN_ART=$DOCKER_REGISTRY_ART/git-defenders
+
+        - DOCKER_REMOTE_DOMAINS="$DOCKER_DOMAIN_ICR $DOCKER_DOMAIN_ART"
+        - DOCKER_IMAGES="dsl detect-secrets detect-secrets-hook"
+        - DOCKER_DOMAIN_LOCAL=git-defenders
         - DOCKER_IMAGE_TAG="$TRAVIS_BUILD_NUMBER-id-$TRAVIS_BUILD_ID-time-$(date +%s)"
-        - DOCKER_IMAGE_TAG_MVP="$TRAVIS_BRANCH-$TRAVIS_BUILD_NUMBER-id-$TRAVIS_BUILD_ID-time-$(date +%s)"
+        - DOCKER_IMAGE_TAG_DSS="$TRAVIS_BRANCH-$TRAVIS_BUILD_NUMBER-id-$TRAVIS_BUILD_ID-time-$(date +%s)"
 deploy:
 - provider: script
-  script: docker login $DOCKER_IMAGE_HOST --username $DOCKER_USER --password $DOCKER_PASS
-    && docker push $DOCKER_IMAGE:$DOCKER_IMAGE_TAG
-    && docker tag $DOCKER_IMAGE:$DOCKER_IMAGE_TAG $DOCKER_IMAGE:latest
-    && docker push $DOCKER_IMAGE:latest
+  # dpl would concatenate all lines into single string
+  script: >-
+        set -e;
+        echo ${IBM_CLOUD_API_KEY} | docker login -u $DOCKER_USER_ICR --password-stdin $DOCKER_REGISTRY_ICR;
+        echo ${ART_API_KEY} | docker login -u $DOCKER_USER_ART --password-stdin $DOCKER_REGISTRY_ART;
+        ./tag-and-publish.sh "$DOCKER_IMAGES" "$DOCKER_DOMAIN_LOCAL" "$DOCKER_REMOTE_DOMAINS" "$DOCKER_IMAGE_TAG latest";
   on:
-    branch: master
+      branch: master
 - provider: script
-  script: docker login $DOCKER_IMAGE_HOST --username $DOCKER_USER --password $DOCKER_PASS
-    && docker tag $DOCKER_IMAGE:$DOCKER_IMAGE_TAG $DOCKER_IMAGE:$DOCKER_IMAGE_TAG_MVP
-    && docker push $DOCKER_IMAGE:$DOCKER_IMAGE_TAG_MVP
-    && docker tag $DOCKER_IMAGE:$DOCKER_IMAGE_TAG_MVP $DOCKER_IMAGE:mvp-latest
-    && docker push $DOCKER_IMAGE:mvp-latest
+  # dpl would concatenate all lines into single string
+  script: >-
+        set -e;
+        echo ${IBM_CLOUD_API_KEY} | docker login -u $DOCKER_USER_ICR --password-stdin $DOCKER_REGISTRY_ICR;
+        echo ${ART_API_KEY} | docker login -u $DOCKER_USER_ART --password-stdin $DOCKER_REGISTRY_ART;
+        ./tag-and-publish.sh "$DOCKER_IMAGES" "$DOCKER_DOMAIN_LOCAL" "$DOCKER_REMOTE_DOMAINS" "$DOCKER_IMAGE_TAG_DSS dss-latest";
   on:
-    branch: mvp
+      branch: dss
 matrix:
     include:
         - env: TOXENV=py27
@@ -39,7 +51,18 @@ matrix:
         #   python: pypy
 install:
     - pip install tox
-script: make test && docker build -t $DOCKER_IMAGE:$DOCKER_IMAGE_TAG --no-cache .
+# normal script supports multiple line syntax
+script: |
+      make test
+      ./build-dockerfiles.sh
+
+      # Publish image and labled with tag when a git tag is pushed
+      if [ -n "$TRAVIS_TAG" ]; then
+        echo ${IBM_CLOUD_API_KEY} | docker login -u $DOCKER_USER_ICR --password-stdin $DOCKER_REGISTRY_ICR
+        echo ${ART_API_KEY} | docker login -u $DOCKER_USER_ART --password-stdin $DOCKER_REGISTRY_ART
+
+        ./tag-and-publish.sh "$DOCKER_IMAGES" "$DOCKER_DOMAIN_LOCAL" "$DOCKER_REMOTE_DOMAINS" "$TRAVIS_TAG"
+      fi
 cache:
     directories:
         - $HOME/.cache/pre-commit

Dockerfiles/00.python.Dockerfile

@@ -0,0 +1,7 @@
+FROM python:3
+LABEL maintainer="squad:git-defenders" url="https://github.ibm.com/whitewater/whitewater-detect-secrets"
+
+RUN apt-get -y remove --purge mysql*
+# Remediate CVE-2019-18218
+RUN apt-get update && apt-get install file -y
+RUN pip install --upgrade pip

Dockerfiles/01.cli.Dockerfile

@@ -0,0 +1,13 @@
+FROM git-defenders/python
+
+# Auto adjust line ending. Support running scan on Windows platform
+RUN git config --global core.autocrlf true
+
+COPY README.md /code/
+COPY setup.py /code/
+COPY setup.cfg /code/
+COPY detect_secrets /code/detect_secrets
+
+RUN pip install /code
+
+WORKDIR /code

Dockerfiles/01.dsl.Dockerfile

@@ -0,0 +1,9 @@
+FROM git-defenders/python
+
+RUN apt-get update && apt-get install -y jq
+RUN mkdir -p /code
+COPY . /usr/src/app
+WORKDIR /usr/src/app
+RUN pip install /usr/src/app
+WORKDIR /code
+ENTRYPOINT [ "/usr/src/app/run-scan.sh" ]

Dockerfiles/02.detect-secrets-hook.Dockerfile

@@ -0,0 +1,4 @@
+FROM git-defenders/cli
+
+RUN git config --global core.safecrlf false
+ENTRYPOINT [ "detect-secrets-hook" ]

Dockerfiles/02.detect-secrets.Dockerfile

@@ -0,0 +1,3 @@
+FROM git-defenders/cli
+ENTRYPOINT [ "detect-secrets" ]
+CMD [ "scan", "/code" ]

build-dockerfiles.sh

@@ -0,0 +1,20 @@
+#!/bin/bash -ex
+
+CUR_DIR=$(dirname $0)
+pushd "${CUR_DIR}"
+
+IMAGE_DOMAIN=git-defenders
+
+# build images
+for dockerfile in Dockerfiles/*.Dockerfile
+do
+    image_name=$(echo -e $(basename ${dockerfile}) | cut -d\. -f2)
+    docker build -f "${dockerfile}" -t $IMAGE_DOMAIN/$image_name .
+done
+
+# test images
+docker run -it --entrypoint detect-secrets $IMAGE_DOMAIN/dsl --version
+docker run -it $IMAGE_DOMAIN/detect-secrets --version
+docker run -it $IMAGE_DOMAIN/detect-secrets-hook --version
+
+popd

detect_secrets/core/audit.py

@@ -2,7 +2,6 @@
 import io
 import json
 import os
-import subprocess
 import sys
 from builtins import input
 from collections import defaultdict
@@ -477,7 +476,11 @@ def _comparison_generator(old_list, new_list, compare_fn):
 
 
 def _clear_screen():  # pragma: no cover
-    subprocess.call(['clear'])
+    ttyname = os.ttyname(sys.stdout.fileno())
+    with open(ttyname, 'wb') as tty:
+        # Use escape sequence instead of "clear" command to avoid incorrect
+        # starting line calculation in docker env
+        tty.write(b'\033c')
 
 
 def _print_context(  # pragma: no cover

tag-and-publish.sh

@@ -0,0 +1,25 @@
+#!/bin/bash -e
+
+script_name=$(basename $0)
+
+if [ $# -lt 4 ]; then
+    echo "Usage: $script_name image_names local_domain_name remote_domain_names tags"
+    exit 1
+fi
+
+DOCKER_IMAGES=$1
+DOCKER_DOMAIN_LOCAL=$2
+DOCKER_DOMAIN_REMOTES=$3
+TAG_NAMES=$4
+
+for image_name in $DOCKER_IMAGES
+do
+    for domain_name in $DOCKER_DOMAIN_REMOTES
+    do
+        for tag in $TAG_NAMES
+        do
+            docker tag $DOCKER_DOMAIN_LOCAL/$image_name $domain_name/$image_name:$tag
+            docker push $domain_name/$image_name:$tag
+        done
+    done
+done

update.md

@@ -1,5 +1,5 @@
-Versioning and update
----------------------
+# Versioning and update
+
 <!-- START doctoc generated TOC please keep comment here to allow auto update -->
 <!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
 **Table of Contents**  *generated with [DocToc](https://github.com/thlorenz/doctoc)*
@@ -8,7 +8,6 @@ Versioning and update
 - [How do we version this repo](#how-do-we-version-this-repo)
 - [How do we make release](#how-do-we-make-release)
 
-
 ## How do we keep update to date with upstream
 
 > Rule of thumb: We keep all IBM related commits on top of the upstream code using rebase and/or cherry-pick.
@@ -64,7 +63,8 @@ Format: `<upstream-version>-ibm.<minor>.<fix>`, for example `0.12.0-ibm.3`
 
 Release should be made when we bump to a new version.
 
-1. Use [`hub` tool](https://github.com/github/hub) to create new release. You can view the released version at https://github.ibm.com/Whitewater/whitewater-detect-secrets/releases
+1. Use [`hub` tool](https://github.com/github/hub) to create new release. You can view the released version at <https://github.ibm.com/Whitewater/whitewater-detect-secrets/releases>.
+This process also creates a tag, which triggers Traivs tag build to generate a new version of the docker image labeled with the newly created tag.
 
 ```sh
 # <branch-name>: the latest commit from the branch would be used to create tag.