From 4f14cc6213bfa811f85283124bced8ddb87b4feb Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 30 Oct 2020 10:34:57 -0400 Subject: [PATCH 01/14] Add script to generate cross-platform Dockerfile. --- buildx-dockerfile.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100755 buildx-dockerfile.sh diff --git a/buildx-dockerfile.sh b/buildx-dockerfile.sh new file mode 100755 index 0000000..46710e9 --- /dev/null +++ b/buildx-dockerfile.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +# Create a Dockerfile suitable for a multi-platform build using buildx +# See: https://docs.docker.com/buildx/working-with-buildx/ + +set -o nounset +set -o errexit +set -o pipefail + +DOCKERFILE=Dockerfile +DOCKERFILEX=Dockerfile-x + +# We don't want this expression to expand. +# shellcheck disable=SC2016 +sed 's/^FROM /FROM --platform=$TARGETPLATFORM /g' < $DOCKERFILE > $DOCKERFILEX From ba039e872f453b14731863f6f6fb411ffcd5f33f Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 30 Oct 2020 10:40:21 -0400 Subject: [PATCH 02/14] Add git ignore for generated Dockerfile-x. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 9f81d3f..bceb4ee 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ __pycache__ .mypy_cache .pytest_cache .python-version +Dockerfile-x From 3d0720bae1c0928402b11cf4174291c7929e8e03 Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 30 Oct 2020 10:44:59 -0400 Subject: [PATCH 03/14] Replace previous Docker workflows with a moderinzed, complete-workflow. These changes are based on the "complete workflow" from https://github.com/docker/build-push-action Additionally it adds support for the new "workflow_dispatch" event type. --- .github/workflows/build.yml | 229 ++++++++++++++++++++++++++++++---- .github/workflows/release.yml | 52 -------- push_readme.sh | 4 +- 3 files changed, 207 insertions(+), 78 deletions(-) delete mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bf84602..a489565 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -3,17 +3,37 @@ name: build on: push: + branches: + - '**' + tags: + - 'v*.*.*' pull_request: + schedule: + - cron: '0 10 * * *' # everyday at 10am repository_dispatch: types: [apb] + workflow_dispatch: + inputs: + remote-shell: + description: "Debug with remote shell" + required: true + default: false + image-tag: + description: "Tag to apply to pushed images" + required: true + default: dispatch env: + BUILDX_CACHE_DIR: ~/.cache/buildx IMAGE_NAME: cisagov/example PIP_CACHE_DIR: ~/.cache/pip + PLATFORMS: "linux/amd64,linux/arm/v6,linux/arm/v7,\ + linux/arm64,linux/ppc64le,linux/s390x" PRE_COMMIT_CACHE_DIR: ~/.cache/pre-commit jobs: lint: + name: "Lint sources" runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -46,32 +66,126 @@ jobs: run: pre-commit install-hooks - name: Run pre-commit on all files run: pre-commit run --all-files - build: + prepare: + name: "Prepare build variables" runs-on: ubuntu-latest + outputs: + created: ${{ steps.prep.outputs.created }} + repometa: ${{ steps.repo.outputs.result }} + source_version: ${{ steps.prep.outputs.source_version }} + tags: ${{ steps.prep.outputs.tags }} steps: - uses: actions/checkout@v2 - - name: Determine image version - run: echo IMAGE_VERSION=$(./bump_version.sh show) >> $GITHUB_ENV - - name: Build docker image - run: | - version=${{ env.IMAGE_VERSION }} - docker build \ - --tag "$IMAGE_NAME" \ - --build-arg GIT_COMMIT=$(git log -1 --format=%H) \ - --build-arg GIT_REMOTE=$(git remote get-url origin) \ - --build-arg VERSION=${{ env.IMAGE_VERSION }} \ - . - - name: Save docker image artifact + - name: Gather repository metadata + id: repo + uses: actions/github-script@v3 + with: + script: | + const repo = await github.repos.get(context.repo) + return repo.data + - name: Calculate output values + id: prep run: | - mkdir dist - version=${{ env.IMAGE_VERSION }} - docker save $IMAGE_NAME:latest | gzip > dist/image.tar.gz + VERSION=noop + SEMVER="^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-((0|[1-9][0-9]*|[0-9]*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9][0-9]*|[0-9]*[a-zA-Z-][0-9a-zA-Z-]*))*))?(\+([0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*))?$" + if [ "${{ github.event_name }}" = "schedule" ]; then + VERSION=nightly + elif [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + VERSION=${{ github.event.inputs.image-tag }} + elif [[ $GITHUB_REF == refs/tags/* ]]; then + VERSION=${GITHUB_REF#refs/tags/} + elif [[ $GITHUB_REF == refs/heads/* ]]; then + VERSION=$(echo ${GITHUB_REF#refs/heads/} | sed -r 's#/+#-#g') + if [ "${{ github.event.repository.default_branch }}" = "$VERSION" ]; + then + VERSION=edge + fi + elif [[ $GITHUB_REF == refs/pull/* ]]; then + VERSION=pr-${{ github.event.number }} + fi + if [[ $VERSION =~ $SEMVER ]]; then + VERSION_NO_V=${VERSION#v} + MAJOR="${BASH_REMATCH[1]}" + MINOR="${BASH_REMATCH[2]}" + PATCH="${BASH_REMATCH[3]}" + TAGS="${IMAGE_NAME}:${VERSION_NO_V//+/_},${IMAGE_NAME}:${MAJOR}.${MINOR}.${PATCH},${IMAGE_NAME}:${MAJOR}.${MINOR},${IMAGE_NAME}:${MAJOR},${IMAGE_NAME}:latest" + else + TAGS="${IMAGE_NAME}:${VERSION}" + fi + if [ "${{ github.event_name }}" = "push" ]; then + TAGS="${TAGS},${IMAGE_NAME}:sha-${GITHUB_SHA::8}" + fi + echo ::set-output name=created::$(date -u +'%Y-%m-%dT%H:%M:%SZ') + echo ::set-output name=source_version::$(./bump_version.sh show) + echo ::set-output name=tags::${TAGS} + echo tags=${TAGS} + - name: Setup debug session remote shell + uses: mxschmitt/action-tmate@v3 + if: github.event.inputs.remote-shell == 'true' + build: + name: "Build test image" + runs-on: ubuntu-latest + needs: [prepare] + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Cache Docker layers + uses: actions/cache@v2 + with: + path: ${{ env.BUILDX_CACHE_DIR }} + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + - name: Create dist directory + run: mkdir -p dist + - name: Build image + id: docker_build + uses: docker/build-push-action@v2 + with: + build-args: | + VERSION=${{ needs.prepare.outputs.source_version }} + cache-from: type=local,src=${{ env.BUILDX_CACHE_DIR }} + cache-to: type=local,dest=${{ env.BUILDX_CACHE_DIR }} + context: . + file: ./Dockerfile + outputs: type=docker,dest=dist/image.tar + tags: ${{ env.IMAGE_NAME }}:latest # not to be pushed + labels: "\ + org.opencontainers.image.created=${{ + needs.prepare.outputs.created }} + + org.opencontainers.image.description=${{ + fromJson(needs.prepare.outputs.repometa).description }} + + org.opencontainers.image.licenses=${{ + fromJson(needs.prepare.outputs.repometa).license.spdx_id }} + + org.opencontainers.image.revision=${{ github.sha }} + + org.opencontainers.image.source=${{ + fromJson(needs.prepare.outputs.repometa).clone_url }} + + org.opencontainers.image.title=${{ + fromJson(needs.prepare.outputs.repometa).name }} + + org.opencontainers.image.url=${{ + fromJson(needs.prepare.outputs.repometa).html_url }} + + org.opencontainers.image.version=${{ + needs.prepare.outputs.source_version }}" + - name: Compress image + run: gzip dist/image.tar - name: Upload artifacts uses: actions/upload-artifact@v2 with: name: dist path: dist test: + name: "Test image" runs-on: ubuntu-latest needs: [build] steps: @@ -79,21 +193,21 @@ jobs: - uses: actions/setup-python@v2 with: python-version: 3.9 - - name: Cache testing environments - uses: actions/cache@v2 + - name: Cache pip test requirements + uses: actions/cache@v1 with: path: ${{ env.PIP_CACHE_DIR }} - key: "test-${{ runner.os }}-\ - ${{ hashFiles('**/requirements-test.txt') }}-\ - ${{ hashFiles('**/requirements.txt') }}" + key: "${{ runner.os }}-pip-test-\ + ${{ hashFiles('**/requirements-test.txt') }}" restore-keys: | - test-${{ runner.os }}- + ${{ runner.os }}-pip-test- + ${{ runner.os }}-pip- - name: Install dependencies run: | python -m pip install --upgrade pip pip install --upgrade --requirement requirements-test.txt - name: Download docker image artifact - uses: actions/download-artifact@v2.0.5 + uses: actions/download-artifact@v2 with: name: dist path: dist @@ -102,4 +216,71 @@ jobs: - name: Run tests env: RELEASE_TAG: ${{ github.event.release.tag_name }} - run: pytest + run: pytest --runslow + build-push-all: + name: "Build and push all platforms" + runs-on: ubuntu-latest + needs: [prepare, test] + if: github.event_name != 'pull_request' + steps: + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Checkout + uses: actions/checkout@v2 + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Cache Docker layers + uses: actions/cache@v2 + with: + path: ${{ env.BUILDX_CACHE_DIR }} + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + - name: Create cross-platform support Dockerfile-x + run: ./buildx-dockerfile.sh + - name: Build and push platform images to Docker Hub + id: docker_build + uses: docker/build-push-action@v2 + with: + build-args: | + VERSION=${{ needs.prepare.outputs.source_version }} + cache-from: type=local,src=${{ env.BUILDX_CACHE_DIR }} + cache-to: type=local,dest=${{ env.BUILDX_CACHE_DIR }} + context: . + file: ./Dockerfile-x + platforms: ${{ env.PLATFORMS }} + push: true + tags: ${{ needs.prepare.outputs.tags }} + labels: "\ + org.opencontainers.image.created=${{ + needs.prepare.outputs.created }} + + org.opencontainers.image.description=${{ + fromJson(needs.prepare.outputs.repometa).description }} + + org.opencontainers.image.licenses=${{ + fromJson(needs.prepare.outputs.repometa).license.spdx_id }} + + org.opencontainers.image.revision=${{ github.sha }} + + org.opencontainers.image.source=${{ + fromJson(needs.prepare.outputs.repometa).clone_url }} + + org.opencontainers.image.title=${{ + fromJson(needs.prepare.outputs.repometa).name }} + + org.opencontainers.image.url=${{ + fromJson(needs.prepare.outputs.repometa).html_url }} + + org.opencontainers.image.version=${{ + needs.prepare.outputs.source_version }}" + - name: Publish README.md to Docker Hub + env: + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + run: ./push_readme.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index dca4c36..0000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,52 +0,0 @@ ---- -name: release - -on: - release: - types: [prereleased, released] - -env: - IMAGE_NAME: cisagov/example - DOCKER_PW: ${{ secrets.DOCKER_PW }} - DOCKER_USER: ${{ secrets.DOCKER_USER }} - -jobs: - release: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: 3.9 - - name: Determine image version - run: echo IMAGE_VERSION=$(./bump_version.sh show) >> $GITHUB_ENV - - name: Build Docker image - run: | - version=${{ env.IMAGE_VERSION }} - docker build \ - --tag "$IMAGE_NAME" \ - --build-arg GIT_COMMIT=$(git log -1 --format=%H) \ - --build-arg GIT_REMOTE=$(git remote get-url origin) \ - --build-arg VERSION=${{ env.IMAGE_VERSION }} \ - . - - name: Tag Docker image - run: | - IFS='.' read -r -a version_array \ - <<< "${{ env.IMAGE_VERSION }}" - docker login --username "$DOCKER_USER" --password "$DOCKER_PW" - docker tag "$IMAGE_NAME" "${IMAGE_NAME}:latest" - docker tag "$IMAGE_NAME" \ - "${IMAGE_NAME}:${{ env.IMAGE_VERSION }}" - docker tag "$IMAGE_NAME" \ - "${IMAGE_NAME}:${version_array[0]}.${version_array[1]}" - docker tag "$IMAGE_NAME" "${IMAGE_NAME}:${version_array[0]}" - - name: Publish image to Docker Hub - run: | - IFS='.' read -r -a version_array \ - <<< "${{ env.IMAGE_VERSION }}" - docker push "${IMAGE_NAME}:latest" - docker push "${IMAGE_NAME}:${{ env.IMAGE_VERSION }}" - docker push "${IMAGE_NAME}:${version_array[0]}.${version_array[1]}" - docker push "${IMAGE_NAME}:${version_array[0]}" - - name: Publish README.md to Docker Hub - run: ./push_readme.sh diff --git a/push_readme.sh b/push_readme.sh index 0b6d07a..4532701 100755 --- a/push_readme.sh +++ b/push_readme.sh @@ -3,7 +3,7 @@ # Push the README.md file to the docker hub repository # Requires the following environment variables to be set: -# DOCKER_PW, DOCKER_USER, IMAGE_NAME +# DOCKER_PASSWORD, DOCKER_USERNAME, IMAGE_NAME set -o nounset set -o errexit @@ -11,7 +11,7 @@ set -o pipefail token=$(curl -s -X POST \ -H "Content-Type: application/json" \ - -d '{"username": "'"$DOCKER_USER"'", "password": "'"$DOCKER_PW"'"}' \ + -d '{"username": "'"$DOCKER_USERNAME"'", "password": "'"$DOCKER_PASSWORD"'"}' \ https://hub.docker.com/v2/users/login/ | jq -r .token) code=$(jq -n --arg msg "$( Date: Fri, 30 Oct 2020 10:55:24 -0400 Subject: [PATCH 04/14] Add CodeQL workflow. Needed to check tests written in Python. --- .github/workflows/codeql-analysis.yml | 75 +++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 .github/workflows/codeql-analysis.yml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 0000000..561e8c1 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,75 @@ +--- + +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +name: "CodeQL" + +on: + push: + pull_request: + # The branches below must be a subset of the branches above + branches: [develop] + schedule: + - cron: '0 21 * * 6' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + # Override automatic language detection by changing the below list + # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', + # 'python'] + language: ['python'] + # Learn more... + # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + with: + # We must fetch at least the immediate parents so that if this is + # a pull request then we can checkout the head. + fetch-depth: 2 + + # If this run was triggered by a pull request event, then checkout + # the head of the pull request instead of the merge commit. + - run: git checkout HEAD^2 + if: ${{ github.event_name == 'pull_request' }} + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a + # config file. By default, queries listed here will override any + # specified in a config file. Prefix the list here with "+" to use + # these queries and those in the config file. queries: + # ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or + # Java). If this step fails, then you should remove it and run the build + # manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following + # three lines and modify them (or add more) to build your code if your + # project uses a compiled language + + # - run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 From 7530392ec4ab87301360f91577f80dbc064da4bd Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 30 Oct 2020 14:54:59 -0400 Subject: [PATCH 05/14] Update README publisher to use new Docker URL. --- push_readme.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/push_readme.sh b/push_readme.sh index 4532701..2e55623 100755 --- a/push_readme.sh +++ b/push_readme.sh @@ -9,15 +9,17 @@ set -o nounset set -o errexit set -o pipefail +echo "Logging in and requesting JWT..." token=$(curl -s -X POST \ -H "Content-Type: application/json" \ -d '{"username": "'"$DOCKER_USERNAME"'", "password": "'"$DOCKER_PASSWORD"'"}' \ https://hub.docker.com/v2/users/login/ | jq -r .token) +echo "Pushing README file..." code=$(jq -n --arg msg "$( Date: Fri, 30 Oct 2020 15:06:29 -0400 Subject: [PATCH 06/14] Expand command option names to long form. --- push_readme.sh | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/push_readme.sh b/push_readme.sh index 2e55623..13155cd 100755 --- a/push_readme.sh +++ b/push_readme.sh @@ -10,19 +10,20 @@ set -o errexit set -o pipefail echo "Logging in and requesting JWT..." -token=$(curl -s -X POST \ - -H "Content-Type: application/json" \ - -d '{"username": "'"$DOCKER_USERNAME"'", "password": "'"$DOCKER_PASSWORD"'"}' \ - https://hub.docker.com/v2/users/login/ | jq -r .token) +token=$(curl --silent --request POST \ + --header "Content-Type: application/json" \ + --data \ + '{"username": "'"$DOCKER_USERNAME"'", "password": "'"$DOCKER_PASSWORD"'"}' \ + https://hub.docker.com/v2/users/login/ | jq --raw-output .token) echo "Pushing README file..." -code=$(jq -n --arg msg "$( Date: Fri, 30 Oct 2020 18:44:23 -0400 Subject: [PATCH 07/14] Update Dockerfile to use opencontainers metadata and build-time labels. --- Dockerfile | 11 ++--------- tests/container_test.py | 2 +- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index b4f4e61..c9184c8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,18 +1,11 @@ -ARG GIT_COMMIT=unspecified -ARG GIT_REMOTE=unspecified ARG VERSION=unspecified FROM python:3.9-alpine -ARG GIT_COMMIT -ARG GIT_REMOTE ARG VERSION -LABEL git_commit=${GIT_COMMIT} -LABEL git_remote=${GIT_REMOTE} -LABEL maintainer="mark.feldhousen@trio.dhs.gov" -LABEL vendor="Cyber and Infrastructure Security Agency" -LABEL version=${VERSION} +LABEL org.opencontainers.image.authors="mark.feldhousen@cisa.dhs.gov" +LABEL org.opencontainers.image.vendor="Cyber and Infrastructure Security Agency" ARG CISA_UID=421 ENV CISA_HOME="/home/cisa" diff --git a/tests/container_test.py b/tests/container_test.py index 90ea1a3..6153028 100644 --- a/tests/container_test.py +++ b/tests/container_test.py @@ -89,5 +89,5 @@ def test_container_version_label_matches(version_container): exec(f.read(), pkg_vars) # nosec project_version = pkg_vars["__version__"] assert ( - version_container.labels["version"] == project_version + version_container.labels["org.opencontainers.image.version"] == project_version ), "Dockerfile version label does not match project version" From e01481ba5b532010969242ff5d51d386c1ad56b7 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 2 Nov 2020 10:44:02 -0500 Subject: [PATCH 08/14] Prevent image pushes when linter checks fail. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a489565..7b79adb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -220,7 +220,7 @@ jobs: build-push-all: name: "Build and push all platforms" runs-on: ubuntu-latest - needs: [prepare, test] + needs: [lint, prepare, test] if: github.event_name != 'pull_request' steps: - name: Login to DockerHub From 297e52758195303ed6aa193f9b40acec7a56a98a Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 2 Nov 2020 12:13:17 -0500 Subject: [PATCH 09/14] Modify cache keys to follow team-established naming conventions. Co-authored-by: Shane Frasier --- .github/workflows/build.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7b79adb..7bb0a2f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -137,9 +137,9 @@ jobs: uses: actions/cache@v2 with: path: ${{ env.BUILDX_CACHE_DIR }} - key: ${{ runner.os }}-buildx-${{ github.sha }} + key: buildx-${{ runner.os }}-${{ github.sha }} restore-keys: | - ${{ runner.os }}-buildx- + buildx-${{ runner.os }}- - name: Create dist directory run: mkdir -p dist - name: Build image @@ -194,14 +194,14 @@ jobs: with: python-version: 3.9 - name: Cache pip test requirements - uses: actions/cache@v1 + uses: actions/cache@v2 with: path: ${{ env.PIP_CACHE_DIR }} - key: "${{ runner.os }}-pip-test-\ - ${{ hashFiles('**/requirements-test.txt') }}" + key: "test-${{ runner.os }}-\ + ${{ hashFiles('**/requirements-test.txt') }}-\ + ${{ hashFiles('**/requirements.txt') }}" restore-keys: | - ${{ runner.os }}-pip-test- - ${{ runner.os }}-pip- + test-${{ runner.os }}- - name: Install dependencies run: | python -m pip install --upgrade pip @@ -238,9 +238,9 @@ jobs: uses: actions/cache@v2 with: path: ${{ env.BUILDX_CACHE_DIR }} - key: ${{ runner.os }}-buildx-${{ github.sha }} + key: buildx-${{ runner.os }}-${{ github.sha }} restore-keys: | - ${{ runner.os }}-buildx- + buildx-${{ runner.os }}- - name: Create cross-platform support Dockerfile-x run: ./buildx-dockerfile.sh - name: Build and push platform images to Docker Hub From f1e097a8641df648bd31673bfe911aae8540a05e Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 2 Nov 2020 12:29:06 -0500 Subject: [PATCH 10/14] Add comments pointing to the opencontainers image-spec documentation. --- .github/workflows/build.yml | 2 ++ Dockerfile | 3 +++ 2 files changed, 5 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7bb0a2f..e80f5d4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -256,6 +256,8 @@ jobs: platforms: ${{ env.PLATFORMS }} push: true tags: ${{ needs.prepare.outputs.tags }} + # For a list of pre-defined annotation keys and value types see: + # https://github.com/opencontainers/image-spec/blob/master/annotations.md labels: "\ org.opencontainers.image.created=${{ needs.prepare.outputs.created }} diff --git a/Dockerfile b/Dockerfile index c9184c8..8819053 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,6 +4,9 @@ FROM python:3.9-alpine ARG VERSION +# For a list of pre-defined annotation keys and value types see: +# https://github.com/opencontainers/image-spec/blob/master/annotations.md +# Note: Additional labels are added by the build workflow. LABEL org.opencontainers.image.authors="mark.feldhousen@cisa.dhs.gov" LABEL org.opencontainers.image.vendor="Cyber and Infrastructure Security Agency" From a88d550bd9ea10b572b58945981a76a804c00263 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 2 Nov 2020 16:36:43 -0500 Subject: [PATCH 11/14] Add comments to help document and break up a monolitic file. --- .github/workflows/build.yml | 54 +++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e80f5d4..b5dd214 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -11,6 +11,7 @@ on: schedule: - cron: '0 10 * * *' # everyday at 10am repository_dispatch: + # Respond to rebuild requests. See: https://github.com/cisagov/action-apb/ types: [apb] workflow_dispatch: inputs: @@ -33,6 +34,8 @@ env: jobs: lint: + # Checks out the source and runs pre-commit hooks. Detects coding errors + # and style deviations. name: "Lint sources" runs-on: ubuntu-latest steps: @@ -66,7 +69,45 @@ jobs: run: pre-commit install-hooks - name: Run pre-commit on all files run: pre-commit run --all-files + prepare: + # Calculates and publishes outputs that are used by other jobs. + # + # Outputs: + # created: + # The current date-time in RFC3339 format. + # repometa: + # The json metadata describing this repository. + # source_version: + # The source version as reported by the `bump_version.sh show` command. + # tags: + # A comma separated list of Docker tags to be applied to the images on + # DockerHub. The tags will vary depending on: + # - The event that triggered the build. + # - The branch the build is based upon. + # - The git tag the build is based upon. + # + # When a build is based on a git tag of the form `v*.*.*` the image will + # be tagged on DockerHub with multiple levels of version specificity. + # For example, a git tag of `v1.2.3+a` will generate Docker tags of + # `:1.2.3_a`, `:1.2.3`, `:1.2`, `:1`, and `:latest`. + # + # Builds targeting the default branch will be tagged with `:edge`. + # + # Builds from other branches will be tagged with the branch name. + # Solidi `/` in branch names are replaced with hyphens `-` in the Docker + # tag. + # + # Builds triggered by a push event are tagged with a short hash in the + # form: sha-12345678 + # + # Builds triggered by a pull request are tagged with the pull request + # number in the form pr-123. + # + # Builds triggered using the GitHub GUI (workflow_dispatch) are tagged + # with the value specified by the user. + # + # Scheduled builds are tagged with `:nightly`. name: "Prepare build variables" runs-on: ubuntu-latest outputs: @@ -122,7 +163,10 @@ jobs: - name: Setup debug session remote shell uses: mxschmitt/action-tmate@v3 if: github.event.inputs.remote-shell == 'true' + build: + # Builds a single test image for the native platform. This image is saved + # as an artifact and loaded by the test job. name: "Build test image" runs-on: ubuntu-latest needs: [prepare] @@ -154,6 +198,8 @@ jobs: file: ./Dockerfile outputs: type=docker,dest=dist/image.tar tags: ${{ env.IMAGE_NAME }}:latest # not to be pushed + # For a list of pre-defined annotation keys and value types see: + # https://github.com/opencontainers/image-spec/blob/master/annotations.md labels: "\ org.opencontainers.image.created=${{ needs.prepare.outputs.created }} @@ -184,7 +230,9 @@ jobs: with: name: dist path: dist + test: + # Executes tests on the single-platform image created in the "build" job. name: "Test image" runs-on: ubuntu-latest needs: [build] @@ -217,7 +265,13 @@ jobs: env: RELEASE_TAG: ${{ github.event.release.tag_name }} run: pytest --runslow + build-push-all: + # Builds the final set of images for each of the platforms listed in + # PLATFORMS environment variable. These images are tagged with the Docker + # tags calculated in the "prepare" job and pushed to DockerHub. The + # contents of README.md is pushed as the image's description. This job is + # skipped when the triggering event is a pull request. name: "Build and push all platforms" runs-on: ubuntu-latest needs: [lint, prepare, test] From a29dc98d6f00c9d6ad6e05fc9f6b99163eb512db Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 5 Nov 2020 11:54:24 -0500 Subject: [PATCH 12/14] Apply correct typographic designation to character replacement comment. --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b5dd214..63afd50 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -94,8 +94,8 @@ jobs: # # Builds targeting the default branch will be tagged with `:edge`. # - # Builds from other branches will be tagged with the branch name. - # Solidi `/` in branch names are replaced with hyphens `-` in the Docker + # Builds from other branches will be tagged with the branch name. Solidi + # `/` in branch names are replaced with hyphen-minuses `-` in the Docker # tag. # # Builds triggered by a push event are tagged with a short hash in the From e9426727634b5a8c198f83395262966290364f85 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 5 Nov 2020 12:22:58 -0500 Subject: [PATCH 13/14] Modernize Docker shields in README to point to working links. --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index abfe21e..99246a9 100644 --- a/README.md +++ b/README.md @@ -6,8 +6,9 @@ ## Docker Image ## -![MicroBadger Layers](https://img.shields.io/microbadger/layers/cisagov/example.svg) -![MicroBadger Size](https://img.shields.io/microbadger/image-size/cisagov/example.svg) +[![Docker Pulls](https://img.shields.io/docker/pulls/cisagov/example)](https://hub.docker.com/r/cisagov/example) +[![Docker Image Size (latest by date)](https://img.shields.io/docker/image-size/cisagov/example)](https://hub.docker.com/r/cisagov/example) +[![Platforms](https://img.shields.io/badge/platforms-amd64%20%7C%20arm%2Fv6%20%7C%20arm%2Fv7%20%7C%20arm64%20%7C%20ppc64le%20%7C%20s390x-blue)](https://hub.docker.com/r/cisagov/skeleton-docker/tags) This is a docker skeleton project that can be used to quickly get a new [cisagov](https://github.com/cisagov) GitHub docker project From da037f5045c34927038f264733abc51816e2b449 Mon Sep 17 00:00:00 2001 From: Mark Feldhousen Date: Thu, 5 Nov 2020 13:04:26 -0500 Subject: [PATCH 14/14] Elaborate on the Solidus. Add some additional comments in case anyone is confused about the nature of the solidus and its place within our tagging system. Co-authored-by: Shane Frasier --- .github/workflows/build.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 63afd50..d56e3eb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -95,8 +95,11 @@ jobs: # Builds targeting the default branch will be tagged with `:edge`. # # Builds from other branches will be tagged with the branch name. Solidi - # `/` in branch names are replaced with hyphen-minuses `-` in the Docker - # tag. + # (`/` characters - commonly known as slashes) in branch names are + # replaced with hyphen-minuses (`-` characters) in the Docker tag. For + # more information about the solidus see these links: + # * https://www.compart.com/en/unicode/U+002F + # * https://en.wikipedia.org/wiki/Slash_(punctuation)#Encoding # # Builds triggered by a push event are tagged with a short hash in the # form: sha-12345678