diff --git a/.github/codeql.yml b/.github/codeql.yml new file mode 100644 index 0000000..9771ca0 --- /dev/null +++ b/.github/codeql.yml @@ -0,0 +1,52 @@ +name: "Code scanning - action" + +on: + push: + pull_request: + schedule: + - cron: '0 19 * * 0' + +jobs: + CodeQL-Build: + + # CodeQL runs on ubuntu-latest and windows-latest + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + with: + # We must fetch at least the immediate parents so that if this is + # a pull request then we can checkout the head. + fetch-depth: 2 + + # If this run was triggered by a pull request event, then checkout + # the head of the pull request instead of the merge commit. + - run: git checkout HEAD^2 + if: ${{ github.event_name == 'pull_request' }} + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + # Override language selection by uncommenting this and choosing your languages + # with: + # languages: go, javascript, csharp, python, cpp, java + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..2659fed --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,14 @@ +version: 2 +updates: +- package-ecosystem: docker + directory: "/" + schedule: + interval: weekly + time: '11:00' + open-pull-requests-limit: 10 +- package-ecosystem: pip + directory: "/" + schedule: + interval: daily + time: '11:00' + open-pull-requests-limit: 10 diff --git a/.github/workflows/ReleaseCycleProposal.md b/.github/workflows/ReleaseCycleProposal.md new file mode 100644 index 0000000..e4c554c --- /dev/null +++ b/.github/workflows/ReleaseCycleProposal.md @@ -0,0 +1,95 @@ +## New PR & Testing Process Proposal + + +### Goals + +Design a development process and write related automation that: + +1. Allows for rapid parallel development +1. Prevents cross-developer collisions +1. Creates a full audit trail of all changes, tests, and deployments +1. Tests small changes (features and bug fixes) individually +1. Can be enabled on most repos with little to no modification +1. Provides multiple test/check points througout the development process including: + - Build tests that are created at the beginning of the code review (PR) process _pre_-merge + - Images that can be tested both before and after merges to the `develop` branch + - Release candidate images that can be tested prior to production releases (aka merges to `master`) +1. Reduces excessive or redundant build testing through the use of Draft/WIP pull requests +1. Enables manual build triggering as needed through the Draft/WIP vs. "Ready for Review" pull requests +1. Allows for easy ad-hoc deployment of images to any environment (assuming above safeguards are in place) +1. Provides a basis for potential future continuous deployment + +### Process + +(Will start at the end of the cycle, as it oddly makes things more clear) + +#### Release Cycle Pt. 1: + +- An official "release" (version bump) is merged from develop to master. +- The production Docker image `:latest` is then updated and pushed to GitHub Packages. +- After this release update is complete, a new Draft/WIP PR for the next release is created (again a merge from develop to master). + +#### Development Cycle: + +- Once development on a new release begins, developers create their short-lived feature branches off of develop. +- Once their feature is complete, they create a new PR to merge from their feature branch to develop. +- When the PR is created, an `:pr#` image is autodamically built uploaded to GitHub Packages. + - Note that this image is intentionally separate from the prod `` image. +- (Optional) - After the `-develop:pr#` image build & upload is complete, some process (PR labeling or a PR comment) "enables" this image to be tested in appdev, ci, and/or next. +- Through some process, we test the image in these environments. + - Note that if we simply use the `-develop:pr#` nomenclature, the above "test environment enabling" may not be needed. + - Simply specifying which PR# tag you want to pull in each environment would suffice. +- Once all testing is complete, the code is merged to develop. + +#### Release Cycle Pt. 2 + +- Once all development for a release is complete, the aforementioned "Release PR" (aka the long-running Draft/WIP PR that will merge from develop to master) is taken out of WIP/Draft mode. +- Taking the Release PR out of Draft/WIP will trigger one final build test from the develop branch. +- Once this image has passed the initial build test, the repo owner can merge the Release PR to master. +- Once merged, the final build image will be uploaded to the prod `:latest` image. +- Process starts again by creating a new Draft/WIP Release PR for the next release. + +### Discussion + +#### Process Advantages + +- Using the `-develop:pr#` naming scheme allows us to test multiple feature improvements at will, allowing for higher velocity development. +- Separating features in this way allows us to run build tests on smaller changes, reducing errors. +- The final Release PR build ensures that the final production image still builds correctly without error. +- Keeping separate prod & `-develop` images allows us to pull development, pre-release (release candidate), and production images at will. +- Note that although the `-develop` image will have a fair number of tags (one per PR), it's extremely easy to pull the correct test image by simply specifying the `:pr-#` tag. + + +#### Image Naming + + +| Environment | Image | Contains | Note | +| ----------- | ----------- |----------- |----------- | +| Dev | _appname_-develop:pr-## | Pre-merge dev PR build| \*See `Improvements` section| +| Pre-Stage | _appname_-develop:latest | Latest post-mere dev PR build | | +| Stage | _appname_:pr-## | Pre-merge dev -> master release candidate | | +| Prod | _appname_:latest | Production (post-merge master) build | | + +#### Deployment + +Our current process of deploying to active environmets (appdev, ci, next, & prod) utilizes Docker image tags. + +To implement this tagging in the new process, a workflow script is being developed to trigger retags of the above images (typically just the `:latest` test and prod images) manually via an web hook. This hook can be triggered by an internal (behind the firewall): + +- CI tool like Jenkins +- Chatops bot connected to Slack +- Other tool such as Postman + +#### Improvements + +This builds on the Deployment section above... + +- \*Currently pulling a specific `development` build (pre-merge) woul require going into the Rancher UI and updating the `_appname_-develop:pr-##` number to reflect the PR you wish to test. + - This is fine as an MVP, given the core devs have access to our Appdev Rancher cluster. + - For future Rancher environments, it is possible that fewer devs will have direct access, neccessitating some improvements to the process. +- A potential improvement might be to implement a bit of "chat-ops": + - When a new PR build completes, a notification is posted in the related Slack channel (e.g. \#narrative-build). + - _Only_ devs who can update the dev enviroment are members of this locked channel. + - Once a PR build notification is posted, one of the devs/devops members can deploy with something like `.appdev deploy pr-##`. + - A chat bot in _our_ infrastructure would listen for these chatops commands and use Rancher's API to update the image url (`_appname_-develop:pr-##`) and reload the service in question. +- Depending on the difficulty of deploying multiple frontends (in the case of `narrative`) to connect to the same appdev backend, it could be possible to test multiple PRs simultaneously, using URLs like `pr##.appdev.kbase.us`. :shrug: diff --git a/.github/workflows/build_prodrc_pr.yaml b/.github/workflows/build_prodrc_pr.yaml new file mode 100644 index 0000000..ca5c715 --- /dev/null +++ b/.github/workflows/build_prodrc_pr.yaml @@ -0,0 +1,27 @@ +--- +name: Build Prod RC Image +'on': + pull_request: + branches: + - master + types: + - opened + - synchronize + - ready_for_review +jobs: + docker_build: + runs-on: ubuntu-latest + steps: + - name: Check out GitHub Repo + if: github.event.pull_request.draft == false + with: + ref: "${{ github.event.pull_request.head.sha }}" + uses: actions/checkout@v2 + - name: Build and Push to Packages + if: github.event.pull_request.draft == false + env: + PR: "${{ github.event.pull_request.number }}" + SHA: "${{ github.event.pull_request.head.sha }}" + DOCKER_ACTOR: "jsfillman" + DOCKER_TOKEN: "${{ secrets.GHCR_TOKEN }}" + run: "./.github/workflows/scripts/build_prodrc_pr.sh\n" diff --git a/.github/workflows/build_test_pr.yaml b/.github/workflows/build_test_pr.yaml new file mode 100644 index 0000000..377de96 --- /dev/null +++ b/.github/workflows/build_test_pr.yaml @@ -0,0 +1,27 @@ +--- +name: Build Test Image +'on': + pull_request: + branches: + - develop + types: + - opened + - synchronize + - ready_for_review +jobs: + docker_build: + runs-on: ubuntu-latest + steps: + - name: Check out GitHub Repo + if: github.event.pull_request.draft == false + with: + ref: "${{ github.event.pull_request.head.sha }}" + uses: actions/checkout@v2 + - name: Build and Push to Packages + if: github.event.pull_request.draft == false + env: + PR: "${{ github.event.pull_request.number }}" + SHA: "${{ github.event.pull_request.head.sha }}" + DOCKER_ACTOR: "jsfillman" + DOCKER_TOKEN: "${{ secrets.GHCR_TOKEN }}" + run: "./.github/workflows/scripts/build_test_pr.sh\n" diff --git a/.github/workflows/scripts/build_prodrc_pr.sh b/.github/workflows/scripts/build_prodrc_pr.sh new file mode 100755 index 0000000..d888fc9 --- /dev/null +++ b/.github/workflows/scripts/build_prodrc_pr.sh @@ -0,0 +1,16 @@ +#! /usr/bin/env bash + +export MY_ORG=$(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $1}') +export MY_APP=$(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $2}') +export DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") +export BUILD_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") +export COMMIT=$(echo "$SHA" | cut -c -7) + +docker login -u "$DOCKER_ACTOR" -p "$DOCKER_TOKEN" ghcr.io +docker build --build-arg BUILD_DATE="$DATE" \ + --build-arg COMMIT="$COMMIT" \ + --build-arg BRANCH="$GITHUB_HEAD_REF" \ + --build-arg PULL_REQUEST="$PR" \ + --label us.kbase.vcs-pull-req="$PR" \ + -t ghcr.io/"$MY_ORG"/"$MY_APP":"pr-""$PR" . +docker push ghcr.io/"$MY_ORG"/"$MY_APP":"pr-""$PR" diff --git a/.github/workflows/scripts/build_test_pr.sh b/.github/workflows/scripts/build_test_pr.sh new file mode 100755 index 0000000..546b1b4 --- /dev/null +++ b/.github/workflows/scripts/build_test_pr.sh @@ -0,0 +1,17 @@ +#! /usr/bin/env bash + +export MY_ORG=$(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $1}') +export MY_APP=$(echo $(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $2}')"-develop") +export DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") +export BUILD_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") +export COMMIT=$(echo "$SHA" | cut -c -7) + +echo $DOCKER_TOKEN | docker login ghcr.io -u $DOCKER_ACTOR --password-stdin +docker build --build-arg BUILD_DATE="$DATE" \ + --build-arg COMMIT="$COMMIT" \ + --build-arg BRANCH="$GITHUB_HEAD_REF" \ + --build-arg PULL_REQUEST="$PR" \ + --label us.kbase.vcs-pull-req="$PR" \ + -t ghcr.io/"$MY_ORG"/"$MY_APP":"pr-""$PR" . +docker push ghcr.io/"$MY_ORG"/"$MY_APP":"pr-""$PR" + \ No newline at end of file diff --git a/.github/workflows/scripts/deploy_tag.sh b/.github/workflows/scripts/deploy_tag.sh new file mode 100755 index 0000000..5fb928a --- /dev/null +++ b/.github/workflows/scripts/deploy_tag.sh @@ -0,0 +1,34 @@ +#! /usr/bin/env bash + +# Usage: ./deploy_tag.sh -e TARGET -o ORG -r REPO -s DEV_PROD -t IMAGE_TAG +# +# Example 1: ./deploy_tag.sh -o "kbase" -r "narrative-traefiker" -s "dev" -t "pr-9001" -e "ci" +# Example 2: ./deploy_tag.sh -o "kbase" -r "narrative" -s "prod" -t "latest" -e "next" +# +# Where: +# -o ORG is the organization (`kbase`, `kbaseapps`, etc.) +# -r REPO is the repository (e.g. `narrative`) +# -s DEV_PROD determines whether to pull the development {APPNAME}-develop or production {APPNAME} image. +# -t IMAGE_TAG is the *current* Docker image tag, typically `pr-#` or `latest` +# -e TARGET is one of: `appdsshev`, `ci`, or `next` +# +# Be sure to set $TOKEN first! +# See: https://docs.github.com/en/packages/getting-started-with-github-container-registry/migrating-to-github-container-registry-for-docker-images#authenticating-with-the-container-registry + + +while getopts e:o:r:s:t: option + do + case "${option}" + in + e) TARGET=${OPTARG};; + o) ORG=${OPTARG};; + r) REPO=${OPTARG};; + s) DEV_PROD=${OPTARG};; + t) IMAGE_TAG=${OPTARG};; + esac +done + +curl -H "Authorization: token $TOKEN" \ + -H 'Accept: application/vnd.github.everest-preview+json' \ + "https://api.github.com/repos/$ORG/$REPO/dispatches" \ + -d '{"event_type":"Tag '"$DEV_PROD"' '"$IMAGE_TAG"' for '"$TARGET"'", "client_payload": {"image_tag": "'"$IMAGE_TAG"'","target": "'"$TARGET"'","dev_prod": "'"$DEV_PROD"'"}}' diff --git a/.github/workflows/scripts/tag_environments.sh b/.github/workflows/scripts/tag_environments.sh new file mode 100755 index 0000000..b39732a --- /dev/null +++ b/.github/workflows/scripts/tag_environments.sh @@ -0,0 +1,22 @@ + +#! /usr/bin/env bash +# Add vars for PR & environments to yaml, as called from external script + +export MY_ORG=$(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $1}') +export MY_APP=$(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $2}') +export DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") +export BUILD_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") +export COMMIT=$(echo "$SHA" | cut -c -7) + +if [ $DEV_PROD = "dev" ] || [ $DEV_PROD = "develop" ] +then + IMAGE=$MY_APP"-develop" +else + IMAGE=$MY_APP +fi + +echo "Dev or Prod:" $DEV_PROD +docker login -u "$DOCKER_ACTOR" -p "$DOCKER_TOKEN" ghcr.io +docker pull ghcr.io/"$MY_ORG"/"$IMAGE":"$IMAGE_TAG" +docker tag ghcr.io/"$MY_ORG"/"$IMAGE":"$IMAGE_TAG" ghcr.io/"$MY_ORG"/"$IMAGE":"$TARGET" +docker push ghcr.io/"$MY_ORG"/"$IMAGE":"$TARGET" diff --git a/.github/workflows/scripts/tag_prod_latest.sh b/.github/workflows/scripts/tag_prod_latest.sh new file mode 100755 index 0000000..1390fd1 --- /dev/null +++ b/.github/workflows/scripts/tag_prod_latest.sh @@ -0,0 +1,12 @@ +#! /usr/bin/env bash + +export MY_ORG=$(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $1}') +export MY_APP=$(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $2}') +export DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") +export BUILD_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") +export COMMIT=$(echo "$SHA" | cut -c -7) + +docker login -u "$DOCKER_ACTOR" -p "$DOCKER_TOKEN" ghcr.io +docker pull ghcr.io/"$MY_ORG"/"$MY_APP":"pr-""$PR" +docker tag ghcr.io/"$MY_ORG"/"$MY_APP":"pr-""$PR" ghcr.io/"$MY_ORG"/"$MY_APP":"latest" +docker push ghcr.io/"$MY_ORG"/"$MY_APP":"latest" diff --git a/.github/workflows/scripts/tag_test_latest.sh b/.github/workflows/scripts/tag_test_latest.sh new file mode 100755 index 0000000..c0dc504 --- /dev/null +++ b/.github/workflows/scripts/tag_test_latest.sh @@ -0,0 +1,12 @@ +#! /usr/bin/env bash + +export MY_ORG=$(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $1}') +export MY_APP=$(echo $(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $2}')"-develop") +export DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") +export BUILD_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") +export COMMIT=$(echo "$SHA" | cut -c -7) + +docker login -u "$DOCKER_ACTOR" -p "$DOCKER_TOKEN" ghcr.io +docker pull ghcr.io/"$MY_ORG"/"$MY_APP":"pr-""$PR" +docker tag ghcr.io/"$MY_ORG"/"$MY_APP":"pr-""$PR" ghcr.io/"$MY_ORG"/"$MY_APP":"latest" +docker push ghcr.io/"$MY_ORG"/"$MY_APP":"latest" diff --git a/.github/workflows/tag_environments.yaml b/.github/workflows/tag_environments.yaml new file mode 100644 index 0000000..8a8a70b --- /dev/null +++ b/.github/workflows/tag_environments.yaml @@ -0,0 +1,19 @@ +--- +name: Tag Image For Deploy +'on': + repository_dispatch +jobs: + tag_environments: + runs-on: ubuntu-latest + steps: + - name: Check out GitHub Repo + uses: actions/checkout@v2 + - name: Tag Deploy Environments + env: + DOCKER_ACTOR: jsfillman + DOCKER_TOKEN: ${{ secrets.GHCR_TOKEN }} + IMAGE_TAG: ${{ github.event.client_payload.image_tag }} + SHA: ${{ github.event.pull_request.head.sha }} + TARGET: ${{ github.event.client_payload.target }} + DEV_PROD: ${{ github.event.client_payload.dev_prod }} + run: './.github/workflows/scripts/tag_environments.sh' diff --git a/.github/workflows/tag_prod_latest.yaml b/.github/workflows/tag_prod_latest.yaml new file mode 100644 index 0000000..10de45f --- /dev/null +++ b/.github/workflows/tag_prod_latest.yaml @@ -0,0 +1,26 @@ +--- +name: Tag Prod Latest +'on': + pull_request: + branches: + - master + types: + - closed +jobs: + docker_tag: + runs-on: ubuntu-latest + steps: + - name: Check out GitHub Repo + if: github.event_name == 'pull_request' && github.event.action == 'closed' && + github.event.pull_request.merged == true + with: + ref: "${{ github.event.pull_request.head.sha }}" + uses: actions/checkout@v2 + - name: Build and Push to Packages + if: github.event.pull_request.draft == false + env: + PR: "${{ github.event.pull_request.number }}" + SHA: "${{ github.event.pull_request.head.sha }}" + DOCKER_ACTOR: "jsfillman" + DOCKER_TOKEN: "${{ secrets.GHCR_TOKEN }}" + run: "./.github/workflows/scripts/tag_prod_latest.sh\n" diff --git a/.github/workflows/tag_test_latest.yaml b/.github/workflows/tag_test_latest.yaml new file mode 100644 index 0000000..63eb825 --- /dev/null +++ b/.github/workflows/tag_test_latest.yaml @@ -0,0 +1,26 @@ +--- +name: Tag Latest Test Image +'on': + pull_request: + branches: + - develop + types: + - closed +jobs: + docker_tag: + runs-on: ubuntu-latest + steps: + - name: Check out GitHub Repo + if: github.event_name == 'pull_request' && github.event.action == 'closed' && + github.event.pull_request.merged == true + with: + ref: "${{ github.event.pull_request.head.sha }}" + uses: actions/checkout@v2 + - name: Build and Push to Packages + if: github.event.pull_request.draft == false + env: + PR: "${{ github.event.pull_request.number }}" + SHA: "${{ github.event.pull_request.head.sha }}" + DOCKER_ACTOR: "jsfillman" + DOCKER_TOKEN: "${{ secrets.GHCR_TOKEN }}" + run: "./.github/workflows/scripts/tag_test_latest.sh\n" diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..1747a3a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,26 @@ +FROM bitnami/minideb:latest + +# Install Base Packages +RUN \ + install_packages \ + ansible \ + bash \ + curl \ + git \ + less \ + mongo-tools \ + net-tools \ + ssh \ + tzdata \ + unzip \ + vim-tiny \ + wget + +# Install Mongo +RUN \ + mkdir -p /data/db/tmp && cd /data/db/tmp/ \ + wget -q https://repo.mongodb.org/apt/debian/dists/buster/mongodb-org/4.4/main/binary-amd64/mongodb-org-shell_4.4.4_amd64.deb; \ + wget -q https://repo.mongodb.org/apt/debian/dists/buster/mongodb-org/4.4/main/binary-amd64/mongodb-org-server_4.4.4_amd64.deb; \ + apt install /data/db/tmp/mongodb-org-shell_4.4.4_amd64.deb; \ + apt install /data/db/tmp/mongodb-org-server_4.4.4_amd64.deb; \ + rm /data/db/tmp/*.deb \ No newline at end of file