Merge pull request #2883 from kbase/develop

Merge to main for 5.0.3 release

.dockerignore

@@ -0,0 +1,11 @@
+Dockerfile
+.husky
+.github
+.vscode
+js-coverage/
+python-coverage/
+narrbase-image/
+node_modules/
+**/__pycache__
+karma-result.json
+install.log

.eslintignore

@@ -1,8 +1,12 @@
 .git
 .github
+.husky
+cover
 docs
 js-coverage
 kbase-extension/static/ext_components
+kbase-extension/static/ext_modules
 kbase-extension/static/ext_packages
-kbase-extension/static/kbase/js/patched-components
 node_modules
+python-coverage
+venv

.flake8

@@ -1,10 +1,16 @@
 [flake8]
-# https://ljvmiranda921.github.io/notebook/2018/06/21/precommits-using-black-and-flake8/
-ignore = E203, E266, E501, W503, F403, F401, E402, C901
-select = B,C,E,F,W,T4,B9
-exclude =
-.git,
+max-line-length = 88
+# E203: whitespace before ‘,’, ‘;’, or ‘:’
+# E501: line length
+# W503: line break after binary operator
+ignore = E203, E501, W503
+extend-exclude =
+*.pyc,
 .github,
+.husky,
 deployment,
 docs,
-node_modules
+node_modules,
+js-coverage,
+python-coverage,
+src/build

.github/workflows/ReleaseCycleProposal.md

@@ -13,7 +13,7 @@ Design a development process and write related automation that:
 1. Provides multiple test/check points througout the development process including:
     - Build tests that are created at the beginning of the code review (PR) process _pre_-merge
     - Images that can be tested both before and after merges to the `develop` branch
-    - Release candidate images that can be tested prior to production releases (aka merges to `master`)
+    - Release candidate images that can be tested prior to production releases (aka merges to `main`)
 1. Reduces excessive or redundant build testing through the use of Draft/WIP pull requests
 1. Enables manual build triggering as needed through the Draft/WIP vs. "Ready for Review" pull requests
 1. Allows for easy ad-hoc deployment of images to any environment (assuming above safeguards are in place)
@@ -25,9 +25,9 @@ Design a development process and write related automation that:
 
 #### Release Cycle Pt. 1:
 
-- An official "release" (version bump) is merged from develop to master.
+- An official "release" (version bump) is merged from develop to main.
 - The production Docker image `<appname>:latest` is then updated and pushed to GitHub Packages.
-- After this release update is complete, a new Draft/WIP PR for the next release is created (again a merge from develop to master). 
+- After this release update is complete, a new Draft/WIP PR for the next release is created (again a merge from develop to main).
 
 #### Development Cycle:
 
@@ -37,15 +37,15 @@ Design a development process and write related automation that:
     - Note that this image is intentionally separate from the prod `<appname>` image.
 - (Optional) - After the `<appname>-develop:pr#` image build & upload is complete, some process (PR labeling or a PR comment) "enables" this image to be tested in appdev, ci, and/or next.
 - Through some process, we test the image in these environments.
-    - Note that if we simply use the `<appname>-develop:pr#` nomenclature, the above "test environment enabling" may not be needed. 
+    - Note that if we simply use the `<appname>-develop:pr#` nomenclature, the above "test environment enabling" may not be needed.
     - Simply specifying which PR# tag you want to pull in each environment would suffice.
 - Once all testing is complete, the code is merged to develop.
 
 #### Release Cycle Pt. 2
 
-- Once all development for a release is complete, the aforementioned "Release PR" (aka the long-running Draft/WIP PR that will merge from develop to master) is taken out of WIP/Draft mode.
+- Once all development for a release is complete, the aforementioned "Release PR" (aka the long-running Draft/WIP PR that will merge from develop to main) is taken out of WIP/Draft mode.
 - Taking the Release PR out of Draft/WIP will trigger one final build test from the develop branch.
-- Once this image has passed the initial build test, the repo owner can merge the Release PR to master.
+- Once this image has passed the initial build test, the repo owner can merge the Release PR to main.
 - Once merged, the final build image will be uploaded to the prod `<appname>:latest` image.
 - Process starts again by creating a new Draft/WIP Release PR for the next release.
 
@@ -67,8 +67,8 @@ Design a development process and write related automation that:
 | ----------- | ----------- |----------- |----------- |
 | Dev      | _appname_-develop:pr-## | Pre-merge dev PR build| \*See `Improvements` section|
 | Pre-Stage   | _appname_-develop:latest | Latest post-mere dev PR build | |
-| Stage      | _appname_:pr-##       | Pre-merge dev -> master release candidate | |
-| Prod   | _appname_:latest        | Production (post-merge master) build | |
+| Stage      | _appname_:pr-##       | Pre-merge dev -> main release candidate | |
+| Prod   | _appname_:latest        | Production (post-merge main) build | |
 
 #### Deployment
 

.github/workflows/build_and_push_image.yml

@@ -0,0 +1,82 @@
+name: Build and push narrative images
+
+on:
+  workflow_call:
+    secrets:
+      GHCR_USERNAME:
+        required: true
+      GHCR_TOKEN:
+        required: true
+
+jobs:
+  docker_build_image:
+    if: github.event.pull_request.draft == false
+    runs-on: ubuntu-latest
+    steps:
+    -
+      name: Check out GitHub Repo
+      with:
+        ref: "${{ github.event.pull_request.head.sha }}"
+      uses: actions/checkout@v2
+    -
+      name: Set up environment
+      run: |
+        if [ ${{ github.base_ref }} == 'main' ]; then
+            branch=''
+        else
+            branch="-""${{ github.base_ref }}"
+        fi
+        narrative_version=`grep '\"version\":' src/config.json.templ | awk '{print $2}' | sed 's/"//g'`
+        narrative_git_hash=`grep '\"git_commit_hash\":' src/config.json.templ | awk '{print $2}' | sed 's/"//g' | sed 's/,//'`
+        echo "APP_IMAGE_TAG=ghcr.io/${{ github.repository }}${branch}:pr-${{ github.event.pull_request.number }}" >> $GITHUB_ENV
+        echo "APP_VERSION_IMAGE_TAG=ghcr.io/${{ github.repository }}${branch}_version:pr-${{ github.event.pull_request.number }}" >> $GITHUB_ENV
+        echo "BRANCH=${branch}" >> $GITHUB_ENV
+        echo "BUILD_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ")" >> $GITHUB_ENV
+        echo "NARRATIVE_GIT_HASH=${narrative_git_hash}" >> $GITHUB_ENV
+        echo "NARRATIVE_VERSION=${narrative_version}" >> $GITHUB_ENV
+        echo "PR=pr-${{ github.event.pull_request.number }}" >> $GITHUB_ENV
+        echo "VCS_REF=$(echo "${{ github.event.pull_request.head.sha }}" | cut -c -7)" >> $GITHUB_ENV
+    -
+      name: Set up QEMU
+      uses: docker/setup-qemu-action@v1
+    -
+      name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v1
+    -
+      name: Log in to the Container registry
+      uses: docker/login-action@v1
+      with:
+        registry: ghcr.io
+        username: ${{ secrets.GHCR_USERNAME }}
+        password: ${{ secrets.GHCR_TOKEN }}
+    -
+      name: Build narrative image
+      uses: docker/build-push-action@v2
+      with:
+        context: .
+        push: true
+        tags: ${{ env.APP_IMAGE_TAG }}
+        build-args: |
+          BRANCH=${{ github.head_ref }}
+          BUILD_DATE=${{ env.BUILD_DATE }}
+          NARRATIVE_VERSION=${{ env.NARRATIVE_VERSION }}
+          VCS_REF=${{ env.VCS_REF }}
+        labels: |
+          us.kbase.vcs-pull-req=pr-${{ env.PR }}
+    -
+      name: Build version image
+      uses: docker/build-push-action@v2
+      if: ${{ github.base_ref }} == 'main' || ${{ github.base_ref }} == 'develop'
+      with:
+        context: .
+        push: true
+        file: Dockerfile2
+        tags: ${{ env.APP_VERSION_IMAGE_TAG }}
+        build-args: |
+          BRANCH=${{ github.head_ref }}
+          BUILD_DATE=${{ env.BUILD_DATE }}
+          NARRATIVE_VERSION=${{ env.NARRATIVE_VERSION }}
+          NARRATIVE_GIT_HASH=${{ env.NARRATIVE_GIT_HASH }}
+          VCS_REF=${{ env.VCS_REF }}
+        labels: |
+          us.kbase.vcs-pull-req=pr-${{ env.PR }}

.github/workflows/build_prodrc_pr.yaml

@@ -1,28 +1,32 @@
 ---
 name: Build Prod RC Image
-'on':
+on:
   pull_request:
     branches:
-    - master
     - main
     types:
     - opened
     - synchronize
     - ready_for_review
+
 jobs:
-  docker_build:
-    runs-on: ubuntu-latest
-    steps:
-    - name: Check out GitHub Repo
-      if: github.event.pull_request.draft == false
-      with:
-        ref: "${{ github.event.pull_request.head.sha }}"
-      uses: actions/checkout@v2
-    - name: Build and Push to Packages
-      if: github.event.pull_request.draft == false
-      env:
-        PR: "${{ github.event.pull_request.number }}"
-        SHA: "${{ github.event.pull_request.head.sha }}"
-        DOCKER_ACTOR: "${{ secrets.GHCR_USERNAME }}"
-        DOCKER_TOKEN: "${{ secrets.GHCR_TOKEN }}"
-      run: "./.github/workflows/scripts/build_prodrc_pr.sh\n"
+  run_build_and_push:
+    uses: ./.github/workflows/build_and_push_image.yml
+    secrets:
+      GHCR_USERNAME: ${{ secrets.GHCR_USERNAME }}
+      GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }}
+
+  run_unit_tests:
+    uses: ./.github/workflows/unit_test.yml
+    needs: run_build_and_push
+    secrets:
+      GHCR_USERNAME: ${{ secrets.GHCR_USERNAME }}
+      GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }}
+
+  run_integration_tests:
+    uses: ./.github/workflows/integration_test.yml
+    needs: run_build_and_push
+    secrets:
+      GHCR_USERNAME: ${{ secrets.GHCR_USERNAME }}
+      GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }}
+      NARRATIVE_TEST_TOKEN: ${{ secrets.NARRATIVE_TEST_TOKEN }}

.github/workflows/build_test_pr.yaml

@@ -1,27 +1,32 @@
 ---
-name: Build Test Image
-'on':
+name: Build Dev Image
+on:
   pull_request:
     branches:
     - develop
     types:
     - opened
     - synchronize
     - ready_for_review
+
 jobs:
-  docker_build:
-    runs-on: ubuntu-latest
-    steps:
-    - name: Check out GitHub Repo
-      if: github.event.pull_request.draft == false
-      with:
-        ref: "${{ github.event.pull_request.head.sha }}"
-      uses: actions/checkout@v2
-    - name: Build and Push to Packages
-      if: github.event.pull_request.draft == false
-      env:
-        PR: "${{ github.event.pull_request.number }}"
-        SHA: "${{ github.event.pull_request.head.sha }}"
-        DOCKER_ACTOR: "${{ secrets.GHCR_USERNAME }}"
-        DOCKER_TOKEN: "${{ secrets.GHCR_TOKEN }}"
-      run: "./.github/workflows/scripts/build_test_pr.sh\n"
+  run_build_and_push:
+    uses: ./.github/workflows/build_and_push_image.yml
+    secrets:
+      GHCR_USERNAME: ${{ secrets.GHCR_USERNAME }}
+      GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }}
+
+  run_unit_tests:
+    uses: ./.github/workflows/unit_test.yml
+    needs: run_build_and_push
+    secrets:
+      GHCR_USERNAME: ${{ secrets.GHCR_USERNAME }}
+      GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }}
+
+  run_integration_tests:
+    uses: ./.github/workflows/integration_test.yml
+    needs: run_build_and_push
+    secrets:
+      GHCR_USERNAME: ${{ secrets.GHCR_USERNAME }}
+      GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }}
+      NARRATIVE_TEST_TOKEN: ${{ secrets.NARRATIVE_TEST_TOKEN }}

.github/workflows/build_truss_pr.yaml

@@ -1,27 +1,32 @@
 ---
 name: Build Truss Branch Image
-'on':
+on:
   pull_request:
     branches:
     - truss
     types:
     - opened
     - synchronize
     - ready_for_review
+
 jobs:
-  docker_build:
-    runs-on: ubuntu-latest
-    steps:
-    - name: Check out GitHub Repo
-      if: github.event.pull_request.draft == false
-      with:
-        ref: "${{ github.event.pull_request.head.sha }}"
-      uses: actions/checkout@v2
-    - name: Build and Push to Packages
-      if: github.event.pull_request.draft == false
-      env:
-        PR: "${{ github.event.pull_request.number }}"
-        SHA: "${{ github.event.pull_request.head.sha }}"
-        DOCKER_ACTOR: "${{ secrets.GHCR_USERNAME }}"
-        DOCKER_TOKEN: "${{ secrets.GHCR_TOKEN }}"
-      run: "./.github/workflows/scripts/build_truss_pr.sh\n"
+  run_build_and_push:
+    uses: ./.github/workflows/build_and_push_image.yml
+    secrets:
+      GHCR_USERNAME: ${{ secrets.GHCR_USERNAME }}
+      GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }}
+
+  run_unit_tests:
+    uses: ./.github/workflows/unit_test.yml
+    needs: run_build_and_push
+    secrets:
+      GHCR_USERNAME: ${{ secrets.GHCR_USERNAME }}
+      GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }}
+
+  run_integration_tests:
+    uses: ./.github/workflows/integration_test.yml
+    needs: run_build_and_push
+    secrets:
+      GHCR_USERNAME: ${{ secrets.GHCR_USERNAME }}
+      GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }}
+      NARRATIVE_TEST_TOKEN: ${{ secrets.NARRATIVE_TEST_TOKEN }}

.github/workflows/build_truss_push.yaml

@@ -1,23 +1,46 @@
 ---
-name: Build Truss Image Push
-'on':
+name: Build and push Truss image
+on:
   push:
     branches:
     - truss
+
 jobs:
   docker_build:
+    if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest
     steps:
-    - name: Check out GitHub Repo
-      if: github.event.pull_request.draft == false
+    -
+      name: Check out GitHub Repo
       with:
         ref: "${{ github.event.pull_request.head.sha }}"
       uses: actions/checkout@v2
-    - name: Build and Push to Packages
-      if: github.event.pull_request.draft == false
-      env:
-        PR: "${{ github.event.pull_request.number }}"
-        SHA: "${{ github.event.pull_request.head.sha }}"
-        DOCKER_ACTOR: "${{ secrets.GHCR_USERNAME }}"
-        DOCKER_TOKEN: "${{ secrets.GHCR_TOKEN }}"
-      run: "./.github/workflows/scripts/build_truss_push.sh\n"
+    -
+      name: Set up environment
+      run: |
+        echo "BUILD_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ")" >> $GITHUB_ENV
+        echo "VCS_REF=$(echo "${{ github.event.pull_request.head.sha }}" | cut -c -7)" >> $GITHUB_ENV
+    -
+      name: Set up QEMU
+      uses: docker/setup-qemu-action@v1
+    -
+      name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v1
+    -
+      name: Log in to the Container registry
+      uses: docker/login-action@v1
+      with:
+        registry: ghcr.io
+        username: ${{ secrets.GHCR_USERNAME }}
+        password: ${{ secrets.GHCR_TOKEN }}
+    -
+      name: Build narrative image
+      uses: docker/build-push-action@v2
+      with:
+        context: .
+        push: true
+        tags: ghcr.io/${{ github.repository }}-truss:latest
+        build-args: |
+          BUILD_DATE=${{ env.BUILD_DATE }}
+          VCS_REF=${{ env.VCS_REF }}
+          BRANCH=${{ github.head_ref }}