Merge pull request #32 from ACCESS-NRI/25-simple-config-ci-checks

Add pytests and CI for quality assurance (QA) configuration checks

.github/workflows/call-pr-1-ci.yml

@@ -12,7 +12,6 @@ on:
       - tools/**
       - doc/**
       - .*
-      - metadata.yaml
       - README.md
 jobs:
   call:

.github/workflows/pr-1-ci.yml

@@ -1,6 +1,22 @@
 name: PR Checks
 on:
   workflow_call:
+    inputs:
+      qa-pytest-markers:
+        type: string
+        required: false
+        default: config
+        description: List of markers for the pytest QA CI checks
+      qa-pytest-add-model-markers:
+        type: boolean
+        required: false
+        default: true
+        description: Whether to run model-specific tests
+      repro-pytest-markers:
+        type: string
+        required: false
+        default: checksum
+        description: List of markers for the pytest repro CI checks
   # Workflows that call this workflow use the following triggers:
   # pull_request:
   #   branches:
@@ -12,7 +28,6 @@ on:
   #     - tools/**
   #     - doc/**
   #     - .*
-  #     - metadata.yaml
   #     - README.md
 jobs:
   commit-check:
@@ -35,10 +50,15 @@ jobs:
 
   branch-check:
     name: PR Source Branch Check
+    # This check is used as a precursor to any repro-ci checks - which are only fired
+    # on dev-* -> release-* PRs.
     # This check is run to confirm that the source branch is of the form `dev-<config>`
     # and the target branch is of the form `release-<config>`. We are being especially
     # concerned with branch names because deployment to GitHub Environments can only
-    # be done on source branches with a certain pattern. See #20.
+    # be done on source branches with a certain pattern. See ACCESS-NRI/access-om2-configs#20.
+    if: needs.commit-check.outputs.authorship != vars.GH_ACTIONS_BOT_GIT_USER_NAME && startsWith(github.base_ref, 'release-') && startsWith(github.head_ref, 'dev-')
+    needs:
+      - commit-check
     runs-on: ubuntu-latest
     permissions:
       pull-requests: write
@@ -47,20 +67,6 @@ jobs:
         with:
           ref: ${{ github.head_ref }}
 
-      - name: Check Source
-        run: |
-          if [[ "${{ startsWith(github.head_ref, 'dev-') }}" == "false" ]]; then
-            echo "::error::Source branch ${{ github.head_ref }} doesn't match 'dev-*'"
-            exit 1
-          fi
-
-      - name: Check Target
-        run: |
-          if [[ "${{ startsWith(github.base_ref, 'release-') }}" == "false" ]]; then
-            echo "::error::Target branch ${{ github.base_ref }} doesn't match 'release-*'"
-            exit 1
-          fi
-
       - name: Compare Source and Target Config Names
         # In this step, we cut the 'dev-' and 'release-' to compare config names directly.
         run: |
@@ -77,24 +83,89 @@ jobs:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           GH_REPO: ${{ github.repository }}
           BODY: |
-            :x: Automated testing cannot be run on this branch :x:
+            :x: Automated Reproducibility testing cannot be run on this branch :x:
             Source and Target branches must be of the form `dev-<config>` and `release-<config>` respectively, and `<config>` must match between them.
             Rename the Source branch or check the Target branch, and try again.
         run: gh pr comment --body '${{ env.BODY }}'
 
+  qa-ci:
+    # Run quick, non-HPC tests on the runner.
+    name: QA CI Checks
+    needs:
+      - commit-check
+    if: needs.commit-check.outputs.authorship != vars.GH_ACTIONS_BOT_GIT_USER_NAME
+    runs-on: ubuntu-latest
+    permissions:
+      checks: write
+    steps:
+      - name: Checkout PR ${{ github.event.action.pull_request.number }}
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.head_ref }}
+          path: pr
+
+      - name: Checkout Tests
+        uses: actions/checkout@v4
+        with:
+          ref: main
+          path: pytest
+
+      - name: Setup Python 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.11
+          cache: pip
+
+      - name: Install requirements.txt
+        working-directory: ./pytest
+        run: pip install -r ./test/requirements.txt
+
+      - name: Get Final List of Markers
+        id: pytest-markers
+        run: |
+          if [[ ${{ inputs.qa-pytest-add-model-markers }} == "true" ]]; then
+            echo "markers=${{ inputs.qa-pytest-markers }} or access_om2" >> $GITHUB_OUTPUT
+          else
+            echo "markers=${{ inputs.qa-pytest-markers }}" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Invoke Simple CI Pytests
+        # We continue on error because we will let the checks generated in
+        # the next step speak to the state of the testing
+        continue-on-error: true
+        working-directory: ./pr
+        run: |
+          echo "Running pytest using '-m ${{ steps.pytest-markers.outputs.markers }}'"
+          pytest ../pytest/test \
+            -m '${{ steps.pytest-markers.outputs.markers }}' \
+            --target-branch '${{ github.base_ref }}' \
+            --junitxml=./test_report.xml
+
+      - name: Parse Test Report
+        id: tests
+        uses: EnricoMi/publish-unit-test-result-action/composite@e780361cd1fc1b1a170624547b3ffda64787d365  #v2.12.0
+        with:
+          files: ./pr/test_report.xml
+          comment_mode: off
+          check_run: true
+          check_name: QA Test Results
+          compare_to_earlier_commit: false
+          report_individual_runs: true
+          report_suite_logs: any
+
   repro-ci:
-    # run the given config on the deployment GitHub Environment (`environment-name`) and
+    # Run the given config on the deployment GitHub Environment (`environment-name`) and
     # upload the checksums and test details
     needs:
       - commit-check
       - branch-check
-    if: needs.commit-check.outputs.authorship != 'github-actions' && needs.branch-check.result == 'success'
+    if: needs.commit-check.outputs.authorship != vars.GH_ACTIONS_BOT_GIT_USER_NAME && needs.branch-check.result == 'success'
     uses: access-nri/reproducibility/.github/workflows/checks.yml@main
     with:
       model-name: access-om2
       environment-name: Gadi
       config-tag: ${{ github.head_ref }}
-      test-markers: checksum
+      test-markers: ${{ inputs.repro-pytest-markers }}
     secrets: inherit
     permissions:
       contents: write
@@ -138,6 +209,7 @@ jobs:
           files: ${{ env.TESTING_LOCAL_LOCATION }}/checksum/test_report.xml
           comment_mode: off
           check_run: true
+          check_name: Repro Test Results
           compare_to_earlier_commit: false
           report_individual_runs: true
           report_suite_logs: any
@@ -153,6 +225,38 @@ jobs:
             echo "result=pass" >> $GITHUB_OUTPUT
           fi
 
+  bump-check:
+    name: Version Bump Check
+    # Check that the `.version` in the metadata.yaml has been modified in
+    # this PR.
+    needs:
+      - repro-ci
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout PR Target
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.base_ref }}
+          path: target
+
+      - name: Checkout PR Source
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.head_ref }}
+          path: source
+
+      - name: Modification Check
+        run: |
+          target=$(yq e '.version' ./target/metadata.yaml)
+          source=$(yq e '.version' ./source/metadata.yaml)
+
+          if [[ "${source}" != "${target}" && "${source}" != "null" ]]; then
+            echo "::notice::The version has been modified to ${target}. Merging is now availible"
+          else
+            echo "::error::The version has not been modified in this PR. Merging is disallowed until an appropriate '!bump' is issued"
+            exit 1
+          fi
+
   result:
     name: Repro Result Notifier
     # Notify the PR of the result of the Repro check
@@ -171,35 +275,19 @@ jobs:
           ref: ${{ github.head_ref }}
 
       - name: Successful Release Comment
-        if: needs.check-checksum.outputs.result == 'pass' && startsWith(github.base_ref, 'release-')
+        if: needs.check-checksum.outputs.result == 'pass'
         env:
           BODY: |
             :white_check_mark: The Bitwise Reproducibility check succeeded when comparing against `${{ needs.check-checksum.outputs.compared-checksum-version }}` for this `Release` config. :white_check_mark:
             For further information, the experiment can be found on Gadi at ${{ needs.repro-ci.outputs.experiment-location }}, and the test results at ${{ needs.check-checksum.outputs.check-run-url }}.
-            Consider bumping the minor version of `access-om2-configs` - to bump the version, comment `!bump minor`. The meaning of these version bumps is explained in the README.md, under `Config Tags`.
-        run: gh pr comment --body '${{ env.BODY }}'
-
-      - name: Successful Dev Comment
-        if: needs.check-checksum.outputs.result == 'pass' && startsWith(github.base_ref, 'dev-')
-        env:
-          BODY: |
-            :white_check_mark: The Bitwise Reproducibility check succeeded when comparing against `${{ needs.check-checksum.outputs.compared-checksum-version }}` for this `Dev` config. :white_check_mark:
-            For further information, the experiment can be found on Gadi at ${{ needs.repro-ci.outputs.experiment-location }}, and the test results at ${{ needs.check-checksum.outputs.check-run-url }}.
+            You must bump the minor version of `access-om2-configs` - to bump the version, comment `!bump minor` or modify the `version` in `metadata.yaml`. The meaning of these version bumps is explained in the README.md, under `Config Tags`.
         run: gh pr comment --body '${{ env.BODY }}'
 
       - name: Failed Release Comment
-        if: needs.check-checksum.outputs.result == 'fail' && startsWith(github.base_ref, 'release-')
+        if: needs.check-checksum.outputs.result == 'fail'
         env:
           BODY: |
             :x: The Bitwise Reproducibility check failed when comparing against `${{ needs.check-checksum.outputs.compared-checksum-version }}` for this `Release` config. :x:
             For further information, the experiment can be found on Gadi at ${{ needs.repro-ci.outputs.experiment-location }}, and the test results at ${{ needs.check-checksum.outputs.check-run-url }}.
-            You must bump the major version of `access-om2-configs` before this PR is merged to account for this - to bump the version, comment `!bump major`. The meaning of these version bumps is explained in the README.md, under `Config Tags`.
-        run: gh pr comment --body '${{ env.BODY }}'
-
-      - name: Failed Dev Comment
-        if: needs.check-checksum.outputs.result == 'fail' && startsWith(github.base_ref, 'dev-')
-        env:
-          BODY: |
-            :warning: The Bitwise Reproducibility check failed when comparing against `${{ needs.check-checksum.outputs.compared-checksum-version }}` for this `Dev` config. :warning:
-            For further information, the experiment can be found on Gadi at ${{ needs.repro-ci.outputs.experiment-location }}, and the test results at ${{ needs.check-checksum.outputs.check-run-url }}.
+            You must bump the major version of `access-om2-configs` before this PR is merged to account for this - to bump the version, comment `!bump major`or modify the `version` in `metadata.yaml`. The meaning of these version bumps is explained in the README.md, under `Config Tags`.
         run: gh pr comment --body '${{ env.BODY }}'

.github/workflows/pr-2-confirm.yml

@@ -58,6 +58,12 @@ jobs:
         id: bump
         run: |
           version=$(yq '.version' metadata.yaml)
+
+          if [[ "${version}" == "null" ]]; then
+            echo "before=null" >> $GITHUB_OUTPUT
+            echo "after=1.0" >> $GITHUB_OUTPUT
+          fi
+
           regex="([0-9]+)\.([0-9]+)"
           if [[ $version =~ $regex ]]; then
             major_version="${BASH_REMATCH[1]}"
@@ -118,8 +124,8 @@ jobs:
 
       - name: Commit and Push Updates
         run: |
-          git config user.name github-actions
-          git config user.email github-actions@github.com
+          git config user.name ${{ vars.GH_ACTIONS_BOT_GIT_USER_NAME }}
+          git config user.email ${{ vars.GH_ACTIONS_BOT_GIT_USER_EMAIL }}
 
           if [[ "${{ needs.bump-version.outputs.type }}" == "minor" ]]; then
             git commit -am "Bumped version to ${{ needs.bump-version.outputs.after }} as part of ${{ env.RUN_URL }}"

.github/workflows/schedule-2-start.yml

@@ -80,7 +80,7 @@ jobs:
       MODEL_URL: https://github.com/ACCESS-NRI/access-om2
       CONFIG: access-om2-configs
       CONFIG_URL: https://github.com/ACCESS-NRI/access-om2-configs
-      TAG_URL: https://github.com/ACCESS-NRI/access-om2-configs/tags/${{ needs.check-checksum.outputs.compared-checksum-version }}
+      TAG_URL: https://github.com/ACCESS-NRI/access-om2-configs/releases/tag/${{ needs.check-checksum.outputs.compared-checksum-version }}
       RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
     steps:
       - name: Create issue

README-DEV.md

@@ -10,9 +10,9 @@ The Reproducibility CI is comprised of two main triggers: on Pull Request and Sc
 
 ### Triggered On Pull Request
 
-This pipeline uses the `pr-1-ci.yml` workflow for the majority of the PR lifecycle.
+This pipeline uses the `pr-1-ci.yml` workflow for the majority of the PR lifecycle. The `main` version of this workflow is called by `call-pr-1-ci.yml` workflow on the config branch.
 
-It also uses `pr-2-confirm.yml`, and `pr-3-bump-tag.yml` to control supplementary actions like bumping the `.version` field in the `metadata.yaml` file correctly and committing the checksums, and updating the config tag on merge.
+It also uses `pr-2-confirm.yml`, and `pr-3-bump-tag.yml` (via `call-pr-3-bump-tag.yml` on the config branch) to control supplementary actions like bumping the `.version` field in the `metadata.yaml` file correctly and committing the checksums, and updating the config tag on merge.
 
 The overall pipeline looks like this:
 
@@ -22,13 +22,28 @@ Merge: `pr-3-bump-tag.yml`
 #### The PR CI Lifecycle: `pr-1-ci.yml`
 
 This file does the bulk of the handling of the PR.
+It contains the following inputs, when the PR-triggered `call-pr-1-ci.yml` calls it:
+
+| Name | Type | Description | Required | Default | Example |
+| ---- | ---- | ----------- | -------- | ------- | ------- |
+| `qa-pytest-markers` | `string` | Markers used for the pytest QA CI checks, in the python format | `false` | `config or metadata` | `config or metadata or highres` |
+| `qa-pytest-add-model-markers` | `boolean` | Markers used for the pytest QA CI checks, in the python format | `false` | `false` | `true` |
+| `repro-pytest-markers` | `string` | Markers used for the pytest repro CI checks, in the python format | `false` | `checksum` | `checksum or performance` |
 
 ##### `commit-check`
 
 The first job, `commit-check`, is used to short-circuit execution of this workflow in the case where the latest commit was authored by `github-actions`, so there isn't a recursive cycle of checking and adding to the PR.
 
 `github-actions` authors commit to the `testing` directory and `metadata.yaml` files in the PR.
 
+##### `branch-check`
+
+This job is used as a check before running [`repro-ci`](#repro-ci) checks, which are only run on PRs `dev-*` -> `release-*`. It also makes sure that the branches are formatted correctly.
+
+##### `qa-ci`
+
+These checks are runner-hosted, quick configuration sanity checks that will always fire on PRs into `release-*` or `dev-*`.
+
 ##### `repro-ci`
 
 This job uses a reusable workflow that runs a given model on Gadi using a particular config tag (in this case, it'll be the source PR branch), and uploads the checksum as an artifact named `<model>-<config-tag>`, for example, `access-om2-release-config-fix`.