Merge branch 'develop' into jkan2/test-lstn

zeta-chain · Nov 8, 2024 · bc456ba · bc456ba
2 parents 94e1ab1 + 7c70809
commit bc456ba
Show file tree

Hide file tree

Showing 134 changed files with 2,647 additions and 2,702 deletions.
diff --git a/.coderabbit.yaml b/.coderabbit.yaml
@@ -33,6 +33,10 @@ reviews:
       instructions: >-
         Review the shell scripts, point out issues relative to security,
         performance, and maintainability.
+    - path: '{*e2e*,*_test.go}'
+      instructions: >-
+        Errors in tests may be handled via require.NoError(t, err) rather
+        than explicitly returning error.
 
   auto_review:
     drafts: false

diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
@@ -28,13 +28,90 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  # this cannot run on forks as forks cannot push packages in pull request context
+  # forked pull request will fall back to slow build
+  build-zetanode:
+    runs-on: ubuntu-22.04
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'zeta-chain/node'
+    env:
+      DOCKER_IMAGE: ghcr.io/${{ github.repository_owner }}/zetanode
+      DOCKER_TAG: ${{ github.ref == 'refs/heads/develop' && 'develop' || github.sha }}
+    outputs:
+      image: ${{ fromJson(steps.build.outputs.metadata)['image.name'] }}
+    steps:
+      - uses: actions/checkout@v4
+
+      # configure docker to use the containerd snapshotter
+      # so that we can use the buildkit cache
+      - uses: depot/use-containerd-snapshotter-action@v1
+
+      - name: Login to Docker Hub registry
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_READ_ONLY }}
+
+      - name: Login to github docker registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Restore go cache
+        uses: actions/cache@v4
+        id: restore-go-cache
+        with:
+          path: |
+            go-cache
+          key: cache-${{ hashFiles('go.sum') }}
+
+      - name: Inject go cache into docker
+        uses: reproducible-containers/buildkit-cache-dance@v3.1.2
+        with:
+          cache-map: |
+            {
+              "go-cache": "/root/.cache/go-build"
+            }
+          skip-extraction: ${{ steps.restore-go-cache.outputs.cache-hit || github.event_name != 'push' }}
+
+      # this ensures that the version is consistent between cache build and make build
+      - name: Set version for cache
+        run: |
+          NODE_VERSION=$(./version.sh)
+          echo "NODE_VERSION=$NODE_VERSION" >> $GITHUB_ENV
+          NODE_COMMIT=$(git log -1 --format='%H')
+          echo "NODE_COMMIT=$NODE_COMMIT" >> $GITHUB_ENV
+
+      # build zetanode with cache options
+      - name: Build zetanode for cache
+        id: build
+        uses: docker/build-push-action@v6
+        env:
+          CACHE_FROM_CONFIG: "type=registry,ref=ghcr.io/${{ github.repository }}:buildcache"
+          CACHE_TO_CONFIG: "type=registry,ref=ghcr.io/${{ github.repository }}:buildcache,mode=max"
+        with:
+          context: .
+          file: ./Dockerfile-localnet
+          push: true
+          tags: ${{ env.DOCKER_IMAGE }}:${{ env.DOCKER_TAG }}
+          cache-from: ${{ env.CACHE_FROM_CONFIG }}
+          cache-to: ${{ github.event_name == 'push' && env.CACHE_TO_CONFIG || '' }}
+          target: latest-runtime
+          build-args: |
+            NODE_VERSION=${{ env.NODE_VERSION }}
+            NODE_COMMIT=${{ env.NODE_COMMIT }}
+
   matrix-conditionals:
+    needs: build-zetanode
+    if: always()
     runs-on: ubuntu-22.04
     env:
       GH_TOKEN: ${{ github.token }}
     outputs:
       DEFAULT_TESTS: ${{ steps.matrix-conditionals.outputs.DEFAULT_TESTS }}
       UPGRADE_TESTS: ${{ steps.matrix-conditionals.outputs.UPGRADE_TESTS }}
+      CONSENSUS_TESTS: ${{ steps.matrix-conditionals.outputs.CONSENSUS_TESTS }}
       UPGRADE_LIGHT_TESTS: ${{ steps.matrix-conditionals.outputs.UPGRADE_LIGHT_TESTS }}
       UPGRADE_IMPORT_MAINNET_TESTS: ${{ steps.matrix-conditionals.outputs.UPGRADE_IMPORT_MAINNET_TESTS }}
       ADMIN_TESTS: ${{ steps.matrix-conditionals.outputs.ADMIN_TESTS }}
@@ -52,16 +129,22 @@ jobs:
         uses: actions/github-script@v7
         with:
           script: |
-            if (context.eventName === 'pull_request') {
+            const getPrLabels = async (pull_number) => {
               const { data: pr } = await github.rest.pulls.get({
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                pull_number: context.payload.pull_request.number,
-              });
-              const labels = pr.labels.map(label => label.name);
-              console.log("labels:", labels);
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  pull_number: pull_number,
+                });
+                const labels = pr.labels.map(label => label.name);
+                console.log(`labels for ${pull_number}:`, labels);
+                return labels;
+            }
+
+            if (context.eventName === 'pull_request') {
+              const labels = await getPrLabels(context.payload.pull_request.number);
               core.setOutput('DEFAULT_TESTS', true);
               core.setOutput('UPGRADE_TESTS', labels.includes('UPGRADE_TESTS'));
+              core.setOutput('CONSENSUS_TESTS', labels.includes('CONSENSUS_TESTS'));
               core.setOutput('UPGRADE_LIGHT_TESTS', labels.includes('UPGRADE_LIGHT_TESTS'));
               core.setOutput('UPGRADE_IMPORT_MAINNET_TESTS', labels.includes('UPGRADE_IMPORT_MAINNET_TESTS'));
               core.setOutput('ADMIN_TESTS', labels.includes('ADMIN_TESTS'));
@@ -74,8 +157,20 @@ jobs:
               core.setOutput('V2_MIGRATION_TESTS', labels.includes('V2_MIGRATION_TESTS')); // for v2 tests, TODO: remove this once we fully migrate to v2 (https://github.com/zeta-chain/node/issues/2627)
               core.setOutput('ENABLE_MONITORING', labels.includes('ENABLE_MONITORING'));
             } else if (context.eventName === 'merge_group') {
+              // default mergequeue tests
               core.setOutput('DEFAULT_TESTS', true);
               core.setOutput('UPGRADE_LIGHT_TESTS', true);
+
+              // conditional tests based on PR labels
+              const commit_message = context.payload.merge_group.head_commit.message;
+              const pr_match = commit_message.split('\n')[0].match(/\(#(\d+)\)$/);
+              if (!pr_match) {
+                console.error("unable to extract PR number from mergequeue commit message");
+                return;
+              }
+              const pr_number = pr_match[1];
+              const pr_labels = await getPrLabels(pr_number);
+              core.setOutput('CONSENSUS_TESTS', !pr_labels.includes('CONSENSUS_BREAKING_ACK'));
             } else if (context.eventName === 'push' && context.ref === 'refs/heads/develop') {
               core.setOutput('DEFAULT_TESTS', true);
             } else if (context.eventName === 'push' && context.ref.startsWith('refs/heads/release/')) {
@@ -119,14 +214,20 @@ jobs:
             }
 
   e2e:
-    needs: matrix-conditionals
+    needs:
+      - build-zetanode
+      - matrix-conditionals
+    if: always()
     strategy:
       fail-fast: false
       matrix:
         include:
           - make-target: "start-e2e-test"
             runs-on: ubuntu-20.04
             run: ${{ needs.matrix-conditionals.outputs.DEFAULT_TESTS == 'true' }}
+          - make-target: "start-e2e-consensus-test"
+            runs-on: ubuntu-20.04
+            run: ${{ needs.matrix-conditionals.outputs.CONSENSUS_TESTS == 'true' }}
           - make-target: "start-upgrade-test"
             runs-on: ubuntu-20.04
             run: ${{ needs.matrix-conditionals.outputs.UPGRADE_TESTS == 'true' }}
@@ -170,12 +271,14 @@ jobs:
       runs-on: ${{ matrix.runs-on}}
       run: ${{ matrix.run }}
       timeout-minutes: "${{ matrix.timeout-minutes || 25 }}"
+      zetanode-image: ${{ needs.build-zetanode.outputs.image }}
       enable-monitoring: ${{ needs.matrix-conditionals.outputs.ENABLE_MONITORING == 'true' }}
     secrets: inherit
   # this allows you to set a required status check
   e2e-ok:
     runs-on: ubuntu-22.04
     needs:
+      - build-zetanode
       - matrix-conditionals
       - e2e
     if: always()
@@ -224,6 +327,10 @@ jobs:
 
 
       - run: |
+          result="${{ needs.build-zetanode.result }}"
+          if [[ $result == "failed" ]]; then
+            exit 1
+          fi
           result="${{ needs.e2e.result }}"
           if [[ $result == "success" || $result == "skipped" ]]; then
             exit 0

diff --git a/.github/workflows/reusable-e2e.yml b/.github/workflows/reusable-e2e.yml
@@ -19,6 +19,10 @@ on:
         required: true
         type: string
         default: 'ubuntu-20.04'
+      zetanode-image:
+        description: 'docker image to use for zetanode'
+        required: true
+        type: string
       enable-monitoring:
         description: 'Enable the monitoring stack for this run'
         type: boolean
@@ -31,12 +35,10 @@ jobs:
     timeout-minutes: ${{ inputs.timeout-minutes }}
     strategy:
       fail-fast: false
+    env:
+      ZETANODE_IMAGE: ${{ inputs.zetanode-image }}
     steps:
       - uses: actions/checkout@v4
-
-      # configure docker to use the containerd snapshotter
-      # so that we can use the buildkit cache
-      - uses: depot/use-containerd-snapshotter-action@v1
 
       - name: Login to Docker Hub registry
         uses: docker/login-action@v3
@@ -51,50 +53,7 @@ jobs:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Restore go cache
-        uses: actions/cache@v4
-        id: restore-go-cache
-        with:
-          path: |
-            go-cache
-          key: cache-${{ hashFiles('go.sum') }}
-
-      - name: Inject go cache into docker
-        uses: reproducible-containers/buildkit-cache-dance@v3.1.2
-        with:
-          cache-map: |
-            {
-              "go-cache": "/root/.cache/go-build"
-            }
-          skip-extraction: ${{ steps.restore-go-cache.outputs.cache-hit || github.event_name != 'push' }}
-
-      # this ensures that the version is consistent between cache build and make build
-      - name: Set version for cache
-        run: |
-          NODE_VERSION=$(./version.sh)
-          echo "NODE_VERSION=$NODE_VERSION" >> $GITHUB_ENV
-          NODE_COMMIT=$(git log -1 --format='%H')
-          echo "NODE_COMMIT=$NODE_COMMIT" >> $GITHUB_ENV
-
-      # build zetanode with cache options
-      - name: Build zetanode for cache
-        uses: docker/build-push-action@v6
-        env:
-          CACHE_FROM_CONFIG: "type=registry,ref=ghcr.io/${{ github.repository }}:buildcache"
-          CACHE_TO_CONFIG: "type=registry,ref=ghcr.io/${{ github.repository }}:buildcache,mode=max"
-        with:
-          context: .
-          file: ./Dockerfile-localnet
-          push: false
-          tags: zetanode:latest
-          cache-from: ${{ env.CACHE_FROM_CONFIG }}
-          cache-to: ${{ github.event_name == 'push' && env.CACHE_TO_CONFIG || '' }}
-          target: latest-runtime
-          build-args: |
-            NODE_VERSION=${{ env.NODE_VERSION }}
-            NODE_COMMIT=${{ env.NODE_COMMIT }}
-
+
       - name: Enable monitoring
         if: inputs.enable-monitoring
         run: |

diff --git a/Dockerfile-localnet b/Dockerfile-localnet
@@ -67,7 +67,8 @@ COPY --from=latest-build /go/bin/zetacored /go/bin/zetaclientd /go/bin/zetaclien
 
 # Optional old version build (from source). This old build is used as the genesis version in the upgrade tests. 
 # Use --target latest-runtime to skip.
-FROM base-build AS old-build-source
+# you must have already built the latest image (which the Makefile does)
+FROM zetanode:latest AS old-build-source
 
 ARG OLD_VERSION
 RUN git clone https://github.com/zeta-chain/node.git
@@ -84,13 +85,12 @@ COPY --from=latest-build /go/bin/zetaclientd-supervisor /usr/local/bin
 
 # Optional old version build (from binary).
 # Use --target latest-runtime to skip.
-FROM base-runtime AS old-runtime
+# you must have already built the latest image (which the Makefile does)
+FROM zetanode:latest AS old-runtime
 
 ARG OLD_VERSION
 ARG BUILDARCH
 
-COPY --from=cosmovisor-build /go/bin/cosmovisor /usr/local/bin
-COPY --from=latest-build /go/bin/zetaclientd-supervisor /usr/local/bin
 RUN curl -Lo /usr/local/bin/zetacored ${OLD_VERSION}/zetacored-linux-${BUILDARCH} && \
     chmod 755 /usr/local/bin/zetacored && \
     curl -Lo /usr/local/bin/zetaclientd ${OLD_VERSION}/zetaclientd-linux-${BUILDARCH} && \