From cb173e27119620bd08ea1c9799e0b24f251568fa Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 25 Oct 2024 09:28:08 -0400 Subject: [PATCH 1/9] Update action.yml --- .github/ensure-tester/action.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/ensure-tester/action.yml b/.github/ensure-tester/action.yml index ca85b7da853..ec5a7b61a11 100644 --- a/.github/ensure-tester/action.yml +++ b/.github/ensure-tester/action.yml @@ -8,11 +8,12 @@ inputs: run: # command to run once tester available required: false - ttl: required: false description: "Time to live for the tester instance in minutes" default: 30 + spot_strategy: + default: BestEffort runs: # define an action, runs in OS of caller using: composite @@ -23,7 +24,7 @@ runs: run: | TYPE=${{ inputs.runner_type }} # Try to use spot for every runner type for now - echo "spot_strategy=BestEffort" >> $GITHUB_OUTPUT + echo "spot_strategy=${{ inputs.spot_strategy }}" >> $GITHUB_OUTPUT echo "runner_label=$USERNAME-$runner_type" >> $GITHUB_OUTPUT echo "ami_id=ami-04d8422a9ba4de80f" >> $GITHUB_OUTPUT # no github runners, 'bare spot' in action code From 760fe9c6a2a26218e737e68ff305ac910cd6de41 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 25 Oct 2024 09:29:06 -0400 Subject: [PATCH 2/9] Update action.yml --- .github/ensure-tester-with-images/action.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/ensure-tester-with-images/action.yml b/.github/ensure-tester-with-images/action.yml index b9e1ceddb89..307473513c8 100644 --- a/.github/ensure-tester-with-images/action.yml +++ b/.github/ensure-tester-with-images/action.yml @@ -5,6 +5,8 @@ description: "Reusable setup steps" inputs: runner_type: required: true + spot_strategy: + default: BestEffort builder_type: required: true builder_images_to_copy: @@ -59,6 +61,7 @@ runs: if: ${{ env.CACHE_SUCCESS != 'true' }} with: runner_type: ${{ inputs.runner_type}} + spot_strategy: ${{ inputs.spot_strategy }} ttl: ${{ inputs.tester_ttl }} run: | set -eux From 82e9c56ad43accc3bab884cb9f181ca59594f155 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 25 Oct 2024 11:05:15 -0400 Subject: [PATCH 3/9] Update action.yml From da2fbb71eb7adf01b53bb2fa4baaa792d7cc522e Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 25 Oct 2024 12:44:47 -0400 Subject: [PATCH 4/9] Allow 4epochs to fail true --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ba1846d355a..ed1bc5253e9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -673,7 +673,9 @@ jobs: cd yarn-project/end-to-end echo ${{ secrets.DOCKERHUB_PASSWORD }} | docker login -u aztecprotocolci --password-stdin test=${{ matrix.test }} - NAMESPACE="${test%.test.ts}" FRESH_INSTALL=true VALUES_FILE=${values_file:-default.yaml} ./scripts/network_test.sh ./src/spartan/$test + fallback_result=false + [ $test = 4epochs.test.ts ] && fallback_result=true + NAMESPACE="${test%.test.ts}" FRESH_INSTALL=true VALUES_FILE=${values_file:-default.yaml} ./scripts/network_test.sh ./src/spartan/$test || $fallback_result l1-contracts-test: needs: [build, configure] From 0e2ba2ab1532f0865c2b22ca6c4836dfca272fb6 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 25 Oct 2024 13:17:56 -0400 Subject: [PATCH 5/9] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ed1bc5253e9..da7ed936eed 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -660,7 +660,7 @@ jobs: env: USERNAME: ${{ needs.configure.outputs.username }} with: - runner_type: 16core-tester-x86 + runner_type: 8core-tester-x86 spot_strategy: None # use on-demand machines builder_type: builder-x86 # these are copied to the tester and expected by the earthly command below From 36c0d1c29fb5402caf89bbf724f43321606b1db2 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 25 Oct 2024 18:09:33 +0000 Subject: [PATCH 6/9] try diff kill patern --- .github/workflows/ci.yml | 4 +--- yarn-project/end-to-end/scripts/network_test.sh | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index da7ed936eed..7134e49cbe3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -673,9 +673,7 @@ jobs: cd yarn-project/end-to-end echo ${{ secrets.DOCKERHUB_PASSWORD }} | docker login -u aztecprotocolci --password-stdin test=${{ matrix.test }} - fallback_result=false - [ $test = 4epochs.test.ts ] && fallback_result=true - NAMESPACE="${test%.test.ts}" FRESH_INSTALL=true VALUES_FILE=${values_file:-default.yaml} ./scripts/network_test.sh ./src/spartan/$test || $fallback_result + NAMESPACE="${test%.test.ts}" FRESH_INSTALL=true VALUES_FILE=${values_file:-default.yaml} ./scripts/network_test.sh ./src/spartan/$test l1-contracts-test: needs: [build, configure] diff --git a/yarn-project/end-to-end/scripts/network_test.sh b/yarn-project/end-to-end/scripts/network_test.sh index b2a83d4b8ef..bcedb8d238c 100755 --- a/yarn-project/end-to-end/scripts/network_test.sh +++ b/yarn-project/end-to-end/scripts/network_test.sh @@ -65,9 +65,9 @@ show_status_until_pxe_ready & function cleanup() { # kill everything in our process group except our process - trap - SIGTERM && kill $(pgrep -g $$ | grep -v $$) $(jobs -p) &>/dev/null || true + kill $(pgrep -g $$ | grep -v $$) $(jobs -p) &>/dev/null || true } -trap cleanup SIGINT SIGTERM EXIT +trap cleanup EXIT # Install the Helm chart helm upgrade --install spartan "$REPO/spartan/aztec-network/" \ --namespace "$NAMESPACE" \ From 2121b90a57a1f171eb61521680ea4af338f5025a Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 25 Oct 2024 18:54:58 +0000 Subject: [PATCH 7/9] try another approach --- .github/workflows/ci.yml | 2 +- yarn-project/end-to-end/scripts/network_test.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7134e49cbe3..b4de019d4e0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -668,8 +668,8 @@ jobs: builder_images_to_copy: aztecprotocol/aztec:${{ env.GIT_COMMIT }} aztecprotocol/end-to-end:${{ env.GIT_COMMIT }} # command to produce the images in case they don't exist builder_command: scripts/earthly-ci ./yarn-project+export-e2e-test-images + tester_ttl: 60 run: | - sudo shutdown -P 60 # extend ttl cd yarn-project/end-to-end echo ${{ secrets.DOCKERHUB_PASSWORD }} | docker login -u aztecprotocolci --password-stdin test=${{ matrix.test }} diff --git a/yarn-project/end-to-end/scripts/network_test.sh b/yarn-project/end-to-end/scripts/network_test.sh index bcedb8d238c..b2a83d4b8ef 100755 --- a/yarn-project/end-to-end/scripts/network_test.sh +++ b/yarn-project/end-to-end/scripts/network_test.sh @@ -65,9 +65,9 @@ show_status_until_pxe_ready & function cleanup() { # kill everything in our process group except our process - kill $(pgrep -g $$ | grep -v $$) $(jobs -p) &>/dev/null || true + trap - SIGTERM && kill $(pgrep -g $$ | grep -v $$) $(jobs -p) &>/dev/null || true } -trap cleanup EXIT +trap cleanup SIGINT SIGTERM EXIT # Install the Helm chart helm upgrade --install spartan "$REPO/spartan/aztec-network/" \ --namespace "$NAMESPACE" \ From ae4170236d1a4f94cebd15449895712c0e3c21c3 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 25 Oct 2024 18:56:38 +0000 Subject: [PATCH 8/9] size fix --- .github/ensure-tester/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ensure-tester/action.yml b/.github/ensure-tester/action.yml index ec5a7b61a11..4525f2df30a 100644 --- a/.github/ensure-tester/action.yml +++ b/.github/ensure-tester/action.yml @@ -32,7 +32,7 @@ runs: echo "ttl=${{ inputs.ttl }}" >> $GITHUB_OUTPUT SIZE=large if [[ $TYPE == 4core-* ]]; then - SIZE=large + SIZE=xlarge elif [[ $TYPE == 8core-* ]]; then SIZE=2xlarge elif [[ $TYPE == 16core-* ]]; then From 3f62f5f930ae8509cbb7972d0275c20ce56d9b9b Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 25 Oct 2024 15:48:39 -0400 Subject: [PATCH 9/9] Update ci.yml --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b4de019d4e0..b1bc3fe17d8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -634,10 +634,10 @@ jobs: with: concurrency_key: network-test-${{ matrix.test }} - name: "Setup and Local Network Tests" - timeout-minutes: 40 + timeout-minutes: 60 # Only allow transfer test to run on every (non-network-all) PR if: matrix.test == 'test-transfer.sh' || github.ref_name == 'master' || contains(github.event.pull_request.labels.*.name, 'network-all') - run: earthly-ci --no-output ./yarn-project/+network-test --test=./${{ matrix.test }} + run: sudo shutdown -P 60 ; earthly-ci --no-output ./yarn-project/+network-test --test=./${{ matrix.test }} # note: proving disabled kind-network-test: @@ -646,7 +646,7 @@ jobs: strategy: fail-fast: false matrix: - test: [smoke.test.ts, transfer.test.ts, 4epochs.test.ts] + test: [smoke.test.ts, transfer.test.ts] # TODO reinstate: 4epochs.test.ts steps: - uses: actions/checkout@v4 with: { ref: "${{ env.GIT_COMMIT }}" }