Merge branch 'master' into sync-noir

* master: (62 commits)
  chore: dont show aws creds in docker fast  (#9465)
  fix: make sure kind tests run every master commit (#9478)
  feat: simulate latency with network chaos (#9469)
  feat: fixed number of pub inputs for databus commitment propagation (#9336)
  chore(CI): remove end-to-end/Earthfile (#9364)
  chore!: use Brillig opcode when possible for less-than operations on fields (#9416)
  fix: remove unnecessary ivpk's from aztec-nr (#9460)
  feat: Sync from noir (#9425)
  chore!: split base rollup and remove public kernel proving (#9434)
  refactor: clean up note processor after changes due to address (#9401)
  feat(avm): merkle tree gadget (#9205)
  fix: docker fast (#9467)
  chore(avm:): Fix execution tests in proving mode (#9466)
  chore: remove unnecessary trait (#9437)
  fix: yarn build:fast (#9464)
  git subrepo push --branch=master noir-projects/aztec-nr
  git_subrepo.sh: Fix parent in .gitrepo file. [skip ci]
  chore: replace relative paths to noir-protocol-circuits
  git subrepo push --branch=master barretenberg
  fix: cleanup of janky encryption apis (#9390)
  ...

.github/ci-setup-action/action.yml

@@ -35,6 +35,10 @@ runs:
     # TODO reconsider how jq gets into image
     - name: Setup jq
       uses: dcarbone/install-jq-action@v2.1.0
+    - name: Setup yq
+      uses: dcarbone/install-yq-action@v1.1.1
+      with:
+        version: v4.44.3
 
     - name: Setup Earthly
       uses: earthly/actions-setup@v1

.github/ensure-tester-with-images/action.yml

@@ -5,6 +5,8 @@ description: "Reusable setup steps"
 inputs:
   runner_type:
     required: true
+  spot_strategy:
+    default: BestEffort
   builder_type:
     required: true
   builder_images_to_copy:
@@ -59,6 +61,7 @@ runs:
       if: ${{ env.CACHE_SUCCESS != 'true' }}
       with:
         runner_type: ${{ inputs.runner_type}}
+        spot_strategy: ${{ inputs.spot_strategy }}
         ttl: ${{ inputs.tester_ttl }}
         run: |
           set -eux

.github/ensure-tester/action.yml

@@ -8,11 +8,12 @@ inputs:
   run:
     # command to run once tester available
     required: false
-
   ttl:
     required: false
     description: "Time to live for the tester instance in minutes"
     default: 30
+  spot_strategy:
+    default: BestEffort
 runs:
   # define an action, runs in OS of caller
   using: composite
@@ -23,15 +24,15 @@ runs:
       run: |
         TYPE=${{ inputs.runner_type }}
         # Try to use spot for every runner type for now
-        echo "spot_strategy=BestEffort" >> $GITHUB_OUTPUT
+        echo "spot_strategy=${{ inputs.spot_strategy }}" >> $GITHUB_OUTPUT
         echo "runner_label=$USERNAME-$runner_type" >> $GITHUB_OUTPUT
         echo "ami_id=ami-04d8422a9ba4de80f" >> $GITHUB_OUTPUT
         # no github runners, 'bare spot' in action code
         echo "runner_concurrency=0" >> $GITHUB_OUTPUT
         echo "ttl=${{ inputs.ttl }}" >> $GITHUB_OUTPUT
         SIZE=large
         if [[ $TYPE == 4core-* ]]; then
-          SIZE=large
+          SIZE=xlarge
         elif [[ $TYPE == 8core-* ]]; then
           SIZE=2xlarge
         elif [[ $TYPE == 16core-* ]]; then
@@ -94,6 +95,16 @@ runs:
           " || true
         fi
 
+    - name: Setup yq
+      shell: bash
+      run: |
+        set -x
+        scripts/run_on_tester "
+          wget https://github.com/mikefarah/yq/releases/download/v4.44.3/yq_linux_amd64 -O ./yq
+          chmod +x ./yq
+          sudo mv ./yq /usr/bin/yq
+        "
+
     - name: Ensure Tester Cleanup
       uses: gacts/run-and-post-run@v1
       with:

.github/workflows/ci-arm.yml

@@ -3,7 +3,7 @@ on:
   push:
     branches:
       - master
-      - '*/*arm-build'
+      - "*/*arm-build"
   workflow_dispatch:
     inputs: {}
 concurrency:
@@ -101,8 +101,9 @@ jobs:
       # prepare images locally, tagged by commit hash
       - name: "Build E2E Image"
         timeout-minutes: 40
+        working-directory: ./yarn-project/end-to-end
         run: |
-          earthly-ci ./yarn-project/end-to-end+uniswap-trade-on-l1-from-l2
+          ./scripts/e2e_compose_test.sh uniswap_trade_on_l1_from_l2.test.ts
 
   rerun-check:
     runs-on: ubuntu-20.04

.github/workflows/ci.yml

@@ -4,7 +4,7 @@ on:
     branches:
       - master
       - provernet
-      - '*/ci-push*'
+      - "*/ci-push*"
   pull_request:
     types:
       - opened
@@ -51,6 +51,7 @@ jobs:
       barretenberg-cpp: ${{ steps.filter.outputs.barretenberg-cpp }}
       noir: ${{ steps.filter.outputs.noir }}
       noir-projects: ${{ steps.filter.outputs.noir-projects }}
+      yarn-project: ${{ steps.filter.outputs.yarn-project }}
       txe: ${{ steps.filter.outputs.txe }}
       l1-contracts: ${{ steps.filter.outputs.l1-contracts }}
       non-docs: ${{ steps.filter.outputs.non-docs }}
@@ -95,6 +96,8 @@ jobs:
               - 'l1-contracts/**'
             noir-projects:
               - 'noir-projects/**'
+            yarn-project:
+              - 'yarn-project/**'
             txe:
               - 'yarn-project/txe/**'
             non-barretenberg-cpp:
@@ -155,7 +158,7 @@ jobs:
       # prepare images locally, tagged by commit hash
       - name: "Build E2E Image"
         timeout-minutes: 40
-        if: (needs.configure.outputs.non-docs == 'true' && needs.configure.outputs.non-misc-ci == 'true' && needs.configure.outputs.non-barretenberg-cpp == 'true') || github.ref_name == 'master'
+        if: (needs.configure.outputs.non-docs == 'true' && needs.configure.outputs.non-barretenberg-cpp == 'true') || github.ref_name == 'master'
         run: |
           earthly-ci ./yarn-project+export-e2e-test-images
       # We base our e2e list used in e2e-x86 off the targets in ./yarn-project/end-to-end
@@ -169,6 +172,7 @@ jobs:
             LABELS=""
           fi
           echo Labels: $LABELS
+          echo "list=$(./scripts/ci/get_e2e_jobs.sh ${{github.ref_name}} "$LABELS")"
           echo "list=$(./scripts/ci/get_e2e_jobs.sh ${{github.ref_name}} "$LABELS")" >> $GITHUB_OUTPUT
       - name: Create list of bench end-to-end jobs
         id: bench_list
@@ -184,7 +188,7 @@ jobs:
   # all the non-bench end-to-end integration tests for aztec
   e2e:
     needs: [build, configure]
-    if: (needs.configure.outputs.non-docs == 'true' && needs.configure.outputs.non-misc-ci == 'true' && needs.configure.outputs.non-barretenberg-cpp == 'true') || github.ref_name == 'master' || contains(github.event.pull_request.labels.*.name, 'e2e')
+    if: (needs.configure.outputs.non-docs == 'true' && needs.configure.outputs.non-barretenberg-cpp == 'true') || github.ref_name == 'master' || contains(github.event.pull_request.labels.*.name, 'e2e')
     runs-on: ubuntu-20.04
     strategy:
       fail-fast: false
@@ -194,29 +198,31 @@ jobs:
       - uses: actions/checkout@v4
         with: { ref: "${{ env.GIT_COMMIT }}" }
       - uses: ./.github/ci-setup-action
+      - name: Determine runner type
+        id: runner_type
+        run: |
+          runner=$(./scripts/ci/select_runner.sh ${{ matrix.test }})
+          echo "type=$runner" >> $GITHUB_OUTPUT
       - name: Setup and Test
         timeout-minutes: 40
         uses: ./.github/ensure-tester-with-images
         env:
           USERNAME: ${{ needs.configure.outputs.username }}
         with:
-          runner_type: ${{ contains(matrix.test, 'prover') && '64core-tester-x86' || '8core-tester-x86' }}
+          runner_type: ${{ steps.runner_type.outputs.type }}
           builder_type: builder-x86
-          # these are copied to the tester and expected by the earthly command below
-          # if they fail to copy, it will try to build them on the tester and fail
           builder_images_to_copy: aztecprotocol/aztec:${{ env.GIT_COMMIT }} aztecprotocol/end-to-end:${{ env.GIT_COMMIT }}
-          # command to produce the images in case they don't exist
           builder_command: scripts/earthly-ci ./yarn-project+export-e2e-test-images
           run: |
             set -eux
             cd ./yarn-project/end-to-end/
             export FORCE_COLOR=1
-            ../../scripts/earthly-ci -P --no-output +${{ matrix.test }}
+            ./scripts/e2e_test.sh ${{ matrix.test }}
 
   # all the benchmarking end-to-end integration tests for aztec (not required to merge)
   bench-e2e:
     needs: [build, configure]
-    if: needs.build.outputs.bench_list != '[]' && ((needs.configure.outputs.non-docs == 'true' && needs.configure.outputs.non-misc-ci == 'true' && needs.configure.outputs.non-barretenberg-cpp == 'true') || github.ref_name == 'master' || contains(github.event.pull_request.labels.*.name, 'bench'))
+    if: needs.build.outputs.bench_list != '[]' && ((needs.configure.outputs.non-docs == 'true' && needs.configure.outputs.non-barretenberg-cpp == 'true') || github.ref_name == 'master' || contains(github.event.pull_request.labels.*.name, 'bench'))
     runs-on: ubuntu-20.04
     strategy:
       fail-fast: false
@@ -226,13 +232,18 @@ jobs:
       - uses: actions/checkout@v4
         with: { ref: "${{ env.GIT_COMMIT }}" }
       - uses: ./.github/ci-setup-action
+      - name: Determine runner type
+        id: runner_type
+        run: |
+          runner=$(./scripts/ci/select_runner.sh ${{ matrix.test }})
+          echo "type=$runner" >> $GITHUB_OUTPUT
       - name: Setup and Test
         uses: ./.github/ensure-tester-with-images
         env:
           USERNAME: ${{ needs.configure.outputs.username }}
         timeout-minutes: 45
         with:
-          runner_type: ${{ contains(matrix.test, 'prover') && '64core-tester-x86' || '16core-tester-x86' }}
+          runner_type: ${{ steps.runner_type.outputs.type }}
           builder_type: builder-x86
           # these are copied to the tester and expected by the earthly command below
           # if they fail to copy, it will try to build them on the tester and fail
@@ -245,7 +256,9 @@ jobs:
             cd ./yarn-project/end-to-end/
             export FORCE_COLOR=1
             export EARTHLY_BUILD_ARGS="${{ env.EARTHLY_BUILD_ARGS }}"
-            ../../scripts/earthly-ci --exec-stats -P --no-output +${{ matrix.test }}
+            ./scripts/e2e_test.sh ${{ matrix.test }}
+
+            earthly-ci +UPLOAD_LOGS --PULL_REQUEST=${{ github.event.pull_request.number }} --BRANCH=${{ github.ref_name }} --COMMIT_HASH=${{ env.GIT_COMMIT }}
 
   acir-bench:
     runs-on: ubuntu-20.04
@@ -525,7 +538,7 @@ jobs:
   noir-examples:
     needs: [build, configure]
     runs-on: ${{ needs.configure.outputs.username }}-x86
-    if: needs.configure.outputs.barretenberg == 'true' || needs.configure.outputs.noir == 'true'
+    if: needs.configure.outputs.noir == 'true'
     steps:
       - uses: actions/checkout@v4
         with: { ref: "${{ env.GIT_COMMIT }}" }
@@ -552,7 +565,7 @@ jobs:
   noir-projects:
     needs: [build, configure]
     runs-on: ${{ needs.configure.outputs.username }}-x86
-    if: needs.configure.outputs.barretenberg == 'true' || needs.configure.outputs.noir == 'true' || needs.configure.outputs.noir-projects == 'true' || needs.configure.outputs.txe == 'true'
+    if: needs.configure.outputs.noir == 'true' || needs.configure.outputs.noir-projects == 'true' || needs.configure.outputs.txe == 'true'
     steps:
       - uses: actions/checkout@v4
         with: { ref: "${{ env.GIT_COMMIT }}" }
@@ -581,6 +594,7 @@ jobs:
 
   yarn-project-formatting:
     needs: [build, configure]
+    if: needs.configure.outputs.yarn-project == 'true' || github.ref_name == 'master'
     runs-on: ${{ needs.configure.outputs.username }}-x86
     steps:
       - uses: actions/checkout@v4
@@ -594,6 +608,7 @@ jobs:
 
   yarn-project-test:
     needs: [build, configure]
+    if: needs.configure.outputs.yarn-project == 'true' || github.ref_name == 'master'
     runs-on: ${{ needs.configure.outputs.username }}-x86
     steps:
       - uses: actions/checkout@v4
@@ -608,6 +623,7 @@ jobs:
   prover-client-test:
     needs: [build, configure]
     runs-on: ${{ needs.configure.outputs.username }}-x86
+    if: needs.configure.outputs.yarn-project == 'true' || github.ref_name == 'master'
     steps:
       - uses: actions/checkout@v4
         with: { ref: "${{ env.GIT_COMMIT }}" }
@@ -621,8 +637,10 @@ jobs:
   # proving disabled
   network-test:
     needs: [build, configure]
+    if: needs.configure.outputs.yarn-project == 'true' || github.ref_name == 'master'
     runs-on: ${{ needs.configure.outputs.username }}-x86
     strategy:
+      max-parallel: 1
       fail-fast: false
       matrix:
         test: [test-transfer.sh, test-4epochs.sh]
@@ -633,19 +651,20 @@ jobs:
         with:
           concurrency_key: network-test-${{ matrix.test }}
       - name: "Setup and Local Network Tests"
-        timeout-minutes: 40
+        timeout-minutes: 60
         # Only allow transfer test to run on every (non-network-all) PR
         if: matrix.test == 'test-transfer.sh' || github.ref_name == 'master' || contains(github.event.pull_request.labels.*.name, 'network-all')
-        run: earthly-ci --no-output ./yarn-project/+network-test --test=./${{ matrix.test }}
+        run: sudo shutdown -P 60 ; earthly-ci --no-output ./yarn-project/+network-test --test=./${{ matrix.test }}
 
   # note: proving disabled
   kind-network-test:
     needs: [build, configure]
+    if: needs.configure.outputs.yarn-project == 'true' || github.ref_name == 'master'
     runs-on: ${{ needs.configure.outputs.username }}-x86
     strategy:
       fail-fast: false
       matrix:
-        test: [smoke.test.ts, transfer.test.ts, 4epochs.test.ts]
+        test: [smoke.test.ts, transfer.test.ts] # TODO reinstate: 4epochs.test.ts
     steps:
       - uses: actions/checkout@v4
         with: { ref: "${{ env.GIT_COMMIT }}" }
@@ -667,8 +686,8 @@ jobs:
           builder_images_to_copy: aztecprotocol/aztec:${{ env.GIT_COMMIT }} aztecprotocol/end-to-end:${{ env.GIT_COMMIT }}
           # command to produce the images in case they don't exist
           builder_command: scripts/earthly-ci ./yarn-project+export-e2e-test-images
+          tester_ttl: 60
           run: |
-            sudo shutdown -P 60 # extend ttl
             cd yarn-project/end-to-end
             echo ${{ secrets.DOCKERHUB_PASSWORD }} | docker login -u aztecprotocolci --password-stdin
             test=${{ matrix.test }}
@@ -796,7 +815,7 @@ jobs:
 
   protocol-circuits-gates-report:
     needs: [build, configure]
-    if: needs.configure.outputs.non-docs == 'true' && needs.configure.outputs.non-misc-ci == 'true'
+    if: needs.configure.outputs.non-docs == 'true' && needs.configure.outputs.non-barretenberg-cpp == 'true'
     runs-on: ${{ needs.configure.outputs.username }}-x86
     permissions:
       pull-requests: write

.github/workflows/nightly-kind-test.yml

@@ -74,8 +74,7 @@ jobs:
             set -eux
             ./spartan/scripts/setup_local_k8s.sh
             export FORCE_COLOR=1
-            export EARTHLY_BUILD_ARGS="${{ env.EARTHLY_BUILD_ARGS }}"
-            ./scripts/earthly-ci --exec-stats -P --no-output ./yarn-project/end-to-end/+kind-network-transfer --values-file=${{ matrix.values_file }}
+            NAMESPACE=transfer FRESH_INSTALL=true VALUES_FILE=${{ matrix.values_file }} ./scripts/network_test.sh ./src/spartan/transfer.test.ts || true
 
   success-check:
     runs-on: ubuntu-20.04

.github/workflows/publish-aztec-packages.yml

@@ -299,6 +299,33 @@ jobs:
             --VERSION=$VERSION \
             --DRY_RUN=${{ (github.event.inputs.publish == 'false') && '1' || '0' }}
 
+  publish-aztec-up:
+    needs: [configure, publish-manifests]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ env.GIT_COMMIT }}
+      - uses: hashicorp/setup-terraform@v3
+        with:
+          terraform_version: 1.7.5
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }}
+          aws-region: eu-west-2
+
+      - name: Publish aztec-up
+        working-directory: ./aztec-up/terraform
+        run: |
+          terraform init
+          TAG=${{ env.DEPLOY_TAG }}
+          export TF_VAR_VERSION=${TAG#aztec-packages-v}
+          terraform apply -auto-approve
+
   # Sometimes runners get killed because they can be spot, we try once more for good measure
   rerun-check:
     runs-on: ubuntu-latest

.github/workflows/sepolia-test.yml

@@ -54,11 +54,11 @@ jobs:
           set -eux
           cd ./yarn-project/end-to-end/
           export FORCE_COLOR=1
-          ../../scripts/earthly-ci -P --no-output +e2e-public-testnet \
-            --SEQ_PUBLISHER_PRIVATE_KEY=${{ secrets.SEPOLIA_SEQ_PRIVATE_KEY }} \
-            --PROVER_PUBLISHER_PRIVATE_KEY=${{ secrets.SEPOLIA_PROVER_PRIVATE_KEY }} \
-            --ETHEREUM_HOST="https://sepolia.infura.io/v3/${{ secrets.SEPOLIA_API_KEY }}" \
-            --L1_CHAIN_ID=${{ env.L1_CHAIN_ID }}
+          SEQ_PUBLISHER_PRIVATE_KEY=${{ secrets.SEPOLIA_SEQ_PRIVATE_KEY }} \
+          PROVER_PUBLISHER_PRIVATE_KEY=${{ secrets.SEPOLIA_PROVER_PRIVATE_KEY }} \
+          ETHEREUM_HOST="https://sepolia.infura.io/v3/${{ secrets.SEPOLIA_API_KEY }}" \
+          L1_CHAIN_ID=${{ env.L1_CHAIN_ID }} \
+          ./scripts/e2e_test_public_testnet.sh ./src/public-testnet
 
   success-check:
     runs-on: ubuntu-20.04

Earthfile

@@ -20,9 +20,6 @@ build:
     # yarn-project has the entry point to Aztec
     BUILD ./yarn-project/+build
 
-test-end-to-end:
-    BUILD ./yarn-project/end-to-end+e2e-tests
-
 release-meta:
     COPY .release-please-manifest.json /usr/src/.release-please-manifest.json
     SAVE ARTIFACT /usr/src /usr/src
@@ -46,7 +43,7 @@ UPLOAD_LOGS:
     ENV BRANCH=$BRANCH
     ENV COMMIT_HASH=$COMMIT_HASH
     RUN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY /usr/src/scripts/logs/upload_logs_to_s3.sh /usr/var/log
-    
+
 base-log-uploader:
     # Install awscli on a fresh ubuntu, and copy the repo "scripts" folder, which we'll use to upload logs
     # Note that we cannot do this LOCALLY because Earthly does not support using secrets locally