diff --git a/.github/workflows/tests-evm-sync.yml b/.github/workflows/tests-evm-sync.yml new file mode 100644 index 00000000000..1b3506c33dd --- /dev/null +++ b/.github/workflows/tests-evm-sync.yml @@ -0,0 +1,157 @@ +name: Tests - EVM Sync +on: + workflow_dispatch: + inputs: + block_ranges: + description: 'Optionally restrict to specific block ranges. Should be + formatted as multiple start block, such as "350000 50000 750000". + Defaults to all block ranges' + required: false + name: + description: 'Workflow run custom name' + required: false + pull_request: + branches: + - master + types: [labeled, opened, reopened, synchronize] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref || github.run_id }} + cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} + +run-name: ${{ inputs.name || github.event.pull_request.title || github.ref_name }} + +jobs: + build: + if: contains(github.event.pull_request.labels.*.name, 'ci/sync') || github.event_name == 'workflow_dispatch' + runs-on: [self-hosted, linux, x64, builder] + steps: + - uses: actions/checkout@v3 + + - name: Populate environment + run: ./make.sh ci-export-vars + + - name: Setup dependencies + run: sudo ./make.sh ci-setup-deps + + - name: Setup dependencies for target + run: ./make.sh ci-setup-deps-target + + - uses: Swatinem/rust-cache@v2 + with: + workspaces: lib -> ../build/lib/target + save-if: ${{ github.ref == 'refs/heads/master' }} + + # TODO: Switch this to a docker build later and this builds on the native + # VM toolchain and loses disparity with the CI release builds + - name: Build binaries + run: ./make.sh build + + - name: Upload binaries + uses: actions/upload-artifact@v3 + with: + name: defi-bins + path: | + build/src/defid + build/src/defi-cli + + - name: Upload shell commands + uses: actions/upload-artifact@v3 + with: + name: sync + path: ci/sync/evm_main.sh + + generate-matrix: + if: contains(github.event.pull_request.labels.*.name, 'ci/sync') || github.event_name == 'workflow_dispatch' + runs-on: [self-hosted, linux, x64] + + # Add "id-token" with the intended permissions. + permissions: + contents: 'read' + id-token: 'write' + + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + + steps: + - uses: actions/checkout@v3 + + - id: 'auth' + name: 'Authenticate to Google Cloud' + uses: 'google-github-actions/auth@v0' + with: + workload_identity_provider: 'projects/965426322273/locations/global/workloadIdentityPools/br-blockchains-pool/providers/br-blockchains-provider' + service_account: 'blockchain-dev-service@br-blockchains-dev.iam.gserviceaccount.com' + + - name: 'Set up Cloud SDK' + uses: 'google-github-actions/setup-gcloud@v0' + + - id: set-matrix + name: 'Set matrix output' + run: | + if [ -n "${{ inputs.block_ranges }}" ]; then + BLOCKS=$(echo ${{ inputs.block_ranges }} | tr ' ' '\n') + else + SNAPSHOTS=$(gsutil ls gs://team-drop/master-evm-datadir) + BLOCKS=$(echo "$SNAPSHOTS" | sed -e 's/.*\-\(.*\)\.tar.*/\1/' | grep -v gs | sort -n | head -n -1) + fi + MATRIX_JSON=$(jq -n -c -M --arg blocks "$BLOCKS" '{blocks: ($blocks | split("\n") | .[] |= tonumber | to_entries | map({start: .value , stop: (.value + 50000)}))}') + echo "MATRIX=$MATRIX_JSON" >> $GITHUB_OUTPUT + + sync: + runs-on: [self-hosted, linux, x64] + needs: [build, generate-matrix] + + strategy: + matrix: ${{fromJson(needs.generate-matrix.outputs.matrix)}} + + continue-on-error: true + env: + BASE_PATH: "https://storage.googleapis.com" + BUCKET: "team-drop" + DATADIR : datadir-${{matrix.blocks.start}} + STOP_BLOCK: ${{matrix.blocks.stop}} + START_BLOCK: ${{matrix.blocks.start}} + DEFID_BIN: ./defid + DEFI_CLI_BIN: ./defi-cli + BASE_REF: master + timeout-minutes: 4320 + + steps: + - uses: actions/checkout@v3 + + - name: Download Snapshot + run: aria2c -x16 -s16 ${{ env.BASE_PATH }}/${{ env.BUCKET }}/${{ BASE_REF }}-evm-datadir/datadir-${{matrix.blocks.start}}.tar.gz + + - name: Create datadir + run: mkdir $DATADIR && tar -C $DATADIR -xvf datadir-${{matrix.blocks.start}}.tar.gz + + - name: Download Binaries + uses: actions/download-artifact@v3 + with: + name: defi-bins + + - name: Download Shell Commands + uses: actions/download-artifact@v3 + with: + name: sync + + - name: Rename Artifact + run: mv evm_main.sh sync.sh + + - name: Set Permissions + run: | + chmod 777 defid + chmod 777 defi-cli + chmod 777 sync.sh + + - name: Sync + run: ./sync.sh + + - name: Diff rollback log + run: diff pre-rollback.log post-rollback.log + if: ${{ failure() || success() }} + + - name: Show post-rollback.log + run: cat post-rollback.log + if: ${{ failure() || success() }} diff --git a/ci/sync/evm_main.sh b/ci/sync/evm_main.sh index c1da72c2f21..d8ec4a3e637 100644 --- a/ci/sync/evm_main.sh +++ b/ci/sync/evm_main.sh @@ -12,8 +12,8 @@ setup_vars() { DATADIR=${DATADIR:-".defi"} DEBUG_FILE="${DATADIR}/debug.log" CONF_FILE="${DATADIR}/defi.conf" - PRE_ROLLBACK_STATE_LOG="debug-pre-rollback.log" - POST_ROLLBACK_STATE_LOG="debug-post-rollback.log" + PRE_ROLLBACK_STATE_LOG="pre-rollback.log" + POST_ROLLBACK_STATE_LOG="post-rollback.log" BASE_REF=${BASE_REF:-"master"} BASE_PATH=${BASE_PATH:-"https://storage.googleapis.com"} BUCKET=${BUCKET:-"team-drop"} @@ -50,7 +50,7 @@ print_info() { ${REF_LOG_PATH} - snapshot: - ${BASE_PATH}/${BUCKET}/${BASE_REF}-evmdatadir/datadir-${START_BLOCK}.tar.gz + ${BASE_PATH}/${BUCKET}/${BASE_REF}-evm-datadir/datadir-${START_BLOCK}.tar.gz - defid: ${DEFID_CMD}