From cd4a7164efc93717814b494b02037bce9188addf Mon Sep 17 00:00:00 2001 From: Max Revitt Date: Fri, 28 Jun 2024 08:12:10 +0100 Subject: [PATCH 01/27] tweak(seq): typo fix (#674) --- zk/stages/stage_sequence_execute_blocks.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zk/stages/stage_sequence_execute_blocks.go b/zk/stages/stage_sequence_execute_blocks.go index 1d2a4ce6c89..36c358f2dc7 100644 --- a/zk/stages/stage_sequence_execute_blocks.go +++ b/zk/stages/stage_sequence_execute_blocks.go @@ -205,7 +205,7 @@ func postBlockStateHandling( ) error { blockNumber := header.Number.Uint64() - blokInfoRootHash, err := blockinfo.BuildBlockInfoTree( + blockInfoRootHash, err := blockinfo.BuildBlockInfoTree( &header.Coinbase, blockNumber, header.Time, @@ -220,10 +220,10 @@ func postBlockStateHandling( return err } - ibs.PostExecuteStateSet(cfg.chainConfig, header.Number.Uint64(), blokInfoRootHash) + ibs.PostExecuteStateSet(cfg.chainConfig, header.Number.Uint64(), blockInfoRootHash) // store a reference to this block info root against the block number - return hermezDb.WriteBlockInfoRoot(header.Number.Uint64(), *blokInfoRootHash) + return hermezDb.WriteBlockInfoRoot(header.Number.Uint64(), *blockInfoRootHash) } func addSenders( From 8c135c68ec81d4d5c509762dfc7b63e849900cfa Mon Sep 17 00:00:00 2001 From: Max Revitt Date: Fri, 28 Jun 2024 08:21:06 +0100 Subject: [PATCH 02/27] fix(zkevm_api): getbatchbynumber quick fix inforoot nil (#675) --- cmd/rpcdaemon/commands/zkevm_api.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cmd/rpcdaemon/commands/zkevm_api.go b/cmd/rpcdaemon/commands/zkevm_api.go index 7cbb7296702..ac5510f67c1 100644 --- a/cmd/rpcdaemon/commands/zkevm_api.go +++ b/cmd/rpcdaemon/commands/zkevm_api.go @@ -378,10 +378,6 @@ func (api *ZkEvmAPIImpl) GetBatchByNumber(ctx context.Context, batchNumber rpc.B batch.RollupExitRoot = infoTreeUpdate.RollupExitRoot } - if infoTreeUpdate == nil { - return nil, errors.New("infoTreeUpdate is nil") - } - // batch l2 data - must build on the fly forkId, err := hermezDb.GetForkId(bn) if err != nil { From 63ea3b72cbfafe5abae6c03199bcd71689049aea Mon Sep 17 00:00:00 2001 From: Valentin Staykov <79150443+V-Staykov@users.noreply.github.com> Date: Fri, 28 Jun 2024 10:21:36 +0300 Subject: [PATCH 03/27] commented out overflowing and eip1559 txs (#671) --- .../roles/eip-1559/tasks/main.yml | 81 ++++++++++--------- .../roles/geth-automated-tests/tasks/main.yml | 24 +++--- .../roles/polycli-cases/tasks/main.yml | 50 ++++++------ .../roles/snowball/tasks/main.yml | 10 +-- 4 files changed, 83 insertions(+), 82 deletions(-) diff --git a/zk/tests/ansible/evm-rpc-tests/roles/eip-1559/tasks/main.yml b/zk/tests/ansible/evm-rpc-tests/roles/eip-1559/tasks/main.yml index 8d851d530df..ff909e5747a 100644 --- a/zk/tests/ansible/evm-rpc-tests/roles/eip-1559/tasks/main.yml +++ b/zk/tests/ansible/evm-rpc-tests/roles/eip-1559/tasks/main.yml @@ -11,51 +11,52 @@ args: executable: /bin/bash -- name: Fill three blocks - ansible.builtin.shell: | - cast send --private-key {{ private_key }} --rpc-url {{ rpc_url }} --gas-limit {{ max_block_size }} -j --create "0x60005B60010180405060405A1163000000025700" >> {{ log_file }} - cast send --private-key {{ private_key }} --rpc-url {{ rpc_url }} --gas-limit {{ max_block_size }} -j --create "0x60005B60010180405060405A1163000000025700" >> {{ log_file }} - cast send --private-key {{ private_key }} --rpc-url {{ rpc_url }} --gas-limit {{ max_block_size }} -j --create "0x60005B60010180405060405A1163000000025700" >> {{ log_file }} - args: - executable: /bin/bash +# - name: Fill three blocks # overflows counters +# ansible.builtin.shell: | +# cast send --private-key {{ private_key }} --rpc-url {{ rpc_url }} --gas-limit {{ max_block_size }} -j --create "0x60005B60010180405060405A1163000000025700" >> {{ log_file }} +# cast send --private-key {{ private_key }} --rpc-url {{ rpc_url }} --gas-limit {{ max_block_size }} -j --create "0x60005B60010180405060405A1163000000025700" >> {{ log_file }} +# cast send --private-key {{ private_key }} --rpc-url {{ rpc_url }} --gas-limit {{ max_block_size }} -j --create "0x60005B60010180405060405A1163000000025700" >> {{ log_file }} +# args: +# executable: /bin/bash -- name: Try to send tx below base fee and replace it - ansible.builtin.shell: | - cast send --async --private-key {{ private_key }} --rpc-url {{ rpc_url }} \ - --gas-price 1 --priority-gas-price 0 \ - --gas-limit 100000 -j {{ eth_address }} "0x646174613a2c7b226d7367223a226569702d3135353920756e646572707269636564227d" &>> {{ log_file }} +# - name: Try to send tx below base fee and replace it # 1559 and tips are disabled for Hermez +# ansible.builtin.shell: | +# cast send --async --private-key {{ private_key }} --rpc-url {{ rpc_url }} \ +# --gas-price 1 --priority-gas-price 0 \ +# --gas-limit 100000 -j {{ eth_address }} "0x646174613a2c7b226d7367223a226569702d3135353920756e646572707269636564227d" &>> {{ log_file }} - sleep 2 +# sleep 2 - cast send --private-key {{ private_key }} --rpc-url {{ rpc_url }} \ - --gas-limit 100000 -j {{ eth_address }} "0x646174613a2c7b226d7367223a226569702d31353539207265706c6163656d656e74227d" &>> {{ log_file }} - args: - executable: /bin/bash +# cast send --private-key {{ private_key }} --rpc-url {{ rpc_url }} \ +# --tip 50000000 +# --gas-limit 100000 -j {{ eth_address }} "0x646174613a2c7b226d7367223a226569702d31353539207265706c6163656d656e74227d" &>> {{ log_file }} +# args: +# executable: /bin/bash -- name: Try to send a tx where the priority fee and base fee are equal - ansible.builtin.shell: | - gas_price="$(cast gas-price --rpc-url {{ rpc_url }})" - cast send --async --private-key {{ private_key }} --rpc-url {{ rpc_url }} \ - --gas-price $gas_price --priority-gas-price $gas_price \ - --gas-limit 100000 -j {{ eth_address }} "0x646174613a2c7b226d7367223a226569702d3135353920657175616c20666565227d" &>> {{ log_file }} - args: - executable: /bin/bash +# - name: Try to send a tx where the priority fee and base fee are equal # 1559 and tips are disabled for Hermez +# ansible.builtin.shell: | +# gas_price="$(cast gas-price --rpc-url {{ rpc_url }})" +# cast send --async --private-key {{ private_key }} --rpc-url {{ rpc_url }} \ +# --gas-price $gas_price --priority-gas-price $gas_price \ +# --gas-limit 100000 -j {{ eth_address }} "0x646174613a2c7b226d7367223a226569702d3135353920657175616c20666565227d" &>> {{ log_file }} +# args: +# executable: /bin/bash -- name: Try to send a tx where the priority fee is above the cap - ansible.builtin.shell: | - gas_price="$(cast gas-price --rpc-url {{ rpc_url }})" - cast send --async --private-key {{ private_key }} --rpc-url {{ rpc_url }} \ - --gas-price $gas_price --priority-gas-price $(( gas_price * 2)) \ - --gas-limit 100000 -j {{ eth_address }} "0x646174613a2c7b226d7367223a226569702d31353539206d6178207072696f7269747920666565207065722067617320686967686572207468616e206d6178206665652070657220676173227d" &>> {{ log_file }} - retVal=$? - if [[ $retVal -ne 0 ]]; then - echo "Task failed successfully" - else - echo "Expected failure, but somehow this worked" - exit 1; - fi - args: - executable: /bin/bash +# - name: Try to send a tx where the priority fee is above the cap # 1559 and tips are disabled for Hermez +# ansible.builtin.shell: | +# gas_price="$(cast gas-price --rpc-url {{ rpc_url }})" +# cast send --async --private-key {{ private_key }} --rpc-url {{ rpc_url }} \ +# --gas-price $gas_price --priority-gas-price $(( gas_price * 2)) \ +# --gas-limit 100000 -j {{ eth_address }} "0x646174613a2c7b226d7367223a226569702d31353539206d6178207072696f7269747920666565207065722067617320686967686572207468616e206d6178206665652070657220676173227d" &>> {{ log_file }} +# retVal=$? +# if [[ $retVal -ne 0 ]]; then +# echo "Task failed successfully" +# else +# echo "Expected failure, but somehow this worked" +# exit 1; +# fi +# args: +# executable: /bin/bash - name: Add inscription to mark test end ansible.builtin.include_role: diff --git a/zk/tests/ansible/evm-rpc-tests/roles/geth-automated-tests/tasks/main.yml b/zk/tests/ansible/evm-rpc-tests/roles/geth-automated-tests/tasks/main.yml index ae45588ba37..0e8d00cf35b 100644 --- a/zk/tests/ansible/evm-rpc-tests/roles/geth-automated-tests/tasks/main.yml +++ b/zk/tests/ansible/evm-rpc-tests/roles/geth-automated-tests/tasks/main.yml @@ -5,18 +5,18 @@ src: geth-contracts dest: "{{ work_dir }}" -- name: Deploy and call each of the geth contracts - ansible.builtin.shell: | - pushd {{ work_dir }} || exit 1 - find geth-contracts -type f -name '*.bin' | sort | while read -r contract; do - cast send --legacy --from {{ eth_address }} --private-key {{ private_key }} --rpc-url {{ rpc_url }} -j --create \ - "$(cat $contract)" | jq '.' > out.tmp.json - cast send --legacy --gas-limit 2000000 --private-key {{ private_key }} --rpc-url {{ rpc_url }} \ - "$(jq -r '.contractAddress' out.tmp.json)" -j "0xDEADBEEF" | jq '.' > $contract.run.json - cat $contract.run.json > {{ log_file }} - done - args: - executable: /bin/bash +# - name: Deploy and call each of the geth contracts # some of the contracts are overflowing counters +# ansible.builtin.shell: | +# pushd {{ work_dir }} || exit 1 +# find geth-contracts -type f -name '*.bin' | sort | while read -r contract; do +# cast send --legacy --from {{ eth_address }} --private-key {{ private_key }} --rpc-url {{ rpc_url }} -j --create \ +# "$(cat $contract)" | jq '.' > out.tmp.json +# cast send --legacy --gas-limit 2000000 --private-key {{ private_key }} --rpc-url {{ rpc_url }} \ +# "$(jq -r '.contractAddress' out.tmp.json)" -j "0xDEADBEEF" | jq '.' > $contract.run.json +# cat $contract.run.json > {{ log_file }} +# done +# args: +# executable: /bin/bash - name: Deploy and call each of the geth contracts ansible.builtin.shell: | diff --git a/zk/tests/ansible/evm-rpc-tests/roles/polycli-cases/tasks/main.yml b/zk/tests/ansible/evm-rpc-tests/roles/polycli-cases/tasks/main.yml index 947b73ebe3e..58d4d303e63 100644 --- a/zk/tests/ansible/evm-rpc-tests/roles/polycli-cases/tasks/main.yml +++ b/zk/tests/ansible/evm-rpc-tests/roles/polycli-cases/tasks/main.yml @@ -13,12 +13,12 @@ args: executable: /bin/bash -- name: Deploy polycli ERC721 Contract - ansible.builtin.shell: | - pushd {{ work_dir }} || exit 1 - cast send -j {{ legacy_flag }} --private-key {{ private_key }} --rpc-url {{ rpc_url }} --create "$(cat polycli-contracts/ERC721.bin)" > erc721.out.json - args: - executable: /bin/bash +# - name: Deploy polycli ERC721 Contract # This overflows +# ansible.builtin.shell: | +# pushd {{ work_dir }} || exit 1 +# cast send -j {{ legacy_flag }} --private-key {{ private_key }} --rpc-url {{ rpc_url }} --create "$(cat polycli-contracts/ERC721.bin)" > erc721.out.json +# args: +# executable: /bin/bash - name: Deploy polycli Loadtest Contract ansible.builtin.shell: | @@ -34,25 +34,25 @@ args: executable: /bin/bash -- name: Perform some ERC721 Mints - ansible.builtin.shell: | - pushd {{ work_dir }} || exit 1 - polycli loadtest {{ legacy_flag }} --private-key {{ private_key }} --mode 7 --rate-limit 500 --requests 2 --concurrency 2 \ - --iterations 1 --erc721-address "$(jq -r '.contractAddress' erc721.out.json)" --rpc-url {{ rpc_url }} &>> {{ log_file }} - polycli loadtest {{ legacy_flag }} --private-key {{ private_key }} --mode 7 --rate-limit 500 --requests 2 --concurrency 2 \ - --iterations 2 --erc721-address "$(jq -r '.contractAddress' erc721.out.json)" --rpc-url {{ rpc_url }} &>> {{ log_file }} - polycli loadtest {{ legacy_flag }} --private-key {{ private_key }} --mode 7 --rate-limit 500 --requests 2 --concurrency 2 \ - --iterations 4 --erc721-address "$(jq -r '.contractAddress' erc721.out.json)" --rpc-url {{ rpc_url }} &>> {{ log_file }} - polycli loadtest {{ legacy_flag }} --private-key {{ private_key }} --mode 7 --rate-limit 500 --requests 2 --concurrency 2 \ - --iterations 8 --erc721-address "$(jq -r '.contractAddress' erc721.out.json)" --rpc-url {{ rpc_url }} &>> {{ log_file }} - polycli loadtest {{ legacy_flag }} --private-key {{ private_key }} --mode 7 --rate-limit 500 --requests 2 --concurrency 2 \ - --iterations 16 --erc721-address "$(jq -r '.contractAddress' erc721.out.json)" --rpc-url {{ rpc_url }} &>> {{ log_file }} - polycli loadtest {{ legacy_flag }} --private-key {{ private_key }} --mode 7 --rate-limit 500 --requests 2 --concurrency 2 \ - --iterations 32 --erc721-address "$(jq -r '.contractAddress' erc721.out.json)" --rpc-url {{ rpc_url }} &>> {{ log_file }} - polycli loadtest {{ legacy_flag }} --private-key {{ private_key }} --mode 7 --rate-limit 500 --requests 2 --concurrency 2 \ - --iterations 64 --erc721-address "$(jq -r '.contractAddress' erc721.out.json)" --rpc-url {{ rpc_url }} &>> {{ log_file }} - args: - executable: /bin/bash +# - name: Perform some ERC721 Mints +# ansible.builtin.shell: | +# pushd {{ work_dir }} || exit 1 +# polycli loadtest {{ legacy_flag }} --private-key {{ private_key }} --mode 7 --rate-limit 500 --requests 2 --concurrency 2 \ +# --iterations 1 --erc721-address "$(jq -r '.contractAddress' erc721.out.json)" --rpc-url {{ rpc_url }} &>> {{ log_file }} +# polycli loadtest {{ legacy_flag }} --private-key {{ private_key }} --mode 7 --rate-limit 500 --requests 2 --concurrency 2 \ +# --iterations 2 --erc721-address "$(jq -r '.contractAddress' erc721.out.json)" --rpc-url {{ rpc_url }} &>> {{ log_file }} +# polycli loadtest {{ legacy_flag }} --private-key {{ private_key }} --mode 7 --rate-limit 500 --requests 2 --concurrency 2 \ +# --iterations 4 --erc721-address "$(jq -r '.contractAddress' erc721.out.json)" --rpc-url {{ rpc_url }} &>> {{ log_file }} +# polycli loadtest {{ legacy_flag }} --private-key {{ private_key }} --mode 7 --rate-limit 500 --requests 2 --concurrency 2 \ +# --iterations 8 --erc721-address "$(jq -r '.contractAddress' erc721.out.json)" --rpc-url {{ rpc_url }} &>> {{ log_file }} +# polycli loadtest {{ legacy_flag }} --private-key {{ private_key }} --mode 7 --rate-limit 500 --requests 2 --concurrency 2 \ +# --iterations 16 --erc721-address "$(jq -r '.contractAddress' erc721.out.json)" --rpc-url {{ rpc_url }} &>> {{ log_file }} +# polycli loadtest {{ legacy_flag }} --private-key {{ private_key }} --mode 7 --rate-limit 500 --requests 2 --concurrency 2 \ +# --iterations 32 --erc721-address "$(jq -r '.contractAddress' erc721.out.json)" --rpc-url {{ rpc_url }} &>> {{ log_file }} +# polycli loadtest {{ legacy_flag }} --private-key {{ private_key }} --mode 7 --rate-limit 500 --requests 2 --concurrency 2 \ +# --iterations 64 --erc721-address "$(jq -r '.contractAddress' erc721.out.json)" --rpc-url {{ rpc_url }} &>> {{ log_file }} +# args: +# executable: /bin/bash - name: Perform some Storage calls in the load tester contract ansible.builtin.shell: | diff --git a/zk/tests/ansible/evm-rpc-tests/roles/snowball/tasks/main.yml b/zk/tests/ansible/evm-rpc-tests/roles/snowball/tasks/main.yml index e3c60b7ce9b..b9b9912a60e 100644 --- a/zk/tests/ansible/evm-rpc-tests/roles/snowball/tasks/main.yml +++ b/zk/tests/ansible/evm-rpc-tests/roles/snowball/tasks/main.yml @@ -44,11 +44,11 @@ - { "primes": 128 } - { "primes": 256 } - { "primes": 512 } - - { "primes": 1024 } - - { "primes": 2048 } - - { "primes": 4096 } - - { "primes": 8192 } - - { "primes": 16384 } + # - { "primes": 1024 } # This overflows + # - { "primes": 2048 } + # - { "primes": 4096 } + # - { "primes": 8192 } + # - { "primes": 16384 } - name: Add inscription to mark test end ansible.builtin.include_role: From 6bd608da2ed66a5e1b9d2606d48283aee686f5f6 Mon Sep 17 00:00:00 2001 From: Jerry Date: Fri, 28 Jun 2024 00:23:04 -0700 Subject: [PATCH 04/27] Add bridge test to zkevm CI (#668) * Add bridge test to CI * Remove matrix --------- Co-authored-by: Igor Mandrigin --- .github/workflows/ci_zkevm.yml | 69 +++++++++++++++++++++++++++++++--- 1 file changed, 64 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci_zkevm.yml b/.github/workflows/ci_zkevm.yml index daea0279e52..a44c020465a 100644 --- a/.github/workflows/ci_zkevm.yml +++ b/.github/workflows/ci_zkevm.yml @@ -57,6 +57,10 @@ jobs: kurtosis-cdk: runs-on: ubuntu-latest + # TODO: Add "cdk-validium" once it's ready + # strategy: + # matrix: + # da-mode: [ "rollup" ] steps: - name: Checkout cdk-erigon uses: actions/checkout@v4 @@ -67,12 +71,21 @@ jobs: uses: actions/checkout@v4 with: repository: 0xPolygon/kurtosis-cdk - ref: feat/cdk-erigon-zkevm + ref: main path: kurtosis-cdk - name: Install Kurtosis CDK tools uses: ./kurtosis-cdk/.github/actions/setup-kurtosis-cdk + - name: Install Foundry + uses: foundry-rs/foundry-toolchain@v1 + + - name: Install yq + run: | + sudo curl -L https://github.com/mikefarah/yq/releases/download/v4.44.2/yq_linux_amd64 -o /usr/local/bin/yq + sudo chmod +x /usr/local/bin/yq + /usr/local/bin/yq --version + - name: Build docker image working-directory: ./cdk-erigon run: docker build -t cdk-erigon:local --file Dockerfile . @@ -80,17 +93,63 @@ jobs: - name: Configure Kurtosis CDK working-directory: ./kurtosis-cdk run: | - yq -Y --in-place '.args.data_availability_mode = "rollup"' params.yml - yq -Y --in-place '.args.cdk_erigon_node_image = "cdk-erigon:local"' params.yml + /usr/local/bin/yq -i '.args.data_availability_mode = "rollup"' cdk-erigon-sequencer-params.yml + /usr/local/bin/yq -i '.args.cdk_erigon_node_image = "cdk-erigon:local"' cdk-erigon-sequencer-params.yml + /usr/local/bin/yq -i '.args.zkevm_bridge_service_image = "hermeznetwork/zkevm-bridge-service:v0.5.0-RC8"' cdk-erigon-sequencer-params.yml - name: Deploy Kurtosis CDK package working-directory: ./kurtosis-cdk - run: kurtosis run --enclave cdk-v1 --args-file params.yml --image-download always . + run: kurtosis run --enclave cdk-v1 --args-file cdk-erigon-sequencer-params.yml --image-download always . - name: Monitor verified batches working-directory: ./kurtosis-cdk shell: bash - run: .github/actions/monitor-cdk-verified-batches/batch_verification_monitor.sh 20 600 cdk-erigon-node-001 + run: .github/actions/monitor-cdk-verified-batches/batch_verification_monitor.sh 20 900 cdk-erigon-node-001 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Set up envs + run: | + kurtosis files download cdk-v1 bridge-config-artifact + echo "BRIDGE_ADDRESS=$(/usr/local/bin/yq '.NetworkConfig.PolygonBridgeAddress' bridge-config-artifact/bridge-config.toml)" >> $GITHUB_ENV + echo "ETH_RPC_URL=$(kurtosis port print cdk-v1 el-1-geth-lighthouse rpc)" >> $GITHUB_ENV + echo "L2_RPC_URL=$(kurtosis port print cdk-v1 cdk-erigon-node-001 http-rpc)" >> $GITHUB_ENV + echo "BRIDGE_API_URL=$(kurtosis port print cdk-v1 zkevm-bridge-service-001 rpc)" >> $GITHUB_ENV + + - name: Fund claim tx manager + run: | + cast send --legacy --private-key "0x12d7de8621a77640c9241b2595ba78ce443d05e94090365ab3bb5e19df82c625" --value 5ether 0x5f5dB0D4D58310F53713eF4Df80ba6717868A9f8 --gas-limit 50000 --rpc-url $L2_RPC_URL + + - name: Clone bridge repository + run: git clone --recurse-submodules -j8 https://github.com/0xPolygonHermez/zkevm-bridge-service.git -b develop bridge + + - name: Build docker image + run: | + cd bridge + make build-docker-e2e-real_network + + - name: Run test ERC20 Bridge + run: | + #!/bin/bash + cd bridge + mkdir tmp + cat < ./tmp/test.toml + TestAddrPrivate= "0x12d7de8621a77640c9241b2595ba78ce443d05e94090365ab3bb5e19df82c625" + [ConnectionConfig] + L1NodeURL="${ETH_RPC_URL}" + L2NodeURL="${L2_RPC_URL}" + BridgeURL="${BRIDGE_API_URL}" + L1BridgeAddr="${BRIDGE_ADDRESS}" + L2BridgeAddr="${BRIDGE_ADDRESS}" + EOF + docker run --network=host --volume "./tmp/:/config/" --env BRIDGE_TEST_CONFIG_FILE=/config/test.toml bridge-e2e-realnetwork-erc20 + + - name: Upload logs + uses: actions/upload-artifact@v3 + with: + name: evm-rpc-tests-logs-${{ github.run_id }} + path: ./cdk-erigon/logs/evm-rpc-tests.log - name: Prepare logs working-directory: ./kurtosis-cdk From 0d658023881d7fae3da145e759039f68c1bf315b Mon Sep 17 00:00:00 2001 From: Valentin Staykov <79150443+V-Staykov@users.noreply.github.com> Date: Fri, 28 Jun 2024 12:12:41 +0300 Subject: [PATCH 05/27] added steps for forkid 10 and 11 (#677) --- core/vm/zk_counters_limits.go | 44 ++++++++++++++++++++++++----------- zk/constants/constants.go | 2 ++ 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/core/vm/zk_counters_limits.go b/core/vm/zk_counters_limits.go index b78466e09a6..311c37c3d88 100644 --- a/core/vm/zk_counters_limits.go +++ b/core/vm/zk_counters_limits.go @@ -7,16 +7,9 @@ import ( ) var ( - defaultTotalSteps = 1 << 23 - defaultCounterLimits = counterLimits{ - totalSteps: defaultTotalSteps, - arith: defaultTotalSteps >> 5, - binary: defaultTotalSteps >> 4, - memAlign: defaultTotalSteps >> 5, - keccaks: int(math.Floor(float64(defaultTotalSteps)/155286) * 44), - padding: int(math.Floor(float64(defaultTotalSteps) / 56)), - poseidon: int(math.Floor(float64(defaultTotalSteps) / 30)), - } + defaultTotalSteps = 1 << 23 + forkId10TotalSteps = 1 << 24 + forkId11TotalSteps = 1 << 25 unlimitedCounters = counterLimits{ totalSteps: math.MaxInt32, @@ -26,6 +19,7 @@ var ( keccaks: math.MaxInt32, padding: math.MaxInt32, poseidon: math.MaxInt32, + sha256: math.MaxInt32, } ) @@ -80,9 +74,33 @@ func createCountrsByLimits(c counterLimits) *Counters { // tp ne used on next forkid counters func getCounterLimits(forkId uint16) *Counters { - if forkId <= uint16(zk_consts.ForkID9Elderberry2) { - return createCountrsByLimits(defaultCounterLimits) + totalSteps := getTotalSteps(forkId) + + counterLimits := counterLimits{ + totalSteps: totalSteps, + arith: totalSteps >> 5, + binary: totalSteps >> 4, + memAlign: totalSteps >> 5, + keccaks: int(math.Floor(float64(totalSteps)/155286) * 44), + padding: int(math.Floor(float64(totalSteps) / 56)), + poseidon: int(math.Floor(float64(totalSteps) / 30)), + sha256: int(math.Floor(float64(totalSteps-1)/31488)) * 7, + } + + return createCountrsByLimits(counterLimits) +} + +func getTotalSteps(forkId uint16) int { + var totalSteps int + + switch forkId { + case uint16(zk_consts.ForkID10): + totalSteps = forkId10TotalSteps + case uint16(zk_consts.ForkID11): + totalSteps = forkId11TotalSteps + default: + totalSteps = defaultTotalSteps } - return createCountrsByLimits(defaultCounterLimits) + return totalSteps } diff --git a/zk/constants/constants.go b/zk/constants/constants.go index 719d1484736..39e99154ac6 100644 --- a/zk/constants/constants.go +++ b/zk/constants/constants.go @@ -9,4 +9,6 @@ const ( ForkID7Etrog ForkId = 7 ForkID8Elderberry ForkId = 8 ForkID9Elderberry2 ForkId = 9 + ForkID10 ForkId = 10 + ForkID11 ForkId = 11 ) From 29f6b8602f8ce4183202e883a3d1617fe76b98a8 Mon Sep 17 00:00:00 2001 From: hexoscott <70711990+hexoscott@users.noreply.github.com> Date: Mon, 1 Jul 2024 14:31:06 +0100 Subject: [PATCH 06/27] unwind to batch start rather than block height in stream when error is detected (#682) --- zk/datastream/server/data_stream_server.go | 13 +++++++++++++ zk/datastream/server/datastream_populate.go | 17 ++++++++++++++++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/zk/datastream/server/data_stream_server.go b/zk/datastream/server/data_stream_server.go index 55483332e0b..5b7829f9439 100644 --- a/zk/datastream/server/data_stream_server.go +++ b/zk/datastream/server/data_stream_server.go @@ -534,6 +534,19 @@ func (srv *DataStreamServer) UnwindToBlock(blockNumber uint64) error { return srv.stream.TruncateFile(entryNum + 1) } +func (srv *DataStreamServer) UnwindToBatchStart(batchNumber uint64) error { + bookmark := types.NewBookmarkProto(batchNumber, datastream.BookmarkType_BOOKMARK_TYPE_BATCH) + marshalled, err := bookmark.Marshal() + if err != nil { + return err + } + entryNum, err := srv.stream.GetBookmark(marshalled) + if err != nil { + return err + } + return srv.stream.TruncateFile(entryNum) +} + const ( PACKET_TYPE_DATA = 2 // NOOP_ENTRY_NUMBER is used because we don't care about the entry number when feeding an atrificial diff --git a/zk/datastream/server/datastream_populate.go b/zk/datastream/server/datastream_populate.go index 1e7ea0e6910..18aea4b72c3 100644 --- a/zk/datastream/server/datastream_populate.go +++ b/zk/datastream/server/datastream_populate.go @@ -79,9 +79,24 @@ func WriteBlocksToStream( } if highestDatastreamBlock > from { - if err := srv.UnwindToBlock(from); err != nil { + // now we need to unwind the batch right back to the point of the start of the batch we are going to write again + batch, err := reader.GetBatchNoByL2Block(from) + if err != nil { + + } + if err := srv.UnwindToBatchStart(batch); err != nil { + return err + } + + // now find the first block in the batch we just unwound to + firstBlockInBatch, found, err := reader.GetLowestBlockInBatch(batch) + if err != nil { return err } + if !found { + return fmt.Errorf("could not find the first block in the batch %v", batch) + } + from = firstBlockInBatch } logTicker := time.NewTicker(10 * time.Second) From ca47f99cad6e919abf4e6ec28e85d3e84a813f5e Mon Sep 17 00:00:00 2001 From: Max Revitt Date: Tue, 2 Jul 2024 11:17:33 +0100 Subject: [PATCH 07/27] fix(zkevm_api): getBatchByNumber, ler, other fixes (#673) * fix(zkevm_api): getBatchByNumber, ler, other fixes * fix(zkevm_api): getBatchByNumber match zkevm node when batch not found * feat(zkevm_api): getBatchDataByNumbers impl closes #679 * tweak(log): log info when sync limit reached * fix(flags): flag check fix, bali addrs, stream * fix(zkevm_api): getBatchDataByNumbers output format * remove check for data streamer timeout - this is optional --------- Co-authored-by: Scott Fairclough --- cmd/rpcdaemon/commands/zkevm_api.go | 184 ++++++++++++++++++-- eth/stagedsync/stage_execute_zkevm.go | 1 + hermezconfig-bali.yaml.example | 8 +- turbo/cli/flags_zkevm.go | 17 +- zk/datastream/server/data_stream_server.go | 33 +--- zk/datastream/server/datastream_populate.go | 3 +- zk/hermez_db/db.go | 14 ++ zk/rpcdaemon/types_zkevm.go | 6 + zk/stages/stage_batches.go | 8 + zk/stages/stage_sequence_execute.go | 10 ++ zk/utils/utils.go | 43 +++++ 11 files changed, 277 insertions(+), 50 deletions(-) diff --git a/cmd/rpcdaemon/commands/zkevm_api.go b/cmd/rpcdaemon/commands/zkevm_api.go index ac5510f67c1..3203a343d8e 100644 --- a/cmd/rpcdaemon/commands/zkevm_api.go +++ b/cmd/rpcdaemon/commands/zkevm_api.go @@ -28,10 +28,11 @@ import ( types "github.com/ledgerwatch/erigon/zk/rpcdaemon" "github.com/ledgerwatch/erigon/zk/sequencer" "github.com/ledgerwatch/erigon/zk/syncer" + zktx "github.com/ledgerwatch/erigon/zk/tx" + "github.com/ledgerwatch/erigon/zk/utils" "github.com/ledgerwatch/erigon/zk/witness" "github.com/ledgerwatch/erigon/zkevm/hex" "github.com/ledgerwatch/erigon/zkevm/jsonrpc/client" - zktx "github.com/ledgerwatch/erigon/zk/tx" ) var sha3UncleHash = common.HexToHash("0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347") @@ -229,6 +230,125 @@ func (api *ZkEvmAPIImpl) VerifiedBatchNumber(ctx context.Context) (hexutil.Uint6 return hexutil.Uint64(highestVerifiedBatchNo), nil } +// GetBatchDataByNumbers returns the batch data for the given batch numbers +func (api *ZkEvmAPIImpl) GetBatchDataByNumbers(ctx context.Context, batchNumbers []rpc.BlockNumber) (json.RawMessage, error) { + tx, err := api.db.BeginRo(ctx) + if err != nil { + return nil, err + } + defer tx.Rollback() + + hermezDb := hermez_db.NewHermezDbReader(tx) + + bds := make([]*types.BatchDataSlim, 0, len(batchNumbers)) + + for _, batchNumber := range batchNumbers { + bd := &types.BatchDataSlim{ + Number: uint64(batchNumber.Int64()), + Empty: false, + } + + // looks weird but we're using the rpc.BlockNumber type to represent the batch number, LatestBlockNumber represents latest batch + if batchNumber == rpc.LatestBlockNumber { + highestBlock, err := rawdb.ReadLastBlockSynced(tx) + if err != nil { + return nil, err + } + highestBatchNo, err := hermezDb.GetBatchNoByL2Block(highestBlock.NumberU64()) + if err != nil { + return nil, err + } + batchNumber = rpc.BlockNumber(highestBatchNo) + } + + batchNo := uint64(batchNumber.Int64()) + + _, found, err := hermezDb.GetLowestBlockInBatch(batchNo) + if err != nil { + return nil, err + } + if !found { + // not found - set to empty and append + bd.Empty = true + bds = append(bds, bd) + continue + } + + // block numbers in batch + blocksInBatch, err := hermezDb.GetL2BlockNosByBatch(batchNo) + if err != nil { + return nil, err + } + + // todo: max - take out shared logic with getBatchByNumber + // collect blocks in batch + var batchBlocks []*eritypes.Block + var batchTxs []eritypes.Transaction + // handle genesis - not in the hermez tables so requires special treament + if batchNumber == 0 { + blk, err := api.ethApi.BaseAPI.blockByNumberWithSenders(tx, 0) + if err != nil { + return nil, err + } + batchBlocks = append(batchBlocks, blk) + // no txs in genesis + } + for _, blkNo := range blocksInBatch { + blk, err := api.ethApi.BaseAPI.blockByNumberWithSenders(tx, blkNo) + if err != nil { + return nil, err + } + batchBlocks = append(batchBlocks, blk) + for _, btx := range blk.Transactions() { + batchTxs = append(batchTxs, btx) + } + } + + // batch l2 data - must build on the fly + forkId, err := hermezDb.GetForkId(batchNo) + if err != nil { + return nil, err + } + + // last batch last block for deltaTimestamp calc + lastBlockNoInPreviousBatch := batchBlocks[0].NumberU64() - 1 + lastBlockInPreviousBatch, err := rawdb.ReadBlockByNumber(tx, lastBlockNoInPreviousBatch) + if err != nil { + return nil, err + } + + var batchL2Data []byte + for i := 0; i < len(batchBlocks); i++ { + var dTs uint32 + if i == 0 { + dTs = uint32(batchBlocks[i].Time() - lastBlockInPreviousBatch.Time()) + } else { + dTs = uint32(batchBlocks[i].Time() - batchBlocks[i-1].Time()) + } + iti, err := hermezDb.GetBlockL1InfoTreeIndex(batchBlocks[i].NumberU64()) + + egTx := make(map[common.Hash]uint8) + for _, txn := range batchBlocks[i].Transactions() { + eg, err := hermezDb.GetEffectiveGasPricePercentage(txn.Hash()) + if err != nil { + return nil, err + } + egTx[txn.Hash()] = eg + } + + bl2d, err := zktx.GenerateBlockBatchL2Data(uint16(forkId), dTs, uint32(iti), batchBlocks[i].Transactions(), egTx) + if err != nil { + return nil, err + } + batchL2Data = append(batchL2Data, bl2d...) + } + bd.BatchL2Data = batchL2Data + bds = append(bds, bd) + } + + return populateBatchDataSlimDetails(bds) +} + // GetBatchByNumber returns a batch from the current canonical chain. If number is nil, the // latest known batch is returned. func (api *ZkEvmAPIImpl) GetBatchByNumber(ctx context.Context, batchNumber rpc.BlockNumber, fullTx *bool) (json.RawMessage, error) { @@ -252,14 +372,23 @@ func (api *ZkEvmAPIImpl) GetBatchByNumber(ctx context.Context, batchNumber rpc.B batchNumber = rpc.BlockNumber(highestBatchNo) } - bn := uint64(batchNumber.Int64()) + batchNo := uint64(batchNumber.Int64()) batch := &types.Batch{ - Number: types.ArgUint64(bn), + Number: types.ArgUint64(batchNo), + } + + // mimic zkevm node null response if we don't have the batch + _, found, err := hermezDb.GetLowestBlockInBatch(batchNo) + if err != nil { + return nil, err + } + if !found { + return nil, nil } // highest block in batch - blockNo, err := hermezDb.GetHighestBlockInBatch(bn) + blockNo, err := hermezDb.GetHighestBlockInBatch(batchNo) if err != nil { return nil, err } @@ -275,7 +404,7 @@ func (api *ZkEvmAPIImpl) GetBatchByNumber(ctx context.Context, batchNumber rpc.B batch.Timestamp = types.ArgUint64(block.Time()) // block numbers in batch - blocksInBatch, err := hermezDb.GetL2BlockNosByBatch(bn) + blocksInBatch, err := hermezDb.GetL2BlockNosByBatch(batchNo) if err != nil { return nil, err } @@ -308,19 +437,19 @@ func (api *ZkEvmAPIImpl) GetBatchByNumber(ctx context.Context, batchNumber rpc.B } } - //for consistency with leagacy node, return nil if no transactions + // for consistency with legacy node, return nil if no transactions if len(batch.Transactions) == 0 { batch.Transactions = nil } // global exit root of batch - batchGer, foundBatchGerNumber, err := hermezDb.GetLastBatchGlobalExitRoot(bn) + batchGer, foundBatchGerNumber, err := hermezDb.GetLastBatchGlobalExitRoot(batchNo) if err != nil { return nil, err } // get last block in batch - lastBlockInbatch, err := hermezDb.GetHighestBlockInBatch(bn) + lastBlockInbatch, err := hermezDb.GetHighestBlockInBatch(batchNo) if err != nil { return nil, err } @@ -350,17 +479,22 @@ func (api *ZkEvmAPIImpl) GetBatchByNumber(ctx context.Context, batchNumber rpc.B } // sequence - seq, err := hermezDb.GetSequenceByBatchNo(bn) + seq, err := hermezDb.GetSequenceByBatchNo(batchNo) if err != nil { return nil, err } if seq != nil { batch.SendSequencesTxHash = &seq.L1TxHash } - batch.Closed = seq != nil || bn == 0 || bn == 1 // sequenced, genesis or injected batch 1 - special batches 0,1 will always be closed + _, found, err = hermezDb.GetLowestBlockInBatch(batchNo + 1) + if err != nil { + return nil, err + } + // sequenced, genesis or injected batch 1 - special batches 0,1 will always be closed, if next batch has blocks, bn must be closed + batch.Closed = seq != nil || batchNo == 0 || batchNo == 1 || found // verification - ver, err := hermezDb.GetVerificationByBatchNo(bn) + ver, err := hermezDb.GetVerificationByBatchNo(batchNo) if err != nil { return nil, err } @@ -368,7 +502,7 @@ func (api *ZkEvmAPIImpl) GetBatchByNumber(ctx context.Context, batchNumber rpc.B batch.VerifyBatchTxHash = &ver.L1TxHash } - // exit roots + // exit roots (MainnetExitRoot, RollupExitRoot) infoTreeUpdate, err := hermezDb.GetL1InfoTreeUpdateByGer(batch.GlobalExitRoot) if err != nil { return nil, err @@ -378,8 +512,15 @@ func (api *ZkEvmAPIImpl) GetBatchByNumber(ctx context.Context, batchNumber rpc.B batch.RollupExitRoot = infoTreeUpdate.RollupExitRoot } + // local exit root + localExitRoot, err := utils.GetBatchLocalExitRoot(batchNo, hermezDb, tx) + if err != nil { + return nil, err + } + batch.LocalExitRoot = localExitRoot + // batch l2 data - must build on the fly - forkId, err := hermezDb.GetForkId(bn) + forkId, err := hermezDb.GetForkId(batchNo) if err != nil { return nil, err } @@ -425,8 +566,6 @@ func (api *ZkEvmAPIImpl) GetBatchByNumber(ctx context.Context, batchNumber rpc.B //} //batch.AccInputHash = oaih - //batch.LocalExitRoot = ver.LocalExitRoot - return populateBatchDetails(batch) } @@ -985,3 +1124,18 @@ func populateBatchDetails(batch *types.Batch) (json.RawMessage, error) { return json.Marshal(jBatch) } + +func populateBatchDataSlimDetails(batches []*types.BatchDataSlim) (json.RawMessage, error) { + jBatches := make([]map[string]interface{}, 0, len(batches)) + for _, b := range batches { + jBatch := map[string]interface{}{} + jBatch["number"] = b.Number + jBatch["empty"] = b.Empty + if !b.Empty { + jBatch["batchL2Data"] = b.BatchL2Data + } + jBatches = append(jBatches, jBatch) + } + + return json.Marshal(jBatches) +} diff --git a/eth/stagedsync/stage_execute_zkevm.go b/eth/stagedsync/stage_execute_zkevm.go index 37962251ca4..6a69f702b0d 100644 --- a/eth/stagedsync/stage_execute_zkevm.go +++ b/eth/stagedsync/stage_execute_zkevm.go @@ -113,6 +113,7 @@ func SpawnExecuteBlocksStageZk(s *StageState, u Unwinder, tx kv.RwTx, toBlock ui Loop: for blockNum := s.BlockNumber + 1; blockNum <= to; blockNum++ { if cfg.zk.SyncLimit > 0 && blockNum > cfg.zk.SyncLimit { + log.Info(fmt.Sprintf("[%s] Sync limit reached", s.LogPrefix()), "block", blockNum) break } diff --git a/hermezconfig-bali.yaml.example b/hermezconfig-bali.yaml.example index 577862a2cc1..65edf80526f 100644 --- a/hermezconfig-bali.yaml.example +++ b/hermezconfig-bali.yaml.example @@ -8,10 +8,10 @@ zkevm.l2-datastreamer-url: datastream.internal.zkevm-rpc.com:6900 zkevm.l1-chain-id: 11155111 zkevm.l1-rpc-url: https://rpc.sepolia.org -zkevm.address-sequencer: "" -zkevm.address-zkevm: "" -zkevm.address-admin: "0xE2EF6215aDc132Df6913C8DD16487aBF118d1764" -zkevm.address-rollup: "0x89BA0Ed947a88fe43c22Ae305C0713eC8a7Eb361" +zkevm.address-sequencer: "0x9aeCf44E36f20DC407d1A580630c9a2419912dcB" +zkevm.address-zkevm: "0x89BA0Ed947a88fe43c22Ae305C0713eC8a7Eb361" +zkevm.address-admin: "0x229A5bDBb09d8555f9214F7a6784804999BA4E0D" +zkevm.address-rollup: "0xE2EF6215aDc132Df6913C8DD16487aBF118d1764" zkevm.address-ger-manager: "0x2968D6d736178f8FE7393CC33C87f29D9C287e78" zkevm.default-gas-price: 10000000 diff --git a/turbo/cli/flags_zkevm.go b/turbo/cli/flags_zkevm.go index 6db6cd0f7f2..ddccbeb8a20 100644 --- a/turbo/cli/flags_zkevm.go +++ b/turbo/cli/flags_zkevm.go @@ -38,6 +38,22 @@ func ApplyFlagsForZkConfig(ctx *cli.Context, cfg *ethconfig.Config) { if v == 0 { panic(fmt.Sprintf("Flag not set: %s", flagName)) } + case []string: + if len(v) == 0 { + panic(fmt.Sprintf("Flag not set: %s", flagName)) + } + case libcommon.Address: + if v == (libcommon.Address{}) { + panic(fmt.Sprintf("Flag not set: %s", flagName)) + } + case time.Duration: + if v == 0 { + panic(fmt.Sprintf("Flag not set: %s", flagName)) + } + case bool: + // nothing to check + default: + panic(fmt.Sprintf("Unsupported type for flag check: %T", value)) } } @@ -145,7 +161,6 @@ func ApplyFlagsForZkConfig(ctx *cli.Context, cfg *ethconfig.Config) { if !sequencer.IsSequencer() { checkFlag(utils.L2RpcUrlFlag.Name, cfg.L2RpcUrl) checkFlag(utils.L2DataStreamerUrlFlag.Name, cfg.L2DataStreamerUrl) - checkFlag(utils.L2DataStreamerTimeout.Name, cfg.L2DataStreamerTimeout) } else { checkFlag(utils.SequencerInitialForkId.Name, cfg.SequencerInitialForkId) checkFlag(utils.ExecutorUrls.Name, cfg.ExecutorUrls) diff --git a/zk/datastream/server/data_stream_server.go b/zk/datastream/server/data_stream_server.go index 5b7829f9439..4ede7d808d4 100644 --- a/zk/datastream/server/data_stream_server.go +++ b/zk/datastream/server/data_stream_server.go @@ -6,11 +6,11 @@ import ( "github.com/0xPolygonHermez/zkevm-data-streamer/datastreamer" libcommon "github.com/gateway-fm/cdk-erigon-lib/common" "github.com/gateway-fm/cdk-erigon-lib/kv" - "github.com/ledgerwatch/erigon/core/state" eritypes "github.com/ledgerwatch/erigon/core/types" "github.com/ledgerwatch/erigon/zk/datastream/proto/github.com/0xPolygonHermez/zkevm-node/state/datastream" "github.com/ledgerwatch/erigon/zk/datastream/types" "github.com/ledgerwatch/erigon/zk/hermez_db" + "github.com/ledgerwatch/erigon/zk/utils" ) type BookmarkType byte @@ -274,7 +274,7 @@ func (srv *DataStreamServer) CreateStreamEntriesProto( } // seal off the last batch - localExitRoot, err := srv.getLocalExitRoot(workingBatch, reader, tx) + localExitRoot, err := utils.GetBatchLocalExitRoot(workingBatch, reader, tx) if err != nil { return nil, err } @@ -372,7 +372,7 @@ func (srv *DataStreamServer) CreateStreamEntriesProto( } } - localExitRoot, err := srv.getLocalExitRoot(batchNumber, reader, tx) + localExitRoot, err := utils.GetBatchLocalExitRoot(batchNumber, reader, tx) if err != nil { return nil, err } @@ -414,31 +414,6 @@ func (srv *DataStreamServer) addBatchStartEntries(reader *hermez_db.HermezDbRead return nil } -func (srv *DataStreamServer) getLocalExitRoot(batch uint64, reader *hermez_db.HermezDbReader, tx kv.Tx) (libcommon.Hash, error) { - // now to fetch the LER for the batch - based on the last block of the batch - var localExitRoot libcommon.Hash - if batch > 0 { - checkBatch := batch - for ; checkBatch > 0; checkBatch-- { - lastBlockNumber, err := reader.GetHighestBlockInBatch(checkBatch) - if err != nil { - return libcommon.Hash{}, err - } - stateReader := state.NewPlainState(tx, lastBlockNumber, nil) - rawLer, err := stateReader.ReadAccountStorage(state.GER_MANAGER_ADDRESS, 1, &state.GLOBAL_EXIT_ROOT_POS_1) - if err != nil { - return libcommon.Hash{}, err - } - stateReader.Close() - localExitRoot = libcommon.BytesToHash(rawLer) - if localExitRoot != (libcommon.Hash{}) { - break - } - } - } - return localExitRoot, nil -} - func (srv *DataStreamServer) CreateAndBuildStreamEntryBytesProto( block *eritypes.Block, reader *hermez_db.HermezDbReader, @@ -525,7 +500,7 @@ func (srv *DataStreamServer) UnwindToBlock(blockNumber uint64) error { if err != nil { return err } - if entry.Type == datastreamer.EntryType(3) { + if entry.Type == datastreamer.EntryType(datastream.EntryType_ENTRY_TYPE_L2_BLOCK) { break } entryNum -= 1 diff --git a/zk/datastream/server/datastream_populate.go b/zk/datastream/server/datastream_populate.go index 18aea4b72c3..356950761fa 100644 --- a/zk/datastream/server/datastream_populate.go +++ b/zk/datastream/server/datastream_populate.go @@ -13,6 +13,7 @@ import ( "github.com/ledgerwatch/erigon/zk/datastream/types" "github.com/ledgerwatch/erigon/zk/hermez_db" "github.com/ledgerwatch/log/v3" + "github.com/ledgerwatch/erigon/zk/utils" ) const GenesisForkId = 0 // genesis fork is always 0 in the datastream @@ -217,7 +218,7 @@ func WriteGenesisToStream( l2Block := srv.CreateL2BlockProto(genesis, genesis.Hash().Bytes(), batchNo, ger, 0, 0, common.Hash{}, 0, common.Hash{}) batchStart := srv.CreateBatchStartProto(batchNo, chainId, GenesisForkId, datastream.BatchType_BATCH_TYPE_REGULAR) - ler, err := srv.getLocalExitRoot(0, reader, tx) + ler, err := utils.GetBatchLocalExitRoot(0, reader, tx) if err != nil { return err } diff --git a/zk/hermez_db/db.go b/zk/hermez_db/db.go index 091bf1cf06b..1c690d5f344 100644 --- a/zk/hermez_db/db.go +++ b/zk/hermez_db/db.go @@ -45,6 +45,7 @@ const SMT_DEPTHS = "smt_depths" // block const L1_INFO_LEAVES = "l1_info_leaves" // l1 info tree index -> l1 info tree leaf const L1_INFO_ROOTS = "l1_info_roots" // root hash -> l1 info tree index const INVALID_BATCHES = "invalid_batches" // batch number -> true +const LOCAL_EXIT_ROOTS = "local_exit_roots" // l2 block number -> local exit root type HermezDb struct { tx kv.RwTx @@ -100,6 +101,7 @@ func CreateHermezBuckets(tx kv.RwTx) error { L1_INFO_LEAVES, L1_INFO_ROOTS, INVALID_BATCHES, + LOCAL_EXIT_ROOTS, } for _, t := range tables { if err := tx.CreateBucket(t); err != nil { @@ -1498,3 +1500,15 @@ func (db *HermezDbReader) GetInvalidBatch(batchNo uint64) (bool, error) { } return len(v) > 0, nil } + +func (db *HermezDb) WriteLocalExitRootForBatchNo(batchNo uint64, root common.Hash) error { + return db.tx.Put(LOCAL_EXIT_ROOTS, Uint64ToBytes(batchNo), root.Bytes()) +} + +func (db *HermezDbReader) GetLocalExitRootForBatchNo(batchNo uint64) (common.Hash, error) { + v, err := db.tx.GetOne(LOCAL_EXIT_ROOTS, Uint64ToBytes(batchNo)) + if err != nil { + return common.Hash{}, err + } + return common.BytesToHash(v), nil +} diff --git a/zk/rpcdaemon/types_zkevm.go b/zk/rpcdaemon/types_zkevm.go index 3262823f2d3..b3f1252b5c8 100644 --- a/zk/rpcdaemon/types_zkevm.go +++ b/zk/rpcdaemon/types_zkevm.go @@ -21,3 +21,9 @@ type Batch struct { Transactions []interface{} `json:"transactions"` BatchL2Data ArgBytes `json:"batchL2Data"` } + +type BatchDataSlim struct { + Number uint64 `json:"number"` + BatchL2Data ArgBytes `json:"batchL2Data,omitempty"` + Empty bool `json:"empty"` +} diff --git a/zk/stages/stage_batches.go b/zk/stages/stage_batches.go index e4e70ccbed4..89d84aa6488 100644 --- a/zk/stages/stage_batches.go +++ b/zk/stages/stage_batches.go @@ -69,6 +69,7 @@ type HermezDb interface { WriteIntermediateTxStateRoot(l2BlockNumber uint64, txHash common.Hash, rpcRoot common.Hash) error WriteBlockL1InfoTreeIndex(blockNumber uint64, l1Index uint64) error WriteLatestUsedGer(batchNo uint64, ger common.Hash) error + WriteLocalExitRootForBatchNo(batchNo uint64, localExitRoot common.Hash) error } type DatastreamClient interface { @@ -831,6 +832,13 @@ func writeL2Block(eriDb ErigonDb, hermezDb HermezDb, l2Block *types.FullL2Block, return fmt.Errorf("write ger for l1 block hash error: %v", err) } } + + // LER per batch - write the ler of the last block in the batch + if l2Block.BatchEnd && l2Block.LocalExitRoot != emptyHash { + if err := hermezDb.WriteLocalExitRootForBatchNo(l2Block.BatchNumber, l2Block.LocalExitRoot); err != nil { + return fmt.Errorf("write local exit root for l1 block hash error: %v", err) + } + } } } diff --git a/zk/stages/stage_sequence_execute.go b/zk/stages/stage_sequence_execute.go index db2fefe76ad..0e0bf060029 100644 --- a/zk/stages/stage_sequence_execute.go +++ b/zk/stages/stage_sequence_execute.go @@ -494,6 +494,16 @@ func SpawnSequencingStage( return err } + // Local Exit Root (ler): read s/c storage every batch to store the LER for the highest block in the batch + ler, err := utils.GetBatchLocalExitRootFromSCStorage(thisBatch, sdb.hermezDb.HermezDbReader, tx) + if err != nil { + return err + } + // write ler to hermezdb + if err = sdb.hermezDb.WriteLocalExitRootForBatchNo(thisBatch, ler); err != nil { + return err + } + log.Info(fmt.Sprintf("[%s] Finish batch %d...", logPrefix, thisBatch)) if freshTx { diff --git a/zk/utils/utils.go b/zk/utils/utils.go index 5946895ba75..4d9393a83e7 100644 --- a/zk/utils/utils.go +++ b/zk/utils/utils.go @@ -9,6 +9,8 @@ import ( "github.com/ledgerwatch/erigon/zk/constants" "github.com/ledgerwatch/erigon/zk/hermez_db" "github.com/ledgerwatch/log/v3" + libcommon "github.com/gateway-fm/cdk-erigon-lib/common" + "github.com/ledgerwatch/erigon/core/state" ) // if current sync is before verified batch - short circuit to verified batch, otherwise to enx of next batch @@ -123,3 +125,44 @@ func RecoverySetBlockConfigForks(blockNum uint64, forkId uint64, cfg ForkConfigW return nil } + +func GetBatchLocalExitRoot(batchNo uint64, db *hermez_db.HermezDbReader, tx kv.Tx) (libcommon.Hash, error) { + // check db first + localExitRoot, err := db.GetLocalExitRootForBatchNo(batchNo) + if err != nil { + return libcommon.Hash{}, err + } + + if localExitRoot != (libcommon.Hash{}) { + return localExitRoot, nil + } + + return GetBatchLocalExitRootFromSCStorage(batchNo, db, tx) +} + +func GetBatchLocalExitRootFromSCStorage(batchNo uint64, db *hermez_db.HermezDbReader, tx kv.Tx) (libcommon.Hash, error) { + var localExitRoot libcommon.Hash + + if batchNo > 0 { + checkBatch := batchNo + for ; checkBatch > 0; checkBatch-- { + blockNo, err := db.GetHighestBlockInBatch(checkBatch) + if err != nil { + return libcommon.Hash{}, err + } + stateReader := state.NewPlainState(tx, blockNo, nil) + rawLer, err := stateReader.ReadAccountStorage(state.GER_MANAGER_ADDRESS, 1, &state.GLOBAL_EXIT_ROOT_POS_1) + if err != nil { + stateReader.Close() + return libcommon.Hash{}, err + } + stateReader.Close() + localExitRoot = libcommon.BytesToHash(rawLer) + if localExitRoot != (libcommon.Hash{}) { + break + } + } + } + + return localExitRoot, nil +} From a302eef6ddad2c8efe3a9693a69f608a30ab8579 Mon Sep 17 00:00:00 2001 From: Valentin Staykov <79150443+V-Staykov@users.noreply.github.com> Date: Tue, 2 Jul 2024 14:09:10 +0300 Subject: [PATCH 08/27] 685 rpc debug tracetransaction memory fields contains extra zeroed memory values (#686) * added test log contract * don't print empty byte lines in trace memory after the last one with a value != 0 --- eth/tracers/logger/json_stream_zkevm.go | 36 ++++++++++++------- .../.gitignore | 0 .../README.md | 0 .../contracts/BigLoop.sol | 0 .../contracts/Counter.sol | 0 .../test-contracts/contracts/EmitLog.sol | 23 ++++++++++++ .../contracts/MulMod.sol | 0 .../hardhat.config.js.example | 0 .../package-lock.json | 0 .../package.json | 5 ++- .../scripts/bigloop.js | 0 .../scripts/counter.js | 0 .../test-contracts/scripts/emitlog.js | 26 ++++++++++++++ .../scripts/mulmod.js | 2 +- 14 files changed, 78 insertions(+), 14 deletions(-) rename zk/debug_tools/{modx-contract => test-contracts}/.gitignore (100%) rename zk/debug_tools/{modx-contract => test-contracts}/README.md (100%) rename zk/debug_tools/{modx-contract => test-contracts}/contracts/BigLoop.sol (100%) rename zk/debug_tools/{modx-contract => test-contracts}/contracts/Counter.sol (100%) create mode 100644 zk/debug_tools/test-contracts/contracts/EmitLog.sol rename zk/debug_tools/{modx-contract => test-contracts}/contracts/MulMod.sol (100%) rename zk/debug_tools/{modx-contract => test-contracts}/hardhat.config.js.example (100%) rename zk/debug_tools/{modx-contract => test-contracts}/package-lock.json (100%) rename zk/debug_tools/{modx-contract => test-contracts}/package.json (71%) rename zk/debug_tools/{modx-contract => test-contracts}/scripts/bigloop.js (100%) rename zk/debug_tools/{modx-contract => test-contracts}/scripts/counter.js (100%) create mode 100644 zk/debug_tools/test-contracts/scripts/emitlog.js rename zk/debug_tools/{modx-contract => test-contracts}/scripts/mulmod.js (90%) diff --git a/eth/tracers/logger/json_stream_zkevm.go b/eth/tracers/logger/json_stream_zkevm.go index 25571a9487e..a6d51a6201f 100644 --- a/eth/tracers/logger/json_stream_zkevm.go +++ b/eth/tracers/logger/json_stream_zkevm.go @@ -200,26 +200,38 @@ func (l *JsonStreamLogger_ZkEvm) writeMemory(memory *vm.Memory) { if !l.cfg.DisableMemory { memData := memory.Data() - //[zkevm] don't print empty bytes in memory array - isMemoryEmpty := true - for i := 0; i < len(memData); i++ { - if memData[i] != 0 { - isMemoryEmpty = false - break + //[zkevm] don't print empty bytes in memory array after the last non-empty byte line + filteredByteLines := [][]byte{} + foundValueLine := false + for i := len(memData); i-32 >= 0; i -= 32 { + bytes := memData[i-32 : i] + + isEmpty := true + if !foundValueLine { + for _, b := range bytes { + if b != 0 { + isEmpty = false + foundValueLine = true + break + } + } + } + + if !isEmpty || foundValueLine { + filteredByteLines = append(filteredByteLines, bytes) } } l.stream.WriteMore() l.stream.WriteObjectField("memory") l.stream.WriteArrayStart() - if !isMemoryEmpty { - for i := 0; i+32 <= len(memData); i += 32 { - if i > 0 { - l.stream.WriteMore() - } - l.stream.WriteString(string(l.hexEncodeBuf[0:hex.Encode(l.hexEncodeBuf[:], memData[i:i+32])])) + for i := len(filteredByteLines) - 1; i >= 0; i-- { + if i != len(filteredByteLines)-1 { + l.stream.WriteMore() } + l.stream.WriteString(string(l.hexEncodeBuf[0:hex.Encode(l.hexEncodeBuf[:], filteredByteLines[i])])) } + l.stream.WriteArrayEnd() } } diff --git a/zk/debug_tools/modx-contract/.gitignore b/zk/debug_tools/test-contracts/.gitignore similarity index 100% rename from zk/debug_tools/modx-contract/.gitignore rename to zk/debug_tools/test-contracts/.gitignore diff --git a/zk/debug_tools/modx-contract/README.md b/zk/debug_tools/test-contracts/README.md similarity index 100% rename from zk/debug_tools/modx-contract/README.md rename to zk/debug_tools/test-contracts/README.md diff --git a/zk/debug_tools/modx-contract/contracts/BigLoop.sol b/zk/debug_tools/test-contracts/contracts/BigLoop.sol similarity index 100% rename from zk/debug_tools/modx-contract/contracts/BigLoop.sol rename to zk/debug_tools/test-contracts/contracts/BigLoop.sol diff --git a/zk/debug_tools/modx-contract/contracts/Counter.sol b/zk/debug_tools/test-contracts/contracts/Counter.sol similarity index 100% rename from zk/debug_tools/modx-contract/contracts/Counter.sol rename to zk/debug_tools/test-contracts/contracts/Counter.sol diff --git a/zk/debug_tools/test-contracts/contracts/EmitLog.sol b/zk/debug_tools/test-contracts/contracts/EmitLog.sol new file mode 100644 index 00000000000..1b36be3fb96 --- /dev/null +++ b/zk/debug_tools/test-contracts/contracts/EmitLog.sol @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.15; + +contract EmitLog { + event Log(); + event LogA(uint256 indexed a); + event LogAB(uint256 indexed a, uint256 indexed b); + event LogABC(uint256 indexed a, uint256 indexed b, uint256 indexed c); + event LogABCD(uint256 indexed a, uint256 indexed b, uint256 indexed c, uint256 d); + + function emitLogs() public { + emit Log(); + emit LogA(1); + emit LogAB(1, 2); + emit LogABC(1, 2, 3); + emit LogABCD(1, 2, 3, 4); + emit LogABCD(4, 3, 2, 1); + emit LogABC(3, 2, 1); + emit LogAB(2, 1); + emit LogA(1); + emit Log(); + } +} \ No newline at end of file diff --git a/zk/debug_tools/modx-contract/contracts/MulMod.sol b/zk/debug_tools/test-contracts/contracts/MulMod.sol similarity index 100% rename from zk/debug_tools/modx-contract/contracts/MulMod.sol rename to zk/debug_tools/test-contracts/contracts/MulMod.sol diff --git a/zk/debug_tools/modx-contract/hardhat.config.js.example b/zk/debug_tools/test-contracts/hardhat.config.js.example similarity index 100% rename from zk/debug_tools/modx-contract/hardhat.config.js.example rename to zk/debug_tools/test-contracts/hardhat.config.js.example diff --git a/zk/debug_tools/modx-contract/package-lock.json b/zk/debug_tools/test-contracts/package-lock.json similarity index 100% rename from zk/debug_tools/modx-contract/package-lock.json rename to zk/debug_tools/test-contracts/package-lock.json diff --git a/zk/debug_tools/modx-contract/package.json b/zk/debug_tools/test-contracts/package.json similarity index 71% rename from zk/debug_tools/modx-contract/package.json rename to zk/debug_tools/test-contracts/package.json index f3144d273ce..7d223095400 100644 --- a/zk/debug_tools/modx-contract/package.json +++ b/zk/debug_tools/test-contracts/package.json @@ -9,7 +9,10 @@ "counter:local": "npx hardhat compile && npx hardhat run scripts/counter.js --network local", "counter:bali": "npx hardhat compile && npx hardhat run scripts/counter.js --network bali", "counter:cardona": "npx hardhat compile && npx hardhat run scripts/counter.js --network cardona", - "counter:mainnet": "npx hardhat compile && npx hardhat run scripts/counter.js --network mainnet" + "counter:mainnet": "npx hardhat compile && npx hardhat run scripts/counter.js --network mainnet", + "emitlog:bali": "npx hardhat compile && npx hardhat run scripts/emitlog.js --network bali", + "emitlog:cardona": "npx hardhat compile && npx hardhat run scripts/emitlog.js --network cardona", + "emitlog:mainnet": "npx hardhat compile && npx hardhat run scripts/emitlog.js --network mainnet" }, "keywords": [], "author": "", diff --git a/zk/debug_tools/modx-contract/scripts/bigloop.js b/zk/debug_tools/test-contracts/scripts/bigloop.js similarity index 100% rename from zk/debug_tools/modx-contract/scripts/bigloop.js rename to zk/debug_tools/test-contracts/scripts/bigloop.js diff --git a/zk/debug_tools/modx-contract/scripts/counter.js b/zk/debug_tools/test-contracts/scripts/counter.js similarity index 100% rename from zk/debug_tools/modx-contract/scripts/counter.js rename to zk/debug_tools/test-contracts/scripts/counter.js diff --git a/zk/debug_tools/test-contracts/scripts/emitlog.js b/zk/debug_tools/test-contracts/scripts/emitlog.js new file mode 100644 index 00000000000..921be881342 --- /dev/null +++ b/zk/debug_tools/test-contracts/scripts/emitlog.js @@ -0,0 +1,26 @@ +async function main() { +try { + // Get the ContractFactory of your SimpleContract + const SimpleContract = await hre.ethers.getContractFactory("EmitLog"); + + // Deploy the contract + const contract = await SimpleContract.deploy(); + // Wait for the deployment transaction to be mined + await contract.waitForDeployment(); + + console.log(`EmitLog deployed to: ${await contract.getAddress()}`); + + const result = await contract.emitLogs(); + console.log('emitLogs method call transaction: ', result.hash); + } catch (error) { + console.error(error); + process.exit(1); + } +} + +main() + .then(() => process.exit(0)) + .catch(error => { + console.error(error); + process.exit(1); + }); \ No newline at end of file diff --git a/zk/debug_tools/modx-contract/scripts/mulmod.js b/zk/debug_tools/test-contracts/scripts/mulmod.js similarity index 90% rename from zk/debug_tools/modx-contract/scripts/mulmod.js rename to zk/debug_tools/test-contracts/scripts/mulmod.js index 5e4b1d3697b..e9657786291 100644 --- a/zk/debug_tools/modx-contract/scripts/mulmod.js +++ b/zk/debug_tools/test-contracts/scripts/mulmod.js @@ -11,7 +11,7 @@ try { console.log(`MulMod deployed to: ${await contract.getAddress()}`); const result = await contract.modx(); - console.log(result); + console.log('modx method call transaction: ', result.hash); } catch (error) { console.error(error); process.exit(1); From 393e639e7c6039ba8495f46201390098a933647a Mon Sep 17 00:00:00 2001 From: Max Revitt Date: Tue, 2 Jul 2024 13:06:19 +0100 Subject: [PATCH 09/27] Nightly Test Tweaks (#684) * test(nightly): add jobs to nightly html page * tweak(ci): split build/run, exitcode 1 on timeout --- .github/workflows/nightly-l1-recovery.yml | 5 ++++- docs/nightly.html | 5 ++++- zk/debug_tools/nightly-block-compare-wait/main.go | 4 ++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/nightly-l1-recovery.yml b/.github/workflows/nightly-l1-recovery.yml index 05efbd59412..0038e72b7de 100644 --- a/.github/workflows/nightly-l1-recovery.yml +++ b/.github/workflows/nightly-l1-recovery.yml @@ -21,6 +21,9 @@ jobs: - name: Set up QEMU uses: docker/setup-qemu-action@v2 + - name: Build Docker Images + run: docker compose -f docker-compose-nightly.yml build + - name: Run Docker Compose working-directory: ./zk/tests/nightly-l1-recovery - run: docker compose -f docker-compose-nightly.yml up --build --exit-code-from=block-checker + run: docker compose -f docker-compose-nightly.yml up --exit-code-from=block-checker diff --git a/docs/nightly.html b/docs/nightly.html index 8877f429c3e..f17ad5f27d1 100644 --- a/docs/nightly.html +++ b/docs/nightly.html @@ -65,8 +65,11 @@

Nightly GitHub Actions Status