From 8577d29bf41b2eff49dce524f31024bcc239c92d Mon Sep 17 00:00:00 2001 From: Max Revitt Date: Mon, 15 Jul 2024 08:16:27 +0100 Subject: [PATCH 01/19] docs(readme): debug.timers flag docs and examples comments (#788) --- README.md | 1 + hermezconfig-bali.yaml.example | 2 ++ hermezconfig-cardona.yaml.example | 2 ++ hermezconfig-mainnet.yaml.example | 2 ++ 4 files changed, 7 insertions(+) diff --git a/README.md b/README.md index 7165c500580..a561d9507d7 100644 --- a/README.md +++ b/README.md @@ -192,6 +192,7 @@ Resource Utilisation config: Useful config entries: - `zkevm.sync-limit`: This will ensure the network only syncs to a given block height. +- `debug.timers`: This will enable debug timers in the logs to help with performance tuning. Recording timings of witness generation, etc. at INFO level. *** diff --git a/hermezconfig-bali.yaml.example b/hermezconfig-bali.yaml.example index 4901a1687cf..ab619354838 100644 --- a/hermezconfig-bali.yaml.example +++ b/hermezconfig-bali.yaml.example @@ -25,6 +25,8 @@ txpool.disable: true torrent.port: 42070 zkevm.datastream-version: 2 +# debug.timers: true # Uncomment to enable timers + externalcl: true http.api: [eth, debug, net, trace, web3, erigon, zkevm] http.addr: 0.0.0.0 diff --git a/hermezconfig-cardona.yaml.example b/hermezconfig-cardona.yaml.example index 4434274bce1..69236d04898 100644 --- a/hermezconfig-cardona.yaml.example +++ b/hermezconfig-cardona.yaml.example @@ -27,6 +27,8 @@ txpool.disable: true torrent.port: 42070 zkevm.datastream-version: 2 +# debug.timers: true # Uncomment to enable timers + externalcl: true http.api: [eth, debug, net, trace, web3, erigon, zkevm] http.addr: 0.0.0.0 diff --git a/hermezconfig-mainnet.yaml.example b/hermezconfig-mainnet.yaml.example index a49d97208f7..8342b11d755 100644 --- a/hermezconfig-mainnet.yaml.example +++ b/hermezconfig-mainnet.yaml.example @@ -25,6 +25,8 @@ zkevm.l1-first-block: 16896700 zkevm.rpc-ratelimit: 250 zkevm.datastream-version: 2 +# debug.timers: true # Uncomment to enable timers + externalcl: true http.api: [eth, debug, net, trace, web3, erigon, zkevm] http.addr: 0.0.0.0 From 8d43b89a79c617fa6d70ead6885f2351a0bd01ba Mon Sep 17 00:00:00 2001 From: Kamen Stoykov <24619432+kstoykov@users.noreply.github.com> Date: Mon, 15 Jul 2024 11:24:00 +0300 Subject: [PATCH 02/19] query-l1-headers (#784) * query-l1-headers * add a condition --- zk/syncer/l1_syncer.go | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/zk/syncer/l1_syncer.go b/zk/syncer/l1_syncer.go index ba510a39983..36504705b43 100644 --- a/zk/syncer/l1_syncer.go +++ b/zk/syncer/l1_syncer.go @@ -224,18 +224,19 @@ func (s *L1Syncer) GetOldAccInputHash(ctx context.Context, addr *common.Address, } func (s *L1Syncer) L1QueryHeaders(logs []ethTypes.Log) (map[uint64]*ethTypes.Header, error) { - // more thread causes error on remote rpc server - headers := make([]*ethTypes.Header, 0) + logsSize := len(logs) // queue up all the logs - logQueue := make(chan ethTypes.Log, len(logs)) + logQueue := make(chan *ethTypes.Log, logsSize) defer close(logQueue) - for i := 0; i < len(logs); i++ { - logQueue <- logs[i] + for i := 0; i < logsSize; i++ { + logQueue <- &logs[i] } var wg sync.WaitGroup - wg.Add(len(logs)) + wg.Add(logsSize) + + headersQueue := make(chan *ethTypes.Header, logsSize) process := func(em IEtherman) { ctx := context.Background() @@ -252,7 +253,7 @@ func (s *L1Syncer) L1QueryHeaders(logs []ethTypes.Log) (map[uint64]*ethTypes.Hea logQueue <- l continue } - headers = append(headers, header) + headersQueue <- header wg.Done() } } @@ -265,10 +266,11 @@ func (s *L1Syncer) L1QueryHeaders(logs []ethTypes.Log) (map[uint64]*ethTypes.Hea } wg.Wait() + close(headersQueue) headersMap := map[uint64]*ethTypes.Header{} - for i := 0; i < len(headers); i++ { - headersMap[headers[i].Number.Uint64()] = headers[i] + for header := range headersQueue { + headersMap[header.Number.Uint64()] = header } return headersMap, nil From b2d303e51f38df7338b2a58333f0997ce7c8b3a8 Mon Sep 17 00:00:00 2001 From: Valentin Staykov <79150443+V-Staykov@users.noreply.github.com> Date: Mon, 15 Jul 2024 12:28:33 +0300 Subject: [PATCH 03/19] set path array capascity to remove resizes (#791) --- smt/pkg/utils/utils.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/smt/pkg/utils/utils.go b/smt/pkg/utils/utils.go index a39bc1b35ab..9f6386b54de 100644 --- a/smt/pkg/utils/utils.go +++ b/smt/pkg/utils/utils.go @@ -10,9 +10,9 @@ import ( "sort" - poseidon "github.com/gateway-fm/vectorized-poseidon-gold/src/vectorizedposeidongold" "github.com/gateway-fm/cdk-erigon-lib/common" "github.com/gateway-fm/cdk-erigon-lib/common/length" + poseidon "github.com/gateway-fm/vectorized-poseidon-gold/src/vectorizedposeidongold" ) const ( @@ -365,7 +365,7 @@ func ScalarToNodeValue8(scalarIn *big.Int) NodeValue8 { } func (nk *NodeKey) GetPath() []int { - res := make([]int, 0) + res := make([]int, 0, 256) auxk := [4]uint64{nk[0], nk[1], nk[2], nk[3]} for j := 0; j < 64; j++ { From a2826597582fb109d0900de808763e896a1891c0 Mon Sep 17 00:00:00 2001 From: Valentin Staykov <79150443+V-Staykov@users.noreply.github.com> Date: Mon, 15 Jul 2024 17:39:03 +0300 Subject: [PATCH 04/19] do not reinitialize state reader on each block for sequencer (#782) --- core/state/plain_readonly_storage_zkevm.go | 85 +++++++++++++++++++ zk/datastream/server/data_stream_server.go | 2 + .../server/data_stream_server_utils.go | 31 +------ zk/utils/utils.go | 25 ++++-- 4 files changed, 108 insertions(+), 35 deletions(-) create mode 100644 core/state/plain_readonly_storage_zkevm.go diff --git a/core/state/plain_readonly_storage_zkevm.go b/core/state/plain_readonly_storage_zkevm.go new file mode 100644 index 00000000000..346f38c4ff6 --- /dev/null +++ b/core/state/plain_readonly_storage_zkevm.go @@ -0,0 +1,85 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package state + +import ( + "fmt" + + libcommon "github.com/gateway-fm/cdk-erigon-lib/common" + "github.com/gateway-fm/cdk-erigon-lib/kv" + "github.com/gateway-fm/cdk-erigon-lib/kv/kvcfg" + + "github.com/ledgerwatch/erigon/common/dbutils" + "github.com/ledgerwatch/erigon/core/state/historyv2read" +) + +// State at the beginning of blockNr +type PlainStateReadAccountStorage struct { + storageHistoryC kv.Cursor + storageChangesC kv.CursorDupSort + tx kv.Tx + blockNr uint64 + trace bool +} + +func NewPlainStateReadAccountStorage(tx kv.Tx, blockNr uint64) *PlainStateReadAccountStorage { + histV3, _ := kvcfg.HistoryV3.Enabled(tx) + if histV3 { + panic("Please use HistoryStateReaderV3 with HistoryV3") + } + ps := &PlainStateReadAccountStorage{ + tx: tx, + blockNr: blockNr, + } + + c2, _ := tx.Cursor(kv.StorageHistory) + c4, _ := tx.CursorDupSort(kv.StorageChangeSet) + + ps.storageHistoryC = c2 + ps.storageChangesC = c4 + return ps +} + +func (s *PlainStateReadAccountStorage) Close() { + s.storageHistoryC.Close() + s.storageChangesC.Close() +} + +func (s *PlainStateReadAccountStorage) SetBlockNr(blockNr uint64) { + s.blockNr = blockNr + s.storageChangesC.First() + s.storageHistoryC.First() +} + +func (s *PlainStateReadAccountStorage) SetTrace(trace bool) { + s.trace = trace +} + +func (s *PlainStateReadAccountStorage) ReadAccountStorage(address libcommon.Address, incarnation uint64, key *libcommon.Hash) ([]byte, error) { + compositeKey := dbutils.PlainGenerateCompositeStorageKey(address.Bytes(), incarnation, key.Bytes()) + enc, err := historyv2read.GetAsOf(s.tx, s.storageHistoryC, s.storageChangesC, true /* storage */, compositeKey, s.blockNr) + if err != nil { + return nil, err + } + if s.trace { + fmt.Printf("ReadAccountStorage [%x] [%x] => [%x]\n", address, *key, enc) + } + if len(enc) == 0 { + return nil, nil + } + return enc, nil +} diff --git a/zk/datastream/server/data_stream_server.go b/zk/datastream/server/data_stream_server.go index 91d5f34900f..367e63f07c2 100644 --- a/zk/datastream/server/data_stream_server.go +++ b/zk/datastream/server/data_stream_server.go @@ -4,6 +4,7 @@ import ( "github.com/0xPolygonHermez/zkevm-data-streamer/datastreamer" zktypes "github.com/ledgerwatch/erigon/zk/types" + "github.com/gateway-fm/cdk-erigon-lib/common" libcommon "github.com/gateway-fm/cdk-erigon-lib/common" "github.com/gateway-fm/cdk-erigon-lib/kv" eritypes "github.com/ledgerwatch/erigon/core/types" @@ -14,6 +15,7 @@ import ( ) type DbReader interface { + GetLocalExitRootForBatchNo(batchNo uint64) (common.Hash, error) GetBatchGlobalExitRootsProto(lastBatchNumber, batchNumber uint64) ([]types.GerUpdateProto, error) GetForkId(batchNumber uint64) (uint64, error) GetBlockGlobalExitRoot(blockNumber uint64) (libcommon.Hash, error) diff --git a/zk/datastream/server/data_stream_server_utils.go b/zk/datastream/server/data_stream_server_utils.go index cb13c7d30fd..8395aaa499b 100644 --- a/zk/datastream/server/data_stream_server_utils.go +++ b/zk/datastream/server/data_stream_server_utils.go @@ -6,10 +6,10 @@ import ( libcommon "github.com/gateway-fm/cdk-erigon-lib/common" "github.com/gateway-fm/cdk-erigon-lib/kv" - "github.com/ledgerwatch/erigon/core/state" eritypes "github.com/ledgerwatch/erigon/core/types" "github.com/ledgerwatch/erigon/zk/datastream/proto/github.com/0xPolygonHermez/zkevm-node/state/datastream" "github.com/ledgerwatch/erigon/zk/datastream/types" + "github.com/ledgerwatch/erigon/zk/utils" ) func newBatchBookmarkEntryProto(batchNo uint64) *types.BookmarkProto { @@ -164,7 +164,7 @@ func createBatchStartEntriesProto( } // seal off the last batch - if localExitRoot, err = getLocalExitRoot(workingBatch, reader, tx); err != nil { + if localExitRoot, err = utils.GetBatchLocalExitRootFromSCStorage(workingBatch, reader, tx); err != nil { return nil, err } entries = append(entries, newBatchEndProto(localExitRoot, root, workingBatch)) @@ -196,7 +196,7 @@ func addBatchEndEntriesProto( } } - localExitRoot, err := getLocalExitRoot(batchNumber, reader, tx) + localExitRoot, err := utils.GetBatchLocalExitRootFromSCStorage(batchNumber, reader, tx) if err != nil { return nil, err } @@ -235,31 +235,6 @@ func addBatchStartEntries(reader DbReader, batchNum, chainId uint64) ([]DataStre return entries, nil } -func getLocalExitRoot(batch uint64, reader DbReader, tx kv.Tx) (libcommon.Hash, error) { - // now to fetch the LER for the batch - based on the last block of the batch - var localExitRoot libcommon.Hash - if batch > 0 { - checkBatch := batch - for ; checkBatch > 0; checkBatch-- { - lastBlockNumber, err := reader.GetHighestBlockInBatch(checkBatch) - if err != nil { - return libcommon.Hash{}, err - } - stateReader := state.NewPlainState(tx, lastBlockNumber, nil) - rawLer, err := stateReader.ReadAccountStorage(state.GER_MANAGER_ADDRESS, 1, &state.GLOBAL_EXIT_ROOT_POS_1) - if err != nil { - return libcommon.Hash{}, err - } - stateReader.Close() - localExitRoot = libcommon.BytesToHash(rawLer) - if localExitRoot != (libcommon.Hash{}) { - break - } - } - } - return localExitRoot, nil -} - func filterTransactionByIndexes( filteredTransactions eritypes.Transactions, transactionsToIncludeByIndex []int, diff --git a/zk/utils/utils.go b/zk/utils/utils.go index 4d9393a83e7..32d144444b6 100644 --- a/zk/utils/utils.go +++ b/zk/utils/utils.go @@ -3,14 +3,15 @@ package utils import ( "fmt" + "github.com/gateway-fm/cdk-erigon-lib/common" + libcommon "github.com/gateway-fm/cdk-erigon-lib/common" "github.com/gateway-fm/cdk-erigon-lib/kv" "github.com/ledgerwatch/erigon/chain" + "github.com/ledgerwatch/erigon/core/state" "github.com/ledgerwatch/erigon/eth/stagedsync/stages" "github.com/ledgerwatch/erigon/zk/constants" "github.com/ledgerwatch/erigon/zk/hermez_db" "github.com/ledgerwatch/log/v3" - libcommon "github.com/gateway-fm/cdk-erigon-lib/common" - "github.com/ledgerwatch/erigon/core/state" ) // if current sync is before verified batch - short circuit to verified batch, otherwise to enx of next batch @@ -80,6 +81,11 @@ type ForkConfigWriter interface { SetForkIdBlock(forkId constants.ForkId, blockNum uint64) error } +type DbReader interface { + GetLocalExitRootForBatchNo(batchNo uint64) (common.Hash, error) + GetHighestBlockInBatch(batchNo uint64) (uint64, error) +} + func UpdateZkEVMBlockCfg(cfg ForkConfigWriter, hermezDb ForkReader, logPrefix string) error { var lastSetBlockNum uint64 = 0 var foundAny bool = false @@ -126,7 +132,7 @@ func RecoverySetBlockConfigForks(blockNum uint64, forkId uint64, cfg ForkConfigW return nil } -func GetBatchLocalExitRoot(batchNo uint64, db *hermez_db.HermezDbReader, tx kv.Tx) (libcommon.Hash, error) { +func GetBatchLocalExitRoot(batchNo uint64, db DbReader, tx kv.Tx) (libcommon.Hash, error) { // check db first localExitRoot, err := db.GetLocalExitRootForBatchNo(batchNo) if err != nil { @@ -140,23 +146,28 @@ func GetBatchLocalExitRoot(batchNo uint64, db *hermez_db.HermezDbReader, tx kv.T return GetBatchLocalExitRootFromSCStorage(batchNo, db, tx) } -func GetBatchLocalExitRootFromSCStorage(batchNo uint64, db *hermez_db.HermezDbReader, tx kv.Tx) (libcommon.Hash, error) { +func GetBatchLocalExitRootFromSCStorage(batchNo uint64, db DbReader, tx kv.Tx) (libcommon.Hash, error) { var localExitRoot libcommon.Hash if batchNo > 0 { checkBatch := batchNo + + stateReader := state.NewPlainStateReadAccountStorage(tx, 0) + defer stateReader.Close() + for ; checkBatch > 0; checkBatch-- { blockNo, err := db.GetHighestBlockInBatch(checkBatch) if err != nil { return libcommon.Hash{}, err } - stateReader := state.NewPlainState(tx, blockNo, nil) + + // stateReader := state.NewPlainStateReadAccountStorage(tx, blockNo) + // defer stateReader.Close() + stateReader.SetBlockNr(blockNo) rawLer, err := stateReader.ReadAccountStorage(state.GER_MANAGER_ADDRESS, 1, &state.GLOBAL_EXIT_ROOT_POS_1) if err != nil { - stateReader.Close() return libcommon.Hash{}, err } - stateReader.Close() localExitRoot = libcommon.BytesToHash(rawLer) if localExitRoot != (libcommon.Hash{}) { break From 4e8c7dddae8b5806a89607f62e1ce5adb1d61778 Mon Sep 17 00:00:00 2001 From: Louis Liu <35095310+louisliu2048@users.noreply.github.com> Date: Mon, 15 Jul 2024 22:43:00 +0800 Subject: [PATCH 05/19] update zkevm-data-streamer version to solve the deadlock bug introduced by datastream conn write timeout (#767) Co-authored-by: Valentin Staykov <79150443+V-Staykov@users.noreply.github.com> --- cmd/erigon-el/backend/backend.go | 2 +- cmd/rpcdaemon/cli/httpcfg/http_cfg.go | 7 ++++--- cmd/utils/flags.go | 5 +++++ eth/backend.go | 2 +- eth/ethconfig/config_zkevm.go | 1 + go.mod | 3 +-- go.sum | 18 ++---------------- turbo/cli/default_flags.go | 1 + turbo/cli/flags.go | 7 ++++--- turbo/cli/flags_zkevm.go | 2 ++ zk/debug_tools/datastream-host/main.go | 8 +++++--- 11 files changed, 27 insertions(+), 29 deletions(-) diff --git a/cmd/erigon-el/backend/backend.go b/cmd/erigon-el/backend/backend.go index 5834bb06c42..ae38fdcc3ce 100644 --- a/cmd/erigon-el/backend/backend.go +++ b/cmd/erigon-el/backend/backend.go @@ -667,7 +667,7 @@ func NewBackend(stack *node.Node, config *ethconfig.Config, logger log.Logger) ( Level: "warn", Outputs: nil, } - backend.dataStream, err = datastreamer.NewServer(uint16(httpRpcCfg.DataStreamPort), uint8(2), 1, datastreamer.StreamType(1), file, logConfig) + backend.dataStream, err = datastreamer.NewServer(uint16(httpRpcCfg.DataStreamPort), uint8(2), 1, datastreamer.StreamType(1), file, httpRpcCfg.DataStreamWriteTimeout, logConfig) if err != nil { return nil, err } diff --git a/cmd/rpcdaemon/cli/httpcfg/http_cfg.go b/cmd/rpcdaemon/cli/httpcfg/http_cfg.go index 7da1f9b5176..45dd7dc9e90 100644 --- a/cmd/rpcdaemon/cli/httpcfg/http_cfg.go +++ b/cmd/rpcdaemon/cli/httpcfg/http_cfg.go @@ -66,7 +66,8 @@ type HttpCfg struct { ReturnDataLimit int // Maximum number of bytes returned from calls (like eth_call) // zkevm - DataStreamPort int - DataStreamHost string - L2RpcUrl string + DataStreamPort int + DataStreamHost string + DataStreamWriteTimeout time.Duration + L2RpcUrl string } diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 4cb1c1a20be..4ca7d6e693b 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -521,6 +521,11 @@ var ( Usage: "Define the host used for the zkevm data stream", Value: "", } + DataStreamWriteTimeout = cli.DurationFlag{ + Name: "zkevm.data-stream-writeTimeout", + Usage: "Define the TCP write timeout when sending data to a datastream client", + Value: 5 * time.Second, + } Limbo = cli.BoolFlag{ Name: "zkevm.limbo", Usage: "Enable limbo processing on batches that failed verification", diff --git a/eth/backend.go b/eth/backend.go index 73b10a9f351..3be28bef3c6 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -728,7 +728,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { Outputs: nil, } // todo [zkevm] read the stream version from config and figure out what system id is used for - backend.dataStream, err = datastreamer.NewServer(uint16(httpCfg.DataStreamPort), uint8(backend.config.DatastreamVersion), 1, datastreamer.StreamType(1), file, logConfig) + backend.dataStream, err = datastreamer.NewServer(uint16(httpCfg.DataStreamPort), uint8(backend.config.DatastreamVersion), 1, datastreamer.StreamType(1), file, httpCfg.DataStreamWriteTimeout, logConfig) if err != nil { return nil, err } diff --git a/eth/ethconfig/config_zkevm.go b/eth/ethconfig/config_zkevm.go index 748d67d2e5b..4941420e9fc 100644 --- a/eth/ethconfig/config_zkevm.go +++ b/eth/ethconfig/config_zkevm.go @@ -49,6 +49,7 @@ type Zk struct { DAUrl string DataStreamHost string DataStreamPort uint + DataStreamWriteTimeout time.Duration RebuildTreeAfter uint64 IncrementTreeAlways bool diff --git a/go.mod b/go.mod index 7fd513b2374..814bbb63ade 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( ) require ( - github.com/0xPolygonHermez/zkevm-data-streamer v0.2.2 + github.com/0xPolygonHermez/zkevm-data-streamer v0.2.3-RC4 github.com/99designs/gqlgen v0.17.29 github.com/Giulio2002/bls v0.0.0-20230217173148-c87a29266b6c github.com/RoaringBitmap/roaring v1.2.3 @@ -158,7 +158,6 @@ require ( github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/gopacket v1.1.19 // indirect github.com/google/pprof v0.0.0-20230207041349-798e818bf904 // indirect diff --git a/go.sum b/go.sum index e39079cd552..e74fb82299d 100644 --- a/go.sum +++ b/go.sum @@ -12,10 +12,9 @@ dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBr dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4= dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU= filippo.io/edwards25519 v1.0.0-rc.1 h1:m0VOOB23frXZvAOK44usCgLWvtsxIoMCTBGJZlpmGfU= -filippo.io/edwards25519 v1.0.0-rc.1/go.mod h1:N1IkdkCkiLB6tki+MYJoSx2JTY9NUlxZE7eHn5EwJns= git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= -github.com/0xPolygonHermez/zkevm-data-streamer v0.2.2 h1:XRMTk+W6vtJVGVjuEznfWyNt7HkRkkuSmlN5Y6p60Sc= -github.com/0xPolygonHermez/zkevm-data-streamer v0.2.2/go.mod h1:0QkAXcFa92mFJrCbN3UPUJGJYes851yEgYHLONnaosE= +github.com/0xPolygonHermez/zkevm-data-streamer v0.2.3-RC4 h1:+4K+xSzv0ImbK30B/T9FauNTrTFUmWcNKYhIgwsE4C4= +github.com/0xPolygonHermez/zkevm-data-streamer v0.2.3-RC4/go.mod h1:0QkAXcFa92mFJrCbN3UPUJGJYes851yEgYHLONnaosE= github.com/99designs/gqlgen v0.17.29 h1:z2MrNOFATCVgQLRCF6Uufz4uz2IQLB5BBMwPUMafJOA= github.com/99designs/gqlgen v0.17.29/go.mod h1:i4rEatMrzzu6RXaHydq1nmEPZkb3bKQsnxNRHS4DQB4= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -46,7 +45,6 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alexflint/go-scalar v1.1.0/go.mod h1:LoFvNMqS1CPrMVltza4LvnGKhaSpc3oyLEBUZVhhS2o= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/anacrolix/chansync v0.3.0 h1:lRu9tbeuw3wl+PhMu/r+JJCRu5ArFXIluOgdF0ao6/U= @@ -58,7 +56,6 @@ github.com/anacrolix/envpprof v1.0.0/go.mod h1:KgHhUaQMc8cC0+cEflSgCFNFbKwi5h54g github.com/anacrolix/envpprof v1.1.0/go.mod h1:My7T5oSqVfEn4MD4Meczkw/f5lSIndGAKu/0SM/rkf4= github.com/anacrolix/envpprof v1.2.1 h1:25TJe6t/i0AfzzldiGFKCpD+s+dk8lONBcacJZB2rdE= github.com/anacrolix/envpprof v1.2.1/go.mod h1:My7T5oSqVfEn4MD4Meczkw/f5lSIndGAKu/0SM/rkf4= -github.com/anacrolix/fuse v0.2.0/go.mod h1:Kfu02xBwnySDpH3N23BmrP3MDfwAQGRLUCj6XyeOvBQ= github.com/anacrolix/generics v0.0.0-20220618083756-f99e35403a60 h1:k4/h2B1gGF+PJGyGHxs8nmHHt1pzWXZWBj6jn4OBlRc= github.com/anacrolix/generics v0.0.0-20220618083756-f99e35403a60/go.mod h1:ff2rHB/joTV03aMSSn/AZNnaIpUw0h3njetGsaXcMy8= github.com/anacrolix/go-libutp v1.2.0 h1:sjxoB+/ARiKUR7IK/6wLWyADIBqGmu1fm0xo+8Yy7u0= @@ -89,7 +86,6 @@ github.com/anacrolix/mmsg v1.0.0 h1:btC7YLjOn29aTUAExJiVUhQOuf/8rhm+/nWCMAnL3Hg= github.com/anacrolix/mmsg v1.0.0/go.mod h1:x8kRaJY/dCrY9Al0PEcj1mb/uFHwP6GCJ9fLl4thEPc= github.com/anacrolix/multiless v0.3.0 h1:5Bu0DZncjE4e06b9r1Ap2tUY4Au0NToBP5RpuEngSis= github.com/anacrolix/multiless v0.3.0/go.mod h1:TrCLEZfIDbMVfLoQt5tOoiBS/uq4y8+ojuEVVvTNPX4= -github.com/anacrolix/publicip v0.2.0/go.mod h1:67G1lVkLo8UjdEcJkwScWVTvlJ35OCDsRJoWXl/wi4g= github.com/anacrolix/stm v0.2.0/go.mod h1:zoVQRvSiGjGoTmbM0vSLIiaKjWtNPeTvXUSdJQA4hsg= github.com/anacrolix/stm v0.4.0 h1:tOGvuFwaBjeu1u9X1eIh9TX8OEedEiEQ1se1FjhFnXY= github.com/anacrolix/stm v0.4.0/go.mod h1:GCkwqWoAsP7RfLW+jw+Z0ovrt2OO7wRzcTtFYMYY5t8= @@ -100,7 +96,6 @@ github.com/anacrolix/sync v0.4.0/go.mod h1:BbecHL6jDSExojhNtgTFSBcdGerzNc64tz3DC github.com/anacrolix/tagflag v0.0.0-20180109131632-2146c8d41bf0/go.mod h1:1m2U/K6ZT+JZG0+bdMK6qauP49QT4wE5pmhJXOKKCHw= github.com/anacrolix/tagflag v1.0.0/go.mod h1:1m2U/K6ZT+JZG0+bdMK6qauP49QT4wE5pmhJXOKKCHw= github.com/anacrolix/tagflag v1.1.0/go.mod h1:Scxs9CV10NQatSmbyjqmqmeQNwGzlNe0CMUMIxqHIG8= -github.com/anacrolix/tagflag v1.3.0/go.mod h1:Scxs9CV10NQatSmbyjqmqmeQNwGzlNe0CMUMIxqHIG8= github.com/anacrolix/torrent v1.48.1-0.20230219022425-e8971ea0f1bf h1:gQCApNMI+lbXYLRiiiC5S2mU9k2BZT9FNnRr//eUzXc= github.com/anacrolix/torrent v1.48.1-0.20230219022425-e8971ea0f1bf/go.mod h1:5OY82KVPu5Fq+P0HefdTQKRt0gfBXeHeRUE04VaSoQo= github.com/anacrolix/upnp v0.1.3-0.20220123035249-922794e51c96 h1:QAVZ3pN/J4/UziniAhJR2OZ9Ox5kOY2053tBbbqUPYA= @@ -186,7 +181,6 @@ github.com/deckarep/golang-set v1.8.0/go.mod h1:5nI87KwE7wgsBU1F4GKAw2Qod7p5kyS3 github.com/deckarep/golang-set/v2 v2.3.0 h1:qs18EKUfHm2X9fA50Mr/M5hccg2tNnVqsiBImnyDs0g= github.com/deckarep/golang-set/v2 v2.3.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0= -github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1owhMVTHFZIlnvd4= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2UO1+lbSKsdiOoYi9Zzey7Fc= github.com/dgravesa/go-parallel v0.6.0 h1:3KU5uzHn92Upq9jPVQ+ILLechgUKigCo72TScC7U3qE= @@ -219,7 +213,6 @@ github.com/edsrzf/mmap-go v1.1.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8E github.com/elastic/gosigar v0.12.0/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= github.com/elastic/gosigar v0.14.2 h1:Dg80n8cr90OZ7x+bAax/QjoW/XqTI11RmA79ZwIm9/4= github.com/elastic/gosigar v0.14.2/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= -github.com/elliotchance/orderedmap v1.4.0/go.mod h1:wsDwEaX5jEoyhbs7x93zk2H/qv0zwuhg4inXhDkYqys= github.com/emicklei/dot v1.0.0 h1:yyObALINBOuI1GdCRwVea2IPtGtVgh0NQgJDrE03Tqc= github.com/emicklei/dot v1.0.0/go.mod h1:DeV7GvQtIw4h2u73RKBkkFdvVAz0D9fzeJrgPW6gy/s= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= @@ -271,7 +264,6 @@ github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vb github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -315,8 +307,6 @@ github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3 h1:zN2lZNZRflqFyxVaTIU61KNKQ9C0055u9CAfpmqUvo4= -github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3/go.mod h1:nPpo7qLxd6XL3hWJG/O60sR8ZKfMCiIoNap5GvD12KU= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -482,8 +472,6 @@ github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+ github.com/jbenet/go-temp-err-catcher v0.1.0 h1:zpb3ZH6wIE8Shj2sKS+khgRvf7T7RABoLk/+KKHggpk= github.com/jbenet/go-temp-err-catcher v0.1.0/go.mod h1:0kJRvmDZXNMIiJirNPEYfhpPwbGVtZVWC34vc5WLsDk= github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= -github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -659,7 +647,6 @@ github.com/multiformats/go-varint v0.0.1/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXS github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= github.com/nsf/jsondiff v0.0.0-20230430225905-43f6cf3098c1 h1:dOYG7LS/WK00RWZc8XGgcUTlTxpp3mKhdR2Q9z9HbXM= @@ -1282,7 +1269,6 @@ modernc.org/ccgo/v3 v3.16.13 h1:Mkgdzl46i5F/CNR/Kj80Ri59hC8TKAhZrYSaqvkwzUw= modernc.org/ccgo/v3 v3.16.13/go.mod h1:2Quk+5YgpImhPjv2Qsob1DnZ/4som1lJTodubIcoUkY= modernc.org/ccorpus v1.11.6 h1:J16RXiiqiCgua6+ZvQot4yUuUy8zxgqbqEEUuGPlISk= modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM= -modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM= modernc.org/libc v1.22.3 h1:D/g6O5ftAfavceqlLOFwaZuA5KYafKwmr30A6iSqoyY= modernc.org/libc v1.22.3/go.mod h1:MQrloYP209xa2zHome2a8HLiLm6k0UT8CoHpV74tOFw= modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ= diff --git a/turbo/cli/default_flags.go b/turbo/cli/default_flags.go index 6e15b10f85b..12b8595369a 100644 --- a/turbo/cli/default_flags.go +++ b/turbo/cli/default_flags.go @@ -208,6 +208,7 @@ var DefaultFlags = []cli.Flag{ &utils.GasPriceFactor, &utils.DataStreamHost, &utils.DataStreamPort, + &utils.DataStreamWriteTimeout, &utils.WitnessFullFlag, &utils.SyncLimit, &utils.SupportGasless, diff --git a/turbo/cli/flags.go b/turbo/cli/flags.go index b1155bd8556..d1a9a4c818c 100644 --- a/turbo/cli/flags.go +++ b/turbo/cli/flags.go @@ -402,9 +402,10 @@ func setEmbeddedRpcDaemon(ctx *cli.Context, cfg *nodecfg.Config) { StateCache: kvcache.DefaultCoherentConfig, - DataStreamPort: ctx.Int(utils.DataStreamPort.Name), - DataStreamHost: ctx.String(utils.DataStreamHost.Name), - L2RpcUrl: ctx.String(utils.L2RpcUrlFlag.Name), + DataStreamPort: ctx.Int(utils.DataStreamPort.Name), + DataStreamHost: ctx.String(utils.DataStreamHost.Name), + DataStreamWriteTimeout: ctx.Duration(utils.DataStreamWriteTimeout.Name), + L2RpcUrl: ctx.String(utils.L2RpcUrlFlag.Name), } if ctx.IsSet(utils.HttpCompressionFlag.Name) { c.HttpCompression = ctx.Bool(utils.HttpCompressionFlag.Name) diff --git a/turbo/cli/flags_zkevm.go b/turbo/cli/flags_zkevm.go index f29e4933441..5a29a1f6017 100644 --- a/turbo/cli/flags_zkevm.go +++ b/turbo/cli/flags_zkevm.go @@ -156,6 +156,7 @@ func ApplyFlagsForZkConfig(ctx *cli.Context, cfg *ethconfig.Config) { DAUrl: ctx.String(utils.DAUrl.Name), DataStreamHost: ctx.String(utils.DataStreamHost.Name), DataStreamPort: ctx.Uint(utils.DataStreamPort.Name), + DataStreamWriteTimeout: ctx.Duration(utils.DataStreamWriteTimeout.Name), } utils2.EnableTimer(cfg.DebugTimers) @@ -169,6 +170,7 @@ func ApplyFlagsForZkConfig(ctx *cli.Context, cfg *ethconfig.Config) { checkFlag(utils.ExecutorStrictMode.Name, cfg.ExecutorStrictMode) checkFlag(utils.DataStreamHost.Name, cfg.DataStreamHost) checkFlag(utils.DataStreamPort.Name, cfg.DataStreamPort) + checkFlag(utils.DataStreamWriteTimeout.Name, cfg.DataStreamWriteTimeout) if cfg.DeprecatedTxPool.Disable { panic("You need tx-pool in order to run a sequencer. Enable it using txpool.disable: false") diff --git a/zk/debug_tools/datastream-host/main.go b/zk/debug_tools/datastream-host/main.go index 4b4e40740d2..c042d818dcf 100644 --- a/zk/debug_tools/datastream-host/main.go +++ b/zk/debug_tools/datastream-host/main.go @@ -1,12 +1,14 @@ package main import ( - "github.com/0xPolygonHermez/zkevm-data-streamer/datastreamer" "flag" - log2 "github.com/0xPolygonHermez/zkevm-data-streamer/log" "fmt" "os" "os/signal" + "time" + + "github.com/0xPolygonHermez/zkevm-data-streamer/datastreamer" + log2 "github.com/0xPolygonHermez/zkevm-data-streamer/log" ) var file = "" @@ -21,7 +23,7 @@ func main() { Outputs: []string{"stdout"}, } - stream, err := datastreamer.NewServer(uint16(6900), uint8(3), 1, datastreamer.StreamType(1), file, logConfig) + stream, err := datastreamer.NewServer(uint16(6900), uint8(3), 1, datastreamer.StreamType(1), file, 5*time.Second, logConfig) if err != nil { fmt.Println("Error creating datastream server:", err) return From b8ab17dcb1efcd5f4e792930edb2137e90a9df68 Mon Sep 17 00:00:00 2001 From: Valentin Staykov <79150443+V-Staykov@users.noreply.github.com> Date: Mon, 15 Jul 2024 18:15:20 +0300 Subject: [PATCH 06/19] =?UTF-8?q?utilize=20ibs.RevertToSnapshot=20to=20not?= =?UTF-8?q?=20reexecute=20transactions=20and=20fix=20da=E2=80=A6=20(#756)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * utilize ibs.RevertToSnapshot to not reexecute transactions and fix datastream last entry search * fix merge * fix merge --- cmd/rpcdaemon/commands/tracing_zkevm.go | 2 +- .../commands/zkevm_block_info_tree.go | 5 +- core/blockchain_zkevm.go | 2 +- core/state_processor_zkevm.go | 12 +- smt/pkg/blockinfo/block_info.go | 1 + turbo/transactions/tracing_zkevm.go | 2 +- zk/datastream/server/data_stream_server.go | 23 +- zk/datastream/test/test.go | 10 +- zk/stages/stage_sequence_execute.go | 322 +++++++----------- .../stage_sequence_execute_transactions.go | 37 +- zk/tests/zk_counters_test.go | 1 + 11 files changed, 194 insertions(+), 223 deletions(-) diff --git a/cmd/rpcdaemon/commands/tracing_zkevm.go b/cmd/rpcdaemon/commands/tracing_zkevm.go index c4af99f532c..154eb371b5d 100644 --- a/cmd/rpcdaemon/commands/tracing_zkevm.go +++ b/cmd/rpcdaemon/commands/tracing_zkevm.go @@ -301,7 +301,7 @@ func (api *PrivateDebugAPIImpl) TraceCallMany(ctx context.Context, bundles []Bun return err } - if _, _, err := core.ApplyTransaction_zkevm(chainConfig, api.engine(), evm, gp, st, state.NewNoopWriter(), parent, txn, nil, effectiveGasPricePercentage); err != nil { + if _, _, err := core.ApplyTransaction_zkevm(chainConfig, api.engine(), evm, gp, st, state.NewNoopWriter(), parent, txn, nil, effectiveGasPricePercentage, true); err != nil { stream.WriteNil() return err } diff --git a/cmd/rpcdaemon/commands/zkevm_block_info_tree.go b/cmd/rpcdaemon/commands/zkevm_block_info_tree.go index 23784f4aaa9..38e9c17bc1f 100644 --- a/cmd/rpcdaemon/commands/zkevm_block_info_tree.go +++ b/cmd/rpcdaemon/commands/zkevm_block_info_tree.go @@ -16,9 +16,10 @@ import ( "github.com/ledgerwatch/erigon/turbo/transactions" "github.com/ledgerwatch/erigon/zk/hermez_db" + "errors" + "github.com/gateway-fm/cdk-erigon-lib/common" "github.com/holiman/uint256" - "errors" ) type TxInfo struct { @@ -119,7 +120,7 @@ func (api *ZkEvmAPIImpl) GetL2BlockInfoTree(ctx context.Context, blockNum rpc.Bl return nil, err } - receipt, execResult, err := core.ApplyTransaction_zkevm(chainConfig, api.ethApi._engine, evm, gp, ibs, state.NewNoopWriter(), block.Header(), tx, usedGas, effectiveGasPricePercentage) + receipt, execResult, err := core.ApplyTransaction_zkevm(chainConfig, api.ethApi._engine, evm, gp, ibs, state.NewNoopWriter(), block.Header(), tx, usedGas, effectiveGasPricePercentage, true) if err != nil { return nil, err } diff --git a/core/blockchain_zkevm.go b/core/blockchain_zkevm.go index e211fcbdc2e..1208cb6782a 100644 --- a/core/blockchain_zkevm.go +++ b/core/blockchain_zkevm.go @@ -104,7 +104,7 @@ func ExecuteBlockEphemerallyZk( return nil, err } - receipt, execResult, err := ApplyTransaction_zkevm(chainConfig, engine, evm, gp, ibs, state.NewNoopWriter(), header, tx, usedGas, effectiveGasPricePercentage) + receipt, execResult, err := ApplyTransaction_zkevm(chainConfig, engine, evm, gp, ibs, state.NewNoopWriter(), header, tx, usedGas, effectiveGasPricePercentage, true) if err != nil { return nil, err } diff --git a/core/state_processor_zkevm.go b/core/state_processor_zkevm.go index 6b5f0bf0010..ae3ff31a339 100644 --- a/core/state_processor_zkevm.go +++ b/core/state_processor_zkevm.go @@ -65,7 +65,7 @@ func GetTxContext(config *chain.Config, engine consensus.EngineReader, ibs *stat // and uses the input parameters for its environment. It returns the receipt // for the transaction, gas used and an error if the transaction failed, // indicating the block was invalid. -func ApplyMessageWithTxContext(msg types.Message, txContext evmtypes.TxContext, gp *GasPool, ibs *state.IntraBlockState, stateWriter state.StateWriter, blockNumber *big.Int, tx types.Transaction, usedGas *uint64, evm vm.VMInterface) (*types.Receipt, *ExecutionResult, error) { +func ApplyMessageWithTxContext(msg types.Message, txContext evmtypes.TxContext, gp *GasPool, ibs *state.IntraBlockState, stateWriter state.StateWriter, blockNumber *big.Int, tx types.Transaction, usedGas *uint64, evm vm.VMInterface, shouldFinalizeIbs bool) (*types.Receipt, *ExecutionResult, error) { rules := evm.ChainRules() if evm.Config().TraceJumpDest { @@ -81,10 +81,11 @@ func ApplyMessageWithTxContext(msg types.Message, txContext evmtypes.TxContext, } // Update the state with pending changes - if err = ibs.FinalizeTx(rules, stateWriter); err != nil { - return nil, nil, err + if shouldFinalizeIbs { + if err = ibs.FinalizeTx(rules, stateWriter); err != nil { + return nil, nil, err + } } - if usedGas != nil { *usedGas += result.UsedGas } @@ -167,11 +168,12 @@ func ApplyTransaction_zkevm( tx types.Transaction, usedGas *uint64, effectiveGasPricePercentage uint8, + shouldFinalizeIbs bool, ) (*types.Receipt, *ExecutionResult, error) { // Create a new context to be used in the EVM environment msg, txContext, err := GetTxContext(config, engine, ibs, header, tx, evm, effectiveGasPricePercentage) if err != nil { return nil, nil, err } - return ApplyMessageWithTxContext(msg, txContext, gp, ibs, stateWriter, header.Number, tx, usedGas, evm) + return ApplyMessageWithTxContext(msg, txContext, gp, ibs, stateWriter, header.Number, tx, usedGas, evm, shouldFinalizeIbs) } diff --git a/smt/pkg/blockinfo/block_info.go b/smt/pkg/blockinfo/block_info.go index 05199ffe6a8..ae62318fa17 100644 --- a/smt/pkg/blockinfo/block_info.go +++ b/smt/pkg/blockinfo/block_info.go @@ -42,6 +42,7 @@ func BuildBlockInfoTree( "previousStateRoot", previousStateRoot.String(), "coinbase", coinbase.String(), "blockGasLimit", blockGasLimit, + "blockGasUsed", blockGasUsed, "blockTime", blockTime, "ger", ger.String(), "l1BlockHash", l1BlockHash.String(), diff --git a/turbo/transactions/tracing_zkevm.go b/turbo/transactions/tracing_zkevm.go index d319e7b981f..3b5531cdada 100644 --- a/turbo/transactions/tracing_zkevm.go +++ b/turbo/transactions/tracing_zkevm.go @@ -128,7 +128,7 @@ func ComputeTxEnv_ZkEvm(ctx context.Context, engine consensus.EngineReader, bloc return txEnv, nil } - if _, _, err := core.ApplyMessageWithTxContext(msg, txContext, gp, statedb, reader.(*state.PlainState), header.Number, txn, nil, vmenv); err != nil { + if _, _, err := core.ApplyMessageWithTxContext(msg, txContext, gp, statedb, reader.(*state.PlainState), header.Number, txn, nil, vmenv, true); err != nil { return TxEnv{}, err } diff --git a/zk/datastream/server/data_stream_server.go b/zk/datastream/server/data_stream_server.go index 367e63f07c2..8a982d24c5d 100644 --- a/zk/datastream/server/data_stream_server.go +++ b/zk/datastream/server/data_stream_server.go @@ -326,20 +326,29 @@ func (srv *DataStreamServer) GetHighestBlockNumber() (uint64, error) { if err != nil { return 0, err } - if entry.Type == datastreamer.EntryType(2) { + if uint32(entry.Type) == uint32(types.EntryTypeL2Block) || uint32(entry.Type) == uint32(types.EntryTypeL2Tx) { break } entryNum -= 1 } - l2Block, err := types.UnmarshalL2Block(entry.Data) - if err != nil { - return 0, err - } + if uint32(entry.Type) == uint32(types.EntryTypeL2Block) { + l2Block, err := types.UnmarshalL2Block(entry.Data) + if err != nil { + return 0, err + } - srv.highestBlockWritten = &l2Block.L2BlockNumber + return l2Block.L2BlockNumber, nil + } else if uint32(entry.Type) == uint32(types.EntryTypeL2Tx) { + tx, err := types.UnmarshalTx(entry.Data) + if err != nil { + return 0, err + } + + return tx.L2BlockNumber, nil + } - return l2Block.L2BlockNumber, nil + return 0, nil } func (srv *DataStreamServer) GetHighestBatchNumber() (uint64, error) { diff --git a/zk/datastream/test/test.go b/zk/datastream/test/test.go index c6e7bc0de58..9a1e69047b1 100644 --- a/zk/datastream/test/test.go +++ b/zk/datastream/test/test.go @@ -17,12 +17,12 @@ const dataStreamCardona = "datastream.cardona.zkevm-rpc.com:6900" const dataStreamBali = "datastream.internal.zkevm-rpc.com:6900" const datastreamMainnet = "stream.zkevm-rpc.com:6900" const estest = "34.175.214.161:6900" -const local = "localhost:6910" +const localhost = "localhost:6910" // This code downloads headers and blocks from a datastream server. func main() { // Create client - c := client.NewClient(context.Background(), local, 0, 0, 0) + c := client.NewClient(context.Background(), localhost, 0, 0, 0) // Start client (connect to the server) defer c.Stop() @@ -31,10 +31,10 @@ func main() { } // create bookmark - bookmark := types.NewBookmarkProto(9756, datastream.BookmarkType_BOOKMARK_TYPE_BATCH) + bookmark := types.NewBookmarkProto(188312, datastream.BookmarkType_BOOKMARK_TYPE_L2_BLOCK) // Read all entries from server - blocksRead, _, _, entriesReadAmount, _, err := c.ReadEntries(bookmark, 100) + blocksRead, _, _, entriesReadAmount, _, err := c.ReadEntries(bookmark, 1) if err != nil { panic(err) } @@ -43,7 +43,7 @@ func main() { // forkId := uint16(0) for _, dsBlock := range *blocksRead { - fmt.Println(dsBlock.BatchNumber) + fmt.Println(len(dsBlock.L2Txs)) } } diff --git a/zk/stages/stage_sequence_execute.go b/zk/stages/stage_sequence_execute.go index 2ddc6562ef6..526d4ee869a 100644 --- a/zk/stages/stage_sequence_execute.go +++ b/zk/stages/stage_sequence_execute.go @@ -114,11 +114,6 @@ func SpawnSequencingStage( var header *types.Header var parentBlock *types.Block - var addedTransactions []types.Transaction - var addedReceipts []*types.Receipt - var addedExecutionResults []*core.ExecutionResult - var clonedBatchCounters *vm.BatchCounterCollector - var decodedBlock zktx.DecodedBatchL2Data var deltaTimestamp uint64 = math.MaxUint64 var blockTransactions []types.Transaction @@ -254,10 +249,9 @@ func SpawnSequencingStage( batchDataOverflow := false var block *types.Block - var thisBlockNumber uint64 - for blockNumber := executionAt; runLoopBlocks; blockNumber++ { + for blockNumber := executionAt + 1; runLoopBlocks; blockNumber++ { if l1Recovery { - decodedBlocksIndex := blockNumber - executionAt + decodedBlocksIndex := blockNumber - (executionAt + 1) if decodedBlocksIndex == decodedBlocksSize { runLoopBlocks = false break @@ -276,37 +270,22 @@ func SpawnSequencingStage( log.Info(fmt.Sprintf("[%s] Starting block %d (forkid %v)...", logPrefix, blockNumber+1, forkId)) - reRunBlockAfterOverflow := blockNumber == lastStartedBn lastStartedBn = blockNumber - if !reRunBlockAfterOverflow { - clonedBatchCounters = batchCounters.Clone() - addedTransactions = []types.Transaction{} - addedReceipts = []*types.Receipt{} - addedExecutionResults = []*core.ExecutionResult{} - effectiveGases = []uint8{} - header, parentBlock, err = prepareHeader(tx, blockNumber, deltaTimestamp, limboHeaderTimestamp, forkId, nextBatchData.Coinbase) - if err != nil { - return err - } + addedTransactions := []types.Transaction{} + addedReceipts := []*types.Receipt{} + effectiveGases = []uint8{} + addedExecutionResults := []*core.ExecutionResult{} - // run this only once the first time, do not add it on rerun - if batchDataOverflow = blockDataSizeChecker.AddBlockStartData(uint32(prevHeader.Time-header.Time), uint32(l1InfoIndex)); batchDataOverflow { - log.Info(fmt.Sprintf("[%s] BatchL2Data limit reached. Stopping.", logPrefix), "blockNumber", blockNumber) - break - } - } else { - batchCounters = clonedBatchCounters - - // create a copy of the header otherwise the executor will return "state root mismatch error" - header = &types.Header{ - ParentHash: header.ParentHash, - Coinbase: header.Coinbase, - Difficulty: header.Difficulty, - Number: header.Number, - GasLimit: header.GasLimit, - Time: header.Time, - } + header, parentBlock, err = prepareHeader(tx, blockNumber-1, deltaTimestamp, limboHeaderTimestamp, forkId, nextBatchData.Coinbase) + if err != nil { + return err + } + + // run this only once the first time, do not add it on rerun + if batchDataOverflow = blockDataSizeChecker.AddBlockStartData(uint32(prevHeader.Time-header.Time), uint32(l1InfoIndex)); batchDataOverflow { + log.Info(fmt.Sprintf("[%s] BatchL2Data limit reached. Stopping.", logPrefix), "blockNumber", blockNumber) + break } overflowOnNewBlock, err := batchCounters.StartNewBlock(l1InfoIndex != 0) @@ -317,8 +296,6 @@ func SpawnSequencingStage( break } - thisBlockNumber = header.Number.Uint64() - infoTreeIndexProgress, l1TreeUpdate, l1TreeUpdateIndex, l1BlockHash, ger, shouldWriteGerToContract, err := prepareL1AndInfoTreeRelatedStuff(sdb, &decodedBlock, l1Recovery, header.Time) if err != nil { return err @@ -333,7 +310,7 @@ func SpawnSequencingStage( cfg.chainConfig, sdb.hermezDb, ibs, - thisBlockNumber, + blockNumber, thisBatch, header.Time, &parentRoot, @@ -343,135 +320,120 @@ func SpawnSequencingStage( return err } - if !reRunBlockAfterOverflow { - // start waiting for a new transaction to arrive - if !isAnyRecovery { - log.Info(fmt.Sprintf("[%s] Waiting for txs from the pool...", logPrefix)) - } + // start waiting for a new transaction to arrive + if !isAnyRecovery { + log.Info(fmt.Sprintf("[%s] Waiting for txs from the pool...", logPrefix)) + } - // we don't care about defer order here we just need to make sure the tickers are stopped to - // avoid a leak - logTicker := time.NewTicker(10 * time.Second) - defer logTicker.Stop() - blockTicker := time.NewTicker(cfg.zk.SequencerBlockSealTime) - defer blockTicker.Stop() - reRunBlock := false - overflow := false - // start to wait for transactions to come in from the pool and attempt to add them to the current batch. Once we detect a counter - // overflow we revert the IBS back to the previous snapshot and don't add the transaction/receipt to the collection that will - // end up in the finalised block - LOOP_TRANSACTIONS: - for { - select { - case <-logTicker.C: - if !isAnyRecovery { - log.Info(fmt.Sprintf("[%s] Waiting some more for txs from the pool...", logPrefix)) - } - case <-blockTicker.C: - if !isAnyRecovery { - break LOOP_TRANSACTIONS - } - case <-batchTicker.C: - if !isAnyRecovery { - runLoopBlocks = false - break LOOP_TRANSACTIONS - } - case <-nonEmptyBatchTimer.C: - if !isAnyRecovery && hasAnyTransactionsInThisBatch { - runLoopBlocks = false - break LOOP_TRANSACTIONS - } - default: - if limboRecovery { - cfg.txPool.LockFlusher() - blockTransactions, err = getLimboTransaction(cfg, limboTxHash) - if err != nil { - cfg.txPool.UnlockFlusher() - return err - } + // we don't care about defer order here we just need to make sure the tickers are stopped to + // avoid a leak + logTicker := time.NewTicker(10 * time.Second) + defer logTicker.Stop() + blockTicker := time.NewTicker(cfg.zk.SequencerBlockSealTime) + defer blockTicker.Stop() + var anyOverflow bool + // start to wait for transactions to come in from the pool and attempt to add them to the current batch. Once we detect a counter + // overflow we revert the IBS back to the previous snapshot and don't add the transaction/receipt to the collection that will + // end up in the finalised block + LOOP_TRANSACTIONS: + for { + select { + case <-logTicker.C: + if !isAnyRecovery { + log.Info(fmt.Sprintf("[%s] Waiting some more for txs from the pool...", logPrefix)) + } + case <-blockTicker.C: + if !isAnyRecovery { + break LOOP_TRANSACTIONS + } + case <-batchTicker.C: + if !isAnyRecovery { + runLoopBlocks = false + break LOOP_TRANSACTIONS + } + case <-nonEmptyBatchTimer.C: + if !isAnyRecovery && hasAnyTransactionsInThisBatch { + runLoopBlocks = false + break LOOP_TRANSACTIONS + } + default: + if limboRecovery { + cfg.txPool.LockFlusher() + blockTransactions, err = getLimboTransaction(cfg, limboTxHash) + if err != nil { cfg.txPool.UnlockFlusher() - } else if !l1Recovery { - cfg.txPool.LockFlusher() - blockTransactions, err = getNextPoolTransactions(cfg, executionAt, forkId, yielded) - if err != nil { - cfg.txPool.UnlockFlusher() - return err - } + return err + } + cfg.txPool.UnlockFlusher() + } else if !l1Recovery { + cfg.txPool.LockFlusher() + blockTransactions, err = getNextPoolTransactions(cfg, executionAt, forkId, yielded) + if err != nil { cfg.txPool.UnlockFlusher() + return err } + cfg.txPool.UnlockFlusher() + } - var receipt *types.Receipt - var execResult *core.ExecutionResult - for i, transaction := range blockTransactions { - var effectiveGas uint8 + var receipt *types.Receipt + var execResult *core.ExecutionResult + for i, transaction := range blockTransactions { + txHash := transaction.Hash() - if l1Recovery { - effectiveGas = l1EffectiveGases[i] - } else { - effectiveGas = DeriveEffectiveGasPrice(cfg, transaction) - } + var effectiveGas uint8 - if !batchDataOverflow { - receipt, execResult, overflow, err = attemptAddTransaction(cfg, sdb, ibs, batchCounters, &blockContext, header, transaction, effectiveGas, l1Recovery, forkId, l1InfoIndex, blockDataSizeChecker) - if err != nil { - if limboRecovery { - panic("limbo transaction has already been executed once so they must not fail while re-executing") - } - - // if we are in recovery just log the error as a warning. If the data is on the L1 then we should consider it as confirmed. - // The executor/prover would simply skip a TX with an invalid nonce for example so we don't need to worry about that here. - if l1Recovery { - log.Warn(fmt.Sprintf("[%s] error adding transaction to batch during recovery: %v", logPrefix, err), - "hash", transaction.Hash(), - "to", transaction.GetTo(), - ) - continue - } - - i++ // leave current tx in yielded set - reRunBlock = true - } - } else { - log.Info(fmt.Sprintf("[%s] BatchL2Data limit reached. Not adding last transaction", logPrefix), "txHash", transaction.Hash()) - } + if l1Recovery { + effectiveGas = l1EffectiveGases[i] + } else { + effectiveGas = DeriveEffectiveGasPrice(cfg, transaction) + } - anyOverflow := overflow || batchDataOverflow + backupDataSizeChecker := *blockDataSizeChecker + if receipt, execResult, anyOverflow, err = attemptAddTransaction(cfg, sdb, ibs, batchCounters, &blockContext, header, transaction, effectiveGas, l1Recovery, forkId, l1InfoIndex, &backupDataSizeChecker); err != nil { + if limboRecovery { + panic("limbo transaction has already been executed once so they must not fail while re-executing") + } - if !reRunBlock && anyOverflow { - if limboRecovery { - panic("limbo transaction has already been executed once so they must not overflow counters while re-executing") - } + // if we are in recovery just log the error as a warning. If the data is on the L1 then we should consider it as confirmed. + // The executor/prover would simply skip a TX with an invalid nonce for example so we don't need to worry about that here. + if l1Recovery { + log.Warn(fmt.Sprintf("[%s] error adding transaction to batch during recovery: %v", logPrefix, err), + "hash", txHash, + "to", transaction.GetTo(), + ) + continue + } + } - if !l1Recovery { - log.Info(fmt.Sprintf("[%s] overflowed adding transaction to batch", logPrefix), "batch", thisBatch, "tx-hash", transaction.Hash(), "has any transactions in this batch", hasAnyTransactionsInThisBatch) - /* - There are two cases when overflow could occur. - 1. The block DOES not contains any transactions. - In this case it means that a single tx overflow entire zk-counters. - In this case we mark it so. Once marked it will be discarded from the tx-pool async (once the tx-pool process the creation of a new batch) - NB: The tx SHOULD not be removed from yielded set, because if removed, it will be picked again on next block. That's why there is i++. It ensures that removing from yielded will start after the problematic tx - 2. The block contains transactions. - In this case, we just have to remove the transaction that overflowed the zk-counters and all transactions after it, from the yielded set. - This removal will ensure that these transaction could be added in the next block(s) - */ - if !hasAnyTransactionsInThisBatch { - i++ // leave current tx in yielded set - cfg.txPool.MarkForDiscardFromPendingBest(transaction.Hash()) - log.Trace(fmt.Sprintf("single transaction %s overflow counters", transaction.Hash())) - } - - reRunBlock = true - } + if anyOverflow { + if limboRecovery { + panic("limbo transaction has already been executed once so they must not overflow counters while re-executing") } - if reRunBlock { - txSize := len(blockTransactions) - for ; i < txSize; i++ { - yielded.Remove(transaction.Hash()) + if !l1Recovery { + log.Info(fmt.Sprintf("[%s] overflowed adding transaction to batch", logPrefix), "batch", thisBatch, "tx-hash", txHash, "has any transactions in this batch", hasAnyTransactionsInThisBatch) + /* + There are two cases when overflow could occur. + 1. The block DOES not contains any transactions. + In this case it means that a single tx overflow entire zk-counters. + In this case we mark it so. Once marked it will be discarded from the tx-pool async (once the tx-pool process the creation of a new batch) + NB: The tx SHOULD not be removed from yielded set, because if removed, it will be picked again on next block. That's why there is i++. It ensures that removing from yielded will start after the problematic tx + 2. The block contains transactions. + In this case, we just have to remove the transaction that overflowed the zk-counters and all transactions after it, from the yielded set. + This removal will ensure that these transaction could be added in the next block(s) + */ + if !hasAnyTransactionsInThisBatch { + cfg.txPool.MarkForDiscardFromPendingBest(txHash) + log.Trace(fmt.Sprintf("single transaction %s overflow counters", txHash)) } - break LOOP_TRANSACTIONS } + break LOOP_TRANSACTIONS + } + + if err == nil { + blockDataSizeChecker = &backupDataSizeChecker + yielded.Remove(txHash) addedTransactions = append(addedTransactions, transaction) addedReceipts = append(addedReceipts, receipt) addedExecutionResults = append(addedExecutionResults, execResult) @@ -479,46 +441,28 @@ func SpawnSequencingStage( hasAnyTransactionsInThisBatch = true nonEmptyBatchTimer.Reset(cfg.zk.SequencerNonEmptyBatchSealTime) + log.Debug(fmt.Sprintf("[%s] Finish block %d with %s transaction", logPrefix, blockNumber, txHash.Hex())) } + } - if l1Recovery { - // just go into the normal loop waiting for new transactions to signal that the recovery - // has finished as far as it can go - if len(blockTransactions) == 0 && !nextBatchData.IsWorkRemaining { - log.Info(fmt.Sprintf("[%s] L1 recovery no more transactions to recover", logPrefix)) - } - - break LOOP_TRANSACTIONS + if l1Recovery { + // just go into the normal loop waiting for new transactions to signal that the recovery + // has finished as far as it can go + if len(blockTransactions) == 0 && !nextBatchData.IsWorkRemaining { + log.Info(fmt.Sprintf("[%s] L1 recovery no more transactions to recover", logPrefix)) } - if limboRecovery { - runLoopBlocks = false - break LOOP_TRANSACTIONS - } + break LOOP_TRANSACTIONS } - } - if reRunBlock { - blockNumber-- // in order to trigger reRunBlockAfterOverflow check - continue // lets execute the same block again - } - } else { - for idx, transaction := range addedTransactions { - effectiveGas := effectiveGases[idx] - receipt, execResult, innerOverflow, err := attemptAddTransaction(cfg, sdb, ibs, batchCounters, &blockContext, header, transaction, effectiveGas, false, forkId, l1InfoIndex, blockDataSizeChecker) - if err != nil { - return err - } - if innerOverflow { - // kill the node at this stage to prevent a batch being created that can't be proven - panic(fmt.Sprintf("overflowed twice during execution while adding tx with index %d", idx)) + + if limboRecovery { + runLoopBlocks = false + break LOOP_TRANSACTIONS } - addedReceipts[idx] = receipt - addedExecutionResults[idx] = execResult } - runLoopBlocks = false // close the batch because there are no counters left } - if err = sdb.hermezDb.WriteBlockL1InfoTreeIndex(thisBlockNumber, l1TreeUpdateIndex); err != nil { + if err = sdb.hermezDb.WriteBlockL1InfoTreeIndex(blockNumber, l1TreeUpdateIndex); err != nil { return err } @@ -532,14 +476,10 @@ func SpawnSequencingStage( cfg.txPool.UpdateLimboRootByTxHash(limboTxHash, &stateRoot) return fmt.Errorf("[%s] %w: %s = %s", s.LogPrefix(), zk.ErrLimboState, limboTxHash.Hex(), stateRoot.Hex()) } else { - log.Debug(fmt.Sprintf("[%s] state root at block %d = %s", s.LogPrefix(), thisBlockNumber, block.Root().Hex())) - } - - for _, tx := range addedTransactions { - log.Debug(fmt.Sprintf("[%s] Finish block %d with %s transaction", logPrefix, thisBlockNumber, tx.Hash().Hex())) + log.Debug(fmt.Sprintf("[%s] state root at block %d = %s", s.LogPrefix(), blockNumber, block.Root().Hex())) } - log.Info(fmt.Sprintf("[%s] Finish block %d with %d transactions...", logPrefix, thisBlockNumber, len(addedTransactions))) + log.Info(fmt.Sprintf("[%s] Finish block %d with %d transactions...", logPrefix, blockNumber, len(addedTransactions))) if !hasExecutorForThisBatch { // save counters midbatch @@ -557,7 +497,7 @@ func SpawnSequencingStage( return err } - if err = cfg.datastreamServer.WriteBlockToStream(logPrefix, tx, sdb.hermezDb, thisBatch, lastBatch, thisBlockNumber); err != nil { + if err = cfg.datastreamServer.WriteBlockToStream(logPrefix, tx, sdb.hermezDb, thisBatch, lastBatch, blockNumber); err != nil { return err } diff --git a/zk/stages/stage_sequence_execute_transactions.go b/zk/stages/stage_sequence_execute_transactions.go index 17263ca8f46..aa9130e7fe9 100644 --- a/zk/stages/stage_sequence_execute_transactions.go +++ b/zk/stages/stage_sequence_execute_transactions.go @@ -185,11 +185,13 @@ func attemptAddTransaction( forkId, l1InfoIndex uint64, blockDataSizeChecker *BlockDataChecker, ) (*types.Receipt, *core.ExecutionResult, bool, error) { + var batchDataOverflow, overflow bool + var err error + txCounters := vm.NewTransactionCounter(transaction, sdb.smt.GetDepth(), uint16(forkId), cfg.zk.VirtualCountersSmtReduction, cfg.zk.ShouldCountersBeUnlimited(l1Recovery)) - overflow, err := batchCounters.AddNewTransactionCounters(txCounters) + overflow, err = batchCounters.AddNewTransactionCounters(txCounters) // run this only once the first time, do not add it on rerun - var batchDataOverflow bool if blockDataSizeChecker != nil { txL2Data, err := txCounters.GetL2DataCache() if err != nil { @@ -215,9 +217,12 @@ func attemptAddTransaction( // TODO: possibly inject zero tracer here! + snapshot := ibs.Snapshot() ibs.Prepare(transaction.Hash(), common.Hash{}, 0) evm := vm.NewZkEVM(*blockContext, evmtypes.TxContext{}, ibs, cfg.chainConfig, *cfg.zkVmConfig) + gasUsed := header.GasUsed + receipt, execResult, err := core.ApplyTransaction_zkevm( cfg.chainConfig, cfg.engine, @@ -227,27 +232,39 @@ func attemptAddTransaction( noop, header, transaction, - &header.GasUsed, + &gasUsed, effectiveGasPrice, + false, ) if err != nil { return nil, nil, false, err } - // we need to keep hold of the effective percentage used - // todo [zkevm] for now we're hard coding to the max value but we need to calc this properly - if err = sdb.hermezDb.WriteEffectiveGasPricePercentage(transaction.Hash(), effectiveGasPrice); err != nil { + if err = txCounters.ProcessTx(ibs, execResult.ReturnData); err != nil { return nil, nil, false, err } - err = txCounters.ProcessTx(ibs, execResult.ReturnData) - if err != nil { + // now that we have executed we can check again for an overflow + if overflow, err = batchCounters.CheckForOverflow(l1InfoIndex != 0); err != nil { return nil, nil, false, err } - // now that we have executed we can check again for an overflow - overflow, err = batchCounters.CheckForOverflow(l1InfoIndex != 0) + if overflow { + ibs.RevertToSnapshot(snapshot) + return nil, nil, true, err + } + + // add the gas only if not reverted. This should not be moved above the overflow check + header.GasUsed = gasUsed + + // we need to keep hold of the effective percentage used + // todo [zkevm] for now we're hard coding to the max value but we need to calc this properly + if err = sdb.hermezDb.WriteEffectiveGasPricePercentage(transaction.Hash(), effectiveGasPrice); err != nil { + return nil, nil, false, err + } + + ibs.FinalizeTx(evm.ChainRules(), noop) return receipt, execResult, overflow, err } diff --git a/zk/tests/zk_counters_test.go b/zk/tests/zk_counters_test.go index ad387354f3c..d47b4a7b6a2 100644 --- a/zk/tests/zk_counters_test.go +++ b/zk/tests/zk_counters_test.go @@ -322,6 +322,7 @@ func runTest(t *testing.T, test vector, err error, fileName string, idx int) { transaction, &header.GasUsed, zktypes.EFFECTIVE_GAS_PRICE_PERCENTAGE_MAXIMUM, + true, ) if err != nil { From 9f304395ccdc666d16bb6667534bbbaca5f0b04d Mon Sep 17 00:00:00 2001 From: Max Revitt Date: Tue, 16 Jul 2024 08:16:58 +0100 Subject: [PATCH 07/19] tweak(nightly): l1 recovery 3h timeout (#789) --- zk/tests/nightly-l1-recovery/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zk/tests/nightly-l1-recovery/docker-compose.yml b/zk/tests/nightly-l1-recovery/docker-compose.yml index 4bbec6a64eb..30b04b7ea91 100644 --- a/zk/tests/nightly-l1-recovery/docker-compose.yml +++ b/zk/tests/nightly-l1-recovery/docker-compose.yml @@ -40,7 +40,7 @@ services: block-checker: image: golang:1.19 - command: ["go", "run", "/repo/zk/debug_tools/nightly-block-compare-wait/main.go", "--compare=http://34.175.214.161:8505", "--compare2=http://erigon-sync:8123", "--duration=1h", "--interval=10s"] + command: ["go", "run", "/repo/zk/debug_tools/nightly-block-compare-wait/main.go", "--compare=http://34.175.214.161:8505", "--compare2=http://erigon-sync:8123", "--duration=3h", "--interval=10s"] volumes: - ../../../:/repo working_dir: /repo From b13885db11120a0cdff76e816fc160c3ac3b3ff9 Mon Sep 17 00:00:00 2001 From: Valentin Staykov <79150443+V-Staykov@users.noreply.github.com> Date: Tue, 16 Jul 2024 11:18:22 +0300 Subject: [PATCH 08/19] migrate whole blockinfotree to bulk insert (#785) * migrate whole blockinfotree to bulk insert * fix tests --- smt/pkg/blockinfo/block_info.go | 362 +++++------------- smt/pkg/blockinfo/block_info_test.go | 183 +++++---- smt/pkg/blockinfo/keys.go | 24 +- smt/pkg/blockinfo/keys_test.go | 9 +- smt/pkg/utils/utils.go | 14 +- zk/datastream/server/data_stream_server.go | 9 +- .../server/data_stream_server_utils.go | 10 +- zk/datastream/server/datastream_populate.go | 5 +- zk/stages/stage_sequence_execute.go | 3 +- 9 files changed, 253 insertions(+), 366 deletions(-) diff --git a/smt/pkg/blockinfo/block_info.go b/smt/pkg/blockinfo/block_info.go index ae62318fa17..575b2ffff22 100644 --- a/smt/pkg/blockinfo/block_info.go +++ b/smt/pkg/blockinfo/block_info.go @@ -34,9 +34,11 @@ func BuildBlockInfoTree( transactionInfos *[]ExecutedTxInfo, ) (*common.Hash, error) { infoTree := NewBlockInfoTree() - if err := infoTree.InitBlockHeader(&previousStateRoot, coinbase, blockNumber, blockGasLimit, blockTime, &ger, &l1BlockHash); err != nil { + keys, vals, err := infoTree.GenerateBlockHeader(&previousStateRoot, coinbase, blockNumber, blockGasLimit, blockTime, &ger, &l1BlockHash) + if err != nil { return nil, err } + log.Trace("info-tree-header", "blockNumber", blockNumber, "previousStateRoot", previousStateRoot.String(), @@ -47,10 +49,7 @@ func BuildBlockInfoTree( "ger", ger.String(), "l1BlockHash", l1BlockHash.String(), ) - var err error var logIndex int64 = 0 - var keys []*utils.NodeKey - var vals []*utils.NodeValue8 for i, txInfo := range *transactionInfos { receipt := txInfo.Receipt t := txInfo.Tx @@ -112,287 +111,119 @@ func (b *BlockInfoTree) GetRoot() *big.Int { return b.smt.LastRoot() } -func (b *BlockInfoTree) InitBlockHeader(oldBlockHash *common.Hash, coinbase *common.Address, blockNumber, gasLimit, timestamp uint64, ger, l1BlochHash *common.Hash) error { - _, err := setL2BlockHash(b.smt, oldBlockHash) - if err != nil { - return err - } - _, err = setCoinbase(b.smt, coinbase) - if err != nil { - return err - } - - _, err = setBlockNumber(b.smt, blockNumber) - if err != nil { - return err - } - - _, err = setGasLimit(b.smt, gasLimit) - if err != nil { - return err - } - _, err = setTimestamp(b.smt, timestamp) - if err != nil { - return err - } - _, err = setGer(b.smt, ger) - if err != nil { - return err - } - _, err = setL1BlockHash(b.smt, l1BlochHash) - if err != nil { - return err - } - return nil -} +func (b *BlockInfoTree) GenerateBlockHeader(oldBlockHash *common.Hash, coinbase *common.Address, blockNumber, gasLimit, timestamp uint64, ger, l1BlochHash *common.Hash) (keys []*utils.NodeKey, vals []*utils.NodeValue8, err error) { + keys = make([]*utils.NodeKey, 7) + vals = make([]*utils.NodeValue8, 7) -func (b *BlockInfoTree) SetBlockTx( - l2TxHash *common.Hash, - txIndex int, - receipt *ethTypes.Receipt, - logIndex int64, - cumulativeGasUsed uint64, - effectivePercentage uint8, -) (*big.Int, error) { - txIndexBig := big.NewInt(int64(txIndex)) - _, err := setL2TxHash(b.smt, txIndexBig, l2TxHash.Big()) - if err != nil { - return nil, err - } - - bigStatus := big.NewInt(0).SetUint64(receipt.Status) - _, err = setTxStatus(b.smt, txIndexBig, bigStatus) - if err != nil { - return nil, err + if keys[0], vals[0], err = generateL2BlockHash(oldBlockHash); err != nil { + return nil, nil, err } - bigCumulativeGasUsed := big.NewInt(0).SetUint64(cumulativeGasUsed) - _, err = setCumulativeGasUsed(b.smt, txIndexBig, bigCumulativeGasUsed) - if err != nil { - return nil, err + if keys[1], vals[1], err = generateCoinbase(coinbase); err != nil { + return nil, nil, err } - log.Trace("info-tree-tx-inner", - "tx-index", txIndex, - "log-index", logIndex, - "cumulativeGasUsed", cumulativeGasUsed, - "effective-percentage", effectivePercentage, - "receipt-status", receipt.Status, - ) - - // now encode the logs - for _, rLog := range receipt.Logs { - reducedTopics := "" - for _, topic := range rLog.Topics { - reducedTopics += fmt.Sprintf("%x", topic) - } - - logToEncode := fmt.Sprintf("0x%x%s", rLog.Data, reducedTopics) - - hash, err := utils.HashContractBytecode(logToEncode) - if err != nil { - return nil, err - } - - logEncodedBig := utils.ConvertHexToBigInt(hash) - _, err = setTxLog(b.smt, txIndexBig, big.NewInt(logIndex), logEncodedBig) - if err != nil { - return nil, err - } - - log.Trace("info-tree-tx-receipt-log", - "topics", reducedTopics, - "to-encode", logToEncode, - "log-index", logIndex, - ) - - // increment log index - logIndex += 1 + if keys[2], vals[2], err = generateBlockNumber(blockNumber); err != nil { + return nil, nil, err } - bigEffectivePercentage := big.NewInt(0).SetUint64(uint64(effectivePercentage)) - root, err := setTxEffectivePercentage(b.smt, txIndexBig, bigEffectivePercentage) - if err != nil { - return nil, err + if keys[3], vals[3], err = generateGasLimit(gasLimit); err != nil { + return nil, nil, err } - return root, nil -} - -func (b *BlockInfoTree) SetBlockGasUsed(gasUsed uint64) (*big.Int, error) { - key, err := KeyBlockHeaderParams(big.NewInt(IndexBlockHeaderParamGasUsed)) - if err != nil { - return nil, err - } - gasUsedBig := big.NewInt(0).SetUint64(gasUsed) - resp, err := b.smt.InsertKA(key, gasUsedBig) - if err != nil { - return nil, err + if keys[4], vals[4], err = generateTimestamp(timestamp); err != nil { + return nil, nil, err } - return resp.NewRootScalar.ToBigInt(), nil -} - -func setL2TxHash(smt *smt.SMT, txIndex *big.Int, l2TxHash *big.Int) (*big.Int, error) { - key, err := KeyTxHash(txIndex) - if err != nil { - return nil, err - } - resp, err := smt.InsertKA(key, l2TxHash) - if err != nil { - return nil, err + if keys[5], vals[5], err = generateGer(ger); err != nil { + return nil, nil, err } - return resp.NewRootScalar.ToBigInt(), nil -} - -func setTxStatus(smt *smt.SMT, txIndex *big.Int, status *big.Int) (*big.Int, error) { - key, err := KeyTxStatus(txIndex) - if err != nil { - return nil, err - } - resp, err := smt.InsertKA(key, status) - if err != nil { - return nil, err + if keys[6], vals[6], err = generateL1BlockHash(l1BlochHash); err != nil { + return nil, nil, err } - return resp.NewRootScalar.ToBigInt(), nil + return keys, vals, nil } -func setCumulativeGasUsed(smt *smt.SMT, txIndex, cumulativeGasUsed *big.Int) (*big.Int, error) { - key, err := KeyCumulativeGasUsed(txIndex) - if err != nil { - return nil, err +func generateL2BlockHash(blockHash *common.Hash) (key *utils.NodeKey, value *utils.NodeValue8, err error) { + if key, err = KeyBlockHeaderParams(big.NewInt(IndexBlockHeaderParamBlockHash)); err != nil { + return nil, nil, err } - resp, err := smt.InsertKA(key, cumulativeGasUsed) - if err != nil { - return nil, err + if value, err = bigInt2NodeVal8(blockHash.Big()); err != nil { + return nil, nil, err } - - return resp.NewRootScalar.ToBigInt(), nil + return key, value, nil } -func setTxEffectivePercentage(smt *smt.SMT, txIndex, effectivePercentage *big.Int) (*big.Int, error) { - key, err := KeyEffectivePercentage(txIndex) - if err != nil { - return nil, err - } - resp, err := smt.InsertKA(key, effectivePercentage) - if err != nil { - return nil, err +func generateCoinbase(coinbase *common.Address) (key *utils.NodeKey, value *utils.NodeValue8, err error) { + if key, err = KeyBlockHeaderParams(big.NewInt(IndexBlockHeaderParamCoinbase)); err != nil { + return nil, nil, err } - return resp.NewRootScalar.ToBigInt(), nil -} - -func setTxLog(smt *smt.SMT, txIndex *big.Int, logIndex *big.Int, log *big.Int) (*big.Int, error) { - key, err := KeyTxLogs(txIndex, logIndex) - if err != nil { - return nil, err - } - resp, err := smt.InsertKA(key, log) - if err != nil { - return nil, err + if value, err = bigInt2NodeVal8(coinbase.Hash().Big()); err != nil { + return nil, nil, err } - return resp.NewRootScalar.ToBigInt(), nil + return key, value, nil } -func setL2BlockHash(smt *smt.SMT, blockHash *common.Hash) (*big.Int, error) { - key, err := KeyBlockHeaderParams(big.NewInt(IndexBlockHeaderParamBlockHash)) - if err != nil { - return nil, err - } - resp, err := smt.InsertKA(key, blockHash.Big()) - if err != nil { - return nil, err +func generateGasLimit(gasLimit uint64) (key *utils.NodeKey, value *utils.NodeValue8, err error) { + if key, err = KeyBlockHeaderParams(big.NewInt(IndexBlockHeaderParamGasLimit)); err != nil { + return nil, nil, err } - return resp.NewRootScalar.ToBigInt(), nil -} - -func setCoinbase(smt *smt.SMT, coinbase *common.Address) (*big.Int, error) { - key, err := KeyBlockHeaderParams(big.NewInt(IndexBlockHeaderParamCoinbase)) - if err != nil { - return nil, err - } - resp, err := smt.InsertKA(key, coinbase.Hash().Big()) - if err != nil { - return nil, err + if value, err = bigInt2NodeVal8(big.NewInt(0).SetUint64(gasLimit)); err != nil { + return nil, nil, err } - - return resp.NewRootScalar.ToBigInt(), nil + return key, value, nil } -func setGasLimit(smt *smt.SMT, gasLimit uint64) (*big.Int, error) { - key, err := KeyBlockHeaderParams(big.NewInt(IndexBlockHeaderParamGasLimit)) - if err != nil { - return nil, err +func generateBlockNumber(blockNumber uint64) (key *utils.NodeKey, value *utils.NodeValue8, err error) { + if key, err = KeyBlockHeaderParams(big.NewInt(IndexBlockHeaderParamNumber)); err != nil { + return nil, nil, err } - gasLimitBig := big.NewInt(0).SetUint64(gasLimit) - resp, err := smt.InsertKA(key, gasLimitBig) - if err != nil { - return nil, err + if value, err = bigInt2NodeVal8(big.NewInt(0).SetUint64(blockNumber)); err != nil { + return nil, nil, err } - - return resp.NewRootScalar.ToBigInt(), nil + return key, value, nil } -func setBlockNumber(smt *smt.SMT, blockNumber uint64) (*big.Int, error) { - key, err := KeyBlockHeaderParams(big.NewInt(IndexBlockHeaderParamNumber)) - if err != nil { - return nil, err +func generateTimestamp(timestamp uint64) (key *utils.NodeKey, value *utils.NodeValue8, err error) { + if key, err = KeyBlockHeaderParams(big.NewInt(IndexBlockHeaderParamTimestamp)); err != nil { + return nil, nil, err } - blockNumberBig := big.NewInt(0).SetUint64(blockNumber) - resp, err := smt.InsertKA(key, blockNumberBig) - if err != nil { - return nil, err + if value, err = bigInt2NodeVal8(big.NewInt(0).SetUint64(timestamp)); err != nil { + return nil, nil, err } - return resp.NewRootScalar.ToBigInt(), nil + return key, value, nil } -func setTimestamp(smt *smt.SMT, timestamp uint64) (*big.Int, error) { - key, err := KeyBlockHeaderParams(big.NewInt(IndexBlockHeaderParamTimestamp)) - if err != nil { - return nil, err +func generateGer(ger *common.Hash) (key *utils.NodeKey, value *utils.NodeValue8, err error) { + if key, err = KeyBlockHeaderParams(big.NewInt(IndexBlockHeaderParamGer)); err != nil { + return nil, nil, err } - timestampBig := big.NewInt(0).SetUint64(timestamp) - resp, err := smt.InsertKA(key, timestampBig) - if err != nil { - return nil, err + if value, err = bigInt2NodeVal8(ger.Big()); err != nil { + return nil, nil, err } - return resp.NewRootScalar.ToBigInt(), nil + return key, value, nil } -func setGer(smt *smt.SMT, ger *common.Hash) (*big.Int, error) { - key, err := KeyBlockHeaderParams(big.NewInt(IndexBlockHeaderParamGer)) - if err != nil { - return nil, err - } - resp, err := smt.InsertKA(key, ger.Big()) - if err != nil { - return nil, err +func generateL1BlockHash(blockHash *common.Hash) (key *utils.NodeKey, value *utils.NodeValue8, err error) { + if key, err = KeyBlockHeaderParams(big.NewInt(IndexBlockHeaderParamBlockHashL1)); err != nil { + return nil, nil, err } - return resp.NewRootScalar.ToBigInt(), nil -} - -func setL1BlockHash(smt *smt.SMT, blockHash *common.Hash) (*big.Int, error) { - key, err := KeyBlockHeaderParams(big.NewInt(IndexBlockHeaderParamBlockHashL1)) - if err != nil { - return nil, err - } - resp, err := smt.InsertKA(key, blockHash.Big()) - if err != nil { - return nil, err + if value, err = bigInt2NodeVal8(blockHash.Big()); err != nil { + return nil, nil, err } - return resp.NewRootScalar.ToBigInt(), nil + return key, value, nil } func bigInt2NodeVal8(val *big.Int) (*utils.NodeValue8, error) { @@ -405,82 +236,70 @@ func bigInt2NodeVal8(val *big.Int) (*utils.NodeValue8, error) { return v, nil } -func generateL2TxHash(txIndex *big.Int, l2TxHash *big.Int) (*utils.NodeKey, *utils.NodeValue8, error) { - key, err := KeyTxHash(txIndex) - if err != nil { +func generateL2TxHash(txIndex *big.Int, l2TxHash *big.Int) (key *utils.NodeKey, value *utils.NodeValue8, err error) { + if key, err = KeyTxHash(txIndex); err != nil { return nil, nil, err } - val, err := bigInt2NodeVal8(l2TxHash) - if err != nil { + if value, err = bigInt2NodeVal8(l2TxHash); err != nil { return nil, nil, err } - return &key, val, nil + return key, value, nil } -func generateTxStatus(txIndex *big.Int, status *big.Int) (*utils.NodeKey, *utils.NodeValue8, error) { - key, err := KeyTxStatus(txIndex) - if err != nil { +func generateTxStatus(txIndex *big.Int, status *big.Int) (key *utils.NodeKey, value *utils.NodeValue8, err error) { + if key, err = KeyTxStatus(txIndex); err != nil { return nil, nil, err } - val, err := bigInt2NodeVal8(status) - if err != nil { + if value, err = bigInt2NodeVal8(status); err != nil { return nil, nil, err } - return &key, val, nil + return key, value, nil } -func generateCumulativeGasUsed(txIndex, cumulativeGasUsed *big.Int) (*utils.NodeKey, *utils.NodeValue8, error) { - key, err := KeyCumulativeGasUsed(txIndex) - if err != nil { +func generateCumulativeGasUsed(txIndex, cumulativeGasUsed *big.Int) (key *utils.NodeKey, value *utils.NodeValue8, err error) { + if key, err = KeyCumulativeGasUsed(txIndex); err != nil { return nil, nil, err } - val, err := bigInt2NodeVal8(cumulativeGasUsed) - if err != nil { + if value, err = bigInt2NodeVal8(cumulativeGasUsed); err != nil { return nil, nil, err } - return &key, val, nil + return key, value, nil } -func generateTxLog(txIndex *big.Int, logIndex *big.Int, log *big.Int) (*utils.NodeKey, *utils.NodeValue8, error) { - key, err := KeyTxLogs(txIndex, logIndex) - if err != nil { +func generateTxLog(txIndex *big.Int, logIndex *big.Int, log *big.Int) (key *utils.NodeKey, value *utils.NodeValue8, err error) { + if key, err = KeyTxLogs(txIndex, logIndex); err != nil { return nil, nil, err } - val, err := bigInt2NodeVal8(log) - if err != nil { + if value, err = bigInt2NodeVal8(log); err != nil { return nil, nil, err } - return &key, val, nil + return key, value, nil } -func generateTxEffectivePercentage(txIndex, effectivePercentage *big.Int) (*utils.NodeKey, *utils.NodeValue8, error) { - key, err := KeyEffectivePercentage(txIndex) - if err != nil { +func generateTxEffectivePercentage(txIndex, effectivePercentage *big.Int) (key *utils.NodeKey, value *utils.NodeValue8, err error) { + if key, err = KeyEffectivePercentage(txIndex); err != nil { return nil, nil, err } - val, err := bigInt2NodeVal8(effectivePercentage) - if err != nil { + if value, err = bigInt2NodeVal8(effectivePercentage); err != nil { return nil, nil, err } - return &key, val, nil + return key, value, nil } -func generateBlockGasUsed(gasUsed uint64) (*utils.NodeKey, *utils.NodeValue8, error) { - key, err := KeyBlockHeaderParams(big.NewInt(IndexBlockHeaderParamGasUsed)) - if err != nil { +func generateBlockGasUsed(gasUsed uint64) (key *utils.NodeKey, value *utils.NodeValue8, err error) { + if key, err = KeyBlockHeaderParams(big.NewInt(IndexBlockHeaderParamGasUsed)); err != nil { return nil, nil, err } gasUsedBig := big.NewInt(0).SetUint64(gasUsed) - val, err := bigInt2NodeVal8(gasUsedBig) - if err != nil { + if value, err = bigInt2NodeVal8(gasUsedBig); err != nil { return nil, nil, err } - return &key, val, nil + return key, value, nil } func (b *BlockInfoTree) GenerateBlockTxKeysVals( @@ -491,8 +310,8 @@ func (b *BlockInfoTree) GenerateBlockTxKeysVals( cumulativeGasUsed uint64, effectivePercentage uint8, ) ([]*utils.NodeKey, []*utils.NodeValue8, error) { - var keys []*utils.NodeKey - var vals []*utils.NodeValue8 + var keys []*utils.NodeKey = make([]*utils.NodeKey, 0, 4+len(receipt.Logs)) + var vals []*utils.NodeValue8 = make([]*utils.NodeValue8, 0, 4+len(receipt.Logs)) txIndexBig := big.NewInt(int64(txIndex)) key, val, err := generateL2TxHash(txIndexBig, l2TxHash.Big()) @@ -542,6 +361,9 @@ func (b *BlockInfoTree) GenerateBlockTxKeysVals( logEncodedBig := utils.ConvertHexToBigInt(hash) key, val, err = generateTxLog(txIndexBig, big.NewInt(logIndex), logEncodedBig) + if err != nil { + return nil, nil, err + } keys = append(keys, key) vals = append(vals, val) diff --git a/smt/pkg/blockinfo/block_info_test.go b/smt/pkg/blockinfo/block_info_test.go index 8d85bf137b1..1c1f13dd3f9 100644 --- a/smt/pkg/blockinfo/block_info_test.go +++ b/smt/pkg/blockinfo/block_info_test.go @@ -9,6 +9,7 @@ import ( "github.com/ledgerwatch/erigon/core/types" ethTypes "github.com/ledgerwatch/erigon/core/types" "github.com/ledgerwatch/erigon/smt/pkg/smt" + smtutils "github.com/ledgerwatch/erigon/smt/pkg/utils" "github.com/ledgerwatch/erigon/zk/utils" ) @@ -81,7 +82,7 @@ func TestBlockInfoHeader(t *testing.T) { ger := common.HexToHash(test.FinalGER) l1BlochHash := common.HexToHash(test.L1BlochHash) - err := infoTree.InitBlockHeader( + keys, vals, err := infoTree.GenerateBlockHeader( &blockHash, &coinbaseAddress, test.NewBlockNumber, @@ -94,10 +95,14 @@ func TestBlockInfoHeader(t *testing.T) { t.Fatal(err) } - root := common.BigToHash(infoTree.GetRoot()).Hex() + root, err := infoTree.smt.InsertBatch(context.Background(), "", keys, vals, nil, nil) + if err != nil { + t.Fatal(err) + } + rootHash := common.BigToHash(root.NewRootScalar.ToBigInt()).Hex() - if root != test.FinalBlockInfoRoot { - t.Fatalf("expected root %s, got %s", test.FinalBlockInfoRoot, root) + if rootHash != test.FinalBlockInfoRoot { + t.Fatalf("expected root %s, got %s", test.FinalBlockInfoRoot, rootHash) } } } @@ -197,8 +202,7 @@ func TestSetBlockTx(t *testing.T) { for _, test := range tests { infoTree := NewBlockInfoTree() - - root, err := infoTree.SetBlockTx( + keys, vals, err := infoTree.GenerateBlockTxKeysVals( &test.l2TxHash, test.txIndex, &test.receipt, @@ -209,30 +213,13 @@ func TestSetBlockTx(t *testing.T) { if err != nil { t.Fatal(err) } - rootHex := common.BigToHash(root).Hex() - infoTree2 := NewBlockInfoTree() - keys, vals, err := infoTree2.GenerateBlockTxKeysVals( - &test.l2TxHash, - test.txIndex, - &test.receipt, - test.logIndex, - test.cumulativeGasUsed, - test.effectivePercentage, - ) - if err != nil { - t.Fatal(err) - } - - root2, err2 := infoTree2.smt.InsertBatch(context.Background(), "", keys, vals, nil, nil) + root, err2 := infoTree.smt.InsertBatch(context.Background(), "", keys, vals, nil, nil) if err2 != nil { t.Fatal(err2) } - rootHex2 := common.BigToHash(root2.NewRootScalar.ToBigInt()).Hex() - if rootHex != rootHex2 { - t.Fatalf("generate different root, raw method root is %s, new method root %s", rootHex, rootHex2) - } + rootHex := common.BigToHash(root.NewRootScalar.ToBigInt()).Hex() if rootHex != test.finalBlockInfoRoot { t.Fatalf("expected root %s, got %s", test.finalBlockInfoRoot, rootHex) @@ -260,14 +247,21 @@ func TestBlockComulativeGasUsed(t *testing.T) { for i, test := range tests { infoTree := NewBlockInfoTree() - root, err := infoTree.SetBlockGasUsed(test.gasUsed) + key, val, err := generateBlockGasUsed(test.gasUsed) if err != nil { t.Fatal(err) } - actualRoot := common.BigToHash(root).Hex() + + root, err2 := infoTree.smt.InsertKA(*key, smtutils.NodeValue8ToBigInt(val)) + if err2 != nil { + t.Fatal(err2) + } + + rootHex := common.BigToHash(root.NewRootScalar.ToBigInt()).Hex() + // root taken from JS implementation - if actualRoot != test.expectedRoot { - t.Fatalf("Test %d expected root %s, got %s", i+1, test.expectedRoot, actualRoot) + if rootHex != test.expectedRoot { + t.Fatalf("Test %d expected root %s, got %s", i+1, test.expectedRoot, rootHex) } } } @@ -288,15 +282,20 @@ func TestSetL2BlockHash(t *testing.T) { } for i, test := range tests { - smt := smt.NewSMT(nil) blockHash := common.HexToHash(test.blockHash) - root, err := setL2BlockHash(smt, &blockHash) + key, val, err := generateL2BlockHash(&blockHash) if err != nil { t.Fatal(err) } - actualRoot := common.BigToHash(root).Hex() - // root taken from JS implementation + smt := smt.NewSMT(nil) + + root, err2 := smt.InsertKA(*key, smtutils.NodeValue8ToBigInt(val)) + if err2 != nil { + t.Fatal(err2) + } + + actualRoot := common.BigToHash(root.NewRootScalar.ToBigInt()).Hex() if actualRoot != test.expectedRoot { t.Fatalf("Test %d expected root %s, got %s", i+1, test.expectedRoot, actualRoot) } @@ -318,12 +317,16 @@ func TestSetCoinbase(t *testing.T) { smt := smt.NewSMT(nil) coinbaseAddress := common.HexToAddress(test.coinbaseAddress) - root, err := setCoinbase(smt, &coinbaseAddress) + key, val, err := generateCoinbase(&coinbaseAddress) if err != nil { t.Fatal(err) } - actualRoot := common.BigToHash(root).Hex() - // root taken from JS implementation + root, err2 := smt.InsertKA(*key, smtutils.NodeValue8ToBigInt(val)) + if err2 != nil { + t.Fatal(err2) + } + + actualRoot := common.BigToHash(root.NewRootScalar.ToBigInt()).Hex() if actualRoot != test.expectedRoot { t.Fatalf("Test %d expected root %s, got %s", i+1, test.expectedRoot, actualRoot) } @@ -347,11 +350,16 @@ func TestSetBlockNumber(t *testing.T) { for i, test := range tests { smt := smt.NewSMT(nil) - root, err := setBlockNumber(smt, test.blockNum) + key, val, err := generateBlockNumber(test.blockNum) if err != nil { t.Fatal(err) } - actualRoot := common.BigToHash(root).Hex() + root, err2 := smt.InsertKA(*key, smtutils.NodeValue8ToBigInt(val)) + if err2 != nil { + t.Fatal(err2) + } + + actualRoot := common.BigToHash(root.NewRootScalar.ToBigInt()).Hex() // root taken from JS implementation if actualRoot != test.expectedRoot { t.Fatalf("Test %d expected root %s, got %s", i+1, test.expectedRoot, actualRoot) @@ -373,11 +381,16 @@ func TestSetGasLimit(t *testing.T) { for i, test := range tests { smt := smt.NewSMT(nil) - root, err := setGasLimit(smt, test.gasLimit) + key, val, err := generateGasLimit(test.gasLimit) if err != nil { t.Fatal(err) } - actualRoot := common.BigToHash(root).Hex() + root, err2 := smt.InsertKA(*key, smtutils.NodeValue8ToBigInt(val)) + if err2 != nil { + t.Fatal(err2) + } + + actualRoot := common.BigToHash(root.NewRootScalar.ToBigInt()).Hex() // root taken from JS implementation if actualRoot != test.expectedRoot { t.Fatalf("Test %d expected root %s, got %s", i+1, test.expectedRoot, actualRoot) @@ -399,11 +412,16 @@ func TestSetTimestamp(t *testing.T) { for i, test := range tests { smt := smt.NewSMT(nil) - root, err := setTimestamp(smt, test.timestamp) + key, val, err := generateTimestamp(test.timestamp) if err != nil { t.Fatal(err) } - actualRoot := common.BigToHash(root).Hex() + root, err2 := smt.InsertKA(*key, smtutils.NodeValue8ToBigInt(val)) + if err2 != nil { + t.Fatal(err2) + } + + actualRoot := common.BigToHash(root.NewRootScalar.ToBigInt()).Hex() // root taken from JS implementation if actualRoot != test.expectedRoot { t.Fatalf("Test %d expected root %s, got %s", i+1, test.expectedRoot, actualRoot) @@ -432,11 +450,16 @@ func TestSetGer(t *testing.T) { smt := smt.NewSMT(nil) ger := common.HexToHash(test.ger) - root, err := setGer(smt, &ger) + key, val, err := generateGer(&ger) if err != nil { t.Fatal(err) } - actualRoot := common.BigToHash(root).Hex() + root, err2 := smt.InsertKA(*key, smtutils.NodeValue8ToBigInt(val)) + if err2 != nil { + t.Fatal(err2) + } + + actualRoot := common.BigToHash(root.NewRootScalar.ToBigInt()).Hex() // root taken from JS implementation if actualRoot != test.expectedRoot { t.Fatalf("Test %d expected root %s, got %s", i+1, test.expectedRoot, actualRoot) @@ -465,11 +488,16 @@ func TestSetL1BlockHash(t *testing.T) { smt := smt.NewSMT(nil) l1BlockHash := common.HexToHash(test.l1BlockHash) - root, err := setL1BlockHash(smt, &l1BlockHash) + key, val, err := generateL1BlockHash(&l1BlockHash) if err != nil { t.Fatal(err) } - actualRoot := common.BigToHash(root).Hex() + root, err2 := smt.InsertKA(*key, smtutils.NodeValue8ToBigInt(val)) + if err2 != nil { + t.Fatal(err2) + } + + actualRoot := common.BigToHash(root.NewRootScalar.ToBigInt()).Hex() // root taken from JS implementation if actualRoot != test.expectedRoot { t.Fatalf("Test %d expected root %s, got %s", i+1, test.expectedRoot, actualRoot) @@ -482,15 +510,19 @@ func TestSetL2TxHash(t *testing.T) { txIndex := big.NewInt(1) l2TxHash := common.HexToHash("0x000000000000000000000000000000005Ca1aB1E").Big() - root, err := setL2TxHash(smt, txIndex, l2TxHash) + key, val, err := generateL2TxHash(txIndex, l2TxHash) if err != nil { t.Fatal(err) } - // root taken from JS implementation - expectedRoot := "a9127a157cee3cd2452a194e4efc2f8a5612cfc36c66e768700727ede4d0e2e6" - actualRoot := root.Text(16) + root, err2 := smt.InsertKA(*key, smtutils.NodeValue8ToBigInt(val)) + if err2 != nil { + t.Fatal(err2) + } + actualRoot := common.BigToHash(root.NewRootScalar.ToBigInt()).Hex() + // root taken from JS implementation + expectedRoot := "0xa9127a157cee3cd2452a194e4efc2f8a5612cfc36c66e768700727ede4d0e2e6" if actualRoot != expectedRoot { t.Fatalf("expected root %s, got %s", expectedRoot, actualRoot) } @@ -501,15 +533,19 @@ func TestSetTxStatus(t *testing.T) { txIndex := big.NewInt(1) status := common.HexToHash("0x000000000000000000000000000000005Ca1aB1E").Big() - root, err := setTxStatus(smt, txIndex, status) + key, val, err := generateTxStatus(txIndex, status) if err != nil { t.Fatal(err) } + root, err2 := smt.InsertKA(*key, smtutils.NodeValue8ToBigInt(val)) + if err2 != nil { + t.Fatal(err2) + } - // root taken from JS implementation - expectedRoot := "7cb6a0928f5165a422cfbe5f93d1cc9eda3f686715639823f6087818465fcbb8" - actualRoot := root.Text(16) + actualRoot := common.BigToHash(root.NewRootScalar.ToBigInt()).Hex() + // root taken from JS implementation + expectedRoot := "0x7cb6a0928f5165a422cfbe5f93d1cc9eda3f686715639823f6087818465fcbb8" if actualRoot != expectedRoot { t.Fatalf("expected root %s, got %s", expectedRoot, actualRoot) } @@ -520,15 +556,20 @@ func TestSetCumulativeGasUsed(t *testing.T) { txIndex := big.NewInt(1) cgu := common.HexToHash("0x000000000000000000000000000000005Ca1aB1E").Big() - root, err := setCumulativeGasUsed(smt, txIndex, cgu) + key, val, err := generateCumulativeGasUsed(txIndex, cgu) if err != nil { t.Fatal(err) } - // root taken from JS implementation - expectedRoot := "c07ff46f07be5b81465c30848202acc4bf82805961d8a9f9ffe74e820e4bca68" - actualRoot := root.Text(16) + root, err2 := smt.InsertKA(*key, smtutils.NodeValue8ToBigInt(val)) + if err2 != nil { + t.Fatal(err2) + } + + actualRoot := common.BigToHash(root.NewRootScalar.ToBigInt()).Hex() + // root taken from JS implementation + expectedRoot := "0xc07ff46f07be5b81465c30848202acc4bf82805961d8a9f9ffe74e820e4bca68" if actualRoot != expectedRoot { t.Fatalf("expected root %s, got %s", expectedRoot, actualRoot) } @@ -539,15 +580,20 @@ func TestSetTxEffectivePercentage(t *testing.T) { txIndex := big.NewInt(1) egp := common.HexToHash("0x000000000000000000000000000000005Ca1aB1E").Big() - root, err := setTxEffectivePercentage(smt, txIndex, egp) + key, val, err := generateTxEffectivePercentage(txIndex, egp) if err != nil { t.Fatal(err) } - // root taken from JS implementation - expectedRoot := "f6b3130ecdd23bd9e47c4dda0fdde6bd0e0446c6d6927778e57e80016fa9fa23" - actualRoot := root.Text(16) + root, err2 := smt.InsertKA(*key, smtutils.NodeValue8ToBigInt(val)) + if err2 != nil { + t.Fatal(err2) + } + actualRoot := common.BigToHash(root.NewRootScalar.ToBigInt()).Hex() + + // root taken from JS implementation + expectedRoot := "0xf6b3130ecdd23bd9e47c4dda0fdde6bd0e0446c6d6927778e57e80016fa9fa23" if actualRoot != expectedRoot { t.Fatalf("expected root %s, got %s", expectedRoot, actualRoot) } @@ -559,15 +605,20 @@ func TestSetTxLogs(t *testing.T) { logIndex := big.NewInt(1) log := common.HexToHash("0x000000000000000000000000000000005Ca1aB1E").Big() - root, err := setTxLog(smt, txIndex, logIndex, log) + key, val, err := generateTxLog(txIndex, logIndex, log) if err != nil { t.Fatal(err) } - // root taken from JS implementation - expectedRoot := "aff38141ae4538baf61f08efe3019ef2d219f30b98b1d40a9813d502f6bacb12" - actualRoot := root.Text(16) + root, err2 := smt.InsertKA(*key, smtutils.NodeValue8ToBigInt(val)) + if err2 != nil { + t.Fatal(err2) + } + actualRoot := common.BigToHash(root.NewRootScalar.ToBigInt()).Hex() + + // root taken from JS implementation + expectedRoot := "0xaff38141ae4538baf61f08efe3019ef2d219f30b98b1d40a9813d502f6bacb12" if actualRoot != expectedRoot { t.Fatalf("expected root %s, got %s", expectedRoot, actualRoot) } diff --git a/smt/pkg/blockinfo/keys.go b/smt/pkg/blockinfo/keys.go index 9924f40346e..de3a8a4d17c 100644 --- a/smt/pkg/blockinfo/keys.go +++ b/smt/pkg/blockinfo/keys.go @@ -25,13 +25,13 @@ const IndexBlockHeaderCumulativeGasUsed = 10 const IndexBlockHeaderLogs = 11 const IndexBlockHeaderEffectivePercentage = 12 -func KeyBlockHeaderParams(paramKey *big.Int) (utils.NodeKey, error) { +func KeyBlockHeaderParams(paramKey *big.Int) (*utils.NodeKey, error) { return utils.KeyBig(paramKey, IndexBlockHeaderParam) } -func KeyTxLogs(txIndex, logIndex *big.Int) (utils.NodeKey, error) { +func KeyTxLogs(txIndex, logIndex *big.Int) (*utils.NodeKey, error) { if txIndex == nil || logIndex == nil { - return [4]uint64{}, errors.New("nil key") + return nil, errors.New("nil key") } txIndexKey := utils.ScalarToArrayBig(txIndex) @@ -40,29 +40,33 @@ func KeyTxLogs(txIndex, logIndex *big.Int) (utils.NodeKey, error) { logIndexArray := utils.ScalarToArrayBig(logIndex) lia, err := utils.NodeValue8FromBigIntArray(logIndexArray) if err != nil { - return [4]uint64{}, err + return nil, err } hk0, err := utils.Hash(lia.ToUintArray(), utils.BranchCapacity) if err != nil { - return [4]uint64{}, err + return nil, err + } + hkRes, err := utils.Hash(key1.ToUintArray(), hk0) + if err != nil { + return nil, err } - return utils.Hash(key1.ToUintArray(), hk0) + return &utils.NodeKey{hkRes[0], hkRes[1], hkRes[2], hkRes[3]}, nil } -func KeyTxStatus(paramKey *big.Int) (utils.NodeKey, error) { +func KeyTxStatus(paramKey *big.Int) (*utils.NodeKey, error) { return utils.KeyBig(paramKey, IndexBlockHeaderStatus) } -func KeyCumulativeGasUsed(paramKey *big.Int) (utils.NodeKey, error) { +func KeyCumulativeGasUsed(paramKey *big.Int) (*utils.NodeKey, error) { return utils.KeyBig(paramKey, IndexBlockHeaderCumulativeGasUsed) } -func KeyTxHash(paramKey *big.Int) (utils.NodeKey, error) { +func KeyTxHash(paramKey *big.Int) (*utils.NodeKey, error) { return utils.KeyBig(paramKey, IndexBlockHeaderTransactionHash) } -func KeyEffectivePercentage(paramKey *big.Int) (utils.NodeKey, error) { +func KeyEffectivePercentage(paramKey *big.Int) (*utils.NodeKey, error) { return utils.KeyBig(paramKey, IndexBlockHeaderEffectivePercentage) } diff --git a/smt/pkg/blockinfo/keys_test.go b/smt/pkg/blockinfo/keys_test.go index ecf34972375..dcd70fa8f8f 100644 --- a/smt/pkg/blockinfo/keys_test.go +++ b/smt/pkg/blockinfo/keys_test.go @@ -1,10 +1,11 @@ package blockinfo import ( - "github.com/ledgerwatch/erigon/smt/pkg/utils" - "github.com/stretchr/testify/assert" "math/big" "testing" + + "github.com/ledgerwatch/erigon/smt/pkg/utils" + "github.com/stretchr/testify/assert" ) func TestKeyBlockHeaderParams(t *testing.T) { @@ -62,7 +63,7 @@ func TestKeyBlockHeaderParams(t *testing.T) { assert.Error(t, err) } else { assert.NoError(t, err) - assert.Equal(t, scenario.expected, val) + assert.Equal(t, scenario.expected, *val) } }) } @@ -90,7 +91,7 @@ func TestKeyTxLogs(t *testing.T) { assert.Error(t, err) } else { assert.NoError(t, err) - assert.Equal(t, scenario.expected, val) + assert.Equal(t, scenario.expected, *val) } }) } diff --git a/smt/pkg/utils/utils.go b/smt/pkg/utils/utils.go index 9f6386b54de..9fdb6fc5de9 100644 --- a/smt/pkg/utils/utils.go +++ b/smt/pkg/utils/utils.go @@ -203,6 +203,11 @@ func NodeValue8FromBigInt(value *big.Int) (*NodeValue8, error) { return NodeValue8FromBigIntArray(x) } +func NodeValue8ToBigInt(value *NodeValue8) *big.Int { + x := BigIntArrayFromNodeValue8(value) + return ArrayBigToScalar(x) +} + func NodeValue8FromBigIntArray(arr []*big.Int) (*NodeValue8, error) { if len(arr) != 8 { return &NodeValue8{}, fmt.Errorf("invalid array length") @@ -602,19 +607,20 @@ func Key(ethAddr string, c int) (NodeKey, error) { return Hash(key1.ToUintArray(), key1Capacity) } -func KeyBig(k *big.Int, c int) ([4]uint64, error) { +func KeyBig(k *big.Int, c int) (*NodeKey, error) { if k == nil { - return [4]uint64{}, errors.New("nil key") + return nil, errors.New("nil key") } add := ScalarToArrayBig(k) key1 := NodeValue8{add[0], add[1], add[2], add[3], add[4], add[5], big.NewInt(int64(c)), big.NewInt(0)} key1Capacity, err := StringToH4(HASH_POSEIDON_ALL_ZEROES) if err != nil { - return NodeKey{}, err + return nil, err } - return Hash(key1.ToUintArray(), key1Capacity) + hk0, err := Hash(key1.ToUintArray(), key1Capacity) + return &NodeKey{hk0[0], hk0[1], hk0[2], hk0[3]}, err } func StrValToBigInt(v string) (*big.Int, bool) { diff --git a/zk/datastream/server/data_stream_server.go b/zk/datastream/server/data_stream_server.go index 8a982d24c5d..40ea2f3933e 100644 --- a/zk/datastream/server/data_stream_server.go +++ b/zk/datastream/server/data_stream_server.go @@ -3,6 +3,7 @@ package server import ( "github.com/0xPolygonHermez/zkevm-data-streamer/datastreamer" zktypes "github.com/ledgerwatch/erigon/zk/types" + "github.com/ledgerwatch/erigon/zk/utils" "github.com/gateway-fm/cdk-erigon-lib/common" libcommon "github.com/gateway-fm/cdk-erigon-lib/common" @@ -151,7 +152,6 @@ func createBlockWithBatchCheckStreamEntriesProto( filteredTransactions := filterTransactionByIndexes(block.Transactions(), transactionsToIncludeByIndex) blockNum := block.NumberU64() - // batch start // BATCH BOOKMARK if isBatchStart { @@ -178,7 +178,12 @@ func createBlockWithBatchCheckStreamEntriesProto( blockEntriesProto = blockEntries.Entries() if isBatchEnd { - if endEntriesProto, err = addBatchEndEntriesProto(reader, tx, batchNumber, lastBatchNumber, block.Root(), gers); err != nil { + localExitRoot, err := utils.GetBatchLocalExitRootFromSCStorage(batchNumber, reader, tx) + if err != nil { + return nil, err + } + blockRoot := block.Root() + if endEntriesProto, err = addBatchEndEntriesProto(tx, batchNumber, lastBatchNumber, &blockRoot, gers, &localExitRoot); err != nil { return nil, err } } diff --git a/zk/datastream/server/data_stream_server_utils.go b/zk/datastream/server/data_stream_server_utils.go index 8395aaa499b..71dcc6b7a03 100644 --- a/zk/datastream/server/data_stream_server_utils.go +++ b/zk/datastream/server/data_stream_server_utils.go @@ -180,11 +180,11 @@ func createBatchStartEntriesProto( } func addBatchEndEntriesProto( - reader DbReader, tx kv.Tx, batchNumber, lastBatchNumber uint64, - root libcommon.Hash, + root *libcommon.Hash, gers []types.GerUpdateProto, + localExitRoot *libcommon.Hash, ) ([]DataStreamEntryProto, error) { entries := make([]DataStreamEntryProto, 0, len(gers)+1) @@ -196,12 +196,8 @@ func addBatchEndEntriesProto( } } - localExitRoot, err := utils.GetBatchLocalExitRootFromSCStorage(batchNumber, reader, tx) - if err != nil { - return nil, err - } // seal off the last batch - entries = append(entries, newBatchEndProto(localExitRoot, root, batchNumber)) + entries = append(entries, newBatchEndProto(*localExitRoot, *root, batchNumber)) return entries, nil } diff --git a/zk/datastream/server/datastream_populate.go b/zk/datastream/server/datastream_populate.go index 7665c4f7f55..fd6ed247691 100644 --- a/zk/datastream/server/datastream_populate.go +++ b/zk/datastream/server/datastream_populate.go @@ -178,7 +178,8 @@ func (srv *DataStreamServer) WriteBatchEnd( reader DbReader, batchNumber, lastBatchNumber uint64, - stateRoot common.Hash, + stateRoot *common.Hash, + localExitRoot *common.Hash, ) (err error) { gers, err := reader.GetBatchGlobalExitRootsProto(lastBatchNumber, batchNumber) if err != nil { @@ -189,7 +190,7 @@ func (srv *DataStreamServer) WriteBatchEnd( return err } - batchEndEntries, err := addBatchEndEntriesProto(reader, tx, batchNumber, lastBatchNumber, stateRoot, gers) + batchEndEntries, err := addBatchEndEntriesProto(tx, batchNumber, lastBatchNumber, stateRoot, gers, localExitRoot) if err != nil { return err } diff --git a/zk/stages/stage_sequence_execute.go b/zk/stages/stage_sequence_execute.go index 526d4ee869a..762587d5d98 100644 --- a/zk/stages/stage_sequence_execute.go +++ b/zk/stages/stage_sequence_execute.go @@ -548,7 +548,8 @@ func SpawnSequencingStage( log.Info(fmt.Sprintf("[%s] Finish batch %d...", logPrefix, thisBatch)) if !hasExecutorForThisBatch { - if err = cfg.datastreamServer.WriteBatchEnd(logPrefix, tx, sdb.hermezDb, thisBatch, lastBatch, block.Root()); err != nil { + blockRoot := block.Root() + if err = cfg.datastreamServer.WriteBatchEnd(logPrefix, tx, sdb.hermezDb, thisBatch, lastBatch, &blockRoot, &ler); err != nil { return err } } From 7fb66e790185be2786c9129c7542a58164bfa1c4 Mon Sep 17 00:00:00 2001 From: Max Revitt Date: Thu, 18 Jul 2024 07:45:26 +0100 Subject: [PATCH 09/19] BatchL2Data Fix (zkevm_getBatchByNumber) (#808) --- cmd/rpcdaemon/commands/zkevm_api.go | 110 ++++++++++++----------- zk/debug_tools/rpc-batch-compare/main.go | 4 +- zk/hermez_db/db.go | 3 + zk/tx/tx.go | 2 +- 4 files changed, 65 insertions(+), 54 deletions(-) diff --git a/cmd/rpcdaemon/commands/zkevm_api.go b/cmd/rpcdaemon/commands/zkevm_api.go index 60657886572..9cbe843f957 100644 --- a/cmd/rpcdaemon/commands/zkevm_api.go +++ b/cmd/rpcdaemon/commands/zkevm_api.go @@ -244,6 +244,31 @@ func (api *ZkEvmAPIImpl) GetBatchDataByNumbers(ctx context.Context, batchNumbers hermezDb := hermez_db.NewHermezDbReader(tx) + // use inbuilt rpc.BlockNumber type to implement the 'latest' behaviour + // highest block/batch tied to last block synced + // unless the node is still syncing - in which case 'current block' is used + // this is the batch number of stage progress of the Finish stage + + highestBlock, err := rawdb.ReadLastBlockSynced(tx) + if err != nil { + return nil, err + } + + highestBatchNo, err := hermezDb.GetBatchNoByL2Block(highestBlock.NumberU64()) + if err != nil { + return nil, err + } + + // check sync status of node + syncing, err := api.ethApi.Syncing(ctx) + if err != nil { + return nil, err + } + if syncing != nil && syncing != false { + bn := syncing.(map[string]interface{})["currentBlock"] + highestBatchNo, err = hermezDb.GetBatchNoByL2Block(uint64(bn.(hexutil.Uint64))) + } + bds := make([]*types.BatchDataSlim, 0, len(batchNumbers)) for _, batchNumber := range batchNumbers { @@ -252,15 +277,6 @@ func (api *ZkEvmAPIImpl) GetBatchDataByNumbers(ctx context.Context, batchNumbers Empty: false, } - highestBlock, err := rawdb.ReadLastBlockSynced(tx) - if err != nil { - return nil, err - } - highestBatchNo, err := hermezDb.GetBatchNoByL2Block(highestBlock.NumberU64()) - if err != nil { - return nil, err - } - // return null if we're not at this block height yet if batchNumber > rpc.BlockNumber(highestBatchNo) { bd.Empty = true @@ -387,16 +403,30 @@ func (api *ZkEvmAPIImpl) GetBatchByNumber(ctx context.Context, batchNumber rpc.B defer tx.Rollback() hermezDb := hermez_db.NewHermezDbReader(tx) + // use inbuilt rpc.BlockNumber type to implement the 'latest' behaviour + // highest block/batch tied to last block synced + // unless the node is still syncing - in which case 'current block' is used + // this is the batch number of stage progress of the Finish stage + highestBlock, err := rawdb.ReadLastBlockSynced(tx) if err != nil { return nil, err } + highestBatchNo, err := hermezDb.GetBatchNoByL2Block(highestBlock.NumberU64()) if err != nil { return nil, err } - // return null if we're not at this block height yet + // check sync status of node + syncing, err := api.ethApi.Syncing(ctx) + if err != nil { + return nil, err + } + if syncing != nil && syncing != false { + bn := syncing.(map[string]interface{})["currentBlock"] + highestBatchNo, err = hermezDb.GetBatchNoByL2Block(uint64(bn.(hexutil.Uint64))) + } if batchNumber > rpc.BlockNumber(highestBatchNo) { return nil, nil } @@ -435,6 +465,8 @@ func (api *ZkEvmAPIImpl) GetBatchByNumber(ctx context.Context, batchNumber rpc.B // last block in batch data batch.Coinbase = block.Coinbase() batch.StateRoot = block.Root() + + // TODO: this logic is wrong it is the L1 verification timestamp we need batch.Timestamp = types.ArgUint64(block.Time()) // block numbers in batch @@ -540,40 +572,12 @@ func (api *ZkEvmAPIImpl) GetBatchByNumber(ctx context.Context, batchNumber rpc.B } // global exit root of batch - batchGer, foundBatchGerNumber, err := hermezDb.GetLastBatchGlobalExitRoot(batchNo) - if err != nil { - return nil, err - } - - // get last block in batch - lastBlockInbatch, err := hermezDb.GetHighestBlockInBatch(batchNo) - if err != nil { - return nil, err - } - - // get latest found ger by block - latestBlockHer, blockNum, err := hermezDb.GetLastBlockGlobalExitRoot(lastBlockInbatch) - if err != nil { - return nil, err - } - - //get latest block ger batch number - latestBlockGerBatchNumber, err := hermezDb.GetBatchNoByL2Block(blockNum) + batchGer, _, err := hermezDb.GetLastBlockGlobalExitRoot(blockNo) if err != nil { return nil, err } - var ger *common.Hash - if batchGer != nil { - ger = &batchGer.GlobalExitRoot - } - if foundBatchGerNumber < latestBlockGerBatchNumber { - ger = &latestBlockHer - } - - if ger != nil { - batch.GlobalExitRoot = *ger - } + batch.GlobalExitRoot = batchGer // sequence seq, err := hermezDb.GetSequenceByBatchNo(batchNo) @@ -600,7 +604,7 @@ func (api *ZkEvmAPIImpl) GetBatchByNumber(ctx context.Context, batchNumber rpc.B } // exit roots (MainnetExitRoot, RollupExitRoot) - infoTreeUpdate, err := hermezDb.GetL1InfoTreeUpdateByGer(batch.GlobalExitRoot) + infoTreeUpdate, err := hermezDb.GetL1InfoTreeUpdateByGer(batchGer) if err != nil { return nil, err } @@ -640,22 +644,24 @@ func (api *ZkEvmAPIImpl) GetBatchByNumber(ctx context.Context, batchNumber rpc.B if forkId < 12 { // get the previous batches exit roots prevBatchNo := batchNo - 1 - prevBatchGer, _, err := hermezDb.GetLastBatchGlobalExitRoot(prevBatchNo) + prevBatchHighestBlock, err := hermezDb.GetHighestBlockInBatch(prevBatchNo) + if err != nil { + return nil, err + } + prevBatchGer, _, err := hermezDb.GetLastBlockGlobalExitRoot(prevBatchHighestBlock) if err != nil { return nil, err } - if prevBatchGer != nil { - itu, err := hermezDb.GetL1InfoTreeUpdateByGer(prevBatchGer.GlobalExitRoot) - if err != nil { - return nil, err - } + itu, err := hermezDb.GetL1InfoTreeUpdateByGer(prevBatchGer) + if err != nil { + return nil, err + } - if itu == nil || batch.MainnetExitRoot == itu.MainnetExitRoot { - batch.MainnetExitRoot = common.Hash{} - batch.RollupExitRoot = common.Hash{} - batch.GlobalExitRoot = common.Hash{} - } + if itu == nil || batch.MainnetExitRoot == itu.MainnetExitRoot { + batch.MainnetExitRoot = common.Hash{} + batch.RollupExitRoot = common.Hash{} + batch.GlobalExitRoot = common.Hash{} } } diff --git a/zk/debug_tools/rpc-batch-compare/main.go b/zk/debug_tools/rpc-batch-compare/main.go index b5524d26ac1..1db9c1b6bb5 100644 --- a/zk/debug_tools/rpc-batch-compare/main.go +++ b/zk/debug_tools/rpc-batch-compare/main.go @@ -10,8 +10,9 @@ import ( "net/http" "os" - "github.com/google/go-cmp/cmp" "io" + + "github.com/google/go-cmp/cmp" ) func getBatchNumber(url string) (*big.Int, error) { @@ -162,6 +163,7 @@ func main() { log.Printf("Skipping %d batches\n", *skip) for i := 0; i < *numBatches; i++ { + log.Println("Checking batch", i+1, "of", *numBatches) batchNumber := new(big.Int).Sub(startBatch, big.NewInt(int64(i**skip))) diff, err := compareBatches(*erigonURL, *legacyURL, batchNumber) if err != nil { diff --git a/zk/hermez_db/db.go b/zk/hermez_db/db.go index 5a94e0832e5..b087dfd85a4 100644 --- a/zk/hermez_db/db.go +++ b/zk/hermez_db/db.go @@ -757,6 +757,7 @@ func (db *HermezDb) WriteBatchGlobalExitRoot(batchNumber uint64, ger dstypes.Ger return db.tx.Put(GLOBAL_EXIT_ROOTS_BATCHES, Uint64ToBytes(batchNumber), ger.EncodeToBytes()) } +// deprecated: post etrog this will not work func (db *HermezDbReader) GetBatchGlobalExitRoots(fromBatchNum, toBatchNum uint64) (*[]dstypes.GerUpdate, error) { c, err := db.tx.Cursor(GLOBAL_EXIT_ROOTS_BATCHES) if err != nil { @@ -784,6 +785,7 @@ func (db *HermezDbReader) GetBatchGlobalExitRoots(fromBatchNum, toBatchNum uint6 return &gers, err } +// GetLastBatchGlobalExitRoot deprecated: post etrog this will not work func (db *HermezDbReader) GetLastBatchGlobalExitRoot(batchNum uint64) (*dstypes.GerUpdate, uint64, error) { c, err := db.tx.Cursor(GLOBAL_EXIT_ROOTS_BATCHES) if err != nil { @@ -829,6 +831,7 @@ func (db *HermezDbReader) GetBatchGlobalExitRootsProto(fromBatchNum, toBatchNum return gersProto, nil } +// GetBatchGlobalExitRoot deprecated: post etrog this will not work func (db *HermezDbReader) GetBatchGlobalExitRoot(batchNum uint64) (*dstypes.GerUpdate, error) { gerUpdateBytes, err := db.tx.GetOne(GLOBAL_EXIT_ROOTS_BATCHES, Uint64ToBytes(batchNum)) if err != nil { diff --git a/zk/tx/tx.go b/zk/tx/tx.go index 0711d43a82d..1b4025d25c4 100644 --- a/zk/tx/tx.go +++ b/zk/tx/tx.go @@ -322,7 +322,7 @@ func TransactionToL2Data(tx types.Transaction, forkId uint16, efficiencyPercenta removeLeadingZeroesFromBytes(gas), to, // don't remove leading 0s from addr removeLeadingZeroesFromBytes(valueBytes), - removeLeadingZeroesFromBytes(tx.GetData()), + tx.GetData(), } if !tx.GetChainID().Eq(uint256.NewInt(0)) || !(v.Eq(uint256.NewInt(27)) || v.Eq(uint256.NewInt(28))) { From 60dbd6c04b680e55b12997fd5dfc719064376b5f Mon Sep 17 00:00:00 2001 From: Valentin Staykov <79150443+V-Staykov@users.noreply.github.com> Date: Thu, 18 Jul 2024 11:40:41 +0300 Subject: [PATCH 10/19] big int to hex conversion optimized (#811) * big int to hex conversion optimized * arrays copy fixes --- cmd/hack/rpc_checker/main.go | 7 ++-- smt/pkg/blockinfo/block_info.go | 6 ++-- smt/pkg/db/mem-db.go | 2 +- smt/pkg/utils/utils.go | 33 ++++++------------- zk/debug_tools/l1-block-finder/main.go | 11 ++++--- zk/debug_tools/l1-info-tree-builder/main.go | 18 +++++----- .../nightly-block-compare-wait/main.go | 2 +- zk/debug_tools/nightly-block-compare/main.go | 5 +-- zk/legacy_executor_verifier/executor.go | 5 +-- zk/stages/stage_interhashes.go | 6 ++-- 10 files changed, 45 insertions(+), 50 deletions(-) diff --git a/cmd/hack/rpc_checker/main.go b/cmd/hack/rpc_checker/main.go index 1953c4e8d21..7ff22f255bd 100644 --- a/cmd/hack/rpc_checker/main.go +++ b/cmd/hack/rpc_checker/main.go @@ -9,6 +9,7 @@ import ( "io/ioutil" "log" "net/http" + "strconv" "sync" ) @@ -65,7 +66,7 @@ func getBlockHash(nodeURL string, blockNumber int, wg *sync.WaitGroup, resultCha request := JSONRPCRequest{ Jsonrpc: "2.0", Method: "eth_getBlockByNumber", - Params: []interface{}{fmt.Sprintf("0x%x", blockNumber), true}, + Params: []interface{}{"0x" + strconv.FormatInt(int64(blockNumber), 16), true}, ID: 1, } @@ -183,7 +184,9 @@ func compareReceipts(nodeURL1, nodeURL2 string, txHashes []string) (bool, error) return true, nil } -/* EXAMPLE USAGE: +/* + EXAMPLE USAGE: + go run cmd/hack/rpc_checker/main.go -node1=http://your-node1-url -node2=http://your-node2-url -fromBlock=3000000 -step=1000 -compare-receipts=true */ func main() { diff --git a/smt/pkg/blockinfo/block_info.go b/smt/pkg/blockinfo/block_info.go index 575b2ffff22..3f2e97bbdf3 100644 --- a/smt/pkg/blockinfo/block_info.go +++ b/smt/pkg/blockinfo/block_info.go @@ -2,7 +2,7 @@ package blockinfo import ( "context" - "fmt" + "encoding/hex" "math/big" ethTypes "github.com/ledgerwatch/erigon/core/types" @@ -349,10 +349,10 @@ func (b *BlockInfoTree) GenerateBlockTxKeysVals( for _, rLog := range receipt.Logs { reducedTopics := "" for _, topic := range rLog.Topics { - reducedTopics += fmt.Sprintf("%x", topic) + reducedTopics += topic.Hex()[2:] } - logToEncode := fmt.Sprintf("0x%x%s", rLog.Data, reducedTopics) + logToEncode := "0x" + hex.EncodeToString(rLog.Data) + reducedTopics hash, err := utils.HashContractBytecode(logToEncode) if err != nil { diff --git a/smt/pkg/db/mem-db.go b/smt/pkg/db/mem-db.go index f342033f22f..a3d38be8626 100644 --- a/smt/pkg/db/mem-db.go +++ b/smt/pkg/db/mem-db.go @@ -240,7 +240,7 @@ func (m *MemDb) GetCode(codeHash []byte) ([]byte, error) { codeHash = utils.ResizeHashTo32BytesByPrefixingWithZeroes(codeHash) - s, ok := m.DbCode[fmt.Sprintf("0x%x", codeHash)] + s, ok := m.DbCode["0x"+hex.EncodeToString(codeHash)] if !ok { return nil, fmt.Errorf("key not found") diff --git a/smt/pkg/utils/utils.go b/smt/pkg/utils/utils.go index 9fdb6fc5de9..befe688a3da 100644 --- a/smt/pkg/utils/utils.go +++ b/smt/pkg/utils/utils.go @@ -181,9 +181,7 @@ func IsArrayUint64Empty(arr []uint64) bool { func Value8FromBigIntArray(arr []*big.Int) NodeValue8 { nv := [8]*big.Int{} - for i, v := range arr { - nv[i] = v - } + copy(nv[:], arr) return nv } @@ -192,9 +190,7 @@ func NodeValue12FromBigIntArray(arr []*big.Int) (*NodeValue12, error) { return &NodeValue12{}, fmt.Errorf("invalid array length") } nv := NodeValue12{} - for i, v := range arr { - nv[i] = v - } + copy(nv[:], arr) return &nv, nil } @@ -213,17 +209,15 @@ func NodeValue8FromBigIntArray(arr []*big.Int) (*NodeValue8, error) { return &NodeValue8{}, fmt.Errorf("invalid array length") } nv := NodeValue8{} - for i, v := range arr { - nv[i] = v - } + copy(nv[:], arr) return &nv, nil } func BigIntArrayFromNodeValue8(nv *NodeValue8) []*big.Int { arr := make([]*big.Int, 8) - for i, v := range nv { - arr[i] = v - } + + copy(arr, nv[:]) + return arr } @@ -248,8 +242,7 @@ func (nv *NodeValue12) IsFinalNode() bool { } func ConvertBigIntToHex(n *big.Int) string { - hex := fmt.Sprintf("0x%0x", n) - return hex + return "0x" + n.Text(16) } func ConvertHexToBigInt(hex string) *big.Int { @@ -407,9 +400,7 @@ func BinaryKey(key NodeKey) string { func ConcatArrays4(a, b [4]uint64) [8]uint64 { result := [8]uint64{} - for i, v := range a { - result[i] = v - } + copy(result[:], a[:]) for i, v := range b { result[i+4] = v @@ -705,14 +696,10 @@ func HashContractBytecode(bc string) (string, error) { } var in [8]uint64 - for i, value := range elementsToHash[4:12] { - in[i] = value - } + copy(in[:], elementsToHash[4:12]) var capacity [4]uint64 - for i, value := range elementsToHash[:4] { - capacity[i] = value - } + copy(capacity[:], elementsToHash[:4]) tmpHash, err = Hash(in, capacity) if err != nil { diff --git a/zk/debug_tools/l1-block-finder/main.go b/zk/debug_tools/l1-block-finder/main.go index 99ac41ae9ca..c09387a48d8 100644 --- a/zk/debug_tools/l1-block-finder/main.go +++ b/zk/debug_tools/l1-block-finder/main.go @@ -1,13 +1,14 @@ package main import ( + "encoding/json" "flag" "fmt" - "net/http" - "strings" - "encoding/json" "io" "math/big" + "net/http" + "strconv" + "strings" ) const SequenceBatchesTopic = "0x303446e6a8cb73c83dff421c0b1d5e5ce0719dab1bff13660fc254e58cc17fce" @@ -101,8 +102,8 @@ func main() { } func makeRequest(startBlock, endBlock uint64, contracts string) (*Response, error) { - startHex := fmt.Sprintf("0x%x", startBlock) - endHex := fmt.Sprintf("0x%x", endBlock) + startHex := "0x" + strconv.FormatUint(startBlock, 16) + endHex := "0x" + strconv.FormatUint(endBlock, 16) reqBody := fmt.Sprintf(req, startHex, endHex, contracts, SequenceBatchesTopic, SequenceBatchesTopic2) diff --git a/zk/debug_tools/l1-info-tree-builder/main.go b/zk/debug_tools/l1-info-tree-builder/main.go index a19991a647f..f326c0ef3e0 100644 --- a/zk/debug_tools/l1-info-tree-builder/main.go +++ b/zk/debug_tools/l1-info-tree-builder/main.go @@ -1,17 +1,19 @@ package main import ( - "flag" - "net/http" - "strings" "encoding/json" - "io" + "flag" "fmt" + "io" "math/big" + "net/http" "sort" - "github.com/ledgerwatch/erigon/zk/l1infotree" - "github.com/iden3/go-iden3-crypto/keccak256" + "strconv" + "strings" + libcommon "github.com/gateway-fm/cdk-erigon-lib/common" + "github.com/iden3/go-iden3-crypto/keccak256" + "github.com/ledgerwatch/erigon/zk/l1infotree" ) type BlockNumberResponse struct { @@ -119,8 +121,8 @@ func main() { stop = true } fmt.Println("Requesting logs from", from, "to", end) - fromAsHex := fmt.Sprintf("0x%x", from) - endAsHex := fmt.Sprintf("0x%x", end) + fromAsHex := "0x" + strconv.FormatUint(from, 16) + endAsHex := "0x" + strconv.FormatUint(end, 16) reqBody := fmt.Sprintf(logReq, fromAsHex, endAsHex, address) req, err := http.NewRequest("POST", endpoint, strings.NewReader(reqBody)) if err != nil { diff --git a/zk/debug_tools/nightly-block-compare-wait/main.go b/zk/debug_tools/nightly-block-compare-wait/main.go index 43ada1d37ce..22a9348f417 100644 --- a/zk/debug_tools/nightly-block-compare-wait/main.go +++ b/zk/debug_tools/nightly-block-compare-wait/main.go @@ -165,7 +165,7 @@ func checkBlocks(erigonNodeURL, compareNodeURL string, maxBlockDiff int) (int64, return 0, false } - lowestBlockHex := fmt.Sprintf("0x%x", lowestBlockNumber) + lowestBlockHex := "0x" + strconv.FormatInt(lowestBlockNumber, 16) erigonBlock, err := getBlockByNumber(erigonNodeURL, lowestBlockHex) if err != nil { diff --git a/zk/debug_tools/nightly-block-compare/main.go b/zk/debug_tools/nightly-block-compare/main.go index 6184acecada..bc65479cfff 100644 --- a/zk/debug_tools/nightly-block-compare/main.go +++ b/zk/debug_tools/nightly-block-compare/main.go @@ -10,8 +10,9 @@ import ( "net/http" "os" - "github.com/google/go-cmp/cmp" "io" + + "github.com/google/go-cmp/cmp" ) func getLatestBlockNumber(url string) (*big.Int, error) { @@ -58,7 +59,7 @@ func getBlockByNumber(url string, number *big.Int) (map[string]interface{}, erro requestBody, _ := json.Marshal(map[string]interface{}{ "jsonrpc": "2.0", "method": "eth_getBlockByNumber", - "params": []interface{}{fmt.Sprintf("0x%x", number), false}, + "params": []interface{}{"0x" + number.Text(16), false}, "id": 1, }) diff --git a/zk/legacy_executor_verifier/executor.go b/zk/legacy_executor_verifier/executor.go index 79de831d1eb..5bb839af274 100644 --- a/zk/legacy_executor_verifier/executor.go +++ b/zk/legacy_executor_verifier/executor.go @@ -7,6 +7,7 @@ import ( "fmt" "time" + "encoding/hex" "encoding/json" "os" "path" @@ -193,14 +194,14 @@ func (e *Executor) Verify(p *Payload, request *VerifierRequest, oldStateRoot com // now save the witness as a hex string along with the datastream // this is to allow for easy debugging of the witness and datastream witnessHexFile := path.Join(e.outputLocation, fmt.Sprintf("witness_%d.hex", request.BatchNumber)) - witnessAsHex := fmt.Sprintf("0x%x", p.Witness) + witnessAsHex := "0x" + hex.EncodeToString(p.Witness) err = os.WriteFile(witnessHexFile, []byte(witnessAsHex), 0644) if err != nil { return false, nil, err } dataStreamHexFile := path.Join(e.outputLocation, fmt.Sprintf("datastream_%d.hex", request.BatchNumber)) - dataStreamAsHex := fmt.Sprintf("0x%x", p.DataStream) + dataStreamAsHex := "0x" + hex.EncodeToString(p.DataStream) err = os.WriteFile(dataStreamHexFile, []byte(dataStreamAsHex), 0644) if err != nil { return false, nil, err diff --git a/zk/stages/stage_interhashes.go b/zk/stages/stage_interhashes.go index 93c77c598ca..e4f25df258e 100644 --- a/zk/stages/stage_interhashes.go +++ b/zk/stages/stage_interhashes.go @@ -414,7 +414,7 @@ func zkIncrementIntermediateHashes(ctx context.Context, logPrefix string, s *sta ach := hexutils.BytesToHex(cc) if len(ach) > 0 { - hexcc := fmt.Sprintf("0x%s", ach) + hexcc := "0x" + ach codeChanges[addr] = hexcc if err != nil { return trie.EmptyRoot, err @@ -582,7 +582,7 @@ func unwindZkSMT(ctx context.Context, logPrefix string, from, to uint64, db kv.R ach := hexutils.BytesToHex(cc) hexcc := "" if len(ach) > 0 { - hexcc = fmt.Sprintf("0x%s", ach) + hexcc = "0x" + ach } codeChanges[addr] = hexcc } @@ -668,7 +668,7 @@ func processAccount(db smt.DB, a *accounts.Account, as map[string]string, inc ui ach := hexutils.BytesToHex(cc) if len(ach) > 0 { - hexcc := fmt.Sprintf("0x%s", ach) + hexcc := "0x" + ach keys, err = insertContractBytecodeToKV(db, keys, addr.String(), hexcc) if err != nil { return []utils.NodeKey{}, err From 0e7df876a730bf5a07c58e8ea2ab4b93d4b30987 Mon Sep 17 00:00:00 2001 From: Valentin Staykov <79150443+V-Staykov@users.noreply.github.com> Date: Thu, 18 Jul 2024 17:18:09 +0300 Subject: [PATCH 11/19] 768 sequencerrpc l1 info tree duplicate logs (#821) * save l1 info tree leaf only if not already written * add db method --- zk/hermez_db/db.go | 23 ++++++++++++++++++ zk/stages/stage_l1_info_tree.go | 42 ++++++++++++++++++++------------- 2 files changed, 48 insertions(+), 17 deletions(-) diff --git a/zk/hermez_db/db.go b/zk/hermez_db/db.go index b087dfd85a4..efda404ac85 100644 --- a/zk/hermez_db/db.go +++ b/zk/hermez_db/db.go @@ -1,6 +1,7 @@ package hermez_db import ( + "bytes" "errors" "fmt" "math" @@ -1447,6 +1448,28 @@ func (db *HermezDb) WriteL1InfoTreeLeaf(l1Index uint64, leaf common.Hash) error return db.tx.Put(L1_INFO_LEAVES, Uint64ToBytes(l1Index), leaf.Bytes()) } +func (db *HermezDbReader) IsL1InfoTreeLeafSaves(leaf common.Hash) (bool, error) { + c, err := db.tx.Cursor(L1_INFO_LEAVES) + if err != nil { + return false, err + } + defer c.Close() + + leafBytes := leaf.Bytes() + + for k, v, err := c.First(); k != nil; k, v, err = c.Next() { + if err != nil { + return false, err + } + + if bytes.Equal(v, leafBytes) { + return true, nil + } + } + + return false, nil +} + func (db *HermezDbReader) GetAllL1InfoTreeLeaves() ([]common.Hash, error) { c, err := db.tx.Cursor(L1_INFO_LEAVES) if err != nil { diff --git a/zk/stages/stage_l1_info_tree.go b/zk/stages/stage_l1_info_tree.go index 211a486ba29..e9d39c50a90 100644 --- a/zk/stages/stage_l1_info_tree.go +++ b/zk/stages/stage_l1_info_tree.go @@ -148,27 +148,35 @@ LOOP: leafHash := l1infotree.HashLeafData(latestUpdate.GER, latestUpdate.ParentHash, latestUpdate.Timestamp) - err = hermezDb.WriteL1InfoTreeLeaf(latestUpdate.Index, leafHash) + leafFoundInDb, err := hermezDb.IsL1InfoTreeLeafSaves(leafHash) if err != nil { return err } - newRoot, err := tree.AddLeaf(uint32(latestUpdate.Index), leafHash) - if err != nil { - return err - } - log.Debug("New L1 Index", - "index", latestUpdate.Index, - "root", newRoot.String(), - "mainnet", latestUpdate.MainnetExitRoot.String(), - "rollup", latestUpdate.RollupExitRoot.String(), - "ger", latestUpdate.GER.String(), - "parent", latestUpdate.ParentHash.String(), - ) - - err = hermezDb.WriteL1InfoTreeRoot(common.BytesToHash(newRoot[:]), latestUpdate.Index) - if err != nil { - return err + if leafFoundInDb { + log.Warn("Leaf already saved in db", "index", latestUpdate.Index, "hash", leafHash) + } else { + if err = hermezDb.WriteL1InfoTreeLeaf(latestUpdate.Index, leafHash); err != nil { + return err + } + + newRoot, err := tree.AddLeaf(uint32(latestUpdate.Index), leafHash) + if err != nil { + return err + } + log.Debug("New L1 Index", + "index", latestUpdate.Index, + "root", newRoot.String(), + "mainnet", latestUpdate.MainnetExitRoot.String(), + "rollup", latestUpdate.RollupExitRoot.String(), + "ger", latestUpdate.GER.String(), + "parent", latestUpdate.ParentHash.String(), + ) + + err = hermezDb.WriteL1InfoTreeRoot(common.BytesToHash(newRoot[:]), latestUpdate.Index) + if err != nil { + return err + } } processed++ From af4ccd3946f14d7c0db93b7d1e0f8b23538fd677 Mon Sep 17 00:00:00 2001 From: Valentin Staykov <79150443+V-Staykov@users.noreply.github.com> Date: Thu, 18 Jul 2024 17:22:25 +0300 Subject: [PATCH 12/19] Block info tree optimizations2 (#820) * create variables for constants that were always recomputed before * ewmove hash save on block info tree build * fix tests --- core/genesis_write.go | 6 +-- smt/pkg/blockinfo/block_info.go | 48 ++++------------- smt/pkg/blockinfo/block_info_test.go | 24 ++++----- smt/pkg/blockinfo/keys.go | 12 +++++ smt/pkg/blockinfo/keys_test.go | 15 ++++++ smt/pkg/smt/entity_storage_mdbx_test.go | 12 ++--- smt/pkg/smt/entity_storage_test.go | 6 +-- smt/pkg/smt/smt.go | 36 +++++++------ smt/pkg/smt/smt_batch_test.go | 66 ++++++++++++++++------- smt/pkg/smt/smt_create_test.go | 8 +-- smt/pkg/smt/smt_test.go | 14 ++--- smt/pkg/smt/witness_test.go | 4 +- smt/pkg/utils/util_test.go | 7 +++ smt/pkg/utils/utils.go | 20 ++++--- zk/stages/stage_interhashes.go | 4 +- zk/stages/stage_sequence_execute_utils.go | 2 +- zk/tx/tx.go | 8 ++- zk/tx/tx_test.go | 21 ++++++++ zk/witness/witness.go | 7 +-- 19 files changed, 189 insertions(+), 131 deletions(-) diff --git a/core/genesis_write.go b/core/genesis_write.go index e90c5781da6..feca2c0c68e 100644 --- a/core/genesis_write.go +++ b/core/genesis_write.go @@ -27,12 +27,12 @@ import ( "sync" "github.com/c2h5oh/datasize" - "github.com/holiman/uint256" erigonchain "github.com/gateway-fm/cdk-erigon-lib/chain" libcommon "github.com/gateway-fm/cdk-erigon-lib/common" "github.com/gateway-fm/cdk-erigon-lib/kv" "github.com/gateway-fm/cdk-erigon-lib/kv/mdbx" "github.com/gateway-fm/cdk-erigon-lib/kv/rawdbv3" + "github.com/holiman/uint256" "github.com/ledgerwatch/erigon/chain" "github.com/ledgerwatch/log/v3" @@ -49,8 +49,8 @@ import ( "github.com/ledgerwatch/erigon/eth/ethconfig" "github.com/ledgerwatch/erigon/params" "github.com/ledgerwatch/erigon/params/networkname" - "github.com/ledgerwatch/erigon/smt/pkg/smt" eridb "github.com/ledgerwatch/erigon/smt/pkg/db" + "github.com/ledgerwatch/erigon/smt/pkg/smt" "golang.org/x/exp/slices" ) @@ -525,7 +525,7 @@ func GenesisToBlock(g *types.Genesis, tmpDir string) (*types.Block, *state.Intra wg.Add(1) var err error sparseDb := eridb.NewMemDb() - sparseTree := smt.NewSMT(sparseDb) + sparseTree := smt.NewSMT(sparseDb, false) go func() { // we may run inside write tx, can't open 2nd write tx in same goroutine // TODO(yperbasis): use memdb.MemoryMutation instead defer wg.Done() diff --git a/smt/pkg/blockinfo/block_info.go b/smt/pkg/blockinfo/block_info.go index 3f2e97bbdf3..612df1504b3 100644 --- a/smt/pkg/blockinfo/block_info.go +++ b/smt/pkg/blockinfo/block_info.go @@ -104,7 +104,7 @@ type BlockInfoTree struct { func NewBlockInfoTree() *BlockInfoTree { return &BlockInfoTree{ - smt: smt.NewSMT(nil), + smt: smt.NewSMT(nil, true), } } func (b *BlockInfoTree) GetRoot() *big.Int { @@ -147,83 +147,56 @@ func (b *BlockInfoTree) GenerateBlockHeader(oldBlockHash *common.Hash, coinbase } func generateL2BlockHash(blockHash *common.Hash) (key *utils.NodeKey, value *utils.NodeValue8, err error) { - if key, err = KeyBlockHeaderParams(big.NewInt(IndexBlockHeaderParamBlockHash)); err != nil { - return nil, nil, err - } if value, err = bigInt2NodeVal8(blockHash.Big()); err != nil { return nil, nil, err } - return key, value, nil + return &BlockHeaderBlockHashKey, value, nil } func generateCoinbase(coinbase *common.Address) (key *utils.NodeKey, value *utils.NodeValue8, err error) { - if key, err = KeyBlockHeaderParams(big.NewInt(IndexBlockHeaderParamCoinbase)); err != nil { - return nil, nil, err - } - if value, err = bigInt2NodeVal8(coinbase.Hash().Big()); err != nil { return nil, nil, err } - return key, value, nil + return &BlockHeaderCoinbaseKey, value, nil } func generateGasLimit(gasLimit uint64) (key *utils.NodeKey, value *utils.NodeValue8, err error) { - if key, err = KeyBlockHeaderParams(big.NewInt(IndexBlockHeaderParamGasLimit)); err != nil { - return nil, nil, err - } - if value, err = bigInt2NodeVal8(big.NewInt(0).SetUint64(gasLimit)); err != nil { return nil, nil, err } - return key, value, nil + return &BlockHeaderGasLimitKey, value, nil } func generateBlockNumber(blockNumber uint64) (key *utils.NodeKey, value *utils.NodeValue8, err error) { - if key, err = KeyBlockHeaderParams(big.NewInt(IndexBlockHeaderParamNumber)); err != nil { - return nil, nil, err - } - if value, err = bigInt2NodeVal8(big.NewInt(0).SetUint64(blockNumber)); err != nil { return nil, nil, err } - return key, value, nil + return &BlockHeaderNumberKey, value, nil } func generateTimestamp(timestamp uint64) (key *utils.NodeKey, value *utils.NodeValue8, err error) { - if key, err = KeyBlockHeaderParams(big.NewInt(IndexBlockHeaderParamTimestamp)); err != nil { - return nil, nil, err - } - if value, err = bigInt2NodeVal8(big.NewInt(0).SetUint64(timestamp)); err != nil { return nil, nil, err } - return key, value, nil + return &BlockHeaderTimestampKey, value, nil } func generateGer(ger *common.Hash) (key *utils.NodeKey, value *utils.NodeValue8, err error) { - if key, err = KeyBlockHeaderParams(big.NewInt(IndexBlockHeaderParamGer)); err != nil { - return nil, nil, err - } - if value, err = bigInt2NodeVal8(ger.Big()); err != nil { return nil, nil, err } - return key, value, nil + return &BlockHeaderGerKey, value, nil } func generateL1BlockHash(blockHash *common.Hash) (key *utils.NodeKey, value *utils.NodeValue8, err error) { - if key, err = KeyBlockHeaderParams(big.NewInt(IndexBlockHeaderParamBlockHashL1)); err != nil { - return nil, nil, err - } - if value, err = bigInt2NodeVal8(blockHash.Big()); err != nil { return nil, nil, err } - return key, value, nil + return &BlockHeaderBlockHashL1Key, value, nil } func bigInt2NodeVal8(val *big.Int) (*utils.NodeValue8, error) { @@ -291,15 +264,12 @@ func generateTxEffectivePercentage(txIndex, effectivePercentage *big.Int) (key * } func generateBlockGasUsed(gasUsed uint64) (key *utils.NodeKey, value *utils.NodeValue8, err error) { - if key, err = KeyBlockHeaderParams(big.NewInt(IndexBlockHeaderParamGasUsed)); err != nil { - return nil, nil, err - } gasUsedBig := big.NewInt(0).SetUint64(gasUsed) if value, err = bigInt2NodeVal8(gasUsedBig); err != nil { return nil, nil, err } - return key, value, nil + return &BlockHeaderGasUsedKey, value, nil } func (b *BlockInfoTree) GenerateBlockTxKeysVals( diff --git a/smt/pkg/blockinfo/block_info_test.go b/smt/pkg/blockinfo/block_info_test.go index 1c1f13dd3f9..42ddfaea093 100644 --- a/smt/pkg/blockinfo/block_info_test.go +++ b/smt/pkg/blockinfo/block_info_test.go @@ -288,7 +288,7 @@ func TestSetL2BlockHash(t *testing.T) { if err != nil { t.Fatal(err) } - smt := smt.NewSMT(nil) + smt := smt.NewSMT(nil, true) root, err2 := smt.InsertKA(*key, smtutils.NodeValue8ToBigInt(val)) if err2 != nil { @@ -314,7 +314,7 @@ func TestSetCoinbase(t *testing.T) { } for i, test := range tests { - smt := smt.NewSMT(nil) + smt := smt.NewSMT(nil, true) coinbaseAddress := common.HexToAddress(test.coinbaseAddress) key, val, err := generateCoinbase(&coinbaseAddress) @@ -348,7 +348,7 @@ func TestSetBlockNumber(t *testing.T) { } for i, test := range tests { - smt := smt.NewSMT(nil) + smt := smt.NewSMT(nil, true) key, val, err := generateBlockNumber(test.blockNum) if err != nil { @@ -379,7 +379,7 @@ func TestSetGasLimit(t *testing.T) { } for i, test := range tests { - smt := smt.NewSMT(nil) + smt := smt.NewSMT(nil, true) key, val, err := generateGasLimit(test.gasLimit) if err != nil { @@ -410,7 +410,7 @@ func TestSetTimestamp(t *testing.T) { } for i, test := range tests { - smt := smt.NewSMT(nil) + smt := smt.NewSMT(nil, true) key, val, err := generateTimestamp(test.timestamp) if err != nil { @@ -447,7 +447,7 @@ func TestSetGer(t *testing.T) { } for i, test := range tests { - smt := smt.NewSMT(nil) + smt := smt.NewSMT(nil, true) ger := common.HexToHash(test.ger) key, val, err := generateGer(&ger) @@ -485,7 +485,7 @@ func TestSetL1BlockHash(t *testing.T) { } for i, test := range tests { - smt := smt.NewSMT(nil) + smt := smt.NewSMT(nil, true) l1BlockHash := common.HexToHash(test.l1BlockHash) key, val, err := generateL1BlockHash(&l1BlockHash) @@ -506,7 +506,7 @@ func TestSetL1BlockHash(t *testing.T) { } func TestSetL2TxHash(t *testing.T) { - smt := smt.NewSMT(nil) + smt := smt.NewSMT(nil, true) txIndex := big.NewInt(1) l2TxHash := common.HexToHash("0x000000000000000000000000000000005Ca1aB1E").Big() @@ -529,7 +529,7 @@ func TestSetL2TxHash(t *testing.T) { } func TestSetTxStatus(t *testing.T) { - smt := smt.NewSMT(nil) + smt := smt.NewSMT(nil, true) txIndex := big.NewInt(1) status := common.HexToHash("0x000000000000000000000000000000005Ca1aB1E").Big() @@ -552,7 +552,7 @@ func TestSetTxStatus(t *testing.T) { } func TestSetCumulativeGasUsed(t *testing.T) { - smt := smt.NewSMT(nil) + smt := smt.NewSMT(nil, true) txIndex := big.NewInt(1) cgu := common.HexToHash("0x000000000000000000000000000000005Ca1aB1E").Big() @@ -576,7 +576,7 @@ func TestSetCumulativeGasUsed(t *testing.T) { } func TestSetTxEffectivePercentage(t *testing.T) { - smt := smt.NewSMT(nil) + smt := smt.NewSMT(nil, true) txIndex := big.NewInt(1) egp := common.HexToHash("0x000000000000000000000000000000005Ca1aB1E").Big() @@ -600,7 +600,7 @@ func TestSetTxEffectivePercentage(t *testing.T) { } func TestSetTxLogs(t *testing.T) { - smt := smt.NewSMT(nil) + smt := smt.NewSMT(nil, true) txIndex := big.NewInt(1) logIndex := big.NewInt(1) log := common.HexToHash("0x000000000000000000000000000000005Ca1aB1E").Big() diff --git a/smt/pkg/blockinfo/keys.go b/smt/pkg/blockinfo/keys.go index de3a8a4d17c..af9cfdef475 100644 --- a/smt/pkg/blockinfo/keys.go +++ b/smt/pkg/blockinfo/keys.go @@ -17,6 +17,18 @@ const IndexBlockHeaderParamGer = 5 const IndexBlockHeaderParamBlockHashL1 = 6 const IndexBlockHeaderParamGasUsed = 7 +// generated by KeyBlockHeaderParams so we don't calculate them every time +var ( + BlockHeaderBlockHashKey = utils.NodeKey{17540094328570681229, 15492539097581145461, 7686481670809850401, 16577991319572125169} + BlockHeaderCoinbaseKey = utils.NodeKey{13866806033333411216, 11510953292839890698, 8274877395843603978, 9372332419316597113} + BlockHeaderNumberKey = utils.NodeKey{6024064788222257862, 13049342112699253445, 12127984136733687200, 8398043461199794462} + BlockHeaderGasLimitKey = utils.NodeKey{5319681466197319121, 14057433120745733551, 5638531288094714593, 17204828339478940337} + BlockHeaderTimestampKey = utils.NodeKey{7890158832167317866, 11032486557242372179, 9653801891436451408, 2062577087515942703} + BlockHeaderGerKey = utils.NodeKey{16031278424721309229, 4132999715765882778, 6388713709192801251, 10826219431775251904} + BlockHeaderBlockHashL1Key = utils.NodeKey{5354929451503733866, 3129555839551084896, 2132809659008379950, 8230742270813566472} + BlockHeaderGasUsedKey = utils.NodeKey{8577769200631379655, 8682051454686970557, 5016656739138242322, 16717481432904730287} +) + // SMT block header constant keys const IndexBlockHeaderParam = 7 const IndexBlockHeaderTransactionHash = 8 diff --git a/smt/pkg/blockinfo/keys_test.go b/smt/pkg/blockinfo/keys_test.go index dcd70fa8f8f..6b52b834d9e 100644 --- a/smt/pkg/blockinfo/keys_test.go +++ b/smt/pkg/blockinfo/keys_test.go @@ -11,44 +11,58 @@ import ( func TestKeyBlockHeaderParams(t *testing.T) { scenarios := map[string]struct { param *big.Int + constKey utils.NodeKey expected utils.NodeKey shouldFail bool }{ "KeyBlockHash": { param: big.NewInt(IndexBlockHeaderParamBlockHash), + constKey: BlockHeaderBlockHashKey, expected: utils.NodeKey{17540094328570681229, 15492539097581145461, 7686481670809850401, 16577991319572125169}, shouldFail: false, }, "KeyCoinbase": { param: big.NewInt(IndexBlockHeaderParamCoinbase), + constKey: BlockHeaderCoinbaseKey, expected: utils.NodeKey{13866806033333411216, 11510953292839890698, 8274877395843603978, 9372332419316597113}, shouldFail: false, }, "KeyBlockNumber": { param: big.NewInt(IndexBlockHeaderParamNumber), + constKey: BlockHeaderNumberKey, expected: utils.NodeKey{6024064788222257862, 13049342112699253445, 12127984136733687200, 8398043461199794462}, shouldFail: false, }, "KeyGasLimit": { param: big.NewInt(IndexBlockHeaderParamGasLimit), + constKey: BlockHeaderGasLimitKey, expected: utils.NodeKey{5319681466197319121, 14057433120745733551, 5638531288094714593, 17204828339478940337}, shouldFail: false, }, "KeyTimestamp": { param: big.NewInt(IndexBlockHeaderParamTimestamp), + constKey: BlockHeaderTimestampKey, expected: utils.NodeKey{7890158832167317866, 11032486557242372179, 9653801891436451408, 2062577087515942703}, shouldFail: false, }, "KeyGer": { param: big.NewInt(IndexBlockHeaderParamGer), + constKey: BlockHeaderGerKey, expected: utils.NodeKey{16031278424721309229, 4132999715765882778, 6388713709192801251, 10826219431775251904}, shouldFail: false, }, "KeyBlockHashL1": { param: big.NewInt(IndexBlockHeaderParamBlockHashL1), + constKey: BlockHeaderBlockHashL1Key, expected: utils.NodeKey{5354929451503733866, 3129555839551084896, 2132809659008379950, 8230742270813566472}, shouldFail: false, }, + "KeyGasUsed": { + param: big.NewInt(IndexBlockHeaderParamGasUsed), + constKey: BlockHeaderGasUsedKey, + expected: utils.NodeKey{8577769200631379655, 8682051454686970557, 5016656739138242322, 16717481432904730287}, + shouldFail: false, + }, "NilKey": { param: nil, expected: utils.NodeKey{}, @@ -64,6 +78,7 @@ func TestKeyBlockHeaderParams(t *testing.T) { } else { assert.NoError(t, err) assert.Equal(t, scenario.expected, *val) + assert.Equal(t, scenario.constKey, *val) } }) } diff --git a/smt/pkg/smt/entity_storage_mdbx_test.go b/smt/pkg/smt/entity_storage_mdbx_test.go index ac16e48b570..22c34bc87ca 100644 --- a/smt/pkg/smt/entity_storage_mdbx_test.go +++ b/smt/pkg/smt/entity_storage_mdbx_test.go @@ -27,7 +27,7 @@ func TestSMT_Mdbx_AddRemove1Element(t *testing.T) { } //defer dbi.Close() - s := NewSMT(sdb) + s := NewSMT(sdb, false) r, _ := s.InsertBI(big.NewInt(1), big.NewInt(2)) if r.Mode != "insertNotFound" { @@ -50,7 +50,7 @@ func TestSMT_Mdbx_AddRemove3Elements(t *testing.T) { t.Errorf("Failed to create temp db: %v", err) } - s := NewSMT(sdb) + s := NewSMT(sdb, false) N := 3 var r *SMTResponse @@ -82,7 +82,7 @@ func TestSMT_Mdbx_AddRemove128Elements(t *testing.T) { t.Errorf("Failed to create temp db: %v", err) } - s := NewSMT(sdb) + s := NewSMT(sdb, false) N := 128 var r *SMTResponse @@ -163,7 +163,7 @@ func TestSMT_Mdbx_MultipleInsert(t *testing.T) { tr = x } - s := NewSMT(sdb) + s := NewSMT(sdb, false) s.SetLastRoot(tr) r, err := s.InsertBI(testCase.key, testCase.value) @@ -217,7 +217,7 @@ func runGenesisTestMdbx(tb testing.TB, filename string) { tb.Fatal("Failed to create db buckets: ", err) } - smt := NewSMT(sdb) + smt := NewSMT(sdb, false) for _, addr := range genesis.Genesis { fmt.Println(addr.ContractName) @@ -275,7 +275,7 @@ func runTestVectorsMdbx(t *testing.T, filename string) { for k, tc := range testCases { t.Run(strconv.Itoa(k), func(t *testing.T) { - smt := NewSMT(nil) + smt := NewSMT(nil, false) for _, addr := range tc.Addresses { bal, _ := new(big.Int).SetString(addr.Balance, 10) diff --git a/smt/pkg/smt/entity_storage_test.go b/smt/pkg/smt/entity_storage_test.go index 3688f590e59..ca057a5674e 100644 --- a/smt/pkg/smt/entity_storage_test.go +++ b/smt/pkg/smt/entity_storage_test.go @@ -72,7 +72,7 @@ func runGenesisTest(tb testing.TB, filename string) { tb.Fatal("Failed to parse json: ", err) } - smt := NewSMT(nil) + smt := NewSMT(nil, false) for _, addr := range genesis.Genesis { fmt.Println(addr.ContractName) @@ -130,7 +130,7 @@ func runTestVectors(t *testing.T, filename string) { for k, tc := range testCases { t.Run(strconv.Itoa(k), func(t *testing.T) { - smt := NewSMT(nil) + smt := NewSMT(nil, false) for _, addr := range tc.Addresses { bal, _ := new(big.Int).SetString(addr.Balance, 10) @@ -163,7 +163,7 @@ func runTestVectors(t *testing.T, filename string) { } func Test_FullGenesisTest_Deprecated(t *testing.T) { - s := NewSMT(nil) + s := NewSMT(nil, false) e := utils.NodeKey{ 13946701032480821596, diff --git a/smt/pkg/smt/smt.go b/smt/pkg/smt/smt.go index 31175259a9d..2a2ad956edc 100644 --- a/smt/pkg/smt/smt.go +++ b/smt/pkg/smt/smt.go @@ -51,7 +51,8 @@ type DebuggableDB interface { } type SMT struct { - Db DB + noSaveOnInsert bool + Db DB *RoSMT } @@ -65,14 +66,15 @@ type SMTResponse struct { Mode string } -func NewSMT(database DB) *SMT { +func NewSMT(database DB, noSaveOnInsert bool) *SMT { if database == nil { database = db.NewMemDb() } return &SMT{ - Db: database, - RoSMT: NewRoSMT(database), + noSaveOnInsert: noSaveOnInsert, + Db: database, + RoSMT: NewRoSMT(database), } } @@ -536,19 +538,23 @@ func (s *SMT) insert(k utils.NodeKey, v utils.NodeValue8, newValH [4]uint64, old } func (s *SMT) hashSave(in [8]uint64, capacity, h [4]uint64) ([4]uint64, error) { - var sl []uint64 - sl = append(sl, in[:]...) - sl = append(sl, capacity[:]...) + if !s.noSaveOnInsert { + var sl []uint64 + sl = append(sl, in[:]...) + sl = append(sl, capacity[:]...) + + v := utils.NodeValue12{} + for i, val := range sl { + b := new(big.Int) + v[i] = b.SetUint64(val) + } - v := utils.NodeValue12{} - for i, val := range sl { - b := new(big.Int) - v[i] = b.SetUint64(val) + err := s.Db.Insert(h, v) + if err != nil { + return [4]uint64{}, err + } } - - err := s.Db.Insert(h, v) - - return h, err + return h, nil } func (s *SMT) hashcalcAndSave(in [8]uint64, capacity [4]uint64) ([4]uint64, error) { diff --git a/smt/pkg/smt/smt_batch_test.go b/smt/pkg/smt/smt_batch_test.go index 30cba8fd343..31b74328d6e 100644 --- a/smt/pkg/smt/smt_batch_test.go +++ b/smt/pkg/smt/smt_batch_test.go @@ -55,8 +55,9 @@ func TestBatchSimpleInsert(t *testing.T) { keyPointers := []*utils.NodeKey{} valuePointers := []*utils.NodeValue8{} - smtIncremental := smt.NewSMT(nil) - smtBatch := smt.NewSMT(nil) + smtIncremental := smt.NewSMT(nil, false) + smtBatch := smt.NewSMT(nil, false) + smtBatchNoSave := smt.NewSMT(nil, true) for i := range keysRaw { k := utils.ScalarToNodeKey(keysRaw[i]) @@ -72,6 +73,9 @@ func TestBatchSimpleInsert(t *testing.T) { _, err := smtBatch.InsertBatch(context.Background(), "", keyPointers, valuePointers, nil, nil) assert.NilError(t, err) + _, err = smtBatchNoSave.InsertBatch(context.Background(), "", keyPointers, valuePointers, nil, nil) + assert.NilError(t, err) + smtIncremental.DumpTree() fmt.Println() smtBatch.DumpTree() @@ -81,7 +85,9 @@ func TestBatchSimpleInsert(t *testing.T) { smtIncrementalRootHash, _ := smtIncremental.Db.GetLastRoot() smtBatchRootHash, _ := smtBatch.Db.GetLastRoot() + smtBatchNoSaveRootHash, _ := smtBatchNoSave.Db.GetLastRoot() assert.Equal(t, utils.ConvertBigIntToHex(smtBatchRootHash), utils.ConvertBigIntToHex(smtIncrementalRootHash)) + assert.Equal(t, utils.ConvertBigIntToHex(smtBatchRootHash), utils.ConvertBigIntToHex(smtBatchNoSaveRootHash)) assertSmtDbStructure(t, smtBatch, false) } @@ -121,7 +127,7 @@ func BenchmarkIncrementalInsert(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - smtIncremental := smt.NewSMT(nil) + smtIncremental := smt.NewSMT(nil, false) incrementalInsert(smtIncremental, keys, vals) } } @@ -139,7 +145,25 @@ func BenchmarkBatchInsert(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - smtBatch := smt.NewSMT(nil) + smtBatch := smt.NewSMT(nil, false) + batchInsert(smtBatch, keys, vals) + } +} + +func BenchmarkBatchInsertNoSave(b *testing.B) { + keys := []*big.Int{} + vals := []*big.Int{} + for i := 0; i < 1000; i++ { + rand.Seed(time.Now().UnixNano()) + keys = append(keys, big.NewInt(int64(rand.Intn(10000)))) + + rand.Seed(time.Now().UnixNano()) + vals = append(vals, big.NewInt(int64(rand.Intn(10000)))) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + smtBatch := smt.NewSMT(nil, true) batchInsert(smtBatch, keys, vals) } } @@ -155,15 +179,21 @@ func TestBatchSimpleInsert2(t *testing.T) { vals = append(vals, big.NewInt(int64(rand.Intn(10000)))) } - smtIncremental := smt.NewSMT(nil) + smtIncremental := smt.NewSMT(nil, false) incrementalInsert(smtIncremental, keys, vals) - smtBatch := smt.NewSMT(nil) + smtBatch := smt.NewSMT(nil, false) batchInsert(smtBatch, keys, vals) + smtBatchNoSave := smt.NewSMT(nil, false) + batchInsert(smtBatchNoSave, keys, vals) + smtIncrementalRootHash, _ := smtIncremental.Db.GetLastRoot() smtBatchRootHash, _ := smtBatch.Db.GetLastRoot() + smtBatchNoSaveRootHash, _ := smtBatchNoSave.Db.GetLastRoot() + assert.Equal(t, utils.ConvertBigIntToHex(smtBatchRootHash), utils.ConvertBigIntToHex(smtIncrementalRootHash)) + assert.Equal(t, utils.ConvertBigIntToHex(smtBatchRootHash), utils.ConvertBigIntToHex(smtBatchNoSaveRootHash)) } func TestBatchWitness(t *testing.T) { @@ -327,8 +357,8 @@ func TestBatchWitness(t *testing.T) { }) } - smtIncremental := smt.NewSMT(nil) - smtBatch := smt.NewSMT(nil) + smtIncremental := smt.NewSMT(nil, false) + smtBatch := smt.NewSMT(nil, false) for i, k := range keys { smtIncremental.Insert(k, values[i]) @@ -391,8 +421,8 @@ func TestBatchDelete(t *testing.T) { }) } - smtIncremental := smt.NewSMT(nil) - smtBatch := smt.NewSMT(nil) + smtIncremental := smt.NewSMT(nil, false) + smtBatch := smt.NewSMT(nil, false) for i, k := range keys { smtIncremental.Insert(k, values[i]) @@ -419,8 +449,8 @@ func TestBatchRawInsert(t *testing.T) { keysForIncremental := []utils.NodeKey{} valuesForIncremental := []utils.NodeValue8{} - smtIncremental := smt.NewSMT(nil) - smtBatch := smt.NewSMT(nil) + smtIncremental := smt.NewSMT(nil, false) + smtBatch := smt.NewSMT(nil, false) rand.Seed(1) size := 1 << 10 @@ -506,9 +536,9 @@ func TestCompareAllTreesInsertTimesAndFinalHashesUsingDiskDb(t *testing.T) { batchDbPath := "/tmp/smt-batch" smtBatchDb, smtBatchTx, smtBatchSmtDb := initDb(t, batchDbPath) - smtIncremental := smt.NewSMT(smtIncrementalSmtDb) - smtBulk := smt.NewSMT(smtBulkSmtDb) - smtBatch := smt.NewSMT(smtBatchSmtDb) + smtIncremental := smt.NewSMT(smtIncrementalSmtDb, false) + smtBulk := smt.NewSMT(smtBulkSmtDb, false) + smtBatch := smt.NewSMT(smtBatchSmtDb, false) compareAllTreesInsertTimesAndFinalHashes(t, smtIncremental, smtBulk, smtBatch) @@ -526,9 +556,9 @@ func TestCompareAllTreesInsertTimesAndFinalHashesUsingDiskDb(t *testing.T) { } func TestCompareAllTreesInsertTimesAndFinalHashesUsingInMemoryDb(t *testing.T) { - smtIncremental := smt.NewSMT(nil) - smtBulk := smt.NewSMT(nil) - smtBatch := smt.NewSMT(nil) + smtIncremental := smt.NewSMT(nil, false) + smtBulk := smt.NewSMT(nil, false) + smtBatch := smt.NewSMT(nil, false) compareAllTreesInsertTimesAndFinalHashes(t, smtIncremental, smtBulk, smtBatch) } diff --git a/smt/pkg/smt/smt_create_test.go b/smt/pkg/smt/smt_create_test.go index 886514a94c5..b5d45b13910 100644 --- a/smt/pkg/smt/smt_create_test.go +++ b/smt/pkg/smt/smt_create_test.go @@ -59,7 +59,7 @@ func TestSMT_Create_Insert(t *testing.T) { for _, scenario := range testCases { t.Run(scenario.name, func(t *testing.T) { - s := NewSMT(nil) + s := NewSMT(nil, false) keys := []utils.NodeKey{} for k, v := range scenario.kvMap { if !v.IsZero() { @@ -92,7 +92,7 @@ func TestSMT_Create_CompareWithRandomData(t *testing.T) { //build and benchmark the tree the first way startTime := time.Now() - s1 := NewSMT(nil) + s1 := NewSMT(nil, false) var root1 *big.Int for k, v := range kvMap { @@ -110,7 +110,7 @@ func TestSMT_Create_CompareWithRandomData(t *testing.T) { //build the tree the from kvbulk startTime = time.Now() - s2 := NewSMT(nil) + s2 := NewSMT(nil, false) // set scenario old root if fail keys := []utils.NodeKey{} for k, v := range kvMap { @@ -150,7 +150,7 @@ func TestSMT_Create_Benchmark(t *testing.T) { //build and benchmark the tree the first way startTime := time.Now() //build the tree the from kvbulk - s := NewSMT(nil) + s := NewSMT(nil, false) // set scenario old root if fail keys := []utils.NodeKey{} for k, v := range kvMap { diff --git a/smt/pkg/smt/smt_test.go b/smt/pkg/smt/smt_test.go index 2fdd122e61e..a0faba89f42 100644 --- a/smt/pkg/smt/smt_test.go +++ b/smt/pkg/smt/smt_test.go @@ -44,7 +44,7 @@ func TestSMT_SingleInsert(t *testing.T) { for _, scenario := range scenarios { t.Run(scenario.name, func(t *testing.T) { - s := NewSMT(nil) + s := NewSMT(nil, false) // set scenario old root if fail newRoot, err := s.InsertBI(scenario.k, scenario.v) if err != nil { @@ -60,7 +60,7 @@ func TestSMT_SingleInsert(t *testing.T) { } func TestSMT_MultipleInsert(t *testing.T) { - s := NewSMT(nil) + s := NewSMT(nil, false) testCases := []struct { root *big.Int key *big.Int @@ -122,7 +122,7 @@ func TestSMT_MultipleInsert(t *testing.T) { } func TestSMT_MultipleInsert3(t *testing.T) { - s := NewSMT(nil) + s := NewSMT(nil, false) testCases := []struct { root *big.Int key *big.Int @@ -170,7 +170,7 @@ func TestSMT_MultipleInsert3(t *testing.T) { } func TestSMT_UpdateElement1(t *testing.T) { - s := NewSMT(nil) + s := NewSMT(nil, false) testCases := []struct { root *big.Int key *big.Int @@ -224,7 +224,7 @@ func TestSMT_UpdateElement1(t *testing.T) { } func TestSMT_AddSharedElement2(t *testing.T) { - s := NewSMT(nil) + s := NewSMT(nil, false) r1, err := s.InsertBI(big.NewInt(8), big.NewInt(2)) if err != nil { @@ -249,7 +249,7 @@ func TestSMT_AddSharedElement2(t *testing.T) { } func TestSMT_AddRemove128Elements(t *testing.T) { - s := NewSMT(nil) + s := NewSMT(nil, false) N := 128 var r *SMTResponse @@ -272,7 +272,7 @@ func TestSMT_AddRemove128Elements(t *testing.T) { } func TestSMT_MultipleInsert2(t *testing.T) { - s := NewSMT(nil) + s := NewSMT(nil, false) testCases := []struct { root *big.Int key utils.NodeKey diff --git a/smt/pkg/smt/witness_test.go b/smt/pkg/smt/witness_test.go index 8e17cf3967a..a46d928bcf5 100644 --- a/smt/pkg/smt/witness_test.go +++ b/smt/pkg/smt/witness_test.go @@ -7,9 +7,9 @@ import ( "fmt" "testing" - "github.com/holiman/uint256" libcommon "github.com/gateway-fm/cdk-erigon-lib/common" "github.com/gateway-fm/cdk-erigon-lib/kv/memdb" + "github.com/holiman/uint256" "github.com/ledgerwatch/erigon/chain" "github.com/ledgerwatch/erigon/core/state" "github.com/ledgerwatch/erigon/smt/pkg/db" @@ -58,7 +58,7 @@ func prepareSMT(t *testing.T) (*smt.SMT, *trie.RetainList) { memdb := db.NewMemDb() - smtTrie := smt.NewSMT(memdb) + smtTrie := smt.NewSMT(memdb, false) smtTrie.SetAccountState(contract.String(), balance.ToBig(), uint256.NewInt(1).ToBig()) smtTrie.SetContractBytecode(contract.String(), hex.EncodeToString(code)) diff --git a/smt/pkg/utils/util_test.go b/smt/pkg/utils/util_test.go index 715c4f7d0ec..a1b718b3a87 100644 --- a/smt/pkg/utils/util_test.go +++ b/smt/pkg/utils/util_test.go @@ -141,6 +141,13 @@ func TestScalarToArrayBig(t *testing.T) { } } +func BenchmarkScalarToArrayBig(b *testing.B) { + scalar := big.NewInt(0x1234567890ABCDEF) + for i := 0; i < b.N; i++ { + ScalarToArrayBig(scalar) + } +} + func TestArrayBigToScalar(t *testing.T) { scalar := big.NewInt(0x1234567890ABCDEF) diff --git a/smt/pkg/utils/utils.go b/smt/pkg/utils/utils.go index befe688a3da..d40363a1fb9 100644 --- a/smt/pkg/utils/utils.go +++ b/smt/pkg/utils/utils.go @@ -480,33 +480,31 @@ func ScalarToArrayBig12(scalar *big.Int) []*big.Int { return []*big.Int{r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, r10, r11} } -func ScalarToArrayBig(scalar *big.Int) []*big.Int { - scalar = new(big.Int).Set(scalar) - mask := new(big.Int) - mask.SetString("FFFFFFFF", 16) +var mask = big.NewInt(4294967295) +func ScalarToArrayBig(scalar *big.Int) []*big.Int { r0 := new(big.Int).And(scalar, mask) r1 := new(big.Int).Rsh(scalar, 32) - r1 = new(big.Int).And(r1, mask) + r1.And(r1, mask) r2 := new(big.Int).Rsh(scalar, 64) - r2 = new(big.Int).And(r2, mask) + r2.And(r2, mask) r3 := new(big.Int).Rsh(scalar, 96) - r3 = new(big.Int).And(r3, mask) + r3.And(r3, mask) r4 := new(big.Int).Rsh(scalar, 128) - r4 = new(big.Int).And(r4, mask) + r4.And(r4, mask) r5 := new(big.Int).Rsh(scalar, 160) - r5 = new(big.Int).And(r5, mask) + r5.And(r5, mask) r6 := new(big.Int).Rsh(scalar, 192) - r6 = new(big.Int).And(r6, mask) + r6.And(r6, mask) r7 := new(big.Int).Rsh(scalar, 224) - r7 = new(big.Int).And(r7, mask) + r7.And(r7, mask) return []*big.Int{r0, r1, r2, r3, r4, r5, r6, r7} } diff --git a/zk/stages/stage_interhashes.go b/zk/stages/stage_interhashes.go index e4f25df258e..c63e06e537d 100644 --- a/zk/stages/stage_interhashes.go +++ b/zk/stages/stage_interhashes.go @@ -127,7 +127,7 @@ func SpawnZkIntermediateHashesStage(s *stagedsync.StageState, u stagedsync.Unwin shouldRegenerate := to > s.BlockNumber && to-s.BlockNumber > cfg.zk.RebuildTreeAfter eridb := db2.NewEriDb(tx) - smt := smt.NewSMT(eridb) + smt := smt.NewSMT(eridb, false) if cfg.zk.IncrementTreeAlways { // increment only behaviour @@ -475,7 +475,7 @@ func unwindZkSMT(ctx context.Context, logPrefix string, from, to uint64, db kv.R defer log.Info(fmt.Sprintf("[%s] Unwind ended", logPrefix)) eridb := db2.NewEriDb(db) - dbSmt := smt.NewSMT(eridb) + dbSmt := smt.NewSMT(eridb, false) log.Info(fmt.Sprintf("[%s]", logPrefix), "last root", common.BigToHash(dbSmt.LastRoot())) diff --git a/zk/stages/stage_sequence_execute_utils.go b/zk/stages/stage_sequence_execute_utils.go index c7a6f2cb239..a93ae224d2b 100644 --- a/zk/stages/stage_sequence_execute_utils.go +++ b/zk/stages/stage_sequence_execute_utils.go @@ -180,7 +180,7 @@ func (sdb *stageDb) SetTx(tx kv.RwTx) { sdb.hermezDb = hermez_db.NewHermezDb(tx) sdb.eridb = db2.NewEriDb(tx) sdb.stateReader = state.NewPlainStateReader(tx) - sdb.smt = smtNs.NewSMT(sdb.eridb) + sdb.smt = smtNs.NewSMT(sdb.eridb, false) } type nextBatchL1Data struct { diff --git a/zk/tx/tx.go b/zk/tx/tx.go index 1b4025d25c4..9b0dfe072ba 100644 --- a/zk/tx/tx.go +++ b/zk/tx/tx.go @@ -504,6 +504,8 @@ func ComputeL2TxHash( return common.HexToHash(hashed), nil } +var re = regexp.MustCompile("^[0-9a-fA-F]*$") + func formatL2TxHashParam(param interface{}, paramLength int) (string, error) { var paramStr string @@ -560,11 +562,7 @@ func formatL2TxHashParam(param interface{}, paramLength int) (string, error) { paramStr = "0" + paramStr } - matched, err := regexp.MatchString("^[0-9a-fA-F]+$", paramStr) - if err != nil { - return "", err - } - if !matched { + if !re.MatchString(paramStr) { return "", fmt.Errorf("invalid hex string") } diff --git a/zk/tx/tx_test.go b/zk/tx/tx_test.go index 994e4bfb3db..c120f16abfc 100644 --- a/zk/tx/tx_test.go +++ b/zk/tx/tx_test.go @@ -318,6 +318,21 @@ func TestComputeL2TxHashScenarios(t *testing.T) { } +func BenchmarkComputeL2TxHashSt(b *testing.B) { + chainId := big.NewInt(2440) + nonce := uint64(87) + gasPrice := uint256.NewInt(493000000) + gasLimit := uint64(100000) + value := uint256.NewInt(100) + data := []byte{} + to := common.HexToAddress("0x5751D5b29dA14d5C334A9453cF04181f417aBe4c") + from := common.HexToAddress("0x5751D5b29dA14d5C334A9453cF04181f417aBe4c") + + for i := 0; i < b.N; i++ { + _, _ = ComputeL2TxHash(chainId, value, gasPrice, nonce, gasLimit, &to, &from, data) + } +} + type testCase struct { param interface{} paramLength int @@ -361,6 +376,12 @@ func TestFormatL2TxHashParam(t *testing.T) { } } +func BenchmarkFormatL2TxHashParam(b *testing.B) { + for i := 0; i < b.N; i++ { + _, _ = formatL2TxHashParam(uint256.NewInt(1000), 8) + } +} + func Test_EncodeToBatchL2DataAndBack(t *testing.T) { toAddress := common.HexToAddress("0x1") tx := &types.LegacyTx{ diff --git a/zk/witness/witness.go b/zk/witness/witness.go index 3febd76b384..450b6ed3833 100644 --- a/zk/witness/witness.go +++ b/zk/witness/witness.go @@ -6,6 +6,8 @@ import ( "errors" "fmt" + "math/big" + libcommon "github.com/gateway-fm/cdk-erigon-lib/common" "github.com/gateway-fm/cdk-erigon-lib/common/datadir" "github.com/gateway-fm/cdk-erigon-lib/kv" @@ -27,11 +29,10 @@ import ( "github.com/ledgerwatch/erigon/turbo/trie" dstypes "github.com/ledgerwatch/erigon/zk/datastream/types" "github.com/ledgerwatch/erigon/zk/hermez_db" + "github.com/ledgerwatch/erigon/zk/l1_data" zkStages "github.com/ledgerwatch/erigon/zk/stages" zkUtils "github.com/ledgerwatch/erigon/zk/utils" "github.com/ledgerwatch/log/v3" - "github.com/ledgerwatch/erigon/zk/l1_data" - "math/big" ) var ( @@ -327,7 +328,7 @@ func (g *Generator) generateWitness(tx kv.Tx, ctx context.Context, blocks []*eri } eridb := db2.NewEriDb(batch) - smtTrie := smt.NewSMT(eridb) + smtTrie := smt.NewSMT(eridb, false) witness, err := smt.BuildWitness(smtTrie, rl, ctx) if err != nil { From 09c160e5e99e63e677d464f166a69451d7fb5a34 Mon Sep 17 00:00:00 2001 From: Max Revitt Date: Thu, 18 Jul 2024 15:33:25 +0100 Subject: [PATCH 13/19] fix(execution): return early if no blocks (#823) --- eth/stagedsync/stage_execute_zkevm.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/eth/stagedsync/stage_execute_zkevm.go b/eth/stagedsync/stage_execute_zkevm.go index 6a69f702b0d..b9e7c8170e0 100644 --- a/eth/stagedsync/stage_execute_zkevm.go +++ b/eth/stagedsync/stage_execute_zkevm.go @@ -269,6 +269,11 @@ func getExecRange(cfg ExecuteBlockCfg, tx kv.RwTx, stageProgress, toBlock uint64 return to, total, nil } + // skip if no progress + if toBlock == 0 { + return 0, 0, nil + } + shouldShortCircuit, noProgressTo, err := utils.ShouldShortCircuitExecution(tx, logPrefix) if err != nil { return 0, 0, err From ff1259342064fa203a3902f744c998e52a98cb75 Mon Sep 17 00:00:00 2001 From: Max Revitt Date: Thu, 18 Jul 2024 17:07:04 +0100 Subject: [PATCH 14/19] Timing Logs: additional timing, log format improvement (#818) * tweak(perf): record timing of executor verify closes #815 closes #816 * fix(timers): log single string * tweak(timers): sequencer execute evm + smt timing closes #814 --- .../legacy_executor_verifier.go | 12 ++++++++++++ zk/stages/stage_sequence_execute.go | 16 +++++++++++++++- zk/utils/debug_timer.go | 17 ++++++++++------- 3 files changed, 37 insertions(+), 8 deletions(-) diff --git a/zk/legacy_executor_verifier/legacy_executor_verifier.go b/zk/legacy_executor_verifier/legacy_executor_verifier.go index 57c6b128291..f51e32f96b0 100644 --- a/zk/legacy_executor_verifier/legacy_executor_verifier.go +++ b/zk/legacy_executor_verifier/legacy_executor_verifier.go @@ -22,6 +22,7 @@ import ( "github.com/ledgerwatch/erigon/zk/syncer" "github.com/ledgerwatch/log/v3" "sync" + "github.com/ledgerwatch/erigon/zk/utils" ) var ErrNoExecutorAvailable = fmt.Errorf("no executor available") @@ -152,6 +153,9 @@ func (v *LegacyExecutorVerifier) VerifySync(tx kv.Tx, request *VerifierRequest, return ErrNoExecutorAvailable } + t := utils.StartTimer("legacy-executor-verifier", "verify-sync") + defer t.LogTimer() + e.AquireAccess() defer e.ReleaseAccess() @@ -176,6 +180,8 @@ func (v *LegacyExecutorVerifier) AddRequestUnsafe(request *VerifierRequest, sequ return verifierBundle, ErrNoExecutorAvailable } + t := utils.StartTimer("legacy-executor-verifier", "add-request-unsafe") + e.AquireAccess() defer e.ReleaseAccess() if v.cancelAllVerifications.Load() { @@ -272,6 +278,9 @@ func (v *LegacyExecutorVerifier) AddRequestUnsafe(request *VerifierRequest, sequ } } + // log timing w/o stream write + t.LogTimer() + if err = v.checkAndWriteToStream(tx, hermezDb, request.BatchNumber); err != nil { log.Error("error writing data to stream", "err", err) } @@ -295,6 +304,9 @@ func (v *LegacyExecutorVerifier) AddRequestUnsafe(request *VerifierRequest, sequ } func (v *LegacyExecutorVerifier) checkAndWriteToStream(tx kv.Tx, hdb *hermez_db.HermezDbReader, newBatch uint64) error { + t := utils.StartTimer("legacy-executor-verifier", "check-and-write-to-stream") + defer t.LogTimer() + v.responsesMtx.Lock() defer v.responsesMtx.Unlock() diff --git a/zk/stages/stage_sequence_execute.go b/zk/stages/stage_sequence_execute.go index 762587d5d98..4ebf439de32 100644 --- a/zk/stages/stage_sequence_execute.go +++ b/zk/stages/stage_sequence_execute.go @@ -288,6 +288,9 @@ func SpawnSequencingStage( break } + // timer: evm + smt + t := utils.StartTimer("stage_sequence_execute", "evm", "smt") + overflowOnNewBlock, err := batchCounters.StartNewBlock(l1InfoIndex != 0) if err != nil { return err @@ -471,6 +474,13 @@ func SpawnSequencingStage( return err } + t.LogTimer() + gasPerSecond := float64(0) + elapsedSeconds := t.Elapsed().Seconds() + if elapsedSeconds != 0 { + gasPerSecond = float64(block.GasUsed()) / elapsedSeconds + } + if limboRecovery { stateRoot := block.Root() cfg.txPool.UpdateLimboRootByTxHash(limboTxHash, &stateRoot) @@ -479,7 +489,11 @@ func SpawnSequencingStage( log.Debug(fmt.Sprintf("[%s] state root at block %d = %s", s.LogPrefix(), blockNumber, block.Root().Hex())) } - log.Info(fmt.Sprintf("[%s] Finish block %d with %d transactions...", logPrefix, blockNumber, len(addedTransactions))) + if gasPerSecond != 0 { + log.Info(fmt.Sprintf("[%s] Finish block %d with %d transactions... (%d gas/s)", logPrefix, blockNumber, len(addedTransactions), int(gasPerSecond))) + } else { + log.Info(fmt.Sprintf("[%s] Finish block %d with %d transactions...", logPrefix, blockNumber, len(addedTransactions))) + } if !hasExecutorForThisBatch { // save counters midbatch diff --git a/zk/utils/debug_timer.go b/zk/utils/debug_timer.go index 0b31e7369fc..0acb96ce55e 100644 --- a/zk/utils/debug_timer.go +++ b/zk/utils/debug_timer.go @@ -11,6 +11,7 @@ var timerEnabled bool type Timer struct { start time.Time + elapsed time.Duration taskNames []string } @@ -19,12 +20,10 @@ func EnableTimer(enable bool) { } func StartTimer(taskNames ...string) *Timer { - if !timerEnabled { - return nil - } return &Timer{ start: time.Now(), taskNames: taskNames, + elapsed: 0, } } @@ -33,12 +32,16 @@ func (t *Timer) LogTimer() { return } - elapsed := time.Since(t.start) - logArgs := []interface{}{"duration", elapsed, "task", t.taskNames[0]} + t.elapsed = time.Since(t.start) + logMessage := fmt.Sprintf("duration: %s, task: %s", t.elapsed, t.taskNames[0]) for i, task := range t.taskNames[1:] { - logArgs = append(logArgs, fmt.Sprintf("subtask%d", i+1), task) + logMessage += fmt.Sprintf(", subtask%d: %s", i+1, task) } - log.Info("[cdk-metric]", logArgs...) + log.Info("[cdk-metric] " + logMessage) +} + +func (t *Timer) Elapsed() time.Duration { + return t.elapsed } From 839ddbf0a313101f7a1f0cae500b2f285d953a1d Mon Sep 17 00:00:00 2001 From: Jerry Date: Thu, 18 Jul 2024 09:23:22 -0700 Subject: [PATCH 15/19] Fix L1 info tree due to duplicated logs (#787) * Fix L1 info tree due to duplicated logs In some cases, l1 info tree will receive l1 info logs which were processed before. It leads to the node creating a duplicated l1 info leaf, which results in invalid batch. This commit resolve this issue by checking l1 info leaf hash and skip the update if it already presents in the l1 info tree. * Delete debugging log --- zk/l1infotree/tree.go | 23 +++++++++++- zk/l1infotree/tree_test.go | 36 +++++++++++++++++-- zk/stages/stage_l1_info_tree.go | 30 ++++++++++++++-- zk/stages/stage_l1_sequencer_sync.go | 52 ++++++++++------------------ 4 files changed, 102 insertions(+), 39 deletions(-) diff --git a/zk/l1infotree/tree.go b/zk/l1infotree/tree.go index b29b12a693d..77b05d400da 100644 --- a/zk/l1infotree/tree.go +++ b/zk/l1infotree/tree.go @@ -3,8 +3,8 @@ package l1infotree import ( "fmt" - "github.com/ledgerwatch/log/v3" "github.com/gateway-fm/cdk-erigon-lib/common" + "github.com/ledgerwatch/log/v3" ) // L1InfoTree provides methods to compute L1InfoTree @@ -14,6 +14,7 @@ type L1InfoTree struct { count uint32 siblings [][32]byte currentRoot common.Hash + allLeaves map[[32]byte]struct{} } // NewL1InfoTree creates new L1InfoTree. @@ -29,6 +30,12 @@ func NewL1InfoTree(height uint8, initialLeaves [][32]byte) (*L1InfoTree, error) log.Error("error initializing siblings. Error: ", err) return nil, err } + + mt.allLeaves = make(map[[32]byte]struct{}) + for _, leaf := range initialLeaves { + mt.allLeaves[leaf] = struct{}{} + } + log.Debug("Initial count: ", mt.count) log.Debug("Initial root: ", mt.currentRoot) return mt, nil @@ -48,6 +55,12 @@ func (mt *L1InfoTree) ResetL1InfoTree(initialLeaves [][32]byte) (*L1InfoTree, er log.Error("error initializing siblings. Error: ", err) return nil, err } + + newMT.allLeaves = make(map[[32]byte]struct{}) + for _, leaf := range initialLeaves { + newMT.allLeaves[leaf] = struct{}{} + } + log.Debug("Reset initial count: ", newMT.count) log.Debug("Reset initial root: ", newMT.currentRoot) return newMT, nil @@ -163,11 +176,19 @@ func (mt *L1InfoTree) AddLeaf(index uint32, leaf [32]byte) (common.Hash, error) // the sibling of 0 bit should be the zero hash, since we are in the last node of the tree } } + + mt.allLeaves[leaf] = struct{}{} + mt.currentRoot = cur mt.count++ return cur, nil } +func (mt *L1InfoTree) LeafExists(leaf [32]byte) bool { + _, ok := mt.allLeaves[leaf] + return ok +} + // initSiblings returns the siblings of the node at the given index. // it is used to initialize the siblings array in the beginning. func (mt *L1InfoTree) initSiblings(initialLeaves [][32]byte) ([][32]byte, common.Hash, error) { diff --git a/zk/l1infotree/tree_test.go b/zk/l1infotree/tree_test.go index 47b795dccdb..111b6254702 100644 --- a/zk/l1infotree/tree_test.go +++ b/zk/l1infotree/tree_test.go @@ -6,11 +6,11 @@ import ( "os" "testing" - "github.com/stretchr/testify/require" + "github.com/gateway-fm/cdk-erigon-lib/common" "github.com/ledgerwatch/erigon/zk/l1infotree" - "github.com/ledgerwatch/erigon/zkevm/log" l1infotree2 "github.com/ledgerwatch/erigon/zk/tests/vectors/l1infotree" - "github.com/gateway-fm/cdk-erigon-lib/common" + "github.com/ledgerwatch/erigon/zkevm/log" + "github.com/stretchr/testify/require" ) func TestComputeTreeRoot(t *testing.T) { @@ -128,3 +128,33 @@ func TestAddLeaf2(t *testing.T) { require.Equal(t, testVector.NewRoot, newRoot) } } + +func TestLeafExists(t *testing.T) { + data, err := os.ReadFile("../tests/vectors/l1infotree/proof-vectors.json") + require.NoError(t, err) + var mtTestVectors []l1infotree2.L1InfoTreeProof + err = json.Unmarshal(data, &mtTestVectors) + require.NoError(t, err) + testVector := mtTestVectors[3] + var leaves [][32]byte + mt, err := l1infotree.NewL1InfoTree(uint8(32), leaves) + require.NoError(t, err) + for _, leaf := range testVector.Leaves { + _, count, _ := mt.GetCurrentRootCountAndSiblings() + _, err := mt.AddLeaf(count, leaf) + require.NoError(t, err) + } + log.Debugf("%d leaves added successfully", len(testVector.Leaves)) + root, _, _ := mt.GetCurrentRootCountAndSiblings() + require.Equal(t, testVector.Root, root) + log.Debug("Final root: ", root) + + for _, leaf := range testVector.Leaves { + exists := mt.LeafExists(leaf) + require.True(t, exists) + } + + nonExistentLeaf := common.HexToHash("0x1234") + exists := mt.LeafExists(nonExistentLeaf) + require.False(t, exists) +} diff --git a/zk/stages/stage_l1_info_tree.go b/zk/stages/stage_l1_info_tree.go index e9d39c50a90..c4f291e6085 100644 --- a/zk/stages/stage_l1_info_tree.go +++ b/zk/stages/stage_l1_info_tree.go @@ -140,13 +140,39 @@ LOOP: treeInitialised = true } - latestUpdate, err = HandleL1InfoTreeUpdate(cfg.syncer, hermezDb, l, latestUpdate, found, header) + if header == nil { + header, err = cfg.syncer.GetHeader(l.BlockNumber) + if err != nil { + return err + } + } + + tmpUpdate, err := CreateL1InfoTreeUpdate(l, header) + if err != nil { return err } + + leafHash := l1infotree.HashLeafData(tmpUpdate.GER, tmpUpdate.ParentHash, tmpUpdate.Timestamp) + + if tree.LeafExists(leafHash) { + log.Warn("Skipping log as L1 Info Tree leaf already exists", "hash", leafHash) + continue + } + + if found { + tmpUpdate.Index = latestUpdate.Index + 1 + } else { + tmpUpdate.Index = 0 + } + found = true + latestUpdate = tmpUpdate - leafHash := l1infotree.HashLeafData(latestUpdate.GER, latestUpdate.ParentHash, latestUpdate.Timestamp) + err = HandleL1InfoTreeUpdate(hermezDb, latestUpdate) + if err != nil { + return err + } leafFoundInDb, err := hermezDb.IsL1InfoTreeLeafSaves(leafHash) if err != nil { diff --git a/zk/stages/stage_l1_sequencer_sync.go b/zk/stages/stage_l1_sequencer_sync.go index 09ecd04e2cd..a1bc3ba8e15 100644 --- a/zk/stages/stage_l1_sequencer_sync.go +++ b/zk/stages/stage_l1_sequencer_sync.go @@ -168,19 +168,14 @@ Loop: return nil } -func HandleL1InfoTreeUpdate( - syncer IL1Syncer, - hermezDb *hermez_db.HermezDb, - l ethTypes.Log, - latestUpdate *types.L1InfoTreeUpdate, - found bool, - header *ethTypes.Header, -) (*types.L1InfoTreeUpdate, error) { +func CreateL1InfoTreeUpdate(l ethTypes.Log, header *ethTypes.Header) (*types.L1InfoTreeUpdate, error) { if len(l.Topics) != 3 { - log.Warn("Received log for info tree that did not have 3 topics") - return nil, nil + return nil, fmt.Errorf("received log for info tree that did not have 3 topics") + } + + if l.BlockNumber != header.Number.Uint64() { + return nil, fmt.Errorf("received log for info tree that did not match the block number") } - var err error mainnetExitRoot := l.Topics[1] rollupExitRoot := l.Topics[2] @@ -190,35 +185,26 @@ func HandleL1InfoTreeUpdate( GER: common.BytesToHash(ger), MainnetExitRoot: mainnetExitRoot, RollupExitRoot: rollupExitRoot, + BlockNumber: l.BlockNumber, + Timestamp: header.Time, + ParentHash: header.ParentHash, } - if !found { - // this is a special case, so we need to start at index 0 - update.Index = 0 - } else { - // increment the index from the previous entry - update.Index = latestUpdate.Index + 1 - } - - // now we need the block timestamp and the parent hash information for the block tied - // to this event - if header == nil { - header, err = syncer.GetHeader(l.BlockNumber) - if err != nil { - return nil, err - } - } - update.ParentHash = header.ParentHash - update.Timestamp = header.Time - update.BlockNumber = l.BlockNumber + return update, nil +} +func HandleL1InfoTreeUpdate( + hermezDb *hermez_db.HermezDb, + update *types.L1InfoTreeUpdate, +) error { + var err error if err = hermezDb.WriteL1InfoTreeUpdate(update); err != nil { - return nil, err + return err } if err = hermezDb.WriteL1InfoTreeUpdateToGer(update); err != nil { - return nil, err + return err } - return update, nil + return nil } const ( From 9f18eb1b67c8f914f65b0d15e4f8d83b071a8892 Mon Sep 17 00:00:00 2001 From: Valentin Staykov <79150443+V-Staykov@users.noreply.github.com> Date: Fri, 19 Jul 2024 11:17:41 +0300 Subject: [PATCH 16/19] fix no progress check (#826) --- eth/stagedsync/stage_execute_zkevm.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/eth/stagedsync/stage_execute_zkevm.go b/eth/stagedsync/stage_execute_zkevm.go index b9e7c8170e0..4c9f4376598 100644 --- a/eth/stagedsync/stage_execute_zkevm.go +++ b/eth/stagedsync/stage_execute_zkevm.go @@ -269,11 +269,6 @@ func getExecRange(cfg ExecuteBlockCfg, tx kv.RwTx, stageProgress, toBlock uint64 return to, total, nil } - // skip if no progress - if toBlock == 0 { - return 0, 0, nil - } - shouldShortCircuit, noProgressTo, err := utils.ShouldShortCircuitExecution(tx, logPrefix) if err != nil { return 0, 0, err @@ -283,6 +278,11 @@ func getExecRange(cfg ExecuteBlockCfg, tx kv.RwTx, stageProgress, toBlock uint64 return 0, 0, err } + // skip if no progress + if prevStageProgress == 0 && toBlock == 0 { + return 0, 0, nil + } + to := prevStageProgress if toBlock > 0 { to = cmp.Min(prevStageProgress, toBlock) From 4438e122fd7a5ea55ccce8c0be1c9cda211d7ad8 Mon Sep 17 00:00:00 2001 From: Kamen Stoykov <24619432+kstoykov@users.noreply.github.com> Date: Fri, 19 Jul 2024 14:23:51 +0300 Subject: [PATCH 17/19] Smt flags and cursors (#825) * smt-flags-and-cursors * remove addition reader.Close * set correct block number in the log --- cmd/hack/hack.go | 3 +- cmd/state/commands/check_change_sets.go | 6 +- smt/pkg/smt/entity_storage.go | 7 ++- zk/stages/stage_interhashes.go | 80 +++++++++++-------------- zk/stages/stage_sequence_execute.go | 2 +- zk/witness/witness.go | 6 +- 6 files changed, 51 insertions(+), 53 deletions(-) diff --git a/cmd/hack/hack.go b/cmd/hack/hack.go index 828841c334b..fa4e4ad5176 100644 --- a/cmd/hack/hack.go +++ b/cmd/hack/hack.go @@ -19,7 +19,6 @@ import ( "time" "github.com/RoaringBitmap/roaring/roaring64" - "github.com/holiman/uint256" libcommon "github.com/gateway-fm/cdk-erigon-lib/common" "github.com/gateway-fm/cdk-erigon-lib/common/hexutility" "github.com/gateway-fm/cdk-erigon-lib/common/length" @@ -30,6 +29,7 @@ import ( "github.com/gateway-fm/cdk-erigon-lib/recsplit" "github.com/gateway-fm/cdk-erigon-lib/recsplit/eliasfano32" librlp "github.com/gateway-fm/cdk-erigon-lib/rlp" + "github.com/holiman/uint256" "github.com/ledgerwatch/log/v3" "golang.org/x/exp/slices" @@ -216,6 +216,7 @@ func readAccountAtVersion(chaindata string, account string, block uint64) error defer tx.Rollback() ps := state.NewPlainState(tx, block, nil) + defer ps.Close() addr := libcommon.HexToAddress(account) acc, err := ps.ReadAccountData(addr) diff --git a/cmd/state/commands/check_change_sets.go b/cmd/state/commands/check_change_sets.go index 8109e2f7d2e..875f496c3d4 100644 --- a/cmd/state/commands/check_change_sets.go +++ b/cmd/state/commands/check_change_sets.go @@ -119,8 +119,10 @@ func CheckChangeSets(genesis *types.Genesis, logger log.Logger, blockNum uint64, engine := initConsensusEngine(chainConfig, allSnapshots) - for !interrupt { + reader := state.NewPlainState(historyTx, blockNum, systemcontracts.SystemContractCodeLookup[chainConfig.ChainName]) + defer reader.Close() + for !interrupt { if blockNum > execAt { log.Warn(fmt.Sprintf("Force stop: because trying to check blockNumber=%d higher than Exec stage=%d", blockNum, execAt)) break @@ -142,7 +144,7 @@ func CheckChangeSets(genesis *types.Genesis, logger log.Logger, blockNum uint64, if b == nil { break } - reader := state.NewPlainState(historyTx, blockNum, systemcontracts.SystemContractCodeLookup[chainConfig.ChainName]) + reader.SetBlockNr(blockNum) //reader.SetTrace(blockNum == uint64(block)) intraBlockState := state.New(reader) csw := state.NewChangeSetWriterPlain(nil /* db */, blockNum) diff --git a/smt/pkg/smt/entity_storage.go b/smt/pkg/smt/entity_storage.go index f3b5215885e..ca2ce595e5c 100644 --- a/smt/pkg/smt/entity_storage.go +++ b/smt/pkg/smt/entity_storage.go @@ -206,7 +206,12 @@ func (s *SMT) SetContractStorage(ethAddr string, storage map[string]string, prog func (s *SMT) SetStorage(ctx context.Context, logPrefix string, accChanges map[libcommon.Address]*accounts.Account, codeChanges map[libcommon.Address]string, storageChanges map[libcommon.Address]map[string]string) ([]*utils.NodeKey, []*utils.NodeValue8, error) { var isDelete bool - initialCapacity := len(accChanges) + len(codeChanges) + len(storageChanges) + storageChangesInitialCapacity := 0 + for _, storage := range storageChanges { + storageChangesInitialCapacity += len(storage) + } + + initialCapacity := len(accChanges)*2 + len(codeChanges)*2 + storageChangesInitialCapacity keysBatchStorage := make([]*utils.NodeKey, 0, initialCapacity) valuesBatchStorage := make([]*utils.NodeValue8, 0, initialCapacity) diff --git a/zk/stages/stage_interhashes.go b/zk/stages/stage_interhashes.go index c63e06e537d..28de6015226 100644 --- a/zk/stages/stage_interhashes.go +++ b/zk/stages/stage_interhashes.go @@ -126,76 +126,60 @@ func SpawnZkIntermediateHashesStage(s *stagedsync.StageState, u stagedsync.Unwin } shouldRegenerate := to > s.BlockNumber && to-s.BlockNumber > cfg.zk.RebuildTreeAfter + shouldIncrementBecauseOfAFlag := cfg.zk.IncrementTreeAlways + shouldIncrementBecauseOfExecutionConditions := s.BlockNumber > 0 && !shouldRegenerate + shouldIncrement := shouldIncrementBecauseOfAFlag || shouldIncrementBecauseOfExecutionConditions + eridb := db2.NewEriDb(tx) smt := smt.NewSMT(eridb, false) - if cfg.zk.IncrementTreeAlways { - // increment only behaviour + if cfg.zk.SmtRegenerateInMemory { + log.Info(fmt.Sprintf("[%s] SMT using mapmutation", logPrefix)) eridb.OpenBatch(quit) - log.Debug(fmt.Sprintf("[%s] IncrementTreeAlways true - incrementing tree", logPrefix), "previousRootHeight", s.BlockNumber, "calculatingRootHeight", to) + } else { + log.Info(fmt.Sprintf("[%s] SMT not using mapmutation", logPrefix)) + } + + if shouldIncrement { + if shouldIncrementBecauseOfAFlag { + log.Debug(fmt.Sprintf("[%s] IncrementTreeAlways true - incrementing tree", logPrefix), "previousRootHeight", s.BlockNumber, "calculatingRootHeight", to) + } if root, err = zkIncrementIntermediateHashes(ctx, logPrefix, s, tx, eridb, smt, s.BlockNumber, to); err != nil { return trie.EmptyRoot, err } } else { - // default behaviour - if s.BlockNumber == 0 || shouldRegenerate { - if cfg.zk.SmtRegenerateInMemory { - log.Info(fmt.Sprintf("[%s] SMT using mapmutation", logPrefix)) - eridb.OpenBatch(quit) - } else { - log.Info(fmt.Sprintf("[%s] SMT not using mapmutation", logPrefix)) - } - if root, err = regenerateIntermediateHashes(logPrefix, tx, eridb, smt, to); err != nil { - return trie.EmptyRoot, err - } - } else { - eridb.OpenBatch(quit) - if root, err = zkIncrementIntermediateHashes(ctx, logPrefix, s, tx, eridb, smt, s.BlockNumber, to); err != nil { - return trie.EmptyRoot, err - } + if root, err = regenerateIntermediateHashes(logPrefix, tx, eridb, smt, to); err != nil { + return trie.EmptyRoot, err } } log.Info(fmt.Sprintf("[%s] Trie root", logPrefix), "hash", root.Hex()) if cfg.checkRoot { - var expectedRootHash common.Hash - var headerHash common.Hash var syncHeadHeader *types.Header - syncHeadHeader, err = cfg.blockReader.HeaderByNumber(ctx, tx, to) - if err != nil { + if syncHeadHeader, err = cfg.blockReader.HeaderByNumber(ctx, tx, to); err != nil { return trie.EmptyRoot, err } if syncHeadHeader == nil { return trie.EmptyRoot, fmt.Errorf("no header found with number %d", to) } - expectedRootHash = syncHeadHeader.Root - headerHash = syncHeadHeader.Hash() + expectedRootHash := syncHeadHeader.Root + headerHash := syncHeadHeader.Hash() if root != expectedRootHash { if cfg.zk.SmtRegenerateInMemory { eridb.RollbackBatch() } panic(fmt.Sprintf("[%s] Wrong trie root of block %d: %x, expected (from header): %x. Block hash: %x", logPrefix, to, root, expectedRootHash, headerHash)) - - if cfg.badBlockHalt { - return trie.EmptyRoot, fmt.Errorf("wrong trie root") - } - if cfg.hd != nil { - cfg.hd.ReportBadHeaderPoS(headerHash, syncHeadHeader.ParentHash) - } - // if to > s.BlockNumber { - //unwindTo := (to + s.BlockNumber) / 2 // Binary search for the correct block, biased to the lower numbers - //log.Warn("Unwinding due to incorrect root hash", "to", unwindTo) - //u.UnwindTo(unwindTo, headerHash) - // } - } else { - log.Info(fmt.Sprintf("[%s] State root matches", logPrefix)) } + + log.Info(fmt.Sprintf("[%s] State root matches", logPrefix)) } - if err := eridb.CommitBatch(); err != nil { - return trie.EmptyRoot, err + if cfg.zk.SmtRegenerateInMemory { + if err := eridb.CommitBatch(); err != nil { + return trie.EmptyRoot, err + } } if err = s.Update(tx, to); err != nil { @@ -389,11 +373,13 @@ func zkIncrementIntermediateHashes(ctx context.Context, logPrefix string, s *sta // NB: changeset tables are zero indexed // changeset tables contain historical value at N-1, so we look up values from plainstate + // i+1 to get state at the beginning of the next batch + psr := state2.NewPlainState(db, from+1, systemcontracts.SystemContractCodeLookup["Hermez"]) + defer psr.Close() + for i := from; i <= to; i++ { dupSortKey := dbutils.EncodeBlockNumber(i) - - // i+1 to get state at the beginning of the next batch - psr := state2.NewPlainState(db, i+1, systemcontracts.SystemContractCodeLookup["Hermez"]) + psr.SetBlockNr(i + 1) // collect changes to accounts and code for _, v, err := ac.SeekExact(dupSortKey); err == nil && v != nil; _, v, err = ac.NextDup() { @@ -526,6 +512,9 @@ func unwindZkSMT(ctx context.Context, logPrefix string, from, to uint64, db kv.R accChanges[addr] = deletedAcc } + psr := state2.NewPlainState(db, from, systemcontracts.SystemContractCodeLookup["Hermez"]) + defer psr.Close() + for i := from; i >= to+1; i-- { select { case <-ctx.Done(): @@ -533,7 +522,7 @@ func unwindZkSMT(ctx context.Context, logPrefix string, from, to uint64, db kv.R default: } - psr := state2.NewPlainState(db, i, systemcontracts.SystemContractCodeLookup["Hermez"]) + psr.SetBlockNr(i) dupSortKey := dbutils.EncodeBlockNumber(i) @@ -617,7 +606,6 @@ func unwindZkSMT(ctx context.Context, logPrefix string, from, to uint64, db kv.R } progressChan <- 1 - psr.Close() } stopPrinter() diff --git a/zk/stages/stage_sequence_execute.go b/zk/stages/stage_sequence_execute.go index 4ebf439de32..6919f3d9784 100644 --- a/zk/stages/stage_sequence_execute.go +++ b/zk/stages/stage_sequence_execute.go @@ -268,7 +268,7 @@ func SpawnSequencingStage( return err } - log.Info(fmt.Sprintf("[%s] Starting block %d (forkid %v)...", logPrefix, blockNumber+1, forkId)) + log.Info(fmt.Sprintf("[%s] Starting block %d (forkid %v)...", logPrefix, blockNumber, forkId)) lastStartedBn = blockNumber diff --git a/zk/witness/witness.go b/zk/witness/witness.go index 450b6ed3833..6afc14f9a2a 100644 --- a/zk/witness/witness.go +++ b/zk/witness/witness.go @@ -242,9 +242,12 @@ func (g *Generator) generateWitness(tx kv.Tx, ctx context.Context, blocks []*eri prevStateRoot := prevHeader.Root + reader := state.NewPlainState(tx, blocks[0].NumberU64(), systemcontracts.SystemContractCodeLookup[g.chainCfg.ChainName]) + defer reader.Close() + for _, block := range blocks { blockNum := block.NumberU64() - reader := state.NewPlainState(tx, blockNum, systemcontracts.SystemContractCodeLookup[g.chainCfg.ChainName]) + reader.SetBlockNr(blockNum) tds.SetStateReader(reader) @@ -313,7 +316,6 @@ func (g *Generator) generateWitness(tx kv.Tx, ctx context.Context, blocks []*eri } prevStateRoot = block.Root() - reader.Close() // close the cursors created by the plainstate } var rl trie.RetainDecider From a2c8d8bdb5989e395f8891e73b0a66081886d1ec Mon Sep 17 00:00:00 2001 From: Kamen Stoykov <24619432+kstoykov@users.noreply.github.com> Date: Fri, 19 Jul 2024 17:23:20 +0300 Subject: [PATCH 18/19] Merge both fixes for l1infotree duplicates (#829) * merge both fixes for l1infotree-duplicates * 1-time init of info tree --- zk/hermez_db/db.go | 23 ----------- zk/stages/stage_l1_info_tree.go | 73 +++++++++++---------------------- 2 files changed, 24 insertions(+), 72 deletions(-) diff --git a/zk/hermez_db/db.go b/zk/hermez_db/db.go index efda404ac85..b087dfd85a4 100644 --- a/zk/hermez_db/db.go +++ b/zk/hermez_db/db.go @@ -1,7 +1,6 @@ package hermez_db import ( - "bytes" "errors" "fmt" "math" @@ -1448,28 +1447,6 @@ func (db *HermezDb) WriteL1InfoTreeLeaf(l1Index uint64, leaf common.Hash) error return db.tx.Put(L1_INFO_LEAVES, Uint64ToBytes(l1Index), leaf.Bytes()) } -func (db *HermezDbReader) IsL1InfoTreeLeafSaves(leaf common.Hash) (bool, error) { - c, err := db.tx.Cursor(L1_INFO_LEAVES) - if err != nil { - return false, err - } - defer c.Close() - - leafBytes := leaf.Bytes() - - for k, v, err := c.First(); k != nil; k, v, err = c.Next() { - if err != nil { - return false, err - } - - if bytes.Equal(v, leafBytes) { - return true, nil - } - } - - return false, nil -} - func (db *HermezDbReader) GetAllL1InfoTreeLeaves() ([]common.Hash, error) { c, err := db.tx.Cursor(L1_INFO_LEAVES) if err != nil { diff --git a/zk/stages/stage_l1_info_tree.go b/zk/stages/stage_l1_info_tree.go index c4f291e6085..50e470bee68 100644 --- a/zk/stages/stage_l1_info_tree.go +++ b/zk/stages/stage_l1_info_tree.go @@ -63,7 +63,7 @@ func SpawnL1InfoTreeStage( progress = cfg.zkCfg.L1FirstBlock - 1 } - latestUpdate, found, err := hermezDb.GetLatestL1InfoTreeUpdate() + latestUpdate, _, err := hermezDb.GetLatestL1InfoTreeUpdate() if err != nil { return err } @@ -112,8 +112,10 @@ LOOP: defer ticker.Stop() processed := 0 - var tree *l1infotree.L1InfoTree - treeInitialised := false + tree, err := initialiseL1InfoTree(hermezDb) + if err != nil { + return err + } // process the logs in chunks for _, chunk := range chunks { @@ -129,17 +131,9 @@ LOOP: } for _, l := range chunk { - header := headersMap[l.BlockNumber] switch l.Topics[0] { case contracts.UpdateL1InfoTreeTopic: - if !treeInitialised { - tree, err = initialiseL1InfoTree(hermezDb) - if err != nil { - return err - } - treeInitialised = true - } - + header := headersMap[l.BlockNumber] if header == nil { header, err = cfg.syncer.GetHeader(l.BlockNumber) if err != nil { @@ -148,61 +142,42 @@ LOOP: } tmpUpdate, err := CreateL1InfoTreeUpdate(l, header) - if err != nil { return err } leafHash := l1infotree.HashLeafData(tmpUpdate.GER, tmpUpdate.ParentHash, tmpUpdate.Timestamp) - if tree.LeafExists(leafHash) { log.Warn("Skipping log as L1 Info Tree leaf already exists", "hash", leafHash) continue } - if found { + if latestUpdate != nil { tmpUpdate.Index = latestUpdate.Index + 1 - } else { - tmpUpdate.Index = 0 - } - - found = true + } // if latestUpdate is nil then Index = 0 which is the default value so no need to set it latestUpdate = tmpUpdate - err = HandleL1InfoTreeUpdate(hermezDb, latestUpdate) + newRoot, err := tree.AddLeaf(uint32(latestUpdate.Index), leafHash) if err != nil { return err } - - leafFoundInDb, err := hermezDb.IsL1InfoTreeLeafSaves(leafHash) - if err != nil { + log.Debug("New L1 Index", + "index", latestUpdate.Index, + "root", newRoot.String(), + "mainnet", latestUpdate.MainnetExitRoot.String(), + "rollup", latestUpdate.RollupExitRoot.String(), + "ger", latestUpdate.GER.String(), + "parent", latestUpdate.ParentHash.String(), + ) + + if err = HandleL1InfoTreeUpdate(hermezDb, latestUpdate); err != nil { return err } - - if leafFoundInDb { - log.Warn("Leaf already saved in db", "index", latestUpdate.Index, "hash", leafHash) - } else { - if err = hermezDb.WriteL1InfoTreeLeaf(latestUpdate.Index, leafHash); err != nil { - return err - } - - newRoot, err := tree.AddLeaf(uint32(latestUpdate.Index), leafHash) - if err != nil { - return err - } - log.Debug("New L1 Index", - "index", latestUpdate.Index, - "root", newRoot.String(), - "mainnet", latestUpdate.MainnetExitRoot.String(), - "rollup", latestUpdate.RollupExitRoot.String(), - "ger", latestUpdate.GER.String(), - "parent", latestUpdate.ParentHash.String(), - ) - - err = hermezDb.WriteL1InfoTreeRoot(common.BytesToHash(newRoot[:]), latestUpdate.Index) - if err != nil { - return err - } + if err = hermezDb.WriteL1InfoTreeLeaf(latestUpdate.Index, leafHash); err != nil { + return err + } + if err = hermezDb.WriteL1InfoTreeRoot(common.BytesToHash(newRoot[:]), latestUpdate.Index); err != nil { + return err } processed++ From 859d3735009873645444ba5b1e98257eed1cbe36 Mon Sep 17 00:00:00 2001 From: Kamen Stoykov <24619432+kstoykov@users.noreply.github.com> Date: Fri, 19 Jul 2024 17:40:24 +0300 Subject: [PATCH 19/19] remove-old-db-functions (#830) --- zk/hermez_db/db.go | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/zk/hermez_db/db.go b/zk/hermez_db/db.go index b087dfd85a4..65f768d85c8 100644 --- a/zk/hermez_db/db.go +++ b/zk/hermez_db/db.go @@ -37,7 +37,6 @@ const INTERMEDIATE_TX_STATEROOTS = "hermez_intermediate_tx_stateRoots" // l2bloc const BATCH_WITNESSES = "hermez_batch_witnesses" // batch number -> witness const BATCH_COUNTERS = "hermez_batch_counters" // batch number -> counters const L1_BATCH_DATA = "l1_batch_data" // batch number -> l1 batch data from transaction call data -const L1_INFO_TREE_HIGHEST_BLOCK = "l1_info_tree_highest_block" // highest l1 block number found with L1 info tree updates const REUSED_L1_INFO_TREE_INDEX = "reused_l1_info_tree_index" // block number => const 1 const LATEST_USED_GER = "latest_used_ger" // batch number -> GER latest used GER const BATCH_BLOCKS = "batch_blocks" // batch number -> block numbers (concatenated together) @@ -96,7 +95,6 @@ func CreateHermezBuckets(tx kv.RwTx) error { BATCH_WITNESSES, BATCH_COUNTERS, L1_BATCH_DATA, - L1_INFO_TREE_HIGHEST_BLOCK, REUSED_L1_INFO_TREE_INDEX, LATEST_USED_GER, BATCH_BLOCKS, @@ -1325,18 +1323,6 @@ func (db *HermezDbReader) GetLastL1BatchData() (uint64, error) { return BytesToUint64(k), nil } -func (db *HermezDb) WriteL1InfoTreeHighestBlock(blockNumber uint64) error { - return db.tx.Put(L1_INFO_TREE_HIGHEST_BLOCK, []byte{}, Uint64ToBytes(blockNumber)) -} - -func (db *HermezDbReader) GetL1InfoTreeHighestBlock() (uint64, error) { - data, err := db.tx.GetOne(L1_INFO_TREE_HIGHEST_BLOCK, []byte{}) - if err != nil { - return 0, err - } - return BytesToUint64(data), nil -} - func (db *HermezDb) WriteLatestUsedGer(batchNo uint64, ger common.Hash) error { batchBytes := Uint64ToBytes(batchNo) return db.tx.Put(LATEST_USED_GER, batchBytes, ger.Bytes())