Merge remote-tracking branch 'origin/zkevm' into zkevm-2.60

.github/workflows/ci_zkevm.yml

@@ -23,9 +23,9 @@ jobs:
   tests:
     strategy:
       matrix:
-        os: [ ubuntu-22.04, macos-14 ] # list of os: https://github.com/actions/virtual-environments
+        os: [ ubuntu-22.04, macos-14-xlarge ] # list of os: https://github.com/actions/virtual-environments
     runs-on: ${{ matrix.os }}
-    timeout-minutes: ${{ matrix.os == 'macos-14' && 40 || 30 }}
+    timeout-minutes: ${{ matrix.os == 'macos-14-xlarge' && 40 || 30 }}
 
     steps:
       - uses: actions/checkout@v3

Makefile

@@ -167,7 +167,7 @@ test-erigon-ext:
 
 ## test:                              run unit tests with a 100s timeout
 test:
-	$(GOTEST) --timeout 200s
+	$(GOTEST) --timeout 10m
 
 test3:
 	$(GOTEST) --timeout 200s -tags $(BUILD_TAGS),erigon3

cmd/integration/commands/flags.go

@@ -35,6 +35,7 @@ var (
 	unwindTypes                             []string
 	chain                                   string // Which chain to use (mainnet, goerli, sepolia, etc.)
 	outputCsvFile                           string
+	config                                  string
 
 	commitmentMode string
 	commitmentTrie string
@@ -53,7 +54,7 @@ func must(err error) {
 }
 
 func withConfig(cmd *cobra.Command) {
-	cmd.Flags().String("config", "", "yaml/toml config file location")
+	cmd.Flags().StringVar(&config, "config", "", "yaml/toml config file location")
 }
 
 func withMining(cmd *cobra.Command) {

cmd/integration/commands/stage_stages_zkevm.go

@@ -9,11 +9,8 @@ import (
 	common2 "github.com/ledgerwatch/erigon-lib/common"
 	"github.com/ledgerwatch/erigon-lib/kv"
 	"github.com/ledgerwatch/erigon-lib/wrap"
-	"github.com/ledgerwatch/erigon/core"
-	"github.com/ledgerwatch/erigon/eth/ethconfig"
 	"github.com/ledgerwatch/erigon/eth/stagedsync/stages"
 	smtdb "github.com/ledgerwatch/erigon/smt/pkg/db"
-	erigoncli "github.com/ledgerwatch/erigon/turbo/cli"
 	"github.com/ledgerwatch/erigon/zk/hermez_db"
 	"github.com/ledgerwatch/log/v3"
 	"github.com/spf13/cobra"
@@ -29,9 +26,6 @@ state_stages_zkevm --datadir=/datadirs/hermez-mainnet --unwind-batch-no=2 --chai
 	Example: "go run ./cmd/integration state_stages_zkevm --config=... --verbosity=3 --unwind-batch-no=100",
 	Run: func(cmd *cobra.Command, args []string) {
 		ctx, _ := common2.RootContext()
-		ethConfig := &ethconfig.Defaults
-		ethConfig.Genesis = core.GenesisBlockByChainName(chain)
-		erigoncli.ApplyFlagsForEthConfigCobra(cmd.Flags(), ethConfig)
 		logger := log.New()
 		db, err := openDB(dbCfg(kv.ChainDB, chaindata), true, logger)
 		if err != nil {

cmd/integration/commands/stages_zkevm.go

@@ -2,13 +2,20 @@ package commands
 
 import (
 	"context"
+	"encoding/json"
+	"math/big"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
 
 	"github.com/c2h5oh/datasize"
 	chain3 "github.com/ledgerwatch/erigon-lib/chain"
 	"github.com/ledgerwatch/erigon-lib/common/datadir"
 	"github.com/ledgerwatch/erigon-lib/kv"
 	"github.com/ledgerwatch/erigon-lib/kv/kvcfg"
 	"github.com/ledgerwatch/erigon/cmd/hack/tool/fromdb"
+	"github.com/ledgerwatch/erigon/cmd/utils"
 	"github.com/ledgerwatch/erigon/consensus"
 	"github.com/ledgerwatch/erigon/core"
 	"github.com/ledgerwatch/erigon/core/types"
@@ -17,6 +24,7 @@ import (
 	"github.com/ledgerwatch/erigon/eth/stagedsync"
 	"github.com/ledgerwatch/erigon/p2p/sentry"
 	"github.com/ledgerwatch/erigon/p2p/sentry/sentry_multi_client"
+	"github.com/ledgerwatch/erigon/params"
 	"github.com/ledgerwatch/erigon/turbo/shards"
 	stages2 "github.com/ledgerwatch/erigon/turbo/stages"
 	"github.com/ledgerwatch/erigon/zk/sequencer"
@@ -29,7 +37,36 @@ func newSyncZk(ctx context.Context, db kv.RwDB) (consensus.Engine, *vm.Config, *
 
 	vmConfig := &vm.Config{}
 
-	genesis := core.GenesisBlockByChainName(chain)
+	var genesis *types.Genesis
+
+	if strings.HasPrefix(chain, "dynamic") {
+		if config == "" {
+			panic("Config file is required for dynamic chain")
+		}
+
+		params.DynamicChainConfigPath = filepath.Dir(config)
+		genesis = core.GenesisBlockByChainName(chain)
+		filename := path.Join(params.DynamicChainConfigPath, chain+"-conf.json")
+
+		dConf := utils.DynamicConfig{}
+
+		if _, err := os.Stat(filename); err == nil {
+			dConfBytes, err := os.ReadFile(filename)
+			if err != nil {
+				panic(err)
+			}
+			if err := json.Unmarshal(dConfBytes, &dConf); err != nil {
+				panic(err)
+			}
+		}
+
+		genesis.Timestamp = dConf.Timestamp
+		genesis.GasLimit = dConf.GasLimit
+		genesis.Difficulty = big.NewInt(dConf.Difficulty)
+	} else {
+		genesis = core.GenesisBlockByChainName(chain)
+	}
+
 	chainConfig, genesisBlock, genesisErr := core.CommitGenesisBlock(db, genesis, "", log.New())
 	if _, ok := genesisErr.(*chain3.ConfigCompatError); genesisErr != nil && !ok {
 		panic(genesisErr)

cmd/utils/flags.go

@@ -496,6 +496,11 @@ var (
 		Usage: "First block to start syncing from on the L1",
 		Value: 0,
 	}
+	L1FinalizedBlockRequirementFlag = cli.Uint64Flag{
+		Name:  "zkevm.l1-finalized-block-requirement",
+		Usage: "The given block must be finalized before sequencer L1 sync continues",
+		Value: 0,
+	}
 	L1ContractAddressCheckFlag = cli.BoolFlag{
 		Name:  "zkevm.l1-contract-address-check",
 		Usage: "Check the contract address on the L1",
@@ -541,6 +546,21 @@ var (
 		Usage: "Halt the sequencer on this batch number",
 		Value: 0,
 	}
+	SequencerResequence = cli.BoolFlag{
+		Name:  "zkevm.sequencer-resequence",
+		Usage: "When enabled, the sequencer will automatically resequence unseen batches stored in data stream",
+		Value: false,
+	}
+	SequencerResequenceStrict = cli.BoolFlag{
+		Name:  "zkevm.sequencer-resequence-strict",
+		Usage: "Strictly resequence the rolledback batches",
+		Value: true,
+	}
+	SequencerResequenceReuseL1InfoIndex = cli.BoolFlag{
+		Name:  "zkevm.sequencer-resequence-reuse-l1-info-index",
+		Usage: "Reuse the L1 info index for resequencing",
+		Value: true,
+	}
 	ExecutorUrls = cli.StringFlag{
 		Name:  "zkevm.executor-urls",
 		Usage: "A comma separated list of grpc addresses that host executors",

core/state/intra_block_state.go

@@ -369,7 +369,7 @@ func (sdb *IntraBlockState) SetCode(addr libcommon.Address, code []byte) {
 			return
 		}
 
-		hashedBytecode, _ := utils.HashContractBytecode(hex.EncodeToString(code))
+		hashedBytecode := utils.HashContractBytecode(hex.EncodeToString(code))
 		stateObject.SetCode(libcommon.HexToHash(hashedBytecode), code)
 	}
 }

core/state/trie_db.go

@@ -913,48 +913,24 @@ func (tds *TrieDbState) ResolveSMTRetainList() (*trie.RetainList, error) {
 	for _, addrHash := range accountTouches {
 		addr := common.BytesToAddress(tds.preimageMap[addrHash]).String()
 
-		nonceKey, err := utils.KeyEthAddrNonce(addr)
-
-		if err != nil {
-			return nil, err
-		}
-
+		nonceKey := utils.KeyEthAddrNonce(addr)
 		keys = append(keys, nonceKey.GetPath())
 
-		balanceKey, err := utils.KeyEthAddrBalance(addr)
-
-		if err != nil {
-			return nil, err
-		}
-
+		balanceKey := utils.KeyEthAddrBalance(addr)
 		keys = append(keys, balanceKey.GetPath())
 
-		codeKey, err := utils.KeyContractCode(addr)
-
-		if err != nil {
-			return nil, err
-		}
-
+		codeKey := utils.KeyContractCode(addr)
 		keys = append(keys, codeKey.GetPath())
 
-		codeLengthKey, err := utils.KeyContractLength(addr)
-
-		if err != nil {
-			return nil, err
-		}
-
+		codeLengthKey := utils.KeyContractLength(addr)
 		keys = append(keys, codeLengthKey.GetPath())
 	}
 
 	getSMTPath := func(ethAddr string, key string) ([]int, error) {
 		a := utils.ConvertHexToBigInt(ethAddr)
 		addr := utils.ScalarToArrayBig(a)
 
-		storageKey, err := utils.KeyContractStorage(addr, key)
-
-		if err != nil {
-			return nil, err
-		}
+		storageKey := utils.KeyContractStorage(addr, key)
 
 		return storageKey.GetPath(), nil
 	}

eth/ethconfig/config_zkevm.go

@@ -28,6 +28,7 @@ type Zk struct {
 	L1HighestBlockType                     string
 	L1MaticContractAddress                 common.Address
 	L1FirstBlock                           uint64
+	L1FinalizedBlockRequirement            uint64
 	L1CacheEnabled                         bool
 	L1CachePort                            uint
 	RpcRateLimits                          int
@@ -38,6 +39,9 @@ type Zk struct {
 	SequencerBatchVerificationTimeout      time.Duration
 	SequencerTimeoutOnEmptyTxPool          time.Duration
 	SequencerHaltOnBatchNumber             uint64
+	SequencerResequence                    bool
+	SequencerResequenceStrict              bool
+	SequencerResequenceReuseL1InfoIndex    bool
 	ExecutorUrls                           []string
 	ExecutorStrictMode                     bool
 	ExecutorRequestTimeout                 time.Duration

eth/tracers/logger/json_stream_zkevm.go

@@ -33,6 +33,7 @@ type JsonStreamLogger_ZkEvm struct {
 
 	counterCollector *vm.CounterCollector
 	stateClosed      bool
+	memSize          int
 }
 
 // NewStructLogger returns a new logger
@@ -204,36 +205,23 @@ func (l *JsonStreamLogger_ZkEvm) writeMemory(memory *vm.Memory) {
 	if !l.cfg.DisableMemory {
 		memData := memory.Data()
 
-		//[zkevm] don't print empty bytes in memory array after the last non-empty byte line
-		filteredByteLines := [][]byte{}
-		foundValueLine := false
-		for i := len(memData); i-32 >= 0; i -= 32 {
-			bytes := memData[i-32 : i]
-
-			isEmpty := true
-			if !foundValueLine {
-				for _, b := range bytes {
-					if b != 0 {
-						isEmpty = false
-						foundValueLine = true
-						break
-					}
-				}
-			}
-
-			if !isEmpty || foundValueLine {
-				filteredByteLines = append(filteredByteLines, bytes)
-			}
+		// on first occurance don't expand memory
+		// this is because in interpreter we expand the memory before we execute the opcode
+		// and the state for traced opcode should be before the execution of the opcode
+		if l.memSize < len(memData) {
+			size := len(memData)
+			memData = memData[:l.memSize]
+			l.memSize = size
 		}
 
 		l.stream.WriteMore()
 		l.stream.WriteObjectField("memory")
 		l.stream.WriteArrayStart()
-		for i := len(filteredByteLines) - 1; i >= 0; i-- {
-			if i != len(filteredByteLines)-1 {
+		for i := len(memData); i-32 >= 0; i -= 32 {
+			if i != len(memData) { // first 32 bytes, don't add a comma
 				l.stream.WriteMore()
 			}
-			l.stream.WriteString(string(l.hexEncodeBuf[0:hex.Encode(l.hexEncodeBuf[:], filteredByteLines[i])]))
+			l.stream.WriteString(string(l.hexEncodeBuf[0:hex.Encode(l.hexEncodeBuf[:], memData[i-32:i])]))
 		}
 
 		l.stream.WriteArrayEnd()

rpc/types.go

@@ -480,3 +480,11 @@ func (ts *Timestamp) UnmarshalJSON(data []byte) error {
 	return nil
 
 }
+
+type ForkInterval struct {
+	ForkId          hexutil.Uint64 `json:"forkId"`
+	FromBatchNumber hexutil.Uint64 `json:"fromBatchNumber"`
+	ToBatchNumber   hexutil.Uint64 `json:"toBatchNumber"`
+	Version         string         `json:"version"`
+	BlockNumber     hexutil.Uint64 `json:"blockNumber"`
+}

smt/pkg/blockinfo/block_info.go

@@ -325,11 +325,7 @@ func (b *BlockInfoTree) GenerateBlockTxKeysVals(
 
 		logToEncode := "0x" + hex.EncodeToString(rLog.Data) + reducedTopics
 
-		hash, err := utils.HashContractBytecode(logToEncode)
-		if err != nil {
-			return nil, nil, err
-		}
-
+		hash := utils.HashContractBytecode(logToEncode)
 		logEncodedBig := utils.ConvertHexToBigInt(hash)
 		key, val, err = generateTxLog(txIndexBig, big.NewInt(logIndex), logEncodedBig)
 		if err != nil {

smt/pkg/blockinfo/keys.go

@@ -55,14 +55,8 @@ func KeyTxLogs(txIndex, logIndex *big.Int) (*utils.NodeKey, error) {
 		return nil, err
 	}
 
-	hk0, err := utils.Hash(lia.ToUintArray(), utils.BranchCapacity)
-	if err != nil {
-		return nil, err
-	}
-	hkRes, err := utils.Hash(key1.ToUintArray(), hk0)
-	if err != nil {
-		return nil, err
-	}
+	hk0 := utils.Hash(lia.ToUintArray(), utils.BranchCapacity)
+	hkRes := utils.Hash(key1.ToUintArray(), hk0)
 
 	return &utils.NodeKey{hkRes[0], hkRes[1], hkRes[2], hkRes[3]}, nil
 }

smt/pkg/db/mem-db.go

@@ -253,11 +253,7 @@ func (m *MemDb) AddCode(code []byte) error {
 	m.lock.Lock()         // Lock for writing
 	defer m.lock.Unlock() // Make sure to unlock when done
 
-	codeHash, err := utils.HashContractBytecode(hex.EncodeToString(code))
-	if err != nil {
-		return err
-	}
-
+	codeHash := utils.HashContractBytecode(hex.EncodeToString(code))
 	m.DbCode[codeHash] = code
 	return nil
 }