enabling witness generation for bad batches (#646)

* enabling witness generation for bad batches

* bad rebase of bad witness branch

* rename of function for getting batch witness by block range

* removing some magic numbers for block gas and some misleading comments

* fix circular imports for tests

cl/clparams/config.go

@@ -27,6 +27,7 @@ import (
 	"github.com/ledgerwatch/erigon/cl/cltypes/ssz"
 	"github.com/ledgerwatch/erigon/cl/utils"
 	"github.com/ledgerwatch/erigon/params/networkname"
+	zkUtils "github.com/ledgerwatch/erigon/zk/utils"
 )
 
 type NetworkType int
@@ -718,7 +719,7 @@ var MainnetBeaconConfig BeaconChainConfig = BeaconChainConfig{
 	MinSyncCommitteeParticipants: 1,
 
 	// Bellatrix
-	TerminalBlockHashActivationEpoch: 18446744073709551615,
+	TerminalBlockHashActivationEpoch: zkUtils.ForkId7BlockGasLimit,
 	TerminalBlockHash:                [32]byte{},
 	TerminalTotalDifficulty:          "58750000000000000000000", // Estimated: Sept 15, 2022
 	EthBurnAddressHex:                "0x0000000000000000000000000000000000000000",

cmd/rpcdaemon/commands/daemon.go

@@ -46,7 +46,7 @@ func APIList(db kv.RoDB, borDb kv.RoDB, eth rpchelper.ApiBackend, txPool txpool.
 	borImpl := NewBorAPI(base, db, borDb) // bor (consensus) specific
 	otsImpl := NewOtterscanAPI(base, db)
 	gqlImpl := NewGraphQLAPI(base, db)
-	zkEvmImpl := NewZkEvmAPI(ethImpl, db, cfg.ReturnDataLimit, ethCfg, l1Syncer)
+	zkEvmImpl := NewZkEvmAPI(ethImpl, db, cfg.ReturnDataLimit, ethCfg, l1Syncer, rpcUrl)
 
 	if cfg.GraphQLEnabled {
 		list = append(list, rpc.API{

cmd/rpcdaemon/commands/zkevm_api.go

@@ -30,6 +30,7 @@ import (
 	"github.com/ledgerwatch/erigon/zk/syncer"
 	"github.com/ledgerwatch/erigon/zk/witness"
 	"github.com/ledgerwatch/erigon/zkevm/hex"
+	"github.com/ledgerwatch/erigon/zkevm/jsonrpc/client"
 )
 
 var sha3UncleHash = common.HexToHash("0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347")
@@ -51,7 +52,7 @@ type ZkEvmAPI interface {
 	// GetBroadcastURI(ctx context.Context) (string, error)
 	GetWitness(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash, mode *WitnessMode, debug *bool) (hexutility.Bytes, error)
 	GetBlockRangeWitness(ctx context.Context, startBlockNrOrHash rpc.BlockNumberOrHash, endBlockNrOrHash rpc.BlockNumberOrHash, mode *WitnessMode, debug *bool) (hexutility.Bytes, error)
-	GetBatchWitness(ctx context.Context, batchNumber uint64, mode *WitnessMode) (hexutility.Bytes, error)
+	GetBatchWitness(ctx context.Context, batchNumber uint64, mode *WitnessMode) (interface{}, error)
 	GetProverInput(ctx context.Context, batchNumber uint64, mode *WitnessMode, debug *bool) (*legacy_executor_verifier.RpcPayload, error)
 	GetLatestGlobalExitRoot(ctx context.Context) (common.Hash, error)
 	GetExitRootsByGER(ctx context.Context, globalExitRoot common.Hash) (*ZkExitRoots, error)
@@ -69,6 +70,7 @@ type ZkEvmAPIImpl struct {
 	ReturnDataLimit int
 	config          *ethconfig.Config
 	l1Syncer        *syncer.L1Syncer
+	l2SequencerUrl  string
 }
 
 // NewEthAPI returns ZkEvmAPIImpl instance
@@ -78,13 +80,15 @@ func NewZkEvmAPI(
 	returnDataLimit int,
 	zkConfig *ethconfig.Config,
 	l1Syncer *syncer.L1Syncer,
+	l2SequencerUrl string,
 ) *ZkEvmAPIImpl {
 	return &ZkEvmAPIImpl{
 		ethApi:          base,
 		db:              db,
 		ReturnDataLimit: returnDataLimit,
 		config:          zkConfig,
 		l1Syncer:        l1Syncer,
+		l2SequencerUrl:  l2SequencerUrl,
 	}
 }
 
@@ -500,35 +504,24 @@ func (api *ZkEvmAPIImpl) GetBlockRangeWitness(ctx context.Context, startBlockNrO
 	return api.getBlockRangeWitness(ctx, api.db, startBlockNrOrHash, endBlockNrOrHash, dbg, checkedMode)
 }
 
-// Get witness for a range of blocks [startBlockNrOrHash, endBlockNrOrHash] (inclusive)
-func (api *ZkEvmAPIImpl) getBlockRangeWitness(ctx context.Context, db kv.RoDB, startBlockNrOrHash rpc.BlockNumberOrHash, endBlockNrOrHash rpc.BlockNumberOrHash, debug bool, witnessMode WitnessMode) (hexutility.Bytes, error) {
-	tx, err := db.BeginRo(ctx)
-	if err != nil {
-		return nil, err
-	}
-	defer tx.Rollback()
+func (api *ZkEvmAPIImpl) getBatchWitness(ctx context.Context, tx kv.Tx, batchNum uint64, debug bool, mode WitnessMode) (hexutility.Bytes, error) {
 	if api.ethApi.historyV3(tx) {
 		return nil, fmt.Errorf("not supported by Erigon3")
 	}
 
-	blockNr, _, _, err := rpchelper.GetCanonicalBlockNumber(startBlockNrOrHash, tx, api.ethApi.filters) // DoCall cannot be executed on non-canonical blocks
+	generator, fullWitness, err := api.buildGenerator(tx, mode)
 	if err != nil {
 		return nil, err
 	}
 
-	endBlockNr, _, _, err := rpchelper.GetCanonicalBlockNumber(endBlockNrOrHash, tx, api.ethApi.filters) // DoCall cannot be executed on non-canonical blocks
+	return generator.GetWitnessByBatch(tx, ctx, batchNum, debug, fullWitness)
 
-	if err != nil {
-		return nil, err
-	}
-
-	if blockNr > endBlockNr {
-		return nil, fmt.Errorf("start block number must be less than or equal to end block number, start=%d end=%d", blockNr, endBlockNr)
-	}
+}
 
+func (api *ZkEvmAPIImpl) buildGenerator(tx kv.Tx, witnessMode WitnessMode) (*witness.Generator, bool, error) {
 	chainConfig, err := api.ethApi.chainConfig(tx)
 	if err != nil {
-		return nil, err
+		return nil, false, err
 	}
 
 	generator := witness.NewGenerator(
@@ -547,7 +540,41 @@ func (api *ZkEvmAPIImpl) getBlockRangeWitness(ctx context.Context, db kv.RoDB, s
 		fullWitness = true
 	}
 
-	return generator.GenerateWitness(tx, ctx, blockNr, endBlockNr, debug, fullWitness)
+	return generator, fullWitness, nil
+}
+
+// Get witness for a range of blocks [startBlockNrOrHash, endBlockNrOrHash] (inclusive)
+func (api *ZkEvmAPIImpl) getBlockRangeWitness(ctx context.Context, db kv.RoDB, startBlockNrOrHash rpc.BlockNumberOrHash, endBlockNrOrHash rpc.BlockNumberOrHash, debug bool, witnessMode WitnessMode) (hexutility.Bytes, error) {
+	tx, err := db.BeginRo(ctx)
+	if err != nil {
+		return nil, err
+	}
+	defer tx.Rollback()
+	if api.ethApi.historyV3(tx) {
+		return nil, fmt.Errorf("not supported by Erigon3")
+	}
+
+	blockNr, _, _, err := rpchelper.GetCanonicalBlockNumber(startBlockNrOrHash, tx, api.ethApi.filters) // DoCall cannot be executed on non-canonical blocks
+	if err != nil {
+		return nil, err
+	}
+
+	endBlockNr, _, _, err := rpchelper.GetCanonicalBlockNumber(endBlockNrOrHash, tx, api.ethApi.filters) // DoCall cannot be executed on non-canonical blocks
+
+	if err != nil {
+		return nil, err
+	}
+
+	if blockNr > endBlockNr {
+		return nil, fmt.Errorf("start block number must be less than or equal to end block number, start=%d end=%d", blockNr, endBlockNr)
+	}
+
+	generator, fullWitness, err := api.buildGenerator(tx, witnessMode)
+	if err != nil {
+		return nil, err
+	}
+
+	return generator.GetWitnessByBlockRange(tx, ctx, blockNr, endBlockNr, debug, fullWitness)
 }
 
 type WitnessMode string
@@ -558,20 +585,32 @@ const (
 	WitnessModeTrimmed WitnessMode = "trimmed"
 )
 
-func (api *ZkEvmAPIImpl) GetBatchWitness(ctx context.Context, batchNumber uint64, mode *WitnessMode) (hexutility.Bytes, error) {
+func (api *ZkEvmAPIImpl) GetBatchWitness(ctx context.Context, batchNumber uint64, mode *WitnessMode) (interface{}, error) {
+	tx, err := api.db.BeginRo(ctx)
+	if err != nil {
+		return nil, err
+	}
+	defer tx.Rollback()
+
+	hermezDb := hermez_db.NewHermezDbReader(tx)
+	badBatch, err := hermezDb.GetInvalidBatch(batchNumber)
+	if err != nil {
+		return nil, err
+	}
+
+	if badBatch && !sequencer.IsSequencer() {
+		// we won't have the details in our db if the batch is marked as invalid so we need to check this
+		// here
+		return api.sendGetBatchWitness(api.l2SequencerUrl, batchNumber, mode)
+	}
+
 	checkedMode := WitnessModeNone
 	if mode != nil && *mode != WitnessModeFull && *mode != WitnessModeTrimmed {
 		return nil, errors.New("invalid mode, must be full or trimmed")
 	} else if mode != nil {
 		checkedMode = *mode
 	}
 
-	tx, err := api.db.BeginRo(ctx)
-	if err != nil {
-		return nil, err
-	}
-	defer tx.Rollback()
-
 	// we only want to check the cache if no special run mode has been supplied.  If a run mode is supplied
 	// we need to always regenerate the witness from scratch
 	if checkedMode == WitnessModeNone {
@@ -585,19 +624,7 @@ func (api *ZkEvmAPIImpl) GetBatchWitness(ctx context.Context, batchNumber uint64
 		}
 	}
 
-	blocks, err := getAllBlocksInBatchNumber(tx, batchNumber)
-
-	if err != nil {
-		return nil, err
-	}
-
-	if len(blocks) == 0 {
-		return nil, errors.New("batch not found")
-	}
-
-	startBlock := rpc.BlockNumberOrHashWithNumber(rpc.BlockNumber(blocks[0]))
-	endBlock := rpc.BlockNumberOrHashWithNumber(rpc.BlockNumber(blocks[len(blocks)-1]))
-	return api.getBlockRangeWitness(ctx, api.db, startBlock, endBlock, false, checkedMode)
+	return api.getBatchWitness(ctx, tx, batchNumber, false, checkedMode)
 }
 
 func (api *ZkEvmAPIImpl) GetProverInput(ctx context.Context, batchNumber uint64, mode *WitnessMode, debug *bool) (*legacy_executor_verifier.RpcPayload, error) {
@@ -675,6 +702,15 @@ func (api *ZkEvmAPIImpl) GetLatestGlobalExitRoot(ctx context.Context) (common.Ha
 	return ger, nil
 }
 
+func (api *ZkEvmAPIImpl) sendGetBatchWitness(rpcUrl string, batchNumber uint64, mode *WitnessMode) (json.RawMessage, error) {
+	res, err := client.JSONRPCCall(rpcUrl, "zkevm_getBatchWitness", batchNumber, mode)
+	if err != nil {
+		return nil, err
+	}
+
+	return res.Result, nil
+}
+
 func getLastBlockInBatchNumber(tx kv.Tx, batchNumber uint64) (uint64, error) {
 	reader := hermez_db.NewHermezDbReader(tx)
 	blocks, err := reader.GetL2BlockNosByBatch(batchNumber)

core/blockchain_zkevm.go

@@ -36,6 +36,7 @@ import (
 	"github.com/ledgerwatch/erigon/core/types"
 	"github.com/ledgerwatch/erigon/core/vm"
 	"github.com/ledgerwatch/erigon/core/vm/evmtypes"
+	"github.com/ledgerwatch/erigon/zk/utils"
 )
 
 type EphemeralExecResultZk struct {
@@ -66,9 +67,8 @@ func ExecuteBlockEphemerallyZk(
 	blockTransactions := block.Transactions()
 	blockGasLimit := block.GasLimit()
 
-	//[hack] - on forkid7 this gas limit was used for execution but rpc is now returning forkid8 gas limit
 	if !chainConfig.IsForkID8Elderberry(block.NumberU64()) {
-		blockGasLimit = 18446744073709551615
+		blockGasLimit = utils.ForkId7BlockGasLimit
 	}
 
 	gp := new(GasPool).AddGas(blockGasLimit)
@@ -184,7 +184,6 @@ func ExecuteBlockEphemerallyZk(
 			EffectiveGasPrice: effectiveGasPricePercentage,
 			Signer:            &txSender,
 		})
-
 	}
 
 	var l2InfoRoot *common.Hash

smt/pkg/blockinfo/block_info.go

@@ -14,10 +14,6 @@ import (
 	"github.com/ledgerwatch/log/v3"
 )
 
-const (
-	BlockGasLimit = 18446744073709551615
-)
-
 type ExecutedTxInfo struct {
 	Tx                ethTypes.Transaction
 	EffectiveGasPrice uint8

smt/pkg/blockinfo/block_info_test.go

@@ -8,6 +8,7 @@ import (
 	"github.com/ledgerwatch/erigon/core/types"
 	ethTypes "github.com/ledgerwatch/erigon/core/types"
 	"github.com/ledgerwatch/erigon/smt/pkg/smt"
+	"github.com/ledgerwatch/erigon/zk/utils"
 )
 
 /*
@@ -55,7 +56,7 @@ func TestBlockInfoHeader(t *testing.T) {
 			BlockHash:          "0x045fa48a1342813a61c1dd2d235620d621b59cdda0bd07ff3536c6cf64f5e688",
 			CoinbaseAddress:    "0x9aeCf44E36f20DC407d1A580630c9a2419912dcB",
 			NewBlockNumber:     592221,
-			BlockGasLimit:      1125899906842624,
+			BlockGasLimit:      utils.ForkId8BlockGasLimit,
 			FinalTimestamp:     1708198045,
 			FinalGER:           "0x0000000000000000000000000000000000000000000000000000000000000000",
 			L1BlochHash:        "0x0000000000000000000000000000000000000000000000000000000000000000",
@@ -64,7 +65,7 @@ func TestBlockInfoHeader(t *testing.T) {
 			BlockHash:          "0x268a22af2bae40acd1cc4228896de4420c5f3bc3bbdd8515d6d01b1b99731f82",
 			CoinbaseAddress:    "0x9aeCf44E36f20DC407d1A580630c9a2419912dcB",
 			NewBlockNumber:     592223,
-			BlockGasLimit:      18446744073709551615,
+			BlockGasLimit:      utils.ForkId7BlockGasLimit,
 			FinalTimestamp:     1708198051,
 			FinalGER:           "0x0000000000000000000000000000000000000000000000000000000000000000",
 			L1BlochHash:        "0x0000000000000000000000000000000000000000000000000000000000000000",

smt/pkg/utils/util_test.go

@@ -9,6 +9,8 @@ import (
 	"testing"
 )
 
+const forkId7BlockGasLimit = 18446744073709551615
+
 func TestBinaryStringToInt64(t *testing.T) {
 	testCases := []struct {
 		binary   string
@@ -18,7 +20,7 @@ func TestBinaryStringToInt64(t *testing.T) {
 		{"110011", 51},
 		{"0", 0},
 		{"11111111", 255},
-		{"1111111111111111111111111111111111111111111111111111111111111111", 18446744073709551615}, // max uint64 value
+		{"1111111111111111111111111111111111111111111111111111111111111111", forkId7BlockGasLimit}, // max uint64 value
 		{"10000000000000000000000000000000000000000000000000000000000000000", 0},                   // overflow scenario
 	}
 

zk/l1_data/l1_decoder.go

@@ -11,6 +11,10 @@ import (
 	"github.com/ledgerwatch/erigon/crypto"
 	"github.com/ledgerwatch/erigon/zk/contracts"
 	"github.com/ledgerwatch/erigon/zk/da"
+	"github.com/ledgerwatch/erigon/zk/hermez_db"
+	zktx "github.com/ledgerwatch/erigon/zk/tx"
+	"github.com/gateway-fm/cdk-erigon-lib/common/length"
+	"encoding/binary"
 )
 
 type RollupBaseEtrogBatchData struct {
@@ -155,3 +159,61 @@ func DecodeL1BatchData(txData []byte, daUrl string) ([][]byte, common.Address, u
 
 	return batchL2Datas, coinbase, limitTimstamp, err
 }
+
+type DecodedL1Data struct {
+	DecodedData     []zktx.DecodedBatchL2Data
+	Coinbase        common.Address
+	L1InfoRoot      common.Hash
+	IsWorkRemaining bool
+	LimitTimestamp  uint64
+}
+
+func BreakDownL1DataByBatch(batchNo uint64, forkId uint64, reader *hermez_db.HermezDbReader) (DecodedL1Data, error) {
+	decoded := DecodedL1Data{}
+	// we expect that the batch we're going to load in next should be in the db already because of the l1 block sync
+	// stage, if it is not there we need to panic as we're in a bad state
+	batchData, err := reader.GetL1BatchData(batchNo)
+	if err != nil {
+		return decoded, err
+	}
+
+	if len(batchData) == 0 {
+		// end of the line for batch recovery so return empty
+		return decoded, nil
+	}
+
+	decoded.Coinbase = common.BytesToAddress(batchData[:length.Addr])
+	decoded.L1InfoRoot = common.BytesToHash(batchData[length.Addr : length.Addr+length.Hash])
+	tsBytes := batchData[length.Addr+length.Hash : length.Addr+length.Hash+8]
+	decoded.LimitTimestamp = binary.BigEndian.Uint64(tsBytes)
+	batchData = batchData[length.Addr+length.Hash+8:]
+
+	decoded.DecodedData, err = zktx.DecodeBatchL2Blocks(batchData, forkId)
+	if err != nil {
+		return decoded, err
+	}
+
+	// no data means no more work to do - end of the line
+	if len(decoded.DecodedData) == 0 {
+		return decoded, nil
+	}
+
+	decoded.IsWorkRemaining = true
+	transactionsInBatch := 0
+	for _, batch := range decoded.DecodedData {
+		transactionsInBatch += len(batch.Transactions)
+	}
+	if transactionsInBatch == 0 {
+		// we need to check if this batch should simply be empty or not so we need to check against the
+		// highest known batch number to see if we have work to do still
+		highestKnown, err := reader.GetLastL1BatchData()
+		if err != nil {
+			return decoded, err
+		}
+		if batchNo >= highestKnown {
+			decoded.IsWorkRemaining = false
+		}
+	}
+
+	return decoded, err
+}