Merge pull request #31 from bobanetwork/add-seq-historical-rpcs

Add seq and historical RPC endpoints

Makefile

@@ -149,10 +149,10 @@ test3:
 
 ## test-integration:                  run integration tests with a 30m timeout
 test-integration:
-	$(GOTEST) --timeout 30m -tags $(BUILD_TAGS),integration
+	$(GOTEST) --timeout 60m -tags $(BUILD_TAGS),integration
 
 test3-integration:
-	$(GOTEST) --timeout 30m -tags $(BUILD_TAGS),integration,e3
+	$(GOTEST) --timeout 60m -tags $(BUILD_TAGS),integration,e3
 
 ## lint:                              run golangci-lint with .golangci.yml config file
 lint:

cmd/rpcdaemon/cli/config.go

@@ -120,6 +120,10 @@ func RootCommand() (*cobra.Command, *httpcfg.HttpCfg) {
 	rootCmd.PersistentFlags().IntVar(&cfg.BatchLimit, utils.RpcBatchLimit.Name, utils.RpcBatchLimit.Value, utils.RpcBatchLimit.Usage)
 	rootCmd.PersistentFlags().IntVar(&cfg.ReturnDataLimit, utils.RpcReturnDataLimit.Name, utils.RpcReturnDataLimit.Value, utils.RpcReturnDataLimit.Usage)
 
+	rootCmd.PersistentFlags().StringVar(&cfg.RollupSequencerHTTP, utils.RollupSequencerHTTPFlag.Name, "", "HTTP endpoint for the sequencer mempool")
+	rootCmd.PersistentFlags().StringVar(&cfg.RollupHistoricalRPC, utils.RollupHistoricalRPCFlag.Name, "", "RPC endpoint for historical data")
+	rootCmd.PersistentFlags().DurationVar(&cfg.RollupHistoricalRPCTimeout, utils.RollupHistoricalRPCTimeoutFlag.Name, rpccfg.DefaultHistoricalRPCTimeout, "Timeout for historical RPC requests")
+
 	if err := rootCmd.MarkPersistentFlagFilename("rpc.accessList", "json"); err != nil {
 		panic(err)
 	}

cmd/rpcdaemon/cli/httpcfg/http_cfg.go

@@ -63,4 +63,9 @@ type HttpCfg struct {
 
 	BatchLimit      int // Maximum number of requests in a batch
 	ReturnDataLimit int // Maximum number of bytes returned from calls (like eth_call)
+
+	RollupSequencerHTTP        string
+	RollupHistoricalRPC        string
+	RollupHistoricalRPCTimeout time.Duration
+	RollupDisableTxPoolGossip  bool
 }

cmd/rpcdaemon/main.go

@@ -1,14 +1,18 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"os"
+	"time"
 
 	"github.com/ledgerwatch/erigon-lib/common"
 	"github.com/ledgerwatch/erigon/cmd/rpcdaemon/cli"
 	"github.com/ledgerwatch/erigon/consensus/ethash"
+	"github.com/ledgerwatch/erigon/rpc"
 	"github.com/ledgerwatch/erigon/turbo/debug"
 	"github.com/ledgerwatch/erigon/turbo/jsonrpc"
+	"github.com/ledgerwatch/log/v3"
 	"github.com/spf13/cobra"
 )
 
@@ -28,9 +32,34 @@ func main() {
 			defer borDb.Close()
 		}
 
+		var seqRPCService *rpc.Client
+		var historicalRPCService *rpc.Client
+
+		// Setup sequencer and hsistorical RPC relay services
+		if cfg.RollupSequencerHTTP != "" {
+			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+			client, err := rpc.DialContext(ctx, cfg.RollupSequencerHTTP, logger)
+			cancel()
+			if err != nil {
+				log.Error(err.Error())
+				return nil
+			}
+			seqRPCService = client
+		}
+		if cfg.RollupHistoricalRPC != "" {
+			ctx, cancel := context.WithTimeout(context.Background(), cfg.RollupHistoricalRPCTimeout)
+			client, err := rpc.DialContext(ctx, cfg.RollupHistoricalRPC, logger)
+			cancel()
+			if err != nil {
+				log.Error(err.Error())
+				return nil
+			}
+			historicalRPCService = client
+		}
+
 		// TODO: Replace with correct consensus Engine
 		engine := ethash.NewFaker()
-		apiList := jsonrpc.APIList(db, borDb, backend, engineBackend, txPool, mining, ff, stateCache, blockReader, agg, *cfg, engine, logger)
+		apiList := jsonrpc.APIList(db, borDb, backend, engineBackend, txPool, mining, ff, stateCache, blockReader, agg, *cfg, engine, seqRPCService, historicalRPCService, logger)
 		if err := cli.StartRpcServer(ctx, *cfg, apiList, nil, logger); err != nil {
 			logger.Error(err.Error())
 			return nil

cmd/rpcdaemon/rpcdaemontest/test_util.go

@@ -112,6 +112,53 @@ func CreateTestSentry(t *testing.T) (*stages.MockSentry, *core.ChainPack, []*cor
 	return m, chain, []*core.ChainPack{orphanedChain}
 }
 
+func CreateOptimismTestSentry(t *testing.T) (*stages.MockSentry, *core.ChainPack, []*core.ChainPack) {
+	addresses := makeTestAddresses()
+	var (
+		key      = addresses.key
+		address  = addresses.address
+		address1 = addresses.address1
+		address2 = addresses.address2
+	)
+
+	var (
+		gspec = &types.Genesis{
+			Config: params.TestOptimismChainConfig,
+			Alloc: types.GenesisAlloc{
+				address:  {Balance: big.NewInt(9000000000000000000)},
+				address1: {Balance: big.NewInt(200000000000000000)},
+				address2: {Balance: big.NewInt(300000000000000000)},
+			},
+			GasLimit: 10000000,
+		}
+	)
+	m := stages.MockWithGenesis(t, gspec, key, false)
+
+	contractBackend := backends.NewTestSimulatedBackendWithConfig(t, gspec.Alloc, gspec.Config, gspec.GasLimit)
+	defer contractBackend.Close()
+
+	// Generate empty chain to have some orphaned blocks for tests
+	orphanedChain, err := core.GenerateChain(m.ChainConfig, m.Genesis, m.Engine, m.DB, 5, func(i int, block *core.BlockGen) {
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	chain, err := getChainInstance(&addresses, m.ChainConfig, m.Genesis, m.Engine, m.DB, contractBackend)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err = m.InsertChain(orphanedChain, nil); err != nil {
+		t.Fatal(err)
+	}
+	if err = m.InsertChain(chain, nil); err != nil {
+		t.Fatal(err)
+	}
+
+	return m, chain, []*core.ChainPack{orphanedChain}
+}
+
 var chainInstance *core.ChainPack
 
 func getChainInstance(

cmd/utils/flags.go

@@ -771,6 +771,27 @@ var (
 		Usage: "Port for sentinel",
 		Value: 7777,
 	}
+
+	// Rollup Flags
+	RollupSequencerHTTPFlag = cli.StringFlag{
+		Name:    "rollup.sequencerhttp",
+		Usage:   "HTTP endpoint for the sequencer mempool",
+		EnvVars: []string{"ROLLUP_SEQUENCER_HTTP_ENDPOINT"},
+	}
+	RollupHistoricalRPCFlag = cli.StringFlag{
+		Name:    "rollup.historicalrpc",
+		Usage:   "RPC endpoint for historical data.",
+		EnvVars: []string{"ROLLUP_HISTORICAL_RPC_ENDPOINT"},
+	}
+	RollupHistoricalRPCTimeoutFlag = cli.StringFlag{
+		Name:  "rollup.historicalrpctimeout",
+		Usage: "Timeout for historical RPC requests.",
+		Value: "5s",
+	}
+	RollupDisableTxPoolGossipFlag = cli.BoolFlag{
+		Name:  "rollup.disabletxpoolgossip",
+		Usage: "Disable transaction pool gossip.",
+	}
 )
 
 var MetricFlags = []cli.Flag{&MetricsEnabledFlag, &MetricsHTTPFlag, &MetricsPortFlag}
@@ -1574,6 +1595,18 @@ func SetEthConfig(ctx *cli.Context, nodeConfig *nodecfg.Config, cfg *ethconfig.C
 	if ctx.IsSet(SentryDropUselessPeers.Name) {
 		cfg.DropUselessPeers = ctx.Bool(SentryDropUselessPeers.Name)
 	}
+
+	// Rollup params
+	if ctx.IsSet(RollupSequencerHTTPFlag.Name) && !ctx.IsSet(MiningEnabledFlag.Name) {
+		cfg.RollupSequencerHTTP = ctx.String(RollupSequencerHTTPFlag.Name)
+	}
+	if ctx.IsSet(RollupHistoricalRPCFlag.Name) {
+		cfg.RollupHistoricalRPC = ctx.String(RollupHistoricalRPCFlag.Name)
+	}
+	if ctx.IsSet(RollupDisableTxPoolGossipFlag.Name) {
+		cfg.RollupDisableTxPoolGossip = ctx.Bool(RollupDisableTxPoolGossipFlag.Name)
+	}
+	cfg.RollupHistoricalRPCTimeout = ctx.Duration(RollupHistoricalRPCTimeoutFlag.Name)
 }
 
 // SetDNSDiscoveryDefaults configures DNS discovery with the given URL if

eth/backend.go

@@ -138,10 +138,12 @@ type Ethereum struct {
 	genesisBlock *types.Block
 	genesisHash  libcommon.Hash
 
-	ethBackendRPC      *privateapi.EthBackendServer
-	engineBackendRPC   engine.EngineClient
-	miningRPC          txpool_proto.MiningServer
-	stateChangesClient txpool.StateChangesClient
+	ethBackendRPC        *privateapi.EthBackendServer
+	seqRPCService        *rpc.Client
+	historicalRPCService *rpc.Client
+	engineBackendRPC     engine.EngineClient
+	miningRPC            txpool_proto.MiningServer
+	stateChangesClient   txpool.StateChangesClient
 
 	miningSealingQuit chan struct{}
 	pendingBlocks     chan *types.Block
@@ -488,6 +490,26 @@ func New(stack *node.Node, config *ethconfig.Config, logger log.Logger) (*Ethere
 		return nil, err
 	}
 
+	// Setup sequencer and hsistorical RPC relay services
+	if config.RollupSequencerHTTP != "" {
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		client, err := rpc.DialContext(ctx, config.RollupSequencerHTTP, logger)
+		cancel()
+		if err != nil {
+			return nil, err
+		}
+		backend.seqRPCService = client
+	}
+	if config.RollupHistoricalRPC != "" {
+		ctx, cancel := context.WithTimeout(context.Background(), config.RollupHistoricalRPCTimeout)
+		client, err := rpc.DialContext(ctx, config.RollupHistoricalRPC, logger)
+		cancel()
+		if err != nil {
+			return nil, err
+		}
+		backend.historicalRPCService = client
+	}
+
 	var miningRPC txpool_proto.MiningServer
 	stateDiffClient := direct.NewStateDiffClientDirect(kvRPC)
 	if config.DeprecatedTxPool.Disable {
@@ -784,8 +806,8 @@ func (s *Ethereum) Init(stack *node.Node, config *ethconfig.Config) error {
 	if casted, ok := s.engine.(*bor.Bor); ok {
 		borDb = casted.DB
 	}
-	apiList := jsonrpc.APIList(chainKv, borDb, ethRpcClient, engineClient, txPoolRpcClient, miningRpcClient, ff, stateCache, blockReader, s.agg, httpRpcCfg, s.engine, s.logger)
-	authApiList := jsonrpc.AuthAPIList(chainKv, ethRpcClient, engineClient, txPoolRpcClient, miningRpcClient, ff, stateCache, blockReader, s.agg, httpRpcCfg, s.engine, s.logger)
+	apiList := jsonrpc.APIList(chainKv, borDb, ethRpcClient, engineClient, txPoolRpcClient, miningRpcClient, ff, stateCache, blockReader, s.agg, httpRpcCfg, s.engine, s.seqRPCService, s.historicalRPCService, s.logger)
+	authApiList := jsonrpc.AuthAPIList(chainKv, ethRpcClient, engineClient, txPoolRpcClient, miningRpcClient, ff, stateCache, blockReader, s.agg, httpRpcCfg, s.engine, s.seqRPCService, s.historicalRPCService, s.logger)
 	go func() {
 		if err := cli.StartRpcServer(ctx, httpRpcCfg, apiList, authApiList, s.logger); err != nil {
 			s.logger.Error(err.Error())

eth/ethconfig/config.go

@@ -248,6 +248,11 @@ type Config struct {
 	OverrideShanghaiTime *big.Int `toml:",omitempty"`
 
 	DropUselessPeers bool
+
+	RollupSequencerHTTP        string
+	RollupHistoricalRPC        string
+	RollupHistoricalRPCTimeout time.Duration
+	RollupDisableTxPoolGossip  bool
 }
 
 type Sync struct {

go.mod

@@ -105,7 +105,10 @@ require (
 	pgregory.net/rapid v1.0.0
 )
 
-require github.com/ethereum/go-ethereum v1.10.25
+require (
+	github.com/anacrolix/log v0.14.0
+	github.com/ethereum/go-ethereum v1.10.25
+)
 
 require (
 	crawshaw.io/sqlite v0.3.3-0.20220618202545-d1964889ea3c // indirect
@@ -117,7 +120,6 @@ require (
 	github.com/anacrolix/envpprof v1.2.1 // indirect
 	github.com/anacrolix/generics v0.0.0-20230428105757-683593396d68 // indirect
 	github.com/anacrolix/go-libutp v1.3.0 // indirect
-	github.com/anacrolix/log v0.14.0 // indirect
 	github.com/anacrolix/missinggo v1.3.0 // indirect
 	github.com/anacrolix/missinggo/perf v1.0.0 // indirect
 	github.com/anacrolix/missinggo/v2 v2.7.2-0.20230527121029-a582b4f397b9 // indirect
@@ -271,7 +273,7 @@ require (
 	rsc.io/tmplfunc v0.0.3 // indirect
 )
 
-replace github.com/ledgerwatch/erigon-lib => github.com/bobanetwork/v3-erigon-lib v0.0.0-20230712175420-636c390c9d3c
+replace github.com/ledgerwatch/erigon-lib => github.com/bobanetwork/v3-erigon-lib v0.0.0-20230718202321-182c163bbdde
 
 replace github.com/tendermint/tendermint => github.com/bnb-chain/tendermint v0.31.12
 

go.sum

@@ -122,8 +122,8 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r
 github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
 github.com/bits-and-blooms/bitset v1.5.0 h1:NpE8frKRLGHIcEzkR+gZhiioW1+WbYV6fKwD6ZIpQT8=
 github.com/bits-and-blooms/bitset v1.5.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
-github.com/bobanetwork/v3-erigon-lib v0.0.0-20230712175420-636c390c9d3c h1:1bcJGClKkSv7Cgxif7NuaON6DU8jmuE9RGvxs33xO8c=
-github.com/bobanetwork/v3-erigon-lib v0.0.0-20230712175420-636c390c9d3c/go.mod h1:khV70zMPfHeF5O/lk/2w6phTSi9u4PusY6qkJKjL+/E=
+github.com/bobanetwork/v3-erigon-lib v0.0.0-20230718202321-182c163bbdde h1:/7rqDufJYEmAl4efTZ5aLPD91azBE1/TIhUHJprMMrY=
+github.com/bobanetwork/v3-erigon-lib v0.0.0-20230718202321-182c163bbdde/go.mod h1:khV70zMPfHeF5O/lk/2w6phTSi9u4PusY6qkJKjL+/E=
 github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
 github.com/bradfitz/iter v0.0.0-20140124041915-454541ec3da2/go.mod h1:PyRFw1Lt2wKX4ZVSQ2mk+PeDa1rxyObEDlApuIsUKuo=
 github.com/bradfitz/iter v0.0.0-20190303215204-33e6a9893b0c/go.mod h1:PyRFw1Lt2wKX4ZVSQ2mk+PeDa1rxyObEDlApuIsUKuo=

params/config.go

@@ -161,6 +161,26 @@ var (
 		Aura:                  &chain.AuRaConfig{},
 	}
 
+	TestOptimismChainConfig = &chain.Config{
+		ChainID:               big.NewInt(1337),
+		Consensus:             chain.EtHashConsensus,
+		HomesteadBlock:        big.NewInt(0),
+		TangerineWhistleBlock: big.NewInt(0),
+		SpuriousDragonBlock:   big.NewInt(0),
+		ByzantiumBlock:        big.NewInt(0),
+		ConstantinopleBlock:   big.NewInt(0),
+		PetersburgBlock:       big.NewInt(0),
+		IstanbulBlock:         big.NewInt(0),
+		MuirGlacierBlock:      big.NewInt(0),
+		BerlinBlock:           big.NewInt(0),
+		Ethash:                new(chain.EthashConfig),
+		Optimism: &chain.OptimismConfig{
+			EIP1559Elasticity:  8,
+			EIP1559Denominator: 1,
+		},
+		BedrockBlock: big.NewInt(1000000000000000000),
+	}
+
 	TestRules = TestChainConfig.Rules(0, 0)
 )
 

rpc/errors.go

@@ -30,6 +30,16 @@ var (
 
 const defaultErrorCode = -32000
 
+var ErrNoHistoricalFallback = NoHistoricalFallbackError{}
+
+type NoHistoricalFallbackError struct{}
+
+func (e NoHistoricalFallbackError) ErrorCode() int { return -32801 }
+
+func (e NoHistoricalFallbackError) Error() string {
+	return "no historical RPC is available for this historical (pre-bedrock) execution request"
+}
+
 type methodNotFoundError struct{ method string }
 
 func (e *methodNotFoundError) ErrorCode() int { return -32601 }

rpc/rpccfg/rpccfg.go

@@ -37,3 +37,5 @@ var DefaultHTTPTimeouts = HTTPTimeouts{
 }
 
 const DefaultEvmCallTimeout = 5 * time.Minute
+
+const DefaultHistoricalRPCTimeout = 5 * time.Second

turbo/cli/default_flags.go

@@ -145,6 +145,10 @@ var DefaultFlags = []cli.Flag{
 	&utils.HeimdallgRPCAddressFlag,
 	&utils.EthStatsURLFlag,
 	&utils.OverrideShanghaiTime,
+	&utils.RollupSequencerHTTPFlag,
+	&utils.RollupHistoricalRPCFlag,
+	&utils.RollupHistoricalRPCTimeoutFlag,
+	&utils.RollupDisableTxPoolGossipFlag,
 
 	&utils.ConfigFlag,
 

turbo/cli/flags.go

@@ -400,6 +400,10 @@ func setEmbeddedRpcDaemon(ctx *cli.Context, cfg *nodecfg.Config, logger log.Logg
 		TxPoolApiAddr: ctx.String(utils.TxpoolApiAddrFlag.Name),
 
 		StateCache: kvcache.DefaultCoherentConfig,
+
+		RollupSequencerHTTP:        ctx.String(utils.RollupSequencerHTTPFlag.Name),
+		RollupHistoricalRPC:        ctx.String(utils.RollupHistoricalRPCFlag.Name),
+		RollupHistoricalRPCTimeout: ctx.Duration(utils.RollupHistoricalRPCTimeoutFlag.Name),
 	}
 	if ctx.IsSet(utils.HttpCompressionFlag.Name) {
 		c.HttpCompression = ctx.Bool(utils.HttpCompressionFlag.Name)

turbo/jsonrpc/daemon.go

@@ -18,9 +18,9 @@ import (
 func APIList(db kv.RoDB, borDb kv.RoDB, eth rpchelper.ApiBackend, engineBackend rpchelper.EngineBackend, txPool txpool.TxpoolClient, mining txpool.MiningClient,
 	filters *rpchelper.Filters, stateCache kvcache.Cache,
 	blockReader services.FullBlockReader, agg *libstate.AggregatorV3, cfg httpcfg.HttpCfg, engine consensus.EngineReader,
-	logger log.Logger,
+	seqRPCService, historicalRPCService *rpc.Client, logger log.Logger,
 ) (list []rpc.API) {
-	base := NewBaseApi(filters, stateCache, blockReader, agg, cfg.WithDatadir, cfg.EvmCallTimeout, engine, cfg.Dirs)
+	base := NewBaseApi(filters, stateCache, blockReader, agg, cfg.WithDatadir, cfg.EvmCallTimeout, engine, cfg.Dirs, seqRPCService, historicalRPCService)
 	ethImpl := NewEthAPI(base, db, eth, txPool, mining, cfg.Gascap, cfg.ReturnDataLimit, logger)
 	erigonImpl := NewErigonAPI(base, db, eth)
 	txpoolImpl := NewTxPoolAPI(base, db, txPool)
@@ -142,9 +142,9 @@ func AuthAPIList(db kv.RoDB, eth rpchelper.ApiBackend, engineBackend rpchelper.E
 	filters *rpchelper.Filters, stateCache kvcache.Cache, blockReader services.FullBlockReader,
 	agg *libstate.AggregatorV3,
 	cfg httpcfg.HttpCfg, engine consensus.EngineReader,
-	logger log.Logger,
+	seqRPCService, historicalRPCService *rpc.Client, logger log.Logger,
 ) (list []rpc.API) {
-	base := NewBaseApi(filters, stateCache, blockReader, agg, cfg.WithDatadir, cfg.EvmCallTimeout, engine, cfg.Dirs)
+	base := NewBaseApi(filters, stateCache, blockReader, agg, cfg.WithDatadir, cfg.EvmCallTimeout, engine, cfg.Dirs, seqRPCService, historicalRPCService)
 
 	ethImpl := NewEthAPI(base, db, eth, txPool, mining, cfg.Gascap, cfg.ReturnDataLimit, logger)
 	engineImpl := NewEngineAPI(base, db, engineBackend)