Merge remote-tracking branch 'origin/zkevm' into zkevm-2.60

cmd/cdk-erigon/main.go

@@ -110,13 +110,17 @@ func setFlagsFromConfigFile(ctx *cli.Context, filePath string) error {
 				for i, v := range sliceInterface {
 					s[i] = fmt.Sprintf("%v", v)
 				}
-				err := ctx.Set(key, strings.Join(s, ","))
-				if err != nil {
+				if err := ctx.Set(key, strings.Join(s, ",")); err != nil {
+					if deprecatedFlag, found := erigoncli.DeprecatedFlags[key]; found {
+						return fmt.Errorf("failed setting %s flag Flag is deprecated, use %s instead", key, deprecatedFlag)
+					}
 					return fmt.Errorf("failed setting %s flag with values=%s error=%s", key, s, err)
 				}
 			} else {
-				err := ctx.Set(key, fmt.Sprintf("%v", value))
-				if err != nil {
+				if err := ctx.Set(key, fmt.Sprintf("%v", value)); err != nil {
+					if deprecatedFlag, found := erigoncli.DeprecatedFlags[key]; found {
+						return fmt.Errorf("failed setting %s flag Flag is deprecated, use %s instead", key, deprecatedFlag)
+					}
 					return fmt.Errorf("failed setting %s flag with value=%v error=%s", key, value, err)
 				}
 			}

cmd/rpcdaemon/cli/config.go

@@ -93,7 +93,6 @@ func RootCommand() (*cobra.Command, *httpcfg.HttpCfg) {
 	rootCmd.PersistentFlags().BoolVar(&cfg.GraphQLEnabled, "graphql", false, "enables graphql endpoint (disabled by default)")
 	rootCmd.PersistentFlags().Uint64Var(&cfg.Gascap, "rpc.gascap", 50_000_000, "Sets a cap on gas that can be used in eth_call/estimateGas")
 	rootCmd.PersistentFlags().Uint64Var(&cfg.MaxTraces, "trace.maxtraces", 200, "Sets a limit on traces that can be returned in trace_filter")
-
 	rootCmd.PersistentFlags().StringVar(&cfg.RpcAllowListFilePath, utils.RpcAccessListFlag.Name, "", "Specify granular (method-by-method) API allowlist")
 	rootCmd.PersistentFlags().UintVar(&cfg.RpcBatchConcurrency, utils.RpcBatchConcurrencyFlag.Name, 2, utils.RpcBatchConcurrencyFlag.Usage)
 	rootCmd.PersistentFlags().BoolVar(&cfg.RpcStreamingDisable, utils.RpcStreamingDisableFlag.Name, false, utils.RpcStreamingDisableFlag.Usage)
@@ -124,6 +123,9 @@ func RootCommand() (*cobra.Command, *httpcfg.HttpCfg) {
 	rootCmd.PersistentFlags().BoolVar(&cfg.HttpCompression, "http.compression", true, "Disable http compression")
 	rootCmd.PersistentFlags().BoolVar(&cfg.WebsocketEnabled, "ws", false, "Enable Websockets - Same port as HTTP[S]")
 	rootCmd.PersistentFlags().BoolVar(&cfg.WebsocketCompression, "ws.compression", false, "Enable Websocket compression (RFC 7692)")
+	rootCmd.PersistentFlags().StringVar(&cfg.WebSocketListenAddress, "ws.addr", nodecfg.DefaultHTTPHost, "Websocket server listening interface")
+	rootCmd.PersistentFlags().IntVar(&cfg.WebSocketPort, "ws.port", nodecfg.DefaultHTTPPort, "Websocket server listening port")
+	rootCmd.PersistentFlags().StringSliceVar(&cfg.WebsocketCORSDomain, "ws.corsdomain", []string{}, "Comma separated list of domains from which to accept cross origin requests (browser enforced)")
 
 	rootCmd.PersistentFlags().BoolVar(&cfg.HttpsServerEnabled, "https.enabled", false, "enable http server")
 	rootCmd.PersistentFlags().StringVar(&cfg.HttpsListenAddress, "https.addr", nodecfg.DefaultHTTPHost, "rpc HTTPS server listening interface")
@@ -630,37 +632,14 @@ func startRegularRpcServer(ctx context.Context, cfg *httpcfg.HttpCfg, rpcAPI []r
 	}
 
 	httpHandler := node.NewHTTPHandlerStack(srv, cfg.HttpCORSDomain, cfg.HttpVirtualHost, cfg.HttpCompression)
-	var wsHandler http.Handler
-	if cfg.WebsocketEnabled {
-		wsHandler = srv.WebsocketHandler([]string{"*"}, nil, cfg.WebsocketCompression, logger)
-	}
+
 	graphQLHandler := graphql.CreateHandler(defaultAPIList)
-	apiHandler, err := createHandler(cfg, defaultAPIList, httpHandler, wsHandler, graphQLHandler, nil)
+
+	apiHandler, err := createHandler(*cfg, defaultAPIList, httpHandler, nil, graphQLHandler, nil)
 	if err != nil {
 		return err
 	}
 
-	// Separate Websocket handler if websocket port flag specified
-	if cfg.WebsocketEnabled && cfg.WebsocketPort != cfg.HttpPort {
-		wsEndpoint := fmt.Sprintf("tcp://%s:%d", cfg.HttpListenAddress, cfg.WebsocketPort)
-		wsApiHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			if isWebsocket(r) {
-				wsHandler.ServeHTTP(w, r)
-			}
-		})
-		wsListener, wsAddr, err := node.StartHTTPEndpoint(wsEndpoint, &node.HttpEndpointConfig{Timeouts: cfg.HTTPTimeouts}, wsApiHandler)
-		if err != nil {
-			return fmt.Errorf("could not start separate Websocket RPC api at port %d: %w", cfg.WebsocketPort, err)
-		}
-		info = append(info, "websocket.url", wsAddr)
-		defer func() {
-			shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-			defer cancel()
-			_ = wsListener.Shutdown(shutdownCtx)
-			logger.Info("HTTP endpoint closed", "url", wsAddr)
-		}()
-	}
-
 	if cfg.HttpServerEnabled {
 		httpEndpoint := fmt.Sprintf("tcp://%s:%d", cfg.HttpListenAddress, cfg.HttpPort)
 		if cfg.HttpURL != "" {
@@ -680,6 +659,7 @@ func startRegularRpcServer(ctx context.Context, cfg *httpcfg.HttpCfg, rpcAPI []r
 			logger.Info("HTTP endpoint closed", "url", httpAddr)
 		}()
 	}
+
 	if cfg.HttpsURL != "" {
 		cfg.HttpsServerEnabled = true
 	}
@@ -740,7 +720,72 @@ func startRegularRpcServer(ctx context.Context, cfg *httpcfg.HttpCfg, rpcAPI []r
 		}()
 	}
 
-	logger.Info("JsonRpc endpoint opened", info...)
+	log.Info("HTTP endpoint opened", info...)
+
+	if cfg.WebsocketEnabled {
+		wsSrv := rpc.NewServer(cfg.RpcBatchConcurrency, cfg.TraceRequests, cfg.RpcStreamingDisable, false, logger, cfg.RPCSlowLogThreshold)
+
+		allowListForRPC, err := parseAllowListForRPC(cfg.RpcAllowListFilePath)
+		if err != nil {
+			return err
+		}
+
+		var wsApiFlags []string
+		for _, flag := range cfg.WebSocketApi {
+			if flag != "engine" {
+				wsApiFlags = append(wsApiFlags, flag)
+			}
+		}
+
+		if err := node.RegisterApisFromWhitelist(defaultAPIList, wsApiFlags, wsSrv, false, logger); err != nil {
+			return fmt.Errorf("could not start register WS apis: %w", err)
+		}
+		wsSrv.SetAllowList(allowListForRPC)
+
+		wsSrv.SetBatchLimit(cfg.BatchLimit)
+
+		var defaultAPIList []rpc.API
+
+		for _, api := range rpcAPI {
+			if api.Namespace != "engine" {
+				defaultAPIList = append(defaultAPIList, api)
+			}
+		}
+
+		var apiFlags []string
+		for _, flag := range cfg.API {
+			if flag != "engine" {
+				apiFlags = append(apiFlags, flag)
+			}
+		}
+
+		if err := node.RegisterApisFromWhitelist(defaultAPIList, apiFlags, wsSrv, false, logger); err != nil {
+			return fmt.Errorf("could not start register RPC apis: %w", err)
+		}
+
+		wsEndpoint := fmt.Sprintf("tcp://%s:%d", cfg.WebSocketListenAddress, cfg.WebSocketPort)
+
+		wsHttpHandler := wsSrv.WebsocketHandler(cfg.WebsocketCORSDomain, nil, cfg.WebsocketCompression, logger)
+		wsHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			wsHttpHandler.ServeHTTP(w, r)
+		})
+
+		wsListener, wsHttpAddr, err := node.StartHTTPEndpoint(wsEndpoint, &node.HttpEndpointConfig{
+			Timeouts: cfg.HTTPTimeouts,
+		}, wsHandler)
+		if err != nil {
+			return fmt.Errorf("could not start ws RPC api: %w", err)
+		}
+
+		defer func() {
+			wsSrv.Stop()
+			shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+			defer cancel()
+			_ = wsListener.Shutdown(shutdownCtx)
+			log.Info("WS endpoint closed", "url", wsHttpAddr)
+		}()
+	}
+
 	<-ctx.Done()
 	logger.Info("Exiting...")
 	return nil
@@ -816,7 +861,7 @@ func ObtainJWTSecret(cfg *httpcfg.HttpCfg, logger log.Logger) ([]byte, error) {
 	return jwtSecret, nil
 }
 
-func createHandler(cfg *httpcfg.HttpCfg, apiList []rpc.API, httpHandler http.Handler, wsHandler http.Handler, graphQLHandler http.Handler, jwtSecret []byte) (http.Handler, error) {
+func createHandler(cfg httpcfg.HttpCfg, apiList []rpc.API, httpHandler, wsHandler, graphQLHandler http.Handler, jwtSecret []byte) (http.Handler, error) {
 	var handler http.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if cfg.GraphQLEnabled && graphql.ProcessGraphQLcheckIfNeeded(graphQLHandler, w, r) {
 			return
@@ -861,7 +906,7 @@ func createEngineListener(cfg *httpcfg.HttpCfg, engineApi []rpc.API, logger log.
 
 	graphQLHandler := graphql.CreateHandler(engineApi)
 
-	engineApiHandler, err := createHandler(cfg, engineApi, engineHttpHandler, wsHandler, graphQLHandler, jwtSecret)
+	engineApiHandler, err := createHandler(*cfg, engineApi, engineHttpHandler, wsHandler, graphQLHandler, jwtSecret)
 	if err != nil {
 		return nil, nil, "", err
 	}

cmd/rpcdaemon/cli/httpcfg/http_cfg.go

@@ -48,6 +48,10 @@ type HttpCfg struct {
 	WebsocketEnabled                  bool
 	WebsocketCompression              bool
 	WebsocketSubscribeLogsChannelSize int
+	WebSocketListenAddress            string
+	WebSocketPort                     int
+	WebsocketCORSDomain               []string
+	WebSocketApi                      []string
 	RpcAllowListFilePath              string
 	RpcBatchConcurrency               uint
 	RpcStreamingDisable               bool

cmd/utils/flags.go

@@ -373,7 +373,7 @@ var (
 	HTTPCORSDomainFlag = cli.StringFlag{
 		Name:  "http.corsdomain",
 		Usage: "Comma separated list of domains from which to accept cross origin requests (browser enforced)",
-		Value: "",
+		Value: "*",
 	}
 	HTTPVirtualHostsFlag = cli.StringFlag{
 		Name:  "http.vhosts",
@@ -707,11 +707,6 @@ var (
 		Usage: "Disable the virtual counters. This has an effect on on sequencer node and when external executor is not enabled.",
 		Value: false,
 	}
-	SupportGasless = cli.BoolFlag{
-		Name:  "zkevm.gasless",
-		Usage: "Support gasless transactions",
-		Value: false,
-	}
 	ExecutorPayloadOutput = cli.StringFlag{
 		Name:  "zkevm.executor-payload-output",
 		Usage: "Output the payload of the executor, serialised requests stored to disk by batch number",

consensus/misc/eip1559_zk.go

@@ -9,7 +9,7 @@ import (
 )
 
 func CalcBaseFeeZk(config *chain.Config, parent *types.Header) *big.Int {
-	if config.SupportGasless || parent.Number.Cmp(big.NewInt(0)) == 0 {
+	if config.AllowFreeTransactions || parent.Number.Cmp(big.NewInt(0)) == 0 {
 		// If the parent is the genesis block, the next block will include the initial batch transaction, which is a legacy transaction, so the basefee will be set to 0
 		return big.NewInt(0)
 	}

core/vm/zk_batch_counters.go

@@ -88,6 +88,12 @@ func (bcc *BatchCounterCollector) AddNewTransactionCounters(txCounters *Transact
 	return bcc.CheckForOverflow(false) //no need to calculate the merkle proof here
 }
 
+func (bcc *BatchCounterCollector) RemovePreviousTransactionCounters() {
+	lastTx := bcc.transactions[len(bcc.transactions)-1]
+	bcc.UndoTransactionCountersCache(lastTx)
+	bcc.transactions = bcc.transactions[:len(bcc.transactions)-1]
+}
+
 func (bcc *BatchCounterCollector) ClearTransactionCounters() {
 	bcc.transactions = bcc.transactions[:0]
 }
@@ -154,7 +160,7 @@ func (bcc *BatchCounterCollector) CheckForOverflow(verifyMerkleProof bool) (bool
 		for _, v := range combined {
 			logText += fmt.Sprintf(" %s: initial: %v used: %v (remaining: %v)", v.name, v.initialAmount, v.used, v.remaining)
 		}
-		log.Info(logText)
+		log.Debug(logText)
 	}
 
 	return overflow, nil
@@ -275,3 +281,15 @@ func (bcc *BatchCounterCollector) UpdateExecutionAndProcessingCountersCache(txCo
 		bcc.processingCombinedCounters[k].used += v.used
 	}
 }
+
+func (bcc *BatchCounterCollector) UndoTransactionCountersCache(txCounters *TransactionCounter) {
+	for k, v := range txCounters.rlpCounters.counters {
+		bcc.rlpCombinedCounters[k].used -= v.used
+	}
+	for k, v := range txCounters.executionCounters.counters {
+		bcc.executionCombinedCounters[k].used -= v.used
+	}
+	for k, v := range txCounters.processingCounters.counters {
+		bcc.processingCombinedCounters[k].used -= v.used
+	}
+}

erigon-lib/chain/chain_config.go

@@ -117,7 +117,7 @@ type Config struct {
 	ForkID12BananaBlock     *big.Int `json:"forkID12BananaBlock,omitempty"`
 	NormalcyBlock           *big.Int `json:"normalcyBlock,omitempty"`
 
-	SupportGasless bool `json:"supportGasless,omitempty"`
+	AllowFreeTransactions bool `json:"allowFreeTransactions,omitempty"`
 }
 
 type BorConfig interface {

eth/backend.go

@@ -995,9 +995,7 @@ func New(ctx context.Context, stack *node.Node, config *ethconfig.Config, logger
 		// entering ZK territory!
 		cfg := backend.config
 
-		// update the chain config with the zero gas from the flags
-		backend.chainConfig.SupportGasless = cfg.Gasless
-
+		backend.chainConfig.AllowFreeTransactions = cfg.AllowFreeTransactions
 		l1Urls := strings.Split(cfg.L1RpcUrl, ",")
 
 		if cfg.Zk.L1CacheEnabled {

migrations/migrations.go

@@ -40,6 +40,7 @@ var migrations = map[kv.Label][]Migration{
 		refactorTableLastRoot,
 		ProhibitNewDownloadsLock2,
 		countersToArray,
+		resetL1Sequences,
 	},
 	kv.TxPoolDB: {},
 	kv.SentryDB: {},

migrations/reset_l1sequences.go

@@ -0,0 +1,34 @@
+package migrations
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/ledgerwatch/erigon-lib/common/datadir"
+	"github.com/ledgerwatch/erigon-lib/kv"
+	"github.com/ledgerwatch/erigon/eth/stagedsync/stages"
+	"github.com/ledgerwatch/log/v3"
+)
+
+var resetL1Sequences = Migration{
+	Name: "remove l1 sequences and stage_l1sync progress to download all l1 sequences anew",
+	Up: func(db kv.RwDB, dirs datadir.Dirs, progress []byte, BeforeCommit Callback, logger log.Logger) (err error) {
+		tx, err := db.BeginRw(context.Background())
+		if err != nil {
+			return err
+		}
+		defer tx.Rollback()
+		tx.ClearBucket(kv.L1SEQUENCES)
+
+		// already checked
+		if err := stages.SaveStageProgress(tx, stages.L1Syncer, 0); err != nil {
+			return fmt.Errorf("failed to get highest checked block, %w", err)
+		}
+
+		// This migration is no-op, but it forces the migration mechanism to apply it and thus write the DB schema version info
+		if err := BeforeCommit(tx, nil, true); err != nil {
+			return err
+		}
+		return tx.Commit()
+	},
+}