Merge remote-tracking branch 'origin/zkevm' into zkevm-2.60

.github/workflows/doc-rpc.yml

@@ -0,0 +1,26 @@
+name: RPC endpoint doc
+on:
+  push:
+    branches:
+      - zkevm
+  pull_request:
+    branches:
+      - zkevm
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - ready_for_review
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v4
+        with:
+          go-version: '1.19'
+      - name: Check RPC endpoints doc
+        run: |
+          cd ./docs/endpoints
+          make check-doc

README.md

@@ -133,9 +133,6 @@ Initial SMT build performance can be increased if machine has enough RAM:
 
 ## Configuration Files
 Config files are the easiest way to configure cdk-erigon, there are examples in the repository for each network e.g. `hermezconfig-mainnet.yaml.example`.
-
-Depending on the RPC provider you are using, you may wish to alter `zkevm.rpc-ratelimit`.
-
 ***
 
 ## Running CDK-Erigon
@@ -188,11 +185,11 @@ For a full explanation of the config options, see below:
 - `zkevm.l2-datastreamer-url`: URL for the L2 data streamer.
 - `zkevm.l1-chain-id`: Chain ID for the L1 network.
 - `zkevm.l1-rpc-url`: L1 Ethereum RPC URL.
+- `zkevm.l1-first-block`: The first block on L1 from which we begin syncing (where the rollup begins on the L1). NB: for AggLayer networks this must be the L1 block where the GER Manager contract was deployed.
 - `zkevm.address-sequencer`: The contract address for the sequencer
 - `zkevm.address-zkevm`: The address for the zkevm contract
 - `zkevm.address-rollup`: The address for the rollup contract
 - `zkevm.address-ger-manager`: The address for the GER manager contract
-- `zkevm.rpc-ratelimit`: Rate limit for RPC calls.
 - `zkevm.data-stream-port`: Port for the data stream.  This needs to be set to enable the datastream server
 - `zkevm.data-stream-host`: The host for the data stream i.e. `localhost`.  This must be set to enable the datastream server
 - `zkevm.datastream-version:` Version of the data stream protocol.

cmd/cdk-erigon/main.go

@@ -76,56 +76,108 @@ func runErigon(cliCtx *cli.Context) error {
 }
 
 func setFlagsFromConfigFile(ctx *cli.Context, filePath string) error {
-	fileExtension := filepath.Ext(filePath)
-
-	fileConfig := make(map[string]interface{})
+	cfg, err := fileConfig(filePath)
+	if err != nil {
+		return err
+	}
 
-	if fileExtension == ".yaml" {
-		yamlFile, err := os.ReadFile(filePath)
-		if err != nil {
-			return err
-		}
-		err = yaml.Unmarshal(yamlFile, fileConfig)
-		if err != nil {
-			return err
-		}
-	} else if fileExtension == ".toml" {
-		tomlFile, err := os.ReadFile(filePath)
-		if err != nil {
-			return err
+	for key, value := range cfg {
+		if ctx.IsSet(key) {
+			continue
 		}
-		err = toml.Unmarshal(tomlFile, &fileConfig)
-		if err != nil {
+
+		if err := setFlag(ctx, key, value); err != nil {
 			return err
 		}
-	} else {
-		return errors.New("config files only accepted are .yaml and .toml")
-	}
-	// sets global flags to value in yaml/toml file
-	for key, value := range fileConfig {
-		if !ctx.IsSet(key) {
-			if reflect.ValueOf(value).Kind() == reflect.Slice {
-				sliceInterface := value.([]interface{})
-				s := make([]string, len(sliceInterface))
-				for i, v := range sliceInterface {
-					s[i] = fmt.Sprintf("%v", v)
-				}
-				if err := ctx.Set(key, strings.Join(s, ",")); err != nil {
-					if deprecatedFlag, found := erigoncli.DeprecatedFlags[key]; found {
-						return fmt.Errorf("failed setting %s flag Flag is deprecated, use %s instead", key, deprecatedFlag)
-					}
-					return fmt.Errorf("failed setting %s flag with values=%s error=%s", key, s, err)
-				}
-			} else {
-				if err := ctx.Set(key, fmt.Sprintf("%v", value)); err != nil {
-					if deprecatedFlag, found := erigoncli.DeprecatedFlags[key]; found {
-						return fmt.Errorf("failed setting %s flag Flag is deprecated, use %s instead", key, deprecatedFlag)
-					}
-					return fmt.Errorf("failed setting %s flag with value=%v error=%s", key, value, err)
-				}
-			}
-		}
 	}
 
 	return nil
 }
+
+func fileConfig(filePath string) (map[string]interface{}, error) {
+	fileExtension := filepath.Ext(filePath)
+	switch fileExtension {
+	case ".yaml":
+		return yamlConfig(filePath)
+	case ".toml":
+		return tomlConfig(filePath)
+	default:
+		return nil, errors.New("config files only accepted are .yaml and .toml")
+	}
+}
+
+func yamlConfig(filePath string) (map[string]interface{}, error) {
+	cfg := make(map[string]interface{})
+	yamlFile, err := os.ReadFile(filePath)
+	if err != nil {
+		return nil, err
+	}
+
+	err = yaml.Unmarshal(yamlFile, &cfg)
+	if err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+func tomlConfig(filePath string) (map[string]interface{}, error) {
+	cfg := make(map[string]interface{})
+
+	tomlFile, err := os.ReadFile(filePath)
+	if err != nil {
+		return nil, err
+	}
+
+	err = toml.Unmarshal(tomlFile, &cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	return cfg, nil
+}
+
+func setFlag(ctx *cli.Context, key string, value interface{}) error {
+	isSlice := reflect.ValueOf(value).Kind() == reflect.Slice
+	if isSlice {
+		return setMultiValueFlag(ctx, key, value)
+	}
+	return setSingleValueFlag(ctx, key, value)
+}
+
+func setMultiValueFlag(ctx *cli.Context, key string, value interface{}) error {
+	sliceInterface := value.([]interface{})
+	slice := make([]string, len(sliceInterface))
+	for i, v := range sliceInterface {
+		slice[i] = fmt.Sprintf("%v", v)
+	}
+
+	return setFlagInContext(ctx, key, strings.Join(slice, ","))
+}
+
+func setSingleValueFlag(ctx *cli.Context, key string, value interface{}) error {
+	return setFlagInContext(ctx, key, fmt.Sprintf("%v", value))
+}
+
+func setFlagInContext(ctx *cli.Context, key, value string) error {
+	if err := ctx.Set(key, value); err != nil {
+		return handleFlagError(key, value, err)
+	}
+	return nil
+}
+
+func handleFlagError(key, value string, err error) error {
+	if deprecatedFlag, found := erigoncli.DeprecatedFlags[key]; found {
+		if deprecatedFlag == "" {
+			return fmt.Errorf("failed setting %s flag: it is deprecated, remove it", key)
+		}
+		return fmt.Errorf("failed setting %s flag: it is deprecated, use %s instead", key, deprecatedFlag)
+	}
+
+	errUnknownFlag := fmt.Errorf("no such flag -%s", key)
+	if err.Error() == errUnknownFlag.Error() {
+		log.Warn("🚨 failed setting flag: unknown flag provided", "key", key, "value", value)
+		return nil
+	}
+
+	return fmt.Errorf("failed setting %s flag with value=%s, error=%s", key, value, err)
+}

cmd/utils/flags.go

@@ -506,6 +506,11 @@ var (
 		Usage: "Check the contract address on the L1",
 		Value: true,
 	}
+	L1ContractAddressRetrieveFlag = cli.BoolFlag{
+		Name:  "zkevm.l1-contract-address-retrieve",
+		Usage: "Retrieve the contracts addresses from the L1",
+		Value: true,
+	}
 	RebuildTreeAfterFlag = cli.Uint64Flag{
 		Name:  "zkevm.rebuild-tree-after",
 		Usage: "Rebuild the state tree after this many blocks behind",
@@ -533,9 +538,14 @@ var (
 	}
 	SequencerBatchVerificationTimeout = cli.StringFlag{
 		Name:  "zkevm.sequencer-batch-verification-timeout",
-		Usage: "This is a maximum time that a batch verification could take. Including retries. This could be interpreted as maximum that that the sequencer can run without executor. Setting it to 0s will mean infinite timeout. Defaults to 30min",
+		Usage: "This is a maximum time that a batch verification could take in terms of executors' errors. Including retries. This could be interpreted as `maximum that that the sequencer can run without executor`. Setting it to 0s will mean infinite timeout. Defaults to 30min",
 		Value: "30m",
 	}
+	SequencerBatchVerificationRetries = cli.StringFlag{
+		Name:  "zkevm.sequencer-batch-verification-retries",
+		Usage: "Number of attempts that a batch will re-run in case of an internal (not executors') error. This could be interpreted as `maximum attempts to send a batch for verification`. Setting it to -1 will mean unlimited attempts. Defaults to 3",
+		Value: "3",
+	}
 	SequencerTimeoutOnEmptyTxPool = cli.StringFlag{
 		Name:  "zkevm.sequencer-timeout-on-empty-tx-pool",
 		Usage: "Timeout before requesting txs from the txpool if none were found before. Defaults to 250ms",

docs/endpoints/Makefile

@@ -0,0 +1,13 @@
+DOC_NAME:="endpoints.md"
+
+.PHONY: gen-doc
+gen-doc: 
+	go run main.go $(DOC_NAME)
+
+.PHONY: check-doc
+check-doc: 
+	go run main.go tmp$(DOC_NAME)
+	cmp -s ./$(DOC_NAME) ./tmp$(DOC_NAME); \
+	RETVAL=$$?; \
+	rm ./tmp$(DOC_NAME); \
+	exit $$RETVAL

docs/endpoints/README.md

@@ -0,0 +1,19 @@
+# JSON RPC Endpoints
+
+This tool is used to generate the list of supported endpoints provided by the JSON-RPC server as a markdown document.
+
+It uses reflection to go through all API interfaces and then merge the information with the content of the [template.md](./template.md) as the base to generate the [endpoints.md](./endpoints.md) file.
+
+To generate the file ensure you have `make` and `go` installed, then run available on this directory:
+
+```bash
+make gen-doc
+```
+
+The [endpoints.md](./endpoints.md) must always be generated when a new endpoint is added, removed or changed so we can have this file pushed to the repo for further use.
+
+There is also a command to check if the current file is compatible with the current code. This command is meant to be used in the CI do ensure the doc is updated.
+
+```bash
+make check-doc
+```