Tests - Env checker, batch compare (#729)

* test(hourly): add hourly envs check

* test(nightly): batch checks bali/int5

* test(nightly): l1 recovery tweak

* test(nightly): tweak to continue on error

.github/workflows/hourly-env-checker.yml

@@ -0,0 +1,37 @@
+name: Hourly - Envs Checker
+
+on:
+  schedule:
+    - cron: '0 * * * *' # hourly
+  workflow_dispatch:
+
+jobs:
+  test-node:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    strategy:
+      matrix:
+        include:
+          - name: "Integration 5 - Legacy"
+            network: "integration-5"
+            rpc: "http://34.175.214.161:8505"
+          - name: "Integration 5 - Erigon RPC"
+            network: "integration-5"
+            rpc: "http://34.175.214.161:8500"
+          - name: "Integration 5 - Erigon Sequencer"
+            network: "integration-5"
+            rpc: "http://34.175.214.161:8005"
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: '1.20'
+
+      - name: Build Go application
+        run: go build -o check_node
+
+      - name: Run tests
+        run: ./check_node -rpcURL=${{ matrix.rpc }} -nodeName=${{ matrix.name }}

.github/workflows/nightly-ansible.yml

@@ -8,6 +8,7 @@ on:
 jobs:
   build:
     runs-on: ubuntu-latest
+    continue-on-error: true
     strategy:
       matrix:
         include:

.github/workflows/nightly-l1-recovery.yml

@@ -23,4 +23,4 @@ jobs:
 
       - name: Run Docker Compose
         working-directory: ./zk/tests/nightly-l1-recovery
-        run: docker compose -f docker-compose-nightly.yml up --build --quiet --exit-code-from=block-checker
+        run: docker compose build --quiet && docker compose up --exit-code-from=block-checker

.github/workflows/nightly-node-compare.yml

@@ -8,7 +8,7 @@ on:
 jobs:
   compare-blocks:
     runs-on: ubuntu-latest
-
+    continue-on-error: true
     strategy:
       matrix:
         nodes:

.github/workflows/nightly-rpc-batch-compare.yml

@@ -0,0 +1,33 @@
+name: Nightly - RPC Batch Compare
+
+on:
+  schedule:
+    - cron: '0 23 * * *'
+  workflow_dispatch:
+
+
+jobs:
+  run-rpc-batch-compare:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    strategy:
+      matrix:
+        include:
+          - name: bali
+            legacy: "https://rpc.zkevm-internal.com"
+            erigon: "https://rpc.internal.zkevm-rpc.com"
+          - name: integration5
+            legacy: "http://34.175.214.161:8505"
+            erigon: "http://34.175.214.161:8500"
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: '1.20'
+
+      - name: Run RPC Batch Compare
+        run: go run ./zk/debug_tools/rpc-batch-compare -erigon="${{ matrix.erigon }}" -legacy="${{ matrix.legacy }}" -skip=100 -offset=1

zk/debug_tools/env-checker/main.go

@@ -0,0 +1,120 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"log"
+	"net/http"
+	"flag"
+	"os"
+	"io"
+)
+
+type JSONRPCRequest struct {
+	Jsonrpc string        `json:"jsonrpc"`
+	Method  string        `json:"method"`
+	Params  []interface{} `json:"params"`
+	ID      int           `json:"id"`
+}
+
+type JSONRPCResponse struct {
+	Jsonrpc string      `json:"jsonrpc"`
+	ID      int         `json:"id"`
+	Result  interface{} `json:"result"`
+}
+
+func main() {
+	rpcURL := flag.String("rpcURL", "", "RPC URL")
+	nodeName := flag.String("nodeName", "", "Node name")
+	flag.Parse()
+
+	if *rpcURL == "" {
+		log.Fatal("rpcURL flag not set")
+	}
+	if *nodeName == "" {
+		log.Fatal("nodeName flag not set")
+	}
+
+	fmt.Printf("Checking node: %s\n", *nodeName)
+	err := checkClientVersion(*rpcURL)
+	if err != nil {
+		fmt.Printf("Node is down: %s\n", *nodeName)
+		os.Exit(1)
+	}
+	err = checkBlockHeight(*rpcURL)
+	if err != nil {
+		fmt.Printf("Node is down: %s\n", *nodeName)
+		os.Exit(1)
+	}
+	err = checkBatchNumber(*rpcURL)
+	if err != nil {
+		fmt.Printf("Node is down: %s\n", *nodeName)
+		os.Exit(1)
+	}
+
+	fmt.Printf("Node is up: %s\n", *nodeName)
+	os.Exit(0)
+}
+
+func makeRequest(rpcURL, method string) (*JSONRPCResponse, error) {
+	reqBody := JSONRPCRequest{
+		Jsonrpc: "2.0",
+		Method:  method,
+		Params:  []interface{}{},
+		ID:      1,
+	}
+
+	reqBytes, _ := json.Marshal(reqBody)
+	resp, err := http.Post(rpcURL, "application/json", bytes.NewBuffer(reqBytes))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	var rpcResp JSONRPCResponse
+	err = json.Unmarshal(body, &rpcResp)
+	if err != nil {
+		return nil, err
+	}
+
+	if rpcResp.Result == nil {
+		return &rpcResp, fmt.Errorf("%s method returned an error: %v", method, rpcResp)
+	}
+
+	return &rpcResp, nil
+}
+
+func checkClientVersion(rpcURL string) error {
+	resp, err := makeRequest(rpcURL, "web3_clientVersion")
+	if err != nil {
+		fmt.Println("Client version retrieval failed")
+		return err
+	}
+	fmt.Printf("Client version: %v\n", resp.Result)
+	return nil
+}
+
+func checkBlockHeight(rpcURL string) error {
+	resp, err := makeRequest(rpcURL, "eth_blockNumber")
+	if err != nil {
+		fmt.Println("Block height retrieval failed")
+	}
+	fmt.Printf("Block height: %v\n", resp.Result)
+	return nil
+}
+
+func checkBatchNumber(rpcURL string) error {
+	resp, err := makeRequest(rpcURL, "zkevm_batchNumber")
+	if err != nil {
+		fmt.Println("Batch number retrieval failed")
+		return err
+	}
+	fmt.Printf("Batch number: %v\n", resp.Result)
+	return nil
+}

zk/debug_tools/rpc-batch-compare/main.go

@@ -0,0 +1,180 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"log"
+	"math/big"
+	"net/http"
+	"os"
+
+	"github.com/google/go-cmp/cmp"
+	"io"
+)
+
+func getBatchNumber(url string) (*big.Int, error) {
+	requestBody, _ := json.Marshal(map[string]interface{}{
+		"jsonrpc": "2.0",
+		"method":  "zkevm_batchNumber",
+		"params":  []interface{}{},
+		"id":      1,
+	})
+
+	resp, err := http.Post(url, "application/json", bytes.NewBuffer(requestBody))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read response body: %v", err)
+	}
+
+	var result map[string]interface{}
+	if err := json.Unmarshal(body, &result); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal response: %v", err)
+	}
+
+	if errorField, ok := result["error"]; ok {
+		return nil, fmt.Errorf("node error: %v", errorField)
+	}
+
+	batchNumberHex, ok := result["result"].(string)
+	if !ok {
+		return nil, fmt.Errorf("invalid response format")
+	}
+
+	batchNumber := new(big.Int)
+	if _, ok := batchNumber.SetString(batchNumberHex[2:], 16); !ok {
+		return nil, fmt.Errorf("failed to convert batch number to big.Int")
+	}
+	return batchNumber, nil
+}
+
+func getBatchByNumber(url string, number *big.Int) (map[string]interface{}, error) {
+	requestBody, _ := json.Marshal(map[string]interface{}{
+		"jsonrpc": "2.0",
+		"method":  "zkevm_getBatchByNumber",
+		"params":  []interface{}{number.String(), true},
+		"id":      1,
+	})
+
+	resp, err := http.Post(url, "application/json", bytes.NewBuffer(requestBody))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read response body: %v", err)
+	}
+
+	var result map[string]interface{}
+	if err := json.Unmarshal(body, &result); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal response: %v", err)
+	}
+
+	if errorField, ok := result["error"]; ok {
+		return nil, fmt.Errorf("node error: %v", errorField)
+	}
+
+	batchData, ok := result["result"].(map[string]interface{})
+	if !ok || batchData == nil {
+		return nil, fmt.Errorf("batch not found")
+	}
+
+	return batchData, nil
+}
+
+func compareBatches(erigonURL, legacyURL string, batchNumber *big.Int) (string, error) {
+	batch1, err := getBatchByNumber(erigonURL, batchNumber)
+	if err != nil {
+		return "", fmt.Errorf("Error getting batch %d from Erigon node: %v", batchNumber, err)
+	}
+
+	batch2, err := getBatchByNumber(legacyURL, batchNumber)
+	if err != nil {
+		return "", fmt.Errorf("Error getting batch %d from Legacy node: %v", batchNumber, err)
+	}
+
+	// ignore list
+	il := []string{
+		"timestamp",
+		"verifyBatchTxHash",
+		"sendSequencesTxHash",
+		"accInputHash",
+		"globalExitRoot",
+		"mainnetExitRoot",
+		"rollupExitRoot",
+	}
+	for _, i := range il {
+		delete(batch1, i)
+		delete(batch2, i)
+	}
+
+	if !cmp.Equal(batch1, batch2) {
+		return fmt.Sprintf("Mismatch at batch %d:\nErigon vs Legacy:\n%s",
+			batchNumber, cmp.Diff(batch1, batch2)), nil
+	}
+	return "", nil
+}
+
+func main() {
+	erigonURL := flag.String("erigon", "http://localhost:8545", "RPC URL of the Erigon node")
+	legacyURL := flag.String("legacy", "http://localhost:8546", "RPC URL of the Legacy node")
+	skip := flag.Int("skip", 1, "Number of batches to skip between each check")
+	numBatches := flag.Int("batches", 1000, "Number of batches to check")
+	startOffset := flag.Int("offset", 0, "Offset from highest getBatchNumber")
+	overrideStartAt := flag.Int("override", 0, "Override start batch number")
+	flag.Parse()
+
+	erigonLatestBatch, err := getBatchNumber(*erigonURL)
+	if err != nil {
+		log.Fatalf("Failed to get latest batch number from Erigon node: %v", err)
+	}
+	log.Println("Erigon latest batch number: ", erigonLatestBatch)
+
+	legacyLatestBatch, err := getBatchNumber(*legacyURL)
+	if err != nil {
+		log.Fatalf("Failed to get latest batch number from Legacy node: %v", err)
+	}
+	log.Println("Legacy latest batch number: ", legacyLatestBatch)
+
+	startBatch := legacyLatestBatch
+	if erigonLatestBatch.Cmp(startBatch) < 0 {
+		startBatch = erigonLatestBatch
+	}
+
+	// offset start batch
+	startBatch = new(big.Int).Sub(startBatch, big.NewInt(int64(*startOffset)))
+
+	if *overrideStartAt != 0 {
+		startBatch = big.NewInt(int64(*overrideStartAt))
+		log.Println("Overriding start batch to", startBatch)
+	}
+
+	log.Printf("Checking %d batches\n", *numBatches)
+	log.Printf("Starting from batch %d\n", startBatch)
+	log.Printf("Skipping %d batches\n", *skip)
+
+	for i := 0; i < *numBatches; i++ {
+		batchNumber := new(big.Int).Sub(startBatch, big.NewInt(int64(i**skip)))
+		diff, err := compareBatches(*erigonURL, *legacyURL, batchNumber)
+		if err != nil {
+			log.Println(err)
+			os.Exit(1)
+		}
+		if diff != "" {
+			log.Println(diff)
+			os.Exit(1)
+		}
+		log.Println("Batch", batchNumber, "matches")
+	}
+
+	log.Println("No differences found")
+	os.Exit(0)
+}