diff --git a/.github/workflows/hourly-env-checker.yml b/.github/workflows/hourly-env-checker.yml new file mode 100644 index 00000000000..e25edadbe71 --- /dev/null +++ b/.github/workflows/hourly-env-checker.yml @@ -0,0 +1,37 @@ +name: Hourly - Envs Checker + +on: + schedule: + - cron: '0 * * * *' # hourly + workflow_dispatch: + +jobs: + test-node: + runs-on: ubuntu-latest + continue-on-error: true + strategy: + matrix: + include: + - name: "Integration 5 - Legacy" + network: "integration-5" + rpc: "http://34.175.214.161:8505" + - name: "Integration 5 - Erigon RPC" + network: "integration-5" + rpc: "http://34.175.214.161:8500" + - name: "Integration 5 - Erigon Sequencer" + network: "integration-5" + rpc: "http://34.175.214.161:8005" + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: '1.20' + + - name: Build Go application + run: go build -o check_node + + - name: Run tests + run: ./check_node -rpcURL=${{ matrix.rpc }} -nodeName=${{ matrix.name }} \ No newline at end of file diff --git a/.github/workflows/nightly-ansible.yml b/.github/workflows/nightly-ansible.yml index 0b4e1841099..a74367a7988 100644 --- a/.github/workflows/nightly-ansible.yml +++ b/.github/workflows/nightly-ansible.yml @@ -8,6 +8,7 @@ on: jobs: build: runs-on: ubuntu-latest + continue-on-error: true strategy: matrix: include: diff --git a/.github/workflows/nightly-l1-recovery.yml b/.github/workflows/nightly-l1-recovery.yml index dc3e64d17b7..b521ad14373 100644 --- a/.github/workflows/nightly-l1-recovery.yml +++ b/.github/workflows/nightly-l1-recovery.yml @@ -23,4 +23,4 @@ jobs: - name: Run Docker Compose working-directory: ./zk/tests/nightly-l1-recovery - run: docker compose -f docker-compose-nightly.yml up --build --quiet --exit-code-from=block-checker + run: docker compose build --quiet && docker compose up --exit-code-from=block-checker diff --git a/.github/workflows/nightly-node-compare.yml b/.github/workflows/nightly-node-compare.yml index a9848fd466b..b0c45a21754 100644 --- a/.github/workflows/nightly-node-compare.yml +++ b/.github/workflows/nightly-node-compare.yml @@ -8,7 +8,7 @@ on: jobs: compare-blocks: runs-on: ubuntu-latest - + continue-on-error: true strategy: matrix: nodes: diff --git a/.github/workflows/nightly-rpc-batch-compare.yml b/.github/workflows/nightly-rpc-batch-compare.yml new file mode 100644 index 00000000000..e79cc817e01 --- /dev/null +++ b/.github/workflows/nightly-rpc-batch-compare.yml @@ -0,0 +1,33 @@ +name: Nightly - RPC Batch Compare + +on: + schedule: + - cron: '0 23 * * *' + workflow_dispatch: + + +jobs: + run-rpc-batch-compare: + runs-on: ubuntu-latest + continue-on-error: true + strategy: + matrix: + include: + - name: bali + legacy: "https://rpc.zkevm-internal.com" + erigon: "https://rpc.internal.zkevm-rpc.com" + - name: integration5 + legacy: "http://34.175.214.161:8505" + erigon: "http://34.175.214.161:8500" + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: '1.20' + + - name: Run RPC Batch Compare + run: go run ./zk/debug_tools/rpc-batch-compare -erigon="${{ matrix.erigon }}" -legacy="${{ matrix.legacy }}" -skip=100 -offset=1 diff --git a/zk/debug_tools/env-checker/main.go b/zk/debug_tools/env-checker/main.go new file mode 100644 index 00000000000..2c98363b83e --- /dev/null +++ b/zk/debug_tools/env-checker/main.go @@ -0,0 +1,120 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "log" + "net/http" + "flag" + "os" + "io" +) + +type JSONRPCRequest struct { + Jsonrpc string `json:"jsonrpc"` + Method string `json:"method"` + Params []interface{} `json:"params"` + ID int `json:"id"` +} + +type JSONRPCResponse struct { + Jsonrpc string `json:"jsonrpc"` + ID int `json:"id"` + Result interface{} `json:"result"` +} + +func main() { + rpcURL := flag.String("rpcURL", "", "RPC URL") + nodeName := flag.String("nodeName", "", "Node name") + flag.Parse() + + if *rpcURL == "" { + log.Fatal("rpcURL flag not set") + } + if *nodeName == "" { + log.Fatal("nodeName flag not set") + } + + fmt.Printf("Checking node: %s\n", *nodeName) + err := checkClientVersion(*rpcURL) + if err != nil { + fmt.Printf("Node is down: %s\n", *nodeName) + os.Exit(1) + } + err = checkBlockHeight(*rpcURL) + if err != nil { + fmt.Printf("Node is down: %s\n", *nodeName) + os.Exit(1) + } + err = checkBatchNumber(*rpcURL) + if err != nil { + fmt.Printf("Node is down: %s\n", *nodeName) + os.Exit(1) + } + + fmt.Printf("Node is up: %s\n", *nodeName) + os.Exit(0) +} + +func makeRequest(rpcURL, method string) (*JSONRPCResponse, error) { + reqBody := JSONRPCRequest{ + Jsonrpc: "2.0", + Method: method, + Params: []interface{}{}, + ID: 1, + } + + reqBytes, _ := json.Marshal(reqBody) + resp, err := http.Post(rpcURL, "application/json", bytes.NewBuffer(reqBytes)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + var rpcResp JSONRPCResponse + err = json.Unmarshal(body, &rpcResp) + if err != nil { + return nil, err + } + + if rpcResp.Result == nil { + return &rpcResp, fmt.Errorf("%s method returned an error: %v", method, rpcResp) + } + + return &rpcResp, nil +} + +func checkClientVersion(rpcURL string) error { + resp, err := makeRequest(rpcURL, "web3_clientVersion") + if err != nil { + fmt.Println("Client version retrieval failed") + return err + } + fmt.Printf("Client version: %v\n", resp.Result) + return nil +} + +func checkBlockHeight(rpcURL string) error { + resp, err := makeRequest(rpcURL, "eth_blockNumber") + if err != nil { + fmt.Println("Block height retrieval failed") + } + fmt.Printf("Block height: %v\n", resp.Result) + return nil +} + +func checkBatchNumber(rpcURL string) error { + resp, err := makeRequest(rpcURL, "zkevm_batchNumber") + if err != nil { + fmt.Println("Batch number retrieval failed") + return err + } + fmt.Printf("Batch number: %v\n", resp.Result) + return nil +} diff --git a/zk/debug_tools/rpc-batch-compare/main.go b/zk/debug_tools/rpc-batch-compare/main.go new file mode 100644 index 00000000000..b5524d26ac1 --- /dev/null +++ b/zk/debug_tools/rpc-batch-compare/main.go @@ -0,0 +1,180 @@ +package main + +import ( + "bytes" + "encoding/json" + "flag" + "fmt" + "log" + "math/big" + "net/http" + "os" + + "github.com/google/go-cmp/cmp" + "io" +) + +func getBatchNumber(url string) (*big.Int, error) { + requestBody, _ := json.Marshal(map[string]interface{}{ + "jsonrpc": "2.0", + "method": "zkevm_batchNumber", + "params": []interface{}{}, + "id": 1, + }) + + resp, err := http.Post(url, "application/json", bytes.NewBuffer(requestBody)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response body: %v", err) + } + + var result map[string]interface{} + if err := json.Unmarshal(body, &result); err != nil { + return nil, fmt.Errorf("failed to unmarshal response: %v", err) + } + + if errorField, ok := result["error"]; ok { + return nil, fmt.Errorf("node error: %v", errorField) + } + + batchNumberHex, ok := result["result"].(string) + if !ok { + return nil, fmt.Errorf("invalid response format") + } + + batchNumber := new(big.Int) + if _, ok := batchNumber.SetString(batchNumberHex[2:], 16); !ok { + return nil, fmt.Errorf("failed to convert batch number to big.Int") + } + return batchNumber, nil +} + +func getBatchByNumber(url string, number *big.Int) (map[string]interface{}, error) { + requestBody, _ := json.Marshal(map[string]interface{}{ + "jsonrpc": "2.0", + "method": "zkevm_getBatchByNumber", + "params": []interface{}{number.String(), true}, + "id": 1, + }) + + resp, err := http.Post(url, "application/json", bytes.NewBuffer(requestBody)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response body: %v", err) + } + + var result map[string]interface{} + if err := json.Unmarshal(body, &result); err != nil { + return nil, fmt.Errorf("failed to unmarshal response: %v", err) + } + + if errorField, ok := result["error"]; ok { + return nil, fmt.Errorf("node error: %v", errorField) + } + + batchData, ok := result["result"].(map[string]interface{}) + if !ok || batchData == nil { + return nil, fmt.Errorf("batch not found") + } + + return batchData, nil +} + +func compareBatches(erigonURL, legacyURL string, batchNumber *big.Int) (string, error) { + batch1, err := getBatchByNumber(erigonURL, batchNumber) + if err != nil { + return "", fmt.Errorf("Error getting batch %d from Erigon node: %v", batchNumber, err) + } + + batch2, err := getBatchByNumber(legacyURL, batchNumber) + if err != nil { + return "", fmt.Errorf("Error getting batch %d from Legacy node: %v", batchNumber, err) + } + + // ignore list + il := []string{ + "timestamp", + "verifyBatchTxHash", + "sendSequencesTxHash", + "accInputHash", + "globalExitRoot", + "mainnetExitRoot", + "rollupExitRoot", + } + for _, i := range il { + delete(batch1, i) + delete(batch2, i) + } + + if !cmp.Equal(batch1, batch2) { + return fmt.Sprintf("Mismatch at batch %d:\nErigon vs Legacy:\n%s", + batchNumber, cmp.Diff(batch1, batch2)), nil + } + return "", nil +} + +func main() { + erigonURL := flag.String("erigon", "http://localhost:8545", "RPC URL of the Erigon node") + legacyURL := flag.String("legacy", "http://localhost:8546", "RPC URL of the Legacy node") + skip := flag.Int("skip", 1, "Number of batches to skip between each check") + numBatches := flag.Int("batches", 1000, "Number of batches to check") + startOffset := flag.Int("offset", 0, "Offset from highest getBatchNumber") + overrideStartAt := flag.Int("override", 0, "Override start batch number") + flag.Parse() + + erigonLatestBatch, err := getBatchNumber(*erigonURL) + if err != nil { + log.Fatalf("Failed to get latest batch number from Erigon node: %v", err) + } + log.Println("Erigon latest batch number: ", erigonLatestBatch) + + legacyLatestBatch, err := getBatchNumber(*legacyURL) + if err != nil { + log.Fatalf("Failed to get latest batch number from Legacy node: %v", err) + } + log.Println("Legacy latest batch number: ", legacyLatestBatch) + + startBatch := legacyLatestBatch + if erigonLatestBatch.Cmp(startBatch) < 0 { + startBatch = erigonLatestBatch + } + + // offset start batch + startBatch = new(big.Int).Sub(startBatch, big.NewInt(int64(*startOffset))) + + if *overrideStartAt != 0 { + startBatch = big.NewInt(int64(*overrideStartAt)) + log.Println("Overriding start batch to", startBatch) + } + + log.Printf("Checking %d batches\n", *numBatches) + log.Printf("Starting from batch %d\n", startBatch) + log.Printf("Skipping %d batches\n", *skip) + + for i := 0; i < *numBatches; i++ { + batchNumber := new(big.Int).Sub(startBatch, big.NewInt(int64(i**skip))) + diff, err := compareBatches(*erigonURL, *legacyURL, batchNumber) + if err != nil { + log.Println(err) + os.Exit(1) + } + if diff != "" { + log.Println(diff) + os.Exit(1) + } + log.Println("Batch", batchNumber, "matches") + } + + log.Println("No differences found") + os.Exit(0) +} diff --git a/zk/tests/nightly-l1-recovery/docker-compose-nightly.yml b/zk/tests/nightly-l1-recovery/docker-compose.yml similarity index 100% rename from zk/tests/nightly-l1-recovery/docker-compose-nightly.yml rename to zk/tests/nightly-l1-recovery/docker-compose.yml