From b6c1e2b99eaca5886edeada12987122e397f7ccd Mon Sep 17 00:00:00 2001 From: g11tech Date: Tue, 2 Aug 2022 18:31:54 +0530 Subject: [PATCH] Optimize merge tracker and add merge metrics (#4350) * Optimize merge tracker and add merge metrics * early return if genesis is merge block * add metric for latest block time * grafana dashboard * cleanup merge update * expanded/compressed merge logging * fix type check in tests * improve logging * expanded merge at 12hrs * Review PR * Fix tests * Update prettyTimeDiffSec test * merge time and dashboard fixes * fix the comment * change timeseries input to seconds from ms * merge found fix * change last block timestamp collection to seconds Co-authored-by: dapplion <35266934+dapplion@users.noreply.github.com> --- dashboards/lodestar_general.json | 959 +++++++++++++++++- packages/beacon-node/src/chain/chain.ts | 13 +- .../src/chain/factory/block/body.ts | 10 +- .../src/eth1/eth1MergeBlockTracker.ts | 467 +++++---- packages/beacon-node/src/eth1/index.ts | 49 +- packages/beacon-node/src/eth1/interface.ts | 36 +- .../src/metrics/metrics/lodestar.ts | 48 + packages/beacon-node/src/node/nodejs.ts | 12 +- packages/beacon-node/src/node/notifier.ts | 80 +- packages/beacon-node/src/util/enum.ts | 17 + packages/beacon-node/src/util/time.ts | 3 +- packages/beacon-node/src/util/timeSeries.ts | 19 +- .../e2e/eth1/eth1ForBlockProduction.test.ts | 2 - .../e2e/eth1/eth1MergeBlockTracker.test.ts | 21 +- .../test/spec/presets/fork_choice.ts | 16 +- .../unit/eth1/eth1MergeBlockTracker.test.ts | 132 ++- .../beacon-node/test/unit/util/time.test.ts | 28 +- .../test/unit/util/timeSeries.test.ts | 8 +- 18 files changed, 1491 insertions(+), 429 deletions(-) create mode 100644 packages/beacon-node/src/util/enum.ts diff --git a/dashboards/lodestar_general.json b/dashboards/lodestar_general.json index 93ba7cd76480..2841eaffcb49 100644 --- a/dashboards/lodestar_general.json +++ b/dashboards/lodestar_general.json @@ -4846,13 +4846,930 @@ "title": "Eth1 Stats", "type": "row" }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 437, + "panels": [ + { + "datasource": null, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "0": { + "index": 0, + "text": "STOPPED" + }, + "1": { + "index": 1, + "text": "SEARCHING" + }, + "2": { + "index": 2, + "text": "FOUND" + }, + "3": { + "index": 3, + "text": "MERGE_COMPLETE" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 439, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "value" + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "exemplar": false, + "expr": "lodestar_eth1_merge_status", + "hide": false, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Merge Status", + "type": "stat" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 459, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "exemplar": false, + "expr": "lodestar_eth1_merge_ttd", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Target TTD", + "type": "stat" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 1 + }, + "id": 461, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "exemplar": false, + "expr": "lodestar_eth1_latest_block_number", + "hide": false, + "interval": "", + "legendFormat": "latest block ttd", + "refId": "A" + } + ], + "title": "Latest block num", + "type": "stat" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 1 + }, + "id": 462, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "value" + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "exemplar": false, + "expr": "lodestar_eth1_parent_blocks_fetched_total", + "hide": false, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Parents fetched", + "type": "stat" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 1 + }, + "id": 456, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "name" + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "exemplar": false, + "expr": "lodestar_eth1_merge_block_details", + "format": "time_series", + "instant": true, + "interval": "", + "legendFormat": "{{terminalBlockNumber}}", + "refId": "A" + } + ], + "title": "Terminal Block Num", + "type": "stat" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 1 + }, + "id": 464, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "name" + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "exemplar": false, + "expr": "lodestar_eth1_merge_block_details", + "format": "time_series", + "instant": true, + "interval": "", + "legendFormat": "{{terminalBlockNumber}}", + "refId": "A" + } + ], + "title": "Terminal Block Time", + "type": "stat" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 5 + }, + "id": 460, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "exemplar": false, + "expr": "lodestar_eth1_merge_td_factor", + "hide": false, + "interval": "", + "legendFormat": "td factor", + "refId": "A" + } + ], + "title": "TD factor", + "type": "stat" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 5 + }, + "id": 450, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "exemplar": false, + "expr": "lodestar_eth1_latest_block_ttd", + "hide": false, + "interval": "", + "legendFormat": "latest block ttd", + "refId": "A" + } + ], + "title": "Latest block ttd", + "type": "stat" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "dateTimeFromNow" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 5 + }, + "id": 463, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "exemplar": false, + "expr": "lodestar_eth1_latest_block_timestamp * 1000", + "hide": false, + "interval": "", + "legendFormat": "latest block ttd", + "refId": "A" + } + ], + "title": "Latest block time", + "type": "stat" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 5 + }, + "id": 467, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "exemplar": false, + "expr": "lodestar_eth1_get_terminal_pow_block_promise_cache_hit_total", + "hide": false, + "interval": "", + "legendFormat": "cache hits", + "refId": "A" + } + ], + "title": "Promise cache hits", + "type": "stat" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 8, + "x": 16, + "y": 5 + }, + "id": 455, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "name" + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "exemplar": false, + "expr": "lodestar_eth1_merge_block_details", + "format": "time_series", + "instant": true, + "interval": "", + "legendFormat": "{{terminalBlockHash}}", + "refId": "A" + } + ], + "title": "Terminal Block Hash", + "type": "stat" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 8, + "x": 16, + "y": 7 + }, + "id": 466, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": false, + "expr": "", + "format": "time_series", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": " ", + "type": "timeseries" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 444, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.2.2", + "targets": [ + { + "exemplar": false, + "expr": "rate(lodestar_eth1_poll_merge_block_errors_total[$__rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "polling errors", + "refId": "A" + } + ], + "title": "Error rate", + "type": "timeseries" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 458, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.2.2", + "targets": [ + { + "exemplar": false, + "expr": "lodestar_eth1_latest_block_ttd", + "hide": false, + "interval": "", + "legendFormat": "lastest block td", + "refId": "A" + }, + { + "exemplar": false, + "expr": "lodestar_eth1_merge_ttd", + "hide": false, + "interval": "", + "legendFormat": "merge ttd", + "refId": "B" + } + ], + "title": "Latest TD vs TTD", + "type": "timeseries" + } + ], + "title": "Merge Metrics", + "type": "row" + }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 21 + "y": 22 }, "id": 208, "panels": [ @@ -5854,7 +6771,7 @@ "h": 1, "w": 24, "x": 0, - "y": 22 + "y": 23 }, "id": 108, "panels": [ @@ -6403,7 +7320,7 @@ "h": 1, "w": 24, "x": 0, - "y": 23 + "y": 24 }, "id": 92, "panels": [ @@ -7449,7 +8366,7 @@ "h": 1, "w": 24, "x": 0, - "y": 24 + "y": 25 }, "id": 25, "panels": [ @@ -7970,7 +8887,7 @@ "h": 1, "w": 24, "x": 0, - "y": 25 + "y": 26 }, "id": 110, "panels": [ @@ -8482,7 +9399,7 @@ "h": 1, "w": 24, "x": 0, - "y": 26 + "y": 27 }, "id": 136, "panels": [ @@ -9074,7 +9991,7 @@ "h": 1, "w": 24, "x": 0, - "y": 27 + "y": 28 }, "id": 75, "panels": [ @@ -9786,7 +10703,7 @@ "h": 1, "w": 24, "x": 0, - "y": 28 + "y": 29 }, "id": 86, "panels": [ @@ -9961,7 +10878,7 @@ "h": 1, "w": 24, "x": 0, - "y": 29 + "y": 30 }, "id": 28, "panels": [ @@ -10711,7 +11628,7 @@ "h": 1, "w": 24, "x": 0, - "y": 30 + "y": 31 }, "id": 66, "panels": [ @@ -11380,7 +12297,7 @@ "h": 1, "w": 24, "x": 0, - "y": 31 + "y": 32 }, "id": 232, "panels": [ @@ -12607,7 +13524,7 @@ "h": 1, "w": 24, "x": 0, - "y": 32 + "y": 33 }, "id": 164, "panels": [ @@ -12789,7 +13706,7 @@ "h": 1, "w": 24, "x": 0, - "y": 33 + "y": 34 }, "id": 166, "panels": [ @@ -13019,7 +13936,7 @@ "h": 1, "w": 24, "x": 0, - "y": 34 + "y": 35 }, "id": 374, "panels": [ @@ -13213,7 +14130,7 @@ "h": 1, "w": 24, "x": 0, - "y": 35 + "y": 36 }, "id": 188, "panels": [ @@ -13652,7 +14569,7 @@ "h": 1, "w": 24, "x": 0, - "y": 36 + "y": 37 }, "id": 214, "panels": [ @@ -14179,7 +15096,7 @@ "h": 1, "w": 24, "x": 0, - "y": 37 + "y": 38 }, "id": 270, "panels": [ @@ -14880,7 +15797,7 @@ "h": 1, "w": 24, "x": 0, - "y": 38 + "y": 39 }, "id": 337, "panels": [ @@ -15450,7 +16367,7 @@ "h": 1, "w": 24, "x": 0, - "y": 39 + "y": 40 }, "id": 252, "panels": [ @@ -15879,7 +16796,7 @@ "h": 1, "w": 24, "x": 0, - "y": 40 + "y": 41 }, "id": 309, "panels": [ @@ -16301,7 +17218,7 @@ "h": 1, "w": 24, "x": 0, - "y": 41 + "y": 42 }, "id": 313, "panels": [ diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index 6ba14c387301..ca09a86fefde 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -13,7 +13,7 @@ import { } from "@lodestar/state-transition"; import {IBeaconConfig} from "@lodestar/config"; import {allForks, UintNum64, Root, phase0, Slot, RootHex, Epoch, ValidatorIndex} from "@lodestar/types"; -import {CheckpointWithHex, IForkChoice, ProtoBlock} from "@lodestar/fork-choice"; +import {CheckpointWithHex, ExecutionStatus, IForkChoice, ProtoBlock} from "@lodestar/fork-choice"; import {ILogger, toHex} from "@lodestar/utils"; import {CompositeTypeAny, fromHexString, TreeView, Type} from "@chainsafe/ssz"; import {GENESIS_EPOCH, ZERO_HASH} from "../constants/index.js"; @@ -521,6 +521,17 @@ export class BeaconChain implements IBeaconChain { this.seenAggregatedAttestations.prune(epoch); this.seenBlockAttesters.prune(epoch); this.beaconProposerCache.prune(epoch); + + // Poll for merge block in the background to speed-up block production. Only if: + // - after BELLATRIX_FORK_EPOCH + // - Beacon node synced + // - head state not isMergeTransitionComplete + if (this.config.BELLATRIX_FORK_EPOCH - epoch < 1) { + const head = this.forkChoice.getHead(); + if (epoch - computeEpochAtSlot(head.slot) < 5 && head.executionStatus === ExecutionStatus.PreMerge) { + this.eth1.startPollingMergeBlock(); + } + } } private onForkChoiceHead(head: ProtoBlock): void { diff --git a/packages/beacon-node/src/chain/factory/block/body.ts b/packages/beacon-node/src/chain/factory/block/body.ts index bf7f2d2e0359..52e7709e3418 100644 --- a/packages/beacon-node/src/chain/factory/block/body.ts +++ b/packages/beacon-node/src/chain/factory/block/body.ts @@ -183,7 +183,7 @@ export async function prepareExecutionPayload( state: CachedBeaconStateBellatrix, suggestedFeeRecipient: string ): Promise<{isPremerge: true} | {isPremerge: false; payloadId: PayloadId}> { - const parentHashRes = getExecutionPayloadParentHash(chain, state); + const parentHashRes = await getExecutionPayloadParentHash(chain, state); if (parentHashRes.isPremerge) { // Return null only if the execution is pre-merge return {isPremerge: true}; @@ -237,7 +237,7 @@ async function prepareExecutionPayloadHeader( throw Error("executionBuilder required"); } - const parentHashRes = getExecutionPayloadParentHash(chain, state); + const parentHashRes = await getExecutionPayloadParentHash(chain, state); if (parentHashRes.isPremerge) { // TODO: Is this okay? @@ -248,13 +248,13 @@ async function prepareExecutionPayloadHeader( return chain.executionBuilder.getPayloadHeader(state.slot, parentHash, proposerPubKey); } -function getExecutionPayloadParentHash( +async function getExecutionPayloadParentHash( chain: { eth1: IEth1ForBlockProduction; config: IChainForkConfig; }, state: CachedBeaconStateBellatrix -): {isPremerge: true} | {isPremerge: false; parentHash: Root} { +): Promise<{isPremerge: true} | {isPremerge: false; parentHash: Root}> { // Use different POW block hash parent for block production based on merge status. // Returned value of null == using an empty ExecutionPayload value if (isMergeTransitionComplete(state)) { @@ -271,7 +271,7 @@ function getExecutionPayloadParentHash( }, actual: ${getCurrentEpoch(state)}` ); - const terminalPowBlockHash = chain.eth1.getTerminalPowBlock(); + const terminalPowBlockHash = await chain.eth1.getTerminalPowBlock(); if (terminalPowBlockHash === null) { // Pre-merge, no prepare payload call is needed return {isPremerge: true}; diff --git a/packages/beacon-node/src/eth1/eth1MergeBlockTracker.ts b/packages/beacon-node/src/eth1/eth1MergeBlockTracker.ts index 7424835a7e70..03a30325b473 100644 --- a/packages/beacon-node/src/eth1/eth1MergeBlockTracker.ts +++ b/packages/beacon-node/src/eth1/eth1MergeBlockTracker.ts @@ -1,112 +1,144 @@ import {IChainConfig} from "@lodestar/config"; -import {Epoch, RootHex} from "@lodestar/types"; -import {ILogger, isErrorAborted, sleep} from "@lodestar/utils"; +import {RootHex} from "@lodestar/types"; +import {ILogger} from "@lodestar/utils"; import {toHexString} from "@chainsafe/ssz"; -import {SLOTS_PER_EPOCH} from "@lodestar/params"; +import {IMetrics} from "../metrics/index.js"; import {ZERO_HASH_HEX} from "../constants/index.js"; -import {IEth1Provider, EthJsonRpcBlockRaw, PowMergeBlock} from "./interface.js"; +import {enumToIndexMap} from "../util/enum.js"; +import {pruneSetToMax} from "../util/map.js"; +import {IEth1Provider, EthJsonRpcBlockRaw, PowMergeBlock, PowMergeBlockTimestamp, TDProgress} from "./interface.js"; import {quantityToNum, quantityToBigint, dataToRootHex} from "./provider/utils.js"; export enum StatusCode { - PRE_MERGE = "PRE_MERGE", + STOPPED = "STOPPED", SEARCHING = "SEARCHING", FOUND = "FOUND", - POST_MERGE = "POST_MERGE", } -/** Numbers of epochs in advance of merge fork condition to start looking for merge block */ -const START_EPOCHS_IN_ADVANCE = 5; +type Status = + | {code: StatusCode.STOPPED} + | {code: StatusCode.SEARCHING} + | {code: StatusCode.FOUND; mergeBlock: PowMergeBlock}; + +/** For metrics, index order = declaration order of StatusCode */ +const statusCodeIdx = enumToIndexMap(StatusCode); /** * Bounds `blocksByHashCache` cache, imposing a max distance between highest and lowest block numbers. * In case of extreme forking the cache might grow unbounded. */ -const MAX_CACHE_POW_HEIGHT_DISTANCE = 1024; - -/** Number of blocks to request at once in a getBlocksByNumber() request */ -const MAX_BLOCKS_PER_PAST_REQUEST = 1000; +const MAX_CACHE_POW_BLOCKS = 1024; -/** Prevent infinite loops on error by sleeping after each error */ -const SLEEP_ON_ERROR_MS = 3000; +const MAX_TD_RENDER_VALUE = Number.MAX_SAFE_INTEGER; export type Eth1MergeBlockTrackerModules = { config: IChainConfig; logger: ILogger; signal: AbortSignal; - clockEpoch: Epoch; - isMergeTransitionComplete: boolean; + metrics: IMetrics | null; }; +// get_pow_block_at_total_difficulty + /** * Follows the eth1 chain to find a (or multiple?) merge blocks that cross the threshold of total terminal difficulty + * + * Finding the mergeBlock could be done in demand when proposing pre-merge blocks. However, that would slow block + * production during the weeks between BELLATRIX_EPOCH and TTD. */ export class Eth1MergeBlockTracker { private readonly config: IChainConfig; private readonly logger: ILogger; - private readonly signal: AbortSignal; + private readonly metrics: IMetrics | null; - /** - * First found mergeBlock. - * TODO: Accept multiple, but then handle long backwards searches properly after crossing TTD - */ - private mergeBlock: PowMergeBlock | null = null; private readonly blocksByHashCache = new Map(); - - private status: StatusCode = StatusCode.PRE_MERGE; private readonly intervals: NodeJS.Timeout[] = []; + private status: Status; + private latestEth1Block: PowMergeBlockTimestamp | null = null; + private getTerminalPowBlockFromEth1Promise: Promise | null = null; + private readonly safeTDFactor: bigint; + constructor( - {config, logger, signal, clockEpoch, isMergeTransitionComplete}: Eth1MergeBlockTrackerModules, + {config, logger, signal, metrics}: Eth1MergeBlockTrackerModules, private readonly eth1Provider: IEth1Provider ) { this.config = config; this.logger = logger; - this.signal = signal; + this.metrics = metrics; - // If merge has already happened, disable - if (isMergeTransitionComplete) { - this.status = StatusCode.POST_MERGE; - return; - } - - // If merge is still not programed, skip - if (config.BELLATRIX_FORK_EPOCH >= Infinity) { - return; - } + // eth1MergeStatus + this.logger.info("Starting search for terminal POW block", { + // eslint-disable-next-line @typescript-eslint/naming-convention + TERMINAL_TOTAL_DIFFICULTY: this.config.TERMINAL_TOTAL_DIFFICULTY, + }); - const startEpoch = this.config.BELLATRIX_FORK_EPOCH - START_EPOCHS_IN_ADVANCE; - if (startEpoch <= clockEpoch) { - // Start now - void this.startFinding(); - } else { - // Set a timer to start in the future - const intervalToStart = setInterval(() => { - void this.startFinding(); - }, (startEpoch - clockEpoch) * SLOTS_PER_EPOCH * config.SECONDS_PER_SLOT * 1000); - this.intervals.push(intervalToStart); - } + this.status = {code: StatusCode.STOPPED}; signal.addEventListener("abort", () => this.close(), {once: true}); + + this.safeTDFactor = getSafeTDFactor(this.config.TERMINAL_TOTAL_DIFFICULTY); + const scaledTTD = this.config.TERMINAL_TOTAL_DIFFICULTY / this.safeTDFactor; + + // Only run metrics if necessary + if (metrics) { + // TTD can't be dynamically changed during execution, register metric once + metrics.eth1.eth1MergeTTD.set(Number(scaledTTD as bigint)); + metrics.eth1.eth1MergeTDFactor.set(Number(this.safeTDFactor as bigint)); + + metrics.eth1.eth1MergeStatus.addCollect(() => { + // Set merge ttd, merge status and merge block status + metrics.eth1.eth1MergeStatus.set(statusCodeIdx[this.status.code]); + + if (this.latestEth1Block !== null) { + // Set latestBlock stats + metrics.eth1.eth1LatestBlockNumber.set(this.latestEth1Block.number); + metrics.eth1.eth1LatestBlockTD.set(Number(this.latestEth1Block.totalDifficulty / this.safeTDFactor)); + metrics.eth1.eth1LatestBlockTimestamp.set(this.latestEth1Block.timestamp); + } + }); + } } /** * Returns the most recent POW block that satisfies the merge block condition */ - getTerminalPowBlock(): PowMergeBlock | null { - // For better debugging in case this module stops searching too early - if (this.mergeBlock === null && this.status === StatusCode.POST_MERGE) { - throw Error("Eth1MergeBlockFinder is on POST_MERGE status and found no mergeBlock"); + async getTerminalPowBlock(): Promise { + switch (this.status.code) { + case StatusCode.STOPPED: + // If not module is not polling fetch the mergeBlock explicitly + return this.getTerminalPowBlockFromEth1(); + + case StatusCode.SEARCHING: + // Assume that polling would have found the block + return null; + + case StatusCode.FOUND: + return this.status.mergeBlock; } - - return this.mergeBlock; } - /** - * Call when merge is irrevocably completed to stop polling unnecessary data from the eth1 node - */ - mergeCompleted(): void { - this.status = StatusCode.POST_MERGE; - this.close(); + getTDProgress(): TDProgress | null { + if (this.latestEth1Block === null) { + return this.latestEth1Block; + } + + const tdDiff = this.config.TERMINAL_TOTAL_DIFFICULTY - this.latestEth1Block.totalDifficulty; + + if (tdDiff > BigInt(0)) { + return { + ttdHit: false, + tdFactor: this.safeTDFactor, + tdDiffScaled: Number((tdDiff / this.safeTDFactor) as bigint), + ttd: this.config.TERMINAL_TOTAL_DIFFICULTY, + td: this.latestEth1Block.totalDifficulty, + timestamp: this.latestEth1Block.timestamp, + }; + } else { + return { + ttdHit: true, + }; + } } /** @@ -115,231 +147,171 @@ export class Eth1MergeBlockTracker { async getPowBlock(powBlockHash: string): Promise { // Check cache first const cachedBlock = this.blocksByHashCache.get(powBlockHash); - if (cachedBlock) return cachedBlock; + if (cachedBlock) { + return cachedBlock; + } // Fetch from node const blockRaw = await this.eth1Provider.getBlockByHash(powBlockHash); if (blockRaw) { const block = toPowBlock(blockRaw); - this.blocksByHashCache.set(block.blockHash, block); + this.cacheBlock(block); return block; } return null; } - private close(): void { - this.intervals.forEach(clearInterval); - } - - private setTerminalPowBlock(mergeBlock: PowMergeBlock): void { - this.logger.info("Terminal POW block found!", { - hash: mergeBlock.blockHash, - number: mergeBlock.number, - totalDifficulty: mergeBlock.totalDifficulty, - }); - this.mergeBlock = mergeBlock; - this.status = StatusCode.FOUND; - this.close(); - } - - private async startFinding(): Promise { - if (this.status !== StatusCode.PRE_MERGE) return; - - // Terminal block hash override takes precedence over terminal total difficulty - const terminalBlockHash = toHexString(this.config.TERMINAL_BLOCK_HASH); - if (terminalBlockHash !== ZERO_HASH_HEX) { - try { - const powBlockOverride = await this.getPowBlock(terminalBlockHash); - if (powBlockOverride) { - this.setTerminalPowBlock(powBlockOverride); - } - } catch (e) { - if (!isErrorAborted(e)) { - this.logger.error("Error fetching POW block from terminal block hash", {terminalBlockHash}, e as Error); - } - } - // if a TERMINAL_BLOCK_HASH other than ZERO_HASH is configured and we can't find it, return NONE + /** + * Should only start polling for mergeBlock if: + * - after BELLATRIX_FORK_EPOCH + * - Beacon node synced + * - head state not isMergeTransitionComplete + */ + startPollingMergeBlock(): void { + if (this.status.code !== StatusCode.STOPPED) { return; } - this.status = StatusCode.SEARCHING; + this.status = {code: StatusCode.SEARCHING}; this.logger.info("Starting search for terminal POW block", { // eslint-disable-next-line @typescript-eslint/naming-convention TERMINAL_TOTAL_DIFFICULTY: this.config.TERMINAL_TOTAL_DIFFICULTY, }); - // 1. Fetch current head chain until finding a block with total difficulty less than `transitionStore.terminalTotalDifficulty` - this.fetchPreviousBlocks().catch((e) => { - if (!isErrorAborted(e)) this.logger.error("Error fetching past POW blocks", {}, e); - }); - - // 2. Subscribe to eth1 blocks and recursively fetch potential POW blocks - const intervalPoll = setInterval(() => { - this.pollLatestBlock().catch((e) => { - if (!isErrorAborted(e)) this.logger.error("Error fetching latest POW block", {}, e); + const interval = setInterval(() => { + // Pre-emptively try to find merge block and cache it if found. + // Future callers of getTerminalPowBlock() will re-use the cached found mergeBlock. + this.getTerminalPowBlockFromEth1().catch((e) => { + this.logger.error("Error on findMergeBlock", {}, e as Error); + this.metrics?.eth1.eth1PollMergeBlockErrors.inc(); }); }, this.config.SECONDS_PER_ETH1_BLOCK * 1000); - // 3. Prune roughly every epoch - const intervalPrune = setInterval(() => { - this.prune(); - }, 32 * this.config.SECONDS_PER_SLOT * 1000); - - // Register interval to clean them on close() - this.intervals.push(intervalPoll, intervalPrune); + this.intervals.push(interval); } - private async fetchPreviousBlocks(): Promise { - // If latest block is under TTD, stop. Subscriptions will pick future blocks - // If latest block is over TTD, go backwards until finding a merge block - // Note: Must ensure parent relationship - - // Fast path for pre-merge scenario - const latestBlockRaw = await this.eth1Provider.getBlockByNumber("latest"); - if (!latestBlockRaw) { - throw Error("getBlockByNumber('latest') returned null"); - } - - const latestBlock = toPowBlock(latestBlockRaw); - // TTD not reached yet, stop looking at old blocks and expect the subscription to find merge block - if (latestBlock.totalDifficulty < this.config.TERMINAL_TOTAL_DIFFICULTY) { - return; - } - - // TTD already reached, search blocks backwards - let minFetchedBlockNumber = latestBlock.number; - // eslint-disable-next-line no-constant-condition - while (true) { - const from = Math.max(0, minFetchedBlockNumber - MAX_BLOCKS_PER_PAST_REQUEST); - // Re-fetch same block to have the full chain of parent-child nodes - const to = minFetchedBlockNumber; - - try { - const blocksRaw = await this.eth1Provider.getBlocksByNumber(from, to); - const blocks = blocksRaw.map(toPowBlock); - - // Should never happen - if (blocks.length < 2) { - throw Error(`getBlocksByNumber(${from}, ${to}) returned less than 2 results`); - } + private close(): void { + this.intervals.forEach(clearInterval); + } - for (let i = 0; i < blocks.length - 1; i++) { - const childBlock = blocks[i + 1]; - const parentBlock = blocks[i]; - if ( - childBlock.totalDifficulty >= this.config.TERMINAL_TOTAL_DIFFICULTY && - parentBlock.totalDifficulty < this.config.TERMINAL_TOTAL_DIFFICULTY - ) { - // Is terminal total difficulty block - if (childBlock.parentHash === parentBlock.blockHash) { - // AND has verified block -> parent relationship - return this.setTerminalPowBlock(childBlock); - } else { - // WARNING! Re-org while doing getBlocksByNumber() call. Ensure that this block is the merge block - // and not some of its parents. - return await this.fetchPotentialMergeBlock(childBlock); + private async getTerminalPowBlockFromEth1(): Promise { + if (!this.getTerminalPowBlockFromEth1Promise) { + this.getTerminalPowBlockFromEth1Promise = this.internalGetTerminalPowBlockFromEth1() + .then((mergeBlock) => { + // Persist found merge block here to affect both caller paths: + // - internal searcher + // - external caller if STOPPED + if (mergeBlock && this.status.code != StatusCode.FOUND) { + if (this.status.code === StatusCode.SEARCHING) { + this.close(); } - } - } - // On next round - minFetchedBlockNumber = Math.min(to, ...blocks.map((block) => block.number)); + this.logger.info("Terminal POW block found!", { + hash: mergeBlock.blockHash, + number: mergeBlock.number, + totalDifficulty: mergeBlock.totalDifficulty, + }); + + this.status = {code: StatusCode.FOUND, mergeBlock}; + this.metrics?.eth1.eth1MergeBlockDetails.set( + { + terminalBlockHash: mergeBlock.blockHash, + // Convert all number/bigints to string labels + terminalBlockNumber: mergeBlock.number.toString(10), + terminalBlockTD: mergeBlock.totalDifficulty.toString(10), + }, + 1 + ); + } - // Scanned the entire blockchain - if (minFetchedBlockNumber <= 0) { - return; - } - } catch (e) { - if (!isErrorAborted(e)) this.logger.error("Error on fetchPreviousBlocks range", {from, to}, e as Error); - await sleep(SLEEP_ON_ERROR_MS, this.signal); - } + return mergeBlock; + }) + .finally(() => { + this.getTerminalPowBlockFromEth1Promise = null; + }); + } else { + // This should no happen, since getTerminalPowBlockFromEth1() should resolve faster than SECONDS_PER_ETH1_BLOCK. + // else something is wrong: the el-cl comms are two slow, or the backsearch got stuck in a deep search. + this.metrics?.eth1.getTerminalPowBlockPromiseCacheHit.inc(); } + + return this.getTerminalPowBlockFromEth1Promise; } /** - * Fetches the current latest block according the execution client. - * If the latest block has totalDifficulty over TTD, it will backwards recursive search the merge block. - * TODO: How to prevent doing long recursive search after the merge block has happened? + * **internal** + **unsafe** since it can create multiple backward searches that overload the eth1 client. + * Must be called in a wrapper to ensure that there's only once concurrent call to this fn. */ - private async pollLatestBlock(): Promise { - const latestBlockRaw = await this.eth1Provider.getBlockByNumber("latest"); - if (!latestBlockRaw) { - throw Error("getBlockByNumber('latest') returned null"); + private async internalGetTerminalPowBlockFromEth1(): Promise { + // Search merge block by hash + // Terminal block hash override takes precedence over terminal total difficulty + const terminalBlockHash = toHexString(this.config.TERMINAL_BLOCK_HASH); + if (terminalBlockHash !== ZERO_HASH_HEX) { + const block = await this.getPowBlock(terminalBlockHash); + if (block) { + return block; + } else { + // if a TERMINAL_BLOCK_HASH other than ZERO_HASH is configured and we can't find it, return NONE + return null; + } } - const latestBlock = toPowBlock(latestBlockRaw); - await this.fetchPotentialMergeBlock(latestBlock); - } + // Search merge block by TTD + else { + const latestBlockRaw = await this.eth1Provider.getBlockByNumber("latest"); + if (!latestBlockRaw) { + throw Error("getBlockByNumber('latest') returned null"); + } - /** - * Potential merge block, do a backwards search with parent hashes. - * De-duplicates code between pollLatestBlock() and fetchPreviousBlocks(). - */ - private async fetchPotentialMergeBlock(block: PowMergeBlock): Promise { - this.logger.debug("Potential terminal POW block", { - number: block.number, - hash: block.blockHash, - totalDifficulty: block.totalDifficulty, - }); - // Persist block for future searches - this.blocksByHashCache.set(block.blockHash, block); + let block = toPowBlock(latestBlockRaw); + this.latestEth1Block = {...block, timestamp: quantityToNum(latestBlockRaw.timestamp)}; + this.cacheBlock(block); + + // This code path to look backwards for the merge block is only necessary if: + // - The network has not yet found the merge block + // - There are descendants of the merge block in the eth1 chain + // For the search below to require more than a few hops, multiple block proposers in a row must fail to detect + // an existing merge block. Such situation is extremely unlikely, so this search is left un-optimized. Since + // this class can start eagerly looking for the merge block when not necessary, startPollingMergeBlock() should + // only be called when there is certainity that a mergeBlock search is necessary. + + // eslint-disable-next-line no-constant-condition + while (true) { + if (block.totalDifficulty < this.config.TERMINAL_TOTAL_DIFFICULTY) { + // TTD not reached yet + return null; + } - // Check if this block is already visited + // else block.totalDifficulty >= this.config.TERMINAL_TOTAL_DIFFICULTY + // Potential mergeBlock! Must find the first block that passes TTD - while (block.totalDifficulty >= this.config.TERMINAL_TOTAL_DIFFICULTY) { - if (block.parentHash === ZERO_HASH_HEX) { - // Allow genesis block to reach TTD - // https://github.com/ethereum/consensus-specs/pull/2719 - return this.setTerminalPowBlock(block); - } + // Allow genesis block to reach TTD https://github.com/ethereum/consensus-specs/pull/2719 + if (block.parentHash === ZERO_HASH_HEX) { + return block; + } - const parent = await this.getPowBlock(block.parentHash); - // Unknown parent - if (!parent) { - return; - } + const parent = await this.getPowBlock(block.parentHash); + if (!parent) { + throw Error(`Unknown parent of block with TD>TTD ${block.parentHash}`); + } - if ( - block.totalDifficulty >= this.config.TERMINAL_TOTAL_DIFFICULTY && - parent.totalDifficulty < this.config.TERMINAL_TOTAL_DIFFICULTY - ) { - // Is terminal total difficulty block AND has verified block -> parent relationship - return this.setTerminalPowBlock(block); - } + this.metrics?.eth1.eth1ParentBlocksFetched.inc(); - // Guard against infinite loops - if (parent.blockHash === block.blockHash) { - throw Error("Infinite loop: parent.blockHash === block.blockHash"); + // block.td > TTD && parent.td < TTD => block is mergeBlock + if (parent.totalDifficulty < this.config.TERMINAL_TOTAL_DIFFICULTY) { + // Is terminal total difficulty block AND has verified block -> parent relationship + return block; + } else { + block = parent; + } } - - // Fetch parent's parent - block = parent; } } - /** - * Prune blocks to have at max MAX_CACHE_POW_HEIGHT_DISTANCE between the highest block number in the cache - * and the lowest block number in the cache. - * - * Call every once in a while, i.e. once per epoch - */ - private prune(): void { - // Find the heightest block number in the cache - let maxBlockNumber = 0; - for (const block of this.blocksByHashCache.values()) { - if (block.number > maxBlockNumber) { - maxBlockNumber = block.number; - } - } - - // Prune blocks below the max distance - const minHeight = maxBlockNumber - MAX_CACHE_POW_HEIGHT_DISTANCE; - for (const [key, block] of this.blocksByHashCache.entries()) { - if (block.number < minHeight) { - this.blocksByHashCache.delete(key); - } - } + private cacheBlock(block: PowMergeBlock): void { + this.blocksByHashCache.set(block.blockHash, block); + pruneSetToMax(this.blocksByHashCache, MAX_CACHE_POW_BLOCKS); } } @@ -352,3 +324,20 @@ export function toPowBlock(block: EthJsonRpcBlockRaw): PowMergeBlock { totalDifficulty: quantityToBigint(block.totalDifficulty), }; } + +/** + * TTD values can be very large, for xDAI > 1e45. So scale down. + * To be good, TTD should be rendered as a number < Number.MAX_TD_RENDER_VALUE ~= 9e15 + */ +export function getSafeTDFactor(ttd: bigint): bigint { + const safeIntegerMult = ttd / BigInt(MAX_TD_RENDER_VALUE); + + // TTD < MAX_TD_RENDER_VALUE, no need to scale down + if (safeIntegerMult === BigInt(0)) { + return BigInt(1); + } + + // Return closest power of 10 to ensure TD < max + const safeIntegerMultDigits = safeIntegerMult.toString(10).length; + return BigInt(10) ** BigInt(safeIntegerMultDigits); +} diff --git a/packages/beacon-node/src/eth1/index.ts b/packages/beacon-node/src/eth1/index.ts index 30e942a68352..1e23b3016334 100644 --- a/packages/beacon-node/src/eth1/index.ts +++ b/packages/beacon-node/src/eth1/index.ts @@ -1,14 +1,7 @@ -import { - BeaconStateAllForks, - CachedBeaconStateAllForks, - computeEpochAtSlot, - getCurrentSlot, - isBellatrixStateType, - isMergeTransitionComplete, -} from "@lodestar/state-transition"; +import {CachedBeaconStateAllForks} from "@lodestar/state-transition"; import {Root} from "@lodestar/types"; import {fromHexString} from "@chainsafe/ssz"; -import {IEth1ForBlockProduction, Eth1DataAndDeposits, IEth1Provider, PowMergeBlock} from "./interface.js"; +import {IEth1ForBlockProduction, Eth1DataAndDeposits, IEth1Provider, PowMergeBlock, TDProgress} from "./interface.js"; import {Eth1DepositDataTracker, Eth1DepositDataTrackerModules} from "./eth1DepositDataTracker.js"; import {Eth1MergeBlockTracker, Eth1MergeBlockTrackerModules} from "./eth1MergeBlockTracker.js"; import {Eth1Options} from "./options.js"; @@ -49,8 +42,7 @@ export {IEth1ForBlockProduction, IEth1Provider, Eth1Provider}; export function initializeEth1ForBlockProduction( opts: Eth1Options, - modules: Pick, - anchorState: BeaconStateAllForks + modules: Pick ): IEth1ForBlockProduction { if (opts.enabled) { return new Eth1ForBlockProduction(opts, { @@ -59,13 +51,12 @@ export function initializeEth1ForBlockProduction( metrics: modules.metrics, logger: modules.logger, signal: modules.signal, - clockEpoch: computeEpochAtSlot(getCurrentSlot(modules.config, anchorState.genesisTime)), - isMergeTransitionComplete: isBellatrixStateType(anchorState) && isMergeTransitionComplete(anchorState), }); } else { return new Eth1ForBlockProductionDisabled(); } } + export class Eth1ForBlockProduction implements IEth1ForBlockProduction { private readonly eth1DepositDataTracker: Eth1DepositDataTracker | null; private readonly eth1MergeBlockTracker: Eth1MergeBlockTracker; @@ -92,18 +83,22 @@ export class Eth1ForBlockProduction implements IEth1ForBlockProduction { } } - getTerminalPowBlock(): Root | null { - const block = this.eth1MergeBlockTracker.getTerminalPowBlock(); + async getTerminalPowBlock(): Promise { + const block = await this.eth1MergeBlockTracker.getTerminalPowBlock(); return block && fromHexString(block.blockHash); } - mergeCompleted(): void { - this.eth1MergeBlockTracker.mergeCompleted(); - } - getPowBlock(powBlockHash: string): Promise { return this.eth1MergeBlockTracker.getPowBlock(powBlockHash); } + + getTDProgress(): TDProgress | null { + return this.eth1MergeBlockTracker.getTDProgress(); + } + + startPollingMergeBlock(): void { + return this.eth1MergeBlockTracker.startPollingMergeBlock(); + } } /** @@ -122,16 +117,20 @@ export class Eth1ForBlockProductionDisabled implements IEth1ForBlockProduction { /** * Will miss the oportunity to propose the merge block but will still produce valid blocks */ - getTerminalPowBlock(): Root | null { + async getTerminalPowBlock(): Promise { return null; } - mergeCompleted(): void { - // Ignore - } - /** Will not be able to validate the merge block */ - async getPowBlock(): Promise { + async getPowBlock(_powBlockHash: string): Promise { throw Error("eth1 must be enabled to verify merge block"); } + + getTDProgress(): TDProgress | null { + return null; + } + + startPollingMergeBlock(): void { + // Ignore + } } diff --git a/packages/beacon-node/src/eth1/interface.ts b/packages/beacon-node/src/eth1/interface.ts index d6031955f392..862a373f1294 100644 --- a/packages/beacon-node/src/eth1/interface.ts +++ b/packages/beacon-node/src/eth1/interface.ts @@ -40,11 +40,20 @@ export interface IEth1ForBlockProduction { getEth1DataAndDeposits(state: CachedBeaconStateAllForks): Promise; /** Returns the most recent POW block that satisfies the merge block condition */ - getTerminalPowBlock(): Root | null; - /** Call when merge is irrevocably completed to stop polling unnecessary data from the eth1 node */ - mergeCompleted(): void; + getTerminalPowBlock(): Promise; /** Get a POW block by hash checking the local cache first */ getPowBlock(powBlockHash: string): Promise; + + /** Get current TD progress for log notifier */ + getTDProgress(): TDProgress | null; + + /** + * Should only start polling for mergeBlock if: + * - after BELLATRIX_FORK_EPOCH + * - Beacon node synced + * - head state not isMergeTransitionComplete + */ + startPollingMergeBlock(): void; } /** Different Eth1Block from phase0.Eth1Block with blockHash */ @@ -61,6 +70,27 @@ export type PowMergeBlock = { totalDifficulty: bigint; }; +export type PowMergeBlockTimestamp = PowMergeBlock & { + /** in seconds */ + timestamp: number; +}; + +export type TDProgress = + | { + ttdHit: false; + /** Power of ten by which tdDiffScaled is scaled down */ + tdFactor: bigint; + /** (TERMINAL_TOTAL_DIFFICULTY - block.totalDifficulty) / tdFactor */ + tdDiffScaled: number; + /** TERMINAL_TOTAL_DIFFICULTY */ + ttd: bigint; + /** totalDifficulty of latest fetched eth1 block */ + td: bigint; + /** timestamp in sec of latest fetched eth1 block */ + timestamp: number; + } + | {ttdHit: true}; + export interface IBatchDepositEvents { depositEvents: phase0.DepositEvent[]; blockNumber: number; diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index 9053a181834b..3c710168d178 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -1052,6 +1052,54 @@ export function createLodestarMetrics( name: "lodestar_eth1_follow_distance_dynamic", help: "Eth1 dynamic follow distance changed by the deposit tracker if blocks are slow", }), + + // Merge Search info + eth1MergeStatus: register.gauge({ + name: "lodestar_eth1_merge_status", + help: "Eth1 Merge Status 0 PRE_MERGE 1 SEARCHING 2 FOUND 3 POST_MERGE", + }), + eth1MergeTDFactor: register.gauge({ + name: "lodestar_eth1_merge_td_factor", + help: "TTD set for the merge", + }), + eth1MergeTTD: register.gauge({ + name: "lodestar_eth1_merge_ttd", + help: "TTD set for the merge scaled down by td_factor", + }), + + eth1PollMergeBlockErrors: register.gauge({ + name: "lodestar_eth1_poll_merge_block_errors_total", + help: "Total count of errors polling merge block", + }), + getTerminalPowBlockPromiseCacheHit: register.gauge({ + name: "lodestar_eth1_get_terminal_pow_block_promise_cache_hit_total", + help: "Total count of skipped runs in poll merge block, because a previous promise existed", + }), + eth1ParentBlocksFetched: register.gauge({ + name: "lodestar_eth1_parent_blocks_fetched_total", + help: "Total count of parent blocks fetched searching for merge block", + }), + + // Latest block details + eth1LatestBlockTD: register.gauge({ + name: "lodestar_eth1_latest_block_ttd", + help: "Eth1 latest Block td scaled down by td_factor", + }), + eth1LatestBlockNumber: register.gauge({ + name: "lodestar_eth1_latest_block_number", + help: "Eth1 latest block number", + }), + eth1LatestBlockTimestamp: register.gauge({ + name: "lodestar_eth1_latest_block_timestamp", + help: "Eth1 latest block timestamp", + }), + + // Merge details + eth1MergeBlockDetails: register.gauge<"terminalBlockHash" | "terminalBlockNumber" | "terminalBlockTD">({ + name: "lodestar_eth1_merge_block_details", + help: "If found then 1 with terminal block details", + labelNames: ["terminalBlockHash", "terminalBlockNumber", "terminalBlockTD"], + }), }, eth1HttpClient: { diff --git a/packages/beacon-node/src/node/nodejs.ts b/packages/beacon-node/src/node/nodejs.ts index 02664d2c74ec..a56776b415ea 100644 --- a/packages/beacon-node/src/node/nodejs.ts +++ b/packages/beacon-node/src/node/nodejs.ts @@ -137,11 +137,13 @@ export class BeaconNode { logger: logger.child(opts.logger.chain), metrics, anchorState, - eth1: initializeEth1ForBlockProduction( - opts.eth1, - {config, db, metrics, logger: logger.child(opts.logger.eth1), signal}, - anchorState - ), + eth1: initializeEth1ForBlockProduction(opts.eth1, { + config, + db, + metrics, + logger: logger.child(opts.logger.eth1), + signal, + }), executionEngine: initializeExecutionEngine(opts.executionEngine, {metrics, signal}), executionBuilder: opts.executionBuilder.enabled ? initializeExecutionBuilder(opts.executionBuilder, config) diff --git a/packages/beacon-node/src/node/notifier.ts b/packages/beacon-node/src/node/notifier.ts index fd93f006b0f2..1e31a2b681da 100644 --- a/packages/beacon-node/src/node/notifier.ts +++ b/packages/beacon-node/src/node/notifier.ts @@ -1,36 +1,38 @@ import {IBeaconConfig} from "@lodestar/config"; +import {Epoch} from "@lodestar/types"; +import {CachedBeaconStateAllForks} from "@lodestar/state-transition"; +import {ProtoBlock} from "@lodestar/fork-choice"; import {ErrorAborted, ILogger, sleep, prettyBytes} from "@lodestar/utils"; import {EPOCHS_PER_SYNC_COMMITTEE_PERIOD, SLOTS_PER_EPOCH} from "@lodestar/params"; import {computeEpochAtSlot, isBellatrixCachedStateType, isMergeTransitionComplete} from "@lodestar/state-transition"; import {IBeaconChain} from "../chain/index.js"; import {INetwork} from "../network/index.js"; import {IBeaconSync, SyncState} from "../sync/index.js"; -import {prettyTimeDiff} from "../util/time.js"; +import {prettyTimeDiffSec} from "../util/time.js"; import {TimeSeries} from "../util/timeSeries.js"; /** Create a warning log whenever the peer count is at or below this value */ const WARN_PEER_COUNT = 1; -/** - * Runs a notifier service that periodically logs information about the node. - */ -export async function runNodeNotifier({ - network, - chain, - sync, - config, - logger, - signal, -}: { +type NodeNotifierModules = { network: INetwork; chain: IBeaconChain; sync: IBeaconSync; config: IBeaconConfig; logger: ILogger; signal: AbortSignal; -}): Promise { +}; + +/** + * Runs a notifier service that periodically logs information about the node. + */ +export async function runNodeNotifier(modules: NodeNotifierModules): Promise { + const {network, chain, sync, config, logger, signal} = modules; + + const headSlotTimeSeries = new TimeSeries({maxPoints: 10}); + const tdTimeSeries = new TimeSeries({maxPoints: 50}); + const SLOTS_PER_SYNC_COMMITTEE_PERIOD = SLOTS_PER_EPOCH * EPOCHS_PER_SYNC_COMMITTEE_PERIOD; - const timeSeries = new TimeSeries({maxPoints: 10}); let hasLowPeerCount = false; // Only log once try { @@ -54,32 +56,41 @@ export async function runNodeNotifier({ const finalizedEpoch = headState.finalizedCheckpoint.epoch; const finalizedRoot = headState.finalizedCheckpoint.root; const headSlot = headInfo.slot; - timeSeries.addPoint(headSlot, Date.now()); + headSlotTimeSeries.addPoint(headSlot, Math.floor(Date.now() / 1000)); const peersRow = `peers: ${connectedPeerCount}`; const finalizedCheckpointRow = `finalized: ${prettyBytes(finalizedRoot)}:${finalizedEpoch}`; const headRow = `head: ${headInfo.slot} ${prettyBytes(headInfo.blockRoot)}`; - const executionInfo = - isBellatrixCachedStateType(headState) && isMergeTransitionComplete(headState) - ? [ - `execution: ${headInfo.executionStatus.toLowerCase()}(${prettyBytes( - headInfo.executionPayloadBlockHash ?? "empty" - )})`, - ] - : []; // Give info about empty slots if head < clock const skippedSlots = clockSlot - headInfo.slot; const clockSlotRow = `slot: ${clockSlot}` + (skippedSlots > 0 ? ` (skipped ${skippedSlots})` : ""); + // Log in TD progress in separate line to not clutter regular status update. + // This line will only exist between BELLATRIX_FORK_EPOCH and TTD, a window of some days / weeks max. + // Notifier log lines must be kept at a reasonable max width otherwise it's very hard to read + const tdProgress = chain.eth1.getTDProgress(); + if (tdProgress !== null && !tdProgress.ttdHit) { + tdTimeSeries.addPoint(tdProgress.tdDiffScaled, tdProgress.timestamp); + + const timestampTDD = tdTimeSeries.computeY0Point(); + // It is possible to get ttd estimate with an error at imminent merge + const secToTTD = Math.max(Math.floor(timestampTDD - Date.now() / 1000), 0); + const timeLeft = isFinite(secToTTD) ? prettyTimeDiffSec(secToTTD) : "?"; + + logger.info(`TTD in ${timeLeft} current TD ${tdProgress.td} / ${tdProgress.ttd}`); + } + + const executionInfo = getExecutionInfo(config, clockEpoch, headState, headInfo); + let nodeState: string[]; switch (sync.state) { case SyncState.SyncingFinalized: case SyncState.SyncingHead: { - const slotsPerSecond = timeSeries.computeLinearSpeed(); + const slotsPerSecond = headSlotTimeSeries.computeLinearSpeed(); const distance = Math.max(clockSlot - headSlot, 0); const secondsLeft = distance / slotsPerSecond; - const timeLeft = isFinite(secondsLeft) ? prettyTimeDiff(1000 * secondsLeft) : "?"; + const timeLeft = isFinite(secondsLeft) ? prettyTimeDiffSec(secondsLeft) : "?"; // Syncing - time left - speed - head - finalized - clock - peers nodeState = [ "Syncing", @@ -134,3 +145,22 @@ function timeToNextHalfSlot(config: IBeaconConfig, chain: IBeaconChain): number const msToNextSlot = msPerSlot - (msFromGenesis % msPerSlot); return msToNextSlot > msPerSlot / 2 ? msToNextSlot - msPerSlot / 2 : msToNextSlot + msPerSlot / 2; } + +function getExecutionInfo( + config: IBeaconConfig, + clockEpoch: Epoch, + headState: CachedBeaconStateAllForks, + headInfo: ProtoBlock +): string[] { + if (clockEpoch < config.BELLATRIX_FORK_EPOCH) { + return []; + } else { + const executionStatusStr = headInfo.executionStatus.toLowerCase(); + + if (isBellatrixCachedStateType(headState) && isMergeTransitionComplete(headState)) { + return [`execution: ${executionStatusStr}(${prettyBytes(headInfo.executionPayloadBlockHash ?? "empty")})`]; + } else { + return [`execution: ${executionStatusStr}`]; + } + } +} diff --git a/packages/beacon-node/src/util/enum.ts b/packages/beacon-node/src/util/enum.ts new file mode 100644 index 000000000000..eeb942d9d328 --- /dev/null +++ b/packages/beacon-node/src/util/enum.ts @@ -0,0 +1,17 @@ +/** + * Given an enum + * ```ts + * enum A { + * a = "a", + * b = "b" + * } + * ``` + * returns + * ```ts + * { a: 0, + * b: 1 } + * ``` + */ +export function enumToIndexMap(enumVar: T): Record { + return Object.fromEntries(Object.keys(enumVar).map((key, i) => [key, i])) as Record; +} diff --git a/packages/beacon-node/src/util/time.ts b/packages/beacon-node/src/util/time.ts index 9227aa672570..c342e344b5a6 100644 --- a/packages/beacon-node/src/util/time.ts +++ b/packages/beacon-node/src/util/time.ts @@ -1,8 +1,7 @@ /** * Render a time difference in human readable form */ -export function prettyTimeDiff(diffMs: number): string { - const secDiff = diffMs / 1000; +export function prettyTimeDiffSec(secDiff: number): string { const minDiff = secDiff / 60; const hourDiff = minDiff / 60; const daysDiff = hourDiff / 24; diff --git a/packages/beacon-node/src/util/timeSeries.ts b/packages/beacon-node/src/util/timeSeries.ts index 523955857cda..45e8547bba4b 100644 --- a/packages/beacon-node/src/util/timeSeries.ts +++ b/packages/beacon-node/src/util/timeSeries.ts @@ -11,9 +11,9 @@ export class TimeSeries { } /** Add TimeSeries entry for value at current time */ - addPoint(value: number, timeMs = Date.now()): void { + addPoint(value: number, timeSec = Math.floor(Date.now() / 1000)): void { // Substract initial time so x values are not big and cause rounding errors - const time = timeMs / 1000 - this.startTimeSec; + const time = timeSec - this.startTimeSec; this.points.push([time, value]); // Limit length by removing old entries @@ -27,10 +27,25 @@ export class TimeSeries { return linearRegression(this.points).m; } + /** + * Compute x point at which y = 0. + * From eq `y = b + m*x` then solve for `0 = b + m*x` + */ + computeY0Point(): number { + const {m, b} = linearRegression(this.points); + // The X cordinate system has been shifted left by startTimeSec, so return the + // projection in original coordinated system + return -b / m + this.startTimeSec; + } + /** Remove all entries */ clear(): void { this.points = []; } + + numPoints(): number { + return this.points.length; + } } /** diff --git a/packages/beacon-node/test/e2e/eth1/eth1ForBlockProduction.test.ts b/packages/beacon-node/test/e2e/eth1/eth1ForBlockProduction.test.ts index fe336e38e5a6..37bf19e5229e 100644 --- a/packages/beacon-node/test/e2e/eth1/eth1ForBlockProduction.test.ts +++ b/packages/beacon-node/test/e2e/eth1/eth1ForBlockProduction.test.ts @@ -76,8 +76,6 @@ describe("eth1 / Eth1Provider", function () { logger, signal: controller.signal, eth1Provider, - clockEpoch: 0, - isMergeTransitionComplete: false, }); // Resolves when Eth1ForBlockProduction has fetched both blocks and deposits diff --git a/packages/beacon-node/test/e2e/eth1/eth1MergeBlockTracker.test.ts b/packages/beacon-node/test/e2e/eth1/eth1MergeBlockTracker.test.ts index c11899dc8d27..3b9c80476879 100644 --- a/packages/beacon-node/test/e2e/eth1/eth1MergeBlockTracker.test.ts +++ b/packages/beacon-node/test/e2e/eth1/eth1MergeBlockTracker.test.ts @@ -59,15 +59,14 @@ describe.skip("eth1 / Eth1MergeBlockTracker", function () { config, logger, signal: controller.signal, - clockEpoch: 0, - isMergeTransitionComplete: false, + metrics: null, }, eth1Provider as IEth1Provider ); // Wait for Eth1MergeBlockTracker to find at least one merge block while (!controller.signal.aborted) { - if (eth1MergeBlockTracker.getTerminalPowBlock()) break; + if (await eth1MergeBlockTracker.getTerminalPowBlock()) break; await sleep(500, controller.signal); } @@ -75,7 +74,7 @@ describe.skip("eth1 / Eth1MergeBlockTracker", function () { expect(eth1MergeBlockTracker["status"]).to.equal(StatusCode.FOUND, "Wrong StatusCode"); // Given the total difficulty offset the block that has TTD is the `difficultyOffset`nth block - const mergeBlock = eth1MergeBlockTracker.getTerminalPowBlock(); + const mergeBlock = await eth1MergeBlockTracker.getTerminalPowBlock(); if (!mergeBlock) throw Error("terminal pow block not found"); expect(mergeBlock.totalDifficulty).to.equal( quantityToBigint(latestBlock.totalDifficulty), @@ -96,15 +95,14 @@ describe.skip("eth1 / Eth1MergeBlockTracker", function () { config: getConfig(terminalTotalDifficulty), logger, signal: controller.signal, - clockEpoch: 0, - isMergeTransitionComplete: false, + metrics: null, }, eth1Provider as IEth1Provider ); // Wait for Eth1MergeBlockTracker to find at least one merge block while (!controller.signal.aborted) { - if (eth1MergeBlockTracker.getTerminalPowBlock()) break; + if (await eth1MergeBlockTracker.getTerminalPowBlock()) break; await sleep(500, controller.signal); } @@ -112,7 +110,7 @@ describe.skip("eth1 / Eth1MergeBlockTracker", function () { expect(eth1MergeBlockTracker["status"]).to.equal(StatusCode.FOUND, "Wrong StatusCode"); // Given the total difficulty offset the block that has TTD is the `difficultyOffset`nth block - const mergeBlock = eth1MergeBlockTracker.getTerminalPowBlock(); + const mergeBlock = await eth1MergeBlockTracker.getTerminalPowBlock(); if (!mergeBlock) throw Error("mergeBlock not found"); expect(mergeBlock.totalDifficulty >= terminalTotalDifficulty).to.equal( true, @@ -133,15 +131,14 @@ describe.skip("eth1 / Eth1MergeBlockTracker", function () { config: getConfig(terminalTotalDifficulty), logger, signal: controller.signal, - clockEpoch: 0, - isMergeTransitionComplete: false, + metrics: null, }, eth1Provider as IEth1Provider ); // Wait for Eth1MergeBlockTracker to find at least one merge block while (!controller.signal.aborted) { - if (eth1MergeBlockTracker.getTerminalPowBlock()) break; + if (await eth1MergeBlockTracker.getTerminalPowBlock()) break; await sleep(500, controller.signal); } @@ -149,7 +146,7 @@ describe.skip("eth1 / Eth1MergeBlockTracker", function () { expect(eth1MergeBlockTracker["status"]).to.equal(StatusCode.FOUND, "Wrong StatusCode"); // Given the total difficulty offset the block that has TTD is the `difficultyOffset`nth block - const mergeBlock = eth1MergeBlockTracker.getTerminalPowBlock(); + const mergeBlock = await eth1MergeBlockTracker.getTerminalPowBlock(); if (!mergeBlock) throw Error("mergeBlock not found"); expect(mergeBlock.totalDifficulty >= terminalTotalDifficulty).to.equal( true, diff --git a/packages/beacon-node/test/spec/presets/fork_choice.ts b/packages/beacon-node/test/spec/presets/fork_choice.ts index 57045e656628..702c845471a6 100644 --- a/packages/beacon-node/test/spec/presets/fork_choice.ts +++ b/packages/beacon-node/test/spec/presets/fork_choice.ts @@ -11,7 +11,7 @@ import {createCachedBeaconStateTest} from "../../utils/cachedBeaconState.js"; import {testLogger} from "../../utils/logger.js"; import {getConfig} from "../utils/getConfig.js"; import {TestRunnerFn} from "../utils/types.js"; -import {IEth1ForBlockProduction} from "../../../src/eth1/index.js"; +import {Eth1ForBlockProductionDisabled} from "../../../src/eth1/index.js"; import {ExecutionEngineMock} from "../../../src/execution/index.js"; import {defaultChainOptions} from "../../../src/chain/options.js"; import {getStubbedBeaconDb} from "../../utils/mocks/db.js"; @@ -347,19 +347,9 @@ function isCheck(step: Step): step is Checks { return typeof (step as Checks).checks === "object"; } -class Eth1ForBlockProductionMock implements IEth1ForBlockProduction { +// Extend Eth1ForBlockProductionDisabled to not have to re-implement new methods +class Eth1ForBlockProductionMock extends Eth1ForBlockProductionDisabled { private items = new Map(); - async getEth1DataAndDeposits(): Promise { - throw Error("Not implemented"); - } - - getTerminalPowBlock(): never { - throw Error("Not implemented"); - } - - mergeCompleted(): never { - throw Error("Not implemented"); - } async getPowBlock(powBlockHash: string): Promise { return this.items.get(powBlockHash) ?? null; diff --git a/packages/beacon-node/test/unit/eth1/eth1MergeBlockTracker.test.ts b/packages/beacon-node/test/unit/eth1/eth1MergeBlockTracker.test.ts index 02aa8c3388fc..16e6993b5905 100644 --- a/packages/beacon-node/test/unit/eth1/eth1MergeBlockTracker.test.ts +++ b/packages/beacon-node/test/unit/eth1/eth1MergeBlockTracker.test.ts @@ -64,23 +64,23 @@ describe("eth1 / Eth1MergeBlockTracker", () => { config, logger, signal: controller.signal, - clockEpoch: 0, - isMergeTransitionComplete: false, + metrics: null, }, eth1Provider as IEth1Provider ); + eth1MergeBlockTracker.startPollingMergeBlock(); // Wait for Eth1MergeBlockTracker to find at least one merge block while (!controller.signal.aborted) { - if (eth1MergeBlockTracker.getTerminalPowBlock()) break; + if (await eth1MergeBlockTracker.getTerminalPowBlock()) break; await sleep(10, controller.signal); } // Status should acknowlege merge block is found - expect(eth1MergeBlockTracker["status"]).to.equal(StatusCode.FOUND, "Wrong StatusCode"); + expect(eth1MergeBlockTracker["status"].code).to.equal(StatusCode.FOUND, "Wrong StatusCode"); // Given the total difficulty offset the block that has TTD is the `difficultyOffset`nth block - expect(eth1MergeBlockTracker.getTerminalPowBlock()).to.deep.equal( + expect(await eth1MergeBlockTracker.getTerminalPowBlock()).to.deep.equal( terminalPowBlock, "Wrong found terminal pow block" ); @@ -89,25 +89,24 @@ describe("eth1 / Eth1MergeBlockTracker", () => { it("Should find terminal pow block polling future 'latest' blocks", async () => { // Set current network totalDifficulty to behind terminalTotalDifficulty by 5. // Then on each call to getBlockByNumber("latest") increase totalDifficulty by 1. - const difficultyOffset = 5; - const totalDifficulty = terminalTotalDifficulty - difficultyOffset; + const numOfBlocks = 5; + const difficulty = 1; let latestBlockPointer = 0; + const blocks: EthJsonRpcBlockRaw[] = []; const blocksByHash = new Map(); - function getLatestBlock(i: number): EthJsonRpcBlockRaw { + for (let i = 0; i < numOfBlocks + 1; i++) { const block: EthJsonRpcBlockRaw = { number: toHex(i), hash: toRootHex(i + 1), parentHash: toRootHex(i), // Latest block is under TTD, so past block search is stopped - totalDifficulty: toHex(totalDifficulty + i), + totalDifficulty: toHex(terminalTotalDifficulty - numOfBlocks * difficulty + i * difficulty), timestamp: "0x0", }; blocks.push(block); - blocksByHash.set(block.hash, block); - return block; } const eth1Provider: IEth1Provider = { @@ -115,7 +114,13 @@ describe("eth1 / Eth1MergeBlockTracker", () => { getBlockNumber: async () => 0, getBlockByNumber: async (blockNumber) => { // On each call simulate that the eth1 chain advances 1 block with +1 totalDifficulty - if (blockNumber === "latest") return getLatestBlock(latestBlockPointer++); + if (blockNumber === "latest") { + if (latestBlockPointer >= blocks.length) { + throw Error("Fetched too many blocks"); + } else { + return blocks[latestBlockPointer++]; + } + } return blocks[blockNumber]; }, getBlockByHash: async (blockHashHex) => blocksByHash.get(blockHashHex) ?? null, @@ -130,67 +135,77 @@ describe("eth1 / Eth1MergeBlockTracker", () => { }, }; - const eth1MergeBlockTracker = new Eth1MergeBlockTracker( - { - config, - logger, - signal: controller.signal, - clockEpoch: 0, - isMergeTransitionComplete: false, - }, - eth1Provider as IEth1Provider - ); + await runFindMergeBlockTest(eth1Provider, blocks[blocks.length - 1]); + }); - // Wait for Eth1MergeBlockTracker to find at least one merge block - while (!controller.signal.aborted) { - if (eth1MergeBlockTracker.getTerminalPowBlock()) break; - await sleep(10, controller.signal); + it("Should find terminal pow block fetching past blocks", async () => { + // Set current network totalDifficulty to behind terminalTotalDifficulty by 5. + // Then on each call to getBlockByNumber("latest") increase totalDifficulty by 1. + + const numOfBlocks = 5; + const difficulty = 1; + const ttdOffset = 1 * difficulty; + const hashOffset = 100; + const blocks: EthJsonRpcBlockRaw[] = []; + + for (let i = 0; i < numOfBlocks * 2; i++) { + const block: EthJsonRpcBlockRaw = { + number: toHex(hashOffset + i), + hash: toRootHex(hashOffset + i + 1), + parentHash: toRootHex(hashOffset + i), + // Latest block is under TTD, so past block search is stopped + totalDifficulty: toHex(terminalTotalDifficulty + i * difficulty - ttdOffset), + timestamp: "0x0", + }; + blocks.push(block); } - // Status should acknowlege merge block is found - expect(eth1MergeBlockTracker["status"]).to.equal(StatusCode.FOUND, "Wrong StatusCode"); + // Before last block (with ttdOffset = 1) is the merge block + const expectedMergeBlock = blocks[ttdOffset]; - // Given the total difficulty offset the block that has TTD is the `difficultyOffset`nth block - expect(eth1MergeBlockTracker.getTerminalPowBlock()).to.deep.equal( - toPowBlock(blocks[difficultyOffset]), - "Wrong found terminal pow block" - ); + const eth1Provider = mockEth1ProviderFromBlocks(blocks); + await runFindMergeBlockTest(eth1Provider, expectedMergeBlock); }); - it("Should find terminal pow block fetching past blocks", async () => { - // Set current network totalDifficulty to behind terminalTotalDifficulty by 5. - // Then on each call to getBlockByNumber("latest") increase totalDifficulty by 1. - const difficultyOffset = 5; - const totalDifficulty = terminalTotalDifficulty - difficultyOffset; + it("Should find terminal pow block fetching past blocks till genesis", async () => { + // There's no block with TD < TTD, searcher should stop at genesis block + const numOfBlocks = 5; + const difficulty = 1; const blocks: EthJsonRpcBlockRaw[] = []; - const blocksByHash = new Map(); - for (let i = 0; i < difficultyOffset * 2; i++) { + for (let i = 0; i < numOfBlocks * 2; i++) { const block: EthJsonRpcBlockRaw = { number: toHex(i), hash: toRootHex(i + 1), parentHash: toRootHex(i), // Latest block is under TTD, so past block search is stopped - totalDifficulty: toHex(totalDifficulty + i), + totalDifficulty: toHex(terminalTotalDifficulty + i * difficulty + 1), timestamp: "0x0", }; blocks.push(block); - blocksByHash.set(block.hash, block); } - // Return a latest block that's over TTD but its parent doesn't exit to cancel future searches - const latestBlock: EthJsonRpcBlockRaw = { - ...blocks[blocks.length - 1], - parentHash: toRootHex(0xffffffff), - }; + // Merge block must be genesis block + const expectedMergeBlock = blocks[0]; - const eth1Provider: IEth1Provider = { + const eth1Provider = mockEth1ProviderFromBlocks(blocks); + await runFindMergeBlockTest(eth1Provider, expectedMergeBlock); + }); + + function mockEth1ProviderFromBlocks(blocks: EthJsonRpcBlockRaw[]): IEth1Provider { + const blocksByHash = new Map(); + + for (const block of blocks) { + blocksByHash.set(block.hash, block); + } + + return { deployBlock: 0, getBlockNumber: async () => 0, getBlockByNumber: async (blockNumber) => { // Always return the same block with totalDifficulty > TTD and unknown parent - if (blockNumber === "latest") return latestBlock; + if (blockNumber === "latest") return blocks[blocks.length - 1]; return blocks[blockNumber]; }, getBlockByHash: async (blockHashHex) => blocksByHash.get(blockHashHex) ?? null, @@ -202,33 +217,38 @@ describe("eth1 / Eth1MergeBlockTracker", () => { throw Error("Not implemented"); }, }; + } + async function runFindMergeBlockTest( + eth1Provider: IEth1Provider, + expectedMergeBlock: EthJsonRpcBlockRaw + ): Promise { const eth1MergeBlockTracker = new Eth1MergeBlockTracker( { config, logger, signal: controller.signal, - clockEpoch: 0, - isMergeTransitionComplete: false, + metrics: null, }, eth1Provider as IEth1Provider ); + eth1MergeBlockTracker.startPollingMergeBlock(); // Wait for Eth1MergeBlockTracker to find at least one merge block while (!controller.signal.aborted) { - if (eth1MergeBlockTracker.getTerminalPowBlock()) break; + if (await eth1MergeBlockTracker.getTerminalPowBlock()) break; await sleep(10, controller.signal); } // Status should acknowlege merge block is found - expect(eth1MergeBlockTracker["status"]).to.equal(StatusCode.FOUND, "Wrong StatusCode"); + expect(eth1MergeBlockTracker["status"].code).to.equal(StatusCode.FOUND, "Wrong StatusCode"); // Given the total difficulty offset the block that has TTD is the `difficultyOffset`nth block - expect(eth1MergeBlockTracker.getTerminalPowBlock()).to.deep.equal( - toPowBlock(blocks[difficultyOffset]), + expect(await eth1MergeBlockTracker.getTerminalPowBlock()).to.deep.equal( + toPowBlock(expectedMergeBlock), "Wrong found terminal pow block" ); - }); + } }); function toHex(num: number | bigint): string { diff --git a/packages/beacon-node/test/unit/util/time.test.ts b/packages/beacon-node/test/unit/util/time.test.ts index f635d5941a8b..2fa50f27df15 100644 --- a/packages/beacon-node/test/unit/util/time.test.ts +++ b/packages/beacon-node/test/unit/util/time.test.ts @@ -1,21 +1,21 @@ import {expect} from "chai"; -import {prettyTimeDiff} from "../../../src/util/time.js"; +import {prettyTimeDiffSec} from "../../../src/util/time.js"; -describe("util / time / prettyTimeDiff", () => { - const testCases: {diffMs: number; res: string}[] = [ - {diffMs: 1500, res: "1.5 seconds"}, - {diffMs: 15000, res: "15 seconds"}, - {diffMs: 100000, res: "1.7 minutes"}, - {diffMs: 1000000, res: "17 minutes"}, - {diffMs: 10000000, res: "2.8 hours"}, - {diffMs: 50000000, res: "14 hours"}, - {diffMs: 100000000, res: "1.2 days"}, - {diffMs: 500000000, res: "5.8 days"}, +describe("util / time / prettyTimeDiffSec", () => { + const testCases: {diffSec: number; res: string}[] = [ + {diffSec: 1.5, res: "1.5 seconds"}, + {diffSec: 15, res: "15 seconds"}, + {diffSec: 100, res: "1.7 minutes"}, + {diffSec: 1000, res: "17 minutes"}, + {diffSec: 10000, res: "2.8 hours"}, + {diffSec: 50000, res: "14 hours"}, + {diffSec: 100000, res: "1.2 days"}, + {diffSec: 500000, res: "5.8 days"}, ]; - for (const {diffMs, res} of testCases) { - it(`pretty ${diffMs}`, () => { - expect(prettyTimeDiff(diffMs)).to.equal(res); + for (const {diffSec, res} of testCases) { + it(`pretty ${diffSec}`, () => { + expect(prettyTimeDiffSec(diffSec)).to.equal(res); }); } }); diff --git a/packages/beacon-node/test/unit/util/timeSeries.test.ts b/packages/beacon-node/test/unit/util/timeSeries.test.ts index 1e114493d69c..727f8a91b250 100644 --- a/packages/beacon-node/test/unit/util/timeSeries.test.ts +++ b/packages/beacon-node/test/unit/util/timeSeries.test.ts @@ -8,9 +8,9 @@ describe.skip("util / TimeSeries", () => { it("Should correctly compute a linear sequence", () => { const timeSeries = new TimeSeries(); - const startTime = 1610190386014; + const startTime = 1610190386; for (let i = 0; i < 4; i++) { - timeSeries.addPoint(100 + i, startTime + i * 1000); + timeSeries.addPoint(100 + i, startTime + i); } const valuePerSec = timeSeries.computeLinearSpeed(); @@ -21,10 +21,10 @@ describe.skip("util / TimeSeries", () => { it("Should correctly do a linear regression", () => { const timeSeries = new TimeSeries(); - const startTime = 1610190386014; + const startTime = 1610190386; for (let i = 1; i < 10; i++) { // Add +1 or -1 so points are not in a perfect line but a regression should return 1 - timeSeries.addPoint(100 + i + Math.pow(-1, i), startTime + i * 1000); + timeSeries.addPoint(100 + i + Math.pow(-1, i), startTime + i); } const valuePerSec = timeSeries.computeLinearSpeed();