diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 671434fe19ad..e515bef7f92a 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -33,7 +33,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: - node-version: 22.4 + node-version: 22 check-latest: true cache: yarn - name: Node.js version diff --git a/.github/workflows/binaries.yml b/.github/workflows/binaries.yml index 722894424b91..469b0803c378 100644 --- a/.github/workflows/binaries.yml +++ b/.github/workflows/binaries.yml @@ -42,7 +42,7 @@ jobs: sudo apt-get install -y build-essential python3 - uses: "./.github/actions/setup-and-build" with: - node: 22.4 + node: 22 - run: | mkdir -p dist yarn global add caxa@3.0.1 diff --git a/.github/workflows/docs-check.yml b/.github/workflows/docs-check.yml index bd7310995d62..180f1c16fdaa 100644 --- a/.github/workflows/docs-check.yml +++ b/.github/workflows/docs-check.yml @@ -15,7 +15,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: - node-version: 22.4 + node-version: 22 cache: yarn - name: Node.js version id: node diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index a4c0f18cdbe3..cec2bca86b4b 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -31,7 +31,7 @@ jobs: - uses: actions/setup-node@v4 with: - node-version: 22.4 + node-version: 22 check-latest: true cache: yarn diff --git a/.github/workflows/publish-dev.yml b/.github/workflows/publish-dev.yml index 4e8c76e0dfdd..cb08c5c8fae0 100644 --- a/.github/workflows/publish-dev.yml +++ b/.github/workflows/publish-dev.yml @@ -20,7 +20,7 @@ jobs: fetch-depth: 0 - uses: actions/setup-node@v4 with: - node-version: 22.4 + node-version: 22 registry-url: "https://registry.npmjs.org" check-latest: true cache: yarn diff --git a/.github/workflows/publish-rc.yml b/.github/workflows/publish-rc.yml index 936072de42c9..e16cbce814dc 100644 --- a/.github/workflows/publish-rc.yml +++ b/.github/workflows/publish-rc.yml @@ -61,7 +61,7 @@ jobs: - uses: "./.github/actions/setup-and-build" with: - node: 22.4 + node: 22 - name: Generate changelog run: node scripts/generate_changelog.mjs ${{ needs.tag.outputs.prev_tag }} ${{ needs.tag.outputs.tag }} CHANGELOG.md diff --git a/.github/workflows/publish-stable.yml b/.github/workflows/publish-stable.yml index c2909a7e4e24..26fee0ba6052 100644 --- a/.github/workflows/publish-stable.yml +++ b/.github/workflows/publish-stable.yml @@ -67,7 +67,7 @@ jobs: - uses: "./.github/actions/setup-and-build" with: - node: 22.4 + node: 22 - name: Generate changelog run: node scripts/generate_changelog.mjs ${{ needs.tag.outputs.prev_tag }} ${{ needs.tag.outputs.tag }} CHANGELOG.md diff --git a/.github/workflows/test-sim-merge.yml b/.github/workflows/test-sim-merge.yml index 0042a9337bc3..ad79bc2c0035 100644 --- a/.github/workflows/test-sim-merge.yml +++ b/.github/workflows/test-sim-merge.yml @@ -30,7 +30,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: - node-version: 22.4 + node-version: 22 check-latest: true cache: yarn - name: Node.js version diff --git a/.github/workflows/test-sim.yml b/.github/workflows/test-sim.yml index fbe2691da637..ff28149537d3 100644 --- a/.github/workflows/test-sim.yml +++ b/.github/workflows/test-sim.yml @@ -31,7 +31,7 @@ jobs: - uses: actions/checkout@v4 - uses: "./.github/actions/setup-and-build" with: - node: 22.4 + node: 22 sim-test-multifork: name: Multifork sim test @@ -42,7 +42,7 @@ jobs: - uses: actions/checkout@v4 - uses: "./.github/actions/setup-and-build" with: - node: 22.4 + node: 22 - name: Load env variables uses: ./.github/actions/dotenv - name: Download required docker images before running tests @@ -71,7 +71,7 @@ jobs: - uses: actions/checkout@v4 - uses: "./.github/actions/setup-and-build" with: - node: 22.4 + node: 22 - name: Load env variables uses: ./.github/actions/dotenv - name: Download required docker images before running tests @@ -100,7 +100,7 @@ jobs: - uses: actions/checkout@v4 - uses: "./.github/actions/setup-and-build" with: - node: 22.4 + node: 22 - name: Load env variables uses: ./.github/actions/dotenv - name: Download required docker images before running tests @@ -129,7 +129,7 @@ jobs: - uses: actions/checkout@v4 - uses: "./.github/actions/setup-and-build" with: - node: 22.4 + node: 22 - name: Load env variables uses: ./.github/actions/dotenv - name: Download required docker images before running tests @@ -158,7 +158,7 @@ jobs: - uses: actions/checkout@v4 - uses: "./.github/actions/setup-and-build" with: - node: 22.4 + node: 22 - name: Load env variables uses: ./.github/actions/dotenv - name: Download required docker images before running tests diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5514f6e896b7..47e17c56f042 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,7 +19,7 @@ jobs: strategy: fail-fast: false matrix: - node: [22.4] + node: [22] steps: # - Uses YAML anchors in the future - uses: actions/checkout@v4 @@ -42,7 +42,7 @@ jobs: strategy: fail-fast: false matrix: - node: [22.4] + node: [22] steps: - uses: actions/checkout@v4 - uses: "./.github/actions/setup-and-build" @@ -71,7 +71,7 @@ jobs: strategy: fail-fast: false matrix: - node: [22.4] + node: [22] steps: - uses: actions/checkout@v4 @@ -92,7 +92,7 @@ jobs: strategy: fail-fast: false matrix: - node: [22.4] + node: [22] steps: - uses: actions/checkout@v4 - uses: "./.github/actions/setup-and-build" @@ -131,7 +131,7 @@ jobs: strategy: fail-fast: false matrix: - node: [22.4] + node: [22] steps: # - Uses YAML anchors in the future - uses: actions/checkout@v4 @@ -168,7 +168,7 @@ jobs: strategy: fail-fast: false matrix: - node: [22.4] + node: [22] steps: # - Uses YAML anchors in the future - uses: actions/checkout@v4 @@ -192,7 +192,7 @@ jobs: strategy: fail-fast: false matrix: - node: [22.4] + node: [22] steps: - uses: actions/checkout@v4 - uses: "./.github/actions/setup-and-build" diff --git a/.gitignore b/.gitignore index 52d9bc66e5b6..e54f7440864d 100644 --- a/.gitignore +++ b/.gitignore @@ -52,6 +52,7 @@ docs/pages/libraries/lightclient-prover/lightclient.md docs/pages/libraries/lightclient-prover/prover.md docs/pages/api/api-reference.md docs/pages/contribution/getting-started.md +docs/static/install ## Docusaurus docs/.docusaurus/ docs/build/ diff --git a/.vscode/settings.json b/.vscode/settings.json index fbc0552fe61c..d36535917835 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,6 +1,5 @@ { "window.title": "${activeEditorShort}${separator}${rootName}${separator}${profileName}${separator}[${activeRepositoryBranchName}]", - "editor.defaultFormatter": "esbenp.prettier-vscode", // For `sysoev.vscode-open-in-github` extension "openInGitHub.defaultBranch": "unstable", "editor.formatOnSave": true, @@ -18,5 +17,11 @@ }, "[jsonc]": { "editor.defaultFormatter": "biomejs.biome" - } + }, + "[yaml]": { + "editor.defaultFormatter": "esbenp.prettier-vscode" + }, + "[markdown]": { + "editor.defaultFormatter": "esbenp.prettier-vscode" + }, } diff --git a/.wordlist.txt b/.wordlist.txt index 2c28fc784061..5070aafcf9a5 100644 --- a/.wordlist.txt +++ b/.wordlist.txt @@ -110,6 +110,7 @@ Vitalik Vitest Wagyu api +args async backfill beaconcha @@ -244,3 +245,4 @@ xRelayPubKey xcode yaml yamux +yml diff --git a/Dockerfile b/Dockerfile index 0ee8083c85e2..5a1f51bfcee6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # --platform=$BUILDPLATFORM is used build javascript source with host arch # Otherwise TS builds on emulated archs and can be extremely slow (+1h) -FROM --platform=${BUILDPLATFORM:-amd64} node:22.4-slim AS build_src +FROM --platform=${BUILDPLATFORM:-amd64} node:22-slim AS build_src ARG COMMIT WORKDIR /usr/app RUN apt-get update && apt-get install -y g++ make python3 python3-setuptools && apt-get clean && rm -rf /var/lib/apt/lists/* @@ -21,7 +21,7 @@ RUN cd packages/cli && GIT_COMMIT=${COMMIT} yarn write-git-data # Copy built src + node_modules to build native packages for archs different than host. # Note: This step is redundant for the host arch -FROM node:22.4-slim AS build_deps +FROM node:22-slim AS build_deps WORKDIR /usr/app RUN apt-get update && apt-get install -y g++ make python3 python3-setuptools && apt-get clean && rm -rf /var/lib/apt/lists/* @@ -35,7 +35,7 @@ RUN cd node_modules/classic-level && yarn rebuild # Copy built src + node_modules to a new layer to prune unnecessary fs # Previous layer weights 7.25GB, while this final 488MB (as of Oct 2020) -FROM node:22.4-slim +FROM node:22-slim WORKDIR /usr/app COPY --from=build_deps /usr/app . diff --git a/docs/pages/run/getting-started/installation.md b/docs/pages/run/getting-started/installation.md index 40c4865f7726..b2b4003f90a5 100644 --- a/docs/pages/run/getting-started/installation.md +++ b/docs/pages/run/getting-started/installation.md @@ -4,6 +4,12 @@ Binaries can be downloaded from the Lodestar [release page](https://github.com/ChainSafe/lodestar/releases/latest) under the `Assets` section. +Run the following command to install the latest version + +```bash +curl -fsSL https://chainsafe.github.io/lodestar/install | bash +``` + ## Docker Installation The [`chainsafe/lodestar`](https://hub.docker.com/r/chainsafe/lodestar) Docker Hub repository is maintained actively. It contains the `lodestar` CLI preinstalled. diff --git a/packages/api/src/beacon/routes/lodestar.ts b/packages/api/src/beacon/routes/lodestar.ts index 81191efd2696..de05b7f2bb44 100644 --- a/packages/api/src/beacon/routes/lodestar.ts +++ b/packages/api/src/beacon/routes/lodestar.ts @@ -1,8 +1,11 @@ +import {ContainerType, ValueOf} from "@chainsafe/ssz"; import {ChainForkConfig} from "@lodestar/config"; -import {Epoch, RootHex, Slot} from "@lodestar/types"; +import {Epoch, RootHex, Slot, ssz} from "@lodestar/types"; import { + ArrayOf, EmptyArgs, EmptyMeta, + EmptyMetaCodec, EmptyRequest, EmptyRequestCodec, EmptyResponseCodec, @@ -10,6 +13,7 @@ import { JsonOnlyResponseCodec, } from "../../utils/codecs.js"; import {Endpoint, RouteDefinitions, Schema} from "../../utils/index.js"; +import {StateArgs} from "./beacon/state.js"; import {FilterGetPeers, NodePeer, PeerDirection, PeerState} from "./node.js"; // See /packages/api/src/routes/index.ts for reasoning and instructions to add new routes @@ -75,6 +79,16 @@ export type LodestarNodePeer = NodePeer & { export type LodestarThreadType = "main" | "network" | "discv5"; +const HistoricalSummariesResponseType = new ContainerType( + { + historicalSummaries: ssz.capella.HistoricalSummaries, + proof: ArrayOf(ssz.Bytes8), + }, + {jsonCase: "eth2"} +); + +export type HistoricalSummariesResponse = ValueOf; + export type Endpoints = { /** Trigger to write a heapdump to disk at `dirpath`. May take > 1min */ writeHeapdump: Endpoint< @@ -214,6 +228,16 @@ export type Endpoints = { {count: number} >; + /** Returns historical summaries and proof for a given state ID */ + getHistoricalSummaries: Endpoint< + // ⏎ + "GET", + StateArgs, + {params: {state_id: string}}, + HistoricalSummariesResponse, + EmptyMeta + >; + /** Dump Discv5 Kad values */ discv5GetKadValues: Endpoint< // ⏎ @@ -365,6 +389,21 @@ export function getDefinitions(_config: ChainForkConfig): RouteDefinitions ({params: {state_id: stateId.toString()}}), + parseReq: ({params}) => ({stateId: params.state_id}), + schema: { + params: {state_id: Schema.StringRequired}, + }, + }, + resp: { + data: HistoricalSummariesResponseType, + meta: EmptyMetaCodec, + }, + }, discv5GetKadValues: { url: "/eth/v1/debug/discv5_kad_values", method: "GET", diff --git a/packages/api/test/unit/beacon/genericServerTest/lodestar.test.ts b/packages/api/test/unit/beacon/genericServerTest/lodestar.test.ts new file mode 100644 index 000000000000..b0f78f4bd62c --- /dev/null +++ b/packages/api/test/unit/beacon/genericServerTest/lodestar.test.ts @@ -0,0 +1,53 @@ +import {config} from "@lodestar/config/default"; +import {FastifyInstance} from "fastify"; +import {afterAll, beforeAll, describe, expect, it, vi} from "vitest"; +import {getClient} from "../../../../src/beacon/client/lodestar.js"; +import {Endpoints, getDefinitions} from "../../../../src/beacon/routes/lodestar.js"; +import {getRoutes} from "../../../../src/beacon/server/lodestar.js"; +import {HttpClient} from "../../../../src/utils/client/httpClient.js"; +import {AnyEndpoint} from "../../../../src/utils/codecs.js"; +import {FastifyRoute} from "../../../../src/utils/server/index.js"; +import {WireFormat} from "../../../../src/utils/wireFormat.js"; +import {getMockApi, getTestServer} from "../../../utils/utils.js"; + +describe("beacon / lodestar", () => { + describe("get HistoricalSummaries as json", () => { + const mockApi = getMockApi(getDefinitions(config)); + let baseUrl: string; + let server: FastifyInstance; + + beforeAll(async () => { + const res = getTestServer(); + server = res.server; + for (const route of Object.values(getRoutes(config, mockApi))) { + server.route(route as FastifyRoute); + } + baseUrl = await res.start(); + }); + + afterAll(async () => { + if (server !== undefined) await server.close(); + }); + + it("getHistoricalSummaries", async () => { + mockApi.getHistoricalSummaries.mockResolvedValue({ + data: { + historicalSummaries: [], + proof: [], + }, + }); + + const httpClient = new HttpClient({baseUrl}); + const client = getClient(config, httpClient); + + const res = await client.getHistoricalSummaries({stateId: "head"}, {responseWireFormat: WireFormat.json}); + + expect(res.ok).toBe(true); + expect(res.wireFormat()).toBe(WireFormat.json); + expect(res.json().data).toStrictEqual({ + historical_summaries: [], + proof: [], + }); + }); + }); +}); diff --git a/packages/beacon-node/src/api/impl/beacon/pool/index.ts b/packages/beacon-node/src/api/impl/beacon/pool/index.ts index 763d7036cba5..d0c35b1c0cd8 100644 --- a/packages/beacon-node/src/api/impl/beacon/pool/index.ts +++ b/packages/beacon-node/src/api/impl/beacon/pool/index.ts @@ -278,7 +278,7 @@ export function getBeaconPoolApi({ } failures.push({index: i, message: (e as Error).message}); - logger.debug( + logger.error( `Error on submitPoolSyncCommitteeSignatures [${i}]`, {slot: signature.slot, validatorIndex: signature.validatorIndex}, e as Error diff --git a/packages/beacon-node/src/api/impl/debug/index.ts b/packages/beacon-node/src/api/impl/debug/index.ts index e6c104272237..a004bd80e8f2 100644 --- a/packages/beacon-node/src/api/impl/debug/index.ts +++ b/packages/beacon-node/src/api/impl/debug/index.ts @@ -2,7 +2,7 @@ import {routes} from "@lodestar/api"; import {ApplicationMethods} from "@lodestar/api/server"; import {ExecutionStatus} from "@lodestar/fork-choice"; import {ZERO_HASH_HEX} from "@lodestar/params"; -import {BeaconState} from "@lodestar/types"; +import {BeaconState, ssz} from "@lodestar/types"; import {isOptimisticBlock} from "../../../util/forkChoice.js"; import {getStateSlotFromBytes} from "../../../util/multifork.js"; import {getStateResponseWithRegen} from "../beacon/state/utils.js"; diff --git a/packages/beacon-node/src/api/impl/lodestar/index.ts b/packages/beacon-node/src/api/impl/lodestar/index.ts index 10787194f5f5..aeef2e11a83e 100644 --- a/packages/beacon-node/src/api/impl/lodestar/index.ts +++ b/packages/beacon-node/src/api/impl/lodestar/index.ts @@ -1,11 +1,12 @@ import fs from "node:fs"; import path from "node:path"; +import {Tree} from "@chainsafe/persistent-merkle-tree"; import {routes} from "@lodestar/api"; import {ApplicationMethods} from "@lodestar/api/server"; import {ChainForkConfig} from "@lodestar/config"; import {Repository} from "@lodestar/db"; -import {SLOTS_PER_EPOCH} from "@lodestar/params"; -import {getLatestWeakSubjectivityCheckpointEpoch} from "@lodestar/state-transition"; +import {ForkSeq, SLOTS_PER_EPOCH} from "@lodestar/params"; +import {BeaconStateCapella, getLatestWeakSubjectivityCheckpointEpoch, loadState} from "@lodestar/state-transition"; import {ssz} from "@lodestar/types"; import {toHex, toRootHex} from "@lodestar/utils"; import {BeaconChain} from "../../../chain/index.js"; @@ -13,6 +14,7 @@ import {QueuedStateRegenerator, RegenRequest} from "../../../chain/regen/index.j import {IBeaconDb} from "../../../db/interface.js"; import {GossipType} from "../../../network/index.js"; import {profileNodeJS, writeHeapSnapshot} from "../../../util/profile.js"; +import {getStateResponseWithRegen} from "../beacon/state/utils.js"; import {ApiModules} from "../types.js"; export function getLodestarApi({ @@ -187,6 +189,29 @@ export function getLodestarApi({ async dumpDbStateIndex() { return {data: await db.stateArchive.dumpRootIndexEntries()}; }, + + async getHistoricalSummaries({stateId}) { + const {state} = await getStateResponseWithRegen(chain, stateId); + + const stateView = ( + state instanceof Uint8Array ? loadState(config, chain.getHeadState(), state).state : state.clone() + ) as BeaconStateCapella; + + const fork = config.getForkName(stateView.slot); + if (ForkSeq[fork] < ForkSeq.capella) { + throw new Error("Historical summaries are not supported before Capella"); + } + + const {gindex} = ssz[fork].BeaconState.getPathInfo(["historicalSummaries"]); + const proof = new Tree(stateView.node).getSingleProof(gindex); + + return { + data: { + historicalSummaries: stateView.historicalSummaries.toValue(), + proof: proof, + }, + }; + }, }; } diff --git a/packages/beacon-node/src/chain/blocks/utils/giraffeBanner.ts b/packages/beacon-node/src/chain/blocks/utils/giraffeBanner.ts new file mode 100644 index 000000000000..d3987831a064 --- /dev/null +++ b/packages/beacon-node/src/chain/blocks/utils/giraffeBanner.ts @@ -0,0 +1,27 @@ +export const ELECTRA_GIRAFFE_BANNER = String.raw` + 2048 + :--: + :-@==+-: + :-=++#+#++#> + :-=+=#+: + ::+*--@-*: + :-+=%*#%@-: + MAXEB**=+%*+: + :-*###+*#*=-: + :--=+*+==#*=-: + :-*=+#*=*-#*+%@%%%#*+: + -=+-+**#+#%%%*#@@+%%#=#% + 32 -*=*+#+=%*#%*#%#+*##***-: + : #+**+*+=*+*%*%%*==++@**=: + =++++=: ::----:: +=-@*: + +++=- -++ =+: + -=@ :+- -+ + :-: :+: :- + :+ := -= + := - @ + - @ : + -: -: - + *: :- =- + :- --: =: + ::*-: :::: :-: +`; diff --git a/packages/beacon-node/src/chain/blocks/verifyBlock.ts b/packages/beacon-node/src/chain/blocks/verifyBlock.ts index 5ead67a720f7..2ce78eb5f3f1 100644 --- a/packages/beacon-node/src/chain/blocks/verifyBlock.ts +++ b/packages/beacon-node/src/chain/blocks/verifyBlock.ts @@ -15,6 +15,7 @@ import {BlockProcessOpts} from "../options.js"; import {RegenCaller} from "../regen/index.js"; import {BlockInput, BlockInputType, ImportBlockOpts} from "./types.js"; import {DENEB_BLOWFISH_BANNER} from "./utils/blowfishBanner.js"; +import {ELECTRA_GIRAFFE_BANNER} from "./utils/giraffeBanner.js"; import {CAPELLA_OWL_BANNER} from "./utils/ownBanner.js"; import {POS_PANDA_MERGE_TRANSITION_BANNER} from "./utils/pandaMergeTransitionBanner.js"; import {verifyBlocksDataAvailability} from "./verifyBlocksDataAvailability.js"; @@ -157,6 +158,11 @@ export async function verifyBlocksInEpoch( this.logger.info("Activating blobs", {epoch: this.config.DENEB_FORK_EPOCH}); break; + case ForkName.electra: + this.logger.info(ELECTRA_GIRAFFE_BANNER); + this.logger.info("Activating maxEB", {epoch: this.config.ELECTRA_FORK_EPOCH}); + break; + default: } } diff --git a/packages/beacon-node/src/chain/prepareNextSlot.ts b/packages/beacon-node/src/chain/prepareNextSlot.ts index 40cadb3774c7..f78c1842bd78 100644 --- a/packages/beacon-node/src/chain/prepareNextSlot.ts +++ b/packages/beacon-node/src/chain/prepareNextSlot.ts @@ -114,7 +114,12 @@ export class PrepareNextSlotScheduler { // the slot 0 of next epoch will likely use this Previous Root Checkpoint state for state transition so we transfer cache here // the resulting state with cache will be cached in Checkpoint State Cache which is used for the upcoming block processing // for other slots dontTransferCached=true because we don't run state transition on this state - {dontTransferCache: !isEpochTransition}, + // + // Shuffling calculation will be done asynchronously when passing asyncShufflingCalculation=true. Shuffling will be queued in + // beforeProcessEpoch and should theoretically be ready immediately after the synchronous epoch transition finished and the + // event loop is free. In long periods of non-finality too many forks will cause the shufflingCache to throw an error for + // too many queued shufflings so only run async during normal epoch transition. See issue ChainSafe/lodestar#7244 + {dontTransferCache: !isEpochTransition, asyncShufflingCalculation: true}, RegenCaller.precomputeEpoch ); diff --git a/packages/beacon-node/src/chain/regen/interface.ts b/packages/beacon-node/src/chain/regen/interface.ts index b70fbc059875..b9a4e38b5b68 100644 --- a/packages/beacon-node/src/chain/regen/interface.ts +++ b/packages/beacon-node/src/chain/regen/interface.ts @@ -28,8 +28,12 @@ export enum RegenFnName { getCheckpointState = "getCheckpointState", } -export type StateCloneOpts = { +export type StateRegenerationOpts = { dontTransferCache: boolean; + /** + * Do not queue shuffling calculation async. Forces sync JIT calculation in afterProcessEpoch if not passed as `true` + */ + asyncShufflingCalculation?: boolean; }; export interface IStateRegenerator extends IStateRegeneratorInternal { @@ -56,7 +60,11 @@ export interface IStateRegeneratorInternal { * Return a valid pre-state for a beacon block * This will always return a state in the latest viable epoch */ - getPreState(block: BeaconBlock, opts: StateCloneOpts, rCaller: RegenCaller): Promise; + getPreState( + block: BeaconBlock, + opts: StateRegenerationOpts, + rCaller: RegenCaller + ): Promise; /** * Return a valid checkpoint state @@ -64,7 +72,7 @@ export interface IStateRegeneratorInternal { */ getCheckpointState( cp: phase0.Checkpoint, - opts: StateCloneOpts, + opts: StateRegenerationOpts, rCaller: RegenCaller ): Promise; @@ -74,12 +82,12 @@ export interface IStateRegeneratorInternal { getBlockSlotState( blockRoot: RootHex, slot: Slot, - opts: StateCloneOpts, + opts: StateRegenerationOpts, rCaller: RegenCaller ): Promise; /** * Return the exact state with `stateRoot` */ - getState(stateRoot: RootHex, rCaller: RegenCaller, opts?: StateCloneOpts): Promise; + getState(stateRoot: RootHex, rCaller: RegenCaller, opts?: StateRegenerationOpts): Promise; } diff --git a/packages/beacon-node/src/chain/regen/queued.ts b/packages/beacon-node/src/chain/regen/queued.ts index b5084d593356..9069b384fd58 100644 --- a/packages/beacon-node/src/chain/regen/queued.ts +++ b/packages/beacon-node/src/chain/regen/queued.ts @@ -8,7 +8,13 @@ import {JobItemQueue} from "../../util/queue/index.js"; import {CheckpointHex, toCheckpointHex} from "../stateCache/index.js"; import {BlockStateCache, CheckpointStateCache} from "../stateCache/types.js"; import {RegenError, RegenErrorCode} from "./errors.js"; -import {IStateRegenerator, IStateRegeneratorInternal, RegenCaller, RegenFnName, StateCloneOpts} from "./interface.js"; +import { + IStateRegenerator, + IStateRegeneratorInternal, + RegenCaller, + RegenFnName, + StateRegenerationOpts, +} from "./interface.js"; import {RegenModules, StateRegenerator} from "./regen.js"; const REGEN_QUEUE_MAX_LEN = 256; @@ -86,7 +92,7 @@ export class QueuedStateRegenerator implements IStateRegenerator { */ getPreStateSync( block: BeaconBlock, - opts: StateCloneOpts = {dontTransferCache: true} + opts: StateRegenerationOpts = {dontTransferCache: true} ): CachedBeaconStateAllForks | null { const parentRoot = toRootHex(block.parentRoot); const parentBlock = this.forkChoice.getBlockHex(parentRoot); @@ -212,7 +218,7 @@ export class QueuedStateRegenerator implements IStateRegenerator { */ async getPreState( block: BeaconBlock, - opts: StateCloneOpts, + opts: StateRegenerationOpts, rCaller: RegenCaller ): Promise { this.metrics?.regenFnCallTotal.inc({caller: rCaller, entrypoint: RegenFnName.getPreState}); @@ -231,7 +237,7 @@ export class QueuedStateRegenerator implements IStateRegenerator { async getCheckpointState( cp: phase0.Checkpoint, - opts: StateCloneOpts, + opts: StateRegenerationOpts, rCaller: RegenCaller ): Promise { this.metrics?.regenFnCallTotal.inc({caller: rCaller, entrypoint: RegenFnName.getCheckpointState}); @@ -256,7 +262,7 @@ export class QueuedStateRegenerator implements IStateRegenerator { async getBlockSlotState( blockRoot: RootHex, slot: Slot, - opts: StateCloneOpts, + opts: StateRegenerationOpts, rCaller: RegenCaller ): Promise { this.metrics?.regenFnCallTotal.inc({caller: rCaller, entrypoint: RegenFnName.getBlockSlotState}); @@ -268,7 +274,7 @@ export class QueuedStateRegenerator implements IStateRegenerator { async getState( stateRoot: RootHex, rCaller: RegenCaller, - opts: StateCloneOpts = {dontTransferCache: true} + opts: StateRegenerationOpts = {dontTransferCache: true} ): Promise { this.metrics?.regenFnCallTotal.inc({caller: rCaller, entrypoint: RegenFnName.getState}); diff --git a/packages/beacon-node/src/chain/regen/regen.ts b/packages/beacon-node/src/chain/regen/regen.ts index 073556d8162f..06d1cee71332 100644 --- a/packages/beacon-node/src/chain/regen/regen.ts +++ b/packages/beacon-node/src/chain/regen/regen.ts @@ -20,7 +20,7 @@ import {getCheckpointFromState} from "../blocks/utils/checkpoint.js"; import {ChainEvent, ChainEventEmitter} from "../emitter.js"; import {BlockStateCache, CheckpointStateCache} from "../stateCache/types.js"; import {RegenError, RegenErrorCode} from "./errors.js"; -import {IStateRegeneratorInternal, RegenCaller, StateCloneOpts} from "./interface.js"; +import {IStateRegeneratorInternal, RegenCaller, StateRegenerationOpts} from "./interface.js"; export type RegenModules = { db: IBeaconDb; @@ -51,7 +51,7 @@ export class StateRegenerator implements IStateRegeneratorInternal { */ async getPreState( block: BeaconBlock, - opts: StateCloneOpts, + opts: StateRegenerationOpts, regenCaller: RegenCaller ): Promise { const parentBlock = this.modules.forkChoice.getBlock(block.parentRoot); @@ -84,7 +84,7 @@ export class StateRegenerator implements IStateRegeneratorInternal { */ async getCheckpointState( cp: phase0.Checkpoint, - opts: StateCloneOpts, + opts: StateRegenerationOpts, regenCaller: RegenCaller, allowDiskReload = false ): Promise { @@ -99,7 +99,7 @@ export class StateRegenerator implements IStateRegeneratorInternal { async getBlockSlotState( blockRoot: RootHex, slot: Slot, - opts: StateCloneOpts, + opts: StateRegenerationOpts, regenCaller: RegenCaller, allowDiskReload = false ): Promise { @@ -146,7 +146,7 @@ export class StateRegenerator implements IStateRegeneratorInternal { async getState( stateRoot: RootHex, caller: RegenCaller, - opts?: StateCloneOpts, + opts?: StateRegenerationOpts, // internal option, don't want to expose to external caller allowDiskReload = false ): Promise { @@ -322,7 +322,7 @@ async function processSlotsByCheckpoint( preState: CachedBeaconStateAllForks, slot: Slot, regenCaller: RegenCaller, - opts: StateCloneOpts + opts: StateRegenerationOpts ): Promise { let postState = await processSlotsToNearestCheckpoint(modules, preState, slot, regenCaller, opts); if (postState.slot < slot) { @@ -343,7 +343,7 @@ async function processSlotsToNearestCheckpoint( preState: CachedBeaconStateAllForks, slot: Slot, regenCaller: RegenCaller, - opts: StateCloneOpts + opts: StateRegenerationOpts ): Promise { const preSlot = preState.slot; const postSlot = slot; diff --git a/packages/beacon-node/src/chain/stateCache/blockStateCacheImpl.ts b/packages/beacon-node/src/chain/stateCache/blockStateCacheImpl.ts index f57c9a411923..7d87675b7bbc 100644 --- a/packages/beacon-node/src/chain/stateCache/blockStateCacheImpl.ts +++ b/packages/beacon-node/src/chain/stateCache/blockStateCacheImpl.ts @@ -3,7 +3,7 @@ import {CachedBeaconStateAllForks} from "@lodestar/state-transition"; import {Epoch, RootHex} from "@lodestar/types"; import {toRootHex} from "@lodestar/utils"; import {Metrics} from "../../metrics/index.js"; -import {StateCloneOpts} from "../regen/interface.js"; +import {StateRegenerationOpts} from "../regen/interface.js"; import {MapTracker} from "./mapMetrics.js"; import {BlockStateCache} from "./types.js"; @@ -39,7 +39,7 @@ export class BlockStateCacheImpl implements BlockStateCache { } } - get(rootHex: RootHex, opts?: StateCloneOpts): CachedBeaconStateAllForks | null { + get(rootHex: RootHex, opts?: StateRegenerationOpts): CachedBeaconStateAllForks | null { this.metrics?.lookups.inc(); const item = this.head?.stateRoot === rootHex ? this.head.state : this.cache.get(rootHex); if (!item) { diff --git a/packages/beacon-node/src/chain/stateCache/fifoBlockStateCache.ts b/packages/beacon-node/src/chain/stateCache/fifoBlockStateCache.ts index eec1fce5d6c2..a119efe66887 100644 --- a/packages/beacon-node/src/chain/stateCache/fifoBlockStateCache.ts +++ b/packages/beacon-node/src/chain/stateCache/fifoBlockStateCache.ts @@ -4,7 +4,7 @@ import {RootHex} from "@lodestar/types"; import {toRootHex} from "@lodestar/utils"; import {Metrics} from "../../metrics/index.js"; import {LinkedList} from "../../util/array.js"; -import {StateCloneOpts} from "../regen/interface.js"; +import {StateRegenerationOpts} from "../regen/interface.js"; import {MapTracker} from "./mapMetrics.js"; import {BlockStateCache} from "./types.js"; @@ -93,7 +93,7 @@ export class FIFOBlockStateCache implements BlockStateCache { /** * Get a state from this cache given a state root hex. */ - get(rootHex: RootHex, opts?: StateCloneOpts): CachedBeaconStateAllForks | null { + get(rootHex: RootHex, opts?: StateRegenerationOpts): CachedBeaconStateAllForks | null { this.metrics?.lookups.inc(); const item = this.cache.get(rootHex); if (!item) { diff --git a/packages/beacon-node/src/chain/stateCache/inMemoryCheckpointsCache.ts b/packages/beacon-node/src/chain/stateCache/inMemoryCheckpointsCache.ts index 4caa6779f697..81562d669365 100644 --- a/packages/beacon-node/src/chain/stateCache/inMemoryCheckpointsCache.ts +++ b/packages/beacon-node/src/chain/stateCache/inMemoryCheckpointsCache.ts @@ -3,7 +3,7 @@ import {CachedBeaconStateAllForks} from "@lodestar/state-transition"; import {Epoch, RootHex, phase0} from "@lodestar/types"; import {MapDef, toRootHex} from "@lodestar/utils"; import {Metrics} from "../../metrics/index.js"; -import {StateCloneOpts} from "../regen/interface.js"; +import {StateRegenerationOpts} from "../regen/interface.js"; import {MapTracker} from "./mapMetrics.js"; import {CacheItemType, CheckpointStateCache} from "./types.js"; @@ -42,7 +42,7 @@ export class InMemoryCheckpointStateCache implements CheckpointStateCache { this.maxEpochs = maxEpochs; } - async getOrReload(cp: CheckpointHex, opts?: StateCloneOpts): Promise { + async getOrReload(cp: CheckpointHex, opts?: StateRegenerationOpts): Promise { return this.get(cp, opts); } @@ -54,7 +54,7 @@ export class InMemoryCheckpointStateCache implements CheckpointStateCache { async getOrReloadLatest( rootHex: string, maxEpoch: number, - opts?: StateCloneOpts + opts?: StateRegenerationOpts ): Promise { return this.getLatest(rootHex, maxEpoch, opts); } @@ -64,7 +64,7 @@ export class InMemoryCheckpointStateCache implements CheckpointStateCache { return 0; } - get(cp: CheckpointHex, opts?: StateCloneOpts): CachedBeaconStateAllForks | null { + get(cp: CheckpointHex, opts?: StateRegenerationOpts): CachedBeaconStateAllForks | null { this.metrics?.lookups.inc(); const cpKey = toCheckpointKey(cp); const item = this.cache.get(cpKey); @@ -98,7 +98,7 @@ export class InMemoryCheckpointStateCache implements CheckpointStateCache { /** * Searches for the latest cached state with a `root`, starting with `epoch` and descending */ - getLatest(rootHex: RootHex, maxEpoch: Epoch, opts?: StateCloneOpts): CachedBeaconStateAllForks | null { + getLatest(rootHex: RootHex, maxEpoch: Epoch, opts?: StateRegenerationOpts): CachedBeaconStateAllForks | null { // sort epochs in descending order, only consider epochs lte `epoch` const epochs = Array.from(this.epochIndex.keys()) .sort((a, b) => b - a) diff --git a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts index a7e0a7cdecfe..9a08aaa75461 100644 --- a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts +++ b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts @@ -7,7 +7,7 @@ import {Logger, MapDef, fromHex, sleep, toHex, toRootHex} from "@lodestar/utils" import {Metrics} from "../../metrics/index.js"; import {AllocSource, BufferPool, BufferWithKey} from "../../util/bufferPool.js"; import {IClock} from "../../util/clock.js"; -import {StateCloneOpts} from "../regen/interface.js"; +import {StateRegenerationOpts} from "../regen/interface.js"; import {serializeState} from "../serializeState.js"; import {CPStateDatastore, DatastoreKey} from "./datastore/index.js"; import {MapTracker} from "./mapMetrics.js"; @@ -168,7 +168,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { * - Get block for processing * - Regen head state */ - async getOrReload(cp: CheckpointHex, opts?: StateCloneOpts): Promise { + async getOrReload(cp: CheckpointHex, opts?: StateRegenerationOpts): Promise { const stateOrStateBytesData = await this.getStateOrLoadDb(cp, opts); if (stateOrStateBytesData === null || isCachedBeaconState(stateOrStateBytesData)) { return stateOrStateBytesData?.clone(opts?.dontTransferCache) ?? null; @@ -240,7 +240,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { */ async getStateOrLoadDb( cp: CheckpointHex, - opts?: StateCloneOpts + opts?: StateRegenerationOpts ): Promise { const cpKey = toCacheKey(cp); const inMemoryState = this.get(cpKey, opts); @@ -272,7 +272,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { /** * Similar to get() api without reloading from disk */ - get(cpOrKey: CheckpointHex | string, opts?: StateCloneOpts): CachedBeaconStateAllForks | null { + get(cpOrKey: CheckpointHex | string, opts?: StateRegenerationOpts): CachedBeaconStateAllForks | null { this.metrics?.cpStateCache.lookups.inc(); const cpKey = typeof cpOrKey === "string" ? cpOrKey : toCacheKey(cpOrKey); const cacheItem = this.cache.get(cpKey); @@ -323,7 +323,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { /** * Searches in-memory state for the latest cached state with a `root` without reload, starting with `epoch` and descending */ - getLatest(rootHex: RootHex, maxEpoch: Epoch, opts?: StateCloneOpts): CachedBeaconStateAllForks | null { + getLatest(rootHex: RootHex, maxEpoch: Epoch, opts?: StateRegenerationOpts): CachedBeaconStateAllForks | null { // sort epochs in descending order, only consider epochs lte `epoch` const epochs = Array.from(this.epochIndex.keys()) .sort((a, b) => b - a) @@ -349,7 +349,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { async getOrReloadLatest( rootHex: RootHex, maxEpoch: Epoch, - opts?: StateCloneOpts + opts?: StateRegenerationOpts ): Promise { // sort epochs in descending order, only consider epochs lte `epoch` const epochs = Array.from(this.epochIndex.keys()) diff --git a/packages/beacon-node/src/chain/stateCache/types.ts b/packages/beacon-node/src/chain/stateCache/types.ts index 403b469dd352..19f05c23ee35 100644 --- a/packages/beacon-node/src/chain/stateCache/types.ts +++ b/packages/beacon-node/src/chain/stateCache/types.ts @@ -1,7 +1,7 @@ import {routes} from "@lodestar/api"; import {CachedBeaconStateAllForks} from "@lodestar/state-transition"; import {Epoch, RootHex, phase0} from "@lodestar/types"; -import {StateCloneOpts} from "../regen/interface.js"; +import {StateRegenerationOpts} from "../regen/interface.js"; export type CheckpointHex = {epoch: Epoch; rootHex: RootHex}; @@ -21,7 +21,7 @@ export type CheckpointHex = {epoch: Epoch; rootHex: RootHex}; * The cache key is state root */ export interface BlockStateCache { - get(rootHex: RootHex, opts?: StateCloneOpts): CachedBeaconStateAllForks | null; + get(rootHex: RootHex, opts?: StateRegenerationOpts): CachedBeaconStateAllForks | null; add(item: CachedBeaconStateAllForks): void; setHeadState(item: CachedBeaconStateAllForks | null): void; /** @@ -60,15 +60,15 @@ export interface BlockStateCache { */ export interface CheckpointStateCache { init?: () => Promise; - getOrReload(cp: CheckpointHex, opts?: StateCloneOpts): Promise; + getOrReload(cp: CheckpointHex, opts?: StateRegenerationOpts): Promise; getStateOrBytes(cp: CheckpointHex): Promise; - get(cpOrKey: CheckpointHex | string, opts?: StateCloneOpts): CachedBeaconStateAllForks | null; + get(cpOrKey: CheckpointHex | string, opts?: StateRegenerationOpts): CachedBeaconStateAllForks | null; add(cp: phase0.Checkpoint, state: CachedBeaconStateAllForks): void; - getLatest(rootHex: RootHex, maxEpoch: Epoch, opts?: StateCloneOpts): CachedBeaconStateAllForks | null; + getLatest(rootHex: RootHex, maxEpoch: Epoch, opts?: StateRegenerationOpts): CachedBeaconStateAllForks | null; getOrReloadLatest( rootHex: RootHex, maxEpoch: Epoch, - opts?: StateCloneOpts + opts?: StateRegenerationOpts ): Promise; updatePreComputedCheckpoint(rootHex: RootHex, epoch: Epoch): number | null; prune(finalizedEpoch: Epoch, justifiedEpoch: Epoch): void; diff --git a/packages/beacon-node/src/execution/engine/http.ts b/packages/beacon-node/src/execution/engine/http.ts index ea064d2fe816..0d2656ae46e2 100644 --- a/packages/beacon-node/src/execution/engine/http.ts +++ b/packages/beacon-node/src/execution/engine/http.ts @@ -2,6 +2,7 @@ import {Logger} from "@lodestar/logger"; import {ForkName, ForkSeq, SLOTS_PER_EPOCH} from "@lodestar/params"; import {ExecutionPayload, ExecutionRequests, Root, RootHex, Wei} from "@lodestar/types"; import {BlobAndProof} from "@lodestar/types/deneb"; +import {strip0xPrefix} from "@lodestar/utils"; import { ErrorJsonRpcResponse, HttpRpcError, @@ -522,11 +523,11 @@ export class ExecutionEngineHttp implements IExecutionEngine { const response = await this.rpc.fetchWithRetries< EngineApiRpcReturnTypes[typeof method], EngineApiRpcParamTypes[typeof method] - >({method, params: [clientVersion]}); + >({method, params: [{...clientVersion, commit: `0x${clientVersion.commit}`}]}); const clientVersions = response.map((cv) => { const code = cv.code in ClientCode ? ClientCode[cv.code as keyof typeof ClientCode] : ClientCode.XX; - return {code, name: cv.name, version: cv.version, commit: cv.commit}; + return {code, name: cv.name, version: cv.version, commit: strip0xPrefix(cv.commit)}; }); if (clientVersions.length === 0) { diff --git a/packages/beacon-node/src/util/kzg.ts b/packages/beacon-node/src/util/kzg.ts index 36a7d19f8d2b..696eeae73370 100644 --- a/packages/beacon-node/src/util/kzg.ts +++ b/packages/beacon-node/src/util/kzg.ts @@ -1,7 +1,7 @@ import fs from "node:fs"; import path from "node:path"; import {fileURLToPath} from "node:url"; -import {fromHex, toHex} from "@lodestar/utils"; +import {fromHex, strip0xPrefix, toHex} from "@lodestar/utils"; // "c-kzg" has hardcoded the mainnet value, do not use params export const FIELD_ELEMENTS_PER_BLOB_MAINNET = 4096; @@ -154,10 +154,3 @@ export function trustedSetupJsonToTxt(data: TrustedSetupJSON): TrustedSetupTXT { ...data.setup_G2.map(strip0xPrefix), ].join("\n"); } - -function strip0xPrefix(hex: string): string { - if (hex.startsWith("0x")) { - return hex.slice(2); - } - return hex; -} diff --git a/packages/cli/src/cmds/beacon/options.ts b/packages/cli/src/cmds/beacon/options.ts index a623ca99512c..7f2cdddfc4ff 100644 --- a/packages/cli/src/cmds/beacon/options.ts +++ b/packages/cli/src/cmds/beacon/options.ts @@ -33,7 +33,6 @@ export const beaconExtraOptions: CliCommandOptions = { }, genesisStateFile: { - hidden: true, description: "Path or URL to download a genesis state file in ssz-encoded format", type: "string", }, @@ -80,7 +79,6 @@ export const beaconExtraOptions: CliCommandOptions = { ignoreWeakSubjectivityCheck: { description: "Ignore the checkpoint sync state failing the weak subjectivity check. This is relevant in testnets where the weak subjectivity period is too small for even few epochs of non finalization causing last finalized to be out of range. This flag is not recommended for mainnet use.", - hidden: true, type: "boolean", group: "weak subjectivity", }, @@ -120,7 +118,6 @@ export const beaconExtraOptions: CliCommandOptions = { }, persistNetworkIdentity: { - hidden: true, description: "Whether to reuse the same peer-id across restarts", type: "boolean", }, diff --git a/packages/cli/src/networks/holesky.ts b/packages/cli/src/networks/holesky.ts index 63bc6e07f8f2..ed1dd7c78462 100644 --- a/packages/cli/src/networks/holesky.ts +++ b/packages/cli/src/networks/holesky.ts @@ -3,7 +3,7 @@ export {holeskyChainConfig as chainConfig} from "@lodestar/config/networks"; export const depositContractDeployBlock = 0; export const genesisFileUrl = "https://media.githubusercontent.com/media/eth-clients/holesky/main/metadata/genesis.ssz"; export const bootnodesFileUrl = - "https://raw.githubusercontent.com/eth-clients/holesky/main/metadata/bootstrap_nodes.txt"; + "https://raw.githubusercontent.com/eth-clients/holesky/main/metadata/bootstrap_nodes.yaml"; export const bootEnrs = [ "enr:-Ku4QFo-9q73SspYI8cac_4kTX7yF800VXqJW4Lj3HkIkb5CMqFLxciNHePmMt4XdJzHvhrCC5ADI4D_GkAsxGJRLnQBh2F0dG5ldHOIAAAAAAAAAACEZXRoMpAhnTT-AQFwAP__________gmlkgnY0gmlwhLKAiOmJc2VjcDI1NmsxoQORcM6e19T1T9gi7jxEZjk_sjVLGFscUNqAY9obgZaxbIN1ZHCCIyk", diff --git a/packages/cli/src/networks/sepolia.ts b/packages/cli/src/networks/sepolia.ts index 9dfd5dc20a0f..6900ca493057 100644 --- a/packages/cli/src/networks/sepolia.ts +++ b/packages/cli/src/networks/sepolia.ts @@ -3,7 +3,7 @@ export {sepoliaChainConfig as chainConfig} from "@lodestar/config/networks"; export const depositContractDeployBlock = 1273020; export const genesisFileUrl = "https://github.com/eth-clients/sepolia/raw/main/metadata/genesis.ssz"; export const bootnodesFileUrl = - "https://raw.githubusercontent.com/eth-clients/sepolia/main/metadata/bootstrap_nodes.txt"; + "https://raw.githubusercontent.com/eth-clients/sepolia/main/metadata/bootstrap_nodes.yaml"; export const bootEnrs = [ "enr:-KO4QP7MmB3juk8rUjJHcUoxZDU9Np4FlW0HyDEGIjSO7GD9PbSsabu7713cWSUWKDkxIypIXg1A-6lG7ySRGOMZHeGCAmuEZXRoMpDTH2GRkAAAc___________gmlkgnY0gmlwhBSoyGOJc2VjcDI1NmsxoQNta5b_bexSSwwrGW2Re24MjfMntzFd0f2SAxQtMj3ueYN0Y3CCIyiDdWRwgiMo", diff --git a/packages/cli/src/options/globalOptions.ts b/packages/cli/src/options/globalOptions.ts index 58ae396b67d1..6a6998dc76cc 100644 --- a/packages/cli/src/options/globalOptions.ts +++ b/packages/cli/src/options/globalOptions.ts @@ -10,6 +10,7 @@ type GlobalSingleArgs = { paramsFile?: string; preset: string; presetFile?: string; + rcConfig?: string; }; export const defaultNetwork: NetworkName = "mainnet"; @@ -44,11 +45,16 @@ const globalSingleOptions: CliCommandOptions = { description: "Preset configuration file to override the active preset with custom values", type: "string", }, + + rcConfig: { + description: "RC file to supplement command line args, accepted formats: .yml, .yaml, .json", + type: "string", + }, }; export const rcConfigOption: [string, string, (configPath: string) => Record] = [ "rcConfig", - "RC file to supplement command line args, accepted formats: .yml, .yaml, .json", + globalSingleOptions.rcConfig.description as string, (configPath: string): Record => readFile(configPath, ["json", "yml", "yaml"]), ]; diff --git a/packages/config/src/genesisConfig/index.ts b/packages/config/src/genesisConfig/index.ts index dad79df1a98d..992185461b58 100644 --- a/packages/config/src/genesisConfig/index.ts +++ b/packages/config/src/genesisConfig/index.ts @@ -1,6 +1,6 @@ import {DOMAIN_VOLUNTARY_EXIT, ForkName, SLOTS_PER_EPOCH} from "@lodestar/params"; import {DomainType, ForkDigest, Root, Slot, Version, phase0, ssz} from "@lodestar/types"; -import {toHex} from "@lodestar/utils"; +import {strip0xPrefix, toHex} from "@lodestar/utils"; import {ChainForkConfig} from "../beaconConfig.js"; import {CachedGenesis, ForkDigestHex} from "./types.js"; export type {ForkDigestContext} from "./types.js"; @@ -142,10 +142,6 @@ function toHexStringNoPrefix(hex: string | Uint8Array): string { return strip0xPrefix(typeof hex === "string" ? hex : toHex(hex)); } -function strip0xPrefix(hex: string): string { - return hex.startsWith("0x") ? hex.slice(2) : hex; -} - function computeForkDigest(currentVersion: Version, genesisValidatorsRoot: Root): ForkDigest { return computeForkDataRoot(currentVersion, genesisValidatorsRoot).slice(0, 4); } diff --git a/packages/state-transition/src/block/processConsolidationRequest.ts b/packages/state-transition/src/block/processConsolidationRequest.ts index f0e2da3f3571..ade22c7454db 100644 --- a/packages/state-transition/src/block/processConsolidationRequest.ts +++ b/packages/state-transition/src/block/processConsolidationRequest.ts @@ -3,7 +3,7 @@ import {electra, ssz} from "@lodestar/types"; import {CachedBeaconStateElectra} from "../types.js"; import {hasEth1WithdrawalCredential} from "../util/capella.js"; -import {hasExecutionWithdrawalCredential, switchToCompoundingValidator} from "../util/electra.js"; +import {hasExecutionWithdrawalCredential, isPubkeyKnown, switchToCompoundingValidator} from "../util/electra.js"; import {computeConsolidationEpochAndUpdateChurn} from "../util/epoch.js"; import {getConsolidationChurnLimit, getPendingBalanceToWithdraw, isActiveValidator} from "../util/validator.js"; @@ -13,6 +13,10 @@ export function processConsolidationRequest( consolidationRequest: electra.ConsolidationRequest ): void { const {sourcePubkey, targetPubkey, sourceAddress} = consolidationRequest; + if (!isPubkeyKnown(state, sourcePubkey) || !isPubkeyKnown(state, targetPubkey)) { + return; + } + const sourceIndex = state.epochCtx.getValidatorIndex(sourcePubkey); const targetIndex = state.epochCtx.getValidatorIndex(targetPubkey); @@ -107,6 +111,7 @@ function isValidSwitchToCompoundRequest( // Verify pubkey exists if (sourceIndex === null) { + // this check is mainly to make the compiler happy, pubkey is checked by the consumer already return false; } diff --git a/packages/state-transition/src/cache/epochCache.ts b/packages/state-transition/src/cache/epochCache.ts index 90b4a3550ab4..c975331c7dfb 100644 --- a/packages/state-transition/src/cache/epochCache.ts +++ b/packages/state-transition/src/cache/epochCache.ts @@ -49,6 +49,7 @@ import { import {computeBaseRewardPerIncrement, computeSyncParticipantReward} from "../util/syncCommittee.js"; import {sumTargetUnslashedBalanceIncrements} from "../util/targetUnslashedBalance.js"; import {EffectiveBalanceIncrements, getEffectiveBalanceIncrementsWithLen} from "./effectiveBalanceIncrements.js"; +import {EpochTransitionCache} from "./epochTransitionCache.js"; import {Index2PubkeyCache, syncPubkeys} from "./pubkeyCache.js"; import {CachedBeaconStateAllForks} from "./stateCache.js"; import { @@ -605,14 +606,7 @@ export class EpochCache { * Steps for afterProcessEpoch * 1) update previous/current/next values of cached items */ - afterProcessEpoch( - state: CachedBeaconStateAllForks, - epochTransitionCache: { - nextShufflingDecisionRoot: RootHex; - nextShufflingActiveIndices: Uint32Array; - nextEpochTotalActiveBalanceByIncrement: number; - } - ): void { + afterProcessEpoch(state: CachedBeaconStateAllForks, epochTransitionCache: EpochTransitionCache): void { // Because the slot was incremented before entering this function the "next epoch" is actually the "current epoch" // in this context but that is not actually true because the state transition happens in the last 4 seconds of the // epoch. For the context of this function "upcoming epoch" is used to denote the epoch that will begin after this @@ -657,28 +651,35 @@ export class EpochCache { this.nextDecisionRoot = epochTransitionCache.nextShufflingDecisionRoot; this.nextActiveIndices = epochTransitionCache.nextShufflingActiveIndices; if (this.shufflingCache) { - this.nextShuffling = null; - // This promise will resolve immediately after the synchronous code of the state-transition runs. Until - // the build is done on a worker thread it will be calculated immediately after the epoch transition - // completes. Once the work is done concurrently it should be ready by time this get runs so the promise - // will resolve directly on the next spin of the event loop because the epoch transition and shuffling take - // about the same time to calculate so theoretically its ready now. Do not await here though in case it - // is not ready yet as the transition must not be asynchronous. - this.shufflingCache - .get(epochAfterUpcoming, this.nextDecisionRoot) - .then((shuffling) => { - if (!shuffling) { - throw new Error("EpochShuffling not returned from get in afterProcessEpoch"); - } - this.nextShuffling = shuffling; - }) - .catch((err) => { - this.shufflingCache?.logger?.error( - "EPOCH_CONTEXT_SHUFFLING_BUILD_ERROR", - {epoch: epochAfterUpcoming, decisionRoot: epochTransitionCache.nextShufflingDecisionRoot}, - err - ); + if (!epochTransitionCache.asyncShufflingCalculation) { + this.nextShuffling = this.shufflingCache.getSync(epochAfterUpcoming, this.nextDecisionRoot, { + state, + activeIndices: this.nextActiveIndices, }); + } else { + this.nextShuffling = null; + // This promise will resolve immediately after the synchronous code of the state-transition runs. Until + // the build is done on a worker thread it will be calculated immediately after the epoch transition + // completes. Once the work is done concurrently it should be ready by time this get runs so the promise + // will resolve directly on the next spin of the event loop because the epoch transition and shuffling take + // about the same time to calculate so theoretically its ready now. Do not await here though in case it + // is not ready yet as the transition must not be asynchronous. + this.shufflingCache + .get(epochAfterUpcoming, this.nextDecisionRoot) + .then((shuffling) => { + if (!shuffling) { + throw new Error("EpochShuffling not returned from get in afterProcessEpoch"); + } + this.nextShuffling = shuffling; + }) + .catch((err) => { + this.shufflingCache?.logger?.error( + "EPOCH_CONTEXT_SHUFFLING_BUILD_ERROR", + {epoch: epochAfterUpcoming, decisionRoot: epochTransitionCache.nextShufflingDecisionRoot}, + err + ); + }); + } } else { // Only for testing. shufflingCache should always be available in prod this.nextShuffling = computeEpochShuffling(state, this.nextActiveIndices, epochAfterUpcoming); diff --git a/packages/state-transition/src/cache/epochTransitionCache.ts b/packages/state-transition/src/cache/epochTransitionCache.ts index b21f940c28d5..d159ca5f3763 100644 --- a/packages/state-transition/src/cache/epochTransitionCache.ts +++ b/packages/state-transition/src/cache/epochTransitionCache.ts @@ -33,6 +33,10 @@ export type EpochTransitionCacheOpts = { * Assert progressive balances the same to EpochTransitionCache */ assertCorrectProgressiveBalances?: boolean; + /** + * Do not queue shuffling calculation async. Forces sync JIT calculation in afterProcessEpoch + */ + asyncShufflingCalculation?: boolean; }; /** @@ -176,6 +180,12 @@ export interface EpochTransitionCache { */ nextEpochTotalActiveBalanceByIncrement: number; + /** + * Compute the shuffling sync or async. Defaults to synchronous. Need to pass `true` with the + * `EpochTransitionCacheOpts` + */ + asyncShufflingCalculation: boolean; + /** * Track by validator index if it's active in the prev epoch. * Used in metrics @@ -387,7 +397,11 @@ export function beforeProcessEpoch( for (let i = 0; i < nextEpochShufflingActiveIndicesLength; i++) { nextShufflingActiveIndices[i] = nextEpochShufflingActiveValidatorIndices[i]; } - state.epochCtx.shufflingCache?.build(epochAfterNext, nextShufflingDecisionRoot, state, nextShufflingActiveIndices); + + const asyncShufflingCalculation = opts?.asyncShufflingCalculation ?? false; + if (asyncShufflingCalculation) { + state.epochCtx.shufflingCache?.build(epochAfterNext, nextShufflingDecisionRoot, state, nextShufflingActiveIndices); + } if (totalActiveStakeByIncrement < 1) { totalActiveStakeByIncrement = 1; @@ -514,6 +528,7 @@ export function beforeProcessEpoch( indicesToEject, nextShufflingDecisionRoot, nextShufflingActiveIndices, + asyncShufflingCalculation, // to be updated in processEffectiveBalanceUpdates nextEpochTotalActiveBalanceByIncrement: 0, isActivePrevEpoch, diff --git a/packages/state-transition/src/epoch/processPendingDeposits.ts b/packages/state-transition/src/epoch/processPendingDeposits.ts index d925aa1cc741..441d5601a380 100644 --- a/packages/state-transition/src/epoch/processPendingDeposits.ts +++ b/packages/state-transition/src/epoch/processPendingDeposits.ts @@ -3,7 +3,7 @@ import {PendingDeposit} from "@lodestar/types/lib/electra/types.js"; import {addValidatorToRegistry, isValidDepositSignature} from "../block/processDeposit.js"; import {CachedBeaconStateElectra, EpochTransitionCache} from "../types.js"; import {increaseBalance} from "../util/balance.js"; -import {hasCompoundingWithdrawalCredential} from "../util/electra.js"; +import {hasCompoundingWithdrawalCredential, isValidatorKnown} from "../util/electra.js"; import {computeStartSlotAtEpoch} from "../util/epoch.js"; import {getActivationExitChurnLimit} from "../util/validator.js"; @@ -51,7 +51,7 @@ export function processPendingDeposits(state: CachedBeaconStateElectra, cache: E let isValidatorWithdrawn = false; const validatorIndex = state.epochCtx.getValidatorIndex(deposit.pubkey); - if (validatorIndex !== null) { + if (isValidatorKnown(state, validatorIndex)) { const validator = state.validators.getReadonly(validatorIndex); isValidatorExited = validator.exitEpoch < FAR_FUTURE_EPOCH; isValidatorWithdrawn = validator.withdrawableEpoch < nextEpoch; @@ -103,7 +103,7 @@ function applyPendingDeposit( const {pubkey, withdrawalCredentials, amount, signature} = deposit; const cachedBalances = cache.balances; - if (validatorIndex === null) { + if (!isValidatorKnown(state, validatorIndex)) { // Verify the deposit signature (proof of possession) which is not checked by the deposit contract if (isValidDepositSignature(state.config, pubkey, withdrawalCredentials, amount, signature)) { addValidatorToRegistry(ForkSeq.electra, state, pubkey, withdrawalCredentials, amount); diff --git a/packages/state-transition/src/util/electra.ts b/packages/state-transition/src/util/electra.ts index f5b899eadcab..a9736dc8161e 100644 --- a/packages/state-transition/src/util/electra.ts +++ b/packages/state-transition/src/util/electra.ts @@ -45,3 +45,20 @@ export function queueExcessActiveBalance(state: CachedBeaconStateElectra, index: state.pendingDeposits.push(pendingDeposit); } } + +/** + * Since we share pubkey2index, pubkey maybe added by other epoch transition but we don't have that validator in this state + */ +export function isPubkeyKnown(state: CachedBeaconStateElectra, pubkey: Uint8Array): boolean { + return isValidatorKnown(state, state.epochCtx.getValidatorIndex(pubkey)); +} + +/** + * Since we share pubkey2index, validatorIndex maybe not null but we don't have that validator in this state + */ +export function isValidatorKnown( + state: CachedBeaconStateElectra, + index: ValidatorIndex | null +): index is ValidatorIndex { + return index !== null && index < state.validators.length; +} diff --git a/packages/types/src/capella/sszTypes.ts b/packages/types/src/capella/sszTypes.ts index 3110e59111d9..057bf97650fe 100644 --- a/packages/types/src/capella/sszTypes.ts +++ b/packages/types/src/capella/sszTypes.ts @@ -125,6 +125,10 @@ export const HistoricalSummary = new ContainerType( {typeName: "HistoricalSummary", jsonCase: "eth2"} ); +export const HistoricalSummaries = new ListCompositeType(HistoricalSummary, HISTORICAL_ROOTS_LIMIT, { + typeName: "HistoricalSummaries", +}); + // we don't reuse bellatrix.BeaconState fields since we need to replace some keys // and we cannot keep order doing that export const BeaconState = new ContainerType( @@ -168,7 +172,7 @@ export const BeaconState = new ContainerType( nextWithdrawalIndex: WithdrawalIndex, // [New in Capella] nextWithdrawalValidatorIndex: ValidatorIndex, // [New in Capella] // Deep history valid from Capella onwards - historicalSummaries: new ListCompositeType(HistoricalSummary, HISTORICAL_ROOTS_LIMIT), // [New in Capella] + historicalSummaries: HistoricalSummaries, // [New in Capella] }, {typeName: "BeaconState", jsonCase: "eth2"} ); diff --git a/packages/utils/src/format.ts b/packages/utils/src/format.ts index b36412072720..94bf0d635ced 100644 --- a/packages/utils/src/format.ts +++ b/packages/utils/src/format.ts @@ -65,3 +65,10 @@ export function prettyMsToTime(timeMs: number): string { const date = new Date(0, 0, 0, 0, 0, 0, timeMs); return `${date.getHours()}:${date.getMinutes()}:${date.getSeconds()}.${date.getMilliseconds()}`; } + +/** + * Remove 0x prefix from a string + */ +export function strip0xPrefix(hex: string): string { + return hex.startsWith("0x") ? hex.slice(2) : hex; +} diff --git a/scripts/install-binary.sh b/scripts/install-binary.sh new file mode 100755 index 000000000000..f6db9948eeda --- /dev/null +++ b/scripts/install-binary.sh @@ -0,0 +1,91 @@ +#!/bin/bash + +# ASCII art +echo " _ _ _ " +echo " | | | | | | " +echo " | | ___ __| | ___ ___| |_ __ _ _ __ " +echo " | |/ _ \ / _ |/ _ \/ __| __/ _ | __|" +echo " | | (_) | (_| | __/\__ \ || (_| | | " +echo " |_|\___/ \__ _|\___||___/\__\__ _|_| " +echo "" + +# Declare directories +TEMP_DIR=$(mktemp -d) +LOCAL_BIN="$HOME/.local/bin" + +# Ensure ~/.local/bin exists +mkdir -p "$LOCAL_BIN" + +# Inform the user about temporary directory usage +echo "Using temporary directory: $TEMP_DIR" + +# Fetch the latest release tag from GitHub +echo "Fetching the latest version information..." +VERSION=$(curl -s "https://api.github.com/repos/ChainSafe/lodestar/releases/latest" | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/') + +# Check if VERSION is empty +if [ -z "$VERSION" ]; then + echo "Failed to fetch the latest version. Exiting." + exit 1 +fi + +echo "Latest version detected: $VERSION" + +# Detect the operating system and architecture +OS=$(uname -s) +ARCH=$(uname -m) + +# Translate architecture to expected format +case $ARCH in + x86_64) ARCH="amd64" ;; + aarch64|arm64) ARCH="arm64" ;; + *) + echo "Unsupported architecture: $ARCH. Exiting." + exit 1 + ;; +esac + +# Translate OS to expected format +case $OS in + Linux) PLATFORM="linux-$ARCH" ;; + *) + echo "Unsupported operating system: $OS. Exiting." + exit 1 + ;; +esac + +# Construct the download URL +URL="https://github.com/ChainSafe/lodestar/releases/download/$VERSION/lodestar-$VERSION-$PLATFORM.tar.gz" +echo "Downloading from: $URL" + +# Download the tarball +if ! wget "$URL" -O "$TEMP_DIR/lodestar-$VERSION-$PLATFORM.tar.gz"; then + echo "Download failed. Exiting." + exit 1 +fi + +# Extract the tarball +echo "Extracting the binary..." +if ! tar -xzf "$TEMP_DIR/lodestar-$VERSION-$PLATFORM.tar.gz" -C "$TEMP_DIR"; then + echo "Extraction failed. Exiting." + exit 1 +fi + +# Move the binary to ~/.local/bin +echo "Moving the binary to $LOCAL_BIN..." +mv "$TEMP_DIR/lodestar" "$LOCAL_BIN/" +chmod +x "$LOCAL_BIN/lodestar" + +# Verify if ~/.local/bin is in PATH +if [[ ":$PATH:" != *":$LOCAL_BIN:"* ]]; then + echo "Adding $LOCAL_BIN to PATH..." + echo 'export PATH="$PATH:$HOME/.local/bin"' >> "$HOME/.bashrc" + echo "Run 'source ~/.bashrc' to apply changes to your shell." +fi + +# Clean up the temporary directory +rm -rf "$TEMP_DIR" + +# Inform the user of successful installation +echo "Installation complete!" +echo "Run 'lodestar --help' to get started." diff --git a/scripts/prepare-docs.sh b/scripts/prepare-docs.sh index d2bac519a7f2..2fa16a7cc759 100755 --- a/scripts/prepare-docs.sh +++ b/scripts/prepare-docs.sh @@ -17,3 +17,6 @@ cp -r packages/prover/README.md $DOCS_DIR/pages/libraries/lightclient-prover/pro # Copy visual assets rm -rf $DOCS_DIR/pages/assets $DOCS_DIR/pages/images cp -r $ASSETS_DIR $DOCS_DIR/pages/assets + +# Copy binary install script to docs +cp scripts/install-binary.sh $DOCS_DIR/static/install