fix: docker build issue for c-kzg

wip: REPLACE THIS COMMIT

commit yarn lock

rebase fixes

fix: update c-zkg install workflow

feat: add trustedSetupPrecompute cli flag

fix: update trusted-setup for testing

fix: update c-zkg install workflow to remove sudo

fix: add rsync to apk deps

Dockerfile

@@ -4,7 +4,7 @@
 FROM --platform=${BUILDPLATFORM:-amd64} node:22.4-slim as build_src
 ARG COMMIT
 WORKDIR /usr/app
-RUN apt-get update && apt-get install -y g++ make python3 python3-setuptools && apt-get clean && rm -rf /var/lib/apt/lists/*
+RUN apt-get update && apt-get install -y git rsync g++ make python3 python3-setuptools && apt-get clean && rm -rf /var/lib/apt/lists/*
 
 COPY . .
 
@@ -23,7 +23,7 @@ RUN cd packages/cli && GIT_COMMIT=${COMMIT} yarn write-git-data
 # Note: This step is redundant for the host arch
 FROM node:22.4-slim as build_deps
 WORKDIR /usr/app
-RUN apt-get update && apt-get install -y g++ make python3 python3-setuptools && apt-get clean && rm -rf /var/lib/apt/lists/*
+RUN apt-get update && apt-get install -y git rsync g++ make python3 python3-setuptools && apt-get clean && rm -rf /var/lib/apt/lists/*
 
 COPY --from=build_src /usr/app .
 

packages/beacon-node/package.json

@@ -132,7 +132,7 @@
     "@lodestar/utils": "^1.21.0",
     "@lodestar/validator": "^1.21.0",
     "@multiformats/multiaddr": "^12.1.3",
-    "c-kzg": "matthewkeil/c-kzg-4844#67bf9367817f0fa5ebd390aeb8c3ae88bdbc170e",
+    "c-kzg": "matthewkeil/c-kzg-4844#13aa01464479aa7c1ccafa64d52cbc17699ffa07",
     "datastore-core": "^9.1.1",
     "datastore-level": "^10.1.1",
     "deepmerge": "^4.3.1",

packages/beacon-node/src/chain/blocks/types.ts

@@ -48,7 +48,7 @@ export type DataColumnsCacheMap = Map<
   {dataColumnSidecar: electra.DataColumnSidecar; dataColumnBytes: Uint8Array | null}
 >;
 
-type ForkBlobsInfo = {fork: ForkName.deneb | ForkName.electra};
+type ForkBlobsInfo = {fork: ForkName.deneb};
 type BlobsData = {blobs: deneb.BlobSidecars; blobsBytes: (Uint8Array | null)[]; blobsSource: BlobsSource};
 export type BlockInputDataBlobs = ForkBlobsInfo & BlobsData;
 

packages/beacon-node/src/chain/options.ts

@@ -31,6 +31,14 @@ export type IChainOptions = BlockProcessOpts &
     maxCachedBlobSidecars?: number;
     /** Max number of produced block roots (blinded or full) cached for broadcast validations */
     maxCachedProducedRoots?: number;
+    /*
+     * This is the window size for the windowed multiplication in proof
+     * generation. The larger wbits is, the faster the MSM will be, but the
+     * size of the precomputed table will grow exponentially. With 8 bits, the
+     * tables are 96 MiB; with 9 bits, the tables are 192 MiB and so forth.
+     * From our testing, there are diminishing returns after 8 bits.
+     */
+    trustedSetupPrecompute?: number;
     /** Option to load a custom kzg trusted setup in txt format */
     trustedSetup?: string;
     broadcastValidationStrictness?: string;
@@ -106,6 +114,9 @@ export const defaultChainOptions: IChainOptions = {
   // for gossip block validation, it's unlikely we see a reorg with 32 slots
   // for attestation validation, having this value ensures we don't have to regen states most of the time
   maxSkipSlots: 32,
+  // TODO: (@matthewkeil) this is a plug number and directly affects memory size
+  // but also has security implications.  Needs to be sorted out.
+  trustedSetupPrecompute: 8,
   broadcastValidationStrictness: "warn",
   // should be less than or equal to MIN_SIGNATURE_SETS_TO_BATCH_VERIFY
   // batching too much may block the I/O thread so if useWorker=false, suggest this value to be 32

packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRange.ts

@@ -163,7 +163,7 @@ export function matchBlockWithDataColumns(
   peerId: PeerIdStr,
   config: ChainForkConfig,
   custodyConfig: CustodyConfig,
-  allBlocks: WithBytes<allForks.SignedBeaconBlock>[],
+  allBlocks: WithBytes<SignedBeaconBlock>[],
   allDataColumnSidecars: electra.DataColumnSidecar[],
   endSlot: Slot,
   blockSource: BlockSource,

packages/beacon-node/src/node/nodejs.ts

@@ -164,7 +164,7 @@ export class BeaconNode {
     // If deneb is configured, load the trusted setup
     if (config.DENEB_FORK_EPOCH < Infinity) {
       await initCKZG();
-      loadEthereumTrustedSetup(TrustedFileMode.Txt, opts.chain.trustedSetup);
+      loadEthereumTrustedSetup(TrustedFileMode.Txt, opts.chain.trustedSetupPrecompute, opts.chain.trustedSetup);
     }
 
     // Prune hot db repos

packages/beacon-node/src/util/blobs.ts

@@ -1,7 +1,7 @@
 import {digest as sha256Digest} from "@chainsafe/as-sha256";
 import {Tree} from "@chainsafe/persistent-merkle-tree";
 import {VERSIONED_HASH_VERSION_KZG, KZG_COMMITMENT_GINDEX0, KZG_COMMITMENTS_GINDEX, ForkName, ForkAll, NUMBER_OF_COLUMNS} from "@lodestar/params";
-import {deneb, ssz, BeaconBlockBody, SignedBeaconBlock, SSZTypesFor} from "@lodestar/types";
+import {deneb, ssz, BeaconBlockBody, SignedBeaconBlock, SSZTypesFor, electra} from "@lodestar/types";
 import {ChainForkConfig} from "@lodestar/config";
 import {signedBlockToSignedHeader} from "@lodestar/state-transition";
 import {ckzg} from "./kzg.js";

packages/beacon-node/src/util/kzg.ts

@@ -14,7 +14,7 @@ function ckzgNotLoaded(): never {
 
 export let ckzg: {
   freeTrustedSetup(): void;
-  loadTrustedSetup(filePath: string): void;
+  loadTrustedSetup(precompute: number, filePath: string): void;
   blobToKzgCommitment(blob: Uint8Array): Uint8Array;
   computeBlobKzgProof(blob: Uint8Array, commitment: Uint8Array): Uint8Array;
   verifyBlobKzgProof(blob: Uint8Array, commitment: Uint8Array, proof: Uint8Array): boolean;
@@ -77,7 +77,11 @@ export enum TrustedFileMode {
  * We persist the trusted setup as serialized bytes to save space over TXT or JSON formats.
  * However the current c-kzg API **requires** to read from a file with a specific .txt format
  */
-export function loadEthereumTrustedSetup(mode: TrustedFileMode = TrustedFileMode.Txt, filePath?: string): void {
+export function loadEthereumTrustedSetup(
+  mode: TrustedFileMode = TrustedFileMode.Txt,
+  precompute = 0, // default to 0 for testing
+  filePath?: string
+): void {
   try {
     let setupFilePath;
     if (mode === TrustedFileMode.Bin) {
@@ -93,7 +97,7 @@ export function loadEthereumTrustedSetup(mode: TrustedFileMode = TrustedFileMode
 
     try {
       // in unit tests, calling loadTrustedSetup() twice has error so we have to free and retry
-      ckzg.loadTrustedSetup(setupFilePath);
+      ckzg.loadTrustedSetup(precompute, setupFilePath);
     } catch (e) {
       if ((e as Error).message !== "Error trusted setup is already loaded") {
         throw e;

packages/beacon-node/test/e2e/network/mdns.test.ts

@@ -86,7 +86,7 @@ describe.skip("mdns", function () {
 
     const opts = await getOpts(peerId);
 
-    const modules: Omit<NetworkInitModules, "opts" | "peerId" | "logger"> = {
+    const modules: Omit<NetworkInitModules, "opts" | "peerId" | "logger" | "nodeId"> = {
       config,
       chain,
       db,

packages/beacon-node/test/e2e/network/onWorker/dataSerialization.test.ts

@@ -79,7 +79,7 @@ describe.skip("data serialization through worker boundary", function () {
 
   // Defining tests in this notation ensures that any event data is tested and probably safe to send
   const networkEventData = filterByUsedEvents<NetworkEventData>(networkEventDirection, {
-    [NetworkEvent.peerConnected]: {peer, status: statusZero},
+    [NetworkEvent.peerConnected]: {peer, status: statusZero, dataColumns: []},
     [NetworkEvent.peerDisconnected]: {peer},
     [NetworkEvent.reqRespRequest]: {
       request: {method: ReqRespMethod.Status, body: statusZero},

packages/beacon-node/test/perf/chain/produceBlock/produceBlockBody.test.ts

@@ -1,3 +1,4 @@
+import crypto from "node:crypto";
 import {fromHexString} from "@chainsafe/ssz";
 import {itBench} from "@dapplion/benchmark";
 import {config} from "@lodestar/config/default";
@@ -39,6 +40,7 @@ describe("produceBlockBody", () => {
         minSameMessageSignatureSetsToBatch: 32,
       },
       {
+        nodeId: Buffer.alloc(32, crypto.randomBytes(32)),
         config: state.config,
         db,
         logger,

packages/beacon-node/test/perf/chain/verifyImportBlocks.test.ts

@@ -1,3 +1,4 @@
+import crypto from "node:crypto";
 import {itBench, setBenchOpts} from "@dapplion/benchmark";
 import {config} from "@lodestar/config/default";
 import {SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY, SLOTS_PER_EPOCH} from "@lodestar/params";
@@ -95,6 +96,7 @@ describe.skip("verify+import blocks - range sync perf test", () => {
           minSameMessageSignatureSetsToBatch: 32,
         },
         {
+          nodeId: Buffer.alloc(32, crypto.randomBytes(32)),
           config: state.config,
           db,
           logger,

packages/beacon-node/test/spec/presets/fork_choice.test.ts

@@ -1,4 +1,5 @@
 import path from "node:path";
+import crypto from "node:crypto";
 import {expect} from "vitest";
 import {toHexString} from "@chainsafe/ssz";
 import {BeaconStateAllForks, isExecutionStateType, signedBlockToSignedHeader} from "@lodestar/state-transition";
@@ -101,6 +102,7 @@ const forkChoiceTest =
             proposerBoostReorg: true,
           },
           {
+            nodeId: Buffer.alloc(32, crypto.randomBytes(32)),
             config: createBeaconConfig(config, state.genesisValidatorsRoot),
             db: getMockedBeaconDb(),
             logger,

packages/beacon-node/test/unit/chain/seenCache/seenGossipBlockInput.test.ts

@@ -15,7 +15,11 @@ describe("SeenGossipBlockInput", () => {
   });
   const genesisValidatorsRoot = Buffer.alloc(32, 0xaa);
   const config = createBeaconConfig(chainConfig, genesisValidatorsRoot);
-  const seenGossipBlockInput = new SeenGossipBlockInput();
+  const seenGossipBlockInput = new SeenGossipBlockInput({
+    custodyColumns: [],
+    custodyColumnsIndex: [],
+    custodyColumnsLen: 0,
+  });
 
   // array of numBlobs, events where events are array of
   // [block|blob11|blob2, pd | bp | null | error string reflecting the expected result]

packages/beacon-node/test/unit/db/api/repositories/dataColumn.test.ts

@@ -53,7 +53,10 @@ describe("block archive repository", function () {
     singedBlock.message.body.blobKzgCommitments.push(commitment);
     singedBlock.message.body.blobKzgCommitments.push(commitment);
     singedBlock.message.body.blobKzgCommitments.push(commitment);
-    const dataColumnSidecars = computeDataColumnSidecars(config, singedBlock, {blobs: [blob, blob, blob]});
+    const dataColumnSidecars = computeDataColumnSidecars(config, singedBlock, {
+      blobs: [blob, blob, blob],
+      kzgProofs: [commitment, commitment, commitment],
+    });
     for (let j = 0; j < dataColumnSidecars.length; j++) {
       dataColumnSidecars[j].index = j;
     }

packages/beacon-node/test/utils/networkWithMockDb.ts

@@ -1,3 +1,4 @@
+import crypto from "node:crypto";
 import {createSecp256k1PeerId} from "@libp2p/peer-id-factory";
 import {ChainForkConfig, createBeaconConfig} from "@lodestar/config";
 import {ssz} from "@lodestar/types";
@@ -56,6 +57,7 @@ export async function getNetworkForTest(
       minSameMessageSignatureSetsToBatch: 32,
     },
     {
+      nodeId: Buffer.alloc(32, crypto.randomBytes(32)),
       config: beaconConfig,
       db,
       logger,