feat: implement blob sidecars with inclusion proof instead of signtaures

packages/api/src/beacon/routes/beacon/block.ts

@@ -1,17 +1,7 @@
 import {ContainerType} from "@chainsafe/ssz";
 import {ForkName} from "@lodestar/params";
 import {ChainForkConfig} from "@lodestar/config";
-import {
-  phase0,
-  allForks,
-  Slot,
-  Root,
-  ssz,
-  RootHex,
-  deneb,
-  isSignedBlockContents,
-  isSignedBlindedBlockContents,
-} from "@lodestar/types";
+import {phase0, allForks, Slot, Root, ssz, RootHex, deneb, isSignedBlockContents} from "@lodestar/types";
 
 import {
   RoutesData,
@@ -30,10 +20,7 @@ import {
 import {HttpStatusCode} from "../../../utils/client/httpStatusCode.js";
 import {parseAcceptHeader, writeAcceptHeader} from "../../../utils/acceptHeader.js";
 import {ApiClientResponse, ResponseFormat} from "../../../interfaces.js";
-import {
-  allForksSignedBlockContentsReqSerializer,
-  allForksSignedBlindedBlockContentsReqSerializer,
-} from "../../../utils/routes.js";
+import {allForksSignedBlockContentsReqSerializer} from "../../../utils/routes.js";
 
 // See /packages/api/src/routes/index.ts for reasoning and instructions to add new routes
 
@@ -207,7 +194,7 @@ export type Api = {
    * Publish a signed blinded block by submitting it to the mev relay and patching in the block
    * transactions beacon node gets in response.
    */
-  publishBlindedBlock(blindedBlockOrContents: allForks.SignedBlindedBeaconBlockOrContents): Promise<
+  publishBlindedBlock(blindedBlock: allForks.SignedBlindedBeaconBlock): Promise<
     ApiClientResponse<
       {
         [HttpStatusCode.OK]: void;
@@ -218,7 +205,7 @@ export type Api = {
   >;
 
   publishBlindedBlockV2(
-    blindedBlockOrContents: allForks.SignedBlindedBeaconBlockOrContents,
+    blindedBlockOrContents: allForks.SignedBlindedBeaconBlock,
     opts: {broadcastValidation?: BroadcastValidation}
   ): Promise<
     ApiClientResponse<
@@ -315,16 +302,9 @@ export function getReqSerializers(config: ChainForkConfig): ReqSerializers<Api,
   ): allForks.AllForksBlindedSSZTypes["SignedBeaconBlock"] =>
     config.getBlindedForkTypes(data.message.slot).SignedBeaconBlock;
 
-  const AllForksSignedBlindedBlockOrContents: TypeJson<allForks.SignedBlindedBeaconBlockOrContents> = {
-    toJson: (data) =>
-      isSignedBlindedBlockContents(data)
-        ? allForksSignedBlindedBlockContentsReqSerializer(getSignedBlindedBeaconBlockType).toJson(data)
-        : getSignedBlindedBeaconBlockType(data).toJson(data),
-
-    fromJson: (data) =>
-      (data as {signed_blinded_block: unknown}).signed_blinded_block !== undefined
-        ? allForksSignedBlindedBlockContentsReqSerializer(getSignedBlindedBeaconBlockType).fromJson(data)
-        : getSignedBlindedBeaconBlockType(data as allForks.SignedBlindedBeaconBlock).fromJson(data),
+  const AllForksSignedBlindedBlock: TypeJson<allForks.SignedBlindedBeaconBlock> = {
+    toJson: (data) => getSignedBlindedBeaconBlockType(data).toJson(data),
+    fromJson: (data) => getSignedBlindedBeaconBlockType(data as allForks.SignedBlindedBeaconBlock).fromJson(data),
   };
 
   return {
@@ -353,14 +333,14 @@ export function getReqSerializers(config: ChainForkConfig): ReqSerializers<Api,
         query: {broadcast_validation: Schema.String},
       },
     },
-    publishBlindedBlock: reqOnlyBody(AllForksSignedBlindedBlockOrContents, Schema.Object),
+    publishBlindedBlock: reqOnlyBody(AllForksSignedBlindedBlock, Schema.Object),
     publishBlindedBlockV2: {
       writeReq: (item, {broadcastValidation}) => ({
-        body: AllForksSignedBlindedBlockOrContents.toJson(item),
+        body: AllForksSignedBlindedBlock.toJson(item),
         query: {broadcast_validation: broadcastValidation},
       }),
       parseReq: ({body, query}) => [
-        AllForksSignedBlindedBlockOrContents.fromJson(body),
+        AllForksSignedBlindedBlock.fromJson(body),
         {broadcastValidation: query.broadcast_validation as BroadcastValidation},
       ],
       schema: {

packages/api/src/beacon/routes/validator.ts

@@ -1,5 +1,5 @@
 import {ContainerType, fromHexString, toHexString, Type} from "@chainsafe/ssz";
-import {ForkName, ForkBlobs, isForkBlobs, isForkExecution, ForkPreBlobs} from "@lodestar/params";
+import {ForkName, ForkBlobs, isForkBlobs, isForkExecution, ForkPreBlobs, ForkExecution} from "@lodestar/params";
 import {
   allForks,
   altair,
@@ -37,7 +37,7 @@ import {
   TypeJson,
 } from "../../utils/index.js";
 import {fromU64Str, fromGraffitiHex, toU64Str, U64Str, toGraffitiHex} from "../../utils/serdes.js";
-import {allForksBlockContentsResSerializer, allForksBlindedBlockContentsResSerializer} from "../../utils/routes.js";
+import {allForksBlockContentsResSerializer} from "../../utils/routes.js";
 import {ExecutionOptimistic} from "./beacon/block.js";
 
 export enum BuilderSelection {
@@ -59,14 +59,14 @@ export type ProduceBlockOrContentsRes = {executionPayloadValue: Wei; consensusBl
   | {data: allForks.BeaconBlock; version: ForkPreBlobs}
   | {data: allForks.BlockContents; version: ForkBlobs}
 );
-export type ProduceBlindedBlockOrContentsRes = {executionPayloadValue: Wei; consensusBlockValue: Gwei} & (
-  | {data: allForks.BlindedBeaconBlock; version: ForkPreBlobs}
-  | {data: allForks.BlindedBlockContents; version: ForkBlobs}
-);
+export type ProduceBlindedBlockRes = {executionPayloadValue: Wei; consensusBlockValue: Gwei} & {
+  data: allForks.BlindedBeaconBlock;
+  version: ForkExecution;
+};
 
 export type ProduceFullOrBlindedBlockOrContentsRes =
   | (ProduceBlockOrContentsRes & {executionPayloadBlinded: false})
-  | (ProduceBlindedBlockOrContentsRes & {executionPayloadBlinded: true});
+  | (ProduceBlindedBlockRes & {executionPayloadBlinded: true});
 
 // See /packages/api/src/routes/index.ts for reasoning and instructions to add new routes
 
@@ -287,7 +287,7 @@ export type Api = {
   ): Promise<
     ApiClientResponse<
       {
-        [HttpStatusCode.OK]: ProduceBlindedBlockOrContentsRes;
+        [HttpStatusCode.OK]: ProduceBlindedBlockRes;
       },
       HttpStatusCode.BAD_REQUEST | HttpStatusCode.SERVICE_UNAVAILABLE
     >
@@ -721,13 +721,11 @@ export function getReturnTypes(): ReturnTypes<Api> {
       isForkBlobs(fork) ? allForksBlockContentsResSerializer(fork) : ssz[fork].BeaconBlock
     )
   ) as TypeJson<ProduceBlockOrContentsRes>;
-  const produceBlindedBlockOrContents = WithBlockValues(
-    WithVersion<allForks.BlindedBeaconBlockOrContents>((fork: ForkName) =>
-      isForkBlobs(fork)
-        ? allForksBlindedBlockContentsResSerializer(fork)
-        : ssz.allForksBlinded[isForkExecution(fork) ? fork : ForkName.bellatrix].BeaconBlock
+  const produceBlindedBlock = WithBlockValues(
+    WithVersion<allForks.BlindedBeaconBlock>(
+      (fork: ForkName) => ssz.allForksBlinded[isForkExecution(fork) ? fork : ForkName.bellatrix].BeaconBlock
     )
-  ) as TypeJson<ProduceBlindedBlockOrContentsRes>;
+  ) as TypeJson<ProduceBlindedBlockRes>;
 
   return {
     getAttesterDuties: WithDependentRootExecutionOptimistic(ArrayOf(AttesterDuty)),
@@ -741,7 +739,7 @@ export function getReturnTypes(): ReturnTypes<Api> {
         if (data.executionPayloadBlinded) {
           return {
             execution_payload_blinded: true,
-            ...(produceBlindedBlockOrContents.toJson(data) as Record<string, unknown>),
+            ...(produceBlindedBlock.toJson(data) as Record<string, unknown>),
           };
         } else {
           return {
@@ -752,13 +750,13 @@ export function getReturnTypes(): ReturnTypes<Api> {
       },
       fromJson: (data) => {
         if ((data as {execution_payload_blinded: true}).execution_payload_blinded) {
-          return {executionPayloadBlinded: true, ...produceBlindedBlockOrContents.fromJson(data)};
+          return {executionPayloadBlinded: true, ...produceBlindedBlock.fromJson(data)};
         } else {
           return {executionPayloadBlinded: false, ...produceBlockOrContents.fromJson(data)};
         }
       },
     },
-    produceBlindedBlock: produceBlindedBlockOrContents,
+    produceBlindedBlock,
 
     produceAttestationData: ContainerData(ssz.phase0.AttestationData),
     produceSyncCommitteeContribution: ContainerData(ssz.altair.SyncCommitteeContribution),

packages/api/src/builder/routes.ts

@@ -34,7 +34,7 @@ export type Api = {
       HttpStatusCode.NOT_FOUND | HttpStatusCode.BAD_REQUEST
     >
   >;
-  submitBlindedBlock(signedBlock: allForks.SignedBlindedBeaconBlockOrContents): Promise<
+  submitBlindedBlock(signedBlock: allForks.SignedBlindedBeaconBlock): Promise<
     ApiClientResponse<
       {
         [HttpStatusCode.OK]: {

packages/api/src/utils/routes.ts

@@ -11,12 +11,14 @@ export function allForksSignedBlockContentsReqSerializer(
   return {
     toJson: (data) => ({
       signed_block: blockSerializer(data.signedBlock).toJson(data.signedBlock),
-      signed_blob_sidecars: ssz.deneb.SignedBlobSidecars.toJson(data.signedBlobSidecars),
+      kzg_proofs: ssz.deneb.KZGProofs.toJson(data.kzgProofs),
+      blobs: ssz.deneb.Blobs.toJson(data.blobs),
     }),
 
-    fromJson: (data: {signed_block: unknown; signed_blob_sidecars: unknown}) => ({
+    fromJson: (data: {signed_block: unknown; kzg_proofs: unknown; blobs: unknown}) => ({
       signedBlock: blockSerializer(data.signed_block as allForks.SignedBeaconBlock).fromJson(data.signed_block),
-      signedBlobSidecars: ssz.deneb.SignedBlobSidecars.fromJson(data.signed_blob_sidecars),
+      kzgProofs: ssz.deneb.KZGProofs.fromJson(data.kzg_proofs),
+      blobs: ssz.deneb.Blobs.fromJson(data.blobs),
     }),
   };
 }
@@ -25,44 +27,13 @@ export function allForksBlockContentsResSerializer(fork: ForkBlobs): TypeJson<al
   return {
     toJson: (data) => ({
       block: (ssz.allForks[fork].BeaconBlock as allForks.AllForksSSZTypes["BeaconBlock"]).toJson(data.block),
-      blob_sidecars: ssz.deneb.BlobSidecars.toJson(data.blobSidecars),
+      kzg_proofs: ssz.deneb.KZGProofs.toJson(data.kzgProofs),
+      blobs: ssz.deneb.Blobs.toJson(data.blobs),
     }),
-    fromJson: (data: {block: unknown; blob_sidecars: unknown}) => ({
+    fromJson: (data: {block: unknown; blob_sidecars: unknown; kzg_proofs: unknown; blobs: unknown}) => ({
       block: ssz.allForks[fork].BeaconBlock.fromJson(data.block),
-      blobSidecars: ssz.deneb.BlobSidecars.fromJson(data.blob_sidecars),
-    }),
-  };
-}
-
-export function allForksSignedBlindedBlockContentsReqSerializer(
-  blockSerializer: (data: allForks.SignedBlindedBeaconBlock) => TypeJson<allForks.SignedBlindedBeaconBlock>
-): TypeJson<allForks.SignedBlindedBlockContents> {
-  return {
-    toJson: (data) => ({
-      signed_blinded_block: blockSerializer(data.signedBlindedBlock).toJson(data.signedBlindedBlock),
-      signed_blinded_blob_sidecars: ssz.deneb.SignedBlindedBlobSidecars.toJson(data.signedBlindedBlobSidecars),
-    }),
-
-    fromJson: (data: {signed_blinded_block: unknown; signed_blinded_blob_sidecars: unknown}) => ({
-      signedBlindedBlock: blockSerializer(data.signed_blinded_block as allForks.SignedBlindedBeaconBlock).fromJson(
-        data.signed_blinded_block
-      ),
-      signedBlindedBlobSidecars: ssz.deneb.SignedBlindedBlobSidecars.fromJson(data.signed_blinded_blob_sidecars),
-    }),
-  };
-}
-
-export function allForksBlindedBlockContentsResSerializer(fork: ForkBlobs): TypeJson<allForks.BlindedBlockContents> {
-  return {
-    toJson: (data) => ({
-      blinded_block: (ssz.allForksBlinded[fork].BeaconBlock as allForks.AllForksBlindedSSZTypes["BeaconBlock"]).toJson(
-        data.blindedBlock
-      ),
-      blinded_blob_sidecars: ssz.deneb.BlindedBlobSidecars.toJson(data.blindedBlobSidecars),
-    }),
-    fromJson: (data: {blinded_block: unknown; blinded_blob_sidecars: unknown}) => ({
-      blindedBlock: ssz.allForksBlinded[fork].BeaconBlock.fromJson(data.blinded_block),
-      blindedBlobSidecars: ssz.deneb.BlindedBlobSidecars.fromJson(data.blinded_blob_sidecars),
+      kzgProofs: ssz.deneb.KZGProofs.fromJson(data.kzg_proofs),
+      blobs: ssz.deneb.Blobs.fromJson(data.blobs),
     }),
   };
 }

packages/beacon-node/src/api/impl/beacon/blocks/index.ts

@@ -1,16 +1,13 @@
 import {fromHexString, toHexString} from "@chainsafe/ssz";
 import {routes, ServerApi, ResponseFormat} from "@lodestar/api";
-import {
-  computeTimeAtSlot,
-  parseSignedBlindedBlockOrContents,
-  reconstructFullBlockOrContents,
-} from "@lodestar/state-transition";
+import {computeTimeAtSlot, reconstructFullBlockOrContents} from "@lodestar/state-transition";
 import {SLOTS_PER_HISTORICAL_ROOT} from "@lodestar/params";
 import {sleep, toHex} from "@lodestar/utils";
 import {allForks, deneb, isSignedBlockContents, ProducedBlockSource} from "@lodestar/types";
 import {BlockSource, getBlockInput, ImportBlockOpts, BlockInput} from "../../../../chain/blocks/types.js";
 import {promiseAllMaybeAsync} from "../../../../util/promises.js";
 import {isOptimisticBlock} from "../../../../util/forkChoice.js";
+import {computeBlobSidecars} from "../../../../util/blobs.js";
 import {BlockError, BlockErrorCode} from "../../../../chain/errors/index.js";
 import {OpSource} from "../../../../metrics/validatorMonitor.js";
 import {NetworkEvent} from "../../../../network/index.js";
@@ -45,22 +42,23 @@ export function getBeaconBlockApi({
     opts: PublishBlockOpts = {}
   ) => {
     const seenTimestampSec = Date.now() / 1000;
-    let blockForImport: BlockInput, signedBlock: allForks.SignedBeaconBlock, signedBlobs: deneb.SignedBlobSidecars;
+    let blockForImport: BlockInput, signedBlock: allForks.SignedBeaconBlock, blobSidecars: deneb.BlobSidecars;
 
     if (isSignedBlockContents(signedBlockOrContents)) {
-      ({signedBlock, signedBlobSidecars: signedBlobs} = signedBlockOrContents);
+      ({signedBlock} = signedBlockOrContents);
+      blobSidecars = computeBlobSidecars(config, signedBlock, signedBlockOrContents);
       blockForImport = getBlockInput.postDeneb(
         config,
         signedBlock,
         BlockSource.api,
-        signedBlobs.map((sblob) => sblob.message),
+        blobSidecars,
         // don't bundle any bytes for block and blobs
         null,
-        signedBlobs.map(() => null)
+        blobSidecars.map(() => null)
       );
     } else {
       signedBlock = signedBlockOrContents;
-      signedBlobs = [];
+      blobSidecars = [];
       // TODO: Once API supports submitting data as SSZ, replace null with blockBytes
       blockForImport = getBlockInput.preDeneb(config, signedBlock, BlockSource.api, null);
     }
@@ -188,18 +186,15 @@ export function getBeaconBlockApi({
           }
           throw e;
         }),
-      ...signedBlobs.map((signedBlob) => () => network.publishBlobSidecar(signedBlob)),
+      ...blobSidecars.map((blobSidecar) => () => network.publishBlobSidecar(blobSidecar)),
     ];
     await promiseAllMaybeAsync(publishPromises);
   };
 
   const publishBlindedBlock: ServerApi<routes.beacon.block.Api>["publishBlindedBlock"] = async (
-    signedBlindedBlockOrContents,
+    signedBlindedBlock,
     opts: PublishBlockOpts = {}
   ) => {
-    const {signedBlindedBlock, signedBlindedBlobSidecars} =
-      parseSignedBlindedBlockOrContents(signedBlindedBlockOrContents);
-
     const slot = signedBlindedBlock.message.slot;
     const blockRoot = toHex(
       chain.config
@@ -210,27 +205,31 @@ export function getBeaconBlockApi({
     // Either the payload/blobs are cached from i) engine locally or ii) they are from the builder
     //
     // executionPayload can be null or a real payload in locally produced so check for presence of root
-    const source = chain.producedBlockRoot.has(blockRoot) ? ProducedBlockSource.engine : ProducedBlockSource.builder;
-
-    const executionPayload = chain.producedBlockRoot.get(blockRoot) ?? null;
-    const blobSidecars = executionPayload
-      ? chain.producedBlobSidecarsCache.get(toHex(executionPayload.blockHash))
-      : undefined;
-    const blobs = blobSidecars ? blobSidecars.map((blobSidecar) => blobSidecar.blob) : null;
-
-    chain.logger.debug("Assembling blinded block for publishing", {source, blockRoot, slot});
+    const executionPayload = chain.producedBlockRoot.get(blockRoot);
+    if (executionPayload !== undefined) {
+      const source = ProducedBlockSource.engine;
+      chain.logger.debug("Reconstructing  signedBlockOrContents", {blockRoot, slot, source});
+
+      const contents = executionPayload
+        ? chain.producedContentsCache.get(toHex(executionPayload.blockHash)) ?? null
+        : null;
+      const signedBlockOrContents = reconstructFullBlockOrContents(signedBlindedBlock, {executionPayload, contents});
+
+      chain.logger.info("Publishing assembled block", {blockRoot, slot, source});
+      return publishBlock(signedBlockOrContents, opts);
+    } else {
+      const source = ProducedBlockSource.builder;
+      chain.logger.debug("Reconstructing  signedBlockOrContents", {blockRoot, slot, source});
 
-    const signedBlockOrContents =
-      source === ProducedBlockSource.engine
-        ? reconstructFullBlockOrContents({signedBlindedBlock, signedBlindedBlobSidecars}, {executionPayload, blobs})
-        : await reconstructBuilderBlockOrContents(chain, signedBlindedBlockOrContents);
+      const signedBlockOrContents = await reconstructBuilderBlockOrContents(chain, signedBlindedBlock);
 
-    // the full block is published by relay and it's possible that the block is already known to us
-    // by gossip
-    //
-    // see: https://github.com/ChainSafe/lodestar/issues/5404
-    chain.logger.info("Publishing assembled block", {blockRoot, slot, source});
-    return publishBlock(signedBlockOrContents, {...opts, ignoreIfKnown: true});
+      // the full block is published by relay and it's possible that the block is already known to us
+      // by gossip
+      //
+      // see: https://github.com/ChainSafe/lodestar/issues/5404
+      chain.logger.info("Publishing assembled block", {blockRoot, slot, source});
+      return publishBlock(signedBlockOrContents, {...opts, ignoreIfKnown: true});
+    }
   };
 
   return {
@@ -424,13 +423,13 @@ export function getBeaconBlockApi({
 
 async function reconstructBuilderBlockOrContents(
   chain: ApiModules["chain"],
-  signedBlindedBlockOrContents: allForks.SignedBlindedBeaconBlockOrContents
+  signedBlindedBlock: allForks.SignedBlindedBeaconBlock
 ): Promise<allForks.SignedBeaconBlockOrContents> {
   const executionBuilder = chain.executionBuilder;
   if (!executionBuilder) {
     throw Error("exeutionBuilder required to publish SignedBlindedBeaconBlock");
   }
 
-  const signedBlockOrContents = await executionBuilder.submitBlindedBlock(signedBlindedBlockOrContents);
+  const signedBlockOrContents = await executionBuilder.submitBlindedBlock(signedBlindedBlock);
   return signedBlockOrContents;
 }