feat: implement blob sidecars with inclusion proof instead of signtau…

…res (#6089) modify the api fix validator handling fix val build refactor the beacon node impl for the new blobs architecture get the build working with no blobs fix the blob sidecars transmisssion relocate compute blob sidecars relocate verify merkle branch verify inclusion proof fix tests and types fix unit test update the spec test versions skip newly required merkle proof runner change the minimal/mainnet preset based constant strategy add other constants to tests apply feedback apply feedback relocate utils
ChainSafe · Dec 25, 2023 · 6f4a9d6 · 6f4a9d6
1 parent 61cf1a8
commit 6f4a9d6
Show file tree

Hide file tree

Showing 61 changed files with 521 additions and 808 deletions.
diff --git a/packages/api/src/beacon/routes/beacon/block.ts b/packages/api/src/beacon/routes/beacon/block.ts
@@ -1,17 +1,7 @@
 import {ContainerType} from "@chainsafe/ssz";
 import {ForkName} from "@lodestar/params";
 import {ChainForkConfig} from "@lodestar/config";
-import {
-  phase0,
-  allForks,
-  Slot,
-  Root,
-  ssz,
-  RootHex,
-  deneb,
-  isSignedBlockContents,
-  isSignedBlindedBlockContents,
-} from "@lodestar/types";
+import {phase0, allForks, Slot, Root, ssz, RootHex, deneb, isSignedBlockContents} from "@lodestar/types";
 
 import {
   RoutesData,
@@ -30,10 +20,7 @@ import {
 import {HttpStatusCode} from "../../../utils/client/httpStatusCode.js";
 import {parseAcceptHeader, writeAcceptHeader} from "../../../utils/acceptHeader.js";
 import {ApiClientResponse, ResponseFormat} from "../../../interfaces.js";
-import {
-  allForksSignedBlockContentsReqSerializer,
-  allForksSignedBlindedBlockContentsReqSerializer,
-} from "../../../utils/routes.js";
+import {allForksSignedBlockContentsReqSerializer} from "../../../utils/routes.js";
 
 // See /packages/api/src/routes/index.ts for reasoning and instructions to add new routes
 
@@ -207,7 +194,7 @@ export type Api = {
    * Publish a signed blinded block by submitting it to the mev relay and patching in the block
    * transactions beacon node gets in response.
    */
-  publishBlindedBlock(blindedBlockOrContents: allForks.SignedBlindedBeaconBlockOrContents): Promise<
+  publishBlindedBlock(blindedBlock: allForks.SignedBlindedBeaconBlock): Promise<
     ApiClientResponse<
       {
         [HttpStatusCode.OK]: void;
@@ -218,7 +205,7 @@ export type Api = {
   >;
 
   publishBlindedBlockV2(
-    blindedBlockOrContents: allForks.SignedBlindedBeaconBlockOrContents,
+    blindedBlockOrContents: allForks.SignedBlindedBeaconBlock,
     opts: {broadcastValidation?: BroadcastValidation}
   ): Promise<
     ApiClientResponse<
@@ -315,16 +302,9 @@ export function getReqSerializers(config: ChainForkConfig): ReqSerializers<Api,
   ): allForks.AllForksBlindedSSZTypes["SignedBeaconBlock"] =>
     config.getBlindedForkTypes(data.message.slot).SignedBeaconBlock;
 
-  const AllForksSignedBlindedBlockOrContents: TypeJson<allForks.SignedBlindedBeaconBlockOrContents> = {
-    toJson: (data) =>
-      isSignedBlindedBlockContents(data)
-        ? allForksSignedBlindedBlockContentsReqSerializer(getSignedBlindedBeaconBlockType).toJson(data)
-        : getSignedBlindedBeaconBlockType(data).toJson(data),
-
-    fromJson: (data) =>
-      (data as {signed_blinded_block: unknown}).signed_blinded_block !== undefined
-        ? allForksSignedBlindedBlockContentsReqSerializer(getSignedBlindedBeaconBlockType).fromJson(data)
-        : getSignedBlindedBeaconBlockType(data as allForks.SignedBlindedBeaconBlock).fromJson(data),
+  const AllForksSignedBlindedBlock: TypeJson<allForks.SignedBlindedBeaconBlock> = {
+    toJson: (data) => getSignedBlindedBeaconBlockType(data).toJson(data),
+    fromJson: (data) => getSignedBlindedBeaconBlockType(data as allForks.SignedBlindedBeaconBlock).fromJson(data),
   };
 
   return {
@@ -353,14 +333,14 @@ export function getReqSerializers(config: ChainForkConfig): ReqSerializers<Api,
         query: {broadcast_validation: Schema.String},
       },
     },
-    publishBlindedBlock: reqOnlyBody(AllForksSignedBlindedBlockOrContents, Schema.Object),
+    publishBlindedBlock: reqOnlyBody(AllForksSignedBlindedBlock, Schema.Object),
     publishBlindedBlockV2: {
       writeReq: (item, {broadcastValidation}) => ({
-        body: AllForksSignedBlindedBlockOrContents.toJson(item),
+        body: AllForksSignedBlindedBlock.toJson(item),
         query: {broadcast_validation: broadcastValidation},
       }),
       parseReq: ({body, query}) => [
-        AllForksSignedBlindedBlockOrContents.fromJson(body),
+        AllForksSignedBlindedBlock.fromJson(body),
         {broadcastValidation: query.broadcast_validation as BroadcastValidation},
       ],
       schema: {

diff --git a/packages/api/src/beacon/routes/validator.ts b/packages/api/src/beacon/routes/validator.ts
@@ -1,5 +1,5 @@
 import {ContainerType, fromHexString, toHexString, Type} from "@chainsafe/ssz";
-import {ForkName, ForkBlobs, isForkBlobs, isForkExecution, ForkPreBlobs} from "@lodestar/params";
+import {ForkName, ForkBlobs, isForkBlobs, isForkExecution, ForkPreBlobs, ForkExecution} from "@lodestar/params";
 import {
   allForks,
   altair,
@@ -37,7 +37,7 @@ import {
   TypeJson,
 } from "../../utils/index.js";
 import {fromU64Str, fromGraffitiHex, toU64Str, U64Str, toGraffitiHex} from "../../utils/serdes.js";
-import {allForksBlockContentsResSerializer, allForksBlindedBlockContentsResSerializer} from "../../utils/routes.js";
+import {allForksBlockContentsResSerializer} from "../../utils/routes.js";
 import {ExecutionOptimistic} from "./beacon/block.js";
 
 export enum BuilderSelection {
@@ -59,14 +59,14 @@ export type ProduceBlockOrContentsRes = {executionPayloadValue: Wei; consensusBl
   | {data: allForks.BeaconBlock; version: ForkPreBlobs}
   | {data: allForks.BlockContents; version: ForkBlobs}
 );
-export type ProduceBlindedBlockOrContentsRes = {executionPayloadValue: Wei; consensusBlockValue: Gwei} & (
-  | {data: allForks.BlindedBeaconBlock; version: ForkPreBlobs}
-  | {data: allForks.BlindedBlockContents; version: ForkBlobs}
-);
+export type ProduceBlindedBlockRes = {executionPayloadValue: Wei; consensusBlockValue: Gwei} & {
+  data: allForks.BlindedBeaconBlock;
+  version: ForkExecution;
+};
 
 export type ProduceFullOrBlindedBlockOrContentsRes =
   | (ProduceBlockOrContentsRes & {executionPayloadBlinded: false})
-  | (ProduceBlindedBlockOrContentsRes & {executionPayloadBlinded: true});
+  | (ProduceBlindedBlockRes & {executionPayloadBlinded: true});
 
 // See /packages/api/src/routes/index.ts for reasoning and instructions to add new routes
 
@@ -287,7 +287,7 @@ export type Api = {
   ): Promise<
     ApiClientResponse<
       {
-        [HttpStatusCode.OK]: ProduceBlindedBlockOrContentsRes;
+        [HttpStatusCode.OK]: ProduceBlindedBlockRes;
       },
       HttpStatusCode.BAD_REQUEST | HttpStatusCode.SERVICE_UNAVAILABLE
     >
@@ -721,13 +721,11 @@ export function getReturnTypes(): ReturnTypes<Api> {
       isForkBlobs(fork) ? allForksBlockContentsResSerializer(fork) : ssz[fork].BeaconBlock
     )
   ) as TypeJson<ProduceBlockOrContentsRes>;
-  const produceBlindedBlockOrContents = WithBlockValues(
-    WithVersion<allForks.BlindedBeaconBlockOrContents>((fork: ForkName) =>
-      isForkBlobs(fork)
-        ? allForksBlindedBlockContentsResSerializer(fork)
-        : ssz.allForksBlinded[isForkExecution(fork) ? fork : ForkName.bellatrix].BeaconBlock
+  const produceBlindedBlock = WithBlockValues(
+    WithVersion<allForks.BlindedBeaconBlock>(
+      (fork: ForkName) => ssz.allForksBlinded[isForkExecution(fork) ? fork : ForkName.bellatrix].BeaconBlock
     )
-  ) as TypeJson<ProduceBlindedBlockOrContentsRes>;
+  ) as TypeJson<ProduceBlindedBlockRes>;
 
   return {
     getAttesterDuties: WithDependentRootExecutionOptimistic(ArrayOf(AttesterDuty)),
@@ -741,7 +739,7 @@ export function getReturnTypes(): ReturnTypes<Api> {
         if (data.executionPayloadBlinded) {
           return {
             execution_payload_blinded: true,
-            ...(produceBlindedBlockOrContents.toJson(data) as Record<string, unknown>),
+            ...(produceBlindedBlock.toJson(data) as Record<string, unknown>),
           };
         } else {
           return {
@@ -752,13 +750,13 @@ export function getReturnTypes(): ReturnTypes<Api> {
       },
       fromJson: (data) => {
         if ((data as {execution_payload_blinded: true}).execution_payload_blinded) {
-          return {executionPayloadBlinded: true, ...produceBlindedBlockOrContents.fromJson(data)};
+          return {executionPayloadBlinded: true, ...produceBlindedBlock.fromJson(data)};
         } else {
           return {executionPayloadBlinded: false, ...produceBlockOrContents.fromJson(data)};
         }
       },
     },
-    produceBlindedBlock: produceBlindedBlockOrContents,
+    produceBlindedBlock,
 
     produceAttestationData: ContainerData(ssz.phase0.AttestationData),
     produceSyncCommitteeContribution: ContainerData(ssz.altair.SyncCommitteeContribution),

diff --git a/packages/api/src/builder/routes.ts b/packages/api/src/builder/routes.ts
@@ -34,7 +34,7 @@ export type Api = {
       HttpStatusCode.NOT_FOUND | HttpStatusCode.BAD_REQUEST
     >
   >;
-  submitBlindedBlock(signedBlock: allForks.SignedBlindedBeaconBlockOrContents): Promise<
+  submitBlindedBlock(signedBlock: allForks.SignedBlindedBeaconBlock): Promise<
     ApiClientResponse<
       {
         [HttpStatusCode.OK]: {

diff --git a/packages/api/src/utils/routes.ts b/packages/api/src/utils/routes.ts
@@ -11,12 +11,14 @@ export function allForksSignedBlockContentsReqSerializer(
   return {
     toJson: (data) => ({
       signed_block: blockSerializer(data.signedBlock).toJson(data.signedBlock),
-      signed_blob_sidecars: ssz.deneb.SignedBlobSidecars.toJson(data.signedBlobSidecars),
+      kzg_proofs: ssz.deneb.KZGProofs.toJson(data.kzgProofs),
+      blobs: ssz.deneb.Blobs.toJson(data.blobs),
     }),
 
-    fromJson: (data: {signed_block: unknown; signed_blob_sidecars: unknown}) => ({
+    fromJson: (data: {signed_block: unknown; kzg_proofs: unknown; blobs: unknown}) => ({
       signedBlock: blockSerializer(data.signed_block as allForks.SignedBeaconBlock).fromJson(data.signed_block),
-      signedBlobSidecars: ssz.deneb.SignedBlobSidecars.fromJson(data.signed_blob_sidecars),
+      kzgProofs: ssz.deneb.KZGProofs.fromJson(data.kzg_proofs),
+      blobs: ssz.deneb.Blobs.fromJson(data.blobs),
     }),
   };
 }
@@ -25,44 +27,13 @@ export function allForksBlockContentsResSerializer(fork: ForkBlobs): TypeJson<al
   return {
     toJson: (data) => ({
       block: (ssz.allForks[fork].BeaconBlock as allForks.AllForksSSZTypes["BeaconBlock"]).toJson(data.block),
-      blob_sidecars: ssz.deneb.BlobSidecars.toJson(data.blobSidecars),
+      kzg_proofs: ssz.deneb.KZGProofs.toJson(data.kzgProofs),
+      blobs: ssz.deneb.Blobs.toJson(data.blobs),
     }),
-    fromJson: (data: {block: unknown; blob_sidecars: unknown}) => ({
+    fromJson: (data: {block: unknown; blob_sidecars: unknown; kzg_proofs: unknown; blobs: unknown}) => ({
       block: ssz.allForks[fork].BeaconBlock.fromJson(data.block),
-      blobSidecars: ssz.deneb.BlobSidecars.fromJson(data.blob_sidecars),
-    }),
-  };
-}
-
-export function allForksSignedBlindedBlockContentsReqSerializer(
-  blockSerializer: (data: allForks.SignedBlindedBeaconBlock) => TypeJson<allForks.SignedBlindedBeaconBlock>
-): TypeJson<allForks.SignedBlindedBlockContents> {
-  return {
-    toJson: (data) => ({
-      signed_blinded_block: blockSerializer(data.signedBlindedBlock).toJson(data.signedBlindedBlock),
-      signed_blinded_blob_sidecars: ssz.deneb.SignedBlindedBlobSidecars.toJson(data.signedBlindedBlobSidecars),
-    }),
-
-    fromJson: (data: {signed_blinded_block: unknown; signed_blinded_blob_sidecars: unknown}) => ({
-      signedBlindedBlock: blockSerializer(data.signed_blinded_block as allForks.SignedBlindedBeaconBlock).fromJson(
-        data.signed_blinded_block
-      ),
-      signedBlindedBlobSidecars: ssz.deneb.SignedBlindedBlobSidecars.fromJson(data.signed_blinded_blob_sidecars),
-    }),
-  };
-}
-
-export function allForksBlindedBlockContentsResSerializer(fork: ForkBlobs): TypeJson<allForks.BlindedBlockContents> {
-  return {
-    toJson: (data) => ({
-      blinded_block: (ssz.allForksBlinded[fork].BeaconBlock as allForks.AllForksBlindedSSZTypes["BeaconBlock"]).toJson(
-        data.blindedBlock
-      ),
-      blinded_blob_sidecars: ssz.deneb.BlindedBlobSidecars.toJson(data.blindedBlobSidecars),
-    }),
-    fromJson: (data: {blinded_block: unknown; blinded_blob_sidecars: unknown}) => ({
-      blindedBlock: ssz.allForksBlinded[fork].BeaconBlock.fromJson(data.blinded_block),
-      blindedBlobSidecars: ssz.deneb.BlindedBlobSidecars.fromJson(data.blinded_blob_sidecars),
+      kzgProofs: ssz.deneb.KZGProofs.fromJson(data.kzg_proofs),
+      blobs: ssz.deneb.Blobs.fromJson(data.blobs),
     }),
   };
 }
diff --git a/packages/beacon-node/src/api/impl/beacon/blocks/index.ts b/packages/beacon-node/src/api/impl/beacon/blocks/index.ts
@@ -1,16 +1,13 @@
 import {fromHexString, toHexString} from "@chainsafe/ssz";
 import {routes, ServerApi, ResponseFormat} from "@lodestar/api";
-import {
-  computeTimeAtSlot,
-  parseSignedBlindedBlockOrContents,
-  reconstructFullBlockOrContents,
-} from "@lodestar/state-transition";
+import {computeTimeAtSlot, reconstructFullBlockOrContents} from "@lodestar/state-transition";
 import {SLOTS_PER_HISTORICAL_ROOT} from "@lodestar/params";
 import {sleep, toHex} from "@lodestar/utils";
 import {allForks, deneb, isSignedBlockContents, ProducedBlockSource} from "@lodestar/types";
 import {BlockSource, getBlockInput, ImportBlockOpts, BlockInput} from "../../../../chain/blocks/types.js";
 import {promiseAllMaybeAsync} from "../../../../util/promises.js";
 import {isOptimisticBlock} from "../../../../util/forkChoice.js";
+import {computeBlobSidecars} from "../../../../util/blobs.js";
 import {BlockError, BlockErrorCode} from "../../../../chain/errors/index.js";
 import {OpSource} from "../../../../metrics/validatorMonitor.js";
 import {NetworkEvent} from "../../../../network/index.js";
@@ -45,22 +42,23 @@ export function getBeaconBlockApi({
     opts: PublishBlockOpts = {}
   ) => {
     const seenTimestampSec = Date.now() / 1000;
-    let blockForImport: BlockInput, signedBlock: allForks.SignedBeaconBlock, signedBlobs: deneb.SignedBlobSidecars;
+    let blockForImport: BlockInput, signedBlock: allForks.SignedBeaconBlock, blobSidecars: deneb.BlobSidecars;
 
     if (isSignedBlockContents(signedBlockOrContents)) {
-      ({signedBlock, signedBlobSidecars: signedBlobs} = signedBlockOrContents);
+      ({signedBlock} = signedBlockOrContents);
+      blobSidecars = computeBlobSidecars(config, signedBlock, signedBlockOrContents);
       blockForImport = getBlockInput.postDeneb(
         config,
         signedBlock,
         BlockSource.api,
-        signedBlobs.map((sblob) => sblob.message),
+        blobSidecars,
         // don't bundle any bytes for block and blobs
         null,
-        signedBlobs.map(() => null)
+        blobSidecars.map(() => null)
       );
     } else {
       signedBlock = signedBlockOrContents;
-      signedBlobs = [];
+      blobSidecars = [];
       // TODO: Once API supports submitting data as SSZ, replace null with blockBytes
       blockForImport = getBlockInput.preDeneb(config, signedBlock, BlockSource.api, null);
     }
@@ -188,18 +186,15 @@ export function getBeaconBlockApi({
           }
           throw e;
         }),
-      ...signedBlobs.map((signedBlob) => () => network.publishBlobSidecar(signedBlob)),
+      ...blobSidecars.map((blobSidecar) => () => network.publishBlobSidecar(blobSidecar)),
     ];
     await promiseAllMaybeAsync(publishPromises);
   };
 
   const publishBlindedBlock: ServerApi<routes.beacon.block.Api>["publishBlindedBlock"] = async (
-    signedBlindedBlockOrContents,
+    signedBlindedBlock,
     opts: PublishBlockOpts = {}
   ) => {
-    const {signedBlindedBlock, signedBlindedBlobSidecars} =
-      parseSignedBlindedBlockOrContents(signedBlindedBlockOrContents);
-
     const slot = signedBlindedBlock.message.slot;
     const blockRoot = toHex(
       chain.config
@@ -210,27 +205,31 @@ export function getBeaconBlockApi({
     // Either the payload/blobs are cached from i) engine locally or ii) they are from the builder
     //
     // executionPayload can be null or a real payload in locally produced so check for presence of root
-    const source = chain.producedBlockRoot.has(blockRoot) ? ProducedBlockSource.engine : ProducedBlockSource.builder;
-
-    const executionPayload = chain.producedBlockRoot.get(blockRoot) ?? null;
-    const blobSidecars = executionPayload
-      ? chain.producedBlobSidecarsCache.get(toHex(executionPayload.blockHash))
-      : undefined;
-    const blobs = blobSidecars ? blobSidecars.map((blobSidecar) => blobSidecar.blob) : null;
-
-    chain.logger.debug("Assembling blinded block for publishing", {source, blockRoot, slot});
+    const executionPayload = chain.producedBlockRoot.get(blockRoot);
+    if (executionPayload !== undefined) {
+      const source = ProducedBlockSource.engine;
+      chain.logger.debug("Reconstructing  signedBlockOrContents", {blockRoot, slot, source});
+
+      const contents = executionPayload
+        ? chain.producedContentsCache.get(toHex(executionPayload.blockHash)) ?? null
+        : null;
+      const signedBlockOrContents = reconstructFullBlockOrContents(signedBlindedBlock, {executionPayload, contents});
+
+      chain.logger.info("Publishing assembled block", {blockRoot, slot, source});
+      return publishBlock(signedBlockOrContents, opts);
+    } else {
+      const source = ProducedBlockSource.builder;
+      chain.logger.debug("Reconstructing  signedBlockOrContents", {blockRoot, slot, source});
 
-    const signedBlockOrContents =
-      source === ProducedBlockSource.engine
-        ? reconstructFullBlockOrContents({signedBlindedBlock, signedBlindedBlobSidecars}, {executionPayload, blobs})
-        : await reconstructBuilderBlockOrContents(chain, signedBlindedBlockOrContents);
+      const signedBlockOrContents = await reconstructBuilderBlockOrContents(chain, signedBlindedBlock);
 
-    // the full block is published by relay and it's possible that the block is already known to us
-    // by gossip
-    //
-    // see: https://github.com/ChainSafe/lodestar/issues/5404
-    chain.logger.info("Publishing assembled block", {blockRoot, slot, source});
-    return publishBlock(signedBlockOrContents, {...opts, ignoreIfKnown: true});
+      // the full block is published by relay and it's possible that the block is already known to us
+      // by gossip
+      //
+      // see: https://github.com/ChainSafe/lodestar/issues/5404
+      chain.logger.info("Publishing assembled block", {blockRoot, slot, source});
+      return publishBlock(signedBlockOrContents, {...opts, ignoreIfKnown: true});
+    }
   };
 
   return {
@@ -424,13 +423,13 @@ export function getBeaconBlockApi({
 
 async function reconstructBuilderBlockOrContents(
   chain: ApiModules["chain"],
-  signedBlindedBlockOrContents: allForks.SignedBlindedBeaconBlockOrContents
+  signedBlindedBlock: allForks.SignedBlindedBeaconBlock
 ): Promise<allForks.SignedBeaconBlockOrContents> {
   const executionBuilder = chain.executionBuilder;
   if (!executionBuilder) {
     throw Error("exeutionBuilder required to publish SignedBlindedBeaconBlock");
   }
 
-  const signedBlockOrContents = await executionBuilder.submitBlindedBlock(signedBlindedBlockOrContents);
+  const signedBlockOrContents = await executionBuilder.submitBlindedBlock(signedBlindedBlock);
   return signedBlockOrContents;
 }