feat: download blocks as ssz (#5923)

* feat: getBlock api to support application/octet-stream header

* fix: use specific handler for getBlock getBlockV2

* fix: build error

* Revert "fix: build error"

This reverts commit fbeb88a.

* fix: infer returned type for getBlock apis

* feat: add getBlock() client api supporting ssz

* chore: remove comments

packages/api/src/beacon/client/beacon.ts

@@ -1,13 +1,46 @@
 import {ChainForkConfig} from "@lodestar/config";
-import {Api, ReqTypes, routesData, getReqSerializers, getReturnTypes} from "../routes/beacon/index.js";
-import {IHttpClient, generateGenericJsonClient} from "../../utils/client/index.js";
+import {Api, ReqTypes, routesData, getReqSerializers, getReturnTypes, BlockId} from "../routes/beacon/index.js";
+import {IHttpClient, generateGenericJsonClient, getFetchOptsSerializers} from "../../utils/client/index.js";
+import {ResponseFormat} from "../../interfaces.js";
+import {BlockResponse, BlockV2Response} from "../routes/beacon/block.js";
 
 /**
  * REST HTTP client for beacon routes
  */
 export function getClient(config: ChainForkConfig, httpClient: IHttpClient): Api {
   const reqSerializers = getReqSerializers(config);
   const returnTypes = getReturnTypes();
-  // All routes return JSON, use a client auto-generator
-  return generateGenericJsonClient<Api, ReqTypes>(routesData, reqSerializers, returnTypes, httpClient);
+  // Some routes return JSON, use a client auto-generator
+  const client = generateGenericJsonClient<Api, ReqTypes>(routesData, reqSerializers, returnTypes, httpClient);
+  const fetchOptsSerializer = getFetchOptsSerializers<Api, ReqTypes>(routesData, reqSerializers);
+
+  return {
+    ...client,
+    async getBlock<T extends ResponseFormat = "json">(blockId: BlockId, format?: T) {
+      if (format === "ssz") {
+        const res = await httpClient.arrayBuffer({
+          ...fetchOptsSerializer.getBlock(blockId, format),
+        });
+        return {
+          ok: true,
+          response: new Uint8Array(res.body),
+          status: res.status,
+        } as BlockResponse<T>;
+      }
+      return client.getBlock(blockId, format);
+    },
+    async getBlockV2<T extends ResponseFormat = "json">(blockId: BlockId, format?: T) {
+      if (format === "ssz") {
+        const res = await httpClient.arrayBuffer({
+          ...fetchOptsSerializer.getBlockV2(blockId, format),
+        });
+        return {
+          ok: true,
+          response: new Uint8Array(res.body),
+          status: res.status,
+        } as BlockV2Response<T>;
+      }
+      return client.getBlockV2(blockId, format);
+    },
+  };
 }

packages/api/src/beacon/routes/beacon/block.ts

@@ -18,7 +18,7 @@ import {
   ContainerData,
 } from "../../../utils/index.js";
 import {HttpStatusCode} from "../../../utils/client/httpStatusCode.js";
-import {ApiClientResponse} from "../../../interfaces.js";
+import {ApiClientResponse, ResponseFormat} from "../../../interfaces.js";
 import {
   SignedBlockContents,
   SignedBlindedBlockContents,
@@ -31,6 +31,7 @@ import {
 // See /packages/api/src/routes/index.ts for reasoning and instructions to add new routes
 
 export type BlockId = RootHex | Slot | "head" | "genesis" | "finalized";
+export const mimeTypeSSZ = "application/octet-stream";
 
 /**
  * True if the response references an unverified execution payload. Optimistic information may be invalidated at
@@ -51,6 +52,26 @@ export enum BroadcastValidation {
   consensusAndEquivocation = "consensus_and_equivocation",
 }
 
+export type BlockResponse<T extends ResponseFormat = "json"> = T extends "ssz"
+  ? ApiClientResponse<{[HttpStatusCode.OK]: Uint8Array}, HttpStatusCode.BAD_REQUEST | HttpStatusCode.NOT_FOUND>
+  : ApiClientResponse<
+      {[HttpStatusCode.OK]: {data: allForks.SignedBeaconBlock}},
+      HttpStatusCode.BAD_REQUEST | HttpStatusCode.NOT_FOUND
+    >;
+
+export type BlockV2Response<T extends ResponseFormat = "json"> = T extends "ssz"
+  ? ApiClientResponse<{[HttpStatusCode.OK]: Uint8Array}, HttpStatusCode.BAD_REQUEST | HttpStatusCode.NOT_FOUND>
+  : ApiClientResponse<
+      {
+        [HttpStatusCode.OK]: {
+          data: allForks.SignedBeaconBlock;
+          executionOptimistic: ExecutionOptimistic;
+          version: ForkName;
+        };
+      },
+      HttpStatusCode.BAD_REQUEST | HttpStatusCode.NOT_FOUND
+    >;
+
 export type Api = {
   /**
    * Get block
@@ -60,26 +81,15 @@ export type Api = {
    * @param blockId Block identifier.
    * Can be one of: "head" (canonical head in node's view), "genesis", "finalized", \<slot\>, \<hex encoded blockRoot with 0x prefix\>.
    */
-  getBlock(blockId: BlockId): Promise<ApiClientResponse<{[HttpStatusCode.OK]: {data: allForks.SignedBeaconBlock}}>>;
+  getBlock<T extends ResponseFormat = "json">(blockId: BlockId, format?: T): Promise<BlockResponse<T>>;
 
   /**
    * Get block
    * Retrieves block details for given block id.
    * @param blockId Block identifier.
    * Can be one of: "head" (canonical head in node's view), "genesis", "finalized", \<slot\>, \<hex encoded blockRoot with 0x prefix\>.
    */
-  getBlockV2(blockId: BlockId): Promise<
-    ApiClientResponse<
-      {
-        [HttpStatusCode.OK]: {
-          data: allForks.SignedBeaconBlock;
-          executionOptimistic: ExecutionOptimistic;
-          version: ForkName;
-        };
-      },
-      HttpStatusCode.BAD_REQUEST | HttpStatusCode.NOT_FOUND
-    >
-  >;
+  getBlockV2<T extends ResponseFormat = "json">(blockId: BlockId, format?: T): Promise<BlockV2Response<T>>;
 
   /**
    * Get block attestations
@@ -246,11 +256,12 @@ export const routesData: RoutesData<Api> = {
 
 /* eslint-disable @typescript-eslint/naming-convention */
 
+type GetBlockReq = {params: {block_id: string}; headers: {accept?: string}};
 type BlockIdOnlyReq = {params: {block_id: string}};
 
 export type ReqTypes = {
-  getBlock: BlockIdOnlyReq;
-  getBlockV2: BlockIdOnlyReq;
+  getBlock: GetBlockReq;
+  getBlockV2: GetBlockReq;
   getBlockAttestations: BlockIdOnlyReq;
   getBlockHeader: BlockIdOnlyReq;
   getBlockHeaders: {query: {slot?: number; parent_root?: string}};
@@ -263,12 +274,21 @@ export type ReqTypes = {
 };
 
 export function getReqSerializers(config: ChainForkConfig): ReqSerializers<Api, ReqTypes> {
-  const blockIdOnlyReq: ReqSerializer<Api["getBlock"], BlockIdOnlyReq> = {
+  const blockIdOnlyReq: ReqSerializer<Api["getBlockHeader"], BlockIdOnlyReq> = {
     writeReq: (block_id) => ({params: {block_id: String(block_id)}}),
     parseReq: ({params}) => [params.block_id],
     schema: {params: {block_id: Schema.StringRequired}},
   };
 
+  const getBlockReq: ReqSerializer<Api["getBlock"], GetBlockReq> = {
+    writeReq: (block_id, format) => ({
+      params: {block_id: String(block_id)},
+      headers: {accept: format === "ssz" ? mimeTypeSSZ : "application/json"},
+    }),
+    parseReq: ({params, headers}) => [params.block_id, headers.accept === mimeTypeSSZ ? "ssz" : "json"],
+    schema: {params: {block_id: Schema.StringRequired}},
+  };
+
   // Compute block type from JSON payload. See https://github.com/ethereum/eth2.0-APIs/pull/142
   const getSignedBeaconBlockType = (data: allForks.SignedBeaconBlock): allForks.AllForksSSZTypes["SignedBeaconBlock"] =>
     config.getForkTypes(data.message.slot).SignedBeaconBlock;
@@ -304,8 +324,8 @@ export function getReqSerializers(config: ChainForkConfig): ReqSerializers<Api,
     };
 
   return {
-    getBlock: blockIdOnlyReq,
-    getBlockV2: blockIdOnlyReq,
+    getBlock: getBlockReq,
+    getBlockV2: getBlockReq,
     getBlockAttestations: blockIdOnlyReq,
     getBlockHeader: blockIdOnlyReq,
     getBlockHeaders: {

packages/api/src/beacon/server/beacon.ts

@@ -4,6 +4,41 @@ import {ServerRoutes, getGenericJsonServer} from "../../utils/server/index.js";
 import {ServerApi} from "../../interfaces.js";
 
 export function getRoutes(config: ChainForkConfig, api: ServerApi<Api>): ServerRoutes<Api, ReqTypes> {
-  // All routes return JSON, use a server auto-generator
-  return getGenericJsonServer<ServerApi<Api>, ReqTypes>({routesData, getReturnTypes, getReqSerializers}, config, api);
+  const reqSerializers = getReqSerializers(config);
+  const returnTypes = getReturnTypes();
+
+  // Most of routes return JSON, use a server auto-generator
+  const serverRoutes = getGenericJsonServer<ServerApi<Api>, ReqTypes>(
+    {routesData, getReturnTypes, getReqSerializers},
+    config,
+    api
+  );
+  return {
+    ...serverRoutes,
+    // Non-JSON routes. Return JSON or binary depending on "accept" header
+    getBlock: {
+      ...serverRoutes.getBlock,
+      handler: async (req) => {
+        const response = await api.getBlock(...reqSerializers.getBlock.parseReq(req));
+        if (response instanceof Uint8Array) {
+          // Fastify 3.x.x will automatically add header `Content-Type: application/octet-stream` if Buffer
+          return Buffer.from(response);
+        } else {
+          return returnTypes.getBlock.toJson(response);
+        }
+      },
+    },
+    getBlockV2: {
+      ...serverRoutes.getBlockV2,
+      handler: async (req) => {
+        const response = await api.getBlockV2(...reqSerializers.getBlockV2.parseReq(req));
+        if (response instanceof Uint8Array) {
+          // Fastify 3.x.x will automatically add header `Content-Type: application/octet-stream` if Buffer
+          return Buffer.from(response);
+        } else {
+          return returnTypes.getBlockV2.toJson(response);
+        }
+      },
+    },
+  };
 }

packages/api/src/interfaces.ts

@@ -3,6 +3,7 @@ import {Resolves} from "./utils/types.js";
 
 /* eslint-disable @typescript-eslint/no-explicit-any */
 
+export type ResponseFormat = "json" | "ssz";
 export type APIClientHandler = (...args: any) => PromiseLike<ApiClientResponse>;
 export type APIServerHandler = (...args: any) => PromiseLike<unknown>;
 

packages/api/test/unit/beacon/testData/beacon.ts

@@ -30,11 +30,11 @@ export const testData: GenericServerTestCases<Api> = {
   // block
 
   getBlock: {
-    args: ["head"],
+    args: ["head", "json"],
     res: {data: ssz.phase0.SignedBeaconBlock.defaultValue()},
   },
   getBlockV2: {
-    args: ["head"],
+    args: ["head", "json"],
     res: {executionOptimistic: true, data: ssz.bellatrix.SignedBeaconBlock.defaultValue(), version: ForkName.bellatrix},
   },
   getBlockAttestations: {

packages/beacon-node/src/api/impl/beacon/blocks/index.ts

@@ -1,5 +1,5 @@
 import {fromHexString, toHexString} from "@chainsafe/ssz";
-import {routes, ServerApi, isSignedBlockContents, isSignedBlindedBlockContents} from "@lodestar/api";
+import {routes, ServerApi, isSignedBlockContents, isSignedBlindedBlockContents, ResponseFormat} from "@lodestar/api";
 import {computeTimeAtSlot} from "@lodestar/state-transition";
 import {SLOTS_PER_HISTORICAL_ROOT} from "@lodestar/params";
 import {sleep, toHex} from "@lodestar/utils";
@@ -243,15 +243,21 @@ export function getBeaconBlockApi({
       };
     },
 
-    async getBlock(blockId) {
+    async getBlock(blockId, format?: ResponseFormat) {
       const {block} = await resolveBlockId(chain, blockId);
+      if (format === "ssz") {
+        return config.getForkTypes(block.message.slot).SignedBeaconBlock.serialize(block);
+      }
       return {
         data: block,
       };
     },
 
-    async getBlockV2(blockId) {
+    async getBlockV2(blockId, format?: ResponseFormat) {
       const {block, executionOptimistic} = await resolveBlockId(chain, blockId);
+      if (format === "ssz") {
+        return config.getForkTypes(block.message.slot).SignedBeaconBlock.serialize(block);
+      }
       return {
         executionOptimistic,
         data: block,

packages/beacon-node/test/sim/mergemock.test.ts

@@ -5,7 +5,7 @@ import {LogLevel, sleep} from "@lodestar/utils";
 import {TimestampFormatCode} from "@lodestar/logger";
 import {SLOTS_PER_EPOCH} from "@lodestar/params";
 import {ChainConfig} from "@lodestar/config";
-import {Epoch, bellatrix} from "@lodestar/types";
+import {Epoch, allForks, bellatrix} from "@lodestar/types";
 import {ValidatorProposerConfig, BuilderSelection} from "@lodestar/validator";
 import {routes} from "@lodestar/api";
 
@@ -210,7 +210,9 @@ describe("executionEngine / ExecutionEngineHttp", function () {
     let builderBlocks = 0;
     await new Promise<void>((resolve, _reject) => {
       bn.chain.emitter.on(routes.events.EventType.block, async (blockData) => {
-        const {data: fullOrBlindedBlock} = await bn.api.beacon.getBlockV2(blockData.block);
+        const {data: fullOrBlindedBlock} = (await bn.api.beacon.getBlockV2(blockData.block)) as {
+          data: allForks.SignedBeaconBlock;
+        };
         if (fullOrBlindedBlock !== undefined) {
           const blockFeeRecipient = toHexString(
             (fullOrBlindedBlock as bellatrix.SignedBeaconBlock).message.body.executionPayload.feeRecipient

packages/beacon-node/test/sim/withdrawal-interop.test.ts

@@ -6,7 +6,7 @@ import {TimestampFormatCode} from "@lodestar/logger";
 import {SLOTS_PER_EPOCH, ForkName} from "@lodestar/params";
 import {ChainConfig} from "@lodestar/config";
 import {computeStartSlotAtEpoch} from "@lodestar/state-transition";
-import {Epoch, capella, Slot} from "@lodestar/types";
+import {Epoch, capella, Slot, allForks} from "@lodestar/types";
 import {ValidatorProposerConfig} from "@lodestar/validator";
 
 import {ExecutionPayloadStatus, PayloadAttributes} from "../../src/execution/engine/interface.js";
@@ -369,7 +369,10 @@ async function retrieveCanonicalWithdrawals(bn: BeaconNode, fromSlot: Slot, toSl
     });
 
     if (block) {
-      if ((block.data as capella.SignedBeaconBlock).message.body.executionPayload?.withdrawals.length > 0) {
+      if (
+        ((block as {data: allForks.SignedBeaconBlock}).data as capella.SignedBeaconBlock).message.body.executionPayload
+          ?.withdrawals.length > 0
+      ) {
         withdrawalsBlocks++;
       }
     }

packages/state-transition/test/perf/analyzeBlocks.ts

@@ -1,5 +1,6 @@
 import {getClient, ApiError} from "@lodestar/api";
 import {config} from "@lodestar/config/default";
+import {allForks} from "@lodestar/types";
 import {getInfuraBeaconUrl} from "../utils/infura.js";
 
 // Analyze how Ethereum Consensus blocks are in a target network to prepare accurate performance states and blocks
@@ -52,7 +53,7 @@ async function run(): Promise<void> {
       }
       ApiError.assert(result.value);
 
-      const block = result.value.response.data;
+      const block = (result.value.response as {data: allForks.SignedBeaconBlock}).data;
 
       blocks++;
       attestations += block.message.body.attestations.length;