From 8f308c09fe75b582160f121895d160ad1cf902ab Mon Sep 17 00:00:00 2001 From: tuyennhv Date: Mon, 11 Sep 2023 22:14:04 +0700 Subject: [PATCH] feat: download blocks as ssz (#5923) * feat: getBlock api to support application/octet-stream header * fix: use specific handler for getBlock getBlockV2 * fix: build error * Revert "fix: build error" This reverts commit fbeb88a2fddeeb375e69307a976007d6680b6430. * fix: infer returned type for getBlock apis * feat: add getBlock() client api supporting ssz * chore: remove comments --- packages/api/src/beacon/client/beacon.ts | 41 +++++++++++-- .../api/src/beacon/routes/beacon/block.ts | 58 +++++++++++++------ packages/api/src/beacon/server/beacon.ts | 39 ++++++++++++- packages/api/src/interfaces.ts | 1 + .../api/test/unit/beacon/testData/beacon.ts | 4 +- .../src/api/impl/beacon/blocks/index.ts | 12 +++- .../beacon-node/test/sim/mergemock.test.ts | 6 +- .../test/sim/withdrawal-interop.test.ts | 7 ++- .../test/perf/analyzeBlocks.ts | 3 +- 9 files changed, 136 insertions(+), 35 deletions(-) diff --git a/packages/api/src/beacon/client/beacon.ts b/packages/api/src/beacon/client/beacon.ts index 875cd32c0465..7a92afe15c6f 100644 --- a/packages/api/src/beacon/client/beacon.ts +++ b/packages/api/src/beacon/client/beacon.ts @@ -1,6 +1,8 @@ import {ChainForkConfig} from "@lodestar/config"; -import {Api, ReqTypes, routesData, getReqSerializers, getReturnTypes} from "../routes/beacon/index.js"; -import {IHttpClient, generateGenericJsonClient} from "../../utils/client/index.js"; +import {Api, ReqTypes, routesData, getReqSerializers, getReturnTypes, BlockId} from "../routes/beacon/index.js"; +import {IHttpClient, generateGenericJsonClient, getFetchOptsSerializers} from "../../utils/client/index.js"; +import {ResponseFormat} from "../../interfaces.js"; +import {BlockResponse, BlockV2Response} from "../routes/beacon/block.js"; /** * REST HTTP client for beacon routes @@ -8,6 +10,37 @@ import {IHttpClient, generateGenericJsonClient} from "../../utils/client/index.j export function getClient(config: ChainForkConfig, httpClient: IHttpClient): Api { const reqSerializers = getReqSerializers(config); const returnTypes = getReturnTypes(); - // All routes return JSON, use a client auto-generator - return generateGenericJsonClient(routesData, reqSerializers, returnTypes, httpClient); + // Some routes return JSON, use a client auto-generator + const client = generateGenericJsonClient(routesData, reqSerializers, returnTypes, httpClient); + const fetchOptsSerializer = getFetchOptsSerializers(routesData, reqSerializers); + + return { + ...client, + async getBlock(blockId: BlockId, format?: T) { + if (format === "ssz") { + const res = await httpClient.arrayBuffer({ + ...fetchOptsSerializer.getBlock(blockId, format), + }); + return { + ok: true, + response: new Uint8Array(res.body), + status: res.status, + } as BlockResponse; + } + return client.getBlock(blockId, format); + }, + async getBlockV2(blockId: BlockId, format?: T) { + if (format === "ssz") { + const res = await httpClient.arrayBuffer({ + ...fetchOptsSerializer.getBlockV2(blockId, format), + }); + return { + ok: true, + response: new Uint8Array(res.body), + status: res.status, + } as BlockV2Response; + } + return client.getBlockV2(blockId, format); + }, + }; } diff --git a/packages/api/src/beacon/routes/beacon/block.ts b/packages/api/src/beacon/routes/beacon/block.ts index a36f4505dc5f..c281c69047c7 100644 --- a/packages/api/src/beacon/routes/beacon/block.ts +++ b/packages/api/src/beacon/routes/beacon/block.ts @@ -18,7 +18,7 @@ import { ContainerData, } from "../../../utils/index.js"; import {HttpStatusCode} from "../../../utils/client/httpStatusCode.js"; -import {ApiClientResponse} from "../../../interfaces.js"; +import {ApiClientResponse, ResponseFormat} from "../../../interfaces.js"; import { SignedBlockContents, SignedBlindedBlockContents, @@ -31,6 +31,7 @@ import { // See /packages/api/src/routes/index.ts for reasoning and instructions to add new routes export type BlockId = RootHex | Slot | "head" | "genesis" | "finalized"; +export const mimeTypeSSZ = "application/octet-stream"; /** * True if the response references an unverified execution payload. Optimistic information may be invalidated at @@ -51,6 +52,26 @@ export enum BroadcastValidation { consensusAndEquivocation = "consensus_and_equivocation", } +export type BlockResponse = T extends "ssz" + ? ApiClientResponse<{[HttpStatusCode.OK]: Uint8Array}, HttpStatusCode.BAD_REQUEST | HttpStatusCode.NOT_FOUND> + : ApiClientResponse< + {[HttpStatusCode.OK]: {data: allForks.SignedBeaconBlock}}, + HttpStatusCode.BAD_REQUEST | HttpStatusCode.NOT_FOUND + >; + +export type BlockV2Response = T extends "ssz" + ? ApiClientResponse<{[HttpStatusCode.OK]: Uint8Array}, HttpStatusCode.BAD_REQUEST | HttpStatusCode.NOT_FOUND> + : ApiClientResponse< + { + [HttpStatusCode.OK]: { + data: allForks.SignedBeaconBlock; + executionOptimistic: ExecutionOptimistic; + version: ForkName; + }; + }, + HttpStatusCode.BAD_REQUEST | HttpStatusCode.NOT_FOUND + >; + export type Api = { /** * Get block @@ -60,7 +81,7 @@ export type Api = { * @param blockId Block identifier. * Can be one of: "head" (canonical head in node's view), "genesis", "finalized", \, \. */ - getBlock(blockId: BlockId): Promise>; + getBlock(blockId: BlockId, format?: T): Promise>; /** * Get block @@ -68,18 +89,7 @@ export type Api = { * @param blockId Block identifier. * Can be one of: "head" (canonical head in node's view), "genesis", "finalized", \, \. */ - getBlockV2(blockId: BlockId): Promise< - ApiClientResponse< - { - [HttpStatusCode.OK]: { - data: allForks.SignedBeaconBlock; - executionOptimistic: ExecutionOptimistic; - version: ForkName; - }; - }, - HttpStatusCode.BAD_REQUEST | HttpStatusCode.NOT_FOUND - > - >; + getBlockV2(blockId: BlockId, format?: T): Promise>; /** * Get block attestations @@ -246,11 +256,12 @@ export const routesData: RoutesData = { /* eslint-disable @typescript-eslint/naming-convention */ +type GetBlockReq = {params: {block_id: string}; headers: {accept?: string}}; type BlockIdOnlyReq = {params: {block_id: string}}; export type ReqTypes = { - getBlock: BlockIdOnlyReq; - getBlockV2: BlockIdOnlyReq; + getBlock: GetBlockReq; + getBlockV2: GetBlockReq; getBlockAttestations: BlockIdOnlyReq; getBlockHeader: BlockIdOnlyReq; getBlockHeaders: {query: {slot?: number; parent_root?: string}}; @@ -263,12 +274,21 @@ export type ReqTypes = { }; export function getReqSerializers(config: ChainForkConfig): ReqSerializers { - const blockIdOnlyReq: ReqSerializer = { + const blockIdOnlyReq: ReqSerializer = { writeReq: (block_id) => ({params: {block_id: String(block_id)}}), parseReq: ({params}) => [params.block_id], schema: {params: {block_id: Schema.StringRequired}}, }; + const getBlockReq: ReqSerializer = { + writeReq: (block_id, format) => ({ + params: {block_id: String(block_id)}, + headers: {accept: format === "ssz" ? mimeTypeSSZ : "application/json"}, + }), + parseReq: ({params, headers}) => [params.block_id, headers.accept === mimeTypeSSZ ? "ssz" : "json"], + schema: {params: {block_id: Schema.StringRequired}}, + }; + // Compute block type from JSON payload. See https://github.com/ethereum/eth2.0-APIs/pull/142 const getSignedBeaconBlockType = (data: allForks.SignedBeaconBlock): allForks.AllForksSSZTypes["SignedBeaconBlock"] => config.getForkTypes(data.message.slot).SignedBeaconBlock; @@ -304,8 +324,8 @@ export function getReqSerializers(config: ChainForkConfig): ReqSerializers): ServerRoutes { - // All routes return JSON, use a server auto-generator - return getGenericJsonServer, ReqTypes>({routesData, getReturnTypes, getReqSerializers}, config, api); + const reqSerializers = getReqSerializers(config); + const returnTypes = getReturnTypes(); + + // Most of routes return JSON, use a server auto-generator + const serverRoutes = getGenericJsonServer, ReqTypes>( + {routesData, getReturnTypes, getReqSerializers}, + config, + api + ); + return { + ...serverRoutes, + // Non-JSON routes. Return JSON or binary depending on "accept" header + getBlock: { + ...serverRoutes.getBlock, + handler: async (req) => { + const response = await api.getBlock(...reqSerializers.getBlock.parseReq(req)); + if (response instanceof Uint8Array) { + // Fastify 3.x.x will automatically add header `Content-Type: application/octet-stream` if Buffer + return Buffer.from(response); + } else { + return returnTypes.getBlock.toJson(response); + } + }, + }, + getBlockV2: { + ...serverRoutes.getBlockV2, + handler: async (req) => { + const response = await api.getBlockV2(...reqSerializers.getBlockV2.parseReq(req)); + if (response instanceof Uint8Array) { + // Fastify 3.x.x will automatically add header `Content-Type: application/octet-stream` if Buffer + return Buffer.from(response); + } else { + return returnTypes.getBlockV2.toJson(response); + } + }, + }, + }; } diff --git a/packages/api/src/interfaces.ts b/packages/api/src/interfaces.ts index 166596297ace..4d57bcbc8934 100644 --- a/packages/api/src/interfaces.ts +++ b/packages/api/src/interfaces.ts @@ -3,6 +3,7 @@ import {Resolves} from "./utils/types.js"; /* eslint-disable @typescript-eslint/no-explicit-any */ +export type ResponseFormat = "json" | "ssz"; export type APIClientHandler = (...args: any) => PromiseLike; export type APIServerHandler = (...args: any) => PromiseLike; diff --git a/packages/api/test/unit/beacon/testData/beacon.ts b/packages/api/test/unit/beacon/testData/beacon.ts index 85068fb83e48..bb9697cf9587 100644 --- a/packages/api/test/unit/beacon/testData/beacon.ts +++ b/packages/api/test/unit/beacon/testData/beacon.ts @@ -30,11 +30,11 @@ export const testData: GenericServerTestCases = { // block getBlock: { - args: ["head"], + args: ["head", "json"], res: {data: ssz.phase0.SignedBeaconBlock.defaultValue()}, }, getBlockV2: { - args: ["head"], + args: ["head", "json"], res: {executionOptimistic: true, data: ssz.bellatrix.SignedBeaconBlock.defaultValue(), version: ForkName.bellatrix}, }, getBlockAttestations: { diff --git a/packages/beacon-node/src/api/impl/beacon/blocks/index.ts b/packages/beacon-node/src/api/impl/beacon/blocks/index.ts index f6121390d079..11b96d29fa3b 100644 --- a/packages/beacon-node/src/api/impl/beacon/blocks/index.ts +++ b/packages/beacon-node/src/api/impl/beacon/blocks/index.ts @@ -1,5 +1,5 @@ import {fromHexString, toHexString} from "@chainsafe/ssz"; -import {routes, ServerApi, isSignedBlockContents, isSignedBlindedBlockContents} from "@lodestar/api"; +import {routes, ServerApi, isSignedBlockContents, isSignedBlindedBlockContents, ResponseFormat} from "@lodestar/api"; import {computeTimeAtSlot} from "@lodestar/state-transition"; import {SLOTS_PER_HISTORICAL_ROOT} from "@lodestar/params"; import {sleep, toHex} from "@lodestar/utils"; @@ -243,15 +243,21 @@ export function getBeaconBlockApi({ }; }, - async getBlock(blockId) { + async getBlock(blockId, format?: ResponseFormat) { const {block} = await resolveBlockId(chain, blockId); + if (format === "ssz") { + return config.getForkTypes(block.message.slot).SignedBeaconBlock.serialize(block); + } return { data: block, }; }, - async getBlockV2(blockId) { + async getBlockV2(blockId, format?: ResponseFormat) { const {block, executionOptimistic} = await resolveBlockId(chain, blockId); + if (format === "ssz") { + return config.getForkTypes(block.message.slot).SignedBeaconBlock.serialize(block); + } return { executionOptimistic, data: block, diff --git a/packages/beacon-node/test/sim/mergemock.test.ts b/packages/beacon-node/test/sim/mergemock.test.ts index d2dc37f893f5..d835aafa6a44 100644 --- a/packages/beacon-node/test/sim/mergemock.test.ts +++ b/packages/beacon-node/test/sim/mergemock.test.ts @@ -5,7 +5,7 @@ import {LogLevel, sleep} from "@lodestar/utils"; import {TimestampFormatCode} from "@lodestar/logger"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {ChainConfig} from "@lodestar/config"; -import {Epoch, bellatrix} from "@lodestar/types"; +import {Epoch, allForks, bellatrix} from "@lodestar/types"; import {ValidatorProposerConfig, BuilderSelection} from "@lodestar/validator"; import {routes} from "@lodestar/api"; @@ -210,7 +210,9 @@ describe("executionEngine / ExecutionEngineHttp", function () { let builderBlocks = 0; await new Promise((resolve, _reject) => { bn.chain.emitter.on(routes.events.EventType.block, async (blockData) => { - const {data: fullOrBlindedBlock} = await bn.api.beacon.getBlockV2(blockData.block); + const {data: fullOrBlindedBlock} = (await bn.api.beacon.getBlockV2(blockData.block)) as { + data: allForks.SignedBeaconBlock; + }; if (fullOrBlindedBlock !== undefined) { const blockFeeRecipient = toHexString( (fullOrBlindedBlock as bellatrix.SignedBeaconBlock).message.body.executionPayload.feeRecipient diff --git a/packages/beacon-node/test/sim/withdrawal-interop.test.ts b/packages/beacon-node/test/sim/withdrawal-interop.test.ts index 4243d9175f14..8976ae9e89d0 100644 --- a/packages/beacon-node/test/sim/withdrawal-interop.test.ts +++ b/packages/beacon-node/test/sim/withdrawal-interop.test.ts @@ -6,7 +6,7 @@ import {TimestampFormatCode} from "@lodestar/logger"; import {SLOTS_PER_EPOCH, ForkName} from "@lodestar/params"; import {ChainConfig} from "@lodestar/config"; import {computeStartSlotAtEpoch} from "@lodestar/state-transition"; -import {Epoch, capella, Slot} from "@lodestar/types"; +import {Epoch, capella, Slot, allForks} from "@lodestar/types"; import {ValidatorProposerConfig} from "@lodestar/validator"; import {ExecutionPayloadStatus, PayloadAttributes} from "../../src/execution/engine/interface.js"; @@ -369,7 +369,10 @@ async function retrieveCanonicalWithdrawals(bn: BeaconNode, fromSlot: Slot, toSl }); if (block) { - if ((block.data as capella.SignedBeaconBlock).message.body.executionPayload?.withdrawals.length > 0) { + if ( + ((block as {data: allForks.SignedBeaconBlock}).data as capella.SignedBeaconBlock).message.body.executionPayload + ?.withdrawals.length > 0 + ) { withdrawalsBlocks++; } } diff --git a/packages/state-transition/test/perf/analyzeBlocks.ts b/packages/state-transition/test/perf/analyzeBlocks.ts index 8bd472d76ba5..f9e26b4f5238 100644 --- a/packages/state-transition/test/perf/analyzeBlocks.ts +++ b/packages/state-transition/test/perf/analyzeBlocks.ts @@ -1,5 +1,6 @@ import {getClient, ApiError} from "@lodestar/api"; import {config} from "@lodestar/config/default"; +import {allForks} from "@lodestar/types"; import {getInfuraBeaconUrl} from "../utils/infura.js"; // Analyze how Ethereum Consensus blocks are in a target network to prepare accurate performance states and blocks @@ -52,7 +53,7 @@ async function run(): Promise { } ApiError.assert(result.value); - const block = result.value.response.data; + const block = (result.value.response as {data: allForks.SignedBeaconBlock}).data; blocks++; attestations += block.message.body.attestations.length;