From b0f53c5ce97928213ff84cf510851f034e97adda Mon Sep 17 00:00:00 2001 From: harkamal Date: Sun, 20 Aug 2023 21:32:59 +0530 Subject: [PATCH] implement produce blockv3 --- packages/api/src/beacon/routes/validator.ts | 8 +- .../src/api/impl/validator/index.ts | 156 ++++++++++++++++-- 2 files changed, 150 insertions(+), 14 deletions(-) diff --git a/packages/api/src/beacon/routes/validator.ts b/packages/api/src/beacon/routes/validator.ts index 7e13bd2625dd..6ee489d72f17 100644 --- a/packages/api/src/beacon/routes/validator.ts +++ b/packages/api/src/beacon/routes/validator.ts @@ -55,6 +55,10 @@ export type ProduceBlindedBlockOrContentsRes = {executionPayloadValue: Wei} & ( | {data: BlindedBlockContents; version: ForkBlobs} ); +export type ProduceFullOrBlindedBlockOrContentsRes = + | (ProduceBlockOrContentsRes & {executionPayloadBlinded: false}) + | (ProduceBlindedBlockOrContentsRes & {executionPayloadBlinded: true}); + // See /packages/api/src/routes/index.ts for reasoning and instructions to add new routes export type BeaconCommitteeSubscription = { @@ -263,9 +267,7 @@ export type Api = { ): Promise< ApiClientResponse< { - [HttpStatusCode.OK]: - | (ProduceBlockOrContentsRes & {executionPayloadBlinded: false}) - | (ProduceBlindedBlockOrContentsRes & {executionPayloadBlinded: true}); + [HttpStatusCode.OK]: ProduceFullOrBlindedBlockOrContentsRes; }, HttpStatusCode.BAD_REQUEST | HttpStatusCode.SERVICE_UNAVAILABLE > diff --git a/packages/beacon-node/src/api/impl/validator/index.ts b/packages/beacon-node/src/api/impl/validator/index.ts index d67f755b8d5d..a99b87106f14 100644 --- a/packages/beacon-node/src/api/impl/validator/index.ts +++ b/packages/beacon-node/src/api/impl/validator/index.ts @@ -28,7 +28,7 @@ import { BLSSignature, } from "@lodestar/types"; import {ExecutionStatus} from "@lodestar/fork-choice"; -import {toHex} from "@lodestar/utils"; +import {toHex, racePromisesWithCutoff, RaceEvent} from "@lodestar/utils"; import {AttestationError, AttestationErrorCode, GossipAction, SyncCommitteeError} from "../../../chain/errors/index.js"; import {validateApiAggregateAndProof} from "../../../chain/validation/index.js"; import {ZERO_HASH} from "../../../constants/index.js"; @@ -61,6 +61,25 @@ import {computeSubnetForCommitteesAtSlot, getPubkeysForIndices} from "./utils.js */ const SYNC_TOLERANCE_EPOCHS = 1; +export enum BuilderSelection { + BuilderAlways = "builderalways", + MaxProfit = "maxprofit", + /** Only activate builder flow for DVT block proposal protocols */ + BuilderOnly = "builderonly", +} + +/** + * Cutoff time to wait for execution and builder block production apis to resolve + * Post this time, race execution and builder to pick whatever resolves first + * + * Emprically the builder block resolves in ~1.5+ seconds, and executon should resolve <1 sec. + * So lowering the cutoff to 2 sec from 3 seconds to publish faster for successful proposal + * as proposals post 4 seconds into the slot seems to be not being included + */ +const BLOCK_PRODUCTION_RACE_CUTOFF_MS = 2_000; +/** Overall timeout for execution and block production apis */ +const BLOCK_PRODUCTION_RACE_TIMEOUT_MS = 12_000; + /** * Server implementation for handling validator duties. * See `@lodestar/validator/src/api` for the client implementation). @@ -245,9 +264,16 @@ export function getValidatorApi({ const produceBlindedBlockOrContents = async function produceBlindedBlockOrContents( slot: Slot, randaoReveal: BLSSignature, - graffiti: string + graffiti: string, + feeRecipient?: string, + {strictFeeRecipientCheck}: {strictFeeRecipientCheck?: boolean} = {} ): Promise { const source = ProducedBlockSource.builder; + // TODO PR + if (strictFeeRecipientCheck) { + throw Error(`strictFeeRecipientCheck not implemented yet for source=${source}`); + } + let timer; metrics?.blockProductionRequests.inc({source}); try { @@ -303,9 +329,15 @@ export function getValidatorApi({ slot: Slot, randaoReveal: BLSSignature, graffiti: string, - feeRecipient?: string + feeRecipient?: string, + {strictFeeRecipientCheck}: {strictFeeRecipientCheck?: boolean} = {} ): Promise { const source = ProducedBlockSource.engine; + // TODO PR + if (strictFeeRecipientCheck) { + throw Error(`strictFeeRecipientCheck not implemented yet for source=${source}`); + } + let timer; metrics?.blockProductionRequests.inc({source}); try { @@ -354,11 +386,20 @@ export function getValidatorApi({ graffiti, feeRecipient, // TODO deneb: skip randao verification - _skipRandaoVerification?: boolean + _skipRandaoVerification?: boolean, + { + builderSelection, + isBuilderEnabled, + strictFeeRecipientCheck, + }: {builderSelection?: BuilderSelection; isBuilderEnabled?: boolean; strictFeeRecipientCheck?: boolean} = {} ) { + // set some sensible opts + builderSelection = builderSelection ?? BuilderSelection.MaxProfit; + isBuilderEnabled = isBuilderEnabled ?? chain.executionBuilder !== undefined; + // Start calls for building execution and builder blocks const blindedBlockPromise = chain.executionBuilder - ? produceBlindedBlockOrContents(slot, randaoReveal, graffiti) + ? produceBlindedBlockOrContents(slot, randaoReveal, graffiti, feeRecipient, {strictFeeRecipientCheck}) : null; const fullBlockPromise = // At any point either the builder or execution or both flows should be active. @@ -368,17 +409,110 @@ export function getValidatorApi({ // (TODO: independently make sure such an options update is not successful for a validator pubkey) // // So if builder is disabled ignore builder selection of builderonly if caused by user mistake - !chain.executionBuilder + !isBuilderEnabled || builderSelection !== BuilderSelection.BuilderOnly ? // TODO deneb: builderSelection needs to be figured out if to be done beacon side // || builderSelection !== BuilderSelection.BuilderOnly - produceFullBlockOrContents(slot, randaoReveal, graffiti, feeRecipient) + produceFullBlockOrContents(slot, randaoReveal, graffiti, feeRecipient, {strictFeeRecipientCheck}) : null; - // just throw random error for now - if (fullBlockPromise === null || blindedBlockPromise === null) { - throw Error("random error"); + let blindedBlock, fullBlock; + if (blindedBlockPromise !== null && fullBlockPromise !== null) { + // reference index of promises in the race + const promisesOrder = [ProducedBlockSource.builder, ProducedBlockSource.engine]; + [blindedBlock, fullBlock] = await racePromisesWithCutoff< + routes.validator.ProduceBlockOrContentsRes | routes.validator.ProduceBlindedBlockOrContentsRes + >( + [blindedBlockPromise, fullBlockPromise], + BLOCK_PRODUCTION_RACE_CUTOFF_MS, + BLOCK_PRODUCTION_RACE_TIMEOUT_MS, + // Callback to log the race events for better debugging capability + (event: RaceEvent, delayMs: number, index?: number) => { + const eventRef = index !== undefined ? {source: promisesOrder[index]} : {}; + logger.debug("Block production race (builder vs execution)", { + event, + ...eventRef, + delayMs, + cutoffMs: BLOCK_PRODUCTION_RACE_CUTOFF_MS, + timeoutMs: BLOCK_PRODUCTION_RACE_TIMEOUT_MS, + }); + } + ); + if (blindedBlock instanceof Error) { + // error here means race cutoff exceeded + logger.error("Failed to produce builder block", {}, blindedBlock); + blindedBlock = null; + } + if (fullBlock instanceof Error) { + logger.error("Failed to produce execution block", {}, fullBlock); + fullBlock = null; + } + } else if (blindedBlockPromise !== null && fullBlockPromise === null) { + blindedBlock = await blindedBlockPromise; + fullBlock = null; + } else if (blindedBlockPromise === null && fullBlockPromise !== null) { + blindedBlock = null; + fullBlock = await fullBlockPromise; } else { - throw Error("random error"); + throw Error( + `Internal Error: Neither builder nor execution proposal flow activated isBuilderEnabled=${isBuilderEnabled} builderSelection=${builderSelection}` + ); + } + + const builderPayloadValue = blindedBlock?.executionPayloadValue ?? BigInt(0); + const enginePayloadValue = fullBlock?.executionPayloadValue ?? BigInt(0); + + let selectedSource: ProducedBlockSource | null = null; + + if (fullBlock && blindedBlock) { + switch (builderSelection) { + case BuilderSelection.MaxProfit: { + // If executionPayloadValues are zero, than choose builder as most likely beacon didn't provide executionPayloadValue + // and builder blocks are most likely thresholded by a min bid + if (enginePayloadValue >= builderPayloadValue && enginePayloadValue !== BigInt(0)) { + selectedSource = ProducedBlockSource.engine; + } else { + selectedSource = ProducedBlockSource.builder; + } + break; + } + + // For everything else just select the builder + default: { + selectedSource = ProducedBlockSource.builder; + } + } + logger.debug(`Selected ${selectedSource} block`, { + builderSelection, + // winston logger doesn't like bigint + enginePayloadValue: `${enginePayloadValue}`, + builderPayloadValue: `${builderPayloadValue}`, + }); + } else if (fullBlock && !blindedBlock) { + selectedSource = ProducedBlockSource.engine; + logger.debug("Selected engine block: no builder block produced", { + // winston logger doesn't like bigint + enginePayloadValue: `${enginePayloadValue}`, + }); + } else if (blindedBlock && !fullBlock) { + selectedSource = ProducedBlockSource.builder; + logger.debug("Selected builder block: no engine block produced", { + // winston logger doesn't like bigint + builderPayloadValue: `${builderPayloadValue}`, + }); + } + + if (selectedSource === null) { + throw Error("Failed to produce engine or builder block"); + } + + if (selectedSource === ProducedBlockSource.engine) { + return {...fullBlock, executionPayloadBlinded: false} as routes.validator.ProduceBlockOrContentsRes & { + executionPayloadBlinded: false; + }; + } else { + return {...blindedBlock, executionPayloadBlinded: true} as routes.validator.ProduceBlindedBlockOrContentsRes & { + executionPayloadBlinded: true; + }; } };