feat: track block production selection results and payload values

packages/beacon-node/src/api/impl/validator/index.ts

@@ -47,7 +47,15 @@ import {
   getValidatorStatus,
 } from "@lodestar/types";
 import {ExecutionStatus, DataAvailabilityStatus} from "@lodestar/fork-choice";
-import {fromHex, toHex, resolveOrRacePromises, prettyWeiToEth, toRootHex} from "@lodestar/utils";
+import {
+  fromHex,
+  toHex,
+  resolveOrRacePromises,
+  prettyWeiToEth,
+  toRootHex,
+  TimeoutError,
+  formatWeiToEth,
+} from "@lodestar/utils";
 import {
   AttestationError,
   AttestationErrorCode,
@@ -115,6 +123,31 @@ type ProduceFullOrBlindedBlockOrContentsRes = {executionPayloadSource: ProducedB
   | (ProduceBlindedBlockRes & {executionPayloadBlinded: true})
 );
 
+/**
+ * Engine block selection reasons tracked in metrics
+ */
+export enum EngineBlockSelectionReason {
+  BuilderDisabled = "builder_disabled",
+  BuilderError = "builder_error",
+  BuilderTimeout = "builder_timeout",
+  BuilderPending = "builder_pending",
+  BuilderNoBid = "builder_no_bid",
+  BuilderCensorship = "builder_censorship",
+  BlockValue = "block_value",
+  EnginePreferred = "engine_preferred",
+}
+
+/**
+ * Builder block selection reasons tracked in metrics
+ */
+export enum BuilderBlockSelectionReason {
+  EngineDisabled = "engine_disabled",
+  EngineError = "engine_error",
+  EnginePending = "engine_pending",
+  BlockValue = "block_value",
+  BuilderPreferred = "builder_preferred",
+}
+
 /**
  * Server implementation for handling validator duties.
  * See `@lodestar/validator/src/api` for the client implementation).
@@ -417,6 +450,7 @@ export function getValidatorApi(
 
       metrics?.blockProductionSuccess.inc({source});
       metrics?.blockProductionNumAggregated.observe({source}, block.body.attestations.length);
+      metrics?.blockProductionExecutionPayloadValue.observe({source}, Number(formatWeiToEth(executionPayloadValue)));
       logger.verbose("Produced blinded block", {
         slot,
         executionPayloadValue,
@@ -491,6 +525,7 @@ export function getValidatorApi(
 
       metrics?.blockProductionSuccess.inc({source});
       metrics?.blockProductionNumAggregated.observe({source}, block.body.attestations.length);
+      metrics?.blockProductionExecutionPayloadValue.observe({source}, Number(formatWeiToEth(executionPayloadValue)));
       logger.verbose("Produced execution block", {
         slot,
         executionPayloadValue,
@@ -694,6 +729,11 @@ export function getValidatorApi(
         ...getBlockValueLogInfo(engine.value),
       });
 
+      metrics?.blockProductionSelectionResults.inc({
+        source: ProducedBlockSource.engine,
+        reason: EngineBlockSelectionReason.BuilderCensorship,
+      });
+
       return {...engine.value, executionPayloadBlinded: false, executionPayloadSource: ProducedBlockSource.engine};
     }
 
@@ -704,6 +744,16 @@ export function getValidatorApi(
         ...getBlockValueLogInfo(builder.value),
       });
 
+      metrics?.blockProductionSelectionResults.inc({
+        source: ProducedBlockSource.builder,
+        reason:
+          isEngineEnabled === false
+            ? BuilderBlockSelectionReason.EngineDisabled
+            : engine.status === "pending"
+              ? BuilderBlockSelectionReason.EnginePending
+              : BuilderBlockSelectionReason.EngineError,
+      });
+
       return {...builder.value, executionPayloadBlinded: true, executionPayloadSource: ProducedBlockSource.builder};
     }
 
@@ -714,16 +764,33 @@ export function getValidatorApi(
         ...getBlockValueLogInfo(engine.value),
       });
 
+      metrics?.blockProductionSelectionResults.inc({
+        source: ProducedBlockSource.engine,
+        reason:
+          isBuilderEnabled === false
+            ? EngineBlockSelectionReason.BuilderDisabled
+            : builder.status === "pending"
+              ? EngineBlockSelectionReason.BuilderPending
+              : builder.reason instanceof NoBidReceived
+                ? EngineBlockSelectionReason.BuilderNoBid
+                : builder.reason instanceof TimeoutError
+                  ? EngineBlockSelectionReason.BuilderTimeout
+                  : EngineBlockSelectionReason.BuilderError,
+      });
+
       return {...engine.value, executionPayloadBlinded: false, executionPayloadSource: ProducedBlockSource.engine};
     }
 
     if (engine.status === "fulfilled" && builder.status === "fulfilled") {
-      const executionPayloadSource = selectBlockProductionSource({
-        builderBlockValue: builder.value.executionPayloadValue + builder.value.consensusBlockValue,
-        engineBlockValue: engine.value.executionPayloadValue + engine.value.consensusBlockValue,
-        builderBoostFactor,
-        builderSelection,
-      });
+      const executionPayloadSource = selectBlockProductionSource(
+        {
+          builderBlockValue: builder.value.executionPayloadValue + builder.value.consensusBlockValue,
+          engineBlockValue: engine.value.executionPayloadValue + engine.value.consensusBlockValue,
+          builderBoostFactor,
+          builderSelection,
+        },
+        metrics
+      );
 
       logger.info(`Selected ${executionPayloadSource} block`, {
         ...loggerContext,

packages/beacon-node/src/api/impl/validator/utils.ts

@@ -3,6 +3,8 @@ import {ATTESTATION_SUBNET_COUNT} from "@lodestar/params";
 import {routes} from "@lodestar/api";
 import {BLSPubkey, CommitteeIndex, ProducedBlockSource, Slot, ValidatorIndex} from "@lodestar/types";
 import {MAX_BUILDER_BOOST_FACTOR} from "@lodestar/validator";
+import {Metrics} from "../../../metrics/index.js";
+import {BuilderBlockSelectionReason, EngineBlockSelectionReason} from "./index.js";
 
 export function computeSubnetForCommitteesAtSlot(
   slot: Slot,
@@ -44,31 +46,70 @@ export function getPubkeysForIndices(
   return pubkeys;
 }
 
-export function selectBlockProductionSource({
-  builderSelection,
-  engineBlockValue,
-  builderBlockValue,
-  builderBoostFactor,
-}: {
-  builderSelection: routes.validator.BuilderSelection;
-  engineBlockValue: bigint;
-  builderBlockValue: bigint;
-  builderBoostFactor: bigint;
-}): ProducedBlockSource {
+export function selectBlockProductionSource(
+  {
+    builderSelection,
+    engineBlockValue,
+    builderBlockValue,
+    builderBoostFactor,
+  }: {
+    builderSelection: routes.validator.BuilderSelection;
+    engineBlockValue: bigint;
+    builderBlockValue: bigint;
+    builderBoostFactor: bigint;
+  },
+  metrics: Metrics | null
+): ProducedBlockSource {
   switch (builderSelection) {
     case routes.validator.BuilderSelection.ExecutionAlways:
-    case routes.validator.BuilderSelection.ExecutionOnly:
+    case routes.validator.BuilderSelection.ExecutionOnly: {
+      metrics?.blockProductionSelectionResults.inc({
+        source: ProducedBlockSource.engine,
+        reason: EngineBlockSelectionReason.EnginePreferred,
+      });
       return ProducedBlockSource.engine;
+    }
 
     case routes.validator.BuilderSelection.Default:
-    case routes.validator.BuilderSelection.MaxProfit:
-      return builderBoostFactor !== MAX_BUILDER_BOOST_FACTOR &&
-        (builderBoostFactor === BigInt(0) || engineBlockValue >= (builderBlockValue * builderBoostFactor) / BigInt(100))
-        ? ProducedBlockSource.engine
-        : ProducedBlockSource.builder;
+    case routes.validator.BuilderSelection.MaxProfit: {
+      if (builderBoostFactor === MAX_BUILDER_BOOST_FACTOR) {
+        metrics?.blockProductionSelectionResults.inc({
+          source: ProducedBlockSource.builder,
+          reason: BuilderBlockSelectionReason.BuilderPreferred,
+        });
+        return ProducedBlockSource.builder;
+      }
+
+      if (builderBoostFactor === BigInt(0)) {
+        metrics?.blockProductionSelectionResults.inc({
+          source: ProducedBlockSource.engine,
+          reason: EngineBlockSelectionReason.EnginePreferred,
+        });
+        return ProducedBlockSource.engine;
+      }
+
+      if (engineBlockValue >= (builderBlockValue * builderBoostFactor) / BigInt(100)) {
+        metrics?.blockProductionSelectionResults.inc({
+          source: ProducedBlockSource.engine,
+          reason: EngineBlockSelectionReason.BlockValue,
+        });
+        return ProducedBlockSource.engine;
+      }
+
+      metrics?.blockProductionSelectionResults.inc({
+        source: ProducedBlockSource.builder,
+        reason: BuilderBlockSelectionReason.BlockValue,
+      });
+      return ProducedBlockSource.builder;
+    }
 
     case routes.validator.BuilderSelection.BuilderAlways:
-    case routes.validator.BuilderSelection.BuilderOnly:
+    case routes.validator.BuilderSelection.BuilderOnly: {
+      metrics?.blockProductionSelectionResults.inc({
+        source: ProducedBlockSource.builder,
+        reason: BuilderBlockSelectionReason.BuilderPreferred,
+      });
       return ProducedBlockSource.builder;
+    }
   }
 }

packages/beacon-node/src/metrics/metrics/beacon.ts

@@ -3,6 +3,7 @@ import {NotReorgedReason} from "@lodestar/fork-choice/lib/forkChoice/interface.j
 import {UpdateHeadOpt} from "@lodestar/fork-choice";
 import {RegistryMetricCreator} from "../utils/registryMetricCreator.js";
 import {BlockProductionStep, PayloadPreparationType} from "../../chain/produceBlock/index.js";
+import {BuilderBlockSelectionReason, EngineBlockSelectionReason} from "../../api/impl/validator/index.js";
 
 export type BeaconMetrics = ReturnType<typeof createBeaconMetrics>;
 
@@ -160,12 +161,26 @@ export function createBeaconMetrics(register: RegistryMetricCreator) {
       help: "Count of blocks successfully produced",
       labelNames: ["source"],
     }),
+    blockProductionSelectionResults: register.gauge<
+      | {source: ProducedBlockSource.engine; reason: EngineBlockSelectionReason}
+      | {source: ProducedBlockSource.builder; reason: BuilderBlockSelectionReason}
+    >({
+      name: "beacon_block_production_selection_results_total",
+      help: "Count of all block production selection results",
+      labelNames: ["source", "reason"],
+    }),
     blockProductionNumAggregated: register.histogram<{source: ProducedBlockSource}>({
       name: "beacon_block_production_num_aggregated_total",
       help: "Count of all aggregated attestations in our produced block",
       buckets: [32, 64, 96, 128],
       labelNames: ["source"],
     }),
+    blockProductionExecutionPayloadValue: register.histogram<{source: ProducedBlockSource}>({
+      name: "beacon_block_production_execution_payload_value",
+      help: "Execution payload value denominated in ETH of produced blocks",
+      buckets: [0.001, 0.005, 0.01, 0.03, 0.05, 0.07, 0.1, 0.3, 0.5, 1],
+      labelNames: ["source"],
+    }),
 
     blockProductionCaches: {
       producedBlockRoot: register.gauge({

packages/utils/src/format.ts

@@ -44,11 +44,18 @@ export function formatBigDecimal(numerator: bigint, denominator: bigint, maxDeci
 // display upto 5 decimal places
 const MAX_DECIMAL_FACTOR = BigInt("100000");
 
+/**
+ * Format wei as ETH, with up to 5 decimals
+ */
+export function formatWeiToEth(wei: bigint): string {
+  return formatBigDecimal(wei, ETH_TO_WEI, MAX_DECIMAL_FACTOR);
+}
+
 /**
  * Format wei as ETH, with up to 5 decimals and append ' ETH'
  */
 export function prettyWeiToEth(wei: bigint): string {
-  return `${formatBigDecimal(wei, ETH_TO_WEI, MAX_DECIMAL_FACTOR)} ETH`;
+  return `${formatWeiToEth(wei)} ETH`;
 }
 
 /**