diff --git a/packages/api/src/utils/client/metrics.ts b/packages/api/src/utils/client/metrics.ts index c8bc3c0637a4..65089e92e7ec 100644 --- a/packages/api/src/utils/client/metrics.ts +++ b/packages/api/src/utils/client/metrics.ts @@ -1,49 +1,9 @@ +import {Gauge, GaugeExtra, Histogram} from "@lodestar/utils"; + export type Metrics = { - requestTime: Histogram<"routeId">; - streamTime: Histogram<"routeId">; - requestErrors: Gauge<"routeId">; - requestToFallbacks: Gauge<"routeId">; - urlsScore: Gauge<"urlIndex">; + requestTime: Histogram<{routeId: string}>; + streamTime: Histogram<{routeId: string}>; + requestErrors: Gauge<{routeId: string}>; + requestToFallbacks: Gauge<{routeId: string}>; + urlsScore: GaugeExtra<{urlIndex: number}>; }; - -type LabelValues = Partial>; -type CollectFn = (metric: Gauge) => void; - -export interface Gauge { - /** - * Increment gauge for given labels - * @param labels Object with label keys and values - * @param value The value to increment with - */ - inc(labels: LabelValues, value?: number): void; - - /** - * Increment gauge - * @param value The value to increment with - */ - inc(value?: number): void; - - /** - * Set gauge value for labels - * @param labels Object with label keys and values - * @param value The value to set - */ - set(labels: LabelValues, value: number): void; - - /** - * Set gauge value - * @param value The value to set - */ - set(value: number): void; - - addCollect(collectFn: CollectFn): void; -} - -export interface Histogram { - /** - * Start a timer where the value in seconds will observed - * @param labels Object with label keys and values - * @return Function to invoke when timer should be stopped - */ - startTimer(labels?: LabelValues): (labels?: LabelValues) => number; -} diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index 71b2f0d5bc00..396dac8c54eb 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -145,7 +145,7 @@ "jwt-simple": "0.5.6", "libp2p": "0.46.12", "multiformats": "^11.0.1", - "prom-client": "^14.2.0", + "prom-client": "^15.1.0", "qs": "^6.11.1", "snappyjs": "^0.7.0", "strict-event-emitter-types": "^2.0.0", diff --git a/packages/beacon-node/src/api/rest/activeSockets.ts b/packages/beacon-node/src/api/rest/activeSockets.ts index ba8a35c80119..9f1b0f1a78a3 100644 --- a/packages/beacon-node/src/api/rest/activeSockets.ts +++ b/packages/beacon-node/src/api/rest/activeSockets.ts @@ -1,12 +1,11 @@ import http, {Server} from "node:http"; import {Socket} from "node:net"; -import {waitFor} from "@lodestar/utils"; -import {IGauge} from "../../metrics/index.js"; +import {Gauge, GaugeExtra, waitFor} from "@lodestar/utils"; export type SocketMetrics = { - activeSockets: IGauge; - socketsBytesRead: IGauge; - socketsBytesWritten: IGauge; + activeSockets: GaugeExtra; + socketsBytesRead: Gauge; + socketsBytesWritten: Gauge; }; // Use relatively short timeout to speed up shutdown diff --git a/packages/beacon-node/src/api/rest/base.ts b/packages/beacon-node/src/api/rest/base.ts index 24049677b5de..3ddb5354a897 100644 --- a/packages/beacon-node/src/api/rest/base.ts +++ b/packages/beacon-node/src/api/rest/base.ts @@ -3,9 +3,8 @@ import fastify, {FastifyInstance} from "fastify"; import fastifyCors from "@fastify/cors"; import bearerAuthPlugin from "@fastify/bearer-auth"; import {RouteConfig} from "@lodestar/api/beacon/server"; -import {ErrorAborted, Logger} from "@lodestar/utils"; +import {ErrorAborted, Gauge, Histogram, Logger} from "@lodestar/utils"; import {isLocalhostIP} from "../../util/ip.js"; -import {IGauge, IHistogram} from "../../metrics/index.js"; import {ApiError, NodeIsSyncing} from "../impl/errors.js"; import {HttpActiveSocketsTracker, SocketMetrics} from "./activeSockets.js"; @@ -25,9 +24,9 @@ export type RestApiServerModules = { }; export type RestApiServerMetrics = SocketMetrics & { - requests: IGauge<"operationId">; - responseTime: IHistogram<"operationId">; - errors: IGauge<"operationId">; + requests: Gauge<{operationId: string}>; + responseTime: Histogram<{operationId: string}>; + errors: Gauge<{operationId: string}>; }; /** diff --git a/packages/beacon-node/src/chain/blocks/verifyBlocksStateTransitionOnly.ts b/packages/beacon-node/src/chain/blocks/verifyBlocksStateTransitionOnly.ts index 786d426d70e9..7d15d4e4f6ce 100644 --- a/packages/beacon-node/src/chain/blocks/verifyBlocksStateTransitionOnly.ts +++ b/packages/beacon-node/src/chain/blocks/verifyBlocksStateTransitionOnly.ts @@ -3,6 +3,7 @@ import { stateTransition, ExecutionPayloadStatus, DataAvailableStatus, + StateHashTreeRootSource, } from "@lodestar/state-transition"; import {ErrorAborted, Logger, sleep} from "@lodestar/utils"; import {Metrics} from "../../metrics/index.js"; @@ -57,7 +58,9 @@ export async function verifyBlocksStateTransitionOnly( metrics ); - const hashTreeRootTimer = metrics?.stateHashTreeRootTime.startTimer({source: "block_transition"}); + const hashTreeRootTimer = metrics?.stateHashTreeRootTime.startTimer({ + source: StateHashTreeRootSource.blockTransition, + }); const stateRoot = postState.hashTreeRoot(); hashTreeRootTimer?.(); diff --git a/packages/beacon-node/src/chain/bls/index.ts b/packages/beacon-node/src/chain/bls/index.ts index 3ee72ac66cbd..f9898b13776b 100644 --- a/packages/beacon-node/src/chain/bls/index.ts +++ b/packages/beacon-node/src/chain/bls/index.ts @@ -1,4 +1,4 @@ export type {IBlsVerifier} from "./interface.js"; -export type {BlsMultiThreadWorkerPoolModules} from "./multithread/index.js"; +export type {BlsMultiThreadWorkerPoolModules, JobQueueItemType} from "./multithread/index.js"; export {BlsMultiThreadWorkerPool} from "./multithread/index.js"; export {BlsSingleThreadVerifier} from "./singleThread.js"; diff --git a/packages/beacon-node/src/chain/bls/multithread/index.ts b/packages/beacon-node/src/chain/bls/multithread/index.ts index 9b0006566253..235ec1536be7 100644 --- a/packages/beacon-node/src/chain/bls/multithread/index.ts +++ b/packages/beacon-node/src/chain/bls/multithread/index.ts @@ -41,6 +41,8 @@ export type BlsMultiThreadWorkerPoolOptions = { blsVerifyAllMultiThread?: boolean; }; +export type {JobQueueItemType}; + // 1 worker for the main thread const blsPoolSize = Math.max(defaultPoolSize - 1, 1); diff --git a/packages/beacon-node/src/chain/opPools/opPool.ts b/packages/beacon-node/src/chain/opPools/opPool.ts index cee8d0614c30..bb436319cd53 100644 --- a/packages/beacon-node/src/chain/opPools/opPool.ts +++ b/packages/beacon-node/src/chain/opPools/opPool.ts @@ -19,6 +19,7 @@ import {IBeaconDb} from "../../db/index.js"; import {SignedBLSToExecutionChangeVersioned} from "../../util/types.js"; import {BlockType} from "../interface.js"; import {Metrics} from "../../metrics/metrics.js"; +import {BlockProductionStep} from "../produceBlock/produceBlockBody.js"; import {isValidBlsToExecutionChangeForBlockInclusion} from "./utils.js"; type HexRoot = string; @@ -201,7 +202,7 @@ export class OpPool { } } endProposerSlashing?.({ - step: "proposerSlashing", + step: BlockProductionStep.proposerSlashing, }); const endAttesterSlashings = stepsMetrics?.startTimer(); @@ -235,7 +236,7 @@ export class OpPool { } } endAttesterSlashings?.({ - step: "attesterSlashings", + step: BlockProductionStep.attesterSlashings, }); const endVoluntaryExits = stepsMetrics?.startTimer(); @@ -256,7 +257,7 @@ export class OpPool { } } endVoluntaryExits?.({ - step: "voluntaryExits", + step: BlockProductionStep.voluntaryExits, }); const endBlsToExecutionChanges = stepsMetrics?.startTimer(); @@ -270,7 +271,7 @@ export class OpPool { } } endBlsToExecutionChanges?.({ - step: "blsToExecutionChanges", + step: BlockProductionStep.blsToExecutionChanges, }); return [attesterSlashings, proposerSlashings, voluntaryExits, blsToExecutionChanges]; diff --git a/packages/beacon-node/src/chain/prepareNextSlot.ts b/packages/beacon-node/src/chain/prepareNextSlot.ts index ce8e720cd766..e2bffd5bc8c6 100644 --- a/packages/beacon-node/src/chain/prepareNextSlot.ts +++ b/packages/beacon-node/src/chain/prepareNextSlot.ts @@ -1,4 +1,9 @@ -import {computeEpochAtSlot, isExecutionStateType, computeTimeAtSlot} from "@lodestar/state-transition"; +import { + computeEpochAtSlot, + isExecutionStateType, + computeTimeAtSlot, + StateHashTreeRootSource, +} from "@lodestar/state-transition"; import {ChainForkConfig} from "@lodestar/config"; import {ForkSeq, SLOTS_PER_EPOCH, ForkExecution} from "@lodestar/params"; import {Slot} from "@lodestar/types"; @@ -106,7 +111,9 @@ export class PrepareNextSlotScheduler { // cache HashObjects for faster hashTreeRoot() later, especially for computeNewStateRoot() if we need to produce a block at slot 0 of epoch // see https://github.com/ChainSafe/lodestar/issues/6194 - const hashTreeRootTimer = this.metrics?.stateHashTreeRootTime.startTimer({source: "prepare_next_slot"}); + const hashTreeRootTimer = this.metrics?.stateHashTreeRootTime.startTimer({ + source: StateHashTreeRootSource.prepareNextSlot, + }); prepareState.hashTreeRoot(); hashTreeRootTimer?.(); diff --git a/packages/beacon-node/src/chain/produceBlock/computeNewStateRoot.ts b/packages/beacon-node/src/chain/produceBlock/computeNewStateRoot.ts index f5d02dbf9b6f..ccc0595d0db6 100644 --- a/packages/beacon-node/src/chain/produceBlock/computeNewStateRoot.ts +++ b/packages/beacon-node/src/chain/produceBlock/computeNewStateRoot.ts @@ -2,6 +2,7 @@ import { CachedBeaconStateAllForks, DataAvailableStatus, ExecutionPayloadStatus, + StateHashTreeRootSource, stateTransition, } from "@lodestar/state-transition"; import {allForks, Gwei, Root} from "@lodestar/types"; @@ -44,7 +45,9 @@ export function computeNewStateRoot( const {attestations, syncAggregate, slashing} = postState.proposerRewards; const proposerReward = BigInt(attestations + syncAggregate + slashing); - const hashTreeRootTimer = metrics?.stateHashTreeRootTime.startTimer({source: "compute_new_state_root"}); + const hashTreeRootTimer = metrics?.stateHashTreeRootTime.startTimer({ + source: StateHashTreeRootSource.computeNewStateRoot, + }); const newStateRoot = postState.hashTreeRoot(); hashTreeRootTimer?.(); diff --git a/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts b/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts index 97f45c0ee289..3c2bec223eca 100644 --- a/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts +++ b/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts @@ -39,10 +39,26 @@ import {validateBlobsAndKzgCommitments} from "./validateBlobsAndKzgCommitments.j // Time to provide the EL to generate a payload from new payload id const PAYLOAD_GENERATION_TIME_MS = 500; -enum PayloadPreparationType { + +export enum PayloadPreparationType { Fresh = "Fresh", Cached = "Cached", Reorged = "Reorged", + Blinded = "Blinded", +} + +/** + * Block production steps tracked in metrics + */ +export enum BlockProductionStep { + proposerSlashing = "proposerSlashing", + attesterSlashings = "attesterSlashings", + voluntaryExits = "voluntaryExits", + blsToExecutionChanges = "blsToExecutionChanges", + attestations = "attestations", + eth1DataAndDeposits = "eth1DataAndDeposits", + syncAggregate = "syncAggregate", + executionPayload = "executionPayload", } export type BlockAttributes = { @@ -131,13 +147,13 @@ export async function produceBlockBody( const endAttestations = stepsMetrics?.startTimer(); const attestations = this.aggregatedAttestationPool.getAttestationsForBlock(this.forkChoice, currentState); endAttestations?.({ - step: "attestations", + step: BlockProductionStep.attestations, }); const endEth1DataAndDeposits = stepsMetrics?.startTimer(); const {eth1Data, deposits} = await this.eth1.getEth1DataAndDeposits(currentState); endEth1DataAndDeposits?.({ - step: "eth1DataAndDeposits", + step: BlockProductionStep.eth1DataAndDeposits, }); const blockBody: phase0.BeaconBlockBody = { @@ -162,7 +178,7 @@ export async function produceBlockBody( (blockBody as altair.BeaconBlockBody).syncAggregate = syncAggregate; } endSyncAggregate?.({ - step: "syncAggregate", + step: BlockProductionStep.syncAggregate, }); Object.assign(logMeta, { @@ -218,7 +234,7 @@ export async function produceBlockBody( executionPayloadValue = builderRes.executionPayloadValue; const fetchedTime = Date.now() / 1000 - computeTimeAtSlot(this.config, blockSlot, this.genesisTime); - const prepType = "blinded"; + const prepType = PayloadPreparationType.Blinded; this.metrics?.blockPayload.payloadFetchedTime.observe({prepType}, fetchedTime); this.logger.verbose("Fetched execution payload header from builder", { slot: blockSlot, @@ -343,7 +359,7 @@ export async function produceBlockBody( executionPayloadValue = BigInt(0); } endExecutionPayload?.({ - step: "executionPayload", + step: BlockProductionStep.executionPayload, }); if (ForkSeq[fork] >= ForkSeq.capella) { diff --git a/packages/beacon-node/src/chain/regen/queued.ts b/packages/beacon-node/src/chain/regen/queued.ts index 5305502c8c05..dfda56cc1eea 100644 --- a/packages/beacon-node/src/chain/regen/queued.ts +++ b/packages/beacon-node/src/chain/regen/queued.ts @@ -221,7 +221,7 @@ export class QueuedStateRegenerator implements IStateRegenerator { private jobQueueProcessor = async (regenRequest: RegenRequest): Promise => { const metricsLabels = { caller: regenRequest.args[regenRequest.args.length - 1] as RegenCaller, - entrypoint: regenRequest.key, + entrypoint: regenRequest.key as RegenFnName, }; let timer; try { diff --git a/packages/beacon-node/src/chain/reprocess.ts b/packages/beacon-node/src/chain/reprocess.ts index 3ab6056fb3af..4c91ef07ff69 100644 --- a/packages/beacon-node/src/chain/reprocess.ts +++ b/packages/beacon-node/src/chain/reprocess.ts @@ -11,7 +11,7 @@ export const REPROCESS_MIN_TIME_TO_NEXT_SLOT_SEC = 2; /** * Reprocess status for metrics */ -enum ReprocessStatus { +export enum ReprocessStatus { /** * There are too many attestations that have unknown block root. */ @@ -140,7 +140,10 @@ export class ReprocessController { for (const awaitingPromise of awaitingPromisesByRoot.values()) { const {resolve, addedTimeMs} = awaitingPromise; resolve(false); - this.metrics?.reprocessApiAttestations.waitSecBeforeReject.set((now - addedTimeMs) / 1000); + this.metrics?.reprocessApiAttestations.waitSecBeforeReject.set( + {reason: ReprocessStatus.expired}, + (now - addedTimeMs) / 1000 + ); this.metrics?.reprocessApiAttestations.reject.inc({reason: ReprocessStatus.expired}); } diff --git a/packages/beacon-node/src/chain/seenCache/seenAttestationData.ts b/packages/beacon-node/src/chain/seenCache/seenAttestationData.ts index ded54a5b4a54..a19476497e9f 100644 --- a/packages/beacon-node/src/chain/seenCache/seenAttestationData.ts +++ b/packages/beacon-node/src/chain/seenCache/seenAttestationData.ts @@ -17,7 +17,7 @@ export type AttestationDataCacheEntry = { subnet: number; }; -enum RejectReason { +export enum RejectReason { // attestation data reaches MAX_CACHE_SIZE_PER_SLOT reached_limit = "reached_limit", // attestation data is too old diff --git a/packages/beacon-node/src/chain/stateCache/mapMetrics.ts b/packages/beacon-node/src/chain/stateCache/mapMetrics.ts index eb52755bfc00..bb33323015d4 100644 --- a/packages/beacon-node/src/chain/stateCache/mapMetrics.ts +++ b/packages/beacon-node/src/chain/stateCache/mapMetrics.ts @@ -1,8 +1,8 @@ -import {IAvgMinMax} from "../../metrics/index.js"; +import {AvgMinMax} from "@lodestar/utils"; type MapTrackerMetrics = { - reads: IAvgMinMax; - secondsSinceLastRead: IAvgMinMax; + reads: AvgMinMax; + secondsSinceLastRead: AvgMinMax; }; export class MapTracker extends Map { diff --git a/packages/beacon-node/src/chain/validation/attestation.ts b/packages/beacon-node/src/chain/validation/attestation.ts index 31e105911ab4..eae171631025 100644 --- a/packages/beacon-node/src/chain/validation/attestation.ts +++ b/packages/beacon-node/src/chain/validation/attestation.ts @@ -541,7 +541,7 @@ export function verifyHeadBlockAndTargetRoot( targetRoot: Root, attestationSlot: Slot, attestationEpoch: Epoch, - caller: string, + caller: RegenCaller, maxSkipSlots?: number ): ProtoBlock { const headBlock = verifyHeadBlockIsKnown(chain, beaconBlockRoot); diff --git a/packages/beacon-node/src/eth1/provider/jsonRpcHttpClient.ts b/packages/beacon-node/src/eth1/provider/jsonRpcHttpClient.ts index 3a1b4ddb0ce1..faa4e310e10a 100644 --- a/packages/beacon-node/src/eth1/provider/jsonRpcHttpClient.ts +++ b/packages/beacon-node/src/eth1/provider/jsonRpcHttpClient.ts @@ -1,8 +1,7 @@ import {EventEmitter} from "events"; import StrictEventEmitter from "strict-event-emitter-types"; import {fetch} from "@lodestar/api"; -import {ErrorAborted, TimeoutError, isValidHttpUrl, retry} from "@lodestar/utils"; -import {IGauge, IHistogram} from "../../metrics/interface.js"; +import {ErrorAborted, Gauge, Histogram, TimeoutError, isValidHttpUrl, retry} from "@lodestar/utils"; import {IJson, RpcPayload} from "../interface.js"; import {JwtClaim, encodeJwtToken} from "./jwt.js"; @@ -58,13 +57,13 @@ export type ReqOpts = { }; export type JsonRpcHttpClientMetrics = { - requestTime: IHistogram<"routeId">; - streamTime: IHistogram<"routeId">; - requestErrors: IGauge<"routeId">; - requestUsedFallbackUrl: IGauge<"routeId">; - activeRequests: IGauge<"routeId">; - configUrlsCount: IGauge; - retryCount: IGauge<"routeId">; + requestTime: Histogram<{routeId: string}>; + streamTime: Histogram<{routeId: string}>; + requestErrors: Gauge<{routeId: string}>; + requestUsedFallbackUrl: Gauge<{routeId: string}>; + activeRequests: Gauge<{routeId: string}>; + configUrlsCount: Gauge; + retryCount: Gauge<{routeId: string}>; }; export interface IJsonRpcHttpClient { diff --git a/packages/beacon-node/src/metrics/index.ts b/packages/beacon-node/src/metrics/index.ts index fb2781333d66..a56591a04090 100644 --- a/packages/beacon-node/src/metrics/index.ts +++ b/packages/beacon-node/src/metrics/index.ts @@ -1,5 +1,4 @@ export * from "./metrics.js"; export * from "./server/index.js"; -export * from "./interface.js"; export * from "./nodeJsMetrics.js"; export {RegistryMetricCreator} from "./utils/registryMetricCreator.js"; diff --git a/packages/beacon-node/src/metrics/interface.ts b/packages/beacon-node/src/metrics/interface.ts deleted file mode 100644 index 2e2a267ca13c..000000000000 --- a/packages/beacon-node/src/metrics/interface.ts +++ /dev/null @@ -1,14 +0,0 @@ -import {Gauge, Histogram} from "prom-client"; - -type CollectFn = (metric: IGauge) => void; - -export type IGauge = Pick, "inc" | "dec" | "set"> & { - addCollect: (collectFn: CollectFn) => void; -}; - -export type IHistogram = Pick, "observe" | "startTimer">; - -export type IAvgMinMax = { - addGetValuesFn(getValuesFn: () => number[]): void; - set(values: number[]): void; -}; diff --git a/packages/beacon-node/src/metrics/metrics/beacon.ts b/packages/beacon-node/src/metrics/metrics/beacon.ts index e2eed75adb70..9366174ef6c6 100644 --- a/packages/beacon-node/src/metrics/metrics/beacon.ts +++ b/packages/beacon-node/src/metrics/metrics/beacon.ts @@ -1,4 +1,6 @@ +import {ProducedBlockSource} from "@lodestar/types"; import {RegistryMetricCreator} from "../utils/registryMetricCreator.js"; +import {BlockProductionStep, PayloadPreparationType} from "../../chain/produceBlock/index.js"; export type BeaconMetrics = ReturnType; @@ -46,7 +48,7 @@ export function createBeaconMetrics(register: RegistryMetricCreator) { // Additional Metrics // TODO: Implement - currentValidators: register.gauge<"status">({ + currentValidators: register.gauge<{status: string}>({ name: "beacon_current_validators", labelNames: ["status"], help: "number of validators in current epoch", @@ -115,55 +117,35 @@ export function createBeaconMetrics(register: RegistryMetricCreator) { buckets: [1, 2, 3, 5, 7, 10, 20, 30, 50, 100], }), - blockProductionTime: register.histogram<"source">({ + blockProductionTime: register.histogram<{source: ProducedBlockSource}>({ name: "beacon_block_production_seconds", help: "Full runtime of block production", buckets: [0.1, 1, 2, 4, 10], labelNames: ["source"], }), - executionBlockProductionTimeSteps: register.histogram<"step">({ + executionBlockProductionTimeSteps: register.histogram<{step: BlockProductionStep}>({ name: "beacon_block_production_execution_steps_seconds", help: "Detailed steps runtime of execution block production", buckets: [0.01, 0.1, 0.2, 0.5, 1], - /** - * - proposerSlashing - * - attesterSlashings - * - voluntaryExits - * - blsToExecutionChanges - * - attestations - * - eth1DataAndDeposits - * - syncAggregate - * - executionPayload - */ labelNames: ["step"], }), - builderBlockProductionTimeSteps: register.histogram<"step">({ + builderBlockProductionTimeSteps: register.histogram<{step: BlockProductionStep}>({ name: "beacon_block_production_builder_steps_seconds", help: "Detailed steps runtime of builder block production", buckets: [0.01, 0.1, 0.2, 0.5, 1], - /** - * - proposerSlashing - * - attesterSlashings - * - voluntaryExits - * - blsToExecutionChanges - * - attestations - * - eth1DataAndDeposits - * - syncAggregate - * - executionPayload - */ labelNames: ["step"], }), - blockProductionRequests: register.gauge<"source">({ + blockProductionRequests: register.gauge<{source: ProducedBlockSource}>({ name: "beacon_block_production_requests_total", help: "Count of all block production requests", labelNames: ["source"], }), - blockProductionSuccess: register.gauge<"source">({ + blockProductionSuccess: register.gauge<{source: ProducedBlockSource}>({ name: "beacon_block_production_successes_total", help: "Count of blocks successfully produced", labelNames: ["source"], }), - blockProductionNumAggregated: register.histogram<"source">({ + blockProductionNumAggregated: register.histogram<{source: ProducedBlockSource}>({ name: "beacon_block_production_num_aggregated_total", help: "Count of all aggregated attestations in our produced block", buckets: [32, 64, 96, 128], @@ -173,11 +155,11 @@ export function createBeaconMetrics(register: RegistryMetricCreator) { blockProductionCaches: { producedBlockRoot: register.gauge({ name: "beacon_blockroot_produced_cache_total", - help: "Count of cached produded block roots", + help: "Count of cached produced block roots", }), producedBlindedBlockRoot: register.gauge({ name: "beacon_blinded_blockroot_produced_cache_total", - help: "Count of cached produded blinded block roots", + help: "Count of cached produced blinded block roots", }), producedContentsCache: register.gauge({ name: "beacon_contents_produced_cache_total", @@ -188,15 +170,15 @@ export function createBeaconMetrics(register: RegistryMetricCreator) { blockPayload: { payloadAdvancePrepTime: register.histogram({ name: "beacon_block_payload_prepare_time", - help: "Time for perparing payload in advance", + help: "Time for preparing payload in advance", buckets: [0.1, 1, 3, 5, 10], }), - payloadFetchedTime: register.histogram<"prepType">({ + payloadFetchedTime: register.histogram<{prepType: PayloadPreparationType}>({ name: "beacon_block_payload_fetched_time", help: "Time to fetch the payload from EL", labelNames: ["prepType"], }), - emptyPayloads: register.gauge<"prepType">({ + emptyPayloads: register.gauge<{prepType: PayloadPreparationType}>({ name: "beacon_block_payload_empty_total", help: "Count of payload with empty transactions", labelNames: ["prepType"], diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index 31c4156e5430..6c000685556c 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -1,6 +1,21 @@ +import {EpochTransitionStep, StateCloneSource, StateHashTreeRootSource} from "@lodestar/state-transition"; import {allForks} from "@lodestar/types"; -import {RegistryMetricCreator} from "../utils/registryMetricCreator.js"; +import {BlockSource} from "../../chain/blocks/types.js"; +import {JobQueueItemType} from "../../chain/bls/index.js"; +import {BlockErrorCode} from "../../chain/errors/index.js"; +import {InsertOutcome} from "../../chain/opPools/types.js"; +import {RegenCaller, RegenFnName} from "../../chain/regen/interface.js"; +import {ReprocessStatus} from "../../chain/reprocess.js"; +import {RejectReason} from "../../chain/seenCache/seenAttestationData.js"; +import {ExecutionPayloadStatus} from "../../execution/index.js"; +import {GossipType} from "../../network/index.js"; +import {CannotAcceptWorkReason, ReprocessRejectReason} from "../../network/processor/index.js"; +import {BackfillSyncMethod} from "../../sync/backfill/backfill.js"; +import {PendingBlockType} from "../../sync/interface.js"; +import {PeerSyncType, RangeSyncType} from "../../sync/utils/remoteSyncType.js"; import {LodestarMetadata} from "../options.js"; +import {RegistryMetricCreator} from "../utils/registryMetricCreator.js"; +import {OpSource} from "../validatorMonitor.js"; export type LodestarMetrics = ReturnType; @@ -14,7 +29,7 @@ export function createLodestarMetrics( anchorState?: Pick ) { if (metadata) { - register.static({ + register.static({ name: "lodestar_version", help: "Lodestar version", value: metadata, @@ -33,34 +48,34 @@ export function createLodestarMetrics( return { gossipValidationQueue: { - length: register.gauge<"topic">({ + length: register.gauge<{topic: GossipType}>({ name: "lodestar_gossip_validation_queue_length", help: "Count of total gossip validation queue length", labelNames: ["topic"], }), - keySize: register.gauge<"topic">({ + keySize: register.gauge<{topic: GossipType}>({ name: "lodestar_gossip_validation_queue_key_size", help: "Count of total gossip validation queue key size", labelNames: ["topic"], }), - droppedJobs: register.gauge<"topic">({ + droppedJobs: register.gauge<{topic: GossipType}>({ name: "lodestar_gossip_validation_queue_dropped_jobs_total", help: "Count of total gossip validation queue dropped jobs", labelNames: ["topic"], }), - jobTime: register.histogram<"topic">({ + jobTime: register.histogram<{topic: GossipType}>({ name: "lodestar_gossip_validation_queue_job_time_seconds", help: "Time to process gossip validation queue job in seconds", labelNames: ["topic"], buckets: [0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2, 5, 10], }), - jobWaitTime: register.histogram<"topic">({ + jobWaitTime: register.histogram<{topic: GossipType}>({ name: "lodestar_gossip_validation_queue_job_wait_time_seconds", help: "Time from job added to the queue to starting the job in seconds", labelNames: ["topic"], buckets: [0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2, 5, 10], }), - concurrency: register.gauge<"topic">({ + concurrency: register.gauge<{topic: GossipType}>({ name: "lodestar_gossip_validation_queue_concurrency", help: "Current count of jobs being run on network processor for topic", labelNames: ["topic"], @@ -79,22 +94,22 @@ export function createLodestarMetrics( }, networkProcessor: { - gossipValidationAccept: register.gauge<"topic">({ + gossipValidationAccept: register.gauge<{topic: GossipType}>({ name: "lodestar_gossip_validation_accept_total", help: "Count of total gossip validation accept", labelNames: ["topic"], }), - gossipValidationIgnore: register.gauge<"topic">({ + gossipValidationIgnore: register.gauge<{topic: GossipType}>({ name: "lodestar_gossip_validation_ignore_total", help: "Count of total gossip validation ignore", labelNames: ["topic"], }), - gossipValidationReject: register.gauge<"topic">({ + gossipValidationReject: register.gauge<{topic: GossipType}>({ name: "lodestar_gossip_validation_reject_total", help: "Count of total gossip validation reject", labelNames: ["topic"], }), - gossipValidationError: register.gauge<"topic" | "error">({ + gossipValidationError: register.gauge<{topic: GossipType; error: string}>({ name: "lodestar_gossip_validation_error_total", help: "Count of total gossip validation errors detailed", labelNames: ["topic", "error"], @@ -108,7 +123,7 @@ export function createLodestarMetrics( help: "Total calls to network processor execute work fn", buckets: [0, 1, 5, 128], }), - canNotAcceptWork: register.gauge<"reason">({ + canNotAcceptWork: register.gauge<{reason: CannotAcceptWorkReason}>({ name: "lodestar_network_processor_can_not_accept_work_total", help: "Total times network processor can not accept work on executeWork", labelNames: ["reason"], @@ -121,7 +136,7 @@ export function createLodestarMetrics( help: "Current count of pending items in reqRespBridgeReqCaller data structure", }), }, - networkWorkerWireEventsOnMainThreadLatency: register.histogram<"eventName">({ + networkWorkerWireEventsOnMainThreadLatency: register.histogram<{eventName: string}>({ name: "lodestar_network_worker_wire_events_on_main_thread_latency_seconds", help: "Latency in seconds to transmit network events to main thread across worker port", labelNames: ["eventName"], @@ -206,19 +221,19 @@ export function createLodestarMetrics( }, apiRest: { - responseTime: register.histogram<"operationId">({ + responseTime: register.histogram<{operationId: string}>({ name: "lodestar_api_rest_response_time_seconds", help: "REST API time to fulfill a request by operationId", labelNames: ["operationId"], // Request times range between 1ms to 100ms in normal conditions. Can get to 1-5 seconds if overloaded buckets: [0.01, 0.1, 1], }), - requests: register.gauge<"operationId">({ + requests: register.gauge<{operationId: string}>({ name: "lodestar_api_rest_requests_total", help: "REST API total count requests by operationId", labelNames: ["operationId"], }), - errors: register.gauge<"operationId">({ + errors: register.gauge<{operationId: string}>({ name: "lodestar_api_rest_errors_total", help: "REST API total count of errors by operationId", labelNames: ["operationId"], @@ -286,7 +301,7 @@ export function createLodestarMetrics( help: "Time to call commit after process a single epoch transition in seconds", buckets: [0.01, 0.05, 0.1, 0.2, 0.5, 0.75, 1], }), - epochTransitionStepTime: register.histogram<"step">({ + epochTransitionStepTime: register.histogram<{step: EpochTransitionStep}>({ name: "lodestar_stfn_epoch_transition_step_seconds", help: "Time to call each step of epoch transition in seconds", labelNames: ["step"], @@ -304,28 +319,28 @@ export function createLodestarMetrics( help: "Time to call commit after process a single block in seconds", buckets: [0.005, 0.01, 0.02, 0.05, 0.1, 1], }), - stateHashTreeRootTime: register.histogram<"source">({ + stateHashTreeRootTime: register.histogram<{source: StateHashTreeRootSource}>({ name: "lodestar_stfn_hash_tree_root_seconds", help: "Time to compute the hash tree root of a post state in seconds", buckets: [0.05, 0.1, 0.2, 0.5, 1, 1.5], labelNames: ["source"], }), - preStateBalancesNodesPopulatedMiss: register.gauge<"source">({ + preStateBalancesNodesPopulatedMiss: register.gauge<{source: StateCloneSource}>({ name: "lodestar_stfn_balances_nodes_populated_miss_total", help: "Total count state.balances nodesPopulated is false on stfn", labelNames: ["source"], }), - preStateBalancesNodesPopulatedHit: register.gauge<"source">({ + preStateBalancesNodesPopulatedHit: register.gauge<{source: StateCloneSource}>({ name: "lodestar_stfn_balances_nodes_populated_hit_total", help: "Total count state.balances nodesPopulated is true on stfn", labelNames: ["source"], }), - preStateValidatorsNodesPopulatedMiss: register.gauge<"source">({ + preStateValidatorsNodesPopulatedMiss: register.gauge<{source: StateCloneSource}>({ name: "lodestar_stfn_validators_nodes_populated_miss_total", help: "Total count state.validators nodesPopulated is false on stfn", labelNames: ["source"], }), - preStateValidatorsNodesPopulatedHit: register.gauge<"source">({ + preStateValidatorsNodesPopulatedHit: register.gauge<{source: StateCloneSource}>({ name: "lodestar_stfn_validators_nodes_populated_hit_total", help: "Total count state.validators nodesPopulated is true on stfn", labelNames: ["source"], @@ -362,7 +377,7 @@ export function createLodestarMetrics( }, blsThreadPool: { - jobsWorkerTime: register.gauge<"workerId">({ + jobsWorkerTime: register.gauge<{workerId: number}>({ name: "lodestar_bls_thread_pool_time_seconds_sum", help: "Total time spent verifying signature sets measured on the worker", labelNames: ["workerId"], @@ -371,7 +386,7 @@ export function createLodestarMetrics( name: "lodestar_bls_thread_pool_success_jobs_signature_sets_count", help: "Count of total verified signature sets", }), - errorAggregateSignatureSetsCount: register.gauge<"type">({ + errorAggregateSignatureSetsCount: register.gauge<{type: JobQueueItemType}>({ name: "lodestar_bls_thread_pool_error_aggregate_signature_sets_count", help: "Count of error when aggregating pubkeys or signatures", labelNames: ["type"], @@ -397,12 +412,12 @@ export function createLodestarMetrics( name: "lodestar_bls_thread_pool_job_groups_started_total", help: "Count of total jobs groups started in bls thread pool, job groups include +1 jobs", }), - totalJobsStarted: register.gauge<"type">({ + totalJobsStarted: register.gauge<{type: JobQueueItemType}>({ name: "lodestar_bls_thread_pool_jobs_started_total", help: "Count of total jobs started in bls thread pool, jobs include +1 signature sets", labelNames: ["type"], }), - totalSigSetsStarted: register.gauge<"type">({ + totalSigSetsStarted: register.gauge<{type: JobQueueItemType}>({ name: "lodestar_bls_thread_pool_sig_sets_started_total", help: "Count of total signature sets started in bls thread pool, sig sets include 1 pk, msg, sig", labelNames: ["type"], @@ -493,29 +508,29 @@ export function createLodestarMetrics( name: "lodestar_sync_status", help: "Range sync status: [Stalled, SyncingFinalized, SyncingHead, Synced]", }), - syncPeersBySyncType: register.gauge<"syncType">({ + syncPeersBySyncType: register.gauge<{syncType: PeerSyncType}>({ name: "lodestar_sync_range_sync_peers", help: "Count of peers by sync type [FullySynced, Advanced, Behind]", labelNames: ["syncType"], }), - syncSwitchGossipSubscriptions: register.gauge<"action">({ + syncSwitchGossipSubscriptions: register.gauge<{action: string}>({ name: "lodestar_sync_switch_gossip_subscriptions", help: "Sync switched gossip subscriptions on/off", labelNames: ["action"], }), syncRange: { - syncChainsEvents: register.gauge<"syncType" | "event">({ + syncChainsEvents: register.gauge<{syncType: RangeSyncType; event: string}>({ name: "lodestar_sync_chains_events_total", help: "Total number of sync chains events events, labeled by syncType", labelNames: ["syncType", "event"], }), - syncChains: register.gauge<"syncType">({ + syncChains: register.gauge<{syncType: RangeSyncType}>({ name: "lodestar_sync_chains_count", help: "Count of sync chains by syncType", labelNames: ["syncType"], }), - syncChainsPeers: register.histogram<"syncType">({ + syncChainsPeers: register.histogram<{syncType: RangeSyncType}>({ name: "lodestar_sync_chains_peer_count_by_type", help: "Count of sync chain peers by syncType", labelNames: ["syncType"], @@ -528,12 +543,12 @@ export function createLodestarMetrics( }, syncUnknownBlock: { - switchNetworkSubscriptions: register.gauge<"action">({ + switchNetworkSubscriptions: register.gauge<{action: string}>({ name: "lodestar_sync_unknown_block_network_subscriptions_count", help: "Switch network subscriptions on/off", labelNames: ["action"], }), - requests: register.gauge<"type">({ + requests: register.gauge<{type: PendingBlockType}>({ name: "lodestar_sync_unknown_block_requests_total", help: "Total number of unknown block events or requests", labelNames: ["type"], @@ -587,43 +602,43 @@ export function createLodestarMetrics( // Gossip attestation gossipAttestation: { - useHeadBlockState: register.gauge<"caller">({ + useHeadBlockState: register.gauge<{caller: RegenCaller}>({ name: "lodestar_gossip_attestation_use_head_block_state_count", help: "Count of gossip attestation verification using head block state", labelNames: ["caller"], }), - useHeadBlockStateDialedToTargetEpoch: register.gauge<"caller">({ + useHeadBlockStateDialedToTargetEpoch: register.gauge<{caller: RegenCaller}>({ name: "lodestar_gossip_attestation_use_head_block_state_dialed_to_target_epoch_count", help: "Count of gossip attestation verification using head block state and dialed to target epoch", labelNames: ["caller"], }), - headSlotToAttestationSlot: register.histogram<"caller">({ + headSlotToAttestationSlot: register.histogram<{caller: RegenCaller}>({ name: "lodestar_gossip_attestation_head_slot_to_attestation_slot", help: "Slot distance between attestation slot and head slot", labelNames: ["caller"], buckets: [0, 1, 2, 4, 8, 16, 32, 64], }), - shufflingCacheHit: register.gauge<"caller">({ + shufflingCacheHit: register.gauge<{caller: RegenCaller}>({ name: "lodestar_gossip_attestation_shuffling_cache_hit_count", help: "Count of gossip attestation verification shuffling cache hit", labelNames: ["caller"], }), - shufflingCacheMiss: register.gauge<"caller">({ + shufflingCacheMiss: register.gauge<{caller: RegenCaller}>({ name: "lodestar_gossip_attestation_shuffling_cache_miss_count", help: "Count of gossip attestation verification shuffling cache miss", labelNames: ["caller"], }), - shufflingCacheRegenHit: register.gauge<"caller">({ + shufflingCacheRegenHit: register.gauge<{caller: RegenCaller}>({ name: "lodestar_gossip_attestation_shuffling_cache_regen_hit_count", help: "Count of gossip attestation verification shuffling cache regen hit", labelNames: ["caller"], }), - shufflingCacheRegenMiss: register.gauge<"caller">({ + shufflingCacheRegenMiss: register.gauge<{caller: RegenCaller}>({ name: "lodestar_gossip_attestation_shuffling_cache_regen_miss_count", help: "Count of gossip attestation verification shuffling cache regen miss", labelNames: ["caller"], }), - attestationSlotToClockSlot: register.histogram<"caller">({ + attestationSlotToClockSlot: register.histogram<{caller: RegenCaller}>({ name: "lodestar_gossip_attestation_attestation_slot_to_clock_slot", help: "Slot distance between clock slot and attestation slot", labelNames: ["caller"], @@ -672,7 +687,7 @@ export function createLodestarMetrics( help: "Time elapsed between block received and execution payload verification", buckets: [0.05, 0.1, 0.3, 0.5, 0.7, 1, 1.3, 1.6, 2, 2.5, 3, 3.5, 4], }), - receivedToBlobsAvailabilityTime: register.histogram<"numBlobs">({ + receivedToBlobsAvailabilityTime: register.histogram<{numBlobs: number}>({ name: "lodestar_gossip_block_received_to_blobs_availability_time", help: "Time elapsed between block received and blobs became available", buckets: [0.05, 0.1, 0.3, 0.5, 0.7, 1, 1.3, 1.6, 2, 2.5, 3, 3.5, 4], @@ -683,7 +698,7 @@ export function createLodestarMetrics( help: "Time elapsed between block received and fully verified state, signatures and payload", buckets: [0.05, 0.1, 0.3, 0.5, 0.7, 1, 1.3, 1.6, 2, 2.5, 3, 3.5, 4], }), - verifiedToBlobsAvailabiltyTime: register.histogram<"numBlobs">({ + verifiedToBlobsAvailabiltyTime: register.histogram<{numBlobs: number}>({ name: "lodestar_gossip_block_verified_to_blobs_availability_time", help: "Time elapsed between block verified and blobs became available", buckets: [0.05, 0.1, 0.3, 0.5, 0.7, 1, 1.3, 1.6, 2, 2.5, 3, 3.5, 4], @@ -694,7 +709,7 @@ export function createLodestarMetrics( help: "Time elapsed between block received and block import", buckets: [0.05, 0.1, 0.3, 0.5, 0.7, 1, 1.3, 1.6, 2, 2.5, 3, 3.5, 4], }), - processBlockErrors: register.gauge<"error">({ + processBlockErrors: register.gauge<{error: BlockErrorCode | "NOT_BLOCK_ERROR"}>({ name: "lodestar_gossip_block_process_block_errors", help: "Count of errors, by error type, while processing blocks", labelNames: ["error"], @@ -725,13 +740,13 @@ export function createLodestarMetrics( name: "lodestar_import_block_set_head_after_first_interval_total", help: "Total times an imported block is set as head after the first slot interval", }), - bySource: register.gauge<"source">({ + bySource: register.gauge<{source: BlockSource}>({ name: "lodestar_import_block_by_source_total", help: "Total number of imported blocks by source", labelNames: ["source"], }), }, - engineNotifyNewPayloadResult: register.gauge<"result">({ + engineNotifyNewPayloadResult: register.gauge<{result: ExecutionPayloadStatus}>({ name: "lodestar_execution_engine_notify_new_payload_result_total", help: "The total result of calling notifyNewPayload execution engine api", labelNames: ["result"], @@ -745,7 +760,7 @@ export function createLodestarMetrics( name: "lodestar_backfill_prev_fin_or_ws_slot", help: "Slot of previous finalized or wsCheckpoint block to be validated", }), - totalBlocks: register.gauge<"method">({ + totalBlocks: register.gauge<{method: BackfillSyncMethod}>({ name: "lodestar_backfill_sync_blocks_total", help: "Total amount of backfilled blocks", labelNames: ["method"], @@ -776,7 +791,7 @@ export function createLodestarMetrics( name: "lodestar_oppool_attestation_pool_size", help: "Current size of the AttestationPool = total attestations unique by data and slot", }), - attestationPoolInsertOutcome: register.counter<"insertOutcome">({ + attestationPoolInsertOutcome: register.counter<{insertOutcome: InsertOutcome}>({ name: "lodestar_attestation_pool_insert_outcome_total", help: "Total number of InsertOutcome as a result of adding an attestation in a pool", labelNames: ["insertOutcome"], @@ -801,7 +816,7 @@ export function createLodestarMetrics( name: "lodestar_oppool_sync_committee_message_pool_size", help: "Current size of the SyncCommitteeMessagePool unique by slot subnet and block root", }), - syncCommitteeMessagePoolInsertOutcome: register.counter<"insertOutcome">({ + syncCommitteeMessagePoolInsertOutcome: register.counter<{insertOutcome: InsertOutcome}>({ name: "lodestar_oppool_sync_committee_message_insert_outcome_total", help: "Total number of InsertOutcome as a result of adding a SyncCommitteeMessage to pool", labelNames: ["insertOutcome"], @@ -827,7 +842,7 @@ export function createLodestarMetrics( // Validator Monitor Metrics (per-epoch summaries) // Only track prevEpochOnChainBalance per index - prevEpochOnChainBalance: register.gauge<"index">({ + prevEpochOnChainBalance: register.gauge<{index: number}>({ name: "validator_monitor_prev_epoch_on_chain_balance", help: "Balance of validator after an epoch", labelNames: ["index"], @@ -936,12 +951,12 @@ export function createLodestarMetrics( help: "The count of times a sync signature was seen inside an aggregate", buckets: [0, 1, 2, 3, 5, 10], }), - prevEpochAttestationSummary: register.gauge<"summary">({ + prevEpochAttestationSummary: register.gauge<{summary: string}>({ name: "validator_monitor_prev_epoch_attestation_summary", help: "Best guess of the node of the result of previous epoch validators attestation actions and causality", labelNames: ["summary"], }), - prevEpochBlockProposalSummary: register.gauge<"summary">({ + prevEpochBlockProposalSummary: register.gauge<{summary: string}>({ name: "validator_monitor_prev_epoch_block_proposal_summary", help: "Best guess of the node of the result of previous epoch validators block proposal actions and causality", labelNames: ["summary"], @@ -949,12 +964,12 @@ export function createLodestarMetrics( // Validator Monitor Metrics (real-time) - unaggregatedAttestationTotal: register.gauge<"src">({ + unaggregatedAttestationTotal: register.gauge<{src: OpSource}>({ name: "validator_monitor_unaggregated_attestation_total", help: "Number of unaggregated attestations seen", labelNames: ["src"], }), - unaggregatedAttestationDelaySeconds: register.histogram<"src">({ + unaggregatedAttestationDelaySeconds: register.histogram<{src: OpSource}>({ name: "validator_monitor_unaggregated_attestation_delay_seconds", help: "The delay between when the validator should send the attestation and when it was received", labelNames: ["src"], @@ -968,23 +983,23 @@ export function createLodestarMetrics( // refine if we want more reasonable values buckets: [0, 10, 20, 30], }), - aggregatedAttestationTotal: register.gauge<"src">({ + aggregatedAttestationTotal: register.gauge<{src: OpSource}>({ name: "validator_monitor_aggregated_attestation_total", help: "Number of aggregated attestations seen", labelNames: ["src"], }), - aggregatedAttestationDelaySeconds: register.histogram<"src">({ + aggregatedAttestationDelaySeconds: register.histogram<{src: OpSource}>({ name: "validator_monitor_aggregated_attestation_delay_seconds", help: "The delay between then the validator should send the aggregate and when it was received", labelNames: ["src"], buckets: [0.1, 0.25, 0.5, 1, 2, 5, 10], }), - attestationInAggregateTotal: register.gauge<"src">({ + attestationInAggregateTotal: register.gauge<{src: OpSource}>({ name: "validator_monitor_attestation_in_aggregate_total", help: "Number of times an attestation has been seen in an aggregate", labelNames: ["src"], }), - attestationInAggregateDelaySeconds: register.histogram<"src">({ + attestationInAggregateDelaySeconds: register.histogram<{src: OpSource}>({ name: "validator_monitor_attestation_in_aggregate_delay_seconds", help: "The delay between when the validator should send the aggregate and when it was received", labelNames: ["src"], @@ -1008,12 +1023,12 @@ export function createLodestarMetrics( name: "validator_monitor_sync_signature_in_aggregate_total", help: "Number of times a sync signature has been seen in an aggregate", }), - beaconBlockTotal: register.gauge<"src">({ + beaconBlockTotal: register.gauge<{src: OpSource}>({ name: "validator_monitor_beacon_block_total", help: "Total number of beacon blocks seen", labelNames: ["src"], }), - beaconBlockDelaySeconds: register.histogram<"src">({ + beaconBlockDelaySeconds: register.histogram<{src: OpSource}>({ name: "validator_monitor_beacon_block_delay_seconds", help: "The delay between when the validator should send the block and when it was received", labelNames: ["src"], @@ -1115,7 +1130,7 @@ export function createLodestarMetrics( name: "lodestar_balances_cache_misses_total", help: "Total number of balances cache misses", }), - closestStateResult: register.counter<"stateId">({ + closestStateResult: register.counter<{stateId: string}>({ name: "lodestar_balances_cache_closest_state_result_total", help: "Total number of stateIds returned as closest justified balances state by id", labelNames: ["stateId"], @@ -1193,7 +1208,7 @@ export function createLodestarMetrics( name: "lodestar_seen_cache_attestation_data_miss_total", help: "Total number of attestation data miss in SeenAttestationData", }), - reject: register.gauge<"reason">({ + reject: register.gauge<{reason: RejectReason}>({ name: "lodestar_seen_cache_attestation_data_reject_total", help: "Total number of attestation data rejected in SeenAttestationData", labelNames: ["reason"], @@ -1201,23 +1216,23 @@ export function createLodestarMetrics( }, }, - regenFnCallTotal: register.gauge<"entrypoint" | "caller">({ + regenFnCallTotal: register.gauge<{entrypoint: RegenFnName; caller: RegenCaller}>({ name: "lodestar_regen_fn_call_total", help: "Total number of calls for regen functions", labelNames: ["entrypoint", "caller"], }), - regenFnQueuedTotal: register.gauge<"entrypoint" | "caller">({ + regenFnQueuedTotal: register.gauge<{entrypoint: RegenFnName; caller: RegenCaller}>({ name: "lodestar_regen_fn_queued_total", help: "Total number of calls queued for regen functions", labelNames: ["entrypoint", "caller"], }), - regenFnCallDuration: register.histogram<"entrypoint" | "caller">({ + regenFnCallDuration: register.histogram<{entrypoint: RegenFnName; caller: RegenCaller}>({ name: "lodestar_regen_fn_call_duration", help: "regen function duration", labelNames: ["entrypoint", "caller"], buckets: [0.1, 1, 10, 100], }), - regenFnTotalErrors: register.gauge<"entrypoint" | "caller">({ + regenFnTotalErrors: register.gauge<{entrypoint: RegenFnName; caller: RegenCaller}>({ name: "lodestar_regen_fn_errors_total", help: "regen function total errors", labelNames: ["entrypoint", "caller"], @@ -1229,7 +1244,7 @@ export function createLodestarMetrics( // Precompute next epoch transition precomputeNextEpochTransition: { - count: register.counter<"result">({ + count: register.counter<{result: string}>({ name: "lodestar_precompute_next_epoch_transition_result_total", labelNames: ["result"], help: "Total number of precomputeNextEpochTransition runs by result", @@ -1258,14 +1273,15 @@ export function createLodestarMetrics( name: "lodestar_reprocess_attestations_wait_time_resolve_seconds", help: "Time to wait for unknown block in seconds", }), - reject: register.gauge<"reason">({ + reject: register.gauge<{reason: ReprocessStatus}>({ name: "lodestar_reprocess_attestations_reject_total", help: "Total number of attestations are rejected to reprocess", labelNames: ["reason"], }), - waitSecBeforeReject: register.gauge<"reason">({ + waitSecBeforeReject: register.gauge<{reason: ReprocessStatus}>({ name: "lodestar_reprocess_attestations_wait_time_reject_seconds", help: "Time to wait for unknown block before being rejected", + labelNames: ["reason"], }), }, @@ -1287,24 +1303,25 @@ export function createLodestarMetrics( name: "lodestar_reprocess_gossip_attestations_wait_time_resolve_seconds", help: "Time to wait for unknown block in seconds", }), - reject: register.gauge<"reason">({ + reject: register.gauge<{reason: ReprocessRejectReason}>({ name: "lodestar_reprocess_gossip_attestations_reject_total", help: "Total number of attestations are rejected to reprocess", labelNames: ["reason"], }), - waitSecBeforeReject: register.gauge<"reason">({ + waitSecBeforeReject: register.gauge<{reason: ReprocessRejectReason}>({ name: "lodestar_reprocess_gossip_attestations_wait_time_reject_seconds", help: "Time to wait for unknown block before being rejected", + labelNames: ["reason"], }), }, lightclientServer: { - onSyncAggregate: register.gauge<"event">({ + onSyncAggregate: register.gauge<{event: string}>({ name: "lodestar_lightclient_server_on_sync_aggregate_event_total", help: "Total number of relevant events onSyncAggregate fn", labelNames: ["event"], }), - highestSlot: register.gauge<"item">({ + highestSlot: register.gauge<{item: string}>({ name: "lodestar_lightclient_server_highest_slot", help: "Current highest slot of items stored by LightclientServer", labelNames: ["item"], @@ -1415,7 +1432,11 @@ export function createLodestarMetrics( }), // Merge details - eth1MergeBlockDetails: register.gauge<"terminalBlockHash" | "terminalBlockNumber" | "terminalBlockTD">({ + eth1MergeBlockDetails: register.gauge<{ + terminalBlockHash: string; + terminalBlockNumber: string; + terminalBlockTD: string; + }>({ name: "lodestar_eth1_merge_block_details", help: "If found then 1 with terminal block details", labelNames: ["terminalBlockHash", "terminalBlockNumber", "terminalBlockTD"], @@ -1423,36 +1444,36 @@ export function createLodestarMetrics( }, eth1HttpClient: { - requestTime: register.histogram<"routeId">({ + requestTime: register.histogram<{routeId: string}>({ name: "lodestar_eth1_http_client_request_time_seconds", help: "eth1 JsonHttpClient - histogram or roundtrip request times", labelNames: ["routeId"], // Provide max resolution on problematic values around 1 second buckets: [0.1, 0.5, 1, 2, 5, 15], }), - streamTime: register.histogram<"routeId">({ + streamTime: register.histogram<{routeId: string}>({ name: "lodestar_eth1_http_client_stream_time_seconds", help: "eth1 JsonHttpClient - streaming time by routeId", labelNames: ["routeId"], // Provide max resolution on problematic values around 1 second buckets: [0.1, 0.5, 1, 2, 5, 15], }), - requestErrors: register.gauge<"routeId">({ + requestErrors: register.gauge<{routeId: string}>({ name: "lodestar_eth1_http_client_request_errors_total", help: "eth1 JsonHttpClient - total count of request errors", labelNames: ["routeId"], }), - retryCount: register.gauge<"routeId">({ + retryCount: register.gauge<{routeId: string}>({ name: "lodestar_eth1_http_client_request_retries_total", help: "eth1 JsonHttpClient - total count of request retries", labelNames: ["routeId"], }), - requestUsedFallbackUrl: register.gauge({ + requestUsedFallbackUrl: register.gauge<{routeId: string}>({ name: "lodestar_eth1_http_client_request_used_fallback_url_total", help: "eth1 JsonHttpClient - total count of requests on fallback url(s)", labelNames: ["routeId"], }), - activeRequests: register.gauge({ + activeRequests: register.gauge<{routeId: string}>({ name: "lodestar_eth1_http_client_active_requests", help: "eth1 JsonHttpClient - current count of active requests", labelNames: ["routeId"], @@ -1464,36 +1485,36 @@ export function createLodestarMetrics( }, executionEnginerHttpClient: { - requestTime: register.histogram<"routeId">({ + requestTime: register.histogram<{routeId: string}>({ name: "lodestar_execution_engine_http_client_request_time_seconds", help: "ExecutionEngineHttp client - histogram or roundtrip request times", labelNames: ["routeId"], // Provide max resolution on problematic values around 1 second buckets: [0.1, 0.5, 1, 2, 5, 15], }), - streamTime: register.histogram<"routeId">({ + streamTime: register.histogram<{routeId: string}>({ name: "lodestar_execution_engine_http_client_stream_time_seconds", help: "ExecutionEngineHttp client - streaming time by routeId", labelNames: ["routeId"], // Provide max resolution on problematic values around 1 second buckets: [0.1, 0.5, 1, 2, 5, 15], }), - requestErrors: register.gauge<"routeId">({ + requestErrors: register.gauge<{routeId: string}>({ name: "lodestar_execution_engine_http_client_request_errors_total", help: "ExecutionEngineHttp client - total count of request errors", labelNames: ["routeId"], }), - retryCount: register.gauge<"routeId">({ + retryCount: register.gauge<{routeId: string}>({ name: "lodestar_execution_engine_http_client_request_retries_total", help: "ExecutionEngineHttp client - total count of request retries", labelNames: ["routeId"], }), - requestUsedFallbackUrl: register.gauge({ + requestUsedFallbackUrl: register.gauge<{routeId: string}>({ name: "lodestar_execution_engine_http_client_request_used_fallback_url_total", help: "ExecutionEngineHttp client - total count of requests on fallback url(s)", labelNames: ["routeId"], }), - activeRequests: register.gauge({ + activeRequests: register.gauge<{routeId: string}>({ name: "lodestar_execution_engine_http_client_active_requests", help: "ExecutionEngineHttp client - current count of active requests", labelNames: ["routeId"], @@ -1505,32 +1526,32 @@ export function createLodestarMetrics( }, builderHttpClient: { - requestTime: register.histogram<"routeId">({ + requestTime: register.histogram<{routeId: string}>({ name: "lodestar_builder_http_client_request_time_seconds", help: "Histogram of builder http client request time by routeId", labelNames: ["routeId"], // Expected times are ~ 50-500ms, but in an overload NodeJS they can be greater buckets: [0.01, 0.1, 1, 5], }), - streamTime: register.histogram<"routeId">({ + streamTime: register.histogram<{routeId: string}>({ name: "lodestar_builder_http_client_stream_time_seconds", help: "Builder api - streaming time by routeId", labelNames: ["routeId"], // Provide max resolution on problematic values around 1 second buckets: [0.1, 0.5, 1, 2, 5, 15], }), - requestErrors: register.gauge<"routeId">({ + requestErrors: register.gauge<{routeId: string}>({ name: "lodestar_builder_http_client_request_errors_total", help: "Total count of errors on builder http client requests by routeId", labelNames: ["routeId"], }), - requestToFallbacks: register.gauge<"routeId">({ + requestToFallbacks: register.gauge<{routeId: string}>({ name: "lodestar_builder_http_client_request_to_fallbacks_total", help: "Total count of requests to fallback URLs on builder http API by routeId", labelNames: ["routeId"], }), - urlsScore: register.gauge<"urlIndex">({ + urlsScore: register.gauge<{urlIndex: number}>({ name: "lodestar_builder_http_client_urls_score", help: "Current score of builder http URLs by url index", labelNames: ["urlIndex"], @@ -1538,22 +1559,22 @@ export function createLodestarMetrics( }, db: { - dbReadReq: register.gauge<"bucket">({ + dbReadReq: register.gauge<{bucket: string}>({ name: "lodestar_db_read_req_total", help: "Total count of db read requests, may read 0 or more items", labelNames: ["bucket"], }), - dbReadItems: register.gauge<"bucket">({ + dbReadItems: register.gauge<{bucket: string}>({ name: "lodestar_db_read_items_total", help: "Total count of db read items, item = key | value | entry", labelNames: ["bucket"], }), - dbWriteReq: register.gauge<"bucket">({ + dbWriteReq: register.gauge<{bucket: string}>({ name: "lodestar_db_write_req_total", help: "Total count of db write requests, may write 0 or more items", labelNames: ["bucket"], }), - dbWriteItems: register.gauge<"bucket">({ + dbWriteItems: register.gauge<{bucket: string}>({ name: "lodestar_db_write_items_total", help: "Total count of db write items", labelNames: ["bucket"], diff --git a/packages/beacon-node/src/metrics/server/http.ts b/packages/beacon-node/src/metrics/server/http.ts index b699471e07d5..d8fbb289e951 100644 --- a/packages/beacon-node/src/metrics/server/http.ts +++ b/packages/beacon-node/src/metrics/server/http.ts @@ -15,6 +15,11 @@ export type HttpMetricsServer = { close(): Promise; }; +enum RequestStatus { + success = "success", + error = "error", +} + export async function getHttpMetricsServer( opts: HttpMetricsServerOpts, { @@ -26,7 +31,7 @@ export async function getHttpMetricsServer( // New registry to metric the metrics. Using the same registry would deadlock the .metrics promise const httpServerRegister = new RegistryMetricCreator(); - const scrapeTimeMetric = httpServerRegister.histogram<"status">({ + const scrapeTimeMetric = httpServerRegister.histogram<{status: RequestStatus}>({ name: "lodestar_metrics_scrape_seconds", help: "Lodestar metrics server async time to scrape metrics", labelNames: ["status"], @@ -40,7 +45,7 @@ export async function getHttpMetricsServer( if (req.method === "GET" && req.url && req.url.includes("/metrics")) { const timer = scrapeTimeMetric.startTimer(); const metricsRes = await Promise.all([wrapError(register.metrics()), getOtherMetrics()]); - timer({status: metricsRes[0].err ? "error" : "success"}); + timer({status: metricsRes[0].err ? RequestStatus.error : RequestStatus.success}); // Ensure we only writeHead once if (metricsRes[0].err) { diff --git a/packages/beacon-node/src/metrics/utils/avgMinMax.ts b/packages/beacon-node/src/metrics/utils/avgMinMax.ts index 43f51c821790..709c83ee38d6 100644 --- a/packages/beacon-node/src/metrics/utils/avgMinMax.ts +++ b/packages/beacon-node/src/metrics/utils/avgMinMax.ts @@ -1,21 +1,21 @@ import {GaugeConfiguration} from "prom-client"; +import {AvgMinMax as IAvgMinMax, LabelKeys, LabelsGeneric} from "@lodestar/utils"; import {GaugeExtra} from "./gauge.js"; type GetValuesFn = () => number[]; -type Labels = Partial>; /** * Special non-standard "Histogram" that captures the avg, min and max of values */ -export class AvgMinMax { - private readonly sum: GaugeExtra; - private readonly avg: GaugeExtra; - private readonly min: GaugeExtra; - private readonly max: GaugeExtra; +export class AvgMinMax implements IAvgMinMax { + private readonly sum: GaugeExtra; + private readonly avg: GaugeExtra; + private readonly min: GaugeExtra; + private readonly max: GaugeExtra; private getValuesFn: GetValuesFn | null = null; - constructor(configuration: GaugeConfiguration) { + constructor(configuration: GaugeConfiguration>) { this.sum = new GaugeExtra({...configuration, name: `${configuration.name}_sum`}); this.avg = new GaugeExtra({...configuration, name: `${configuration.name}_avg`}); this.min = new GaugeExtra({...configuration, name: `${configuration.name}_min`}); @@ -33,8 +33,8 @@ export class AvgMinMax { } set(values: number[]): void; - set(labels: Labels, values: number[]): void; - set(arg1?: Labels | number[], arg2?: number[]): void { + set(labels: Labels, values: number[]): void; + set(arg1?: Labels | number[], arg2?: number[]): void { if (arg2 === undefined) { const values = arg1 as number[]; const {sum, avg, min, max} = getStats(values); @@ -44,7 +44,7 @@ export class AvgMinMax { this.max.set(max); } else { const values = (arg2 !== undefined ? arg2 : arg1) as number[]; - const labels = arg1 as Labels; + const labels = arg1 as Labels; const {sum, avg, min, max} = getStats(values); this.sum.set(labels, sum); this.avg.set(labels, avg); diff --git a/packages/beacon-node/src/metrics/utils/gauge.ts b/packages/beacon-node/src/metrics/utils/gauge.ts index fb95fe25d24d..1f527adfcb64 100644 --- a/packages/beacon-node/src/metrics/utils/gauge.ts +++ b/packages/beacon-node/src/metrics/utils/gauge.ts @@ -1,29 +1,16 @@ -import {Gauge, GaugeConfiguration} from "prom-client"; -import {IGauge} from "../interface.js"; - -type CollectFn = (metric: IGauge) => void; -type Labels = Partial>; +import {Gauge} from "prom-client"; +import {CollectFn, Gauge as IGauge, LabelKeys, LabelsGeneric} from "@lodestar/utils"; /** - * Extends the prom-client Gauge with extra features: - * - Add multiple collect functions after instantiation - * - Create child gauges with fixed labels + * Extends the prom-client Gauge to be able to add multiple collect functions after instantiation */ -export class GaugeExtra extends Gauge implements IGauge { - private collectFns: CollectFn[] = []; - - constructor(configuration: GaugeConfiguration) { - super(configuration); - } +export class GaugeExtra extends Gauge> implements IGauge { + private collectFns: CollectFn[] = []; - addCollect(collectFn: CollectFn): void { + addCollect(collectFn: CollectFn): void { this.collectFns.push(collectFn); } - child(labels: Labels): GaugeChild { - return new GaugeChild(labels, this); - } - /** * @override Metric.collect */ @@ -33,48 +20,3 @@ export class GaugeExtra extends Gauge implements IGauge { } } } - -export class GaugeChild implements IGauge { - gauge: GaugeExtra; - labelsParent: Labels; - constructor(labelsParent: Labels, gauge: GaugeExtra) { - this.gauge = gauge; - this.labelsParent = labelsParent; - } - - // Sorry for this mess, `prom-client` API choices are not great - // If the function signature was `inc(value: number, labels?: Labels)`, this would be simpler - inc(value?: number): void; - inc(labels: Labels, value?: number): void; - inc(arg1?: Labels | number, arg2?: number): void { - if (typeof arg1 === "object") { - this.gauge.inc({...this.labelsParent, ...arg1}, arg2 ?? 1); - } else { - this.gauge.inc(this.labelsParent, arg1 ?? 1); - } - } - - dec(value?: number): void; - dec(labels: Labels, value?: number): void; - dec(arg1?: Labels | number, arg2?: number): void { - if (typeof arg1 === "object") { - this.gauge.dec({...this.labelsParent, ...arg1}, arg2 ?? 1); - } else { - this.gauge.dec(this.labelsParent, arg1 ?? 1); - } - } - - set(value: number): void; - set(labels: Labels, value: number): void; - set(arg1?: Labels | number, arg2?: number): void { - if (typeof arg1 === "object") { - this.gauge.set({...this.labelsParent, ...arg1}, arg2 ?? 0); - } else { - this.gauge.set(this.labelsParent, arg1 ?? 0); - } - } - - addCollect(collectFn: CollectFn): void { - this.gauge.addCollect(() => collectFn(this)); - } -} diff --git a/packages/beacon-node/src/metrics/utils/histogram.ts b/packages/beacon-node/src/metrics/utils/histogram.ts deleted file mode 100644 index 4490929629f2..000000000000 --- a/packages/beacon-node/src/metrics/utils/histogram.ts +++ /dev/null @@ -1,48 +0,0 @@ -import {Histogram, HistogramConfiguration} from "prom-client"; -import {IHistogram} from "../interface.js"; - -type Labels = Partial>; - -/** - * Extends the prom-client Histogram with extra features: - * - Add multiple collect functions after instantiation - * - Create child histograms with fixed labels - */ -export class HistogramExtra extends Histogram implements IHistogram { - constructor(configuration: HistogramConfiguration) { - super(configuration); - } - - child(labels: Labels): HistogramChild { - return new HistogramChild(labels, this); - } -} - -export class HistogramChild implements IHistogram { - histogram: HistogramExtra; - labelsParent: Labels; - constructor(labelsParent: Labels, histogram: HistogramExtra) { - this.histogram = histogram; - this.labelsParent = labelsParent; - } - - // Sorry for this mess, `prom-client` API choices are not great - // If the function signature was `observe(value: number, labels?: Labels)`, this would be simpler - observe(value?: number): void; - observe(labels: Labels, value?: number): void; - observe(arg1?: Labels | number, arg2?: number): void { - if (typeof arg1 === "object") { - this.histogram.observe({...this.labelsParent, ...arg1}, arg2 ?? 0); - } else { - this.histogram.observe(this.labelsParent, arg1 ?? 0); - } - } - - startTimer(arg1?: Labels): (labels?: Labels) => number { - if (typeof arg1 === "object") { - return this.histogram.startTimer({...this.labelsParent, ...arg1}); - } else { - return this.histogram.startTimer(this.labelsParent); - } - } -} diff --git a/packages/beacon-node/src/metrics/utils/registryMetricCreator.ts b/packages/beacon-node/src/metrics/utils/registryMetricCreator.ts index 8864eb2c74c4..adec6f984702 100644 --- a/packages/beacon-node/src/metrics/utils/registryMetricCreator.ts +++ b/packages/beacon-node/src/metrics/utils/registryMetricCreator.ts @@ -1,33 +1,41 @@ -import {Gauge, GaugeConfiguration, Registry, HistogramConfiguration, CounterConfiguration, Counter} from "prom-client"; +import {Gauge, Registry, Counter, Histogram} from "prom-client"; +import { + AvgMinMaxConfig, + CounterConfig, + GaugeConfig, + HistogramConfig, + AvgMinMax as IAvgMinMax, + Counter as ICounter, + GaugeExtra as IGaugeExtra, + Histogram as IHistogram, + LabelKeys, + LabelsGeneric, + MetricsRegisterCustom, + NoLabels, + StaticConfig, +} from "@lodestar/utils"; import {AvgMinMax} from "./avgMinMax.js"; import {GaugeExtra} from "./gauge.js"; -import {HistogramExtra} from "./histogram.js"; -type StaticConfiguration = { - name: GaugeConfiguration["name"]; - help: GaugeConfiguration["help"]; - value: Record; -}; - -export class RegistryMetricCreator extends Registry { - gauge(configuration: GaugeConfiguration): GaugeExtra { - return new GaugeExtra({...configuration, registers: [this]}); +export class RegistryMetricCreator extends Registry implements MetricsRegisterCustom { + gauge(configuration: GaugeConfig): IGaugeExtra { + return new GaugeExtra({...configuration, registers: [this]}); } - histogram(configuration: HistogramConfiguration): HistogramExtra { - return new HistogramExtra({...configuration, registers: [this]}); + histogram(configuration: HistogramConfig): IHistogram { + return new Histogram>({...configuration, registers: [this]}); } - avgMinMax(configuration: GaugeConfiguration): AvgMinMax { - return new AvgMinMax({...configuration, registers: [this]}); + avgMinMax(configuration: AvgMinMaxConfig): IAvgMinMax { + return new AvgMinMax({...configuration, registers: [this]}); } /** Static metric to send string-based data such as versions, config params, etc */ - static({name, help, value}: StaticConfiguration): void { + static({name, help, value}: StaticConfig): void { new Gauge({name, help, labelNames: Object.keys(value), registers: [this]}).set(value, 1); } - counter(configuration: CounterConfiguration): Counter { - return new Counter({...configuration, registers: [this]}); + counter(configuration: CounterConfig): ICounter { + return new Counter>({...configuration, registers: [this]}); } } diff --git a/packages/beacon-node/src/monitoring/service.ts b/packages/beacon-node/src/monitoring/service.ts index f50f992ebe1f..9581c5f11c92 100644 --- a/packages/beacon-node/src/monitoring/service.ts +++ b/packages/beacon-node/src/monitoring/service.ts @@ -1,8 +1,7 @@ import {Registry} from "prom-client"; import {fetch} from "@lodestar/api"; -import {ErrorAborted, Logger, TimeoutError} from "@lodestar/utils"; +import {ErrorAborted, Histogram, Logger, TimeoutError} from "@lodestar/utils"; import {RegistryMetricCreator} from "../metrics/index.js"; -import {HistogramExtra} from "../metrics/utils/histogram.js"; import {defaultMonitoringOptions, MonitoringOptions} from "./options.js"; import {createClientStats} from "./clientStats.js"; import {ClientStats} from "./types.js"; @@ -25,6 +24,11 @@ enum Status { Closed = "closed", } +enum SendDataStatus { + Success = "success", + Error = "error", +} + export type Client = "beacon" | "validator"; /** @@ -38,8 +42,8 @@ export class MonitoringService { private readonly register: Registry; private readonly logger: Logger; - private readonly collectDataMetric: HistogramExtra; - private readonly sendDataMetric: HistogramExtra<"status">; + private readonly collectDataMetric: Histogram; + private readonly sendDataMetric: Histogram<{status: SendDataStatus}>; private status = Status.Started; private initialDelayTimeout?: NodeJS.Timeout; @@ -193,7 +197,7 @@ export class MonitoringService { throw e; } } finally { - timer({status: res?.ok ? "success" : "error"}); + timer({status: res?.ok ? SendDataStatus.Success : SendDataStatus.Error}); clearTimeout(timeout); } } diff --git a/packages/beacon-node/src/network/core/metrics.ts b/packages/beacon-node/src/network/core/metrics.ts index 4f416ad4fba2..0137ce1f0540 100644 --- a/packages/beacon-node/src/network/core/metrics.ts +++ b/packages/beacon-node/src/network/core/metrics.ts @@ -1,4 +1,8 @@ import {RegistryMetricCreator} from "../../metrics/utils/registryMetricCreator.js"; +import {SubnetType} from "../metadata.js"; +import {DiscoveredPeerStatus} from "../peers/discover.js"; +import {SubnetSource} from "../subnets/attnetsService.js"; +import {DLLSubnetSource} from "../subnets/dllAttnetsService.js"; export type NetworkCoreMetrics = ReturnType; @@ -13,12 +17,12 @@ export function createNetworkCoreMetrics(register: RegistryMetricCreator) { name: "libp2p_peers", help: "number of connected peers", }), - peersByDirection: register.gauge<"direction">({ + peersByDirection: register.gauge<{direction: string}>({ name: "lodestar_peers_by_direction_count", help: "number of peers, labeled by direction", labelNames: ["direction"], }), - peersByClient: register.gauge<"client">({ + peersByClient: register.gauge<{client: string}>({ name: "lodestar_peers_by_client_count", help: "number of peers, labeled by client", labelNames: ["client"], @@ -28,14 +32,14 @@ export function createNetworkCoreMetrics(register: RegistryMetricCreator) { help: "Histogram of current count of long lived attnets of connected peers", buckets: [0, 4, 16, 32, 64], }), - peerScoreByClient: register.histogram<"client">({ + peerScoreByClient: register.histogram<{client: string}>({ name: "lodestar_app_peer_score", help: "Current peer score at lodestar app side", // Min score = -100, max score = 100, disconnect = -20, ban = -50 buckets: [-100, -50, -20, 0, 25], labelNames: ["client"], }), - peerGossipScoreByClient: register.histogram<"client">({ + peerGossipScoreByClient: register.histogram<{client: string}>({ name: "lodestar_gossip_score_by_client", help: "Gossip peer score by client", labelNames: ["client"], @@ -53,27 +57,27 @@ export function createNetworkCoreMetrics(register: RegistryMetricCreator) { name: "lodestar_peers_sync_count", help: "Current count of peers useful for sync", }), - peerConnectedEvent: register.gauge<"direction" | "status">({ + peerConnectedEvent: register.gauge<{direction: string; status: string}>({ name: "lodestar_peer_connected_total", help: "Total number of peer:connected event, labeled by direction", labelNames: ["direction", "status"], }), - peerDisconnectedEvent: register.gauge<"direction">({ + peerDisconnectedEvent: register.gauge<{direction: string}>({ name: "lodestar_peer_disconnected_total", help: "Total number of peer:disconnected event, labeled by direction", labelNames: ["direction"], }), - peerGoodbyeReceived: register.gauge<"reason">({ + peerGoodbyeReceived: register.gauge<{reason: string}>({ name: "lodestar_peer_goodbye_received_total", help: "Total number of goodbye received, labeled by reason", labelNames: ["reason"], }), - peerLongConnectionDisconnect: register.gauge<"reason">({ + peerLongConnectionDisconnect: register.gauge<{reason: string}>({ name: "lodestar_peer_long_connection_disconnect_total", help: "For peers with long connection, track disconnect reason", labelNames: ["reason"], }), - peerGoodbyeSent: register.gauge<"reason">({ + peerGoodbyeSent: register.gauge<{reason: string}>({ name: "lodestar_peer_goodbye_sent_total", help: "Total number of goodbye sent, labeled by reason", labelNames: ["reason"], @@ -82,22 +86,22 @@ export function createNetworkCoreMetrics(register: RegistryMetricCreator) { name: "lodestar_peers_requested_total_to_connect", help: "Prioritization results total peers count requested to connect", }), - peersRequestedToDisconnect: register.gauge<"reason">({ + peersRequestedToDisconnect: register.gauge<{reason: string}>({ name: "lodestar_peers_requested_total_to_disconnect", help: "Prioritization results total peers count requested to disconnect", labelNames: ["reason"], }), - peersRequestedSubnetsToQuery: register.gauge<"type">({ + peersRequestedSubnetsToQuery: register.gauge<{type: SubnetType}>({ name: "lodestar_peers_requested_total_subnets_to_query", help: "Prioritization results total subnets to query and discover peers in", labelNames: ["type"], }), - peersRequestedSubnetsPeerCount: register.gauge<"type">({ + peersRequestedSubnetsPeerCount: register.gauge<{type: SubnetType}>({ name: "lodestar_peers_requested_total_subnets_peers_count", help: "Prioritization results total peers in subnets to query and discover peers in", labelNames: ["type"], }), - peersReportPeerCount: register.gauge<"reason">({ + peersReportPeerCount: register.gauge<{reason: string}>({ name: "lodestar_peers_report_peer_count", help: "network.reportPeer count by reason", labelNames: ["reason"], @@ -115,12 +119,12 @@ export function createNetworkCoreMetrics(register: RegistryMetricCreator) { name: "lodestar_discovery_peers_to_connect", help: "Current peers to connect count from discoverPeers requests", }), - subnetPeersToConnect: register.gauge<"type">({ + subnetPeersToConnect: register.gauge<{type: SubnetType}>({ name: "lodestar_discovery_subnet_peers_to_connect", help: "Current peers to connect count from discoverPeers requests", labelNames: ["type"], }), - subnetsToConnect: register.gauge<"type">({ + subnetsToConnect: register.gauge<{type: SubnetType}>({ name: "lodestar_discovery_subnets_to_connect", help: "Current subnets to connect count from discoverPeers requests", labelNames: ["type"], @@ -129,7 +133,7 @@ export function createNetworkCoreMetrics(register: RegistryMetricCreator) { name: "lodestar_discovery_cached_enrs_size", help: "Current size of the cachedENRs Set", }), - findNodeQueryRequests: register.gauge<"action">({ + findNodeQueryRequests: register.gauge<{action: string}>({ name: "lodestar_discovery_find_node_query_requests_total", help: "Total count of find node queries started", labelNames: ["action"], @@ -143,7 +147,7 @@ export function createNetworkCoreMetrics(register: RegistryMetricCreator) { name: "lodestar_discovery_find_node_query_enrs_total", help: "Total count of found ENRs in queries", }), - discoveredStatus: register.gauge<"status">({ + discoveredStatus: register.gauge<{status: DiscoveredPeerStatus}>({ name: "lodestar_discovery_discovered_status_total_count", help: "Total count of status results of PeerDiscovery.onDiscovered() function", labelNames: ["status"], @@ -152,7 +156,7 @@ export function createNetworkCoreMetrics(register: RegistryMetricCreator) { name: "lodestar_discovery_total_dial_attempts", help: "Total dial attempts by peer discovery", }), - dialTime: register.histogram<"status">({ + dialTime: register.histogram<{status: string}>({ name: "lodestar_discovery_dial_time_seconds", help: "Time to dial peers in seconds", labelNames: ["status"], @@ -161,62 +165,13 @@ export function createNetworkCoreMetrics(register: RegistryMetricCreator) { }, reqResp: { - rateLimitErrors: register.gauge<"method">({ + rateLimitErrors: register.gauge<{method: string}>({ name: "beacon_reqresp_rate_limiter_errors_total", help: "Count rate limiter errors", labelNames: ["method"], }), }, - gossipValidationAccept: register.gauge<"topic">({ - name: "lodestar_gossip_validation_accept_total", - help: "Count of total gossip validation accept", - labelNames: ["topic"], - }), - gossipValidationIgnore: register.gauge<"topic">({ - name: "lodestar_gossip_validation_ignore_total", - help: "Count of total gossip validation ignore", - labelNames: ["topic"], - }), - gossipValidationReject: register.gauge<"topic">({ - name: "lodestar_gossip_validation_reject_total", - help: "Count of total gossip validation reject", - labelNames: ["topic"], - }), - gossipValidationError: register.gauge<"topic" | "error">({ - name: "lodestar_gossip_validation_error_total", - help: "Count of total gossip validation errors detailed", - labelNames: ["topic", "error"], - }), - - gossipValidationQueueLength: register.gauge<"topic">({ - name: "lodestar_gossip_validation_queue_length", - help: "Count of total gossip validation queue length", - labelNames: ["topic"], - }), - gossipValidationQueueDroppedJobs: register.gauge<"topic">({ - name: "lodestar_gossip_validation_queue_dropped_jobs_total", - help: "Count of total gossip validation queue dropped jobs", - labelNames: ["topic"], - }), - gossipValidationQueueJobTime: register.histogram<"topic">({ - name: "lodestar_gossip_validation_queue_job_time_seconds", - help: "Time to process gossip validation queue job in seconds", - labelNames: ["topic"], - buckets: [0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2, 5, 10], - }), - gossipValidationQueueJobWaitTime: register.histogram<"topic">({ - name: "lodestar_gossip_validation_queue_job_wait_time_seconds", - help: "Time from job added to the queue to starting the job in seconds", - labelNames: ["topic"], - buckets: [0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2, 5, 10], - }), - gossipValidationQueueConcurrency: register.gauge<"topic">({ - name: "lodestar_gossip_validation_queue_concurrency", - help: "Current count of jobs being run on network processor for topic", - labelNames: ["topic"], - }), - discv5: { decodeEnrAttemptCount: register.counter({ name: "lodestar_discv5_decode_enr_attempt_count", @@ -237,14 +192,14 @@ export function createNetworkCoreMetrics(register: RegistryMetricCreator) { name: "lodestar_attnets_service_committee_subscriptions_total", help: "Count of committee subscriptions", }), - subscriptionsCommitteeMeshPeers: register.histogram<"subnet">({ + subscriptionsCommitteeMeshPeers: register.histogram<{subnet: number}>({ name: "lodestar_attnets_service_committee_subscriptions_mesh_peers", help: "Histogram of mesh peers per committee subscription", labelNames: ["subnet"], // Dlow = 6, D = 8, DHi = 12 plus 2 more buckets buckets: [0, 4, 6, 8, 12], }), - subscriptionsCommitteeTimeToStableMesh: register.histogram<"subnet">({ + subscriptionsCommitteeTimeToStableMesh: register.histogram<{subnet: number}>({ name: "lodestar_attnets_service_committee_subscriptions_time_to_stable_mesh_seconds", help: "Histogram of time until committee subscription is considered healthy (>= 6 mesh peers)", labelNames: ["subnet"], @@ -259,12 +214,12 @@ export function createNetworkCoreMetrics(register: RegistryMetricCreator) { name: "lodestar_attnets_service_long_lived_subscriptions_total", help: "Count of long lived subscriptions", }), - subscribeSubnets: register.gauge<"subnet" | "src">({ + subscribeSubnets: register.gauge<{subnet: number; src: SubnetSource | DLLSubnetSource}>({ name: "lodestar_attnets_service_subscribe_subnets_total", help: "Count of subscribe_subnets calls", labelNames: ["subnet", "src"], }), - unsubscribeSubnets: register.gauge<"subnet" | "src">({ + unsubscribeSubnets: register.gauge<{subnet: number; src: SubnetSource | DLLSubnetSource}>({ name: "lodestar_attnets_service_unsubscribe_subnets_total", help: "Count of unsubscribe_subnets calls", labelNames: ["subnet", "src"], @@ -280,12 +235,12 @@ export function createNetworkCoreMetrics(register: RegistryMetricCreator) { name: "lodestar_syncnets_service_committee_subscriptions_total", help: "Count of syncnet committee subscriptions", }), - subscribeSubnets: register.gauge<"subnet">({ + subscribeSubnets: register.gauge<{subnet: number}>({ name: "lodestar_syncnets_service_subscribe_subnets_total", help: "Count of syncnet subscribe_subnets calls", labelNames: ["subnet"], }), - unsubscribeSubnets: register.gauge<"subnet">({ + unsubscribeSubnets: register.gauge<{subnet: number}>({ name: "lodestar_syncnets_service_unsubscribe_subnets_total", help: "Count of syncnet unsubscribe_subnets calls", labelNames: ["subnet"], @@ -303,7 +258,7 @@ export function getNetworkCoreWorkerMetrics(register: RegistryMetricCreator) { name: "lodestar_network_worker_reqresp_bridge_caller_pending_count", help: "Current count of pending elements in respBridgeCaller", }), - networkWorkerWireEventsOnWorkerThreadLatency: register.histogram<"eventName">({ + networkWorkerWireEventsOnWorkerThreadLatency: register.histogram<{eventName: string}>({ name: "lodestar_network_worker_wire_events_on_worker_thread_latency_seconds", help: "Latency in seconds to transmit network events to worker thread across parent port", labelNames: ["eventName"], diff --git a/packages/beacon-node/src/network/discv5/worker.ts b/packages/beacon-node/src/network/discv5/worker.ts index 1b50ee86aa29..e09b063d13d1 100644 --- a/packages/beacon-node/src/network/discv5/worker.ts +++ b/packages/beacon-node/src/network/discv5/worker.ts @@ -3,12 +3,20 @@ import path from "node:path"; import fs from "node:fs"; import {createFromProtobuf} from "@libp2p/peer-id-factory"; import {Multiaddr, multiaddr} from "@multiformats/multiaddr"; -import {Gauge} from "prom-client"; import {expose} from "@chainsafe/threads/worker"; import {Observable, Subject} from "@chainsafe/threads/observable"; -import {createKeypairFromPeerId, Discv5, ENR, ENRData, SignableENR, SignableENRData} from "@chainsafe/discv5"; +import { + createKeypairFromPeerId, + Discv5, + ENR, + ENRData, + IDiscv5CreateOptions, + SignableENR, + SignableENRData, +} from "@chainsafe/discv5"; import {createBeaconConfig} from "@lodestar/config"; import {getNodeLogger} from "@lodestar/logger/node"; +import {Gauge} from "@lodestar/utils"; import {RegistryMetricCreator} from "../../metrics/index.js"; import {collectNodeJSMetrics} from "../../metrics/nodeJsMetrics.js"; import {profileNodeJS, writeHeapSnapshot} from "../../util/profile.js"; @@ -28,14 +36,14 @@ const logger = getNodeLogger(workerData.loggerOpts); // Set up metrics, nodejs and discv5-specific let metricsRegistry: RegistryMetricCreator | undefined; -let enrRelevanceMetric: Gauge<"status"> | undefined; +let enrRelevanceMetric: Gauge<{status: string}> | undefined; let closeMetrics: () => void | undefined; if (workerData.metrics) { metricsRegistry = new RegistryMetricCreator(); closeMetrics = collectNodeJSMetrics(metricsRegistry, "discv5_worker_"); // add enr relevance metric - enrRelevanceMetric = metricsRegistry.gauge<"status">({ + enrRelevanceMetric = metricsRegistry.gauge<{status: string}>({ name: "lodestar_discv5_discovered_status_total_count", help: "Total count of status results of enrRelevance() function", labelNames: ["status"], @@ -56,7 +64,7 @@ const discv5 = Discv5.create({ ip6: workerData.bindAddrs.ip6 ? multiaddr(workerData.bindAddrs.ip6) : undefined, }, config: workerData.config, - metricsRegistry, + metricsRegistry: metricsRegistry as IDiscv5CreateOptions["metricsRegistry"], }); // Load boot enrs diff --git a/packages/beacon-node/src/network/gossip/metrics.ts b/packages/beacon-node/src/network/gossip/metrics.ts index 3711669edddf..c2b5d0b32338 100644 --- a/packages/beacon-node/src/network/gossip/metrics.ts +++ b/packages/beacon-node/src/network/gossip/metrics.ts @@ -1,4 +1,6 @@ +import {ForkName} from "@lodestar/params"; import {RegistryMetricCreator} from "../../metrics/index.js"; +import {GossipType} from "./interface.js"; export type Eth2GossipsubMetrics = ReturnType; @@ -6,12 +8,12 @@ export type Eth2GossipsubMetrics = ReturnType export function createEth2GossipsubMetrics(register: RegistryMetricCreator) { return { gossipPeer: { - scoreByThreshold: register.gauge<"threshold">({ + scoreByThreshold: register.gauge<{threshold: string}>({ name: "lodestar_gossip_peer_score_by_threshold_count", help: "Gossip peer score by threshold", labelNames: ["threshold"], }), - meshPeersByClient: register.gauge<"client">({ + meshPeersByClient: register.gauge<{client: string}>({ name: "lodestar_gossip_mesh_peers_by_client_count", help: "number of mesh peers, labeled by client", labelNames: ["client"], @@ -22,34 +24,34 @@ export function createEth2GossipsubMetrics(register: RegistryMetricCreator) { }), }, gossipMesh: { - peersByType: register.gauge<"type" | "fork">({ + peersByType: register.gauge<{type: GossipType; fork: ForkName}>({ name: "lodestar_gossip_mesh_peers_by_type_count", help: "Number of connected mesh peers per gossip type", labelNames: ["type", "fork"], }), - peersByBeaconAttestationSubnet: register.gauge<"subnet" | "fork">({ + peersByBeaconAttestationSubnet: register.gauge<{subnet: string; fork: ForkName}>({ name: "lodestar_gossip_mesh_peers_by_beacon_attestation_subnet_count", help: "Number of connected mesh peers per beacon attestation subnet", labelNames: ["subnet", "fork"], }), - peersBySyncCommitteeSubnet: register.gauge<"subnet" | "fork">({ + peersBySyncCommitteeSubnet: register.gauge<{subnet: number; fork: ForkName}>({ name: "lodestar_gossip_mesh_peers_by_sync_committee_subnet_count", help: "Number of connected mesh peers per sync committee subnet", labelNames: ["subnet", "fork"], }), }, gossipTopic: { - peersByType: register.gauge<"type" | "fork">({ + peersByType: register.gauge<{type: GossipType; fork: ForkName}>({ name: "lodestar_gossip_topic_peers_by_type_count", help: "Number of connected topic peers per gossip type", labelNames: ["type", "fork"], }), - peersByBeaconAttestationSubnet: register.gauge<"subnet" | "fork">({ + peersByBeaconAttestationSubnet: register.gauge<{subnet: string; fork: ForkName}>({ name: "lodestar_gossip_topic_peers_by_beacon_attestation_subnet_count", help: "Number of connected topic peers per beacon attestation subnet", labelNames: ["subnet", "fork"], }), - peersBySyncCommitteeSubnet: register.gauge<"subnet" | "fork">({ + peersBySyncCommitteeSubnet: register.gauge<{subnet: number; fork: ForkName}>({ name: "lodestar_gossip_topic_peers_by_sync_committee_subnet_count", help: "Number of connected topic peers per sync committee subnet", labelNames: ["subnet", "fork"], diff --git a/packages/beacon-node/src/network/peers/discover.ts b/packages/beacon-node/src/network/peers/discover.ts index 2090e8bedab6..2805f67b4763 100644 --- a/packages/beacon-node/src/network/peers/discover.ts +++ b/packages/beacon-node/src/network/peers/discover.ts @@ -43,7 +43,7 @@ enum QueryStatusCode { } type QueryStatus = {code: QueryStatusCode.NotActive} | {code: QueryStatusCode.Active; count: number}; -enum DiscoveredPeerStatus { +export enum DiscoveredPeerStatus { bad_score = "bad_score", already_connected = "already_connected", already_dialing = "already_dialing", diff --git a/packages/beacon-node/src/network/processor/index.ts b/packages/beacon-node/src/network/processor/index.ts index 1d1fd82a4522..3d067c626f76 100644 --- a/packages/beacon-node/src/network/processor/index.ts +++ b/packages/beacon-node/src/network/processor/index.ts @@ -93,7 +93,7 @@ const PROCESS_UNKNOWN_BLOCK_GOSSIP_OBJECTS_YIELD_EVERY_MS = 50; /** * Reprocess reject reason for metrics */ -enum ReprocessRejectReason { +export enum ReprocessRejectReason { /** * There are too many attestations that have unknown block root. */ @@ -107,9 +107,9 @@ enum ReprocessRejectReason { /** * Cannot accept work reason for metrics */ -enum CannotAcceptWorkReason { +export enum CannotAcceptWorkReason { /** - * Validating or procesing gossip block at current slot. + * Validating or processing gossip block at current slot. */ processingCurrentSlotBlock = "processing_current_slot_block", /** @@ -344,7 +344,10 @@ export class NetworkProcessor { for (const gossipMessages of gossipMessagesByRoot.values()) { for (const message of gossipMessages) { this.metrics?.reprocessGossipAttestations.reject.inc({reason: ReprocessRejectReason.expired}); - this.metrics?.reprocessGossipAttestations.waitSecBeforeReject.set(nowSec - message.seenTimestampSec); + this.metrics?.reprocessGossipAttestations.waitSecBeforeReject.set( + {reason: ReprocessRejectReason.expired}, + nowSec - message.seenTimestampSec + ); // TODO: Should report the dropped job to gossip? It will be eventually pruned from the mcache } } diff --git a/packages/beacon-node/src/network/subnets/attnetsService.ts b/packages/beacon-node/src/network/subnets/attnetsService.ts index d76e56677ac6..7eabc2e4114c 100644 --- a/packages/beacon-node/src/network/subnets/attnetsService.ts +++ b/packages/beacon-node/src/network/subnets/attnetsService.ts @@ -34,7 +34,7 @@ const LAST_SEEN_VALIDATOR_TIMEOUT = 150; const gossipType = GossipType.beacon_attestation; -enum SubnetSource { +export enum SubnetSource { committee = "committee", random = "random", } diff --git a/packages/beacon-node/src/network/subnets/dllAttnetsService.ts b/packages/beacon-node/src/network/subnets/dllAttnetsService.ts index f7ae0e8d09c2..7236695cb11a 100644 --- a/packages/beacon-node/src/network/subnets/dllAttnetsService.ts +++ b/packages/beacon-node/src/network/subnets/dllAttnetsService.ts @@ -20,7 +20,7 @@ import {computeSubscribedSubnet} from "./util.js"; const gossipType = GossipType.beacon_attestation; -enum SubnetSource { +export enum DLLSubnetSource { committee = "committee", longLived = "long_lived", } @@ -179,7 +179,7 @@ export class DLLAttnetsService implements IAttnetsService { if (dutiedSlot === clockSlot + this.opts.slotsToSubscribeBeforeAggregatorDuty) { // Trigger gossip subscription first, in batch if (dutiedInfo.size > 0) { - this.subscribeToSubnets(Array.from(dutiedInfo.keys()), SubnetSource.committee); + this.subscribeToSubnets(Array.from(dutiedInfo.keys()), DLLSubnetSource.committee); } // Then, register the subscriptions for (const subnet of dutiedInfo.keys()) { @@ -276,7 +276,7 @@ export class DLLAttnetsService implements IAttnetsService { } // First, tell gossip to subscribe to the subnets if not connected already - this.subscribeToSubnets(newSubnets, SubnetSource.longLived); + this.subscribeToSubnets(newSubnets, DLLSubnetSource.longLived); // then update longLivedSubscriptions for (const subnet of toRemoveSubnets) { @@ -289,7 +289,7 @@ export class DLLAttnetsService implements IAttnetsService { } // Only tell gossip to unsubsribe last, longLivedSubscriptions has the latest state - this.unsubscribeSubnets(toRemoveSubnets, this.clock.currentSlot, SubnetSource.longLived); + this.unsubscribeSubnets(toRemoveSubnets, this.clock.currentSlot, DLLSubnetSource.longLived); this.updateMetadata(); } @@ -300,7 +300,7 @@ export class DLLAttnetsService implements IAttnetsService { private unsubscribeExpiredCommitteeSubnets(slot: Slot): void { const expired = this.shortLivedSubscriptions.getExpired(slot); if (expired.length > 0) { - this.unsubscribeSubnets(expired, slot, SubnetSource.committee); + this.unsubscribeSubnets(expired, slot, DLLSubnetSource.committee); } } @@ -333,7 +333,7 @@ export class DLLAttnetsService implements IAttnetsService { * Trigger a gossip subcription only if not already subscribed * shortLivedSubscriptions or longLivedSubscriptions should be updated right AFTER this called **/ - private subscribeToSubnets(subnets: number[], src: SubnetSource): void { + private subscribeToSubnets(subnets: number[], src: DLLSubnetSource): void { const forks = getActiveForks(this.config, this.clock.currentEpoch); for (const subnet of subnets) { if (!this.shortLivedSubscriptions.has(subnet) && !this.longLivedSubscriptions.has(subnet)) { @@ -349,7 +349,7 @@ export class DLLAttnetsService implements IAttnetsService { * Trigger a gossip un-subscription only if no-one is still subscribed * If unsubscribe long lived subnets, longLivedSubscriptions should be updated right BEFORE this called **/ - private unsubscribeSubnets(subnets: number[], slot: Slot, src: SubnetSource): void { + private unsubscribeSubnets(subnets: number[], slot: Slot, src: DLLSubnetSource): void { // No need to unsubscribeTopic(). Return early to prevent repetitive extra work if (this.opts.subscribeAllSubnets) return; diff --git a/packages/beacon-node/src/util/queue/options.ts b/packages/beacon-node/src/util/queue/options.ts index c3846cd8be1f..e55d413088e3 100644 --- a/packages/beacon-node/src/util/queue/options.ts +++ b/packages/beacon-node/src/util/queue/options.ts @@ -1,4 +1,4 @@ -import {IGauge, IHistogram} from "../../metrics/index.js"; +import {Gauge, GaugeExtra, Histogram} from "@lodestar/utils"; export enum QueueType { FIFO = "FIFO", @@ -19,12 +19,12 @@ export type JobQueueOpts = { }; export type QueueMetrics = { - length: IGauge; - droppedJobs: IGauge; + length: GaugeExtra; + droppedJobs: Gauge; /** Compute async utilization rate with `rate(metrics_name[1m])` */ - jobTime: IHistogram; - jobWaitTime: IHistogram; - concurrency: IGauge; + jobTime: Histogram; + jobWaitTime: Histogram; + concurrency: Gauge; }; export const defaultQueueOpts: Required< diff --git a/packages/beacon-node/test/unit/monitoring/properties.test.ts b/packages/beacon-node/test/unit/monitoring/properties.test.ts index 639161eefc9e..1a2e2c58377a 100644 --- a/packages/beacon-node/test/unit/monitoring/properties.test.ts +++ b/packages/beacon-node/test/unit/monitoring/properties.test.ts @@ -91,7 +91,11 @@ describe("monitoring / properties", () => { const labelValue = "test_label_value"; const metricValue = 10; - const metric = metrics.register.gauge({name: metricName, help: "withLabel test", labelNames: [labelName]}); + const metric = metrics.register.gauge<{[labelName]: string}>({ + name: metricName, + help: "withLabel test", + labelNames: [labelName], + }); metric.set({[labelName]: "different_value"}, metricValue + 1); metric.set({[labelName]: labelValue}, metricValue); diff --git a/packages/beacon-node/test/unit/monitoring/service.test.ts b/packages/beacon-node/test/unit/monitoring/service.test.ts index 27b8ca35c307..068f35634f81 100644 --- a/packages/beacon-node/test/unit/monitoring/service.test.ts +++ b/packages/beacon-node/test/unit/monitoring/service.test.ts @@ -1,7 +1,7 @@ import {describe, it, expect, beforeEach, beforeAll, afterAll, vi, afterEach, MockInstance} from "vitest"; +import {Histogram} from "prom-client"; import {ErrorAborted, TimeoutError} from "@lodestar/utils"; import {RegistryMetricCreator} from "../../../src/index.js"; -import {HistogramExtra} from "../../../src/metrics/utils/histogram.js"; import {MonitoringService} from "../../../src/monitoring/service.js"; import {MonitoringOptions} from "../../../src/monitoring/options.js"; import {sleep} from "../../utils/sleep.js"; @@ -41,8 +41,8 @@ describe("monitoring / service", () => { it("should register metrics for collecting and sending data", () => { service = new MonitoringService("beacon", {endpoint}, {register, logger}); - expect(register.getSingleMetric("lodestar_monitoring_collect_data_seconds")).toBeInstanceOf(HistogramExtra); - expect(register.getSingleMetric("lodestar_monitoring_send_data_seconds")).toBeInstanceOf(HistogramExtra); + expect(register.getSingleMetric("lodestar_monitoring_collect_data_seconds")).toBeInstanceOf(Histogram); + expect(register.getSingleMetric("lodestar_monitoring_send_data_seconds")).toBeInstanceOf(Histogram); }); it("should log a warning message if insecure monitoring endpoint is provided ", () => { diff --git a/packages/cli/package.json b/packages/cli/package.json index 4162cbbfb2f6..d566258e7f5d 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -88,7 +88,7 @@ "js-yaml": "^4.1.0", "lockfile": "^1.0.4", "lodash": "^4.17.21", - "prom-client": "^14.2.0", + "prom-client": "^15.1.0", "rimraf": "^4.4.1", "source-map-support": "^0.5.21", "uint8arrays": "^4.0.3", diff --git a/packages/cli/src/cmds/bootnode/handler.ts b/packages/cli/src/cmds/bootnode/handler.ts index be639eb1bf4b..7bf9169cfdc6 100644 --- a/packages/cli/src/cmds/bootnode/handler.ts +++ b/packages/cli/src/cmds/bootnode/handler.ts @@ -1,6 +1,6 @@ import path from "node:path"; import {Multiaddr, multiaddr} from "@multiformats/multiaddr"; -import {Discv5, ENR} from "@chainsafe/discv5"; +import {Discv5, ENR, IDiscv5CreateOptions} from "@chainsafe/discv5"; import {ErrorAborted} from "@lodestar/utils"; import {HttpMetricsServer, RegistryMetricCreator, getHttpMetricsServer} from "@lodestar/beacon-node"; @@ -58,7 +58,7 @@ export async function bootnodeHandler(args: BootnodeArgs & GlobalArgs): Promise< ip6: bindAddrs.ip6 ? multiaddr(bindAddrs.ip6) : undefined, }, config: {enrUpdate: !enr.ip && !enr.ip6}, - metricsRegistry, + metricsRegistry: metricsRegistry as IDiscv5CreateOptions["metricsRegistry"], }); // If there are any bootnodes, add them to the routing table diff --git a/packages/cli/src/cmds/validator/handler.ts b/packages/cli/src/cmds/validator/handler.ts index 7713bf5a6dd1..69e4610bff0e 100644 --- a/packages/cli/src/cmds/validator/handler.ts +++ b/packages/cli/src/cmds/validator/handler.ts @@ -9,7 +9,7 @@ import { defaultOptions, } from "@lodestar/validator"; import {routes} from "@lodestar/api"; -import {getMetrics, MetricsRegister} from "@lodestar/validator"; +import {getMetrics} from "@lodestar/validator"; import { RegistryMetricCreator, collectNodeJSMetrics, @@ -112,7 +112,7 @@ export async function validatorHandler(args: IValidatorCliArgs & GlobalArgs): Pr // Send version and network data for static registries const register = args["metrics"] || args["monitoring.endpoint"] ? new RegistryMetricCreator() : null; - const metrics = register && getMetrics(register as unknown as MetricsRegister, {version, commit, network}); + const metrics = register && getMetrics(register, {version, commit, network}); // Start metrics server if metrics are enabled. // Collect NodeJS metrics defined in the Lodestar repo diff --git a/packages/db/src/controller/metrics.ts b/packages/db/src/controller/metrics.ts index b4b8a0bf0963..4827d6fb4515 100644 --- a/packages/db/src/controller/metrics.ts +++ b/packages/db/src/controller/metrics.ts @@ -1,26 +1,10 @@ +import {Counter, Gauge, Histogram} from "@lodestar/utils"; + export type LevelDbControllerMetrics = { - dbReadReq: Counter<"bucket">; - dbReadItems: Counter<"bucket">; - dbWriteReq: Counter<"bucket">; - dbWriteItems: Counter<"bucket">; + dbReadReq: Counter<{bucket: string}>; + dbReadItems: Counter<{bucket: string}>; + dbWriteReq: Counter<{bucket: string}>; + dbWriteItems: Counter<{bucket: string}>; dbSizeTotal: Gauge; dbApproximateSizeTime: Histogram; }; - -type Labels = Partial>; - -interface Counter { - inc(value?: number): void; - inc(labels: Labels, value?: number): void; - inc(arg1?: Labels | number, arg2?: number): void; -} - -interface Gauge { - set(value: number): void; - set(labels: Labels, value: number): void; - set(arg1?: Labels | number, arg2?: number): void; -} - -interface Histogram { - startTimer(): () => number; -} diff --git a/packages/reqresp/src/ReqResp.ts b/packages/reqresp/src/ReqResp.ts index e79b5737bc91..671df3c83662 100644 --- a/packages/reqresp/src/ReqResp.ts +++ b/packages/reqresp/src/ReqResp.ts @@ -2,8 +2,8 @@ import {setMaxListeners} from "node:events"; import {Connection, Stream} from "@libp2p/interface/connection"; import {PeerId} from "@libp2p/interface/peer-id"; import type {Libp2p} from "libp2p"; -import {Logger} from "@lodestar/utils"; -import {getMetrics, Metrics, MetricsRegister} from "./metrics.js"; +import {Logger, MetricsRegister} from "@lodestar/utils"; +import {getMetrics, Metrics} from "./metrics.js"; import {RequestError, RequestErrorCode, sendRequest, SendRequestOpts} from "./request/index.js"; import {handleRequest} from "./response/index.js"; import { diff --git a/packages/reqresp/src/index.ts b/packages/reqresp/src/index.ts index 9bb07c1a4fce..d31960fdcd89 100644 --- a/packages/reqresp/src/index.ts +++ b/packages/reqresp/src/index.ts @@ -1,7 +1,7 @@ export {ReqResp} from "./ReqResp.js"; export type {ReqRespOpts} from "./ReqResp.js"; export {getMetrics} from "./metrics.js"; -export type {Metrics, MetricsRegister} from "./metrics.js"; +export type {Metrics} from "./metrics.js"; export {Encoding as ReqRespEncoding} from "./types.js"; // Expose enums renamed export * from "./types.js"; export * from "./interface.js"; diff --git a/packages/reqresp/src/metrics.ts b/packages/reqresp/src/metrics.ts index c4474d0d61b7..4af18a782322 100644 --- a/packages/reqresp/src/metrics.ts +++ b/packages/reqresp/src/metrics.ts @@ -1,62 +1,7 @@ -type LabelValues = Partial>; - -interface Gauge { - // Sorry for this mess, `prom-client` API choices are not great - // If the function signature was `inc(value: number, labels?: Labels)`, this would be simpler - inc(value?: number): void; - inc(labels: LabelValues, value?: number): void; - inc(arg1?: LabelValues | number, arg2?: number): void; - - dec(value?: number): void; - dec(labels: LabelValues, value?: number): void; - dec(arg1?: LabelValues | number, arg2?: number): void; - - set(value: number): void; - set(labels: LabelValues, value: number): void; - set(arg1?: LabelValues | number, arg2?: number): void; - - addCollect: (collectFn: () => void) => void; -} - -interface Histogram { - startTimer(arg1?: LabelValues): (labels?: LabelValues) => number; - - observe(value: number): void; - observe(labels: LabelValues, values: number): void; - observe(arg1: LabelValues | number, arg2?: number): void; - - reset(): void; -} - -type GaugeConfig = { - name: string; - help: string; - labelNames?: T[]; -}; - -type HistogramConfig = { - name: string; - help: string; - labelNames?: T[]; - buckets?: number[]; -}; - -export interface MetricsRegister { - gauge(config: GaugeConfig): Gauge; - histogram(config: HistogramConfig): Histogram; -} +import {MetricsRegister} from "@lodestar/utils"; export type Metrics = ReturnType; -export type LodestarGitData = { - /** "0.16.0 developer/feature-1 ac99f2b5" */ - version: string; - /** "4f816b16dfde718e2d74f95f2c8292596138c248" */ - commit: string; - /** "goerli" */ - network: string; -}; - /** * A collection of metrics used throughout the Gossipsub behaviour. */ @@ -65,48 +10,48 @@ export function getMetrics(register: MetricsRegister) { // Using function style instead of class to prevent having to re-declare all MetricsPrometheus types. return { - outgoingRequests: register.gauge<"method">({ + outgoingRequests: register.gauge<{method: string}>({ name: "beacon_reqresp_outgoing_requests_total", help: "Counts total requests done per method", labelNames: ["method"], }), - outgoingRequestRoundtripTime: register.histogram<"method">({ + outgoingRequestRoundtripTime: register.histogram<{method: string}>({ name: "beacon_reqresp_outgoing_request_roundtrip_time_seconds", help: "Histogram of outgoing requests round-trip time", labelNames: ["method"], // Spec sets RESP_TIMEOUT = 10 sec buckets: [0.1, 0.2, 0.5, 1, 5, 10, 15, 60], }), - outgoingErrors: register.gauge<"method">({ + outgoingErrors: register.gauge<{method: string}>({ name: "beacon_reqresp_outgoing_requests_error_total", help: "Counts total failed requests done per method", labelNames: ["method"], }), - incomingRequests: register.gauge<"method">({ + incomingRequests: register.gauge<{method: string}>({ name: "beacon_reqresp_incoming_requests_total", help: "Counts total responses handled per method", labelNames: ["method"], }), - incomingRequestHandlerTime: register.histogram<"method">({ + incomingRequestHandlerTime: register.histogram<{method: string}>({ name: "beacon_reqresp_incoming_request_handler_time_seconds", help: "Histogram of incoming requests internal handling time", labelNames: ["method"], // Spec sets RESP_TIMEOUT = 10 sec buckets: [0.1, 0.2, 0.5, 1, 5, 10], }), - incomingErrors: register.gauge<"method">({ + incomingErrors: register.gauge<{method: string}>({ name: "beacon_reqresp_incoming_requests_error_total", help: "Counts total failed responses handled per method", labelNames: ["method"], }), - outgoingResponseTTFB: register.histogram<"method">({ + outgoingResponseTTFB: register.histogram<{method: string}>({ name: "beacon_reqresp_outgoing_response_ttfb_seconds", help: "Time to first byte (TTFB) for outgoing responses", labelNames: ["method"], // Spec sets TTFB_TIMEOUT = 5 sec buckets: [0.1, 1, 5], }), - incomingResponseTTFB: register.histogram<"method">({ + incomingResponseTTFB: register.histogram<{method: string}>({ name: "beacon_reqresp_incoming_response_ttfb_seconds", help: "Time to first byte (TTFB) for incoming responses", labelNames: ["method"], diff --git a/packages/state-transition/src/epoch/index.ts b/packages/state-transition/src/epoch/index.ts index 05c8b55d0435..b55ebe291fb9 100644 --- a/packages/state-transition/src/epoch/index.ts +++ b/packages/state-transition/src/epoch/index.ts @@ -51,6 +51,22 @@ export {computeUnrealizedCheckpoints} from "./computeUnrealizedCheckpoints.js"; const maxValidatorsPerStateSlashing = SLOTS_PER_EPOCH * MAX_ATTESTER_SLASHINGS * MAX_VALIDATORS_PER_COMMITTEE; const maxSafeValidators = Math.floor(Number.MAX_SAFE_INTEGER / MAX_EFFECTIVE_BALANCE); +/** + * Epoch transition steps tracked in metrics + */ +export enum EpochTransitionStep { + beforeProcessEpoch = "beforeProcessEpoch", + afterProcessEpoch = "afterProcessEpoch", + processJustificationAndFinalization = "processJustificationAndFinalization", + processInactivityUpdates = "processInactivityUpdates", + processRegistryUpdates = "processRegistryUpdates", + processSlashings = "processSlashings", + processRewardsAndPenalties = "processRewardsAndPenalties", + processEffectiveBalanceUpdates = "processEffectiveBalanceUpdates", + processParticipationFlagUpdates = "processParticipationFlagUpdates", + processSyncCommitteeUpdates = "processSyncCommitteeUpdates", +} + export function processEpoch( fork: ForkSeq, state: CachedBeaconStateAllForks, @@ -67,14 +83,14 @@ export function processEpoch( { const timer = metrics?.epochTransitionStepTime.startTimer({ - step: "processJustificationAndFinalization", + step: EpochTransitionStep.processJustificationAndFinalization, }); processJustificationAndFinalization(state, cache); timer?.(); } if (fork >= ForkSeq.altair) { - const timer = metrics?.epochTransitionStepTime.startTimer({step: "processInactivityUpdates"}); + const timer = metrics?.epochTransitionStepTime.startTimer({step: EpochTransitionStep.processInactivityUpdates}); processInactivityUpdates(state as CachedBeaconStateAltair, cache); timer?.(); } @@ -83,7 +99,7 @@ export function processEpoch( // after processSlashings() to update balances only once // processRewardsAndPenalties(state, cache); { - const timer = metrics?.epochTransitionStepTime.startTimer({step: "processRegistryUpdates"}); + const timer = metrics?.epochTransitionStepTime.startTimer({step: EpochTransitionStep.processRegistryUpdates}); processRegistryUpdates(state, cache); timer?.(); } @@ -91,13 +107,13 @@ export function processEpoch( // accumulate slashing penalties and only update balances once in processRewardsAndPenalties() let slashingPenalties: number[]; { - const timer = metrics?.epochTransitionStepTime.startTimer({step: "processSlashings"}); + const timer = metrics?.epochTransitionStepTime.startTimer({step: EpochTransitionStep.processSlashings}); slashingPenalties = processSlashings(state, cache, false); timer?.(); } { - const timer = metrics?.epochTransitionStepTime.startTimer({step: "processRewardsAndPenalties"}); + const timer = metrics?.epochTransitionStepTime.startTimer({step: EpochTransitionStep.processRewardsAndPenalties}); processRewardsAndPenalties(state, cache, slashingPenalties); timer?.(); } @@ -106,7 +122,7 @@ export function processEpoch( { const timer = metrics?.epochTransitionStepTime.startTimer({ - step: "processEffectiveBalanceUpdates", + step: EpochTransitionStep.processEffectiveBalanceUpdates, }); processEffectiveBalanceUpdates(state, cache); timer?.(); @@ -126,7 +142,7 @@ export function processEpoch( } else { { const timer = metrics?.epochTransitionStepTime.startTimer({ - step: "processParticipationFlagUpdates", + step: EpochTransitionStep.processParticipationFlagUpdates, }); processParticipationFlagUpdates(state as CachedBeaconStateAltair); timer?.(); @@ -134,7 +150,7 @@ export function processEpoch( { const timer = metrics?.epochTransitionStepTime.startTimer({ - step: "processSyncCommitteeUpdates", + step: EpochTransitionStep.processSyncCommitteeUpdates, }); processSyncCommitteeUpdates(state as CachedBeaconStateAltair); timer?.(); diff --git a/packages/state-transition/src/index.ts b/packages/state-transition/src/index.ts index 3cb93803447f..b8506ab44309 100644 --- a/packages/state-transition/src/index.ts +++ b/packages/state-transition/src/index.ts @@ -2,6 +2,7 @@ export * from "./stateTransition.js"; export * from "./constants/index.js"; export * from "./util/index.js"; export * from "./signatureSets/index.js"; +export type {EpochTransitionStep} from "./epoch/index.js"; export type {BeaconStateTransitionMetrics} from "./metrics.js"; export type { diff --git a/packages/state-transition/src/metrics.ts b/packages/state-transition/src/metrics.ts index 681bb2b910cf..62062bbfc539 100644 --- a/packages/state-transition/src/metrics.ts +++ b/packages/state-transition/src/metrics.ts @@ -1,18 +1,21 @@ import {Epoch} from "@lodestar/types"; +import {Gauge, Histogram} from "@lodestar/utils"; import {CachedBeaconStateAllForks} from "./types.js"; +import {StateCloneSource, StateHashTreeRootSource} from "./stateTransition.js"; import {AttesterStatus} from "./util/attesterStatus.js"; +import {EpochTransitionStep} from "./epoch/index.js"; export type BeaconStateTransitionMetrics = { epochTransitionTime: Histogram; epochTransitionCommitTime: Histogram; - epochTransitionStepTime: Histogram<"step">; + epochTransitionStepTime: Histogram<{step: EpochTransitionStep}>; processBlockTime: Histogram; processBlockCommitTime: Histogram; - stateHashTreeRootTime: Histogram; - preStateBalancesNodesPopulatedMiss: Gauge<"source">; - preStateBalancesNodesPopulatedHit: Gauge<"source">; - preStateValidatorsNodesPopulatedMiss: Gauge<"source">; - preStateValidatorsNodesPopulatedHit: Gauge<"source">; + stateHashTreeRootTime: Histogram<{source: StateHashTreeRootSource}>; + preStateBalancesNodesPopulatedMiss: Gauge<{source: StateCloneSource}>; + preStateBalancesNodesPopulatedHit: Gauge<{source: StateCloneSource}>; + preStateValidatorsNodesPopulatedMiss: Gauge<{source: StateCloneSource}>; + preStateValidatorsNodesPopulatedHit: Gauge<{source: StateCloneSource}>; preStateClonedCount: Histogram; postStateBalancesNodesPopulatedMiss: Gauge; postStateBalancesNodesPopulatedHit: Gauge; @@ -21,26 +24,10 @@ export type BeaconStateTransitionMetrics = { registerValidatorStatuses: (currentEpoch: Epoch, statuses: AttesterStatus[], balances?: number[]) => void; }; -type LabelValues = Partial>; - -interface Histogram { - startTimer(labels?: LabelValues): (labels?: LabelValues) => number; - - observe(value: number): void; - observe(labels: LabelValues, values: number): void; - observe(arg1: LabelValues | number, arg2?: number): void; -} - -interface Gauge { - inc(value?: number): void; - inc(labels: LabelValues, value?: number): void; - inc(arg1?: LabelValues | number, arg2?: number): void; -} - export function onStateCloneMetrics( state: CachedBeaconStateAllForks, metrics: BeaconStateTransitionMetrics, - source: "stateTransition" | "processSlots" + source: StateCloneSource ): void { metrics.preStateClonedCount.observe(state.clonedCount); diff --git a/packages/state-transition/src/stateTransition.ts b/packages/state-transition/src/stateTransition.ts index cdb8878c87fa..b3f3b41eb865 100644 --- a/packages/state-transition/src/stateTransition.ts +++ b/packages/state-transition/src/stateTransition.ts @@ -20,7 +20,7 @@ import { upgradeStateToDeneb, } from "./slot/index.js"; import {processBlock} from "./block/index.js"; -import {processEpoch} from "./epoch/index.js"; +import {EpochTransitionStep, processEpoch} from "./epoch/index.js"; import {BlockExternalData, DataAvailableStatus, ExecutionPayloadStatus} from "./block/externalData.js"; import {ProcessBlockOpts} from "./block/types.js"; @@ -36,6 +36,24 @@ export type StateTransitionOpts = BlockExternalData & dontTransferCache?: boolean; }; +/** + * `state.clone()` invocation source tracked in metrics + */ +export enum StateCloneSource { + stateTransition = "stateTransition", + processSlots = "processSlots", +} + +/** + * `state.hashTreeRoot()` invocation source tracked in metrics + */ +export enum StateHashTreeRootSource { + stateTransition = "state_transition", + blockTransition = "block_transition", + prepareNextSlot = "prepare_next_slot", + computeNewStateRoot = "compute_new_state_root", +} + /** * Implementation Note: follows the optimizations in protolambda's eth2fastspec (https://github.com/protolambda/eth2fastspec) */ @@ -58,7 +76,7 @@ export function stateTransition( let postState = state.clone(options.dontTransferCache); if (metrics) { - onStateCloneMetrics(postState, metrics, "stateTransition"); + onStateCloneMetrics(postState, metrics, StateCloneSource.stateTransition); } // State is already a ViewDU, which won't commit changes. Equivalent to .setStateCachesAsTransient() @@ -96,7 +114,9 @@ export function stateTransition( // Verify state root if (verifyStateRoot) { - const hashTreeRootTimer = metrics?.stateHashTreeRootTime.startTimer(); + const hashTreeRootTimer = metrics?.stateHashTreeRootTime.startTimer({ + source: StateHashTreeRootSource.stateTransition, + }); const stateRoot = postState.hashTreeRoot(); hashTreeRootTimer?.(); @@ -127,7 +147,7 @@ export function processSlots( let postState = state.clone(epochTransitionCacheOpts?.dontTransferCache); if (metrics) { - onStateCloneMetrics(postState, metrics, "processSlots"); + onStateCloneMetrics(postState, metrics, StateCloneSource.processSlots); } // State is already a ViewDU, which won't commit changes. Equivalent to .setStateCachesAsTransient() @@ -167,7 +187,7 @@ function processSlotsWithTransientCache( let epochTransitionCache: EpochTransitionCache; { - const timer = metrics?.epochTransitionStepTime.startTimer({step: "beforeProcessEpoch"}); + const timer = metrics?.epochTransitionStepTime.startTimer({step: EpochTransitionStep.beforeProcessEpoch}); epochTransitionCache = beforeProcessEpoch(postState, epochTransitionCacheOpts); timer?.(); } @@ -180,7 +200,7 @@ function processSlotsWithTransientCache( postState.slot++; { - const timer = metrics?.epochTransitionStepTime.startTimer({step: "afterProcessEpoch"}); + const timer = metrics?.epochTransitionStepTime.startTimer({step: EpochTransitionStep.afterProcessEpoch}); postState.epochCtx.afterProcessEpoch(postState, epochTransitionCache); timer?.(); } diff --git a/packages/utils/package.json b/packages/utils/package.json index 43daecc2d71c..8137cd496a0a 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -27,7 +27,7 @@ "build:watch": "yarn run build --watch", "build:release": "yarn clean && yarn build", "check-build": "node -e \"(async function() { await import('./lib/index.js') })()\"", - "check-types": "tsc", + "check-types": "tsc && vitest --run --typecheck --dir test/types/", "lint": "eslint --color --ext .ts src/ test/", "lint:fix": "yarn run lint --fix", "pretest": "yarn run check-types", @@ -50,6 +50,7 @@ "devDependencies": { "@types/js-yaml": "^4.0.5", "@types/triple-beam": "^1.3.2", + "prom-client": "^15.1.0", "triple-beam": "^1.3.0" }, "keywords": [ diff --git a/packages/utils/src/index.ts b/packages/utils/src/index.ts index 637b965af682..fcff789f9c56 100644 --- a/packages/utils/src/index.ts +++ b/packages/utils/src/index.ts @@ -8,6 +8,7 @@ export * from "./format.js"; export * from "./logger.js"; export * from "./map.js"; export * from "./math.js"; +export * from "./metrics.js"; export * from "./objects.js"; export {retry, type RetryOptions} from "./retry.js"; export * from "./notNullish.js"; diff --git a/packages/utils/src/metrics.ts b/packages/utils/src/metrics.ts new file mode 100644 index 000000000000..a25518280ee1 --- /dev/null +++ b/packages/utils/src/metrics.ts @@ -0,0 +1,71 @@ +export type NoLabels = Record; +export type LabelsGeneric = Record; +export type LabelKeys = Extract; +export type CollectFn = (metric: Gauge) => void; + +export interface Gauge { + inc: NoLabels extends Labels ? (value?: number) => void : (labels: Labels, value?: number) => void; + dec: NoLabels extends Labels ? (value?: number) => void : (labels: Labels, value?: number) => void; + set: NoLabels extends Labels ? (value: number) => void : (labels: Labels, value: number) => void; + + collect?(): void; +} + +export interface GaugeExtra extends Omit, "collect"> { + addCollect(collectFn: CollectFn): void; +} + +export interface Histogram { + startTimer(): NoLabels extends Labels ? () => number : (labels: Labels) => number; + startTimer>( + labels?: NoLabels extends Labels ? never : L + ): keyof Omit extends never ? () => number : (labels: Omit) => number; + + observe: NoLabels extends Labels ? (value: number) => void : (labels: Labels, value: number) => void; + + reset(): void; +} + +export interface AvgMinMax { + addGetValuesFn(getValuesFn: () => number[]): void; + + set: NoLabels extends Labels ? (values: number[]) => void : (labels: Labels, values: number[]) => void; +} + +export interface Counter { + inc: NoLabels extends Labels ? (value?: number) => void : (labels: Labels, value?: number) => void; +} + +export type GaugeConfig = { + name: string; + help: string; +} & (NoLabels extends Labels ? {labelNames?: never} : {labelNames: [LabelKeys, ...LabelKeys[]]}); + +export type HistogramConfig = GaugeConfig & { + buckets?: number[]; +}; + +export type AvgMinMaxConfig = GaugeConfig; + +export type CounterConfig = GaugeConfig; + +export type StaticConfig = { + name: GaugeConfig["name"]; + help: GaugeConfig["help"]; + value: Record, string>; +}; + +export interface MetricsRegister { + gauge(config: GaugeConfig): Gauge; + histogram(config: HistogramConfig): Histogram; + counter(config: CounterConfig): Counter; +} + +export interface MetricsRegisterExtra extends MetricsRegister { + gauge(config: GaugeConfig): GaugeExtra; +} + +export interface MetricsRegisterCustom extends MetricsRegisterExtra { + avgMinMax(config: AvgMinMaxConfig): AvgMinMax; + static(config: StaticConfig): void; +} diff --git a/packages/utils/test/types/metrics.test-d.ts b/packages/utils/test/types/metrics.test-d.ts new file mode 100644 index 000000000000..2f008618e648 --- /dev/null +++ b/packages/utils/test/types/metrics.test-d.ts @@ -0,0 +1,114 @@ +import {describe, it, expectTypeOf} from "vitest"; +import {Counter as PromCounter, Gauge as PromGauge, Histogram as PromHistogram} from "prom-client"; +import {Counter, Gauge, Histogram, MetricsRegister} from "../../src/metrics.js"; + +describe("Metric types", () => { + type Labels = {label: string}; + type MultipleLabels = {label1: string; label2: string}; + + describe("MetricsRegister", () => { + const register = {} as MetricsRegister; + + it("should require name and help to be defined on each metric", () => { + expectTypeOf(register.gauge).parameter(0).toHaveProperty("name").toBeString(); + expectTypeOf(register.gauge).parameter(0).toHaveProperty("help").toBeString(); + }); + + it("should require to set labelNames if metric has defined labels", () => { + expectTypeOf(register.gauge) + .parameter(0) + .toHaveProperty("labelNames") + .toMatchTypeOf<"label"[]>(); + + expectTypeOf(register.gauge) + .parameter(0) + .toHaveProperty("labelNames") + .toMatchTypeOf<("label1" | "label2")[]>(); + }); + + it("should not require to set labelNames if metric has no labels", () => { + expectTypeOf(register.gauge).parameter(0).toHaveProperty("labelNames").toEqualTypeOf(); + }); + }); + + describe("Gauge", () => { + it("should be compatible with prom-client type", () => { + expectTypeOf().toMatchTypeOf(); + }); + + it("should require to set labels if metric has defined labels", () => { + const gauge = {} as Gauge; + + expectTypeOf(gauge.inc).toEqualTypeOf<(labels: Labels, value?: number | undefined) => void>(); + expectTypeOf(gauge.dec).toEqualTypeOf<(labels: Labels, value?: number | undefined) => void>(); + expectTypeOf(gauge.set).toEqualTypeOf<(labels: Labels, value: number) => void>(); + }); + + it("should not require to set labels if metric has no labels", () => { + const gauge = {} as Gauge; + + expectTypeOf(gauge.inc).toEqualTypeOf<(value?: number | undefined) => void>(); + expectTypeOf(gauge.dec).toEqualTypeOf<(value?: number | undefined) => void>(); + expectTypeOf(gauge.set).toEqualTypeOf<(value: number) => void>(); + }); + }); + + describe("Histogram", () => { + it("should be compatible with prom-client type", () => { + expectTypeOf().toMatchTypeOf(); + }); + + it("should require to set labels if metric has defined labels", () => { + const histogram = {} as Histogram; + + expectTypeOf(histogram.startTimer).toMatchTypeOf<(labels: Labels) => () => number>(); + expectTypeOf(histogram.observe).toEqualTypeOf<(labels: Labels, value: number) => void>(); + }); + + it("should require to set labels in timer if not set in startTimer", () => { + const histogram = {} as Histogram; + + const timer = histogram.startTimer(); + expectTypeOf(timer).toEqualTypeOf<(labels: MultipleLabels) => number>(); + }); + + it("should not require to set labels in timer if already set in startTimer", () => { + const histogram = {} as Histogram; + + const timer = histogram.startTimer({label1: "value1", label2: "label2"}); + expectTypeOf(timer).toEqualTypeOf<() => number>(); + }); + + it("should allow to set labels in either startTimer or timer", () => { + const histogram = {} as Histogram; + + const timer = histogram.startTimer({label1: "value1"}); + expectTypeOf(timer).toEqualTypeOf<(labels: {label2: string}) => number>(); + }); + + it("should not require to set labels if metric has no labels", () => { + const histogram = {} as Histogram; + + expectTypeOf(histogram.startTimer).toMatchTypeOf<() => () => number>(); + expectTypeOf(histogram.observe).toEqualTypeOf<(value: number) => void>(); + }); + }); + + describe("Counter", () => { + it("should be compatible with prom-client type", () => { + expectTypeOf().toMatchTypeOf(); + }); + + it("should require to set labels if metric has defined labels", () => { + const counter = {} as Counter; + + expectTypeOf(counter.inc).toEqualTypeOf<(labels: Labels, value?: number | undefined) => void>(); + }); + + it("should not require to set labels if metric has no labels", () => { + const counter = {} as Counter; + + expectTypeOf(counter.inc).toEqualTypeOf<(value?: number | undefined) => void>(); + }); + }); +}); diff --git a/packages/validator/src/index.ts b/packages/validator/src/index.ts index 39a331af6657..381db59b7c85 100644 --- a/packages/validator/src/index.ts +++ b/packages/validator/src/index.ts @@ -8,7 +8,7 @@ export type { ProposerConfig, } from "./services/validatorStore.js"; export {waitForGenesis} from "./genesis.js"; -export {getMetrics, type Metrics, type MetricsRegister} from "./metrics.js"; +export {getMetrics, type Metrics} from "./metrics.js"; // Remote signer client export { diff --git a/packages/validator/src/metrics.ts b/packages/validator/src/metrics.ts index 5bc3895414a2..4acf66955769 100644 --- a/packages/validator/src/metrics.ts +++ b/packages/validator/src/metrics.ts @@ -1,3 +1,5 @@ +import {MetricsRegisterExtra} from "@lodestar/utils"; + export enum MessageSource { forward = "forward", publish = "publish", @@ -11,64 +13,6 @@ export enum BeaconHealth { ERROR = 4, } -type LabelsGeneric = Record; -type CollectFn = (metric: Gauge) => void; - -interface Gauge { - // Sorry for this mess, `prom-client` API choices are not great - // If the function signature was `inc(value: number, labels?: Labels)`, this would be simpler - inc(value?: number): void; - inc(labels: Labels, value?: number): void; - inc(arg1?: Labels | number, arg2?: number): void; - - dec(value?: number): void; - dec(labels: Labels, value?: number): void; - dec(arg1?: Labels | number, arg2?: number): void; - - set(value: number): void; - set(labels: Labels, value: number): void; - set(arg1?: Labels | number, arg2?: number): void; - - addCollect(collectFn: CollectFn): void; -} - -interface Histogram { - startTimer(): () => number; - - observe(value: number): void; - observe(labels: Labels, values: number): void; - observe(arg1: Labels | number, arg2?: number): void; - - reset(): void; -} - -interface AvgMinMax { - set(values: number[]): void; - set(labels: Labels, values: number[]): void; - set(arg1?: Labels | number[], arg2?: number[]): void; -} - -type GaugeConfig = { - name: string; - help: string; - labelNames?: keyof Labels extends string ? (keyof Labels)[] : undefined; -}; - -type HistogramConfig = { - name: string; - help: string; - labelNames?: (keyof Labels)[]; - buckets?: number[]; -}; - -type AvgMinMaxConfig = GaugeConfig; - -export interface MetricsRegister { - gauge(config: GaugeConfig): Gauge; - histogram(config: HistogramConfig): Histogram; - avgMinMax(config: AvgMinMaxConfig): AvgMinMax; -} - export type Metrics = ReturnType; export type LodestarGitData = { @@ -81,10 +25,10 @@ export type LodestarGitData = { }; /** - * A collection of metrics used throughout the Gossipsub behaviour. + * A collection of metrics used by the validator client */ // eslint-disable-next-line @typescript-eslint/explicit-function-return-type -export function getMetrics(register: MetricsRegister, gitData: LodestarGitData) { +export function getMetrics(register: MetricsRegisterExtra, gitData: LodestarGitData) { // Using function style instead of class to prevent having to re-declare all MetricsPrometheus types. // Track version, same as https://github.com/ChainSafe/lodestar/blob/6df28de64f12ea90b341b219229a47c8a25c9343/packages/lodestar/src/metrics/metrics/lodestar.ts#L17 @@ -92,7 +36,7 @@ export function getMetrics(register: MetricsRegister, gitData: LodestarGitData) .gauge({ name: "lodestar_version", help: "Lodestar version", - labelNames: Object.keys(gitData) as (keyof LodestarGitData)[], + labelNames: Object.keys(gitData) as [keyof LodestarGitData], }) .set(gitData, 1); @@ -367,7 +311,7 @@ export function getMetrics(register: MetricsRegister, gitData: LodestarGitData) labelNames: ["routeId"], }), - urlsScore: register.gauge<{urlIndex: string}>({ + urlsScore: register.gauge<{urlIndex: number}>({ name: "vc_rest_api_client_urls_score", help: "Current score of REST API URLs by url index", labelNames: ["urlIndex"], diff --git a/yarn.lock b/yarn.lock index db39fcfe612c..291225ebedf5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2598,6 +2598,11 @@ resolved "https://registry.npmjs.org/@opentelemetry/api/-/api-1.0.0-rc.0.tgz" integrity sha512-iXKByCMfrlO5S6Oh97BuM56tM2cIBB0XsL/vWF/AtJrJEKx4MC/Xdu0xDsGXMGcNWpqF7ujMsjjnp0+UHBwnDQ== +"@opentelemetry/api@^1.4.0": + version "1.7.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-1.7.0.tgz#b139c81999c23e3c8d3c0a7234480e945920fc40" + integrity sha512-AdY5wvN0P2vXBi3b29hxZgSFvdhdxPB9+f0B6s//P9Q8nibRWeA3cHm8UmLpio9ABigkVHJ5NMPk+Mz8VCCyrw== + "@parcel/watcher@2.0.4": version "2.0.4" resolved "https://registry.yarnpkg.com/@parcel/watcher/-/watcher-2.0.4.tgz#f300fef4cc38008ff4b8c29d92588eced3ce014b" @@ -12503,6 +12508,14 @@ prom-client@^14.2.0: dependencies: tdigest "^0.1.1" +prom-client@^15.1.0: + version "15.1.0" + resolved "https://registry.yarnpkg.com/prom-client/-/prom-client-15.1.0.tgz#816a4a2128da169d0471093baeccc6d2f17a4613" + integrity sha512-cCD7jLTqyPdjEPBo/Xk4Iu8jxjuZgZJ3e/oET3L+ZwOuap/7Cw3dH/TJSsZKs1TQLZ2IHpIlRAKw82ef06kmMw== + dependencies: + "@opentelemetry/api" "^1.4.0" + tdigest "^0.1.1" + promise-inflight@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/promise-inflight/-/promise-inflight-1.0.1.tgz#98472870bf228132fcbdd868129bad12c3c029e3"