Skip to content

Commit

Permalink
Use a histogram for queue lengths
Browse files Browse the repository at this point in the history
  • Loading branch information
@paulhauner
paulhauner committed Jul 18, 2023
1 parent 63de59b commit 93baac2
Show file tree
Hide file tree
Showing 3 changed files with 68 additions and 109 deletions.
98 changes: 56 additions & 42 deletions beacon_node/beacon_processor/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1224,61 +1224,75 @@ impl<E: EthSpec> BeaconProcessor<E> {
&metrics::BEACON_PROCESSOR_WORKERS_ACTIVE_TOTAL,
self.current_workers as i64,
);
metrics::set_gauge(
&metrics::BEACON_PROCESSOR_UNAGGREGATED_ATTESTATION_QUEUE_TOTAL,
attestation_queue.len() as i64,
metrics::observe_histogram_vec(
&metrics::BEACON_PROCESSOR_QUEUE_LENGTHS,
&[GOSSIP_ATTESTATION],
attestation_queue.len() as f64,
);
metrics::set_gauge(
&metrics::BEACON_PROCESSOR_AGGREGATED_ATTESTATION_QUEUE_TOTAL,
aggregate_queue.len() as i64,
metrics::observe_histogram_vec(
&metrics::BEACON_PROCESSOR_QUEUE_LENGTHS,
&[GOSSIP_AGGREGATE],
aggregate_queue.len() as f64,
);
metrics::set_gauge(
&metrics::BEACON_PROCESSOR_SYNC_MESSAGE_QUEUE_TOTAL,
sync_message_queue.len() as i64,
metrics::observe_histogram_vec(
&metrics::BEACON_PROCESSOR_QUEUE_LENGTHS,
&[GOSSIP_SYNC_SIGNATURE],
sync_message_queue.len() as f64,
);
metrics::set_gauge(
&metrics::BEACON_PROCESSOR_SYNC_CONTRIBUTION_QUEUE_TOTAL,
sync_contribution_queue.len() as i64,
metrics::observe_histogram_vec(
&metrics::BEACON_PROCESSOR_QUEUE_LENGTHS,
&[GOSSIP_SYNC_CONTRIBUTION],
sync_contribution_queue.len() as f64,
);
metrics::set_gauge(
&metrics::BEACON_PROCESSOR_GOSSIP_BLOCK_QUEUE_TOTAL,
gossip_block_queue.len() as i64,
metrics::observe_histogram_vec(
&metrics::BEACON_PROCESSOR_QUEUE_LENGTHS,
&[GOSSIP_BLOCK],
gossip_block_queue.len() as f64,
);
metrics::set_gauge(
&metrics::BEACON_PROCESSOR_RPC_BLOCK_QUEUE_TOTAL,
rpc_block_queue.len() as i64,
metrics::observe_histogram_vec(
&metrics::BEACON_PROCESSOR_QUEUE_LENGTHS,
&[RPC_BLOCK],
rpc_block_queue.len() as f64,
);
metrics::set_gauge(
&metrics::BEACON_PROCESSOR_CHAIN_SEGMENT_QUEUE_TOTAL,
chain_segment_queue.len() as i64,
metrics::observe_histogram_vec(
&metrics::BEACON_PROCESSOR_QUEUE_LENGTHS,
&[CHAIN_SEGMENT],
chain_segment_queue.len() as f64,
);
metrics::set_gauge(
&metrics::BEACON_PROCESSOR_BACKFILL_CHAIN_SEGMENT_QUEUE_TOTAL,
backfill_chain_segment.len() as i64,
metrics::observe_histogram_vec(
&metrics::BEACON_PROCESSOR_QUEUE_LENGTHS,
&[CHAIN_SEGMENT_BACKFILL],
backfill_chain_segment.len() as f64,
);
metrics::set_gauge(
&metrics::BEACON_PROCESSOR_EXIT_QUEUE_TOTAL,
gossip_voluntary_exit_queue.len() as i64,
metrics::observe_histogram_vec(
&metrics::BEACON_PROCESSOR_QUEUE_LENGTHS,
&[GOSSIP_VOLUNTARY_EXIT],
gossip_voluntary_exit_queue.len() as f64,
);
metrics::set_gauge(
&metrics::BEACON_PROCESSOR_PROPOSER_SLASHING_QUEUE_TOTAL,
gossip_proposer_slashing_queue.len() as i64,
metrics::observe_histogram_vec(
&metrics::BEACON_PROCESSOR_QUEUE_LENGTHS,
&[GOSSIP_PROPOSER_SLASHING],
gossip_proposer_slashing_queue.len() as f64,
);
metrics::set_gauge(
&metrics::BEACON_PROCESSOR_ATTESTER_SLASHING_QUEUE_TOTAL,
gossip_attester_slashing_queue.len() as i64,
metrics::observe_histogram_vec(
&metrics::BEACON_PROCESSOR_QUEUE_LENGTHS,
&[GOSSIP_ATTESTER_SLASHING],
gossip_attester_slashing_queue.len() as f64,
);
metrics::set_gauge(
&metrics::BEACON_PROCESSOR_BLS_TO_EXECUTION_CHANGE_QUEUE_TOTAL,
gossip_bls_to_execution_change_queue.len() as i64,
metrics::observe_histogram_vec(
&metrics::BEACON_PROCESSOR_QUEUE_LENGTHS,
&[GOSSIP_BLS_TO_EXECUTION_CHANGE],
gossip_bls_to_execution_change_queue.len() as f64,
);
metrics::set_gauge(
&metrics::BEACON_PROCESSOR_API_REQUEST_P0_QUEUE_TOTAL,
api_request_p0_queue.len() as i64,
metrics::observe_histogram_vec(
&metrics::BEACON_PROCESSOR_QUEUE_LENGTHS,
&[API_REQUEST_P0],
api_request_p0_queue.len() as f64,
);
metrics::set_gauge(
&metrics::BEACON_PROCESSOR_API_REQUEST_P1_QUEUE_TOTAL,
api_request_p1_queue.len() as i64,
metrics::observe_histogram_vec(
&metrics::BEACON_PROCESSOR_QUEUE_LENGTHS,
&[API_REQUEST_P1],
api_request_p1_queue.len() as f64,
);

if aggregate_queue.is_full() && aggregate_debounce.elapsed() {
Expand Down
72 changes: 5 additions & 67 deletions beacon_node/beacon_processor/src/metrics.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,73 +41,11 @@ lazy_static::lazy_static! {
"beacon_processor_event_handling_seconds",
"Time spent handling a new message and allocating it to a queue or worker."
);
// Gossip blocks.
pub static ref BEACON_PROCESSOR_GOSSIP_BLOCK_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
"beacon_processor_gossip_block_queue_total",
"Count of blocks from gossip waiting to be verified."
);
// Gossip Exits.
pub static ref BEACON_PROCESSOR_EXIT_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
"beacon_processor_exit_queue_total",
"Count of exits from gossip waiting to be verified."
);
// Gossip proposer slashings.
pub static ref BEACON_PROCESSOR_PROPOSER_SLASHING_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
"beacon_processor_proposer_slashing_queue_total",
"Count of proposer slashings from gossip waiting to be verified."
);
// Gossip attester slashings.
pub static ref BEACON_PROCESSOR_ATTESTER_SLASHING_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
"beacon_processor_attester_slashing_queue_total",
"Count of attester slashings from gossip waiting to be verified."
);
// Gossip BLS to execution changes.
pub static ref BEACON_PROCESSOR_BLS_TO_EXECUTION_CHANGE_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
"beacon_processor_bls_to_execution_change_queue_total",
"Count of address changes from gossip waiting to be verified."
);
// Rpc blocks.
pub static ref BEACON_PROCESSOR_RPC_BLOCK_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
"beacon_processor_rpc_block_queue_total",
"Count of blocks from the rpc waiting to be verified."
);
// Chain segments.
pub static ref BEACON_PROCESSOR_CHAIN_SEGMENT_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
"beacon_processor_chain_segment_queue_total",
"Count of chain segments from the rpc waiting to be verified."
);
pub static ref BEACON_PROCESSOR_BACKFILL_CHAIN_SEGMENT_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
"beacon_processor_backfill_chain_segment_queue_total",
"Count of backfill chain segments from the rpc waiting to be verified."
);
// Unaggregated attestations.
pub static ref BEACON_PROCESSOR_UNAGGREGATED_ATTESTATION_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
"beacon_processor_unaggregated_attestation_queue_total",
"Count of unagg. attestations waiting to be processed."
);
// Aggregated attestations.
pub static ref BEACON_PROCESSOR_AGGREGATED_ATTESTATION_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
"beacon_processor_aggregated_attestation_queue_total",
"Count of agg. attestations waiting to be processed."
);
// Sync committee messages.
pub static ref BEACON_PROCESSOR_SYNC_MESSAGE_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
"beacon_processor_sync_message_queue_total",
"Count of sync committee messages waiting to be processed."
);
// Sync contribution.
pub static ref BEACON_PROCESSOR_SYNC_CONTRIBUTION_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
"beacon_processor_sync_contribution_queue_total",
"Count of sync committee contributions waiting to be processed."
);
// HTTP API requests.
pub static ref BEACON_PROCESSOR_API_REQUEST_P0_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
"beacon_processor_api_request_p0_queue_total",
"Count of P0 HTTP requesets waiting to be processed."
);
pub static ref BEACON_PROCESSOR_API_REQUEST_P1_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
"beacon_processor_api_request_p1_queue_total",
"Count of P1 HTTP requesets waiting to be processed."
pub static ref BEACON_PROCESSOR_QUEUE_LENGTHS: Result<HistogramVec> = try_create_histogram_vec_with_buckets(
"beacon_processor_queue_lengths",
"The lengths of beacon processor event queues.",
Ok(vec![0.0, 1.0, 4.0, 16.0, 64.0, 256.0, 1024.0, 4096.0, 16384.0, 32768.0]),
&["type"]
);

/*
Expand Down
7 changes: 7 additions & 0 deletions common/lighthouse_metrics/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -280,6 +280,13 @@ pub fn observe_timer_vec(vec: &Result<HistogramVec>, name: &[&str], duration: Du
}
}

/// Starts a timer on `vec` with the given `name`.
pub fn observe_histogram_vec(vec: &Result<HistogramVec>, name: &[&str], value: f64) {
if let Some(h) = get_histogram(vec, name) {
h.observe(value)
}
}

/// Stops a timer created with `start_timer(..)`.
pub fn stop_timer(timer: Option<HistogramTimer>) {
if let Some(t) = timer {
Expand Down

0 comments on commit 93baac2

Please sign in to comment.