Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

tmp(statsd): Disable high-volume timers to assess performance #3955

Merged
merged 2 commits into from
Aug 28, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions relay-metrics/src/aggregator.rs
Original file line number Diff line number Diff line change
Expand Up @@ -468,11 +468,11 @@ fn get_flush_time(
let initial_flush = bucket_key.timestamp + config.bucket_interval() + config.initial_delay();
let backdated = initial_flush <= now;

let delay = now.as_secs() as i64 - bucket_key.timestamp.as_secs() as i64;
relay_statsd::metric!(
histogram(MetricHistograms::BucketsDelay) = delay as f64,
backdated = if backdated { "true" } else { "false" },
);
// let delay = now.as_secs() as i64 - bucket_key.timestamp.as_secs() as i64;
// relay_statsd::metric!(
// histogram(MetricHistograms::BucketsDelay) = delay as f64,
// backdated = if backdated { "true" } else { "false" },
// );

let flush_timestamp = if backdated {
// If the initial flush time has passed or can't be represented, we want to treat the
Expand Down
1 change: 1 addition & 0 deletions relay-metrics/src/statsd.rs
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,7 @@ pub enum MetricHistograms {
/// This metric is tagged with:
/// - `backdated`: A flag indicating whether the metric was reported within the `initial_delay`
/// time period (`false`) or after the initial delay has expired (`true`).
#[allow(dead_code)] // TODO: Temporarily disabled for a performance measurement.
BucketsDelay,

/// Distribution of invalid bucket timestamps observed, relative to the time of observation.
Expand Down
25 changes: 8 additions & 17 deletions relay-server/src/services/metrics/aggregator.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ use relay_metrics::aggregator::AggregateMetricsError;
use relay_metrics::{aggregator, Bucket, UnixTimestamp};
use relay_system::{Controller, FromMessage, Interface, NoResponse, Recipient, Service, Shutdown};

use crate::statsd::{RelayCounters, RelayHistograms, RelayTimers};
use crate::statsd::{RelayCounters, RelayHistograms};

/// Aggregator for metric buckets.
///
Expand All @@ -31,17 +31,6 @@ pub enum Aggregator {
BucketCountInquiry(BucketCountInquiry, relay_system::Sender<usize>),
}

impl Aggregator {
/// Returns the name of the message variant.
pub fn variant(&self) -> &'static str {
match self {
Aggregator::MergeBuckets(_) => "MergeBuckets",
#[cfg(test)]
Aggregator::BucketCountInquiry(_, _) => "BucketCountInquiry",
}
}
}

impl Interface for Aggregator {}

impl FromMessage<MergeBuckets> for Aggregator {
Expand Down Expand Up @@ -220,10 +209,11 @@ impl AggregatorService {
}

fn handle_message(&mut self, message: Aggregator) {
let ty = message.variant();
relay_statsd::metric!(
timer(RelayTimers::AggregatorServiceDuration),
message = ty,
// let ty = message.variant();
// relay_statsd::metric!(
// timer(RelayTimers::AggregatorServiceDuration),
// message = ty,
{
{
match message {
Aggregator::MergeBuckets(msg) => self.handle_merge_buckets(msg),
Expand All @@ -233,7 +223,8 @@ impl AggregatorService {
}
}
}
)
}
// )
}

fn handle_shutdown(&mut self, message: Shutdown) {
Expand Down
13 changes: 7 additions & 6 deletions relay-server/src/services/metrics/router.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ use relay_system::{Addr, NoResponse, Recipient, Service};
use crate::services::metrics::{
Aggregator, AggregatorHandle, AggregatorService, FlushBuckets, MergeBuckets,
};
use crate::statsd::RelayTimers;
use crate::utils;

/// Service that routes metrics & metric buckets to the appropriate aggregator.
Expand Down Expand Up @@ -105,18 +104,20 @@ impl StartedRouter {
}

fn handle_message(&mut self, message: Aggregator) {
let ty = message.variant();
relay_statsd::metric!(
timer(RelayTimers::MetricRouterServiceDuration),
message = ty,
// let ty = message.variant();
// relay_statsd::metric!(
// timer(RelayTimers::MetricRouterServiceDuration),
// message = ty,
{
{
match message {
Aggregator::MergeBuckets(msg) => self.handle_merge_buckets(msg),
#[cfg(test)]
Aggregator::BucketCountInquiry(_, _sender) => (), // not supported
}
}
)
}
// )
}

fn handle_merge_buckets(&mut self, message: MergeBuckets) {
Expand Down
8 changes: 5 additions & 3 deletions relay-server/src/services/processor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2904,10 +2904,11 @@ impl EnvelopeProcessorService {
}

fn handle_message(&self, message: EnvelopeProcessor) {
let ty = message.variant();
// let ty = message.variant();
let feature_weights = self.feature_weights(&message);

metric!(timer(RelayTimers::ProcessMessageDuration), message = ty, {
// metric!(timer(RelayTimers::ProcessMessageDuration), message = ty, {
{
let mut cogs = self.inner.cogs.timed(ResourceId::Relay, feature_weights);

match message {
Expand All @@ -2924,7 +2925,8 @@ impl EnvelopeProcessorService {
EnvelopeProcessor::SubmitEnvelope(m) => self.handle_submit_envelope(*m),
EnvelopeProcessor::SubmitClientReports(m) => self.handle_submit_client_reports(*m),
}
});
}
// });
}

fn feature_weights(&self, message: &EnvelopeProcessor) -> FeatureWeights {
Expand Down
12 changes: 7 additions & 5 deletions relay-server/src/services/project_cache.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1263,10 +1263,11 @@ impl ProjectCacheBroker {
}

fn handle_message(&mut self, message: ProjectCache) {
let ty = message.variant();
metric!(
timer(RelayTimers::ProjectCacheMessageDuration),
message = ty,
// let ty = message.variant();
// metric!(
// timer(RelayTimers::ProjectCacheMessageDuration),
// message = ty,
{
{
match message {
ProjectCache::RequestUpdate(message) => self.handle_request_update(message),
Expand All @@ -1291,7 +1292,8 @@ impl ProjectCacheBroker {
}
}
}
)
}
// )
}
}

Expand Down
6 changes: 6 additions & 0 deletions relay-server/src/statsd.rs
Original file line number Diff line number Diff line change
Expand Up @@ -464,11 +464,13 @@ pub enum RelayTimers {
/// This metric is tagged with:
///
/// - `message`: The type of message that was processed.
#[allow(dead_code)] // TODO: Temporarily disabled for a performance measurement.
ProcessMessageDuration,
/// Timing in milliseconds for handling a project cache message.
///
/// This metric is tagged with:
/// - `message`: The type of message that was processed.
#[allow(dead_code)] // TODO: Temporarily disabled for a performance measurement.
ProjectCacheMessageDuration,
/// Timing in milliseconds for processing a message in the buffer service.
///
Expand Down Expand Up @@ -502,11 +504,13 @@ pub enum RelayTimers {
///
/// This metric is tagged with:
/// - `message`: The type of message that was processed.
#[allow(dead_code)] // TODO: Temporarily disabled for a performance measurement.
AggregatorServiceDuration,
/// Timing in milliseconds for processing a message in the metric router service.
///
/// This metric is tagged with:
/// - `message`: The type of message that was processed.
#[allow(dead_code)] // TODO: Temporarily disabled for a performance measurement.
MetricRouterServiceDuration,
/// Timing in milliseconds for processing a message in the metric store service.
///
Expand Down Expand Up @@ -551,7 +555,9 @@ impl TimerMetric for RelayTimers {
RelayTimers::HealthCheckDuration => "health.message.duration",
#[cfg(feature = "processing")]
RelayTimers::RateLimitBucketsDuration => "processor.rate_limit_buckets",
#[allow(dead_code)] // TODO: Temporarily disabled for a performance measurement.
RelayTimers::AggregatorServiceDuration => "metrics.aggregator.message.duration",
#[allow(dead_code)] // TODO: Temporarily disabled for a performance measurement.
RelayTimers::MetricRouterServiceDuration => "metrics.router.message.duration",
#[cfg(feature = "processing")]
RelayTimers::StoreServiceDuration => "store.message.duration",
Expand Down
Loading