feat(spooler): Implement eviction mechanism when buffer is full

CHANGELOG.md

@@ -27,6 +27,7 @@
 - Remove the OTEL spans endpoint in favor of Envelopes. ([#3973](https://github.com/getsentry/relay/pull/3973))
 - Remove the `generate-schema` tool. Relay no longer exposes JSON schema for the event protocol. Consult the Rust type documentation of the `relay-event-schema` crate instead. ([#3974](https://github.com/getsentry/relay/pull/3974))
 - Allow creation of `SqliteEnvelopeBuffer` from config, and load existing stacks from db on startup. ([#3967](https://github.com/getsentry/relay/pull/3967))
+- Implement eviction policy when the `EnvelopeBuffer` doesn't have enough capacity. ([#4014](https://github.com/getsentry/relay/pull/4014)) 
 
 ## 24.8.0
 

relay-config/src/config.rs

@@ -903,6 +903,7 @@ fn spool_envelopes_stack_max_batches() -> usize {
  2
 }
 
+/// Default maximum time between receiving the envelope and processing it.
 fn spool_envelopes_max_envelope_delay_secs() -> u64 {
  24 * 60 * 60
 }
@@ -912,6 +913,16 @@ fn spool_disk_usage_refresh_frequency_ms() -> u64 {
  100
 }
 
+/// Default percentage of envelope stacks that can be evicted in the buffer.
+fn spool_envelopes_evictable_stacks_percentage() -> f32 {
+ 0.1
+}
+
+/// Default maximum number of envelopes that can be evicted for each stack.
+fn spool_envelopes_max_evictable_envelopes() -> usize {
+ 100
+}
+
 /// Persistent buffering configuration for incoming envelopes.
 #[derive(Debug, Serialize, Deserialize)]
 pub struct EnvelopeSpool {
@@ -955,6 +966,12 @@ pub struct EnvelopeSpool {
  /// internal page stats.
  #[serde(default = "spool_disk_usage_refresh_frequency_ms")]
  disk_usage_refresh_frequency_ms: u64,
+ /// Percentage of envelope stacks that can be evicted in the buffer.
+ #[serde(default = "spool_envelopes_evictable_stacks_percentage")]
+ evictable_stacks_percentage: f32,
+ /// Maximum number of envelopes that can be evicted for each stack.
+ #[serde(default = "spool_envelopes_max_evictable_envelopes")]
+ max_evictable_envelopes: usize,
  /// Version of the spooler.
  #[serde(default)]
  version: EnvelopeSpoolVersion,
@@ -991,6 +1008,8 @@ impl Default for EnvelopeSpool {
  max_batches: spool_envelopes_stack_max_batches(),
  max_envelope_delay_secs: spool_envelopes_max_envelope_delay_secs(),
  disk_usage_refresh_frequency_ms: spool_disk_usage_refresh_frequency_ms(),
+ evictable_stacks_percentage: spool_envelopes_evictable_stacks_percentage(),
+ max_evictable_envelopes: spool_envelopes_max_evictable_envelopes(),
  version: EnvelopeSpoolVersion::default(),
  }
  }
@@ -2194,6 +2213,16 @@ impl Config {
  self.values.spool.envelopes.max_batches
  }
 
+ /// Maximum number of stacks that can be evicted.
+ pub fn spool_envelopes_evictable_stacks_percentage(&self) -> f32 {
+ self.values.spool.envelopes.evictable_stacks_percentage
+ }
+
+ /// Maximum number of envelopes that can be evicted per stack.
+ pub fn spool_envelopes_stack_max_evictable_envelopes(&self) -> usize {
+ self.values.spool.envelopes.max_evictable_envelopes
+ }
+
  /// Returns `true` if version 2 of the spooling mechanism is used.
  pub fn spool_v2(&self) -> bool {
  matches!(

relay-server/benches/benches.rs

@@ -101,6 +101,7 @@ fn benchmark_sqlite_envelope_stack(c: &mut Criterion) {
  envelope_store.clone(),
  disk_batch_size,
  2,
+ 10,
  ProjectKey::parse("e12d836b15bb49d7bbf99e64295d995b").unwrap(),
  ProjectKey::parse("e12d836b15bb49d7bbf99e64295d995b").unwrap(),
  true,
@@ -138,6 +139,7 @@ fn benchmark_sqlite_envelope_stack(c: &mut Criterion) {
  envelope_store.clone(),
  disk_batch_size,
  2,
+ 10,
  ProjectKey::parse("e12d836b15bb49d7bbf99e64295d995b").unwrap(),
  ProjectKey::parse("e12d836b15bb49d7bbf99e64295d995b").unwrap(),
  true,
@@ -179,6 +181,7 @@ fn benchmark_sqlite_envelope_stack(c: &mut Criterion) {
  envelope_store.clone(),
  disk_batch_size,
  2,
+ 10,
  ProjectKey::parse("e12d836b15bb49d7bbf99e64295d995b").unwrap(),
  ProjectKey::parse("e12d836b15bb49d7bbf99e64295d995b").unwrap(),
  true,