feat(spooler): Implement eviction mechanism when buffer is full

CHANGELOG.md

@@ -27,6 +27,7 @@
 - Remove the OTEL spans endpoint in favor of Envelopes. ([#3973](https://github.com/getsentry/relay/pull/3973))
 - Remove the `generate-schema` tool. Relay no longer exposes JSON schema for the event protocol. Consult the Rust type documentation of the `relay-event-schema` crate instead. ([#3974](https://github.com/getsentry/relay/pull/3974))
 - Allow creation of `SqliteEnvelopeBuffer` from config, and load existing stacks from db on startup. ([#3967](https://github.com/getsentry/relay/pull/3967))
+- Implement eviction policy when the `EnvelopeBuffer` doesn't have enough capacity. ([#4014](https://github.com/getsentry/relay/pull/4014)) 
 
 ## 24.8.0
 

relay-config/src/config.rs

@@ -903,6 +903,7 @@ fn spool_envelopes_stack_max_batches() -> usize {
     2
 }
 
+/// Default maximum time between receiving the envelope and processing it.
 fn spool_envelopes_max_envelope_delay_secs() -> u64 {
     24 * 60 * 60
 }
@@ -912,6 +913,16 @@ fn spool_disk_usage_refresh_frequency_ms() -> u64 {
     100
 }
 
+/// Default percentage of envelope stacks that can be evicted in the buffer.
+fn spool_envelopes_evictable_stacks_percentage() -> f32 {
+    0.1
+}
+
+/// Default maximum number of envelopes that can be evicted for each stack.
+fn spool_envelopes_max_evictable_envelopes() -> usize {
+    100
+}
+
 /// Persistent buffering configuration for incoming envelopes.
 #[derive(Debug, Serialize, Deserialize)]
 pub struct EnvelopeSpool {
@@ -955,6 +966,12 @@ pub struct EnvelopeSpool {
     /// internal page stats.
     #[serde(default = "spool_disk_usage_refresh_frequency_ms")]
     disk_usage_refresh_frequency_ms: u64,
+    /// Percentage of envelope stacks that can be evicted in the buffer.
+    #[serde(default = "spool_envelopes_evictable_stacks_percentage")]
+    evictable_stacks_percentage: f32,
+    /// Maximum number of envelopes that can be evicted for each stack.
+    #[serde(default = "spool_envelopes_max_evictable_envelopes")]
+    max_evictable_envelopes: usize,
     /// Version of the spooler.
     #[serde(default)]
     version: EnvelopeSpoolVersion,
@@ -991,6 +1008,8 @@ impl Default for EnvelopeSpool {
             max_batches: spool_envelopes_stack_max_batches(),
             max_envelope_delay_secs: spool_envelopes_max_envelope_delay_secs(),
             disk_usage_refresh_frequency_ms: spool_disk_usage_refresh_frequency_ms(),
+            evictable_stacks_percentage: spool_envelopes_evictable_stacks_percentage(),
+            max_evictable_envelopes: spool_envelopes_max_evictable_envelopes(),
             version: EnvelopeSpoolVersion::default(),
         }
     }
@@ -2194,6 +2213,16 @@ impl Config {
         self.values.spool.envelopes.max_batches
     }
 
+    /// Maximum number of stacks that can be evicted.
+    pub fn spool_envelopes_evictable_stacks_percentage(&self) -> f32 {
+        self.values.spool.envelopes.evictable_stacks_percentage
+    }
+
+    /// Maximum number of envelopes that can be evicted per stack.
+    pub fn spool_envelopes_stack_max_evictable_envelopes(&self) -> usize {
+        self.values.spool.envelopes.max_evictable_envelopes
+    }
+
     /// Returns `true` if version 2 of the spooling mechanism is used.
     pub fn spool_v2(&self) -> bool {
         matches!(

relay-server/benches/benches.rs

@@ -101,6 +101,7 @@ fn benchmark_sqlite_envelope_stack(c: &mut Criterion) {
                                 envelope_store.clone(),
                                 disk_batch_size,
                                 2,
+                                10,
                                 ProjectKey::parse("e12d836b15bb49d7bbf99e64295d995b").unwrap(),
                                 ProjectKey::parse("e12d836b15bb49d7bbf99e64295d995b").unwrap(),
                                 true,
@@ -138,6 +139,7 @@ fn benchmark_sqlite_envelope_stack(c: &mut Criterion) {
                                     envelope_store.clone(),
                                     disk_batch_size,
                                     2,
+                                    10,
                                     ProjectKey::parse("e12d836b15bb49d7bbf99e64295d995b").unwrap(),
                                     ProjectKey::parse("e12d836b15bb49d7bbf99e64295d995b").unwrap(),
                                     true,
@@ -179,6 +181,7 @@ fn benchmark_sqlite_envelope_stack(c: &mut Criterion) {
                                 envelope_store.clone(),
                                 disk_batch_size,
                                 2,
+                                10,
                                 ProjectKey::parse("e12d836b15bb49d7bbf99e64295d995b").unwrap(),
                                 ProjectKey::parse("e12d836b15bb49d7bbf99e64295d995b").unwrap(),
                                 true,