feat(normalization): bypass host scrubbing via allow list (#3813)

As discussed on #3572.

### Legal Boilerplate

Look, I get it. The entity doing business as "Sentry" was incorporated
in the State of Delaware in 2015 as Functional Software, Inc. and is
gonna need some rights from me in order to utilize my contributions in
this here PR. So here's the deal: I retain all rights, title and
interest in and to my contributions, and by keeping this boilerplate
intact I confirm that Sentry can use, modify, copy, and redistribute my
contributions, under Sentry's choice of terms.

---------

Co-authored-by: Joris Bayer <joris.bayer@sentry.io>

CHANGELOG.md

@@ -41,6 +41,9 @@
 - Report outcomes for spans when transactions are rate limited. ([#3749](https://github.com/getsentry/relay/pull/3749))
 - Only transfer valid profile ids. ([#3809](https://github.com/getsentry/relay/pull/3809))
 
+**Features**:
+- Allow list for excluding certain host/IPs from scrubbing in spans. ([#3813](https://github.com/getsentry/relay/pull/3813))
+
 **Internal**:
 
 - Aggregate metrics before rate limiting. ([#3746](https://github.com/getsentry/relay/pull/3746))

relay-cabi/src/processing.rs

@@ -273,6 +273,7 @@ pub unsafe extern "C" fn relay_store_normalizer_normalize_event(
         measurements: None,
         normalize_spans: config.normalize_spans,
         replay_id: config.replay_id,
+        span_allowed_hosts: &[], // only supported in relay
     };
     normalize_event(&mut event, &normalization_config);
 

relay-dynamic-config/Cargo.toml

@@ -30,6 +30,7 @@ relay-quotas = { workspace = true }
 relay-sampling = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
+url = { workspace = true }
 
 [dev-dependencies]
 similar-asserts = { workspace = true }

relay-dynamic-config/src/global.rs

@@ -10,6 +10,7 @@ use relay_filter::GenericFiltersConfig;
 use relay_quotas::Quota;
 use serde::{de, Deserialize, Serialize};
 use serde_json::Value;
+use url::Host;
 
 use crate::{defaults, ErrorBoundary, MetricExtractionGroup, MetricExtractionGroups};
 
@@ -225,6 +226,16 @@ pub struct Options {
     )]
     pub extrapolation_duplication_limit: usize,
 
+    /// List of values on span description that are allowed to be sent to Sentry without being scrubbed.
+    ///
+    /// At this point, it doesn't accept IP addresses in CIDR format.. yet.
+    #[serde(
+        rename = "relay.span-normalization.allowed_hosts",
+        deserialize_with = "default_on_error",
+        skip_serializing_if = "Vec::is_empty"
+    )]
+    pub http_span_allowed_hosts: Vec<Host>,
+
     /// All other unknown options.
     #[serde(flatten)]
     other: HashMap<String, Value>,

relay-event-normalization/src/event.rs

@@ -25,6 +25,7 @@ use relay_protocol::{
     RemarkType, Value,
 };
 use smallvec::SmallVec;
+use url::Host;
 use uuid::Uuid;
 
 use crate::normalize::request;
@@ -154,6 +155,9 @@ pub struct NormalizationConfig<'a> {
     ///
     /// It is persisted into the event payload for correlation.
     pub replay_id: Option<Uuid>,
+
+    /// Controls list of hosts to be excluded from scrubbing
+    pub span_allowed_hosts: &'a [Host],
 }
 
 impl<'a> Default for NormalizationConfig<'a> {
@@ -185,6 +189,7 @@ impl<'a> Default for NormalizationConfig<'a> {
             measurements: None,
             normalize_spans: true,
             replay_id: Default::default(),
+            span_allowed_hosts: Default::default(),
         }
     }
 }
@@ -323,7 +328,11 @@ fn normalize(event: &mut Event, meta: &mut Meta, config: &NormalizationConfig) {
     }
 
     if config.enrich_spans {
-        extract_span_tags_from_event(event, config.max_tag_value_length);
+        extract_span_tags_from_event(
+            event,
+            config.max_tag_value_length,
+            config.span_allowed_hosts,
+        );
     }
 
     if let Some(context) = event.context_mut::<TraceContext>() {

relay-event-normalization/src/normalize/span/description/mod.rs

@@ -38,6 +38,7 @@ const DOMAIN_ALLOW_LIST: &[&str] = &["localhost"];
 /// Returns `None` if no scrubbing can be performed.
 pub(crate) fn scrub_span_description(
     span: &Span,
+    span_allowed_hosts: &[Host],
 ) -> (Option<String>, Option<Vec<sqlparser::ast::Statement>>) {
     let Some(description) = span.description.as_str() else {
         return (None, None);
@@ -56,7 +57,7 @@ pub(crate) fn scrub_span_description(
         .as_str()
         .map(|op| op.split_once('.').unwrap_or((op, "")))
         .and_then(|(op, sub)| match (op, sub) {
-            ("http", _) => scrub_http(description),
+            ("http", _) => scrub_http(description, span_allowed_hosts),
             ("cache", _) | ("db", "redis") => scrub_redis_keys(description),
             ("db", _) if db_system == Some("redis") => scrub_redis_keys(description),
             ("db", sub) => {
@@ -166,7 +167,7 @@ fn scrub_supabase(string: &str) -> Option<String> {
     Some(DB_SUPABASE_REGEX.replace_all(string, "{%s}").into())
 }
 
-fn scrub_http(string: &str) -> Option<String> {
+fn scrub_http(string: &str, allow_list: &[Host]) -> Option<String> {
     let (method, url) = string.split_once(' ')?;
     if !HTTP_METHOD_EXTRACTOR_REGEX.is_match(method) {
         return None;
@@ -179,7 +180,7 @@ fn scrub_http(string: &str) -> Option<String> {
     let scrubbed = match Url::parse(url) {
         Ok(url) => {
             let scheme = url.scheme();
-            let scrubbed_host = url.host().map(scrub_host);
+            let scrubbed_host = url.host().map(|host| scrub_host(host, allow_list));
             let domain = concatenate_host_and_port(scrubbed_host.as_deref(), url.port());
 
             format!("{method} {scheme}://{domain}")
@@ -222,10 +223,15 @@ fn scrub_file(description: &str) -> Option<String> {
 /// use std::net::{Ipv4Addr, Ipv6Addr};
 /// use relay_event_normalization::span::description::scrub_host;
 ///
-/// assert_eq!(scrub_host(Host::Domain("foo.bar.baz")), "*.bar.baz");
-/// assert_eq!(scrub_host(Host::Ipv4(Ipv4Addr::LOCALHOST)), "127.0.0.1");
+/// assert_eq!(scrub_host(Host::Domain("foo.bar.baz"), &[]), "*.bar.baz");
+/// assert_eq!(scrub_host(Host::Ipv4(Ipv4Addr::LOCALHOST), &[]), "127.0.0.1");
+/// assert_eq!(scrub_host(Host::Ipv4(Ipv4Addr::new(8, 8, 8, 8)), &[Host::parse("8.8.8.8").unwrap()]), "8.8.8.8");
 /// ```
-pub fn scrub_host(host: Host<&str>) -> Cow<'_, str> {
+pub fn scrub_host<'a>(host: Host<&'a str>, allow_list: &'a [Host]) -> Cow<'a, str> {
+    if allow_list.iter().any(|allowed_host| &host == allowed_host) {
+        return host.to_string().into();
+    }
+
     match host {
         Host::Ipv4(ip) => Cow::Borrowed(scrub_ipv4(ip)),
         Host::Ipv6(ip) => Cow::Borrowed(scrub_ipv6(ip)),
@@ -394,7 +400,7 @@ fn scrub_resource(resource_type: &str, string: &str) -> Option<String> {
             return Some("browser-extension://*".to_owned());
         }
         scheme => {
-            let scrubbed_host = url.host().map(scrub_host);
+            let scrubbed_host = url.host().map(|host| scrub_host(host, &[]));
             let domain = concatenate_host_and_port(scrubbed_host.as_deref(), url.port());
 
             let segment_count = url.path_segments().map(|s| s.count()).unwrap_or_default();
@@ -562,7 +568,7 @@ mod tests {
                     .description
                     .set_value(Some($description_in.into()));
 
-                let scrubbed = scrub_span_description(span.value_mut().as_mut().unwrap());
+                let scrubbed = scrub_span_description(span.value_mut().as_mut().unwrap(), &[]);
 
                 if $expected == "" {
                     assert!(scrubbed.0.is_none());
@@ -1121,7 +1127,7 @@ mod tests {
 
         let mut span = Annotated::<Span>::from_json(json).unwrap();
         let span = span.value_mut().as_mut().unwrap();
-        let scrubbed = scrub_span_description(span);
+        let scrubbed = scrub_span_description(span, &[]);
         assert_eq!(scrubbed.0.as_deref(), Some("SELECT %s"));
     }
 
@@ -1134,7 +1140,7 @@ mod tests {
 
         let mut span = Annotated::<Span>::from_json(json).unwrap();
 
-        let scrubbed = scrub_span_description(span.value_mut().as_mut().unwrap());
+        let scrubbed = scrub_span_description(span.value_mut().as_mut().unwrap(), &[]);
 
         // When db.system is missing, no scrubbed description (i.e. no group) is set.
         assert!(scrubbed.0.is_none());
@@ -1152,7 +1158,7 @@ mod tests {
 
         let mut span = Annotated::<Span>::from_json(json).unwrap();
 
-        let scrubbed = scrub_span_description(span.value_mut().as_mut().unwrap());
+        let scrubbed = scrub_span_description(span.value_mut().as_mut().unwrap(), &[]);
 
         // Can be scrubbed with db system.
         assert_eq!(scrubbed.0.as_deref(), Some("SELECT a FROM b"));
@@ -1170,7 +1176,7 @@ mod tests {
 
         let mut span = Annotated::<Span>::from_json(json).unwrap();
 
-        let scrubbed = scrub_span_description(span.value_mut().as_mut().unwrap());
+        let scrubbed = scrub_span_description(span.value_mut().as_mut().unwrap(), &[]);
 
         // NOTE: this should return `DEL *`, but we cannot detect lowercase command names yet.
         assert_eq!(scrubbed.0.as_deref(), Some("*"));
@@ -1186,7 +1192,7 @@ mod tests {
 
         let mut span = Annotated::<Span>::from_json(json).unwrap();
 
-        let scrubbed = scrub_span_description(span.value_mut().as_mut().unwrap());
+        let scrubbed = scrub_span_description(span.value_mut().as_mut().unwrap(), &[]);
 
         assert_eq!(scrubbed.0.as_deref(), Some("INSERTED * 'UAEventData'"));
     }
@@ -1201,7 +1207,7 @@ mod tests {
 
         let mut span = Annotated::<Span>::from_json(json).unwrap();
 
-        let scrubbed = scrub_span_description(span.value_mut().as_mut().unwrap());
+        let scrubbed = scrub_span_description(span.value_mut().as_mut().unwrap(), &[]);
 
         assert_eq!(
             scrubbed.0.as_deref(),
@@ -1221,7 +1227,7 @@ mod tests {
 
         let mut span = Annotated::<Span>::from_json(json).unwrap();
 
-        let scrubbed = scrub_span_description(span.value_mut().as_mut().unwrap());
+        let scrubbed = scrub_span_description(span.value_mut().as_mut().unwrap(), &[]);
 
         // Can be scrubbed with db system.
         assert_eq!(scrubbed.0.as_deref(), Some("my-component-name"));