chore(metrics): Normalize db.redis operation (#2268)

This change add normalization to the `db.redis` operation, reusing regexes which were done for `cache` op. Also make sure not to run SQL normalization for this op.
getsentry · Jun 29, 2023 · e16db1f · e16db1f
1 parent 9dffb22
commit e16db1f
Show file tree

Hide file tree

Showing 5 changed files with 322 additions and 4 deletions.
diff --git a/relay-general/src/store/regexes.rs b/relay-general/src/store/regexes.rs
@@ -95,7 +95,9 @@ pub static CACHE_NORMALIZER_REGEX: Lazy<Regex> = Lazy::new(|| {
         # Capture hex.
         (([\s.+:/\-])+(?P<hex>[a-fA-F0-9]+\b)+) |
         # Capture segments, in form of`:{hi}:`
-        (([\s.+:/\-])+(?P<segment>\{[^\}]*\})+)
+        (([\s.+:/\-])+(?P<segment>\{[^\}]*\})+) |
+        # Capture everything if above regex could not match.
+        ([\s]+(?P<zero>[^:/\-]+$))
     "#,
     )
     .unwrap()

diff --git a/relay-general/src/store/transactions/processor.rs b/relay-general/src/store/transactions/processor.rs
@@ -440,7 +440,7 @@ fn scrub_sql_queries(string: &mut Annotated<String>) -> Result<bool, ProcessingA
     scrub_identifiers_with_regex(string, &SQL_NORMALIZER_REGEX, "%s")
 }
 
-fn scrub_cache_keys(string: &mut Annotated<String>) -> Result<bool, ProcessingAction> {
+fn scrub_redis_keys(string: &mut Annotated<String>) -> Result<bool, ProcessingAction> {
     scrub_identifiers_with_regex(string, &CACHE_NORMALIZER_REGEX, "*")
 }
 
@@ -620,8 +620,8 @@ fn scrub_span_description(span: &mut Span) -> Result<(), ProcessingAction> {
 
     let did_scrub = match span.op.value() {
         Some(op) if op.starts_with("http") => scrub_identifiers(&mut scrubbed)?,
-        Some(op) if op.starts_with("db") => scrub_sql_queries(&mut scrubbed)?,
-        Some(op) if op.starts_with("cache") => scrub_cache_keys(&mut scrubbed)?,
+        Some(op) if op.starts_with("cache") || op == "db.redis" => scrub_redis_keys(&mut scrubbed)?,
+        Some(op) if op.starts_with("db") && op != "db.redis" => scrub_sql_queries(&mut scrubbed)?,
         Some(op) if op.starts_with("resource") => scrub_resource_identifiers(&mut scrubbed)?,
         _ => false,
     };

diff --git a/relay-server/src/metrics_extraction/spans/mod.rs b/relay-server/src/metrics_extraction/spans/mod.rs
@@ -1001,6 +1001,28 @@ mod tests {
                         "cache.hit": false
                     }
                 },
+                {
+                    "description": "GET test:123:def",
+                    "op": "db.redis",
+                    "parent_span_id": "8f5a2b8768cafb4e",
+                    "span_id": "bb7af8b99e95af5f",
+                    "start_timestamp": 1597976300.0000000,
+                    "timestamp": 1597976302.0000000,
+                    "trace_id": "ff62a8b040f340bda5d830223def1d81",
+                    "status": "ok",
+                    "data": {}
+                },
+                {
+                    "description": "GET lkjasdlkasjdlasjdlkasjdlkasjd",
+                    "op": "db.redis",
+                    "parent_span_id": "8f5a2b8768cafb4e",
+                    "span_id": "bb7af8b99e95af5f",
+                    "start_timestamp": 1597976300.0000000,
+                    "timestamp": 1597976302.0000000,
+                    "trace_id": "ff62a8b040f340bda5d830223def1d81",
+                    "status": "ok",
+                    "data": {}
+                },
                 {
                     "description": "http://domain/static/myscript-v1.9.23.js",
                     "op": "resource.script",

diff --git a/...ans/snapshots/relay_server__metrics_extraction__spans__tests__extract_span_metrics-2.snap b/...ans/snapshots/relay_server__metrics_extraction__spans__tests__extract_span_metrics-2.snap
@@ -1624,6 +1624,164 @@ expression: metrics
             "transaction.op": "myop",
         },
     },
+    Metric {
+        name: "s:spans/user@none",
+        value: Set(
+            933084975,
+        ),
+        timestamp: UnixTimestamp(1619420400),
+        tags: {
+            "environment": "fake_environment",
+            "http.status_code": "200",
+            "span.category": "db",
+            "span.description": "GET test:*:*",
+            "span.group": "2c1e35c1f77934da",
+            "span.module": "db",
+            "span.op": "db.redis",
+            "span.status": "ok",
+            "transaction": "gEt /api/:version/users/",
+            "transaction.method": "POST",
+            "transaction.op": "myop",
+        },
+    },
+    Metric {
+        name: "d:spans/exclusive_time@millisecond",
+        value: Distribution(
+            2000.0,
+        ),
+        timestamp: UnixTimestamp(1619420400),
+        tags: {
+            "environment": "fake_environment",
+            "http.status_code": "200",
+            "span.category": "db",
+            "span.description": "GET test:*:*",
+            "span.group": "2c1e35c1f77934da",
+            "span.module": "db",
+            "span.op": "db.redis",
+            "span.status": "ok",
+            "transaction": "gEt /api/:version/users/",
+            "transaction.method": "POST",
+            "transaction.op": "myop",
+        },
+    },
+    Metric {
+        name: "d:spans/exclusive_time_light@millisecond",
+        value: Distribution(
+            2000.0,
+        ),
+        timestamp: UnixTimestamp(1619420400),
+        tags: {
+            "environment": "fake_environment",
+            "http.status_code": "200",
+            "span.category": "db",
+            "span.description": "GET test:*:*",
+            "span.group": "2c1e35c1f77934da",
+            "span.module": "db",
+            "span.op": "db.redis",
+            "span.status": "ok",
+            "transaction.method": "POST",
+            "transaction.op": "myop",
+        },
+    },
+    Metric {
+        name: "d:spans/duration@millisecond",
+        value: Distribution(
+            2000.0,
+        ),
+        timestamp: UnixTimestamp(1619420400),
+        tags: {
+            "environment": "fake_environment",
+            "http.status_code": "200",
+            "span.category": "db",
+            "span.description": "GET test:*:*",
+            "span.group": "2c1e35c1f77934da",
+            "span.module": "db",
+            "span.op": "db.redis",
+            "span.status": "ok",
+            "transaction": "gEt /api/:version/users/",
+            "transaction.method": "POST",
+            "transaction.op": "myop",
+        },
+    },
+    Metric {
+        name: "s:spans/user@none",
+        value: Set(
+            933084975,
+        ),
+        timestamp: UnixTimestamp(1619420400),
+        tags: {
+            "environment": "fake_environment",
+            "http.status_code": "200",
+            "span.category": "db",
+            "span.description": "GET *",
+            "span.group": "37e3d9fab1ae9162",
+            "span.module": "db",
+            "span.op": "db.redis",
+            "span.status": "ok",
+            "transaction": "gEt /api/:version/users/",
+            "transaction.method": "POST",
+            "transaction.op": "myop",
+        },
+    },
+    Metric {
+        name: "d:spans/exclusive_time@millisecond",
+        value: Distribution(
+            2000.0,
+        ),
+        timestamp: UnixTimestamp(1619420400),
+        tags: {
+            "environment": "fake_environment",
+            "http.status_code": "200",
+            "span.category": "db",
+            "span.description": "GET *",
+            "span.group": "37e3d9fab1ae9162",
+            "span.module": "db",
+            "span.op": "db.redis",
+            "span.status": "ok",
+            "transaction": "gEt /api/:version/users/",
+            "transaction.method": "POST",
+            "transaction.op": "myop",
+        },
+    },
+    Metric {
+        name: "d:spans/exclusive_time_light@millisecond",
+        value: Distribution(
+            2000.0,
+        ),
+        timestamp: UnixTimestamp(1619420400),
+        tags: {
+            "environment": "fake_environment",
+            "http.status_code": "200",
+            "span.category": "db",
+            "span.description": "GET *",
+            "span.group": "37e3d9fab1ae9162",
+            "span.module": "db",
+            "span.op": "db.redis",
+            "span.status": "ok",
+            "transaction.method": "POST",
+            "transaction.op": "myop",
+        },
+    },
+    Metric {
+        name: "d:spans/duration@millisecond",
+        value: Distribution(
+            2000.0,
+        ),
+        timestamp: UnixTimestamp(1619420400),
+        tags: {
+            "environment": "fake_environment",
+            "http.status_code": "200",
+            "span.category": "db",
+            "span.description": "GET *",
+            "span.group": "37e3d9fab1ae9162",
+            "span.module": "db",
+            "span.op": "db.redis",
+            "span.status": "ok",
+            "transaction": "gEt /api/:version/users/",
+            "transaction.method": "POST",
+            "transaction.op": "myop",
+        },
+    },
     Metric {
         name: "s:spans/user@none",
         value: Set(

diff --git a/...spans/snapshots/relay_server__metrics_extraction__spans__tests__extract_span_metrics.snap b/...spans/snapshots/relay_server__metrics_extraction__spans__tests__extract_span_metrics.snap
@@ -1441,6 +1441,142 @@ expression: event.value().unwrap().spans
         },
         other: {},
     },
+    Span {
+        timestamp: Timestamp(
+            2020-08-21T02:18:22Z,
+        ),
+        start_timestamp: Timestamp(
+            2020-08-21T02:18:20Z,
+        ),
+        exclusive_time: 2000.0,
+        description: "GET test:123:def",
+        op: "db.redis",
+        span_id: SpanId(
+            "bb7af8b99e95af5f",
+        ),
+        parent_span_id: SpanId(
+            "8f5a2b8768cafb4e",
+        ),
+        trace_id: TraceId(
+            "ff62a8b040f340bda5d830223def1d81",
+        ),
+        status: Ok,
+        tags: ~,
+        origin: ~,
+        data: {
+            "description.scrubbed": String(
+                "GET test:*:*",
+            ),
+            "environment": String(
+                "fake_environment",
+            ),
+            "http.status_code": String(
+                "200",
+            ),
+            "release": String(
+                "1.2.3",
+            ),
+            "span.category": String(
+                "db",
+            ),
+            "span.description": String(
+                "GET test:*:*",
+            ),
+            "span.group": String(
+                "2c1e35c1f77934da",
+            ),
+            "span.module": String(
+                "db",
+            ),
+            "span.op": String(
+                "db.redis",
+            ),
+            "span.status": String(
+                "ok",
+            ),
+            "transaction": String(
+                "gEt /api/:version/users/",
+            ),
+            "transaction.method": String(
+                "POST",
+            ),
+            "transaction.op": String(
+                "myop",
+            ),
+            "user": String(
+                "id:user123",
+            ),
+        },
+        other: {},
+    },
+    Span {
+        timestamp: Timestamp(
+            2020-08-21T02:18:22Z,
+        ),
+        start_timestamp: Timestamp(
+            2020-08-21T02:18:20Z,
+        ),
+        exclusive_time: 2000.0,
+        description: "GET lkjasdlkasjdlasjdlkasjdlkasjd",
+        op: "db.redis",
+        span_id: SpanId(
+            "bb7af8b99e95af5f",
+        ),
+        parent_span_id: SpanId(
+            "8f5a2b8768cafb4e",
+        ),
+        trace_id: TraceId(
+            "ff62a8b040f340bda5d830223def1d81",
+        ),
+        status: Ok,
+        tags: ~,
+        origin: ~,
+        data: {
+            "description.scrubbed": String(
+                "GET *",
+            ),
+            "environment": String(
+                "fake_environment",
+            ),
+            "http.status_code": String(
+                "200",
+            ),
+            "release": String(
+                "1.2.3",
+            ),
+            "span.category": String(
+                "db",
+            ),
+            "span.description": String(
+                "GET *",
+            ),
+            "span.group": String(
+                "37e3d9fab1ae9162",
+            ),
+            "span.module": String(
+                "db",
+            ),
+            "span.op": String(
+                "db.redis",
+            ),
+            "span.status": String(
+                "ok",
+            ),
+            "transaction": String(
+                "gEt /api/:version/users/",
+            ),
+            "transaction.method": String(
+                "POST",
+            ),
+            "transaction.op": String(
+                "myop",
+            ),
+            "user": String(
+                "id:user123",
+            ),
+        },
+        other: {},
+    },
     Span {
         timestamp: Timestamp(
             2020-08-21T02:18:22Z,