Fix Prometheus metrics (#773)

* chore: replace dots in ipv4 addresses with slashes Signed-off-by: Rolf Ahrenberg <rolf.ahrenberg@saunalahti.fi> * chore: tweak default mapper configuration for Prometheus Signed-off-by: Rolf Ahrenberg <rolf.ahrenberg@saunalahti.fi> --------- Signed-off-by: Rolf Ahrenberg <rolf.ahrenberg@saunalahti.fi>
envoyproxy · Jan 15, 2025 · 60d8e81 · 60d8e81
1 parent fccc165
commit 60d8e81
Show file tree

Hide file tree

Showing 3 changed files with 37 additions and 16 deletions.
diff --git a/src/stats/prom/default_mapper.yaml b/src/stats/prom/default_mapper.yaml
@@ -24,35 +24,53 @@ mappings:
     labels:
       domain: "$1"
       key1: "$2"
+  - match: "ratelimit.service.rate_limit.*.*.shadow_mode"
+    name: "ratelimit_service_rate_limit_shadow_mode"
+    timer_type: "histogram"
+    labels:
+      domain: "$1"
+      key1: "$2"
 
-  - match: "ratelimit.service.rate_limit.*.*.*.near_limit"
+  - match: "ratelimit\\.service\\.rate_limit\\.([^\\.]*)\\.([^\\.]*)\\.([^\\.]*)(\\..*)?\\.near_limit"
+    match_type: regex
     name: "ratelimit_service_rate_limit_near_limit"
     timer_type: "histogram"
     labels:
       domain: "$1"
       key1: "$2"
       key2: "$3"
-  - match: "ratelimit.service.rate_limit.*.*.*.over_limit"
+  - match: "ratelimit\\.service\\.rate_limit\\.([^\\.]*)\\.([^\\.]*)\\.([^\\.]*)(\\..*)?\\.over_limit"
+    match_type: regex
     name: "ratelimit_service_rate_limit_over_limit"
     timer_type: "histogram"
     labels:
       domain: "$1"
       key1: "$2"
       key2: "$3"
-  - match: "ratelimit.service.rate_limit.*.*.*.total_hits"
+  - match: "ratelimit\\.service\\.rate_limit\\.([^\\.]*)\\.([^\\.]*)\\.([^\\.]*)(\\..*)?\\.total_hits"
+    match_type: regex
     name: "ratelimit_service_rate_limit_total_hits"
     timer_type: "histogram"
     labels:
       domain: "$1"
       key1: "$2"
       key2: "$3"
-  - match: "ratelimit.service.rate_limit.*.*.*.within_limit"
+  - match: "ratelimit\\.service\\.rate_limit\\.([^\\.]*)\\.([^\\.]*)\\.([^\\.]*)(\\..*)?\\.within_limit"
+    match_type: regex
     name: "ratelimit_service_rate_limit_within_limit"
     timer_type: "histogram"
     labels:
       domain: "$1"
       key1: "$2"
       key2: "$3"
+  - match: "ratelimit\\.service\\.rate_limit\\.([^\\.]*)\\.([^\\.]*)\\.([^\\.]*)(\\..*)?\\.shadow_mode"
+    match_type: regex
+    name: "ratelimit_service_rate_limit_shadow_mode"
+    timer_type: "histogram"
+    labels:
+      domain: "$1"
+      key1: "$2"
+      key2: "$3"
 
   - match: "ratelimit.service.call.should_rate_limit.*"
     name: "ratelimit_service_should_rate_limit_error"
@@ -65,7 +83,6 @@ mappings:
     match_metric_type: counter
     labels:
       grpc_method: "$1"
-
   - match: "ratelimit_server.*.response_time"
     name: "ratelimit_service_response_time_seconds"
     timer_type: histogram
@@ -75,15 +92,6 @@ mappings:
   - match: "ratelimit.service.config_load_success"
     name: "ratelimit_service_config_load_success"
     match_metric_type: counter
-
   - match: "ratelimit.service.config_load_error"
     name: "ratelimit_service_config_load_error"
     match_metric_type: counter
-
-  - match: "ratelimit.service.rate_limit.*.*.*.shadow_mode"
-    name: "ratelimit_service_rate_limit_shadow_mode"
-    timer_type: "histogram"
-    labels:
-      domain: "$1"
-      key1: "$2"
-      key2: "$3"
diff --git a/src/stats/prom/prometheus_sink_test.go b/src/stats/prom/prometheus_sink_test.go
@@ -45,6 +45,8 @@ func toMap(labels []*dto.LabelPair) map[string]string {
 func TestFlushCounterWithDifferentLabels(t *testing.T) {
 	s.FlushCounter("ratelimit.service.rate_limit.domain1.key1_val1.over_limit", 1)
 	s.FlushCounter("ratelimit.service.rate_limit.domain1.key1_val1.key2_val2.over_limit", 2)
+	s.FlushCounter("ratelimit.service.rate_limit.domain1.key3_val3.key4_val4.over_limit", 1)
+	s.FlushCounter("ratelimit.service.rate_limit.domain1.key3_val3.key4_val4.key5_val5.over_limit", 2)
 	assert.Eventually(t, func() bool {
 		metricFamilies, err := prometheus.DefaultGatherer.Gather()
 		require.NoError(t, err)
@@ -56,7 +58,7 @@ func TestFlushCounterWithDifferentLabels(t *testing.T) {
 
 		m, ok := metrics["ratelimit_service_rate_limit_over_limit"]
 		require.True(t, ok)
-		require.Len(t, m.Metric, 2)
+		require.Len(t, m.Metric, 3)
 		require.Equal(t, 1.0, *m.Metric[0].Counter.Value)
 		require.Equal(t, map[string]string{
 			"domain": "domain1",
@@ -68,6 +70,12 @@ func TestFlushCounterWithDifferentLabels(t *testing.T) {
 			"key1":   "key1_val1",
 			"key2":   "key2_val2",
 		}, toMap(m.Metric[1].Label))
+		require.Equal(t, 3.0, *m.Metric[2].Counter.Value)
+		require.Equal(t, map[string]string{
+			"domain": "domain1",
+			"key1":   "key3_val3",
+			"key2":   "key4_val4",
+		}, toMap(m.Metric[2].Label))
 		return true
 	}, time.Second, time.Millisecond)
 }

diff --git a/src/utils/utilities.go b/src/utils/utilities.go
@@ -1,6 +1,7 @@
 package utils
 
 import (
+	"regexp"
 	"strings"
 
 	pb "github.com/envoyproxy/go-control-plane/envoy/service/ratelimit/v3"
@@ -61,10 +62,14 @@ func MaskCredentialsInUrl(url string) string {
 	return strings.Join(urls, ",")
 }
 
+var ipv4Regex = regexp.MustCompile(`\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}`)
+
 // Remove invalid characters from the stat name.
 func SanitizeStatName(s string) string {
 	r := strings.NewReplacer(":", "_", "|", "_")
-	return r.Replace(s)
+	return ipv4Regex.ReplaceAllStringFunc(r.Replace(s), func(ip string) string {
+		return strings.ReplaceAll(ip, ".", "_")
+	})
 }
 
 func GetHitsAddends(request *pb.RateLimitRequest) []uint64 {