Skip to content

Commit

Permalink
Merge pull request #305 from 3scale-ops/feat/server-alias-for-sentine…
Browse files Browse the repository at this point in the history 
…l-metrics

Add a label with the server alias information for all Sentinel generated metrics
  • Loading branch information
@3scale-robot
3scale-robot authored Sep 6, 2024
2 parents a10ed06 + ac17a97 commit 60cb73c
Show file tree
Hide file tree
Showing 5 changed files with 143 additions and 79 deletions.
66 changes: 33 additions & 33 deletions pkg/assets/dashboards/redis-sentinel.json.gtpl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,38 +98,38 @@
"targets": [
{
"exemplar": true,
"expr": "saas_redis_sentinel_server_info and on(namespace,sentinel,redis_server) rate(saas_redis_sentinel_sdown_count{namespace='$namespace',sentinel=~\"[[sentinel]].*\"}[1m]) > 0",
"expr": "saas_redis_sentinel_server_info and on(namespace,sentinel,redis_server_alias) rate(saas_redis_sentinel_sdown_count{namespace='$namespace',sentinel=~\"[[sentinel]].*\"}[1m]) > 0",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{`down: {{shard}}/{{redis_server}}/{{role}}`}}",
"legendFormat": "{{`down: {{shard}}/{{redis_server_alias}}/{{role}}`}}",
"refId": "A"
},
{
"exemplar": true,
"expr": "saas_redis_sentinel_server_info and on(namespace,sentinel,redis_server) rate(saas_redis_sentinel_sdown_cleared_count{namespace='$namespace',sentinel=~\"[[sentinel]].*\"}[1m]) > 0",
"expr": "saas_redis_sentinel_server_info and on(namespace,sentinel,redis_server_alias) rate(saas_redis_sentinel_sdown_cleared_count{namespace='$namespace',sentinel=~\"[[sentinel]].*\"}[1m]) > 0",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{`up: {{shard}}/{{redis_server}}/{{role}}`}}",
"legendFormat": "{{`up: {{shard}}/{{redis_server_alias}}/{{role}}`}}",
"refId": "B"
},
{
"exemplar": true,
"expr": "saas_redis_sentinel_server_info and on(namespace,sentinel,redis_server) rate(saas_redis_sentinel_sdown_sentinel_count{namespace='$namespace',sentinel=~\"[[sentinel]].*\"}[1m]) > 0",
"expr": "saas_redis_sentinel_server_info and on(namespace,sentinel,redis_server_alias) rate(saas_redis_sentinel_sdown_sentinel_count{namespace='$namespace',sentinel=~\"[[sentinel]].*\"}[1m]) > 0",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "down: sentinel/{{`{{redis_server}}`}}",
"legendFormat": "down: sentinel/{{`{{redis_server_alias}}`}}",
"refId": "C"
},
{
"exemplar": true,
"expr": "saas_redis_sentinel_server_info and on(namespace,sentinel,redis_server) rate(saas_redis_sentinel_sdown_cleared_sentinel_count{namespace='$namespace',sentinel=~\"[[sentinel]].*\"}[1m]) > 0",
"expr": "saas_redis_sentinel_server_info and on(namespace,sentinel,redis_server_alias) rate(saas_redis_sentinel_sdown_cleared_sentinel_count{namespace='$namespace',sentinel=~\"[[sentinel]].*\"}[1m]) > 0",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "up: sentinel/{{`{{redis_server}}`}}",
"legendFormat": "up: sentinel/{{`{{redis_server_alias}}`}}",
"refId": "D"
}
],
Expand Down Expand Up @@ -232,11 +232,11 @@
"targets": [
{
"exemplar": true,
"expr": "saas_redis_sentinel_server_info{role='master'} and on(namespace,sentinel,shard,redis_server) rate(saas_redis_sentinel_switch_master_count{namespace='$namespace',sentinel=~\"[[sentinel]].*\"}[1m]) > 0",
"expr": "saas_redis_sentinel_server_info{role='master'} and on(namespace,sentinel,shard,redis_server_alias) rate(saas_redis_sentinel_switch_master_count{namespace='$namespace',sentinel=~\"[[sentinel]].*\"}[1m]) > 0",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{`{{shard}}/{{redis_server}}`}}",
"legendFormat": "{{`{{shard}}/{{redis_server_alias}}`}}",
"refId": "A"
}
],
Expand Down Expand Up @@ -450,7 +450,7 @@
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{`{{shard}}/{{redis_server}}/{{role}}`}}",
"legendFormat": "{{`{{shard}}/{{redis_server_alias}}/{{role}}`}}",
"refId": "A"
}
],
Expand Down Expand Up @@ -565,7 +565,7 @@
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{`{{shard}}/{{redis_server}}/{{role}}`}}",
"legendFormat": "{{`{{shard}}/{{redis_server_alias}}/{{role}}`}}",
"refId": "A"
}
],
Expand Down Expand Up @@ -670,7 +670,7 @@
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{`{{shard}}/{{redis_server}}/{{role}}`}}",
"legendFormat": "{{`{{shard}}/{{redis_server_alias}}/{{role}}`}}",
"refId": "A"
}
],
Expand Down Expand Up @@ -777,7 +777,7 @@
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{`{{shard}}/{{redis_server}}/{{role}}`}}",
"legendFormat": "{{`{{shard}}/{{redis_server_alias}}/{{role}}`}}",
"refId": "A"
}
],
Expand Down Expand Up @@ -1096,7 +1096,7 @@
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{`{{shard}}/{{redis_server}}/{{role}}`}}",
"legendFormat": "{{`{{shard}}/{{redis_server_alias}}/{{role}}`}}",
"refId": "A"
}
],
Expand Down Expand Up @@ -1213,20 +1213,20 @@
"targets": [
{
"exemplar": true,
"expr": "saas_redis_sentinel_server_info and on(namespace,sentinel,redis_server) rate(saas_redis_sentinel_sdown_count{namespace='$namespace',sentinel=~\"[[sentinel]].*\"}[1m]) > 0",
"expr": "saas_redis_sentinel_server_info and on(namespace,sentinel,redis_server_alias) rate(saas_redis_sentinel_sdown_count{namespace='$namespace',sentinel=~\"[[sentinel]].*\"}[1m]) > 0",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{`{{shard}}/{{redis_server}}`}}",
"legendFormat": "{{`{{shard}}/{{redis_server_alias}}`}}",
"refId": "A"
},
{
"exemplar": true,
"expr": "saas_redis_sentinel_server_info and on(namespace,sentinel,redis_server) rate(saas_redis_sentinel_sdown_sentinel_count{namespace='$namespace',sentinel=~\"[[sentinel]].*\"}[1m]) > 0",
"expr": "saas_redis_sentinel_server_info and on(namespace,sentinel,redis_server_alias) rate(saas_redis_sentinel_sdown_sentinel_count{namespace='$namespace',sentinel=~\"[[sentinel]].*\"}[1m]) > 0",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "sentinel/{{`{{redis_server}}`}}",
"legendFormat": "sentinel/{{`{{redis_server_alias}}`}}",
"refId": "B"
}
],
Expand Down Expand Up @@ -1556,7 +1556,7 @@
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{`{{shard}}/{{redis_server}}/{{role}}`}}",
"legendFormat": "{{`{{shard}}/{{redis_server_alias}}/{{role}}`}}",
"refId": "A"
}
],
Expand Down Expand Up @@ -1674,7 +1674,7 @@
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{`{{shard}}/{{redis_server}}/{{role}}`}}",
"legendFormat": "{{`{{shard}}/{{redis_server_alias}}/{{role}}`}}",
"refId": "A"
}
],
Expand Down Expand Up @@ -1782,7 +1782,7 @@
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{`{{shard}}/{{redis_server}}/{{role}}`}}",
"legendFormat": "{{`{{shard}}/{{redis_server_alias}}/{{role}}`}}",
"refId": "A"
}
],
Expand Down Expand Up @@ -1892,7 +1892,7 @@
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{`{{shard}}/{{redis_server}}/{{role}}`}}",
"legendFormat": "{{`{{shard}}/{{redis_server_alias}}/{{role}}`}}",
"refId": "A"
}
],
Expand Down Expand Up @@ -2220,7 +2220,7 @@
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{`{{shard}}/{{redis_server}}/{{role}}`}}",
"legendFormat": "{{`{{shard}}/{{redis_server_alias}}/{{role}}`}}",
"refId": "A"
}
],
Expand Down Expand Up @@ -2350,20 +2350,20 @@
"targets": [
{
"exemplar": true,
"expr": "saas_redis_sentinel_server_info and on(namespace,sentinel,redis_server) rate(saas_redis_sentinel_sdown_count{namespace='$namespace',sentinel=~\"[[sentinel]].*\"}[1m]) > 0",
"expr": "saas_redis_sentinel_server_info and on(namespace,sentinel,redis_server_alias) rate(saas_redis_sentinel_sdown_count{namespace='$namespace',sentinel=~\"[[sentinel]].*\"}[1m]) > 0",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{`{{shard}}-{{redis_server}}`}}",
"legendFormat": "{{`{{shard}}-{{redis_server_alias}}`}}",
"refId": "A"
},
{
"exemplar": true,
"expr": "saas_redis_sentinel_server_info and on(namespace,sentinel,redis_server) rate(saas_redis_sentinel_sdown_sentinel_count{namespace='$namespace',sentinel=~\"[[sentinel]].*\"}[1m]) > 0",
"expr": "saas_redis_sentinel_server_info and on(namespace,sentinel,redis_server_alias) rate(saas_redis_sentinel_sdown_sentinel_count{namespace='$namespace',sentinel=~\"[[sentinel]].*\"}[1m]) > 0",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "sentinel-{{`{{redis_server}}`}}",
"legendFormat": "sentinel-{{`{{redis_server_alias}}`}}",
"refId": "B"
}
],
Expand Down Expand Up @@ -2693,7 +2693,7 @@
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{`{{shard}}/{{redis_server}}/{{role}}`}}",
"legendFormat": "{{`{{shard}}/{{redis_server_alias}}/{{role}}`}}",
"refId": "A"
}
],
Expand Down Expand Up @@ -2811,7 +2811,7 @@
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{`{{shard}}/{{redis_server}}/{{role}}`}}",
"legendFormat": "{{`{{shard}}/{{redis_server_alias}}/{{role}}`}}",
"refId": "A"
}
],
Expand Down Expand Up @@ -2919,7 +2919,7 @@
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{`{{shard}}/{{redis_server}}/{{role}}`}}",
"legendFormat": "{{`{{shard}}/{{redis_server_alias}}/{{role}}`}}",
"refId": "A"
}
],
Expand Down Expand Up @@ -3029,7 +3029,7 @@
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{`{{shard}}/{{redis_server}}/{{role}}`}}",
"legendFormat": "{{`{{shard}}/{{redis_server_alias}}/{{role}}`}}",
"refId": "A"
}
],
Expand Down Expand Up @@ -3357,7 +3357,7 @@
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{`{{shard}}/{{redis_server}}/{{role}}`}}",
"legendFormat": "{{`{{shard}}/{{redis_server_alias}}/{{role}}`}}",
"refId": "A"
}
],
Expand Down
32 changes: 20 additions & 12 deletions pkg/redis/events/watcher.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,31 +38,31 @@ var (
Namespace: "saas_redis_sentinel",
Help: "+sdown (https://redis.io/topics/sentinel#sentinel-api)",
},
[]string{"sentinel", "shard", "redis_server"},
[]string{"sentinel", "shard", "redis_server_host", "redis_server_alias"},
)
sdownSentinelCount = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "sdown_sentinel_count",
Namespace: "saas_redis_sentinel",
Help: "+sdown (https://redis.io/topics/sentinel#sentinel-api)",
},
[]string{"sentinel", "shard", "redis_server"},
[]string{"sentinel", "shard", "redis_server_host", "redis_server_alias"},
)
sdownClearedCount = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "sdown_cleared_count",
Namespace: "saas_redis_sentinel",
Help: "-sdown (https://redis.io/topics/sentinel#sentinel-api)",
},
[]string{"sentinel", "shard", "redis_server"},
[]string{"sentinel", "shard", "redis_server_host", "redis_server_alias"},
)
sdownClearedSentinelCount = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "sdown_cleared_sentinel_count",
Namespace: "saas_redis_sentinel",
Help: "-sdown (https://redis.io/topics/sentinel#sentinel-api)",
},
[]string{"sentinel", "shard", "redis_server"},
[]string{"sentinel", "shard", "redis_server_host", "redis_server_alias"},
)
)

Expand Down Expand Up @@ -208,14 +208,16 @@ func (sew *SentinelEventWatcher) metricsFromEvent(rem RedisEventMessage) {
sdownSentinelCount.With(
prometheus.Labels{
"sentinel": sew.sentinelURI, "shard": rem.master.name,
"redis_server": fmt.Sprintf("%s:%s", rem.target.ip, rem.target.port),
"redis_server_host": fmt.Sprintf("%s:%s", rem.target.ip, rem.target.port),
"redis_server_alias": sew.topology.GetPool().GetServerAlias(fmt.Sprintf("%s:%s", rem.target.ip, rem.target.port)),
},
).Add(1)
default:
sdownCount.With(
prometheus.Labels{
"sentinel": sew.sentinelURI, "shard": rem.master.name,
"redis_server": fmt.Sprintf("%s:%s", rem.target.ip, rem.target.port),
"redis_server_host": fmt.Sprintf("%s:%s", rem.target.ip, rem.target.port),
"redis_server_alias": sew.topology.GetPool().GetServerAlias(fmt.Sprintf("%s:%s", rem.target.ip, rem.target.port)),
},
).Add(1)
}
Expand All @@ -225,14 +227,16 @@ func (sew *SentinelEventWatcher) metricsFromEvent(rem RedisEventMessage) {
sdownClearedSentinelCount.With(
prometheus.Labels{
"sentinel": sew.sentinelURI, "shard": rem.master.name,
"redis_server": fmt.Sprintf("%s:%s", rem.target.ip, rem.target.port),
"redis_server_host": fmt.Sprintf("%s:%s", rem.target.ip, rem.target.port),
"redis_server_alias": sew.topology.GetPool().GetServerAlias(fmt.Sprintf("%s:%s", rem.target.ip, rem.target.port)),
},
).Add(1)
default:
sdownClearedCount.With(
prometheus.Labels{
"sentinel": sew.sentinelURI, "shard": rem.master.name,
"redis_server": fmt.Sprintf("%s:%s", rem.target.ip, rem.target.port),
"redis_server_host": fmt.Sprintf("%s:%s", rem.target.ip, rem.target.port),
"redis_server_alias": sew.topology.GetPool().GetServerAlias(fmt.Sprintf("%s:%s", rem.target.ip, rem.target.port)),
},
).Add(1)
}
Expand All @@ -258,25 +262,29 @@ func (sew *SentinelEventWatcher) initCounters() {
sdownSentinelCount.With(
prometheus.Labels{
"sentinel": sew.sentinelURI, "shard": shard.Name,
"redis_server": server.ID(),
"redis_server_host": server.ID(),
"redis_server_alias": sew.topology.GetPool().GetServerAlias(server.ID()),
},
).Add(0)
sdownCount.With(
prometheus.Labels{
"sentinel": sew.sentinelURI, "shard": shard.Name,
"redis_server": server.ID(),
"redis_server_host": server.ID(),
"redis_server_alias": sew.topology.GetPool().GetServerAlias(server.ID()),
},
).Add(0)
sdownClearedSentinelCount.With(
prometheus.Labels{
"sentinel": sew.sentinelURI, "shard": shard.Name,
"redis_server": server.ID(),
"redis_server_host": server.ID(),
"redis_server_alias": sew.topology.GetPool().GetServerAlias(server.ID()),
},
).Add(0)
sdownClearedCount.With(
prometheus.Labels{
"sentinel": sew.sentinelURI, "shard": shard.Name,
"redis_server": server.ID(),
"redis_server_host": server.ID(),
"redis_server_alias": sew.topology.GetPool().GetServerAlias(server.ID()),
},
).Add(0)
}
Expand Down
Loading

0 comments on commit 60cb73c

Please sign in to comment.