From c56849bcc3bd92891b2967640e0ddf43a97ee0c0 Mon Sep 17 00:00:00 2001 From: Sebastian Gaiser Date: Thu, 5 Dec 2024 20:55:20 +0000 Subject: [PATCH 1/6] fix(alerts): use sum by instance max for 'KubeAggregatedAPIErrors' Signed-off-by: Sebastian Gaiser --- alerts/kube_apiserver.libsonnet | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerts/kube_apiserver.libsonnet b/alerts/kube_apiserver.libsonnet index 558c2acd2..37bb693fd 100644 --- a/alerts/kube_apiserver.libsonnet +++ b/alerts/kube_apiserver.libsonnet @@ -88,7 +88,7 @@ local utils = import '../lib/utils.libsonnet'; { alert: 'KubeAggregatedAPIErrors', expr: ||| - sum by(name, namespace, %(clusterLabel)s)(increase(aggregator_unavailable_apiservice_total{%(kubeApiserverSelector)s}[10m])) > 4 + sum by (instance) (max by(name, namespace, %(clusterLabel)s)(increase(aggregator_unavailable_apiservice_total{%(kubeApiserverSelector)s}[10m]))) > 4 ||| % $._config, labels: { severity: 'warning', From 9f6803e685ad50808c06b7067c99dbe88ca170b8 Mon Sep 17 00:00:00 2001 From: Sebastian Gaiser Date: Mon, 9 Dec 2024 16:22:25 +0100 Subject: [PATCH 2/6] Update alerts/kube_apiserver.libsonnet Co-authored-by: Stephen Lang --- alerts/kube_apiserver.libsonnet | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerts/kube_apiserver.libsonnet b/alerts/kube_apiserver.libsonnet index 37bb693fd..1fdda4fb7 100644 --- a/alerts/kube_apiserver.libsonnet +++ b/alerts/kube_apiserver.libsonnet @@ -88,7 +88,7 @@ local utils = import '../lib/utils.libsonnet'; { alert: 'KubeAggregatedAPIErrors', expr: ||| - sum by (instance) (max by(name, namespace, %(clusterLabel)s)(increase(aggregator_unavailable_apiservice_total{%(kubeApiserverSelector)s}[10m]))) > 4 + sum by (%(clusterLabel)s, instance, name, reason)(increase(aggregator_unavailable_apiservice_total{%(kubeApiserverSelector)s}[1m])) > 0 ||| % $._config, labels: { severity: 'warning', From 13a4c2ed2ab62c7a9982d0af9c1cd6205a34e9b9 Mon Sep 17 00:00:00 2001 From: Sebastian Gaiser Date: Mon, 9 Dec 2024 16:22:45 +0100 Subject: [PATCH 3/6] Update alerts/kube_apiserver.libsonnet --- alerts/kube_apiserver.libsonnet | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerts/kube_apiserver.libsonnet b/alerts/kube_apiserver.libsonnet index 1fdda4fb7..b6e93217e 100644 --- a/alerts/kube_apiserver.libsonnet +++ b/alerts/kube_apiserver.libsonnet @@ -88,7 +88,7 @@ local utils = import '../lib/utils.libsonnet'; { alert: 'KubeAggregatedAPIErrors', expr: ||| - sum by (%(clusterLabel)s, instance, name, reason)(increase(aggregator_unavailable_apiservice_total{%(kubeApiserverSelector)s}[1m])) > 0 + sum by(%(clusterLabel)s, instance, name, reason)(increase(aggregator_unavailable_apiservice_total{%(kubeApiserverSelector)s}[1m])) > 0 ||| % $._config, labels: { severity: 'warning', From 394294190791ef4fac5b0bc92a5a733d71cbfc1a Mon Sep 17 00:00:00 2001 From: Sebastian Gaiser Date: Mon, 9 Dec 2024 16:24:07 +0100 Subject: [PATCH 4/6] Update alerts/kube_apiserver.libsonnet --- alerts/kube_apiserver.libsonnet | 1 + 1 file changed, 1 insertion(+) diff --git a/alerts/kube_apiserver.libsonnet b/alerts/kube_apiserver.libsonnet index b6e93217e..65497b607 100644 --- a/alerts/kube_apiserver.libsonnet +++ b/alerts/kube_apiserver.libsonnet @@ -90,6 +90,7 @@ local utils = import '../lib/utils.libsonnet'; expr: ||| sum by(%(clusterLabel)s, instance, name, reason)(increase(aggregator_unavailable_apiservice_total{%(kubeApiserverSelector)s}[1m])) > 0 ||| % $._config, + 'for': '10m', labels: { severity: 'warning', }, From 7e839b293b0116b91df8a6c21c767f9ed456bedc Mon Sep 17 00:00:00 2001 From: Sebastian Gaiser Date: Mon, 9 Dec 2024 15:29:30 +0000 Subject: [PATCH 5/6] fix(alerts): use sum by instance max for 'KubeAggregatedAPIErrors' - adjust description Signed-off-by: Sebastian Gaiser --- alerts/kube_apiserver.libsonnet | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/alerts/kube_apiserver.libsonnet b/alerts/kube_apiserver.libsonnet index 65497b607..e7348d488 100644 --- a/alerts/kube_apiserver.libsonnet +++ b/alerts/kube_apiserver.libsonnet @@ -95,7 +95,9 @@ local utils = import '../lib/utils.libsonnet'; severity: 'warning', }, annotations: { - description: 'Kubernetes aggregated API {{ $labels.name }}/{{ $labels.namespace }} has reported errors. It has appeared unavailable {{ $value | humanize }} times averaged over the past 10m.', + description: 'Kubernetes aggregated API {{ $labels.instance }}/{{ $labels.name }} has reported {{ $labels.reason }} errors%s.' % [ + (utils.ifShowMultiCluster($._config, ' on cluster {{ $labels.%(clusterLabel)s }}' % $._config), + ], summary: 'Kubernetes aggregated API has reported errors.', }, }, From 21c0b8720e952aa3a02436f22926b1453a7707f4 Mon Sep 17 00:00:00 2001 From: Stephen Lang Date: Tue, 10 Dec 2024 18:15:19 +0000 Subject: [PATCH 6/6] fix(build): remove extra parenthsis Signed-off-by: Stephen Lang --- alerts/kube_apiserver.libsonnet | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerts/kube_apiserver.libsonnet b/alerts/kube_apiserver.libsonnet index e7348d488..d67bbf3ed 100644 --- a/alerts/kube_apiserver.libsonnet +++ b/alerts/kube_apiserver.libsonnet @@ -96,7 +96,7 @@ local utils = import '../lib/utils.libsonnet'; }, annotations: { description: 'Kubernetes aggregated API {{ $labels.instance }}/{{ $labels.name }} has reported {{ $labels.reason }} errors%s.' % [ - (utils.ifShowMultiCluster($._config, ' on cluster {{ $labels.%(clusterLabel)s }}' % $._config), + utils.ifShowMultiCluster($._config, ' on cluster {{ $labels.%(clusterLabel)s }}' % $._config), ], summary: 'Kubernetes aggregated API has reported errors.', },