From de8329f2b672920421fa49f5be21050aff58e3a4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan-Otto=20Kr=C3=B6pke?= <mail@jkroepke.de>
Date: Thu, 23 Nov 2023 19:45:25 +0100
Subject: [PATCH] Storage Alerts: Missing cluster label
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Jan-Otto Kröpke <mail@jkroepke.de>
---
 alerts/storage_alerts.libsonnet | 26 +++++++++++++-------------
 rules/node.libsonnet            |  4 ++--
 tests.yaml                      | 18 +++++++++---------
 3 files changed, 24 insertions(+), 24 deletions(-)

diff --git a/alerts/storage_alerts.libsonnet b/alerts/storage_alerts.libsonnet
index ce028683f..ebeed2b85 100644
--- a/alerts/storage_alerts.libsonnet
+++ b/alerts/storage_alerts.libsonnet
@@ -26,9 +26,9 @@
               ) < 0.03
               and
               kubelet_volume_stats_used_bytes{%(prefixedNamespaceSelector)s%(kubeletSelector)s} > 0
-              unless on(namespace, persistentvolumeclaim)
+              unless on(%(clusterLabel)s, namespace, persistentvolumeclaim)
               kube_persistentvolumeclaim_access_mode{%(prefixedNamespaceSelector)s access_mode="ReadOnlyMany"} == 1
-              unless on(namespace, persistentvolumeclaim)
+              unless on(%(clusterLabel)s, namespace, persistentvolumeclaim)
               kube_persistentvolumeclaim_labels{%(prefixedNamespaceSelector)s%(pvExcludedSelector)s} == 1
             ||| % $._config,
             'for': '1m',
@@ -36,7 +36,7 @@
               severity: 'critical',
             },
             annotations: {
-              description: 'The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} is only {{ $value | humanizePercentage }} free.',
+              description: 'The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} on Cluster {{ $labels.%(clusterLabel)s }} is only {{ $value | humanizePercentage }} free.' % $._config,
               summary: 'PersistentVolume is filling up.',
             },
           },
@@ -52,9 +52,9 @@
               kubelet_volume_stats_used_bytes{%(prefixedNamespaceSelector)s%(kubeletSelector)s} > 0
               and
               predict_linear(kubelet_volume_stats_available_bytes{%(prefixedNamespaceSelector)s%(kubeletSelector)s}[%(volumeFullPredictionSampleTime)s], 4 * 24 * 3600) < 0
-              unless on(namespace, persistentvolumeclaim)
+              unless on(%(clusterLabel)s, namespace, persistentvolumeclaim)
               kube_persistentvolumeclaim_access_mode{%(prefixedNamespaceSelector)s access_mode="ReadOnlyMany"} == 1
-              unless on(namespace, persistentvolumeclaim)
+              unless on(%(clusterLabel)s, namespace, persistentvolumeclaim)
               kube_persistentvolumeclaim_labels{%(prefixedNamespaceSelector)s%(pvExcludedSelector)s} == 1
             ||| % $._config,
             'for': '1h',
@@ -62,7 +62,7 @@
               severity: 'warning',
             },
             annotations: {
-              description: 'Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} is expected to fill up within four days. Currently {{ $value | humanizePercentage }} is available.',
+              description: 'Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} on Cluster {{ $labels.%(clusterLabel)s }} is expected to fill up within four days. Currently {{ $value | humanizePercentage }} is available.' % $._config,
               summary: 'PersistentVolume is filling up.',
             },
           },
@@ -76,9 +76,9 @@
               ) < 0.03
               and
               kubelet_volume_stats_inodes_used{%(prefixedNamespaceSelector)s%(kubeletSelector)s} > 0
-              unless on(namespace, persistentvolumeclaim)
+              unless on(%(clusterLabel)s, namespace, persistentvolumeclaim)
               kube_persistentvolumeclaim_access_mode{%(prefixedNamespaceSelector)s access_mode="ReadOnlyMany"} == 1
-              unless on(namespace, persistentvolumeclaim)
+              unless on(%(clusterLabel)s, namespace, persistentvolumeclaim)
               kube_persistentvolumeclaim_labels{%(prefixedNamespaceSelector)s%(pvExcludedSelector)s} == 1
             ||| % $._config,
             'for': '1m',
@@ -86,7 +86,7 @@
               severity: 'critical',
             },
             annotations: {
-              description: 'The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} only has {{ $value | humanizePercentage }} free inodes.',
+              description: 'The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} on Cluster {{ $labels.%(clusterLabel)s }} only has {{ $value | humanizePercentage }} free inodes.' % $._config,
               summary: 'PersistentVolumeInodes are filling up.',
             },
           },
@@ -102,9 +102,9 @@
               kubelet_volume_stats_inodes_used{%(prefixedNamespaceSelector)s%(kubeletSelector)s} > 0
               and
               predict_linear(kubelet_volume_stats_inodes_free{%(prefixedNamespaceSelector)s%(kubeletSelector)s}[%(volumeFullPredictionSampleTime)s], 4 * 24 * 3600) < 0
-              unless on(namespace, persistentvolumeclaim)
+              unless on(%(clusterLabel)s, namespace, persistentvolumeclaim)
               kube_persistentvolumeclaim_access_mode{%(prefixedNamespaceSelector)s access_mode="ReadOnlyMany"} == 1
-              unless on(namespace, persistentvolumeclaim)
+              unless on(%(clusterLabel)s, namespace, persistentvolumeclaim)
               kube_persistentvolumeclaim_labels{%(prefixedNamespaceSelector)s%(pvExcludedSelector)s} == 1
             ||| % $._config,
             'for': '1h',
@@ -112,7 +112,7 @@
               severity: 'warning',
             },
             annotations: {
-              description: 'Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} is expected to run out of inodes within four days. Currently {{ $value | humanizePercentage }} of its inodes are free.',
+              description: 'Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} on Cluster {{ $labels.%(clusterLabel)s }} is expected to run out of inodes within four days. Currently {{ $value | humanizePercentage }} of its inodes are free.' % $._config,
               summary: 'PersistentVolumeInodes are filling up.',
             },
           },
@@ -126,7 +126,7 @@
               severity: 'critical',
             },
             annotations: {
-              description: 'The persistent volume {{ $labels.persistentvolume }} has status {{ $labels.phase }}.',
+              description: 'The persistent volume {{ $labels.persistentvolume }} on Cluster {{ $labels.%(clusterLabel)s }} has status {{ $labels.phase }}.' % $._config,
               summary: 'PersistentVolume is having issues with provisioning.',
             },
           },
diff --git a/rules/node.libsonnet b/rules/node.libsonnet
index e4e949bd7..453e3720b 100644
--- a/rules/node.libsonnet
+++ b/rules/node.libsonnet
@@ -32,8 +32,8 @@
             expr: |||
               count by (%(clusterLabel)s, node) (
                 node_cpu_seconds_total{mode="idle",%(nodeExporterSelector)s}
-                * on (namespace, %(podLabel)s) group_left(node)
-                topk by(namespace, %(podLabel)s) (1, node_namespace_pod:kube_pod_info:)
+                * on (%(clusterLabel)s, namespace, %(podLabel)s) group_left(node)
+                topk by(%(clusterLabel)s, namespace, %(podLabel)s) (1, node_namespace_pod:kube_pod_info:)
               )
             ||| % $._config,
           },
diff --git a/tests.yaml b/tests.yaml
index a2fb0bb0f..e97a17d7e 100644
--- a/tests.yaml
+++ b/tests.yaml
@@ -33,7 +33,7 @@ tests:
         severity: critical
       exp_annotations:
         summary: "PersistentVolume is filling up."
-        description: 'The PersistentVolume claimed by somepvc in Namespace monitoring is only 1.562% free.'
+        description: 'The PersistentVolume claimed by somepvc in Namespace monitoring on Cluster  is only 1.562% free.'
         runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumefillingup
 
 # Don't alert when PVC access_mode is ReadOnlyMany
@@ -119,7 +119,7 @@ tests:
         severity: critical
       exp_annotations:
         summary: "PersistentVolume is filling up."
-        description: 'The PersistentVolume claimed by somepvc in Namespace monitoring is only 1.294% free.'
+        description: 'The PersistentVolume claimed by somepvc in Namespace monitoring on Cluster  is only 1.294% free.'
         runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumefillingup
 
 - interval: 1m
@@ -143,7 +143,7 @@ tests:
         severity: warning
       exp_annotations:
         summary: "PersistentVolume is filling up."
-        description: 'Based on recent sampling, the PersistentVolume claimed by somepvc in Namespace monitoring is expected to fill up within four days. Currently 1.263% is available.'
+        description: 'Based on recent sampling, the PersistentVolume claimed by somepvc in Namespace monitoring on Cluster  is expected to fill up within four days. Currently 1.263% is available.'
         runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumefillingup
     - exp_labels:
         job: kubelet
@@ -152,7 +152,7 @@ tests:
         severity: critical
       exp_annotations:
         summary: "PersistentVolume is filling up."
-        description: 'The PersistentVolume claimed by somepvc in Namespace monitoring is only 1.263% free.'
+        description: 'The PersistentVolume claimed by somepvc in Namespace monitoring on Cluster  is only 1.263% free.'
         runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumefillingup
 
 # Block volume mounts can report 0 for the kubelet_volume_stats_used_bytes metric but it shouldn't trigger the KubePersistentVolumeFillingUp alert.
@@ -228,7 +228,7 @@ tests:
         severity: critical
       exp_annotations:
         summary: "PersistentVolumeInodes are filling up."
-        description: 'The PersistentVolume claimed by somepvc in Namespace monitoring only has 1.562% free inodes.'
+        description: 'The PersistentVolume claimed by somepvc in Namespace monitoring on Cluster  only has 1.562% free inodes.'
         runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumeinodesfillingup
 
 # Don't alert when PVC access_mode is ReadOnlyMany
@@ -314,7 +314,7 @@ tests:
         severity: critical
       exp_annotations:
         summary: "PersistentVolumeInodes are filling up."
-        description: 'The PersistentVolume claimed by somepvc in Namespace monitoring only has 1.294% free inodes.'
+        description: 'The PersistentVolume claimed by somepvc in Namespace monitoring on Cluster  only has 1.294% free inodes.'
         runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumeinodesfillingup
 
 - interval: 1m
@@ -338,7 +338,7 @@ tests:
         severity: warning
       exp_annotations:
         summary: "PersistentVolumeInodes are filling up."
-        description: 'Based on recent sampling, the PersistentVolume claimed by somepvc in Namespace monitoring is expected to run out of inodes within four days. Currently 1.263% of its inodes are free.'
+        description: 'Based on recent sampling, the PersistentVolume claimed by somepvc in Namespace monitoring on Cluster  is expected to run out of inodes within four days. Currently 1.263% of its inodes are free.'
         runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumeinodesfillingup
     - exp_labels:
         job: kubelet
@@ -347,7 +347,7 @@ tests:
         severity: critical
       exp_annotations:
         summary: "PersistentVolumeInodes are filling up."
-        description: 'The PersistentVolume claimed by somepvc in Namespace monitoring only has 1.263% free inodes.'
+        description: 'The PersistentVolume claimed by somepvc in Namespace monitoring on Cluster  only has 1.263% free inodes.'
         runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumeinodesfillingup
 
 # Block volume mounts can report 0 for the kubelet_volume_stats_inodes_used metric but it shouldn't trigger the KubePersistentVolumeInodesFillingUp alert.
@@ -1137,7 +1137,7 @@ tests:
         runbook_url: "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodcrashlooping"
         summary: "Pod is crash looping."
   - eval_time: 20m
-    alertname: KubePodCrashLooping   # alert fired for a period of 5 minutes after resolution because the alert looks back at the last 5 minutes of data and the range vector doesn't take stale samples into account 
+    alertname: KubePodCrashLooping   # alert fired for a period of 5 minutes after resolution because the alert looks back at the last 5 minutes of data and the range vector doesn't take stale samples into account
     exp_alerts:
       - exp_labels:
           severity: "warning"