Skip to content

Commit

Permalink
Merge pull request #885 from jkroepke/missing-cluster-label
Browse files Browse the repository at this point in the history 
Storage Alerts: Missing cluster label
  • Loading branch information
@povilasv
povilasv authored Dec 11, 2023
2 parents 2dbe4f9 + 6212c1e commit c72ac03
Show file tree
Hide file tree
Showing 3 changed files with 279 additions and 250 deletions.
26 changes: 13 additions & 13 deletions alerts/storage_alerts.libsonnet
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,17 +26,17 @@
) < 0.03
and
kubelet_volume_stats_used_bytes{%(prefixedNamespaceSelector)s%(kubeletSelector)s} > 0
unless on(namespace, persistentvolumeclaim)
unless on(%(clusterLabel)s, namespace, persistentvolumeclaim)
kube_persistentvolumeclaim_access_mode{%(prefixedNamespaceSelector)s access_mode="ReadOnlyMany"} == 1
unless on(namespace, persistentvolumeclaim)
unless on(%(clusterLabel)s, namespace, persistentvolumeclaim)
kube_persistentvolumeclaim_labels{%(prefixedNamespaceSelector)s%(pvExcludedSelector)s} == 1
||| % $._config,
'for': '1m',
labels: {
severity: 'critical',
},
annotations: {
description: 'The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} is only {{ $value | humanizePercentage }} free.',
description: 'The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} on Cluster {{ $labels.%(clusterLabel)s }} is only {{ $value | humanizePercentage }} free.' % $._config,
summary: 'PersistentVolume is filling up.',
},
},
Expand All @@ -52,17 +52,17 @@
kubelet_volume_stats_used_bytes{%(prefixedNamespaceSelector)s%(kubeletSelector)s} > 0
and
predict_linear(kubelet_volume_stats_available_bytes{%(prefixedNamespaceSelector)s%(kubeletSelector)s}[%(volumeFullPredictionSampleTime)s], 4 * 24 * 3600) < 0
unless on(namespace, persistentvolumeclaim)
unless on(%(clusterLabel)s, namespace, persistentvolumeclaim)
kube_persistentvolumeclaim_access_mode{%(prefixedNamespaceSelector)s access_mode="ReadOnlyMany"} == 1
unless on(namespace, persistentvolumeclaim)
unless on(%(clusterLabel)s, namespace, persistentvolumeclaim)
kube_persistentvolumeclaim_labels{%(prefixedNamespaceSelector)s%(pvExcludedSelector)s} == 1
||| % $._config,
'for': '1h',
labels: {
severity: 'warning',
},
annotations: {
description: 'Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} is expected to fill up within four days. Currently {{ $value | humanizePercentage }} is available.',
description: 'Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} on Cluster {{ $labels.%(clusterLabel)s }} is expected to fill up within four days. Currently {{ $value | humanizePercentage }} is available.' % $._config,
summary: 'PersistentVolume is filling up.',
},
},
Expand All @@ -76,17 +76,17 @@
) < 0.03
and
kubelet_volume_stats_inodes_used{%(prefixedNamespaceSelector)s%(kubeletSelector)s} > 0
unless on(namespace, persistentvolumeclaim)
unless on(%(clusterLabel)s, namespace, persistentvolumeclaim)
kube_persistentvolumeclaim_access_mode{%(prefixedNamespaceSelector)s access_mode="ReadOnlyMany"} == 1
unless on(namespace, persistentvolumeclaim)
unless on(%(clusterLabel)s, namespace, persistentvolumeclaim)
kube_persistentvolumeclaim_labels{%(prefixedNamespaceSelector)s%(pvExcludedSelector)s} == 1
||| % $._config,
'for': '1m',
labels: {
severity: 'critical',
},
annotations: {
description: 'The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} only has {{ $value | humanizePercentage }} free inodes.',
description: 'The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} on Cluster {{ $labels.%(clusterLabel)s }} only has {{ $value | humanizePercentage }} free inodes.' % $._config,
summary: 'PersistentVolumeInodes are filling up.',
},
},
Expand All @@ -102,17 +102,17 @@
kubelet_volume_stats_inodes_used{%(prefixedNamespaceSelector)s%(kubeletSelector)s} > 0
and
predict_linear(kubelet_volume_stats_inodes_free{%(prefixedNamespaceSelector)s%(kubeletSelector)s}[%(volumeFullPredictionSampleTime)s], 4 * 24 * 3600) < 0
unless on(namespace, persistentvolumeclaim)
unless on(%(clusterLabel)s, namespace, persistentvolumeclaim)
kube_persistentvolumeclaim_access_mode{%(prefixedNamespaceSelector)s access_mode="ReadOnlyMany"} == 1
unless on(namespace, persistentvolumeclaim)
unless on(%(clusterLabel)s, namespace, persistentvolumeclaim)
kube_persistentvolumeclaim_labels{%(prefixedNamespaceSelector)s%(pvExcludedSelector)s} == 1
||| % $._config,
'for': '1h',
labels: {
severity: 'warning',
},
annotations: {
description: 'Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} is expected to run out of inodes within four days. Currently {{ $value | humanizePercentage }} of its inodes are free.',
description: 'Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} on Cluster {{ $labels.%(clusterLabel)s }} is expected to run out of inodes within four days. Currently {{ $value | humanizePercentage }} of its inodes are free.' % $._config,
summary: 'PersistentVolumeInodes are filling up.',
},
},
Expand All @@ -126,7 +126,7 @@
severity: 'critical',
},
annotations: {
description: 'The persistent volume {{ $labels.persistentvolume }} has status {{ $labels.phase }}.',
description: 'The persistent volume {{ $labels.persistentvolume }} on Cluster {{ $labels.%(clusterLabel)s }} has status {{ $labels.phase }}.' % $._config,
summary: 'PersistentVolume is having issues with provisioning.',
},
},
Expand Down
4 changes: 2 additions & 2 deletions rules/node.libsonnet
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,8 @@
expr: |||
count by (%(clusterLabel)s, node) (
node_cpu_seconds_total{mode="idle",%(nodeExporterSelector)s}
* on (namespace, %(podLabel)s) group_left(node)
topk by(namespace, %(podLabel)s) (1, node_namespace_pod:kube_pod_info:)
* on (%(clusterLabel)s, namespace, %(podLabel)s) group_left(node)
topk by(%(clusterLabel)s, namespace, %(podLabel)s) (1, node_namespace_pod:kube_pod_info:)
)
||| % $._config,
},
Expand Down
Loading

0 comments on commit c72ac03

Please sign in to comment.