Merge pull request #16 from adinhodovic/add-alerts-for-config-reload-…

…fails feat: Add alerts for config reload failures
adinhodovic · Mar 24, 2024 · d1be06c · d1be06c
2 parents bd02e45 + a7fb606
commit d1be06c
Show file tree

Hide file tree

Showing 6 changed files with 495 additions and 135 deletions.
diff --git a/alerts/alerts.libsonnet b/alerts/alerts.libsonnet
@@ -4,6 +4,24 @@
  {
  name: 'nginx.rules',
  rules: [
+ {
+ alert: 'NginxConfigReloadFailed',
+ expr: |||
+ sum(
+ nginx_ingress_controller_config_last_reload_successful{%(ingressNginxSelector)s}
+ ) by (job, controller_class)
+ == 0
+ ||| % $._config,
+ 'for': '5m',
+ labels: {
+ severity: 'warning',
+ },
+ annotations: {
+ summary: 'Nginx config reload failed.',
+ description: 'Nginx config reload failed for the controller with the class {{ $labels.controller_class }}.',
+ dashboard_url: $._config.overviewDashboardUrl + '?var-job={{ $labels.job }}&var-controller_class={{ $labels.controller_class }}',
+ },
+ },
  {
  alert: 'NginxHighHttp4xxErrorRate',
  expr: |||

diff --git a/dashboards_out/ingress-nginx-overview.json b/dashboards_out/ingress-nginx-overview.json
@@ -63,7 +63,7 @@
  ]
  }
  },
- "pluginVersion": "v10.2.0",
+ "pluginVersion": "v10.4.0",
  "targets": [
  {
  "datasource": {
@@ -112,7 +112,7 @@
  ]
  }
  },
- "pluginVersion": "v10.2.0",
+ "pluginVersion": "v10.4.0",
  "targets": [
  {
  "datasource": {
@@ -165,7 +165,7 @@
  ]
  }
  },
- "pluginVersion": "v10.2.0",
+ "pluginVersion": "v10.4.0",
  "targets": [
  {
  "datasource": {
@@ -210,7 +210,7 @@
  ]
  }
  },
- "pluginVersion": "v10.2.0",
+ "pluginVersion": "v10.4.0",
  "targets": [
  {
  "datasource": {
@@ -259,7 +259,7 @@
  ]
  }
  },
- "pluginVersion": "v10.2.0",
+ "pluginVersion": "v10.4.0",
  "targets": [
  {
  "datasource": {
@@ -325,7 +325,7 @@
  "sort": "desc"
  }
  },
- "pluginVersion": "v10.2.0",
+ "pluginVersion": "v10.4.0",
  "targets": [
  {
  "datasource": {
@@ -376,7 +376,7 @@
  "sort": "desc"
  }
  },
- "pluginVersion": "v10.2.0",
+ "pluginVersion": "v10.4.0",
  "targets": [
  {
  "datasource": {
@@ -463,7 +463,7 @@
  }
  ]
  },
- "pluginVersion": "v10.2.0",
+ "pluginVersion": "v10.4.0",
  "targets": [
  {
  "datasource": {
@@ -638,7 +638,7 @@
  }
  ]
  },
- "pluginVersion": "v10.2.0",
+ "pluginVersion": "v10.4.0",
  "targets": [
  {
  "datasource": {

diff --git a/dashboards_out/ingress-nginx-request-handling-performance.json b/dashboards_out/ingress-nginx-request-handling-performance.json
@@ -64,7 +64,7 @@
  "sort": "desc"
  }
  },
- "pluginVersion": "v10.2.0",
+ "pluginVersion": "v10.4.0",
  "targets": [
  {
  "datasource": {
@@ -131,7 +131,7 @@
  "sort": "desc"
  }
  },
- "pluginVersion": "v10.2.0",
+ "pluginVersion": "v10.4.0",
  "targets": [
  {
  "datasource": {
@@ -214,7 +214,7 @@
  "sort": "desc"
  }
  },
- "pluginVersion": "v10.2.0",
+ "pluginVersion": "v10.4.0",
  "targets": [
  {
  "datasource": {
@@ -265,7 +265,7 @@
  "sort": "desc"
  }
  },
- "pluginVersion": "v10.2.0",
+ "pluginVersion": "v10.4.0",
  "targets": [
  {
  "datasource": {
@@ -320,7 +320,7 @@
  "sort": "desc"
  }
  },
- "pluginVersion": "v10.2.0",
+ "pluginVersion": "v10.4.0",
  "targets": [
  {
  "datasource": {
@@ -371,7 +371,7 @@
  "sort": "desc"
  }
  },
- "pluginVersion": "v10.2.0",
+ "pluginVersion": "v10.4.0",
  "targets": [
  {
  "datasource": {
@@ -426,7 +426,7 @@
  "sort": "desc"
  }
  },
- "pluginVersion": "v10.2.0",
+ "pluginVersion": "v10.4.0",
  "targets": [
  {
  "datasource": {
@@ -481,7 +481,7 @@
  "sort": "desc"
  }
  },
- "pluginVersion": "v10.2.0",
+ "pluginVersion": "v10.4.0",
  "targets": [
  {
  "datasource": {