Merge pull request #237 from vshn/redis/sla_report

Add redis sla reporting and fix non-working maintenance exclusion
vshn · Oct 4, 2023 · 786d434 · 786d434
2 parents 60fad89 + 1f1e0b2
commit 786d434
Show file tree

Hide file tree

Showing 29 changed files with 87 additions and 67 deletions.
diff --git a/class/defaults.yml b/class/defaults.yml
@@ -39,7 +39,7 @@ parameters:
       appcat:
         registry: ghcr.io
         repository: vshn/appcat
-        tag: v4.33.0
+        tag: v4.34.0
       apiserver:
         registry: ghcr.io
         repository: vshn/appcat-apiserver
@@ -381,6 +381,7 @@ parameters:
           bucket_region: "lpg"
           grpcEndpoint: ${appcat:grpcEndpoint}
           defaultPlan: standard-1
+          sla: 99.25
           plans:
             standard-512m:
               size:

diff --git a/component/main.jsonnet b/component/main.jsonnet
@@ -122,7 +122,7 @@ local maintenanceRule = kube._Object('monitoring.coreos.com/v1', 'PrometheusRule
         name: 'appcat-cluster-maintenance',
         rules: [
           {
-            expr: 'scalar(max(max_over_time(openshift_upgrade_controller_upgradejob_state{state="active"}[10m])) or vector(0))',
+            expr: 'max(max_over_time(openshift_upgrade_controller_upgradejob_state{state="active"}[10m])) or vector(0)',
             record: 'appcat:cluster:maintenance',
           },
         ],

diff --git a/component/slos.libsonnet b/component/slos.libsonnet
@@ -66,12 +66,12 @@ local prometheusRule(name) =
         sli: {
           events: {
             // The  0*rate(...) makes sure that the query reports an error rate for all instances, even if that instance has never produced a single error
-            error_query: '(sum(rate(appcat_probes_seconds_count{reason!="success", service="VSHNPostgreSQL", ha="false"}[{{.window}}]) or 0*rate(appcat_probes_seconds_count{service="VSHNPostgreSQL"}[{{.window}}])) by (service, namespace, name, organization, sla) or vector(0)) - appcat:cluster:maintenance > 0',
-            total_query: '(sum(rate(appcat_probes_seconds_count{service="VSHNPostgreSQL", ha="false"}[{{.window}}])) by (service, namespace, name, organization, sla) or vector(0)) - appcat:cluster:maintenance > 0',
+            error_query: '(sum(rate(appcat_probes_seconds_count{reason!="success", service="VSHNPostgreSQL", ha="false"}[{{.window}}]) or 0*rate(appcat_probes_seconds_count{service="VSHNPostgreSQL"}[{{.window}}])) by (service, namespace, name, organization, sla) or vector(0)) - scalar(appcat:cluster:maintenance) > 0',
+            total_query: '(sum(rate(appcat_probes_seconds_count{service="VSHNPostgreSQL", ha="false"}[{{.window}}])) by (service, namespace, name, organization, sla) or vector(0)) - scalar(appcat:cluster:maintenance) > 0',
           },
         },
         alerting+: {
-          name: 'SLO_AppCat_VSHNPosgtreSQLUptime',
+          name: 'SLO_AppCat_VSHNPostgreSQLUptime',
           annotations+: {
             summary: 'Probes to PostgreSQL by VSHN instance fail',
           },
@@ -110,8 +110,8 @@ local prometheusRule(name) =
         sli: {
           events: {
             // The  0*rate(...) makes sure that the query reports an error rate for all instances, even if that instance has never produced a single error
-            error_query: '(sum(rate(appcat_probes_seconds_count{reason!="success", service="VSHNRedis"}[{{.window}}]) or 0*rate(appcat_probes_seconds_count{service="VSHNRedis"}[{{.window}}]))  by (service, namespace, name, organization, sla) or vector(0)) - appcat:cluster:maintenance > 0',
-            total_query: '(sum(rate(appcat_probes_seconds_count{service="VSHNRedis"}[{{.window}}])) by (service, namespace, name, organization, sla) or vector(0)) - appcat:cluster:maintenance > 0',
+            error_query: '(sum(rate(appcat_probes_seconds_count{reason!="success", service="VSHNRedis"}[{{.window}}]) or 0*rate(appcat_probes_seconds_count{service="VSHNRedis"}[{{.window}}]))  by (service, namespace, name, organization, sla) or vector(0)) - scalar(appcat:cluster:maintenance) > 0',
+            total_query: '(sum(rate(appcat_probes_seconds_count{service="VSHNRedis"}[{{.window}}])) by (service, namespace, name, organization, sla) or vector(0)) - scalar(appcat:cluster:maintenance) > 0',
           },
         },
         alerting+: {

diff --git a/component/vshn_redis.jsonnet b/component/vshn_redis.jsonnet
@@ -45,6 +45,8 @@ local xrd = xrds.XRDFromCRD(
   connectionSecretKeys=connectionSecretKeys,
 ) + xrds.WithPlanDefaults(redisPlans, redisParams.defaultPlan);
 
+local promRuleRedisSLA = common.PromRuleSLA(params.services.vshn.redis.sla, 'VSHNRedis');
+
 local restoreServiceAccount = kube.ServiceAccount('redisrestoreserviceaccount') + {
   metadata+: {
     namespace: params.services.controlNamespace,
@@ -691,5 +693,6 @@ if params.services.vshn.enabled && redisParams.enabled then {
   '20_rbac_vshn_redis_resize': [ resizeClusterRole, resizeServiceAccount, resizeClusterRoleBinding ],
   '20_plans_vshn_redis': plansCM,
   '21_composition_vshn_redis': composition,
+  '22_prom_rule_sla_redis': promRuleRedisSLA,
   [if isOpenshift then '21_openshift_template_redis_vshn']: osTemplate,
 } else {}
diff --git a/tests/golden/apiserver/appcat/appcat/10_appcat_maintenance_recording_rule.yaml b/tests/golden/apiserver/appcat/appcat/10_appcat_maintenance_recording_rule.yaml
@@ -10,6 +10,6 @@ spec:
   groups:
     - name: appcat-cluster-maintenance
       rules:
-        - expr: scalar(max(max_over_time(openshift_upgrade_controller_upgradejob_state{state="active"}[10m]))
-            or vector(0))
+        - expr: max(max_over_time(openshift_upgrade_controller_upgradejob_state{state="active"}[10m]))
+            or vector(0)
           record: appcat:cluster:maintenance
diff --git a/tests/golden/cloudscale/appcat/appcat/10_appcat_maintenance_recording_rule.yaml b/tests/golden/cloudscale/appcat/appcat/10_appcat_maintenance_recording_rule.yaml
@@ -10,6 +10,6 @@ spec:
   groups:
     - name: appcat-cluster-maintenance
       rules:
-        - expr: scalar(max(max_over_time(openshift_upgrade_controller_upgradejob_state{state="active"}[10m]))
-            or vector(0))
+        - expr: max(max_over_time(openshift_upgrade_controller_upgradejob_state{state="active"}[10m]))
+            or vector(0)
           record: appcat:cluster:maintenance
diff --git a/tests/golden/controllers/appcat/appcat/10_appcat_maintenance_recording_rule.yaml b/tests/golden/controllers/appcat/appcat/10_appcat_maintenance_recording_rule.yaml
@@ -10,6 +10,6 @@ spec:
   groups:
     - name: appcat-cluster-maintenance
       rules:
-        - expr: scalar(max(max_over_time(openshift_upgrade_controller_upgradejob_state{state="active"}[10m]))
-            or vector(0))
+        - expr: max(max_over_time(openshift_upgrade_controller_upgradejob_state{state="active"}[10m]))
+            or vector(0)
           record: appcat:cluster:maintenance
diff --git a/tests/golden/controllers/appcat/appcat/controllers/appcat/30_deployment.yaml b/tests/golden/controllers/appcat/appcat/controllers/appcat/30_deployment.yaml
@@ -23,7 +23,7 @@ spec:
           env:
             - name: PLANS_NAMESPACE
               value: syn-appcat
-          image: ghcr.io/vshn/appcat:v4.33.0
+          image: ghcr.io/vshn/appcat:v4.34.0
           livenessProbe:
             httpGet:
               path: /healthz

diff --git a/tests/golden/defaults/appcat/appcat/10_appcat_maintenance_recording_rule.yaml b/tests/golden/defaults/appcat/appcat/10_appcat_maintenance_recording_rule.yaml
@@ -10,6 +10,6 @@ spec:
   groups:
     - name: appcat-cluster-maintenance
       rules:
-        - expr: scalar(max(max_over_time(openshift_upgrade_controller_upgradejob_state{state="active"}[10m]))
-            or vector(0))
+        - expr: max(max_over_time(openshift_upgrade_controller_upgradejob_state{state="active"}[10m]))
+            or vector(0)
           record: appcat:cluster:maintenance
diff --git a/.../appcat/appcat/sli_exporter/apps_v1_deployment_appcat-sliexporter-controller-manager.yaml b/.../appcat/appcat/sli_exporter/apps_v1_deployment_appcat-sliexporter-controller-manager.yaml
@@ -28,7 +28,7 @@ spec:
           value: "false"
         - name: APPCAT_SLI_VSHNREDIS
           value: "false"
-        image: ghcr.io/vshn/appcat:v4.33.0
+        image: ghcr.io/vshn/appcat:v4.34.0
         livenessProbe:
           httpGet:
             path: /healthz

diff --git a/tests/golden/exoscale/appcat/appcat/10_appcat_maintenance_recording_rule.yaml b/tests/golden/exoscale/appcat/appcat/10_appcat_maintenance_recording_rule.yaml
@@ -10,6 +10,6 @@ spec:
   groups:
     - name: appcat-cluster-maintenance
       rules:
-        - expr: scalar(max(max_over_time(openshift_upgrade_controller_upgradejob_state{state="active"}[10m]))
-            or vector(0))
+        - expr: max(max_over_time(openshift_upgrade_controller_upgradejob_state{state="active"}[10m]))
+            or vector(0)
           record: appcat:cluster:maintenance
diff --git a/tests/golden/minio/appcat/appcat/10_appcat_maintenance_recording_rule.yaml b/tests/golden/minio/appcat/appcat/10_appcat_maintenance_recording_rule.yaml
@@ -10,6 +10,6 @@ spec:
   groups:
     - name: appcat-cluster-maintenance
       rules:
-        - expr: scalar(max(max_over_time(openshift_upgrade_controller_upgradejob_state{state="active"}[10m]))
-            or vector(0))
+        - expr: max(max_over_time(openshift_upgrade_controller_upgradejob_state{state="active"}[10m]))
+            or vector(0)
           record: appcat:cluster:maintenance
diff --git a/tests/golden/minio/appcat/appcat/21_composition_vshn_minio.yaml b/tests/golden/minio/appcat/appcat/21_composition_vshn_minio.yaml
@@ -25,7 +25,7 @@ spec:
         data:
           controlNamespace: syn-appcat-control
           defaultPlan: standard-1
-          imageTag: v4.33.0
+          imageTag: v4.34.0
           maintenanceSA: helm-based-service-maintenance
           minioChartRepository: https://charts.min.io
           minioChartVersion: 5.0.13

diff --git a/tests/golden/minio/appcat/appcat/controllers/appcat/30_deployment.yaml b/tests/golden/minio/appcat/appcat/controllers/appcat/30_deployment.yaml
@@ -23,7 +23,7 @@ spec:
           env:
             - name: PLANS_NAMESPACE
               value: syn-appcat
-          image: ghcr.io/vshn/appcat:v4.33.0
+          image: ghcr.io/vshn/appcat:v4.34.0
           livenessProbe:
             httpGet:
               path: /healthz

diff --git a/tests/golden/minio/appcat/appcat/sla_reporter/01_cronjob.yaml b/tests/golden/minio/appcat/appcat/sla_reporter/01_cronjob.yaml
@@ -30,7 +30,7 @@ spec:
               envFrom:
                 - secretRef:
                     name: appcat-sla-reports-creds
-              image: ghcr.io/vshn/appcat:v4.33.0
+              image: ghcr.io/vshn/appcat:v4.34.0
               name: sla-reporter
               resources:
                 limits:

diff --git a/.../appcat/appcat/sli_exporter/apps_v1_deployment_appcat-sliexporter-controller-manager.yaml b/.../appcat/appcat/sli_exporter/apps_v1_deployment_appcat-sliexporter-controller-manager.yaml
@@ -28,7 +28,7 @@ spec:
           value: "false"
         - name: APPCAT_SLI_VSHNREDIS
           value: "false"
-        image: ghcr.io/vshn/appcat:v4.33.0
+        image: ghcr.io/vshn/appcat:v4.34.0
         livenessProbe:
           httpGet:
             path: /healthz

diff --git a/tests/golden/openshift/appcat/appcat/10_appcat_maintenance_recording_rule.yaml b/tests/golden/openshift/appcat/appcat/10_appcat_maintenance_recording_rule.yaml
@@ -10,6 +10,6 @@ spec:
   groups:
     - name: appcat-cluster-maintenance
       rules:
-        - expr: scalar(max(max_over_time(openshift_upgrade_controller_upgradejob_state{state="active"}[10m]))
-            or vector(0))
+        - expr: max(max_over_time(openshift_upgrade_controller_upgradejob_state{state="active"}[10m]))
+            or vector(0)
           record: appcat:cluster:maintenance
diff --git a/.../appcat/appcat/sli_exporter/apps_v1_deployment_appcat-sliexporter-controller-manager.yaml b/.../appcat/appcat/sli_exporter/apps_v1_deployment_appcat-sliexporter-controller-manager.yaml
@@ -28,7 +28,7 @@ spec:
           value: "false"
         - name: APPCAT_SLI_VSHNREDIS
           value: "false"
-        image: ghcr.io/vshn/appcat:v4.33.0
+        image: ghcr.io/vshn/appcat:v4.34.0
         livenessProbe:
           httpGet:
             path: /healthz

diff --git a/tests/golden/vshn/appcat/appcat/10_appcat_maintenance_recording_rule.yaml b/tests/golden/vshn/appcat/appcat/10_appcat_maintenance_recording_rule.yaml
@@ -10,6 +10,6 @@ spec:
   groups:
     - name: appcat-cluster-maintenance
       rules:
-        - expr: scalar(max(max_over_time(openshift_upgrade_controller_upgradejob_state{state="active"}[10m]))
-            or vector(0))
+        - expr: max(max_over_time(openshift_upgrade_controller_upgradejob_state{state="active"}[10m]))
+            or vector(0)
           record: appcat:cluster:maintenance
diff --git a/tests/golden/vshn/appcat/appcat/21_composition_vshn_minio.yaml b/tests/golden/vshn/appcat/appcat/21_composition_vshn_minio.yaml
@@ -25,7 +25,7 @@ spec:
         data:
           controlNamespace: syn-appcat-control
           defaultPlan: standard-1
-          imageTag: v4.33.0
+          imageTag: v4.34.0
           maintenanceSA: helm-based-service-maintenance
           minioChartRepository: https://charts.min.io
           minioChartVersion: 5.0.13

diff --git a/tests/golden/vshn/appcat/appcat/21_composition_vshn_postgres.yaml b/tests/golden/vshn/appcat/appcat/21_composition_vshn_postgres.yaml
@@ -33,7 +33,7 @@ spec:
           emailAlertingSmtpHost: smtp.eu.mailgun.org:465
           emailAlertingSmtpUsername: myuser@example.com
           externalDatabaseConnectionsEnabled: 'true'
-          imageTag: v4.33.0
+          imageTag: v4.34.0
           quotasEnabled: 'false'
           sgNamespace: stackgres
           sideCars: '{"clusterController": {"limits": {"cpu": "600m", "memory": "768Mi"},

diff --git a/tests/golden/vshn/appcat/appcat/21_composition_vshn_postgresrestore.yaml b/tests/golden/vshn/appcat/appcat/21_composition_vshn_postgresrestore.yaml
@@ -33,7 +33,7 @@ spec:
           emailAlertingSmtpHost: smtp.eu.mailgun.org:465
           emailAlertingSmtpUsername: myuser@example.com
           externalDatabaseConnectionsEnabled: 'true'
-          imageTag: v4.33.0
+          imageTag: v4.34.0
           quotasEnabled: 'false'
           sgNamespace: stackgres
           sideCars: '{"clusterController": {"limits": {"cpu": "600m", "memory": "768Mi"},

diff --git a/tests/golden/vshn/appcat/appcat/21_composition_vshn_redis.yaml b/tests/golden/vshn/appcat/appcat/21_composition_vshn_redis.yaml
@@ -30,7 +30,7 @@ spec:
         data:
           bucketRegion: lpg
           controlNamespace: syn-appcat-control
-          imageTag: v4.33.0
+          imageTag: v4.34.0
           maintenanceSA: helm-based-service-maintenance
           quotasEnabled: 'false'
           restoreSA: redisrestoreserviceaccount

diff --git a/tests/golden/vshn/appcat/appcat/22_prom_rule_sla_redis.yaml b/tests/golden/vshn/appcat/appcat/22_prom_rule_sla_redis.yaml
@@ -0,0 +1,16 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  annotations: {}
+  labels:
+    name: vshn-vshnredis-sla
+  name: vshn-vshnredis-sla
+  namespace: appcat-slos
+spec:
+  groups:
+    - name: appcat-vshnredis-sla-target
+      rules:
+        - expr: vector(99.25)
+          labels:
+            service: VSHNRedis
+          record: sla:objective:ratio
diff --git a/tests/golden/vshn/appcat/appcat/controllers/appcat/30_deployment.yaml b/tests/golden/vshn/appcat/appcat/controllers/appcat/30_deployment.yaml
@@ -23,7 +23,7 @@ spec:
           env:
             - name: PLANS_NAMESPACE
               value: syn-appcat
-          image: ghcr.io/vshn/appcat:v4.33.0
+          image: ghcr.io/vshn/appcat:v4.34.0
           livenessProbe:
             httpGet:
               path: /healthz

diff --git a/tests/golden/vshn/appcat/appcat/sla_reporter/01_cronjob.yaml b/tests/golden/vshn/appcat/appcat/sla_reporter/01_cronjob.yaml
@@ -30,7 +30,7 @@ spec:
               envFrom:
                 - secretRef:
                     name: appcat-sla-reports-creds
-              image: ghcr.io/vshn/appcat:v4.33.0
+              image: ghcr.io/vshn/appcat:v4.34.0
               name: sla-reporter
               resources:
                 limits: