Alert manger controller (#138)

* bookmark

* bookmark

* initial commit

* adding OwnerReferences to webhooks

* adding deletion logic

* adding e2e tests

* changing tests folder names

* fixing tests

* fixing test

* refactoring

* adding support for RepeatInterval

* adding support for group-by, and seperating routes to different notification-groups

* skipping alert and webhooks in a case of an error

* fixing tests

Makefile

@@ -111,6 +111,8 @@ endif
 install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config.
  $(KUSTOMIZE) build config/crd | kubectl apply -f -
  kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/master/example/prometheus-operator-crd-full/monitoring.coreos.com_prometheusrules.yaml
+ kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/main/jsonnet/prometheus-operator/alertmanagerconfigs-crd.json
+
 
 .PHONY: uninstall
 uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.

apis/coralogix/v1alpha1/outboundwebhook_types.go

@@ -287,7 +287,9 @@ var (
 )
 
 type Slack struct {
- Digests []SlackConfigDigest `json:"digests"`
+ // +optional
+ Digests []SlackConfigDigest `json:"digests"`
+ // +optional
  Attachments []SlackConfigAttachment `json:"attachments"`
  Url string `json:"url"`
 }

config/crd/bases/coralogix.com_outboundwebhooks.yaml

@@ -178,8 +178,6 @@ spec:
  url:
  type: string
  required:
- - attachments
- - digests
  - url
  type: object
  type: object
@@ -342,8 +340,6 @@ spec:
  url:
  type: string
  required:
- - attachments
- - digests
  - url
  type: object
  type: object

config/samples/alertmanger/example-alertmanager.yaml

@@ -0,0 +1,48 @@
+apiVersion: monitoring.coreos.com/v1alpha1
+kind: AlertmanagerConfig
+metadata:
+ name: config-example
+ labels:
+ alertmanagerConfig: example
+ app.coralogix.com/track-alertmanger-config: "true"
+spec:
+ route:
+ groupBy:
+ - alertname
+ - cluster
+ - alert_group
+ - job
+ - namespace
+ - severity
+ - priority
+ receiver: slack-default
+ repeatInterval: 3h
+ routes:
+ - receiver: slack-general
+ matchers:
+ - matchType: "=~"
+ name: slack_channel
+ value: ".+"
+ continue: true
+ - receiver: opsgenie-general
+ groupBy:
+ - alertname
+ matchers:
+ - matchType: "=~"
+ name: opsgenie_team
+ value: ".+"
+ repeatInterval: 4m
+ receivers:
+ - name: slack-general
+ slackConfigs:
+ - apiURL:
+ name: "slack-webhook-secret" # Name of the Kubernetes Secret
+ key: "webhook-url" # Key in the Kubernetes Secret
+ - name: opsgenie-general
+ opsgenieConfigs:
+ - apiURL: https://api.opsgenie.com/v2/alerts
+ - name: slack-default
+ slackConfigs:
+ - apiURL:
+ name: "slack-webhook-secret" # Name of the Kubernetes Secret
+ key: "webhook-url" # Key in the Kubernetes Secret

config/samples/alertmanger/slack-url-secret.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Secret
+metadata:
+ name: slack-webhook-secret # Name of the secret
+ namespace: default # Namespace where the secret is created
+type: Opaque
+data:
+ webhook-url: aHR0cHM6Ly9zbGFjay5jb20vYXBpL2NoYXQucG9zdE1lc3NhZ2U= # Base64 encoded API URL

config/samples/prometheusrules/monitoring.coreos.com_prometheusrules.yaml

@@ -7,6 +7,7 @@ metadata:
  role: alert-rules
  app.coralogix.com/track-recording-rules: "true"
  app.coralogix.com/track-alerting-rules: "true"
+ app.coralogix.com/managed-by-alertmanger-config: "true"
  name: prometheus-example-rules
 spec:
  groups:
@@ -24,6 +25,7 @@ spec:
  cxMinNonNullValuesPercentage: "20"
  labels:
  severity: critical
+ slack_channel: "#observability"
  - name: example.rules2
  interval: "70s"
  rules:
@@ -37,4 +39,11 @@ spec:
  annotations:
  cxMinNonNullValuesPercentage: "20"
  labels:
- severity: info
+ severity: info
+ slack_channel: "#observability"
+ opsgenie_team: "team1"
+ - alert: app-latency
+ expr: histogram_quantile(0.99, sum(irate(istio_request_duration_seconds_bucket{reporter="source",destination_service=~"ingress-annotation-test-svc.example-app.svc.cluster.local"}[1m])) by (le, destination_workload)) > 0.2
+ for: 5m
+ annotations:
+ cxMinNonNullValuesPercentage: "20"