kube-prometheus-stack.yaml

---
apiVersion: helm.toolkit.fluxcd.io/v2beta1
kind: HelmRelease
metadata:
  name: kube-prometheus-stack
  namespace: monitoring
spec:
  interval: 15m
  chart:
    spec:
      # renovate: registryUrl=https://prometheus-community.github.io/helm-charts
      chart: kube-prometheus-stack
      version: 68.3.0
      sourceRef:
        kind: HelmRepository
        name: prometheus-community-charts
        namespace: flux-system
  maxHistory: 3
  install:
    createNamespace: true
    remediation:
      retries: 3
  upgrade:
    cleanupOnFail: true
    remediation:
      retries: 3
  uninstall:
    keepHistory: false
  values:

    crds:
      enabled: true

    cleanPrometheusOperatorObjectNames: true

    defaultRules:
      disabled:
        HostHighCpuLoad: true

    alertmanager:
      config:
        global:
          resolve_timeout: 5m
          slack_api_url: ${SECRET_KPS_SLACK_API_URL}
        receivers:
        - name: 'null'
        - name: 'slack-notifications'
          slack_configs:
            - channel: '#notifications'
              icon_url: https://avatars3.githubusercontent.com/u/3380462
              username: 'Alertmanager'
              send_resolved: true
              title: |-
                [{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ if ne .CommonAnnotations.summary ""}}{{ .CommonAnnotations.summary }} {{ else if ne .CommonAnnotations.message ""}}{{ .CommonAnnotations.message }} {{ else if ne .CommonAnnotations.description ""}}{{ .CommonAnnotations.description }} {{ else }}{{ .CommonLabels.alertname }}{{ end }}
              text: >-
                {{ range .Alerts -}}
                  *Alert:* {{ .Annotations.title }}{{ if .Labels.severity }} - `{{ .Labels.severity }}`{{ end }}

                {{ if ne .Annotations.summary ""}}*Summary:* {{ .Annotations.summary }} {{ else if ne .Annotations.message ""}}*Message:* {{ .Annotations.message }} {{ else if ne .Annotations.description ""}}*Description:* {{ .Annotations.description }}{{ end }}

                *Details:*
                  {{ range .Labels.SortedPairs }} • *{{ .Name }}:* `{{ .Value }}`
                  {{ end }}
                {{ end }}
        - name: 'pagerduty'
          pagerduty_configs:
            - routing_key: ${SECRET_KPS_PAGERDUTY_KEY}
              description: |-
                {{ template "pagerduty.custom.description" . }}
        route:
          group_by: ["alertname", "job"]
          group_wait: 45s
          group_interval: 10m
          repeat_interval: 12h
          receiver: 'slack-notifications'
          routes:
            - receiver: "null" # quote
              matchers:
                - alertname =~ "Watchdog"
            - receiver: "null" # quote
              matchers:
                - alertname =~ "InfoInhibitor"
            # - receiver: 'pagerduty'
            #   matchers:
            #     - severity = "critical"
            #   continue: true
            - receiver: 'slack-notifications'
        inhibit_rules:
          - source_matchers:
              - severity = "critical"
            target_matchers:
              - severity = "warning"
            equal: ["alertname", "namespace"]

        templates: ["*.tmpl"]
      templateFiles:
        pagerduty-custom.tmpl: |-
          {{- define "pagerduty.custom.description" -}}[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ if ne .CommonAnnotations.summary ""}}{{ .CommonAnnotations.summary }} {{ else if ne .CommonAnnotations.message ""}}{{ .CommonAnnotations.message }} {{ else if ne .CommonAnnotations.description ""}}{{ .CommonAnnotations.description }} {{ else }}{{ .CommonLabels.alertname }}{{ end }}{{- end -}}
      ingress:
        enabled: true
        pathType: ImplementationSpecific
        ingressClassName: nginx-internal
        annotations:
          cert-manager.io/cluster-issuer: letsencrypt-prod
          nginx.ingress.kubernetes.io/auth-signin: "https://auth.${SECRET_DOMAIN}"
          nginx.ingress.kubernetes.io/auth-url: "https://auth.${SECRET_DOMAIN}/api/authz/auth-request"
        hosts: ["prom-alert.k.${SECRET_DOMAIN}"]
        tls:
        - hosts:
          - "prom-alert.k.${SECRET_DOMAIN}"
          secretName: prom-alert-cert
      alertmanagerSpec:
        storage:
          volumeClaimTemplate:
            spec:
              storageClassName: rook-ceph-block
              resources:
                requests:
                  storage: 1Gi
        tolerations:
        - key: "arm"
          operator: "Exists"

    grafana:
      enabled: false
      forceDeployDashboards: true

    kubelet:
      enabled: true
      serviceMonitor:
        metricRelabelings:
          - action: replace
            sourceLabels: ["node"]
            targetLabel: instance

    kube-state-metrics:
      metricLabelsAllowlist: ["persistentvolumeclaims=[*]"]
      prometheus:
        monitor:
          enabled: true
          relabelings:
            - action: replace
              regex: (.*)
              replacement: $1
              sourceLabels: ["__meta_kubernetes_pod_node_name"]
              targetLabel: kubernetes_node

    kubeApiServer:
      enabled: true

    kubeControllerManager:
      enabled: false
      endpoints:
        - 10.10.40.40
      service:
        enabled: true
        port: 10257
        targetPort: 10257
      serviceMonitor:
        enabled: true
        https: true
        insecureSkipVerify: true

    kubeScheduler:
      enabled: false
      endpoints:
        - 10.10.40.40
      service:
        enabled: true
        port: 10259
        targetPort: 10259
      serviceMonitor:
        enabled: true
        https: true
        insecureSkipVerify: true

    kubeProxy:
      enabled: false
      endpoints:
        - 10.10.40.40

    kubeEtcd:
      enabled: false
      endpoints:
        - 10.10.40.40
      service:
        enabled: true
        port: 2381
        targetPort: 2381

    prometheusOperator:
      createCustomResource: true
      # prometheusConfigReloaderImage:
      #   repository: quay.io/coreos/prometheus-config-reloader
      #   tag: v0.39.0
      # configmapReloadImage:
      #   repository: jimmidyson/configmap-reload
      #   tag: v0.4.0
      prometheusConfigReloader:
        resources:
          limits:
            cpu: 200m
      admissionWebhooks:
        enabled: false
        certManager:
          enabled: true

    prometheus:
      ingress:
        enabled: true
        pathType: ImplementationSpecific
        ingressClassName: nginx-internal
        annotations:
          cert-manager.io/cluster-issuer: letsencrypt-prod
          nginx.ingress.kubernetes.io/auth-signin: "https://auth.${SECRET_DOMAIN}"
          nginx.ingress.kubernetes.io/auth-url: "https://auth.${SECRET_DOMAIN}/api/authz/auth-request"
        hosts: ["prom-server.k.${SECRET_DOMAIN}"]
        tls:
        - hosts:
          - "prom-server.k.${SECRET_DOMAIN}"
          secretName: prom-server-cert
      thanosService:
        enabled: true
      thanosServiceMonitor:
        enabled: true
      prometheusSpec:
        replicas: 2
        replicaExternalLabelName: "replica"
        thanos:
          image: quay.io/thanos/thanos:v0.37.2
          version: v0.30.1
          objectStorageConfig:
            name: thanos
            key: object-store.yaml
        podAntiAffinity: hard
        ruleSelectorNilUsesHelmValues: false
        serviceMonitorSelectorNilUsesHelmValues: false
        podMonitorSelectorNilUsesHelmValues: false
        probeSelectorNilUsesHelmValues: false
        retention: 2d
        enableAdminAPI: true
        walCompression: true
        storageSpec:
          volumeClaimTemplate:
            spec:
              storageClassName: rook-ceph-block
              resources:
                requests:
                  storage: 10Gi
        resources:
          requests:
            memory: 2000Mi
            cpu: 250m
          limits:
            memory: 5000Mi

    nodeExporter:
      enabled: true

    prometheus-node-exporter:
      prometheus:
        monitor:
          enabled: true
          # relabelings:
          #   - action: replace
          #     regex: (.*)
          #     replacement: $1
          #     sourceLabels:
          #       - __meta_kubernetes_pod_node_name
          #     targetLabel: instance
      tolerations:
      - key: "arm"
        operator: "Exists"
      - key: "armhf"
        operator: "Exists"
      - key: "node-role.kubernetes.io/master"
        operator: "Exists"