Setup OTEL Collector for metric orchestration

DiamondLightSource · Mar 14, 2024 · a95470d · a95470d
1 parent f8a4935
commit a95470d
Show file tree

Hide file tree

Showing 4 changed files with 156 additions and 92 deletions.
diff --git a/.github/workflows/charts.yml b/.github/workflows/charts.yml
@@ -25,6 +25,7 @@ jobs:
         run: |
           helm repo add prometheus https://prometheus-community.github.io/helm-charts
           helm repo add jaegertracing https://jaegertracing.github.io/helm-charts
+          helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts
 
       - name: Lint
         run: >

diff --git a/charts/data-gateway/Chart.lock b/charts/data-gateway/Chart.lock
@@ -5,5 +5,8 @@ dependencies:
 - name: jaeger
   repository: https://jaegertracing.github.io/helm-charts
   version: 2.0.0
-digest: sha256:350282c3114464a73c0ded6e66072f7530e4bce8298d90cb73dd19119fc9a123
-generated: "2024-03-14T10:53:42.342035905Z"
+- name: opentelemetry-collector
+  repository: https://open-telemetry.github.io/opentelemetry-helm-charts
+  version: 0.83.0
+digest: sha256:c64dbb83ab863fb6f4c83dd8fc4505508072fda7b83f3ae1d1ac96af4b3b0c5f
+generated: "2024-03-14T11:22:20.559143777Z"
diff --git a/charts/data-gateway/Chart.yaml b/charts/data-gateway/Chart.yaml
@@ -14,3 +14,7 @@ dependencies:
     repository: https://jaegertracing.github.io/helm-charts
     version: 2.0.0
     condition: jaeger.enabled
+  - name: opentelemetry-collector
+    repository: https://open-telemetry.github.io/opentelemetry-helm-charts
+    version: 0.83.0
+    condition: opentelemetry-collector.enabled
diff --git a/charts/data-gateway/values.yaml b/charts/data-gateway/values.yaml
@@ -37,96 +37,10 @@ prometheus:
           static_configs:
             - targets:
                 - localhost:9090
-        - job_name: "kubernetes-service-endpoints"
-          kubernetes_sd_configs:
-            - role: endpoints
-              namespaces:
-                own_namespace: true
-          relabel_configs:
-            - source_labels:
-                [__meta_kubernetes_service_annotation_prometheus_io_scrape]
-              action: keep
-              regex: true
-            - source_labels:
-                [__meta_kubernetes_service_annotation_prometheus_io_scheme]
-              action: replace
-              target_label: __scheme__
-              regex: (https?)
-            - source_labels:
-                [__meta_kubernetes_service_annotation_prometheus_io_path]
-              action: replace
-              target_label: __metrics_path__
-              regex: (.+)
-            - source_labels:
-                [
-                  __address__,
-                  __meta_kubernetes_service_annotation_prometheus_io_port,
-                ]
-              action: replace
-              target_label: __address__
-              regex: (.+?)(?::\d+)?;(\d+)
-              replacement: $1:$2
-            - action: labelmap
-              regex: __meta_kubernetes_service_annotation_prometheus_io_param_(.+)
-              replacement: __param_$1
-            - action: labelmap
-              regex: __meta_kubernetes_service_label_(.+)
-            - source_labels: [__meta_kubernetes_namespace]
-              action: replace
-              target_label: namespace
-            - source_labels: [__meta_kubernetes_service_name]
-              action: replace
-              target_label: service
-            - source_labels: [__meta_kubernetes_pod_node_name]
-              action: replace
-              target_label: node
-        - job_name: "kubernetes-pods"
-          kubernetes_sd_configs:
-            - role: pod
-              namespaces:
-                own_namespace: true
-          relabel_configs:
-            - source_labels:
-                [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
-              action: keep
-              regex: true
-            - source_labels:
-                [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
-              action: keep
-              regex: true
-            - source_labels:
-                [__meta_kubernetes_pod_annotation_prometheus_io_scheme]
-              action: replace
-              regex: (https?)
-              target_label: __scheme__
-            - source_labels:
-                [__meta_kubernetes_pod_annotation_prometheus_io_path]
-              action: replace
-              target_label: __metrics_path__
-              regex: (.+)
-            - source_labels:
-                [
-                  __address__,
-                  __meta_kubernetes_pod_annotation_prometheus_io_port,
-                ]
-              action: replace
-              regex: ([^:]+)(?::\d+)?;(\d+)
-              replacement: $1:$2
-              target_label: __address__
-            - action: labelmap
-              regex: __meta_kubernetes_pod_annotation_prometheus_io_param_(.+)
-              replacement: __param_$1
-            - action: labelmap
-              regex: __meta_kubernetes_pod_label_(.+)
-            - source_labels: [__meta_kubernetes_namespace]
-              action: replace
-              target_label: namespace
-            - source_labels: [__meta_kubernetes_pod_name]
-              action: replace
-              target_label: pod
-            - source_labels: [__meta_kubernetes_pod_phase]
-              regex: Pending|Succeeded|Failed|Completed
-              action: drop
+        - job_name: otel-collector
+          static_configs:
+            - targets:
+              - "{{ .Release.Name }}-opentelemetry-collector:9090"
 
 jaeger:
   enabled: true
@@ -142,3 +56,145 @@ jaeger:
     enabled: false
   query:
     enabled: false
+
+opentelemetry-collector:
+  enabled: true
+  mode: deployment
+  ports:
+    jaeger-compact:
+      enabled: false
+    jaeger-thrift:
+      enabled: false
+    jaeger-grpc:
+      enabled: false
+    zipkin:
+      enabled: false
+  config:
+    receivers:
+      jaeger: null
+      zipkin: null
+      prometheus:
+        config:
+          scrape_configs:
+          - job_name: opentelemetry-collector
+            scrape_interval: 10s
+            static_configs:
+              - targets:
+                  - ${env:MY_POD_IP}:8888
+          - job_name: kubernetes-service-endpoints
+            kubernetes_sd_configs:
+              - role: endpoints
+                namespaces:
+                  own_namespace: true
+            relabel_configs:
+              - source_labels:
+                  [__meta_kubernetes_service_annotation_prometheus_io_scrape]
+                action: keep
+                regex: true
+              - source_labels:
+                  [__meta_kubernetes_service_annotation_prometheus_io_scheme]
+                action: replace
+                target_label: __scheme__
+                regex: (https?)
+              - source_labels:
+                  [__meta_kubernetes_service_annotation_prometheus_io_path]
+                action: replace
+                target_label: __metrics_path__
+                regex: (.+)
+              - source_labels:
+                  [
+                    __address__,
+                    __meta_kubernetes_service_annotation_prometheus_io_port,
+                  ]
+                action: replace
+                target_label: __address__
+                regex: (.+?)(?::\d+)?;(\d+)
+                replacement: $1:$2
+              - action: labelmap
+                regex: __meta_kubernetes_service_annotation_prometheus_io_param_(.+)
+                replacement: __param_$1
+              - action: labelmap
+                regex: __meta_kubernetes_service_label_(.+)
+              - source_labels: [__meta_kubernetes_namespace]
+                action: replace
+                target_label: namespace
+              - source_labels: [__meta_kubernetes_service_name]
+                action: replace
+                target_label: service
+              - source_labels: [__meta_kubernetes_pod_node_name]
+                action: replace
+                target_label: node
+          - job_name: kubernetes-pods
+            kubernetes_sd_configs:
+              - role: pod
+                namespaces:
+                  own_namespace: true
+            relabel_configs:
+              - source_labels:
+                  [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
+                action: keep
+                regex: true
+              - source_labels:
+                  [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
+                action: keep
+                regex: true
+              - source_labels:
+                  [__meta_kubernetes_pod_annotation_prometheus_io_scheme]
+                action: replace
+                regex: (https?)
+                target_label: __scheme__
+              - source_labels:
+                  [__meta_kubernetes_pod_annotation_prometheus_io_path]
+                action: replace
+                target_label: __metrics_path__
+                regex: (.+)
+              - source_labels:
+                  [
+                    __address__,
+                    __meta_kubernetes_pod_annotation_prometheus_io_port,
+                  ]
+                action: replace
+                regex: ([^:]+)(?::\d+)?;(\d+)
+                replacement: $1:$2
+                target_label: __address__
+              - action: labelmap
+                regex: __meta_kubernetes_pod_annotation_prometheus_io_param_(.+)
+                replacement: __param_$1
+              - action: labelmap
+                regex: __meta_kubernetes_pod_label_(.+)
+              - source_labels: [__meta_kubernetes_namespace]
+                action: replace
+                target_label: namespace
+              - source_labels: [__meta_kubernetes_pod_name]
+                action: replace
+                target_label: pod
+              - source_labels: [__meta_kubernetes_pod_phase]
+                regex: Pending|Succeeded|Failed|Completed
+                action: drop
+    exporters:
+      otlp/jaeger:
+        endpoint: "{{ .Release.Name }}-jaeger-collector:4317"
+        tls:
+          insecure: true
+      prometheus:
+        endpoint: :9090
+    service:
+      pipelines:
+        metrics:
+          receivers:
+            - otlp
+            - prometheus
+          processors:
+            - memory_limiter
+            - batch
+          exporters:
+            - prometheus
+        traces:
+          receivers:
+            - otlp
+          processors:
+            - memory_limiter
+            - batch
+          exporters:
+            - otlp/jaeger
+