test: introduce integration test for OTel collector (#3211)

Signed-off-by: odubajDT <ondrej.dubaj@dynatrace.com>

.github/actions/spelling/expect.txt

@@ -686,3 +686,6 @@ Yash
 yml
 YOURNAME
 yourregistry
+GOGC
+otelcol
+serviceapp

Makefile

@@ -35,6 +35,7 @@ integration-test:
 	chainsaw test --test-dir ./test/chainsaw/testcertificate/
 	chainsaw test --test-dir ./test/chainsaw/non-blocking-deployment/
 	chainsaw test --test-dir ./test/chainsaw/timeout-failure-deployment/
+	chainsaw test --test-dir ./test/chainsaw/traces/
 
 .PHONY: integration-test-local #these tests should run on a real cluster!
 integration-test-local:
@@ -44,6 +45,8 @@ integration-test-local:
 	chainsaw test --test-dir ./test/chainsaw/testanalysis/ --config ./.chainsaw-local.yaml
 	chainsaw test --test-dir ./test/chainsaw/testcertificate/ --config ./.chainsaw-local.yaml
 	chainsaw test --test-dir ./test/chainsaw/non-blocking-deployment/ --config ./.chainsaw-local.yaml
+	chainsaw test --test-dir ./test/chainsaw/timeout-failure-deployment/ --config ./.chainsaw-local.yaml
+	chainsaw test --test-dir ./test/chainsaw/traces/ --config ./.chainsaw-local.yaml
 
 .PHONY: integration-test-scheduling-gates #these tests should run on a real cluster!
 integration-test-scheduling-gates:

test/chainsaw/traces/00-assert.yaml

@@ -0,0 +1,19 @@
+apiVersion: lifecycle.keptn.sh/v1beta1
+kind: KeptnAppVersion
+metadata:
+  name: keptnapp-0.0.1-6b86b273
+spec:
+  appName: keptnapp
+  revision: 1
+  version: 0.0.1
+  workloads:
+    - name: nginx
+      version: 0.0.1
+status:
+  currentPhase: Completed
+  postDeploymentEvaluationStatus: Succeeded
+  postDeploymentStatus: Succeeded
+  preDeploymentEvaluationStatus: Succeeded
+  preDeploymentStatus: Succeeded
+  status: Succeeded
+  workloadOverallStatus: Succeeded

test/chainsaw/traces/00-install.yaml

@@ -0,0 +1,35 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment
+  labels:
+    app.kubernetes.io/name: nginx
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: nginx
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/part-of: keptnapp
+        app.kubernetes.io/name: nginx
+        app.kubernetes.io/version: 0.0.1
+    spec:
+      containers:
+        - name: nginx
+          image: nginx:latest
+          ports:
+            - containerPort: 80
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: nginx
+spec:
+  selector:
+    app.kubernetes.io/name: nginx
+  ports:
+    - protocol: TCP
+      port: 8080
+      targetPort: 80

test/chainsaw/traces/chainsaw-test.yaml

@@ -0,0 +1,39 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: traces
+spec:
+  namespaceTemplate:
+    metadata:
+      annotations:
+        keptn.sh/lifecycle-toolkit: enabled
+  steps:
+    - name: step-00
+      try:
+        - apply:
+            file: keptnconfig.yaml
+    - name: step-01
+      try:
+        - apply:
+            file: collector.yaml
+    - name: step-02
+      try:
+        - script:
+            content: ./../common/verify-keptnconfig.sh
+        - sleep:
+            duration: 30s
+    - name: step-03
+      try:
+        - apply:
+            file: 00-install.yaml
+        - assert:
+            file: 00-assert.yaml
+    - name: step-04
+      try:
+        - sleep:
+            duration: 30s
+    - name: step-05
+      try:
+        - script:
+            content: ./verify-traces.sh

test/chainsaw/traces/collector.yaml

@@ -0,0 +1,141 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: otel-collector-conf
+  namespace: keptn-system
+  labels:
+    app: opentelemetry
+    component: otel-collector-conf
+data:
+  otel-collector-config: |
+    receivers:
+      # Make sure to add the otlp receiver.
+      # This will open up the receiver on port 4317
+      otlp:
+        protocols:
+          grpc:
+            endpoint: "0.0.0.0:4317"
+    processors:
+    extensions:
+      health_check: {}
+    exporters:
+      otlp:
+        endpoint: "0.0.0.0:4317"
+        tls:
+          insecure: true
+      logging:
+        verbosity: detailed
+
+    service:
+      extensions: [health_check]
+      pipelines:
+        traces:
+          receivers: [otlp]
+          processors: []
+          exporters: [otlp, logging]
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: otel-collector
+  namespace: keptn-system
+  labels:
+    app: opentelemetry
+    component: otel-collector
+spec:
+  ports:
+    - name: otlp # Default endpoint for otlp receiver.
+      port: 4317
+      protocol: TCP
+      targetPort: 4317
+      nodePort: 30080
+    - name: metrics # Default endpoint for metrics.
+      port: 8889
+      protocol: TCP
+      targetPort: 8889
+  selector:
+    component: otel-collector
+  type: NodePort
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: otel-collector
+  namespace: keptn-system
+  labels:
+    app: opentelemetry
+    component: otel-collector
+spec:
+  selector:
+    matchLabels:
+      app: opentelemetry
+      component: otel-collector
+  minReadySeconds: 5
+  progressDeadlineSeconds: 120
+  replicas: 1
+  template:
+    metadata:
+      annotations:
+        prometheus.io/path: "/metrics"
+        prometheus.io/port: "8889"
+        prometheus.io/scrape: "true"
+      labels:
+        app: opentelemetry
+        component: otel-collector
+    spec:
+      containers:
+        - command:
+            - "/otelcol"
+            - "--config=/conf/otel-collector-config.yaml"
+          env:
+            - name: GOGC
+              value: "80"
+          image: otel/opentelemetry-collector:0.95.0
+          name: otel-collector
+          resources:
+            limits:
+              cpu: 400m
+              memory: 1Gi
+            requests:
+              cpu: 75m
+              memory: 200Mi
+          ports:
+            - containerPort: 4317 # Default endpoint for otlp receiver.
+            - containerPort: 8889 # Default endpoint for querying metrics.
+          volumeMounts:
+            - name: otel-collector-config-vol
+              mountPath: /conf
+          livenessProbe:
+            httpGet:
+              path: /
+              port: 13133 # Health Check extension default port.
+          readinessProbe:
+            httpGet:
+              path: /
+              port: 13133 # Health Check extension default port.
+      volumes:
+        - configMap:
+            name: otel-collector-conf
+            items:
+              - key: otel-collector-config
+                path: otel-collector-config.yaml
+          name: otel-collector-config-vol
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    serviceapp: otel-collector
+  name: otel-collector
+  namespace: keptn-system
+spec:
+  endpoints:
+    - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+      interval: 30s
+      port: metrics
+  namespaceSelector:
+    matchNames:
+      - keptn-system
+  selector:
+    matchLabels:
+      app: opentelemetry

test/chainsaw/traces/keptnconfig.yaml

@@ -0,0 +1,6 @@
+apiVersion: options.keptn.sh/v1alpha1
+kind: KeptnConfig
+metadata:
+  name: keptnconfig
+spec:
+  OTelCollectorUrl: 'otel-collector.keptn-system.svc.cluster.local:4317'

test/chainsaw/traces/verify-traces.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+KEPTN_NAMESPACE="keptn-system"
+RETRY_COUNT=5
+SLEEP_TIME=5
+# span name which we expect to find in the logs of otel-collector
+EXPECTED_SUBSTRINGS=(
+    Name : keptnapp-0.0.1-6b86b273
+    Name : AppPreDeployTasks
+    Name : AppPreDeployEvaluations
+    Name : AppDeploy
+    Name : keptnapp-nginx/WorkloadDeploy
+    Name : keptnapp-nginx/WorkloadPreDeployTasks
+    Name : keptnapp-nginx/WorkloadPreDeployEvaluations
+    Name : keptnapp-nginx/WorkloadDeploy
+    Name : keptnapp-nginx/WorkloadPostDeployTasks
+    Name : keptnapp-nginx/WorkloadPostDeployEvaluations
+    Name : AppPostDeployTasks
+    Name : AppPostDeployEvaluations
+    keptn.deployment.app.namespace: $NAMESPACE
+)
+
+check_variable_contains_substrings() {
+    # remove unneeded whitespaces
+    local variable2=$(echo "$1" | sed 's/\s\+/ /g')
+    local variable=$(echo "$variable2" | sed "s/Str($NAMESPACE)/$NAMESPACE/g")
+    local -a substrings=("${@:2}")
+
+    local contains_all=true
+
+    # Iterate over the substrings array
+    for substring in "${substrings[@]}"; do
+        # Check if the variable contains the current substring
+        if [[ ! $variable =~ $substring ]]; then
+            # If the substring is not found, set the flag to false and break the loop
+            contains_all=false
+            break
+        fi
+    done
+
+    # Return the flag
+    echo "$contains_all"
+}
+
+for i in $(seq 1 $RETRY_COUNT); do
+    VAR=$(kubectl logs -n "$KEPTN_NAMESPACE" deployment/otel-collector)
+    result=$(check_variable_contains_substrings "$VAR" "${EXPECTED_SUBSTRINGS[@]}")
+    # shellcheck disable=SC1072
+    if [ "$result" = true ]; then
+        echo "All traces found, test passed!"
+        exit 0
+    fi
+    if [ "$i" -lt "$RETRY_COUNT" ]; then
+            echo "Sleeping for ${SLEEP_TIME} seconds before retrying..."
+            sleep ${SLEEP_TIME}
+    fi
+done
+
+echo "Retried ${RETRY_COUNT} times, but correct traces were not found. Exiting..."
+exit 1