Merge remote-tracking branch 'origin/improvement/ZENKO-4876' into imp…

…rovement/ZENKO-4919

.github/actions/archive-artifacts/action.yaml

@@ -74,26 +74,29 @@ runs:
         set -exu
 
         KAFKA=$(kubectl get pods -n ${NAMESPACE} -lkafka_cr=${ZENKO_NAME}-base-queue -o jsonpath='{.items[0].metadata.name}')
+        KAFKA_PATH="/tmp/artifacts/data/${STAGE}/kafka"
+        
+        mkdir -p ${KAFKA_PATH}
 
         kubectl exec -in ${NAMESPACE} ${KAFKA} -c kafka -- \
             env KAFKA_OPTS= kafka-topics.sh --bootstrap-server :9092 --list \
-            > /tmp/artifacts/data/${STAGE}/kafka-topics.log
+            > ${KAFKA_PATH}/kafka-topics.log
 
         kubectl exec -in ${NAMESPACE} ${KAFKA} -c kafka -- \
             env KAFKA_OPTS= kafka-consumer-groups.sh --bootstrap-server :9092 --list \
-            > /tmp/artifacts/data/${STAGE}/kafka-consumer-groups.log
+            > ${KAFKA_PATH}/kafka-consumer-groups.log
 
         kubectl exec -in ${NAMESPACE} ${KAFKA} -c kafka  -- \
                    env KAFKA_OPTS= kafka-consumer-groups.sh --bootstrap-server :9092 --describe --all-groups \
-                   > /tmp/artifacts/data/${STAGE}/kafka-offsets.log
+                   > ${KAFKA_PATH}/kafka-offsets.log
 
         KAFKA_SERVICE=$(kubectl get services -n ${NAMESPACE} -lkafka_cr=${ZENKO_NAME}-base-queue -o jsonpath='{.items[0].metadata.name}')
         kubectl run -n ${NAMESPACE} kcat --image=edenhill/kcat:1.7.1 --restart=Never --command -- sleep 300        
         kubectl wait -n ${NAMESPACE} pod kcat --for=condition=ready
-        cat /tmp/artifacts/data/${STAGE}/kafka-topics.log | grep -v '^__' | xargs -P 15 -I {} \
+        cat ${KAFKA_PATH}/kafka-topics.log | grep -v '^__' | xargs -P 15 -I {} \
             sh -c "kubectl exec -i -n ${NAMESPACE} kcat -- \
                        kcat -L -b ${KAFKA_SERVICE} -t {}  -C -o beginning -e -q -J \
-                       > /tmp/artifacts/data/${STAGE}/kafka-messages-{}.log"
+                       > ${KAFKA_PATH}/kafka-messages-{}.log"
       env:
         STAGE: ${{ inputs.stage }}
         NAMESPACE: ${{ inputs.zenko-namespace }}

.github/actions/deploy/action.yaml

@@ -64,9 +64,10 @@ runs:
         docker pull ${OPERATOR_IMAGE_NAME}:${OPERATOR_IMAGE_TAG}
         kind load docker-image ${OPERATOR_IMAGE_NAME}:${OPERATOR_IMAGE_TAG}
         cd ./.github/scripts/end2end
-        git clone https://${GIT_ACCESS_TOKEN}@github.com/scality/zenko-operator.git operator
+        git init operator
         cd operator
-        git checkout ${OPERATOR_IMAGE_TAG}
+        git fetch --depth 1 --no-tags https://${GIT_ACCESS_TOKEN}@github.com/scality/zenko-operator.git ${OPERATOR_IMAGE_TAG}
+        git checkout FETCH_HEAD
         tilt ci
       env:
         OPERATOR_IMAGE_TAG: ${{ inputs.zkop_tag }}

.github/scripts/end2end/configs/prometheus.yaml

@@ -36,13 +36,19 @@ spec:
   evaluationInterval: 30s
   logFormat: logfmt
   logLevel: info
+  serviceMonitorNamespaceSelector: {}
+  serviceMonitorSelector:
+    matchLabels:
+      metalk8s.scality.com/monitor: ""
   podMonitorNamespaceSelector: {}
   podMonitorSelector:
     matchLabels:
       metalk8s.scality.com/monitor: ""
+  probeNamespaceSelector: {}
   probeSelector:
     matchLabels:
       metalk8s.scality.com/monitor: ""
+  ruleNamespaceSelector: {}
   ruleSelector:
     matchLabels:
       metalk8s.scality.com/monitor: ""

.github/scripts/end2end/install-kind-dependencies.sh

@@ -58,14 +58,15 @@ kubectl rollout status -n ingress-nginx deployment/ingress-nginx-controller --ti
 
 # cert-manager
 kubectl apply --validate=false -f https://github.com/jetstack/cert-manager/releases/download/${CERT_MANAGER_VERSION}/cert-manager.yaml --wait
-# kubectl apply --validate=false -f - <<EOF
-# apiVersion: cert-manager.io/v1
-# kind: ClusterIssuer
-# metadata:
-#   name: artesca-root-ca-issuer
-# spec:
-#   selfSigned: {}
-# EOF
+kubectl rollout status -n cert-manager deployment/cert-manager-webhook --timeout=10m
+kubectl apply -f - <<EOF
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: artesca-root-ca-issuer
+spec:
+  selfSigned: {}
+EOF
 
 # prometheus
 # last-applied-configuration can end up larger than 256kB  which is too large for an annotation

.github/scripts/end2end/patch-coredns.sh

@@ -31,6 +31,7 @@ corefile="
     rewrite name exact s3.dr.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
     rewrite name exact sts.dr.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
     rewrite name exact iam.dr.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
+    rewrite name exact prom.dr.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
     rewrite name exact shell-ui.dr.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
     rewrite name exact website.mywebsite.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
     kubernetes cluster.local in-addr.arpa ip6.arpa {

.github/workflows/alerts.yaml

@@ -1,10 +1,11 @@
 name: Test alerts
 
 on:
-  push:
-    branches-ignore:
-      - 'development/**'
-      - 'q/*'
+  workflow_call:
+    secrets:
+      GIT_ACCESS_TOKEN:
+        description: 'GitHub token'
+        required: true
 
 jobs:
   run-alert-tests:

.github/workflows/end2end.yaml

@@ -121,6 +121,10 @@ jobs:
       - name: Verify monitoring dashboard versions
         run: bash ./.github/scripts/check_versions.sh
 
+  check-alerts:
+    uses: ./.github/workflows/alerts.yaml
+    secrets: inherit
+
   check-workflows:
     runs-on: ubuntu-22.04
     steps:
@@ -379,7 +383,7 @@ jobs:
           cache-to: type=gha,mode=max,scope=end2end-ctst
 
   end2end-http:
-    needs: [build-kafka, build-test-image, check-dashboard-versions]
+    needs: [build-kafka, build-test-image]
     runs-on:
       - ubuntu
       - focal
@@ -428,7 +432,7 @@ jobs:
         run: kind delete cluster
 
   end2end-pra:
-    needs: [build-kafka, check-dashboard-versions, lint-and-build-ctst]
+    needs: [build-kafka, lint-and-build-ctst]
     runs-on: ubuntu-22.04-16core
     env:
       GIT_ACCESS_TOKEN: ${{ secrets.GIT_ACCESS_TOKEN }}
@@ -488,7 +492,7 @@ jobs:
         run: kind delete cluster
 
   end2end-https:
-    needs: [build-kafka, build-test-image, check-dashboard-versions]
+    needs: [build-kafka, build-test-image]
     runs-on:
       - ubuntu
       - focal
@@ -540,7 +544,7 @@ jobs:
         run: kind delete cluster
 
   end2end-sharded:
-    needs: [build-kafka, build-test-image, check-dashboard-versions]
+    needs: [build-kafka, build-test-image]
     runs-on:
       - ubuntu-22.04-8core
       # Enable this for Ring-based tests
@@ -580,7 +584,7 @@ jobs:
         run: kind delete cluster
 
   ctst-end2end-sharded:
-    needs: [build-kafka, lint-and-build-ctst, check-dashboard-versions]
+    needs: [build-kafka, lint-and-build-ctst]
     runs-on:
       - ubuntu-22.04-8core
     steps:
@@ -629,6 +633,7 @@ jobs:
   write-final-status:
     runs-on: ubuntu-latest
     needs:
+      - check-alerts
       - check-dashboard-versions
       - check-workflows
       - build-doc

tests/ctst/features/pra.feature

@@ -18,7 +18,8 @@ Feature: PRA operations
     Given a DR installed
     Then the DR source should be in phase "Running"
     And the DR sink should be in phase "Running"
-    Then the kafka DR volume exists
+    And the kafka DR volume exists
+    And prometheus should scrap federated metrics from DR sink
 
     # Check that objects are transitioned in the DR site
     Given access keys for the replicated account

tests/ctst/steps/pra.ts

@@ -15,6 +15,7 @@ import {
 } from 'steps/utils/utils';
 import { CacheHelper, Constants, Identity, IdentityEnum, SuperAdmin, Utils } from 'cli-testing';
 import { safeJsonParse } from 'common/utils';
+import { PrometheusDriver } from 'prometheus-query';
 import assert from 'assert';
 import { EntityType } from 'world/Zenko';
 
@@ -70,9 +71,8 @@ async function installPRA(world: Zenko, sinkS3Endpoint = 'http://s3.zenko.local'
         sourceS3Endpoint: 'http://s3.zenko.local',
         sinkS3Endpoint,
         prometheusService: world.parameters.PrometheusService,
-        // prometheusHostname: 'prom.dr.zenko.local', // could be any name, cert will be auto-generated
+        prometheusHostname: 'prom.dr.zenko.local',
         prometheusExternalIpsDiscovery: true,
-        prometheusDisableTls: true,
         forceRotateServiceCredentials: (CacheHelper.savedAcrossTests[Zenko.PRA_INSTALL_COUNT_KEY] as number) > 0,
         ...kafkaExternalIpOption,
         timeout,
@@ -328,8 +328,25 @@ Then('the kafka DR volume exists', { timeout: volumeTimeout + 2000 }, async func
     assert(volumeParsed.result!['volume phase'] === 'Bound');
 });
 
+Then('prometheus should scrap federated metrics from DR sink', { timeout: 70000 }, async function (this: Zenko) {
+    const prom = new PrometheusDriver({
+        endpoint: `http://${this.parameters.PrometheusService}:9090`,
+        baseURL: '/api/v1',
+    });
+
+    for (;;) {
+        const t = Date.now();
+        const metrics = await prom.series('{drSinkInstance="end2end-pra-sink"}', t - 60 * 1000, t);
+        if (metrics.length > 0) {
+            break;
+        }
+
+        await Utils.sleep(1000);
+    }
+});
+
 const failoverTimeout = 360000;
-When ('I request the failover state for the DR', { timeout: failoverTimeout + 2000 }, async function (this: Zenko) {
+When('I request the failover state for the DR', { timeout: failoverTimeout + 2000 }, async function (this: Zenko) {
     await this.zenkoDrCtl?.failover({
         sinkZenkoDrNamespace: 'default',
         sinkZenkoDrInstance: 'end2end-pra-sink',
@@ -339,7 +356,7 @@ When ('I request the failover state for the DR', { timeout: failoverTimeout + 20
 });
 
 const failbackTimeout = 360000;
-When ('I resume operations for the DR', { timeout: failbackTimeout + 2000 }, async function (this: Zenko) {
+When('I resume operations for the DR', { timeout: failbackTimeout + 2000 }, async function (this: Zenko) {
     await this.zenkoDrCtl?.failback({
         sinkZenkoDrNamespace: 'default',
         sinkZenkoDrInstance: 'end2end-pra-sink',