diff --git a/.github/actions/archive-artifacts/action.yaml b/.github/actions/archive-artifacts/action.yaml index 6d02f751da..747e3b1471 100644 --- a/.github/actions/archive-artifacts/action.yaml +++ b/.github/actions/archive-artifacts/action.yaml @@ -74,26 +74,29 @@ runs: set -exu KAFKA=$(kubectl get pods -n ${NAMESPACE} -lkafka_cr=${ZENKO_NAME}-base-queue -o jsonpath='{.items[0].metadata.name}') + KAFKA_PATH="/tmp/artifacts/data/${STAGE}/kafka" + + mkdir -p ${KAFKA_PATH} kubectl exec -in ${NAMESPACE} ${KAFKA} -c kafka -- \ env KAFKA_OPTS= kafka-topics.sh --bootstrap-server :9092 --list \ - > /tmp/artifacts/data/${STAGE}/kafka-topics.log + > ${KAFKA_PATH}/kafka-topics.log kubectl exec -in ${NAMESPACE} ${KAFKA} -c kafka -- \ env KAFKA_OPTS= kafka-consumer-groups.sh --bootstrap-server :9092 --list \ - > /tmp/artifacts/data/${STAGE}/kafka-consumer-groups.log + > ${KAFKA_PATH}/kafka-consumer-groups.log kubectl exec -in ${NAMESPACE} ${KAFKA} -c kafka -- \ env KAFKA_OPTS= kafka-consumer-groups.sh --bootstrap-server :9092 --describe --all-groups \ - > /tmp/artifacts/data/${STAGE}/kafka-offsets.log + > ${KAFKA_PATH}/kafka-offsets.log KAFKA_SERVICE=$(kubectl get services -n ${NAMESPACE} -lkafka_cr=${ZENKO_NAME}-base-queue -o jsonpath='{.items[0].metadata.name}') kubectl run -n ${NAMESPACE} kcat --image=edenhill/kcat:1.7.1 --restart=Never --command -- sleep 300 kubectl wait -n ${NAMESPACE} pod kcat --for=condition=ready - cat /tmp/artifacts/data/${STAGE}/kafka-topics.log | grep -v '^__' | xargs -P 15 -I {} \ + cat ${KAFKA_PATH}/kafka-topics.log | grep -v '^__' | xargs -P 15 -I {} \ sh -c "kubectl exec -i -n ${NAMESPACE} kcat -- \ kcat -L -b ${KAFKA_SERVICE} -t {} -C -o beginning -e -q -J \ - > /tmp/artifacts/data/${STAGE}/kafka-messages-{}.log" + > ${KAFKA_PATH}/kafka-messages-{}.log" env: STAGE: ${{ inputs.stage }} NAMESPACE: ${{ inputs.zenko-namespace }} diff --git a/.github/actions/deploy/action.yaml b/.github/actions/deploy/action.yaml index 1fc7708f0c..5c4a76cfce 100644 --- a/.github/actions/deploy/action.yaml +++ b/.github/actions/deploy/action.yaml @@ -64,9 +64,10 @@ runs: docker pull ${OPERATOR_IMAGE_NAME}:${OPERATOR_IMAGE_TAG} kind load docker-image ${OPERATOR_IMAGE_NAME}:${OPERATOR_IMAGE_TAG} cd ./.github/scripts/end2end - git clone https://${GIT_ACCESS_TOKEN}@github.com/scality/zenko-operator.git operator + git init operator cd operator - git checkout ${OPERATOR_IMAGE_TAG} + git fetch --depth 1 --no-tags https://${GIT_ACCESS_TOKEN}@github.com/scality/zenko-operator.git ${OPERATOR_IMAGE_TAG} + git checkout FETCH_HEAD tilt ci env: OPERATOR_IMAGE_TAG: ${{ inputs.zkop_tag }} diff --git a/.github/scripts/end2end/configs/prometheus.yaml b/.github/scripts/end2end/configs/prometheus.yaml index e885d6325b..6a22731175 100644 --- a/.github/scripts/end2end/configs/prometheus.yaml +++ b/.github/scripts/end2end/configs/prometheus.yaml @@ -36,13 +36,19 @@ spec: evaluationInterval: 30s logFormat: logfmt logLevel: info + serviceMonitorNamespaceSelector: {} + serviceMonitorSelector: + matchLabels: + metalk8s.scality.com/monitor: "" podMonitorNamespaceSelector: {} podMonitorSelector: matchLabels: metalk8s.scality.com/monitor: "" + probeNamespaceSelector: {} probeSelector: matchLabels: metalk8s.scality.com/monitor: "" + ruleNamespaceSelector: {} ruleSelector: matchLabels: metalk8s.scality.com/monitor: "" diff --git a/.github/scripts/end2end/install-kind-dependencies.sh b/.github/scripts/end2end/install-kind-dependencies.sh index 2db3fa4a94..5a4b35e5b5 100755 --- a/.github/scripts/end2end/install-kind-dependencies.sh +++ b/.github/scripts/end2end/install-kind-dependencies.sh @@ -58,14 +58,21 @@ kubectl rollout status -n ingress-nginx deployment/ingress-nginx-controller --ti # cert-manager kubectl apply --validate=false -f https://github.com/jetstack/cert-manager/releases/download/${CERT_MANAGER_VERSION}/cert-manager.yaml --wait -# kubectl apply --validate=false -f - < 0, ...kafkaExternalIpOption, timeout, @@ -328,8 +328,25 @@ Then('the kafka DR volume exists', { timeout: volumeTimeout + 2000 }, async func assert(volumeParsed.result!['volume phase'] === 'Bound'); }); +Then('prometheus should scrap federated metrics from DR sink', { timeout: 70000 }, async function (this: Zenko) { + const prom = new PrometheusDriver({ + endpoint: `http://${this.parameters.PrometheusService}:9090`, + baseURL: '/api/v1', + }); + + for (;;) { + const t = Date.now(); + const metrics = await prom.series('{drSinkInstance="end2end-pra-sink"}', t - 60 * 1000, t); + if (metrics.length > 0) { + break; + } + + await Utils.sleep(1000); + } +}); + const failoverTimeout = 360000; -When ('I request the failover state for the DR', { timeout: failoverTimeout + 2000 }, async function (this: Zenko) { +When('I request the failover state for the DR', { timeout: failoverTimeout + 2000 }, async function (this: Zenko) { await this.zenkoDrCtl?.failover({ sinkZenkoDrNamespace: 'default', sinkZenkoDrInstance: 'end2end-pra-sink', @@ -339,7 +356,7 @@ When ('I request the failover state for the DR', { timeout: failoverTimeout + 20 }); const failbackTimeout = 360000; -When ('I resume operations for the DR', { timeout: failbackTimeout + 2000 }, async function (this: Zenko) { +When('I resume operations for the DR', { timeout: failbackTimeout + 2000 }, async function (this: Zenko) { await this.zenkoDrCtl?.failback({ sinkZenkoDrNamespace: 'default', sinkZenkoDrInstance: 'end2end-pra-sink', diff --git a/tests/zenko_tests/node_tests/backbeat/ReplicationUtility.js b/tests/zenko_tests/node_tests/backbeat/ReplicationUtility.js index d80bf5e1d4..3e5c8abaa1 100644 --- a/tests/zenko_tests/node_tests/backbeat/ReplicationUtility.js +++ b/tests/zenko_tests/node_tests/backbeat/ReplicationUtility.js @@ -1,6 +1,7 @@ const assert = require('assert'); const crypto = require('crypto'); const async = require('async'); +const { jsutil } = require('arsenal'); const { scalityS3Client, awsS3Client } = require('../s3SDK'); @@ -589,28 +590,28 @@ class ReplicationUtility { // Continue getting head object while the status is PENDING or PROCESSING. waitUntilReplicated(bucketName, key, versionId, cb) { - let status; return async.doWhilst( callback => this.s3.headObject({ Bucket: bucketName, Key: key, VersionId: versionId, }, (err, data) => { + const cbOnce = jsutil.once(callback); if (err) { - return callback(err); + return cbOnce(err); } - status = data.ReplicationStatus; + const status = data.ReplicationStatus; assert.notStrictEqual( status, 'FAILED', `Unexpected CRR failure occurred: ${JSON.stringify(data)}`, ); if (status === 'PENDING' || status === 'PROCESSING') { - return setTimeout(callback, 2000); + return setTimeout(() => cbOnce(null, status), 2000); } - return callback(); + return cbOnce(null, status); }), - () => (status === 'PENDING' || status === 'PROCESSING'), + status => (status === 'PENDING' || status === 'PROCESSING'), cb, ); } @@ -622,14 +623,15 @@ class ReplicationUtility { const expectedCode = client === 'azure' ? 'BlobNotFound' : 'NoSuchKey'; return async.doWhilst( callback => this[method](bucketName, key, err => { + const cbOnce = jsutil.once(callback); if (err && err.code !== expectedCode) { - return callback(err); + return cbOnce(err); } objectExists = err === null; if (!objectExists) { - return callback(); + return cbOnce(); } - return setTimeout(callback, 2000); + return setTimeout(cbOnce, 2000); }), () => objectExists, cb, @@ -644,8 +646,9 @@ class ReplicationUtility { Bucket: bucketName, Key: key, }, (err, data) => { + const cbOnce = jsutil.once(callback); if (err) { - return callback(err); + return cbOnce(err); } const statuses = []; // We cannot rely on the global status for one-to-many, so check @@ -657,9 +660,9 @@ class ReplicationUtility { }); shouldContinue = statuses.includes('PENDING'); if (shouldContinue) { - return setTimeout(callback, 2000); + return setTimeout(cbOnce, 2000); } - return callback(); + return cbOnce(); }), () => shouldContinue, cb, @@ -674,14 +677,15 @@ class ReplicationUtility { Bucket: bucketName, Key: key, }, (err, data) => { + const cbOnce = jsutil.once(callback); if (err) { - return callback(err); + return cbOnce(err); } shouldContinue = data.ReplicationStatus === 'FAILED'; if (shouldContinue) { - return setTimeout(callback, 2000); + return setTimeout(cbOnce, 2000); } - return callback(); + return cbOnce(); }), () => shouldContinue, cb,