diff --git a/.github/actions/archive-artifacts/action.yaml b/.github/actions/archive-artifacts/action.yaml index 6d02f751da..747e3b1471 100644 --- a/.github/actions/archive-artifacts/action.yaml +++ b/.github/actions/archive-artifacts/action.yaml @@ -74,26 +74,29 @@ runs: set -exu KAFKA=$(kubectl get pods -n ${NAMESPACE} -lkafka_cr=${ZENKO_NAME}-base-queue -o jsonpath='{.items[0].metadata.name}') + KAFKA_PATH="/tmp/artifacts/data/${STAGE}/kafka" + + mkdir -p ${KAFKA_PATH} kubectl exec -in ${NAMESPACE} ${KAFKA} -c kafka -- \ env KAFKA_OPTS= kafka-topics.sh --bootstrap-server :9092 --list \ - > /tmp/artifacts/data/${STAGE}/kafka-topics.log + > ${KAFKA_PATH}/kafka-topics.log kubectl exec -in ${NAMESPACE} ${KAFKA} -c kafka -- \ env KAFKA_OPTS= kafka-consumer-groups.sh --bootstrap-server :9092 --list \ - > /tmp/artifacts/data/${STAGE}/kafka-consumer-groups.log + > ${KAFKA_PATH}/kafka-consumer-groups.log kubectl exec -in ${NAMESPACE} ${KAFKA} -c kafka -- \ env KAFKA_OPTS= kafka-consumer-groups.sh --bootstrap-server :9092 --describe --all-groups \ - > /tmp/artifacts/data/${STAGE}/kafka-offsets.log + > ${KAFKA_PATH}/kafka-offsets.log KAFKA_SERVICE=$(kubectl get services -n ${NAMESPACE} -lkafka_cr=${ZENKO_NAME}-base-queue -o jsonpath='{.items[0].metadata.name}') kubectl run -n ${NAMESPACE} kcat --image=edenhill/kcat:1.7.1 --restart=Never --command -- sleep 300 kubectl wait -n ${NAMESPACE} pod kcat --for=condition=ready - cat /tmp/artifacts/data/${STAGE}/kafka-topics.log | grep -v '^__' | xargs -P 15 -I {} \ + cat ${KAFKA_PATH}/kafka-topics.log | grep -v '^__' | xargs -P 15 -I {} \ sh -c "kubectl exec -i -n ${NAMESPACE} kcat -- \ kcat -L -b ${KAFKA_SERVICE} -t {} -C -o beginning -e -q -J \ - > /tmp/artifacts/data/${STAGE}/kafka-messages-{}.log" + > ${KAFKA_PATH}/kafka-messages-{}.log" env: STAGE: ${{ inputs.stage }} NAMESPACE: ${{ inputs.zenko-namespace }} diff --git a/.github/actions/deploy/action.yaml b/.github/actions/deploy/action.yaml index 1fc7708f0c..5c4a76cfce 100644 --- a/.github/actions/deploy/action.yaml +++ b/.github/actions/deploy/action.yaml @@ -64,9 +64,10 @@ runs: docker pull ${OPERATOR_IMAGE_NAME}:${OPERATOR_IMAGE_TAG} kind load docker-image ${OPERATOR_IMAGE_NAME}:${OPERATOR_IMAGE_TAG} cd ./.github/scripts/end2end - git clone https://${GIT_ACCESS_TOKEN}@github.com/scality/zenko-operator.git operator + git init operator cd operator - git checkout ${OPERATOR_IMAGE_TAG} + git fetch --depth 1 --no-tags https://${GIT_ACCESS_TOKEN}@github.com/scality/zenko-operator.git ${OPERATOR_IMAGE_TAG} + git checkout FETCH_HEAD tilt ci env: OPERATOR_IMAGE_TAG: ${{ inputs.zkop_tag }} diff --git a/.github/scripts/end2end/configs/prometheus.yaml b/.github/scripts/end2end/configs/prometheus.yaml index e885d6325b..6a22731175 100644 --- a/.github/scripts/end2end/configs/prometheus.yaml +++ b/.github/scripts/end2end/configs/prometheus.yaml @@ -36,13 +36,19 @@ spec: evaluationInterval: 30s logFormat: logfmt logLevel: info + serviceMonitorNamespaceSelector: {} + serviceMonitorSelector: + matchLabels: + metalk8s.scality.com/monitor: "" podMonitorNamespaceSelector: {} podMonitorSelector: matchLabels: metalk8s.scality.com/monitor: "" + probeNamespaceSelector: {} probeSelector: matchLabels: metalk8s.scality.com/monitor: "" + ruleNamespaceSelector: {} ruleSelector: matchLabels: metalk8s.scality.com/monitor: "" diff --git a/.github/scripts/end2end/install-kind-dependencies.sh b/.github/scripts/end2end/install-kind-dependencies.sh index a2c392411e..523f25a62c 100755 --- a/.github/scripts/end2end/install-kind-dependencies.sh +++ b/.github/scripts/end2end/install-kind-dependencies.sh @@ -58,14 +58,15 @@ kubectl rollout status -n ingress-nginx deployment/ingress-nginx-controller --ti # cert-manager kubectl apply --validate=false -f https://github.com/jetstack/cert-manager/releases/download/${CERT_MANAGER_VERSION}/cert-manager.yaml --wait -# kubectl apply --validate=false -f - <" bucket + And a transition workflow to "e2e-cold" location + And objects "obj" of size bytes + Then object "obj-1" should be "transitioned" and have the storage class "e2e-cold" + And dmf volume should contain objects + Given mpu objects "obj" of size bytes + Then object "obj-1" should be "transitioned" and have the storage class "e2e-cold" + And dmf volume should contain 1 objects + + Examples: + | versioningConfiguration | objectCount | objectSize | + | Non versioned | 1 | 100 | + | Suspended | 1 | 100 | + + @2.7.0 + @PreMerge + @Dmf + @ColdStorage + Scenario Outline: Overwriting of a cold object with copyObject + Given a "" bucket + And a transition workflow to "e2e-cold" location + And 2 objects "obj" of size bytes + Then object "obj-1" should be "transitioned" and have the storage class "e2e-cold" + And object "obj-2" should be "transitioned" and have the storage class "e2e-cold" + And dmf volume should contain 2 objects + When i restore object "obj-1" for 5 days + Then object "obj-1" should be "restored" and have the storage class "e2e-cold" + Given "obj-1" is copied to "obj-2" + Then object "obj-2" should be "transitioned" and have the storage class "e2e-cold" + And dmf volume should contain 2 objects + + Examples: + | versioningConfiguration | objectSize | + | Non versioned | 100 | + | Suspended | 100 | + + @2.7.0 + @PreMerge + @Dmf + @ColdStorage + Scenario Outline: Overwriting of a cold object with mpu + Given a "" bucket + And a transition workflow to "e2e-cold" location + And objects "obj" of size bytes + Then object "obj-1" should be "transitioned" and have the storage class "e2e-cold" + And dmf volume should contain objects + Given mpu objects "obj" of size bytes + Then object "obj-1" should be "transitioned" and have the storage class "e2e-cold" + And dmf volume should contain 1 objects + + Examples: + | versioningConfiguration | objectCount | objectSize | + | Non versioned | 1 | 100 | + | Suspended | 1 | 100 | + + @2.7.0 + @PreMerge + @Dmf + @ColdStorage + Scenario Outline: Overwriting of a cold object with copyObject + Given a "" bucket + And a transition workflow to "e2e-cold" location + And 2 objects "obj" of size bytes + Then object "obj-1" should be "transitioned" and have the storage class "e2e-cold" + And object "obj-2" should be "transitioned" and have the storage class "e2e-cold" + And dmf volume should contain 2 objects + When i restore object "obj-1" for 5 days + Then object "obj-1" should be "restored" and have the storage class "e2e-cold" + Given "obj-1" is copied to "obj-2" + Then object "obj-2" should be "transitioned" and have the storage class "e2e-cold" + And dmf volume should contain 2 objects + + Examples: + | versioningConfiguration | objectSize | + | Non versioned | 100 | + | Suspended | 100 | @2.7.0 @PreMerge @@ -125,4 +206,4 @@ Feature: DMF | versioningConfiguration | objectCount | objectSize | restoreDays | | Non versioned | 2 | 100 | 15 | | Versioned | 2 | 100 | 15 | - | Suspended | 2 | 100 | 15 | \ No newline at end of file + | Suspended | 2 | 100 | 15 | diff --git a/tests/ctst/features/pra.feature b/tests/ctst/features/pra.feature index f0aede1e32..0199b137b2 100644 --- a/tests/ctst/features/pra.feature +++ b/tests/ctst/features/pra.feature @@ -18,7 +18,8 @@ Feature: PRA operations Given a DR installed Then the DR source should be in phase "Running" And the DR sink should be in phase "Running" - Then the kafka DR volume exists + And the kafka DR volume exists + And prometheus should scrap federated metrics from DR sink # Check that objects are transitioned in the DR site Given access keys for the replicated account diff --git a/tests/ctst/steps/pra.ts b/tests/ctst/steps/pra.ts index f8cc5a1725..6f35d19425 100644 --- a/tests/ctst/steps/pra.ts +++ b/tests/ctst/steps/pra.ts @@ -15,6 +15,7 @@ import { } from 'steps/utils/utils'; import { CacheHelper, Constants, Identity, IdentityEnum, SuperAdmin, Utils } from 'cli-testing'; import { safeJsonParse } from 'common/utils'; +import { PrometheusDriver } from 'prometheus-query'; import assert from 'assert'; import { EntityType } from 'world/Zenko'; @@ -70,9 +71,8 @@ async function installPRA(world: Zenko, sinkS3Endpoint = 'http://s3.zenko.local' sourceS3Endpoint: 'http://s3.zenko.local', sinkS3Endpoint, prometheusService: world.parameters.PrometheusService, - // prometheusHostname: 'prom.dr.zenko.local', // could be any name, cert will be auto-generated + prometheusHostname: 'prom.dr.zenko.local', prometheusExternalIpsDiscovery: true, - prometheusDisableTls: true, forceRotateServiceCredentials: (CacheHelper.savedAcrossTests[Zenko.PRA_INSTALL_COUNT_KEY] as number) > 0, ...kafkaExternalIpOption, timeout, @@ -328,8 +328,25 @@ Then('the kafka DR volume exists', { timeout: volumeTimeout + 2000 }, async func assert(volumeParsed.result!['volume phase'] === 'Bound'); }); +Then('prometheus should scrap federated metrics from DR sink', { timeout: 70000 }, async function (this: Zenko) { + const prom = new PrometheusDriver({ + endpoint: `http://${this.parameters.PrometheusService}:9090`, + baseURL: '/api/v1', + }); + + for (;;) { + const t = Date.now(); + const metrics = await prom.series('{drSinkInstance="end2end-pra-sink"}', t - 60 * 1000, t); + if (metrics.length > 0) { + break; + } + + await Utils.sleep(1000); + } +}); + const failoverTimeout = 360000; -When ('I request the failover state for the DR', { timeout: failoverTimeout + 2000 }, async function (this: Zenko) { +When('I request the failover state for the DR', { timeout: failoverTimeout + 2000 }, async function (this: Zenko) { await this.zenkoDrCtl?.failover({ sinkZenkoDrNamespace: 'default', sinkZenkoDrInstance: 'end2end-pra-sink', @@ -339,7 +356,7 @@ When ('I request the failover state for the DR', { timeout: failoverTimeout + 20 }); const failbackTimeout = 360000; -When ('I resume operations for the DR', { timeout: failbackTimeout + 2000 }, async function (this: Zenko) { +When('I resume operations for the DR', { timeout: failbackTimeout + 2000 }, async function (this: Zenko) { await this.zenkoDrCtl?.failback({ sinkZenkoDrNamespace: 'default', sinkZenkoDrInstance: 'end2end-pra-sink', diff --git a/tests/ctst/steps/utils/utils.ts b/tests/ctst/steps/utils/utils.ts index dd1745bc25..9469a974c1 100644 --- a/tests/ctst/steps/utils/utils.ts +++ b/tests/ctst/steps/utils/utils.ts @@ -199,6 +199,84 @@ async function createBucketWithConfiguration( } } +async function putMpuObject(world: Zenko, parts: number = 2, objectName: string, content?: string) { + const key = objectName || `${Utils.randomString()}`; + const bucket = world.getSaved('bucketName'); + + world.resetCommand(); + world.addToSaved('objectName', objectName); + world.logger.debug('Adding mpu object', { objectName }); + world.addCommandParameter({ key }); + world.addCommandParameter({ bucket }); + const userMetadata = world.getSaved('userMetadata'); + if (userMetadata) { + world.addCommandParameter({ metadata: JSON.stringify(userMetadata) }); + } + + const initiateMPUResult = await S3.createMultipartUpload(world.getCommandParameters()); + assert.ifError(initiateMPUResult.stderr || initiateMPUResult.err); + const uploadId = extractPropertyFromResults(initiateMPUResult, 'UploadId'); + + await uploadSetup(world, 'UploadPart', content); + const body = world.getSaved('tempFileName'); + + const uploadedParts = []; + for (let i = 0; i < parts; i++) { + world.resetCommand(); + world.addCommandParameter({ key }); + world.addCommandParameter({ bucket }); + world.addCommandParameter({ partNumber: (i+1).toString() }); + world.addCommandParameter({ uploadId }); + if (body) { + world.addCommandParameter({ body }); + } + + const uploadPartResult = await S3.uploadPart(world.getCommandParameters()); + assert.ifError(uploadPartResult.stderr || uploadPartResult.err); + + uploadedParts.push({ + ETag: extractPropertyFromResults(uploadPartResult, 'ETag'), + PartNumber: (i+1).toString(), + }); + } + + await uploadTeardown(world, 'UploadPart'); + + world.resetCommand(); + world.addCommandParameter({ key }); + world.addCommandParameter({ bucket }); + world.addCommandParameter({ uploadId }); + world.addCommandParameter({ multipartUpload: JSON.stringify({ Parts: uploadedParts }) }); + + const result = await S3.completeMultipartUpload(world.getCommandParameters()); + const versionId = extractPropertyFromResults(result, 'VersionId'); + world.saveCreatedObject(objectName, versionId || ''); + world.setResult(result); + return result; +} + +async function copyObject(world: Zenko, srcObjectName?: string, dstObjectName?: string) { + const bucket = world.getSaved('bucketName'); + const key = dstObjectName || world.getSaved('objectName'); + const copySource = `${bucket}/${srcObjectName || world.getSaved('objectName')}`; + + world.resetCommand(); + world.addCommandParameter({ copySource }); + world.addCommandParameter({ bucket }); + world.addCommandParameter({ key }); + + const userMetadata = world.getSaved('userMetadata'); + if (userMetadata) { + world.addCommandParameter({ metadata: JSON.stringify(userMetadata) }); + } + + const result = await S3.copyObject(world.getCommandParameters()); + const versionId = extractPropertyFromResults(result, 'VersionId'); + world.saveCreatedObject(key, versionId || ''); + world.setResult(result); + return result; +} + async function putObject(world: Zenko, objectName?: string, content?: string) { world.resetCommand(); let finalObjectName = objectName; @@ -394,6 +472,8 @@ export { runActionAgainstBucket, createBucketWithConfiguration, getAuthorizationConfiguration, + putMpuObject, + copyObject, putObject, emptyNonVersionedBucket, emptyVersionedBucket, diff --git a/tests/zenko_tests/node_tests/backbeat/ReplicationUtility.js b/tests/zenko_tests/node_tests/backbeat/ReplicationUtility.js index d80bf5e1d4..3e5c8abaa1 100644 --- a/tests/zenko_tests/node_tests/backbeat/ReplicationUtility.js +++ b/tests/zenko_tests/node_tests/backbeat/ReplicationUtility.js @@ -1,6 +1,7 @@ const assert = require('assert'); const crypto = require('crypto'); const async = require('async'); +const { jsutil } = require('arsenal'); const { scalityS3Client, awsS3Client } = require('../s3SDK'); @@ -589,28 +590,28 @@ class ReplicationUtility { // Continue getting head object while the status is PENDING or PROCESSING. waitUntilReplicated(bucketName, key, versionId, cb) { - let status; return async.doWhilst( callback => this.s3.headObject({ Bucket: bucketName, Key: key, VersionId: versionId, }, (err, data) => { + const cbOnce = jsutil.once(callback); if (err) { - return callback(err); + return cbOnce(err); } - status = data.ReplicationStatus; + const status = data.ReplicationStatus; assert.notStrictEqual( status, 'FAILED', `Unexpected CRR failure occurred: ${JSON.stringify(data)}`, ); if (status === 'PENDING' || status === 'PROCESSING') { - return setTimeout(callback, 2000); + return setTimeout(() => cbOnce(null, status), 2000); } - return callback(); + return cbOnce(null, status); }), - () => (status === 'PENDING' || status === 'PROCESSING'), + status => (status === 'PENDING' || status === 'PROCESSING'), cb, ); } @@ -622,14 +623,15 @@ class ReplicationUtility { const expectedCode = client === 'azure' ? 'BlobNotFound' : 'NoSuchKey'; return async.doWhilst( callback => this[method](bucketName, key, err => { + const cbOnce = jsutil.once(callback); if (err && err.code !== expectedCode) { - return callback(err); + return cbOnce(err); } objectExists = err === null; if (!objectExists) { - return callback(); + return cbOnce(); } - return setTimeout(callback, 2000); + return setTimeout(cbOnce, 2000); }), () => objectExists, cb, @@ -644,8 +646,9 @@ class ReplicationUtility { Bucket: bucketName, Key: key, }, (err, data) => { + const cbOnce = jsutil.once(callback); if (err) { - return callback(err); + return cbOnce(err); } const statuses = []; // We cannot rely on the global status for one-to-many, so check @@ -657,9 +660,9 @@ class ReplicationUtility { }); shouldContinue = statuses.includes('PENDING'); if (shouldContinue) { - return setTimeout(callback, 2000); + return setTimeout(cbOnce, 2000); } - return callback(); + return cbOnce(); }), () => shouldContinue, cb, @@ -674,14 +677,15 @@ class ReplicationUtility { Bucket: bucketName, Key: key, }, (err, data) => { + const cbOnce = jsutil.once(callback); if (err) { - return callback(err); + return cbOnce(err); } shouldContinue = data.ReplicationStatus === 'FAILED'; if (shouldContinue) { - return setTimeout(callback, 2000); + return setTimeout(cbOnce, 2000); } - return callback(); + return cbOnce(); }), () => shouldContinue, cb,