From 47e6917bdd4b2f2caedaf1773bffe3a87e1e38c8 Mon Sep 17 00:00:00 2001 From: Francois Ferrand Date: Wed, 13 Nov 2024 19:53:58 +0100 Subject: [PATCH 1/2] Add test for archive object replacement Add replacement with MPU or copyObject. Issue: ZENKO-4928 --- tests/ctst/common/common.ts | 23 +++++++++- tests/ctst/features/dmf.feature | 41 +++++++++++++++++ tests/ctst/steps/utils/utils.ts | 79 +++++++++++++++++++++++++++++++++ 3 files changed, 141 insertions(+), 2 deletions(-) diff --git a/tests/ctst/common/common.ts b/tests/ctst/common/common.ts index 78926f1ae8..a5066165b5 100644 --- a/tests/ctst/common/common.ts +++ b/tests/ctst/common/common.ts @@ -7,6 +7,8 @@ import assert from 'assert'; import { Admin, Kafka } from 'kafkajs'; import { createBucketWithConfiguration, + putMpuObject, + copyObject, putObject, runActionAgainstBucket, getObjectNameWithBackendFlakiness, @@ -62,7 +64,7 @@ export async function cleanS3Bucket( } async function addMultipleObjects(this: Zenko, numberObjects: number, - objectName: string, sizeBytes: number, userMD?: string) { + objectName: string, sizeBytes: number, userMD?: string, parts?: number) { let lastResult = null; for (let i = 1; i <= numberObjects; i++) { this.resetCommand(); @@ -74,7 +76,9 @@ async function addMultipleObjects(this: Zenko, numberObjects: number, if (userMD) { this.addToSaved('userMetadata', userMD); } - lastResult = await putObject(this, objectNameFinal); + lastResult = parts === undefined + ? await putObject(this, objectNameFinal) + : await putMpuObject(this, parts, objectNameFinal); } return lastResult; } @@ -147,6 +151,16 @@ Given('{int} objects {string} of size {int} bytes', await addMultipleObjects.call(this, numberObjects, objectName, sizeBytes); }); +Given('{int} mpu objects {string} of size {int} bytes', + async function (this: Zenko, numberObjects: number, objectName: string, sizeBytes: number) { + await addMultipleObjects.call(this, numberObjects, objectName, sizeBytes, undefined, 1); + }); + +Given('{string} is copied to {string}', + async function (this: Zenko, sourceObject: string, destinationObject: string) { + await copyObject(this, sourceObject, destinationObject); + }); + Given('{int} objects {string} of size {int} bytes on {string} site', async function (this: Zenko, numberObjects: number, objectName: string, sizeBytes: number, site: string) { this.resetCommand(); @@ -164,6 +178,11 @@ Given('{int} objects {string} of size {int} bytes with user metadata {string}', await addMultipleObjects.call(this, numberObjects, objectName, sizeBytes, userMD); }); +Given('{int} mpu objects {string} of size {int} bytes with user metadata {string}', + async function (this: Zenko, numberObjects: number, objectName: string, sizeBytes: number, userMD: string) { + await addMultipleObjects.call(this, numberObjects, objectName, sizeBytes, userMD); + }); + Given('a tag on object {string} with key {string} and value {string}', async function (this: Zenko, objectName: string, tagKey: string, tagValue: string) { this.resetCommand(); diff --git a/tests/ctst/features/dmf.feature b/tests/ctst/features/dmf.feature index 175553e87b..7571e0cb96 100644 --- a/tests/ctst/features/dmf.feature +++ b/tests/ctst/features/dmf.feature @@ -92,3 +92,44 @@ Feature: DMF | versioningConfiguration | objectCount | objectSize | | Non versioned | 1 | 100 | | Suspended | 1 | 100 | + + @2.7.0 + @PreMerge + @Dmf + @ColdStorage + Scenario Outline: Overwriting of a cold object with mpu + Given a "" bucket + And a transition workflow to "e2e-cold" location + And objects "obj" of size bytes + Then object "obj-1" should be "transitioned" and have the storage class "e2e-cold" + And dmf volume should contain objects + Given mpu objects "obj" of size bytes + Then object "obj-1" should be "transitioned" and have the storage class "e2e-cold" + And dmf volume should contain 1 objects + + Examples: + | versioningConfiguration | objectCount | objectSize | + | Non versioned | 1 | 100 | + | Suspended | 1 | 100 | + + @2.7.0 + @PreMerge + @Dmf + @ColdStorage + Scenario Outline: Overwriting of a cold object with copyObject + Given a "" bucket + And a transition workflow to "e2e-cold" location + And 2 objects "obj" of size bytes + Then object "obj-1" should be "transitioned" and have the storage class "e2e-cold" + And object "obj-2" should be "transitioned" and have the storage class "e2e-cold" + And dmf volume should contain 2 objects + When i restore object "obj-1" for 5 days + Then object "obj-1" should be "restored" and have the storage class "e2e-cold" + Given "obj-1" is copied to "obj-2" + Then object "obj-2" should be "transitioned" and have the storage class "e2e-cold" + And dmf volume should contain 2 objects + + Examples: + | versioningConfiguration | objectSize | + | Non versioned | 100 | + | Suspended | 100 | diff --git a/tests/ctst/steps/utils/utils.ts b/tests/ctst/steps/utils/utils.ts index dd1745bc25..394b012c02 100644 --- a/tests/ctst/steps/utils/utils.ts +++ b/tests/ctst/steps/utils/utils.ts @@ -199,6 +199,83 @@ async function createBucketWithConfiguration( } } +async function putMpuObject(world: Zenko, parts: number = 2, objectName: string, content?: string) { + const key = objectName || `${Utils.randomString()}`; + const bucket = world.getSaved('bucketName'); + + world.resetCommand(); + world.addToSaved('objectName', objectName); + world.logger.debug('Adding mpu object', { objectName }); + world.addCommandParameter({ key }); + world.addCommandParameter({ bucket }); + const userMetadata = world.getSaved('userMetadata'); + if (userMetadata) { + world.addCommandParameter({ metadata: JSON.stringify(userMetadata) }); + } + + const initiateMPUResult = await S3.createMultipartUpload(world.getCommandParameters()); + assert.ifError(initiateMPUResult.stderr || initiateMPUResult.err); + const uploadId = extractPropertyFromResults(initiateMPUResult, 'UploadId'); + + await uploadSetup(world, 'UploadPart', content); + const body = world.getSaved('tempFileName'); + + const uploadedParts = []; + for (let i = 0; i < parts; i++) { + world.resetCommand(); + world.addCommandParameter({ key }); + world.addCommandParameter({ bucket }); + world.addCommandParameter({ partNumber: (i+1).toString() }); + world.addCommandParameter({ uploadId }); + if (body) { + world.addCommandParameter({ body }); + } + + const uploadPartResult = await S3.uploadPart(world.getCommandParameters()); + assert.ifError(uploadPartResult.stderr || uploadPartResult.err); + + uploadedParts.push({ + ETag: extractPropertyFromResults(uploadPartResult, 'ETag'), + PartNumber: (i+1).toString(), + }); + } + + await uploadTeardown(world, 'UploadPart'); + + world.resetCommand(); + world.addCommandParameter({ key }); + world.addCommandParameter({ bucket }); + world.addCommandParameter({ uploadId }); + world.addCommandParameter({ multipartUpload: JSON.stringify({ Parts: uploadedParts }) }); + + const result = await S3.completeMultipartUpload(world.getCommandParameters()); + const versionId = extractPropertyFromResults(result, 'VersionId'); + world.saveCreatedObject(objectName, versionId || ''); + world.setResult(result); + return result; +} + +async function copyObject(world: Zenko, srcObjectName?: string, dstObjectName?: string) { + const bucket = world.getSaved('bucketName'); + const key = dstObjectName || world.getSaved('objectName'); + const copySource = `${bucket}/${srcObjectName || world.getSaved('objectName')}`; + + world.resetCommand(); + world.addCommandParameter({ copySource }); + world.addCommandParameter({ bucket }); + world.addCommandParameter({ key }); + + const userMetadata = world.getSaved('userMetadata'); + if (userMetadata) { + world.addCommandParameter({ metadata: JSON.stringify(userMetadata) }); + } + + const result = await S3.copyObject(world.getCommandParameters()); + const versionId = extractPropertyFromResults(result, 'VersionId'); + world.saveCreatedObject(key, versionId || ''); + world.setResult(result); +} + async function putObject(world: Zenko, objectName?: string, content?: string) { world.resetCommand(); let finalObjectName = objectName; @@ -394,6 +471,8 @@ export { runActionAgainstBucket, createBucketWithConfiguration, getAuthorizationConfiguration, + putMpuObject, + copyObject, putObject, emptyNonVersionedBucket, emptyVersionedBucket, From dfa1a659ddaa0312f692a3f843c2a421678d4e0f Mon Sep 17 00:00:00 2001 From: Francois Ferrand Date: Mon, 18 Nov 2024 16:18:14 +0100 Subject: [PATCH 2/2] ctst: check result of object creation steps Issue: ZENKO-4928 --- tests/ctst/common/common.ts | 18 ++++++++++++------ tests/ctst/steps/utils/utils.ts | 1 + 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/tests/ctst/common/common.ts b/tests/ctst/common/common.ts index a5066165b5..eebaaea941 100644 --- a/tests/ctst/common/common.ts +++ b/tests/ctst/common/common.ts @@ -148,17 +148,20 @@ Given('an existing bucket {string} {string} versioning, {string} ObjectLock {str Given('{int} objects {string} of size {int} bytes', async function (this: Zenko, numberObjects: number, objectName: string, sizeBytes: number) { - await addMultipleObjects.call(this, numberObjects, objectName, sizeBytes); + const result = await addMultipleObjects.call(this, numberObjects, objectName, sizeBytes); + assert.ifError(result?.stderr || result?.err); }); Given('{int} mpu objects {string} of size {int} bytes', async function (this: Zenko, numberObjects: number, objectName: string, sizeBytes: number) { - await addMultipleObjects.call(this, numberObjects, objectName, sizeBytes, undefined, 1); + const result = await addMultipleObjects.call(this, numberObjects, objectName, sizeBytes, undefined, 1); + assert.ifError(result?.stderr || result?.err); }); Given('{string} is copied to {string}', async function (this: Zenko, sourceObject: string, destinationObject: string) { - await copyObject(this, sourceObject, destinationObject); + const result = await copyObject(this, sourceObject, destinationObject); + assert.ifError(result?.stderr || result?.err); }); Given('{int} objects {string} of size {int} bytes on {string} site', @@ -170,17 +173,20 @@ Given('{int} objects {string} of size {int} bytes on {string} site', } else { Identity.useIdentity(IdentityEnum.ACCOUNT, Zenko.sites['source'].accountName); } - await addMultipleObjects.call(this, numberObjects, objectName, sizeBytes); + const result = await addMultipleObjects.call(this, numberObjects, objectName, sizeBytes); + assert.ifError(result?.stderr || result?.err); }); Given('{int} objects {string} of size {int} bytes with user metadata {string}', async function (this: Zenko, numberObjects: number, objectName: string, sizeBytes: number, userMD: string) { - await addMultipleObjects.call(this, numberObjects, objectName, sizeBytes, userMD); + const result = await addMultipleObjects.call(this, numberObjects, objectName, sizeBytes, userMD); + assert.ifError(result?.stderr || result?.err); }); Given('{int} mpu objects {string} of size {int} bytes with user metadata {string}', async function (this: Zenko, numberObjects: number, objectName: string, sizeBytes: number, userMD: string) { - await addMultipleObjects.call(this, numberObjects, objectName, sizeBytes, userMD); + const result = await addMultipleObjects.call(this, numberObjects, objectName, sizeBytes, userMD); + assert.ifError(result?.stderr || result?.err); }); Given('a tag on object {string} with key {string} and value {string}', diff --git a/tests/ctst/steps/utils/utils.ts b/tests/ctst/steps/utils/utils.ts index 394b012c02..9469a974c1 100644 --- a/tests/ctst/steps/utils/utils.ts +++ b/tests/ctst/steps/utils/utils.ts @@ -274,6 +274,7 @@ async function copyObject(world: Zenko, srcObjectName?: string, dstObjectName?: const versionId = extractPropertyFromResults(result, 'VersionId'); world.saveCreatedObject(key, versionId || ''); world.setResult(result); + return result; } async function putObject(world: Zenko, objectName?: string, content?: string) {