Skip to content

Commit

Permalink
Merge branch 'w/2.8/improvement/ZENKO-4928' into w/2.9/improvement/ZE…
Browse files Browse the repository at this point in the history 
…NKO-4928
  • Loading branch information
@francoisferrand
francoisferrand committed Nov 18, 2024
2 parents 2a31a02 + ec77cea commit 4004d0c
Show file tree
Hide file tree
Showing 3 changed files with 192 additions and 6 deletions.
35 changes: 30 additions & 5 deletions tests/ctst/common/common.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ import assert from 'assert';
import { Admin, Kafka } from 'kafkajs';
import {
createBucketWithConfiguration,
putMpuObject,
copyObject,
putObject,
runActionAgainstBucket,
getObjectNameWithBackendFlakiness,
Expand Down Expand Up @@ -62,7 +64,7 @@ export async function cleanS3Bucket(
}

async function addMultipleObjects(this: Zenko, numberObjects: number,
objectName: string, sizeBytes: number, userMD?: string) {
objectName: string, sizeBytes: number, userMD?: string, parts?: number) {
let lastResult = null;
for (let i = 1; i <= numberObjects; i++) {
this.resetCommand();
Expand All @@ -74,7 +76,9 @@ async function addMultipleObjects(this: Zenko, numberObjects: number,
if (userMD) {
this.addToSaved('userMetadata', userMD);
}
lastResult = await putObject(this, objectNameFinal);
lastResult = parts === undefined
? await putObject(this, objectNameFinal)
: await putMpuObject(this, parts, objectNameFinal);
}
return lastResult;
}
Expand Down Expand Up @@ -144,7 +148,20 @@ Given('an existing bucket {string} {string} versioning, {string} ObjectLock {str

Given('{int} objects {string} of size {int} bytes',
async function (this: Zenko, numberObjects: number, objectName: string, sizeBytes: number) {
await addMultipleObjects.call(this, numberObjects, objectName, sizeBytes);
const result = await addMultipleObjects.call(this, numberObjects, objectName, sizeBytes);
assert.ifError(result?.stderr || result?.err);
});

Given('{int} mpu objects {string} of size {int} bytes',
async function (this: Zenko, numberObjects: number, objectName: string, sizeBytes: number) {
const result = await addMultipleObjects.call(this, numberObjects, objectName, sizeBytes, undefined, 1);
assert.ifError(result?.stderr || result?.err);
});

Given('{string} is copied to {string}',
async function (this: Zenko, sourceObject: string, destinationObject: string) {
const result = await copyObject(this, sourceObject, destinationObject);
assert.ifError(result?.stderr || result?.err);
});

Given('{int} objects {string} of size {int} bytes on {string} site',
Expand All @@ -156,12 +173,20 @@ Given('{int} objects {string} of size {int} bytes on {string} site',
} else {
Identity.useIdentity(IdentityEnum.ACCOUNT, Zenko.sites['source'].accountName);
}
await addMultipleObjects.call(this, numberObjects, objectName, sizeBytes);
const result = await addMultipleObjects.call(this, numberObjects, objectName, sizeBytes);
assert.ifError(result?.stderr || result?.err);
});

Given('{int} objects {string} of size {int} bytes with user metadata {string}',
async function (this: Zenko, numberObjects: number, objectName: string, sizeBytes: number, userMD: string) {
await addMultipleObjects.call(this, numberObjects, objectName, sizeBytes, userMD);
const result = await addMultipleObjects.call(this, numberObjects, objectName, sizeBytes, userMD);
assert.ifError(result?.stderr || result?.err);
});

Given('{int} mpu objects {string} of size {int} bytes with user metadata {string}',
async function (this: Zenko, numberObjects: number, objectName: string, sizeBytes: number, userMD: string) {
const result = await addMultipleObjects.call(this, numberObjects, objectName, sizeBytes, userMD);
assert.ifError(result?.stderr || result?.err);
});

Given('a tag on object {string} with key {string} and value {string}',
Expand Down
83 changes: 82 additions & 1 deletion tests/ctst/features/dmf.feature
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,87 @@ Feature: DMF
| Non versioned | 1 | 100 |
| Suspended | 1 | 100 |

@2.7.0
@PreMerge
@Dmf
@ColdStorage
Scenario Outline: Overwriting of a cold object with mpu
Given a "<versioningConfiguration>" bucket
And a transition workflow to "e2e-cold" location
And <objectCount> objects "obj" of size <objectSize> bytes
Then object "obj-1" should be "transitioned" and have the storage class "e2e-cold"
And dmf volume should contain <objectCount> objects
Given <objectCount> mpu objects "obj" of size <objectSize> bytes
Then object "obj-1" should be "transitioned" and have the storage class "e2e-cold"
And dmf volume should contain 1 objects

Examples:
| versioningConfiguration | objectCount | objectSize |
| Non versioned | 1 | 100 |
| Suspended | 1 | 100 |

@2.7.0
@PreMerge
@Dmf
@ColdStorage
Scenario Outline: Overwriting of a cold object with copyObject
Given a "<versioningConfiguration>" bucket
And a transition workflow to "e2e-cold" location
And 2 objects "obj" of size <objectSize> bytes
Then object "obj-1" should be "transitioned" and have the storage class "e2e-cold"
And object "obj-2" should be "transitioned" and have the storage class "e2e-cold"
And dmf volume should contain 2 objects
When i restore object "obj-1" for 5 days
Then object "obj-1" should be "restored" and have the storage class "e2e-cold"
Given "obj-1" is copied to "obj-2"
Then object "obj-2" should be "transitioned" and have the storage class "e2e-cold"
And dmf volume should contain 2 objects

Examples:
| versioningConfiguration | objectSize |
| Non versioned | 100 |
| Suspended | 100 |

@2.7.0
@PreMerge
@Dmf
@ColdStorage
Scenario Outline: Overwriting of a cold object with mpu
Given a "<versioningConfiguration>" bucket
And a transition workflow to "e2e-cold" location
And <objectCount> objects "obj" of size <objectSize> bytes
Then object "obj-1" should be "transitioned" and have the storage class "e2e-cold"
And dmf volume should contain <objectCount> objects
Given <objectCount> mpu objects "obj" of size <objectSize> bytes
Then object "obj-1" should be "transitioned" and have the storage class "e2e-cold"
And dmf volume should contain 1 objects

Examples:
| versioningConfiguration | objectCount | objectSize |
| Non versioned | 1 | 100 |
| Suspended | 1 | 100 |

@2.7.0
@PreMerge
@Dmf
@ColdStorage
Scenario Outline: Overwriting of a cold object with copyObject
Given a "<versioningConfiguration>" bucket
And a transition workflow to "e2e-cold" location
And 2 objects "obj" of size <objectSize> bytes
Then object "obj-1" should be "transitioned" and have the storage class "e2e-cold"
And object "obj-2" should be "transitioned" and have the storage class "e2e-cold"
And dmf volume should contain 2 objects
When i restore object "obj-1" for 5 days
Then object "obj-1" should be "restored" and have the storage class "e2e-cold"
Given "obj-1" is copied to "obj-2"
Then object "obj-2" should be "transitioned" and have the storage class "e2e-cold"
And dmf volume should contain 2 objects

Examples:
| versioningConfiguration | objectSize |
| Non versioned | 100 |
| Suspended | 100 |

@2.7.0
@PreMerge
Expand Down Expand Up @@ -125,4 +206,4 @@ Feature: DMF
| versioningConfiguration | objectCount | objectSize | restoreDays |
| Non versioned | 2 | 100 | 15 |
| Versioned | 2 | 100 | 15 |
| Suspended | 2 | 100 | 15 |
| Suspended | 2 | 100 | 15 |
80 changes: 80 additions & 0 deletions tests/ctst/steps/utils/utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,84 @@ async function createBucketWithConfiguration(
}
}

async function putMpuObject(world: Zenko, parts: number = 2, objectName: string, content?: string) {
const key = objectName || `${Utils.randomString()}`;
const bucket = world.getSaved<string>('bucketName');

world.resetCommand();
world.addToSaved('objectName', objectName);
world.logger.debug('Adding mpu object', { objectName });
world.addCommandParameter({ key });
world.addCommandParameter({ bucket });
const userMetadata = world.getSaved<string>('userMetadata');
if (userMetadata) {
world.addCommandParameter({ metadata: JSON.stringify(userMetadata) });
}

const initiateMPUResult = await S3.createMultipartUpload(world.getCommandParameters());
assert.ifError(initiateMPUResult.stderr || initiateMPUResult.err);
const uploadId = extractPropertyFromResults<string>(initiateMPUResult, 'UploadId');

await uploadSetup(world, 'UploadPart', content);
const body = world.getSaved<string>('tempFileName');

const uploadedParts = [];
for (let i = 0; i < parts; i++) {
world.resetCommand();
world.addCommandParameter({ key });
world.addCommandParameter({ bucket });
world.addCommandParameter({ partNumber: (i+1).toString() });
world.addCommandParameter({ uploadId });
if (body) {
world.addCommandParameter({ body });
}

const uploadPartResult = await S3.uploadPart(world.getCommandParameters());
assert.ifError(uploadPartResult.stderr || uploadPartResult.err);

uploadedParts.push({
ETag: extractPropertyFromResults<string>(uploadPartResult, 'ETag'),
PartNumber: (i+1).toString(),
});
}

await uploadTeardown(world, 'UploadPart');

world.resetCommand();
world.addCommandParameter({ key });
world.addCommandParameter({ bucket });
world.addCommandParameter({ uploadId });
world.addCommandParameter({ multipartUpload: JSON.stringify({ Parts: uploadedParts }) });

const result = await S3.completeMultipartUpload(world.getCommandParameters());
const versionId = extractPropertyFromResults<string>(result, 'VersionId');
world.saveCreatedObject(objectName, versionId || '');
world.setResult(result);
return result;
}

async function copyObject(world: Zenko, srcObjectName?: string, dstObjectName?: string) {
const bucket = world.getSaved<string>('bucketName');
const key = dstObjectName || world.getSaved<string>('objectName');
const copySource = `${bucket}/${srcObjectName || world.getSaved<string>('objectName')}`;

world.resetCommand();
world.addCommandParameter({ copySource });
world.addCommandParameter({ bucket });
world.addCommandParameter({ key });

const userMetadata = world.getSaved<string>('userMetadata');
if (userMetadata) {
world.addCommandParameter({ metadata: JSON.stringify(userMetadata) });
}

const result = await S3.copyObject(world.getCommandParameters());
const versionId = extractPropertyFromResults<string>(result, 'VersionId');
world.saveCreatedObject(key, versionId || '');
world.setResult(result);
return result;
}

async function putObject(world: Zenko, objectName?: string, content?: string) {
world.resetCommand();
let finalObjectName = objectName;
Expand Down Expand Up @@ -394,6 +472,8 @@ export {
runActionAgainstBucket,
createBucketWithConfiguration,
getAuthorizationConfiguration,
putMpuObject,
copyObject,
putObject,
emptyNonVersionedBucket,
emptyVersionedBucket,
Expand Down

0 comments on commit 4004d0c

Please sign in to comment.