From 29ae88077c8ec45a7e6aa324e8ddde8fe1919398 Mon Sep 17 00:00:00 2001 From: Nicolas Humbert Date: Wed, 13 Sep 2023 17:12:36 +0200 Subject: [PATCH] ARSN-366 Limit lifecycle listing on scanned entries --- lib/algos/list/delimiterCurrent.ts | 16 ++++--------- lib/algos/list/delimiterNonCurrent.js | 20 +++++----------- lib/algos/list/delimiterOrphanDeleteMarker.js | 18 +++++--------- .../unit/algos/list/delimiterCurrent.spec.js | 20 +++++++--------- .../algos/list/delimiterNonCurrent.spec.js | 23 +++++++----------- .../list/delimiterOrphanDeleteMarker.spec.js | 24 +++++++------------ 6 files changed, 44 insertions(+), 77 deletions(-) diff --git a/lib/algos/list/delimiterCurrent.ts b/lib/algos/list/delimiterCurrent.ts index 0cb247c27..2970132d4 100644 --- a/lib/algos/list/delimiterCurrent.ts +++ b/lib/algos/list/delimiterCurrent.ts @@ -10,8 +10,6 @@ type ResultObject = { NextMarker ?: string; }; -const DELIMITER_TIMEOUT_MS = 10 * 1000; // 10s - /** * Handle object listing with parameters. This extends the base class DelimiterMaster * to return the master/current versions. @@ -22,6 +20,7 @@ class DelimiterCurrent extends DelimiterMaster { * @param {Object} parameters - listing parameters * @param {String} parameters.beforeDate - limit the response to keys older than beforeDate * @param {String} parameters.excludedDataStoreName - excluded datatore name + * @param {Number} parameters.maxScannedLifecycleListingEntries - max number of entries to be scanned * @param {RequestLogger} logger - The logger of the request * @param {String} [vFormat] - versioning key format */ @@ -30,8 +29,7 @@ class DelimiterCurrent extends DelimiterMaster { this.beforeDate = parameters.beforeDate; this.excludedDataStoreName = parameters.excludedDataStoreName; - // used for monitoring - this.start = null; + this.maxScannedLifecycleListingEntries = parameters.maxScannedLifecycleListingEntries; this.scannedKeys = 0; } @@ -51,10 +49,6 @@ class DelimiterCurrent extends DelimiterMaster { } } - // The genMDParamsV1() function calls genMDParamsV0() in the Delimiter class, - // making sure that this.start is set for both v0 and v1 bucket formats - this.start = Date.now(); - return params; } @@ -80,11 +74,11 @@ class DelimiterCurrent extends DelimiterMaster { return FILTER_END; } - if (this.start && Date.now() - this.start > DELIMITER_TIMEOUT_MS) { + if (this.scannedKeys >= this.maxScannedLifecycleListingEntries) { this.IsTruncated = true; - this.logger.info('listing stopped after expected internal timeout', + this.logger.info('listing stopped due to reaching the maximum scanned entries limit', { - timeoutMs: DELIMITER_TIMEOUT_MS, + maxScannedLifecycleListingEntries: this.maxScannedLifecycleListingEntries, scannedKeys: this.scannedKeys, }); return FILTER_END; diff --git a/lib/algos/list/delimiterNonCurrent.js b/lib/algos/list/delimiterNonCurrent.js index 31997223a..2358d292e 100644 --- a/lib/algos/list/delimiterNonCurrent.js +++ b/lib/algos/list/delimiterNonCurrent.js @@ -1,8 +1,6 @@ const { DelimiterVersions } = require('./delimiterVersions'); const { FILTER_END, FILTER_SKIP } = require('./tools'); -// TODO: find an acceptable timeout value. -const DELIMITER_TIMEOUT_MS = 10 * 1000; // 10s const TRIM_METADATA_MIN_BLOB_SIZE = 10000; /** @@ -17,7 +15,8 @@ class DelimiterNonCurrent extends DelimiterVersions { * @param {String} parameters.versionIdMarker - version id marker * @param {String} parameters.beforeDate - limit the response to keys with stale date older than beforeDate. * “stale date” is the date on when a version becomes non-current. - * @param {String} parameters.excludedDataStoreName - exclude dataStoreName matches from the versions + * @param {Number} parameters.maxScannedLifecycleListingEntries - max number of entries to be scanned + * @param {String} parameters.excludedDataStoreName - exclude dataStoreName matches from the versions * @param {RequestLogger} logger - The logger of the request * @param {String} [vFormat] - versioning key format */ @@ -26,22 +25,15 @@ class DelimiterNonCurrent extends DelimiterVersions { this.beforeDate = parameters.beforeDate; this.excludedDataStoreName = parameters.excludedDataStoreName; + this.maxScannedLifecycleListingEntries = parameters.maxScannedLifecycleListingEntries; // internal state this.prevKey = null; this.staleDate = null; - // used for monitoring this.scannedKeys = 0; } - genMDParamsV0() { - // The genMDParamsV1() function calls genMDParamsV0()in the DelimiterVersions class, - // making sure that this.start is set for both v0 and v1 bucket formats - this.start = Date.now(); - return super.genMDParamsV0(); - } - getLastModified(value) { let lastModified; try { @@ -78,11 +70,11 @@ class DelimiterNonCurrent extends DelimiterVersions { } filter(obj) { - if (this.start && Date.now() - this.start > DELIMITER_TIMEOUT_MS) { + if (this.scannedKeys >= this.maxScannedLifecycleListingEntries) { this.IsTruncated = true; - this.logger.info('listing stopped after expected internal timeout', + this.logger.info('listing stopped due to reaching the maximum scanned entries limit', { - timeoutMs: DELIMITER_TIMEOUT_MS, + maxScannedLifecycleListingEntries: this.maxScannedLifecycleListingEntries, scannedKeys: this.scannedKeys, }); return FILTER_END; diff --git a/lib/algos/list/delimiterOrphanDeleteMarker.js b/lib/algos/list/delimiterOrphanDeleteMarker.js index 62f0e8c6f..410a93729 100644 --- a/lib/algos/list/delimiterOrphanDeleteMarker.js +++ b/lib/algos/list/delimiterOrphanDeleteMarker.js @@ -1,6 +1,5 @@ const DelimiterVersions = require('./delimiterVersions').DelimiterVersions; const { FILTER_END } = require('./tools'); -const DELIMITER_TIMEOUT_MS = 10 * 1000; // 10s const TRIM_METADATA_MIN_BLOB_SIZE = 10000; /** * Handle object listing with parameters. This extends the base class DelimiterVersions @@ -13,6 +12,7 @@ class DelimiterOrphanDeleteMarker extends DelimiterVersions { * Delimiter listing of orphan delete markers. * @param {Object} parameters - listing parameters * @param {String} parameters.beforeDate - limit the response to keys older than beforeDate + * @param {Number} parameters.maxScannedLifecycleListingEntries - max number of entries to be scanned * @param {RequestLogger} logger - The logger of the request * @param {String} [vFormat] - versioning key format */ @@ -22,6 +22,7 @@ class DelimiterOrphanDeleteMarker extends DelimiterVersions { maxKeys, prefix, beforeDate, + maxScannedLifecycleListingEntries, } = parameters; const versionParams = { @@ -33,20 +34,13 @@ class DelimiterOrphanDeleteMarker extends DelimiterVersions { }; super(versionParams, logger, vFormat); + this.maxScannedLifecycleListingEntries = maxScannedLifecycleListingEntries; this.beforeDate = beforeDate; this.keyName = null; this.value = null; - // used for monitoring this.scannedKeys = 0; } - genMDParamsV0() { - // The genMDParamsV1() function calls genMDParamsV0() in the DelimiterVersions class, - // making sure that this.start is set for both v0 and v1 bucket formats - this.start = Date.now(); - return super.genMDParamsV0(); - } - _reachedMaxKeys() { if (this.keys >= this.maxKeys) { return true; @@ -108,12 +102,12 @@ class DelimiterOrphanDeleteMarker extends DelimiterVersions { } filter(obj) { - if (this.start && Date.now() - this.start > DELIMITER_TIMEOUT_MS) { + if (this.scannedKeys >= this.maxScannedLifecycleListingEntries) { this.nextKeyMarker = this.keyName; this.IsTruncated = true; - this.logger.info('listing stopped after expected internal timeout', + this.logger.info('listing stopped due to reaching the maximum scanned entries limit', { - timeoutMs: DELIMITER_TIMEOUT_MS, + maxScannedLifecycleListingEntries: this.maxScannedLifecycleListingEntries, scannedKeys: this.scannedKeys, }); return FILTER_END; diff --git a/tests/unit/algos/list/delimiterCurrent.spec.js b/tests/unit/algos/list/delimiterCurrent.spec.js index e789e30da..84153bc62 100644 --- a/tests/unit/algos/list/delimiterCurrent.spec.js +++ b/tests/unit/algos/list/delimiterCurrent.spec.js @@ -13,8 +13,6 @@ const VSConst = require('../../../../lib/versioning/constants').VersioningConstants; const { DbPrefixes } = VSConst; -const DELIMITER_TIMEOUT_MS = 10 * 1000; // 10s - const EmptyResult = { Contents: [], IsTruncated: false, @@ -46,11 +44,13 @@ function getListingKey(key, vFormat) { const marker = 'premark'; const beforeDate = '1970-01-01T00:00:00.005Z'; const excludedDataStoreName = 'location1'; + const maxScannedLifecycleListingEntries = 2; const delimiter = new DelimiterCurrent({ prefix, marker, beforeDate, excludedDataStoreName, + maxScannedLifecycleListingEntries, }, fakeLogger, v); const expectedParams = { @@ -64,7 +64,7 @@ function getListingKey(key, vFormat) { lt: getListingKey('prf', v), }; assert.deepStrictEqual(delimiter.genMDParams(), expectedParams); - assert(delimiter.start); + assert.strictEqual(delimiter.maxScannedLifecycleListingEntries, 2); }); it('should accept entry starting with prefix', () => { @@ -353,9 +353,10 @@ function getListingKey(key, vFormat) { assert.deepStrictEqual(delimiter.result(), expectedResult); }); - it('should return the objects pushed before timeout', () => { + it('should return the objects pushed before max scanned entries value is reached', () => { const beforeDate = '1970-01-01T00:00:00.003Z'; - const delimiter = new DelimiterCurrent({ beforeDate }, fakeLogger, v); + const maxScannedLifecycleListingEntries = 2; + const delimiter = new DelimiterCurrent({ beforeDate, maxScannedLifecycleListingEntries }, fakeLogger, v); const masterKey1 = 'key1'; const date1 = '1970-01-01T00:00:00.000Z'; @@ -375,8 +376,6 @@ function getListingKey(key, vFormat) { value: value2, }), FILTER_ACCEPT); - delimiter.start = Date.now() - (DELIMITER_TIMEOUT_MS + 1); - const masterKey3 = 'key3'; const date3 = '1970-01-01T00:00:00.002Z'; const value3 = `{"last-modified": "${date3}"}`; @@ -404,9 +403,10 @@ function getListingKey(key, vFormat) { assert.deepStrictEqual(delimiter.result(), expectedResult); }); - it('should return empty content after timeout', () => { + it('should return empty content after max scanned entries value is reached', () => { const beforeDate = '1970-01-01T00:00:00.003Z'; - const delimiter = new DelimiterCurrent({ beforeDate }, fakeLogger, v); + const maxScannedLifecycleListingEntries = 2; + const delimiter = new DelimiterCurrent({ beforeDate, maxScannedLifecycleListingEntries }, fakeLogger, v); const masterKey1 = 'key1'; const date1 = '1970-01-01T00:00:00.004Z'; @@ -426,8 +426,6 @@ function getListingKey(key, vFormat) { value: value2, }), FILTER_ACCEPT); - delimiter.start = Date.now() - (DELIMITER_TIMEOUT_MS + 1); - const masterKey3 = 'key3'; const date3 = '1970-01-01T00:00:00.006Z'; const value3 = `{"last-modified": "${date3}"}`; diff --git a/tests/unit/algos/list/delimiterNonCurrent.spec.js b/tests/unit/algos/list/delimiterNonCurrent.spec.js index 335c3e071..029ed6e80 100644 --- a/tests/unit/algos/list/delimiterNonCurrent.spec.js +++ b/tests/unit/algos/list/delimiterNonCurrent.spec.js @@ -12,9 +12,6 @@ const VSConst = require('../../../../lib/versioning/constants').VersioningConstants; const { DbPrefixes } = VSConst; -// TODO: find an acceptable timeout value. -const DELIMITER_TIMEOUT_MS = 10 * 1000; // 10s - const VID_SEP = VSConst.VersionId.Separator; const EmptyResult = { Contents: [], @@ -48,10 +45,12 @@ function getListingKey(key, vFormat) { const prefix = 'pre'; const keyMarker = 'premark'; const versionIdMarker = 'vid1'; + const maxScannedLifecycleListingEntries = 2; const delimiter = new DelimiterNonCurrent({ prefix, keyMarker, versionIdMarker, + maxScannedLifecycleListingEntries, }, fakeLogger, v); let expectedParams; @@ -70,7 +69,7 @@ function getListingKey(key, vFormat) { ]; } assert.deepStrictEqual(delimiter.genMDParams(), expectedParams); - assert(delimiter.start); + assert.strictEqual(delimiter.maxScannedLifecycleListingEntries, 2); }); it('should accept entry starting with prefix', () => { const delimiter = new DelimiterNonCurrent({ prefix: 'prefix' }, fakeLogger, v); @@ -286,8 +285,9 @@ function getListingKey(key, vFormat) { assert.deepStrictEqual(delimiter.result(), expectedResult); }); - it('should end filtering if delimiter timeout', () => { - const delimiter = new DelimiterNonCurrent({ }, fakeLogger, v); + it('should return the non-current versions pushed before max scanned entries value is reached', () => { + const maxScannedLifecycleListingEntries = 2; + const delimiter = new DelimiterNonCurrent({ maxScannedLifecycleListingEntries }, fakeLogger, v); const masterKey = 'key'; @@ -313,9 +313,6 @@ function getListingKey(key, vFormat) { value: value2, }), FILTER_ACCEPT); - // force delimiter to timeout. - delimiter.start = Date.now() - (DELIMITER_TIMEOUT_MS + 1); - // filter third version const versionId3 = 'version3'; const versionKey3 = `${masterKey}${VID_SEP}${versionId3}`; @@ -343,8 +340,9 @@ function getListingKey(key, vFormat) { assert.deepStrictEqual(delimiter.result(), expectedResult); }); - it('should end filtering if delimiter timeout with empty content', () => { - const delimiter = new DelimiterNonCurrent({ }, fakeLogger, v); + it('should return empty content after max scanned entries value is reached', () => { + const maxScannedLifecycleListingEntries = 2; + const delimiter = new DelimiterNonCurrent({ maxScannedLifecycleListingEntries }, fakeLogger, v); // filter current version const masterKey1 = 'key1'; @@ -370,9 +368,6 @@ function getListingKey(key, vFormat) { value: value2, }), FILTER_ACCEPT); - // force delimiter to timeout. - delimiter.start = Date.now() - (DELIMITER_TIMEOUT_MS + 1); - // filter current version const masterKey3 = 'key3'; const versionId3 = 'version3'; diff --git a/tests/unit/algos/list/delimiterOrphanDeleteMarker.spec.js b/tests/unit/algos/list/delimiterOrphanDeleteMarker.spec.js index de824fa0f..e654861c3 100644 --- a/tests/unit/algos/list/delimiterOrphanDeleteMarker.spec.js +++ b/tests/unit/algos/list/delimiterOrphanDeleteMarker.spec.js @@ -13,10 +13,6 @@ const VSConst = require('../../../../lib/versioning/constants').VersioningConstants; const { DbPrefixes } = VSConst; -// TODO: find an acceptable timeout value. -const DELIMITER_TIMEOUT_MS = 10 * 1000; // 10s - - const VID_SEP = VSConst.VersionId.Separator; const EmptyResult = { Contents: [], @@ -49,9 +45,11 @@ function getListingKey(key, vFormat) { it('should return expected metadata parameters', () => { const prefix = 'pre'; const marker = 'premark'; + const maxScannedLifecycleListingEntries = 2; const delimiter = new DelimiterOrphanDeleteMarker({ prefix, marker, + maxScannedLifecycleListingEntries, }, fakeLogger, v); let expectedParams; @@ -70,7 +68,7 @@ function getListingKey(key, vFormat) { ]; } assert.deepStrictEqual(delimiter.genMDParams(), expectedParams); - assert(delimiter.start); + assert.strictEqual(delimiter.maxScannedLifecycleListingEntries, 2); }); it('should accept entry starting with prefix', () => { const delimiter = new DelimiterOrphanDeleteMarker({ prefix: 'prefix' }, fakeLogger, v); @@ -305,8 +303,9 @@ function getListingKey(key, vFormat) { assert.deepStrictEqual(delimiter.result(), expectedResult); }); - it('should end filtering if delimiter timeout', () => { - const delimiter = new DelimiterOrphanDeleteMarker({ }, fakeLogger, v); + it('should end filtering if max scanned entries value is reached', () => { + const maxScannedLifecycleListingEntries = 1; + const delimiter = new DelimiterOrphanDeleteMarker({ maxScannedLifecycleListingEntries }, fakeLogger, v); // filter the first orphan delete marker const masterKey1 = 'key1'; @@ -320,9 +319,6 @@ function getListingKey(key, vFormat) { value: value1, }), FILTER_ACCEPT); - // force delimiter to timeout. - delimiter.start = Date.now() - (DELIMITER_TIMEOUT_MS + 1); - // filter the second orphan delete marker const masterKey2 = 'key2'; const versionId2 = 'version2'; @@ -349,8 +345,9 @@ function getListingKey(key, vFormat) { assert.deepStrictEqual(delimiter.result(), expectedResult); }); - it('should end filtering if delimiter timeout with empty content', () => { - const delimiter = new DelimiterOrphanDeleteMarker({ }, fakeLogger, v); + it('should end filtering with empty content if max scanned entries value is reached', () => { + const maxScannedLifecycleListingEntries = 2; + const delimiter = new DelimiterOrphanDeleteMarker({ maxScannedLifecycleListingEntries }, fakeLogger, v); const masterKey1 = 'key1'; const versionId1 = 'version1'; @@ -374,9 +371,6 @@ function getListingKey(key, vFormat) { value: value2, }), FILTER_ACCEPT); - // force delimiter to timeout. - delimiter.start = Date.now() - (DELIMITER_TIMEOUT_MS + 1); - // filter the third orphan delete marker const masterKey3 = 'key3'; const versionId3 = 'version3';