diff --git a/packages/spacecat-shared-data-access/docs/schema.json b/packages/spacecat-shared-data-access/docs/schema.json old mode 100644 new mode 100755 index 4a6a1e7e..5d15e622 --- a/packages/spacecat-shared-data-access/docs/schema.json +++ b/packages/spacecat-shared-data-access/docs/schema.json @@ -40,7 +40,7 @@ ], "GlobalSecondaryIndexes": [ { - "IndexName": "spacecat-data-ApiKey-byHashedApiKey", + "IndexName": "spacecat-data-gsi1pk-gsi1sk", "KeyAttributes": { "PartitionKey": { "AttributeName": "gsi1pk", "AttributeType": "S" }, "SortKey": { "AttributeName": "gsi1sk", "AttributeType": "S" } @@ -48,7 +48,7 @@ "Projection": { "ProjectionType": "ALL" } }, { - "IndexName": "spacecat-data-ApiKey-byImsOrgIdAndImsUserId", + "IndexName": "spacecat-data-gsi2pk-gsi2sk", "KeyAttributes": { "PartitionKey": { "AttributeName": "gsi2pk", "AttributeType": "S" }, "SortKey": { "AttributeName": "gsi2sk", "AttributeType": "S" } @@ -56,47 +56,7 @@ "Projection": { "ProjectionType": "ALL" } }, { - "IndexName": "spacecat-data-Opportunity-byAuditId", - "KeyAttributes": { - "PartitionKey": { "AttributeName": "gsi1pk", "AttributeType": "S" }, - "SortKey": { "AttributeName": "gsi1sk", "AttributeType": "S" } - }, - "Projection": { "ProjectionType": "ALL" } - }, - { - "IndexName": "spacecat-data-Opportunity-bySiteId", - "KeyAttributes": { - "PartitionKey": { "AttributeName": "gsi2pk", "AttributeType": "S" }, - "SortKey": { "AttributeName": "gsi2sk", "AttributeType": "S" } - }, - "Projection": { "ProjectionType": "ALL" } - }, - { - "IndexName": "spacecat-data-Suggestion-byOpportunityId", - "KeyAttributes": { - "PartitionKey": { "AttributeName": "gsi1pk", "AttributeType": "S" }, - "SortKey": { "AttributeName": "gsi1sk", "AttributeType": "S" } - }, - "Projection": { "ProjectionType": "ALL" } - }, - { - "IndexName": "spacecat-data-Site-all", - "KeyAttributes": { - "PartitionKey": { "AttributeName": "gsi1pk", "AttributeType": "S" }, - "SortKey": { "AttributeName": "gsi1sk", "AttributeType": "S" } - }, - "Projection": { "ProjectionType": "ALL" } - }, - { - "IndexName": "spacecat-data-Site-byOrganizationId", - "KeyAttributes": { - "PartitionKey": { "AttributeName": "gsi2pk", "AttributeType": "S" }, - "SortKey": { "AttributeName": "gsi2sk", "AttributeType": "S" } - }, - "Projection": { "ProjectionType": "ALL" } - }, - { - "IndexName": "spacecat-data-Site-byDeliveryType", + "IndexName": "spacecat-data-gsi3pk-gsi3sk", "KeyAttributes": { "PartitionKey": { "AttributeName": "gsi3pk", "AttributeType": "S" }, "SortKey": { "AttributeName": "gsi3sk", "AttributeType": "S" } @@ -104,90 +64,18 @@ "Projection": { "ProjectionType": "ALL" } }, { - "IndexName": "spacecat-data-Organization-all", + "IndexName": "spacecat-data-gsi4pk-gsi4sk", "KeyAttributes": { - "PartitionKey": { "AttributeName": "gsi1pk", "AttributeType": "S" }, - "SortKey": { "AttributeName": "gsi1sk", "AttributeType": "S" } - }, - "Projection": { "ProjectionType": "ALL" } - }, - { - "IndexName": "spacecat-data-Audit-bySiteId", - "KeyAttributes": { - "PartitionKey": { "AttributeName": "gsi1pk", "AttributeType": "S" }, - "SortKey": { "AttributeName": "gsi1sk", "AttributeType": "S" } - }, - "Projection": { "ProjectionType": "ALL" } - }, - { - "IndexName": "spacecat-data-Experiment-bySiteId", - "KeyAttributes": { - "PartitionKey": { "AttributeName": "gsi1pk", "AttributeType": "S" }, - "SortKey": { "AttributeName": "gsi1sk", "AttributeType": "S" } + "PartitionKey": { "AttributeName": "gsi4pk", "AttributeType": "S" }, + "SortKey": { "AttributeName": "gsi4sk", "AttributeType": "S" } }, "Projection": { "ProjectionType": "ALL" } }, { - "IndexName": "spacecat-data-KeyEvent-bySiteId", + "IndexName": "spacecat-data-gsi5pk-gsi5sk", "KeyAttributes": { - "PartitionKey": { "AttributeName": "gsi1pk", "AttributeType": "S" }, - "SortKey": { "AttributeName": "gsi1sk", "AttributeType": "S" } - }, - "Projection": { "ProjectionType": "ALL" } - }, - { - "IndexName": "spacecat-data-SiteCandidate-all", - "KeyAttributes": { - "PartitionKey": { "AttributeName": "gsi1pk", "AttributeType": "S" }, - "SortKey": { "AttributeName": "gsi1sk", "AttributeType": "S" } - }, - "Projection": { "ProjectionType": "ALL" } - }, - { - "IndexName": "spacecat-data-SiteCandidate-bySiteId", - "KeyAttributes": { - "PartitionKey": { "AttributeName": "gsi2pk", "AttributeType": "S" }, - "SortKey": { "AttributeName": "gsi2sk", "AttributeType": "S" } - }, - "Projection": { "ProjectionType": "ALL" } - }, - { - "IndexName": "spacecat-data-SiteTopPage-bySiteId", - "KeyAttributes": { - "PartitionKey": { "AttributeName": "gsi1pk", "AttributeType": "S" }, - "SortKey": { "AttributeName": "gsi1sk", "AttributeType": "S" } - }, - "Projection": { "ProjectionType": "ALL" } - }, - { - "IndexName": "spacecat-data-Configuration-all", - "KeyAttributes": { - "PartitionKey": { "AttributeName": "gsi1pk", "AttributeType": "S" }, - "SortKey": { "AttributeName": "version", "AttributeType": "N" } - }, - "Projection": { "ProjectionType": "ALL" } - }, - { - "IndexName": "spacecat-data-ImportJob-all", - "KeyAttributes": { - "PartitionKey": { "AttributeName": "gsi1pk", "AttributeType": "S" }, - "SortKey": { "AttributeName": "gsi1sk", "AttributeType": "S" } - }, - "Projection": { "ProjectionType": "ALL" } - }, - { - "IndexName": "spacecat-data-ImportJob-byStatus", - "KeyAttributes": { - "PartitionKey": { "AttributeName": "gsi2pk", "AttributeType": "S" }, - "SortKey": { "AttributeName": "gsi2sk", "AttributeType": "S" } - }, - "Projection": { "ProjectionType": "ALL" } - }, - { - "IndexName": "spacecat-data-ImportUrl-byImportJobId", - "KeyAttributes": { - "PartitionKey": { "AttributeName": "gsi1pk", "AttributeType": "S" }, - "SortKey": { "AttributeName": "gsi1sk", "AttributeType": "S" } + "PartitionKey": { "AttributeName": "gsi5pk", "AttributeType": "S" }, + "SortKey": { "AttributeName": "gsi5sk", "AttributeType": "S" } }, "Projection": { "ProjectionType": "ALL" } } diff --git a/packages/spacecat-shared-data-access/src/v2/models/api-key/api-key.schema.js b/packages/spacecat-shared-data-access/src/v2/models/api-key/api-key.schema.js index 358557b1..668d140e 100644 --- a/packages/spacecat-shared-data-access/src/v2/models/api-key/api-key.schema.js +++ b/packages/spacecat-shared-data-access/src/v2/models/api-key/api-key.schema.js @@ -69,12 +69,10 @@ const schema = new SchemaBuilder(ApiKey, ApiKeyCollection) }, }) .addIndex( - 'byHashedApiKey', { composite: ['hashedApiKey'] }, { composite: ['updatedAt'] }, ) .addIndex( - 'byImsOrgIdAndImsUserId', { composite: ['imsOrgId', 'imsUserId'] }, { composite: ['updatedAt'] }, ); diff --git a/packages/spacecat-shared-data-access/src/v2/models/base/base.collection.js b/packages/spacecat-shared-data-access/src/v2/models/base/base.collection.js index 663b0331..929e2464 100755 --- a/packages/spacecat-shared-data-access/src/v2/models/base/base.collection.js +++ b/packages/spacecat-shared-data-access/src/v2/models/base/base.collection.js @@ -24,7 +24,6 @@ import { guardId } from '../../util/guards.js'; import { entityNameToAllPKValue, isNonEmptyArray, - keyNamesToIndexName, removeElectroProperties, } from '../../util/util.js'; import { INDEX_TYPES } from './constants.js'; @@ -40,26 +39,24 @@ function isValidParent(parent, child) { } /** - * Attempts to find an index name matching a generated name from the given keyNames. - * If no exact match is found, it progressively shortens the keyNames by removing the last one - * and tries again. If still no match, it tries the "all" index, and then "primary". - * - * @param {object} indexes - The available indexes, keyed by their names. - * @param {object} keys - The keys to find an index name for. - * @returns {object} The found index. + * Finds the index name by the keys provided. The index is searched + * keys to match the combination of partition and sort keys. If no + * index is found, we fall back to the "all" index, then the "primary". + * @param {Schema} schema - The schema to search for the index. + * @param {Object} keys - The keys to search for. + * @return {*|string} - The index name. */ -function findIndexNameByKeys(indexes, keys) { +function findIndexNameByKeys(schema, keys) { const keyNames = Object.keys(keys); - for (let { length } = keyNames; length > 0; length -= 1) { - const subKeyNames = keyNames.slice(0, length); - const candidateName = keyNamesToIndexName(subKeyNames); - if (indexes[candidateName]) { - return candidateName; - } + + const index = schema.findIndexBySortKeys(keyNames); + if (index) { + return index.index; } - if (indexes.all) { - return INDEX_TYPES.ALL; + const allIndex = schema.findIndexByType(INDEX_TYPES.ALL); + if (allIndex) { + return allIndex.index; } return INDEX_TYPES.PRIMARY; @@ -176,11 +173,11 @@ class BaseCollection { throw new Error(message); } - const indexName = options.index || findIndexNameByKeys(this.entity.query, keys); + const indexName = options.index || findIndexNameByKeys(this.schema, keys); const index = this.entity.query[indexName]; if (!index) { - const message = `Failed to query [${this.entityName}]: index [${indexName}] not found`; + const message = `Failed to query [${this.entityName}]: query proxy [${indexName}] not found`; this.log.error(message); throw new Error(message); } @@ -243,7 +240,8 @@ class BaseCollection { * Finds a single entity from the "all" index. Requires an index named "all" with a partition key * named "pk" with a static value of "ALL_". * @param {Object} [sortKeys] - The sort keys to use for the query. - * @param {{index?: string, attributes?: string[]}} [options] - Additional options for the query. + * @param {{index?: string, attributes?: string[], order?: string}} [options] - + * Additional options for the query. * @return {Promise|null>} */ async findByAll(sortKeys = {}, options = {}) { @@ -254,7 +252,7 @@ class BaseCollection { } const keys = { pk: entityNameToAllPKValue(this.entityName), ...sortKeys }; - return this.#queryByIndexKeys(keys, { ...options, index: INDEX_TYPES.ALL, limit: 1 }); + return this.#queryByIndexKeys(keys, { ...options, limit: 1 }); } /** diff --git a/packages/spacecat-shared-data-access/src/v2/models/base/index.d.ts b/packages/spacecat-shared-data-access/src/v2/models/base/index.d.ts index b759ea09..09bb84ec 100644 --- a/packages/spacecat-shared-data-access/src/v2/models/base/index.d.ts +++ b/packages/spacecat-shared-data-access/src/v2/models/base/index.d.ts @@ -58,12 +58,20 @@ export interface Reference { isRemoveDependents(): boolean; } +export interface IndexAccessor { + indexName: string; + keySets: string[][]; +} + export interface Schema { + findIndexBySortKeys(sortKeys: string[]): object | null; + findIndexByType(type: string): object | null; getAttribute(name: string): object; getAttributes(): object; getCollectionName(): string; getEntityName(): string; getIdName(): string; + getIndexAccessors(): Array; getIndexes(): object; getIndexKeys(indexName: string): string[]; getModelClass(): object; @@ -75,8 +83,7 @@ export interface Schema { export interface SchemaBuilder { addAttribute(name: string, data: object): SchemaBuilder; - addAllIndexWithComposite(...attributeNames: string[]): SchemaBuilder - addAllIndexWithTemplateField(fieldName: string, template: string): SchemaBuilder; + addAllIndex(sortKeys: string[]): SchemaBuilder; addIndex(name: string, partitionKey: object, sortKey: object): SchemaBuilder; addReference(referenceType: string, entityName: string, sortKeys?: string[]): SchemaBuilder; build(): Schema; diff --git a/packages/spacecat-shared-data-access/src/v2/models/base/schema.builder.js b/packages/spacecat-shared-data-access/src/v2/models/base/schema.builder.js index ff7cbf44..77a9693c 100755 --- a/packages/spacecat-shared-data-access/src/v2/models/base/schema.builder.js +++ b/packages/spacecat-shared-data-access/src/v2/models/base/schema.builder.js @@ -15,10 +15,10 @@ import { hasText, isInteger, isNonEmptyObject } from '@adobe/spacecat-shared-uti import { v4 as uuid, validate as uuidValidate } from 'uuid'; import { - capitalize, decapitalize, entityNameToAllPKValue, - entityNameToIdName, isNonEmptyArray, + entityNameToIdName, + isNonEmptyArray, } from '../../util/util.js'; import { INDEX_TYPES } from './constants.js'; @@ -70,52 +70,6 @@ const UPDATED_AT_ATTRIBUTE_DATA = { set: () => new Date().toISOString(), }; -/** Certain index names (primary, all) are reserved and cannot be reused. */ -const RESERVED_INDEX_NAMES = [INDEX_TYPES.PRIMARY, INDEX_TYPES.ALL]; - -/** - * Constructs a fully qualified index name. - * @param {string} service - The name of the service. - * @param {string} entity - The name of the entity. - * @param {string} name - The index name (e.g., 'all', 'byForeignKey'). - * @returns {string} The fully qualified index name. - */ -const createdIndexName = (service, entity, name) => `${service.toLowerCase()}-data-${entity}-${name}`; - -/** - * Sorts an indexes object by its keys alphabetically. - * @param {object} indexes - An object whose keys are index names and values are index definitions. - * @returns {object} A new object with the same entries, but keys sorted alphabetically. - */ -const sortIndexes = (indexes) => Object.fromEntries( - Object.entries(indexes).sort((a, b) => a[0].localeCompare(b[0])), -); - -/** - * Assigns GSI field names to indexes that don't have them yet. - * Ensures that if an "all" index exists, it uses gsi1 (already assigned) - * and other indexes continue numbering from gsi2 onwards. - * - * @param {object} indexes - Object of indexes that require naming. - * @param {object|null} all - The "all" index object if present, null otherwise. - */ -const numberGSIsIndexes = (indexes, all) => { - // if there's an "all" index, we start indexing subsequent GSIs from 2, - // because "all" index already occupies gsi1. - // if no "all" index exists, start from 1. - let gsiCounter = isNonEmptyObject(all) ? 1 : 0; - - Object.values(indexes).forEach((index) => { /* eslint-disable no-param-reassign */ - // only assign new field names and number through if none are provided. - if (!index.pk.field || !index.sk.field) { - gsiCounter += 1; - } - - index.pk.field = index.pk.field || `gsi${gsiCounter}pk`; - index.sk.field = index.sk.field || `gsi${gsiCounter}sk`; - }); -}; - /** * The SchemaBuilder class allows for constructing a schema definition * including attributes, indexes, and references to other entities. @@ -159,9 +113,9 @@ class SchemaBuilder { this.rawIndexes = { primary: null, - all: null, - belongs_to: {}, - other: {}, + all: [], + belongs_to: [], + other: [], }; this.attributes = {}; @@ -189,16 +143,14 @@ class SchemaBuilder { }; } - #internalAddIndex(name, partitionKey, sortKey, type) { - const indexFullName = createdIndexName(this.serviceName, this.entityName, name); - + #internalAddIndex(partitionKey, sortKey, type) { // store index config without assigning fields yet // the fields will be assigned in build phase based on sorting and presence of "all" index - this.rawIndexes[type][name] = { - ...(indexFullName && { index: indexFullName }), + this.rawIndexes[type].push({ + type, pk: { ...partitionKey }, sk: { ...sortKey }, - }; + }); } /** @@ -224,52 +176,24 @@ class SchemaBuilder { } /** - * Adds an "all" index based on composite attributes. - * The "all" index is a special index listing all entities, sorted by given attributes. - * Useful for global queries across all entities of this type. - * Will overwrite any existing "all" index. + * Adds an "all" index with composite partition and sort keys, or a template-based sort key. + * Useful for querying all entities of this type. Only one "all" index is allowed and a + * pre-existing "all" index will be overwritten. * - * @param {...string} attributeNames - The attribute names forming the composite sort key. + * @param {Array} sortKeys - The attributes to form the sort key. * @returns {SchemaBuilder} Returns this builder for method chaining. - * @throws {Error} If no attribute names are provided. + * @throws {Error} If composite attribute names or template are not provided. */ - addAllIndexWithComposite(...attributeNames) { - if (attributeNames.length === 0) { - throw new Error('At least one composite attribute name is required.'); + addAllIndex(sortKeys) { + if (!isNonEmptyArray(sortKeys)) { + throw new Error('Sort keys are required and must be a non-empty array.'); } - this.rawIndexes.all = { - index: createdIndexName(this.serviceName, this.entityName, INDEX_TYPES.ALL), - pk: { field: 'gsi1pk', template: entityNameToAllPKValue(this.entityName) }, - sk: { field: 'gsi1sk', composite: attributeNames }, - }; - - return this; - } - - /** - * Adds an "all" index with a template-based sort key. - * Useful if a single value template defines how entries are sorted. - * - * @param {string} fieldName - The sort key field name. - * @param {string} template - A template string defining how to generate the sort key value. - * @returns {SchemaBuilder} Returns this builder for method chaining. - * @throws {Error} If fieldName or template are not valid strings. - */ - addAllIndexWithTemplateField(fieldName, template) { - if (!hasText(fieldName)) { - throw new Error('fieldName is required and must be a non-empty string.'); - } - - if (!hasText(template)) { - throw new Error('template is required and must be a non-empty string.'); - } - - this.rawIndexes.all = { - index: createdIndexName(this.serviceName, this.entityName, 'all'), - pk: { field: 'gsi1pk', template: entityNameToAllPKValue(this.entityName) }, - sk: { field: fieldName, template }, - }; + this.#internalAddIndex( + { template: entityNameToAllPKValue(this.entityName) }, + { composite: sortKeys }, + INDEX_TYPES.ALL, + ); return this; } @@ -277,22 +201,13 @@ class SchemaBuilder { /** * Adds a generic secondary index (GSI). * - * @param {string} name - The index name. Cannot be 'primary' or 'all'. * @param {object} partitionKey - The partition key definition * (e.g., { composite: [attributeName] }). * @param {object} sortKey - The sort key definition. * @returns {SchemaBuilder} Returns this builder for method chaining. * @throws {Error} If index name is reserved or pk/sk configs are invalid. */ - addIndex(name, partitionKey, sortKey) { - if (!hasText(name)) { - throw new Error('Index name is required and must be a non-empty string.'); - } - - if (RESERVED_INDEX_NAMES.includes(name)) { - throw new Error(`Index name "${name}" is reserved.`); - } - + addIndex(partitionKey, sortKey) { if (!isNonEmptyObject(partitionKey)) { throw new Error('Partition key configuration (pk) is required and must be a non-empty object.'); } @@ -301,7 +216,7 @@ class SchemaBuilder { throw new Error('Sort key configuration (sk) is required and must be a non-empty object.'); } - this.#internalAddIndex(name, partitionKey, sortKey, INDEX_TYPES.OTHER); + this.#internalAddIndex(partitionKey, sortKey, INDEX_TYPES.OTHER); return this; } @@ -357,7 +272,6 @@ class SchemaBuilder { }); this.#internalAddIndex( - `by${capitalize(foreignKeyName)}`, { composite: [decapitalize(foreignKeyName)] }, { composite: isNonEmptyArray(sortKeys) ? sortKeys : ['updatedAt'] }, INDEX_TYPES.BELONGS_TO, @@ -378,21 +292,37 @@ class SchemaBuilder { */ #buildIndexes() { // eslint-disable-next-line camelcase - const { belongs_to, other } = this.rawIndexes; + const { all, belongs_to, other } = this.rawIndexes; + + // set the order of indexes + const orderedIndexes = [ + ...all, + // eslint-disable-next-line camelcase + ...belongs_to, + ...other, + ]; + + if (orderedIndexes.length > 5) { + throw new Error('Cannot have more than 5 indexes.'); + } - // belongs_to indexes come before other indexes - const indexes = { - ...sortIndexes(belongs_to), - ...sortIndexes(other), - }; + this.indexes = { primary: this.rawIndexes.primary }; - numberGSIsIndexes(indexes, this.rawIndexes.all); + let indexCounter = 0; + Object.values(orderedIndexes).forEach((index) => { + indexCounter += 1; - this.indexes = { - primary: this.rawIndexes.primary, - ...(this.rawIndexes.all && { all: this.rawIndexes.all }), - ...indexes, - }; + const pkFieldName = `gsi${indexCounter}pk`; + const skFieldName = `gsi${indexCounter}sk`; + const indexName = `${this.serviceName.toLowerCase()}-data-${pkFieldName}-${skFieldName}`; + + this.indexes[indexName] = { + index: indexName, + indexType: index.type, + pk: { field: pkFieldName, ...index.pk }, + sk: { field: skFieldName, ...index.sk }, + }; + }); } /** diff --git a/packages/spacecat-shared-data-access/src/v2/models/base/schema.js b/packages/spacecat-shared-data-access/src/v2/models/base/schema.js index e6cb3b75..8894ab70 100644 --- a/packages/spacecat-shared-data-access/src/v2/models/base/schema.js +++ b/packages/spacecat-shared-data-access/src/v2/models/base/schema.js @@ -140,6 +140,31 @@ class Schema { return this.indexes[indexName]; } + findIndexBySortKeys(sortKeys) { + // find index that has same sort keys, then remove the last sort key + // and find the index that has the remaining sort keys, etc. + for (let { length } = sortKeys; length > 0; length -= 1) { + const subKeyNames = sortKeys.slice(0, length); + const index = Object.values(this.indexes).find((candidate) => { + const { pk, sk } = candidate; + const allKeys = [...(pk?.facets || []), ...(sk?.facets || [])]; + + // check if all keys in the index are in the sort keys + return subKeyNames.every((key) => allKeys.includes(key)); + }); + + if (isNonEmptyObject(index)) { + return index; + } + } + + return null; + } + + findIndexByType(type) { + return Object.values(this.indexes).find((index) => index.indexType === type) || null; + } + /** * Returns the indexes for the schema. By default, this returns all indexes. * You can use the `exclude` parameter to exclude certain indexes. @@ -170,7 +195,7 @@ class Schema { } const pkKeys = Array.isArray(index.pk?.facets) ? index.pk.facets : []; - const skKeys = Array.isArray(index.sk?.facets) ? index.sk.facets : [index.sk?.field]; + const skKeys = Array.isArray(index.sk?.facets) ? index.sk.facets : []; return [...pkKeys, ...skKeys]; } diff --git a/packages/spacecat-shared-data-access/src/v2/models/configuration/configuration.collection.js b/packages/spacecat-shared-data-access/src/v2/models/configuration/configuration.collection.js index 5914d44b..4070c89d 100755 --- a/packages/spacecat-shared-data-access/src/v2/models/configuration/configuration.collection.js +++ b/packages/spacecat-shared-data-access/src/v2/models/configuration/configuration.collection.js @@ -10,7 +10,7 @@ * governing permissions and limitations under the License. */ -import { incrementVersion, sanitizeIdAndAuditFields } from '../../util/util.js'; +import { incrementVersion, sanitizeIdAndAuditFields, zeroPad } from '../../util/util.js'; import BaseCollection from '../base/base.collection.js'; /** @@ -31,6 +31,10 @@ class ConfigurationCollection extends BaseCollection { return super.create(sanitizedData); } + async findByVersion(version) { + return this.findByAll({ versionString: zeroPad(version, 10) }); + } + async findLatest() { return this.findByAll({}, { order: 'desc' }); } diff --git a/packages/spacecat-shared-data-access/src/v2/models/configuration/configuration.schema.js b/packages/spacecat-shared-data-access/src/v2/models/configuration/configuration.schema.js index be30795d..89d4145d 100755 --- a/packages/spacecat-shared-data-access/src/v2/models/configuration/configuration.schema.js +++ b/packages/spacecat-shared-data-access/src/v2/models/configuration/configuration.schema.js @@ -19,6 +19,7 @@ import Joi from 'joi'; import SchemaBuilder from '../base/schema.builder.js'; import Configuration from './configuration.model.js'; import ConfigurationCollection from './configuration.collection.js'; +import { zeroPad } from '../../util/util.js'; const handlerSchema = Joi.object().pattern(Joi.string(), Joi.object( { @@ -97,7 +98,13 @@ const schema = new SchemaBuilder(Configuration, ConfigurationCollection) required: true, readOnly: true, }) - // eslint-disable-next-line no-template-curly-in-string - .addAllIndexWithTemplateField('version', '${version}'); + .addAttribute('versionString', { // used for indexing/sorting + type: 'string', + required: true, + readOnly: true, + default: '0', // setting the default forces set() to run, to transform the version number to a string + set: (value, all) => zeroPad(all.version, 10), + }) + .addAllIndex(['versionString']); export default schema.build(); diff --git a/packages/spacecat-shared-data-access/src/v2/models/import-job/import-job.schema.js b/packages/spacecat-shared-data-access/src/v2/models/import-job/import-job.schema.js index 055af993..7781c4d8 100755 --- a/packages/spacecat-shared-data-access/src/v2/models/import-job/import-job.schema.js +++ b/packages/spacecat-shared-data-access/src/v2/models/import-job/import-job.schema.js @@ -142,9 +142,8 @@ const schema = new SchemaBuilder(ImportJob, ImportJobCollection) default: 0, validate: (value) => !value || isInteger(value), }) - .addAllIndexWithComposite('startedAt') + .addAllIndex(['startedAt']) .addIndex( - 'byStatus', { composite: ['status'] }, { composite: ['updatedAt'] }, ); diff --git a/packages/spacecat-shared-data-access/src/v2/models/organization/organization.schema.js b/packages/spacecat-shared-data-access/src/v2/models/organization/organization.schema.js index 36b120df..753ececf 100644 --- a/packages/spacecat-shared-data-access/src/v2/models/organization/organization.schema.js +++ b/packages/spacecat-shared-data-access/src/v2/models/organization/organization.schema.js @@ -46,6 +46,6 @@ const schema = new SchemaBuilder(Organization, OrganizationCollection) type: 'any', validate: (value) => !value || isNonEmptyObject(value), }) - .addAllIndexWithComposite('imsOrgId'); + .addAllIndex(['imsOrgId']); export default schema.build(); diff --git a/packages/spacecat-shared-data-access/src/v2/models/site-candidate/site-candidate.schema.js b/packages/spacecat-shared-data-access/src/v2/models/site-candidate/site-candidate.schema.js index 5557f16a..5c5b80ee 100755 --- a/packages/spacecat-shared-data-access/src/v2/models/site-candidate/site-candidate.schema.js +++ b/packages/spacecat-shared-data-access/src/v2/models/site-candidate/site-candidate.schema.js @@ -54,6 +54,6 @@ const schema = new SchemaBuilder(SiteCandidate, SiteCandidateCollection) .addAttribute('updatedBy', { type: 'string', }) - .addAllIndexWithComposite('baseURL'); + .addAllIndex(['baseURL']); export default schema.build(); diff --git a/packages/spacecat-shared-data-access/src/v2/models/site/site.schema.js b/packages/spacecat-shared-data-access/src/v2/models/site/site.schema.js index e757db73..a2baec66 100755 --- a/packages/spacecat-shared-data-access/src/v2/models/site/site.schema.js +++ b/packages/spacecat-shared-data-access/src/v2/models/site/site.schema.js @@ -81,9 +81,8 @@ const schema = new SchemaBuilder(Site, SiteCollection) set: () => new Date().toISOString(), validate: (value) => !value || isIsoDate(value), }) - .addAllIndexWithComposite('baseURL') + .addAllIndex(['baseURL']) .addIndex( - 'byDeliveryType', { composite: ['deliveryType'] }, { composite: ['updatedAt'] }, ); diff --git a/packages/spacecat-shared-data-access/src/v2/readme.md b/packages/spacecat-shared-data-access/src/v2/readme.md index e9cb2c05..c0610202 100755 --- a/packages/spacecat-shared-data-access/src/v2/readme.md +++ b/packages/spacecat-shared-data-access/src/v2/readme.md @@ -135,7 +135,7 @@ const userSchema = new SchemaBuilder(User, UserCollection) validate: (value) => value.includes('@'), }) .addAttribute('name', { type: 'string', required: true }) - .addAllIndexWithComposite('email') + .addAllIndex(['email']) .addReference('belongs_to', 'Organization') // Adds organizationId and byOrganizationId index .build(); diff --git a/packages/spacecat-shared-data-access/src/v2/util/util.js b/packages/spacecat-shared-data-access/src/v2/util/util.js index 1ec8f81c..ac860799 100644 --- a/packages/spacecat-shared-data-access/src/v2/util/util.js +++ b/packages/spacecat-shared-data-access/src/v2/util/util.js @@ -82,6 +82,13 @@ const incrementVersion = (version) => (isInteger(version) ? parseInt(version, 10 const isNonEmptyArray = (value) => Array.isArray(value) && value.length > 0; +const zeroPad = (num, length) => { + const str = String(num); + return str.length >= length + ? str + : '0'.repeat(length - str.length) + str; +}; + export { capitalize, classExtends, @@ -101,4 +108,5 @@ export { removeElectroProperties, sanitizeIdAndAuditFields, sanitizeTimestamps, + zeroPad, }; diff --git a/packages/spacecat-shared-data-access/test/it/configuration/configuration.test.js b/packages/spacecat-shared-data-access/test/it/configuration/configuration.test.js index e334c8bb..27e30432 100644 --- a/packages/spacecat-shared-data-access/test/it/configuration/configuration.test.js +++ b/packages/spacecat-shared-data-access/test/it/configuration/configuration.test.js @@ -18,7 +18,7 @@ import chaiAsPromised from 'chai-as-promised'; import { getDataAccess } from '../util/db.js'; import { seedDatabase } from '../util/seed.js'; -import { sanitizeIdAndAuditFields, sanitizeTimestamps } from '../../../src/v2/util/util.js'; +import { sanitizeIdAndAuditFields, sanitizeTimestamps, zeroPad } from '../../../src/v2/util/util.js'; use(chaiAsPromised); @@ -91,6 +91,7 @@ describe('Configuration IT', async () => { test: data, }, version: configuration.getVersion() + 1, + versionString: zeroPad(configuration.getVersion() + 1, 10), }; configuration.addHandler('test', data); diff --git a/packages/spacecat-shared-data-access/test/unit/v2/models/base/base.collection.test.js b/packages/spacecat-shared-data-access/test/unit/v2/models/base/base.collection.test.js index c2cef26c..a6824f8d 100755 --- a/packages/spacecat-shared-data-access/test/unit/v2/models/base/base.collection.test.js +++ b/packages/spacecat-shared-data-access/test/unit/v2/models/base/base.collection.test.js @@ -58,7 +58,7 @@ describe('BaseCollection', () => { let baseCollectionInstance; let mockElectroService; let mockEntityRegistry; - let mockIndexes = { primary: {} }; + let mockIndexes; let mockLogger; const mockRecord = { @@ -70,6 +70,7 @@ describe('BaseCollection', () => { }; beforeEach(() => { + mockIndexes = { primary: {}, all: { index: 'all', indexType: 'all' } }; mockEntityRegistry = { getCollection: stub(), }; @@ -167,7 +168,7 @@ describe('BaseCollection', () => { it('creates accessors for partition and sort key attributes', () => { mockIndexes = { - bySomeKey: { pk: { facets: ['someKey'] }, sk: { facets: ['someOtherKey'] } }, + bySomeKey: { index: 'bySomeKey', pk: { facets: ['someKey'] }, sk: { facets: ['someOtherKey'] } }, }; const instance = createInstance( @@ -188,7 +189,7 @@ describe('BaseCollection', () => { { go: () => Promise.resolve({ data: [] }) }, ); mockIndexes = { - bySomeKey: { pk: { facets: ['someKey'] }, sk: { facets: ['someOtherKey'] } }, + bySomeKey: { index: 'bySomeKey', pk: { facets: ['someKey'] }, sk: { facets: ['someOtherKey'] } }, }; mockElectroService.entities.mockEntityModel.model.schema = { @@ -257,7 +258,7 @@ describe('BaseCollection', () => { it('throws error if index is not found', async () => { await expect(baseCollectionInstance.findByIndexKeys({ someKey: 'someValue' }, { index: 'none' })) - .to.be.rejectedWith('Failed to query [mockEntityModel]: index [none] not found'); + .to.be.rejectedWith('Failed to query [mockEntityModel]: query proxy [none] not found'); expect(mockLogger.error).to.have.been.calledOnce; }); }); @@ -565,11 +566,24 @@ describe('BaseCollection', () => { it('successfully queries entities by index keys', async () => { const mockFindResult = { data: [mockRecord] }; + + mockIndexes = { + bySomeKey: { index: 'bySomeKey', pk: { facets: ['someKey'] }, sk: { facets: ['someOtherKey'] } }, + }; + mockElectroService.entities.mockEntityModel.query.bySomeKey.returns( { go: () => Promise.resolve(mockFindResult) }, ); - const result = await baseCollectionInstance.allByIndexKeys({ someKey: 'someValue' }); + const instance = createInstance( + mockElectroService, + mockEntityRegistry, + mockIndexes, + mockLogger, + ); + + const result = await instance.allByIndexKeys({ someKey: 'someValue' }); + expect(result).to.be.an('array').that.has.length(1); expect(result[0].record).to.deep.include(mockRecord); expect(mockElectroService.entities.mockEntityModel.query.bySomeKey) @@ -578,14 +592,24 @@ describe('BaseCollection', () => { it('successfully queries entities by primary index keys', async () => { const mockFindResult = { data: [mockRecord] }; + delete mockElectroService.entities.mockEntityModel.query.all; delete mockElectroService.entities.mockEntityModel.query.bySomeKey; + delete mockIndexes.all; mockElectroService.entities.mockEntityModel.query.primary.returns( { go: () => Promise.resolve(mockFindResult) }, ); - const result = await baseCollectionInstance.allByIndexKeys({ someKey: 'someValue' }); + const instance = createInstance( + mockElectroService, + mockEntityRegistry, + mockIndexes, + mockLogger, + ); + + const result = await instance.allByIndexKeys({ someKey: 'someValue' }); + expect(result).to.be.an('array').that.has.length(1); expect(result[0].record).to.deep.include(mockRecord); expect(mockElectroService.entities.mockEntityModel.query.primary) diff --git a/packages/spacecat-shared-data-access/test/unit/v2/models/base/base.model.test.js b/packages/spacecat-shared-data-access/test/unit/v2/models/base/base.model.test.js index 1c6be4e6..d12fdd70 100755 --- a/packages/spacecat-shared-data-access/test/unit/v2/models/base/base.model.test.js +++ b/packages/spacecat-shared-data-access/test/unit/v2/models/base/base.model.test.js @@ -80,7 +80,10 @@ describe('BaseModel', () => { }, suggestion: { entity: suggestionEntity, - query: { primary: stub().returns({ go: stub().resolves({ data: [mockRecord] }) }) }, + query: { + primary: stub().returns({ go: stub().resolves({ data: [mockRecord] }) }), + 'spacecat-data-gsi1pk-gsi1sk': stub().returns({ go: stub().resolves({ data: [mockRecord] }) }), + }, remove: stub().returns({ go: stub().resolves() }), indexes: { primary: {}, diff --git a/packages/spacecat-shared-data-access/test/unit/v2/models/base/schema.builder.test.js b/packages/spacecat-shared-data-access/test/unit/v2/models/base/schema.builder.test.js index 76b63c2b..b01d8f5d 100755 --- a/packages/spacecat-shared-data-access/test/unit/v2/models/base/schema.builder.test.js +++ b/packages/spacecat-shared-data-access/test/unit/v2/models/base/schema.builder.test.js @@ -95,9 +95,9 @@ describe('SchemaBuilder', () => { pk: { composite: ['mockModelId'], field: 'pk' }, sk: { composite: [], field: 'sk' }, }, - all: null, - belongs_to: {}, - other: {}, + all: [], + belongs_to: [], + other: [], }); }); }); @@ -135,84 +135,40 @@ describe('SchemaBuilder', () => { }); }); - describe('addAllIndexWithComposite', () => { - it('throws error if attribute name is not provided', () => { - expect(() => instance.addAllIndexWithComposite()) - .to.throw('At least one composite attribute name is required.'); - }); - - it('successfully adds an all index', () => { - const result = instance.addAllIndexWithComposite('test'); - - expect(result).to.equal(instance); - expect(instance.rawIndexes.all).to.deep.equal({ - index: 'spacecat-data-MockModel-all', - pk: { field: 'gsi1pk', template: 'ALL_MOCKMODELS' }, - sk: { composite: ['test'], field: 'gsi1sk' }, - }); - }); - }); - - describe('addAllIndexWithTemplateField', () => { - it('throws error if field name is not provided', () => { - expect(() => instance.addAllIndexWithTemplateField()) - .to.throw('fieldName is required and must be a non-empty string.'); - }); - - it('throws error if template is not provided', () => { - expect(() => instance.addAllIndexWithTemplateField('test')) - .to.throw('template is required and must be a non-empty string.'); - }); - - it('successfully adds an all index', () => { /* eslint-disable no-template-curly-in-string */ - const result = instance.addAllIndexWithTemplateField('test', '${test}'); - - expect(result).to.equal(instance); - expect(instance.rawIndexes.all).to.deep.equal({ - index: 'spacecat-data-MockModel-all', - pk: { field: 'gsi1pk', template: 'ALL_MOCKMODELS' }, - sk: { field: 'test', template: '${test}' }, - }); + describe('addAllIndex', () => { + it('throws error if no sort keys are provided', () => { + expect(() => instance.addAllIndex()) + .to.throw('Sort keys are required and must be a non-empty array.'); + expect(() => instance.addAllIndex('test')) + .to.throw('Sort keys are required and must be a non-empty array.'); }); }); describe('addIndex', () => { - it('throws error if index name is not provided', () => { - expect(() => instance.addIndex()) - .to.throw('Index name is required and must be a non-empty string.'); - }); - - it('throws error if index name is reserved', () => { - expect(() => instance.addIndex('all')) - .to.throw('Index name "all" is reserved.'); - expect(() => instance.addIndex('primary')) - .to.throw('Index name "primary" is reserved.'); - }); - it('throws error if pk is not provided', () => { - expect(() => instance.addIndex('test')) + expect(() => instance.addIndex()) .to.throw('Partition key configuration (pk) is required and must be a non-empty object.'); - expect(() => instance.addIndex('test', 'pk')) + expect(() => instance.addIndex('pk')) .to.throw('Partition key configuration (pk) is required and must be a non-empty object.'); - expect(() => instance.addIndex('test', {})) + expect(() => instance.addIndex({})) .to.throw('Partition key configuration (pk) is required and must be a non-empty object.'); }); it('throws error if sk is not provided', () => { - expect(() => instance.addIndex('test', { composite: ['test'] })) + expect(() => instance.addIndex({ composite: ['test'] })) .to.throw('Sort key configuration (sk) is required and must be a non-empty object.'); - expect(() => instance.addIndex('test', { composite: ['test'] }, 'sk')) + expect(() => instance.addIndex({ composite: ['test'] }, 'sk')) .to.throw('Sort key configuration (sk) is required and must be a non-empty object.'); - expect(() => instance.addIndex('test', { composite: ['test'] }, {})) + expect(() => instance.addIndex({ composite: ['test'] }, {})) .to.throw('Sort key configuration (sk) is required and must be a non-empty object.'); }); it('successfully adds an index', () => { - const result = instance.addIndex('test', { composite: ['test'] }, { composite: ['test'] }); + const result = instance.addIndex({ composite: ['test'] }, { composite: ['test'] }); expect(result).to.equal(instance); - expect(instance.rawIndexes.other.test).to.deep.equal({ - index: 'spacecat-data-MockModel-test', + expect(instance.rawIndexes.other[0]).to.deep.equal({ + type: 'other', pk: { composite: ['test'] }, sk: { composite: ['test'] }, }); @@ -288,8 +244,8 @@ describe('SchemaBuilder', () => { type: 'string', validate: instance.attributes.someEntityId.validate, }); - expect(instance.rawIndexes.belongs_to.bySomeEntityId).to.deep.equal({ - index: 'spacecat-data-MockModel-bySomeEntityId', + expect(instance.rawIndexes.belongs_to[0]).to.deep.equal({ + type: 'belongs_to', pk: { composite: ['someEntityId'] }, sk: { composite: ['updatedAt'] }, }); @@ -313,8 +269,8 @@ describe('SchemaBuilder', () => { type: 'string', validate: instance.attributes.someEntityId.validate, }); - expect(instance.rawIndexes.belongs_to.bySomeEntityId).to.deep.equal({ - index: 'spacecat-data-MockModel-bySomeEntityId', + expect(instance.rawIndexes.belongs_to[0]).to.deep.equal({ + type: 'belongs_to', pk: { composite: ['someEntityId'] }, sk: { composite: ['updatedAt'] }, }); @@ -362,10 +318,9 @@ describe('SchemaBuilder', () => { required: true, validate: () => true, }); - instance.addAllIndexWithComposite('baseURL'); - instance.addAllIndexWithTemplateField('test', '${test}'); - instance.addIndex('byDeliveryType', { composite: ['deliveryType'] }, { composite: ['updatedAt'] }); - instance.addIndex('bySomeField', { field: 'someField', composite: ['deliveryType'] }, { composite: ['updatedAt'] }); + instance.addAllIndex(['baseURL']); + instance.addIndex({ composite: ['deliveryType'] }, { composite: ['updatedAt'] }); + instance.addIndex({ field: 'someField', composite: ['deliveryType'] }, { composite: ['updatedAt'] }); const schema = instance.build(); @@ -417,28 +372,33 @@ describe('SchemaBuilder', () => { pk: { field: 'pk', composite: ['mockModelId'] }, sk: { field: 'sk', composite: [] }, }, - all: { - index: 'spacecat-data-MockModel-all', + 'spacecat-data-gsi1pk-gsi1sk': { + index: 'spacecat-data-gsi1pk-gsi1sk', + indexType: 'all', pk: { field: 'gsi1pk', template: 'ALL_MOCKMODELS' }, - sk: { field: 'test', template: '${test}' }, + sk: { field: 'gsi1sk', composite: ['baseURL'] }, }, - byOrganizationId: { - index: 'spacecat-data-MockModel-byOrganizationId', + 'spacecat-data-gsi2pk-gsi2sk': { + index: 'spacecat-data-gsi2pk-gsi2sk', + indexType: 'belongs_to', pk: { composite: ['organizationId'], field: 'gsi2pk' }, sk: { composite: ['updatedAt'], field: 'gsi2sk' }, }, - bySiteId: { - index: 'spacecat-data-MockModel-bySiteId', + 'spacecat-data-gsi3pk-gsi3sk': { + index: 'spacecat-data-gsi3pk-gsi3sk', + indexType: 'belongs_to', pk: { composite: ['siteId'], field: 'gsi3pk' }, sk: { composite: ['someField'], field: 'gsi3sk' }, }, - byDeliveryType: { - index: 'spacecat-data-MockModel-byDeliveryType', + 'spacecat-data-gsi4pk-gsi4sk': { + index: 'spacecat-data-gsi4pk-gsi4sk', + indexType: 'other', pk: { composite: ['deliveryType'], field: 'gsi4pk' }, sk: { composite: ['updatedAt'], field: 'gsi4sk' }, }, - bySomeField: { - index: 'spacecat-data-MockModel-bySomeField', + 'spacecat-data-gsi5pk-gsi5sk': { + index: 'spacecat-data-gsi5pk-gsi5sk', + indexType: 'other', pk: { composite: ['deliveryType'], field: 'someField' }, sk: { composite: ['updatedAt'], field: 'gsi5sk' }, }, @@ -471,5 +431,15 @@ describe('SchemaBuilder', () => { ], }); }); + + it('throws error if more than 5 indexes are added', () => { + instance.addAllIndex(['baseURL']); + instance.addIndex({ composite: ['deliveryType'] }, { composite: ['updatedAt'] }); + instance.addIndex({ field: 'someField', composite: ['deliveryType'] }, { composite: ['updatedAt'] }); + instance.addIndex({ field: 'someField', composite: ['deliveryType'] }, { composite: ['updatedAt'] }); + instance.addIndex({ field: 'someField', composite: ['deliveryType'] }, { composite: ['updatedAt'] }); + instance.addIndex({ field: 'someField', composite: ['deliveryType'] }, { composite: ['updatedAt'] }); + expect(() => instance.build()).to.throw('Cannot have more than 5 indexes.'); + }); }); }); diff --git a/packages/spacecat-shared-data-access/test/unit/v2/models/base/schema.test.js b/packages/spacecat-shared-data-access/test/unit/v2/models/base/schema.test.js index eb3c643d..8a257844 100644 --- a/packages/spacecat-shared-data-access/test/unit/v2/models/base/schema.test.js +++ b/packages/spacecat-shared-data-access/test/unit/v2/models/base/schema.test.js @@ -49,7 +49,7 @@ describe('Schema', () => { }, indexes: { primary: { pk: { composite: ['id'] } }, - byOrganizationId: { sk: { facets: ['organizationId'] } }, + byOrganizationId: { sk: { facets: ['organizationId'] }, indexType: 'belongs_to' }, }, references: [new Reference('belongs_to', 'Organization')], }; @@ -137,6 +137,21 @@ describe('Schema', () => { expect(instance.getIdName()).to.equal('mockEntityModelId'); }); + it('findIndexByType returns null if no index is found', () => { + expect(instance.findIndexByType('other')).to.equal(null); + }); + + it('findIndexByType returns index', () => { + expect(instance.findIndexByType('belongs_to')).to.deep.equal({ + indexType: 'belongs_to', + sk: { + facets: [ + 'organizationId', + ], + }, + }); + }); + it('getIndexAccessors', () => { expect(instance.getIndexAccessors()).to.deep.equal([{ indexName: 'byOrganizationId', @@ -154,7 +169,7 @@ describe('Schema', () => { it('getIndexes with exclusion', () => { expect(instance.getIndexes(['primary'])).to.deep.equal({ - byOrganizationId: { sk: { facets: ['organizationId'] } }, + byOrganizationId: { sk: { facets: ['organizationId'] }, indexType: 'belongs_to' }, }); }); diff --git a/packages/spacecat-shared-data-access/test/unit/v2/models/configuration/configuration.collection.test.js b/packages/spacecat-shared-data-access/test/unit/v2/models/configuration/configuration.collection.test.js index 433fa74d..e2a06e4f 100755 --- a/packages/spacecat-shared-data-access/test/unit/v2/models/configuration/configuration.collection.test.js +++ b/packages/spacecat-shared-data-access/test/unit/v2/models/configuration/configuration.collection.test.js @@ -92,6 +92,19 @@ describe('ConfigurationCollection', () => { }); }); + describe('findByVersion', () => { + it('finds configuration by version', async () => { + const mockResult = { configurationId: 's12345' }; + + instance.findByAll = stub().resolves(mockResult); + + const result = await instance.findByVersion(3); + + expect(result).to.deep.equal(mockResult); + expect(instance.findByAll).to.have.been.calledWithExactly({ versionString: '0000000003' }); + }); + }); + describe('findLatest', () => { it('returns the latest configuration', async () => { const mockResult = { configurationId: 's12345' }; diff --git a/packages/spacecat-shared-data-access/test/unit/v2/util/util.test.js b/packages/spacecat-shared-data-access/test/unit/v2/util/util.test.js index edfdaa77..5302e3b6 100644 --- a/packages/spacecat-shared-data-access/test/unit/v2/util/util.test.js +++ b/packages/spacecat-shared-data-access/test/unit/v2/util/util.test.js @@ -32,6 +32,7 @@ import { referenceToBaseMethodName, sanitizeIdAndAuditFields, sanitizeTimestamps, + zeroPad, } from '../../../../src/v2/util/util.js'; import Reference from '../../../../src/v2/models/base/reference.js'; @@ -217,4 +218,13 @@ describe('Utilities', () => { expect(sanitizeIdAndAuditFields('User', data)).to.deep.equal({ foo: 'bar' }); }); }); + + describe('zeroPad', () => { + it('adds leading zeros to a number', () => { + expect(zeroPad(123, 5)).to.equal('00123'); + }); + it('skips padding when number is longer than length', () => { + expect(zeroPad(123, 1)).to.equal('123'); + }); + }); });