fix: reduce number of indexes (#499)

adobe · Dec 19, 2024 · 95601cc · 95601cc
1 parent e2b5192
commit 95601cc
Show file tree

Hide file tree

Showing 21 changed files with 269 additions and 370 deletions.
diff --git a/packages/spacecat-shared-data-access/docs/schema.json b/packages/spacecat-shared-data-access/docs/schema.json
@@ -40,154 +40,42 @@
       ],
       "GlobalSecondaryIndexes": [
         {
-          "IndexName": "spacecat-data-ApiKey-byHashedApiKey",
+          "IndexName": "spacecat-data-gsi1pk-gsi1sk",
           "KeyAttributes": {
             "PartitionKey": { "AttributeName": "gsi1pk", "AttributeType": "S" },
             "SortKey": { "AttributeName": "gsi1sk", "AttributeType": "S" }
           },
           "Projection": { "ProjectionType": "ALL" }
         },
         {
-          "IndexName": "spacecat-data-ApiKey-byImsOrgIdAndImsUserId",
+          "IndexName": "spacecat-data-gsi2pk-gsi2sk",
           "KeyAttributes": {
             "PartitionKey": { "AttributeName": "gsi2pk", "AttributeType": "S" },
             "SortKey": { "AttributeName": "gsi2sk", "AttributeType": "S" }
           },
           "Projection": { "ProjectionType": "ALL" }
         },
         {
-          "IndexName": "spacecat-data-Opportunity-byAuditId",
-          "KeyAttributes": {
-            "PartitionKey": { "AttributeName": "gsi1pk", "AttributeType": "S" },
-            "SortKey": { "AttributeName": "gsi1sk", "AttributeType": "S" }
-          },
-          "Projection": { "ProjectionType": "ALL" }
-        },
-        {
-          "IndexName": "spacecat-data-Opportunity-bySiteId",
-          "KeyAttributes": {
-            "PartitionKey": { "AttributeName": "gsi2pk", "AttributeType": "S" },
-            "SortKey": { "AttributeName": "gsi2sk", "AttributeType": "S" }
-          },
-          "Projection": { "ProjectionType": "ALL" }
-        },
-        {
-          "IndexName": "spacecat-data-Suggestion-byOpportunityId",
-          "KeyAttributes": {
-            "PartitionKey": { "AttributeName": "gsi1pk", "AttributeType": "S" },
-            "SortKey": { "AttributeName": "gsi1sk", "AttributeType": "S" }
-          },
-          "Projection": { "ProjectionType": "ALL" }
-        },
-        {
-          "IndexName": "spacecat-data-Site-all",
-          "KeyAttributes": {
-            "PartitionKey": { "AttributeName": "gsi1pk", "AttributeType": "S" },
-            "SortKey": { "AttributeName": "gsi1sk", "AttributeType": "S" }
-          },
-          "Projection": { "ProjectionType": "ALL" }
-        },
-        {
-          "IndexName": "spacecat-data-Site-byOrganizationId",
-          "KeyAttributes": {
-            "PartitionKey": { "AttributeName": "gsi2pk", "AttributeType": "S" },
-            "SortKey": { "AttributeName": "gsi2sk", "AttributeType": "S" }
-          },
-          "Projection": { "ProjectionType": "ALL" }
-        },
-        {
-          "IndexName": "spacecat-data-Site-byDeliveryType",
+          "IndexName": "spacecat-data-gsi3pk-gsi3sk",
           "KeyAttributes": {
             "PartitionKey": { "AttributeName": "gsi3pk", "AttributeType": "S" },
             "SortKey": { "AttributeName": "gsi3sk", "AttributeType": "S" }
           },
           "Projection": { "ProjectionType": "ALL" }
         },
         {
-          "IndexName": "spacecat-data-Organization-all",
+          "IndexName": "spacecat-data-gsi4pk-gsi4sk",
           "KeyAttributes": {
-            "PartitionKey": { "AttributeName": "gsi1pk", "AttributeType": "S" },
-            "SortKey": { "AttributeName": "gsi1sk", "AttributeType": "S" }
-          },
-          "Projection": { "ProjectionType": "ALL" }
-        },
-        {
-          "IndexName": "spacecat-data-Audit-bySiteId",
-          "KeyAttributes": {
-            "PartitionKey": { "AttributeName": "gsi1pk", "AttributeType": "S" },
-            "SortKey": { "AttributeName": "gsi1sk", "AttributeType": "S" }
-          },
-          "Projection": { "ProjectionType": "ALL" }
-        },
-        {
-          "IndexName": "spacecat-data-Experiment-bySiteId",
-          "KeyAttributes": {
-            "PartitionKey": { "AttributeName": "gsi1pk", "AttributeType": "S" },
-            "SortKey": { "AttributeName": "gsi1sk", "AttributeType": "S" }
+            "PartitionKey": { "AttributeName": "gsi4pk", "AttributeType": "S" },
+            "SortKey": { "AttributeName": "gsi4sk", "AttributeType": "S" }
           },
           "Projection": { "ProjectionType": "ALL" }
         },
         {
-          "IndexName": "spacecat-data-KeyEvent-bySiteId",
+          "IndexName": "spacecat-data-gsi5pk-gsi5sk",
           "KeyAttributes": {
-            "PartitionKey": { "AttributeName": "gsi1pk", "AttributeType": "S" },
-            "SortKey": { "AttributeName": "gsi1sk", "AttributeType": "S" }
-          },
-          "Projection": { "ProjectionType": "ALL" }
-        },
-        {
-          "IndexName": "spacecat-data-SiteCandidate-all",
-          "KeyAttributes": {
-            "PartitionKey": { "AttributeName": "gsi1pk", "AttributeType": "S" },
-            "SortKey": { "AttributeName": "gsi1sk", "AttributeType": "S" }
-          },
-          "Projection": { "ProjectionType": "ALL" }
-        },
-        {
-          "IndexName": "spacecat-data-SiteCandidate-bySiteId",
-          "KeyAttributes": {
-            "PartitionKey": { "AttributeName": "gsi2pk", "AttributeType": "S" },
-            "SortKey": { "AttributeName": "gsi2sk", "AttributeType": "S" }
-          },
-          "Projection": { "ProjectionType": "ALL" }
-        },
-        {
-          "IndexName": "spacecat-data-SiteTopPage-bySiteId",
-          "KeyAttributes": {
-            "PartitionKey": { "AttributeName": "gsi1pk", "AttributeType": "S" },
-            "SortKey": { "AttributeName": "gsi1sk", "AttributeType": "S" }
-          },
-          "Projection": { "ProjectionType": "ALL" }
-        },
-        {
-          "IndexName": "spacecat-data-Configuration-all",
-          "KeyAttributes": {
-            "PartitionKey": { "AttributeName": "gsi1pk", "AttributeType": "S" },
-            "SortKey": { "AttributeName": "version", "AttributeType": "N" }
-          },
-          "Projection": { "ProjectionType": "ALL" }
-        },
-        {
-          "IndexName": "spacecat-data-ImportJob-all",
-          "KeyAttributes": {
-            "PartitionKey": { "AttributeName": "gsi1pk", "AttributeType": "S" },
-            "SortKey": { "AttributeName": "gsi1sk", "AttributeType": "S" }
-          },
-          "Projection": { "ProjectionType": "ALL" }
-        },
-        {
-          "IndexName": "spacecat-data-ImportJob-byStatus",
-          "KeyAttributes": {
-            "PartitionKey": { "AttributeName": "gsi2pk", "AttributeType": "S" },
-            "SortKey": { "AttributeName": "gsi2sk", "AttributeType": "S" }
-          },
-          "Projection": { "ProjectionType": "ALL" }
-        },
-        {
-          "IndexName": "spacecat-data-ImportUrl-byImportJobId",
-          "KeyAttributes": {
-            "PartitionKey": { "AttributeName": "gsi1pk", "AttributeType": "S" },
-            "SortKey": { "AttributeName": "gsi1sk", "AttributeType": "S" }
+            "PartitionKey": { "AttributeName": "gsi5pk", "AttributeType": "S" },
+            "SortKey": { "AttributeName": "gsi5sk", "AttributeType": "S" }
           },
           "Projection": { "ProjectionType": "ALL" }
         }

diff --git a/packages/spacecat-shared-data-access/src/v2/models/api-key/api-key.schema.js b/packages/spacecat-shared-data-access/src/v2/models/api-key/api-key.schema.js
@@ -69,12 +69,10 @@ const schema = new SchemaBuilder(ApiKey, ApiKeyCollection)
     },
   })
   .addIndex(
-    'byHashedApiKey',
     { composite: ['hashedApiKey'] },
     { composite: ['updatedAt'] },
   )
   .addIndex(
-    'byImsOrgIdAndImsUserId',
     { composite: ['imsOrgId', 'imsUserId'] },
     { composite: ['updatedAt'] },
   );

diff --git a/packages/spacecat-shared-data-access/src/v2/models/base/base.collection.js b/packages/spacecat-shared-data-access/src/v2/models/base/base.collection.js
@@ -24,7 +24,6 @@ import { guardId } from '../../util/guards.js';
 import {
   entityNameToAllPKValue,
   isNonEmptyArray,
-  keyNamesToIndexName,
   removeElectroProperties,
 } from '../../util/util.js';
 import { INDEX_TYPES } from './constants.js';
@@ -40,26 +39,24 @@ function isValidParent(parent, child) {
 }
 
 /**
- * Attempts to find an index name matching a generated name from the given keyNames.
- * If no exact match is found, it progressively shortens the keyNames by removing the last one
- * and tries again. If still no match, it tries the "all" index, and then "primary".
- *
- * @param {object} indexes - The available indexes, keyed by their names.
- * @param {object} keys - The keys to find an index name for.
- * @returns {object} The found index.
+ * Finds the index name by the keys provided. The index is searched
+ * keys to match the combination of partition and sort keys. If no
+ * index is found, we fall back to the "all" index, then the "primary".
+ * @param {Schema} schema - The schema to search for the index.
+ * @param {Object} keys - The keys to search for.
+ * @return {*|string} - The index name.
  */
-function findIndexNameByKeys(indexes, keys) {
+function findIndexNameByKeys(schema, keys) {
   const keyNames = Object.keys(keys);
-  for (let { length } = keyNames; length > 0; length -= 1) {
-    const subKeyNames = keyNames.slice(0, length);
-    const candidateName = keyNamesToIndexName(subKeyNames);
-    if (indexes[candidateName]) {
-      return candidateName;
-    }
+
+  const index = schema.findIndexBySortKeys(keyNames);
+  if (index) {
+    return index.index;
   }
 
-  if (indexes.all) {
-    return INDEX_TYPES.ALL;
+  const allIndex = schema.findIndexByType(INDEX_TYPES.ALL);
+  if (allIndex) {
+    return allIndex.index;
   }
 
   return INDEX_TYPES.PRIMARY;
@@ -176,11 +173,11 @@ class BaseCollection {
       throw new Error(message);
     }
 
-    const indexName = options.index || findIndexNameByKeys(this.entity.query, keys);
+    const indexName = options.index || findIndexNameByKeys(this.schema, keys);
     const index = this.entity.query[indexName];
 
     if (!index) {
-      const message = `Failed to query [${this.entityName}]: index [${indexName}] not found`;
+      const message = `Failed to query [${this.entityName}]: query proxy [${indexName}] not found`;
       this.log.error(message);
       throw new Error(message);
     }
@@ -243,7 +240,8 @@ class BaseCollection {
    * Finds a single entity from the "all" index. Requires an index named "all" with a partition key
    * named "pk" with a static value of "ALL_<ENTITYNAME>".
    * @param {Object} [sortKeys] - The sort keys to use for the query.
-   * @param {{index?: string, attributes?: string[]}} [options] - Additional options for the query.
+   * @param {{index?: string, attributes?: string[], order?: string}} [options] -
+   * Additional options for the query.
    * @return {Promise<BaseModel|Array<BaseModel>|null>}
    */
   async findByAll(sortKeys = {}, options = {}) {
@@ -254,7 +252,7 @@ class BaseCollection {
     }
 
     const keys = { pk: entityNameToAllPKValue(this.entityName), ...sortKeys };
-    return this.#queryByIndexKeys(keys, { ...options, index: INDEX_TYPES.ALL, limit: 1 });
+    return this.#queryByIndexKeys(keys, { ...options, limit: 1 });
   }
 
   /**

diff --git a/packages/spacecat-shared-data-access/src/v2/models/base/index.d.ts b/packages/spacecat-shared-data-access/src/v2/models/base/index.d.ts
@@ -58,12 +58,20 @@ export interface Reference {
   isRemoveDependents(): boolean;
 }
 
+export interface IndexAccessor {
+  indexName: string;
+  keySets: string[][];
+}
+
 export interface Schema {
+  findIndexBySortKeys(sortKeys: string[]): object | null;
+  findIndexByType(type: string): object | null;
   getAttribute(name: string): object;
   getAttributes(): object;
   getCollectionName(): string;
   getEntityName(): string;
   getIdName(): string;
+  getIndexAccessors(): Array<IndexAccessor>;
   getIndexes(): object;
   getIndexKeys(indexName: string): string[];
   getModelClass(): object;
@@ -75,8 +83,7 @@ export interface Schema {
 
 export interface SchemaBuilder {
   addAttribute(name: string, data: object): SchemaBuilder;
-  addAllIndexWithComposite(...attributeNames: string[]): SchemaBuilder
-  addAllIndexWithTemplateField(fieldName: string, template: string): SchemaBuilder;
+  addAllIndex(sortKeys: string[]): SchemaBuilder;
   addIndex(name: string, partitionKey: object, sortKey: object): SchemaBuilder;
   addReference(referenceType: string, entityName: string, sortKeys?: string[]): SchemaBuilder;
   build(): Schema;