feat(dynamo): scan support

packages/spacecat-shared-dynamo/src/index.d.ts

@@ -11,7 +11,7 @@
  */
 
 import { DynamoDB } from '@aws-sdk/client-dynamodb';
-import { DynamoDBDocumentClient, QueryCommandInput } from '@aws-sdk/lib-dynamodb';
+import { DynamoDBDocumentClient, QueryCommandInput, ScanCommandInput } from '@aws-sdk/lib-dynamodb';
 
 export declare interface Logger {
   error(message: string, ...args: unknown[]): void;
@@ -24,6 +24,7 @@ export declare interface DynamoDbKey {
 }
 
 export declare interface DynamoDbClient {
+  scan(originalParams: ScanCommandInput): Promise<object[]>;
   query(originalParams: QueryCommandInput): Promise<object[]>;
   getItem(tableName: string, key: DynamoDbKey): Promise<object>;
   putItem(tableName: string, item: object): Promise<{ message: string }>;

packages/spacecat-shared-dynamo/src/index.js

@@ -14,6 +14,7 @@ import AWSXray from 'aws-xray-sdk';
 import { DynamoDB } from '@aws-sdk/client-dynamodb';
 import { DynamoDBDocument } from '@aws-sdk/lib-dynamodb';
 
+import scan from './modules/scan.js';
 import query from './modules/query.js';
 import getItem from './modules/getItem.js';
 import putItem from './modules/putItem.js';
@@ -37,6 +38,7 @@ const createClient = (
     },
   }),
 ) => ({
+  scan: (params) => scan(docClient, params, log),
   query: (params) => query(docClient, params, log),
   getItem: (tableName, key) => getItem(docClient, tableName, key, log),
   putItem: (tableName, item) => putItem(docClient, tableName, item, log),

packages/spacecat-shared-dynamo/src/modules/scan.js

@@ -0,0 +1,62 @@
+/*
+ * Copyright 2024 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+import { performance } from 'perf_hooks';
+
+/**
+ * Scans DynamoDB and automatically handles pagination to retrieve all items.
+ *
+ * @param {DynamoDBDocumentClient} docClient - The AWS SDK DynamoDB Document client instance.
+ * @param {Object} originalParams - The parameters for the DynamoDB scan.
+ * @param {Logger} log - The logging object, defaults to console.
+ * @returns {Promise<Array>} A promise that resolves to an array of items retrieved from DynamoDB.
+ * @throws {Error} Throws an error if the DynamoDB scan operation fails.
+ */
+async function scan(docClient, originalParams, log = console) {
+  let items = [];
+  const params = { ...originalParams };
+
+  let totalTime = 0;
+  let paginationCount = 0;
+
+  try {
+    let data;
+    if (params.Limit && params.Limit <= 1) {
+      const result = await docClient.scan(params);
+      return result.Items;
+    }
+    do {
+      const startTime = performance.now();
+
+      // eslint-disable-next-line no-await-in-loop
+      data = await docClient.scan(params);
+
+      const endTime = performance.now(); // End timing
+      const duration = endTime - startTime;
+      totalTime += duration;
+      paginationCount += 1;
+
+      log.info(`Pagination ${paginationCount} scan time: ${duration.toFixed(2)} ms`);
+
+      items = items.concat(data.Items);
+      params.ExclusiveStartKey = data.LastEvaluatedKey;
+    } while (data.LastEvaluatedKey);
+  } catch (error) {
+    log.error('DB Scan Error:', error);
+    throw error;
+  }
+
+  log.info(`Total scan time: ${totalTime.toFixed(2)} ms with ${paginationCount} paginations for scan: ${JSON.stringify(params)}`);
+
+  return items;
+}
+
+export default scan;

packages/spacecat-shared-dynamo/test/index.test.js

@@ -25,6 +25,12 @@ describe('createClient', () => {
     docClient = DynamoDBDocumentClient.from(dbClient);
   });
 
+  it('should create a DynamoDB client with scan method', () => {
+    const client = createClient(console, dbClient, docClient);
+    expect(client).to.have.property('scan');
+    expect(client.query).to.be.a('function');
+  });
+
   it('should create a DynamoDB client with query method', () => {
     const client = createClient(console, dbClient, docClient);
     expect(client).to.have.property('query');
@@ -51,6 +57,6 @@ describe('createClient', () => {
 
   it('should use default parameters if none are provided', () => {
     const client = createClient();
-    expect(client).to.have.all.keys('query', 'getItem', 'putItem', 'removeItem');
+    expect(client).to.have.all.keys('scan', 'query', 'getItem', 'putItem', 'removeItem');
   });
 });

packages/spacecat-shared-dynamo/test/modules/scan.test.js

@@ -0,0 +1,77 @@
+/*
+ * Copyright 2024 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+/* eslint-env mocha */
+
+import { expect } from 'chai';
+import { createClient } from '../../src/index.js';
+
+describe('scan', () => {
+  const scanParams = {
+    TableName: 'TestTable',
+  };
+  const scanParamsWithLimit = {
+    TableName: 'TestTable',
+    Limit: 1,
+  };
+
+  let dynamoDbClient;
+  let mockDocClient;
+
+  beforeEach(() => {
+    mockDocClient = {
+      scan: async (params) => {
+        if (params.Limit) {
+          return { Items: ['item1'] };
+        }
+        // Check if LastEvaluatedKey is provided and simulate pagination
+        if (params.ExclusiveStartKey === 'key2') {
+          return { Items: ['item3'], LastEvaluatedKey: undefined };
+        } else {
+          return { Items: ['item1', 'item2'], LastEvaluatedKey: 'key2' };
+        }
+      },
+    };
+
+    dynamoDbClient = createClient(console, undefined, mockDocClient);
+  });
+
+  it('scans items from the database', async () => {
+    const result = await dynamoDbClient.scan(scanParams);
+    expect(result).to.be.an('array');
+  });
+
+  it('scans items from the database with pagination', async () => {
+    const result = await dynamoDbClient.scan(scanParams);
+    expect(result).to.have.lengthOf(3);
+    expect(result).to.deep.equal(['item1', 'item2', 'item3']);
+  });
+
+  it('scans items from the database with limit', async () => {
+    const result = await dynamoDbClient.scan(scanParamsWithLimit);
+    expect(result).to.have.lengthOf(1);
+    expect(result).to.deep.equal(['item1']);
+  });
+
+  it('handles errors in scan', async () => {
+    mockDocClient.scan = async () => {
+      throw new Error('Scan failed');
+    };
+
+    try {
+      await dynamoDbClient.scan(scanParams);
+      expect.fail('scanDb did not throw as expected');
+    } catch (error) {
+      expect(error.message).to.equal('Scan failed');
+    }
+  });
+});