From 6d82b57627062442d0d9f7d178e1c701855ed28d Mon Sep 17 00:00:00 2001 From: David Date: Tue, 30 Jul 2024 12:12:47 -0700 Subject: [PATCH] V3 updates, Trivy fix, QA changes for 291 --- .github/workflows/analysis.yaml | 73 +++--- .github/workflows/deploy_dev.yaml | 2 +- .github/workflows/deploy_prod.yaml | 2 +- .github/workflows/deploy_test.yaml | 2 +- .github/workflows/on-pr.yaml | 2 +- .trivyignore | 2 + arSam/__tests__/mock_data.json | 1 + arSam/__tests__/settings.js | 8 +- arSam/__tests__/setup.js | 78 ++++-- arSam/handlers/activity/POST/index.js | 18 +- .../activity/__tests__/activity.test.js | 80 +++--- arSam/handlers/export-variance/GET/index.js | 24 +- .../handlers/export-variance/GET/package.json | 11 - .../__tests__/export-variance.tests.js | 63 ++--- .../export-variance/invokable/index.js | 17 +- .../export-variance/invokable/package.json | 9 - arSam/handlers/export/GET/index.js | 25 +- arSam/handlers/export/GET/package.json | 11 - .../handlers/export/__tests__/export.test.js | 79 +++--- arSam/handlers/export/invokable/index.js | 4 +- arSam/handlers/export/invokable/package.json | 1 - arSam/handlers/fiscalYearEnd/POST/index.js | 9 +- .../handlers/fiscalYearEnd/POST/package.json | 1 - .../__tests__/fiscalYearEnd.test.js | 56 +++-- .../nameUpdate/__tests__/name-update.test.js | 129 ++++++---- arSam/handlers/nameUpdate/index.js | 22 +- arSam/handlers/nameUpdate/package.json | 3 +- arSam/handlers/park/POST/index.js | 9 +- arSam/handlers/park/__tests__/park.test.js | 54 ++-- arSam/handlers/subArea/DELETE/index.js | 30 ++- arSam/handlers/subArea/DELETE/package.json | 9 - arSam/handlers/subArea/POST/index.js | 5 +- .../subArea/__tests__/subArea.test.js | 237 ++++++++---------- arSam/handlers/variance/PUT/index.js | 4 +- .../variance/__tests__/variance.test.js | 132 +++++----- .../__tests__/dynamoLayer.test.js | 119 ++++----- arSam/layers/baseLayer/baseLayer.js | 51 +++- arSam/layers/baseLayer/nodejs/package.json | 3 + .../__tests__/constantsLayer.test.js | 2 +- .../__tests__/formulasLayer.test.js | 2 +- arSam/layers/functionsLayer/Makefile | 2 +- arSam/layers/functionsLayer/functionsLayer.js | 5 +- .../layers/functionsLayer/nodejs/package.json | 5 +- .../__tests__/keycloakLayer.test.js | 12 +- .../__tests__/subAreaLayer.test.js | 6 +- arSam/package.json | 18 +- 46 files changed, 742 insertions(+), 695 deletions(-) create mode 100644 .trivyignore delete mode 100644 arSam/handlers/export-variance/GET/package.json delete mode 100644 arSam/handlers/export-variance/invokable/package.json delete mode 100644 arSam/handlers/export/GET/package.json delete mode 100644 arSam/handlers/subArea/DELETE/package.json rename arSam/layers/{ => baseLayer}/__tests__/dynamoLayer.test.js (59%) rename arSam/layers/{ => constantsLayer}/__tests__/constantsLayer.test.js (98%) rename arSam/layers/{ => formulaLayer}/__tests__/formulasLayer.test.js (95%) rename arSam/layers/{ => keycloakLayer}/__tests__/keycloakLayer.test.js (87%) rename arSam/layers/{ => subAreaLayer}/__tests__/subAreaLayer.test.js (93%) diff --git a/.github/workflows/analysis.yaml b/.github/workflows/analysis.yaml index 354418c..19494a5 100644 --- a/.github/workflows/analysis.yaml +++ b/.github/workflows/analysis.yaml @@ -1,38 +1,43 @@ -# name: Analysis +name: Analysis -# on: -# push: -# branches: [main] -# pull_request: -# types: [opened, reopened, synchronize, ready_for_review, converted_to_draft] -# schedule: -# - cron: "0 11 * * 0" # 3 AM PST = 12 PM UDT, runs sundays -# workflow_dispatch: +on: + push: + branches: [main] + pull_request: + types: [opened, reopened, synchronize, ready_for_review, converted_to_draft] + schedule: + - cron: "29 22 * * 4" -# concurrency: -# group: ${{ github.workflow }}-${{ github.ref }} -# cancel-in-progress: true +permissions: + contents: read -# jobs: -# # https://github.com/marketplace/actions/aqua-security-trivy -# trivy: -# name: Trivy Security Scan -# if: ${{ ! github.event.pull_request.draft }} -# runs-on: ubuntu-22.04 -# timeout-minutes: 1 -# steps: -# - uses: actions/checkout@v4 -# - name: Run Trivy vulnerability scanner in repo mode -# uses: aquasecurity/trivy-action@0.22.0 -# with: -# format: "sarif" -# output: "trivy-results.sarif" -# ignore-unfixed: true -# scan-type: "fs" -# scanners: "vuln,secret,config" -# severity: "CRITICAL,HIGH" +jobs: + trivy: + permissions: + contents: read + security-events: write + name: Trivy Security Scan + if: ${{ ! github.event.pull_request.draft }} + runs-on: ubuntu-22.04 + timeout-minutes: 1 + steps: + - name: Checkout code + uses: actions/checkout@v3 -# - name: Upload Trivy scan results to GitHub Security tab -# uses: github/codeql-action/upload-sarif@v3 -# with: -# sarif_file: "trivy-results.sarif" + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@0.22.0 + with: + format: "sarif" + output: "trivy-results.sarif" + ignore-unfixed: true + scan-type: "fs" + scanners: "vuln,secret,config" + severity: "CRITICAL,HIGH" + + - name: Print SARIF file + run: cat trivy-results.sarif + + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: "trivy-results.sarif" diff --git a/.github/workflows/deploy_dev.yaml b/.github/workflows/deploy_dev.yaml index 894ffaa..7cc61d3 100644 --- a/.github/workflows/deploy_dev.yaml +++ b/.github/workflows/deploy_dev.yaml @@ -18,7 +18,7 @@ jobs: environment: dev strategy: matrix: - node-version: [18.x] + node-version: [20.x] defaults: run: diff --git a/.github/workflows/deploy_prod.yaml b/.github/workflows/deploy_prod.yaml index 45ba591..bf9336f 100644 --- a/.github/workflows/deploy_prod.yaml +++ b/.github/workflows/deploy_prod.yaml @@ -20,7 +20,7 @@ jobs: environment: prod strategy: matrix: - node-version: [18.x] + node-version: [20.x] defaults: run: diff --git a/.github/workflows/deploy_test.yaml b/.github/workflows/deploy_test.yaml index 1e76f2f..24fa58a 100644 --- a/.github/workflows/deploy_test.yaml +++ b/.github/workflows/deploy_test.yaml @@ -23,7 +23,7 @@ jobs: environment: test strategy: matrix: - node-version: [18.x] + node-version: [20.x] defaults: run: diff --git a/.github/workflows/on-pr.yaml b/.github/workflows/on-pr.yaml index 8fab930..0fd695e 100644 --- a/.github/workflows/on-pr.yaml +++ b/.github/workflows/on-pr.yaml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - node-version: [18.x] + node-version: [20.x] defaults: run: working-directory: "./arSam" diff --git a/.trivyignore b/.trivyignore new file mode 100644 index 0000000..b222856 --- /dev/null +++ b/.trivyignore @@ -0,0 +1,2 @@ +# Ignore possible issue with empty URI +AVD-AWS-0112 diff --git a/arSam/__tests__/mock_data.json b/arSam/__tests__/mock_data.json index 183d8de..f248e7d 100644 --- a/arSam/__tests__/mock_data.json +++ b/arSam/__tests__/mock_data.json @@ -30,6 +30,7 @@ }, { "sk": "HIST", + "orcs": "NA", "subAreas": [ { "name": "LEGACY SUBAREA", diff --git a/arSam/__tests__/settings.js b/arSam/__tests__/settings.js index b9ccd06..52ff7f9 100644 --- a/arSam/__tests__/settings.js +++ b/arSam/__tests__/settings.js @@ -1,13 +1,15 @@ -const REGION = process.env.AWS_REGION || 'local'; -const ENDPOINT = 'http://localhost:8000'; +const REGION = process.env.AWS_REGION || 'local-env'; +const ENDPOINT = 'http://172.17.0.2:8000'; const TABLE_NAME = process.env.TABLE_NAME || 'ParksAr-tests'; const CONFIG_TABLE_NAME = process.env.CONFIG_TABLE_NAME || 'ConfigAr-tests'; const NAME_CACHE_TABLE_NAME = process.env.NAME_CACHE_TABLE_NAME || 'NameCacheAr-tests'; + module.exports = { REGION, ENDPOINT, TABLE_NAME, CONFIG_TABLE_NAME, - NAME_CACHE_TABLE_NAME, + NAME_CACHE_TABLE_NAME }; + \ No newline at end of file diff --git a/arSam/__tests__/setup.js b/arSam/__tests__/setup.js index bbc21ce..91db9a3 100644 --- a/arSam/__tests__/setup.js +++ b/arSam/__tests__/setup.js @@ -1,19 +1,16 @@ -const { DynamoDB } = require('@aws-sdk/client-dynamodb'); +const { DynamoDBClient, CreateTableCommand, DeleteTableCommand } = require('@aws-sdk/client-dynamodb'); +const { REGION, ENDPOINT } = require('./settings'); +const crypto = require('crypto'); -const { REGION, ENDPOINT, TABLE_NAME, CONFIG_TABLE_NAME, NAME_CACHE_TABLE_NAME } = require('./settings'); -module.exports = async () => { - dynamoDb = new DynamoDB({ +async function createDB (TABLE_NAME, NAME_CACHE_TABLE_NAME, CONFIG_TABLE_NAME) { + dynamoDb = new DynamoDBClient({ region: REGION, endpoint: ENDPOINT }); - // TODO: This should pull in the JSON version of our serverless.yml! - try { - console.log("Creating main table."); - await dynamoDb - .createTable({ + let params = { TableName: TABLE_NAME, KeySchema: [ { @@ -61,11 +58,12 @@ module.exports = async () => { } } ] - }); + } + + + await dynamoDb.send(new CreateTableCommand(params)); - console.log("Creating name-cache table."); - await dynamoDb - .createTable({ + params = { TableName: NAME_CACHE_TABLE_NAME, KeySchema: [ { @@ -83,11 +81,11 @@ module.exports = async () => { ReadCapacityUnits: 1, WriteCapacityUnits: 1 } - }); - console.log("Creating config table."); - await dynamoDb - .createTable({ + } + await dynamoDb.send(new CreateTableCommand(params)) + + params = { TableName: CONFIG_TABLE_NAME, KeySchema: [ { @@ -105,8 +103,52 @@ module.exports = async () => { ReadCapacityUnits: 1, WriteCapacityUnits: 1 } - }); + } + await dynamoDb.send(new CreateTableCommand(params)); + } catch (err) { console.log(err); } }; + +function getHashedText(text) { + return crypto.createHash('md5').update(text).digest('hex'); +} + +async function deleteDB(TABLE_NAME, NAME_CACHE_TABLE_NAME, CONFIG_TABLE_NAME) { + const dynamoDb = new DynamoDBClient({ + region: REGION, + endpoint: ENDPOINT + }); + + try { + //Delete Main Table + let param = { + TableName: TABLE_NAME + }; + + await dynamoDb.send(new DeleteTableCommand(param)); + + //Delete NameChache Table + param = { + TableName: NAME_CACHE_TABLE_NAME + }; + await dynamoDb.send(new DeleteTableCommand(param)); + + //Delete Config Table + param = { + TableName: CONFIG_TABLE_NAME + }; + await dynamoDb.send(new DeleteTableCommand(param)); + + + } catch (err) { + console.log(err); + } +} + +module.exports = { + createDB, + getHashedText, + deleteDB +} \ No newline at end of file diff --git a/arSam/handlers/activity/POST/index.js b/arSam/handlers/activity/POST/index.js index 697f31b..27c9da2 100644 --- a/arSam/handlers/activity/POST/index.js +++ b/arSam/handlers/activity/POST/index.js @@ -1,7 +1,9 @@ -const { marshall, unmarshall } = require('@aws-sdk/util-dynamodb'); const { DateTime } = require('luxon'); const { - dynamodb, + dynamoClient, + PutItemCommand, + DeleteItemCommand, + UpdateItemCommand, runQuery, TABLE_NAME, getOne, @@ -9,7 +11,9 @@ const { TIMEZONE, sendResponse, logger, - calculateVariance + calculateVariance, + marshall, + unmarshall, } = require('/opt/baseLayer'); const { EXPORT_VARIANCE_CONFIG } = require('/opt/constantsLayer'); @@ -132,7 +136,7 @@ async function deleteVariance(body) { logger.info('Deleting variance record:', params); - await dynamodb.deleteItem(params); + await dynamoClient.send(new DeleteItemCommand(params)); } async function checkVarianceTrigger(body) { @@ -228,7 +232,7 @@ async function createVariance(body, fields) { TableName: TABLE_NAME, Item: newObject, }; - await dynamodb.putItem(putObj); + await dynamoClient.send(new PutItemCommand(putObj)); } catch (e) { logger.error(e); } @@ -318,7 +322,7 @@ async function handleLockUnlock(record, lock, context) { ReturnValues: 'ALL_NEW', }; try { - const res = await dynamodb.updateItem(updateObj); + const res = await dynamoClient.send(new UpdateItemCommand(updateObj)); logger.info(`Updated record pk: ${record.pk}, sk: ${record.sk} `); const s = lock ? 'locked' : 'unlocked'; return sendResponse(200, { @@ -380,7 +384,7 @@ async function handleActivity(body, lock = false, context) { Item: newObject, }; - await dynamodb.putItem(putObject); + await dynamoClient.send(new PutItemCommand(putObject)); logger.info('Activity Updated.'); return sendResponse(200, body, context); } catch (err) { diff --git a/arSam/handlers/activity/__tests__/activity.test.js b/arSam/handlers/activity/__tests__/activity.test.js index 6b7d797..a7b4cdf 100644 --- a/arSam/handlers/activity/__tests__/activity.test.js +++ b/arSam/handlers/activity/__tests__/activity.test.js @@ -1,6 +1,6 @@ const { DynamoDBClient, PutItemCommand, GetItemCommand } = require('@aws-sdk/client-dynamodb'); const { marshall, unmarshall } = require('@aws-sdk/util-dynamodb'); -const { REGION, ENDPOINT, TABLE_NAME } = require("../../../__tests__/settings"); +const { REGION, ENDPOINT } = require("../../../__tests__/settings"); const { PARKSLIST, SUBAREAS, @@ -9,6 +9,7 @@ const { FISCAL_YEAR_LOCKS, } = require("../../../__tests__/mock_data.json"); +const { getHashedText, deleteDB, createDB } = require("../../../__tests__/setup"); const jwt = require("jsonwebtoken"); const token = jwt.sign( { resource_access: { "attendance-and-revenue": { roles: ["sysadmin"] } } }, @@ -18,52 +19,63 @@ const emptyRole = { resource_access: { "attendance-and-revenue": { roles: [""] } }, }; -async function setupDb() { - docClient = new DynamoDBClient({ - region: REGION, - endpoint: ENDPOINT, - convertEmptyValues: true, - }); +async function setupDb(tableName) { for (const item of PARKSLIST) { - await genericPutDocument(item); + await genericPutDocument(item, tableName); } for (const item of SUBAREAS) { - await genericPutDocument(item); + await genericPutDocument(item, tableName); } for (const item of SUBAREA_ENTRIES) { - await genericPutDocument(item); + await genericPutDocument(item, tableName); } for (const item of CONFIG_ENTRIES) { - await genericPutDocument(item); + await genericPutDocument(item, tableName); } for (const item of FISCAL_YEAR_LOCKS) { - await genericPutDocument(item); + await genericPutDocument(item, tableName); } } -async function genericPutDocument(item) { +async function genericPutDocument(item, TABLE_NAME) { + + const dynamoClient = new DynamoDBClient({ + region: REGION, + endpoint: ENDPOINT + }); + const input = { - Item: marshall(item, { convertEmptyValues: true }), + Item: marshall(item), TableName: TABLE_NAME, }; const command = new PutItemCommand(input); - return await docClient.send(command); + return await dynamoClient.send(command); } describe("Activity Test", () => { const OLD_ENV = process.env; + let hash + let TABLE_NAME + let NAME_CACHE_TABLE_NAME + let CONFIG_TABLE_NAME + beforeEach(async () => { jest.resetModules(); process.env = { ...OLD_ENV }; // Make a copy of environment + hash = getHashedText(expect.getState().currentTestName); + process.env.TABLE_NAME = hash + TABLE_NAME = process.env.TABLE_NAME; + NAME_CACHE_TABLE_NAME = TABLE_NAME.concat("-nameCache"); + CONFIG_TABLE_NAME = TABLE_NAME.concat("-config"); + await createDB(TABLE_NAME, NAME_CACHE_TABLE_NAME, CONFIG_TABLE_NAME); + await setupDb(TABLE_NAME); }); afterEach(() => { + deleteDB(TABLE_NAME, NAME_CACHE_TABLE_NAME, CONFIG_TABLE_NAME); process.env = OLD_ENV; // Restore old environment - }); - - beforeAll(async () => { - return await setupDb(); + }); test("Handler - 200 GET specific activity entry", async () => { @@ -172,6 +184,10 @@ describe("Activity Test", () => { }); test("HandlePost - 200 POST handle Activity/Variances", async () => { + const dynamoClient = new DynamoDBClient({ + region: REGION, + endpoint: ENDPOINT + }); const activityPOST = require("../POST/index"); // Setup the first record const response = await activityPOST.handlePost( @@ -216,11 +232,9 @@ describe("Activity Test", () => { TableName: TABLE_NAME, }; const command = new GetItemCommand(input); - const doc = await docClient.send(command); + const doc = await dynamoClient.send(command); expect(doc?.Item).toBe(undefined); - - // Change year and create a new record const secondResponse = await activityPOST.handlePost( { @@ -264,7 +278,7 @@ describe("Activity Test", () => { TableName: TABLE_NAME, }; const command2 = new GetItemCommand(input2); - const doc2 = await docClient.send(command2); + const doc2 = await dynamoClient.send(command2); expect(unmarshall(doc2?.Item)).toEqual({ parkName: 'Cultus Lake Park', orcs: '0041', @@ -344,7 +358,7 @@ describe("Activity Test", () => { // note: CONFIG POST disabled 2022-09-27 - test("HandlePost - 400 POST handle Activity", async () => { + test("HandlePost - 400 POST handle Activity with Body", async () => { const activityPOST = require("../POST/index"); const response = await activityPOST.handlePost( { @@ -433,11 +447,10 @@ describe("Activity Test", () => { }, null ); - expect(response.statusCode).toBe(400); }); - test("HandleLock - 200 POST lock record", async () => { + test("HandleLock/PostToLocked/Unlock - 200-409-200", async () => { const activityPOST = require("../POST/index"); const response = await activityPOST.handleLock( { @@ -461,11 +474,8 @@ describe("Activity Test", () => { null ); expect(response.statusCode).toBe(200); - }); - test("HandlePost - 409 POST to locked record", async () => { - const activityPOST = require("../POST/index"); - const response = await activityPOST.handlePost( + const response2 = await activityPOST.handlePost( { headers: { Authorization: "Bearer " + token, @@ -486,12 +496,10 @@ describe("Activity Test", () => { }, null ); - expect(response.statusCode).toBe(409); - }); + expect(response2.statusCode).toBe(409); - test("HandleUnlock - 200 POST unlock record", async () => { - const activityPOST = require("../POST/index"); - const response = await activityPOST.handleUnlock( + + const response3 = await activityPOST.handleUnlock( { headers: { Authorization: "Bearer " + token, @@ -512,7 +520,7 @@ describe("Activity Test", () => { }, null ); - expect(response.statusCode).toBe(200); + expect(response3.statusCode).toBe(200); }); test("Handler - 403 POST to locked fiscal year", async () => { diff --git a/arSam/handlers/export-variance/GET/index.js b/arSam/handlers/export-variance/GET/index.js index 78e12db..440fea8 100644 --- a/arSam/handlers/export-variance/GET/index.js +++ b/arSam/handlers/export-variance/GET/index.js @@ -1,10 +1,3 @@ -const { Lambda } = require("@aws-sdk/client-lambda"); -const { getSignedUrl } = require("@aws-sdk/s3-request-presigner"); -const { S3Client, GetObjectCommand } = require("@aws-sdk/client-s3"); -const { marshall } = require('@aws-sdk/util-dynamodb'); - -const defaultRegion = process.env.AWS_REGION || "ca-central-1"; -const s3Client = new S3Client({ region: defaultRegion }); const bucket = process.env.S3_BUCKET_DATA || "parks-ar-assets-tools"; const IS_OFFLINE = @@ -17,9 +10,18 @@ if (IS_OFFLINE) { options.endpoint = "http://localhost:3002"; } -const lambda = new Lambda(options); - -const { runQuery, TABLE_NAME, dynamodb, sendResponse, logger } = require("/opt/baseLayer"); +const { runQuery, + TABLE_NAME, + dynamoClient, + PutItemCommand, + marshall, + lambda, + s3Client, + GetObjectCommand, + getSignedUrl, + sendResponse, + logger +} = require("/opt/baseLayer"); const { createHash } = require('node:crypto'); const VARIANCE_EXPORT_FUNCTION_NAME = @@ -151,7 +153,7 @@ exports.handler = async (event, context) => { logger.debug('Creating new job:', varianceExportPutObj); let newJob; try { - newJob = await dynamodb.putItem(varianceExportPutObj); + newJob = await dynamoClient.send(new PutItemCommand(varianceExportPutObj)); // Check if there's already a report being generated. // If there are is no instance of a job or the job is 100% complete, generate a report. logger.debug('New job created:', newJob); diff --git a/arSam/handlers/export-variance/GET/package.json b/arSam/handlers/export-variance/GET/package.json deleted file mode 100644 index 3396848..0000000 --- a/arSam/handlers/export-variance/GET/package.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "name": "varianceExportGetFunction", - "version": "1.0.0", - "author": "Digitalspace ", - "dependencies": { - "@aws-sdk/client-lambda": "^3.568.0", - "@aws-sdk/client-s3": "^3.568.0", - "@aws-sdk/util-dynamodb": "^3.614.0", - "@aws-sdk/s3-request-presigner": "^3.568.0" - } -} diff --git a/arSam/handlers/export-variance/__tests__/export-variance.tests.js b/arSam/handlers/export-variance/__tests__/export-variance.tests.js index f007470..52f6c72 100644 --- a/arSam/handlers/export-variance/__tests__/export-variance.tests.js +++ b/arSam/handlers/export-variance/__tests__/export-variance.tests.js @@ -1,68 +1,69 @@ -const AWS = require("aws-sdk"); -const { DocumentClient } = require("aws-sdk/clients/dynamodb"); -const { REGION, ENDPOINT, TABLE_NAME } = require("../../../__tests__/settings"); +const { DynamoDBClient, PutItemCommand } = require('@aws-sdk/client-dynamodb'); +const { REGION, ENDPOINT } = require("../../../__tests__/settings"); const { PARKSLIST, SUBAREAS, VARIANCE_JOBSLIST, VARIANCE_MOCKJOB } = require("../../../__tests__/mock_data.json"); - +const { getHashedText, deleteDB, createDB } = require("../../../__tests__/setup"); +const { marshall } = require('@aws-sdk/util-dynamodb'); const jwt = require("jsonwebtoken"); const tokenContent = { resource_access: { "attendance-and-revenue": { roles: ["sysadmin"] } }, }; const token = jwt.sign(tokenContent, "defaultSecret"); -async function setupDb() { - new AWS.DynamoDB({ - region: REGION, - endpoint: ENDPOINT, - }); - docClient = new DocumentClient({ +async function setupDb(TABLE_NAME) { + const dynamoClient = new DynamoDBClient({ region: REGION, - endpoint: ENDPOINT, - convertEmptyValues: true, + endpoint: ENDPOINT }); for (const park of PARKSLIST) { - await docClient - .put({ + let params = { TableName: TABLE_NAME, - Item: park, - }) - .promise(); + Item: marshall(park), + } + await dynamoClient.send(new PutItemCommand(params)); } for (const subarea of SUBAREAS) { - await docClient - .put({ + params = { TableName: TABLE_NAME, - Item: subarea, - }) - .promise(); + Item: marshall(subarea), + } + await dynamoClient.send(new PutItemCommand(params)); } for (const job of VARIANCE_JOBSLIST) { - await docClient - .put({ + params = { TableName: TABLE_NAME, - Item: job, - }) - .promise(); + Item: marshall(job), + } + await dynamoClient.send(new PutItemCommand(params)); } } describe("Export Variance Report", () => { const OLD_ENV = process.env; + let hash + let TABLE_NAME + let NAME_CACHE_TABLE_NAME + let CONFIG_TABLE_NAME + beforeEach(async () => { jest.resetModules(); process.env = { ...OLD_ENV }; // Make a copy of environment + hash = getHashedText(expect.getState().currentTestName); + process.env.TABLE_NAME = hash + TABLE_NAME = process.env.TABLE_NAME; + NAME_CACHE_TABLE_NAME = TABLE_NAME.concat("-nameCache"); + CONFIG_TABLE_NAME = TABLE_NAME.concat("-config"); + await createDB(TABLE_NAME, NAME_CACHE_TABLE_NAME, CONFIG_TABLE_NAME); + await setupDb(TABLE_NAME); }); afterEach(() => { + deleteDB(TABLE_NAME, NAME_CACHE_TABLE_NAME, CONFIG_TABLE_NAME); process.env = OLD_ENV; // Restore old environment }); - beforeAll(async () => { - return await setupDb(); - }); - test("Handler - 403 GET Invalid Auth", async () => { const event = { headers: { diff --git a/arSam/handlers/export-variance/invokable/index.js b/arSam/handlers/export-variance/invokable/index.js index efc7009..d8c83ac 100644 --- a/arSam/handlers/export-variance/invokable/index.js +++ b/arSam/handlers/export-variance/invokable/index.js @@ -1,10 +1,15 @@ -const { S3Client, PutObjectCommand } = require('@aws-sdk/client-s3'); -const s3Client = new S3Client({}); -const { marshall } = require('@aws-sdk/util-dynamodb'); const fs = require('fs'); - const { VARIANCE_CSV_SCHEMA, VARIANCE_STATE_DICTIONARY } = require("/opt/constantsLayer"); -const { getParks, TABLE_NAME, dynamodb, runQuery, logger } = require("/opt/baseLayer"); +const { getParks, + TABLE_NAME, + dynamoClient, + PutItemCommand, + marshall, + s3Client, + PutObjectCommand, + runQuery, + logger +} = require("/opt/baseLayer"); const FILE_PATH = process.env.FILE_PATH || "/tmp/"; const FILE_NAME = process.env.FILE_NAME || "A&R_Variance_Report"; @@ -139,7 +144,7 @@ async function updateJobEntry(jobObj) { TableName: TABLE_NAME, Item: marshall(jobObj) } - await dynamodb.putItem(putObj); + await dynamoClient.send(new PutItemCommand(putObj)); } async function getVarianceRecords(fiscalYearEnd, roles) { diff --git a/arSam/handlers/export-variance/invokable/package.json b/arSam/handlers/export-variance/invokable/package.json deleted file mode 100644 index 30be9ac..0000000 --- a/arSam/handlers/export-variance/invokable/package.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "name": "varianceExportInvokableFunction", - "version": "1.0.0", - "author": "Digitalspace ", - "dependencies": { - "@aws-sdk/client-s3": "^3.568.0", - "@aws-sdk/util-dynamodb": "^3.614.0" - } -} diff --git a/arSam/handlers/export/GET/index.js b/arSam/handlers/export/GET/index.js index faeb4d0..b38c19d 100644 --- a/arSam/handlers/export/GET/index.js +++ b/arSam/handlers/export/GET/index.js @@ -1,10 +1,4 @@ -const { Lambda } = require("@aws-sdk/client-lambda"); -const { getSignedUrl } = require("@aws-sdk/s3-request-presigner"); -const { S3Client, GetObjectCommand } = require("@aws-sdk/client-s3"); -const { marshall } = require('@aws-sdk/util-dynamodb'); -const defaultRegion = process.env.AWS_REGION || "ca-central-1"; -const s3Client = new S3Client({ region: defaultRegion }); const bucket = process.env.S3_BUCKET_DATA || "parks-ar-assets-tools"; const IS_OFFLINE = @@ -16,13 +10,22 @@ if (IS_OFFLINE) { // For local we use port 3002 because we're hitting an invokable options.endpoint = "http://localhost:3002"; } -const lambda = new Lambda(options); -const { runQuery, dynamodb, TABLE_NAME, sendResponse, logger } = require("/opt/baseLayer"); +const { runQuery, + dynamoClient, + PutItemCommand, + GetObjectCommand, + getSignedUrl, + s3Client, + marshall, + lambda, + TABLE_NAME, + sendResponse, + logger +} = require("/opt/baseLayer"); const { convertRolesToMD5 } = require("/opt/functionsLayer"); -const EXPORT_FUNCTION_NAME = - process.env.EXPORT_FUNCTION_NAME || "ar-api-ExportInvokableFunction"; +const EXPORT_FUNCTION_NAME = process.env.EXPORT_FUNCTION_NAME || "ar-api-ExportInvokableFunction"; const EXPIRY_TIME = process.env.EXPORT_EXPIRY_TIME ? Number(process.env.EXPORT_EXPIRY_TIME) @@ -134,7 +137,7 @@ exports.handler = async (event, context) => { logger.debug(putObject); let newJob; try { - newJob = await dynamodb.putItem(putObject); + newJob = await dynamoClient.send(new PutItemCommand(putObject)); // Check if there's already a report being generated. // If there are is no instance of a job or the job is 100% complete, generate a report. logger.debug("Creating a new export job.", newJob); diff --git a/arSam/handlers/export/GET/package.json b/arSam/handlers/export/GET/package.json deleted file mode 100644 index 9f2c2d8..0000000 --- a/arSam/handlers/export/GET/package.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "name": "exportGetFunction", - "version": "1.0.0", - "author": "Digitalspace ", - "dependencies": { - "@aws-sdk/client-lambda": "^3.568.0", - "@aws-sdk/client-s3": "^3.568.0", - "@aws-sdk/util-dynamodb": "^3.614.0", - "@aws-sdk/s3-request-presigner": "^3.568.0" - } -} diff --git a/arSam/handlers/export/__tests__/export.test.js b/arSam/handlers/export/__tests__/export.test.js index 3bb9bf5..d3ba53d 100644 --- a/arSam/handlers/export/__tests__/export.test.js +++ b/arSam/handlers/export/__tests__/export.test.js @@ -1,7 +1,8 @@ -const AWS = require("aws-sdk"); -const { DocumentClient } = require("aws-sdk/clients/dynamodb"); -const { REGION, ENDPOINT, TABLE_NAME } = require("../../../__tests__/settings"); +const { DynamoDBClient, PutItemCommand } = require('@aws-sdk/client-dynamodb'); +const { REGION, ENDPOINT } = require("../../../__tests__/settings"); const { PARKSLIST, SUBAREAS, JOBSLIST, MOCKJOB } = require("../../../__tests__/mock_data.json"); +const { getHashedText, deleteDB, createDB } = require("../../../__tests__/setup"); +const { marshall } = require('@aws-sdk/util-dynamodb'); const jwt = require("jsonwebtoken"); const tokenContent = { @@ -9,60 +10,61 @@ const tokenContent = { }; const token = jwt.sign(tokenContent, "defaultSecret"); -async function setupDb() { - new AWS.DynamoDB({ +async function setupDb(tableName) { + const dynamoClient = new DynamoDBClient({ region: REGION, - endpoint: ENDPOINT, - }); - docClient = new DocumentClient({ - region: REGION, - endpoint: ENDPOINT, - convertEmptyValues: true, + endpoint: ENDPOINT }); for (const park of PARKSLIST) { - await docClient - .put({ - TableName: TABLE_NAME, - Item: park, - }) - .promise(); + let params = { + TableName: tableName, + Item: marshall(park), + } + await dynamoClient.send(new PutItemCommand(params)) } for (const subarea of SUBAREAS) { - await docClient - .put({ - TableName: TABLE_NAME, - Item: subarea, - }) - .promise(); - } + let params = { + TableName: tableName, + Item: marshall(subarea), + } + await dynamoClient.send(new PutItemCommand(params)) + } for (const job of JOBSLIST) { - await docClient - .put({ - TableName: TABLE_NAME, - Item: job, - }) - .promise(); + let params = { + TableName: tableName, + Item: marshall(job), + } + await dynamoClient.send(new PutItemCommand(params)) } } describe("Export Report", () => { const OLD_ENV = process.env; + let hash + let TABLE_NAME + let NAME_CACHE_TABLE_NAME + let CONFIG_TABLE_NAME + beforeEach(async () => { jest.resetModules(); process.env = { ...OLD_ENV }; // Make a copy of environment + hash = getHashedText(expect.getState().currentTestName); + process.env.TABLE_NAME = hash + TABLE_NAME = process.env.TABLE_NAME; + NAME_CACHE_TABLE_NAME = TABLE_NAME.concat("-nameCache"); + CONFIG_TABLE_NAME = TABLE_NAME.concat("-config"); + await createDB(TABLE_NAME, NAME_CACHE_TABLE_NAME, CONFIG_TABLE_NAME); + await setupDb(TABLE_NAME); }); afterEach(() => { + deleteDB(TABLE_NAME, NAME_CACHE_TABLE_NAME, CONFIG_TABLE_NAME); process.env = OLD_ENV; // Restore old environment }); - - beforeAll(async () => { - return await setupDb(); - }); - + test("Handler - 403 GET Invalid Auth", async () => { const event = { headers: { @@ -108,14 +110,15 @@ describe("Export Report", () => { }; const exportGET = require("../GET/index"); - const result = await exportGET.handler(event, null) + const result = await exportGET.handler(event, null); let body; try { body = JSON.parse(result.body) } catch (e) { + console.log("In this dumb catch") body = 'fail' } - + expect(result).toEqual( expect.objectContaining({ headers: { @@ -128,7 +131,7 @@ describe("Export Report", () => { statusCode: 200, }), ); - expect(body.jobObj[dateField]).toMatch(JOBSLIST[0][dateField]) + expect(body.jobObj[dateField]).toMatch(JOBSLIST[0][dateField]) }) test("Handler - 200 GET, generate report", async () => { diff --git a/arSam/handlers/export/invokable/index.js b/arSam/handlers/export/invokable/index.js index dbcfb90..8e04578 100644 --- a/arSam/handlers/export/invokable/index.js +++ b/arSam/handlers/export/invokable/index.js @@ -1,5 +1,3 @@ -const { S3Client, PutObjectCommand } = require("@aws-sdk/client-s3"); -const s3Client = new S3Client({}); const fs = require("fs"); const writeXlsxFile = require("write-excel-file/node"); const { @@ -8,6 +6,8 @@ const { getSubAreas, getRecords, logger, + s3Client, + PutObjectCommand } = require("/opt/baseLayer"); const { EXPORT_NOTE_KEYS, diff --git a/arSam/handlers/export/invokable/package.json b/arSam/handlers/export/invokable/package.json index 74d8990..81419b2 100644 --- a/arSam/handlers/export/invokable/package.json +++ b/arSam/handlers/export/invokable/package.json @@ -3,7 +3,6 @@ "version": "1.0.0", "author": "Digitalspace ", "dependencies": { - "@aws-sdk/client-s3": "^3.568.0", "write-excel-file": "^1.3.16" } } diff --git a/arSam/handlers/fiscalYearEnd/POST/index.js b/arSam/handlers/fiscalYearEnd/POST/index.js index b343429..9d6d6ee 100644 --- a/arSam/handlers/fiscalYearEnd/POST/index.js +++ b/arSam/handlers/fiscalYearEnd/POST/index.js @@ -1,11 +1,12 @@ -const { marshall } = require('@aws-sdk/util-dynamodb'); const { TABLE_NAME, - dynamodb, + dynamoClient, + PutItemCommand, TIMEZONE, FISCAL_YEAR_FINAL_MONTH, sendResponse, - logger + logger, + marshall } = require("/opt/baseLayer"); const { DateTime } = require("luxon"); @@ -93,7 +94,7 @@ async function putFiscalYear(isLocked, params) { TableName: TABLE_NAME, Item: marshall(newObject), }; - await dynamodb.putItem(putObj); + await dynamoClient.send(new PutItemCommand(putObj)); logger.debug("Updated fiscalYearEnd Object:", newObject); return newObject; } catch (err) { diff --git a/arSam/handlers/fiscalYearEnd/POST/package.json b/arSam/handlers/fiscalYearEnd/POST/package.json index c9bf047..d55698d 100644 --- a/arSam/handlers/fiscalYearEnd/POST/package.json +++ b/arSam/handlers/fiscalYearEnd/POST/package.json @@ -3,7 +3,6 @@ "version": "1.0.0", "author": "Digitalspace ", "dependencies": { - "@aws-sdk/util-dynamodb": "^3.614.0", "luxon": "^3.2.1" } } diff --git a/arSam/handlers/fiscalYearEnd/__tests__/fiscalYearEnd.test.js b/arSam/handlers/fiscalYearEnd/__tests__/fiscalYearEnd.test.js index 80e85e1..b7b5bfb 100644 --- a/arSam/handlers/fiscalYearEnd/__tests__/fiscalYearEnd.test.js +++ b/arSam/handlers/fiscalYearEnd/__tests__/fiscalYearEnd.test.js @@ -1,51 +1,57 @@ -const AWS = require('aws-sdk'); -const { DocumentClient } = require('aws-sdk/clients/dynamodb'); -const { REGION, ENDPOINT, TABLE_NAME } = require('../../../__tests__/settings'); +const { DynamoDBClient, PutItemCommand } = require('@aws-sdk/client-dynamodb'); +const { marshall } = require('@aws-sdk/util-dynamodb'); +const { getHashedText, deleteDB, createDB } = require("../../../__tests__/setup"); +const { REGION, ENDPOINT } = require('../../../__tests__/settings'); const { FISCAL_YEAR_LOCKS2 } = require('../../../__tests__/mock_data.json'); const jwt = require('jsonwebtoken'); const token = jwt.sign({ resource_access: { 'attendance-and-revenue': { roles: ['sysadmin'] } } }, 'defaultSecret'); -async function setupDb() { - new AWS.DynamoDB({ - region: REGION, - endpoint: ENDPOINT - }); - docClient = new DocumentClient({ - region: REGION, - endpoint: ENDPOINT, - convertEmptyValues: true - }); - +async function setupDb(TABLE_NAME) { + for (const item of FISCAL_YEAR_LOCKS2) { - await (genericPutDocument(item)); + await (genericPutDocument(item, TABLE_NAME)); } } -async function genericPutDocument(item) { - return await docClient - .put({ +async function genericPutDocument(item, TABLE_NAME) { + const dynamoClient = new DynamoDBClient({ + region: REGION, + endpoint: ENDPOINT + }); + + const params = { TableName: TABLE_NAME, - Item: item - }) - .promise(); + Item: marshall(item) + } + await dynamoClient.send(new PutItemCommand(params)); + } describe('Fiscal Year End Test', () => { const OLD_ENV = process.env; + let hash + let TABLE_NAME + let NAME_CACHE_TABLE_NAME + let CONFIG_TABLE_NAME + beforeEach(async () => { jest.resetModules(); process.env = { ...OLD_ENV }; // Make a copy of environment + hash = getHashedText(expect.getState().currentTestName); + process.env.TABLE_NAME = hash + TABLE_NAME = process.env.TABLE_NAME; + NAME_CACHE_TABLE_NAME = TABLE_NAME.concat("-nameCache"); + CONFIG_TABLE_NAME = TABLE_NAME.concat("-config"); + await createDB(TABLE_NAME, NAME_CACHE_TABLE_NAME, CONFIG_TABLE_NAME); + await setupDb(TABLE_NAME); }); afterEach(() => { + deleteDB(TABLE_NAME, NAME_CACHE_TABLE_NAME, CONFIG_TABLE_NAME); process.env = OLD_ENV; // Restore old environment }); - beforeAll(async () => { - return await setupDb(); - }); - test('Handler - 200 GET fiscal year end', async () => { const fiscalYearEndGET = require('../GET/index'); const obj = await fiscalYearEndGET.handler( diff --git a/arSam/handlers/nameUpdate/__tests__/name-update.test.js b/arSam/handlers/nameUpdate/__tests__/name-update.test.js index ff8a1e9..aa0895f 100644 --- a/arSam/handlers/nameUpdate/__tests__/name-update.test.js +++ b/arSam/handlers/nameUpdate/__tests__/name-update.test.js @@ -1,17 +1,18 @@ -const AWS = require("aws-sdk"); -const { DocumentClient } = require("aws-sdk/clients/dynamodb"); -const { REGION, ENDPOINT, TABLE_NAME, NAME_CACHE_TABLE_NAME } = require("../../../__tests__/settings"); -const docClient = new DocumentClient({ - region: REGION, - endpoint: ENDPOINT, - convertEmptyValues: true, -}); -async function setupDb() { +const { DynamoDBClient, PutItemCommand, GetItemCommand, UpdateItemCommand } = require('@aws-sdk/client-dynamodb'); +const { REGION, ENDPOINT } = require("../../../__tests__/settings"); +const { getHashedText, deleteDB, createDB } = require("../../../__tests__/setup"); +const { marshall, unmarshall } = require('@aws-sdk/util-dynamodb'); + + +async function setupDb(TABLE_NAME) { // Insert a document for the handler to now find and update. - await docClient - .put({ + const dynamoClient = new DynamoDBClient({ + region: REGION, + endpoint: ENDPOINT + }); + const params = { TableName: TABLE_NAME, - Item: { + Item: marshall({ "pk": "0673::Backcountry Cabins", "sk": "201702", "activity": "Backcountry Cabins", @@ -26,29 +27,47 @@ async function setupDb() { "orcs": "0001", "parkName": "Strathcona Park", "subAreaId": "0673" - } - }) - .promise(); + }) + } + await dynamoClient.send(new PutItemCommand(params)); } describe("Name Update Tests", () => { const OLD_ENV = process.env; + let hash + let TABLE_NAME + let NAME_CACHE_TABLE_NAME + let CONFIG_TABLE_NAME + beforeEach(async () => { jest.resetModules(); process.env = { ...OLD_ENV }; // Make a copy of environment + hash = getHashedText(expect.getState().currentTestName); + process.env.TABLE_NAME = hash + TABLE_NAME = process.env.TABLE_NAME; + NAME_CACHE_TABLE_NAME = TABLE_NAME.concat("-nameCache"); + CONFIG_TABLE_NAME = TABLE_NAME.concat("-config"); + process.env.NAME_CACHE_TABLE_NAME = NAME_CACHE_TABLE_NAME; + process.env.TABLE_NAME = TABLE_NAME; + process.env.CONFIG_TABLE_NAME = CONFIG_TABLE_NAME; + await createDB(TABLE_NAME, NAME_CACHE_TABLE_NAME, CONFIG_TABLE_NAME); + await setupDb(TABLE_NAME); }); afterEach(() => { + deleteDB(TABLE_NAME, NAME_CACHE_TABLE_NAME, CONFIG_TABLE_NAME); process.env = OLD_ENV; // Restore old environment }); - beforeAll(async () => { - return await setupDb(); - }); - test("updateLocalCache", async () => { const axios = require('axios'); jest.mock("axios"); + + const dynamoClient = new DynamoDBClient({ + region: REGION, + endpoint: ENDPOINT + }); + axios.get.mockImplementation( () => Promise.resolve({ statusCode: 200, @@ -77,74 +96,82 @@ describe("Name Update Tests", () => { // Cached document keys const CACHED_DOCUMENT = { TableName: NAME_CACHE_TABLE_NAME, - Key: { + Key: marshall({ pk: "1", - }, + }), }; // AR document key - const AR_DOCUMENT_KEY = { + const AR_DOCUMENT_KEY = marshall({ pk: "0673::Backcountry Cabins", sk: "201702" - }; + }); // Ensure this doesn't exist yet. - const notFoundDoc = await docClient.get(CACHED_DOCUMENT).promise(); - expect(notFoundDoc).toStrictEqual({}); + const notFoundDoc = await dynamoClient.send(new GetItemCommand(CACHED_DOCUMENT)); + const responseItem = notFoundDoc.Item; + expect(responseItem).toBeUndefined(); // Call the handler, it will have cache-miss await nameUpdateHandler.handler({}, null); // Expect the cache to be updated. - const doc = await docClient.get(CACHED_DOCUMENT).promise(); - expect(doc.Item.pk).toBe("1"); + const docRes = await dynamoClient.send(new GetItemCommand(CACHED_DOCUMENT)); + const doc = unmarshall(docRes.Item); + expect(doc.pk).toBe("1"); // Change the last cached item to be different in order to trigger a displayName // change on the handler. const params = { TableName: NAME_CACHE_TABLE_NAME, - Key: { pk: '1' }, + Key: marshall({ pk: '1' }), UpdateExpression: 'set displayName =:displayName', - ExpressionAttributeValues: { + ExpressionAttributeValues: marshall({ ':displayName': 'some other park name' - } + }) }; - await docClient.update(params).promise(); - const cachedDocumentSet = await docClient.get(CACHED_DOCUMENT).promise(); - expect(cachedDocumentSet.Item.displayName).toBe('some other park name'); + + await dynamoClient.send(new UpdateItemCommand(params)); + const cachedDocumentSetRes = await dynamoClient.send(new GetItemCommand(CACHED_DOCUMENT)); + const cachedDocumentSet = unmarshall(cachedDocumentSetRes.Item); + expect(cachedDocumentSet.displayName).toBe('some other park name'); // Also update the backcountry cabin record in the main table const params2 = { TableName: TABLE_NAME, Key: AR_DOCUMENT_KEY, UpdateExpression: 'set parkName =:parkName', - ExpressionAttributeValues: { + ExpressionAttributeValues: marshall({ ':parkName': 'some other park name' - } + }) }; - await docClient.update(params2).promise(); - const arDocumentSetParkName = await docClient.get({ - TableName: TABLE_NAME, - Key: AR_DOCUMENT_KEY, - }).promise(); + await dynamoClient.send(new UpdateItemCommand(params2)); - expect(arDocumentSetParkName.Item.parkName).toBe('some other park name'); + const params3 = { + TableName: TABLE_NAME, + Key: AR_DOCUMENT_KEY, + } + const arDocumentSetParkNameRes = await dynamoClient.send(new GetItemCommand(params3)); + const arDocumentSetParkName = unmarshall(arDocumentSetParkNameRes.Item); + expect(arDocumentSetParkName.parkName).toBe('some other park name'); // Run the update await nameUpdateHandler.handler({}, null); - // Fetch the updated cache and check that it has been udpated - const cachedDocument = await docClient.get(CACHED_DOCUMENT).promise(); - - // Ensure it was updated - expect(cachedDocument.Item.displayName).toBe('Strathcona Park'); + // Fetch the updated cache and check that it has been updated + const cachedDocumentRes = await dynamoClient.send(new GetItemCommand(CACHED_DOCUMENT)); + const cachedDocument = unmarshall(cachedDocumentRes.Item); + expect(cachedDocument.displayName).toBe('Strathcona Park'); // Fetch the updated AR document and check that it has been udpated - const arDocument = await docClient.get({ - TableName: TABLE_NAME, - Key: AR_DOCUMENT_KEY, - }).promise(); - expect(arDocument.Item.parkName).toBe('Strathcona Park'); + const fetchParams = { + TableName: TABLE_NAME, + Key: AR_DOCUMENT_KEY, + } + + const arDocumentRes = await dynamoClient.send(new GetItemCommand(fetchParams)); + const arDocument = unmarshall(arDocumentRes.Item); + expect(arDocument.parkName).toBe('Strathcona Park'); }); }); diff --git a/arSam/handlers/nameUpdate/index.js b/arSam/handlers/nameUpdate/index.js index 3c6a4a6..16b106e 100644 --- a/arSam/handlers/nameUpdate/index.js +++ b/arSam/handlers/nameUpdate/index.js @@ -1,6 +1,16 @@ const axios = require('axios'); -const { marshall } = require('@aws-sdk/util-dynamodb'); -const { runQuery, runScan, NAME_CACHE_TABLE_NAME, TABLE_NAME, ORCS_INDEX, dynamodb, logger } = require("/opt/baseLayer"); +const { runQuery, + runScan, + NAME_CACHE_TABLE_NAME, + TABLE_NAME, + ORCS_INDEX, + dynamoClient, + PutItemCommand, + BatchWriteItemCommand, + UpdateItemCommand, + marshall, + logger +} = require("/opt/baseLayer"); const DATA_REGISTER_NAME_ENDPOINT = process.env.DATA_REGISTER_NAME_ENDPOINT || 'https://zloys5cfvf.execute-api.ca-central-1.amazonaws.com/api/parks/names?status=established'; const DATA_REGISTER_NAME_API_KEY = process.env.DATA_REGISTER_NAME_API_KEY; const ESTABLISHED_STATE = 'established'; @@ -82,7 +92,7 @@ async function updateLocalCache(item) { TableName: NAME_CACHE_TABLE_NAME, Item: marshall(item, { removeUndefinedValues: true }) }; - await dynamodb.putItem(putItem); + await dynamoClient.send(new PutItemCommand(putItem)); logger.info("Update complete") } @@ -104,7 +114,7 @@ async function batchWriteCache(records) { if (batch.RequestItems[NAME_CACHE_TABLE_NAME].length === 25) { batchCount++; // Write the current batch and reset the batch - await dynamodb.batchWriteItem(batch); + await dynamoClient.send(new BatchWriteItemCommand(batch)); process.stdout.write(`.`); batch.RequestItems[NAME_CACHE_TABLE_NAME] = []; } @@ -114,7 +124,7 @@ async function batchWriteCache(records) { if (batch.RequestItems[NAME_CACHE_TABLE_NAME].length > 0) { batchCount++; logger.info(`writing final batch #${batchCount}`); - await dynamodb.batchWriteItem(batch); + await dynamoClient.send(new BatchWriteItemCommand(batch)); logger.info(`Complete.`); } } @@ -161,7 +171,7 @@ async function updateRecords(recordsToUpdate, updateObj) { try { process.stdout.write(`.`); - await dynamodb.updateItem(params); + await dynamoClient.send(new UpdateItemCommand(params)); } catch (e) { logger.info(e); // TODO: Fall through, but record error somehow? diff --git a/arSam/handlers/nameUpdate/package.json b/arSam/handlers/nameUpdate/package.json index efe8065..0f0ba16 100644 --- a/arSam/handlers/nameUpdate/package.json +++ b/arSam/handlers/nameUpdate/package.json @@ -3,8 +3,7 @@ "version": "1.0.0", "author": "Digitalspace ", "dependencies": { - "axios": "^0.21.1", - "@aws-sdk/util-dynamodb": "^3.614.0" + "axios": "^0.21.1" } } \ No newline at end of file diff --git a/arSam/handlers/park/POST/index.js b/arSam/handlers/park/POST/index.js index 030ed34..c1bc697 100644 --- a/arSam/handlers/park/POST/index.js +++ b/arSam/handlers/park/POST/index.js @@ -1,4 +1,9 @@ -const { dynamodb, TABLE_NAME, logger, sendResponse } = require("/opt/baseLayer"); +const { dynamoClient, + PutItemCommand, + TABLE_NAME, + logger, + sendResponse +} = require("/opt/baseLayer"); exports.handler = async (event, context) => { logger.debug("Park POST:", event); @@ -38,7 +43,7 @@ exports.handler = async (event, context) => { }; logger.debug("Creating park:", postObj); - const res = await dynamodb.putItem(postObj); + const res = await dynamoClient.send(new PutItemCommand(postObj)); logger.info("Park Created"); logger.debug("Result:", res); return sendResponse(200, res); diff --git a/arSam/handlers/park/__tests__/park.test.js b/arSam/handlers/park/__tests__/park.test.js index cc02c94..b3422b0 100644 --- a/arSam/handlers/park/__tests__/park.test.js +++ b/arSam/handlers/park/__tests__/park.test.js @@ -1,7 +1,8 @@ -const AWS = require("aws-sdk"); -const { DocumentClient } = require("aws-sdk/clients/dynamodb"); -const { REGION, ENDPOINT, TABLE_NAME } = require("../../../__tests__/settings"); +const { DynamoDBClient, PutItemCommand } = require('@aws-sdk/client-dynamodb');const { DocumentClient } = require("aws-sdk/clients/dynamodb"); +const { REGION, ENDPOINT } = require("../../../__tests__/settings"); const { PARKSLIST, SUBAREAS } = require("../../../__tests__/mock_data.json"); +const { getHashedText, deleteDB, createDB } = require("../../../__tests__/setup"); +const { marshall } = require('@aws-sdk/util-dynamodb'); const jwt = require("jsonwebtoken"); const tokenContent = { @@ -9,34 +10,27 @@ const tokenContent = { }; const token = jwt.sign(tokenContent, "defaultSecret"); -async function setupDb() { - new AWS.DynamoDB({ +async function setupDb(TABLE_NAME) { + const dynamoClient = new DynamoDBClient({ region: REGION, - endpoint: ENDPOINT, - }); - docClient = new DocumentClient({ - region: REGION, - endpoint: ENDPOINT, - convertEmptyValues: true, + endpoint: ENDPOINT }); for (const park of PARKSLIST) { - await docClient - .put({ + const params = { TableName: TABLE_NAME, - Item: park, - }) - .promise(); + Item: marshall(park) + } + await dynamoClient.send(new PutItemCommand(params)) } for (const subarea of SUBAREAS) { - await docClient - .put({ + const params = { TableName: TABLE_NAME, - Item: subarea, - }) - .promise(); - } + Item: marshall(subarea) + } + await dynamoClient.send(new PutItemCommand(params)) + } } describe("Park Test", () => { @@ -50,18 +44,28 @@ describe("Park Test", () => { }; const OLD_ENV = process.env; + let hash + let TABLE_NAME + let NAME_CACHE_TABLE_NAME + let CONFIG_TABLE_NAME + beforeEach(async () => { jest.resetModules(); process.env = { ...OLD_ENV }; // Make a copy of environment + hash = getHashedText(expect.getState().currentTestName); + process.env.TABLE_NAME = hash + TABLE_NAME = process.env.TABLE_NAME; + NAME_CACHE_TABLE_NAME = TABLE_NAME.concat("-nameCache"); + CONFIG_TABLE_NAME = TABLE_NAME.concat("-config"); + await createDB(TABLE_NAME, NAME_CACHE_TABLE_NAME, CONFIG_TABLE_NAME); + await setupDb(TABLE_NAME); }); afterEach(() => { + deleteDB(TABLE_NAME, NAME_CACHE_TABLE_NAME, CONFIG_TABLE_NAME); process.env = OLD_ENV; // Restore old environment }); - beforeAll(async () => { - return await setupDb(); - }); test("Handler - 200 Received list of parks", async () => { const event = { diff --git a/arSam/handlers/subArea/DELETE/index.js b/arSam/handlers/subArea/DELETE/index.js index e8056c6..4301130 100644 --- a/arSam/handlers/subArea/DELETE/index.js +++ b/arSam/handlers/subArea/DELETE/index.js @@ -1,6 +1,14 @@ -const { marshall } = require('@aws-sdk/util-dynamodb'); const { requirePermissions } = require("/opt/permissionLayer"); -const { getOne, TABLE_NAME, dynamodb, runQuery, sendResponse, logger } = require("/opt/baseLayer"); +const { getOne, + TABLE_NAME, + dynamoClient, + UpdateItemCommand, + DeleteItemCommand, + marshall, + runQuery, + sendResponse, + logger +} = require("/opt/baseLayer"); exports.handler = async (event, context) => { logger.info("SubArea delete"); @@ -13,7 +21,6 @@ exports.handler = async (event, context) => { ) { return sendResponse(400, { msg: "Bad Request." }, context); } - // Check if the user is authenticated and has admin permissions. try { await requirePermissions(event, { "isAuthenticated": true, "isAdmin": true }); @@ -21,7 +28,6 @@ exports.handler = async (event, context) => { logger.error(e); return sendResponse(e.statusCode || 400, e.msg, context); } - // Check if query string has archive flag set to true try { if (event.queryStringParameters.archive === "true") { @@ -41,13 +47,12 @@ exports.handler = async (event, context) => { async function deleteSubArea(subAreaId, orcs, context) { // Update the park object. await deleteSubAreaFromPark(subAreaId, orcs, context); - // Remove subarea records. const activitiesSet = await deleteSubAreaRecords(subAreaId, orcs, context); + // Remove activity records. let activities = [...activitiesSet]; logger.info("activities:", activities); - - // Remove activity records. + if (activities.length > 0) { await deleteActivityRecords(subAreaId, activities, context); } @@ -67,7 +72,7 @@ async function deleteActivityRecords(subAreaId, activities, context) { } }; logger.info("Deleting activity config:", params); - const response = await dynamodb.deleteItem(params); + const response = await dynamoClient.send(new DeleteItemCommand(params)); logger.info("Response:", response); } @@ -102,7 +107,7 @@ async function deleteActivityRecord(pk, sk) { logger.info("Deleting activity record:", params); - const response = await dynamodb.deleteItem(params); + const response = await dynamoClient.send(new DeleteItemCommand(params)); logger.info("Response:", response); return response; @@ -119,7 +124,7 @@ async function deleteSubAreaRecords(subAreaId, orcs, context) { ReturnValues: 'ALL_OLD' }; logger.info("Deleting subArea records:", params); - const response = await dynamodb.deleteItem(params); + const response = await dynamoClient.send(new DeleteItemCommand(params)); logger.info("Activities deleted:", response.Attributes?.activities.SS); return response.Attributes?.activities.SS; } @@ -140,7 +145,7 @@ async function archiveSubAreaRecord(subAreaId, orcs, context) { ReturnValues: 'ALL_NEW' }; logger.info("Archiving subArea records:", params); - const response = await dynamodb.updateItem(params); + const response = await dynamoClient.send(new UpdateItemCommand(params)); logger.info("response:", response); return response; } @@ -173,7 +178,7 @@ async function deleteSubAreaFromPark(subAreaId, orcs, context) { ReturnValues: 'ALL_NEW' }; - const response = await dynamodb.updateItem(updateParkObject); + const response = await dynamoClient.send(new UpdateItemCommand(updateParkObject)); logger.info("Park Object after update:", response.Attributes); return response.Attributes; } @@ -183,7 +188,6 @@ async function archiveSubArea(subAreaId, orcs, context) { logger.info("Removing subarea from park"); await deleteSubAreaFromPark(subAreaId, orcs, context); logger.info("Removed. Archiving Subarea."); - // Go throught the subarea records and flag them as archived. await archiveSubAreaRecord(subAreaId, orcs, context); logger.info("Archived."); diff --git a/arSam/handlers/subArea/DELETE/package.json b/arSam/handlers/subArea/DELETE/package.json deleted file mode 100644 index 684e735..0000000 --- a/arSam/handlers/subArea/DELETE/package.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "name": "subAreaDeleteFunction", - "version": "1.0.0", - "author": "Digitalspace ", - "dependencies": { - "@aws-sdk/util-dynamodb": "^3.614.0" - } -} - \ No newline at end of file diff --git a/arSam/handlers/subArea/POST/index.js b/arSam/handlers/subArea/POST/index.js index 8b2e3b7..d9f453e 100644 --- a/arSam/handlers/subArea/POST/index.js +++ b/arSam/handlers/subArea/POST/index.js @@ -1,5 +1,6 @@ const { - dynamodb, + dynamoClient, + TransactWriteItemsCommand, incrementAndGetNextSubAreaID, getOne, logger, @@ -92,7 +93,7 @@ exports.handler = async (event, context) => { }); } - const res = await dynamodb.transactWriteItems(transactionObj); + const res = await dynamoClient.send(new TransactWriteItemsCommand(transactionObj)); logger.debug('res:', res); // Add Keycloak role diff --git a/arSam/handlers/subArea/__tests__/subArea.test.js b/arSam/handlers/subArea/__tests__/subArea.test.js index 0cd99f7..de3b661 100644 --- a/arSam/handlers/subArea/__tests__/subArea.test.js +++ b/arSam/handlers/subArea/__tests__/subArea.test.js @@ -1,12 +1,8 @@ -const AWS = require("aws-sdk"); -const { DocumentClient } = require("aws-sdk/clients/dynamodb"); -const { - REGION, - ENDPOINT, - TABLE_NAME, - CONFIG_TABLE_NAME, -} = require("../../../__tests__/settings"); +const { DynamoDBClient, PutItemCommand, GetItemCommand } = require('@aws-sdk/client-dynamodb'); +const { REGION, ENDPOINT } = require("../../../__tests__/settings"); const { PARKSLIST } = require("../../../__tests__/mock_data.json"); +const { getHashedText, deleteDB, createDB } = require("../../../__tests__/setup"); +const { marshall, unmarshall } = require('@aws-sdk/util-dynamodb'); const jwt = require("jsonwebtoken"); const tokenContent = { @@ -14,93 +10,80 @@ const tokenContent = { }; const token = jwt.sign(tokenContent, "defaultSecret"); -const suffix = "-subAreaTest"; + const testParkList = []; -async function setupDb() { - new AWS.DynamoDB({ - region: REGION, - endpoint: ENDPOINT, - }); - docClient = new DocumentClient({ +async function setupDb(TABLE_NAME) { + const dynamoClient = new DynamoDBClient({ region: REGION, - endpoint: ENDPOINT, - convertEmptyValues: true, + endpoint: ENDPOINT }); for await (let park of PARKSLIST) { - park.sk = park.sk + suffix; - park.orcs = park.orcs + suffix; + park.sk = park.sk; + park.orcs = park.orcs; subAreaParkIdToDelete = park; let modifiedSubAreas = []; for await (let subArea of park.subAreas) { - subArea.id = subArea.id + suffix; + subArea.id = subArea.id; subAreaToDelete = subArea; modifiedSubAreas.push(subArea); // Add the sub area record - console.log("subarea record:", { - pk: `park::${park.orcs}`, - sk: `${subArea.id}`, - activities: docClient.createSet([ - 'Day Use' - ]) - }); - await docClient - .put({ - TableName: TABLE_NAME, - Item: { - pk: `park::${park.orcs}`, - sk: `${subArea.id}`, - activities: docClient.createSet([ - 'Day Use' - ]) - } - }) - .promise(); + // console.log("subarea record:", { + // pk: `park::${park.orcs}`, + // sk: `${subArea.id}`, + // activities: { SS : ['Day Use'] } + // }); + let params1 = { + TableName: TABLE_NAME, + Item: { + pk: {S: `park::${park.orcs}`}, + sk: {S: `${subArea.id}`}, + activities: { SS : ['Day Use'] } + } + }; + await dynamoClient.send(new PutItemCommand(params1)) // Add the activity config - await docClient - .put({ - TableName: TABLE_NAME, - Item: { - pk: `config::${subArea.id}`, - sk: `Day Use` - } + let params2 = { + TableName: TABLE_NAME, + Item: marshall({ + pk: `config::${subArea.id}`, + sk: `Day Use` }) - .promise(); + }; + await dynamoClient.send(new PutItemCommand(params2)) - console.log("activity config", { - pk: `config::${subArea.id}`, - sk: `Day Use` - }) + // console.log("activity config", { + // pk: `config::${subArea.id}`, + // sk: `Day Use` + // }) // Add the activity record - await docClient - .put({ - TableName: TABLE_NAME, - Item: { - pk: `${subArea.id}::Day Use`, - sk: `202201` - } + let params3 = { + TableName: TABLE_NAME, + Item: marshall({ + pk: `${subArea.id}::Day Use`, + sk: `202201` }) - .promise(); + }; + await dynamoClient.send(new PutItemCommand(params3)) - console.log("activity record", { - pk: `${subArea.id}::Day Use`, - sk: `202201` - }) + // console.log("activity record", { + // pk: `${subArea.id}::Day Use`, + // sk: `202201` + // }) } park.subAreas = modifiedSubAreas; // Add the park record - await docClient - .put({ - TableName: TABLE_NAME, - Item: park, - }) - .promise(); + let params4 = { + TableName: TABLE_NAME, + Item: marshall(park), + } + await dynamoClient.send(new PutItemCommand(params4)) testParkList.push(park); } @@ -115,34 +98,54 @@ describe("Sub Area Test", () => { }) ) }; + const OLD_ENV = process.env; + let hash + let TABLE_NAME + let NAME_CACHE_TABLE_NAME + let CONFIG_TABLE_NAME + beforeEach(async () => { jest.resetModules(); process.env = { ...OLD_ENV }; // Make a copy of environment - }); + hash = getHashedText(expect.getState().currentTestName); + process.env.TABLE_NAME = hash + TABLE_NAME = process.env.TABLE_NAME; + NAME_CACHE_TABLE_NAME = TABLE_NAME.concat("-nameCache"); + CONFIG_TABLE_NAME = TABLE_NAME.concat("-config"); + process.env.NAME_CACHE_TABLE_NAME = NAME_CACHE_TABLE_NAME + process.env.CONFIG_TABLE_NAME= CONFIG_TABLE_NAME + process.env.TABLE_NAME = TABLE_NAME + await createDB(TABLE_NAME, NAME_CACHE_TABLE_NAME, CONFIG_TABLE_NAME); + await setupDb(TABLE_NAME); + }, 20000); afterEach(() => { + deleteDB(TABLE_NAME, NAME_CACHE_TABLE_NAME, CONFIG_TABLE_NAME); process.env = OLD_ENV; // Restore old environment }); - beforeAll(async () => { - return await setupDb(); - }); - test("Handler - 200 Sub Area POST Success", async () => { jest.mock('/opt/keycloakLayer', () => { return mockKeycloakRoles }); - let config = await docClient - .get({ + const dynamoClient = new DynamoDBClient({ + region: REGION, + endpoint: ENDPOINT + }); + + let params = { TableName: CONFIG_TABLE_NAME, - Key: { + Key: marshall({ pk: "subAreaID", - }, - }) - .promise(); - const lastID = Object.keys(config).length === 0 ? 0 : config.Item.lastID; + }), + } + const config = await dynamoClient.send(new GetItemCommand(params)); + + const lastID = config.Item === undefined ? 0 : config.Item.lastID; + + // TODO: need to unmarshall? const subAreaPOST = require("../POST/index"); const response = await subAreaPOST.handler( @@ -159,7 +162,7 @@ describe("Sub Area Test", () => { }, body: JSON.stringify({ activities: ["Day Use"], - orcs: "0041" + suffix, + orcs: "0041", managementArea: "South Fraser", section: "South Coast", region: "South Coast", @@ -171,52 +174,16 @@ describe("Sub Area Test", () => { ); expect(response.statusCode).toBe(200); - config = await docClient - .get({ - TableName: CONFIG_TABLE_NAME, - Key: { - pk: "subAreaID", - }, - }) - .promise(); - + let configParams2 = { + TableName: CONFIG_TABLE_NAME, + Key: marshall({ + pk: "subAreaID", + }), + }; + const config2Res = await dynamoClient.send(new GetItemCommand(configParams2)) + const config2 = unmarshall(config2Res.Item); // check for incremented subAreaID - expect(config.Item.lastID).toBeGreaterThan(lastID); - }); - - test("Handler - 403 Sub Area POST Unauthenticated", async () => { - const axios = require("axios"); - jest.mock("axios"); - axios.post.mockImplementation(() => - Promise.resolve({ statusCode: 200, data: {} }) - ); - - const subAreaPOST = require("../POST/index"); - const response = await subAreaPOST.handler( - { - headers: { - Authorization: "Bearer " + token, - }, - requestContext: { - authorizer: { - roles: "[\"public\"]", - isAdmin: false, - isAuthenticated: false, - }, - }, - body: JSON.stringify({ - activities: ["Day Use"], - orcs: "0041" + suffix, - managementArea: "South Fraser", - section: "South Coast", - region: "South Coast", - bundle: "South Fraser", - subAreaName: "Clear Creek", - }), - }, - null - ); - expect(response.statusCode).toBe(403); + expect(config2.lastID).toBeGreaterThan(lastID); }); test("Handler - 403 Sub Area POST Unauthenticated Invalid User", async () => { @@ -241,7 +208,7 @@ describe("Sub Area Test", () => { }, body: JSON.stringify({ activities: ["Day Use"], - orcs: "0041" + suffix, + orcs: "0041", managementArea: "South Fraser", section: "South Coast", region: "South Coast", @@ -254,7 +221,7 @@ describe("Sub Area Test", () => { expect(response.statusCode).toBe(403); }); - test("Handler - 403 Sub Area POST Unauthenticated", async () => { + test("Handler - 403 Sub Area POST Unauthenticated ", async () => { const axios = require("axios"); jest.mock("axios"); axios.post.mockImplementation(() => @@ -276,7 +243,7 @@ describe("Sub Area Test", () => { }, body: JSON.stringify({ activities: ["Day Use"], - orcs: "0041" + suffix, + orcs: "0041", managementArea: "South Fraser", section: "South Coast", region: "South Coast", @@ -312,7 +279,7 @@ describe("Sub Area Test", () => { }, body: JSON.stringify({ activities: ["Day Use"], - orcs: "0041" + suffix, + orcs: "0041", managementArea: "South Fraser", section: "South Coast", region: "South Coast", @@ -438,7 +405,7 @@ describe("Sub Area Test", () => { Authorization: "Bearer " + token, }, queryStringParameters: { - orcs: "0041" + suffix, + orcs: "0041", archive: "false", subAreaId: "fakeSubAreaId" }, @@ -473,7 +440,7 @@ describe("Sub Area Test", () => { Authorization: "Bearer " + token, }, queryStringParameters: { - orcs: "0041" + suffix, + orcs: "0041", archive: "false", subAreaId: "fakeSubAreaId" }, @@ -508,7 +475,7 @@ describe("Sub Area Test", () => { Authorization: "Bearer " + token, }, queryStringParameters: { - orcs: "0041" + suffix, + orcs: "0041", archive: "true", subAreaId: "fakeSubAreaId" }, @@ -576,7 +543,7 @@ describe("Sub Area Test", () => { const subAreaDELETE = require("../DELETE/index"); // Delete the first subarea from PARKSLIST - const parkObject = PARKSLIST[0]; + const parkObject = PARKSLIST[0]; //add activities const qsp = { orcs: parkObject.orcs, archive: "false", diff --git a/arSam/handlers/variance/PUT/index.js b/arSam/handlers/variance/PUT/index.js index a3f370e..71137bd 100644 --- a/arSam/handlers/variance/PUT/index.js +++ b/arSam/handlers/variance/PUT/index.js @@ -1,4 +1,4 @@ -const { dynamodb, TABLE_NAME, logger, sendResponse } = require("/opt/baseLayer"); +const { dynamoClient, UpdateItemCommand, TABLE_NAME, logger, sendResponse } = require("/opt/baseLayer"); exports.handler = async (event, context) => { logger.debug("Variance PUT:", event); @@ -66,7 +66,7 @@ exports.handler = async (event, context) => { params.UpdateExpression = `SET ${updateExpressions.join(', ')}`; } - const res = await dynamodb.updateItem(params); + const res = await dynamoClient.send(new UpdateItemCommand(params)); logger.info("Variance updated"); logger.debug("Result:", res); return sendResponse(200, res); diff --git a/arSam/handlers/variance/__tests__/variance.test.js b/arSam/handlers/variance/__tests__/variance.test.js index 1c0c035..1d3c5b6 100644 --- a/arSam/handlers/variance/__tests__/variance.test.js +++ b/arSam/handlers/variance/__tests__/variance.test.js @@ -1,10 +1,7 @@ -const AWS = require("aws-sdk"); -const { DocumentClient } = require("aws-sdk/clients/dynamodb"); -const { - REGION, - ENDPOINT, - TABLE_NAME -} = require("../../../__tests__/settings"); +const { DynamoDBClient, PutItemCommand } = require('@aws-sdk/client-dynamodb'); +const { REGION, ENDPOINT } = require("../../../__tests__/settings"); +const { getHashedText, deleteDB, createDB } = require("../../../__tests__/setup"); +const { marshall, unmarshall } = require('@aws-sdk/util-dynamodb'); const jwt = require("jsonwebtoken"); const tokenContent = { @@ -12,58 +9,73 @@ const tokenContent = { }; const token = jwt.sign(tokenContent, "defaultSecret"); -async function setupDb() { - new AWS.DynamoDB({ +async function setupDb(TABLE_NAME) { + const dynamoClient = new DynamoDBClient({ region: REGION, - endpoint: ENDPOINT, - }); - docClient = new DocumentClient({ - region: REGION, - endpoint: ENDPOINT, - convertEmptyValues: true, + endpoint: ENDPOINT }); - await docClient - .put({ - TableName: TABLE_NAME, - Item: { - pk: `variance::0001::202201`, - sk: `0403::Day Use`, - fields: docClient.createSet(["peopleAndVehiclesVehicle"]), - notes: "A Note", - resolved: false, - }, - }) - .promise(); - - await docClient - .put({ - TableName: TABLE_NAME, - Item: { - pk: `variance::0001::202201`, - sk: `0403::Frontcountry Camping`, - fields: docClient.createSet(["peopleAndVehiclesVehicle"]), - notes: "A different note", - resolved: false, - }, - }) - .promise(); - - await docClient - .put({ - TableName: TABLE_NAME, - Item: { - pk: `variance::0001::202202`, - sk: `0403::Day Use`, - fields: docClient.createSet(["peopleAndVehiclesVehicle"]), - notes: "A Note", - resolved: false, - }, - }) - .promise(); + let params1 = { + TableName: TABLE_NAME, + Item: marshall({ + pk: `variance::0001::202201`, + sk: `0403::Day Use`, + fields: { SS: ["peopleAndVehiclesVehicle"] }, + notes: "A Note", + resolved: false, + }), + }; + await dynamoClient.send(new PutItemCommand(params1)) + + let params2 = { + TableName: TABLE_NAME, + Item: marshall({ + pk: `variance::0001::202201`, + sk: `0403::Frontcountry Camping`, + fields: { SS: ["peopleAndVehiclesVehicle"] }, + notes: "A different note", + resolved: false, + }), + }; + await dynamoClient.send(new PutItemCommand(params2)) + + let params3 = { + TableName: TABLE_NAME, + Item: marshall({ + pk: `variance::0001::202202`, + sk: `0403::Day Use`, + fields: { SS: ["peopleAndVehiclesVehicle"] }, + notes: "A Note", + resolved: false, + }), + } + await dynamoClient.send(new PutItemCommand(params3)) } describe("Variance Test", () => { + const OLD_ENV = process.env; + let hash + let TABLE_NAME + let NAME_CACHE_TABLE_NAME + let CONFIG_TABLE_NAME + + beforeEach(async () => { + jest.resetModules(); + process.env = { ...OLD_ENV }; // Make a copy of environment + hash = getHashedText(expect.getState().currentTestName); + process.env.TABLE_NAME = hash + TABLE_NAME = process.env.TABLE_NAME; + NAME_CACHE_TABLE_NAME = TABLE_NAME.concat("-nameCache"); + CONFIG_TABLE_NAME = TABLE_NAME.concat("-config"); + await createDB(TABLE_NAME, NAME_CACHE_TABLE_NAME, CONFIG_TABLE_NAME); + await setupDb(TABLE_NAME); + }, 20000); + + afterEach(() => { + deleteDB(TABLE_NAME, NAME_CACHE_TABLE_NAME, CONFIG_TABLE_NAME); + process.env = OLD_ENV; // Restore old environment + }); + const mockedUnauthenticatedInvalidUser = { roleFilter: jest.fn((records, roles) => { return {} @@ -82,20 +94,6 @@ describe("Variance Test", () => { }), }; - const OLD_ENV = process.env; - beforeEach(async () => { - jest.resetModules(); - process.env = { ...OLD_ENV }; // Make a copy of environment - }); - - afterEach(() => { - process.env = OLD_ENV; // Restore old environment - }); - - beforeAll(async () => { - return await setupDb(); - }); - test("Variance GET Single PK Success", async () => { jest.mock("/opt/permissionLayer", () => { return mockedSysadmin; diff --git a/arSam/layers/__tests__/dynamoLayer.test.js b/arSam/layers/baseLayer/__tests__/dynamoLayer.test.js similarity index 59% rename from arSam/layers/__tests__/dynamoLayer.test.js rename to arSam/layers/baseLayer/__tests__/dynamoLayer.test.js index 64b49b3..a02a346 100644 --- a/arSam/layers/__tests__/dynamoLayer.test.js +++ b/arSam/layers/baseLayer/__tests__/dynamoLayer.test.js @@ -1,102 +1,67 @@ -const AWS = require("aws-sdk"); -const { DocumentClient } = require("aws-sdk/clients/dynamodb"); -const { REGION, ENDPOINT, TABLE_NAME } = require("../../__tests__/settings"); -const { PARKSLIST, SUBAREAS, SUBAREA_ENTRIES } = require("../../__tests__/mock_data.json"); - -const CONFIG_TABLE_NAME = "ConfigsAr-tests-dynamo"; - -async function setupDb() { - const dynamoDb = new AWS.DynamoDB({ - region: REGION, - endpoint: ENDPOINT, - }); - docClient = new DocumentClient({ +const { DynamoDBClient, PutItemCommand } = require("@aws-sdk/client-dynamodb"); +const { REGION, ENDPOINT } = require("../../../__tests__/settings"); +const { PARKSLIST, SUBAREAS, SUBAREA_ENTRIES } = require("../../../__tests__/mock_data.json"); +const { getHashedText, deleteDB, createDB } = require("../../../__tests__/setup"); +const { marshall } = require('@aws-sdk/util-dynamodb'); + +async function setupDb(TABLE_NAME) { + const dynamoClient = new DynamoDBClient({ region: REGION, - endpoint: ENDPOINT, - convertEmptyValues: true, + endpoint: ENDPOINT }); - await dynamoDb - .createTable({ - TableName: CONFIG_TABLE_NAME, - KeySchema: [ - { - AttributeName: "pk", - KeyType: "HASH", - }, - ], - AttributeDefinitions: [ - { - AttributeName: "pk", - AttributeType: "S", - }, - ], - ProvisionedThroughput: { - ReadCapacityUnits: 1, - WriteCapacityUnits: 1, - }, - }) - .promise(); - for (const park of PARKSLIST) { - await docClient - .put({ - TableName: TABLE_NAME, - Item: park, - }) - .promise(); + let params = { + TableName: TABLE_NAME, + Item: marshall(park), + } + await dynamoClient.send(new PutItemCommand(params)) } for (const subarea of SUBAREAS) { - await docClient - .put({ - TableName: TABLE_NAME, - Item: subarea, - }) - .promise(); + let params = { + TableName: TABLE_NAME, + Item: marshall(subarea), + } + await dynamoClient.send(new PutItemCommand(params)) } for (const subEntry of SUBAREA_ENTRIES) { - await docClient - .put({ - TableName: TABLE_NAME, - Item: subEntry, - }) - .promise(); + let params = { + TableName: TABLE_NAME, + Item: marshall(subEntry), + } + await dynamoClient.send(new PutItemCommand(params)) } } describe("Pass Succeeds", () => { const OLD_ENV = process.env; + let hash + let TABLE_NAME + let NAME_CACHE_TABLE_NAME + let CONFIG_TABLE_NAME + beforeEach(async () => { jest.resetModules(); process.env = { ...OLD_ENV }; // Make a copy of environment + hash = getHashedText(expect.getState().currentTestName); + process.env.TABLE_NAME = hash + TABLE_NAME = process.env.TABLE_NAME; + NAME_CACHE_TABLE_NAME = TABLE_NAME.concat("-nameCache"); + CONFIG_TABLE_NAME = TABLE_NAME.concat("-config"); process.env.CONFIG_TABLE_NAME = CONFIG_TABLE_NAME; + await createDB(TABLE_NAME, NAME_CACHE_TABLE_NAME, CONFIG_TABLE_NAME); + await setupDb(TABLE_NAME); }); afterEach(() => { + deleteDB(TABLE_NAME, NAME_CACHE_TABLE_NAME, CONFIG_TABLE_NAME); process.env = OLD_ENV; // Restore old environment }); - beforeAll(async () => { - return await setupDb(); - }); - - afterAll(async () => { - const dynamoDb = new AWS.DynamoDB({ - region: REGION, - endpoint: ENDPOINT, - }); - - await dynamoDb - .deleteTable({ - TableName: CONFIG_TABLE_NAME, - }) - .promise(); - }); - test("dynamoUtil - runScan", async () => { - const utils = require("../baseLayer/baseLayer"); + const utils = require("../baseLayer"); let queryObj = { TableName: TABLE_NAME, @@ -120,7 +85,7 @@ describe("Pass Succeeds", () => { }); test("dynamoUtil - getParks", async () => { - const utils = require("../baseLayer/baseLayer"); + const utils = require("../baseLayer"); const result = await utils.getParks(); @@ -137,7 +102,7 @@ describe("Pass Succeeds", () => { }); test("dynamoUtil - getSubAreas", async () => { - const utils = require("../baseLayer/baseLayer"); + const utils = require("../baseLayer"); let orc = "0041"; let specificSubAreas = []; @@ -161,7 +126,7 @@ describe("Pass Succeeds", () => { }); test("dynamoUtil - getRecords", async () => { - const utils = require("../baseLayer/baseLayer"); + const utils = require("../baseLayer"); const result = await utils.getRecords(SUBAREAS[0]); @@ -175,7 +140,7 @@ describe("Pass Succeeds", () => { }); test("dynamoUtil - incrementAndGetNextSubAreaID works with and without an entry in the DB", async () => { - const utils = require("../baseLayer/baseLayer"); + const utils = require("../baseLayer"); const result = await utils.incrementAndGetNextSubAreaID(); expect(result).toEqual("1"); diff --git a/arSam/layers/baseLayer/baseLayer.js b/arSam/layers/baseLayer/baseLayer.js index a9c04f7..c7905d5 100644 --- a/arSam/layers/baseLayer/baseLayer.js +++ b/arSam/layers/baseLayer/baseLayer.js @@ -86,8 +86,20 @@ function calculateVariance( } // DynamoUtils -const { DynamoDB } = require('@aws-sdk/client-dynamodb'); +const { DynamoDBClient, + GetItemCommand, + QueryCommand, + PutItemCommand, + UpdateItemCommand, + BatchWriteItemCommand, + TransactWriteItemsCommand, + ScanCommand, + DeleteItemCommand +} = require('@aws-sdk/client-dynamodb'); const { marshall, unmarshall } = require('@aws-sdk/util-dynamodb'); +const { S3Client, PutObjectCommand, GetObjectCommand } = require("@aws-sdk/client-s3"); +const { getSignedUrl } = require("@aws-sdk/s3-request-presigner"); +const { Lambda } = require("@aws-sdk/client-lambda"); const TABLE_NAME = process.env.TABLE_NAME || "ParksAr-tests"; const ORCS_INDEX = process.env.ORCS_INDEX || "orcs-index"; @@ -130,9 +142,9 @@ const RECORD_ACTIVITY_LIST = [ "Boating", ]; -const dynamodb = new DynamoDB(options); - -exports.dynamodb = new DynamoDB(); +const dynamoClient = new DynamoDBClient(options); +const s3Client = new S3Client({region: AWS_REGION}); +const lambda = new Lambda({region: AWS_REGION}); // simple way to return a single Item by primary key. async function getOne(pk, sk) { @@ -141,7 +153,7 @@ async function getOne(pk, sk) { TableName: TABLE_NAME, Key: marshall({ pk, sk }), }; - let item = await dynamodb.getItem(params); + let item = await dynamoClient.send(new GetItemCommand(params)); if (item?.Item) { return unmarshall(item.Item); } @@ -151,17 +163,18 @@ async function getOne(pk, sk) { // (1MB) unless they are explicitly specified to retrieve more. // TODO: Ensure the returned object has the same structure whether results are paginated or not. async function runQuery(query, paginated = false) { - logger.debug("query:", query); + logger.info('query:', query); let data = []; let pageData = []; let page = 0; - + const command = new QueryCommand(query); + do { page++; if (pageData?.LastEvaluatedKey) { - query.ExclusiveStartKey = pageData.LastEvaluatedKey; + command.input.ExclusiveStartKey = pageData.LastEvaluatedKey; } - pageData = await dynamodb.query(query); + pageData = await dynamoClient.send(command); data = data.concat( pageData.Items.map((item) => { return unmarshall(item); @@ -203,7 +216,7 @@ async function runScan(query, paginated = false) { if (pageData?.LastEvaluatedKey) { query.ExclusiveStartKey = pageData.LastEvaluatedKey; } - pageData = await dynamodb.scan(query); + pageData = await dynamoClient.send(new ScanCommand(query)); data = data.concat( pageData.Items.map((item) => { return unmarshall(item); @@ -276,7 +289,7 @@ async function batchWrite(items, action = 'put') { } } try { - await dynamodb.batchWriteItem(batchChunk); + await dynamoClient.send(new BatchWriteItemCommand(batchChunk)); } catch (err) { for (const item of items) { logger.info('item.fields:', item.fields); @@ -350,7 +363,7 @@ async function incrementAndGetNextSubAreaID() { }, ReturnValues: "UPDATED_NEW", }; - const response = await dynamodb.updateItem(configUpdateObj); + const response = await dynamoClient.send(new UpdateItemCommand(configUpdateObj)); return response?.Attributes?.lastID?.N; } @@ -371,7 +384,19 @@ module.exports = { TABLE_NAME, ORCS_INDEX, NAME_CACHE_TABLE_NAME, - dynamodb, + dynamoClient, + PutItemCommand, + UpdateItemCommand, + DeleteItemCommand, + BatchWriteItemCommand, + TransactWriteItemsCommand, + s3Client, + PutObjectCommand, + GetObjectCommand, + marshall, + unmarshall, + getSignedUrl, + lambda, runQuery, runScan, getOne, diff --git a/arSam/layers/baseLayer/nodejs/package.json b/arSam/layers/baseLayer/nodejs/package.json index 812e823..caad788 100644 --- a/arSam/layers/baseLayer/nodejs/package.json +++ b/arSam/layers/baseLayer/nodejs/package.json @@ -6,6 +6,9 @@ "dependencies": { "@aws-sdk/client-dynamodb": "^3.614.0", "@aws-sdk/util-dynamodb": "^3.614.0", + "@aws-sdk/s3-request-presigner": "^3.568.0", + "@aws-sdk/client-s3": "^3.568.0", + "@aws-sdk/client-lambda": "^3.568.0", "winston": "^3.8.0" } } diff --git a/arSam/layers/__tests__/constantsLayer.test.js b/arSam/layers/constantsLayer/__tests__/constantsLayer.test.js similarity index 98% rename from arSam/layers/__tests__/constantsLayer.test.js rename to arSam/layers/constantsLayer/__tests__/constantsLayer.test.js index 1e2017b..90c7386 100644 --- a/arSam/layers/__tests__/constantsLayer.test.js +++ b/arSam/layers/constantsLayer/__tests__/constantsLayer.test.js @@ -7,7 +7,7 @@ describe("Constants Test", () => { test("Handler - Constants has items", async () => { - const constants = require("../constantsLayer/constantsLayer"); + const constants = require("../constantsLayer"); // Checks to ensure the value functions returns the data we pass through to it based on the attribute. expect(constants.CSV_SYSADMIN_SCHEMA.length).toEqual(96); for(const row of constants.CSV_SYSADMIN_SCHEMA) { diff --git a/arSam/layers/__tests__/formulasLayer.test.js b/arSam/layers/formulaLayer/__tests__/formulasLayer.test.js similarity index 95% rename from arSam/layers/__tests__/formulasLayer.test.js rename to arSam/layers/formulaLayer/__tests__/formulasLayer.test.js index 7c7fffd..08fa5f6 100644 --- a/arSam/layers/__tests__/formulasLayer.test.js +++ b/arSam/layers/formulaLayer/__tests__/formulasLayer.test.js @@ -4,7 +4,7 @@ describe("keycloak utility tests", () => { }); test("Creates Update Park with New Sub Area Object", async () => { - const utils = require("../formulaLayer/formulaLayer"); + const utils = require("../formulaLayer"); const response = await utils.createPutFormulaConfigObj( ["Day Use", "Backcountry Cabins", "Fake Garbage"], "test-id", diff --git a/arSam/layers/functionsLayer/Makefile b/arSam/layers/functionsLayer/Makefile index f0e0178..3ca068c 100644 --- a/arSam/layers/functionsLayer/Makefile +++ b/arSam/layers/functionsLayer/Makefile @@ -8,4 +8,4 @@ build-FunctionsLayer: # symlink to node_modules folder for testing purposes cd "$(ARTIFACTS_DIR)" && ln -s "$(ARTIFACTS_DIR)"/nodejs/node_modules node_modules # remove package.json to avoid rebuilding when changes don't relate to dependencies - rm "$(ARTIFACTS_DIR)/nodejs/package.json" + rm "$(ARTIFACTS_DIR)/nodejs/package.json" diff --git a/arSam/layers/functionsLayer/functionsLayer.js b/arSam/layers/functionsLayer/functionsLayer.js index e461d05..2dab6f7 100644 --- a/arSam/layers/functionsLayer/functionsLayer.js +++ b/arSam/layers/functionsLayer/functionsLayer.js @@ -1,5 +1,4 @@ -const { marshall } = require('@aws-sdk/util-dynamodb'); -const { dynamodb } = require("/opt/baseLayer"); +const { dynamoClient, PutItemCommand, marshall } = require("/opt/baseLayer"); const { createHash } = require("node:crypto"); function convertRolesToMD5(roles, prefix = "") { @@ -24,7 +23,7 @@ async function updateJobEntry(jobObj, tableName) { TableName: tableName, Item: newObject, }; - await dynamodb.putItem(putObject); + await dynamoClient.send(new PutItemCommand(putObject)); } module.exports = { diff --git a/arSam/layers/functionsLayer/nodejs/package.json b/arSam/layers/functionsLayer/nodejs/package.json index 415e89d..6f5617d 100644 --- a/arSam/layers/functionsLayer/nodejs/package.json +++ b/arSam/layers/functionsLayer/nodejs/package.json @@ -2,8 +2,5 @@ "name": "functionsLayer", "version": "0.0.1", "author": "Digitalspace ", - "license": "MIT", - "dependencies": { - "@aws-sdk/util-dynamodb": "^3.614.0" - } + "license": "MIT" } diff --git a/arSam/layers/__tests__/keycloakLayer.test.js b/arSam/layers/keycloakLayer/__tests__/keycloakLayer.test.js similarity index 87% rename from arSam/layers/__tests__/keycloakLayer.test.js rename to arSam/layers/keycloakLayer/__tests__/keycloakLayer.test.js index 60f15fc..04ea449 100644 --- a/arSam/layers/__tests__/keycloakLayer.test.js +++ b/arSam/layers/keycloakLayer/__tests__/keycloakLayer.test.js @@ -8,7 +8,7 @@ describe("keycloak utility tests", () => { const axios = require('axios'); jest.mock("axios"); axios.post.mockImplementation(() => Promise.resolve({ statusCode: 200, data: {} })); - const utils = require("../keycloakLayer/keycloakLayer"); + const utils = require("../keycloakLayer"); const response = await utils.createKeycloakRole('http://localhost/', 'client-id', 'sometoken', '0001:0001', 'Some description'); expect(response).toEqual({}); }); @@ -17,7 +17,7 @@ describe("keycloak utility tests", () => { const axios = require('axios'); jest.mock("axios"); axios.post.mockImplementation(() => Promise.reject({ statusCode: 400, data: {} })); - const utils = require("../keycloakLayer/keycloakLayer"); + const utils = require("../keycloakLayer"); try { await utils.createKeycloakRole('http://localhost/', 'client-id', 'sometoken', '0001:0001', 'Some description'); } catch (error) { @@ -29,7 +29,7 @@ describe("keycloak utility tests", () => { const axios = require('axios'); jest.mock("axios"); axios.delete.mockImplementation(() => Promise.resolve({ statusCode: 200, data: { } })); - const utils = require("../keycloakLayer/keycloakLayer"); + const utils = require("../keycloakLayer"); const response = await utils.deleteKeycloakRole('http://localhost/', 'client-id', 'sometoken', '0001:0001'); expect(response).toEqual({}); }); @@ -38,7 +38,7 @@ describe("keycloak utility tests", () => { const axios = require('axios'); jest.mock("axios"); axios.delete.mockImplementation(() => Promise.reject({ statusCode: 400, data: {} })); - const utils = require("../keycloakLayer/keycloakLayer"); + const utils = require("../keycloakLayer"); try { await utils.deleteKeycloakRole('http://localhost/', 'client-id', 'sometoken', '0001:0001', 'Some description'); } catch (error) { @@ -53,7 +53,7 @@ describe("keycloak utility tests", () => { const axios = require('axios'); jest.mock("axios"); axios.get.mockImplementation(() => Promise.resolve({ statusCode: 200, data: theRole })); - const utils = require("../keycloakLayer/keycloakLayer"); + const utils = require("../keycloakLayer"); const response = await utils.getKeycloakRole('http://localhost/', 'client-id', 'sometoken', '0001:0001'); expect(response).toEqual(theRole); }); @@ -62,7 +62,7 @@ describe("keycloak utility tests", () => { const axios = require('axios'); jest.mock("axios"); axios.get.mockImplementation(() => Promise.reject({ statusCode: 400, data: {} })); - const utils = require("../keycloakLayer/keycloakLayer"); + const utils = require("../keycloakLayer"); try { await utils.getKeycloakRole('http://localhost/', 'client-id', 'sometoken', '0001:0001'); } catch (error) { diff --git a/arSam/layers/__tests__/subAreaLayer.test.js b/arSam/layers/subAreaLayer/__tests__/subAreaLayer.test.js similarity index 93% rename from arSam/layers/__tests__/subAreaLayer.test.js rename to arSam/layers/subAreaLayer/__tests__/subAreaLayer.test.js index d1890b1..ade8285 100644 --- a/arSam/layers/__tests__/subAreaLayer.test.js +++ b/arSam/layers/subAreaLayer/__tests__/subAreaLayer.test.js @@ -16,7 +16,7 @@ describe('keycloak utility tests', () => { }; test('Creates Update Park with New Sub Area Object', async () => { - const utils = require('../subAreaLayer/subAreaLayer'); + const utils = require('../subAreaLayer'); const response = await utils.createUpdateParkWithNewSubAreaObj('test-name', 'test-id', false, 'test-orcs'); expect(response).toEqual({ @@ -46,7 +46,7 @@ describe('keycloak utility tests', () => { }); test('Creates Put Sub Area Obj', async () => { - const utils = require('../subAreaLayer/subAreaLayer'); + const utils = require('../subAreaLayer'); const response = await utils.createPutSubAreaObj(testSubAreaObj, 'test-id', 'test-name'); expect(response).toEqual({ @@ -69,7 +69,7 @@ describe('keycloak utility tests', () => { }); test('Creates Valid Sub Area Object', async () => { - const utils = require('../subAreaLayer/subAreaLayer'); + const utils = require('../subAreaLayer'); const garbage = { test: 'fake', whatever: [] }; const testSubAreaObjWithGarbage = { ...testSubAreaObj, ...garbage }; diff --git a/arSam/package.json b/arSam/package.json index eeb1487..26d3892 100644 --- a/arSam/package.json +++ b/arSam/package.json @@ -5,14 +5,13 @@ "build": "sam build", "test": "npm run build && jest --coverage" }, - "jest": { + "jest": { "verbose": true, - "globalSetup": "./__tests__/setup.js", - "globalTeardown": "./__tests__/teardown.js", + "testTimeout": 10000, "modulePathIgnorePatterns": [ "/__tests__", "/.aws-sam/" - ], + ], "moduleNameMapper": { "^/opt/baseLayer": "/.aws-sam/build/BaseLayer/baseLayer", "^/opt/constantsLayer": "/.aws-sam/build/ConstantsLayer/constantsLayer", @@ -26,22 +25,23 @@ "devDependencies": { "@digitalspace/dynamodb-migrate": "^1.0.6", "aws-sdk-mock": "^5.4.0", - "jest": "^29.5.0", + "jest": "^29.7.0", + "luxon": "^3.2.1", "read-excel-file": "^5.3.4", "serverless": "^3.18.1", "serverless-dotenv-plugin": "^6.0.0", "serverless-offline": "^12.0.4", - "serverless-plugin-include-dependencies": "^5.0.0", - "luxon": "^3.2.1" + "serverless-plugin-include-dependencies": "^5.0.0" }, "dependencies": { "@aws-sdk/client-dynamodb": "^3.614.0", "@aws-sdk/client-lambda": "^3.568.0", "@aws-sdk/client-s3": "^3.568.0", - "@aws-sdk/util-dynamodb": "^3.614.0", "@aws-sdk/s3-request-presigner": "^3.568.0", + "@aws-sdk/util-dynamodb": "^3.614.0", "@babel/traverse": "7.23.2", "axios": "^1.4.0", + "crypto": "1.0.1", "jsonwebtoken": "^9.0.0", "jwks-rsa": "^3.0.1", "node-jose": "^2.2.0", @@ -51,4 +51,4 @@ "winston": "^3.8.0", "write-excel-file": "^1.3.16" } -} +}