From 6a47132aa8f5d3dff0dfeceff119dfcbb9a1ad22 Mon Sep 17 00:00:00 2001 From: Maria <79389256+marythedev@users.noreply.github.com> Date: Mon, 4 Dec 2023 22:02:02 -0500 Subject: [PATCH] added dynamodb - app fully supports aws --- .github/workflows/ci.yml | 28 +- docker-compose.yml | 7 +- fragments-definition.json | 273 ++++++------ package-lock.json | 554 +++++++++++++++++++++++++ package.json | 2 + src/model/data/aws/ddbDocClient.js | 66 +++ src/model/data/aws/index.js | 128 ++++-- src/model/fragment.js | 11 +- src/routes/api/byId.js | 2 +- src/routes/api/delete.js | 2 +- src/routes/api/get.js | 2 +- src/routes/api/info.js | 9 +- src/routes/api/post.js | 4 +- tests/integration/lab-10-dynamodb.hurl | 126 ++++++ tests/integration/lab-9-s3.hurl | 9 +- 15 files changed, 1041 insertions(+), 182 deletions(-) create mode 100644 src/model/data/aws/ddbDocClient.js create mode 100644 tests/integration/lab-10-dynamodb.hurl diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7fdb6d8..00b2bb6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,9 +68,33 @@ jobs: #2. Use `install-ci-test` to do it in a single command, see https://docs.npmjs.com/cli/v8/commands/npm-install-ci-test run: npm install-ci-test + integration-tests: + name: Integration Tests + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v3 + - name: Setup node + uses: actions/setup-node@v3 + with: + node-version: '18' + cache: 'npm' + - name: Install node dependencies + # NOTE: we need to install dev dependencies too vs. production only for hurl + run: npm install + - name: Build Containers + run: docker compose up -d + - name: Setup Local AWS Resources + # NOTE: this file needs to be made executable *before* you check into git: + # $ chmod +x ./scripts/local-aws-setup.sh + run: ./scripts/local-aws-setup.sh + - name: Run Hurl Tests + run: npm run test:integration + + docker-hub: name: Build and Push Image to Docker Hub - needs: [lint, dockerfile-lint, unit-tests] + needs: [lint, dockerfile-lint, unit-tests, integration-tests] runs-on: ubuntu-latest steps: # Set up buildx for optimal Docker Builds, see: @@ -95,4 +119,4 @@ jobs: with: push: true # Use 3 tags: :latest-commit-sha, :main, and :latest - tags: ${{ env.DOCKERHUB_REPO }}:${{ env.SHA_TAG }}, ${{ env.DOCKERHUB_REPO }}:main, ${{ env.DOCKERHUB_REPO }}:latest + tags: ${{ env.DOCKERHUB_REPO }}:${{ env.SHA_TAG }}, ${{ env.DOCKERHUB_REPO }}:main, ${{ env.DOCKERHUB_REPO }}:latest \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index f6e85c3..e61a014 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,7 +8,7 @@ services: environment: - API_URL=http://localhost:8080 # API is running on http://localhost:8080 - HTPASSWD_FILE=tests/.htpasswd # Use Basic Auth (for running tests, CI) - - LOG_LEVEL=${LOG_LEVEL:-info} + - LOG_LEVEL=${LOG_LEVEL:-debug} - AWS_REGION=us-east-1 # Use the LocalStack endpoint vs. AWS for S3 AWS SDK clients. # NOTE: we use Docker's internal network to the localstack container @@ -19,6 +19,9 @@ services: # default to 'fragments' as the name, unless something else is defined in the env. - AWS_S3_BUCKET_NAME=${AWS_S3_BUCKET_NAME:-fragments} - AWS_DYNAMODB_TABLE_NAME=${AWS_DYNAMODB_TABLE_NAME:-fragments} + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} + - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} + - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN} # Ports to publish ports: - '8080:8080' @@ -30,7 +33,7 @@ services: - '8000:8000' # Run the database in memory, see: # https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/DynamoDBLocal.UsageNotes.html - command: ['-jar', 'DynamoDBLocal.jar', '-inMemory'] + command: ['-jar', 'DynamoDBLocal.jar', '-inMemory', '-sharedDb'] # LocalStack for S3, see https://docs.localstack.cloud/get-started/#docker-compose # Interact via awscli-local, see https://docs.localstack.cloud/integrations/aws-cli/#installation diff --git a/fragments-definition.json b/fragments-definition.json index 9e33898..52fcf6d 100644 --- a/fragments-definition.json +++ b/fragments-definition.json @@ -1,127 +1,150 @@ { - "ipcMode": null, - "executionRoleArn": "arn:aws:iam::390240750368:role/LabRole", - "containerDefinitions": [ - { - "dnsSearchDomains": null, - "environmentFiles": null, - "logConfiguration": { - "logDriver": "awslogs", - "secretOptions": null, - "options": { - "awslogs-group": "/ecs/fragments-task", - "awslogs-region": "us-east-1", - "awslogs-stream-prefix": "ecs" - } - }, - "entryPoint": [], - "portMappings": [ - { - "hostPort": 8080, - "protocol": "tcp", - "containerPort": 8080 - } - ], - "command": [], - "linuxParameters": null, - "cpu": 256, - "environment": [ - { - "name": "PORT", - "value": "8080" - }, { - "name": "AWS_S3_BUCKET_NAME", - "value": "maria.dmytrenko-fragments" - } - ], - "resourceRequirements": null, - "ulimits": null, - "dnsServers": null, - "mountPoints": [], - "workingDirectory": null, - "secrets": null, - "dockerSecurityOptions": null, - "memory": null, - "memoryReservation": 512, - "volumesFrom": [], - "stopTimeout": null, - "image": "mdmytrenko/fragments:latest", - "startTimeout": null, - "firelensConfiguration": null, - "dependsOn": null, - "disableNetworking": null, - "interactive": null, - "healthCheck": null, - "essential": true, - "links": [], - "hostname": null, - "extraHosts": null, - "pseudoTerminal": null, - "user": null, - "readonlyRootFilesystem": null, - "dockerLabels": null, - "systemControls": null, - "privileged": null, - "name": "fragments" - } - ], - "placementConstraints": [], - "memory": "512", - "taskRoleArn": "arn:aws:iam::390240750368:role/LabRole", - "compatibilities": [ - "EC2", - "FARGATE" - ], - "taskDefinitionArn": "arn:aws:ecs:us-east-1:390240750368:task-definition/fragments-task:2", - "family": "fragments-task", - "requiresAttributes": [ - { - "targetId": null, - "targetType": null, - "value": null, - "name": "com.amazonaws.ecs.capability.logging-driver.awslogs" - }, - { - "targetId": null, - "targetType": null, - "value": null, - "name": "ecs.capability.execution-role-awslogs" - }, - { - "targetId": null, - "targetType": null, - "value": null, - "name": "com.amazonaws.ecs.capability.docker-remote-api.1.19" - }, - { - "targetId": null, - "targetType": null, - "value": null, - "name": "com.amazonaws.ecs.capability.docker-remote-api.1.21" + "ipcMode": null, + "executionRoleArn": "arn:aws:iam::390240750368:role/LabRole", + "containerDefinitions": [ + { + "dnsSearchDomains": null, + "environmentFiles": null, + "logConfiguration": { + "logDriver": "awslogs", + "secretOptions": null, + "options": { + "awslogs-group": "/ecs/fragments-task", + "awslogs-region": "us-east-1", + "awslogs-stream-prefix": "ecs" + } }, - { - "targetId": null, - "targetType": null, - "value": null, - "name": "com.amazonaws.ecs.capability.docker-remote-api.1.18" - }, - { - "targetId": null, - "targetType": null, - "value": null, - "name": "ecs.capability.task-eni" - } - ], - "pidMode": null, - "requiresCompatibilities": [ - "FARGATE" - ], - "networkMode": "awsvpc", - "runtimePlatform": null, - "cpu": "256", - "revision": 2, - "status": "ACTIVE", - "inferenceAccelerators": null, - "proxyConfiguration": null, - "volumes": [] - } \ No newline at end of file + "entryPoint": null, + "portMappings": [ + { + "hostPort": 8080, + "protocol": "tcp", + "containerPort": 8080 + } + ], + "command": null, + "linuxParameters": null, + "cpu": 256, + "environment": [ + { + "name": "AWS_S3_BUCKET_NAME", + "value": "maria.dmytrenko-fragments" + }, + { + "name": "AWS_DYNAMODB_TABLE_NAME", + "value": "fragments" + }, + { + "name": "PORT", + "value": "8080" + } + ], + "resourceRequirements": null, + "ulimits": null, + "dnsServers": null, + "mountPoints": [], + "workingDirectory": null, + "secrets": null, + "dockerSecurityOptions": null, + "memory": null, + "memoryReservation": 512, + "volumesFrom": [], + "stopTimeout": null, + "image": "390240750368.dkr.ecr.us-east-1.amazonaws.com/fragments:v0.9.0", + "startTimeout": null, + "firelensConfiguration": null, + "dependsOn": null, + "disableNetworking": null, + "interactive": null, + "healthCheck": null, + "essential": true, + "links": null, + "hostname": null, + "extraHosts": null, + "pseudoTerminal": null, + "user": null, + "readonlyRootFilesystem": null, + "dockerLabels": null, + "systemControls": null, + "privileged": null, + "name": "fragments" + } + ], + "placementConstraints": [], + "memory": "512", + "taskRoleArn": "arn:aws:iam::390240750368:role/LabRole", + "compatibilities": [ + "EC2", + "FARGATE" + ], + "taskDefinitionArn": "arn:aws:ecs:us-east-1:390240750368:task-definition/fragments-task:5", + "family": "fragments-task", + "requiresAttributes": [ + { + "targetId": null, + "targetType": null, + "value": null, + "name": "com.amazonaws.ecs.capability.logging-driver.awslogs" + }, + { + "targetId": null, + "targetType": null, + "value": null, + "name": "ecs.capability.execution-role-awslogs" + }, + { + "targetId": null, + "targetType": null, + "value": null, + "name": "com.amazonaws.ecs.capability.ecr-auth" + }, + { + "targetId": null, + "targetType": null, + "value": null, + "name": "com.amazonaws.ecs.capability.docker-remote-api.1.19" + }, + { + "targetId": null, + "targetType": null, + "value": null, + "name": "com.amazonaws.ecs.capability.docker-remote-api.1.21" + }, + { + "targetId": null, + "targetType": null, + "value": null, + "name": "com.amazonaws.ecs.capability.task-iam-role" + }, + { + "targetId": null, + "targetType": null, + "value": null, + "name": "ecs.capability.execution-role-ecr-pull" + }, + { + "targetId": null, + "targetType": null, + "value": null, + "name": "com.amazonaws.ecs.capability.docker-remote-api.1.18" + }, + { + "targetId": null, + "targetType": null, + "value": null, + "name": "ecs.capability.task-eni" + } + ], + "pidMode": null, + "requiresCompatibilities": [ + "FARGATE" + ], + "networkMode": "awsvpc", + "runtimePlatform": null, + "cpu": "256", + "revision": 5, + "status": "ACTIVE", + "inferenceAccelerators": null, + "proxyConfiguration": null, + "volumes": [] +} \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 8645391..cf89332 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,9 @@ "version": "0.9.0", "license": "UNLICENSED", "dependencies": { + "@aws-sdk/client-dynamodb": "^3.465.0", "@aws-sdk/client-s3": "^3.458.0", + "@aws-sdk/lib-dynamodb": "^3.465.0", "aws-jwt-verify": "^4.0.0", "compression": "^1.7.4", "content-type": "^1.0.5", @@ -184,6 +186,474 @@ "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" }, + "node_modules/@aws-sdk/client-dynamodb": { + "version": "3.465.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-dynamodb/-/client-dynamodb-3.465.0.tgz", + "integrity": "sha512-ruYGPuz6IliiPm7dBZhibzJWN15corMTM10xIGG4YZH1igrvM5uvu1ab6OdNi3JfjNuG0eKHK8Q7vN9bSYcQeQ==", + "dependencies": { + "@aws-crypto/sha256-browser": "3.0.0", + "@aws-crypto/sha256-js": "3.0.0", + "@aws-sdk/client-sts": "3.465.0", + "@aws-sdk/core": "3.465.0", + "@aws-sdk/credential-provider-node": "3.465.0", + "@aws-sdk/middleware-endpoint-discovery": "3.465.0", + "@aws-sdk/middleware-host-header": "3.465.0", + "@aws-sdk/middleware-logger": "3.465.0", + "@aws-sdk/middleware-recursion-detection": "3.465.0", + "@aws-sdk/middleware-signing": "3.465.0", + "@aws-sdk/middleware-user-agent": "3.465.0", + "@aws-sdk/region-config-resolver": "3.465.0", + "@aws-sdk/types": "3.465.0", + "@aws-sdk/util-endpoints": "3.465.0", + "@aws-sdk/util-user-agent-browser": "3.465.0", + "@aws-sdk/util-user-agent-node": "3.465.0", + "@smithy/config-resolver": "^2.0.18", + "@smithy/fetch-http-handler": "^2.2.6", + "@smithy/hash-node": "^2.0.15", + "@smithy/invalid-dependency": "^2.0.13", + "@smithy/middleware-content-length": "^2.0.15", + "@smithy/middleware-endpoint": "^2.2.0", + "@smithy/middleware-retry": "^2.0.20", + "@smithy/middleware-serde": "^2.0.13", + "@smithy/middleware-stack": "^2.0.7", + "@smithy/node-config-provider": "^2.1.5", + "@smithy/node-http-handler": "^2.1.9", + "@smithy/protocol-http": "^3.0.9", + "@smithy/smithy-client": "^2.1.15", + "@smithy/types": "^2.5.0", + "@smithy/url-parser": "^2.0.13", + "@smithy/util-base64": "^2.0.1", + "@smithy/util-body-length-browser": "^2.0.0", + "@smithy/util-body-length-node": "^2.1.0", + "@smithy/util-defaults-mode-browser": "^2.0.19", + "@smithy/util-defaults-mode-node": "^2.0.25", + "@smithy/util-endpoints": "^1.0.4", + "@smithy/util-retry": "^2.0.6", + "@smithy/util-utf8": "^2.0.2", + "@smithy/util-waiter": "^2.0.13", + "tslib": "^2.5.0", + "uuid": "^8.3.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/client-sso": { + "version": "3.465.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.465.0.tgz", + "integrity": "sha512-JXDBa3Sl+LS0KEOs0PZoIjpNKEEGfeyFwdnRxi8Y1hMXNEKyJug1cI2Psqu2olpn4KeXwoP1BuITppZYdolOew==", + "dependencies": { + "@aws-crypto/sha256-browser": "3.0.0", + "@aws-crypto/sha256-js": "3.0.0", + "@aws-sdk/core": "3.465.0", + "@aws-sdk/middleware-host-header": "3.465.0", + "@aws-sdk/middleware-logger": "3.465.0", + "@aws-sdk/middleware-recursion-detection": "3.465.0", + "@aws-sdk/middleware-user-agent": "3.465.0", + "@aws-sdk/region-config-resolver": "3.465.0", + "@aws-sdk/types": "3.465.0", + "@aws-sdk/util-endpoints": "3.465.0", + "@aws-sdk/util-user-agent-browser": "3.465.0", + "@aws-sdk/util-user-agent-node": "3.465.0", + "@smithy/config-resolver": "^2.0.18", + "@smithy/fetch-http-handler": "^2.2.6", + "@smithy/hash-node": "^2.0.15", + "@smithy/invalid-dependency": "^2.0.13", + "@smithy/middleware-content-length": "^2.0.15", + "@smithy/middleware-endpoint": "^2.2.0", + "@smithy/middleware-retry": "^2.0.20", + "@smithy/middleware-serde": "^2.0.13", + "@smithy/middleware-stack": "^2.0.7", + "@smithy/node-config-provider": "^2.1.5", + "@smithy/node-http-handler": "^2.1.9", + "@smithy/protocol-http": "^3.0.9", + "@smithy/smithy-client": "^2.1.15", + "@smithy/types": "^2.5.0", + "@smithy/url-parser": "^2.0.13", + "@smithy/util-base64": "^2.0.1", + "@smithy/util-body-length-browser": "^2.0.0", + "@smithy/util-body-length-node": "^2.1.0", + "@smithy/util-defaults-mode-browser": "^2.0.19", + "@smithy/util-defaults-mode-node": "^2.0.25", + "@smithy/util-endpoints": "^1.0.4", + "@smithy/util-retry": "^2.0.6", + "@smithy/util-utf8": "^2.0.2", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/client-sts": { + "version": "3.465.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sts/-/client-sts-3.465.0.tgz", + "integrity": "sha512-rHi9ba6ssNbVjlWSdhi4C5newEhGhzkY9UE4KB+/Tj21zXfEP8r6uIltnQXPtun2SdA95Krh/yS1qQ4MRuzqyA==", + "dependencies": { + "@aws-crypto/sha256-browser": "3.0.0", + "@aws-crypto/sha256-js": "3.0.0", + "@aws-sdk/core": "3.465.0", + "@aws-sdk/credential-provider-node": "3.465.0", + "@aws-sdk/middleware-host-header": "3.465.0", + "@aws-sdk/middleware-logger": "3.465.0", + "@aws-sdk/middleware-recursion-detection": "3.465.0", + "@aws-sdk/middleware-sdk-sts": "3.465.0", + "@aws-sdk/middleware-signing": "3.465.0", + "@aws-sdk/middleware-user-agent": "3.465.0", + "@aws-sdk/region-config-resolver": "3.465.0", + "@aws-sdk/types": "3.465.0", + "@aws-sdk/util-endpoints": "3.465.0", + "@aws-sdk/util-user-agent-browser": "3.465.0", + "@aws-sdk/util-user-agent-node": "3.465.0", + "@smithy/config-resolver": "^2.0.18", + "@smithy/fetch-http-handler": "^2.2.6", + "@smithy/hash-node": "^2.0.15", + "@smithy/invalid-dependency": "^2.0.13", + "@smithy/middleware-content-length": "^2.0.15", + "@smithy/middleware-endpoint": "^2.2.0", + "@smithy/middleware-retry": "^2.0.20", + "@smithy/middleware-serde": "^2.0.13", + "@smithy/middleware-stack": "^2.0.7", + "@smithy/node-config-provider": "^2.1.5", + "@smithy/node-http-handler": "^2.1.9", + "@smithy/protocol-http": "^3.0.9", + "@smithy/smithy-client": "^2.1.15", + "@smithy/types": "^2.5.0", + "@smithy/url-parser": "^2.0.13", + "@smithy/util-base64": "^2.0.1", + "@smithy/util-body-length-browser": "^2.0.0", + "@smithy/util-body-length-node": "^2.1.0", + "@smithy/util-defaults-mode-browser": "^2.0.19", + "@smithy/util-defaults-mode-node": "^2.0.25", + "@smithy/util-endpoints": "^1.0.4", + "@smithy/util-retry": "^2.0.6", + "@smithy/util-utf8": "^2.0.2", + "fast-xml-parser": "4.2.5", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/core": { + "version": "3.465.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.465.0.tgz", + "integrity": "sha512-fHSIw/Rgex3KbrEKn6ZrUc2VcsOTpdBMeyYtfmsTOLSyDDOG9k3jelOvVbCbrK5N6uEUSM8hrnySEKg94UB0cg==", + "dependencies": { + "@smithy/smithy-client": "^2.1.15", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/credential-provider-env": { + "version": "3.465.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.465.0.tgz", + "integrity": "sha512-fku37AgkB9KhCuWHE6mfvbWYU0X84Df6MQ60nYH7s/PiNEhkX2cVI6X6kOKjP1MNIwRcYt+oQDvplVKdHume+A==", + "dependencies": { + "@aws-sdk/types": "3.465.0", + "@smithy/property-provider": "^2.0.0", + "@smithy/types": "^2.5.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/credential-provider-ini": { + "version": "3.465.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.465.0.tgz", + "integrity": "sha512-B1MFufvdToAEMtfszilVnKer2S7P/OfMhkCizq2zuu8aU/CquRyHvKEQgWdvqunUDrFnVTc0kUZgsbBY0uPjLg==", + "dependencies": { + "@aws-sdk/credential-provider-env": "3.465.0", + "@aws-sdk/credential-provider-process": "3.465.0", + "@aws-sdk/credential-provider-sso": "3.465.0", + "@aws-sdk/credential-provider-web-identity": "3.465.0", + "@aws-sdk/types": "3.465.0", + "@smithy/credential-provider-imds": "^2.0.0", + "@smithy/property-provider": "^2.0.0", + "@smithy/shared-ini-file-loader": "^2.0.6", + "@smithy/types": "^2.5.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/credential-provider-node": { + "version": "3.465.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.465.0.tgz", + "integrity": "sha512-R3VA9yJ0BvezvrDxcgPTv9VHbVPbzchLTrX5jLFSVuW/lPPYLUi/Cjtyg9C9Y7qRfoQS4fNMvSRhwO5/TF68gA==", + "dependencies": { + "@aws-sdk/credential-provider-env": "3.465.0", + "@aws-sdk/credential-provider-ini": "3.465.0", + "@aws-sdk/credential-provider-process": "3.465.0", + "@aws-sdk/credential-provider-sso": "3.465.0", + "@aws-sdk/credential-provider-web-identity": "3.465.0", + "@aws-sdk/types": "3.465.0", + "@smithy/credential-provider-imds": "^2.0.0", + "@smithy/property-provider": "^2.0.0", + "@smithy/shared-ini-file-loader": "^2.0.6", + "@smithy/types": "^2.5.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/credential-provider-process": { + "version": "3.465.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.465.0.tgz", + "integrity": "sha512-YE6ZrRYwvb8969hWQnr4uvOJ8RU0JrNsk3vWTe/czly37ioZUEhi8jmpQp4f2mX/6U6buoFGWu5Se3VCdw2SFQ==", + "dependencies": { + "@aws-sdk/types": "3.465.0", + "@smithy/property-provider": "^2.0.0", + "@smithy/shared-ini-file-loader": "^2.0.6", + "@smithy/types": "^2.5.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/credential-provider-sso": { + "version": "3.465.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.465.0.tgz", + "integrity": "sha512-tLIP/4JQIJpn8yIg6RZRQ2nmvj5i4wLZvYvY4RtaFv2JrQUkmmTfyOZJuOBrIFRwJjx0fHmFu8DJjcOhMzllIQ==", + "dependencies": { + "@aws-sdk/client-sso": "3.465.0", + "@aws-sdk/token-providers": "3.465.0", + "@aws-sdk/types": "3.465.0", + "@smithy/property-provider": "^2.0.0", + "@smithy/shared-ini-file-loader": "^2.0.6", + "@smithy/types": "^2.5.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/credential-provider-web-identity": { + "version": "3.465.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.465.0.tgz", + "integrity": "sha512-B4Y75fMTZIniEU0yyqat+9NsQbYlXdqP5Y3bShkaG3pGLOHzF/xMlWuG+D3kkQ806PLYi+BgfVls4BcO+NyVcA==", + "dependencies": { + "@aws-sdk/types": "3.465.0", + "@smithy/property-provider": "^2.0.0", + "@smithy/types": "^2.5.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/middleware-host-header": { + "version": "3.465.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.465.0.tgz", + "integrity": "sha512-nnGva8eplwEJqdVzcb+xF2Fwua0PpiwxMEvpnIy73gNbetbJdgFIprryMLYes00xzJEqnew+LWdpcd3YyS34ZA==", + "dependencies": { + "@aws-sdk/types": "3.465.0", + "@smithy/protocol-http": "^3.0.9", + "@smithy/types": "^2.5.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/middleware-logger": { + "version": "3.465.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.465.0.tgz", + "integrity": "sha512-aGMx1aSlzDDgjZ7fSxLhGD5rkyCfHwq04TSB5fQAgDBqUjj4IQXZwmNglX0sLRmArXZtDglUVESOfKvTANJTPg==", + "dependencies": { + "@aws-sdk/types": "3.465.0", + "@smithy/types": "^2.5.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/middleware-recursion-detection": { + "version": "3.465.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.465.0.tgz", + "integrity": "sha512-ol3dlsTnryBhV5qkUvK5Yg3dRaV1NXIxYJaIkShrl8XAv4wRNcDJDmO5NYq5eVZ3zgV1nv6xIpZ//dDnnf6Z+g==", + "dependencies": { + "@aws-sdk/types": "3.465.0", + "@smithy/protocol-http": "^3.0.9", + "@smithy/types": "^2.5.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/middleware-sdk-sts": { + "version": "3.465.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-sts/-/middleware-sdk-sts-3.465.0.tgz", + "integrity": "sha512-PmTM5ycUe1RLAPrQXLCR8JzKamJuKDB0aIW4rx4/skurzWsEGRI47WHggf9N7sPie41IBGUhRbXcf7sfPjvI3Q==", + "dependencies": { + "@aws-sdk/middleware-signing": "3.465.0", + "@aws-sdk/types": "3.465.0", + "@smithy/types": "^2.5.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/middleware-signing": { + "version": "3.465.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-signing/-/middleware-signing-3.465.0.tgz", + "integrity": "sha512-d90KONWXSC3jA0kqJ6u8ygS4LoMg1TmSM7bPhHyibJVAEhnrlB4Aq1CWljNbbtphGpdKy5/XRM9O0/XCXWKQ4w==", + "dependencies": { + "@aws-sdk/types": "3.465.0", + "@smithy/property-provider": "^2.0.0", + "@smithy/protocol-http": "^3.0.9", + "@smithy/signature-v4": "^2.0.0", + "@smithy/types": "^2.5.0", + "@smithy/util-middleware": "^2.0.6", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/middleware-user-agent": { + "version": "3.465.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.465.0.tgz", + "integrity": "sha512-1MvIWMj2nktLOJN8Kh4jiTK28oL85fTeoXHZ+V8xYMzont6C6Y8gQPtg7ka+RotHwqWMrovfnANisnX8EzEP/Q==", + "dependencies": { + "@aws-sdk/types": "3.465.0", + "@aws-sdk/util-endpoints": "3.465.0", + "@smithy/protocol-http": "^3.0.9", + "@smithy/types": "^2.5.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/region-config-resolver": { + "version": "3.465.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/region-config-resolver/-/region-config-resolver-3.465.0.tgz", + "integrity": "sha512-h0Phd2Ae873dsPSWuxqxz2yRC5NMeeWxQiJPh4j42HF8g7dZK7tMQPkYznAoA/BzSBsEX87sbr3MmigquSyUTA==", + "dependencies": { + "@smithy/node-config-provider": "^2.1.5", + "@smithy/types": "^2.5.0", + "@smithy/util-config-provider": "^2.0.0", + "@smithy/util-middleware": "^2.0.6", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/token-providers": { + "version": "3.465.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.465.0.tgz", + "integrity": "sha512-NaZbsyLs3whzRHGV27hrRwEdXB/tEK6tqn/aCNBy862LhVzocY1A+eYLKrnrvpraOOd2vyAuOtvvB3RMIdiL6g==", + "dependencies": { + "@aws-crypto/sha256-browser": "3.0.0", + "@aws-crypto/sha256-js": "3.0.0", + "@aws-sdk/middleware-host-header": "3.465.0", + "@aws-sdk/middleware-logger": "3.465.0", + "@aws-sdk/middleware-recursion-detection": "3.465.0", + "@aws-sdk/middleware-user-agent": "3.465.0", + "@aws-sdk/region-config-resolver": "3.465.0", + "@aws-sdk/types": "3.465.0", + "@aws-sdk/util-endpoints": "3.465.0", + "@aws-sdk/util-user-agent-browser": "3.465.0", + "@aws-sdk/util-user-agent-node": "3.465.0", + "@smithy/config-resolver": "^2.0.18", + "@smithy/fetch-http-handler": "^2.2.6", + "@smithy/hash-node": "^2.0.15", + "@smithy/invalid-dependency": "^2.0.13", + "@smithy/middleware-content-length": "^2.0.15", + "@smithy/middleware-endpoint": "^2.2.0", + "@smithy/middleware-retry": "^2.0.20", + "@smithy/middleware-serde": "^2.0.13", + "@smithy/middleware-stack": "^2.0.7", + "@smithy/node-config-provider": "^2.1.5", + "@smithy/node-http-handler": "^2.1.9", + "@smithy/property-provider": "^2.0.0", + "@smithy/protocol-http": "^3.0.9", + "@smithy/shared-ini-file-loader": "^2.0.6", + "@smithy/smithy-client": "^2.1.15", + "@smithy/types": "^2.5.0", + "@smithy/url-parser": "^2.0.13", + "@smithy/util-base64": "^2.0.1", + "@smithy/util-body-length-browser": "^2.0.0", + "@smithy/util-body-length-node": "^2.1.0", + "@smithy/util-defaults-mode-browser": "^2.0.19", + "@smithy/util-defaults-mode-node": "^2.0.25", + "@smithy/util-endpoints": "^1.0.4", + "@smithy/util-retry": "^2.0.6", + "@smithy/util-utf8": "^2.0.2", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/types": { + "version": "3.465.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.465.0.tgz", + "integrity": "sha512-Clqu2eD50OOzwSftGpzJrIOGev/7VJhJpc02SeS4cqFgI9EVd+rnFKS/Ux0kcwjLQBMiPcCLtql3KAHApFHAIA==", + "dependencies": { + "@smithy/types": "^2.5.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/util-endpoints": { + "version": "3.465.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.465.0.tgz", + "integrity": "sha512-lDpBN1faVw8Udg5hIo+LJaNfllbBF86PCisv628vfcggO8/EArL/v2Eos0KeqVT8yaINXCRSagwfo5TNTuW0KQ==", + "dependencies": { + "@aws-sdk/types": "3.465.0", + "@smithy/util-endpoints": "^1.0.4", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/util-user-agent-browser": { + "version": "3.465.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.465.0.tgz", + "integrity": "sha512-RM+LjkIsmUCBJ4yQeBnkJWJTjPOPqcNaKv8bpZxatIHdvzGhXLnWLNi3qHlBsJB2mKtKRet6nAUmKmzZR1sDzA==", + "dependencies": { + "@aws-sdk/types": "3.465.0", + "@smithy/types": "^2.5.0", + "bowser": "^2.11.0", + "tslib": "^2.5.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/util-user-agent-node": { + "version": "3.465.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.465.0.tgz", + "integrity": "sha512-XsHbq7gLCiGdy6FQ7/5nGslK0ij3Iuh051djuIICvNurlds5cqKLiBe63gX3IUUwxJcrKh4xBGviQJ52KdVSeg==", + "dependencies": { + "@aws-sdk/types": "3.465.0", + "@smithy/node-config-provider": "^2.1.5", + "@smithy/types": "^2.5.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + }, + "peerDependencies": { + "aws-crt": ">=1.0.0" + }, + "peerDependenciesMeta": { + "aws-crt": { + "optional": true + } + } + }, "node_modules/@aws-sdk/client-s3": { "version": "3.458.0", "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.458.0.tgz", @@ -460,6 +930,35 @@ "node": ">=14.0.0" } }, + "node_modules/@aws-sdk/endpoint-cache": { + "version": "3.465.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/endpoint-cache/-/endpoint-cache-3.465.0.tgz", + "integrity": "sha512-0cuotk23hVSrqxHkJ3TTWC9MVMRgwlUvCatyegJEauJnk8kpLSGXE5KVdExlUBwShGNlj7ac29okZ9m17iTi5Q==", + "dependencies": { + "mnemonist": "0.38.3", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/lib-dynamodb": { + "version": "3.465.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/lib-dynamodb/-/lib-dynamodb-3.465.0.tgz", + "integrity": "sha512-l/jxVkdoetTsBaWxffC8X9Gp0k5m7I6yM0dMow+p0fCzOip5JVkSBI08dgQZGD0ECzxBEwW8vxdu84BLn2sS0A==", + "dependencies": { + "@aws-sdk/util-dynamodb": "3.465.0", + "@smithy/smithy-client": "^2.1.15", + "@smithy/types": "^2.5.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-dynamodb": "^3.0.0" + } + }, "node_modules/@aws-sdk/middleware-bucket-endpoint": { "version": "3.451.0", "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-bucket-endpoint/-/middleware-bucket-endpoint-3.451.0.tgz", @@ -477,6 +976,34 @@ "node": ">=14.0.0" } }, + "node_modules/@aws-sdk/middleware-endpoint-discovery": { + "version": "3.465.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-endpoint-discovery/-/middleware-endpoint-discovery-3.465.0.tgz", + "integrity": "sha512-11Eq9BIKBHTi+y8ClgFMDbv4NyFEqUHm9CuAaPHy1PBYItp6ajqjjZi7yIjsM6KHba1br9SqtsuGfaFnWxWRSg==", + "dependencies": { + "@aws-sdk/endpoint-cache": "3.465.0", + "@aws-sdk/types": "3.465.0", + "@smithy/node-config-provider": "^2.1.5", + "@smithy/protocol-http": "^3.0.9", + "@smithy/types": "^2.5.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/middleware-endpoint-discovery/node_modules/@aws-sdk/types": { + "version": "3.465.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.465.0.tgz", + "integrity": "sha512-Clqu2eD50OOzwSftGpzJrIOGev/7VJhJpc02SeS4cqFgI9EVd+rnFKS/Ux0kcwjLQBMiPcCLtql3KAHApFHAIA==", + "dependencies": { + "@smithy/types": "^2.5.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/@aws-sdk/middleware-expect-continue": { "version": "3.451.0", "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-expect-continue/-/middleware-expect-continue-3.451.0.tgz", @@ -738,6 +1265,20 @@ "node": ">=14.0.0" } }, + "node_modules/@aws-sdk/util-dynamodb": { + "version": "3.465.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-dynamodb/-/util-dynamodb-3.465.0.tgz", + "integrity": "sha512-Zi3ThmR7zfNH32CV5nsVf4Ce6gkX1N/wnxMdTQv2qnu1zoA/6o7RPzrGs92EafDHugvRXJFoWM5oHjYBreK61Q==", + "dependencies": { + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-dynamodb": "^3.0.0" + } + }, "node_modules/@aws-sdk/util-endpoints": { "version": "3.451.0", "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.451.0.tgz", @@ -6258,6 +6799,14 @@ "node": ">=10" } }, + "node_modules/mnemonist": { + "version": "0.38.3", + "resolved": "https://registry.npmjs.org/mnemonist/-/mnemonist-0.38.3.tgz", + "integrity": "sha512-2K9QYubXx/NAjv4VLq1d1Ly8pWNC5L3BrixtdkyTegXWJIqY+zLNDhhX/A+ZwWt70tB1S8H4BE8FLYEFyNoOBw==", + "dependencies": { + "obliterator": "^1.6.1" + } + }, "node_modules/ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", @@ -6400,6 +6949,11 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/obliterator": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/obliterator/-/obliterator-1.6.1.tgz", + "integrity": "sha512-9WXswnqINnnhOG/5SLimUlzuU1hFJUc8zkwyD59Sd+dPOMf05PmnYG/d6Q7HZ+KmgkZJa1PxRso6QdM3sTNHig==" + }, "node_modules/on-exit-leak-free": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/on-exit-leak-free/-/on-exit-leak-free-2.1.0.tgz", diff --git a/package.json b/package.json index 0027bf8..3e79ca6 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,9 @@ "supertest": "^6.3.3" }, "dependencies": { + "@aws-sdk/client-dynamodb": "^3.465.0", "@aws-sdk/client-s3": "^3.458.0", + "@aws-sdk/lib-dynamodb": "^3.465.0", "aws-jwt-verify": "^4.0.0", "compression": "^1.7.4", "content-type": "^1.0.5", diff --git a/src/model/data/aws/ddbDocClient.js b/src/model/data/aws/ddbDocClient.js new file mode 100644 index 0000000..70ee2ba --- /dev/null +++ b/src/model/data/aws/ddbDocClient.js @@ -0,0 +1,66 @@ +const { DynamoDBClient } = require('@aws-sdk/client-dynamodb'); +// Helper library for working with converting DynamoDB types to/from JS +const { DynamoDBDocumentClient } = require('@aws-sdk/lib-dynamodb'); + +const logger = require('../../../logger'); + +/** + * If AWS credentials are configured in the environment, use them. Normally when we connect to DynamoDB from a deployment in AWS, we won't bother with this. But if you're testing locally, you'll need + * these, or if you're connecting to LocalStack or DynamoDB Local + * @returns Object | undefined + */ +const getCredentials = () => { + if (process.env.AWS_ACCESS_KEY_ID && process.env.AWS_SECRET_ACCESS_KEY) { + // See https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-dynamodb/interfaces/dynamodbclientconfig.html#credentials + const credentials = { + accessKeyId: process.env.AWS_ACCESS_KEY_ID, + secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY, + // Optionally include the AWS Session Token, too (e.g., if you're connecting to AWS from your laptop). + // Not all situations require this, so we won't check for it above, just use it if it is present. + sessionToken: process.env.AWS_SESSION_TOKEN, + }; + logger.debug('Using extra DynamoDB Credentials'); + return credentials; + } +}; + +/** + * If an AWS DynamoDB Endpoint is configured in the environment, use it. + * @returns string | undefined + */ +const getDynamoDBEndpoint = () => { + if (process.env.AWS_DYNAMODB_ENDPOINT_URL) { + logger.debug( + { endpoint: process.env.AWS_DYNAMODB_ENDPOINT_URL }, + 'Using alternate DynamoDB endpoint' + ); + return process.env.AWS_DYNAMODB_ENDPOINT_URL; + } +}; + +// Create and configure an Amazon DynamoDB client object. +const ddbClient = new DynamoDBClient({ + region: process.env.AWS_REGION, + endpoint: getDynamoDBEndpoint(), + credentials: getCredentials(), +}); + +// Instead of exposing the ddbClient directly, we'll wrap it with a helper +// that will simplify converting data to/from DynamoDB and JavaScript (i.e. +// marshalling and unmarshalling typed attribute data) +const ddbDocClient = DynamoDBDocumentClient.from(ddbClient, { + marshallOptions: { + // Whether to automatically convert empty strings, blobs, and sets to `null`. + convertEmptyValues: false, // false, by default. + // Whether to remove undefined values while marshalling. + removeUndefinedValues: false, // false, by default. + // Whether to convert typeof object to map attribute. + convertClassInstanceToMap: true, // we have to set this to `true` for LocalStack + }, + unmarshallOptions: { + // Whether to return numbers as a string instead of converting them to native JavaScript numbers. + wrapNumbers: false, // false, by default. + }, +}); + +module.exports = ddbDocClient; \ No newline at end of file diff --git a/src/model/data/aws/index.js b/src/model/data/aws/index.js index c4f2789..8634d2b 100644 --- a/src/model/data/aws/index.js +++ b/src/model/data/aws/index.js @@ -1,38 +1,50 @@ // temporary use of memory-db until we add DynamoDB -const MemoryDB = require('../memory/memory-db'); +const ddbDocClient = require('./ddbDocClient'); const s3Client = require('./s3Client'); +const { PutCommand, GetCommand, QueryCommand, DeleteCommand } = require('@aws-sdk/lib-dynamodb'); const { PutObjectCommand, GetObjectCommand, DeleteObjectCommand } = require('@aws-sdk/client-s3'); const logger = require('../../../logger'); -// Create two in-memory databases: one for fragment metadata and the other for raw data -const metadata = new MemoryDB(); - -// Write a fragment's metadata to memory db. Returns a Promise -async function writeFragment(fragment) { - // Create the PUT API params from our details +// Writes a fragment to DynamoDB. Returns a Promise. +function writeFragment(fragment) { + // Configure our PUT params, with the name of the table and item (attributes and keys) const params = { - Bucket: process.env.AWS_S3_BUCKET_NAME, - Key: `${fragment.ownerId}/${fragment.id}`, - Body: JSON.stringify(fragment), + TableName: process.env.AWS_DYNAMODB_TABLE_NAME, + Item: fragment, }; - // Create a PUT Object command to send to S3 - const command = new PutObjectCommand(params); + // Create a PUT command to send to DynamoDB + const command = new PutCommand(params); try { - // Use our client to send the command - await s3Client.send(command); + return ddbDocClient.send(command); } catch (err) { - // If anything goes wrong, log info to debug - const { Bucket, Key } = params; - logger.error({ err, Bucket, Key }, 'Error uploading fragment data to S3'); - throw new Error('unable to upload fragment data'); + logger.warn({ err, params, fragment }, 'error writing fragment to DynamoDB'); + throw err; } } -// Read a fragment's metadata from memory db. Returns a Promise -function readFragment(ownerId, id) { - return metadata.get(ownerId, id); +// Reads a fragment from DynamoDB. Returns a Promise +async function readFragment(ownerId, id) { + // Configure our GET params, with the name of the table and key (partition key + sort key) + const params = { + TableName: process.env.AWS_DYNAMODB_TABLE_NAME, + Key: { ownerId, id }, + }; + + // Create a GET command to send to DynamoDB + const command = new GetCommand(params); + + try { + // Wait for the data to come back from AWS + const data = await ddbDocClient.send(command); + // We may or may not get back any data (e.g., no item found for the given key). + // If we get back an item (fragment), we'll return it. Otherwise we'll return `undefined`. + return data?.Item; + } catch (err) { + logger.warn({ err, params }, 'error reading fragment from DynamoDB'); + throw err; + } } // Write a fragment's data buffer to S3 Object in a Bucket @@ -98,39 +110,81 @@ const streamToBuffer = (stream) => stream.on('end', () => resolve(Buffer.concat(chunks))); }); -// Get a list of fragment ids/objects for the given user from memory db. Returns a Promise +// Get a list of fragments, either ids-only, or full Objects, for the given user. +// Returns a Promise|Array|undefined> async function listFragments(ownerId, expand = false) { - const fragments = await metadata.query(ownerId); + // Configure our QUERY params, with the name of the table and the query expression + const params = { + TableName: process.env.AWS_DYNAMODB_TABLE_NAME, + // Specify that we want to get all items where the ownerId is equal to the + // `:ownerId` that we'll define below in the ExpressionAttributeValues. + KeyConditionExpression: 'ownerId = :ownerId', + // Use the `ownerId` value to do the query + ExpressionAttributeValues: { + ':ownerId': ownerId, + }, + }; - // If we don't get anything back, or are supposed to give expanded fragments, return - if (expand || !fragments) { - return fragments; + // Limit to only `id` if we aren't supposed to expand. Without doing this + // we'll get back every attribute. The projection expression defines a list + // of attributes to return, see: + // https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Expressions.ProjectionExpressions.html + if (!expand) { + params.ProjectionExpression = 'id'; } - // Otherwise, map to only send back the ids - return fragments.map((fragment) => fragment.id); + // Create a QUERY command to send to DynamoDB + const command = new QueryCommand(params); + + try { + // Wait for the data to come back from AWS + const data = await ddbDocClient.send(command); + + // If we haven't expanded to include all attributes, remap this array from + // [ {"id":"b9e7a264-630f-436d-a785-27f30233faea"}, {"id":"dad25b07-8cd6-498b-9aaf-46d358ea97fe"} ,... ] to + // [ "b9e7a264-630f-436d-a785-27f30233faea", "dad25b07-8cd6-498b-9aaf-46d358ea97fe", ... ] + return !expand ? data?.Items.map((item) => item.id) : data?.Items + } catch (err) { + logger.error({ err, params }, 'error getting all fragments for user from DynamoDB'); + throw err; + } } // Delete a fragment's metadata and data from S3 async function deleteFragment(ownerId, id) { - // Create the DELETE API params from our details - const params = { + // Create the DELETE API params from our details for S3 and DynamoDB + const S3Params = { Bucket: process.env.AWS_S3_BUCKET_NAME, Key: `${ownerId}/${id}`, }; - // Create a DELETE Object command to send to S3 - const command = new DeleteObjectCommand(params); + const DynamoParams = { + TableName: process.env.AWS_DYNAMODB_TABLE_NAME, + Key: { ownerId, id } + }; + + // Create a DELETE Object command to send to S3 and DynamoDB + const S3Command = new DeleteObjectCommand(S3Params); + const DynamoCommand = new DeleteCommand(DynamoParams); try { - readFragmentData(ownerId, id); // check if fragment exists - // Use our client to send the command - await s3Client.send(command); + // Use our client to send the DynamoDB command + await ddbDocClient.send(DynamoCommand); } catch (err) { // If anything goes wrong, log info to debug - const { Bucket, Key } = params; + const { TableName, Key } = DynamoParams; + logger.error({ err, TableName, Key }, 'Error deleting fragment data from DynamoDB'); + throw new Error('unable to delete fragment data from DynamoDB'); + } + + try { + // Use our client to send the S3 command + await s3Client.send(S3Command); + } catch (err) { + // If anything goes wrong, log info to debug + const { Bucket, Key } = S3Params; logger.error({ err, Bucket, Key }, 'Error deleting fragment data from S3'); - throw new Error('unable to delete fragment data'); + throw new Error('unable to delete fragment data from S3'); } } diff --git a/src/model/fragment.js b/src/model/fragment.js index b2badc9..74c4b67 100644 --- a/src/model/fragment.js +++ b/src/model/fragment.js @@ -43,7 +43,6 @@ class Fragment { this.ownerId = ownerId; this.created = new Date(created); this.updated = new Date(updated); - if (Fragment.isSupportedType(type) == true) this.type = type; @@ -157,6 +156,16 @@ class Fragment { } } + /** + * Updates the fragment's dates to the string format + * @param none + * @returns nothing + */ + async convertDatestoDateString() { + this.created = this.created.toDateString(); + this.updated = this.updated.toDateString(); + } + /** * Returns the mime type (e.g., without encoding) for the fragment's type: * "text/html; charset=utf-8" -> "text/html" diff --git a/src/routes/api/byId.js b/src/routes/api/byId.js index 506f01d..3781dc0 100644 --- a/src/routes/api/byId.js +++ b/src/routes/api/byId.js @@ -1,4 +1,4 @@ -const { readFragment, readFragmentData } = require('../../model/data/memory'); +const { readFragment, readFragmentData } = require('../../model/data'); const { createErrorResponse } = require('../../response'); const logger = require('../../logger'); var MarkdownIt = require('markdown-it'); diff --git a/src/routes/api/delete.js b/src/routes/api/delete.js index 925cd99..cb09a58 100644 --- a/src/routes/api/delete.js +++ b/src/routes/api/delete.js @@ -1,5 +1,5 @@ const { createSuccessResponse, createErrorResponse } = require('../../response'); -const { deleteFragment } = require('../../model/data/memory'); +const { deleteFragment } = require('../../model/data'); const logger = require('../../logger'); // Delete fragment for the current user diff --git a/src/routes/api/get.js b/src/routes/api/get.js index 47403f4..71dc050 100644 --- a/src/routes/api/get.js +++ b/src/routes/api/get.js @@ -1,5 +1,5 @@ const { createSuccessResponse } = require('../../response'); -const { listFragments } = require('../../model/data/memory'); +const { listFragments } = require('../../model/data'); // Gets a list of fragments for the current user module.exports = async (req, res) => { diff --git a/src/routes/api/info.js b/src/routes/api/info.js index 5bf0d77..94f4e01 100644 --- a/src/routes/api/info.js +++ b/src/routes/api/info.js @@ -1,5 +1,5 @@ -const { readFragment } = require('../../model/data/memory'); -const { createErrorResponse } = require('../../response'); +const { readFragment } = require('../../model/data'); +const { createSuccessResponse, createErrorResponse } = require('../../response'); const logger = require('../../logger'); // Gets a fragment based on its id @@ -7,7 +7,10 @@ module.exports = async (req, res) => { const fragmentMetadata = await readFragment(req.user, req.params.id); if (fragmentMetadata) { - res.status(200).json(fragmentMetadata); + let data = { + fragment: fragmentMetadata + }; + res.status(200).json(createSuccessResponse(data)); } else { logger.warn(`Requested fragment does not exist in the memory.`); logger.debug(`Fragment not found with ID ${req.params.id}`); diff --git a/src/routes/api/post.js b/src/routes/api/post.js index 3ca20c9..d5c1bb4 100644 --- a/src/routes/api/post.js +++ b/src/routes/api/post.js @@ -1,7 +1,7 @@ const { createSuccessResponse, createErrorResponse } = require('../../response'); const { Fragment } = require('../../model/fragment'); const contentType = require('content-type'); -const { writeFragment, writeFragmentData } = require('../../model/data/memory'); +const { writeFragment, writeFragmentData } = require('../../model/data'); const logger = require('../../logger'); // Creates a fragment for the current user @@ -21,6 +21,8 @@ module.exports = async (req, res) => { const ownerId = req.user; const fragment = new Fragment({ ownerId, type }); fragment.updateSize(req.body); + if (process.env.AWS_REGION) //save dates as strings when storing with AWS + fragment.convertDatestoDateString(); await writeFragment(fragment); await writeFragmentData(ownerId, fragment.id, req.body); diff --git a/tests/integration/lab-10-dynamodb.hurl b/tests/integration/lab-10-dynamodb.hurl new file mode 100644 index 0000000..1d6311d --- /dev/null +++ b/tests/integration/lab-10-dynamodb.hurl @@ -0,0 +1,126 @@ + +# POST a new JSON fragment to http://localhost:8080 as an authorized user. +#The fragment's body should be the JSON value, { "service": "DynamoDB" }. +POST http://localhost:8080/v1/fragments +# Send a plain text fragment +Content-Type: application/json +# Include HTTP Basic Auth credentials using the [BasicAuth] section +[BasicAuth] +user1@email.com:password1 + +{ "service": "DynamoDB" } + + +# Confirm that the server returns a 201 +# Capture the Location header value and the fragment's id in variables named fragment1_url and fragment1_id. +HTTP/1.1 201 +[Captures] +fragment1_url: header "Location" +fragment1_id: jsonpath "$.fragment.id" + + +# GET the fragment info (i.e., metadata) for the fragment you just created using the Location URL/info as an authorized user +GET {{fragment1_url}}/info +# Include HTTP Basic Auth credentials using the [BasicAuth] section +[BasicAuth] +user1@email.com:password1 + + +# Confirm that the server returns a 200 and that all of the metadata properties match what you expect. +HTTP/1.1 200 +[Asserts] +jsonpath "$.status" == "ok" +jsonpath "$.fragment.id" == {{fragment1_id}} +# Our ownerId hash is a hex encoded string +jsonpath "$.fragment.ownerId" matches "^[0-9a-fA-F]+$" +jsonpath "$.fragment.created" isString +jsonpath "$.fragment.updated" isString +jsonpath "$.fragment.type" == "application/json" +jsonpath "$.fragment.size" == 25 + + +# POST a second Markdown fragment to http://localhost:8080 as the same authorized user. +# The fragment's body should be the Markdown value, DynamoDB is **great**. +POST http://localhost:8080/v1/fragments +# Send a plain text fragment +Content-Type: text/markdown +# Include HTTP Basic Auth credentials using the [BasicAuth] section +[BasicAuth] +user1@email.com:password1 + +`DynamoDB is **great**.` + + +# Confirm that the server returns a 201 +# Capture the Location header value and the second id in variables named fragment2_url and fragment2_id +HTTP/1.1 201 +[Captures] +fragment2_url: header "Location" +fragment2_id: jsonpath "$.fragment.id" + + +# GET the fragment info (i.e., metadata) you just created using the url/info as an authorized user +GET {{fragment2_url}}/info +# Include HTTP Basic Auth credentials using the [BasicAuth] section +[BasicAuth] +user1@email.com:password1 + + +# Confirm that the server returns a 200 and that all of the metadata properties match what you expect. +HTTP/1.1 200 +[Asserts] +jsonpath "$.status" == "ok" +jsonpath "$.fragment.id" == {{fragment2_id}} +# Our ownerId hash is a hex encoded string +jsonpath "$.fragment.ownerId" matches "^[0-9a-fA-F]+$" +jsonpath "$.fragment.created" isString +jsonpath "$.fragment.updated" isString +jsonpath "$.fragment.type" == "text/markdown" +jsonpath "$.fragment.size" == 22 + + +# GET all of the fragments for the same authorized user without expanding them (i.e., just get back the IDs) +GET http://localhost:8080/v1/fragments +[BasicAuth] +user1@email.com:password1 + + +# Confirm that the list of fragments includes the two id values you captured above +HTTP/1.1 200 +[Asserts] +jsonpath "$.status" == "ok" +jsonpath "$.fragments" includes {{fragment1_id}} +jsonpath "$.fragments" includes {{fragment2_id}} + + +# DELETE the first fragment you created above +DELETE {{fragment1_url}} +[BasicAuth] +user1@email.com:password1 + + +# Confirm that the server returns a 200 +HTTP/1.1 200 + + +# Try to GET the first fragment again using the url you captured above as the authorized user +GET {{fragment1_url}}/info +[BasicAuth] +user1@email.com:password1 + + +# Confirm that the server returns a 404, since the fragment should be deleted +HTTP/1.1 404 + + +# GET all of the fragments for the same authorized user without expanding them (i.e., just get back the IDs) +GET http://localhost:8080/v1/fragments +[BasicAuth] +user1@email.com:password1 + + +# Confirm that the first id is NOT included but that the second id is (i.e., that the second was not deleted). +HTTP/1.1 200 +[Asserts] +jsonpath "$.fragments" not includes {{fragment1_id}} +jsonpath "$.fragments" includes {{fragment2_id}} \ No newline at end of file diff --git a/tests/integration/lab-9-s3.hurl b/tests/integration/lab-9-s3.hurl index 71bbfa8..af6486c 100644 --- a/tests/integration/lab-9-s3.hurl +++ b/tests/integration/lab-9-s3.hurl @@ -46,11 +46,4 @@ user1@email.com:password1 # Confirm that the server returns a 404, since the fragment should be deleted. -HTTP/1.1 404 - - -#DELETE $url -#Authorization: Bearer - -# Confirm that the server returns a 404 since the fragment should be deleted -#ASSERT status 404 \ No newline at end of file +HTTP/1.1 404 \ No newline at end of file