feature-kinesis-lambda-cdk-typescript

kinesis-lambda-cdk-typescript/README.md

@@ -0,0 +1,65 @@
+# Amazon Kinesis Data Stream triggers AWS Lambda with simple Event Source Mapping
+
+This pattern creates an Amazon [Kinesis Data Stream](https://docs.aws.amazon.com/streams/latest/dev/introduction.html) and an AWS [Lambda function](https://docs.aws.amazon.com/lambda/latest/dg/welcome.html), using the AWS Cloud Development Kit (AWS CDK) in TypeScript.
+
+Learn more about this pattern at Serverless Land Patterns: [https://serverlessland.com/patterns/kinesis-lambda-cdk-typescript](https://serverlessland.com/patterns/kinesis-lambda-cdk-typescript).
+
+Important: this application uses various AWS services and there are costs associated with these services after the Free Tier usage - please see the [AWS Pricing page](https://aws.amazon.com/pricing/) for details. You are responsible for any AWS costs incurred. No warranty is implied in this example.
+
+## Requirements
+
+* [Create an AWS account](https://portal.aws.amazon.com/gp/aws/developer/registration/index.html) if you do not already have one and log in. The IAM user that you use must have sufficient permissions to make necessary AWS service calls and manage AWS resources.
+* [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html) installed and configured
+* [Git Installed](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git)
+* [AWS CDK Toolkit](https://docs.aws.amazon.com/cdk/latest/guide/cli.html) installed and configured
+* [Node 18+](https://nodejs.org/en/download/current) installed
+* [Docker](https://docs.docker.com/get-docker/) Installed
+
+## Deployment Instructions
+
+1. Create a new directory, navigate to that directory in a terminal and clone the GitHub repository:
+    ```
+    git clone https://github.com/aws-samples/serverless-patterns
+    ```
+2. Change directory to the pattern directory:
+    ```
+    cd serverless-patterns/kinesis-lambda-cdk-typescript
+    ```
+3. From the command line, use AWS CDK to deploy the AWS resources for the serverless application as specified in the MyServerlessApplicationStack.java file:
+    ```
+    cdk deploy
+    ```
+
+## How it works
+
+When data is sent to the Kinesis data stream, the Lambda function will be triggered and process data from the stream.
+
+## Testing
+
+From the command line, run the following command to send a single data record to the Kinesis data stream. Note that you must edit the {MyServerlessApplicationStack.KinesisLambda-KinesisStream} with the Kinesis data stream ARN that is deployed. This is provided in the MyServerlessApplicationStack deployment outputs.
+
+```
+aws kinesis put-record --stream-name {MyServerlessApplicationStack.KinesisLambda-KinesisStream} --partition-key 123 --data testdata
+```
+
+You will see output as below:
+```
+{
+    "ShardId": "shardId-000000000000",
+    "SequenceNumber":"49644233305086035939298432546400484303012441668144070658",
+    "EncryptionType": "KMS"
+}
+```
+
+This will trigger the lambda which you can verify by checking the CloudWatch logs under tab Monitor.
+
+## Cleanup
+
+1. From the command line the AWS CDK to delete the Serverless application stack
+   ```
+   cdk destroy
+   ```
+----
+Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+
+SPDX-License-Identifier: MIT-0

kinesis-lambda-cdk-typescript/example-pattern.json

@@ -0,0 +1,59 @@
+{
+  "title": "Amazon Kinesis Data Streams triggers AWS Lambda",
+  "description": "Create a serverless app using Amazon Kinesis Data Streams and AWS Lambda.",
+  "language": "TypeScript",
+  "level": "200",
+  "framework": "CDK",
+  "introBox": {
+    "headline": "How it works",
+    "text": [
+      "This pattern creates an Amazon Kinesis Data Stream that integrates with AWS Lambda using the AWS CDK in TypeScript.",
+      "In this example, the Kinesis data stream is used as an event source for the AWS Lambda (Node) function which",
+      "processes the data from the stream and catches any failures."
+    ]
+  },
+  "gitHub": {
+    "template": {
+      "repoURL": "https://github.com/aws-samples/serverless-patterns/tree/main/kinesis-lambda-cdk-typescript",
+      "templateURL": "patterns/kinesis-lambda-cdk-typescript",
+      "projectFolder": "kinesis-lambda-cdk-typescript",
+      "templateFile": "kinesis-lambda-cdk-typescript/src/bin/app.ts"
+    }
+  },
+  "resources": {
+    "bullets": [
+      {
+        "text": "Using AWS Lambda with Amazon Kinesis",
+        "link": "https://docs.aws.amazon.com/lambda/latest/dg/with-kinesis.html"
+      },
+      {
+        "text": "Working with the AWS CDK in TypeScript",
+        "link": "https://docs.aws.amazon.com/cdk/v2/guide/work-with-cdk-typescript.html"
+      }
+    ]
+  },
+  "deploy": {
+    "text": [
+      "sam deploy"
+    ]
+  },
+  "testing": {
+    "text": [
+      "See the Github repo for detailed testing instructions."
+    ]
+  },
+  "cleanup": {
+    "text": [
+      "Delete the stack: <code>cdk delete</code>."
+    ]
+  },
+  "authors": [
+    {
+      "name": "Jayesh Shinde",
+      "image": "https://media.licdn.com/dms/image/D5603AQHVz5yI3aST-Q/profile-displayphoto-shrink_800_800/0/1681449439639?e=1699488000&v=beta&t=3kF_wIbfR5gSK5ZaLS_gmBJ3D4tqt_wUdUWEr3jK-Rg",
+      "bio": "Sr. Application Architect @AWS",
+      "linkedin": "https://www.linkedin.com/in/jayeshshinde/",
+      "twitter": "@jayeshashinde"
+    }
+  ]
+}

kinesis-lambda-cdk-typescript/src/.gitignore

@@ -0,0 +1,8 @@
+*.js
+!jest.config.js
+*.d.ts
+node_modules
+
+# CDK asset staging directory
+.cdk.staging
+cdk.out

kinesis-lambda-cdk-typescript/src/.npmignore

@@ -0,0 +1,6 @@
+*.ts
+!*.d.ts
+
+# CDK asset staging directory
+.cdk.staging
+cdk.out

kinesis-lambda-cdk-typescript/src/bin/app.ts

@@ -0,0 +1,21 @@
+#!/usr/bin/env node
+import 'source-map-support/register';
+import * as cdk from 'aws-cdk-lib';
+import { AppStack } from '../lib/app-stack';
+
+const app = new cdk.App();
+new AppStack(app, 'AppStack', {
+  /* If you don't specify 'env', this stack will be environment-agnostic.
+   * Account/Region-dependent features and context lookups will not work,
+   * but a single synthesized template can be deployed anywhere. */
+
+  /* Uncomment the next line to specialize this stack for the AWS Account
+   * and Region that are implied by the current CLI configuration. */
+  // env: { account: process.env.CDK_DEFAULT_ACCOUNT, region: process.env.CDK_DEFAULT_REGION },
+
+  /* Uncomment the next line if you know exactly what Account and Region you
+   * want to deploy the stack to. */
+  // env: { account: '123456789012', region: 'us-east-1' },
+
+  /* For more information, see https://docs.aws.amazon.com/cdk/latest/guide/environments.html */
+});

kinesis-lambda-cdk-typescript/src/cdk.json

@@ -0,0 +1,57 @@
+{
+  "app": "npx ts-node --prefer-ts-exts bin/app.ts",
+  "watch": {
+    "include": [
+      "**"
+    ],
+    "exclude": [
+      "README.md",
+      "cdk*.json",
+      "**/*.d.ts",
+      "**/*.js",
+      "tsconfig.json",
+      "package*.json",
+      "yarn.lock",
+      "node_modules",
+      "test"
+    ]
+  },
+  "context": {
+    "@aws-cdk/aws-lambda:recognizeLayerVersion": true,
+    "@aws-cdk/core:checkSecretUsage": true,
+    "@aws-cdk/core:target-partitions": [
+      "aws",
+      "aws-cn"
+    ],
+    "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true,
+    "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true,
+    "@aws-cdk/aws-ecs:arnFormatIncludesClusterName": true,
+    "@aws-cdk/aws-iam:minimizePolicies": true,
+    "@aws-cdk/core:validateSnapshotRemovalPolicy": true,
+    "@aws-cdk/aws-codepipeline:crossAccountKeyAliasStackSafeResourceName": true,
+    "@aws-cdk/aws-s3:createDefaultLoggingPolicy": true,
+    "@aws-cdk/aws-sns-subscriptions:restrictSqsDescryption": true,
+    "@aws-cdk/aws-apigateway:disableCloudWatchRole": true,
+    "@aws-cdk/core:enablePartitionLiterals": true,
+    "@aws-cdk/aws-events:eventsTargetQueueSameAccount": true,
+    "@aws-cdk/aws-iam:standardizedServicePrincipals": true,
+    "@aws-cdk/aws-ecs:disableExplicitDeploymentControllerForCircuitBreaker": true,
+    "@aws-cdk/aws-iam:importedRoleStackSafeDefaultPolicyName": true,
+    "@aws-cdk/aws-s3:serverAccessLogsUseBucketPolicy": true,
+    "@aws-cdk/aws-route53-patters:useCertificate": true,
+    "@aws-cdk/customresources:installLatestAwsSdkDefault": false,
+    "@aws-cdk/aws-rds:databaseProxyUniqueResourceName": true,
+    "@aws-cdk/aws-codedeploy:removeAlarmsFromDeploymentGroup": true,
+    "@aws-cdk/aws-apigateway:authorizerChangeDeploymentLogicalId": true,
+    "@aws-cdk/aws-ec2:launchTemplateDefaultUserData": true,
+    "@aws-cdk/aws-secretsmanager:useAttachedSecretResourcePolicyForSecretTargetAttachments": true,
+    "@aws-cdk/aws-redshift:columnId": true,
+    "@aws-cdk/aws-stepfunctions-tasks:enableEmrServicePolicyV2": true,
+    "@aws-cdk/aws-ec2:restrictDefaultSecurityGroup": true,
+    "@aws-cdk/aws-apigateway:requestValidatorUniqueId": true,
+    "@aws-cdk/aws-kms:aliasNameRef": true,
+    "@aws-cdk/aws-autoscaling:generateLaunchTemplateInsteadOfLaunchConfig": true,
+    "@aws-cdk/core:includePrefixInUniqueNameGeneration": true,
+    "@aws-cdk/aws-opensearchservice:enableOpensearchMultiAzWithStandby": true
+  }
+}

kinesis-lambda-cdk-typescript/src/jest.config.js

@@ -0,0 +1,8 @@
+module.exports = {
+  testEnvironment: 'node',
+  roots: ['<rootDir>/test'],
+  testMatch: ['**/*.test.ts'],
+  transform: {
+    '^.+\\.tsx?$': 'ts-jest'
+  }
+};

kinesis-lambda-cdk-typescript/src/lambda_code/package.json

@@ -0,0 +1,20 @@
+{
+    "name": "sample_test",
+    "version": "1.0.0",
+    "description": "",
+    "main": "app.js",
+    "author": "SAM CLI",
+    "license": "MIT",
+    "dependencies": {
+        "aws-sdk": "^2.1450.0"
+    },
+    "scripts": {
+        "test": "mocha tests/unit/",
+        "debug": "node --inspect=5859 app.js",
+        "locally": "node app.js"
+    },
+    "devDependencies": {
+        "chai": "^4.2.0",
+        "mocha": "^8.2.1"
+    }
+}

kinesis-lambda-cdk-typescript/src/lib/app-stack.ts

@@ -0,0 +1,51 @@
+import * as cdk from 'aws-cdk-lib';
+import { Construct } from 'constructs';
+import * as path from 'path';
+import { Function, AssetCode, Architecture } from 'aws-cdk-lib/aws-lambda';
+import { Runtime, StartingPosition } from 'aws-cdk-lib/aws-lambda';
+import { Stream } from 'aws-cdk-lib/aws-kinesis';
+import { KinesisEventSource } from 'aws-cdk-lib/aws-lambda-event-sources';
+import { RetentionDays } from 'aws-cdk-lib/aws-logs';
+
+
+export class AppStack extends cdk.Stack {
+  constructor(scope: Construct, id: string, props?: cdk.StackProps) {
+    super(scope, id, props);
+
+    const prefix = 'test-';
+
+    const kStream = new Stream(this, prefix + 'Kinesis-Stream', {
+      streamName: prefix + 'KinesisLambda-Stream',
+      retentionPeriod: cdk.Duration.days(7),
+      shardCount: 1,
+    });
+
+    // write lambda cdk construct to pick from lambda_code folder
+    const lambdaFn = new Function(this, prefix + 'Lambda-Function', {
+      functionName: prefix + 'KinesisLambda-Function',
+      runtime: Runtime.NODEJS_18_X,
+      handler: 'app.lambdaHandler',
+      code: new AssetCode(path.join(__dirname, '../lambda_code')),
+      environment: {
+        'STREAM_NAME': prefix + 'KinesisLambda-Stream',
+        'STRAM_ARN': kStream.streamArn,
+      },
+      architecture: Architecture.ARM_64,
+      memorySize: 128,
+      logRetention: RetentionDays.FIVE_DAYS,
+    });
+
+    // add lambda eventsouce to lambdaFn with kinesis stream
+    lambdaFn.addEventSource(new KinesisEventSource(kStream, {
+      startingPosition: StartingPosition.TRIM_HORIZON,
+      batchSize: 1,
+      maxBatchingWindow: cdk.Duration.seconds(1),
+      bisectBatchOnError: false,
+      retryAttempts: 0,
+      maxRecordAge: cdk.Duration.seconds(120),
+      enabled: true,
+      parallelizationFactor: 1,
+      reportBatchItemFailures: false,
+    }));
+  }
+}

kinesis-lambda-cdk-typescript/src/package.json

@@ -0,0 +1,28 @@
+{
+  "name": "app",
+  "version": "0.1.0",
+  "bin": {
+    "app": "bin/app.js"
+  },
+  "scripts": {
+    "build": "tsc",
+    "watch": "tsc -w",
+    "test": "jest",
+    "cdk": "cdk",
+    "lambda": "cd ./lambda_code && npm i"
+  },
+  "devDependencies": {
+    "@types/jest": "^29.5.3",
+    "@types/node": "20.4.5",
+    "aws-cdk": "^2.90.0",
+    "jest": "^29.6.2",
+    "ts-jest": "^29.1.1",
+    "ts-node": "^10.9.1",
+    "typescript": "~5.1.6"
+  },
+  "dependencies": {
+    "aws-cdk-lib": "2.90.0",
+    "constructs": "^10.0.0",
+    "source-map-support": "^0.5.21"
+  }
+}

kinesis-lambda-cdk-typescript/src/test/app.test.ts

@@ -0,0 +1,17 @@
+// import * as cdk from 'aws-cdk-lib';
+// import { Template } from 'aws-cdk-lib/assertions';
+// import * as App from '../lib/app-stack';
+
+// example test. To run these tests, uncomment this file along with the
+// example resource in lib/app-stack.ts
+test('SQS Queue Created', () => {
+//   const app = new cdk.App();
+//     // WHEN
+//   const stack = new App.AppStack(app, 'MyTestStack');
+//     // THEN
+//   const template = Template.fromStack(stack);
+
+//   template.hasResourceProperties('AWS::SQS::Queue', {
+//     VisibilityTimeout: 300
+//   });
+});

kinesis-lambda-cdk-typescript/src/tsconfig.json

@@ -0,0 +1,31 @@
+{
+  "compilerOptions": {
+    "target": "ES2020",
+    "module": "commonjs",
+    "lib": [
+      "es2020",
+      "dom"
+    ],
+    "declaration": true,
+    "strict": true,
+    "noImplicitAny": true,
+    "strictNullChecks": true,
+    "noImplicitThis": true,
+    "alwaysStrict": true,
+    "noUnusedLocals": false,
+    "noUnusedParameters": false,
+    "noImplicitReturns": true,
+    "noFallthroughCasesInSwitch": false,
+    "inlineSourceMap": true,
+    "inlineSources": true,
+    "experimentalDecorators": true,
+    "strictPropertyInitialization": false,
+    "typeRoots": [
+      "./node_modules/@types"
+    ]
+  },
+  "exclude": [
+    "node_modules",
+    "cdk.out"
+  ]
+}