Merge pull request #2385 from chaitanyagummadi/chaitanyagummadi-featu…

…re-dynamodb-pipes-sqs New serverless pattern - dynamodb-sqs-evbpipe-terraform
aws-samples · Aug 21, 2024 · 7fe178c · 7fe178c
2 parents fdfa497 + e05f803
commit 7fe178c
Show file tree

Hide file tree

Showing 5 changed files with 380 additions and 0 deletions.
diff --git a/eventbridge-pipes-dynamodbstream-to-sqs-terraform/Architecture.jpg b/eventbridge-pipes-dynamodbstream-to-sqs-terraform/Architecture.jpg
diff --git a/eventbridge-pipes-dynamodbstream-to-sqs-terraform/README.md b/eventbridge-pipes-dynamodbstream-to-sqs-terraform/README.md
@@ -0,0 +1,92 @@
+# Amazon DynamoDB Stream to Amazon SQS using Amazon EventBridge Pipes
+
+This pattern sends DynamoDB Stream records to an SQS queue using EventBridge Pipes.
+
+
+
+Learn more about this pattern at Serverless Land Patterns: https://serverlessland.com/patterns/eventbridge-pipes-dynamodbstream-to-sqs-terraform
+
+Important: this application uses various AWS services and there are costs associated with these services after the Free Tier usage - please see the [AWS Pricing page](https://aws.amazon.com/pricing/) for details. You are responsible for any AWS costs incurred. No warranty is implied in this example.
+
+## Requirements
+
+* [Create an AWS account](https://portal.aws.amazon.com/gp/aws/developer/registration/index.html) if you do not already have one and log in. The IAM user that you use must have sufficient permissions to make necessary AWS service calls and manage AWS resources.
+* [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html) installed and configured
+* [Git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) installed and configured
+* [Terraform](https://learn.hashicorp.com/tutorials/terraform/install-cli)  installed
+
+## Deployment Instructions
+
+1. Create a new directory, navigate to that directory in a terminal and clone the GitHub repository:
+    ``` 
+    git clone https://github.com/aws-samples/serverless-patterns
+    ```
+2. Change directory to the pattern directory:
+    ```
+    cd serverless-patterns/eventbridge-pipes-dynamodbstream-to-sqs-terraform
+    ```
+3. Run the follow command to initialize, download, and install the defined providers. If you are unfamiliar with the Terraform CLI, refer Terraform [documentation](https://www.terraform.io/cli/commands) to learn more about the available commands.
+    ```
+    terraform init
+    ```
+4.  Deploy the AWS resources for the pattern as specified in the `main.tf` file. 
+
+    Use the below command to review the changes before deploying.
+    ```
+    terraform plan
+    ```
+    Deploy:
+    ```
+    terraform apply --auto-approve
+    ```
+5. Note the output from the Terraform deployment process. These contain the resource names and/or ARNs which are used for testing.
+
+## How it works
+
+Please refer to the architecture diagram below:
+![End to End Architecture](Architecture.jpg)
+
+## Testing
+
+Once this stack is deployed in your AWS account, copy the DynamoDBTableName value from the output.
+
+Then, insert two records to the DynamoDB table as follows:
+Record1:
+```sh
+    aws dynamodb put-item \
+        --table-name <DynamoDBTableName> \
+        --item '{"Id":{"S":"Test1"}}'
+```
+
+Record2:
+```sh
+    aws dynamodb put-item \
+        --table-name <DynamoDBTableName> \
+        --item '{"Id":{"S":"Test2"}}'
+```
+
+When you poll the messages from the SQS queue, you would see the two message from DynamoDB stream.
+```sh
+    aws sqs receive-message --queue-url <SQS_url> --attribute-names All --message-attribute-names All --max-number-of-messages 10
+```
+
+## Cleanup
+
+1. Change to the below directory inside the cloned git repo:
+    ```
+    cd serverless-patterns/eventbridge-pipes-dynamodbstream-to-sqs-terraform
+    ```
+2. Delete the resources
+    ```bash
+    terraform destroy
+    ```
+3. Enter 'yes' when prompted.
+
+4. Check if all the resources were deleted successfully.
+    ```bash
+    terraform show
+    ```
+----
+Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+
+SPDX-License-Identifier: MIT-0
diff --git a/...ge-pipes-dynamodbstream-to-sqs-terraform/eventbridge-pipes-ddb-streams-sqs-terraform.json b/...ge-pipes-dynamodbstream-to-sqs-terraform/eventbridge-pipes-ddb-streams-sqs-terraform.json
@@ -0,0 +1,87 @@
+{
+    "title": "Amazon DynamoDB Streams to Amazon SQS queue using AWS EventBridge Pipes",
+    "description": "This pattern creates an AWS EventBridge Pipe to deliver messages from an Amazon DynamoDB Stream to Amazon SQS.",
+    "language": "",
+    "level": "200",
+    "framework": "Terraform",
+    "introBox": {
+        "headline": "How it works",
+        "text": [
+            "Whenever there is a change (insert, update, or delete) to an item in a DynamoDB table, a new record is added to the DynamoDB Stream. The stream captures the details of the event, and the item data.",
+            "This pattern allows you to react to changes in your DynamoDB table by filtering (optional) and delivering relevant events to an SQS queue. The EventBridge Pipe acts as a bridge between the DynamoDB Stream and the SQS queue.",
+            "Then, other services or applications can consume these events from the SQS queue."
+        ]
+    },
+    "gitHub": {
+        "template": {
+            "repoURL": "https://github.com/aws-samples/serverless-patterns/tree/main/eventbridge-pipes-dynamodbstream-to-sqs-terraform",
+            "templateURL": "serverless-patterns/eventbridge-pipes-dynamodbstream-to-sqs-terraform",
+            "projectFolder": "eventbridge-pipes-dynamodbstream-to-sqs-terraform",
+            "templateFile": "main.tf"
+        }
+    },
+    "resources": {
+        "bullets": [
+            {
+                "text": "EventBridge Pipes with DynamoDB stream as a source",
+                "link": "https://docs.aws.amazon.com/eventbridge/latest/userguide/eb-pipes-dynamodb.html"
+            },
+            {
+                "text": "Amazon EventBridge Pipes architectural patterns",
+                "link": "https://aws.amazon.com/blogs/compute/implementing-architectural-patterns-with-amazon-eventbridge-pipes/"
+            }
+        ]
+    },
+    "deploy": {
+        "text": [
+            "terraform init",
+            "terraform apply"
+        ]
+    },
+    "testing": {
+        "text": [
+            "See the README in the GitHub repo for detailed testing instructions."
+        ]
+    },
+    "cleanup": {
+        "text": [
+            "terraform destroy"
+        ]
+    },
+    "authors": [
+        {
+            "name": "Chaitanya Gummadi",
+            "image": "https://gravatar.com/avatar/a9b30c6e727f613f3f44a44016e36998719404ea7451c750f402a21f2dd72937.jpg?size=256",
+            "bio": "Chaitanya is a Sr. Cloud Support Engineer with Amazon Web Services (AWS) based in Texas.",
+            "linkedin": "cgummadi"
+        }
+    ],
+    "patternArch": {
+        "icon1": {
+            "x": 20,
+            "y": 50,
+            "service": "dynamodb-stream",
+            "label": "Amazon DynamoDB Streams"
+        },
+        "icon2": {
+            "x": 55,
+            "y": 50,
+            "service": "eventbridge-pipes",
+            "label": "AWS EventBridge Pipes"
+        },
+        "icon3": {
+            "x": 90,
+            "y": 50,
+            "service": "sqs",
+            "label": "Amazon SQS"
+        },
+        "line1": {
+            "from": "icon1",
+            "to": "icon2"
+        },
+        "line2": {
+            "from": "icon2",
+            "to": "icon3"
+        }
+    }
+}
diff --git a/eventbridge-pipes-dynamodbstream-to-sqs-terraform/example-pattern.json b/eventbridge-pipes-dynamodbstream-to-sqs-terraform/example-pattern.json
@@ -0,0 +1,87 @@
+{
+    "title": "Amazon DynamoDB Streams to Amazon SQS queue using AWS EventBridge Pipes",
+    "description": "This pattern creates an AWS EventBridge Pipe to deliver messages from an Amazon DynamoDB Stream to Amazon SQS.",
+    "language": "",
+    "level": "200",
+    "framework": "Terraform",
+    "introBox": {
+        "headline": "How it works",
+        "text": [
+            "Whenever there is a change (insert, update, or delete) to an item in a DynamoDB table, a new record is added to the DynamoDB Stream. The stream captures the details of the event, and the item data.",
+            "This pattern allows you to react to changes in your DynamoDB table by filtering (optional) and delivering relevant events to an SQS queue. The EventBridge Pipe acts as a bridge between the DynamoDB Stream and the SQS queue.",
+            "Then, other services or applications can consume these events from the SQS queue."
+        ]
+    },
+    "gitHub": {
+        "template": {
+            "repoURL": "https://github.com/aws-samples/serverless-patterns/tree/main/eventbridge-pipes-dynamodbstream-to-sqs-terraform",
+            "templateURL": "serverless-patterns/eventbridge-pipes-dynamodbstream-to-sqs-terraform",
+            "projectFolder": "eventbridge-pipes-dynamodbstream-to-sqs-terraform",
+            "templateFile": "main.tf"
+        }
+    },
+    "resources": {
+        "bullets": [
+            {
+                "text": "EventBridge Pipes with DynamoDB stream as a source",
+                "link": "https://docs.aws.amazon.com/eventbridge/latest/userguide/eb-pipes-dynamodb.html"
+            },
+            {
+                "text": "Amazon EventBridge Pipes architectural patterns",
+                "link": "https://aws.amazon.com/blogs/compute/implementing-architectural-patterns-with-amazon-eventbridge-pipes/"
+            }
+        ]
+    },
+    "deploy": {
+        "text": [
+            "terraform init",
+            "terraform apply"
+        ]
+    },
+    "testing": {
+        "text": [
+            "See the README in the GitHub repo for detailed testing instructions."
+        ]
+    },
+    "cleanup": {
+        "text": [
+            "terraform destroy"
+        ]
+    },
+    "authors": [
+        {
+            "name": "Chaitanya Gummadi",
+            "image": "https://gravatar.com/avatar/a9b30c6e727f613f3f44a44016e36998719404ea7451c750f402a21f2dd72937.jpg?size=256",
+            "bio": "Chaitanya is a Sr. Cloud Support Engineer with Amazon Web Services (AWS) based in Texas.",
+            "linkedin": "cgummadi"
+        }
+    ],
+    "patternArch": {
+        "icon1": {
+            "x": 20,
+            "y": 50,
+            "service": "dynamodb-stream",
+            "label": "Amazon DynamoDB Streams"
+        },
+        "icon2": {
+            "x": 55,
+            "y": 50,
+            "service": "eventbridge-pipes",
+            "label": "AWS EventBridge Pipes"
+        },
+        "icon3": {
+            "x": 90,
+            "y": 50,
+            "service": "sqs",
+            "label": "Amazon SQS"
+        },
+        "line1": {
+            "from": "icon1",
+            "to": "icon2"
+        },
+        "line2": {
+            "from": "icon2",
+            "to": "icon3"
+        }
+    }
+}
diff --git a/eventbridge-pipes-dynamodbstream-to-sqs-terraform/main.tf b/eventbridge-pipes-dynamodbstream-to-sqs-terraform/main.tf
@@ -0,0 +1,114 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 5.0"
+    }
+  }
+}
+provider "aws" {
+  profile = "default"
+  region  = "us-east-1"
+}
+
+
+# Create DynamoDB table
+resource "aws_dynamodb_table" "source" {
+  name           = "demo_table"
+  billing_mode   = "PAY_PER_REQUEST"
+  hash_key       = "Id"
+
+  attribute {
+    name = "Id"
+    type = "S"
+  }
+
+  stream_enabled   = true
+  stream_view_type = "NEW_AND_OLD_IMAGES"
+}
+
+# Create SQS queue
+resource "aws_sqs_queue" "target" {
+  name = "demo_queue"
+}
+
+# Create EventBridge Pipe
+resource "aws_pipes_pipe" "demo_pipe" {
+  name     = "demo_pipe"
+  role_arn = aws_iam_role.demo_role.arn
+  source   = aws_dynamodb_table.source.stream_arn
+  target   = aws_sqs_queue.target.arn
+  source_parameters {
+    dynamodb_stream_parameters {
+      starting_position = "TRIM_HORIZON"
+      batch_size = 1
+    }
+  }
+}
+# Create IAM role for EventBridge Pipe
+resource "aws_iam_role" "demo_role" {
+  name = "demo-role"
+
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Action = "sts:AssumeRole"
+        Effect = "Allow"
+        Sid    = "PipesPermissions"
+        Principal = {
+          Service = "pipes.amazonaws.com"
+        }
+      },
+    ]
+  })
+}
+
+# Create an IAM policy for EventBridge Pipe
+resource "aws_iam_policy" "demo_policy" {
+  name = "demo_policy"
+  policy = jsonencode({
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Sid": "dynamodbPermissions",
+            "Effect": "Allow",
+            "Action": [
+                "dynamodb:DescribeStream",
+                "dynamodb:GetRecords",
+                "dynamodb:GetShardIterator",
+                "dynamodb:ListStreams"
+            ],
+            "Resource": [
+                aws_dynamodb_table.source.stream_arn
+            ]
+        },
+        {
+            "Sid": "sqsPermissions",
+            "Effect": "Allow",
+            "Action": [
+                "sqs:SendMessage"
+            ],
+            "Resource": [
+                aws_sqs_queue.target.arn
+            ]
+        }
+    ]
+})
+
+# Attach the IAM policy to the IAM role
+}
+resource "aws_iam_role_policy_attachment" "demo-attach" {
+  role       = aws_iam_role.demo_role.name
+  policy_arn = aws_iam_policy.demo_policy.arn
+}
+
+output "SQS_arn" {
+  value       = aws_sqs_queue.target.id
+  description = "The ARN of the SQS queue URL."
+}
+
+output "DynamoDB_arn" {
+  value       = aws_dynamodb_table.source.arn
+  description = "The ARN of the DynamoDB table."
+}