diff --git a/s3-eventbridge-fargate-cdk/.gitignore b/s3-eventbridge-fargate-cdk/.gitignore new file mode 100644 index 000000000..a05e3ebb3 --- /dev/null +++ b/s3-eventbridge-fargate-cdk/.gitignore @@ -0,0 +1,10 @@ +*.js +!jest.config.js +*.d.ts +node_modules + +# CDK asset staging directory +.cdk.staging +cdk.out + +.idea \ No newline at end of file diff --git a/s3-eventbridge-fargate-cdk/.npmignore b/s3-eventbridge-fargate-cdk/.npmignore new file mode 100644 index 000000000..c1d6d45dc --- /dev/null +++ b/s3-eventbridge-fargate-cdk/.npmignore @@ -0,0 +1,6 @@ +*.ts +!*.d.ts + +# CDK asset staging directory +.cdk.staging +cdk.out diff --git a/s3-eventbridge-fargate-cdk/README.md b/s3-eventbridge-fargate-cdk/README.md new file mode 100644 index 000000000..f2ac220db --- /dev/null +++ b/s3-eventbridge-fargate-cdk/README.md @@ -0,0 +1,83 @@ +# Amazon S3 to AWS Fargate + +This pattern demonstrates how to invoke an AWS Fargate task when an object is uploaded to Amazon S3. +This pattern is commonly implemented with an AWS Lambda function, but this is not always possible: +- Processing > 15 min +- Docker image > 10G +- GPU required + +When you need more (more time, more memory, more power), but still want to use serverless service, you can look at Fargate. + +Important: this application uses various AWS services and there are costs associated with these services after the Free Tier usage - please see the [AWS Pricing page](https://aws.amazon.com/pricing/) for details. You are responsible for any AWS costs incurred. No warranty is implied in this example. + +## Requirements + +* [Create an AWS account](https://portal.aws.amazon.com/gp/aws/developer/registration/index.html) if you do not already have one and log in. The IAM user that you use must have sufficient permissions to make necessary AWS service calls and manage AWS resources. +* [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html) installed and configured +* [Git Installed](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) +* [AWS Cloud Development Kit](https://docs.aws.amazon.com/cdk/v2/guide/getting_started.html) (AWS CDK) installed + +## Deployment Instructions + +1. Create a new directory, navigate to that directory in a terminal and clone the GitHub repository: + ``` + git clone https://github.com/aws-samples/serverless-patterns + ``` +2. Change directory to the pattern directory: + ``` + cd serverless-patterns/s3-eventbridge-fargate-cdk + ``` +3. From the command line, use AWS SAM to deploy the AWS resources for the pattern as specified in the template.yml file: + ``` + npm install && npx cdk deploy + ``` +4. During the prompts: + * Allow CDK to create resources and IAM roles with the required permissions. + +5. Note the outputs from the CDK deployment process. These contain the resource names and/or ARNs which are used for testing. + +## How it works + +![Architecture](doc/archi.png) + +1. When a file is uploaded to S3, in the specified bucket (eventually with a specific prefix), an event is triggered +2. EventBridge catch this event and triggers the ECS Fargate task +3. Fargate bootstrap a container to run the task + +## Testing + +1. Once CDK / CloudFormation deployed the stack, upload a file to S3 using the following command. + (replace `your-bucket-name` with the CDK output `S3TriggerFargateTaskStack.IngestionBucket`) : + +```shell + aws s3api put-object --bucket your-bucket-name --key file-name --body file-name +``` + +2. The S3 file upload will trigger the EventBridge Rule which will run the ECS task. The ECS task will execute and print the content of the document in CloudWatch. + +3. Check the CloudWatch log group to see the ECS task execution details. Logs can be found in `/ecs/doc-ingestion` log stream. +Use the following commands to get the logs: + +```shell +# get log streams +aws logs describe-log-streams --log-group-name /ecs/doc-ingestion +# use the latest log stream name in the next command to get logs +aws logs get-log-events --log-group-name /ecs/doc-ingestion --log-stream-name doc-ingestion-logs/DocIngestion/... +``` + +4. The file contents are displayed in the log. + +## Cleanup + +1. Delete the stack + ```bash + npx cdk destroy + ``` +2. Confirm the stack has been deleted + ```bash + aws cloudformation list-stacks --query "StackSummaries[?contains(StackName,'S3TriggerFargateTaskStack')].StackStatus" + ``` +---- +Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved. + +SPDX-License-Identifier: MIT-0 diff --git a/s3-eventbridge-fargate-cdk/cdk.json b/s3-eventbridge-fargate-cdk/cdk.json new file mode 100644 index 000000000..259b4b6ab --- /dev/null +++ b/s3-eventbridge-fargate-cdk/cdk.json @@ -0,0 +1,65 @@ +{ + "app": "npx ts-node --prefer-ts-exts src/s3-trigger-fargate-task.ts", + "watch": { + "include": [ + "**" + ], + "exclude": [ + "README.md", + "cdk*.json", + "**/*.d.ts", + "**/*.js", + "tsconfig.json", + "package*.json", + "yarn.lock", + "node_modules", + "test" + ] + }, + "context": { + "@aws-cdk/aws-lambda:recognizeLayerVersion": true, + "@aws-cdk/core:checkSecretUsage": true, + "@aws-cdk/core:target-partitions": [ + "aws", + "aws-cn" + ], + "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true, + "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true, + "@aws-cdk/aws-ecs:arnFormatIncludesClusterName": true, + "@aws-cdk/aws-iam:minimizePolicies": true, + "@aws-cdk/core:validateSnapshotRemovalPolicy": true, + "@aws-cdk/aws-codepipeline:crossAccountKeyAliasStackSafeResourceName": true, + "@aws-cdk/aws-s3:createDefaultLoggingPolicy": true, + "@aws-cdk/aws-sns-subscriptions:restrictSqsDescryption": true, + "@aws-cdk/aws-apigateway:disableCloudWatchRole": true, + "@aws-cdk/core:enablePartitionLiterals": true, + "@aws-cdk/aws-events:eventsTargetQueueSameAccount": true, + "@aws-cdk/aws-iam:standardizedServicePrincipals": true, + "@aws-cdk/aws-ecs:disableExplicitDeploymentControllerForCircuitBreaker": true, + "@aws-cdk/aws-iam:importedRoleStackSafeDefaultPolicyName": true, + "@aws-cdk/aws-s3:serverAccessLogsUseBucketPolicy": true, + "@aws-cdk/aws-route53-patters:useCertificate": true, + "@aws-cdk/customresources:installLatestAwsSdkDefault": false, + "@aws-cdk/aws-rds:databaseProxyUniqueResourceName": true, + "@aws-cdk/aws-codedeploy:removeAlarmsFromDeploymentGroup": true, + "@aws-cdk/aws-apigateway:authorizerChangeDeploymentLogicalId": true, + "@aws-cdk/aws-ec2:launchTemplateDefaultUserData": true, + "@aws-cdk/aws-secretsmanager:useAttachedSecretResourcePolicyForSecretTargetAttachments": true, + "@aws-cdk/aws-redshift:columnId": true, + "@aws-cdk/aws-stepfunctions-tasks:enableEmrServicePolicyV2": true, + "@aws-cdk/aws-ec2:restrictDefaultSecurityGroup": true, + "@aws-cdk/aws-apigateway:requestValidatorUniqueId": true, + "@aws-cdk/aws-kms:aliasNameRef": true, + "@aws-cdk/aws-autoscaling:generateLaunchTemplateInsteadOfLaunchConfig": true, + "@aws-cdk/core:includePrefixInUniqueNameGeneration": true, + "@aws-cdk/aws-efs:denyAnonymousAccess": true, + "@aws-cdk/aws-opensearchservice:enableOpensearchMultiAzWithStandby": true, + "@aws-cdk/aws-lambda-nodejs:useLatestRuntimeVersion": true, + "@aws-cdk/aws-efs:mountTargetOrderInsensitiveLogicalId": true, + "@aws-cdk/aws-rds:auroraClusterChangeScopeOfInstanceParameterGroupWithEachParameters": true, + "@aws-cdk/aws-appsync:useArnForSourceApiAssociationIdentifier": true, + "@aws-cdk/aws-rds:preventRenderingDeprecatedCredentials": true, + "@aws-cdk/aws-codepipeline-actions:useNewDefaultBranchForCodeCommitSource": true, + "@aws-cdk/aws-cloudwatch-actions:changeLambdaPermissionLogicalIdForLambdaAction": true + } +} diff --git a/s3-eventbridge-fargate-cdk/doc/archi.drawio b/s3-eventbridge-fargate-cdk/doc/archi.drawio new file mode 100644 index 000000000..4a20a6c9e --- /dev/null +++ b/s3-eventbridge-fargate-cdk/doc/archi.drawio @@ -0,0 +1 @@ +7Vptd5owFP41fpwHCAh+LKjd69nO3DnbPu2kEiErEBei1f763UBAMNTVM/qyWauWPHnhJs997g1pByhIt5ccr+IPLCTJwDLC7QBNBpY1tkfwLYFdCYxcqwQiTsMSMvfAnN4SBRoKXdOQ5K2GgrFE0FUbXLAsIwvRwjDn7KbdbMmS9l1XOCIaMF/gREe/0lDEJepZ7h5/TWgUV3c2R+OyJsVVYzWTPMYhu2lAaDpAAWdMlFfpNiCJXLtqXcp+sztqa8M4ycR9OsxFaH3z32a/bj8a32eX7y5fkY+vbGWb2FUTJiHMXxUZFzGLWIaT6R71OVtnIZGjGlDat3nP2ApAE8CfRIidIhOvBQMoFmmiasFgvvum+heF77IwdKriZNusnOyapU+E05QIwhVYTkBafee6KChna74gRxaj8i/MIyKOtEM1e+D1hIE1fAf9OEmwoJu2HVj5X1S321MEF4qlExhT425wslZ30ijMr4lYxGp9VoxmojDC8eENPhKUHweaBhIZWk4H2IW5OmjqzeCX2XWHQ7ALc3XQ1JvJUmV1G+zCXEe3+LC32dHbPOgNb3D3tUhoRoI63Mg1XrJMBCxhvFh/BD8zyagfcRxS0qobGRcIuY26CeUwEGUZ1GdSTHI8miTN8VwX4grgueDsmjRqlsULakKcx7UkN4QLCiHsPb4iySeWUzX8FROCpY0GFwmNZIWQwvWxKi3AKimvpmTlDJWeTasqK4+Tt8T5qlyOJd1KO3yIditZmW4jmReG+Ca3h5yUGnyzkPb4UCyv2q1yVMtaGkq2x4WtC7HqMFKRd3dQvtkHctcrobgRwyusd+lamnQvUnwLC2AZc6SpGCYu2iRoBB3ymNIwLEM0yektvqrpaQcB6ccyKueKUM2zMpaRAzdUUB+0OOMWLRbyNFo8Q6elyqO90zJ6ihzYY9pC90xbzlOmLfSStv7ltDWb2TPPPy1t+YGJnPNJW2QDBlzBQ0zUU6BEzy1/OS8a/qc17HpTwz5NwxPDCUz3bDS8hByKRU/6dZ6bft0T9ftXDtexhyzyyHhsGHdtOJ+dKxURjPCpDO15tVXuci+yyH8InF/34zkj58+eg4yhh3TnsR9ql+zd/fBSLI9fZr5zeYpByP7jU4yJHvMxpjoxfVp5q23ifyZvvk7Ij56OJWz3QNyO7jm2M/Q6MsODids89VTxxXfunxo2hWP04Tr1ucgx1+mIOQ/nN0eOtKbBHL5nakd1LnnBOcwLtp66Hzkv6EcvhdOeDSWW67RV00GJ86iM2BojnyHBnA0hWgbsIKQrjD0cIfrJxhe5mT8XQkaw37Ato36ZbXo6trY90QPF/R/Ai7rGfxGg6W8= \ No newline at end of file diff --git a/s3-eventbridge-fargate-cdk/doc/archi.png b/s3-eventbridge-fargate-cdk/doc/archi.png new file mode 100644 index 000000000..c54f47b82 Binary files /dev/null and b/s3-eventbridge-fargate-cdk/doc/archi.png differ diff --git a/s3-eventbridge-fargate-cdk/docker/doc-ingestion/Dockerfile b/s3-eventbridge-fargate-cdk/docker/doc-ingestion/Dockerfile new file mode 100644 index 000000000..41ca27e01 --- /dev/null +++ b/s3-eventbridge-fargate-cdk/docker/doc-ingestion/Dockerfile @@ -0,0 +1,12 @@ +FROM public.ecr.aws/docker/library/python:3.10-slim + +RUN apt-get update + +RUN python -m pip install --upgrade setuptools pip + +COPY requirements.txt ./ +RUN pip install --default-timeout=100 -r requirements.txt +CMD ["pip", "freeze"] + +COPY app.py ./ +CMD [ "python", "app.py" ] diff --git a/s3-eventbridge-fargate-cdk/docker/doc-ingestion/app.py b/s3-eventbridge-fargate-cdk/docker/doc-ingestion/app.py new file mode 100644 index 000000000..f2d7a6ffe --- /dev/null +++ b/s3-eventbridge-fargate-cdk/docker/doc-ingestion/app.py @@ -0,0 +1,18 @@ +import os +import boto3 + +s3_bucket = os.environ['S3_BUCKET'] +s3_object = os.environ['S3_OBJECT_KEY'] + +s3_client = boto3.client('s3') +try: + s3_response = s3_client.get_object(Bucket=s3_bucket, Key=s3_object) + data = s3_response['Body'].read().decode('utf-8').splitlines(True) + print(data) + + # implement business logic to process file + +except Exception as e: + print(f"Error: {e}") + raise e + diff --git a/s3-eventbridge-fargate-cdk/docker/doc-ingestion/requirements.txt b/s3-eventbridge-fargate-cdk/docker/doc-ingestion/requirements.txt new file mode 100644 index 000000000..22ca9575f --- /dev/null +++ b/s3-eventbridge-fargate-cdk/docker/doc-ingestion/requirements.txt @@ -0,0 +1,2 @@ +boto3 +botocore diff --git a/s3-eventbridge-fargate-cdk/jest.config.js b/s3-eventbridge-fargate-cdk/jest.config.js new file mode 100644 index 000000000..08263b895 --- /dev/null +++ b/s3-eventbridge-fargate-cdk/jest.config.js @@ -0,0 +1,8 @@ +module.exports = { + testEnvironment: 'node', + roots: ['/test'], + testMatch: ['**/*.test.ts'], + transform: { + '^.+\\.tsx?$': 'ts-jest' + } +}; diff --git a/s3-eventbridge-fargate-cdk/package.json b/s3-eventbridge-fargate-cdk/package.json new file mode 100644 index 000000000..63550ef57 --- /dev/null +++ b/s3-eventbridge-fargate-cdk/package.json @@ -0,0 +1,27 @@ +{ + "name": "s3-trigger-fargate-task", + "version": "0.1.0", + "bin": { + "s3-trigger-fargate-task": "src/s3-trigger-fargate-task.js" + }, + "scripts": { + "build": "tsc", + "watch": "tsc -w", + "test": "jest", + "cdk": "cdk" + }, + "devDependencies": { + "@types/jest": "^29.5.11", + "@types/node": "20.11.14", + "jest": "^29.7.0", + "ts-jest": "^29.1.2", + "aws-cdk": "2.126.0", + "ts-node": "^10.9.2", + "typescript": "~5.3.3" + }, + "dependencies": { + "aws-cdk-lib": "2.126.0", + "constructs": "^10.0.0", + "source-map-support": "^0.5.21" + } +} \ No newline at end of file diff --git a/s3-eventbridge-fargate-cdk/pattern.json b/s3-eventbridge-fargate-cdk/pattern.json new file mode 100644 index 000000000..fda556685 --- /dev/null +++ b/s3-eventbridge-fargate-cdk/pattern.json @@ -0,0 +1,62 @@ +{ + "title": "Amazon S3 to AWS Fargate", + "description": "Trigger an AWS Fargate Task when a file is uploaded to S3", + "language": "Typescript", + "level": "200", + "framework": "CDK", + "introBox": { + "headline": "How it works", + "text": [ + "This pattern demonstrates how to trigger an AWS Fargate task when an object is uploaded to Amazon S3.", + "This pattern is commonly implemented with an AWS Lambda function, but this is not always possible:", + " - Processing > 15 min", + " - Docker image > 10G", + " - GPU required", + "Thanks to EventBridge, S3 events can be used to trigger an ECS/Fargate task." + ] + }, + "gitHub": { + "template": { + "repoURL": "https://github.com/aws-samples/serverless-patterns/tree/main/s3-eventbridge-fargate-cdk", + "templateURL": "serverless-patterns/s3-eventbridge-fargate-cdk", + "projectFolder": "s3-eventbridge-fargate-cdk", + "templateFile": "src/s3-trigger-fargate-task-stack.ts" + } + }, + "resources": { + "bullets": [ + { + "text": "Using EventBridge to handle S3 events", + "link": "https://docs.aws.amazon.com/AmazonS3/latest/userguide/EventBridge.html" + }, + { + "text": "ECS Task as a target for EventBridge", + "link": "https://docs.aws.amazon.com/eventbridge/latest/userguide/eb-targets.html#targets-specifics-ecs-task" + } + ] + }, + "deploy": { + "text": [ + "npm install && cdk deploy" + ] + }, + "testing": { + "text": [ + "See the GitHub repo for detailed testing instructions." + ] + }, + "cleanup": { + "text": [ + "Delete the stack: cdk destroy." + ] + }, + "authors": [ + { + "name": "Jerome Van Der Linden", + "image": "https://serverlessland.com/assets/images/resources/contributors/jerome-van-der-linden.jpg", + "bio": "Jerome is a Solutions Architect Builder at AWS. Passionate about building stuff using the AWS services, and especially the serverless ones.", + "linkedin": "jeromevdl", + "twitter": "jeromevdl" + } + ] +} \ No newline at end of file diff --git a/s3-eventbridge-fargate-cdk/s3-eventbirdge-fargate-cdk.json b/s3-eventbridge-fargate-cdk/s3-eventbirdge-fargate-cdk.json new file mode 100644 index 000000000..38376b7ad --- /dev/null +++ b/s3-eventbridge-fargate-cdk/s3-eventbirdge-fargate-cdk.json @@ -0,0 +1,92 @@ +{ + "title": "Amazon S3 to AWS Fargate", + "description": "Start an AWS Fargate Task when a file is uploaded to S3", + "language": "TypeScript", + "level": "200", + "framework": "CDK", + "introBox": { + "headline": "How it works", + "text": [ + "This pattern demonstrates how to start an AWS Fargate task when an object is uploaded to Amazon S3.", + "This pattern is commonly implemented with an AWS Lambda function, but this is not always possible:", + " - Processing > 15 min", + " - Docker image > 10G", + " - GPU required", + "Thanks to Amazon EventBridge, S3 events can start an ECS/Fargate task." + ] + }, + "gitHub": { + "template": { + "repoURL": "https://github.com/aws-samples/serverless-patterns/tree/main/s3-eventbridge-fargate-cdk", + "templateURL": "serverless-patterns/s3-eventbridge-fargate-cdk/src", + "projectFolder": "s3-eventbridge-fargate-cdk", + "templateFile": "src/s3-trigger-fargate-task-stack.ts" + } + }, + "resources": { + "bullets": [ + { + "text": "Using EventBridge to handle S3 events", + "link": "https://docs.aws.amazon.com/AmazonS3/latest/userguide/EventBridge.html" + }, + { + "text": "ECS Task as a target for EventBridge", + "link": "https://docs.aws.amazon.com/eventbridge/latest/userguide/eb-targets.html#targets-specifics-ecs-task" + } + ] + }, + "deploy": { + "text": [ + "npm install && cdk deploy" + ] + }, + "testing": { + "text": [ + "See the GitHub repo for detailed testing instructions." + ] + }, + "cleanup": { + "text": [ + "Delete the stack: cdk destroy." + ] + }, + "authors": [ + { + "name": "Jerome Van Der Linden", + "image": "https://serverlessland.com/assets/images/resources/contributors/jerome-van-der-linden.jpg", + "bio": "Jerome is a Solutions Architect Builder at AWS. Passionate about building stuff using the AWS services, and especially the serverless ones.", + "linkedin": "jeromevdl", + "twitter": "jeromevdl" + } + ], + "patternArch": { + "icon1": { + "x": 20, + "y": 50, + "service": "s3", + "label": "Amazon S3" + }, + "icon2": { + "x": 50, + "y": 50, + "service": "eventbridge", + "label": "Amazon EventBridge" + }, + "icon3": { + "x": 80, + "y": 50, + "service": "fargate", + "label": "AWS Fargate" + }, + "line1": { + "from": "icon1", + "to": "icon2", + "label": "Event" + }, + "line2": { + "from": "icon2", + "to": "icon3", + "label": "Rule" + } + } +} diff --git a/s3-eventbridge-fargate-cdk/src/s3-trigger-fargate-task-stack.ts b/s3-eventbridge-fargate-cdk/src/s3-trigger-fargate-task-stack.ts new file mode 100644 index 000000000..59c220c1d --- /dev/null +++ b/s3-eventbridge-fargate-cdk/src/s3-trigger-fargate-task-stack.ts @@ -0,0 +1,167 @@ +import * as cdk from 'aws-cdk-lib'; +import { Construct } from 'constructs'; +import * as s3 from 'aws-cdk-lib/aws-s3'; +import * as ec2 from 'aws-cdk-lib/aws-ec2'; +import * as iam from 'aws-cdk-lib/aws-iam'; +import * as logs from 'aws-cdk-lib/aws-logs'; +import { RetentionDays } from 'aws-cdk-lib/aws-logs'; +import * as ecs from 'aws-cdk-lib/aws-ecs'; +import * as events from 'aws-cdk-lib/aws-events'; +import * as targets from "aws-cdk-lib/aws-events-targets"; +import * as path from "node:path"; +import { DockerImageAsset } from "aws-cdk-lib/aws-ecr-assets"; +import { CfnOutput, Duration } from "aws-cdk-lib"; + +export class S3TriggerFargateTaskStack extends cdk.Stack { + constructor(scope: Construct, id: string, props?: cdk.StackProps) { + super(scope, id, props); + + // Ingestion bucket + const bucket = new s3.Bucket(this, 'IngestionBucket', { + eventBridgeEnabled: true, + blockPublicAccess: s3.BlockPublicAccess.BLOCK_ALL, + removalPolicy: cdk.RemovalPolicy.DESTROY, + autoDeleteObjects: true, + enforceSSL: true, + publicReadAccess: false + }); + + // VPC for the ECS Cluster + const vpc = new ec2.Vpc(this, "VPC", { + maxAzs: 2, + natGateways: 1, + subnetConfiguration: [ + { + cidrMask: 24, + name: "public", + subnetType: ec2.SubnetType.PUBLIC, + }, + { + cidrMask: 24, + name: "private", + subnetType: ec2.SubnetType.PRIVATE_WITH_EGRESS, + }, + ], + }); + + // S3 Gateway Endpoint to avoid accessing S3 from Internet + const s3GatewayEndpoint = vpc.addGatewayEndpoint('S3GatewayEndpoint', { + service: ec2.GatewayVpcEndpointAwsService.S3 + }); + + // IAM policy to only enable access to S3 through vpc endpoint (OPTIONAL) + const bucketDenyPolicy = new iam.PolicyStatement({ + effect: iam.Effect.DENY, + principals: [new iam.AnyPrincipal()], + actions: ['s3:GetObject'], // adapt to your security requirements + resources: [ + `arn:aws:s3:::${bucket.bucketName}`, + `arn:aws:s3:::${bucket.bucketName}/*`, + ], + conditions: { + 'StringNotEquals': { + 'aws:SourceVpce': s3GatewayEndpoint.vpcEndpointId, + }, + }, + }); + bucket.addToResourcePolicy(bucketDenyPolicy); + + const cluster = new ecs.Cluster(this, "Cluster", { + vpc: vpc, + }); + + // ECS Task definition + const taskDefinition = new ecs.FargateTaskDefinition(this, "TaskDefinition", { + cpu: 512, + memoryLimitMiB: 2048, + runtimePlatform: { + cpuArchitecture: ecs.CpuArchitecture.X86_64, + operatingSystemFamily: ecs.OperatingSystemFamily.LINUX + }, + }); + + // Container logs + const ecsLogGroup = new logs.LogGroup(this, 'ContainerLogGroup', { + logGroupName: '/ecs/doc-ingestion', + removalPolicy: cdk.RemovalPolicy.DESTROY, + retention: RetentionDays.FIVE_DAYS + }); + + // Container image + const container = taskDefinition.addContainer('DocIngestion', { + image: ecs.ContainerImage.fromDockerImageAsset(new DockerImageAsset(this, 'DocIngestionImage', { + directory: path.join(__dirname, '../docker/doc-ingestion'), + })), + logging: new ecs.AwsLogDriver({ streamPrefix: 'doc-ingestion-logs', logGroup: ecsLogGroup}), + }); + + // Define the ECS Task as a target for the EventBridge rule + // We extract information from the event and store them in environment variables + const ecsTask = new targets.EcsTask({ + taskDefinition, + cluster, + containerOverrides: [ + { + containerName: container.containerName, + environment: [ + { + name: 'S3_BUCKET', + value: events.EventField.fromPath("$.detail.bucket.name"), + }, + { + name: 'S3_OBJECT_KEY', + value:events. EventField.fromPath("$.detail.object.key"), + } + ] + } + ], + maxEventAge: Duration.hours(2), + retryAttempts: 3 + }); + + // EventBridge rule definition + const rule = new events.Rule(this, 'rule', { + eventPattern: { + source: ['aws.s3'], + detailType: [ + 'Object Created' + ], + detail: { + bucket: { + name: [ + bucket.bucketName + ] + }, + // OPTIONAL: if there is a specific prefix to watch for + // object: { + // key: [{ + // prefix: "prefix" + // }] + // } + } + }, + targets: [ + ecsTask, + // just for debugging purpose, log events in cloudwatch + // new targets.CloudWatchLogGroup(new logs.LogGroup(this, 'RuleLogs', { + // logGroupName: '/aws/events/doc-ingestion', + // removalPolicy: cdk.RemovalPolicy.DESTROY, + // retention: RetentionDays.FIVE_DAYS + // })) + ] + }); + + // Allow ECS task to access the bucket + bucket.grantRead(taskDefinition.taskRole); + + new CfnOutput(this, 'IngestionBucketOut', { + key: 'IngestionBucket', + value: bucket.bucketName + }); + + new CfnOutput(this, 'ECSLogsOut', { + key: 'ECSLogs', + value: ecsLogGroup.logGroupName + }); + } +} diff --git a/s3-eventbridge-fargate-cdk/src/s3-trigger-fargate-task.ts b/s3-eventbridge-fargate-cdk/src/s3-trigger-fargate-task.ts new file mode 100644 index 000000000..108cbde7e --- /dev/null +++ b/s3-eventbridge-fargate-cdk/src/s3-trigger-fargate-task.ts @@ -0,0 +1,8 @@ +#!/usr/bin/env node +import 'source-map-support/register'; +import * as cdk from 'aws-cdk-lib'; +import { S3TriggerFargateTaskStack } from './s3-trigger-fargate-task-stack'; +const description = "S3-Fargate Pattern (uksb-1tthgi812) (tag:s3-eventbridge-fargate-cdk)"; +const app = new cdk.App(); +new S3TriggerFargateTaskStack(app, 'S3TriggerFargateTaskStack', {description:description +}); diff --git a/s3-eventbridge-fargate-cdk/tsconfig.json b/s3-eventbridge-fargate-cdk/tsconfig.json new file mode 100644 index 000000000..aaa7dc510 --- /dev/null +++ b/s3-eventbridge-fargate-cdk/tsconfig.json @@ -0,0 +1,31 @@ +{ + "compilerOptions": { + "target": "ES2020", + "module": "commonjs", + "lib": [ + "es2020", + "dom" + ], + "declaration": true, + "strict": true, + "noImplicitAny": true, + "strictNullChecks": true, + "noImplicitThis": true, + "alwaysStrict": true, + "noUnusedLocals": false, + "noUnusedParameters": false, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": false, + "inlineSourceMap": true, + "inlineSources": true, + "experimentalDecorators": true, + "strictPropertyInitialization": false, + "typeRoots": [ + "./node_modules/@types" + ] + }, + "exclude": [ + "node_modules", + "cdk.out" + ] +}