template.yml

AWSTemplateFormatVersion: '2010-09-09'
Transform: 'AWS::Serverless-2016-10-31'
Description: AWS AI RAG project
Parameters:
  timePeriodValue:
    Description: "The time period of the scheduled scrape event (without unit)"
    Type: Number
    MinValue: 1
    MaxValue: 255
  timePeriodUnit:
    Description: "The unit for the time period of the scheduled event, e.g. 'minute'"
    Type: String
    MinLength: 1
    MaxLength: 50
  timeout:
    Description: Timeout in seconds for the lambdas
    Type: Number
    MinValue: 1
    Default: 180
  logLevel:
    Description: The log level for the lambda functions, e.g. DEBUG
    Type: String
    MinLength: 1
    MaxLength: 50
    Default: INFO
  bucketName:
    Description: The S3 bucket which will contain the uploaded resources
    Type: String
    MinLength: 1
    MaxLength: 100

Resources:

  ossDomain:
    Type: AWS::OpenSearchService::Domain
    Properties:
      DomainName:
        !Join
        - '-'
        - - !Ref "AWS::StackName"
          - 'oss-domain'
      EngineVersion: OpenSearch_2.13
      EBSOptions:
        EBSEnabled: true
        VolumeSize: 10
        VolumeType: standard
      ClusterConfig:
        InstanceCount: 1
        InstanceType: t2.small.search
      AccessPolicies:
        Version: '2012-10-17'
        Statement:
          -
            Effect: 'Allow'
            Principal:
              AWS: !Ref "AWS::AccountId"
            Action: 'es:*'
            Resource: !Sub 'arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/${AWS::StackName}-oss-domain/*'

  mainScrapeLog:
    Type: AWS::Logs::LogGroup
    Properties:
      LogGroupClass: STANDARD
      RetentionInDays: 3
      LogGroupName:
        !Join
        - '-'
        - - !Ref "AWS::StackName"
          - 'main_scrape_lambda-log'

  accessPolicyToSQS:
    Type: AWS::IAM::ManagedPolicy
    Properties:
      Description: Allow writing to SQS
      PolicyDocument:
        Version: "2012-10-17"
        Statement:
          - Effect: Allow
            Action: 'sqs:SendMessage'
            Resource: !GetAtt queue.Arn

  mainScrape:
    Type: AWS::Serverless::Function
    Properties:
      Environment:
        Variables:
          SQS_URL: !GetAtt queue.QueueUrl
          LOG_LEVEL: !Ref logLevel
      FunctionName:
        !Join
        - '-'
        - - !Ref "AWS::StackName"
          - 'main_scrape_lambda'
      Architectures:
        - x86_64
      MemorySize: 128
      PackageType: Zip
      Handler: main_scrape_lambda.lambda_function.lambda_handler
      Runtime: python3.10
      CodeUri:
        Bucket: !Ref bucketName
        Key: main_scrape_lambda
      Description: Do a periodic scrape of the main page
      Timeout: !Ref timeout
      Policies:
        - AWSLambdaBasicExecutionRole
        - AWSLambda_ReadOnlyAccess
        - !Ref accessPolicyToSQS
      Layers:
        - Ref: pythonLibLayer
      Events:
        ScheduledEvent:
          Type: ScheduleV2
          Description: Repeating trigger for the Lambda
          Properties:
            State: DISABLED
            RetryPolicy:
              MaximumRetryAttempts: 2
              MaximumEventAgeInSeconds: 240
            ScheduleExpression:
              !Join
              - ''
              - - 'rate('
                - !Ref timePeriodValue
                - ' '
                - !Ref timePeriodUnit
                - ')'
            Name:
              !Join
              - '-'
              - - !Ref "AWS::StackName"
                - 'schedule'
      LoggingConfig:
        LogFormat: Text
        LogGroup: !Ref mainScrapeLog

  pythonLibLayer:
    Type: AWS::Serverless::LayerVersion
    Properties:
      LayerName:
        !Join
        - '-'
        - - !Ref "AWS::StackName"
          - 'layer'
      Description: Python packages used by the lambdas.
      ContentUri:
        Bucket: !Ref bucketName
        Key: layer
      CompatibleArchitectures:
        - x86_64
      CompatibleRuntimes:
        - python3.10

  newsScrapeLog:
    Type: AWS::Logs::LogGroup
    Properties:
      LogGroupClass: STANDARD
      RetentionInDays: 3
      LogGroupName:
        !Join
        - '-'
        - - !Ref "AWS::StackName"
          - 'news_scrape_lambda-log'

  queue:
    Type: AWS::SQS::Queue
    Properties:
      MessageRetentionPeriod: 43200
      QueueName:
        !Join
        - '-'
        - - !Ref "AWS::StackName"
          - 'queue'
      RedriveAllowPolicy:
        redrivePermission: denyAll
      RedrivePolicy:
        deadLetterTargetArn: !GetAtt DLQ.Arn
        maxReceiveCount: 3
      VisibilityTimeout:
        # Make this 10x the lambda timeout
        !Join
        - ''
        - - !Ref timeout
          - '0'

  DLQ:
    Type: AWS::SQS::Queue
    Properties:
      MessageRetentionPeriod: 432000
      QueueName:
        !Join
        - '-'
        - - !Ref "AWS::StackName"
          - 'dlq'

  newsScrape:
    Type: AWS::Serverless::Function
    Properties:
      Environment:
        Variables:
          OPENSEARCH_URL: !GetAtt ossDomain.DomainEndpoint
          LOG_LEVEL: !Ref logLevel
      FunctionName:
        !Join
        - '-'
        - - !Ref "AWS::StackName"
          - 'news_scrape_lambda'
      Architectures:
        - x86_64
      MemorySize: 128
      PackageType: Zip
      Handler: news_scrape_lambda.lambda_function.lambda_handler
      Runtime: python3.10
      CodeUri:
        Bucket: !Ref bucketName
        Key: news_scrape_lambda
      Description: Read events from SQS and process
      Timeout: !Ref timeout
      Policies:
        - AWSLambdaBasicExecutionRole
        - AWSLambda_ReadOnlyAccess
        - AmazonOpenSearchServiceFullAccess
        - AmazonBedrockFullAccess
        - AWSLambdaSQSQueueExecutionRole
      Layers:
        - Ref: pythonLibLayer
      Events:
        SQSEvent:
          Type: SQS
          Description: Run the Lambda with an SQS event
          Properties:
            BatchSize: 1
            MaximumBatchingWindowInSeconds: 0
            Enabled: true
            Queue: !GetAtt queue.Arn
      LoggingConfig:
        LogFormat: Text
        LogGroup: !Ref newsScrapeLog

  fastapiLambda:
    Type: AWS::Serverless::Function
    Properties:
      Environment:
        Variables:
          QUERY_LAMBDA_ARN: !GetAtt queryLambda.Arn
          LOG_LEVEL: !Ref logLevel
          DB_TABLE_NAME: !Ref dynamoDB
      FunctionName:
        !Join
        - '-'
        - - !Ref "AWS::StackName"
          - 'fastapi_lambda'
      Architectures:
        - x86_64
      MemorySize: 128
      PackageType: Zip
      Handler: fastapi_lambda.lambda_function.lambda_handler
      Runtime: python3.10
      CodeUri:
        Bucket: !Ref bucketName
        Key: fastapi_lambda
      Description: Provide an API
      Timeout: !Ref timeout
      Policies:
        - AWSLambdaBasicExecutionRole
        - AWSLambda_ReadOnlyAccess
        - AmazonOpenSearchServiceFullAccess
        - !Ref dynamoDbAccess
        - !Ref invokeQueryLambdaPolicy
      Layers:
        - Ref: pythonLibLayer
      LoggingConfig:
        LogFormat: Text
        LogGroup: !Ref fastapiLambdaLog

  fastapiLambdaLog:
    Type: AWS::Logs::LogGroup
    Properties:
      LogGroupClass: STANDARD
      RetentionInDays: 3
      LogGroupName:
        !Join
        - '-'
        - - !Ref "AWS::StackName"
          - 'fastapi_lambda-log'

  queryLambdaLog:
    Type: AWS::Logs::LogGroup
    Properties:
      LogGroupClass: STANDARD
      RetentionInDays: 3
      LogGroupName:
        !Join
        - '-'
        - - !Ref "AWS::StackName"
          - 'query_lambda-log'

  queryLambda:
    Type: AWS::Serverless::Function
    Properties:
      Environment:
        Variables:
          OPENSEARCH_URL: !GetAtt ossDomain.DomainEndpoint
          LOG_LEVEL: !Ref logLevel
          DB_TABLE_NAME: !Ref dynamoDB
      FunctionName:
        !Join
        - '-'
        - - !Ref "AWS::StackName"
          - 'query_lambda'
      Architectures:
        - x86_64
      MemorySize: 128
      PackageType: Zip
      Handler: query_lambda.lambda_function.lambda_handler
      Runtime: python3.10
      CodeUri:
        Bucket: !Ref bucketName
        Key: query_lambda
      Description: Query the LLM
      Timeout: !Ref timeout
      Policies:
        - AWSLambdaBasicExecutionRole
        - AWSLambda_ReadOnlyAccess
        - AmazonOpenSearchServiceFullAccess
        - AmazonBedrockFullAccess
        - !Ref dynamoDbAccess
      Layers:
        - Ref: pythonLibLayer
      LoggingConfig:
        LogFormat: Text
        LogGroup: !Ref queryLambdaLog

  dynamoDB:
    Type: AWS::DynamoDB::Table
    Properties:
      AttributeDefinitions:
        -
          AttributeName: "id"
          AttributeType: "S"
      KeySchema:
        -
          AttributeName: "id"
          KeyType: "HASH"
      ProvisionedThroughput:
        ReadCapacityUnits: "3"
        WriteCapacityUnits: "3"
      TableName:
        !Join
        - '-'
        - - !Ref "AWS::StackName"
          - 'dynamodb'
      TimeToLiveSpecification:
        AttributeName: expiryTimestamp
        Enabled: true

  dynamoDbAccess:
    Type: AWS::IAM::ManagedPolicy
    Properties:
      Description: Give lambdas CRUD access to dynamoDB
      PolicyDocument:
        Version: "2012-10-17"
        Statement:
          - Effect: Allow
            Action:
              - "dynamodb:DeleteItem"
              - "dynamodb:GetItem"
              - "dynamodb:PutItem"
              - "dynamodb:Query"
              - "dynamodb:Scan"
              - "dynamodb:UpdateItem"
              - "dynamodb:BatchWriteItem"
            Resource:
              - !GetAtt dynamoDB.Arn
              - !Sub ${dynamoDB.Arn}/index/*

  invokeQueryLambdaPolicy:
    Type: AWS::IAM::ManagedPolicy
    Properties:
      Description: Give access to query_lambda
      PolicyDocument:
        Version: "2012-10-17"
        Statement:
          - Effect: Allow
            Action:
              - "lambda:InvokeFunction"
            Resource:
              - !GetAtt queryLambda.Arn

  deletionLambdaLog:
    Type: AWS::Logs::LogGroup
    Properties:
      LogGroupClass: STANDARD
      RetentionInDays: 3
      LogGroupName:
        !Join
        - '-'
        - - !Ref "AWS::StackName"
          - 'deletion_lambda-log'

  deletionLambda:
    Type: AWS::Serverless::Function
    Properties:
      Environment:
        Variables:
          OPENSEARCH_URL: !GetAtt ossDomain.DomainEndpoint
          LOG_LEVEL: !Ref logLevel
      FunctionName:
        !Join
        - '-'
        - - !Ref "AWS::StackName"
          - 'deletion_lambda'
      Architectures:
        - x86_64
      MemorySize: 128
      PackageType: Zip
      Handler: deletion_lambda.lambda_function.lambda_handler
      Runtime: python3.10
      CodeUri:
        Bucket: !Ref bucketName
        Key: deletion_lambda
      Description: Do a periodic deletion of old OSS documents
      Timeout: !Ref timeout
      Policies:
        - AWSLambdaBasicExecutionRole
        - AWSLambda_ReadOnlyAccess
        - AmazonOpenSearchServiceFullAccess
      Layers:
        - Ref: pythonLibLayer
      Events:
        ScheduledEvent:
          Type: ScheduleV2
          Description: Repeating trigger for the Lambda at 2am every day
          Properties:
            State: ENABLED
            RetryPolicy:
              MaximumRetryAttempts: 2
              MaximumEventAgeInSeconds: 240
            ScheduleExpression: "cron(0 2 * * ? *)"
            Name:
              !Join
              - '-'
              - - !Ref "AWS::StackName"
                - 'deletion_lambda-schedule'
      LoggingConfig:
        LogFormat: Text
        LogGroup: !Ref deletionLambdaLog