template.yaml

AWSTemplateFormatVersion: '2010-09-09'
Transform: AWS::Serverless-2016-10-31

Description: |
  Send logs to Coralogix from AWS (S3, Cloudtrail, Cloudwatch, msk, SNS, SQS, Kinesis and more)
  Please report any issues to: github.com/coralogix/coralogix-aws-shipper/issues
Metadata:
  AWS::ServerlessRepo::Application:
    Name: coralogix-aws-shipper
    Description: Send logs to Coralogix from AWS (S3, Cloudtrail, Cloudwatch, msk, SNS, SQS, Kinesis and more)
    Author: Coralogix
    SpdxLicenseId: Apache-2.0
    LicenseUrl: LICENSE
    ReadmeUrl: README.md
    # NOTE: there is a 10 Label limit for metadata in the AWS Serverless Repo metadata
    Labels:
      - coralogix
      - logs
      - sns
      - s3
      - cloudwatch
      - cloudtrail
      - vpc
      - sqs
      - kinesis
      - cloudfront
    HomePageUrl: https://coralogix.com
    SemanticVersion: 1.0.13
    SourceCodeUrl: https://github.com/coralogix/coralogix-aws-shipper

  AWS::CloudFormation::Interface:
    ParameterGroups:
      - Label:
          default: Coralogix configuration
        Parameters:
          - IntegrationType
          - CoralogixRegion
          - CustomDomain
          - ApplicationName
          - SubsystemName
          - ApiKey
          - StoreAPIKeyInSecretsManager
      - Label:
          default: S3/CloudTrail/VpcFlow/S3Csv configuration
        Parameters:
          - S3BucketName
          - S3KeyPrefix
          - S3KeySuffix
          - NewlinePattern
          - SNSTopicArn
          - SQSTopicArn
          - CSVDelimiter
      - Label:
          default: Kafka & MSK configuration
        Parameters:
          - KafkaTopic
          - KafkaBrokers
          - KafkaSubnets
          - KafkaSecurityGroups
          - MSKClusterArn
          - KafkaBatchSize
      - Label:
          default: Cloudwatch configuration
        Parameters:
          - CloudWatchLogGroupName
          - CloudWatchLogGroupPrefix
      - Label:
          default: SNS configuration
        Parameters:
          - SNSIntegrationTopicArn
      - Label:
          default: SQS configuration
        Parameters:
          - SQSIntegrationTopicArn
      - Label:
          default: Kinesis configuration
        Parameters:
          - KinesisStreamArn
      - Label:
          default: Generic Config
        Parameters:
          - NotificationEmail
          - BlockingPattern
          - SamplingRate
          - AddMetadata
          - CustomMetadata
      - Label:
          default: Lambda configuration
        Parameters:
          - FunctionMemorySize
          - FunctionTimeout
          - LogLevel
          - LambdaLogRetention
          - FunctionRunTime
          - FunctionArchitectures
      - Label:
          default: VPC configuration Optional
        Parameters:
          - LambdaSubnetID
          - LambdaSecurityGroupID
          - UsePrivateLink

Parameters:
  CoralogixRegion:
    Type: String
    Description: |
      The Coralogix location region, possible options are [EU1, EU2, AP1, AP2, US1, US2, Custom]
      If this value is set to Custom you must specify the Custom Domain to use via the CustomDomain parameter.
    AllowedValues:
      - EU1
      - EU2
      - AP1
      - AP2
      - US1
      - US2
      - Custom
    Default: Custom
  
  CustomDomain:
    Type: String
    Description: The Custom Domain. If set, will be the domain used to send telemetry (e.g. cx123.coralogix.com)
    Default: ''

  ApiKey:
    Type: String
    Description: |
      Your Coralogix Send Your Data - API Key which is used to validate your authenticity (https://coralogix.com/docs/send-your-data-api-key/)
      This value can be a Coralogix API Key or an AWS Secret Manager ARN that holds the API Key
    MinLength: 1
    NoEcho: true

  ApplicationName:
    Type: String
    Description: The name of your application (https://coralogix.com/docs/application-and-subsystem-names/)
    MinLength: 1

  SubsystemName:
    Type: String
    Description: The subsystem name of your application (https://coralogix.com/docs/application-and-subsystem-names/)
    Default: ''

  NewlinePattern:
    Type: String
    Description: Regular expression to detect a new log line for multiline logs from S3 source, e.g., use expression \n(?=\d{2}\-\d{2}\s\d{2}\:\d{2}\:\d{2}\.\d{3})
    Default: ''
  
  AddMetadata:
    Type: String
    Description: |
      Add metadata to the log message. Expects comma separated values. Options are bucket_name,key_name,stream_name
    Default: ''

  CustomMetadata:
    Type: String
    Description: |
      Add custom metadata to the log message. Expects comma separated values. Options are key1=value1,key2=value2
    Default: ''

  BlockingPattern:
    Type: String
    Description: Regular expression to detect lines that should be excluded from sent to Coralogix
    Default: ''

  SamplingRate:
    Type: Number
    Description: Send messages with specific rate (1 out of N) e.g., put the value 10 if you want to send every 10th log 
    MinValue: 1
    Default: 1

  LogLevel:
    Type: String
    Description: 'Log level for the Lambda function. Can be one of: INFO, WARN, ERROR, DEBUG'
    AllowedValues:
      - "INFO"
      - "WARN"
      - "ERROR"
      - "DEBUG"
    Default: "WARN"

  S3BucketName:
    Type: String
    Description: |
      The name of the AWS S3 bucket to watch
    AllowedPattern: '^[0-9A-Za-z\.\-_]*(?<!\.)$'
    Default: "none"
    MaxLength: 63

  CSVDelimiter:
    Type: String
    Description: |
      The delimiter used in the CSV file to process
      This value is applied when the S3Csv integration type is selected
    MaxLength: 1
    Default: ','

  S3KeyPrefix:
    Type: String
    Description: |
      The AWS S3 path prefix to watch. This value is ignored
      when the SNSTopicArn / SQSTopicArn parameter is provided.
    MaxLength: 1024
    Default: ''

  S3KeySuffix:
    Type: String
    Description: |
      The AWS S3 path suffix to watch. This value is ignored
      when the SNSTopicArn parameter is provided.
    MaxLength: 1024
    Default: ''

  FunctionMemorySize:
    Type: Number
    Description: Memory size for lambda function
    MinValue: 128
    MaxValue: 10240
    Default: 1024

  FunctionTimeout:
    Type: Number
    Description: Timeout for the lambda function
    MinValue: 30
    MaxValue: 900
    Default: 300
  
  FunctionRunTime:
    Type: String
    Description: Type of runtime for the lambda, allowd values are provided.al2023 or provided.al2
    Default: provided.al2023
    AllowedValues:
      - provided.al2023
      - provided.al2

  FunctionArchitectures:
    Type: String
    Description: Architectures for the lambda function, allowd values are arm64 or x86_64
    Default: 'arm64'
    AllowedValues:
      - arm64
      - x86_64

  SNSTopicArn:
    Type: String
    Description: The ARN for the SNS topic that contains the SNS subscription responsible for retrieving logs from Amazon S3
    # placeholder value to avoid cfn-lint issues
    Default: 'arn:aws:sns:us-east-1:123456789012:placeholder'

  SQSTopicArn:
    Type: String
    Description: The ARN for the SQS topic that contains the SQS subscription responsible for retrieving logs from Amazon S3
    # placeholder value to avoid cfn-lint issues
    Default: 'arn:aws:sqs:us-east-1:123456789012:placeholder'

  SQSIntegrationTopicArn:
    Type: String
    Description: The ARN of SQS topic to subscribe to retrieving messages
    # placeholder value to avoid cfn-lint issues
    Default: 'arn:aws:sqs:us-east-1:123456789012:placeholder'

  SNSIntegrationTopicArn:
    Type: String
    Description: The ARN of SNS topic to subscribe to retrieving messages
    # placeholder value to avoid cfn-lint issues
    Default: 'arn:aws:sns:us-east-1:123456789012:placeholder'

  KinesisStreamArn:
    Type: String
    Description: The ARN of Kinesis stream to subscribe to retrieving messages
    Default: ''

  IntegrationType:
    Type: String
    Description: 'The integration type. Can be one of: S3, CloudTrail, VpcFlow, CloudWatch, S3Csv, Sns, Sqs, Kinesis, CloudFront, Kafka, MSK, EcrScan'
    AllowedValues:
      - S3
      - CloudTrail
      - CloudWatch
      - VpcFlow
      - S3Csv
      - Sns
      - Sqs
      - Kinesis
      - CloudFront
      - Kafka
      - MSK
      - EcrScan
    Default: S3

  CloudWatchLogGroupName:
    Type: String
    Description: 'A comma separated list of CloudWatch log groups names to watch  e.g, (log-group1,log-group2,log-group3)'
    Default: ''

  CloudWatchLogGroupPrefix:
    Type: String
    Description: 'Prefix of the CloudWatch log groups that will trigger the lambda, in case that your log groups are log-group1, log-group2, log-group3 then you can set the value to `log-group`. When using this variable you will not be able to see the log groups as trigger for the lambda. The parameter dose not replace CloudWatchLogGroupName parameter'
    Default: ''

  LambdaLogRetention:
    Type: Number
    Description: CloudWatch log retention days for logs generated by the Lambda function
    MinValue: 1
    Default: 5

  NotificationEmail:
    Type: String
    Description: Failure notification email address 
    MaxLength: 320
    Default: ''

  StoreAPIKeyInSecretsManager:
    Type: String
    Description: |
      Store the API key in AWS Secrets Manager. ApiKeys are stored in secret manager
      by default. If this option is set to false, the ApiKey will apeear in plain text as an 
      environment variable in the lambda function console.
    AllowedValues:
      - true
      - false
    Default: true

  LambdaSubnetID:
    Type: String
    Description: ID of Subnet into which to deploy the integration
    Default: ''

  LambdaSecurityGroupID:
    Type: String
    Description: ID of the SecurityGroup into which to deploy the integration
    Default: ''

  UsePrivateLink:
    Type: String
    Description: Will you be using our PrivateLink?
    AllowedValues:
      - true
      - false
    Default: false
  
  MSKClusterArn:
    Type: String
    Description: The ARN of the MSK cluster to subscribe to retrieving messages
    Default: ''

  KafkaTopic:
    Type: String
    Description: The name of the Kafka topic to subscribe to retrieving messages
    Default: ''

  KafkaBatchSize:
    Type: Number
    Description: The maximum number of records to retrieve per batch from Kafka
    MinValue: 1
    MaxValue: 10000
    Default: 100

  KafkaBrokers:
    Type: CommaDelimitedList
    Description: |
      Comma-delimited list of host and port pair addresses of your Kafka brokers
      [Not Required when using MSK]
    Default: ''
  
  KafkaSubnets:
    Type: CommaDelimitedList
    Description: |
      The subnets associated with your VPC for each Kafka broker
      [Not Required when using MSK]
    Default: ""

  KafkaSecurityGroups:
    Type: CommaDelimitedList
    Description: |
      The security groups associated with your VPC for each Kafka broker
      [Not Required when using MSK]
    Default: ""

  EnableDLQ:
    Type: String
    Description: Enable Dead Letter Queue for the Lambda function
    AllowedValues:
      - true
      - false
    Default: false

  DLQRetryLimit:
    Type: Number
    Description: The maximum number of times to retry the function execution in case of failure
    MinValue: 0
    MaxValue: 5
    Default: 3

  DLQRetryDelay:
    Type: Number
    Description: The delay in seconds between retries
    MinValue: 0
    MaxValue: 900
    Default: 900
  
  DLQS3Bucket:
    Type: String
    Description: The S3 bucket to store the DLQ failed messages after retry limit is reached
    # placeholder value to avoid cfn-lint issues
    Default: 'none'

Mappings:
  CoralogixRegionMap:
    EU1:
      Domain: coralogix.com
    EU2:
      Domain: eu2.coralogix.com
    AP1:
      Domain: coralogix.in
    AP2:
      Domain: coralogixsg.com
    US1:
      Domain: coralogix.us
    US2:
      Domain: cx498.coralogix.com
    Custom:
      Domain: ""

Conditions:
  # DLQ conditions
  DLQEnabled: !Equals [ !Ref EnableDLQ, 'true' ]
  IsKafkaIntegration: !Equals [ !Ref IntegrationType, 'Kafka' ]
  BlockPatternNotSet: !Equals [ !Ref BlockingPattern, '' ]
  CSVDelimiterUse: !Equals [!Ref IntegrationType, 'S3Csv']
  NewlinePatternNotSet: !Equals [ !Ref NewlinePattern, '' ]
  AddMetadataNotSet: !Equals [ !Ref AddMetadata, '']
  CustomMetadataNotSet: !Equals [ !Ref CustomMetadata, '']
  IsSNSIntegration: !Equals [ !Ref IntegrationType, 'Sns' ]
  UseECRScan: !Equals [ !Ref IntegrationType, 'EcrScan' ]
  IsSQSIntegration: !Equals [ !Ref IntegrationType, 'Sqs' ]
  IsKinesisIntegration: !Equals [ !Ref IntegrationType, 'Kinesis' ]
  IsNotificationEnabled: !Not [ !Equals [ !Ref NotificationEmail, '' ] ]
  IsCustomDomain: !Equals [ !Ref CoralogixRegion, Custom ]
  S3KeyPrefixIsSet: !Not [ !Equals [ !Ref S3KeyPrefix, '' ] ]
  S3SuffixIsSet: !Not [ !Equals [ !Ref S3KeySuffix, '' ] ]
  IsApiKeyNotArn: !Equals [!Ref ApiKey , !Select [0,!Split [":" , !Ref ApiKey]]]
  ApiKeyIsArn: !Not [!Condition IsApiKeyNotArn]
  UseAWSDefaultPrefix: !Or 
    - !Equals [ !Ref IntegrationType, 'VpcFlow' ]
    - !Equals [ !Ref IntegrationType, 'CloudTrail' ]
  UseAWSDefaultVpcFlowSuffix: !Equals [!Ref IntegrationType, 'VpcFlow']
  UseAWSDefaultCloudTrailSuffix: !Equals [!Ref IntegrationType, 'CloudTrail']
  StoreAPIKeyInSecretsManager: !And
    - !Equals [ !Ref StoreAPIKeyInSecretsManager, 'true' ]
    - !Condition IsApiKeyNotArn
  IsPrivateLink: !Equals [ !Ref UsePrivateLink, 'true' ]

  SNSIsSet:
    !Or
    - !Not [ !Equals [ !Ref SNSTopicArn, 'arn:aws:sns:us-east-1:123456789012:placeholder' ] ]
    - !Not [ !Equals [ !Ref SNSIntegrationTopicArn, 'arn:aws:sns:us-east-1:123456789012:placeholder' ] ]
  
  SQSIsSet:
    !Or
    - !Not [ !Equals [ !Ref SQSTopicArn, 'arn:aws:sqs:us-east-1:123456789012:placeholder' ] ]
    - !Not [ !Equals [ !Ref SQSIntegrationTopicArn, 'arn:aws:sqs:us-east-1:123456789012:placeholder' ] ]

  UseKinesisStreamARN: !And 
      - !Not [ !Equals [ !Ref KinesisStreamArn, '' ] ]
      - !Equals [ !Ref CloudWatchLogGroupName, '' ]
  UseVpcConfig: !And 
    - !Not [ !Equals [ !Ref LambdaSubnetID, '' ] ]
    - !Not [ !Equals [ !Ref LambdaSecurityGroupID, '' ] ]
  UseMSK: !And
    - !Not [ !Equals [ !Ref MSKClusterArn, '' ] ]
    - !Not [ !Equals [ !Ref KafkaTopic, '' ] ]
    - !Equals [ !Ref IntegrationType, 'MSK' ]

  S3BucketNameIsSet: !Not [ !Equals [ !Ref S3BucketName, 'none' ] ]

Rules:
  ValidateDLQ:
    RuleCondition: !Equals [ !Ref EnableDLQ, 'true' ]
    Assertions:
      - Assert: !Not [ !Equals [ !Ref DLQS3Bucket, 'none' ] ]
        AssertDescription: DLQS3Bucket must be set when EnableDLQ is set to true

  ValidateCloudWatchLogs:
    RuleCondition: !Not [ !Equals [ !Ref CloudWatchLogGroupName, '' ] ]
    Assertions:
      - Assert: !Not [ !Equals [ !Ref CloudWatchLogGroupName, '' ] ]
        AssertDescription: CloudWatchLogGroupName must be set when IntegrationType is set to cloudwatch
      - Assert: !Equals [ !Ref S3BucketName, 'none' ]
        AssertDescription: S3Bucket parameter must be empty when IntegrationType is set to cloudwatch 

  ValidateCustomDomain:
    RuleCondition: !Equals [ !Ref CoralogixRegion, Custom ] 
    Assertions:
      - Assert: !Not [ !Equals [ !Ref CustomDomain, '' ] ]
        AssertDescription: CustomDomain must be set when CoralogixRegion is set to Custom

  ValidatePrivateLinkConfig:
    RuleCondition: !Equals [ !Ref UsePrivateLink, 'true']
    Assertions:
      - Assert: !And
        - !Not [ !Equals [ !Ref LambdaSubnetID, '' ] ]
        - !Not [ !Equals [ !Ref LambdaSecurityGroupID, '' ] ]
        AssertDescription: Subnet ID and Security Group ID must be set when UsePrivateLink is set to true
  
  ValidationS3Integrations:
    RuleCondition: !Or
      - !Equals [ !Ref IntegrationType, 'S3' ]
      - !Equals [ !Ref IntegrationType, 'CloudTrail' ]
      - !Equals [ !Ref IntegrationType, 'VpcFlow' ]
      - !Equals [ !Ref IntegrationType, 'S3Csv' ]
    Assertions:
      - Assert: !Not [ !Equals [ !Ref S3BucketName, 'none' ] ]
        AssertDescription: S3BucketName must be set when IntegrationType is set to S3, CloudTrail, VpcFlow or S3Csv

  ValidateKafkaIntegrationParams:
    RuleCondition: !Equals [ !Ref IntegrationType, 'Kafka' ]
    Assertions:
      - Assert: !Not [ !Equals [ !Ref KafkaTopic, '' ] ]
        AssertDescription: KafkaTopic must be set when IntegrationType is set to Kafka

  ValidateMSKIntegrationParams:
    RuleCondition: !Equals [ !Ref IntegrationType, 'MSK' ]
    Assertions:
      - Assert: !Not [ !Equals [ !Ref MSKClusterArn, '' ] ]
        AssertDescription: MSKClusterArn must be set when IntegrationType is set to MSK
      - Assert: !Not [ !Equals [ !Ref KafkaTopic, '' ] ]
        AssertDescription: KafkaTopic must be set when IntegrationType is set to MSK
        
  ValidateSNSIntegrationParams:
    RuleCondition: !Equals [ !Ref IntegrationType, 'Sns' ]
    Assertions:
      - Assert: !Not [ !Equals [ !Ref SNSIntegrationTopicArn, 'arn:aws:sns:us-east-1:123456789012:placeholder' ] ]
        AssertDescription: SNSIntegrationTopicArn must be set when IntegrationType is set to Sns

  ValidateSQSIntegrationParams:
    RuleCondition: !Equals [ !Ref IntegrationType, 'Sqs' ]
    Assertions:
      - Assert: !Not [ !Equals [ !Ref SQSIntegrationTopicArn, 'arn:aws:sqs:us-east-1:123456789012:placeholder' ] ]
        AssertDescription: SQSIntegrationTopicArn must be set when IntegrationType is set to Sqs  

Resources:
  Secret:
    Type: 'AWS::SecretsManager::Secret'
    Condition: StoreAPIKeyInSecretsManager
    DeletionPolicy: Delete
    UpdateReplacePolicy: Delete
    Properties:
      Description: Coralogix API Key Secret
      SecretString: !Ref ApiKey
      # using StackId here as using the function name creates a circular dependency
      Name: !Sub 
        - 'lambda/coralogix/${AWS::Region}/coralogix-aws-shipper/${stack}'
        - stack: !Select
            - 1
            - !Split
                - "/"
                - !Select
                    - 5
                    - !Split
                        - ":"
                        - !Ref "AWS::StackId"          

  LambdaFunction:
    Type: AWS::Serverless::Function
    Metadata:
      SamResourceId: LambdaFunction
      BuildMethod: makefile
    Properties:
      Description: Send logs to Coralogix.
      Handler: bootstrap
      Runtime: !Ref FunctionRunTime
      Architectures: 
        - !Ref FunctionArchitectures
      MemorySize: !Ref FunctionMemorySize
      Timeout: !Ref FunctionTimeout
      CodeUri: .
      Policies:
        - !If 
          - S3BucketNameIsSet
          - S3ReadPolicy: 
              BucketName: !Ref S3BucketName
          - !Ref AWS::NoValue

        # if SQS topic ARN is set, add policy
        - !If
          - SQSIsSet
          - SQSPollerPolicy:
              QueueName: !Select
                - 5
                - !Split
                  - ":"
                  - !If
                    - IsSQSIntegration
                    - !Ref SQSIntegrationTopicArn
                    - !Ref SQSTopicArn
          - !Ref AWS::NoValue

        # if kafka/msk integration is set, add policy
        - !If
          - IsKafkaIntegration
          - Statement:
              - Effect: Allow
                Action:
                  - 'ec2:CreateNetworkInterface'
                  - 'ec2:DescribeNetworkInterfaces'
                  - 'ec2:DescribeVpcs'
                  - 'ec2:DeleteNetworkInterface'
                  - 'ec2:DescribeSubnets'
                  - 'ec2:DescribeSecurityGroups'
                Resource: '*'
          - !Ref AWS::NoValue
          
        # if kinesis integration is set, add policy
        - !If
          - IsKinesisIntegration
          - KinesisStreamReadPolicy:
              StreamName: !Select [1, !Split ["/", !Select [5, !Split [":", !Ref KinesisStreamArn]]]]
          - !Ref AWS::NoValue

        # if msk integration is set, add policy
        - !If
          - UseMSK
          - Statement:
              - Effect: Allow
                Action:
                  - 'kafka:DescribeCluster'
                  - 'kafka:GetBootstrapBrokers'
                  - 'kafka:DescribeClusterV2'
                  - 'ec2:DescribeSecurityGroups'
                Resource: !Ref MSKClusterArn
              - Effect: Allow
                Action:
                  - 'ec2:CreateNetworkInterface'
                  - 'ec2:DescribeNetworkInterfaces'
                  - 'ec2:DescribeVpcs'
                  - 'ec2:DeleteNetworkInterface'
                  - 'ec2:DescribeSubnets'
                  - 'ec2:DescribeSecurityGroups'
                Resource: '*'
          - !Ref AWS::NoValue
        - !If
          - ApiKeyIsArn
          - Statement:
              - Effect: Allow
                Action:
                  - 'secretsmanager:GetSecretValue'
                Resource: !Ref ApiKey
          - !If
            - StoreAPIKeyInSecretsManager
            - Statement:
                - Effect: Allow
                  Action:
                    - 'secretsmanager:GetSecretValue'
                  Resource: !Ref Secret
            # Note, this is a hack to get around the fact that you can't have a condition on a policy
            # If the Apikey is not an ARn or we are not storing the key in secrets manager, then we don't need access
            # to secrets manager
            - Statement:
                - Effect: Deny
                  Action:
                    - 'secretsmanager:GetSecretValue'
                  Resource: '*'

        # if ecrscan is enabled, add policy
        - !If
          - UseECRScan
          - Statement:
            - Effect: Allow
              Action:
                - 'ecr:DescribeImageScanFindings'
              Resource: !Sub 'arn:aws:ecr:${AWS::Region}:${AWS::AccountId}:repository/*'
            - Effect: Allow
              Action:
                - 'events:PutEvents'
              Resource: "*"
          - !Ref AWS::NoValue

        # if dlq is enabled, add policy
        - !If
          - DLQEnabled
          - Statement:
              - Effect: Allow
                Action:
                  - 'sqs:SendMessage'
                  - 'sqs:GetQueueAttributes'
                  - 'sqs:GetQueueUrl'
                  - 'sqs:ReceiveMessage'
                  - 'sqs:DeleteMessage'
                Resource: !GetAtt DeadLetterQueue.Arn
              - Effect: Allow
                Action:
                  - 's3:PutObject'
                  - 's3:PutObjectAcl'
                  - 's3:AbortMultipartUpload'
                  - 's3:DeleteObject'
                  - 's3:PutObjectTagging'
                  - 's3:PutObjectVersionTagging'
                Resource: 
                  - !Sub arn:aws:s3:::${DLQS3Bucket}
                  - !Sub arn:aws:s3:::${DLQS3Bucket}/*
          - !Ref AWS::NoValue
    
      VpcConfig: !If
        - UseVpcConfig
        - SecurityGroupIds:
            - !Ref LambdaSecurityGroupID
          SubnetIds:
            - !Ref LambdaSubnetID
        - !Ref AWS::NoValue

      # EventInvokeConfig is place here by default
      # however, the  notification subscription will
      # only take place if the UseSNSTopicARN condition is met
      EventInvokeConfig: 
        DestinationConfig:
          OnFailure:
            Type: SNS

      Environment:
        Variables:
          CORALOGIX_ENDPOINT: !If
            - IsCustomDomain
            - !If
              - IsPrivateLink
              - !Sub
                - https://ingress.private.${domain}
                - domain: !Ref CustomDomain
              - !Sub
                - https://ingress.${domain}
                - domain: !Ref CustomDomain
            - !If
              - IsPrivateLink
              - !Sub
                - https://ingress.private.${domain}
                - domain: !FindInMap [CoralogixRegionMap, !Ref CoralogixRegion , Domain]
              - !Sub
                - https://ingress.${domain}
                - domain: !FindInMap [CoralogixRegionMap, !Ref CoralogixRegion , Domain]
          CORALOGIX_API_KEY: !If
            - StoreAPIKeyInSecretsManager
            - !Ref Secret
            - !Ref ApiKey
          APP_NAME: !Ref ApplicationName
          SUB_NAME: !Ref SubsystemName
          SAMPLING: !Ref SamplingRate
          INTEGRATION_TYPE: !Ref IntegrationType
          RUST_LOG: !Ref LogLevel
          ADD_METADATA: !If
            - AddMetadataNotSet
            - !Ref AWS::NoValue
            - !Ref AddMetadata
          CUSTOM_METADATA: !If
            - CustomMetadataNotSet
            - !Ref AWS::NoValue
            - !Ref CustomMetadata
          NEWLINE_PATTERN: !If
            - NewlinePatternNotSet
            - !Ref AWS::NoValue
            - !Ref NewlinePattern
          BLOCKING_PATTERN: !If
            - BlockPatternNotSet
            - !Ref AWS::NoValue
            - !Ref BlockingPattern
          CSV_DELIMITER: !If
            - CSVDelimiterUse
            - !Ref CSVDelimiter
            - !Ref AWS::NoValue

          DLQ_RETRY_LIMIT: !Ref DLQRetryLimit
          DLQ_ARN: !If
            - DLQEnabled
            - !GetAtt DeadLetterQueue.Arn
            - !Ref AWS::NoValue
          DLQ_URL:
            Fn::If:
              - DLQEnabled
              - !Ref DeadLetterQueue
              - !Ref AWS::NoValue
          DLQ_S3_BUCKET: !If
            - DLQEnabled
            - !Ref DLQS3Bucket
            - !Ref AWS::NoValue

  SNSNotificationSubscription:
    Type: AWS::SNS::Subscription
    Condition: IsNotificationEnabled
    Properties:
      Protocol: email
      Endpoint: !Ref NotificationEmail
      TopicArn: !Ref LambdaFunction.DestinationTopic

  ConfigureLambda:
    Type: Custom::ConfigureLambda
    Properties:
      ServiceToken: !GetAtt CustomResourceFunction.Arn
      LambdaArn: !GetAtt LambdaFunction.Arn
      DLQ:
        EnableDLQ: !Ref EnableDLQ
        RetryLimit: !Ref DLQRetryLimit
        RetryDelay: !Ref DLQRetryDelay
        DLQArn: !If
          - DLQEnabled
          - !GetAtt DeadLetterQueue.Arn
          - !Ref AWS::NoValue
        DLQS3Bucket: !If
          - DLQEnabled
          - !Ref DLQS3Bucket
          - !Ref AWS::NoValue
        DLQ_URL: !If
          - DLQEnabled
          - !Ref DeadLetterQueue
          - !Ref AWS::NoValue
      Parameters:
        CoralogixRegion: !Ref CoralogixRegion
        CustomDomain: !Ref CustomDomain
        ApiKey: !If
          - StoreAPIKeyInSecretsManager
          - !Ref Secret
          - !Ref ApiKey
        ApplicationName: !Ref ApplicationName
        SubsystemName: !Ref SubsystemName
        AddMetadata: !Ref AddMetadata
        CustomMetadata: !Ref CustomMetadata
        NewlinePattern: !Ref NewlinePattern
        BlockingPattern: !Ref BlockingPattern
        SamplingRate: !Ref SamplingRate
        LogLevel: !Ref LogLevel
        S3BucketName: !Ref S3BucketName
        CSVDelimiter: !Ref CSVDelimiter
        FunctionMemorySize: !Ref FunctionMemorySize
        FunctionTimeout: !Ref FunctionTimeout
        SNSTopicArn: !Ref SNSTopicArn
        SQSTopicArn: !Ref SQSTopicArn
        SQSIntegrationTopicArn: !Ref SQSIntegrationTopicArn
        SNSIntegrationTopicArn: !Ref SNSIntegrationTopicArn
        KinesisStreamArn: !Ref KinesisStreamArn
        IntegrationType: !Ref IntegrationType
        CloudWatchLogGroupName: !Ref CloudWatchLogGroupName
        CloudWatchLogGroupPrefix: !Ref CloudWatchLogGroupPrefix
        LambdaLogRetention: !Ref LambdaLogRetention
        NotificationEmail: !Ref NotificationEmail
        StoreAPIKeyInSecretsManager: !Ref StoreAPIKeyInSecretsManager
        LambdaSubnetID: !Ref LambdaSubnetID
        LambdaSecurityGroupID: !Ref LambdaSecurityGroupID
        UsePrivateLink: !Ref UsePrivateLink
        MSKClusterArn: !Ref MSKClusterArn
        KafkaTopic: !Ref KafkaTopic
        KafkaBatchSize: !Ref KafkaBatchSize
        KafkaBrokers: !Ref KafkaBrokers
        KafkaSubnets: !Ref KafkaSubnets
        KafkaSecurityGroups: !Ref KafkaSecurityGroups
        S3KeyPrefix: !If
          - S3KeyPrefixIsSet
          - !Ref S3KeyPrefix
          - !If
            - UseAWSDefaultPrefix
            - 'AWSLogs/'
            - ''
        S3KeySuffix: !If
          - S3SuffixIsSet
          - !Ref S3KeySuffix
          - !If
            - UseAWSDefaultCloudTrailSuffix
            - '.json.gz'
            - !If
              - UseAWSDefaultVpcFlowSuffix
              - '.log.gz'
              - ""
        CoralogixEndpoint: !If
          - IsCustomDomain
          - !If
            - IsPrivateLink
            - !Sub
              - https://ingress.private.${domain}
              - domain: !Ref CustomDomain
            - !Sub
              - https://ingress.${domain}
              - domain: !Ref CustomDomain
          - !If
            - IsPrivateLink
            - !Sub
              - https://ingress.private.${domain}
              - domain: !FindInMap
                  - CoralogixRegionMap
                  - !Ref CoralogixRegion
                  - Domain
            - !Sub
              - https://ingress.${domain}
              - domain: !FindInMap
                  - CoralogixRegionMap
                  - !Ref CoralogixRegion
                  - Domain
        CoralogixAPIKey: !If
          - StoreAPIKeyInSecretsManager
          - !Ref Secret
          - !Ref ApiKey
    Metadata:
      SamResourceId: ConfigureLambda

  LambdaLogGroup:
    DeletionPolicy: Retain
    UpdateReplacePolicy: Retain
    Type: 'AWS::Logs::LogGroup'
    Properties:
      LogGroupName: !Sub '/aws/lambda/${LambdaFunction}'
      RetentionInDays: !Ref LambdaLogRetention

##################################
# ------------ DLQ --------------#
##################################

  DeadLetterQueue:
    DeletionPolicy: Delete
    UpdateReplacePolicy: Delete
    Condition: DLQEnabled
    Type: AWS::SQS::Queue
    Properties: 
      QueueName: !Sub coralogix-aws-shipper-dlq-${AWS::StackName}
      DelaySeconds: !Ref DLQRetryDelay
      MessageRetentionPeriod: 1209600
      VisibilityTimeout: !Ref FunctionTimeout

##################################
# ---------- Kinesis ------------#
##################################

  KinesisEventSourceMapping:
    Condition: UseKinesisStreamARN
    Type: AWS::Lambda::EventSourceMapping
    Properties:
      BatchSize: 100
      EventSourceArn: !Ref KinesisStreamArn
      FunctionName: !GetAtt LambdaFunction.Arn
      StartingPosition: LATEST


##################################
# ------------ MSK-- ------------#
##################################

  MSKEventSourceMapping:
    Condition: UseMSK
    Type: AWS::Lambda::EventSourceMapping
    Properties:
      BatchSize: !Ref KafkaBatchSize
      EventSourceArn: !Ref MSKClusterArn
      FunctionName: !GetAtt LambdaFunction.Arn
      StartingPosition: LATEST
      Topics: !Split [",", !Ref KafkaTopic]

##################################
# ------------ SQS-- ------------#
##################################

  SQSEventSourceMapping:
    Condition: SQSIsSet
    Type: AWS::Lambda::EventSourceMapping
    Properties:
      BatchSize: 10
      FunctionName: !GetAtt LambdaFunction.Arn
      # StartingPosition: LATEST
      EventSourceArn: !If
        - IsSQSIntegration
        - !Ref SQSIntegrationTopicArn
        - !Ref SQSTopicArn


  SQSEventPermission:
    Condition: SQSIsSet
    Type: AWS::Lambda::Permission
    Properties:
      Action: lambda:InvokeFunction
      FunctionName: !GetAtt LambdaFunction.Arn
      Principal: sns.amazonaws.com
      SourceArn: !If
        - IsSQSIntegration
        - !Ref SQSIntegrationTopicArn
        - !Ref SQSTopicArn
  
##################################
# ------------ SNS-- ------------#
##################################

  SNSSubscription:
    Condition: SNSIsSet
    Type: AWS::SNS::Subscription
    Properties:
      Protocol: lambda
      Endpoint: !GetAtt LambdaFunction.Arn
      TopicArn: !If
        - IsSNSIntegration
        - !Ref SNSIntegrationTopicArn
        - !Ref SNSTopicArn

  SNSEventPermission:
    Condition: SNSIsSet
    Type: AWS::Lambda::Permission
    Properties:
      Action: lambda:InvokeFunction
      FunctionName: !GetAtt LambdaFunction.Arn
      Principal: sns.amazonaws.com
      SourceArn: !If
        - IsSNSIntegration
        - !Ref SNSIntegrationTopicArn
        - !Ref SNSTopicArn


##################################
# -------  ECR Scan Image -------#
##################################

  LambdaFunctionEcrInvokePermission:
    Condition: UseECRScan
    Type: AWS::Lambda::Permission
    Properties:
      FunctionName: !GetAtt LambdaFunction.Arn
      Action: lambda:InvokeFunction
      Principal: events.amazonaws.com
      SourceAccount: !Ref AWS::AccountId
      SourceArn: !GetAtt ECRScanTrigger.Arn

  ECRScanTrigger:
    Condition: UseECRScan
    Type: AWS::Events::Rule
    Properties:
      Description: "Trigger for ECR image scan completion"
      State: ENABLED
      EventPattern:
        source:
          - "aws.ecr"
        detail-type:
          - "ECR Image Scan"
        detail:
          scan-status:
            - "COMPLETE"
      Targets:
        - Arn: !GetAtt LambdaFunction.Arn
          Id: "TargetFunctionV1"

##################################
# -- Custom Resource Definition -#
##################################
  CustomResourceFunction:
    Type: AWS::Serverless::Function
    Metadata:
      SamResourceId: CustomResourceFunction
    Properties:
      Handler: index.lambda_handler
      Runtime: python3.12
      Timeout: 900
      Policies:
      - Statement:
        - Sid: EC2Access
          Effect: Allow
          Action:
            - 'ec2:CreateNetworkInterface'
            - 'ec2:DescribeNetworkInterfaces'
            - 'ec2:DescribeVpcs'
            - 'ec2:DeleteNetworkInterface'
            - 'ec2:DescribeSubnets'
            - 'ec2:DescribeSecurityGroups'
          Resource: '*'
      - Statement:
        - Sid: LambdaAccess
          Effect: Allow
          Action:
          - lambda:*
          Resource: !GetAtt LambdaFunction.Arn
      - Statement:
        - Sid: EventSourceMappings
          Effect: Allow
          Action:
          - lambda:ListEventSourceMappings
          - lambda:CreateEventSourceMapping
          - lambda:DeleteEventSourceMapping
          - lambda:UpdateEventSourceMapping
          Resource: '*'
      - Statement:
        - Sid: S3NotificationPolicy
          Effect: Allow
          Action:
          - s3:GetBucketNotification
          - s3:PutBucketNotification
          Resource:
            Fn::Sub: arn:aws:s3:::${S3BucketName}
        - Sid: PutSubscriptionFilter
          Effect: Allow
          Action:
          - logs:PutSubscriptionFilter
          - logs:DeleteSubscriptionFilter
          - logs:DescribeSubscriptionFilters
          Resource: !Sub arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:*:*
      CodeUri: custom-resource/