template.yml

AWSTemplateFormatVersion: '2010-09-09'
Transform: AWS::Serverless-2016-10-31

Parameters:
  ExecutionRoleArn:
    Type: String
    Default: arn:aws:iam::117819748843:role/service-role

  DeploymentZone:
    Type: String
    Default: deployment-zone-117819748843-us-east-1

  EmrZone:
    Type: String
    Default: log-zone-117819748843-us-east-1

  DockerImageUri:
    Type: String
    Default: 117819748843.dkr.ecr.us-east-1.amazonaws.com/emr-serverless:weather-etl

  SecurityGroupIds:
    Type: CommaDelimitedList
    Default: sg-002c9a50cf295d5ad

  SubnetIds:
    Type: CommaDelimitedList
    Default: subnet-07fa6b708d8df66b4

Resources:
  ETLJobTrigger:
    Type: AWS::Events::Rule
    Properties: 
      # Description: String
      Name: weather-etl-trigger
      RoleArn: !Ref ExecutionRoleArn
      ScheduleExpression: "cron(0 6 * * ? *)"
      State: ENABLED
      Targets:
        - Id: ETLStepFunction
          Arn: !GetAtt StepFunctionStateMachine.Arn
          Input: "{\n  \"input\": \"{}\"\n}"
          RetryPolicy:
            MaximumEventAgeInSeconds: 300
            MaximumRetryAttempts: 3
          RoleArn: !Ref ExecutionRoleArn

  EMRServerlessApplication:
    Type: AWS::EMRServerless::Application
    Properties: 
      Name: weather-etl
      Architecture: X86_64
      AutoStartConfiguration: 
        Enabled: true
      AutoStopConfiguration: 
        Enabled: true
        IdleTimeoutMinutes: 1
      MaximumCapacity: 
        Cpu: 4vCPU
        Disk: 100GB
        Memory: 16GB
      ReleaseLabel: "emr-6.12.0"
      Type: Spark
      ImageConfiguration:
        ImageUri: !Ref DockerImageUri
      NetworkConfiguration:
        SecurityGroupIds: !Ref SecurityGroupIds
        SubnetIds: !Ref SubnetIds

  
  StepFunctionStateMachine:
    Type: AWS::StepFunctions::StateMachine
    Properties:
      StateMachineName: weather-etl
      RoleArn: !Ref ExecutionRoleArn
      Definition: {
            "Comment": "Runs EMR Serverless job for Weather ETL",
            "StartAt": "StartJobRun",
            "States": {
                "StartJobRun": {
                  "Type": "Task",
                  "End": true,
                  "Parameters": {
                      "Name": "Weather ETL",
                      "ClientToken.$": "States.UUID()",
                      "ApplicationId": !Ref EMRServerlessApplication,
                      "ExecutionRoleArn": !Ref ExecutionRoleArn,
                      "JobDriver": {
                        "SparkSubmit": {
                            "EntryPoint": !Sub "s3://${DeploymentZone}/weather_etl/spark_entrypoint.py",
                            "SparkSubmitParameters": !Sub "
                                --driver-memory 2G 
                                --driver-cores 2 
                                --num-executors 1 
                                --executor-memory 2G
                                --executor-cores 2
                                --conf spark.dynamicAllocation.enabled=false
                                --conf spark.sql.execution.arrow.pyspark.enabled=true
                                --jars https://repo1.maven.org/maven2/org/postgresql/postgresql/42.6.0/postgresql-42.6.0.jar
                              "
                        }
                      },
                      "ConfigurationOverrides": {
                        "MonitoringConfiguration": {
                          "S3MonitoringConfiguration": {
                            "LogUri": !Sub "s3://${EmrZone}/logging/"
                          }
                        }
                      },
                      "ExecutionTimeoutMinutes": 600
                  },
                  "Resource": "arn:aws:states:::aws-sdk:emrserverless:startJobRun"
                }
            }
        }