BRS-275: Invocation error trigger for Lambda (#88)

lambda/cloudwatchAlarm/index.js

@@ -0,0 +1,76 @@
+const axios = require('axios');
+const { utcToZonedTime } = require('date-fns-tz');
+const { timeZone } = require('../dynamoUtil');
+const ROCKETCHAT_URL = process.env.ROCKETCHAT_URL;
+const ROCKETCHAT_BEARER_TOKEN = process.env.ROCKETCHAT_BEARER_TOKEN;
+const AWS_ACCOUNT_LIST = JSON.parse(process.env.AWS_ACCOUNT_LIST);
+
+exports.handler = async (event, context) => {
+  console.log('Cloudwatch Alarm Event:', event, context);
+  try {
+    // parse through the records
+    for(const record of event.Records) {
+      // Event this to Rocket.cat
+      console.log("record.body.Subject:", record.body);
+      const body = JSON.parse(record.body);
+      console.log("body:", body);
+      const message = JSON.parse(body.Message);
+
+      // Build the message fields.
+      let fields = [];
+      fields.push({
+        "title": "Alarm Description",
+        "value": message.AlarmDescription,
+        "short": true
+      });
+      fields.push({
+        "title": "AWS Account ID",
+        "value": message.AWSAccountId,
+        "short": true
+      });
+      fields.push({
+        "title": "Date (America/Vancouver Time)",
+        "value": utcToZonedTime(message.StateChangeTime, timeZone),
+        "short": true
+      });
+      fields.push({
+        "title": "Date (UTC Time)",
+        "value": message.StateChangeTime,
+        "short": true
+      });
+      fields.push({
+        "title": "ARN",
+        "value": message.AlarmArn,
+        "short": true
+      });
+
+      try {
+        await axios({
+          method: 'post',
+          url: ROCKETCHAT_URL,
+          headers: {
+            Authorization: ROCKETCHAT_BEARER_TOKEN,
+            'Content-Type': 'application/json'
+          },
+          data: {
+            "emoji": ":interrobang:",
+            "text": record.body.Subject,
+            "attachments": [
+              {
+                "title": `${AWS_ACCOUNT_LIST[message.AWSAccountId]} Errors`,
+                "fields": fields,
+                "color": "#eb1414"
+              }
+            ]
+          }
+        });
+      } catch (e) {
+        console.log("Error, couldn't send notification.", e);
+      }
+    }
+  } catch (e) {
+    console.log("Error parsing cloudwatch alarm data!", e);
+  }
+
+    return {};
+};

serverless.yml

@@ -45,6 +45,9 @@ functions:
           path: /captcha/audio
           cors: true
 
+  cloudwatchAlarm:
+    handler: lambda/cloudwatchAlarm/index.handler
+
   ###########
   # config
   ###########

terraform/src/cloudwatchAlarms.tf

@@ -0,0 +1,136 @@
+resource "aws_lambda_function" "cloudwatch_alarm" {
+  function_name = "cloudwatchAlarm"
+
+  filename         = "artifacts/cloudwatchAlarm.zip"
+  source_code_hash = filebase64sha256("artifacts/cloudwatchAlarm.zip")
+
+  handler = "lambda/cloudwatchAlarm/index.handler"
+  runtime = "nodejs14.x"
+  timeout = 30
+  publish = "true"
+
+  role = aws_iam_role.readRole.arn
+
+  environment {
+    variables = {
+      AWS_ACCOUNT_LIST = data.aws_ssm_parameter.aws_account_list.value,
+      ROCKETCHAT_URL = data.aws_ssm_parameter.rocketchat_url.value,
+      ROCKETCHAT_BEARER_TOKEN = data.aws_ssm_parameter.rocketchat_bearer_token.value,
+    }
+  }
+}
+
+resource "aws_sns_topic" "cloudwatch_error_alarm" {
+  name            = "lambda-error-topic"
+}
+
+data "aws_iam_policy_document" "sns-topic-policy" {
+  policy_id = "__default_policy_ID"
+
+  statement {
+    sid    = "__default_statement_ID"
+    effect = "Allow"
+
+    actions = [
+      "SNS:Subscribe",
+      "SNS:SetTopicAttributes",
+      "SNS:RemovePermission",
+      "SNS:Receive",
+      "SNS:Publish",
+      "SNS:ListSubscriptionsByTopic",
+      "SNS:GetTopicAttributes",
+      "SNS:DeleteTopic",
+      "SNS:AddPermission",
+    ]
+
+    condition {
+      test     = "StringEquals"
+      variable = "AWS:SourceOwner"
+      values   = ["${var.target_aws_account_id}"]
+    }
+
+    principals {
+      type        = "AWS"
+      identifiers = ["*"]
+    }
+
+    resources = ["${aws_sns_topic.cloudwatch_error_alarm.arn}"]
+  }
+
+  statement {
+    sid     = "AWSEvents_capture-autoscaling-events_SendToSNS"
+    effect  = "Allow"
+    actions = ["SNS:Publish"]
+
+    principals {
+      type        = "Service"
+      identifiers = ["cloudwatch.amazonaws.com"]
+    }
+
+    resources = ["${aws_sns_topic.cloudwatch_error_alarm.arn}"]
+  }
+}
+
+
+resource "aws_sns_topic_policy" "default" {
+  arn = aws_sns_topic.cloudwatch_error_alarm.arn
+  policy = "${data.aws_iam_policy_document.sns-topic-policy.json}"
+}
+
+resource "aws_cloudwatch_metric_alarm" "lambda_alert" {
+  alarm_name                = "lambda-error-alert"
+  comparison_operator       = "GreaterThanThreshold"
+  evaluation_periods        = "1"
+  metric_name               = "Errors"
+  namespace                 = "AWS/Lambda"
+  period                    = "10"
+  statistic                 = "Sum"
+  threshold                 = "0"
+  alarm_description         = "This metric monitors all Lambda function invocation errors"
+  datapoints_to_alarm       = "1"
+  insufficient_data_actions = []
+  alarm_actions             = [aws_sns_topic.cloudwatch_error_alarm.arn]
+}
+
+resource "aws_sqs_queue" "alarm_queue" {
+  name                      = "cloudwatch-alarm-queue"
+  message_retention_seconds = 86400
+}
+
+resource "aws_sns_topic_subscription" "user_updates_sqs_target" {
+  topic_arn = aws_sns_topic.cloudwatch_error_alarm.arn
+  protocol  = "sqs"
+  endpoint  = aws_sqs_queue.alarm_queue.arn
+}
+
+resource "aws_lambda_event_source_mapping" "event_source_mapping" {
+  event_source_arn = aws_sqs_queue.alarm_queue.arn
+  enabled          = true
+  function_name    = aws_lambda_function.cloudwatch_alarm.arn
+  batch_size       = 1
+}
+
+resource "aws_sqs_queue_policy" "sqs_queue_policy" {
+  queue_url = aws_sqs_queue.alarm_queue.id
+
+  policy = <<POLICY
+{
+  "Version": "2012-10-17",
+  "Id": "sqspolicy",
+  "Statement": [
+    {
+      "Sid": "First",
+      "Effect": "Allow",
+      "Principal": "*",
+      "Action": [
+         "sqs:DeleteMessage",
+         "sqs:GetQueueAttributes",
+         "sqs:ReceiveMessage",
+         "sqs:SendMessage"
+      ],
+      "Resource": "${aws_sqs_queue.alarm_queue.arn}"
+    }
+  ]
+}
+POLICY
+}

terraform/src/params.tf

@@ -33,3 +33,15 @@ data "aws_ssm_parameter" "s3_bucket_data" {
 data "aws_ssm_parameter" "public_url" {
   name = "/parks-reso-public/url"
 }
+
+data "aws_ssm_parameter" "aws_account_list" {
+  name = "/parks-reso-api/aws_account_list"
+}
+
+data "aws_ssm_parameter" "rocketchat_url" {
+  name = "/parks-reso-api/rocketchat_url"
+}
+
+data "aws_ssm_parameter" "rocketchat_bearer_token" {
+  name = "/parks-reso-api/rocketchat_bearer_token"
+}