Signoz alert manager setup for SMTP

README.md

@@ -10,24 +10,32 @@ This repo is used to deploy an EKS cluster to AWS. CI/CD is managed through Spac
 │   └── policies: Rego policies that can be attached to 0..* spacelift stacks
 ├── dev: Development/sandbox environment
 │   ├── spacelift: Terraform scripts to manage spacelift resources
-│   │   └── dpe-sandbox: Spacelift specific resources to manage the CI/CD pipeline
+│   │   └── dpe-k8s/dpe-sandbox: Spacelift specific resources to manage the CI/CD pipeline
 │   └── stacks: The deployable cloud resources
+│       ├── dpe-auth0: Stack used to provision and setup auth0 IDP (Identity Provider) settings
 │       ├── dpe-sandbox-k8s: K8s + supporting AWS resources
 │       └── dpe-sandbox-k8s-deployments: Resources deployed inside of a K8s cluster
 └── modules: Templatized collections of terraform resources that are used in a stack
     ├── apache-airflow: K8s deployment for apache airflow
     │   └── templates: Resources used during deployment of airflow
     ├── argo-cd: K8s deployment for Argo CD, a declarative, GitOps continuous delivery tool for Kubernetes.
     │   └── templates: Resources used during deployment of this helm chart
-    ├── trivy-operator: K8s deployment for trivy, along with a few supporting charts for security scanning
-    │   └── templates: Resources used during deployment of these helm charts
-    ├── victoria-metrics: K8s deployment for victoria metrics, a promethus like tool for cluster metric collection
-    │   └── templates: Resources used during deployment of these helm charts
+    ├── cert-manager: Handles provisioning TLS certificates for the cluster
+    ├── envoy-gateway: API Gateway for the cluster securing and providing secure traffic into the cluster
+    ├── postgres-cloud-native: Used to provision a postgres instance
+    ├── postgres-cloud-native-operator: Operator that manages the lifecycle of postgres instances on the cluster
     ├── demo-network-policies: K8s deployment for a demo showcasing how to use network policies
     ├── demo-pod-level-security-groups-strict: K8s deployment for a demo showcasing how to use pod level security groups in strict mode
     ├── sage-aws-eks: Sage specific EKS cluster for AWS
+    ├── sage-aws-eks-addons: Sets up additional resources that need to be installed post creation of the EKS cluster
     ├── sage-aws-k8s-node-autoscaler: K8s node autoscaler using spotinst ocean
-    └── sage-aws-vpc: Sage specific VPC for AWS
+    ├── sage-aws-ses: AWS SES (Simple email service) setup
+    ├── sage-aws-vpc: Sage specific VPC for AWS
+    ├── signoz: SigNoz provides APM, logs, traces, metrics, exceptions, & alerts in a single tool
+    ├── trivy-operator: K8s deployment for trivy, along with a few supporting charts for security scanning
+    │   └── templates: Resources used during deployment of these helm charts
+    ├── victoria-metrics: K8s deployment for victoria metrics, a promethus like tool for cluster metric collection
+    │   └── templates: Resources used during deployment of these helm charts
 ```
 
 This root `main.tf` contains all the "Things" that are going to be deployed. 
@@ -283,10 +291,27 @@ This document describes the abbreviated process below:
 				"iam:*PolicyVersion",
 				"iam:*OpenIDConnectProvider",
 				"iam:*InstanceProfile",
-				"iam:ListPolicyVersions"
+				"iam:ListPolicyVersions",
+				"iam:ListGroupsForUser",
+                "iam:ListAttachedUserPolicies"
 			],
 			"Resource": "*"
-		}
+		},
+        {
+            "Effect": "Allow",
+            "Action": [
+                "iam:CreateUser",
+                "iam:AttachUserPolicy",
+                "iam:ListPolicies",
+                "iam:TagUser",
+                "iam:GetUser",
+                "iam:DeleteUser",
+                "iam:CreateAccessKey",
+                "iam:ListAccessKeys",
+                "iam:DeleteAccessKeys"
+            ],
+            "Resource": "arn:aws:iam::{{AWS ACCOUNT ID}}:user/smtp_user"
+        }
 	]
 }
 ```

deployments/main.tf

@@ -70,7 +70,13 @@ module "dpe-sandbox-spacelift-development" {
   enable_cluster_ingress = true
   enable_otel_ingress    = true
   ssl_hostname           = "a09a38cc5a8d6497ea69c6bf6318701b-1974793757.us-east-1.elb.amazonaws.com"
+
   auth0_jwks_uri         = "https://dev-sage-dpe.us.auth0.com/.well-known/jwks.json"
+
+  ses_email_identities = ["aws-dpe-dev@sagebase.org"]
+  ses_email_domains    = ["sagebase.org"]
+  # Defines the email address that will be used as the sender of the email alerts
+  smtp_from            = "aws-dpe-dev@sagebase.org"
 }
 
 module "dpe-sandbox-spacelift-production" {
@@ -115,4 +121,7 @@ module "dpe-sandbox-spacelift-production" {
   enable_otel_ingress    = false
   ssl_hostname           = ""
   auth0_jwks_uri         = ""
+
+  ses_email_identities = []
+  ses_email_domains    = []
 }

deployments/spacelift/dpe-k8s/main.tf

@@ -11,6 +11,8 @@ locals {
     private_subnet_cidrs_eks_worker_nodes  = var.private_subnet_cidrs_eks_worker_nodes
     azs_eks_control_plane                  = var.azs_eks_control_plane
     azs_eks_worker_nodes                   = var.azs_eks_worker_nodes
+    ses_email_identities                   = var.ses_email_identities
+    ses_email_domains                      = var.ses_email_domains
   }
 
   k8s_stack_deployments_variables = {
@@ -25,6 +27,7 @@ locals {
     enable_otel_ingress    = var.enable_otel_ingress
     ssl_hostname           = var.ssl_hostname
     auth0_jwks_uri         = var.auth0_jwks_uri
+    smtp_from              = var.smtp_from
   }
 
   auth0_stack_variables = {
@@ -39,6 +42,8 @@ locals {
     private_subnet_ids_eks_worker_nodes = "TF_VAR_private_subnet_ids_eks_worker_nodes"
     node_security_group_id              = "TF_VAR_node_security_group_id"
     pod_to_node_dns_sg_id               = "TF_VAR_pod_to_node_dns_sg_id"
+    smtp_user                           = "TF_VAR_smtp_user"
+    smtp_password                       = "TF_VAR_smtp_password"
   }
 }
 

deployments/spacelift/dpe-k8s/variables.tf

@@ -180,4 +180,33 @@ variable "auth0_clients" {
     description = string
     app_type    = string
   }))
-}
+}
+
+variable "ses_email_identities" {
+  type        = list(string)
+  description = "List of email identities to be added to SES"
+}
+
+variable "ses_email_domains" {
+  type        = list(string)
+  description = "List of email domains to be added to SES"
+}
+
+
+variable "smtp_user" {
+  description = "The SMTP user. Required if smtp_user, smtp_password, and smtp_from are set"
+  type        = string
+  default     = ""
+}
+
+variable "smtp_password" {
+  description = "The SMTP password. Required if smtp_user, smtp_password, and smtp_from are set"
+  type        = string
+  default     = ""
+}
+
+variable "smtp_from" {
+  description = "The SMTP from address. Required if smtp_user, smtp_password, and smtp_from are set"
+  type        = string
+  default     = ""
+}

deployments/stacks/dpe-k8s-deployments/main.tf

@@ -89,6 +89,9 @@ module "signoz" {
   gateway_namespace    = "envoy-gateway"
   cluster_name         = var.cluster_name
   auth0_jwks_uri       = var.auth0_jwks_uri
+  smtp_password        = var.smtp_password
+  smtp_user            = var.smtp_user
+  smtp_from            = var.smtp_from
 }
 
 module "envoy-gateway" {

deployments/stacks/dpe-k8s-deployments/variables.tf

@@ -85,3 +85,26 @@ variable "auth0_jwks_uri" {
   description = "The JWKS URI for Auth0"
   type        = string
 }
+
+variable "auth0_domain" {
+  description = "Auth0 domain"
+  type        = string
+}
+
+variable "smtp_user" {
+  description = "The SMTP user. Required if smtp_user, smtp_password, and smtp_from are set"
+  type        = string
+  default     = ""
+}
+
+variable "smtp_password" {
+  description = "The SMTP password. Required if smtp_user, smtp_password, and smtp_from are set"
+  type        = string
+  default     = ""
+}
+
+variable "smtp_from" {
+  description = "The SMTP from address. Required if smtp_user, smtp_password, and smtp_from are set"
+  type        = string
+  default     = ""
+}

deployments/stacks/dpe-k8s/main.tf

@@ -36,3 +36,10 @@ module "sage-aws-eks" {
   private_subnet_ids_eks_control_plane = module.sage-aws-vpc.private_subnet_ids_eks_control_plane
   private_subnet_ids_eks_worker_nodes  = module.sage-aws-vpc.private_subnet_ids_eks_worker_nodes
 }
+
+module "sage-aws-ses" {
+  source = "../../../modules/sage-aws-ses"
+
+  email_identities = var.ses_email_identities
+  email_domains    = var.ses_email_domains
+}

deployments/stacks/dpe-k8s/outputs.tf

@@ -37,3 +37,12 @@ output "region" {
 output "cluster_name" {
   value = module.sage-aws-eks.cluster_name
 }
+
+output "smtp_user" {
+  value = module.sage-aws-ses.smtp_user
+}
+
+output "smtp_password" {
+  sensitive = true
+  value     = module.sage-aws-ses.smtp_password
+}

deployments/stacks/dpe-k8s/variables.tf

@@ -54,3 +54,13 @@ variable "azs_eks_worker_nodes" {
   type        = list(string)
   description = "Availability Zones for the EKS worker nodes"
 }
+
+variable "ses_email_identities" {
+  type        = list(string)
+  description = "List of email identities to be added to SES"
+}
+
+variable "ses_email_domains" {
+  type        = list(string)
+  description = "List of email domains to be added to SES"
+}

modules/sage-aws-ses/README.md

@@ -0,0 +1,32 @@
+# Purpose
+This module is used to set up SES (Simple email service) in AWS.
+
+By setting a few variables we are able to create a number of Email addresses and Domains
+to AWS SES. The variables to be set are:
+
+- `email_identities`, example: `["example@sagebase.org"]`
+- `email_domains`, example `["sagebase.org"]`
+
+# Manual steps required
+After running this module a number of manual steps are required as they are external
+processes that need to happen:
+
+## Verify Email address
+1) Navigate to Amazon SES in the web console
+2) Navigate to `identities`
+3) Choose the Identity to verify
+4) Send a test email and click the link recieved to verify the email
+
+Optional: Send a test email after verifying to confirm you may recieve emails
+
+
+## Verify Sending domain
+This is required for each AWS account where AWS SES is going to be set up.
+
+Reading: <https://docs.aws.amazon.com/ses/latest/dg/creating-identities.html#just-verify-domain-proc>
+
+1) Navigate to Amazon SES in the web console
+2) Navigate to `identities`
+3) Choose the Domain to verify
+4) Download the DKIM under `Publish DNS records` and create an IT ticket to add the records
+5) Example IT ticket for reference: <https://sagebionetworks.jira.com/browse/IT-3965>

modules/sage-aws-ses/data.tf

@@ -0,0 +1,6 @@
+data "aws_iam_policy_document" "ses_sender" {
+  statement {
+    actions   = ["ses:SendRawEmail"]
+    resources = ["*"]
+  }
+}

modules/sage-aws-ses/main.tf

@@ -0,0 +1,28 @@
+resource "aws_ses_email_identity" "identities" {
+  for_each = { for identity in var.email_identities : identity => identity }
+  email = each.value
+}
+
+resource "aws_ses_domain_identity" "identities" {
+  for_each = { for identity in var.email_domains : identity => identity }
+  domain = each.value
+}
+
+resource "aws_iam_user" "smtp_user" {
+  name = "smtp_user"
+}
+
+resource "aws_iam_access_key" "smtp_user" {
+  user = aws_iam_user.smtp_user.name
+}
+
+resource "aws_iam_policy" "ses_sender" {
+  name        = "ses_sender"
+  description = "Allows sending of e-mails via Simple Email Service"
+  policy      = data.aws_iam_policy_document.ses_sender.json
+}
+
+resource "aws_iam_user_policy_attachment" "test-attach" {
+  user       = aws_iam_user.smtp_user.name
+  policy_arn = aws_iam_policy.ses_sender.arn
+}

modules/sage-aws-ses/ouputs.tf

@@ -0,0 +1,9 @@
+
+output "smtp_user" {
+  value = aws_iam_access_key.smtp_user.id
+}
+
+output "smtp_password" {
+  sensitive = true
+  value     = aws_iam_access_key.smtp_user.ses_smtp_password_v4
+}

modules/sage-aws-ses/variables.tf

@@ -0,0 +1,17 @@
+variable "email_identities" {
+  type        = list(string)
+  description = "List of email identities to be added to SES"
+}
+
+variable "email_domains" {
+  type        = list(string)
+  description = "List of email domains to be added to SES"
+}
+
+variable "tags" {
+  description = "AWS Resource Tags"
+  type        = map(string)
+  default = {
+    "CostCenter" = "No Program / 000000"
+  }
+}

modules/sage-aws-ses/versions.tf

@@ -0,0 +1,8 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 5.0"
+    }
+  }
+}

modules/signoz/README.md

@@ -13,6 +13,14 @@ A number of items are needed:
 - Set up ingress to the cluster/collector to send data to: https://sagebionetworks.jira.com/browse/IBCDPE-1095
 - Set up accounts and access to the service decleratively
 
+
+## Setting up SMTP for alertmanager
+Alertmanager is an additional tool that is deployed to the kubernetes cluster that
+handles forwarding an alert out to 1 or more streams that will receive the alert.
+Alert manager is set to to send emails through AWS SES (Simple Email Service) set up
+by the `modules/sage-aws-ses` terraform scripts. See that module for more information
+about the setup of AWS SES.
+
 ## Accessing signoz (Internet)
 
 #### Sending data into signoz (From internet)

modules/signoz/main.tf

@@ -1,3 +1,6 @@
+locals {
+  alertmanager_enabled = var.smtp_from != "" && var.smtp_user != "" && var.smtp_password != ""
+}
 
 resource "kubernetes_namespace" "signoz" {
   metadata {
@@ -7,7 +10,7 @@ resource "kubernetes_namespace" "signoz" {
 
 resource "kubectl_manifest" "signoz-deployment" {
   depends_on = [kubernetes_namespace.signoz]
-
+  
   yaml_body = <<YAML
 apiVersion: argoproj.io/v1alpha1
 kind: Application
@@ -31,6 +34,19 @@ spec:
       parameters:
       - name: "clickhouse.password"
         value: ${random_password.clickhouse-admin-password.result}
+      %{if local.alertmanager_enabled}
+      - name: "alertmanager.enabled"
+        value: "true"
+      - name: "alertmanager.additionalEnvs.ALERTMANAGER_SMTP_FROM"
+        value: ${var.smtp_from}
+      - name: "alertmanager.additionalEnvs.ALERTMANAGER_SMTP_AUTH_USERNAME"
+        value: ${var.smtp_user}
+      - name: "alertmanager.additionalEnvs.ALERTMANAGER_SMTP_AUTH_PASSWORD"
+        value: ${var.smtp_password}
+      %{else}
+      - name: "alertmanager.enabled"
+        value: "false"
+      %{endif}
       valueFiles:
       - $values/modules/signoz/templates/values.yaml
   - repoURL: 'https://github.com/Sage-Bionetworks-Workflows/eks-stack.git'