Signoz alert manager setup for SMTP (#41)

* Set up SMTP settings with test email & Move to lets encrypt

README.md

@@ -10,24 +10,32 @@ This repo is used to deploy an EKS cluster to AWS. CI/CD is managed through Spac
 │   └── policies: Rego policies that can be attached to 0..* spacelift stacks
 ├── dev: Development/sandbox environment
 │   ├── spacelift: Terraform scripts to manage spacelift resources
-│   │   └── dpe-sandbox: Spacelift specific resources to manage the CI/CD pipeline
+│   │   └── dpe-k8s/dpe-sandbox: Spacelift specific resources to manage the CI/CD pipeline
 │   └── stacks: The deployable cloud resources
+│       ├── dpe-auth0: Stack used to provision and setup auth0 IDP (Identity Provider) settings
 │       ├── dpe-sandbox-k8s: K8s + supporting AWS resources
 │       └── dpe-sandbox-k8s-deployments: Resources deployed inside of a K8s cluster
 └── modules: Templatized collections of terraform resources that are used in a stack
     ├── apache-airflow: K8s deployment for apache airflow
     │   └── templates: Resources used during deployment of airflow
     ├── argo-cd: K8s deployment for Argo CD, a declarative, GitOps continuous delivery tool for Kubernetes.
     │   └── templates: Resources used during deployment of this helm chart
-    ├── trivy-operator: K8s deployment for trivy, along with a few supporting charts for security scanning
-    │   └── templates: Resources used during deployment of these helm charts
-    ├── victoria-metrics: K8s deployment for victoria metrics, a promethus like tool for cluster metric collection
-    │   └── templates: Resources used during deployment of these helm charts
+    ├── cert-manager: Handles provisioning TLS certificates for the cluster
+    ├── envoy-gateway: API Gateway for the cluster securing and providing secure traffic into the cluster
+    ├── postgres-cloud-native: Used to provision a postgres instance
+    ├── postgres-cloud-native-operator: Operator that manages the lifecycle of postgres instances on the cluster
     ├── demo-network-policies: K8s deployment for a demo showcasing how to use network policies
     ├── demo-pod-level-security-groups-strict: K8s deployment for a demo showcasing how to use pod level security groups in strict mode
     ├── sage-aws-eks: Sage specific EKS cluster for AWS
+    ├── sage-aws-eks-addons: Sets up additional resources that need to be installed post creation of the EKS cluster
     ├── sage-aws-k8s-node-autoscaler: K8s node autoscaler using spotinst ocean
-    └── sage-aws-vpc: Sage specific VPC for AWS
+    ├── sage-aws-ses: AWS SES (Simple email service) setup
+    ├── sage-aws-vpc: Sage specific VPC for AWS
+    ├── signoz: SigNoz provides APM, logs, traces, metrics, exceptions, & alerts in a single tool
+    ├── trivy-operator: K8s deployment for trivy, along with a few supporting charts for security scanning
+    │   └── templates: Resources used during deployment of these helm charts
+    ├── victoria-metrics: K8s deployment for victoria metrics, a promethus like tool for cluster metric collection
+    │   └── templates: Resources used during deployment of these helm charts
 ```
 
 This root `main.tf` contains all the "Things" that are going to be deployed. 
@@ -283,10 +291,27 @@ This document describes the abbreviated process below:
 				"iam:*PolicyVersion",
 				"iam:*OpenIDConnectProvider",
 				"iam:*InstanceProfile",
-				"iam:ListPolicyVersions"
+				"iam:ListPolicyVersions",
+				"iam:ListGroupsForUser",
+                "iam:ListAttachedUserPolicies"
 			],
 			"Resource": "*"
-		}
+		},
+        {
+            "Effect": "Allow",
+            "Action": [
+                "iam:CreateUser",
+                "iam:AttachUserPolicy",
+                "iam:ListPolicies",
+                "iam:TagUser",
+                "iam:GetUser",
+                "iam:DeleteUser",
+                "iam:CreateAccessKey",
+                "iam:ListAccessKeys",
+                "iam:DeleteAccessKeys"
+            ],
+            "Resource": "arn:aws:iam::{{AWS ACCOUNT ID}}:user/smtp_user"
+        }
 	]
 }
 ```

deployments/main.tf

@@ -69,8 +69,11 @@ module "dpe-sandbox-spacelift-development" {
 
   enable_cluster_ingress = true
   enable_otel_ingress    = true
-  ssl_hostname           = "a09a38cc5a8d6497ea69c6bf6318701b-1974793757.us-east-1.elb.amazonaws.com"
+  ssl_hostname           = "dev.sagedpe.org"
   auth0_jwks_uri         = "https://dev-sage-dpe.us.auth0.com/.well-known/jwks.json"
+  ses_email_identities   = ["aws-dpe-dev@sagebase.org"]
+  # Defines the email address that will be used as the sender of the email alerts
+  smtp_from = "aws-dpe-dev@sagebase.org"
 }
 
 module "dpe-sandbox-spacelift-production" {
@@ -115,4 +118,6 @@ module "dpe-sandbox-spacelift-production" {
   enable_otel_ingress    = false
   ssl_hostname           = ""
   auth0_jwks_uri         = ""
+
+  ses_email_identities = []
 }

deployments/spacelift/dpe-k8s/main.tf

@@ -11,6 +11,7 @@ locals {
     private_subnet_cidrs_eks_worker_nodes  = var.private_subnet_cidrs_eks_worker_nodes
     azs_eks_control_plane                  = var.azs_eks_control_plane
     azs_eks_worker_nodes                   = var.azs_eks_worker_nodes
+    ses_email_identities                   = var.ses_email_identities
   }
 
   k8s_stack_deployments_variables = {
@@ -25,12 +26,13 @@ locals {
     enable_otel_ingress    = var.enable_otel_ingress
     ssl_hostname           = var.ssl_hostname
     auth0_jwks_uri         = var.auth0_jwks_uri
+    smtp_from              = var.smtp_from
   }
 
   auth0_stack_variables = {
-    cluster_name         = var.cluster_name
-    auth0_domain         = var.auth0_domain
-    auth0_clients        = var.auth0_clients
+    cluster_name  = var.cluster_name
+    auth0_domain  = var.auth0_domain
+    auth0_clients = var.auth0_clients
   }
 
   # Variables to be passed from the k8s stack to the deployments stack
@@ -39,6 +41,8 @@ locals {
     private_subnet_ids_eks_worker_nodes = "TF_VAR_private_subnet_ids_eks_worker_nodes"
     node_security_group_id              = "TF_VAR_node_security_group_id"
     pod_to_node_dns_sg_id               = "TF_VAR_pod_to_node_dns_sg_id"
+    smtp_user                           = "TF_VAR_smtp_user"
+    smtp_password                       = "TF_VAR_smtp_password"
   }
 }
 
@@ -250,4 +254,4 @@ resource "spacelift_environment_variable" "auth0-stack-environment-variables" {
   name       = "TF_VAR_${each.key}"
   value      = try(tostring(each.value), jsonencode(each.value))
   write_only = false
-}
+}

deployments/spacelift/dpe-k8s/variables.tf

@@ -180,4 +180,27 @@ variable "auth0_clients" {
     description = string
     app_type    = string
   }))
-}
+}
+
+variable "ses_email_identities" {
+  type        = list(string)
+  description = "List of email identities to be added to SES"
+}
+
+variable "smtp_user" {
+  description = "The SMTP user. Required if smtp_user, smtp_password, and smtp_from are set"
+  type        = string
+  default     = ""
+}
+
+variable "smtp_password" {
+  description = "The SMTP password. Required if smtp_user, smtp_password, and smtp_from are set"
+  type        = string
+  default     = ""
+}
+
+variable "smtp_from" {
+  description = "The SMTP from address. Required if smtp_user, smtp_password, and smtp_from are set"
+  type        = string
+  default     = ""
+}

deployments/stacks/dpe-k8s-deployments/main.tf

@@ -89,6 +89,9 @@ module "signoz" {
   gateway_namespace    = "envoy-gateway"
   cluster_name         = var.cluster_name
   auth0_jwks_uri       = var.auth0_jwks_uri
+  smtp_password        = var.smtp_password
+  smtp_user            = var.smtp_user
+  smtp_from            = var.smtp_from
 }
 
 module "envoy-gateway" {
@@ -102,11 +105,8 @@ module "envoy-gateway" {
   git_revision         = var.git_revision
   namespace            = "envoy-gateway"
   argo_deployment_name = "envoy-gateway"
-  cluster_issuer_name  = "selfsigned"
-  # To determine more elegant ways to fill in these values, for example, if we have
-  # a pre-defined DNS name for the cluster (https://sagebionetworks.jira.com/browse/IT-3931)
-  ssl_hostname   = var.ssl_hostname
-  auth0_jwks_uri = var.auth0_jwks_uri
+  cluster_issuer_name  = "lets-encrypt-prod"
+  ssl_hostname         = var.ssl_hostname
 }
 
 module "cert-manager" {

deployments/stacks/dpe-k8s-deployments/variables.tf

@@ -85,3 +85,21 @@ variable "auth0_jwks_uri" {
   description = "The JWKS URI for Auth0"
   type        = string
 }
+
+variable "smtp_user" {
+  description = "The SMTP user. Required if smtp_user, smtp_password, and smtp_from are set"
+  type        = string
+  default     = ""
+}
+
+variable "smtp_password" {
+  description = "The SMTP password. Required if smtp_user, smtp_password, and smtp_from are set"
+  type        = string
+  default     = ""
+}
+
+variable "smtp_from" {
+  description = "The SMTP from address. Required if smtp_user, smtp_password, and smtp_from are set"
+  type        = string
+  default     = ""
+}

deployments/stacks/dpe-k8s/main.tf

@@ -36,3 +36,9 @@ module "sage-aws-eks" {
   private_subnet_ids_eks_control_plane = module.sage-aws-vpc.private_subnet_ids_eks_control_plane
   private_subnet_ids_eks_worker_nodes  = module.sage-aws-vpc.private_subnet_ids_eks_worker_nodes
 }
+
+module "sage-aws-ses" {
+  source = "../../../modules/sage-aws-ses"
+
+  email_identities = var.ses_email_identities
+}

deployments/stacks/dpe-k8s/outputs.tf

@@ -37,3 +37,12 @@ output "region" {
 output "cluster_name" {
   value = module.sage-aws-eks.cluster_name
 }
+
+output "smtp_user" {
+  value = module.sage-aws-ses.smtp_user
+}
+
+output "smtp_password" {
+  sensitive = true
+  value     = module.sage-aws-ses.smtp_password
+}

deployments/stacks/dpe-k8s/variables.tf

@@ -54,3 +54,8 @@ variable "azs_eks_worker_nodes" {
   type        = list(string)
   description = "Availability Zones for the EKS worker nodes"
 }
+
+variable "ses_email_identities" {
+  type        = list(string)
+  description = "List of email identities to be added to SES"
+}

modules/envoy-gateway/main.tf

@@ -63,15 +63,6 @@ spec:
           - op: replace
             path: /metadata/name
             value: ${var.cluster_issuer_name}
-      - target:
-          kind: SecurityPolicy
-        patch: |-
-          - op: replace
-            path: /spec/jwt/providers
-            value:
-              - name: auth0
-                remoteJWKS:
-                  uri: ${var.auth0_jwks_uri}
   destination:
     server: 'https://kubernetes.default.svc'
     namespace: ${var.namespace}

modules/envoy-gateway/resources/cert-issuer.yaml

@@ -1,25 +1,18 @@
 # To implement using something like letsencrypt
-# apiVersion: cert-manager.io/v1
-# kind: ClusterIssuer
-# metadata:
-#   name: letsencrypt-staging
-# spec:
-#   acme:
-#     server: https://acme-staging-v02.api.letsencrypt.org/directory
-#     email: "bryan.fauble@sagebase.org"
-#     privateKeySecretRef:
-#       name: letsencrypt-staging-account-key
-#     solvers:
-#     - http01:
-#         gatewayHTTPRoute:
-#           parentRefs:
-#           - kind: Gateway
-#             name: eg
-#             namespace: envoy-gateway
----
 apiVersion: cert-manager.io/v1
 kind: ClusterIssuer
 metadata:
-  name: selfsigned
+  name: lets-encrypt-prod
 spec:
-  selfSigned: {}
+  acme:
+    server: https://acme-v02.api.letsencrypt.org/directory
+    email: "dpe@sagebase.org"
+    privateKeySecretRef:
+      name: letsencrypt-prod-account-key
+    solvers:
+    - http01:
+        gatewayHTTPRoute:
+          parentRefs:
+          - kind: Gateway
+            name: eg
+            namespace: envoy-gateway

modules/envoy-gateway/resources/http-to-https-redirect.yaml

@@ -0,0 +1,16 @@
+# Upgrades HTTP requests to HTTPS
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: tls-redirect
+spec:
+  parentRefs:
+    - name: eg
+      sectionName: http
+  hostnames:
+    - "*.sagedpe.org"
+  rules:
+    - filters:
+        - type: RequestRedirect
+          requestRedirect:
+            scheme: https

modules/envoy-gateway/resources/kustomization.yaml

@@ -6,4 +6,4 @@ resources:
 - envoy-proxy.yaml
 - gateway.yaml
 - traffic-policy.yaml
-- security-policy.yaml
+- http-to-https-redirect.yaml

modules/envoy-gateway/variables.tf

@@ -35,8 +35,3 @@ variable "ssl_hostname" {
   description = "The hostname to use for the SSL certificate"
   type        = string
 }
-
-variable "auth0_jwks_uri" {
-  description = "The JWKS URI for Auth0"
-  type        = string
-}

modules/sage-aws-ses/README.md

@@ -0,0 +1,35 @@
+# Purpose
+This module is used to set up SES (Simple email service) in AWS.
+
+By setting a few variables we are able to create a number of Email addresses
+to AWS SES. The variables to be set are:
+
+- `email_identities`, example: `["example@sagebase.org"]`
+
+# Manual steps required
+After running this module a number of manual steps are required as they are external
+processes that need to happen:
+
+## Verify Email address
+1) Navigate to Amazon SES in the web console
+2) Navigate to `identities`
+3) Choose the Identity to verify
+4) Send a test email and click the link received to verify the email
+
+Optional: Send a test email after verifying to confirm you may receive emails
+
+# Request production access
+After creating AWS SES settings the first time you will be in "Sandbox" mode. In order
+to request production access follow the following document: <https://docs.aws.amazon.com/ses/latest/dg/request-production-access.html>
+under the section "To request that your account be removed from the Amazon SES sandbox using the AWS CLI".
+
+The command will look something like:
+
+```
+aws sesv2 put-account-details \
+--production-access-enabled \
+--mail-type TRANSACTIONAL \
+--website-url https://www.synapse.org/ \
+--additional-contact-email-addresses dpe@sagebase.org \
+--contact-language EN
+```

modules/sage-aws-ses/data.tf

@@ -0,0 +1,6 @@
+data "aws_iam_policy_document" "ses_sender" {
+  statement {
+    actions   = ["ses:SendRawEmail"]
+    resources = ["*"]
+  }
+}