[IBCDPE-938] Deploy Signoz (OTEL Visualization) to kubernetes cluster

.gitignore

@@ -1,4 +1,5 @@
 *.tfstate*
 .terraform
 terraform.tfvars
-settings.json
+settings.json
+temporary_files*

README.md

@@ -10,24 +10,32 @@ This repo is used to deploy an EKS cluster to AWS. CI/CD is managed through Spac
 │   └── policies: Rego policies that can be attached to 0..* spacelift stacks
 ├── dev: Development/sandbox environment
 │   ├── spacelift: Terraform scripts to manage spacelift resources
-│   │   └── dpe-sandbox: Spacelift specific resources to manage the CI/CD pipeline
+│   │   └── dpe-k8s/dpe-sandbox: Spacelift specific resources to manage the CI/CD pipeline
 │   └── stacks: The deployable cloud resources
+│       ├── dpe-auth0: Stack used to provision and setup auth0 IDP (Identity Provider) settings
 │       ├── dpe-sandbox-k8s: K8s + supporting AWS resources
 │       └── dpe-sandbox-k8s-deployments: Resources deployed inside of a K8s cluster
 └── modules: Templatized collections of terraform resources that are used in a stack
     ├── apache-airflow: K8s deployment for apache airflow
     │   └── templates: Resources used during deployment of airflow
     ├── argo-cd: K8s deployment for Argo CD, a declarative, GitOps continuous delivery tool for Kubernetes.
     │   └── templates: Resources used during deployment of this helm chart
-    ├── trivy-operator: K8s deployment for trivy, along with a few supporting charts for security scanning
-    │   └── templates: Resources used during deployment of these helm charts
-    ├── victoria-metrics: K8s deployment for victoria metrics, a promethus like tool for cluster metric collection
-    │   └── templates: Resources used during deployment of these helm charts
+    ├── cert-manager: Handles provisioning TLS certificates for the cluster
+    ├── envoy-gateway: API Gateway for the cluster securing and providing secure traffic into the cluster
+    ├── postgres-cloud-native: Used to provision a postgres instance
+    ├── postgres-cloud-native-operator: Operator that manages the lifecycle of postgres instances on the cluster
     ├── demo-network-policies: K8s deployment for a demo showcasing how to use network policies
     ├── demo-pod-level-security-groups-strict: K8s deployment for a demo showcasing how to use pod level security groups in strict mode
     ├── sage-aws-eks: Sage specific EKS cluster for AWS
+    ├── sage-aws-eks-addons: Sets up additional resources that need to be installed post creation of the EKS cluster
     ├── sage-aws-k8s-node-autoscaler: K8s node autoscaler using spotinst ocean
-    └── sage-aws-vpc: Sage specific VPC for AWS
+    ├── sage-aws-ses: AWS SES (Simple email service) setup
+    ├── sage-aws-vpc: Sage specific VPC for AWS
+    ├── signoz: SigNoz provides APM, logs, traces, metrics, exceptions, & alerts in a single tool
+    ├── trivy-operator: K8s deployment for trivy, along with a few supporting charts for security scanning
+    │   └── templates: Resources used during deployment of these helm charts
+    ├── victoria-metrics: K8s deployment for victoria metrics, a promethus like tool for cluster metric collection
+    │   └── templates: Resources used during deployment of these helm charts
 ```
 
 This root `main.tf` contains all the "Things" that are going to be deployed. 
@@ -164,7 +172,7 @@ allow us to review for any security advisories.
 
 ### Deploying an application to the kubernetes cluster
 Deployment of applications to the kubernetes cluster is handled through the combination
-of terraform (.tf) scripts, spacelift (CICD tool), and ArgoCd (Declarative definitions 
+of terraform (.tf) scripts, spacelift (CICD tool), and ArgoCd or Flux CD (Declarative definitions 
 for applications).
 
 To start of the deployment journey the first step is to create a new terraform module
@@ -283,10 +291,27 @@ This document describes the abbreviated process below:
 				"iam:*PolicyVersion",
 				"iam:*OpenIDConnectProvider",
 				"iam:*InstanceProfile",
-				"iam:ListPolicyVersions"
+				"iam:ListPolicyVersions",
+				"iam:ListGroupsForUser",
+                "iam:ListAttachedUserPolicies"
 			],
 			"Resource": "*"
-		}
+		},
+        {
+            "Effect": "Allow",
+            "Action": [
+                "iam:CreateUser",
+                "iam:AttachUserPolicy",
+                "iam:ListPolicies",
+                "iam:TagUser",
+                "iam:GetUser",
+                "iam:DeleteUser",
+                "iam:CreateAccessKey",
+                "iam:ListAccessKeys",
+                "iam:DeleteAccessKeys"
+            ],
+            "Resource": "arn:aws:iam::{{AWS ACCOUNT ID}}:user/smtp_user"
+        }
 	]
 }
 ```

deployments/main.tf

@@ -31,24 +31,55 @@ module "dpe-sandbox-spacelift-development" {
   k8s_stack_deployments_name         = "DPE DEV Kubernetes Deployments"
   k8s_stack_deployments_project_root = "deployments/stacks/dpe-k8s-deployments"
 
+  auth0_stack_name         = "DPE DEV Auth0"
+  auth0_stack_project_root = "deployments/stacks/dpe-auth0"
+  auth0_domain             = "dev-sage-dpe.us.auth0.com"
+  auth0_clients = [
+    {
+      name        = "bfauble - automation"
+      description = "App for testing signoz"
+      app_type    = "non_interactive"
+      scopes      = ["write:telemetry"]
+    },
+    {
+      name        = "schematic - Github Actions"
+      description = "Client for Github Actions to export telemetry data"
+      app_type    = "non_interactive"
+      scopes      = ["write:telemetry"]
+    },
+    {
+      name        = "schematic - Dev"
+      description = "Client for schematic deployed to AWS DEV to export telemetry data"
+      app_type    = "non_interactive"
+      scopes      = ["write:telemetry"]
+    },
+  ]
+  auth0_identifier = "https://dev.sagedpe.org"
+
   aws_account_id = "631692904429"
   region         = "us-east-1"
 
   cluster_name = "dpe-k8-sandbox"
   vpc_name     = "dpe-sandbox"
 
-  vpc_cidr_block = "10.51.0.0/16"
-  # public_subnet_cidrs  = ["10.51.1.0/24", "10.51.2.0/24", "10.51.3.0/24"]
-  # private_subnet_cidrs = ["10.51.4.0/24", "10.51.5.0/24", "10.51.6.0/24"]
-  # azs                  = ["us-east-1a", "us-east-1b", "us-east-1c"]
-  # For now, we are only using one public and one private subnet. This is due to how
-  # EBS can only be mounted to a single AZ. We will need to revisit this if we want to
-  # allow usage of EFS ($$$$), or add some kind of EBS volume replication.
-  # Note: EKS requires at least two subnets in different AZs. However, we are only using
-  # a single subnet for node deployment.
-  public_subnet_cidrs  = ["10.51.1.0/24", "10.51.2.0/24"]
-  private_subnet_cidrs = ["10.51.4.0/24", "10.51.5.0/24"]
-  azs                  = ["us-east-1a", "us-east-1b"]
+  vpc_cidr_block = "10.52.16.0/20"
+  # A public subnet is required for each AZ in which the worker nodes are deployed
+  public_subnet_cidrs                    = ["10.52.16.0/24", "10.52.17.0/24", "10.52.19.0/24"]
+  private_subnet_cidrs_eks_control_plane = ["10.52.18.0/28", "10.52.18.16/28"]
+  azs_eks_control_plane                  = ["us-east-1a", "us-east-1b"]
+
+  private_subnet_cidrs_eks_worker_nodes = ["10.52.28.0/22", "10.52.24.0/22", "10.52.20.0/22"]
+  azs_eks_worker_nodes                  = ["us-east-1c", "us-east-1b", "us-east-1a"]
+
+  enable_cluster_ingress = true
+  enable_otel_ingress    = true
+  ssl_hostname           = "dev.sagedpe.org"
+  auth0_jwks_uri         = "https://dev-sage-dpe.us.auth0.com/.well-known/jwks.json"
+  deploy_auth0           = true
+
+  ses_email_identities = ["aws-dpe-dev@sagebase.org"]
+  # Defines the email address that will be used as the sender of the email alerts
+  smtp_from = "aws-dpe-dev@sagebase.org"
 }
 
 module "dpe-sandbox-spacelift-production" {
@@ -69,22 +100,32 @@ module "dpe-sandbox-spacelift-production" {
   k8s_stack_deployments_name         = "DPE Kubernetes Deployments"
   k8s_stack_deployments_project_root = "deployments/stacks/dpe-k8s-deployments"
 
+  auth0_stack_name         = "DPE Auth0"
+  auth0_stack_project_root = "deployments/stacks/dpe-auth0"
+  auth0_domain             = ""
+  auth0_clients            = []
+  auth0_identifier         = ""
+
   aws_account_id = "766808016710"
   region         = "us-east-1"
 
   cluster_name = "dpe-k8"
   vpc_name     = "dpe-k8"
 
-  vpc_cidr_block = "10.52.0.0/16"
-  # public_subnet_cidrs  = ["10.52.1.0/24", "10.52.2.0/24", "10.52.3.0/24"]
-  # private_subnet_cidrs = ["10.52.4.0/24", "10.52.5.0/24", "10.52.6.0/24"]
-  # azs                  = ["us-east-1a", "us-east-1b", "us-east-1c"]
-  # For now, we are only using one public and one private subnet. This is due to how
-  # EBS can only be mounted to a single AZ. We will need to revisit this if we want to
-  # allow usage of EFS ($$$$), or add some kind of EBS volume replication.
-  # Note: EKS requires at least two subnets in different AZs. However, we are only using
-  # a single subnet for node deployment.
-  public_subnet_cidrs  = ["10.52.1.0/24", "10.52.2.0/24"]
-  private_subnet_cidrs = ["10.52.4.0/24", "10.52.5.0/24"]
-  azs                  = ["us-east-1a", "us-east-1b"]
+  vpc_cidr_block = "10.52.0.0/20"
+  # A public subnet is required for each AZ in which the worker nodes are deployed
+  public_subnet_cidrs                    = ["10.52.0.0/24", "10.52.1.0/24", "10.52.3.0/24"]
+  private_subnet_cidrs_eks_control_plane = ["10.52.2.0/28", "10.52.2.16/28"]
+  azs_eks_control_plane                  = ["us-east-1a", "us-east-1b"]
+
+  private_subnet_cidrs_eks_worker_nodes = ["10.52.12.0/22", "10.52.8.0/22", "10.52.4.0/22"]
+  azs_eks_worker_nodes                  = ["us-east-1c", "us-east-1b", "us-east-1a"]
+
+  enable_cluster_ingress = false
+  enable_otel_ingress    = false
+  ssl_hostname           = ""
+  auth0_jwks_uri         = ""
+  deploy_auth0           = false
+
+  ses_email_identities = []
 }

deployments/spacelift/dpe-k8s/main.tf

@@ -1,32 +1,51 @@
 locals {
   k8s_stack_environment_variables = {
-    aws_account_id                    = var.aws_account_id
-    region                            = var.region
-    pod_security_group_enforcing_mode = var.pod_security_group_enforcing_mode
-    cluster_name                      = var.cluster_name
-    vpc_name                          = var.vpc_name
-    vpc_cidr_block                    = var.vpc_cidr_block
-    public_subnet_cidrs               = var.public_subnet_cidrs
-    private_subnet_cidrs              = var.private_subnet_cidrs
-    azs                               = var.azs
+    aws_account_id                         = var.aws_account_id
+    region                                 = var.region
+    pod_security_group_enforcing_mode      = var.pod_security_group_enforcing_mode
+    cluster_name                           = var.cluster_name
+    vpc_name                               = var.vpc_name
+    vpc_cidr_block                         = var.vpc_cidr_block
+    public_subnet_cidrs                    = var.public_subnet_cidrs
+    private_subnet_cidrs_eks_control_plane = var.private_subnet_cidrs_eks_control_plane
+    private_subnet_cidrs_eks_worker_nodes  = var.private_subnet_cidrs_eks_worker_nodes
+    azs_eks_control_plane                  = var.azs_eks_control_plane
+    azs_eks_worker_nodes                   = var.azs_eks_worker_nodes
+    ses_email_identities                   = var.ses_email_identities
   }
 
   k8s_stack_deployments_variables = {
-    spotinst_account = var.spotinst_account
-    vpc_cidr_block   = var.vpc_cidr_block
+    spotinst_account       = var.spotinst_account
+    vpc_cidr_block         = var.vpc_cidr_block
+    cluster_name           = var.cluster_name
+    auto_deploy            = var.auto_deploy
+    auto_prune             = var.auto_prune
+    git_revision           = var.git_branch
+    aws_account_id         = var.aws_account_id
+    enable_cluster_ingress = var.enable_cluster_ingress
+    enable_otel_ingress    = var.enable_otel_ingress
+    ssl_hostname           = var.ssl_hostname
+    auth0_jwks_uri         = var.auth0_jwks_uri
+    smtp_from              = var.smtp_from
+    auth0_identifier       = var.auth0_identifier
+  }
+
+  auth0_stack_variables = {
     cluster_name     = var.cluster_name
-    auto_deploy      = var.auto_deploy
-    auto_prune       = var.auto_prune
-    git_revision     = var.git_branch
-    aws_account_id   = var.aws_account_id
+    auth0_domain     = var.auth0_domain
+    auth0_clients    = var.auth0_clients
+    auth0_identifier = var.auth0_identifier
   }
 
   # Variables to be passed from the k8s stack to the deployments stack
   k8s_stack_to_deployment_variables = {
-    vpc_id                 = "TF_VAR_vpc_id"
-    private_subnet_ids     = "TF_VAR_private_subnet_ids"
-    node_security_group_id = "TF_VAR_node_security_group_id"
-    pod_to_node_dns_sg_id  = "TF_VAR_pod_to_node_dns_sg_id"
+    vpc_id                              = "TF_VAR_vpc_id"
+    private_subnet_ids_eks_worker_nodes = "TF_VAR_private_subnet_ids_eks_worker_nodes"
+    node_security_group_id              = "TF_VAR_node_security_group_id"
+    pod_to_node_dns_sg_id               = "TF_VAR_pod_to_node_dns_sg_id"
+    smtp_user                           = "TF_VAR_smtp_user"
+    smtp_password                       = "TF_VAR_smtp_password"
+    cluster_oidc_provider_arn           = "TF_VAR_cluster_oidc_provider_arn"
   }
 }
 
@@ -160,31 +179,6 @@ resource "spacelift_stack_dependency_reference" "cluster-name" {
 #   stack_id  = spacelift_stack.k8s-stack.id
 # }
 
-resource "spacelift_stack_destructor" "k8s-stack-deployments-destructor" {
-  depends_on = [
-    spacelift_stack.k8s-stack,
-    spacelift_aws_integration_attachment.k8s-deployments-aws-integration-attachment,
-    spacelift_context_attachment.k8s-kubeconfig-hooks,
-    spacelift_stack_dependency_reference.cluster-name,
-    spacelift_stack_dependency_reference.region-name,
-    spacelift_environment_variable.k8s-stack-deployments-environment-variables
-  ]
-
-  stack_id = spacelift_stack.k8s-stack-deployments.id
-}
-
-resource "spacelift_stack_destructor" "k8s-stack-destructor" {
-  depends_on = [
-    spacelift_aws_integration_attachment.k8s-aws-integration-attachment,
-    spacelift_context_attachment.k8s-kubeconfig-hooks,
-    spacelift_stack_dependency_reference.cluster-name,
-    spacelift_stack_dependency_reference.region-name,
-    spacelift_environment_variable.k8s-stack-environment-variables
-  ]
-
-  stack_id = spacelift_stack.k8s-stack.id
-}
-
 resource "spacelift_aws_integration_attachment" "k8s-aws-integration-attachment" {
   integration_id = var.aws_integration_id
   stack_id       = spacelift_stack.k8s-stack.id
@@ -199,3 +193,42 @@ resource "spacelift_aws_integration_attachment" "k8s-deployments-aws-integration
   read           = true
   write          = true
 }
+
+resource "spacelift_stack" "auth0" {
+  count = var.deploy_auth0 ? 1 : 0
+  github_enterprise {
+    namespace = "Sage-Bionetworks-Workflows"
+    id        = "sage-bionetworks-workflows-gh"
+  }
+
+  depends_on = [
+    spacelift_space.dpe-space
+  ]
+
+  administrative          = false
+  autodeploy              = var.auto_deploy
+  branch                  = var.git_branch
+  description             = "Stack used to create and manage Auth0 for authentication"
+  name                    = var.auth0_stack_name
+  project_root            = var.auth0_stack_project_root
+  repository              = "eks-stack"
+  terraform_version       = var.opentofu_version
+  terraform_workflow_tool = "OPEN_TOFU"
+  space_id                = spacelift_space.dpe-space.id
+  additional_project_globs = [
+    "deployments/"
+  ]
+}
+
+resource "spacelift_environment_variable" "auth0-stack-environment-variables" {
+  depends_on = [
+    spacelift_stack.auth0
+  ]
+
+  for_each = { for k, v in local.auth0_stack_variables : k => v if var.deploy_auth0 }
+
+  stack_id   = spacelift_stack.auth0[0].id
+  name       = "TF_VAR_${each.key}"
+  value      = try(tostring(each.value), jsonencode(each.value))
+  write_only = false
+}