[IBCDPE-1095] Secure cluster ingress for telemetry data

deployments/main.tf

@@ -31,16 +31,46 @@ module "dpe-sandbox-spacelift-development" {
   k8s_stack_deployments_name         = "DPE DEV Kubernetes Deployments"
   k8s_stack_deployments_project_root = "deployments/stacks/dpe-k8s-deployments"
 
+  auth0_stack_name         = "DPE DEV Auth0"
+  auth0_stack_project_root = "deployments/stacks/dpe-auth0"
+  auth0_domain             = "dev-57n3awu5je6q653y.us.auth0.com"
+  auth0_clients = [
+    {
+      name        = "bfauble - automation"
+      description = "App for testing signoz"
+      app_type    = "non_interactive"
+    },
+    {
+      name        = "schematic - Github Actions"
+      description = "Client for Github Actions to export telemetry data"
+      app_type    = "non_interactive"
+    },
+    {
+      name        = "schematic - Dev"
+      description = "Client for schematic deployed to AWS DEV to export telemetry data"
+      app_type    = "non_interactive"
+    },
+  ]
+
   aws_account_id = "631692904429"
   region         = "us-east-1"
 
   cluster_name = "dpe-k8-sandbox"
   vpc_name     = "dpe-sandbox"
 
-  vpc_cidr_block       = "10.51.0.0/16"
-  public_subnet_cidrs  = ["10.51.1.0/24", "10.51.2.0/24"]
-  private_subnet_cidrs = ["10.51.4.0/24", "10.51.5.0/24"]
-  azs                  = ["us-east-1a", "us-east-1b"]
+  vpc_cidr_block = "10.52.16.0/20"
+  # A public subnet is required for each AZ in which the worker nodes are deployed
+  public_subnet_cidrs                    = ["10.52.16.0/24", "10.52.17.0/24", "10.52.19.0/24"]
+  private_subnet_cidrs_eks_control_plane = ["10.52.18.0/28", "10.52.18.16/28"]
+  azs_eks_control_plane                  = ["us-east-1a", "us-east-1b"]
+
+  private_subnet_cidrs_eks_worker_nodes = ["10.52.28.0/22", "10.52.24.0/22", "10.52.20.0/22"]
+  azs_eks_worker_nodes                  = ["us-east-1c", "us-east-1b", "us-east-1a"]
+
+  enable_cluster_ingress = true
+  enable_otel_ingress    = true
+  ssl_hostname           = "a09a38cc5a8d6497ea69c6bf6318701b-1974793757.us-east-1.elb.amazonaws.com"
+  auth0_jwks_uri         = "https://dev-57n3awu5je6q653y.us.auth0.com/.well-known/jwks.json"
 }
 
 module "dpe-sandbox-spacelift-production" {
@@ -61,14 +91,28 @@ module "dpe-sandbox-spacelift-production" {
   k8s_stack_deployments_name         = "DPE Kubernetes Deployments"
   k8s_stack_deployments_project_root = "deployments/stacks/dpe-k8s-deployments"
 
+  auth0_stack_name         = "DPE Auth0"
+  auth0_stack_project_root = "deployments/stacks/dpe-auth0"
+  auth0_domain             = ""
+  auth0_clients            = []
+
   aws_account_id = "766808016710"
   region         = "us-east-1"
 
   cluster_name = "dpe-k8"
   vpc_name     = "dpe-k8"
 
-  vpc_cidr_block       = "10.52.0.0/16"
-  public_subnet_cidrs  = ["10.52.1.0/24", "10.52.2.0/24"]
-  private_subnet_cidrs = ["10.52.4.0/24", "10.52.5.0/24"]
-  azs                  = ["us-east-1a", "us-east-1b"]
+  vpc_cidr_block = "10.52.0.0/20"
+  # A public subnet is required for each AZ in which the worker nodes are deployed
+  public_subnet_cidrs                    = ["10.52.0.0/24", "10.52.1.0/24", "10.52.3.0/24"]
+  private_subnet_cidrs_eks_control_plane = ["10.52.2.0/28", "10.52.2.16/28"]
+  azs_eks_control_plane                  = ["us-east-1a", "us-east-1b"]
+
+  private_subnet_cidrs_eks_worker_nodes = ["10.52.12.0/22", "10.52.8.0/22", "10.52.4.0/22"]
+  azs_eks_worker_nodes                  = ["us-east-1c", "us-east-1b", "us-east-1a"]
+
+  enable_cluster_ingress = false
+  enable_otel_ingress    = false
+  ssl_hostname           = ""
+  auth0_jwks_uri         = ""
 }

deployments/spacelift/dpe-k8s/main.tf

@@ -1,32 +1,44 @@
 locals {
   k8s_stack_environment_variables = {
-    aws_account_id                    = var.aws_account_id
-    region                            = var.region
-    pod_security_group_enforcing_mode = var.pod_security_group_enforcing_mode
-    cluster_name                      = var.cluster_name
-    vpc_name                          = var.vpc_name
-    vpc_cidr_block                    = var.vpc_cidr_block
-    public_subnet_cidrs               = var.public_subnet_cidrs
-    private_subnet_cidrs              = var.private_subnet_cidrs
-    azs                               = var.azs
+    aws_account_id                         = var.aws_account_id
+    region                                 = var.region
+    pod_security_group_enforcing_mode      = var.pod_security_group_enforcing_mode
+    cluster_name                           = var.cluster_name
+    vpc_name                               = var.vpc_name
+    vpc_cidr_block                         = var.vpc_cidr_block
+    public_subnet_cidrs                    = var.public_subnet_cidrs
+    private_subnet_cidrs_eks_control_plane = var.private_subnet_cidrs_eks_control_plane
+    private_subnet_cidrs_eks_worker_nodes  = var.private_subnet_cidrs_eks_worker_nodes
+    azs_eks_control_plane                  = var.azs_eks_control_plane
+    azs_eks_worker_nodes                   = var.azs_eks_worker_nodes
   }
 
   k8s_stack_deployments_variables = {
-    spotinst_account = var.spotinst_account
-    vpc_cidr_block   = var.vpc_cidr_block
-    cluster_name     = var.cluster_name
-    auto_deploy      = var.auto_deploy
-    auto_prune       = var.auto_prune
-    git_revision     = var.git_branch
-    aws_account_id   = var.aws_account_id
+    spotinst_account       = var.spotinst_account
+    vpc_cidr_block         = var.vpc_cidr_block
+    cluster_name           = var.cluster_name
+    auto_deploy            = var.auto_deploy
+    auto_prune             = var.auto_prune
+    git_revision           = var.git_branch
+    aws_account_id         = var.aws_account_id
+    enable_cluster_ingress = var.enable_cluster_ingress
+    enable_otel_ingress    = var.enable_otel_ingress
+    ssl_hostname           = var.ssl_hostname
+    auth0_jwks_uri         = var.auth0_jwks_uri
+  }
+
+  auth0_stack_variables = {
+    cluster_name         = var.cluster_name
+    auth0_domain         = var.auth0_domain
+    auth0_clients        = var.auth0_clients
   }
 
   # Variables to be passed from the k8s stack to the deployments stack
   k8s_stack_to_deployment_variables = {
-    vpc_id                 = "TF_VAR_vpc_id"
-    private_subnet_ids     = "TF_VAR_private_subnet_ids"
-    node_security_group_id = "TF_VAR_node_security_group_id"
-    pod_to_node_dns_sg_id  = "TF_VAR_pod_to_node_dns_sg_id"
+    vpc_id                              = "TF_VAR_vpc_id"
+    private_subnet_ids_eks_worker_nodes = "TF_VAR_private_subnet_ids_eks_worker_nodes"
+    node_security_group_id              = "TF_VAR_node_security_group_id"
+    pod_to_node_dns_sg_id               = "TF_VAR_pod_to_node_dns_sg_id"
   }
 }
 
@@ -199,3 +211,43 @@ resource "spacelift_aws_integration_attachment" "k8s-deployments-aws-integration
   read           = true
   write          = true
 }
+
+
+resource "spacelift_stack" "auth0" {
+  github_enterprise {
+    namespace = "Sage-Bionetworks-Workflows"
+    id        = "sage-bionetworks-workflows-gh"
+  }
+
+  depends_on = [
+    spacelift_space.dpe-space
+  ]
+
+  administrative          = false
+  autodeploy              = var.auto_deploy
+  branch                  = var.git_branch
+  description             = "Stack used to create and manage Auth0 for authentication"
+  name                    = var.auth0_stack_name
+  project_root            = var.auth0_stack_project_root
+  repository              = "eks-stack"
+  terraform_version       = var.opentofu_version
+  terraform_workflow_tool = "OPEN_TOFU"
+  space_id                = spacelift_space.dpe-space.id
+  additional_project_globs = [
+    "deployments/"
+  ]
+}
+
+resource "spacelift_stack_destructor" "auth0-stack-destructor" {
+  stack_id = spacelift_stack.auth0.id
+}
+
+
+resource "spacelift_environment_variable" "auth0-stack-environment-variables" {
+  for_each = local.auth0_stack_variables
+
+  stack_id   = spacelift_stack.auth0.id
+  name       = "TF_VAR_${each.key}"
+  value      = try(tostring(each.value), jsonencode(each.value))
+  write_only = false
+}

deployments/spacelift/dpe-k8s/variables.tf

@@ -118,12 +118,66 @@ variable "public_subnet_cidrs" {
   description = "Public Subnet CIDR values"
 }
 
-variable "private_subnet_cidrs" {
+variable "private_subnet_cidrs_eks_control_plane" {
   type        = list(string)
-  description = "Private Subnet CIDR values"
+  description = "Private Subnet CIDR values for the EKS control plane"
 }
 
-variable "azs" {
+variable "private_subnet_cidrs_eks_worker_nodes" {
   type        = list(string)
-  description = "Availability Zones"
+  description = "Private Subnet CIDR values for the EKS worker nodes"
 }
+
+variable "azs_eks_control_plane" {
+  type        = list(string)
+  description = "Availability Zones for the EKS control plane"
+}
+
+variable "azs_eks_worker_nodes" {
+  type        = list(string)
+  description = "Availability Zones for the EKS worker nodes"
+}
+
+variable "enable_cluster_ingress" {
+  description = "Enable cluster ingress"
+  type        = bool
+}
+
+variable "enable_otel_ingress" {
+  description = "Enable OpenTelemetry ingress, used to send traces to SigNoz"
+  type        = bool
+}
+
+variable "ssl_hostname" {
+  description = "The hostname to use for the SSL certificate"
+  type        = string
+}
+
+variable "auth0_jwks_uri" {
+  description = "The JWKS URI for Auth0"
+  type        = string
+}
+
+variable "auth0_stack_name" {
+  description = "Name of the auth0 stack"
+  type        = string
+}
+
+variable "auth0_stack_project_root" {
+  description = "Project root of the auth0 stack"
+  type        = string
+}
+
+variable "auth0_domain" {
+  description = "Auth0 domain"
+  type        = string
+}
+
+variable "auth0_clients" {
+  description = "List of clients to create in Auth0."
+  type = list(object({
+    name        = string
+    description = string
+    app_type    = string
+  }))
+}

deployments/stacks/dpe-auth0/main.tf

@@ -0,0 +1,43 @@
+# Used to create the Auth0 resources for the DPE stack
+resource "auth0_resource_server" "k8s-cluster-telemetry" {
+  name        = "${var.cluster_name}-telemetry"
+  identifier  = "${var.cluster_name}-telemetry"
+  signing_alg = "RS256"
+
+  allow_offline_access                            = false
+  token_lifetime                                  = 86400
+  skip_consent_for_verifiable_first_party_clients = true
+  # https://registry.terraform.io/providers/auth0/auth0/latest/docs/resources/resource_server_scopes
+  # Says to use the following, however it errors out:
+  # This object has no argument, nested block, or exported attribute named "scopes".
+  # lifecycle {
+  #   ignore_changes = [scopes]
+  # }
+}
+
+resource "auth0_client" "oauth2_clients" {
+  for_each = { for client in var.auth0_clients : client.name => client }
+
+  name        = each.value.name
+  description = each.value.description
+  app_type    = each.value.app_type
+
+  jwt_configuration {
+    alg = "RS256"
+  }
+}
+
+resource "auth0_client_credentials" "client_secrets" {
+  for_each = { for client in auth0_client.oauth2_clients : client.name => client }
+
+  client_id             = auth0_client.oauth2_clients[each.key].id
+  authentication_method = "client_secret_post"
+}
+
+resource "auth0_client_grant" "access_to_k8s_cluster" {
+  for_each = { for client in var.auth0_clients : client.name => client }
+
+  client_id = auth0_client.oauth2_clients[each.key].id
+  audience  = auth0_resource_server.k8s-cluster-telemetry.identifier
+  scopes    = []
+}

deployments/stacks/dpe-auth0/provider.tf

@@ -0,0 +1,10 @@
+# Requires manually setting id and secret in the stack environment variables in the Spacelift UI
+# These come from auth0 > Applications > Applications > API Explorer Application > Settings
+# TF_VAR_auth0_client_id
+# TF_VAR_auth0_client_secret
+# TF_VAR_auth0_domain
+provider "auth0" {
+  domain        = var.auth0_domain
+  client_id     = var.auth0_client_id
+  client_secret = var.auth0_client_secret
+}

deployments/stacks/dpe-auth0/variables.tf

@@ -0,0 +1,28 @@
+variable "cluster_name" {
+  description = "EKS cluster name"
+  type        = string
+}
+
+variable "auth0_domain" {
+  description = "Auth0 domain"
+  type        = string
+}
+
+variable "auth0_client_id" {
+  description = "Auth0 client ID"
+  type        = string
+}
+
+variable "auth0_client_secret" {
+  description = "Auth0 client secret"
+  type        = string
+}
+
+variable "auth0_clients" {
+  description = "List of clients to create in Auth0."
+  type = list(object({
+    name        = string
+    description = string
+    app_type    = string
+  }))
+}

deployments/stacks/dpe-auth0/versions.tf

@@ -0,0 +1,8 @@
+terraform {
+  required_providers {
+    auth0 = {
+      source = "auth0/auth0"
+      version = "1.7.1"
+    }
+  }
+}