[IBCDPE] Split out and start creating items as individual modules

modules/apache-airflow/data.tf

@@ -0,0 +1,7 @@
+data "aws_eks_cluster" "cluster" {
+  name = var.cluster_name
+}
+
+data "aws_eks_cluster_auth" "cluster" {
+  name = var.cluster_name
+}

modules/apache-airflow/main.tf

@@ -0,0 +1,101 @@
+resource "kubernetes_namespace" "airflow" {
+  metadata {
+    name = "airflow"
+  }
+}
+
+resource "random_password" "airflow" {
+  length           = 16
+  special          = true
+  override_special = "!#$%&*()-_=+[]{}<>:?"
+}
+
+resource "kubernetes_secret" "airflow_webserver_secret" {
+  metadata {
+    name      = "airflow-webserver-secret"
+    namespace = "airflow"
+  }
+
+  data = {
+    "webserver-secret-key" = random_password.airflow.result
+  }
+
+  depends_on = [kubernetes_namespace.airflow]
+}
+
+# TODO: Should a long-term deployment use a managed RDS instance?
+# https://github.com/apache/airflow/blob/main/chart/values.yaml#L2321-L2329
+resource "helm_release" "airflow" {
+  name       = "apache-airflow"
+  repository = "https://airflow.apache.org"
+  chart      = "airflow"
+  namespace  = "airflow"
+  version    = "1.11.0"
+  depends_on = [kubernetes_namespace.airflow]
+
+  # https://github.com/hashicorp/terraform-provider-helm/issues/683#issuecomment-830872443
+  wait = false
+
+  set {
+    name  = "config.webserver.expose_config"
+    value = "true"
+  }
+
+  set {
+    name  = "config.secrets.backend"
+    value = "airflow.providers.amazon.aws.secrets.secrets_manager.SecretsManagerBackend"
+  }
+
+  set {
+    name  = "webserver.service.type"
+    value = "LoadBalancer"
+  }
+
+  set {
+    name  = "webserverSecretKeySecretName"
+    value = "airflow-webserver-secret"
+  }
+
+  set {
+    name  = "airflowVersion"
+    value = "2.7.1"
+  }
+
+  set {
+    name  = "defaultAirflowRepository"
+    value = "bfaublesage/airflow"
+  }
+
+  set {
+    name  = "defaultAirflowTag"
+    value = "2.7.1-python-3.10"
+  }
+
+  set {
+    name  = "dags.persistence.enabled"
+    value = "false"
+  }
+
+  set {
+    name  = "dags.gitSync.enabled"
+    value = "true"
+  }
+
+  set {
+    name  = "dags.gitSync.repo"
+    value = "https://github.com/Sage-Bionetworks-Workflows/orca-recipes"
+  }
+
+  set {
+    name  = "dags.gitSync.subPath"
+    value = "dags"
+  }
+
+  set {
+    name  = "dags.gitSync.branch"
+    value = "main"
+  }
+
+
+  values = [templatefile("${path.module}/templates/airflow-values.yaml", {})]
+}

modules/apache-airflow/provider.tf

@@ -0,0 +1,16 @@
+provider "aws" {
+  region = var.region
+}
+
+provider "kubernetes" {
+  config_path            = var.kube_config_path
+  host                   = data.aws_eks_cluster.cluster.endpoint
+  cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data)
+  token                  = data.aws_eks_cluster_auth.cluster.token
+}
+
+provider "helm" {
+  kubernetes {
+    config_path = var.kube_config_path
+  }
+}

modules/apache-airflow/templates/airflow-values.yaml

@@ -0,0 +1,20 @@
+config:
+    secrets:
+        backend_kwargs: '{"connections_prefix": "airflow/connections", "variables_prefix": "airflow/variables", "region_name": "us-east-1"}'
+    # webserver:
+    #     authenticate: true
+    #     auth_backend: airflow.contrib.auth.backends.google_auth
+    #     web_server_ssl_cert = <path to cert>
+    #     web_server_ssl_key = <path to key>
+    #     web_server_port = 443
+    #     base_url = http://<hostname or IP>:443
+    # celery:
+    #     ssl_active = True
+    #     ssl_key = <path to key>
+    #     ssl_cert = <path to cert>
+    #     ssl_cacert = <path to cacert>
+
+# service:
+#   type: LoadBalancer # or another type as needed
+#   annotations:
+#     alb.ingress.kubernetes.io/scheme: "internal"

modules/apache-airflow/variables.tf

@@ -0,0 +1,18 @@
+variable "cluster_name" {
+  description = "Name of K8 cluster"
+  type        = string
+  default     = "dpe-k8"
+}
+
+variable "kube_config_path" {
+  description = "Kube config path"
+  type        = string
+  default     = "~/.kube/config"
+}
+
+variable "region" {
+  description = "AWS region"
+  type        = string
+  default     = "us-east-1"
+}
+

modules/apache-airflow/versions.tf

@@ -0,0 +1,3 @@
+terraform {
+  required_version = "<= 1.5.7"
+}

modules/internal-k8-infra/data.tf

@@ -1,9 +1,9 @@
 data "aws_eks_cluster" "cluster" {
-  name = "dpe-k8"
+  name = var.cluster_name
 }
 
 data "aws_eks_cluster_auth" "cluster" {
-  name = "dpe-k8"
+  name = var.cluster_name
 }
 
 data "aws_secretsmanager_secret" "spotinst_token" {

modules/internal-k8-infra/main.tf

@@ -1,16 +1,15 @@
-module "kubernetes-controller" {
-  source  = "spotinst/kubernetes-controller/ocean"
-  version = "0.0.2"
+module "ocean-controller" {
+  source = "spotinst/ocean-controller/spotinst"
+  version = "0.54.0"
 
-  # Credentials
+  # Credentials.
   spotinst_token   = data.aws_secretsmanager_secret_version.secret_credentials.secret_string
   spotinst_account = var.spotinst_account
 
-  # Configuration
+  # Configuration.
   cluster_identifier = var.cluster_name
 }
 
-
 module "ocean-aws-k8s" {
   source  = "spotinst/ocean-aws-k8s/spotinst"
   version = "1.2.0"

modules/k8s-node-autoscaler/README.md

@@ -0,0 +1 @@
+# The use of this module is experimental and is a WIP

modules/k8s-node-autoscaler/data.tf

@@ -0,0 +1,39 @@
+data "aws_eks_cluster" "cluster" {
+  name = var.cluster_name
+}
+
+data "aws_eks_cluster_auth" "cluster" {
+  name = var.cluster_name
+}
+
+data "aws_secretsmanager_secret" "spotinst_token" {
+  name = "spotinst_token"
+}
+
+data "aws_secretsmanager_secret_version" "secret_credentials" {
+  secret_id = data.aws_secretsmanager_secret.spotinst_token.id
+}
+
+# TODO: This should search for the VPC using some other value as ID would change
+# on first startup and teardown/restart
+data "aws_subnets" "node_subnets" {
+  filter {
+    name   = "vpc-id"
+    values = ["vpc-0f30cfca319ebc521"]
+  }
+}
+
+data "aws_eks_node_groups" "node_groups" {
+  cluster_name = var.cluster_name
+}
+
+data "aws_eks_node_group" "node_group" {
+  cluster_name    = var.cluster_name
+  node_group_name = data.aws_eks_node_groups.node_groups[0].id
+}
+
+data "aws_security_group" "eks_cluster_security_group" {
+  tags = {
+    Name = "${var.cluster_name}-node"
+  }
+}

modules/k8s-node-autoscaler/main.tf

@@ -0,0 +1,27 @@
+module "ocean-controller" {
+  source = "spotinst/ocean-controller/spotinst"
+  version = "0.54.0"
+
+  # Credentials.
+  spotinst_token   = data.aws_secretsmanager_secret_version.secret_credentials.secret_string
+  spotinst_account = var.spotinst_account
+
+  # Configuration.
+  cluster_identifier = var.cluster_name
+}
+
+module "ocean-aws-k8s" {
+  source  = "spotinst/ocean-aws-k8s/spotinst"
+  version = "1.2.0"
+  # worker_instance_profile_arn      = "arn:aws:iam::766808016710:role/airflow-node-group-eks-node-group-20240517054613935800000001"
+
+  # Configuration
+  cluster_name                     = var.cluster_name
+  region                           = var.region
+  subnet_ids                       = data.aws_subnets.node_subnets.ids
+  worker_instance_profile_arn      = tolist(data.aws_eks_node_group.node_group.node_role_arn)[0]
+  security_groups                  = [data.aws_security_group.eks_cluster_security_group.id]
+  is_aggressive_scale_down_enabled = true
+  max_scale_down_percentage        = 33
+  tags                             = var.tags
+}

modules/k8s-node-autoscaler/provider.tf

@@ -0,0 +1,21 @@
+provider "aws" {
+  region = var.region
+}
+
+provider "spotinst" {
+  account = var.spotinst_account
+  token   = data.aws_secretsmanager_secret_version.secret_credentials.secret_string
+}
+
+provider "kubernetes" {
+  config_path            = var.kube_config_path
+  host                   = data.aws_eks_cluster.cluster.endpoint
+  cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data)
+  token                  = data.aws_eks_cluster_auth.cluster.token
+}
+
+provider "helm" {
+  kubernetes {
+    config_path = var.kube_config_path
+  }
+}

modules/k8s-node-autoscaler/variables.tf

@@ -0,0 +1,31 @@
+variable "cluster_name" {
+  description = "Name of K8 cluster"
+  type        = string
+  default     = "dpe-k8"
+}
+
+variable "region" {
+  description = "AWS region"
+  type        = string
+  default     = "us-east-1"
+}
+
+variable "kube_config_path" {
+  description = "Kube config path"
+  type        = string
+  default     = "~/.kube/config"
+}
+
+variable "spotinst_account" {
+  description = "Spot.io account"
+  type        = string
+  default     = "act-ac6522b4"
+}
+
+variable "tags" {
+  description = "AWS Resource Tags"
+  type        = map(string)
+  default = {
+    "CostCenter" = "No Program / 000000"
+  }
+}

modules/k8s-node-autoscaler/versions.tf

@@ -0,0 +1,9 @@
+terraform {
+  required_version = "<= 1.5.7"
+  required_providers {
+    spotinst = {
+      source  = "spotinst/spotinst"
+      version = "1.172.0" # Specify the version you wish to use
+    }
+  }
+}

spacelift/main.tf

@@ -0,0 +1,31 @@
+resource "spacelift_stack" "root_administrative_stack" {
+  github_enterprise {
+    namespace = "Sage-Bionetworks-Workflows"
+    id        = "sage-bionetworks-workflows-gh"
+  }
+
+  administrative    = true
+  autodeploy        = false
+  branch            = "main"
+  description       = "Manages other spacelift resources"
+  name              = "Root Spacelift Administrative Stack"
+  project_root      = "spacelift"
+  repository        = "eks-stack"
+  terraform_version = "1.5.7"
+  space_id          = "root"
+}
+
+module "policies" {
+  source = "./modules/policies"
+}
+
+module "policy-attachments" {
+  source = "./modules/policy-attachments"
+  depends_on = [
+    module.policies
+  ]
+}
+
+module "stacks" {
+  source = "./modules/stacks"
+}

spacelift/modules/policies/check-estimated-cloud-spend.rego

@@ -0,0 +1,20 @@
+package spacelift
+
+# Warn if changes that will cause the monthly cost to go above a certain threshold
+warn[sprintf("monthly cost greater than $%d ($%.2f)", [threshold, monthly_cost])] {
+  threshold := 100
+  monthly_cost := to_number(input.third_party_metadata.infracost.projects[0].breakdown.totalMonthlyCost)
+  monthly_cost > threshold
+}
+
+# Warn if the monthly costs increase more than a certain percentage
+warn[sprintf("monthly cost increase greater than %d%% (%.2f%%)", [threshold, percentage_increase])] {
+  threshold := 5
+  previous_cost := to_number(input.third_party_metadata.infracost.projects[0].pastBreakdown.totalMonthlyCost)
+  previous_cost > 0
+
+  monthly_cost := to_number(input.third_party_metadata.infracost.projects[0].breakdown.totalMonthlyCost)
+  percentage_increase := ((monthly_cost - previous_cost) / previous_cost) * 100
+
+  percentage_increase > threshold
+}