DRAFT: Split eks creation to own module (#9)

* Push EKS creation to it's own module

modules/k8s-node-autoscaler/data.tf

@@ -14,26 +14,33 @@ data "aws_secretsmanager_secret_version" "secret_credentials" {
   secret_id = data.aws_secretsmanager_secret.spotinst_token.id
 }
 
-# TODO: This should search for the VPC using some other value as ID would change
-# on first startup and teardown/restart
-data "aws_subnets" "node_subnets" {
+data "aws_vpc" "selected" {
   filter {
-    name   = "vpc-id"
-    values = ["vpc-0f30cfca319ebc521"]
+    name   = "tag:Name"
+    values = ["spacelift-created-vpc"]
   }
 }
 
-data "aws_eks_node_groups" "node_groups" {
-  cluster_name = var.cluster_name
-}
+data "aws_subnets" "private" {
+  filter {
+    name   = "vpc-id"
+    values = [data.aws_vpc.selected.id]
+  }
 
-data "aws_eks_node_group" "node_group" {
-  cluster_name    = var.cluster_name
-  node_group_name = data.aws_eks_node_groups.node_groups[0].id
+  filter {
+    name   = "tag:Name"
+    values = ["private"]
+  }
 }
 
-data "aws_security_group" "eks_cluster_security_group" {
+data "aws_security_group" "eks_node_security_group" {
   tags = {
     Name = "${var.cluster_name}-node"
   }
 }
+
+data "kubernetes_storage_class" "existing" {
+  metadata {
+    name = "gp2"
+  }
+}

modules/k8s-node-autoscaler/main.tf

@@ -1,3 +1,62 @@
+resource "aws_iam_role" "work_profile_iam_role" {
+  name = "work_profile_iam_role_${var.cluster_name}"
+
+  assume_role_policy = jsonencode({
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Sid": "EKSNodeAssumeRole",
+            "Effect": "Allow",
+            "Principal": {
+                "Service": "ec2.amazonaws.com"
+            },
+            "Action": "sts:AssumeRole"
+        }
+    ]
+  })
+
+  tags = var.tags
+}
+
+
+resource "aws_iam_role_policy_attachment" "a1" {
+  role       = aws_iam_role.work_profile_iam_role.name
+  policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy"
+}
+
+resource "aws_iam_role_policy_attachment" "a2" {
+  role       = aws_iam_role.work_profile_iam_role.name
+  policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly"
+}
+
+resource "aws_iam_role_policy_attachment" "a3" {
+  role       = aws_iam_role.work_profile_iam_role.name
+  policy_arn = "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy"
+}
+
+resource "aws_iam_role_policy_attachment" "a4" {
+  role       = aws_iam_role.work_profile_iam_role.name
+  policy_arn = "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy"
+}
+
+resource "aws_iam_role_policy_attachment" "a5" {
+  role       = aws_iam_role.work_profile_iam_role.name
+  policy_arn = "arn:aws:iam::aws:policy/SecretsManagerReadWrite"
+}
+
+resource "aws_iam_instance_profile" "profile" {
+  name = "eks_profile_${var.cluster_name}"
+  role = aws_iam_role.work_profile_iam_role.name
+  tags = var.tags
+}
+
+resource "aws_eks_access_entry" "example" {
+  cluster_name      = var.cluster_name
+  principal_arn     = aws_iam_role.work_profile_iam_role.arn
+  type              = "EC2_LINUX"
+  tags = var.tags
+}
+
 module "ocean-controller" {
   source = "spotinst/ocean-controller/spotinst"
   version = "0.54.0"
@@ -18,10 +77,50 @@ module "ocean-aws-k8s" {
   # Configuration
   cluster_name                     = var.cluster_name
   region                           = var.region
-  subnet_ids                       = data.aws_subnets.node_subnets.ids
-  worker_instance_profile_arn      = tolist(data.aws_eks_node_group.node_group.node_role_arn)[0]
-  security_groups                  = [data.aws_security_group.eks_cluster_security_group.id]
+  subnet_ids                       = data.aws_subnets.private.ids
+  worker_instance_profile_arn      = aws_iam_instance_profile.profile.arn
+  security_groups                  = [data.aws_security_group.eks_node_security_group.id]
   is_aggressive_scale_down_enabled = true
   max_scale_down_percentage        = 33
   tags                             = var.tags
 }
+
+resource "aws_eks_addon" "coredns" {
+  cluster_name = var.cluster_name
+  addon_name   = "coredns"
+  tags         = var.tags
+
+  depends_on = [
+    module.ocean-controller,
+    module.ocean-aws-k8s,
+  ]
+}
+
+resource "aws_eks_addon" "ebs-csi-driver" {
+  cluster_name = var.cluster_name
+  addon_name   = "aws-ebs-csi-driver"
+  tags         = var.tags
+
+  depends_on = [
+    module.ocean-controller,
+    module.ocean-aws-k8s,
+  ]
+}
+
+resource "kubernetes_storage_class" "default" {
+  depends_on = [aws_eks_addon.ebs-csi-driver]
+
+  metadata {
+    name = "gp2-default"
+    annotations = {
+      "storageclass.kubernetes.io/is-default-class" = "true"
+    }
+  }
+
+  storage_provisioner     = data.kubernetes_storage_class.existing.storage_provisioner
+  reclaim_policy          = data.kubernetes_storage_class.existing.reclaim_policy
+  parameters              = data.kubernetes_storage_class.existing.parameters
+  volume_binding_mode     = data.kubernetes_storage_class.existing.volume_binding_mode
+  allow_volume_expansion  = data.kubernetes_storage_class.existing.allow_volume_expansion
+  mount_options           = data.kubernetes_storage_class.existing.mount_options
+}

modules/k8s-node-autoscaler/provider.tf

@@ -13,9 +13,3 @@ provider "kubernetes" {
   cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data)
   token                  = data.aws_eks_cluster_auth.cluster.token
 }
-
-provider "helm" {
-  kubernetes {
-    config_path = var.kube_config_path
-  }
-}

modules/k8s-node-autoscaler/variables.tf

@@ -1,7 +1,7 @@
 variable "cluster_name" {
   description = "Name of K8 cluster"
   type        = string
-  default     = "dpe-k8"
+  default     = "dpe-k8-dev"
 }
 
 variable "region" {

modules/sage-aws-eks/data.tf

@@ -0,0 +1,38 @@
+data "aws_eks_cluster" "cluster" {
+  depends_on = [module.eks.cluster_id]
+  name       = module.eks.cluster_name
+}
+
+data "aws_eks_cluster_auth" "cluster" {
+  depends_on = [module.eks.cluster_id]
+  name       = module.eks.cluster_name
+}
+
+data "aws_vpc" "selected" {
+  filter {
+    name   = "tag:Name"
+    values = ["spacelift-created-vpc"]
+  }
+}
+
+data "aws_subnets" "private" {
+  filter {
+    name   = "vpc-id"
+    values = [data.aws_vpc.selected.id]
+  }
+
+  filter {
+    name   = "tag:Name"
+    values = ["private"]
+  }
+}
+
+data "aws_security_group" "vpc" {
+  vpc_id = data.aws_vpc.selected.id
+
+  filter {
+    name   = "tag:Name"
+    values = ["spacelift-created-vpc-default"]
+  }
+
+}

modules/sage-aws-eks/main.tf

@@ -0,0 +1,126 @@
+resource "aws_iam_role" "admin_role" {
+  name = "eks_admin_role_${var.cluster_name}"
+
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Effect = "Allow"
+        Principal = {
+          AWS = "arn:aws:iam::766808016710:root" # Replace YOUR_AWS_ACCOUNT_ID with your actual AWS account ID
+        }
+        Action = "sts:AssumeRole"
+      },
+    ]
+  })
+
+  tags = var.tags
+}
+
+resource "aws_iam_role" "viewer_role" {
+  name = "eks_viewer_role_${var.cluster_name}"
+
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Effect = "Allow"
+        Principal = {
+          AWS = "arn:aws:sts::766808016710:assumed-role/AWSReservedSSO_Developer_92af2c086e7e7f38/bryan.fauble@sagebase.org"
+        }
+        Action = "sts:AssumeRole"
+      },
+    ]
+  })
+
+  tags = var.tags
+}
+
+resource "aws_iam_role_policy_attachment" "admin_policy" {
+  role       = aws_iam_role.admin_role.name
+  policy_arn = "arn:aws:iam::aws:policy/PowerUserAccess"
+}
+
+module "eks" {
+  source  = "terraform-aws-modules/eks/aws"
+  version = "~> 20.12"
+
+  cluster_name    = var.cluster_name
+  cluster_version = var.cluster_version
+
+  cluster_endpoint_public_access = true
+
+  cluster_addons = {
+    # coredns = {
+    #   most_recent = true
+    # }
+    kube-proxy = {
+      most_recent = true
+    }
+    vpc-cni = {
+      most_recent = true
+    }
+    # TODO When the cluster is created we need to set the gp2 storageclass as default:
+    # kubectl patch storageclass gp2 -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'
+    # This way any PVC that is created will use gp2 as the default storage class
+    # aws-ebs-csi-driver = {
+    #   most_recent = true
+    # }
+  }
+# TODO: The AWS EBS CSI driver is not working right for some reason. PVC are made, but storage is not being allocated. Determine why
+  vpc_id                    = data.aws_vpc.selected.id
+  subnet_ids                = data.aws_subnets.private.ids
+  # TODO
+  # control_plane_subnet_ids  = data.vpc.intra_subnets module.vpc.intra_subnets
+  cluster_security_group_id = data.aws_security_group.vpc.id
+
+  iam_role_additional_policies = {
+    AmazonEBSCSIDriverPolicy = "arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy",
+    SecretsManagerReadWrite  = "arn:aws:iam::aws:policy/SecretsManagerReadWrite"
+    WorkerNodePolicy         = "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy"
+  }
+
+  # Cluster access entry
+  # To add the current caller identity as an administrator
+  enable_cluster_creator_admin_permissions = true
+  authentication_mode                      = "API"
+
+
+  access_entries = {
+    # One access entry with a policy associated
+    eks_admin_role = {
+      kubernetes_groups = []
+      principal_arn     = aws_iam_role.admin_role.arn
+
+      policy_associations = {
+        eks_admin_role = {
+          policy_arn = "arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy"
+          access_scope = {
+            type = "cluster"
+          }
+        }
+      }
+    }
+    eks_viewer_role = {
+      kubernetes_groups = []
+      principal_arn     = aws_iam_role.viewer_role.arn
+
+      policy_associations = {
+        eks_admin_role = {
+          policy_arn = "arn:aws:eks::aws:cluster-access-policy/AmazonEKSViewPolicy"
+          access_scope = {
+            type = "cluster"
+          }
+        }
+      }
+    }
+    # https://docs.aws.amazon.com/eks/latest/userguide/access-policies.html#access-policy-permissions
+    # TODO: Additional roles that need to be created:
+    # AmazonEKSAdminViewPolicy?
+    # AmazonEKSEditPolicy
+    # AmazonEKSViewPolicy
+
+  }
+  tags = var.tags
+}
+

modules/sage-aws-eks/provider.tf

@@ -0,0 +1,3 @@
+provider "aws" {
+  region = var.region
+}

modules/sage-aws-eks/variables.tf

@@ -0,0 +1,25 @@
+variable "cluster_name" {
+  description = "Name of K8 cluster"
+  type        = string
+  default     = "dpe-k8-dev"
+}
+
+variable "cluster_version" {
+  description = "Version of K8 cluster"
+  type        = string
+  default     = "1.30"
+}
+
+variable "region" {
+  description = "AWS region"
+  type        = string
+  default     = "us-east-1"
+}
+
+variable "tags" {
+  description = "AWS Resource Tags"
+  type        = map(string)
+  default = {
+    "CostCenter" = "No Program / 000000"
+  }
+}

modules/sage-aws-eks/versions.tf

@@ -0,0 +1,3 @@
+terraform {
+  required_version = "<= 1.5.7"
+}