[IBCDPE-939] Correcting some issues

.github/CODEOWNERS

@@ -0,0 +1 @@
+* @Sage-Bionetworks-Workflows/dpe

.github/workflows/tfsec_pr_commenter.yml

@@ -0,0 +1,19 @@
+name: tfsec-pr-commenter
+on:
+  pull_request:
+jobs:
+  tfsec:
+    name: tfsec PR commenter
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: read
+      pull-requests: write
+
+    steps:
+      - name: Clone repo
+        uses: actions/checkout@master
+      - name: tfsec
+        uses: aquasecurity/tfsec-pr-commenter-action@v1.2.0
+        with:
+          github_token: ${{ github.token }}

README.md

@@ -1,15 +1,27 @@
 # EKS-stack
 
-Leveraging spot.io, we spin up an EKS stack behind an existing private VPC that has scale-to-zero capabilities. To deploy this stack
+Leveraging spot.io, we spin up an EKS stack behind an existing private VPC that has scale-to-zero capabilities. To deploy this stack:
 
-1. log into dpe-prod via jumpcloud and export the credentials (you must have admin)
+TODO: Instructions need to be re-writen. Deployment is occuring through spacelift.io
+
+<!-- 1. log into dpe-prod via jumpcloud and export the credentials (you must have admin)
 2. run `terraform apply`
 3. This will deploy the terraform stack.  The terraform backend state is stored in an S3 bucket.  The terraform state is stored in the S3 bucket `s3://dpe-terraform-bucket`
 4. The spot.io account token is stored in AWS secrets manager: `spotinst_token`
-5. Add `AmazonEBSCSIDriverPolicy` and `SecretsManagerReadWrite` to the IAM policy
-
-```
-aws eks update-kubeconfig --name tyu-spot-ocean
+5. Add `AmazonEBSCSIDriverPolicy` and `SecretsManagerReadWrite` to the IAM policy -->
+
+To connect to the EKS stack running in AWS you'll need to make sure that you have
+SSO setup for the account you'll be using. Once setup run the commands below:
+```
+# Login with the profile you're using to authenticate. For example mine is called 
+# `dpe-prod-admin`
+aws sso login --profile dpe-prod-admin
+
+# Update your kubeconfig with the proper values. This is saying "Authenticate with 
+# AWS using my SSO session for the profile `dpe-prod-admin`. After authenticated 
+# assuming that we want to use the `role/eks_admin_role` to connect to the k8s 
+# cluster". This will update your kubeconfig with permissions to access the cluster.
+aws eks update-kubeconfig --region us-east-1 --name dpe-k8 --role-arn arn:aws:iam::766808016710:role/eks_admin_role --profile dpe-prod-admin
 ```
 
 ## Future work

main.tf

@@ -19,7 +19,7 @@ resource "aws_iam_role" "admin_role" {
 
 resource "aws_iam_role_policy_attachment" "admin_policy" {
   role       = aws_iam_role.admin_role.name
-  policy_arn = "arn:aws:iam::aws:policy/AdministratorAccess"
+  policy_arn = "arn:aws:iam::aws:policy/PowerUserAccess"
 }
 
 
@@ -132,44 +132,20 @@ module "eks" {
 
       policy_associations = {
         eks_admin_role = {
-          policy_arn = "arn:aws:eks::aws:cluster-access-policy/AmazonEKSAdminPolicy"
+          policy_arn = "arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy"
           access_scope = {
             type = "cluster"
           }
         }
       }
     }
+    # https://docs.aws.amazon.com/eks/latest/userguide/access-policies.html#access-policy-permissions
+    # TODO: Additional roles that need to be created:
+    # AmazonEKSAdminViewPolicy?
+    # AmazonEKSEditPolicy
+    # AmazonEKSViewPolicy
+
   }
   tags = var.tags
 }
 
-module "ocean-controller" {
-  source = "spotinst/ocean-controller/spotinst"
-
-  # Credentials.
-  spotinst_token   = data.aws_secretsmanager_secret_version.secret_credentials.secret_string
-  spotinst_account = var.spotinst_account
-
-  # Configuration.
-  cluster_identifier = var.cluster_name
-}
-
-module "ocean-aws-k8s" {
-  source  = "spotinst/ocean-aws-k8s/spotinst"
-  version = "1.2.0"
-
-  depends_on = [module.eks, module.vpc]
-
-  # Configuration
-  cluster_name                     = var.cluster_name
-  region                           = var.region
-  subnet_ids                       = module.vpc.private_subnets
-  worker_instance_profile_arn      = tolist(data.aws_iam_instance_profiles.profile.arns)[0]
-  security_groups                  = [module.eks.node_security_group_id]
-  is_aggressive_scale_down_enabled = true
-  max_scale_down_percentage        = 33
-  # Overwrite Name Tag and add additional
-  # tags = {
-  #   "kubernetes.io/cluster/tyu-spot-ocean" = "owned"
-  # }
-}

modules/internal-k8-infra/data.tf

@@ -13,3 +13,24 @@ data "aws_secretsmanager_secret" "spotinst_token" {
 data "aws_secretsmanager_secret_version" "secret_credentials" {
   secret_id = data.aws_secretsmanager_secret.spotinst_token.id
 }
+
+# TODO: This should search for the VPC using some other value as ID would change
+# on first startup and teardown/restart
+data "aws_subnets" "node_subnets" {
+  filter {
+    name   = "vpc-id"
+    values = ["vpc-0f30cfca319ebc521"]
+  }
+}
+
+# TODO: This should dynamically search for the node group
+data "aws_eks_node_group" "profile" {
+  cluster_name    = var.cluster_name
+  node_group_name = "airflow-node-group-20240517054615841200000009"
+}
+
+data "aws_security_group" "eks_cluster_security_group" {
+  tags = {
+    Name = "${var.cluster_name}-node"
+  }
+}

modules/internal-k8-infra/main.tf

@@ -10,6 +10,22 @@ module "kubernetes-controller" {
   cluster_identifier = var.cluster_name
 }
 
+
+module "ocean-aws-k8s" {
+  source  = "spotinst/ocean-aws-k8s/spotinst"
+  version = "1.2.0"
+
+  # Configuration
+  cluster_name                     = var.cluster_name
+  region                           = var.region
+  subnet_ids                       = data.aws_subnets.node_subnets.ids
+  worker_instance_profile_arn      = data.aws_eks_node_group.profile.node_role_arn
+  security_groups                  = [data.aws_security_group.eks_cluster_security_group.id]
+  is_aggressive_scale_down_enabled = true
+  max_scale_down_percentage        = 33
+  tags                             = var.tags
+}
+
 resource "kubernetes_namespace" "airflow" {
   metadata {
     name = "airflow"

modules/internal-k8-infra/provider.tf

@@ -8,14 +8,14 @@ provider "spotinst" {
 }
 
 provider "kubernetes" {
-  config_path            = "~/.kube/config"
+  config_path            = var.kube_config_path
   host                   = data.aws_eks_cluster.cluster.endpoint
   cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data)
   token                  = data.aws_eks_cluster_auth.cluster.token
 }
 
 provider "helm" {
   kubernetes {
-    config_path = "~/.kube/config"
+    config_path = var.kube_config_path
   }
 }

modules/internal-k8-infra/variables.tf

@@ -1,28 +1,31 @@
-variable "cluster_name" {
-  description = "Name of K8 cluster"
-  type        = string
-  default     = "dpe-k8"
-}
-
-variable "region" {
-  description = "AWS region"
-  type        = string
-  default     = "us-east-1"
-}
-
-variable "spotinst_account" {
-  description = "Spot.io account"
-  type        = string
-  default     = "act-ac6522b4"
-}
-
-variable "tags" {
-  description = "AWS Resource Tags"
-  type        = map(string)
-  default = {
-    "CostCenter" = "No Program / 000000"
-    # "kubernetes.io/cluster/tyu-spot-ocean" = "owned",
-    # "key"   = "kubernetes.io/cluster/tyu-spot-ocean",
-    # "value" = "owned"
-  }
-}
+variable "cluster_name" {
+  description = "Name of K8 cluster"
+  type        = string
+  default     = "dpe-k8"
+}
+
+variable "kube_config_path" {
+  description = "Kube config path"
+  type        = string
+  default     = "~/.kube/config"
+}
+
+variable "region" {
+  description = "AWS region"
+  type        = string
+  default     = "us-east-1"
+}
+
+variable "spotinst_account" {
+  description = "Spot.io account"
+  type        = string
+  default     = "act-ac6522b4"
+}
+
+variable "tags" {
+  description = "AWS Resource Tags"
+  type        = map(string)
+  default = {
+    "CostCenter" = "No Program / 000000"
+  }
+}

modules/internal-k8-infra/versions.tf

@@ -1,8 +1,9 @@
-terraform {
-  required_providers {
-    spotinst = {
-      source  = "spotinst/spotinst"
-      version = "1.172.0" # Specify the version you wish to use
-    }
-  }
-}
+terraform {
+  required_version = "<= 1.5.7"
+  required_providers {
+    spotinst = {
+      source  = "spotinst/spotinst"
+      version = "1.172.0" # Specify the version you wish to use
+    }
+  }
+}

provider.tf

@@ -8,6 +8,7 @@ provider "spotinst" {
 }
 
 provider "kubernetes" {
+  config_path            = var.kube_config_path
   host                   = data.aws_eks_cluster.cluster.endpoint
   cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data)
   token                  = data.aws_eks_cluster_auth.cluster.token