From 00fb9ca4c2cad820a0f682cdb48f5889c687e7fd Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Mon, 20 May 2024 10:20:46 -0700 Subject: [PATCH 1/7] Correcting some issues --- .github/CODEOWNERS | 1 + .github/workflows/tfsec_pr_commenter.yml | 19 +++ README.md | 24 +++- main.tf | 12 +- modules/internal-k8-infra/main.tf | 12 -- modules/internal-k8-infra/variables.tf | 53 ++++----- modules/internal-k8-infra/versions.tf | 17 +-- variables.tf | 141 +++++++++++------------ versions.tf | 15 +-- 9 files changed, 158 insertions(+), 136 deletions(-) create mode 100644 .github/CODEOWNERS create mode 100644 .github/workflows/tfsec_pr_commenter.yml diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000..0eb6f53e --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @Sage-Bionetworks-Workflows/dpe \ No newline at end of file diff --git a/.github/workflows/tfsec_pr_commenter.yml b/.github/workflows/tfsec_pr_commenter.yml new file mode 100644 index 00000000..0d968acd --- /dev/null +++ b/.github/workflows/tfsec_pr_commenter.yml @@ -0,0 +1,19 @@ +name: tfsec-pr-commenter +on: + pull_request: +jobs: + tfsec: + name: tfsec PR commenter + runs-on: ubuntu-latest + + permissions: + contents: read + pull-requests: write + + steps: + - name: Clone repo + uses: actions/checkout@master + - name: tfsec + uses: aquasecurity/tfsec-pr-commenter-action@v1.2.0 + with: + github_token: ${{ github.token }} \ No newline at end of file diff --git a/README.md b/README.md index b9178acb..36ba6e5b 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,27 @@ # EKS-stack -Leveraging spot.io, we spin up an EKS stack behind an existing private VPC that has scale-to-zero capabilities. To deploy this stack +Leveraging spot.io, we spin up an EKS stack behind an existing private VPC that has scale-to-zero capabilities. To deploy this stack: -1. log into dpe-prod via jumpcloud and export the credentials (you must have admin) +TODO: Instructions need to be re-writen. Deployment is occuring through spacelift.io + + + +To connect to the EKS stack running in AWS you'll need to make sure that you have +SSO setup for the account you'll be using. Once setup run the commands below: +``` +# Login with the profile you're using to authenticate. For example mine is called +# `dpe-prod-admin` +aws sso login --profile dpe-prod-admin + +# Update your kubeconfig with the proper values. This is saying "Authenticate with +# AWS using my SSO session for the profile `dpe-prod-admin`. After authenticated +# assuming that we want to use the `role/eks_admin_role` to connect to the k8s +# cluster". This will update your kubeconfig with permissions to access the cluster. +aws eks update-kubeconfig --region us-east-1 --name dpe-k8 --role-arn arn:aws:iam::766808016710:role/eks_admin_role --profile dpe-prod-admin ``` ## Future work diff --git a/main.tf b/main.tf index 9182ba13..4f18bd67 100644 --- a/main.tf +++ b/main.tf @@ -19,7 +19,7 @@ resource "aws_iam_role" "admin_role" { resource "aws_iam_role_policy_attachment" "admin_policy" { role = aws_iam_role.admin_role.name - policy_arn = "arn:aws:iam::aws:policy/AdministratorAccess" + policy_arn = "arn:aws:iam::aws:policy/PowerUserAccess" } @@ -132,13 +132,19 @@ module "eks" { policy_associations = { eks_admin_role = { - policy_arn = "arn:aws:eks::aws:cluster-access-policy/AmazonEKSAdminPolicy" + policy_arn = "arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy" access_scope = { type = "cluster" } } } - } + } + # https://docs.aws.amazon.com/eks/latest/userguide/access-policies.html#access-policy-permissions + # TODO: Additional roles that need to be created: + # AmazonEKSAdminViewPolicy? + # AmazonEKSEditPolicy + # AmazonEKSViewPolicy + } tags = var.tags } diff --git a/modules/internal-k8-infra/main.tf b/modules/internal-k8-infra/main.tf index f5557790..256bbd7d 100644 --- a/modules/internal-k8-infra/main.tf +++ b/modules/internal-k8-infra/main.tf @@ -1,15 +1,3 @@ -module "kubernetes-controller" { - source = "spotinst/kubernetes-controller/ocean" - version = "0.0.2" - - # Credentials - spotinst_token = data.aws_secretsmanager_secret_version.secret_credentials.secret_string - spotinst_account = var.spotinst_account - - # Configuration - cluster_identifier = var.cluster_name -} - resource "kubernetes_namespace" "airflow" { metadata { name = "airflow" diff --git a/modules/internal-k8-infra/variables.tf b/modules/internal-k8-infra/variables.tf index f9ba8126..113e2d23 100644 --- a/modules/internal-k8-infra/variables.tf +++ b/modules/internal-k8-infra/variables.tf @@ -1,28 +1,25 @@ -variable "cluster_name" { - description = "Name of K8 cluster" - type = string - default = "dpe-k8" -} - -variable "region" { - description = "AWS region" - type = string - default = "us-east-1" -} - -variable "spotinst_account" { - description = "Spot.io account" - type = string - default = "act-ac6522b4" -} - -variable "tags" { - description = "AWS Resource Tags" - type = map(string) - default = { - "CostCenter" = "No Program / 000000" - # "kubernetes.io/cluster/tyu-spot-ocean" = "owned", - # "key" = "kubernetes.io/cluster/tyu-spot-ocean", - # "value" = "owned" - } -} +variable "cluster_name" { + description = "Name of K8 cluster" + type = string + default = "dpe-k8" +} + +variable "region" { + description = "AWS region" + type = string + default = "us-east-1" +} + +variable "spotinst_account" { + description = "Spot.io account" + type = string + default = "act-ac6522b4" +} + +variable "tags" { + description = "AWS Resource Tags" + type = map(string) + default = { + "CostCenter" = "No Program / 000000" + } +} diff --git a/modules/internal-k8-infra/versions.tf b/modules/internal-k8-infra/versions.tf index 09f88186..5508ca73 100644 --- a/modules/internal-k8-infra/versions.tf +++ b/modules/internal-k8-infra/versions.tf @@ -1,8 +1,9 @@ -terraform { - required_providers { - spotinst = { - source = "spotinst/spotinst" - version = "1.172.0" # Specify the version you wish to use - } - } -} +terraform { + required_version = "<= 1.5.7" + required_providers { + spotinst = { + source = "spotinst/spotinst" + version = "1.172.0" # Specify the version you wish to use + } + } +} diff --git a/variables.tf b/variables.tf index 30b7d7cc..c3e737d4 100644 --- a/variables.tf +++ b/variables.tf @@ -1,72 +1,69 @@ -variable "cluster_name" { - description = "Name of K8 cluster" - type = string - default = "dpe-k8" -} - -variable "cluster_version" { - description = "Version of K8 cluster" - type = string - default = "1.29" -} - -variable "cidr" { - description = "CIDR block for the VPC" - type = string - default = "10.51.0.0/16" -} - -variable "public_subnet_cidrs" { - type = list(string) - description = "Public Subnet CIDR values" - default = ["10.51.1.0/24", "10.51.2.0/24", "10.51.3.0/24"] -} - -variable "private_subnet_cidrs" { - type = list(string) - description = "Private Subnet CIDR values" - default = ["10.51.4.0/24", "10.51.5.0/24", "10.51.6.0/24"] -} - -variable "create_igw" { - description = "Controls if an Internet Gateway is created for public subnets and the related routes that connect them" - type = string - default = "false" -} - -variable "azs" { - - type = list(string) - description = "Availability Zones" - default = ["us-east-1a", "us-east-1b", "us-east-1c"] - -} - -variable "region" { - description = "AWS region" - type = string - default = "us-east-1" -} - -variable "spotinst_account" { - description = "Spot.io account" - type = string - default = "act-ac6522b4" -} - -variable "eks_nodeGroup" { - description = "EKS node group name" - type = string - default = "airflow-node-group" -} - -variable "tags" { - description = "AWS Resource Tags" - type = map(string) - default = { - "CostCenter" = "No Program / 000000" - # "kubernetes.io/cluster/tyu-spot-ocean" = "owned", - # "key" = "kubernetes.io/cluster/tyu-spot-ocean", - # "value" = "owned" - } -} +variable "cluster_name" { + description = "Name of K8 cluster" + type = string + default = "dpe-k8" +} + +variable "cluster_version" { + description = "Version of K8 cluster" + type = string + default = "1.29" +} + +variable "cidr" { + description = "CIDR block for the VPC" + type = string + default = "10.51.0.0/16" +} + +variable "public_subnet_cidrs" { + type = list(string) + description = "Public Subnet CIDR values" + default = ["10.51.1.0/24", "10.51.2.0/24", "10.51.3.0/24"] +} + +variable "private_subnet_cidrs" { + type = list(string) + description = "Private Subnet CIDR values" + default = ["10.51.4.0/24", "10.51.5.0/24", "10.51.6.0/24"] +} + +variable "create_igw" { + description = "Controls if an Internet Gateway is created for public subnets and the related routes that connect them" + type = string + default = "false" +} + +variable "azs" { + + type = list(string) + description = "Availability Zones" + default = ["us-east-1a", "us-east-1b", "us-east-1c"] + +} + +variable "region" { + description = "AWS region" + type = string + default = "us-east-1" +} + +variable "spotinst_account" { + description = "Spot.io account" + type = string + default = "act-ac6522b4" +} + +variable "eks_nodeGroup" { + description = "EKS node group name" + type = string + default = "airflow-node-group" +} + +variable "tags" { + description = "AWS Resource Tags" + type = map(string) + default = { + "CostCenter" = "No Program / 000000" + } +} diff --git a/versions.tf b/versions.tf index 2cec693f..fc844662 100644 --- a/versions.tf +++ b/versions.tf @@ -1,7 +1,8 @@ -terraform { - required_providers { - spotinst = { - source = "spotinst/spotinst" - } - } -} +terraform { + required_version = "<= 1.5.7" + required_providers { + spotinst = { + source = "spotinst/spotinst" + } + } +} From cd63eec8b80f91663b1b98880a9e34326ce361b0 Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Mon, 20 May 2024 11:47:13 -0700 Subject: [PATCH 2/7] Shift in which module the ocean controller is deployed --- main.tf | 42 +++++-------------------------- modules/internal-k8-infra/data.tf | 16 ++++++++++++ modules/internal-k8-infra/main.tf | 28 +++++++++++++++++++++ 3 files changed, 50 insertions(+), 36 deletions(-) diff --git a/main.tf b/main.tf index 4f18bd67..d208b71a 100644 --- a/main.tf +++ b/main.tf @@ -138,44 +138,14 @@ module "eks" { } } } - } - # https://docs.aws.amazon.com/eks/latest/userguide/access-policies.html#access-policy-permissions - # TODO: Additional roles that need to be created: - # AmazonEKSAdminViewPolicy? - # AmazonEKSEditPolicy - # AmazonEKSViewPolicy + } + # https://docs.aws.amazon.com/eks/latest/userguide/access-policies.html#access-policy-permissions + # TODO: Additional roles that need to be created: + # AmazonEKSAdminViewPolicy? + # AmazonEKSEditPolicy + # AmazonEKSViewPolicy } tags = var.tags } -module "ocean-controller" { - source = "spotinst/ocean-controller/spotinst" - - # Credentials. - spotinst_token = data.aws_secretsmanager_secret_version.secret_credentials.secret_string - spotinst_account = var.spotinst_account - - # Configuration. - cluster_identifier = var.cluster_name -} - -module "ocean-aws-k8s" { - source = "spotinst/ocean-aws-k8s/spotinst" - version = "1.2.0" - - depends_on = [module.eks, module.vpc] - - # Configuration - cluster_name = var.cluster_name - region = var.region - subnet_ids = module.vpc.private_subnets - worker_instance_profile_arn = tolist(data.aws_iam_instance_profiles.profile.arns)[0] - security_groups = [module.eks.node_security_group_id] - is_aggressive_scale_down_enabled = true - max_scale_down_percentage = 33 - # Overwrite Name Tag and add additional - # tags = { - # "kubernetes.io/cluster/tyu-spot-ocean" = "owned" - # } -} diff --git a/modules/internal-k8-infra/data.tf b/modules/internal-k8-infra/data.tf index f069b694..17235fcb 100644 --- a/modules/internal-k8-infra/data.tf +++ b/modules/internal-k8-infra/data.tf @@ -13,3 +13,19 @@ data "aws_secretsmanager_secret" "spotinst_token" { data "aws_secretsmanager_secret_version" "secret_credentials" { secret_id = data.aws_secretsmanager_secret.spotinst_token.id } + +# TODO: This should search for the VPC using some other value as ID would change +# on first startup and teardown/restart +data "aws_vpc" "selected" { + id = "spacelift-created-vpc vpc-0f30cfca319ebc521" +} + +data "aws_eks_node_group" "profile" { + cluster_name = var.cluster_name + node_group_name = "one" +} + +# TODO: This may be wrong +data "aws_iam_instance_profiles" "profile" { + role_name = data.aws_eks_cluster.eks_managed_node_groups["one"].iam_role_name +} diff --git a/modules/internal-k8-infra/main.tf b/modules/internal-k8-infra/main.tf index 256bbd7d..0ee4ec08 100644 --- a/modules/internal-k8-infra/main.tf +++ b/modules/internal-k8-infra/main.tf @@ -1,3 +1,31 @@ +module "kubernetes-controller" { + source = "spotinst/kubernetes-controller/ocean" + version = "0.0.2" + + # Credentials + spotinst_token = data.aws_secretsmanager_secret_version.secret_credentials.secret_string + spotinst_account = var.spotinst_account + + # Configuration + cluster_identifier = var.cluster_name +} + + +module "ocean-aws-k8s" { + source = "spotinst/ocean-aws-k8s/spotinst" + version = "1.2.0" + + # Configuration + cluster_name = var.cluster_name + region = var.region + subnet_ids = data.aws_vpc.private_subnets + worker_instance_profile_arn = data.aws_eks_node_group.profile.node_role_arn + security_groups = [data.aws_eks_cluster.cluster.node_security_group_id] + is_aggressive_scale_down_enabled = true + max_scale_down_percentage = 33 + tags = var.tags +} + resource "kubernetes_namespace" "airflow" { metadata { name = "airflow" From c6b9e376c1a4aebb7de09aadbcba772570e4a97b Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Mon, 20 May 2024 12:06:15 -0700 Subject: [PATCH 3/7] Try different values --- modules/internal-k8-infra/data.tf | 12 +++---- modules/internal-k8-infra/main.tf | 56 +++++++++++++++---------------- 2 files changed, 32 insertions(+), 36 deletions(-) diff --git a/modules/internal-k8-infra/data.tf b/modules/internal-k8-infra/data.tf index 17235fcb..c74b2769 100644 --- a/modules/internal-k8-infra/data.tf +++ b/modules/internal-k8-infra/data.tf @@ -20,12 +20,8 @@ data "aws_vpc" "selected" { id = "spacelift-created-vpc vpc-0f30cfca319ebc521" } -data "aws_eks_node_group" "profile" { - cluster_name = var.cluster_name - node_group_name = "one" -} - -# TODO: This may be wrong -data "aws_iam_instance_profiles" "profile" { - role_name = data.aws_eks_cluster.eks_managed_node_groups["one"].iam_role_name +data "aws_security_group" "eks_cluster_security_group" { + tags = { + Name = "${var.cluster_name}-node" + } } diff --git a/modules/internal-k8-infra/main.tf b/modules/internal-k8-infra/main.tf index 0ee4ec08..5aac701c 100644 --- a/modules/internal-k8-infra/main.tf +++ b/modules/internal-k8-infra/main.tf @@ -1,31 +1,31 @@ -module "kubernetes-controller" { - source = "spotinst/kubernetes-controller/ocean" - version = "0.0.2" - - # Credentials - spotinst_token = data.aws_secretsmanager_secret_version.secret_credentials.secret_string - spotinst_account = var.spotinst_account - - # Configuration - cluster_identifier = var.cluster_name -} - - -module "ocean-aws-k8s" { - source = "spotinst/ocean-aws-k8s/spotinst" - version = "1.2.0" - - # Configuration - cluster_name = var.cluster_name - region = var.region - subnet_ids = data.aws_vpc.private_subnets - worker_instance_profile_arn = data.aws_eks_node_group.profile.node_role_arn - security_groups = [data.aws_eks_cluster.cluster.node_security_group_id] - is_aggressive_scale_down_enabled = true - max_scale_down_percentage = 33 - tags = var.tags -} - +module "kubernetes-controller" { + source = "spotinst/kubernetes-controller/ocean" + version = "0.0.2" + + # Credentials + spotinst_token = data.aws_secretsmanager_secret_version.secret_credentials.secret_string + spotinst_account = var.spotinst_account + + # Configuration + cluster_identifier = var.cluster_name +} + + +module "ocean-aws-k8s" { + source = "spotinst/ocean-aws-k8s/spotinst" + version = "1.2.0" + + # Configuration + cluster_name = var.cluster_name + region = var.region + subnet_ids = data.aws_vpc.selected.private_subnets + worker_instance_profile_arn = data.aws_eks_node_group.profile.node_role_arn + security_groups = [data.aws_security_group.eks_cluster_security_group.id] + is_aggressive_scale_down_enabled = true + max_scale_down_percentage = 33 + tags = var.tags +} + resource "kubernetes_namespace" "airflow" { metadata { name = "airflow" From ed67b8600855727e4b8ebd83564e4762eba91826 Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Mon, 20 May 2024 12:10:54 -0700 Subject: [PATCH 4/7] Corrections --- modules/internal-k8-infra/data.tf | 12 ++++++++++-- modules/internal-k8-infra/main.tf | 2 +- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/modules/internal-k8-infra/data.tf b/modules/internal-k8-infra/data.tf index c74b2769..fb63bda4 100644 --- a/modules/internal-k8-infra/data.tf +++ b/modules/internal-k8-infra/data.tf @@ -16,8 +16,16 @@ data "aws_secretsmanager_secret_version" "secret_credentials" { # TODO: This should search for the VPC using some other value as ID would change # on first startup and teardown/restart -data "aws_vpc" "selected" { - id = "spacelift-created-vpc vpc-0f30cfca319ebc521" +data "aws_subnets" "node_subnets" { + filter { + name = "vpc-id" + values = ["vpc-0f30cfca319ebc521"] + } +} + +data "aws_eks_node_group" "profile" { + cluster_name = var.cluster_name + node_group_name = "one" } data "aws_security_group" "eks_cluster_security_group" { diff --git a/modules/internal-k8-infra/main.tf b/modules/internal-k8-infra/main.tf index 5aac701c..7ba02144 100644 --- a/modules/internal-k8-infra/main.tf +++ b/modules/internal-k8-infra/main.tf @@ -18,7 +18,7 @@ module "ocean-aws-k8s" { # Configuration cluster_name = var.cluster_name region = var.region - subnet_ids = data.aws_vpc.selected.private_subnets + subnet_ids = data.aws_subnets.node_subnets.ids worker_instance_profile_arn = data.aws_eks_node_group.profile.node_role_arn security_groups = [data.aws_security_group.eks_cluster_security_group.id] is_aggressive_scale_down_enabled = true From 21b6e893da72e1f93813ad0fea8f5da345d5290a Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Mon, 20 May 2024 12:13:16 -0700 Subject: [PATCH 5/7] Correct node_group_name --- modules/internal-k8-infra/data.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/internal-k8-infra/data.tf b/modules/internal-k8-infra/data.tf index fb63bda4..559cab03 100644 --- a/modules/internal-k8-infra/data.tf +++ b/modules/internal-k8-infra/data.tf @@ -25,7 +25,7 @@ data "aws_subnets" "node_subnets" { data "aws_eks_node_group" "profile" { cluster_name = var.cluster_name - node_group_name = "one" + node_group_name = "airflow-node-group" } data "aws_security_group" "eks_cluster_security_group" { From 275e98a2ac9a5ab55895544659da9c91bb003944 Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Mon, 20 May 2024 12:18:49 -0700 Subject: [PATCH 6/7] Try static node group ID --- modules/internal-k8-infra/data.tf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/internal-k8-infra/data.tf b/modules/internal-k8-infra/data.tf index 559cab03..36c24cac 100644 --- a/modules/internal-k8-infra/data.tf +++ b/modules/internal-k8-infra/data.tf @@ -23,9 +23,10 @@ data "aws_subnets" "node_subnets" { } } +# TODO: This should dynamically search for the node group data "aws_eks_node_group" "profile" { cluster_name = var.cluster_name - node_group_name = "airflow-node-group" + node_group_name = "airflow-node-group-20240517054615841200000009" } data "aws_security_group" "eks_cluster_security_group" { From fe149a318179912c9757d1bb85bd4a885f7470d4 Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Mon, 20 May 2024 12:39:13 -0700 Subject: [PATCH 7/7] Set kube context via config path --- modules/internal-k8-infra/provider.tf | 4 ++-- modules/internal-k8-infra/variables.tf | 6 ++++++ provider.tf | 1 + variables.tf | 6 ++++++ 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/modules/internal-k8-infra/provider.tf b/modules/internal-k8-infra/provider.tf index d771695a..451c9b98 100644 --- a/modules/internal-k8-infra/provider.tf +++ b/modules/internal-k8-infra/provider.tf @@ -8,7 +8,7 @@ provider "spotinst" { } provider "kubernetes" { - config_path = "~/.kube/config" + config_path = var.kube_config_path host = data.aws_eks_cluster.cluster.endpoint cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data) token = data.aws_eks_cluster_auth.cluster.token @@ -16,6 +16,6 @@ provider "kubernetes" { provider "helm" { kubernetes { - config_path = "~/.kube/config" + config_path = var.kube_config_path } } diff --git a/modules/internal-k8-infra/variables.tf b/modules/internal-k8-infra/variables.tf index 113e2d23..8c8cde6f 100644 --- a/modules/internal-k8-infra/variables.tf +++ b/modules/internal-k8-infra/variables.tf @@ -4,6 +4,12 @@ variable "cluster_name" { default = "dpe-k8" } +variable "kube_config_path" { + description = "Kube config path" + type = string + default = "~/.kube/config" +} + variable "region" { description = "AWS region" type = string diff --git a/provider.tf b/provider.tf index 02104066..e57ccf43 100644 --- a/provider.tf +++ b/provider.tf @@ -8,6 +8,7 @@ provider "spotinst" { } provider "kubernetes" { + config_path = var.kube_config_path host = data.aws_eks_cluster.cluster.endpoint cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) token = data.aws_eks_cluster_auth.cluster.token diff --git a/variables.tf b/variables.tf index c3e737d4..9eb7308d 100644 --- a/variables.tf +++ b/variables.tf @@ -10,6 +10,12 @@ variable "cluster_version" { default = "1.29" } +variable "kube_config_path" { + description = "Kube config path" + type = string + default = "~/.kube/config" +} + variable "cidr" { description = "CIDR block for the VPC" type = string