diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000..0eb6f53e --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @Sage-Bionetworks-Workflows/dpe \ No newline at end of file diff --git a/.github/workflows/tfsec_pr_commenter.yml b/.github/workflows/tfsec_pr_commenter.yml new file mode 100644 index 00000000..0d968acd --- /dev/null +++ b/.github/workflows/tfsec_pr_commenter.yml @@ -0,0 +1,19 @@ +name: tfsec-pr-commenter +on: + pull_request: +jobs: + tfsec: + name: tfsec PR commenter + runs-on: ubuntu-latest + + permissions: + contents: read + pull-requests: write + + steps: + - name: Clone repo + uses: actions/checkout@master + - name: tfsec + uses: aquasecurity/tfsec-pr-commenter-action@v1.2.0 + with: + github_token: ${{ github.token }} \ No newline at end of file diff --git a/README.md b/README.md index b9178acb..36ba6e5b 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,27 @@ # EKS-stack -Leveraging spot.io, we spin up an EKS stack behind an existing private VPC that has scale-to-zero capabilities. To deploy this stack +Leveraging spot.io, we spin up an EKS stack behind an existing private VPC that has scale-to-zero capabilities. To deploy this stack: -1. log into dpe-prod via jumpcloud and export the credentials (you must have admin) +TODO: Instructions need to be re-writen. Deployment is occuring through spacelift.io + + + +To connect to the EKS stack running in AWS you'll need to make sure that you have +SSO setup for the account you'll be using. Once setup run the commands below: +``` +# Login with the profile you're using to authenticate. For example mine is called +# `dpe-prod-admin` +aws sso login --profile dpe-prod-admin + +# Update your kubeconfig with the proper values. This is saying "Authenticate with +# AWS using my SSO session for the profile `dpe-prod-admin`. After authenticated +# assuming that we want to use the `role/eks_admin_role` to connect to the k8s +# cluster". This will update your kubeconfig with permissions to access the cluster. +aws eks update-kubeconfig --region us-east-1 --name dpe-k8 --role-arn arn:aws:iam::766808016710:role/eks_admin_role --profile dpe-prod-admin ``` ## Future work diff --git a/main.tf b/main.tf index 9182ba13..d208b71a 100644 --- a/main.tf +++ b/main.tf @@ -19,7 +19,7 @@ resource "aws_iam_role" "admin_role" { resource "aws_iam_role_policy_attachment" "admin_policy" { role = aws_iam_role.admin_role.name - policy_arn = "arn:aws:iam::aws:policy/AdministratorAccess" + policy_arn = "arn:aws:iam::aws:policy/PowerUserAccess" } @@ -132,44 +132,20 @@ module "eks" { policy_associations = { eks_admin_role = { - policy_arn = "arn:aws:eks::aws:cluster-access-policy/AmazonEKSAdminPolicy" + policy_arn = "arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy" access_scope = { type = "cluster" } } } } + # https://docs.aws.amazon.com/eks/latest/userguide/access-policies.html#access-policy-permissions + # TODO: Additional roles that need to be created: + # AmazonEKSAdminViewPolicy? + # AmazonEKSEditPolicy + # AmazonEKSViewPolicy + } tags = var.tags } -module "ocean-controller" { - source = "spotinst/ocean-controller/spotinst" - - # Credentials. - spotinst_token = data.aws_secretsmanager_secret_version.secret_credentials.secret_string - spotinst_account = var.spotinst_account - - # Configuration. - cluster_identifier = var.cluster_name -} - -module "ocean-aws-k8s" { - source = "spotinst/ocean-aws-k8s/spotinst" - version = "1.2.0" - - depends_on = [module.eks, module.vpc] - - # Configuration - cluster_name = var.cluster_name - region = var.region - subnet_ids = module.vpc.private_subnets - worker_instance_profile_arn = tolist(data.aws_iam_instance_profiles.profile.arns)[0] - security_groups = [module.eks.node_security_group_id] - is_aggressive_scale_down_enabled = true - max_scale_down_percentage = 33 - # Overwrite Name Tag and add additional - # tags = { - # "kubernetes.io/cluster/tyu-spot-ocean" = "owned" - # } -} diff --git a/modules/internal-k8-infra/data.tf b/modules/internal-k8-infra/data.tf index f069b694..36c24cac 100644 --- a/modules/internal-k8-infra/data.tf +++ b/modules/internal-k8-infra/data.tf @@ -13,3 +13,24 @@ data "aws_secretsmanager_secret" "spotinst_token" { data "aws_secretsmanager_secret_version" "secret_credentials" { secret_id = data.aws_secretsmanager_secret.spotinst_token.id } + +# TODO: This should search for the VPC using some other value as ID would change +# on first startup and teardown/restart +data "aws_subnets" "node_subnets" { + filter { + name = "vpc-id" + values = ["vpc-0f30cfca319ebc521"] + } +} + +# TODO: This should dynamically search for the node group +data "aws_eks_node_group" "profile" { + cluster_name = var.cluster_name + node_group_name = "airflow-node-group-20240517054615841200000009" +} + +data "aws_security_group" "eks_cluster_security_group" { + tags = { + Name = "${var.cluster_name}-node" + } +} diff --git a/modules/internal-k8-infra/main.tf b/modules/internal-k8-infra/main.tf index f5557790..7ba02144 100644 --- a/modules/internal-k8-infra/main.tf +++ b/modules/internal-k8-infra/main.tf @@ -10,6 +10,22 @@ module "kubernetes-controller" { cluster_identifier = var.cluster_name } + +module "ocean-aws-k8s" { + source = "spotinst/ocean-aws-k8s/spotinst" + version = "1.2.0" + + # Configuration + cluster_name = var.cluster_name + region = var.region + subnet_ids = data.aws_subnets.node_subnets.ids + worker_instance_profile_arn = data.aws_eks_node_group.profile.node_role_arn + security_groups = [data.aws_security_group.eks_cluster_security_group.id] + is_aggressive_scale_down_enabled = true + max_scale_down_percentage = 33 + tags = var.tags +} + resource "kubernetes_namespace" "airflow" { metadata { name = "airflow" diff --git a/modules/internal-k8-infra/provider.tf b/modules/internal-k8-infra/provider.tf index d771695a..451c9b98 100644 --- a/modules/internal-k8-infra/provider.tf +++ b/modules/internal-k8-infra/provider.tf @@ -8,7 +8,7 @@ provider "spotinst" { } provider "kubernetes" { - config_path = "~/.kube/config" + config_path = var.kube_config_path host = data.aws_eks_cluster.cluster.endpoint cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data) token = data.aws_eks_cluster_auth.cluster.token @@ -16,6 +16,6 @@ provider "kubernetes" { provider "helm" { kubernetes { - config_path = "~/.kube/config" + config_path = var.kube_config_path } } diff --git a/modules/internal-k8-infra/variables.tf b/modules/internal-k8-infra/variables.tf index f9ba8126..8c8cde6f 100644 --- a/modules/internal-k8-infra/variables.tf +++ b/modules/internal-k8-infra/variables.tf @@ -1,28 +1,31 @@ -variable "cluster_name" { - description = "Name of K8 cluster" - type = string - default = "dpe-k8" -} - -variable "region" { - description = "AWS region" - type = string - default = "us-east-1" -} - -variable "spotinst_account" { - description = "Spot.io account" - type = string - default = "act-ac6522b4" -} - -variable "tags" { - description = "AWS Resource Tags" - type = map(string) - default = { - "CostCenter" = "No Program / 000000" - # "kubernetes.io/cluster/tyu-spot-ocean" = "owned", - # "key" = "kubernetes.io/cluster/tyu-spot-ocean", - # "value" = "owned" - } -} +variable "cluster_name" { + description = "Name of K8 cluster" + type = string + default = "dpe-k8" +} + +variable "kube_config_path" { + description = "Kube config path" + type = string + default = "~/.kube/config" +} + +variable "region" { + description = "AWS region" + type = string + default = "us-east-1" +} + +variable "spotinst_account" { + description = "Spot.io account" + type = string + default = "act-ac6522b4" +} + +variable "tags" { + description = "AWS Resource Tags" + type = map(string) + default = { + "CostCenter" = "No Program / 000000" + } +} diff --git a/modules/internal-k8-infra/versions.tf b/modules/internal-k8-infra/versions.tf index 09f88186..5508ca73 100644 --- a/modules/internal-k8-infra/versions.tf +++ b/modules/internal-k8-infra/versions.tf @@ -1,8 +1,9 @@ -terraform { - required_providers { - spotinst = { - source = "spotinst/spotinst" - version = "1.172.0" # Specify the version you wish to use - } - } -} +terraform { + required_version = "<= 1.5.7" + required_providers { + spotinst = { + source = "spotinst/spotinst" + version = "1.172.0" # Specify the version you wish to use + } + } +} diff --git a/provider.tf b/provider.tf index 02104066..e57ccf43 100644 --- a/provider.tf +++ b/provider.tf @@ -8,6 +8,7 @@ provider "spotinst" { } provider "kubernetes" { + config_path = var.kube_config_path host = data.aws_eks_cluster.cluster.endpoint cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) token = data.aws_eks_cluster_auth.cluster.token diff --git a/variables.tf b/variables.tf index 30b7d7cc..9eb7308d 100644 --- a/variables.tf +++ b/variables.tf @@ -1,72 +1,75 @@ -variable "cluster_name" { - description = "Name of K8 cluster" - type = string - default = "dpe-k8" -} - -variable "cluster_version" { - description = "Version of K8 cluster" - type = string - default = "1.29" -} - -variable "cidr" { - description = "CIDR block for the VPC" - type = string - default = "10.51.0.0/16" -} - -variable "public_subnet_cidrs" { - type = list(string) - description = "Public Subnet CIDR values" - default = ["10.51.1.0/24", "10.51.2.0/24", "10.51.3.0/24"] -} - -variable "private_subnet_cidrs" { - type = list(string) - description = "Private Subnet CIDR values" - default = ["10.51.4.0/24", "10.51.5.0/24", "10.51.6.0/24"] -} - -variable "create_igw" { - description = "Controls if an Internet Gateway is created for public subnets and the related routes that connect them" - type = string - default = "false" -} - -variable "azs" { - - type = list(string) - description = "Availability Zones" - default = ["us-east-1a", "us-east-1b", "us-east-1c"] - -} - -variable "region" { - description = "AWS region" - type = string - default = "us-east-1" -} - -variable "spotinst_account" { - description = "Spot.io account" - type = string - default = "act-ac6522b4" -} - -variable "eks_nodeGroup" { - description = "EKS node group name" - type = string - default = "airflow-node-group" -} - -variable "tags" { - description = "AWS Resource Tags" - type = map(string) - default = { - "CostCenter" = "No Program / 000000" - # "kubernetes.io/cluster/tyu-spot-ocean" = "owned", - # "key" = "kubernetes.io/cluster/tyu-spot-ocean", - # "value" = "owned" - } -} +variable "cluster_name" { + description = "Name of K8 cluster" + type = string + default = "dpe-k8" +} + +variable "cluster_version" { + description = "Version of K8 cluster" + type = string + default = "1.29" +} + +variable "kube_config_path" { + description = "Kube config path" + type = string + default = "~/.kube/config" +} + +variable "cidr" { + description = "CIDR block for the VPC" + type = string + default = "10.51.0.0/16" +} + +variable "public_subnet_cidrs" { + type = list(string) + description = "Public Subnet CIDR values" + default = ["10.51.1.0/24", "10.51.2.0/24", "10.51.3.0/24"] +} + +variable "private_subnet_cidrs" { + type = list(string) + description = "Private Subnet CIDR values" + default = ["10.51.4.0/24", "10.51.5.0/24", "10.51.6.0/24"] +} + +variable "create_igw" { + description = "Controls if an Internet Gateway is created for public subnets and the related routes that connect them" + type = string + default = "false" +} + +variable "azs" { + + type = list(string) + description = "Availability Zones" + default = ["us-east-1a", "us-east-1b", "us-east-1c"] + +} + +variable "region" { + description = "AWS region" + type = string + default = "us-east-1" +} + +variable "spotinst_account" { + description = "Spot.io account" + type = string + default = "act-ac6522b4" +} + +variable "eks_nodeGroup" { + description = "EKS node group name" + type = string + default = "airflow-node-group" +} + +variable "tags" { + description = "AWS Resource Tags" + type = map(string) + default = { + "CostCenter" = "No Program / 000000" + } +} diff --git a/versions.tf b/versions.tf index 2cec693f..fc844662 100644 --- a/versions.tf +++ b/versions.tf @@ -1,7 +1,8 @@ -terraform { - required_providers { - spotinst = { - source = "spotinst/spotinst" - } - } -} +terraform { + required_version = "<= 1.5.7" + required_providers { + spotinst = { + source = "spotinst/spotinst" + } + } +}