[IBCDPE-935] VPC Updates & VPC CNI Exploration

README.md

@@ -17,6 +17,8 @@ This repo is used to deploy an EKS cluster to AWS. CI/CD is managed through Spac
 └── modules: Templatized collections of terraform resources that are used in a stack
     ├── apache-airflow: K8s deployment for apache airflow
     │   └── templates: Resources used during deployment of airflow
+    ├── demo-network-policies: K8s deployment for a demo showcasing how to use network policies
+    ├── demo-pod-level-security-groups-strict: K8s deployment for a demo showcasing how to use pod level security groups in strict mode
     ├── sage-aws-eks: Sage specific EKS cluster for AWS
     ├── sage-aws-k8s-node-autoscaler: K8s node autoscaler using spotinst ocean
     └── sage-aws-vpc: Sage specific VPC for AWS
@@ -54,6 +56,10 @@ configurable parameters in order to run a number of workloads.
 #### EKS API access
 API access to the kubernetes cluster endpoint is set to `Public and private`. 
 
+Reading:
+
+- <https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/network_connectivity.md>
+
 ##### Public
 This allows one outside of the VPC to connect via `kubectl` and related tools to 
 interact with kubernetes resources. By default, this API server endpoint is public to 
@@ -78,9 +84,13 @@ Kubernetes nodes and configuring the necessary networking for Pods on each node.
 Allows us to assign EC2 security groups directly to pods running in AWS EKS clusters.
 This can be used as an alternative or in conjunction with `Kubernetes network policies`.
 
+See `modules/demo-pod-level-security-groups-strict` for more context on how this works.
+
 #### Kubernetes network policies
 Controls network traffic within the cluster, for example pod to pod traffic.
 
+See `modules/demo-network-policies` for more context on how this works.
+
 Further reading:
 - https://docs.aws.amazon.com/eks/latest/userguide/cni-network-policy.html
 - https://docs.aws.amazon.com/eks/latest/userguide/security-groups-for-pods.html
@@ -90,11 +100,26 @@ Further reading:
 
 #### EKS Autoscaler
 
-Us use spot.io to manage the nodes attached to each of the EKS cluster. This tool has
+We use spot.io to manage the nodes attached to each of the EKS cluster. This tool has
 scale-to-zerio capabilities and will dynamically add or removes nodes from the cluster
 depending on the required demand. The autoscaler is templatized and provided as a
 terraform module to be used within an EKS stack.
 
+Setup of spotio (Manual per AWS Account):
+
+* Subscribe through the AWS Marketplace: <https://aws.amazon.com/marketplace/saas/ordering?productId=bc241ac2-7b41-4fdd-89d1-6928ec6dae15>
+* "Set up your account" on the spotio website and link it to an existing organization
+* Link the account through the AWS UI:
+* Create a policy (See the JSON in the spotio UI)
+* Create a role (See instructions in the spotio UI)
+
+After this has been setup the last item is to get an API token from the spotio UI and
+add it to the AWS secret manager.
+
+* Log into the spot UI and go to <https://console.spotinst.com/settings/v2/tokens/permanent>
+* Create a new Permanent token, name it `{AWS-Account-Name}-token` or similar
+* Copy the token and create an `AWS Secrets Manager` Plaintext secret named `spotinst_token` with a description `Spot.io token`
+
 
 #### Connecting to an EKS cluster for kubectl commands
 
@@ -111,3 +136,68 @@ aws sso login --profile dpe-prod-admin
 # cluster". This will update your kubeconfig with permissions to access the cluster.
 aws eks update-kubeconfig --region us-east-1 --name dpe-k8 --role-arn arn:aws:iam::766808016710:role/eks_admin_role --profile dpe-prod-admin
 ```
+
+### Spacelift
+Here are some instructions on setting up spacelift.
+
+
+#### Connecting a new AWS account for cloud integration
+
+This document describes the abbreviated process below:
+<https://docs.spacelift.io/integrations/cloud-providers/aws#setup-guide>
+
+- Create a new role and set it's name to something unique within the account, such as `spacelift-admin-role`
+- Description: "Role for spacelift CICD to assume when deploying resources managed by terraform"
+- Use the custom trust policy below:
+
+```
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Principal": {
+                "AWS": "arn:aws:iam::324880187172:root"
+            },
+            "Action": "sts:AssumeRole",
+            "Condition": {
+                "StringLike": {
+                    "sts:ExternalId": "sagebionetworks@*"
+                }
+            }
+        },
+        {
+            "Effect": "Allow",
+            "Principal": {
+                "AWS": "arn:aws:iam::{{AWS ACCOUNT ID}}:root"
+            },
+            "Action": "sts:AssumeRole"
+        }
+    ]
+}
+```
+
+- Attach a few policies to the role:
+  - `PowerUserAccess`
+  - Create an inline policy to allow interaction with IAM (Needed if TF is going to be creating, editing, and deleting IAM roles/policies):
+```
+{
+	"Version": "2012-10-17",
+	"Statement": [
+		{
+			"Effect": "Allow",
+			"Action": [
+				"iam:*Role",
+				"iam:*RolePolicy",
+				"iam:*RolePolicies",
+				"iam:*Policy",
+				"iam:*PolicyVersion",
+				"iam:*OpenIDConnectProvider",
+				"iam:*InstanceProfile"
+			],
+			"Resource": "*"
+		}
+	]
+}
+```
+- Add a new `spacelift_aws_integration` resources to the `common-resources/aws-integrations` directory.

common-resources/aws-integrations/main.tf

@@ -0,0 +1,8 @@
+# Resources derived from: https://registry.terraform.io/providers/spacelift-io/spacelift/latest/docs/resources/aws_integration
+resource "spacelift_aws_integration" "org-sagebase-dnt-dev-aws-integration" {
+  name                           = "org-sagebase-dnt-dev-aws-integration"
+  role_arn                       = "arn:aws:iam::631692904429:role/spacelift-admin-role"
+  generate_credentials_in_worker = false
+  duration_seconds               = 3600
+  space_id                       = "root"
+}

common-resources/aws-integrations/versions.tf

@@ -0,0 +1,8 @@
+terraform {
+  required_providers {
+    spacelift = {
+      source  = "spacelift-io/spacelift"
+      version = "1.13.0"
+    }
+  }
+}

common-resources/main.tf

@@ -5,3 +5,7 @@ module "policies" {
 module "contexts" {
   source = "./contexts"
 }
+
+module "aws-integrations" {
+  source = "./aws-integrations"
+}

dev/spacelift/dpe-sandbox/main.tf

@@ -13,7 +13,7 @@ resource "spacelift_stack" "k8s-stack" {
 
   administrative          = false
   autodeploy              = true
-  branch                  = "ibcdpe-935-vpc-updates"
+  branch                  = "main"
   description             = "Infrastructure to support deploying to an EKS cluster"
   name                    = "DPE DEV Kubernetes Infrastructure"
   project_root            = "dev/stacks/dpe-sandbox-k8s"
@@ -31,7 +31,7 @@ resource "spacelift_stack" "k8s-stack-deployments" {
 
   administrative          = false
   autodeploy              = true
-  branch                  = "ibcdpe-935-vpc-updates"
+  branch                  = "main"
   description             = "Deployments internal to an EKS cluster"
   name                    = "DPE DEV Kubernetes Deployments"
   project_root            = "dev/stacks/dpe-sandbox-k8s-deployments"
@@ -69,6 +69,12 @@ resource "spacelift_stack_dependency_reference" "security-group-id-reference" {
   input_name          = "TF_VAR_node_security_group_id"
 }
 
+resource "spacelift_stack_dependency_reference" "pod-to-node-security-group-id-reference" {
+  stack_dependency_id = spacelift_stack_dependency.k8s-stack-to-deployments.id
+  output_name         = "pod_to_node_dns_sg_id"
+  input_name          = "TF_VAR_pod_to_node_dns_sg_id"
+}
+
 resource "spacelift_stack_dependency_reference" "vpc-cidr-block-reference" {
   stack_dependency_id = spacelift_stack_dependency.k8s-stack-to-deployments.id
   output_name         = "vpc_cidr_block"
@@ -111,14 +117,16 @@ resource "spacelift_stack_destructor" "k8s-stack-destructor" {
 }
 
 resource "spacelift_aws_integration_attachment" "k8s-aws-integration-attachment" {
-  integration_id = "01HXW154N60KJ8NCC93H1VYPNM"
+  # org-sagebase-dnt-dev-aws-integration
+  integration_id = "01J3DNYVM4AWWSDY3QEVRMQ076"
   stack_id       = spacelift_stack.k8s-stack.id
   read           = true
   write          = true
 }
 
 resource "spacelift_aws_integration_attachment" "k8s-deployments-aws-integration-attachment" {
-  integration_id = "01HXW154N60KJ8NCC93H1VYPNM"
+  # org-sagebase-dnt-dev-aws-integration
+  integration_id = "01J3DNYVM4AWWSDY3QEVRMQ076"
   stack_id       = spacelift_stack.k8s-stack-deployments.id
   read           = true
   write          = true

dev/stacks/dpe-sandbox-k8s-deployments/data.tf

@@ -13,4 +13,3 @@ data "aws_secretsmanager_secret" "spotinst_token" {
 data "aws_secretsmanager_secret_version" "secret_credentials" {
   secret_id = data.aws_secretsmanager_secret.spotinst_token.id
 }
-