Merge pull request #165 from kbst/workload-identity-support

Integrate K8s workloads into Cloud IAM
kbst · Mar 13, 2021 · 5416c6e · 5416c6e
2 parents cce1239 + 6d22f49
commit 5416c6e
Show file tree

Hide file tree

Showing 19 changed files with 141 additions and 34 deletions.
diff --git a/.gitignore b/.gitignore
@@ -8,3 +8,8 @@
 # Ignore build and dist
 _build/
 _dist/
+
+# No version locking for tests,
+# we want them to blow up in case
+# of issues
+tests/.terraform.lock.hcl
diff --git a/aws/_modules/eks/openid_connect.tf b/aws/_modules/eks/openid_connect.tf
@@ -0,0 +1,17 @@
+data "aws_partition" "current" {
+  count = var.disable_openid_connect_provider == false ? 1 : 0
+}
+
+data "tls_certificate" "current" {
+  count = var.disable_openid_connect_provider == false ? 1 : 0
+
+  url = aws_eks_cluster.current.identity[0].oidc[0].issuer
+}
+
+resource "aws_iam_openid_connect_provider" "current" {
+  count = var.disable_openid_connect_provider == false ? 1 : 0
+
+  client_id_list  = ["sts.${data.aws_partition.current[0].dns_suffix}"]
+  thumbprint_list = [data.tls_certificate.current[0].certificates[0].sha1_fingerprint]
+  url             = aws_eks_cluster.current.identity[0].oidc[0].issuer
+}
diff --git a/aws/_modules/eks/variables.tf b/aws/_modules/eks/variables.tf
@@ -79,3 +79,8 @@ variable "enabled_cluster_log_types" {
   type        = list(string)
   description = "List of cluster log types to enable."
 }
+
+variable "disable_openid_connect_provider" {
+  type        = bool
+  description = "Whether to disable the OpenID connect provider."
+}
diff --git a/aws/_modules/eks/versions.tf b/aws/_modules/eks/versions.tf
@@ -18,6 +18,12 @@ terraform {
       source  = "hashicorp/kubernetes"
       version = "~> 2.0.2"
     }
+
+    tls = {
+      # https://registry.terraform.io/providers/hashicorp/tls/latest
+      source  = "hashicorp/tls"
+      version = "~> 3.1.0"
+    }
   }
 
   required_version = ">= 0.13"

diff --git a/aws/cluster/configuration.tf b/aws/cluster/configuration.tf
@@ -38,4 +38,6 @@ locals {
 
   enabled_cluster_log_types_lookup = lookup(local.cfg, "enabled_cluster_log_types", "api,audit,authenticator,controllerManager,scheduler")
   enabled_cluster_log_types        = split(",", local.enabled_cluster_log_types_lookup)
+
+  disable_openid_connect_provider = lookup(local.cfg, "disable_openid_connect_provider", false)
 }
diff --git a/aws/cluster/main.tf b/aws/cluster/main.tf
@@ -36,4 +36,6 @@ module "cluster" {
   disable_default_ingress = local.disable_default_ingress
 
   enabled_cluster_log_types = local.enabled_cluster_log_types
+
+  disable_openid_connect_provider = local.disable_openid_connect_provider
 }
diff --git a/azurerm/_modules/aks/main.tf b/azurerm/_modules/aks/main.tf
@@ -27,24 +27,38 @@ resource "azurerm_kubernetes_cluster" "current" {
 
     vm_size         = var.default_node_pool_vm_size
     os_disk_size_gb = var.default_node_pool_os_disk_size_gb
-    
+
     vnet_subnet_id = var.network_plugin == "azure" ? azurerm_subnet.current[0].id : null
     max_pods       = var.max_pods
   }
 
   network_profile {
-    network_plugin      = var.network_plugin
-    network_policy      = var.network_policy
+    network_plugin = var.network_plugin
+    network_policy = var.network_policy
+
+    docker_bridge_cidr = "172.17.0.1/16"
+    service_cidr       = var.service_cidr
+    dns_service_ip     = var.dns_service_ip
+    pod_cidr           = var.network_plugin == "azure" ? null : var.pod_cidr
+  }
+
+  dynamic "identity" {
+    for_each = var.disable_managed_identities == true ? toset([]) : toset([1])
 
-    docker_bridge_cidr  = "172.17.0.1/16"
-    service_cidr        = var.service_cidr
-    dns_service_ip      = var.dns_service_ip
-    pod_cidr            = var.network_plugin == "azure" ? null : var.pod_cidr
+    content {
+      type = var.user_assigned_identity_id == null ? "SystemAssigned" : "UserAssigned"
+
+      user_assigned_identity_id = var.user_assigned_identity_id
+    }
   }
 
-  service_principal {
-    client_id     = azuread_application.current.application_id
-    client_secret = azuread_service_principal_password.current.value
+  dynamic "service_principal" {
+    for_each = var.disable_managed_identities == true ? toset([1]) : toset([])
+
+    content {
+      client_id     = azuread_application.current[0].application_id
+      client_secret = azuread_service_principal_password.current[0].value
+    }
   }
 
   addon_profile {

diff --git a/azurerm/_modules/aks/service_principal.tf b/azurerm/_modules/aks/service_principal.tf
@@ -1,18 +1,26 @@
 resource "azuread_application" "current" {
+  count = var.disable_managed_identities == true ? 1 : 0
+
   display_name = var.metadata_name
 }
 
 resource "azuread_service_principal" "current" {
-  application_id = azuread_application.current.application_id
+  count = var.disable_managed_identities == true ? 1 : 0
+
+  application_id = azuread_application.current[0].application_id
 }
 
 resource "random_string" "password" {
+  count = var.disable_managed_identities == true ? 1 : 0
+
   length  = 64
   special = true
 }
 
 resource "azuread_service_principal_password" "current" {
-  service_principal_id = azuread_service_principal.current.id
-  value                = random_string.password.result
+  count = var.disable_managed_identities == true ? 1 : 0
+
+  service_principal_id = azuread_service_principal.current[0].id
+  value                = random_string.password[0].result
   end_date_relative    = var.service_principal_end_date_relative
 }
diff --git a/azurerm/_modules/aks/variables.tf b/azurerm/_modules/aks/variables.tf
@@ -136,3 +136,15 @@ variable "service_principal_end_date_relative" {
   type        = string
   description = "Relative time in hours for which the service principal password is valid. Defaults to 1 year."
 }
+
+variable "disable_managed_identities" {
+  type        = bool
+  description = "Keep using legacy service principal instead of new managed identities."
+  default     = false
+}
+
+variable "user_assigned_identity_id" {
+  type        = string
+  description = "ID of the UserAssigned identity to use."
+  default     = null
+}
diff --git a/azurerm/cluster/configuration.tf b/azurerm/cluster/configuration.tf
@@ -17,9 +17,9 @@ locals {
 
   dns_prefix = lookup(local.cfg, "dns_prefix", "api")
 
-  vnet_address_space        = split(",", lookup(local.cfg, "vnet_address_space", "10.0.0.0/8"))
-  subnet_address_prefixes   = split(",", lookup(local.cfg, "subnet_address_prefixes", "10.1.0.0/16"))
-  subnet_service_endpoints  = split(",", lookup(local.cfg, "subnet_service_endpoints", ""))
+  vnet_address_space       = split(",", lookup(local.cfg, "vnet_address_space", "10.0.0.0/8"))
+  subnet_address_prefixes  = split(",", lookup(local.cfg, "subnet_address_prefixes", "10.1.0.0/16"))
+  subnet_service_endpoints = split(",", lookup(local.cfg, "subnet_service_endpoints", ""))
 
   network_plugin = lookup(local.cfg, "network_plugin", "kubenet")
   network_policy = lookup(local.cfg, "network_policy", "calico")
@@ -46,4 +46,7 @@ locals {
   disable_default_ingress = lookup(local.cfg, "disable_default_ingress", false)
 
   service_principal_end_date_relative = lookup(local.cfg, "service_principal_end_date_relative", "8766h")
+
+  disable_managed_identities = lookup(local.cfg, "disable_managed_identities", false)
+  user_assigned_identity_id  = lookup(local.cfg, "user_assigned_identity_id", null)
 }
diff --git a/azurerm/cluster/main.tf b/azurerm/cluster/main.tf
@@ -27,16 +27,16 @@ module "cluster" {
 
   dns_prefix = local.dns_prefix
 
-  vnet_address_space        = local.vnet_address_space
-  subnet_address_prefixes   = local.subnet_address_prefixes
-  subnet_service_endpoints  = local.subnet_service_endpoints
+  vnet_address_space       = local.vnet_address_space
+  subnet_address_prefixes  = local.subnet_address_prefixes
+  subnet_service_endpoints = local.subnet_service_endpoints
 
-  network_plugin            = local.network_plugin
-  network_policy            = local.network_policy
-  service_cidr              = local.service_cidr
-  dns_service_ip            = local.dns_service_ip
-  pod_cidr                  = local.pod_cidr
-  max_pods                  = local.max_pods
+  network_plugin = local.network_plugin
+  network_policy = local.network_policy
+  service_cidr   = local.service_cidr
+  dns_service_ip = local.dns_service_ip
+  pod_cidr       = local.pod_cidr
+  max_pods       = local.max_pods
 
   default_node_pool_name = local.default_node_pool_name
   default_node_pool_type = local.default_node_pool_type
@@ -54,4 +54,7 @@ module "cluster" {
   disable_default_ingress = local.disable_default_ingress
 
   service_principal_end_date_relative = local.service_principal_end_date_relative
+
+  disable_managed_identities = local.disable_managed_identities
+  user_assigned_identity_id  = local.user_assigned_identity_id
 }
diff --git a/google/_modules/gke/cluster.tf b/google/_modules/gke/cluster.tf
@@ -23,6 +23,13 @@ resource "google_container_cluster" "current" {
 
   network = google_compute_network.current.self_link
 
+  dynamic "workload_identity_config" {
+    for_each = var.disable_workload_identity == false ? toset([1]) : toset([])
+    content {
+      identity_namespace = "${var.project}.svc.id.goog"
+    }
+  }
+
   #
   #
   # Addon config

diff --git a/google/_modules/gke/node_pool.tf b/google/_modules/gke/node_pool.tf
@@ -22,5 +22,6 @@ module "node_pool" {
   disk_type    = var.disk_type
   image_type   = var.image_type
   machine_type = var.machine_type
-}
 
+  node_workload_metadata_config = var.node_workload_metadata_config
+}
diff --git a/google/_modules/gke/node_pool/main.tf b/google/_modules/gke/node_pool/main.tf
@@ -29,6 +29,10 @@ resource "google_container_node_pool" "current" {
     labels = var.metadata_labels
 
     tags = var.metadata_tags
+
+    workload_metadata_config {
+      node_metadata = var.node_workload_metadata_config
+    }
   }
 
   management {

diff --git a/google/_modules/gke/node_pool/variables.tf b/google/_modules/gke/node_pool/variables.tf
@@ -97,3 +97,7 @@ variable "auto_upgrade" {
   default     = true
 }
 
+variable "node_workload_metadata_config" {
+  description = "How to expose the node metadata to the workload running on the node."
+  type        = string
+}
diff --git a/google/_modules/gke/variables.tf b/google/_modules/gke/variables.tf
@@ -128,3 +128,13 @@ variable "enable_cloud_nat" {
   type        = bool
   description = "Whether to enable cloud nat and allow internet access for private nodes."
 }
+
+variable "disable_workload_identity" {
+  description = "Wheter to disable workload identity support."
+  type        = bool
+}
+
+variable "node_workload_metadata_config" {
+  description = "How to expose the node metadata to the workload running on the node."
+  type        = string
+}
diff --git a/google/cluster/configuration.tf b/google/cluster/configuration.tf
@@ -57,9 +57,13 @@ locals {
 
   disable_default_ingress = lookup(local.cfg, "disable_default_ingress", false)
 
-  enable_private_nodes    = lookup(local.cfg, "enable_private_nodes", true)
-  master_cidr_block       = lookup(local.cfg, "master_cidr_block", "172.16.0.32/28")
+  enable_private_nodes = lookup(local.cfg, "enable_private_nodes", true)
+  master_cidr_block    = lookup(local.cfg, "master_cidr_block", "172.16.0.32/28")
 
   # by default include cloud_nat when private nodes are enabled
   enable_cloud_nat = lookup(local.cfg, "enable_cloud_nat", local.enable_private_nodes)
+
+  disable_workload_identity             = lookup(local.cfg, "disable_workload_identity", false)
+  default_node_workload_metadata_config = tobool(local.disable_workload_identity) == false ? "GKE_METADATA_SERVER" : "UNSPECIFIED"
+  node_workload_metadata_config         = lookup(local.cfg, "node_workload_metadata_config", local.default_node_workload_metadata_config)
 }
diff --git a/google/cluster/main.tf b/google/cluster/main.tf
@@ -48,7 +48,10 @@ module "cluster" {
 
   disable_default_ingress = local.disable_default_ingress
 
-  enable_private_nodes    = local.enable_private_nodes
-  master_cidr_block       = local.master_cidr_block
-  enable_cloud_nat        = local.enable_cloud_nat
+  enable_private_nodes = local.enable_private_nodes
+  master_cidr_block    = local.master_cidr_block
+  enable_cloud_nat     = local.enable_cloud_nat
+
+  disable_workload_identity     = local.disable_workload_identity
+  node_workload_metadata_config = local.node_workload_metadata_config
 }
diff --git a/quickstart/src/configurations/aks/clusters.tf b/quickstart/src/configurations/aks/clusters.tf
@@ -2,7 +2,4 @@ module "aks_zero" {
   source = "github.com/kbst/terraform-kubestack//azurerm/cluster?ref={{version}}"
 
   configuration = var.clusters["aks_zero"]
-
-  # vnet_subnet_id = azurerm_subnet.external.id  # uncomment and populate with an externally-created
-                                                 # subnet's ID when using CNI/advanced networking
 }