fix: ebs csi driver addon (#367)

fixes jenkins-x/jx#8575

Signed-off-by: tgelpi <ted.gelpi@gmail.com>

README.md

@@ -29,6 +29,7 @@ The module makes use of the [Terraform EKS cluster Module](https://github.com/te
       - [Transitioning from Worker Groups to Worker Groups Launch Templates](#transitioning-from-worker-groups-to-worker-groups-launch-templates)
     - [EKS node groups](#eks-node-groups)
       - [Custom EKS node groups](#custom-eks-node-groups)
+    - [EBS CSI DRIVER](#ebs-csi-driver)
     - [AWS Auth](#aws-auth)
       - [`map_users`](#map_users)
       - [`map_roles`](#map_roles)
@@ -547,6 +548,18 @@ module "eks-jx" {
 
 :warning: **Note**: EKS node groups are supported in kubernetes v1.14+ and platform version eks.3
 
+### EBS CSI Driver
+In version 1.23 the Kubernetes in-tree to container storage interface (CSI) volume migration feature is enabled. This feature enables the replacement of existing Kubernetes in-tree storage plugins for Amazon EBS with a corresponding Amazon EBS CSI driver. If you use Amazon EBS volumes install the Amazon EBS CSI driver in your cluster before you update/create your cluster to/in version 1.23 [Kubernetes 1.23 Ref](https://docs.aws.amazon.com/eks/latest/userguide/kubernetes-versions.html#kubernetes-1.23).
+
+An add-on is software that provides supporting operational capabilities to Kubernetes applications, but is not specific to the application. This includes software like observability agents or Kubernetes drivers that allow the cluster to interact with underlying AWS resources for networking, compute, and storage. [EKS Addons Guide](https://docs.aws.amazon.com/eks/latest/userguide/eks-add-ons.html)
+
+To enable the EBS CSI Driver (aws-ebs-csi-driver) set variables `enable_ebs_addon`and `create_addon_role` both  to true. The version of the driver addon is defined in the string variable `ebs_addon_version`
+To determine what versions of EBS CSI driver are supported use the command:
+```
+aws eks describe-addon-versions --addon-name "aws-ebs-csi-driver" | jq -r '.addons[].addonVersions[].addonVersion'
+```
+
+:warning: **Note**: It is imperative that you export the environment variable `AWS_REGION` with the appropriate region value (i.e. us-west-2). 
 ### AWS Auth
 
 When running EKS, authentication for the cluster is controlled by a `configmap` called `aws-auth`. By default, that should look something like this:
@@ -714,6 +727,7 @@ Each example generates a valid _jx-requirements.yml_ file that can be used to bo
 | <a name="input_cluster_in_private_subnet"></a> [cluster\_in\_private\_subnet](#input\_cluster\_in\_private\_subnet) | Flag to enable installation of cluster on private subnets | `bool` | `false` | no |
 | <a name="input_cluster_name"></a> [cluster\_name](#input\_cluster\_name) | Variable to provide your desired name for the cluster. The script will create a random name if this is empty | `string` | `""` | no |
 | <a name="input_cluster_version"></a> [cluster\_version](#input\_cluster\_version) | Kubernetes version to use for the EKS cluster. | `string` | n/a | yes |
+| <a name="input_create_addon_role"></a> [create\_addon\_role](#input\_create\_addon\_role) | Flag to control ebscsi addon iam role creation | `bool` | `true` | no |
 | <a name="input_create_and_configure_subdomain"></a> [create\_and\_configure\_subdomain](#input\_create\_and\_configure\_subdomain) | Flag to create an NS record set for the subdomain in the apex domain's Hosted Zone | `bool` | `false` | no |
 | <a name="input_create_asm_role"></a> [create\_asm\_role](#input\_create\_asm\_role) | Flag to control AWS Secrets Manager iam roles creation | `bool` | `false` | no |
 | <a name="input_create_autoscaler_role"></a> [create\_autoscaler\_role](#input\_create\_autoscaler\_role) | Flag to control cluster autoscaler iam role creation | `bool` | `true` | no |
@@ -732,7 +746,9 @@ Each example generates a valid _jx-requirements.yml_ file that can be used to bo
 | <a name="input_create_vpc"></a> [create\_vpc](#input\_create\_vpc) | Controls if VPC and related resources should be created. If you have an existing vpc for jx, set it to false | `bool` | `true` | no |
 | <a name="input_desired_node_count"></a> [desired\_node\_count](#input\_desired\_node\_count) | The number of worker nodes to use for the cluster | `number` | `3` | no |
 | <a name="input_enable_acl"></a> [enable\_acl](#input\_enable\_acl) | Flag to enable ACL along with bucket ownership controls for S3 storage | `bool` | `false` | no |
+| <a name="input_ebs_addon_version"></a> [ebs\_addon\_version](#input\_ebs\_addon\_version) | EBS Addon aws-ebs-csi-driver Version | `string` |  `"v1.21.0-ebsbuild.1"`| no |
 | <a name="input_enable_backup"></a> [enable\_backup](#input\_enable\_backup) | Whether or not Velero backups should be enabled | `bool` | `false` | no |
+| <a name="input_enable_ebs_addon"></a> [enable\_ebs\_addon](#input\_enable\_ebs\_addon) | Flag to enable or disable EBS CSI driver addon | `bool` | `false` | no |
 | <a name="input_enable_external_dns"></a> [enable\_external\_dns](#input\_enable\_external\_dns) | Flag to enable or disable External DNS in the final `jx-requirements.yml` file | `bool` | `false` | no |
 | <a name="input_enable_key_name"></a> [enable\_key\_name](#input\_enable\_key\_name) | Flag to enable ssh key pair name | `bool` | `false` | no |
 | <a name="input_enable_key_rotation"></a> [enable\_key\_rotation](#input\_enable\_key\_rotation) | Flag to enable kms key rotation | `bool` | `true` | no |

main.tf

@@ -90,6 +90,7 @@ module "cluster" {
   create_ctrlb_role                     = var.create_ctrlb_role
   create_exdns_role                     = var.create_exdns_role
   create_pipeline_vis_role              = var.create_pipeline_vis_role
+  create_addon_role                     = var.create_addon_role
   create_asm_role                       = var.create_asm_role
   create_ssm_role                       = var.create_ssm_role
   create_tekton_role                    = var.create_tekton_role
@@ -105,6 +106,8 @@ module "cluster" {
   use_asm                               = var.use_asm
   boot_iam_role                         = "${var.asm_role}${var.boot_iam_role}"
   enable_acl                            = var.enable_acl
+  enable_ebs_addon                      = var.enable_ebs_addon
+  ebs_addon_version                     = var.ebs_addon_version
 }
 
 // ----------------------------------------------------------------------------

modules/cluster/irsa.tf

@@ -469,3 +469,22 @@ module "iam_assumable_role_secrets-system-manager" {
   role_policy_arns              = [var.create_ssm_role ? aws_iam_policy.system-manager[0].arn : ""]
   oidc_fully_qualified_subjects = ["system:serviceaccount:${local.secret-infra-namespace}:kubernetes-external-secrets"]
 }
+
+// ----------------------------------------------------------------------------
+// EBS CSI Driver Addon
+// Terraform submodule IAM Role for Service Accounts in EKS
+// ----------------------------------------------------------------------------
+
+module "ebs_csi_irsa_role" {
+  source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks"
+  version                       = "~> v5.30.1"
+  role_name                     = "${local.cluster_trunc}-ebscsi-addon"
+  create_role                   = var.create_addon_role
+  attach_ebs_csi_policy         = true   // Attaches AmazonEBSCSIDriverPolicy
+  oidc_providers = {
+     main  = {
+        provider_arn               = module.eks.oidc_provider_arn
+        namespace_service_accounts = ["kube-system:ebs-csi-controller-sa"]
+     }
+  }
+}

modules/cluster/main.tf

@@ -217,3 +217,16 @@ resource "kubernetes_config_map" "jenkins_x_requirements" {
     module.eks
   ]
 }
+// ----------------------------------------------------------------------------
+// Include aws-ebs-csi-driver addon if enabled
+// ----------------------------------------------------------------------------
+
+resource "aws_eks_addon" "ebs_addon" {
+  count             = var.enable_ebs_addon ? 1 : 0
+  cluster_name      = var.cluster_name
+  addon_name        = "aws-ebs-csi-driver"
+  addon_version     = var.ebs_addon_version
+  resolve_conflicts = "OVERWRITE"
+  service_account_role_arn = module.ebs_csi_irsa_role.iam_role_arn
+}
+

modules/cluster/variables.tf

@@ -447,3 +447,21 @@ variable "enable_acl" {
   description = "Flag to enable ACL instead of bucket ownership for S3 storage"
   type        = bool
 }
+
+variable "create_addon_role" {
+  description = "Flag to control addon iam roles creation"
+  type        = bool
+  default     = false
+} 
+
+variable "enable_ebs_addon" {
+  description = "Flag to enable or disable EBS CSI driver addon"
+  type        = bool
+  default     = false
+}
+
+variable "ebs_addon_version" {
+  description = "EBS CSI driver addon (aws-ebs-csi-driver) version"
+  type = string
+  default = "v1.21.0-eksbuild.1"
+}

outputs.tf

@@ -100,6 +100,7 @@ output "cluster_asm_iam_role" {
 output "cluster_ssm_iam_role" {
   value       = module.cluster.cluster_ssm_iam_role
   description = "The IAM Role that the External Secrets pod will assume to authenticate (Parameter Store)"
+
 }
 
 // ----------------------------------------------------------------------------

test/terraform_eks_test.go

@@ -105,6 +105,12 @@ func TestTerraformEksJX(t *testing.T) {
 	})
 	assert.NoError(t, err)
 
+	addRole := terraform.Output(t, tfOptions, "ebscsi_addon_iam_role")
+	_, err = iamClient.GetRole(context.TODO(), &iam.GetRoleInput{
+		RoleName: aws.String(addRole),
+	})
+	assert.NoError(t, err)
+
 	// Vault
 	vaultBucket := terraform.Output(t, tfOptions, "vault_unseal_bucket")
 	aws2.AssertS3BucketExists(t, region, vaultBucket)

variables.tf

@@ -653,3 +653,21 @@ variable "enable_acl" {
   type        = bool
   default     = false
 }
+
+variable "create_addon_role" {
+  description = "Flag to control addon iam roles creation"
+  type        = bool
+  default     = false
+}
+
+variable "enable_ebs_addon" {
+  description = "Flag to enable or disable EBS CSI driver addon"
+  type        = bool
+  default     = false
+}
+
+variable "ebs_addon_version" {
+  description = "EBS CSI driver addon (aws-ebs-csi-driver) version"
+  type = string
+  default = "v1.21.0-eksbuild.1"
+}