AWS Elastic Kubernetes Service Terraform module

Terraform module which creates EKS Cluster and dependent resources on AWS.

Notice

This module install several Helm charts with limited inputs for their configuration, in order to keep it easy and simple. For a more accurate configuration we recommend to read their documentation and make your own installation of your desired Helm charts:

• Metrics Server Helm chart
• Ingress NGINX Helm Chart
• Cluster Autoscaler Helm Chart
• Cert Manager Helm Chart
• Kube Prometheus Stack Helm Chart
• Loki Distributed Helm Chart
• Fluent Bit Helm Chart
• Tempo Distributed Helm Chart
• Grafana Helm Chart

Usage

Terraform required version >= 0.14.8

Elastic Kubernetes Service

EKS Cluster with ELB:

locals {
  configmap_roles = [
    {
      "role_arn" = aws_iam_role.jobs_runner.arn 
      "k8s_user" = "jobs-runner"
      "k8s_groups" = [
        "system:masters"
        ]
      }
    ]

  configmap_users = [
    {
      "user_arn" = "arn:aws:iam::123456789123:user/demo"
      "k8s_user" = "demo"
      "k8s_groups" = [
        "system:masters",
        "system:developers"
        ]
      }
    ]

  prometheus_additional_scrape_configs = file("${path.root}/${var.prometheus_additional_scrape_configs_path}")
}


module "eks_main" {
  source                                      = "github.com/nimbux911/terraform-aws-eks.git"
  environment                                 = "dev"
  cluster_name                                = "dev-eks-main"
  cluster_version                             = "1.23"
  vpc_id                                      = "vpc-abcd1234"
  subnets_ids                                 = ["subnet-abc1234", "subnet-efgh5678"]
  eks_api_private                             = true
  enabled_cluster_log_types                   = ["api", "audit", "authenticator", "controllerManager", "scheduler"]
  aws_auth_ignore_changes                     = false
  add_configmap_roles                         = local.configmap_roles
  target_group_arns                           = local.tg_arns
  eks_tags                                    = var.eks_tags
  health_check_type                           = "ELB"  

  # Example for ingress-nginx log format in logfmt using real source ip as client_ip
  ingress_custom_configuration                = "log-format-upstream: timestamp=$time_iso8601 client_ip=$http_x_forwarded_for method=$request_method uri=$request_uri status=$status http_user_agent=$http_user_agent request_length=$request_length request_time=$request_time proxy_upstream_name=$proxy_upstream_name upstream_addr=$upstream_addr upstream_response_length=$upstream_response_length upstream_response_time=$upstream_response_time upstream_status=$upstream_status req_id=$req_id" 

  managed_node_groups = [
    { 
      name    = "monitoring-${var.cluster_name}"
      values  = {
        ami_id            = var.eks_worker_ami_id,
        instance_type     = "m6a.large",
        asg_min           = 3,
        asg_max           = 4,
        subnets_ids       = ["subnet-abc1234", "subnet-efgh5678"],
        volume_type       = "gp3",
        volume_size       = 100,
        volume_iops       = 4000,
        k8s_labels        = {
          nodegroup       = "monitoring-${var.cluster_name}"
        }
      }
    }
  ]

  custom_node_groups = [
    {
      name    = "${var.environment}-${var.cluster_name}"
      values  = {
        ami_id            = var.eks_worker_ami_id,
        instance_type     = "t3.medium",
        asg_min           = 4,
        asg_max           = 8,
        subnets_ids       = ["subnet-abc1234", "subnet-efgh5678"],
        volume_type       = "gp2",
        volume_size       = 100,
        asg_tags          = var.asg_tags
        k8s_labels        = {
          nodegroup       = "${var.environment}-${var.cluster_name}"
        spot_nodes_enabled = true, // Just for custom node groups on Launch templates: https://docs.aws.amazon.com/eks/latest/APIReference/API_LaunchTemplateSpecification.html
        spot_options       = {
          max_price  = "0.0416" # t3.medium on-demand price
        }
      }
    }
  ]

  helm_ingress_nginx_enabled           = true 
  helm_cluster_autoscaler_enabled      = true
  cluster_autoscaler_extra_helm_values = file("${path.root}/resources/helm/values/cluster-autoscaler.yaml")
  helm_metrics_server_enabled          = true 
  helm_cert_manager_enabled            = true
  ingress_node_affinity                = {
    enabled     = true,
    label_key   = "nodegroup",
    label_value = "${var.env}-eks-spot"
  }

  create_ebs_csi_role = true

  eks_addons = {
    vpc-cni = {
      version              = "v1.12.6-eksbuild.1"
      configuration_values = {
        env = {
          ENABLE_PREFIX_DELEGATION = "true"
        }
      }
    },
    coredns = {
      version = "v1.8.7-eksbuild.4"
    },
    kube-proxy = {
      version = "v1.22.17-eksbuild.2"
    },
    aws-ebs-csi-driver = {
      version = "v1.17.0-eksbuild.1"
      service_account_role_arn = "arn:aws:iam::${var.current_account_id}:role/test-eks-main-eks-ebs-csi-controller"
    }
  }

# ================== loki-distributed ================= #
  helm_loki_enabled                     = true
  loki_storage_s3_bucket                = "my-bucket-loki-logs"
  loki_s3_bucket_region                 = "us-east-1"
  loki_ingester_replicas                = 3
  loki_ingester_node_selector           = { "eks\\.amazonaws\\.com/nodegroup" = "monitoring-${var.cluster_name}" }
  loki_distributor_min_replicas         = 2
  loki_distributor_node_selector        = { "eks\\.amazonaws\\.com/nodegroup" = "monitoring-${var.cluster_name}" }
  loki_distributor_max_replicas         = 4
  loki_querier_min_replicas             = 2
  loki_querier_max_replicas             = 4
  loki_querier_node_selector            = { "eks\\.amazonaws\\.com/nodegroup" = "monitoring-${var.cluster_name}" }
  loki_query_frontend_min_replicas      = 2
  loki_query_frontend_max_replicas      = 4
  loki_query_frontend_node_selector     = { "eks\\.amazonaws\\.com/nodegroup" = "monitoring-${var.cluster_name}" }
  loki_gateway_enabled                  = true
  loki_gateway_min_replicas             = 2
  loki_gateway_max_replicas             = 4
  loki_gateway_node_selector            = { "eks\\.amazonaws\\.com/nodegroup" = "monitoring-${var.cluster_name}" }
  loki_gateway_ingress_enabled          = true
  loki_gateway_ingress_host             = "loki.example.com"
  loki_compactor_enabled                = true
  loki_compactor_node_selector          = { "eks\\.amazonaws\\.com/nodegroup" = "monitoring-${var.cluster_name}" }
  loki_index_gateway_enabled            = true
  loki_index_gateway_replicas           = 1
  loki_index_gateway_node_selector      = { "eks\\.amazonaws\\.com/nodegroup" = "monitoring-${var.cluster_name}" }

# ================== fluent-bit ================== #
  helm_fluent_bit_enabled = true

# ================== prometheus ================== #
  helm_prometheus_enabled               = true
  prometheus_replicas                   = 2
  prometheus_ingress_enabled            = true
  prometheus_ingress_host               = "prometheus.example.com"
  prometheus_requests_cpu               = "200m"
  prometheus_requests_memory            = "1024Mi"
  prometheus_limits_cpu                 = "500m"
  prometheus_limits_memory              = "2048Mi"
  prometheus_node_selector              = { "eks\\.amazonaws\\.com/nodegroup" = "monitoring-${var.cluster_name}" }
  prometheus_additional_scrape_configs  = local.prometheus_additional_scrape_configs

# ================== tempo ================== #
  helm_tempo_enabled            = true
  tempo_storage_s3_bucket       = "my-bucket-tempo-traces"
  tempo_s3_bucket_region        = "us-east-1"
  tempo_gateway_enabled         = true
  tempo_gateway_ingress_enabled = true
  tempo_gateway_ingress_host    = "tempo.example.com"

# open-telemetry
  k8s_opentelemetry_enabled = true

# =================== grafana ================== #
  helm_grafana_enabled    = true
  grafana_ingress_enabled = true
  grafana_ingress_host    = "grafana.example.com"


}

Inputs

Name	Description	Type	Default	Required
environment	Environment name of the resources.	string	""	yes
cluster_name	Cluster name	string	""	yes
cluster_version	Kubernetes version of the cluster.	string	""	yes
managed_node_groups	AWS managed node groups configurations	object(...)	null	no
custom_node_groups	Custom node groups configurations	object(...)	null	no
k8s_auth_api	Kubernetes authentication API for Terraform providers.	string	client.authentication.k8s.io/v1beta1	no
vpc_id	VPC ID where cluster will be deployed.	string	""	yes
subnets_ids	Subnets ids from the VPC ID where the workers will be deployed. They must be, at least, from 2 differents AZs.	list[string]	[]	yes
max_pods_per_node	Max pods per Kubernetes worker node.	string	"100"	no
target_group_arns	ARNs of the target groups for using the worker nodes behind of ELB	list[string]	[]	no
health_check_type	Health check type for the worker nodes.	string	"EC2"	no
on_demand_percentage_above_base_capacity	Percentage split between on-demand and Spot instances above the base on-demand capacity.	number	100	no
spot_allocation_strategy		string	"capacity-optimized"	no
spot_instance_pools	Number of Spot pools per availability zone to allocate capacity. EC2 Auto Scaling selects the cheapest Spot pools and evenly allocates Spot capacity across the number of Spot pools that you specify.	number	2	no
eks_tags	Tags to add to all resources except the autoscaling group.	map	{}	no
eks_api_private	Defines it the Kubernetes API will be private or public.	bool	false	no
eks_addons	Adds EKS addons.	map(map(string))	{}	no
enable_irsa	Determines whether to create an OpenID Connect Provider for EKS to enable IRSA.	bool	true	no
openid_connect_audiences	List of OpenID Connect audience client IDs to add to the IRSA provider.	list[string]	[]	no
custom_oidc_thumbprints	Additional list of server certificate thumbprints for the OpenID Connect (OIDC) identity provider's server certificate(s).	list[string]	[]	no
create_ebs_csi_role	Indicates whether or not to create an IAM assumable role with oidc for EKS ebs-csi-controller. If true, the role will be created with the following name: <cluster_name>-ebs-csi-controller	bool	false	no
add_configmap_roles	List of maps with the information of the IAM roles to be added to aws-auth configmap.	list[map]	[]	no
add_configmap_users	List of maps with the information of the IAM users to be added to aws-auth configmap.	list[map]	[]	no
aws_auth_ignore_changes	Set if aws-auth configmap will be managed by Terraform or ignored.	bool	true	no
eks_worker_max_pods_enabled	Enable --max-pods flag in workers bootstrap	bool	false	no
eks_worker_ssh_cidrs	Add SSH ingress rule to eks workers	list	[]	no
enabled_cluster_log_types	Enable CloudWatch Logs for control plane components	list[string]	[]	no
helm_ingress_nginx_enabled	Set if ingress-nginx Helm chart will be installed on the cluster.	bool	false	no
ingress_chart_version	Set the version for the chart	string	4.0.18	no
ingress_custom_configuration	Add custom configuration options (see example above in module call inputs and https://github.com/kubernetes/ingress-nginx/blob/main/charts/ingress-nginx/values.yaml#L52)	string	null	no
ingress_http_nodeport	Set port for ingress http nodePort	int	32080	no
ingress_https_nodeport	Set port for ingress https nodePort	int	32443	no
ingress_https_traffic_enabled	Set https traffic for ingress	bool	false	no
ingress_node_affinity	Set nodeAffinity for ingress	map	{ enabled = false, label_key = null, label_value = null}	no
ingress_requests_cpu	Set how much cpu will be assigned to the request	string	100m	no
ingress_requests_memory	Set how much memory will be assigned to the request	string	90Mi	no
ingress_service_monitor_enabled	Enable serviceMonitor for ingress-nginx helm chart	bool	false	no
ingress_priority_class_name	allows you to set a priority class	string	""	no
ingress_replicacount	Minimum Replicas count of ingress	number	"1"	no
helm_ingress_nginx_additional_enabled	Set if additional ingress-nginx Helm chart will be installed on the cluster.	bool	false	no
ingress_additional_chart_version	Set the version for the chart	string	4.0.18	no
ingress_additional_http_nodeport	Set port for additional ingress http nodePort	int	31080	no
ingress_additional_https_nodeport	Set port for additional ingress https nodePort	int	31443	no
ingress_additional_https_traffic_enabled	Set https traffic for additional ingress	bool	false	no
ingress_additional_requests_cpu	Set how much cpu will be assigned to the request	string	100m	no
ingress_additional_requests_memory	Set how much memory will be assigned to the request	string	90Mi	no
ingress_additional_priority_class_name	allows you to set a priority class	string	""	no
ingress_additional_replicacount	Minimum Replicas count of ingress additional	number	"1"	no
helm_cluster_autoscaler_enabled	Set if cluster-autoscaler Helm chart will be installed on the cluster.	bool	false	no
cluster_autoscaler_chart_version	Set the version for the chart	string	9.16.1	no
cluster_autoscaler_priority_class_name	allows you to set a priority class	string	""	no
cluster_autoscaler_extra_helm_values	Raw YAML containing additional cluster-autoscaler Helm values	string	""	no
helm_metrics_server_enabled	Set if metrics-server Helm chart will be installed on the cluster.	bool	false	no
metrics_server_chart_version	Set the version for the chart	string	1.6.1	no
metrics_server_priority_class_name	allows you to set a priority class	string	""	no
helm_cert_manager_enabled	Set if cert-manager helm chart will be installed on the cluster	bool	false	no
cert_manager_chart_version	Set the version for the chart	string	6.0.5	no
cert_manager_priority_class_name	allows you to set a priority class	string	""	no
helm_loki_enabled	Set if loki-stack Helm chart will be installed on the cluster.	bool	false	no
loki_chart_version	Set the version for the chart	string	0.48.3	no
loki_storage_s3_bucket	s3 bucket for loki logs	string	""	yes
loki_s3_bucket_region	s3 bucket for loki logs region	string	""	yes
loki_logs_retention_enabled	Enable logs retention. If s3 storage never stop growing	bool	false	no
loki_logs_retention	Set logs retention period	string	744h	no
loki_ingester_replicas	Loki ingester replicas	int	1	no
loki_ingester_node_selector	Loki ingester nodeSelector	map{}	null	no
loki_ingester_storage_class	storageClass for ingesters pv	string	gp2	no
loki_ingester_storage_size	size of ingesters pv	string	10Gi	no
loki_ingester_requests_cpu	resources config for kubernetes pod	string	null	no
loki_ingester_requests_memory	resources config for kubernetes pod	string	null	no
loki_ingester_limits_cpu	resources config for kubernetes pod	string	null	no
loki_ingester_limits_memory	resources config for kubernetes pod	string	null	no
loki_distributor_node_selector	Loki distributor nodeSelector	map{}	null	no
loki_distributor_min_replicas	loki distributor hpa min replicas	int	1	no
loki_distributor_requests_cpu	resources config for kubernetes pod	string	null	no
loki_distributor_requests_memory	resources config for kubernetes pod	string	null	no
loki_distributor_limits_cpu	resources config for kubernetes pod	string	null	no
loki_distributor_limits_memory	resources config for kubernetes pod	string	null	no
loki_distributor_max_replicas	loki distributor hpa max replicas	int	1	no
loki_querier_node_selector	Loki querier nodeSelector	map{}	null	no
loki_querier_min_replicas	loki querier hpa min replicas	int	1	no
loki_querier_max_replicas	loki querier hpa max replicas	int	1	no
loki_querier_requests_cpu	resources config for kubernetes pod	string	null	no
loki_querier_requests_memory	resources config for kubernetes pod	string	null	no
loki_querier_limits_cpu	resources config for kubernetes pod	string	null	no
loki_querier_limits_memory	resources config for kubernetes pod	string	null	no
loki_query_frontend_node_selector	Loki query-frontend nodeSelector	map{}	null	no
loki_query_frontend_min_replicas	loki query-frontend hpa min replicas	int	1	no
loki_query_frontend_max_replicas	loki query-frontend hpa max replicas	int	1	no
loki_query_frontend_requests_cpu	resources config for kubernetes pod	string	null	no
loki_query_frontend_requests_memory	resources config for kubernetes pod	string	null	no
loki_query_frontend_limits_cpu	resources config for kubernetes pod	string	null	no
loki_query_frontend_limits_memory	resources config for kubernetes pod	string	null	no
loki_max_query_length	The limit to length of chunk store queries	string	721h	no
loki_gateway_enabled	Enable loki gateway	bool	false	no
loki_gateway_node_selector	Loki gateway nodeSelector	map{}	null	no
loki_gateway_min_replicas	loki gateway hpa min replicas	int	1	no
loki_gateway_max_replicas	loki gateway hpa max replicas	int	1	no
loki_gateway_ingress_enabled	Enable ingress for loki gateway	bool	false	no
loki_gateway_ingress_host	Host for ingress rule	string	""	no
loki_gateway_ingress_path	Path for ingress rule	string	/	no
loki_gateway_ingress_path_type	Path type for ingress rule	string	Prefix	no
loki_gateway_ingress_class_name	Set ingress class name	string	nginx	no
loki_gateway_requests_cpu	resources config for kubernetes pod	string	null	no
loki_gateway_requests_memory	resources config for kubernetes pod	string	null	no
loki_gateway_limits_cpu	resources config for kubernetes pod	string	null	no
loki_gateway_limits_memory	resources config for kubernetes pod	string	null	no
loki_compactor_enabled	Enable loki compactor	bool	false	no
loki_compactor_node_selector	Loki compactor nodeSelector	map{}	null	no
loki_compactor_requests_cpu	resources config for kubernetes pod	string	null	no
loki_compactor_requests_memory	resources config for kubernetes pod	string	null	no
loki_compactor_limits_cpu	resources config for kubernetes pod	string	null	no
loki_compactor_limits_memory	resources config for kubernetes pod	string	null	no
loki_index_gateway_enabled	Enable loki index gateway	bool	false	no
loki_index_gateway_node_selector	Loki _index gateway nodeSelector	map{}	null	no
loki_index_gateway_replicas	Set loki index gateway replicas	int	1	no
loki_index_gateway_storage_class	storageClass for index gateway pv	string	gp2	no
loki_index_gateway_storage_size	storage size for index gateway pv	string	10Gi	no
loki_index_gateway_requests_cpu	resources config for kubernetes pod	string	null	no
loki_index_gateway_requests_memory	resources config for kubernetes pod	string	null	no
loki_index_gateway_limits_cpu	resources config for kubernetes pod	string	null	no
loki_index_gateway_limits_memory	resources config for kubernetes pod	string	null	no
loki_priority_class_name	allows you to set a priority class	string	""	no
helm_fluent_bit_enabled	install fluent-bit helm chart	bool	false	no
fluent_bit_chart_version	Set the version for the chart	string	0.19.24	no
fluent_bit_priority_class_name	allows you to set a priority class	string	""	no
k8s_opentelemetry_enabled	install opentelemetry manifests	bool	false	no
helm_prometheus_enabled	install kube-prometheus-stack helm chart	bool	false	no
prometheus_chart_version	Set the version for the chart	string	35.0.3	no
prometheus_node_selector	Prometheus components nodeSelector	map{}	null	no
prometheus_replicas	prometheus server replicas	int	1	no
prometheus_requests_cpu	resources config for kubernetes pod	string	null	no
prometheus_requests_memory	resources config for kubernetes pod	string	null	no
prometheus_limits_cpu	resources config for kubernetes pod	string	null	no
prometheus_limits_memory	resources config for kubernetes pod	string	null	no
prometheus_ingress_enabled	Enable ingress for prometheus server	bool	false	no
prometheus_ingress_host	Host for ingress rule	string	""	no
prometheus_ingress_path	Path for ingress rule	string	/	no
prometheus_ingress_path_type	Path type for ingress rule	string	Prefix	no
prometheus_ingress_class_name	Prometheus Ingress className	string	nginx	no
prometheus_storage_class_name	Prometheus storage className for pv	string	gp2	no
prometheus_storage_size	Prometheus storage size	string	20Gi	no
prometheus_metrics_retention	Prometheus metrics period retention	string	14d	no
prometheus_priority_class_name	allows you to set a priority class	string	""	no
prometheus_additional_scrape_configs	allows you to set a additional scrape config	string	""	no
helm_tempo_enabled	Install tempo-distributed helm chart	bool	false	no
tempo_chart_version	Set the version for the chart	string	0.17.1	no
tempo_compactor_requests_cpu	resources config for kubernetes pod	string	null	no
tempo_compactor_requests_memory	resources config for kubernetes pod	string	null	no
tempo_compactor_limits_cpu	resources config for kubernetes pod	string	null	no
tempo_compactor_limits_memory	resources config for kubernetes pod	string	null	no
tempo_distributor_requests_cpu	resources config for kubernetes pod	string	null	no
tempo_distributor_requests_memory	resources config for kubernetes pod	string	null	no
tempo_distributor_limits_cpu	resources config for kubernetes pod	string	null	no
tempo_distributor_limits_memory	resources config for kubernetes pod	string	null	no
tempo_storage_s3_bucket	s3 bucket for tempo traces	string	""	no
tempo_s3_bucket_region	s3 bucket regino for tempo traces	string	""	no
tempo_ingester_requests_cpu	resources config for kubernetes pod	string	null	no
tempo_ingester_requests_memory	resources config for kubernetes pod	string	null	no
tempo_ingester_limits_cpu	resources config for kubernetes pod	string	null	no
tempo_ingester_limits_memory	resources config for kubernetes pod	string	null	no
tempo_querier_requests_cpu	resources config for kubernetes pod	string	null	no
tempo_querier_requests_memory	resources config for kubernetes pod	string	null	no
tempo_querier_limits_cpu	resources config for kubernetes pod	string	null	no
tempo_querier_limits_memory	resources config for kubernetes pod	string	null	no
tempo_query_frontend_requests_cpu	resources config for kubernetes pod	string	null	no
tempo_query_frontend_requests_memory	resources config for kubernetes pod	string	null	no
tempo_query_frontend_limits_cpu	resources config for kubernetes pod	string	null	no
tempo_query_frontend_limits_memory	resources config for kubernetes pod	string	null	no
tempo_gateway_enabled	enable tempo gateway	bool	false	no
tempo_gateway_requests_cpu	resources config for kubernetes pod	string	null	no
tempo_gateway_requests_memory	resources config for kubernetes pod	string	null	no
tempo_gateway_limits_cpu	resources config for kubernetes pod	string	null	no
tempo_gateway_limits_memory	resources config for kubernetes pod	string	null	no
tempo_gateway_ingress_enabled	Enable ingress for tempo gateway	bool	false	no
tempo_gateway_ingress_host	Host for ingress rule	string	""	no
tempo_gateway_ingress_path	Path for ingress rule	string	/	no
tempo_ingress_path_type	Path type for ingress rule	string	Prefix	no
tempo_ingress_class_name	ingress className	string	nginx	no
tempo_priority_class_name	allows you to set a priority class	string	""	no
helm_grafana_enabled	install grafana helm chart	bool	false	no
grafana_chart_version	Set the version for the chart	string	6.45.0	no
grafana_ingress_enabled	Enable ingress for grafana	bool	false	no
grafana_ingress_host	Host for ingress rule	string	""	no
grafana_ingress_path	Path for ingress rule	string	/	no
grafana_ingress_path_type	Path type for ingress rule	string	Prefix	no
grafana_ingress_class_name	ingress className	string	nginx	no
grafana_persistence_enabled	Persistent volume	bool	false	no
grafana_priority_class_name	allows you to set a priority class	string	""	no
k8s_image_registry	Kubernetes image registry.	string	registry.k8s.io	no

Outputs

Name	Description
security_group_worker_arn	The ARN of the workers security group.
worker_role_arn	The ARN of the workers IAM Role.
worker_role_id	The ID of the workers IAM Role.
asg_name	Name of the of the workers Autoscaling Group.
eks_certificate_authority	Cluster's certificate authority.
eks_endpoint	Cluster's endpoint.
eks_managed_node_groups_autoscaling_group_names	List of the autoscaling group names created by EKS managed node groups.
oidc_provider	The OpenID Connect identity provider (issuer URL without leading https://).
oidc_provider_arn	The ARN of the OIDC Provider if enable_irsa = true.
cluster_tls_certificate_sha1_fingerprint	The SHA1 fingerprint of the public key of the cluster's certificate.
ebs_csi_iam_role_arn	The arn of the role created for ebs csi driver.