feat(logging): add support for publishing logs to CloudWatch (#45)

* enable management of logging

* pre-commit udpates

---------

Signed-off-by: Steve Teuber <steve.teuber@idealo.de>
Co-authored-by: mstewart7 <mstewart@tomonetworks.com>
Co-authored-by: Steve Teuber <steve.teuber@idealo.de>

README.md

@@ -88,6 +88,8 @@ Here is a working example of using this Terraform module:
 
 | Name | Type |
 |------|------|
+| [aws_cloudwatch_log_group.opensearch](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource |
+| [aws_cloudwatch_log_resource_policy.allow_logging](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_resource_policy) | resource |
 | [aws_elasticsearch_domain.opensearch](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/elasticsearch_domain) | resource |
 | [aws_elasticsearch_domain_saml_options.opensearch](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/elasticsearch_domain_saml_options) | resource |
 | [aws_iam_service_linked_role.es](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_service_linked_role) | resource |
@@ -101,6 +103,7 @@ Here is a working example of using this Terraform module:
 | [elasticsearch_opensearch_roles_mapping.role_mapping](https://registry.terraform.io/providers/phillbaker/elasticsearch/latest/docs/resources/opensearch_roles_mapping) | resource |
 | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source |
 | [aws_iam_policy_document.access_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
+| [aws_iam_policy_document.allow_logging](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
 | [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source |
 | [aws_route53_zone.opensearch](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/route53_zone) | data source |
 
@@ -140,6 +143,7 @@ Here is a working example of using this Terraform module:
 | <a name="input_indices"></a> [indices](#input\_indices) | A map of all indices to create. | `map(any)` | `{}` | no |
 | <a name="input_ism_policies"></a> [ism\_policies](#input\_ism\_policies) | A map of all ISM policies to create. | `map(any)` | `{}` | no |
 | <a name="input_ism_policy_files"></a> [ism\_policy\_files](#input\_ism\_policy\_files) | A set of all ISM policy files to create. | `set(string)` | `[]` | no |
+| <a name="input_log_streams_enabled"></a> [log\_streams\_enabled](#input\_log\_streams\_enabled) | Configuration for which log streams to enable sending logs to CloudWatch. | `map(string)` | <pre>{<br>  "AUDIT_LOGS": "false",<br>  "ES_APPLICATION_LOGS": "false",<br>  "INDEX_SLOW_LOGS": "false",<br>  "SEARCH_SLOW_LOGS": "false"<br>}</pre> | no |
 | <a name="input_master_instance_count"></a> [master\_instance\_count](#input\_master\_instance\_count) | The number of dedicated master nodes in the cluster. | `number` | `3` | no |
 | <a name="input_master_instance_enabled"></a> [master\_instance\_enabled](#input\_master\_instance\_enabled) | Indicates whether dedicated master nodes are enabled for the cluster. | `bool` | `true` | no |
 | <a name="input_master_instance_type"></a> [master\_instance\_type](#input\_master\_instance\_type) | The type of EC2 instances to run for each master node. A list of available instance types can you find at https://aws.amazon.com/en/opensearch-service/pricing/#On-Demand_instance_pricing | `string` | `"r6gd.large.elasticsearch"` | no |

cloudwatch.tf

@@ -0,0 +1,16 @@
+locals {
+  log_prefix = "/aws/OpenSearchService/domains/${var.cluster_name}"
+}
+
+resource "aws_cloudwatch_log_group" "opensearch" {
+  for_each = { for k, v in var.log_streams_enabled : k => v if v == "true" }
+
+  name              = "${local.log_prefix}/${each.key}"
+  retention_in_days = 14
+}
+
+resource "aws_cloudwatch_log_resource_policy" "allow_logging" {
+  count           = anytrue(values(var.log_streams_enabled)) ? 1 : 0
+  policy_document = data.aws_iam_policy_document.allow_logging.json
+  policy_name     = "opensearch-${var.cluster_name}-logs"
+}

data.tf

@@ -19,3 +19,20 @@ data "aws_iam_policy_document" "access_policy" {
     }
   }
 }
+
+data "aws_iam_policy_document" "allow_logging" {
+  statement {
+    actions = [
+      "logs:CreateLogStream",
+      "logs:PutLogEvents",
+      "logs:PutLogEventsBatch",
+    ]
+
+    resources = ["arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:${local.log_prefix}/*:*"]
+
+    principals {
+      identifiers = ["es.amazonaws.com"]
+      type        = "Service"
+    }
+  }
+}

main.tf

@@ -114,9 +114,21 @@ resource "aws_elasticsearch_domain" "opensearch" {
     }
   }
 
+  dynamic "log_publishing_options" {
+    for_each = { for k, v in var.log_streams_enabled : k => v if v == "true" }
+    content {
+      log_type                 = log_publishing_options.key
+      enabled                  = tobool(log_publishing_options.value)
+      cloudwatch_log_group_arn = try(aws_cloudwatch_log_group.opensearch[log_publishing_options.key].arn, "")
+    }
+  }
+
   tags = var.tags
 
-  depends_on = [aws_iam_service_linked_role.es]
+  depends_on = [
+    aws_iam_service_linked_role.es,
+    aws_cloudwatch_log_group.opensearch
+  ]
 }
 
 resource "aws_elasticsearch_domain_saml_options" "opensearch" {

variables.tf

@@ -342,3 +342,14 @@ variable "auto_tune_enabled" {
   type        = bool
   default     = true
 }
+
+variable "log_streams_enabled" {
+  description = "Configuration for which log streams to enable sending logs to CloudWatch."
+  type        = map(string)
+  default = {
+    "INDEX_SLOW_LOGS"     = "false"
+    "SEARCH_SLOW_LOGS"    = "false"
+    "ES_APPLICATION_LOGS" = "false"
+    "AUDIT_LOGS"          = "false"
+  }
+}