feat: Enable Prometheus sidecar in Cloud Run v2. (#253)

Co-authored-by: Bharath KKB <bharathkrishnakb@gmail.com>
Co-authored-by: abhishek kumar tiwari <abhiwa@google.com>

examples/v2_with_gmp/README.md

@@ -0,0 +1,64 @@
+# Cloud Run Service using v2 API and Prometheus Sidecar Example
+
+This example showcases the basic deployment of containerized applications with Prometheus Sidecar to provide observability  on Cloud Run and IAM policy for the service.
+
+The resources/services/activations/deletions that this example will create/trigger are:
+
+* Creates a Cloud Run service with provided name and container.
+* Add Prometheus Sidecar container to the Cloud Run instance.
+* Creates a Service Account capable of writing metrics to be used by Cloud Run Service.
+
+## Assumptions and Prerequisites
+
+This example assumes that below mentioned prerequisites are in place before consuming the example.
+
+* All required APIs are enabled in the GCP Project
+
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| project\_id | The project ID to deploy to | `string` | n/a | yes |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| observed\_generation | The generation of this Service currently serving traffic. |
+| project\_id | Project ID of the service |
+| service\_id | Unique Identifier for the created service with format projects/{{project}}/locations/{{location}}/services/{{name}} |
+| service\_location | Location in which the Cloud Run service was created |
+| service\_name | Name of the created service |
+| service\_uri | The URL on which the deployed service is available |
+| traffic\_statuses | Detailed status information for corresponding traffic targets. |
+
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+
+## Requirements
+
+These sections describe requirements for using this example.
+
+### Software
+
+* [Terraform](https://www.terraform.io/downloads.html) ~> v0.13+
+* [Terraform Provider for GCP](https://github.com/terraform-providers/terraform-provider-google) ~> v5.0+
+* [Terraform Provider for GCP Beta](https://github.com/terraform-providers/terraform-provider-google-beta) ~>
+  v5.0+
+
+### Service Account
+
+A service account can be used with required roles to execute this example:
+
+* Cloud Run Admin: `roles/run.admin`
+
+Know more about [Cloud Run Deployment Permissions](https://cloud.google.com/run/docs/reference/iam/roles#additional-configuration).
+
+The [Project Factory module](https://registry.terraform.io/modules/terraform-google-modules/project-factory/google/latest) and the
+[IAM module](https://registry.terraform.io/modules/terraform-google-modules/iam/google/latest) may be used in combination to provision a service account with the necessary roles applied.
+
+### APIs
+
+A project with the following APIs enabled must be used to host the main resource of this example:
+
+* Google Cloud Run: `run.googleapis.com`

examples/v2_with_gmp/main.tf

@@ -0,0 +1,31 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+module "cloud_run_v2" {
+  source  = "GoogleCloudPlatform/cloud-run/google//modules/v2"
+  version = "~> 0.14"
+
+  service_name = "ci-cloud-run-v2-gmp"
+  project_id   = var.project_id
+  location     = "us-central1"
+  containers = [
+    {
+      container_image = "us-docker.pkg.dev/cloudrun/container/hello"
+      container_name  = "hello-world"
+    }
+  ]
+  enable_prometheus_sidecar = true
+}

examples/v2_with_gmp/outputs.tf

@@ -0,0 +1,50 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+output "project_id" {
+  value       = module.cloud_run_v2.project_id
+  description = "Project ID of the service"
+}
+
+output "service_name" {
+  value       = module.cloud_run_v2.service_name
+  description = "Name of the created service"
+}
+
+output "service_uri" {
+  value       = module.cloud_run_v2.service_uri
+  description = "The URL on which the deployed service is available"
+}
+
+output "service_id" {
+  value       = module.cloud_run_v2.service_id
+  description = "Unique Identifier for the created service with format projects/{{project}}/locations/{{location}}/services/{{name}}"
+}
+
+output "service_location" {
+  value       = module.cloud_run_v2.location
+  description = "Location in which the Cloud Run service was created"
+}
+
+output "traffic_statuses" {
+  value       = module.cloud_run_v2.traffic_statuses
+  description = "Detailed status information for corresponding traffic targets."
+}
+
+output "observed_generation" {
+  value       = module.cloud_run_v2.observed_generation
+  description = "The generation of this Service currently serving traffic."
+}

examples/v2_with_gmp/variables.tf

@@ -0,0 +1,20 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+variable "project_id" {
+  description = "The project ID to deploy to"
+  type        = string
+}

metadata.yaml

@@ -62,6 +62,8 @@ spec:
         location: examples/simple_job_exec
       - name: v2
         location: examples/v2
+      - name: v2_with_gmp
+        location: examples/v2_with_gmp
   interfaces:
     variables:
       - name: project_id

modules/v2/README.md

@@ -41,10 +41,11 @@ Functional examples are included in the
 | binary\_authorization | Settings for the Binary Authorization feature. | <pre>object({<br>    breakglass_justification = optional(bool) # If present, indicates to use Breakglass using this justification. If useDefault is False, then it must be empty. For more information on breakglass, see https://cloud.google.com/binary-authorization/docs/using-breakglass<br>    use_default              = optional(bool) #If True, indicates to use the default project's binary authorization policy. If False, binary authorization will be disabled.<br>  })</pre> | `null` | no |
 | client | Arbitrary identifier for the API client and version identifier | <pre>object({<br>    name    = optional(string, null)<br>    version = optional(string, null)<br>  })</pre> | `{}` | no |
 | cloud\_run\_deletion\_protection | This field prevents Terraform from destroying or recreating the Cloud Run v2 Jobs and Services | `bool` | `true` | no |
-| containers | Map of container images for the service | <pre>list(object({<br>    container_name       = optional(string, null)<br>    container_image      = string<br>    working_dir          = optional(string, null)<br>    depends_on_container = optional(list(string), null)<br>    container_args       = optional(list(string), null)<br>    container_command    = optional(list(string), null)<br>    env_vars             = optional(map(string), {})<br>    env_secret_vars = optional(map(object({<br>      secret  = string<br>      version = string<br>    })), {})<br>    volume_mounts = optional(list(object({<br>      name       = string<br>      mount_path = string<br>    })), [])<br>    ports = optional(object({<br>      name           = optional(string, "http1")<br>      container_port = optional(number, 8080)<br>    }), {})<br>    resources = optional(object({<br>      limits = optional(object({<br>        cpu    = optional(string)<br>        memory = optional(string)<br>      }))<br>      cpu_idle          = optional(bool, true)<br>      startup_cpu_boost = optional(bool, false)<br>    }), {})<br>    startup_probe = optional(object({<br>      failure_threshold     = optional(number, null)<br>      initial_delay_seconds = optional(number, null)<br>      timeout_seconds       = optional(number, null)<br>      period_seconds        = optional(number, null)<br>      http_get = optional(object({<br>        path = optional(string)<br>        port = optional(string)<br>        http_headers = optional(list(object({<br>          name  = string<br>          value = string<br>        })), [])<br>      }), null)<br>      tcp_socket = optional(object({<br>        port = optional(number)<br>      }), null)<br>      grpc = optional(object({<br>        port    = optional(number)<br>        service = optional(string)<br>      }), null)<br>    }), null)<br>    liveness_probe = optional(object({<br>      failure_threshold     = optional(number, null)<br>      initial_delay_seconds = optional(number, null)<br>      timeout_seconds       = optional(number, null)<br>      period_seconds        = optional(number, null)<br>      http_get = optional(object({<br>        path = optional(string)<br>        port = optional(string)<br>        http_headers = optional(list(object({<br>          name  = string<br>          value = string<br>        })), null)<br>      }), null)<br>      grpc = optional(object({<br>        port    = optional(number)<br>        service = optional(string)<br>      }), null)<br>    }), null)<br>  }))</pre> | n/a | yes |
+| containers | Map of container images for the service | <pre>list(object({<br>    container_name       = optional(string, null)<br>    container_image      = string<br>    working_dir          = optional(string, null)<br>    depends_on_container = optional(list(string), null)<br>    container_args       = optional(list(string), null)<br>    container_command    = optional(list(string), null)<br>    env_vars             = optional(map(string), {})<br>    env_secret_vars = optional(map(object({<br>      secret  = string<br>      version = string<br>    })), {})<br>    volume_mounts = optional(list(object({<br>      name       = string<br>      mount_path = string<br>    })), [])<br>    ports = optional(object({<br>      name           = optional(string, "http1")<br>      container_port = optional(number, 8080)<br>    }), {})<br>    resources = optional(object({<br>      limits = optional(object({<br>        cpu    = optional(string)<br>        memory = optional(string)<br>      }))<br>      cpu_idle          = optional(bool, true)<br>      startup_cpu_boost = optional(bool, false)<br>    }), {})<br>    startup_probe = optional(object({<br>      failure_threshold     = optional(number, null)<br>      initial_delay_seconds = optional(number, null)<br>      timeout_seconds       = optional(number, null)<br>      period_seconds        = optional(number, null)<br>      http_get = optional(object({<br>        path = optional(string)<br>        port = optional(string)<br>        http_headers = optional(list(object({<br>          name  = string<br>          value = string<br>        })), [])<br>      }), null)<br>      tcp_socket = optional(object({<br>        port = optional(number)<br>      }), null)<br>      grpc = optional(object({<br>        port    = optional(number)<br>        service = optional(string)<br>      }), null)<br>    }), null)<br>    liveness_probe = optional(object({<br>      failure_threshold     = optional(number, null)<br>      initial_delay_seconds = optional(number, null)<br>      timeout_seconds       = optional(number, null)<br>      period_seconds        = optional(number, null)<br>      http_get = optional(object({<br>        path = optional(string)<br>        port = optional(string)<br>        http_headers = optional(list(object({<br>          name  = string<br>          value = string<br>        })), null)<br>      }), null)<br>      tcp_socket = optional(object({<br>        port = optional(number)<br>      }), null)<br>      grpc = optional(object({<br>        port    = optional(number)<br>        service = optional(string)<br>      }), null)<br>    }), null)<br>  }))</pre> | n/a | yes |
 | create\_service\_account | Create a new service account for cloud run service | `bool` | `true` | no |
 | custom\_audiences | One or more custom audiences that you want this service to support. Specify each custom audience as the full URL in a string. Refer https://cloud.google.com/run/docs/configuring/custom-audiences | `list(string)` | `null` | no |
 | description | Cloud Run service description. This field currently has a 512-character limit. | `string` | `null` | no |
+| enable\_prometheus\_sidecar | Enable Prometheus sidecar in Cloud Run instance. | `bool` | `false` | no |
 | encryption\_key | A reference to a customer managed encryption key (CMEK) to use to encrypt this container image. | `string` | `null` | no |
 | execution\_environment | The sandbox environment to host this Revision. | `string` | `"EXECUTION_ENVIRONMENT_GEN2"` | no |
 | ingress | Provides the ingress settings for this Service. On output, returns the currently observed ingress settings, or INGRESS\_TRAFFIC\_UNSPECIFIED if no revision is active. | `string` | `"INGRESS_TRAFFIC_ALL"` | no |

modules/v2/main.tf

@@ -32,6 +32,31 @@ locals {
     email  = google_service_account.sa[0].email,
     member = google_service_account.sa[0].member
   } : {}
+
+  ingress_container = try(
+    [for container in var.containers : container if length(try(container.ports, {})) > 0][0],
+    null
+  )
+  prometheus_sidecar_container = [{
+    container_name  = "collector"
+    container_image = "us-docker.pkg.dev/cloud-ops-agents-artifacts/cloud-run-gmp-sidecar/cloud-run-gmp-sidecar:1.1.1"
+    # Set default values for the sidecar container
+    ports                = {}
+    working_dir          = null
+    depends_on_container = [local.ingress_container.container_name]
+    container_args       = null
+    container_command    = null
+    env_vars             = {}
+    env_secret_vars      = {}
+    volume_mounts        = []
+    resources = {
+      cpu_idle          = true
+      startup_cpu_boost = false
+      limits            = {}
+    }
+    startup_probe  = []
+    liveness_probe = []
+  }]
 }
 
 resource "google_service_account" "sa" {
@@ -42,10 +67,14 @@ resource "google_service_account" "sa" {
 }
 
 resource "google_project_iam_member" "roles" {
-  for_each = toset(var.service_account_project_roles)
-  project  = var.project_id
-  role     = each.value
-  member   = "serviceAccount:${local.service_account}"
+  for_each = toset(distinct(concat(
+    var.service_account_project_roles,
+    var.enable_prometheus_sidecar ? ["roles/monitoring.metricWriter"] : []
+  )))
+
+  project = var.project_id
+  role    = each.value
+  member  = "serviceAccount:${local.service_account}"
 }
 
 resource "google_cloud_run_v2_service" "main" {
@@ -96,18 +125,21 @@ resource "google_cloud_run_v2_service" "main" {
     }
 
     dynamic "containers" {
-      for_each = var.containers
+      for_each = concat(var.containers,
+      var.enable_prometheus_sidecar ? local.prometheus_sidecar_container : [])
       content {
         name        = containers.value.container_name
         image       = containers.value.container_image
         command     = containers.value.container_command
         args        = containers.value.container_args
         working_dir = containers.value.working_dir
         depends_on  = containers.value.depends_on_container
-
-        ports {
-          name           = containers.value.ports["name"]
-          container_port = containers.value.ports["container_port"]
+        dynamic "ports" {
+          for_each = lookup(containers.value, "ports", {}) != {} ? [containers.value.ports] : []
+          content {
+            name           = ports.value["name"]
+            container_port = ports.value["container_port"]
+          }
         }
 
         resources {

modules/v2/metadata.display.yaml

@@ -46,6 +46,12 @@ spec:
         description:
           name: description
           title: Description
+        enable_prometheus_sidecar:
+          name: enable_prometheus_sidecar
+          title: Enable Prometheus Sidecar
+          altDefaults:
+            - type: ALTERNATE_TYPE_DC
+              value: true
         encryption_key:
           name: encryption_key
           title: Encryption Key

modules/v2/metadata.yaml

@@ -46,6 +46,8 @@ spec:
         location: examples/simple_job_exec
       - name: v2
         location: examples/v2
+      - name: v2_with_gmp
+        location: examples/v2_with_gmp
   interfaces:
     variables:
       - name: project_id
@@ -138,7 +140,7 @@ spec:
         connections:
           - source:
               source: github.com/terraform-google-modules/terraform-google-network//modules/vpc-serverless-connector-beta
-              version: ~> 9.1.0
+              version: ~> 9.1
             spec:
               outputExpr: connector_ids
               inputPath: connector
@@ -291,6 +293,9 @@ spec:
                     value = string
                   })), null)
                 }), null)
+                tcp_socket = optional(object({
+                  port = optional(number)
+                }), null)
                 grpc = optional(object({
                   port    = optional(number)
                   service = optional(string)
@@ -341,6 +346,10 @@ spec:
         description: Roles to grant to the newly created cloud run SA in specified project. Should be used with create_service_account set to true and no input for service_account
         varType: list(string)
         defaultValue: []
+      - name: enable_prometheus_sidecar
+        description: Enable Pormetheus sidecar in Cloud Run instance.
+        varType: bool
+        defaultValue: false
       - name: cloud_run_deletion_protection
         description: This field prevents Terraform from destroying or recreating the Cloud Run v2 Jobs and Services
         varType: bool

modules/v2/variables.tf

@@ -293,6 +293,9 @@ variable "containers" {
           value = string
         })), null)
       }), null)
+      tcp_socket = optional(object({
+        port = optional(number)
+      }), null)
       grpc = optional(object({
         port    = optional(number)
         service = optional(string)
@@ -326,3 +329,10 @@ variable "cloud_run_deletion_protection" {
   description = "This field prevents Terraform from destroying or recreating the Cloud Run v2 Jobs and Services"
   default     = true
 }
+
+// Prometheus sidecar
+variable "enable_prometheus_sidecar" {
+  type        = bool
+  description = "Enable Prometheus sidecar in Cloud Run instance."
+  default     = false
+}

test/setup/main.tf

@@ -34,6 +34,7 @@ module "project" {
     "cloudkms.googleapis.com",
     "iam.googleapis.com",
     "accesscontextmanager.googleapis.com",
-    "cloudbilling.googleapis.com"
+    "cloudbilling.googleapis.com",
+    "monitoring.googleapis.com"
   ]
 }