Merge pull request #4 from a5chin/feature/log2bq

BQ へのログ保存を行う

.dockerignore

@@ -0,0 +1,37 @@
+# Local .terraform directories
+**/.terraform/*
+
+# .tfstate files
+**/*.tfstate
+**/*.tfstate.*
+
+# Crash log files
+**/crash.log
+**/crash.*.log
+
+# Exclude all .tfvars files, which are likely to contain sensitive data, such as
+# password, private keys, and other secrets. These should not be part of version
+# control as they are data points which are potentially sensitive and subject
+# to change depending on the environment.
+**/*.tfvars
+**/*.tfvars.json
+
+# Ignore override files as they are usually used to override resources locally and so
+# are not checked in
+**/override.tf
+**/override.tf.json
+**/*_override.tf
+**/*_override.tf.json
+
+# Ignore transient lock info files created by terraform apply
+**/.terraform.tfstate.lock.info
+
+# Include override files you do wish to add to version control using negated pattern
+# !example_override.tf
+
+# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan
+# example: *tfplan*
+
+# Ignore CLI configuration files
+**/.terraformrc
+**/terraform.rc

.pre-commit-config.yaml

@@ -23,6 +23,20 @@ repos:
             "./modules/gcs2spanner/",
           ]
 
+  - repo: https://github.com/terraform-docs/terraform-docs
+    rev: "v0.18.0"
+    hooks:
+      - id: terraform-docs-go
+        name: terraform-docs-log2bq
+        args:
+          [
+            "markdown",
+            "table",
+            "--output-file",
+            "./README.md",
+            "./modules/log2bq/",
+          ]
+
   - repo: https://github.com/terraform-docs/terraform-docs
     rev: "v0.18.0"
     hooks:

environments/dev/main.tf

@@ -1,4 +1,4 @@
 locals {
-  project_id = "project=id"
+  project_id = "project_id"
   location   = "location"
 }

environments/prod/main.tf

@@ -1,4 +1,4 @@
 locals {
-  project_id = "project=id"
+  project_id = "project_id"
   location   = "location"
 }

environments/stg/main.tf

@@ -1,4 +1,4 @@
 locals {
-  project_id = "project=id"
+  project_id = "project_id"
   location   = "location"
 }

modules/log2bq/README.md

@@ -0,0 +1,44 @@
+<!-- BEGIN_TF_DOCS -->
+## Requirements
+
+| Name | Version |
+|------|---------|
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >=1.7 |
+| <a name="requirement_archive"></a> [archive](#requirement\_archive) | >=2.4.0 |
+| <a name="requirement_google"></a> [google](#requirement\_google) | >= 5.22.0 |
+| <a name="requirement_google-beta"></a> [google-beta](#requirement\_google-beta) | >= 5.22.0 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_google"></a> [google](#provider\_google) | >= 5.22.0 |
+
+## Modules
+
+No modules.
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [google_bigquery_dataset.main](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/bigquery_dataset) | resource |
+| [google_bigquery_table.main](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/bigquery_table) | resource |
+| [google_logging_project_sink.main](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/logging_project_sink) | resource |
+| [google_project_iam_member.main](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource |
+| [google_project_service.main](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_service) | resource |
+| [google_project.main](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_bigquery"></a> [bigquery](#input\_bigquery) | The bigquery settings | <pre>object({<br>    dataset         = string<br>    table           = string<br>    view            = string<br>    expiration_days = number<br>  })</pre> | n/a | yes |
+| <a name="input_logging"></a> [logging](#input\_logging) | n/a | <pre>object({<br>    target = string<br>    filter = string<br>  })</pre> | n/a | yes |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| <a name="output_enabled_apis"></a> [enabled\_apis](#output\_enabled\_apis) | Already enabled APIs list. |
+<!-- END_TF_DOCS -->

modules/log2bq/bigquery.tf

@@ -0,0 +1,23 @@
+resource "google_bigquery_dataset" "main" {
+  dataset_id                      = "cloud_logging_sinked_${var.logging.target}"
+  description                     = "Cloud Logging sinked dataset"
+  location                        = "US"
+  default_partition_expiration_ms = 60 * 1000 * 24 * var.bigquery.expiration_days
+  storage_billing_model           = "PHYSICAL"
+}
+
+resource "google_bigquery_table" "main" {
+  dataset_id = google_bigquery_dataset.main.dataset_id
+  table_id   = var.bigquery.view
+
+  view {
+    query          = <<EOF
+      SELECT *
+      FROM `${google_bigquery_dataset.main.dataset_id}.${var.bigquery.table}`
+      WEHRE ${var.logging.target}
+    EOF
+    use_legacy_sql = false
+  }
+
+  require_partition_filter = false
+}

modules/log2bq/logging.tf

@@ -0,0 +1,17 @@
+resource "google_logging_project_sink" "main" {
+  name        = "${var.logging.target}-to-bigquery-sink"
+  destination = "bigquery.googleapi.com/${google_bigquery_dataset.main.id}"
+  filter      = var.logging.filter
+
+  bigquery_options {
+    use_partitioned_tables = true
+  }
+
+  unique_writer_identity = true
+}
+
+resource "google_project_iam_member" "main" {
+  project = data.google_project.main.project_id
+  role    = "roles/bigquery.dataEditor"
+  member  = google_logging_project_sink.main.writer_identity
+}

modules/log2bq/main.tf

@@ -0,0 +1,16 @@
+locals {
+  apis = toset([
+    "bigquery.googleapis.com",
+    "logging.googleapis.com",
+  ])
+}
+
+data "google_project" "main" {}
+
+resource "google_project_service" "main" {
+  for_each = local.apis
+
+  project            = data.google_project.main.project_id
+  service            = each.value
+  disable_on_destroy = false
+}

modules/log2bq/outputs.tf

@@ -0,0 +1,4 @@
+output "enabled_apis" {
+  description = "Already enabled APIs list."
+  value       = [for api in google_project_service.main : api.service]
+}

modules/log2bq/provider.tf

@@ -0,0 +1,18 @@
+terraform {
+  required_version = ">=1.7"
+
+  required_providers {
+    archive = {
+      source  = "hashicorp/google"
+      version = ">=2.4.0"
+    }
+    google = {
+      source  = "hashicorp/google"
+      version = ">= 5.22.0"
+    }
+    google-beta = {
+      source  = "hashicorp/google"
+      version = ">= 5.22.0"
+    }
+  }
+}

modules/log2bq/variables.tf

@@ -0,0 +1,16 @@
+variable "bigquery" {
+  description = "The bigquery settings"
+  type = object({
+    dataset         = string
+    table           = string
+    view            = string
+    expiration_days = number
+  })
+}
+
+variable "logging" {
+  type = object({
+    target = string
+    filter = string
+  })
+}