merge documentation and terraform

README.md

@@ -9,7 +9,8 @@ This README is organized into four parts:  <br>
 - Part 1: [Deploying Tag Engine v2](#deploy) <br>
 - Part 2: [Testing your Setup with a User Account](#testa)  <br>
 - Part 3: [Testing your Setup with a Service Account](#testb)  <br>
-- Part 4: [What To Do Next](#next)  <br> 
+- Part 4: [Troubleshooting](#troubleshooting)  <br>
+- Part 5: [What To Do Next](#next)  <br> 
 
 ### <a name="deploy"></a> Part 1: Deploying Tag Engine v2
 
@@ -111,7 +112,7 @@ python create_template.py $TAG_ENGINE_PROJECT $TAG_ENGINE_REGION data_governance
 ```
 <br>
 
-2. Authorize a user account to use $TAG_CREATOR_SA and to invoke the Tag Engine Cloud Run service:
+2. Authorize a user account to use $TAG_CREATOR_SA and to invoke the Tag Engine API Cloud Run service:
 
 ```
 export USER_ACCOUNT="username@example.com"
@@ -120,7 +121,7 @@ gcloud iam service-accounts add-iam-policy-binding $TAG_CREATOR_SA \
     --member=user:$USER_ACCOUNT --role=roles/iam.serviceAccountUser 
 
 
-gcloud run services add-iam-policy-binding tag-engine \
+gcloud run services add-iam-policy-binding tag-engine-api \
     --member=user:$USER_ACCOUNT --role=roles/run.invoker \
     --region=$TAG_ENGINE_REGION	
 ```
@@ -233,7 +234,7 @@ e) View the job status:
 ```
 <br>
 
-2. Authorize a service account to use $TAG_CREATOR_SA and to invoke the Tag Engine Cloud Run service:
+2. Authorize a service account to use $TAG_CREATOR_SA and to invoke the Tag Engine API Cloud Run service:
 
 ```
 	export CLIENT_SA="tag-engine-client@<PROJECT>.iam.gserviceaccount.com"
@@ -242,7 +243,7 @@ e) View the job status:
 	    --member=serviceAccount:$CLIENT_SA --role=roles/iam.serviceAccountUser 
 
 
-	gcloud run services add-iam-policy-binding tag-engine \
+	gcloud run services add-iam-policy-binding tag-engine-api \
 	    --member=serviceAccount:$CLIENT_SA --role=roles/run.invoker \
 	    --region=$TAG_ENGINE_REGION	
 ```
@@ -260,7 +261,6 @@ e) View the job status:
 4. Generate an IAM token (aka Bearer token) for authenticating to the Tag Engine Cloud Run service:
 
 ```
-	gcloud auth login
 	export IAM_TOKEN=$(gcloud auth print-identity-token)
 ```
 <br>
@@ -332,7 +332,32 @@ e) View the job status:
    Open the Data Catalog UI and verify that your tag was successfully created. If not, open the Cloud Run logs and investigate the problem. 
 <br><br>
 
-### <a name="next"></a> Part 4: Next Steps
+### <a name="troubleshooting"></a> Part 4: Troubleshooting
+
+If you encounter the error `The requested URL was not found on this server` after running the Terraform, the issue is that the Cloud Run API service didn't get built correctly. Try to rebuild and redeploy the Cloud Run API service with this command:
+
+```
+cd datacatalog-tag-engine
+gcloud beta run deploy tag-engine-api \
+ 	--source . \
+ 	--platform managed \
+ 	--region $TAG_ENGINE_REGION \
+ 	--no-allow-unauthenticated \
+ 	--ingress=all \
+ 	--memory=1024Mi \
+ 	--service-account=$TAG_ENGINE_SA
+```
+
+Then, call the `ping` endpoint as follows:
+```
+curl $TAG_ENGINE_URL/ping -H "Authorization: Bearer $IAM_TOKEN" -H "oauth_token: $OAUTH_TOKEN"
+```
+You should see the following response:
+```
+Tag Engine is alive
+```
+
+### <a name="next"></a> Part 5: Next Steps
 
 1. Explore additional API methods and run them through curl commands:
 

deploy/external_load_balancer/cloud_run.tf

@@ -67,7 +67,7 @@ resource "google_cloud_run_v2_service" "api_service" {
 	    }
      }
   }
-  depends_on = [null_resource.build_api_image]
+  depends_on = [google_project_service.tag_engine_project, null_resource.build_api_image]
 }
 
 output "api_service_uri" {
@@ -148,7 +148,7 @@ resource "google_cloud_run_v2_service" "ui_service" {
 	    egress = "PRIVATE_RANGES_ONLY"
 	}
   }
-  depends_on = [null_resource.build_ui_image]
+  depends_on = [google_project_service.tag_engine_project, null_resource.build_ui_image]
 }
 
 output "ui_service_uri" {

deploy/external_load_balancer/firestore.tf

@@ -4,16 +4,32 @@
 # created automatically when the API is enabled. 
 # ************************************************************ #
 
-#resource "google_firestore_database" "create" {
-  #project     = var.tag_engine_project
-  #name        = "(default)"
-  #location_id = var.firestore_region
-  #type        = "FIRESTORE_NATIVE"
+resource "google_firestore_database" "create" {
+  project     = var.tag_engine_project
+  name        = "(default)"
+  location_id = var.firestore_region
+  type        = "FIRESTORE_NATIVE"
 
-  #depends_on = [google_project_service.tag_engine_project]
-  #}
+  depends_on = [google_project_service.tag_engine_project]
+}
+
+
+# ************************************************************ #
+# Install python packages
+# ************************************************************ #
+resource "null_resource" "install_packages" {
 
+provisioner "local-exec" {
+  command = "/bin/bash install_packages.sh"
+}
+
+triggers = {
+  always_run = timestamp()
+}
 
+depends_on = [google_cloud_run_v2_service.api_service, google_cloud_run_v2_service.ui_service]
+}
+
 # ************************************************************ #
 # Create the firestore indexes
 # ************************************************************ #
@@ -24,6 +40,6 @@ resource "null_resource" "firestore_indexes" {
     command = "python create_indexes.py ${var.tag_engine_project}"
   }
 
-  depends_on = [google_project_service.tag_engine_project]
+  depends_on = [google_firestore_database.create, null_resource.install_packages]
 }
 

deploy/external_load_balancer/iam_bindings.tf

@@ -140,20 +140,19 @@ resource "google_project_iam_binding" "loggingViewer" {
   depends_on = [google_project_service.tag_engine_project]
 }
 
-resource "google_project_iam_binding" "PolicyTagReader" {
-  project = var.tag_engine_project
-  role    = "projects/${var.tag_engine_project}/roles/PolicyTagReader"
-  members = ["serviceAccount:${var.tag_creator_sa}"]
-  depends_on = [google_project_service.tag_engine_project]
-}
-
 resource "google_project_iam_binding" "BigQuerySchemaUpdate" {
   project = var.bigquery_project
   role    = "projects/${var.bigquery_project}/roles/BigQuerySchemaUpdate"
   members = ["serviceAccount:${var.tag_creator_sa}"]
-  depends_on = [google_project_service.tag_engine_project]
+  depends_on = [google_project_iam_custom_role.bigquery_schema_update]
 }
 
+resource "google_project_iam_binding" "PolicyTagReader" {
+  project = var.tag_engine_project
+  role    = "projects/${var.tag_engine_project}/roles/PolicyTagReader"
+  members = ["serviceAccount:${var.tag_creator_sa}"]
+  depends_on = [google_project_iam_custom_role.policy_tag_reader]
+}
 
 # ************************************************************ #
 # Create the service account policy bindings for tag_engine_sa

deploy/external_load_balancer/variables.tf

@@ -1,7 +1,7 @@
 variable "required_apis" {
 	type = list
 	description = "list of required GCP services"
-	default = ["cloudresourcemanager.googleapis.com", "iam.googleapis.com", "cloudresourcemanager.googleapis.com", "cloudbuild.googleapis.com", "artifactregistry.googleapis.com", "vpcaccess.googleapis.com", "cloudtasks.googleapis.com", "firestore.googleapis.com", "datacatalog.googleapis.com", "iap.googleapis.com"] 
+	default = ["cloudresourcemanager.googleapis.com", "iam.googleapis.com", "cloudresourcemanager.googleapis.com", "cloudbuild.googleapis.com", "artifactregistry.googleapis.com", "run.googleapis.com", "vpcaccess.googleapis.com", "cloudtasks.googleapis.com", "firestore.googleapis.com", "datacatalog.googleapis.com", "iap.googleapis.com"] 
 }
 
 variable "tag_engine_project" {
@@ -96,4 +96,4 @@ variable "oauth_client_secret" {
 variable "authorized_user_accounts" {
   type = list(string)
   description = "The list of users you want to authorize to use the Tag Engine UI. Provide the email address for each user, which must be a google identity." 
-}
+}

deploy/without_load_balancer/cloud_run.tf

@@ -66,7 +66,7 @@ resource "google_cloud_run_v2_service" "api_service" {
 	    }
      }
   }
-  depends_on = [null_resource.build_api_image]
+  depends_on = [google_project_service.tag_engine_project, null_resource.build_api_image]
 }
 
 output "api_service_uri" {
@@ -134,7 +134,7 @@ resource "google_cloud_run_v2_service" "ui_service" {
 	    }
      }
   }
-  depends_on = [null_resource.build_ui_image]
+  depends_on = [google_project_service.tag_engine_project, null_resource.build_ui_image]
 }
 
 output "ui_service_uri" {

deploy/without_load_balancer/firestore.tf

@@ -4,16 +4,32 @@
 # created automatically when the API is enabled. 
 # ************************************************************ #
 
-#resource "google_firestore_database" "create" {
-  #project     = var.tag_engine_project
-  #name        = "(default)"
-  #location_id = var.firestore_region
-  #type        = "FIRESTORE_NATIVE"
+resource "google_firestore_database" "create" {
+  project     = var.tag_engine_project
+  name        = "(default)"
+  location_id = var.firestore_region
+  type        = "FIRESTORE_NATIVE"
 
-  #depends_on = [google_project_service.tag_engine_project]
-  #}
+  depends_on = [google_project_service.tag_engine_project]
+}
+
+
+# ************************************************************ #
+# Install python packages
+# ************************************************************ #
+resource "null_resource" "install_packages" {
 
+provisioner "local-exec" {
+  command = "/bin/bash install_packages.sh"
+}
+
+triggers = {
+  always_run = timestamp()
+}
 
+depends_on = [google_cloud_run_v2_service.api_service, google_cloud_run_v2_service.ui_service]
+}
+
 # ************************************************************ #
 # Create the firestore indexes
 # ************************************************************ #
@@ -24,6 +40,6 @@ resource "null_resource" "firestore_indexes" {
     command = "python create_indexes.py ${var.tag_engine_project}"
   }
 
-  depends_on = [google_project_service.tag_engine_project]
+  depends_on = [google_firestore_database.create, null_resource.install_packages]
 }
 

deploy/without_load_balancer/iam_bindings.tf

@@ -140,20 +140,19 @@ resource "google_project_iam_binding" "loggingViewer" {
   depends_on = [google_project_service.tag_engine_project]
 }
 
-resource "google_project_iam_binding" "PolicyTagReader" {
-  project = var.tag_engine_project
-  role    = "projects/${var.tag_engine_project}/roles/PolicyTagReader"
-  members = ["serviceAccount:${var.tag_creator_sa}"]
-  depends_on = [google_project_service.tag_engine_project]
-}
-
 resource "google_project_iam_binding" "BigQuerySchemaUpdate" {
   project = var.bigquery_project
   role    = "projects/${var.bigquery_project}/roles/BigQuerySchemaUpdate"
   members = ["serviceAccount:${var.tag_creator_sa}"]
-  depends_on = [google_project_service.tag_engine_project]
+  depends_on = [google_project_iam_custom_role.bigquery_schema_update]
 }
 
+resource "google_project_iam_binding" "PolicyTagReader" {
+  project = var.tag_engine_project
+  role    = "projects/${var.tag_engine_project}/roles/PolicyTagReader"
+  members = ["serviceAccount:${var.tag_creator_sa}"]
+  depends_on = [google_project_iam_custom_role.policy_tag_reader]
+}
 
 # ************************************************************ #
 # Create the service account policy bindings for tag_engine_sa

deploy/without_load_balancer/variables.tf

@@ -1,7 +1,7 @@
 variable "required_apis" {
 	type = list
 	description = "list of required GCP services"
-	default = ["cloudresourcemanager.googleapis.com", "iam.googleapis.com", "cloudresourcemanager.googleapis.com", "cloudbuild.googleapis.com", "artifactregistry.googleapis.com", "vpcaccess.googleapis.com", "cloudtasks.googleapis.com", "firestore.googleapis.com", "datacatalog.googleapis.com", "iap.googleapis.com"] 
+	default = ["cloudresourcemanager.googleapis.com", "iam.googleapis.com", "cloudresourcemanager.googleapis.com", "cloudbuild.googleapis.com", "artifactregistry.googleapis.com", "cloudtasks.googleapis.com", "firestore.googleapis.com", "datacatalog.googleapis.com", "run.googleapis.com"] 
 }
 
 variable "tag_engine_project" {