From dcafdcabfb08a6b3a7219b11638e44847a1e31e5 Mon Sep 17 00:00:00 2001 From: Shirley Cohen Date: Sun, 22 Oct 2023 15:51:06 -0500 Subject: [PATCH 1/6] fixed minor formatting bugs --- docs/manual_deployment.md | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/docs/manual_deployment.md b/docs/manual_deployment.md index a21a2e1..7426bfa 100644 --- a/docs/manual_deployment.md +++ b/docs/manual_deployment.md @@ -184,11 +184,11 @@ Note: If you plan to create tags from CSV files, you also need to ensure that `T `storage.buckets.get` permission on the GCS bucket where the CSV files are stored. To do that, you can create a custom role with this permission or assign the `storage.legacyBucketReader` role: - ``` +``` gcloud storage buckets add-iam-policy-binding gs:// \ --member=serviceAccount:$TAG_CREATOR_SA' \ --role=roles/storage.legacyBucketReader - ``` +```
@@ -209,19 +209,19 @@ gcloud alpha firestore databases create --project=$TAG_ENGINE_PROJECT --location First, you must download a private key for your `$TAG_ENGINE_SA`: - ``` + ``` gcloud iam service-accounts keys create private_key.json --iam-account=$TAG_ENGINE_SA export GOOGLE_APPLICATION_CREDENTIALS="private_key.json" - ``` + ``` Second, create the composite indexes which are needed for serving multiple read requests: - ``` + ``` pip install google-cloud-firestore cd deploy/external_load_balancer python create_indexes.py $TAG_ENGINE_PROJECT cd .. - ``` + ``` Note: the above script is expected to run for 10-12 minutes. As the indexes get created, you will see them show up in the Firestore console. There should be about 36 indexes.

@@ -232,7 +232,7 @@ gcloud alpha firestore databases create --project=$TAG_ENGINE_PROJECT --location The next two commands require `gcloud beta`. You can install `gcloud beta` by running `gcloud components install beta`. - ``` + ``` gcloud beta run deploy tag-engine-api \ --source . \ --platform managed \ @@ -241,7 +241,7 @@ gcloud alpha firestore databases create --project=$TAG_ENGINE_PROJECT --location --ingress=all \ --memory=1024Mi \ --service-account=$TAG_ENGINE_SA - ``` + ``` To deploy the UI service without IAP: @@ -263,7 +263,7 @@ gcloud alpha firestore databases create --project=$TAG_ENGINE_PROJECT --location Create a VPC access connector before running the next command. This connector is used to send requests to your VPC network from Cloud Run using internal DNS and internal IP addresses as opposed to going through the public internet. To create a connector, consult [this page](https://cloud.google.com/vpc/docs/configure-serverless-vpc-access#gcloud). - ``` + ``` gcloud beta run deploy tag-engine-ui \ --source . \ --platform managed \ @@ -277,7 +277,7 @@ gcloud alpha firestore databases create --project=$TAG_ENGINE_PROJECT --location --service-account=$TAG_ENGINE_SA \ --vpc-connector=projects/$TAG_ENGINE_PROJECT/locations/$TAG_ENGINE_REGION/connectors/$VPC_CONNECTOR \ --vpc-egress=private-ranges-only - ``` + ```
@@ -285,17 +285,17 @@ gcloud alpha firestore databases create --project=$TAG_ENGINE_PROJECT --location If you are deploying the API, run: - ``` - export API_SERVICE_URL=`gcloud run services describe tag-engine-api --format="value(status.url)"` - gcloud run services update tag-engine-api --set-env-vars SERVICE_URL=$API_SERVICE_URL - ``` + ``` + export API_SERVICE_URL=`gcloud run services describe tag-engine-api --format="value(status.url)"` + gcloud run services update tag-engine-api --set-env-vars SERVICE_URL=$API_SERVICE_URL + ``` If you are deploying the UI, run: - ``` - export UI_SERVICE_URL=`gcloud run services describe tag-engine-ui --format="value(status.url)"` - gcloud run services update tag-engine-ui --set-env-vars SERVICE_URL=$UI_SERVICE_URL - ``` + ``` + export UI_SERVICE_URL=`gcloud run services describe tag-engine-ui --format="value(status.url)"` + gcloud run services update tag-engine-ui --set-env-vars SERVICE_URL=$UI_SERVICE_URL + ```
From 064f8d6b593f5f6af7cc2e0dfb8ec32a3ea2f792 Mon Sep 17 00:00:00 2001 From: Shirley Cohen Date: Sun, 29 Oct 2023 14:38:15 -0500 Subject: [PATCH 2/6] fixed terraform dependencies --- deploy/external_load_balancer/cloud_run.tf | 4 +-- deploy/external_load_balancer/firestore.tf | 32 ++++++++++++++----- deploy/external_load_balancer/iam_bindings.tf | 15 ++++----- deploy/external_load_balancer/variables.tf | 4 +-- deploy/without_load_balancer/cloud_run.tf | 4 +-- deploy/without_load_balancer/firestore.tf | 32 ++++++++++++++----- deploy/without_load_balancer/iam_bindings.tf | 15 ++++----- deploy/without_load_balancer/variables.tf | 2 +- 8 files changed, 69 insertions(+), 39 deletions(-) diff --git a/deploy/external_load_balancer/cloud_run.tf b/deploy/external_load_balancer/cloud_run.tf index 548a052..01dae40 100644 --- a/deploy/external_load_balancer/cloud_run.tf +++ b/deploy/external_load_balancer/cloud_run.tf @@ -67,7 +67,7 @@ resource "google_cloud_run_v2_service" "api_service" { } } } - depends_on = [null_resource.build_api_image] + depends_on = [google_project_service.tag_engine_project, null_resource.build_api_image] } output "api_service_uri" { @@ -148,7 +148,7 @@ resource "google_cloud_run_v2_service" "ui_service" { egress = "PRIVATE_RANGES_ONLY" } } - depends_on = [null_resource.build_ui_image] + depends_on = [google_project_service.tag_engine_project, null_resource.build_ui_image] } output "ui_service_uri" { diff --git a/deploy/external_load_balancer/firestore.tf b/deploy/external_load_balancer/firestore.tf index 4bfc3f4..41b7fa2 100644 --- a/deploy/external_load_balancer/firestore.tf +++ b/deploy/external_load_balancer/firestore.tf @@ -4,16 +4,32 @@ # created automatically when the API is enabled. # ************************************************************ # -#resource "google_firestore_database" "create" { - #project = var.tag_engine_project - #name = "(default)" - #location_id = var.firestore_region - #type = "FIRESTORE_NATIVE" +resource "google_firestore_database" "create" { + project = var.tag_engine_project + name = "(default)" + location_id = var.firestore_region + type = "FIRESTORE_NATIVE" - #depends_on = [google_project_service.tag_engine_project] - #} + depends_on = [google_project_service.tag_engine_project] +} + + +# ************************************************************ # +# Install python packages +# ************************************************************ # +resource "null_resource" "install_packages" { +provisioner "local-exec" { + command = "/bin/bash install_packages.sh" +} + +triggers = { + always_run = timestamp() +} +depends_on = [google_cloud_run_v2_service.api_service, google_cloud_run_v2_service.ui_service] +} + # ************************************************************ # # Create the firestore indexes # ************************************************************ # @@ -24,6 +40,6 @@ resource "null_resource" "firestore_indexes" { command = "python create_indexes.py ${var.tag_engine_project}" } - depends_on = [google_project_service.tag_engine_project] + depends_on = [google_firestore_database.create, null_resource.install_packages] } diff --git a/deploy/external_load_balancer/iam_bindings.tf b/deploy/external_load_balancer/iam_bindings.tf index 9189491..2577b24 100644 --- a/deploy/external_load_balancer/iam_bindings.tf +++ b/deploy/external_load_balancer/iam_bindings.tf @@ -140,20 +140,19 @@ resource "google_project_iam_binding" "loggingViewer" { depends_on = [google_project_service.tag_engine_project] } -resource "google_project_iam_binding" "PolicyTagReader" { - project = var.tag_engine_project - role = "projects/${var.tag_engine_project}/roles/PolicyTagReader" - members = ["serviceAccount:${var.tag_creator_sa}"] - depends_on = [google_project_service.tag_engine_project] -} - resource "google_project_iam_binding" "BigQuerySchemaUpdate" { project = var.bigquery_project role = "projects/${var.bigquery_project}/roles/BigQuerySchemaUpdate" members = ["serviceAccount:${var.tag_creator_sa}"] - depends_on = [google_project_service.tag_engine_project] + depends_on = [google_project_iam_custom_role.bigquery_schema_update] } +resource "google_project_iam_binding" "PolicyTagReader" { + project = var.tag_engine_project + role = "projects/${var.tag_engine_project}/roles/PolicyTagReader" + members = ["serviceAccount:${var.tag_creator_sa}"] + depends_on = [google_project_iam_custom_role.policy_tag_reader] +} # ************************************************************ # # Create the service account policy bindings for tag_engine_sa diff --git a/deploy/external_load_balancer/variables.tf b/deploy/external_load_balancer/variables.tf index 5451cab..5b348a5 100644 --- a/deploy/external_load_balancer/variables.tf +++ b/deploy/external_load_balancer/variables.tf @@ -1,7 +1,7 @@ variable "required_apis" { type = list description = "list of required GCP services" - default = ["cloudresourcemanager.googleapis.com", "iam.googleapis.com", "cloudresourcemanager.googleapis.com", "cloudbuild.googleapis.com", "artifactregistry.googleapis.com", "vpcaccess.googleapis.com", "cloudtasks.googleapis.com", "firestore.googleapis.com", "datacatalog.googleapis.com", "iap.googleapis.com"] + default = ["cloudresourcemanager.googleapis.com", "iam.googleapis.com", "cloudresourcemanager.googleapis.com", "cloudbuild.googleapis.com", "artifactregistry.googleapis.com", "run.googleapis.com", "vpcaccess.googleapis.com", "cloudtasks.googleapis.com", "firestore.googleapis.com", "datacatalog.googleapis.com", "iap.googleapis.com"] } variable "tag_engine_project" { @@ -96,4 +96,4 @@ variable "oauth_client_secret" { variable "authorized_user_accounts" { type = list(string) description = "The list of users you want to authorize to use the Tag Engine UI. Provide the email address for each user, which must be a google identity." -} \ No newline at end of file +} diff --git a/deploy/without_load_balancer/cloud_run.tf b/deploy/without_load_balancer/cloud_run.tf index 20aa2ee..6d1d11a 100644 --- a/deploy/without_load_balancer/cloud_run.tf +++ b/deploy/without_load_balancer/cloud_run.tf @@ -66,7 +66,7 @@ resource "google_cloud_run_v2_service" "api_service" { } } } - depends_on = [null_resource.build_api_image] + depends_on = [google_project_service.tag_engine_project, null_resource.build_api_image] } output "api_service_uri" { @@ -134,7 +134,7 @@ resource "google_cloud_run_v2_service" "ui_service" { } } } - depends_on = [null_resource.build_ui_image] + depends_on = [google_project_service.tag_engine_project, null_resource.build_ui_image] } output "ui_service_uri" { diff --git a/deploy/without_load_balancer/firestore.tf b/deploy/without_load_balancer/firestore.tf index 4bfc3f4..41b7fa2 100644 --- a/deploy/without_load_balancer/firestore.tf +++ b/deploy/without_load_balancer/firestore.tf @@ -4,16 +4,32 @@ # created automatically when the API is enabled. # ************************************************************ # -#resource "google_firestore_database" "create" { - #project = var.tag_engine_project - #name = "(default)" - #location_id = var.firestore_region - #type = "FIRESTORE_NATIVE" +resource "google_firestore_database" "create" { + project = var.tag_engine_project + name = "(default)" + location_id = var.firestore_region + type = "FIRESTORE_NATIVE" - #depends_on = [google_project_service.tag_engine_project] - #} + depends_on = [google_project_service.tag_engine_project] +} + + +# ************************************************************ # +# Install python packages +# ************************************************************ # +resource "null_resource" "install_packages" { +provisioner "local-exec" { + command = "/bin/bash install_packages.sh" +} + +triggers = { + always_run = timestamp() +} +depends_on = [google_cloud_run_v2_service.api_service, google_cloud_run_v2_service.ui_service] +} + # ************************************************************ # # Create the firestore indexes # ************************************************************ # @@ -24,6 +40,6 @@ resource "null_resource" "firestore_indexes" { command = "python create_indexes.py ${var.tag_engine_project}" } - depends_on = [google_project_service.tag_engine_project] + depends_on = [google_firestore_database.create, null_resource.install_packages] } diff --git a/deploy/without_load_balancer/iam_bindings.tf b/deploy/without_load_balancer/iam_bindings.tf index 9189491..2577b24 100644 --- a/deploy/without_load_balancer/iam_bindings.tf +++ b/deploy/without_load_balancer/iam_bindings.tf @@ -140,20 +140,19 @@ resource "google_project_iam_binding" "loggingViewer" { depends_on = [google_project_service.tag_engine_project] } -resource "google_project_iam_binding" "PolicyTagReader" { - project = var.tag_engine_project - role = "projects/${var.tag_engine_project}/roles/PolicyTagReader" - members = ["serviceAccount:${var.tag_creator_sa}"] - depends_on = [google_project_service.tag_engine_project] -} - resource "google_project_iam_binding" "BigQuerySchemaUpdate" { project = var.bigquery_project role = "projects/${var.bigquery_project}/roles/BigQuerySchemaUpdate" members = ["serviceAccount:${var.tag_creator_sa}"] - depends_on = [google_project_service.tag_engine_project] + depends_on = [google_project_iam_custom_role.bigquery_schema_update] } +resource "google_project_iam_binding" "PolicyTagReader" { + project = var.tag_engine_project + role = "projects/${var.tag_engine_project}/roles/PolicyTagReader" + members = ["serviceAccount:${var.tag_creator_sa}"] + depends_on = [google_project_iam_custom_role.policy_tag_reader] +} # ************************************************************ # # Create the service account policy bindings for tag_engine_sa diff --git a/deploy/without_load_balancer/variables.tf b/deploy/without_load_balancer/variables.tf index 1d27425..390e3e2 100644 --- a/deploy/without_load_balancer/variables.tf +++ b/deploy/without_load_balancer/variables.tf @@ -1,7 +1,7 @@ variable "required_apis" { type = list description = "list of required GCP services" - default = ["cloudresourcemanager.googleapis.com", "iam.googleapis.com", "cloudresourcemanager.googleapis.com", "cloudbuild.googleapis.com", "artifactregistry.googleapis.com", "vpcaccess.googleapis.com", "cloudtasks.googleapis.com", "firestore.googleapis.com", "datacatalog.googleapis.com", "iap.googleapis.com"] + default = ["cloudresourcemanager.googleapis.com", "iam.googleapis.com", "cloudresourcemanager.googleapis.com", "cloudbuild.googleapis.com", "artifactregistry.googleapis.com", "cloudtasks.googleapis.com", "firestore.googleapis.com", "datacatalog.googleapis.com", "run.googleapis.com"] } variable "tag_engine_project" { From 4b808f93b94db67a802c9d8da6fdde5900a0b222 Mon Sep 17 00:00:00 2001 From: Shirley Cohen Date: Sun, 29 Oct 2023 14:51:19 -0500 Subject: [PATCH 3/6] fixed minor bugs in test procedure --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index b3c870a..0fdc68c 100644 --- a/README.md +++ b/README.md @@ -111,7 +111,7 @@ python create_template.py $TAG_ENGINE_PROJECT $TAG_ENGINE_REGION data_governance ```
-2. Authorize a user account to use $TAG_CREATOR_SA and to invoke the Tag Engine Cloud Run service: +2. Authorize a user account to use $TAG_CREATOR_SA and to invoke the Tag Engine API Cloud Run service: ``` export USER_ACCOUNT="username@example.com" @@ -120,7 +120,7 @@ gcloud iam service-accounts add-iam-policy-binding $TAG_CREATOR_SA \ --member=user:$USER_ACCOUNT --role=roles/iam.serviceAccountUser -gcloud run services add-iam-policy-binding tag-engine \ +gcloud run services add-iam-policy-binding tag-engine-api \ --member=user:$USER_ACCOUNT --role=roles/run.invoker \ --region=$TAG_ENGINE_REGION ``` @@ -233,7 +233,7 @@ e) View the job status: ```
-2. Authorize a service account to use $TAG_CREATOR_SA and to invoke the Tag Engine Cloud Run service: +2. Authorize a service account to use $TAG_CREATOR_SA and to invoke the Tag Engine API Cloud Run service: ``` export CLIENT_SA="tag-engine-client@.iam.gserviceaccount.com" @@ -242,7 +242,7 @@ e) View the job status: --member=serviceAccount:$CLIENT_SA --role=roles/iam.serviceAccountUser - gcloud run services add-iam-policy-binding tag-engine \ + gcloud run services add-iam-policy-binding tag-engine-api \ --member=serviceAccount:$CLIENT_SA --role=roles/run.invoker \ --region=$TAG_ENGINE_REGION ``` From a77c719bd6c2df916159aefab13855929874d87f Mon Sep 17 00:00:00 2001 From: Shirley Cohen Date: Sun, 29 Oct 2023 15:03:31 -0500 Subject: [PATCH 4/6] fixed minor bug in README.md --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 0fdc68c..c5be22b 100644 --- a/README.md +++ b/README.md @@ -260,7 +260,6 @@ e) View the job status: 4. Generate an IAM token (aka Bearer token) for authenticating to the Tag Engine Cloud Run service: ``` - gcloud auth login export IAM_TOKEN=$(gcloud auth print-identity-token) ```
From dcc38adc22c0091b97409ca250925d59da0ae08b Mon Sep 17 00:00:00 2001 From: Shirley Cohen Date: Sun, 29 Oct 2023 15:15:09 -0500 Subject: [PATCH 5/6] added troubleshooting section --- README.md | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c5be22b..bc5dbf2 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,8 @@ This README is organized into four parts:
- Part 1: [Deploying Tag Engine v2](#deploy)
- Part 2: [Testing your Setup with a User Account](#testa)
- Part 3: [Testing your Setup with a Service Account](#testb)
-- Part 4: [What To Do Next](#next)
+- Part 4: [Troubleshooting](#troubleshooting)
+- Part 5: [What To Do Next](#next)
### Part 1: Deploying Tag Engine v2 @@ -331,7 +332,23 @@ e) View the job status: Open the Data Catalog UI and verify that your tag was successfully created. If not, open the Cloud Run logs and investigate the problem.

-### Part 4: Next Steps +### Part 4: Troubleshooting + +If you encounter the error `The requested URL was not found on this server`, you'll need to redeploy the Cloud Run API service as follows: + +``` +cd datacatalog-tag-engine +gcloud beta run deploy tag-engine-api \ + --source . \ + --platform managed \ + --region $TAG_ENGINE_REGION \ + --no-allow-unauthenticated \ + --ingress=all \ + --memory=1024Mi \ + --service-account=$TAG_ENGINE_SA +``` + +### Part 5: Next Steps 1. Explore additional API methods and run them through curl commands: From 3e40500bc53c00815a9ac179e4d3ffd61f3e01eb Mon Sep 17 00:00:00 2001 From: Shirley Cohen Date: Sun, 29 Oct 2023 17:02:49 -0500 Subject: [PATCH 6/6] added more details to troubleshooting step in README --- README.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index bc5dbf2..3ce411a 100644 --- a/README.md +++ b/README.md @@ -334,7 +334,7 @@ e) View the job status: ### Part 4: Troubleshooting -If you encounter the error `The requested URL was not found on this server`, you'll need to redeploy the Cloud Run API service as follows: +If you encounter the error `The requested URL was not found on this server` after running the Terraform, the issue is that the Cloud Run API service didn't get built correctly. Try to rebuild and redeploy the Cloud Run API service with this command: ``` cd datacatalog-tag-engine @@ -348,6 +348,15 @@ gcloud beta run deploy tag-engine-api \ --service-account=$TAG_ENGINE_SA ``` +Then, call the `ping` endpoint as follows: +``` +curl $TAG_ENGINE_URL/ping -H "Authorization: Bearer $IAM_TOKEN" -H "oauth_token: $OAUTH_TOKEN" +``` +You should see the following response: +``` +Tag Engine is alive +``` + ### Part 5: Next Steps 1. Explore additional API methods and run them through curl commands: