diff --git a/README.md b/README.md index 8c86ba2..87742bc 100644 --- a/README.md +++ b/README.md @@ -196,20 +196,41 @@ These permissions may vary depending on the setup of the project. Consider the f ### Configure the Cloud Build account -In the source project, navigate to the [Cloud Build](https://console.cloud.google.com/cloud-build/settings/service-account) and locate the account that will execute the deployment process. +You need to grant the Cloud Build service account permissions to deploy Cortex. -![cloud build service account](images/5.png "image_tooltip") +Cloud Build uses a service account to execute builds on your behalf. [Cloud Build service account](https://cloud.google.com/build/docs/cloud-build-service-account) describes how Cloud Build uses the default service account. -Locate the build account in [IAM](https://pantheon.corp.google.com/iam-admin/iam) (make sure it says _cloudbuild_): +To grant the required permissions, perform the following steps: -![Cloud build service account in IAM](images/6.png "image_tooltip") +1. Find the default Cloud Build service account by opening [Cloud Shell](https://shell.cloud.google.com/?show=terminal) and executing the following gcloud command: + ```bash + gcloud builds get-default-service-account --project + ``` + +2. You should see a response formatted as either: + + `serviceAccountEmail: projects//serviceAccounts/-compute@developer.gserviceaccount.com` + + or + + `serviceAccountEmail: projects//serviceAccounts/@cloudbuild.gserviceaccount.com` + + Note the last part, `-compute@developer.gserviceaccount.com` or `@cloudbuild.gserviceaccount.com`, This is your default Cloud Build service account. + + +3. Locate this service account in [IAM](https://console.cloud.google.com/iam-admin/iam): + ![Cloud build service account in IAM](images/6.png "Cloud Build service account") + + or + + ![Cloud build compute service account in IAM](images/cloudbuild_compute_sa.png "Cloud Build Compute service account") -Grant the following permissions to the Cloud Build service account in both the source and target projects if they are different: +4. Grant the following permissions to the Cloud Build service account in the source project (and the target project if deploying to a separate target): -- BigQuery Data Editor -- BigQuery Job User + - BigQuery Data Editor + - BigQuery Job User -\[Optional\] If changing the default values for Data Mesh in `config/config.json` to implement features beyond descriptions, the executing account (Cloud Build service account) will need to have the following permissions: +\[Optional\] If changing the default values for Data Mesh in `config/config.json` to implement features beyond descriptions, the executing account (Cloud Build service account) will need to have the following additional permissions: - Policy Tag Admin - Data Catalog TagTemplate Owner - Dataplex Editor diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 5208664..dfddb50 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,3 +1,14 @@ +## June 2024 - Release 5.4.2 +The Cloud Build default service account behavior has changed and will now default to one of two types of service accounts: +* The legacy Cloud Build service account `@cloudbuild.gserviceaccount.com`. +* The project's compute service account `-compute@developer.gserviceaccount.com`. + +This release updates the [Quick demo deployment](README.md#quick-demo-deployment) to use either default account. + +The guidance in [Configure the Cloud Build account](README.md#configure-the-cloud-build-account) has also been updated with the latest instructions to determine your Cloud Build default account. + +Additional details about the Cloud Build change can be found at [Cloud Build Service Account Change](https://cloud.google.com/build/docs/cloud-build-service-account-updates). + ## May 2024 - Release 5.4.1 ### Marketing * Table schema directory location re-aligned for Google Ads, CM360 and TikTok: moved from `src/marketing/src/SOURCE/src/table_schema` to `src/marketing/src/SOURCE/config/table_schema` matching other Marketing sources. diff --git a/images/5.png b/images/5.png deleted file mode 100644 index a0b98c5..0000000 Binary files a/images/5.png and /dev/null differ diff --git a/images/cloudbuild_compute_sa.png b/images/cloudbuild_compute_sa.png new file mode 100644 index 0000000..9f106dd Binary files /dev/null and b/images/cloudbuild_compute_sa.png differ diff --git a/src/utils/interactive/apply_config.py b/src/utils/interactive/apply_config.py index 14f7c0d..8031360 100644 --- a/src/utils/interactive/apply_config.py +++ b/src/utils/interactive/apply_config.py @@ -1,4 +1,4 @@ -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,21 +17,36 @@ import logging import typing +from google.api_core import retry +from google.api_core.exceptions import Conflict +from google.api_core.exceptions import Forbidden +from google.api_core.exceptions import Unauthorized +from google import auth as gauth +from google.auth.transport import requests +from google.cloud import bigquery +from google.cloud import storage +from google.cloud.bigquery.enums import EntityTypes import googleapiclient.discovery from googleapiclient.errors import HttpError -from google.api_core.exceptions import Unauthorized, Forbidden, Conflict -from google.cloud import bigquery, storage -from google.cloud.bigquery.enums import EntityTypes +_RETRY_TIMEOUT_SEC = 60.0 # Timeout for API retries SOURCE_PROJECT_APIS = ["cloudresourcemanager", "storage-component", "bigquery", "cloudbuild"] TARGET_PROJECT_APIS = ["storage-component", "bigquery"] PROJECT_ROLES = ["roles/bigquery.user"] + +@retry.Retry(predicate=retry.if_exception_type(KeyError, HttpError), + timeout=_RETRY_TIMEOUT_SEC) def get_cloud_build_account(project_id: str) -> str: - """Retrieves GCP project Cloud Build account principal by project name/id. + """ + Retrieves GCP project Cloud Build account principal by project name/id. + + Since this gets called soon after the Cloud Build API is enabled, + the @retry.Retry dectorator is called to ensure the API is available before + this function proceeds with retrieving the serivce account. Args: project_id (str): project id @@ -39,11 +54,18 @@ def get_cloud_build_account(project_id: str) -> str: Returns: str: Cloud Build account principal """ - crm = googleapiclient.discovery.build("cloudresourcemanager", "v1", - cache_discovery=False) - project = crm.projects().get(projectId=project_id).execute() - project_number = project["projectNumber"] - return f"{project_number}@cloudbuild.gserviceaccount.com" + + # Get default Cloud Build account + cloudbuild_account_url =( + "https://cloudbuild.googleapis.com/v1/projects/" + f"{project_id}/locations/global/defaultServiceAccount") + + credentials,_ = gauth.default(quota_project_id=project_id) + session = requests.AuthorizedSession(credentials) + response_json = session.get(cloudbuild_account_url).json() + sa_email = response_json["serviceAccountEmail"] + + return sa_email.split("/")[-1] def add_bq_roles(client: bigquery.Client, dataset: bigquery.Dataset, @@ -228,7 +250,7 @@ def add_bucket_roles(client: storage.Client, bucket: storage.Bucket, else: if service_account_name not in role_binding["members"]: modified = True - role_binding["members"].append(service_account_name) + role_binding["members"].add(service_account_name) if modified: try: