From f06cfdb0ed0b21467e4cc04291a4f846217095bf Mon Sep 17 00:00:00 2001 From: dark0dave Date: Mon, 27 Nov 2023 19:11:24 +0000 Subject: [PATCH 1/2] fix(cred): Add cred lookup Signed-off-by: dark0dave --- pyproject.toml | 2 +- storages/backends/gcloud.py | 38 ++++++++++++++++++++++++------------- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 66dded8d..a01a6c96 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ dropbox = [ "dropbox>=7.2.1; python_version<'3.12'", ] google = [ - "google-cloud-storage>=1.27", + "google-cloud-storage>=2.14", ] libcloud = [ "apache-libcloud", diff --git a/storages/backends/gcloud.py b/storages/backends/gcloud.py index ace69244..77b3e144 100644 --- a/storages/backends/gcloud.py +++ b/storages/backends/gcloud.py @@ -1,4 +1,5 @@ import gzip +import logging import io import mimetypes from datetime import timedelta @@ -20,6 +21,8 @@ from storages.utils import to_bytes try: + from google import auth + from google.auth.transport import requests from google.cloud.exceptions import NotFound from google.cloud.storage import Blob from google.cloud.storage import Client @@ -34,6 +37,7 @@ CONTENT_ENCODING = "content_encoding" CONTENT_TYPE = "content_type" +_LOGGER = logging.getLogger(__name__) class GoogleCloudFile(CompressedFileMixin, File): @@ -142,12 +146,19 @@ def get_default_settings(self): # roll over. "max_memory_size": setting("GS_MAX_MEMORY_SIZE", 0), "blob_chunk_size": setting("GS_BLOB_CHUNK_SIZE"), + "sa_email": setting("GS_SA_SIGNING_EMAIL") } @property def client(self): + credentials, project_id = auth.default(scopes=['https://www.googleapis.com/auth/cloud-platform']) + credentials.refresh(requests.Request()) + if not hasattr(credentials, "service_account_email"): + credentials.service_account_email = self.sa_email + _LOGGER.debug(f"Signing email: {credentials.service_account_email}") + self.credentials = credentials if self._client is None: - self._client = Client(project=self.project_id, credentials=self.credentials) + self._client = Client(project=project_id, credentials=self.credentials) return self._client @property @@ -318,19 +329,20 @@ def url(self, name, parameters=None): storage_base_url=self.custom_endpoint, quoted_name=_quote(name, safe=b"/~"), ) + elif not self.custom_endpoint: + return blob.generate_signed_url(**self.signed_url_extra()) else: - default_params = { - "bucket_bound_hostname": self.custom_endpoint, - "expiration": self.expiration, - "version": "v4", - } - params = parameters or {} - - for key, value in default_params.items(): - if value and key not in params: - params[key] = value - - return blob.generate_signed_url(**params) + return blob.generate_signed_url( + api_access_endpoint=self.custom_endpoint, + **self.signed_url_extra() + ) + + def signed_url_extra(self): + return { + "service_account_email": self.credentials.service_account_email, + "access_token": self.credentials.token, + "credentials": self.credentials, + } def get_available_name(self, name, max_length=None): name = clean_name(name) From 8a7ef0cb2ce6f8ed4c8996fa441d7b23e321c753 Mon Sep 17 00:00:00 2001 From: dark0dave Date: Mon, 22 Jan 2024 16:10:46 +0000 Subject: [PATCH 2/2] fix(expiration): Add expiration back Signed-off-by: dark0dave --- docs/backends/gcloud.rst | 10 ++++++++++ storages/backends/gcloud.py | 37 +++++++++++++++---------------------- 2 files changed, 25 insertions(+), 22 deletions(-) diff --git a/docs/backends/gcloud.rst b/docs/backends/gcloud.rst index 962298eb..0c18f27e 100644 --- a/docs/backends/gcloud.rst +++ b/docs/backends/gcloud.rst @@ -61,6 +61,7 @@ For development use cases, or other instances outside Google infrastructure: Alternatively, you can use the setting ``credentials`` or ``GS_CREDENTIALS`` as described below. +It is also now possible to use workload identity by providing the service account via ``GS_SA_SIGNING_EMAIL``. Settings ~~~~~~~~ @@ -219,3 +220,12 @@ Settings It supports `timedelta`, `datetime`, or `integer` seconds since epoch time. Note: The maximum value for this option is 7 days (604800 seconds) in version `v4` (See this `Github issue `_) + +``sa_email`` or ``GS_SA_SIGNING_EMAIL`` + + default: ``''`` + + This is the signing email if it is not fetched from the credentials. Or if you wish to sign the signed urls with a different service_account. + + As above please note that, Default Google Compute Engine (GCE) Service accounts are + `unable to sign urls `_. diff --git a/storages/backends/gcloud.py b/storages/backends/gcloud.py index 77b3e144..46a40069 100644 --- a/storages/backends/gcloud.py +++ b/storages/backends/gcloud.py @@ -1,5 +1,4 @@ import gzip -import logging import io import mimetypes from datetime import timedelta @@ -37,7 +36,6 @@ CONTENT_ENCODING = "content_encoding" CONTENT_TYPE = "content_type" -_LOGGER = logging.getLogger(__name__) class GoogleCloudFile(CompressedFileMixin, File): @@ -151,14 +149,13 @@ def get_default_settings(self): @property def client(self): - credentials, project_id = auth.default(scopes=['https://www.googleapis.com/auth/cloud-platform']) - credentials.refresh(requests.Request()) - if not hasattr(credentials, "service_account_email"): - credentials.service_account_email = self.sa_email - _LOGGER.debug(f"Signing email: {credentials.service_account_email}") - self.credentials = credentials if self._client is None: - self._client = Client(project=project_id, credentials=self.credentials) + project_id, credentials = self.project_id, self.credentials + if project_id is None and credentials is None: + credentials, project_id = auth.default(scopes=['https://www.googleapis.com/auth/cloud-platform']) + if not hasattr(credentials, "service_account_email"): + credentials.service_account_email = self.sa_email + self._client = Client(project=project_id, credentials=credentials) return self._client @property @@ -329,20 +326,16 @@ def url(self, name, parameters=None): storage_base_url=self.custom_endpoint, quoted_name=_quote(name, safe=b"/~"), ) - elif not self.custom_endpoint: - return blob.generate_signed_url(**self.signed_url_extra()) else: - return blob.generate_signed_url( - api_access_endpoint=self.custom_endpoint, - **self.signed_url_extra() - ) - - def signed_url_extra(self): - return { - "service_account_email": self.credentials.service_account_email, - "access_token": self.credentials.token, - "credentials": self.credentials, - } + params = { + "service_account_email": self.credentials.service_account_email, + "access_token": self.credentials.token, + "credentials": self.credentials, + "expiration": self.expiration, + } + if self.custom_endpoint: + params["api_access_endpoint"] = self.custom_endpoint + return blob.generate_signed_url(**params) def get_available_name(self, name, max_length=None): name = clean_name(name)