Skip to content

Commit

Permalink
Add GCS signed URL support (#5300)
Browse files Browse the repository at this point in the history
* add util function for gcp sign url

* add string to sign and other sign functions

* add GoogleCloudStorageConfig::new and config and move functions to client

* add more code and rearrange struct

* add client_email for credential and return the signed url

* clean some code

* add client email for AuthorizedUserCredentials

* tidy some code

* format doc

* Add GcpSigningCredentialProvider for getting email

* add test

* Move some functions which shared by aws and gcp to utils.

* fix some bug and make it can get proper result

* remoe useless code

* tidy some code

* do not export host

* add sign_by_key

* Cleanup

* Add ServiceAccountKey

* Further tweaks

* add more scope for signing.

* tidy

* Tweak and add test

* Retry and handle errors for signBlob

---------

Co-authored-by: Raphael Taylor-Davies <r.taylordavies@googlemail.com>
  • Loading branch information
l1nxy and tustvold authored Apr 4, 2024
1 parent eddef43 commit 5a0baf1
Show file tree
Hide file tree
Showing 7 changed files with 677 additions and 81 deletions.
19 changes: 1 addition & 18 deletions object_store/src/aws/credential.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ use crate::aws::{AwsCredentialProvider, STORE, STRICT_ENCODE_SET, STRICT_PATH_EN
use crate::client::retry::RetryExt;
use crate::client::token::{TemporaryToken, TokenCache};
use crate::client::TokenProvider;
use crate::util::hmac_sha256;
use crate::util::{hex_digest, hex_encode, hmac_sha256};
use crate::{CredentialProvider, Result, RetryConfig};
use async_trait::async_trait;
use bytes::Buf;
Expand Down Expand Up @@ -342,23 +342,6 @@ impl CredentialExt for RequestBuilder {
}
}

/// Computes the SHA256 digest of `body` returned as a hex encoded string
fn hex_digest(bytes: &[u8]) -> String {
let digest = ring::digest::digest(&ring::digest::SHA256, bytes);
hex_encode(digest.as_ref())
}

/// Returns `bytes` as a lower-case hex encoded string
fn hex_encode(bytes: &[u8]) -> String {
use std::fmt::Write;
let mut out = String::with_capacity(bytes.len() * 2);
for byte in bytes {
// String writing is infallible
let _ = write!(out, "{byte:02x}");
}
out
}

/// Canonicalizes query parameters into the AWS canonical form
///
/// <https://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html>
Expand Down
11 changes: 1 addition & 10 deletions object_store/src/aws/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ use crate::client::list::ListClientExt;
use crate::client::CredentialProvider;
use crate::multipart::{MultipartStore, PartId};
use crate::signer::Signer;
use crate::util::STRICT_ENCODE_SET;
use crate::{
Error, GetOptions, GetResult, ListResult, MultipartId, MultipartUpload, ObjectMeta,
ObjectStore, Path, PutMode, PutOptions, PutResult, Result, UploadPart,
Expand All @@ -64,16 +65,6 @@ pub use dynamo::DynamoCommit;
pub use precondition::{S3ConditionalPut, S3CopyIfNotExists};
pub use resolve::resolve_bucket_region;

// http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
//
// Do not URI-encode any of the unreserved characters that RFC 3986 defines:
// A-Z, a-z, 0-9, hyphen ( - ), underscore ( _ ), period ( . ), and tilde ( ~ ).
pub(crate) const STRICT_ENCODE_SET: percent_encoding::AsciiSet = percent_encoding::NON_ALPHANUMERIC
.remove(b'-')
.remove(b'.')
.remove(b'_')
.remove(b'~');

/// This struct is used to maintain the URI path encoding
const STRICT_PATH_ENCODE_SET: percent_encoding::AsciiSet = STRICT_ENCODE_SET.remove(b'/');

Expand Down
55 changes: 47 additions & 8 deletions object_store/src/gcp/builder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,19 @@ use crate::gcp::credential::{
ApplicationDefaultCredentials, InstanceCredentialProvider, ServiceAccountCredentials,
DEFAULT_GCS_BASE_URL,
};
use crate::gcp::{credential, GcpCredential, GcpCredentialProvider, GoogleCloudStorage, STORE};
use crate::gcp::{
credential, GcpCredential, GcpCredentialProvider, GcpSigningCredential,
GcpSigningCredentialProvider, GoogleCloudStorage, STORE,
};
use crate::{ClientConfigKey, ClientOptions, Result, RetryConfig, StaticCredentialProvider};
use serde::{Deserialize, Serialize};
use snafu::{OptionExt, ResultExt, Snafu};
use std::str::FromStr;
use std::sync::Arc;
use url::Url;

use super::credential::{AuthorizedUserSigningCredentials, InstanceSigningCredentialProvider};

#[derive(Debug, Snafu)]
enum Error {
#[snafu(display("Missing bucket name"))]
Expand Down Expand Up @@ -107,6 +112,8 @@ pub struct GoogleCloudStorageBuilder {
client_options: ClientOptions,
/// Credentials
credentials: Option<GcpCredentialProvider>,
/// Credentials for sign url
signing_cedentials: Option<GcpSigningCredentialProvider>,
}

/// Configuration keys for [`GoogleCloudStorageBuilder`]
Expand Down Expand Up @@ -202,6 +209,7 @@ impl Default for GoogleCloudStorageBuilder {
client_options: ClientOptions::new().with_allow_http(true),
url: None,
credentials: None,
signing_cedentials: None,
}
}
}
Expand Down Expand Up @@ -452,13 +460,13 @@ impl GoogleCloudStorageBuilder {
Arc::new(StaticCredentialProvider::new(GcpCredential {
bearer: "".to_string(),
})) as _
} else if let Some(credentials) = service_account_credentials {
} else if let Some(credentials) = service_account_credentials.clone() {
Arc::new(TokenCredentialProvider::new(
credentials.token_provider()?,
self.client_options.client()?,
self.retry_config.clone(),
)) as _
} else if let Some(credentials) = application_default_credentials {
} else if let Some(credentials) = application_default_credentials.clone() {
match credentials {
ApplicationDefaultCredentials::AuthorizedUser(token) => {
Arc::new(TokenCredentialProvider::new(
Expand All @@ -483,13 +491,44 @@ impl GoogleCloudStorageBuilder {
)) as _
};

let config = GoogleCloudStorageConfig {
base_url: gcs_base_url,
let signing_credentials = if let Some(signing_credentials) = self.signing_cedentials {
signing_credentials
} else if disable_oauth {
Arc::new(StaticCredentialProvider::new(GcpSigningCredential {
email: "".to_string(),
private_key: None,
})) as _
} else if let Some(credentials) = service_account_credentials.clone() {
credentials.signing_credentials()?
} else if let Some(credentials) = application_default_credentials.clone() {
match credentials {
ApplicationDefaultCredentials::AuthorizedUser(token) => {
Arc::new(TokenCredentialProvider::new(
AuthorizedUserSigningCredentials::from(token)?,
self.client_options.client()?,
self.retry_config.clone(),
)) as _
}
ApplicationDefaultCredentials::ServiceAccount(token) => {
token.signing_credentials()?
}
}
} else {
Arc::new(TokenCredentialProvider::new(
InstanceSigningCredentialProvider::default(),
self.client_options.metadata_client()?,
self.retry_config.clone(),
)) as _
};

let config = GoogleCloudStorageConfig::new(
gcs_base_url,
credentials,
signing_credentials,
bucket_name,
retry_config: self.retry_config,
client_options: self.client_options,
};
self.retry_config,
self.client_options,
);

Ok(GoogleCloudStorage {
client: Arc::new(GoogleCloudStorageClient::new(config)?),
Expand Down
105 changes: 103 additions & 2 deletions object_store/src/gcp/client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,19 +24,22 @@ use crate::client::s3::{
ListResponse,
};
use crate::client::GetOptionsExt;
use crate::gcp::{GcpCredential, GcpCredentialProvider, STORE};
use crate::gcp::{GcpCredential, GcpCredentialProvider, GcpSigningCredentialProvider, STORE};
use crate::multipart::PartId;
use crate::path::{Path, DELIMITER};
use crate::util::hex_encode;
use crate::{
ClientOptions, GetOptions, ListResult, MultipartId, PutMode, PutOptions, PutResult, Result,
RetryConfig,
};
use async_trait::async_trait;
use base64::prelude::BASE64_STANDARD;
use base64::Engine;
use bytes::{Buf, Bytes};
use percent_encoding::{percent_encode, utf8_percent_encode, NON_ALPHANUMERIC};
use reqwest::header::HeaderName;
use reqwest::{header, Client, Method, RequestBuilder, Response, StatusCode};
use serde::Serialize;
use serde::{Deserialize, Serialize};
use snafu::{OptionExt, ResultExt, Snafu};
use std::sync::Arc;

Expand Down Expand Up @@ -101,6 +104,15 @@ enum Error {

#[snafu(display("Got invalid multipart response: {}", source))]
InvalidMultipartResponse { source: quick_xml::de::DeError },

#[snafu(display("Error signing blob: {}", source))]
SignBlobRequest { source: crate::client::retry::Error },

#[snafu(display("Got invalid signing blob repsonse: {}", source))]
InvalidSignBlobResponse { source: reqwest::Error },

#[snafu(display("Got invalid signing blob signature: {}", source))]
InvalidSignBlobSignature { source: base64::DecodeError },
}

impl From<Error> for crate::Error {
Expand All @@ -123,13 +135,39 @@ pub struct GoogleCloudStorageConfig {

pub credentials: GcpCredentialProvider,

pub signing_credentials: GcpSigningCredentialProvider,

pub bucket_name: String,

pub retry_config: RetryConfig,

pub client_options: ClientOptions,
}

impl GoogleCloudStorageConfig {
pub fn new(
base_url: String,
credentials: GcpCredentialProvider,
signing_credentials: GcpSigningCredentialProvider,
bucket_name: String,
retry_config: RetryConfig,
client_options: ClientOptions,
) -> Self {
Self {
base_url,
credentials,
signing_credentials,
bucket_name,
retry_config,
client_options,
}
}

pub fn path_url(&self, path: &Path) -> String {
format!("{}/{}/{}", self.base_url, self.bucket_name, path)
}
}

/// A builder for a put request allowing customisation of the headers and query string
pub struct PutRequest<'a> {
path: &'a Path,
Expand Down Expand Up @@ -163,6 +201,21 @@ impl<'a> PutRequest<'a> {
}
}

/// Sign Blob Request Body
#[derive(Debug, Serialize)]
struct SignBlobBody {
/// The payload to sign
payload: String,
}

/// Sign Blob Response
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
struct SignBlobResponse {
/// The signature for the payload
signed_blob: String,
}

#[derive(Debug)]
pub struct GoogleCloudStorageClient {
config: GoogleCloudStorageConfig,
Expand Down Expand Up @@ -197,6 +250,54 @@ impl GoogleCloudStorageClient {
self.config.credentials.get_credential().await
}

/// Create a signature from a string-to-sign using Google Cloud signBlob method.
/// form like:
/// ```plaintext
/// curl -X POST --data-binary @JSON_FILE_NAME \
/// -H "Authorization: Bearer OAUTH2_TOKEN" \
/// -H "Content-Type: application/json" \
/// "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/SERVICE_ACCOUNT_EMAIL:signBlob"
/// ```
///
/// 'JSON_FILE_NAME' is a file containing the following JSON object:
/// ```plaintext
/// {
/// "payload": "REQUEST_INFORMATION"
/// }
/// ```
pub async fn sign_blob(&self, string_to_sign: &str, client_email: &str) -> Result<String> {
let credential = self.get_credential().await?;
let body = SignBlobBody {
payload: BASE64_STANDARD.encode(string_to_sign),
};

let url = format!(
"https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/{}:signBlob",
client_email
);

let response = self
.client
.post(&url)
.bearer_auth(&credential.bearer)
.json(&body)
.send_retry(&self.config.retry_config)
.await
.context(SignBlobRequestSnafu)?;

//If successful, the signature is returned in the signedBlob field in the response.
let response = response
.json::<SignBlobResponse>()
.await
.context(InvalidSignBlobResponseSnafu)?;

let signed_blob = BASE64_STANDARD
.decode(response.signed_blob)
.context(InvalidSignBlobSignatureSnafu)?;

Ok(hex_encode(&signed_blob))
}

pub fn object_url(&self, path: &Path) -> String {
let encoded = utf8_percent_encode(path.as_ref(), NON_ALPHANUMERIC);
format!(
Expand Down
Loading

0 comments on commit 5a0baf1

Please sign in to comment.