From f71c3c4b9e8899345c6dc9e720b9ab99d6e064db Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Tue, 10 Sep 2024 17:16:24 -0700 Subject: [PATCH] s3blob/blob: support additional endpoint query parameters This commit adds the following query parameters for AWS: 1. `dualstack` 2. `fips` 3. `accelerate` (S3-only) This avoids the need for users to specify `endpoint`. For example, if my AWS S3 bucket is `my-bucket` in `us-east-1`, and you want to enable transfer acceleration, dual-stack support, and/or FIPS, you would need to configure `endpoint` with one of the following: 1. `my-bucket.s3-accelerate.amazonaws.com` 2. `my-bucket.s3-accelerate.dualstack.amazonaws.com` 3. `my-bucket.s3-fips.us-gov-east-1.amazonaws.com` 4. `my-bucket.s3-fips.dualstack.us-east-1.amazonaws.com` For example, for the last option, users can use `s3://my-bucket?fips=true&dualstack=true`. Closes https://github.com/google/go-cloud/issues/3484 --- aws/aws.go | 37 +++++++++++++++++++++++++++++++++++++ blob/s3blob/s3blob.go | 21 ++++++++++++++++++--- blob/s3blob/s3blob_test.go | 12 ++++++++++++ 3 files changed, 67 insertions(+), 3 deletions(-) diff --git a/aws/aws.go b/aws/aws.go index 0f2d9c61ff..1f3ba19cd5 100644 --- a/aws/aws.go +++ b/aws/aws.go @@ -26,6 +26,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/client" "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/endpoints" "github.com/aws/aws-sdk-go/aws/session" "github.com/google/wire" ) @@ -102,6 +103,22 @@ func ConfigFromURLParams(q url.Values) (*aws.Config, error) { return nil, fmt.Errorf("invalid value for query parameter %q: %v", param, err) } cfg.S3ForcePathStyle = aws.Bool(b) + case "dualstack": + b, err := strconv.ParseBool(value) + if err != nil { + return nil, fmt.Errorf("invalid value for query parameter %q: %v", param, err) + } + cfg.UseDualStack = aws.Bool(b) + case "fips": + b, err := strconv.ParseBool(value) + if err != nil { + return nil, fmt.Errorf("invalid value for query parameter %q: %v", param, err) + } + fips := endpoints.FIPSEndpointStateDisabled + if b { + fips = endpoints.FIPSEndpointStateEnabled + } + cfg.UseFIPSEndpoint = fips case "awssdk": // ignore, should be handled before this default: @@ -197,6 +214,26 @@ func V2ConfigFromURLParams(ctx context.Context, q url.Values) (awsv2.Config, err endpoint = value case "profile": opts = append(opts, awsv2cfg.WithSharedConfigProfile(value)) + case "dualstack": + dualStackParam, err := strconv.ParseBool(value) + if err != nil { + return awsv2.Config{}, fmt.Errorf("invalid value for dualstack: %w", err) + } + dualStack := awsv2.DualStackEndpointStateDisabled + if dualStackParam { + dualStack = awsv2.DualStackEndpointStateEnabled + } + opts = append(opts, awsv2cfg.WithUseDualStackEndpoint(dualStack)) + case "fips": + fipsParam, err := strconv.ParseBool(value) + if err != nil { + return awsv2.Config{}, fmt.Errorf("invalid value for fips: %w", err) + } + fips := awsv2.FIPSEndpointStateDisabled + if fipsParam { + fips = awsv2.FIPSEndpointStateEnabled + } + opts = append(opts, awsv2cfg.WithUseFIPSEndpoint(fips)) case "awssdk": // ignore, should be handled before this default: diff --git a/blob/s3blob/s3blob.go b/blob/s3blob/s3blob.go index d3e80cf024..264cd38330 100644 --- a/blob/s3blob/s3blob.go +++ b/blob/s3blob/s3blob.go @@ -145,8 +145,9 @@ type URLOpener struct { } const ( - sseTypeParamKey = "ssetype" - kmsKeyIdParamKey = "kmskeyid" + sseTypeParamKey = "ssetype" + kmsKeyIdParamKey = "kmskeyid" + accelerateParamKey = "accelerate" ) func toServerSideEncryptionType(value string) (typesv2.ServerSideEncryption, error) { @@ -178,12 +179,24 @@ func (o *URLOpener) OpenBucketURL(ctx context.Context, u *url.URL) (*blob.Bucket o.Options.KMSEncryptionID = kmsKeyID } + accelerate := false + if accelerateParam := q.Get(accelerateParamKey); accelerateParam != "" { + q.Del(accelerateParamKey) + var err error + accelerate, err = strconv.ParseBool(accelerateParam) + if err != nil { + return nil, fmt.Errorf("invalid value for %q: %v", accelerateParamKey, err) + } + } + if o.UseV2 { cfg, err := gcaws.V2ConfigFromURLParams(ctx, q) if err != nil { return nil, fmt.Errorf("open bucket %v: %v", u, err) } - clientV2 := s3v2.NewFromConfig(cfg) + clientV2 := s3v2.NewFromConfig(cfg, func(o *s3v2.Options) { + o.UseAccelerate = accelerate + }) return OpenBucketV2(ctx, clientV2, u.Host, &o.Options) } @@ -194,6 +207,8 @@ func (o *URLOpener) OpenBucketURL(ctx context.Context, u *url.URL) (*blob.Bucket if err != nil { return nil, fmt.Errorf("open bucket %v: %v", u, err) } + + overrideCfg.S3UseAccelerate = &accelerate configProvider.Configs = append(configProvider.Configs, overrideCfg) return OpenBucket(ctx, configProvider, u.Host, &o.Options) diff --git a/blob/s3blob/s3blob_test.go b/blob/s3blob/s3blob_test.go index 381156d0a4..6872a4ca0f 100644 --- a/blob/s3blob/s3blob_test.go +++ b/blob/s3blob/s3blob_test.go @@ -476,6 +476,18 @@ func TestOpenBucketFromURL(t *testing.T) { {"s3://mybucket?awssdk=v2", false}, // OK, use KMS Server Side Encryption {"s3://mybucket?ssetype=aws:kms&kmskeyid=arn:aws:us-east-1:12345:key/1-a-2-b", false}, + // OK, use S3 Transfer acceleration + {"s3://mybucket?accelerate=true", false}, + // OK, use dual stack endpoints + {"s3://mybucket?dualstack=true", false}, + // OK, use FIPS endpoints + {"s3://mybucket?fips=true", false}, + // OK, use S3 Transfer acceleration (v1) + {"s3://mybucket?awssdk=v1&accelerate=true", false}, + // OK, use dual stack endpoints (v1) + {"s3://mybucket?awssdk=v1&dualstack=true", false}, + // OK, use FIPS endpoints (v1) + {"s3://mybucket?awssdk=v1&fips=true", false}, // Invalid ssetype {"s3://mybucket?ssetype=aws:notkmsoraes&kmskeyid=arn:aws:us-east-1:12345:key/1-a-2-b", true}, // Invalid parameter together with a valid one.