From aeb964fdbfd47bbbb63e4d0976dd5c3fd794bfd5 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Wed, 11 Sep 2024 09:55:15 -0700 Subject: [PATCH] aws: support additional endpoint query parameters "dualstack", "fips", "accelerate" (#3486) --- aws/aws.go | 36 ++++++++++++++++++++++++++++++++++++ aws/aws_test.go | 4 ++++ blob/s3blob/s3blob.go | 26 +++++++++++++++++++++++--- blob/s3blob/s3blob_test.go | 20 ++++++++++++++++++++ 4 files changed, 83 insertions(+), 3 deletions(-) diff --git a/aws/aws.go b/aws/aws.go index 0f2d9c61ff..2e4cbafdef 100644 --- a/aws/aws.go +++ b/aws/aws.go @@ -26,6 +26,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/client" "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/endpoints" "github.com/aws/aws-sdk-go/aws/session" "github.com/google/wire" ) @@ -81,6 +82,8 @@ func (co ConfigOverrider) ClientConfig(serviceName string, cfgs ...*aws.Config) // - endpoint: The endpoint URL (hostname only or fully qualified URI); sets aws.Config.Endpoint. // - disableSSL: A value of "true" disables SSL when sending requests; sets aws.Config.DisableSSL. // - s3ForcePathStyle: A value of "true" forces the request to use path-style addressing; sets aws.Config.S3ForcePathStyle. +// - dualstack: A value of "true" enables dual stack (IPv4 and IPv6) endpoints +// - fips: A value of "true" enables the use of FIPS endpoints func ConfigFromURLParams(q url.Values) (*aws.Config, error) { var cfg aws.Config for param, values := range q { @@ -102,6 +105,20 @@ func ConfigFromURLParams(q url.Values) (*aws.Config, error) { return nil, fmt.Errorf("invalid value for query parameter %q: %v", param, err) } cfg.S3ForcePathStyle = aws.Bool(b) + case "dualstack": + b, err := strconv.ParseBool(value) + if err != nil { + return nil, fmt.Errorf("invalid value for query parameter %q: %v", param, err) + } + cfg.UseDualStack = aws.Bool(b) + case "fips": + b, err := strconv.ParseBool(value) + if err != nil { + return nil, fmt.Errorf("invalid value for query parameter %q: %v", param, err) + } + if b { + cfg.UseFIPSEndpoint = endpoints.FIPSEndpointStateEnabled + } case "awssdk": // ignore, should be handled before this default: @@ -178,6 +195,8 @@ func NewDefaultV2Config(ctx context.Context) (awsv2.Config, error) { // - profile: The shared config profile to use; sets SharedConfigProfile. // - endpoint: The AWS service endpoint to send HTTP request. // - hostname_immutable: Make the hostname immutable, only works if endpoint is also set. +// - dualstack: A value of "true" enables dual stack (IPv4 and IPv6) endpoints. +// - fips: A value of "true" enables the use of FIPS endpoints. func V2ConfigFromURLParams(ctx context.Context, q url.Values) (awsv2.Config, error) { var endpoint string var hostnameImmutable bool @@ -197,6 +216,22 @@ func V2ConfigFromURLParams(ctx context.Context, q url.Values) (awsv2.Config, err endpoint = value case "profile": opts = append(opts, awsv2cfg.WithSharedConfigProfile(value)) + case "dualstack": + dualStack, err := strconv.ParseBool(value) + if err != nil { + return awsv2.Config{}, fmt.Errorf("invalid value for dualstack: %w", err) + } + if dualStack { + opts = append(opts, awsv2cfg.WithUseDualStackEndpoint(awsv2.DualStackEndpointStateEnabled)) + } + case "fips": + fips, err := strconv.ParseBool(value) + if err != nil { + return awsv2.Config{}, fmt.Errorf("invalid value for fips: %w", err) + } + if fips { + opts = append(opts, awsv2cfg.WithUseFIPSEndpoint(awsv2.FIPSEndpointStateEnabled)) + } case "awssdk": // ignore, should be handled before this default: @@ -215,5 +250,6 @@ func V2ConfigFromURLParams(ctx context.Context, q url.Values) (awsv2.Config, err }) opts = append(opts, awsv2cfg.WithEndpointResolverWithOptions(customResolver)) } + return awsv2cfg.LoadDefaultConfig(ctx, opts...) } diff --git a/aws/aws_test.go b/aws/aws_test.go index 29dbee1ae5..8ad2984ed4 100644 --- a/aws/aws_test.go +++ b/aws/aws_test.go @@ -184,6 +184,10 @@ func TestV2ConfigFromURLParams(t *testing.T) { HostnameImmutable: true, }, }, + { + name: "FIPS and dual stack", + query: url.Values{"fips": {"true"}, "dualstack": {"true"}}, + }, // Can't test "profile", since AWS validates that the profile exists. } diff --git a/blob/s3blob/s3blob.go b/blob/s3blob/s3blob.go index e2d42daa7d..7ed47e444e 100644 --- a/blob/s3blob/s3blob.go +++ b/blob/s3blob/s3blob.go @@ -130,6 +130,11 @@ const Scheme = "s3" // Use "awssdk=v1" to force using AWS SDK v1, "awssdk=v2" to force using AWS SDK v2, // or anything else to accept the default. // +// The following S3-specific query options are also supported: +// - ssetype: The type of server side encryption used (AES256, aws:kms, aws:kms:dsse) +// - kmskeyid: The KMS key ID for server side encryption +// - accelerate: A value of "true" uses the S3 Transfer Accleration endpoints +// // For V1, see gocloud.dev/aws/ConfigFromURLParams for supported query parameters // for overriding the aws.Session from the URL. // For V2, see gocloud.dev/aws/V2ConfigFromURLParams. @@ -145,8 +150,9 @@ type URLOpener struct { } const ( - sseTypeParamKey = "ssetype" - kmsKeyIdParamKey = "kmskeyid" + sseTypeParamKey = "ssetype" + kmsKeyIdParamKey = "kmskeyid" + accelerateParamKey = "accelerate" ) func toServerSideEncryptionType(value string) (typesv2.ServerSideEncryption, error) { @@ -178,12 +184,24 @@ func (o *URLOpener) OpenBucketURL(ctx context.Context, u *url.URL) (*blob.Bucket o.Options.KMSEncryptionID = kmsKeyID } + accelerate := false + if accelerateParam := q.Get(accelerateParamKey); accelerateParam != "" { + q.Del(accelerateParamKey) + var err error + accelerate, err = strconv.ParseBool(accelerateParam) + if err != nil { + return nil, fmt.Errorf("invalid value for %q: %v", accelerateParamKey, err) + } + } + if o.UseV2 { cfg, err := gcaws.V2ConfigFromURLParams(ctx, q) if err != nil { return nil, fmt.Errorf("open bucket %v: %v", u, err) } - clientV2 := s3v2.NewFromConfig(cfg) + clientV2 := s3v2.NewFromConfig(cfg, func(o *s3v2.Options) { + o.UseAccelerate = accelerate + }) return OpenBucketV2(ctx, clientV2, u.Host, &o.Options) } @@ -194,6 +212,8 @@ func (o *URLOpener) OpenBucketURL(ctx context.Context, u *url.URL) (*blob.Bucket if err != nil { return nil, fmt.Errorf("open bucket %v: %v", u, err) } + + overrideCfg.S3UseAccelerate = &accelerate configProvider.Configs = append(configProvider.Configs, overrideCfg) return OpenBucket(ctx, configProvider, u.Host, &o.Options) diff --git a/blob/s3blob/s3blob_test.go b/blob/s3blob/s3blob_test.go index 381156d0a4..8d54c521ca 100644 --- a/blob/s3blob/s3blob_test.go +++ b/blob/s3blob/s3blob_test.go @@ -476,6 +476,26 @@ func TestOpenBucketFromURL(t *testing.T) { {"s3://mybucket?awssdk=v2", false}, // OK, use KMS Server Side Encryption {"s3://mybucket?ssetype=aws:kms&kmskeyid=arn:aws:us-east-1:12345:key/1-a-2-b", false}, + // OK, use S3 Transfer acceleration and dual stack endpoints + {"s3://mybucket?accelerate=true&dualstack=true", false}, + // OK, use FIPS endpoints + {"s3://mybucket?fips=true", false}, + // OK, use S3 Transfer accleration and dual stack endpoints (v1) + {"s3://mybucket?awssdk=v1&accelerate=true&dualstack=true", false}, + // OK, use FIPS endpoints (v1) + {"s3://mybucket?awssdk=v1&fips=true", false}, + // Invalid accelerate (v1) + {"s3://mybucket?awssdk=v1&accelerate=bogus", true}, + // Invalid accelerate (v2) + {"s3://mybucket?accelerate=bogus", true}, + // Invalid FIPS (v1) + {"s3://mybucket?awssdk=v1&fips=bogus", true}, + // Invalid FIPS (v2) + {"s3://mybucket?fips=bogus", true}, + // Invalid dualstack (v1) + {"s3://mybucket?awssdk=v1&dualstack=bad", true}, + // Invalid dualstack (v2) + {"s3://mybucket?dualstack=bad", true}, // Invalid ssetype {"s3://mybucket?ssetype=aws:notkmsoraes&kmskeyid=arn:aws:us-east-1:12345:key/1-a-2-b", true}, // Invalid parameter together with a valid one.