s3blob/blob: support additional endpoint query parameters

This commit adds the following query parameters for AWS:

1. `dualstack`
2. `fips`
3. `accelerate` (S3-only)

This avoids the need for users to specify `endpoint`. For example, if
my AWS S3 bucket is `my-bucket` in `us-east-1`, and you want to enable
transfer acceleration, dual-stack support, and/or FIPS, you would need
to configure `endpoint` with one of the following:

1. `my-bucket.s3-accelerate.amazonaws.com`
2. `my-bucket.s3-accelerate.dualstack.amazonaws.com`
3. `my-bucket.s3-fips.us-gov-east-1.amazonaws.com`
4. `my-bucket.s3-fips.dualstack.us-east-1.amazonaws.com`

For example, for the last option, users can use
`s3://my-bucket?fips=true&dualstack=true`.

Closes google#3484

aws/aws.go

@@ -26,6 +26,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/client"
 	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/endpoints"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/google/wire"
 )
@@ -102,6 +103,22 @@ func ConfigFromURLParams(q url.Values) (*aws.Config, error) {
 				return nil, fmt.Errorf("invalid value for query parameter %q: %v", param, err)
 			}
 			cfg.S3ForcePathStyle = aws.Bool(b)
+		case "dualstack":
+			b, err := strconv.ParseBool(value)
+			if err != nil {
+				return nil, fmt.Errorf("invalid value for query parameter %q: %v", param, err)
+			}
+			cfg.UseDualStack = aws.Bool(b)
+		case "fips":
+			b, err := strconv.ParseBool(value)
+			if err != nil {
+				return nil, fmt.Errorf("invalid value for query parameter %q: %v", param, err)
+			}
+			fips := endpoints.FIPSEndpointStateDisabled
+			if b {
+				fips = endpoints.FIPSEndpointStateEnabled
+			}
+			cfg.UseFIPSEndpoint = fips
 		case "awssdk":
 			// ignore, should be handled before this
 		default:
@@ -197,6 +214,26 @@ func V2ConfigFromURLParams(ctx context.Context, q url.Values) (awsv2.Config, err
 			endpoint = value
 		case "profile":
 			opts = append(opts, awsv2cfg.WithSharedConfigProfile(value))
+		case "dualstack":
+			dualStackParam, err := strconv.ParseBool(value)
+			if err != nil {
+				return awsv2.Config{}, fmt.Errorf("invalid value for dualstack: %w", err)
+			}
+			dualStack := awsv2.DualStackEndpointStateDisabled
+			if dualStackParam {
+				dualStack = awsv2.DualStackEndpointStateEnabled
+			}
+			opts = append(opts, awsv2cfg.WithUseDualStackEndpoint(dualStack))
+		case "fips":
+			fipsParam, err := strconv.ParseBool(value)
+			if err != nil {
+				return awsv2.Config{}, fmt.Errorf("invalid value for fips: %w", err)
+			}
+			fips := awsv2.FIPSEndpointStateDisabled
+			if fipsParam {
+				fips = awsv2.FIPSEndpointStateEnabled
+			}
+			opts = append(opts, awsv2cfg.WithUseFIPSEndpoint(fips))
 		case "awssdk":
 			// ignore, should be handled before this
 		default:

blob/s3blob/s3blob.go

@@ -145,8 +145,9 @@ type URLOpener struct {
 }
 
 const (
-	sseTypeParamKey  = "ssetype"
-	kmsKeyIdParamKey = "kmskeyid"
+	sseTypeParamKey    = "ssetype"
+	kmsKeyIdParamKey   = "kmskeyid"
+	accelerateParamKey = "accelerate"
 )
 
 func toServerSideEncryptionType(value string) (typesv2.ServerSideEncryption, error) {
@@ -178,12 +179,24 @@ func (o *URLOpener) OpenBucketURL(ctx context.Context, u *url.URL) (*blob.Bucket
 		o.Options.KMSEncryptionID = kmsKeyID
 	}
 
+	accelerate := false
+	if accelerateParam := q.Get(accelerateParamKey); accelerateParam != "" {
+		q.Del(accelerateParamKey)
+		var err error
+		accelerate, err = strconv.ParseBool(accelerateParam)
+		if err != nil {
+			return nil, fmt.Errorf("invalid value for %q: %v", accelerateParamKey, err)
+		}
+	}
+
 	if o.UseV2 {
 		cfg, err := gcaws.V2ConfigFromURLParams(ctx, q)
 		if err != nil {
 			return nil, fmt.Errorf("open bucket %v: %v", u, err)
 		}
-		clientV2 := s3v2.NewFromConfig(cfg)
+		clientV2 := s3v2.NewFromConfig(cfg, func(o *s3v2.Options) {
+			o.UseAccelerate = accelerate
+		})
 
 		return OpenBucketV2(ctx, clientV2, u.Host, &o.Options)
 	}
@@ -194,6 +207,8 @@ func (o *URLOpener) OpenBucketURL(ctx context.Context, u *url.URL) (*blob.Bucket
 	if err != nil {
 		return nil, fmt.Errorf("open bucket %v: %v", u, err)
 	}
+
+	overrideCfg.S3UseAccelerate = &accelerate
 	configProvider.Configs = append(configProvider.Configs, overrideCfg)
 
 	return OpenBucket(ctx, configProvider, u.Host, &o.Options)

blob/s3blob/s3blob_test.go

@@ -476,6 +476,18 @@ func TestOpenBucketFromURL(t *testing.T) {
 		{"s3://mybucket?awssdk=v2", false},
 		// OK, use KMS Server Side Encryption
 		{"s3://mybucket?ssetype=aws:kms&kmskeyid=arn:aws:us-east-1:12345:key/1-a-2-b", false},
+		// OK, use S3 Transfer acceleration
+		{"s3://mybucket?accelerate=true", false},
+		// OK, use dual stack endpoints
+		{"s3://mybucket?dualstack=true", false},
+		// OK, use FIPS endpoints
+		{"s3://mybucket?fips=true", false},
+		// OK, use S3 Transfer acceleration (v1)
+		{"s3://mybucket?awssdk=v1&accelerate=true", false},
+		// OK, use dual stack endpoints (v1)
+		{"s3://mybucket?awssdk=v1&dualstack=true", false},
+		// OK, use FIPS endpoints (v1)
+		{"s3://mybucket?awssdk=v1&fips=true", false},
 		// Invalid ssetype
 		{"s3://mybucket?ssetype=aws:notkmsoraes&kmskeyid=arn:aws:us-east-1:12345:key/1-a-2-b", true},
 		// Invalid parameter together with a valid one.