Load s3 options from url

Signed-off-by: Tamal Saha <tamal@appscode.com>

go.mod

@@ -3,40 +3,42 @@ module gomodules.xyz/blobfs
 go 1.18
 
 require (
-	github.com/aws/aws-sdk-go v1.43.31
-	gocloud.dev v0.26.0
+	github.com/aws/aws-sdk-go v1.54.18
+	gocloud.dev v0.37.0
 )
 
 require (
-	github.com/aws/aws-sdk-go-v2 v1.16.2 // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.1 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.15.3 // indirect
-	github.com/aws/aws-sdk-go-v2/credentials v1.11.2 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.3 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.3 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.9 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.3 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.10 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.26.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.11.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.16.3 // indirect
-	github.com/aws/smithy-go v1.11.2 // indirect
+	github.com/aws/aws-sdk-go-v2 v1.25.3 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.27.7 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.7 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.3 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.9 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.3 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.3 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.51.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.20.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.28.4 // indirect
+	github.com/aws/smithy-go v1.20.1 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/google/wire v0.5.0 // indirect
-	github.com/googleapis/gax-go/v2 v2.2.0 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
+	github.com/google/wire v0.6.0 // indirect
+	github.com/googleapis/gax-go/v2 v2.12.2 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
-	go.opencensus.io v0.23.0 // indirect
-	golang.org/x/net v0.0.0-20220401154927-543a649e0bdd // indirect
-	golang.org/x/sys v0.0.0-20220330033206-e17cdc41300f // indirect
-	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
-	google.golang.org/api v0.74.0 // indirect
-	google.golang.org/genproto v0.0.0-20220401170504-314d38edb7de // indirect
-	google.golang.org/grpc v1.45.0 // indirect
-	google.golang.org/protobuf v1.28.0 // indirect
+	go.opencensus.io v0.24.0 // indirect
+	golang.org/x/net v0.22.0 // indirect
+	golang.org/x/sys v0.18.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
+	google.golang.org/api v0.169.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7 // indirect
+	google.golang.org/grpc v1.62.1 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
 )

lib.go

@@ -7,18 +7,15 @@ import (
 	"crypto/x509"
 	"fmt"
 	"io"
+	"net"
 	"net/http"
+	"net/url"
 	"os"
 	"path"
 	"strings"
 
 	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/credentials"
-	"github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds"
-	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
-	"github.com/aws/aws-sdk-go/aws/ec2metadata"
-	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/sts"
+	gcaws "gocloud.dev/aws"
 	"gocloud.dev/blob"
 	_ "gocloud.dev/blob/fileblob"
 	_ "gocloud.dev/blob/memblob"
@@ -34,10 +31,7 @@ type BlobFS struct {
 	storageURL string
 	prefix     string
 
-	CACert      []byte
-	InsecureTLS bool
-	Endpoint    string
-	Region      string
+	CACert []byte
 }
 
 func New(storageURL string, prefix ...string) *BlobFS {
@@ -70,7 +64,9 @@ func (fs *BlobFS) WriteFile(ctx context.Context, filepath string, data []byte) e
 	}
 	defer bucket.Close()
 
-	w, err := bucket.NewWriter(ctx, filename, nil)
+	w, err := bucket.NewWriter(ctx, filename, &blob.WriterOptions{
+		DisableContentTypeDetection: true,
+	})
 	if err != nil {
 		return err
 	}
@@ -143,13 +139,42 @@ func (fs *BlobFS) SignedURL(ctx context.Context, filepath string, opts *blob.Sig
 
 func (fs *BlobFS) OpenBucket(ctx context.Context, dir string) (*blob.Bucket, error) {
 	var bucket *blob.Bucket
-	var err error
-	if strings.HasPrefix(fs.storageURL, "s3://") {
-		sess, err := fs.getS3Session()
+
+	u, err := url.Parse(fs.storageURL)
+	if err != nil {
+		return nil, err
+	}
+	if u.Scheme == s3blob.Scheme {
+		sess, rest, err := gcaws.NewSessionFromURLParams(u.Query())
 		if err != nil {
+			return nil, fmt.Errorf("open bucket %v: %v", u, err)
+		}
+		configProvider := &gcaws.ConfigOverrider{
+			Base: sess,
+		}
+		overrideCfg, err := gcaws.ConfigFromURLParams(rest)
+		if err != nil {
+			return nil, fmt.Errorf("open bucket %v: %v", u, err)
+		}
+
+		var insecureTLS bool
+		if overrideCfg.Endpoint != nil {
+			u, err := url.Parse(*overrideCfg.Endpoint)
+			if err != nil {
+				return nil, err
+			}
+			// use InsecureSkipVerify, if IP address is used for baseURL host
+			if ip := net.ParseIP(u.Hostname()); ip != nil && u.Scheme == "https" {
+				insecureTLS = true
+			}
+		}
+		if err := configureTLS(overrideCfg, fs.CACert, insecureTLS); err != nil {
 			return nil, err
 		}
-		bucket, err = s3blob.OpenBucket(ctx, sess, fs.storageURL, nil)
+
+		configProvider.Configs = append(configProvider.Configs, overrideCfg)
+
+		bucket, err = s3blob.OpenBucket(ctx, configProvider, u.Host, nil)
 		if err != nil {
 			return nil, err
 		}
@@ -167,35 +192,6 @@ func (fs *BlobFS) OpenBucket(ctx context.Context, dir string) (*blob.Bucket, err
 	return blob.PrefixedBucket(bucket, prefix), nil
 }
 
-func (fs *BlobFS) getS3Session() (*session.Session, error) {
-	config := &aws.Config{
-		Region:                        aws.String(fs.Region),
-		CredentialsChainVerboseErrors: aws.Bool(true),
-		Endpoint:                      aws.String(fs.Endpoint),
-		S3ForcePathStyle:              aws.Bool(true),
-	}
-	if err := configureTLS(config, fs.CACert, fs.InsecureTLS); err != nil {
-		return nil, err
-	}
-
-	sess := session.Must(session.NewSession())
-	config.WithCredentials(credentials.NewChainCredentials([]credentials.Provider{
-		&credentials.EnvProvider{},
-		&credentials.SharedCredentialsProvider{},
-		// Required for IRSA
-		stscreds.NewWebIdentityRoleProviderWithOptions(
-			sts.New(sess),
-			os.Getenv(awsRoleArn),
-			"",
-			stscreds.FetchTokenPath(os.Getenv(awsWebIdentityTokenFile)),
-		),
-		&ec2rolecreds.EC2RoleProvider{
-			Client: ec2metadata.New(sess),
-		},
-	}))
-	return session.NewSession(config)
-}
-
 func configureTLS(config *aws.Config, caCert []byte, insecureTLS bool) error {
 	tlsConfig := &tls.Config{
 		InsecureSkipVerify: insecureTLS,