From 82fceb3ade6cbad6aa75c7e47e513d8047404c97 Mon Sep 17 00:00:00 2001 From: Avi Deitcher Date: Thu, 7 Nov 2024 15:00:53 +0200 Subject: [PATCH] standardize S3 endpoint handling and add option for path-style Signed-off-by: Avi Deitcher --- cmd/root.go | 6 +++- docs/configuration.md | 12 ++++--- go.mod | 3 +- go.sum | 38 ---------------------- pkg/config/local.go | 4 +++ pkg/storage/credentials/creds.go | 1 + pkg/storage/parse.go | 3 ++ pkg/storage/s3/s3.go | 56 +++++++++++++------------------- 8 files changed, 44 insertions(+), 79 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 2248d957..f21350bf 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -51,11 +51,13 @@ func rootCmd(execs execs) (*cobra.Command, error) { Short: "backup or restore one or more mysql-compatible databases", Long: `Backup or restore one or more mysql-compatible databases. In addition to the provided command-line flag options and environment variables, - when using s3-storage, supports the standard AWS options: + when using s3-storage, supports the following AWS options: AWS_ACCESS_KEY_ID: AWS Key ID AWS_SECRET_ACCESS_KEY: AWS Secret Access Key AWS_REGION: Region in which the bucket resides + AWS_ENDPOINT_URL: Endpoint URL to use instead of default s3..amazonaws.com + AWS_PATH_STYLE: Use path-style URLs for S3 requests instead of virtual-hosted-style URLs `, PersistentPreRunE: func(c *cobra.Command, args []string) error { bindFlags(cmd, v) @@ -146,6 +148,7 @@ func rootCmd(execs execs) (*cobra.Command, error) { cmdConfig.creds = credentials.Creds{ AWS: credentials.AWSCreds{ Endpoint: v.GetString("aws-endpoint-url"), + PathStyle: v.GetBool("aws-path-style"), AccessKeyID: v.GetString("aws-access-key-id"), SecretAccessKey: v.GetString("aws-secret-access-key"), Region: v.GetString("aws-region"), @@ -187,6 +190,7 @@ func rootCmd(execs execs) (*cobra.Command, error) { // aws options pflags.String("aws-endpoint-url", "", "Specify an alternative endpoint for s3 interoperable systems e.g. Digitalocean; ignored if not using s3.") + pflags.Bool("aws-path-style", false, "Use path-style addressing of buckets instead of default virtual-host-style; ignored if not using s3.") pflags.String("aws-access-key-id", "", "Access Key for s3 and s3 interoperable systems; ignored if not using s3.") pflags.String("aws-secret-access-key", "", "Secret Access Key for s3 and s3 interoperable systems; ignored if not using s3.") pflags.String("aws-region", "", "Region for s3 and s3 interoperable systems; ignored if not using s3.") diff --git a/docs/configuration.md b/docs/configuration.md index 787f65ba..d5a5e747 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -84,6 +84,7 @@ The following are the environment variables, CLI flags and configuration file op | AWS secret access key, used only if a target does not have one | BRP | `aws-secret-access-key` | `AWS_SECRET_ACCESS_KEY` | `dump.targets[s3-target].secretAccessKey` | | | AWS default region, used only if a target does not have one | BRP | `aws-region` | `AWS_REGION` | `dump.targets[s3-target].region` | | | alternative endpoint URL for S3-interoperable systems, used only if a target does not have one | BR | `aws-endpoint-url` | `AWS_ENDPOINT_URL` | `dump.targets[s3-target].endpoint` | | +| path-style addressing for S3 bucket instead of default virtual-host-style addressing | BR | `aws-path-style` | `AWS_PATH_STYLE` | `dump.targets[s3-target].pathStyle` | | | SMB username, used only if a target does not have one | BRP | `smb-user` | `SMB_USER` | `dump.targets[smb-target].username` | | | SMB password, used only if a target does not have one | BRP | `smb-pass` | `SMB_PASS` | `dump.targets[smb-target].password` | | | compression to use, one of: `bzip2`, `gzip` | BP | `compression` | `DB_DUMP_COMPRESSION` | `dump.compression` | `gzip` | @@ -159,12 +160,13 @@ for details of each. * Type s3: * `region`: the region * `endpoint`: the endpoint - * `accessKeyId`: the access key ID (s3) - * `secretAccessKey`: the secret access key (s3) + * `pathStyle` (boolean): use path-style bucket addressing instead of virtual-host style bucket addressing, see [AWS docs](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html) + * `accessKeyId`: the access key ID + * `secretAccessKey`: the secret access key * Type smb: - * `domain`: the domain (smb) - * `username`: the username (smb) - * `password`: the password (smb) + * `domain`: the domain + * `username`: the username + * `password`: the password * `logging`: the log level, one of: error,warning,info,debug,trace; default is info * `telemetry`: configuration for sending telemetry data (optional) * `url`: URL to telemetry service diff --git a/go.mod b/go.mod index e80fd92b..9b427921 100644 --- a/go.mod +++ b/go.mod @@ -34,7 +34,6 @@ require ( github.com/felixge/httpsnoop v1.0.3 // indirect github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/sys/user v0.2.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 // indirect @@ -59,7 +58,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/sso v1.24.3 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.3 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.32.3 // indirect - github.com/aws/smithy-go v1.22.0 + github.com/aws/smithy-go v1.22.0 // indirect github.com/containerd/containerd v1.7.11 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/docker/go-units v0.5.0 // indirect diff --git a/go.sum b/go.sum index 5d14bd3c..f285ad3e 100644 --- a/go.sum +++ b/go.sum @@ -14,80 +14,42 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/aws/aws-sdk-go v1.44.256 h1:O8VH+bJqgLDguqkH/xQBFz5o/YheeZqgcOYIgsTVWY4= github.com/aws/aws-sdk-go v1.44.256/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= -github.com/aws/aws-sdk-go-v2 v1.30.3 h1:jUeBtG0Ih+ZIFH0F4UkmL9w3cSpaMv9tYYDbzILP8dY= -github.com/aws/aws-sdk-go-v2 v1.30.3/go.mod h1:nIQjQVp5sfpQcTc9mPSr1B0PaWK5ByX9MOoDadSN4lc= github.com/aws/aws-sdk-go-v2 v1.32.3 h1:T0dRlFBKcdaUPGNtkBSwHZxrtis8CQU17UpNBZYd0wk= github.com/aws/aws-sdk-go-v2 v1.32.3/go.mod h1:2SK5n0a2karNTv5tbP1SjsX0uhttou00v/HpXKM1ZUo= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3 h1:tW1/Rkad38LA15X4UQtjXZXNKsCgkshC3EbmcUmghTg= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3/go.mod h1:UbnqO+zjqk3uIt9yCACHJ9IVNhyhOCnYk8yA19SAWrM= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 h1:pT3hpW0cOHRJx8Y0DfJUEQuqPild8jRGmSFmBgvydr0= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6/go.mod h1:j/I2++U0xX+cr44QjHay4Cvxj6FUbnxrgmqN3H1jTZA= -github.com/aws/aws-sdk-go-v2/config v1.27.27 h1:HdqgGt1OAP0HkEDDShEl0oSYa9ZZBSOmKpdpsDMdO90= -github.com/aws/aws-sdk-go-v2/config v1.27.27/go.mod h1:MVYamCg76dFNINkZFu4n4RjDixhVr51HLj4ErWzrVwg= github.com/aws/aws-sdk-go-v2/config v1.28.1 h1:oxIvOUXy8x0U3fR//0eq+RdCKimWI900+SV+10xsCBw= github.com/aws/aws-sdk-go-v2/config v1.28.1/go.mod h1:bRQcttQJiARbd5JZxw6wG0yIK3eLeSCPdg6uqmmlIiI= -github.com/aws/aws-sdk-go-v2/credentials v1.17.27 h1:2raNba6gr2IfA0eqqiP2XiQ0UVOpGPgDSi0I9iAP+UI= -github.com/aws/aws-sdk-go-v2/credentials v1.17.27/go.mod h1:gniiwbGahQByxan6YjQUMcW4Aov6bLC3m+evgcoN4r4= github.com/aws/aws-sdk-go-v2/credentials v1.17.42 h1:sBP0RPjBU4neGpIYyx8mkU2QqLPl5u9cmdTWVzIpHkM= github.com/aws/aws-sdk-go-v2/credentials v1.17.42/go.mod h1:FwZBfU530dJ26rv9saAbxa9Ej3eF/AK0OAY86k13n4M= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 h1:KreluoV8FZDEtI6Co2xuNk/UqI9iwMrOx/87PBNIKqw= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11/go.mod h1:SeSUYBLsMYFoRvHE0Tjvn7kbxaUhl75CJi1sbfhMxkU= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.18 h1:68jFVtt3NulEzojFesM/WVarlFpCaXLKaBxDpzkQ9OQ= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.18/go.mod h1:Fjnn5jQVIo6VyedMc0/EhPpfNlPl7dHV916O6B+49aE= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.9 h1:TC2vjvaAv1VNl9A0rm+SeuBjrzXnrlwk6Yop+gKRi38= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.9/go.mod h1:WPv2FRnkIOoDv/8j2gSUsI4qDc7392w5anFB/I89GZ8= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.35 h1:ihPPdcCVSN0IvBByXwqVp28/l4VosBZ6sDulcvU2J7w= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.35/go.mod h1:JkgEhs3SVF51Dj3m1Bj+yL8IznpxzkwlA3jLg3x7Kls= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 h1:SoNJ4RlFEQEbtDcCEt+QG56MY4fm4W8rYirAmq+/DdU= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15/go.mod h1:U9ke74k1n2bf+RIgoX1SXFed1HLs51OgUSs+Ph0KJP8= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.22 h1:Jw50LwEkVjuVzE1NzkhNKkBf9cRN7MtE1F/b2cOKTUM= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.22/go.mod h1:Y/SmAyPcOTmpeVaWSzSKiILfXTVJwrGmYZhcRbhWuEY= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15 h1:C6WHdGnTDIYETAm5iErQUiVNsclNx9qbJVPIt03B6bI= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15/go.mod h1:ZQLZqhcu+JhSrA9/NXRm8SkDvsycE+JkV3WGY41e+IM= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.22 h1:981MHwBaRZM7+9QSR6XamDzF/o7ouUGxFzr+nVSIhrs= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.22/go.mod h1:1RA1+aBEfn+CAB/Mh0MB6LsdCYCnjZm7tKXtnk499ZQ= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.15 h1:Z5r7SycxmSllHYmaAZPpmN8GviDrSGhMS6bldqtXZPw= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.15/go.mod h1:CetW7bDE00QoGEmPUoZuRog07SGVAUVW6LFpNP0YfIg= github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.22 h1:yV+hCAHZZYJQcwAaszoBNwLbPItHvApxT0kVIw6jRgs= github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.22/go.mod h1:kbR1TL8llqB1eGnVbybcA4/wgScxdylOdyAd51yxPdw= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 h1:dT3MqvGhSoaIhRseqw2I0yH81l7wiR2vjs57O51EAm8= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3/go.mod h1:GlAeCkHwugxdHaueRr4nhPuY+WW+gR8UjlcqzPr1SPI= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 h1:TToQNkvGguu209puTojY/ozlqy2d/SFNcoLIqTFi42g= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0/go.mod h1:0jp+ltwkf+SwG2fm/PKo8t4y8pJSgOCO4D8Lz3k0aHQ= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.17 h1:YPYe6ZmvUfDDDELqEKtAd6bo8zxhkm+XEFEzQisqUIE= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.17/go.mod h1:oBtcnYua/CgzCWYN7NZ5j7PotFDaFSUjCYVTtfyn7vw= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.3 h1:kT6BcZsmMtNkP/iYMcRG+mIEA/IbeiUimXtGmqF39y0= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.3/go.mod h1:Z8uGua2k4PPaGOYn66pK02rhMrot3Xk3tpBuUFPomZU= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 h1:HGErhhrxZlQ044RiM+WdoZxp0p+EGM62y3L6pwA4olE= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17/go.mod h1:RkZEx4l0EHYDJpWppMJ3nD9wZJAa8/0lq9aVC+r2UII= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.3 h1:qcxX0JYlgWH3hpPUnd6U0ikcl6LLA9sLkXE2w1fpMvY= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.3/go.mod h1:cLSNEmI45soc+Ef8K/L+8sEA3A3pYFEYf5B5UI+6bH4= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.15 h1:246A4lSTXWJw/rmlQI+TT2OcqeDMKBdyjEQrafMaQdA= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.15/go.mod h1:haVfg3761/WF7YPuJOER2MP0k4UAXyHaLclKXB6usDg= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.3 h1:ZC7Y/XgKUxwqcdhO5LE8P6oGP1eh6xlQReWNKfhvJno= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.3/go.mod h1:WqfO7M9l9yUAw0HcHaikwRd/H6gzYdz7vjejCA5e2oY= -github.com/aws/aws-sdk-go-v2/service/s3 v1.58.2 h1:sZXIzO38GZOU+O0C+INqbH7C2yALwfMWpd64tONS/NE= -github.com/aws/aws-sdk-go-v2/service/s3 v1.58.2/go.mod h1:Lcxzg5rojyVPU/0eFwLtcyTaek/6Mtic5B1gJo7e/zE= github.com/aws/aws-sdk-go-v2/service/s3 v1.66.2 h1:p9TNFL8bFUMd+38YIpTAXpoxyz0MxC7FlbFEH4P4E1U= github.com/aws/aws-sdk-go-v2/service/s3 v1.66.2/go.mod h1:fNjyo0Coen9QTwQLWeV6WO2Nytwiu+cCcWaTdKCAqqE= -github.com/aws/aws-sdk-go-v2/service/sso v1.22.4 h1:BXx0ZIxvrJdSgSvKTZ+yRBeSqqgPM89VPlulEcl37tM= -github.com/aws/aws-sdk-go-v2/service/sso v1.22.4/go.mod h1:ooyCOXjvJEsUw7x+ZDHeISPMhtwI3ZCB7ggFMcFfWLU= github.com/aws/aws-sdk-go-v2/service/sso v1.24.3 h1:UTpsIf0loCIWEbrqdLb+0RxnTXfWh2vhw4nQmFi4nPc= github.com/aws/aws-sdk-go-v2/service/sso v1.24.3/go.mod h1:FZ9j3PFHHAR+w0BSEjK955w5YD2UwB/l/H0yAK3MJvI= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4 h1:yiwVzJW2ZxZTurVbYWA7QOrAaCYQR72t0wrSBfoesUE= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4/go.mod h1:0oxfLkpz3rQ/CHlx5hB7H69YUpFiI1tql6Q6Ne+1bCw= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.3 h1:2YCmIXv3tmiItw0LlYf6v7gEHebLY45kBEnPezbUKyU= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.3/go.mod h1:u19stRyNPxGhj6dRm+Cdgu6N75qnbW7+QN0q0dsAk58= -github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 h1:ZsDKRLXGWHk8WdtyYMoGNO7bTudrvuKpDKgMVRlepGE= -github.com/aws/aws-sdk-go-v2/service/sts v1.30.3/go.mod h1:zwySh8fpFyXp9yOr/KVzxOl8SRqgf/IDw5aUt9UKFcQ= github.com/aws/aws-sdk-go-v2/service/sts v1.32.3 h1:wVnQ6tigGsRqSWDEEyH6lSAJ9OyFUsSnbaUWChuSGzs= github.com/aws/aws-sdk-go-v2/service/sts v1.32.3/go.mod h1:VZa9yTFyj4o10YGsmDO4gbQJUvvhY72fhumT8W4LqsE= -github.com/aws/smithy-go v1.20.3 h1:ryHwveWzPV5BIof6fyDvor6V3iUL7nTfiTKXHiW05nE= -github.com/aws/smithy-go v1.20.3/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= github.com/aws/smithy-go v1.22.0 h1:uunKnWlcoL3zO7q+gG2Pk53joueEOsnNB28QdMsmiMM= github.com/aws/smithy-go v1.22.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= diff --git a/pkg/config/local.go b/pkg/config/local.go index 3004d013..904d5811 100644 --- a/pkg/config/local.go +++ b/pkg/config/local.go @@ -133,6 +133,7 @@ type S3Target struct { URL string `yaml:"url"` Region string `yaml:"region"` Endpoint string `yaml:"endpoint"` + PathStyle bool `yaml:"pathStyle"` Credentials AWSCredentials `yaml:"credentials"` } @@ -148,6 +149,9 @@ func (s S3Target) Storage() (storage.Storage, error) { if s.Endpoint != "" { opts = append(opts, s3.WithEndpoint(s.Endpoint)) } + if s.PathStyle { + opts = append(opts, s3.WithPathStyle()) + } if s.Credentials.AccessKeyId != "" { opts = append(opts, s3.WithAccessKeyId(s.Credentials.AccessKeyId)) } diff --git a/pkg/storage/credentials/creds.go b/pkg/storage/credentials/creds.go index 68745a5f..e361127c 100644 --- a/pkg/storage/credentials/creds.go +++ b/pkg/storage/credentials/creds.go @@ -15,5 +15,6 @@ type AWSCreds struct { AccessKeyID string SecretAccessKey string Endpoint string + PathStyle bool Region string } diff --git a/pkg/storage/parse.go b/pkg/storage/parse.go index 13975b51..d5126b26 100644 --- a/pkg/storage/parse.go +++ b/pkg/storage/parse.go @@ -48,6 +48,9 @@ func ParseURL(url string, creds credentials.Creds) (Storage, error) { if creds.AWS.SecretAccessKey != "" { opts = append(opts, s3.WithSecretAccessKey(creds.AWS.SecretAccessKey)) } + if creds.AWS.PathStyle { + opts = append(opts, s3.WithPathStyle()) + } store = s3.New(*u, opts...) default: return nil, fmt.Errorf("unknown url protocol: %s", u.Scheme) diff --git a/pkg/storage/s3/s3.go b/pkg/storage/s3/s3.go index 450af656..d187dc34 100644 --- a/pkg/storage/s3/s3.go +++ b/pkg/storage/s3/s3.go @@ -15,7 +15,6 @@ import ( "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/feature/s3/manager" "github.com/aws/aws-sdk-go-v2/service/s3" - smithyendpoints "github.com/aws/smithy-go/endpoints" log "github.com/sirupsen/logrus" ) @@ -190,36 +189,45 @@ func (s *S3) Remove(target string, logger *log.Entry) error { func (s *S3) getClient(logger *log.Entry) (*s3.Client, error) { // Get the AWS config - var ( - opts []func(*config.LoadOptions) error // global client options - s3opts []func(*s3.Options) // s3 client options - ) - if s.endpoint != "" { - cleanEndpoint := getEndpoint(s.endpoint) - s3opts = append(s3opts, - s3.WithEndpointResolverV2(&staticResolver{endpoint: cleanEndpoint}), - ) - } + var configOpts []func(*config.LoadOptions) error // global client options if logger.Level == log.TraceLevel { - opts = append(opts, config.WithClientLogMode(aws.LogRequestWithBody|aws.LogResponse)) + configOpts = append(configOpts, config.WithClientLogMode(aws.LogRequestWithBody|aws.LogResponse)) } if s.region != "" { - opts = append(opts, config.WithRegion(s.region)) + configOpts = append(configOpts, config.WithRegion(s.region)) } if s.accessKeyId != "" { - opts = append(opts, config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider( + configOpts = append(configOpts, config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider( s.accessKeyId, s.secretAccessKey, "", ))) } cfg, err := config.LoadDefaultConfig(context.TODO(), - opts..., + configOpts..., ) if err != nil { return nil, fmt.Errorf("failed to load AWS config: %v", err) } + // Get the S3 client + var s3opts []func(*s3.Options) // s3 client options + if s.endpoint != "" { + cleanEndpoint := getEndpoint(s.endpoint) + s3opts = append(s3opts, + func(o *s3.Options) { + o.BaseEndpoint = &cleanEndpoint + }, + ) + } + if s.pathStyle { + s3opts = append(s3opts, + func(o *s3.Options) { + o.UsePathStyle = true + }, + ) + } + // Create a new S3 service client return s3.NewFromConfig(cfg, s3opts...), nil } @@ -243,24 +251,6 @@ func getEndpoint(endpoint string) string { return e } -// endpointResolver is a custom endpoint resolver that always returns the same endpoint. -type staticResolver struct { - endpoint string -} - -func (s *staticResolver) ResolveEndpoint(ctx context.Context, params s3.EndpointParameters) ( - smithyendpoints.Endpoint, error, -) { - // This value will be used as-is when making the request. - u, err := url.Parse(s.endpoint) - if err != nil { - return smithyendpoints.Endpoint{}, err - } - return smithyendpoints.Endpoint{ - URI: *u, - }, nil -} - type s3FileInfo struct { name string lastModified time.Time