diff --git a/adapter.ts b/adapter.ts index d958743..f504ae2 100644 --- a/adapter.ts +++ b/adapter.ts @@ -261,8 +261,15 @@ export interface PutObjectOption { accelerateUploading?: boolean; } +export enum UrlStyle { + Path = 'path', + VirtualHost = 'virtualHost', + BucketEndpoint = 'bucketEndpoint', +} + export interface GetObjectStreamOption { rangeStart?: number; rangeEnd?: number; abortSignal?: AbortSignal; + urlStyle?: UrlStyle, } diff --git a/downloader.ts b/downloader.ts index ccc1ad4..0714f36 100644 --- a/downloader.ts +++ b/downloader.ts @@ -5,7 +5,7 @@ import { ThrottleGroup, ThrottleOptions } from 'stream-throttle'; import { Ref } from './types'; import { Progress, ProgressStream, SpeedMonitor } from './progress-stream'; -import { Adapter, Domain, ObjectHeader, StorageObject } from './adapter'; +import { Adapter, Domain, ObjectHeader, StorageObject, UrlStyle } from './adapter'; import { HttpClient } from './http-client'; export class Downloader { @@ -92,6 +92,7 @@ export class Downloader { domain, { rangeStart: recoveredFrom, + urlStyle: getFileOption?.urlStyle, }, ); pipeList.unshift(reader); @@ -154,6 +155,7 @@ export class Downloader { domain: Domain | undefined, option?: { rangeStart?: number, + urlStyle?: UrlStyle, }, ): Promise { // default values @@ -167,6 +169,7 @@ export class Downloader { { rangeStart: start, abortSignal: this.abortController?.signal, + urlStyle: option?.urlStyle, } ); } @@ -265,4 +268,5 @@ export interface GetFileOption { downloadThrottleGroup?: ThrottleGroup; downloadThrottleOption?: ThrottleOptions; getCallback?: GetCallback; + urlStyle?: UrlStyle; } diff --git a/kodo.ts b/kodo.ts index 1bc5eeb..be65ccb 100644 --- a/kodo.ts +++ b/kodo.ts @@ -41,6 +41,7 @@ import { StorageObject, TransferObject, UploadPartOutput, + UrlStyle, } from './adapter'; import { KodoHttpClient, RequestOptions, ServiceName } from './kodo-http-client'; import { RequestStats, URLRequestOptions } from './http-client'; @@ -676,7 +677,13 @@ export class Kodo implements Adapter { headers.Range = `bytes=${option?.rangeStart ?? ''}-${option?.rangeEnd ?? ''}`; } - const url = await this.getObjectURL(s3RegionId, object, domain); + const url = await this.getObjectURL( + s3RegionId, + object, + domain, + undefined, + option?.urlStyle, + ); const response = await this.callUrl( [ url.toString(), @@ -704,7 +711,7 @@ export class Kodo implements Adapter { object: StorageObject, domain?: Domain, deadline?: Date, - style: 'path' | 'virtualHost' | 'bucketEndpoint' = 'bucketEndpoint', + style: UrlStyle = UrlStyle.BucketEndpoint, ): Promise { if (!domain) { const domains = await this._listDomains(s3RegionId, object.bucket); @@ -714,7 +721,7 @@ export class Kodo implements Adapter { domain = domains[0]; } - if (style !== 'bucketEndpoint') { + if (style !== UrlStyle.BucketEndpoint) { throw new Error('Only support "bucketEndpoint" style for now'); } diff --git a/s3.ts b/s3.ts index ed51018..d5fd768 100644 --- a/s3.ts +++ b/s3.ts @@ -36,6 +36,7 @@ import { StorageObject, TransferObject, UploadPartOutput, + UrlStyle, } from './adapter'; import { ErrorRequestUplogEntry, @@ -65,8 +66,15 @@ export class S3 extends Kodo { protected clientsLock = new AsyncLock(); protected listKodoBucketsPromise?: Promise; - private async getClient(s3RegionId?: string, s3ForcePathStyle = true): Promise { - const cacheKey = [s3RegionId ?? '', s3ForcePathStyle ? 's3ForcePathStyle' : ''].join(':'); + /** + * if domain exists, the urlStyle will be forced to 'bucketEndpoint' + */ + private async getClient( + s3RegionId?: string, + urlStyle: UrlStyle = UrlStyle.Path, + domain?: Domain, + ): Promise { + const cacheKey = [s3RegionId ?? '', urlStyle, domain?.name ?? ''].join(':'); if (this.clients[cacheKey]) { return this.clients[cacheKey]; } @@ -77,12 +85,26 @@ export class S3 extends Kodo { userAgent += `/${this.adapterOption.appendedUserAgent}`; } const s3IdEndpoint = await this.regionService.getS3Endpoint(s3RegionId, this.getRegionRequestOptions()); + const urlStyleOptions: { + endpoint: string, + s3ForcePathStyle?: boolean, + s3BucketEndpoint?: boolean, + } = { + endpoint: !domain + ? s3IdEndpoint.s3Endpoint + : `${domain.protocol}://${domain.name}`, + }; + if (urlStyle === UrlStyle.BucketEndpoint) { + urlStyleOptions.s3BucketEndpoint = true; + } else { + urlStyleOptions.s3ForcePathStyle = urlStyle === UrlStyle.Path; + } + return new AWS.S3({ apiVersion: '2006-03-01', customUserAgent: userAgent, computeChecksums: true, region: s3IdEndpoint.s3Id, - endpoint: s3IdEndpoint.s3Endpoint, maxRetries: 10, signatureVersion: 'v4', useDualstack: true, @@ -94,11 +116,11 @@ export class S3 extends Kodo { httpOptions: { connectTimeout: 30000, timeout: 300000, - agent: s3IdEndpoint.s3Endpoint.startsWith('https://') + agent: urlStyleOptions.endpoint.startsWith('https://') ? HttpClient.httpsKeepaliveAgent : HttpClient.httpKeepaliveAgent, }, - s3ForcePathStyle + ...urlStyleOptions }); }); this.clients[cacheKey] = client; @@ -562,11 +584,11 @@ export class S3 extends Kodo { async getObjectStream( s3RegionId: string, object: StorageObject, - _domain?: Domain, + domain?: Domain, option?: GetObjectStreamOption, ): Promise { const [s3, bucketId] = await Promise.all([ - this.getClient(s3RegionId), + this.getClient(s3RegionId, option?.urlStyle, domain), this.fromKodoBucketNameToS3BucketId(object.bucket), ]); let range: string | undefined; @@ -595,33 +617,10 @@ export class S3 extends Kodo { object: StorageObject, domain?: Domain, deadline?: Date, - style: 'path' | 'virtualHost' | 'bucketEndpoint' = 'path', + style: UrlStyle = UrlStyle.Path, ): Promise { - let s3Promise: Promise; - // if domain is not undefined, use the domain, else use the default s3 endpoint - if (domain) { - if (style !== 'bucketEndpoint') { - throw new Error('Custom S3 endpoint only support "bucketEndpoint" style'); - } - s3Promise = Promise.resolve(new AWS.S3({ - apiVersion: '2006-03-01', - region: s3RegionId, - endpoint: `${domain.protocol}://${domain.name}`, - credentials: { - accessKeyId: this.adapterOption.accessKey, - secretAccessKey: this.adapterOption.secretKey, - }, - signatureVersion: 'v4', - s3BucketEndpoint: true, // use bucketEndpoint style - })); - } else { - if (style === 'bucketEndpoint') { - throw new Error('Default S3 endpoint not support "bucketEndpoint" style'); - } - s3Promise = this.getClient(s3RegionId, style === 'path'); - } const [s3, bucketId] = await Promise.all([ - s3Promise, + this.getClient(s3RegionId, style, domain), this.fromKodoBucketNameToS3BucketId(object.bucket), ]); const expires = deadline