diff --git a/adapter.ts b/adapter.ts index 34679a1..d958743 100644 --- a/adapter.ts +++ b/adapter.ts @@ -3,8 +3,11 @@ import { URL } from 'url'; import { Readable } from 'stream'; import { OutgoingHttpHeaders } from 'http'; import { NatureLanguage } from './uplog'; +import { KodoHttpClient } from './kodo-http-client'; export abstract class Adapter { + abstract readonly client: KodoHttpClient; + abstract storageClasses: StorageClass[]; abstract enter(sdkApiName: string, f: (scope: Adapter) => Promise, sdkUplogOption?: EnterUplogOption): Promise; diff --git a/kodo-http-client.ts b/kodo-http-client.ts index 2c98633..e2ac617 100644 --- a/kodo-http-client.ts +++ b/kodo-http-client.ts @@ -49,9 +49,15 @@ export interface SharedRequestOptions extends AdapterOption { apiType: 'kodo' | 's3', } +const regionsCache: Map = new Map(); +const regionsCacheLock = new AsyncLock(); + export class KodoHttpClient { - private readonly regionsCache: { [key: string]: Region; } = {}; - private readonly regionsCacheLock = new AsyncLock(); + static clearCache(): void { + regionsCache.clear(); + RegionService.clearCache(); + } + private readonly regionService: RegionService; private static logClientId: string | undefined = undefined; private readonly uplogBuffer: UplogBuffer; @@ -69,7 +75,12 @@ export class KodoHttpClient { } async call(options: RequestOptions): Promise> { - const urls = await this.getServiceUrls(options.serviceName, options.bucketName, options.s3RegionId, options.stats); + const urls = await this.getServiceUrls({ + serviceName: options.serviceName, + bucketName: options.bucketName, + s3RegionId: options.s3RegionId, + stats: options.stats + }); return await this.callUrls(urls, { method: options.method, path: options.path, @@ -95,50 +106,82 @@ export class KodoHttpClient { } clearCache() { - Object.keys(this.regionsCache).forEach((key) => { delete this.regionsCache[key]; }); - this.regionService.clearCache(); + KodoHttpClient.clearCache(); } - private async getServiceUrls( + private async getRegion( + serviceName: ServiceName, + bucketName?: string, + s3RegionId?: string, + stats?: RequestStats, + ): Promise { + let key: string; + if (s3RegionId) { + key = `${this.sharedOptions.ucUrl}/${s3RegionId}`; + } else { + key = `${this.sharedOptions.ucUrl}/${this.sharedOptions.accessKey}/${bucketName}`; + } + if (serviceName === ServiceName.UpAcc) { + key += `/${ServiceName.UpAcc}`; + } + const cachedRegion = regionsCache.get(key); + if (cachedRegion?.validated) { + return cachedRegion; + } + return await regionsCacheLock.acquire(key, async (): Promise => { + // re-check cache by others may fetch it + const cachedRegion = regionsCache.get(key); + if (cachedRegion?.validated) { + return cachedRegion; + } + + try { + const fetchedRegion = await this.fetchRegion( + bucketName, + s3RegionId, + stats, + ); + if ( + bucketName && + serviceName === ServiceName.UpAcc && + !fetchedRegion.upAccUrls?.length + ) { + // no upAccUrls, do not cache + return fetchedRegion; + } + regionsCache.set(key, fetchedRegion); + return fetchedRegion; + } catch (err) { + // when err, still return expired cached region + if (cachedRegion && !cachedRegion.validated) { + return cachedRegion; + } + throw err; + } + }); + } + + async getServiceUrls({ + serviceName, + bucketName, + s3RegionId, + stats, + withFallback = true, + }:{ serviceName: ServiceName, bucketName?: string, s3RegionId?: string, stats?: RequestStats, - ): Promise { - let key: string; - if (s3RegionId) { - key = `${this.sharedOptions.ucUrl}/${s3RegionId}`; - } else { - key = `${this.sharedOptions.ucUrl}/${this.sharedOptions.accessKey}/${bucketName}`; - } - if (this.regionsCache[key]?.validated) { - return this.getUrlsFromRegion(serviceName, this.regionsCache[key]); - } - const region: Region = await this.regionsCacheLock.acquire(key, async (): Promise => { - // re-check cache by others may fetch it - const cachedRegion = this.regionsCache[key]; - if (cachedRegion?.validated) { - return cachedRegion; - } - - try { - const fetchedRegion = await this.fetchRegion( - bucketName, - s3RegionId, - stats, - ); - this.regionsCache[key] = fetchedRegion; - return fetchedRegion; - } catch (err) { - // when err, still return expired cached region - if (cachedRegion && !cachedRegion.validated) { - return cachedRegion; - } - throw err; - } - }); + withFallback?: boolean, + }): Promise { + const region = await this.getRegion( + serviceName, + bucketName, + s3RegionId, + stats, + ); - return this.getUrlsFromRegion(serviceName, region); + return this.getUrlsFromRegion(serviceName, region, withFallback); } private async fetchRegion( @@ -231,15 +274,25 @@ export class KodoHttpClient { } } - private getUrlsFromRegion(serviceName: ServiceName, region: Region): string[] { + private getUrlsFromRegion( + serviceName: ServiceName, + region: Region, + withFallback = true, + ): string[] { switch (serviceName) { case ServiceName.Up: return [...region.upUrls]; case ServiceName.UpAcc: + if (!withFallback) { + return [...region.upAccUrls]; + } if (!region.upAccUrls.length) { return [...region.upUrls]; } - return [...region.upAccUrls]; + return [ + ...region.upAccUrls, + ...region.upUrls, + ]; case ServiceName.Uc: return [...region.ucUrls]; case ServiceName.Rs: diff --git a/kodo.ts b/kodo.ts index f325b9b..3ac6776 100644 --- a/kodo.ts +++ b/kodo.ts @@ -55,7 +55,7 @@ interface KodoAdapterOption { export class Kodo implements Adapter { storageClasses: StorageClass[] = []; - protected readonly client: KodoHttpClient; + readonly client: KodoHttpClient; protected readonly regionService: RegionService; protected bucketDomainsCache: Record = {}; protected bucketDomainsCacheLock = new AsyncLock(); diff --git a/package.json b/package.json index 45bba67..59e3487 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "kodo-s3-adapter-sdk", - "version": "0.6.1", + "version": "0.7.0", "description": "Adapter for Kodo & S3 API", "main": "dist/index.js", "repository": "github.com/bachue/kodo-s3-adapter-sdk", diff --git a/region.ts b/region.ts index 10948c9..82d4dc0 100644 --- a/region.ts +++ b/region.ts @@ -62,12 +62,16 @@ export class Region { readonly label?: string, readonly translatedLabels: { [lang: string]: string; } = {}, readonly storageClasses: RegionStorageClass[] = [], - readonly ttl: number = 0, + public ttl: number = 0, readonly createTime: number = Date.now(), ) {} get validated(): boolean { - return Date.now() < (this.createTime + this.ttl * 1000); + if (this.ttl < 0) { + return true; + } + const liveTime = Math.round((Date.now() - this.createTime) / 1000); + return liveTime < this.ttl; } private static requestAll(options: GetAllOptions): Promise> { diff --git a/region_service.ts b/region_service.ts index cf03cee..b7b7a54 100644 --- a/region_service.ts +++ b/region_service.ts @@ -14,6 +14,9 @@ const regionCache: Map = new Map(); const queryRegionLock = new AsyncLock(); export class RegionService { + static clearCache() { + regionCache.clear(); + } constructor(private readonly adapterOption: AdapterOption) { } @@ -59,7 +62,7 @@ export class RegionService { } clearCache() { - regionCache.clear(); + RegionService.clearCache(); } async getS3Endpoint(s3RegionId?: string, options?: GetAllRegionsOptions): Promise {