Use archive tokens when downloading blobs when possible

close : #8069

Co-authored-by: ivk <ivk@tutao.de>

schemas/storage.json

@@ -1,4 +1,15 @@
 {
 	"app": "storage",
-	"versions": []
+	"versions": [
+		{
+			"version": 10,
+			"changes": [
+				{
+					"name": "AddValue",
+					"sourceType": "BlobServerAccessInfo",
+					"info": "AddValue BlobServerAccessInfo/tokenKind/208."
+				}
+			]
+		}
+	]
 }

src/common/api/common/TutanotaConstants.ts

@@ -1233,6 +1233,11 @@ export enum PublicKeyIdentifierType {
 	GROUP_ID = "1",
 }
 
+export enum BlobAccessTokenKind {
+	Archive = "0",
+	Instances = "1",
+}
+
 export function asPublicKeyIdentifier(maybe: NumberString): PublicKeyIdentifierType {
 	if (Object.values(PublicKeyIdentifierType).includes(maybe as PublicKeyIdentifierType)) {
 		return maybe as PublicKeyIdentifierType

src/common/api/entities/storage/ModelInfo.ts

@@ -1,6 +1,6 @@
 const modelInfo = {
-	version: 9,
-	compatibleSince: 0,
+	version: 10,
+	compatibleSince: 10,
 }
 
 export default modelInfo

src/common/api/entities/storage/TypeModels.js

@@ -57,7 +57,7 @@ export const typeModels = {
 			}
 		},
 		"app": "storage",
-		"version": "9"
+		"version": "10"
 	},
 	"BlobAccessTokenPostOut": {
 		"name": "BlobAccessTokenPostOut",
@@ -91,7 +91,7 @@ export const typeModels = {
 			}
 		},
 		"app": "storage",
-		"version": "9"
+		"version": "10"
 	},
 	"BlobArchiveRef": {
 		"name": "BlobArchiveRef",
@@ -152,7 +152,7 @@ export const typeModels = {
 			}
 		},
 		"app": "storage",
-		"version": "9"
+		"version": "10"
 	},
 	"BlobGetIn": {
 		"name": "BlobGetIn",
@@ -204,7 +204,7 @@ export const typeModels = {
 			}
 		},
 		"app": "storage",
-		"version": "9"
+		"version": "10"
 	},
 	"BlobId": {
 		"name": "BlobId",
@@ -236,7 +236,7 @@ export const typeModels = {
 		},
 		"associations": {},
 		"app": "storage",
-		"version": "9"
+		"version": "10"
 	},
 	"BlobPostOut": {
 		"name": "BlobPostOut",
@@ -268,7 +268,7 @@ export const typeModels = {
 		},
 		"associations": {},
 		"app": "storage",
-		"version": "9"
+		"version": "10"
 	},
 	"BlobReadData": {
 		"name": "BlobReadData",
@@ -320,7 +320,7 @@ export const typeModels = {
 			}
 		},
 		"app": "storage",
-		"version": "9"
+		"version": "10"
 	},
 	"BlobReferenceDeleteIn": {
 		"name": "BlobReferenceDeleteIn",
@@ -381,7 +381,7 @@ export const typeModels = {
 			}
 		},
 		"app": "storage",
-		"version": "9"
+		"version": "10"
 	},
 	"BlobReferencePutIn": {
 		"name": "BlobReferencePutIn",
@@ -442,7 +442,7 @@ export const typeModels = {
 			}
 		},
 		"app": "storage",
-		"version": "9"
+		"version": "10"
 	},
 	"BlobServerAccessInfo": {
 		"name": "BlobServerAccessInfo",
@@ -479,6 +479,15 @@ export const typeModels = {
 				"type": "Date",
 				"cardinality": "One",
 				"encrypted": false
+			},
+			"tokenKind": {
+				"final": false,
+				"name": "tokenKind",
+				"id": 208,
+				"since": 10,
+				"type": "Number",
+				"cardinality": "One",
+				"encrypted": false
 			}
 		},
 		"associations": {
@@ -494,7 +503,7 @@ export const typeModels = {
 			}
 		},
 		"app": "storage",
-		"version": "9"
+		"version": "10"
 	},
 	"BlobServerUrl": {
 		"name": "BlobServerUrl",
@@ -526,7 +535,7 @@ export const typeModels = {
 		},
 		"associations": {},
 		"app": "storage",
-		"version": "9"
+		"version": "10"
 	},
 	"BlobWriteData": {
 		"name": "BlobWriteData",
@@ -558,7 +567,7 @@ export const typeModels = {
 		},
 		"associations": {},
 		"app": "storage",
-		"version": "9"
+		"version": "10"
 	},
 	"InstanceId": {
 		"name": "InstanceId",
@@ -590,6 +599,6 @@ export const typeModels = {
 		},
 		"associations": {},
 		"app": "storage",
-		"version": "9"
+		"version": "10"
 	}
 }

src/common/api/entities/storage/TypeRefs.ts

@@ -146,6 +146,7 @@ export type BlobServerAccessInfo = {
 	_id: Id;
 	blobAccessToken: string;
 	expires: Date;
+	tokenKind: NumberString;
 
 	servers: BlobServerUrl[];
 }

src/common/api/worker/facades/BlobAccessTokenFacade.ts

@@ -1,12 +1,12 @@
-import { ArchiveDataType } from "../../common/TutanotaConstants"
+import { ArchiveDataType, BlobAccessTokenKind } from "../../common/TutanotaConstants"
 import { assertWorkerOrNode } from "../../common/Env"
 import { BlobAccessTokenService } from "../../entities/storage/Services"
 import { IServiceExecutor } from "../../common/ServiceRequest"
 import { BlobServerAccessInfo, createBlobAccessTokenPostIn, createBlobReadData, createBlobWriteData, createInstanceId } from "../../entities/storage/TypeRefs"
 import { DateProvider } from "../../common/DateProvider.js"
 import { resolveTypeReference } from "../../common/EntityFunctions.js"
 import { AuthDataProvider } from "./UserFacade.js"
-import { deduplicate, getFirstOrThrow, isEmpty, lazyMemoized, TypeRef } from "@tutao/tutanota-utils"
+import { deduplicate, first, isEmpty, lazyMemoized, TypeRef } from "@tutao/tutanota-utils"
 import { ProgrammingError } from "../../common/error/ProgrammingError.js"
 import { BlobLoadOptions } from "./lazy/BlobFacade.js"
 import { BlobReferencingInstance } from "../../common/utils/BlobUtils.js"
@@ -21,16 +21,14 @@ assertWorkerOrNode()
  */
 export class BlobAccessTokenFacade {
 	// cache for blob access tokens that are valid for the whole archive (key:<archiveId>)
-	private readonly readArchiveCache: BlobAccessTokenCache<string>
 	// cache for blob access tokens that are valid for blobs from a given instance were the user does not own the archive (key:<instanceElementId>).
-	private readonly readBlobCache: BlobAccessTokenCache<string>
+	private readonly readCache: BlobAccessTokenCache
 	// cache for upload requests are valid for the whole archive (key:<ownerGroup + archiveDataType>).
-	private readonly writeCache: BlobAccessTokenCache<string>
+	private readonly writeCache: BlobAccessTokenCache
 
 	constructor(private readonly serviceExecutor: IServiceExecutor, private readonly authDataProvider: AuthDataProvider, dateProvider: DateProvider) {
-		this.readArchiveCache = new BlobAccessTokenCache<Id>(dateProvider)
-		this.readBlobCache = new BlobAccessTokenCache<Id>(dateProvider)
-		this.writeCache = new BlobAccessTokenCache<string>(dateProvider)
+		this.readCache = new BlobAccessTokenCache(dateProvider)
+		this.writeCache = new BlobAccessTokenCache(dateProvider)
 	}
 
 	/**
@@ -51,7 +49,7 @@ export class BlobAccessTokenFacade {
 			return blobAccessInfo
 		}
 		const key = this.makeWriteCacheKey(ownerGroupId, archiveDataType)
-		return this.writeCache.getToken(key, requestNewToken)
+		return this.writeCache.getToken(key, [], requestNewToken)
 	}
 
 	private makeWriteCacheKey(ownerGroupId: string, archiveDataType: ArchiveDataType) {
@@ -65,7 +63,7 @@ export class BlobAccessTokenFacade {
 	 */
 	evictWriteToken(archiveDataType: ArchiveDataType, ownerGroupId: Id): void {
 		const key = this.makeWriteCacheKey(ownerGroupId, archiveDataType)
-		this.writeCache.evict(key)
+		this.writeCache.evictArchiveOrGroupKey(key)
 	}
 
 	/**
@@ -90,8 +88,9 @@ export class BlobAccessTokenFacade {
 			throw new ProgrammingError("All referencing instances must be part of the same list")
 		}
 
+		const archiveId = this.getArchiveId(referencingInstances)
+
 		const requestNewToken = lazyMemoized(async () => {
-			const archiveId = this.getArchiveId(referencingInstances)
 			const instanceIds = referencingInstances.map(({ elementId }) => createInstanceId({ instanceId: elementId }))
 			const tokenRequest = createBlobAccessTokenPostIn({
 				archiveDataType,
@@ -106,7 +105,8 @@ export class BlobAccessTokenFacade {
 			return blobAccessInfo
 		})
 
-		return this.readBlobCache.getTokenMultiple(
+		return this.readCache.getToken(
+			archiveId,
 			referencingInstances.map((instance) => instance.elementId),
 			requestNewToken,
 		)
@@ -124,8 +124,8 @@ export class BlobAccessTokenFacade {
 		referencingInstance: BlobReferencingInstance,
 		blobLoadOptions: BlobLoadOptions,
 	): Promise<BlobServerAccessInfo> {
+		const archiveId = this.getArchiveId([referencingInstance])
 		const requestNewToken = async () => {
-			const archiveId = this.getArchiveId([referencingInstance])
 			const instanceListId = referencingInstance.listId
 			const instanceId = referencingInstance.elementId
 			const instanceIds = [createInstanceId({ instanceId })]
@@ -141,23 +141,27 @@ export class BlobAccessTokenFacade {
 			const { blobAccessInfo } = await this.serviceExecutor.post(BlobAccessTokenService, tokenRequest, blobLoadOptions)
 			return blobAccessInfo
 		}
-		return this.readBlobCache.getToken(referencingInstance.elementId, requestNewToken)
+		return this.readCache.getToken(archiveId, [referencingInstance.elementId], requestNewToken)
 	}
 
 	/**
 	 * Remove a given read blobs token from the cache.
 	 * @param referencingInstance
 	 */
 	evictReadBlobsToken(referencingInstance: BlobReferencingInstance): void {
-		this.readBlobCache.evict(referencingInstance.elementId)
+		this.readCache.evictInstanceId(referencingInstance.elementId)
+		const archiveId = this.getArchiveId([referencingInstance])
+		this.readCache.evictArchiveOrGroupKey(archiveId)
 	}
 
 	/**
 	 * Remove a given read blobs token from the cache.
 	 * @param referencingInstances
 	 */
 	evictReadBlobsTokenMultipleBlobs(referencingInstances: BlobReferencingInstance[]): void {
-		this.readBlobCache.evictAll(referencingInstances.map((instance) => instance.elementId))
+		this.readCache.evictAll(referencingInstances.map((instance) => instance.elementId))
+		const archiveId = this.getArchiveId(referencingInstances)
+		this.readCache.evictArchiveOrGroupKey(archiveId)
 	}
 
 	/**
@@ -178,15 +182,15 @@ export class BlobAccessTokenFacade {
 			const { blobAccessInfo } = await this.serviceExecutor.post(BlobAccessTokenService, tokenRequest)
 			return blobAccessInfo
 		}
-		return this.readArchiveCache.getToken(archiveId, requestNewToken)
+		return this.readCache.getToken(archiveId, [], requestNewToken)
 	}
 
 	/**
 	 * Remove a given read archive token from the cache.
 	 * @param archiveId
 	 */
 	evictArchiveToken(archiveId: Id): void {
-		this.readArchiveCache.evict(archiveId)
+		this.readCache.evictArchiveOrGroupKey(archiveId)
 	}
 
 	private getArchiveId(referencingInstances: readonly BlobReferencingInstance[]): Id {
@@ -237,41 +241,54 @@ function canBeUsedForAnotherRequest(blobServerAccessInfo: BlobServerAccessInfo,
 	return blobServerAccessInfo.expires.getTime() > dateProvider.now()
 }
 
-class BlobAccessTokenCache<IdType> {
-	private cache: Map<IdType, BlobServerAccessInfo>
-	private dateProvider: DateProvider
+class BlobAccessTokenCache {
+	private readonly instanceMap: Map<Id, BlobServerAccessInfo> = new Map()
+	private readonly archiveMap: Map<Id, BlobServerAccessInfo> = new Map()
 
-	constructor(dateProvider: DateProvider) {
-		this.cache = new Map<IdType, BlobServerAccessInfo>()
-		this.dateProvider = dateProvider
-	}
+	constructor(private readonly dateProvider: DateProvider) {}
 
-	public async getToken(id: IdType, loader: () => Promise<BlobServerAccessInfo>): Promise<BlobServerAccessInfo> {
-		return this.getTokenMultiple([id], loader)
-	}
-
-	public async getTokenMultiple(ids: IdType[], loader: () => Promise<BlobServerAccessInfo>): Promise<BlobServerAccessInfo> {
-		const tokens = deduplicate(ids.map((id) => this.cache.get(id) ?? null))
-		const firstTokenFound = getFirstOrThrow(tokens)
+	/**
+	 * Get a token from the cache or from {@param loader}.
+	 * First will try to use the token keyed by {@param archiveOrGroupKey}, otherwise it will try to find a token valid for all of {@param instanceIds}.
+	 */
+	public async getToken(
+		archiveOrGroupKey: Id | null,
+		instanceIds: readonly Id[],
+		loader: () => Promise<BlobServerAccessInfo>,
+	): Promise<BlobServerAccessInfo> {
+		const archiveToken = archiveOrGroupKey ? this.archiveMap.get(archiveOrGroupKey) : null
+		if (archiveToken != null && canBeUsedForAnotherRequest(archiveToken, this.dateProvider)) {
+			return archiveToken
+		}
 
+		const tokens = deduplicate(instanceIds.map((id) => this.instanceMap.get(id) ?? null))
+		const firstTokenFound = first(tokens)
 		if (tokens.length != 1 || firstTokenFound == null || !canBeUsedForAnotherRequest(firstTokenFound, this.dateProvider)) {
 			const newToken = await loader()
-			for (const id of ids) {
-				this.cache.set(id, newToken)
+			if (archiveOrGroupKey != null && newToken.tokenKind === BlobAccessTokenKind.Archive) {
+				this.archiveMap.set(archiveOrGroupKey, newToken)
+			} else {
+				for (const id of instanceIds) {
+					this.instanceMap.set(id, newToken)
+				}
 			}
 			return newToken
 		} else {
 			return firstTokenFound
 		}
 	}
 
-	public evict(id: IdType): void {
+	public evictInstanceId(id: Id): void {
 		this.evictAll([id])
 	}
 
-	public evictAll(ids: IdType[]): void {
+	public evictArchiveOrGroupKey(id: Id): void {
+		this.archiveMap.delete(id)
+	}
+
+	public evictAll(ids: Id[]): void {
 		for (const id of ids) {
-			this.cache.delete(id)
+			this.instanceMap.delete(id)
 		}
 	}
 }