-
Notifications
You must be signed in to change notification settings - Fork 532
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Use mailbox export tokens, Extract MailExportTokenFacade
#8098 Make `createTestEntity` deterministic and fix tests related to it. Co-authored-by: bir <bir@tutao.de>
- Loading branch information
Showing
22 changed files
with
483 additions
and
401 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| import { File as TutanotaFile } from "../../entities/tutanota/TypeRefs.js" | ||
| import { elementIdPart, listIdPart } from "./EntityUtils.js" | ||
| import { Blob } from "../../entities/sys/TypeRefs.js" | ||
| import { SomeEntity } from "../EntityTypes.js" | ||
|
|
||
| /** | ||
| * Common interface for instances that are referencing blobs. Main purpose is to have a proper way to access the attribute for the Blob aggregated type | ||
| * because the name of the attribute can be different for each instance. | ||
| * | ||
| */ | ||
| export type BlobReferencingInstance = { | ||
| elementId: Id | ||
|
|
||
| listId: Id | null | ||
|
|
||
| blobs: Blob[] | ||
|
|
||
| entity: SomeEntity | ||
| } | ||
|
|
||
| export function createReferencingInstance(tutanotaFile: TutanotaFile): BlobReferencingInstance { | ||
| return { | ||
| blobs: tutanotaFile.blobs, | ||
| elementId: elementIdPart(tutanotaFile._id), | ||
| listId: listIdPart(tutanotaFile._id), | ||
| entity: tutanotaFile, | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
156 changes: 45 additions & 111 deletions
156
src/common/api/worker/facades/lazy/MailExportFacade.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,142 +1,76 @@ | ||
| import { MailExportTokenService } from "../../../entities/tutanota/Services" | ||
| import { File as TutanotaFile, Mail } from "../../../entities/tutanota/TypeRefs" | ||
| import { assertWorkerOrNode } from "../../../common/Env" | ||
| import { IServiceExecutor } from "../../../common/ServiceRequest" | ||
| import { EntityClient } from "../../../common/EntityClient" | ||
| import { CacheMode, EntityRestClientLoadOptions } from "../../rest/EntityRestClient" | ||
| import type { ListElementEntity, SomeEntity } from "../../../common/EntityTypes" | ||
| import { assertNotNull, TypeRef } from "@tutao/tutanota-utils" | ||
| import { AccessExpiredError } from "../../../common/error/RestError" | ||
| import { isNotNull, promiseMap } from "@tutao/tutanota-utils" | ||
| import { NotFoundError } from "../../../common/error/RestError" | ||
| import { BulkMailLoader, MailWithMailDetails } from "../../../../../mail-app/workerUtils/index/BulkMailLoader.js" | ||
| import { convertToDataFile, DataFile } from "../../../common/DataFile.js" | ||
| import { ArchiveDataType } from "../../../common/TutanotaConstants.js" | ||
| import { BlobFacade } from "./BlobFacade.js" | ||
| import { CryptoFacade } from "../../crypto/CryptoFacade.js" | ||
| import { createReferencingInstance } from "../../../common/utils/BlobUtils.js" | ||
| import { MailExportTokenFacade } from "./MailExportTokenFacade.js" | ||
|
|
||
| assertWorkerOrNode() | ||
|
|
||
| const TAG = "[MailExportFacade]" | ||
|
|
||
| /** | ||
| * Denotes the header that will have the mail export token. | ||
| */ | ||
| export const MAIL_EXPORT_TOKEN_HEADER = "mailExportToken" | ||
|
|
||
| /** | ||
| * Denotes an export token. This is internally just a string, but we want the TypeScript compiler to enforce strong | ||
| * typing. | ||
| */ | ||
| type MailExportToken = string & { _exportToken: undefined } | ||
|
|
||
| /** | ||
| * Mail exporter functions | ||
| * Wraps bulk loading of mails for mail export. | ||
| * | ||
| * This implements loadForMailGroup and loadRangeForMailGroup which uses mail export tokens retrieved from the server | ||
| * and does not write to cache. Note that no loadAll method is implemented since tokens expire after a short period of | ||
| * time, and it is better to process in batches. | ||
| * Takes care of using mail export tokens. | ||
| */ | ||
| export class MailExportFacade { | ||
| // This will only be set if a request is in progress | ||
| private currentExportTokenRequest: Promise<MailExportToken> | null = null | ||
| // Set when we have a known valid token | ||
| private currentExportToken: MailExportToken | null = null | ||
|
|
||
| constructor(private readonly serviceExecutor: IServiceExecutor, private readonly entityClient: EntityClient) {} | ||
|
|
||
| /** | ||
| * Load a single element for export, (re-)generating a mail export token if needed | ||
| */ | ||
| async load<T extends SomeEntity>(typeRef: TypeRef<T>, id: PropertyType<T, "_id">): Promise<T> { | ||
| return this.handleRequest((options) => this.entityClient.load(typeRef, id, options)) | ||
| constructor( | ||
| private readonly mailExportTokenFacade: MailExportTokenFacade, | ||
| private readonly bulkMailLoader: BulkMailLoader, | ||
| private readonly blobFacade: BlobFacade, | ||
| private readonly cryptoFacade: CryptoFacade, | ||
| ) {} | ||
|
|
||
| async loadFixedNumberOfMailsWithCache(mailListId: Id, startId: Id): Promise<Mail[]> { | ||
| return this.mailExportTokenFacade.loadWithToken((token) => | ||
| this.bulkMailLoader.loadFixedNumberOfMailsWithCache(mailListId, startId, this.options(token)), | ||
| ) | ||
| } | ||
|
|
||
| /** | ||
| * Load a multiple elements for export, (re-)generating a mail export token if needed | ||
| */ | ||
| async loadMultiple<T extends SomeEntity>(typeRef: TypeRef<T>, listId: Id | null, elementIds: Id[]): Promise<T[]> { | ||
| return this.handleRequest((options) => this.entityClient.loadMultiple(typeRef, listId, elementIds, undefined, options)) | ||
| async loadMailDetails(mails: readonly Mail[]): Promise<MailWithMailDetails[]> { | ||
| return this.mailExportTokenFacade.loadWithToken((token) => this.bulkMailLoader.loadMailDetails(mails, this.options(token))) | ||
| } | ||
|
|
||
| /** | ||
| * Load a range of elements for export, (re-)generating a mail export token if needed | ||
| */ | ||
| async loadRange<T extends ListElementEntity>(typeRef: TypeRef<T>, listId: Id, firstId: Id, count: number, reverse: boolean): Promise<T[]> { | ||
| return this.handleRequest((options) => this.entityClient.loadRange<T>(typeRef, listId, firstId, count, reverse, options)) | ||
| async loadAttachments(mails: readonly Mail[]): Promise<TutanotaFile[]> { | ||
| return this.mailExportTokenFacade.loadWithToken((token) => this.bulkMailLoader.loadAttachments(mails, this.options(token))) | ||
| } | ||
|
|
||
| /** | ||
| * Runs `request`. | ||
| * | ||
| * If `AccessExpiredError` is thrown, delete the cached token and re-run it again. | ||
| * @param request function to run | ||
| * @private | ||
| */ | ||
| private async handleRequest<T>(request: (options: EntityRestClientLoadOptions) => Promise<T>): Promise<T> { | ||
| const token = this.currentExportToken ?? (await this.requestNewToken()) | ||
| try { | ||
| const options = this.applyExportOptions(token) | ||
| return await request(options) | ||
| } catch (e) { | ||
| // We only allow one retry | ||
| if (e instanceof AccessExpiredError) { | ||
| let newToken | ||
| if (this.currentExportToken === token) { | ||
| console.log(TAG, `token expired for exporting and will be renewed`) | ||
| newToken = await this.requestNewToken() | ||
| async loadAttachmentData(mail: Mail, attachments: readonly TutanotaFile[]): Promise<DataFile[]> { | ||
| const attachmentsWithKeys = await this.cryptoFacade.enforceSessionKeyUpdateIfNeeded(mail, attachments) | ||
| // TODO: download attachments efficiently. | ||
| // - download multiple blobs at once if possible | ||
| // - use file references instead of data files (introduce a similar type to MailBundle or change MailBundle) | ||
| const attachmentData = await promiseMap(attachmentsWithKeys, async (attachment) => { | ||
| try { | ||
| const bytes = await this.mailExportTokenFacade.loadWithToken((token) => | ||
| this.blobFacade.downloadAndDecrypt(ArchiveDataType.Attachments, createReferencingInstance(attachment), this.options(token)), | ||
| ) | ||
| return convertToDataFile(attachment, bytes) | ||
| } catch (e) { | ||
| if (e instanceof NotFoundError) { | ||
| return null | ||
| } else { | ||
| // Already a request going on... wait for that to finish | ||
| newToken = this.currentExportToken ?? (await this.requestNewToken()) | ||
| throw e | ||
| } | ||
|
|
||
| const options = this.applyExportOptions(newToken) | ||
| return await request(options) | ||
| } else { | ||
| throw e | ||
| } | ||
| } | ||
| }) | ||
| return attachmentData.filter(isNotNull) | ||
| } | ||
|
|
||
| private applyExportOptions(token: MailExportToken): EntityRestClientLoadOptions { | ||
| private options(token: string): { extraHeaders: Dict } { | ||
| return { | ||
| cacheMode: CacheMode.ReadOnly, | ||
| extraHeaders: { | ||
| [MAIL_EXPORT_TOKEN_HEADER]: token, | ||
| }, | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Request a new token and write it to the tokenCache. | ||
| * | ||
| * This token will be valid for the mail group and current user for a short amount of time, after which you will get | ||
| * an `AccessExpiredError` when using the token (or `NotAuthorizedError` if the user lost access to the group in the | ||
| * meantime). | ||
| * @throws TooManyRequestsError the user cannot request any more tokens right now | ||
| * @return the token | ||
| */ | ||
| private requestNewToken(): Promise<MailExportToken> { | ||
| if (this.currentExportTokenRequest) { | ||
| return this.currentExportTokenRequest | ||
| } | ||
|
|
||
| this.currentExportToken = null | ||
| this.currentExportTokenRequest = this.serviceExecutor.post(MailExportTokenService, null).then( | ||
| (result) => { | ||
| this.currentExportToken = result.mailExportToken as MailExportToken | ||
| this.currentExportTokenRequest = null | ||
| return this.currentExportToken | ||
| }, | ||
| (error) => { | ||
| // Re-initialize in case MailExportTokenService won't fail on a future request | ||
| this.currentExportTokenRequest = null | ||
| throw error | ||
| }, | ||
| ) | ||
| return this.currentExportTokenRequest | ||
| } | ||
|
|
||
| // @VisibleForTesting | ||
| _setCurrentExportToken(token: string) { | ||
| this.currentExportToken = token as MailExportToken | ||
| this.currentExportTokenRequest = null | ||
| } | ||
|
|
||
| // @VisibleForTesting | ||
| _getCurrentExportToken(): string | null { | ||
| return this.currentExportToken | ||
| } | ||
| } |
Oops, something went wrong.