Add locally persistent caching of input files

src/index.html

@@ -131,7 +131,7 @@ <h3>Encoder settings</h3>
  </div>
 
  <div class="mb-3">
- <div>Input video loading progress:</div>
+ <div>Input video loading progress (<span id="loaded-bytes">-</span>/<span id="total-bytes">-</span> bytes loaded):</div>
 
  <div class="progress">
  <div id="loading-progress" class="progress-bar" role="progressbar" style="width: 0%" aria-label="loading progress"></div>

src/index.ts

@@ -10,17 +10,32 @@ const DEFAULT_FRAMERATE = 30;
 const TOTAL_TIME_MICROS = 596380000; 
 
 const encoderProgressControl = $('#encoder-progress');
-const loadingProgressControl = $('#loading-progress');
-
 const exportTimeText = $('#export-time');
 const outputSizeText = $('#output-size');
 
-function setProgress(timestampMicros: number, durationSeconds: number, outputSizeBytes: number): void {
+
+function setExportProgress(timestampMicros: number, durationSeconds: number, outputSizeBytes: number): void {
  encoderProgressControl.css('width', `${Math.round(timestampMicros / TOTAL_TIME_MICROS * 100)}%`);
  exportTimeText.text(`${durationSeconds.toFixed(3)}`);
  outputSizeText.text(`${outputSizeBytes}`);
 }
 
+const loadingProgressControl = $('#loading-progress');
+const loadedBytesText = $('#loaded-bytes');
+const totalBytesText = $('#total-bytes');
+
+function setLoadingProgress(loadedBytes: number, totalBytes: number): void {
+ if (totalBytes === 0) {
+ loadingProgressControl.css('width', `0%`);
+ loadedBytesText.text('-');
+ totalBytesText.text('-');
+ } else {
+ loadingProgressControl.css('width', `${Math.round(loadedBytes / totalBytes * 100)}%`);
+ loadedBytesText.text(`${loadedBytes}`);
+ totalBytesText.text(`${totalBytes}`);
+ }
+}
+
 $('button#run-benchmark').click(async () => {
  $('button#run-benchmark').attr('disabled', 'disabled');
  $('select').attr('disabled', 'disabled');
@@ -45,9 +60,7 @@ $('button#run-benchmark').click(async () => {
  exportTimeText.val('Loading video.');
  outputSizeText.text('0');
 
- const inputFile = await loadInputFile(inputFileName, readProgress => {
- loadingProgressControl.css('width', `${(readProgress * 100).toFixed(1)}%`);
- });
+ const inputFile = await loadInputFile(inputFileName, setLoadingProgress);
  exportTimeText.val('Running...');
 
  const startTimeMillis = performance.now();
@@ -86,13 +99,14 @@ $('button#run-benchmark').click(async () => {
  ++encodedPackets;
 
  if (showEncodingProgress) {
- setProgress(chunk.timestamp, performance.now() - startTimeMillis, outputSizeBytes);
+ setExportProgress(chunk.timestamp, performance.now() - startTimeMillis, outputSizeBytes);
  }
  }
 
  const durationSeconds = (performance.now() - startTimeMillis) / 1000;
 
- setProgress(0, durationSeconds, outputSizeBytes);
+ setLoadingProgress(0, 0);
+ setExportProgress(0, durationSeconds, outputSizeBytes);
  alert(`Benchmark finished in ${durationSeconds.toFixed(3)} seconds.\n${decodedFrames} frames decoded, ${encodedPackets} packets encoded (${outputSizeBytes} bytes).`);
  $('button#run-benchmark').removeAttr('disabled');
  $('select').removeAttr('disabled');

src/shared/file-cache.ts

@@ -0,0 +1,124 @@
+const DEFAULT_QUOTA = 2 * 1024 * 1024 * 1024; // 2 GB
+const INMEMORY_CACHE: { [ fileName: string]: File} = {};
+
+interface WindowWithFileSystem extends Window {
+ readonly TEMPORARY: number;
+ readonly PERSISTENT: number;
+
+ webkitRequestFileSystem(
+ type: WindowWithFileSystem["PERSISTENT"] | WindowWithFileSystem["TEMPORARY"],
+ size: number,
+ successCallback: (fs: FileSystem) => void,
+ errorCallback: (error: Error) => void
+ ): void;
+}
+
+function isWindowWithFileSystem(window: Window): window is WindowWithFileSystem {
+ return (
+ 'TEMPORARY' in window && typeof window.TEMPORARY === 'number' &&
+ 'PERSISTENT' in window && typeof window.PERSISTENT === 'number' &&
+ 'webkitRequestFileSystem' in window && typeof window.webkitRequestFileSystem === 'function'
+ );
+}
+
+export interface FileWriter {
+ seek(position: number): void;
+ write(data: Blob): void;
+ truncate(length: number): void;
+
+ onabort: null | ((event: Event) => void);
+ onerror: null | ((event: Event) => void);
+ onwrite: null | ((event: Event) => void);
+ onwriteend: null | ((event: Event) => void);
+}
+
+interface FileSystemFileEntryWithWriter extends FileSystemFileEntry {
+ createWriter(sucessCallback: (writer: FileWriter) => void, errorCallback: (error: Error) => void): void;
+}
+
+function isFileSystemFileEntryWithWriter(entry: FileSystemFileEntry): entry is FileSystemFileEntryWithWriter {
+ return 'createWriter' in entry && typeof entry.createWriter === 'function';
+}
+
+
+function requestFileSystem(window: WindowWithFileSystem): Promise<FileSystem> {
+ return new Promise<FileSystem>((resolve, reject) => {
+ window.webkitRequestFileSystem(window.TEMPORARY, DEFAULT_QUOTA, resolve, reject);
+ });
+}
+
+export async function lookupFile(name: string): Promise<File | undefined> {
+ if (!isWindowWithFileSystem(window)) {
+ return INMEMORY_CACHE[name];
+ }
+
+ const fs = await requestFileSystem(window);
+
+ const entry = await new Promise<FileSystemFileEntry | undefined>((resolve, reject) => {
+ fs.root.getFile(name, {
+ create: false,
+ }, entry => {
+ if (entry.isFile) {
+ resolve(entry as FileSystemFileEntry);
+ } else {
+ reject(new Error(`Requested to lookup file "${name}", which is a directory.`))
+ }
+ }, error => {
+ console.warn(`Error looking up file ${name}: ${error.message}`);
+ resolve(undefined);
+ });
+ });
+
+ if (entry === undefined) {
+ // Not found in cache
+ return undefined;
+ }
+
+ return await new Promise<File>((resolve, reject) => {
+ entry.file(resolve, reject);
+ });
+}
+
+export async function cacheFile(file: File): Promise<void> {
+ if (!isWindowWithFileSystem(window)) {
+ INMEMORY_CACHE[file.name] = file;
+ return;
+ }
+
+ const fs = await requestFileSystem(window);
+
+ const entry = await new Promise<FileSystemFileEntry>((resolve, reject) => {
+ fs.root.getFile(file.name, {
+ create: true,
+ exclusive: false,
+ }, entry => {
+ if (entry.isFile) {
+ resolve(entry as FileSystemFileEntry);
+ } else {
+ reject(new Error(`Requested to create file "${file.name}", got a non-file instead.`))
+ }
+ }, reject);
+ });
+
+ if (!isFileSystemFileEntryWithWriter(entry)) {
+ throw new Error('Incomplete legacy FileSystem API - cannot create writer');
+ }
+
+ const writer = await new Promise<FileWriter>((resolve, reject) => {
+ entry.createWriter(resolve, reject);
+ });
+
+ await new Promise<void>((resolve, reject) => {
+ writer.onerror = event => {
+ console.error('Error caching file', event);
+ reject(new Error(`Error caching file: ${event}`));
+ };
+
+ writer.onwrite = () => {
+ resolve();
+ };
+
+ writer.seek(0);
+ writer.write(file);
+ });
+}

src/shared/input-files.ts

@@ -1,14 +1,14 @@
+import { cacheFile, lookupFile } from "./file-cache";
+
 const AZURE_BLOBSTORE_ENDPOINT = 'https://exportperformance.blob.core.windows.net';
 const AZURE_BLOBSTORE_CONTAINER = 'videos';
 
-const CACHED_VIDEO_FILES: { [ fileName: string]: File } = {};
+export async function loadInputFile(inputFileName: string, progressCallback: (loaded: number, size: number) => void): Promise<File> {
+ const cachedFile = await lookupFile(inputFileName);
 
-export async function loadInputFile(inputFileName: string, progressCallback?: (progress: number) => void): Promise<File> {
- if (inputFileName in CACHED_VIDEO_FILES) {
- if (progressCallback !== undefined) {
- progressCallback(1);
- }
- return CACHED_VIDEO_FILES[inputFileName];
+ if (cachedFile !== undefined) {
+ progressCallback(cachedFile.size, cachedFile.size);
+ return cachedFile;
  }
 
  const response = await fetch(`${AZURE_BLOBSTORE_ENDPOINT}/${AZURE_BLOBSTORE_CONTAINER}/${inputFileName}`);
@@ -22,6 +22,7 @@ export async function loadInputFile(inputFileName: string, progressCallback?: (p
  }
 
  const fileSize = Number.parseInt(response.headers.get('Content-Length')!);
+ progressCallback(0, fileSize);
 
  const reader = response.body.getReader();
 
@@ -33,13 +34,13 @@ export async function loadInputFile(inputFileName: string, progressCallback?: (p
  length += value.byteLength;
  chunks.push(value);
 
- if (progressCallback !== undefined) {
- progressCallback(length / fileSize);
- }
+ progressCallback(length, fileSize);
  }
  }
 
- const file = new File(chunks, inputFileName, { type: response.headers.get('Content-Type') ?? 'video/mp4' });
- CACHED_VIDEO_FILES[inputFileName] = file;
- return file;
+ const downloadedFile = new File(chunks, inputFileName, { type: response.headers.get('Content-Type') ?? 'video/mp4' });
+
+ await cacheFile(downloadedFile);
+
+ return downloadedFile;
 }