L:: v.0.5.1

Use WebAssembly.Memory instead of raw ArrayBuffer. Typo correction in the build template: proper 'use strict;'. Better memory management in the bundle() method and in ByteStreamWriter. Docs update. Returned to the old behaviour: wait for the completion of a forgotten task, before sending the next to the Web Worker.
RussCoder · Jan 9, 2021 · 2e53947 · 2e53947
1 parent eb2dbad
commit 2e53947
Show file tree

Hide file tree

Showing 8 changed files with 68 additions and 42 deletions.
diff --git a/extension/manifest.json b/extension/manifest.json
@@ -2,7 +2,7 @@
     "manifest_version": 2,
     "name": "DjVu.js Viewer",
     "short_name": "DV",
-    "version": "0.5.3.0",
+    "version": "0.5.3.1",
     "author": "RussCoder",
     "homepage_url": "https://github.com/RussCoder/djvujs",
     "description": "Opens links to .djvu files. Allows opening .djvu files from a local disk. Processes djvu <object> & <embed> tags.",

diff --git a/library/API.md b/library/API.md
@@ -263,11 +263,11 @@ The `DjVuWorker` instance has the following methods and props:
   queue.
 - `isTaskInProcess(promise: Promise): boolean` - checks whether the task has
   been already started.
-- `revokeObjectURL(url: string): void` - the sad fact is that if an ObjectURL
-  has been created inside a worker it can be revoked only inside this very
-  worker. I checked it by myself. Maybe it will change in the future, but you
-  should prefer this method to the `URL.revokeObjectURL()` when you want to
-  revoke a URL from the worker (e.g. a URL to a PNG image or a whole document).
+- `revokeObjectURL(url: string): void` - formerly, if an ObjectURL had been
+  created inside a worker it could be revoked only inside this very worker. I
+  checked it by myself, but now it seems that this behavior has been fixed and
+  usual `URL.revokeObjectURL()` works too. But this method is still available if
+  you wish to revoke the URL inside the worker in which it was created.
 - `reset(): void` - recreates the worker.
 
 ## Additional Notes

diff --git a/library/CHANGELOG.md b/library/CHANGELOG.md
@@ -1,5 +1,14 @@
 # DjVu.js Library's Changelog
 
+## v.0.5.1 (09.01.2021)
+
+- More robust memory management for document creation (usage of
+  `WebAssembly.Memory` with its `grow()` method instead of manual `ArrayBuffer`
+  expansion).
+- `'use strict';` in the Web Worker (typo correction).
+- Returned to the old behaviour: wait for the completion of a forgotten task,
+  before sending the next to the Web Worker.
+
 ## v.0.5.0 (06.12.2020)
 
 - Feature: bundle indirect djvu documents.

diff --git a/library/rollup.config.js b/library/rollup.config.js
@@ -5,7 +5,7 @@ const commonjs = require('rollup-plugin-commonjs');
 const outputTemplate = {
     format: 'iife',
     name: 'DjVu',
-    intro: "function DjVuScript() {\n'use strict;'",
+    intro: "function DjVuScript() {\n'use strict';",
     outro: "}\nreturn Object.assign(DjVuScript(), {DjVuScript});"
 };
 

diff --git a/library/src/ByteStreamWriter.js b/library/src/ByteStreamWriter.js
@@ -1,15 +1,29 @@
 import { stringToCodePoints, codePointsToUtf8 } from './DjVu';
 
+const pageSize = 64 * 1024;
+const growthLimit = 20 * 1024 * 1024 / pageSize;
+
 export default class ByteStreamWriter {
-    constructor(length) {
-        //размер шага роста используемой памяти
-        this.growthStep = length || 4096; // not used now
-        this.buffer = new ArrayBuffer(this.growthStep);
-        this.viewer = new DataView(this.buffer);
+    constructor(length = 0) {
+        // As the practice has shown, usage of WebAssembly.Memory and its grow() method
+        // is more robust than the manual expansion of ArrayBuffer 
+        // via `new Uint8Array(newBuffer).set(new Uint8Array(oldBuffer))`.
+        // In particular, with WebAssembly.Memory it's possible to download and bundle
+        // a document that is about 1.7 GB in size, while with raw ArrayBuffers 
+        // a browser tab crashes (in Chrome) when the buffer reaches about 1.5 GB 
+        // (or there is an error that a buffer cannot be allocated).
+        this.memory = new WebAssembly.Memory({ initial: Math.ceil(length / pageSize), maximum: 65536 });
+        this.assignBufferFromMemory();
+
         this.offset = 0;
         this.offsetMarks = {};
     }
 
+    assignBufferFromMemory() {
+        this.buffer = this.memory.buffer;
+        this.viewer = new DataView(this.buffer);
+    }
+
     /**
      * Переводит смещение на начало и зачищает сохраненные смещения
      */
@@ -24,7 +38,7 @@ export default class ByteStreamWriter {
     }
 
     writeByte(byte) {
-        this.checkOffset();
+        this.checkOffset(1);
         this.viewer.setUint8(this.offset++, byte);
         return this;
     }
@@ -35,7 +49,7 @@ export default class ByteStreamWriter {
     }
 
     writeInt32(val) {
-        this.checkOffset(3);
+        this.checkOffset(4);
         this.viewer.setInt32(this.offset, val);
         this.offset += 4;
         return this;
@@ -71,33 +85,26 @@ export default class ByteStreamWriter {
     }
 
     checkOffset(requiredBytesNumber = 0) {
-        var bool = this.offset + requiredBytesNumber >= this.buffer.byteLength;
+        const bool = this.offset + requiredBytesNumber > this.buffer.byteLength;
         if (bool) {
             this._expand(requiredBytesNumber);
         }
         return bool;
     }
 
     _expand(requiredBytesNumber) {
-        //Globals.Timer.start("expandTime");
-
-        var newLength = 2 * this.buffer.byteLength; // perhaps it's not the best strategy but still it works
-        if (newLength < this.buffer.byteLength + requiredBytesNumber) {
-            newLength += requiredBytesNumber;
-        }
-        var nb = new ArrayBuffer(newLength);
-        new Uint8Array(nb).set(new Uint8Array(this.buffer)); // быстрое копирование ArrayBuffer
-        this.buffer = nb;
-        this.viewer = new DataView(this.buffer);
-
-        //Globals.Timer.end("expandTime");
+        this.memory.grow(Math.max(
+            Math.ceil(requiredBytesNumber / pageSize),
+            Math.min(this.memory.buffer.byteLength / pageSize, growthLimit)
+        ));
+        this.assignBufferFromMemory();
     }
 
     //смещение на length байт
     jump(length) {
         length = +length;
         if (length > 0) {
-            this.checkOffset(length - 1);
+            this.checkOffset(length);
         }
         this.offset += length;
         return this;
@@ -108,7 +115,7 @@ export default class ByteStreamWriter {
     }
 
     writeArray(arr) {
-        while (this.checkOffset(arr.length - 1)) { }
+        while (this.checkOffset(arr.length)) { }
         new Uint8Array(this.buffer).set(arr, this.offset);
         this.offset += arr.length;
     }
@@ -123,14 +130,14 @@ export default class ByteStreamWriter {
     }
 
     writeInt16(val) {
-        this.checkOffset(1);
+        this.checkOffset(2);
         this.viewer.setInt16(this.offset, val);
         this.offset += 2;
         return this;
     }
 
     writeUint16(val) {
-        this.checkOffset(1);
+        this.checkOffset(2);
         this.viewer.setUint16(this.offset, val);
         this.offset += 2;
         return this;

diff --git a/library/src/DjVu.js b/library/src/DjVu.js
@@ -1,5 +1,5 @@
 var DjVu = {
-    VERSION: '0.5.0',
+    VERSION: '0.5.1',
     IS_DEBUG: false,
     setDebugMode: (flag) => DjVu.IS_DEBUG = flag
 };

diff --git a/library/src/DjVuWorker.js b/library/src/DjVuWorker.js
@@ -66,10 +66,6 @@ export default class DjVuWorker {
         this.currentPromise = null;
         this.promiseCallbacks = null;
         this.currentCommandId = null;
-        // reset the flag, although the worker doesn't stop. But it's more robust,
-        // than to wait till the forgotten task finishes. Just because the message can not to come from the worker,
-        // e.g. when it contains data which cannot be cloned, like functions ((although it shouldn't happen).
-        this.isWorking = false;
     }
 
     emptyTaskQueue() {
@@ -165,15 +161,28 @@ export default class DjVuWorker {
     messageHandler({ data: obj }) {
         if (obj.action) return this.processAction(obj);
 
+        this.isWorking = false;
         const callbacks = this.promiseCallbacks;
         const commandId = obj.sendBackData && obj.sendBackData.commandId;
 
-        if (commandId === this.currentCommandId
-            || this.currentCommandId === null) { // technically, we can forget the current task, but still have a task queue
-            this.isWorking = false;
+        // either a result or a forgotten command returned
+        if (commandId === this.currentCommandId || this.currentCommandId === null) {
+            // in fact, this invocation is essential, since this.isWorking
+            // isn't reset when all tasks are cancelled.
+            // So we still wait for a cancelled task to finish - it's important, because otherwise
+            // cancelAllTasks() would have no sense - the real worker's queue would be overwhelmed with "current" tasks,
+            // which cannot be cancelled once sent, while now it's possible to really cancel all tasks several times
+            // while some other task is being processed in the worker.
+            // Real example: a user is quickly turning over pages in the single page mode in the viewer.
+            // commandIds only prevent us from forgetting current task
+            // in case when something comes from the worker and it's not an action
+            // (it shouldn't happen, just an additional measure)
             this.runNextTask();
         } else {
-            return; // in case if we cancel a task, but its result came afterwards
+            // it shouldn't happen, it means that one more task has been already sent
+            // without waiting for a forgotten one. Or an action is sent incorrectly.
+            console.warn('DjVu.js: Something strange came from the worker.', obj);
+            return;
         }
 
         if (!callbacks) return; // in case of all tasks cancellation

diff --git a/library/src/methods/bundle.js b/library/src/methods/bundle.js
@@ -77,8 +77,9 @@ export default async function bundle(progressCallback = () => { }) {
 
     progressCallback((totalOperations - 2) / totalOperations);
 
-    for (const chunkByteStream of chunkByteStreams) {
-        djvuWriter.writeFormChunkBS(chunkByteStream);
+    for (let i = 0; i < chunkByteStreams.length; i++) {
+        djvuWriter.writeFormChunkBS(chunkByteStreams[i]);
+        chunkByteStreams[i] = null; // release memory
     }
 
     progressCallback((totalOperations - 1) / totalOperations);