ported the full output optimization code from VSCode (#13137)

* ported the full output optimization code from VSCode

Signed-off-by: Jonah Iden <jonah.iden@typefox.io>

* added commit id to licence header

Signed-off-by: Jonah Iden <jonah.iden@typefox.io>

* added file to comment

Signed-off-by: Jonah Iden <jonah.iden@typefox.io>

* fixed multiple outputs now updating correctly

Signed-off-by: Jonah Iden <jonah.iden@typefox.io>

---------

Signed-off-by: Jonah Iden <jonah.iden@typefox.io>

packages/notebook/src/browser/notebook-output-utils.ts

@@ -0,0 +1,119 @@
+// *****************************************************************************
+// Copyright (C) 2023 TypeFox and others.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+// *****************************************************************************
+/*---------------------------------------------------------------------------------------------
+ * Copyright (c) Microsoft Corporation. All rights reserved.
+ * Licensed under the MIT License. See License.txt in the project root for license information.
+ * Copied from commit 18b2c92451b076943e5b508380e0eba66ba7d934 from file src\vs\workbench\contrib\notebook\common\notebookCommon.ts
+ *--------------------------------------------------------------------------------------------*/
+
+import { BinaryBuffer } from '@theia/core/lib/common/buffer';
+
+const textDecoder = new TextDecoder();
+
+/**
+ * Given a stream of individual stdout outputs, this function will return the compressed lines, escaping some of the common terminal escape codes.
+ * E.g. some terminal escape codes would result in the previous line getting cleared, such if we had 3 lines and
+ * last line contained such a code, then the result string would be just the first two lines.
+ * @returns a single VSBuffer with the concatenated and compressed data, and whether any compression was done.
+ */
+export function compressOutputItemStreams(outputs: Uint8Array[]): { data: BinaryBuffer, didCompression: boolean } {
+ const buffers: Uint8Array[] = [];
+ let startAppending = false;
+
+ // Pick the first set of outputs with the same mime type.
+ for (const output of outputs) {
+ if ((buffers.length === 0 || startAppending)) {
+ buffers.push(output);
+ startAppending = true;
+ }
+ }
+
+ let didCompression = compressStreamBuffer(buffers);
+ const concatenated = BinaryBuffer.concat(buffers.map(buffer => BinaryBuffer.wrap(buffer)));
+ const data = formatStreamText(concatenated);
+ didCompression = didCompression || data.byteLength !== concatenated.byteLength;
+ return { data, didCompression };
+}
+
+export const MOVE_CURSOR_1_LINE_COMMAND = `${String.fromCharCode(27)}[A`;
+const MOVE_CURSOR_1_LINE_COMMAND_BYTES = MOVE_CURSOR_1_LINE_COMMAND.split('').map(c => c.charCodeAt(0));
+const LINE_FEED = 10;
+function compressStreamBuffer(streams: Uint8Array[]): boolean {
+ let didCompress = false;
+ streams.forEach((stream, index) => {
+ if (index === 0 || stream.length < MOVE_CURSOR_1_LINE_COMMAND.length) {
+ return;
+ }
+
+ const previousStream = streams[index - 1];
+
+ // Remove the previous line if required.
+ const command = stream.subarray(0, MOVE_CURSOR_1_LINE_COMMAND.length);
+ if (command[0] === MOVE_CURSOR_1_LINE_COMMAND_BYTES[0] && command[1] === MOVE_CURSOR_1_LINE_COMMAND_BYTES[1] && command[2] === MOVE_CURSOR_1_LINE_COMMAND_BYTES[2]) {
+ const lastIndexOfLineFeed = previousStream.lastIndexOf(LINE_FEED);
+ if (lastIndexOfLineFeed === -1) {
+ return;
+ }
+
+ didCompress = true;
+ streams[index - 1] = previousStream.subarray(0, lastIndexOfLineFeed);
+ streams[index] = stream.subarray(MOVE_CURSOR_1_LINE_COMMAND.length);
+ }
+ });
+ return didCompress;
+}
+
+const BACKSPACE_CHARACTER = '\b'.charCodeAt(0);
+const CARRIAGE_RETURN_CHARACTER = '\r'.charCodeAt(0);
+function formatStreamText(buffer: BinaryBuffer): BinaryBuffer {
+ // We have special handling for backspace and carriage return characters.
+ // Don't unnecessary decode the bytes if we don't need to perform any processing.
+ if (!buffer.buffer.includes(BACKSPACE_CHARACTER) && !buffer.buffer.includes(CARRIAGE_RETURN_CHARACTER)) {
+ return buffer;
+ }
+ // Do the same thing jupyter is doing
+ return BinaryBuffer.fromString(fixCarriageReturn(fixBackspace(textDecoder.decode(buffer.buffer))));
+}
+
+/**
+ * Took this from jupyter/notebook
+ * https://github.com/jupyter/notebook/blob/b8b66332e2023e83d2ee04f83d8814f567e01a4e/notebook/static/base/js/utils.js
+ * Remove characters that are overridden by backspace characters
+ */
+function fixBackspace(txt: string): string {
+ let tmp = txt;
+ do {
+ txt = tmp;
+ // Cancel out anything-but-newline followed by backspace
+ tmp = txt.replace(/[^\n]\x08/gm, '');
+ } while (tmp.length < txt.length);
+ return txt;
+}
+
+/**
+ * Remove chunks that should be overridden by the effect of carriage return characters
+ * From https://github.com/jupyter/notebook/blob/master/notebook/static/base/js/utils.js
+ */
+function fixCarriageReturn(txt: string): string {
+ txt = txt.replace(/\r+\n/gm, '\n'); // \r followed by \n --> newline
+ while (txt.search(/\r[^$]/g) > -1) {
+ const base = txt.match(/^(.*)\r+/m)![1];
+ let insert = txt.match(/\r+(.*)$/m)![1];
+ insert = insert + base.slice(insert.length, base.length);
+ txt = txt.replace(/\r+.*$/m, '\r').replace(/^.*\r/m, insert);
+ }
+ return txt;
+}

packages/notebook/src/browser/view-model/notebook-cell-output-model.ts

@@ -15,8 +15,8 @@
 // *****************************************************************************
 
 import { Disposable, Emitter } from '@theia/core';
-import { BinaryBuffer } from '@theia/core/lib/common/buffer';
 import { CellOutput, CellOutputItem, isTextStreamMime } from '../../common';
+import { compressOutputItemStreams } from '../notebook-output-utils';
 
 export class NotebookCellOutputModel implements Disposable {
 
@@ -73,24 +73,25 @@ export class NotebookCellOutputModel implements Disposable {
  if (this.outputs.length > 1 && this.outputs.every(item => isTextStreamMime(item.mime))) {
  // Look for the mimes in the items, and keep track of their order.
  // Merge the streams into one output item, per mime type.
- const mimeOutputs = new Map<string, BinaryBuffer[]>();
+ const mimeOutputs = new Map<string, Uint8Array[]>();
  const mimeTypes: string[] = [];
  this.outputs.forEach(item => {
- let items: BinaryBuffer[];
+ let items: Uint8Array[];
  if (mimeOutputs.has(item.mime)) {
  items = mimeOutputs.get(item.mime)!;
  } else {
  items = [];
  mimeOutputs.set(item.mime, items);
  mimeTypes.push(item.mime);
  }
- items.push(item.data);
+ items.push(item.data.buffer);
  });
  this.outputs.length = 0;
  mimeTypes.forEach(mime => {
+ const compressionResult = compressOutputItemStreams(mimeOutputs.get(mime)!);
  this.outputs.push({
  mime,
- data: BinaryBuffer.concat(mimeOutputs.get(mime)!)
+ data: compressionResult.data
  });
  });
  }

packages/plugin-ext/src/main/browser/notebooks/renderers/cell-output-webview.tsx

@@ -76,7 +76,7 @@ export class CellOutputWebviewImpl implements CellOutputWebview, Disposable {
  protected async init(): Promise<void> {
  this.cell.onDidChangeOutputs(outputChange => this.updateOutput(outputChange));
  this.cell.onDidChangeOutputItems(output => {
- this.updateOutput({start: this.cell.outputs.findIndex(o => o.getData().outputId === o.outputId), deleteCount: 1, newOutputs: [output]});
+ this.updateOutput({start: this.cell.outputs.findIndex(o => o.outputId === output.outputId), deleteCount: 1, newOutputs: [output]});
  });
 
  this.webviewWidget = await this.widgetManager.getOrCreateWidget(WebviewWidget.FACTORY_ID, { id: this.id });