From 1f3a8c2658e9c2ddf5d3bb4fd9b25d07cf2b3e4a Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Mon, 25 Nov 2024 11:49:54 +0000 Subject: [PATCH] feat!: add new outputs node --- packages/myst-cli/src/process/mdast.ts | 2 + packages/myst-cli/src/process/notebook.ts | 21 +- .../src/transforms/crossReferences.ts | 13 +- packages/myst-cli/src/transforms/index.ts | 1 + .../myst-cli/src/transforms/outputs.spec.ts | 49 ++-- packages/myst-cli/src/transforms/outputs.ts | 209 ++++++++++-------- packages/myst-cli/src/transforms/schema.ts | 121 ++++++++++ packages/myst-directives/src/code.ts | 9 +- packages/myst-execute/src/execute.ts | 19 +- packages/myst-execute/tests/execute.yml | 209 ++++++------------ packages/myst-execute/tests/run.spec.ts | 1 + packages/myst-spec-ext/src/types.ts | 6 + .../notebook-fig-embed/outputs/index.json | 4 + 13 files changed, 398 insertions(+), 266 deletions(-) create mode 100644 packages/myst-cli/src/transforms/schema.ts diff --git a/packages/myst-cli/src/process/mdast.ts b/packages/myst-cli/src/process/mdast.ts index fb50a03cd..7d5dc691d 100644 --- a/packages/myst-cli/src/process/mdast.ts +++ b/packages/myst-cli/src/process/mdast.ts @@ -69,6 +69,7 @@ import { transformFilterOutputStreams, transformLiftCodeBlocksInJupytext, transformMystXRefs, + internalASTToExternal, } from '../transforms/index.js'; import type { ImageExtensions } from '../utils/resolveExtension.js'; import { logMessagesFromVFile } from '../utils/logging.js'; @@ -431,5 +432,6 @@ export async function finalizeMdast( postData.widgets = cache.$getMdast(file)?.pre.widgets; updateFileInfoFromFrontmatter(session, file, frontmatter); } + internalASTToExternal(mdast); logMessagesFromVFile(session, vfile); } diff --git a/packages/myst-cli/src/process/notebook.ts b/packages/myst-cli/src/process/notebook.ts index ce8323f39..60b01e39c 100644 --- a/packages/myst-cli/src/process/notebook.ts +++ b/packages/myst-cli/src/process/notebook.ts @@ -165,17 +165,20 @@ export async function processNotebookFull( value: ensureString(cell.source), }; - // Embed outputs in an output block - const output: { type: 'output'; id: string; data: IOutput[] } = { - type: 'output', + const outputsChildren = (cell.outputs as IOutput[]).map((output) => { + // Embed outputs in an output block + const result = { + type: 'output', + jupyter_data: output, + }; + return result; + }); + const outputs = { + type: 'outputs', id: nanoid(), - data: [], + children: outputsChildren, }; - - if (cell.outputs && (cell.outputs as IOutput[]).length > 0) { - output.data = cell.outputs as IOutput[]; - } - return acc.concat(blockParent(cell, [code, output])); + return acc.concat(blockParent(cell, [code, outputs])); } return acc; }, diff --git a/packages/myst-cli/src/transforms/crossReferences.ts b/packages/myst-cli/src/transforms/crossReferences.ts index 470c1a956..ba595a626 100644 --- a/packages/myst-cli/src/transforms/crossReferences.ts +++ b/packages/myst-cli/src/transforms/crossReferences.ts @@ -1,5 +1,6 @@ import type { VFile } from 'vfile'; import { selectAll } from 'unist-util-select'; +import { visit, SKIP } from 'unist-util-visit'; import type { FrontmatterParts, GenericNode, GenericParent, References } from 'myst-common'; import { RuleId, fileWarn, plural, selectMdastNodes } from 'myst-common'; import { computeHash, tic } from 'myst-cli-utils'; @@ -9,6 +10,8 @@ import type { CrossReference, Dependency, Link, SourceFileKind } from 'myst-spec import type { ISession } from '../session/types.js'; import { loadFromCache, writeToCache } from '../session/cache.js'; import type { SiteAction, SiteExport } from 'myst-config'; +import type { IOutput } from '@jupyterlab/nbformat'; +import { externalASTToInternal } from './schema.js'; export const XREF_MAX_AGE = 1; // in days @@ -32,6 +35,13 @@ export type MystData = { references?: References; }; +function upgradeAndDowngradeMystData(data: MystData): MystData { + if (data.mdast) { + externalASTToInternal(data.mdast); + } + return data; +} + async function fetchMystData( session: ISession, dataUrl: string | undefined, @@ -48,7 +58,8 @@ async function fetchMystData( try { const resp = await session.fetch(dataUrl); if (resp.ok) { - const data = (await resp.json()) as MystData; + const data = upgradeAndDowngradeMystData((await resp.json()) as MystData); + writeToCache(session, filename, JSON.stringify(data)); return data; } diff --git a/packages/myst-cli/src/transforms/index.ts b/packages/myst-cli/src/transforms/index.ts index 6bdd85242..8c26e3370 100644 --- a/packages/myst-cli/src/transforms/index.ts +++ b/packages/myst-cli/src/transforms/index.ts @@ -9,5 +9,6 @@ export * from './include.js'; export * from './links.js'; export * from './mdast.js'; export * from './outputs.js'; +export * from './schema.js'; export * from './inlineExpressions.js'; export * from './types.js'; diff --git a/packages/myst-cli/src/transforms/outputs.spec.ts b/packages/myst-cli/src/transforms/outputs.spec.ts index f15bdd674..361df3552 100644 --- a/packages/myst-cli/src/transforms/outputs.spec.ts +++ b/packages/myst-cli/src/transforms/outputs.spec.ts @@ -20,9 +20,14 @@ describe('reduceOutputs', () => { ], }, { - type: 'output', - id: 'abc123', - data: [], + type: 'outputs', + children: [ + { + type: 'output', + id: 'abc123', + jupyter_data: null, + }, + ], }, ], }, @@ -49,18 +54,21 @@ describe('reduceOutputs', () => { ], }, { - type: 'output', + type: 'outputs', id: 'abc123', - data: [ + children: [ { - output_type: 'display_data', - execution_count: 3, - metadata: {}, - data: { - 'application/octet-stream': { - content_type: 'application/octet-stream', - hash: 'def456', - path: '/my/path/def456.png', + type: 'output', + jupyter_data: { + output_type: 'display_data', + execution_count: 3, + metadata: {}, + data: { + 'application/octet-stream': { + content_type: 'application/octet-stream', + hash: 'def456', + path: '/my/path/def456.png', + }, }, }, }, @@ -91,14 +99,19 @@ describe('reduceOutputs', () => { ], }, { - type: 'output', + type: 'outputs', id: 'abc123', - data: [], children: [ { - type: 'image', - placeholder: true, - url: 'placeholder.png', + type: 'output', + jupyter_data: null, + children: [ + { + type: 'image', + placeholder: true, + url: 'placeholder.png', + }, + ], }, ], }, diff --git a/packages/myst-cli/src/transforms/outputs.ts b/packages/myst-cli/src/transforms/outputs.ts index ea014c752..f7dd5e5be 100644 --- a/packages/myst-cli/src/transforms/outputs.ts +++ b/packages/myst-cli/src/transforms/outputs.ts @@ -1,7 +1,7 @@ import fs from 'node:fs'; import { dirname, join, relative } from 'node:path'; import { computeHash } from 'myst-cli-utils'; -import type { Image, SourceFileKind } from 'myst-spec-ext'; +import type { Image, SourceFileKind, Output } from 'myst-spec-ext'; import { liftChildren, fileError, RuleId, fileWarn } from 'myst-common'; import type { GenericNode, GenericParent } from 'myst-common'; import type { ProjectSettings } from 'myst-frontmatter'; @@ -34,17 +34,29 @@ export async function transformOutputsToCache( kind: SourceFileKind, opts?: { minifyMaxCharacters?: number }, ) { - const outputs = selectAll('output', mdast) as GenericNode[]; - if (!outputs.length) return; + const outputsNodes = selectAll('outputs', mdast) as GenericNode[]; + if (!outputsNodes.length) return; const cache = castSession(session); await Promise.all( - outputs - .filter((output) => output.visibility !== 'remove') + outputsNodes + // Ignore outputs that are hidden + .filter((outputs) => outputs.visibility !== 'remove') + // Pull out children + .map((outputs) => outputs.children as Output[]) + .flat() + // Filter outputs with no data + // TODO: fix type + .filter((output) => (output as any).jupyter_data !== undefined) + // Minify output data .map(async (output) => { - output.data = await minifyCellOutput(output.data as IOutput[], cache.$outputs, { - computeHash, - maxCharacters: opts?.minifyMaxCharacters, - }); + [(output as any).jupyter_data] = await minifyCellOutput( + [(output as any).jupyter_data] as IOutput[], + cache.$outputs, + { + computeHash, + maxCharacters: opts?.minifyMaxCharacters, + }, + ); }), ); } @@ -77,7 +89,9 @@ export function transformFilterOutputStreams( const outputs = selectAll('output', block) as GenericNode[]; // There should be only one output in the block outputs.forEach((output) => { - output.data = output.data.filter((data: IStream | MinifiedMimeOutput) => { + const shouldKeepOutput = ( + data: IStream | MinifiedMimeOutput, // TODO: output-refactoring -- drop to single output in future + ) => { if ( (stderr !== 'show' || blockRemoveStderr) && data.output_type === 'stream' && @@ -144,9 +158,14 @@ export function transformFilterOutputStreams( return !doRemove; } return true; - }); + }; + const keepData = shouldKeepOutput(output.jupyter_data); + if (!keepData) { + output.type = '__delete__'; + } }); }); + remove(mdast, { cascade: false }, '__delete__'); } function writeCachedOutputToFile( @@ -192,16 +211,19 @@ export function transformOutputsToFile( const outputs = selectAll('output', mdast) as GenericNode[]; const cache = castSession(session); - outputs.forEach((node) => { - walkOutputs(node.data, (obj) => { - const { hash } = obj; - if (!hash || !cache.$outputs[hash]) return undefined; - obj.path = writeCachedOutputToFile(session, hash, cache.$outputs[hash], writeFolder, { - ...opts, - node, + outputs + .filter((output) => !!output.jupyter_data) + .forEach((node) => { + // TODO: output-refactoring -- drop to single output in future + walkOutputs([node.jupyter_data], (obj) => { + const { hash } = obj; + if (!hash || !cache.$outputs[hash]) return undefined; + obj.path = writeCachedOutputToFile(session, hash, cache.$outputs[hash], writeFolder, { + ...opts, + node, + }); }); }); - }); } /** @@ -233,79 +255,90 @@ export function reduceOutputs( writeFolder: string, opts?: { altOutputFolder?: string; vfile?: VFile }, ) { - const outputs = selectAll('output', mdast) as GenericNode[]; + const outputsNodes = selectAll('outputs', mdast) as GenericNode[]; const cache = castSession(session); - outputs.forEach((node) => { - if (!node.data?.length && !node.children?.length) { - node.type = '__delete__'; - return; - } - node.type = '__lift__'; - if (node.children?.length) return; - const selectedOutputs: { content_type: string; hash: string }[] = []; - node.data.forEach((output: MinifiedOutput) => { - let selectedOutput: { content_type: string; hash: string } | undefined; - walkOutputs([output], (obj: any) => { - const { output_type, content_type, hash } = obj; - if (!hash) return undefined; - if (!selectedOutput || isPreferredOutputType(content_type, selectedOutput.content_type)) { - if (['error', 'stream'].includes(output_type)) { - selectedOutput = { content_type: 'text/plain', hash }; - } else if (typeof content_type === 'string') { - if ( - content_type.startsWith('image/') || - content_type === 'text/plain' || - content_type === 'text/html' - ) { - selectedOutput = { content_type, hash }; + outputsNodes.forEach((outputsNode) => { + const outputs = outputsNode.children as GenericNode[]; + + outputs.forEach((outputNode) => { + // TODO: output-refactoring -- drop to single output in future + if (!outputNode.jupyter_data && !outputNode.children?.length) { + outputNode.type = '__delete__'; + return; + } + outputNode.type = '__lift__'; + if (outputNode.children?.length) return; + const selectedOutputs: { content_type: string; hash: string }[] = []; + // TODO: output-refactoring -- drop to single output in future + const selectOutput = (output: MinifiedOutput) => { + let selectedOutput: { content_type: string; hash: string } | undefined; + walkOutputs([output], (obj: any) => { + const { output_type, content_type, hash } = obj; + if (!hash) return undefined; + if (!selectedOutput || isPreferredOutputType(content_type, selectedOutput.content_type)) { + if (['error', 'stream'].includes(output_type)) { + selectedOutput = { content_type: 'text/plain', hash }; + } else if (typeof content_type === 'string') { + if ( + content_type.startsWith('image/') || + content_type === 'text/plain' || + content_type === 'text/html' + ) { + selectedOutput = { content_type, hash }; + } } } - } - }); - if (selectedOutput) selectedOutputs.push(selectedOutput); + }); + if (selectedOutput) selectedOutputs.push(selectedOutput); + }; + if (outputNode.jupyter_data) { + selectOutput(outputNode.jupyter_data); + } + const children: (Image | GenericNode)[] = selectedOutputs + .map((output): Image | GenericNode | GenericNode[] | undefined => { + const { content_type, hash } = output ?? {}; + if (!hash || !cache.$outputs[hash]) return undefined; + if (content_type === 'text/html') { + const htmlTree = { + type: 'root', + children: [ + { + type: 'html', + value: cache.$outputs[hash][0], + }, + ], + }; + htmlTransform(htmlTree); + return htmlTree.children; + } else if (content_type.startsWith('image/')) { + const path = writeCachedOutputToFile(session, hash, cache.$outputs[hash], writeFolder, { + ...opts, + node: outputNode, + }); + if (!path) return undefined; + const relativePath = relative(dirname(file), path); + return { + type: 'image', + data: { type: 'output' }, + url: relativePath, + urlSource: relativePath, + }; + } else if (content_type === 'text/plain' && cache.$outputs[hash]) { + const [content] = cache.$outputs[hash]; + return { + type: 'code', + data: { type: 'output' }, + value: stripAnsi(content), + }; + } + return undefined; + }) + .flat() + .filter((output): output is Image | GenericNode => !!output); + outputNode.children = children; }); - const children: (Image | GenericNode)[] = selectedOutputs - .map((output): Image | GenericNode | GenericNode[] | undefined => { - const { content_type, hash } = output ?? {}; - if (!hash || !cache.$outputs[hash]) return undefined; - if (content_type === 'text/html') { - const htmlTree = { - type: 'root', - children: [ - { - type: 'html', - value: cache.$outputs[hash][0], - }, - ], - }; - htmlTransform(htmlTree); - return htmlTree.children; - } else if (content_type.startsWith('image/')) { - const path = writeCachedOutputToFile(session, hash, cache.$outputs[hash], writeFolder, { - ...opts, - node, - }); - if (!path) return undefined; - const relativePath = relative(dirname(file), path); - return { - type: 'image', - data: { type: 'output' }, - url: relativePath, - urlSource: relativePath, - }; - } else if (content_type === 'text/plain' && cache.$outputs[hash]) { - const [content] = cache.$outputs[hash]; - return { - type: 'code', - data: { type: 'output' }, - value: stripAnsi(content), - }; - } - return undefined; - }) - .flat() - .filter((output): output is Image | GenericNode => !!output); - node.children = children; + // Erase the outputs node + outputsNode.type = '__lift__'; }); remove(mdast, '__delete__'); liftChildren(mdast, '__lift__'); diff --git a/packages/myst-cli/src/transforms/schema.ts b/packages/myst-cli/src/transforms/schema.ts new file mode 100644 index 000000000..839dd096c --- /dev/null +++ b/packages/myst-cli/src/transforms/schema.ts @@ -0,0 +1,121 @@ +import { selectAll } from 'unist-util-select'; +import { visit, SKIP } from 'unist-util-visit'; +import type { GenericNode, GenericParent } from 'myst-common'; +import type { IOutput } from '@jupyterlab/nbformat'; + +/** + * Convert from a "published" external MyST AST to our internal representation. + * + * Sometimes we may need to introduce new features in our AST whose naive implementation would + * break existing AST consumers. An example is the Output AST refactoring, which exposes individual code-cell + * outputs to MyST's AST transforms; if we publish this updated AST, consumers may not understand + * the new node types or renamed fields. + * + * In the MDAST world, the only non-breaking change for AST consumers is to _add_ a new field. We can therefore + * represent new features in our published AST through temporary fields, allowing for new features to be deployed, + * before actually introducing a breaking schema evolution. + * + * This routine presently handles mapping from a "classic" MyST AST (that represents multiple outputs as a single Output node) + * to a "future" AST that will represent these outputs as individual Output nodes. In order to defer this breaking change as long + * as is possible, we will externally publish a "backwards & forwards compatible" AST that encodes this new state in a temporary field. + * + * Once the "future" AST becomes "contemporary" AST, this routine can be removed. + * + * type CodeBlock = { + * type: "block"; + * kind: "notebook-code", + * children: [ + * Code, + * Outputs + * ] + * } + * + * type Outputs = { + * type: "outputs"; + * children: Output[]; + * visibility: ...; + * identifier: ...; + * html_id: ...; + * } + * + * type Output = { + * type: "output"; + * children: GenericNode[]; + * jupyter_data: IOutput; + * } + * + */ +export function externalASTToInternal(ast: GenericParent) { + // TODO: output-refactoring -- rewrite this function + visit( + ast as any, + 'output', + (node: GenericNode, index: number | null, parent: GenericParent | null) => { + // Case 1: convert "classic" AST to "future" AST + // 1. nest `Output` under `Outputs` + // 2. lift `identifier` and `html_id` labels to `Outputs` + // 3. lift `visibility` to `Outputs` + if (parent && parent.type !== 'outputs' && !('_future_ast' in node)) { + // assert node.children.length === 1 + const outputsChildren = node.data.map((outputData: IOutput) => { + return { + type: 'output', + jupyter_data: outputData, + children: [], // FIXME: ignoring children here + }; + }); + // Nest `output` under `outputs` (1) + const outputs = { + type: 'outputs', + children: outputsChildren, + // Lift `Output.visibility` to `Outputs` (3) + visibility: node.visibility, + // Lift `Output.identifier` and `Output.html_id` to `Outputs` (2) + identifier: node.identifier, + html_id: node.html_id, + }; + parent.children[index!] = outputs; + return SKIP; + } + // Case 2: "compatible" AST + else if (parent && parent.type !== 'outputs' && '_future_ast' in node) { + const outputs = JSON.parse(node._future_ast); + parent.children[index!] = outputs; + return SKIP; + } + // Case 3: "future" AST + else { + // Don't do anything! + } + }, + ); +} + +/** + * Convert from an internal MyST AST to a "published" external representation. + * + * See the docstring for externalASTToInternal + * + * type Output = { + * type: "output"; + * children: GenericNode[]; + * data: IOutput[]; + * visibility: ...; + * identifier: ...; + * html_id: ...; + * } + * + */ +export function internalASTToExternal(ast: GenericParent) { + const outputsNodes = selectAll('outputs', ast) as GenericParent[]; + outputsNodes.forEach((outputsNode) => { + outputsNode._future_ast = JSON.stringify(outputsNode); + + outputsNode.type = 'output'; + outputsNode.data = outputsNode.children + .map((output) => output.jupyter_data) + .filter((datum) => !!datum); + // Do not publish any children + outputsNode.children = []; + }); +} diff --git a/packages/myst-directives/src/code.ts b/packages/myst-directives/src/code.ts index b7da9b0f6..0eef01cd3 100644 --- a/packages/myst-directives/src/code.ts +++ b/packages/myst-directives/src/code.ts @@ -210,15 +210,14 @@ export const codeCellDirective: DirectiveSpec = { executable: true, value: (data.body ?? '') as string, }; - const output = { - type: 'output', - id: nanoid(), - data: [], + const outputs = { + type: 'outputs', + children: [], }; const block: GenericNode = { type: 'block', kind: NotebookCell.code, - children: [code, output], + children: [code, outputs], data: {}, }; addCommonDirectiveOptions(data, block); diff --git a/packages/myst-execute/src/execute.ts b/packages/myst-execute/src/execute.ts index 5f14286c8..359d31936 100644 --- a/packages/myst-execute/src/execute.ts +++ b/packages/myst-execute/src/execute.ts @@ -2,7 +2,7 @@ import { select, selectAll } from 'unist-util-select'; import type { Logger } from 'myst-cli-utils'; import type { PageFrontmatter, KernelSpec } from 'myst-frontmatter'; import type { Kernel, KernelMessage, Session, SessionManager } from '@jupyterlab/services'; -import type { Block, Code, InlineExpression, Output } from 'myst-spec-ext'; +import type { Block, Code, InlineExpression, Output, Outputs } from 'myst-spec-ext'; import type { IOutput } from '@jupyterlab/nbformat'; import type { GenericNode, GenericParent, IExpressionResult, IExpressionError } from 'myst-common'; import { NotebookCell, fileError } from 'myst-common'; @@ -166,7 +166,7 @@ type CodeBlock = Block & { * @param node node to test */ function isCellBlock(node: GenericNode): node is CodeBlock { - return node.type === 'block' && select('code', node) !== null && select('output', node) !== null; + return node.type === 'block' && select('code', node) !== null && select('outputs', node) !== null; } /** @@ -282,14 +282,17 @@ function applyComputedOutputsToNodes( const thisResult = computedResult.shift(); if (isCellBlock(matchedNode)) { - // Pull out output to set data - const output = select('output', matchedNode) as unknown as { data: IOutput[] }; - // Set the output array to empty if we don't have a result (e.g. due to a kernel error) - output.data = thisResult === undefined ? [] : (thisResult as IOutput[]); + const rawOutputData = (thisResult as IOutput[]) ?? []; + // Pull out outputs to set data + const outputs = select('outputs', matchedNode) as Outputs; + // Ensure that whether this fails or succeeds, we write to `children` (e.g. due to a kernel error) + outputs.children = rawOutputData.map((data) => { + return { type: 'output', children: [], jupyter_data: data as any }; + }); } else if (isInlineExpression(matchedNode)) { + const rawOutputData = thisResult as Record | undefined; // Set data of expression to the result, or empty if we don't have one - matchedNode.result = // TODO: FIXME .data - thisResult === undefined ? undefined : (thisResult as unknown as Record); + matchedNode.result = rawOutputData; } else { // This should never happen throw new Error('Node must be either code block or inline expression.'); diff --git a/packages/myst-execute/tests/execute.yml b/packages/myst-execute/tests/execute.yml index f768992aa..664d53067 100644 --- a/packages/myst-execute/tests/execute.yml +++ b/packages/myst-execute/tests/execute.yml @@ -27,22 +27,18 @@ cases: - type: block kind: notebook-code data: - id: nb-cell-0 - identifier: nb-cell-0 - label: nb-cell-0 - html_id: nb-cell-0 + children: - type: code lang: python executable: true value: print('abc') - identifier: nb-cell-0-code enumerator: 1 - html_id: nb-cell-0-code - - type: output - id: T7FMDqDm8dM2bOT1tKeeM - identifier: nb-cell-0-output - html_id: nb-cell-0-output + - type: outputs + children: + - type: output + children: [] + after: type: root children: @@ -54,27 +50,22 @@ cases: - type: block kind: notebook-code data: - id: nb-cell-0 - identifier: nb-cell-0 - label: nb-cell-0 - html_id: nb-cell-0 + children: - type: code lang: python executable: true value: print('abc') - identifier: nb-cell-0-code enumerator: 1 - html_id: nb-cell-0-code - - type: output - id: T7FMDqDm8dM2bOT1tKeeM - identifier: nb-cell-0-output - html_id: nb-cell-0-output - data: - - output_type: stream - name: stdout - text: | - abc + - type: outputs + children: + - type: output + children: [] + jupyter_data: + output_type: stream + name: stdout + text: | + abc - title: tree with inline expression is evaluated before: type: root @@ -115,100 +106,72 @@ cases: - type: block kind: notebook-code data: - id: nb-cell-0 - identifier: nb-cell-0 - label: nb-cell-0 - html_id: nb-cell-0 children: - type: code lang: python executable: true value: print('abc') - identifier: nb-cell-0-code enumerator: 1 - html_id: nb-cell-0-code - - type: output - id: T7FMDqDm8dM2bOT1tKeeM - identifier: nb-cell-0-output - html_id: nb-cell-0-output - data: + - type: outputs + children: + - type: output + jupyter_data: - type: block kind: notebook-code data: - id: nb-cell-0 - identifier: nb-cell-0 - label: nb-cell-0 - html_id: nb-cell-0 children: - type: code lang: python executable: true value: raise ValueError - identifier: nb-cell-0-code enumerator: 1 - html_id: nb-cell-0-code - - type: output - id: T7FMDqDm8dM2bOT1tKeeM - identifier: nb-cell-0-output - html_id: nb-cell-0-output - data: + - type: outputs + children: + - type: output + jupyter_data: after: type: root children: - type: block kind: notebook-code data: - id: nb-cell-0 - identifier: nb-cell-0 - label: nb-cell-0 - html_id: nb-cell-0 children: - type: code lang: python executable: true value: print('abc') - identifier: nb-cell-0-code enumerator: 1 - html_id: nb-cell-0-code - - type: output - id: T7FMDqDm8dM2bOT1tKeeM - identifier: nb-cell-0-output - html_id: nb-cell-0-output - data: - - output_type: stream - name: stdout - text: | - abc + - type: outputs + children: + - type: output + jupyter_data: + output_type: stream + name: stdout + text: | + abc - type: block kind: notebook-code data: - id: nb-cell-0 - identifier: nb-cell-0 - label: nb-cell-0 - html_id: nb-cell-0 children: - type: code lang: python executable: true value: raise ValueError - identifier: nb-cell-0-code enumerator: 1 - html_id: nb-cell-0-code - - type: output - id: T7FMDqDm8dM2bOT1tKeeM - identifier: nb-cell-0-output - html_id: nb-cell-0-output - data: - - output_type: error - # Note this traceback can be different on various machines - # Not including it means we still validate an error, just don't care about the traceback - # traceback: - # - "\e[0;31m---------------------------------------------------------------------------\e[0m" - # - "\e[0;31mValueError\e[0m Traceback (most recent call last)" - # - "Cell \e[0;32mIn[2], line 1\e[0m\n\e[0;32m----> 1\e[0m \e[38;5;28;01mraise\e[39;00m \e[38;5;167;01mValueError\e[39;00m\n" - # - "\e[0;31mValueError\e[0m: " - ename: ValueError - evalue: '' + - type: outputs + children: + - type: output + jupyter_data: + output_type: error + # Note this traceback can be different on various machines + # Not including it means we still validate an error, just don't care about the traceback + # traceback: + # - "\e[0;31m---------------------------------------------------------------------------\e[0m" + # - "\e[0;31mValueError\e[0m Traceback (most recent call last)" + # - "Cell \e[0;32mIn[2], line 1\e[0m\n\e[0;32m----> 1\e[0m \e[38;5;28;01mraise\e[39;00m \e[38;5;167;01mValueError\e[39;00m\n" + # - "\e[0;31mValueError\e[0m: " + ename: ValueError + evalue: '' - title: tree with bad executable code and `raises-exception` is evaluated and passes before: type: root @@ -216,58 +179,44 @@ cases: - type: block kind: notebook-code data: - id: nb-cell-0 tags: raises-exception - identifier: nb-cell-0 - label: nb-cell-0 - html_id: nb-cell-0 children: - type: code lang: python executable: true value: raise ValueError - identifier: nb-cell-0-code enumerator: 1 - html_id: nb-cell-0-code - - type: output - id: T7FMDqDm8dM2bOT1tKeeM - identifier: nb-cell-0-output - html_id: nb-cell-0-output - data: + - type: outputs + children: + - type: output + jupyter_data: after: type: root children: - type: block kind: notebook-code data: - id: nb-cell-0 tags: raises-exception - identifier: nb-cell-0 - label: nb-cell-0 - html_id: nb-cell-0 children: - type: code lang: python executable: true value: raise ValueError - identifier: nb-cell-0-code enumerator: 1 - html_id: nb-cell-0-code - - type: output - id: T7FMDqDm8dM2bOT1tKeeM - identifier: nb-cell-0-output - html_id: nb-cell-0-output - data: - - output_type: error - # Note this traceback can be different on various machines - # Not including it means we still validate an error, just don't care about the traceback - # traceback: - # - "\e[0;31m---------------------------------------------------------------------------\e[0m" - # - "\e[0;31mValueError\e[0m Traceback (most recent call last)" - # - "Cell \e[0;32mIn[2], line 1\e[0m\n\e[0;32m----> 1\e[0m \e[38;5;28;01mraise\e[39;00m \e[38;5;167;01mValueError\e[39;00m\n" - # - "\e[0;31mValueError\e[0m: " - ename: ValueError - evalue: '' + - type: outputs + children: + - type: output + jupyter_data: + output_type: error + # Note this traceback can be different on various machines + # Not including it means we still validate an error, just don't care about the traceback + # traceback: + # - "\e[0;31m---------------------------------------------------------------------------\e[0m" + # - "\e[0;31mValueError\e[0m Traceback (most recent call last)" + # - "Cell \e[0;32mIn[2], line 1\e[0m\n\e[0;32m----> 1\e[0m \e[38;5;28;01mraise\e[39;00m \e[38;5;167;01mValueError\e[39;00m\n" + # - "\e[0;31mValueError\e[0m: " + ename: ValueError + evalue: '' - title: tree with bad executable code and `skip-execution` is not evaluated before: type: root @@ -275,45 +224,31 @@ cases: - type: block kind: notebook-code data: - id: nb-cell-0 tags: skip-execution - identifier: nb-cell-0 - label: nb-cell-0 - html_id: nb-cell-0 children: - type: code lang: python executable: true value: raise ValueError - identifier: nb-cell-0-code enumerator: 1 - html_id: nb-cell-0-code - - type: output - id: T7FMDqDm8dM2bOT1tKeeM - identifier: nb-cell-0-output - html_id: nb-cell-0-output - data: + - type: outputs + children: + - type: output + jupyter_data: after: type: root children: - type: block kind: notebook-code data: - id: nb-cell-0 tags: skip-execution - identifier: nb-cell-0 - label: nb-cell-0 - html_id: nb-cell-0 children: - type: code lang: python executable: true value: raise ValueError - identifier: nb-cell-0-code enumerator: 1 - html_id: nb-cell-0-code - - type: output - id: T7FMDqDm8dM2bOT1tKeeM - identifier: nb-cell-0-output - html_id: nb-cell-0-output - data: + - type: outputs + children: + - type: output + jupyter_data: diff --git a/packages/myst-execute/tests/run.spec.ts b/packages/myst-execute/tests/run.spec.ts index e6a5d159f..ab4f0bbfb 100644 --- a/packages/myst-execute/tests/run.spec.ts +++ b/packages/myst-execute/tests/run.spec.ts @@ -96,6 +96,7 @@ casesList.forEach(({ title, cases }) => { expect.arrayContaining([expect.stringMatching(throws)]), ); } + console.log(JSON.stringify(after, null, 2)); expect(before).toMatchObject(after); }, { timeout: 30_000 }, diff --git a/packages/myst-spec-ext/src/types.ts b/packages/myst-spec-ext/src/types.ts index b808b82be..ed7a4b4e4 100644 --- a/packages/myst-spec-ext/src/types.ts +++ b/packages/myst-spec-ext/src/types.ts @@ -251,6 +251,12 @@ export type Container = Omit & { parentEnumerator?: string; }; +export type Outputs = Node & + Target & { + type: 'outputs'; + children: Output[]; + }; + export type Output = Node & Target & { type: 'output'; diff --git a/packages/mystmd/tests/notebook-fig-embed/outputs/index.json b/packages/mystmd/tests/notebook-fig-embed/outputs/index.json index efa957528..763891c6e 100644 --- a/packages/mystmd/tests/notebook-fig-embed/outputs/index.json +++ b/packages/mystmd/tests/notebook-fig-embed/outputs/index.json @@ -55,6 +55,7 @@ "children": [ { "type": "output", + "_future_ast": "{\"type\":\"outputs\",\"children\":[{\"type\":\"output\",\"id\":\"Ffu3R5cJjHblGUbO46YxC\",\"jupyter_data\":{\"output_type\":\"execute_result\",\"execution_count\":2,\"metadata\":{},\"data\":{\"text/html\":{\"content_type\":\"text/html\",\"hash\":\"a16fcedcd26437c820ccfc05d1f48a57\",\"path\":\"/a16fcedcd26437c820ccfc05d1f48a57.html\"},\"text/plain\":{\"content\":\"alt.Chart(...)\",\"content_type\":\"text/plain\"}}},\"key\":\"aicFc0saqi\"}],\"key\":\"dSx0jlrAGq\"}", "data": [ { "output_type": "execute_result", @@ -144,6 +145,7 @@ }, { "type": "output", + "_future_ast": "{\"type\":\"outputs\",\"children\":[{\"type\":\"output\",\"id\":\"Ffu3R5cJjHblGUbO46YxC\",\"jupyter_data\":{\"output_type\":\"execute_result\",\"execution_count\":2,\"metadata\":{},\"data\":{\"text/html\":{\"content_type\":\"text/html\",\"hash\":\"a16fcedcd26437c820ccfc05d1f48a57\",\"path\":\"/a16fcedcd26437c820ccfc05d1f48a57.html\"},\"text/plain\":{\"content\":\"alt.Chart(...)\",\"content_type\":\"text/plain\"}}},\"key\":\"Na254yM4lm\"}],\"key\":\"hODUzanXjz\"}", "data": [ { "output_type": "execute_result", @@ -190,6 +192,7 @@ "children": [ { "type": "output", + "_future_ast": "{\"type\":\"outputs\",\"children\":[{\"type\":\"output\",\"jupyter_data\":{\"output_type\":\"execute_result\",\"execution_count\":2,\"metadata\":{},\"data\":{\"text/html\":{\"content_type\":\"text/html\",\"hash\":\"a85dae82213ec48ea05754a345bab5ce\",\"path\":\"https://cdn.curvenote.com/0192bff5-9c9d-722f-92bf-e702aa8e1f46/public/a85dae82213ec48ea05754a345bab5ce.html\"},\"text/plain\":{\"content\":\"alt.VConcatChart(...)\",\"content_type\":\"text/plain\"}}},\"children\":[],\"key\":\"yzzP9Mn8wY\"}],\"visibility\":\"show\",\"key\":\"f7fQ28USQb\"}", "data": [ { "output_type": "execute_result", @@ -286,6 +289,7 @@ }, { "type": "output", + "_future_ast": "{\"type\":\"outputs\",\"children\":[{\"type\":\"output\",\"jupyter_data\":{\"output_type\":\"execute_result\",\"execution_count\":2,\"metadata\":{},\"data\":{\"text/html\":{\"content_type\":\"text/html\",\"hash\":\"a85dae82213ec48ea05754a345bab5ce\",\"path\":\"https://cdn.curvenote.com/0192bff5-9c9d-722f-92bf-e702aa8e1f46/public/a85dae82213ec48ea05754a345bab5ce.html\"},\"text/plain\":{\"content\":\"alt.VConcatChart(...)\",\"content_type\":\"text/plain\"}}},\"children\":[],\"key\":\"gKECmUtMaA\"}],\"visibility\":\"show\",\"key\":\"Mt6Gpq3tTq\"}", "data": [ { "output_type": "execute_result",