⬆️🤫 Lift and parse Markdown/text outputs from Jupyter outputs (option 2)

packages/myst-cli/src/process/mdast.ts

@@ -68,7 +68,9 @@ import {
   transformImagesToDisk,
   transformFilterOutputStreams,
   transformLiftCodeBlocksInJupytext,
+  transformMarkdownOutputs,
   transformMystXRefs,
+  internalASTToExternal,
 } from '../transforms/index.js';
 import type { ImageExtensions } from '../utils/resolveExtension.js';
 import { logMessagesFromVFile } from '../utils/logging.js';
@@ -239,6 +241,9 @@ export async function transformMdast(
       log: session.log,
     });
   }
+  await transformMarkdownOutputs(mdast, {
+    parser: (content: string) => parseMyst(session, content, file),
+  });
   transformRenderInlineExpressions(mdast, vfile);
   await transformOutputsToCache(session, mdast, kind, { minifyMaxCharacters });
   transformFilterOutputStreams(mdast, vfile, frontmatter.settings);
@@ -380,6 +385,7 @@ export async function finalizeMdast(
 ) {
   const vfile = new VFile(); // Collect errors on this file
   vfile.path = file;
+
   if (simplifyFigures) {
     // Transform output nodes to images / text
     reduceOutputs(session, mdast, file, imageWriteFolder, {
@@ -431,5 +437,6 @@ export async function finalizeMdast(
     postData.widgets = cache.$getMdast(file)?.pre.widgets;
     updateFileInfoFromFrontmatter(session, file, frontmatter);
   }
+  internalASTToExternal(mdast);
   logMessagesFromVFile(session, vfile);
 }

packages/myst-cli/src/process/notebook.ts

@@ -165,17 +165,20 @@ export async function processNotebookFull(
           value: ensureString(cell.source),
         };
 
-        // Embed outputs in an output block
-        const output: { type: 'output'; id: string; data: IOutput[] } = {
-          type: 'output',
-          id: nanoid(),
-          data: [],
+        const outputsChildren = (cell.outputs as IOutput[]).map((output) => {
+          // Embed outputs in an output block
+          const result: { type: 'output'; id: string; jupyter_data: IOutput } = {
+            type: 'output',
+            id: nanoid(),
+            jupyter_data: output,
+          };
+          return result;
+        });
+        const outputs = {
+          type: 'outputs',
+          children: outputsChildren,
         };
-
-        if (cell.outputs && (cell.outputs as IOutput[]).length > 0) {
-          output.data = cell.outputs as IOutput[];
-        }
-        return acc.concat(blockParent(cell, [code, output]));
+        return acc.concat(blockParent(cell, [code, outputs]));
       }
       return acc;
     },

packages/myst-cli/src/transforms/crossReferences.ts

@@ -1,5 +1,6 @@
 import type { VFile } from 'vfile';
 import { selectAll } from 'unist-util-select';
+import { visit, SKIP } from 'unist-util-visit';
 import type { FrontmatterParts, GenericNode, GenericParent, References } from 'myst-common';
 import { RuleId, fileWarn, plural, selectMdastNodes } from 'myst-common';
 import { computeHash, tic } from 'myst-cli-utils';
@@ -9,6 +10,8 @@ import type { CrossReference, Dependency, Link, SourceFileKind } from 'myst-spec
 import type { ISession } from '../session/types.js';
 import { loadFromCache, writeToCache } from '../session/cache.js';
 import type { SiteAction, SiteExport } from 'myst-config';
+import type { IOutput } from '@jupyterlab/nbformat';
+import { externalASTToInternal } from './schema.js';
 
 export const XREF_MAX_AGE = 1; // in days
 
@@ -32,6 +35,13 @@ export type MystData = {
   references?: References;
 };
 
+function upgradeAndDowngradeMystData(data: MystData): MystData {
+  if (data.mdast) {
+    externalASTToInternal(data.mdast);
+  }
+  return data;
+}
+
 async function fetchMystData(
   session: ISession,
   dataUrl: string | undefined,
@@ -48,7 +58,8 @@ async function fetchMystData(
     try {
       const resp = await session.fetch(dataUrl);
       if (resp.ok) {
-        const data = (await resp.json()) as MystData;
+        const data = upgradeAndDowngradeMystData((await resp.json()) as MystData);
+
         writeToCache(session, filename, JSON.stringify(data));
         return data;
       }

packages/myst-cli/src/transforms/index.ts

@@ -9,5 +9,6 @@ export * from './include.js';
 export * from './links.js';
 export * from './mdast.js';
 export * from './outputs.js';
+export * from './schema.js';
 export * from './inlineExpressions.js';
 export * from './types.js';

packages/myst-cli/src/transforms/outputs.spec.ts

@@ -20,9 +20,14 @@ describe('reduceOutputs', () => {
               ],
             },
             {
-              type: 'output',
-              id: 'abc123',
-              data: [],
+              type: 'outputs',
+              children: [
+                {
+                  type: 'output',
+                  id: 'abc123',
+                  jupyter_data: null,
+                },
+              ],
             },
           ],
         },
@@ -49,18 +54,21 @@ describe('reduceOutputs', () => {
               ],
             },
             {
-              type: 'output',
+              type: 'outputs',
               id: 'abc123',
-              data: [
+              children: [
                 {
-                  output_type: 'display_data',
-                  execution_count: 3,
-                  metadata: {},
-                  data: {
-                    'application/octet-stream': {
-                      content_type: 'application/octet-stream',
-                      hash: 'def456',
-                      path: '/my/path/def456.png',
+                  type: 'output',
+                  jupyter_data: {
+                    output_type: 'display_data',
+                    execution_count: 3,
+                    metadata: {},
+                    data: {
+                      'application/octet-stream': {
+                        content_type: 'application/octet-stream',
+                        hash: 'def456',
+                        path: '/my/path/def456.png',
+                      },
                     },
                   },
                 },
@@ -91,14 +99,19 @@ describe('reduceOutputs', () => {
               ],
             },
             {
-              type: 'output',
+              type: 'outputs',
               id: 'abc123',
-              data: [],
               children: [
                 {
-                  type: 'image',
-                  placeholder: true,
-                  url: 'placeholder.png',
+                  type: 'output',
+                  jupyter_data: null,
+                  children: [
+                    {
+                      type: 'image',
+                      placeholder: true,
+                      url: 'placeholder.png',
+                    },
+                  ],
                 },
               ],
             },