✏️ Parse markdown outputs into AST

packages/myst-cli/src/process/file.ts

@@ -36,6 +36,7 @@ type LoadFileOptions = {
   preFrontmatter?: Record<string, any>;
   keepTitleNode?: boolean;
   kind?: SourceFileKind;
+  minifyMaxCharacters?: number;
 };
 
 export type LoadFileResult = {
@@ -85,7 +86,7 @@ export async function loadNotebookFile(
     mdast,
     frontmatter: nbFrontmatter,
     widgets,
-  } = await processNotebookFull(session, file, content);
+  } = await processNotebookFull(session, file, content, opts?.minifyMaxCharacters);
   const { frontmatter: cellFrontmatter, identifiers } = getPageFrontmatter(
     session,
     mdast,

packages/myst-cli/src/process/mdast.ts

@@ -52,13 +52,13 @@ import {
   transformImageFormats,
   transformLinkedDOIs,
   transformLinkedRORs,
-  transformOutputsToCache,
   transformRenderInlineExpressions,
   transformThumbnail,
   StaticFileTransformer,
   propagateBlockDataToCode,
   transformBanner,
-  reduceOutputs,
+  transformReduceOutputs,
+  transformLiftOutputs,
   transformPlaceholderImages,
   transformDeleteBase64UrlSource,
   transformWebp,
@@ -76,7 +76,6 @@ import { combineCitationRenderers } from './citations.js';
 import { bibFilesInDir, selectFile } from './file.js';
 import { parseMyst } from './myst.js';
 import { kernelExecutionTransform, LocalDiskCache } from 'myst-execute';
-import type { IOutput } from '@jupyterlab/nbformat';
 import { rawDirectiveTransform } from '../transforms/raw.js';
 
 const LINKS_SELECTOR = 'link,card,linkBlock';
@@ -198,16 +197,36 @@ export async function transformMdast(
     .use(abbreviationPlugin, { abbreviations: frontmatter.abbreviations })
     .use(indexIdentifierPlugin)
     .use(enumerateTargetsPlugin, { state }); // This should be after math/container transforms
-  // Load custom transform plugins
-  session.plugins?.transforms.forEach((t) => {
-    if (t.stage !== 'document') return;
-    pipe.use(t.plugin, undefined, pluginUtils);
-  });
   await pipe.run(mdast, vfile);
 
   // This needs to come after basic transformations since meta tags are added there
   propagateBlockDataToCode(session, vfile, mdast);
 
+  if (execute) {
+    const cachePath = path.join(session.buildPath(), 'execute');
+    await kernelExecutionTransform(mdast, vfile, {
+      basePath: session.sourcePath(),
+      cache: new LocalDiskCache(cachePath),
+      sessionFactory: () => session.jupyterSessionManager(),
+      frontmatter: frontmatter,
+      ignoreCache: false,
+      errorIsFatal: false,
+      log: session.log,
+      outputCache: cache.$outputs,
+      minifyMaxCharacters,
+    });
+  }
+  transformRenderInlineExpressions(mdast, vfile);
+
+  // Run document plugins
+  const pluginPipe = unified();
+  // Load custom transform plugins
+  session.plugins?.transforms.forEach((t) => {
+    if (t.stage !== 'document') return;
+    pluginPipe.use(t.plugin, undefined, pluginUtils);
+  });
+  await pluginPipe.run(mdast, vfile);
+
   // Initialize citation renderers for this (non-bib) file
   cache.$citationRenderers[file] = await transformLinkedDOIs(
     session,
@@ -225,21 +244,6 @@ export async function transformMdast(
   }
   // Combine file-specific citation renderers with project renderers from bib files
   const fileCitationRenderer = combineCitationRenderers(cache, ...rendererFiles);
-
-  if (execute) {
-    const cachePath = path.join(session.buildPath(), 'execute');
-    await kernelExecutionTransform(mdast, vfile, {
-      basePath: session.sourcePath(),
-      cache: new LocalDiskCache<(IExpressionResult | IOutput[])[]>(cachePath),
-      sessionFactory: () => session.jupyterSessionManager(),
-      frontmatter: frontmatter,
-      ignoreCache: false,
-      errorIsFatal: false,
-      log: session.log,
-    });
-  }
-  transformRenderInlineExpressions(mdast, vfile);
-  await transformOutputsToCache(session, mdast, kind, { minifyMaxCharacters });
   transformFilterOutputStreams(mdast, vfile, frontmatter.settings);
   transformCitations(session, file, mdast, fileCitationRenderer, references);
   await unified()
@@ -382,9 +386,10 @@ export async function finalizeMdast(
   vfile.path = file;
   if (simplifyFigures) {
     // Transform output nodes to images / text
-    reduceOutputs(session, mdast, file, imageWriteFolder, {
+    transformReduceOutputs(session, mdast, file, imageWriteFolder, {
       altOutputFolder: simplifyFigures ? undefined : imageAltOutputFolder,
     });
+    transformLiftOutputs(mdast);
   }
   transformOutputsToFile(session, mdast, imageWriteFolder, {
     altOutputFolder: simplifyFigures ? undefined : imageAltOutputFolder,

packages/myst-cli/src/process/notebook.ts

@@ -7,14 +7,15 @@ import type {
   ICell,
   IMimeBundle,
   INotebookContent,
-  INotebookMetadata,
   IOutput,
   MultilineString,
 } from '@jupyterlab/nbformat';
-import { CELL_TYPES, ensureString } from 'nbtx';
+import { CELL_TYPES, ensureString, minifyCellOutput } from 'nbtx';
 import { VFile } from 'vfile';
 import { logMessagesFromVFile } from '../utils/logging.js';
 import type { ISession } from '../session/types.js';
+import { castSession } from '../session/cache.js';
+import { computeHash } from 'myst-cli-utils';
 import { BASE64_HEADER_SPLIT } from '../transforms/images.js';
 import { parseMyst } from './myst.js';
 import type { Code, InlineExpression } from 'myst-spec-ext';
@@ -88,8 +89,10 @@ export async function processNotebookFull(
   session: ISession,
   file: string,
   content: string,
+  minifyMaxCharacters?: number,
 ): Promise<{ mdast: GenericParent; frontmatter: PageFrontmatter; widgets: Record<string, any> }> {
   const { log } = session;
+  const cache = castSession(session);
   const { metadata, cells } = JSON.parse(content) as INotebookContent;
   // notebook will be empty, use generateNotebookChildren, generateNotebookOrder here if we want to populate those
 
@@ -165,15 +168,22 @@ export async function processNotebookFull(
           value: ensureString(cell.source),
         };
 
-        const outputsChildren = (cell.outputs as IOutput[]).map((output) => {
-          // Embed outputs in an output block
-          const result = {
-            type: 'output',
-            jupyter_data: output,
-            children: [],
-          };
-          return result;
-        });
+        const outputsChildren = await Promise.all(
+          (cell.outputs as IOutput[]).map(async (output) => {
+            // Minify the output
+            const [minifiedOutput] = await minifyCellOutput([output] as IOutput[], cache.$outputs, {
+              computeHash,
+              maxCharacters: minifyMaxCharacters,
+            });
+            // Embed outputs in an output block
+            const result = {
+              type: 'output',
+              jupyter_data: minifiedOutput,
+              children: [],
+            };
+            return result;
+          }),
+        );
         const outputs = {
           type: 'outputs',
           id: nanoid(),