feat: add build-time execution

packages/myst-cli/package.json

@@ -43,6 +43,7 @@
   },
   "dependencies": {
     "@jupyterlab/nbformat": "3.5.2",
+    "@jupyterlab/services": "^7.0.0",
     "@reduxjs/toolkit": "^1.7.2",
     "adm-zip": "^0.5.10",
     "boxen": "^7.1.1",

packages/myst-cli/src/process/mdast.ts

@@ -9,6 +9,7 @@
   basicTransformationsPlugin,
   htmlPlugin,
   footnotesPlugin,
+  blockMetadataPlugin,
   ReferenceState,
   MultiPageReferenceState,
   resolveReferencesTransform,
@@ -53,7 +54,6 @@
   transformImageFormats,
   transformThumbnail,
   StaticFileTransformer,
-  inlineExpressionsPlugin,
   propagateBlockDataToCode,
   transformBanner,
   reduceOutputs,
@@ -66,6 +66,7 @@
   transformImagesToDisk,
   transformFilterOutputStreams,
   transformLiftCodeBlocksInJupytext,
+  renderInlineExpressionsTransform,
 } from '../transforms/index.js';
 import type { ImageExtensions } from '../utils/resolveExtension.js';
 import { logMessagesFromVFile } from '../utils/logMessagesFromVFile.js';
@@ -74,6 +75,13 @@
 import { loadIntersphinx } from './intersphinx.js';
 import { frontmatterPartsTransform } from '../transforms/parts.js';
 import { parseMyst } from './myst.js';
+import {
+  findExistingJupyterServer,
+  JupyterServerSettings,
+  launchJupyterServer,
+  transformKernelExecution,
+} from '../transforms/execute.js';
+import { ServerConnection, KernelManager, SessionManager } from '@jupyterlab/services';
 
 const LINKS_SELECTOR = 'link,card,linkBlock';
 
@@ -174,7 +182,7 @@
 
   const pipe = unified()
     .use(reconstructHtmlPlugin) // We need to group and link the HTML first
-    .use(inlineExpressionsPlugin) // Happens before math and images!
+    .use(blockMetadataPlugin) // Happens before math and images!
     .use(htmlPlugin, { htmlHandlers }) // Some of the HTML plugins need to operate on the transformed html, e.g. figure caption transforms
     .use(basicTransformationsPlugin, {
       parser: (content: string) => parseMyst(session, content, file),
@@ -223,8 +231,11 @@
   // Combine file-specific citation renderers with project renderers from bib files
   const fileCitationRenderer = combineCitationRenderers(cache, ...rendererFiles);
 
+  await transformKernelExecution(session, mdast, frontmatter, false, vfile, false);
+
   transformFilterOutputStreams(mdast, vfile, frontmatter.settings);
   await transformOutputsToCache(session, mdast, kind, { minifyMaxCharacters });
+
   transformCitations(mdast, fileCitationRenderer, references);
   await unified()
     .use(codePlugin, { lang: frontmatter?.kernelspec?.language })
@@ -348,6 +359,7 @@
       altOutputFolder: simplifyFigures ? undefined : imageAltOutputFolder,
     });
   }
+  renderInlineExpressionsTransform(mdast, vfile);
   transformOutputsToFile(session, mdast, imageWriteFolder, {
     altOutputFolder: simplifyFigures ? undefined : imageAltOutputFolder,
     vfile,

packages/myst-cli/src/process/notebook.ts

@@ -19,7 +19,13 @@
 import type { ISession } from '../session/types.js';
 import { BASE64_HEADER_SPLIT } from '../transforms/images.js';
 import { parseMyst } from './myst.js';
-import type { Code } from 'myst-spec-ext';
+import type { Code, InlineExpression } from 'myst-spec-ext';
+import {
+  findExpression,
+  IUserExpressionMetadata,
+  metadataSection,
+  renderExpression,
+} from '../transforms/index.js';
 
 function blockParent(cell: ICell, children: GenericNode[]) {
   const type = cell.cell_type === CELL_TYPES.code ? NotebookCell.code : NotebookCell.content;
@@ -86,6 +92,9 @@
     end = -1;
   }
 
+  const vfile = new VFile();
+  vfile.path = file;
+
   const items = await cells?.slice(0, end).reduce(
     async (P, cell: ICell, index) => {
       const acc = await P;
@@ -100,7 +109,20 @@
         if (omitBlockDivider) {
           return acc.concat(...cellMdast.children);
         }
-        return acc.concat(blockParent(cell, cellMdast.children));
+        const block = blockParent(cell, cellMdast.children) as GenericNode;
+
+        // Embed expression results into expression
+        const userExpressions = block.data?.[metadataSection] as IUserExpressionMetadata[];
+        const inlineNodes = selectAll('inlineExpression', block) as InlineExpression[];
+        let count = 0;
+        inlineNodes.forEach((inlineExpression) => {
+          const data = findExpression(userExpressions, inlineExpression.value);
+          if (!data) return;
+          count += 1;
+          inlineExpression.identifier = `eval-${index}-${count}`;
+          inlineExpression.data = data.result as unknown as Record<string, unknown>;
+        });
+        return acc.concat(block);
       }
       if (cell.cell_type === CELL_TYPES.raw) {
         const raw: Code = {
@@ -118,19 +140,14 @@
           value: ensureString(cell.source),
         };
 
-        const output: { type: 'output'; id: string; data: MinifiedOutput[] } = {
+        // Embed outputs into output block
+        const output: { type: 'output'; id: string; data: IOutput[] } = {
           type: 'output',
           id: nanoid(),
           data: [],
         };
-
         if (cell.outputs && (cell.outputs as IOutput[]).length > 0) {
-          const minified: MinifiedOutput[] = await minifyCellOutput(
-            cell.outputs as IOutput[],
-            cache.$outputs,
-            { computeHash, maxCharacters: opts?.minifyMaxCharacters },
-          );
-          output.data = minified;
+          output.data = cell.outputs as IOutput[];
         }
         return acc.concat(blockParent(cell, [code, output]));
       }

packages/myst-cli/src/session/session.ts

@@ -19,6 +19,12 @@
 import { rootReducer } from '../store/reducers.js';
 import version from '../version.js';
 import type { ISession } from './types.js';
+import { KernelManager, ServerConnection, SessionManager } from '@jupyterlab/services';
+import {
+  findExistingJupyterServer,
+  JupyterServerSettings,
+  launchJupyterServer,
+} from '../transforms/execute.js';
 
 const CONFIG_FILES = ['myst.yml'];
 const API_URL = 'https://api.mystmd.org';
@@ -62,6 +68,7 @@
 
   _shownUpgrade = false;
   _latestVersion?: string;
+  _jupyterSessionManager: SessionManager | undefined | null = null;
 
   get log(): Logger {
     return this.$logger;
@@ -157,4 +164,36 @@
     });
     return warnings;
   }
+
+  async jupyterSessionManager(): Promise<SessionManager | undefined> {
+    if (this._jupyterSessionManager !== null) {
+      return Promise.resolve(this._jupyterSessionManager);
+    }
+    try {
+      const partialServerSettings = await new Promise<JupyterServerSettings>(
+        async (resolve, reject) => {
+          if (process.env.JUPYTER_BASE_URL === undefined) {
+            resolve(
+              findExistingJupyterServer() ||
+                (await launchJupyterServer(this.contentPath(), this.log)),
+            );
+          } else {
+            resolve({
+              baseUrl: process.env.JUPYTER_BASE_URL,
+              token: process.env.JUPYTER_TOKEN,
+            });
+          }
+        },
+      );
+      const serverSettings = ServerConnection.makeSettings(partialServerSettings);
+      const kernelManager = new KernelManager({ serverSettings });
+      const manager = new SessionManager({ kernelManager, serverSettings });
+      // TODO: this is a race condition, even though we shouldn't hit if if this promise is actually awaited
+      this._jupyterSessionManager = manager;
+      return manager;
+    } catch {
+      this._jupyterSessionManager = undefined;
+      return undefined;
+    }
+  }
 }

packages/myst-cli/src/session/types.ts

@@ -8,6 +8,7 @@
 
 import type { BuildWarning, RootState } from '../store/index.js';
 import type { PreRendererData, RendererData, SingleCitationRenderer } from '../transforms/types.js';
+import { SessionManager } from '@jupyterlab/services';
 
 export type ISession = {
   API_URL: string;
@@ -24,6 +25,7 @@
   plugins: MystPlugin | undefined;
   loadPlugins(): Promise<MystPlugin>;
   getAllWarnings(ruleId: RuleId): (BuildWarning & { file: string })[];
+  jupyterSessionManager(): Promise<SessionManager | undefined>;
 };
 
 export type ISessionWithCache = ISession & {