feat: add support for auto-detecting the running server

packages/myst-cli/src/process/mdast.ts

@@ -65,7 +65,8 @@ import {
   transformImagesWithoutExt,
   transformImagesToDisk,
   transformFilterOutputStreams,
-  transformLiftCodeBlocksInJupytext, renderInlineExpressionsTransform
+  transformLiftCodeBlocksInJupytext,
+  renderInlineExpressionsTransform,
 } from '../transforms/index.js';
 import type { ImageExtensions } from '../utils/resolveExtension.js';
 import { logMessagesFromVFile } from '../utils/logMessagesFromVFile.js';
@@ -74,7 +75,12 @@ import { bibFilesInDir, selectFile } from './file.js';
 import { loadIntersphinx } from './intersphinx.js';
 import { frontmatterPartsTransform } from '../transforms/parts.js';
 import { parseMyst } from './myst.js';
-import { transformKernelExecution } from '../transforms/execute.js';
+import {
+  findExistingJupyterServer,
+  JupyterServerSettings,
+  launchJupyterServer,
+  transformKernelExecution,
+} from '../transforms/execute.js';
 import { ServerConnection, KernelManager, SessionManager } from '@jupyterlab/services';
 
 const LINKS_SELECTOR = 'link,card,linkBlock';
@@ -225,13 +231,7 @@ export async function transformMdast(
   // Combine file-specific citation renderers with project renderers from bib files
   const fileCitationRenderer = combineCitationRenderers(cache, ...rendererFiles);
 
-  const serverSettings = ServerConnection.makeSettings({
-    baseUrl: process.env.JUPYTER_BASE_URL,
-    token: process.env.JUPYTER_TOKEN,
-  });
-  const kernelManager = new KernelManager({ serverSettings });
-  const sessionManager = new SessionManager({ kernelManager, serverSettings });
-  await transformKernelExecution(session, sessionManager, mdast, frontmatter, false, vfile, false);
+  await transformKernelExecution(session, mdast, frontmatter, false, vfile, false);
 
   transformFilterOutputStreams(mdast, vfile, frontmatter.settings);
   await transformOutputsToCache(session, mdast, kind, { minifyMaxCharacters });

packages/myst-cli/src/session/session.ts

@@ -19,6 +19,12 @@ import type { RootState } from '../store/reducers.js';
 import { rootReducer } from '../store/reducers.js';
 import version from '../version.js';
 import type { ISession } from './types.js';
+import { KernelManager, ServerConnection, SessionManager } from '@jupyterlab/services';
+import {
+  findExistingJupyterServer,
+  JupyterServerSettings,
+  launchJupyterServer,
+} from '../transforms/execute.js';
 
 const CONFIG_FILES = ['myst.yml'];
 const API_URL = 'https://api.mystmd.org';
@@ -62,6 +68,7 @@ export class Session implements ISession {
 
   _shownUpgrade = false;
   _latestVersion?: string;
+  _jupyterSessionManager: SessionManager | undefined | null = null;
 
   get log(): Logger {
     return this.$logger;
@@ -157,4 +164,33 @@ export class Session implements ISession {
     });
     return warnings;
   }
+
+  async jupyterSessionManager(): Promise<SessionManager | undefined> {
+    if (this._jupyterSessionManager !== null) {
+      return Promise.resolve(this._jupyterSessionManager);
+    }
+    try {
+      const partialServerSettings = await new Promise<JupyterServerSettings>(
+        async (resolve, reject) => {
+          if (process.env.JUPYTER_BASE_URL === undefined) {
+            resolve(findExistingJupyterServer() || (await launchJupyterServer(this.contentPath(), this.log)));
+          } else {
+            resolve({
+              baseUrl: process.env.JUPYTER_BASE_URL,
+              token: process.env.JUPYTER_TOKEN,
+            });
+          }
+        },
+      );
+      const serverSettings = ServerConnection.makeSettings(partialServerSettings);
+      const kernelManager = new KernelManager({ serverSettings });
+      const manager = new SessionManager({ kernelManager, serverSettings });
+      // TODO: this is a race condition, even though we shouldn't hit if if this promise is actually awaited
+      this._jupyterSessionManager = manager;
+      return manager;
+    } catch {
+      this._jupyterSessionManager = undefined;
+      return undefined;
+    }
+  }
 }

packages/myst-cli/src/session/types.ts

@@ -8,6 +8,7 @@ import type { Store } from 'redux';
 
 import type { BuildWarning, RootState } from '../store/index.js';
 import type { PreRendererData, RendererData, SingleCitationRenderer } from '../transforms/types.js';
+import { SessionManager } from '@jupyterlab/services';
 
 export type ISession = {
   API_URL: string;
@@ -24,6 +25,7 @@ export type ISession = {
   plugins: MystPlugin | undefined;
   loadPlugins(): Promise<MystPlugin>;
   getAllWarnings(ruleId: RuleId): (BuildWarning & { file: string })[];
+  jupyterSessionManager(): Promise<SessionManager | undefined>;
 };
 
 export type ISessionWithCache = ISession & {

packages/myst-cli/src/transforms/execute.ts

@@ -1,6 +1,12 @@
 import { select, selectAll } from 'unist-util-select';
 import type { PageFrontmatter } from 'myst-frontmatter';
-import { Kernel, KernelMessage, Session, SessionManager } from '@jupyterlab/services';
+import {
+  Kernel,
+  KernelMessage,
+  ServerConnection,
+  Session,
+  SessionManager,
+} from '@jupyterlab/services';
 import type { IExpressionResult } from './inlineExpressions.js';
 import type { Code, InlineExpression } from 'myst-spec-ext';
 import type { IOutput } from '@jupyterlab/nbformat';
@@ -11,6 +17,64 @@ import assert from 'node:assert';
 import { createHash } from 'node:crypto';
 import type { ISession } from '../session/index.js';
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import which from 'which';
+import { spawn, spawnSync } from 'node:child_process';
+import { Logger } from 'myst-cli-utils';
+
+export type JupyterServerSettings = Partial<ServerConnection.ISettings> & {
+  dispose?: () => void;
+};
+
+export function findExistingJupyterServer(): JupyterServerSettings | undefined {
+  const pythonPath = which.sync('python');
+  const listProc = spawnSync(pythonPath, ['-m', 'jupyter_server', 'list', '--jsonlist']);
+  if (listProc.status !== 0) {
+    return undefined;
+  }
+  const servers = JSON.parse(listProc.stdout.toString());
+  if (servers.length === 0) {
+    return undefined;
+  }
+  const server = servers.pop()!;
+  return {
+    baseUrl: server.url,
+    token: server.token,
+  };
+}
+
+export function launchJupyterServer(
+  contentPath: string,
+  log: Logger,
+): Promise<JupyterServerSettings> {
+  const pythonPath = which.sync('python');
+  const proc = spawn(pythonPath, ['-m', 'jupyter_server', '--ServerApp.root_dir', contentPath]);
+  const promise = new Promise<JupyterServerSettings>((resolve, reject) => {
+    // proc.stderr.on('data', (data) => console.log({err: data.toString()}))
+    proc.stderr.on('data', (buf) => {
+      const data = buf.toString();
+      // Wait for server to declare itself up
+      const match = data.match(/([^\s]*?)\?token=([^\s]*)/);
+      if (match === null) {
+        return;
+      }
+      proc.stdout.removeAllListeners('data');
+
+      const [_, addr, token] = match;
+
+      // Determine details
+      resolve({
+        baseUrl: addr,
+        token: token,
+        dispose: () => proc.kill('SIGINT'),
+      });
+    });
+    setTimeout(reject, 20_000); // Fail after 20 seconds of nothing happening
+  });
+  promise.then((settings) =>
+    log.info(`Started up Jupyter Server on ${settings.baseUrl}?token=${settings.token}`),
+  );
+  return promise;
+}
 
 /**
  * Interpret an IOPub message as an IOutput object
@@ -102,21 +166,17 @@ async function evaluateExpression(kernel: Kernel.IKernelConnection, expr: string
   return { status: result.status, result };
 }
 
-type CacheItem = (IExpressionResult | IOutput[])[];
-
-type HashableCacheKeyItem = {
-  kind: string;
-  content: string;
-};
-
 /**
  * Build a cache key from an array of executable nodes
  *
  * @param nodes array of executable ndoes
  */
 function buildCacheKey(nodes: (ICellBlock | InlineExpression)[]): string {
   // Build an array of hashable items from an array of nodes
-  const hashableItems: HashableCacheKeyItem[] = [];
+  const hashableItems: {
+    kind: string;
+    content: string;
+  }[] = [];
   for (const node of nodes) {
     if (isCellBlock(node)) {
       hashableItems.push({
@@ -300,7 +360,6 @@ function applyComputedOutputsToNodes(
  * Transform an AST to include the outputs of executing the given notebook
  *
  * @param session
- * @param sessionManager
  * @param mdast
  * @param frontmatter
  * @param ignoreCache
@@ -309,7 +368,6 @@ function applyComputedOutputsToNodes(
  */
 export async function transformKernelExecution(
   session: ISession,
-  sessionManager: SessionManager,
   mdast: GenericParent,
   frontmatter: PageFrontmatter,
   ignoreCache: boolean,
@@ -346,32 +404,39 @@ export async function transformKernelExecution(
         ? 'Code cells and expressions will be re-evaluated, as the cache is being ignored'
         : 'Code cells and expressions will be re-evaluated, as there is no entry in the execution cache',
     );
-    // Boot up a kernel, and execute each cell
-    let sessionConnection: Session.ISessionConnection | undefined;
-    return await sessionManager
-      .startNew(options)
-      .then(async (conn) => {
-        sessionConnection = conn;
-        assert(conn.kernel);
-        fileInfo(vfile, `Connected to kernel ${conn.kernel.name}`);
-        // Execute notebook
-        const { results, errorOccurred } = await computeExecutableNodes(
-          conn.kernel,
-          executableNodes,
-          vfile,
-        );
-        // Populate cache if things were successful
-        if (!errorOccurred) {
-          cache.set(cacheKey, results);
-        } else {
-          // Otherwise, keep tabs on the error
-          fileError(vfile, 'An error occurred during kernel execution', { fatal: errorIsFatal });
-        }
-        // Refer to these computed results
-        cachedResults = results;
-      })
-      // Ensure that we shut-down the kernel
-      .finally(async () => sessionConnection !== undefined && sessionConnection.shutdown());
+    const sessionManager = await session.jupyterSessionManager();
+    // Do we not have a working session?
+    if (sessionManager === undefined) {
+      cachedResults = [];
+    }
+    // Otherwise, boot up a kernel, and execute each cell
+    else {
+      let sessionConnection: Session.ISessionConnection | undefined;
+      await sessionManager
+        .startNew(options)
+        .then(async (conn) => {
+          sessionConnection = conn;
+          assert(conn.kernel);
+          fileInfo(vfile, `Connected to kernel ${conn.kernel.name}`);
+          // Execute notebook
+          const { results, errorOccurred } = await computeExecutableNodes(
+            conn.kernel,
+            executableNodes,
+            vfile,
+          );
+          // Populate cache if things were successful
+          if (!errorOccurred) {
+            cache.set(cacheKey, results);
+          } else {
+            // Otherwise, keep tabs on the error
+            fileError(vfile, 'An error occurred during kernel execution', { fatal: errorIsFatal });
+          }
+          // Refer to these computed results
+          cachedResults = results;
+        })
+        // Ensure that we shut-down the kernel
+        .finally(async () => sessionConnection !== undefined && sessionConnection.shutdown());
+    }
   }
   assert(cachedResults !== undefined);