diff --git a/packages/myst-cli/src/build/build.ts b/packages/myst-cli/src/build/build.ts index 589578f79..73eb39d80 100644 --- a/packages/myst-cli/src/build/build.ts +++ b/packages/myst-cli/src/build/build.ts @@ -27,6 +27,7 @@ export type BuildOpts = { output?: string; checkLinks?: boolean; ci?: boolean; + execute?: boolean; }; export function hasAnyExplicitExportFormat(opts: BuildOpts): boolean { diff --git a/packages/myst-cli/src/build/clean.ts b/packages/myst-cli/src/build/clean.ts index b46afb542..5db0c9185 100644 --- a/packages/myst-cli/src/build/clean.ts +++ b/packages/myst-cli/src/build/clean.ts @@ -27,6 +27,7 @@ export type CleanOptions = { html?: boolean; temp?: boolean; exports?: boolean; + execute?: boolean; templates?: boolean; all?: boolean; yes?: boolean; @@ -43,6 +44,7 @@ const ALL_OPTS: CleanOptions = { html: true, temp: true, exports: true, + execute: true, templates: true, }; const DEFAULT_OPTS: CleanOptions = { @@ -56,10 +58,12 @@ const DEFAULT_OPTS: CleanOptions = { html: true, temp: true, exports: true, + execute: true, }; function coerceOpts(opts: CleanOptions) { - const { docx, pdf, tex, xml, md, meca, site, html, temp, exports, templates, all } = opts; + const { docx, pdf, tex, xml, md, meca, site, html, temp, exports, execute, templates, all } = + opts; if (all) return { ...opts, ...ALL_OPTS }; if ( !docx && @@ -72,6 +76,7 @@ function coerceOpts(opts: CleanOptions) { !html && !temp && !exports && + !execute && !templates ) { return { ...opts, ...DEFAULT_OPTS }; @@ -112,7 +117,7 @@ function deduplicatePaths(paths: string[]) { export async function clean(session: ISession, files: string[], opts: CleanOptions) { opts = coerceOpts(opts); - const { site, html, temp, exports, templates, yes } = opts; + const { site, html, temp, exports, execute, templates, yes } = opts; let pathsToDelete: string[] = []; const exportOptionsList = await collectAllBuildExportOptions(session, files, opts); if (exports) { @@ -125,7 +130,7 @@ export async function clean(session: ISession, files: string[], opts: CleanOptio }); } let buildFolders: string[] = []; - if (temp || exports || templates || html) { + if (temp || exports || execute || templates || html) { const projectPaths = [ ...getProjectPaths(session), ...exportOptionsList.map((exp) => exp.$project), @@ -138,12 +143,13 @@ export async function clean(session: ISession, files: string[], opts: CleanOptio buildFolders.push(session.buildPath()); } buildFolders = [...new Set(buildFolders)].sort(); - if (temp || exports || templates || html) { + if (temp || exports || templates || execute || html) { buildFolders.forEach((folder) => { if (temp) pathsToDelete.push(path.join(folder, 'temp')); if (exports) pathsToDelete.push(path.join(folder, 'exports')); if (templates) pathsToDelete.push(path.join(folder, 'templates')); if (html) pathsToDelete.push(path.join(folder, 'html')); + if (execute) pathsToDelete.push(path.join(folder, 'execute')); }); } if (site) { diff --git a/packages/myst-cli/src/build/site/prepare.ts b/packages/myst-cli/src/build/site/prepare.ts index 4388bed69..1baa54984 100644 --- a/packages/myst-cli/src/build/site/prepare.ts +++ b/packages/myst-cli/src/build/site/prepare.ts @@ -12,6 +12,7 @@ export type Options = { checkLinks?: boolean; yes?: boolean; port?: number; + execute?: boolean; serverPort?: number; writeToc?: boolean; keepHost?: boolean; @@ -38,8 +39,15 @@ export function ensureBuildFoldersExist(session: ISession): void { } export async function buildSite(session: ISession, opts: Options) { - const { writeToc, strict, checkLinks, extraLinkTransformers, extraTransforms, defaultTemplate } = - opts; + const { + writeToc, + strict, + checkLinks, + extraLinkTransformers, + extraTransforms, + defaultTemplate, + execute, + } = opts; ensureBuildFoldersExist(session); await processSite(session, { writeToc, @@ -48,5 +56,6 @@ export async function buildSite(session: ISession, opts: Options) { extraLinkTransformers, extraTransforms, defaultTemplate, + execute, }); } diff --git a/packages/myst-cli/src/build/site/start.ts b/packages/myst-cli/src/build/site/start.ts index 144b432eb..f220f18d2 100644 --- a/packages/myst-cli/src/build/site/start.ts +++ b/packages/myst-cli/src/build/site/start.ts @@ -112,12 +112,13 @@ export async function startServer( if (!opts.headless) await installSiteTemplate(session, mystTemplate); await buildSite(session, opts); const server = await startContentServer(session, opts); - const { extraLinkTransformers, extraTransforms, defaultTemplate } = opts; + const { extraLinkTransformers, extraTransforms, defaultTemplate, execute } = opts; if (!opts.buildStatic) { watchContent(session, server.reload, { extraLinkTransformers, extraTransforms, defaultTemplate, + execute, }); } if (opts.headless) { diff --git a/packages/myst-cli/src/build/site/watch.ts b/packages/myst-cli/src/build/site/watch.ts index 85b35afac..0287e0598 100644 --- a/packages/myst-cli/src/build/site/watch.ts +++ b/packages/myst-cli/src/build/site/watch.ts @@ -15,6 +15,7 @@ type TransformOptions = { extraTransforms?: TransformFn[]; defaultTemplate?: string; reloadProject?: boolean; + execute?: boolean; }; function watchConfigAndPublic(session: ISession, serverReload: () => void, opts: TransformOptions) { diff --git a/packages/myst-cli/src/process/mdast.ts b/packages/myst-cli/src/process/mdast.ts index ddeaa9d4a..7d570823c 100644 --- a/packages/myst-cli/src/process/mdast.ts +++ b/packages/myst-cli/src/process/mdast.ts @@ -17,6 +17,7 @@ import { enumerateTargetsPlugin, keysTransform, linksTransform, + MultiPageReferenceState, MystTransformer, WikiTransformer, GithubTransformer, @@ -115,6 +116,7 @@ export async function transformMdast( pageSlug?: string; imageExtensions?: ImageExtensions[]; watchMode?: boolean; + execute?: boolean; extraTransforms?: TransformFn[]; minifyMaxCharacters?: number; index?: string; @@ -132,6 +134,7 @@ export async function transformMdast( minifyMaxCharacters, index, titleDepth, + execute, } = opts; const toc = tic(); const { store, log } = session; @@ -230,14 +233,17 @@ export async function transformMdast( // Combine file-specific citation renderers with project renderers from bib files const fileCitationRenderer = combineCitationRenderers(cache, ...rendererFiles); - const cachePath = path.join(session.buildPath(), 'execute'); - await kernelExecutionTransform(mdast, vfile, { - cache: new LocalDiskCache<(IExpressionResult | IOutput[])[]>(cachePath), - sessionFactory: () => session.jupyterSessionManager(), - frontmatter: frontmatter, - ignoreCache: false, - errorIsFatal: false, - }); + if (execute) { + const cachePath = path.join(session.buildPath(), 'execute'); + await kernelExecutionTransform(mdast, vfile, { + cache: new LocalDiskCache<(IExpressionResult | IOutput[])[]>(cachePath), + sessionFactory: () => session.jupyterSessionManager(), + frontmatter: frontmatter, + ignoreCache: false, + errorIsFatal: false, + log: session.log, + }); + } transformRenderInlineExpressions(mdast, vfile); transformFilterOutputStreams(mdast, vfile, frontmatter.settings); diff --git a/packages/myst-cli/src/process/site.ts b/packages/myst-cli/src/process/site.ts index 55e35331c..0d14cfd94 100644 --- a/packages/myst-cli/src/process/site.ts +++ b/packages/myst-cli/src/process/site.ts @@ -48,6 +48,8 @@ type ProcessOptions = { defaultTemplate?: string; reloadProject?: boolean; minifyMaxCharacters?: number; + /** Execute flag for notebooks */ + execute?: boolean; }; /** @@ -206,6 +208,7 @@ export async function fastProcessFile( extraLinkTransformers, extraTransforms, defaultTemplate, + execute, }: { file: string; pageSlug: string; @@ -214,6 +217,7 @@ export async function fastProcessFile( extraLinkTransformers?: LinkTransformer[]; extraTransforms?: TransformFn[]; defaultTemplate?: string; + execute?: boolean; }, ) { const toc = tic(); @@ -228,6 +232,7 @@ export async function fastProcessFile( watchMode: true, extraTransforms, index: project.index, + execute, }); const pageReferenceStates = selectPageReferenceStates(session, pages); await postProcessMdast(session, { @@ -267,6 +272,7 @@ export async function processProject( writeFiles = true, reloadProject, minifyMaxCharacters, + execute, } = opts || {}; if (!siteProject.path) { const slugSuffix = siteProject.slug ? `: ${siteProject.slug}` : ''; @@ -283,9 +289,7 @@ export async function processProject( // Load all citations (.bib) ...project.bibliography.map((path) => loadFile(session, path, siteProject.path, '.bib')), // Load all content (.md and .ipynb) - ...pages.map((page) => - loadFile(session, page.file, siteProject.path, undefined), - ), + ...pages.map((page) => loadFile(session, page.file, siteProject.path, undefined)), // Load up all the intersphinx references loadIntersphinx(session, { projectPath: siteProject.path }) as Promise, ]); @@ -306,6 +310,7 @@ export async function processProject( watchMode, extraTransforms: opts?.extraTransforms, index: project.index, + execute, }), ), ); diff --git a/packages/myst-cli/src/session/session.ts b/packages/myst-cli/src/session/session.ts index bebcf5c23..20ba21b33 100644 --- a/packages/myst-cli/src/session/session.ts +++ b/packages/myst-cli/src/session/session.ts @@ -185,9 +185,14 @@ export class Session implements ISession { token: process.env.JUPYTER_TOKEN, }; } else { - partialServerSettings = - (await findExistingJupyterServer()) || - (await launchJupyterServer(this.contentPath(), this.log)); + const existing = await findExistingJupyterServer(); + if (existing) { + this.log.debug(`Found existing server on: ${existing.appUrl}`); + partialServerSettings = existing; + } else { + this.log.debug(`Launching jupyter server on ${this.contentPath()}`); + partialServerSettings = await launchJupyterServer(this.contentPath(), this.log); + } } const serverSettings = ServerConnection.makeSettings(partialServerSettings); diff --git a/packages/myst-cli/src/utils/addWarningForFile.ts b/packages/myst-cli/src/utils/addWarningForFile.ts index 29b830d87..d9575dfff 100644 --- a/packages/myst-cli/src/utils/addWarningForFile.ts +++ b/packages/myst-cli/src/utils/addWarningForFile.ts @@ -28,7 +28,7 @@ export function addWarningForFile( const formatted = `${message}${note}${url}`; switch (kind) { case 'info': - session.log.info(`ℹ️ ${prefix}${formatted}`); + session.log.info(`ℹ️ ${prefix}${formatted}`); break; case 'error': session.log.error(`⛔️ ${prefix}${formatted}`); diff --git a/packages/myst-execute/package.json b/packages/myst-execute/package.json index 25e558745..b02da18d7 100644 --- a/packages/myst-execute/package.json +++ b/packages/myst-execute/package.json @@ -32,6 +32,7 @@ "url": "https://github.com/executablebooks/mystmd/issues" }, "dependencies": { + "chalk": "^5.2.0", "myst-common": "^1.1.21", "vfile": "^5.3.7", "@jupyterlab/services": "^7.0.0", diff --git a/packages/myst-execute/src/execute.ts b/packages/myst-execute/src/execute.ts index 11467b570..4453fa740 100644 --- a/packages/myst-execute/src/execute.ts +++ b/packages/myst-execute/src/execute.ts @@ -1,10 +1,11 @@ import { select, selectAll } from 'unist-util-select'; +import type { Logger } from 'myst-cli-utils'; import type { PageFrontmatter } from 'myst-frontmatter'; import type { Kernel, KernelMessage, Session, SessionManager } from '@jupyterlab/services'; import type { Code, InlineExpression } from 'myst-spec-ext'; import type { IOutput } from '@jupyterlab/nbformat'; import type { GenericNode, GenericParent, IExpressionResult } from 'myst-common'; -import { fileError, fileInfo, fileWarn } from 'myst-common'; +import { fileError, fileWarn } from 'myst-common'; import type { VFile } from 'vfile'; import path from 'node:path'; import assert from 'node:assert'; @@ -30,7 +31,12 @@ function IOPubAsOutput(msg: KernelMessage.IIOPubMessage): IOutput { * @param kernel connection to an active kernel * @param code code to execute */ -async function executeCode(kernel: Kernel.IKernelConnection, code: string) { +async function executeCode( + kernel: Kernel.IKernelConnection, + code: string, + opts?: { log?: Logger }, +) { + const log = opts?.log ?? console; const future = kernel.requestExecute({ code: code, }); @@ -103,7 +109,7 @@ async function evaluateExpression(kernel: Kernel.IKernelConnection, expr: string /** * Build a cache key from an array of executable nodes * - * @param nodes array of executable ndoes + * @param nodes array of executable nodes */ function buildCacheKey(nodes: (ICellBlock | InlineExpression)[]): string { // Build an array of hashable items from an array of nodes @@ -125,7 +131,7 @@ function buildCacheKey(nodes: (ICellBlock | InlineExpression)[]): string { }); } } - // Serialise the array into JSON, and compute the hash + // Serialize the array into JSON, and compute the hash const hashableString = JSON.stringify(hashableItems); return createHash('md5').update(hashableString).digest('hex'); } @@ -158,7 +164,7 @@ function isInlineExpression(node: GenericNode): node is InlineExpression { /** * For each executable node, perform a kernel execution request and return the results. - * Return an additional boolean indicating whether an error occured. + * Return an additional boolean indicating whether an error occurred. * * @param kernel * @param nodes @@ -167,7 +173,7 @@ function isInlineExpression(node: GenericNode): node is InlineExpression { async function computeExecutableNodes( kernel: Kernel.IKernelConnection, nodes: (ICellBlock | InlineExpression)[], - vfile: VFile, + opts: { vfile: VFile; log?: Logger }, ): Promise<{ results: (IOutput[] | IExpressionResult)[]; errorOccurred: boolean; @@ -179,7 +185,7 @@ async function computeExecutableNodes( if (isCellBlock(matchedNode)) { // Pull out code to execute const code = select('code', matchedNode) as Code; - const { status, outputs } = await executeCode(kernel, code.value); + const { status, outputs } = await executeCode(kernel, code.value, { log: opts.log }); // Cache result results.push(outputs); @@ -187,7 +193,13 @@ async function computeExecutableNodes( const metadata = matchedNode.data || {}; const allowErrors = !!metadata?.tags?.['raises-exception']; if (status === 'error' && !allowErrors) { - fileWarn(vfile, 'An exception occurred during code execution, halting further execution'); + fileWarn( + opts.vfile, + 'An exception occurred during code execution, halting further execution', + { + node: matchedNode, + }, + ); // Make a note of the failure errorOccurred = true; break; @@ -199,8 +211,9 @@ async function computeExecutableNodes( // Check for errors if (status === 'error') { fileWarn( - vfile, + opts.vfile, 'An exception occurred during expression evaluation, halting further execution', + { node: matchedNode }, ); // Make a note of the failure errorOccurred = true; @@ -253,16 +266,18 @@ export type Options = { frontmatter: PageFrontmatter; ignoreCache?: boolean; errorIsFatal?: boolean; + log?: Logger; }; /** * Transform an AST to include the outputs of executing the given notebook * * @param tree - * @param file + * @param vfile * @param opts */ -export async function kernelExecutionTransform(tree: GenericParent, file: VFile, opts: Options) { +export async function kernelExecutionTransform(tree: GenericParent, vfile: VFile, opts: Options) { + const log = opts.log ?? console; // Pull out code-like nodes const executableNodes = selectAll( 'block:has(code[executable=true]):has(output),inlineExpression', @@ -280,16 +295,15 @@ export async function kernelExecutionTransform(tree: GenericParent, file: VFile, // Do we need to re-execute notebook? if (opts.ignoreCache || cachedResults === undefined) { - fileInfo( - file, - opts.ignoreCache - ? 'Code cells and expressions will be re-evaluated, as the cache is being ignored' - : 'Code cells and expressions will be re-evaluated, as there is no entry in the execution cache', + log.info( + `💿 Executing Notebook (${vfile.path}) ${ + opts.ignoreCache ? '[cache ignored]' : '[no execution cache found]' + }`, ); const sessionManager = await opts.sessionFactory(); // Do we not have a working session? if (sessionManager === undefined) { - fileError(file, `Could not load Jupyter session manager to run executable nodes`, { + fileError(vfile, `Could not load Jupyter session manager to run executable nodes`, { fatal: opts.errorIsFatal, }); } @@ -297,31 +311,36 @@ export async function kernelExecutionTransform(tree: GenericParent, file: VFile, else { let sessionConnection: Session.ISessionConnection | undefined; const sessionOpts = { - path: file.path, + path: vfile.path, type: 'notebook', - name: path.basename(file.path), + name: path.basename(vfile.path), kernel: { name: opts.frontmatter?.kernelspec?.name ?? 'python3', }, }; await sessionManager .startNew(sessionOpts) + .catch((err) => { + log.debug((err as Error).stack); + log.error('Jupyter connection error'); + }) .then(async (conn) => { + if (!conn) return; sessionConnection = conn; assert(conn.kernel); - fileInfo(file, `Connected to kernel ${conn.kernel.name}`); + log.debug(`Connected to kernel ${conn.kernel.name}`); // Execute notebook const { results, errorOccurred } = await computeExecutableNodes( conn.kernel, executableNodes, - file, + { vfile, log }, ); // Populate cache if things were successful if (!errorOccurred) { opts.cache.set(cacheKey, results); } else { // Otherwise, keep tabs on the error - fileError(file, 'An error occurred during kernel execution', { + fileError(vfile, 'An error occurred during kernel execution', { fatal: opts.errorIsFatal, }); } @@ -331,6 +350,9 @@ export async function kernelExecutionTransform(tree: GenericParent, file: VFile, // Ensure that we shut-down the kernel .finally(async () => sessionConnection !== undefined && sessionConnection.shutdown()); } + } else { + // We found the cache, adding them in below! + log.info(`💾 Adding Cached Notebook Outputs (${vfile.path})`); } if (cachedResults) { diff --git a/packages/myst-execute/src/manager.ts b/packages/myst-execute/src/manager.ts index fc907c8c6..7f00fbb68 100644 --- a/packages/myst-execute/src/manager.ts +++ b/packages/myst-execute/src/manager.ts @@ -3,6 +3,7 @@ import which from 'which'; import { spawn } from 'node:child_process'; import * as readline from 'node:readline'; import type { Logger } from 'myst-cli-utils'; +import chalk from 'chalk'; export type JupyterServerSettings = Partial & { dispose?: () => void; @@ -48,6 +49,7 @@ export async function findExistingJupyterServer(): Promise a.pid - b.pid); const server = servers.pop()!; + // TODO: We should ping the server to ensure that it actually is up! return { baseUrl: server.url, token: server.token, @@ -64,6 +66,7 @@ export function launchJupyterServer( contentPath: string, log: Logger, ): Promise { + log.info(`🚀 ${chalk.yellowBright('Starting new Jupyter server')}`); const pythonPath = which.sync('python'); const proc = spawn(pythonPath, ['-m', 'jupyter_server', '--ServerApp.root_dir', contentPath]); const promise = new Promise((resolve, reject) => { @@ -76,7 +79,7 @@ export function launchJupyterServer( } // Pull out the match information - const [_, addr, token] = match; + const [, addr, token] = match; // Resolve the promise resolve({ @@ -90,8 +93,9 @@ export function launchJupyterServer( setTimeout(reject, 20_000); // Fail after 20 seconds of nothing happening }); // Inform log - promise.then((settings) => - log.info(`Started up Jupyter Server on ${settings.baseUrl}?token=${settings.token}`), - ); + promise.then((settings) => { + const url = `${settings.baseUrl}?token=${settings.token}`; + log.info(`🪐 ${chalk.greenBright('Jupyter Server Started')}\n ${chalk.dim(url)}`); + }); return promise; } diff --git a/packages/mystmd/src/build.ts b/packages/mystmd/src/build.ts index 0bae1ce58..6b113de6c 100644 --- a/packages/mystmd/src/build.ts +++ b/packages/mystmd/src/build.ts @@ -18,12 +18,14 @@ import { makeTypstOption, makeWatchOption, makeCIOption, + makeExecuteOption, } from './options.js'; export function makeBuildCLI(program: Command) { const command = new Command('build') .description('Build PDF, LaTeX, Word and website exports from MyST files') .argument('[files...]', 'list of files to export') + .addOption(makeExecuteOption('Execute Notebooks')) .addOption(makePdfOption('Build PDF output')) .addOption(makeTexOption('Build LaTeX outputs')) .addOption(makeTypstOption('Build Typst outputs')) diff --git a/packages/mystmd/src/clean.ts b/packages/mystmd/src/clean.ts index 4141646b1..99136b91d 100644 --- a/packages/mystmd/src/clean.ts +++ b/packages/mystmd/src/clean.ts @@ -12,6 +12,7 @@ import { makeSiteOption, makeTexOption, makeTypstOption, + makeExecuteOption, makeYesOption, } from './options.js'; @@ -49,6 +50,7 @@ export function makeCleanCLI(program: Command) { .addOption(makeMecaOptions('Clean MECA zip output')) .addOption(makeSiteOption('Clean MyST site content')) .addOption(makeHtmlOption('Clean static HTML site content')) + .addOption(makeExecuteOption('Clean execute cache')) .addOption(makeTempOption()) .addOption(makeExportsOption()) .addOption(makeTemplatesOption()) diff --git a/packages/mystmd/src/options.ts b/packages/mystmd/src/options.ts index fdd8f6c48..754eade8d 100644 --- a/packages/mystmd/src/options.ts +++ b/packages/mystmd/src/options.ts @@ -42,6 +42,10 @@ export function makeHtmlOption(description: string) { return new Option('--html', description).default(false); } +export function makeExecuteOption(description: string) { + return new Option('--execute', description).default(false); +} + export function makeProjectOption(description: string) { return new Option('--project', description).default(false); } diff --git a/packages/mystmd/src/site.ts b/packages/mystmd/src/site.ts index af8746f51..fb2ac125d 100644 --- a/packages/mystmd/src/site.ts +++ b/packages/mystmd/src/site.ts @@ -6,11 +6,13 @@ import { makeHeadlessOption, makePortOption, makeServerPortOption, + makeExecuteOption, } from './options.js'; export function makeStartCLI(program: Command) { const command = new Command('start') .description('Start the current project as a website') + .addOption(makeExecuteOption('Execute Notebooks')) .addOption(makeKeepHostOption()) .addOption(makeHeadlessOption()) .addOption(makePortOption())