📠 Add up/downgrade for outputs AST change

.changeset/cold-beers-scream.md

@@ -0,0 +1,5 @@
+---
+"myst-migrate": patch
+---
+
+Add support for upgrade/downgrade of outputs

packages/myst-migrate/src/migrations.ts

@@ -1,5 +1,6 @@
 import type { Migration } from './types.js';
 import * as v1 from './v1_footnotes.js';
 import * as v2 from './v2_blockClasses.js';
+import * as v3 from './v3_outputs.js';
 
-export const MIGRATIONS: Migration[] = [v1, v2];
+export const MIGRATIONS: Migration[] = [v1, v2, v3];

packages/myst-migrate/src/tests/version_2_3.spec.ts

@@ -0,0 +1,154 @@
+import { describe, expect, it } from 'vitest';
+import { migrate } from '../index';
+import type { Parent } from 'mdast';
+
+const SIMPLE_AST: Parent = {
+  type: 'root',
+  children: [
+    {
+      // @ts-expect-error: unknown type
+      type: 'block',
+      children: [
+        {
+          // @ts-expect-error: invalid child type
+          type: 'paragraph',
+          children: [
+            {
+              type: 'text',
+              value: 'This is an ',
+            },
+            {
+              type: 'emphasis',
+              children: [
+                {
+                  type: 'text',
+                  value: 'interesting',
+                },
+              ],
+            },
+            {
+              type: 'text',
+              value: ' file. See ',
+            },
+            {
+              type: 'link',
+              url: '#other',
+              children: [],
+              // @ts-expect-error: unknown field
+              urlSource: '#other',
+            },
+            {
+              type: 'text',
+              value: ' for more.',
+            },
+          ],
+        },
+        {
+          // @ts-expect-error: invalid child type
+          type: 'code',
+          lang: 'python',
+          value: 'Some code',
+        },
+      ],
+    },
+  ],
+};
+
+const SIMPLE_V3_AST_WITH_OUTPUT: Parent = {
+  type: 'root',
+  children: [
+    {
+      // @ts-expect-error: unknown type
+      type: 'block',
+      children: [
+        {
+          // @ts-expect-error: invalid child type
+          type: 'outputs',
+          id: 'abc123',
+          children: [
+            {
+              // @ts-expect-error: invalid child type
+              type: 'output',
+              children: [],
+              jupyter_data: {
+                output_type: 'display_data',
+                execution_count: 3,
+                metadata: {},
+                data: {
+                  'application/octet-stream': {
+                    content_type: 'application/octet-stream',
+                    hash: 'def456',
+                    path: '/my/path/def456.png',
+                  },
+                },
+              },
+            },
+          ],
+        },
+      ],
+    },
+  ],
+};
+
+const SIMPLE_V2_AST_WITH_OUTPUT: Parent = {
+  type: 'root',
+  children: [
+    {
+      // @ts-expect-error: unknown type
+      type: 'block',
+      children: [
+        {
+          // @ts-expect-error: invalid child type
+          type: 'output',
+          id: 'abc123',
+          // @ts-expect-error: invalid type
+          data: [
+            {
+              output_type: 'display_data',
+              execution_count: 3,
+              metadata: {},
+              data: {
+                'application/octet-stream': {
+                  content_type: 'application/octet-stream',
+                  hash: 'def456',
+                  path: '/my/path/def456.png',
+                },
+              },
+            },
+          ],
+          children: [],
+        },
+      ],
+    },
+  ],
+};
+
+describe('downgrade 3->2', () => {
+  it('leaves a simple AST unchanged', async () => {
+    const mdast = structuredClone(SIMPLE_AST) as any;
+    const result = await migrate({ version: 3, mdast }, { to: 2 });
+    expect(result.version).toBe(2);
+    expect(mdast).toStrictEqual(SIMPLE_AST);
+  });
+  it('downgrades an AST with outputs', async () => {
+    const mdast = structuredClone(SIMPLE_V3_AST_WITH_OUTPUT);
+    const result = await migrate({ version: 3, mdast }, { to: 2 });
+    expect(result.version).toBe(2);
+    expect(mdast).toStrictEqual(SIMPLE_V2_AST_WITH_OUTPUT);
+  });
+});
+
+describe('upgrade 3->2', () => {
+  it('leaves a simple AST unchanged', async () => {
+    const mdast = structuredClone(SIMPLE_AST) as any;
+    const result = await migrate({ version: 2, mdast }, { to: 3 });
+    expect(result.version).toBe(3);
+    expect(mdast).toStrictEqual(SIMPLE_AST);
+  });
+  it('upgrades an AST with output', async () => {
+    const mdast = structuredClone(SIMPLE_V2_AST_WITH_OUTPUT);
+    const result = await migrate({ version: 2, mdast }, { to: 3 });
+    expect(result.version).toBe(3);
+    expect(mdast).toStrictEqual(SIMPLE_V3_AST_WITH_OUTPUT);
+  });
+});

packages/myst-migrate/src/v3_outputs.ts

@@ -0,0 +1,93 @@
+import { selectAll } from 'unist-util-select';
+import { assert } from 'console';
+import type { IFile } from './types.js';
+
+export const description = `
+Output nodes previously could not represent AST trees for each output. 
+Now, the Outputs node has Output children with a 1:1 correspondence to IOutput bundles.
+`;
+
+type OutputV2 = {
+  type: 'output';
+  children?: any[];
+
+  data?: any[];
+  visibility?: any;
+
+  html_id?: string;
+  label?: string;
+  identifier?: string;
+
+  id?: string;
+};
+
+type OutputV3 = {
+  type: 'output';
+  children: any[];
+
+  jupyter_data?: any;
+
+  label?: string;
+  identifier?: string;
+  html_id?: string;
+};
+
+export function upgrade(file: IFile): IFile {
+  const { version, mdast } = file;
+  assert(version === 2, 'Version must be 2');
+  const nodes = selectAll('output', mdast) as OutputV2[];
+  nodes.forEach((node) => {
+    // We can only correlate output children with the IOutput objects if there's only one IOutput
+    // Additionally, there may be placeholders that need to be removed
+    const numOutputs = node.data?.length ?? 0;
+    const children = numOutputs === 1 ? node.children ?? [] : [];
+    const placeholders = children.filter((child) => !!child.placeholder);
+    const notPlaceholders = children.filter((child) => !child.placeholder);
+
+    const outputsChildren = (node.data ?? []).map((outputData) => {
+      const result: OutputV3 = {
+        type: 'output',
+        jupyter_data: outputData,
+        children: notPlaceholders, // FIXME: ignoring children here
+      };
+      notPlaceholders.length = 0;
+      return result;
+    });
+
+    // Restore placeholders at the end
+    outputsChildren.push(...placeholders);
+
+    // Convert Output into Outputs
+    if (node.data !== undefined) {
+      delete node.data;
+    }
+    (node as any).type = 'outputs';
+    (node as any).children = outputsChildren;
+  });
+  return file;
+}
+
+export function downgrade(file: IFile): IFile {
+  const { version, mdast } = file;
+  assert(version === 3, 'Version must be 3');
+  const nodes = selectAll('outputs', mdast) as OutputV2[];
+  nodes.forEach((node) => {
+    const outputsChildren = node.children as any[];
+    const data = outputsChildren
+      .filter((output) => output.type === 'output')
+      .map((output: any) => output.jupyter_data)
+      .filter((datum) => !!datum);
+
+    const notPlaceholders = outputsChildren.filter((child: any) => !child.placeholder);
+    const children = notPlaceholders.map((output) => (output as OutputV3).children ?? []).flat();
+    const placeholders = outputsChildren.filter((child: any) => !!child.placeholder);
+    children.push(...placeholders);
+
+    // Convert Outputs into Output
+    (node as any).data = data;
+
+    (node as any).type = 'output';
+    (node as any).children = children;
+  });
+  return file;
+}