Bump shikiji, use transformers API, expose transformers API

.changeset/six-scissors-worry.md

@@ -0,0 +1,5 @@
+---
+"@astrojs/markdown-remark": minor
+---
+
+Bump shikiji, use transformers API

packages/astro/package.json

@@ -166,7 +166,7 @@
     "resolve": "^1.22.4",
     "semver": "^7.5.4",
     "server-destroy": "^1.0.1",
-    "shikiji": "^0.6.13",
+    "shikiji": "^0.9.17",
     "string-width": "^7.0.0",
     "strip-ansi": "^7.1.0",
     "tsconfck": "^3.0.0",
@@ -224,6 +224,7 @@
     "remark-code-titles": "^0.1.2",
     "rollup": "^4.5.0",
     "sass": "^1.69.5",
+    "shikiji-core": "^0.9.17",
     "srcset-parse": "^1.1.0",
     "unified": "^11.0.4"
   },

packages/astro/src/core/config/schema.ts

@@ -17,9 +17,11 @@ import { appendForwardSlash, prependForwardSlash, removeTrailingForwardSlash } f
 // This import is required to appease TypeScript!
 // See https://github.com/withastro/astro/pull/8762
 import 'mdast-util-to-hast';
+import 'shikiji-core';
 
 type ShikiLangs = NonNullable<ShikiConfig['langs']>;
 type ShikiTheme = NonNullable<ShikiConfig['theme']>;
+type ShikiTransformers = ShikiConfig['transformers'];
 
 const ASTRO_CONFIG_DEFAULTS = {
 	root: '.',
@@ -274,6 +276,7 @@ export const AstroConfigSchema = z.object({
 						)
 						.default(ASTRO_CONFIG_DEFAULTS.markdown.shikiConfig.experimentalThemes!),
 					wrap: z.boolean().or(z.null()).default(ASTRO_CONFIG_DEFAULTS.markdown.shikiConfig.wrap!),
+					transformers: z.custom<ShikiTransformers>(),
 				})
 				.default({}),
 			remarkPlugins: z

packages/integrations/markdoc/package.json

@@ -71,7 +71,7 @@
     "gray-matter": "^4.0.3",
     "htmlparser2": "^9.0.0",
     "kleur": "^4.1.5",
-    "shikiji": "^0.6.13",
+    "shikiji": "^0.9.17",
     "zod": "^3.22.4"
   },
   "peerDependencies": {

packages/markdown/remark/package.json

@@ -38,7 +38,7 @@
     "remark-parse": "^11.0.0",
     "remark-rehype": "^11.0.0",
     "remark-smartypants": "^2.0.0",
-    "shikiji": "^0.6.13",
+    "shikiji": "^0.9.17",
     "unified": "^11.0.4",
     "unist-util-visit": "^5.0.0",
     "vfile": "^6.0.1"

packages/markdown/remark/src/shiki.ts

@@ -1,4 +1,4 @@
-import { bundledLanguages, getHighlighter } from 'shikiji';
+import { bundledLanguages, createCssVariablesTheme, getHighlighter } from 'shikiji';
 import { visit } from 'unist-util-visit';
 import type { ShikiConfig } from './types.js';
 
@@ -7,31 +7,29 @@ export interface ShikiHighlighter {
 }
 
 const ASTRO_COLOR_REPLACEMENTS: Record<string, string> = {
-	'#000001': 'var(--astro-code-color-text)',
-	'#000002': 'var(--astro-code-color-background)',
-	'#000004': 'var(--astro-code-token-constant)',
-	'#000005': 'var(--astro-code-token-string)',
-	'#000006': 'var(--astro-code-token-comment)',
-	'#000007': 'var(--astro-code-token-keyword)',
-	'#000008': 'var(--astro-code-token-parameter)',
-	'#000009': 'var(--astro-code-token-function)',
-	'#000010': 'var(--astro-code-token-string-expression)',
-	'#000011': 'var(--astro-code-token-punctuation)',
-	'#000012': 'var(--astro-code-token-link)',
+	'--astro-code-foreground': '--astro-code-color-text',
+	'--astro-code-background': '--astro-code-color-background',
 };
 const COLOR_REPLACEMENT_REGEX = new RegExp(
 	`(${Object.keys(ASTRO_COLOR_REPLACEMENTS).join('|')})`,
 	'g'
 );
 
+const cssVariablesTheme = createCssVariablesTheme({
+	variablePrefix: '--astro-code-',
+});
+
 export async function createShikiHighlighter({
 	langs = [],
 	theme = 'github-dark',
 	experimentalThemes = {},
 	wrap = false,
+	transformers = [],
 }: ShikiConfig = {}): Promise<ShikiHighlighter> {
 	const themes = experimentalThemes;
 
+	theme = theme === 'css-variables' ? cssVariablesTheme : theme;
+
 	const highlighter = await getHighlighter({
 		langs: langs.length ? langs : Object.keys(bundledLanguages),
 		themes: Object.values(themes).length ? Object.values(themes) : [theme],
@@ -53,74 +51,77 @@ export async function createShikiHighlighter({
 			return highlighter.codeToHtml(code, {
 				...themeOptions,
 				lang,
-				transforms: {
-					pre(node) {
-						// Swap to `code` tag if inline
-						if (inline) {
-							node.tagName = 'code';
-						}
-
-						// Cast to string as shikiji will always pass them as strings instead of any other types
-						const classValue = (node.properties.class as string) ?? '';
-						const styleValue = (node.properties.style as string) ?? '';
-
-						// Replace "shiki" class naming with "astro-code"
-						node.properties.class = classValue.replace(/shiki/g, 'astro-code');
-
-						// Handle code wrapping
-						// if wrap=null, do nothing.
-						if (wrap === false) {
-							node.properties.style = styleValue + '; overflow-x: auto;';
-						} else if (wrap === true) {
-							node.properties.style =
-								styleValue + '; overflow-x: auto; white-space: pre-wrap; word-wrap: break-word;';
-						}
-					},
-					line(node) {
-						// Add "user-select: none;" for "+"/"-" diff symbols.
-						// Transform `<span class="line"><span style="...">+ something</span></span>
-						// into      `<span class="line"><span style="..."><span style="user-select: none;">+</span> something</span></span>`
-						if (lang === 'diff') {
-							const innerSpanNode = node.children[0];
-							const innerSpanTextNode =
-								innerSpanNode?.type === 'element' && innerSpanNode.children?.[0];
-
-							if (innerSpanTextNode && innerSpanTextNode.type === 'text') {
-								const start = innerSpanTextNode.value[0];
-								if (start === '+' || start === '-') {
-									innerSpanTextNode.value = innerSpanTextNode.value.slice(1);
-									innerSpanNode.children.unshift({
-										type: 'element',
-										tagName: 'span',
-										properties: { style: 'user-select: none;' },
-										children: [{ type: 'text', value: start }],
-									});
-								}
+				transformers: [
+					{
+						pre(node) {
+							// Swap to `code` tag if inline
+							if (inline) {
+								node.tagName = 'code';
 							}
-						}
-					},
-					code(node) {
-						if (inline) {
-							return node.children[0] as typeof node;
-						}
-					},
-					root(node) {
-						if (Object.values(experimentalThemes).length) {
-							return;
-						}
-
-						// theme.id for shiki -> shikiji compat
-						const themeName = typeof theme === 'string' ? theme : theme.name;
-						if (themeName === 'css-variables') {
-							// Replace special color tokens to CSS variables
-							visit(node as any, 'element', (child) => {
-								if (child.properties?.style) {
-									child.properties.style = replaceCssVariables(child.properties.style);
+
+							// Cast to string as shikiji will always pass them as strings instead of any other types
+							const classValue = (node.properties.class as string) ?? '';
+							const styleValue = (node.properties.style as string) ?? '';
+
+							// Replace "shiki" class naming with "astro-code"
+							node.properties.class = classValue.replace(/shiki/g, 'astro-code');
+
+							// Handle code wrapping
+							// if wrap=null, do nothing.
+							if (wrap === false) {
+								node.properties.style = styleValue + '; overflow-x: auto;';
+							} else if (wrap === true) {
+								node.properties.style =
+									styleValue + '; overflow-x: auto; white-space: pre-wrap; word-wrap: break-word;';
+							}
+						},
+						line(node) {
+							// Add "user-select: none;" for "+"/"-" diff symbols.
+							// Transform `<span class="line"><span style="...">+ something</span></span>
+							// into      `<span class="line"><span style="..."><span style="user-select: none;">+</span> something</span></span>`
+							if (lang === 'diff') {
+								const innerSpanNode = node.children[0];
+								const innerSpanTextNode =
+									innerSpanNode?.type === 'element' && innerSpanNode.children?.[0];
+
+								if (innerSpanTextNode && innerSpanTextNode.type === 'text') {
+									const start = innerSpanTextNode.value[0];
+									if (start === '+' || start === '-') {
+										innerSpanTextNode.value = innerSpanTextNode.value.slice(1);
+										innerSpanNode.children.unshift({
+											type: 'element',
+											tagName: 'span',
+											properties: { style: 'user-select: none;' },
+											children: [{ type: 'text', value: start }],
+										});
+									}
 								}
-							});
-						}
+							}
+						},
+						code(node) {
+							if (inline) {
+								return node.children[0] as typeof node;
+							}
+						},
+						root(node) {
+							if (Object.values(experimentalThemes).length) {
+								return;
+							}
+
+							// theme.id for shiki -> shikiji compat
+							const themeName = typeof theme === 'string' ? theme : theme.name;
+							if (themeName === 'css-variables') {
+								// Replace special color tokens to CSS variables
+								visit(node as any, 'element', (child) => {
+									if (child.properties?.style) {
+										child.properties.style = replaceCssVariables(child.properties.style);
+									}
+								});
+							}
+						},
 					},
-				},
+					...transformers,
+				],
 			});
 		},
 	};

packages/markdown/remark/src/types.ts

@@ -4,6 +4,7 @@ import type { Options as RemarkRehypeOptions } from 'remark-rehype';
 import type {
 	BuiltinTheme,
 	LanguageRegistration,
+	ShikijiTransformer,
 	ThemeRegistration,
 	ThemeRegistrationRaw,
 } from 'shikiji';
@@ -32,11 +33,14 @@ export type RehypePlugins = (string | [string, any] | RehypePlugin | [RehypePlug
 
 export type RemarkRehype = RemarkRehypeOptions;
 
+export type ThemePresets = BuiltinTheme | 'css-variables';
+
 export interface ShikiConfig {
 	langs?: LanguageRegistration[];
-	theme?: BuiltinTheme | ThemeRegistration | ThemeRegistrationRaw;
-	experimentalThemes?: Record<string, BuiltinTheme | ThemeRegistration | ThemeRegistrationRaw>;
+	theme?: ThemePresets | ThemeRegistration | ThemeRegistrationRaw;
+	experimentalThemes?: Record<string, ThemePresets | ThemeRegistration | ThemeRegistrationRaw>;
 	wrap?: boolean | null;
+	transformers?: ShikijiTransformer[];
 }
 
 export interface AstroMarkdownOptions {