From e9a72d9a91a3741566866bcaab11172cb0dc7d31 Mon Sep 17 00:00:00 2001 From: Degreat Date: Wed, 17 Jan 2024 13:13:05 +0000 Subject: [PATCH] Bump shikiji, use transformers API, expose transformers API (#9643) * Bump shikiji, use transformers API, expose transformers API * update astro config schema * include shikiji-core * Use default import * address css-variables theme * Remove shikiji markdoc * Improve schema transformers handling * Fix tests * Update changeset * bump shikiji version * Update .changeset/six-scissors-worry.md Co-authored-by: Sarah Rainsberger * Update wording Co-authored-by: Sarah Rainsberger --------- Co-authored-by: bluwy Co-authored-by: Emanuele Stoppa Co-authored-by: Sarah Rainsberger --- .changeset/polite-dogs-join.md | 5 + .changeset/six-scissors-worry.md | 8 ++ packages/astro/package.json | 3 +- packages/astro/src/core/config/schema.ts | 6 + packages/astro/src/core/errors/dev/vite.ts | 19 ++- packages/astro/src/core/errors/overlay.ts | 4 +- packages/integrations/markdoc/package.json | 1 - packages/markdown/remark/package.json | 2 +- packages/markdown/remark/src/index.ts | 1 + packages/markdown/remark/src/shiki.ts | 157 +++++++++++---------- packages/markdown/remark/src/types.ts | 8 +- pnpm-lock.yaml | 23 +-- 12 files changed, 132 insertions(+), 105 deletions(-) create mode 100644 .changeset/polite-dogs-join.md create mode 100644 .changeset/six-scissors-worry.md diff --git a/.changeset/polite-dogs-join.md b/.changeset/polite-dogs-join.md new file mode 100644 index 000000000000..3bb0128d620f --- /dev/null +++ b/.changeset/polite-dogs-join.md @@ -0,0 +1,5 @@ +--- +"@astrojs/markdoc": patch +--- + +Removes unnecessary `shikiji` dependency diff --git a/.changeset/six-scissors-worry.md b/.changeset/six-scissors-worry.md new file mode 100644 index 000000000000..ebba0da66d63 --- /dev/null +++ b/.changeset/six-scissors-worry.md @@ -0,0 +1,8 @@ +--- +"@astrojs/markdown-remark": minor +"astro": minor +--- + +Adds a new `markdown.shikiConfig.transformers` config option. You can use this option to transform the Shikiji hast (AST format of the generated HTML) to customize the final HTML. Also updates Shikiji to the latest stable version. + +See [Shikiji's documentation](https://shikiji.netlify.app/guide/transformers) for more details about creating your own custom transformers, and [a list of common transformers](https://shikiji.netlify.app/packages/transformers) you can add directly to your project. diff --git a/packages/astro/package.json b/packages/astro/package.json index 7ca4923678c2..e1743f8f6421 100644 --- a/packages/astro/package.json +++ b/packages/astro/package.json @@ -166,7 +166,7 @@ "resolve": "^1.22.4", "semver": "^7.5.4", "server-destroy": "^1.0.1", - "shikiji": "^0.6.13", + "shikiji": "^0.9.18", "string-width": "^7.0.0", "strip-ansi": "^7.1.0", "tsconfck": "^3.0.0", @@ -224,6 +224,7 @@ "remark-code-titles": "^0.1.2", "rollup": "^4.5.0", "sass": "^1.69.5", + "shikiji-core": "^0.9.18", "srcset-parse": "^1.1.0", "unified": "^11.0.4" }, diff --git a/packages/astro/src/core/config/schema.ts b/packages/astro/src/core/config/schema.ts index 7be8043c08ab..1f021f568033 100644 --- a/packages/astro/src/core/config/schema.ts +++ b/packages/astro/src/core/config/schema.ts @@ -17,9 +17,11 @@ import { appendForwardSlash, prependForwardSlash, removeTrailingForwardSlash } f // This import is required to appease TypeScript! // See https://github.com/withastro/astro/pull/8762 import 'mdast-util-to-hast'; +import 'shikiji-core'; type ShikiLangs = NonNullable; type ShikiTheme = NonNullable; +type ShikiTransformers = NonNullable; const ASTRO_CONFIG_DEFAULTS = { root: '.', @@ -275,6 +277,10 @@ export const AstroConfigSchema = z.object({ ) .default(ASTRO_CONFIG_DEFAULTS.markdown.shikiConfig.experimentalThemes!), wrap: z.boolean().or(z.null()).default(ASTRO_CONFIG_DEFAULTS.markdown.shikiConfig.wrap!), + transformers: z + .custom() + .array() + .default(ASTRO_CONFIG_DEFAULTS.markdown.shikiConfig.transformers!), }) .default({}), remarkPlugins: z diff --git a/packages/astro/src/core/errors/dev/vite.ts b/packages/astro/src/core/errors/dev/vite.ts index 1bd448145123..bfdd043a7567 100644 --- a/packages/astro/src/core/errors/dev/vite.ts +++ b/packages/astro/src/core/errors/dev/vite.ts @@ -1,7 +1,6 @@ -import { replaceCssVariables } from '@astrojs/markdown-remark'; import * as fs from 'node:fs'; import { fileURLToPath } from 'node:url'; -import { codeToHtml } from 'shikiji'; +import { codeToHtml, createCssVariablesTheme } from 'shikiji'; import type { ErrorPayload } from 'vite'; import type { ModuleLoader } from '../../module-loader/index.js'; import { FailedToLoadModuleSSR, InvalidGlob, MdxIntegrationMissingError } from '../errors-data.js'; @@ -124,7 +123,11 @@ export interface AstroErrorPayload { // Map these to `.js` during error highlighting. const ALTERNATIVE_JS_EXTS = ['cjs', 'mjs']; const ALTERNATIVE_MD_EXTS = ['mdoc']; -const INLINE_STYLE_SELECTOR_GLOBAL = /style="(.*?)"/g; + +let _cssVariablesTheme: ReturnType; +const cssVariablesTheme = () => + _cssVariablesTheme ?? + (_cssVariablesTheme = createCssVariablesTheme({ variablePrefix: '--astro-code-' })); /** * Generate a payload for Vite's error overlay @@ -147,21 +150,15 @@ export async function getViteErrorPayload(err: ErrorWithMetadata): Promise - replaceCssVariables(m) - ); - } - return { type: 'error', err: { diff --git a/packages/astro/src/core/errors/overlay.ts b/packages/astro/src/core/errors/overlay.ts index 1062680bd9da..282c79cd6a21 100644 --- a/packages/astro/src/core/errors/overlay.ts +++ b/packages/astro/src/core/errors/overlay.ts @@ -68,7 +68,7 @@ const style = /* css */ ` --toggle-border-color: #C3CADB; /* Syntax Highlighting */ - --astro-code-color-text: #000000; + --astro-code-foreground: #000000; --astro-code-token-constant: #4ca48f; --astro-code-token-string: #9f722a; --astro-code-token-comment: #8490b5; @@ -121,7 +121,7 @@ const style = /* css */ ` --toggle-border-color: #3D4663; /* Syntax Highlighting */ - --astro-code-color-text: #ffffff; + --astro-code-foreground: #ffffff; --astro-code-token-constant: #90f4e3; --astro-code-token-string: #f4cf90; --astro-code-token-comment: #8490b5; diff --git a/packages/integrations/markdoc/package.json b/packages/integrations/markdoc/package.json index d3edca0afc0b..0c6a997f03d0 100644 --- a/packages/integrations/markdoc/package.json +++ b/packages/integrations/markdoc/package.json @@ -71,7 +71,6 @@ "gray-matter": "^4.0.3", "htmlparser2": "^9.0.0", "kleur": "^4.1.5", - "shikiji": "^0.6.13", "zod": "^3.22.4" }, "peerDependencies": { diff --git a/packages/markdown/remark/package.json b/packages/markdown/remark/package.json index c65a582e77a7..8594fcb2fd17 100644 --- a/packages/markdown/remark/package.json +++ b/packages/markdown/remark/package.json @@ -38,7 +38,7 @@ "remark-parse": "^11.0.0", "remark-rehype": "^11.0.0", "remark-smartypants": "^2.0.0", - "shikiji": "^0.6.13", + "shikiji": "^0.9.18", "unified": "^11.0.4", "unist-util-visit": "^5.0.0", "vfile": "^6.0.1" diff --git a/packages/markdown/remark/src/index.ts b/packages/markdown/remark/src/index.ts index 14301584793c..7881614e5dbd 100644 --- a/packages/markdown/remark/src/index.ts +++ b/packages/markdown/remark/src/index.ts @@ -36,6 +36,7 @@ export const markdownConfigDefaults: Required = { theme: 'github-dark', experimentalThemes: {}, wrap: false, + transformers: [], }, remarkPlugins: [], rehypePlugins: [], diff --git a/packages/markdown/remark/src/shiki.ts b/packages/markdown/remark/src/shiki.ts index 477ab2184b83..04b1bbcf6a72 100644 --- a/packages/markdown/remark/src/shiki.ts +++ b/packages/markdown/remark/src/shiki.ts @@ -1,4 +1,4 @@ -import { bundledLanguages, getHighlighter } from 'shikiji'; +import { bundledLanguages, createCssVariablesTheme, getHighlighter } from 'shikiji'; import { visit } from 'unist-util-visit'; import type { ShikiConfig } from './types.js'; @@ -6,32 +6,32 @@ export interface ShikiHighlighter { highlight(code: string, lang?: string, options?: { inline?: boolean }): string; } +// TODO: Remove this special replacement in Astro 5 const ASTRO_COLOR_REPLACEMENTS: Record = { - '#000001': 'var(--astro-code-color-text)', - '#000002': 'var(--astro-code-color-background)', - '#000004': 'var(--astro-code-token-constant)', - '#000005': 'var(--astro-code-token-string)', - '#000006': 'var(--astro-code-token-comment)', - '#000007': 'var(--astro-code-token-keyword)', - '#000008': 'var(--astro-code-token-parameter)', - '#000009': 'var(--astro-code-token-function)', - '#000010': 'var(--astro-code-token-string-expression)', - '#000011': 'var(--astro-code-token-punctuation)', - '#000012': 'var(--astro-code-token-link)', + '--astro-code-foreground': '--astro-code-color-text', + '--astro-code-background': '--astro-code-color-background', }; const COLOR_REPLACEMENT_REGEX = new RegExp( `(${Object.keys(ASTRO_COLOR_REPLACEMENTS).join('|')})`, 'g' ); +let _cssVariablesTheme: ReturnType; +const cssVariablesTheme = () => + _cssVariablesTheme ?? + (_cssVariablesTheme = createCssVariablesTheme({ variablePrefix: '--astro-code-' })); + export async function createShikiHighlighter({ langs = [], theme = 'github-dark', experimentalThemes = {}, wrap = false, + transformers = [], }: ShikiConfig = {}): Promise { const themes = experimentalThemes; + theme = theme === 'css-variables' ? cssVariablesTheme() : theme; + const highlighter = await getHighlighter({ langs: langs.length ? langs : Object.keys(bundledLanguages), themes: Object.values(themes).length ? Object.values(themes) : [theme], @@ -53,74 +53,77 @@ export async function createShikiHighlighter({ return highlighter.codeToHtml(code, { ...themeOptions, lang, - transforms: { - pre(node) { - // Swap to `code` tag if inline - if (inline) { - node.tagName = 'code'; - } - - // Cast to string as shikiji will always pass them as strings instead of any other types - const classValue = (node.properties.class as string) ?? ''; - const styleValue = (node.properties.style as string) ?? ''; - - // Replace "shiki" class naming with "astro-code" - node.properties.class = classValue.replace(/shiki/g, 'astro-code'); - - // Handle code wrapping - // if wrap=null, do nothing. - if (wrap === false) { - node.properties.style = styleValue + '; overflow-x: auto;'; - } else if (wrap === true) { - node.properties.style = - styleValue + '; overflow-x: auto; white-space: pre-wrap; word-wrap: break-word;'; - } - }, - line(node) { - // Add "user-select: none;" for "+"/"-" diff symbols. - // Transform `+ something - // into `+ something` - if (lang === 'diff') { - const innerSpanNode = node.children[0]; - const innerSpanTextNode = - innerSpanNode?.type === 'element' && innerSpanNode.children?.[0]; - - if (innerSpanTextNode && innerSpanTextNode.type === 'text') { - const start = innerSpanTextNode.value[0]; - if (start === '+' || start === '-') { - innerSpanTextNode.value = innerSpanTextNode.value.slice(1); - innerSpanNode.children.unshift({ - type: 'element', - tagName: 'span', - properties: { style: 'user-select: none;' }, - children: [{ type: 'text', value: start }], - }); - } + transformers: [ + { + pre(node) { + // Swap to `code` tag if inline + if (inline) { + node.tagName = 'code'; } - } - }, - code(node) { - if (inline) { - return node.children[0] as typeof node; - } - }, - root(node) { - if (Object.values(experimentalThemes).length) { - return; - } - - // theme.id for shiki -> shikiji compat - const themeName = typeof theme === 'string' ? theme : theme.name; - if (themeName === 'css-variables') { - // Replace special color tokens to CSS variables - visit(node as any, 'element', (child) => { - if (child.properties?.style) { - child.properties.style = replaceCssVariables(child.properties.style); + + // Cast to string as shikiji will always pass them as strings instead of any other types + const classValue = (node.properties.class as string) ?? ''; + const styleValue = (node.properties.style as string) ?? ''; + + // Replace "shiki" class naming with "astro-code" + node.properties.class = classValue.replace(/shiki/g, 'astro-code'); + + // Handle code wrapping + // if wrap=null, do nothing. + if (wrap === false) { + node.properties.style = styleValue + '; overflow-x: auto;'; + } else if (wrap === true) { + node.properties.style = + styleValue + '; overflow-x: auto; white-space: pre-wrap; word-wrap: break-word;'; + } + }, + line(node) { + // Add "user-select: none;" for "+"/"-" diff symbols. + // Transform `+ something + // into `+ something` + if (lang === 'diff') { + const innerSpanNode = node.children[0]; + const innerSpanTextNode = + innerSpanNode?.type === 'element' && innerSpanNode.children?.[0]; + + if (innerSpanTextNode && innerSpanTextNode.type === 'text') { + const start = innerSpanTextNode.value[0]; + if (start === '+' || start === '-') { + innerSpanTextNode.value = innerSpanTextNode.value.slice(1); + innerSpanNode.children.unshift({ + type: 'element', + tagName: 'span', + properties: { style: 'user-select: none;' }, + children: [{ type: 'text', value: start }], + }); + } } - }); - } + } + }, + code(node) { + if (inline) { + return node.children[0] as typeof node; + } + }, + root(node) { + if (Object.values(experimentalThemes).length) { + return; + } + + // theme.id for shiki -> shikiji compat + const themeName = typeof theme === 'string' ? theme : theme.name; + if (themeName === 'css-variables') { + // Replace special color tokens to CSS variables + visit(node as any, 'element', (child) => { + if (child.properties?.style) { + child.properties.style = replaceCssVariables(child.properties.style); + } + }); + } + }, }, - }, + ...transformers, + ], }); }, }; diff --git a/packages/markdown/remark/src/types.ts b/packages/markdown/remark/src/types.ts index ab5af8ed134b..69e6d52009f7 100644 --- a/packages/markdown/remark/src/types.ts +++ b/packages/markdown/remark/src/types.ts @@ -4,6 +4,7 @@ import type { Options as RemarkRehypeOptions } from 'remark-rehype'; import type { BuiltinTheme, LanguageRegistration, + ShikijiTransformer, ThemeRegistration, ThemeRegistrationRaw, } from 'shikiji'; @@ -32,11 +33,14 @@ export type RehypePlugins = (string | [string, any] | RehypePlugin | [RehypePlug export type RemarkRehype = RemarkRehypeOptions; +export type ThemePresets = BuiltinTheme | 'css-variables'; + export interface ShikiConfig { langs?: LanguageRegistration[]; - theme?: BuiltinTheme | ThemeRegistration | ThemeRegistrationRaw; - experimentalThemes?: Record; + theme?: ThemePresets | ThemeRegistration | ThemeRegistrationRaw; + experimentalThemes?: Record; wrap?: boolean | null; + transformers?: ShikijiTransformer[]; } export interface AstroMarkdownOptions { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a27e8015488d..8027ea434640 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -645,8 +645,8 @@ importers: specifier: ^1.0.1 version: 1.0.1 shikiji: - specifier: ^0.6.13 - version: 0.6.13 + specifier: ^0.9.18 + version: 0.9.18 string-width: specifier: ^7.0.0 version: 7.0.0 @@ -808,6 +808,9 @@ importers: sass: specifier: ^1.69.5 version: 1.69.6 + shikiji-core: + specifier: ^0.9.18 + version: 0.9.18 srcset-parse: specifier: ^1.1.0 version: 1.1.0 @@ -3844,9 +3847,6 @@ importers: kleur: specifier: ^4.1.5 version: 4.1.5 - shikiji: - specifier: ^0.6.13 - version: 0.6.13 zod: specifier: ^3.22.4 version: 3.22.4 @@ -5017,8 +5017,8 @@ importers: specifier: ^2.0.0 version: 2.0.0 shikiji: - specifier: ^0.6.13 - version: 0.6.13 + specifier: ^0.9.18 + version: 0.9.18 unified: specifier: ^11.0.4 version: 11.0.4 @@ -14288,10 +14288,13 @@ packages: vscode-textmate: 5.2.0 dev: true - /shikiji@0.6.13: - resolution: {integrity: sha512-4T7X39csvhT0p7GDnq9vysWddf2b6BeioiN3Ymhnt3xcy9tXmDcnsEFVxX18Z4YcQgEE/w48dLJ4pPPUcG9KkA==} + /shikiji-core@0.9.18: + resolution: {integrity: sha512-PKTXptbrp/WEDjNHV8OFG9KkfhmR0pSd161kzlDDlgQ0HXAnqJYNDSjqsy1CYZMx5bSvLMy42yJj9oFTqmkNTQ==} + + /shikiji@0.9.18: + resolution: {integrity: sha512-/tFMIdV7UQklzN13VjF0/XFzmii6C606Jc878hNezvB8ZR8FG8FW9j0I4J9EJre0owlnPntgLVPpHqy27Gs+DQ==} dependencies: - hast-util-to-html: 9.0.0 + shikiji-core: 0.9.18 dev: false /side-channel@1.0.4: