From b3cb05cd09fb191241d6dade94d70bf1c8c92807 Mon Sep 17 00:00:00 2001 From: Anthony Fu Date: Mon, 14 Aug 2023 19:24:36 +0200 Subject: [PATCH] feat: add `codeToTokensWithThemes`, fix #7 Co-authored-by: starknt --- packages/shikiji/src/core/bundle-factory.ts | 25 ++- packages/shikiji/src/core/core.ts | 50 +++--- .../shikiji/src/core/renderer-html-themes.ts | 30 ++-- packages/shikiji/src/core/themedTokenizer.ts | 19 +-- packages/shikiji/src/index.ts | 1 + packages/shikiji/src/types.ts | 38 +++-- packages/shikiji/test/singleton.test.ts | 8 - packages/shikiji/test/themes.test.ts | 148 +++++++++++++++++- 8 files changed, 244 insertions(+), 75 deletions(-) diff --git a/packages/shikiji/src/core/bundle-factory.ts b/packages/shikiji/src/core/bundle-factory.ts index 401d8846f..b9a814ca2 100644 --- a/packages/shikiji/src/core/bundle-factory.ts +++ b/packages/shikiji/src/core/bundle-factory.ts @@ -1,4 +1,4 @@ -import type { BundledHighlighterOptions, CodeToHtmlThemesOptions, CodeToHtmlOptions, CodeToThemedTokensOptions, HighlighterCoreOptions, HighlighterGeneric, LanguageInput, MaybeArray, PlainTextLanguage, RequireKeys, ThemeInput } from '../types' +import type { BundledHighlighterOptions, CodeToHtmlOptions, CodeToHtmlThemesOptions, CodeToThemedTokensOptions, CodeToTokensWithThemesOptions, HighlighterCoreOptions, HighlighterGeneric, LanguageInput, MaybeArray, PlainTextLanguage, RequireKeys, ThemeInput } from '../types' import { isPlaintext, toArray } from './utils' import { getHighlighterCore } from './core' @@ -71,7 +71,7 @@ export function createdBundledHighlighter(getHighlighter: GetHighlighterFactory) { let _shiki: ReturnType - async function getShikiWithThemeLang(options: { theme: MaybeArray; lang: MaybeArray }) { + async function _getHighlighter(options: { theme: MaybeArray; lang: MaybeArray }) { if (!_shiki) { _shiki = getHighlighter({ themes: toArray(options.theme), @@ -96,7 +96,7 @@ export function createSingletonShorthands(g * Differences from `shiki.codeToHtml()`, this function is async. */ async function codeToHtml(code: string, options: RequireKeys, 'theme' | 'lang'>) { - const shiki = await getShikiWithThemeLang(options) + const shiki = await _getHighlighter(options) return shiki.codeToHtml(code, options) } @@ -107,7 +107,7 @@ export function createSingletonShorthands(g * Differences from `shiki.codeToThemedTokens()`, this function is async. */ async function codeToThemedTokens(code: string, options: RequireKeys, 'theme' | 'lang'>) { - const shiki = await getShikiWithThemeLang(options) + const shiki = await _getHighlighter(options) return shiki.codeToThemedTokens(code, options) } @@ -118,16 +118,31 @@ export function createSingletonShorthands(g * Differences from `shiki.codeToHtmlThemes()`, this function is async. */ async function codeToHtmlThemes(code: string, options: RequireKeys, 'themes' | 'lang'>) { - const shiki = await getShikiWithThemeLang({ + const shiki = await _getHighlighter({ lang: options.lang, theme: Object.values(options.themes).filter(Boolean) as T[], }) return shiki.codeToHtmlThemes(code, options) } + /** + * Shorthand for `codeToTokensWithThemes` with auto-loaded theme and language. + * A singleton highlighter it maintained internally. + * + * Differences from `shiki.codeToTokensWithThemes()`, this function is async. + */ + async function codeToTokensWithThemes(code: string, options: RequireKeys, 'themes' | 'lang'>) { + const shiki = await _getHighlighter({ + lang: options.lang, + theme: Object.values(options.themes).filter(Boolean) as T[], + }) + return shiki.codeToTokensWithThemes(code, options) + } + return { codeToHtml, codeToHtmlThemes, codeToThemedTokens, + codeToTokensWithThemes, } } diff --git a/packages/shikiji/src/core/core.ts b/packages/shikiji/src/core/core.ts index 1eb238823..9d21fa1f5 100644 --- a/packages/shikiji/src/core/core.ts +++ b/packages/shikiji/src/core/core.ts @@ -1,11 +1,11 @@ -import type { CodeToHtmlOptions, CodeToHtmlThemesOptions, CodeToThemedTokensOptions, HighlighterCoreOptions, HighlighterGeneric, LanguageInput, MaybeGetter, ThemeInput, ThemeRegisteration, ThemedToken } from '../types' +import type { CodeToHtmlOptions, CodeToHtmlThemesOptions, CodeToThemedTokensOptions, CodeToTokensWithThemesOptions, HighlighterCoreOptions, HighlighterGeneric, LanguageInput, MaybeGetter, ThemeInput, ThemedToken } from '../types' import { createOnigScanner, createOnigString, loadWasm } from '../oniguruma' import { Registry } from './registry' import { Resolver } from './resolver' import { tokenizeWithTheme } from './themedTokenizer' import { renderToHtml } from './renderer-html' import { isPlaintext } from './utils' -import { renderToHtmlThemes } from './renderer-html-themes' +import { renderToHtmlThemes, syncThemesTokenization } from './renderer-html-themes' export type HighlighterCore = HighlighterGeneric @@ -85,6 +85,9 @@ export async function getHighlighterCore(options: HighlighterCoreOptions = {}): } } + /** + * Get highlighted code in HTML. + */ function codeToHtml(code: string, options: CodeToHtmlOptions = {}): string { const tokens = codeToThemedTokens(code, { ...options, @@ -99,27 +102,38 @@ export async function getHighlighterCore(options: HighlighterCoreOptions = {}): }) } + /** + * Get tokens with multiple themes, with synced + */ + function codeToTokensWithThemes(code: string, options: CodeToTokensWithThemesOptions) { + const themes = Object.entries(options.themes) + .filter(i => i[1]) as [string, string][] + + const tokens = syncThemesTokenization(...themes.map(t => codeToThemedTokens(code, { + ...options, + theme: t[1], + includeExplanation: false, + }))) + + return themes.map(([color, theme], idx) => [ + color, + theme, + tokens[idx], + ] as [string, string, ThemedToken[][]]) + } + function codeToHtmlThemes(code: string, options: CodeToHtmlThemesOptions): string { const { - defaultColor = 'light', cssVariablePrefix = '--shiki-', + defaultColor = 'light', + cssVariablePrefix = '--shiki-', } = options - const themes = Object.entries(options.themes) - .filter(i => i[1]) + const tokens = codeToTokensWithThemes(code, options) .sort(a => a[0] === defaultColor ? -1 : 1) - const tokens = themes.map(([color, theme]) => [ - color, - codeToThemedTokens(code, { - ...options, - theme, - includeExplanation: false, - }), - getTheme(theme), - ] as [string, ThemedToken[][], ThemeRegisteration]) - return renderToHtmlThemes( tokens, + tokens.map(i => getTheme(i[1])), cssVariablePrefix, defaultColor !== false, options, @@ -132,15 +146,15 @@ export async function getHighlighterCore(options: HighlighterCoreOptions = {}): async function loadTheme(...themes: ThemeInput[]) { await Promise.all( - themes.map(async theme => _registry.loadTheme(await normalizeGetter(theme)), - ), + themes.map(async theme => _registry.loadTheme(await normalizeGetter(theme))), ) } return { - codeToThemedTokens, codeToHtml, codeToHtmlThemes, + codeToThemedTokens, + codeToTokensWithThemes, loadLanguage, loadTheme, getLoadedThemes: () => _registry.getLoadedThemes(), diff --git a/packages/shikiji/src/core/renderer-html-themes.ts b/packages/shikiji/src/core/renderer-html-themes.ts index bedd6a91c..a3d1b772d 100644 --- a/packages/shikiji/src/core/renderer-html-themes.ts +++ b/packages/shikiji/src/core/renderer-html-themes.ts @@ -2,19 +2,19 @@ import type { HtmlRendererOptions, ThemeRegisteration, ThemedToken } from '../ty import { renderToHtml } from './renderer-html' /** - * Break tokens from multiple themes into same length. + * Break tokens from multiple themes into same tokenization. * * For example, given two themes that tokenize `console.log("hello")` as: * * - `console . log (" hello ")` (6 tokens) * - `console .log ( "hello" )` (5 tokens) * - * This function break both themes into same tokenization, so later the can be rendered in pairs. + * This function will return: * * - `console . log ( " hello " )` (8 tokens) * - `console . log ( " hello " )` (8 tokens) */ -export function _syncThemedTokens(...themes: ThemedToken[][][]) { +export function syncThemesTokenization(...themes: ThemedToken[][][]) { const outThemes = themes.map(() => []) const count = themes.length @@ -55,35 +55,35 @@ export function _syncThemedTokens(...themes: ThemedToken[][][]) { } export function renderToHtmlThemes( - themes: [string, ThemedToken[][], ThemeRegisteration][], + themeTokens: [string, string, ThemedToken[][]][], + themeRegs: ThemeRegisteration[], cssVariablePrefix = '--shiki-', defaultColor = true, options: HtmlRendererOptions = {}, ) { - const synced = _syncThemedTokens(...themes.map(t => t[1])) - + const themeMap = themeTokens.map(t => t[2]) const merged: ThemedToken[][] = [] - for (let i = 0; i < synced[0].length; i++) { - const lines = synced.map(t => t[i]) + for (let i = 0; i < themeMap[0].length; i++) { + const lineMap = themeMap.map(t => t[i]) const lineout: any[] = [] merged.push(lineout) - for (let j = 0; j < lines[0].length; j++) { - const tokens = lines.map(t => t[j]) - const colors = tokens.map((t, idx) => `${idx === 0 && defaultColor ? '' : `${cssVariablePrefix + themes[idx][0]}:`}${t.color || 'inherit'}`).join(';') + for (let j = 0; j < lineMap[0].length; j++) { + const tokenMap = lineMap.map(t => t[j]) + const colors = tokenMap.map((t, idx) => `${idx === 0 && defaultColor ? '' : `${cssVariablePrefix + themeTokens[idx][0]}:`}${t.color || 'inherit'}`).join(';') lineout.push({ - ...tokens[0], + ...tokenMap[0], color: colors, htmlStyle: defaultColor ? undefined : colors, }) } } - const fg = options.fg || themes.map((t, idx) => (idx === 0 && defaultColor ? '' : `${cssVariablePrefix + t[0]}:`) + t[2].fg).join(';') - const bg = options.bg || themes.map((t, idx) => (idx === 0 && defaultColor ? '' : `${cssVariablePrefix + t[0]}-bg:`) + t[2].bg).join(';') + const fg = options.fg || themeTokens.map((t, idx) => (idx === 0 && defaultColor ? '' : `${cssVariablePrefix + t[0]}:`) + themeRegs[idx].fg).join(';') + const bg = options.bg || themeTokens.map((t, idx) => (idx === 0 && defaultColor ? '' : `${cssVariablePrefix + t[0]}-bg:`) + themeRegs[idx].bg).join(';') return renderToHtml(merged, { fg, bg, - themeName: `shiki-themes ${themes.map(t => t[2].name).join(' ')}`, + themeName: `shiki-themes ${themeRegs.map(t => t.name).join(' ')}`, rootStyle: defaultColor ? undefined : [fg, bg].join(';'), ...options, }) diff --git a/packages/shikiji/src/core/themedTokenizer.ts b/packages/shikiji/src/core/themedTokenizer.ts index 82a7aa341..a9f199681 100644 --- a/packages/shikiji/src/core/themedTokenizer.ts +++ b/packages/shikiji/src/core/themedTokenizer.ts @@ -5,7 +5,7 @@ import type { IGrammar, IRawTheme } from 'vscode-textmate' import { INITIAL } from 'vscode-textmate' -import type { ThemedToken, ThemedTokenExplanation, ThemedTokenScopeExplanation } from '../types' +import type { ThemedToken, ThemedTokenScopeExplanation } from '../types' import type { FontStyle } from './stackElementMetadata' import { StackElementMetadata } from './stackElementMetadata' @@ -58,8 +58,14 @@ export function tokenizeWithTheme( const foregroundColor = colorMap[foreground] const fontStyle: FontStyle = StackElementMetadata.getFontStyle(metadata) - const explanation: ThemedTokenExplanation[] = [] + const token: ThemedToken = { + content: line.substring(startIndex, nextStartIndex), + color: foregroundColor, + fontStyle, + } + if (options.includeExplanation) { + token.explanation = [] let offset = 0 while (startIndex + offset < nextStartIndex) { const tokenWithScopes = tokensWithScopes![tokensWithScopesIndex!] @@ -69,7 +75,7 @@ export function tokenizeWithTheme( tokenWithScopes.endIndex, ) offset += tokenWithScopesText.length - explanation.push({ + token.explanation.push({ content: tokenWithScopesText, scopes: explainThemeScopes(theme, tokenWithScopes.scopes), }) @@ -78,12 +84,7 @@ export function tokenizeWithTheme( } } - actual.push({ - content: line.substring(startIndex, nextStartIndex), - color: foregroundColor, - fontStyle, - explanation, - }) + actual.push(token) } final.push(actual) actual = [] diff --git a/packages/shikiji/src/index.ts b/packages/shikiji/src/index.ts index a2100c090..ecf29d822 100644 --- a/packages/shikiji/src/index.ts +++ b/packages/shikiji/src/index.ts @@ -24,6 +24,7 @@ export const { codeToHtml, codeToHtmlThemes, codeToThemedTokens, + codeToTokensWithThemes, } = /* @__PURE__ */ createSingletonShorthands< BuiltinLanguages, BuiltinThemes diff --git a/packages/shikiji/src/types.ts b/packages/shikiji/src/types.ts index e8bdcc028..c2eb76caa 100644 --- a/packages/shikiji/src/types.ts +++ b/packages/shikiji/src/types.ts @@ -42,6 +42,10 @@ export interface HighlighterGeneric, ResolveBundleKey> ): ThemedToken[][] + codeToTokensWithThemes( + code: string, + options: CodeToTokensWithThemesOptions, ResolveBundleKey> + ): [color: string, theme: string, tokens: ThemedToken[][]][] loadTheme(...themes: (ThemeInput | BundledThemeKeys)[]): Promise loadLanguage(...langs: (LanguageInput | BundledLangKeys | PlainTextLanguage)[]): Promise getLoadedLanguages(): string[] @@ -84,13 +88,26 @@ export interface LanguageRegistration extends IRawGrammar { unbalancedBracketSelectors?: string[] } -export interface CodeToHtmlOptions { +export interface CodeToThemedTokensOptions { lang?: Languages | PlainTextLanguage theme?: Themes + /** + * Include explanation of why a token is given a color. + * + * @default true + */ + includeExplanation?: boolean +} + +export interface CodeToHtmlBasicOptions { lineOptions?: LineOption[] } -export interface CodeToHtmlThemesOptions { +export interface CodeToHtmlOptions + extends Omit, 'includeExplanation'>, CodeToHtmlBasicOptions { +} + +export interface CodeToTokensWithThemesOptions { lang?: Languages | PlainTextLanguage /** @@ -112,6 +129,10 @@ export interface CodeToHtmlThemesOptions { light: Themes dark: Themes } & Partial> +} + +export interface CodeToHtmlThemesOptions + extends CodeToTokensWithThemesOptions, CodeToHtmlBasicOptions { /** * The default theme applied to the code (via inline `color` style). @@ -141,19 +162,6 @@ export interface CodeToHtmlThemesOptions { * @default '--shiki-' */ cssVariablePrefix?: string - - lineOptions?: LineOption[] -} - -export interface CodeToThemedTokensOptions { - lang?: Languages | PlainTextLanguage - theme?: Themes - /** - * Include explanation of why a token is given a color. - * - * @default true - */ - includeExplanation?: boolean } export interface LineOption { diff --git a/packages/shikiji/test/singleton.test.ts b/packages/shikiji/test/singleton.test.ts index 5743e6c62..3d67ba278 100644 --- a/packages/shikiji/test/singleton.test.ts +++ b/packages/shikiji/test/singleton.test.ts @@ -18,49 +18,41 @@ describe('should', () => { { "color": "#BD976A", "content": "console", - "explanation": [], "fontStyle": 0, }, { "color": "#666666", "content": ".", - "explanation": [], "fontStyle": 0, }, { "color": "#80A665", "content": "log", - "explanation": [], "fontStyle": 0, }, { "color": "#666666", "content": "(", - "explanation": [], "fontStyle": 0, }, { "color": "#C98A7D99", "content": "\\"", - "explanation": [], "fontStyle": 0, }, { "color": "#C98A7D", "content": "hello", - "explanation": [], "fontStyle": 0, }, { "color": "#C98A7D99", "content": "\\"", - "explanation": [], "fontStyle": 0, }, { "color": "#666666", "content": ")", - "explanation": [], "fontStyle": 0, }, ], diff --git a/packages/shikiji/test/themes.test.ts b/packages/shikiji/test/themes.test.ts index 5ceadf665..0d2f0ac6d 100644 --- a/packages/shikiji/test/themes.test.ts +++ b/packages/shikiji/test/themes.test.ts @@ -1,9 +1,9 @@ import { describe, expect, it } from 'vitest' import type { ThemedToken } from '../src' -import { codeToHtmlThemes, codeToThemedTokens } from '../src' -import { _syncThemedTokens } from '../src/core/renderer-html-themes' +import { codeToHtmlThemes, codeToThemedTokens, codeToTokensWithThemes } from '../src' +import { syncThemesTokenization } from '../src/core/renderer-html-themes' -describe('syncThemedTokens', () => { +describe('syncThemesTokenization', () => { function stringifyTokens(tokens: ThemedToken[][]) { return tokens.map(line => line.map(token => token.content).join(' ')).join('\n') } @@ -17,7 +17,7 @@ describe('syncThemedTokens', () => { expect(stringifyTokens(lines2)) .toMatchInlineSnapshot('"console .log ( \\"hello\\" )"') - const [out1, out2] = _syncThemedTokens(lines1, lines2) + const [out1, out2] = syncThemesTokenization(lines1, lines2) expect(stringifyTokens(out1)) .toBe(stringifyTokens(out2)) @@ -35,7 +35,7 @@ describe('syncThemedTokens', () => { expect(stringifyTokens(lines3)) .toMatchInlineSnapshot('"console . log ( \\" hello \\" ) ;"') - const [out1, out2, out3] = _syncThemedTokens(lines1, lines2, lines3) + const [out1, out2, out3] = syncThemesTokenization(lines1, lines2, lines3) expect(stringifyTokens(out1)) .toBe(stringifyTokens(out2)) @@ -168,3 +168,141 @@ function toggleTheme() { .toMatchFileSnapshot('./out/multiple-themes-no-default.html') }) }) + +describe('codeToTokensWithThemes', () => { + it('generates', async () => { + const themes = { + 'light': 'vitesse-light', + 'dark': 'vitesse-dark', + 'nord': 'nord', + 'min-dark': 'min-dark', + 'min-light': 'min-light', + } as const + + const code = await codeToTokensWithThemes('a.b', { + lang: 'js', + themes, + }) + + expect(code) + .toMatchInlineSnapshot(` + [ + [ + "light", + "vitesse-light", + [ + [ + { + "color": "#B07D48", + "content": "a", + "fontStyle": 0, + }, + { + "color": "#999999", + "content": ".", + "fontStyle": 0, + }, + { + "color": "#B07D48", + "content": "b", + "fontStyle": 0, + }, + ], + ], + ], + [ + "dark", + "vitesse-dark", + [ + [ + { + "color": "#BD976A", + "content": "a", + "fontStyle": 0, + }, + { + "color": "#666666", + "content": ".", + "fontStyle": 0, + }, + { + "color": "#BD976A", + "content": "b", + "fontStyle": 0, + }, + ], + ], + ], + [ + "nord", + "nord", + [ + [ + { + "color": "#D8DEE9", + "content": "a", + "fontStyle": 0, + }, + { + "color": "#ECEFF4", + "content": ".", + "fontStyle": 0, + }, + { + "color": "#D8DEE9", + "content": "b", + "fontStyle": 0, + }, + ], + ], + ], + [ + "min-dark", + "min-dark", + [ + [ + { + "color": "#79B8FF", + "content": "a", + "fontStyle": 0, + }, + { + "color": "#B392F0", + "content": ".", + "fontStyle": 0, + }, + { + "color": "#B392F0", + "content": "b", + "fontStyle": 0, + }, + ], + ], + ], + [ + "min-light", + "min-light", + [ + [ + { + "color": "#1976D2", + "content": "a", + "fontStyle": 0, + }, + { + "color": "#24292EFF", + "content": ".", + "fontStyle": 0, + }, + { + "color": "#24292EFF", + "content": "b", + "fontStyle": 0, + }, + ], + ], + ], + ] + `) + }) +})