fix(twoslash): support targeting multiple tokens a node, more accurat…

…e result

packages/shikiji-twoslash/src/core.ts

@@ -131,28 +131,37 @@ export function createTransformerFactory(
           codeEl.children.splice(index + 1, 0, ...nodes)
         }
 
-        const locateTextToken = (
-          line: number,
-          character: number,
-        ) => {
-          const lineEl = this.lines[line]
-          if (!lineEl) {
-            if (throws)
-              throw new Error(`[shikiji-twoslash] Cannot find line ${line} in code element`)
-            return
-          }
-          const textNodes = lineEl.children.flatMap(i => i.type === 'element' ? i.children || [] : []) as (Text | Element)[]
+        // Build a map of tokens to their line and character position
+        const tokensMap: [line: number, charStart: number, charEnd: number, token: Element | Text][] = []
+        this.lines.forEach((lineEl, line) => {
           let index = 0
-          for (const token of textNodes) {
-            if ('value' in token && typeof token.value === 'string')
+          for (const token of lineEl.children.flatMap(i => i.type === 'element' ? i.children || [] : []) as (Text | Element)[]) {
+            if ('value' in token && typeof token.value === 'string') {
+              tokensMap.push([line, index, index + token.value.length, token])
               index += token.value.length
-
-            if (index > character)
-              return token
+            }
           }
+        })
 
-          if (throws)
-            throw new Error(`[shikiji-twoslash] Cannot find token at L${line}:${character}`)
+        // Find tokens are in range of a node, it can may multiple tokens.
+        const locateTextTokens = (
+          line: number,
+          character: number,
+          length: number,
+        ) => {
+          const start = character
+          const end = character + length
+          // When the length is 0 (completion), we find the token that contains it
+          if (length === 0) {
+            return tokensMap
+              .filter(([l, s, e]) => l === line && s < start && start <= e)
+              .map(i => i[3])
+          }
+          // Otherwise we find the tokens that are completely inside the range
+          // Because we did the breakpoints earlier, we can safely assume that there will be no across-boundary tokens
+          return tokensMap
+            .filter(([l, s, e]) => l === line && (start <= s && s < end) && (start < e && e <= end))
+            .map(i => i[3])
         }
 
         const tokensSkipHover = new Set<Element | Text>()
@@ -165,57 +174,66 @@ export function createTransformerFactory(
             continue
           }
 
-          const token = locateTextToken(node.line, node.character)
+          const tokens = locateTextTokens(node.line, node.character, node.length)
 
           switch (node.type) {
             case 'error': {
-              if (token && renderer.nodeError) {
-                tokensSkipHover.add(token)
-                const clone = { ...token }
-                Object.assign(token, renderer.nodeError.call(this, node, clone))
+              if (renderer.nodeError) {
+                tokens.forEach((token) => {
+                  tokensSkipHover.add(token)
+                  const clone = { ...token }
+                  Object.assign(token, renderer.nodeError!.call(this, node, clone))
+                })
               }
               if (renderer.lineError)
                 insertAfterLine(node.line, renderer.lineError.call(this, node))
               break
             }
             case 'query': {
+              const token = tokens[0]
               if (token && renderer.nodeQuery) {
                 tokensSkipHover.add(token)
                 const clone = { ...token }
-                Object.assign(token, renderer.nodeQuery.call(this, node, clone))
+                Object.assign(token, renderer.nodeQuery!.call(this, node, clone))
               }
               if (renderer.lineQuery)
                 insertAfterLine(node.line, renderer.lineQuery.call(this, node, token))
               break
             }
             case 'completion': {
-              if (token && renderer.nodeCompletion) {
-                tokensSkipHover.add(token)
-                const clone = { ...token }
-                Object.assign(token, renderer.nodeCompletion.call(this, node, clone))
+              if (renderer.nodeCompletion) {
+                tokens.forEach((token) => {
+                  tokensSkipHover.add(token)
+                  const clone = { ...token }
+                  Object.assign(token, renderer.nodeCompletion!.call(this, node, clone))
+                })
               }
               if (renderer.lineCompletion)
                 insertAfterLine(node.line, renderer.lineCompletion.call(this, node))
               break
             }
             case 'highlight': {
-              if (token && renderer.nodeHightlight) {
-                tokensSkipHover.add(token)
-                const clone = { ...token }
-                Object.assign(token, renderer.nodeHightlight.call(this, node, clone))
+              if (renderer.nodeHightlight) {
+                tokens.forEach((token) => {
+                  tokensSkipHover.add(token)
+                  const clone = { ...token }
+                  Object.assign(token, renderer.nodeHightlight!.call(this, node, clone))
+                })
               }
               if (renderer.lineHighlight)
                 insertAfterLine(node.line, renderer.lineHighlight.call(this, node))
               break
             }
             case 'hover': {
               // Hover will be handled after all other nodes are processed
-              if (token && renderer.nodeStaticInfo) {
+              if (renderer.nodeStaticInfo) {
                 postActions.push(() => {
-                  if (tokensSkipHover.has(token))
-                    return
-                  const clone = { ...token }
-                  Object.assign(token, renderer.nodeStaticInfo.call(this, node, clone))
+                  tokens.forEach((token) => {
+                    if (tokensSkipHover.has(token))
+                      return
+                    const clone = { ...token }
+                    Object.assign(token, renderer.nodeStaticInfo.call(this, node, clone))
+                  })
                 })
               }
               break

packages/shikiji-twoslash/test/out/error-multi-tokens.html

@@ -0,0 +1,2 @@
+<link rel="stylesheet" href="../../style-rich.css" />
+<pre class="shiki vitesse-dark twoslash lsp" style="background-color:#121212;color:#dbd7caee" tabindex="0"><code><span class="line"><span style="color:#CB7676">const</span><span> </span><span style="color:#BD976A"><span class="twoslash-hover"><span class="twoslash-popup-info"><span class="line"><span style="color:#CB7676">const</span><span> </span><span style="color:#BD976A">x</span><span style="color:#666666">: [</span><span style="color:#5DA994">number</span><span style="color:#666666">]</span></span></span>x</span></span><span style="color:#666666">: [</span><span style="color:#5DA994">number</span><span style="color:#666666">] =</span><span style="color:#CB7676"> </span><span style="color:#666666">[</span><span style="color:#C98A7D99"><span class="twoslash-error">"</span></span><span style="color:#C98A7D"><span class="twoslash-error">hello</span></span><span style="color:#C98A7D99"><span class="twoslash-error">"</span></span><span style="color:#666666">]</span></span><div class="twoslash-meta-line twoslash-error-line">Type 'string' is not assignable to type 'number'.</div></code></pre>

packages/shikiji-twoslash/test/target-multi-tokens.test.ts

@@ -0,0 +1,44 @@
+import { codeToHtml, codeToThemedTokens } from 'shikiji'
+import { transformerTwoSlash } from 'shikiji-twoslash'
+import { expect, it } from 'vitest'
+
+const code = `const x: [number] = ["hello"]`
+
+it('verify theme behavior', async () => {
+  const tokens = await codeToThemedTokens(code, {
+    lang: 'ts',
+    theme: 'vitesse-dark',
+  })
+
+  // `vitesse-dark` separates the the quotes ints tokens, where the error is targeting all strings
+  expect.soft(tokens.find(i => i.find(j => j.content === '"')))
+    .toBeDefined()
+  expect.soft(tokens.find(i => i.find(j => j.content === '"ref"')))
+    .not.toBeDefined()
+})
+
+it('should split tokens correctly', async () => {
+  const html = await codeToHtml(code, {
+    lang: 'ts',
+    theme: 'vitesse-dark',
+    transformers: [
+      transformerTwoSlash({
+        twoslashOptions: {
+          handbookOptions: {
+            errors: [2322],
+          },
+        },
+      }),
+    ],
+  })
+
+  expect.soft(
+    [...html.matchAll(/<span class="twoslash-error">(.*?)<\/span>/g)].map(i => i[1]),
+  )
+    .toEqual(['"', 'hello', '"'])
+
+  expect(
+    `<link rel="stylesheet" href="../../style-rich.css" />\n${html}`,
+  )
+    .toMatchFileSnapshot('./out/error-multi-tokens.html')
+})