feat: upgrade oniguruma-to-js, support more languages

docs/references/engine-js-compat.md

@@ -1,8 +1,8 @@
 # JavaScript RegExp Engine Compatibility References
 
-> Genreated on Sunday, September 1, 2024
+> Genreated on Wednesday, September 4, 2024
 >
-> Version `1.15.2`
+> Version `1.16.1`
 >
 > Runtime: Node.js v20.12.2
 
@@ -11,9 +11,9 @@
 |                 |                             Count |
 | :-------------- | --------------------------------: |
 | Total Languages |                               213 |
-| Fully Supported | [178](#fully-supported-languages) |
-| Mismatched      |       [30](#mismatched-languages) |
-| Unsupported     |       [5](#unsupported-languages) |
+| Fully Supported | [187](#fully-supported-languages) |
+| Mismatched      |       [18](#mismatched-languages) |
+| Unsupported     |       [8](#unsupported-languages) |
 
 ## Fully Supported Languages
 
@@ -28,6 +28,7 @@ Languages that works with the JavaScript RegExp engine, and will produce the sam
 | apl                | ✅ OK           |               179 |               - |
 | applescript        | ✅ OK           |               151 |               - |
 | ara                | ✅ OK           |                54 |               - |
+| asciidoc           | ✅ OK           |               262 |               - |
 | asm                | ✅ OK           |               297 |               - |
 | astro              | ✅ OK           |                59 |               - |
 | awk                | ✅ OK           |                36 |               - |
@@ -38,6 +39,7 @@ Languages that works with the JavaScript RegExp engine, and will produce the sam
 | bibtex             | ✅ OK           |                19 |               - |
 | bicep              | ✅ OK           |                28 |               - |
 | c                  | ✅ OK           |               158 |               - |
+| cadence            | ✅ OK           |                71 |               - |
 | clarity            | ✅ OK           |                43 |               - |
 | clj                | ✅ OK           |                38 |               - |
 | clojure            | ✅ OK           |                38 |               - |
@@ -47,10 +49,13 @@ Languages that works with the JavaScript RegExp engine, and will produce the sam
 | codeql             | ✅ OK           |               150 |               - |
 | coffee             | ✅ OK           |               120 |               - |
 | common-lisp        | ✅ OK           |                57 |               - |
+| coq                | ✅ OK           |                25 |               - |
 | cpp                | ✅ OK           |               220 |               - |
 | crystal            | ✅ OK           |               140 |               - |
 | css                | ✅ OK           |               141 |               - |
 | csv                | ✅ OK           |                 1 |               - |
+| cue                | ✅ OK           |                85 |               - |
+| cypher             | ✅ OK           |                39 |               - |
 | d                  | ✅ OK           |               270 |               - |
 | dart               | ✅ OK           |                71 |               - |
 | dax                | ✅ OK           |                23 |               - |
@@ -133,6 +138,8 @@ Languages that works with the JavaScript RegExp engine, and will produce the sam
 | perl               | ✅ OK           |               156 |               - |
 | plsql              | ✅ OK           |                43 |               - |
 | postcss            | ✅ OK           |                47 |               - |
+| powerquery         | ✅ OK           |                30 |               - |
+| powershell         | ✅ OK           |                88 |               - |
 | prisma             | ✅ OK           |                26 |               - |
 | prolog             | ✅ OK           |                26 |               - |
 | proto              | ✅ OK           |                33 |               - |
@@ -171,6 +178,7 @@ Languages that works with the JavaScript RegExp engine, and will produce the sam
 | tcl                | ✅ OK           |                33 |               - |
 | templ              | ✅ OK           |                74 |               - |
 | terraform          | ✅ OK           |                62 |               - |
+| tex                | ✅ OK           |                38 |               - |
 | toml               | ✅ OK           |                40 |               - |
 | ts-tags            | ✅ OK           |                 - |               - |
 | tsv                | ✅ OK           |                 1 |               - |
@@ -196,6 +204,7 @@ Languages that works with the JavaScript RegExp engine, and will produce the sam
 | wolfram            | ✅ OK           |               501 |               - |
 | xml                | ✅ OK           |                30 |               - |
 | xsl                | ✅ OK           |                 5 |               - |
+| yaml               | ✅ OK           |                46 |               - |
 | zenscript          | ✅ OK           |                21 |               - |
 | zig                | ✅ OK           |                51 |               - |
 | zsh                | ✅ OK           |               146 |               - |
@@ -208,43 +217,34 @@ Languages that does not throw with the JavaScript RegExp engine, but will produc
 | ------------ | :-------------- | ----------------: | --------------: |
 | angular-html | ⚠️ Mismatch     |                 2 |               - |
 | apex         | ⚠️ Mismatch     |               189 |               - |
-| asciidoc     | ⚠️ Mismatch     |               262 |               - |
 | beancount    | ⚠️ Mismatch     |                39 |               - |
 | blade        | ⚠️ Mismatch     |               330 |               - |
-| cadence      | ⚠️ Mismatch     |                71 |               - |
-| coq          | ⚠️ Mismatch     |                25 |               - |
-| cue          | ⚠️ Mismatch     |                85 |               - |
-| cypher       | ⚠️ Mismatch     |                39 |               - |
 | haml         | ⚠️ Mismatch     |                64 |               - |
-| haskell      | ⚠️ Mismatch     |               157 |               - |
 | kusto        | ⚠️ Mismatch     |                60 |               - |
 | latex        | ⚠️ Mismatch     |               183 |               - |
 | markdown     | ⚠️ Mismatch     |               103 |               - |
 | mdc          | ⚠️ Mismatch     |                27 |               - |
-| mdx          | ⚠️ Mismatch     |               180 |               - |
 | mermaid      | ⚠️ Mismatch     |               129 |               - |
 | nginx        | ⚠️ Mismatch     |               102 |               - |
 | php          | ⚠️ Mismatch     |               328 |               - |
 | po           | ⚠️ Mismatch     |                23 |               - |
-| powerquery   | ⚠️ Mismatch     |                30 |               - |
-| powershell   | ⚠️ Mismatch     |                88 |               - |
 | pug          | ⚠️ Mismatch     |                92 |               - |
-| purescript   | ⚠️ Mismatch     |                72 |               - |
 | rst          | ⚠️ Mismatch     |                61 |               - |
 | splunk       | ⚠️ Mismatch     |                17 |               - |
 | stata        | ⚠️ Mismatch     |               189 |               - |
 | systemd      | ⚠️ Mismatch     |                32 |               - |
-| tex          | ⚠️ Mismatch     |                38 |               - |
-| yaml         | ⚠️ Mismatch     |                46 |               - |
 
 ## Unsupported Languages
 
 Languages that throws with the JavaScript RegExp engine (contains syntaxes that we can't polyfill yet). If you need to use these languages, please use the Oniguruma engine.
 
-| Language | Highlight Match | Patterns Parsable | Patterns Failed |
-| -------- | :-------------- | ----------------: | --------------: |
-| ada      | ✅ OK           |               199 |               1 |
-| csharp   | ⚠️ Mismatch     |               298 |               1 |
-| razor    | ⚠️ Mismatch     |                83 |               2 |
-| swift    | ❌ Error        |               302 |               4 |
-| julia    | ❌ Error        |                77 |              18 |
+| Language   | Highlight Match | Patterns Parsable | Patterns Failed |
+| ---------- | :-------------- | ----------------: | --------------: |
+| ada        | ✅ OK           |               199 |               1 |
+| csharp     | ⚠️ Mismatch     |               298 |               1 |
+| razor      | ⚠️ Mismatch     |                83 |               2 |
+| mdx        | ❌ Error        |               177 |               4 |
+| julia      | ❌ Error        |                90 |               5 |
+| swift      | ❌ Error        |               301 |               5 |
+| purescript | ❌ Error        |                64 |               8 |
+| haskell    | ❌ Error        |               113 |              44 |

packages/compat/test/fixtures.test.ts

@@ -8,7 +8,8 @@ import getHighlighter from '../src'
 describe('fixtures', () => {
   const files = import.meta.glob('./input/*.*', { as: 'raw', eager: true })
   const filter = process.env.FILTER
-  Object.entries(files)
+  Object
+    .entries(files)
     .forEach(([path, content]) => {
       const run = !filter || path.includes(filter)
         ? it

packages/core/src/highlight/code-to-tokens-ansi.ts

@@ -70,7 +70,8 @@ function dimColor(color: string) {
   if (hexMatch) {
     if (hexMatch[3]) {
       // convert from #rrggbbaa to #rrggbb(aa/2)
-      const alpha = Math.round(Number.parseInt(hexMatch[3], 16) / 2)
+      const alpha = Math
+        .round(Number.parseInt(hexMatch[3], 16) / 2)
         .toString(16)
         .padStart(2, '0')
       return `#${hexMatch[1]}${hexMatch[2]}${alpha}`
@@ -81,7 +82,8 @@ function dimColor(color: string) {
     }
     else {
       // convert from #rgb to #rrggbb80
-      return `#${Array.from(hexMatch[1])
+      return `#${Array
+        .from(hexMatch[1])
         .map(x => `${x}${x}`)
         .join('')}80`
     }

packages/core/src/highlight/code-to-tokens-themes.ts

@@ -14,7 +14,8 @@ export function codeToTokensWithThemes(
   code: string,
   options: CodeToTokensWithThemesOptions,
 ): ThemedTokenWithVariants[][] {
-  const themes = Object.entries(options.themes)
+  const themes = Object
+    .entries(options.themes)
     .filter(i => i[1])
     .map(i => ({ color: i[0], theme: i[1]! }))
 

packages/core/src/highlight/code-to-tokens.ts

@@ -26,7 +26,8 @@ export function codeToTokens(
       cssVariablePrefix = '--shiki-',
     } = options
 
-    const themes = Object.entries(options.themes)
+    const themes = Object
+      .entries(options.themes)
       .filter(i => i[1])
       .map(i => ({ color: i[0], theme: i[1]! }))
       .sort((a, b) => a.color === defaultColor ? -1 : b.color === defaultColor ? 1 : 0)

packages/core/src/utils.ts

@@ -137,7 +137,8 @@ export function splitToken<
 export function splitTokens<
   T extends Pick<ThemedToken, 'content' | 'offset'>,
 >(tokens: T[][], breakpoints: number[] | Set<number>) {
-  const sorted = Array.from(breakpoints instanceof Set ? breakpoints : new Set(breakpoints))
+  const sorted = Array
+    .from(breakpoints instanceof Set ? breakpoints : new Set(breakpoints))
     .sort((a, b) => a - b)
 
   if (!sorted.length)

packages/rehype/test/core.test.ts

@@ -48,7 +48,6 @@ it('run with rehype-raw', async () => {
     ],
   })
 
-  // eslint-disable-next-line unicorn/consistent-function-scoping
   const rehypeMetaString = () => (tree: Root) => {
     visit(tree, 'element', (node) => {
       if (node.tagName === 'code' && node.data?.meta) {

packages/shiki/scripts/prepare/langs.ts

@@ -118,12 +118,13 @@ export default langs
 
     const bundled = Array.from(bundledIds).map(id => grammars.find(i => i.name === id)!).filter(Boolean)
 
-    const info = bundled.map(i => ({
-      id: i.name,
-      name: i.displayName || i.name,
-      aliases: i.aliases,
-      import: `__(() => import('./langs/${i.name}')) as DynamicImportLanguageRegistration__`,
-    }) as const)
+    const info = bundled
+      .map(i => ({
+        id: i.name,
+        name: i.displayName || i.name,
+        aliases: i.aliases,
+        import: `__(() => import('./langs/${i.name}')) as DynamicImportLanguageRegistration__`,
+      }) as const)
       .sort((a, b) => a.id.localeCompare(b.id))
 
     const type = info.flatMap(i => [...i.aliases || [], i.id]).sort().map(i => `  | '${i}'`).join('\n')

packages/shiki/test/core.test.ts

@@ -158,7 +158,9 @@ describe('errors', () => {
     })
 
     const code = shiki.codeToHtml('console.log("Hi")', { lang: 'javascript', theme: mtp })
-    expect.soft(code)
+
+    expect
+      .soft(code)
       .toMatchInlineSnapshot(`"<pre class="shiki material-theme-palenight" style="background-color:#292D3E;color:#babed8" tabindex="0"><code><span class="line"><span style="color:#BABED8">console</span><span style="color:#89DDFF">.</span><span style="color:#82AAFF">log</span><span style="color:#BABED8">(</span><span style="color:#89DDFF">"</span><span style="color:#C3E88D">Hi</span><span style="color:#89DDFF">"</span><span style="color:#BABED8">)</span></span></code></pre>"`)
 
     expect.soft(shiki.getLoadedThemes()).toContain('material-theme-palenight')

packages/shiki/test/decorations.test.ts

@@ -105,7 +105,8 @@ describe('decorations errors', () => {
           { start: 10, end: 0 },
         ],
       })
-    }).rejects
+    })
+      .rejects
       .toThrowErrorMatchingInlineSnapshot(`[ShikiError: Invalid decoration range: {"line":1,"character":6,"offset":10} - {"line":0,"character":0,"offset":0}]`)
   })
 
@@ -119,8 +120,7 @@ describe('decorations errors', () => {
           { start: 5, end: 15 },
         ],
       })
-    }).rejects
-      .toThrowErrorMatchingInlineSnapshot(`[ShikiError: Decorations {"line":0,"character":0,"offset":0} and {"line":1,"character":1,"offset":5} intersect.]`)
+    }).rejects.toThrowErrorMatchingInlineSnapshot(`[ShikiError: Decorations {"line":0,"character":0,"offset":0} and {"line":1,"character":1,"offset":5} intersect.]`)
   })
 
   it('throws when lines overflow', async () => {
@@ -132,8 +132,7 @@ describe('decorations errors', () => {
           { start: { line: 100, character: 0 }, end: { line: 100, character: 1 } },
         ],
       })
-    }).rejects
-      .toThrowErrorMatchingInlineSnapshot(`[ShikiError: Invalid decoration position {"line":100,"character":0}. Lines length: 12]`)
+    }).rejects.toThrowErrorMatchingInlineSnapshot(`[ShikiError: Invalid decoration position {"line":100,"character":0}. Lines length: 12]`)
   })
 
   it('throws when chars overflow', async () => {
@@ -145,8 +144,7 @@ describe('decorations errors', () => {
           { start: { line: 0, character: 0 }, end: { line: 0, character: 10 } },
         ],
       })
-    }).rejects
-      .toThrowErrorMatchingInlineSnapshot(`[ShikiError: Invalid decoration position {"line":0,"character":10}. Line 0 length: 4]`)
+    }).rejects.toThrowErrorMatchingInlineSnapshot(`[ShikiError: Invalid decoration position {"line":0,"character":10}. Line 0 length: 4]`)
 
     expect(async () => {
       await codeToHtml(code, {
@@ -159,8 +157,7 @@ describe('decorations errors', () => {
           },
         ],
       })
-    }).rejects
-      .toThrowErrorMatchingInlineSnapshot(`[ShikiError: Invalid decoration position {"line":1,"character":36}. Line 1 length: 33]`)
+    }).rejects.toThrowErrorMatchingInlineSnapshot(`[ShikiError: Invalid decoration position {"line":1,"character":36}. Line 1 length: 33]`)
   })
 
   it('throws when offset underflows/overflows', async () => {
@@ -170,16 +167,14 @@ describe('decorations errors', () => {
         lang: 'ts',
         decorations: [{ start: 1, end: 1000 }],
       })
-    }).rejects
-      .toThrowErrorMatchingInlineSnapshot(`[ShikiError: Invalid decoration offset: 1000. Code length: 252]`)
+    }).rejects.toThrowErrorMatchingInlineSnapshot(`[ShikiError: Invalid decoration offset: 1000. Code length: 252]`)
 
     expect(async () => {
       await codeToHtml(code, {
         theme: 'vitesse-light',
         lang: 'ts',
         decorations: [{ start: -3, end: 5 }],
       })
-    }).rejects
-      .toThrowErrorMatchingInlineSnapshot(`[ShikiError: Invalid decoration offset: -3. Code length: 252]`)
+    }).rejects.toThrowErrorMatchingInlineSnapshot(`[ShikiError: Invalid decoration offset: -3. Code length: 252]`)
   })
 })

packages/shiki/test/engine-js/compare.test.ts

@@ -183,7 +183,8 @@ describe('cases', async () => {
         ])
       }
 
-      await expect.soft(JSON.stringify(engineWasm.instances, null, 2))
+      await expect
+        .soft(JSON.stringify(engineWasm.instances, null, 2))
         .toMatchFileSnapshot(`./__records__/${c.c.name}.json`)
 
       compare.forEach(([a, b]) => {

packages/shiki/test/grammar-state.test.ts

@@ -41,13 +41,17 @@ it('getLastGrammarState', async () => {
     grammarState: state,
   })
 
-  expect.soft(highlightedNatural)
-    .not.toEqual(highlightedContext)
+  expect
+    .soft(highlightedNatural)
+    .not
+    .toEqual(highlightedContext)
 
-  expect.soft(highlightedContext)
+  expect
+    .soft(highlightedContext)
     .toEqual(highlightedContext2)
 
-  expect.soft(highlightedNatural)
+  expect
+    .soft(highlightedNatural)
     .toMatchInlineSnapshot(`
       {
         "bg": "#ffffff",
@@ -115,8 +119,7 @@ it('getLastGrammarState', async () => {
       }
     `)
 
-  expect.soft(highlightedContext)
-    .toMatchInlineSnapshot(`
+  expect.soft(highlightedContext).toMatchInlineSnapshot(`
       {
         "bg": "#ffffff",
         "fg": "#393a34",
@@ -203,7 +206,8 @@ it('grammarContextCode', async () => {
     .toMatchInlineSnapshot(`"<span style="color:#999999">&#x3C;</span><span style="color:#1E754F">div</span><span style="color:#999999"> :</span><span style="color:#59873A">value</span><span style="color:#999999">=</span><span style="color:#999999">"</span><span style="color:#2F798A">1</span><span style="color:#AB5959"> +</span><span style="color:#2F798A"> 2</span><span style="color:#999999">"</span><span style="color:#999999">></span><span style="color:#393A34">&#x3C;button /></span><span style="color:#999999">&#x3C;/</span><span style="color:#1E754F">div</span><span style="color:#999999">></span>"`)
 
   expect(highlightedVueTemplate)
-    .not.toEqual(highlightedVueBare)
+    .not
+    .toEqual(highlightedVueBare)
 })
 
 describe('errors', () => {

packages/shiki/test/wasm4.test.ts

@@ -11,7 +11,6 @@ it('wasm', async () => {
   const shiki = await createHighlighterCore({
     themes: [nord],
     langs: [js as any],
-    // eslint-disable-next-line unicorn/consistent-function-scoping
     loadWasm: Promise.resolve().then(() => obj => WebAssembly.instantiate(wasmBinary, obj).then(r => r.instance)),
   })
 

packages/transformers/src/transformers/transformer-meta-highlight.ts

@@ -6,7 +6,8 @@ export function parseMetaHighlightString(meta: string) {
   const match = meta.match(/\{([\d,-]+)\}/)
   if (!match)
     return null
-  const lines = match[1].split(',')
+  const lines = match[1]
+    .split(',')
     .flatMap((v) => {
       const num = v.split('-').map(v => Number.parseInt(v, 10))
       if (num.length === 1)

packages/twoslash/src/renderer-rich.ts

@@ -419,7 +419,8 @@ export function rendererRich(options: RendererRichOptions = {}): TwoslashRendere
       if (node.type !== 'text')
         throw new ShikiTwoslashError(`Renderer hook nodeCompletion only works on text nodes, got ${node.type}`)
 
-      const items: Element[] = query.completions
+      const items: Element[] = query
+        .completions
         .map((i) => {
           const kind = i.kind || 'default'
           const isDeprecated = 'kindModifiers' in i && typeof i.kindModifiers === 'string' && i.kindModifiers?.split(',').includes('deprecated')

packages/twoslash/test/fixtures.test.ts

@@ -30,11 +30,13 @@ describe('fixtures', () => {
 
       const html = hastToHtml(hast)
 
-      expect.soft(JSON.stringify(hast, null, 2))
+      expect
+        .soft(JSON.stringify(hast, null, 2))
         .toMatchFileSnapshot(`./out/${name}.json`)
 
       const style = '<link rel="stylesheet" href="../../style-rich.css" />'
-      expect.soft(style + html)
+      expect
+        .soft(style + html)
         .toMatchFileSnapshot(`./out/${name}.html`)
     })
   }