fix(engine): regex flags propagation

shikijs · Aug 30, 2024 · 55526bb · 55526bb
1 parent 61a6bf2
commit 55526bb
Show file tree

Hide file tree

Showing 11 changed files with 2,706 additions and 83 deletions.
diff --git a/packages/core/src/engines/javascript.ts b/packages/core/src/engines/javascript.ts
@@ -27,7 +27,10 @@ export class JavaScriptScanner implements PatternScanner {
           p
             // YAML specific handling; TODO: move to tm-grammars
             .replaceAll('[^\\s[-?:,\\[\\]{}#&*!|>\'"%@`]]', '[^\\s\\-?:,\\[\\]{}#&*!|>\'"%@`]'),
-          { flags: 'dgm' },
+          {
+            flags: 'dgm',
+            ignoreContiguousAnchors: true,
+          },
         )
         cache?.set(p, regex)
         return regex

diff --git a/packages/shiki/test/engine-js/__records__/jsonc.json b/packages/shiki/test/engine-js/__records__/jsonc.json
@@ -0,0 +1,241 @@
+[
+  {
+    "constractor": [
+      [
+        "\\b(?:true|false|null)\\b",
+        "-?(?:0|[1-9]\\d*)(?:(?:\\.\\d+)?(?:[eE][+-]?\\d+)?)?",
+        "\"",
+        "\\[",
+        "\\{",
+        "/\\*\\*(?!/)",
+        "/\\*",
+        "(//).*$\\n?"
+      ]
+    ],
+    "executions": [
+      {
+        "args": [
+          "// comment\n",
+          0
+        ],
+        "result": {
+          "index": 7,
+          "captureIndices": [
+            {
+              "start": 0,
+              "end": 11,
+              "length": 11
+            },
+            {
+              "start": 0,
+              "end": 2,
+              "length": 2
+            }
+          ]
+        }
+      },
+      {
+        "args": [
+          "// comment\n",
+          11
+        ],
+        "result": null
+      },
+      {
+        "args": [
+          "{\"foo\":\"bar\"}\n",
+          0
+        ],
+        "result": {
+          "index": 4,
+          "captureIndices": [
+            {
+              "start": 0,
+              "end": 1,
+              "length": 1
+            }
+          ]
+        }
+      },
+      {
+        "args": [
+          "{\"foo\":\"bar\"}\n",
+          13
+        ],
+        "result": null
+      }
+    ]
+  },
+  {
+    "constractor": [
+      [
+        "\\}",
+        "\"",
+        "/\\*\\*(?!/)",
+        "/\\*",
+        "(//).*$\\n?",
+        ":",
+        "[^\\s\\}]"
+      ]
+    ],
+    "executions": [
+      {
+        "args": [
+          "{\"foo\":\"bar\"}\n",
+          1
+        ],
+        "result": {
+          "index": 1,
+          "captureIndices": [
+            {
+              "start": 1,
+              "end": 2,
+              "length": 1
+            }
+          ]
+        }
+      },
+      {
+        "args": [
+          "{\"foo\":\"bar\"}\n",
+          6
+        ],
+        "result": {
+          "index": 5,
+          "captureIndices": [
+            {
+              "start": 6,
+              "end": 7,
+              "length": 1
+            }
+          ]
+        }
+      },
+      {
+        "args": [
+          "{\"foo\":\"bar\"}\n",
+          12
+        ],
+        "result": {
+          "index": 0,
+          "captureIndices": [
+            {
+              "start": 12,
+              "end": 13,
+              "length": 1
+            }
+          ]
+        }
+      }
+    ]
+  },
+  {
+    "constractor": [
+      [
+        "\"",
+        "\\\\(?:[\"\\\\/bfnrt]|u[0-9a-fA-F]{4})",
+        "\\\\."
+      ]
+    ],
+    "executions": [
+      {
+        "args": [
+          "{\"foo\":\"bar\"}\n",
+          2
+        ],
+        "result": {
+          "index": 0,
+          "captureIndices": [
+            {
+              "start": 5,
+              "end": 6,
+              "length": 1
+            }
+          ]
+        }
+      }
+    ]
+  },
+  {
+    "constractor": [
+      [
+        "(,)|(?=\\})",
+        "\\b(?:true|false|null)\\b",
+        "-?(?:0|[1-9]\\d*)(?:(?:\\.\\d+)?(?:[eE][+-]?\\d+)?)?",
+        "\"",
+        "\\[",
+        "\\{",
+        "/\\*\\*(?!/)",
+        "/\\*",
+        "(//).*$\\n?",
+        "[^\\s,]"
+      ]
+    ],
+    "executions": [
+      {
+        "args": [
+          "{\"foo\":\"bar\"}\n",
+          7
+        ],
+        "result": {
+          "index": 3,
+          "captureIndices": [
+            {
+              "start": 7,
+              "end": 8,
+              "length": 1
+            }
+          ]
+        }
+      },
+      {
+        "args": [
+          "{\"foo\":\"bar\"}\n",
+          12
+        ],
+        "result": {
+          "index": 0,
+          "captureIndices": [
+            {
+              "start": 12,
+              "end": 12,
+              "length": 0
+            },
+            {
+              "start": 4294967295,
+              "end": 4294967295,
+              "length": 0
+            }
+          ]
+        }
+      }
+    ]
+  },
+  {
+    "constractor": [
+      [
+        "\"",
+        "\\\\(?:[\"\\\\/bfnrt]|u[0-9a-fA-F]{4})",
+        "\\\\."
+      ]
+    ],
+    "executions": [
+      {
+        "args": [
+          "{\"foo\":\"bar\"}\n",
+          8
+        ],
+        "result": {
+          "index": 0,
+          "captureIndices": [
+            {
+              "start": 11,
+              "end": 12,
+              "length": 1
+            }
+          ]
+        }
+      }
+    ]
+  }
+]
diff --git a/packages/shiki/test/engine-js/__records__/sql.json b/packages/shiki/test/engine-js/__records__/sql.json