fallback unknown lang to text

JSerZANP · Jul 16, 2024 · 701f562 · 701f562
1 parent 727226c
commit 701f562
Show file tree

Hide file tree

Showing 3 changed files with 43 additions and 0 deletions.
diff --git a/...shaku-code-annotate-shiki-transformer/src/__tests__/__snapshots__/codeToHtml.test.ts.snap b/...shaku-code-annotate-shiki-transformer/src/__tests__/__snapshots__/codeToHtml.test.ts.snap
@@ -126,3 +126,15 @@ exports[`codeToHtml() 4`] = `"<pre class=\\"shiki github-light shaku\\" style=\\
 exports[`codeToHtml() 5`] = `"<pre class=\\"shiki github-light shaku\\" style=\\"background-color:#fff;color:#24292e\\" tabindex=\\"0\\"><code><span class=\\"line\\"></span><span class=\\"line\\"><span style=\\"color:#D73A49\\">function</span><span style=\\"color:#6F42C1\\"> useSomeEffect</span><span style=\\"color:#24292E\\">({</span><span style=\\"color:#E36209\\">blog</span><span style=\\"color:#24292E\\">}) {</span></span><span class=\\"line\\"><span style=\\"color:#6F42C1\\">  useEffect</span><span style=\\"color:#24292E\\">(() </span><span style=\\"color:#D73A49\\">=></span><span style=\\"color:#24292E\\"> {</span></span><span class=\\"line\\" data-a=\\"1\\"><span style=\\"color:#D73A49\\">      return</span><span style=\\"color:#24292E\\"> () </span><span style=\\"color:#D73A49\\">=></span><span style=\\"color:#24292E\\"> {</span></span><span class=\\"line\\" data-a-b-c=\\"1-2-3\\" data-beg-1=\\"hello-2\\"><span style=\\"color:#24292E\\">        location.href </span><span style=\\"color:#D73A49\\">=</span><span style=\\"color:#032F62\\"> 'https://jser.dev'</span></span><span class=\\"line\\"><span style=\\"color:#24292E\\">      }</span></span><span class=\\"line\\"><span style=\\"color:#24292E\\">    }, [blog])</span></span><span class=\\"line\\"><span style=\\"color:#24292E\\">}</span></span><span class=\\"line\\"><span style=\\"color:#24292E\\">  </span></span><span class=\\"line\\"></span></code></pre>"`;
 
 exports[`codeToHtml() 6`] = `"<pre class=\\"shiki github-light shaku\\" style=\\"background-color:#fff;color:#24292e\\" tabindex=\\"0\\"><code><span class=\\"line\\"></span><span class=\\"line\\"><span style=\\"color:#D73A49\\">  function</span><span style=\\"color:#6F42C1\\"> useSomeEffect</span><span style=\\"color:#24292E\\">({</span><span style=\\"color:#E36209\\">blog</span><span style=\\"color:#24292E\\">}) {</span></span><span class=\\"line\\"><span style=\\"color:#24292E\\">          location.href </span><span style=\\"color:#D73A49\\">=</span><span style=\\"color:#032F62\\"> 'https://jser.dev'</span></span><span class=\\"line\\"><span style=\\"color:#24292E\\">  }</span></span><span class=\\"line\\"><span style=\\"color:#24292E\\">    </span></span><span class=\\"line\\"></span></code></pre>"`;
+
+exports[`unknown lang should fallback to text 1`] = `"<pre class=\\"shiki github-light shaku\\" style=\\"background-color:#fff;color:#24292e\\" tabindex=\\"0\\"><code><span class=\\"line\\"><span></span></span><span class=\\"line\\"><span>function ChatRoom({ roomId }) {</span></span><span class=\\"line\\"><span>  const [serverUrl, setServerUrl] = useState('https://localhost:1234');</span></span><span class=\\"line\\"><span>  //       ^</span></span><span class=\\"line\\"><span>  // [&#x3C;a href=\\"https:jser.dev\\">jser.dev&#x3C;/a>]</span></span><span class=\\"line\\"><span></span></span><span class=\\"line\\"><span>  useEffect(() => {</span></span><span class=\\"line\\"><span>     // (                                                     )</span></span><span class=\\"line\\"><span>        const connection = createConnection(serverUrl, roomId);</span></span><span class=\\"line\\"><span>     // (1                )</span></span><span class=\\"line\\"><span>        connection.connect();</span></span><span class=\\"line\\"><span>    </span></span><span class=\\"line\\"><span>  	return () => {</span></span><span class=\\"line\\"><span>           //(2                    )</span></span><span class=\\"line\\"><span>             connection.disconnect();</span></span><span class=\\"line\\"><span>  	};</span></span><span class=\\"line\\"><span>   //(3                 )</span></span><span class=\\"line\\"><span>  }, [serverUrl, roomId]);</span></span><span class=\\"line\\"><span>}</span></span><span class=\\"line\\"><span></span></span><span class=\\"line\\"><span></span></span></code></pre>"`;
+
+exports[`unknown lang should fallback to text 2`] = `"<pre class=\\"shiki github-light shaku\\" style=\\"background-color:#fff;color:#24292e\\" tabindex=\\"0\\"><code><span class=\\"line\\"><span></span></span><span class=\\"line\\"><span>function hello() {</span></span><span class=\\"line\\"><span>  //        ^!</span></span><span class=\\"line\\"><span>  //    [Hello world!]!</span></span><span class=\\"line\\"><span>  //        ^</span></span><span class=\\"line\\"><span>  //    [Shaku &#x3C;b>Shaku&#x3C;/b>]</span></span><span class=\\"line\\"><span>  const blog = \\"https://jser.dev\\"</span></span><span class=\\"line\\"><span>}</span></span><span class=\\"line\\"><span>  </span></span><span class=\\"line\\"><span></span></span></code></pre>"`;
+
+exports[`unknown lang should fallback to text 3`] = `"<pre class=\\"shiki github-light shaku\\" style=\\"background-color:#fff;color:#24292e\\" tabindex=\\"0\\"><code><span class=\\"line\\"><span></span></span><span class=\\"line\\"><span>function useSomeEffect({blog}) {</span></span><span class=\\"line\\"><span>  useEffect(() => {</span></span><span class=\\"line\\"><span>    //  do some stuff</span></span><span class=\\"line\\"><span>      // @fold start</span></span><span class=\\"line\\"><span>      return () => {</span></span><span class=\\"line\\"><span>        // @highlight</span></span><span class=\\"line\\"><span>        location.href = 'https://jser.dev'</span></span><span class=\\"line\\"><span>      }</span></span><span class=\\"line\\"><span>      // @fold ^</span></span><span class=\\"line\\"><span>    }, [blog])</span></span><span class=\\"line\\"><span>}</span></span><span class=\\"line\\"><span>  </span></span><span class=\\"line\\"><span></span></span></code></pre>"`;
+
+exports[`unknown lang should fallback to text 4`] = `"<pre class=\\"shiki github-light shaku\\" style=\\"background-color:#fff;color:#24292e\\" tabindex=\\"0\\"><code><span class=\\"line\\"><span></span></span><span class=\\"line\\"><span>// @class abc h123</span></span><span class=\\"line\\"><span>// @highlight</span></span><span class=\\"line\\"><span>function useSomeEffect({blog}) {</span></span><span class=\\"line\\"><span>  useEffect(() => {</span></span><span class=\\"line\\"><span>      return () => {</span></span><span class=\\"line\\"><span>        location.href = 'https://jser.dev'</span></span><span class=\\"line\\"><span>      }</span></span><span class=\\"line\\"><span>    }, [blog])</span></span><span class=\\"line\\"><span>}</span></span><span class=\\"line\\"><span>  </span></span><span class=\\"line\\"><span></span></span></code></pre>"`;
+
+exports[`unknown lang should fallback to text 5`] = `"<pre class=\\"shiki github-light shaku\\" style=\\"background-color:#fff;color:#24292e\\" tabindex=\\"0\\"><code><span class=\\"line\\"><span></span></span><span class=\\"line\\"><span>function useSomeEffect({blog}) {</span></span><span class=\\"line\\"><span>  useEffect(() => {</span></span><span class=\\"line\\"><span>    // @data a=1</span></span><span class=\\"line\\"><span>      return () => {</span></span><span class=\\"line\\"><span>        // @data a-b-c=1-2-3 beg-1=hello-2</span></span><span class=\\"line\\"><span>        location.href = 'https://jser.dev'</span></span><span class=\\"line\\"><span>      }</span></span><span class=\\"line\\"><span>    }, [blog])</span></span><span class=\\"line\\"><span>}</span></span><span class=\\"line\\"><span>  </span></span><span class=\\"line\\"><span></span></span></code></pre>"`;
+
+exports[`unknown lang should fallback to text 6`] = `"<pre class=\\"shiki github-light shaku\\" style=\\"background-color:#fff;color:#24292e\\" tabindex=\\"0\\"><code><span class=\\"line\\"><span></span></span><span class=\\"line\\"><span>  function useSomeEffect({blog}) {</span></span><span class=\\"line\\"><span>    // @cut v</span></span><span class=\\"line\\"><span>    useEffect(() => {</span></span><span class=\\"line\\"><span>        return () => {</span></span><span class=\\"line\\"><span>          // @cut ^</span></span><span class=\\"line\\"><span>          location.href = 'https://jser.dev'</span></span><span class=\\"line\\"><span>          // @cut start</span></span><span class=\\"line\\"><span>        }</span></span><span class=\\"line\\"><span>      }, [blog])</span></span><span class=\\"line\\"><span>      // @cut end</span></span><span class=\\"line\\"><span>    // @cut</span></span><span class=\\"line\\"><span>    // a comment to cut</span></span><span class=\\"line\\"><span>  }</span></span><span class=\\"line\\"><span>    </span></span><span class=\\"line\\"><span></span></span></code></pre>"`;
diff --git a/packages/shaku-code-annotate-shiki-transformer/src/__tests__/codeToHtml.test.ts b/packages/shaku-code-annotate-shiki-transformer/src/__tests__/codeToHtml.test.ts
@@ -129,3 +129,18 @@ test("codeToHtml() + raw HTML + explicit trigger", async () => {
     expect(html).toMatchSnapshot();
   }
 });
+
+test("unknown lang should fallback to text", async () => {
+  const processor = await fetchProcessor(
+    "javascript",
+    true /* useDangerousRawHtml */
+  );
+
+  for (const snippet of snippets) {
+    const html = await processor
+      .process(`\`\`\`unknown \n${snippet}\n\`\`\``)
+      .then((result) => result.toString());
+
+    expect(html).toMatchSnapshot();
+  }
+});
diff --git a/packages/shaku-code-annotate-shiki-transformer/src/__tests__/util.ts b/packages/shaku-code-annotate-shiki-transformer/src/__tests__/util.ts
@@ -5,12 +5,27 @@ import remarkRehype from "remark-rehype";
 import { unified } from "unified";
 import shakuCodeAnnotateShikiTransformer from "..";
 import * as shiki from "shiki";
+import { visit } from "unist-util-visit";
+import { commentPatterns } from "../commentPatterns";
 
 export const unifiedProcessor = unified()
   .use(remarkParse)
   .use(remarkRehype) // this sanitize html by default
   .use(rehypeStringify);
 
+function remarkFallbackLang() {
+  return (tree) => {
+    visit(tree, "code", (node) => {
+      if (node.lang) {
+        node.lang = node.lang.toLowerCase();
+        if (!(node.lang in commentPatterns)) {
+          node.lang = "text";
+        }
+      }
+    });
+  };
+}
+
 export function fetchProcessor(
   lang: string,
   useDangerousRawHtml: boolean,
@@ -24,6 +39,7 @@ export function fetchProcessor(
     .then((highlighter) =>
       unified()
         .use(remarkParse)
+        .use(remarkFallbackLang)
         .use(remarkRehype)
         .use(rehypeShiki, {
           transformers: [