From 3d47510c977b9ac5f474393889fe6348393fe7f6 Mon Sep 17 00:00:00 2001 From: JSer <69352453+JSerZANP@users.noreply.github.com> Date: Fri, 5 Jul 2024 17:38:20 +0900 Subject: [PATCH] close #48 --- .../__snapshots__/codeToHtml.test.ts.snap | 24 +- .../src/__tests__/langs/liquid/output.txt | 2 +- .../src/__tests__/langs/mermaid/output.txt | 2 +- .../src/__tests__/langs/objc/output.txt | 2 +- .../src/__tests__/langs/prolog/output.txt | 2 +- .../src/__tests__/langs/razor/output.txt | 2 +- .../src/__tests__/langs/smalltalk/output.txt | 2 +- .../src/__tests__/langs/stata/output.txt | 2 +- .../src/__tests__/langs/tsx/output.txt | 2 +- .../src/commentPatterns.ts | 183 +++++++++ .../src/defaultCode.ts | 55 ++- .../src/index.ts | 383 ++---------------- .../src/defaultCode.ts | 52 +-- 13 files changed, 288 insertions(+), 425 deletions(-) create mode 100644 packages/shaku-code-annotate-shiki-transformer/src/commentPatterns.ts diff --git a/packages/shaku-code-annotate-shiki-transformer/src/__tests__/__snapshots__/codeToHtml.test.ts.snap b/packages/shaku-code-annotate-shiki-transformer/src/__tests__/__snapshots__/codeToHtml.test.ts.snap index 5192e2e..c230423 100644 --- a/packages/shaku-code-annotate-shiki-transformer/src/__tests__/__snapshots__/codeToHtml.test.ts.snap +++ b/packages/shaku-code-annotate-shiki-transformer/src/__tests__/__snapshots__/codeToHtml.test.ts.snap @@ -1,25 +1,25 @@ // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html -exports[`codeToHtml() + raw HTML 1`] = `"
function ChatRoom({ roomId }) {  const [serverUrl, setServerUrl] = useState('https://localhost:1234');
jser.dev
  useEffect(() => {        const connection = createConnection(serverUrl, roomId);        connection.connect();      	return () => {             connection.disconnect();  	};  }, [serverUrl, roomId]);}
"`; +exports[`codeToHtml() + raw HTML 1`] = `"
function ChatRoom({ roomId }) {  const [serverUrl, setServerUrl] = useState('https://localhost:1234');  //       ^  // [<a href=\\"https:jser.dev\\">jser.dev</a>]  useEffect(() => {     // (                                                     )        const connection = createConnection(serverUrl, roomId);     // (1                )        connection.connect();      	return () => {           //(2                    )             connection.disconnect();  	};   //(3                 )  }, [serverUrl, roomId]);}
"`; -exports[`codeToHtml() + raw HTML 2`] = `"
function hello() {  //        ^  //    [Hello world!]
Shaku Shaku
  const blog = \\"https://jser.dev\\"}
"`; +exports[`codeToHtml() + raw HTML 2`] = `"
function hello() {  //        ^!  //    [Hello world!]!  //        ^  //    [Shaku <b>Shaku</b>]  const blog = \\"https://jser.dev\\"}
"`; -exports[`codeToHtml() + raw HTML 3`] = `"
function useSomeEffect({blog}) {  useEffect(() => {    //  do some stuff
{...}      return () => {        location.href = 'https://jser.dev'      }
    }, [blog])}
"`; +exports[`codeToHtml() + raw HTML 3`] = `"
function useSomeEffect({blog}) {  useEffect(() => {    //  do some stuff      // @fold start      return () => {        // @highlight        location.href = 'https://jser.dev'      }      // @fold ^    }, [blog])}
"`; -exports[`codeToHtml() + raw HTML 4`] = `"
function useSomeEffect({blog}) {  useEffect(() => {      return () => {        location.href = 'https://jser.dev'      }    }, [blog])}
"`; +exports[`codeToHtml() + raw HTML 4`] = `"
// @class abc h123// @highlightfunction useSomeEffect({blog}) {  useEffect(() => {      return () => {        location.href = 'https://jser.dev'      }    }, [blog])}
"`; -exports[`codeToHtml() + raw HTML 5`] = `"
function useSomeEffect({blog}) {  useEffect(() => {      return () => {        location.href = 'https://jser.dev'      }    }, [blog])}
"`; +exports[`codeToHtml() + raw HTML 5`] = `"
function useSomeEffect({blog}) {  useEffect(() => {    // @data a=1      return () => {        // @data a-b-c=1-2-3 beg-1=hello-2        location.href = 'https://jser.dev'      }    }, [blog])}
"`; -exports[`codeToHtml() + raw HTML 6`] = `"
  function useSomeEffect({blog}) {          location.href = 'https://jser.dev'  }
"`; +exports[`codeToHtml() + raw HTML 6`] = `"
  function useSomeEffect({blog}) {    // @cut v    useEffect(() => {        return () => {          // @cut ^          location.href = 'https://jser.dev'          // @cut start        }      }, [blog])      // @cut end    // @cut    // a comment to cut  }
"`; -exports[`codeToHtml() 1`] = `"
function ChatRoom({ roomId }) {  const [serverUrl, setServerUrl] = useState('https://localhost:1234');
&lt;p&gt;jser.dev&lt;/p&gt;
  useEffect(() => {        const connection = createConnection(serverUrl, roomId);        connection.connect();      	return () => {             connection.disconnect();  	};  }, [serverUrl, roomId]);}
"`; +exports[`codeToHtml() 1`] = `"
function ChatRoom({ roomId }) {  const [serverUrl, setServerUrl] = useState('https://localhost:1234');  //       ^  // [<a href=\\"https:jser.dev\\">jser.dev</a>]  useEffect(() => {     // (                                                     )        const connection = createConnection(serverUrl, roomId);     // (1                )        connection.connect();      	return () => {           //(2                    )             connection.disconnect();  	};   //(3                 )  }, [serverUrl, roomId]);}
"`; -exports[`codeToHtml() 2`] = `"
function hello() {  //        ^  //    [Hello world!]
&lt;p&gt;Shaku Shaku&lt;/p&gt;
  const blog = \\"https://jser.dev\\"}
"`; +exports[`codeToHtml() 2`] = `"
function hello() {  //        ^!  //    [Hello world!]!  //        ^  //    [Shaku <b>Shaku</b>]  const blog = \\"https://jser.dev\\"}
"`; -exports[`codeToHtml() 3`] = `"
function useSomeEffect({blog}) {  useEffect(() => {    //  do some stuff
{...}      return () => {        location.href = 'https://jser.dev'      }
    }, [blog])}
"`; +exports[`codeToHtml() 3`] = `"
function useSomeEffect({blog}) {  useEffect(() => {    //  do some stuff      // @fold start      return () => {        // @highlight        location.href = 'https://jser.dev'      }      // @fold ^    }, [blog])}
"`; -exports[`codeToHtml() 4`] = `"
function useSomeEffect({blog}) {  useEffect(() => {      return () => {        location.href = 'https://jser.dev'      }    }, [blog])}
"`; +exports[`codeToHtml() 4`] = `"
// @class abc h123// @highlightfunction useSomeEffect({blog}) {  useEffect(() => {      return () => {        location.href = 'https://jser.dev'      }    }, [blog])}
"`; -exports[`codeToHtml() 5`] = `"
function useSomeEffect({blog}) {  useEffect(() => {      return () => {        location.href = 'https://jser.dev'      }    }, [blog])}
"`; +exports[`codeToHtml() 5`] = `"
function useSomeEffect({blog}) {  useEffect(() => {    // @data a=1      return () => {        // @data a-b-c=1-2-3 beg-1=hello-2        location.href = 'https://jser.dev'      }    }, [blog])}
"`; -exports[`codeToHtml() 6`] = `"
  function useSomeEffect({blog}) {          location.href = 'https://jser.dev'  }
"`; +exports[`codeToHtml() 6`] = `"
  function useSomeEffect({blog}) {    // @cut v    useEffect(() => {        return () => {          // @cut ^          location.href = 'https://jser.dev'          // @cut start        }      }, [blog])      // @cut end    // @cut    // a comment to cut  }
"`; diff --git a/packages/shaku-code-annotate-shiki-transformer/src/__tests__/langs/liquid/output.txt b/packages/shaku-code-annotate-shiki-transformer/src/__tests__/langs/liquid/output.txt index ced215f..9a48b35 100644 --- a/packages/shaku-code-annotate-shiki-transformer/src/__tests__/langs/liquid/output.txt +++ b/packages/shaku-code-annotate-shiki-transformer/src/__tests__/langs/liquid/output.txt @@ -1 +1 @@ -
This is a multi-line comment {% comment %}
----------
Hello World!
{% endcomment %}
\ No newline at end of file +
This is a multi-line comment {% comment %}          ----------     ^  [Hello World!]{% endcomment %}
\ No newline at end of file diff --git a/packages/shaku-code-annotate-shiki-transformer/src/__tests__/langs/mermaid/output.txt b/packages/shaku-code-annotate-shiki-transformer/src/__tests__/langs/mermaid/output.txt index 0ace835..820a0d4 100644 --- a/packages/shaku-code-annotate-shiki-transformer/src/__tests__/langs/mermaid/output.txt +++ b/packages/shaku-code-annotate-shiki-transformer/src/__tests__/langs/mermaid/output.txt @@ -1 +1 @@ -
graph LR    A[Start] --> B[Process]    %% @highlight    B --> C[Process]    C --> D[End]    %%    -----  %%      ^  %% [Hello Shaku!]
\ No newline at end of file +
graph LR    A[Start] --> B[Process]    B --> C[Process]    C --> D[End]
-----
Hello Shaku!
\ No newline at end of file diff --git a/packages/shaku-code-annotate-shiki-transformer/src/__tests__/langs/objc/output.txt b/packages/shaku-code-annotate-shiki-transformer/src/__tests__/langs/objc/output.txt index 8110f44..9992546 100644 --- a/packages/shaku-code-annotate-shiki-transformer/src/__tests__/langs/objc/output.txt +++ b/packages/shaku-code-annotate-shiki-transformer/src/__tests__/langs/objc/output.txt @@ -1 +1 @@ -
#import <Foundation/Foundation.h>// This is a single-line comment
------
Hello World!
\ No newline at end of file +
#import <Foundation/Foundation.h>// @highlight// This is a single-line comment// ------//  ^// [Hello World!]
\ No newline at end of file diff --git a/packages/shaku-code-annotate-shiki-transformer/src/__tests__/langs/prolog/output.txt b/packages/shaku-code-annotate-shiki-transformer/src/__tests__/langs/prolog/output.txt index 0f686a9..5a26cab 100644 --- a/packages/shaku-code-annotate-shiki-transformer/src/__tests__/langs/prolog/output.txt +++ b/packages/shaku-code-annotate-shiki-transformer/src/__tests__/langs/prolog/output.txt @@ -1 +1 @@ -
% Predicates for family relationships% @highlightparent(john, sarah).%     -----%       ^%  [Hello World!]
\ No newline at end of file +
% Predicates for family relationshipsparent(john, sarah).
-----
Hello World!
\ No newline at end of file diff --git a/packages/shaku-code-annotate-shiki-transformer/src/__tests__/langs/razor/output.txt b/packages/shaku-code-annotate-shiki-transformer/src/__tests__/langs/razor/output.txt index 0ac4fd0..4de8a42 100644 --- a/packages/shaku-code-annotate-shiki-transformer/src/__tests__/langs/razor/output.txt +++ b/packages/shaku-code-annotate-shiki-transformer/src/__tests__/langs/razor/output.txt @@ -1 +1 @@ -
<h1>Welcome to our website!</h1>  @{
----
Hello World!
  }
\ No newline at end of file +
<h1>Welcome to our website!</h1>  @{    // ----    // ^    //[Hello World!]  }
\ No newline at end of file diff --git a/packages/shaku-code-annotate-shiki-transformer/src/__tests__/langs/smalltalk/output.txt b/packages/shaku-code-annotate-shiki-transformer/src/__tests__/langs/smalltalk/output.txt index f7f1531..86fd734 100644 --- a/packages/shaku-code-annotate-shiki-transformer/src/__tests__/langs/smalltalk/output.txt +++ b/packages/shaku-code-annotate-shiki-transformer/src/__tests__/langs/smalltalk/output.txt @@ -1 +1 @@ -
Object subclass: MyClass [  "This is a simple class named MyClass."    | instanceVariable |    MyClass class >> classMethod [      "This is a class method of MyClass."            ^ 'Hello from class method!'
-----
Hello Shaku!
  ]    MyClass >> instanceMethod [      "This is an instance method of MyClass."            ^ 'Hello from instance method!'  ]]
\ No newline at end of file +
Object subclass: MyClass [  "This is a simple class named MyClass."    | instanceVariable |    "@highlight"  MyClass class >> classMethod [      "This is a class method of MyClass."            ^ 'Hello from class method!'      "  -----"      "  ^ "  " [Hello Shaku!]"  ]    MyClass >> instanceMethod [      "This is an instance method of MyClass."            ^ 'Hello from instance method!'  ]]
\ No newline at end of file diff --git a/packages/shaku-code-annotate-shiki-transformer/src/__tests__/langs/stata/output.txt b/packages/shaku-code-annotate-shiki-transformer/src/__tests__/langs/stata/output.txt index e991874..2e576e9 100644 --- a/packages/shaku-code-annotate-shiki-transformer/src/__tests__/langs/stata/output.txt +++ b/packages/shaku-code-annotate-shiki-transformer/src/__tests__/langs/stata/output.txt @@ -1 +1 @@ -
* @highlightuse "mydata.dta"
------
Hello Shaku!
\ No newline at end of file +
use "mydata.dta"
------
Hello Shaku!
\ No newline at end of file diff --git a/packages/shaku-code-annotate-shiki-transformer/src/__tests__/langs/tsx/output.txt b/packages/shaku-code-annotate-shiki-transformer/src/__tests__/langs/tsx/output.txt index e0c24a8..d6d42ec 100644 --- a/packages/shaku-code-annotate-shiki-transformer/src/__tests__/langs/tsx/output.txt +++ b/packages/shaku-code-annotate-shiki-transformer/src/__tests__/langs/tsx/output.txt @@ -1 +1 @@ -
const MyComponent: React.FC = () => {  return (    {/* @highlight */}    <div>      {/* This is a comment inside JSX */}      <h1>Hello, World!</h1>
-----
Hello World!
    </div>  );};
\ No newline at end of file +
const MyComponent: React.FC = () => {  return (    <div>      {/* This is a comment inside JSX */}      <h1>Hello, World!</h1>
-----
Hello World!
    </div>  );};
\ No newline at end of file diff --git a/packages/shaku-code-annotate-shiki-transformer/src/commentPatterns.ts b/packages/shaku-code-annotate-shiki-transformer/src/commentPatterns.ts new file mode 100644 index 0000000..8d89120 --- /dev/null +++ b/packages/shaku-code-annotate-shiki-transformer/src/commentPatterns.ts @@ -0,0 +1,183 @@ +import { supportedLangs } from "./defaultCode"; + +type Lang = (typeof supportedLangs)[number]; + +const PatternAsterisk = /^(?\s*)(?\*)(?.*)$/; +const PatternDoubleSlash = + /^(?\s*)(?\/\/)(?.*)$/; +const PatternDoubleDash = + /^(?\s*)(?\-\-)(?.*)$/; +const PatternHash = /^(?\s*)(?#)(?.*)$/; +const PatternSemiColon = /^(?\s*)(?;)(?.*)$/; +const PatternDoubleSemiColon = + /^(?\s*)(?;;)(?.*)$/; +const PatternSFC = + /^(?\s*)(?(\/\/|\<\!\-\-|\/\*))(?.*?)(?(\-\-\>|\*\/))?$/; +const PatternXML = + /^(?\s*)(?\<\!\-\-)(?.*)(?\-\-\>)$/; +const PatternPercentage = /^(?\s*)(?%)(?.*)$/; +const PatternDoublePercentage = + /^(?\s*)(?%%)(?.*)$/; +const PatternSlashAsterisk = + /^(?\s*)(?\/\*)(?.*)(?\*\/)$/; +const PatternJSWithJSX = + /^(?\s*)(?(\/\*|\/\/|\{\/\*))(?.*?)(?(\*\/|\*\/\}))?$/; +const PatternHBS = + /^(?\s*)(?\{\{\!\-\-)(?.*)(?\-\-\}\})$/; +const PatternBatch = /^(?\s*)(?REM)(?.*)$/i; +const PatternDoubleQuote = + /^(?\s*)(?\")(?.*)$/; + +export const commentPatterns: Record = { + abap: PatternAsterisk, + "actionscript-3": PatternDoubleSlash, + ada: PatternDoubleDash, + apache: PatternHash, + apex: PatternDoubleSlash, + apl: /^(?\s*)(?⍝)(?.*)$/, + applescript: PatternDoubleDash, + ara: PatternDoubleSlash, + astro: + /^(?\s*)(?(\.*)(?(\-\-\>|\*\/\}))$/, + awk: PatternHash, + ballerina: PatternDoubleSlash, + batch: PatternBatch, + berry: PatternHash, + bicep: PatternDoubleSlash, + blade: + /^(?\s*)(?\{\{\-\-)(?.*)(?\-\-\}\})$/, + cadence: PatternDoubleSlash, + clarity: PatternDoubleSemiColon, + clojure: PatternSemiColon, + cmake: PatternHash, + asm: PatternSemiColon, + c: PatternDoubleSlash, + cobol: PatternAsterisk, + coffee: PatternHash, + crystal: PatternHash, + css: PatternSlashAsterisk, + cue: PatternDoubleSlash, + d: PatternDoubleSlash, + dart: PatternDoubleSlash, + dax: PatternDoubleSlash, + dockerfile: PatternHash, + "dream-maker": PatternDoubleSlash, + elixir: PatternHash, + elm: PatternDoubleDash, + erb: /^(?\s*)(?\<%#)(?.*)(?%\>)$/, + erlang: PatternDoublePercentage, + fish: PatternHash, + "f#": PatternDoubleSlash, + "git-commit": PatternHash, + glsl: PatternDoubleSlash, + gnuplot: PatternHash, + graphql: PatternHash, + groovy: PatternDoubleSlash, + hack: PatternDoubleSlash, + haml: /^(?\s*)(?-#)(?.*)$/, + handlebars: PatternHBS, + haskell: PatternDoubleDash, + hcl: PatternHash, + hlsl: PatternDoubleSlash, + html: PatternXML, + cpp: PatternDoubleSlash, + "c#": PatternDoubleSlash, + go: PatternDoubleSlash, + java: PatternDoubleSlash, + javascript: PatternJSWithJSX, + imba: PatternHash, + ini: PatternSemiColon, + properties: PatternHash, + json5: PatternDoubleSlash, + jsonc: PatternDoubleSlash, + jsonnet: PatternDoubleSlash, + jssm: PatternDoubleSlash, + jsx: PatternJSWithJSX, + kotlin: PatternDoubleSlash, + kusto: PatternDoubleSlash, + kql: PatternDoubleSlash, + julia: PatternHash, + matlab: PatternPercentage, + latex: PatternPercentage, + less: /^(?\s*)(?(\/\*|\/\/))(?.*?)(?(\*\/))?$/, + php: PatternDoubleSlash, + lisp: PatternSemiColon, + logo: PatternSemiColon, + lua: PatternDoubleDash, + make: PatternHash, + makefile: PatternHash, + markdown: PatternXML, + marko: PatternXML, + mdx: /^(?\s*)(?\{\/\*)(?.*)(?\*\/\})$/, + mermaid: /^(?\s*)(?%%)(?.*)$/, + nginx: PatternHash, + nim: PatternHash, + nix: PatternHash, + "objective-c": PatternDoubleSlash, + "objective-cpp": PatternDoubleSlash, + ocaml: + /^(?\s*)(?\(\*)(?.*)(?\*\))$/, + pascal: + /^(?\s*)(?\{)(?.*)(?\})$/, + perl: PatternHash, + powerquery: PatternDoubleSlash, + powershell: PatternHash, + prisma: PatternDoubleSlash, + prolog: PatternPercentage, + proto: PatternDoubleSlash, + plsql: PatternDoubleDash, + purescript: PatternDoubleDash, + raku: PatternHash, + perl6: PatternHash, + reg: PatternSemiColon, + rel: PatternDoubleSlash, + riscv: PatternHash, + sas: PatternSlashAsterisk, + sass: PatternDoubleSlash, + scala: PatternDoubleSlash, + scheme: PatternSemiColon, + shaderlab: PatternDoubleSlash, + shader: PatternDoubleSlash, + scss: PatternSlashAsterisk, + shellscript: PatternHash, + bash: PatternHash, + sh: PatternHash, + shell: PatternHash, + zsh: PatternHash, + solidity: PatternDoubleSlash, + r: PatternHash, + sparql: PatternHash, + stata: PatternAsterisk, + stylus: PatternDoubleSlash, + svelte: PatternSFC, + python: PatternHash, + ruby: PatternHash, + rust: PatternDoubleSlash, + sql: PatternDoubleDash, + "system-verilog": PatternDoubleSlash, + tcl: PatternHash, + tex: PatternPercentage, + toml: PatternHash, + tsx: PatternJSWithJSX, + turtle: PatternHash, + swift: PatternDoubleSlash, + twig: /^(?\s*)(?\{#)(?.*)(?#\})$/, + typescript: PatternJSWithJSX, + vb: /^(?\s*)(?\')(?.*)$/, + verilog: PatternDoubleSlash, + vhdl: PatternDoubleDash, + viml: PatternDoubleQuote, + vimscript: PatternDoubleQuote, + vue: PatternSFC, + wasm: PatternDoubleSemiColon, + wenyan: /^(?\s*)(?注曰。)(?.*)$/, + wgsl: PatternDoubleSlash, + wolfram: + /^(?\s*)(?\(\*)(?.*)(?\*\))$/, + xml: PatternXML, + xsl: PatternXML, + yaml: PatternHash, + zenscript: PatternDoubleSlash, + jison: PatternDoubleSlash, + "ssh-config": PatternHash, +} satisfies Record; diff --git a/packages/shaku-code-annotate-shiki-transformer/src/defaultCode.ts b/packages/shaku-code-annotate-shiki-transformer/src/defaultCode.ts index 8b3184b..98174d5 100644 --- a/packages/shaku-code-annotate-shiki-transformer/src/defaultCode.ts +++ b/packages/shaku-code-annotate-shiki-transformer/src/defaultCode.ts @@ -1248,7 +1248,6 @@ description = "An example of using comments in TOML" const MyComponent: React.FC = () => { return ( {/* @highlight */} -
{/* This is a comment inside JSX */}

Hello, World!

@@ -1531,10 +1530,10 @@ export const supportedLangs = [ "astro", "awk", "ballerina", - "bat", + // "bat", "batch", "berry", - "be", + // "be", // "bibtex", "bicep", "blade", @@ -1546,31 +1545,31 @@ export const supportedLangs = [ // "clj", "cmake", "cobol", - "codeql", // nothing to test + // "codeql", // nothing to test // "ql", "coffee", "cpp", "crystal", "c#", - "cs", + // "cs", "css", "cue", "d", "dart", "dax", - "diff", // nothing to test - "docker", + // "diff", // nothing to test + // "docker", "dockerfile", "dream-maker", "elixir", "elm", "erb", "erlang", - "erl", + // "erl", "fish", - "fsharp", + // "fsharp", "f#", - "fs", + // "fs", // "gdresource", // "gdscript", // "gdshader", @@ -1585,9 +1584,9 @@ export const supportedLangs = [ "hack", "haml", "handlebars", - "hbs", + // "hbs", "haskell", - "hs", + // "hs", "hcl", "hlsl", "html", @@ -1597,15 +1596,15 @@ export const supportedLangs = [ "properties", "java", "javascript", - "js", + // "js", // "jinja-html", "jison", - "json", + // "json", "json5", "jsonc", "jsonnet", "jssm", - "fsl", + // "fsl", "jsx", "julia", "kotlin", @@ -1613,14 +1612,14 @@ export const supportedLangs = [ "kql", "latex", "less", - "liquid", + // "liquid", "lisp", "logo", "lua", "make", "makefile", "markdown", - "md", + // "md", "marko", "matlab", "mdx", @@ -1629,7 +1628,7 @@ export const supportedLangs = [ "nim", "nix", "objective-c", - "objc", + // "objc", "objective-cpp", "ocaml", "pascal", @@ -1639,7 +1638,7 @@ export const supportedLangs = [ // "postcss", "powerquery", "powershell", - "ps", + // "ps", // "ps1", "prisma", "prolog", @@ -1652,15 +1651,15 @@ export const supportedLangs = [ "r", "raku", "perl6", - "razor", + // "razor", "reg", "rel", "riscv", - "rst", + // "rst", "ruby", - "rb", + // "rb", "rust", - "rs", + // "rs", "sas", "sass", "scala", @@ -1674,14 +1673,14 @@ export const supportedLangs = [ "sh", "shell", "zsh", - "smalltalk", + // "smalltalk", "solidity", "sparql", "sql", "ssh-config", "stata", "stylus", - "styl", + // "styl", "svelte", "swift", "system-verilog", @@ -1693,7 +1692,7 @@ export const supportedLangs = [ "turtle", "twig", "typescript", - "ts", + // "ts", // "v", "vb", // "cmd", @@ -1712,6 +1711,6 @@ export const supportedLangs = [ "xml", "xsl", "yaml", - "yml", + // "yml", "zenscript", -]; +] as const; diff --git a/packages/shaku-code-annotate-shiki-transformer/src/index.ts b/packages/shaku-code-annotate-shiki-transformer/src/index.ts index 68d6086..8ec9983 100644 --- a/packages/shaku-code-annotate-shiki-transformer/src/index.ts +++ b/packages/shaku-code-annotate-shiki-transformer/src/index.ts @@ -5,6 +5,7 @@ import { renderComponentHast, } from "shaku-code-annotate-core"; import { Element, ElementContent } from "hast"; +import { commentPatterns } from "./commentPatterns"; declare module "shiki" { export interface ShikiTransformerContextMeta { @@ -154,7 +155,7 @@ const shakuCodeAnnotateShikiTransformer: ( config: { offset: minOffset, arrowOffset: directiveOffset - minOffset, - contents: contents.join(""), + contents: contents.join(useDangerousRawHtml ? "" : "\n"), }, }, { @@ -582,134 +583,41 @@ const shakuCodeAnnotateShikiTransformer: ( export default shakuCodeAnnotateShikiTransformer; /** - * different kinds of comments have different interpretations - * Below are some common examples, these are not exhaustive - * I'm not sure if there are other cases for different languages - * - * "// aaa" => [{content: "// aaa", explanation: [{content: '//'}, {content: ' aaa'}]}] - * "/* aaa *\/" => [{content: "/* aaa *\/", explanation: [{content: '/*'}, {content: ' aaa '}, content: '*\/'}]}] - * " // aaa" => [{content: " "}, {content: "// aaa", explanation: [{content: '//'}, {content: ' aaa'}]}] - * " /* aaa" => [{content: " "}, {content: "/* aaa", explanation: [{content: '/*'}, {content: ' aaa'}]}] - * " aaa" => [{content: " aaa"}] - * " *\/" => [{content: " *\/", explanation: [{content: ' '}, {content: '*\/'}]}] - * - * one exception is jsx, there is no comment line but following form which should be treated as one - * " { /* aaa *\/ }" => [{content: " { "}, {content: "/* a *\/"}, {content: " }"}] - * - * So the idea is: - * 1. if all tokens has explanation of comment or "whitespace", then it is comment line - * 2. "whitespace" means token is whitespace or has `punctuation.section.embedded` and `meta.jsx.children`. - * 3. if the first token is "whitespace", then it used to calculate the offset - * 4. the first meaningful token has the comment body - * - find the first explanation that is not `punctuation.definition`. + * using token explanation is not performant and flaky + * now here just use boring regexp */ function parseComment( - line: ThemedToken[], + text: string, lang?: null | string ): null | { offset: number; body: string; } { - if (line.length === 0) return null; - // comments start from the beginning - const isCommentLine = line.every( - (token) => - shouldBeTreatedAsWhitespace(token, lang) || - token.explanation?.some((exp) => - exp.scopes.some((scope) => { - if (lang === "astro") { - return ( - scope.scopeName.startsWith("comment") && - scope.scopeName !== "comment" - ); - } - return scope.scopeName.startsWith("comment"); - }) - ) - ); - if (!isCommentLine) return null; - - const shouldTreatFirstTokenOffset = shouldBeTreatedAsWhitespace( - line[0], - lang - ); - let offset = shouldTreatFirstTokenOffset ? line[0].content.length : 0; - - let body = ""; - - // special case for some languages that one comment has multiple tokens - // TODO: maybe we should give up the "clever" approach because it is not solid - // rather we can just try to trim for each lang? - if ( - lang != null && - [ - "ada", - "berry", - "elm", - "haml", - "handlebars", - "hlsl", - "jssm", - "kotlin", - "nix", - "ocaml", - "prisma", - "proto", - "sas", - "sass", - "shaderlab", - "shader", - "solidity", - "viml", - "vimscript", - "wenyan", - ].includes(lang) - ) { - body = line - .slice(shouldTreatFirstTokenOffset ? 1 : 0) - .map((token) => token.content) - .join(""); - } else { - const tokenWithBody = shouldTreatFirstTokenOffset ? line[1] : line[0]; + if (lang == null) { + return null; + } + const reg = commentPatterns[lang]; - if (tokenWithBody != null) { - const explanations = tokenWithBody.explanation ?? []; - // there is a chance that the leading spaces are not in the first explanations - if (explanations.length > 0) { - const lengthOfExplanations = explanations.reduce( - (acc, explanation) => acc + explanation.content.length, - 0 - ); - offset += tokenWithBody.content.length - lengthOfExplanations; - } + if (reg == null) { + return null; + } - for (let i = 0; i < explanations.length; i++) { - const explanation = explanations[i]; - // find the first explanation that is not element tag - if ( - explanation.scopes.every( - (scope) => - !scope.scopeName.startsWith("punctuation.definition") && - !isWhitespace(explanation.content) - ) - ) { - body = explanation.content; - break; - } - // for other none tokens, treat them as offset - offset += explanation.content.length; - } + const matches = text.match(reg); + if (matches) { + const body = matches.groups?.body; + const leadingSpaces = matches.groups?.leadingSpaces; + const markerStart = matches.groups?.markerStart; + if (body == null || leadingSpaces == null || markerStart == null) { + return null; } - } - // for some languages, we are not able to extract body from above logic - // so we have to trim manually - const { trimmedBody, offset: extraOffset } = trimCommentBody(body, lang); + return { + offset: leadingSpaces.length + markerStart.length, + body, + }; + } - return { - offset: offset + extraOffset, - body: trimmedBody, - }; + return null; } function parseLines( @@ -719,7 +627,7 @@ function parseLines( ) { return lines.map((line) => { if (isShakuSyntaxEnabled) { - const parsedComment = parseComment(line, lang); + const parsedComment = parseComment(getTextOfElement(line), lang); if (parsedComment != null) { const { body, offset } = parsedComment; const shakuLine = parseLine(body); @@ -750,245 +658,12 @@ function parseLines( }); } -type ParsedLine = ReturnType[number]; - function hasShakuDirectiveFocus(lines: ReturnType) { return lines.some( (line) => line.type === "shaku" && line.line.type === "DirectiveFocus" ); } -function isWhitespace(str: string) { - return /^\s+$/.test(str); -} - -function shouldBeTreatedAsWhitespace(token: ThemedToken, lang?: string | null) { - if (isWhitespace(token.content)) return true; - - if ( - !["javascript", "jsx", "tsx", "astro", "mdx", "batch"].includes(lang ?? "") - ) { - return false; - } - if ( - token.explanation?.every((explanation) => { - return ( - isWhitespace(explanation.content) || - (explanation.scopes.some((scope) => - scope.scopeName.startsWith("meta.jsx.children") - ) && - explanation.scopes.some( - (scope) => - scope.scopeName.startsWith( - "punctuation.section.embedded.begin" - ) || - scope.scopeName.startsWith("punctuation.section.embedded.end") - )) || - (explanation.scopes.some((scope) => - scope.scopeName.startsWith("source.astro") - ) && - explanation.scopes.some( - (scope) => - scope.scopeName.startsWith( - "punctuation.section.embedded.begin.astro" - ) || - scope.scopeName.startsWith( - "punctuation.section.embedded.end.astro" - ) - )) || - explanation.scopes.some( - (scope) => scope.scopeName === "keyword.command.rem.batchfile" - ) || - (explanation.scopes.some((scope) => scope.scopeName === "source.mdx") && - explanation.scopes.some((scope) => - scope.scopeName.startsWith("string.other.begin.expression.mdx") - )) - ); - }) - ) { - return true; - } - return false; -} - -const commentMarkers: Record = { - abap: { - head: /^\s*\*/, - }, - ada: { - head: /^\s*\-\-/, - }, - apache: { - head: /^\s*#/, - }, - "actionscript-3": { - head: /^\s*\/\//, - }, - asm: { - head: /^\s*;/, - }, - awk: { - head: /^\s*#/, - }, - ballerina: { - head: /^\s*\/\//, - }, - berry: { - head: /^\s*#/, - }, - bicep: { - head: /^\s*\/\//, - }, - clarity: { - head: /^\s*;;/, - }, - cmake: { - head: /^\s*#/, - }, - cobol: { - head: /^\s*\*/, - }, - d: { - head: /^\s*\/\//, - }, - elm: { - head: /^\s*--/, - }, - dart: { - head: /^\s*\/\//, - }, - erlang: { - head: /^\s*%/, - }, - fsharp: { - head: /^\s*\/\//, - }, - "f#": { - head: /^\s*\/\//, - }, - "git-commit": { - head: /^\s*#/, - }, - graphql: { - head: /^\s*#/, - }, - haml: { - head: /^\s*-#/, - }, - handlebars: { - head: /^\s*\{\{!--/, - tail: /--\}\}\s*$/, - }, - hlsl: { - head: /^\s*\/\//, - }, - json5: { - head: /^\s*\/\//, - }, - jsonnet: { - head: /^\s*\/\//, - }, - jssm: { - head: /^\s*\/\//, - }, - rust: { - head: /^\s*\/\//, - }, - kotlin: { - head: /^\s*\/\//, - }, - kusto: { - head: /^\s*\/\//, - }, - kql: { - head: /^\s*\/\//, - }, - mermaid: { - head: /^\s*%%/, - }, - nginx: { - head: /^\s*#/, - }, - nix: { - head: /^\s*#/, - }, - ocaml: { - head: /^\s*\(\*/, - tail: /\s*\*\)$/, - }, - plsql: { - head: /^\s*--/, - }, - powerquery: { - head: /^\s*\/\//, - }, - prisma: { - head: /^\s*\/\//, - }, - proto: { - head: /^\s*\/\//, - }, - sas: { - head: /^\s*\/\*/, - tail: /\s*\*\/$/, - }, - sass: { - head: /^\s*\/\//, - }, - shaderlab: { - head: /^\s*\/\//, - }, - shader: { - head: /^\s*\/\//, - }, - solidity: { - head: /^\s*\/\//, - }, - sparql: { - head: /^\s*#/, - }, - turtle: { - head: /^\s*#/, - }, - vhdl: { - head: /^\s*--/, - }, - viml: { - head: /^\s*"/, - }, - vimscript: { - head: /^\s*"/, - }, - wenyan: { - head: /^\s*注曰。/, - }, - wgsl: { - head: /^\s*\/\//, - }, - zenscript: { - head: /^\s*\/\//, - }, -}; - -function trimCommentBody(body: string, lang?: string | null) { - let trimmedBody = body; - let offset = 0; - if (lang != null && lang in commentMarkers) { - const { head, tail } = commentMarkers[lang]; - if (head != null) { - trimmedBody = trimmedBody.replace(head, ""); - offset = body.length - trimmedBody.length; - } - if (tail != null) { - trimmedBody = trimmedBody.replace(tail, ""); - } - } - return { - trimmedBody, - offset, - }; -} - function assertsNever(data: never) { throw new Error("expected never but got: " + data); } @@ -1002,6 +677,12 @@ function getLeadingSpaceCount(str: string) { return 0; } +function getTextOfElement(line: ThemedToken[]): string { + return line.reduce((acc, token) => { + return acc + token.content; + }, ""); +} + function getTextSizeOfElement(token: Element): number { return token.children.reduce((acc, child) => { if (child.type === "text") { diff --git a/packages/shaku-code-annotate-shiki/src/defaultCode.ts b/packages/shaku-code-annotate-shiki/src/defaultCode.ts index 8b3184b..cca6cd9 100644 --- a/packages/shaku-code-annotate-shiki/src/defaultCode.ts +++ b/packages/shaku-code-annotate-shiki/src/defaultCode.ts @@ -1531,10 +1531,10 @@ export const supportedLangs = [ "astro", "awk", "ballerina", - "bat", + // "bat", "batch", "berry", - "be", + // "be", // "bibtex", "bicep", "blade", @@ -1546,31 +1546,31 @@ export const supportedLangs = [ // "clj", "cmake", "cobol", - "codeql", // nothing to test + // "codeql", // nothing to test // "ql", "coffee", "cpp", "crystal", "c#", - "cs", + // "cs", "css", "cue", "d", "dart", "dax", - "diff", // nothing to test - "docker", + // "diff", // nothing to test + // "docker", "dockerfile", "dream-maker", "elixir", "elm", "erb", "erlang", - "erl", + // "erl", "fish", - "fsharp", + // "fsharp", "f#", - "fs", + // "fs", // "gdresource", // "gdscript", // "gdshader", @@ -1585,9 +1585,9 @@ export const supportedLangs = [ "hack", "haml", "handlebars", - "hbs", + // "hbs", "haskell", - "hs", + // "hs", "hcl", "hlsl", "html", @@ -1597,15 +1597,15 @@ export const supportedLangs = [ "properties", "java", "javascript", - "js", + // "js", // "jinja-html", "jison", - "json", + // "json", "json5", "jsonc", "jsonnet", "jssm", - "fsl", + // "fsl", "jsx", "julia", "kotlin", @@ -1613,14 +1613,14 @@ export const supportedLangs = [ "kql", "latex", "less", - "liquid", + // "liquid", "lisp", "logo", "lua", "make", "makefile", "markdown", - "md", + // "md", "marko", "matlab", "mdx", @@ -1629,7 +1629,7 @@ export const supportedLangs = [ "nim", "nix", "objective-c", - "objc", + // "objc", "objective-cpp", "ocaml", "pascal", @@ -1639,7 +1639,7 @@ export const supportedLangs = [ // "postcss", "powerquery", "powershell", - "ps", + // "ps", // "ps1", "prisma", "prolog", @@ -1652,15 +1652,15 @@ export const supportedLangs = [ "r", "raku", "perl6", - "razor", + // "razor", "reg", "rel", "riscv", - "rst", + // "rst", "ruby", - "rb", + // "rb", "rust", - "rs", + // "rs", "sas", "sass", "scala", @@ -1674,14 +1674,14 @@ export const supportedLangs = [ "sh", "shell", "zsh", - "smalltalk", + // "smalltalk", "solidity", "sparql", "sql", "ssh-config", "stata", "stylus", - "styl", + // "styl", "svelte", "swift", "system-verilog", @@ -1693,7 +1693,7 @@ export const supportedLangs = [ "turtle", "twig", "typescript", - "ts", + // "ts", // "v", "vb", // "cmd", @@ -1712,6 +1712,6 @@ export const supportedLangs = [ "xml", "xsl", "yaml", - "yml", + // "yml", "zenscript", ];