🎨 improve(#23): support new parser option 'formatUrl' to resolve urls…

… in the ast (#24) * ⬆️ chore: upgrade devDependencies * 🎨 improve: support new parser option 'formatUrl' to resolve urls in the ast * 🔧 chore: fix tokenizer template
yozorajs · Jan 26, 2024 · a2c5ac8 · a2c5ac8
1 parent f8ed6c9
commit a2c5ac8
Show file tree

Hide file tree

Showing 22 changed files with 1,915 additions and 1,400 deletions.
diff --git a/.eslintrc b/.eslintrc
@@ -6,7 +6,7 @@
   },
   "overrides": [
     {
-      "files": ["**/*.js"],
+      "files": ["**/*.js", "**/*.mjs"],
       "parser": "@babel/eslint-parser",
       "parserOptions": {
         "ecmaVersion": 2018,

diff --git a/jest.config.mjs b/jest.config.mjs
@@ -31,6 +31,5 @@ export default async function () {
       },
     },
     extensionsToTreatAsEsm: ['.ts', '.mts'],
-    prettierPath: require.resolve('prettier-2'),
   }
 }
diff --git a/package.json b/package.json
@@ -48,34 +48,33 @@
     "tokenizers/*"
   ],
   "devDependencies": {
-    "@babel/core": "^7.22.20",
-    "@babel/eslint-parser": "^7.22.15",
-    "@guanghechen/conventional-changelog": "^5.0.1",
-    "@guanghechen/helper-jest": "^5.0.7",
-    "@guanghechen/helper-string": "^5.0.7",
-    "@guanghechen/jest-config": "^5.0.7",
-    "@guanghechen/rollup-config": "^5.3.0",
-    "@guanghechen/script-doc-link": "^5.0.9",
-    "@types/jest": "29.5.5",
-    "@types/node": "^18.17.17",
+    "@babel/core": "^7.23.9",
+    "@babel/eslint-parser": "^7.23.9",
+    "@guanghechen/conventional-changelog": "^6.0.0-alpha.0",
+    "@guanghechen/helper-jest": "^6.0.0-alpha.2",
+    "@guanghechen/helper-string": "^6.0.0-alpha.2",
+    "@guanghechen/jest-config": "^6.0.0-alpha.2",
+    "@guanghechen/rollup-config": "^6.0.0-alpha.3",
+    "@guanghechen/script-doc-link": "^6.0.0-alpha.11",
+    "@types/jest": "29.5.11",
+    "@types/node": "^18.19.10",
     "cross-env": "^7.0.3",
-    "eslint": "8.49.0",
+    "eslint": "8.56.0",
     "handlebars": "4.7.8",
-    "husky": "8.0.3",
+    "husky": "9.0.6",
     "is-ci": "3.0.1",
     "jest": "^29.7.0",
-    "lerna": "7.3.0",
-    "lint-staged": "14.0.1",
+    "lerna": "8.0.2",
+    "lint-staged": "15.2.0",
     "npm-run-all": "4.1.5",
     "pinst": "3.0.0",
-    "prettier": "3.0.3",
-    "prettier-2": "npm:prettier@^2",
-    "rimraf": "^5.0.1",
-    "rollup": "^3.29.2",
-    "ts-jest": "29.1.1",
-    "ts-node": "10.9.1",
+    "prettier": "3.2.4",
+    "rimraf": "^5.0.5",
+    "rollup": "^4.9.6",
+    "ts-jest": "29.1.2",
+    "ts-node": "10.9.2",
     "tsconfig-paths": "4.2.0",
-    "typescript": "5.2.2"
+    "typescript": "5.3.3"
   },
   "lint-staged": {
     "*.{js,jsx,ts,tsx}": [

diff --git a/packages/core-parser/src/parser.ts b/packages/core-parser/src/parser.ts
@@ -7,7 +7,7 @@ import type {
   IInlineTokenizer,
   ITokenizer,
 } from '@yozora/core-tokenizer'
-import { TokenizerType } from '@yozora/core-tokenizer'
+import { TokenizerType, encodeLinkDestination } from '@yozora/core-tokenizer'
 import { createProcessor } from './processor'
 import type { IParseOptions, IParser } from './types'
 import { createPhrasingLineGenerator } from './util/phrasing-line'
@@ -115,12 +115,13 @@ export class DefaultParser implements IParser {
       presetDefinitions: [],
       presetFootnoteDefinitions: [],
       shouldReservePosition: false,
+      formatUrl: encodeLinkDestination,
       ...options,
     }
   }
 
   public parse(contents: Iterable<string> | string, options: IParseOptions = {}): Root {
-    const { shouldReservePosition, presetDefinitions, presetFootnoteDefinitions } = {
+    const { shouldReservePosition, presetDefinitions, presetFootnoteDefinitions, formatUrl } = {
       ...this.defaultParseOptions,
       ...options,
     }
@@ -138,6 +139,7 @@ export class DefaultParser implements IParser {
       shouldReservePosition,
       presetDefinitions,
       presetFootnoteDefinitions,
+      formatUrl,
     })
     const root: Root = processor.process(linesIterator)
     return root

diff --git a/packages/core-parser/src/processor/index.ts b/packages/core-parser/src/processor/index.ts
@@ -29,6 +29,7 @@ export function createProcessor(options: IProcessorOptions): IProcessor {
     shouldReservePosition,
     presetDefinitions,
     presetFootnoteDefinitions,
+    formatUrl,
   } = options
 
   let isIdentifierRegisterAvailable = false
@@ -51,6 +52,7 @@ export function createProcessor(options: IProcessorOptions): IProcessor {
     },
     parseBlockApi: {
       shouldReservePosition,
+      formatUrl,
       processInlines,
       parseBlockTokens,
     },
@@ -64,14 +66,15 @@ export function createProcessor(options: IProcessorOptions): IProcessor {
     },
     parseInlineApi: {
       shouldReservePosition,
-      getNodePoints: () => _nodePoints,
-      hasDefinition: identifier => definitionIdentifierSet.has(identifier),
-      hasFootnoteDefinition: identifier => footnoteIdentifierSet.has(identifier),
-      parseInlineTokens,
       calcPosition: token => ({
         start: calcStartPoint(_nodePoints, token.startIndex),
         end: calcEndPoint(_nodePoints, token.endIndex - 1),
       }),
+      formatUrl,
+      getNodePoints: () => _nodePoints,
+      hasDefinition: identifier => definitionIdentifierSet.has(identifier),
+      hasFootnoteDefinition: identifier => footnoteIdentifierSet.has(identifier),
+      parseInlineTokens,
     },
   })
 

diff --git a/packages/core-parser/src/processor/types.ts b/packages/core-parser/src/processor/types.ts
@@ -24,6 +24,7 @@ export interface IProcessorOptions {
   readonly shouldReservePosition: boolean
   readonly presetDefinitions: Association[]
   readonly presetFootnoteDefinitions: Association[]
+  readonly formatUrl: (url: string) => string
 }
 
 /**

diff --git a/packages/core-parser/src/types.ts b/packages/core-parser/src/types.ts
@@ -20,6 +20,13 @@ export interface IParseOptions {
    * Preset footnote definition meta data list.
    */
   readonly presetFootnoteDefinitions?: Association[]
+
+  /**
+   * Format url.
+   * @param url
+   * @returns
+   */
+  readonly formatUrl?: (url: string) => string
 }
 
 /**

diff --git a/packages/core-tokenizer/src/constant.ts b/packages/core-tokenizer/src/constant.ts
@@ -1,3 +1,4 @@
+/* eslint-disable @typescript-eslint/no-duplicate-enum-values */
 export enum TokenizerType {
   // Block tokenizer
   BLOCK = 'block',

diff --git a/packages/core-tokenizer/src/types/parse-block/api.ts b/packages/core-tokenizer/src/types/parse-block/api.ts
@@ -10,6 +10,11 @@ export interface IParseBlockPhaseApi {
    * Whether it is necessary to reserve the position in the Node produced.
    */
   readonly shouldReservePosition: boolean
+  /**
+   * Format url.
+   * @param url
+   */
+  formatUrl(url: string): string
   /**
    * Process node points into inline nodes.
    * @param nodePoints

diff --git a/packages/core-tokenizer/src/types/parse-inline/api.ts b/packages/core-tokenizer/src/types/parse-inline/api.ts
@@ -10,15 +10,20 @@ export interface IParseInlinePhaseApi {
    * Whether it is necessary to reserve the position in the Node produced.
    */
   readonly shouldReservePosition: boolean
-  /**
-   * Get the node points.
-   */
-  getNodePoints(): ReadonlyArray<INodePoint>
   /**
    * Calculate position of token.
    * @param interval
    */
   calcPosition(interval: Readonly<INodeInterval>): Position
+  /**
+   * Format url.
+   * @param url
+   */
+  formatUrl(url: string): string
+  /**
+   * Get the node points.
+   */
+  getNodePoints(): ReadonlyArray<INodePoint>
   /**
    * Check if there is exists a definition with the given identifier.
    * @param identifier

diff --git a/packages/invariant/package.json b/packages/invariant/package.json
@@ -42,7 +42,7 @@
   "devDependencies": {
     "cross-env": "^7.0.3",
     "jest": "^29.7.0",
-    "rimraf": "^5.0.1",
-    "rollup": "^3.29.2"
+    "rimraf": "^5.0.5",
+    "rollup": "^4.9.6"
   }
 }
diff --git a/packages/parser/package.json b/packages/parser/package.json
@@ -84,7 +84,7 @@
   "devDependencies": {
     "cross-env": "^7.0.3",
     "jest": "^29.7.0",
-    "rimraf": "^5.0.1",
-    "rollup": "^3.29.2"
+    "rimraf": "^5.0.5",
+    "rollup": "^4.9.6"
   }
 }
diff --git a/scaffolds/eslint-config/package.json b/scaffolds/eslint-config/package.json
@@ -35,11 +35,11 @@
     "README.md"
   ],
   "dependencies": {
-    "@guanghechen/eslint-config": "^5.0.2",
-    "@guanghechen/eslint-config-ts": "^5.0.7",
-    "eslint-config-prettier": "^9.0.0",
-    "eslint-plugin-jest": "^27.4.0",
-    "eslint-plugin-prettier": "^5.0.0"
+    "@guanghechen/eslint-config": "^6.0.0-alpha.1",
+    "@guanghechen/eslint-config-ts": "^6.0.0-alpha.5",
+    "eslint-config-prettier": "^9.1.0",
+    "eslint-plugin-jest": "^27.6.3",
+    "eslint-plugin-prettier": "^5.1.3"
   },
   "peerDependencies": {
     "eslint": ">=8.0.0",

diff --git a/scaffolds/jest-for-tokenizer/package.json b/scaffolds/jest-for-tokenizer/package.json
@@ -41,7 +41,7 @@
     "@yozora/ast": "^2.3.0",
     "@yozora/ast-util": "^2.3.0",
     "@yozora/invariant": "^2.3.0",
-    "globby": "^13.2.2",
+    "globby": "^14.0.0",
     "jest": "^29.7.0"
   },
   "devDependencies": {

diff --git a/scaffolds/template-tokenizer/cli.mjs b/scaffolds/template-tokenizer/cli.mjs
@@ -2,7 +2,10 @@
 
 import { launch } from '@guanghechen/plop-helper'
 import path from 'node:path'
+import url from 'node:url'
+
+const __dirname = path.dirname(url.fileURLToPath(import.meta.url))
 
 launch(process.argv, args => ({
-  configPath: args.plopfile || path.join(__dirname, 'index.js'),
+  configPath: args.plopfile || path.join(__dirname, 'index.mjs'),
 }))
diff --git a/scaffolds/template-tokenizer/index.mjs b/scaffolds/template-tokenizer/index.mjs
@@ -7,23 +7,23 @@ import {
 import { toKebabCase, toTrim } from '@guanghechen/helper-string'
 import path from 'node:path'
 import url from 'node:url'
-import manifest from './package.json' assert { type: 'json' }
+import manifest from './package.json'
 
 const __dirname = path.dirname(url.fileURLToPath(import.meta.url))
 
 const transformers = {
   tokenizerName: toTrim,
 }
 
-export default function (plop) {
-  const preAnswers = resolveNpmPackagePreAnswers({
+export default async function (plop) {
+  const preAnswers = await resolveNpmPackagePreAnswers({
     isMonorepo: convertToBoolean(process.env.DEBUG_IS_MONOREPO),
   })
-  const defaultAnswers = { packageVersion: manifest.version }
   const { cwd, isMonorepo } = preAnswers
   const tokenizerPackageNameRegex = /^(?:[^\\/]+\/)tokenizer-([\w-]+)$/
 
-  const prompts = createNpmPackagePrompts(preAnswers, defaultAnswers)
+  const defaultAnswers = { packageVersion: manifest.version }
+  const prompts = await createNpmPackagePrompts(preAnswers, defaultAnswers)
   prompts.splice(
     1,
     0,

diff --git a/scaffolds/template-tokenizer/package.json b/scaffolds/template-tokenizer/package.json
@@ -34,9 +34,9 @@
     "test:update": "yarn test -u"
   },
   "dependencies": {
-    "@guanghechen/helper-option": "^5.0.7",
-    "@guanghechen/helper-plop": "^5.0.8",
-    "@guanghechen/helper-string": "^5.0.7"
+    "@guanghechen/helper-option": "^6.0.0-alpha.1",
+    "@guanghechen/helper-plop": "^6.0.0-alpha.3",
+    "@guanghechen/helper-string": "^6.0.0-alpha.2"
   },
   "devDependencies": {
     "cross-env": "^7.0.3",

diff --git a/tokenizers/autolink/src/parse.ts b/tokenizers/autolink/src/parse.ts
@@ -3,7 +3,6 @@ import { LinkType } from '@yozora/ast'
 import type { INodePoint } from '@yozora/character'
 import { calcStringFromNodePoints } from '@yozora/character'
 import type { IParseInlineHookCreator } from '@yozora/core-tokenizer'
-import { encodeLinkDestination } from '@yozora/core-tokenizer'
 import type { INode, IThis, IToken, T } from './types'
 
 export const parse: IParseInlineHookCreator<T, IToken, INode, IThis> = function (api) {
@@ -20,7 +19,7 @@ export const parse: IParseInlineHookCreator<T, IToken, INode, IThis> = function
           url = 'mailto:' + url
         }
 
-        const encodedUrl = encodeLinkDestination(url)
+        const encodedUrl: string = api.formatUrl(url)
         const children: Node[] = api.parseInlineTokens(token.children)
         const node: INode = api.shouldReservePosition
           ? { type: LinkType, position: api.calcPosition(token), url: encodedUrl, children }

diff --git a/tokenizers/definition/src/parse.ts b/tokenizers/definition/src/parse.ts
@@ -2,7 +2,6 @@ import { DefinitionType } from '@yozora/ast'
 import type { INodePoint } from '@yozora/character'
 import { AsciiCodePoint, calcEscapedStringFromNodePoints } from '@yozora/character'
 import type { IParseBlockHookCreator } from '@yozora/core-tokenizer'
-import { encodeLinkDestination } from '@yozora/core-tokenizer'
 import type { INode, IThis, IToken, T } from './types'
 
 export const parse: IParseBlockHookCreator<T, IToken, INode, IThis> = function (api) {
@@ -26,7 +25,7 @@ export const parse: IParseBlockHookCreator<T, IToken, INode, IThis> = function (
                 true,
               )
             : calcEscapedStringFromNodePoints(destinationPoints, 0, destinationPoints.length, true)
-        const url = encodeLinkDestination(destination)
+        const url = api.formatUrl(destination)
 
         /**
          * Resolve link title

diff --git a/tokenizers/image/src/parse.ts b/tokenizers/image/src/parse.ts
@@ -3,7 +3,6 @@ import { ImageType } from '@yozora/ast'
 import type { INodePoint } from '@yozora/character'
 import { AsciiCodePoint, calcEscapedStringFromNodePoints } from '@yozora/character'
 import type { IParseInlineHookCreator } from '@yozora/core-tokenizer'
-import { encodeLinkDestination } from '@yozora/core-tokenizer'
 import type { INode, IThis, IToken, T } from './types'
 import { calcImageAlt } from './util'
 
@@ -27,7 +26,7 @@ export const parse: IParseInlineHookCreator<T, IToken, INode, IThis> = function
             endIndex,
             true,
           )
-          url = encodeLinkDestination(destination)
+          url = api.formatUrl(destination)
         }
 
         // calc alt

diff --git a/tokenizers/link/src/parse.ts b/tokenizers/link/src/parse.ts
@@ -3,7 +3,6 @@ import { LinkType } from '@yozora/ast'
 import type { INodePoint } from '@yozora/character'
 import { AsciiCodePoint, calcEscapedStringFromNodePoints } from '@yozora/character'
 import type { IParseInlineHookCreator } from '@yozora/core-tokenizer'
-import { encodeLinkDestination } from '@yozora/core-tokenizer'
 import type { INode, IThis, IToken, T } from './types'
 
 export const parse: IParseInlineHookCreator<T, IToken, INode, IThis> = function (api) {
@@ -26,7 +25,7 @@ export const parse: IParseInlineHookCreator<T, IToken, INode, IThis> = function
             endIndex,
             true,
           )
-          url = encodeLinkDestination(destination)
+          url = api.formatUrl(destination)
         }
 
         // calc title