feat: perf scanner logic

.github/workflows/e2e.yaml

@@ -7,7 +7,7 @@ jobs:
     strategy:
       matrix:
         version: [16, 18]
-        os: [ubuntu-latest, windows-latest]
+        os: [ubuntu-latest]
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4
@@ -28,7 +28,7 @@ jobs:
     strategy:
       matrix:
         version: [18,20]
-        os: [ubuntu-latest, windows-latest]
+        os: [ubuntu-latest]
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4

.yarnrc.yml

@@ -1,2 +1,11 @@
-yarnPath: .yarn/releases/yarn-3.6.3.cjs
-nodeLinker: pnpm
+compressionLevel: mixed
+
+enableGlobalCache: false
+
+httpProxy: "http://localhost:7890"
+
+httpsProxy: "https://localhost:7890"
+
+nodeLinker: pnpm
+
+yarnPath: .yarn/releases/yarn-4.1.1.cjs

__tests__/inject.spec.ts

@@ -2,9 +2,9 @@ import test from 'ava'
 import { len } from '../src/shared'
 import { createInjectScript } from '../src/inject'
 import { jsdelivr } from '../src/resolver/jsdelivr'
-import type { TrackModule } from '../src'
+import type { IModule } from '../src'
 
-interface MockIIFEMdoule extends TrackModule {
+interface MockIIFEMdoule extends IModule {
   relativeModule: string
   version: string
   bindings: Set<string>

__tests__/scanner.spec.ts

@@ -10,7 +10,7 @@ test('scanner dependencies', (t) => {
 test('scanner failed', (t) => {
   const scanner = createScanner(['vue', 'react'])
   scanner.scanAllDependencies()
-  t.is(scanner.failedModules.has('react'), true)
+  t.is(scanner.failedMessages[0], '[scanner error]: can\'t find any iife file from package \'react\',please check it.')
   t.is(scanner.dependencies.has('vue'), true)
 })
 

package.json

@@ -77,7 +77,7 @@
     "ava": "^5.2.0",
     "c8": "^7.12.0",
     "eslint": "^8.51.0",
-    "eslint-config-kagura": "^2.0.1",
+    "eslint-config-kagura": "^2.1.1",
     "playwright": "^1.35.1",
     "react": "^18.2.0",
     "tsup": "^7.1.0",
@@ -89,10 +89,9 @@
   "dependencies": {
     "@babel/core": "^7.22.5",
     "@rollup/pluginutils": "^5.0.2",
-    "@xn-sakina/rml-wasm": "^2.1.1",
     "debug": "^4.3.4",
-    "magic-string": "^0.30.5",
-    "rs-module-lexer": "^2.1.1"
+    "es-module-lexer": "^1.4.1",
+    "magic-string": "^0.30.5"
   },
   "ava": {
     "files": [
@@ -162,5 +161,5 @@
     "which-typed-array": "npm:@nolyfill/which-typed-array@latest",
     "tsup@^7.1.0": "patch:tsup@npm%3A7.1.0#./.yarn/patches/tsup-npm-7.1.0-3fe66cd74a.patch"
   },
-  "packageManager": "yarn@3.6.3"
+  "packageManager": "yarn@4.1.1"
 }

src/index.ts

@@ -1,6 +1,8 @@
 import { createFilter } from '@rollup/pluginutils'
 import type { Plugin } from 'vite'
+import { searchForWorkspaceRoot } from 'vite'
 import _debug from 'debug'
+import { parse } from 'es-module-lexer'
 import { createScanner, getPackageExports, serializationExportsFields } from './scanner'
 import { createInjectScript } from './inject'
 import { isSupportThreads, len, transformCJSRequire } from './shared'
@@ -11,30 +13,13 @@ import { jsdelivr } from './resolver/jsdelivr'
 const debug = _debug('vite-plugin-cdn2')
 
 const NODE_MODULES = 'node_modules'
-// rs-module-lexer can't cover all platforms.
-// But it provide a wasm bindings. So we provide
-// a wrapper func to cover most of scence.
-// WASM support at least node15
-
-async function createRsModuleLexer() {
-  try {
-    const { parse } = (await import('rs-module-lexer')).default
-    return parse
-  } catch (error) {
-    const { parse } = await import('@xn-sakina/rml-wasm').catch(() => {
-      throw new Error('rs-module-lexer can\'t work on you current machine.')
-    })
-    return parse
-  }
-}
 
 function createDependency() {
   const dependency: Record<string, ModuleInfo> = {}
 
-  const filter = (code: string, id: string, dependencyWithAlias: Record<string, string>, lex: Awaited<ReturnType<typeof createRsModuleLexer>>) => {
-    const { output } = lex({ input: [{ filename: id, code }] })
-    if (!len(output)) return false
-    const { imports } = output[0]
+  const filter = (code: string, id: string, dependencyWithAlias: Record<string, string>) => {
+    const [imports] = parse(code, id)
+    if (!len(imports)) return false
     const modules = Array.from(new Set([...imports.map(i => i.n)]))
     for (const m of modules) {
       if (dependencyWithAlias[m]) return true
@@ -45,7 +30,6 @@ function createDependency() {
 
   return {
     dependency,
-    lex: null,
     get dependencyWithAlias() {
       const traverse = (aliases: string[], name: string) => aliases.reduce((acc, cur) => ({ ...acc, [cur]: name }), {})
       return Object.values(this.dependency).reduce((acc, cur) => {
@@ -54,7 +38,7 @@ function createDependency() {
       }, {})
     },
     filter: function (code: string, id: string) {
-      return filter(code, id, this.dependencyWithAlias, this.lex)
+      return filter(code, id, this.dependencyWithAlias)
     }
   }
 }
@@ -92,16 +76,14 @@ function cdn(opts: CDNPluginOptions = {}): Plugin[] {
         const [isSupport, version] = isSupportThreads()
         try {
           if (!isSupport) throw new Error(`vite-plugin-cdn2 can't work with nodejs ${version}.`)
-          const esModuleLexer = await createRsModuleLexer()
-          api.dependency.lex = esModuleLexer
-          const defaultWd = config.root
+          const defaultWd = searchForWorkspaceRoot(config.root)
           scanner.setDefaultWd(defaultWd)
           debug('start scanning')
           scanner.scanAllDependencies()
           debug('scanning done', scanner.dependencies)
           api.dependency.dependency = Object.fromEntries(scanner.dependencies)
           if (logLevel === 'warn') {
-            scanner.failedModules.forEach((errorMessage, name) => config.logger.error(`vite-plugin-cdn2: ${name} ${errorMessage ? errorMessage : 'resolved failed.Please check it.'}`))
+            scanner.failedMessages.forEach(msg => config.logger.error(msg))
           }
           // work for serve mode
           // https://vitejs.dev/config/dep-optimization-options.html
@@ -146,8 +128,6 @@ function external(opts: ExternalPluginOptions = {}): Plugin {
     name: 'vite-plugin-external',
     async buildStart() {
       try {
-        const esModuleLexer = await createRsModuleLexer()
-        dependency.lex = esModuleLexer
         debug('start check modules')
         for (const module of modules) {
           if (!module.global) throw new Error(`vite-plugin-external: missing global for module ${module.name}`)
@@ -190,4 +170,4 @@ export { defineScript, defineLink } from './resolve'
 
 export default cdn
 
-export type { TrackModule, CDNPluginOptions, ExternalPluginOptions, ExternalModule } from './interface'
+export type { IModule, CDNPluginOptions, ExternalPluginOptions, ExternalModule } from './interface'

src/interface.ts

@@ -38,13 +38,13 @@ export interface Module {
   global?: string
 }
 
-export interface TrackModule extends Module {
+export interface IModule extends Module {
   spare?: Array<ScriptSpare | LinkSpare> | string
   relativeModule?: string
   aliases?: Array<string>
 }
 
-export interface IIFEModuleInfo extends TrackModule {
+export interface IIFEModuleInfo extends IModule {
   version: string
   unpkg?: string
   jsdelivr?: string
@@ -57,8 +57,6 @@ export interface ModuleInfo extends IIFEModuleInfo {
   code?: string
 }
 
-export type IModule = TrackModule
-
 export type ExternalModule = Required<Module> & {
   aliases?: Array<string>
 }

src/scanner.ts

@@ -6,7 +6,7 @@ import worker_threads from 'worker_threads'
 import type { MessagePort } from 'worker_threads'
 import { tryScanGlobalName } from './transform'
 import { MAX_CONCURRENT, _import, createConcurrentQueue, is, len, lookup } from './shared'
-import type { IIFEModuleInfo, IModule, Module, ModuleInfo, TrackModule } from './interface'
+import type { IModule, Module, ModuleInfo } from './interface'
 
 // TODO
 // https://github.com/nodejs/node/issues/43304
@@ -29,12 +29,12 @@ interface WorkerData {
 }
 
 interface ScannerModule {
-  modules: Array<TrackModule>
+  modules: Array<IModule>
 }
 
 interface ThreadMessage {
   bindings: Map<string, ModuleInfo>, 
-  failedModules: Map<string, string>
+  failedMessages: string[]
   id: number
   error: Error | AggregateError
 }
@@ -57,8 +57,8 @@ function createWorkerThreads(scannerModule: ScannerModule, defaultWd: string) {
     const { message }: { message: ThreadMessage } = worker_threads.receiveMessageOnPort(mainPort)
     if (message.id !== id) throw new Error(`Internal error: Expected id ${id} but got id ${message.id}`)
     if (message.error) throw message.error
-    const { bindings, failedModules } = message
-    return { dependencies: bindings, failedModules }
+    const { bindings, failedMessages } = message
+    return { dependencies: bindings, failedMessages }
   }
   return runSync()
 }
@@ -96,82 +96,55 @@ export async function getPackageExports(...argvs: [string | Module, string?]) {
   return bindings
 }
 
-async function tryResolveModule(
-  module: IModule,
-  dependenciesMap: Map<string, ModuleInfo>,
-  failedModules: Map<string, string>,
-  defaultWd: string
-) {
-  const { name: moduleName, relativeModule, aliases, ...rest } = module
+async function tryResolvePackage(module: IModule, defaultWd: string, handler: (moduleInfo: ModuleInfo, errorMessage: string) => void) {
+  const { name: pkgName, ...rest } = module
+  let errorMessage = ''
+  const moduleInfo: ModuleInfo = Object.create(null)
   try {
-    const modulePath = _require.resolve(moduleName, { paths: [defaultWd] })
-    const packageJsonPath = lookup(modulePath, 'package.json')
-    const str = await fsp.readFile(packageJsonPath, 'utf8')
-    const packageJSON: IIFEModuleInfo = JSON.parse(str)
-    const { version, name, unpkg, jsdelivr } = packageJSON
-    const meta: ModuleInfo = Object.create(null)
-    // Most of package has jsdelivr or unpkg field
-    // but a small part is not. so we should accept user define.
-    const iifeRelativePath = relativeModule || jsdelivr || unpkg 
-    if (!iifeRelativePath) throw new Error('try resolve file failed.')
-    if (rest.global) {
-      Object.assign(meta, { name, version, relativeModule: iifeRelativePath, aliases: serializationExportsFields(name, aliases), ...rest })
+    const pkgPath = _require.resolve(pkgName, { paths: [defaultWd] })
+    const pkgJsonPath = _require.resolve(path.join(pkgName, 'package.json'), { paths: [defaultWd] })
+    const { version, name, unpkg, jsdelivr } = _require(pkgJsonPath)
+    const iifeRelativePath = rest.relativeModule || jsdelivr || unpkg
+    is(!!iifeRelativePath, `[scanner error]: can't find any iife file from package '${pkgName}',please check it.`, 'normal')
+    if ('global' in rest) {
+      Object.assign(moduleInfo, { ...rest, name, version, relativeModule: iifeRelativePath, aliases: serializationExportsFields(name, rest.aliases) })
     } else {
-      const iifeFilePath = lookup(packageJsonPath, iifeRelativePath)
+      const iifeFilePath = lookup(pkgJsonPath, iifeRelativePath)
       const code = await fsp.readFile(iifeFilePath, 'utf8')
-      Object.assign(meta, { name, version, code, relativeModule: iifeRelativePath, aliases: serializationExportsFields(name, aliases), ...rest })
+      const global = await tryScanGlobalName(code)
+      is(!!global, `[scanner error]: unable to guess the global name form '${pkgName}', please enter manually.`, 'normal')
+      Object.assign(moduleInfo, { ...rest, name, version, relativeModule: iifeRelativePath, aliases: serializationExportsFields(name, rest.aliases), global })
     }
-    const bindings = await getPackageExports(modulePath)
-    dependenciesMap.set(name, { ...meta, bindings })
+    moduleInfo.bindings = await getPackageExports(pkgPath)
   } catch (error) {
-    const message = (() => {
-      if (error instanceof Error) {
-        if ('code' in error) {
-          if (error.code === 'MODULE_NOT_FOUND') return 'can\'t find module.'
-        }
-        return error.message
-      }
-      return error
-    })()
-    failedModules.set(moduleName, message)
+    errorMessage = `[scanner error]: invalid package '${pkgName}'.`
+    if (error instanceof Error && 'code' in error) {
+      if (error.code === 'normal') errorMessage = error.message
+    }
   }
+  handler(moduleInfo, errorMessage)
 }
 
 function startSyncThreads() {
   if (!worker_threads.workerData.internalThread) return
   const { workerPort, scannerModule, defaultWd } = worker_threads.workerData as WorkerData
   const { parentPort } = worker_threads
-  const dependenciesMap: Map<string, ModuleInfo> = new Map()
-  const failedModules: Map<string, string> = new Map()
+  const bindings: Map<string, ModuleInfo> = new Map()
+  const failedMessages: string[] = []
   parentPort?.on('message', (msg) => {
     (async () => {
       const { id, sharedBuffer } = msg
       const sharedBufferView = new Int32Array(sharedBuffer)
+      const queue = createConcurrentQueue(MAX_CONCURRENT)
       try {
-        const bindings: Map<string, ModuleInfo> = new Map()
-        const queue = createConcurrentQueue(MAX_CONCURRENT)
         for (const module of scannerModule) {
-          queue.enqueue(() => tryResolveModule(module, dependenciesMap, failedModules, defaultWd))
+          queue.enqueue(() => tryResolvePackage(module, defaultWd, (info, msg) => {
+            if (msg) return failedMessages.push(msg)
+            bindings.set(info.name, info)
+          }))
         }
         await queue.wait()
-        for (const module of scannerModule) {
-          const { name } = module
-          if (dependenciesMap.has(name)) {
-            const { code, ...rest } = dependenciesMap.get(name)
-            if (!code) {
-              bindings.set(name, rest)
-              continue
-            }
-            const globalName = await tryScanGlobalName(code)
-            if (!globalName) {
-              failedModules.set(name, 'try resolve global name failed.')
-            } else {
-              bindings.set(name, { ...rest, global: globalName })
-            }
-          }
-        }
-        dependenciesMap.clear()
-        workerPort.postMessage({ bindings, id, failedModules })
+        workerPort.postMessage({ bindings, id, failedMessages })
       } catch (error) {
         workerPort.postMessage({ error, id })
       }
@@ -188,12 +161,12 @@ if (worker_threads.workerData?.internalThread) {
 class Scanner {
   modules: Array<IModule | string>
   dependencies: Map<string, ModuleInfo>
-  failedModules: Map<string, string>
+  failedMessages: string[]
   private defaultWd: string
   constructor(modules: Array<IModule | string>) {
     this.modules = modules
     this.dependencies = new Map()
-    this.failedModules = new Map()
+    this.failedMessages = []
     this.defaultWd = process.cwd()
   }
 
@@ -203,9 +176,9 @@ class Scanner {
 
   public scanAllDependencies() {
     // we won't throw any exceptions inside this task.
-    const res = createWorkerThreads(this.serialization(this.modules), this.defaultWd)
-    this.dependencies = res.dependencies
-    this.failedModules = res.failedModules
+    const { failedMessages, dependencies } = createWorkerThreads(this.serialization(this.modules), this.defaultWd)
+    this.dependencies = dependencies
+    this.failedMessages = failedMessages
   }
 
   private serialization(input: Array<IModule | string>) {

src/shared.ts

@@ -25,9 +25,11 @@ export function isSupportThreads(): [boolean, string] {
   return [true, `${major}.${minor}`]
 }
 
-export function is(condit: boolean, message: string) {
+export function is(condit: boolean, message: string, code?: number | string) {
   if (!condit) {
-    throw new Error(message)
+    const error = new Error(message)
+    if (code !== undefined) (error as any).code = code
+    throw error
   }
 }