feat!: support running tests using VM context (#3203)

Co-authored-by: Anjorin Damilare <damilareanjorin1@gmail.com>

.github/workflows/ci.yml

@@ -84,6 +84,9 @@ jobs:
       - name: Test Single Thread
         run: pnpm run test:ci:single-thread
 
+      - name: Test Vm Threads
+        run: pnpm run test:ci:vm-threads
+
   test-ui:
     runs-on: ubuntu-latest
 

docs/config/index.md

@@ -144,7 +144,11 @@ Handling for dependencies resolution.
 - **Type:** `(string | RegExp)[]`
 - **Default:** `[/\/node_modules\//]`
 
-Externalize means that Vite will bypass the package to native Node. Externalized dependencies will not be applied Vite's transformers and resolvers, so they do not support HMR on reload. All packages under `node_modules` are externalized.
+Externalize means that Vite will bypass the package to the native Node. Externalized dependencies will not be applied to Vite's transformers and resolvers, so they do not support HMR on reload. By default, all packages inside `node_modules` are externalized.
+
+These options support package names as they are written in `node_modules` or specified inside [`deps.moduleDirectories`](#deps-moduledirectories). For example, package `@company/some-name` located inside `packages/some-name` should be specified as `some-name`, and `packages` should be included in `deps.moduleDirectories`. Basically, Vitest always checks the file path, not the actual package name.
+
+If regexp is used, Vitest calls it on the _file path_, not the package name.
 
 #### server.deps.inline
 
@@ -421,6 +425,7 @@ import type { Environment } from 'vitest'
 
 export default <Environment>{
   name: 'custom',
+  transformMode: 'ssr',
   setup() {
     // custom setup
     return {
@@ -468,7 +473,7 @@ export default defineConfig({
 
 ### poolMatchGlobs
 
-- **Type:** `[string, 'browser' | 'threads' | 'child_process'][]`
+- **Type:** `[string, 'threads' | 'child_process' | 'experimentalVmThreads'][]`
 - **Default:** `[]`
 - **Version:** Since Vitest 0.29.4
 
@@ -542,6 +547,69 @@ Custom reporters for output. Reporters can be [a Reporter instance](https://gith
 Write test results to a file when the `--reporter=json`, `--reporter=html` or `--reporter=junit` option is also specified.
 By providing an object instead of a string you can define individual outputs when using multiple reporters.
 
+### experimentalVmThreads
+
+- **Type:** `boolean`
+- **CLI:** `--experimentalVmThreads`, `--experimental-vm-threads`
+- **Version:** Since Vitest 0.34.0
+
+Run tests using [VM context](https://nodejs.org/api/vm.html) (inside a sandboxed environment) in a worker pool.
+
+This makes tests run faster, but the VM module is unstable when running [ESM code](https://github.com/nodejs/node/issues/37648). Your tests will [leak memory](https://github.com/nodejs/node/issues/33439) - to battle that, consider manually editing [`experimentalVmWorkerMemoryLimit`](#experimentalvmworkermemorylimit) value.
+
+::: warning
+Running code in a sandbox has some advantages (faster tests), but also comes with a number of disadvantages.
+
+- The globals within native modules, such as (`fs`, `path`, etc), differ from the globals present in your test environment. As a result, any error thrown by these native modules will reference a different Error constructor compared to the one used in your code:
+
+```ts
+try {
+  fs.writeFileSync('/doesnt exist')
+}
+catch (err) {
+  console.log(err instanceof Error) // false
+}
+```
+
+- Importing ES modules caches them indefinitely which introduces memory leaks if you have a lot of contexts (test files). There is no API in Node.js that clears that cache.
+- Accessing globals [takes longer](https://github.com/nodejs/node/issues/31658) in a sandbox environment.
+
+Please, be aware of these issues when using this option. Vitest team cannot fix any of the issues on our side.
+:::
+
+### experimentalVmWorkerMemoryLimit
+
+- **Type:** `string | number`
+- **CLI:** `--experimentalVmWorkerMemoryLimit`, `--experimental-vm-worker-memory-limit`
+- **Default:** `1 / CPU Cores`
+- **Version:** Since Vitest 0.34.0
+
+Specifies the memory limit for workers before they are recycled. This value heavily depends on your environment, so it's better to specify it manually instead of relying on the default.
+
+This option only affects workers that run tests in [VM context](#experimentalvmthreads).
+
+::: tip
+The implementation is based on Jest's [`workerIdleMemoryLimit`](https://jestjs.io/docs/configuration#workeridlememorylimit-numberstring).
+
+The limit can be specified in a number of different ways and whatever the result is `Math.floor` is used to turn it into an integer value:
+
+- `<= 1` - The value is assumed to be a percentage of system memory. So 0.5 sets the memory limit of the worker to half of the total system memory
+- `\> 1` - Assumed to be a fixed byte value. Because of the previous rule if you wanted a value of 1 byte (I don't know why) you could use 1.1.
+- With units
+  - `50%` - As above, a percentage of total system memory
+  - `100KB`, `65MB`, etc - With units to denote a fixed memory limit.
+    - `K` / `KB` - Kilobytes (x1000)
+    - `KiB` - Kibibytes (x1024)
+    - `M` / `MB` - Megabytes
+    - `MiB` - Mebibytes
+    - `G` / `GB` - Gigabytes
+    - `GiB` - Gibibytes
+:::
+
+::: warning
+Percentage based memory limit [does not work on Linux CircleCI](https://github.com/jestjs/jest/issues/11956#issuecomment-1212925677) workers due to incorrect system memory being reported.
+:::
+
 ### threads
 
 - **Type:** `boolean`
@@ -708,6 +776,8 @@ Make sure that your files are not excluded by `watchExclude`.
 
 Isolate environment for each test file. Does not work if you disable [`--threads`](#threads).
 
+This options has no effect on [`experimentalVmThreads`](#experimentalvmthreads).
+
 ### coverage<NonProjectOption />
 
 You can use [`v8`](https://v8.dev/blog/javascript-code-coverage), [`istanbul`](https://istanbul.js.org/) or [a custom coverage solution](/guide/coverage#custom-coverage-provider) for coverage collection.

docs/guide/cli.md

@@ -69,7 +69,10 @@ Run only [benchmark](https://vitest.dev/guide/features.html#benchmarking-experim
 | `--ui` | Enable UI |
 | `--open` | Open the UI automatically if enabled (default: `true`) |
 | `--api [api]` | Serve API, available options: `--api.port <port>`, `--api.host [host]` and `--api.strictPort` |
-| `--threads` | Enable Threads (default: `true`) |
+| `--threads` | Enable Threads (default: `true`) |   
+| `--single-thread` | Run tests inside a single thread, requires --threads (default: `false`) |
+| `--experimental-vm-threads` | Run tests in a worker pool using VM isolation (default: `false`) |
+| `--experimental-vm-worker-memory-limit` | Set the maximum allowed memory for a worker. When reached, a new worker will be created instead |
 | `--silent` | Silent console output from tests |
 | `--isolate` | Isolate environment for each test file (default: `true`) |
 | `--reporter <name>` | Select reporter: `default`, `verbose`, `dot`, `junit`, `json`, or a path to a custom reporter |

docs/guide/environment.md

@@ -27,13 +27,27 @@ Or you can also set [`environmentMatchGlobs`](https://vitest.dev/config/#environ
 
 ## Custom Environment
 
-Starting from 0.23.0, you can create your own package to extend Vitest environment. To do so, create package with the name `vitest-environment-${name}`. That package should export an object with the shape of `Environment`:
+Starting from 0.23.0, you can create your own package to extend Vitest environment. To do so, create package with the name `vitest-environment-${name}` or specify a path to a valid JS file (supported since 0.34.0). That package should export an object with the shape of `Environment`:
 
 ```ts
 import type { Environment } from 'vitest'
 
 export default <Environment>{
   name: 'custom',
+  transformMode: 'ssr',
+  // optional - only if you support "experimental-vm" pool
+  async setupVM() {
+    const vm = await import('node:vm')
+    const context = vm.createContext()
+    return {
+      getVmContext() {
+        return context
+      },
+      teardown() {
+        // called after all tests with this env have been run
+      }
+    }
+  },
   setup() {
     // custom setup
     return {
@@ -45,6 +59,10 @@ export default <Environment>{
 }
 ```
 
+::: warning
+Since 0.34.0 Vitest requires `transformMode` option on environment object. It should be equal to `ssr` or `web`. This value determines how plugins will transform source code. If it's set to `ssr`, plugin hooks will receive `ssr: true` when transforming or resolving files. Otherwise, `ssr` is set to `false`.
+:::
+
 You also have access to default Vitest environments through `vitest/environments` entry:
 
 ```ts

examples/mocks/vite.config.ts

@@ -29,8 +29,12 @@ export default defineConfig({
   test: {
     globals: true,
     environment: 'node',
+    server: {
+      deps: {
+        external: [/src\/external/],
+      },
+    },
     deps: {
-      external: [/src\/external/],
       interopDefault: true,
       moduleDirectories: ['node_modules', 'projects'],
     },

examples/solid/test/__snapshots__/Hello.test.jsx.snap

@@ -1,4 +1,4 @@
-// Vitest Snapshot v1
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
 
 exports[`<Hello /> > renders 1`] = `"<div>4 x 2 = 8</div><button>x1</button>"`;
 

examples/svelte/package.json

@@ -9,7 +9,7 @@
   },
   "devDependencies": {
     "@sveltejs/vite-plugin-svelte": "latest",
-    "@testing-library/svelte": "latest",
+    "@testing-library/svelte": "^4.0.3",
     "@vitest/ui": "latest",
     "jsdom": "latest",
     "svelte": "latest",

examples/vitesse/src/auto-import.d.ts

@@ -1,6 +1,7 @@
 /* eslint-disable */
 /* prettier-ignore */
 // @ts-nocheck
+// noinspection JSUnusedGlobalSymbols
 // Generated by unplugin-auto-import
 export {}
 declare global {

package.json

@@ -23,6 +23,7 @@
     "test:run": "vitest run -r test/core",
     "test:all": "CI=true pnpm -r --stream run test --allowOnly",
     "test:ci": "CI=true pnpm -r --stream --filter !test-fails --filter !test-browser --filter !test-esm --filter !test-browser run test --allowOnly",
+    "test:ci:vm-threads": "CI=true pnpm -r --stream --filter !test-fails --filter !test-single-thread --filter !test-browser --filter !test-esm --filter !test-browser run test --allowOnly --experimental-vm-threads",
     "test:ci:single-thread": "CI=true pnpm -r --stream --filter !test-fails --filter !test-coverage --filter !test-watch --filter !test-bail --filter !test-esm --filter !test-browser run test --allowOnly --no-threads",
     "typecheck": "tsc --noEmit",
     "typecheck:why": "tsc --noEmit --explainFiles > explainTypes.txt",

packages/browser/src/client/main.ts

@@ -68,6 +68,9 @@ ws.addEventListener('open', async () => {
   globalThis.__vitest_worker__ = {
     config,
     browserHashMap,
+    environment: {
+      name: 'browser',
+    },
     // @ts-expect-error untyped global for internal use
     moduleCache: globalThis.__vi_module_cache__,
     rpc: client.rpc,

packages/runner/src/types/runner.ts

@@ -29,7 +29,7 @@ export interface VitestRunnerConstructor {
   new(config: VitestRunnerConfig): VitestRunner
 }
 
-export type CancelReason = 'keyboard-input' | 'test-failure' | string & {}
+export type CancelReason = 'keyboard-input' | 'test-failure' | string & Record<string, never>
 
 export interface VitestRunner {
   /**

packages/vite-node/package.json

@@ -46,6 +46,11 @@
       "require": "./dist/source-map.cjs",
       "import": "./dist/source-map.mjs"
     },
+    "./constants": {
+      "types": "./dist/constants.d.ts",
+      "require": "./dist/constants.cjs",
+      "import": "./dist/constants.mjs"
+    },
     "./*": "./*"
   },
   "main": "./dist/index.mjs",

packages/vite-node/rollup.config.js

@@ -15,6 +15,7 @@ const entries = {
   'client': 'src/client.ts',
   'utils': 'src/utils.ts',
   'cli': 'src/cli.ts',
+  'constants': 'src/constants.ts',
   'hmr': 'src/hmr/index.ts',
   'source-map': 'src/source-map.ts',
 }
@@ -29,6 +30,7 @@ const external = [
   'vite/types/hot',
   'node:url',
   'node:events',
+  'node:vm',
 ]
 
 const plugins = [

packages/vite-node/src/client.ts

@@ -17,7 +17,7 @@ const debugNative = createDebug('vite-node:client:native')
 
 const clientStub = {
   injectQuery: (id: string) => id,
-  createHotContext() {
+  createHotContext: () => {
     return {
       accept: () => {},
       prune: () => {},
@@ -28,33 +28,11 @@ const clientStub = {
       send: () => {},
     }
   },
-  updateStyle(id: string, css: string) {
-    if (typeof document === 'undefined')
-      return
-
-    const element = document.querySelector(`[data-vite-dev-id="${id}"]`)
-    if (element) {
-      element.textContent = css
-      return
-    }
-
-    const head = document.querySelector('head')
-    const style = document.createElement('style')
-    style.setAttribute('type', 'text/css')
-    style.setAttribute('data-vite-dev-id', id)
-    style.textContent = css
-    head?.appendChild(style)
-  },
-  removeStyle(id: string) {
-    if (typeof document === 'undefined')
-      return
-    const sheet = document.querySelector(`[data-vite-dev-id="${id}"]`)
-    if (sheet)
-      document.head.removeChild(sheet)
-  },
+  updateStyle: () => {},
+  removeStyle: () => {},
 }
 
-export const DEFAULT_REQUEST_STUBS: Record<string, unknown> = {
+export const DEFAULT_REQUEST_STUBS: Record<string, Record<string, unknown>> = {
   '/@vite/client': clientStub,
   '@vite/client': clientStub,
 }
@@ -304,7 +282,6 @@ export class ViteNodeRunner {
     const requestStubs = this.options.requestStubs || DEFAULT_REQUEST_STUBS
     if (id in requestStubs)
       return requestStubs[id]
-
     let { code: transformed, externalize } = await this.options.fetchModule(id)
 
     if (externalize) {
@@ -317,6 +294,8 @@ export class ViteNodeRunner {
     if (transformed == null)
       throw new Error(`[vite-node] Failed to load "${id}" imported from ${callstack[callstack.length - 2]}`)
 
+    const { Object, Reflect, Symbol } = this.getContextPrimitives()
+
     const modulePath = cleanUrl(moduleId)
     // disambiguate the `<UNIT>:/` on windows: see nodejs/node#31710
     const href = pathToFileURL(modulePath).href
@@ -416,18 +395,27 @@ export class ViteNodeRunner {
     if (transformed[0] === '#')
       transformed = transformed.replace(/^\#\!.*/, s => ' '.repeat(s.length))
 
+    await this.runModule(context, transformed)
+
+    return exports
+  }
+
+  protected getContextPrimitives() {
+    return { Object, Reflect, Symbol }
+  }
+
+  protected async runModule(context: Record<string, any>, transformed: string) {
     // add 'use strict' since ESM enables it by default
     const codeDefinition = `'use strict';async (${Object.keys(context).join(',')})=>{{`
     const code = `${codeDefinition}${transformed}\n}}`
-    const fn = vm.runInThisContext(code, {
-      filename: __filename,
+    const options = {
+      filename: context.__filename,
       lineOffset: 0,
       columnOffset: -codeDefinition.length,
-    })
+    }
 
+    const fn = vm.runInThisContext(code, options)
     await fn(...Object.values(context))
-
-    return exports
   }
 
   prepareContext(context: Record<string, any>) {
@@ -446,11 +434,15 @@ export class ViteNodeRunner {
     return !path.endsWith('.mjs') && 'default' in mod
   }
 
+  protected importExternalModule(path: string) {
+    return import(path)
+  }
+
   /**
    * Import a module and interop it
    */
   async interopedImport(path: string) {
-    const importedModule = await import(path)
+    const importedModule = await this.importExternalModule(path)
 
     if (!this.shouldInterop(path, importedModule))
       return importedModule