test(perf): add per test file overhead benchmark

.gitignore

@@ -27,6 +27,8 @@
 /website/translated_docs
 /website/i18n/*
 
+/benchmarks/*/node_modules/
+
 /reports/*
 
 coverage

benchmarks/test-file-overhead/.gitignore

@@ -0,0 +1 @@
+[1-9]*.test.js

benchmarks/test-file-overhead/0.test.js

@@ -0,0 +1 @@
+it('is fast', () => {});

benchmarks/test-file-overhead/README.md

@@ -0,0 +1,6 @@
+First, run `./prepare.sh` to generate the benchmark files. On Windows, use a Bash (WSL, Git, Cygwinm ...) to do this, but you can use CMD for the actual benchmark run if the CMD environment is what you want to benchmark for.
+
+To run the benchmark, use a benchmarking tool such as [hyperfine](https://github.com/sharkdp/hyperfine):
+```sh
+hyperfine -w 3 -m 10 ../../jest /tmp/other-jest-clone-to-compare-against/jest
+```

benchmarks/test-file-overhead/package.json

@@ -0,0 +1,5 @@
+{
+  "jest": {
+    "testEnvironment": "node"
+  }
+}

benchmarks/test-file-overhead/prepare.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env sh
+for i in {1..499}; do
+  cp 0.test.js $i.test.js
+done

benchmarks/test-file-overhead/yarn.lock

@@ -0,0 +1,11 @@
+# This file is generated by running "yarn install" inside your project.
+# Manual changes might be lost - proceed with caution!
+
+__metadata:
+  version: 4
+
+"root-workspace-0b6124@workspace:.":
+  version: 0.0.0-use.local
+  resolution: "root-workspace-0b6124@workspace:."
+  languageName: unknown
+  linkType: soft

packages/jest-runtime/src/index.ts

@@ -100,6 +100,7 @@ type ModuleRegistry = Map<string, InitialModule | Module>;
 // * take sufficiently long to require to warrant an optimization.
 // When required from the outside, they use the worker's require cache and are thus
 // only loaded once per worker, not once per test file.
+// Use /benchmarks/test-file-overhead to measure the impact.
 // Note that this only applies when they are required in an internal context;
 // users who require one of these modules in their tests will still get the module from inside the VM.
 // Prefer listing a module here only if it is impractical to use the jest-resolve-outside-vm-option where it is required,