perf: End-to-end Linter and Formatter benchmarks (#3570)

rome · Nov 7, 2022 · fec159f · fec159f
1 parent abdb58d
commit fec159f
Show file tree

Hide file tree

Showing 6 changed files with 2,287 additions and 0 deletions.
diff --git a/benchmark/README.md b/benchmark/README.md
@@ -0,0 +1,43 @@
+# Benchmarks
+## Getting Started
+1. Install hyperfine: `cargo install hyperfine`
+2. Install node modules: `npm i`
+3. Run the benchmarks: `node run.js`
+
+## Results
+Setup: MacBook Pro (13-inch, M1, 2020)
+
+### Formatting
+* Rome's ~25 times faster than Prettier
+* Rome's ~7 times faster when restricting it to a single core
+
+### Linting
+* Rome's 2-3 times faster than ESLint
+* Rome is ~20-40% slower than ESLint when restricting it to a single core.
+
+The speed-ups for the multithreaded benchmarks can vary significantly depending on the setup. For example, Rome is 100 times faster than Prettier on an M1 Max with 10 cores.
+
+## Analysis
+### Formatter
+* Rome's formatter is fast :).
+* It should be possible to speed up Prettier. Rome's architecture isn't that different, and native has its advantages, but Prettier should be able to compete in single-threaded mode.
+
+### Linting
+* Rome's linter spends significant time building the semantic model, the control flow graph, and matching queries. I'm convinced there's room for improvement ([3565](https://github.com/rome/tools/pull/3565), [3569](https://github.com/rome/tools/pull/3569)).
+* Computing the diff for code fixes is expensive. Rome can spend up to 3s computing diffs (not measured by these benchmarks, see explanations below)
+* Hypothesis: Rome doesn't use async IO to read files. That's why the single-threaded Rome issues the file read commands one by one. ESLint may be faster in single-threaded linting because it can issue all reads with async IO (so that the OS loads the files in the background while other files are linted). I have yet to verify if ESLint indeed does use async IO.
+
+## Notes
+
+We've been careful to create fair benchmarks. This section explains some of the decisions behind the benchmark setup and how these decisions affect the results. Please [let us know](https://github.com/rome/tools/issues) if you have ideas on how to make the benchmarks fairer or if there's a mistake with our setup.
+
+### Formatting
+* Compares the wall time of Rome and Prettier to format all files in a project where all files are correctly formatted.
+* The benchmark limits Prettier to only format JavaScript and TypeScript files because Rome doesn't support other file types.
+* Rome only prints a summary with the number of formatted files. The prettier benchmark uses `--loglevel=error` for a fairer benchmark so that Prettier doesn't print every filename.
+
+### Linting
+* Compares the wall time of Rome and ESLint to check all files in a project.
+* Only enables rules that both Rome and ESLint support.
+* Rome prints rich diffs for each lint diagnostic, whereas ESLint only shows the name and description of the lint rule. That's why the benchmark uses `--max-diagnostics=0` when running Rome because Rome then only counts diagnostics without generating the diffs. Overall, this results in a more accurate comparison but has the downside that Rome only prints the diagnostic count, whereas ESLint prints a line for each lint error.
+
diff --git a/benchmark/bench.eslint.js b/benchmark/bench.eslint.js
@@ -0,0 +1,27 @@
+module.exports = {
+	rules: {
+		"no-extra-boolean-cast": "error",
+		"prefer-rest-params": "error",
+		"no-async-promise-executor": "error",
+		"no-ex-assign": "error",
+		"no-compare-neg-zero": "error",
+		"no-debugger": "error",
+		"no-delete-var": "error",
+		eqeqeq: "error",
+		"no-dupe-args": "error",
+		"no-empty-pattern": "error",
+		"no-func-assign": "error",
+		"no-import-assign": "error",
+		"no-label-var": "error",
+		"no-new-symbol": "error",
+		"no-restricted-globals": "error",
+		"no-shadow-restricted-names": "error",
+		"no-sparse-arrays": "error",
+		"no-unreachable": "error",
+		"no-unsafe-negation": "error",
+		"no-unused-vars": "error",
+		"valid-typeof": "error",
+		"no-const-assign": "warn",
+		"for-direction": "warn",
+	},
+};
diff --git a/benchmark/bench.rome.json b/benchmark/bench.rome.json
@@ -0,0 +1,38 @@
+{
+    "linter": {
+        "enabled": true,
+        "rules": {
+            "recommended": false,
+            "complexity": {
+                "noExtraBooleanCast": "error"
+            },
+            "correctness": {
+                "noArguments": "error",
+                "noAsyncPromiseExecutor": "error",
+                "noCatchAssign": "error",
+                "noCompareNegZero": "error",
+                "noDebugger": "error",
+                "noDelete": "error",
+                "noDoubleEquals": "error",
+                "noDupeArgs": "error",
+                "noEmptyPattern": "error",
+                "noFunctionAssign": "error",
+                "noImportAssign": "error",
+                "noLabelVar": "error",
+                "noMultipleSpacesInRegularExpressionLiterals": "error",
+                "noNewSymbol": "error",
+                "noRestrictedGlobals": "error",
+                "noShadowRestrictedNames": "error",
+                "noSparseArray": "error",
+                "noUnreachable": "error",
+                "noUnsafeNegation": "error",
+                "noUnusedVariables": "error",
+                "useValidTypeof": "error"
+            },
+            "nursery": {
+                "noConstAssign": "warn",
+                "useValidForDirection": "warn"
+            }
+        }
+    }
+}