perf: Benchmark expression evaluation

.github/workflows/benchmark-workflow.yml

@@ -0,0 +1,44 @@
+name: Benchmark workflow package
+
+on:
+  pull_request:
+    paths:
+      - 'packages/workflow/**'
+  workflow_dispatch:
+
+jobs:
+  benchmark:
+    name: Benchmark
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4.1.1
+
+      - run: corepack enable
+
+      - uses: actions/setup-node@v4.0.1
+        with:
+          node-version: 18.x
+          cache: pnpm
+
+      - run: pnpm install --frozen-lockfile
+
+      - name: Build
+        if: ${{ inputs.cacheKey == '' }}
+        run: pnpm build:backend
+
+      - name: Restore cached build artifacts
+        if: ${{ inputs.cacheKey != '' }}
+        uses: actions/cache/restore@v4.0.0
+        with:
+          path: ./packages/**/dist
+          key: ${{ inputs.cacheKey }}
+
+      - name: Build benchmarks
+        working-directory: packages/workflow
+        run: pnpm benchmark:build
+
+      - name: Benchmark workflow package
+        uses: CodSpeedHQ/action@v2
+        with:
+          working-directory: packages/workflow
+          run: pnpm benchmark:run

packages/workflow/package.json

@@ -24,6 +24,8 @@
     "./*": "./*"
   },
   "scripts": {
+    "benchmark:build": "tsc -p tsconfig.benchmark.json > /dev/null; tsc-alias -p tsconfig.benchmark.json",
+    "benchmark:run": "node dist/test/benchmark.js",
     "clean": "rimraf dist .turbo",
     "dev": "pnpm watch",
     "typecheck": "tsc",
@@ -39,15 +41,18 @@
     "dist/**/*"
   ],
   "devDependencies": {
+    "@codspeed/tinybench-plugin": "^3.1.0",
     "@types/deep-equal": "^1.0.1",
     "@types/express": "^4.17.21",
     "@types/jmespath": "^0.15.0",
     "@types/lodash": "^4.14.195",
     "@types/luxon": "^3.2.0",
     "@types/md5": "^2.3.5",
-    "@types/xml2js": "^0.4.14"
+    "@types/xml2js": "^0.4.14",
+    "tinybench": "^2.8.0"
   },
   "dependencies": {
+    "@langchain/core": "0.1.61",
     "@n8n/tournament": "1.0.2",
     "@n8n_io/riot-tmpl": "4.0.0",
     "ast-types": "0.15.2",
@@ -65,7 +70,6 @@
     "recast": "0.21.5",
     "title-case": "3.0.3",
     "transliteration": "2.3.5",
-    "xml2js": "0.6.2",
-    "@langchain/core": "0.1.61"
+    "xml2js": "0.6.2"
   }
 }

packages/workflow/test/Expression.test.ts

@@ -3,44 +3,19 @@
  */
 
 import { DateTime, Duration, Interval } from 'luxon';
-import { Workflow } from '@/Workflow';
-import * as Helpers from './Helpers';
 import type { ExpressionTestEvaluation, ExpressionTestTransform } from './ExpressionFixtures/base';
 import { baseFixtures } from './ExpressionFixtures/base';
-import type { INodeExecutionData } from '@/Interfaces';
 import { extendSyntax } from '@/Extensions/ExpressionExtension';
 import { ExpressionError } from '@/errors/expression.error';
 import { setDifferEnabled, setEvaluator } from '@/ExpressionEvaluatorProxy';
-import { workflow } from './ExpressionExtensions/Helpers';
+import { evaluate } from './evaluate';
 
 setDifferEnabled(true);
 
 for (const evaluator of ['tmpl', 'tournament'] as const) {
 	setEvaluator(evaluator);
 	describe(`Expression (with ${evaluator})`, () => {
 		describe('getParameterValue()', () => {
-			const nodeTypes = Helpers.NodeTypes();
-			const workflow = new Workflow({
-				id: '1',
-				nodes: [
-					{
-						name: 'node',
-						typeVersion: 1,
-						type: 'test.set',
-						id: 'uuid-1234',
-						position: [0, 0],
-						parameters: {},
-					},
-				],
-				connections: {},
-				active: false,
-				nodeTypes,
-			});
-			const expression = workflow.expression;
-
-			const evaluate = (value: string) =>
-				expression.getParameterValue(value, null, 0, 0, 'node', [], 'manual', {});
-
 			it('should not be able to use global built-ins from denylist', () => {
 				expect(evaluate('={{document}}')).toEqual({});
 				expect(evaluate('={{window}}')).toEqual({});
@@ -173,13 +148,6 @@ for (const evaluator of ['tmpl', 'tournament'] as const) {
 		});
 
 		describe('Test all expression value fixtures', () => {
-			const expression = workflow.expression;
-
-			const evaluate = (value: string, data: INodeExecutionData[]) => {
-				const itemIndex = data.length === 0 ? -1 : 0;
-				return expression.getParameterValue(value, null, 0, itemIndex, 'node', data, 'manual', {});
-			};
-
 			for (const t of baseFixtures) {
 				if (!t.tests.some((test) => test.type === 'evaluation')) {
 					continue;

packages/workflow/test/benchmark.ts

@@ -0,0 +1,31 @@
+import { Bench } from 'tinybench';
+import { withCodSpeed } from '@codspeed/tinybench-plugin';
+import { baseFixtures } from './ExpressionFixtures/base';
+import { evaluate } from './evaluate';
+import type { INodeExecutionData } from '@/index';
+
+function addExpressionEvaluationTasks(bench: Bench) {
+	for (const fixture of baseFixtures) {
+		for (const test of fixture.tests) {
+			if ('error' in test) continue;
+
+			if (test.type === 'evaluation') {
+				const input = test.input.map((d) => ({ json: d })) as INodeExecutionData[];
+				bench.add(fixture.expression, () => evaluate(fixture.expression, input));
+			}
+		}
+	}
+}
+
+async function main() {
+	const bench = withCodSpeed(new Bench());
+
+	addExpressionEvaluationTasks(bench);
+
+	await bench.warmup();
+	await bench.run();
+
+	console.table(bench.table());
+}
+
+void main();

packages/workflow/test/evaluate.ts

@@ -0,0 +1,6 @@
+import { workflow } from './ExpressionExtensions/Helpers';
+import type { INodeExecutionData } from '@/Interfaces';
+
+export const evaluate = (expression: string, data: INodeExecutionData[] = []) => {
+	return workflow.expression.getParameterValue(expression, null, 0, 0, 'node', data, 'manual', {});
+};

packages/workflow/tsconfig.benchmark.json

@@ -0,0 +1,13 @@
+{
+	"extends": ["./tsconfig.json", "../../tsconfig.build.json"],
+	"compilerOptions": {
+		"outDir": "dist",
+		"tsBuildInfoFile": "dist/benchmark.tsbuildinfo",
+		"baseUrl": ".",
+		"paths": {
+			"@/*": ["./src/*"]
+		}
+	},
+	"include": ["test/**/*.ts"],
+	"exclude": ["test/**/*.test.ts"]
+}