diff --git a/.github/workflows/benchmark-workflow.yml b/.github/workflows/benchmark-workflow.yml new file mode 100644 index 0000000000000..0aae1263783bb --- /dev/null +++ b/.github/workflows/benchmark-workflow.yml @@ -0,0 +1,44 @@ +name: Benchmark workflow package + +on: + pull_request: + paths: + - 'packages/workflow/**' + workflow_dispatch: + +jobs: + benchmark: + name: Benchmark + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4.1.1 + + - run: corepack enable + + - uses: actions/setup-node@v4.0.1 + with: + node-version: 18.x + cache: pnpm + + - run: pnpm install --frozen-lockfile + + - name: Build + if: ${{ inputs.cacheKey == '' }} + run: pnpm build:backend + + - name: Restore cached build artifacts + if: ${{ inputs.cacheKey != '' }} + uses: actions/cache/restore@v4.0.0 + with: + path: ./packages/**/dist + key: ${{ inputs.cacheKey }} + + - name: Build benchmarks + working-directory: packages/workflow + run: pnpm benchmark:build + + - name: Benchmark workflow package + uses: CodSpeedHQ/action@v2 + with: + working-directory: packages/workflow + run: pnpm benchmark:run diff --git a/packages/workflow/package.json b/packages/workflow/package.json index d3839c66b71b9..0cd0d39311d34 100644 --- a/packages/workflow/package.json +++ b/packages/workflow/package.json @@ -24,6 +24,8 @@ "./*": "./*" }, "scripts": { + "benchmark:build": "tsc -p tsconfig.benchmark.json > /dev/null; tsc-alias -p tsconfig.benchmark.json", + "benchmark:run": "node dist/test/benchmark.js", "clean": "rimraf dist .turbo", "dev": "pnpm watch", "typecheck": "tsc", @@ -39,15 +41,18 @@ "dist/**/*" ], "devDependencies": { + "@codspeed/tinybench-plugin": "^3.1.0", "@types/deep-equal": "^1.0.1", "@types/express": "^4.17.21", "@types/jmespath": "^0.15.0", "@types/lodash": "^4.14.195", "@types/luxon": "^3.2.0", "@types/md5": "^2.3.5", - "@types/xml2js": "^0.4.14" + "@types/xml2js": "^0.4.14", + "tinybench": "^2.8.0" }, "dependencies": { + "@langchain/core": "0.1.61", "@n8n/tournament": "1.0.2", "@n8n_io/riot-tmpl": "4.0.0", "ast-types": "0.15.2", @@ -65,7 +70,6 @@ "recast": "0.21.5", "title-case": "3.0.3", "transliteration": "2.3.5", - "xml2js": "0.6.2", - "@langchain/core": "0.1.61" + "xml2js": "0.6.2" } } diff --git a/packages/workflow/test/Expression.test.ts b/packages/workflow/test/Expression.test.ts index f9d15fb0af5f3..1051b983456f0 100644 --- a/packages/workflow/test/Expression.test.ts +++ b/packages/workflow/test/Expression.test.ts @@ -7,11 +7,10 @@ import { Workflow } from '@/Workflow'; import * as Helpers from './Helpers'; import type { ExpressionTestEvaluation, ExpressionTestTransform } from './ExpressionFixtures/base'; import { baseFixtures } from './ExpressionFixtures/base'; -import type { INodeExecutionData } from '@/Interfaces'; import { extendSyntax } from '@/Extensions/ExpressionExtension'; import { ExpressionError } from '@/errors/expression.error'; import { setDifferEnabled, setEvaluator } from '@/ExpressionEvaluatorProxy'; -import { workflow } from './ExpressionExtensions/Helpers'; +import { evaluate } from './evaluate'; setDifferEnabled(true); @@ -173,13 +172,6 @@ for (const evaluator of ['tmpl', 'tournament'] as const) { }); describe('Test all expression value fixtures', () => { - const expression = workflow.expression; - - const evaluate = (value: string, data: INodeExecutionData[]) => { - const itemIndex = data.length === 0 ? -1 : 0; - return expression.getParameterValue(value, null, 0, itemIndex, 'node', data, 'manual', {}); - }; - for (const t of baseFixtures) { if (!t.tests.some((test) => test.type === 'evaluation')) { continue; diff --git a/packages/workflow/test/benchmark.ts b/packages/workflow/test/benchmark.ts new file mode 100644 index 0000000000000..33972ef6b4a13 --- /dev/null +++ b/packages/workflow/test/benchmark.ts @@ -0,0 +1,29 @@ +import { Bench } from 'tinybench'; +import { withCodSpeed } from '@codspeed/tinybench-plugin'; +import { baseFixtures } from './ExpressionFixtures/base'; +import { evaluate } from './evaluate'; +import type { INodeExecutionData } from '@/index'; + +function addExpressionEvaluationTasks(bench: Bench) { + for (const fixture of baseFixtures) { + for (const test of fixture.tests) { + if ('error' in test) continue; + + if (test.type === 'evaluation') { + const input = test.input.map((d) => ({ json: d })) as INodeExecutionData[]; + bench.add(fixture.expression, () => evaluate(fixture.expression, input)); + } + } + } +} + +async function main() { + const bench = withCodSpeed(new Bench({ time: 0, iterations: 1 })); // @TODO temp config + + addExpressionEvaluationTasks(bench); + + await bench.warmup(); + await bench.run(); +} + +void main(); diff --git a/packages/workflow/test/evaluate.ts b/packages/workflow/test/evaluate.ts new file mode 100644 index 0000000000000..8285ce1074787 --- /dev/null +++ b/packages/workflow/test/evaluate.ts @@ -0,0 +1,16 @@ +import { workflow } from './ExpressionExtensions/Helpers'; +import type { INodeExecutionData } from '@/Interfaces'; + +export const evaluate = (value: string, data: INodeExecutionData[]) => { + const itemIndex = data.length === 0 ? -1 : 0; + return workflow.expression.getParameterValue( + value, + null, + 0, + itemIndex, + 'node', + data, + 'manual', + {}, + ); +}; diff --git a/packages/workflow/tsconfig.benchmark.json b/packages/workflow/tsconfig.benchmark.json new file mode 100644 index 0000000000000..a7dcfb1d90298 --- /dev/null +++ b/packages/workflow/tsconfig.benchmark.json @@ -0,0 +1,13 @@ +{ + "extends": ["./tsconfig.json", "../../tsconfig.build.json"], + "compilerOptions": { + "outDir": "dist", + "tsBuildInfoFile": "dist/benchmark.tsbuildinfo", + "baseUrl": ".", + "paths": { + "@/*": ["./src/*"] + } + }, + "include": ["test/**/*.ts"], + "exclude": ["test/**/*.test.ts"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index aa2e0d688c10d..dd233e6e7dad6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -184,7 +184,7 @@ importers: dependencies: axios: specifier: 1.6.7 - version: 1.6.7(debug@4.3.4) + version: 1.6.7(debug@3.2.7) packages/@n8n/imap: dependencies: @@ -527,7 +527,7 @@ importers: version: 1.11.0 axios: specifier: 1.6.7 - version: 1.6.7(debug@4.3.4) + version: 1.6.7(debug@3.2.7) basic-auth: specifier: 2.0.1 version: 2.0.1 @@ -864,7 +864,7 @@ importers: version: 1.11.0 axios: specifier: 1.6.7 - version: 1.6.7(debug@4.3.4) + version: 1.6.7(debug@3.2.7) concat-stream: specifier: 2.0.0 version: 2.0.0 @@ -1121,7 +1121,7 @@ importers: version: 10.5.0(vue@3.4.21) axios: specifier: 1.6.7 - version: 1.6.7(debug@4.3.4) + version: 1.6.7(debug@3.2.7) chart.js: specifier: ^4.4.0 version: 4.4.0 @@ -1589,7 +1589,7 @@ importers: version: 0.15.2 axios: specifier: 1.6.7 - version: 1.6.7(debug@4.3.4) + version: 1.6.7(debug@3.2.7) callsites: specifier: 3.1.0 version: 3.1.0 @@ -1633,6 +1633,9 @@ importers: specifier: 0.6.2 version: 0.6.2 devDependencies: + '@codspeed/tinybench-plugin': + specifier: ^3.1.0 + version: 3.1.0(tinybench@2.8.0) '@types/deep-equal': specifier: ^1.0.1 version: 1.0.1 @@ -1654,6 +1657,9 @@ importers: '@types/xml2js': specifier: ^0.4.14 version: 0.4.14 + tinybench: + specifier: ^2.8.0 + version: 2.8.0 packages: @@ -9249,7 +9255,7 @@ packages: ts-dedent: 2.2.0 type-fest: 2.19.0 vue: 3.4.21(typescript@5.4.2) - vue-component-type-helpers: 2.0.16 + vue-component-type-helpers: 2.0.17 transitivePeerDependencies: - encoding - supports-color @@ -11512,7 +11518,7 @@ packages: /axios@1.6.7: resolution: {integrity: sha512-/hDJGff6/c7u0hDkvkGxR/oy6CbCs8ziCsC7SqmhjfozqiJGc8Z11wrv9z9lYfY4K8l+H9TpjcMDX0xOZmx+RA==} dependencies: - follow-redirects: 1.15.6(debug@4.3.4) + follow-redirects: 1.15.6(debug@3.2.7) form-data: 4.0.0 proxy-from-env: 1.1.0 transitivePeerDependencies: @@ -14976,7 +14982,6 @@ packages: optional: true dependencies: debug: 3.2.7(supports-color@5.5.0) - dev: false /follow-redirects@1.15.6(debug@4.3.4): resolution: {integrity: sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==} @@ -23124,10 +23129,6 @@ packages: resolution: {integrity: sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg==} dev: true - /tinybench@2.5.1: - resolution: {integrity: sha512-65NKvSuAVDP/n4CqH+a9w2kTlLReS9vhsAP06MWx+/89nMinJyB2icyl58RIcqCmIggpojIGeuJGhjU1aGMBSg==} - dev: true - /tinybench@2.8.0: resolution: {integrity: sha512-1/eK7zUnIklz4JUUlL+658n58XO2hHLQfSk1Zf2LKieUjxidN16eKFEoDEfjHc3ohofSSqK3X5yO6VGb6iW8Lw==} dev: true @@ -24213,7 +24214,7 @@ packages: picocolors: 1.0.0 std-env: 3.6.0 strip-literal: 2.0.0 - tinybench: 2.5.1 + tinybench: 2.8.0 tinypool: 0.8.2 vite: 5.1.6(sass@1.64.1) vite-node: 1.3.1 @@ -24321,8 +24322,8 @@ packages: resolution: {integrity: sha512-0vOfAtI67UjeO1G6UiX5Kd76CqaQ67wrRZiOe7UAb9Jm6GzlUr/fC7CV90XfwapJRjpCMaZFhv1V0ajWRmE9Dg==} dev: true - /vue-component-type-helpers@2.0.16: - resolution: {integrity: sha512-qisL/iAfdO++7w+SsfYQJVPj6QKvxp4i1MMxvsNO41z/8zu3KuAw9LkhKUfP/kcOWGDxESp+pQObWppXusejCA==} + /vue-component-type-helpers@2.0.17: + resolution: {integrity: sha512-2car49m8ciqg/JjgMBkx7o/Fd2A7fHESxNqL/2vJYFLXm4VwYO4yH0rexOi4a35vwNgDyvt17B07Vj126l9rAQ==} dev: true /vue-demi@0.14.5(vue@3.4.21):