From 82c986a8a2bd672c030ce7475b22c8c9080cf6ba Mon Sep 17 00:00:00 2001 From: Tomi Turtiainen <10324676+tomi@users.noreply.github.com> Date: Mon, 26 Aug 2024 11:38:38 +0300 Subject: [PATCH] feat(benchmark): Add benchmarking of pooled sqlite --- .github/workflows/benchmark-nightly.yml | 4 +- .../@n8n/benchmark/scripts/runInCloud.mjs | 58 ++++++++++++++----- .../benchmark/scripts/runOnVm/bootstrap.sh | 4 +- .../sqlite-legacy/docker-compose.yml | 17 ++++++ .../n8nSetups/sqlite/docker-compose.yml | 2 + .../benchmark/scripts/runOnVm/runOnVm.mjs | 49 +++++++++++----- packages/@n8n/benchmark/src/commands/run.ts | 1 + packages/@n8n/benchmark/src/config/config.ts | 6 ++ .../benchmark/src/testExecution/k6Executor.ts | 23 +++++--- .../src/testExecution/scenarioRunner.ts | 17 +++++- 10 files changed, 137 insertions(+), 44 deletions(-) create mode 100644 packages/@n8n/benchmark/scripts/runOnVm/n8nSetups/sqlite-legacy/docker-compose.yml diff --git a/.github/workflows/benchmark-nightly.yml b/.github/workflows/benchmark-nightly.yml index eb5aacbd0f81b..0100d12a5d634 100644 --- a/.github/workflows/benchmark-nightly.yml +++ b/.github/workflows/benchmark-nightly.yml @@ -60,10 +60,10 @@ jobs: - name: Run the benchmark with debug logging if: github.event.inputs.debug == 'true' - run: pnpm run-in-cloud sqlite --debug + run: pnpm run-in-cloud --debug working-directory: packages/@n8n/benchmark - name: Run the benchmark if: github.event.inputs.debug != 'true' - run: pnpm run-in-cloud sqlite + run: pnpm run-in-cloud working-directory: packages/@n8n/benchmark diff --git a/packages/@n8n/benchmark/scripts/runInCloud.mjs b/packages/@n8n/benchmark/scripts/runInCloud.mjs index 9d6f2b9556da3..e1c71d37d2614 100755 --- a/packages/@n8n/benchmark/scripts/runInCloud.mjs +++ b/packages/@n8n/benchmark/scripts/runInCloud.mjs @@ -15,7 +15,7 @@ // @ts-check import fs from 'fs'; import minimist from 'minimist'; -import { $, sleep, which } from 'zx'; +import { sleep, which } from 'zx'; import path from 'path'; import { SshClient } from './sshClient.mjs'; import { TerraformClient } from './terraformClient.mjs'; @@ -61,7 +61,6 @@ async function ensureDependencies() { } /** - * * @param {Config} config * @param {BenchmarkEnv} benchmarkEnv */ @@ -86,7 +85,32 @@ async function runBenchmarksOnVm(config, benchmarkEnv) { // Give some time for the VM to be ready await sleep(1000); - console.log('Running benchmarks...'); + if (config.n8nSetupToUse === 'all') { + const availableSetups = readAvailableN8nSetups(); + + for (const n8nSetup of availableSetups) { + await runBenchmarkForN8nSetup({ + config, + sshClient, + scriptsDir, + n8nSetup, + }); + } + } else { + await runBenchmarkForN8nSetup({ + config, + sshClient, + scriptsDir, + n8nSetup: config.n8nSetupToUse, + }); + } +} + +/** + * @param {{ config: Config; sshClient: any; scriptsDir: string; n8nSetup: string; }} opts + */ +async function runBenchmarkForN8nSetup({ config, sshClient, scriptsDir, n8nSetup }) { + console.log(`Running benchmarks for ${n8nSetup}...`); const runScriptPath = path.join(scriptsDir, 'runOnVm.mjs'); const flags = { @@ -100,7 +124,7 @@ async function runBenchmarksOnVm(config, benchmarkEnv) { .map(([key, value]) => `--${key}=${value}`) .join(' '); - await sshClient.ssh(`npx zx ${runScriptPath} ${flagsString} ${config.n8nSetupToUse}`, { + await sshClient.ssh(`npx zx ${runScriptPath} ${flagsString} ${n8nSetup}`, { // Test run should always log its output verbose: true, }); @@ -138,10 +162,15 @@ function readAvailableN8nSetups() { * @returns {Promise} */ async function parseAndValidateConfig() { - const args = minimist(process.argv.slice(2), { - boolean: ['debug'], + const args = minimist(process.argv.slice(3), { + boolean: ['debug', 'help'], }); + if (args.help) { + printUsage(); + process.exit(0); + } + const n8nSetupToUse = await getAndValidateN8nSetup(args); const isVerbose = args.debug || false; const n8nTag = args.n8nTag || process.env.N8N_DOCKER_TAG || 'latest'; @@ -163,10 +192,8 @@ async function parseAndValidateConfig() { async function getAndValidateN8nSetup(args) { // Last parameter is the n8n setup to use const n8nSetupToUse = args._[args._.length - 1]; - - if (!n8nSetupToUse) { - printUsage(); - process.exit(1); + if (!n8nSetupToUse || n8nSetupToUse === 'all') { + return 'all'; } const availableSetups = readAvailableN8nSetups(); @@ -182,19 +209,20 @@ async function getAndValidateN8nSetup(args) { function printUsage() { const availableSetups = readAvailableN8nSetups(); - console.log('Usage: zx scripts/runInCloud.mjs '); - console.log(' eg: zx scripts/runInCloud.mjs sqlite'); + console.log('Usage: zx scripts/runInCloud.mjs [n8n setup name]'); + console.log(' eg: zx scripts/runInCloud.mjs'); console.log(''); console.log('Options:'); + console.log( + ` [n8n setup name] Against which n8n setup to run the benchmarks. One of: ${['all', ...availableSetups].join(', ')}. Default is all`, + ); console.log(' --debug Enable verbose output'); console.log(' --n8nTag Docker tag for n8n image. Default is latest'); console.log(' --benchmarkTag Docker tag for benchmark cli image. Default is latest'); console.log( - ' --k6ApiToken API token for k6 cloud. Default is read from K6_API_TOKEN env var. If omitted, k6 cloud will not be used.', + ' --k6ApiToken API token for k6 cloud. Default is read from K6_API_TOKEN env var. If omitted, k6 cloud will not be used', ); console.log(''); - console.log('Available setups:'); - console.log(` ${availableSetups.join(', ')}`); } main().catch(console.error); diff --git a/packages/@n8n/benchmark/scripts/runOnVm/bootstrap.sh b/packages/@n8n/benchmark/scripts/runOnVm/bootstrap.sh index 9a5ffbe25319e..665e46d87722e 100644 --- a/packages/@n8n/benchmark/scripts/runOnVm/bootstrap.sh +++ b/packages/@n8n/benchmark/scripts/runOnVm/bootstrap.sh @@ -10,8 +10,8 @@ CURRENT_USER=$(whoami) # Mount the data disk if [ -d "/n8n" ]; then echo "Data disk already mounted. Clearing it..." - rm -rf /n8n/* - rm -rf /n8n/.[!.]* + sudo rm -rf /n8n/* + sudo rm -rf /n8n/.[!.]* else sudo mkdir -p /n8n sudo parted /dev/sdc --script mklabel gpt mkpart xfspart xfs 0% 100% diff --git a/packages/@n8n/benchmark/scripts/runOnVm/n8nSetups/sqlite-legacy/docker-compose.yml b/packages/@n8n/benchmark/scripts/runOnVm/n8nSetups/sqlite-legacy/docker-compose.yml new file mode 100644 index 0000000000000..02b61961f1b4a --- /dev/null +++ b/packages/@n8n/benchmark/scripts/runOnVm/n8nSetups/sqlite-legacy/docker-compose.yml @@ -0,0 +1,17 @@ +services: + n8n: + image: ghcr.io/n8n-io/n8n:${N8N_VERSION:-latest} + environment: + - N8N_DIAGNOSTICS_ENABLED=false + - N8N_USER_FOLDER=/n8n + ports: + - 5678:5678 + volumes: + - /n8n:/n8n + benchmark: + image: ghcr.io/n8n-io/n8n-benchmark:${N8N_BENCHMARK_VERSION:-latest} + depends_on: + - n8n + environment: + - N8N_BASE_URL=http://n8n:5678 + - K6_API_TOKEN=${K6_API_TOKEN} diff --git a/packages/@n8n/benchmark/scripts/runOnVm/n8nSetups/sqlite/docker-compose.yml b/packages/@n8n/benchmark/scripts/runOnVm/n8nSetups/sqlite/docker-compose.yml index 02b61961f1b4a..3d953b04d25bf 100644 --- a/packages/@n8n/benchmark/scripts/runOnVm/n8nSetups/sqlite/docker-compose.yml +++ b/packages/@n8n/benchmark/scripts/runOnVm/n8nSetups/sqlite/docker-compose.yml @@ -4,6 +4,8 @@ services: environment: - N8N_DIAGNOSTICS_ENABLED=false - N8N_USER_FOLDER=/n8n + - DB_SQLITE_POOL_SIZE=3 + - DB_SQLITE_ENABLE_WAL=true ports: - 5678:5678 volumes: diff --git a/packages/@n8n/benchmark/scripts/runOnVm/runOnVm.mjs b/packages/@n8n/benchmark/scripts/runOnVm/runOnVm.mjs index e3402e70c58bd..d8d3537dec398 100755 --- a/packages/@n8n/benchmark/scripts/runOnVm/runOnVm.mjs +++ b/packages/@n8n/benchmark/scripts/runOnVm/runOnVm.mjs @@ -2,23 +2,19 @@ /** * This script runs the benchmarks using a given docker compose setup */ +// @ts-check +import path from 'path'; +import { $, argv, fs } from 'zx'; -import { $ } from 'zx'; - -const [n8nSetupToUse] = argv._; - -if (!n8nSetupToUse) { - printUsage(); - process.exit(1); -} - -function printUsage() { - console.log('Usage: zx runOnVm.mjs '); - console.log(' eg: zx runOnVm.mjs sqlite'); -} +const paths = { + n8nSetupsDir: path.join(__dirname, 'n8nSetups'), +}; async function main() { - const composeFilePath = path.join(__dirname, 'n8nSetups', n8nSetupToUse); + const [n8nSetupToUse] = argv._; + validateN8nSetup(n8nSetupToUse); + + const composeFilePath = path.join(paths.n8nSetupsDir, n8nSetupToUse); const n8nTag = argv.n8nDockerTag || process.env.N8N_DOCKER_TAG || 'latest'; const benchmarkTag = argv.benchmarkDockerTag || process.env.BENCHMARK_DOCKER_TAG || 'latest'; const k6ApiToken = argv.k6ApiToken || process.env.K6_API_TOKEN || undefined; @@ -30,6 +26,7 @@ async function main() { N8N_VERSION: n8nTag, BENCHMARK_VERSION: benchmarkTag, K6_API_TOKEN: k6ApiToken, + N8N_BENCHMARK_SCENARIO_NAME_PREFIX: n8nSetupToUse, }, }); @@ -52,4 +49,28 @@ async function dumpN8nInstanceLogs($$) { await $$`docker-compose logs n8n`; } +function printUsage() { + const availableSetups = getAllN8nSetups(); + console.log('Usage: zx runOnVm.mjs '); + console.log(` eg: zx runOnVm.mjs ${availableSetups[0]}`); + console.log(''); + console.log('Available setups:'); + console.log(availableSetups.join(', ')); +} + +/** + * @returns {string[]} + */ +function getAllN8nSetups() { + return fs.readdirSync(paths.n8nSetupsDir); +} + +function validateN8nSetup(givenSetup) { + const availableSetups = getAllN8nSetups(); + if (!availableSetups.includes(givenSetup)) { + printUsage(); + process.exit(1); + } +} + main(); diff --git a/packages/@n8n/benchmark/src/commands/run.ts b/packages/@n8n/benchmark/src/commands/run.ts index 3d9a3bf803e45..4476dbfbe0465 100644 --- a/packages/@n8n/benchmark/src/commands/run.ts +++ b/packages/@n8n/benchmark/src/commands/run.ts @@ -34,6 +34,7 @@ export default class RunCommand extends Command { email: config.get('n8n.user.email'), password: config.get('n8n.user.password'), }, + config.get('scenarioNamePrefix'), ); const allScenarios = scenarioLoader.loadAll(config.get('testScenariosPath')); diff --git a/packages/@n8n/benchmark/src/config/config.ts b/packages/@n8n/benchmark/src/config/config.ts index 2fa8d9249bf8f..ad8be8b4ea398 100644 --- a/packages/@n8n/benchmark/src/config/config.ts +++ b/packages/@n8n/benchmark/src/config/config.ts @@ -31,6 +31,12 @@ const configSchema = { }, }, }, + scenarioNamePrefix: { + doc: 'Prefix for the scenario name', + format: String, + default: 'Unnamed', + env: 'N8N_BENCHMARK_SCENARIO_NAME_PREFIX', + }, k6: { executablePath: { doc: 'The path to the k6 binary', diff --git a/packages/@n8n/benchmark/src/testExecution/k6Executor.ts b/packages/@n8n/benchmark/src/testExecution/k6Executor.ts index a9ab325b5f572..07c0e4787e102 100644 --- a/packages/@n8n/benchmark/src/testExecution/k6Executor.ts +++ b/packages/@n8n/benchmark/src/testExecution/k6Executor.ts @@ -9,6 +9,11 @@ export type K6ExecutorOpts = { n8nApiBaseUrl: string; }; +export type K6RunOpts = { + /** Name of the scenario run. Used e.g. when the run is reported to k6 cloud */ + scenarioRunName: string; +}; + /** * Flag for the k6 CLI. * @example ['--duration', '1m'] @@ -36,8 +41,8 @@ export function handleSummary(data) { constructor(private readonly opts: K6ExecutorOpts) {} - async executeTestScenario(scenario: Scenario) { - const augmentedTestScriptPath = this.augmentSummaryScript(scenario); + async executeTestScenario(scenario: Scenario, { scenarioRunName }: K6RunOpts) { + const augmentedTestScriptPath = this.augmentSummaryScript(scenario, scenarioRunName); const runDirPath = path.dirname(augmentedTestScriptPath); const flags: K6CliFlag[] = [['--quiet'], ['--duration', '1m'], ['--vus', '5']]; @@ -62,7 +67,7 @@ export function handleSummary(data) { console.log((chunk as Buffer).toString()); } - this.loadEndOfTestSummary(runDirPath, scenario.name); + this.loadEndOfTestSummary(runDirPath, scenarioRunName); } /** @@ -70,24 +75,24 @@ export function handleSummary(data) { * * @returns Absolute path to the augmented test script */ - private augmentSummaryScript(scenario: Scenario) { + private augmentSummaryScript(scenario: Scenario, scenarioRunName: string) { const fullTestScriptPath = path.join(scenario.scenarioDirPath, scenario.scriptPath); const testScript = fs.readFileSync(fullTestScriptPath, 'utf8'); - const summaryScript = this.handleSummaryScript.replace('{{scenarioName}}', scenario.name); + const summaryScript = this.handleSummaryScript.replace('{{scenarioName}}', scenarioRunName); const augmentedTestScript = `${testScript}\n\n${summaryScript}`; - const tempFilePath = tmpfile(`${scenario.name}.ts`, augmentedTestScript); + const tempFilePath = tmpfile(`${scenarioRunName}.ts`, augmentedTestScript); return tempFilePath; } - private loadEndOfTestSummary(dir: string, scenarioName: string): K6EndOfTestSummary { - const summaryReportPath = path.join(dir, `${scenarioName}.summary.json`); + private loadEndOfTestSummary(dir: string, scenarioRunName: string): K6EndOfTestSummary { + const summaryReportPath = path.join(dir, `${scenarioRunName}.summary.json`); const summaryReport = fs.readFileSync(summaryReportPath, 'utf8'); try { - return JSON.parse(summaryReport); + return JSON.parse(summaryReport) as K6EndOfTestSummary; } catch (error) { throw new Error(`Failed to parse the summary report at ${summaryReportPath}`); } diff --git a/packages/@n8n/benchmark/src/testExecution/scenarioRunner.ts b/packages/@n8n/benchmark/src/testExecution/scenarioRunner.ts index e9a9dfc396642..4d70520910870 100644 --- a/packages/@n8n/benchmark/src/testExecution/scenarioRunner.ts +++ b/packages/@n8n/benchmark/src/testExecution/scenarioRunner.ts @@ -17,6 +17,7 @@ export class ScenarioRunner { email: string; password: string; }, + private readonly scenarioPrefix: string, ) {} async runManyScenarios(scenarios: Scenario[]) { @@ -38,13 +39,25 @@ export class ScenarioRunner { } private async runSingleTestScenario(testDataImporter: ScenarioDataImporter, scenario: Scenario) { - console.log('Running scenario:', scenario.name); + const scenarioRunName = this.formTestScenarioRunName(scenario); + console.log('Running scenario:', scenarioRunName); console.log('Loading and importing data'); const testData = await this.dataLoader.loadDataForScenario(scenario); await testDataImporter.importTestScenarioData(testData.workflows); console.log('Executing scenario script'); - await this.k6Executor.executeTestScenario(scenario); + await this.k6Executor.executeTestScenario(scenario, { + scenarioRunName, + }); + } + + /** + * Forms a name for the scenario by combining prefix and scenario name. + * The benchmarks are ran against different n8n setups, so we use the + * prefix to differentiate between them. + */ + private formTestScenarioRunName(scenario: Scenario) { + return `${this.scenarioPrefix}-${scenario.name}`; } }