From 08e1ea6978fc0994b149ad26cfa83948b39f21e2 Mon Sep 17 00:00:00 2001 From: Ravi Sawlani Date: Tue, 14 Mar 2023 17:03:30 +0530 Subject: [PATCH] Features/use worker threads (#3592) --- lib/reporter/global-reporter.js | 6 +- lib/reporter/index.js | 12 +- lib/runner/cli/cli.js | 6 +- lib/runner/concurrency/index.js | 220 +++++++++++--- lib/runner/concurrency/task.js | 23 ++ lib/runner/concurrency/worker-process.js | 57 ++++ lib/runner/concurrency/worker-task.js | 95 ++++++ lib/settings/defaults.js | 4 + lib/settings/settings.js | 10 +- lib/testsuite/index.js | 31 +- .../service-builders/base-service.js | 2 +- package-lock.json | 128 ++++++++ package.json | 1 + test/src/cli/testParallelExecution.js | 287 ++++++++++++++++-- test/src/cli/testParallelExecutionExitCode.js | 59 +++- test/src/index/testProgrammaticApis.js | 2 +- .../index/transport/testSeleniumTransport.js | 6 +- test/src/runner/cli/testCliRunnerParallel.js | 40 ++- test/src/runner/testRunWithGlobalHooks.js | 122 ++++++-- 19 files changed, 981 insertions(+), 130 deletions(-) create mode 100644 lib/runner/concurrency/task.js create mode 100644 lib/runner/concurrency/worker-process.js create mode 100644 lib/runner/concurrency/worker-task.js diff --git a/lib/reporter/global-reporter.js b/lib/reporter/global-reporter.js index a01b8896ba..fdc2969b6c 100644 --- a/lib/reporter/global-reporter.js +++ b/lib/reporter/global-reporter.js @@ -81,7 +81,7 @@ module.exports = class GlobalReporter { setupChildProcessListener(emitter) { if (!Concurrency.isTestWorker()) { emitter.on('message', data => { - data = JSON.parse(data); + data = this.settings.use_child_process ? JSON.parse(data) : data; this.addTestSuiteResults(data.results, data.httpOutput); }); } @@ -108,7 +108,7 @@ module.exports = class GlobalReporter { } print() { - if (Concurrency.isChildProcess() || !this.settings.output) { + if (Concurrency.isWorker() || !this.settings.output) { return this; } @@ -381,7 +381,7 @@ module.exports = class GlobalReporter { } save() { - if (Concurrency.isChildProcess()) { + if (Concurrency.isWorker()) { return Promise.resolve(); } diff --git a/lib/reporter/index.js b/lib/reporter/index.js index de41920d2a..a0f967babc 100644 --- a/lib/reporter/index.js +++ b/lib/reporter/index.js @@ -288,9 +288,9 @@ class Reporter extends SimplifiedReporter { let currentTestResult = this.testResults.currentTestResult; const Concurrency = require('../runner/concurrency'); - const isChildProcess = Concurrency.isChildProcess(); - if (isChildProcess || !this.settings.detailed_output || this.unitTestsMode) { - this.printSimplifiedTestResult(ok, elapsedTime, isChildProcess); + const isWorker = Concurrency.isWorker(); + if (isWorker || !this.settings.detailed_output || this.unitTestsMode) { + this.printSimplifiedTestResult(ok, elapsedTime, isWorker); return; } @@ -310,14 +310,14 @@ class Reporter extends SimplifiedReporter { /** * @param {boolean} ok * @param {number} elapsedTime - * @param {boolean} isChildProcess + * @param {boolean} isWorker */ - printSimplifiedTestResult(ok, elapsedTime, isChildProcess) { + printSimplifiedTestResult(ok, elapsedTime, isWorker) { const {currentTest} = this; let result = [colors[ok ? 'green': 'red'](Utils.symbols[ok ? 'ok' : 'fail'])]; if (!this.unitTestsMode) { - if (isChildProcess) { + if (isWorker) { result.push(colors.white(process.env.__NIGHTWATCH_ENV, colors.background.black)); } diff --git a/lib/runner/cli/cli.js b/lib/runner/cli/cli.js index b15baf8c03..cbe454eaba 100644 --- a/lib/runner/cli/cli.js +++ b/lib/runner/cli/cli.js @@ -354,7 +354,7 @@ class CliRunner { } runGlobalHook(key, args = [], isParallelHook = false) { - if (isParallelHook && Concurrency.isChildProcess() || !isParallelHook && !Concurrency.isChildProcess()) { + if (isParallelHook && Concurrency.isWorker() || !isParallelHook && !Concurrency.isWorker()) { return this.globals.runGlobalHook(key, args); } @@ -395,7 +395,7 @@ class CliRunner { if (isMobile(desiredCapabilities)) { - if (Concurrency.isChildProcess()) { + if (Concurrency.isWorker()) { Logger.info('Disabling parallelism while running tests on mobile platform'); } @@ -595,7 +595,7 @@ class CliRunner { return err; }) .then(errorOrFailed => { - if (typeof done == 'function' && !Concurrency.isChildProcess()) { + if (typeof done == 'function' && !Concurrency.isWorker()) { if (errorOrFailed instanceof Error) { return done(errorOrFailed); } diff --git a/lib/runner/concurrency/index.js b/lib/runner/concurrency/index.js index e1f625b497..1eb6890eb4 100644 --- a/lib/runner/concurrency/index.js +++ b/lib/runner/concurrency/index.js @@ -1,8 +1,9 @@ const EventEmitter = require('events'); const Utils = require('../../utils'); -const ChildProcess = require('./child-process.js'); const {isObject, isNumber} = Utils; const lodashClone = require('lodash.clone'); +const ChildProcess = require('./child-process.js'); +const WorkerPool = require('./worker-process.js'); const {Logger} = Utils; class Concurrency extends EventEmitter { @@ -11,6 +12,7 @@ class Concurrency extends EventEmitter { this.argv = argv; this.settings = lodashClone(settings, true); + this.useChildProcess = settings.use_child_process; this.childProcessOutput = {}; this.globalExitCode = 0; this.testWorkersEnabled = typeof isTestWorkerEnabled !== undefined ? isTestWorkerEnabled : settings.testWorkersEnabled; @@ -18,7 +20,7 @@ class Concurrency extends EventEmitter { } static getChildProcessArgs(envs) { - let childProcessArgs = []; + const childProcessArgs = []; let arg; for (let i = 2; i < process.argv.length; i++) { @@ -33,13 +35,101 @@ class Concurrency extends EventEmitter { return childProcessArgs; } + + /** + * + * @param {String} label + * @param {Array} args + * @param {Array} extraArgs + * @param {Number} index + * @return {ChildProcess} + */ + createChildProcess(label, args = [], extraArgs = [], index = 0) { + this.childProcessOutput[label] = []; + const childArgs = args.slice().concat(extraArgs); + + const childProcess = new ChildProcess(label, index, this.childProcessOutput[label], this.settings, childArgs); + childProcess.on('message', data => { + this.emit('message', data); + }); + + return childProcess; + } + + /** + * + * @param {ChildProcess} childProcess + * @param {String} outputLabel + * @param {Array} availColors + * @param {String} type + * @return {Promise} + */ + runChildProcess(childProcess, outputLabel, availColors, type) { + return childProcess.setLabel(outputLabel) + .run(availColors, type) + .then(exitCode => { + if (exitCode > 0) { + this.globalExitCode = exitCode; + } + }); + } + + + /** + * + * @param {Array} envs + * @param {Array} [modules] + * @return {Promise} + */ + runChildProcesses(envs, modules) { + const availColors = Concurrency.getAvailableColors(); + const args = Concurrency.getChildProcessArgs(envs); + + if (this.testWorkersEnabled) { + const jobs = []; + if (envs.length > 0) { + envs.forEach((env) => { + const envModules = modules[env]; + const jobList = envModules.map((module) => { + return { + env, + module + }; + }); + jobs.push(...jobList); + }); + } else { + const jobList = modules.map((module) => { + return { + module + }; + }); + jobs.push(...jobList); + } + + return this.runMultipleTestProcess(jobs, args, availColors); + } + + return this.runProcessTestEnvironment(envs, args, availColors); + } + /** * * @param {Array} envs * @param {Array} modules */ runMultiple(envs = [], modules) { - return this.runChildProcesses(envs, modules) + if (this.useChildProcess) { + return this.runChildProcesses(envs, modules) + .then(_ => { + return this.globalExitCode; + }); + } + + return this.runWorkerProcesses(envs, modules) + .catch(_ => { + this.globalExitCode = 1; + }) .then(_ => { return this.globalExitCode; }); @@ -51,9 +141,8 @@ class Concurrency extends EventEmitter { * @param {Array} [modules] * @return {Promise} */ - runChildProcesses(envs, modules) { + runWorkerProcesses(envs, modules) { const availColors = Concurrency.getAvailableColors(); - const args = Concurrency.getChildProcessArgs(envs); if (this.testWorkersEnabled) { const jobs = []; @@ -77,10 +166,10 @@ class Concurrency extends EventEmitter { jobs.push(...jobList); } - return this.runMultipleTestWorkers(jobs, args, availColors); + return this.runMultipleTestWorkers(jobs, availColors); } - return this.runTestEnvironments(envs, args, availColors); + return this.runWorkerTestEnvironments(envs, availColors); } /** @@ -90,7 +179,7 @@ class Concurrency extends EventEmitter { * @param {Array} availColors * @return {Promise} */ - runTestEnvironments(envs, args, availColors) { + runProcessTestEnvironment(envs, args, availColors) { return Promise.all(envs.map((environment, index) => { const extraArgs = ['--env', environment]; if (this.isSafariEnvPresent) { @@ -104,11 +193,38 @@ class Concurrency extends EventEmitter { /** * - * @param {Array} modules - * @param {Array} args + * @param {Array} envs * @param {Array} availColors + * @return {Promise} */ - runMultipleTestWorkers(modules, args, availColors) { + runWorkerTestEnvironments(envs, availColors) { + + let maxWorkerCount = this.getTestWorkersCount(); + const remaining = envs.length; + + maxWorkerCount = Math.min(maxWorkerCount, remaining); + const workerPool = this.setupWorkerPool(['--parallel-mode'], this.settings, maxWorkerCount); + + envs.forEach((env) => { + const workerArgv = {...this.argv}; + workerArgv.env = workerArgv.e = env; + workerPool.addTask({ + argv: workerArgv, + settings: this.settings, + label: `${env} environment`, + colors: availColors + }); + }); + + return Promise.all(workerPool.tasks); + } + /** + * + * @param {Array} modules + * @param {Array} args + * @param {Array} availColors + */ + runMultipleTestProcess(modules, args, availColors) { let maxWorkerCount = this.getTestWorkersCount(); let remaining = modules.length; @@ -145,43 +261,58 @@ class Concurrency extends EventEmitter { }); } - /** * - * @param {ChildProcess} childProcess - * @param {String} outputLabel + * @param {Array} modules * @param {Array} availColors - * @param {String} type - * @return {Promise} */ - runChildProcess(childProcess, outputLabel, availColors, type) { - return childProcess.setLabel(outputLabel) - .run(availColors, type) - .then(exitCode => { - if (exitCode > 0) { - this.globalExitCode = exitCode; - } + runMultipleTestWorkers(modules, availColors) { + let maxWorkerCount = this.getTestWorkersCount(); + const remaining = modules.length; + + maxWorkerCount = Math.min(maxWorkerCount, remaining); + + if (this.settings.output) { + Logger.info(`Launching up to ${maxWorkerCount} concurrent test worker processes...\n`); + } + + const workerPool = this.setupWorkerPool(['--test-worker', '--parallel-mode'], this.settings, maxWorkerCount); + + modules.forEach(({module, env}) => { + let outputLabel = Utils.getModuleKey(module, this.settings.src_folders, modules); + outputLabel = env ? `${env}: ${outputLabel}` : outputLabel; + + const workerArgv = {...this.argv}; + workerArgv._source = [module]; + workerArgv.env = env; + workerArgv['test-worker'] = true; + + return workerPool.addTask({ + argv: workerArgv, + settings: this.settings, + label: outputLabel, + colors: availColors }); + }); + + + return Promise.all(workerPool.tasks); } + /** - * - * @param {String} label - * @param {Array} args - * @param {Array} extraArgs - * @param {Number} index - * @return {ChildProcess} + * + * @param {Array} args + * @param {Object} settings + * @param {Number} maxWorkerCount */ - createChildProcess(label, args = [], extraArgs = [], index = 0) { - this.childProcessOutput[label] = []; - const childArgs = args.slice().concat(extraArgs); - - const childProcess = new ChildProcess(label, index, this.childProcessOutput[label], this.settings, childArgs); - childProcess.on('message', data => { + setupWorkerPool(args, settings, maxWorkerCount) { + const workerPool = new WorkerPool(args, settings, maxWorkerCount); + workerPool.on('message', data => { this.emit('message', data); }); - return childProcess; + return workerPool; } /** @@ -220,11 +351,12 @@ class Concurrency extends EventEmitter { let workers = require('os').cpus().length; if (isObject(test_workers) && isNumber(test_workers.workers)) { - if (workers < test_workers) { + if (workers < test_workers.workers) { // eslint-disable-next-line no-console Logger.warn(`Number of max workers is set to ${test_workers.workers} while the number of cpu cores is ${workers}. This can cause performance issues...\n`); + } else { + workers = test_workers.workers; } - workers = test_workers.workers; } return workers; @@ -251,7 +383,7 @@ class Concurrency extends EventEmitter { workers += 1; if (queue.length) { - let item = queue.shift(); + const item = queue.shift(); fn(item.env, item.module, index++, next); } else { done(); @@ -263,7 +395,7 @@ class Concurrency extends EventEmitter { } static getAvailableColors() { - let availColorPairs = [ + const availColorPairs = [ ['red', 'light_gray'], ['green', 'black'], ['blue', 'light_gray'], @@ -287,16 +419,16 @@ class Concurrency extends EventEmitter { return availColorPairs; } - static isChildProcess() { - return process.env.__NIGHTWATCH_PARALLEL_MODE === '1'; + static isWorker() { + return process.env.__NIGHTWATCH_PARALLEL_MODE === '1' || WorkerPool.isWorkerThread; } static isTestWorker(argv = {}) { - return Concurrency.isChildProcess() && argv['test-worker']; + return Concurrency.isWorker() && argv['test-worker']; } static isMasterProcess() { - return !Concurrency.isChildProcess(); + return !Concurrency.isWorker(); } } diff --git a/lib/runner/concurrency/task.js b/lib/runner/concurrency/task.js new file mode 100644 index 0000000000..25b89aaf44 --- /dev/null +++ b/lib/runner/concurrency/task.js @@ -0,0 +1,23 @@ +const Nightwatch = require('../../index'); +function runWorkerTask({argv, port1}) { + const writeData = (data) => { + data = data.toString().trim(); + + if (data){ + port1.postMessage({ + type: 'stdout', + data: data + }); + } + }; + + process.stdout.write = writeData; + process.stderr.write = writeData; + + //send reports to main thread using message port + process.port = port1; + + return Nightwatch.runTests(argv, {}); +} + +module.exports = runWorkerTask; \ No newline at end of file diff --git a/lib/runner/concurrency/worker-process.js b/lib/runner/concurrency/worker-process.js new file mode 100644 index 0000000000..a890a74386 --- /dev/null +++ b/lib/runner/concurrency/worker-process.js @@ -0,0 +1,57 @@ +const path = require('path'); +const Piscina= require('piscina'); +const {isWorkerThread} = Piscina; +const EventEmitter = require('events'); +const WorkerTask = require('./worker-task'); + +const WORKER_FILE = path.resolve(__dirname, 'task.js'); + +class WorkerPool extends EventEmitter { + + + static get isWorkerThread() { + return isWorkerThread; + } + + get tasks() { + return this.__tasks; + } + + set tasks(tasks) { + this.__tasks = tasks; + } + + constructor(args, settings, maxWorkerCount) { + super(); + + this.settings = settings; + this.piscina = new Piscina({ + filename: WORKER_FILE, + maxThreads: maxWorkerCount, + argv: args, + env: { + __NIGHTWATCH_PARALLEL_MODE: '1' + } + }); + + this.__tasks = []; + this.index = 0; + } + + /** + * adds a task to running worker queue + */ + addTask({label, argv, colors} = {}) { + const workerTask = new WorkerTask({piscina: this.piscina, index: this.index, label, argv, settings: this.settings}); + + workerTask.on('message', (data) => { + this.emit('message', data); + }); + + this.index++; + this.__tasks.push(workerTask.runWorkerTask(colors)); + + } +} + +module.exports = WorkerPool; diff --git a/lib/runner/concurrency/worker-task.js b/lib/runner/concurrency/worker-task.js new file mode 100644 index 0000000000..31901906d1 --- /dev/null +++ b/lib/runner/concurrency/worker-task.js @@ -0,0 +1,95 @@ +const boxen = require('boxen'); +const {MessageChannel} = require('worker_threads'); +const {Logger, symbols} = require('../../utils'); +const EventEmitter = require('events'); + +let prevIndex = 0; + +class WorkerTask extends EventEmitter { + + static get prevIndex() { + return prevIndex; + } + + static set prevIndex(val) { + prevIndex = val; + } + + constructor({piscina, index, label, settings, argv, task_ouput}) { + super(); + + this.task_output = task_ouput || []; + this.piscina = piscina; + this.label = label; + this.settings = settings; + this.argv = argv; + this.index = index; + this.task_label = ''; + } + + printLog(msg) { + if (this.settings.output) { + // eslint-disable-next-line no-console + console.info(msg); + } + } + + setlabel(colorPair) { + this.task_label = this.settings.disable_colors ? ` ${this.label} ` : Logger.colors[colorPair[1]](` ${this.label} `, Logger.colors.background[colorPair[0]]); + } + + writeToStdOut(data) { + let output = ''; + + if (WorkerTask.prevIndex !== this.index) { + WorkerTask.prevIndex = this.index; + if (this.settings.live_output) { + output += '\n'; + } + } + + if (this.settings.output && (this.settings.detailed_ouput || !this.settings.silent)) { + const lines = data.split('\n').map(line => this.env_label + ' ' + line + ' '); + data = lines.join('\n'); + } + + output += data; + + if (this.settings.live_output) { + process.stdout.write(output + '\n'); + } else { + this.task_output.push(output); + } + } + + async runWorkerTask(colors, type) { + this.availColors = colors; + const colorPair = this.availColors[this.index%4]; + this.setlabel(colorPair); + + this.printLog('Running '+ Logger.colors[colorPair[1]](` ${this.task_label} `, Logger.colors.background[colorPair[0]])); + + const {port1, port2} = new MessageChannel(); + port2.onmessage = ({data: result}) => { + switch (result.type) { + case 'testsuite_finished': + result.itemKey = this.label, + this.emit('message', result); + break; + + case 'stdout': + this.writeToStdOut(result.data); + break; + } + }; + port2.unref(); + + return this.piscina.run({argv: this.argv, port1}, {transferList: [port1]}) + .then(failures => { + // eslint-disable-next-line no-console + console.log(boxen(this.task_output.join('\n'), {title: `────────────────── ${failures ? symbols.fail : symbols.ok} ${this.task_label}`, padding: 1, borderColor: 'cyan'})); + }); + } +} + +module.exports = WorkerTask; \ No newline at end of file diff --git a/lib/settings/defaults.js b/lib/settings/defaults.js index b9423adc8e..85f60e4350 100644 --- a/lib/settings/defaults.js +++ b/lib/settings/defaults.js @@ -206,6 +206,10 @@ module.exports = { // Specifies which test runner to use: default|mocha test_runner: 'default', + // Specifies which implementation to use for Concurrency: child-process|worker-threads + use_child_process: false, + + // Defines options used to connect to the WebDriver/Selenium server webdriver: { start_process: false, diff --git a/lib/settings/settings.js b/lib/settings/settings.js index 9905a2c238..e7828e7036 100644 --- a/lib/settings/settings.js +++ b/lib/settings/settings.js @@ -103,13 +103,7 @@ class Settings { get testWorkersEnabled() { const {test_workers} = this.settings; - if (this.argv.parallel === false || this.argv.parallel === 'false' - || this.argv.serial === true - || isNumber(this.argv.workers) && this.argv.workers === 1) { - return false; - } - - return this.argv.parallel === true || test_workers === true || isObject(test_workers) && test_workers.enabled; + return this.argv.serial ? false : (this.argv.parallel || test_workers === true || (test_workers && test_workers.enabled && test_workers.workers !== 1)); } /** @@ -306,7 +300,7 @@ class Settings { setParallelMode() { const Concurrency = require('../runner/concurrency'); - if (Concurrency.isChildProcess()) { + if (Concurrency.isWorker()) { this.settings.parallel_mode = true; } diff --git a/lib/testsuite/index.js b/lib/testsuite/index.js index f9cf976779..89eb041678 100644 --- a/lib/testsuite/index.js +++ b/lib/testsuite/index.js @@ -644,19 +644,31 @@ class TestSuite { return this.testSuiteFinished(failures); } - async testSuiteFinished(failures) { - this.reporter.testSuiteFinished(); - this.currentRunnable = null; - - if (Concurrency.isChildProcess() && typeof process.send == 'function') { + sendReportToParentWorker() { + if (this.settings.use_child_process && typeof process.send === 'function') { process.send(SafeJSON.stringify({ type: 'testsuite_finished', itemKey: process.env.__NIGHTWATCH_ENV_LABEL, results: this.reporter.exportResults(), httpOutput: Logger.collectOutput() })); + } else if (process.port && typeof process.port.postMessage === 'function') { + process.port.postMessage({ + type: 'testsuite_finished', + results: this.reporter.exportResults(), + httpOutput: Logger.collectOutput() + }); } + } + + async testSuiteFinished(failures) { + this.reporter.testSuiteFinished(); + this.currentRunnable = null; + if (Concurrency.isWorker()) { + this.sendReportToParentWorker(); + } + try { let lastError = failures; if (lastError === true) { @@ -1062,13 +1074,8 @@ class TestSuite { console.log(Logger.colors.green(`Testsuite "${this.context.moduleName}" is disabled, skipping...`)); // send report even if test is skipped - if (Concurrency.isChildProcess() && typeof process.send == 'function') { - process.send(SafeJSON.stringify({ - type: 'testsuite_finished', - itemKey: process.env.__NIGHTWATCH_ENV_LABEL, - results: this.reporter.exportResults(), - httpOutput: Logger.collectOutput() - })); + if (Concurrency.isWorker()) { + this.sendReportToParentWorker(); } return Promise.resolve(); diff --git a/lib/transport/selenium-webdriver/service-builders/base-service.js b/lib/transport/selenium-webdriver/service-builders/base-service.js index 7cf7f991c7..fee27a5520 100644 --- a/lib/transport/selenium-webdriver/service-builders/base-service.js +++ b/lib/transport/selenium-webdriver/service-builders/base-service.js @@ -217,7 +217,7 @@ class BaseService { } needsSinkProcess() { - return !Concurrency.isChildProcess(); + return !Concurrency.isWorker(); } hasSinkSupport() { diff --git a/package-lock.json b/package-lock.json index ad05366c5c..ac041716ef 100644 --- a/package-lock.json +++ b/package-lock.json @@ -37,6 +37,7 @@ "nightwatch-axe-verbose": "^2.1.0", "open": "8.4.0", "ora": "5.4.1", + "piscina": "^3.2.0", "selenium-webdriver": "4.6.1", "semver": "7.3.5", "stacktrace-parser": "0.1.10", @@ -102,6 +103,11 @@ "node": ">=6.0.0" } }, + "node_modules/@assemblyscript/loader": { + "version": "0.10.1", + "resolved": "https://registry.npmjs.org/@assemblyscript/loader/-/loader-0.10.1.tgz", + "integrity": "sha512-H71nDOOL8Y7kWRLqf6Sums+01Q5msqBW2KhDUTemh1tvY04eSkSXrK0uj/4mmY0Xr16/3zyZmsrxN7CKuRbNRg==" + }, "node_modules/@babel/code-frame": { "version": "7.18.6", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.18.6.tgz", @@ -2837,6 +2843,11 @@ "node": ">=0.10.0" } }, + "node_modules/eventemitter-asyncresource": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/eventemitter-asyncresource/-/eventemitter-asyncresource-1.0.0.tgz", + "integrity": "sha512-39F7TBIV0G7gTelxwbEqnwhp90eqCPON1k0NwNfwhgKn4Co4ybUbj2pECcXT0B3ztRKZ7Pw1JujUUgmQJHcVAQ==" + }, "node_modules/ext": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/ext/-/ext-1.7.0.tgz", @@ -3407,6 +3418,21 @@ "node": ">=8" } }, + "node_modules/hdr-histogram-js": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/hdr-histogram-js/-/hdr-histogram-js-2.0.3.tgz", + "integrity": "sha512-Hkn78wwzWHNCp2uarhzQ2SGFLU3JY8SBDDd3TAABK4fc30wm+MuPOrg5QVFVfkKOQd6Bfz3ukJEI+q9sXEkK1g==", + "dependencies": { + "@assemblyscript/loader": "^0.10.1", + "base64-js": "^1.2.0", + "pako": "^1.0.3" + } + }, + "node_modules/hdr-histogram-percentiles-obj": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/hdr-histogram-percentiles-obj/-/hdr-histogram-percentiles-obj-3.0.0.tgz", + "integrity": "sha512-7kIufnBqdsBGcSZLPJwqHT3yhk1QTsSlFsVD3kx5ixH/AlgBs9yM1q6DPhXZ8f8gtdqgh7N7/5btRLpQsS2gHw==" + }, "node_modules/he": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", @@ -5139,6 +5165,20 @@ "integrity": "sha512-CXdUiJembsNjuToQvxayPZF9Vqht7hewsvy2sOWafLvi2awflj9mOC6bHIg50orX8IJvWKY9wYQ/zB2kogPslQ==", "dev": true }, + "node_modules/nice-napi": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/nice-napi/-/nice-napi-1.0.2.tgz", + "integrity": "sha512-px/KnJAJZf5RuBGcfD+Sp2pAKq0ytz8j+1NehvgIGFkvtvFrDM3T8E4x/JJODXK9WZow8RRGrbA9QQ3hs+pDhA==", + "hasInstallScript": true, + "optional": true, + "os": [ + "!win32" + ], + "dependencies": { + "node-addon-api": "^3.0.0", + "node-gyp-build": "^4.2.2" + } + }, "node_modules/nightwatch-axe-verbose": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/nightwatch-axe-verbose/-/nightwatch-axe-verbose-2.1.0.tgz", @@ -5172,6 +5212,23 @@ "node": ">= 10.13" } }, + "node_modules/node-addon-api": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-3.2.1.tgz", + "integrity": "sha512-mmcei9JghVNDYydghQmeDX8KoAm0FAiYyIcUt/N4nhyAipB17pllZQDOJD2fotxABnt4Mdz+dKTO7eftLg4d0A==", + "optional": true + }, + "node_modules/node-gyp-build": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.6.0.tgz", + "integrity": "sha512-NTZVKn9IylLwUzaKjkas1e4u2DLNcV4rdYagA4PWdPwW87Bi7z+BznyKSRwS/761tV/lzCGXplWsiaMjLqP2zQ==", + "optional": true, + "bin": { + "node-gyp-build": "bin.js", + "node-gyp-build-optional": "optional.js", + "node-gyp-build-test": "build-test.js" + } + }, "node_modules/node-preload": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/node-preload/-/node-preload-0.2.1.tgz", @@ -5656,6 +5713,19 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/piscina": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/piscina/-/piscina-3.2.0.tgz", + "integrity": "sha512-yn/jMdHRw+q2ZJhFhyqsmANcbF6V2QwmD84c6xRau+QpQOmtrBCoRGdvTfeuFDYXB5W2m6MfLkjkvQa9lUSmIA==", + "dependencies": { + "eventemitter-asyncresource": "^1.0.0", + "hdr-histogram-js": "^2.0.1", + "hdr-histogram-percentiles-obj": "^3.0.0" + }, + "optionalDependencies": { + "nice-napi": "^1.0.2" + } + }, "node_modules/pkg-dir": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", @@ -7213,6 +7283,11 @@ "@jridgewell/trace-mapping": "^0.3.9" } }, + "@assemblyscript/loader": { + "version": "0.10.1", + "resolved": "https://registry.npmjs.org/@assemblyscript/loader/-/loader-0.10.1.tgz", + "integrity": "sha512-H71nDOOL8Y7kWRLqf6Sums+01Q5msqBW2KhDUTemh1tvY04eSkSXrK0uj/4mmY0Xr16/3zyZmsrxN7CKuRbNRg==" + }, "@babel/code-frame": { "version": "7.18.6", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.18.6.tgz", @@ -9328,6 +9403,11 @@ "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==" }, + "eventemitter-asyncresource": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/eventemitter-asyncresource/-/eventemitter-asyncresource-1.0.0.tgz", + "integrity": "sha512-39F7TBIV0G7gTelxwbEqnwhp90eqCPON1k0NwNfwhgKn4Co4ybUbj2pECcXT0B3ztRKZ7Pw1JujUUgmQJHcVAQ==" + }, "ext": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/ext/-/ext-1.7.0.tgz", @@ -9739,6 +9819,21 @@ } } }, + "hdr-histogram-js": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/hdr-histogram-js/-/hdr-histogram-js-2.0.3.tgz", + "integrity": "sha512-Hkn78wwzWHNCp2uarhzQ2SGFLU3JY8SBDDd3TAABK4fc30wm+MuPOrg5QVFVfkKOQd6Bfz3ukJEI+q9sXEkK1g==", + "requires": { + "@assemblyscript/loader": "^0.10.1", + "base64-js": "^1.2.0", + "pako": "^1.0.3" + } + }, + "hdr-histogram-percentiles-obj": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/hdr-histogram-percentiles-obj/-/hdr-histogram-percentiles-obj-3.0.0.tgz", + "integrity": "sha512-7kIufnBqdsBGcSZLPJwqHT3yhk1QTsSlFsVD3kx5ixH/AlgBs9yM1q6DPhXZ8f8gtdqgh7N7/5btRLpQsS2gHw==" + }, "he": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", @@ -11143,6 +11238,16 @@ "integrity": "sha512-CXdUiJembsNjuToQvxayPZF9Vqht7hewsvy2sOWafLvi2awflj9mOC6bHIg50orX8IJvWKY9wYQ/zB2kogPslQ==", "dev": true }, + "nice-napi": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/nice-napi/-/nice-napi-1.0.2.tgz", + "integrity": "sha512-px/KnJAJZf5RuBGcfD+Sp2pAKq0ytz8j+1NehvgIGFkvtvFrDM3T8E4x/JJODXK9WZow8RRGrbA9QQ3hs+pDhA==", + "optional": true, + "requires": { + "node-addon-api": "^3.0.0", + "node-gyp-build": "^4.2.2" + } + }, "nightwatch-axe-verbose": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/nightwatch-axe-verbose/-/nightwatch-axe-verbose-2.1.0.tgz", @@ -11173,6 +11278,18 @@ "propagate": "^2.0.0" } }, + "node-addon-api": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-3.2.1.tgz", + "integrity": "sha512-mmcei9JghVNDYydghQmeDX8KoAm0FAiYyIcUt/N4nhyAipB17pllZQDOJD2fotxABnt4Mdz+dKTO7eftLg4d0A==", + "optional": true + }, + "node-gyp-build": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.6.0.tgz", + "integrity": "sha512-NTZVKn9IylLwUzaKjkas1e4u2DLNcV4rdYagA4PWdPwW87Bi7z+BznyKSRwS/761tV/lzCGXplWsiaMjLqP2zQ==", + "optional": true + }, "node-preload": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/node-preload/-/node-preload-0.2.1.tgz", @@ -11541,6 +11658,17 @@ "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==" }, + "piscina": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/piscina/-/piscina-3.2.0.tgz", + "integrity": "sha512-yn/jMdHRw+q2ZJhFhyqsmANcbF6V2QwmD84c6xRau+QpQOmtrBCoRGdvTfeuFDYXB5W2m6MfLkjkvQa9lUSmIA==", + "requires": { + "eventemitter-asyncresource": "^1.0.0", + "hdr-histogram-js": "^2.0.1", + "hdr-histogram-percentiles-obj": "^3.0.0", + "nice-napi": "^1.0.2" + } + }, "pkg-dir": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", diff --git a/package.json b/package.json index 72103ab992..80b0716143 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,7 @@ "nightwatch-axe-verbose": "^2.1.0", "open": "8.4.0", "ora": "5.4.1", + "piscina": "^3.2.0", "selenium-webdriver": "4.6.1", "semver": "7.3.5", "stacktrace-parser": "0.1.10", diff --git a/test/src/cli/testParallelExecution.js b/test/src/cli/testParallelExecution.js index 0a86e289af..a4b899a093 100644 --- a/test/src/cli/testParallelExecution.js +++ b/test/src/cli/testParallelExecution.js @@ -6,6 +6,8 @@ const common = require('../../common.js'); const Nightwatch = require('../../lib/nightwatch.js'); describe('test Parallel Execution', function() { + const workerPoolArgv = []; + const taskArgv= []; const allArgs = []; const allOpts = []; @@ -40,6 +42,23 @@ describe('test Parallel Execution', function() { return new ChildProcess(); } }); + mockery.registerMock('./worker-process.js', class WorkerProcess extends EventEmitter { + constructor(args, settings, maxWorkerCount) { + super(); + + this.tasks = []; + this.index = 0; + + workerPoolArgv.push(args); + } + + addTask({argv}) { + + taskArgv.push(argv); + this.tasks.push(Promise.resolve()); + Promise.resolve(); + } + }); const {platform, constants, homedir, release, type, tmpdir} = require('os'); mockery.registerMock('os', { @@ -60,23 +79,25 @@ describe('test Parallel Execution', function() { mockery.resetCache(); mockery.disable(); allArgs.length = 0; + workerPoolArgv.length = 0; + taskArgv.length = 0; allOpts.length = 0; process.env.__NIGHTWATCH_PARALLEL_MODE = null; }); - it('testParallelExecution', function() { + + it('testParallelExecution - child-process', function() { const CliRunner = common.require('runner/cli/cli.js'); let originalCwd = process.cwd(); process.chdir(path.join(__dirname, '../../extra/')); - let runner = new CliRunner({ config: './nightwatch.json', env: 'default,mixed', reporter: 'junit' }); - - runner.setup(); - + runner.setup({ + use_child_process: true + }); assert.ok(runner.parallelMode()); assert.strictEqual(runner.testEnv, 'default,mixed'); assert.deepStrictEqual(runner.availableTestEnvs, ['default', 'mixed']); @@ -92,38 +113,91 @@ describe('test Parallel Execution', function() { }); }); - it('test parallel execution with workers defaults', function() { + it('testParallelExecution -- worker', function() { + const CliRunner = common.require('runner/cli/cli.js'); + const originalCwd = process.cwd(); + process.chdir(path.join(__dirname, '../../extra/')); + + const runner = new CliRunner({ + config: './nightwatch.json', + env: 'default,mixed', + reporter: 'junit' + }); + + runner.setup({ + use_child_process: false + }); + + assert.ok(runner.parallelMode()); + assert.strictEqual(runner.testEnv, 'default,mixed'); + assert.deepStrictEqual(runner.availableTestEnvs, ['default', 'mixed']); + + return runner.runTests().then(_ => { + assert.ok(runner.parallelMode()); + assert.strictEqual(runner.concurrency.globalExitCode, 0); + assert.strictEqual(workerPoolArgv.length, 1); + assert.ok(workerPoolArgv[0].join(' ').includes('--parallel-mode')); + assert.strictEqual(taskArgv.length, 2); + assert.strictEqual(taskArgv[0].env, 'default'); + assert.strictEqual(taskArgv[1].env, 'mixed'); + process.chdir(originalCwd); + }); + }); + + + it('test parallel execution with workers defaults -- child process', function(){ + const CliRunner = common.require('runner/cli/cli.js'); + const runner = new CliRunner({ + reporter: 'junit', + config: path.join(__dirname, '../../extra/parallelism.json') + }); + runner.setup({ + use_child_process: true + }); + runner.test_settings.globals.retryAssertionTimeout = 10; + runner.test_settings.globals.waitForConditionTimeout = 10; + runner.test_settings.globals.waitForConditionPollInterval = 9; + assert.ok(runner.test_settings.test_workers); + + return runner.runTests().then(_ => { + assert.strictEqual(allArgs.length, 4); + assert.strictEqual(runner.concurrency.globalExitCode, 0); + }); + }); + + it('test parallel execution with workers defaults -- worker thread', function() { const CliRunner = common.require('runner/cli/cli.js'); let runner = new CliRunner({ reporter: 'junit', config: path.join(__dirname, '../../extra/parallelism.json') }); - runner.setup(); + runner.setup({ + use_child_process: false + }); runner.test_settings.globals.retryAssertionTimeout = 10; runner.test_settings.globals.waitForConditionTimeout = 10; runner.test_settings.globals.waitForConditionPollInterval = 9; assert.ok(runner.test_settings.test_workers); return runner.runTests().then(_ => { - assert.strictEqual(allArgs.length, 4); + assert.strictEqual(taskArgv.length, 4); assert.strictEqual(runner.concurrency.globalExitCode, 0); }); }); - it('testParallelExecutionSameEnv', function() { + it('testParallelExecutionSameEnv - child-process', function() { let originalCwd = process.cwd(); process.chdir(path.join(__dirname, '../../extra/')); - const CliRunner = common.require('runner/cli/cli.js'); let runner = new CliRunner({ config: './nightwatch.json', reporter: 'junit', env: 'mixed,mixed' }); - - runner.setup(); - + runner.setup({ + use_child_process: true + }); assert.ok(runner.parallelMode()); assert.strictEqual(runner.testEnv, 'mixed,mixed'); assert.deepStrictEqual(runner.availableTestEnvs, ['default', 'mixed']); @@ -136,14 +210,43 @@ describe('test Parallel Execution', function() { }); }); - it('testParallelExecutionWithWorkersAuto', function() { + it('testParallelExecutionSameEnv - worker threads', function() { + let originalCwd = process.cwd(); + process.chdir(path.join(__dirname, '../../extra/')); + const CliRunner = common.require('runner/cli/cli.js'); let runner = new CliRunner({ + config: './nightwatch.json', reporter: 'junit', - config: path.join(__dirname, '../../extra/parallelism-auto.json') + env: 'mixed,mixed' }); - runner.setup(); + runner.setup({ + use_child_process: false + }); + + assert.ok(runner.parallelMode()); + assert.strictEqual(runner.testEnv, 'mixed,mixed'); + assert.deepStrictEqual(runner.availableTestEnvs, ['default', 'mixed']); + + return runner.runTests().then(_ => { + assert.strictEqual(taskArgv.length, 2); + assert.strictEqual(taskArgv[0].env, 'mixed'); + assert.strictEqual(taskArgv[1].env, 'mixed'); + process.chdir(originalCwd); + }); + }); + + + it('testParallelExecutionWithWorkersAuto - child process', function() { + const CliRunner = common.require('runner/cli/cli.js'); + const runner = new CliRunner({ + reporter: 'junit', + config: path.join(__dirname, '../../extra/parallelism-auto.json') + }); + runner.setup({ + use_child_process: true + }); assert.deepStrictEqual(runner.test_settings.test_workers, { enabled: true, workers: 'auto' @@ -154,7 +257,27 @@ describe('test Parallel Execution', function() { }); }); - it('testParallelExecutionWithWorkers and multiple environments', function() { + it('testParallelExecutionWithWorkersAuto - worker threads', function() { + const CliRunner = common.require('runner/cli/cli.js'); + const runner = new CliRunner({ + reporter: 'junit', + config: path.join(__dirname, '../../extra/parallelism-auto.json') + }); + + runner.setup({ + use_child_process: false + }); + assert.deepStrictEqual(runner.test_settings.test_workers, { + enabled: true, + workers: 'auto' + }); + + return runner.runTests().then(_ => { + assert.strictEqual(taskArgv.length, 4); + }); + }); + + it('testParallelExecutionWithWorkers and multiple environments - child process', function() { const CliRunner = common.require('runner/cli/cli.js'); let runner = new CliRunner({ reporter: 'junit', @@ -162,19 +285,38 @@ describe('test Parallel Execution', function() { env: 'default,default' }); - runner.setup(); + runner.setup({ + use_child_process: true + }); assert.strictEqual(runner.test_settings.test_workers.enabled, true); assert.ok(runner.test_settings.testWorkersEnabled); }); - it('test parallel execution with workers count', function() { + it('testParallelExecutionWithWorkers and multiple environments - worker threads', function() { const CliRunner = common.require('runner/cli/cli.js'); let runner = new CliRunner({ reporter: 'junit', - config: path.join(__dirname, '../../extra/parallelism-count.json') + config: path.join(__dirname, '../../extra/parallelism-auto.json'), + env: 'default,default' }); - runner.setup(); + runner.setup({ + use_child_process: false + }); + assert.strictEqual(runner.test_settings.test_workers.enabled, true); + assert.ok(runner.test_settings.testWorkersEnabled); + }); + + + it('test parallel execution with workers count - child process', function() { + const CliRunner = common.require('runner/cli/cli.js'); + let runner = new CliRunner({ + reporter: 'junit', + config: path.join(__dirname, '../../extra/parallelism-count.json') + }); + runner.setup({ + use_child_process: true + }); assert.deepStrictEqual(runner.test_settings.test_workers, { enabled: true, workers: 6 @@ -185,6 +327,26 @@ describe('test Parallel Execution', function() { }); }); + it('test parallel execution with workers count - worker threads', function() { + const CliRunner = common.require('runner/cli/cli.js'); + let runner = new CliRunner({ + reporter: 'junit', + config: path.join(__dirname, '../../extra/parallelism-count.json') + }); + + runner.setup({ + use_child_process: false + }); + assert.deepStrictEqual(runner.test_settings.test_workers, { + enabled: true, + workers: 6 + }); + + return runner.runTests().then(_ => { + assert.strictEqual(taskArgv.length, 4); + }); + }); + it('test parallel execution with workers=count arg', function() { const CliRunner = common.require('runner/cli/cli.js'); let runner = new CliRunner({ @@ -257,7 +419,56 @@ describe('test Parallel Execution', function() { assert.strictEqual(runner.isConcurrencyEnabled(), true); }); - it('test parallel execution using selenium server', function() { + it('test parallel execution using selenium server - child process', function() { + mockery.registerMock('geckodriver', { + path: '/path/to/geckodriver' + }); + mockery.registerMock('chromedriver', { + path: '/path/to/chromedriver' + }); + mockery.registerMock('@nightwatch/selenium-server', { + path: '/path/to/selenium-server-standalone.3.0.jar' + }); + mockery.registerMock('./service-builders/selenium.js', class SeleniumServer { + constructor(settings) { + this.settings = settings; + this.service = { + kill() { + return Promise.resolve(); + } + }; + } + async init() { + this.initCalled = true; + } + stop() { + this.stopped = true; + } + setOutputFile(filename) { + this.outfilename = filename; + } + }); + const CliRunner = common.require('runner/cli/cli.js'); + const runner = new CliRunner({ + config: path.join(__dirname, '../../extra/parallelism-selenium-server.json') + }); + runner.setup({ + use_child_process: true + }); + + return runner.runTests().then(_ => { + assert.ok(runner.parallelMode()); + assert.strictEqual(runner.concurrency.globalExitCode, 0); + assert.strictEqual(allArgs.length, 4); + assert.strictEqual(runner.seleniumService.initCalled, true); + assert.strictEqual(runner.seleniumService.stopped, true); + assert.strictEqual(runner.seleniumService.outfilename, ''); + assert.ok(allArgs[0].join(' ').includes('--test-worker --parallel-mode')); + assert.ok(allArgs[1].join(' ').includes('--test-worker --parallel-mode')); + }); + }); + + it('test parallel execution using selenium server - worker threads', function() { mockery.registerMock('geckodriver', { path: '/path/to/geckodriver' }); @@ -297,20 +508,44 @@ describe('test Parallel Execution', function() { config: path.join(__dirname, '../../extra/parallelism-selenium-server.json') }); - runner.setup(); + runner.setup({ + use_child_process: false + }); return runner.runTests().then(_ => { assert.ok(runner.parallelMode()); assert.strictEqual(runner.concurrency.globalExitCode, 0); - assert.strictEqual(allArgs.length, 4); + assert.strictEqual(taskArgv.length, 4); assert.strictEqual(runner.seleniumService.initCalled, true); assert.strictEqual(runner.seleniumService.stopped, true); assert.strictEqual(runner.seleniumService.outfilename, ''); - assert.ok(allArgs[0].join(' ').includes('--test-worker --parallel-mode')); - assert.ok(allArgs[1].join(' ').includes('--test-worker --parallel-mode')); + assert.ok(workerPoolArgv[0].join(' ').includes('--test-worker --parallel-mode')); }); }); + it('test concurrency with --headless', function() { + const CliRunner = common.require('runner/cli/cli.js'); + const runner = new CliRunner({ + reporter: 'junit', + config: path.join(__dirname, '../../extra/parallelism-count.json'), + env: 'default', + headless: true + }); + + runner.setup({ + use_child_process: false + }); + assert.deepStrictEqual(runner.test_settings.test_workers, { + enabled: true, + workers: 6 + }); + + return runner.runTests().then(_ => { + assert.strictEqual(taskArgv[0].env, 'default'); + assert.strictEqual(taskArgv[0].headless, true); + }); + }); + it('test Concurrency.getChildProcessArgs with --env=chrome,firefox', function() { const argv = process.argv.slice(0); process.argv = ['node', 'runner.js', '--env=chrome,firefox', '--headless']; diff --git a/test/src/cli/testParallelExecutionExitCode.js b/test/src/cli/testParallelExecutionExitCode.js index 651dacaf03..eb8e965a37 100644 --- a/test/src/cli/testParallelExecutionExitCode.js +++ b/test/src/cli/testParallelExecutionExitCode.js @@ -42,6 +42,18 @@ describe('test Parallel Execution Exit Code', function() { return new Child(); } }); + mockery.registerMock('./worker-process.js', class WorkerProcess extends events { + constructor(args, settings, maxWorkerCount) { + super(); + this.tasks = []; + this.index = 0; + } + + addTask({argv}) { + this.tasks.push(Promise.reject()); + Promise.resolve(); + } + }); const {platform, constants, homedir, release} = require('os'); mockery.registerMock('os', { @@ -70,7 +82,9 @@ describe('test Parallel Execution Exit Code', function() { config: path.join(__dirname, '../../extra/parallelism-count.json') }); - runner.setup(); + runner.setup({ + use_child_process: true + }); let setExitCode = runner.processListener.setExitCode; runner.processListener.setExitCode = function(code) { @@ -88,7 +102,48 @@ describe('test Parallel Execution Exit Code', function() { env: 'env1,env2' }); - runner.setup(); + runner.setup({ + use_child_process: true + }); + + let setExitCode = runner.processListener.setExitCode; + runner.processListener.setExitCode = function(code) { + runner.processListener.setExitCode = setExitCode; + assert.strictEqual(code, 1); + }; + + return runner.runTests(); + }); + + it('test parallel execution with code non zero test workers - worker threads', function() { + const CliRunner = common.require('runner/cli/cli.js'); + let runner = new CliRunner({ + config: path.join(__dirname, '../../extra/parallelism-count.json') + }); + + runner.setup({ + use_child_process: false + }); + + let setExitCode = runner.processListener.setExitCode; + runner.processListener.setExitCode = function(code) { + runner.processListener.setExitCode = setExitCode; + assert.strictEqual(code, 1); + }; + + return runner.runTests(); + }); + + it('test parallel execution with code non zero envs - worker threads', function() { + const CliRunner = common.require('runner/cli/cli.js'); + let runner = new CliRunner({ + config: path.join(__dirname, '../../extra/parallelism-envs.json'), + env: 'env1,env2' + }); + + runner.setup({ + use_child_proces: true + }); let setExitCode = runner.processListener.setExitCode; runner.processListener.setExitCode = function(code) { diff --git a/test/src/index/testProgrammaticApis.js b/test/src/index/testProgrammaticApis.js index 5f8a6c665d..27b2307e38 100644 --- a/test/src/index/testProgrammaticApis.js +++ b/test/src/index/testProgrammaticApis.js @@ -526,7 +526,7 @@ describe('test programmatic apis', function () { const Concurrency = common.require('runner/concurrency'); const Nightwatch = common.require('index.js'); - Concurrency.isChildProcess = function() { + Concurrency.isWorker = function() { return true; }; diff --git a/test/src/index/transport/testSeleniumTransport.js b/test/src/index/transport/testSeleniumTransport.js index 383c165d70..7d90fea865 100644 --- a/test/src/index/transport/testSeleniumTransport.js +++ b/test/src/index/transport/testSeleniumTransport.js @@ -7,9 +7,9 @@ const MockServer = require('../../../lib/mockserver.js'); const CommandGlobals = require('../../../lib/globals/commands.js'); describe('Selenium Server Transport', function () { - const fn = Concurrency.isChildProcess; + const fn = Concurrency.isWorker; before(function(done) { - Concurrency.isChildProcess = function() { + Concurrency.isWorker = function() { return true; } @@ -21,7 +21,7 @@ describe('Selenium Server Transport', function () { }); after(function(done) { - Concurrency.isChildProcess = fn; + Concurrency.isWorker = fn; CommandGlobals.afterEach.call(this, done); }); diff --git a/test/src/runner/cli/testCliRunnerParallel.js b/test/src/runner/cli/testCliRunnerParallel.js index 2a919d336a..91d1ed0eee 100644 --- a/test/src/runner/cli/testCliRunnerParallel.js +++ b/test/src/runner/cli/testCliRunnerParallel.js @@ -6,10 +6,11 @@ const NightwatchClient = common.require('index.js'); describe('Test CLI Runner in Parallel', function () { const ChildProcess = common.require('runner/concurrency/child-process.js'); + const WorkerProcess = common.require('runner/concurrency/worker-process.js'); const RunnerBase = common.require('runner/runner.js'); const filtered = Object.keys(require.cache).filter(item => ( - item.endsWith('runner/runner.js') || item.endsWith('runner/concurrency/child-process.js') + item.endsWith('runner/runner.js') || item.endsWith('runner/concurrency/child-process.js') || item.endsWith('runner/concurrency/worker-process.js') )); if (filtered && filtered.length > 0) { @@ -28,7 +29,7 @@ describe('Test CLI Runner in Parallel', function () { mockery.disable(); }); - it('test run geckodriver with concurrency', function () { + it('test run geckodriver with concurrency - child process', function () { class RunnerBaseMock extends RunnerBase { static readTestSource(settings, argv) { assert.strictEqual(settings.testWorkersEnabled, true); @@ -57,6 +58,41 @@ describe('Test CLI Runner in Parallel', function () { env: 'default', config: path.join(__dirname, '../../../extra/withgeckodriver-concurrent.json') }, { + use_child_process: true, + silent: false, + output: false, + output_folder: false + }); + }); + + it('test run geckodriver with concurrency - worker threads', function () { + class RunnerBaseMock extends RunnerBase { + static readTestSource(settings, argv) { + assert.strictEqual(settings.testWorkersEnabled, true); + + return [ + 'test_file_1.js', + 'test_file_2.js' + ]; + } + } + + class WorkerProcessMock extends WorkerProcess { + addTask({colors}) { + assert.strictEqual(colors.length, 4); + + return Promise.resolve(0); + } + } + + mockery.registerMock('./worker-process.js', WorkerProcessMock); + mockery.registerMock('../runner.js', RunnerBaseMock); + + return NightwatchClient.runTests({ + env: 'default', + config: path.join(__dirname, '../../../extra/withgeckodriver-concurrent.json') + }, { + use_child_process: false, silent: false, output: false, output_folder: false diff --git a/test/src/runner/testRunWithGlobalHooks.js b/test/src/runner/testRunWithGlobalHooks.js index 9f90ac7f7b..38786c7c8a 100644 --- a/test/src/runner/testRunWithGlobalHooks.js +++ b/test/src/runner/testRunWithGlobalHooks.js @@ -5,11 +5,11 @@ const CommandGlobals = require('../../lib/globals/commands.js'); const MockServer = require('../../lib/mockserver.js'); const {settings} = common; const {runTests} = common.require('index.js'); +const mockery = require('mockery'); describe('testRunWithGlobalHooks', function() { before(function(done) { this.server = MockServer.init(); - this.server.on('listening', () => { done(); }); @@ -20,6 +20,7 @@ describe('testRunWithGlobalHooks', function() { }); beforeEach(function() { + mockery.enable({useCleanCache: true, warnOnUnregistered: false}); process.removeAllListeners('exit'); process.removeAllListeners('uncaughtException'); process.removeAllListeners('unhandledRejection'); @@ -27,17 +28,20 @@ describe('testRunWithGlobalHooks', function() { afterEach(function() { process.env.__NIGHTWATCH_PARALLEL_MODE = null; + mockery.deregisterAll(); + mockery.resetCache(); + mockery.disable(); Object.keys(require.cache).filter(i => i.includes('/sampletests')).forEach(function(module) { delete require.cache[module]; }); }); it('testRunner with globalBefore and after', function() { - let testsPath = path.join(__dirname, '../../sampletests/before-after'); + const testsPath = path.join(__dirname, '../../sampletests/before-after'); let beforeEachCount = 0; let afterEachCount = 0; - let globals = { + const globals = { calls: 0, beforeEach() { @@ -75,8 +79,8 @@ describe('testRunWithGlobalHooks', function() { this.timeout(10000); it('testRunner with global async beforeEach and afterEach', function() { - let testsPath = path.join(__dirname, '../../sampletests/before-after'); - let globals = { + const testsPath = path.join(__dirname, '../../sampletests/before-after'); + const globals = { calls: 0, beforeEach(cb) { setTimeout(function() { @@ -106,8 +110,8 @@ describe('testRunWithGlobalHooks', function() { }); it('testRunner with global async beforeEach and afterEach with api argument', function() { - let testsPath = path.join(__dirname, '../../sampletests/before-after'); - let globals = { + const testsPath = path.join(__dirname, '../../sampletests/before-after'); + const globals = { calls: 0, beforeEach(client, done) { assert.deepStrictEqual(client.globals, this); @@ -140,7 +144,7 @@ describe('testRunWithGlobalHooks', function() { it('test run with global async beforeEach and assert failure', function() { let beforeEachCount = 0; - let testsPath = path.join(__dirname, '../../sampletests/before-after'); + const testsPath = path.join(__dirname, '../../sampletests/before-after'); return runTests(testsPath, settings({ globals: { @@ -162,7 +166,7 @@ describe('testRunWithGlobalHooks', function() { }); it('test run with global async beforeEach and exception', function() { - let testsPath = path.join(__dirname, '../../sampletests/before-after/'); + const testsPath = path.join(__dirname, '../../sampletests/before-after/'); return runTests(testsPath, settings({ output: false, @@ -193,7 +197,7 @@ describe('testRunWithGlobalHooks', function() { }); it('test run with global async beforeEach and timeout error', async function() { - let testsPath = path.join(__dirname, '../../sampletests/before-after'); + const testsPath = path.join(__dirname, '../../sampletests/before-after'); let expectedErr; @@ -214,7 +218,7 @@ describe('testRunWithGlobalHooks', function() { }); it('test run with global async beforeEach and done(err);', function() { - let testsPath = path.join(__dirname, '../../sampletests/before-after'); + const testsPath = path.join(__dirname, '../../sampletests/before-after'); return runTests(testsPath, settings({ globals: { @@ -233,8 +237,8 @@ describe('testRunWithGlobalHooks', function() { }); it('test currentTest in global beforeEach/afterEach', function() { - let testsPath = path.join(__dirname, '../../sampletests/withfailures'); - let globals = { + const testsPath = path.join(__dirname, '../../sampletests/withfailures'); + const globals = { calls: 0, waitForConditionPollInterval: 5, waitForConditionTimeout: 5, @@ -281,10 +285,46 @@ describe('testRunWithGlobalHooks', function() { })); }); - it('test global child process hooks', function() { - let testsPath = path.join(__dirname, '../../sampletests/before-after'); + it('test global child process hooks - child process', function() { + const testsPath = path.join(__dirname, '../../sampletests/before-after'); process.env.__NIGHTWATCH_PARALLEL_MODE = '1'; - let globals = { + const globals = { + calls: 0, + beforeChildProcess() { + globals.calls++; + }, + afterChildProcess() { + globals.calls++; + }, + reporter(results, cb) { + assert.strictEqual(globals.calls, 20); + cb(); + } + }; + + return runTests(testsPath, settings({ + output: false, + use_child_process: true, + globals + }) + ); + }); + + it('test global child process hooks - worker threads', function() { + mockery.registerMock('./worker-process.js', class WorkerProcess { + static get isWorkerThread() { + return true; + } + }); + + const processPort = process.port; + process.port = { + postMessage: function(){} + }; + + const testsPath = path.join(__dirname, '../../sampletests/before-after'); + + const globals = { calls: 0, beforeChildProcess() { globals.calls++; @@ -294,21 +334,63 @@ describe('testRunWithGlobalHooks', function() { }, reporter(results, cb) { assert.strictEqual(globals.calls, 20); + process.port = processPort; cb(); } }; return runTests(testsPath, settings({ + use_child_process: false, output: false, globals }) ); }); - it('test global async child process hooks', function() { - let testsPath = path.join(__dirname, '../../sampletests/before-after'); + + it('test global async child process hooks - child process', function() { + const testsPath = path.join(__dirname, '../../sampletests/before-after'); process.env.__NIGHTWATCH_PARALLEL_MODE = '1'; - let globals = { + const globals = { + calls: 0, + beforeChildProcess(_, done) { + setTimeout(function() { + globals.calls++; + done(); + }, 10); + }, + afterChildProcess(_, done) { + setTimeout(function() { + globals.calls++; + done(); + }, 15); + }, + reporter(_, cb) { + assert.strictEqual(globals.calls, 20); + cb(); + } + }; + + return runTests(testsPath, settings({ + output: false, + use_child_process: true, + globals + })); + }); + + it('test global async child process hooks - worker thread', function() { + mockery.registerMock('./worker-process.js', class WorkerProcess { + static get isWorkerThread() { + return true; + } + }); + const processPort = process.port; + process.port = { + postMessage: function(){} + }; + + const testsPath = path.join(__dirname, '../../sampletests/before-after'); + const globals = { calls: 0, beforeChildProcess(_, done) { setTimeout(function() { @@ -324,12 +406,14 @@ describe('testRunWithGlobalHooks', function() { }, reporter(_, cb) { assert.strictEqual(globals.calls, 20); + process.port = processPort; cb(); } }; return runTests(testsPath, settings({ output: false, + use_child_process: false, globals })); });