benchmark: add test and all options and improve errors

This adds a new `test` option. Using it automatically uses a single
minimal option matrix to verify the benchmark works as expected.

Using the new `all` option makes sure all test suites are run.

On top of that the benchmarks will from now on report properly
what category might have a typo, if any.

The http duration was also refactored to use a option instead of
relying on a configuration setting.

The fixture folder is ignored as test suite from now on.

PR-URL: nodejs#31396
Fixes: nodejs#31083
Reviewed-By: Sam Roberts <vieuxtech@gmail.com>
Reviewed-By: Rich Trott <rtrott@gmail.com>

benchmark/_cli.js

@@ -6,22 +6,24 @@ const path = require('path');
 // Create an object of all benchmark scripts
 const benchmarks = {};
 fs.readdirSync(__dirname)
-  .filter((name) => fs.statSync(path.resolve(__dirname, name)).isDirectory())
+  .filter((name) => {
+    return name !== 'fixtures' &&
+           fs.statSync(path.resolve(__dirname, name)).isDirectory();
+  })
   .forEach((category) => {
     benchmarks[category] = fs.readdirSync(path.resolve(__dirname, category))
       .filter((filename) => filename[0] !== '.' && filename[0] !== '_');
   });
 
 function CLI(usage, settings) {
-  if (!(this instanceof CLI)) return new CLI(usage, settings);
-
   if (process.argv.length < 3) {
     this.abort(usage); // Abort will exit the process
   }
 
   this.usage = usage;
   this.optional = {};
   this.items = [];
+  this.test = false;
 
   for (const argName of settings.arrayArgs) {
     this.optional[argName] = [];
@@ -34,7 +36,7 @@ function CLI(usage, settings) {
     if (arg === '--') {
       // Only items can follow --
       mode = 'item';
-    } else if ('both' === mode && arg[0] === '-') {
+    } else if (mode === 'both' && arg[0] === '-') {
       // Optional arguments declaration
 
       if (arg[1] === '-') {
@@ -61,6 +63,8 @@ function CLI(usage, settings) {
 
       // The next value can be either an option or an item
       mode = 'both';
+    } else if (arg === 'test') {
+      this.test = true;
     } else if (['both', 'item'].includes(mode)) {
       // item arguments
       this.items.push(arg);
@@ -83,9 +87,15 @@ CLI.prototype.abort = function(msg) {
 CLI.prototype.benchmarks = function() {
   const paths = [];
 
+  if (this.items.includes('all')) {
+    this.items = Object.keys(benchmarks);
+  }
+
   for (const category of this.items) {
-    if (benchmarks[category] === undefined)
-      continue;
+    if (benchmarks[category] === undefined) {
+      console.error(`The "${category}" category does not exist.`);
+      process.exit(1);
+    }
     for (const scripts of benchmarks[category]) {
       if (this.shouldSkip(scripts)) continue;
 

benchmark/_http-benchmarkers.js

@@ -43,9 +43,8 @@ class AutocannonBenchmarker {
     }
     if (!result || !result.requests || !result.requests.average) {
       return undefined;
-    } else {
-      return result.requests.average;
     }
+    return result.requests.average;
   }
 }
 
@@ -77,9 +76,8 @@ class WrkBenchmarker {
     const throughput = match && +match[1];
     if (!isFinite(throughput)) {
       return undefined;
-    } else {
-      return throughput;
     }
+    return throughput;
   }
 }
 
@@ -89,18 +87,20 @@ class WrkBenchmarker {
  */
 class TestDoubleBenchmarker {
   constructor(type) {
-    // `type` is the type ofbenchmarker. Possible values are 'http' and 'http2'.
+    // `type` is the type of benchmarker. Possible values are 'http' and
+    // 'http2'.
     this.name = `test-double-${type}`;
     this.executable = path.resolve(__dirname, '_test-double-benchmarker.js');
     this.present = fs.existsSync(this.executable);
     this.type = type;
   }
 
   create(options) {
-    const env = Object.assign({
+    const env = {
       duration: options.duration,
       test_url: `http://127.0.0.1:${options.port}${options.path}`,
-    }, process.env);
+      ...process.env
+    };
 
     const child = child_process.fork(this.executable,
                                      [this.type],
@@ -189,13 +189,14 @@ http_benchmarkers.forEach((benchmarker) => {
 });
 
 exports.run = function(options, callback) {
-  options = Object.assign({
+  options = {
     port: exports.PORT,
     path: '/',
     connections: 100,
     duration: 5,
     benchmarker: exports.default_http_benchmarker,
-  }, options);
+    ...options
+  };
   if (!options.benchmarker) {
     callback(new Error('Could not locate required http benchmarker. See ' +
                        `${requirementsURL} for further instructions.`));
@@ -212,6 +213,7 @@ exports.run = function(options, callback) {
                        'is  not installed'));
     return;
   }
+  process.env.duration = process.env.duration || options.duration || 5;
 
   const benchmarker_start = process.hrtime();
 
@@ -220,7 +222,8 @@ exports.run = function(options, callback) {
   child.stderr.pipe(process.stderr);
 
   let stdout = '';
-  child.stdout.on('data', (chunk) => stdout += chunk.toString());
+  child.stdout.setEncoding('utf8');
+  child.stdout.on('data', (chunk) => stdout += chunk);
 
   child.once('close', (code) => {
     const elapsed = process.hrtime(benchmarker_start);

benchmark/_test-double-benchmarker.js

@@ -7,7 +7,7 @@ if (!['http', 'http2'].includes(myModule)) {
 
 const http = require(myModule);
 
-const duration = process.env.duration || 0;
+const duration = +process.env.duration;
 const url = process.env.test_url;
 
 const start = process.hrtime();
@@ -18,13 +18,15 @@ function request(res, client) {
   res.on('error', () => {});
   res.on('end', () => {
     throughput++;
-    const diff = process.hrtime(start);
-    if (duration > 0 && diff[0] < duration) {
+    const [sec, nanosec] = process.hrtime(start);
+    const ms = sec * 1000 + nanosec / 1e6;
+    if (ms < duration * 1000) {
       run();
     } else {
       console.log(JSON.stringify({ throughput }));
       if (client) {
         client.destroy();
+        process.exit(0);
       }
     }
   });
@@ -33,7 +35,7 @@ function request(res, client) {
 function run() {
   if (http.get) { // HTTP
     http.get(url, request);
-  } else {        // HTTP/2
+  } else { // HTTP/2
     const client = http.connect(url);
     client.on('error', (e) => { throw e; });
     request(client.request(), client);

benchmark/async_hooks/http-server.js

@@ -3,10 +3,11 @@ const common = require('../common.js');
 
 const bench = common.createBenchmark(main, {
   asyncHooks: ['init', 'before', 'after', 'all', 'disabled', 'none'],
-  connections: [50, 500]
+  connections: [50, 500],
+  duration: 5
 });
 
-function main({ asyncHooks, connections }) {
+function main({ asyncHooks, connections, duration }) {
   if (asyncHooks !== 'none') {
     let hooks = {
       init() {},
@@ -33,6 +34,7 @@ function main({ asyncHooks, connections }) {
       bench.http({
         connections,
         path,
+        duration
       }, () => {
         server.close();
       });

benchmark/buffers/buffer-base64-encode.js

@@ -25,6 +25,8 @@ const common = require('../common.js');
 const bench = common.createBenchmark(main, {
   len: [64 * 1024 * 1024],
   n: [32]
+}, {
+  test: { len: 256 }
 });
 
 function main({ n, len }) {

benchmark/buffers/buffer-swap.js

@@ -7,6 +7,8 @@ const bench = common.createBenchmark(main, {
   method: ['swap16', 'swap32', 'swap64'/* , 'htons', 'htonl', 'htonll' */],
   len: [64, 256, 768, 1024, 2056, 8192],
   n: [1e6]
+}, {
+  test: { len: 16 }
 });
 
 // The htons and htonl methods below are used to benchmark the