test_runner: add initial CLI runner (#4)

* test_runner: add initial CLI runner

* add eslint dev dependency for node 14

* pkg: add missing bin

* remove direct eslint dependency

* pkg: refactor, recreate package-lock.json

README.md

@@ -20,7 +20,7 @@ Differences from the core implementation:
 
 ## Docs
 
-> https://github.com/nodejs/node/blob/54819f08e0c469528901d81a9cee546ea518a5c3/doc/api/test.md
+> https://github.com/cjihrig/node/blob/484d35402de36d8d5756b244c8e5fbb8aa4c6afd/doc/api/test.md
 
 # Test runner
 
@@ -232,6 +232,66 @@ test('a test that creates asynchronous activity', t => {
 })
 ```
 
+## Running tests from the command line
+
+The Node.js test runner can be invoked from the command line:
+
+```bash
+node-core-test
+```
+
+By default, Node.js will recursively search the current directory for
+JavaScript source files matching a specific naming convention. Matching files
+are executed as test files. More information on the expected test file naming
+convention and behavior can be found in the [test runner execution model][]
+section.
+
+Alternatively, one or more paths can be provided as the final argument(s) to
+the Node.js command, as shown below.
+
+```bash
+node-core-test test1.js test2.mjs custom_test_dir/
+```
+
+In this example, the test runner will execute the files `test1.js` and
+`test2.mjs`. The test runner will also recursively search the
+`custom_test_dir/` directory for test files to execute.
+
+### Test runner execution model
+
+When searching for test files to execute, the test runner behaves as follows:
+
+- Any files explicitly provided by the user are executed.
+- If the user did not explicitly specify any paths, the current working
+  directory is recursively searched for files as specified in the following
+  steps.
+- `node_modules` directories are skipped unless explicitly provided by the
+  user.
+- If a directory named `test` is encountered, the test runner will search it
+  recursively for all all `.js`, `.cjs`, and `.mjs` files. All of these files
+  are treated as test files, and do not need to match the specific naming
+  convention detailed below. This is to accommodate projects that place all of
+  their tests in a single `test` directory.
+- In all other directories, `.js`, `.cjs`, and `.mjs` files matching the
+  following patterns are treated as test files:
+  - `^test$` - Files whose basename is the string `'test'`. Examples:
+    `test.js`, `test.cjs`, `test.mjs`.
+  - `^test-.+` - Files whose basename starts with the string `'test-'`
+    followed by one or more characters. Examples: `test-example.js`,
+    `test-another-example.mjs`.
+  - `.+[\.\-\_]test$` - Files whose basename ends with `.test`, `-test`, or
+    `_test`, preceded by one or more characters. Examples: `example.test.js`,
+    `example-test.cjs`, `example_test.mjs`.
+  - Other file types understood by Node.js such as `.node` and `.json` are not
+    automatically executed by the test runner, but are supported if explicitly
+    provided on the command line.
+
+Each matching test file is executed in a separate child process. If the child
+process finishes with an exit code of 0, the test is considered passing.
+Otherwise, the test is considered to be a failure. Test files must be
+executable by Node.js, but are not required to use the `node:test` module
+internally.
+
 ## `test([name][, options][, fn])`
 
 - `name` {string} The name of the test, which is displayed when reporting test
@@ -354,6 +414,7 @@ behaves in the same fashion as the top level [`test()`][] function.
 [tap]: https://testanything.org/
 [`testcontext`]: #class-testcontext
 [`test()`]: #testname-options-fn
+[test runner execution model]: #test-runner-execution-model
 
 ## Kudos
 

bin/test_runner.js

@@ -0,0 +1,143 @@
+#!/usr/bin/env node
+// https://github.com/cjihrig/node/blob/484d35402de36d8d5756b244c8e5fbb8aa4c6afd/lib/internal/main/test_runner.js
+'use strict'
+const {
+  ArrayFrom,
+  ArrayPrototypePush,
+  ArrayPrototypeSlice,
+  ArrayPrototypeSort,
+  Promise,
+  SafeSet
+} = require('../lib/primordials')
+const { spawn } = require('child_process')
+const { readdirSync, statSync } = require('fs')
+const {
+  codes: { ERR_TEST_FAILURE }
+} = require('../lib/errors')
+const test = require('../lib/harness')
+const { kSubtestsFailed } = require('../lib/test')
+const { isSupportedFileType, doesPathMatchFilter } = require('../lib/utils')
+const { basename, join, resolve } = require('path')
+
+// TODO(cjihrig): Replace this with recursive readdir once it lands.
+function processPath (path, testFiles, options) {
+  const stats = statSync(path)
+
+  if (stats.isFile()) {
+    if (
+      options.userSupplied ||
+      (options.underTestDir && isSupportedFileType(path)) ||
+      doesPathMatchFilter(path)
+    ) {
+      testFiles.add(path)
+    }
+  } else if (stats.isDirectory()) {
+    const name = basename(path)
+
+    if (!options.userSupplied && name === 'node_modules') {
+      return
+    }
+
+    // 'test' directories get special treatment. Recursively add all .js,
+    // .cjs, and .mjs files in the 'test' directory.
+    const isTestDir = name === 'test'
+    const { underTestDir } = options
+    const entries = readdirSync(path)
+
+    if (isTestDir) {
+      options.underTestDir = true
+    }
+
+    options.userSupplied = false
+
+    for (let i = 0; i < entries.length; i++) {
+      processPath(join(path, entries[i]), testFiles, options)
+    }
+
+    options.underTestDir = underTestDir
+  }
+}
+
+function createTestFileList () {
+  const cwd = process.cwd()
+  const hasUserSuppliedPaths = process.argv.length > 2
+  const testPaths = hasUserSuppliedPaths
+    ? ArrayPrototypeSlice(process.argv, 2)
+    : [cwd]
+  const testFiles = new SafeSet()
+
+  try {
+    for (let i = 0; i < testPaths.length; i++) {
+      const absolutePath = resolve(testPaths[i])
+
+      processPath(absolutePath, testFiles, { userSupplied: true })
+    }
+  } catch (err) {
+    if (err?.code === 'ENOENT') {
+      console.error(`Could not find '${err.path}'`)
+      process.exit(1)
+    }
+
+    throw err
+  }
+
+  return ArrayPrototypeSort(ArrayFrom(testFiles))
+}
+
+function runTestFile (path) {
+  return test(path, () => {
+    return new Promise((resolve, reject) => {
+      const args = [...process.execArgv]
+      ArrayPrototypePush(args, path)
+
+      const child = spawn(process.execPath, args)
+      // TODO(cjihrig): Implement a TAP parser to read the child's stdout
+      // instead of just displaying it all if the child fails.
+      let stdout = ''
+      let stderr = ''
+      let err
+
+      child.on('error', error => {
+        err = error
+      })
+
+      child.stdout.setEncoding('utf8')
+      child.stderr.setEncoding('utf8')
+
+      child.stdout.on('data', chunk => {
+        stdout += chunk
+      })
+
+      child.stderr.on('data', chunk => {
+        stderr += chunk
+      })
+
+      child.once('exit', (code, signal) => {
+        if (code !== 0 || signal !== null) {
+          if (!err) {
+            err = new ERR_TEST_FAILURE('test failed', kSubtestsFailed)
+            err.exitCode = code
+            err.signal = signal
+            err.stdout = stdout
+            err.stderr = stderr
+            // The stack will not be useful since the failures came from tests
+            // in a child process.
+            err.stack = undefined
+          }
+
+          return reject(err)
+        }
+
+        resolve()
+      })
+    })
+  })
+}
+
+;(async function main () {
+  const testFiles = createTestFileList()
+
+  for (let i = 0; i < testFiles.length; i++) {
+    runTestFile(testFiles[i])
+  }
+})()

lib/errors.js

@@ -3,26 +3,110 @@
 'use strict'
 
 const assert = require('assert')
-const { lazyInternalUtilInspect } = require('./primordials')
+const {
+  ArrayPrototypeUnshift,
+  lazyInternalUtilInspect,
+  ObjectDefineProperties,
+  ReflectApply,
+  SafeMap,
+  StringPrototypeMatch
+} = require('./primordials')
 
-class ERR_TEST_FAILURE extends Error {
-  constructor (error, failureType) {
-    const message = error?.message ?? lazyInternalUtilInspect().inspect(error)
-    super(message)
+function inspectWithNoCustomRetry (obj, options) {
+  const utilInspect = lazyInternalUtilInspect()
 
+  try {
+    return utilInspect.inspect(obj, options)
+  } catch {
+    return utilInspect.inspect(obj, { ...options, customInspect: false })
+  }
+}
+
+const kIsNodeError = 'kIsNodeError'
+const messages = new SafeMap()
+const codes = {}
+
+function getMessage (key, args, self) {
+  const msg = messages.get(key)
+
+  if (typeof msg === 'function') {
     assert(
-      typeof failureType === 'string',
-      "The 'failureType' argument must be of type string."
+      msg.length <= args.length, // Default options do not count.
+      `Code: ${key}; The provided arguments length (${args.length}) does not ` +
+        `match the required ones (${msg.length}).`
     )
+    return ReflectApply(msg, self, args)
+  }
+
+  const expectedLength =
+    (StringPrototypeMatch(msg, /%[dfijoOs]/g) || []).length
+  assert(
+    expectedLength === args.length,
+    `Code: ${key}; The provided arguments length (${args.length}) does not ` +
+      `match the required ones (${expectedLength}).`
+  )
+  if (args.length === 0) { return msg }
+
+  ArrayPrototypeUnshift(args, msg)
+  return ReflectApply(lazyInternalUtilInspect().format, null, args)
+}
 
-    this.failureType = error?.failureType ?? failureType
-    this.cause = error
-    this.code = 'ERR_TEST_FAILURE'
+function makeNodeErrorWithCode (Base, key) {
+  return function NodeError (...args) {
+    const error = new Base()
+    const message = getMessage(key, args, error)
+    ObjectDefineProperties(error, {
+      [kIsNodeError]: {
+        value: true,
+        enumerable: false,
+        writable: false,
+        configurable: true
+      },
+      message: {
+        value: message,
+        enumerable: false,
+        writable: true,
+        configurable: true
+      },
+      toString: {
+        value () {
+          return `${this.name} [${key}]: ${this.message}`
+        },
+        enumerable: false,
+        writable: true,
+        configurable: true
+      }
+    })
+    error.code = key
+    return error
   }
 }
 
-module.exports = {
-  codes: {
-    ERR_TEST_FAILURE
+// Utility function for registering the error codes. Only used here. Exported
+// *only* to allow for testing.
+function E (sym, val, def) {
+  messages.set(sym, val)
+  def = makeNodeErrorWithCode(def, sym)
+  codes[sym] = def
+}
+
+E('ERR_TEST_FAILURE', function (error, failureType) {
+  assert(typeof failureType === 'string',
+    "The 'failureType' argument must be of type string.")
+
+  let msg = error?.message ?? error
+
+  if (typeof msg !== 'string') {
+    msg = inspectWithNoCustomRetry(msg)
   }
+
+  this.failureType = failureType
+  this.cause = error
+  return msg
+}, Error)
+
+module.exports = {
+  codes,
+  inspectWithNoCustomRetry,
+  kIsNodeError
 }

lib/primordials.js

@@ -3,18 +3,32 @@
 const util = require('util')
 const replaceAll = require('string.prototype.replaceall')
 
+exports.ArrayFrom = it => Array.from(it)
+// exports.ArrayPrototypeFilter = (arr, fn) => arr.filter(fn)
 exports.ArrayPrototypeForEach = (arr, fn) => arr.forEach(fn)
+// exports.ArrayPrototypeIncludes = (arr, el) => arr.includes(el)
 exports.ArrayPrototypeJoin = (arr, str) => arr.join(str)
 exports.ArrayPrototypePush = (arr, el) => arr.push(el)
 exports.ArrayPrototypeShift = arr => arr.shift()
+exports.ArrayPrototypeSlice = (arr, offset) => arr.slice(offset)
+exports.ArrayPrototypeSort = (arr, fn) => arr.sort(fn)
+exports.ArrayPrototypeUnshift = (arr, el) => arr.unshift(el)
 exports.FunctionPrototype = () => {}
 exports.FunctionPrototypeBind = (fn, obj) => fn.bind(obj)
 exports.lazyInternalUtilInspect = () => util
 exports.Number = Number
 exports.ObjectCreate = obj => Object.create(obj)
+exports.ObjectDefineProperties = (obj, props) => Object.defineProperties(obj, props)
 exports.ObjectEntries = obj => Object.entries(obj)
+exports.ReflectApply = (target, self, args) => Reflect.apply(target, self, args)
+exports.Promise = Promise
 exports.SafeMap = Map
-exports.StringPrototypeReplace = (str, search, replacement) =>
-  str.replace(search, replacement)
+exports.SafeSet = Set
+exports.StringPrototypeMatch = (str, reg) => str.match(reg)
+// exports.StringPrototypeReplace = (str, search, replacement) =>
+//   str.replace(search, replacement)
 exports.StringPrototypeReplaceAll = replaceAll
 exports.StringPrototypeSplit = (str, search) => str.split(search)
+exports.RegExpPrototypeExec = (reg, str) => reg.exec(str)
+exports.RegExpPrototypeSymbolReplace = (regexp, str, replacement) =>
+  str.replace(regexp, replacement)