feat!: new output format for "test" command

index.js

@@ -24,8 +24,12 @@ if (process.argv.includes('test')) {
 		throw new Error('Could not find any rules.')
 	}
 
-	if (test(module.exports.rules) === false) {
-		process.exit(1)
+	const errorCount = test(module.exports.rules, {
+		log: console.log,
+		err: console.error,
+	})
+	if (errorCount > 0) {
+		process.exit(errorCount)
 	}
 }
 

test.js

@@ -4,10 +4,7 @@ const { RuleTester } = require('eslint')
 const chalk = require('chalk')
 
 /**
- * @typedef {{
- * 	valid: Array<import('eslint').RuleTester.ValidTestCase>
- * 	invalid: Array<import('eslint').RuleTester.InvalidTestCase>
- * }} TestCases
+ * @typedef {import('eslint').RuleTester.ValidTestCase | import('eslint').RuleTester.InvalidTestCase} TestCase
  */
 
 const Exclusiveness = Symbol('exclusivenessToken')
@@ -32,10 +29,13 @@ function only(item) {
 }
 
 /**
- * @param {Record<string, import('eslint').Rule.RuleModule & { tests?: TestCases }>} rules
- * @returns {void | false}
+ * @param {Record<string, import('eslint').Rule.RuleModule & { tests?: { valid: Array<import('eslint').RuleTester.ValidTestCase>, invalid: Array<import('eslint').RuleTester.InvalidTestCase> } }>} rules
+ * @returns {number} number of error test cases
  */
-module.exports = function test(rules) {
+module.exports = function test(
+	rules,
+	{ log, err } = { log: console.log, err: console.error }
+) {
 	// See https://eslint.org/docs/latest/integrate/nodejs-api#ruletester
 	const tester = new RuleTester()
 
@@ -44,47 +44,74 @@ module.exports = function test(rules) {
 		ruleModule.tests?.invalid.some(testCase => testCase[Exclusiveness])
 	)
 
+	const stats = { pass: 0, fail: 0, skip: 0 }
 	for (const ruleName in rules) {
 		const ruleModule = rules[ruleName]
 		if (!ruleModule.tests || typeof ruleModule.tests !== 'object') {
-			console.log('⚪ ' + ruleName)
+			log('⚪ ' + ruleName)
 			continue
 		}
 
-		const validItems = ruleModule.tests.valid.map(testCase => (
-			{ testCase, valid: [testCase], invalid: [] }
-		))
-		const invalidItems = ruleModule.tests.invalid.map(testCase => (
-			{ testCase, valid: [], invalid: [testCase] }
-		))
-		const totalItems = [...validItems, ...invalidItems]
-		const nonSkippedItems = totalItems.filter(({ testCase }) => oneOrMoreTestCaseIsSkipped ? !!testCase[Exclusiveness] : true)
+		for (const testCase of ruleModule.tests.invalid) {
+			testCase.errors = testCase.errors ?? []
+		}
+
+		/**
+		 * @type {Array<TestCase>}
+		 */
+		const totalItems = [...ruleModule.tests.valid, ...ruleModule.tests.invalid]
+		const runningItems = totalItems.filter(testCase => oneOrMoreTestCaseIsSkipped ? !!testCase[Exclusiveness] : true)
 
-		for (const { testCase, valid, invalid } of nonSkippedItems) {
+		const errors = runningItems.reduce((results, testCase) => {
 			try {
-				tester.run(ruleName, ruleModule, { valid, invalid })
+				tester.run(
+					ruleName,
+					ruleModule,
+					'errors' in testCase ? { valid: [], invalid: [testCase] } : { valid: [testCase], invalid: [] }
+				)
 
 			} catch (error) {
-				console.log('🔴 ' + chalk.bold(ruleName))
+				results.push({ testCase, error })
+			}
+
+			return results
+		}, /** @type {Array<{ testCase: TestCase, error: Error }>} */([]))
+
+		stats.skip += totalItems.length - runningItems.length
+		stats.fail += errors.length
+		stats.pass += runningItems.length - errors.length
+
+		if (errors.length > 0) {
+			err('🔴 ' + ruleName)
+			for (const { testCase, error } of errors) {
+				err('')
+				err(offset(getPrettyCode(testCase.code), chalk.bgRed))
+				err('')
+				err(offset(error.message, chalk.red))
+				return 1
+			}
 
-				console.log('')
-				console.log(offset(getPrettyCode(testCase.code), chalk.bgRed))
-				console.log('')
+		} else if (totalItems.length === runningItems.length) {
+			log('🟢 ' + ruleName)
 
-				console.error(offset(error.message, chalk.red))
-				if (error.stack) {
-					console.error(offset(error.stack, chalk.red))
-				}
+		} else if (runningItems.length > 0) {
+			log('🟡 ' + ruleName)
 
-				return false
-			}
+		} else {
+			log('⏩ ' + ruleName)
 		}
+	}
 
-		console.log((totalItems.length === nonSkippedItems.length ? '🟢' : '🟡') + ' ' + ruleName)
+	log('')
+	log(chalk.bgGreen(chalk.bold(' PASS ')) + ' ' + stats.pass.toLocaleString())
+	if (stats.fail > 0) {
+		log(chalk.bgRed(chalk.bold(' FAIL ')) + ' ' + stats.fail.toLocaleString())
+	}
+	if (stats.skip > 0) {
+		log(chalk.bgHex('#0CAAEE')(chalk.bold(' SKIP ')) + ' ' + stats.skip.toLocaleString())
 	}
 
-	console.log('')
-	console.log(`Done testing ${Object.keys(rules).length.toLocaleString()} rule${Object.keys(rules).length === 1 ? '' : 's'}.`)
+	return stats.fail
 }
 
 global.only = only

test.test.js

@@ -4,11 +4,10 @@ jest.mock('chalk', () => ({
 	bold: (text) => text,
 	red: (text) => text,
 	bgRed: (text) => text,
+	bgGreen: (text) => text,
+	bgHex: () => (text) => text,
 }))
 
-jest.spyOn(console, 'log').mockImplementation(() => { })
-jest.spyOn(console, 'error').mockImplementation(() => { })
-
 afterEach(() => {
 	jest.clearAllMocks()
 })
@@ -19,7 +18,7 @@ afterAll(() => {
 
 const test = require('./test')
 
-it('does not return false, given no failing test case', () => {
+it('returns zero errors, given no failing test case', () => {
 	const rules = {
 		foo: {
 			create(context) {
@@ -41,12 +40,20 @@ it('does not return false, given no failing test case', () => {
 		}
 	}
 
-	expect(test(rules)).not.toBe(false)
-	expect(console.log).toHaveBeenCalledWith(expect.stringMatching(/Done testing 1 rule/))
-	expect(console.error).not.toHaveBeenCalled()
+	const log = jest.fn()
+	const err = jest.fn()
+	const errorCount = test(rules, { log, err })
+
+	expect(errorCount).toBe(0)
+	expect(log.mock.calls.join('\n')).toMatchInlineSnapshot(`
+"🟢 foo
+
+ PASS  2"
+`)
+	expect(err).not.toHaveBeenCalled()
 })
 
-it('returns false, given a failing test case', () => {
+it('returns non-zero errors, given any failing test case', () => {
 	const rules = {
 		foo: {
 			create(context) {
@@ -68,9 +75,30 @@ it('returns false, given a failing test case', () => {
 		}
 	}
 
-	expect(test(rules)).toBe(false)
-	expect(console.log).toHaveBeenCalledWith('🔴 foo')
-	expect(console.error).toHaveBeenCalledWith(expect.stringMatching(/Should have no errors but had 1/))
+	const log = jest.fn()
+	const err = jest.fn()
+	const errorCount = test(rules, { log, err })
+
+	expect(errorCount).toBe(1)
+	expect(log.mock.calls.join('\n')).toMatchInlineSnapshot(`""`)
+	expect(err.mock.calls.join('\n')).toMatchInlineSnapshot(`
+"🔴 foo
+
+   void(0)
+
+   Should have no errors but had 1: [
+     {
+       ruleId: 'foo',
+       severity: 1,
+       message: 'bar',
+       line: 1,
+       column: 1,
+       nodeType: 'Program',
+       endLine: 1,
+       endColumn: 8
+     }
+   ] (1 strictEqual 0)"
+`)
 })
 
 it('runs only the test case wrapped with `only` function', () => {
@@ -102,8 +130,18 @@ it('runs only the test case wrapped with `only` function', () => {
 		}
 	}
 
-	expect(test(rules)).not.toBe(false)
-	expect(console.error).not.toHaveBeenCalled()
+	const log = jest.fn()
+	const err = jest.fn()
+	const errorCount = test(rules, { log, err })
+
+	expect(errorCount).toBe(0)
 	expect(rules.foo.create).toHaveBeenCalled()
 	expect(rules.loo.create).not.toHaveBeenCalled()
+	expect(log.mock.calls.join('\n')).toMatchInlineSnapshot(`
+"🟡 foo
+⏩ loo
+
+ PASS  1
+ SKIP  3"
+`)
 })