Skip to content

Commit

Permalink
feat!: better rule testing outputs
Browse files Browse the repository at this point in the history
  • Loading branch information
@ThisIsManta
ThisIsManta committed Aug 4, 2024
1 parent 6f12ba2 commit 80b987b
Show file tree
Hide file tree
Showing 3 changed files with 160 additions and 104 deletions.
2 changes: 1 addition & 1 deletion executable.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ if (process.argv[2] === 'test') {
silent: false,
})

// Do not move this below as `global.only` must be injected before anything
// Do not move this line any lower as `global.only` must be injected before anything
const testRunner = require('./testRunner')

const { rules } = require('./index')
Expand Down
223 changes: 140 additions & 83 deletions testRunner.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,25 +3,36 @@
const { RuleTester } = require('eslint')
const chalk = require('chalk')

const Exclusiveness = Symbol('exclusivenessToken')
const gray = chalk.hex('#BDBDBD')

/**
* @param {Array<import('./types').TestCase> | import('./types').TestCase} item
* @param {{ valid?: Array<import('./types').TestCase>, invalid?: Array<import('./types').TestCase> } | Array<import('./types').TestCase> | import('./types').TestCase} testCaseOrGroup
*/
function only(item) {
function only(testCaseOrGroup) {
// Disallow test case exclusiveness in CI
if (!process.env.CI) {
if (Array.isArray(item)) {
if (typeof testCaseOrGroup === 'string') {
// Not support
} else if (Array.isArray(testCaseOrGroup)) {
// Support `valid: only([...])` and `invalid: only([...])`
for (const listItem of item) {
for (const listItem of testCaseOrGroup) {
only(listItem)
}
} else if (typeof item === 'object' && item !== null) {
item[Exclusiveness] = true
} else if (typeof testCaseOrGroup === 'object' && testCaseOrGroup !== null) {
if ('code' in testCaseOrGroup) {
testCaseOrGroup.only = true
} else {
if ('valid' in testCaseOrGroup && Array.isArray(testCaseOrGroup.valid)) {
only(testCaseOrGroup.valid)
}
if ('invalid' in testCaseOrGroup && Array.isArray(testCaseOrGroup.invalid)) {
only(testCaseOrGroup.invalid)
}
}
}
}

return item
return testCaseOrGroup
}

/**
Expand All @@ -42,85 +53,144 @@ function testRunner(
})

const oneOrMoreTestCaseIsSkipped = Object.values(rules).some(ruleModule =>
ruleModule.tests?.valid?.some(testCase => testCase[Exclusiveness]) ||
ruleModule.tests?.invalid?.some(testCase => testCase[Exclusiveness])
ruleModule.tests?.valid?.some(testCase =>
typeof testCase === 'object' && testCase.only
) ||
ruleModule.tests?.invalid?.some(testCase =>
testCase.only
)
)

const ruleList = Object.entries(rules).map(([ruleName, ruleModule]) => {
/**
* @type {Array<import('./types').TestCase>}
*/
const totalTestCases = [
...(ruleModule.tests?.valid || []).map(testCase =>
typeof testCase === 'string' ? { code: testCase } : testCase
),
...(ruleModule.tests?.invalid || []),
]

const selectTestCases = totalTestCases.filter(testCase =>
oneOrMoreTestCaseIsSkipped ? testCase.only : true
)

return {
ruleName,
ruleModule,
totalTestCases,
selectTestCases,
}
})

// Put rules that have zero and all-skipped test cases at the top respectively
ruleList.sort((left, right) => {
if (left.totalTestCases.length === 0 && right.totalTestCases.length === 0) {
return 0
} else if (left.totalTestCases.length === 0) {
return -1
} else if (right.totalTestCases.length === 0) {
return 1
}

if (left.selectTestCases.length === 0 && right.selectTestCases.length === 0) {
return 0
} else if (left.selectTestCases.length === 0) {
return -1
} else if (right.selectTestCases.length === 0) {
return 1
}

return 0
})

const stats = { pass: 0, fail: 0, skip: 0 }
for (const ruleName in rules) {
const ruleModule = rules[ruleName]
if (
!ruleModule.tests ||
typeof ruleModule.tests !== 'object' ||
!ruleModule.tests.valid && !ruleModule.tests.invalid
) {

for (const { ruleName, ruleModule, totalTestCases, selectTestCases } of ruleList) {
if (totalTestCases.length === 0) {
log('⚪ ' + ruleName)
continue
}

for (const testCase of ruleModule.tests.invalid || []) {
testCase.errors = testCase.errors ?? []
stats.skip += totalTestCases.length - selectTestCases.length

if (selectTestCases.length === 0) {
log('⏩ ' + ruleName)
continue
}

/**
* @type {Array<import('./types').TestCase>}
*/
const totalItems = [
...(ruleModule.tests.valid || []).map(testCase => typeof testCase === 'string' ? { code: testCase } : testCase),
...(ruleModule.tests.invalid || []),
]
const runningItems = totalItems.filter(testCase => oneOrMoreTestCaseIsSkipped ? !!testCase[Exclusiveness] : true)

const errors = runningItems.reduce((results, testCase) => {
try {
tester.run(
ruleName,
ruleModule,
'errors' in testCase ? { valid: [], invalid: [testCase] } : { valid: [testCase], invalid: [] }
)

} catch (error) {
results.push({ testCase, error })
}
const failingTestResults = selectTestCases.reduce(
/**
* @param {Array<import('./types').TestCase & { error: Error }>} results
*/
(results, { only, ...testCase }) => {
try {
tester.run(
ruleName,
ruleModule,
// Run one test case at a time
'errors' in testCase
? { valid: [], invalid: [testCase] }
: { valid: [testCase], invalid: [] }
)

} catch (error) {
results.push({ ...testCase, error })
}

return results
}, [])

if (failingTestResults.length > 0) {
log('🔴 ' + ruleName)
for (const failingTestCase of failingTestResults) {
if (failingTestCase !== failingTestResults[0]) {
// Add a blank line between test cases
log('')
}

return results
}, /** @type {Array<{ testCase: import('./types').TestCase, error: Error }>} */([]))
log(offset(failingTestCase.code, true, chalk.bgHex('#E0E0E0')))

stats.skip += totalItems.length - runningItems.length
stats.fail += errors.length
stats.pass += runningItems.length - errors.length
// See https://eslint.org/docs/latest/integrate/nodejs-api#ruletester
if (failingTestCase.name !== undefined) {
log(gray(' name: ') + failingTestCase.name)
}
if (failingTestCase.filename !== undefined) {
log(gray(' filename: ') + failingTestCase.filename)
}
if (failingTestCase.options !== undefined) {
log(gray(' options: ') + offset(JSON.stringify(failingTestCase.options, null, 2)).replace(/^\s*/, ''))
}

if (errors.length > 0) {
err('🔴 ' + ruleName)
for (const { testCase, error } of errors) {
err('')
err(offset(getPrettyCode(testCase.code), chalk.bgRed))
err('')
err(offset(error.message, chalk.red))
err(offset(failingTestCase.error.message))

if (bail) {
return 1
}
}

} else if (totalItems.length === runningItems.length) {
} else if (totalTestCases.length === selectTestCases.length) {
log('🟢 ' + ruleName)

} else if (runningItems.length > 0) {
log('🟡 ' + ruleName)

} else {
log('⏩ ' + ruleName)
log('🟡 ' + ruleName)
}

stats.pass += selectTestCases.length - failingTestResults.length
stats.fail += failingTestResults.length
}

log('')
log(chalk.bgGreen(chalk.bold(' PASS ')) + ' ' + stats.pass.toLocaleString())
if (stats.fail > 0) {
log(chalk.bgRed(chalk.bold(' FAIL ')) + ' ' + stats.fail.toLocaleString())
}

if (stats.skip > 0) {
log(chalk.bgHex('#0CAAEE')(chalk.bold(' SKIP ')) + ' ' + stats.skip.toLocaleString())
log(chalk.bgHex('#0CAAEE')(chalk.white.bold(' SKIP ')) + ' ' + stats.skip.toLocaleString())
}

log(chalk.bgGreen(chalk.white.bold(' PASS ')) + ' ' + stats.pass.toLocaleString())

if (stats.fail > 0) {
log(chalk.bgRed(chalk.white.bold(' FAIL ')) + ' ' + stats.fail.toLocaleString())
}

return stats.fail
Expand All @@ -134,26 +204,13 @@ module.exports.only = only
* @param {string} text
* @param {(line: string) => string} [decorateLine=line => line]
*/
function offset(text, decorateLine = line => line) {
return text.split('\n').map(line => ' ' + decorateLine(line)).join('\n')
}

/**
* @param {string} text
* @returns {string}
*/
function getPrettyCode(text) {
const trimmedCode = text.split('\n').filter((line, rank, list) =>
(rank === 0 || rank === list.length - 1) ? line.trim().length > 0 : true
)

const indent = trimmedCode
.filter(line => line.trim().length > 0)
.map(line => line.match(/^(\t|\s)+/)?.at(0) || '')
.reduce((output, indent) => indent.length < output.length ? indent : output, '')

return trimmedCode.map(line => line
.replace(new RegExp('^' + indent), '')
.replace(/^\t+/, tabs => ' '.repeat(tabs.length))
).join('\n')
function offset(text, lineNumberVisible = false, decorateLine = line => line) {
const lines = text.split('\n')
const lastLineDigitCount = Math.max(lines.length.toString().length, 2)
return lines.map((line, lineIndex) => {
const lineNumber = gray(
(lineIndex + 1).toString().padStart(lastLineDigitCount, ' ')
)
return (lineNumberVisible ? lineNumber : ' ') + ' ' + decorateLine(line)
}).join('\n')
}
39 changes: 19 additions & 20 deletions testRunner.test.js
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,14 @@
const { jest, afterEach, afterAll, it, expect } = require('@jest/globals')

jest.mock('chalk', () => ({
bold: (text) => text,
red: (text) => text,
bgRed: (text) => text,
bgGreen: (text) => text,
hex: () => (text) => text,
bgHex: () => (text) => text,
white: {
bold: (text) => text
}
}))

afterEach(() => {
Expand Down Expand Up @@ -81,16 +84,16 @@ it('returns non-zero errors, given any failing test case', () => {

expect(errorCount).toBe(2)
expect(log.mock.calls.join('\n')).toMatchInlineSnapshot(`
"
"🔴 foo
1 void(0)
1
PASS 0
FAIL 2"
`)
expect(err.mock.calls.join('\n')).toMatchInlineSnapshot(`
"🔴 foo
void(0)
Should have no errors but had 1: [
" Should have no errors but had 1: [
{
ruleId: 'rule-to-test/foo',
severity: 1,
Expand All @@ -102,9 +105,6 @@ it('returns non-zero errors, given any failing test case', () => {
endColumn: 8
}
] (1 strictEqual 0)
Should have 1 error but had 0: [] (0 strictEqual 1)"
`)
})
Expand Down Expand Up @@ -136,13 +136,12 @@ it('returns at most one error, given bailing out', () => {
const errorCount = testRunner(rules, { bail: true, log, err })

expect(errorCount).toBe(1)
expect(log.mock.calls.join('\n')).toMatchInlineSnapshot(`""`)
expect(err.mock.calls.join('\n')).toMatchInlineSnapshot(`
expect(log.mock.calls.join('\n')).toMatchInlineSnapshot(`
"🔴 foo
void(0)
Should have no errors but had 1: [
1 void(0)"
`)
expect(err.mock.calls.join('\n')).toMatchInlineSnapshot(`
" Should have no errors but had 1: [
{
ruleId: 'rule-to-test/foo',
severity: 1,
Expand Down Expand Up @@ -194,11 +193,11 @@ it('runs only the test case wrapped with `only` function', () => {
expect(rules.foo.create).toHaveBeenCalled()
expect(rules.loo.create).not.toHaveBeenCalled()
expect(log.mock.calls.join('\n')).toMatchInlineSnapshot(`
"🟡 foo
⏩ loo
"⏩ loo
🟡 foo
PASS 1
SKIP 3"
SKIP 3
PASS 1"
`)
})

Expand Down

0 comments on commit 80b987b

Please sign in to comment.