Skip to content

Commit

Permalink
feat(config): add failOnSkippedTests option. (#3374)
Browse files Browse the repository at this point in the history
Recently jasmine added support for reporting pending (incl due to fit()) tests.
This appears in results as skipped tests and this option allows us to fail tests
with skipped tests'
  • Loading branch information
johnjbarton authored Oct 17, 2019
1 parent c7bab36 commit 4ed3af0
Show file tree
Hide file tree
Showing 3 changed files with 54 additions and 18 deletions.
10 changes: 10 additions & 0 deletions docs/config/01-configuration-file.md
Original file line number Diff line number Diff line change
Expand Up @@ -383,6 +383,16 @@ The server can be stopped using the `karma stop` command.
**Description:** Enable or disable failure on running empty test-suites. If disabled the program
will return exit-code `0` and display a warning.

## failOnSkippedTests
**Type:** Boolean

**Default:** `false`

**CLI:** `--fail-on-skipped-tests`, `--no-fail-on-skipped-tests`

**Description:** Enable or disable failure on tests deliberately disabled, eg fit() or xit() tests in jasmine.
Use this to prevent accidental disabling tests needed to validate production.

## failOnFailingTestSuite
**Type:** Boolean

Expand Down
25 changes: 16 additions & 9 deletions lib/browser_collection.js
Original file line number Diff line number Diff line change
Expand Up @@ -38,30 +38,37 @@ class BrowserCollection {
return this.browsers.map((browser) => browser.serialize())
}

calculateExitCode (results, singleRunBrowserNotCaptured, failOnEmptyTestSuite, failOnFailingTestSuite) {
calculateExitCode (results, singleRunBrowserNotCaptured, config) {
config = config || {}
if (results.disconnected || singleRunBrowserNotCaptured) {
return 1
} else if (results.success + results.failed === 0 && !failOnEmptyTestSuite) {
}
if (results.skipped && config.failOnSkippedTests) {
return 1
}
if (results.success + results.failed === 0 && !config.failOnEmptyTestSuite) {
return 0
} else if (results.error) {
}
if (results.error) {
return 1
} else if (failOnFailingTestSuite === false) {
}
if (config.failOnFailingTestSuite === false) {
return 0 // Tests executed without infrastructure error, exit with 0 independent of test status.
} else {
return results.failed ? 1 : 0
}
return results.failed ? 1 : 0
}

getResults (singleRunBrowserNotCaptured, failOnEmptyTestSuite, failOnFailingTestSuite) {
const results = { success: 0, failed: 0, error: false, disconnected: false, exitCode: 0 }
getResults (singleRunBrowserNotCaptured, config) {
const results = { success: 0, failed: 0, skipped: 0, error: false, disconnected: false, exitCode: 0 }
this.browsers.forEach((browser) => {
results.success += browser.lastResult.success
results.failed += browser.lastResult.failed
results.skipped += browser.lastResult.skipped
results.error = results.error || browser.lastResult.error
results.disconnected = results.disconnected || browser.lastResult.disconnected
})

results.exitCode = this.calculateExitCode(results, singleRunBrowserNotCaptured, failOnEmptyTestSuite, failOnFailingTestSuite)
results.exitCode = this.calculateExitCode(results, singleRunBrowserNotCaptured, config)
return results
}

Expand Down
37 changes: 28 additions & 9 deletions test/unit/browser_collection.spec.js
Original file line number Diff line number Diff line change
Expand Up @@ -281,15 +281,15 @@ describe('BrowserCollection', () => {
expect(res).to.be.equal(EXIT_CODE_SUCCESS)
})
it('shall pass if failOnEmptyTestSuite is false', () => {
const res = collection.calculateExitCode(results, false, false)
const res = collection.calculateExitCode(results, false, { failOnEmptyTestSuite: false })
expect(res).to.be.equal(EXIT_CODE_SUCCESS)
})
it('shall fail if failOnEmptyTestSuite is true', () => {
const res = collection.calculateExitCode(results, false, true)
const res = collection.calculateExitCode(results, false, { failOnEmptyTestSuite: true })
expect(res).to.be.equal(EXIT_CODE_ERROR)
})
it('shall fail if failOnFailingTestSuite is set', () => {
const res = collection.calculateExitCode(results, false, true, true)
const res = collection.calculateExitCode(results, false, { failOnEmptyTestSuite: true, failOnFailingTestSuite: true })
expect(res).to.be.equal(EXIT_CODE_ERROR)
})
})
Expand All @@ -308,15 +308,15 @@ describe('BrowserCollection', () => {
expect(res).to.be.equal(EXIT_CODE_SUCCESS)
})
it('shall pass if failOnEmptyTestSuite not is set', () => {
const res = collection.calculateExitCode(results, false, false)
const res = collection.calculateExitCode(results, false, { failOnEmptyTestSuite: false })
expect(res).to.be.equal(EXIT_CODE_SUCCESS)
})
it('shall pass if failOnFailingTestSuite is true', () => {
const res = collection.calculateExitCode(results, false, true, true)
const res = collection.calculateExitCode(results, false, { failOnEmptyTestSuite: true, failOnFailingTestSuite: true })
expect(res).to.be.equal(EXIT_CODE_SUCCESS)
})
it('shall pass if failOnFailingTestSuite is false', () => {
const res = collection.calculateExitCode(results, false, true, false)
const res = collection.calculateExitCode(results, false, { failOnEmptyTestSuite: true, failOnFailingTestSuite: false })
expect(res).to.be.equal(EXIT_CODE_SUCCESS)
})
})
Expand All @@ -335,15 +335,34 @@ describe('BrowserCollection', () => {
expect(res).to.be.equal(EXIT_CODE_ERROR)
})
it('shall fail if failOnEmptyTestSuite not is set', () => {
const res = collection.calculateExitCode(results, false, false)
const res = collection.calculateExitCode(results, false, { failOnEmptyTestSuite: false })
expect(res).to.be.equal(EXIT_CODE_ERROR)
})
it('shall fail if failOnFailingTestSuite is true', () => {
const res = collection.calculateExitCode(results, false, true, true)
const res = collection.calculateExitCode(results, false, { failOnEmptyTestSuite: true, failOnFailingTestSuite: true })
expect(res).to.be.equal(EXIT_CODE_ERROR)
})
it('shall pass if failOnFailingTestSuite is false', () => {
const res = collection.calculateExitCode(results, false, true, false)
const res = collection.calculateExitCode(results, false, { failOnEmptyTestSuite: true, failOnFailingTestSuite: false })
expect(res).to.be.equal(EXIT_CODE_SUCCESS)
})
})
describe('all test passed, with skipped tests', () => {
const results = {
success: 10,
skipped: 5,
error: false
}
it('shall fail if singleRunBrowserNotCaptured is true', () => {
const res = collection.calculateExitCode(results, true)
expect(res).to.be.equal(EXIT_CODE_ERROR)
})
it('shall fail if failOnSkippedTests is true', () => {
const res = collection.calculateExitCode(results, false, { failOnEmptyTestSuite: true, failOnSkippedTests: true })
expect(res).to.be.equal(EXIT_CODE_ERROR)
})
it('shall pass if failOnSkippedTests is false', () => {
const res = collection.calculateExitCode(results, false, { failOnEmptyTestSuite: true, failOnSkippedTests: false })
expect(res).to.be.equal(EXIT_CODE_SUCCESS)
})
})
Expand Down

0 comments on commit 4ed3af0

Please sign in to comment.