diff --git a/package.json b/package.json index 2dfcfc3..de6e4aa 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,7 @@ "mocha": "^2.2.5", "morgan": "^1.6.1", "mversion": "^1.10.0", - "protractor": "^4.0.11", + "protractor": "^5.1.2", "proxyquire": "^1.6.0", "semantic-release": "^6.3.6", "sinon": "^1.15.4", diff --git a/src/parsers/standard.js b/src/parsers/standard.js index 1b187b7..2a7d6e9 100644 --- a/src/parsers/standard.js +++ b/src/parsers/standard.js @@ -4,7 +4,7 @@ export default { parse (output) { let failedSpecs = new Set() let match = null - let FAILED_LINES = /at (?:\[object Object\]|Object)\.(?:|it) \((([A-Za-z]:\\)?.*?):.*\)/g + let FAILED_LINES = /at (?:\[object Object\]|Object|UserContext)\.(?:|it) \((([A-Za-z]:\\)?.*?):.*\)/g while (match = FAILED_LINES.exec(output)) { // eslint-disable-line no-cond-assign // windows output includes stack traces from // webdriver so we filter those out here diff --git a/test/unit/index.test.js b/test/unit/index.test.js index 016c840..af6992b 100644 --- a/test/unit/index.test.js +++ b/test/unit/index.test.js @@ -5,6 +5,8 @@ import parseOptions from '../../src/parse-options' const failedSingleTestOutput = readFixture('failed-test-output.txt') const failedShardedTestOutput = readFixture('sharded-failed-test-output.txt') +const failedJasmineSpecReporterTestOutput = readFixture('failed-jasmine-spec-reporter-test-output.txt') +const failedShardedJasmineSpecReporterTestOutput = readFixture('multicapabilities-failed-jasmine-spec-reporter-test-output.txt') describe('Protractor Flake', () => { let spawnStub = null @@ -93,6 +95,24 @@ describe('Protractor Flake', () => { expect(spawnStub).to.have.been.calledWith('node', [pathToProtractor(), '--params.flake.retry', true, '--specs', '/tests/a-flakey.test.js']) }) + it('isolates individual failed specs from jasmine-spec-reporter output', () => { + protractorFlake({maxAttempts: 3}) + + spawnStub.dataCallback(failedJasmineSpecReporterTestOutput) + spawnStub.endCallback(1) + + expect(spawnStub).to.have.been.calledWith('node', [pathToProtractor(), '--params.flake.retry', true, '--specs', '/tests/flakey.test.js']) + }) + + it('isolates individual failed specs for sharded jasmine-spec-reporter output', () => { + protractorFlake({maxAttempts: 3}) + + spawnStub.dataCallback(failedShardedJasmineSpecReporterTestOutput) + spawnStub.endCallback(1) + + expect(spawnStub).to.have.been.calledWith('node', [pathToProtractor(), '--params.flake.retry', true, '--specs', '/tests/flakey.test.js']) + }) + it('isolates failed specs for sharded protractor output', () => { protractorFlake({maxAttempts: 3}) diff --git a/test/unit/support/fixtures/failed-jasmine-spec-reporter-test-output.txt b/test/unit/support/fixtures/failed-jasmine-spec-reporter-test-output.txt new file mode 100644 index 0000000..d835c54 --- /dev/null +++ b/test/unit/support/fixtures/failed-jasmine-spec-reporter-test-output.txt @@ -0,0 +1,32 @@ +[08:04:24] I/launcher - Running 1 instances of WebDriver +[08:04:24] I/direct - Using ChromeDriver directly... +Jasmine started + + a flakey integration test + ✗ fails, in a horribly consistent manner + - Expected false to be truthy. + at UserContext. (/tests/flakey.test.js:7:39) + at node_modules/jasminewd2/index.js:112:25 + at new ManagedPromise (node_modules/selenium-webdriver/lib/promise.js:1067:7) + at ControlFlow.promise (node_modules/selenium-webdriver/lib/promise.js:2396:12) + at schedulerExecute (node_modules/jasminewd2/index.js:95:18) + at TaskQueue.execute_ (node_modules/selenium-webdriver/lib/promise.js:2970:14) + at TaskQueue.executeNext_ (node_modules/selenium-webdriver/lib/promise.js:2953:27) + at asyncRun (node_modules/selenium-webdriver/lib/promise.js:2813:27) + at node_modules/selenium-webdriver/lib/promise.js:676:7 + + A test that passes + ✓ passes + +************************************************** +* Failures * +************************************************** + +1) a flakey integration test fails, in a horribly consistent manner + - Expected false to be truthy. + +Executed 2 of 2 specs (1 FAILED) in 0.742 sec. +[08:04:26] I/launcher - 0 instance(s) of WebDriver still running +[08:04:26] I/launcher - chrome #01 failed 1 test(s) +[08:04:26] I/launcher - overall: 1 failed spec(s) +[08:04:26] E/launcher - Process exited with error code 1 \ No newline at end of file diff --git a/test/unit/support/fixtures/multicapabilities-failed-jasmine-spec-reporter-test-output.txt b/test/unit/support/fixtures/multicapabilities-failed-jasmine-spec-reporter-test-output.txt new file mode 100644 index 0000000..58f03ef --- /dev/null +++ b/test/unit/support/fixtures/multicapabilities-failed-jasmine-spec-reporter-test-output.txt @@ -0,0 +1,53 @@ +[14:29:59] I/launcher - Running 2 instances of WebDriver +[14:30:04] I/testLogger - +------------------------------------ + +[14:30:04] I/testLogger - [chrome #01-1] PID: 81285 +[chrome #01-1] Specs: /tests/passing-test.js +[chrome #01-1] +[chrome #01-1] (node:81285) [DEP0022] DeprecationWarning: os.tmpDir() is deprecated. Use os.tmpdir() instead. +[chrome #01-1] [14:30:00] I/sauce - Using SauceLabs selenium server at https://ondemand.saucelabs.com:443/wd/hub +[chrome #01-1] Jasmine started +[chrome #01-1] +[chrome #01-1] A test that passes +[chrome #01-1] ✓ passes +[chrome #01-1] +[chrome #01-1] Executed 1 of 1 spec SUCCESS in 0.014 sec. +[chrome #01-1] [14:30:03] I/sauce - SauceLabs results available at http://saucelabs.com/jobs/93e3737e1fa147ea9d517c0fee7ea76e + +[14:30:04] I/testLogger - + +[14:30:04] I/launcher - 1 instance(s) of WebDriver still running +[14:30:04] I/testLogger - +------------------------------------ +[14:30:04] I/testLogger - [chrome #01-0] PID: 81284 +[chrome #01-0] Specs: /tests/flakey.test.js +[chrome #01-0] +[chrome #01-0] (node:81284) [DEP0022] DeprecationWarning: os.tmpDir() is deprecated. Use os.tmpdir() instead. +[chrome #01-0] [14:30:00] I/sauce - Using SauceLabs selenium server at https://ondemand.saucelabs.com:443/wd/hub +[chrome #01-0] Jasmine started +[chrome #01-0] +[chrome #01-0] a flakey integration test +[chrome #01-0] ✗ fails, in a horribly consistent manner +[chrome #01-0] - Expected false to be truthy. +[chrome #01-0] at UserContext. (/tests/flakey.test.js:7:39) +[chrome #01-0] at /node_modules/protractor/node_modules/jasminewd2/index.js:112:25 +[chrome #01-0] at new ManagedPromise (/node_modules/protractor/node_modules/selenium-webdriver/lib/promise.js:1067:7) +[chrome #01-0] at ControlFlow.promise (/node_modules/protractor/node_modules/selenium-webdriver/lib/promise.js:2396:12) +[chrome #01-0] at schedulerExecute (/node_modules/protractor/node_modules/jasminewd2/index.js:95:18) +[chrome #01-0] at TaskQueue.execute_ (node_modules/protractor/node_modules/selenium-webdriver/lib/promise.js:2970:14) +[chrome #01-0] at TaskQueue.executeNext_ (node_modules/protractor/node_modules/selenium-webdriver/lib/promise.js:2953:27) +[chrome #01-0] at asyncRun (node_modules/protractor/node_modules/selenium-webdriver/lib/promise.js:2813:27) +[chrome #01-0] at node_modules/protractor/node_modules/selenium-webdriver/lib/promise.js:676:7 +[chrome #01-0] +[chrome #01-0] ************************************************** +[chrome #01-0] * Failures * +[chrome #01-0] ************************************************** +[chrome #01-0] +[chrome #01-0] 1) a flakey integration test fails, in a horribly consistent manner +[chrome #01-0] - Expected false to be truthy. +[chrome #01-0] +[chrome #01-0] Executed 1 of 1 spec (1 FAILED) in 0.933 sec. +[chrome #01-0] [14:30:03] I/sauce - SauceLabs results available at http://saucelabs.com/jobs/dfdb1646bae041a08b0c36cdbf3b19a5 + +[14:30:04] I/testLogger -