From ea80f2cd7780bb3947b797a6342e5c961640e1c1 Mon Sep 17 00:00:00 2001 From: brine Date: Wed, 23 Aug 2017 12:46:20 -0500 Subject: [PATCH 1/4] feat: add support for jasmine-spec-reporter (displayStacktrace: true) --- src/parsers/standard.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/parsers/standard.js b/src/parsers/standard.js index 1b187b7..2a7d6e9 100644 --- a/src/parsers/standard.js +++ b/src/parsers/standard.js @@ -4,7 +4,7 @@ export default { parse (output) { let failedSpecs = new Set() let match = null - let FAILED_LINES = /at (?:\[object Object\]|Object)\.(?:|it) \((([A-Za-z]:\\)?.*?):.*\)/g + let FAILED_LINES = /at (?:\[object Object\]|Object|UserContext)\.(?:|it) \((([A-Za-z]:\\)?.*?):.*\)/g while (match = FAILED_LINES.exec(output)) { // eslint-disable-line no-cond-assign // windows output includes stack traces from // webdriver so we filter those out here From 9d446b7bab53f15295b6ffcfe38639ce47109db9 Mon Sep 17 00:00:00 2001 From: brine Date: Thu, 24 Aug 2017 09:36:30 -0500 Subject: [PATCH 2/4] fix: add unit test for jasmine-spec-reporter, and associated accoutrements --- package.json | 3 +- .../jasmine-spec-reporter.conf.js | 51 +++++++++++++++++++ test/unit/index.test.js | 10 ++++ ...iled-jasmine-spec-reporter-test-output.txt | 32 ++++++++++++ 4 files changed, 95 insertions(+), 1 deletion(-) create mode 100644 test/integration/support/protractor-config/jasmine-spec-reporter.conf.js create mode 100644 test/unit/support/fixtures/failed-jasmine-spec-reporter-test-output.txt diff --git a/package.json b/package.json index 2dfcfc3..6d04c0b 100644 --- a/package.json +++ b/package.json @@ -37,10 +37,11 @@ "cookie-parser": "^1.3.5", "express": "^4.13.1", "express-session": "^1.11.3", + "jasmine-spec-reporter": "^4.2.1", "mocha": "^2.2.5", "morgan": "^1.6.1", "mversion": "^1.10.0", - "protractor": "^4.0.11", + "protractor": "^5.1.2", "proxyquire": "^1.6.0", "semantic-release": "^6.3.6", "sinon": "^1.15.4", diff --git a/test/integration/support/protractor-config/jasmine-spec-reporter.conf.js b/test/integration/support/protractor-config/jasmine-spec-reporter.conf.js new file mode 100644 index 0000000..ad95acc --- /dev/null +++ b/test/integration/support/protractor-config/jasmine-spec-reporter.conf.js @@ -0,0 +1,51 @@ +'use strict'; + +var JOB_NUMBER = (process.env.TRAVIS_JOB_NUMBER || '') +var JOB_NAME = 'Flake' + JOB_NUMBER + +exports.config = { + specs: [ + '../flakey-test.js', + '../passing-test.js' + ], + + capabilities: { + browserName: 'chrome', + name: JOB_NAME, + shardTestFiles: false, + maxInstances: 2, + 'tunnel-identifier': JOB_NUMBER + }, + + sauceUser: process.env.SAUCE_USERNAME, + sauceKey: process.env.SAUCE_ACCESS_KEY, + + baseUrl: 'http://localhost:3000/', + + framework: 'jasmine2', + + allScriptsTimeout: 10000, + + getPageTimeout: 3000, + + jasmineNodeOpts: { + defaultTimeoutInterval: 5000, + print: function() {} // remove dot reporter + }, + + onPrepare: function () { + // let protractor know it doesn't need to look for angular on the page + browser.ignoreSynchronization = true; + + // make reports mo' pretty + var SpecReporter = require('jasmine-spec-reporter').SpecReporter; + jasmine.getEnv().addReporter( + new SpecReporter({ + spec: { + displayStacktrace: true + } + }) + ); + } +}; + diff --git a/test/unit/index.test.js b/test/unit/index.test.js index 016c840..332cae6 100644 --- a/test/unit/index.test.js +++ b/test/unit/index.test.js @@ -5,6 +5,7 @@ import parseOptions from '../../src/parse-options' const failedSingleTestOutput = readFixture('failed-test-output.txt') const failedShardedTestOutput = readFixture('sharded-failed-test-output.txt') +const failedJasmineSpecReporterTestOutput = readFixture('failed-jasmine-spec-reporter-test-output.txt') describe('Protractor Flake', () => { let spawnStub = null @@ -93,6 +94,15 @@ describe('Protractor Flake', () => { expect(spawnStub).to.have.been.calledWith('node', [pathToProtractor(), '--params.flake.retry', true, '--specs', '/tests/a-flakey.test.js']) }) + it('isolates individual failed specs from jasmine-spec-reporter output', () => { + protractorFlake({maxAttempts: 3}) + + spawnStub.dataCallback(failedJasmineSpecReporterTestOutput) + spawnStub.endCallback(1) + + expect(spawnStub).to.have.been.calledWith('node', [pathToProtractor(), '--params.flake.retry', true, '--specs', '/tests/flakey.test.js']) + }) + it('isolates failed specs for sharded protractor output', () => { protractorFlake({maxAttempts: 3}) diff --git a/test/unit/support/fixtures/failed-jasmine-spec-reporter-test-output.txt b/test/unit/support/fixtures/failed-jasmine-spec-reporter-test-output.txt new file mode 100644 index 0000000..d835c54 --- /dev/null +++ b/test/unit/support/fixtures/failed-jasmine-spec-reporter-test-output.txt @@ -0,0 +1,32 @@ +[08:04:24] I/launcher - Running 1 instances of WebDriver +[08:04:24] I/direct - Using ChromeDriver directly... +Jasmine started + + a flakey integration test + ✗ fails, in a horribly consistent manner + - Expected false to be truthy. + at UserContext. (/tests/flakey.test.js:7:39) + at node_modules/jasminewd2/index.js:112:25 + at new ManagedPromise (node_modules/selenium-webdriver/lib/promise.js:1067:7) + at ControlFlow.promise (node_modules/selenium-webdriver/lib/promise.js:2396:12) + at schedulerExecute (node_modules/jasminewd2/index.js:95:18) + at TaskQueue.execute_ (node_modules/selenium-webdriver/lib/promise.js:2970:14) + at TaskQueue.executeNext_ (node_modules/selenium-webdriver/lib/promise.js:2953:27) + at asyncRun (node_modules/selenium-webdriver/lib/promise.js:2813:27) + at node_modules/selenium-webdriver/lib/promise.js:676:7 + + A test that passes + ✓ passes + +************************************************** +* Failures * +************************************************** + +1) a flakey integration test fails, in a horribly consistent manner + - Expected false to be truthy. + +Executed 2 of 2 specs (1 FAILED) in 0.742 sec. +[08:04:26] I/launcher - 0 instance(s) of WebDriver still running +[08:04:26] I/launcher - chrome #01 failed 1 test(s) +[08:04:26] I/launcher - overall: 1 failed spec(s) +[08:04:26] E/launcher - Process exited with error code 1 \ No newline at end of file From 1d2d7900e7766c0bd2646f3ae94e8d1f3f16574c Mon Sep 17 00:00:00 2001 From: brine Date: Thu, 24 Aug 2017 12:40:18 -0500 Subject: [PATCH 3/4] fix: removed unneeded config file and associated dependancy --- package.json | 1 - .../jasmine-spec-reporter.conf.js | 51 ------------------- 2 files changed, 52 deletions(-) delete mode 100644 test/integration/support/protractor-config/jasmine-spec-reporter.conf.js diff --git a/package.json b/package.json index 6d04c0b..de6e4aa 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,6 @@ "cookie-parser": "^1.3.5", "express": "^4.13.1", "express-session": "^1.11.3", - "jasmine-spec-reporter": "^4.2.1", "mocha": "^2.2.5", "morgan": "^1.6.1", "mversion": "^1.10.0", diff --git a/test/integration/support/protractor-config/jasmine-spec-reporter.conf.js b/test/integration/support/protractor-config/jasmine-spec-reporter.conf.js deleted file mode 100644 index ad95acc..0000000 --- a/test/integration/support/protractor-config/jasmine-spec-reporter.conf.js +++ /dev/null @@ -1,51 +0,0 @@ -'use strict'; - -var JOB_NUMBER = (process.env.TRAVIS_JOB_NUMBER || '') -var JOB_NAME = 'Flake' + JOB_NUMBER - -exports.config = { - specs: [ - '../flakey-test.js', - '../passing-test.js' - ], - - capabilities: { - browserName: 'chrome', - name: JOB_NAME, - shardTestFiles: false, - maxInstances: 2, - 'tunnel-identifier': JOB_NUMBER - }, - - sauceUser: process.env.SAUCE_USERNAME, - sauceKey: process.env.SAUCE_ACCESS_KEY, - - baseUrl: 'http://localhost:3000/', - - framework: 'jasmine2', - - allScriptsTimeout: 10000, - - getPageTimeout: 3000, - - jasmineNodeOpts: { - defaultTimeoutInterval: 5000, - print: function() {} // remove dot reporter - }, - - onPrepare: function () { - // let protractor know it doesn't need to look for angular on the page - browser.ignoreSynchronization = true; - - // make reports mo' pretty - var SpecReporter = require('jasmine-spec-reporter').SpecReporter; - jasmine.getEnv().addReporter( - new SpecReporter({ - spec: { - displayStacktrace: true - } - }) - ); - } -}; - From f9229b55a850a858a83f7d0487a6231eff64c68b Mon Sep 17 00:00:00 2001 From: brine Date: Thu, 24 Aug 2017 15:01:21 -0500 Subject: [PATCH 4/4] fix: added unit test for sharded jasmine-spec-report --- test/unit/index.test.js | 10 ++++ ...iled-jasmine-spec-reporter-test-output.txt | 53 +++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 test/unit/support/fixtures/multicapabilities-failed-jasmine-spec-reporter-test-output.txt diff --git a/test/unit/index.test.js b/test/unit/index.test.js index 332cae6..af6992b 100644 --- a/test/unit/index.test.js +++ b/test/unit/index.test.js @@ -6,6 +6,7 @@ import parseOptions from '../../src/parse-options' const failedSingleTestOutput = readFixture('failed-test-output.txt') const failedShardedTestOutput = readFixture('sharded-failed-test-output.txt') const failedJasmineSpecReporterTestOutput = readFixture('failed-jasmine-spec-reporter-test-output.txt') +const failedShardedJasmineSpecReporterTestOutput = readFixture('multicapabilities-failed-jasmine-spec-reporter-test-output.txt') describe('Protractor Flake', () => { let spawnStub = null @@ -103,6 +104,15 @@ describe('Protractor Flake', () => { expect(spawnStub).to.have.been.calledWith('node', [pathToProtractor(), '--params.flake.retry', true, '--specs', '/tests/flakey.test.js']) }) + it('isolates individual failed specs for sharded jasmine-spec-reporter output', () => { + protractorFlake({maxAttempts: 3}) + + spawnStub.dataCallback(failedShardedJasmineSpecReporterTestOutput) + spawnStub.endCallback(1) + + expect(spawnStub).to.have.been.calledWith('node', [pathToProtractor(), '--params.flake.retry', true, '--specs', '/tests/flakey.test.js']) + }) + it('isolates failed specs for sharded protractor output', () => { protractorFlake({maxAttempts: 3}) diff --git a/test/unit/support/fixtures/multicapabilities-failed-jasmine-spec-reporter-test-output.txt b/test/unit/support/fixtures/multicapabilities-failed-jasmine-spec-reporter-test-output.txt new file mode 100644 index 0000000..58f03ef --- /dev/null +++ b/test/unit/support/fixtures/multicapabilities-failed-jasmine-spec-reporter-test-output.txt @@ -0,0 +1,53 @@ +[14:29:59] I/launcher - Running 2 instances of WebDriver +[14:30:04] I/testLogger - +------------------------------------ + +[14:30:04] I/testLogger - [chrome #01-1] PID: 81285 +[chrome #01-1] Specs: /tests/passing-test.js +[chrome #01-1] +[chrome #01-1] (node:81285) [DEP0022] DeprecationWarning: os.tmpDir() is deprecated. Use os.tmpdir() instead. +[chrome #01-1] [14:30:00] I/sauce - Using SauceLabs selenium server at https://ondemand.saucelabs.com:443/wd/hub +[chrome #01-1] Jasmine started +[chrome #01-1] +[chrome #01-1] A test that passes +[chrome #01-1] ✓ passes +[chrome #01-1] +[chrome #01-1] Executed 1 of 1 spec SUCCESS in 0.014 sec. +[chrome #01-1] [14:30:03] I/sauce - SauceLabs results available at http://saucelabs.com/jobs/93e3737e1fa147ea9d517c0fee7ea76e + +[14:30:04] I/testLogger - + +[14:30:04] I/launcher - 1 instance(s) of WebDriver still running +[14:30:04] I/testLogger - +------------------------------------ +[14:30:04] I/testLogger - [chrome #01-0] PID: 81284 +[chrome #01-0] Specs: /tests/flakey.test.js +[chrome #01-0] +[chrome #01-0] (node:81284) [DEP0022] DeprecationWarning: os.tmpDir() is deprecated. Use os.tmpdir() instead. +[chrome #01-0] [14:30:00] I/sauce - Using SauceLabs selenium server at https://ondemand.saucelabs.com:443/wd/hub +[chrome #01-0] Jasmine started +[chrome #01-0] +[chrome #01-0] a flakey integration test +[chrome #01-0] ✗ fails, in a horribly consistent manner +[chrome #01-0] - Expected false to be truthy. +[chrome #01-0] at UserContext. (/tests/flakey.test.js:7:39) +[chrome #01-0] at /node_modules/protractor/node_modules/jasminewd2/index.js:112:25 +[chrome #01-0] at new ManagedPromise (/node_modules/protractor/node_modules/selenium-webdriver/lib/promise.js:1067:7) +[chrome #01-0] at ControlFlow.promise (/node_modules/protractor/node_modules/selenium-webdriver/lib/promise.js:2396:12) +[chrome #01-0] at schedulerExecute (/node_modules/protractor/node_modules/jasminewd2/index.js:95:18) +[chrome #01-0] at TaskQueue.execute_ (node_modules/protractor/node_modules/selenium-webdriver/lib/promise.js:2970:14) +[chrome #01-0] at TaskQueue.executeNext_ (node_modules/protractor/node_modules/selenium-webdriver/lib/promise.js:2953:27) +[chrome #01-0] at asyncRun (node_modules/protractor/node_modules/selenium-webdriver/lib/promise.js:2813:27) +[chrome #01-0] at node_modules/protractor/node_modules/selenium-webdriver/lib/promise.js:676:7 +[chrome #01-0] +[chrome #01-0] ************************************************** +[chrome #01-0] * Failures * +[chrome #01-0] ************************************************** +[chrome #01-0] +[chrome #01-0] 1) a flakey integration test fails, in a horribly consistent manner +[chrome #01-0] - Expected false to be truthy. +[chrome #01-0] +[chrome #01-0] Executed 1 of 1 spec (1 FAILED) in 0.933 sec. +[chrome #01-0] [14:30:03] I/sauce - SauceLabs results available at http://saucelabs.com/jobs/dfdb1646bae041a08b0c36cdbf3b19a5 + +[14:30:04] I/testLogger -