feat(cucumber): add cucumberjs multiCapability support, add extra log…

…ging

.gitignore

@@ -3,3 +3,4 @@ node_modules
 test/integration/support/times-flaked
 dist
 !dist/.gitkeep
+.idea

src/index.js

@@ -22,15 +22,17 @@ export default function (options = {}, callback = function noop () {}) {
     if (status === 0) {
       callback(status)
     } else {
+      let failedSpecs = parser.parse(output)
       if (++testAttempt <= parsedOptions.maxAttempts) {
         log('info', `Using ${parser.name} to parse output\n`)
-        let failedSpecs = parser.parse(output)
-
         log('info', `Re-running tests: test attempt ${testAttempt}\n`)
         log('info', 'Re-running the following test files:\n')
         log('info', failedSpecs.join('\n') + '\n')
         return startProtractor(failedSpecs)
       }
+      if (failedSpecs.length === 0) {
+        log('info', '\nTests failed but no specs were found. All specs have been run again.\n\n')
+      }
 
       callback(status, output)
     }

src/parsers/cucumber.multi.js

@@ -0,0 +1,29 @@
+export default {
+  name: 'cucumberMulti',
+  parse (output) {
+    let match = null
+    let failedSpecs = []
+    let testsOutput = output.split('------------------------------------')
+    let RESULT_FAIL = /(.*?) Failures:.*/g
+    let SPECFILE_REG = /(.*?) Specs:\s(.*\.feature)/g
+    testsOutput.forEach(function (test) {
+      let specfile
+      let result = 'passed'
+      // only check specs when RESULT_FAIL, ` Specs: ` is always printed when at least multiple features on 1 instance
+      // are run with `shardTestFiles: true`
+      if (RESULT_FAIL.exec(test)) { // eslint-disable-line no-cond-assign
+        while (match = SPECFILE_REG.exec(test)) { // eslint-disable-line no-cond-assign
+          specfile = match[2]
+          result = 'failed'
+        }
+      }
+      if (specfile && result === 'failed') {
+        if (!/node_modules/.test(specfile)) {
+          failedSpecs.push(specfile)
+        }
+      }
+    })
+    // Remove double values
+    return [...new Set(failedSpecs)]
+  }
+}

src/parsers/index.js

@@ -1,8 +1,9 @@
 import cucumber from './cucumber'
+import cucumberMulti from './cucumber.multi'
 import multi from './multi'
 import standard from './standard'
 
-let all = { cucumber, multi, standard }
+let all = { cucumber, cucumberMulti, multi, standard }
 
 function getParser (name) {
   if (name && all[name]) {

test/unit/parsers/cucumber.multi.test.js

@@ -0,0 +1,105 @@
+import readFixture from '../support/read-fixture'
+import multiParser from '../../../src/parsers/cucumber.multi'
+
+describe('cucumberMultiParser', () => {
+  /**
+   * How it works:
+   * Single:
+   * - 1 webdriver instance + 1 feature + error + shardTestFiles
+   *    No Specs !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+   * - 1 webdriver instance + 1 feature + error + !shardTestFiles
+   *    No Specs !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+   * - 1 webdriver instance + 1 feature + success + shardTestFiles
+   *    No Specs !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+   * - 1 webdriver instance + 1 feature + success + !shardTestFiles
+   *    No Specs !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+   * - 1 webdriver instance + 2 features + error + shardTestFiles
+   *    SPECS ###################################################
+   * - 1 webdriver instance + 2 features + error + !shardTestFiles
+   *    No Specs !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+   * - 1 webdriver instance + 2 features + success + shardTestFiles
+   *    SPECS ###################################################
+   * - 1 webdriver instance + 2 features + success + !shardTestFiles
+   *    No Specs !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+   *
+   * Conclusion:
+   *
+   *
+   * Multi:
+   * - 2 webdriver instances + 1 feature + error + !shardTestFiles
+   *    SPECS for both ############################################
+   * - 2 webdriver instances + 1 feature + error + shardTestFiles
+   *    SPECS for both ############################################
+   * - 2 webdriver instances + 1 feature + success + !shardTestFiles
+   *    SPECS for both ############################################
+   * - 2 webdriver instances + 1 feature + success + shardTestFiles
+   *    SPECS for both ############################################
+   * - 2 webdriver instances + 2 features + error + !shardTestFiles
+   *    NO specs for both !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+   * - 2 webdriver instances + 2 features + error + shardTestFiles
+   *    SPECS for both ############################################
+   * - 2 webdriver instances + 2 features + success + !shardTestFiles
+   *    NO specs for both !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+   * - 2 webdriver instances + 2 features + success + shardTestFiles
+   *    SPECS for both ############################################
+   *
+   * Conclusion:
+   * SHARED needs to be true for multiple. Then it's always printed. First check on `Failures:`, then on `Specs:`   *
+   */
+  describe('#parse', () => {
+    it('properly handles error output in multicapabilities tests with a feature', function () {
+      let output = readFixture('cucumberjs/cucumberjs-multi-output-feature-failure.txt')
+
+      expect(multiParser.parse(output)).to.eql([
+        '/Users/wswebcreation/test/e2e/features/functional/flakey.feature'
+      ])
+    })
+
+    it('properly handles error output in sharded multicapabilities tests with a feature', function () {
+      let output = readFixture('cucumberjs/cucumberjs-multi-output-shared-feature-failure.txt')
+
+      expect(multiParser.parse(output)).to.eql([
+        '/Users/wswebcreation/test/e2e/features/functional/flakey.feature'
+      ])
+    })
+
+    it('returns an empty array if cucumberjs output has no matches for a multicapabilities tests with a feature', function () {
+      let output = readFixture('cucumberjs/cucumberjs-multi-output-feature-success.txt')
+
+      expect(multiParser.parse(output)).to.eql([])
+    })
+
+    it('returns an empty array if cucumberjs output has no matches for a sharded multicapabilities tests with a feature', function () {
+      let output = readFixture('cucumberjs/cucumberjs-multi-output-shared-feature-success.txt')
+
+      expect(multiParser.parse(output)).to.eql([])
+    })
+
+    it('can\'t handles error output in multicapabilities tests with features', function () {
+      let output = readFixture('cucumberjs/cucumberjs-multi-output-features-failures.txt')
+
+      expect(multiParser.parse(output)).to.eql([])
+    })
+
+    it('properly handles error output in sharded multicapabilities tests with features and double failures', function () {
+      let output = readFixture('cucumberjs/cucumberjs-multi-output-shared-features-failures.txt')
+
+      expect(multiParser.parse(output)).to.eql([
+        '/Users/wswebcreation/test/e2e/features/functional/another.flakey.feature',
+        '/Users/wswebcreation/test/e2e/features/functional/flakey.feature'
+      ])
+    })
+
+    it('returns an empty array if cucumberjs output has no matches for a multicapabilities tests with features', function () {
+      let output = readFixture('cucumberjs/cucumberjs-multi-output-features-success.txt')
+
+      expect(multiParser.parse(output)).to.eql([])
+    })
+
+    it('returns an empty array if cucumberjs output has no matches for a sharded multicapabilities tests with features', function () {
+      let output = readFixture('cucumberjs/cucumberjs-multi-output-shared-features-success.txt')
+
+      expect(multiParser.parse(output)).to.eql([])
+    })
+  })
+})

test/unit/support/fixtures/cucumberjs/cucumberjs-multi-output-feature-failure.txt

@@ -0,0 +1,52 @@
+[launcher] Running 2 instances of WebDriver
+.
+------------------------------------
+[chrome #01] PID: 33160
+[chrome #01] Specs: /Users/wswebcreation/test/e2e/features/functional/flakey.feature
+[chrome #01] 
+[chrome #01] Using the selenium server at http://localhost:4444/wd/hub
+[chrome #01] Feature: Flake unit tests
+[chrome #01] 
+[chrome #01]   Scenario: A flakey scenario
+[chrome #01]     Then a flakey integration test fails for firefox
+[chrome #01] 
+[chrome #01] 1 scenario (1 passed)
+[chrome #01] 1 step (1 passed)
+[chrome #01] 0m00.007s
+
+[launcher] 1 instance(s) of WebDriver still running
+F
+------------------------------------
+[firefox #11] PID: 33161
+[firefox #11] Specs: /Users/wswebcreation/test/e2e/features/functional/flakey.feature
+[firefox #11] 
+[firefox #11] Using the selenium server at http://localhost:4444/wd/hub
+[firefox #11] Feature: Flake unit tests
+[firefox #11] 
+[firefox #11]   Scenario: A flakey scenario
+[firefox #11]     Then a flakey integration test fails for firefox
+[firefox #11] 
+[firefox #11] Failures:
+[firefox #11] 
+[firefox #11] 1) Scenario: A flakey scenario - test/e2e/features/functional/flakey.feature:6
+[firefox #11]    Then a flakey integration test fails for firefox - test/e2e/features/functional/flakey.feature:7
+[firefox #11]    Step Definition: test/e2e/step_definitions/functional.steps.js:56
+[firefox #11]    Message:
+[firefox #11]      AssertionError: expected true to equal false
+[firefox #11]          at World.<anonymous> (/Users/wswebcreation/test/e2e/step_definitions/functional.steps.js:59:32)
+[firefox #11]          at doNTCallback0 (node.js:428:9)
+[firefox #11]          at process._tickCallback (node.js:357:13)
+[firefox #11] 
+[firefox #11] 1 scenario (1 failed)
+[firefox #11] 1 step (1 failed)
+[firefox #11] 0m00.131s
+
+[launcher] 0 instance(s) of WebDriver still running
+[launcher] chrome #01 passed
+[launcher] firefox #11 failed 1 test(s)
+[launcher] overall: 1 failed spec(s)
+[launcher] Process exited with error code 1
+>> 
+Warning: Tests failed, protractor exited with code: 1 Use --force to continue.
+
+Aborted due to warnings.

test/unit/support/fixtures/cucumberjs/cucumberjs-multi-output-feature-success.txt

@@ -0,0 +1,37 @@
+[launcher] Running 2 instances of WebDriver
+.
+------------------------------------
+[chrome #01] PID: 33789
+[chrome #01] Specs: /Users/wswebcreation/e2e/features/functional/success.feature
+[chrome #01]
+[chrome #01] Using the selenium server at http://localhost:4444/wd/hub
+[chrome #01] Feature: Working feature
+[chrome #01]
+[chrome #01]   Scenario: A working scenario
+[chrome #01]     Then an integration test succeeds in a consistent manner
+[chrome #01]
+[chrome #01] 1 scenario (1 passed)
+[chrome #01] 1 step (1 passed)
+[chrome #01] 0m00.006s
+
+[launcher] 1 instance(s) of WebDriver still running
+.
+------------------------------------
+[firefox #11] PID: 33790
+[firefox #11] Specs: /Users/wswebcreation/e2e/features/functional/success.feature
+[firefox #11]
+[firefox #11] Using the selenium server at http://localhost:4444/wd/hub
+[firefox #11] Feature: Flake unit tests
+[firefox #11]
+[firefox #11]   Scenario: A flakey scenario
+[firefox #11]     Then a flakey integration test fails, in a horribly consistent manner
+[firefox #11]
+[firefox #11] 1 scenario (1 passed)
+[firefox #11] 1 step (1 passed)
+[firefox #11] 0m00.005s
+
+[launcher] 0 instance(s) of WebDriver still running
+[launcher] chrome #01 passed
+[launcher] firefox #11 passed
+
+Done.

test/unit/support/fixtures/cucumberjs/cucumberjs-multi-output-features-failures.txt

@@ -0,0 +1,78 @@
+[launcher] Running 2 instances of WebDriver
+F.
+------------------------------------
+[chrome #01] PID: 33973
+[chrome #01] Using the selenium server at http://localhost:4444/wd/hub
+[chrome #01] Feature: More flake unit tests
+[chrome #01] 
+[chrome #01]   Scenario: Another flakey scenario
+[chrome #01]     Then another flakey integration test fails, in a horribly consistent manner
+[chrome #01] 
+[chrome #01] Feature: Flake unit tests
+[chrome #01] 
+[chrome #01]   Scenario: A flakey scenario
+[chrome #01]     Then a flakey integration test fails, in a horribly consistent manner
+[chrome #01] 
+[chrome #01] Failures:
+[chrome #01] 
+[chrome #01] 1) Scenario: Another flakey scenario - test/e2e/features/functional/another.flakey.feature:6
+[chrome #01]    Step: Then another flakey integration test fails, in a horribly consistent manner - test/e2e/features/functional/another.flakey.feature:7
+[chrome #01]    Step Definition: test/e2e/step_definitions/functional.steps.js:62
+[chrome #01]    Message:
+[chrome #01]      AssertionError: expected true to equal false
+[chrome #01]          at World.<anonymous> (/Users/wswebcreation/test/e2e/step_definitions/functional.steps.js:63:32)
+[chrome #01]          at doNTCallback0 (node.js:428:9)
+[chrome #01]          at process._tickCallback (node.js:357:13)
+[chrome #01] 
+[chrome #01] 2 scenarios (1 failed, 1 passed)
+[chrome #01] 2 steps (1 failed, 1 passed)
+[chrome #01] 0m00.360s
+
+[launcher] 1 instance(s) of WebDriver still running
+FF
+------------------------------------
+[firefox #11] PID: 33974
+[firefox #11] Using the selenium server at http://localhost:4444/wd/hub
+[firefox #11] Feature: More flake unit tests
+[firefox #11] 
+[firefox #11]   Scenario: Another flakey scenario
+[firefox #11]     Then another flakey integration test fails, in a horribly consistent manner
+[firefox #11] 
+[firefox #11] Feature: Flake unit tests
+[firefox #11] 
+[firefox #11]   Scenario: A flakey scenario
+[firefox #11]     Then a flakey integration test fails, in a horribly consistent manner
+[firefox #11] 
+[firefox #11] Failures:
+[firefox #11] 
+[firefox #11] 1) Scenario: Another flakey scenario - test/e2e/features/functional/another.flakey.feature:6
+[firefox #11]    Step: Then another flakey integration test fails, in a horribly consistent manner - test/e2e/features/functional/another.flakey.feature:7
+[firefox #11]    Step Definition: test/e2e/step_definitions/functional.steps.js:62
+[firefox #11]    Message:
+[firefox #11]      AssertionError: expected true to equal false
+[firefox #11]          at World.<anonymous> (/Users/wswebcreation/test/e2e/step_definitions/functional.steps.js:63:32)
+[firefox #11]          at doNTCallback0 (node.js:428:9)
+[firefox #11]          at process._tickCallback (node.js:357:13)
+[firefox #11] 
+[firefox #11] 2) Scenario: A flakey scenario - test/e2e/features/functional/flakey.feature:6
+[firefox #11]    Step: Then a flakey integration test fails, in a horribly consistent manner - test/e2e/features/functional/flakey.feature:7
+[firefox #11]    Step Definition: test/e2e/step_definitions/functional.steps.js:56
+[firefox #11]    Message:
+[firefox #11]      AssertionError: expected true to equal false
+[firefox #11]          at World.<anonymous> (/Users/wswebcreation/test/e2e/step_definitions/functional.steps.js:59:32)
+[firefox #11]          at doNTCallback0 (node.js:428:9)
+[firefox #11]          at process._tickCallback (node.js:357:13)
+[firefox #11] 
+[firefox #11] 2 scenarios (2 failed)
+[firefox #11] 2 steps (2 failed)
+[firefox #11] 0m00.202s
+
+[launcher] 0 instance(s) of WebDriver still running
+[launcher] chrome #01 failed 1 test(s)
+[launcher] firefox #11 failed 2 test(s)
+[launcher] overall: 3 failed spec(s)
+[launcher] Process exited with error code 1
+>> 
+Warning: Tests failed, protractor exited with code: 1 Use --force to continue.
+
+Aborted due to warnings.

test/unit/support/fixtures/cucumberjs/cucumberjs-multi-output-features-success.txt

@@ -0,0 +1,43 @@
+[launcher] Running 2 instances of WebDriver
+..
+------------------------------------
+[chrome #01] PID: 34165
+[chrome #01] Using the selenium server at http://localhost:4444/wd/hub
+[chrome #01] Feature: More flake unit tests
+[chrome #01]
+[chrome #01]   Scenario: Another flakey scenario
+[chrome #01]     Then another flakey integration test fails, in a horribly consistent manner
+[chrome #01]
+[chrome #01] Feature: Flake unit tests
+[chrome #01]
+[chrome #01]   Scenario: A flakey scenario
+[chrome #01]     Then a flakey integration test fails, in a horribly consistent manner
+[chrome #01]
+[chrome #01] 2 scenarios (2 passed)
+[chrome #01] 2 steps (2 passed)
+[chrome #01] 0m00.007s
+
+[launcher] 1 instance(s) of WebDriver still running
+..
+------------------------------------
+[firefox #11] PID: 34166
+[firefox #11] Using the selenium server at http://localhost:4444/wd/hub
+[firefox #11] Feature: More flake unit tests
+[firefox #11]
+[firefox #11]   Scenario: Another flakey scenario
+[firefox #11]     Then another flakey integration test fails, in a horribly consistent manner
+[firefox #11]
+[firefox #11] Feature: Flake unit tests
+[firefox #11]
+[firefox #11]   Scenario: A flakey scenario
+[firefox #11]     Then a flakey integration test fails, in a horribly consistent manner
+[firefox #11]
+[firefox #11] 2 scenarios (2 passed)
+[firefox #11] 2 steps (2 passed)
+[firefox #11] 0m00.006s
+
+[launcher] 0 instance(s) of WebDriver still running
+[launcher] chrome #01 passed
+[launcher] firefox #11 passed
+
+Done.