feat: support test retries

cli/types/index.d.ts

@@ -2536,8 +2536,22 @@ declare namespace Cypress {
     disableTimersAndAnimations: boolean
     padding: Padding
     scale: boolean
-    beforeScreenshot(doc: Document): void
-    afterScreenshot(doc: Document): void
+    beforeScreenshot: ($el: JQuery<HTMLElement>) => void
+    afterScreenshot: ($el: JQuery<HTMLElement>, props: {
+      path: string,
+      size: 12,
+      dimensions: {
+        width: number,
+        height: number
+      },
+      multipart: boolean,
+      pixelRatio: number,
+      takenAt: string,
+      name: string,
+      blackout: string[],
+      duration: number,
+      testAttemptIndex: number
+    }) => void
   }
 
   interface ScreenshotDefaultsOptions extends ScreenshotOptions {
@@ -4739,7 +4753,7 @@ declare namespace Cypress {
     name: string
     /** Override *name* for display purposes only */
     displayName: string
-    message: any[]
+    message: any[] | string
     /** Return an object that will be printed in the dev tools console */
     consoleProps(): ObjectLike
   }

packages/driver/package.json

@@ -19,6 +19,10 @@
     "@packages/coffee": "*",
     "@packages/runner": "*",
     "@packages/ts": "*",
+    "@types/chalk": "^2.2.0",
+    "@types/common-tags": "^1.8.0",
+    "@types/lodash": "^4.14.123",
+    "@types/mocha": "^5.2.6",
     "angular": "1.7.9",
     "backbone": "1.4.0",
     "basic-auth": "2.0.1",
@@ -56,6 +60,7 @@
     "parse-domain": "bahmutov/parse-domain#fb60bd4",
     "setimmediate": "1.0.5",
     "sinon": "8.1.1",
+    "snap-shot-core": "^7.4.0",
     "text-mask-addons": "3.8.0",
     "underscore": "1.9.1",
     "underscore.string": "3.3.5",

packages/driver/src/cy/aliases.coffee

@@ -32,15 +32,15 @@ validateAlias = (alias) ->
   if alias in blacklist
     $errUtils.throwErrByPath "as.reserved_word", { args: { alias } }
 
-create = (state) ->
+create = (getRemotejQueryInstance, state) ->
   addAlias = (ctx, aliasObj) ->
     { alias, subject } = aliasObj
 
     aliases = state("aliases") ? {}
     aliases[alias] = aliasObj
     state("aliases", aliases)
 
-    remoteSubject = cy.getRemotejQueryInstance(subject)
+    remoteSubject = getRemotejQueryInstance(subject)
 
     ## assign the subject to our runnable ctx
     ctx[alias] = remoteSubject ? subject

packages/driver/src/cy/assertions.js

@@ -1,4 +1,3 @@
-/* global cy Cypress */
 const _ = require('lodash')
 const Promise = require('bluebird')
 
@@ -70,14 +69,14 @@ const parseValueActualAndExpected = (value, actual, expected) => {
   return obj
 }
 
-const create = function (state, queue, retryFn) {
+const create = function (Cypress, cy) {
   const getUpcomingAssertions = () => {
-    const index = state('index') + 1
+    const index = cy.state('index') + 1
 
     const assertions = []
 
     // grab the rest of the queue'd commands
-    for (let cmd of queue.slice(index).get()) {
+    for (let cmd of cy.queue.slice(index).get()) {
       // don't break on utilities, just skip over them
       if (cmd.is('utility')) {
         continue
@@ -112,7 +111,7 @@ const create = function (state, queue, retryFn) {
       cmd.set('assertionIndex', 0)
 
       return cmd.get('fn').originalFn.apply(
-        state('ctx'),
+        cy.state('ctx'),
         [subject].concat(cmd.get('args')),
       )
     })
@@ -135,7 +134,7 @@ const create = function (state, queue, retryFn) {
   const verifyUpcomingAssertions = function (subject, options = {}, callbacks = {}) {
     const cmds = getUpcomingAssertions()
 
-    state('upcomingAssertions', cmds)
+    cy.state('upcomingAssertions', cmds)
 
     // we're applying the default assertion in the
     // case where there are no upcoming assertion commands
@@ -228,7 +227,7 @@ const create = function (state, queue, retryFn) {
       }
 
       if (_.isFunction(onRetry)) {
-        return retryFn(onRetry, options)
+        return cy.retry(onRetry, options)
       }
     }
 
@@ -258,7 +257,7 @@ const create = function (state, queue, retryFn) {
         }
 
         // when we do immediately unbind this function
-        state('onBeforeLog', null)
+        cy.state('onBeforeLog', null)
 
         const insertNewLog = (log) => {
           cmd.log(log)
@@ -341,7 +340,7 @@ const create = function (state, queue, retryFn) {
         return insertNewLog(log)
       }
 
-      state('onBeforeLog', setCommandLog)
+      cy.state('onBeforeLog', setCommandLog)
 
       // send verify=true as the last arg
       return assertFn.apply(this, args.concat(true))
@@ -381,16 +380,16 @@ const create = function (state, queue, retryFn) {
     }
 
     const restore = () => {
-      state('upcomingAssertions', [])
+      cy.state('upcomingAssertions', [])
 
       // no matter what we need to
       // restore the assert fn
-      return state('overrideAssert', undefined)
+      return cy.state('overrideAssert', undefined)
     }
 
     // store this in case our test ends early
     // and we reset between tests
-    state('overrideAssert', overrideAssert)
+    cy.state('overrideAssert', overrideAssert)
 
     return Promise
     .reduce(fns, assertions, [subject])
@@ -459,7 +458,7 @@ const create = function (state, queue, retryFn) {
         // if our current command is an assertion type
         isAssertionType(current) ||
         // are we currently verifying assertions?
-        (state('upcomingAssertions') && state('upcomingAssertions').length > 0) ||
+        (cy.state('upcomingAssertions') && cy.state('upcomingAssertions').length > 0) ||
         // did the function have arguments
         functionHadArguments(current)
     }
@@ -503,7 +502,7 @@ const create = function (state, queue, retryFn) {
   const assert = function (...args) {
     // if we've temporarily overriden assertions
     // then just bail early with this function
-    const fn = state('overrideAssert') || assertFn
+    const fn = cy.state('overrideAssert') || assertFn
 
     return fn.apply(this, args)
   }

packages/driver/src/cy/commands/agents.coffee

@@ -50,7 +50,7 @@ onInvoke = (Cypress, obj, args) ->
     error:    obj.error
     type:     "parent"
     end:      true
-    snapshot: true
+    snapshot: !agent._noSnapshot
     event:    true
     consoleProps: ->
       consoleObj = {}
@@ -187,6 +187,12 @@ module.exports = (Commands, Cypress, cy, state, config) ->
 
       return agent
 
+    ## disable DOM snapshots during log for this agent
+    agent.snapshot = (bool = true) ->
+      agent._noSnapshot = !bool
+
+      return agent
+
     agent.as = (alias) ->
       cy.validateAlias(alias)
       cy.addAlias(ctx, {

packages/driver/src/cy/commands/commands.coffee

@@ -1,18 +1,17 @@
 _ = require("lodash")
 
-$Chainer = require("../../cypress/chainer")
-$errUtils = require("../../cypress/error_utils")
+$utils = require("../../cypress/utils")
 
-command = (ctx, name, args...) ->
-  if not ctx[name]
-    cmds = "\`#{_.keys($Chainer.prototype).join("`, `")}\`"
-    $errUtils.throwErrByPath("miscellaneous.invalid_command", {
-      args: { name, cmds }
-    })
+module.exports = (Commands, Cypress, cy, state, config) ->
+  command = (ctx, name, args...) ->
+    if not ctx[name]
+      cmds = "\`#{_.keys($Chainer.prototype).join("`, `")}\`"
+      $errUtils.throwErrByPath("miscellaneous.invalid_command", {
+        args: { name, cmds }
+      })
 
-  ctx[name].apply(ctx, args)
+    ctx[name].apply(ctx, args)
 
-module.exports = (Commands, Cypress, cy, state, config) ->
   Commands.addChainer({
     command: (chainer, args) ->
       command(chainer, args...)

packages/driver/src/cy/commands/navigation.coffee

@@ -98,7 +98,7 @@ specifyFileByRelativePath = (url, log) ->
     }
   })
 
-aboutBlank = (win) ->
+aboutBlank = (cy, win) ->
   new Promise (resolve) ->
     cy.once("window:load", resolve)
 
@@ -173,7 +173,7 @@ formSubmitted = (Cypress, e) ->
     }
   })
 
-pageLoading = (bool, state) ->
+pageLoading = (bool, Cypress, state) ->
   return if state("pageLoading") is bool
 
   state("pageLoading", bool)
@@ -187,15 +187,15 @@ stabilityChanged = (Cypress, state, config, stable, event) ->
       ## if we're currently visiting about blank
       ## and becoming unstable for the first time
       ## notifiy that we're page loading
-      pageLoading(true, state)
+      pageLoading(true, Cypress, state)
       return
     else
       ## else wait until after we finish visiting
       ## about blank
       return
 
   ## let the world know that the app is page:loading
-  pageLoading(!stable, state)
+  pageLoading(!stable, Cypress, state)
 
   ## if we aren't becoming unstable
   ## then just return now
@@ -825,7 +825,7 @@ module.exports = (Commands, Cypress, cy, state, config) ->
           hasVisitedAboutBlank = true
           currentlyVisitingAboutBlank = true
 
-          aboutBlank(win)
+          aboutBlank(cy, win)
           .then ->
             currentlyVisitingAboutBlank = false
 

packages/driver/src/cy/commands/screenshot.coffee

@@ -39,6 +39,7 @@ automateScreenshot = (state, options = {}) ->
   props = _.extend({
     titles
     testId: runnable.id
+    testAttemptIndex: $utils.getTestFromRunnable(runnable)._currentRetry
     takenPaths: state("screenshotPaths")
   }, _.omit(options, "runnable", "timeout", "log", "subject"))
 
@@ -245,6 +246,7 @@ takeScreenshot = (Cypress, state, screenshotConfig, options = {}) ->
   getOptions = (isOpen) ->
     {
       id: runnable.id
+      testAttemptIndex: $utils.getTestFromRunnable(runnable)._currentRetry
       isOpen: isOpen
       appOnly: isAppOnly(screenshotConfig)
       scale: getShouldScale(screenshotConfig)
@@ -304,6 +306,7 @@ takeScreenshot = (Cypress, state, screenshotConfig, options = {}) ->
       else
         automateScreenshot(state, automationOptions)
   .then (props) ->
+
     onAfterScreenshot and onAfterScreenshot.call(state("ctx"), $el, props)
 
     $Screenshot.onAfterScreenshot($el, props)

packages/driver/src/cypress.js

@@ -256,11 +256,6 @@ class $Cypress {
 
         break
 
-      case 'runner:set:runnable':
-        // when there is a hook / test (runnable) that
-        // is about to be invoked
-        return this.cy.setRunnable(...args)
-
       case 'runner:suite:start':
         // mocha runner started processing a suite
         if (this.config('isTextTerminal')) {
@@ -310,6 +305,8 @@ class $Cypress {
 
       case 'runner:pass':
         // mocha runner calculated a pass
+        // this is delayed from when mocha would normally fire it
+        // since we fire it after all afterEach hooks have ran
         if (this.config('isTextTerminal')) {
           return this.emit('mocha', 'pass', ...args)
         }
@@ -346,15 +343,32 @@ class $Cypress {
 
         break
       }
+      // retry event only fired in mocha version 6+
+      // https://github.com/mochajs/mocha/commit/2a76dd7589e4a1ed14dd2a33ab89f182e4c4a050
+      case 'runner:retry': {
+        // mocha runner calculated a pass
+        if (this.config('isTextTerminal')) {
+          this.emit('mocha', 'retry', ...args)
+        }
+
+        break
+      }
 
-      case 'mocha:runnable:run':
-        return this.runner.onRunnableRun(...args)
+      // case 'mocha:runnable:run':
+      //   return this.runner.onRunnableRun(...args)
 
       case 'runner:test:before:run':
         // get back to a clean slate
         this.cy.reset()
 
-        return this.emit('test:before:run', ...args)
+        if (this.config('isTextTerminal')) {
+          // needed for handling test retries
+          this.emit('mocha', 'test:before:run', args[0])
+        }
+
+        this.emit('test:before:run', ...args)
+
+        break
 
       case 'runner:test:before:run:async':
         // TODO: handle timeouts here? or in the runner?

packages/driver/src/cypress/chai_factory.js

@@ -0,0 +1,39 @@
+const _ = require('lodash')
+const chai = require('chai')
+const utils = require('chai/lib/chai/utils')
+const Assertion = require('chai/lib/chai/assertion')
+const Core = require('chai/lib/chai/core/assertions')
+const Expect = require('chai/lib/chai/interface/expect')
+const Should = require('chai/lib/chai/interface/should')
+const Assert = require('chai/lib/chai/interface/assert')
+
+const CHAI_STATIC_PROPS = ['config', 'version', 'use', 'util', 'AssertionError']
+
+const create = () => {
+  const used = []
+
+  const obj = _.pick(chai, CHAI_STATIC_PROPS)
+
+  obj.use = function (fn) {
+    if (!~used.indexOf(fn)) {
+      fn(this, obj.util)
+      used.push(fn)
+    }
+
+    return this
+  }
+
+  Assertion(obj, obj.util)
+  Core(obj, obj.util)
+  Expect(obj, obj.util)
+  Should(obj, obj.util)
+  Assert(obj, obj.util)
+
+  return obj
+}
+
+module.exports = {
+  utils,
+
+  create,
+}