feat: dashboard orchestration (#14925)

* test orchestration

* bump json-schemas

* fix unit

* fix server-ct build

* add @types/node

* add runtime to resolved config, send resolved config to postInstanceTests

* add missing fixture spec, support sending test config for skipped and proper title

* spec prior., refactor recording tests, update specs

* remove unneded utils, fix e2e_record_spec, rename testConfig, fix unit

* fix bug with spec SKIP not stopping test execution

* bump json-schemas

* bump json schemas 2

* refactor exit early and error logic, add runnerCapabilities + bump json schemas

* update yarn.lock

* fix referenceError

* fix: skipAction -> skipSpecAction, muteAction -> muteTestAction

* exit with non-zero for canceled runs, improve messaging for canceled runs

* fixup unit tests

* colorize run cancelation messages

packages/driver/cypress/integration/commands/angular_spec.js

@@ -28,7 +28,7 @@ describe('src/cy/commands/angular', () => {
           cy.state('window').angular = this.angular
         })
 
-        it('throws when cannot find angular', (done) => {
+        it('throws when cannot find angular', { retries: 2 }, (done) => {
           delete cy.state('window').angular
 
           cy.on('fail', (err) => {

packages/driver/src/cy/testConfigOverrides.ts

@@ -44,28 +44,30 @@ function mutateConfiguration (testConfigOverride, config, env) {
   return restoreConfigFn
 }
 
-function getResolvedTestConfigOverride (test) {
+// this is called during test onRunnable time
+// in order to resolve the test config upfront before test runs
+export function getResolvedTestConfigOverride (test) {
   let curParent = test.parent
 
-  const cfgs = [test.cfg]
+  const testConfig = [test._testConfig]
 
   while (curParent) {
-    if (curParent.cfg) {
-      cfgs.push(curParent.cfg)
+    if (curParent._testConfig) {
+      testConfig.push(curParent._testConfig)
     }
 
     curParent = curParent.parent
   }
 
-  return _.reduceRight(cfgs, (acc, cfg) => _.extend(acc, cfg), {})
+  return _.reduceRight(testConfig, (acc, opts) => _.extend(acc, opts), {})
 }
 
 class TestConfigOverride {
   private restoreTestConfigFn: Nullable<() => void> = null
   restoreAndSetTestConfigOverrides (test, config, env) {
     if (this.restoreTestConfigFn) this.restoreTestConfigFn()
 
-    const resolvedTestConfig = getResolvedTestConfigOverride(test)
+    const resolvedTestConfig = test._testConfig || {}
 
     this.restoreTestConfigFn = mutateConfiguration(resolvedTestConfig, config, env)
   }

packages/driver/src/cypress.js

@@ -134,6 +134,7 @@ class $Cypress {
     _.extend(this, browserInfo(config))
 
     this.state = $SetterGetter.create({})
+    this.originalConfig = _.cloneDeep(config)
     this.config = $SetterGetter.create(config)
     this.env = $SetterGetter.create(env)
     this.getFirefoxGcInterval = $FirefoxForcedGc.createIntervalGetter(this)

packages/driver/src/cypress/mocha.js

@@ -33,9 +33,10 @@ const suiteAfterEach = Suite.prototype.afterEach
 delete window.mocha
 delete window.Mocha
 
-function invokeFnWithOriginalTitle (ctx, originalTitle, mochaArgs, fn) {
+function invokeFnWithOriginalTitle (ctx, originalTitle, mochaArgs, fn, _testConfig) {
   const ret = fn.apply(ctx, mochaArgs)
 
+  ret._testConfig = _testConfig
   ret.originalTitle = originalTitle
 
   return ret
@@ -64,13 +65,11 @@ function overloadMochaFnForConfig (fnName, specWindow) {
       const origFn = subFn ? _fn[subFn] : _fn
 
       if (args.length > 2 && _.isObject(args[1])) {
-        const opts = _.defaults({}, args[1], {
-          browser: null,
-        })
+        const _testConfig = _.extend({}, args[1])
 
         const mochaArgs = [args[0], args[2]]
 
-        const configMatchesBrowser = opts.browser == null || Cypress.isBrowser(opts.browser, `${fnType} config value \`{ browser }\``)
+        const configMatchesBrowser = _testConfig.browser == null || Cypress.isBrowser(_testConfig.browser, `${fnType} config value \`{ browser }\``)
 
         if (!configMatchesBrowser) {
           // TODO: this would mess up the dashboard since it would be registered as a new test
@@ -84,15 +83,15 @@ function overloadMochaFnForConfig (fnName, specWindow) {
               this.skip()
             }
 
-            return invokeFnWithOriginalTitle(this, originalTitle, mochaArgs, origFn)
+            return invokeFnWithOriginalTitle(this, originalTitle, mochaArgs, origFn, _testConfig)
           }
 
-          return invokeFnWithOriginalTitle(this, originalTitle, mochaArgs, _fn['skip'])
+          return invokeFnWithOriginalTitle(this, originalTitle, mochaArgs, _fn['skip'], _testConfig)
         }
 
         const ret = origFn.apply(this, mochaArgs)
 
-        ret.cfg = opts
+        ret._testConfig = _testConfig
 
         return ret
       }
@@ -327,7 +326,7 @@ function patchTestClone () {
     const ret = testClone.apply(this, arguments)
 
     // carry over testConfigOverrides
-    ret.cfg = this.cfg
+    ret._testConfig = this._testConfig
 
     // carry over test.id
     ret.id = this.id

packages/driver/src/cypress/runner.js

@@ -9,6 +9,7 @@ const $Log = require('./log')
 const $utils = require('./utils')
 const $errUtils = require('./error_utils')
 const $stackUtils = require('./stack_utils')
+const { getResolvedTestConfigOverride } = require('../cy/testConfigOverrides')
 
 const mochaCtxKeysRe = /^(_runnable|test)$/
 const betweenQuotesRe = /\"(.+?)\"/
@@ -18,8 +19,7 @@ const TEST_BEFORE_RUN_EVENT = 'runner:test:before:run'
 const TEST_AFTER_RUN_EVENT = 'runner:test:after:run'
 
 const RUNNABLE_LOGS = 'routes agents commands hooks'.split(' ')
-const RUNNABLE_PROPS = 'id order title root hookName hookId err state failedFromHookId body speed type duration wallClockStartedAt wallClockDuration timings file originalTitle invocationDetails final currentRetry retries'.split(' ')
-
+const RUNNABLE_PROPS = '_testConfig id order title _titlePath root hookName hookId err state failedFromHookId body speed type duration wallClockStartedAt wallClockDuration timings file originalTitle invocationDetails final currentRetry retries'.split(' ')
 const debug = require('debug')('cypress:driver:runner')
 
 const fire = (event, runnable, Cypress) => {
@@ -493,6 +493,17 @@ const normalizeAll = (suite, initialTests = {}, setTestsById, setTests, onRunnab
     setTests(testsArr)
   }
 
+  // generate the diff of the config after spec has been executed
+  // e.g. config changes via Cypress.config('...')
+  normalizedSuite.runtimeConfig = {}
+  _.map(Cypress.config(), (v, key) => {
+    if (_.isEqual(v, Cypress.originalConfig[key])) {
+      return null
+    }
+
+    normalizedSuite.runtimeConfig[key] = v
+  })
+
   return normalizedSuite
 }
 
@@ -548,6 +559,17 @@ const normalize = (runnable, tests, initialTests, onRunnable, onLogsById, getRun
     // and collections
     const wrappedRunnable = wrapAll(runnable)
 
+    if (runnable.type === 'test') {
+      const cfg = getResolvedTestConfigOverride(runnable)
+
+      if (_.size(cfg)) {
+        runnable._testConfig = cfg
+        wrappedRunnable._testConfig = cfg
+      }
+
+      wrappedRunnable._titlePath = runnable.titlePath()
+    }
+
     if (prevAttempts) {
       wrappedRunnable.prevAttempts = prevAttempts
     }

packages/runner-ct/package.json

@@ -47,6 +47,7 @@
     "@babel/preset-env": "^7.12.1",
     "@packages/driver": "0.0.0-development",
     "@percy/cypress": "2.3.4",
+    "@types/node": "12.12.50",
     "@types/sockjs-client": "1.1.0",
     "babel-loader": "8.1.0",
     "clean-webpack-plugin": "^3.0.0",

packages/runner-ct/src/lib/event-manager.js

@@ -269,18 +269,20 @@ const eventManager = {
             Cypress.runner.setStartTime(state.startTime)
           }
 
+          if (config.isTextTerminal && !state.currentId) {
+            // we are in run mode and it's the first load
+            // store runnables in backend and maybe send to dashboard
+            return ws.emit('set:runnables:and:maybe:record:tests', runnables, run)
+          }
+
           if (state.currentId) {
             // if we have a currentId it means
             // we need to tell the Cypress to skip
             // ahead to that test
             Cypress.runner.resumeAtTest(state.currentId, state.emissions)
           }
 
-          if (config.isTextTerminal && !state.currentId) {
-            ws.emit('set:runnables', runnables, run)
-          } else {
-            run()
-          }
+          run()
         })
       },
     })

packages/runner/cypress/support/helpers.js

@@ -260,7 +260,7 @@ function createCypress (defaultOptions = {}) {
           url: opts.visitUrl,
         } })
 
-        .withArgs('set:runnables')
+        .withArgs('set:runnables:and:maybe:record:tests')
         .callsFake((...args) => {
           setRunnablesStub(...args)
           _.last(args)()

packages/runner/src/lib/event-manager.js

@@ -340,18 +340,20 @@ const eventManager = {
             Cypress.runner.setStartTime(state.startTime)
           }
 
+          if (config.isTextTerminal && !state.currentId) {
+            // we are in run mode and it's the first load
+            // store runnables in backend and maybe send to dashboard
+            return ws.emit('set:runnables:and:maybe:record:tests', runnables, run)
+          }
+
           if (state.currentId) {
             // if we have a currentId it means
             // we need to tell the Cypress to skip
             // ahead to that test
             Cypress.runner.resumeAtTest(state.currentId, state.emissions)
           }
 
-          if (config.isTextTerminal && !state.currentId) {
-            ws.emit('set:runnables', runnables, run)
-          } else {
-            run()
-          }
+          run()
         })
       },
     })