Capture stderr in e2e tests

cli/lib/exec/spawn.js

@@ -15,15 +15,37 @@ const errors = require('../errors')
 const isXlibOrLibudevRe = /^(?:Xlib|libudev)/
 const isHighSierraWarningRe = /\*\*\* WARNING/
 const isRenderWorkerRe = /\.RenderWorker-/
+// https://github.com/electron/electron/issues/15625
+const isFontconfigWarningRe = /^Fontconfig warning:/
 
-const GARBAGE_WARNINGS = [isXlibOrLibudevRe, isHighSierraWarningRe, isRenderWorkerRe]
+const GARBAGE_WARNINGS = [
+  isXlibOrLibudevRe,
+  isHighSierraWarningRe,
+  isRenderWorkerRe,
+  isFontconfigWarningRe,
+]
 
 const isGarbageLineWarning = (str) => {
   return _.some(GARBAGE_WARNINGS, (re) => {
     return re.test(str)
   })
 }
 
+function shouldShowStderrLine (line, filterFn) {
+  // bail if this is warning line garbage
+  if (isGarbageLineWarning(line)) {
+    return false
+  }
+
+  // if we have a callback and this explictly returns
+  // false then bail
+  if (filterFn && filterFn(line) === false) {
+    return false
+  }
+
+  return true
+}
+
 function isPlatform (platform) {
   return os.platform() === platform
 }
@@ -63,6 +85,8 @@ function getStdio (needsXvfb) {
 module.exports = {
   isGarbageLineWarning,
 
+  shouldShowStderrLine,
+
   start (args, options = {}) {
     const needsXvfb = xvfb.isNeeded()
     let executable = state.getPathToExecutable(state.getBinaryDir())
@@ -186,19 +210,9 @@ module.exports = {
           child.stderr.on('data', (data) => {
             const str = data.toString()
 
-            // bail if this is warning line garbage
-            if (isGarbageLineWarning(str)) {
-              return
+            if (shouldShowStderrLine(str, onStderrData)) {
+              process.stderr.write(data)
             }
-
-            // if we have a callback and this explictly returns
-            // false then bail
-            if (onStderrData && onStderrData(str) === false) {
-              return
-            }
-
-            // else pass it along!
-            process.stderr.write(data)
           })
         }
 

packages/server/__snapshots__/8_reporters_spec.coffee.js

@@ -147,7 +147,7 @@ exports['e2e reporters mochawesome passes with mochawesome@1.5.2 npm custom repo
 ────────────────────────────────────────────────────────────────────────────────────────────────────
 
   Running:  simple_passing_spec.coffee                                                      (1 of 1)
-[mochawesome] Generating report files...
+[mochawesome] Generating report files...
 
 
 
@@ -158,7 +158,7 @@ exports['e2e reporters mochawesome passes with mochawesome@1.5.2 npm custom repo
   1 passing
 
 
-[mochawesome] Report saved to mochawesome-reports/mochawesome.html
+[mochawesome] Report saved to mochawesome-reports/mochawesome.html
 
 
 
@@ -214,7 +214,7 @@ exports['e2e reporters mochawesome fails with mochawesome@1.5.2 npm custom repor
 ────────────────────────────────────────────────────────────────────────────────────────────────────
 
   Running:  simple_failing_hook_spec.coffee                                                 (1 of 1)
-[mochawesome] Generating report files...
+[mochawesome] Generating report files...
 
 
 
@@ -263,7 +263,7 @@ Because this error occurred during a 'after all' hook we are skipping the remain
 
 
 
-[mochawesome] Report saved to mochawesome-reports/mochawesome.html
+[mochawesome] Report saved to mochawesome-reports/mochawesome.html
 
 
 
@@ -339,9 +339,9 @@ exports['e2e reporters mochawesome passes with mochawesome@2.3.1 npm custom repo
 
   1 passing
 
-[mochawesome] Report JSON saved to /foo/bar/.projects/e2e/mochawesome-report/mochawesome.json
+[mochawesome] Report JSON saved to /foo/bar/.projects/e2e/mochawesome-report/mochawesome.json
 
-[mochawesome] Report HTML saved to /foo/bar/.projects/e2e/mochawesome-report/mochawesome.html
+[mochawesome] Report HTML saved to /foo/bar/.projects/e2e/mochawesome-report/mochawesome.html
 
 
   (Results)
@@ -442,9 +442,9 @@ Because this error occurred during a 'after all' hook we are skipping the remain
 
 
 
-[mochawesome] Report JSON saved to /foo/bar/.projects/e2e/mochawesome-report/mochawesome.json
+[mochawesome] Report JSON saved to /foo/bar/.projects/e2e/mochawesome-report/mochawesome.json
 
-[mochawesome] Report HTML saved to /foo/bar/.projects/e2e/mochawesome-report/mochawesome.html
+[mochawesome] Report HTML saved to /foo/bar/.projects/e2e/mochawesome-report/mochawesome.html
 
 
   (Results)

packages/server/test/support/helpers/e2e.js

@@ -95,7 +95,7 @@ const replaceUploadingResults = function (orig, ...rest) {
   return ret
 }
 
-const normalizeStdout = function (str, options = {}) {
+const normalizeOutput = function (str, options = {}) {
   const normalizeOptions = _.defaults({}, options, { normalizeAvailableBrowsers: true })
 
   // remove all of the dynamic parts of stdout
@@ -320,7 +320,7 @@ localItFn.skip = function (title, options) {
 }
 
 module.exports = (e2e = {
-  normalizeStdout,
+  normalizeOutput,
 
   it: localItFn,
 
@@ -331,7 +331,7 @@ module.exports = (e2e = {
     const index = args.length - 1
 
     // normalize the stdout of it
-    args[index] = normalizeStdout(args[index])
+    args[index] = normalizeOutput(args[index])
 
     return snapshot.apply(null, args)
   },
@@ -435,8 +435,11 @@ module.exports = (e2e = {
       browser: 'electron',
       project: e2ePath,
       timeout: options.exit === false ? 3000000 : 120000,
+      onStdout: _.identity,
+      onStderr: _.identity,
       originalTitle: null,
       sanitizeScreenshotDimensions: false,
+      useCli: true,
     })
 
     ctx.timeout(options.timeout)
@@ -462,12 +465,22 @@ module.exports = (e2e = {
 
   args (options = {}) {
     let browser
-    const args = [
+    let args = [
+      'index.js',
       // hides a user warning to go through NPM module
       `--cwd=${process.cwd()}`,
       `--run-project=${options.project}`,
     ]
 
+    if (options.useCli) {
+      args = [
+        path.join(__dirname, '../../../../../cli/bin/cypress'),
+        'run',
+        '--dev',
+        `--project=${options.project}`,
+      ]
+    }
+
     if (options.spec) {
       args.push(`--spec=${options.spec}`)
     }
@@ -527,7 +540,9 @@ module.exports = (e2e = {
       args.push('--output-path', options.outputPath)
     }
 
-    if (options.exit != null) {
+    if (options.useCli && options.exit === false) {
+      args.push('--no-exit')
+    } else if (options.exit != null) {
       args.push('--exit', options.exit)
     }
 
@@ -564,6 +579,7 @@ module.exports = (e2e = {
 
     let stdout = ''
     let stderr = ''
+    let combinedOut = ''
 
     const exit = function (code) {
       let expected
@@ -572,25 +588,14 @@ module.exports = (e2e = {
         expect(code).to.eq(expected, 'expected exit code')
       }
 
-      // snapshot the stdout!
+      // snapshot the combined stdout and stderr!
       if (options.snapshot) {
-        // enable callback to modify stdout
-        let matches; let ostd; let str
-
-        ostd = options.onStdout
-
-        if (ostd) {
-          stdout = ostd(stdout)
-        }
-
         // if we have browser in the stdout make
         // sure its legit
-        matches = browserNameVersionRe.exec(stdout)
+        const matches = browserNameVersionRe.exec(stdout)
 
         if (matches) {
-          // eslint-disable-next-line no-unused-vars
-          const [str, key, customBrowserPath, browserName, version, headless] = matches
-
+          const [customBrowserPath, browserName, version, headless] = matches.slice(2)
           const { browser } = options
 
           if (browser && !customBrowserPath) {
@@ -608,12 +613,12 @@ module.exports = (e2e = {
           }
         }
 
-        str = normalizeStdout(stdout, options)
+        const normalizedCombinedOut = normalizeOutput(combinedOut, options)
 
         if (options.originalTitle) {
-          snapshot(options.originalTitle, str, { allowSharedSnapshot: true })
+          snapshot(options.originalTitle, normalizedCombinedOut, { allowSharedSnapshot: true })
         } else {
-          snapshot(str)
+          snapshot(normalizedCombinedOut)
         }
       }
 
@@ -652,8 +657,20 @@ module.exports = (e2e = {
       sp.stdout.pipe(process.stdout)
       sp.stderr.pipe(process.stderr)
 
-      sp.stdout.on('data', (buf) => stdout += buf.toString())
-      sp.stderr.on('data', (buf) => stderr += buf.toString())
+      sp.stdout.on('data', (buf) => {
+        const line = options.onStdout(buf.toString())
+
+        combinedOut += line
+        stdout += line
+      })
+
+      sp.stderr.on('data', (buf) => {
+        const line = options.onStderr(buf.toString())
+
+        combinedOut += `[e2e stderr]: line`
+        stderr += line
+      })
+
       sp.on('error', reject)
 
       return sp.on('exit', resolve)