fix(testrunner): pass error into test fixtures (#3605)

.gitignore

@@ -1,5 +1,6 @@
 /node_modules/
 /test-results/
+/test-runner/test/test-results/
 /test/coverage-report
 /test/test-user-data-dir*
 .local-browsers/

test-runner/src/builtin.fixtures.ts

@@ -16,10 +16,10 @@
 
 import os from 'os';
 import path from 'path';
-import {promisify} from 'util';
+import { promisify } from 'util';
 import fs from 'fs';
 import rimraf from 'rimraf';
-import {registerFixture} from './fixtures';
+import { registerFixture } from './fixtures';
 
 
 declare global {

test-runner/src/fixtures.ts

@@ -15,7 +15,7 @@
  */
 
 import debug from 'debug';
-import { Test } from './test';
+import { Test, serializeError } from './test';
 
 type Scope = 'test' | 'worker';
 
@@ -159,6 +159,9 @@ export class FixturePool<Config> {
     return async() => {
       try {
         await this.resolveParametersAndRun(callback, timeout, config, test);
+      } catch (e) {
+        test.error = serializeError(e);
+        throw e;
       } finally {
         await this.teardownScope('test');
       }

test-runner/src/index.ts

@@ -81,7 +81,7 @@ export async function runTests(config: RunnerConfig, suite: Suite, reporter: Rep
   // Trial run does not need many workers, use one.
   const jobs = (config.trialRun || config.debug) ? 1 : config.jobs;
   const runner = new Runner(suite, { ...config, jobs }, reporter);
-
+  fs.mkdirSync(config.outputDir, { recursive: true });
   try {
     for (const f of beforeFunctions)
       await f();

test-runner/src/test.ts

@@ -164,3 +164,27 @@ export function serializeConfiguration(configuration: Configuration): string {
       tokens.push(`${name}=${value}`);
   return tokens.join(', ');
 }
+
+export function serializeError(error: Error): any {
+  if (error instanceof Error) {
+    return {
+      message: error.message,
+      stack: error.stack
+    }
+  }
+  return trimCycles(error);
+}
+
+function trimCycles(obj: any): any {
+  const cache = new Set();
+  return JSON.parse(
+    JSON.stringify(obj, function(key, value) {
+      if (typeof value === 'object' && value !== null) {
+        if (cache.has(value))
+          return '' + value;
+        cache.add(value);
+      }
+      return value;
+    })
+  );
+}

test-runner/src/testRunner.ts

@@ -17,7 +17,7 @@
 import { FixturePool, rerunRegistrations, setParameters } from './fixtures';
 import { EventEmitter } from 'events';
 import { setCurrentTestFile } from './expect';
-import { Test, Suite, Configuration } from './test';
+import { Test, Suite, Configuration, serializeError } from './test';
 import { spec } from './spec';
 import { RunnerConfig } from './runnerConfig';
 
@@ -163,27 +163,3 @@ export class TestRunner extends EventEmitter {
     };
   }
 }
-
-function trimCycles(obj: any): any {
-  const cache = new Set();
-  return JSON.parse(
-    JSON.stringify(obj, function(key, value) {
-      if (typeof value === 'object' && value !== null) {
-        if (cache.has(value))
-          return '' + value;
-        cache.add(value);
-      }
-      return value;
-    })
-  );
-}
-
-function serializeError(error: Error): any {
-  if (error instanceof Error) {
-    return {
-      message: error.message,
-      stack: error.stack
-    }
-  }
-  return trimCycles(error);
-}

test-runner/test/assets/test-error-visible-in-fixture.js

@@ -0,0 +1,28 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const { registerFixture } = require('../../');
+const fs = require('fs');
+const path = require('path');
+
+registerFixture('postProcess', async ({}, runTest, config, test) => {
+	await runTest('');
+	fs.writeFileSync(path.join(config.outputDir, 'test-error-visible-in-fixture.txt'), JSON.stringify(test.error, undefined, 2));
+});
+
+it('ensure fixture handles test error', async ({ postProcess }) => {
+  expect(true).toBe(false);
+});

test-runner/test/exit-code.spec.ts

@@ -13,32 +13,52 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import "../lib"
-import { spawnSync } from "child_process";
-import path from 'path';
+
+import { spawnSync } from 'child_process';
+import * as fs from 'fs';
+import * as path from 'path';
+import rimraf from 'rimraf';
+import { promisify } from 'util';
+import '../lib';
+
+const removeFolderAsync = promisify(rimraf);
 
 it('should fail', async() => {
-  const result = runTest('one-failure.js');
+  const result = await runTest('one-failure.js');
   expect(result.exitCode).toBe(1);
   expect(result.passed).toBe(0);
   expect(result.failed).toBe(1);
 });
 
 it('should succeed', async() => {
-  const result = runTest('one-success.js');
+  const result = await runTest('one-success.js');
   expect(result.exitCode).toBe(0);
   expect(result.passed).toBe(1);
   expect(result.failed).toBe(0);
 });
 
-function runTest(filePath: string) {
-  const {output, status} = spawnSync('node', [path.join(__dirname, '..', 'cli.js'), path.join(__dirname, 'assets', filePath)]);
-  const passed = (/  (\d+) passed/.exec(output.toString()) || [])[1];
-  const failed = (/  (\d+) failed/.exec(output.toString()) || [])[1];
+it('should access error in fixture', async() => {
+  const result = await runTest('test-error-visible-in-fixture.js');
+  expect(result.exitCode).toBe(1);
+  const data = JSON.parse(fs.readFileSync(path.join(__dirname, 'test-results', 'test-error-visible-in-fixture.txt')).toString());
+  expect(data.message).toContain('Object.is equality');
+});
+
+async function runTest(filePath: string) {
+  const outputDir = path.join(__dirname, 'test-results')
+  await removeFolderAsync(outputDir).catch(e => {});
+
+  const { output, status } = spawnSync('node', [
+    path.join(__dirname, '..', 'cli.js'),
+    path.join(__dirname, 'assets', filePath),
+    '--output=' + outputDir
+  ]);
+  const passed = (/(\d+) passed/.exec(output.toString()) || [])[1];
+  const failed = (/(\d+) failed/.exec(output.toString()) || [])[1];
   return {
     exitCode: status,
     output,
     passed: parseInt(passed),
     failed: parseInt(failed || '0')
   }
-}
+}