test: collect stdout/stderr in tests (#3615)

microsoft · Aug 25, 2020 · aaff845 · aaff845
1 parent 0af3d8e
commit aaff845
Show file tree

Hide file tree

Showing 12 changed files with 221 additions and 103 deletions.
diff --git a/test-runner/src/reporter.ts b/test-runner/src/reporter.ts
@@ -21,6 +21,8 @@ export interface Reporter {
   onBegin(config: RunnerConfig, suite: Suite): void;
   onTest(test: Test): void;
   onPending(test: Test): void;
+  onStdOut(test: Test, chunk: string | Buffer);
+  onStdErr(test: Test, chunk: string | Buffer);
   onPass(test: Test): void;
   onFail(test: Test): void;
   onEnd(): void;

diff --git a/test-runner/src/reporters/base.ts b/test-runner/src/reporters/base.ts
@@ -58,6 +58,16 @@ export class BaseReporter implements Reporter  {
     this.pending.push(test);
   }
 
+  onStdOut(test: Test, chunk: string | Buffer) {
+    if (!this.config.quiet)
+      process.stdout.write(chunk);
+  }
+
+  onStdErr(test: Test, chunk: string | Buffer) {
+    if (!this.config.quiet)
+      process.stderr.write(chunk);
+  }
+
   onPass(test: Test) {
     this.passes.push(test);
   }
@@ -96,34 +106,40 @@ export class BaseReporter implements Reporter  {
 
   private _printFailures(failures: Test[]) {
     failures.forEach((failure, index) => {
-      const relativePath = path.relative(process.cwd(), failure.file);
-      const header = `  ${index +1}. ${terminalLink(relativePath, `file://${os.hostname()}${failure.file}`)} › ${failure.title}`;
-      console.log(colors.bold(colors.red(header)));
-      const stack = failure.error.stack;
-      if (stack) {
-        console.log('');
-        const messageLocation = failure.error.stack.indexOf(failure.error.message);
-        const preamble = failure.error.stack.substring(0, messageLocation + failure.error.message.length);
-        console.log(indent(preamble, '    '));
-        const position = positionInFile(stack, failure.file);
-        if (position) {
-          const source = fs.readFileSync(failure.file, 'utf8');
-          console.log('');
-          console.log(indent(codeFrameColumns(source, {
-              start: position,
-            },
-            { highlightCode: true}
-          ), '    '));
-        }
-        console.log('');
-        console.log(indent(colors.dim(stack.substring(preamble.length + 1)), '    '));
-      } else {
-        console.log('');
-        console.log(indent(String(failure.error), '    '));
-      }
-      console.log('');
+      console.log(this.formatFailure(failure, index + 1));
     });
   }
+
+  formatFailure(failure: Test, index?: number): string {
+    const tokens: string[] = [];
+    const relativePath = path.relative(process.cwd(), failure.file);
+    const header = `  ${index ? index + ')' : ''} ${terminalLink(relativePath, `file://${os.hostname()}${failure.file}`)} › ${failure.title}`;
+    tokens.push(colors.bold(colors.red(header)));
+    const stack = failure.error.stack;
+    if (stack) {
+      tokens.push('');
+      const messageLocation = failure.error.stack.indexOf(failure.error.message);
+      const preamble = failure.error.stack.substring(0, messageLocation + failure.error.message.length);
+      tokens.push(indent(preamble, '    '));
+      const position = positionInFile(stack, failure.file);
+      if (position) {
+        const source = fs.readFileSync(failure.file, 'utf8');
+        tokens.push('');
+        tokens.push(indent(codeFrameColumns(source, {
+            start: position,
+          },
+          { highlightCode: true}
+        ), '    '));
+      }
+      tokens.push('');
+      tokens.push(indent(colors.dim(stack.substring(preamble.length + 1)), '    '));
+    } else {
+      tokens.push('');
+      tokens.push(indent(String(failure.error), '    '));
+    }
+    tokens.push('');
+    return tokens.join('\n');
+  }
 }
 
 function indent(lines: string, tab: string) {

diff --git a/test-runner/src/reporters/json.ts b/test-runner/src/reporters/json.ts
@@ -54,9 +54,18 @@ class JSONReporter extends BaseReporter {
       slow: test.slow,
       duration: test.duration,
       timeout: test.timeout,
-      error: test.error
+      error: test.error,
+      stdout: test.stdout.map(s => stdioEntry(s)),
+      stderr: test.stderr.map(s => stdioEntry(s)),
+      data: test.data
     };
   }
 }
 
+function stdioEntry(s: string | Buffer): any {
+  if (typeof s === 'string')
+    return { text: s };
+  return { buffer: s.toString('base64') }
+}
+
 export default JSONReporter;
diff --git a/test-runner/src/reporters/multiplexer.ts b/test-runner/src/reporters/multiplexer.ts
@@ -40,6 +40,16 @@ export class Multiplexer implements Reporter {
       reporter.onPending(test);
   }
 
+  onStdOut(test: Test, chunk: string | Buffer) {
+    for (const reporter of this._reporters)
+      reporter.onStdOut(test, chunk);    
+  }
+
+  onStdErr(test: Test, chunk: string | Buffer) {
+    for (const reporter of this._reporters)
+      reporter.onStdErr(test, chunk);    
+  }
+
   onPass(test: Test) {
     for (const reporter of this._reporters)
       reporter.onPass(test);

diff --git a/test-runner/src/reporters/pytest.ts b/test-runner/src/reporters/pytest.ts
@@ -88,12 +88,22 @@ class PytestReporter extends BaseReporter {
     super.onPending(test);
     this._append(test, colors.yellow('∘'));
     this._progress.push('S');
+    this._throttler.schedule();
+  }
+
+  onStdOut(test: Test, chunk: string | Buffer) {
+    this._repaint(chunk);
+  }
+
+  onStdErr(test: Test, chunk: string | Buffer) {
+    this._repaint(chunk);
   }
 
   onPass(test: Test) {
     super.onPass(test);
     this._append(test, colors.green('✓'));
     this._progress.push('P');
+    this._throttler.schedule();
   }
 
   onFail(test: Test) {
@@ -103,13 +113,7 @@ class PytestReporter extends BaseReporter {
     row.failed = true;
     this._failed = true;
     this._progress.push('F');
-  }
-
-  onEnd() {
-    super.onEnd();
-    this._repaint();
-    if (this._failed)
-      this.epilogue();
+    this._repaint(this.formatFailure(test) + '\n');
   }
 
   private _append(test: Test, s: string): Row {
@@ -118,11 +122,10 @@ class PytestReporter extends BaseReporter {
     row.track.push(s);
     if (row.track.length === row.total)
       row.finishTime = Date.now();
-    this._throttler.schedule();
     return row;
   }
 
-  private _repaint() {
+  private _repaint(prependChunk?: string | Buffer) {
     const rowList = [...this._rows.values()];
     const running = rowList.filter(r => r.startTime && !r.finishTime);
     const finished = rowList.filter(r => r.finishTime).sort((a, b) => b.finishTime - a.finishTime);
@@ -160,6 +163,8 @@ class PytestReporter extends BaseReporter {
     lines.push('');
 
     process.stdout.write((cursorPrevLine + eraseLine).repeat(this._visibleRows + statusRows));
+    if (prependChunk)
+      process.stdout.write(prependChunk);
     process.stdout.write(lines.join('\n'));
   }
 

diff --git a/test-runner/src/runner.ts b/test-runner/src/runner.ts
@@ -20,7 +20,7 @@ import path from 'path';
 import { EventEmitter } from 'events';
 import { lookupRegistrations, FixturePool } from './fixtures';
 import { Suite, Test } from './test';
-import { TestRunnerEntry } from './testRunner';
+import { TestRunnerEntry, SerializedTest } from './testRunner';
 import { RunnerConfig } from './runnerConfig';
 import { Reporter } from './reporter';
 
@@ -160,13 +160,19 @@ export class Runner {
     });
     worker.on('fail', params => {
       ++this.stats.failures;
-      const out = worker.takeOut();
-      if (out.length)
-        params.test.error.stack += '\n\x1b[33mstdout: ' + out.join('\n') + '\x1b[0m';
-      const err = worker.takeErr();
-      if (err.length)
-        params.test.error.stack += '\n\x1b[33mstderr: ' + err.join('\n') + '\x1b[0m';
-        this._reporter.onFail(this._updateTest(params.test));
+      this._reporter.onFail(this._updateTest(params.test));
+    });
+    worker.on('stdout', params => {
+      const chunk = chunkFromParams(params);
+      const test = this._testById.get(params.testId);
+      test.stdout.push(chunk);
+      this._reporter.onStdOut(test, chunk);
+    });
+    worker.on('stderr', params => {
+      const chunk = chunkFromParams(params);
+      const test = this._testById.get(params.testId);
+      test.stderr.push(chunk);
+      this._reporter.onStdErr(test, chunk);
     });
     worker.on('exit', () => {
       this._workers.delete(worker);
@@ -182,10 +188,11 @@ export class Runner {
     this._createWorker();
   }
 
-  _updateTest(serialized) {
+  _updateTest(serialized: SerializedTest): Test {
     const test = this._testById.get(serialized.id);
     test.duration = serialized.duration;
     test.error = serialized.error;
+    test.data = serialized.data;
     return test;
   }
 
@@ -238,20 +245,6 @@ class OopWorker extends Worker {
       const { method, params } = message;
       this.emit(method, params);
     });
-    this.stdout = [];
-    this.stderr = [];
-    this.on('stdout', params => {
-      const chunk = chunkFromParams(params);
-      if (!runner._config.quiet)
-        process.stdout.write(chunk);
-      this.stdout.push(chunk);
-    });
-    this.on('stderr', params => {
-      const chunk = chunkFromParams(params);
-      if (!runner._config.quiet)
-        process.stderr.write(chunk);
-      this.stderr.push(chunk);
-    });
   }
 
   async init() {
@@ -267,18 +260,6 @@ class OopWorker extends Worker {
   stop() {
     this.process.send({ method: 'stop' });
   }
-
-  takeOut() {
-    const result = this.stdout;
-    this.stdout = [];
-    return result;
-  }
-
-  takeErr() {
-    const result = this.stderr;
-    this.stderr = [];
-    return result;
-  }
 }
 
 class InProcessWorker extends Worker {
@@ -298,7 +279,7 @@ class InProcessWorker extends Worker {
     delete require.cache[entry.file];
     const { TestRunner } = require('./testRunner');
     const testRunner = new TestRunner(entry, this.runner._config, 0);
-    for (const event of ['test', 'pending', 'pass', 'fail', 'done'])
+    for (const event of ['test', 'pending', 'pass', 'fail', 'done', 'stdout', 'stderr'])
       testRunner.on(event, this.emit.bind(this, event));
     testRunner.run();
   }
@@ -307,19 +288,11 @@ class InProcessWorker extends Worker {
     await this.fixturePool.teardownScope('worker');
     this.emit('exit');
   }
-
-  takeOut() {
-    return [];
-  }
-
-  takeErr() {
-    return [];
-  }
 }
 
-function chunkFromParams(params: string | { buffer: string }): string | Buffer {
-  if (typeof params === 'string')
-    return params;
+function chunkFromParams(params: { testId: string, buffer?: string, text?: string }): string | Buffer {
+  if (typeof params.text === 'string')
+    return params.text;
   return Buffer.from(params.buffer, 'base64');
 }
 

diff --git a/test-runner/src/test.ts b/test-runner/src/test.ts
@@ -27,6 +27,9 @@ export class Test {
   timeout = 0;
   fn: Function;
   error: any;
+  stdout: (string | Buffer)[] = [];
+  stderr: (string | Buffer)[] = [];
+  data: any = {};
 
   _ordinal: number;
   _overriddenFn: Function;