feat(testrunner): better matchers (#1077)

This patch re-implements matching and reporting for test runner.
Among other improvements:
- test failures now show a short snippet from test
- test failures now explicitly say what received and what was expected
- `expect.toBe()` now does text diff when gets strings as input
- `expect.toEqual` now does object diff

.github/workflows/tests.yml

@@ -10,6 +10,8 @@ on:
 
 env:
   CI: true
+  # Force terminal colors. @see https://www.npmjs.com/package/colors
+  FORCE_COLOR: 1
 
 jobs:
   chromium_linux:

package.json

@@ -62,6 +62,7 @@
     "@types/ws": "^6.0.1",
     "@typescript-eslint/eslint-plugin": "^2.6.1",
     "@typescript-eslint/parser": "^2.6.1",
+    "colors": "^1.4.0",
     "commonmark": "^0.28.1",
     "cross-env": "^5.0.5",
     "eslint": "^6.6.0",

test/golden-utils.js

@@ -20,6 +20,7 @@ const Diff = require('text-diff');
 const PNG = require('pngjs').PNG;
 const jpeg = require('jpeg-js');
 const pixelmatch = require('pixelmatch');
+const c = require('colors/safe');
 
 module.exports = {compare};
 
@@ -51,7 +52,7 @@ function compareImages(actualBuffer, expectedBuffer, mimeType) {
   const expected = mimeType === 'image/png' ? PNG.sync.read(expectedBuffer) : jpeg.decode(expectedBuffer);
   if (expected.width !== actual.width || expected.height !== actual.height) {
     return {
-      errorMessage: `Sizes differ: expected image ${expected.width}px X ${expected.height}px, but got ${actual.width}px X ${actual.height}px. `
+      errorMessage: `Sizes differ; expected image ${expected.width}px X ${expected.height}px, but got ${actual.width}px X ${actual.height}px. `
     };
   }
   const diff = new PNG({width: expected.width, height: expected.height});
@@ -110,31 +111,43 @@ function compare(goldenPath, outputPath, actual, goldenName) {
   if (!comparator) {
     return {
       pass: false,
-      message: 'Failed to find comparator with type ' + mimeType + ': '  + goldenName
+      message: 'Failed to find comparator with type ' + mimeType + ': '  + goldenName,
     };
   }
   const result = comparator(actual, expected, mimeType);
   if (!result)
     return { pass: true };
   ensureOutputDir();
+  const output = [
+    c.red(`GOLDEN FAILED: `) + c.yellow('"' + goldenName + '"'),
+  ];
+  if (result.errorMessage)
+    output.push('    ' + result.errorMessage);
+  output.push('');
+  output.push(`Expected: ${c.yellow(expectedPath)}`);
   if (goldenPath === outputPath) {
-    fs.writeFileSync(addSuffix(actualPath, '-actual'), actual);
+    const filepath = addSuffix(actualPath, '-actual');
+    fs.writeFileSync(filepath, actual);
+    output.push(`Received: ${c.yellow(filepath)}`);
   } else {
     fs.writeFileSync(actualPath, actual);
     // Copy expected to the output/ folder for convenience.
     fs.writeFileSync(addSuffix(actualPath, '-expected'), expected);
+    output.push(`Received: ${c.yellow(actualPath)}`);
   }
   if (result.diff) {
     const diffPath = addSuffix(actualPath, '-diff', result.diffExtension);
     fs.writeFileSync(diffPath, result.diff);
+    output.push(`    Diff: ${c.yellow(diffPath)}`);
   }
 
   let message = goldenName + ' mismatch!';
   if (result.errorMessage)
     message += ' ' + result.errorMessage;
   return {
     pass: false,
-    message: message + ' ' + messageSuffix
+    message: message + ' ' + messageSuffix,
+    formatter: () => output.join('\n'),
   };
 
   function ensureOutputDir() {

test/test.js

@@ -124,7 +124,6 @@ if (process.env.CI && testRunner.hasFocusedTestsOrSuites()) {
 new Reporter(testRunner, {
   verbose: process.argv.includes('--verbose'),
   summary: !process.argv.includes('--verbose'),
-  projectFolder: utils.projectRoot(),
   showSlowTests: process.env.CI ? 5 : 0,
   showSkippedTests: 10,
 });

utils/testrunner/Matchers.js

@@ -14,7 +14,11 @@
  * limitations under the License.
  */
 
-module.exports = class Matchers {
+const {getCallerLocation} = require('./utils.js');
+const colors = require('colors/safe');
+const Diff = require('text-diff');
+
+class Matchers {
   constructor(customMatchers = {}) {
     this._matchers = {};
     Object.assign(this._matchers, DefaultMatchers);
@@ -26,99 +30,202 @@ module.exports = class Matchers {
     this._matchers[name] = matcher;
   }
 
-  expect(value) {
-    return new Expect(value, this._matchers);
+  expect(received) {
+    return new Expect(received, this._matchers);
   }
 };
 
+class MatchError extends Error {
+  constructor(message, formatter) {
+    super(message);
+    this.name = this.constructor.name;
+    this.formatter = formatter;
+    this.location = getCallerLocation(__filename);
+    Error.captureStackTrace(this, this.constructor);
+  }
+}
+
+module.exports = {Matchers, MatchError};
+
 class Expect {
-  constructor(value, matchers) {
+  constructor(received, matchers) {
     this.not = {};
     this.not.not = this;
     for (const matcherName of Object.keys(matchers)) {
       const matcher = matchers[matcherName];
-      this[matcherName] = applyMatcher.bind(null, matcherName, matcher, false /* inverse */, value);
-      this.not[matcherName] = applyMatcher.bind(null, matcherName, matcher, true /* inverse */, value);
+      this[matcherName] = applyMatcher.bind(null, matcherName, matcher, false /* inverse */, received);
+      this.not[matcherName] = applyMatcher.bind(null, matcherName, matcher, true /* inverse */, received);
     }
 
-    function applyMatcher(matcherName, matcher, inverse, value, ...args) {
-      const result = matcher.call(null, value, ...args);
+    function applyMatcher(matcherName, matcher, inverse, received, ...args) {
+      const result = matcher.call(null, received, ...args);
       const message = `expect.${inverse ? 'not.' : ''}${matcherName} failed` + (result.message ? `: ${result.message}` : '');
       if (result.pass === inverse)
-        throw new Error(message);
+        throw new MatchError(message, result.formatter || defaultFormatter.bind(null, received));
+    }
+  }
+}
+
+function defaultFormatter(received) {
+  return `Received: ${colors.red(JSON.stringify(received))}`;
+}
+
+function stringFormatter(received, expected) {
+  const diff = new Diff();
+  const result = diff.main(expected, received);
+  diff.cleanupSemantic(result);
+  const highlighted = result.map(([type, text]) => {
+    if (type === -1)
+      return colors.bgRed(text);
+    if (type === 1)
+      return colors.bgGreen.black(text);
+    return text;
+  }).join('');
+  const output = [
+    `Expected: ${expected}`,
+    `Received: ${highlighted}`,
+  ];
+  for (let i = 0; i < Math.min(expected.length, received.length); ++i) {
+    if (expected[i] !== received[i]) {
+      const padding = ' '.repeat('Expected: '.length);
+      const firstDiffCharacter = '~'.repeat(i) + '^';
+      output.push(colors.red(padding + firstDiffCharacter));
+      break;
+    }
+  }
+  return output.join('\n');
+}
+
+function objectFormatter(received, expected) {
+  const receivedLines = received.split('\n');
+  const expectedLines = expected.split('\n');
+  const encodingMap = new Map();
+  const decodingMap = new Map();
+
+  const doEncodeLines = (lines) => {
+    let encoded = '';
+    for (const line of lines) {
+      let code = encodingMap.get(line);
+      if (!code) {
+        code = String.fromCodePoint(encodingMap.size);
+        encodingMap.set(line, code);
+        decodingMap.set(code, line);
+      }
+      encoded += code;
     }
+    return encoded;
+  };
+
+  const doDecodeLines = (text) => {
+    let decoded = [];
+    for (const codepoint of [...text])
+      decoded.push(decodingMap.get(codepoint));
+    return decoded;
   }
+
+  let receivedEncoded = doEncodeLines(received.split('\n'));
+  let expectedEncoded = doEncodeLines(expected.split('\n'));
+
+  const diff = new Diff();
+  const result = diff.main(expectedEncoded, receivedEncoded);
+  diff.cleanupSemantic(result);
+
+  const highlighted = result.map(([type, text]) => {
+    const lines = doDecodeLines(text);
+    if (type === -1)
+      return lines.map(line => '-   ' + colors.bgRed(line));
+    if (type === 1)
+      return lines.map(line => '+   ' + colors.bgGreen.black(line));
+    return lines.map(line => '    ' + line);
+  }).flat().join('\n');
+  return `Received:\n${highlighted}`;
+}
+
+function toBeFormatter(received, expected) {
+  if (typeof expected === 'string' && typeof received === 'string') {
+    return stringFormatter(JSON.stringify(received), JSON.stringify(expected));
+  }
+  return [
+    `Expected: ${JSON.stringify(expected)}`,
+    `Received: ${colors.red(JSON.stringify(received))}`,
+  ].join('\n');
 }
 
 const DefaultMatchers = {
-  toBe: function(value, other, message) {
-    message = message || `${value} == ${other}`;
-    return { pass: value === other, message };
+  toBe: function(received, expected, message) {
+    message = message || `${received} == ${expected}`;
+    return { pass: received === expected, message, formatter: toBeFormatter.bind(null, received, expected) };
   },
 
-  toBeFalsy: function(value, message) {
-    message = message || `${value}`;
-    return { pass: !value, message };
+  toBeFalsy: function(received, message) {
+    message = message || `${received}`;
+    return { pass: !received, message };
   },
 
-  toBeTruthy: function(value, message) {
-    message = message || `${value}`;
-    return { pass: !!value, message };
+  toBeTruthy: function(received, message) {
+    message = message || `${received}`;
+    return { pass: !!received, message };
   },
 
-  toBeGreaterThan: function(value, other, message) {
-    message = message || `${value} > ${other}`;
-    return { pass: value > other, message };
+  toBeGreaterThan: function(received, other, message) {
+    message = message || `${received} > ${other}`;
+    return { pass: received > other, message };
   },
 
-  toBeGreaterThanOrEqual: function(value, other, message) {
-    message = message || `${value} >= ${other}`;
-    return { pass: value >= other, message };
+  toBeGreaterThanOrEqual: function(received, other, message) {
+    message = message || `${received} >= ${other}`;
+    return { pass: received >= other, message };
   },
 
-  toBeLessThan: function(value, other, message) {
-    message = message || `${value} < ${other}`;
-    return { pass: value < other, message };
+  toBeLessThan: function(received, other, message) {
+    message = message || `${received} < ${other}`;
+    return { pass: received < other, message };
   },
 
-  toBeLessThanOrEqual: function(value, other, message) {
-    message = message || `${value} <= ${other}`;
-    return { pass: value <= other, message };
+  toBeLessThanOrEqual: function(received, other, message) {
+    message = message || `${received} <= ${other}`;
+    return { pass: received <= other, message };
   },
 
-  toBeNull: function(value, message) {
-    message = message || `${value} == null`;
-    return { pass: value === null, message };
+  toBeNull: function(received, message) {
+    message = message || `${received} == null`;
+    return { pass: received === null, message };
   },
 
-  toContain: function(value, other, message) {
-    message = message || `${value} ⊇ ${other}`;
-    return { pass: value.includes(other), message };
+  toContain: function(received, other, message) {
+    message = message || `${received} ⊇ ${other}`;
+    return { pass: received.includes(other), message };
   },
 
-  toEqual: function(value, other, message) {
-    const valueJson = stringify(value);
-    const otherJson = stringify(other);
-    message = message || `\n${valueJson} ≈ ${otherJson}`;
-    return { pass: valueJson === otherJson, message };
+  toEqual: function(received, other, message) {
+    let receivedJson = stringify(received);
+    let otherJson = stringify(other);
+    let formatter = objectFormatter.bind(null, receivedJson, otherJson);
+    if (receivedJson.length < 40 && otherJson.length < 40) {
+      receivedJson = receivedJson.split('\n').map(line => line.trim()).join(' ');
+      otherJson = otherJson.split('\n').map(line => line.trim()).join(' ');
+      formatter = stringFormatter.bind(null, receivedJson, otherJson);
+    }
+    message = message || `\n${receivedJson} ≈ ${otherJson}`;
+    return { pass: receivedJson === otherJson, message, formatter };
   },
 
-  toBeCloseTo: function(value, other, precision, message) {
+  toBeCloseTo: function(received, other, precision, message) {
     return {
-      pass: Math.abs(value - other) < Math.pow(10, -precision),
+      pass: Math.abs(received - other) < Math.pow(10, -precision),
       message
     };
   },
 
-  toBeInstanceOf: function(value, other, message) {
-    message = message || `${value.constructor.name} instanceof ${other.name}`;
-    return { pass: value instanceof other, message };
+  toBeInstanceOf: function(received, other, message) {
+    message = message || `${received.constructor.name} instanceof ${other.name}`;
+    return { pass: received instanceof other, message };
   },
 };
 
 function stringify(value) {
   function stabilize(key, object) {
-    if (typeof object !== 'object' || object === undefined || object === null)
+    if (typeof object !== 'object' || object === undefined || object === null || Array.isArray(object))
       return object;
     const result = {};
     for (const key of Object.keys(object).sort())