✨ Add core testing mode to help test SDKs

Currently, the majority of our SDKs (JavaScript SDKs) have access to sdk-utils' test server that
mocks the core API. This is the primary method of testing these SDKs since their implementations use
sdk-utils internally anyway. For other SDKs, we implement the required utils on a per-language
basis, including how they're tested. Testing typically involves mocking HTTP modules or creating
stub APIs to simulate the CLI exec command running.

With minimal changes to our existing core API, we can add a testing mode. This new testing mode
accomplishes two main tasks. First is to silence all logs and enable dry-run mode to disable uploads
and asset discovery. Second is to enable a few extra API endpoints to allow all SDKs to be more
easily tested. New API endpoints include access to raw logs, a simple HTML document to snapshot, and
a set of commands to manipulate how the API responds to test various circumstances such as errors,
disconnects, or missing/invalid core version information.

Other changes made include a small adjustment to client's request util to allow `false` as a JSON
body, and updated percy-css tests which were failing from unrelated, accidentally unstashed, changes.

packages/cli-exec/src/common.js

@@ -7,4 +7,8 @@ export const flags = [{
   parse: Number,
   default: 5338,
   short: 'P'
+}, {
+  name: 'testing',
+  percyrc: 'testing',
+  hidden: true
 }];

packages/client/src/utils.js

@@ -119,7 +119,7 @@ export async function request(url, options = {}, callback) {
   let { proxyAgentFor } = await import('./proxy.js');
 
   // automatically stringify body content
-  if (body && typeof body !== 'string') {
+  if (body !== undefined && typeof body !== 'string') {
     headers = { 'Content-Type': 'application/json', ...headers };
     body = JSON.stringify(body);
   }

packages/core/src/api.js

@@ -11,7 +11,7 @@ export const PERCY_DOM = createRequire(import.meta.url).resolve('@percy/dom');
 export function createPercyServer(percy, port) {
   let pkg = getPackageJSON(import.meta.url);
 
-  return Server.createServer({ port })
+  let server = Server.createServer({ port })
   // facilitate logger websocket connections
     .websocket('/(logger)?', ws => {
       ws.addEventListener('message', ({ data }) => {
@@ -31,7 +31,18 @@ export function createPercyServer(percy, port) {
 
       // add version header
       res.setHeader('Access-Control-Expose-Headers', '*, X-Percy-Core-Version');
-      res.setHeader('X-Percy-Core-Version', pkg.version);
+
+      // skip or change api version header in testing mode
+      if (percy.testing?.version !== false) {
+        res.setHeader('X-Percy-Core-Version', percy.testing?.version ?? pkg.version);
+      }
+
+      // support sabotaging requests in testing mode
+      if (percy.testing?.api?.[req.url.pathname] === 'error') {
+        return res.json(500, { success: false, error: 'Error: testing' });
+      } else if (percy.testing?.api?.[req.url.pathname] === 'disconnect') {
+        return req.connection.destroy();
+      }
 
       // return json errors
       return next().catch(e => res.json(e.status ?? 500, {
@@ -83,6 +94,37 @@ export function createPercyServer(percy, port) {
       setImmediate(() => percy.stop());
       return res.json(200, { success: true });
     });
+
+  // add test endpoints only in testing mode
+  return !percy.testing ? server : server
+  // manipulates testing mode configuration to trigger specific scenarios
+    .route('/test/api/:cmd', ({ body, params: { cmd } }, res) => {
+      body = Buffer.isBuffer(body) ? body.toString() : body;
+
+      if (cmd === 'reset') {
+        // the reset command will reset testing mode to its default options
+        percy.testing = {};
+      } else if (cmd === 'version') {
+        // the version command will update the api version header for testing
+        percy.testing.version = body;
+      } else if (cmd === 'error' || cmd === 'disconnect') {
+        // the error or disconnect commands will cause specific endpoints to fail
+        percy.testing.api = { ...percy.testing.api, [body]: cmd };
+      } else {
+        // 404 for unknown commands
+        return res.send(404);
+      }
+
+      return res.json(200, { testing: percy.testing });
+    })
+  // returns an array of raw logs from the logger
+    .route('get', '/test/logs', (req, res) => res.json(200, {
+      logs: Array.from(logger.instance.messages)
+    }))
+  // serves a very basic html page for testing snapshots
+    .route('get', '/test/snapshot', (req, res) => {
+      return res.send(200, 'text/html', '<p>Snapshot Me!</p>');
+    });
 }
 
 // Create a static server instance with an automatic sitemap

packages/core/src/percy.js

@@ -44,6 +44,8 @@ export class Percy {
     skipUploads,
     // implies `skipUploads` and also skips asset discovery
     dryRun,
+    // implies `dryRun`, silent logs, and adds extra api endpoints
+    testing,
     // configuration filepath
     config,
     // provided to @percy/client
@@ -57,9 +59,11 @@ export class Percy {
     // options which will become accessible via the `.config` property
     ...options
   } = {}) {
+    if (testing) loglevel = 'silent';
     if (loglevel) this.loglevel(loglevel);
 
-    this.dryRun = !!dryRun;
+    this.testing = testing ? {} : null;
+    this.dryRun = !!testing || !!dryRun;
     this.skipUploads = this.dryRun || !!skipUploads;
     this.deferUploads = this.skipUploads || !!deferUploads;
     if (this.deferUploads) this.#uploads.stop();

packages/core/test/api.test.js

@@ -238,4 +238,79 @@ describe('API Server', () => {
       await expectAsync(percy.stop()).toBeResolved();
     });
   });
+
+  describe('when testing mode is enabled', () => {
+    const addr = 'http://localhost:5338';
+    const get = p => request(`${addr}${p}`);
+    const post = (p, body) => request(`${addr}${p}`, { method: 'post', body });
+    const req = p => request(`${addr}${p}`, { retries: 0 }, false);
+
+    beforeEach(async () => {
+      percy = await Percy.start({
+        testing: true
+      });
+    });
+
+    it('implies loglevel silent and dryRun', () => {
+      expect(percy.testing).toBeDefined();
+      expect(percy.loglevel()).toEqual('silent');
+      expect(percy.dryRun).toBeTrue();
+    });
+
+    it('enables several /test/api endpoint commands', async () => {
+      expect(percy.testing).toEqual({});
+      await post('/test/api/version', false);
+      expect(percy.testing).toHaveProperty('version', false);
+      await post('/test/api/version', '0.0.1');
+      expect(percy.testing).toHaveProperty('version', '0.0.1');
+      await post('/test/api/reset');
+      expect(percy.testing).toEqual({});
+      await post('/test/api/error', '/percy/healthcheck');
+      expect(percy.testing).toHaveProperty('api', { '/percy/healthcheck': 'error' });
+      await post('/test/api/disconnect', '/percy/healthcheck');
+      expect(percy.testing).toHaveProperty('api', { '/percy/healthcheck': 'disconnect' });
+      await expectAsync(post('/test/api/foobar')).toBeRejectedWithError('404 Not Found');
+    });
+
+    it('can manipulate the version header via /test/api/version', async () => {
+      let { headers } = await req('/percy/healthcheck');
+      expect(headers['x-percy-core-version']).toBeDefined();
+
+      await post('/test/api/version', false);
+      ({ headers } = await req('/percy/healthcheck'));
+      expect(headers['x-percy-core-version']).toBeUndefined();
+
+      await post('/test/api/version', '0.0.1');
+      ({ headers } = await req('/percy/healthcheck'));
+      expect(headers['x-percy-core-version']).toEqual('0.0.1');
+    });
+
+    it('can make endpoints return server errors via /test/api/error', async () => {
+      let { statusCode } = await req('/percy/healthcheck');
+      expect(statusCode).toEqual(200);
+
+      await post('/test/api/error', '/percy/healthcheck');
+      ({ statusCode } = await req('/percy/healthcheck'));
+      expect(statusCode).toEqual(500);
+    });
+
+    it('can make endpoints destroy connections via /test/api/disconnect', async () => {
+      await expectAsync(req('/percy/healthcheck')).toBeResolved();
+      await post('/test/api/disconnect', '/percy/healthcheck');
+      await expectAsync(req('/percy/healthcheck')).toBeRejected();
+    });
+
+    it('enables a /test/logs endpoint to return raw logs', async () => {
+      percy.log.info('foo bar from test');
+      let { logs } = await get('/test/logs');
+
+      expect(logs).toEqual(jasmine.arrayContaining([
+        jasmine.objectContaining({ message: 'foo bar from test' })
+      ]));
+    });
+
+    it('enables a /test/snapshot endpoint that serves a simple html document', async () => {
+      await expectAsync(get('/test/snapshot')).toBeResolvedTo('<p>Snapshot Me!</p>');
+    });
+  });
 });

packages/core/test/helpers/request.js

@@ -7,7 +7,7 @@ export async function request(url, method = 'GET', handle) {
   try {
     return await request(url, options, cb);
   } catch (error) {
-    if (typeof handle !== 'boolean') throw error;
+    if (!error.response || typeof handle !== 'boolean') throw error;
     return handle ? [error.response.body, error.response] : error.response;
   }
 }

packages/core/test/snapshot.test.js

@@ -851,8 +851,8 @@ describe('Snapshot', () => {
 
   describe('with percy-css', () => {
     let getResourceData = () => (
-      api.requests['/builds/123/snapshots'][0]
-        .body.data.relationships.resources.data);
+      api.requests['/builds/123/snapshots'][0].body.data.relationships.resources.data
+    ).find(r => r.attributes['resource-url'].endsWith('.css'));
 
     beforeEach(() => {
       percy.config.snapshot.percyCSS = 'p { color: purple; }';
@@ -866,14 +866,14 @@ describe('Snapshot', () => {
 
       await percy.idle();
 
-      let resources = getResourceData();
-      expect(resources[1].id).toBe(sha256hash('p { color: purple; }'));
-      expect(resources[1].attributes['resource-url'])
+      let resource = getResourceData();
+      expect(resource.id).toBe(sha256hash('p { color: purple; }'));
+      expect(resource.attributes['resource-url'])
         .toMatch(/localhost:8000\/percy-specific\.\d+\.css$/);
     });
 
     it('creates a resource for per-snapshot percy-css', async () => {
-      percy.config.snapshot.percyCSS = '';
+      percy.setConfig({ snapshot: { percyCSS: '' } });
 
       await percy.snapshot({
         name: 'test snapshot',
@@ -883,9 +883,9 @@ describe('Snapshot', () => {
 
       await percy.idle();
 
-      let resources = getResourceData();
-      expect(resources[1].id).toBe(sha256hash('body { color: purple; }'));
-      expect(resources[1].attributes['resource-url'])
+      let resource = getResourceData();
+      expect(resource.id).toBe(sha256hash('body { color: purple; }'));
+      expect(resource.attributes['resource-url'])
         .toMatch(/localhost:8000\/percy-specific\.\d+\.css$/);
     });
 
@@ -898,10 +898,10 @@ describe('Snapshot', () => {
 
       await percy.idle();
 
-      let resources = getResourceData();
-      expect(resources[1].id)
+      let resource = getResourceData();
+      expect(resource.id)
         .toBe(sha256hash('p { color: purple; }\np { font-size: 2rem; }'));
-      expect(resources[1].attributes['resource-url'])
+      expect(resource.attributes['resource-url'])
         .toMatch(/localhost:8000\/percy-specific\.\d+\.css$/);
     });
 
@@ -918,7 +918,7 @@ describe('Snapshot', () => {
       await percy.idle();
 
       let root = api.requests['/builds/123/resources'][0].body.data;
-      let cssURL = new URL(getResourceData()[1].attributes['resource-url']);
+      let cssURL = new URL(getResourceData().attributes['resource-url']);
       let injectedDOM = testDOM.replace('</body>', (
        `<link data-percy-specific-css rel="stylesheet" href="${cssURL.pathname}"/>`
       ) + '</body>');