From 951ce54eaaeb76a52808c48b88e26c584703c227 Mon Sep 17 00:00:00 2001 From: Wil Wilsman Date: Tue, 5 Jul 2022 12:46:52 -0500 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20Add=20core=20testing=20mode=20to=20?= =?UTF-8?q?help=20test=20SDKs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently, the majority of our SDKs (JavaScript SDKs) have access to sdk-utils' test server that mocks the core API. This is the primary method of testing these SDKs since their implementations use sdk-utils internally anyway. For other SDKs, we implement the required utils on a per-language basis, including how they're tested. Testing typically involves mocking HTTP modules or creating stub APIs to simulate the CLI exec command running. With minimal changes to our existing core API, we can add a testing mode. This new testing mode accomplishes two main tasks. First is to silence all logs and enable dry-run mode to disable uploads and asset discovery. Second is to enable a few extra API endpoints to allow all SDKs to be more easily tested. New API endpoints include access to raw logs, a simple HTML document to snapshot, and a set of commands to manipulate how the API responds to test various circumstances such as errors, disconnects, or missing/invalid core version information. Other changes made include a small adjustment to client's request util to allow `false` as a JSON body, and updated percy-css tests which were failing from unrelated, accidentally unstashed, changes. --- packages/cli-exec/src/common.js | 4 ++ packages/client/src/utils.js | 2 +- packages/core/src/api.js | 46 +++++++++++++++- packages/core/src/percy.js | 6 ++- packages/core/test/api.test.js | 75 +++++++++++++++++++++++++++ packages/core/test/helpers/request.js | 2 +- packages/core/test/snapshot.test.js | 26 +++++----- 7 files changed, 143 insertions(+), 18 deletions(-) diff --git a/packages/cli-exec/src/common.js b/packages/cli-exec/src/common.js index 4dfd81aac..7555baaae 100644 --- a/packages/cli-exec/src/common.js +++ b/packages/cli-exec/src/common.js @@ -7,4 +7,8 @@ export const flags = [{ parse: Number, default: 5338, short: 'P' +}, { + name: 'testing', + percyrc: 'testing', + hidden: true }]; diff --git a/packages/client/src/utils.js b/packages/client/src/utils.js index 92512d3b2..7cae14283 100644 --- a/packages/client/src/utils.js +++ b/packages/client/src/utils.js @@ -119,7 +119,7 @@ export async function request(url, options = {}, callback) { let { proxyAgentFor } = await import('./proxy.js'); // automatically stringify body content - if (body && typeof body !== 'string') { + if (body !== undefined && typeof body !== 'string') { headers = { 'Content-Type': 'application/json', ...headers }; body = JSON.stringify(body); } diff --git a/packages/core/src/api.js b/packages/core/src/api.js index f671bedc6..f55d272c8 100644 --- a/packages/core/src/api.js +++ b/packages/core/src/api.js @@ -11,7 +11,7 @@ export const PERCY_DOM = createRequire(import.meta.url).resolve('@percy/dom'); export function createPercyServer(percy, port) { let pkg = getPackageJSON(import.meta.url); - return Server.createServer({ port }) + let server = Server.createServer({ port }) // facilitate logger websocket connections .websocket('/(logger)?', ws => { ws.addEventListener('message', ({ data }) => { @@ -31,7 +31,18 @@ export function createPercyServer(percy, port) { // add version header res.setHeader('Access-Control-Expose-Headers', '*, X-Percy-Core-Version'); - res.setHeader('X-Percy-Core-Version', pkg.version); + + // skip or change api version header in testing mode + if (percy.testing?.version !== false) { + res.setHeader('X-Percy-Core-Version', percy.testing?.version ?? pkg.version); + } + + // support sabotaging requests in testing mode + if (percy.testing?.api?.[req.url.pathname] === 'error') { + return res.json(500, { success: false, error: 'Error: testing' }); + } else if (percy.testing?.api?.[req.url.pathname] === 'disconnect') { + return req.connection.destroy(); + } // return json errors return next().catch(e => res.json(e.status ?? 500, { @@ -83,6 +94,37 @@ export function createPercyServer(percy, port) { setImmediate(() => percy.stop()); return res.json(200, { success: true }); }); + + // add test endpoints only in testing mode + return !percy.testing ? server : server + // manipulates testing mode configuration to trigger specific scenarios + .route('/test/api/:cmd', ({ body, params: { cmd } }, res) => { + body = Buffer.isBuffer(body) ? body.toString() : body; + + if (cmd === 'reset') { + // the reset command will reset testing mode to its default options + percy.testing = {}; + } else if (cmd === 'version') { + // the version command will update the api version header for testing + percy.testing.version = body; + } else if (cmd === 'error' || cmd === 'disconnect') { + // the error or disconnect commands will cause specific endpoints to fail + percy.testing.api = { ...percy.testing.api, [body]: cmd }; + } else { + // 404 for unknown commands + return res.send(404); + } + + return res.json(200, { testing: percy.testing }); + }) + // returns an array of raw logs from the logger + .route('get', '/test/logs', (req, res) => res.json(200, { + logs: Array.from(logger.instance.messages) + })) + // serves a very basic html page for testing snapshots + .route('get', '/test/snapshot', (req, res) => { + return res.send(200, 'text/html', '

Snapshot Me!

'); + }); } // Create a static server instance with an automatic sitemap diff --git a/packages/core/src/percy.js b/packages/core/src/percy.js index 4b838e031..2e5c44021 100644 --- a/packages/core/src/percy.js +++ b/packages/core/src/percy.js @@ -44,6 +44,8 @@ export class Percy { skipUploads, // implies `skipUploads` and also skips asset discovery dryRun, + // implies `dryRun`, silent logs, and adds extra api endpoints + testing, // configuration filepath config, // provided to @percy/client @@ -57,9 +59,11 @@ export class Percy { // options which will become accessible via the `.config` property ...options } = {}) { + if (testing) loglevel = 'silent'; if (loglevel) this.loglevel(loglevel); - this.dryRun = !!dryRun; + this.testing = testing ? {} : null; + this.dryRun = !!testing || !!dryRun; this.skipUploads = this.dryRun || !!skipUploads; this.deferUploads = this.skipUploads || !!deferUploads; if (this.deferUploads) this.#uploads.stop(); diff --git a/packages/core/test/api.test.js b/packages/core/test/api.test.js index c92992f47..3e72e887a 100644 --- a/packages/core/test/api.test.js +++ b/packages/core/test/api.test.js @@ -238,4 +238,79 @@ describe('API Server', () => { await expectAsync(percy.stop()).toBeResolved(); }); }); + + describe('when testing mode is enabled', () => { + const addr = 'http://localhost:5338'; + const get = p => request(`${addr}${p}`); + const post = (p, body) => request(`${addr}${p}`, { method: 'post', body }); + const req = p => request(`${addr}${p}`, { retries: 0 }, false); + + beforeEach(async () => { + percy = await Percy.start({ + testing: true + }); + }); + + it('implies loglevel silent and dryRun', () => { + expect(percy.testing).toBeDefined(); + expect(percy.loglevel()).toEqual('silent'); + expect(percy.dryRun).toBeTrue(); + }); + + it('enables several /test/api endpoint commands', async () => { + expect(percy.testing).toEqual({}); + await post('/test/api/version', false); + expect(percy.testing).toHaveProperty('version', false); + await post('/test/api/version', '0.0.1'); + expect(percy.testing).toHaveProperty('version', '0.0.1'); + await post('/test/api/reset'); + expect(percy.testing).toEqual({}); + await post('/test/api/error', '/percy/healthcheck'); + expect(percy.testing).toHaveProperty('api', { '/percy/healthcheck': 'error' }); + await post('/test/api/disconnect', '/percy/healthcheck'); + expect(percy.testing).toHaveProperty('api', { '/percy/healthcheck': 'disconnect' }); + await expectAsync(post('/test/api/foobar')).toBeRejectedWithError('404 Not Found'); + }); + + it('can manipulate the version header via /test/api/version', async () => { + let { headers } = await req('/percy/healthcheck'); + expect(headers['x-percy-core-version']).toBeDefined(); + + await post('/test/api/version', false); + ({ headers } = await req('/percy/healthcheck')); + expect(headers['x-percy-core-version']).toBeUndefined(); + + await post('/test/api/version', '0.0.1'); + ({ headers } = await req('/percy/healthcheck')); + expect(headers['x-percy-core-version']).toEqual('0.0.1'); + }); + + it('can make endpoints return server errors via /test/api/error', async () => { + let { statusCode } = await req('/percy/healthcheck'); + expect(statusCode).toEqual(200); + + await post('/test/api/error', '/percy/healthcheck'); + ({ statusCode } = await req('/percy/healthcheck')); + expect(statusCode).toEqual(500); + }); + + it('can make endpoints destroy connections via /test/api/disconnect', async () => { + await expectAsync(req('/percy/healthcheck')).toBeResolved(); + await post('/test/api/disconnect', '/percy/healthcheck'); + await expectAsync(req('/percy/healthcheck')).toBeRejected(); + }); + + it('enables a /test/logs endpoint to return raw logs', async () => { + percy.log.info('foo bar from test'); + let { logs } = await get('/test/logs'); + + expect(logs).toEqual(jasmine.arrayContaining([ + jasmine.objectContaining({ message: 'foo bar from test' }) + ])); + }); + + it('enables a /test/snapshot endpoint that serves a simple html document', async () => { + await expectAsync(get('/test/snapshot')).toBeResolvedTo('

Snapshot Me!

'); + }); + }); }); diff --git a/packages/core/test/helpers/request.js b/packages/core/test/helpers/request.js index 64e4b3c86..3b35050dc 100644 --- a/packages/core/test/helpers/request.js +++ b/packages/core/test/helpers/request.js @@ -7,7 +7,7 @@ export async function request(url, method = 'GET', handle) { try { return await request(url, options, cb); } catch (error) { - if (typeof handle !== 'boolean') throw error; + if (!error.response || typeof handle !== 'boolean') throw error; return handle ? [error.response.body, error.response] : error.response; } } diff --git a/packages/core/test/snapshot.test.js b/packages/core/test/snapshot.test.js index 057862a58..6f61096ba 100644 --- a/packages/core/test/snapshot.test.js +++ b/packages/core/test/snapshot.test.js @@ -851,8 +851,8 @@ describe('Snapshot', () => { describe('with percy-css', () => { let getResourceData = () => ( - api.requests['/builds/123/snapshots'][0] - .body.data.relationships.resources.data); + api.requests['/builds/123/snapshots'][0].body.data.relationships.resources.data + ).find(r => r.attributes['resource-url'].endsWith('.css')); beforeEach(() => { percy.config.snapshot.percyCSS = 'p { color: purple; }'; @@ -866,14 +866,14 @@ describe('Snapshot', () => { await percy.idle(); - let resources = getResourceData(); - expect(resources[1].id).toBe(sha256hash('p { color: purple; }')); - expect(resources[1].attributes['resource-url']) + let resource = getResourceData(); + expect(resource.id).toBe(sha256hash('p { color: purple; }')); + expect(resource.attributes['resource-url']) .toMatch(/localhost:8000\/percy-specific\.\d+\.css$/); }); it('creates a resource for per-snapshot percy-css', async () => { - percy.config.snapshot.percyCSS = ''; + percy.setConfig({ snapshot: { percyCSS: '' } }); await percy.snapshot({ name: 'test snapshot', @@ -883,9 +883,9 @@ describe('Snapshot', () => { await percy.idle(); - let resources = getResourceData(); - expect(resources[1].id).toBe(sha256hash('body { color: purple; }')); - expect(resources[1].attributes['resource-url']) + let resource = getResourceData(); + expect(resource.id).toBe(sha256hash('body { color: purple; }')); + expect(resource.attributes['resource-url']) .toMatch(/localhost:8000\/percy-specific\.\d+\.css$/); }); @@ -898,10 +898,10 @@ describe('Snapshot', () => { await percy.idle(); - let resources = getResourceData(); - expect(resources[1].id) + let resource = getResourceData(); + expect(resource.id) .toBe(sha256hash('p { color: purple; }\np { font-size: 2rem; }')); - expect(resources[1].attributes['resource-url']) + expect(resource.attributes['resource-url']) .toMatch(/localhost:8000\/percy-specific\.\d+\.css$/); }); @@ -918,7 +918,7 @@ describe('Snapshot', () => { await percy.idle(); let root = api.requests['/builds/123/resources'][0].body.data; - let cssURL = new URL(getResourceData()[1].attributes['resource-url']); + let cssURL = new URL(getResourceData().attributes['resource-url']); let injectedDOM = testDOM.replace('', ( `` ) + '');