Percy snapshot tests (#12872)

* Sample percy test added

* Node engine up to 14.x for UI prep

* Force ui test rerun

* Updated config.yml

* Node v upgraded to 14 for docker image

* Expect length in test

* Running ember tests under percy exec

* Percy exec format

* Percy cli added

* Noop to rerun tests with updated percy_token

* Evals full list and details open snapshots

* Pretty legit use of assert so disable the warning

* Jobs list tests

* Snapshots for top-level clients, servers, ACL, topology, and storage lists

* Expect caveat for Topology test

* Stabilizing tests with faker seeded to 1

* Seed-stabilizing any tests with percySnapshots

* Faker import

* Drop unused param

* Assets and test audit using an older node version

* New strategy: avoid seeding, just use percyCSS to hide certain things

.circleci/config.yml

@@ -206,7 +206,7 @@ jobs:
           path: c:\tmp\test-reports
   test-ui:
     docker:
-      - image: docker.mirror.hashicorp.services/circleci/node:12-browsers
+      - image: docker.mirror.hashicorp.services/circleci/node:14-browsers
         environment:
           # See https://git.io/vdao3 for details.
           JOBS: 2
@@ -379,6 +379,7 @@ jobs:
             ./scripts/vagrant-linux-unpriv-ui.sh
 
             export PATH="$GOPATH/bin:/usr/local/go/bin:$PATH"
+            
             source ${BASH_ENV}
             . ~/.nvm/nvm.sh
             cd ui && yarn install --frozen-lockfile && cd ..

.github/workflows/ember-assets.yml

@@ -17,7 +17,7 @@ jobs:
       - name: Use Node.js
         uses: actions/setup-node@v2
         with:
-          node-version: '12'
+          node-version: '14'
       - uses: backspace/ember-asset-size-action@edit-comment
         with:
           repo-token: "${{ secrets.GITHUB_TOKEN }}"

.github/workflows/ember-test-audit.yml

@@ -21,7 +21,7 @@ jobs:
       - name: Use Node.js
         uses: actions/setup-node@v2
         with:
-          node-version: '12'
+          node-version: '14'
       - run: yarn --frozen-lockfile
       - run: mkdir -p /tmp/test-reports
       - run: npx ember-test-audit 3 --json --output ../base-audit.json
@@ -38,7 +38,7 @@ jobs:
       - name: Use Node.js
         uses: actions/setup-node@v2
         with:
-          node-version: '12'
+          node-version: '14'
       - run: yarn --frozen-lockfile
       - run: mkdir -p /tmp/test-reports
       - run: npx ember-test-audit 3 --json --output ../pr-audit.json

scripts/vagrant-linux-unpriv-ui.sh

@@ -10,8 +10,8 @@ export NVM_DIR="${HOME}/.nvm"
 
 # Install Node, Ember CLI, and Phantom for UI development
 # Use exact full version version (e.g. not 12) for reproducibility purposes
-nvm install 12.22.10
-nvm alias default 12.22.10
+nvm install 14.19.2
+nvm alias default 14.19.2
 npm install -g ember-cli
 
 # Install Yarn for front-end dependency management

ui/.percy.yml

@@ -0,0 +1,7 @@
+version: 1
+snapshot:
+  # Hide high-variability data from Percy snapshots; helps make sure that randomized data doesn't cause a visual diff.
+  percy-css: | 
+    table td { 
+      visibility: hidden;
+    }

ui/package.json

@@ -20,7 +20,7 @@
     "build-storybook": "STORYBOOK=true ember build && build-storybook -s dist",
     "storybook": "STORYBOOK=true start-storybook -p 6006 -s dist",
     "test": "npm-run-all lint test:*",
-    "test:ember": "ember test",
+    "test:ember": "percy exec -- ember test",
     "local:qunitdom": "ember test --server --query=dockcontainer",
     "local:exam": "ember exam --server --load-balance --parallel=4"
   },
@@ -169,6 +169,8 @@
     ]
   },
   "dependencies": {
+    "@percy/cli": "^1.1.0",
+    "@percy/ember": "^3.0.0",
     "codemirror": "^5.56.0",
     "curved-arrows": "^0.1.0",
     "d3": "^7.3.0",

ui/tests/acceptance/clients-list-test.js

@@ -6,6 +6,7 @@ import { setupMirage } from 'ember-cli-mirage/test-support';
 import a11yAudit from 'nomad-ui/tests/helpers/a11y-audit';
 import pageSizeSelect from './behaviors/page-size-select';
 import ClientsList from 'nomad-ui/tests/pages/clients/list';
+import percySnapshot from '@percy/ember';
 
 module('Acceptance | clients list', function (hooks) {
   setupApplicationTest(hooks);
@@ -28,12 +29,13 @@ module('Acceptance | clients list', function (hooks) {
   test('/clients should list one page of clients', async function (assert) {
     // Make sure to make more nodes than 1 page to assert that pagination is working
     const nodesCount = ClientsList.pageSize + 1;
-
     server.createList('node', nodesCount);
     server.createList('agent', 1);
 
     await ClientsList.visit();
 
+    await percySnapshot(assert);
+
     assert.equal(ClientsList.nodes.length, ClientsList.pageSize);
     assert.ok(ClientsList.hasPagination, 'Pagination found on the page');
 
@@ -211,6 +213,8 @@ module('Acceptance | clients list', function (hooks) {
 
     await ClientsList.visit();
 
+    await percySnapshot(assert);
+
     assert.ok(ClientsList.isEmpty);
     assert.equal(ClientsList.empty.headline, 'No Clients');
   });

ui/tests/acceptance/evaluations-test.js

@@ -1,3 +1,4 @@
+/* eslint-disable qunit/require-expect */
 import {
   click,
   currentRouteName,
@@ -17,6 +18,7 @@ import {
   clickTrigger,
 } from 'ember-power-select/test-support/helpers';
 import { generateAcceptanceTestEvalMock } from '../../mirage/utils';
+import percySnapshot from '@percy/ember';
 
 const getStandardRes = () => [
   {
@@ -123,6 +125,9 @@ module('Acceptance | evaluations list', function (hooks) {
 
   test('it renders an empty message if there are no evaluations rendered', async function (assert) {
     await visit('/evaluations');
+    assert.expect(2);
+
+    await percySnapshot(assert);
 
     assert
       .dom('[data-test-empty-evaluations-list]')
@@ -134,7 +139,6 @@ module('Acceptance | evaluations list', function (hooks) {
 
   test('it renders a list of evaluations', async function (assert) {
     assert.expect(3);
-
     server.get('/evaluations', function (_server, fakeRequest) {
       assert.deepEqual(
         fakeRequest.queryParams,
@@ -151,6 +155,8 @@ module('Acceptance | evaluations list', function (hooks) {
 
     await visit('/evaluations');
 
+    await percySnapshot(assert);
+
     assert
       .dom('[data-test-eval-table]')
       .exists('Evaluations table should render');
@@ -662,6 +668,8 @@ module('Acceptance | evaluations list', function (hooks) {
       const evalId = '5fb1b8cd';
       await click(`[data-test-evaluation='${evalId}']`);
 
+      await percySnapshot(assert);
+
       assert
         .dom('[data-test-eval-detail-is-open]')
         .exists(

ui/tests/acceptance/jobs-list-test.js

@@ -7,6 +7,7 @@ import a11yAudit from 'nomad-ui/tests/helpers/a11y-audit';
 import pageSizeSelect from './behaviors/page-size-select';
 import JobsList from 'nomad-ui/tests/pages/jobs/list';
 import Layout from 'nomad-ui/tests/pages/layout';
+import percySnapshot from '@percy/ember';
 
 let managementToken, clientToken;
 
@@ -43,6 +44,8 @@ module('Acceptance | jobs list', function (hooks) {
 
     await JobsList.visit();
 
+    await percySnapshot(assert);
+
     const sortedJobs = server.db.jobs.sortBy('modifyIndex').reverse();
     assert.equal(JobsList.jobs.length, JobsList.pageSize);
     JobsList.jobs.forEach((job, index) => {
@@ -116,6 +119,8 @@ module('Acceptance | jobs list', function (hooks) {
   test('when there are no jobs, there is an empty message', async function (assert) {
     await JobsList.visit();
 
+    await percySnapshot(assert);
+
     assert.ok(JobsList.isEmpty, 'There is an empty message');
     assert.equal(
       JobsList.emptyState.headline,

ui/tests/acceptance/servers-list-test.js

@@ -7,6 +7,7 @@ import a11yAudit from 'nomad-ui/tests/helpers/a11y-audit';
 import { findLeader } from '../../mirage/config';
 import ServersList from 'nomad-ui/tests/pages/servers/list';
 import formatHost from 'nomad-ui/utils/format-host';
+import percySnapshot from '@percy/ember';
 
 const minimumSetup = () => {
   server.createList('node', 1);
@@ -41,6 +42,8 @@ module('Acceptance | servers list', function (hooks) {
 
     await ServersList.visit();
 
+    await percySnapshot(assert);
+
     assert.equal(
       ServersList.servers.length,
       ServersList.pageSize,

ui/tests/acceptance/token-test.js

@@ -1,3 +1,4 @@
+/* eslint-disable qunit/require-expect */
 import { currentURL, find, visit } from '@ember/test-helpers';
 import { module, skip, test } from 'qunit';
 import { setupApplicationTest } from 'ember-qunit';
@@ -8,6 +9,7 @@ import Jobs from 'nomad-ui/tests/pages/jobs/list';
 import JobDetail from 'nomad-ui/tests/pages/jobs/detail';
 import ClientDetail from 'nomad-ui/tests/pages/clients/detail';
 import Layout from 'nomad-ui/tests/pages/layout';
+import percySnapshot from '@percy/ember';
 
 let job;
 let node;
@@ -120,6 +122,8 @@ module('Acceptance | tokens', function (hooks) {
     await Tokens.visit();
     await Tokens.secret(secretId).submit();
 
+    await percySnapshot(assert);
+
     assert.ok(Tokens.successMessage, 'Token success message is shown');
     assert.notOk(Tokens.errorMessage, 'Token error message is not shown');
     assert.ok(Tokens.managementMessage, 'Token management message is shown');

ui/tests/acceptance/topology-test.js

@@ -1,3 +1,4 @@
+/* eslint-disable qunit/require-expect */
 import { get } from '@ember/object';
 import { currentURL } from '@ember/test-helpers';
 import { module, test } from 'qunit';
@@ -12,6 +13,7 @@ import {
   formatScheduledHertz,
 } from 'nomad-ui/utils/units';
 import queryString from 'query-string';
+import percySnapshot from '@percy/ember';
 
 const sumResources = (list, dimension) =>
   list.reduce((agg, val) => agg + (get(val, dimension) || 0), 0);
@@ -39,6 +41,9 @@ module('Acceptance | topology', function (hooks) {
     server.createList('allocation', 5);
 
     await Topology.visit();
+
+    await percySnapshot(assert);
+
     assert.equal(Topology.infoPanelTitle, 'Cluster Details');
     assert.notOk(Topology.filteredNodesWarning.isPresent);
 

ui/tests/acceptance/volumes-list-test.js

@@ -7,6 +7,7 @@ import a11yAudit from 'nomad-ui/tests/helpers/a11y-audit';
 import pageSizeSelect from './behaviors/page-size-select';
 import VolumesList from 'nomad-ui/tests/pages/storage/volumes/list';
 import Layout from 'nomad-ui/tests/pages/layout';
+import percySnapshot from '@percy/ember';
 
 const assignWriteAlloc = (volume, alloc) => {
   volume.writeAllocs.add(alloc);
@@ -54,6 +55,8 @@ module('Acceptance | volumes list', function (hooks) {
 
     await VolumesList.visit();
 
+    await percySnapshot(assert);
+
     const sortedVolumes = server.db.csiVolumes.sortBy('id');
     assert.equal(VolumesList.volumes.length, VolumesList.pageSize);
     VolumesList.volumes.forEach((volume, index) => {
@@ -126,6 +129,8 @@ module('Acceptance | volumes list', function (hooks) {
   test('when there are no volumes, there is an empty message', async function (assert) {
     await VolumesList.visit();
 
+    await percySnapshot(assert);
+
     assert.ok(VolumesList.isEmpty);
     assert.equal(VolumesList.emptyState.headline, 'No Volumes');
   });