MWPW-135084 POC Visual Comparison Test

package.json

@@ -26,7 +26,7 @@
   },
   "homepage": "https://github.com/adobecom/college#readme",
   "devDependencies": {
-    "@amwp/platform-ui-automation": "^0.0.2",
+    "@amwp/platform-ui-automation": "^0.0.3",
     "@amwp/platform-ui-lib-adobe": "^0.0.2",
     "@babel/core": "7.17.7",
     "@babel/eslint-parser": "7.17.0",

test/e2e/README.md

@@ -0,0 +1,34 @@
+## Scenario Test
+
+E2E tests that simulate an end user's journey and ensure that UI elements exist as expected. 
+
+```
+npx run test/e2e/frictionless -t "@smoke and @gnav"
+```
+
+## Visual Test
+
+Scenario tests fall short of detecting layout and rendering issues. Visual tests fill the gaps. 
+
+Visual tests leverage the existing user steps. and verify the screenshots of selected elements. 
+
+```
+npx run test/e2e/frictionless -t "@visual and @gnav"
+```
+
+For example, a feature file `test/e2e/frictionless/features/visual/gnav.feature`
+
+```
+  Scenario: Navigate Gnav Menus
+    Given I go to the compress-pdf page
+     Then I screenshot the submenu of the 1st and 3rd menu items
+     Then I should see the same screenshots as baseline
+```
+
+The baseline screenshots are stored in the folder `test/e2e/frictionless/features/visual/gnav`. Each OS platform and browser has its own baseline. When the test is running, the current screenshots are stored in `reports/screenshot/visual/gnav`. The final step compares screenshots just taken with the baseline. The differences are stored under `reports` with the file name format `${platform}_${browser}_${image-name}.png`
+
+It can also compare the rendering of two different browers. For exmaple, the following command will run the test with the Webkit browser and compare to Firefox's baseline. 
+
+```
+npx run test/e2e/frictionless -t "@visual and @gnav" -b webkit --baseBrowser firefox
+```

test/e2e/frictionless/features/visual/gnav.feature

@@ -0,0 +1,12 @@
+Feature: Frictionless Gnav Visual Tests
+
+  Background:
+    Given I have a new browser context
+
+  @MWPW-136087 @visual @gnav
+  Scenario: Navigate Gnav Menus
+    Given I go to the compress-pdf page
+     When I resize the browser window to 1920x1080
+      And I screenshot the submenu of the 1st and 3rd menu items
+     Then I should see the same screenshots as baseline
+

test/e2e/frictionless/profiles.yml

@@ -11,4 +11,7 @@ live:
   baseUrl: https://${branch}--dc--adobecom.hlx.live
 
 preview:
-  baseUrl: https://${branch}--dc--adobecom.hlx.page
+  baseUrl: https://${branch}--dc--adobecom.hlx.page
+
+local:
+  baseUrl: http://localhost:3000

test/e2e/frictionless/step-definitions/dc.steps.js

@@ -23,8 +23,10 @@ import { CompressPdfPage } from "../page-objects/compresspdf.page";
 import { FrictionlessPage } from "../page-objects/frictionless.page";
 import { cardinal } from "../support/cardinal";
 import { expect } from "@playwright/test";
+const os = require("os");
 const path = require("path");
 const fs = require("fs");
+import { getComparator } from 'playwright-core/lib/utils';
 
 async function enableNetworkLogging(page) {
   if (global.config.profile.enableAnalytics) {
@@ -253,20 +255,63 @@ Then(/^I dismiss the extension modal$/, async function () {
   await this.page.closeExtensionModal.click();
 });
 
-Then(/^I should be able to open the submenu of the (.*) menu item(?:|s)$/, async function (items) {
+Then(/^I (screenshot|should be able to open) the submenu of the (.*) menu item(?:|s)$/, async function (action, items) {
   this.context(FrictionlessPage);
+
   let menuItems = items.replace(/ and /g, ",").split(",");
   menuItems = menuItems.map((x) => x.trim()).filter((x) => x.length > 0);
-  for (let item of menuItems) {
+  for (let item of menuItems) {  
     const index = cardinal(item);
     await this.page.openSubMenu(index);
     await expect(this.page.fedsPopup).toBeVisible();
-    await this.page.fedsPopup.screenshot({path: `fedsPopup-${item}.png`});
+    if (action === 'screenshot') {
+      const profile = global.config.profile;
+      let outputDir = this.gherkinDocument.uri.split('/features/')[1].replace('.feature', '');
+      outputDir = `${profile.reportDir}/screenshots/${outputDir}/${os.platform()}/${profile.browser}`;
+      await this.page.fedsPopup.screenshot({path: `${outputDir}/gnav-submenu-${item}.png`});
+    }
     await this.page.closeSubMenu(index);
     await expect(this.page.fedsPopup).not.toBeVisible();
   }
 });
 
+/***
+ * This step is used to compare the current screenshots with the baseline
+ * screenshots. 
+ * 
+ * Baseline Folder: features/${feature-name}/${platform}/${browser}
+ * Current Folder: ${report-dir}/screenshots/${feature-name}/${platform}/${browser} 
+ * Diff Image: ${report-dir}/${platform}_${browser}_${image-name}.png
+ * 
+ * Command line options:
+ * --baseBrowser: Use a different browser to compare with the current browser
+*/
+Then(/^I should see the same screenshots as baseline$/, async function () {
+  const comparator = getComparator('image/png');
+  let baseDir = this.gherkinDocument.uri.replace('.feature', '');
+
+  const profile = global.config.profile;
+  let outputDir = this.gherkinDocument.uri.split('/features/')[1].replace('.feature', '');
+  outputDir = `${profile.reportDir}/screenshots/${outputDir}/${os.platform()}/${profile.browser}`;
+  const images = fs.readdirSync(outputDir).filter(x => x.endsWith('.png'));
+
+  const baseBrowser = profile.baseBrowser || profile.browser;
+
+  const errors = [];
+  for (let image of images) {
+    const baseImage = fs.readFileSync(`${baseDir}/${os.platform()}/${baseBrowser}/${image}`);
+    const currImage = fs.readFileSync(`${outputDir}/${image}`);
+    let diffImage = comparator(baseImage, currImage);
+    if (diffImage) {
+      errors.push(image);
+      fs.writeFileSync(`${profile.reportDir}/${os.platform()}_${profile.browser}_${image}`, diffImage.diff);
+    }
+  }
+  if (errors.length > 0) {
+    throw `Differences found: ${errors.join(', ')}`;
+  }
+});
+
 Then(/^I should be able to use the "([^\"]*)" submenu$/, async function (menu) {
   this.context(FrictionlessPage);
   await this.page.openSubMenu(menu);
@@ -296,14 +341,3 @@ Then(/^I switch to the new page after clicking "Buy now" button in the header$/,
   this.page.native = newPage; 
 });
 
-Then(
-  /^I should not see the address bar contains "([^\"]*)"$/,
-  async function (fragment) {
-    const pattern = fragment.replace(/[|\\{}()[\]^$+*?.]/g, '\\$&');
-    await expect(this.page.native).not.toHaveURL(new RegExp(pattern), {timeout: 10000});
-  }
-);
-
-Then(/^I go back$/, async function () {
-  await this.page.native.goBack();
-});