test(avt): include playwright test report json in @carbon/react (#14538)

Co-authored-by: Andrea Cardona <Andrea.Cardona@ibm.com>
carbon-design-system · Aug 31, 2023 · 78472c6 · 78472c6
1 parent 8b9666a
commit 78472c6
Show file tree

Hide file tree

Showing 6 changed files with 83 additions and 16 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -165,7 +165,7 @@ jobs:
         if: always()
         uses: actions/upload-artifact@v3
         with:
-          name: playwright-report
+          name: playwright-vrt-report
           path: .playwright
 
   vrt:
@@ -221,5 +221,5 @@ jobs:
         if: always()
         uses: actions/upload-artifact@v3
         with:
-          name: playwright-report
+          name: playwright-avt-report
           path: .playwright
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -29,9 +29,40 @@ jobs:
       - name: Install dependencies
         run: yarn install --immutable --immutable-cache --check-cache
 
+      # For generating playwright json report for avt tests
+      - name: Install browsers
+        run: yarn playwright install --with-deps
+
       - name: Build project
         run: yarn build
 
+      # Additional steps for generating playwright json report for avt tests
+      - name: Build storybook
+        run:
+          STORYBOOK_STORE_7=false yarn workspace @carbon/react storybook:build
+      - name: Run storybook
+        id: storybook
+        run: |
+          npx serve -l 3000 packages/react/storybook-static &
+          pid=$!
+          echo ::set-output name=pid::"$pid"
+      - uses: ./actions/wait-for-it
+        with:
+          URL: 'http://localhost:3000'
+        timeout-minutes: 3
+      - name: Run AVT
+        if: github.repository == 'carbon-design-system/carbon'
+        run: |
+          yarn playwright test --project chromium --grep @avt
+      - name: Stop storybook
+        run: kill ${{ steps.storybook.outputs.pid }}
+      - name: Upload test results
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: playwright-avt-report
+          path: .playwright
+
       - name: Run Continuous Integration checks
         run: yarn ci-check
 

diff --git a/docs/testing.md b/docs/testing.md
@@ -37,6 +37,21 @@ in our elements site.
 These tests are authored within the `e2e` directory and match the file pattern:
 `*-test.e2e.js`.
 
+#### Tags
+
+Playwright tests are divided into different tag categories for reporting
+purposes. `@avt` test tags are used to populate accessibility test statuses
+within component pages on https://carbondesignsystem.com
+
+For avt tests, the test title should always include one of the following:
+
+| Tag                    | Description                                                                                                                    |
+| ---------------------- | ------------------------------------------------------------------------------------------------------------------------------ |
+| `@avt`                 | High level/root tag that should wrap all avt tests. This is usually placed in a `describe` block title.                        |
+| `@avt-default-state`   | Sub-tag of `@avt`, used to tag individual tests covering the default state of a component.                                     |
+| `@avt-advanced-states` | Sub-tag of `@avt`, used to tag individual tests covering advanced states of a component (open/close, invalid, expanded, etc.). |
+| `@avt-keyboard-nav`    | Sub-tag of `@avt`, used to tag individual tests covering keyboard navigation flows.                                            |
+
 #### Developing
 
 When working with Playwright locally, it's important to start up the service

diff --git a/e2e/components/accordion/accordion-test.avt.e2e.js b/e2e/components/accordion/accordion-test.avt.e2e.js
@@ -11,40 +11,44 @@ import { expect, test } from '@playwright/test';
 import { visitStory } from '../../test-utils/storybook';
 
 test.describe('Accordion @avt', () => {
-  test('accessibility-checker default', async ({ page }) => {
+  test('@avt-default-state', async ({ page }) => {
     await visitStory(page, {
       component: 'Accordion',
       id: 'components-accordion--default',
       globals: {
         theme: 'white',
       },
     });
-    await expect(page).toHaveNoACViolations('accordion');
+    await expect(page).toHaveNoACViolations('Accordion @avt-default-state');
   });
 
-  test('accessibility-checker skeleton', async ({ page }) => {
+  test('@avt-advanced-states skeleton', async ({ page }) => {
     await visitStory(page, {
       component: 'Accordion',
       id: 'components-accordion--skeleton',
       globals: {
         theme: 'white',
       },
     });
-    await expect(page).toHaveNoACViolations('accordion--skeleton');
+    await expect(page).toHaveNoACViolations(
+      'Accordion @avt-advanced-states skeleton'
+    );
   });
 
-  test('accessibility-checker with layer', async ({ page }) => {
+  test('@avt-advanced-states with layer', async ({ page }) => {
     await visitStory(page, {
       component: 'Accordion',
       id: 'components-accordion--with-layer',
       globals: {
         theme: 'white',
       },
     });
-    await expect(page).toHaveNoACViolations('accordion--with-layer');
+    await expect(page).toHaveNoACViolations(
+      'Accordion @avt-advanced-states with layer'
+    );
   });
 
-  test('accessibility-checker keyboard nav', async ({ page }) => {
+  test('@avt-keyboard-nav', async ({ page }) => {
     await visitStory(page, {
       component: 'Accordion',
       id: 'components-accordion--default',
@@ -78,7 +82,7 @@ test.describe('Accordion @avt', () => {
     await expect(accordin_btn2).toHaveAttribute('aria-expanded', 'false');
   });
 
-  test('accessibility-checker mouse click', async ({ page }) => {
+  test('@avt-advanced-states mouse click', async ({ page }) => {
     await visitStory(page, {
       component: 'Accordion',
       id: 'components-accordion--default',
@@ -91,12 +95,14 @@ test.describe('Accordion @avt', () => {
     await expect(accordin_btn1).toHaveAttribute('aria-expanded', 'true');
 
     // Checking for ACViolation
-    await expect(page).toHaveNoACViolations('accordion-mouse-click');
+    await expect(page).toHaveNoACViolations(
+      'Accordion @avt-advanced-states mouse click'
+    );
     await accordin_btn1.click();
     await expect(accordin_btn1).toHaveAttribute('aria-expanded', 'false');
   });
 
-  test('accessibility-checker mouse hover', async ({ page }) => {
+  test('@avt-advanced-states mouse hover', async ({ page }) => {
     await visitStory(page, {
       component: 'Accordion',
       id: 'components-accordion--default',
@@ -106,10 +112,12 @@ test.describe('Accordion @avt', () => {
     });
     const accordin_btn1 = page.getByRole('button', { name: 'Section 1 title' });
     await accordin_btn1.hover();
-    await expect(page).toHaveNoACViolations('accordion-mouse-hover');
+    await expect(page).toHaveNoACViolations(
+      'Accordion @avt-advanced-states mouse hover'
+    );
   });
 
-  test('accessibility-checker disabled accordion', async ({ page }) => {
+  test('@avt-advanced-states disabled', async ({ page }) => {
     await visitStory(page, {
       component: 'Accordion',
       id: 'components-accordion--playground',
@@ -122,6 +130,8 @@ test.describe('Accordion @avt', () => {
     });
     const accordin_btn1 = page.getByRole('button', { name: 'Section 1 title' });
     await expect(accordin_btn1).toBeDisabled();
-    await expect(page).toHaveNoACViolations('accordion-disabled');
+    await expect(page).toHaveNoACViolations(
+      'Accordion @avt-advanced-states disabled'
+    );
   });
 });
diff --git a/packages/react/package.json b/packages/react/package.json
@@ -18,7 +18,8 @@
     "index.scss",
     "icons/index.js",
     "icons/index.esm.js",
-    "icons/package.json"
+    "icons/package.json",
+    ".playwright/INTERNAL_AVT_REPORT_DO_NOT_USE.json"
   ],
   "keywords": [
     "ibm",

diff --git a/playwright.config.js b/playwright.config.js
@@ -54,6 +54,16 @@ const config = {
         outputFile: path.join(__dirname, '.playwright', 'results.json'),
       },
     ],
+    [
+      'json',
+      {
+        outputFile: path.join(
+          __dirname,
+          'packages/react/.playwright',
+          'INTERNAL_AVT_REPORT_DO_NOT_USE.json'
+        ),
+      },
+    ],
   ],
 };