From ee99f34d33f35229609cf7af40fa9f31cbb154ac Mon Sep 17 00:00:00 2001 From: Dan Bjorge Date: Mon, 27 Jun 2022 14:19:06 -0400 Subject: [PATCH] docs(accessibility-testing): initial accessibility testing guide for js Implements #14112 --- docs/src/accessibility-testing-js.md | 292 +++++++++++++++++++++++++++ 1 file changed, 292 insertions(+) create mode 100644 docs/src/accessibility-testing-js.md diff --git a/docs/src/accessibility-testing-js.md b/docs/src/accessibility-testing-js.md new file mode 100644 index 0000000000000..e5f4f299b7e61 --- /dev/null +++ b/docs/src/accessibility-testing-js.md @@ -0,0 +1,292 @@ +--- +id: accessibility-testing +title: "Accessibility testing" +--- + +Playwright can be used to test your application for many types of accessibility issues. + +A few examples of problems this can catch include: +- Text that would be hard to read for users with vision impairments due to poor color contrast with the background behind it +- UI controls and form elements without labels that a screen reader could identify +- Interactive elements with duplicate IDs which can confuse assistive technologies + +The following examples rely on the [`@axe-core/playwright`](https://npmjs.org/@axe-core/playwright) package which adds support for running the [axe accessibility testing engine](https://www.deque.com/axe/) as part of your Playwright tests. + + + +## Disclaimer + +Automated accessibility tests can detect some common accessibility problems such as missing or invalid properties. But most accessibility problems can only be discovered through manual testing. We recommend using a combination of automated testing, manual accessibility assessments, and inclusive user testing. + +For manual assessments, we recommend [Accessibility Insights for Web](https://accessibilityinsights.io/docs/web/overview/?referrer=playwright-accessibility-testing-js), a free and open source dev tool that walks you through assessing a website for [WCAG 2.1 AA](https://www.w3.org/WAI/WCAG21/quickref/?currentsidebar=%23col_customize&levels=aa) coverage. + +## Example accessibility tests + +Accessibility tests work just like any other Playwright test. You can either create separate test cases for them, or integrate accessibility scans and assertions into your existing test cases. + +The following examples demonstrate a few basic accessibility testing scenarios. + +### Example 1: Scanning an entire page + +This example demonstrates how to test an entire page for automatically detectable accessibility violations. The test: +1. Imports the `@axe-core/playwright` package +2. Uses normal Playwright Test syntax to define a test case +3. Uses normal Playwright syntax to navigate to the page under test +4. Awaits `AxeBuilder.analyze()` to run the accessibility scan against the page +5. Uses normal Playwright Test [expect] syntax to verify that there are no violations in the returned scan results + +```js tab=js-ts +import { test, expect } from '@playwright/test'; +import AxeBuilder from '@axe-core/playwright'; // 1 + +test.describe('homepage', function() { // 2 + test('should not have any automatically detectable accessibility issues', async function({ page }) { + await page.goto('https://your-site.com/'); // 3 + + const accessibilityScanResults = await new AxeBuilder({ page }).analyze(); // 4 + + expect(accessibilityScanResults.violations).toEqual([]); // 5 + }); +}); +``` + +```js tab=js-js +const { test, expect } = require('@playwright/test'); +const AxeBuilder = require('@axe-core/playwright').default; // 1 + +test.describe('homepage', function() { // 2 + test('should not have any automatically detectable accessibility issues', async function({ page }) { + await page.goto('https://your-site.com/'); // 3 + + const accessibilityScanResults = await new AxeBuilder({ page }).analyze(); // 4 + + expect(accessibilityScanResults.violations).toEqual([]); // 5 + }); +}); +``` + +### Example 2: Configuring axe to scan a specific part of a page + +`@axe-core/playwright` supports many configuration options for axe. You can specify these options by using a Builder pattern with the `AxeBuilder` class. + +For example, you can use [`AxeBuilder.include()`](https://github.com/dequelabs/axe-core-npm/blob/develop/packages/playwright/README.md#axebuilderincludeselector-string--string) to constrain an accessibility scan to only run against one specific part of a page. + +`AxeBuilder.analyze()` will scan the page *in its current state* when you call it. To scan parts of a page that are revealed based on UI interactions, use [Locators](./locators.md) to interact with the page before invoking `analyze()`: + +```js +test('navigation menu flyout should not have automatically detectable accessibility violations', async function({ page }) { + await page.goto('https://your-site.com/'); + + await page.locator('button[aria-label="Navigation Menu"]').click(); + + await page.locator('#navigation-menu-flyout').waitFor(); + + const accessibilityScanResults = await new AxeBuilder({ page }) + .include('#navigation-menu-flyout') + .analyze(); + + expect(accessibilityScanResults.violations).toEqual([]); +}); +``` + +### Example 3: Scanning for WCAG violations + +By default, axe checks against a wide variety of accessibility rules. Some of these rules correspond to specific success criteria from the [Web Content Accessibility Guidelines (WCAG)](https://www.w3.org/TR/WCAG21/), and others are "best practice" rules that are not specifically required by any WCAG criteron. + +You can constrain an accessibility scan to only run those rules which are "tagged" as corresponding to specific WCAG success criteria by using [`AxeBuilder.withTags()`](https://github.com/dequelabs/axe-core-npm/blob/develop/packages/playwright/README.md#axebuilderwithtagstags-stringarray). For example, [Accessibility Insights for Web's Automated Checks](https://accessibilityinsights.io/docs/web/getstarted/fastpass/?referrer=playwright-accessibility-testing-js) only include axe rules that test for violations of WCAG A and AA success criteria; to match that behavior, you would use the tags `wcag2a`, `wcag2aa`, `wcag21a`, and `wcag21aa`. + +Note that [automated testing cannot detect all types of WCAG violations](#disclaimer). + +```js +test('should not have any automatically detectable WCAG A or AA violations', async function({ page }) { + await page.goto('https://your-site.com/'); + + const accessibilityScanResults = await new AxeBuilder({ page }) + .withTags(['wcag2a', 'wcag2aa', 'wcag21a', 'wcag21aa']) + .analyze(); + + expect(accessibilityScanResults.violations).toEqual([]); +}); +``` + +You can find a complete listing of the rule tags axe-core supports in [the "Axe-core Tags" section of the axe API documentation](https://www.deque.com/axe/core-documentation/api-documentation/#axe-core-tags). + +## Handling known issues + +A common question when adding accessibility tests to an application is "how do I suppress known violations?" The following examples demonstrate a few techniques you can use. + +### Excluding individual elements from a scan + +If your application contains a few specific elements with known issues, you can use [`AxeBuilder.exclude()`](https://github.com/dequelabs/axe-core-npm/blob/develop/packages/playwright/README.md#axebuilderexcludeselector-string--string) to exclude them from being scanned until you're able to fix the issues. + +This is usually the simplest option, but it has some important downsides: +* `exclude()` will exclude the specified elements *and all of their descendants*. Avoid using it with components that contain many children. +* `exclude()` will prevent *all* rules from running against the specified elements, not just the rules corresponding to known issues. + +Here is an example of excluding one element from being scanned in one specific test: + +```js +test('should not have any accessibility violations outside of elements with known issues', async function({ page }) { + await page.goto('https://your-site.com/page-with-known-issues'); + + const accessibilityScanResults = await new AxeBuilder({ page }) + .exclude('#element-with-known-issue') + .analyze(); + + expect(accessibilityScanResults.violations).toEqual([]); +}); +``` + +If the element in question is used repeatedly in many pages, consider using a [using a test fixture](#using-a-test-fixture-for-common-axe-configuration) to reuse the same `AxeBuilder` configuration across multiple tests. + +### Disabling individual scan rules + +If your application contains many different pre-existing violations of a specific rule, you can use [`AxeBuilder.disableRules()`](https://github.com/dequelabs/axe-core-npm/blob/develop/packages/playwright/README.md#axebuilderdisablerulesrules-stringarray) to temporarily disable individual rules until you're able to fix the issues. + +You can find the rule IDs to pass to `disableRules()` in the `id` property of the violations you want to suppress. A [complete list of axe's rules](https://github.com/dequelabs/axe-core/blob/master/doc/rule-descriptions.md) can be found in `axe-core`'s documentation. + +```js +test('should not have any accessibility violations outside of rules with known issues', async function({ page }) { + await page.goto('https://your-site.com/page-with-known-issues'); + + const accessibilityScanResults = await new AxeBuilder({ page }) + .disableRules(['duplicate-id']) + .analyze(); + + expect(accessibilityScanResults.violations).toEqual([]); +}); +``` + +### Using snapshots to allow specific known issues + +If you would like to allow for a more granular set of known issues, you can use [Snapshots](./test-snapshots.md) to verify that a set of pre-existing violations has not changed. This approach avoids the downsides of using `AxeBuilder.exclude()` at the cost of slightly more complexity and fragility. + +Do not use a snapshot of the entire `accessibilityScanResults.violations` array. It contains implementation details of the elements in question, such as a snippet of their rendered HTML; if you include these in your snapshots, it will make your tests prone to breaking every time one of the components in question changes for an unrelated reason: + +```js +// Don't do this! This is fragile. +expect(accessibilityScanResults.violations).toMatchSnapshot(); +``` + +Instead, create a *fingerprint* of the violation(s) in question that contains only enough information to uniquely identify the issue, and use a snapshot of the fingerprint: + +```js +// This is less fragile than snapshotting the entire violations array. +expect(violationFingerprints(accessibilityScanResults)).toMatchSnapshot(); + +// my-test-utils.js +function violationFingerprints(accessibilityScanResults) { + const violationFingerprints = accessibilityScanResults.violations.map(violation => ({ + rule: violation.id, + // These are CSS selectors which uniquely identify each element with + // a violation of the rule in question. + targets: violation.nodes.map(node => node.target), + })); + + return JSON.stringify(violationFingerprints, null, 2); +} +``` + +## Exporting a report as a test attachment + +There are many tools and libraries which support generating standalone reports for axe scan results. You can use [`testInfo.attach()`](./api/class-testinfo#test-info-attach) to attach these reports to your test results, which [reporters](./test-reporters) can embed or link as part of your test output. + +The following example uses the [`axe-sarif-converter` package](http://npmjs.com/package/axe-sarif-converter) to produce a [SARIF report](https://sarifweb.azurewebsites.net/) and attach it to the test result: + +```js +import { test, expect } from '@playwright/test'; +import { convertAxeToSarif } from 'axe-sarif-converter'; + +test('example with SARIF report attachment', async function({ page }, testInfo) { + await page.goto('https://your-site.com/'); + + const accessibilityScanResults = await new AxeBuilder({ page }).analyze(); + + const sarifReport = convertAxeToSarif(accessibilityScanResults); + await testInfo.attach('sarif-report', { + body: JSON.stringify(sarifReport), + contentType: 'application/json' + }); + + expect(accessibilityScanResults.violations).toEqual([]); +}); +``` + +## Using a test fixture for common axe configuration + +[Test fixtures](./test-fixtures) are a good way to share common `AxeBuilder` configuration across many tests. Some scenarios where this might be useful include: +* Using a common set of rules among all of your tests +* Suppressing a known violation in a common element which appears in many different pages +* Attaching standalone accessibility reports consistently for many scans + +The following example demonstrates creating and using a test fixture that covers each of those scenarios. + +### Creating a fixture + +This example fixture creates an `AxeBuilder` object which is pre-configured with shared `withTags()` and `exclude()` configuration. + +```js tab=js-ts +// axe-test.ts +import { test as base } from '@playwright/test'; +import AxeBuilder from '@axe-core/playwright'; + +type AxeFixture = { + makeAxeBuilder: () => AxeBuilder; +}; + +// Extend base test by providing "makeAxeBuilder" +// +// This new "test" can be used in multiple test files, and each of them will get +// a consistently configured AxeBuilder instance. +export const test = base.extend({ + makeAxeBuilder: async ({ page }, use, testInfo) => { + const makeAxeBuilder = () => new AxeBuilder({ page }) + .withTags(['wcag2a', 'wcag2aa', 'wcag21a', 'wcag21aa']) + .exclude('#commonly-reused-element-with-known-issue'); + + await use(makeAxeBuilder); + } +}); +export { expect } from '@playwright/test'; +``` + +```js tab=js-js +// axe-test.js +const base = require('@playwright/test'); +const AxeBuilder = require('@axe-core/playwright').default; + +// Extend base test by providing "makeAxeBuilder" +// +// This new "test" can be used in multiple test files, and each of them will get +// a consistently configured AxeBuilder instance. +exports.test = base.test.extend({ + makeAxeBuilder: async ({ page }, use, testInfo) => { + const makeAxeBuilder = () => new AxeBuilder({ page }) + .withTags(['wcag2a', 'wcag2aa', 'wcag21a', 'wcag21aa']) + .exclude('#commonly-reused-element-with-known-issue'); + + await use(makeAxeBuilder); + } +}); +exports.expect = base.expect; +``` + +### Using a fixture + +To use the fixture, replace the earlier examples' `new AxeBuilder({ page })` with the newly defined `makeAxeBuilder` fixture: + +```js +const { test, expect } = require('./axe-test'); + +test('example using custom fixture', async function({ page, makeAxeBuilder }) { + await page.goto('https://your-site.com/'); + + const accessibilityScanResults = await makeAxeBuilder() + // Automatically uses the shared AxeBuilder configuration, + // but supports additional test-specific configuration too + .include('#specific-element-under-test') + .analyze(); + + expect(accessibilityScanResults.violations).toEqual([]); +}); +```