Skip to content

Commit

Permalink
Use pytest-regressions to mark expected a11y test failures (pydata#1501)
Browse files Browse the repository at this point in the history
* Use pytest-regressions with a11y tests to mark expected failures

* Update tests/test_a11y.py

* frozen data class

* Revert "frozen data class"

This reverts commit 3b0c73e.

---------

Co-authored-by: Daniel McCloy <dan@mccloy.info>
  • Loading branch information
2 people authored and ivanov committed Jun 4, 2024
1 parent dabf8e3 commit ebfb3fb
Show file tree
Hide file tree
Showing 35 changed files with 111 additions and 65 deletions.
5 changes: 3 additions & 2 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,9 @@ repos:
hooks:
- id: prettier
# Exclude the HTML, since it doesn't understand Jinja2
# exclude also the webpack.congo.js file has it embed complete url dificult to prettify
exclude: .+\.html|webpack\.config\.js
# exclude also the webpack.config.js file has it embed complete url dificult to prettify
# exclude the pytest-regressions folder tests/test_ally
exclude: .+\.html|webpack\.config\.js|tests/test_a11y/

- repo: https://github.com/psf/black
rev: 22.12.0
Expand Down
84 changes: 68 additions & 16 deletions tests/test_a11y.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,6 @@

import pytest

from .utils.pretty_axe_results import pretty_axe_results

# Using importorskip to ensure these tests are only loaded if Playwright is installed.
playwright = pytest.importorskip("playwright")
from playwright.sync_api import Page, expect # noqa: E402
Expand Down Expand Up @@ -72,34 +70,88 @@ def url_base():
process.wait()


def fingerprint_violations(accessibility_page_scan_violations):
"""Create a fingerprint of the Axe violations array.
https://playwright.dev/docs/accessibility-testing#using-snapshots-to-allow-specific-known-issues
"""
return [
{
"id": violation["id"],
"help": violation["help"],
"helpUrl": violation["helpUrl"],
"targets": [node["target"] for node in violation["nodes"]],
}
for violation in accessibility_page_scan_violations
]


@pytest.mark.a11y
@pytest.mark.parametrize("theme", ["light", "dark"])
@pytest.mark.parametrize(
"url_pathname,selector",
[
("/examples/kitchen-sink/admonitions.html", "#admonitions"),
("/examples/kitchen-sink/api.html", "#api-documentation"),
(
"/examples/kitchen-sink/admonitions.html",
"#admonitions",
),
(
"/examples/kitchen-sink/api.html",
"#api-documentation",
),
("/examples/kitchen-sink/blocks.html", "#blocks"),
("/examples/kitchen-sink/generic.html", "#generic-items"),
("/examples/kitchen-sink/images.html", "#images-figures"),
(
"/examples/kitchen-sink/generic.html",
"#generic-items",
),
(
"/examples/kitchen-sink/images.html",
"#images-figures",
),
("/examples/kitchen-sink/lists.html", "#lists"),
("/examples/kitchen-sink/structure.html", "#structural-elements"),
("/examples/kitchen-sink/structure.html", "#structural-elements-2"),
(
"/examples/kitchen-sink/structure.html",
"#structural-elements",
),
(
"/examples/kitchen-sink/structure.html",
"#structural-elements-2",
),
("/examples/kitchen-sink/tables.html", "#tables"),
("/examples/kitchen-sink/typography.html", "#typography"),
(
"/examples/kitchen-sink/typography.html",
"#typography",
),
("/examples/pydata.html", "#pydata-library-styles"),
("/user_guide/theme-elements.html", "#theme-specific-elements"),
("/user_guide/web-components.html", "#sphinx-design-components"),
("/user_guide/extending.html", "#extending-the-theme"),
("/user_guide/styling.html", "#theme-variables-and-css"),
(
"/user_guide/theme-elements.html",
"#theme-specific-elements",
),
(
"/user_guide/web-components.html",
"#sphinx-design-components",
),
(
"/user_guide/extending.html",
"#extending-the-theme",
),
(
"/user_guide/styling.html",
"#theme-variables-and-css",
),
# Using one of the simplest pages on the site, select the whole page for
# testing in order to effectively test repeated website elements like
# nav, sidebars, breadcrumbs, footer
("/user_guide/page-toc.html", ""),
],
)
def test_axe_core(
theme: str, url_base: str, url_pathname: str, selector: str, page: Page
data_regression,
theme: str,
url_base: str,
url_pathname: str,
selector: str,
page: Page,
):
"""Should have no Axe-core violations at the provided theme and page section."""
# Load the page at the provided path
Expand All @@ -118,8 +170,8 @@ def test_axe_core(
# the PyData Sphinx Theme documentation website.)
results = page.evaluate("axe.run()" if selector == "" else f"axe.run('{selector}')")

# Expect Axe-core to have found 0 accessibility violations
assert len(results["violations"]) == 0, pretty_axe_results(results, selector)
# Check found violations against known violations that we do not plan to fix
data_regression.check(fingerprint_violations(results["violations"]))


def test_version_switcher_highlighting(page: Page, url_base: str) -> None:
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[]
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[]
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[]
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[]
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[]
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[]
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[]
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[]
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[]
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[]
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[]
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[]
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[]
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[]
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[]
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[]
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[]
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[]
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[]
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[]
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
- help: Table header text should not be empty
helpUrl: https://dequeuniversity.com/rules/axe/4.6/empty-table-header?application=axeAPI
id: empty-table-header
targets:
- - thead > tr > th:nth-child(1)
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
- help: Table header text should not be empty
helpUrl: https://dequeuniversity.com/rules/axe/4.6/empty-table-header?application=axeAPI
id: empty-table-header
targets:
- - thead > tr > th:nth-child(1)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[]
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[]
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[]
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[]
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[]
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[]
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[]
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[]
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[]
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[]
47 changes: 0 additions & 47 deletions tests/utils/pretty_axe_results.py

This file was deleted.

0 comments on commit ebfb3fb

Please sign in to comment.