Use pytest-regressions to mark expected a11y test failures (#1501)

* Use pytest-regressions with a11y tests to mark expected failures

* Update tests/test_a11y.py

* frozen data class

* Revert "frozen data class"

This reverts commit 3b0c73e.

---------

Co-authored-by: Daniel McCloy <dan@mccloy.info>

.pre-commit-config.yaml

@@ -9,8 +9,9 @@ repos:
     hooks:
       - id: prettier
         # Exclude the HTML, since it doesn't understand Jinja2
-        # exclude also the webpack.congo.js file has it embed complete url dificult to prettify
-        exclude: .+\.html|webpack\.config\.js
+        # exclude also the webpack.config.js file has it embed complete url dificult to prettify
+        # exclude the pytest-regressions folder tests/test_ally
+        exclude: .+\.html|webpack\.config\.js|tests/test_a11y/
 
   - repo: https://github.com/psf/black
     rev: 22.12.0

tests/test_a11y.py

@@ -8,8 +8,6 @@
 
 import pytest
 
-from .utils.pretty_axe_results import pretty_axe_results
-
 # Using importorskip to ensure these tests are only loaded if Playwright is installed.
 playwright = pytest.importorskip("playwright")
 from playwright.sync_api import Page, expect  # noqa: E402
@@ -72,34 +70,88 @@ def url_base():
         process.wait()
 
 
+def fingerprint_violations(accessibility_page_scan_violations):
+    """Create a fingerprint of the Axe violations array.
+
+    https://playwright.dev/docs/accessibility-testing#using-snapshots-to-allow-specific-known-issues
+    """
+    return [
+        {
+            "id": violation["id"],
+            "help": violation["help"],
+            "helpUrl": violation["helpUrl"],
+            "targets": [node["target"] for node in violation["nodes"]],
+        }
+        for violation in accessibility_page_scan_violations
+    ]
+
+
 @pytest.mark.a11y
 @pytest.mark.parametrize("theme", ["light", "dark"])
 @pytest.mark.parametrize(
     "url_pathname,selector",
     [
-        ("/examples/kitchen-sink/admonitions.html", "#admonitions"),
-        ("/examples/kitchen-sink/api.html", "#api-documentation"),
+        (
+            "/examples/kitchen-sink/admonitions.html",
+            "#admonitions",
+        ),
+        (
+            "/examples/kitchen-sink/api.html",
+            "#api-documentation",
+        ),
         ("/examples/kitchen-sink/blocks.html", "#blocks"),
-        ("/examples/kitchen-sink/generic.html", "#generic-items"),
-        ("/examples/kitchen-sink/images.html", "#images-figures"),
+        (
+            "/examples/kitchen-sink/generic.html",
+            "#generic-items",
+        ),
+        (
+            "/examples/kitchen-sink/images.html",
+            "#images-figures",
+        ),
         ("/examples/kitchen-sink/lists.html", "#lists"),
-        ("/examples/kitchen-sink/structure.html", "#structural-elements"),
-        ("/examples/kitchen-sink/structure.html", "#structural-elements-2"),
+        (
+            "/examples/kitchen-sink/structure.html",
+            "#structural-elements",
+        ),
+        (
+            "/examples/kitchen-sink/structure.html",
+            "#structural-elements-2",
+        ),
         ("/examples/kitchen-sink/tables.html", "#tables"),
-        ("/examples/kitchen-sink/typography.html", "#typography"),
+        (
+            "/examples/kitchen-sink/typography.html",
+            "#typography",
+        ),
         ("/examples/pydata.html", "#pydata-library-styles"),
-        ("/user_guide/theme-elements.html", "#theme-specific-elements"),
-        ("/user_guide/web-components.html", "#sphinx-design-components"),
-        ("/user_guide/extending.html", "#extending-the-theme"),
-        ("/user_guide/styling.html", "#theme-variables-and-css"),
+        (
+            "/user_guide/theme-elements.html",
+            "#theme-specific-elements",
+        ),
+        (
+            "/user_guide/web-components.html",
+            "#sphinx-design-components",
+        ),
+        (
+            "/user_guide/extending.html",
+            "#extending-the-theme",
+        ),
+        (
+            "/user_guide/styling.html",
+            "#theme-variables-and-css",
+        ),
         # Using one of the simplest pages on the site, select the whole page for
         # testing in order to effectively test repeated website elements like
         # nav, sidebars, breadcrumbs, footer
         ("/user_guide/page-toc.html", ""),
     ],
 )
 def test_axe_core(
-    theme: str, url_base: str, url_pathname: str, selector: str, page: Page
+    data_regression,
+    theme: str,
+    url_base: str,
+    url_pathname: str,
+    selector: str,
+    page: Page,
 ):
     """Should have no Axe-core violations at the provided theme and page section."""
     # Load the page at the provided path
@@ -118,8 +170,8 @@ def test_axe_core(
     # the PyData Sphinx Theme documentation website.)
     results = page.evaluate("axe.run()" if selector == "" else f"axe.run('{selector}')")
 
-    # Expect Axe-core to have found 0 accessibility violations
-    assert len(results["violations"]) == 0, pretty_axe_results(results, selector)
+    # Check found violations against known violations that we do not plan to fix
+    data_regression.check(fingerprint_violations(results["violations"]))
 
 
 def test_version_switcher_highlighting(page: Page, url_base: str) -> None:

tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_admonitions_html__admonitions_dark_.yml

@@ -0,0 +1 @@
+[]

tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_admonitions_html__admonitions_light_.yml

@@ -0,0 +1 @@
+[]

tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_api_html__api_documentation_dark_.yml

@@ -0,0 +1 @@
+[]

tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_api_html__api_documentation_light_.yml

@@ -0,0 +1 @@
+[]

tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_blocks_html__blocks_dark_.yml

@@ -0,0 +1 @@
+[]

tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_blocks_html__blocks_light_.yml

@@ -0,0 +1 @@
+[]

tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_generic_html__generic_items_dark_.yml

@@ -0,0 +1 @@
+[]

tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_generic_html__generic_items_light_.yml

@@ -0,0 +1 @@
+[]

tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_images_html__images_figures_dark_.yml

@@ -0,0 +1 @@
+[]

tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_images_html__images_figures_light_.yml

@@ -0,0 +1 @@
+[]

tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_lists_html__lists_dark_.yml

@@ -0,0 +1 @@
+[]

tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_lists_html__lists_light_.yml

@@ -0,0 +1 @@
+[]

tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_structure_html__structural_elements_2_dark_.yml

@@ -0,0 +1 @@
+[]

tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_structure_html__structural_elements_2_light_.yml

@@ -0,0 +1 @@
+[]

tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_structure_html__structural_elements_dark_.yml

@@ -0,0 +1 @@
+[]

tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_structure_html__structural_elements_light_.yml

@@ -0,0 +1 @@
+[]

tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_tables_html__tables_dark_.yml

@@ -0,0 +1 @@
+[]

tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_tables_html__tables_light_.yml

@@ -0,0 +1 @@
+[]

tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_typography_html__typography_dark_.yml

@@ -0,0 +1 @@
+[]

tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_typography_html__typography_light_.yml

@@ -0,0 +1 @@
+[]

tests/test_a11y/test_axe_core_chromium__examples_pydata_html__pydata_library_styles_dark_.yml

@@ -0,0 +1,5 @@
+- help: Table header text should not be empty
+  helpUrl: https://dequeuniversity.com/rules/axe/4.6/empty-table-header?application=axeAPI
+  id: empty-table-header
+  targets:
+  - - thead > tr > th:nth-child(1)

tests/test_a11y/test_axe_core_chromium__examples_pydata_html__pydata_library_styles_light_.yml

@@ -0,0 +1,5 @@
+- help: Table header text should not be empty
+  helpUrl: https://dequeuniversity.com/rules/axe/4.6/empty-table-header?application=axeAPI
+  id: empty-table-header
+  targets:
+  - - thead > tr > th:nth-child(1)

tests/test_a11y/test_axe_core_chromium__user_guide_extending_html__extending_the_theme_dark_.yml

@@ -0,0 +1 @@
+[]

tests/test_a11y/test_axe_core_chromium__user_guide_extending_html__extending_the_theme_light_.yml

@@ -0,0 +1 @@
+[]

tests/test_a11y/test_axe_core_chromium__user_guide_page_toc_html__dark_.yml

@@ -0,0 +1 @@
+[]

tests/test_a11y/test_axe_core_chromium__user_guide_page_toc_html__light_.yml

@@ -0,0 +1 @@
+[]

tests/test_a11y/test_axe_core_chromium__user_guide_styling_html__theme_variables_and_css_dark_.yml

@@ -0,0 +1 @@
+[]

tests/test_a11y/test_axe_core_chromium__user_guide_styling_html__theme_variables_and_css_light_.yml

@@ -0,0 +1 @@
+[]

tests/test_a11y/test_axe_core_chromium__user_guide_theme_elements_html__theme_specific_elements_dark_.yml

@@ -0,0 +1 @@
+[]

tests/test_a11y/test_axe_core_chromium__user_guide_theme_elements_html__theme_specific_elements_light_.yml

@@ -0,0 +1 @@
+[]

tests/test_a11y/test_axe_core_chromium__user_guide_web_components_html__sphinx_design_components_dark_.yml

@@ -0,0 +1 @@
+[]

tests/test_a11y/test_axe_core_chromium__user_guide_web_components_html__sphinx_design_components_light_.yml

@@ -0,0 +1 @@
+[]