add a xfail test for the sa11y violation with the intention to genera…

…te a report.

nbconvert_a11y/pytest_axe.py

@@ -26,6 +26,7 @@
 OUTPUTS = ".jp-OutputArea-output"
 NO_ALT = "img:not([alt])"
 PYGMENTS = ".highlight"
+SA11Y = "sa11y-control-panel"
 
 # axe test tags
 # https://github.com/dequelabs/axe-core/blob/develop/doc/API.md#axe-core-tags

tests/test_a11y_baseline.py

@@ -8,12 +8,10 @@
 from pathlib import Path
 
 from pytest import mark, param
-from nbconvert_a11y.pytest_axe import JUPYTER_WIDGETS, MATHJAX
+from nbconvert_a11y.pytest_axe import JUPYTER_WIDGETS, MATHJAX, SA11Y
 
 from tests.test_smoke import CONFIGURATIONS, NOTEBOOKS, SKIPCI, get_target_html
 
-SA11Y = "sa11y-control-panel"
-
 TPL_NOT_ACCESSIBLE = mark.xfail(reason="template is not accessible")
 HERE = Path(__file__).parent
 EXPORTS = HERE / "exports"

tests/test_third.py

@@ -7,23 +7,51 @@
 from functools import partial
 from os import environ
 from pathlib import Path
+from time import sleep
 from unittest import TestCase
+from nbconvert import get_exporter
 
 from pytest import fixture, skip, mark
-from nbconvert_a11y.pytest_axe import JUPYTER_WIDGETS, MATHJAX, NO_ALT, PYGMENTS, AllOf, Violation
+from nbconvert_a11y.pytest_axe import (
+    JUPYTER_WIDGETS,
+    MATHJAX,
+    NO_ALT,
+    PYGMENTS,
+    SA11Y,
+    AllOf,
+    Violation,
+)
+from tests.test_color_themes import LORENZ
 from tests.test_smoke import CONFIGURATIONS, NOTEBOOKS, get_target_html
 
 # only run these tests when the CI environment variables are defined.
 environ.get("CI") or skip(allow_module_level=True)
 xfail = partial(mark.xfail, raises=AllOf, strict=True)
 
 
+@fixture
+def exporter(request):
+    e = get_exporter("html")()
+    yield e
+
+
+@fixture
+def a11y_exporter(request):
+    e = get_exporter("a11y")()
+    e.wcag_priority = "AA"
+    e.include_sa11y = True
+    yield e
+
+
 class DefaultTemplate(TestCase):
     """automated accessibility testing of the default nbconvert light theme."""
 
     # test all of the accessibility violations
     # then incrementally explain them in smaller tests.
-    @xfail(reason="there is a lot of complexity in ammending accessibility in many projects")
+    @xfail(
+        reason="there is a lot of complexity in ammending accessibility in many projects",
+        strict=True,
+    )
     def test_all(self):
         raise self.axe.run().raises_allof(
             Violation["critical-image-alt"],
@@ -33,7 +61,10 @@ def test_all(self):
             Violation["minor-focus-order-semantics"],
         )
 
-    @xfail(reason="the default pygments theme has priority AA and AAA color contrast issues.")
+    @xfail(
+        reason="the default pygments theme has priority AA and AAA color contrast issues.",
+        strict=True,
+    )
     def test_highlight_pygments(self):
         """the default template has two serious color contrast violations.
 
@@ -45,7 +76,7 @@ def test_highlight_pygments(self):
             Violation["serious-color-contrast"],
         )
 
-    @xfail(reason="widgets have not recieved a concerted effort.")
+    @xfail(reason="widgets have not recieved a concerted effort.", raises=AllOf, strict=True)
     def test_widget_display(self):
         """the simple lorenz widget generates one minor and one serious accessibility violation."""
         raise self.axe.run({"include": [JUPYTER_WIDGETS], "exclude": [NO_ALT]}).raises_allof(
@@ -58,10 +89,22 @@ def test_widget_display(self):
     # test pandas
 
     @fixture(autouse=True)
-    def lorenz(
-        self,
-        axe,
-        config=(CONFIGURATIONS / (a := "default")).with_suffix(".py"),
-        notebook=(NOTEBOOKS / (b := "lorenz-executed")).with_suffix(".ipynb"),
-    ):
-        self.axe = axe(Path.as_uri(get_target_html(config, notebook))).configure()
+    def lorenz(self, axe, tmp_path, exporter):
+        tmp = (tmp_path / LORENZ.name).with_suffix(".html")
+        tmp.write_text(exporter.from_filename(LORENZ)[0])
+        self.axe = axe(tmp.as_uri().strip()).configure()
+
+
+class A11yTemplate(TestCase):
+    @xfail(raises=AllOf, strict=True)
+    def test_sa11y(self):
+        """the simple lorenz widget generates one minor and one serious accessibility violation."""
+        raise self.axe.run({"include": [SA11Y]}).raises_allof(
+            Violation["serious-label-content-name-mismatch"]
+        )
+
+    @fixture(autouse=True)
+    def lorenz(self, axe, tmp_path, a11y_exporter):
+        tmp = (tmp_path / LORENZ.name).with_suffix(".html")
+        tmp.write_text(a11y_exporter.from_filename(LORENZ)[0])
+        self.axe = axe(tmp.as_uri().strip()).configure()