Axe

.gitignore

@@ -118,3 +118,10 @@ logs/
 
 # reports
 reports/
+
+# nox
+**.nox/
+
+# playwright stuff
+**/playwright-*/
+**/test-results/

tests/retrolab/README.md

@@ -0,0 +1,32 @@
+# Retrolab accessibility tests
+
+## :package: Requirements
+
+To run the tests in this directory you need the following pre-requisites:
+
+- mamba
+- python > 3.7
+- nox
+- pyyaml
+
+you can install both nox and pyyaml from the command line with the following command:
+
+```bash
+python3 -m  pip install nox pyyaml
+```
+
+## :zap: Running the tests
+
+1. Make sure you are in the correct directory:
+
+    ```bash
+    cd tests/retrolab
+    ```
+
+2. Run the test with nox from the command line:
+
+    ```bash
+    nox -s a11y_tests
+    ```
+
+ If you are invoking the nox command for the first time, this may take while to install the dependencies and run the tests.

tests/retrolab/noxfile.py

@@ -0,0 +1,41 @@
+import nox
+from pathlib import Path
+from yaml import safe_load
+
+# global variables
+FILE = Path(__file__)
+TEST_DIR = FILE.parent
+ENV_FILE = TEST_DIR / "environment.yml"
+
+# parse the env file
+environment = safe_load(ENV_FILE.read_text())
+dependencies = environment.get("dependencies")
+# note this assumes that the last line is a pip package
+# maybe can be improved in the future, but it works for now
+requirements = dependencies.pop(-1).get("pip")
+
+
+def install_environment(session):
+    """Install conda dependencies - instead of creating a conda env through
+    conda env create -f we install dependency into the session venv.
+    This is faster than recreating, updating or pruning the environment using
+    conda env...
+
+    Args:
+        session (nox.session): Nox session calling the function
+    """
+    for conda_pkg in dependencies:
+        # installing conda dependencies
+        session.conda_install(conda_pkg, channel="conda-forge")
+        # installing pip dependencies
+        for pkg in requirements:
+            # We split each line in case there's a space for `-r`
+            session.install(*pkg.split())
+
+
+@nox.session(venv_backend="mamba", reuse_venv=True)
+def a11y_tests(session):
+    install_environment(session)
+    session.run("yarn", "install")
+    session.run("yarn", "playwright", "install", "chromium")
+    session.run("yarn", "run", "test")

tests/retrolab/package.json

@@ -0,0 +1,21 @@
+{
+  "name": "@retrolab/a11ytests",
+  "version": "1.0.0",
+  "private": true,
+  "description": "Run automated accessibility checks against RetroLab",
+  "license": "BSD-3-Clause",
+  "author": "Project Jupyter",
+  "scripts": {
+    "start": "jupyter retro --config test/jupyter_server_config.py",
+    "test": "playwright test",
+    "test:debug": "PWDEBUG=1 playwright test",
+    "test:report": "http-server ./playwright-report -a localhost -o",
+    "test:update": "playwright test --update-snapshots"
+  },
+  "dependencies": {
+    "@jupyterlab/galata": "4.0.2",
+    "axe-html-reporter": "^2.2.3",
+    "axe-playwright": "^1.1.9",
+    "expect-axe-playwright": "^1.2.1"
+  }
+}

tests/retrolab/playwright.config.ts

@@ -0,0 +1,95 @@
+import galataConfig from "@jupyterlab/galata/lib/playwright-config";
+import { PlaywrightTestConfig, devices } from "@playwright/test";
+// playwright.config.ts
+import { expect } from '@playwright/test'
+import { matchers } from 'expect-axe-playwright'
+
+expect.extend(matchers)
+// modified from https://github.com/MarcusFelling/demo.playwright/blob/main/accessibility/playwright.config.ts
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+const config: PlaywrightTestConfig = {
+  ...galataConfig,
+
+  testDir: "./test",
+
+  /* Maximum time one test can run for. */
+  timeout: 60 * 1000,
+
+  expect: {
+    /**
+     * Maximum time expect() should wait for the condition to be met.
+     * For example in `await expect(locator).toHaveText();`
+     */
+    timeout: 5000,
+  },
+
+  /* Fail the build on CI if you accidentally left test.only in the source code. */
+  forbidOnly: !!process.env.CI,
+
+  /* Retry on CI only */
+  retries: process.env.CI ? 2 : 0,
+
+  /* Opt out of parallel tests on CI. */
+  workers: process.env.CI ? 1 : undefined,
+
+  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+  reporter: "html",
+
+  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+  use: {
+    /* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */
+    actionTimeout: 0,
+
+    /* Base URL to use in actions like `await page.goto('/')`. */
+    // baseURL: process.env.BASEURL,
+
+    /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+    trace: "on",
+
+    acceptDownloads: true,
+
+    // from RetroLab's playwright.config: https://github.com/jupyterlab/retrolab/blob/main/ui-tests/playwright.config.ts
+    appPath: "/retro",
+
+    video: "retain-on-failure",
+  },
+
+  /* Configure projects for major browsers */
+  projects: [
+    {
+      name: "chromium",
+
+      /* Project-specific settings. */
+      use: {
+        ...devices["Desktop Chrome"],
+      },
+    },
+
+    // {
+    //   name: "firefox",
+    //   use: {
+    //     ...devices["Desktop Firefox"],
+    //   },
+    // },
+
+    // {
+    //   name: "webkit",
+    //   use: {
+    //     ...devices["Desktop Safari"],
+    //   },
+    // },
+  ],
+
+  // TODO: enable port 8888 and 9323
+  // saves results to playwright-report and test-results
+  webServer: {
+    command: "yarn start",
+    port: 8888,
+    timeout: 120 * 1000,
+    reuseExistingServer: !process.env.CI,
+  },
+};
+
+export default config;