diff --git a/.travis.yml b/.travis.yml index d0de440677a9f..b7158c74a1dfe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -69,8 +69,8 @@ jobs: install: - ./bin/setup-local-env.sh script: - - $( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --listTests > ~/.jest-e2e-tests - npm run build + - $( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --listTests > ~/.jest-e2e-tests - npm run test-e2e -- --ci --cacheDirectory="$HOME/.jest-cache" --runTestsByPath $( awk 'NR % 4 == 0' < ~/.jest-e2e-tests ) - name: E2E tests (Admin with plugins) (2/4) @@ -78,8 +78,8 @@ jobs: install: - ./bin/setup-local-env.sh script: - - $( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --listTests > ~/.jest-e2e-tests - npm run build + - $( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --listTests > ~/.jest-e2e-tests - npm run test-e2e -- --ci --cacheDirectory="$HOME/.jest-cache" --runTestsByPath $( awk 'NR % 4 == 1' < ~/.jest-e2e-tests ) - name: E2E tests (Admin with plugins) (3/4) @@ -87,8 +87,8 @@ jobs: install: - ./bin/setup-local-env.sh script: - - $( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --listTests > ~/.jest-e2e-tests - npm run build + - $( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --listTests > ~/.jest-e2e-tests - npm run test-e2e -- --ci --cacheDirectory="$HOME/.jest-cache" --runTestsByPath $( awk 'NR % 4 == 2' < ~/.jest-e2e-tests ) - name: E2E tests (Admin with plugins) (4/4) @@ -96,8 +96,8 @@ jobs: install: - ./bin/setup-local-env.sh script: - - $( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --listTests > ~/.jest-e2e-tests - npm run build + - $( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --listTests > ~/.jest-e2e-tests - npm run test-e2e -- --ci --cacheDirectory="$HOME/.jest-cache" --runTestsByPath $( awk 'NR % 4 == 3' < ~/.jest-e2e-tests ) - name: E2E tests (Author without plugins) (1/4) @@ -105,8 +105,8 @@ jobs: install: - ./bin/setup-local-env.sh script: - - $( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --listTests > ~/.jest-e2e-tests - npm run build + - $( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --listTests > ~/.jest-e2e-tests - npm run test-e2e -- --ci --cacheDirectory="$HOME/.jest-cache" --runTestsByPath $( awk 'NR % 4 == 0' < ~/.jest-e2e-tests ) - name: E2E tests (Author without plugins) (2/4) @@ -114,8 +114,8 @@ jobs: install: - ./bin/setup-local-env.sh script: - - $( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --listTests > ~/.jest-e2e-tests - npm run build + - $( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --listTests > ~/.jest-e2e-tests - npm run test-e2e -- --ci --cacheDirectory="$HOME/.jest-cache" --runTestsByPath $( awk 'NR % 4 == 1' < ~/.jest-e2e-tests ) - name: E2E tests (Author without plugins) (3/4) @@ -123,8 +123,8 @@ jobs: install: - ./bin/setup-local-env.sh script: - - $( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --listTests > ~/.jest-e2e-tests - npm run build + - $( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --listTests > ~/.jest-e2e-tests - npm run test-e2e -- --ci --cacheDirectory="$HOME/.jest-cache" --runTestsByPath $( awk 'NR % 4 == 2' < ~/.jest-e2e-tests ) - name: E2E tests (Author without plugins) (4/4) @@ -132,9 +132,10 @@ jobs: install: - ./bin/setup-local-env.sh script: - - $( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --listTests > ~/.jest-e2e-tests - npm run build + - $( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --listTests > ~/.jest-e2e-tests - npm run test-e2e -- --ci --cacheDirectory="$HOME/.jest-cache" --runTestsByPath $( awk 'NR % 4 == 3' < ~/.jest-e2e-tests ) + allow_failures: - name: PHP unit tests (PHP 5.3) env: WP_VERSION=latest SWITCH_TO_PHP=5.3 diff --git a/package-lock.json b/package-lock.json index 2ea228aa9e96f..e7af47a4f0dbd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3271,6 +3271,7 @@ "requires": { "@wordpress/e2e-test-utils": "file:packages/e2e-test-utils", "@wordpress/jest-console": "file:packages/jest-console", + "@wordpress/jest-puppeteer-axe": "file:packages/jest-puppeteer-axe", "@wordpress/scripts": "file:packages/scripts", "expect-puppeteer": "^4.0.0", "lodash": "^4.17.11" diff --git a/packages/e2e-tests/CHANGELOG.md b/packages/e2e-tests/CHANGELOG.md index 75b6278875283..397c64f9d17a8 100644 --- a/packages/e2e-tests/CHANGELOG.md +++ b/packages/e2e-tests/CHANGELOG.md @@ -1,3 +1,9 @@ +## Unreleased + +### New features + +- Added Axe (the Accessibility Engine) API integration with e2e tests suite. + ## 1.0.0 (2019-03-06) - Initial release. diff --git a/packages/e2e-tests/config/setup-test-framework.js b/packages/e2e-tests/config/setup-test-framework.js index 4800da23a860f..368f0540920d0 100644 --- a/packages/e2e-tests/config/setup-test-framework.js +++ b/packages/e2e-tests/config/setup-test-framework.js @@ -6,15 +6,14 @@ import { get } from 'lodash'; /** * WordPress dependencies */ -import '@wordpress/jest-console'; import { + activatePlugin, clearLocalStorage, enablePageDialogAccept, setBrowserViewport, - visitAdminPage, - activatePlugin, switchUserToAdmin, switchUserToTest, + visitAdminPage, } from '@wordpress/e2e-test-utils'; /** @@ -148,6 +147,40 @@ function observeConsoleLogging() { } ); } +/** + * Runs Axe tests when the block editor is found on the current page. + * + * @return {?Promise} Promise resolving once Axe texts are finished. + */ +async function runAxeTestsForBlockEditor() { + if ( ! await page.$( '.block-editor' ) ) { + return; + } + + await expect( page ).toPassAxeTests( { + // Temporary disabled rules to enable initial integration. + // See: https://github.com/WordPress/gutenberg/pull/15018. + disabledRules: [ + 'aria-allowed-role', + 'aria-valid-attr-value', + 'button-name', + 'color-contrast', + 'dlitem', + 'duplicate-id', + 'label', + 'link-name', + 'listitem', + 'region', + ], + exclude: [ + // Ignores elements created by metaboxes. + '.edit-post-layout__metaboxes', + // Ignores elements created by TinyMCE. + '.mce-container', + ], + } ); +} + // Before every test suite run, delete all content created by the test. This ensures // other posts/comments/etc. aren't dirtying tests and tests don't depend on // each other's side-effects. @@ -162,6 +195,7 @@ beforeAll( async () => { } ); afterEach( async () => { + await runAxeTestsForBlockEditor(); await setupBrowser(); } ); diff --git a/packages/e2e-tests/jest.config.js b/packages/e2e-tests/jest.config.js index ea4d99ce56ac1..eeb88d9aa0daf 100644 --- a/packages/e2e-tests/jest.config.js +++ b/packages/e2e-tests/jest.config.js @@ -5,6 +5,8 @@ module.exports = { ], setupFilesAfterEnv: [ '/config/setup-test-framework.js', + '@wordpress/jest-console', + '@wordpress/jest-puppeteer-axe', 'expect-puppeteer', ], }; diff --git a/packages/e2e-tests/package.json b/packages/e2e-tests/package.json index 98a3e325e3a0c..302aa7eaf57fb 100644 --- a/packages/e2e-tests/package.json +++ b/packages/e2e-tests/package.json @@ -24,6 +24,7 @@ "dependencies": { "@wordpress/e2e-test-utils": "file:../e2e-test-utils", "@wordpress/jest-console": "file:../jest-console", + "@wordpress/jest-puppeteer-axe": "file:../jest-puppeteer-axe", "@wordpress/scripts": "file:../scripts", "expect-puppeteer": "^4.0.0", "lodash": "^4.17.11" diff --git a/packages/e2e-tests/plugins/format-api/index.js b/packages/e2e-tests/plugins/format-api/index.js index 4d41f7c26c4b3..671767e204926 100644 --- a/packages/e2e-tests/plugins/format-api/index.js +++ b/packages/e2e-tests/plugins/format-api/index.js @@ -18,7 +18,7 @@ props.value, { type: 'my-plugin/link', attributes: { - url: '#test', + url: 'https://example.com', } } ) diff --git a/packages/e2e-tests/specs/plugins/__snapshots__/format-api.test.js.snap b/packages/e2e-tests/specs/plugins/__snapshots__/format-api.test.js.snap index 20a79ccc33efb..e5a006328d097 100644 --- a/packages/e2e-tests/specs/plugins/__snapshots__/format-api.test.js.snap +++ b/packages/e2e-tests/specs/plugins/__snapshots__/format-api.test.js.snap @@ -2,6 +2,6 @@ exports[`Using Format API Clicking the control wraps the selected text properly with HTML code 1`] = ` " -

First paragraph

+

First paragraph

" `; diff --git a/packages/e2e-tests/specs/plugins/meta-attribute-block.test.js b/packages/e2e-tests/specs/plugins/meta-attribute-block.test.js index 45c8256159979..5de9ea1f6c619 100644 --- a/packages/e2e-tests/specs/plugins/meta-attribute-block.test.js +++ b/packages/e2e-tests/specs/plugins/meta-attribute-block.test.js @@ -41,13 +41,14 @@ describe( 'Block with a meta attribute', () => { await page.keyboard.type( 'Meta Value' ); const inputs = await page.$$( '.my-meta-input' ); - await inputs.forEach( async ( input ) => { + await Promise.all( inputs.map( async ( input ) => { // Clicking the input selects the block, // and selecting the block enables the sync data mode - // as otherwise the asynchronous rerendering of unselected blocks + // as otherwise the asynchronous re-rendering of unselected blocks // may cause the input to have not yet been updated for the other blocks await input.click(); - expect( await input.getProperty( 'value' ) ).toBe( 'Meta Value' ); - } ); + const inputValue = await input.getProperty( 'value' ); + expect( await inputValue.jsonValue() ).toBe( 'Meta Value' ); + } ) ); } ); } ); diff --git a/packages/jest-puppeteer-axe/CHANGELOG.md b/packages/jest-puppeteer-axe/CHANGELOG.md index 2256a6340e4fc..c8e5368f48496 100644 --- a/packages/jest-puppeteer-axe/CHANGELOG.md +++ b/packages/jest-puppeteer-axe/CHANGELOG.md @@ -1,3 +1,9 @@ +## Unreleased + +### New features + +- Added optional `disabledRules` option to use with `toPassAxeTests` matcher. + ## 1.0.0 (2019-03-06) - Initial release. diff --git a/packages/jest-puppeteer-axe/README.md b/packages/jest-puppeteer-axe/README.md index 29fd06538ed47..c28b7ed96f752 100644 --- a/packages/jest-puppeteer-axe/README.md +++ b/packages/jest-puppeteer-axe/README.md @@ -38,8 +38,9 @@ test( 'checks the test page with Axe', async () => { ``` It is also possible to pass optional Axe API options to perform customized check: -- `include` - CSS selector to to add the list of elements to include in analysis. -- `exclude` - CSS selector to to add the list of elements to exclude from analysis. +- `include` - CSS selector(s) to to add the list of elements to include in analysis. +- `exclude` - CSS selector(s) to to add the list of elements to exclude from analysis. +- `disabledRules` - the list of [Axe rules](https://github.com/dequelabs/axe-core/blob/master/doc/rule-descriptions.md) to skip from verification. ```js test( 'checks the test component with Axe excluding some button', async () => { @@ -50,6 +51,7 @@ test( 'checks the test component with Axe excluding some button', async () => { await expect( page ).toPassAxeTests( { include: '.test-component', exclude: '.some-button', + disabledRules: [ 'aria-allowed-role' ], } ); } ); ``` diff --git a/packages/jest-puppeteer-axe/src/index.js b/packages/jest-puppeteer-axe/src/index.js index ccf1be7cf0a64..336cec9bcf816 100644 --- a/packages/jest-puppeteer-axe/src/index.js +++ b/packages/jest-puppeteer-axe/src/index.js @@ -11,8 +11,9 @@ import AxePuppeteer from 'axe-puppeteer'; * @return {string} The user friendly message to display when the matcher fails. */ function formatViolations( violations ) { - return violations.map( ( { help, id, nodes } ) => { - let output = `Rule: ${ id } (${ help })\n` + + return violations.map( ( { help, helpUrl, id, nodes } ) => { + let output = `Rule: "${ id }" (${ help })\n` + + `Help: ${ helpUrl }\n` + 'Affected Nodes:\n'; nodes.forEach( ( node ) => { @@ -52,16 +53,17 @@ function formatViolations( violations ) { * * @see https://github.com/dequelabs/axe-puppeteer * - * @param {Page} page Puppeteer's page instance. - * @param {?Object} params Optional Axe API options. - * @param {?string} params.include CSS selector to add to the list of elements - * to include in analysis. - * @param {?string} params.exclude CSS selector to add to the list of elements - * to exclude from analysis. + * @param {Page} page Puppeteer's page instance. + * @param {?Object} params Optional Axe API options. + * @param {?string|Array} params.include CSS selector(s) to add to the list of elements + * to include in analysis. + * @param {?string|Array} params.exclude CSS selector(s) to add to the list of elements + * to exclude from analysis. + * @param {?Array} params.disabledRules The list of Axe rules to skip from verification. * * @return {Object} A matcher object with two keys `pass` and `message`. */ -async function toPassAxeTests( page, { include, exclude } = {} ) { +async function toPassAxeTests( page, { include, exclude, disabledRules } = {} ) { const axe = new AxePuppeteer( page ); if ( include ) { @@ -72,6 +74,10 @@ async function toPassAxeTests( page, { include, exclude } = {} ) { axe.exclude( exclude ); } + if ( disabledRules ) { + axe.disableRules( disabledRules ); + } + const { violations } = await axe.analyze(); const pass = violations.length === 0;