From dcb94081ffbc54a5eeeb527d90ed9145e4a72bdb Mon Sep 17 00:00:00 2001 From: Ben Surgison Date: Fri, 20 Jan 2023 14:10:45 +0000 Subject: [PATCH 1/2] Reorganise and fix plugin tests --- .github/workflows/test-acceptance.yaml | 8 +- .github/workflows/test-download.yaml | 41 ------ .github/workflows/test-plugins.yaml | 56 -------- .../change-service-name.cypress.js | 0 .../clear-data-page.cypress.js | 0 .../edit-home-page.cypress.js | 0 .../management-available.cypress.js | 0 .../default-layout.cypress.js | 0 .../create-new-page-from-template.cypress.js | 128 ------------------ .../install-plugin-test.cypress.js | 0 .../multi-combined-plugin-test.cypress.js | 0 .../multi-plugin-test.cypress.js | 0 .../single-plugin-test.cypress.js | 0 .../install-available-plugin.cypress.js | 8 +- .../preview-template-view.cypress.js | 2 +- .../smoke/0-smoke-tests/index-page.cypress.js | 2 +- 16 files changed, 9 insertions(+), 236 deletions(-) delete mode 100644 .github/workflows/test-download.yaml delete mode 100644 .github/workflows/test-plugins.yaml rename cypress/e2e/dev/{6-management-tests => 5-management-tests}/change-service-name.cypress.js (100%) rename cypress/e2e/dev/{6-management-tests => 5-management-tests}/clear-data-page.cypress.js (100%) rename cypress/e2e/dev/{6-management-tests => 5-management-tests}/edit-home-page.cypress.js (100%) rename cypress/e2e/dev/{6-management-tests => 5-management-tests}/management-available.cypress.js (100%) rename cypress/e2e/dev/{7-layout-tests => 6-layout-tests}/default-layout.cypress.js (100%) delete mode 100644 cypress/e2e/dev/6-management-tests/create-new-page-from-template.cypress.js rename cypress/e2e/{dev/5-plugin-tests => plugins/0-mock-plugin-tests}/install-plugin-test.cypress.js (100%) rename cypress/e2e/{dev/5-plugin-tests => plugins/0-mock-plugin-tests}/multi-combined-plugin-test.cypress.js (100%) rename cypress/e2e/{dev/5-plugin-tests => plugins/0-mock-plugin-tests}/multi-plugin-test.cypress.js (100%) rename cypress/e2e/{dev/5-plugin-tests => plugins/0-mock-plugin-tests}/single-plugin-test.cypress.js (100%) rename cypress/e2e/plugins/{0-plugins-tests => 1-available-plugins-tests}/install-available-plugin.cypress.js (97%) rename cypress/e2e/{dev/6-management-tests => plugins/1-available-plugins-tests}/preview-template-view.cypress.js (95%) diff --git a/.github/workflows/test-acceptance.yaml b/.github/workflows/test-acceptance.yaml index b664c3cd15..8a6befc5a5 100644 --- a/.github/workflows/test-acceptance.yaml +++ b/.github/workflows/test-acceptance.yaml @@ -15,9 +15,9 @@ jobs: matrix: node-version: [18.x] os: [macos-latest, windows-latest, ubuntu-latest] - env: [smoke, dev, prod] + type: [smoke, plugins, dev, prod] - name: Acceptance ${{ matrix.env }} test kit on Node v${{ matrix.node-version }} (${{ matrix.os }}) + name: Acceptance ${{ matrix.type }} test kit on Node v${{ matrix.node-version }} (${{ matrix.os }}) runs-on: ${{ matrix.os }} timeout-minutes: 45 @@ -43,7 +43,7 @@ jobs: - run: npm ci - - run: npm run test:acceptance:${{ matrix.env }} + - run: npm run test:acceptance:${{ matrix.type }} env: CYPRESS_REQUEST_TIMEOUT: 20000 CYPRESS_DEFAULT_COMMAND_TIMEOUT: 40000 @@ -53,6 +53,6 @@ jobs: - if: ${{ failure() }} uses: actions/upload-artifact@v3 with: - name: cypress-screenshots-${{ matrix.env }}-${{ runner.os }}-${{ matrix.node-version }} + name: cypress-screenshots-${{ matrix.type }}-${{ runner.os }}-${{ matrix.node-version }} path: cypress/screenshots/ diff --git a/.github/workflows/test-download.yaml b/.github/workflows/test-download.yaml deleted file mode 100644 index 241608cde0..0000000000 --- a/.github/workflows/test-download.yaml +++ /dev/null @@ -1,41 +0,0 @@ -name: Tests (Download) - -on: - push: - branches: - - main - pull_request: - -jobs: - tests: - - strategy: - fail-fast: false # continue other tests if one test in matrix fails - matrix: - node-version: [16.x] - os: [ubuntu-latest] - - name: Test downloading kit from website - continue-on-error: true - runs-on: ${{ matrix.os }} - timeout-minutes: 25 - - env: - CYPRESS_CACHE_FOLDER: ~/.cache/Cypress - - steps: - - uses: actions/checkout@v3 - - - name: Use Node v${{ matrix.node-version }} - uses: actions/setup-node@v3 - with: - cache: 'npm' - node-version: ${{ matrix.node-version }} - - - name: Cache Cypress binary - uses: actions/cache@v3 - with: - path: ~/.cache/Cypress - key: cypress-${{ runner.os }}-cypress-${{ hashFiles('**/package.json') }} - - - run: npm ci diff --git a/.github/workflows/test-plugins.yaml b/.github/workflows/test-plugins.yaml deleted file mode 100644 index 3d4c1a6fbd..0000000000 --- a/.github/workflows/test-plugins.yaml +++ /dev/null @@ -1,56 +0,0 @@ -name: Tests (Acceptance-Plugins) - -on: - push: - branches: - - main - - support/* - pull_request: - -jobs: - tests: - - strategy: - fail-fast: false # continue other tests if one test in matrix fails - matrix: - node-version: [16.x, 18.x] - os: [macos-latest, windows-latest, ubuntu-latest] - - name: Acceptance test kit on Node v${{ matrix.node-version }} (${{ matrix.os }}) - runs-on: ${{ matrix.os }} - timeout-minutes: 25 - - env: - CYPRESS_CACHE_FOLDER: ~/.cache/Cypress - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Use Node v${{ matrix.node-version }} - uses: actions/setup-node@v3 - with: - cache: 'npm' - node-version: ${{ matrix.node-version }} - - - name: Cache Cypress binary - uses: actions/cache@v3 - with: - path: ~/.cache/Cypress - key: cypress-${{ runner.os }}-cypress-${{ hashFiles('**/package.json') }} - - - run: npm ci - - - run: npm run test:acceptance:plugins - env: - CYPRESS_REQUEST_TIMEOUT: 20000 - CYPRESS_DEFAULT_COMMAND_TIMEOUT: 40000 - CYPRESS_PAGE_LOAD_TIMEOUT: 120000 - CYPRESS_RETRIES: 3 - - - if: ${{ failure() }} - uses: actions/upload-artifact@v3 - with: - name: cypress-screenshots-dev-${{ runner.os }}-${{ matrix.node-version }} - path: cypress/screenshots/ diff --git a/cypress/e2e/dev/6-management-tests/change-service-name.cypress.js b/cypress/e2e/dev/5-management-tests/change-service-name.cypress.js similarity index 100% rename from cypress/e2e/dev/6-management-tests/change-service-name.cypress.js rename to cypress/e2e/dev/5-management-tests/change-service-name.cypress.js diff --git a/cypress/e2e/dev/6-management-tests/clear-data-page.cypress.js b/cypress/e2e/dev/5-management-tests/clear-data-page.cypress.js similarity index 100% rename from cypress/e2e/dev/6-management-tests/clear-data-page.cypress.js rename to cypress/e2e/dev/5-management-tests/clear-data-page.cypress.js diff --git a/cypress/e2e/dev/6-management-tests/edit-home-page.cypress.js b/cypress/e2e/dev/5-management-tests/edit-home-page.cypress.js similarity index 100% rename from cypress/e2e/dev/6-management-tests/edit-home-page.cypress.js rename to cypress/e2e/dev/5-management-tests/edit-home-page.cypress.js diff --git a/cypress/e2e/dev/6-management-tests/management-available.cypress.js b/cypress/e2e/dev/5-management-tests/management-available.cypress.js similarity index 100% rename from cypress/e2e/dev/6-management-tests/management-available.cypress.js rename to cypress/e2e/dev/5-management-tests/management-available.cypress.js diff --git a/cypress/e2e/dev/7-layout-tests/default-layout.cypress.js b/cypress/e2e/dev/6-layout-tests/default-layout.cypress.js similarity index 100% rename from cypress/e2e/dev/7-layout-tests/default-layout.cypress.js rename to cypress/e2e/dev/6-layout-tests/default-layout.cypress.js diff --git a/cypress/e2e/dev/6-management-tests/create-new-page-from-template.cypress.js b/cypress/e2e/dev/6-management-tests/create-new-page-from-template.cypress.js deleted file mode 100644 index 07d50abc12..0000000000 --- a/cypress/e2e/dev/6-management-tests/create-new-page-from-template.cypress.js +++ /dev/null @@ -1,128 +0,0 @@ - -// core dependencies -const path = require('path') - -// local dependencies -const { waitForApplication, getTemplateLink, deleteFile } = require('../../utils') - -const viewFolder = path.join(Cypress.env('projectFolder'), 'app', 'views') - -const startPageView = path.join(viewFolder, 'start.html') -const startPagePath = '/start' - -const validUnicodePageView = path.join(viewFolder, 'brontë.html') -const validUnicodePagePath = '/brontë' - -const manageTemplatesPagePath = '/manage-prototype/templates' - -describe('create new page', () => { - before(() => { - Cypress.on('uncaught:exception', (err, runnable) => { - // we expect an error with message 'Cannot read properties of undefined (reading 'documentReady')' - // and don't want to fail the test so we return false - if (err.message.includes('Cannot read properties of undefined (reading \'documentReady\')')) { - return false - } - // we still want to ensure there are no other unexpected - // errors, so we let them fail the test - }) - deleteFile(startPageView) - deleteFile(validUnicodePageView) - waitForApplication(manageTemplatesPagePath) - }) - - it('View the start page from the management page', () => { - cy.task('log', 'Visit the manage prototype templates page') - cy.get(`a[href="${getTemplateLink('view', 'govuk-prototype-kit', '/lib/nunjucks/templates/start.html')}"]`).click() - - cy.task('log', 'The start page template should be displayed') - cy.get('.govuk-button--start') - .should('contains.text', 'Start now') - }) - - describe('Create the start page from the management page', () => { - const testCreatePage = (pagePath, pageView) => () => { - cy.task('log', 'The start page should not be found') - cy.visit(pagePath, { failOnStatusCode: false }) - cy.get('body') - .should('contains.text', `Page not found: ${pagePath}`) - - cy.task('log', 'Visit the manage prototype templates page') - cy.visit(manageTemplatesPagePath) - - cy.get(`a[href="${getTemplateLink('install', 'govuk-prototype-kit', '/lib/nunjucks/templates/start.html')}"]`).click() - - cy.task('log', 'Create the page') - cy.get('.govuk-heading-l') - .should('contains.text', 'Create new Start page') - cy.get('.govuk-label') - .should('contains.text', 'Path for the new page') - cy.get('#chosen-url') - .type(pagePath) - cy.get('.govuk-button') - .should('contains.text', 'Create page').click() - - cy.task('log', 'Confirmation page') - cy.get('.govuk-heading-l') - .should('contains.text', 'Page created') - - cy.task('log', 'Confirm the page exists') - cy.get(`a[href="${pagePath}"]`) - .should('contains.text', pagePath).click() - cy.task('log', 'The start page should be displayed') - cy.get('.govuk-button--start') - .should('contains.text', 'Start now') - - cy.task('log', 'Confirm the view of the page exists where expected') - cy.task('existsFile', { filename: pageView }) - } - - it(`where url is ${startPagePath}`, testCreatePage(startPagePath, startPageView)) - - it(`where url is ${validUnicodePagePath}`, testCreatePage(validUnicodePagePath, validUnicodePageView)) - }) - - describe('Invalid urls entered', () => { - const testError = (url, error) => () => { - url && cy.get('input').type(url) - cy.get('form').submit() - cy.get('.govuk-error-summary__list').should('contains.text', error) - cy.get('#chosen-url-error').should('contains.text', error) - } - - const errors = { - exists: 'Path already exists', - missing: 'Enter a path', - singleSlash: 'Path must not be a single forward slash (/)', - endsWithSlash: 'Path must not end in a forward slash (/)', - multipleSlashes: 'must not include a slash followed by another slash (//)', - slash: 'Path must begin with a forward slash (/)', - invalid: 'Path must not include !$&\'()*+,;=:?#[]@.% or space' - } - - beforeEach(() => { - cy.visit(manageTemplatesPagePath) - cy.get(`a[href="${getTemplateLink('install', 'govuk-prototype-kit', '/lib/nunjucks/templates/start.html')}"]`).click() - }) - - it('already exists', testError(startPagePath, errors.exists)) - - it('empty', testError('', errors.missing)) - - it('empty (multiple spaces)', testError(' ', errors.missing)) - - it('single slash only', testError('/', errors.singleSlash)) - - it('missing starting slash', testError('foo/bar', errors.slash)) - - it('ends with a slash', testError('/foo/bar/', errors.endsWithSlash)) - - it('contains consecutive slashes', testError('//bar/foo', errors.multipleSlashes)) - - it('invalid (contains a search parameter)', testError('/?param=true', errors.invalid)) - - it('invalid (contains spaces in the url)', testError('/foo bar/baz bar', errors.invalid)) - - it('invalid (random)', testError('/$$fr%%"pp', errors.invalid)) - }) -}) diff --git a/cypress/e2e/dev/5-plugin-tests/install-plugin-test.cypress.js b/cypress/e2e/plugins/0-mock-plugin-tests/install-plugin-test.cypress.js similarity index 100% rename from cypress/e2e/dev/5-plugin-tests/install-plugin-test.cypress.js rename to cypress/e2e/plugins/0-mock-plugin-tests/install-plugin-test.cypress.js diff --git a/cypress/e2e/dev/5-plugin-tests/multi-combined-plugin-test.cypress.js b/cypress/e2e/plugins/0-mock-plugin-tests/multi-combined-plugin-test.cypress.js similarity index 100% rename from cypress/e2e/dev/5-plugin-tests/multi-combined-plugin-test.cypress.js rename to cypress/e2e/plugins/0-mock-plugin-tests/multi-combined-plugin-test.cypress.js diff --git a/cypress/e2e/dev/5-plugin-tests/multi-plugin-test.cypress.js b/cypress/e2e/plugins/0-mock-plugin-tests/multi-plugin-test.cypress.js similarity index 100% rename from cypress/e2e/dev/5-plugin-tests/multi-plugin-test.cypress.js rename to cypress/e2e/plugins/0-mock-plugin-tests/multi-plugin-test.cypress.js diff --git a/cypress/e2e/dev/5-plugin-tests/single-plugin-test.cypress.js b/cypress/e2e/plugins/0-mock-plugin-tests/single-plugin-test.cypress.js similarity index 100% rename from cypress/e2e/dev/5-plugin-tests/single-plugin-test.cypress.js rename to cypress/e2e/plugins/0-mock-plugin-tests/single-plugin-test.cypress.js diff --git a/cypress/e2e/plugins/0-plugins-tests/install-available-plugin.cypress.js b/cypress/e2e/plugins/1-available-plugins-tests/install-available-plugin.cypress.js similarity index 97% rename from cypress/e2e/plugins/0-plugins-tests/install-available-plugin.cypress.js rename to cypress/e2e/plugins/1-available-plugins-tests/install-available-plugin.cypress.js index 5979997034..a91e8d5c57 100644 --- a/cypress/e2e/plugins/0-plugins-tests/install-available-plugin.cypress.js +++ b/cypress/e2e/plugins/1-available-plugins-tests/install-available-plugin.cypress.js @@ -139,7 +139,7 @@ describe('Management plugins: ', () => { cy.get('a[href*="/templates"]') .should('contains.text', 'Templates').click() - cy.get('h2').eq(2).should('contain.text', pluginName) + cy.get('h2').contains(pluginName) cy.task('log', `Create a new ${pluginPageTitle} page`) @@ -150,8 +150,7 @@ describe('Management plugins: ', () => { .should('contains.text', `Create new ${pluginPageTitle} page`) cy.get('#chosen-url') .type(pluginPagePath) - cy.get('.govuk-button') - .should('contains.text', 'Create page').click() + cy.get('.govuk-button').contains('Create page').click() provePluginFunctionalityWorks() }) @@ -217,8 +216,7 @@ describe('Management plugins: ', () => { const pkg = 'invalid-prototype-kit-plugin' const pluginName = 'Invalid Prototype Kit Plugin' cy.visit(`${managePluginsPagePath}/install?package=${encodeURIComponent(pkg)}`) - cy.get('h2') - .should('contains.text', `Install ${pluginName}`) + cy.get('h2').contains(`Install ${pluginName}`) failAction() }) diff --git a/cypress/e2e/dev/6-management-tests/preview-template-view.cypress.js b/cypress/e2e/plugins/1-available-plugins-tests/preview-template-view.cypress.js similarity index 95% rename from cypress/e2e/dev/6-management-tests/preview-template-view.cypress.js rename to cypress/e2e/plugins/1-available-plugins-tests/preview-template-view.cypress.js index 79d6646cdd..4daf2f78b6 100644 --- a/cypress/e2e/dev/6-management-tests/preview-template-view.cypress.js +++ b/cypress/e2e/plugins/1-available-plugins-tests/preview-template-view.cypress.js @@ -25,7 +25,7 @@ describe('Management plugins: ', () => { }) it(`Preview a ${plugin}${version2} template`, () => { - cy.get('h2').eq(2).should('contain.text', pluginName) + cy.get('h2').contains(pluginName) cy.task('log', `Preview the ${pluginPageTitle} template`) diff --git a/cypress/e2e/smoke/0-smoke-tests/index-page.cypress.js b/cypress/e2e/smoke/0-smoke-tests/index-page.cypress.js index 6633ae1ab2..5a2288bc9e 100644 --- a/cypress/e2e/smoke/0-smoke-tests/index-page.cypress.js +++ b/cypress/e2e/smoke/0-smoke-tests/index-page.cypress.js @@ -1,6 +1,6 @@ specify('index page', () => { cy.visit('/') - cy.contains('GOV.UK Prototype Kit') + cy.get('h2').contains('GOV.UK Prototype Kit') }) specify('GOV.UK Frontend fonts loaded', () => { From 9b3eae1c97c6f94e130e71d6313644f9bfdc9f2f Mon Sep 17 00:00:00 2001 From: Ben Surgison Date: Fri, 20 Jan 2023 17:09:54 +0000 Subject: [PATCH 2/2] Reduce timeout now workflows are more balanced. --- .github/workflows/test-acceptance.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-acceptance.yaml b/.github/workflows/test-acceptance.yaml index 8a6befc5a5..e21b74e8a9 100644 --- a/.github/workflows/test-acceptance.yaml +++ b/.github/workflows/test-acceptance.yaml @@ -19,7 +19,7 @@ jobs: name: Acceptance ${{ matrix.type }} test kit on Node v${{ matrix.node-version }} (${{ matrix.os }}) runs-on: ${{ matrix.os }} - timeout-minutes: 45 + timeout-minutes: 20 env: CYPRESS_CACHE_FOLDER: ~/.cache/Cypress