From a969f315f38a4e8a83ddd3290eef622557df41de Mon Sep 17 00:00:00 2001 From: Joe D'Amore Date: Tue, 8 Oct 2024 17:13:16 -0400 Subject: [PATCH 01/25] Delete 'region-1' Docker Compose service, deprecate 'e2e', 'e2e_heimdall', and 'component' services, and add 'cypress_local', 'cypress_remote', and 'cypress_component' --- docker-compose.yml | 60 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 45 insertions(+), 15 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 5285fdd036d..d324c01e97f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -82,14 +82,14 @@ x-e2e-runners: context: . dockerfile: ./packages/manager/Dockerfile target: e2e - depends_on: - web: - condition: service_healthy + # depends_on: + # web: + # condition: service_healthy env_file: ./packages/manager/.env volumes: *default-volumes # TODO Stop using entrypoint, use CMD instead. # (Or just make `yarn` the entrypoint, but either way stop forcing `cy:e2e`). - entrypoint: ['yarn', 'cy:e2e'] + # entrypoint: ['yarn', 'cy:e2e'] services: # Serves a local instance of Cloud Manager for Cypress to use for its tests. @@ -112,6 +112,45 @@ services: timeout: 10s retries: 10 + # Cypress test runner service to run tests against a remotely-served Cloud instance. + # + # This is useful when testing against a standard Cloud Manager environment, + # like Production at cloud.linode.com, but can also be used to run tests against + # pre-Prod environments, PR preview links, and more. + cypress_remote: + <<: *default-runner + environment: + <<: *default-env + MANAGER_OAUTH: ${MANAGER_OAUTH} + + # Cypress test runner service to run tests against a locally-served Cloud instance. + # + # This is useful when testing against a customized or in-development build of + # Cloud Manager. + cypress_local: + <<: *default-runner + environment: + <<: *default-env + MANAGER_OAUTH: ${MANAGER_OAUTH} + depends_on: + web: + condition: service_healthy + + # Cypress component test runner service. + # + # Unlike other Cloud Manager Cypress tests, these tests can be run without + # requiring a Cloud Manager environment. + cypress_component: + <<: *default-runner + environment: + CY_TEST_DISABLE_RETRIES: ${CY_TEST_DISABLE_RETRIES} + CY_TEST_JUNIT_REPORT: ${CY_TEST_JUNIT_REPORT} + + + # --> ! DEPRECATION NOTICE ! <-- + # The services below this line are deprecated, and will be deleted soon. + # Don't build any pipelines are write any scripts that depend on these. + # Generic end-to-end test runner for Cloud's primary testing pipeline. # Configured to run against a local Cloud instance. e2e: @@ -119,6 +158,7 @@ services: environment: <<: *default-env MANAGER_OAUTH: ${MANAGER_OAUTH} + entrypoint: ['yarn', 'cy:e2e'] # Component test runner. # Does not require any Cloud Manager environment to run. @@ -138,14 +178,4 @@ services: environment: <<: *default-env MANAGER_OAUTH: ${MANAGER_OAUTH} - - region-1: - build: - context: . - dockerfile: ./packages/manager/Dockerfile - target: e2e - env_file: ./packages/manager/.env - volumes: *default-volumes - environment: - <<: *default-env - MANAGER_OAUTH: ${MANAGER_OAUTH_1} + entrypoint: ['yarn', 'cy:e2e'] From 7db76b88fd7d242b2b0fbb4ae5cfced936b88120 Mon Sep 17 00:00:00 2001 From: Joe D'Amore Date: Tue, 8 Oct 2024 18:21:53 -0400 Subject: [PATCH 02/25] Remove entrypoint for test runner in Dockerfile --- packages/manager/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/manager/Dockerfile b/packages/manager/Dockerfile index 2cfa22aca4e..466edaf64ea 100644 --- a/packages/manager/Dockerfile +++ b/packages/manager/Dockerfile @@ -47,4 +47,3 @@ ENV CI=1 ENV NO_COLOR=1 ENV HOME=/home/node/ ENV CYPRESS_CACHE_FOLDER=/home/node/.cache/Cypress -ENTRYPOINT yarn cy:ci From 4dec8d3a3096144cd848e166d6eba3fc28432d61 Mon Sep 17 00:00:00 2001 From: Joe D'Amore Date: Tue, 8 Oct 2024 18:40:37 -0400 Subject: [PATCH 03/25] Improve docs, make `yarn` the default entrypoint --- docker-compose.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index d324c01e97f..bad4f57cd54 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -89,7 +89,7 @@ x-e2e-runners: volumes: *default-volumes # TODO Stop using entrypoint, use CMD instead. # (Or just make `yarn` the entrypoint, but either way stop forcing `cy:e2e`). - # entrypoint: ['yarn', 'cy:e2e'] + entrypoint: 'yarn' services: # Serves a local instance of Cloud Manager for Cypress to use for its tests. @@ -149,7 +149,10 @@ services: # --> ! DEPRECATION NOTICE ! <-- # The services below this line are deprecated, and will be deleted soon. - # Don't build any pipelines are write any scripts that depend on these. + # Don't build any pipelines or write any scripts that depend on these. + # Instead, opt to use `cypress_local` in places where you would've used `e2e`, + # use `cypress_remote` in places where you would've used `e2e_heimdall`, and + # use `cypress_component` in places where you would've used `component`. # Generic end-to-end test runner for Cloud's primary testing pipeline. # Configured to run against a local Cloud instance. From f8a98e0362b139725352b3ab246487242c2b074a Mon Sep 17 00:00:00 2001 From: Joe D'Amore Date: Thu, 10 Oct 2024 13:52:14 -0400 Subject: [PATCH 04/25] WIP add temporary console.log --- packages/manager/cypress/support/plugins/split-run.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/manager/cypress/support/plugins/split-run.ts b/packages/manager/cypress/support/plugins/split-run.ts index eccb6c0e153..2482e85c851 100644 --- a/packages/manager/cypress/support/plugins/split-run.ts +++ b/packages/manager/cypress/support/plugins/split-run.ts @@ -57,6 +57,10 @@ export const splitCypressRun: CypressPlugin = (_on, config) => { // Override configuration spec pattern to reflect test subset for this runner... const specs = globSync(config.specPattern); + console.log({ + specPattern: config.specPattern, + specs, + }); let totalWeight = 0; let weightedSpecs: SpecWeight[] = []; From c8c37020779bb40bd4ef18cab966137640f2e91d Mon Sep 17 00:00:00 2001 From: Joe D'Amore Date: Thu, 10 Oct 2024 15:14:00 -0400 Subject: [PATCH 05/25] Remove temporary logging call --- packages/manager/cypress/support/plugins/split-run.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/manager/cypress/support/plugins/split-run.ts b/packages/manager/cypress/support/plugins/split-run.ts index 2482e85c851..eccb6c0e153 100644 --- a/packages/manager/cypress/support/plugins/split-run.ts +++ b/packages/manager/cypress/support/plugins/split-run.ts @@ -57,10 +57,6 @@ export const splitCypressRun: CypressPlugin = (_on, config) => { // Override configuration spec pattern to reflect test subset for this runner... const specs = globSync(config.specPattern); - console.log({ - specPattern: config.specPattern, - specs, - }); let totalWeight = 0; let weightedSpecs: SpecWeight[] = []; From b600561680c6fbfe710c72554c965945b5ccfcfe Mon Sep 17 00:00:00 2001 From: Joe D'Amore Date: Thu, 10 Oct 2024 18:11:03 -0400 Subject: [PATCH 06/25] Allow Cypress Slack notification title to be customized --- scripts/junit-summary/formatters/slack-formatter.ts | 6 ++++-- scripts/junit-summary/index.ts | 1 + scripts/junit-summary/metadata/metadata.ts | 3 +++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts/junit-summary/formatters/slack-formatter.ts b/scripts/junit-summary/formatters/slack-formatter.ts index 4168cd4a9f5..2d11447105c 100644 --- a/scripts/junit-summary/formatters/slack-formatter.ts +++ b/scripts/junit-summary/formatters/slack-formatter.ts @@ -23,9 +23,11 @@ export const slackFormatter: Formatter = ( _junitData: TestSuites[] ) => { const indicator = runInfo.failing ? ':x-mark:' : ':check-mark:'; + const runTitle = metadata.pipelineTitle ?? 'Cypress test results'; + const headline = (metadata.runId && metadata.runUrl) - ? `*Cypress test results for run <${metadata.runUrl}|#${metadata.runId}>*\n` - : `*Cypress test results*\n`; + ? `*${runTitle}* (<${metadata.runUrl}|#${metadata.runId}>)\n` + : `*${runTitle}*\n`; const breakdown = `:small_red_triangle: ${runInfo.failing} Failing | :thumbs_up_green: ${runInfo.passing} Passing | :small_blue_diamond: ${runInfo.skipped} Skipped\n\n`; diff --git a/scripts/junit-summary/index.ts b/scripts/junit-summary/index.ts index acc02fb1677..5808cbac6df 100644 --- a/scripts/junit-summary/index.ts +++ b/scripts/junit-summary/index.ts @@ -24,6 +24,7 @@ program .version('0.1.0') .arguments('') .option('-f, --format ', 'JUnit summary output format', 'json') + .option('--meta:title ', 'Pipeline title') .option('--meta:author-name ', 'Author name') .option('--meta:author-slack ', 'Author Slack name') .option('--meta:author-github ', 'Author GitHub name') diff --git a/scripts/junit-summary/metadata/metadata.ts b/scripts/junit-summary/metadata/metadata.ts index 12d234aa361..79d01d8ec75 100644 --- a/scripts/junit-summary/metadata/metadata.ts +++ b/scripts/junit-summary/metadata/metadata.ts @@ -3,6 +3,9 @@ */ export interface Metadata { + /** Job or pipeline title. */ + pipelineTitle?: string; + /** Code author name. */ authorName?: string; From 34d0c7b6dfa456e45ddb3b7b54ff3723a9bc7968 Mon Sep 17 00:00:00 2001 From: Joe D'Amore Date: Thu, 10 Oct 2024 19:00:21 -0400 Subject: [PATCH 07/25] Allow arbitrary extra information to be displayed in Slack/GitHub notifications --- scripts/junit-summary/formatters/github-formatter.ts | 3 +++ scripts/junit-summary/formatters/slack-formatter.ts | 6 ++++++ scripts/junit-summary/index.ts | 1 + scripts/junit-summary/metadata/metadata.ts | 3 +++ 4 files changed, 13 insertions(+) diff --git a/scripts/junit-summary/formatters/github-formatter.ts b/scripts/junit-summary/formatters/github-formatter.ts index c1161a61cf0..d039db83bed 100644 --- a/scripts/junit-summary/formatters/github-formatter.ts +++ b/scripts/junit-summary/formatters/github-formatter.ts @@ -39,6 +39,8 @@ export const githubFormatter: Formatter = ( const breakdown = `:x: ${runInfo.failing} Failing | :green_heart: ${runInfo.passing} Passing | :arrow_right_hook: ${runInfo.skipped} Skipped | :clock1: ${secondsToTimeString(runInfo.time)}\n\n`; + const extra = metadata.extra ? `${metadata.extra}\n\n` : null; + const failedTestSummary = (() => { const heading = `### Details`; const failedTestHeader = ``; @@ -82,6 +84,7 @@ export const githubFormatter: Formatter = ( headline, '', breakdown, + extra, runInfo.failing > 0 ? failedTestSummary : null, runInfo.failing > 0 ? rerunNote : null, ] diff --git a/scripts/junit-summary/formatters/slack-formatter.ts b/scripts/junit-summary/formatters/slack-formatter.ts index 2d11447105c..6ce9abdb5fe 100644 --- a/scripts/junit-summary/formatters/slack-formatter.ts +++ b/scripts/junit-summary/formatters/slack-formatter.ts @@ -88,6 +88,8 @@ export const slackFormatter: Formatter = ( return `${rerunTip}\n${cypressCommand}`; })(); + const extra = metadata.extra ? `${metadata.extra}\n` : null; + // Display test run details (author, PR number, run number, etc.) when applicable. const footer = (() => { const authorIdentifier = (metadata.authorSlack ? `@${metadata.authorSlack}` : null) @@ -117,6 +119,10 @@ export const slackFormatter: Formatter = ( runInfo.failing > 0 ? `${failedTestSummary}\n` : null, runInfo.failing > 0 ? `${rerunNote}\n` : null, + // If extra information has been supplied, + // display it above the footer. + extra, + // Show run details footer. `:cypress: ${footer}`, ].filter((item) => item !== null).join('\n'); diff --git a/scripts/junit-summary/index.ts b/scripts/junit-summary/index.ts index 5808cbac6df..30c72881930 100644 --- a/scripts/junit-summary/index.ts +++ b/scripts/junit-summary/index.ts @@ -37,6 +37,7 @@ program .option('--meta:artifacts-url ', 'CI artifacts URL') .option('--meta:results-url ', 'CI results URL') .option('--meta:rerun-url ', 'CI rerun URL') + .option('--meta:extra ', 'Extra information to display in output') .action((junitPath) => { return main(junitPath); }); diff --git a/scripts/junit-summary/metadata/metadata.ts b/scripts/junit-summary/metadata/metadata.ts index 79d01d8ec75..add4f54fdce 100644 --- a/scripts/junit-summary/metadata/metadata.ts +++ b/scripts/junit-summary/metadata/metadata.ts @@ -41,4 +41,7 @@ export interface Metadata { /** CI rerun trigger URL. */ rerunUrl?: string; + + /** Arbitrary extra information that can be added to output. */ + extra: string; } From 6b46173c449226d25434f29c077f3f3a22510cb1 Mon Sep 17 00:00:00 2001 From: Joe D'Amore Date: Thu, 10 Oct 2024 20:04:51 -0400 Subject: [PATCH 08/25] Improve formatting wrt PR titles and run IDs --- .../formatters/slack-formatter.ts | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/scripts/junit-summary/formatters/slack-formatter.ts b/scripts/junit-summary/formatters/slack-formatter.ts index 6ce9abdb5fe..b43db1df570 100644 --- a/scripts/junit-summary/formatters/slack-formatter.ts +++ b/scripts/junit-summary/formatters/slack-formatter.ts @@ -23,28 +23,30 @@ export const slackFormatter: Formatter = ( _junitData: TestSuites[] ) => { const indicator = runInfo.failing ? ':x-mark:' : ':check-mark:'; - const runTitle = metadata.pipelineTitle ?? 'Cypress test results'; + const headline = metadata.pipelineTitle + ? `*${metadata.pipelineTitle}*\n` + : '*Cypress test results*\n'; - const headline = (metadata.runId && metadata.runUrl) - ? `*${runTitle}* (<${metadata.runUrl}|#${metadata.runId}>)\n` - : `*${runTitle}*\n`; + const prInfo = (metadata.changeId && metadata.changeUrl && metadata.changeTitle) + ? `:pull-request: _${metadata.changeTitle}_ (<${metadata.changeUrl}|#${metadata.changeId}>)\n` + : null; const breakdown = `:small_red_triangle: ${runInfo.failing} Failing | :thumbs_up_green: ${runInfo.passing} Passing | :small_blue_diamond: ${runInfo.skipped} Skipped\n\n`; // Show a human-readable summary of what was tested and whether it succeeded. const summary = (() => { - const info = !runInfo.failing + const statusInfo = !runInfo.failing ? `> ${indicator} ${runInfo.passing} passing ${pluralize(runInfo.passing, 'test', 'tests')}` : `> ${indicator} ${runInfo.failing} failed ${pluralize(runInfo.failing, 'test', 'tests')}`; - const prInfo = (metadata.changeId && metadata.changeUrl) - ? ` on PR <${metadata.changeUrl}|#${metadata.changeId}>${metadata.changeTitle ? ` - _${metadata.changeTitle}_` : ''}` + const buildInfo = (metadata.runId && metadata.runUrl) + ? `on run <${metadata.runUrl}|#${metadata.runId}>` : ''; const runLength = `(${secondsToTimeString(runInfo.time)})`; const endingPunctuation = !runInfo.failing ? '.' : ':'; - return `${info}${prInfo} ${runLength}${endingPunctuation}` + return `${statusInfo}${buildInfo} ${runLength}${endingPunctuation}` })(); // Display a list of failed tests and collection of actions when applicable. @@ -108,6 +110,7 @@ export const slackFormatter: Formatter = ( return [ headline, + prInfo, breakdown, summary, @@ -119,8 +122,7 @@ export const slackFormatter: Formatter = ( runInfo.failing > 0 ? `${failedTestSummary}\n` : null, runInfo.failing > 0 ? `${rerunNote}\n` : null, - // If extra information has been supplied, - // display it above the footer. + // If extra information has been supplied, display it above the footer. extra, // Show run details footer. From 19a2284e0d70b949eefe3e644b610655d635449a Mon Sep 17 00:00:00 2001 From: Joe D'Amore Date: Thu, 10 Oct 2024 20:07:54 -0400 Subject: [PATCH 09/25] Temporarily force Cypress tests to fail at process level --- packages/manager/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/manager/package.json b/packages/manager/package.json index bd7aa7e825d..5a87b6316a8 100644 --- a/packages/manager/package.json +++ b/packages/manager/package.json @@ -98,8 +98,8 @@ "storybook": "storybook dev -p 6006", "storybook-static": "storybook build -c .storybook -o .out", "build-storybook": "storybook build", - "cy:run": "cypress run -b chrome", - "cy:e2e": "cypress run --headless -b chrome", + "cy:run": "false", + "cy:e2e": "false", "cy:debug": "cypress open --e2e", "cy:component": "cypress open --component", "cy:component:run": "cypress run --component --headless -b chrome", From d0814b6fcc7b28077494e8ada6473bf6bea27ead Mon Sep 17 00:00:00 2001 From: Joe D'Amore Date: Thu, 10 Oct 2024 20:25:13 -0400 Subject: [PATCH 10/25] Only show up to six failures in a Slack notification --- .../junit-summary/formatters/slack-formatter.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/scripts/junit-summary/formatters/slack-formatter.ts b/scripts/junit-summary/formatters/slack-formatter.ts index b43db1df570..5562bc175d7 100644 --- a/scripts/junit-summary/formatters/slack-formatter.ts +++ b/scripts/junit-summary/formatters/slack-formatter.ts @@ -8,6 +8,14 @@ import { secondsToTimeString } from '../util'; import * as path from 'path'; import { cypressRunCommand } from '../util/cypress'; +/** + * The maximum number of failures that will be listed in the Slack notification. + * + * The Slack notification has a maximum character limit, so we must truncate + * the failure results to reduce the risk of hitting that limit. + */ +const FAILURE_SUMMARY_LIMIT = 6; + /** * Outputs test result summary formatted as a Slack message. * @@ -53,11 +61,16 @@ export const slackFormatter: Formatter = ( const failedTestSummary = (() => { const failedTestLines = results .filter((result: TestResult) => result.failing) + .slice(0, FAILURE_SUMMARY_LIMIT) .map((result: TestResult) => { const specFile = path.basename(result.testFilename); return `• \`${specFile}\` — _${result.groupName}_ » _${result.testName}_`; }); + const truncationNote = (runInfo.failing > FAILURE_SUMMARY_LIMIT) + ? `and ${runInfo.failing - FAILURE_SUMMARY_LIMIT} more...\n` + : null; + // When applicable, display actions that can be taken by the user. const failedTestActions = [ metadata.resultsUrl ? `<${metadata.resultsUrl}|View results>` : '', @@ -70,6 +83,7 @@ export const slackFormatter: Formatter = ( return [ '', ...failedTestLines, + truncationNote, '', failedTestActions ? failedTestActions : null, ] From 0fcde2c14fdff5dac287a8c8cf98d750595ed540 Mon Sep 17 00:00:00 2001 From: Joe D'Amore Date: Thu, 10 Oct 2024 20:28:23 -0400 Subject: [PATCH 11/25] Slightly improve truncation note --- scripts/junit-summary/formatters/slack-formatter.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/junit-summary/formatters/slack-formatter.ts b/scripts/junit-summary/formatters/slack-formatter.ts index 5562bc175d7..a9336d2e70c 100644 --- a/scripts/junit-summary/formatters/slack-formatter.ts +++ b/scripts/junit-summary/formatters/slack-formatter.ts @@ -67,8 +67,9 @@ export const slackFormatter: Formatter = ( return `• \`${specFile}\` — _${result.groupName}_ » _${result.testName}_`; }); + const remainingFailures = runInfo.failing - FAILURE_SUMMARY_LIMIT; const truncationNote = (runInfo.failing > FAILURE_SUMMARY_LIMIT) - ? `and ${runInfo.failing - FAILURE_SUMMARY_LIMIT} more...\n` + ? `and ${remainingFailures} more ${pluralize(remainingFailures, 'failure', 'failures')}...\n` : null; // When applicable, display actions that can be taken by the user. From 05e5a7b4a6db8b0cf877d92122e4c16cdd460948 Mon Sep 17 00:00:00 2001 From: Joe D'Amore Date: Thu, 10 Oct 2024 20:32:56 -0400 Subject: [PATCH 12/25] Fix missing space --- scripts/junit-summary/formatters/slack-formatter.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/junit-summary/formatters/slack-formatter.ts b/scripts/junit-summary/formatters/slack-formatter.ts index a9336d2e70c..af4e8e8d74e 100644 --- a/scripts/junit-summary/formatters/slack-formatter.ts +++ b/scripts/junit-summary/formatters/slack-formatter.ts @@ -48,7 +48,7 @@ export const slackFormatter: Formatter = ( : `> ${indicator} ${runInfo.failing} failed ${pluralize(runInfo.failing, 'test', 'tests')}`; const buildInfo = (metadata.runId && metadata.runUrl) - ? `on run <${metadata.runUrl}|#${metadata.runId}>` + ? ` on run <${metadata.runUrl}|#${metadata.runId}>` : ''; const runLength = `(${secondsToTimeString(runInfo.time)})`; From dd60b2d508afe95c8d8e706b024064066b2d5cfb Mon Sep 17 00:00:00 2001 From: Joe D'Amore Date: Thu, 10 Oct 2024 21:22:36 -0400 Subject: [PATCH 13/25] Allow JUnit summary when there are no JUnit files in given path --- scripts/junit-summary/index.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scripts/junit-summary/index.ts b/scripts/junit-summary/index.ts index 30c72881930..b697b420c8c 100644 --- a/scripts/junit-summary/index.ts +++ b/scripts/junit-summary/index.ts @@ -78,10 +78,6 @@ const main = async (junitPath: string) => { return path.resolve(reportPath, dirItem); }); - if (reportFiles.length < 1) { - throw new Error(`No JUnit report files found in '${reportPath}'.`) - } - // Read JUnit report file contents. const loadReportFileContents = reportFiles.map((reportFile: string) => { return fs.readFile(reportFile, 'utf8'); From 756295c4c6c605bb905238c487fd2963e6abf37a Mon Sep 17 00:00:00 2001 From: Joe D'Amore Date: Thu, 10 Oct 2024 21:23:13 -0400 Subject: [PATCH 14/25] Pass extra info to metadata object --- scripts/junit-summary/index.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/junit-summary/index.ts b/scripts/junit-summary/index.ts index b697b420c8c..b0f983bd5c4 100644 --- a/scripts/junit-summary/index.ts +++ b/scripts/junit-summary/index.ts @@ -67,6 +67,8 @@ const main = async (junitPath: string) => { artifactsUrl: program.opts()['meta:artifactsUrl'], resultsUrl: program.opts()['meta:resultsUrl'], rerunUrl: program.opts()['meta:rerunUrl'], + + extra: program.opts()['meta:extra'], }; // Create an array of absolute file paths to JUnit XML report files. From a4e7800cea2d29aeead349ea0d430938364c89ff Mon Sep 17 00:00:00 2001 From: Joe D'Amore Date: Thu, 10 Oct 2024 21:30:36 -0400 Subject: [PATCH 15/25] Restore package scripts --- packages/manager/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/manager/package.json b/packages/manager/package.json index 5a87b6316a8..bd7aa7e825d 100644 --- a/packages/manager/package.json +++ b/packages/manager/package.json @@ -98,8 +98,8 @@ "storybook": "storybook dev -p 6006", "storybook-static": "storybook build -c .storybook -o .out", "build-storybook": "storybook build", - "cy:run": "false", - "cy:e2e": "false", + "cy:run": "cypress run -b chrome", + "cy:e2e": "cypress run --headless -b chrome", "cy:debug": "cypress open --e2e", "cy:component": "cypress open --component", "cy:component:run": "cypress run --component --headless -b chrome", From ff4144366fc075d2546a47f28bfdc78538e2783c Mon Sep 17 00:00:00 2001 From: Joe D'Amore Date: Thu, 10 Oct 2024 21:38:23 -0400 Subject: [PATCH 16/25] Improve PR title formatting --- scripts/junit-summary/formatters/slack-formatter.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/junit-summary/formatters/slack-formatter.ts b/scripts/junit-summary/formatters/slack-formatter.ts index af4e8e8d74e..65b2b3dda1f 100644 --- a/scripts/junit-summary/formatters/slack-formatter.ts +++ b/scripts/junit-summary/formatters/slack-formatter.ts @@ -36,7 +36,7 @@ export const slackFormatter: Formatter = ( : '*Cypress test results*\n'; const prInfo = (metadata.changeId && metadata.changeUrl && metadata.changeTitle) - ? `:pull-request: _${metadata.changeTitle}_ (<${metadata.changeUrl}|#${metadata.changeId}>)\n` + ? `:pull-request: ${metadata.changeTitle} (<${metadata.changeUrl}|#${metadata.changeId}>)\n` : null; const breakdown = `:small_red_triangle: ${runInfo.failing} Failing | :thumbs_up_green: ${runInfo.passing} Passing | :small_blue_diamond: ${runInfo.skipped} Skipped\n\n`; From 94c0d8066898b8d26cca6c84c754400065c01372 Mon Sep 17 00:00:00 2001 From: Joe D'Amore Date: Thu, 10 Oct 2024 23:08:05 -0400 Subject: [PATCH 17/25] Pass pipeline title CLI arg to formatter --- scripts/junit-summary/index.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/junit-summary/index.ts b/scripts/junit-summary/index.ts index b0f983bd5c4..a2d9cbd2d23 100644 --- a/scripts/junit-summary/index.ts +++ b/scripts/junit-summary/index.ts @@ -51,6 +51,8 @@ const main = async (junitPath: string) => { const reportPath = path.resolve(junitPath); const summaryFormat = program.opts().format; const metadata: Metadata = { + pipelineTitle: program.opts()['meta:title'], + authorName: program.opts()['meta:authorName'], authorSlack: program.opts()['meta:authorSlack'], authorGitHub: program.opts()['meta:auhtorGithub'], From d5ec24d56296c0a1b163df5a4f2e922e11c80485 Mon Sep 17 00:00:00 2001 From: Joe D'Amore Date: Fri, 11 Oct 2024 09:36:21 -0400 Subject: [PATCH 18/25] Make `extra` metadata field optional --- scripts/junit-summary/metadata/metadata.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/junit-summary/metadata/metadata.ts b/scripts/junit-summary/metadata/metadata.ts index add4f54fdce..35cde6fd38b 100644 --- a/scripts/junit-summary/metadata/metadata.ts +++ b/scripts/junit-summary/metadata/metadata.ts @@ -43,5 +43,5 @@ export interface Metadata { rerunUrl?: string; /** Arbitrary extra information that can be added to output. */ - extra: string; + extra?: string; } From 87f4242b91161a4dec45d88460c2627045350f81 Mon Sep 17 00:00:00 2001 From: Joe D'Amore Date: Wed, 16 Oct 2024 16:07:16 -0400 Subject: [PATCH 19/25] Move LaunchDarkly URL matchers to constants file --- .../cypress/support/constants/feature-flags.ts | 11 +++++++++++ .../cypress/support/intercepts/feature-flags.ts | 12 ++++-------- 2 files changed, 15 insertions(+), 8 deletions(-) create mode 100644 packages/manager/cypress/support/constants/feature-flags.ts diff --git a/packages/manager/cypress/support/constants/feature-flags.ts b/packages/manager/cypress/support/constants/feature-flags.ts new file mode 100644 index 00000000000..08e996ca15d --- /dev/null +++ b/packages/manager/cypress/support/constants/feature-flags.ts @@ -0,0 +1,11 @@ +/** + * @file Constants related to Cypress's handling of LaunchDarkly feature flags. + */ + +// LaunchDarkly URL pattern for feature flag retrieval. +export const launchDarklyUrlPattern = + 'https://app.launchdarkly.com/sdk/evalx/*/contexts/*'; + +// LaunchDarkly URL pattern for feature flag / event streaming. +export const launchDarklyClientstreamPattern = + 'https://clientstream.launchdarkly.com/eval/*/*'; diff --git a/packages/manager/cypress/support/intercepts/feature-flags.ts b/packages/manager/cypress/support/intercepts/feature-flags.ts index 9bf393efb58..d77ab55f531 100644 --- a/packages/manager/cypress/support/intercepts/feature-flags.ts +++ b/packages/manager/cypress/support/intercepts/feature-flags.ts @@ -3,17 +3,13 @@ */ import { getResponseDataFromMockData } from 'support/util/feature-flags'; +import { + launchDarklyUrlPattern, + launchDarklyClientstreamPattern, +} from 'support/constants/feature-flags'; import type { FeatureFlagMockData } from 'support/util/feature-flags'; -// LaunchDarkly URL pattern for feature flag retrieval. -const launchDarklyUrlPattern = - 'https://app.launchdarkly.com/sdk/evalx/*/contexts/*'; - -// LaunchDarkly URL pattern for feature flag / event streaming. -const launchDarklyClientstreamPattern = - 'https://clientstream.launchdarkly.com/eval/*/*'; - /** * Intercepts GET request to feature flag clientstream URL and mocks the response. * From b39fe0c6c20d349b2913ecff9cb0baa6a9806a3d Mon Sep 17 00:00:00 2001 From: Joe D'Amore Date: Wed, 16 Oct 2024 16:16:56 -0400 Subject: [PATCH 20/25] Allow feature flags to be overridden via CY_TEST_FEATURE_FLAGS environment variable --- docs/development-guide/08-testing.md | 11 +++--- packages/manager/cypress.config.ts | 2 + packages/manager/cypress/support/e2e.ts | 2 + .../support/plugins/feature-flag-override.ts | 39 +++++++++++++++++++ .../setup/mock-feature-flags-request.ts | 37 ++++++++++++++++++ 5 files changed, 86 insertions(+), 5 deletions(-) create mode 100644 packages/manager/cypress/support/plugins/feature-flag-override.ts create mode 100644 packages/manager/cypress/support/setup/mock-feature-flags-request.ts diff --git a/docs/development-guide/08-testing.md b/docs/development-guide/08-testing.md index 5635774c867..9ce93c1169b 100644 --- a/docs/development-guide/08-testing.md +++ b/docs/development-guide/08-testing.md @@ -191,12 +191,13 @@ Environment variables related to the general operation of the Cloud Manager Cypr | `CY_TEST_SUITE` | Name of the Cloud Manager UI test suite to run. Possible values are `core`, `region`, or `synthetic`. | `region` | Unset; defaults to `core` suite | | `CY_TEST_TAGS` | Query identifying tests that should run by specifying allowed and disallowed tags. | `method:e2e` | Unset; all tests run by default | -###### Regions -These environment variables are used by Cloud Manager's UI tests to override region selection behavior. This can be useful for testing Cloud Manager functionality against a specific region. +###### Overriding Behavior +These environment variables can be used to override some behaviors of Cloud Manager's UI tests. This can be useful when testing Cloud Manager for nonstandard or work-in-progress functionality. -| Environment Variable | Description | Example | Default | -|----------------------|-------------------------------------------------|-----------|---------------------------------------| -| `CY_TEST_REGION` | ID of region to test (as used by Linode APIv4). | `us-east` | Unset; regions are selected at random | +| Environment Variable | Description | Example | Default | +|-------------------------|-------------------------------------------------|-----------|--------------------------------------------| +| `CY_TEST_REGION` | ID of region to test (as used by Linode APIv4). | `us-east` | Unset; regions are selected at random | +| `CY_TEST_FEATURE_FLAGS` | JSON string containing feature flag data | `{}` | Unset; feature flag data is not overridden | ###### Run Splitting These environment variables facilitate splitting the Cypress run between multiple runners without the use of any third party services. This can be useful for improving Cypress test performance in some circumstances. For additional performance gains, an optional test weights file can be specified using `CY_TEST_SPLIT_RUN_WEIGHTS` (see `CY_TEST_GENWEIGHTS` to generate test weights). diff --git a/packages/manager/cypress.config.ts b/packages/manager/cypress.config.ts index 09322be48ba..b51562d9f38 100644 --- a/packages/manager/cypress.config.ts +++ b/packages/manager/cypress.config.ts @@ -17,6 +17,7 @@ import { enableJunitReport } from './cypress/support/plugins/junit-report'; import { generateTestWeights } from './cypress/support/plugins/generate-weights'; import { logTestTagInfo } from './cypress/support/plugins/test-tagging-info'; import cypressViteConfig from './cypress/vite.config'; +import { featureFlagOverrides } from './cypress/support/plugins/feature-flag-override'; /** * Exports a Cypress configuration object. @@ -91,6 +92,7 @@ export default defineConfig({ fetchAccount, fetchLinodeRegions, regionOverrideCheck, + featureFlagOverrides, logTestTagInfo, splitCypressRun, enableJunitReport(), diff --git a/packages/manager/cypress/support/e2e.ts b/packages/manager/cypress/support/e2e.ts index 2e21f7ca60b..5996d3d71aa 100644 --- a/packages/manager/cypress/support/e2e.ts +++ b/packages/manager/cypress/support/e2e.ts @@ -60,11 +60,13 @@ chai.use(function (chai, utils) { // Test setup. import { deleteInternalHeader } from './setup/delete-internal-header'; +import { mockFeatureFlagRequests } from './setup/mock-feature-flags-request'; import { mockFeatureFlagClientstream } from './setup/feature-flag-clientstream'; import { mockAccountRequest } from './setup/mock-account-request'; import { trackApiRequests } from './setup/request-tracking'; trackApiRequests(); mockAccountRequest(); +mockFeatureFlagRequests(); mockFeatureFlagClientstream(); deleteInternalHeader(); diff --git a/packages/manager/cypress/support/plugins/feature-flag-override.ts b/packages/manager/cypress/support/plugins/feature-flag-override.ts new file mode 100644 index 00000000000..b8694cc9ff9 --- /dev/null +++ b/packages/manager/cypress/support/plugins/feature-flag-override.ts @@ -0,0 +1,39 @@ +import type { CypressPlugin } from './plugin'; + +/** + * Handles setup related to Launch Darkly feature flag overrides. + * + * Checks if the user has passed overrides via the `CY_TEST_FEATURE_FLAGS` env, + * and validates its value if so by attempting to parse it as JSON. If that + * succeeds, the parsed override object is exposed to Cypress via the + * `featureFlagOverrides` config. + */ +export const featureFlagOverrides: CypressPlugin = (_on, config) => { + const featureFlagOverridesJson = config.env?.['CY_TEST_FEATURE_FLAGS']; + + let featureFlagOverrides = undefined; + if (featureFlagOverridesJson) { + const notice = + 'Feature flag overrides are enabled with the following JSON payload:'; + const jsonWarning = + 'Be aware that malformed or invalid feature flag data can trigger crashes and other unexpected behavior.'; + + console.info(`${notice}\n\n${featureFlagOverridesJson}\n\n${jsonWarning}`); + + try { + featureFlagOverrides = JSON.parse(featureFlagOverridesJson); + } catch (e) { + throw new Error( + `Unable to parse feature flag JSON:\n\n${featureFlagOverridesJson}\n\nPlease double check your 'CY_TEST_FEATURE_FLAGS' value and try again.` + ); + } + } + + return { + ...config, + env: { + ...config.env, + featureFlagOverrides, + }, + }; +}; diff --git a/packages/manager/cypress/support/setup/mock-feature-flags-request.ts b/packages/manager/cypress/support/setup/mock-feature-flags-request.ts new file mode 100644 index 00000000000..2f45f33f0cf --- /dev/null +++ b/packages/manager/cypress/support/setup/mock-feature-flags-request.ts @@ -0,0 +1,37 @@ +/** + * @file Intercepts and mocks Launch Darkly feature flag requests with override data if specified. + */ + +import { launchDarklyUrlPattern } from 'support/constants/feature-flags'; + +/** + * If feature flag overrides have been specified, intercept every LaunchDarkly + * feature flag request and modify the response to contain the override data. + * + * This override happens before other intercepts and mocks (e.g. via `mockGetFeatureFlags` + * and `mockAppendFeatureFlags`), so mocks set up by those functions will take + * priority in the event that both modify the same feature flag value. + */ +export const mockFeatureFlagRequests = () => { + const featureFlagOverrides = Cypress.env('featureFlagOverrides'); + + if (featureFlagOverrides) { + beforeEach(() => { + cy.intercept( + { + middleware: true, + url: launchDarklyUrlPattern, + }, + (req) => { + req.on('before:response', (res) => { + const overriddenFeatureFlagData = { + ...res.body, + ...featureFlagOverrides, + }; + res.body = overriddenFeatureFlagData; + }); + } + ); + }); + } +}; From 40b9bb5f67a81a5bd54c814f4d579591e8888ad6 Mon Sep 17 00:00:00 2001 From: Joe D'Amore Date: Wed, 16 Oct 2024 16:26:45 -0400 Subject: [PATCH 21/25] Pass CY_TEST_FEATURE_FLAGS to Docker containers --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index c68db725dfb..37555efedbf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,6 +35,7 @@ x-e2e-env: # Cloud Manager-specific test configuration. CY_TEST_SUITE: ${CY_TEST_SUITE} CY_TEST_REGION: ${CY_TEST_REGION} + CY_TEST_FEATURE_FLAGS: ${CY_TEST_FEATURE_FLAGS} CY_TEST_TAGS: ${CY_TEST_TAGS} CY_TEST_DISABLE_RETRIES: ${CY_TEST_DISABLE_RETRIES} From 2cfc7ecfca9bf3251babdb0d665c1ebcb6c512af Mon Sep 17 00:00:00 2001 From: Joe D'Amore Date: Thu, 17 Oct 2024 13:47:49 -0400 Subject: [PATCH 22/25] Temporarily force test failure while debugging Slack notification logic --- .../manager/cypress/e2e/core/billing/billing-contact.spec.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/manager/cypress/e2e/core/billing/billing-contact.spec.ts b/packages/manager/cypress/e2e/core/billing/billing-contact.spec.ts index 346023aa089..d3620c035ab 100644 --- a/packages/manager/cypress/e2e/core/billing/billing-contact.spec.ts +++ b/packages/manager/cypress/e2e/core/billing/billing-contact.spec.ts @@ -73,6 +73,7 @@ describe('Billing Contact', () => { }); }); it('Edit Contact Info', () => { + throw new Error('Forcing this to fail temporarily sorry'); // mock the user's account data and confirm that it is displayed correctly upon page load mockGetAccount(accountData).as('getAccount'); cy.visitWithLogin('/account/billing'); From 887be4094c51b3e92cb4e231628b1fa2f6e8590f Mon Sep 17 00:00:00 2001 From: Joe D'Amore Date: Thu, 17 Oct 2024 14:02:37 -0400 Subject: [PATCH 23/25] Remove forced test failure --- .../manager/cypress/e2e/core/billing/billing-contact.spec.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/manager/cypress/e2e/core/billing/billing-contact.spec.ts b/packages/manager/cypress/e2e/core/billing/billing-contact.spec.ts index d3620c035ab..346023aa089 100644 --- a/packages/manager/cypress/e2e/core/billing/billing-contact.spec.ts +++ b/packages/manager/cypress/e2e/core/billing/billing-contact.spec.ts @@ -73,7 +73,6 @@ describe('Billing Contact', () => { }); }); it('Edit Contact Info', () => { - throw new Error('Forcing this to fail temporarily sorry'); // mock the user's account data and confirm that it is displayed correctly upon page load mockGetAccount(accountData).as('getAccount'); cy.visitWithLogin('/account/billing'); From b003b5e47a815af73cde27dcfa5d3b16f0ac15cf Mon Sep 17 00:00:00 2001 From: Joe D'Amore Date: Mon, 21 Oct 2024 10:04:11 -0400 Subject: [PATCH 24/25] Comment clean up --- docker-compose.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 37555efedbf..36bbe54ef43 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -81,13 +81,8 @@ x-e2e-runners: context: . dockerfile: ./packages/manager/Dockerfile target: e2e - # depends_on: - # web: - # condition: service_healthy env_file: ./packages/manager/.env volumes: *default-volumes - # TODO Stop using entrypoint, use CMD instead. - # (Or just make `yarn` the entrypoint, but either way stop forcing `cy:e2e`). entrypoint: 'yarn' services: From a6f1782e523698ee08d1d4f57658c18cfa49546a Mon Sep 17 00:00:00 2001 From: Joe D'Amore Date: Mon, 21 Oct 2024 14:26:47 -0400 Subject: [PATCH 25/25] Add changesets --- .../.changeset/pr-11088-tech-stories-1729535071709.md | 5 +++++ packages/manager/.changeset/pr-11088-tests-1729535093463.md | 5 +++++ packages/manager/.changeset/pr-11088-tests-1729535165205.md | 5 +++++ packages/manager/.changeset/pr-11088-tests-1729535197632.md | 5 +++++ 4 files changed, 20 insertions(+) create mode 100644 packages/manager/.changeset/pr-11088-tech-stories-1729535071709.md create mode 100644 packages/manager/.changeset/pr-11088-tests-1729535093463.md create mode 100644 packages/manager/.changeset/pr-11088-tests-1729535165205.md create mode 100644 packages/manager/.changeset/pr-11088-tests-1729535197632.md diff --git a/packages/manager/.changeset/pr-11088-tech-stories-1729535071709.md b/packages/manager/.changeset/pr-11088-tech-stories-1729535071709.md new file mode 100644 index 00000000000..037e8d3821f --- /dev/null +++ b/packages/manager/.changeset/pr-11088-tech-stories-1729535071709.md @@ -0,0 +1,5 @@ +--- +"@linode/manager": Tech Stories +--- + +Replace 'e2e', 'e2e_heimdall', and 'component' Docker Compose services with 'cypress_local', 'cypress_remote', and 'cypress_component' ([#11088](https://github.com/linode/manager/pull/11088)) diff --git a/packages/manager/.changeset/pr-11088-tests-1729535093463.md b/packages/manager/.changeset/pr-11088-tests-1729535093463.md new file mode 100644 index 00000000000..2dc798af453 --- /dev/null +++ b/packages/manager/.changeset/pr-11088-tests-1729535093463.md @@ -0,0 +1,5 @@ +--- +"@linode/manager": Tests +--- + +Allow overriding feature flags via CY_TEST_FEATURE_FLAGS environment variable ([#11088](https://github.com/linode/manager/pull/11088)) diff --git a/packages/manager/.changeset/pr-11088-tests-1729535165205.md b/packages/manager/.changeset/pr-11088-tests-1729535165205.md new file mode 100644 index 00000000000..a895a08ccd5 --- /dev/null +++ b/packages/manager/.changeset/pr-11088-tests-1729535165205.md @@ -0,0 +1,5 @@ +--- +"@linode/manager": Tests +--- + +Allow pipeline Slack notifications to be customized ([#11088](https://github.com/linode/manager/pull/11088)) diff --git a/packages/manager/.changeset/pr-11088-tests-1729535197632.md b/packages/manager/.changeset/pr-11088-tests-1729535197632.md new file mode 100644 index 00000000000..24c079ce5dc --- /dev/null +++ b/packages/manager/.changeset/pr-11088-tests-1729535197632.md @@ -0,0 +1,5 @@ +--- +"@linode/manager": Tests +--- + +Show PR title in Slack CI notifications ([#11088](https://github.com/linode/manager/pull/11088))
Failing Tests
SpecTest