diff --git a/docker-compose.yml b/docker-compose.yml index 494f64db0de..36bbe54ef43 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,6 +35,7 @@ x-e2e-env: # Cloud Manager-specific test configuration. CY_TEST_SUITE: ${CY_TEST_SUITE} CY_TEST_REGION: ${CY_TEST_REGION} + CY_TEST_FEATURE_FLAGS: ${CY_TEST_FEATURE_FLAGS} CY_TEST_TAGS: ${CY_TEST_TAGS} CY_TEST_DISABLE_RETRIES: ${CY_TEST_DISABLE_RETRIES} @@ -80,14 +81,9 @@ x-e2e-runners: context: . dockerfile: ./packages/manager/Dockerfile target: e2e - depends_on: - web: - condition: service_healthy env_file: ./packages/manager/.env volumes: *default-volumes - # TODO Stop using entrypoint, use CMD instead. - # (Or just make `yarn` the entrypoint, but either way stop forcing `cy:e2e`). - entrypoint: ['yarn', 'cy:e2e'] + entrypoint: 'yarn' services: # Serves a local instance of Cloud Manager for Cypress to use for its tests. @@ -110,6 +106,48 @@ services: timeout: 10s retries: 10 + # Cypress test runner service to run tests against a remotely-served Cloud instance. + # + # This is useful when testing against a standard Cloud Manager environment, + # like Production at cloud.linode.com, but can also be used to run tests against + # pre-Prod environments, PR preview links, and more. + cypress_remote: + <<: *default-runner + environment: + <<: *default-env + MANAGER_OAUTH: ${MANAGER_OAUTH} + + # Cypress test runner service to run tests against a locally-served Cloud instance. + # + # This is useful when testing against a customized or in-development build of + # Cloud Manager. + cypress_local: + <<: *default-runner + environment: + <<: *default-env + MANAGER_OAUTH: ${MANAGER_OAUTH} + depends_on: + web: + condition: service_healthy + + # Cypress component test runner service. + # + # Unlike other Cloud Manager Cypress tests, these tests can be run without + # requiring a Cloud Manager environment. + cypress_component: + <<: *default-runner + environment: + CY_TEST_DISABLE_RETRIES: ${CY_TEST_DISABLE_RETRIES} + CY_TEST_JUNIT_REPORT: ${CY_TEST_JUNIT_REPORT} + + + # --> ! DEPRECATION NOTICE ! <-- + # The services below this line are deprecated, and will be deleted soon. + # Don't build any pipelines or write any scripts that depend on these. + # Instead, opt to use `cypress_local` in places where you would've used `e2e`, + # use `cypress_remote` in places where you would've used `e2e_heimdall`, and + # use `cypress_component` in places where you would've used `component`. + # Generic end-to-end test runner for Cloud's primary testing pipeline. # Configured to run against a local Cloud instance. e2e: @@ -117,6 +155,7 @@ services: environment: <<: *default-env MANAGER_OAUTH: ${MANAGER_OAUTH} + entrypoint: ['yarn', 'cy:e2e'] # Component test runner. # Does not require any Cloud Manager environment to run. @@ -136,14 +175,4 @@ services: environment: <<: *default-env MANAGER_OAUTH: ${MANAGER_OAUTH} - - region-1: - build: - context: . - dockerfile: ./packages/manager/Dockerfile - target: e2e - env_file: ./packages/manager/.env - volumes: *default-volumes - environment: - <<: *default-env - MANAGER_OAUTH: ${MANAGER_OAUTH_1} + entrypoint: ['yarn', 'cy:e2e'] diff --git a/docs/development-guide/08-testing.md b/docs/development-guide/08-testing.md index 5635774c867..9ce93c1169b 100644 --- a/docs/development-guide/08-testing.md +++ b/docs/development-guide/08-testing.md @@ -191,12 +191,13 @@ Environment variables related to the general operation of the Cloud Manager Cypr | `CY_TEST_SUITE` | Name of the Cloud Manager UI test suite to run. Possible values are `core`, `region`, or `synthetic`. | `region` | Unset; defaults to `core` suite | | `CY_TEST_TAGS` | Query identifying tests that should run by specifying allowed and disallowed tags. | `method:e2e` | Unset; all tests run by default | -###### Regions -These environment variables are used by Cloud Manager's UI tests to override region selection behavior. This can be useful for testing Cloud Manager functionality against a specific region. +###### Overriding Behavior +These environment variables can be used to override some behaviors of Cloud Manager's UI tests. This can be useful when testing Cloud Manager for nonstandard or work-in-progress functionality. -| Environment Variable | Description | Example | Default | -|----------------------|-------------------------------------------------|-----------|---------------------------------------| -| `CY_TEST_REGION` | ID of region to test (as used by Linode APIv4). | `us-east` | Unset; regions are selected at random | +| Environment Variable | Description | Example | Default | +|-------------------------|-------------------------------------------------|-----------|--------------------------------------------| +| `CY_TEST_REGION` | ID of region to test (as used by Linode APIv4). | `us-east` | Unset; regions are selected at random | +| `CY_TEST_FEATURE_FLAGS` | JSON string containing feature flag data | `{}` | Unset; feature flag data is not overridden | ###### Run Splitting These environment variables facilitate splitting the Cypress run between multiple runners without the use of any third party services. This can be useful for improving Cypress test performance in some circumstances. For additional performance gains, an optional test weights file can be specified using `CY_TEST_SPLIT_RUN_WEIGHTS` (see `CY_TEST_GENWEIGHTS` to generate test weights). diff --git a/packages/manager/.changeset/pr-11088-tech-stories-1729535071709.md b/packages/manager/.changeset/pr-11088-tech-stories-1729535071709.md new file mode 100644 index 00000000000..037e8d3821f --- /dev/null +++ b/packages/manager/.changeset/pr-11088-tech-stories-1729535071709.md @@ -0,0 +1,5 @@ +--- +"@linode/manager": Tech Stories +--- + +Replace 'e2e', 'e2e_heimdall', and 'component' Docker Compose services with 'cypress_local', 'cypress_remote', and 'cypress_component' ([#11088](https://github.com/linode/manager/pull/11088)) diff --git a/packages/manager/.changeset/pr-11088-tests-1729535093463.md b/packages/manager/.changeset/pr-11088-tests-1729535093463.md new file mode 100644 index 00000000000..2dc798af453 --- /dev/null +++ b/packages/manager/.changeset/pr-11088-tests-1729535093463.md @@ -0,0 +1,5 @@ +--- +"@linode/manager": Tests +--- + +Allow overriding feature flags via CY_TEST_FEATURE_FLAGS environment variable ([#11088](https://github.com/linode/manager/pull/11088)) diff --git a/packages/manager/.changeset/pr-11088-tests-1729535165205.md b/packages/manager/.changeset/pr-11088-tests-1729535165205.md new file mode 100644 index 00000000000..a895a08ccd5 --- /dev/null +++ b/packages/manager/.changeset/pr-11088-tests-1729535165205.md @@ -0,0 +1,5 @@ +--- +"@linode/manager": Tests +--- + +Allow pipeline Slack notifications to be customized ([#11088](https://github.com/linode/manager/pull/11088)) diff --git a/packages/manager/.changeset/pr-11088-tests-1729535197632.md b/packages/manager/.changeset/pr-11088-tests-1729535197632.md new file mode 100644 index 00000000000..24c079ce5dc --- /dev/null +++ b/packages/manager/.changeset/pr-11088-tests-1729535197632.md @@ -0,0 +1,5 @@ +--- +"@linode/manager": Tests +--- + +Show PR title in Slack CI notifications ([#11088](https://github.com/linode/manager/pull/11088)) diff --git a/packages/manager/Dockerfile b/packages/manager/Dockerfile index 2cfa22aca4e..466edaf64ea 100644 --- a/packages/manager/Dockerfile +++ b/packages/manager/Dockerfile @@ -47,4 +47,3 @@ ENV CI=1 ENV NO_COLOR=1 ENV HOME=/home/node/ ENV CYPRESS_CACHE_FOLDER=/home/node/.cache/Cypress -ENTRYPOINT yarn cy:ci diff --git a/packages/manager/cypress.config.ts b/packages/manager/cypress.config.ts index 09322be48ba..b51562d9f38 100644 --- a/packages/manager/cypress.config.ts +++ b/packages/manager/cypress.config.ts @@ -17,6 +17,7 @@ import { enableJunitReport } from './cypress/support/plugins/junit-report'; import { generateTestWeights } from './cypress/support/plugins/generate-weights'; import { logTestTagInfo } from './cypress/support/plugins/test-tagging-info'; import cypressViteConfig from './cypress/vite.config'; +import { featureFlagOverrides } from './cypress/support/plugins/feature-flag-override'; /** * Exports a Cypress configuration object. @@ -91,6 +92,7 @@ export default defineConfig({ fetchAccount, fetchLinodeRegions, regionOverrideCheck, + featureFlagOverrides, logTestTagInfo, splitCypressRun, enableJunitReport(), diff --git a/packages/manager/cypress/support/constants/feature-flags.ts b/packages/manager/cypress/support/constants/feature-flags.ts new file mode 100644 index 00000000000..08e996ca15d --- /dev/null +++ b/packages/manager/cypress/support/constants/feature-flags.ts @@ -0,0 +1,11 @@ +/** + * @file Constants related to Cypress's handling of LaunchDarkly feature flags. + */ + +// LaunchDarkly URL pattern for feature flag retrieval. +export const launchDarklyUrlPattern = + 'https://app.launchdarkly.com/sdk/evalx/*/contexts/*'; + +// LaunchDarkly URL pattern for feature flag / event streaming. +export const launchDarklyClientstreamPattern = + 'https://clientstream.launchdarkly.com/eval/*/*'; diff --git a/packages/manager/cypress/support/e2e.ts b/packages/manager/cypress/support/e2e.ts index 2e21f7ca60b..5996d3d71aa 100644 --- a/packages/manager/cypress/support/e2e.ts +++ b/packages/manager/cypress/support/e2e.ts @@ -60,11 +60,13 @@ chai.use(function (chai, utils) { // Test setup. import { deleteInternalHeader } from './setup/delete-internal-header'; +import { mockFeatureFlagRequests } from './setup/mock-feature-flags-request'; import { mockFeatureFlagClientstream } from './setup/feature-flag-clientstream'; import { mockAccountRequest } from './setup/mock-account-request'; import { trackApiRequests } from './setup/request-tracking'; trackApiRequests(); mockAccountRequest(); +mockFeatureFlagRequests(); mockFeatureFlagClientstream(); deleteInternalHeader(); diff --git a/packages/manager/cypress/support/intercepts/feature-flags.ts b/packages/manager/cypress/support/intercepts/feature-flags.ts index 9bf393efb58..d77ab55f531 100644 --- a/packages/manager/cypress/support/intercepts/feature-flags.ts +++ b/packages/manager/cypress/support/intercepts/feature-flags.ts @@ -3,17 +3,13 @@ */ import { getResponseDataFromMockData } from 'support/util/feature-flags'; +import { + launchDarklyUrlPattern, + launchDarklyClientstreamPattern, +} from 'support/constants/feature-flags'; import type { FeatureFlagMockData } from 'support/util/feature-flags'; -// LaunchDarkly URL pattern for feature flag retrieval. -const launchDarklyUrlPattern = - 'https://app.launchdarkly.com/sdk/evalx/*/contexts/*'; - -// LaunchDarkly URL pattern for feature flag / event streaming. -const launchDarklyClientstreamPattern = - 'https://clientstream.launchdarkly.com/eval/*/*'; - /** * Intercepts GET request to feature flag clientstream URL and mocks the response. * diff --git a/packages/manager/cypress/support/plugins/feature-flag-override.ts b/packages/manager/cypress/support/plugins/feature-flag-override.ts new file mode 100644 index 00000000000..b8694cc9ff9 --- /dev/null +++ b/packages/manager/cypress/support/plugins/feature-flag-override.ts @@ -0,0 +1,39 @@ +import type { CypressPlugin } from './plugin'; + +/** + * Handles setup related to Launch Darkly feature flag overrides. + * + * Checks if the user has passed overrides via the `CY_TEST_FEATURE_FLAGS` env, + * and validates its value if so by attempting to parse it as JSON. If that + * succeeds, the parsed override object is exposed to Cypress via the + * `featureFlagOverrides` config. + */ +export const featureFlagOverrides: CypressPlugin = (_on, config) => { + const featureFlagOverridesJson = config.env?.['CY_TEST_FEATURE_FLAGS']; + + let featureFlagOverrides = undefined; + if (featureFlagOverridesJson) { + const notice = + 'Feature flag overrides are enabled with the following JSON payload:'; + const jsonWarning = + 'Be aware that malformed or invalid feature flag data can trigger crashes and other unexpected behavior.'; + + console.info(`${notice}\n\n${featureFlagOverridesJson}\n\n${jsonWarning}`); + + try { + featureFlagOverrides = JSON.parse(featureFlagOverridesJson); + } catch (e) { + throw new Error( + `Unable to parse feature flag JSON:\n\n${featureFlagOverridesJson}\n\nPlease double check your 'CY_TEST_FEATURE_FLAGS' value and try again.` + ); + } + } + + return { + ...config, + env: { + ...config.env, + featureFlagOverrides, + }, + }; +}; diff --git a/packages/manager/cypress/support/setup/mock-feature-flags-request.ts b/packages/manager/cypress/support/setup/mock-feature-flags-request.ts new file mode 100644 index 00000000000..2f45f33f0cf --- /dev/null +++ b/packages/manager/cypress/support/setup/mock-feature-flags-request.ts @@ -0,0 +1,37 @@ +/** + * @file Intercepts and mocks Launch Darkly feature flag requests with override data if specified. + */ + +import { launchDarklyUrlPattern } from 'support/constants/feature-flags'; + +/** + * If feature flag overrides have been specified, intercept every LaunchDarkly + * feature flag request and modify the response to contain the override data. + * + * This override happens before other intercepts and mocks (e.g. via `mockGetFeatureFlags` + * and `mockAppendFeatureFlags`), so mocks set up by those functions will take + * priority in the event that both modify the same feature flag value. + */ +export const mockFeatureFlagRequests = () => { + const featureFlagOverrides = Cypress.env('featureFlagOverrides'); + + if (featureFlagOverrides) { + beforeEach(() => { + cy.intercept( + { + middleware: true, + url: launchDarklyUrlPattern, + }, + (req) => { + req.on('before:response', (res) => { + const overriddenFeatureFlagData = { + ...res.body, + ...featureFlagOverrides, + }; + res.body = overriddenFeatureFlagData; + }); + } + ); + }); + } +}; diff --git a/scripts/junit-summary/formatters/github-formatter.ts b/scripts/junit-summary/formatters/github-formatter.ts index c1161a61cf0..d039db83bed 100644 --- a/scripts/junit-summary/formatters/github-formatter.ts +++ b/scripts/junit-summary/formatters/github-formatter.ts @@ -39,6 +39,8 @@ export const githubFormatter: Formatter = ( const breakdown = `:x: ${runInfo.failing} Failing | :green_heart: ${runInfo.passing} Passing | :arrow_right_hook: ${runInfo.skipped} Skipped | :clock1: ${secondsToTimeString(runInfo.time)}\n\n`; + const extra = metadata.extra ? `${metadata.extra}\n\n` : null; + const failedTestSummary = (() => { const heading = `### Details`; const failedTestHeader = ``; @@ -82,6 +84,7 @@ export const githubFormatter: Formatter = ( headline, '', breakdown, + extra, runInfo.failing > 0 ? failedTestSummary : null, runInfo.failing > 0 ? rerunNote : null, ] diff --git a/scripts/junit-summary/formatters/slack-formatter.ts b/scripts/junit-summary/formatters/slack-formatter.ts index 4168cd4a9f5..65b2b3dda1f 100644 --- a/scripts/junit-summary/formatters/slack-formatter.ts +++ b/scripts/junit-summary/formatters/slack-formatter.ts @@ -8,6 +8,14 @@ import { secondsToTimeString } from '../util'; import * as path from 'path'; import { cypressRunCommand } from '../util/cypress'; +/** + * The maximum number of failures that will be listed in the Slack notification. + * + * The Slack notification has a maximum character limit, so we must truncate + * the failure results to reduce the risk of hitting that limit. + */ +const FAILURE_SUMMARY_LIMIT = 6; + /** * Outputs test result summary formatted as a Slack message. * @@ -23,37 +31,47 @@ export const slackFormatter: Formatter = ( _junitData: TestSuites[] ) => { const indicator = runInfo.failing ? ':x-mark:' : ':check-mark:'; - const headline = (metadata.runId && metadata.runUrl) - ? `*Cypress test results for run <${metadata.runUrl}|#${metadata.runId}>*\n` - : `*Cypress test results*\n`; + const headline = metadata.pipelineTitle + ? `*${metadata.pipelineTitle}*\n` + : '*Cypress test results*\n'; + + const prInfo = (metadata.changeId && metadata.changeUrl && metadata.changeTitle) + ? `:pull-request: ${metadata.changeTitle} (<${metadata.changeUrl}|#${metadata.changeId}>)\n` + : null; const breakdown = `:small_red_triangle: ${runInfo.failing} Failing | :thumbs_up_green: ${runInfo.passing} Passing | :small_blue_diamond: ${runInfo.skipped} Skipped\n\n`; // Show a human-readable summary of what was tested and whether it succeeded. const summary = (() => { - const info = !runInfo.failing + const statusInfo = !runInfo.failing ? `> ${indicator} ${runInfo.passing} passing ${pluralize(runInfo.passing, 'test', 'tests')}` : `> ${indicator} ${runInfo.failing} failed ${pluralize(runInfo.failing, 'test', 'tests')}`; - const prInfo = (metadata.changeId && metadata.changeUrl) - ? ` on PR <${metadata.changeUrl}|#${metadata.changeId}>${metadata.changeTitle ? ` - _${metadata.changeTitle}_` : ''}` + const buildInfo = (metadata.runId && metadata.runUrl) + ? ` on run <${metadata.runUrl}|#${metadata.runId}>` : ''; const runLength = `(${secondsToTimeString(runInfo.time)})`; const endingPunctuation = !runInfo.failing ? '.' : ':'; - return `${info}${prInfo} ${runLength}${endingPunctuation}` + return `${statusInfo}${buildInfo} ${runLength}${endingPunctuation}` })(); // Display a list of failed tests and collection of actions when applicable. const failedTestSummary = (() => { const failedTestLines = results .filter((result: TestResult) => result.failing) + .slice(0, FAILURE_SUMMARY_LIMIT) .map((result: TestResult) => { const specFile = path.basename(result.testFilename); return `• \`${specFile}\` — _${result.groupName}_ » _${result.testName}_`; }); + const remainingFailures = runInfo.failing - FAILURE_SUMMARY_LIMIT; + const truncationNote = (runInfo.failing > FAILURE_SUMMARY_LIMIT) + ? `and ${remainingFailures} more ${pluralize(remainingFailures, 'failure', 'failures')}...\n` + : null; + // When applicable, display actions that can be taken by the user. const failedTestActions = [ metadata.resultsUrl ? `<${metadata.resultsUrl}|View results>` : '', @@ -66,6 +84,7 @@ export const slackFormatter: Formatter = ( return [ '', ...failedTestLines, + truncationNote, '', failedTestActions ? failedTestActions : null, ] @@ -86,6 +105,8 @@ export const slackFormatter: Formatter = ( return `${rerunTip}\n${cypressCommand}`; })(); + const extra = metadata.extra ? `${metadata.extra}\n` : null; + // Display test run details (author, PR number, run number, etc.) when applicable. const footer = (() => { const authorIdentifier = (metadata.authorSlack ? `@${metadata.authorSlack}` : null) @@ -104,6 +125,7 @@ export const slackFormatter: Formatter = ( return [ headline, + prInfo, breakdown, summary, @@ -115,6 +137,9 @@ export const slackFormatter: Formatter = ( runInfo.failing > 0 ? `${failedTestSummary}\n` : null, runInfo.failing > 0 ? `${rerunNote}\n` : null, + // If extra information has been supplied, display it above the footer. + extra, + // Show run details footer. `:cypress: ${footer}`, ].filter((item) => item !== null).join('\n'); diff --git a/scripts/junit-summary/index.ts b/scripts/junit-summary/index.ts index acc02fb1677..a2d9cbd2d23 100644 --- a/scripts/junit-summary/index.ts +++ b/scripts/junit-summary/index.ts @@ -24,6 +24,7 @@ program .version('0.1.0') .arguments('') .option('-f, --format ', 'JUnit summary output format', 'json') + .option('--meta:title ', 'Pipeline title') .option('--meta:author-name ', 'Author name') .option('--meta:author-slack ', 'Author Slack name') .option('--meta:author-github ', 'Author GitHub name') @@ -36,6 +37,7 @@ program .option('--meta:artifacts-url ', 'CI artifacts URL') .option('--meta:results-url ', 'CI results URL') .option('--meta:rerun-url ', 'CI rerun URL') + .option('--meta:extra ', 'Extra information to display in output') .action((junitPath) => { return main(junitPath); }); @@ -49,6 +51,8 @@ const main = async (junitPath: string) => { const reportPath = path.resolve(junitPath); const summaryFormat = program.opts().format; const metadata: Metadata = { + pipelineTitle: program.opts()['meta:title'], + authorName: program.opts()['meta:authorName'], authorSlack: program.opts()['meta:authorSlack'], authorGitHub: program.opts()['meta:auhtorGithub'], @@ -65,6 +69,8 @@ const main = async (junitPath: string) => { artifactsUrl: program.opts()['meta:artifactsUrl'], resultsUrl: program.opts()['meta:resultsUrl'], rerunUrl: program.opts()['meta:rerunUrl'], + + extra: program.opts()['meta:extra'], }; // Create an array of absolute file paths to JUnit XML report files. @@ -76,10 +82,6 @@ const main = async (junitPath: string) => { return path.resolve(reportPath, dirItem); }); - if (reportFiles.length < 1) { - throw new Error(`No JUnit report files found in '${reportPath}'.`) - } - // Read JUnit report file contents. const loadReportFileContents = reportFiles.map((reportFile: string) => { return fs.readFile(reportFile, 'utf8'); diff --git a/scripts/junit-summary/metadata/metadata.ts b/scripts/junit-summary/metadata/metadata.ts index 12d234aa361..35cde6fd38b 100644 --- a/scripts/junit-summary/metadata/metadata.ts +++ b/scripts/junit-summary/metadata/metadata.ts @@ -3,6 +3,9 @@ */ export interface Metadata { + /** Job or pipeline title. */ + pipelineTitle?: string; + /** Code author name. */ authorName?: string; @@ -38,4 +41,7 @@ export interface Metadata { /** CI rerun trigger URL. */ rerunUrl?: string; + + /** Arbitrary extra information that can be added to output. */ + extra?: string; }
Failing Tests
SpecTest