test: [M3-8072] - Cloud changes for ad-hoc test pipeline

docker-compose.yml

@@ -35,6 +35,7 @@ x-e2e-env:
   # Cloud Manager-specific test configuration.
   CY_TEST_SUITE: ${CY_TEST_SUITE}
   CY_TEST_REGION: ${CY_TEST_REGION}
+  CY_TEST_FEATURE_FLAGS: ${CY_TEST_FEATURE_FLAGS}
   CY_TEST_TAGS: ${CY_TEST_TAGS}
   CY_TEST_DISABLE_RETRIES: ${CY_TEST_DISABLE_RETRIES}
 
@@ -80,14 +81,9 @@ x-e2e-runners:
     context: .
     dockerfile: ./packages/manager/Dockerfile
     target: e2e
-  depends_on:
-    web:
-      condition: service_healthy
   env_file: ./packages/manager/.env
   volumes: *default-volumes
-  # TODO Stop using entrypoint, use CMD instead.
-  # (Or just make `yarn` the entrypoint, but either way stop forcing `cy:e2e`).
-  entrypoint: ['yarn', 'cy:e2e']
+  entrypoint: 'yarn'
 
 services:
   # Serves a local instance of Cloud Manager for Cypress to use for its tests.
@@ -110,13 +106,56 @@ services:
       timeout: 10s
       retries: 10
 
+  # Cypress test runner service to run tests against a remotely-served Cloud instance.
+  #
+  # This is useful when testing against a standard Cloud Manager environment,
+  # like Production at cloud.linode.com, but can also be used to run tests against
+  # pre-Prod environments, PR preview links, and more.
+  cypress_remote:
+    <<: *default-runner
+    environment:
+      <<: *default-env
+      MANAGER_OAUTH: ${MANAGER_OAUTH}
+
+  # Cypress test runner service to run tests against a locally-served Cloud instance.
+  #
+  # This is useful when testing against a customized or in-development build of
+  # Cloud Manager.
+  cypress_local:
+    <<: *default-runner
+    environment:
+      <<: *default-env
+      MANAGER_OAUTH: ${MANAGER_OAUTH}
+    depends_on:
+      web:
+        condition: service_healthy
+
+  # Cypress component test runner service.
+  #
+  # Unlike other Cloud Manager Cypress tests, these tests can be run without
+  # requiring a Cloud Manager environment.
+  cypress_component:
+    <<: *default-runner
+    environment:
+      CY_TEST_DISABLE_RETRIES: ${CY_TEST_DISABLE_RETRIES}
+      CY_TEST_JUNIT_REPORT: ${CY_TEST_JUNIT_REPORT}
+
+
+  # --> ! DEPRECATION NOTICE ! <--
+  # The services below this line are deprecated, and will be deleted soon.
+  # Don't build any pipelines or write any scripts that depend on these.
+  # Instead, opt to use `cypress_local` in places where you would've used `e2e`,
+  # use `cypress_remote` in places where you would've used `e2e_heimdall`, and
+  # use `cypress_component` in places where you would've used `component`.
+
   # Generic end-to-end test runner for Cloud's primary testing pipeline.
   # Configured to run against a local Cloud instance.
   e2e:
     <<: *default-runner
     environment:
       <<: *default-env
       MANAGER_OAUTH: ${MANAGER_OAUTH}
+    entrypoint: ['yarn', 'cy:e2e']
 
   # Component test runner.
   # Does not require any Cloud Manager environment to run.
@@ -136,14 +175,4 @@ services:
     environment:
       <<: *default-env
       MANAGER_OAUTH: ${MANAGER_OAUTH}
-
-  region-1:
-    build:
-      context: .
-      dockerfile: ./packages/manager/Dockerfile
-      target: e2e
-    env_file: ./packages/manager/.env
-    volumes: *default-volumes
-    environment:
-      <<: *default-env
-      MANAGER_OAUTH: ${MANAGER_OAUTH_1}
+    entrypoint: ['yarn', 'cy:e2e']

docs/development-guide/08-testing.md

@@ -191,12 +191,13 @@ Environment variables related to the general operation of the Cloud Manager Cypr
 | `CY_TEST_SUITE`      | Name of the Cloud Manager UI test suite to run. Possible values are `core`, `region`, or `synthetic`. | `region`     | Unset; defaults to `core` suite |
 | `CY_TEST_TAGS`       | Query identifying tests that should run by specifying allowed and disallowed tags.                    | `method:e2e` | Unset; all tests run by default |
 
-###### Regions
-These environment variables are used by Cloud Manager's UI tests to override region selection behavior. This can be useful for testing Cloud Manager functionality against a specific region.
+###### Overriding Behavior
+These environment variables can be used to override some behaviors of Cloud Manager's UI tests. This can be useful when testing Cloud Manager for nonstandard or work-in-progress functionality.
 
-| Environment Variable | Description                                     | Example   | Default                               |
-|----------------------|-------------------------------------------------|-----------|---------------------------------------|
-| `CY_TEST_REGION`     | ID of region to test (as used by Linode APIv4). | `us-east` | Unset; regions are selected at random |
+| Environment Variable    | Description                                     | Example   | Default                                    |
+|-------------------------|-------------------------------------------------|-----------|--------------------------------------------|
+| `CY_TEST_REGION`        | ID of region to test (as used by Linode APIv4). | `us-east` | Unset; regions are selected at random      |
+| `CY_TEST_FEATURE_FLAGS` | JSON string containing feature flag data        | `{}`      | Unset; feature flag data is not overridden |
 
 ###### Run Splitting
 These environment variables facilitate splitting the Cypress run between multiple runners without the use of any third party services. This can be useful for improving Cypress test performance in some circumstances. For additional performance gains, an optional test weights file can be specified using `CY_TEST_SPLIT_RUN_WEIGHTS` (see `CY_TEST_GENWEIGHTS` to generate test weights).

packages/manager/Dockerfile

@@ -47,4 +47,3 @@ ENV CI=1
 ENV NO_COLOR=1
 ENV HOME=/home/node/
 ENV CYPRESS_CACHE_FOLDER=/home/node/.cache/Cypress
-ENTRYPOINT yarn cy:ci

packages/manager/cypress.config.ts

@@ -17,6 +17,7 @@ import { enableJunitReport } from './cypress/support/plugins/junit-report';
 import { generateTestWeights } from './cypress/support/plugins/generate-weights';
 import { logTestTagInfo } from './cypress/support/plugins/test-tagging-info';
 import cypressViteConfig from './cypress/vite.config';
+import { featureFlagOverrides } from './cypress/support/plugins/feature-flag-override';
 
 /**
  * Exports a Cypress configuration object.
@@ -91,6 +92,7 @@ export default defineConfig({
         fetchAccount,
         fetchLinodeRegions,
         regionOverrideCheck,
+        featureFlagOverrides,
         logTestTagInfo,
         splitCypressRun,
         enableJunitReport(),

packages/manager/cypress/support/constants/feature-flags.ts

@@ -0,0 +1,11 @@
+/**
+ * @file Constants related to Cypress's handling of LaunchDarkly feature flags.
+ */
+
+// LaunchDarkly URL pattern for feature flag retrieval.
+export const launchDarklyUrlPattern =
+  'https://app.launchdarkly.com/sdk/evalx/*/contexts/*';
+
+// LaunchDarkly URL pattern for feature flag / event streaming.
+export const launchDarklyClientstreamPattern =
+  'https://clientstream.launchdarkly.com/eval/*/*';

packages/manager/cypress/support/e2e.ts

@@ -60,11 +60,13 @@ chai.use(function (chai, utils) {
 
 // Test setup.
 import { deleteInternalHeader } from './setup/delete-internal-header';
+import { mockFeatureFlagRequests } from './setup/mock-feature-flags-request';
 import { mockFeatureFlagClientstream } from './setup/feature-flag-clientstream';
 import { mockAccountRequest } from './setup/mock-account-request';
 import { trackApiRequests } from './setup/request-tracking';
 
 trackApiRequests();
 mockAccountRequest();
+mockFeatureFlagRequests();
 mockFeatureFlagClientstream();
 deleteInternalHeader();

packages/manager/cypress/support/intercepts/feature-flags.ts

@@ -3,17 +3,13 @@
  */
 
 import { getResponseDataFromMockData } from 'support/util/feature-flags';
+import {
+  launchDarklyUrlPattern,
+  launchDarklyClientstreamPattern,
+} from 'support/constants/feature-flags';
 
 import type { FeatureFlagMockData } from 'support/util/feature-flags';
 
-// LaunchDarkly URL pattern for feature flag retrieval.
-const launchDarklyUrlPattern =
-  'https://app.launchdarkly.com/sdk/evalx/*/contexts/*';
-
-// LaunchDarkly URL pattern for feature flag / event streaming.
-const launchDarklyClientstreamPattern =
-  'https://clientstream.launchdarkly.com/eval/*/*';
-
 /**
  * Intercepts GET request to feature flag clientstream URL and mocks the response.
  *

packages/manager/cypress/support/plugins/feature-flag-override.ts

@@ -0,0 +1,39 @@
+import type { CypressPlugin } from './plugin';
+
+/**
+ * Handles setup related to Launch Darkly feature flag overrides.
+ *
+ * Checks if the user has passed overrides via the `CY_TEST_FEATURE_FLAGS` env,
+ * and validates its value if so by attempting to parse it as JSON. If that
+ * succeeds, the parsed override object is exposed to Cypress via the
+ * `featureFlagOverrides` config.
+ */
+export const featureFlagOverrides: CypressPlugin = (_on, config) => {
+  const featureFlagOverridesJson = config.env?.['CY_TEST_FEATURE_FLAGS'];
+
+  let featureFlagOverrides = undefined;
+  if (featureFlagOverridesJson) {
+    const notice =
+      'Feature flag overrides are enabled with the following JSON payload:';
+    const jsonWarning =
+      'Be aware that malformed or invalid feature flag data can trigger crashes and other unexpected behavior.';
+
+    console.info(`${notice}\n\n${featureFlagOverridesJson}\n\n${jsonWarning}`);
+
+    try {
+      featureFlagOverrides = JSON.parse(featureFlagOverridesJson);
+    } catch (e) {
+      throw new Error(
+        `Unable to parse feature flag JSON:\n\n${featureFlagOverridesJson}\n\nPlease double check your 'CY_TEST_FEATURE_FLAGS' value and try again.`
+      );
+    }
+  }
+
+  return {
+    ...config,
+    env: {
+      ...config.env,
+      featureFlagOverrides,
+    },
+  };
+};

packages/manager/cypress/support/setup/mock-feature-flags-request.ts

@@ -0,0 +1,37 @@
+/**
+ * @file Intercepts and mocks Launch Darkly feature flag requests with override data if specified.
+ */
+
+import { launchDarklyUrlPattern } from 'support/constants/feature-flags';
+
+/**
+ * If feature flag overrides have been specified, intercept every LaunchDarkly
+ * feature flag request and modify the response to contain the override data.
+ *
+ * This override happens before other intercepts and mocks (e.g. via `mockGetFeatureFlags`
+ * and `mockAppendFeatureFlags`), so mocks set up by those functions will take
+ * priority in the event that both modify the same feature flag value.
+ */
+export const mockFeatureFlagRequests = () => {
+  const featureFlagOverrides = Cypress.env('featureFlagOverrides');
+
+  if (featureFlagOverrides) {
+    beforeEach(() => {
+      cy.intercept(
+        {
+          middleware: true,
+          url: launchDarklyUrlPattern,
+        },
+        (req) => {
+          req.on('before:response', (res) => {
+            const overriddenFeatureFlagData = {
+              ...res.body,
+              ...featureFlagOverrides,
+            };
+            res.body = overriddenFeatureFlagData;
+          });
+        }
+      );
+    });
+  }
+};

scripts/junit-summary/formatters/github-formatter.ts

@@ -39,6 +39,8 @@ export const githubFormatter: Formatter = (
 
   const breakdown = `:x: ${runInfo.failing} Failing | :green_heart: ${runInfo.passing} Passing | :arrow_right_hook: ${runInfo.skipped} Skipped | :clock1: ${secondsToTimeString(runInfo.time)}\n\n`;
 
+  const extra = metadata.extra ? `${metadata.extra}\n\n` : null;
+
   const failedTestSummary = (() => {
     const heading = `### Details`;
     const failedTestHeader = `<table><thead><tr><th colspan="3">Failing Tests</th></tr><tr><th></th><th>Spec</th><th>Test</th></tr></thead><tbody>`;
@@ -82,6 +84,7 @@ export const githubFormatter: Formatter = (
     headline,
     '',
     breakdown,
+    extra,
     runInfo.failing > 0 ? failedTestSummary : null,
     runInfo.failing > 0 ? rerunNote : null,
   ]

scripts/junit-summary/formatters/slack-formatter.ts

@@ -8,6 +8,14 @@ import { secondsToTimeString } from '../util';
 import * as path from 'path';
 import { cypressRunCommand } from '../util/cypress';
 
+/**
+ * The maximum number of failures that will be listed in the Slack notification.
+ *
+ * The Slack notification has a maximum character limit, so we must truncate
+ * the failure results to reduce the risk of hitting that limit.
+ */
+const FAILURE_SUMMARY_LIMIT = 6;
+
 /**
  * Outputs test result summary formatted as a Slack message.
  *
@@ -23,37 +31,47 @@ export const slackFormatter: Formatter = (
   _junitData: TestSuites[]
 ) => {
   const indicator = runInfo.failing ? ':x-mark:' : ':check-mark:';
-  const headline = (metadata.runId && metadata.runUrl)
-    ? `*Cypress test results for run <${metadata.runUrl}|#${metadata.runId}>*\n`
-    : `*Cypress test results*\n`;
+  const headline = metadata.pipelineTitle
+    ? `*${metadata.pipelineTitle}*\n`
+    : '*Cypress test results*\n';
+
+  const prInfo = (metadata.changeId && metadata.changeUrl && metadata.changeTitle)
+    ? `:pull-request: ${metadata.changeTitle} (<${metadata.changeUrl}|#${metadata.changeId}>)\n`
+    : null;
 
   const breakdown = `:small_red_triangle: ${runInfo.failing} Failing | :thumbs_up_green: ${runInfo.passing} Passing | :small_blue_diamond: ${runInfo.skipped} Skipped\n\n`;
 
   // Show a human-readable summary of what was tested and whether it succeeded.
   const summary = (() => {
-    const info = !runInfo.failing
+    const statusInfo = !runInfo.failing
      ? `> ${indicator} ${runInfo.passing} passing ${pluralize(runInfo.passing, 'test', 'tests')}`
      : `> ${indicator} ${runInfo.failing} failed ${pluralize(runInfo.failing, 'test', 'tests')}`;
 
-    const prInfo = (metadata.changeId && metadata.changeUrl)
-      ? ` on PR <${metadata.changeUrl}|#${metadata.changeId}>${metadata.changeTitle ? ` - _${metadata.changeTitle}_` : ''}`
+    const buildInfo = (metadata.runId && metadata.runUrl)
+      ? ` on run <${metadata.runUrl}|#${metadata.runId}>`
       : '';
 
     const runLength = `(${secondsToTimeString(runInfo.time)})`;
     const endingPunctuation = !runInfo.failing ? '.' : ':';
 
-    return `${info}${prInfo} ${runLength}${endingPunctuation}`
+    return `${statusInfo}${buildInfo} ${runLength}${endingPunctuation}`
   })();
 
   // Display a list of failed tests and collection of actions when applicable.
   const failedTestSummary = (() => {
     const failedTestLines = results
       .filter((result: TestResult) => result.failing)
+      .slice(0, FAILURE_SUMMARY_LIMIT)
       .map((result: TestResult) => {
         const specFile = path.basename(result.testFilename);
         return `• \`${specFile}\` — _${result.groupName}_ » _${result.testName}_`;
       });
 
+    const remainingFailures = runInfo.failing - FAILURE_SUMMARY_LIMIT;
+    const truncationNote = (runInfo.failing > FAILURE_SUMMARY_LIMIT)
+      ? `and ${remainingFailures} more ${pluralize(remainingFailures, 'failure', 'failures')}...\n`
+      : null;
+
     // When applicable, display actions that can be taken by the user.
     const failedTestActions = [
       metadata.resultsUrl ? `<${metadata.resultsUrl}|View results>` : '',
@@ -66,6 +84,7 @@ export const slackFormatter: Formatter = (
     return [
       '',
       ...failedTestLines,
+      truncationNote,
       '',
       failedTestActions ? failedTestActions : null,
     ]
@@ -86,6 +105,8 @@ export const slackFormatter: Formatter = (
     return `${rerunTip}\n${cypressCommand}`;
   })();
 
+  const extra = metadata.extra ? `${metadata.extra}\n` : null;
+
   // Display test run details (author, PR number, run number, etc.) when applicable.
   const footer = (() => {
     const authorIdentifier = (metadata.authorSlack ? `@${metadata.authorSlack}` : null)
@@ -104,6 +125,7 @@ export const slackFormatter: Formatter = (
 
   return [
     headline,
+    prInfo,
     breakdown,
     summary,
 
@@ -115,6 +137,9 @@ export const slackFormatter: Formatter = (
     runInfo.failing > 0 ? `${failedTestSummary}\n` : null,
     runInfo.failing > 0 ? `${rerunNote}\n` : null,
 
+    // If extra information has been supplied, display it above the footer.
+    extra,
+
     // Show run details footer.
     `:cypress: ${footer}`,
   ].filter((item) => item !== null).join('\n');