test: [M3-8072] - Cloud changes for ad-hoc test pipeline (#11088)

* Delete 'region-1' Docker Compose service, deprecate 'e2e', 'e2e_heimdall', and 'component' services, and add 'cypress_local', 'cypress_remote', and 'cypress_component'

* Set yarn as entrypoint for test runner in Dockerfile

* Allow Cypress Slack notification title to be customized

* Allow extra information to be displayed in Slack/GitHub notifications

* Only show up to six failures in a Slack notification

* Allow JUnit summary when there are no JUnit files in given path

* Move LaunchDarkly URL matchers to constants file

* Allow feature flags to be overridden via CY_TEST_FEATURE_FLAGS environment variable

* Add changesets

docker-compose.yml

@@ -35,6 +35,7 @@ x-e2e-env:
   # Cloud Manager-specific test configuration.
   CY_TEST_SUITE: ${CY_TEST_SUITE}
   CY_TEST_REGION: ${CY_TEST_REGION}
+  CY_TEST_FEATURE_FLAGS: ${CY_TEST_FEATURE_FLAGS}
   CY_TEST_TAGS: ${CY_TEST_TAGS}
   CY_TEST_DISABLE_RETRIES: ${CY_TEST_DISABLE_RETRIES}
 
@@ -80,14 +81,9 @@ x-e2e-runners:
     context: .
     dockerfile: ./packages/manager/Dockerfile
     target: e2e
-  depends_on:
-    web:
-      condition: service_healthy
   env_file: ./packages/manager/.env
   volumes: *default-volumes
-  # TODO Stop using entrypoint, use CMD instead.
-  # (Or just make `yarn` the entrypoint, but either way stop forcing `cy:e2e`).
-  entrypoint: ['yarn', 'cy:e2e']
+  entrypoint: 'yarn'
 
 services:
   # Serves a local instance of Cloud Manager for Cypress to use for its tests.
@@ -110,13 +106,56 @@ services:
       timeout: 10s
       retries: 10
 
+  # Cypress test runner service to run tests against a remotely-served Cloud instance.
+  #
+  # This is useful when testing against a standard Cloud Manager environment,
+  # like Production at cloud.linode.com, but can also be used to run tests against
+  # pre-Prod environments, PR preview links, and more.
+  cypress_remote:
+    <<: *default-runner
+    environment:
+      <<: *default-env
+      MANAGER_OAUTH: ${MANAGER_OAUTH}
+
+  # Cypress test runner service to run tests against a locally-served Cloud instance.
+  #
+  # This is useful when testing against a customized or in-development build of
+  # Cloud Manager.
+  cypress_local:
+    <<: *default-runner
+    environment:
+      <<: *default-env
+      MANAGER_OAUTH: ${MANAGER_OAUTH}
+    depends_on:
+      web:
+        condition: service_healthy
+
+  # Cypress component test runner service.
+  #
+  # Unlike other Cloud Manager Cypress tests, these tests can be run without
+  # requiring a Cloud Manager environment.
+  cypress_component:
+    <<: *default-runner
+    environment:
+      CY_TEST_DISABLE_RETRIES: ${CY_TEST_DISABLE_RETRIES}
+      CY_TEST_JUNIT_REPORT: ${CY_TEST_JUNIT_REPORT}
+
+
+  # --> ! DEPRECATION NOTICE ! <--
+  # The services below this line are deprecated, and will be deleted soon.
+  # Don't build any pipelines or write any scripts that depend on these.
+  # Instead, opt to use `cypress_local` in places where you would've used `e2e`,
+  # use `cypress_remote` in places where you would've used `e2e_heimdall`, and
+  # use `cypress_component` in places where you would've used `component`.
+
   # Generic end-to-end test runner for Cloud's primary testing pipeline.
   # Configured to run against a local Cloud instance.
   e2e:
     <<: *default-runner
     environment:
       <<: *default-env
       MANAGER_OAUTH: ${MANAGER_OAUTH}
+    entrypoint: ['yarn', 'cy:e2e']
 
   # Component test runner.
   # Does not require any Cloud Manager environment to run.
@@ -136,14 +175,4 @@ services:
     environment:
       <<: *default-env
       MANAGER_OAUTH: ${MANAGER_OAUTH}
-
-  region-1:
-    build:
-      context: .
-      dockerfile: ./packages/manager/Dockerfile
-      target: e2e
-    env_file: ./packages/manager/.env
-    volumes: *default-volumes
-    environment:
-      <<: *default-env
-      MANAGER_OAUTH: ${MANAGER_OAUTH_1}
+    entrypoint: ['yarn', 'cy:e2e']

docs/development-guide/08-testing.md

@@ -191,12 +191,13 @@ Environment variables related to the general operation of the Cloud Manager Cypr
 | `CY_TEST_SUITE`      | Name of the Cloud Manager UI test suite to run. Possible values are `core`, `region`, or `synthetic`. | `region`     | Unset; defaults to `core` suite |
 | `CY_TEST_TAGS`       | Query identifying tests that should run by specifying allowed and disallowed tags.                    | `method:e2e` | Unset; all tests run by default |
 
-###### Regions
-These environment variables are used by Cloud Manager's UI tests to override region selection behavior. This can be useful for testing Cloud Manager functionality against a specific region.
+###### Overriding Behavior
+These environment variables can be used to override some behaviors of Cloud Manager's UI tests. This can be useful when testing Cloud Manager for nonstandard or work-in-progress functionality.
 
-| Environment Variable | Description                                     | Example   | Default                               |
-|----------------------|-------------------------------------------------|-----------|---------------------------------------|
-| `CY_TEST_REGION`     | ID of region to test (as used by Linode APIv4). | `us-east` | Unset; regions are selected at random |
+| Environment Variable    | Description                                     | Example   | Default                                    |
+|-------------------------|-------------------------------------------------|-----------|--------------------------------------------|
+| `CY_TEST_REGION`        | ID of region to test (as used by Linode APIv4). | `us-east` | Unset; regions are selected at random      |
+| `CY_TEST_FEATURE_FLAGS` | JSON string containing feature flag data        | `{}`      | Unset; feature flag data is not overridden |
 
 ###### Run Splitting
 These environment variables facilitate splitting the Cypress run between multiple runners without the use of any third party services. This can be useful for improving Cypress test performance in some circumstances. For additional performance gains, an optional test weights file can be specified using `CY_TEST_SPLIT_RUN_WEIGHTS` (see `CY_TEST_GENWEIGHTS` to generate test weights).

packages/manager/.changeset/pr-11088-tech-stories-1729535071709.md

@@ -0,0 +1,5 @@
+---
+"@linode/manager": Tech Stories
+---
+
+Replace 'e2e', 'e2e_heimdall', and 'component' Docker Compose services with 'cypress_local', 'cypress_remote', and 'cypress_component' ([#11088](https://github.com/linode/manager/pull/11088))

packages/manager/.changeset/pr-11088-tests-1729535093463.md

@@ -0,0 +1,5 @@
+---
+"@linode/manager": Tests
+---
+
+Allow overriding feature flags via CY_TEST_FEATURE_FLAGS environment variable ([#11088](https://github.com/linode/manager/pull/11088))

packages/manager/.changeset/pr-11088-tests-1729535165205.md

@@ -0,0 +1,5 @@
+---
+"@linode/manager": Tests
+---
+
+Allow pipeline Slack notifications to be customized ([#11088](https://github.com/linode/manager/pull/11088))

packages/manager/.changeset/pr-11088-tests-1729535197632.md

@@ -0,0 +1,5 @@
+---
+"@linode/manager": Tests
+---
+
+Show PR title in Slack CI notifications ([#11088](https://github.com/linode/manager/pull/11088))

packages/manager/Dockerfile

@@ -47,4 +47,3 @@ ENV CI=1
 ENV NO_COLOR=1
 ENV HOME=/home/node/
 ENV CYPRESS_CACHE_FOLDER=/home/node/.cache/Cypress
-ENTRYPOINT yarn cy:ci

packages/manager/cypress.config.ts

@@ -17,6 +17,7 @@ import { enableJunitReport } from './cypress/support/plugins/junit-report';
 import { generateTestWeights } from './cypress/support/plugins/generate-weights';
 import { logTestTagInfo } from './cypress/support/plugins/test-tagging-info';
 import cypressViteConfig from './cypress/vite.config';
+import { featureFlagOverrides } from './cypress/support/plugins/feature-flag-override';
 
 /**
  * Exports a Cypress configuration object.
@@ -91,6 +92,7 @@ export default defineConfig({
         fetchAccount,
         fetchLinodeRegions,
         regionOverrideCheck,
+        featureFlagOverrides,
         logTestTagInfo,
         splitCypressRun,
         enableJunitReport(),

packages/manager/cypress/support/constants/feature-flags.ts

@@ -0,0 +1,11 @@
+/**
+ * @file Constants related to Cypress's handling of LaunchDarkly feature flags.
+ */
+
+// LaunchDarkly URL pattern for feature flag retrieval.
+export const launchDarklyUrlPattern =
+  'https://app.launchdarkly.com/sdk/evalx/*/contexts/*';
+
+// LaunchDarkly URL pattern for feature flag / event streaming.
+export const launchDarklyClientstreamPattern =
+  'https://clientstream.launchdarkly.com/eval/*/*';

packages/manager/cypress/support/e2e.ts

@@ -60,11 +60,13 @@ chai.use(function (chai, utils) {
 
 // Test setup.
 import { deleteInternalHeader } from './setup/delete-internal-header';
+import { mockFeatureFlagRequests } from './setup/mock-feature-flags-request';
 import { mockFeatureFlagClientstream } from './setup/feature-flag-clientstream';
 import { mockAccountRequest } from './setup/mock-account-request';
 import { trackApiRequests } from './setup/request-tracking';
 
 trackApiRequests();
 mockAccountRequest();
+mockFeatureFlagRequests();
 mockFeatureFlagClientstream();
 deleteInternalHeader();

packages/manager/cypress/support/intercepts/feature-flags.ts

@@ -3,17 +3,13 @@
  */
 
 import { getResponseDataFromMockData } from 'support/util/feature-flags';
+import {
+  launchDarklyUrlPattern,
+  launchDarklyClientstreamPattern,
+} from 'support/constants/feature-flags';
 
 import type { FeatureFlagMockData } from 'support/util/feature-flags';
 
-// LaunchDarkly URL pattern for feature flag retrieval.
-const launchDarklyUrlPattern =
-  'https://app.launchdarkly.com/sdk/evalx/*/contexts/*';
-
-// LaunchDarkly URL pattern for feature flag / event streaming.
-const launchDarklyClientstreamPattern =
-  'https://clientstream.launchdarkly.com/eval/*/*';
-
 /**
  * Intercepts GET request to feature flag clientstream URL and mocks the response.
  *

packages/manager/cypress/support/plugins/feature-flag-override.ts

@@ -0,0 +1,39 @@
+import type { CypressPlugin } from './plugin';
+
+/**
+ * Handles setup related to Launch Darkly feature flag overrides.
+ *
+ * Checks if the user has passed overrides via the `CY_TEST_FEATURE_FLAGS` env,
+ * and validates its value if so by attempting to parse it as JSON. If that
+ * succeeds, the parsed override object is exposed to Cypress via the
+ * `featureFlagOverrides` config.
+ */
+export const featureFlagOverrides: CypressPlugin = (_on, config) => {
+  const featureFlagOverridesJson = config.env?.['CY_TEST_FEATURE_FLAGS'];
+
+  let featureFlagOverrides = undefined;
+  if (featureFlagOverridesJson) {
+    const notice =
+      'Feature flag overrides are enabled with the following JSON payload:';
+    const jsonWarning =
+      'Be aware that malformed or invalid feature flag data can trigger crashes and other unexpected behavior.';
+
+    console.info(`${notice}\n\n${featureFlagOverridesJson}\n\n${jsonWarning}`);
+
+    try {
+      featureFlagOverrides = JSON.parse(featureFlagOverridesJson);
+    } catch (e) {
+      throw new Error(
+        `Unable to parse feature flag JSON:\n\n${featureFlagOverridesJson}\n\nPlease double check your 'CY_TEST_FEATURE_FLAGS' value and try again.`
+      );
+    }
+  }
+
+  return {
+    ...config,
+    env: {
+      ...config.env,
+      featureFlagOverrides,
+    },
+  };
+};

packages/manager/cypress/support/setup/mock-feature-flags-request.ts

@@ -0,0 +1,37 @@
+/**
+ * @file Intercepts and mocks Launch Darkly feature flag requests with override data if specified.
+ */
+
+import { launchDarklyUrlPattern } from 'support/constants/feature-flags';
+
+/**
+ * If feature flag overrides have been specified, intercept every LaunchDarkly
+ * feature flag request and modify the response to contain the override data.
+ *
+ * This override happens before other intercepts and mocks (e.g. via `mockGetFeatureFlags`
+ * and `mockAppendFeatureFlags`), so mocks set up by those functions will take
+ * priority in the event that both modify the same feature flag value.
+ */
+export const mockFeatureFlagRequests = () => {
+  const featureFlagOverrides = Cypress.env('featureFlagOverrides');
+
+  if (featureFlagOverrides) {
+    beforeEach(() => {
+      cy.intercept(
+        {
+          middleware: true,
+          url: launchDarklyUrlPattern,
+        },
+        (req) => {
+          req.on('before:response', (res) => {
+            const overriddenFeatureFlagData = {
+              ...res.body,
+              ...featureFlagOverrides,
+            };
+            res.body = overriddenFeatureFlagData;
+          });
+        }
+      );
+    });
+  }
+};

scripts/junit-summary/formatters/github-formatter.ts

@@ -39,6 +39,8 @@ export const githubFormatter: Formatter = (
 
   const breakdown = `:x: ${runInfo.failing} Failing | :green_heart: ${runInfo.passing} Passing | :arrow_right_hook: ${runInfo.skipped} Skipped | :clock1: ${secondsToTimeString(runInfo.time)}\n\n`;
 
+  const extra = metadata.extra ? `${metadata.extra}\n\n` : null;
+
   const failedTestSummary = (() => {
     const heading = `### Details`;
     const failedTestHeader = `<table><thead><tr><th colspan="3">Failing Tests</th></tr><tr><th></th><th>Spec</th><th>Test</th></tr></thead><tbody>`;
@@ -82,6 +84,7 @@ export const githubFormatter: Formatter = (
     headline,
     '',
     breakdown,
+    extra,
     runInfo.failing > 0 ? failedTestSummary : null,
     runInfo.failing > 0 ? rerunNote : null,
   ]