From f13815352f00f1e673caa9f5f9729e07deeb4366 Mon Sep 17 00:00:00 2001 From: jdamore-linode <97627410+jdamore-linode@users.noreply.github.com> Date: Mon, 4 Mar 2024 12:06:53 -0500 Subject: [PATCH] refactor: [M3-7650] - Refactor Cypress region utils, address region capacity-related flake (#10242) * Refactor Cypress region handling utils, disallow ap-northeast and us-iad from being used * Wait 15 seconds if API responds with 429 during Cypress test setup --- .../pr-10242-tests-1709244555529.md | 5 + .../cypress/support/constants/databases.ts | 6 +- .../cypress/support/setup/defer-command.ts | 10 +- .../manager/cypress/support/util/regions.ts | 183 ++++++++++++++---- 4 files changed, 160 insertions(+), 44 deletions(-) create mode 100644 packages/manager/.changeset/pr-10242-tests-1709244555529.md diff --git a/packages/manager/.changeset/pr-10242-tests-1709244555529.md b/packages/manager/.changeset/pr-10242-tests-1709244555529.md new file mode 100644 index 00000000000..7cd31fd424a --- /dev/null +++ b/packages/manager/.changeset/pr-10242-tests-1709244555529.md @@ -0,0 +1,5 @@ +--- +"@linode/manager": Tests +--- + +Refactor Cypress region utils, address region capacity flake ([#10242](https://github.com/linode/manager/pull/10242)) diff --git a/packages/manager/cypress/support/constants/databases.ts b/packages/manager/cypress/support/constants/databases.ts index bc4e36dc948..4957b0cec4a 100644 --- a/packages/manager/cypress/support/constants/databases.ts +++ b/packages/manager/cypress/support/constants/databases.ts @@ -61,7 +61,7 @@ export const databaseConfigurations: databaseClusterConfiguration[] = [ engine: 'MySQL', label: randomLabel(), linodeType: 'g6-nanode-1', - region: chooseRegion({ capability: 'Managed Databases' }), + region: chooseRegion({ capabilities: ['Managed Databases'] }), version: '8', }, { @@ -70,7 +70,7 @@ export const databaseConfigurations: databaseClusterConfiguration[] = [ engine: 'MySQL', label: randomLabel(), linodeType: 'g6-dedicated-16', - region: chooseRegion({ capability: 'Managed Databases' }), + region: chooseRegion({ capabilities: ['Managed Databases'] }), version: '5', }, // { @@ -89,7 +89,7 @@ export const databaseConfigurations: databaseClusterConfiguration[] = [ engine: 'PostgreSQL', label: randomLabel(), linodeType: 'g6-nanode-1', - region: chooseRegion({ capability: 'Managed Databases' }), + region: chooseRegion({ capabilities: ['Managed Databases'] }), version: '13', }, ]; diff --git a/packages/manager/cypress/support/setup/defer-command.ts b/packages/manager/cypress/support/setup/defer-command.ts index 0d903aa078f..f5f34068674 100644 --- a/packages/manager/cypress/support/setup/defer-command.ts +++ b/packages/manager/cypress/support/setup/defer-command.ts @@ -1,5 +1,6 @@ import type { APIError } from '@linode/api-v4'; import type { AxiosError } from 'axios'; +import { timeout } from 'support/util/backoff'; type LinodeApiV4Error = { errors: APIError[]; @@ -185,7 +186,7 @@ Cypress.Commands.add( return { log: false }; })(); - const timeout = (() => { + const timeoutLength = (() => { if (typeof labelOrOptions !== 'string') { return labelOrOptions?.timeout; } @@ -197,7 +198,7 @@ Cypress.Commands.add( end: false, message: commandLabel, name: 'defer', - timeout, + timeout: timeoutLength, }); // Wraps the given promise in order to update Cypress's log on completion. @@ -207,6 +208,11 @@ Cypress.Commands.add( result = await promise; } catch (e: any) { commandLog.error(e); + // If we're getting rate limited, timeout for 15 seconds so that + // test reattempts do not immediately trigger more 429 responses. + if (isAxiosError(e) && e.response?.status === 429) { + await timeout(15000); + } throw enhanceError(e); } commandLog.end(); diff --git a/packages/manager/cypress/support/util/regions.ts b/packages/manager/cypress/support/util/regions.ts index a16841f2f0f..0eade6285cf 100644 --- a/packages/manager/cypress/support/util/regions.ts +++ b/packages/manager/cypress/support/util/regions.ts @@ -1,7 +1,25 @@ import { randomItem } from 'support/util/random'; +import { buildArray, shuffleArray } from './arrays'; import type { Capabilities, Region } from '@linode/api-v4'; +/** + * Regions that cannot be selected using `chooseRegion()` and `chooseRegions()`. + * + * This is useful for regions which have capabilities that are required for tests, + * but do not have capacity, resulting in 400 responses from the API. + * + * In the future we may be able to leverage the API to dynamically exclude regions + * that are lacking capacity. + */ +const disallowedRegionIds = [ + // Tokyo, JP + 'ap-northeast', + + // Washington, DC + 'us-iad', +]; + /** * Returns an object describing a Cloud Manager region if specified by the user. * @@ -46,10 +64,15 @@ export const getTestableRegions = (): Region[] => { * * If no known region exists with the given ID, an error is thrown. * + * @param id - ID of the region to find. + * @param searchRegions - Optional array of Regions from which to search. + * * @throws When no region exists in the `regions` array with the given ID. */ -export const getRegionById = (id: string) => { - const region = regions.find((findRegion: Region) => findRegion.id === id); +export const getRegionById = (id: string, searchRegions?: Region[]) => { + const region = (searchRegions ?? regions).find( + (findRegion: Region) => findRegion.id === id + ); if (!region) { throw new Error(`Unable to find region by ID. Unknown ID '${id}'.`); } @@ -62,10 +85,13 @@ export const getRegionById = (id: string) => { * If no known region exists with the given human-readable label, an error is * thrown. * + * @param label - Label of the region to find. + * @param searchRegions - Optional array of Regions from which to search. + * * @throws When no region exists in the `regions` array with the given label. */ -export const getRegionByLabel = (label: string) => { - const region = regions.find( +export const getRegionByLabel = (label: string, searchRegions?: Region[]) => { + const region = (searchRegions ?? regions).find( (findRegion: Region) => findRegion.label === label ); if (!region) { @@ -78,38 +104,119 @@ export const getRegionByLabel = (label: string) => { interface ChooseRegionOptions { /** - * If specified, the region returned will support the defined capability + * If specified, the region returned will support the defined capabilities * @example 'Managed Databases' */ - capability?: Capabilities; + capabilities?: Capabilities[]; + + /** + * Regions from which to choose. If unspecified, Regions exposed by the API will be used. + */ + regions?: Region[]; } /** - * Returns a known Cloud Manager region at random, or returns a user-chosen - * region if one was specified. + * Returns `true` if the given Region has all of the given capabilities. * - * Region selection can be configured via the `CY_TEST_REGION` environment - * variable. If defined, the region returned by this function will be - * overridden using the chosen region. + * @param region - Region to check capabilities. + * @param capabilities - Capabilities to check. * - * @returns Object describing a Cloud Manager region to use during tests. + * @returns `true` if `region` has all of the given capabilities. */ -export const chooseRegion = (options?: ChooseRegionOptions): Region => { +const regionHasCapabilities = ( + region: Region, + capabilities: Capabilities[] +): boolean => { + return capabilities.every((capability) => + region.capabilities.includes(capability) + ); +}; + +/** + * Returns an array of Region objects that have all of the given capabilities. + * + * @param regions - Regions from which to search. + * @param capabilities - Capabilities to check. + * + * @returns Array of Region objects containing the required capabilities. + */ +const regionsWithCapabilities = ( + regions: Region[], + capabilities: Capabilities[] +): Region[] => { + return regions.filter((region: Region) => + regionHasCapabilities(region, capabilities) + ); +}; + +/** + * Returns an array of Region objects that meet the given criteria. + * + * @param options - Object describing Region selection criteria. + * @param detectOverrideRegion - Whether override region should be detected and applied. + * + * @throws If no regions meet the desired criteria. + * @throws If an override region is specified which does not meet the given criteria. + * + * @returns Array of Region objects that meet criteria specified by `options` param. + */ +const resolveSearchRegions = ( + options?: ChooseRegionOptions, + detectOverrideRegion: boolean = true +): Region[] => { + const requiredCapabilities = options?.capabilities ?? []; const overrideRegion = getOverrideRegion(); - if (overrideRegion) { - return overrideRegion; + // If the user has specified an override region for this run, it takes precedent + // over any other specified criteria. + if (overrideRegion && detectOverrideRegion) { + // TODO Consider skipping instead of failing when test isn't applicable to override region. + if (!regionHasCapabilities(overrideRegion, requiredCapabilities)) { + throw new Error( + `Override region ${overrideRegion.id} (${ + overrideRegion.label + }) does not support one or more capabilities: ${requiredCapabilities.join( + ', ' + )}` + ); + } + if (disallowedRegionIds.includes(overrideRegion.id)) { + throw new Error( + `Override region ${overrideRegion.id} (${overrideRegion.label}) is disallowed for testing due to capacity limitations.` + ); + } + return [overrideRegion]; } - if (options?.capability) { - const regionsWithCapability = regions.filter((region) => - region.capabilities.includes(options.capability!) - ); + const capableRegions = regionsWithCapabilities( + options?.regions ?? regions, + requiredCapabilities + ).filter((region: Region) => !disallowedRegionIds.includes(region.id)); - return randomItem(regionsWithCapability); + if (!capableRegions.length) { + throw new Error( + `No regions are available with the required capabilities: ${requiredCapabilities.join( + ', ' + )}` + ); } - return randomItem(regions); + return capableRegions; +}; + +/** + * Returns a known Cloud Manager region at random, or returns a user-chosen + * region if one was specified. + * + * Region selection can be overridden via the `CY_TEST_REGION` environment + * variable. + * + * @param options - Region selection options. + * + * @returns Object describing a Cloud Manager region to use during tests. + */ +export const chooseRegion = (options?: ChooseRegionOptions): Region => { + return randomItem(resolveSearchRegions(options)); }; /** @@ -120,38 +227,36 @@ export const chooseRegion = (options?: ChooseRegionOptions): Region => { * subsequent items will be chosen at random. * * @param count - Number of Regions to include in the returned array. + * @param options - Region selection options. * * @throws When `count` is less than 0. * @throws When there are not enough regions to satisfy the given `count`. * * @returns Array of Cloud Manager Region objects. */ -export const chooseRegions = (count: number): Region[] => { +export const chooseRegions = ( + count: number, + options?: ChooseRegionOptions +): Region[] => { if (count < 0) { throw new Error( 'Unable to choose regions. The desired number of regions must be 0 or greater' ); } - if (regions.length < count) { + + const searchRegions = [ + ...shuffleArray(resolveSearchRegions(options, false)), + // If an override region is specified, insert it into the array last so it pops first. + ...(getOverrideRegion() ? resolveSearchRegions(options, true) : []), + ]; + + if (searchRegions.length < count) { throw new Error( - `Unable to choose regions. The desired number of regions exceeds the number of known regions (${regions.length})` + `Unable to choose regions. The desired number of regions exceeds the number of known regions that meet the required criteria (${regions.length})` ); } - const overrideRegion = getOverrideRegion(); - return new Array(count).fill(null).reduce((acc: Region[], _cur, index) => { - const chosenRegion: Region = ((): Region => { - if (index === 0 && overrideRegion) { - return overrideRegion; - } - // Get an array of regions that have not already been selected. - const unusedRegions = regions.filter( - (region: Region) => !acc.includes(region) - ); - return randomItem(unusedRegions); - })(); - acc.push(chosenRegion); - return acc; - }, []); + + return buildArray(count, (i) => searchRegions.pop()!); }; /**