feat: experimental retries (#27930)

* chore: set up feature/test-burn-in feature branch

* feat: add burnIn Configuration option (currently a no-op) (#27377)

* feat: add the burnIn Configuration to the config package. Option
currently is a no-op

* chore: make burn in experimental

* chore: set experimentalBurnIn to false by default

* feat: add new experimental retries configuration (#27412)

* feat: implement the experimental retries configuration options to pair
with test burn in

* [run ci]

* fix cache invalidation [run ci]

* fix snapshot added in v13 for module api to include test burn in experimentalflag

* chore: fix merge conflict

* chore: add burnInTestAction capability (#27768)

* add burnInTestAction capability

* feat: add burn in capability for cloud

* chore: fix snapshot for record_spec

* feat: implement experimental retries (#27826)

* chore: format the retries/runner snapshot files to make diff easier

* feat: implement experimentalRetries strategies 'detect-flake-and-pass-on-threshold' and 'detect-flake-but-always-fail'. This should not be a breaking change, though it does modify mocha and the test object even when the experiment is not configured. This is to exercise the system and make sure things still work as expected even when we go GA. Test updates will follow in following commits.

* chore: update snapshots from system tests and cy-in-cy tests that now have the cypress test metadata property _cypressTestStatusInfo. tests have been added in the fail-with-[before|after]each specs to visually see the suite being skipped when developing.

* chore: add cy-in-cy tests to verify reporter behavior for pass/fail tests, as well as new mocha snapshots to verify attempts. New tests were needed for this as the 'retries' option in testConfigOverrides currently is and will be invalid for experiment and will function as an override. tests run in the cy-in-cy tests are using globally configured experimentalRetries for the given tested project, which showcases the different behavior between attempts/retries and pass/fail status.

* chore: add unit test like driver test to verify the test object in mocha is decorated/handled properly in calculateTestStatus

* chore: add sanity system tests to verify console reporter output for experimental retries logic. Currently there is a bug in the reporter where the logged status doesnt wait for the aftereach to complete, which impacts the total exitCode and printed status.

* fix: aftereach console output. make sure to fail the test in the appropriate spot in runner.ts and not prematurely, which in turn updates the snapshots for cy-in-cy as the fail event comes later."

* chore: address comments from code review

* fix: make sure hook failures print outer status + attempts when the error is the hook itself.

* chore: improve types within calculateTestStatus inside mocha.ts

* Revert "feat: add burnIn Configuration option (currently a no-op) (#27377)"

This reverts commit c428443.

* Revert "chore: add burnInTestAction capability (#27768)"

This reverts commit ae3df1a.

* chore: run snapshot and binary jobs against experimental retries feature branch

* chore: add changelog entry (wip)

* Revert "fix snapshot added in v13 for module api to include test burn in experimentalflag"

This reverts commit bb5046c.

* Fix system tests

* Clear CircleCI cache

* Normalize retries config for test execution

* Fixed some unit tests

* update snapshots for newer test metadata

* Fix cy-in-cy snapshots

* update snapshots

* bump cache version

* chore: ensure legacy retry overrides work; reject exp. retries overrides (#28045)

* update changelog

* flip if statement in experimental retries option validation

* refactor invalid experimental retry override for more useful error msg

* revert testConfigOverrides snapshot

* update snapshots for test override sys test

* Update packages/config/src/validation.ts

Co-authored-by: Chris Breiding <chrisbreiding@users.noreply.github.com>

* succinct changelog entry; links to docs for details

* testConfigOverride system test snapshots

* Update .github/workflows/update_v8_snapshot_cache.yml

Co-authored-by: Ryan Manuel <ryanm@cypress.io>

* Update cli/CHANGELOG.md

Co-authored-by: Ryan Manuel <ryanm@cypress.io>

* Update packages/driver/src/cypress.ts

Co-authored-by: Ryan Manuel <ryanm@cypress.io>

* updating cache-version

* improve typescript usage when appending experimental retry options to experiments in Experimenets.vue

* Revert "improve typescript usage when appending experimental retry options to experiments in Experimenets.vue"

This reverts commit b459aba.

* refactor test config override validation for experimental retry subkeys

* account for error throw differences in browsers in system tests

* bump circle cache

* bump circle cache again

---------

Co-authored-by: astone123 <adams@cypress.io>
Co-authored-by: mabela416 <mabel@cypress.io>
Co-authored-by: Muaz Othman <muaz@cypress.io>
Co-authored-by: Muaz Othman <muazweb@gmail.com>
Co-authored-by: Cacie Prins <cacie@cypress.io>
Co-authored-by: Cacie Prins <cacieprins@users.noreply.github.com>
Co-authored-by: Chris Breiding <chrisbreiding@users.noreply.github.com>
Co-authored-by: Ryan Manuel <ryanm@cypress.io>
Co-authored-by: Matthew Schile <mschile@cypress.io>

.circleci/cache-version.txt

@@ -1,3 +1,3 @@
 # Bump this version to force CI to re-create the cache from scratch.
 
-10-18-23
+10-25-23.1

.circleci/workflows.yml

@@ -29,6 +29,7 @@ mainBuildFilters: &mainBuildFilters
         - develop
         - /^release\/\d+\.\d+\.\d+$/
         # use the following branch as well to ensure that v8 snapshot cache updates are fully tested
+        - 'feature/experimental-retries'
         - 'publish-binary'
         - 'ryanm/fix/better-sqlite3'
 
@@ -41,6 +42,9 @@ macWorkflowFilters: &darwin-workflow-filters
     - equal: [ develop, << pipeline.git.branch >> ]
     # use the following branch as well to ensure that v8 snapshot cache updates are fully tested
     - equal: [ 'update-v8-snapshot-cache-on-develop', << pipeline.git.branch >> ]
+    - equal: [ 'feature/experimental-retries', << pipeline.git.branch >> ]
+    - equal: [ 'chore/update_webpack_deps_to_latest_webpack4_compat', << pipeline.git.branch >> ]
+    - equal: [ 'lerna-optimize-tasks', << pipeline.git.branch >> ]
     - equal: [ 'ryanm/fix/better-sqlite3', << pipeline.git.branch >> ]
     - matches:
         pattern: /^release\/\d+\.\d+\.\d+$/
@@ -52,6 +56,10 @@ linuxArm64WorkflowFilters: &linux-arm64-workflow-filters
     - equal: [ develop, << pipeline.git.branch >> ]
     # use the following branch as well to ensure that v8 snapshot cache updates are fully tested
     - equal: [ 'update-v8-snapshot-cache-on-develop', << pipeline.git.branch >> ]
+    - equal: [ 'feature/experimental-retries', << pipeline.git.branch >> ]
+    - equal: [ 'chore/update_webpack_deps_to_latest_webpack4_compat', << pipeline.git.branch >> ]
+    - equal: [ 'chore/bump_loaders_and_optimize_webpack', << pipeline.git.branch >> ]
+    - equal: [ 'lerna-optimize-tasks', << pipeline.git.branch >> ]
     - equal: [ 'ryanm/fix/better-sqlite3', << pipeline.git.branch >> ]
     - matches:
         pattern: /^release\/\d+\.\d+\.\d+$/
@@ -75,6 +83,9 @@ windowsWorkflowFilters: &windows-workflow-filters
     - equal: [ develop, << pipeline.git.branch >> ]
     # use the following branch as well to ensure that v8 snapshot cache updates are fully tested
     - equal: [ 'update-v8-snapshot-cache-on-develop', << pipeline.git.branch >> ]
+    - equal: [ 'feature/experimental-retries', << pipeline.git.branch >> ]
+    - equal: [ 'chore/update_webpack_deps_to_latest_webpack4_compat', << pipeline.git.branch >> ]
+    - equal: [ 'lerna-optimize-tasks', << pipeline.git.branch >> ]
     - equal: [ 'ryanm/fix/better-sqlite3', << pipeline.git.branch >> ]
     - matches:
         pattern: /^release\/\d+\.\d+\.\d+$/
@@ -145,7 +156,7 @@ commands:
           name: Set environment variable to determine whether or not to persist artifacts
           command: |
             echo "Setting SHOULD_PERSIST_ARTIFACTS variable"
-            echo 'if ! [[ "$CIRCLE_BRANCH" != "develop" && "$CIRCLE_BRANCH" != "release/"* && "$CIRCLE_BRANCH" != "publish-binary" && "$CIRCLE_BRANCH" != "update-v8-snapshot-cache-on-develop" && "$CIRCLE_BRANCH" != "ryanm/fix/better-sqlite3" ]]; then
+            echo 'if ! [[ "$CIRCLE_BRANCH" != "develop" && "$CIRCLE_BRANCH" != "release/"* && "$CIRCLE_BRANCH" != "publish-binary" && "$CIRCLE_BRANCH" != "lerna-optimize-tasks" && "$CIRCLE_BRANCH" != "feature/experimental-retries" ]]; then
                 export SHOULD_PERSIST_ARTIFACTS=true
             fi' >> "$BASH_ENV"
   # You must run `setup_should_persist_artifacts` command and be using bash before running this command

cli/CHANGELOG.md

@@ -1,7 +1,12 @@
 <!-- See the ../guides/writing-the-cypress-changelog.md for details on writing the changelog. -->
-## 13.3.4
 
-_Released 11/7/2023 (PENDING)_
+## 13.4.0
+
+_Released 10/25/2023 (PENDING)_
+
+**Features:**
+
+- Introduced experimental configuration options for advanced retry logic: adds `experimentalStrategy` and `experimentalOptions` keys to the `retry` configuration key. See [Experimental Flake Detection Features](https://docs.cypress.io/guides/references/experiments/#Experimental-Flake-Detection-Features) in the documentation. Addressed in [#27930](https://github.com/cypress-io/cypress/pull/27930).
 
 **Bugfixes:**
 

cli/types/cypress.d.ts

@@ -2851,6 +2851,30 @@ declare namespace Cypress {
     certs: PEMCert[] | PFXCert[]
   }
 
+  type RetryStrategyWithModeSpecs = RetryStrategy & {
+    openMode: boolean; // defaults to false
+    runMode: boolean; // defaults to true
+  }
+
+  type RetryStrategy =
+    | RetryStrategyDetectFlakeAndPassOnThresholdType
+    | RetryStrategyDetectFlakeButAlwaysFailType
+
+  interface RetryStrategyDetectFlakeAndPassOnThresholdType {
+    experimentalStrategy: "detect-flake-and-pass-on-threshold"
+    experimentalOptions?: {
+      maxRetries: number; // defaults to 2 if experimentalOptions is not provided, must be a whole number > 0
+      passesRequired: number; // defaults to 2 if experimentalOptions is not provided, must be a whole number > 0 and <= maxRetries
+    }
+  }
+
+  interface RetryStrategyDetectFlakeButAlwaysFailType {
+    experimentalStrategy: "detect-flake-but-always-fail"
+    experimentalOptions?: {
+      maxRetries: number; // defaults to 2 if experimentalOptions is not provided, must be a whole number > 0
+      stopIfAnyPassed: boolean; // defaults to false if experimentalOptions is not provided
+    }
+  }
   interface ResolvedConfigOptions<ComponentDevServerOpts = any> {
     /**
      * Url used as prefix for [cy.visit()](https://on.cypress.io/visit) or [cy.request()](https://on.cypress.io/request) command's url
@@ -3122,7 +3146,7 @@ declare namespace Cypress {
      * To enable test retries only in runMode, set e.g. `{ openMode: null, runMode: 2 }`
      * @default null
      */
-    retries: Nullable<number | { runMode?: Nullable<number>, openMode?: Nullable<number> }>
+    retries: Nullable<number | ({ runMode?: Nullable<number>, openMode?: Nullable<number> }) | RetryStrategyWithModeSpecs>
     /**
      * Enables including elements within the shadow DOM when using querying
      * commands (e.g. cy.get(), cy.find()). Can be set globally in cypress.config.{js,ts,mjs,cjs},

cli/types/tests/cypress-tests.ts

@@ -1159,6 +1159,39 @@ namespace CypressLocalStorageTests {
   cy.clearAllSessionStorage({ log: 'true' }) // $ExpectError
 }
 
+namespace CypressRetriesSpec {
+  Cypress.config('retries', {
+    openMode: 0,
+    runMode: 1
+  })
+
+  Cypress.config('retries', {
+    openMode: false,
+    runMode: false,
+    experimentalStrategy: "detect-flake-and-pass-on-threshold",
+    experimentalOptions: {
+      maxRetries: 2,
+      passesRequired: 2
+    }
+  })
+
+  Cypress.config('retries', {
+    openMode: false,
+    runMode: false,
+    experimentalStrategy: "detect-flake-but-always-fail",
+    experimentalOptions: {
+      maxRetries: 2,
+      stopIfAnyPassed: true
+    }
+  })
+
+  Cypress.config('retries', { openMode: false, runMode: true, experimentalStrategy: "detect-flake-and-pass-on-threshold", experimentalOptions: { maxRetries: 2} }) // $ExpectError
+  Cypress.config('retries', { openMode: false, runMode: true, experimentalStrategy: "detect-flake-but-always-fail", experimentalOptions: { maxRetries: 2} }) // $ExpectError
+
+  Cypress.config('retries', { openMode: false, runMode: true, experimentalStrategy: "detect-flake-and-pass-on-threshold", experimentalOptions: { passesRequired: 2} }) // $ExpectError
+  Cypress.config('retries', { openMode: false, runMode: true, experimentalStrategy: "detect-flake-but-always-fail", experimentalOptions: { stopIfAnyPassed: true} }) // $ExpectError
+}
+
 namespace CypressTraversalTests {
   cy.wrap({}).prevUntil('a') // $ExpectType Chainable<JQuery<HTMLAnchorElement>>
   cy.wrap({}).prevUntil('#myItem') // $ExpectType Chainable<JQuery<HTMLElement>>