Merge pull request Expensify#30484 from margelo/e2e/increase-reliability

[NoQA] [DEV] E2e/increase reliability

.prettierignore

@@ -15,3 +15,5 @@ package-lock.json
 *.css
 *.scss
 *.md
+# We need to modify the import here specifically, hence we disable prettier to get rid of the sorted imports
+src/libs/E2E/reactNativeLaunchingTest.js

src/libs/E2E/reactNativeLaunchingTest.js

@@ -7,7 +7,6 @@
  */
 import * as Metrics from '@libs/Metrics';
 import Performance from '@libs/Performance';
-import '../../../index';
 import E2EConfig from '../../../tests/e2e/config';
 import E2EClient from './client';
 
@@ -65,5 +64,5 @@ E2EClient.getTestConfig()
 
 // start the usual app
 Performance.markStart('regularAppStart');
-
+import '../../../index';
 Performance.markEnd('regularAppStart');

tests/e2e/compare/compare.js

@@ -24,7 +24,7 @@ const PROBABILITY_CONSIDERED_SIGNIFICANCE = 0.02;
  * This is additional filter, in addition to probability threshold above.
  * Too small duration difference might be result of measurement grain of 1 ms.
  */
-const DURATION_DIFF_THRESHOLD_SIGNIFICANCE = 50;
+const DURATION_DIFF_THRESHOLD_SIGNIFICANCE = 100;
 
 const loadFile = (path) =>
     fs.readFile(path, 'utf8').then((data) => {

tests/e2e/config.js

@@ -31,7 +31,7 @@ module.exports = {
     SERVER_PORT: 4723,
 
     // The amount of times a test should be executed for average performance metrics
-    RUNS: 60,
+    RUNS: 80,
 
     DEFAULT_BASELINE_BRANCH: 'main',
 
@@ -46,6 +46,9 @@ module.exports = {
     // The time in milliseconds after which an operation fails due to timeout
     INTERACTION_TIMEOUT: 300000,
 
+    // Period we wait between each test runs, to let the device cool down
+    COOL_DOWN: 90 * 1000,
+
     TEST_NAMES,
 
     /**

tests/e2e/testRunner.js

@@ -250,6 +250,15 @@ const runTests = async () => {
             }
         }
         testLog.done();
+
+        // If we still have tests add a cool down period
+        if (testIndex < numOfTests - 1) {
+            const coolDownLogs = Logger.progressInfo(`Cooling down for ${config.COOL_DOWN / 1000}s`);
+            coolDownLogs.updateText(`Cooling down for ${config.COOL_DOWN / 1000}s`);
+            // eslint-disable-next-line no-loop-func
+            await new Promise((resolve) => setTimeout(resolve, config.COOL_DOWN));
+            coolDownLogs.done();
+        }
     }
 
     // Calculate statistics and write them to our work file