chore: adds quality gate for rerunning e2e spec files that are new or have been modified

development/lib/retry.js

@@ -58,6 +58,10 @@ async function retry(
     }
   }
 
+  if (retryUntilFailure) {
+    return null;
+  }
+
   throw new Error(rejectionMessage);
 }
 

test/e2e/fetch-changed-files.js

@@ -0,0 +1,33 @@
+const axios = require('axios');
+
+const PR_URL = `${process.env.CIRCLE_PULL_REQUEST}`;
+const PR_NUMBER = PR_URL.match(/\/(\d+)$/u)[1];
+
+const GH_API_URL = `https://api.github.com/repos/MetaMask/metamask-extension/pulls/${PR_NUMBER}/files`;
+const headers = {
+  Accept: 'application/vnd.github+json',
+  Authorization: `Bearer ${process.env.GITHUB_TOKEN}`,
+  'X-GitHub-Api-Version': '2022-11-28',
+};
+
+async function fetchChangedE2eFiles() {
+  try {
+    const response = await axios.get(GH_API_URL, { headers });
+    const filesChanged = response.data
+      .filter(
+        (file) =>
+          file.filename.startsWith('test/e2e/') &&
+          (file.filename.endsWith('.spec.js') ||
+            file.filename.endsWith('.spec.ts')),
+      )
+      .map((file) => file.filename)
+      .join('\n');
+
+    return filesChanged;
+  } catch (error) {
+    console.error('Error making request:', error);
+    throw error;
+  }
+}
+
+module.exports = { fetchChangedE2eFiles };

test/e2e/run-all.js

@@ -6,6 +6,7 @@ const { hideBin } = require('yargs/helpers');
 const { runInShell } = require('../../development/lib/run-command');
 const { exitWithError } = require('../../development/lib/exit-with-error');
 const { loadBuildTypesConfig } = require('../../development/lib/build-type');
+const { fetchChangedE2eFiles } = require('./fetch-changed-files');
 
 // These tests should only be run on Flask for now.
 const FLASK_ONLY_TESTS = ['test-snap-namelookup.spec.js'];
@@ -212,12 +213,31 @@ async function main() {
 
   console.log('My test list:', myTestList);
 
+  let changedOrNewTests = '';
+  try {
+    changedOrNewTests = await fetchChangedE2eFiles();
+  } catch (error) {
+    console.error('Error fetching changed e2e files:', error);
+  }
+  console.log('Spec files that will be re-run:', changedOrNewTests);
+
   // spawn `run-e2e-test.js` for each test in myTestList
   for (let testPath of myTestList) {
     if (testPath !== '') {
       testPath = testPath.replace('\n', ''); // sometimes there's a newline at the end of the testPath
       console.log(`\nExecuting testPath: ${testPath}\n`);
-      await runInShell('node', [...args, testPath]);
+
+      const testFileName = testPath.split('/').pop();
+      const isTestChangedOrNew = changedOrNewTests.includes(testFileName);
+      const retryIndex = args.indexOf('--retries');
+      if (retryIndex !== -1) {
+        args.splice(retryIndex, 2);
+      }
+
+      const extraArgs = isTestChangedOrNew
+        ? ['--retry-until-failure', '--retries=5']
+        : [];
+      await runInShell('node', [...args, ...extraArgs, testPath]);
     }
   }
 }

test/e2e/tests/signature/eth-sign.spec.js

@@ -11,7 +11,7 @@ const {
 const FixtureBuilder = require('../../fixture-builder');
 
 describe('Eth sign', function () {
-  it('will detect if eth_sign is disabled', async function () {
+  it('CHANGED TO TEST CI -will detect if eth_sign is disabled', async function () {
     await withFixtures(
       {
         dapp: true,
@@ -25,7 +25,7 @@ describe('Eth sign', function () {
         await unlockWallet(driver);
 
         await openDapp(driver);
-        await driver.clickElement('#ethSign');
+        await driver.clickElement('#BAD-ID-TO-FAIL-ON-PURPOSE');
 
         await driver.delay(1000);
         const ethSignButton = await driver.findElement('#ethSign');

test/e2e/tests/signature/personal-sign.spec.js

@@ -10,7 +10,7 @@ const {
 const FixtureBuilder = require('../../fixture-builder');
 
 describe('Personal sign', function () {
-  it('can initiate and confirm a personal sign', async function () {
+  it('CHANGED TO TEST CI -can initiate and confirm a personal sign', async function () {
     await withFixtures(
       {
         dapp: true,