From 7532fe8f68f86954b0c5497ca76c93ae7ad6b17e Mon Sep 17 00:00:00 2001 From: Gleb Bahmutov Date: Fri, 13 Oct 2023 09:13:21 -0400 Subject: [PATCH] feat: split specs based on timings from a JSON file (#125) * start work on splitting specs based on timings * testing timings * add readme file --- .github/workflows/ci.yml | 12 +++++ README.md | 33 +++++++++++++ cypress/e2e/timings.cy.js | 98 +++++++++++++++++++++++++++++++++++++++ package.json | 3 +- src/index.js | 49 ++++++++++++++++++-- src/timings.js | 29 ++++++++++++ timings.json | 28 +++++++++++ 7 files changed, 248 insertions(+), 4 deletions(-) create mode 100644 cypress/e2e/timings.cy.js create mode 100644 src/timings.js create mode 100644 timings.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 79a0d51..1bba062 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -129,6 +129,17 @@ jobs: with: command: npm run user-specs + test-timings: + runs-on: ubuntu-22.04 + steps: + - name: Checkout ๐Ÿ›Ž + uses: actions/checkout@v4 + - name: Split specs based on timings json file ๐Ÿงช + # https://github.com/cypress-io/github-action + uses: cypress-io/github-action@v5 + with: + command: npm run timings + test-workflow-e2e: # https://github.com/bahmutov/cypress-workflows uses: bahmutov/cypress-workflows/.github/workflows/split.yml@v1 @@ -173,6 +184,7 @@ jobs: test-user-spec-list, test-subfolder, test-index1, + test-timings, ] runs-on: ubuntu-22.04 steps: diff --git a/README.md b/README.md index b96acd8..17a49c4 100644 --- a/README.md +++ b/README.md @@ -229,6 +229,39 @@ Some CIs provide an agent index that already starts at 1. You can pass it via `S job1: SPLIT=3 SPLIT_INDEX1=1 npx cypress run ``` +## Split specs based on timings + +If you know the spec timings, you can create a JSON file and pass the timings to this plugin. The list of specs will be split into N machines to make the total durations for each machine approximately equal. You can see an example [timings.json](./timings.json) file: + +```json +{ + "durations": [ + { + "spec": "cypress/e2e/chunks.cy.js", + "duration": 300 + }, + { + "spec": "cypress/e2e/spec-a.cy.js", + "duration": 10050 + }, + ... + ] +} +``` + +You can pass the JSON filename via `SPLIT_FILE` environment variable or Cypress`env` variable. + +``` +# split all specs across 3 machines using known spec timings +# loaded from "timings.json" file +$ SPLIT_FILE=timings.json SPLIT=3 npx cypress run + +# the equivalent syntax using Cypress --env argument +$ npx cypress run --env split=3,splitFile=timings.json +``` + +For specs not in the timings file, it will use average duration of the known specs. + ## CI summary To skip GitHub Actions summary, set an environment variable `SPLIT_SUMMARY=false`. By default, this plugin generates the summary. diff --git a/cypress/e2e/timings.cy.js b/cypress/e2e/timings.cy.js new file mode 100644 index 0000000..2bed497 --- /dev/null +++ b/cypress/e2e/timings.cy.js @@ -0,0 +1,98 @@ +/// + +const { splitByDuration } = require('../../src/timings') + +it('splits specs based on timings', () => { + const list = [ + { + spec: 'a', + duration: 1000, + }, + { + spec: 'b', + duration: 6000, + }, + { + spec: 'c', + duration: 6000, + }, + { + spec: 'd', + duration: 1000, + }, + ] + const { chunks, sums } = splitByDuration(2, list) + expect(chunks, 'chunks').to.deep.equal([ + [ + { + spec: 'b', + duration: 6000, + }, + { + spec: 'a', + duration: 1000, + }, + ], + [ + { + spec: 'c', + duration: 6000, + }, + { + spec: 'd', + duration: 1000, + }, + ], + ]) + expect(sums, 'duration sums').to.deep.equal([7000, 7000]) +}) + +it('splits specs based on timings, single result', () => { + const list = [ + { + spec: 'a', + duration: 1000, + }, + { + spec: 'b', + duration: 6000, + }, + { + spec: 'c', + duration: 6000, + }, + { + spec: 'd', + duration: 1000, + }, + ] + const { chunks, sums } = splitByDuration(4, list) + expect(chunks).to.deep.equal([ + [ + { + spec: 'b', + duration: 6000, + }, + ], + [ + { + spec: 'c', + duration: 6000, + }, + ], + [ + { + spec: 'a', + duration: 1000, + }, + ], + + [ + { + spec: 'd', + duration: 1000, + }, + ], + ]) + expect(sums, 'duration sums').to.deep.equal([6000, 6000, 1000, 1000]) +}) diff --git a/package.json b/package.json index d73f240..ad6fa3c 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,8 @@ "test-names": "find-cypress-specs --names", "test-names:component": "find-cypress-specs --component --names", "deps": "npm audit --report --omit dev", - "subfolder": "DEBUG=cypress-split,find-cypress-specs SPLIT=2 SPLIT_INDEX=0 cypress run --config-file examples/my-app/tests/cypress.config.js" + "subfolder": "DEBUG=cypress-split,find-cypress-specs SPLIT=2 SPLIT_INDEX=0 cypress run --config-file examples/my-app/tests/cypress.config.js", + "timings": "DEBUG=cypress-split SPLIT=2 SPLIT_INDEX=0 SPLIT_FILE=timings.json cypress run" }, "repository": { "type": "git", diff --git a/src/index.js b/src/index.js index 5c3df21..34e271b 100644 --- a/src/index.js +++ b/src/index.js @@ -6,6 +6,7 @@ const { getSpecs } = require('find-cypress-specs') const ghCore = require('@actions/core') const cTable = require('console.table') const { getChunk } = require('./chunk') +const { splitByDuration } = require('./timings') const { getEnvironmentFlag } = require('./utils') const path = require('path') const os = require('os') @@ -68,6 +69,7 @@ function cypressSplit(on, config) { let SPLIT = process.env.SPLIT || config.env.split || config.env.SPLIT let SPLIT_INDEX = process.env.SPLIT_INDEX || config.env.splitIndex + let SPLIT_FILE = process.env.SPLIT_FILE || config.env.splitFile // some CI systems like TeamCity provide agent index starting with 1 // let's check for SPLIT_INDEX1 and if it is set, @@ -84,6 +86,7 @@ function cypressSplit(on, config) { // potentially a list of files to run / split let SPEC = process.env.SPEC || config.env.spec || config.env.SPEC + /** @type {string[]|undefined} absolute spec filenames */ let specs if (typeof SPEC === 'string' && SPEC) { specs = SPEC.split(',') @@ -146,12 +149,52 @@ function cypressSplit(on, config) { debug('get chunk %o', { specs, splitN, splitIndex }) /** @type {string[]} absolute spec filenames */ - const splitSpecs = getChunk(specs, splitN, splitIndex) - debug('split specs') - debug(splitSpecs) + let splitSpecs const cwd = process.cwd() console.log('spec from the current directory %s', cwd) + + if (SPLIT_FILE) { + debug('loading split file %s', SPLIT_FILE) + const splitFile = JSON.parse(fs.readFileSync(SPLIT_FILE, 'utf8')) + const previousDurations = splitFile.durations + const averageDuration = + previousDurations + .map((item) => item.duration) + .reduce((sum, duration) => (sum += duration), 0) / + previousDurations.length + const specsWithDurations = specs.map((specName) => { + const relativeSpec = path.relative(cwd, specName) + const foundInfo = previousDurations.find( + (item) => item.spec === relativeSpec, + ) + if (!foundInfo) { + return { + specName, + duration: averageDuration, + } + } else { + return { + specName, + duration: foundInfo.duration, + } + } + }) + debug('splitting by duration %d ways', splitN) + debug(specsWithDurations) + const { chunks, sums } = splitByDuration(splitN, specsWithDurations) + debug('split by duration') + debug(chunks) + debug('sums of durations for chunks') + debug(sums) + + splitSpecs = chunks[splitIndex].map((item) => item.specName) + } else { + splitSpecs = getChunk(specs, splitN, splitIndex) + } + debug('split specs') + debug(splitSpecs) + const nameRows = splitSpecs.map((specName, k) => { const row = [String(k + 1), path.relative(cwd, specName)] return row diff --git a/src/timings.js b/src/timings.js new file mode 100644 index 0000000..55a4c3d --- /dev/null +++ b/src/timings.js @@ -0,0 +1,29 @@ +/** + * Split the list of items into "n" lists by "duration" property + * in each item. Sorts the list first, then round-robin fills + * the lists. Put the item into the list with the smallest sum. + * @param {number} n Number of output lists + * @returns {any} + */ +function splitByDuration(n, list) { + const result = [] + const sums = [] + for (let k = 0; k < n; k += 1) { + result.push([]) + sums.push(0) + } + const sorted = list.sort((a, b) => b.duration - a.duration) + sorted.forEach((item) => { + const smallestIndex = sums.reduce((currentSmallestIndex, value, index) => { + return value < sums[currentSmallestIndex] ? index : currentSmallestIndex + }, 0) + result[smallestIndex].push(item) + sums[smallestIndex] += item.duration + }) + + // console.table(result) + // console.table(sums) + return { chunks: result, sums } +} + +module.exports = { splitByDuration } diff --git a/timings.json b/timings.json new file mode 100644 index 0000000..8e47258 --- /dev/null +++ b/timings.json @@ -0,0 +1,28 @@ +{ + "durations": [ + { + "spec": "cypress/e2e/chunks.cy.js", + "duration": 300 + }, + { + "spec": "cypress/e2e/spec-a.cy.js", + "duration": 10050 + }, + { + "spec": "cypress/e2e/spec-b.cy.js", + "duration": 10100 + }, + { + "spec": "cypress/e2e/spec-c.cy.js", + "duration": 10060 + }, + { + "spec": "cypress/e2e/spec-d.cy.js", + "duration": 10070 + }, + { + "spec": "cypress/e2e/timings.cy.js", + "duration": 100 + } + ] +}