allow list of search patterns in testFiles config option (#5402)

* WIP: log spec search steps

* add multiple files via glob test

* allow testFiles to be a string or a list of strings to match

* failing config tests: testFiles type change

Change `testFiles` from a string to either a string **or** array of strings; following similar pattern as `ignoreTestFiles`

* testFiles type change passing tests

* update verbiage related to testFiles type change

* enable running integration tests locally

See paulmillr/chokidar#855 for more details.

* integration tests; failing

Seems to fail for similar reason as #4543 and is likely resolved with PR #4849

* correct test verbiage

Distinguish between the two tests; as they are subtly different.

* update TS types for testFiles type change

Updated the comments, but forgot to update the actual type. This commit rectifies that.

* correct CLI syntax in test

`--config` was not being passed the appropriate value. Now matches other test cases; such as that found in [args_spec](https://github.com/cypress-io/cypress/blob/92b91fe514e5ff6286b4d4e26d2df23062bdf869/packages/server/test/unit/args_spec.coffee#L210)

* remove unsupported syntax from tests

From looking at other tests, it does not appear that the syntax `--config=testFiles=glob1,glob2` is current supported. Removing the test for this test case.

* test the correct config value

Previously was testing `ignoreTestFiles` instead of `testFies`

* provide more accurate test assertion

* correct config's `testFiles` type

Should be a string or an array of strings.

* remove unnecessary newline

* tweak verbiage to better align with intention

Co-Authored-By: Ben Kucera <14625260+Bkucera@users.noreply.github.com>


Co-authored-by: Andrew Smith <andrew@andrew.codes>

cli/schema/cypress.schema.json

@@ -43,9 +43,12 @@
       "description": "The reporter options used. Supported options depend on the reporter. See https://on.cypress.io/reporters#Reporter-Options"
     },
     "testFiles": {
-      "type": "string",
+      "type": [
+        "string",
+        "array"
+      ],
       "default": "**/*.*",
-      "description": "A String glob pattern of the test files to load"
+      "description": "A String or Array of string glob patterns of the test files to load. See https://on.cypress.io/configuration#Global"
     },
     "watchForFileChanges": {
       "type": "boolean",

cli/types/cypress-npm-api.d.ts

@@ -43,9 +43,9 @@ declare module 'cypress' {
      */
     reporter: string,
     /**
-     * A String glob pattern of the test files to load.
+     * A String or Array of string glob pattern of the test files to load.
      */
-    testFiles: string
+    testFiles: string | string[]
 
     //
     // timeouts
@@ -290,7 +290,7 @@ declare module 'cypress' {
     })
     ```
    */
-  interface CypressOpenOptions extends CypressCommonOptions  {
+  interface CypressOpenOptions extends CypressCommonOptions {
     /**
      * Specify a filesystem path to a custom browser
      */

packages/server/lib/config.coffee

@@ -154,7 +154,7 @@ validationRules = {
   reporter: v.isString
   requestTimeout: v.isNumber
   responseTimeout: v.isNumber
-  testFiles: v.isString
+  testFiles: v.isStringOrArrayOfStrings
   supportFile: v.isStringOrFalse
   taskTimeout: v.isNumber
   trashAssetsBeforeRuns: v.isBoolean

packages/server/lib/util/specs.js

@@ -14,6 +14,7 @@ const path = require('path')
 const check = require('check-more-types')
 const debug = require('debug')('cypress:server:specs')
 const minimatch = require('minimatch')
+const Promise = require('bluebird')
 const glob = require('./glob')
 
 const MINIMATCH_OPTIONS = { dot: true, matchBase: true }
@@ -24,7 +25,10 @@ const getPatternRelativeToProjectRoot = (specPattern, projectRoot) => {
   })
 }
 
-const find = function (config, specPattern) {
+/**
+ * Finds all spec files that pass the config.
+ */
+const find = function findSpecs (config, specPattern) {
   let fixturesFolderPath
 
   la(check.maybe.strings(specPattern), 'invalid spec pattern', specPattern)
@@ -36,6 +40,12 @@ const find = function (config, specPattern) {
     integrationFolderPath
   )
 
+  if (specPattern) {
+    debug('spec pattern "%s"', specPattern)
+  } else {
+    debug('there is no spec pattern')
+  }
+
   //# support files are not automatically
   //# ignored because only _fixtures are hard
   //# coded. the rest is simply whatever is in
@@ -132,17 +142,31 @@ const find = function (config, specPattern) {
     .value()
   }
 
-  //# grab all the files
-  return glob(config.testFiles, options)
+  // grab all the files
+  debug('globbing test files "%s"', config.testFiles)
+  debug('glob options %o', options)
+
+  // ensure we handle either a single string or a list of strings the same way
+  const testFilesPatterns = [].concat(config.testFiles)
+
+  /**
+   * Finds matching files for the given pattern, filters out specs to be ignored.
+   */
+  const findOnePattern = (pattern) => {
+    return glob(pattern, options)
+    .tap(debug)
+
+    // filter out anything that matches our
+    // ignored test files glob
+    .filter(doesNotMatchAllIgnoredPatterns)
+    .filter(matchesSpecPattern)
+    .map(setNameParts)
+    .tap((files) => {
+      return debug('found %d spec files: %o', files.length, files)
+    })
+  }
 
-  //# filter out anything that matches our
-  //# ignored test files glob
-  .filter(doesNotMatchAllIgnoredPatterns)
-  .filter(matchesSpecPattern)
-  .map(setNameParts)
-  .tap((files) => {
-    return debug('found %d spec files: %o', files.length, files)
-  })
+  return Promise.mapSeries(testFilesPatterns, findOnePattern).then(_.flatten)
 }
 
 module.exports = {

packages/server/package.json

@@ -53,7 +53,7 @@
     "chalk": "2.4.2",
     "charset": "1.0.1",
     "check-more-types": "2.24.0",
-    "chokidar": "3.0.2",
+    "chokidar": "3.2.2",
     "chrome-remote-interface": "0.28.0",
     "cjsxify": "0.3.0",
     "cli-table3": "0.5.1",

packages/server/test/integration/cypress_spec.coffee

@@ -336,6 +336,20 @@ describe "lib/cypress", ->
         expect(browsers.open).to.be.calledWithMatch(ELECTRON_BROWSER, {url: "http://localhost:8888/__/#/tests/integration/test2.coffee"})
         @expectExitWith(0)
 
+    it "runs project by limiting spec files via config.testFiles string glob pattern", ->
+      cypress.start(["--run-project=#{@todosPath}", "--config=testFiles=#{@todosPath}/tests/test2.coffee"])
+      .then =>
+        expect(browsers.open).to.be.calledWithMatch(ELECTRON_BROWSER, {url: "http://localhost:8888/__/#/tests/integration/test2.coffee"})
+        @expectExitWith(0)
+
+    it "runs project by limiting spec files via config.testFiles as a JSON array of string glob patterns", ->
+      cypress.start(["--run-project=#{@todosPath}", "--config=testFiles=[\"**/test2.coffee\",\"**/test1.js\"]"])
+      .then =>
+        expect(browsers.open).to.be.calledWithMatch(ELECTRON_BROWSER, {url: "http://localhost:8888/__/#/tests/integration/test2.coffee"})
+      .then =>
+        expect(browsers.open).to.be.calledWithMatch(ELECTRON_BROWSER, {url: "http://localhost:8888/__/#/tests/integration/test1.js"})
+        @expectExitWith(0)
+
     it "does not watch settings or plugins in run mode", ->
       watch = sinon.spy(Watchers.prototype, "watch")
       watchTree = sinon.spy(Watchers.prototype, "watchTree")

packages/server/test/unit/config_spec.coffee

@@ -225,11 +225,11 @@ describe "lib/config", ->
 
         it "fails if not a string or array", ->
           @setup({ignoreTestFiles: 5})
-          @expectValidationFails("be a string or an array of string")
+          @expectValidationFails("be a string or an array of strings")
 
         it "fails if not an array of strings", ->
           @setup({ignoreTestFiles: [5]})
-          @expectValidationFails("be a string or an array of string")
+          @expectValidationFails("be a string or an array of strings")
           @expectValidationFails("the value was: `[5]`")
 
       context "integrationFolder", ->
@@ -322,9 +322,18 @@ describe "lib/config", ->
           @setup({testFiles: "**/*.coffee"})
           @expectValidationPasses()
 
-        it "fails if not a string", ->
+        it "passes if an array of strings", ->
+          @setup({testFiles: ["**/*.coffee", "**/*.jsx"]})
+          @expectValidationPasses()
+
+        it "fails if not a string or array", ->
           @setup({testFiles: 42})
-          @expectValidationFails("be a string")
+          @expectValidationFails("be a string or an array of strings")
+
+        it "fails if not an array of strings", ->
+          @setup({testFiles: [5]})
+          @expectValidationFails("be a string or an array of strings")
+          @expectValidationFails("the value was: `[5]`")
 
       context "supportFile", ->
         it "passes if a string", ->
@@ -435,11 +444,11 @@ describe "lib/config", ->
 
         it "fails if not a string or array", ->
           @setup({blacklistHosts: 5})
-          @expectValidationFails("be a string or an array of string")
+          @expectValidationFails("be a string or an array of strings")
 
         it "fails if not an array of strings", ->
           @setup({blacklistHosts: [5]})
-          @expectValidationFails("be a string or an array of string")
+          @expectValidationFails("be a string or an array of strings")
           @expectValidationFails("the value was: `[5]`")
 
   context ".getConfigKeys", ->

packages/server/test/unit/specs_spec.coffee

@@ -6,6 +6,7 @@ files = require("#{root}lib/files")
 config = require("#{root}lib/config")
 specsUtil = require("#{root}lib/util/specs")
 FixturesHelper = require("#{root}/test/support/helpers/fixtures")
+debug = require("debug")("test")
 
 describe "lib/util/specs", ->
   beforeEach ->
@@ -56,23 +57,48 @@ describe "lib/util/specs", ->
         expect(files.length).to.equal(1)
         expect(files[0].name).to.equal("coffee_spec.coffee")
 
+    it "uses glob to process config.testFiles", ->
+      config.get(FixturesHelper.projectPath("various-file-types"))
+      .then (cfg) ->
+        cfg.testFiles = "{coffee_*.coffee,js_spec.js}"
+        specsUtil.find(cfg)
+      .then (files) ->
+        debug("found spec files %o", files)
+        expect(files.length).to.equal(2)
+        expect(files[0].name).to.equal("coffee_spec.coffee")
+        expect(files[1].name).to.equal("js_spec.js")
+
+    it "allows array in config.testFiles", ->
+      config.get(FixturesHelper.projectPath("various-file-types"))
+      .then (cfg) ->
+        cfg.testFiles = ["coffee_*.coffee", "js_spec.js"]
+        specsUtil.find(cfg)
+      .then (files) ->
+        debug("found spec files %o", files)
+        expect(files.length).to.equal(2)
+        expect(files[0].name).to.equal("coffee_spec.coffee")
+        expect(files[1].name).to.equal("js_spec.js")
+
     it "filters using specPattern", ->
       config.get(FixturesHelper.projectPath("various-file-types"))
       .then (cfg) ->
-        specsUtil.find(cfg, [
+        specPattern = [
           path.join(cfg.projectRoot, "cypress", "integration", "js_spec.js")
-        ])
+        ]
+        specsUtil.find(cfg, specPattern)
       .then (files) ->
         expect(files.length).to.equal(1)
         expect(files[0].name).to.equal("js_spec.js")
 
     it "filters using specPattern as array of glob patterns", ->
       config.get(FixturesHelper.projectPath("various-file-types"))
       .then (cfg) ->
-        specsUtil.find(cfg, [
+        debug("test config testFiles is %o", cfg.testFiles)
+        specPattern = [
           path.join(cfg.projectRoot, "cypress", "integration", "js_spec.js")
           path.join(cfg.projectRoot, "cypress", "integration", "ts*")
-        ])
+        ]
+        specsUtil.find(cfg, specPattern)
       .then (files) ->
         expect(files.length).to.equal(2)
         expect(files[0].name).to.equal("js_spec.js")