refactor to have geckodriver launch the browser and split out webdriv…

…er to own class [run ci]

.circleci/workflows.yml

@@ -30,7 +30,7 @@ mainBuildFilters: &mainBuildFilters
         - /^release\/\d+\.\d+\.\d+$/
         # use the following branch as well to ensure that v8 snapshot cache updates are fully tested
         - 'update-v8-snapshot-cache-on-develop'
-        - 'ryanm/chore/fix-full-snapshot'
+        - 'misc/remove_marionette_for_geckodriver'
         - 'publish-binary'
 
 # usually we don't build Mac app - it takes a long time
@@ -42,7 +42,7 @@ macWorkflowFilters: &darwin-workflow-filters
     - equal: [ develop, << pipeline.git.branch >> ]
     # use the following branch as well to ensure that v8 snapshot cache updates are fully tested
     - equal: [ 'update-v8-snapshot-cache-on-develop', << pipeline.git.branch >> ]
-    - equal: [ 'ryanm/chore/fix-full-snapshot', << pipeline.git.branch >> ]
+    - equal: [ 'misc/remove_marionette_for_geckodriver', << pipeline.git.branch >> ]
     - matches:
         pattern: /^release\/\d+\.\d+\.\d+$/
         value: << pipeline.git.branch >>
@@ -53,7 +53,7 @@ linuxArm64WorkflowFilters: &linux-arm64-workflow-filters
     - equal: [ develop, << pipeline.git.branch >> ]
     # use the following branch as well to ensure that v8 snapshot cache updates are fully tested
     - equal: [ 'update-v8-snapshot-cache-on-develop', << pipeline.git.branch >> ]
-    - equal: [ 'ryanm/chore/fix-full-snapshot', << pipeline.git.branch >> ]
+    - equal: [ 'misc/remove_marionette_for_geckodriver', << pipeline.git.branch >> ]
     - matches:
         pattern: /^release\/\d+\.\d+\.\d+$/
         value: << pipeline.git.branch >>
@@ -76,7 +76,7 @@ windowsWorkflowFilters: &windows-workflow-filters
     - equal: [ develop, << pipeline.git.branch >> ]
     # use the following branch as well to ensure that v8 snapshot cache updates are fully tested
     - equal: [ 'update-v8-snapshot-cache-on-develop', << pipeline.git.branch >> ]
-    - equal: [ 'ryanm/chore/fix-full-snapshot', << pipeline.git.branch >> ]
+    - equal: [ 'misc/remove_marionette_for_geckodriver', << pipeline.git.branch >> ]
     - matches:
         pattern: /^release\/\d+\.\d+\.\d+$/
         value: << pipeline.git.branch >>
@@ -152,7 +152,7 @@ commands:
           name: Set environment variable to determine whether or not to persist artifacts
           command: |
             echo "Setting SHOULD_PERSIST_ARTIFACTS variable"
-            echo 'if ! [[ "$CIRCLE_BRANCH" != "develop" && "$CIRCLE_BRANCH" != "release/"* && "$CIRCLE_BRANCH" != "ryanm/chore/fix-full-snapshot" ]]; then
+            echo 'if ! [[ "$CIRCLE_BRANCH" != "develop" && "$CIRCLE_BRANCH" != "release/"* && "$CIRCLE_BRANCH" != "misc/remove_marionette_for_geckodriver" ]]; then
                 export SHOULD_PERSIST_ARTIFACTS=true
             fi' >> "$BASH_ENV"
   # You must run `setup_should_persist_artifacts` command and be using bash before running this command

packages/launcher/lib/browsers.ts

@@ -28,7 +28,7 @@ export function launch (
 
   const spawnOpts: cp.SpawnOptionsWithStdioTuple<cp.StdioNull, cp.StdioPipe, cp.StdioPipe> = {
     stdio: ['ignore', 'pipe', 'pipe'],
-    // allow setting default env vars such as MOZ_HEADLESS_WIDTH
+    // allow setting default env vars
     // but only if it's not already set by the environment
     env: { ...browserEnv, ...process.env },
   }

packages/server/lib/browsers/firefox-util.ts

@@ -7,8 +7,8 @@ import * as protocol from './protocol'
 import { CdpAutomation } from './cdp_automation'
 import { BrowserCriClient } from './browser-cri-client'
 import type { Automation } from '../automation'
-import { GeckoDriver } from './geckodriver'
 import type { CypressError } from '@packages/errors'
+import type { WebDriverClassic } from './webdriver-classic'
 
 const debug = Debug('cypress:server:browsers:firefox-util')
 
@@ -20,7 +20,7 @@ let timings = {
   collections: [] as any[],
 }
 
-let geckoDriver: GeckoDriver
+let webDriverClassic: WebDriverClassic
 
 const getTabId = (tab) => {
   return _.get(tab, 'browsingContextID')
@@ -103,11 +103,11 @@ async function connectToNewTabClassic () {
   // For versions 124 and above, a new tab is not created, so @packages/extension creates one for us.
   // Since the tab is always available on our behalf,
   // we can connect to it here and navigate it to about:blank to set it up for CDP connection
-  const handles = await geckoDriver.getWindowHandlesWebDriverClassic()
+  const handles = await webDriverClassic.getWindowHandles()
 
-  await geckoDriver.switchToWindowWebDriverClassic(handles[0])
+  await webDriverClassic.switchToWindow(handles[0])
 
-  await geckoDriver.navigateWebdriverClassic('about:blank')
+  await webDriverClassic.navigate('about:blank')
 }
 
 async function connectToNewSpec (options, automation: Automation, browserCriClient: BrowserCriClient) {
@@ -140,7 +140,7 @@ async function setupCDP (remotePort: number, automation: Automation, onError?: (
 }
 
 async function navigateToUrlClassic (url: string) {
-  await geckoDriver.navigateWebdriverClassic(url)
+  await webDriverClassic.navigate(url)
 }
 
 const logGcDetails = () => {
@@ -209,46 +209,23 @@ export default {
 
   async setup ({
     automation,
-    extensions,
     onError,
     url,
     foxdriverPort,
-    marionettePort,
-    geckoDriverPort,
     remotePort,
-    browserName,
-    profilePath,
-    binaryPath,
-    isHeadless,
-    isTextTerminal,
+    webDriverClassic: wdcInstance,
   }: {
     automation: Automation
-    extensions: string[]
     onError?: (err: Error) => void
     url: string
     foxdriverPort: number
-    marionettePort: number
-    geckoDriverPort: number
     remotePort: number
-    browserName: string
-    profilePath: string
-    binaryPath: string
-    isHeadless: boolean
-    isTextTerminal: boolean
+    webDriverClassic: WebDriverClassic
   }): Promise<BrowserCriClient> {
-    const [,, browserCriClient] = await Promise.all([
+    // set the WebDriver classic instance instantiated from geckodriver
+    webDriverClassic = wdcInstance
+    const [, browserCriClient] = await Promise.all([
       this.setupFoxdriver(foxdriverPort),
-      this.setupGeckoDriver({
-        port: geckoDriverPort,
-        marionettePort,
-        remotePort,
-        browserName,
-        extensions,
-        profilePath,
-        binaryPath,
-        isHeadless,
-        isTextTerminal,
-      }),
       setupCDP(remotePort, automation, onError),
     ])
 
@@ -263,32 +240,6 @@ export default {
 
   setupCDP,
 
-  async setupGeckoDriver (opts: {
-    port: number
-    marionettePort: number
-    remotePort: number
-    browserName: string
-    extensions: string[]
-    profilePath: string
-    binaryPath: string
-    isHeadless: boolean
-    isTextTerminal: boolean
-  }) {
-    geckoDriver = await GeckoDriver.create({
-      host: '127.0.0.1',
-      port: opts.port,
-      marionetteHost: '127.0.0.1',
-      marionettePort: opts.marionettePort,
-      remotePort: opts.remotePort,
-      browserName: opts.browserName,
-      extensions: opts.extensions,
-      profilePath: opts.profilePath,
-      binaryPath: opts.binaryPath,
-      isHeadless: opts.isHeadless,
-      isTextTerminal: opts.isTextTerminal,
-    })
-  },
-
   // NOTE: this is going to be removed in Cypress 14. @see https://github.com/cypress-io/cypress/issues/30222
   async setupFoxdriver (port) {
     await protocol._connectAsync({

packages/server/lib/browsers/firefox.ts

@@ -4,7 +4,7 @@ import Debug from 'debug'
 import getPort from 'get-port'
 import path from 'path'
 import urlUtil from 'url'
-import { debug as launcherDebug, launch } from '@packages/launcher/lib/browsers'
+import { debug as launcherDebug } from '@packages/launcher/lib/browsers'
 import { doubleEscape } from '@packages/launcher/lib/windows'
 import FirefoxProfile from 'firefox-profile'
 import * as errors from '../errors'
@@ -15,13 +15,13 @@ import { EventEmitter } from 'events'
 import os from 'os'
 import treeKill from 'tree-kill'
 import mimeDb from 'mime-db'
-import { getRemoteDebuggingPort } from './protocol'
 import type { BrowserCriClient } from './browser-cri-client'
 import type { Automation } from '../automation'
 import { getCtx } from '@packages/data-context'
 import { getError } from '@packages/errors'
 import type { BrowserLaunchOpts, BrowserNewTabOpts, RunModeVideoApi } from '@packages/types'
 import { GeckoDriver } from './geckodriver'
+import { WebDriverClassic } from './webdriver-classic'
 
 const debug = Debug('cypress:server:browsers:firefox')
 
@@ -355,6 +355,7 @@ toolbar {
 `
 
 let browserCriClient: BrowserCriClient | undefined
+let wdcInstance: WebDriverClassic | undefined
 
 export function _createDetachedInstance (browserInstance: BrowserInstance, browserCriClient?: BrowserCriClient): BrowserInstance {
   const detachedInstance: BrowserInstance = new EventEmitter() as BrowserInstance
@@ -368,9 +369,6 @@ export function _createDetachedInstance (browserInstance: BrowserInstance, brows
       clearInstanceState({ gracefulShutdown: true })
     }
 
-    // make sure to close geckodriver
-    GeckoDriver.close()
-
     treeKill(browserInstance.pid as number, (err?, result?) => {
       debug('force-exit of process tree complete %o', { err, result })
       detachedInstance.emit('exit')
@@ -384,14 +382,13 @@ export function _createDetachedInstance (browserInstance: BrowserInstance, brows
 * Clear instance state for the chrome instance, this is normally called in on kill or on exit.
 */
 export function clearInstanceState (options: GracefulShutdownOptions = {}) {
-  debug('closing remote interface client')
+  debug('clearing instance state')
+
   if (browserCriClient) {
+    debug('closing remote interface client')
     browserCriClient.close(options.gracefulShutdown).catch(() => {})
     browserCriClient = undefined
   }
-
-  // make sure to close geckodriver
-  GeckoDriver.close()
 }
 
 export async function connectToNewSpec (browser: Browser, options: BrowserNewTabOpts, automation: Automation) {
@@ -417,28 +414,24 @@ export async function open (browser: Browser, url: string, options: BrowserLaunc
     extensions: [] as string[],
     preferences: _.extend({}, defaultPreferences),
     args: [
-      '-marionette',
       '-new-instance',
-      '-foreground',
       // if testing against older versions of Firefox to determine when a regression may have been introduced, uncomment the '-allow-downgrade' flag.
       // '-allow-downgrade',
       '-start-debugger-server', // uses the port+host defined in devtools.debugger.remote
       '-no-remote', // @see https://github.com/cypress-io/cypress/issues/6380
     ],
   })
 
-  let remotePort
-
-  remotePort = await getRemoteDebuggingPort()
-
-  defaultLaunchOptions.args.push(`--remote-debugging-port=${remotePort}`)
-
   if (browser.isHeadless) {
     defaultLaunchOptions.args.push('-headless')
-    // we don't need to specify width/height since MOZ_HEADLESS_ env vars will be set
-    // and the browser will spawn maximized. The user may still supply these args to override
-    // defaultLaunchOptions.args.push('--width=1920')
-    // defaultLaunchOptions.args.push('--height=1081')
+
+    // sets headless resolution to 1280x720 by default
+    // user can overwrite this default with these env vars or -height, -width arguments
+    if (!defaultLaunchOptions.args.includes('-width') && !defaultLaunchOptions.args.includes('-height')) {
+      // (height must account for firefox url bar, which we can only shrink to 1px ,
+      // and the total size of the window url and tab bar, which is 85 pixels for a total offset of 86 pixels)
+      defaultLaunchOptions.args.push('-width', '1280', '-height', '806')
+    }
   }
 
   debug('firefox open %o', options)
@@ -473,12 +466,13 @@ export async function open (browser: Browser, url: string, options: BrowserLaunc
     foxdriverPort,
     marionettePort,
     geckoDriverPort,
-  ] = await Promise.all([getPort(), getPort(), getPort()])
+    webDriverBiDiPort,
+  ] = await Promise.all([getPort(), getPort(), getPort(), getPort()])
 
   defaultLaunchOptions.preferences['devtools.debugger.remote-port'] = foxdriverPort
   defaultLaunchOptions.preferences['marionette.port'] = marionettePort
 
-  debug('available ports: %o', { foxdriverPort, marionettePort, geckoDriverPort })
+  debug('available ports: %o', { foxdriverPort, marionettePort, geckoDriverPort, webDriverBiDiPort })
 
   const [
     cacheDir,
@@ -550,23 +544,74 @@ export async function open (browser: Browser, url: string, options: BrowserLaunc
     await fs.writeFile(path.join(profileDir, 'chrome', 'userChrome.css'), userCss)
   }
 
-  launchOptions.args = launchOptions.args.concat([
-    '-profile',
-    profile.path(),
-  ])
+  // create the geckodriver process, which we will use WebDriver Classic to open the browser
+  const browserInstance = await GeckoDriver.create({
+    host: '127.0.0.1',
+    port: geckoDriverPort,
+    marionetteHost: '127.0.0.1',
+    marionettePort,
+    webdriverBidiPort: webDriverBiDiPort,
+    profilePath: profile.path(),
+    binaryPath: browser.path,
+  })
+
+  wdcInstance = new WebDriverClassic('127.0.0.1', geckoDriverPort)
 
   debug('launch in firefox', { url, args: launchOptions.args })
 
-  const browserInstance = launch(browser, 'about:blank', remotePort, launchOptions.args, {
-    // sets headless resolution to 1280x720 by default
-    // user can overwrite this default with these env vars or --height, --width arguments
-    MOZ_HEADLESS_WIDTH: '1280',
-    MOZ_HEADLESS_HEIGHT: '721',
-    ...launchOptions.env,
-  })
+  const capabilitiesToSend = {
+    // browser capabilities are going here as exact
+    capabilities: {
+      alwaysMatch: {
+        acceptInsecureCerts: true,
+        // @see https://developer.mozilla.org/en-US/docs/Web/WebDriver/Capabilities/firefoxOptions
+        'moz:firefoxOptions': {
+          binary: browser.path,
+          args: launchOptions.args,
+          env: {
+            // MOZ_REMOTE_SETTINGS_DEVTOOLS: '1',
+            ...launchOptions.env,
+          },
+          prefs: launchOptions.preferences,
+        },
+        // @see https://firefox-source-docs.mozilla.org/testing/geckodriver/Capabilities.html#moz-debuggeraddress
+        // we specify the the debugger address and webdriver, through firefox added capabilities, will return us the CDP port.
+        'moz:debuggerAddress': true,
+      },
+    },
+  }
+
+  debug(`sending capabilities %s`, JSON.stringify(capabilitiesToSend.capabilities))
+
+  // this command starts the webdriver session and actually opens the browser
+  const { capabilities } = await wdcInstance.createSession(capabilitiesToSend)
+
+  debug(`received capabilities %o`, capabilities)
+
+  const cdpPort = parseInt(new URL(`ws://${capabilities['moz:debuggerAddress']}`).port)
+
+  debug(`CDP running on port ${cdpPort}`)
+
+  const browserPID = capabilities['moz:processID']
+
+  debug(`firefox running on pid: ${browserPID}`)
+
+  // makes it so get getRemoteDebuggingPort() is calculated correctly
+  process.env.CYPRESS_REMOTE_DEBUGGING_PORT = cdpPort.toString()
+
+  // install the browser extensions
+  await Promise.all(_.map(launchOptions.extensions, (path) => {
+    debug(`installing extension at path: ${path}`)
+
+    return wdcInstance!.installAddOn({
+      extensionPath: path,
+      isTemporary: true,
+    })
+  }))
 
   try {
-    browserCriClient = await firefoxUtil.setup({ automation, extensions: launchOptions.extensions, url, foxdriverPort, marionettePort, geckoDriverPort, remotePort, browserName: browser.name, profilePath: profile.path(), binaryPath: browser.path, onError: options.onError, isHeadless: options.browser.isHeadless, isTextTerminal: options.isTextTerminal })
+    debug('setting up firefox utils')
+    browserCriClient = await firefoxUtil.setup({ automation, url, foxdriverPort, webDriverClassic: wdcInstance, remotePort: cdpPort, onError: options.onError })
 
     if (os.platform() === 'win32') {
       // override the .kill method for Windows so that the detached Firefox process closes between specs
@@ -583,6 +628,10 @@ export async function open (browser: Browser, url: string, options: BrowserLaunc
 
       debug('closing firefox')
 
+      process.kill(browserPID)
+
+      debug('closing geckodriver')
+
       return originalBrowserKill.apply(browserInstance, args)
     }