ci(e2e): Migrate tests to Jest

.eslintrc.js

@@ -65,28 +65,6 @@ module.exports = {
 				'valid-jsdoc': 'off',
 			},
 		},
-		{
-			plugins: [ 'mocha' ],
-			files: [ 'test/e2e/**/*', 'packages/magellan-mocha-plugin/test_support/**/*' ],
-			rules: {
-				'import/no-nodejs-modules': 'off',
-				'mocha/no-exclusive-tests': 'error',
-				'mocha/handle-done-callback': [ 'error', { ignoreSkipped: true } ],
-				'mocha/no-global-tests': 'error',
-				'mocha/no-async-describe': 'error',
-				'mocha/no-top-level-hooks': 'error',
-				'mocha/max-top-level-suites': [ 'error', { limit: 1 } ],
-				'no-console': 'off',
-				// Disable all rules from "plugin:jest/recommended", as e2e tests use mocha
-				...Object.keys( require( 'eslint-plugin-jest' ).configs.recommended.rules ).reduce(
-					( disabledRules, key ) => ( { ...disabledRules, [ key ]: 'off' } ),
-					{}
-				),
-			},
-			globals: {
-				step: false,
-			},
-		},
 		merge(
 			// ESLint doesn't allow the `extends` field inside `overrides`, so we need to compose
 			// the TypeScript config manually using internal bits from various plugins

.teamcity/_self/projects/WebApp.kt

@@ -66,6 +66,7 @@ object RunCalypsoE2eDesktopTests : BuildType({
 				export NODE_CONFIG_ENV=test
 				export TEST_VIDEO=true
 				export HIGHLIGHT_ELEMENT=true
+				export TEAMCITY_VERSION=2020
 
 				# Instructs Magellan to not hide the output from individual `mocha` processes. This is required for
 				# mocha-teamcity-reporter to work.
@@ -104,7 +105,8 @@ object RunCalypsoE2eDesktopTests : BuildType({
 				export BROWSERLOCALE="en"
 				export NODE_CONFIG="{\"calypsoBaseURL\":\"${'$'}{URL%/}\"}"
 
-				yarn magellan --config=magellan-calypso.json --max_workers=%E2E_WORKERS% --local_browser=chrome --mocha_args="--reporter mocha-multi-reporters --reporter-options configFile=mocha-reporter.json"
+				# --reporters can't be the last arg, see https://github.com/facebook/jest/issues/10257
+				yarn jest --reporters=jest-teamcity --reporters=default --testNamePattern @parallel --maxWorkers=%E2E_WORKERS% specs/specs-calypso
 			""".trimIndent()
 			dockerImage = "%docker_image_e2e%"
 			dockerRunParameters = "-u %env.UID% --security-opt seccomp=.teamcity/docker-seccomp.json --shm-size=8gb"
@@ -228,6 +230,7 @@ object RunCalypsoE2eMobileTests : BuildType({
 				export NODE_CONFIG_ENV=test
 				export TEST_VIDEO=true
 				export HIGHLIGHT_ELEMENT=true
+				export TEAMCITY_VERSION=2020
 
 				# Instructs Magellan to not hide the output from individual `mocha` processes. This is required for
 				# mocha-teamcity-reporter to work.
@@ -266,7 +269,8 @@ object RunCalypsoE2eMobileTests : BuildType({
 				export BROWSERLOCALE="en"
 				export NODE_CONFIG="{\"calypsoBaseURL\":\"${'$'}{URL%/}\"}"
 
-				yarn magellan --config=magellan-calypso.json --max_workers=%E2E_WORKERS% --local_browser=chrome --mocha_args="--reporter mocha-multi-reporters --reporter-options configFile=mocha-reporter.json"
+				# --reporters can't be the last arg, see https://github.com/facebook/jest/issues/10257
+				yarn jest --reporters=jest-teamcity --reporters=default --testNamePattern @parallel --maxWorkers=%E2E_WORKERS% specs/specs-calypso
 			""".trimIndent()
 			dockerImage = "%docker_image_e2e%"
 			dockerRunParameters = "-u %env.UID% --security-opt seccomp=.teamcity/docker-seccomp.json --shm-size=8gb"

test/e2e/.eslintrc.js

@@ -0,0 +1,40 @@
+module.exports = {
+	env: {
+		node: true,
+		mocha: false,
+	},
+	overrides: [
+		{
+			plugins: [ 'mocha' ],
+			files: [
+				'test/e2e/specs/specs-jetpack/*',
+				'test/e2e/specs/specs-playwright/*',
+				'test/e2e/specs/specs-wpcom/*',
+			],
+			rules: {
+				'mocha/no-exclusive-tests': 'error',
+				'mocha/handle-done-callback': [ 'error', { ignoreSkipped: true } ],
+				'mocha/no-global-tests': 'error',
+				'mocha/no-async-describe': 'error',
+				'mocha/no-top-level-hooks': 'error',
+				'mocha/max-top-level-suites': [ 'error', { limit: 1 } ],
+				// Disable all rules from "plugin:jest/recommended", as e2e tests use mocha
+				...Object.keys( require( 'eslint-plugin-jest' ).configs.recommended.rules ).reduce(
+					( disabledRules, key ) => ( { ...disabledRules, [ key ]: 'off' } ),
+					{}
+				),
+			},
+			globals: {
+				step: false,
+			},
+		},
+	],
+	rules: {
+		'import/no-nodejs-modules': 'off',
+		'no-console': 'off',
+
+		// We have many tests that don't make an explicit `expect`, but instead puts the browser
+		// in certain state that will be used by the next test, or asserted by WebDriver
+		'jest/expect-expect': 'off',
+	},
+};

test/e2e/config/default.json

@@ -3,6 +3,7 @@
 	"explicitWaitMS": 20000,
 	"emailWaitMS": 60000,
 	"mochaTimeoutMS": 160000,
+	"jestTimeoutMS": 160000,
 	"mochaDevDocsTimeoutMS": 1000000,
 	"startBrowserTimeoutMS": 60000,
 	"startAppTimeoutMS": 240000,

test/e2e/config/live-branches.json

@@ -2,6 +2,7 @@
 	"calypsoBaseURL": "https://calypso.live",
 	"explicitWaitMS": 40000,
 	"mochaTimeoutMS": 320000,
+	"jestTimeoutMS": 320000,
 	"browser": "chrome",
 	"saveAllScreenshots": true
 }

test/e2e/jest.config.js

@@ -0,0 +1,12 @@
+module.exports = {
+	cacheDirectory: '<rootDir>/../../.cache/jest',
+	testMatch: [ '<rootDir>/specs/**/*.js' ],
+	globalSetup: '<rootDir>/lib/jest/jest-global-setup.js',
+	setupFilesAfterEnv: [ '<rootDir>/lib/jest/jest-setup.js', '<rootDir>/lib/hooks/jest.js' ],
+	verbose: true,
+	transform: {
+		'\\.[jt]sx?$': [ 'babel-jest', { configFile: '../../babel.config.js' } ],
+	},
+	testRunner: 'jest-circus/runner',
+	testEnvironment: '<rootDir>/lib/jest/jest-environment-e2e.js',
+};

test/e2e/lib/driver-manager.js

@@ -11,11 +11,11 @@ import SauceLabs from 'saucelabs';
 import { times } from 'lodash';
 import { getChromeVersion } from '@testim/chrome-version';
 import * as remote from 'selenium-webdriver/remote';
+import path from 'path';
 
 /**
  * Internal dependencies
  */
-import { generatePath } from './test-utils';
 import * as dataHelper from './data-helper';
 
 const webDriverImplicitTimeOutMS = 2000;
@@ -118,6 +118,7 @@ export async function startBrowser( {
 	useCustomUA = true,
 	resizeBrowserWindow = true,
 	disableThirdPartyCookies = false,
+	tempDir = null,
 } = {} ) {
 	const screenSize = currentScreenSize();
 	const locale = currentLocale();
@@ -224,10 +225,10 @@ export async function startBrowser( {
 
 				// eslint-disable-next-line no-case-declarations
 				const service = new chrome.ServiceBuilder( chromedriver.path )
-					.loggingTo( generatePath( 'chromedriver.log' ) )
+					.loggingTo( path.join( tempDir, 'chromedriver.log' ) )
 					.build();
 				chrome.setDefaultService( service );
-				options.setChromeLogFile( generatePath( './chrome.log' ) );
+				options.setChromeLogFile( path.join( tempDir, 'chrome.log' ) );
 				options.addArguments( '--enable-logging' );
 
 				builder = new webdriver.Builder();

test/e2e/lib/flows/login-flow.js

@@ -69,8 +69,6 @@ export default class LoginFlow {
 		// 	return;
 		// }
 
-		console.log( 'Logging in as ' + this.account.username );
-
 		let loginURL = this.account.loginURL;
 		let loginPage;
 

test/e2e/lib/hooks/browser/browser.js

@@ -7,8 +7,8 @@ export const closeBrowser = async function ( { driver } ) {
 	await quitBrowser( driver );
 };
 
-export async function createBrowser() {
-	const driver = await startBrowser();
+export async function createBrowser( { tempDir } ) {
+	const driver = await startBrowser( { tempDir } );
 	await ensureNotLoggedIn( driver );
 	return driver;
 }

test/e2e/lib/hooks/browser/index.js

@@ -8,8 +8,8 @@ export const buildHooks = () => {
 	let driver;
 
 	return {
-		createBrowser: async function () {
-			driver = await createBrowser();
+		createBrowser: async function ( options ) {
+			driver = await createBrowser( options );
 			return driver;
 		},
 		saveBrowserLogs: function ( options ) {

test/e2e/lib/hooks/jest.js

@@ -0,0 +1,60 @@
+/**
+ * External dependencies
+ */
+import config from 'config';
+import path from 'path';
+import { mkdtemp, mkdir } from 'fs/promises';
+import { getState } from 'expect';
+
+/**
+ * Internal dependencies
+ */
+import { buildHooks as buildVideoHooks, isVideoEnabled } from './video';
+import { buildHooks as buildBrowserHooks } from './browser';
+
+let tempDir;
+let driver;
+const startBrowserTimeoutMS = config.get( 'startBrowserTimeoutMS' );
+const browserHooks = buildBrowserHooks();
+const videoHooks = isVideoEnabled() ? buildVideoHooks() : null;
+
+beforeAll( async () => {
+	const { testPath } = getState();
+
+	const sanitizedTestFilename = path.basename( testPath, path.extname( testPath ) );
+	const resultsPath = path.join( __dirname, '../../results' );
+	await mkdir( resultsPath, { recursive: true } );
+	tempDir = await mkdtemp( path.join( resultsPath, sanitizedTestFilename + '-' ) );
+
+	if ( isVideoEnabled() ) {
+		await videoHooks.startFramebuffer();
+		await videoHooks.startVideoRecording( { tempDir } );
+	}
+
+	driver = await browserHooks.createBrowser( { tempDir } );
+	global.__BROWSER__ = driver;
+}, startBrowserTimeoutMS );
+
+afterAll( async () => {
+	if ( isVideoEnabled() ) {
+		if ( global.__CURRENT_TEST_FAILED__ ) {
+			// Sometimes chrome crashes mid test. By wraping `takeScreenshot` in a try/catch we ensure
+			// we at least get a video recording
+			try {
+				videoHooks.takeScreenshot( {
+					tempDir,
+					testName: global.__CURRENT_TEST_NAME__,
+					driver: driver,
+				} );
+			} catch {}
+
+			await videoHooks.saveVideoRecording( { tempDir, testName: global.__CURRENT_TEST_NAME__ } );
+		}
+
+		await videoHooks.stopFramebuffer();
+		await videoHooks.stopVideoRecording();
+	}
+
+	await browserHooks.saveBrowserLogs( { tempDir } );
+	await browserHooks.closeBrowser();
+} );

test/e2e/lib/hooks/mocha.js

@@ -60,7 +60,7 @@ export const mochaHooks = async () => {
 
 	hooks.beforeAll.push( async function createBrowserHook() {
 		this.timeout( startBrowserTimeoutMS );
-		driver = await browserHooks.createBrowser();
+		driver = await browserHooks.createBrowser( { tempDir } );
 		this.driver = driver;
 	} );
 

test/e2e/lib/hooks/video/video-recorder.js

@@ -10,8 +10,13 @@ import { getTestNameWithTime } from '../../test-utils';
 
 const kill = ( proc ) =>
 	new Promise( ( resolve ) => {
-		proc.on( 'close', resolve );
-		proc.kill();
+		if ( proc.killed || proc.exitCode > 0 ) {
+			// Already killed
+			resolve();
+		} else {
+			proc.on( 'close', resolve );
+			proc.kill();
+		}
 	} );
 
 export const buildHooks = ( displayNum ) => {
@@ -46,7 +51,11 @@ export const buildHooks = ( displayNum ) => {
 	async function saveVideoRecording( { tempDir, testName } ) {
 		const newFile = path.join( tempDir, `screenshots/${ getTestNameWithTime( testName ) }.mpg` );
 		await mkdir( path.dirname( newFile ), { recursive: true } );
+		spawn( 'ls', [ '-la', tempDir + '/screenshots' ], { stdio: 'inherit' } );
 		await kill( ffVideo );
+		spawn( 'ls', [ '-la', tempDir + '/screenshots' ], { stdio: 'inherit' } );
+		spawn( 'ls', [ '-la', file ], { stdio: 'inherit' } );
+		spawn( 'ls', [ '-la', path.dirname( file ) ], { stdio: 'inherit' } );
 		await rename( file, newFile );
 	}
 

test/e2e/lib/jest/jest-environment-e2e.js

@@ -0,0 +1,34 @@
+const JestEnvironmentNode = require( 'jest-environment-node' );
+
+class JestEnvironmentE2E extends JestEnvironmentNode {
+	testFailed = false;
+
+	async handleTestEvent( event ) {
+		// Top describe (event.describeBlock.name)
+		// 		event.type=="run_describe_start" && event.describeBlock.parent.name === "ROOT_DESCRIBE_BLOCK"
+
+		// Test (event.test.name)
+		//
+
+		// Save reference to current test
+		if ( event.name === 'test_start' ) {
+			this.global.__CURRENT_TEST_NAME__ = event.test.name;
+			return;
+		}
+
+		// If a hook or a test fails, enter in "failed mode"
+		if ( event.name === 'hook_failure' || event.name === 'test_fn_failure' ) {
+			this.global.__CURRENT_TEST_FAILED__ = true;
+			this.testFailed = true;
+			return;
+		}
+
+		// If a test starts after a test has already failed, skip it
+		if ( this.testFailed && event.name === 'test_start' ) {
+			event.test.mode = 'skip';
+			return;
+		}
+	}
+}
+
+module.exports = JestEnvironmentE2E;

test/e2e/lib/jest/jest-global-setup.js

@@ -0,0 +1,17 @@
+/**
+ * External dependencies
+ */
+import { node } from 'execa';
+
+module.exports = async () => {
+	// Update chromedriver
+	const subprocess = node( require.resolve( 'chromedriver/install.js' ), {
+		env: {
+			CHROMEDRIVER_SKIP_DOWNLOAD: false,
+			DETECT_CHROMEDRIVER_VERSION: true,
+		},
+	} );
+	subprocess.stdout.pipe( process.stdout );
+	subprocess.stderr.pipe( process.stderr );
+	await subprocess;
+};

test/e2e/lib/jest/jest-setup.js

@@ -0,0 +1,7 @@
+/**
+ * External dependencies
+ */
+import config from 'config';
+
+// Default timeout
+jest.setTimeout( config.get( 'jestTimeoutMS' ) );

test/e2e/lib/test-utils.js

@@ -1,12 +1,3 @@
-/**
- * External dependencies
- */
-import path from 'path';
-
-const BASE_PATH = process.env.TEMP_ASSET_PATH || path.join( __dirname, '..' );
-
-export const generatePath = ( name ) => path.join( BASE_PATH, name );
-
 export const getTestNameWithTime = ( testName ) => {
 	const currentTestName = testName.replace( /[^a-z0-9]/gi, '-' ).toLowerCase();
 	const dateTime = new Date().toISOString().split( '.' )[ 0 ].replace( /:/g, '-' );

test/e2e/package.json

@@ -39,6 +39,7 @@
 		"esformatter-quotes": "^1.0.3",
 		"esformatter-semicolons": "^1.1.2",
 		"esformatter-special-bangs": "^1.0.1",
+		"eslint-plugin-jest": "^24.3.6",
 		"esm": "^3.2.25",
 		"execa": "^5.0.0",
 		"ffmpeg-static": "^2.4.0",
@@ -47,6 +48,8 @@
 		"grunt-cli": "^1.2.0",
 		"grunt-concurrent": "^2.3.0",
 		"grunt-shell": "^1.1.2",
+		"jest-environment-node": "^26.6.2",
+		"jest-teamcity": "^1.9.0",
 		"junit-viewer": "^4.9.6",
 		"lodash": "^4.17.20",
 		"mailosaur": "^4.0.0",