POC Playwright test stability

.env

@@ -9,7 +9,7 @@
 ##
 
 # The site will be available at http://localhost:LOCAL_PORT
-LOCAL_PORT=8889
+LOCAL_PORT=80
 
 # Where to run WordPress from. Valid options are 'src' and 'build'.
 LOCAL_DIR=src
@@ -62,4 +62,4 @@ LOCAL_WP_ENVIRONMENT_TYPE=local
 LOCAL_WP_DEVELOPMENT_MODE=core
 
 # The URL to use when running e2e tests.
-WP_BASE_URL=http://localhost:${LOCAL_PORT}
+WP_BASE_URL=http://wordpress-develop

.github/workflows/performance.yml

@@ -12,10 +12,6 @@ on:
       - '![45].[0-9].[0-9]+'
       - '!6.[01].[0-9]+'
   pull_request:
-    branches:
-      - trunk
-      - '6.[2-9]'
-      - '[7-9].[0-9]'
   workflow_dispatch:
 
 # Cancels all previous workflow runs for pull requests that have not completed.
@@ -34,9 +30,14 @@ env:
 
   # Performance testing should be performed in an environment reflecting a standard production environment.
   LOCAL_WP_DEBUG: false
+  LOCAL_WP_DEBUG_LOG: false
+  LOCAL_WP_DEBUG_DISPLAY: false
   LOCAL_SCRIPT_DEBUG: false
   LOCAL_SAVEQUERIES: false
   LOCAL_WP_DEVELOPMENT_MODE: "''"
+  LOCAL_WP_PLAYWRIGHT: true
+  PW_TEST_CONNECT_WS_ENDPOINT: "ws://127.0.0.1:3000/"
+  WP_BASE_URL: http://wordpress-develop
 
   # This workflow takes two sets of measurements — one for the current commit,
   # and another against a consistent version that is used as a baseline measurement.
@@ -60,14 +61,19 @@ jobs:
   # - Install npm dependencies.
   # - Install Playwright browsers.
   # - Build WordPress.
+  # - Add hosts to /etc/hosts.
   # - Start Docker environment.
   # - Log running Docker containers.
   # - Docker debug information.
   # - Install WordPress.
   # - Install WordPress Importer plugin.
   # - Import mock data.
+  # - Deactivate WordPress Importer plugin.
   # - Update permalink structure.
-  # - Install MU plugin.
+  # - Disable external HTTP requests.
+  # - Disable cron.
+  # - List defined constants.
+  # - Install MU plugins.
   # - Run performance tests (current commit).
   # - Print performance tests results.
   # - Check out target commit (target branch or previous commit).
@@ -129,6 +135,10 @@ jobs:
       - name: Build WordPress
         run: npm run build
 
+      - name: Add hosts to /etc/hosts
+        run: |
+          sudo echo "127.0.0.1 wordpress-develop" | sudo tee -a /etc/hosts
+
       - name: Start Docker environment
         run: |
           npm run env:start
@@ -158,21 +168,61 @@ jobs:
           npm run env:cli -- import themeunittestdata.wordpress.xml --authors=create --path=/var/www/${{ env.LOCAL_DIR }}
           rm themeunittestdata.wordpress.xml
 
+      - name: Deactivate WordPress Importer plugin
+        run: npm run env:cli -- plugin deactivate wordpress-importer --path=/var/www/${{ env.LOCAL_DIR }}
+
       - name: Update permalink structure
         run: |
           npm run env:cli -- rewrite structure '/%year%/%monthnum%/%postname%/' --path=/var/www/${{ env.LOCAL_DIR }}
 
-      - name: Install MU plugin
+      - name: Disable external HTTP requests
+        run: |
+          npm run env:cli -- config set WP_HTTP_BLOCK_EXTERNAL true --raw --type=constant
+
+      - name: Disable cron
+        run: |
+          npm run env:cli -- config set DISABLE_WP_CRON true --raw --type=constant
+
+      - name: List defined constants
+        run: |
+          npm run env:cli -- config list --path=/var/www/${{ env.LOCAL_DIR }}
+
+      - name: Install MU plugins
         run: |
           mkdir ./${{ env.LOCAL_DIR }}/wp-content/mu-plugins
           cp ./tests/performance/wp-content/mu-plugins/server-timing.php ./${{ env.LOCAL_DIR }}/wp-content/mu-plugins/server-timing.php
+          cp ./tests/performance/wp-content/mu-plugins/reset-helper.php ./${{ env.LOCAL_DIR }}/wp-content/mu-plugins/reset-helper.php
 
       - name: Run performance tests (current commit)
-        run: npm run test:performance
+        run: |
+          TEST_RUNS=10 npm run test:performance
+          node ./tests/performance/results.js
+          TEST_RUNS=20 npm run test:performance
+          node ./tests/performance/results.js
+          TEST_RUNS=30 npm run test:performance
+          node ./tests/performance/results.js
+          TEST_RUNS=40 npm run test:performance
+          node ./tests/performance/results.js
+          TEST_RUNS=50 npm run test:performance
+          node ./tests/performance/results.js
 
       - name: Print performance tests results
         run: node ./tests/performance/results.js
 
+      - name: Run Server-Timing bash script
+        run: |
+          TEST_RUNS=10 bash tests/performance/servertiming.sh
+          TEST_RUNS=20 bash tests/performance/servertiming.sh
+          TEST_RUNS=30 bash tests/performance/servertiming.sh
+          TEST_RUNS=40 bash tests/performance/servertiming.sh
+          TEST_RUNS=50 bash tests/performance/servertiming.sh
+
+          TEST_RUNS=10 bash tests/performance/servertiming.sh block-theme
+          TEST_RUNS=20 bash tests/performance/servertiming.sh block-theme
+          TEST_RUNS=30 bash tests/performance/servertiming.sh block-theme
+          TEST_RUNS=40 bash tests/performance/servertiming.sh block-theme
+          TEST_RUNS=50 bash tests/performance/servertiming.sh block-theme
+
       - name: Check out target commit (target branch or previous commit)
         run: |
           if [[ -z "$TARGET_REF" ]]; then

docker-compose.yml

@@ -12,7 +12,7 @@ services:
       - wpdevnet
 
     ports:
-      - ${LOCAL_PORT-8889}:80
+      - "80:80"
 
     environment:
       LOCAL_DIR: ${LOCAL_DIR-src}
@@ -129,12 +129,28 @@ services:
       - wpdevnet
 
     ports:
-      - 11211:11211
+      - "11211:11211"
 
     depends_on:
       php:
         condition: service_started
 
+  playwright:
+    build:
+      context: .
+      dockerfile: ./tools/local-env/playwright/Dockerfile
+
+    networks:
+      - wpdevnet
+
+    environment:
+      # GitHub sets CI to true automatically
+      # https://docs.github.com/en/actions/learn-github-actions/variables#default-environment-variables
+      - CI
+
+    ports:
+      - "3000:3000"
+
 volumes:
   # So that sites aren't wiped every time containers are restarted, MySQL uses a persistent volume.
   mysql: {}

tests/e2e/playwright.config.js

@@ -3,6 +3,11 @@
  */
 import path from 'node:path';
 import { defineConfig } from '@playwright/test';
+import dotenv from 'dotenv';
+import dotenvExpand from 'dotenv-expand';
+
+// To ensure env vars like WP_BASE_URL are defined before loading the base config.
+dotenvExpand.expand( dotenv.config() );
 
 /**
  * WordPress dependencies
@@ -20,6 +25,8 @@ const config = defineConfig( {
 	globalSetup: require.resolve( './config/global-setup.js' ),
 	webServer: {
 		...baseConfig.webServer,
+		port: undefined,
+		url: process.env.WP_BASE_URL,
 		command: 'npm run env:start',
 	},
 } );

tests/e2e/specs/hello.test.js

@@ -4,7 +4,7 @@
 import { test, expect } from '@wordpress/e2e-test-utils-playwright';
 
 test.describe( 'Hello World', () => {
-	test( 'Should load properly', async ( { admin, page }) => {
+	test( 'Should load properly', async ( { admin, page } ) => {
 		await admin.visitAdminPage( '/' );
 		await expect(
 			page.getByRole('heading', { name: 'Welcome to WordPress', level: 2 })

tests/performance/playwright.config.js

@@ -3,6 +3,11 @@
  */
 import path from 'node:path';
 import { defineConfig } from '@playwright/test';
+import dotenv from 'dotenv';
+import dotenvExpand from 'dotenv-expand';
+
+// To ensure env vars like WP_BASE_URL are defined before loading the base config.
+dotenvExpand.expand( dotenv.config() );
 
 /**
  * WordPress dependencies
@@ -14,14 +19,12 @@ process.env.STORAGE_STATE_PATH ??= path.join(
 	process.env.WP_ARTIFACTS_PATH,
 	'storage-states/admin.json'
 );
-process.env.TEST_RUNS ??= '20';
+process.env.TEST_RUNS ??= '30';
 
 const config = defineConfig( {
 	...baseConfig,
 	globalSetup: require.resolve( './config/global-setup.js' ),
-	reporter: process.env.CI
-		? './config/performance-reporter.js'
-		: [ [ 'list' ], [ './config/performance-reporter.js' ] ],
+	reporter: [ [ 'list' ], [ './config/performance-reporter.js' ] ],
 	forbidOnly: !! process.env.CI,
 	workers: 1,
 	retries: 0,
@@ -30,10 +33,13 @@ const config = defineConfig( {
 	reportSlowTests: null,
 	webServer: {
 		...baseConfig.webServer,
+		port: undefined,
+		url: process.env.WP_BASE_URL,
 		command: 'npm run env:start',
 	},
 	use: {
 		...baseConfig.use,
+		baseURL: process.env.WP_BASE_URL,
 		video: 'off',
 	},
 } );

tests/performance/results.js

@@ -5,7 +5,7 @@
  */
 const fs = require( 'node:fs' );
 const { join } = require( 'node:path' );
-const { median, getResultsFilename } = require( './utils' );
+const { median, getResultsFilename, standardDeviation, medianAbsoluteDeviation } = require( './utils' );
 
 const testSuites = [
     'home-classic-theme',
@@ -17,6 +17,11 @@ console.log( '\n>> 🎉 Results 🎉 \n' );
 for ( const testSuite of testSuites ) {
     const resultsFileName = getResultsFilename( testSuite + '.test' );
     const resultsPath = join( __dirname, '/specs/', resultsFileName );
+
+	if ( ! fs.existsSync( resultsPath ) ) {
+		continue;
+	}
+
     fs.readFile( resultsPath, "utf8", ( err, data ) => {
         if ( err ) {
             console.log( "File read failed:", err );
@@ -29,8 +34,14 @@ for ( const testSuite of testSuites ) {
         const rawResults = [];
 
         for ( var key in tableData ) {
-            if ( tableData.hasOwnProperty( key ) ) {
-                rawResults[ key ] = median( tableData[ key ] );
+			if ( tableData.hasOwnProperty( key ) && tableData[ key ].length > 0 ) {
+				rawResults.push( {
+					key,
+					iterations: tableData[ key ].length,
+					median: median( tableData[ key ] ),
+					'STD':  standardDeviation( tableData[ key ] ),
+					'MAD':  medianAbsoluteDeviation( tableData[ key ] ),
+				} );
             }
         }
         console.table( rawResults );

tests/performance/servertiming.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+WP_TOTAL=();
+
+if [[ $1 == 'block-theme' ]]; then
+		echo "Block Theme:"
+		npm run env:cli -- theme activate twentytwentythree --quiet --path=/var/www/build
+	else
+		echo "Classic Theme:"
+		npm run env:cli -- theme activate twentytwentyone --quiet --path=/var/www/build
+fi
+
+TEST_RUNS=${TEST_RUNS:-30}
+
+for ((i=1; i<=TEST_RUNS; i++ ))
+do
+	# Clear caches.
+	curl -s http://wordpress-develop?reset_helper
+
+	# Warm caches.
+	curl -s -o /dev/null http://wordpress-develop
+
+	# Actual request for measuring.
+	SERVER_TIMING=$(curl -sI -X GET http://wordpress-develop | tr -d '\r' | sed -En 's/^Server-Timing: (.*)/\1/p')
+
+	IFS=', ' read -r -a ENTRIES <<< "$SERVER_TIMING"
+
+	for ENTRY in "${ENTRIES[@]}"
+  do
+  	IFS=';' read -r -a ENTRY_PARTS <<< "$ENTRY"
+
+  		KEY=${ENTRY_PARTS[0]}
+  		VALUE=${ENTRY_PARTS[1]}
+  		VALUE=${VALUE/dur=/}
+
+  		if [ "$KEY" == "wp-total" ]; then
+  			WP_TOTAL+=( "$VALUE" )
+  		fi
+
+#    	echo "Key: $KEY"
+#    	echo "Value: $VALUE"
+  done
+done
+
+IFS=$'\n'
+MEDIAN=$(awk '{arr[NR]=$1} END {if (NR%2==1) print arr[(NR+1)/2]; else print (arr[NR/2]+arr[NR/2+1])/2}' <<< sort <<< "${WP_TOTAL[*]}")
+unset IFS
+
+STDDEV=$(awk 'NF {sum=0;ssq=0;for (i=1;i<=NF;i++){sum+=$i;ssq+=$i**2}; print (ssq/NF-(sum/NF)**2)**0.5}' <<< "${WP_TOTAL[*]}")
+
+echo "Iterations: ${#WP_TOTAL[@]} / Median: $MEDIAN / Std deviation: $STDDEV"

tests/performance/specs/home-block-theme.test.js

@@ -31,6 +31,14 @@ test.describe( 'Front End - Twenty Twenty Three', () => {
 		await requestUtils.activateTheme( 'twentytwentyone' );
 	} );
 
+	test.beforeEach( async ( { browser } ) => {
+		const page = await browser.newPage();
+		// Clear caches.
+		await page.goto( '/?reset_helper' );
+		// Warm caches.
+		await page.goto( '/' );
+	} );
+
 	const iterations = Number( process.env.TEST_RUNS );
 	for ( let i = 1; i <= iterations; i++ ) {
 		test( `Measure load time metrics (${ i } of ${ iterations })`, async ( {