From 23acde292b23f0b2360563766cf115f776ef9c5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9rgio=20Gomes?= Date: Wed, 8 Feb 2023 16:53:54 +0000 Subject: [PATCH 1/5] E2E perf tests: run each test in a separate page --- .../config/setup-performance-test.js | 21 ++++-- .../specs/performance/post-editor.test.js | 7 +- .../specs/performance/site-editor.test.js | 64 +++++++++++-------- 3 files changed, 58 insertions(+), 34 deletions(-) diff --git a/packages/e2e-tests/config/setup-performance-test.js b/packages/e2e-tests/config/setup-performance-test.js index 172699035571e9..384a88b8a3f2d3 100644 --- a/packages/e2e-tests/config/setup-performance-test.js +++ b/packages/e2e-tests/config/setup-performance-test.js @@ -19,9 +19,11 @@ const PUPPETEER_TIMEOUT = process.env.PUPPETEER_TIMEOUT; // The Jest timeout is increased because these tests are a bit slow. jest.setTimeout( PUPPETEER_TIMEOUT || 100000 ); -async function setupBrowser() { - await clearLocalStorage(); +async function setupPage() { await setBrowserViewport( 'large' ); + await page.emulateMediaFeatures( [ + { name: 'prefers-reduced-motion', value: 'reduce' }, + ] ); } // Before every test suite run, delete all content created by the test. This ensures @@ -32,13 +34,18 @@ beforeAll( async () => { await trashAllPosts(); await trashAllPosts( 'wp_block' ); - await setupBrowser(); + await clearLocalStorage(); + await setupPage(); await activatePlugin( 'gutenberg-test-plugin-disables-the-css-animations' ); - await page.emulateMediaFeatures( [ - { name: 'prefers-reduced-motion', value: 'reduce' }, - ] ); } ); afterEach( async () => { - await setupBrowser(); + // Clear localStorage between tests so that the next test starts clean. + await clearLocalStorage(); + // Close the previous page entirely and create a new page, so that the next test + // isn't affected by page unload work. + await page.close(); + page = await browser.newPage(); + // Set up testing config on new page. + await setupPage(); } ); diff --git a/packages/e2e-tests/specs/performance/post-editor.test.js b/packages/e2e-tests/specs/performance/post-editor.test.js index 10699f921be510..7fbcc6e51989fd 100644 --- a/packages/e2e-tests/specs/performance/post-editor.test.js +++ b/packages/e2e-tests/specs/performance/post-editor.test.js @@ -105,9 +105,14 @@ describe( 'Post Editor Performance', () => { readFile( join( __dirname, '../../assets/large-post.html' ) ) ); await saveDraft(); + const draftURL = await page.url(); + let i = 5; while ( i-- ) { - await page.reload(); + await page.close(); + page = await browser.newPage(); + + await page.goto( draftURL ); await page.waitForSelector( '.edit-post-layout', { timeout: 120000, } ); diff --git a/packages/e2e-tests/specs/performance/site-editor.test.js b/packages/e2e-tests/specs/performance/site-editor.test.js index e3a2ef86c69328..77fece4b4f3a5d 100644 --- a/packages/e2e-tests/specs/performance/site-editor.test.js +++ b/packages/e2e-tests/specs/performance/site-editor.test.js @@ -30,34 +30,29 @@ import { jest.setTimeout( 1000000 ); +const results = { + serverResponse: [], + firstPaint: [], + domContentLoaded: [], + loaded: [], + firstContentfulPaint: [], + firstBlock: [], + type: [], + typeContainer: [], + focus: [], + inserterOpen: [], + inserterHover: [], + inserterSearch: [], + listViewOpen: [], +}; + +let id; + describe( 'Site Editor Performance', () => { beforeAll( async () => { await activateTheme( 'emptytheme' ); await deleteAllTemplates( 'wp_template' ); await deleteAllTemplates( 'wp_template_part' ); - } ); - afterAll( async () => { - await deleteAllTemplates( 'wp_template' ); - await deleteAllTemplates( 'wp_template_part' ); - await activateTheme( 'twentytwentyone' ); - } ); - - it( 'Loading', async () => { - const results = { - serverResponse: [], - firstPaint: [], - domContentLoaded: [], - loaded: [], - firstContentfulPaint: [], - firstBlock: [], - type: [], - typeContainer: [], - focus: [], - inserterOpen: [], - inserterHover: [], - inserterSearch: [], - listViewOpen: [], - }; const html = readFile( join( __dirname, '../../assets/large-post.html' ) @@ -80,17 +75,32 @@ describe( 'Site Editor Performance', () => { }, html ); await saveDraft(); - const id = await page.evaluate( () => + id = await page.evaluate( () => new URL( document.location ).searchParams.get( 'post' ) ); + } ); + afterAll( async () => { + await deleteAllTemplates( 'wp_template' ); + await deleteAllTemplates( 'wp_template_part' ); + await activateTheme( 'twentytwentyone' ); + } ); + + beforeEach( async () => { await visitSiteEditor( { postId: id, postType: 'page' } ); + } ); + + it( 'Loading', async () => { + const editorURL = await page.url(); let i = 3; // Measuring loading time. while ( i-- ) { - await page.reload(); + await page.close(); + page = await browser.newPage(); + + await page.goto( editorURL ); await page.waitForSelector( '.edit-site-visual-editor', { timeout: 120000, } ); @@ -111,7 +121,9 @@ describe( 'Site Editor Performance', () => { results.firstContentfulPaint.push( firstContentfulPaint ); results.firstBlock.push( firstBlock ); } + } ); + it( 'Typing', async () => { // Measuring typing performance inside the post content. await canvas().waitForSelector( '[data-type="core/post-content"] [data-type="core/paragraph"]' @@ -121,7 +133,7 @@ describe( 'Site Editor Performance', () => { '[data-type="core/post-content"] [data-type="core/paragraph"]' ); await insertBlock( 'Paragraph' ); - i = 200; + let i = 200; const traceFile = __dirname + '/trace.json'; await page.tracing.start( { path: traceFile, From 0ef78ce0a04a728467fd9658cfd09e12dda96f08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9rgio=20Gomes?= Date: Thu, 9 Feb 2023 10:41:24 +0000 Subject: [PATCH 2/5] Throw away 1st measurement to avoid caching quirks --- .../specs/performance/post-editor.test.js | 23 ++++++++++----- .../specs/performance/site-editor.test.js | 28 ++++++++++++++----- 2 files changed, 37 insertions(+), 14 deletions(-) diff --git a/packages/e2e-tests/specs/performance/post-editor.test.js b/packages/e2e-tests/specs/performance/post-editor.test.js index 7fbcc6e51989fd..9a7b702220dc1d 100644 --- a/packages/e2e-tests/specs/performance/post-editor.test.js +++ b/packages/e2e-tests/specs/performance/post-editor.test.js @@ -107,7 +107,14 @@ describe( 'Post Editor Performance', () => { await saveDraft(); const draftURL = await page.url(); - let i = 5; + // Number of sample measurements to take. + const samples = 5; + // Number of throwaway measurements to perform before recording samples. + // Having at least one helps ensure that caching quirks don't manifest in + // the results. + const throwaway = 1; + + let i = throwaway + samples; while ( i-- ) { await page.close(); page = await browser.newPage(); @@ -126,12 +133,14 @@ describe( 'Post Editor Performance', () => { firstBlock, } = await getLoadingDurations(); - results.serverResponse.push( serverResponse ); - results.firstPaint.push( firstPaint ); - results.domContentLoaded.push( domContentLoaded ); - results.loaded.push( loaded ); - results.firstContentfulPaint.push( firstContentfulPaint ); - results.firstBlock.push( firstBlock ); + if ( i < samples ) { + results.serverResponse.push( serverResponse ); + results.firstPaint.push( firstPaint ); + results.domContentLoaded.push( domContentLoaded ); + results.loaded.push( loaded ); + results.firstContentfulPaint.push( firstContentfulPaint ); + results.firstBlock.push( firstBlock ); + } } } ); diff --git a/packages/e2e-tests/specs/performance/site-editor.test.js b/packages/e2e-tests/specs/performance/site-editor.test.js index 77fece4b4f3a5d..109cac9a0c2f8b 100644 --- a/packages/e2e-tests/specs/performance/site-editor.test.js +++ b/packages/e2e-tests/specs/performance/site-editor.test.js @@ -93,7 +93,14 @@ describe( 'Site Editor Performance', () => { it( 'Loading', async () => { const editorURL = await page.url(); - let i = 3; + // Number of sample measurements to take. + const samples = 3; + // Number of throwaway measurements to perform before recording samples. + // Having at least one helps ensure that caching quirks don't manifest in + // the results. + const throwaway = 1; + + let i = throwaway + samples; // Measuring loading time. while ( i-- ) { @@ -114,16 +121,23 @@ describe( 'Site Editor Performance', () => { firstBlock, } = await getLoadingDurations(); - results.serverResponse.push( serverResponse ); - results.firstPaint.push( firstPaint ); - results.domContentLoaded.push( domContentLoaded ); - results.loaded.push( loaded ); - results.firstContentfulPaint.push( firstContentfulPaint ); - results.firstBlock.push( firstBlock ); + if ( i < samples ) { + results.serverResponse.push( serverResponse ); + results.firstPaint.push( firstPaint ); + results.domContentLoaded.push( domContentLoaded ); + results.loaded.push( loaded ); + results.firstContentfulPaint.push( firstContentfulPaint ); + results.firstBlock.push( firstBlock ); + } } } ); it( 'Typing', async () => { + await page.waitForSelector( '.edit-site-visual-editor', { + timeout: 120000, + } ); + await canvas().waitForSelector( '.wp-block', { timeout: 120000 } ); + // Measuring typing performance inside the post content. await canvas().waitForSelector( '[data-type="core/post-content"] [data-type="core/paragraph"]' From 2b092e2179d03c30954c5f5eec1eff74f9ee6840 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9rgio=20Gomes?= Date: Fri, 10 Feb 2023 14:01:16 +0000 Subject: [PATCH 3/5] Move page closing to `beforeEach` --- packages/e2e-tests/config/setup-performance-test.js | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/e2e-tests/config/setup-performance-test.js b/packages/e2e-tests/config/setup-performance-test.js index 384a88b8a3f2d3..2c349605026937 100644 --- a/packages/e2e-tests/config/setup-performance-test.js +++ b/packages/e2e-tests/config/setup-performance-test.js @@ -35,17 +35,19 @@ beforeAll( async () => { await trashAllPosts(); await trashAllPosts( 'wp_block' ); await clearLocalStorage(); - await setupPage(); await activatePlugin( 'gutenberg-test-plugin-disables-the-css-animations' ); } ); -afterEach( async () => { - // Clear localStorage between tests so that the next test starts clean. - await clearLocalStorage(); - // Close the previous page entirely and create a new page, so that the next test +beforeEach( async () => { + // Close the previous page entirely and create a new page, so that the test // isn't affected by page unload work. await page.close(); page = await browser.newPage(); // Set up testing config on new page. await setupPage(); } ); + +afterEach( async () => { + // Clear localStorage between tests so that the next test starts clean. + await clearLocalStorage(); +} ); From 383811d04b5787f21dc857dafe8ce08572500dd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9rgio=20Gomes?= Date: Fri, 10 Feb 2023 14:04:41 +0000 Subject: [PATCH 4/5] Move metric collection inside conditional --- .../specs/performance/post-editor.test.js | 17 +++++++++-------- .../specs/performance/site-editor.test.js | 17 +++++++++-------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/packages/e2e-tests/specs/performance/post-editor.test.js b/packages/e2e-tests/specs/performance/post-editor.test.js index 9a7b702220dc1d..597f4f36cd9a3b 100644 --- a/packages/e2e-tests/specs/performance/post-editor.test.js +++ b/packages/e2e-tests/specs/performance/post-editor.test.js @@ -124,16 +124,17 @@ describe( 'Post Editor Performance', () => { timeout: 120000, } ); await canvas().waitForSelector( '.wp-block', { timeout: 120000 } ); - const { - serverResponse, - firstPaint, - domContentLoaded, - loaded, - firstContentfulPaint, - firstBlock, - } = await getLoadingDurations(); if ( i < samples ) { + const { + serverResponse, + firstPaint, + domContentLoaded, + loaded, + firstContentfulPaint, + firstBlock, + } = await getLoadingDurations(); + results.serverResponse.push( serverResponse ); results.firstPaint.push( firstPaint ); results.domContentLoaded.push( domContentLoaded ); diff --git a/packages/e2e-tests/specs/performance/site-editor.test.js b/packages/e2e-tests/specs/performance/site-editor.test.js index 109cac9a0c2f8b..cd071f0be5a85d 100644 --- a/packages/e2e-tests/specs/performance/site-editor.test.js +++ b/packages/e2e-tests/specs/performance/site-editor.test.js @@ -112,16 +112,17 @@ describe( 'Site Editor Performance', () => { timeout: 120000, } ); await canvas().waitForSelector( '.wp-block', { timeout: 120000 } ); - const { - serverResponse, - firstPaint, - domContentLoaded, - loaded, - firstContentfulPaint, - firstBlock, - } = await getLoadingDurations(); if ( i < samples ) { + const { + serverResponse, + firstPaint, + domContentLoaded, + loaded, + firstContentfulPaint, + firstBlock, + } = await getLoadingDurations(); + results.serverResponse.push( serverResponse ); results.firstPaint.push( firstPaint ); results.domContentLoaded.push( domContentLoaded ); From b863d420ef764300759b5d4ee76345374ce9fd50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9rgio=20Gomes?= Date: Fri, 10 Feb 2023 16:39:27 +0000 Subject: [PATCH 5/5] Revert "Move page closing to `beforeEach`" This reverts commit 2b092e2179d03c30954c5f5eec1eff74f9ee6840. --- packages/e2e-tests/config/setup-performance-test.js | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/packages/e2e-tests/config/setup-performance-test.js b/packages/e2e-tests/config/setup-performance-test.js index 2c349605026937..384a88b8a3f2d3 100644 --- a/packages/e2e-tests/config/setup-performance-test.js +++ b/packages/e2e-tests/config/setup-performance-test.js @@ -35,19 +35,17 @@ beforeAll( async () => { await trashAllPosts(); await trashAllPosts( 'wp_block' ); await clearLocalStorage(); + await setupPage(); await activatePlugin( 'gutenberg-test-plugin-disables-the-css-animations' ); } ); -beforeEach( async () => { - // Close the previous page entirely and create a new page, so that the test +afterEach( async () => { + // Clear localStorage between tests so that the next test starts clean. + await clearLocalStorage(); + // Close the previous page entirely and create a new page, so that the next test // isn't affected by page unload work. await page.close(); page = await browser.newPage(); // Set up testing config on new page. await setupPage(); } ); - -afterEach( async () => { - // Clear localStorage between tests so that the next test starts clean. - await clearLocalStorage(); -} );