Run e2e tests in debug/editor mode (in addition to prod mode)

.github/workflows/ci.yml

@@ -40,21 +40,29 @@ jobs:
       - name: Bazel tests
         # NOTE: bazel test //... doesn't work (due to node_modules)
         run: bazel test //mesop/...
-      - name: Run playwright test
+      - name: Run playwright test (prod mode)
         run: yarn playwright test
       - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
         if: always()
         with:
-          name: playwright-report
-          path: playwright-report/
+          name: playwright-report-prod-mode
+          path: playwright-report-prod-mode/
+          retention-days: 30
+      - name: Run playwright test (debug/editor mode)
+        run: MESOP_DEBUG_MODE=true yarn playwright test
+      - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
+        if: always()
+        with:
+          name: playwright-report-debug-mode
+          path: playwright-report-debug-mode/
           retention-days: 30
       - name: Run playwright test (concurrency)
         run: yarn playwright test mesop/tests/e2e/concurrency/state_test.ts --repeat-each=48 --workers=16
       - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
         if: always()
         with:
-          name: playwright-report
-          path: playwright-report/
+          name: playwright-report-concurrency
+          path: playwright-report-concurrency/
           retention-days: 30
       - name: Run playwright test with memory state session
         run: MESOP_STATE_SESSION_BACKEND=memory yarn playwright test

mesop/components/html/e2e/html_test.ts

@@ -1,4 +1,5 @@
 import {test, expect} from '@playwright/test';
+import {testInProdOnly} from '../../../tests/e2e/e2e_helpers';
 
 test('test sanitized html', async ({page}) => {
   await page.goto('/components/html/e2e/html_app');
@@ -50,7 +51,7 @@ test('test sandboxed html - origin is null', async ({page}) => {
   ).toBeVisible();
 });
 
-test('sandbox_iframe.html csp', async ({page}) => {
+testInProdOnly('sandbox_iframe.html csp', async ({page}) => {
   const response = await page.goto('/sandbox_iframe.html');
   const csp = response?.headers()['content-security-policy']!;
   expect(csp).toMatchSnapshot('sandbox_iframe.html-csp.txt');

mesop/examples/e2e/theme_test.ts

@@ -8,13 +8,13 @@ test('theme', async ({page}) => {
   await expect(page.getByText('Theme: dark')).toBeVisible();
   expect(await page.evaluate(hasDarkTheme)).toBeTruthy();
 
-  await page.locator('button').click();
+  await page.getByRole('button', {name: 'toggle theme'}).click();
 
   // Theme mode is light -> light theme
   await expect(page.getByText('Theme: light')).toBeVisible();
   expect(await page.evaluate(hasDarkTheme)).toBeFalsy();
 
-  await page.locator('button').click();
+  await page.getByRole('button', {name: 'toggle theme'}).click();
 
   // Theme mode is dark -> dark theme
   await expect(page.getByText('Theme: dark')).toBeVisible();

mesop/tests/e2e/e2e_helpers.ts

@@ -0,0 +1,11 @@
+import {test as base} from '@playwright/test';
+
+export const testInProdOnly = base.extend({
+  // Skip all tests in this file if MESOP_DEBUG_MODE is 'true'
+  page: async ({page}, use) => {
+    if (process.env.MESOP_DEBUG_MODE === 'true') {
+      base.skip(true, 'Skipping test in debug mode');
+    }
+    await use(page);
+  },
+});

mesop/tests/e2e/late_registration_error_test.ts

@@ -1,27 +1,36 @@
-import {test, expect} from '@playwright/test';
+import {expect} from '@playwright/test';
+import {testInProdOnly} from './e2e_helpers';
 
-test('register page too late should cause an error', async ({page}) => {
-  await page.goto('/testing/error_register_page_too_late');
+testInProdOnly(
+  'register page too late should cause an error',
+  async ({page}) => {
+    await page.goto('/testing/error_register_page_too_late');
 
-  await expect(
-    page.getByText('Sorry, there was an error. Please contact the developer.'),
-  ).toBeVisible();
+    await expect(
+      page.getByText(
+        'Sorry, there was an error. Please contact the developer.',
+      ),
+    ).toBeVisible();
 
-  await page.goto('/testing/error_register_page_too_late/too_late_page');
+    await page.goto('/testing/error_register_page_too_late/too_late_page');
 
-  await expect(
-    page.getByText(
-      'User Error: Accessed path: /testing/error_register_page_too_late/too_late_page not registered',
-    ),
-  ).toBeVisible();
-});
+    await expect(
+      page.getByText(
+        'User Error: Accessed path: /testing/error_register_page_too_late/too_late_page not registered',
+      ),
+    ).toBeVisible();
+  },
+);
 
-test('register web component too late should cause an error', async ({
-  page,
-}) => {
-  await page.goto('/testing/error_register_web_component_too_late');
+testInProdOnly(
+  'register web component too late should cause an error',
+  async ({page}) => {
+    await page.goto('/testing/error_register_web_component_too_late');
 
-  await expect(
-    page.getByText('Sorry, there was an error. Please contact the developer.'),
-  ).toBeVisible();
-});
+    await expect(
+      page.getByText(
+        'Sorry, there was an error. Please contact the developer.',
+      ),
+    ).toBeVisible();
+  },
+);

mesop/tests/e2e/on_load_test.ts

@@ -14,8 +14,10 @@ test('on_load (generator)', async ({page}) => {
 
 test('on_load - navigate triggers on load handler', async ({page}) => {
   await page.goto('/on_load');
-  // Triggers a navigation to /on_load_generator
-  await page.getByRole('button').click();
+
+  await page
+    .getByRole('button', {name: 'navigate to /on_load_generator'})
+    .click();
 
   await assertGeneratorPage(page);
 });

mesop/tests/e2e/viewport_size_test.ts

@@ -17,7 +17,7 @@ test('viewport_size', async ({page}) => {
 test('viewport_size - works for any user event', async ({page}) => {
   await page.goto('/viewport_size');
   expect(await page.getByText('Count:').textContent()).toEqual('Count: 0');
-  await page.getByRole('button').click();
+  await page.getByRole('button', {name: 'on_click should work'}).click();
   // Make sure counter has been incremented to know that user event has been processed.
   expect(await page.getByText('Count: 1').textContent()).toEqual('Count: 1');
 });

mesop/tests/e2e/web_components/csp_violations_test.ts

@@ -1,24 +1,26 @@
-import {test, expect} from '@playwright/test';
+import {expect} from '@playwright/test';
+import {testInProdOnly} from '../e2e_helpers';
 
 // Prevent regression where JS modules were loaded on every page,
 // and not just pages where it was needed.
-test('test CSP violations (from web components) are not shown on home page', async ({
-  page,
-}) => {
-  const cspViolations: string[] = [];
+testInProdOnly(
+  'test CSP violations (from web components) are not shown on home page',
+  async ({page}) => {
+    const cspViolations: string[] = [];
 
-  // Listen for CSP violations
-  page.on('console', (msg) => {
-    if (
-      msg.type() === 'error' &&
-      msg.text().includes('Content Security Policy')
-    ) {
-      cspViolations.push(msg.text());
-    }
-  });
+    // Listen for CSP violations
+    page.on('console', (msg) => {
+      if (
+        msg.type() === 'error' &&
+        msg.text().includes('Content Security Policy')
+      ) {
+        cspViolations.push(msg.text());
+      }
+    });
 
-  await page.goto('/');
-  await page.waitForLoadState('networkidle');
+    await page.goto('/');
+    await page.waitForLoadState('networkidle');
 
-  expect(cspViolations).toHaveLength(0);
-});
+    expect(cspViolations).toHaveLength(0);
+  },
+);

mesop/tests/e2e/web_security_test.ts

@@ -1,19 +1,20 @@
 // http://localhost:32123/plot
-import {test, expect} from '@playwright/test';
+import {expect} from '@playwright/test';
+import {testInProdOnly} from './e2e_helpers';
 
-test('csp: default', async ({page}) => {
+testInProdOnly('csp: default', async ({page}) => {
   const response = await page.goto('/');
   const csp = response?.headers()['content-security-policy']!;
   expect(cleanCsp(csp)).toMatchSnapshot('csp.txt');
 });
 
-test('csp: allowed parent iframe origins', async ({page}) => {
+testInProdOnly('csp: allowed parent iframe origins', async ({page}) => {
   const response = await page.goto('/allowed_iframe_parents');
   const csp = response?.headers()['content-security-policy']!;
   expect(cleanCsp(csp)).toMatchSnapshot('csp_allowed_iframe_parents.txt');
 });
 
-test('csp escaping', async ({page}) => {
+testInProdOnly('csp escaping', async ({page}) => {
   const response = await page.goto('/testing/csp_escaping');
   const csp = response?.headers()['content-security-policy']!;
   expect(cleanCsp(csp)).toMatchSnapshot('csp_escaping.txt');

playwright.config.ts

@@ -53,7 +53,9 @@ export default defineConfig({
   webServer: {
     command: `MESOP_STATE_SESSION_BACKEND=${
       process.env.MESOP_STATE_SESSION_BACKEND || 'none'
-    } bazel run //mesop/cli -- --path=mesop/mesop/example_index.py --prod`,
+    } bazel run //mesop/cli -- --path=mesop/mesop/example_index.py --prod=${
+      process.env.MESOP_DEBUG_MODE === 'true' ? 'false' : 'true'
+    }`,
     url: 'http://127.0.0.1:32123/',
     reuseExistingServer: !process.env.CI,
   },