Feat: Add errorMessageFormatter

.storybook/test-runner.ts

@@ -8,6 +8,9 @@ const skipSnapshots = process.env.SKIP_SNAPSHOTS === 'true';
 
 const config: TestRunnerConfig = {
   logLevel: 'verbose',
+  errorMessageFormatter: (message) => {
+    return message;
+  },
   tags: {
     exclude: ['exclude'],
     include: [],

README.md

@@ -729,6 +729,23 @@ const config: TestRunnerConfig = {
 export default config;
 ```
 
+#### errorMessageFormatter
+
+The `errorMessageFormatter` property defines a function that will pre-format the error messages before they get reported in the CLI:
+
+```ts
+// .storybook/test-runner.ts
+import type { TestRunnerConfig } from '@storybook/test-runner';
+
+const config: TestRunnerConfig = {
+  errorMessageFormatter: (message) => {
+    // manipulate the error message as you like
+    return message;
+  },
+};
+export default config;
+```
+
 ### Utility functions
 
 For more specific use cases, the test runner provides utility functions that could be useful to you.

src/playwright/hooks.ts

@@ -63,6 +63,11 @@ export interface TestRunnerConfig {
    * @default 'info'
    */
   logLevel?: 'info' | 'warn' | 'error' | 'verbose' | 'none';
+
+  /**
+   * Defines a custom function to process the error message. Useful to sanitize error messages or to add additional information.
+   */
+  errorMessageFormatter?: (error: string) => string;
 }
 
 export const setPreVisit = (preVisit: TestHook) => {

src/setup-page-script.ts

@@ -30,7 +30,8 @@ const TEST_RUNNER_DEBUG_PRINT_LIMIT = parseInt('{{debugPrintLimit}}', 10);
 // Type definitions for globals
 declare global {
   // this is defined in setup-page.ts and can be used for logging from the browser to node, helpful for debugging
-  var logToPage: (message: string) => void;
+  var logToPage: (message: string) => Promise<void>;
+  var testRunner_errorMessageFormatter: (message: string) => Promise<string>;
 }
 
 // Type definitions for function parameters and return types
@@ -205,21 +206,39 @@ function addToUserAgent(extra: string): void {
 
 // Custom error class
 class StorybookTestRunnerError extends Error {
-  constructor(storyId: string, errorMessage: string, logs: string[] = []) {
-    super(errorMessage);
+  constructor(
+    storyId: string,
+    errorMessage: string,
+    logs: string[] = [],
+    isMessageFormatted: boolean = false
+  ) {
+    const message = isMessageFormatted
+      ? errorMessage
+      : StorybookTestRunnerError.buildErrorMessage(storyId, errorMessage, logs);
+    super(message);
+
     this.name = 'StorybookTestRunnerError';
+  }
+
+  public static buildErrorMessage(
+    storyId: string,
+    errorMessage: string,
+    logs: string[] = []
+  ): string {
     const storyUrl = `${TEST_RUNNER_STORYBOOK_URL}?path=/story/${storyId}`;
     const finalStoryUrl = `${storyUrl}&addonPanel=storybook/interactions/panel`;
     const separator = '\n\n--------------------------------------------------';
     // The original error message will also be collected in the logs, so we filter it to avoid duplication
-    const finalLogs = logs.filter((err) => !err.includes(errorMessage));
+    const finalLogs = logs.filter((err: string) => !err.includes(errorMessage));
     const extraLogs =
       finalLogs.length > 0 ? separator + '\n\nBrowser logs:\n\n' + finalLogs.join('\n\n') : '';
 
-    this.message = `\nAn error occurred in the following story. Access the link for full output:\n${finalStoryUrl}\n\nMessage:\n ${truncate(
+    const message = `\nAn error occurred in the following story. Access the link for full output:\n${finalStoryUrl}\n\nMessage:\n ${truncate(
       errorMessage,
       TEST_RUNNER_DEBUG_PRINT_LIMIT
     )}\n${extraLogs}`;
+
+    return message;
   }
 }
 
@@ -351,13 +370,32 @@ async function __test(storyId: string): Promise<any> {
   };
 
   return new Promise((resolve, reject) => {
+    const rejectWithFormattedError = (storyId: string, message: string) => {
+      const errorMessage = StorybookTestRunnerError.buildErrorMessage(storyId, message, logs);
+
+      testRunner_errorMessageFormatter(errorMessage)
+        .then((formattedMessage) => {
+          reject(new StorybookTestRunnerError(storyId, formattedMessage, logs, true));
+        })
+        .catch((error) => {
+          reject(
+            new StorybookTestRunnerError(
+              storyId,
+              'There was an error when executing the errorMessageFormatter defiend in your Storybook test-runner config file. Please fix it and rerun the tests:\n\n' +
+                error.message
+            )
+          );
+        });
+    };
+
     const listeners = {
       [TEST_RUNNER_RENDERED_EVENT]: () => {
         cleanup(listeners);
         if (hasErrors) {
-          reject(new StorybookTestRunnerError(storyId, 'Browser console errors', logs));
+          rejectWithFormattedError(storyId, 'Browser console errors');
+        } else {
+          resolve(document.getElementById('root'));
         }
-        resolve(document.getElementById('root'));
       },
 
       storyUnchanged: () => {
@@ -367,34 +405,29 @@ async function __test(storyId: string): Promise<any> {
 
       storyErrored: ({ description }: { description: string }) => {
         cleanup(listeners);
-        reject(new StorybookTestRunnerError(storyId, description, logs));
+        rejectWithFormattedError(storyId, description);
       },
 
       storyThrewException: (error: Error) => {
         cleanup(listeners);
-        reject(new StorybookTestRunnerError(storyId, error.message, logs));
+        rejectWithFormattedError(storyId, error.message);
       },
 
       playFunctionThrewException: (error: Error) => {
         cleanup(listeners);
-        reject(new StorybookTestRunnerError(storyId, error.message, logs));
+
+        rejectWithFormattedError(storyId, error.message);
       },
 
       unhandledErrorsWhilePlaying: ([error]: Error[]) => {
         cleanup(listeners);
-        reject(new StorybookTestRunnerError(storyId, error.message, logs));
+        rejectWithFormattedError(storyId, error.message);
       },
 
       storyMissing: (id: string) => {
         cleanup(listeners);
         if (id === storyId) {
-          reject(
-            new StorybookTestRunnerError(
-              storyId,
-              'The story was missing when trying to access it.',
-              logs
-            )
-          );
+          rejectWithFormattedError(storyId, 'The story was missing when trying to access it.');
         }
       },
     };

src/setup-page.ts

@@ -61,6 +61,14 @@ export const setupPage = async (page: Page, browserContext: BrowserContext) => {
   // if we ever want to log something from the browser to node
   await page.exposeBinding('logToPage', (_, message) => console.log(message));
 
+  await page.exposeBinding('testRunner_errorMessageFormatter', (_, message: string) => {
+    if (testRunnerConfig.errorMessageFormatter) {
+      return testRunnerConfig.errorMessageFormatter(message);
+    }
+
+    return message;
+  });
+
   const finalStorybookUrl = referenceURL ?? targetURL ?? '';
   const testRunnerPackageLocation = await pkgUp({ cwd: __dirname });
   if (!testRunnerPackageLocation) throw new Error('Could not find test-runner package location');

yarn.lock

@@ -437,10 +437,10 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@babel/helper-string-parser@npm:^7.24.1":
-  version: 7.24.1
-  resolution: "@babel/helper-string-parser@npm:7.24.1"
-  checksum: 8404e865b06013979a12406aab4c0e8d2e377199deec09dfe9f57b833b0c9ce7b6e8c1c553f2da8d0bcd240c5005bd7a269f4fef0d628aeb7d5fe035c436fb67
+"@babel/helper-string-parser@npm:^7.24.7":
+  version: 7.24.7
+  resolution: "@babel/helper-string-parser@npm:7.24.7"
+  checksum: 09568193044a578743dd44bf7397940c27ea693f9812d24acb700890636b376847a611cdd0393a928544e79d7ad5b8b916bd8e6e772bc8a10c48a647a96e7b1a
   languageName: node
   linkType: hard
 
@@ -451,6 +451,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@babel/helper-validator-identifier@npm:^7.24.7":
+  version: 7.24.7
+  resolution: "@babel/helper-validator-identifier@npm:7.24.7"
+  checksum: 6799ab117cefc0ecd35cd0b40ead320c621a298ecac88686a14cffceaac89d80cdb3c178f969861bf5fa5e4f766648f9161ea0752ecfe080d8e89e3147270257
+  languageName: node
+  linkType: hard
+
 "@babel/helper-validator-option@npm:^7.23.5":
   version: 7.23.5
   resolution: "@babel/helper-validator-option@npm:7.23.5"
@@ -1686,13 +1693,13 @@ __metadata:
   linkType: hard
 
 "@babel/types@npm:^7.0.0, @babel/types@npm:^7.18.9, @babel/types@npm:^7.20.7, @babel/types@npm:^7.22.15, @babel/types@npm:^7.22.5, @babel/types@npm:^7.23.0, @babel/types@npm:^7.23.4, @babel/types@npm:^7.24.0, @babel/types@npm:^7.24.5, @babel/types@npm:^7.3.3, @babel/types@npm:^7.4.4, @babel/types@npm:^7.8.3":
-  version: 7.24.5
-  resolution: "@babel/types@npm:7.24.5"
+  version: 7.24.7
+  resolution: "@babel/types@npm:7.24.7"
   dependencies:
-    "@babel/helper-string-parser": ^7.24.1
-    "@babel/helper-validator-identifier": ^7.24.5
+    "@babel/helper-string-parser": ^7.24.7
+    "@babel/helper-validator-identifier": ^7.24.7
     to-fast-properties: ^2.0.0
-  checksum: 8eeeacd996593b176e649ee49d8dc3f26f9bb6aa1e3b592030e61a0e58ea010fb018dccc51e5314c8139409ea6cbab02e29b33e674e1f6962d8e24c52da6375b
+  checksum: 3e4437fced97e02982972ce5bebd318c47d42c9be2152c0fd28c6f786cc74086cc0a8fb83b602b846e41df37f22c36254338eada1a47ef9d8a1ec92332ca3ea8
   languageName: node
   linkType: hard