Merge pull request #321 from work933k/console-error-checking

Add ability to fail on console errors

README.md

@@ -151,6 +151,7 @@ Usage: test-storybook [options]
 | `--junit`                         | Indicates that test information should be reported in a junit file. <br/>`test-storybook --**junit**`                                                                         |
 | `--ci`                            | Instead of the regular behavior of storing a new snapshot automatically, it will fail the test and require Jest to be run with `--updateSnapshot`. <br/>`test-storybook --ci` |
 | `--shard [shardIndex/shardCount]` | Splits your test suite across different machines to run in CI. <br/>`test-storybook --shard=1/3`                                                                              |
+| `--failOnConsole`                 | Makes tests fail on browser console errors<br/>`test-storybook --failOnConsole`                                                                                               |
 
 ## Ejecting configuration
 

package.json

@@ -37,7 +37,7 @@
     "build-storybook": "storybook build",
     "release": "yarn build && auto shipit",
     "test-storybook": "node dist/test-storybook",
-    "test-storybook:failures": "SKIP_SNAPSHOTS=true TEST_FAILURES=1 yarn test-storybook --json --outputFile test-results.json",
+    "test-storybook:failures": "SKIP_SNAPSHOTS=true TEST_FAILURES=1 yarn test-storybook --json --failOnConsole --outputFile test-results.json",
     "test-storybook:no-cache": "yarn test-storybook --no-cache",
     "test-storybook:json": "yarn test-storybook --stories-json",
     "test-storybook:ci": "concurrently -k -s first -n \"SB,TEST\" -c \"magenta,blue\" \"yarn build-storybook --quiet && npx serve storybook-static -l 6006\" \"wait-on tcp:6006 && yarn test-storybook\"",

src/setup-page.ts

@@ -51,6 +51,7 @@ const sanitizeURL = (url: string) => {
 
 export const setupPage = async (page: Page, browserContext: BrowserContext) => {
   const targetURL = process.env.TARGET_URL;
+  const failOnConsole = process.env.TEST_CHECK_CONSOLE;
 
   const viewMode = process.env.VIEW_MODE || 'story';
   const renderedEvent = viewMode === 'docs' ? 'docsRendered' : 'storyRendered';
@@ -215,6 +216,9 @@ export const setupPage = async (page: Page, browserContext: BrowserContext) => {
       // end of fast-safe-stringify code
       
       function composeMessage(args) {
+        if (args instanceof Error) {
+          return \`\${args.name}: \${args.message}\\n\${args.stack}\`;
+        }
         if (typeof args === 'undefined') return "undefined";
         if (typeof args === 'string') return args;
         return stringify(args, null, null, { depthLimit: 5, edgesLimit: 100 });
@@ -306,10 +310,14 @@ export const setupPage = async (page: Page, browserContext: BrowserContext) => {
 
         // collect logs to show upon test error
         let logs = [];
+        let hasErrors = false;
 
         const spyOnConsole = (method, name) => {
           const originalFn = console[method];
           return function () {
+            if (\`${failOnConsole}\`==='true' && method==='error') {
+              hasErrors = true;
+            }
             const message = [...arguments].map(composeMessage).join(', ');
             const prefix = \`\${bold(name)}: \`;
             logs.push(prefix + message);
@@ -332,7 +340,12 @@ export const setupPage = async (page: Page, browserContext: BrowserContext) => {
         })
 
         return new Promise((resolve, reject) => {
-          channel.on('${renderedEvent}', () => resolve(document.getElementById('root')));
+          channel.on('${renderedEvent}', () => { 
+            if (hasErrors) {
+              return reject(new StorybookTestRunnerError(storyId, 'Browser console errors', logs));
+            }
+            return resolve(document.getElementById('root'));             
+          });
           channel.on('storyUnchanged', () => resolve(document.getElementById('root')));
           channel.on('storyErrored', ({ description }) => reject(
             new StorybookTestRunnerError(storyId, description, logs))

src/test-storybook.ts

@@ -304,6 +304,10 @@ const main = async () => {
   const { storiesPaths, lazyCompilation } = getStorybookMetadata();
   process.env.STORYBOOK_STORIES_PATTERN = storiesPaths;
 
+  if (runnerOptions.failOnConsole) {
+    process.env.TEST_CHECK_CONSOLE = 'true';
+  }
+
   if (lazyCompilation && isLocalStorybookIp) {
     log(
       `You're running Storybook with lazy compilation enabled, and will likely cause issues with the test runner locally. Consider disabling 'lazyCompilation' in ${runnerOptions.configDir}/main.js when running 'test-storybook' locally.`

src/util/getCliOptions.test.ts

@@ -17,6 +17,15 @@ describe('getCliOptions', () => {
     expect(opts.runnerOptions).toMatchObject(customConfig);
   });
 
+  it('returns failOnConsole option if passed', () => {
+    const customConfig = { failOnConsole: true };
+    jest
+      .spyOn(cliHelper, 'getParsedCliOptions')
+      .mockReturnValue({ options: customConfig, extraArgs: [] });
+    const opts = getCliOptions();
+    expect(opts.runnerOptions).toMatchObject(customConfig);
+  });
+
   it('returns extra args if passed', () => {
     const extraArgs = ['TestName', 'AnotherTestName'];
     // mock argv to avoid side effect from running tests e.g. jest --coverage,

src/util/getCliOptions.ts

@@ -13,6 +13,7 @@ export type CliOptions = {
     coverageDirectory?: string;
     junit?: boolean;
     browsers?: BrowserType | BrowserType[];
+    failOnConsole?: boolean;
   };
   jestOptions: JestOptions;
 };
@@ -28,6 +29,7 @@ const STORYBOOK_RUNNER_COMMANDS: StorybookRunnerCommand[] = [
   'coverage',
   'coverageDirectory',
   'junit',
+  'failOnConsole',
 ];
 
 function copyOption<ObjType extends object, KeyType extends keyof ObjType>(

src/util/getParsedCliOptions.ts

@@ -75,7 +75,8 @@ export const getParsedCliOptions = (): ParsedCliOptions => {
     .option(
       '--shard <shardIndex/shardCount>',
       'Splits your test suite across different machines to run in CI.'
-    );
+    )
+    .option('--failOnConsole', 'Makes tests fail on browser console errors');
 
   program.exitOverride();
 

stories/expected-failures/Failure.stories.tsx

@@ -10,6 +10,11 @@ export default {
 
 const Template = (args) => <Page {...args} />;
 
+export const ComponentLogsErrors = () => {
+  console.error('Console error with a failure');
+  return <div>Hello world</div>;
+};
+
 export const ComponentThrowsErrors = () => {
   throw new Error('Component has a failure');
 };