CI: improve error reporting mechanism

.circleci/config.yml

@@ -176,30 +176,22 @@ commands:
             echo "Canceling workflow as previous step resulted in failure."
             echo "To execute all checks locally, please run yarn ci-tests"
             curl -X POST --header "Content-Type: application/json" "https://circleci.com/api/v2/workflow/${CIRCLE_WORKFLOW_ID}/cancel?circle-token=${WORKFLOW_CANCELER}"
-  list-recent-merges:
-    description: 'Lists the merge commits from the past 24 hours'
-    steps:
-      - run:
-          name: Listing merge commits from the past 24 hours
-          when: on_fail
-          command: |
-            git log --merges --since="24 hours ago" --pretty=format:"\`%h\` %<(12)%ar | %s [%an]" | grep "Merge pull request" | sed "s/Merge pull request #/https:\/\/github.com\/storybookjs\/storybook\/pull\//g"
   report-workflow-on-failure:
     description: 'Reports failures to discord'
     parameters:
       template:
         description: |
-          Which template to report in discord
+          Which template to report in discord. Applicable for parallel sandbox jobs
         type: string
+        default: 'none'
     steps:
       - run:
           when: on_fail
           command: git fetch --unshallow
-      - list-recent-merges
       - discord/status:
-          only_for_branches: next,alpha
+          only_for_branches: main,next,alpha
           fail_only: true
-          failure_message: 'Oh no! The **$CIRCLE_JOB** job has failed for **<< parameters.template >>**.\n\n**Relevant PRs of the last 24h:**\n$(git log --merges --since="24 hours ago" --pretty=format:"\`%h\` %<(12)%ar %s [%an]" | grep "Merge pull request" | sed "s/Merge pull request #/https:\/\/github.com\/storybookjs\/storybook\/pull\//g" | tr "\\n" "\\\\n" | sed "s/\\\\/\\\\n/g")'
+          failure_message: $(yarn get-report-message << pipeline.parameters.workflow >> << parameters.template >>)
 jobs:
   build:
     executor:
@@ -222,6 +214,7 @@ jobs:
           command: |
             cd code
             yarn local-registry --publish
+      - report-workflow-on-failure
       - save_cache:
           name: Save Yarn cache
           key: build-yarn-2-cache-v4--{{ checksum "code/yarn.lock" }}--{{ checksum "scripts/yarn.lock" }}
@@ -274,6 +267,7 @@ jobs:
             rm -rf node_modules
             mkdir node_modules
             npx -p @storybook/bench@next sb-bench 'yarn install' --label cra
+      - report-workflow-on-failure
       - run:
           name: prep artifacts
           when: always
@@ -314,6 +308,7 @@ jobs:
             rm -rf node_modules
             mkdir node_modules
             npx -p @storybook/bench@next sb-bench 'yarn install' --label react-vite
+      - report-workflow-on-failure
       - run:
           name: prep artifacts
           when: always
@@ -335,6 +330,7 @@ jobs:
           command: |
             cd code
             yarn lint
+      - report-workflow-on-failure
       - cancel-workflow-on-failure
   check:
     executor:
@@ -350,6 +346,7 @@ jobs:
           command: |
             yarn task --task check --start-from=auto --no-link --debug
             git diff --exit-code
+      - report-workflow-on-failure
       - cancel-workflow-on-failure
   script-unit-tests:
     executor: sb_node_16_browsers
@@ -365,6 +362,7 @@ jobs:
             yarn test --coverage --ci
       - store_test_results:
           path: scripts/junit.xml
+      - report-workflow-on-failure
       - cancel-workflow-on-failure
   unit-tests:
     executor:
@@ -386,6 +384,7 @@ jobs:
           root: .
           paths:
             - code/coverage
+      - report-workflow-on-failure
       - cancel-workflow-on-failure
   coverage:
     executor:
@@ -401,6 +400,7 @@ jobs:
           command: |
             cd code
             yarn coverage
+      - report-workflow-on-failure
   chromatic-internal-storybooks:
     executor:
       class: medium
@@ -416,6 +416,7 @@ jobs:
             cd code
             yarn storybook:ui:chromatic
             yarn storybook:blocks:chromatic
+      - report-workflow-on-failure
       - store_test_results:
           path: test-results
   ## new workflow
@@ -527,6 +528,8 @@ jobs:
       - run:
           name: Running Chromatic
           command: yarn task --task chromatic --template $(yarn get-template << pipeline.parameters.workflow >> chromatic) --no-link --start-from=never --junit
+      - report-workflow-on-failure:
+          template: $(yarn get-template << pipeline.parameters.workflow >> chromatic)
       - store_test_results:
           path: test-results
   e2e-sandboxes:

package.json

@@ -3,6 +3,7 @@
     "start": "yarn task --task dev --template react-vite/default-ts --start-from=install",
     "task": "echo 'Installing Script Dependencies...'; cd scripts; yarn install >/dev/null; yarn task",
     "get-template": "cd scripts; yarn get-template",
+    "get-report-message": "cd scripts; yarn get-report-message",
     "test": "cd code; yarn test",
     "lint": "cd code; yarn lint",
     "nx": "cd code; yarn nx",

scripts/get-report-message.ts

@@ -0,0 +1,69 @@
+/* eslint-disable no-console */
+import { command } from 'execa';
+import { readJson } from 'fs-extra';
+import { join } from 'path';
+
+type Branch = 'main' | 'next' | 'alpha';
+type Workflow = 'merged' | 'daily';
+
+const getFooter = async (branch: Branch, workflow: Workflow, job: string) => {
+  if (job === 'chromatic-sandboxes') {
+    return `\n\nThis might not necessarily be a bug, it could be a visual diff that you have to review and approve. Please check it!`;
+  }
+
+  if (branch === 'alpha') {
+    const packageJson = await readJson(join(__dirname, '..', 'code', 'package.json'));
+
+    // running in alpha branch we should just show the version which failed
+    return `\n**Version: ${packageJson.version}**`;
+  }
+
+  const mergeCommits =
+    workflow === 'merged'
+      ? // show single merge for merged workflow
+        `git log -1 --pretty=format:"\`%h\` %<(12)%ar %s [%an]"`
+      : // show last 24h merges for daily workflow
+        `git log --merges --since="24 hours ago" --pretty=format:"\`%h\` %<(12)%ar %s [%an]"`;
+
+  const result = await command(mergeCommits, { shell: true });
+  const formattedResult = result.stdout
+    // discord needs escaped line breaks
+    .replace(/\n/g, '\\n')
+    // make links out of pull request ids
+    .replace(/Merge pull request #/g, 'https://github.com/storybookjs/storybook/pull/');
+
+  return `\n\n**Relevant PRs:**\n${formattedResult}`;
+};
+
+// This command is run in Circle CI on failures, to get a rich message to report to Discord
+// Usage: yarn get-report-message type workflow branch
+async function run() {
+  const [, , workflow = '', template = 'none'] = process.argv;
+
+  if (!workflow) {
+    throw new Error('[get-report-message] Missing workflow argument.');
+  }
+
+  const { CIRCLE_BRANCH: currentBranch = '', CIRCLE_JOB: currentJob = '' } = process.env;
+
+  if (!currentBranch || !currentJob) {
+    throw new Error(
+      '[get-report-message] Missing CIRCLE_BRANCH or CIRCLE_JOB environment variables.'
+    );
+  }
+
+  const title = `Oh no! The **${currentJob}** job has failed${
+    template !== 'none' ? ` for **${template}**.` : '.'
+  }`;
+  const body = `\n\n**Branch**: \`${currentBranch}\`\n**Workflow:** ${workflow}`;
+  const footer = await getFooter(currentBranch as Branch, workflow as Workflow, currentJob);
+
+  console.log(`${title}${body}${footer}`.replace(/\n/g, '\\n'));
+}
+
+if (require.main === module) {
+  run().catch((err) => {
+    console.error(err);
+    process.exit(1);
+  });
+}

scripts/package.json

@@ -3,6 +3,7 @@
   "version": "7.0.0-alpha.16",
   "private": true,
   "scripts": {
+    "get-report-message": "ts-node ./get-report-message.ts",
     "get-template": "ts-node ./get-template.ts",
     "lint": "yarn lint:js && yarn lint:md",
     "lint:js": "yarn lint:js:cmd .  --quiet",