Switch cron jobs from pubsub to HTTP triggers (#872)

* Create jobs/ folder

* Move jobs into src/

* Clean up jobs handlers

* Add comments

* Fix tests

* Re-add old pubsub

* Phantom commit

* Try to fix module dependencies

* Undo ts config changes

* Keep one job endpoint & test

* Move stale-bot

* Move slackScores

* Move slackNotifs, undo move slackScores

* Move SlackScores

* Change paths...?

* Change import

* Update test

* Add tests for coverage

src/buildServer.ts

@@ -15,6 +15,7 @@ import { loadBrain } from '@utils/loadBrain';
 
 import * as PubSub from './webhooks/pubsub';
 import { SENTRY_DSN } from './config';
+import { routeJobs } from './jobs';
 import { SlackRouter } from './slack';
 
 export async function buildServer(
@@ -97,6 +98,9 @@ export async function buildServer(
   // TODO: Unify all these webhooks URL patterns!
   server.post('/webhooks/pubsub', PubSub.opts, PubSub.pubSubHandler);
 
+  // Endpoints for Cloud Scheduler webhooks (Cron Jobs)
+  server.register(routeJobs, { prefix: '/jobs' });
+
   server.post<{ Params: { service: string } }>(
     '/slack/:service/webhook',
     {},

src/jobs/README.md

@@ -0,0 +1,20 @@
+# Google PubSub Handler
+
+This handler will be [tied to a Google PubSub topic][pubsub] to receive paylods and then process them.
+
+Currently, the only available type is `stale-triage-notifier` with a payload in the following shape:
+
+```ts
+type PubSubPayload = {
+  name: string;
+  slo?: number;
+  repos?: string[];
+};
+```
+
+This payload will be sent regularly using the [Cloud Scheduler][cloud_scheduler]
+to notify product owners about their issues pending triage over [our SLO][process_doc].
+
+[pubsub]: https://cloud.google.com/run/docs/tutorials/pubsub#integrating-pubsub
+[cloud_scheduler]: https://cloud.google.com/scheduler/docs/tut-pub-sub#create_a_job
+[process_doc]: https://www.notion.so/sentry/Engaging-Customers-177c77ac473e41eabe9ca7b4bf537537#9d7b15dec9c345618b9195fb5c785e53

src/jobs/gocdPausedPipelineBot.test.ts

@@ -0,0 +1,174 @@
+import { OAuth2Client } from 'google-auth-library';
+import moment from 'moment-timezone';
+
+import { GETSENTRY_ORG } from '@/config';
+import { GoCDDashboardResponse } from '@/types/gocd';
+
+import * as gocdAPI from '../api/gocd/index';
+import { bolt } from '../api/slack/__mocks__';
+
+import { triggerPausedPipelineBot } from './gocdPausedPipelineBot';
+
+const NOW = moment('2024-01-01T00:00:00Z');
+
+jest.mock('@/config', () => {
+  const actualEnvVariables = jest.requireActual('@/config');
+  return {
+    ...actualEnvVariables,
+    GOCD_PAUSED_PIPELINE_REMINDERS: [
+      {
+        pipelineName: 'deploy-test',
+        notifyAfter: moment.duration(1, 'hour'),
+        slackChannel: 'test',
+      },
+      {
+        pipelineName: 'deploy-prod',
+        notifyAfter: moment.duration(1, 'hour'),
+        slackChannel: 'test',
+      },
+    ],
+  };
+});
+
+describe('GoCD Paused Pipeline Notifications', function () {
+  let postMessageSpy: jest.SpyInstance;
+  beforeAll(() => {
+    jest
+      .spyOn(OAuth2Client.prototype, 'verifyIdToken')
+      .mockImplementation(jest.fn());
+    postMessageSpy = jest.spyOn(bolt.client.chat, 'postMessage');
+  });
+
+  afterEach(() => {
+    postMessageSpy.mockClear();
+  });
+
+  it('should not send a message if org is not getsentry', async () => {
+    await triggerPausedPipelineBot({} as any, NOW);
+    expect(postMessageSpy).toHaveBeenCalledTimes(0);
+  });
+
+  it('should not send a message if fetchDashboard fails', async () => {
+    jest.spyOn(gocdAPI, 'fetchDashboard').mockImplementation(() => {
+      return Promise.reject('error');
+    });
+    await triggerPausedPipelineBot(GETSENTRY_ORG, NOW);
+    expect(postMessageSpy).toHaveBeenCalledTimes(0);
+  });
+
+  it('should not send a message to slack if a pipeline is not paused', async () => {
+    jest.spyOn(gocdAPI, 'fetchDashboard').mockImplementation(() => {
+      return Promise.resolve({
+        pipelines: [
+          {
+            name: 'deploy-test',
+            pause_info: {
+              paused: false,
+            },
+          },
+        ],
+      } as GoCDDashboardResponse);
+    });
+    await triggerPausedPipelineBot(GETSENTRY_ORG, NOW);
+    expect(postMessageSpy).toHaveBeenCalledTimes(0);
+  });
+
+  it('should send a message to slack if a pipeline is paused', async () => {
+    jest.spyOn(gocdAPI, 'fetchDashboard').mockImplementation(() => {
+      return Promise.resolve({
+        pipelines: [
+          {
+            name: 'deploy-test',
+            pause_info: {
+              paused: true,
+              paused_at: NOW.clone().subtract(2, 'hours').toISOString(),
+            },
+          },
+        ],
+      } as GoCDDashboardResponse);
+    });
+    await triggerPausedPipelineBot(GETSENTRY_ORG, NOW);
+    expect(postMessageSpy).toBeCalledWith({
+      channel: 'test',
+      text: 'A pipeline has been paused for an extended period of time. If this is unintentional, please look into unpausing it if it is safe to do so.',
+      blocks: [
+        {
+          type: 'section',
+          text: {
+            type: 'mrkdwn',
+            text: 'A pipeline has been paused for an extended period of time. If this is unintentional, please look into unpausing it if it is safe to do so.',
+          },
+        },
+        {
+          type: 'section',
+          text: {
+            type: 'mrkdwn',
+            text: ':warning: *<https://deploy.getsentry.net/go/tab/pipeline/history/deploy-test|deploy-test>*: has been paused for 2 hours',
+          },
+        },
+      ],
+    });
+  });
+
+  it('should not send a message to slack if the pipeline has not been paused long enough', async () => {
+    jest.spyOn(gocdAPI, 'fetchDashboard').mockImplementation(() => {
+      return Promise.resolve({
+        pipelines: [
+          {
+            name: 'deploy-test',
+            pause_info: {
+              paused: true,
+              paused_at: NOW.clone().subtract(30, 'minutes').toISOString(),
+            },
+          },
+        ],
+      } as GoCDDashboardResponse);
+    });
+    await triggerPausedPipelineBot(GETSENTRY_ORG, NOW);
+    expect(postMessageSpy).toHaveBeenCalledTimes(0);
+  });
+
+  it('should send a message if multiple pipelines are paused', async () => {
+    jest.spyOn(gocdAPI, 'fetchDashboard').mockImplementation(() => {
+      return Promise.resolve({
+        pipelines: [
+          {
+            name: 'deploy-test',
+            pause_info: {
+              paused: true,
+              paused_at: NOW.clone().subtract(2, 'hours').toISOString(),
+            },
+          },
+          {
+            name: 'deploy-prod',
+            pause_info: {
+              paused: true,
+              paused_at: NOW.clone().subtract(2, 'hours').toISOString(),
+            },
+          },
+        ],
+      } as GoCDDashboardResponse);
+    });
+    await triggerPausedPipelineBot(GETSENTRY_ORG, NOW);
+    expect(postMessageSpy).toBeCalledWith({
+      channel: 'test',
+      text: 'Multiple pipelines have been paused for an extended period of time. If this is unintentional, please look into unpausing them if it is safe to do so.',
+      blocks: [
+        {
+          type: 'section',
+          text: {
+            type: 'mrkdwn',
+            text: 'Multiple pipelines have been paused for an extended period of time. If this is unintentional, please look into unpausing them if it is safe to do so.',
+          },
+        },
+        {
+          type: 'section',
+          text: {
+            type: 'mrkdwn',
+            text: ':warning: *<https://deploy.getsentry.net/go/tab/pipeline/history/deploy-test|deploy-test>*: has been paused for 2 hours\n:warning: *<https://deploy.getsentry.net/go/tab/pipeline/history/deploy-prod|deploy-prod>*: has been paused for 2 hours',
+          },
+        },
+      ],
+    });
+  });
+});

src/jobs/gocdPausedPipelineBot.ts

@@ -0,0 +1,135 @@
+import * as Sentry from '@sentry/node';
+import moment from 'moment-timezone';
+
+import { GitHubOrg } from '@/api/github/org';
+import { fetchDashboard } from '@/api/gocd';
+import { bolt } from '@/api/slack';
+import {
+  GETSENTRY_ORG,
+  GOCD_ORIGIN,
+  GOCD_PAUSED_PIPELINE_REMINDERS,
+} from '@/config';
+import { GoCDDashboardResponse } from '@/types/gocd';
+
+const GOCD_PAUSED_PIPELINE_REMINDER_TEXT_SINGULAR =
+  'A pipeline has been paused for an extended period of time. If this is unintentional, please look into unpausing it if it is safe to do so.';
+const GOCD_PAUSED_PIPELINE_REMINDER_TEXT_PLURAL =
+  'Multiple pipelines have been paused for an extended period of time. If this is unintentional, please look into unpausing them if it is safe to do so.';
+
+type PausedPipelineInfo = {
+  pipelineName: string;
+  durationPaused: moment.Duration;
+};
+
+export const triggerPausedPipelineBot = async (
+  org: GitHubOrg,
+  now: moment.Moment
+) => {
+  if (org !== GETSENTRY_ORG) {
+    return;
+  }
+  let dashboardResult: GoCDDashboardResponse;
+  try {
+    dashboardResult = await fetchDashboard();
+  } catch (err) {
+    Sentry.captureException(err);
+    return;
+  }
+  const remindersByChannel = getRemindersByChannel(dashboardResult, now);
+  for (const [channel, pausedPipelineInfos] of remindersByChannel.entries()) {
+    const pausedPipelineReminderText = getReminderText(pausedPipelineInfos);
+    await postMessageToSlack(
+      channel,
+      pausedPipelineReminderText,
+      pausedPipelineInfos
+    );
+  }
+};
+
+function getRemindersByChannel(
+  dashboardResult: GoCDDashboardResponse,
+  now: moment.Moment
+): Map<string, PausedPipelineInfo[]> {
+  const remindersByChannel = new Map<string, PausedPipelineInfo[]>();
+  for (const pipeline of dashboardResult.pipelines) {
+    const pauseInfo = pipeline.pause_info;
+    const pausedPipelineReminder = GOCD_PAUSED_PIPELINE_REMINDERS.find(
+      (reminder) => reminder.pipelineName === pipeline.name
+    );
+    if (
+      pausedPipelineReminder === undefined ||
+      pauseInfo === undefined ||
+      !pauseInfo.paused ||
+      pauseInfo.paused_at == null
+    ) {
+      continue;
+    }
+    const durationPaused = moment.duration(now.diff(pauseInfo.paused_at));
+    if (durationPaused >= pausedPipelineReminder.notifyAfter) {
+      const pausedPipelineInfos =
+        remindersByChannel.get(pausedPipelineReminder.slackChannel) ?? [];
+      pausedPipelineInfos.push({
+        pipelineName: pipeline.name,
+        durationPaused,
+      });
+      remindersByChannel.set(
+        pausedPipelineReminder.slackChannel,
+        pausedPipelineInfos
+      );
+    }
+  }
+  return remindersByChannel;
+}
+
+async function postMessageToSlack(
+  channel: string,
+  pausedPipelineReminderText: string,
+  pausedPipelineInfos: PausedPipelineInfo[]
+) {
+  const pausedPipelineReminderBlocks = generatePausedPipelineReminderBlocks(
+    pausedPipelineReminderText,
+    pausedPipelineInfos
+  );
+  await bolt.client.chat.postMessage({
+    channel,
+    text: pausedPipelineReminderText,
+    blocks: pausedPipelineReminderBlocks,
+  });
+}
+
+function generatePausedPipelineReminderBlocks(
+  pausedPipelineReminderText: string,
+  pausedPipelines: PausedPipelineInfo[]
+) {
+  const pausedPipelineWarningTexts = pausedPipelines
+    .map((info) => {
+      return `:warning: *<${GOCD_ORIGIN}/go/tab/pipeline/history/${
+        info.pipelineName
+      }|${
+        info.pipelineName
+      }>*: has been paused for ${info.durationPaused.humanize()}`;
+    })
+    .join('\n');
+  return [
+    {
+      type: 'section',
+      text: {
+        type: 'mrkdwn',
+        text: pausedPipelineReminderText,
+      },
+    },
+    {
+      type: 'section',
+      text: {
+        type: 'mrkdwn',
+        text: pausedPipelineWarningTexts,
+      },
+    },
+  ];
+}
+
+function getReminderText(pausedPipelinesInfos: PausedPipelineInfo[]): string {
+  return pausedPipelinesInfos.length > 1
+    ? GOCD_PAUSED_PIPELINE_REMINDER_TEXT_PLURAL
+    : GOCD_PAUSED_PIPELINE_REMINDER_TEXT_SINGULAR;
+}