Github Workflow Monitoring

Signed-off-by: Prudhvi Godithi <pgodithi@amazon.com>

.env.example

@@ -0,0 +1,5 @@
+WEBHOOK_PROXY_URL=
+APP_ID=
+PRIVATE_KEY=
+WEBHOOK_SECRET=
+INSTALLATION_ID=

.github/workflows/build-test.yml

@@ -25,8 +25,10 @@ jobs:
       - name: Verify styling failure msg
         if: steps.style_verify.outcome == 'failure'
         run: echo "Please run 'npm run format' before commiting the code!"
-      - name: Run build
+      - name: Run build test
         run: npm run build
+      - name: Run junit
+        run: npm run junit
       - name: Upload results to Codecov
         uses: codecov/codecov-action@v4
         with:

.github/workflows/check-version-bump.yml

@@ -1,5 +1,5 @@
 ---
-name: Check Version Bump
+name: check-version-bump
 
 on:
   pull_request:

.github/workflows/license-header-checker.yml

@@ -1,5 +1,5 @@
 ---
-name: License Header Checker
+name: license-header-checker
 
 on: [push, pull_request]
 

.github/workflows/releases.yml

@@ -2,12 +2,15 @@ name: Release Automation App
 
 on:
   push:
+    branches:
+      - main
 
 permissions:
   contents: write
 
 jobs:
   release-automation-app:
+    if: github.repository == 'opensearch-project/automation-app'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4

.gitignore

@@ -3,7 +3,7 @@ npm-debug.log
 *.pem
 *.swp*
 !mock-cert.pem
-.env*
+.env
 coverage
 junit.xml
 bin

.licenserc.json

@@ -28,6 +28,7 @@
     ".gz",
     ".toml",
     ".ini",
-    "gradle/wrapper"
+    "gradle/wrapper",
+    ".env.example"
   ]
 }

README.md

@@ -16,7 +16,7 @@ This repository hosts the source code of an automation app to handle the daily a
 
 ## Project Resources
 
-The automation app utilizes the [Probot](https://probot.github.io/) framework and [Octokit](https://docs.github.com/en/rest/using-the-rest-api/libraries-for-the-rest-api?apiVersion=2022-11-28) library to perform user-defined operations on top of the resources within GitHub. See [configs](configs/operations/hello-world.yml) yaml for more information.
+The automation app utilizes the [Probot](https://probot.github.io/) framework and [Octokit](https://docs.github.com/en/rest/using-the-rest-api/libraries-for-the-rest-api?apiVersion=2022-11-28) library to perform user-defined operations on top of the resources within GitHub. See [configs](configs) yaml for more information.
 
 ## Usages
 
@@ -33,8 +33,8 @@ A `Service` is an instance of the app that manages and manipulates specific `Res
 
 To create a service, you need two configuration files:
 
-- **Resource configuration file**: Defines the resources that the service will manage or modify (`configs/resources/sample-resource.yml`).
-- **Operation configuration file**: Defines the operations (tasks) that the service will execute with the resources (`configs/resources/sample-operation.yml`).
+- **Resource configuration file**: Defines the resources that the service will manage or modify. [Sample Resource Config](configs/resources/sample-resource.yml).
+- **Operation configuration file**: Defines the operations (tasks) that the service will execute with the resources. [Sample Operation Config](configs/operations/sample-operation.yml).
 
 ### Start the Service
 
@@ -66,12 +66,12 @@ When you run the above command, the following takes place:
 #### List of Environment Variables (You can use them directly in the startup command, export them, or add them to the `.env` file):
 
 | Name                        | Type    | Default   | Description                                                                                                                                                        | Example                                   |
-| --------------------------- | ------- | --------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------- | --- |
+| --------------------------- | ------- | --------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------- |
 | RESOURCE_CONFIG             | String  | ''        | Path to resource config yaml file.                                                                                                                                 | 'configs/resources/sample-resource.yml'   |
 | OPERATION_CONFIG            | String  | ''        | Path to operation config yaml file.                                                                                                                                | 'configs/operations/sample-operation.yml' |
 | INSTALLATION_ID             | String  | ''        | Installation Id of your GitHub App, must install the App to repositories before retrieving the id.                                                                 | '1234567890'                              |
 | ADDITIONAL_RESOURCE_CONTEXT | Boolean | false     | Setting true will let each resource defined in RESOURCE_CONFIG to call GitHub Rest API and GraphQL for more detailed context (ex: node_id). Increase startup time. | true / false                              |
-| SERVICE_NAME                | String  | 'default' | Set Service Name                                                                                                                                                   | 'My Service'                              | '   |
+| SERVICE_NAME                | String  | 'default' | Set Service Name                                                                                                                                                   | 'My Service'                              |
 
 #### Start the Service with Docker
 

configs/operations/github-workflow-runs-monitor.yml

@@ -9,7 +9,6 @@ tasks:
     call: github-workflow-runs-monitor@default
     args:
       workflows:
-        - 'Publish snapshots to Apache Maven repositories'
         - 'Publish snapshots to maven'
         - 'Run performance benchmark on pull request'
       # https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows

package.json

@@ -1,6 +1,6 @@
 {
   "name": "opensearch-automation-app",
-  "version": "0.1.5",
+  "version": "0.1.7",
   "description": "An Automation App that handles all your GitHub Repository Activities",
   "author": "Peter Zhu",
   "homepage": "https://github.com/opensearch-project/automation-app",
@@ -22,7 +22,8 @@
     "format": "prettier --write src/**/*.ts test/**/*.ts configs/**/*.yml",
     "format-dryrun": "prettier --check src/**/*.ts test/**/*.ts configs/**/*.yml",
     "lint": "eslint --fix \"src/**/*.ts\" --ignore-pattern \"**/*.d.ts\"",
-    "test": "jest --coverage --reporters=jest-junit"
+    "test": "jest --coverage",
+    "junit": "jest --reporters=jest-junit"
   },
   "dependencies": {
     "@aws-sdk/client-cloudwatch": "^3.664.0",

src/call/github-workflow-runs-monitor.ts

@@ -13,10 +13,10 @@
 //  - events     : The list of events to monitor and index, from https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows.
 
 import { Probot } from 'probot';
+import { CloudWatchClient, PutMetricDataCommand } from '@aws-sdk/client-cloudwatch';
 import { Resource } from '../service/resource/resource';
 import { OpensearchClient } from '../utility/opensearch/opensearch-client';
 import { validateResourceConfig } from '../utility/verification/verify-resource';
-import { CloudWatchClient, PutMetricDataCommand } from '@aws-sdk/client-cloudwatch';
 
 interface WorkflowRunMonitorArgs {
   events: string[];
@@ -82,11 +82,8 @@ export default async function githubWorkflowRunsMonitor(
     app.log.error(`Error indexing log data: ${error}`);
   }
 
-  let count: any;
-  if (job?.status === 'completed' && job?.conclusion === 'success') {
-    count = 0;
-  }
-  if (job?.status === 'completed' && job?.conclusion === 'failure') {
+  let count = 0;
+  if (job?.status === 'completed' && (job?.conclusion === 'failure' || job?.conclusion === 'startup_failure')) {
     count = 1;
   }
   try {
@@ -108,7 +105,7 @@ export default async function githubWorkflowRunsMonitor(
       ],
     });
     await cloudWatchClient.send(putMetricDataCommand);
-    app.log.info('CloudWatch metric for workflow failure published.');
+    app.log.info('CloudWatch metric for workflow published.');
   } catch (error) {
     app.log.error(`Error Publishing CloudWatch metric for workflow : ${error}`);
   }

test/call/github-workflow-runs-monitor.test.ts

@@ -10,13 +10,16 @@
 import githubWorkflowRunsMonitor from '../../src/call/github-workflow-runs-monitor';
 import { Probot, Logger } from 'probot';
 import { OpensearchClient } from '../../src/utility/opensearch/opensearch-client';
+import { CloudWatchClient, PutMetricDataCommand } from '@aws-sdk/client-cloudwatch';
 
 jest.mock('../../src/utility/opensearch/opensearch-client');
+jest.mock('@aws-sdk/client-cloudwatch');
 
 describe('githubWorkflowRunsMonitor', () => {
   let app: Probot;
   let context: any;
   let resource: any;
+  let mockCloudWatchClient: any;
 
   beforeEach(() => {
     app = new Probot({ appId: 1, secret: 'test', privateKey: 'test' });
@@ -68,6 +71,11 @@ describe('githubWorkflowRunsMonitor', () => {
         ],
       ]),
     };
+
+    mockCloudWatchClient = {
+      send: jest.fn(),
+    };
+    (CloudWatchClient as jest.Mock).mockImplementation(() => mockCloudWatchClient);
   });
 
   it('should skip indexing when the event is not relevant', async () => {
@@ -132,4 +140,53 @@ describe('githubWorkflowRunsMonitor', () => {
     await githubWorkflowRunsMonitor(app, context, resource, { events, workflows });
     expect(app.log.error).toHaveBeenCalledWith('Error indexing log data: Error: Indexing failed');
   });
+
+  it('should publish CloudWatch metric for a successful workflow run', async () => {
+    const events = ['push', 'pull_request'];
+    const workflows = ['Publish snapshots to maven'];
+
+    mockCloudWatchClient.send.mockResolvedValue({});
+
+    await githubWorkflowRunsMonitor(app, context, resource, { events, workflows });
+
+    expect(mockCloudWatchClient.send).toHaveBeenCalledWith(expect.any(PutMetricDataCommand));
+    expect(app.log.info).toHaveBeenCalledWith('CloudWatch metric for workflow published.');
+  });
+
+  it('should publish CloudWatch metric for a failed workflow run with failure conclusion', async () => {
+    const events = ['push', 'pull_request'];
+    const workflows = ['Publish snapshots to maven'];
+    context.payload.workflow_run.conclusion = 'failure';
+
+    mockCloudWatchClient.send.mockResolvedValue({});
+
+    await githubWorkflowRunsMonitor(app, context, resource, { events, workflows });
+
+    expect(mockCloudWatchClient.send).toHaveBeenCalledWith(expect.any(PutMetricDataCommand));
+    expect(app.log.info).toHaveBeenCalledWith('CloudWatch metric for workflow published.');
+  });
+
+  it('should publish CloudWatch metric for a failed workflow run with startup_failure conclusion', async () => {
+    const events = ['push', 'pull_request'];
+    const workflows = ['Publish snapshots to maven'];
+    context.payload.workflow_run.conclusion = 'startup_failure';
+
+    mockCloudWatchClient.send.mockResolvedValue({});
+
+    await githubWorkflowRunsMonitor(app, context, resource, { events, workflows });
+
+    expect(mockCloudWatchClient.send).toHaveBeenCalledWith(expect.any(PutMetricDataCommand));
+    expect(app.log.info).toHaveBeenCalledWith('CloudWatch metric for workflow published.');
+  });
+
+  it('should log an error if CloudWatch metric publishing fails', async () => {
+    const events = ['push', 'pull_request'];
+    const workflows = ['Publish snapshots to maven'];
+
+    mockCloudWatchClient.send.mockRejectedValue(new Error('CloudWatch error'));
+
+    await githubWorkflowRunsMonitor(app, context, resource, { events, workflows });
+
+    expect(app.log.error).toHaveBeenCalledWith('Error Publishing CloudWatch metric for workflow : Error: CloudWatch error');
+  });
 });