Very basic support for OpenMetrics (aka Prometheus) (#442)

This PR:

- creates an OpenMetrics server that enables collecting performance data from this process by e.g. a Prometheus server;
- exposes as metrics the performance of http requests with MatrixBot.

Further metrics may of course be added.

config/harness.yaml

@@ -177,6 +177,19 @@ health:
     # Defaults to 418.
     unhealthyStatus: 418
 
+  openMetrics:
+    # Whether openMetrics should be enabled (default false, activated for tests)
+    enabled: true
+
+    # The port to expose the webserver on. Defaults to 8081.
+    port: 9090
+
+    # The address to listen for requests on. Defaults to all addresses.
+    address: "0.0.0.0"
+
+    # The path to expose the monitoring endpoint at. Defaults to `/metrics`
+    endpoint: "/metrics"
+
 # Options for exposing web APIs.
 web:
   # Whether to enable web APIs.

package.json

@@ -58,6 +58,7 @@
     "matrix-appservice-bridge": "8.0.0",
     "parse-duration": "^1.0.2",
     "pg": "^8.8.0",
+    "prom-client": "^14.1.0",
     "shell-quote": "^1.7.3",
     "ulidx": "^0.3.0",
     "yaml": "^2.1.1"

src/Mjolnir.ts

@@ -38,6 +38,7 @@ import { ProtectionManager } from "./protections/ProtectionManager";
 import { RoomMemberManager } from "./RoomMembers";
 import ProtectedRoomsConfig from "./ProtectedRoomsConfig";
 import { MatrixEmitter, MatrixSendClient } from "./MatrixEmitter";
+import { OpenMetrics } from "./webapis/OpenMetrics";
 
 export const STATE_NOT_STARTED = "not_started";
 export const STATE_CHECKING_PERMISSIONS = "checking_permissions";
@@ -64,6 +65,7 @@ export class Mjolnir {
     private protectedRoomsConfig: ProtectedRoomsConfig;
     public readonly protectedRoomsTracker: ProtectedRoomsSet;
     private webapis: WebAPIs;
+    private openMetrics: OpenMetrics;
     public taskQueue: ThrottlingQueue;
     /**
      * Reporting back to the management room.
@@ -233,6 +235,7 @@ export class Mjolnir {
         if (config.pollReports) {
             this.reportPoller = new ReportPoller(this, this.reportManager);
         }
+        this.openMetrics = new OpenMetrics(this.config);
         // Setup join/leave listener
         this.roomJoins = new RoomMemberManager(this.matrixEmitter);
         this.taskQueue = new ThrottlingQueue(this, config.backgroundDelayMS);
@@ -261,6 +264,7 @@ export class Mjolnir {
      * Start Mjölnir.
      */
     public async start() {
+        LogService.info("Mjolnir", "Starting Mjolnir instance");
         try {
             // Start the web server.
             console.log("Starting web server");
@@ -279,7 +283,7 @@ export class Mjolnir {
                 }
                 this.reportPoller.start(reportPollSetting.from);
             }
-
+            await this.openMetrics.start();
             // Load the state.
             this.currentState = STATE_CHECKING_PERMISSIONS;
 
@@ -330,6 +334,7 @@ export class Mjolnir {
         this.matrixEmitter.stop();
         this.webapis.stop();
         this.reportPoller?.stop();
+        this.openMetrics.stop();
     }
 
     /**

src/appservice/AppService.ts

@@ -20,6 +20,7 @@ import { DataStore, PgDataStore } from ".//datastore";
 import { Api } from "./Api";
 import { IConfig } from "./config/config";
 import { AccessControl } from "./AccessControl";
+import { OpenMetrics } from "../webapis/OpenMetrics";
 
 const log = new Logger("AppService");
 /**
@@ -29,6 +30,7 @@ const log = new Logger("AppService");
 export class MjolnirAppService {
 
     private readonly api: Api;
+    private readonly openMetrics: OpenMetrics;
 
     /**
      * The constructor is private because we want to ensure intialization steps are followed,
@@ -42,6 +44,7 @@ export class MjolnirAppService {
         private readonly dataStore: DataStore,
     ) {
         this.api = new Api(config.homeserver.url, mjolnirManager);
+        this.openMetrics = new OpenMetrics(config);
     }
 
     /**
@@ -144,6 +147,8 @@ export class MjolnirAppService {
         this.api.start(this.config.webAPI.port);
         await this.bridge.listen(port);
         log.info("MjolnirAppService started successfully");
+
+        await this.openMetrics.start();
     }
 
     /**
@@ -153,6 +158,7 @@ export class MjolnirAppService {
         await this.bridge.close();
         await this.dataStore.close();
         await this.api.close();
+        this.openMetrics.stop();
     }
 
     /**

src/appservice/cli.ts

@@ -1,6 +1,7 @@
 import { Cli } from "matrix-appservice-bridge";
 import { MjolnirAppService } from "./AppService";
 import { IConfig } from "./config/config";
+import * as utils from "../utils";
 
 /**
  * This file provides the entrypoint for the appservice mode for mjolnir.
@@ -20,6 +21,8 @@ const cli = new Cli({
         if (config === null) {
             throw new Error("Couldn't load config");
         }
+        utils.initializeSentry(config);
+        utils.initializeGlobalPerformanceMetrics(config);
         await MjolnirAppService.run(port, config, cli.getRegistrationFilePath());
     }
 });

src/appservice/config/config.ts

@@ -39,6 +39,42 @@ export interface IConfig {
     accessControlList: string,
     /** configuration for matrix-appservice-bridge's Logger */
     logging?: LoggingOpts,
+    health?: {
+        // If specified, attempt to upload any crash statistics to sentry.
+        sentry?: {
+            dsn: string;
+
+            // Frequency of performance monitoring.
+            //
+            // A number in [0.0, 1.0], where 0.0 means "don't bother with tracing"
+            // and 1.0 means "trace performance at every opportunity".
+            tracesSampleRate: number;
+        };
+        openMetrics?: {
+            /**
+             * If `true`, expose a web server for server metrics, e.g. performance.
+             *
+             * Intended to be used with Prometheus or another Open Metrics scrapper.
+             */
+            enabled: boolean;
+            /**
+             * The port on which to expose server metrics.
+             */
+            port: number;
+            /**
+             * The path at which to collect health metrics.
+             *
+             * If unspecified, use `"/metrics"`.
+             */
+            endpoint: string;
+            /**
+             * If specified, only serve this address mask.
+             *
+             * If unspecified, use 0.0.0.0 (accessible by any host).
+             */
+            address: string;
+        }
+    }
 }
 
 export function read(configPath: string): IConfig {

src/config.ts

@@ -19,6 +19,45 @@ import { load } from "js-yaml";
 import { MatrixClient, LogService } from "matrix-bot-sdk";
 import Config from "config";
 
+export interface IHealthConfig {
+    health?: {
+        // If specified, attempt to upload any crash statistics to sentry.
+        sentry?: {
+            dsn: string;
+
+            // Frequency of performance monitoring.
+            //
+            // A number in [0.0, 1.0], where 0.0 means "don't bother with tracing"
+            // and 1.0 means "trace performance at every opportunity".
+            tracesSampleRate: number;
+        };
+        openMetrics?: {
+            /**
+             * If `true`, expose a web server for server metrics, e.g. performance.
+             *
+             * Intended to be used with Prometheus or another Open Metrics scrapper.
+             */
+            enabled: boolean;
+            /**
+             * The port on which to expose server metrics.
+             */
+            port: number;
+            /**
+             * The path at which to collect health metrics.
+             *
+             * If unspecified, use `"/metrics"`.
+             */
+            endpoint: string;
+            /**
+             * If specified, only serve this address mask.
+             *
+             * If unspecified, use 0.0.0.0 (accessible by any host).
+             */
+            address: string;
+        }
+    }
+}
+
 /**
  * The configuration, as read from production.yaml
  *
@@ -99,6 +138,30 @@ export interface IConfig {
             // and 1.0 means "trace performance at every opportunity".
             tracesSampleRate: number;
         };
+        openMetrics?: {
+            /**
+             * If `true`, expose a web server for server metrics, e.g. performance.
+             *
+             * Intended to be used with Prometheus or another Open Metrics scrapper.
+             */
+            enabled: boolean;
+            /**
+             * The port on which to expose server metrics.
+             */
+            port: number;
+            /**
+             * The path at which to collect health metrics.
+             *
+             * If unspecified, use `"/metrics"`.
+             */
+            endpoint: string;
+            /**
+             * If specified, only serve this address mask.
+             *
+             * If unspecified, use 0.0.0.0 (accessible by any host).
+             */
+            address: string;
+        }
     };
     web: {
         enabled: boolean;
@@ -167,6 +230,12 @@ const defaultConfig: IConfig = {
             healthyStatus: 200,
             unhealthyStatus: 418,
         },
+        openMetrics: {
+            enabled: false,
+            port: 9090,
+            address: "0.0.0.0",
+            endpoint: "/metrics",
+        }
     },
     web: {
         enabled: false,

src/index.ts

@@ -29,7 +29,7 @@ import {
 
 import { read as configRead } from "./config";
 import { Mjolnir } from "./Mjolnir";
-import { initializeSentry, patchMatrixClient } from "./utils";
+import { initializeSentry, initializeGlobalPerformanceMetrics, patchMatrixClient } from "./utils";
 
 
 (async function () {
@@ -46,6 +46,9 @@ import { initializeSentry, patchMatrixClient } from "./utils";
     if (config.health.sentry) {
         initializeSentry(config);
     }
+    if (config.health.openMetrics?.enabled) {
+        initializeGlobalPerformanceMetrics(config);
+    }
     const healthz = new Healthz(config);
     healthz.isHealthy = false; // start off unhealthy
     if (config.health.healthz.enabled) {

src/utils.ts

@@ -25,9 +25,10 @@ import { ClientRequest, IncomingMessage } from "http";
 import { default as parseDuration } from "parse-duration";
 import * as Sentry from '@sentry/node';
 import * as _ from '@sentry/tracing'; // Performing the import activates tracing.
+import { collectDefaultMetrics, Counter, Histogram, register } from "prom-client";
 
 import ManagementRoomOutput from "./ManagementRoomOutput";
-import { IConfig } from "./config";
+import { IHealthConfig } from "./config";
 import { MatrixSendClient } from "./MatrixEmitter";
 
 // Define a few aliases to simplify parsing durations.
@@ -401,17 +402,62 @@ export function patchMatrixClient() {
     patchMatrixClientForRetry();
 }
 
+/**
+ * Initialize performance measurements for the matrix client.
+ *
+ * This method is idempotent. If `config` specifies that Open Metrics
+ * should not be used, it does nothing.
+ */
+export function initializeGlobalPerformanceMetrics(config: IHealthConfig) {
+    if (isGlobalPerformanceMetricsCollectorInitialized || !config.health?.openMetrics?.enabled) {
+        return;
+    }
+
+    // Collect the Prometheus-recommended metrics.
+    collectDefaultMetrics({ register });
+
+    // Collect matrix-bot-sdk-related metrics.
+    let originalRequestFn = getRequestFn();
+    let perfHistogram = new Histogram({
+        name: "mjolnir_performance_http_request",
+        help: "Duration of HTTP requests in seconds",
+    });
+    let successfulRequestsCounter = new Counter({
+        name: "mjolnir_status_api_request_pass",
+        help: "Number of successful API requests",
+    });
+    let failedRequestsCounter = new Counter({
+        name: "mjolnir_status_api_request_fail",
+        help: "Number of failed API requests",
+    });
+    setRequestFn(async (params: { [k: string]: any }, cb: any) => {
+        let timer = perfHistogram.startTimer();
+        return await originalRequestFn(params, function(error: object, response: any, body: string) {
+            // Stop timer before calling callback.
+            timer();
+            if (error) {
+                failedRequestsCounter.inc();
+            } else {
+                successfulRequestsCounter.inc();
+            }
+            cb(error, response, body);
+        });
+    });
+    isGlobalPerformanceMetricsCollectorInitialized = true;
+}
+let isGlobalPerformanceMetricsCollectorInitialized = false;
+
 /**
  * Initialize Sentry for error monitoring and reporting.
  *
  * This method is idempotent. If `config` specifies that Sentry
  * should not be used, it does nothing.
  */
-export function initializeSentry(config: IConfig) {
+export function initializeSentry(config: IHealthConfig) {
     if (sentryInitialized) {
         return;
     }
-    if (config.health.sentry) {
+    if (config.health?.sentry) {
         // Configure error monitoring with Sentry.
         let sentry = config.health.sentry;
         Sentry.init({
@@ -423,4 +469,4 @@ export function initializeSentry(config: IConfig) {
 }
 // Set to `true` once we have initialized `Sentry` to ensure
 // that we do not attempt to initialize it more than once.
-let sentryInitialized = false;
+let sentryInitialized = false;