Development monitoring (#2045)

* init monitoring package

* add helper method
Make http request using axios

* create HealthCheck class
this is the generic class for checking if the service is alive
create GridProxy class

* create TFchain health check class with example

* rewrite classe, now implements the interface

* make the code scalable:
add some types.
add initializeServices
add monitoring interval

* add generateString function

* enhance interfaces

* apply interface changes

* add alivenessChecker

* update isAlive interface
now it return ServiceStatus

* update isAlive methods

* add event emitter

* add colorize function and type

* update imports

* update alivenessCheckers

* update alivenessCheckers and add js docs

* update rmb constractor to recive the mnemonic

* add summary event

* update example

* remove rmb error logs

* Revert "remove rmb error logs"

This reverts commit 8d4e930.

* remove rmb error logs

* add disconnect Interface, use it on rmb

* use tfChain client instade of polkadot

* add disconnect method and update example script

* change the example path

* use Chalk: text colorization

* fix: remove body and make header optional in axios get request

* add resolveSErviceStatus function and apply it

* add exit and disconnect function

* update example

* enahnce: rename ILivenessChecker class and rewrite IServiceInfo clase

* enahnce: add ts docs

* enahnce: add info getters to classes

* enahnce: add work around to check if the service implements the IDisconnectHandler

* enahnce: handle axios error message

* rewrite: monitor events now exposing an instance of class that contains all needed events and thier handlers

* cleanUp: remove debugging logs

* rewrite: create serviceMonitor class with some monitore functions,
- update example script
- move disconnect handler to serviceMonitor class

* docs: add ts docs

* enhance: update naming

* tests: add jest tests

* cleanUp: enhance naming and remove mnem

* fix: export GraphQLMonitor, and enhance naming

* fix: update example command

* update:
- remove unnecessary package
- change package name to include @threefold/
- use queryClient in tfchain and update the example

* update: add license apache-2

* update: add graphql to example

* fix: remove comment on RMB

packages/monitoring/example/index.ts

@@ -0,0 +1,26 @@
+import { GraphQLMonitor, GridProxyMonitor, RMBMonitor, ServiceMonitor, TFChainMonitor } from "../src/";
+async function HealthCheck() {
+  try {
+    const services = [
+      new GridProxyMonitor("<FakeURL>"),
+      new GraphQLMonitor("https://graphql.dev.grid.tf/graphql"),
+      new TFChainMonitor("wss://tfchain.dev.grid.tf/ws"),
+      new RMBMonitor("wss://relay.dev.grid.tf", "wss://tfchain.dev.grid.tf/ws", "mnemonic", "sr25519"),
+    ];
+    const serviceMonitor = new ServiceMonitor(services);
+
+    // ping some services to check their liveness
+    // await serviceMonitor.pingService();
+
+    // keep monitoring services with Interval
+    serviceMonitor.interval = 0.25;
+    const monitor = serviceMonitor.monitorService();
+    await new Promise(resolve => setTimeout(resolve, 0.5 * 60 * 1000));
+    await monitor.exitAndDisconnect();
+    process.exit(0);
+  } catch (err) {
+    console.log(err);
+  }
+}
+
+HealthCheck();

packages/monitoring/jest.config.js

@@ -0,0 +1,34 @@
+/** @type {import("ts-jest/dist/types").InitialOptionsTsJest} */
+const { defaults } = require("jest-config");
+
+module.exports = {
+  verbose: true,
+  preset: "ts-jest",
+  testEnvironment: "node",
+  sandboxInjectedGlobals: ["Math"],
+  moduleFileExtensions: [...defaults.moduleFileExtensions, "ts", "tsx"],
+  modulePathIgnorePatterns: ["<rootDir>/dist"],
+
+  testMatch: ["**/__tests__/**/*.[jt]s?(x)", "**/?(*.)+(spec|test).[jt]s?(x)"],
+  transform: {
+    "^.+\\.(js|jsx|ts|tsx)$": ["ts-jest", { tsconfig: "tsconfig.json" }],
+  },
+  transformIgnorePatterns: ["/node_modules/(?!@polkadot|@babel/runtime/helpers/esm/)"],
+  // Don't use it with ts files -> Not supported
+  // globalTeardown: "<rootDir>/tests/global_teardown.ts"
+
+  reporters: [
+    "default",
+    [
+      "jest-junit",
+      {
+        suiteName: "jest tests",
+        outputDirectory: "tests/test-reports",
+        outputName: "report.xml",
+        includeShortConsoleOutput: true,
+        suiteNameTemplate: "{filename}",
+        reportTestSuiteErrors: true,
+      },
+    ],
+  ],
+};

packages/monitoring/package.json

@@ -0,0 +1,32 @@
+{
+  "name": "@threefold/monitoring",
+  "version": "1.0.0",
+  "description": "Threefold monitoring package",
+  "license": "Apache-2.0",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "scripts": {
+    "example": "yarn run ts-node --project tsconfig.json example/index.ts",
+    "build": "tsc",
+    "test": "jest "
+  },
+  "author": "Omar Kassem",
+  "private": false,
+  "publishConfig": {
+    "access": "public"
+  },
+  "dependencies": {
+    "axios": "^0.27.2",
+    "@threefold/types": "^2.3.0-alpha6",
+    "@threefold/rmb_direct_client": "^2.3.0-alpha6",
+    "typescript": "^5.3.3",
+    "ts-node": "^10.9.1",
+    "@threefold/tfchain_client": "^2.3.0-alpha6",
+    "chalk": "4.1.2"
+  },
+  "devDependencies": {
+    "jest": "29.7.0",
+    "ts-jest": "29.1.2",
+    "@types/jest": "29.5.11"
+  }
+}

packages/monitoring/src/helpers/events.ts

@@ -0,0 +1,58 @@
+import chalk from "chalk";
+import { EventEmitter } from "events";
+
+import { MonitorEvents } from "../types";
+
+type ServiceStatus = { [key: string]: boolean };
+
+const ALIVE = chalk.green.bold("Alive");
+const DOWN = chalk.red.bold("Down");
+class MonitorEventEmitter extends EventEmitter {
+  private summary: ServiceStatus = {};
+
+  constructor() {
+    super();
+    this.addListener(MonitorEvents.log, this.monitorLogsHandler);
+    this.addListener(MonitorEvents.serviceDown, this.serviceDownHandler);
+    this.addListener(MonitorEvents.storeStatus, this.addToServiceSummary);
+    this.addListener(MonitorEvents.summarize, this.printStatusSummary);
+  }
+  public log(message: string) {
+    this.emit("MonitorLog", message);
+  }
+  public summarize() {
+    this.emit("MonitorSummarize");
+  }
+  public storeStatus(serviceName: string, isAlive: boolean) {
+    this.emit("MonitorStoreStatus", serviceName, isAlive);
+  }
+  public serviceDown(serviceName: string, error: Error) {
+    this.emit("MonitorServiceDown", serviceName, error);
+  }
+
+  private monitorLogsHandler(msg) {
+    console.log(msg);
+  }
+  private serviceDownHandler(serviceName: string, error: Error) {
+    console.log(`${chalk.red.bold(serviceName + " is Down")}`);
+    console.log(chalk.gray("* Error: " + error.message));
+    this.summary[serviceName] = false;
+  }
+
+  private addToServiceSummary(serviceName: string, serviceIsAlive: boolean) {
+    this.summary[serviceName] = serviceIsAlive;
+  }
+
+  private printStatusSummary() {
+    const serviceNames = Object.keys(this.summary);
+    const maxServiceNameLength = Math.max(...serviceNames.map(entry => entry.length));
+    console.log(chalk.blue.bold("Aliveness check summary:"));
+    for (const service in this.summary) {
+      const padding = " ".repeat(maxServiceNameLength - service.length);
+      console.log(`\t${service}${padding}: ${this.summary[service] ? ALIVE : DOWN}`);
+    }
+  }
+}
+
+const monitorEvents = new MonitorEventEmitter();
+export { monitorEvents };

packages/monitoring/src/helpers/utils.ts

@@ -0,0 +1,33 @@
+import { RequestError } from "@threefold/types";
+import axios, { AxiosError, AxiosRequestConfig } from "axios";
+import { ServiceStatus } from "src/types";
+
+export async function sendGetRequest(url: string, options: AxiosRequestConfig = {}) {
+  try {
+    return await axios.get(url, options);
+  } catch (e) {
+    const { response } = e as AxiosError;
+    const errorMessage = (response?.data as { error: string })?.error || (e as Error).message;
+
+    throw new RequestError(`HTTP request failed ${errorMessage ? "due to " + errorMessage : ""}.`);
+  }
+}
+
+export function generateString(length: number): string {
+  let result = "";
+  const characters = "abcdefghijklmnopqrstuvwxyz0123456789";
+  const charactersLength = characters.length;
+  for (let i = 0; i < length; i++) {
+    result += characters.charAt(Math.floor(Math.random() * charactersLength));
+  }
+  return result;
+}
+
+export async function resolveServiceStatus(promise: Promise<any>): Promise<ServiceStatus> {
+  try {
+    await promise;
+    return { alive: true };
+  } catch (error) {
+    return { alive: false, error };
+  }
+}

packages/monitoring/src/index.ts

@@ -0,0 +1 @@
+export * from "./serviceMonitor/index";

packages/monitoring/src/serviceMonitor/alivenessChecker.ts

@@ -0,0 +1,79 @@
+import { monitorEvents } from "../helpers/events";
+import { IDisconnectHandler, ILivenessChecker } from "../types";
+
+/**
+ * Represents a service monitor that periodically checks the liveness of multiple services.
+ */
+export class ServiceMonitor {
+  /**
+   * Creates an instance of ServiceMonitor.
+   * @param services - An array of services to monitor.
+   * @param interval - The interval, in minutes, between monitoring checks (default is 2 minutes).
+   * @param retries - The number of retries in case a service is determined to be down (default is 2 retries).
+   * @param retryInterval - The interval, in seconds, between retries (default is 2 seconds).
+   */
+  constructor(public services: ILivenessChecker[], public interval = 2, public retries = 2, public retryInterval = 2) {}
+
+  /**
+   * Checks the liveness of each service once and logs events accordingly.
+   * @private
+   */
+  private async checkLivenessOnce(): Promise<void> {
+    for (const service of this.services) {
+      for (let retryCount = 1; retryCount <= this.retries; retryCount++) {
+        const { alive, error } = await service.isAlive();
+        if (alive) {
+          monitorEvents.storeStatus(service.serviceName(), alive);
+          break;
+        }
+        if (retryCount < this.retries) {
+          monitorEvents.log(`${service.serviceName()} seems to be down; Retrying (${retryCount}/${this.retries})...`);
+          await new Promise(resolve => setTimeout(resolve, this.retryInterval * 60));
+        } else monitorEvents.serviceDown(service.serviceName(), error);
+      }
+    }
+    monitorEvents.summarize();
+  }
+
+  /**
+   * Disconnects services that implement the `IDisconnectHandler` interface.
+   * @returns A promise that resolves when all services are disconnected.
+   */
+  public async disconnect(): Promise<void> {
+    for (const service of this.services) {
+      if ("disconnect" in service) {
+        await (service as IDisconnectHandler).disconnect();
+      }
+    }
+  }
+
+  /**
+   * Monitors the services at a regular interval and returns a function to exit and disconnect the monitoring.
+   * @returns An object with a function `exitAndDisconnect` to stop the monitoring and disconnect services.
+   */
+  public monitorService(): { exitAndDisconnect: () => Promise<void> } {
+    if (this.services.length === 0) throw new Error("No services to monitor");
+
+    monitorEvents.log(`Checking services status...`);
+    this.checkLivenessOnce();
+    const intervalId = setInterval(async () => await this.checkLivenessOnce(), this.interval * 60 * 1000);
+
+    /**
+     * Stops the monitoring and disconnects the services.
+     * @returns A promise that resolves when the monitoring is stopped and services are disconnected.
+     */
+    const exitAndDisconnect = async (): Promise<void> => {
+      clearInterval(intervalId);
+      await this.disconnect();
+    };
+    return { exitAndDisconnect };
+  }
+
+  /**
+   * Checks the liveness of each service once and disconnects the services.
+   */
+  public async pingService(): Promise<void> {
+    await this.checkLivenessOnce();
+    await this.disconnect();
+  }
+}

packages/monitoring/src/serviceMonitor/graphql.ts

@@ -0,0 +1,20 @@
+import { resolveServiceStatus, sendGetRequest } from "../helpers/utils";
+import { ILivenessChecker, ServiceStatus } from "../types";
+
+export class GraphQLMonitor implements ILivenessChecker {
+  private readonly name = "GraphQl";
+  private readonly url: string;
+  constructor(graphQlUrl: string) {
+    this.url = graphQlUrl;
+  }
+  serviceName() {
+    return this.name;
+  }
+  serviceUrl() {
+    return this.url;
+  }
+
+  async isAlive(): Promise<ServiceStatus> {
+    return resolveServiceStatus(sendGetRequest(this.url));
+  }
+}

packages/monitoring/src/serviceMonitor/gridproxy.ts

@@ -0,0 +1,19 @@
+import { resolveServiceStatus, sendGetRequest } from "../helpers/utils";
+import { ILivenessChecker, ServiceStatus } from "../types";
+
+export class GridProxyMonitor implements ILivenessChecker {
+  private readonly name = "GridProxy";
+  private url: string;
+  constructor(gridProxyUrl: string) {
+    this.url = gridProxyUrl;
+  }
+  serviceName() {
+    return this.name;
+  }
+  serviceUrl() {
+    return this.url;
+  }
+  async isAlive(): Promise<ServiceStatus> {
+    return resolveServiceStatus(sendGetRequest(this.url));
+  }
+}

packages/monitoring/src/serviceMonitor/index.ts

@@ -0,0 +1,5 @@
+export { GridProxyMonitor } from "./gridproxy";
+export { TFChainMonitor } from "./tfChain";
+export { RMBMonitor } from "./rmb";
+export { ServiceMonitor } from "./alivenessChecker";
+export { GraphQLMonitor } from "./graphql";

packages/monitoring/src/serviceMonitor/rmb.ts

@@ -0,0 +1,31 @@
+import { KeypairType } from "@polkadot/util-crypto/types";
+import { Client as RMBClient } from "@threefold/rmb_direct_client";
+
+import { generateString, resolveServiceStatus } from "../helpers/utils";
+import { IDisconnectHandler, ILivenessChecker, ServiceStatus } from "../types";
+
+export class RMBMonitor implements ILivenessChecker, IDisconnectHandler {
+  private name = "RMB";
+  private url: string;
+  private rmbClient: RMBClient;
+  constructor(relayUrl: string, chainUrl: string, mnemonic: string, keypairType: KeypairType) {
+    this.url = relayUrl;
+    this.rmbClient = new RMBClient(chainUrl, relayUrl, mnemonic, generateString(10), keypairType, 0);
+  }
+  private async setUp() {
+    await this.rmbClient.connect();
+  }
+  public serviceName() {
+    return this.name;
+  }
+  public serviceUrl() {
+    return this.url;
+  }
+  public async isAlive(): Promise<ServiceStatus> {
+    if (!this.rmbClient?.con?.OPEN) await this.setUp();
+    return resolveServiceStatus(this.rmbClient.ping(2));
+  }
+  public async disconnect() {
+    await this.rmbClient.disconnect();
+  }
+}

packages/monitoring/src/serviceMonitor/tfChain.ts

@@ -0,0 +1,38 @@
+import { QueryClient } from "@threefold/tfchain_client";
+
+import { IDisconnectHandler, ILivenessChecker, ServiceStatus } from "../types";
+
+export class TFChainMonitor implements ILivenessChecker, IDisconnectHandler {
+  private name = "TFChain";
+  private url: string;
+  private tfClient: QueryClient;
+  constructor(tfChainUrl: string) {
+    this.url = tfChainUrl;
+    this.tfClient = new QueryClient(this.url);
+  }
+  private async setUp() {
+    await this.tfClient?.connect();
+  }
+  serviceName() {
+    return this.name;
+  }
+  serviceUrl() {
+    return this.url;
+  }
+  public async isAlive(): Promise<ServiceStatus> {
+    try {
+      if (!this.tfClient.api) await this.setUp();
+      return {
+        alive: true,
+      };
+    } catch (error) {
+      return {
+        alive: false,
+        error,
+      };
+    }
+  }
+  public async disconnect() {
+    await this.tfClient.disconnect();
+  }
+}