From 2e6d78e9d630c98c887918293345e91c29ee9b19 Mon Sep 17 00:00:00 2001 From: Omar Kassem Date: Mon, 12 Feb 2024 14:14:59 +0200 Subject: [PATCH] Development monitoring (#2045) * init monitoring package * add helper method Make http request using axios * create HealthCheck class this is the generic class for checking if the service is alive create GridProxy class * create TFchain health check class with example * rewrite classe, now implements the interface * make the code scalable: add some types. add initializeServices add monitoring interval * add generateString function * enhance interfaces * apply interface changes * add alivenessChecker * update isAlive interface now it return ServiceStatus * update isAlive methods * add event emitter * add colorize function and type * update imports * update alivenessCheckers * update alivenessCheckers and add js docs * update rmb constractor to recive the mnemonic * add summary event * update example * remove rmb error logs * Revert "remove rmb error logs" This reverts commit 8d4e930d967b28a1ab01c83006ef3e3af920adb0. * remove rmb error logs * add disconnect Interface, use it on rmb * use tfChain client instade of polkadot * add disconnect method and update example script * change the example path * use Chalk: text colorization * fix: remove body and make header optional in axios get request * add resolveSErviceStatus function and apply it * add exit and disconnect function * update example * enahnce: rename ILivenessChecker class and rewrite IServiceInfo clase * enahnce: add ts docs * enahnce: add info getters to classes * enahnce: add work around to check if the service implements the IDisconnectHandler * enahnce: handle axios error message * rewrite: monitor events now exposing an instance of class that contains all needed events and thier handlers * cleanUp: remove debugging logs * rewrite: create serviceMonitor class with some monitore functions, - update example script - move disconnect handler to serviceMonitor class * docs: add ts docs * enhance: update naming * tests: add jest tests * cleanUp: enhance naming and remove mnem * fix: export GraphQLMonitor, and enhance naming * fix: update example command * update: - remove unnecessary package - change package name to include @threefold/ - use queryClient in tfchain and update the example * update: add license apache-2 * update: add graphql to example * fix: remove comment on RMB --- packages/monitoring/example/index.ts | 26 ++++++ packages/monitoring/jest.config.js | 34 ++++++++ packages/monitoring/package.json | 32 ++++++++ packages/monitoring/src/helpers/events.ts | 58 ++++++++++++++ packages/monitoring/src/helpers/utils.ts | 33 ++++++++ packages/monitoring/src/index.ts | 1 + .../src/serviceMonitor/alivenessChecker.ts | 79 +++++++++++++++++++ .../monitoring/src/serviceMonitor/graphql.ts | 20 +++++ .../src/serviceMonitor/gridproxy.ts | 19 +++++ .../monitoring/src/serviceMonitor/index.ts | 5 ++ packages/monitoring/src/serviceMonitor/rmb.ts | 31 ++++++++ .../monitoring/src/serviceMonitor/tfChain.ts | 38 +++++++++ packages/monitoring/src/types/index.ts | 50 ++++++++++++ .../monitoring/tests/generateStrings.test.ts | 28 +++++++ packages/monitoring/tsconfig.json | 19 +++++ yarn.lock | 5 ++ 16 files changed, 478 insertions(+) create mode 100644 packages/monitoring/example/index.ts create mode 100644 packages/monitoring/jest.config.js create mode 100644 packages/monitoring/package.json create mode 100644 packages/monitoring/src/helpers/events.ts create mode 100644 packages/monitoring/src/helpers/utils.ts create mode 100644 packages/monitoring/src/index.ts create mode 100644 packages/monitoring/src/serviceMonitor/alivenessChecker.ts create mode 100644 packages/monitoring/src/serviceMonitor/graphql.ts create mode 100644 packages/monitoring/src/serviceMonitor/gridproxy.ts create mode 100644 packages/monitoring/src/serviceMonitor/index.ts create mode 100644 packages/monitoring/src/serviceMonitor/rmb.ts create mode 100644 packages/monitoring/src/serviceMonitor/tfChain.ts create mode 100644 packages/monitoring/src/types/index.ts create mode 100644 packages/monitoring/tests/generateStrings.test.ts create mode 100644 packages/monitoring/tsconfig.json diff --git a/packages/monitoring/example/index.ts b/packages/monitoring/example/index.ts new file mode 100644 index 0000000000..418df9897c --- /dev/null +++ b/packages/monitoring/example/index.ts @@ -0,0 +1,26 @@ +import { GraphQLMonitor, GridProxyMonitor, RMBMonitor, ServiceMonitor, TFChainMonitor } from "../src/"; +async function HealthCheck() { + try { + const services = [ + new GridProxyMonitor(""), + new GraphQLMonitor("https://graphql.dev.grid.tf/graphql"), + new TFChainMonitor("wss://tfchain.dev.grid.tf/ws"), + new RMBMonitor("wss://relay.dev.grid.tf", "wss://tfchain.dev.grid.tf/ws", "mnemonic", "sr25519"), + ]; + const serviceMonitor = new ServiceMonitor(services); + + // ping some services to check their liveness + // await serviceMonitor.pingService(); + + // keep monitoring services with Interval + serviceMonitor.interval = 0.25; + const monitor = serviceMonitor.monitorService(); + await new Promise(resolve => setTimeout(resolve, 0.5 * 60 * 1000)); + await monitor.exitAndDisconnect(); + process.exit(0); + } catch (err) { + console.log(err); + } +} + +HealthCheck(); diff --git a/packages/monitoring/jest.config.js b/packages/monitoring/jest.config.js new file mode 100644 index 0000000000..3ba7b556d5 --- /dev/null +++ b/packages/monitoring/jest.config.js @@ -0,0 +1,34 @@ +/** @type {import("ts-jest/dist/types").InitialOptionsTsJest} */ +const { defaults } = require("jest-config"); + +module.exports = { + verbose: true, + preset: "ts-jest", + testEnvironment: "node", + sandboxInjectedGlobals: ["Math"], + moduleFileExtensions: [...defaults.moduleFileExtensions, "ts", "tsx"], + modulePathIgnorePatterns: ["/dist"], + + testMatch: ["**/__tests__/**/*.[jt]s?(x)", "**/?(*.)+(spec|test).[jt]s?(x)"], + transform: { + "^.+\\.(js|jsx|ts|tsx)$": ["ts-jest", { tsconfig: "tsconfig.json" }], + }, + transformIgnorePatterns: ["/node_modules/(?!@polkadot|@babel/runtime/helpers/esm/)"], + // Don't use it with ts files -> Not supported + // globalTeardown: "/tests/global_teardown.ts" + + reporters: [ + "default", + [ + "jest-junit", + { + suiteName: "jest tests", + outputDirectory: "tests/test-reports", + outputName: "report.xml", + includeShortConsoleOutput: true, + suiteNameTemplate: "{filename}", + reportTestSuiteErrors: true, + }, + ], + ], +}; diff --git a/packages/monitoring/package.json b/packages/monitoring/package.json new file mode 100644 index 0000000000..dd22a25eb7 --- /dev/null +++ b/packages/monitoring/package.json @@ -0,0 +1,32 @@ +{ + "name": "@threefold/monitoring", + "version": "1.0.0", + "description": "Threefold monitoring package", + "license": "Apache-2.0", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "scripts": { + "example": "yarn run ts-node --project tsconfig.json example/index.ts", + "build": "tsc", + "test": "jest " + }, + "author": "Omar Kassem", + "private": false, + "publishConfig": { + "access": "public" + }, + "dependencies": { + "axios": "^0.27.2", + "@threefold/types": "^2.3.0-alpha6", + "@threefold/rmb_direct_client": "^2.3.0-alpha6", + "typescript": "^5.3.3", + "ts-node": "^10.9.1", + "@threefold/tfchain_client": "^2.3.0-alpha6", + "chalk": "4.1.2" + }, + "devDependencies": { + "jest": "29.7.0", + "ts-jest": "29.1.2", + "@types/jest": "29.5.11" + } +} diff --git a/packages/monitoring/src/helpers/events.ts b/packages/monitoring/src/helpers/events.ts new file mode 100644 index 0000000000..14ffc56120 --- /dev/null +++ b/packages/monitoring/src/helpers/events.ts @@ -0,0 +1,58 @@ +import chalk from "chalk"; +import { EventEmitter } from "events"; + +import { MonitorEvents } from "../types"; + +type ServiceStatus = { [key: string]: boolean }; + +const ALIVE = chalk.green.bold("Alive"); +const DOWN = chalk.red.bold("Down"); +class MonitorEventEmitter extends EventEmitter { + private summary: ServiceStatus = {}; + + constructor() { + super(); + this.addListener(MonitorEvents.log, this.monitorLogsHandler); + this.addListener(MonitorEvents.serviceDown, this.serviceDownHandler); + this.addListener(MonitorEvents.storeStatus, this.addToServiceSummary); + this.addListener(MonitorEvents.summarize, this.printStatusSummary); + } + public log(message: string) { + this.emit("MonitorLog", message); + } + public summarize() { + this.emit("MonitorSummarize"); + } + public storeStatus(serviceName: string, isAlive: boolean) { + this.emit("MonitorStoreStatus", serviceName, isAlive); + } + public serviceDown(serviceName: string, error: Error) { + this.emit("MonitorServiceDown", serviceName, error); + } + + private monitorLogsHandler(msg) { + console.log(msg); + } + private serviceDownHandler(serviceName: string, error: Error) { + console.log(`${chalk.red.bold(serviceName + " is Down")}`); + console.log(chalk.gray("* Error: " + error.message)); + this.summary[serviceName] = false; + } + + private addToServiceSummary(serviceName: string, serviceIsAlive: boolean) { + this.summary[serviceName] = serviceIsAlive; + } + + private printStatusSummary() { + const serviceNames = Object.keys(this.summary); + const maxServiceNameLength = Math.max(...serviceNames.map(entry => entry.length)); + console.log(chalk.blue.bold("Aliveness check summary:")); + for (const service in this.summary) { + const padding = " ".repeat(maxServiceNameLength - service.length); + console.log(`\t${service}${padding}: ${this.summary[service] ? ALIVE : DOWN}`); + } + } +} + +const monitorEvents = new MonitorEventEmitter(); +export { monitorEvents }; diff --git a/packages/monitoring/src/helpers/utils.ts b/packages/monitoring/src/helpers/utils.ts new file mode 100644 index 0000000000..9855f4c696 --- /dev/null +++ b/packages/monitoring/src/helpers/utils.ts @@ -0,0 +1,33 @@ +import { RequestError } from "@threefold/types"; +import axios, { AxiosError, AxiosRequestConfig } from "axios"; +import { ServiceStatus } from "src/types"; + +export async function sendGetRequest(url: string, options: AxiosRequestConfig = {}) { + try { + return await axios.get(url, options); + } catch (e) { + const { response } = e as AxiosError; + const errorMessage = (response?.data as { error: string })?.error || (e as Error).message; + + throw new RequestError(`HTTP request failed ${errorMessage ? "due to " + errorMessage : ""}.`); + } +} + +export function generateString(length: number): string { + let result = ""; + const characters = "abcdefghijklmnopqrstuvwxyz0123456789"; + const charactersLength = characters.length; + for (let i = 0; i < length; i++) { + result += characters.charAt(Math.floor(Math.random() * charactersLength)); + } + return result; +} + +export async function resolveServiceStatus(promise: Promise): Promise { + try { + await promise; + return { alive: true }; + } catch (error) { + return { alive: false, error }; + } +} diff --git a/packages/monitoring/src/index.ts b/packages/monitoring/src/index.ts new file mode 100644 index 0000000000..66484a7686 --- /dev/null +++ b/packages/monitoring/src/index.ts @@ -0,0 +1 @@ +export * from "./serviceMonitor/index"; diff --git a/packages/monitoring/src/serviceMonitor/alivenessChecker.ts b/packages/monitoring/src/serviceMonitor/alivenessChecker.ts new file mode 100644 index 0000000000..3bfa5769d3 --- /dev/null +++ b/packages/monitoring/src/serviceMonitor/alivenessChecker.ts @@ -0,0 +1,79 @@ +import { monitorEvents } from "../helpers/events"; +import { IDisconnectHandler, ILivenessChecker } from "../types"; + +/** + * Represents a service monitor that periodically checks the liveness of multiple services. + */ +export class ServiceMonitor { + /** + * Creates an instance of ServiceMonitor. + * @param services - An array of services to monitor. + * @param interval - The interval, in minutes, between monitoring checks (default is 2 minutes). + * @param retries - The number of retries in case a service is determined to be down (default is 2 retries). + * @param retryInterval - The interval, in seconds, between retries (default is 2 seconds). + */ + constructor(public services: ILivenessChecker[], public interval = 2, public retries = 2, public retryInterval = 2) {} + + /** + * Checks the liveness of each service once and logs events accordingly. + * @private + */ + private async checkLivenessOnce(): Promise { + for (const service of this.services) { + for (let retryCount = 1; retryCount <= this.retries; retryCount++) { + const { alive, error } = await service.isAlive(); + if (alive) { + monitorEvents.storeStatus(service.serviceName(), alive); + break; + } + if (retryCount < this.retries) { + monitorEvents.log(`${service.serviceName()} seems to be down; Retrying (${retryCount}/${this.retries})...`); + await new Promise(resolve => setTimeout(resolve, this.retryInterval * 60)); + } else monitorEvents.serviceDown(service.serviceName(), error); + } + } + monitorEvents.summarize(); + } + + /** + * Disconnects services that implement the `IDisconnectHandler` interface. + * @returns A promise that resolves when all services are disconnected. + */ + public async disconnect(): Promise { + for (const service of this.services) { + if ("disconnect" in service) { + await (service as IDisconnectHandler).disconnect(); + } + } + } + + /** + * Monitors the services at a regular interval and returns a function to exit and disconnect the monitoring. + * @returns An object with a function `exitAndDisconnect` to stop the monitoring and disconnect services. + */ + public monitorService(): { exitAndDisconnect: () => Promise } { + if (this.services.length === 0) throw new Error("No services to monitor"); + + monitorEvents.log(`Checking services status...`); + this.checkLivenessOnce(); + const intervalId = setInterval(async () => await this.checkLivenessOnce(), this.interval * 60 * 1000); + + /** + * Stops the monitoring and disconnects the services. + * @returns A promise that resolves when the monitoring is stopped and services are disconnected. + */ + const exitAndDisconnect = async (): Promise => { + clearInterval(intervalId); + await this.disconnect(); + }; + return { exitAndDisconnect }; + } + + /** + * Checks the liveness of each service once and disconnects the services. + */ + public async pingService(): Promise { + await this.checkLivenessOnce(); + await this.disconnect(); + } +} diff --git a/packages/monitoring/src/serviceMonitor/graphql.ts b/packages/monitoring/src/serviceMonitor/graphql.ts new file mode 100644 index 0000000000..bbb1207cbf --- /dev/null +++ b/packages/monitoring/src/serviceMonitor/graphql.ts @@ -0,0 +1,20 @@ +import { resolveServiceStatus, sendGetRequest } from "../helpers/utils"; +import { ILivenessChecker, ServiceStatus } from "../types"; + +export class GraphQLMonitor implements ILivenessChecker { + private readonly name = "GraphQl"; + private readonly url: string; + constructor(graphQlUrl: string) { + this.url = graphQlUrl; + } + serviceName() { + return this.name; + } + serviceUrl() { + return this.url; + } + + async isAlive(): Promise { + return resolveServiceStatus(sendGetRequest(this.url)); + } +} diff --git a/packages/monitoring/src/serviceMonitor/gridproxy.ts b/packages/monitoring/src/serviceMonitor/gridproxy.ts new file mode 100644 index 0000000000..cab403be69 --- /dev/null +++ b/packages/monitoring/src/serviceMonitor/gridproxy.ts @@ -0,0 +1,19 @@ +import { resolveServiceStatus, sendGetRequest } from "../helpers/utils"; +import { ILivenessChecker, ServiceStatus } from "../types"; + +export class GridProxyMonitor implements ILivenessChecker { + private readonly name = "GridProxy"; + private url: string; + constructor(gridProxyUrl: string) { + this.url = gridProxyUrl; + } + serviceName() { + return this.name; + } + serviceUrl() { + return this.url; + } + async isAlive(): Promise { + return resolveServiceStatus(sendGetRequest(this.url)); + } +} diff --git a/packages/monitoring/src/serviceMonitor/index.ts b/packages/monitoring/src/serviceMonitor/index.ts new file mode 100644 index 0000000000..34e22fa696 --- /dev/null +++ b/packages/monitoring/src/serviceMonitor/index.ts @@ -0,0 +1,5 @@ +export { GridProxyMonitor } from "./gridproxy"; +export { TFChainMonitor } from "./tfChain"; +export { RMBMonitor } from "./rmb"; +export { ServiceMonitor } from "./alivenessChecker"; +export { GraphQLMonitor } from "./graphql"; diff --git a/packages/monitoring/src/serviceMonitor/rmb.ts b/packages/monitoring/src/serviceMonitor/rmb.ts new file mode 100644 index 0000000000..9a00d96f67 --- /dev/null +++ b/packages/monitoring/src/serviceMonitor/rmb.ts @@ -0,0 +1,31 @@ +import { KeypairType } from "@polkadot/util-crypto/types"; +import { Client as RMBClient } from "@threefold/rmb_direct_client"; + +import { generateString, resolveServiceStatus } from "../helpers/utils"; +import { IDisconnectHandler, ILivenessChecker, ServiceStatus } from "../types"; + +export class RMBMonitor implements ILivenessChecker, IDisconnectHandler { + private name = "RMB"; + private url: string; + private rmbClient: RMBClient; + constructor(relayUrl: string, chainUrl: string, mnemonic: string, keypairType: KeypairType) { + this.url = relayUrl; + this.rmbClient = new RMBClient(chainUrl, relayUrl, mnemonic, generateString(10), keypairType, 0); + } + private async setUp() { + await this.rmbClient.connect(); + } + public serviceName() { + return this.name; + } + public serviceUrl() { + return this.url; + } + public async isAlive(): Promise { + if (!this.rmbClient?.con?.OPEN) await this.setUp(); + return resolveServiceStatus(this.rmbClient.ping(2)); + } + public async disconnect() { + await this.rmbClient.disconnect(); + } +} diff --git a/packages/monitoring/src/serviceMonitor/tfChain.ts b/packages/monitoring/src/serviceMonitor/tfChain.ts new file mode 100644 index 0000000000..2b5e10ea36 --- /dev/null +++ b/packages/monitoring/src/serviceMonitor/tfChain.ts @@ -0,0 +1,38 @@ +import { QueryClient } from "@threefold/tfchain_client"; + +import { IDisconnectHandler, ILivenessChecker, ServiceStatus } from "../types"; + +export class TFChainMonitor implements ILivenessChecker, IDisconnectHandler { + private name = "TFChain"; + private url: string; + private tfClient: QueryClient; + constructor(tfChainUrl: string) { + this.url = tfChainUrl; + this.tfClient = new QueryClient(this.url); + } + private async setUp() { + await this.tfClient?.connect(); + } + serviceName() { + return this.name; + } + serviceUrl() { + return this.url; + } + public async isAlive(): Promise { + try { + if (!this.tfClient.api) await this.setUp(); + return { + alive: true, + }; + } catch (error) { + return { + alive: false, + error, + }; + } + } + public async disconnect() { + await this.tfClient.disconnect(); + } +} diff --git a/packages/monitoring/src/types/index.ts b/packages/monitoring/src/types/index.ts new file mode 100644 index 0000000000..760c613273 --- /dev/null +++ b/packages/monitoring/src/types/index.ts @@ -0,0 +1,50 @@ +/** + * Represents a basic service interface. + */ +interface IServiceBase { + /** + * Returns the name of the service. + * @returns {string} The service name. + */ + serviceName: () => string; + + /** + * Returns the URL of the service. + * @returns {string} The service URL. + */ + serviceUrl: () => string; +} + +/** + * Represents a handler for disconnecting a service. + */ +export interface IDisconnectHandler { + /** + * Performs the disconnection from the service. + * @returns {Promise} A promise that resolves when the disconnection is successful. + */ + disconnect: () => Promise; +} + +/** + * Represents a service with liveness checking capability. + */ +export interface ILivenessChecker extends IServiceBase { + /** + * Checks if the service is alive. + * @returns {Promise} A promise that resolves with the current status of the service. + */ + isAlive: () => Promise; +} + +export type ServiceStatus = { + alive: boolean; + error?: Error; +}; + +export enum MonitorEvents { + "log" = "MonitorLog", + "summarize" = "MonitorSummarize", + "storeStatus" = "MonitorStoreStatus", + "serviceDown" = "MonitorServiceDown", +} diff --git a/packages/monitoring/tests/generateStrings.test.ts b/packages/monitoring/tests/generateStrings.test.ts new file mode 100644 index 0000000000..12edd7d42c --- /dev/null +++ b/packages/monitoring/tests/generateStrings.test.ts @@ -0,0 +1,28 @@ +import { generateString } from "../src/helpers/utils"; + +describe("generateString", () => { + test("should generate a string of the specified length", () => { + const length = 10; + const result = generateString(length); + expect(result).toHaveLength(length); + }); + + test("should generate an empty string if the length is 0", () => { + const result = generateString(0); + expect(result).toBe(""); + }); + + test("should generate unique strings for multiple calls", () => { + const length = 5; + const result1 = generateString(length); + const result2 = generateString(length); + expect(result1).not.toBe(result2); + }); + + test("should generate strings containing only lowercase letters and numbers", () => { + const length = 8; + const result = generateString(length); + const isValid = /^[a-z0-9]+$/.test(result); + expect(isValid).toBe(true); + }); +}); diff --git a/packages/monitoring/tsconfig.json b/packages/monitoring/tsconfig.json new file mode 100644 index 0000000000..9bd3f777a4 --- /dev/null +++ b/packages/monitoring/tsconfig.json @@ -0,0 +1,19 @@ +{ + "compilerOptions": { + "module": "commonjs", + "target": "esnext", + "lib": ["ESNext", "DOM"], + "types": ["node", "jest"], + "declaration": true, + "declarationMap": true, + "outDir": "./dist/node", + "esModuleInterop": true, + "resolveJsonModule": true, + "emitDecoratorMetadata": true, + "experimentalDecorators": true, + "allowJs": true, + "baseUrl": ".", + "skipLibCheck": true /* Skip type checking all .d.ts files. */ + }, + "include": ["src/**/*"] +} diff --git a/yarn.lock b/yarn.lock index 5acf718caf..7a609bd6d4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -19392,6 +19392,11 @@ typescript@*, typescript@^5.0.2: resolved "https://registry.npmjs.org/typescript/-/typescript-4.9.5.tgz" integrity sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g== +typescript@^5.3.3: + version "5.3.3" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.3.3.tgz#b3ce6ba258e72e6305ba66f5c9b452aaee3ffe37" + integrity sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw== + typescript@~4.5.5: version "4.5.5" resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.5.5.tgz#d8c953832d28924a9e3d37c73d729c846c5896f3"