feat: logging; refactoring; formatting; updated dependencies; updated…

… docs

.eslintrc

@@ -0,0 +1,15 @@
+{
+  "env": {
+    "browser": true,
+    "es2021": true
+  },
+  "extends": ["eslint:recommended", "plugin:@typescript-eslint/recommended"],
+  "overrides": [],
+  "parser": "@typescript-eslint/parser",
+  "parserOptions": {
+    "ecmaVersion": "latest",
+    "sourceType": "module"
+  },
+  "plugins": ["@typescript-eslint"],
+  "rules": {}
+}

README.md

@@ -11,12 +11,10 @@ Service that acts as a central component to monitor BullMQ:
 
 The following section will provide a brief overview of the libraries and practices used in the implementation of this service.
 
-
 ### BullMQ Dashboard
 
 Implemented by using [@bull-board](https://github.com/felixmosh/bull-board) and secured using [passport](https://www.passportjs.org/).
 
-
 ### Prometheus Exporter
 
 Strongly influenced by [bull_exporter](https://github.com/UpHabit/bull_exporter). Which uses the old bull library.
@@ -32,8 +30,8 @@ Furthermore, a cron job is executed every n seconds which collects the current s
 
 Thus, the following metrics are collected:
 
-| Metric                    | type      | description |
-|---------------------------|-----------|-------------|
+| Metric                    | type      | description                                             |
+| ------------------------- | --------- | ------------------------------------------------------- |
 | bullmq_processed_duration | histogram | Processing time for completed jobs                      |
 | bullmq_completed_duration | histogram | Completion time for jobs                                |
 | bullmq_completed          | gauge     | Total number of completed jobs                          |
@@ -42,4 +40,27 @@ Thus, the following metrics are collected:
 | bullmq_failed             | gauge     | Total number of failed jobs                             |
 | bullmq_waiting            | gauge     | Total number of jobs waiting to be processed            |
 
-Each metric also has the attribute `queue` which indicated which queue the metric is associated with.
+Each metric also has the attribute `queue` which indicated which queue the metric is associated with.
+
+## How to use
+
+1. Install the dependencies
+
+```
+npm install
+```
+
+2. Default environment is `local`. This can be set using the `NODE_ENV` variable.
+
+```
+export NODE_ENV=production
+```
+
+3. Make sure that the required config file is present: `./configs/config-${NODE_ENV}.json` (see [local](./configs/config-local.json)).
+4. Start the server
+
+```
+npm run dev
+```
+
+5. Access the resources `http://localhost:8080/bullmq/ui/login` or `http://localhost:8080/prometheus/metrics`

configs/config-local.json

@@ -1,6 +1,6 @@
 {
   "redis": {
-    "host": "localhost:49153/",
+    "host": "localhost:6379/",
     "username": "default",
     "password": "redispw",
     "ssl": false
@@ -19,4 +19,4 @@
       "role": "user"
     }
   ]
-}
+}

package.json

@@ -1,5 +1,5 @@
 {
-  "name": "bullmq-prometheus",
+  "name": "bullmq-exporter",
   "version": "1.0.0",
   "description": "Service that can be used to monitor BullMQ by providing Prometheus metrics and a Bullmq dashboard secured behind a login wall.",
   "main": "src/server.ts",
@@ -9,6 +9,7 @@
     "prettier": "prettier -w src/**/*.ts",
     "build": "tsc",
     "run": "node dist/server.js",
+    "dev": "ts-node src/server.ts",
     "ts-node": "ts-node src/server.ts",
     "nodemon": "nodemon",
     "prepare": "node -e \"if (process.env.NODE_ENV !== 'production'){process.exit(1)} \" || husky install"
@@ -17,36 +18,36 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
-    "@bull-board/api": "^4.2.2",
-    "@bull-board/express": "^4.2.2",
-    "bullmq": "^1.87.1",
+    "@bull-board/api": "^4.11.0",
+    "@bull-board/express": "^4.11.0",
+    "bullmq": "^3.6.2",
     "connect-ensure-login": "^0.1.1",
     "ejs": "^3.1.8",
-    "express": "^4.18.1",
+    "express": "^4.18.2",
     "express-session": "^1.17.3",
-    "ioredis": "^5.2.2",
+    "express-winston": "^4.2.0",
+    "ioredis": "^5.3.0",
     "parse-duration": "^1.0.2",
     "passport": "^0.6.0",
     "passport-local": "^1.0.0",
-    "pino": "^8.5.0",
-    "pino-http": "^8.2.0",
-    "prom-client": "^14.0.1"
+    "prom-client": "^14.1.1",
+    "winston": "^3.8.2"
   },
   "devDependencies": {
     "@types/connect-ensure-login": "^0.1.7",
     "@types/ejs": "^3.1.1",
-    "@types/express": "^4.17.13",
+    "@types/express": "^4.17.17",
     "@types/express-session": "^1.17.5",
-    "@types/passport": "^1.0.9",
-    "@types/passport-local": "^1.0.34",
-    "@typescript-eslint/eslint-plugin": "^5.33.0",
-    "@typescript-eslint/parser": "^5.33.0",
-    "eslint": "^8.21.0",
-    "husky": "^8.0.1",
-    "nodemon": "^2.0.19",
-    "prettier": "^2.7.1",
+    "@types/passport": "^1.0.11",
+    "@types/passport-local": "^1.0.35",
+    "@typescript-eslint/eslint-plugin": "^5.50.0",
+    "@typescript-eslint/parser": "^5.50.0",
+    "eslint": "^8.33.0",
+    "husky": "^8.0.3",
+    "nodemon": "^2.0.20",
+    "prettier": "^2.8.3",
     "ts-node": "^10.9.1",
-    "typescript": "^4.7.4"
+    "typescript": "^4.9.5"
   },
   "nodemonConfig": {
     "watch": [

src/app.ts

@@ -1,45 +1,61 @@
-import express from 'express';
-import Redis from 'ioredis';
-import config from './config';
-import { ConfigureRoutes as ConfigureDashboardRoutes, User } from './controllers/dashboard';
-import { ConfigureRoutes as ConfigureMetricsRoute } from './controllers/metrics';
-import logger from './logger';
-import { PrometheusMetricsCollector } from './monitor/promMetricsCollector';
-import { formatConnectionString, handleFutureShutdown } from './utils';
+import express from "express";
+import Redis from "ioredis";
+import expressWinston, {
+  LoggerOptions as ExpressWinstonOpts,
+} from "express-winston";
+import config from "./config";
+import {
+  ConfigureRoutes as ConfigureDashboardRoutes,
+  User,
+} from "./controllers/dashboard";
+import { ConfigureRoutes as ConfigureMetricsRoute } from "./controllers/metrics";
+import logger, { winstonLoggerOpts } from "./logger";
+import { PrometheusMetricsCollector } from "./monitor/promMetricsCollector";
+import { formatConnectionString, handleFutureShutdown } from "./utils";
 
-const log = logger.child({ pkg: "app" })
 export const app = express();
-app.disable('x-powered-by');
+app.disable("x-powered-by");
 
-const pino = require('pino-http')()
-app.use(pino)
+const expressWinstonOpts: ExpressWinstonOpts = {
+  ...(winstonLoggerOpts as ExpressWinstonOpts),
+  meta: false,
+  ignoreRoute: function (req, _res) {
+    return req.path.includes("/health");
+  },
+};
+app.use(expressWinston.logger(expressWinstonOpts));
 
-app.get('/health', async (req, res) => {
-  res.status(200).send('OK');
+app.get("/health", async (_req, res) => {
+  res.status(200).send("OK");
 });
 
-const username = config.redis.username
-const password = config.redis.password
-const host = config.redis.host
+const username = config.redis.username;
+const password = config.redis.password;
+const host = config.redis.host;
 
 if (username === undefined || password === undefined || host === undefined) {
   process.exit(125);
 }
 
-const enableSsl = config.redis.ssl
-const prefix = process.env.NODE_ENV?.toLowerCase() || 'local';
-const cookieSecret = config.cookieSecret
-const cookieMaxAge = config.cookieMaxAge
+const enableSsl = config.redis.ssl;
+const prefix = process.env.NODE_ENV?.toLowerCase() || "local";
+const cookieSecret = config.cookieSecret;
+const cookieMaxAge = config.cookieMaxAge;
 const defaultUsers: Array<User> = [
-  { username: 'admin', password: 'secret', role: 'admin' },
-  { username: 'user', password: 'secret', role: 'user' },
+  { username: "admin", password: "secret", role: "admin" },
+  { username: "user", password: "secret", role: "user" },
 ];
 
-const users = config.users || defaultUsers
+const users = config.users || defaultUsers;
 
-const redisConnString = formatConnectionString(host, username, password, enableSsl);
+const redisConnString = formatConnectionString(
+  host,
+  username,
+  password,
+  enableSsl
+);
 
-export const metricsCollector = new PrometheusMetricsCollector('monitor', {
+export const metricsCollector = new PrometheusMetricsCollector("monitor", {
   bullmqOpts: {
     prefix: prefix,
   },
@@ -50,14 +66,14 @@ export const metricsCollector = new PrometheusMetricsCollector('monitor', {
 handleFutureShutdown(metricsCollector);
 
 const dashboardRouter = express.Router();
-app.use('/bullmq', dashboardRouter);
+app.use("/bullmq", dashboardRouter);
 
 metricsCollector
   .discoverAllQueues()
   .then((queues) => {
-    log.info(`Discovered ${queues.length} queues`);
+    logger.info(`Discovered ${queues.length} queues`);
     ConfigureDashboardRoutes(dashboardRouter, {
-      basePath: '/bullmq',
+      basePath: "/bullmq",
       queues: metricsCollector.monitoredQueues.map((q) => q.queue),
       cookieSecret: cookieSecret,
       cookieMaxAge: cookieMaxAge,

src/config.ts

@@ -1,19 +1,19 @@
-import { readFileSync } from 'fs'
+import { readFileSync } from "fs";
 
 export interface Config {
   redis: {
-    host: string,
-    username: string,
-    password: string,
-    ssl: boolean
-  },
-  cookieSecret: string,
-  cookieMaxAge: string,
-  users?: Array<any>
+    host: string;
+    username: string;
+    password: string;
+    ssl: boolean;
+  };
+  cookieSecret: string;
+  cookieMaxAge: string;
+  users?: Array<any>;
 }
 
-const prefix = process.env.NODE_ENV?.toLowerCase() || 'local';
-const jsonRaw = readFileSync(`./configs/config-${prefix}.json`)
-const config = JSON.parse(jsonRaw.toString())
+const prefix = process.env.NODE_ENV?.toLowerCase() || "local";
+const jsonRaw = readFileSync(`./configs/config-${prefix}.json`);
+const config = JSON.parse(jsonRaw.toString());
 
-export default config as Config;
+export default config as Config;