Skip to content

Commit

Permalink
dynamic prometheus labels
Browse files Browse the repository at this point in the history 
fixes louislam#680
replaces louislam#898
  • Loading branch information
@spali
spali committed Jan 24, 2023
1 parent b05c620 commit 181185f
Show file tree
Hide file tree
Showing 3 changed files with 131 additions and 59 deletions.
19 changes: 6 additions & 13 deletions server/model/monitor.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -196,11 +196,11 @@ class Monitor extends BeanModel {
* Start monitor
* @param {Server} io Socket server instance
*/
start(io) {
async start(io) {
let previousBeat = null;
let retries = 0;

let prometheus = new Prometheus(this);
this.prometheus = await Prometheus.createAndInitMetrics(this);

const beat = async () => {

Expand Down Expand Up @@ -729,7 +729,7 @@ class Monitor extends BeanModel {
await R.store(bean);

log.debug("monitor", `[${this.name}] prometheus.update`);
prometheus.update(bean, tlsInfo);
await this.prometheus.update(bean, tlsInfo);

previousBeat = bean;

Expand Down Expand Up @@ -810,19 +810,12 @@ class Monitor extends BeanModel {
}

/** Stop monitor */
stop() {
async stop() {
clearTimeout(this.heartbeatInterval);
this.isStop = true;

this.prometheus().remove();
}

/**
* Get a new prometheus instance
* @returns {Prometheus}
*/
prometheus() {
return new Prometheus(this);
this.prometheus?.remove();
this.prometheus = null;
}

/**
Expand Down
144 changes: 107 additions & 37 deletions server/prometheus.js
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
const { R } = require("redbean-node");
const PrometheusClient = require("prom-client");
const { log } = require("../src/util");

Expand All @@ -9,52 +10,121 @@ const commonLabels = [
"monitor_port",
];

const monitorCertDaysRemaining = new PrometheusClient.Gauge({
name: "monitor_cert_days_remaining",
help: "The number of days remaining until the certificate expires",
labelNames: commonLabels
});

const monitorCertIsValid = new PrometheusClient.Gauge({
name: "monitor_cert_is_valid",
help: "Is the certificate still valid? (1 = Yes, 0= No)",
labelNames: commonLabels
});
const monitorResponseTime = new PrometheusClient.Gauge({
name: "monitor_response_time",
help: "Monitor Response Time (ms)",
labelNames: commonLabels
});

const monitorStatus = new PrometheusClient.Gauge({
name: "monitor_status",
help: "Monitor Status (1 = UP, 0= DOWN)",
labelNames: commonLabels
});

class Prometheus {
monitorLabelValues = {};

/**
* @param {Object} monitor Monitor object to monitor
* Metric: monitor_cert_days_remaining
* @type {PrometheusClient.Gauge<string> | null}
*/
constructor(monitor) {
static monitorCertDaysRemaining = null;

/**
* Metric: monitor_cert_is_valid
* @type {PrometheusClient.Gauge<string> | null}
*/
static monitorCertIsValid = null;

/**
* Metric: monitor_response_time
* @type {PrometheusClient.Gauge<string> | null}
*/
static monitorResponseTime = null;

/**
* Metric: monitor_status
* @type {PrometheusClient.Gauge<string> | null}
*/
static monitorStatus = null;

/**
* All registered metric labels.
* @type {string[] | null}
*/
static monitorLabelNames = null;

/**
* Monitor labels/values combination.
* @type {{}}
*/
monitorLabelValues;

/**
* Initialize metrics and get all label names the first time called.
*/
static async initMetrics() {
if (!this.monitorLabelNames) {
let labelNames = await R.getCol("SELECT name FROM tag");
this.monitorLabelNames = [ ...commonLabels, ...labelNames ];
}
if (!this.monitorCertDaysRemaining) {
this.monitorCertDaysRemaining = new PrometheusClient.Gauge({
name: "monitor_cert_days_remaining",
help: "The number of days remaining until the certificate expires",
labelNames: this.monitorLabelNames
});
}
if (!this.monitorCertIsValid) {
this.monitorCertIsValid = new PrometheusClient.Gauge({
name: "monitor_cert_is_valid",
help: "Is the certificate still valid? (1 = Yes, 0= No)",
labelNames: this.monitorLabelNames
});
}
if (!this.monitorResponseTime) {
this.monitorResponseTime = new PrometheusClient.Gauge({
name: "monitor_response_time",
help: "Monitor Response Time (ms)",
labelNames: this.monitorLabelNames
});
}
if (!this.monitorStatus) {
this.monitorStatus = new PrometheusClient.Gauge({
name: "monitor_status",
help: "Monitor Status (1 = UP, 0= DOWN)",
labelNames: this.monitorLabelNames
});
}
}

/**
* Wrapper to create a `Prometheus` instance and ensure metrics are initialized.
* @param {Monitor} monitor Monitor object to monitor
*/
static async createAndInitMetrics(monitor) {
await Prometheus.initMetrics();
let tags = await monitor.getTags();
return new Prometheus(monitor, tags);
}

/**
* Creates a prometheus metric instance.
*
* Note: Make sure to call `Prometheus.initMetrics()` once prior creating Prometheus instances.
* @param {Monitor} monitor Monitor object to monitor
* @param {Promise<LooseObject<any>[]>} tags Tags of the monitor
*/
constructor(monitor, tags) {
this.monitorLabelValues = {
monitor_name: monitor.name,
monitor_type: monitor.type,
monitor_url: monitor.url,
monitor_hostname: monitor.hostname,
monitor_port: monitor.port
};
Object.values(tags)
// only label names that were known at first metric creation.
.filter(tag => Prometheus.monitorLabelNames.includes(tag.name))
.forEach(tag => {
this.monitorLabelValues[tag.name] = tag.value;
});
}

/**
* Update the metrics page
* @param {Object} heartbeat Heartbeat details
* @param {Object} tlsInfo TLS details
*/
update(heartbeat, tlsInfo) {

async update(heartbeat, tlsInfo) {
if (typeof tlsInfo !== "undefined") {
try {
let isValid;
Expand All @@ -63,15 +133,15 @@ class Prometheus {
} else {
isValid = 0;
}
monitorCertIsValid.set(this.monitorLabelValues, isValid);
Prometheus.monitorCertIsValid.set(this.monitorLabelValues, isValid);
} catch (e) {
log.error("prometheus", "Caught error");
log.error("prometheus", e);
}

try {
if (tlsInfo.certInfo != null) {
monitorCertDaysRemaining.set(this.monitorLabelValues, tlsInfo.certInfo.daysRemaining);
Prometheus.monitorCertDaysRemaining.set(this.monitorLabelValues, tlsInfo.certInfo.daysRemaining);
}
} catch (e) {
log.error("prometheus", "Caught error");
Expand All @@ -80,18 +150,18 @@ class Prometheus {
}

try {
monitorStatus.set(this.monitorLabelValues, heartbeat.status);
Prometheus.monitorStatus.set(this.monitorLabelValues, heartbeat.status);
} catch (e) {
log.error("prometheus", "Caught error");
log.error("prometheus", e);
}

try {
if (typeof heartbeat.ping === "number") {
monitorResponseTime.set(this.monitorLabelValues, heartbeat.ping);
Prometheus.monitorResponseTime.set(this.monitorLabelValues, heartbeat.ping);
} else {
// Is it good?
monitorResponseTime.set(this.monitorLabelValues, -1);
Prometheus.monitorResponseTime.set(this.monitorLabelValues, -1);
}
} catch (e) {
log.error("prometheus", "Caught error");
Expand All @@ -102,10 +172,10 @@ class Prometheus {
/** Remove monitor from prometheus */
remove() {
try {
monitorCertDaysRemaining.remove(this.monitorLabelValues);
monitorCertIsValid.remove(this.monitorLabelValues);
monitorResponseTime.remove(this.monitorLabelValues);
monitorStatus.remove(this.monitorLabelValues);
Prometheus.monitorCertDaysRemaining?.remove(this.monitorLabelValues);
Prometheus.monitorCertIsValid?.remove(this.monitorLabelValues);
Prometheus.monitorResponseTime?.remove(this.monitorLabelValues);
Prometheus.monitorStatus?.remove(this.monitorLabelValues);
} catch (e) {
console.error(e);
}
Expand Down
27 changes: 18 additions & 9 deletions server/server.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -674,9 +674,6 @@ let needSetup = false;
throw new Error("Permission denied.");
}

// Reset Prometheus labels
server.monitorList[monitor.id]?.prometheus()?.remove();

bean.name = monitor.name;
bean.type = monitor.type;
bean.url = monitor.url;
Expand Down Expand Up @@ -870,7 +867,7 @@ let needSetup = false;
log.info("manage", `Delete Monitor: ${monitorID} User ID: ${socket.userID}`);

if (monitorID in server.monitorList) {
server.monitorList[monitorID].stop();
await server.monitorList[monitorID].stop();
delete server.monitorList[monitorID];
}

Expand Down Expand Up @@ -919,6 +916,8 @@ let needSetup = false;
try {
checkLogin(socket);

log.debug("manage", `Add Tag: ${tag} User ID: ${socket.userID}`);

let bean = R.dispense("tag");
bean.name = tag.name;
bean.color = tag.color;
Expand All @@ -941,6 +940,8 @@ let needSetup = false;
try {
checkLogin(socket);

log.debug("manage", `Edit Tag: ${tag} User ID: ${socket.userID}`);

let bean = await R.findOne("tag", " id = ? ", [ tag.id ]);
if (bean == null) {
callback({
Expand Down Expand Up @@ -971,6 +972,8 @@ let needSetup = false;
try {
checkLogin(socket);

log.debug("manage", `Delete Tag: ${tagID} User ID: ${socket.userID}`);

await R.exec("DELETE FROM tag WHERE id = ? ", [ tagID ]);

callback({
Expand All @@ -990,6 +993,8 @@ let needSetup = false;
try {
checkLogin(socket);

log.debug("manage", `Add Monitor(${monitorID}) Tag(${tagID}): ${value} User ID: ${socket.userID}`);

await R.exec("INSERT INTO monitor_tag (tag_id, monitor_id, value) VALUES (?, ?, ?)", [
tagID,
monitorID,
Expand All @@ -1013,6 +1018,8 @@ let needSetup = false;
try {
checkLogin(socket);

log.debug("manage", `Edit Monitor(${monitorID}) Tag(${tagID}): ${value} User ID: ${socket.userID}`);

await R.exec("UPDATE monitor_tag SET value = ? WHERE tag_id = ? AND monitor_id = ?", [
value,
tagID,
Expand All @@ -1036,6 +1043,8 @@ let needSetup = false;
try {
checkLogin(socket);

log.debug("manage", `Delete Monitor(${monitorID}) Tag(${tagID}): ${value} User ID: ${socket.userID}`);

await R.exec("DELETE FROM monitor_tag WHERE tag_id = ? AND monitor_id = ? AND value = ?", [
tagID,
monitorID,
Expand Down Expand Up @@ -1691,11 +1700,11 @@ async function startMonitor(userID, monitorID) {
]);

if (monitor.id in server.monitorList) {
server.monitorList[monitor.id].stop();
await server.monitorList[monitor.id].stop();
}

server.monitorList[monitor.id] = monitor;
monitor.start(io);
await monitor.start(io);
}

/**
Expand Down Expand Up @@ -1725,7 +1734,7 @@ async function pauseMonitor(userID, monitorID) {
]);

if (monitorID in server.monitorList) {
server.monitorList[monitorID].stop();
await server.monitorList[monitorID].stop();
}
}

Expand All @@ -1738,7 +1747,7 @@ async function startMonitors() {
}

for (let monitor of list) {
monitor.start(io);
await monitor.start(io);
// Give some delays, so all monitors won't make request at the same moment when just start the server.
await sleep(getRandomInt(300, 1000));
}
Expand All @@ -1759,7 +1768,7 @@ async function shutdownFunction(signal) {
log.info("server", "Stopping all monitors");
for (let id in server.monitorList) {
let monitor = server.monitorList[id];
monitor.stop();
await monitor.stop();
}
await sleep(2000);
await Database.close();
Expand Down

0 comments on commit 181185f

Please sign in to comment.