add 4 uptime metrics in prometheus

louislam · Jan 8, 2025 · bbc6e33 · bbc6e33
1 parent be2faf6
commit bbc6e33
Show file tree

Hide file tree

Showing 2 changed files with 110 additions and 3 deletions.
diff --git a/server/model/monitor.js b/server/model/monitor.js
@@ -980,7 +980,15 @@ class Monitor extends BeanModel {
             await R.store(bean);
 
             log.debug("monitor", `[${this.name}] prometheus.update`);
-            this.prometheus?.update(bean, tlsInfo);
+            let uptimeMetrics = {};
+            let data24h = await uptimeCalculator.get24Hour();
+            let data30d = await uptimeCalculator.get30Day();
+            let data1y = await uptimeCalculator.get1Year();
+            uptimeMetrics.avgPing = data24h.avgPing ? Number(data24h.avgPing.toFixed(2)) : null;
+            uptimeMetrics.data24h = data24h.uptime;
+            uptimeMetrics.data30d = data30d.uptime;
+            uptimeMetrics.data1y = data1y.uptime;
+            this.prometheus?.update(bean, tlsInfo, uptimeMetrics);
 
             previousBeat = bean;
 
@@ -1730,7 +1738,7 @@ class Monitor extends BeanModel {
      */
     async handleTlsInfo(tlsInfo) {
         await this.updateTlsInfo(tlsInfo);
-        this.prometheus?.update(null, tlsInfo);
+        this.prometheus?.update(null, tlsInfo, null);
 
         if (!this.getIgnoreTls() && this.isEnabledExpiryNotification()) {
             log.debug("monitor", `[${this.name}] call checkCertExpiryNotifications`);

diff --git a/server/prometheus.js b/server/prometheus.js
@@ -20,6 +20,31 @@ const monitorCertIsValid = new PrometheusClient.Gauge({
     help: "Is the certificate still valid? (1 = Yes, 0= No)",
     labelNames: commonLabels
 });
+
+const monitorUptime1y = new PrometheusClient.Gauge({
+    name: "monitor_uptime_1y",
+    help: "Monitor Uptime 1y (%)",
+    labelNames: commonLabels,
+});
+
+const monitorUptime30d = new PrometheusClient.Gauge({
+    name: "monitor_uptime_30d",
+    help: "Monitor Uptime 30d (%)",
+    labelNames: commonLabels,
+});
+
+const monitorUptime24h = new PrometheusClient.Gauge({
+    name: "monitor_uptime_24h",
+    help: "Monitor Uptime 24h (%)",
+    labelNames: commonLabels,
+});
+
+const monitorAverageResponseTime = new PrometheusClient.Gauge({
+    name: "monitor_average_response_time",
+    help: "Monitor Average Response Time (ms)",
+    labelNames: commonLabels,
+});
+
 const monitorResponseTime = new PrometheusClient.Gauge({
     name: "monitor_response_time",
     help: "Monitor Response Time (ms)",
@@ -54,7 +79,7 @@ class Prometheus {
      * @param {object} tlsInfo TLS details
      * @returns {void}
      */
-    update(heartbeat, tlsInfo) {
+    update(heartbeat, tlsInfo, uptime) {
 
         if (typeof tlsInfo !== "undefined") {
             try {
@@ -80,6 +105,76 @@ class Prometheus {
             }
         }
 
+        if (uptime) {
+            if (typeof uptime.avgPing !== "undefined") {
+                try {
+                    if (typeof uptime.avgPing === "number") {
+                        monitorAverageResponseTime.set(
+                            this.monitorLabelValues,
+                            uptime.avgPing
+                        );
+                    } else {
+                        // Is it good?
+                        monitorAverageResponseTime.set(
+                            this.monitorLabelValues,
+                            -1
+                        );
+                    }
+                } catch (e) {
+                    log.error("prometheus", "Caught error");
+                    log.error("prometheus", e);
+                }
+            }
+            if (typeof uptime.data24h !== "undefined") {
+                try {
+                    if (typeof uptime.data24h === "number") {
+                        monitorUptime24h.set(
+                            this.monitorLabelValues,
+                            uptime.data24h
+                        );
+                    } else {
+                        // Is it good?
+                        monitorUptime24h.set(this.monitorLabelValues, -1);
+                    }
+                } catch (e) {
+                    log.error("prometheus", "Caught error");
+                    log.error("prometheus", e);
+                }
+            }
+            if (typeof uptime.data30d !== "undefined") {
+                try {
+                    if (typeof uptime.data30d === "number") {
+                        monitorUptime30d.set(
+                            this.monitorLabelValues,
+                            uptime.data30d
+                        );
+                    } else {
+                        // Is it good?
+                        monitorUptime30d.set(this.monitorLabelValues, -1);
+                    }
+                } catch (e) {
+                    log.error("prometheus", "Caught error");
+                    log.error("prometheus", e);
+                }
+            }
+            if (typeof uptime.data1y !== "undefined") {
+                try {
+                    if (typeof uptime.data1y === "number") {
+                        monitorUptime1y.set(
+                            this.monitorLabelValues,
+                            uptime.data1y
+                        );
+                    } else {
+                        // Is it good?
+                        monitorUptime1y.set(this.monitorLabelValues, -1);
+                    }
+                } catch (e) {
+                    log.error("prometheus", "Caught error");
+                    log.error("prometheus", e);
+                }
+            }
+        }
+
         if (heartbeat) {
             try {
                 monitorStatus.set(this.monitorLabelValues, heartbeat.status);
@@ -110,6 +205,10 @@ class Prometheus {
         try {
             monitorCertDaysRemaining.remove(this.monitorLabelValues);
             monitorCertIsValid.remove(this.monitorLabelValues);
+            monitorUptime1y.remove(this.monitorLabelValues);
+            monitorUptime30d.remove(this.monitorLabelValues);
+            monitorUptime24h.remove(this.monitorLabelValues);
+            monitorAverageResponseTime.remove(this.monitorLabelValues);
             monitorResponseTime.remove(this.monitorLabelValues);
             monitorStatus.remove(this.monitorLabelValues);
         } catch (e) {