prometheus: active worker gauge

internal/prometheus/worker_metrics.go

@@ -0,0 +1,23 @@
+package prometheus
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var (
+	ActiveWorkers = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Name:      "active_workers",
+		Namespace: Namespace,
+		Subsystem: WorkerSubsystem,
+		Help:      "Active workers",
+	}, []string{"worker", "tenant", "arch"})
+)
+
+func AddActiveWorker(worker, tenant, arch string) {
+	ActiveWorkers.WithLabelValues(worker, tenant, arch).Inc()
+}
+
+func RemoveActiveWorker(worker, tenant, arch string) {
+	ActiveWorkers.WithLabelValues(worker, tenant, arch).Dec()
+}

internal/worker/server.go

@@ -168,7 +168,9 @@ func (s *Server) WatchWorkers() {
 			err = s.jobs.DeleteWorker(w.ID)
 			if err != nil {
 				logrus.Warningf("Unable to remove worker: %v", err)
+				continue
 			}
+			prometheus.RemoveActiveWorker(w.ID.String(), w.Channel, w.Arch)
 		}
 	}
 }
@@ -828,6 +830,7 @@ func (s *Server) RegisterWorker(c, a string) (uuid.UUID, error) {
 	if err != nil {
 		return uuid.Nil, err
 	}
+	prometheus.AddActiveWorker(workerID.String(), c, a)
 	logrus.Infof("Worker (%v) registered", a)
 	return workerID, nil
 }