WIP2

pkg/template/controller/metrics.go

@@ -1,65 +1,71 @@
 package controller
 
 import (
+	"time"
+
 	templateapi "github.com/openshift/origin/pkg/template/apis/template"
 	"github.com/prometheus/client_golang/prometheus"
 	"k8s.io/apimachinery/pkg/labels"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	kapi "k8s.io/kubernetes/pkg/api"
 )
 
-var (
-	templateInstancesTotal = prometheus.NewDesc(
-		"TemplateInstanceController_TemplateInstances_total",
-		"Counts TemplateInstance objects by condition type and status",
+func newTemplateInstancesTotal() *prometheus.GaugeVec {
+	return prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "openshift_template_instance_total",
+			Help: "Counts TemplateInstance objects by condition type and status",
+		},
 		[]string{"type", "status"},
-		nil,
 	)
-	templateInstancesWaiting = prometheus.NewDesc(
-		"TemplateInstanceController_TemplateInstances_waiting_start_time_seconds",
-		"Show the start time in unix epoch form of waiting TemplateInstances by namespace and name",
-		[]string{"namespace", "name"},
-		nil,
+}
+
+func newTemplateInstancesWaiting() prometheus.Histogram {
+	return prometheus.NewHistogram(
+		prometheus.HistogramOpts{
+			Name:    "openshift_template_instance_active_wait_time_seconds",
+			Help:    "Show the distribution of active TemplateInstance wait times",
+			Buckets: []float64{60, 300, 600, 1200, 3600},
+		},
 	)
-)
+}
 
 func (c *TemplateInstanceController) Describe(ch chan<- *prometheus.Desc) {
-	ch <- templateInstancesTotal
-	ch <- templateInstancesWaiting
+	newTemplateInstancesTotal().Describe(ch)
+	newTemplateInstancesWaiting().Describe(ch)
 }
 
 func (c *TemplateInstanceController) Collect(ch chan<- prometheus.Metric) {
+	now := time.Now()
+
 	templateInstances, err := c.lister.List(labels.Everything())
 	if err != nil {
 		utilruntime.HandleError(err)
 		return
 	}
 
-	m := map[templateapi.TemplateInstanceCondition]uint64{}
+	templateInstancesTotal := newTemplateInstancesTotal()
+	templateInstancesWaiting := newTemplateInstancesWaiting()
 
 	for _, templateInstance := range templateInstances {
 		waiting := true
 
-		m[templateapi.TemplateInstanceCondition{}]++
+		templateInstancesTotal.WithLabelValues("", "").Inc()
+
 		for _, cond := range templateInstance.Status.Conditions {
-			m[templateapi.TemplateInstanceCondition{Type: cond.Type, Status: cond.Status}]++
-			if cond.Type == templateapi.TemplateInstanceInstantiateFailure && cond.Status == kapi.ConditionTrue ||
-				cond.Type == templateapi.TemplateInstanceReady && cond.Status == kapi.ConditionTrue {
+			templateInstancesTotal.WithLabelValues(string(cond.Type), string(cond.Status)).Inc()
+
+			if cond.Status == kapi.ConditionTrue &&
+				(cond.Type == templateapi.TemplateInstanceInstantiateFailure || cond.Type == templateapi.TemplateInstanceReady) {
 				waiting = false
 			}
 		}
 
 		if waiting {
-			ch <- prometheus.MustNewConstMetric(templateInstancesWaiting,
-				prometheus.GaugeValue,
-				float64(templateInstance.CreationTimestamp.Unix()),
-				templateInstance.Namespace,
-				templateInstance.Name,
-			)
+			templateInstancesWaiting.Observe(float64(now.Sub(templateInstance.CreationTimestamp.Time) / time.Second))
 		}
 	}
 
-	for cond, count := range m {
-		ch <- prometheus.MustNewConstMetric(templateInstancesTotal, prometheus.GaugeValue, float64(count), string(cond.Type), string(cond.Status))
-	}
+	templateInstancesTotal.Collect(ch)
+	templateInstancesWaiting.Collect(ch)
 }

pkg/template/controller/templateinstance_controller.go

@@ -71,7 +71,7 @@ func NewTemplateInstanceController(config *rest.Config, oc client.Interface, kc
 		templateclient:   templateclient.Template(),
 		lister:           informer.Lister(),
 		informer:         informer.Informer(),
-		queue:            workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "TemplateInstanceController"),
+		queue:            workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "openshift_template_instance_controller_queue"),
 		readinessLimiter: workqueue.NewItemFastSlowRateLimiter(5*time.Second, 20*time.Second, 200),
 	}