-
Notifications
You must be signed in to change notification settings - Fork 4.7k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
implement prometheus metrics for the TemplateInstance controller
- Loading branch information
Jim Minter
committed
Sep 26, 2017
1 parent
e020a09
commit b3451c8
Showing
5 changed files
with
227 additions
and
2 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,65 @@ | ||
package controller | ||
|
||
import ( | ||
templateapi "github.com/openshift/origin/pkg/template/apis/template" | ||
"github.com/prometheus/client_golang/prometheus" | ||
"k8s.io/apimachinery/pkg/labels" | ||
utilruntime "k8s.io/apimachinery/pkg/util/runtime" | ||
kapi "k8s.io/kubernetes/pkg/api" | ||
) | ||
|
||
var templateInstancesTotal = prometheus.NewGaugeVec( | ||
prometheus.GaugeOpts{ | ||
Name: "openshift_template_instance_total", | ||
Help: "Counts TemplateInstance objects by condition type and status", | ||
}, | ||
[]string{"type", "status"}, | ||
) | ||
|
||
var templateInstancesActiveStartTime = prometheus.NewGaugeVec( | ||
prometheus.GaugeOpts{ | ||
Name: "openshift_template_instance_active_start_time_seconds", | ||
Help: "Show the start time in unix epoch form of active TemplateInstance objects by namespace and name", | ||
}, | ||
[]string{"namespace", "name"}, | ||
) | ||
|
||
func (c *TemplateInstanceController) Describe(ch chan<- *prometheus.Desc) { | ||
templateInstancesTotal.Describe(ch) | ||
templateInstancesActiveStartTime.Describe(ch) | ||
} | ||
|
||
func (c *TemplateInstanceController) Collect(ch chan<- prometheus.Metric) { | ||
templateInstances, err := c.lister.List(labels.Everything()) | ||
if err != nil { | ||
utilruntime.HandleError(err) | ||
return | ||
} | ||
|
||
templateInstancesTotal.Reset() | ||
templateInstancesActiveStartTime.Reset() | ||
|
||
templateInstancesTotal.WithLabelValues("", "").Set(0) | ||
|
||
for _, templateInstance := range templateInstances { | ||
waiting := true | ||
|
||
templateInstancesTotal.WithLabelValues("", "").Inc() | ||
|
||
for _, cond := range templateInstance.Status.Conditions { | ||
templateInstancesTotal.WithLabelValues(string(cond.Type), string(cond.Status)).Inc() | ||
|
||
if cond.Status == kapi.ConditionTrue && | ||
(cond.Type == templateapi.TemplateInstanceInstantiateFailure || cond.Type == templateapi.TemplateInstanceReady) { | ||
waiting = false | ||
} | ||
} | ||
|
||
if waiting { | ||
templateInstancesActiveStartTime.WithLabelValues(templateInstance.Namespace, templateInstance.Name).Set(float64(templateInstance.CreationTimestamp.Unix())) | ||
} | ||
} | ||
|
||
templateInstancesTotal.Collect(ch) | ||
templateInstancesActiveStartTime.Collect(ch) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,96 @@ | ||
package controller | ||
|
||
import ( | ||
"bytes" | ||
"net/http" | ||
"testing" | ||
"time" | ||
|
||
templateapi "github.com/openshift/origin/pkg/template/apis/template" | ||
"github.com/openshift/origin/pkg/template/generated/listers/template/internalversion" | ||
|
||
"github.com/prometheus/client_golang/prometheus" | ||
"github.com/prometheus/client_golang/prometheus/promhttp" | ||
|
||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||
"k8s.io/apimachinery/pkg/labels" | ||
kapi "k8s.io/kubernetes/pkg/api" | ||
) | ||
|
||
type fakeLister []*templateapi.TemplateInstance | ||
|
||
func (f fakeLister) List(labels.Selector) ([]*templateapi.TemplateInstance, error) { | ||
return f, nil | ||
} | ||
func (fakeLister) TemplateInstances(string) internalversion.TemplateInstanceNamespaceLister { | ||
return nil | ||
} | ||
|
||
type fakeResponseWriter struct { | ||
bytes.Buffer | ||
statusCode int | ||
header http.Header | ||
} | ||
|
||
func (f *fakeResponseWriter) Header() http.Header { | ||
return f.header | ||
} | ||
|
||
func (f *fakeResponseWriter) WriteHeader(statusCode int) { | ||
f.statusCode = statusCode | ||
} | ||
|
||
func TestMetrics(t *testing.T) { | ||
expectedResponse := `# HELP openshift_template_instance_active_start_time_seconds Show the start time in unix epoch form of active TemplateInstance objects by namespace and name | ||
# TYPE openshift_template_instance_active_start_time_seconds gauge | ||
openshift_template_instance_active_start_time_seconds{name="testname",namespace="testnamespace"} 123 | ||
# HELP openshift_template_instance_total Counts TemplateInstance objects by condition type and status | ||
# TYPE openshift_template_instance_total gauge | ||
openshift_template_instance_total{status="",type=""} 2 | ||
openshift_template_instance_total{status="False",type="Ready"} 1 | ||
openshift_template_instance_total{status="True",type="Ready"} 1 | ||
` | ||
registry := prometheus.NewRegistry() | ||
|
||
c := &TemplateInstanceController{ | ||
lister: &fakeLister{ | ||
{ | ||
Status: templateapi.TemplateInstanceStatus{ | ||
Conditions: []templateapi.TemplateInstanceCondition{ | ||
{ | ||
Type: templateapi.TemplateInstanceReady, | ||
Status: kapi.ConditionTrue, | ||
}, | ||
}, | ||
}, | ||
}, | ||
{ | ||
ObjectMeta: metav1.ObjectMeta{ | ||
Namespace: "testnamespace", | ||
Name: "testname", | ||
CreationTimestamp: metav1.Time{ | ||
Time: time.Unix(123, 0), | ||
}, | ||
}, | ||
Status: templateapi.TemplateInstanceStatus{ | ||
Conditions: []templateapi.TemplateInstanceCondition{ | ||
{ | ||
Type: templateapi.TemplateInstanceReady, | ||
Status: kapi.ConditionFalse, | ||
}, | ||
}, | ||
}, | ||
}, | ||
}, | ||
} | ||
|
||
registry.MustRegister(c) | ||
|
||
h := promhttp.HandlerFor(registry, promhttp.HandlerOpts{ErrorHandling: promhttp.PanicOnError}) | ||
rw := &fakeResponseWriter{header: http.Header{}} | ||
h.ServeHTTP(rw, &http.Request{}) | ||
|
||
if rw.String() != expectedResponse { | ||
t.Error(rw.String()) | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,61 @@ | ||
package integration | ||
|
||
import ( | ||
"regexp" | ||
"testing" | ||
"time" | ||
|
||
"k8s.io/apimachinery/pkg/util/wait" | ||
|
||
testutil "github.com/openshift/origin/test/util" | ||
testserver "github.com/openshift/origin/test/util/server" | ||
) | ||
|
||
var metricsRegexp = regexp.MustCompile("(?m)^# HELP ([^ ]*)") | ||
|
||
func TestMetrics(t *testing.T) { | ||
expectedMetrics := []string{ | ||
"openshift_build_terminal_phase_total", | ||
"openshift_template_instance_total", | ||
} | ||
|
||
masterConfig, clusterAdminKubeConfig, err := testserver.StartTestMaster() | ||
if err != nil { | ||
t.Fatal(err) | ||
} | ||
defer testserver.CleanupMasterEtcd(t, masterConfig) | ||
|
||
clusterAdminClient, err := testutil.GetClusterAdminClient(clusterAdminKubeConfig) | ||
if err != nil { | ||
t.Fatal(err) | ||
} | ||
|
||
var missingMetrics []string | ||
err = wait.Poll(time.Second, 30*time.Second, func() (bool, error) { | ||
missingMetrics = []string{} | ||
|
||
b, err := clusterAdminClient.Get().RequestURI("/metrics").DoRaw() | ||
if err != nil { | ||
return false, err | ||
} | ||
|
||
metrics := map[string]struct{}{} | ||
for _, match := range metricsRegexp.FindAllStringSubmatch(string(b), -1) { | ||
metrics[match[1]] = struct{}{} | ||
} | ||
|
||
for _, metric := range expectedMetrics { | ||
if _, ok := metrics[metric]; !ok { | ||
missingMetrics = append(missingMetrics, metric) | ||
} | ||
} | ||
|
||
return len(missingMetrics) == 0, nil | ||
}) | ||
if len(missingMetrics) > 0 { | ||
t.Error(missingMetrics) | ||
} | ||
if err != nil { | ||
t.Error(err) | ||
} | ||
} |