Skip to content

Commit

Permalink
Merge pull request #130 from brancz/labels
Browse files Browse the repository at this point in the history 
Add Pod/Node Labels metric
  • Loading branch information
@brancz
brancz committed Apr 28, 2017
2 parents 28fcd65 + 1cd6838 commit 007629f
Show file tree
Hide file tree
Showing 15 changed files with 1,736 additions and 46 deletions.
20 changes: 12 additions & 8 deletions main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
"github.com/golang/glog"
"github.com/openshift/origin/pkg/util/proc"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/spf13/pflag"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
Expand All @@ -47,19 +48,21 @@ var (
"pods": struct{}{},
"nodes": struct{}{},
"resourcequotas": struct{}{},
"limitrange": struct{}{},
"limitrange": struct{}{},
"replicasets": struct{}{},
"replicationcontrollers": struct{}{},
"services": struct{}{},
}
availableCollectors = map[string]func(registry prometheus.Registerer, kubeClient clientset.Interface){
"daemonsets": RegisterDaemonSetCollector,
"deployments": RegisterDeploymentCollector,
"pods": RegisterPodCollector,
"nodes": RegisterNodeCollector,
"resourcequotas": RegisterResourceQuotaCollector,
"limitrange": RegisterLimitRangeCollector,
"limitrange": RegisterLimitRangeCollector,
"replicasets": RegisterReplicaSetCollector,
"replicationcontrollers": RegisterReplicationControllerCollector,
"services": RegisterServiceCollector,
}
)

Expand Down Expand Up @@ -160,8 +163,9 @@ func main() {
glog.Fatalf("Failed to create client: %v", err)
}

registerCollectors(kubeClient, collectors)
metricsServer(options.port)
registry := prometheus.NewRegistry()
registerCollectors(registry, kubeClient, collectors)
metricsServer(registry, options.port)
}

func isNotExists(file string) bool {
Expand Down Expand Up @@ -224,13 +228,13 @@ func createKubeClient(inCluster bool, apiserver string, kubeconfig string) (kube
return kubeClient, nil
}

func metricsServer(port int) {
func metricsServer(registry prometheus.Gatherer, port int) {
// Address to listen on for web interface and telemetry
listenAddress := fmt.Sprintf(":%d", port)

glog.Infof("Starting metrics server: %s", listenAddress)
// Add metricsPath
http.Handle(metricsPath, prometheus.UninstrumentedHandler())
http.Handle(metricsPath, promhttp.HandlerFor(registry, promhttp.HandlerOpts{}))
// Add healthzPath
http.HandleFunc(healthzPath, func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(200)
Expand All @@ -254,12 +258,12 @@ func metricsServer(port int) {

// registerCollectors creates and starts informers and initializes and
// registers metrics for collection.
func registerCollectors(kubeClient clientset.Interface, enabledCollectors collectorSet) {
func registerCollectors(registry prometheus.Registerer, kubeClient clientset.Interface, enabledCollectors collectorSet) {
activeCollectors := []string{}
for c, _ := range enabledCollectors {
f, ok := availableCollectors[c]
if ok {
f(prometheus.DefaultRegisterer, kubeClient)
f(registry, kubeClient)
activeCollectors = append(activeCollectors, c)
}
}
Expand Down
22 changes: 22 additions & 0 deletions node.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,10 @@ import (
)

var (
descNodeLabelsName = "kube_node_labels"
descNodeLabelsHelp = "Kubernetes labels converted to Prometheus labels."
descNodeLabelsDefaultLabels = []string{"node"}

descNodeInfo = prometheus.NewDesc(
"kube_node_info",
"Information about a cluster node.",
Expand All @@ -40,6 +44,12 @@ var (
}, nil,
)

descNodeLabels = prometheus.NewDesc(
descNodeLabelsName,
descNodeLabelsHelp,
descNodeLabelsDefaultLabels, nil,
)

descNodeSpecUnschedulable = prometheus.NewDesc(
"kube_node_spec_unschedulable",
"Whether a node can schedule new pods.",
Expand Down Expand Up @@ -144,6 +154,7 @@ type nodeCollector struct {
// Describe implements the prometheus.Collector interface.
func (nc *nodeCollector) Describe(ch chan<- *prometheus.Desc) {
ch <- descNodeInfo
ch <- descNodeLabels
ch <- descNodeSpecUnschedulable
ch <- descNodeStatusReady
ch <- descNodeStatusMemoryPressure
Expand Down Expand Up @@ -171,6 +182,15 @@ func (nc *nodeCollector) Collect(ch chan<- prometheus.Metric) {
}
}

func nodeLabelsDesc(labelKeys []string) *prometheus.Desc {
return prometheus.NewDesc(
descNodeLabelsName,
descNodeLabelsHelp,
append(descNodeLabelsDefaultLabels, labelKeys...),
nil,
)
}

func (nc *nodeCollector) collectNode(ch chan<- prometheus.Metric, n v1.Node) {
addGauge := func(desc *prometheus.Desc, v float64, lv ...string) {
lv = append([]string{n.Name}, lv...)
Expand All @@ -185,6 +205,8 @@ func (nc *nodeCollector) collectNode(ch chan<- prometheus.Metric, n v1.Node) {
n.Status.NodeInfo.KubeletVersion,
n.Status.NodeInfo.KubeProxyVersion,
)
labelKeys, labelValues := kubeLabelsToPrometheusLabels(n.Labels)
addGauge(nodeLabelsDesc(labelKeys), 1, labelValues...)

addGauge(descNodeSpecUnschedulable, boolFloat64(n.Spec.Unschedulable))

Expand Down
19 changes: 13 additions & 6 deletions node_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@ func TestNodeCollector(t *testing.T) {
const metadata = `
# HELP kube_node_info Information about a cluster node.
# TYPE kube_node_info gauge
# HELP kube_node_labels Kubernetes labels converted to Prometheus labels.
# TYPE kube_node_labels gauge
# HELP kube_node_spec_unschedulable Whether a node can schedule new pods.
# TYPE kube_node_spec_unschedulable gauge
# HELP kube_node_status_ready The ready status of a cluster node.
Expand All @@ -55,12 +57,12 @@ func TestNodeCollector(t *testing.T) {
# HELP kube_node_status_allocatable_cpu_cores The CPU resources of a node that are available for scheduling.
# TYPE kube_node_status_allocatable_memory_bytes gauge
# HELP kube_node_status_allocatable_memory_bytes The memory resources of a node that are available for scheduling.
# HELP kube_node_status_memory_pressure Whether the kubelet is under pressure due to insufficient available memory.
# TYPE kube_node_status_memory_pressure gauge
# HELP kube_node_status_disk_pressure Whether the kubelet is under pressure due to insufficient available disk.
# TYPE kube_node_status_disk_pressure gauge
# HELP kube_node_status_network_unavailable Whether the network is correctly configured for the node.
# TYPE kube_node_status_network_unavailable gauge
# HELP kube_node_status_memory_pressure Whether the kubelet is under pressure due to insufficient available memory.
# TYPE kube_node_status_memory_pressure gauge
# HELP kube_node_status_disk_pressure Whether the kubelet is under pressure due to insufficient available disk.
# TYPE kube_node_status_disk_pressure gauge
# HELP kube_node_status_network_unavailable Whether the network is correctly configured for the node.
# TYPE kube_node_status_network_unavailable gauge
`
cases := []struct {
nodes []v1.Node
Expand All @@ -87,6 +89,7 @@ func TestNodeCollector(t *testing.T) {
},
want: metadata + `
kube_node_info{container_runtime_version="rkt",kernel_version="kernel",kubelet_version="kubelet",kubeproxy_version="kubeproxy",node="127.0.0.1",os_image="osimage"} 1
kube_node_labels{node="127.0.0.1"} 1
kube_node_spec_unschedulable{node="127.0.0.1"} 0
`,
},
Expand All @@ -96,6 +99,9 @@ func TestNodeCollector(t *testing.T) {
{
ObjectMeta: v1.ObjectMeta{
Name: "127.0.0.1",
Labels: map[string]string{
"type": "master",
},
},
Spec: v1.NodeSpec{
Unschedulable: true,
Expand Down Expand Up @@ -123,6 +129,7 @@ func TestNodeCollector(t *testing.T) {
},
want: metadata + `
kube_node_info{container_runtime_version="rkt",kernel_version="kernel",kubelet_version="kubelet",kubeproxy_version="kubeproxy",node="127.0.0.1",os_image="osimage"} 1
kube_node_labels{label_type="master",node="127.0.0.1"} 1
kube_node_spec_unschedulable{node="127.0.0.1"} 1
kube_node_status_capacity_cpu_cores{node="127.0.0.1"} 4.3
kube_node_status_capacity_memory_bytes{node="127.0.0.1"} 2e9
Expand Down
53 changes: 51 additions & 2 deletions pod.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,9 @@ package main

import (
"encoding/json"
"fmt"

"fmt"
"regexp"

"github.com/golang/glog"
"github.com/prometheus/client_golang/prometheus"
"golang.org/x/net/context"
Expand All @@ -30,51 +31,71 @@ import (
)

var (
invalidLabelCharRE = regexp.MustCompile(`[^a-zA-Z0-9_]`)
descPodLabelsName = "kube_pod_labels"
descPodLabelsHelp = "Kubernetes labels converted to Prometheus labels."
descPodLabelsDefaultLabels = []string{"namespace", "pod"}

descPodInfo = prometheus.NewDesc(
"kube_pod_info",
"Information about pod.",
[]string{"namespace", "pod", "host_ip", "pod_ip", "node", "created_by"}, nil,
)

descPodLabels = prometheus.NewDesc(
descPodLabelsName,
descPodLabelsHelp,
descPodLabelsDefaultLabels, nil,
)

descPodStatusPhase = prometheus.NewDesc(
"kube_pod_status_phase",
"The pods current phase.",
[]string{"namespace", "pod", "phase"}, nil,
)

descPodStatusReady = prometheus.NewDesc(
"kube_pod_status_ready",
"Describes whether the pod is ready to serve requests.",
[]string{"namespace", "pod", "condition"}, nil,
)

descPodStatusScheduled = prometheus.NewDesc(
"kube_pod_status_scheduled",
"Describes the status of the scheduling process for the pod.",
[]string{"namespace", "pod", "condition"}, nil,
)

descPodContainerInfo = prometheus.NewDesc(
"kube_pod_container_info",
"Information about a container in a pod.",
[]string{"namespace", "pod", "container", "image", "image_id", "container_id"}, nil,
)

descPodContainerStatusWaiting = prometheus.NewDesc(
"kube_pod_container_status_waiting",
"Describes whether the container is currently in waiting state.",
[]string{"namespace", "pod", "container"}, nil,
)

descPodContainerStatusRunning = prometheus.NewDesc(
"kube_pod_container_status_running",
"Describes whether the container is currently in running state.",
[]string{"namespace", "pod", "container"}, nil,
)

descPodContainerStatusTerminated = prometheus.NewDesc(
"kube_pod_container_status_terminated",
"Describes whether the container is currently in terminated state.",
[]string{"namespace", "pod", "container"}, nil,
)

descPodContainerStatusReady = prometheus.NewDesc(
"kube_pod_container_status_ready",
"Describes whether the containers readiness check succeeded.",
[]string{"namespace", "pod", "container"}, nil,
)

descPodContainerStatusRestarts = prometheus.NewDesc(
"kube_pod_container_status_restarts",
"The number of container restarts per container.",
Expand Down Expand Up @@ -140,6 +161,7 @@ type podCollector struct {
// Describe implements the prometheus.Collector interface.
func (pc *podCollector) Describe(ch chan<- *prometheus.Desc) {
ch <- descPodInfo
ch <- descPodLabels
ch <- descPodStatusPhase
ch <- descPodStatusReady
ch <- descPodStatusScheduled
Expand Down Expand Up @@ -179,6 +201,31 @@ func (pc *podCollector) Collect(ch chan<- prometheus.Metric) {
}
}

func kubeLabelsToPrometheusLabels(labels map[string]string) ([]string, []string) {
labelKeys := make([]string, len(labels))
labelValues := make([]string, len(labels))
i := 0
for k, v := range labels {
labelKeys[i] = "label_" + sanitizeLabelName(k)
labelValues[i] = v
i++
}
return labelKeys, labelValues
}

func sanitizeLabelName(s string) string {
return invalidLabelCharRE.ReplaceAllString(s, "_")
}

func podLabelsDesc(labelKeys []string) *prometheus.Desc {
return prometheus.NewDesc(
descPodLabelsName,
descPodLabelsHelp,
append(descPodLabelsDefaultLabels, labelKeys...),
nil,
)
}

func (pc *podCollector) collectPod(ch chan<- prometheus.Metric, p v1.Pod) {
nodeName := p.Spec.NodeName
addConstMetric := func(desc *prometheus.Desc, t prometheus.ValueType, v float64, lv ...string) {
Expand All @@ -193,6 +240,8 @@ func (pc *podCollector) collectPod(ch chan<- prometheus.Metric, p v1.Pod) {
}

addGauge(descPodInfo, 1, p.Status.HostIP, p.Status.PodIP, nodeName, extractCreatedBy(p.Annotations))
labelKeys, labelValues := kubeLabelsToPrometheusLabels(p.Labels)
addGauge(podLabelsDesc(labelKeys), 1, labelValues...)
addGauge(descPodStatusPhase, 1, string(p.Status.Phase))

for _, c := range p.Status.Conditions {
Expand Down
53 changes: 37 additions & 16 deletions pod_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@ func TestPodCollector(t *testing.T) {
const metadata = `
# HELP kube_pod_container_info Information about a container in a pod.
# TYPE kube_pod_container_info gauge
# HELP kube_pod_labels Kubernetes labels converted to Prometheus labels.
# TYPE kube_pod_labels gauge
# HELP kube_pod_container_status_ready Describes whether the containers readiness check succeeded.
# TYPE kube_pod_container_status_ready gauge
# HELP kube_pod_container_status_restarts The number of container restarts per container.
Expand Down Expand Up @@ -468,29 +470,48 @@ func TestPodCollector(t *testing.T) {
},
},
want: metadata + `
kube_pod_container_resource_requests_cpu_cores{container="pod1_con1",namespace="ns1",node="node1",pod="pod1"} 0.2
kube_pod_container_resource_requests_cpu_cores{container="pod1_con2",namespace="ns1",node="node1",pod="pod1"} 0.3
kube_pod_container_resource_requests_cpu_cores{container="pod2_con1",namespace="ns2",node="node2",pod="pod2"} 0.4
kube_pod_container_resource_requests_cpu_cores{container="pod2_con2",namespace="ns2",node="node2",pod="pod2"} 0.5
kube_pod_container_resource_requests_memory_bytes{container="pod1_con1",namespace="ns1",node="node1",pod="pod1"} 1e+08
kube_pod_container_resource_requests_memory_bytes{container="pod1_con2",namespace="ns1",node="node1",pod="pod1"} 2e+08
kube_pod_container_resource_requests_memory_bytes{container="pod2_con1",namespace="ns2",node="node2",pod="pod2"} 3e+08
kube_pod_container_resource_requests_memory_bytes{container="pod2_con2",namespace="ns2",node="node2",pod="pod2"} 4e+08
kube_pod_container_resource_limits_cpu_cores{container="pod1_con1",namespace="ns1",node="node1",pod="pod1"} 0.2
kube_pod_container_resource_limits_cpu_cores{container="pod1_con2",namespace="ns1",node="node1",pod="pod1"} 0.3
kube_pod_container_resource_limits_cpu_cores{container="pod2_con1",namespace="ns2",node="node2",pod="pod2"} 0.4
kube_pod_container_resource_limits_cpu_cores{container="pod2_con2",namespace="ns2",node="node2",pod="pod2"} 0.5
kube_pod_container_resource_limits_memory_bytes{container="pod1_con1",namespace="ns1",node="node1",pod="pod1"} 1e+08
kube_pod_container_resource_limits_memory_bytes{container="pod1_con2",namespace="ns1",node="node1",pod="pod1"} 2e+08
kube_pod_container_resource_limits_memory_bytes{container="pod2_con1",namespace="ns2",node="node2",pod="pod2"} 3e+08
kube_pod_container_resource_limits_memory_bytes{container="pod2_con2",namespace="ns2",node="node2",pod="pod2"} 4e+08
kube_pod_container_resource_requests_cpu_cores{container="pod1_con1",namespace="ns1",node="node1",pod="pod1"} 0.2
kube_pod_container_resource_requests_cpu_cores{container="pod1_con2",namespace="ns1",node="node1",pod="pod1"} 0.3
kube_pod_container_resource_requests_cpu_cores{container="pod2_con1",namespace="ns2",node="node2",pod="pod2"} 0.4
kube_pod_container_resource_requests_cpu_cores{container="pod2_con2",namespace="ns2",node="node2",pod="pod2"} 0.5
kube_pod_container_resource_requests_memory_bytes{container="pod1_con1",namespace="ns1",node="node1",pod="pod1"} 1e+08
kube_pod_container_resource_requests_memory_bytes{container="pod1_con2",namespace="ns1",node="node1",pod="pod1"} 2e+08
kube_pod_container_resource_requests_memory_bytes{container="pod2_con1",namespace="ns2",node="node2",pod="pod2"} 3e+08
kube_pod_container_resource_requests_memory_bytes{container="pod2_con2",namespace="ns2",node="node2",pod="pod2"} 4e+08
kube_pod_container_resource_limits_cpu_cores{container="pod1_con1",namespace="ns1",node="node1",pod="pod1"} 0.2
kube_pod_container_resource_limits_cpu_cores{container="pod1_con2",namespace="ns1",node="node1",pod="pod1"} 0.3
kube_pod_container_resource_limits_cpu_cores{container="pod2_con1",namespace="ns2",node="node2",pod="pod2"} 0.4
kube_pod_container_resource_limits_cpu_cores{container="pod2_con2",namespace="ns2",node="node2",pod="pod2"} 0.5
kube_pod_container_resource_limits_memory_bytes{container="pod1_con1",namespace="ns1",node="node1",pod="pod1"} 1e+08
kube_pod_container_resource_limits_memory_bytes{container="pod1_con2",namespace="ns1",node="node1",pod="pod1"} 2e+08
kube_pod_container_resource_limits_memory_bytes{container="pod2_con1",namespace="ns2",node="node2",pod="pod2"} 3e+08
kube_pod_container_resource_limits_memory_bytes{container="pod2_con2",namespace="ns2",node="node2",pod="pod2"} 4e+08
`,
metrics: []string{
"kube_pod_container_resource_requests_cpu_cores",
"kube_pod_container_resource_requests_memory_bytes",
"kube_pod_container_resource_limits_cpu_cores",
"kube_pod_container_resource_limits_memory_bytes",
},
}, {
pods: []v1.Pod{
{
ObjectMeta: v1.ObjectMeta{
Name: "pod1",
Namespace: "ns1",
Labels: map[string]string{
"app": "example",
},
},
Spec: v1.PodSpec{},
},
},
want: metadata + `
kube_pod_labels{label_app="example",namespace="ns1",pod="pod1"} 1
`,
metrics: []string{
"kube_pod_labels",
},
},
}
for _, c := range cases {
Expand Down
Loading

0 comments on commit 007629f

Please sign in to comment.