Skip to content

Commit

Permalink
Add pod init container metrics
Browse files Browse the repository at this point in the history 
Co-Authored-By: Tariq Ibrahim <tariq181290@gmail.com>
Co-Authored-By: Sumit Vij <sumitvij11+github@gmail.com>
  • Loading branch information
@tariq1890 @thedebugger
tariq1890 and thedebugger committed May 20, 2019
1 parent 43cce2d commit 050ab04
Show file tree
Hide file tree
Showing 4 changed files with 494 additions and 5 deletions.
11 changes: 11 additions & 0 deletions docs/pod-metrics.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,17 @@
| kube_pod_container_resource_limits | Gauge | `resource`=&lt;resource-name&gt; <br> `unit`=&lt;resource-unit&gt; <br> `container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `node`=&lt; node-name&gt; | STABLE |
| kube_pod_container_resource_limits_memory_bytes | Gauge | `container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `node`=&lt; node-name&gt; | DEPRECATED |
| kube_pod_created | Gauge | `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; |
| kube_pod_init_container_info | Gauge | `container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `image`=&lt;image-name&gt; <br> `image_id`=&lt;image-id&gt; <br> `container_id`=&lt;containerid&gt; | STABLE |
| kube_pod_init_container_status_waiting | Gauge | `container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; | STABLE |
| kube_pod_init_container_status_waiting_reason | Gauge | `container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `reason`=&lt;ContainerCreating\|CrashLoopBackOff\|ErrImagePull\|ImagePullBackOff\|CreateContainerConfigError&gt; | STABLE |
| kube_pod_init_container_status_running | Gauge | `container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; | STABLE |
| kube_pod_init_container_status_terminated | Gauge | `container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; | STABLE |
| kube_pod_init_container_status_terminated_reason | Gauge | `container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `reason`=&lt;OOMKilled\|Error\|Completed\|ContainerCannotRun\|DeadlineExceeded&gt; | STABLE |
| kube_pod_init_container_status_last_terminated_reason | Gauge | `container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `reason`=&lt;OOMKilled\|Error\|Completed\|ContainerCannotRun\|DeadlineExceeded&gt; | STABLE |
| kube_pod_init_container_status_ready | Gauge | `container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; | STABLE |
| kube_pod_init_container_status_restarts_total | Counter | `container`=&lt;container-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `pod`=&lt;pod-name&gt; | STABLE |
| kube_pod_init_container_resource_requests | Gauge | `resource`=&lt;resource-name&gt; <br> `unit`=&lt;resource-unit&gt; <br> `container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `node`=&lt; node-name&gt; | STABLE |
| kube_pod_init_container_resource_limits | Gauge | `resource`=&lt;resource-name&gt; <br> `unit`=&lt;resource-unit&gt; <br> `container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `node`=&lt; node-name&gt; | STABLE |
| kube_pod_spec_volumes_persistentvolumeclaims_info | Gauge | `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `volume`=&lt;volume-name&gt; <br> `persistentvolumeclaim`=&lt;persistentvolumeclaim-claimname&gt; | STABLE |
| kube_pod_spec_volumes_persistentvolumeclaims_readonly | Gauge | `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `volume`=&lt;volume-name&gt; <br> `persistentvolumeclaim`=&lt;persistentvolumeclaim-claimname&gt; | STABLE |
| kube_pod_status_scheduled_time | Gauge | `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; | STABLE |
268 changes: 268 additions & 0 deletions internal/collector/pod.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -333,6 +333,27 @@ var (
}
}),
},
{
Name: "kube_pod_init_container_info",
Type: metric.Gauge,
Help: "Information about an init container in a pod.",
GenerateFunc: wrapPodFunc(func(p *v1.Pod) *metric.Family {
ms := make([]*metric.Metric, len(p.Status.InitContainerStatuses))
labelKeys := []string{"container", "image", "image_id", "container_id"}

for i, cs := range p.Status.InitContainerStatuses {
ms[i] = &metric.Metric{
LabelKeys: labelKeys,
LabelValues: []string{cs.Name, cs.Image, cs.ImageID, cs.ContainerID},
Value: 1,
}
}

return &metric.Family{
Metrics: ms,
}
}),
},
{
Name: "kube_pod_container_status_waiting",
Type: metric.Gauge,
Expand All @@ -353,6 +374,26 @@ var (
}
}),
},
{
Name: "kube_pod_init_container_status_waiting",
Type: metric.Gauge,
Help: "Describes whether the init container is currently in waiting state.",
GenerateFunc: wrapPodFunc(func(p *v1.Pod) *metric.Family {
ms := make([]*metric.Metric, len(p.Status.InitContainerStatuses))

for i, cs := range p.Status.InitContainerStatuses {
ms[i] = &metric.Metric{
LabelKeys: []string{"container"},
LabelValues: []string{cs.Name},
Value: boolFloat64(cs.State.Waiting != nil),
}
}

return &metric.Family{
Metrics: ms,
}
}),
},
{
Name: "kube_pod_container_status_waiting_reason",
Type: metric.Gauge,
Expand All @@ -375,6 +416,28 @@ var (
}
}),
},
{
Name: "kube_pod_init_container_status_waiting_reason",
Type: metric.Gauge,
Help: "Describes the reason the init container is currently in waiting state.",
GenerateFunc: wrapPodFunc(func(p *v1.Pod) *metric.Family {
ms := make([]*metric.Metric, len(p.Status.InitContainerStatuses)*len(containerWaitingReasons))

for i, cs := range p.Status.InitContainerStatuses {
for j, reason := range containerWaitingReasons {
ms[i*len(containerWaitingReasons)+j] = &metric.Metric{
LabelKeys: []string{"container", "reason"},
LabelValues: []string{cs.Name, reason},
Value: boolFloat64(waitingReason(cs, reason)),
}
}
}

return &metric.Family{
Metrics: ms,
}
}),
},
{
Name: "kube_pod_container_status_running",
Type: metric.Gauge,
Expand All @@ -395,6 +458,26 @@ var (
}
}),
},
{
Name: "kube_pod_init_container_status_running",
Type: metric.Gauge,
Help: "Describes whether the init container is currently in running state.",
GenerateFunc: wrapPodFunc(func(p *v1.Pod) *metric.Family {
ms := make([]*metric.Metric, len(p.Status.InitContainerStatuses))

for i, cs := range p.Status.InitContainerStatuses {
ms[i] = &metric.Metric{
LabelKeys: []string{"container"},
LabelValues: []string{cs.Name},
Value: boolFloat64(cs.State.Running != nil),
}
}

return &metric.Family{
Metrics: ms,
}
}),
},
{
Name: "kube_pod_container_status_terminated",
Type: metric.Gauge,
Expand All @@ -415,6 +498,26 @@ var (
}
}),
},
{
Name: "kube_pod_init_container_status_terminated",
Type: metric.Gauge,
Help: "Describes whether the init container is currently in terminated state.",
GenerateFunc: wrapPodFunc(func(p *v1.Pod) *metric.Family {
ms := make([]*metric.Metric, len(p.Status.InitContainerStatuses))

for i, cs := range p.Status.InitContainerStatuses {
ms[i] = &metric.Metric{
LabelKeys: []string{"container"},
LabelValues: []string{cs.Name},
Value: boolFloat64(cs.State.Terminated != nil),
}
}

return &metric.Family{
Metrics: ms,
}
}),
},
{
Name: "kube_pod_container_status_terminated_reason",
Type: metric.Gauge,
Expand All @@ -437,6 +540,28 @@ var (
}
}),
},
{
Name: "kube_pod_init_container_status_terminated_reason",
Type: metric.Gauge,
Help: "Describes the reason the init container is currently in terminated state.",
GenerateFunc: wrapPodFunc(func(p *v1.Pod) *metric.Family {
ms := make([]*metric.Metric, len(p.Status.InitContainerStatuses)*len(containerTerminatedReasons))

for i, cs := range p.Status.InitContainerStatuses {
for j, reason := range containerTerminatedReasons {
ms[i*len(containerTerminatedReasons)+j] = &metric.Metric{
LabelKeys: []string{"container", "reason"},
LabelValues: []string{cs.Name, reason},
Value: boolFloat64(terminationReason(cs, reason)),
}
}
}

return &metric.Family{
Metrics: ms,
}
}),
},
{
Name: "kube_pod_container_status_last_terminated_reason",
Type: metric.Gauge,
Expand All @@ -459,6 +584,28 @@ var (
}
}),
},
{
Name: "kube_pod_init_container_status_last_terminated_reason",
Type: metric.Gauge,
Help: "Describes the last reason the init container was in terminated state.",
GenerateFunc: wrapPodFunc(func(p *v1.Pod) *metric.Family {
ms := make([]*metric.Metric, len(p.Status.InitContainerStatuses)*len(containerTerminatedReasons))

for i, cs := range p.Status.InitContainerStatuses {
for j, reason := range containerTerminatedReasons {
ms[i*len(containerTerminatedReasons)+j] = &metric.Metric{
LabelKeys: []string{"container", "reason"},
LabelValues: []string{cs.Name, reason},
Value: boolFloat64(lastTerminationReason(cs, reason)),
}
}
}

return &metric.Family{
Metrics: ms,
}
}),
},
{
Name: "kube_pod_container_status_ready",
Type: metric.Gauge,
Expand All @@ -479,6 +626,26 @@ var (
}
}),
},
{
Name: "kube_pod_init_container_status_ready",
Type: metric.Gauge,
Help: "Describes whether the init containers readiness check succeeded.",
GenerateFunc: wrapPodFunc(func(p *v1.Pod) *metric.Family {
ms := make([]*metric.Metric, len(p.Status.InitContainerStatuses))

for i, cs := range p.Status.InitContainerStatuses {
ms[i] = &metric.Metric{
LabelKeys: []string{"container"},
LabelValues: []string{cs.Name},
Value: boolFloat64(cs.Ready),
}
}

return &metric.Family{
Metrics: ms,
}
}),
},
{
Name: "kube_pod_container_status_restarts_total",
Type: metric.Counter,
Expand All @@ -499,6 +666,26 @@ var (
}
}),
},
{
Name: "kube_pod_init_container_status_restarts_total",
Type: metric.Counter,
Help: "The number of restarts for the init container.",
GenerateFunc: wrapPodFunc(func(p *v1.Pod) *metric.Family {
ms := make([]*metric.Metric, len(p.Status.InitContainerStatuses))

for i, cs := range p.Status.InitContainerStatuses {
ms[i] = &metric.Metric{
LabelKeys: []string{"container"},
LabelValues: []string{cs.Name},
Value: float64(cs.RestartCount),
}
}

return &metric.Family{
Metrics: ms,
}
}),
},
{
Name: "kube_pod_container_resource_requests",
Type: metric.Gauge,
Expand Down Expand Up @@ -615,6 +802,64 @@ var (
}
}),
},
{
Name: "kube_pod_init_container_resource_limits",
Type: metric.Gauge,
Help: "The number of requested limit resource by the init container.",
GenerateFunc: wrapPodFunc(func(p *v1.Pod) *metric.Family {
ms := []*metric.Metric{}

for _, c := range p.Spec.InitContainers {
lim := c.Resources.Limits

for resourceName, val := range lim {
switch resourceName {
case v1.ResourceCPU:
ms = append(ms, &metric.Metric{
Value: float64(val.MilliValue()) / 1000,
LabelValues: []string{c.Name, p.Spec.NodeName, sanitizeLabelName(string(resourceName)), string(constant.UnitCore)},
})
case v1.ResourceStorage:
fallthrough
case v1.ResourceEphemeralStorage:
fallthrough
case v1.ResourceMemory:
ms = append(ms, &metric.Metric{
LabelValues: []string{c.Name, p.Spec.NodeName, sanitizeLabelName(string(resourceName)), string(constant.UnitByte)},
Value: float64(val.Value()),
})
default:
if isHugePageResourceName(resourceName) {
ms = append(ms, &metric.Metric{
LabelValues: []string{c.Name, p.Spec.NodeName, sanitizeLabelName(string(resourceName)), string(constant.UnitByte)},
Value: float64(val.Value()),
})
}
if isAttachableVolumeResourceName(resourceName) {
ms = append(ms, &metric.Metric{
Value: float64(val.Value()),
LabelValues: []string{c.Name, p.Spec.NodeName, sanitizeLabelName(string(resourceName)), string(constant.UnitByte)},
})
}
if isExtendedResourceName(resourceName) {
ms = append(ms, &metric.Metric{
Value: float64(val.Value()),
LabelValues: []string{c.Name, p.Spec.NodeName, sanitizeLabelName(string(resourceName)), string(constant.UnitInteger)},
})
}
}
}
}

for _, metric := range ms {
metric.LabelKeys = []string{"container", "node", "resource", "unit"}
}

return &metric.Family{
Metrics: ms,
}
}),
},
{
Name: "kube_pod_container_resource_requests_cpu_cores",
Type: metric.Gauge,
Expand All @@ -638,6 +883,29 @@ var (
}
}),
},
{
Name: "kube_init_pod_container_resource_requests_cpu_cores",
Type: metric.Gauge,
Help: "The number of requested cpu cores by an init container.",
GenerateFunc: wrapPodFunc(func(p *v1.Pod) *metric.Family {
ms := []*metric.Metric{}

for _, c := range p.Spec.InitContainers {
req := c.Resources.Requests
if cpu, ok := req[v1.ResourceCPU]; ok {
ms = append(ms, &metric.Metric{
LabelKeys: []string{"container", "node"},
LabelValues: []string{c.Name, p.Spec.NodeName},
Value: float64(cpu.MilliValue()) / 1000,
})
}
}

return &metric.Family{
Metrics: ms,
}
}),
},
{
Name: "kube_pod_container_resource_requests_memory_bytes",
Type: metric.Gauge,
Expand Down

0 comments on commit 050ab04

Please sign in to comment.