Skip to content

Commit

Permalink
Merge pull request #535 from jutley/container-last-terminated-reason
Browse files Browse the repository at this point in the history 
Add kube_pod_container_status_last_terminated_reason
  • Loading branch information
@k8s-ci-robot
k8s-ci-robot authored Sep 5, 2018
2 parents 1b79b31 + e2afd1e commit 5119063
Show file tree
Hide file tree
Showing 3 changed files with 82 additions and 0 deletions.
1 change: 1 addition & 0 deletions Documentation/pod-metrics.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
| kube_pod_container_status_running | Gauge | `container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; | STABLE |
| kube_pod_container_status_terminated | Gauge | `container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; | STABLE |
| kube_pod_container_status_terminated_reason | Gauge | `container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `reason`=&lt;OOMKilled\|Error\|Completed\|ContainerCannotRun&gt; | STABLE |
| kube_pod_container_status_last_terminated_reason | Gauge | `container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `reason`=&lt;OOMKilled\|Error\|Completed\|ContainerCannotRun&gt; | STABLE |
| kube_pod_container_status_ready | Gauge | `container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; | STABLE |
| kube_pod_container_status_restarts_total | Counter | `container`=&lt;container-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `pod`=&lt;pod-name&gt; | STABLE |
| kube_pod_container_resource_requests_cpu_cores | Gauge | `container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `node`=&lt; node-name&gt; | STABLE |
Expand Down
18 changes: 18 additions & 0 deletions pkg/collectors/pod.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,13 @@ var (
append(descPodLabelsDefaultLabels, "container", "reason"),
nil,
)
descPodContainerStatusLastTerminatedReason = prometheus.NewDesc(
"kube_pod_container_status_last_terminated_reason",
"Describes the last reason the container was in terminated state.",
append(descPodLabelsDefaultLabels, "container", "reason"),
nil,
)

descPodContainerStatusReady = prometheus.NewDesc(
"kube_pod_container_status_ready",
"Describes whether the containers readiness check succeeded.",
Expand Down Expand Up @@ -251,6 +258,7 @@ func (pc *podCollector) Describe(ch chan<- *prometheus.Desc) {
ch <- descPodContainerStatusRunning
ch <- descPodContainerStatusTerminated
ch <- descPodContainerStatusTerminatedReason
ch <- descPodContainerStatusLastTerminatedReason
ch <- descPodContainerStatusReady
ch <- descPodContainerStatusRestarts
ch <- descPodSpecVolumesPersistentVolumeClaimsInfo
Expand Down Expand Up @@ -380,6 +388,13 @@ func (pc *podCollector) collectPod(ch chan<- prometheus.Metric, p v1.Pod) {
return cs.State.Terminated.Reason == reason
}

lastTerminationReason := func(cs v1.ContainerStatus, reason string) bool {
if cs.LastTerminationState.Terminated == nil {
return false
}
return cs.LastTerminationState.Terminated.Reason == reason
}

var lastFinishTime float64

for _, cs := range p.Status.ContainerStatuses {
Expand All @@ -395,6 +410,9 @@ func (pc *podCollector) collectPod(ch chan<- prometheus.Metric, p v1.Pod) {
for _, reason := range containerTerminatedReasons {
addGauge(descPodContainerStatusTerminatedReason, boolFloat64(terminationReason(cs, reason)), cs.Name, reason)
}
for _, reason := range containerTerminatedReasons {
addGauge(descPodContainerStatusLastTerminatedReason, boolFloat64(lastTerminationReason(cs, reason)), cs.Name, reason)
}
addGauge(descPodContainerStatusReady, boolFloat64(cs.Ready), cs.Name)
addCounter(descPodContainerStatusRestarts, float64(cs.RestartCount), cs.Name)

Expand Down
63 changes: 63 additions & 0 deletions pkg/collectors/pod_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,8 @@ func TestPodCollector(t *testing.T) {
# TYPE kube_pod_container_status_terminated gauge
# HELP kube_pod_container_status_terminated_reason Describes the reason the container is currently in terminated state.
# TYPE kube_pod_container_status_terminated_reason gauge
# HELP kube_pod_container_status_last_terminated_reason Describes the last reason the container was in terminated state.
# TYPE kube_pod_container_status_last_terminated_reason gauge
# HELP kube_pod_container_status_waiting Describes whether the container is currently in waiting state.
# TYPE kube_pod_container_status_waiting gauge
# HELP kube_pod_container_status_waiting_reason Describes the reason the container is currently in waiting state.
Expand Down Expand Up @@ -326,6 +328,27 @@ func TestPodCollector(t *testing.T) {
},
},
},
{
ObjectMeta: metav1.ObjectMeta{
Name: "pod6",
Namespace: "ns6",
},
Status: v1.PodStatus{
ContainerStatuses: []v1.ContainerStatus{
v1.ContainerStatus{
Name: "container7",
State: v1.ContainerState{
Running: &v1.ContainerStateRunning{},
},
LastTerminationState: v1.ContainerState{
Terminated: &v1.ContainerStateTerminated{
Reason: "OOMKilled",
},
},
},
},
},
},
},
want: metadata + `
kube_pod_container_status_running{container="container1",namespace="ns1",pod="pod1"} 1
Expand All @@ -334,12 +357,14 @@ func TestPodCollector(t *testing.T) {
kube_pod_container_status_running{container="container6",namespace="ns5",pod="pod5"} 0
kube_pod_container_status_running{container="container2",namespace="ns2",pod="pod2"} 0
kube_pod_container_status_running{container="container3",namespace="ns2",pod="pod2"} 0
kube_pod_container_status_running{container="container7",namespace="ns6",pod="pod6"} 1
kube_pod_container_status_terminated{container="container1",namespace="ns1",pod="pod1"} 0
kube_pod_container_status_terminated{container="container4",namespace="ns3",pod="pod3"} 0
kube_pod_container_status_terminated{container="container5",namespace="ns4",pod="pod4"} 0
kube_pod_container_status_terminated{container="container6",namespace="ns5",pod="pod5"} 0
kube_pod_container_status_terminated{container="container2",namespace="ns2",pod="pod2"} 1
kube_pod_container_status_terminated{container="container3",namespace="ns2",pod="pod2"} 0
kube_pod_container_status_terminated{container="container7",namespace="ns6",pod="pod6"} 0
kube_pod_container_status_terminated_reason{container="container1",namespace="ns1",pod="pod1",reason="Completed"} 0
kube_pod_container_status_terminated_reason{container="container1",namespace="ns1",pod="pod1",reason="ContainerCannotRun"} 0
kube_pod_container_status_terminated_reason{container="container1",namespace="ns1",pod="pod1",reason="Error"} 0
Expand All @@ -364,12 +389,17 @@ func TestPodCollector(t *testing.T) {
kube_pod_container_status_terminated_reason{container="container3",namespace="ns2",pod="pod2",reason="ContainerCannotRun"} 0
kube_pod_container_status_terminated_reason{container="container3",namespace="ns2",pod="pod2",reason="Error"} 0
kube_pod_container_status_terminated_reason{container="container3",namespace="ns2",pod="pod2",reason="OOMKilled"} 0
kube_pod_container_status_terminated_reason{container="container7",namespace="ns6",pod="pod6",reason="Completed"} 0
kube_pod_container_status_terminated_reason{container="container7",namespace="ns6",pod="pod6",reason="ContainerCannotRun"} 0
kube_pod_container_status_terminated_reason{container="container7",namespace="ns6",pod="pod6",reason="Error"} 0
kube_pod_container_status_terminated_reason{container="container7",namespace="ns6",pod="pod6",reason="OOMKilled"} 0
kube_pod_container_status_waiting{container="container1",namespace="ns1",pod="pod1"} 0
kube_pod_container_status_waiting{container="container2",namespace="ns2",pod="pod2"} 0
kube_pod_container_status_waiting{container="container3",namespace="ns2",pod="pod2"} 1
kube_pod_container_status_waiting{container="container4",namespace="ns3",pod="pod3"} 1
kube_pod_container_status_waiting{container="container5",namespace="ns4",pod="pod4"} 1
kube_pod_container_status_waiting{container="container6",namespace="ns5",pod="pod5"} 1
kube_pod_container_status_waiting{container="container7",namespace="ns6",pod="pod6"} 0
kube_pod_container_status_waiting_reason{container="container1",namespace="ns1",pod="pod1",reason="ContainerCreating"} 0
kube_pod_container_status_waiting_reason{container="container1",namespace="ns1",pod="pod1",reason="ImagePullBackOff"} 0
kube_pod_container_status_waiting_reason{container="container1",namespace="ns1",pod="pod1",reason="CrashLoopBackOff"} 0
Expand All @@ -394,13 +424,46 @@ func TestPodCollector(t *testing.T) {
kube_pod_container_status_waiting_reason{container="container3",namespace="ns2",pod="pod2",reason="ImagePullBackOff"} 0
kube_pod_container_status_waiting_reason{container="container3",namespace="ns2",pod="pod2",reason="CrashLoopBackOff"} 0
kube_pod_container_status_waiting_reason{container="container3",namespace="ns2",pod="pod2",reason="ErrImagePull"} 0
kube_pod_container_status_waiting_reason{container="container7",namespace="ns6",pod="pod6",reason="ContainerCreating"} 0
kube_pod_container_status_waiting_reason{container="container7",namespace="ns6",pod="pod6",reason="ImagePullBackOff"} 0
kube_pod_container_status_waiting_reason{container="container7",namespace="ns6",pod="pod6",reason="CrashLoopBackOff"} 0
kube_pod_container_status_waiting_reason{container="container7",namespace="ns6",pod="pod6",reason="ErrImagePull"} 0
kube_pod_container_status_last_terminated_reason{container="container1",namespace="ns1",pod="pod1",reason="Completed"} 0
kube_pod_container_status_last_terminated_reason{container="container1",namespace="ns1",pod="pod1",reason="ContainerCannotRun"} 0
kube_pod_container_status_last_terminated_reason{container="container1",namespace="ns1",pod="pod1",reason="Error"} 0
kube_pod_container_status_last_terminated_reason{container="container1",namespace="ns1",pod="pod1",reason="OOMKilled"} 0
kube_pod_container_status_last_terminated_reason{container="container4",namespace="ns3",pod="pod3",reason="Completed"} 0
kube_pod_container_status_last_terminated_reason{container="container4",namespace="ns3",pod="pod3",reason="ContainerCannotRun"} 0
kube_pod_container_status_last_terminated_reason{container="container4",namespace="ns3",pod="pod3",reason="Error"} 0
kube_pod_container_status_last_terminated_reason{container="container4",namespace="ns3",pod="pod3",reason="OOMKilled"} 0
kube_pod_container_status_last_terminated_reason{container="container5",namespace="ns4",pod="pod4",reason="Completed"} 0
kube_pod_container_status_last_terminated_reason{container="container5",namespace="ns4",pod="pod4",reason="ContainerCannotRun"} 0
kube_pod_container_status_last_terminated_reason{container="container5",namespace="ns4",pod="pod4",reason="Error"} 0
kube_pod_container_status_last_terminated_reason{container="container5",namespace="ns4",pod="pod4",reason="OOMKilled"} 0
kube_pod_container_status_last_terminated_reason{container="container6",namespace="ns5",pod="pod5",reason="Completed"} 0
kube_pod_container_status_last_terminated_reason{container="container6",namespace="ns5",pod="pod5",reason="ContainerCannotRun"} 0
kube_pod_container_status_last_terminated_reason{container="container6",namespace="ns5",pod="pod5",reason="Error"} 0
kube_pod_container_status_last_terminated_reason{container="container6",namespace="ns5",pod="pod5",reason="OOMKilled"} 0
kube_pod_container_status_last_terminated_reason{container="container2",namespace="ns2",pod="pod2",reason="Completed"} 0
kube_pod_container_status_last_terminated_reason{container="container2",namespace="ns2",pod="pod2",reason="ContainerCannotRun"} 0
kube_pod_container_status_last_terminated_reason{container="container2",namespace="ns2",pod="pod2",reason="Error"} 0
kube_pod_container_status_last_terminated_reason{container="container2",namespace="ns2",pod="pod2",reason="OOMKilled"} 0
kube_pod_container_status_last_terminated_reason{container="container3",namespace="ns2",pod="pod2",reason="Completed"} 0
kube_pod_container_status_last_terminated_reason{container="container3",namespace="ns2",pod="pod2",reason="ContainerCannotRun"} 0
kube_pod_container_status_last_terminated_reason{container="container3",namespace="ns2",pod="pod2",reason="Error"} 0
kube_pod_container_status_last_terminated_reason{container="container3",namespace="ns2",pod="pod2",reason="OOMKilled"} 0
kube_pod_container_status_last_terminated_reason{container="container7",namespace="ns6",pod="pod6",reason="Completed"} 0
kube_pod_container_status_last_terminated_reason{container="container7",namespace="ns6",pod="pod6",reason="ContainerCannotRun"} 0
kube_pod_container_status_last_terminated_reason{container="container7",namespace="ns6",pod="pod6",reason="Error"} 0
kube_pod_container_status_last_terminated_reason{container="container7",namespace="ns6",pod="pod6",reason="OOMKilled"} 1
`,
metrics: []string{
"kube_pod_container_status_running",
"kube_pod_container_status_waiting",
"kube_pod_container_status_waiting_reason",
"kube_pod_container_status_terminated",
"kube_pod_container_status_terminated_reason",
"kube_pod_container_status_last_terminated_reason",
},
}, {
pods: []v1.Pod{
Expand Down

0 comments on commit 5119063

Please sign in to comment.