Merge pull request #1519 from harjas27/container_started_time

capture start time for containers in terminated state
kubernetes · Aug 9, 2021 · 16e8f54 · 16e8f54
2 parents 17204a5 + 6324b16
commit 16e8f54
Show file tree

Hide file tree

Showing 2 changed files with 63 additions and 0 deletions.
diff --git a/internal/store/pod.go b/internal/store/pod.go
@@ -292,6 +292,12 @@ func createPodContainerStateStartedFamilyGenerator() generator.FamilyGenerator {
 						LabelValues: []string{cs.Name},
 						Value:       float64((cs.State.Running.StartedAt).Unix()),
 					})
+				} else if cs.State.Terminated != nil {
+					ms = append(ms, &metric.Metric{
+						LabelKeys:   []string{"container"},
+						LabelValues: []string{cs.Name},
+						Value:       float64((cs.State.Terminated.StartedAt).Unix()),
+					})
 				}
 			}
 

diff --git a/internal/store/pod_test.go b/internal/store/pod_test.go
@@ -365,6 +365,56 @@ func TestPodStore(t *testing.T) {
 				"kube_pod_init_container_status_terminated_reason",
 			},
 		},
+		{
+			Obj: &v1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "pod1",
+					Namespace: "ns1",
+					UID:       "uid1",
+				},
+				Status: v1.PodStatus{
+					ContainerStatuses: []v1.ContainerStatus{
+						{
+							Name: "container1",
+							State: v1.ContainerState{
+								Terminated: &v1.ContainerStateTerminated{
+									StartedAt: metav1.Time{
+										Time: time.Unix(1501777018, 0),
+									},
+									Reason: "Completed",
+								},
+							},
+						},
+					},
+				},
+			},
+			Want: `
+				# HELP kube_pod_container_status_running Describes whether the container is currently in running state.
+				# HELP kube_pod_container_state_started Start time in unix timestamp for a pod container.
+				# HELP kube_pod_container_status_terminated Describes whether the container is currently in terminated state.
+				# HELP kube_pod_container_status_terminated_reason Describes the reason the container is currently in terminated state.
+				# HELP kube_pod_container_status_waiting Describes whether the container is currently in waiting state.
+				# HELP kube_pod_container_status_waiting_reason Describes the reason the container is currently in waiting state.
+				# TYPE kube_pod_container_status_running gauge
+				# TYPE kube_pod_container_state_started gauge
+				# TYPE kube_pod_container_status_terminated gauge
+				# TYPE kube_pod_container_status_terminated_reason gauge
+				# TYPE kube_pod_container_status_waiting gauge
+				# TYPE kube_pod_container_status_waiting_reason gauge
+				kube_pod_container_state_started{container="container1",namespace="ns1",pod="pod1",uid="uid1"} 1.501777018e+09
+				kube_pod_container_status_running{container="container1",namespace="ns1",pod="pod1",uid="uid1"} 0
+				kube_pod_container_status_terminated_reason{container="container1",namespace="ns1",pod="pod1",reason="Completed",uid="uid1"} 1
+				kube_pod_container_status_terminated{container="container1",namespace="ns1",pod="pod1",uid="uid1"} 1
+				kube_pod_container_status_waiting{container="container1",namespace="ns1",pod="pod1",uid="uid1"} 0
+			`,
+			MetricNames: []string{
+				"kube_pod_container_status_running",
+				"kube_pod_container_state_started",
+				"kube_pod_container_status_waiting",
+				"kube_pod_container_status_terminated",
+				"kube_pod_container_status_terminated_reason",
+			},
+		},
 		{
 			Obj: &v1.Pod{
 				ObjectMeta: metav1.ObjectMeta{
@@ -378,6 +428,9 @@ func TestPodStore(t *testing.T) {
 							Name: "container2",
 							State: v1.ContainerState{
 								Terminated: &v1.ContainerStateTerminated{
+									StartedAt: metav1.Time{
+										Time: time.Unix(1501777018, 0),
+									},
 									Reason: "OOMKilled",
 								},
 							},
@@ -395,17 +448,20 @@ func TestPodStore(t *testing.T) {
 			},
 			Want: `
 				# HELP kube_pod_container_status_running Describes whether the container is currently in running state.
+				# HELP kube_pod_container_state_started Start time in unix timestamp for a pod container.
 				# HELP kube_pod_container_status_terminated Describes whether the container is currently in terminated state.
 				# HELP kube_pod_container_status_terminated_reason Describes the reason the container is currently in terminated state.
 				# HELP kube_pod_container_status_waiting Describes whether the container is currently in waiting state.
 				# HELP kube_pod_container_status_waiting_reason Describes the reason the container is currently in waiting state.
 				# TYPE kube_pod_container_status_running gauge
+				# TYPE kube_pod_container_state_started gauge
 				# TYPE kube_pod_container_status_terminated gauge
 				# TYPE kube_pod_container_status_terminated_reason gauge
 				# TYPE kube_pod_container_status_waiting gauge
 				# TYPE kube_pod_container_status_waiting_reason gauge
 				kube_pod_container_status_running{container="container2",namespace="ns2",pod="pod2",uid="uid2"} 0
 				kube_pod_container_status_running{container="container3",namespace="ns2",pod="pod2",uid="uid2"} 0
+				kube_pod_container_state_started{container="container2",namespace="ns2",pod="pod2",uid="uid2"} 1.501777018e+09
 				kube_pod_container_status_terminated_reason{container="container2",namespace="ns2",pod="pod2",reason="OOMKilled",uid="uid2"} 1
 				kube_pod_container_status_terminated{container="container2",namespace="ns2",pod="pod2",uid="uid2"} 1
 				kube_pod_container_status_terminated{container="container3",namespace="ns2",pod="pod2",uid="uid2"} 0
@@ -415,6 +471,7 @@ func TestPodStore(t *testing.T) {
 `,
 			MetricNames: []string{
 				"kube_pod_container_status_running",
+				"kube_pod_container_state_started",
 				"kube_pod_container_status_waiting",
 				"kube_pod_container_status_terminated",
 				"kube_pod_container_status_terminated_reason",