Support pod_ready_time and pod_container_ready_time

Co-authored-by: Szymon Grzemski <sz.grzemski@gmail.com>
Signed-off-by: Lan Liang <gcslyp@gmail.com>

docs/pod-metrics.md

@@ -12,6 +12,8 @@
 | kube_pod_nodeselectors| Gauge | Describes the Pod nodeSelectors | |  `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `nodeselector_NODE_SELECTOR`=&lt;NODE_SELECTOR&gt; <br> `uid`=&lt;pod-uid&gt; | EXPERIMENTAL | Opt-in |
 | kube_pod_status_phase | Gauge | The pods current phase | | `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `phase`=&lt;Pending\|Running\|Succeeded\|Failed\|Unknown&gt; <br> `uid`=&lt;pod-uid&gt; | STABLE | - |
 | kube_pod_status_ready | Gauge | Describes whether the pod is ready to serve requests | | `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `condition`=&lt;true\|false\|unknown&gt; <br> `uid`=&lt;pod-uid&gt; | STABLE | - |
+| kube_pod_status_ready_time | Gauge | Time when pod passed readiness probes. | seconds | `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `uid`=&lt;pod-uid&gt; | STABLE |
+| kube_pod_status_container_ready_time | Gauge | Time when the container of the pod entered Ready state. | seconds | `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `uid`=&lt;pod-uid&gt; | STABLE |
 | kube_pod_status_scheduled | Gauge | Describes the status of the scheduling process for the pod | |`pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `condition`=&lt;true\|false\|unknown&gt; <br> `uid`=&lt;pod-uid&gt; | STABLE | - |
 | kube_pod_container_info | Gauge | Information about a container in a pod | | `container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `image`=&lt;image-name&gt; <br> `image_id`=&lt;image-id&gt; <br> `image_spec`=&lt;image-spec&gt; <br> `container_id`=&lt;containerid&gt; <br> `uid`=&lt;pod-uid&gt; | STABLE | - |
 | kube_pod_container_status_waiting | Gauge | Describes whether the container is currently in waiting state | | `container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `uid`=&lt;pod-uid&gt; | STABLE | - |

internal/store/pod.go

@@ -83,6 +83,8 @@ func podMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generat
 		createPodStartTimeFamilyGenerator(),
 		createPodStatusPhaseFamilyGenerator(),
 		createPodStatusReadyFamilyGenerator(),
+		createPodStatusReadyTimeFamilyGenerator(),
+		createPodStatusContainerReadyTimeFamilyGenerator(),
 		createPodStatusReasonFamilyGenerator(),
 		createPodStatusScheduledFamilyGenerator(),
 		createPodStatusScheduledTimeFamilyGenerator(),
@@ -1317,6 +1319,58 @@ func createPodStatusPhaseFamilyGenerator() generator.FamilyGenerator {
 	)
 }
 
+func createPodStatusContainerReadyTimeFamilyGenerator() generator.FamilyGenerator {
+	return *generator.NewFamilyGenerator(
+		"kube_pod_status_container_ready_time",
+		"Readiness achieved time in unix timestamp for a pod containers.",
+		metric.Gauge,
+		"",
+		wrapPodFunc(func(p *v1.Pod) *metric.Family {
+			ms := []*metric.Metric{}
+
+			for _, c := range p.Status.Conditions {
+				if c.Type == v1.ContainersReady {
+					ms = append(ms, &metric.Metric{
+						LabelKeys:   []string{},
+						LabelValues: []string{},
+						Value:       float64((c.LastTransitionTime).Unix()),
+					})
+				}
+			}
+
+			return &metric.Family{
+				Metrics: ms,
+			}
+		}),
+	)
+}
+
+func createPodStatusReadyTimeFamilyGenerator() generator.FamilyGenerator {
+	return *generator.NewFamilyGenerator(
+		"kube_pod_status_ready_time",
+		"Readiness achieved time in unix timestamp for a pod.",
+		metric.Gauge,
+		"",
+		wrapPodFunc(func(p *v1.Pod) *metric.Family {
+			ms := []*metric.Metric{}
+
+			for _, c := range p.Status.Conditions {
+				if c.Type == v1.PodReady {
+					ms = append(ms, &metric.Metric{
+						LabelKeys:   []string{},
+						LabelValues: []string{},
+						Value:       float64((c.LastTransitionTime).Unix()),
+					})
+				}
+			}
+
+			return &metric.Family{
+				Metrics: ms,
+			}
+		}),
+	)
+}
+
 func createPodStatusReadyFamilyGenerator() generator.FamilyGenerator {
 	return *generator.NewFamilyGeneratorWithStability(
 		"kube_pod_status_ready",

internal/store/pod_test.go

@@ -1426,13 +1426,19 @@ func TestPodStore(t *testing.T) {
 						{
 							Type:   v1.PodReady,
 							Status: v1.ConditionTrue,
+							LastTransitionTime: metav1.Time{
+								Time: time.Unix(1501666018, 0),
+							},
 						},
 					},
 				},
 			},
 			Want: `
 				# HELP kube_pod_status_ready [STABLE] Describes whether the pod is ready to serve requests.
+				# HELP kube_pod_status_ready_time Readiness achieved time in unix timestamp for a pod.
 				# TYPE kube_pod_status_ready gauge
+				# TYPE kube_pod_status_ready_time gauge
+				kube_pod_status_ready_time{namespace="ns1",pod="pod1",uid="uid1"} 1.501666018e+09
 				kube_pod_status_ready{condition="false",namespace="ns1",pod="pod1",uid="uid1"} 0
 				kube_pod_status_ready{condition="true",namespace="ns1",pod="pod1",uid="uid1"} 1
 				kube_pod_status_ready{condition="unknown",namespace="ns1",pod="pod1",uid="uid1"} 0
@@ -1451,13 +1457,19 @@ func TestPodStore(t *testing.T) {
 						{
 							Type:   v1.PodReady,
 							Status: v1.ConditionFalse,
+							LastTransitionTime: metav1.Time{
+								Time: time.Unix(1501666018, 0),
+							},
 						},
 					},
 				},
 			},
 			Want: `
 				# HELP kube_pod_status_ready [STABLE] Describes whether the pod is ready to serve requests.
+				# HELP kube_pod_status_ready_time Readiness achieved time in unix timestamp for a pod.
 				# TYPE kube_pod_status_ready gauge
+				# TYPE kube_pod_status_ready_time gauge
+				kube_pod_status_ready_time{namespace="ns2",pod="pod2",uid="uid2"} 1.501666018e+09
 				kube_pod_status_ready{condition="false",namespace="ns2",pod="pod2",uid="uid2"} 1
 				kube_pod_status_ready{condition="true",namespace="ns2",pod="pod2",uid="uid2"} 0
 				kube_pod_status_ready{condition="unknown",namespace="ns2",pod="pod2",uid="uid2"} 0
@@ -2079,7 +2091,7 @@ func BenchmarkPodStore(b *testing.B) {
 		},
 	}
 
-	expectedFamilies := 47
+	expectedFamilies := 49
 	for n := 0; n < b.N; n++ {
 		families := f(pod)
 		if len(families) != expectedFamilies {

pkg/app/server_test.go

@@ -229,7 +229,9 @@ func TestFullScrapeCycle(t *testing.T) {
 # HELP kube_pod_spec_volumes_persistentvolumeclaims_info [STABLE] Information about persistentvolumeclaim volumes in a pod.
 # HELP kube_pod_spec_volumes_persistentvolumeclaims_readonly [STABLE] Describes whether a persistentvolumeclaim is mounted read only.
 # HELP kube_pod_start_time [STABLE] Start time in unix timestamp for a pod.
+# HELP kube_pod_status_container_ready_time Readiness achieved time in unix timestamp for a pod containers.
 # HELP kube_pod_status_phase [STABLE] The pods current phase.
+# HELP kube_pod_status_ready_time Readiness achieved time in unix timestamp for a pod.
 # HELP kube_pod_status_ready [STABLE] Describes whether the pod is ready to serve requests.
 # HELP kube_pod_status_reason The pod status reasons
 # HELP kube_pod_status_scheduled [STABLE] Describes the status of the scheduling process for the pod.
@@ -275,8 +277,10 @@ func TestFullScrapeCycle(t *testing.T) {
 # TYPE kube_pod_spec_volumes_persistentvolumeclaims_info gauge
 # TYPE kube_pod_spec_volumes_persistentvolumeclaims_readonly gauge
 # TYPE kube_pod_start_time gauge
+# TYPE kube_pod_status_container_ready_time gauge
 # TYPE kube_pod_status_phase gauge
 # TYPE kube_pod_status_ready gauge
+# TYPE kube_pod_status_ready_time gauge
 # TYPE kube_pod_status_reason gauge
 # TYPE kube_pod_status_scheduled gauge
 # TYPE kube_pod_status_scheduled_time gauge