From f36b8e612db5c3dfcafed75732cc6bcf05a79e7d Mon Sep 17 00:00:00 2001 From: Lan Liang Date: Tue, 18 Oct 2022 18:00:29 +0800 Subject: [PATCH] Support pod_ready_time and pod_container_ready_time Co-authored-by: Szymon Grzemski Signed-off-by: Lan Liang --- docs/pod-metrics.md | 2 ++ internal/store/pod.go | 54 ++++++++++++++++++++++++++++++++++++++ internal/store/pod_test.go | 14 +++++++++- pkg/app/server_test.go | 4 +++ 4 files changed, 73 insertions(+), 1 deletion(-) diff --git a/docs/pod-metrics.md b/docs/pod-metrics.md index fcea2027d6..8aa10d900b 100644 --- a/docs/pod-metrics.md +++ b/docs/pod-metrics.md @@ -12,6 +12,8 @@ | kube_pod_nodeselectors| Gauge | Describes the Pod nodeSelectors | | `pod`=<pod-name>
`namespace`=<pod-namespace>
`nodeselector_NODE_SELECTOR`=<NODE_SELECTOR>
`uid`=<pod-uid> | EXPERIMENTAL | Opt-in | | kube_pod_status_phase | Gauge | The pods current phase | | `pod`=<pod-name>
`namespace`=<pod-namespace>
`phase`=<Pending\|Running\|Succeeded\|Failed\|Unknown>
`uid`=<pod-uid> | STABLE | - | | kube_pod_status_ready | Gauge | Describes whether the pod is ready to serve requests | | `pod`=<pod-name>
`namespace`=<pod-namespace>
`condition`=<true\|false\|unknown>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_status_ready_time | Gauge | Time when pod passed readiness probes. | seconds | `pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | +| kube_pod_status_container_ready_time | Gauge | Time when the container of the pod entered Ready state. | seconds | `pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | | kube_pod_status_scheduled | Gauge | Describes the status of the scheduling process for the pod | |`pod`=<pod-name>
`namespace`=<pod-namespace>
`condition`=<true\|false\|unknown>
`uid`=<pod-uid> | STABLE | - | | kube_pod_container_info | Gauge | Information about a container in a pod | | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`image`=<image-name>
`image_id`=<image-id>
`image_spec`=<image-spec>
`container_id`=<containerid>
`uid`=<pod-uid> | STABLE | - | | kube_pod_container_status_waiting | Gauge | Describes whether the container is currently in waiting state | | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | - | diff --git a/internal/store/pod.go b/internal/store/pod.go index 3d09fc137c..90bb11d3c0 100644 --- a/internal/store/pod.go +++ b/internal/store/pod.go @@ -83,6 +83,8 @@ func podMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generat createPodStartTimeFamilyGenerator(), createPodStatusPhaseFamilyGenerator(), createPodStatusReadyFamilyGenerator(), + createPodStatusReadyTimeFamilyGenerator(), + createPodStatusContainerReadyTimeFamilyGenerator(), createPodStatusReasonFamilyGenerator(), createPodStatusScheduledFamilyGenerator(), createPodStatusScheduledTimeFamilyGenerator(), @@ -1317,6 +1319,58 @@ func createPodStatusPhaseFamilyGenerator() generator.FamilyGenerator { ) } +func createPodStatusContainerReadyTimeFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGenerator( + "kube_pod_status_container_ready_time", + "Readiness achieved time in unix timestamp for a pod containers.", + metric.Gauge, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + ms := []*metric.Metric{} + + for _, c := range p.Status.Conditions { + if c.Type == v1.ContainersReady { + ms = append(ms, &metric.Metric{ + LabelKeys: []string{}, + LabelValues: []string{}, + Value: float64((c.LastTransitionTime).Unix()), + }) + } + } + + return &metric.Family{ + Metrics: ms, + } + }), + ) +} + +func createPodStatusReadyTimeFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGenerator( + "kube_pod_status_ready_time", + "Readiness achieved time in unix timestamp for a pod.", + metric.Gauge, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + ms := []*metric.Metric{} + + for _, c := range p.Status.Conditions { + if c.Type == v1.PodReady { + ms = append(ms, &metric.Metric{ + LabelKeys: []string{}, + LabelValues: []string{}, + Value: float64((c.LastTransitionTime).Unix()), + }) + } + } + + return &metric.Family{ + Metrics: ms, + } + }), + ) +} + func createPodStatusReadyFamilyGenerator() generator.FamilyGenerator { return *generator.NewFamilyGeneratorWithStability( "kube_pod_status_ready", diff --git a/internal/store/pod_test.go b/internal/store/pod_test.go index e94ecc3190..dadffc79b7 100644 --- a/internal/store/pod_test.go +++ b/internal/store/pod_test.go @@ -1426,13 +1426,19 @@ func TestPodStore(t *testing.T) { { Type: v1.PodReady, Status: v1.ConditionTrue, + LastTransitionTime: metav1.Time{ + Time: time.Unix(1501666018, 0), + }, }, }, }, }, Want: ` # HELP kube_pod_status_ready [STABLE] Describes whether the pod is ready to serve requests. + # HELP kube_pod_status_ready_time Readiness achieved time in unix timestamp for a pod. # TYPE kube_pod_status_ready gauge + # TYPE kube_pod_status_ready_time gauge + kube_pod_status_ready_time{namespace="ns1",pod="pod1",uid="uid1"} 1.501666018e+09 kube_pod_status_ready{condition="false",namespace="ns1",pod="pod1",uid="uid1"} 0 kube_pod_status_ready{condition="true",namespace="ns1",pod="pod1",uid="uid1"} 1 kube_pod_status_ready{condition="unknown",namespace="ns1",pod="pod1",uid="uid1"} 0 @@ -1451,13 +1457,19 @@ func TestPodStore(t *testing.T) { { Type: v1.PodReady, Status: v1.ConditionFalse, + LastTransitionTime: metav1.Time{ + Time: time.Unix(1501666018, 0), + }, }, }, }, }, Want: ` # HELP kube_pod_status_ready [STABLE] Describes whether the pod is ready to serve requests. + # HELP kube_pod_status_ready_time Readiness achieved time in unix timestamp for a pod. # TYPE kube_pod_status_ready gauge + # TYPE kube_pod_status_ready_time gauge + kube_pod_status_ready_time{namespace="ns2",pod="pod2",uid="uid2"} 1.501666018e+09 kube_pod_status_ready{condition="false",namespace="ns2",pod="pod2",uid="uid2"} 1 kube_pod_status_ready{condition="true",namespace="ns2",pod="pod2",uid="uid2"} 0 kube_pod_status_ready{condition="unknown",namespace="ns2",pod="pod2",uid="uid2"} 0 @@ -2079,7 +2091,7 @@ func BenchmarkPodStore(b *testing.B) { }, } - expectedFamilies := 47 + expectedFamilies := 49 for n := 0; n < b.N; n++ { families := f(pod) if len(families) != expectedFamilies { diff --git a/pkg/app/server_test.go b/pkg/app/server_test.go index cba7e27fe3..345a9dfc87 100644 --- a/pkg/app/server_test.go +++ b/pkg/app/server_test.go @@ -229,7 +229,9 @@ func TestFullScrapeCycle(t *testing.T) { # HELP kube_pod_spec_volumes_persistentvolumeclaims_info [STABLE] Information about persistentvolumeclaim volumes in a pod. # HELP kube_pod_spec_volumes_persistentvolumeclaims_readonly [STABLE] Describes whether a persistentvolumeclaim is mounted read only. # HELP kube_pod_start_time [STABLE] Start time in unix timestamp for a pod. +# HELP kube_pod_status_container_ready_time Readiness achieved time in unix timestamp for a pod containers. # HELP kube_pod_status_phase [STABLE] The pods current phase. +# HELP kube_pod_status_ready_time Readiness achieved time in unix timestamp for a pod. # HELP kube_pod_status_ready [STABLE] Describes whether the pod is ready to serve requests. # HELP kube_pod_status_reason The pod status reasons # HELP kube_pod_status_scheduled [STABLE] Describes the status of the scheduling process for the pod. @@ -275,8 +277,10 @@ func TestFullScrapeCycle(t *testing.T) { # TYPE kube_pod_spec_volumes_persistentvolumeclaims_info gauge # TYPE kube_pod_spec_volumes_persistentvolumeclaims_readonly gauge # TYPE kube_pod_start_time gauge +# TYPE kube_pod_status_container_ready_time gauge # TYPE kube_pod_status_phase gauge # TYPE kube_pod_status_ready gauge +# TYPE kube_pod_status_ready_time gauge # TYPE kube_pod_status_reason gauge # TYPE kube_pod_status_scheduled gauge # TYPE kube_pod_status_scheduled_time gauge