diff --git a/docs/pod-metrics.md b/docs/pod-metrics.md index 74c0bf2d94..e6b8e96743 100644 --- a/docs/pod-metrics.md +++ b/docs/pod-metrics.md @@ -24,6 +24,8 @@ | kube_pod_container_status_last_terminated_reason | Gauge | Describes the last reason the container was in terminated state | |`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`reason`=<last-terminated-reason>
`uid`=<pod-uid> | EXPERIMENTAL | - | | kube_pod_container_status_last_terminated_exitcode | Gauge | Describes the exit code for the last container in terminated state. | | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | EXPERIMENTAL | - | | kube_pod_container_status_ready | Gauge | Describes whether the containers readiness check succeeded | |`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_status_ready_time | Gauge | Time when pod passed readiness probes. | seconds | `pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | EXPERIMENTAL | +| kube_pod_status_container_ready_time | Gauge | Time when the container of the pod entered Ready state. | seconds | `pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | EXPERIMENTAL | | kube_pod_container_status_restarts_total | Counter | The number of container restarts per container | | `container`=<container-name>
`namespace`=<pod-namespace>
`pod`=<pod-name>
`uid`=<pod-uid> | STABLE | - | | kube_pod_container_resource_requests | Gauge | The number of requested request resource by a container. It is recommended to use the `kube_pod_resource_requests` metric exposed by kube-scheduler instead, as it is more precise. | `cpu`=<core>
`memory`=<bytes> |`resource`=<resource-name>
`unit`=<resource-unit>
`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`node`=< node-name>
`uid`=<pod-uid> | EXPERIMENTAL | - | | kube_pod_container_resource_limits | Gauge | The number of requested limit resource by a container. It is recommended to use the `kube_pod_resource_limits` metric exposed by kube-scheduler instead, as it is more precise. | `cpu`=<core>
`memory`=<bytes> |`resource`=<resource-name>
`unit`=<resource-unit>
`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`node`=< node-name>
`uid`=<pod-uid> | EXPERIMENTAL | - | diff --git a/internal/store/pod.go b/internal/store/pod.go index 9d7fbc4f3d..b53f3f9386 100644 --- a/internal/store/pod.go +++ b/internal/store/pod.go @@ -84,6 +84,8 @@ func podMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generat createPodStatusPhaseFamilyGenerator(), createPodStatusQosClassFamilyGenerator(), createPodStatusReadyFamilyGenerator(), + createPodStatusReadyTimeFamilyGenerator(), + createPodStatusContainerReadyTimeFamilyGenerator(), createPodStatusReasonFamilyGenerator(), createPodStatusScheduledFamilyGenerator(), createPodStatusScheduledTimeFamilyGenerator(), @@ -1318,6 +1320,60 @@ func createPodStatusPhaseFamilyGenerator() generator.FamilyGenerator { ) } +func createPodStatusContainerReadyTimeFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_status_container_ready_time", + "Readiness achieved time in unix timestamp for a pod containers.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + ms := []*metric.Metric{} + + for _, c := range p.Status.Conditions { + if c.Type == v1.ContainersReady { + ms = append(ms, &metric.Metric{ + LabelKeys: []string{}, + LabelValues: []string{}, + Value: float64((c.LastTransitionTime).Unix()), + }) + } + } + + return &metric.Family{ + Metrics: ms, + } + }), + ) +} + +func createPodStatusReadyTimeFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_status_ready_time", + "Readiness achieved time in unix timestamp for a pod.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + ms := []*metric.Metric{} + + for _, c := range p.Status.Conditions { + if c.Type == v1.PodReady { + ms = append(ms, &metric.Metric{ + LabelKeys: []string{}, + LabelValues: []string{}, + Value: float64((c.LastTransitionTime).Unix()), + }) + } + } + + return &metric.Family{ + Metrics: ms, + } + }), + ) +} + func createPodStatusQosClassFamilyGenerator() generator.FamilyGenerator { return *generator.NewFamilyGeneratorWithStability( "kube_pod_status_qos_class", diff --git a/internal/store/pod_test.go b/internal/store/pod_test.go index 1f42d0b7c4..35fc0fd996 100644 --- a/internal/store/pod_test.go +++ b/internal/store/pod_test.go @@ -1434,6 +1434,32 @@ func TestPodStore(t *testing.T) { `, MetricNames: []string{"kube_pod_status_reason"}, }, + { + Obj: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "ns1", + UID: "uid1", + }, + Status: v1.PodStatus{ + Conditions: []v1.PodCondition{ + { + Type: v1.ContainersReady, + Status: v1.ConditionTrue, + LastTransitionTime: metav1.Time{ + Time: time.Unix(1501666018, 0), + }, + }, + }, + }, + }, + Want: ` + # HELP kube_pod_status_container_ready_time Readiness achieved time in unix timestamp for a pod containers. + # TYPE kube_pod_status_container_ready_time gauge + kube_pod_status_container_ready_time{namespace="ns1",pod="pod1",uid="uid1"} 1.501666018e+09 + `, + MetricNames: []string{"kube_pod_status_container_ready_time"}, + }, { Obj: &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ @@ -1446,18 +1472,24 @@ func TestPodStore(t *testing.T) { { Type: v1.PodReady, Status: v1.ConditionTrue, + LastTransitionTime: metav1.Time{ + Time: time.Unix(1501666018, 0), + }, }, }, }, }, Want: ` # HELP kube_pod_status_ready [STABLE] Describes whether the pod is ready to serve requests. + # HELP kube_pod_status_ready_time Readiness achieved time in unix timestamp for a pod. # TYPE kube_pod_status_ready gauge + # TYPE kube_pod_status_ready_time gauge + kube_pod_status_ready_time{namespace="ns1",pod="pod1",uid="uid1"} 1.501666018e+09 kube_pod_status_ready{condition="false",namespace="ns1",pod="pod1",uid="uid1"} 0 kube_pod_status_ready{condition="true",namespace="ns1",pod="pod1",uid="uid1"} 1 kube_pod_status_ready{condition="unknown",namespace="ns1",pod="pod1",uid="uid1"} 0 `, - MetricNames: []string{"kube_pod_status_ready"}, + MetricNames: []string{"kube_pod_status_ready_time", "kube_pod_status_ready"}, }, { Obj: &v1.Pod{ @@ -1471,18 +1503,24 @@ func TestPodStore(t *testing.T) { { Type: v1.PodReady, Status: v1.ConditionFalse, + LastTransitionTime: metav1.Time{ + Time: time.Unix(1501666018, 0), + }, }, }, }, }, Want: ` # HELP kube_pod_status_ready [STABLE] Describes whether the pod is ready to serve requests. + # HELP kube_pod_status_ready_time Readiness achieved time in unix timestamp for a pod. # TYPE kube_pod_status_ready gauge + # TYPE kube_pod_status_ready_time gauge + kube_pod_status_ready_time{namespace="ns2",pod="pod2",uid="uid2"} 1.501666018e+09 kube_pod_status_ready{condition="false",namespace="ns2",pod="pod2",uid="uid2"} 1 kube_pod_status_ready{condition="true",namespace="ns2",pod="pod2",uid="uid2"} 0 kube_pod_status_ready{condition="unknown",namespace="ns2",pod="pod2",uid="uid2"} 0 `, - MetricNames: []string{"kube_pod_status_ready"}, + MetricNames: []string{"kube_pod_status_ready_time", "kube_pod_status_ready"}, }, { Obj: &v1.Pod{ @@ -2099,7 +2137,7 @@ func BenchmarkPodStore(b *testing.B) { }, } - expectedFamilies := 48 + expectedFamilies := 50 for n := 0; n < b.N; n++ { families := f(pod) if len(families) != expectedFamilies { diff --git a/pkg/app/server_test.go b/pkg/app/server_test.go index aefeef891a..d1dd6a3068 100644 --- a/pkg/app/server_test.go +++ b/pkg/app/server_test.go @@ -238,8 +238,10 @@ func TestFullScrapeCycle(t *testing.T) { # HELP kube_pod_spec_volumes_persistentvolumeclaims_info [STABLE] Information about persistentvolumeclaim volumes in a pod. # HELP kube_pod_spec_volumes_persistentvolumeclaims_readonly [STABLE] Describes whether a persistentvolumeclaim is mounted read only. # HELP kube_pod_start_time [STABLE] Start time in unix timestamp for a pod. +# HELP kube_pod_status_container_ready_time Readiness achieved time in unix timestamp for a pod containers. # HELP kube_pod_status_qos_class The pods current qosClass. # HELP kube_pod_status_phase [STABLE] The pods current phase. +# HELP kube_pod_status_ready_time Readiness achieved time in unix timestamp for a pod. # HELP kube_pod_status_ready [STABLE] Describes whether the pod is ready to serve requests. # HELP kube_pod_status_reason The pod status reasons # HELP kube_pod_status_scheduled [STABLE] Describes the status of the scheduling process for the pod. @@ -285,9 +287,11 @@ func TestFullScrapeCycle(t *testing.T) { # TYPE kube_pod_spec_volumes_persistentvolumeclaims_info gauge # TYPE kube_pod_spec_volumes_persistentvolumeclaims_readonly gauge # TYPE kube_pod_start_time gauge +# TYPE kube_pod_status_container_ready_time gauge # TYPE kube_pod_status_phase gauge # TYPE kube_pod_status_qos_class gauge # TYPE kube_pod_status_ready gauge +# TYPE kube_pod_status_ready_time gauge # TYPE kube_pod_status_reason gauge # TYPE kube_pod_status_scheduled gauge # TYPE kube_pod_status_scheduled_time gauge