From 4d8c58ad0075e4e56412b1595ab1ee452279135f Mon Sep 17 00:00:00 2001 From: Lan Liang Date: Tue, 18 Oct 2022 18:00:29 +0800 Subject: [PATCH 1/4] Support pod_ready_time and pod_container_ready_time Co-authored-by: Szymon Grzemski Signed-off-by: Lan Liang --- docs/pod-metrics.md | 54 ++++++++++++++++++++------------------ internal/store/pod.go | 54 ++++++++++++++++++++++++++++++++++++++ internal/store/pod_test.go | 14 +++++++++- pkg/app/server_test.go | 4 +++ 4 files changed, 99 insertions(+), 27 deletions(-) diff --git a/docs/pod-metrics.md b/docs/pod-metrics.md index 627ae4f90c..7367d7eb7f 100644 --- a/docs/pod-metrics.md +++ b/docs/pod-metrics.md @@ -1,31 +1,33 @@ # Pod Metrics -| Metric name| Metric type | Description | Unit (where applicable) | Labels/tags | Status | Opt-in | -| ---------- | ----------- |-----------------------------------------------------------------------| ----------------------- | ----------- | ------ | ------ | -| kube_pod_annotations | Gauge | Kubernetes annotations converted to Prometheus labels | | `pod`=<pod-name>
`namespace`=<pod-namespace>
`annotation_POD_ANNOTATION`=<POD_ANNOTATION>
`uid`=<pod-uid> | EXPERIMENTAL | - -| kube_pod_info | Gauge | Information about pod | | `pod`=<pod-name>
`namespace`=<pod-namespace>
`host_ip`=<host-ip>
`pod_ip`=<pod-ip>
`node`=<node-name>
`created_by_kind`=<created_by_kind>
`created_by_name`=<created_by_name>
`uid`=<pod-uid>
`priority_class`=<priority_class>
`host_network`=<host_network>| STABLE | - | -| kube_pod_ips | Gauge | Pod IP addresses | | `pod`=<pod-name>
`namespace`=<pod-namespace>
`ip`=<pod-ip-address>
`ip_family`=<4 OR 6>
`uid`=<pod-uid> | EXPERIMENTAL | - | -| kube_pod_start_time | Gauge | Start time in unix timestamp for a pod | seconds | `pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | - | -| kube_pod_completion_time | Gauge | Completion time in unix timestamp for a pod | seconds | `pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | - | -| kube_pod_owner | Gauge | Information about the Pod's owner | |`pod`=<pod-name>
`namespace`=<pod-namespace>
`owner_kind`=<owner kind>
`owner_name`=<owner name>
`owner_is_controller`=<whether owner is controller>
`uid`=<pod-uid> | STABLE | - | -| kube_pod_labels | Gauge | Kubernetes labels converted to Prometheus labels | | `pod`=<pod-name>
`namespace`=<pod-namespace>
`label_POD_LABEL`=<POD_LABEL>
`uid`=<pod-uid> | STABLE | - | -| kube_pod_nodeselectors| Gauge | Describes the Pod nodeSelectors | | `pod`=<pod-name>
`namespace`=<pod-namespace>
`nodeselector_NODE_SELECTOR`=<NODE_SELECTOR>
`uid`=<pod-uid> | EXPERIMENTAL | Opt-in | -| kube_pod_status_phase | Gauge | The pods current phase | | `pod`=<pod-name>
`namespace`=<pod-namespace>
`phase`=<Pending\|Running\|Succeeded\|Failed\|Unknown>
`uid`=<pod-uid> | STABLE | - | -| kube_pod_status_ready | Gauge | Describes whether the pod is ready to serve requests | | `pod`=<pod-name>
`namespace`=<pod-namespace>
`condition`=<true\|false\|unknown>
`uid`=<pod-uid> | STABLE | - | -| kube_pod_status_scheduled | Gauge | Describes the status of the scheduling process for the pod | |`pod`=<pod-name>
`namespace`=<pod-namespace>
`condition`=<true\|false\|unknown>
`uid`=<pod-uid> | STABLE | - | -| kube_pod_container_info | Gauge | Information about a container in a pod | | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`image`=<image-name>
`image_id`=<image-id>
`image_spec`=<image-spec>
`container_id`=<containerid>
`uid`=<pod-uid> | STABLE | - | -| kube_pod_container_status_waiting | Gauge | Describes whether the container is currently in waiting state | | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | - | -| kube_pod_container_status_waiting_reason | Gauge | Describes the reason the container is currently in waiting state | |`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`reason`=<container-waiting-reason>
`uid`=<pod-uid> | STABLE | - | -| kube_pod_container_status_running | Gauge | Describes whether the container is currently in running state | |`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | - | -| kube_pod_container_state_started | Gauge | Start time in unix timestamp for a pod container | seconds |`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | - | -| kube_pod_container_status_terminated | Gauge | Describes whether the container is currently in terminated state | |`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | - | -| kube_pod_container_status_terminated_reason | Gauge | Describes the reason the container is currently in terminated state | |`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`reason`=<container-terminated-reason>
`uid`=<pod-uid> | EXPERIMENTAL | - | -| kube_pod_container_status_last_terminated_reason | Gauge | Describes the last reason the container was in terminated state | |`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`reason`=<last-terminated-reason>
`uid`=<pod-uid> | EXPERIMENTAL | - | -| kube_pod_container_status_last_terminated_exitcode | Gauge | Describes the exit code for the last container in terminated state. | | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | EXPERIMENTAL | - | -| kube_pod_container_status_ready | Gauge | Describes whether the containers readiness check succeeded | |`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | - | -| kube_pod_container_status_restarts_total | Counter | The number of container restarts per container | | `container`=<container-name>
`namespace`=<pod-namespace>
`pod`=<pod-name>
`uid`=<pod-uid> | STABLE | - | -| kube_pod_container_resource_requests | Gauge | The number of requested request resource by a container. It is recommended to use the `kube_pod_resource_requests` metric exposed by kube-scheduler instead, as it is more precise. | `cpu`=<core>
`memory`=<bytes> |`resource`=<resource-name>
`unit`=<resource-unit>
`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`node`=< node-name>
`uid`=<pod-uid> | EXPERIMENTAL | - | -| kube_pod_container_resource_limits | Gauge | The number of requested limit resource by a container. It is recommended to use the `kube_pod_resource_limits` metric exposed by kube-scheduler instead, as it is more precise. | `cpu`=<core>
`memory`=<bytes> |`resource`=<resource-name>
`unit`=<resource-unit>
`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`node`=< node-name>
`uid`=<pod-uid> | EXPERIMENTAL | - | +| Metric name| Metric type | Description | Unit (where applicable) | Labels/tags | Status | Opt-in | +| ---------- | ----------- | ----------- | ----------------------- | ----------- | ------ | ------ | +| kube_pod_annotations | Gauge | Kubernetes annotations converted to Prometheus labels | | `pod`=<pod-name>
`namespace`=<pod-namespace>
`annotation_POD_ANNOTATION`=<POD_ANNOTATION>
`uid`=<pod-uid> | EXPERIMENTAL | - +| kube_pod_info | Gauge | Information about pod | | `pod`=<pod-name>
`namespace`=<pod-namespace>
`host_ip`=<host-ip>
`pod_ip`=<pod-ip>
`node`=<node-name>
`created_by_kind`=<created_by_kind>
`created_by_name`=<created_by_name>
`uid`=<pod-uid>
`priority_class`=<priority_class>
`host_network`=<host_network>| STABLE | - | +| kube_pod_ips | Gauge | Pod IP addresses | | `pod`=<pod-name>
`namespace`=<pod-namespace>
`ip`=<pod-ip-address>
`ip_family`=<4 OR 6>
`uid`=<pod-uid> | EXPERIMENTAL | - | +| kube_pod_start_time | Gauge | Start time in unix timestamp for a pod | seconds | `pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_completion_time | Gauge | Completion time in unix timestamp for a pod | seconds | `pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_owner | Gauge | Information about the Pod's owner | |`pod`=<pod-name>
`namespace`=<pod-namespace>
`owner_kind`=<owner kind>
`owner_name`=<owner name>
`owner_is_controller`=<whether owner is controller>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_labels | Gauge | Kubernetes labels converted to Prometheus labels | | `pod`=<pod-name>
`namespace`=<pod-namespace>
`label_POD_LABEL`=<POD_LABEL>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_nodeselectors| Gauge | Describes the Pod nodeSelectors | | `pod`=<pod-name>
`namespace`=<pod-namespace>
`nodeselector_NODE_SELECTOR`=<NODE_SELECTOR>
`uid`=<pod-uid> | EXPERIMENTAL | Opt-in | +| kube_pod_status_phase | Gauge | The pods current phase | | `pod`=<pod-name>
`namespace`=<pod-namespace>
`phase`=<Pending\|Running\|Succeeded\|Failed\|Unknown>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_status_ready | Gauge | Describes whether the pod is ready to serve requests | | `pod`=<pod-name>
`namespace`=<pod-namespace>
`condition`=<true\|false\|unknown>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_status_ready_time | Gauge | Time when pod passed readiness probes. | seconds | `pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | +| kube_pod_status_container_ready_time | Gauge | Time when the container of the pod entered Ready state. | seconds | `pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | +| kube_pod_status_scheduled | Gauge | Describes the status of the scheduling process for the pod | |`pod`=<pod-name>
`namespace`=<pod-namespace>
`condition`=<true\|false\|unknown>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_container_info | Gauge | Information about a container in a pod | | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`image`=<image-name>
`image_id`=<image-id>
`image_spec`=<image-spec>
`container_id`=<containerid>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_container_status_waiting | Gauge | Describes whether the container is currently in waiting state | | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_container_status_waiting_reason | Gauge | Describes the reason the container is currently in waiting state | |`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`reason`=<container-waiting-reason>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_container_status_running | Gauge | Describes whether the container is currently in running state | |`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_container_state_started | Gauge | Start time in unix timestamp for a pod container | seconds |`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_container_status_terminated | Gauge | Describes whether the container is currently in terminated state | |`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_container_status_terminated_reason | Gauge | Describes the reason the container is currently in terminated state | |`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`reason`=<container-terminated-reason>
`uid`=<pod-uid> | EXPERIMENTAL | - | +| kube_pod_container_status_last_terminated_reason | Gauge | Describes the last reason the container was in terminated state | |`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`reason`=<last-terminated-reason>
`uid`=<pod-uid> | EXPERIMENTAL | - | +| kube_pod_container_status_last_terminated_exitcode | Gauge | Describes the exit code for the last container in terminated state. | | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | EXPERIMENTAL | - | +| kube_pod_container_status_ready | Gauge | Describes whether the containers readiness check succeeded | |`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_container_status_restarts_total | Counter | The number of container restarts per container | | `container`=<container-name>
`namespace`=<pod-namespace>
`pod`=<pod-name>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_container_resource_requests | Gauge | The number of requested request resource by a container | `cpu`=<core>
`memory`=<bytes> |`resource`=<resource-name>
`unit`=<resource-unit>
`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`node`=< node-name>
`uid`=<pod-uid> | EXPERIMENTAL | - | +| kube_pod_container_resource_limits | Gauge | The number of requested limit resource by a container | `cpu`=<core>
`memory`=<bytes> |`resource`=<resource-name>
`unit`=<resource-unit>
`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`node`=< node-name>
`uid`=<pod-uid> | EXPERIMENTAL | - | | kube_pod_overhead_cpu_cores | Gauge | The pod overhead in regards to cpu cores associated with running a pod | core |`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | EXPERIMENTAL | - | | kube_pod_overhead_memory_bytes | Gauge | The pod overhead in regards to memory associated with running a pod | bytes |`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | EXPERIMENTAL | - | | kube_pod_runtimeclass_name_info | Gauge | The runtimeclass associated with the pod | |`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | EXPERIMENTAL | - | diff --git a/internal/store/pod.go b/internal/store/pod.go index f9bfdd21f2..8d6ca2db31 100644 --- a/internal/store/pod.go +++ b/internal/store/pod.go @@ -83,6 +83,8 @@ func podMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generat createPodStartTimeFamilyGenerator(), createPodStatusPhaseFamilyGenerator(), createPodStatusReadyFamilyGenerator(), + createPodStatusReadyTimeFamilyGenerator(), + createPodStatusContainerReadyTimeFamilyGenerator(), createPodStatusReasonFamilyGenerator(), createPodStatusScheduledFamilyGenerator(), createPodStatusScheduledTimeFamilyGenerator(), @@ -1317,6 +1319,58 @@ func createPodStatusPhaseFamilyGenerator() generator.FamilyGenerator { ) } +func createPodStatusContainerReadyTimeFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGenerator( + "kube_pod_status_container_ready_time", + "Readiness achieved time in unix timestamp for a pod containers.", + metric.Gauge, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + ms := []*metric.Metric{} + + for _, c := range p.Status.Conditions { + if c.Type == v1.ContainersReady { + ms = append(ms, &metric.Metric{ + LabelKeys: []string{}, + LabelValues: []string{}, + Value: float64((c.LastTransitionTime).Unix()), + }) + } + } + + return &metric.Family{ + Metrics: ms, + } + }), + ) +} + +func createPodStatusReadyTimeFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGenerator( + "kube_pod_status_ready_time", + "Readiness achieved time in unix timestamp for a pod.", + metric.Gauge, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + ms := []*metric.Metric{} + + for _, c := range p.Status.Conditions { + if c.Type == v1.PodReady { + ms = append(ms, &metric.Metric{ + LabelKeys: []string{}, + LabelValues: []string{}, + Value: float64((c.LastTransitionTime).Unix()), + }) + } + } + + return &metric.Family{ + Metrics: ms, + } + }), + ) +} + func createPodStatusReadyFamilyGenerator() generator.FamilyGenerator { return *generator.NewFamilyGeneratorWithStability( "kube_pod_status_ready", diff --git a/internal/store/pod_test.go b/internal/store/pod_test.go index fb118f2d1c..438d5bcb2d 100644 --- a/internal/store/pod_test.go +++ b/internal/store/pod_test.go @@ -1426,13 +1426,19 @@ func TestPodStore(t *testing.T) { { Type: v1.PodReady, Status: v1.ConditionTrue, + LastTransitionTime: metav1.Time{ + Time: time.Unix(1501666018, 0), + }, }, }, }, }, Want: ` # HELP kube_pod_status_ready [STABLE] Describes whether the pod is ready to serve requests. + # HELP kube_pod_status_ready_time Readiness achieved time in unix timestamp for a pod. # TYPE kube_pod_status_ready gauge + # TYPE kube_pod_status_ready_time gauge + kube_pod_status_ready_time{namespace="ns1",pod="pod1",uid="uid1"} 1.501666018e+09 kube_pod_status_ready{condition="false",namespace="ns1",pod="pod1",uid="uid1"} 0 kube_pod_status_ready{condition="true",namespace="ns1",pod="pod1",uid="uid1"} 1 kube_pod_status_ready{condition="unknown",namespace="ns1",pod="pod1",uid="uid1"} 0 @@ -1451,13 +1457,19 @@ func TestPodStore(t *testing.T) { { Type: v1.PodReady, Status: v1.ConditionFalse, + LastTransitionTime: metav1.Time{ + Time: time.Unix(1501666018, 0), + }, }, }, }, }, Want: ` # HELP kube_pod_status_ready [STABLE] Describes whether the pod is ready to serve requests. + # HELP kube_pod_status_ready_time Readiness achieved time in unix timestamp for a pod. # TYPE kube_pod_status_ready gauge + # TYPE kube_pod_status_ready_time gauge + kube_pod_status_ready_time{namespace="ns2",pod="pod2",uid="uid2"} 1.501666018e+09 kube_pod_status_ready{condition="false",namespace="ns2",pod="pod2",uid="uid2"} 1 kube_pod_status_ready{condition="true",namespace="ns2",pod="pod2",uid="uid2"} 0 kube_pod_status_ready{condition="unknown",namespace="ns2",pod="pod2",uid="uid2"} 0 @@ -2079,7 +2091,7 @@ func BenchmarkPodStore(b *testing.B) { }, } - expectedFamilies := 47 + expectedFamilies := 49 for n := 0; n < b.N; n++ { families := f(pod) if len(families) != expectedFamilies { diff --git a/pkg/app/server_test.go b/pkg/app/server_test.go index 659b84bf24..263385b8f3 100644 --- a/pkg/app/server_test.go +++ b/pkg/app/server_test.go @@ -238,7 +238,9 @@ func TestFullScrapeCycle(t *testing.T) { # HELP kube_pod_spec_volumes_persistentvolumeclaims_info [STABLE] Information about persistentvolumeclaim volumes in a pod. # HELP kube_pod_spec_volumes_persistentvolumeclaims_readonly [STABLE] Describes whether a persistentvolumeclaim is mounted read only. # HELP kube_pod_start_time [STABLE] Start time in unix timestamp for a pod. +# HELP kube_pod_status_container_ready_time Readiness achieved time in unix timestamp for a pod containers. # HELP kube_pod_status_phase [STABLE] The pods current phase. +# HELP kube_pod_status_ready_time Readiness achieved time in unix timestamp for a pod. # HELP kube_pod_status_ready [STABLE] Describes whether the pod is ready to serve requests. # HELP kube_pod_status_reason The pod status reasons # HELP kube_pod_status_scheduled [STABLE] Describes the status of the scheduling process for the pod. @@ -284,8 +286,10 @@ func TestFullScrapeCycle(t *testing.T) { # TYPE kube_pod_spec_volumes_persistentvolumeclaims_info gauge # TYPE kube_pod_spec_volumes_persistentvolumeclaims_readonly gauge # TYPE kube_pod_start_time gauge +# TYPE kube_pod_status_container_ready_time gauge # TYPE kube_pod_status_phase gauge # TYPE kube_pod_status_ready gauge +# TYPE kube_pod_status_ready_time gauge # TYPE kube_pod_status_reason gauge # TYPE kube_pod_status_scheduled gauge # TYPE kube_pod_status_scheduled_time gauge From 0e4e396e349f65484b038d4708379572a1f0170f Mon Sep 17 00:00:00 2001 From: Ryan Olds Date: Thu, 29 Dec 2022 14:26:57 -0800 Subject: [PATCH 2/4] Addressed feedback --- internal/store/pod.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/store/pod.go b/internal/store/pod.go index 8d6ca2db31..d9260b7aa9 100644 --- a/internal/store/pod.go +++ b/internal/store/pod.go @@ -1320,10 +1320,11 @@ func createPodStatusPhaseFamilyGenerator() generator.FamilyGenerator { } func createPodStatusContainerReadyTimeFamilyGenerator() generator.FamilyGenerator { - return *generator.NewFamilyGenerator( + return *generator.NewFamilyGeneratorWithStability( "kube_pod_status_container_ready_time", "Readiness achieved time in unix timestamp for a pod containers.", metric.Gauge, + basemetrics.ALPHA, "", wrapPodFunc(func(p *v1.Pod) *metric.Family { ms := []*metric.Metric{} @@ -1346,10 +1347,11 @@ func createPodStatusContainerReadyTimeFamilyGenerator() generator.FamilyGenerato } func createPodStatusReadyTimeFamilyGenerator() generator.FamilyGenerator { - return *generator.NewFamilyGenerator( + return *generator.NewFamilyGeneratorWithStability( "kube_pod_status_ready_time", "Readiness achieved time in unix timestamp for a pod.", metric.Gauge, + basemetrics.ALPHA, "", wrapPodFunc(func(p *v1.Pod) *metric.Family { ms := []*metric.Metric{} From dd7202bdc2ce4554a7f191f899edc80d978a530b Mon Sep 17 00:00:00 2001 From: "Ryan R. Olds" Date: Tue, 10 Jan 2023 16:13:18 -0800 Subject: [PATCH 3/4] Fixing bad merge in rebase --- docs/pod-metrics.md | 56 ++++++++++++++++++++++----------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/docs/pod-metrics.md b/docs/pod-metrics.md index 7367d7eb7f..c2d6fd9339 100644 --- a/docs/pod-metrics.md +++ b/docs/pod-metrics.md @@ -1,33 +1,33 @@ # Pod Metrics -| Metric name| Metric type | Description | Unit (where applicable) | Labels/tags | Status | Opt-in | -| ---------- | ----------- | ----------- | ----------------------- | ----------- | ------ | ------ | -| kube_pod_annotations | Gauge | Kubernetes annotations converted to Prometheus labels | | `pod`=<pod-name>
`namespace`=<pod-namespace>
`annotation_POD_ANNOTATION`=<POD_ANNOTATION>
`uid`=<pod-uid> | EXPERIMENTAL | - -| kube_pod_info | Gauge | Information about pod | | `pod`=<pod-name>
`namespace`=<pod-namespace>
`host_ip`=<host-ip>
`pod_ip`=<pod-ip>
`node`=<node-name>
`created_by_kind`=<created_by_kind>
`created_by_name`=<created_by_name>
`uid`=<pod-uid>
`priority_class`=<priority_class>
`host_network`=<host_network>| STABLE | - | -| kube_pod_ips | Gauge | Pod IP addresses | | `pod`=<pod-name>
`namespace`=<pod-namespace>
`ip`=<pod-ip-address>
`ip_family`=<4 OR 6>
`uid`=<pod-uid> | EXPERIMENTAL | - | -| kube_pod_start_time | Gauge | Start time in unix timestamp for a pod | seconds | `pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | - | -| kube_pod_completion_time | Gauge | Completion time in unix timestamp for a pod | seconds | `pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | - | -| kube_pod_owner | Gauge | Information about the Pod's owner | |`pod`=<pod-name>
`namespace`=<pod-namespace>
`owner_kind`=<owner kind>
`owner_name`=<owner name>
`owner_is_controller`=<whether owner is controller>
`uid`=<pod-uid> | STABLE | - | -| kube_pod_labels | Gauge | Kubernetes labels converted to Prometheus labels | | `pod`=<pod-name>
`namespace`=<pod-namespace>
`label_POD_LABEL`=<POD_LABEL>
`uid`=<pod-uid> | STABLE | - | -| kube_pod_nodeselectors| Gauge | Describes the Pod nodeSelectors | | `pod`=<pod-name>
`namespace`=<pod-namespace>
`nodeselector_NODE_SELECTOR`=<NODE_SELECTOR>
`uid`=<pod-uid> | EXPERIMENTAL | Opt-in | -| kube_pod_status_phase | Gauge | The pods current phase | | `pod`=<pod-name>
`namespace`=<pod-namespace>
`phase`=<Pending\|Running\|Succeeded\|Failed\|Unknown>
`uid`=<pod-uid> | STABLE | - | -| kube_pod_status_ready | Gauge | Describes whether the pod is ready to serve requests | | `pod`=<pod-name>
`namespace`=<pod-namespace>
`condition`=<true\|false\|unknown>
`uid`=<pod-uid> | STABLE | - | -| kube_pod_status_ready_time | Gauge | Time when pod passed readiness probes. | seconds | `pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | -| kube_pod_status_container_ready_time | Gauge | Time when the container of the pod entered Ready state. | seconds | `pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | -| kube_pod_status_scheduled | Gauge | Describes the status of the scheduling process for the pod | |`pod`=<pod-name>
`namespace`=<pod-namespace>
`condition`=<true\|false\|unknown>
`uid`=<pod-uid> | STABLE | - | -| kube_pod_container_info | Gauge | Information about a container in a pod | | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`image`=<image-name>
`image_id`=<image-id>
`image_spec`=<image-spec>
`container_id`=<containerid>
`uid`=<pod-uid> | STABLE | - | -| kube_pod_container_status_waiting | Gauge | Describes whether the container is currently in waiting state | | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | - | -| kube_pod_container_status_waiting_reason | Gauge | Describes the reason the container is currently in waiting state | |`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`reason`=<container-waiting-reason>
`uid`=<pod-uid> | STABLE | - | -| kube_pod_container_status_running | Gauge | Describes whether the container is currently in running state | |`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | - | -| kube_pod_container_state_started | Gauge | Start time in unix timestamp for a pod container | seconds |`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | - | -| kube_pod_container_status_terminated | Gauge | Describes whether the container is currently in terminated state | |`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | - | -| kube_pod_container_status_terminated_reason | Gauge | Describes the reason the container is currently in terminated state | |`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`reason`=<container-terminated-reason>
`uid`=<pod-uid> | EXPERIMENTAL | - | -| kube_pod_container_status_last_terminated_reason | Gauge | Describes the last reason the container was in terminated state | |`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`reason`=<last-terminated-reason>
`uid`=<pod-uid> | EXPERIMENTAL | - | -| kube_pod_container_status_last_terminated_exitcode | Gauge | Describes the exit code for the last container in terminated state. | | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | EXPERIMENTAL | - | -| kube_pod_container_status_ready | Gauge | Describes whether the containers readiness check succeeded | |`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | - | -| kube_pod_container_status_restarts_total | Counter | The number of container restarts per container | | `container`=<container-name>
`namespace`=<pod-namespace>
`pod`=<pod-name>
`uid`=<pod-uid> | STABLE | - | -| kube_pod_container_resource_requests | Gauge | The number of requested request resource by a container | `cpu`=<core>
`memory`=<bytes> |`resource`=<resource-name>
`unit`=<resource-unit>
`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`node`=< node-name>
`uid`=<pod-uid> | EXPERIMENTAL | - | -| kube_pod_container_resource_limits | Gauge | The number of requested limit resource by a container | `cpu`=<core>
`memory`=<bytes> |`resource`=<resource-name>
`unit`=<resource-unit>
`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`node`=< node-name>
`uid`=<pod-uid> | EXPERIMENTAL | - | +| Metric name| Metric type | Description | Unit (where applicable) | Labels/tags | Status | Opt-in | +| ---------- | ----------- |-----------------------------------------------------------------------| ----------------------- | ----------- | ------ | ------ | +| kube_pod_annotations | Gauge | Kubernetes annotations converted to Prometheus labels | | `pod`=<pod-name>
`namespace`=<pod-namespace>
`annotation_POD_ANNOTATION`=<POD_ANNOTATION>
`uid`=<pod-uid> | EXPERIMENTAL | - +| kube_pod_info | Gauge | Information about pod | | `pod`=<pod-name>
`namespace`=<pod-namespace>
`host_ip`=<host-ip>
`pod_ip`=<pod-ip>
`node`=<node-name>
`created_by_kind`=<created_by_kind>
`created_by_name`=<created_by_name>
`uid`=<pod-uid>
`priority_class`=<priority_class>
`host_network`=<host_network>| STABLE | - | +| kube_pod_ips | Gauge | Pod IP addresses | | `pod`=<pod-name>
`namespace`=<pod-namespace>
`ip`=<pod-ip-address>
`ip_family`=<4 OR 6>
`uid`=<pod-uid> | EXPERIMENTAL | - | +| kube_pod_start_time | Gauge | Start time in unix timestamp for a pod | seconds | `pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_completion_time | Gauge | Completion time in unix timestamp for a pod | seconds | `pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_owner | Gauge | Information about the Pod's owner | |`pod`=<pod-name>
`namespace`=<pod-namespace>
`owner_kind`=<owner kind>
`owner_name`=<owner name>
`owner_is_controller`=<whether owner is controller>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_labels | Gauge | Kubernetes labels converted to Prometheus labels | | `pod`=<pod-name>
`namespace`=<pod-namespace>
`label_POD_LABEL`=<POD_LABEL>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_nodeselectors| Gauge | Describes the Pod nodeSelectors | | `pod`=<pod-name>
`namespace`=<pod-namespace>
`nodeselector_NODE_SELECTOR`=<NODE_SELECTOR>
`uid`=<pod-uid> | EXPERIMENTAL | Opt-in | +| kube_pod_status_phase | Gauge | The pods current phase | | `pod`=<pod-name>
`namespace`=<pod-namespace>
`phase`=<Pending\|Running\|Succeeded\|Failed\|Unknown>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_status_ready | Gauge | Describes whether the pod is ready to serve requests | | `pod`=<pod-name>
`namespace`=<pod-namespace>
`condition`=<true\|false\|unknown>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_status_scheduled | Gauge | Describes the status of the scheduling process for the pod | |`pod`=<pod-name>
`namespace`=<pod-namespace>
`condition`=<true\|false\|unknown>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_container_info | Gauge | Information about a container in a pod | | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`image`=<image-name>
`image_id`=<image-id>
`image_spec`=<image-spec>
`container_id`=<containerid>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_container_status_waiting | Gauge | Describes whether the container is currently in waiting state | | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_container_status_waiting_reason | Gauge | Describes the reason the container is currently in waiting state | |`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`reason`=<container-waiting-reason>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_container_status_running | Gauge | Describes whether the container is currently in running state | |`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_container_state_started | Gauge | Start time in unix timestamp for a pod container | seconds |`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_container_status_terminated | Gauge | Describes whether the container is currently in terminated state | |`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_container_status_terminated_reason | Gauge | Describes the reason the container is currently in terminated state | |`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`reason`=<container-terminated-reason>
`uid`=<pod-uid> | EXPERIMENTAL | - | +| kube_pod_container_status_last_terminated_reason | Gauge | Describes the last reason the container was in terminated state | |`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`reason`=<last-terminated-reason>
`uid`=<pod-uid> | EXPERIMENTAL | - | +| kube_pod_container_status_last_terminated_exitcode | Gauge | Describes the exit code for the last container in terminated state. | | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | EXPERIMENTAL | - | +| kube_pod_container_status_ready | Gauge | Describes whether the containers readiness check succeeded | |`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_status_ready_time | Gauge | Time when pod passed readiness probes. | seconds | `pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | EXPERIMENTAL | +| kube_pod_status_container_ready_time | Gauge | Time when the container of the pod entered Ready state. | seconds | `pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | EXPERIMENTAL | +| kube_pod_container_status_restarts_total | Counter | The number of container restarts per container | | `container`=<container-name>
`namespace`=<pod-namespace>
`pod`=<pod-name>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_container_resource_requests | Gauge | The number of requested request resource by a container. It is recommended to use the `kube_pod_resource_requests` metric exposed by kube-scheduler instead, as it is more precise. | `cpu`=<core>
`memory`=<bytes> |`resource`=<resource-name>
`unit`=<resource-unit>
`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`node`=< node-name>
`uid`=<pod-uid> | EXPERIMENTAL | - | +| kube_pod_container_resource_limits | Gauge | The number of requested limit resource by a container. It is recommended to use the `kube_pod_resource_limits` metric exposed by kube-scheduler instead, as it is more precise. | `cpu`=<core>
`memory`=<bytes> |`resource`=<resource-name>
`unit`=<resource-unit>
`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`node`=< node-name>
`uid`=<pod-uid> | EXPERIMENTAL | - | | kube_pod_overhead_cpu_cores | Gauge | The pod overhead in regards to cpu cores associated with running a pod | core |`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | EXPERIMENTAL | - | | kube_pod_overhead_memory_bytes | Gauge | The pod overhead in regards to memory associated with running a pod | bytes |`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | EXPERIMENTAL | - | | kube_pod_runtimeclass_name_info | Gauge | The runtimeclass associated with the pod | |`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | EXPERIMENTAL | - | From ad5a7acf4549cf097a956854fc623f1cf6f3b4a3 Mon Sep 17 00:00:00 2001 From: "Ryan R. Olds" Date: Wed, 18 Jan 2023 16:23:44 -0800 Subject: [PATCH 4/4] Added a test for kube_pod_status_container_ready_time --- internal/store/pod_test.go | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/internal/store/pod_test.go b/internal/store/pod_test.go index 25d33d0a8b..35fc0fd996 100644 --- a/internal/store/pod_test.go +++ b/internal/store/pod_test.go @@ -1434,6 +1434,32 @@ func TestPodStore(t *testing.T) { `, MetricNames: []string{"kube_pod_status_reason"}, }, + { + Obj: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "ns1", + UID: "uid1", + }, + Status: v1.PodStatus{ + Conditions: []v1.PodCondition{ + { + Type: v1.ContainersReady, + Status: v1.ConditionTrue, + LastTransitionTime: metav1.Time{ + Time: time.Unix(1501666018, 0), + }, + }, + }, + }, + }, + Want: ` + # HELP kube_pod_status_container_ready_time Readiness achieved time in unix timestamp for a pod containers. + # TYPE kube_pod_status_container_ready_time gauge + kube_pod_status_container_ready_time{namespace="ns1",pod="pod1",uid="uid1"} 1.501666018e+09 + `, + MetricNames: []string{"kube_pod_status_container_ready_time"}, + }, { Obj: &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ @@ -1463,7 +1489,7 @@ func TestPodStore(t *testing.T) { kube_pod_status_ready{condition="true",namespace="ns1",pod="pod1",uid="uid1"} 1 kube_pod_status_ready{condition="unknown",namespace="ns1",pod="pod1",uid="uid1"} 0 `, - MetricNames: []string{"kube_pod_status_ready"}, + MetricNames: []string{"kube_pod_status_ready_time", "kube_pod_status_ready"}, }, { Obj: &v1.Pod{ @@ -1494,7 +1520,7 @@ func TestPodStore(t *testing.T) { kube_pod_status_ready{condition="true",namespace="ns2",pod="pod2",uid="uid2"} 0 kube_pod_status_ready{condition="unknown",namespace="ns2",pod="pod2",uid="uid2"} 0 `, - MetricNames: []string{"kube_pod_status_ready"}, + MetricNames: []string{"kube_pod_status_ready_time", "kube_pod_status_ready"}, }, { Obj: &v1.Pod{