Skip to content

Commit

Permalink
Merge pull request #1938 from ryanrolds/rolds/pod_ready_time
Browse files Browse the repository at this point in the history 
Add metrics of kube_pod_status_ready_time and kube_pod_status_containers_ready_time redux
  • Loading branch information
@k8s-ci-robot
k8s-ci-robot committed Jan 19, 2023
2 parents c752152 + ad5a7ac commit 1cda0bf
Show file tree
Hide file tree
Showing 4 changed files with 103 additions and 3 deletions.
2 changes: 2 additions & 0 deletions docs/pod-metrics.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@
| kube_pod_container_status_last_terminated_reason | Gauge | Describes the last reason the container was in terminated state | |`container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `reason`=&lt;last-terminated-reason&gt; <br> `uid`=&lt;pod-uid&gt; | EXPERIMENTAL | - |
| kube_pod_container_status_last_terminated_exitcode | Gauge | Describes the exit code for the last container in terminated state. | | `container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `uid`=&lt;pod-uid&gt; | EXPERIMENTAL | - |
| kube_pod_container_status_ready | Gauge | Describes whether the containers readiness check succeeded | |`container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `uid`=&lt;pod-uid&gt; | STABLE | - |
| kube_pod_status_ready_time | Gauge | Time when pod passed readiness probes. | seconds | `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `uid`=&lt;pod-uid&gt; | EXPERIMENTAL |
| kube_pod_status_container_ready_time | Gauge | Time when the container of the pod entered Ready state. | seconds | `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `uid`=&lt;pod-uid&gt; | EXPERIMENTAL |
| kube_pod_container_status_restarts_total | Counter | The number of container restarts per container | | `container`=&lt;container-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `pod`=&lt;pod-name&gt; <br> `uid`=&lt;pod-uid&gt; | STABLE | - |
| kube_pod_container_resource_requests | Gauge | The number of requested request resource by a container. It is recommended to use the `kube_pod_resource_requests` metric exposed by kube-scheduler instead, as it is more precise. | `cpu`=&lt;core&gt; <br> `memory`=&lt;bytes&gt; |`resource`=&lt;resource-name&gt; <br> `unit`=&lt;resource-unit&gt; <br> `container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `node`=&lt; node-name&gt; <br> `uid`=&lt;pod-uid&gt; | EXPERIMENTAL | - |
| kube_pod_container_resource_limits | Gauge | The number of requested limit resource by a container. It is recommended to use the `kube_pod_resource_limits` metric exposed by kube-scheduler instead, as it is more precise. | `cpu`=&lt;core&gt; <br> `memory`=&lt;bytes&gt; |`resource`=&lt;resource-name&gt; <br> `unit`=&lt;resource-unit&gt; <br> `container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `node`=&lt; node-name&gt; <br> `uid`=&lt;pod-uid&gt; | EXPERIMENTAL | - |
Expand Down
56 changes: 56 additions & 0 deletions internal/store/pod.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,8 @@ func podMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generat
createPodStatusPhaseFamilyGenerator(),
createPodStatusQosClassFamilyGenerator(),
createPodStatusReadyFamilyGenerator(),
createPodStatusReadyTimeFamilyGenerator(),
createPodStatusContainerReadyTimeFamilyGenerator(),
createPodStatusReasonFamilyGenerator(),
createPodStatusScheduledFamilyGenerator(),
createPodStatusScheduledTimeFamilyGenerator(),
Expand Down Expand Up @@ -1318,6 +1320,60 @@ func createPodStatusPhaseFamilyGenerator() generator.FamilyGenerator {
)
}

func createPodStatusContainerReadyTimeFamilyGenerator() generator.FamilyGenerator {
return *generator.NewFamilyGeneratorWithStability(
"kube_pod_status_container_ready_time",
"Readiness achieved time in unix timestamp for a pod containers.",
metric.Gauge,
basemetrics.ALPHA,
"",
wrapPodFunc(func(p *v1.Pod) *metric.Family {
ms := []*metric.Metric{}

for _, c := range p.Status.Conditions {
if c.Type == v1.ContainersReady {
ms = append(ms, &metric.Metric{
LabelKeys: []string{},
LabelValues: []string{},
Value: float64((c.LastTransitionTime).Unix()),
})
}
}

return &metric.Family{
Metrics: ms,
}
}),
)
}

func createPodStatusReadyTimeFamilyGenerator() generator.FamilyGenerator {
return *generator.NewFamilyGeneratorWithStability(
"kube_pod_status_ready_time",
"Readiness achieved time in unix timestamp for a pod.",
metric.Gauge,
basemetrics.ALPHA,
"",
wrapPodFunc(func(p *v1.Pod) *metric.Family {
ms := []*metric.Metric{}

for _, c := range p.Status.Conditions {
if c.Type == v1.PodReady {
ms = append(ms, &metric.Metric{
LabelKeys: []string{},
LabelValues: []string{},
Value: float64((c.LastTransitionTime).Unix()),
})
}
}

return &metric.Family{
Metrics: ms,
}
}),
)
}

func createPodStatusQosClassFamilyGenerator() generator.FamilyGenerator {
return *generator.NewFamilyGeneratorWithStability(
"kube_pod_status_qos_class",
Expand Down
44 changes: 41 additions & 3 deletions internal/store/pod_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1434,6 +1434,32 @@ func TestPodStore(t *testing.T) {
`,
MetricNames: []string{"kube_pod_status_reason"},
},
{
Obj: &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "pod1",
Namespace: "ns1",
UID: "uid1",
},
Status: v1.PodStatus{
Conditions: []v1.PodCondition{
{
Type: v1.ContainersReady,
Status: v1.ConditionTrue,
LastTransitionTime: metav1.Time{
Time: time.Unix(1501666018, 0),
},
},
},
},
},
Want: `
# HELP kube_pod_status_container_ready_time Readiness achieved time in unix timestamp for a pod containers.
# TYPE kube_pod_status_container_ready_time gauge
kube_pod_status_container_ready_time{namespace="ns1",pod="pod1",uid="uid1"} 1.501666018e+09
`,
MetricNames: []string{"kube_pod_status_container_ready_time"},
},
{
Obj: &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Expand All @@ -1446,18 +1472,24 @@ func TestPodStore(t *testing.T) {
{
Type: v1.PodReady,
Status: v1.ConditionTrue,
LastTransitionTime: metav1.Time{
Time: time.Unix(1501666018, 0),
},
},
},
},
},
Want: `
# HELP kube_pod_status_ready [STABLE] Describes whether the pod is ready to serve requests.
# HELP kube_pod_status_ready_time Readiness achieved time in unix timestamp for a pod.
# TYPE kube_pod_status_ready gauge
# TYPE kube_pod_status_ready_time gauge
kube_pod_status_ready_time{namespace="ns1",pod="pod1",uid="uid1"} 1.501666018e+09
kube_pod_status_ready{condition="false",namespace="ns1",pod="pod1",uid="uid1"} 0
kube_pod_status_ready{condition="true",namespace="ns1",pod="pod1",uid="uid1"} 1
kube_pod_status_ready{condition="unknown",namespace="ns1",pod="pod1",uid="uid1"} 0
`,
MetricNames: []string{"kube_pod_status_ready"},
MetricNames: []string{"kube_pod_status_ready_time", "kube_pod_status_ready"},
},
{
Obj: &v1.Pod{
Expand All @@ -1471,18 +1503,24 @@ func TestPodStore(t *testing.T) {
{
Type: v1.PodReady,
Status: v1.ConditionFalse,
LastTransitionTime: metav1.Time{
Time: time.Unix(1501666018, 0),
},
},
},
},
},
Want: `
# HELP kube_pod_status_ready [STABLE] Describes whether the pod is ready to serve requests.
# HELP kube_pod_status_ready_time Readiness achieved time in unix timestamp for a pod.
# TYPE kube_pod_status_ready gauge
# TYPE kube_pod_status_ready_time gauge
kube_pod_status_ready_time{namespace="ns2",pod="pod2",uid="uid2"} 1.501666018e+09
kube_pod_status_ready{condition="false",namespace="ns2",pod="pod2",uid="uid2"} 1
kube_pod_status_ready{condition="true",namespace="ns2",pod="pod2",uid="uid2"} 0
kube_pod_status_ready{condition="unknown",namespace="ns2",pod="pod2",uid="uid2"} 0
`,
MetricNames: []string{"kube_pod_status_ready"},
MetricNames: []string{"kube_pod_status_ready_time", "kube_pod_status_ready"},
},
{
Obj: &v1.Pod{
Expand Down Expand Up @@ -2099,7 +2137,7 @@ func BenchmarkPodStore(b *testing.B) {
},
}

expectedFamilies := 48
expectedFamilies := 50
for n := 0; n < b.N; n++ {
families := f(pod)
if len(families) != expectedFamilies {
Expand Down
4 changes: 4 additions & 0 deletions pkg/app/server_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -238,8 +238,10 @@ func TestFullScrapeCycle(t *testing.T) {
# HELP kube_pod_spec_volumes_persistentvolumeclaims_info [STABLE] Information about persistentvolumeclaim volumes in a pod.
# HELP kube_pod_spec_volumes_persistentvolumeclaims_readonly [STABLE] Describes whether a persistentvolumeclaim is mounted read only.
# HELP kube_pod_start_time [STABLE] Start time in unix timestamp for a pod.
# HELP kube_pod_status_container_ready_time Readiness achieved time in unix timestamp for a pod containers.
# HELP kube_pod_status_qos_class The pods current qosClass.
# HELP kube_pod_status_phase [STABLE] The pods current phase.
# HELP kube_pod_status_ready_time Readiness achieved time in unix timestamp for a pod.
# HELP kube_pod_status_ready [STABLE] Describes whether the pod is ready to serve requests.
# HELP kube_pod_status_reason The pod status reasons
# HELP kube_pod_status_scheduled [STABLE] Describes the status of the scheduling process for the pod.
Expand Down Expand Up @@ -285,9 +287,11 @@ func TestFullScrapeCycle(t *testing.T) {
# TYPE kube_pod_spec_volumes_persistentvolumeclaims_info gauge
# TYPE kube_pod_spec_volumes_persistentvolumeclaims_readonly gauge
# TYPE kube_pod_start_time gauge
# TYPE kube_pod_status_container_ready_time gauge
# TYPE kube_pod_status_phase gauge
# TYPE kube_pod_status_qos_class gauge
# TYPE kube_pod_status_ready gauge
# TYPE kube_pod_status_ready_time gauge
# TYPE kube_pod_status_reason gauge
# TYPE kube_pod_status_scheduled gauge
# TYPE kube_pod_status_scheduled_time gauge
Expand Down

0 comments on commit 1cda0bf

Please sign in to comment.