Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add metrics of kube_pod_status_ready_time and kube_pod_status_containers_ready_time #1837

Closed
wants to merge 1 commit into from
Closed

Add metrics of kube_pod_status_ready_time and kube_pod_status_containers_ready_time #1837

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions docs/pod-metrics.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@
| kube_pod_nodeselectors| Gauge | Describes the Pod nodeSelectors | | `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `nodeselector_NODE_SELECTOR`=&lt;NODE_SELECTOR&gt; <br> `uid`=&lt;pod-uid&gt; | EXPERIMENTAL | Opt-in |
| kube_pod_status_phase | Gauge | The pods current phase | | `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `phase`=&lt;Pending\|Running\|Succeeded\|Failed\|Unknown&gt; <br> `uid`=&lt;pod-uid&gt; | STABLE | - |
| kube_pod_status_ready | Gauge | Describes whether the pod is ready to serve requests | | `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `condition`=&lt;true\|false\|unknown&gt; <br> `uid`=&lt;pod-uid&gt; | STABLE | - |
| kube_pod_status_ready_time | Gauge | Time when pod passed readiness probes. | seconds | `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `uid`=&lt;pod-uid&gt; | STABLE |
| kube_pod_status_container_ready_time | Gauge | Time when the container of the pod entered Ready state. | seconds | `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `uid`=&lt;pod-uid&gt; | STABLE |
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let's start with making the metrics experimental, and we can graduate them over time.

Copy link
Contributor

@CatherineF-dev CatherineF-dev Oct 21, 2022 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agree.

You can use generator.NewFamilyGeneratorWithStability instead of generator.NewFamilyGenerator to state it's experimental. (basemetrics.ALPHA)
Example PR: #1844

| kube_pod_status_scheduled | Gauge | Describes the status of the scheduling process for the pod | |`pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `condition`=&lt;true\|false\|unknown&gt; <br> `uid`=&lt;pod-uid&gt; | STABLE | - |
| kube_pod_container_info | Gauge | Information about a container in a pod | | `container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `image`=&lt;image-name&gt; <br> `image_id`=&lt;image-id&gt; <br> `image_spec`=&lt;image-spec&gt; <br> `container_id`=&lt;containerid&gt; <br> `uid`=&lt;pod-uid&gt; | STABLE | - |
| kube_pod_container_status_waiting | Gauge | Describes whether the container is currently in waiting state | | `container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `uid`=&lt;pod-uid&gt; | STABLE | - |
Expand Down
54 changes: 54 additions & 0 deletions internal/store/pod.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,8 @@ func podMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generat
createPodStartTimeFamilyGenerator(),
createPodStatusPhaseFamilyGenerator(),
createPodStatusReadyFamilyGenerator(),
createPodStatusReadyTimeFamilyGenerator(),
createPodStatusContainerReadyTimeFamilyGenerator(),
createPodStatusReasonFamilyGenerator(),
createPodStatusScheduledFamilyGenerator(),
createPodStatusScheduledTimeFamilyGenerator(),
Expand Down Expand Up @@ -1317,6 +1319,58 @@ func createPodStatusPhaseFamilyGenerator() generator.FamilyGenerator {
)
}

func createPodStatusContainerReadyTimeFamilyGenerator() generator.FamilyGenerator {
return *generator.NewFamilyGenerator(
"kube_pod_status_container_ready_time",
"Readiness achieved time in unix timestamp for a pod containers.",
metric.Gauge,
"",
wrapPodFunc(func(p *v1.Pod) *metric.Family {
ms := []*metric.Metric{}

for _, c := range p.Status.Conditions {
if c.Type == v1.ContainersReady {
ms = append(ms, &metric.Metric{
LabelKeys: []string{},
LabelValues: []string{},
Value: float64((c.LastTransitionTime).Unix()),
})
}
}

return &metric.Family{
Metrics: ms,
}
}),
)
}

func createPodStatusReadyTimeFamilyGenerator() generator.FamilyGenerator {
return *generator.NewFamilyGenerator(
"kube_pod_status_ready_time",
"Readiness achieved time in unix timestamp for a pod.",
metric.Gauge,
"",
wrapPodFunc(func(p *v1.Pod) *metric.Family {
ms := []*metric.Metric{}

for _, c := range p.Status.Conditions {
if c.Type == v1.PodReady {
ms = append(ms, &metric.Metric{
LabelKeys: []string{},
LabelValues: []string{},
Value: float64((c.LastTransitionTime).Unix()),
})
}
}

return &metric.Family{
Metrics: ms,
}
}),
)
}

func createPodStatusReadyFamilyGenerator() generator.FamilyGenerator {
return *generator.NewFamilyGeneratorWithStability(
"kube_pod_status_ready",
Expand Down
14 changes: 13 additions & 1 deletion internal/store/pod_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1426,13 +1426,19 @@ func TestPodStore(t *testing.T) {
{
Type: v1.PodReady,
Status: v1.ConditionTrue,
LastTransitionTime: metav1.Time{
Time: time.Unix(1501666018, 0),
},
},
},
},
},
Want: `
# HELP kube_pod_status_ready [STABLE] Describes whether the pod is ready to serve requests.
# HELP kube_pod_status_ready_time Readiness achieved time in unix timestamp for a pod.
# TYPE kube_pod_status_ready gauge
# TYPE kube_pod_status_ready_time gauge
kube_pod_status_ready_time{namespace="ns1",pod="pod1",uid="uid1"} 1.501666018e+09
kube_pod_status_ready{condition="false",namespace="ns1",pod="pod1",uid="uid1"} 0
kube_pod_status_ready{condition="true",namespace="ns1",pod="pod1",uid="uid1"} 1
kube_pod_status_ready{condition="unknown",namespace="ns1",pod="pod1",uid="uid1"} 0
Expand All @@ -1451,13 +1457,19 @@ func TestPodStore(t *testing.T) {
{
Type: v1.PodReady,
Status: v1.ConditionFalse,
LastTransitionTime: metav1.Time{
Time: time.Unix(1501666018, 0),
},
},
},
},
},
Want: `
# HELP kube_pod_status_ready [STABLE] Describes whether the pod is ready to serve requests.
# HELP kube_pod_status_ready_time Readiness achieved time in unix timestamp for a pod.
# TYPE kube_pod_status_ready gauge
fpetkovski marked this conversation as resolved.
Show resolved Hide resolved
# TYPE kube_pod_status_ready_time gauge
kube_pod_status_ready_time{namespace="ns2",pod="pod2",uid="uid2"} 1.501666018e+09
kube_pod_status_ready{condition="false",namespace="ns2",pod="pod2",uid="uid2"} 1
kube_pod_status_ready{condition="true",namespace="ns2",pod="pod2",uid="uid2"} 0
kube_pod_status_ready{condition="unknown",namespace="ns2",pod="pod2",uid="uid2"} 0
Expand Down Expand Up @@ -2079,7 +2091,7 @@ func BenchmarkPodStore(b *testing.B) {
},
}

expectedFamilies := 47
expectedFamilies := 49
for n := 0; n < b.N; n++ {
families := f(pod)
if len(families) != expectedFamilies {
Expand Down
4 changes: 4 additions & 0 deletions pkg/app/server_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -229,7 +229,9 @@ func TestFullScrapeCycle(t *testing.T) {
# HELP kube_pod_spec_volumes_persistentvolumeclaims_info [STABLE] Information about persistentvolumeclaim volumes in a pod.
# HELP kube_pod_spec_volumes_persistentvolumeclaims_readonly [STABLE] Describes whether a persistentvolumeclaim is mounted read only.
# HELP kube_pod_start_time [STABLE] Start time in unix timestamp for a pod.
# HELP kube_pod_status_container_ready_time Readiness achieved time in unix timestamp for a pod containers.
# HELP kube_pod_status_phase [STABLE] The pods current phase.
# HELP kube_pod_status_ready_time Readiness achieved time in unix timestamp for a pod.
# HELP kube_pod_status_ready [STABLE] Describes whether the pod is ready to serve requests.
# HELP kube_pod_status_reason The pod status reasons
# HELP kube_pod_status_scheduled [STABLE] Describes the status of the scheduling process for the pod.
Expand Down Expand Up @@ -275,8 +277,10 @@ func TestFullScrapeCycle(t *testing.T) {
# TYPE kube_pod_spec_volumes_persistentvolumeclaims_info gauge
# TYPE kube_pod_spec_volumes_persistentvolumeclaims_readonly gauge
# TYPE kube_pod_start_time gauge
# TYPE kube_pod_status_container_ready_time gauge
# TYPE kube_pod_status_phase gauge
# TYPE kube_pod_status_ready gauge
# TYPE kube_pod_status_ready_time gauge
# TYPE kube_pod_status_reason gauge
# TYPE kube_pod_status_scheduled gauge
# TYPE kube_pod_status_scheduled_time gauge
Expand Down