From 636d3022f6fb0a2cf112cb4b610bef998fd15bf3 Mon Sep 17 00:00:00 2001 From: Maxime Leroy <19607336+maxime1907@users.noreply.github.com> Date: Mon, 22 May 2023 12:15:39 +0200 Subject: [PATCH] feat(pvc): support kube_persistentvolumeclaim_deletion_timestamp Signed-off-by: Maxime Leroy <19607336+maxime1907@users.noreply.github.com> --- docs/persistentvolumeclaim-metrics.md | 30 ++++++++++++++++- internal/store/persistentvolumeclaim.go | 22 +++++++++++++ internal/store/persistentvolumeclaim_test.go | 34 ++++++++++++++++++++ 3 files changed, 85 insertions(+), 1 deletion(-) diff --git a/docs/persistentvolumeclaim-metrics.md b/docs/persistentvolumeclaim-metrics.md index 2c92bc8247..761aa150df 100644 --- a/docs/persistentvolumeclaim-metrics.md +++ b/docs/persistentvolumeclaim-metrics.md @@ -9,8 +9,36 @@ | kube_persistentvolumeclaim_resource_requests_storage_bytes | Gauge | | | `namespace`=<persistentvolumeclaim-namespace>
`persistentvolumeclaim`=<persistentvolumeclaim-name> | STABLE | | kube_persistentvolumeclaim_status_condition | Gauge | | | `namespace` =<persistentvolumeclaim-namespace>
`persistentvolumeclaim`=<persistentvolumeclaim-name>
`type`=<persistentvolumeclaim-condition-type>
`status`=<true\false\unknown> | EXPERIMENTAL | | kube_persistentvolumeclaim_status_phase | Gauge | | | `namespace`=<persistentvolumeclaim-namespace>
`persistentvolumeclaim`=<persistentvolumeclaim-name>
`phase`=<Pending\Bound\Lost> | STABLE | -| kube_persistentvolumeclaim_created | Gauge | Unix Creation Timestamp | seconds | `namespace`=<persistentvolumeclaim-namespace>
`persistentvolumeclaim`=<persistentvolumeclaim-name> | EXPERIMENTAL | +| kube_persistentvolumeclaim_created | Gauge | Unix creation timestamp | seconds | `namespace`=<persistentvolumeclaim-namespace>
`persistentvolumeclaim`=<persistentvolumeclaim-name> | EXPERIMENTAL | +| kube_persistentvolumeclaim_deletion_timestamp | Gauge | Unix deletion timestamp | seconds | `namespace`=<persistentvolumeclaim-namespace>
`persistentvolumeclaim`=<persistentvolumeclaim-name> | EXPERIMENTAL | Note: - An empty string will be used if PVC has no storage class. + +## Useful metrics queries + +### How to retrieve non-standard PVC state + +It is not straightforward to get the PVC states for certain cases like "Terminating" since it is not stored behind a field in the `PersistentVolumeClaim.Status`. + +So to mimic the [logic](https://github.com/kubernetes/kubernetes/blob/v1.17.3/pkg/printers/internalversion/printers.go#L1402) used by the `kubectl` command line, you will need to compose multiple metrics. + +For example: + +* For PVCs in `Terminating` state: `count(kube_persistentvolumeclaim_deletion_timestamp) by (namespace, persistentvolumeclaim) * count(kube_persistentvolumeclaim_status_phase{phase="Bound"} == 1) by (namespace, persistentvolumeclaim)` + +Here is an example of a Prometheus rule that can be used to alert on a PVC that has been in the `Terminating` state for more than `5m`. + +```yaml +groups: +- name: PVC state + rules: + - alert: PVCBlockedInTerminatingState + expr: count(kube_persistentvolumeclaim_deletion_timestamp) by (namespace, persistentvolumeclaim) * count(kube_persistentvolumeclaim_status_phase{phase="Bound"} == 1) by (namespace, persistentvolumeclaim) > 0 + for: 5m + labels: + severity: page + annotations: + summary: PVC {{$labels.namespace}}/{{$labels.persistentvolumeclaim}} blocked in Terminating state. +``` diff --git a/internal/store/persistentvolumeclaim.go b/internal/store/persistentvolumeclaim.go index 637382d260..128809da9f 100644 --- a/internal/store/persistentvolumeclaim.go +++ b/internal/store/persistentvolumeclaim.go @@ -226,6 +226,28 @@ func persistentVolumeClaimMetricFamilies(allowAnnotationsList, allowLabelsList [ }) } + return &metric.Family{ + Metrics: ms, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + "kube_persistentvolumeclaim_deletion_timestamp", + "Unix deletion timestamp", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapPersistentVolumeClaimFunc(func(p *v1.PersistentVolumeClaim) *metric.Family { + ms := []*metric.Metric{} + + if p.DeletionTimestamp != nil && !p.DeletionTimestamp.IsZero() { + ms = append(ms, &metric.Metric{ + LabelKeys: []string{}, + LabelValues: []string{}, + Value: float64(p.DeletionTimestamp.Unix()), + }) + } + return &metric.Family{ Metrics: ms, } diff --git a/internal/store/persistentvolumeclaim_test.go b/internal/store/persistentvolumeclaim_test.go index 39c5c6d824..a226259b7a 100644 --- a/internal/store/persistentvolumeclaim_test.go +++ b/internal/store/persistentvolumeclaim_test.go @@ -276,6 +276,40 @@ func TestPersistentVolumeClaimStore(t *testing.T) { `, MetricNames: []string{"kube_persistentvolumeclaim_created", "kube_persistentvolumeclaim_info", "kube_persistentvolumeclaim_status_phase", "kube_persistentvolumeclaim_resource_requests_storage_bytes", "kube_persistentvolumeclaim_annotations", "kube_persistentvolumeclaim_labels", "kube_persistentvolumeclaim_access_mode", "kube_persistentvolumeclaim_status_condition"}, }, + { + Obj: &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "terminating-data", + CreationTimestamp: metav1.Time{Time: time.Unix(1500000000, 0)}, + DeletionTimestamp: &metav1.Time{Time: time.Unix(1800000000, 0)}, + }, + Spec: v1.PersistentVolumeClaimSpec{ + AccessModes: []v1.PersistentVolumeAccessMode{ + v1.ReadWriteOnce, + }, + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + VolumeName: "pvc-postgresql-data", + }, + Status: v1.PersistentVolumeClaimStatus{ + Phase: v1.ClaimBound, + }, + }, + Want: ` + # HELP kube_persistentvolumeclaim_deletion_timestamp Unix deletion timestamp + # HELP kube_persistentvolumeclaim_status_phase [STABLE] The phase the persistent volume claim is currently in. + # TYPE kube_persistentvolumeclaim_deletion_timestamp gauge + # TYPE kube_persistentvolumeclaim_status_phase gauge + kube_persistentvolumeclaim_deletion_timestamp{namespace="",persistentvolumeclaim="terminating-data"} 1.8e+09 + kube_persistentvolumeclaim_status_phase{namespace="",persistentvolumeclaim="terminating-data",phase="Bound"} 1 + kube_persistentvolumeclaim_status_phase{namespace="",persistentvolumeclaim="terminating-data",phase="Lost"} 0 + kube_persistentvolumeclaim_status_phase{namespace="",persistentvolumeclaim="terminating-data",phase="Pending"} 0 +`, + MetricNames: []string{"kube_persistentvolumeclaim_deletion_timestamp", "kube_persistentvolumeclaim_status_phase"}, + }, } for i, c := range cases { c.Func = generator.ComposeMetricGenFuncs(persistentVolumeClaimMetricFamilies(c.AllowAnnotationsList, c.AllowLabelsList))