feat(pvc): support kube_persistentvolumeclaim_deletion_timestamp

Signed-off-by: Maxime Leroy <19607336+maxime1907@users.noreply.github.com>
kubernetes · Jun 7, 2023 · b3f69e0 · b3f69e0
1 parent 31d6e8f
commit b3f69e0
Show file tree

Hide file tree

Showing 3 changed files with 81 additions and 1 deletion.
diff --git a/docs/persistentvolumeclaim-metrics.md b/docs/persistentvolumeclaim-metrics.md
@@ -9,8 +9,32 @@
 | kube_persistentvolumeclaim_resource_requests_storage_bytes | Gauge | | | `namespace`=&lt;persistentvolumeclaim-namespace&gt; <br> `persistentvolumeclaim`=&lt;persistentvolumeclaim-name&gt; | STABLE |
 | kube_persistentvolumeclaim_status_condition | Gauge | | | `namespace` =&lt;persistentvolumeclaim-namespace&gt; <br> `persistentvolumeclaim`=&lt;persistentvolumeclaim-name&gt; <br> `type`=&lt;persistentvolumeclaim-condition-type&gt; <br> `status`=&lt;true\false\unknown&gt; | EXPERIMENTAL |
 | kube_persistentvolumeclaim_status_phase | Gauge | | | `namespace`=&lt;persistentvolumeclaim-namespace&gt; <br> `persistentvolumeclaim`=&lt;persistentvolumeclaim-name&gt; <br> `phase`=&lt;Pending\Bound\Lost&gt; | STABLE |
-| kube_persistentvolumeclaim_created | Gauge | Unix Creation Timestamp | seconds | `namespace`=&lt;persistentvolumeclaim-namespace&gt; <br> `persistentvolumeclaim`=&lt;persistentvolumeclaim-name&gt; | EXPERIMENTAL |
+| kube_persistentvolumeclaim_created | Gauge | Unix creation timestamp | seconds | `namespace`=&lt;persistentvolumeclaim-namespace&gt; <br> `persistentvolumeclaim`=&lt;persistentvolumeclaim-name&gt; | EXPERIMENTAL |
+| kube_persistentvolumeclaim_deletion_timestamp | Gauge | Unix deletion timestamp | seconds | `namespace`=&lt;persistentvolumeclaim-namespace&gt; <br> `persistentvolumeclaim`=&lt;persistentvolumeclaim-name&gt; | EXPERIMENTAL |
 
 Note:
 
 - An empty string will be used if PVC has no storage class.
+
+## Useful metrics queries
+
+### How to retrieve non-standard PVC state
+
+It is not straightforward to get the PVC states for certain cases like "Terminating" since it is not stored behind a field in the `PersistentVolumeClaim.Status`.
+
+So to mimic the [logic](https://github.com/kubernetes/kubernetes/blob/v1.27.2/pkg/printers/internalversion/printers.go#L1883) used by the `kubectl` command line, you will need to compose multiple metrics.
+
+Here is an example of a Prometheus rule that can be used to alert on a PVC that has been in the `Terminating` state for more than `5m`.
+
+```yaml
+groups:
+- name: PVC state
+  rules:
+  - alert: PVCBlockedInTerminatingState
+    expr: count(kube_persistentvolumeclaim_deletion_timestamp) by (namespace, persistentvolumeclaim) * count(kube_persistentvolumeclaim_status_phase{phase="Bound"} == 1) by (namespace, persistentvolumeclaim) > 0
+    for: 5m
+    labels:
+      severity: critical
+    annotations:
+      summary: PVC {{$labels.namespace}}/{{$labels.persistentvolumeclaim}} blocked in Terminating state.
+```
diff --git a/internal/store/persistentvolumeclaim.go b/internal/store/persistentvolumeclaim.go
@@ -226,6 +226,28 @@ func persistentVolumeClaimMetricFamilies(allowAnnotationsList, allowLabelsList [
 					})
 				}
 
+				return &metric.Family{
+					Metrics: ms,
+				}
+			}),
+		),
+		*generator.NewFamilyGeneratorWithStability(
+			"kube_persistentvolumeclaim_deletion_timestamp",
+			"Unix deletion timestamp",
+			metric.Gauge,
+			basemetrics.ALPHA,
+			"",
+			wrapPersistentVolumeClaimFunc(func(p *v1.PersistentVolumeClaim) *metric.Family {
+				ms := []*metric.Metric{}
+
+				if p.DeletionTimestamp != nil && !p.DeletionTimestamp.IsZero() {
+					ms = append(ms, &metric.Metric{
+						LabelKeys:   []string{},
+						LabelValues: []string{},
+						Value:       float64(p.DeletionTimestamp.Unix()),
+					})
+				}
+
 				return &metric.Family{
 					Metrics: ms,
 				}

diff --git a/internal/store/persistentvolumeclaim_test.go b/internal/store/persistentvolumeclaim_test.go
@@ -276,6 +276,40 @@ func TestPersistentVolumeClaimStore(t *testing.T) {
 `,
 			MetricNames: []string{"kube_persistentvolumeclaim_created", "kube_persistentvolumeclaim_info", "kube_persistentvolumeclaim_status_phase", "kube_persistentvolumeclaim_resource_requests_storage_bytes", "kube_persistentvolumeclaim_annotations", "kube_persistentvolumeclaim_labels", "kube_persistentvolumeclaim_access_mode", "kube_persistentvolumeclaim_status_condition"},
 		},
+		{
+			Obj: &v1.PersistentVolumeClaim{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:              "terminating-data",
+					CreationTimestamp: metav1.Time{Time: time.Unix(1500000000, 0)},
+					DeletionTimestamp: &metav1.Time{Time: time.Unix(1800000000, 0)},
+				},
+				Spec: v1.PersistentVolumeClaimSpec{
+					AccessModes: []v1.PersistentVolumeAccessMode{
+						v1.ReadWriteOnce,
+					},
+					Resources: v1.ResourceRequirements{
+						Requests: v1.ResourceList{
+							v1.ResourceStorage: resource.MustParse("1Gi"),
+						},
+					},
+					VolumeName: "pvc-postgresql-data",
+				},
+				Status: v1.PersistentVolumeClaimStatus{
+					Phase: v1.ClaimBound,
+				},
+			},
+			Want: `
+				# HELP kube_persistentvolumeclaim_deletion_timestamp Unix deletion timestamp
+				# HELP kube_persistentvolumeclaim_status_phase [STABLE] The phase the persistent volume claim is currently in.
+				# TYPE kube_persistentvolumeclaim_deletion_timestamp gauge
+				# TYPE kube_persistentvolumeclaim_status_phase gauge
+				kube_persistentvolumeclaim_deletion_timestamp{namespace="",persistentvolumeclaim="terminating-data"} 1.8e+09
+				kube_persistentvolumeclaim_status_phase{namespace="",persistentvolumeclaim="terminating-data",phase="Bound"} 1
+				kube_persistentvolumeclaim_status_phase{namespace="",persistentvolumeclaim="terminating-data",phase="Lost"} 0
+				kube_persistentvolumeclaim_status_phase{namespace="",persistentvolumeclaim="terminating-data",phase="Pending"} 0
+`,
+			MetricNames: []string{"kube_persistentvolumeclaim_deletion_timestamp", "kube_persistentvolumeclaim_status_phase"},
+		},
 	}
 	for i, c := range cases {
 		c.Func = generator.ComposeMetricGenFuncs(persistentVolumeClaimMetricFamilies(c.AllowAnnotationsList, c.AllowLabelsList))