From 2be0d9e34c1064cb6154252e77bb419001e29499 Mon Sep 17 00:00:00 2001
From: Maxime Leroy <19607336+maxime1907@users.noreply.github.com>
Date: Mon, 22 May 2023 12:40:04 +0200
Subject: [PATCH] feat(pv): support kube_persistentvolume_deletion_timestamp

Signed-off-by: Maxime Leroy <19607336+maxime1907@users.noreply.github.com>
---
 docs/persistentvolume-metrics.md        | 25 ++++++++++++++++++++++++-
 internal/store/persistentvolume.go      | 22 ++++++++++++++++++++++
 internal/store/persistentvolume_test.go | 18 ++++++++++++++++++
 3 files changed, 64 insertions(+), 1 deletion(-)

diff --git a/docs/persistentvolume-metrics.md b/docs/persistentvolume-metrics.md
index a80e045768..9d6093767b 100644
--- a/docs/persistentvolume-metrics.md
+++ b/docs/persistentvolume-metrics.md
@@ -8,5 +8,28 @@
 | kube_persistentvolume_claim_ref | Gauge | | | `persistentvolume`=&lt;pv-name&gt; <br>`claim_namespace`=&lt;<namespace>&gt; <br>`name`=&lt;<name>&gt; | STABLE |
 | kube_persistentvolume_labels | Gauge | | | `persistentvolume`=&lt;persistentvolume-name&gt; <br> `label_PERSISTENTVOLUME_LABEL`=&lt;PERSISTENTVOLUME_LABEL&gt; | STABLE |
 | kube_persistentvolume_info | Gauge | | | `persistentvolume`=&lt;pv-name&gt; <br> `storageclass`=&lt;storageclass-name&gt; <br> `gce_persistent_disk_name`=&lt;pd-name&gt; <br> `host_path`=&lt;path-of-a-host-volume&gt; <br> `host_path_type`=&lt;host-mount-type&gt; <br> `ebs_volume_id`=&lt;ebs-volume-id&gt; <br> `azure_disk_name`=&lt;azure-disk-name&gt; <br> `fc_wwids`=&lt;fc-wwids-comma-separated&gt; <br> `fc_lun`=&lt;fc-lun&gt; <br> `fc_target_wwns`=&lt;fc-target-wwns-comma-separated&gt; <br> `iscsi_target_portal`=&lt;iscsi-target-portal&gt; <br> `iscsi_iqn`=&lt;iscsi-iqn&gt; <br> `iscsi_lun`=&lt;iscsi-lun&gt; <br> `iscsi_initiator_name`=&lt;iscsi-initiator-name&gt; <br> `local_path`=&lt;path-of-a-local-volume&gt; <br> `local_fs`=&lt;local-volume-fs-type&gt; <br> `nfs_server`=&lt;nfs-server&gt; <br> `nfs_path`=&lt;nfs-path&gt; <br> `csi_driver`=&lt;csi-driver&gt; <br> `csi_volume_handle`=&lt;csi-volume-handle&gt; | STABLE |
-| kube_persistentvolume_created | Gauge | Unix Creation Timestamp | seconds | `persistentvolume`=&lt;persistentvolume-name&gt; <br> | EXPERIMENTAL |
+| kube_persistentvolume_created | Gauge | Unix creation timestamp | seconds | `persistentvolume`=&lt;persistentvolume-name&gt; <br> | EXPERIMENTAL |
+| kube_persistentvolume_deletion_timestamp | Gauge | Unix deletion timestamp | seconds | `persistentvolume`=&lt;persistentvolume-name&gt; <br> | EXPERIMENTAL |
 
+## Useful metrics queries
+
+### How to retrieve non-standard PV state
+
+It is not straightforward to get the PV states for certain cases like "Terminating" since it is not stored behind a field in the `PersistentVolume.Status`.
+
+So to mimic the [logic](https://github.com/kubernetes/kubernetes/blob/v1.27.2/pkg/printers/internalversion/printers.go#L1838) used by the `kubectl` command line, you will need to compose multiple metrics.
+
+Here is an example of a Prometheus rule that can be used to alert on a PV that has been in the `Terminating` state for more than `5m`.
+
+```yaml
+groups:
+- name: PV state
+  rules:
+  - alert: PVBlockedInTerminatingState
+    expr: kube_persistentvolume_deletion_timestamp * on(persistentvolume) group_left() (kube_persistentvolume_status_phase{phase="Bound"} == 1) > 0
+    for: 5m
+    labels:
+      severity: warning
+    annotations:
+      summary: PV {{$labels.persistentvolume}} blocked in Terminating state.
+```
diff --git a/internal/store/persistentvolume.go b/internal/store/persistentvolume.go
index 35b7ea071a..e1af75ad23 100644
--- a/internal/store/persistentvolume.go
+++ b/internal/store/persistentvolume.go
@@ -316,6 +316,28 @@ func persistentVolumeMetricFamilies(allowAnnotationsList, allowLabelsList []stri
 					})
 				}
 
+				return &metric.Family{
+					Metrics: ms,
+				}
+			}),
+		),
+		*generator.NewFamilyGeneratorWithStability(
+			"kube_persistentvolume_deletion_timestamp",
+			"Unix deletion timestamp",
+			metric.Gauge,
+			basemetrics.ALPHA,
+			"",
+			wrapPersistentVolumeFunc(func(p *v1.PersistentVolume) *metric.Family {
+				ms := []*metric.Metric{}
+
+				if p.DeletionTimestamp != nil && !p.DeletionTimestamp.IsZero() {
+					ms = append(ms, &metric.Metric{
+						LabelKeys:   []string{},
+						LabelValues: []string{},
+						Value:       float64(p.DeletionTimestamp.Unix()),
+					})
+				}
+
 				return &metric.Family{
 					Metrics: ms,
 				}
diff --git a/internal/store/persistentvolume_test.go b/internal/store/persistentvolume_test.go
index a5686d1d3c..d3f07be9ec 100644
--- a/internal/store/persistentvolume_test.go
+++ b/internal/store/persistentvolume_test.go
@@ -683,6 +683,24 @@ func TestPersistentVolumeStore(t *testing.T) {
 `,
 			MetricNames: []string{"kube_persistentvolume_created"},
 		},
+		{
+			Obj: &v1.PersistentVolume{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:              "test-pv-terminating",
+					CreationTimestamp: metav1.Time{Time: time.Unix(1500000000, 0)},
+					DeletionTimestamp: &metav1.Time{Time: time.Unix(1800000000, 0)},
+				},
+				Status: v1.PersistentVolumeStatus{
+					Phase: v1.VolumeBound,
+				},
+			},
+			Want: `
+				# HELP kube_persistentvolume_deletion_timestamp Unix deletion timestamp
+				# TYPE kube_persistentvolume_deletion_timestamp gauge
+				kube_persistentvolume_deletion_timestamp{persistentvolume="test-pv-terminating"} 1.8e+09
+`,
+			MetricNames: []string{"kube_persistentvolume_deletion_timestamp"},
+		},
 	}
 	for i, c := range cases {
 		c.Func = generator.ComposeMetricGenFuncs(persistentVolumeMetricFamilies(c.AllowAnnotationsList, c.AllowLabelsList))