Add alert for multiple default virt storage classes

Signed-off-by: Alex Kalenyuk <akalenyu@redhat.com>
kubevirt · Oct 3, 2023 · 44f4a6c · 44f4a6c
1 parent 7e94eb5
commit 44f4a6c
Show file tree

Hide file tree

Showing 6 changed files with 76 additions and 5 deletions.
diff --git a/cmd/cdi-controller/controller.go b/cmd/cdi-controller/controller.go
@@ -369,6 +369,7 @@ func getTokenPrivateKey() *rsa.PrivateKey {
 func registerMetrics() {
 	metrics.Registry.MustRegister(controller.IncompleteProfileGauge)
 	controller.IncompleteProfileGauge.Set(-1)
+	metrics.Registry.MustRegister(controller.DefaultVirtStorageClassesGauge)
 	metrics.Registry.MustRegister(controller.DataImportCronOutdatedGauge)
 }
 

diff --git a/doc/metrics.md b/doc/metrics.md
@@ -13,6 +13,8 @@ CDI install ready. Type: Gauge.
 DataImportCron has an outdated import. Type: Gauge.
 ### kubevirt_cdi_dataimportcron_outdated_aggregated
 Total count of outdated DataImportCron imports. Type: Gauge.
+### kubevirt_cdi_default_virt_storageclasses
+Number of default virt storage classes currently configured. Type: Gauge.
 ### kubevirt_cdi_import_pods_high_restart
 The number of CDI import pods with high restart count. Type: Gauge.
 ### kubevirt_cdi_incomplete_storageprofiles

diff --git a/pkg/controller/storageprofile-controller.go b/pkg/controller/storageprofile-controller.go
@@ -41,6 +41,12 @@ var (
 			Name: monitoring.MetricOptsList[monitoring.IncompleteProfile].Name,
 			Help: monitoring.MetricOptsList[monitoring.IncompleteProfile].Help,
 		})
+	// DefaultVirtStorageClassesGauge is the metric we use to alert about multiple default virt storage classes
+	DefaultVirtStorageClassesGauge = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Name: monitoring.MetricOptsList[monitoring.DefaultVirtClasses].Name,
+			Help: monitoring.MetricOptsList[monitoring.DefaultVirtClasses].Help,
+		})
 )
 
 // StorageProfileReconciler members
@@ -72,7 +78,7 @@ func (r *StorageProfileReconciler) Reconcile(_ context.Context, req reconcile.Re
 		return reconcile.Result{}, err
 	}
 
-	return reconcile.Result{}, r.checkIncompleteProfiles()
+	return reconcile.Result{}, r.computeMetrics()
 }
 
 func (r *StorageProfileReconciler) reconcileStorageProfile(sc *storagev1.StorageClass) (reconcile.Result, error) {
@@ -255,7 +261,7 @@ func (r *StorageProfileReconciler) deleteStorageProfile(name string, log logr.Lo
 		return err
 	}
 
-	return r.checkIncompleteProfiles()
+	return r.computeMetrics()
 }
 
 func isNoProvisioner(name string, cl client.Client) bool {
@@ -266,6 +272,33 @@ func isNoProvisioner(name string, cl client.Client) bool {
 	return storageClass.Provisioner == "kubernetes.io/no-provisioner"
 }
 
+func (r *StorageProfileReconciler) computeMetrics() error {
+	if err := r.checkIncompleteProfiles(); err != nil {
+		return err
+	}
+	if err := r.checkDefaultVirtStorageClasses(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (r *StorageProfileReconciler) checkDefaultVirtStorageClasses() error {
+	defaultVirtStorageClassCount := 0
+	storageClassList := &storagev1.StorageClassList{}
+	if err := r.client.List(context.TODO(), storageClassList); err != nil {
+		return err
+	}
+	for _, sc := range storageClassList.Items {
+		if sc.Annotations[cc.AnnDefaultVirtStorageClass] == "true" {
+			defaultVirtStorageClassCount++
+		}
+	}
+	DefaultVirtStorageClassesGauge.Set(float64(defaultVirtStorageClassCount))
+
+	return nil
+}
+
 func (r *StorageProfileReconciler) checkIncompleteProfiles() error {
 	numIncomplete := 0
 	storageProfileList := &cdiv1.StorageProfileList{}

diff --git a/pkg/monitoring/metrics.go b/pkg/monitoring/metrics.go
@@ -25,10 +25,11 @@ type MetricsKey string
 
 // All metrics names for reference
 const (
-	ReadyGauge             MetricsKey = "readyGauge"
-	IncompleteProfile      MetricsKey = "incompleteProfile"
-	DataImportCronOutdated MetricsKey = "dataImportCronOutdated"
 	CloneProgress          MetricsKey = "cloneProgress"
+	DataImportCronOutdated MetricsKey = "dataImportCronOutdated"
+	IncompleteProfile      MetricsKey = "incompleteProfile"
+	ReadyGauge             MetricsKey = "readyGauge"
+	DefaultVirtClasses     MetricsKey = "defaultVirtClasses"
 )
 
 // MetricOptsList list all CDI metrics
@@ -53,6 +54,11 @@ var MetricOptsList = map[MetricsKey]MetricOpts{
 		Help: "CDI install ready",
 		Type: "Gauge",
 	},
+	DefaultVirtClasses: {
+		Name: "kubevirt_cdi_default_virt_storageclasses",
+		Help: "Number of default virt storage classes currently configured",
+		Type: "Gauge",
+	},
 }
 
 // GetRecordRulesDesc returns CDI Prometheus Record Rules

diff --git a/pkg/operator/controller/prometheus.go b/pkg/operator/controller/prometheus.go
@@ -221,6 +221,21 @@ func getAlertRules(runbookURLTemplate string) []promv1.Rule {
 				componentAlertLabelKey:    componentAlertLabelValue,
 			},
 		),
+		generateAlertRule(
+			"CDIMultipleDefaultVirtStorageClasses",
+			"kubevirt_cdi_default_virt_storageclasses > 1",
+			promv1.Duration("5m"),
+			map[string]string{
+				"summary":     "More than one default virtualization StorageClass detected",
+				"runbook_url": fmt.Sprintf(runbookURLTemplate, "CDIMultipleDefaultVirtStorageClasses"),
+			},
+			map[string]string{
+				severityAlertLabelKey:     "warning",
+				healthImpactAlertLabelKey: "warning",
+				partOfAlertLabelKey:       partOfAlertLabelValue,
+				componentAlertLabelKey:    componentAlertLabelValue,
+			},
+		),
 	}
 }
 

diff --git a/tests/monitoring_test.go b/tests/monitoring_test.go
@@ -24,6 +24,7 @@ import (
 	cdiv1 "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1"
 
 	"kubevirt.io/containerized-data-importer/pkg/controller"
+	cc "kubevirt.io/containerized-data-importer/pkg/controller/common"
 	"kubevirt.io/containerized-data-importer/pkg/storagecapabilities"
 	"kubevirt.io/containerized-data-importer/tests/framework"
 	"kubevirt.io/containerized-data-importer/tests/utils"
@@ -172,6 +173,19 @@ var _ = Describe("[Destructive] Monitoring Tests", func() {
 			}
 		})
 
+		It("[test_id:XXXX] CDIMultipleDefaultVirtStorageClasses fired when more than one default virt storage class exists", func() {
+			numAddedStorageClasses = 2
+			for i := 0; i < numAddedStorageClasses; i++ {
+				sc := createUnknownStorageClass(fmt.Sprintf("unknown-sc-%d", i), "kubernetes.io/non-existent-provisioner")
+				cc.AddAnnotation(sc, cc.AnnDefaultVirtStorageClass, "true")
+				_, err := f.K8sClient.StorageV1().StorageClasses().Create(context.TODO(), sc, metav1.CreateOptions{})
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			By("Check that the CDIMultipleDefaultVirtStorageClasses alert is triggered")
+			waitForPrometheusAlert(f, "CDIMultipleDefaultVirtStorageClasses")
+		})
+
 		It("[test_id:9659] StorageProfile incomplete metric expected value remains unchanged for provisioner known to not work", func() {
 			sc, err := f.K8sClient.StorageV1().StorageClasses().Create(context.TODO(), createUnknownStorageClass("unsupported-provisioner", storagecapabilities.ProvisionerNoobaa), metav1.CreateOptions{})
 			Expect(err).ToNot(HaveOccurred())