feat: add a function to retrieve the user's backup size. (#4780)

* Add a function to retrieve the user's backup size. * db backup monitor --------- Co-authored-by: jiahui <bxy4543@gmail.com>
labring · Jun 28, 2024 · a1442f9 · a1442f9
1 parent 7d5f5a1
commit a1442f9
Show file tree

Hide file tree

Showing 3 changed files with 83 additions and 15 deletions.
diff --git a/controllers/pkg/objectstorage/objectstorage.go b/controllers/pkg/objectstorage/objectstorage.go
@@ -240,3 +240,41 @@ func isUsageBytesTargetMetric(name string) bool {
 func getUserWithBucket(bucket string) string {
 	return strings.Split(bucket, "-")[0]
 }
+
+/*
+/pvc-03392f69-a7b0-4a52-a839-82d9c586d96f/ns-eemtkfj3/halo-faxdridb-pg-bd87bb8c-a128-44cc-b076-6228593b16ee/postgresql/halo-faxdridb-pg-yhxnjm/halo-faxdridb-pg-yhxnjm.tar.gz
+	1/pvc-03392f69-a7b0-4a52-a839-82d9c586d96f
+		2/pvc-03392f69-a7b0-4a52-a839-82d9c586d96f
+		2/ns-eemtkfj3
+			3/ns-eemtkfj3
+			3/halo-faxdridb-pg-bd87bb8c-a128-44cc-b076-6228593b16ee
+				4/halo-faxdridb-pg-bd87bb8c-a128-44cc-b076-6228593b16ee
+				4/postgresql
+					5/postgresql
+					5/halo-faxdridb-pg-yhxnjm
+						6/halo-faxdridb-pg-yhxnjm
+						6/halo-faxdridb-pg-yhxnjm.tar.gz
+*/
+
+func GetUserBakFileSize(client *minio.Client) map[string]int64 {
+	bucket := "file-backup"
+	userUsageMap := make(map[string]int64)
+	objectsCh := client.ListObjects(context.Background(), bucket, minio.ListObjectsOptions{Recursive: true})
+	for object := range objectsCh {
+		user := extractNamespace(object.Key)
+		if user != "" {
+			userUsageMap[user] += object.Size
+		}
+	}
+
+	return userUsageMap
+}
+
+func extractNamespace(input string) string {
+	re := regexp.MustCompile(`ns-(\w+)`)
+	matches := re.FindStringSubmatch(input)
+	if len(matches) < 2 {
+		return ""
+	}
+	return matches[1]
+}
diff --git a/controllers/pkg/objectstorage/objectstorage_test.go b/controllers/pkg/objectstorage/objectstorage_test.go
@@ -61,3 +61,15 @@ func TestQueryUserUsage(t *testing.T) {
 		fmt.Println(metric)
 	}
 }
+
+func TestGetUserBakFileSize(t *testing.T) {
+	objStorageClient, err := objectstoragev1.NewOSClient("objectstorageapi.192.168.0.55.nip.io", "username", "passw0rd")
+	if err != nil {
+		t.Fatalf("NewOSClient error: %v", err)
+	}
+	bakFileSize := GetUserBakFileSize(objStorageClient)
+	if err != nil {
+		t.Fatalf("GetUserBakFileSize error: %v", err)
+	}
+	fmt.Println(bakFileSize)
+}
diff --git a/controllers/resources/controllers/monitor_controller.go b/controllers/resources/controllers/monitor_controller.go
@@ -62,20 +62,21 @@ import (
 type MonitorReconciler struct {
 	client.Client
 	logr.Logger
-	Interval                time.Duration
-	Scheme                  *runtime.Scheme
-	stopCh                  chan struct{}
-	wg                      sync.WaitGroup
-	periodicReconcile       time.Duration
-	NvidiaGpu               map[string]gpu.NvidiaGPU
-	DBClient                database.Interface
-	TrafficClient           database.Interface
-	Properties              *resources.PropertyTypeLS
-	PromURL                 string
-	currentObjectMetrics    map[string]objstorage.MetricData
-	ObjStorageClient        *minio.Client
-	ObjStorageMetricsClient *objstorage.MetricsClient
-	ObjectStorageInstance   string
+	Interval                 time.Duration
+	Scheme                   *runtime.Scheme
+	stopCh                   chan struct{}
+	wg                       sync.WaitGroup
+	periodicReconcile        time.Duration
+	NvidiaGpu                map[string]gpu.NvidiaGPU
+	DBClient                 database.Interface
+	TrafficClient            database.Interface
+	Properties               *resources.PropertyTypeLS
+	PromURL                  string
+	currentObjectMetrics     map[string]objstorage.MetricData
+	ObjStorageClient         *minio.Client
+	ObjStorageMetricsClient  *objstorage.MetricsClient
+	ObjStorageUserBackupSize map[string]int64
+	ObjectStorageInstance    string
 }
 
 type quantity struct {
@@ -271,6 +272,8 @@ func (r *MonitorReconciler) preMonitorResourceUsage() error {
 		r.currentObjectMetrics = metrics
 		logger.Info("success query object storage resource usage", "time", time.Now().Format("2006-01-02 15:04:05"))
 	}
+	r.ObjStorageUserBackupSize = objstorage.GetUserBakFileSize(r.ObjStorageClient)
+	fmt.Println("ObjStorageUserBackupSize", r.ObjStorageUserBackupSize)
 	return nil
 }
 
@@ -324,7 +327,10 @@ func (r *MonitorReconciler) monitorResourceUsage(namespace *corev1.Namespace) er
 		return fmt.Errorf("failed to list pvc: %v", err)
 	}
 	for _, pvc := range pvcList.Items {
-		if pvc.Status.Phase != corev1.ClaimBound || pvc.Name == resources.KubeBlocksBackUpName {
+		if pvc.Status.Phase != corev1.ClaimBound {
+			continue
+		}
+		if len(pvc.OwnerReferences) > 0 && pvc.OwnerReferences[0].Kind == "BackupRepo" {
 			continue
 		}
 		pvcRes := resources.NewResourceNamed(&pvc)
@@ -334,6 +340,14 @@ func (r *MonitorReconciler) monitorResourceUsage(namespace *corev1.Namespace) er
 		}
 		resUsed[pvcRes.String()][corev1.ResourceStorage].Add(pvc.Spec.Resources.Requests[corev1.ResourceStorage])
 	}
+	if backupSize := r.ObjStorageUserBackupSize[getBackupObjectStorageName(namespace.Name)]; backupSize > 0 {
+		backupRes := resources.NewObjStorageResourceNamed("DB-BACKUP")
+		if resUsed[backupRes.String()] == nil {
+			resNamed[backupRes.String()] = backupRes
+			resUsed[backupRes.String()] = initResources()
+		}
+		resUsed[backupRes.String()][corev1.ResourceStorage].Add(*resource.NewQuantity(backupSize, resource.BinarySI))
+	}
 	svcList := corev1.ServiceList{}
 	if err := r.List(context.Background(), &svcList, &client.ListOptions{Namespace: namespace.Name}); err != nil {
 		return fmt.Errorf("failed to list svc: %v", err)
@@ -378,6 +392,10 @@ func (r *MonitorReconciler) monitorResourceUsage(namespace *corev1.Namespace) er
 	return r.DBClient.InsertMonitor(context.Background(), monitors...)
 }
 
+func getBackupObjectStorageName(namespace string) string {
+	return strings.TrimPrefix(namespace, "ns-")
+}
+
 func (r *MonitorReconciler) getResourceUsed(podResource map[corev1.ResourceName]*quantity) (bool, map[uint8]int64) {
 	used := map[uint8]int64{}
 	isEmpty := true