feat: 支持清理引用不存在的Node TencentBlueKing#2306

cnlkl · Jun 27, 2024 · 20fe740 · 20fe740
1 parent 2ae546f
commit 20fe740
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 4 deletions.
diff --git a/.../biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/clean/DeletedNodeCleanupJob.kt b/.../biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/clean/DeletedNodeCleanupJob.kt
@@ -168,7 +168,9 @@ class DeletedNodeCleanupJob(
             if (node.sha256.isNullOrEmpty() || node.sha256 == FAKE_SHA256) return
             try {
                 val credentialsKey = getCredentialsKey(node.projectId, node.repoName)
-                if (!decrementFileReferences(node.sha256, credentialsKey)) {
+                val deletedDays = node.deleted?.let { Duration.between(it, LocalDateTime.now()).toDays() } ?: 0
+                val keepRefLostNode = deletedDays < properties.keepRefLostNodeDays
+                if (!decrementFileReferences(node.sha256, credentialsKey, true) && keepRefLostNode) {
                     logger.warn("Clean up node fail collection[$collectionName], node[$node]")
                     return
                 }
@@ -185,7 +187,7 @@ class DeletedNodeCleanupJob(
         context.fileCount.addAndGet(result?.deletedCount ?: 0)
     }
 
-    private fun decrementFileReferences(sha256: String, credentialsKey: String?): Boolean {
+    private fun decrementFileReferences(sha256: String, credentialsKey: String?, createIfNotExists: Boolean): Boolean {
         val collectionName = COLLECTION_FILE_REFERENCE + MongoShardingUtils.shardingSequence(sha256, SHARDING_COUNT)
         val criteria = buildCriteria(sha256, credentialsKey)
         criteria.and(FileReference::count.name).gt(0)
@@ -200,6 +202,12 @@ class DeletedNodeCleanupJob(
 
         val newQuery = Query(buildCriteria(sha256, credentialsKey))
         mongoTemplate.findOne<FileReference>(newQuery, collectionName) ?: run {
+            if (createIfNotExists) {
+                /* 早期FileReferenceCleanupJob在最终存储不存在时，不会判断对应的node是否存在而是直接删除引用，
+                 * 导致出现node存在而引用不存在的情况，此处为这些引用缺失的数据补偿创建引用以清理对应的node及存储
+                 */
+                mongoTemplate.upsert(newQuery, Update().inc(FileReference::count.name, 0), collectionName)
+            }
             logger.error("Failed to decrement reference of file [$sha256] on credentialsKey [$credentialsKey]")
             return false
         }
@@ -253,7 +261,9 @@ class DeletedNodeCleanupJob(
             keySet.add(defaultCredentials.key)
         }
         keySet.forEach {
-            decrementFileReferences(sha256, it)
+            // 由于不确定node在哪个存储，此处无法确定为丢失引用的node创建哪个存储的引用，因此引用丢失时不补偿创建引用
+            // StorageReconcileJob中会为缺少引用的存储文件补偿创建引用
+            decrementFileReferences(sha256, it, false)
         }
     }
 

diff --git a/...c/main/kotlin/com/tencent/bkrepo/job/config/properties/DeletedNodeCleanupJobProperties.kt b/...c/main/kotlin/com/tencent/bkrepo/job/config/properties/DeletedNodeCleanupJobProperties.kt
@@ -33,5 +33,9 @@ import org.springframework.boot.context.properties.ConfigurationProperties
 @ConfigurationProperties(value = "job.deleted-node-cleanup")
 class DeletedNodeCleanupJobProperties(
     override var cron: String = "0 0 2/6 * * ?",
-    var deletedNodeReserveDays: Long = 15L
+    var deletedNodeReserveDays: Long = 15L,
+    /**
+     * 保留引用丢失的node天数
+     */
+    var keepRefLostNodeDays: Long = 15L,
 ) : MongodbJobProperties()