feat: 目录统计支持使用redis TencentBlueKing#2313

zacYL · Jul 5, 2024 · 9479a0a · 9479a0a
1 parent 263948d
commit 9479a0a
Show file tree

Hide file tree

Showing 6 changed files with 262 additions and 6 deletions.
diff --git a/.../src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/ActiveProjectNodeFolderStatJob.kt b/.../src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/ActiveProjectNodeFolderStatJob.kt
@@ -27,10 +27,12 @@
 
 package com.tencent.bkrepo.job.batch.task.stat
 
+import com.tencent.bkrepo.common.api.constant.StringPool
 import com.tencent.bkrepo.job.FOLDER
 import com.tencent.bkrepo.job.batch.base.ActiveProjectService
 import com.tencent.bkrepo.job.batch.base.JobContext
 import com.tencent.bkrepo.job.batch.context.NodeFolderJobContext
+import com.tencent.bkrepo.job.batch.utils.FolderUtils
 import com.tencent.bkrepo.job.config.properties.ActiveProjectNodeFolderStatJobProperties
 import org.slf4j.LoggerFactory
 import org.springframework.boot.context.properties.EnableConfigurationProperties
@@ -46,7 +48,7 @@ import java.time.Duration
 @Component
 @EnableConfigurationProperties(ActiveProjectNodeFolderStatJobProperties::class)
 class ActiveProjectNodeFolderStatJob(
-    properties: ActiveProjectNodeFolderStatJobProperties,
+    val properties: ActiveProjectNodeFolderStatJobProperties,
     executor: ThreadPoolTaskExecutor,
     private val activeProjectService: ActiveProjectService,
     private val mongoTemplate: MongoTemplate,
@@ -65,6 +67,13 @@ class ActiveProjectNodeFolderStatJob(
         return Criteria().and(FOLDER).`is`(false)
     }
 
+    override fun beforeRunProject(projectId: String) {
+        if (properties.userMemory) return
+        // 每次任务启动前要将redis上对应的key清理， 避免干扰
+        val key = KEY_PREFIX + FolderUtils.buildCacheKey(projectId = projectId, repoName = StringPool.EMPTY)
+        nodeFolderStat.removeRedisKey(key)
+    }
+
     override fun runRow(row: StatNode, context: JobContext) {
         require(context is NodeFolderJobContext)
         val node = nodeFolderStat.buildNode(
@@ -76,7 +85,12 @@ class ActiveProjectNodeFolderStatJob(
             folder = row.folder,
             size = row.size
         )
-        nodeFolderStat.collectNode(node, context)
+        nodeFolderStat.collectNode(
+            node = node,
+            context = context,
+            useMemory = properties.userMemory,
+            keyPrefix = KEY_PREFIX
+        )
     }
 
     override fun createJobContext(): NodeFolderJobContext {
@@ -94,8 +108,22 @@ class ActiveProjectNodeFolderStatJob(
      */
     override fun onRunProjectFinished(collection: String, projectId: String, context: JobContext) {
         require(context is NodeFolderJobContext)
+        if (!properties.userMemory) {
+            nodeFolderStat.updateRedisCache(
+                context = context,
+                force = true,
+                keyPrefix = KEY_PREFIX,
+                collectionName = null,
+                projectId = projectId
+            )
+        }
+
         logger.info("store memory cache to db with projectId $projectId")
-        nodeFolderStat.storeMemoryCacheToDB(context, collection, projectId)
+        if (properties.userMemory) {
+            nodeFolderStat.storeMemoryCacheToDB(context, collection, projectId)
+        } else {
+            nodeFolderStat.storeRedisCacheToDB(context, KEY_PREFIX, collection, projectId)
+        }
     }
 
     /**
@@ -105,5 +133,6 @@ class ActiveProjectNodeFolderStatJob(
 
     companion object {
         private val logger = LoggerFactory.getLogger(ActiveProjectNodeFolderStatJob::class.java)
+        private val KEY_PREFIX = "activeProjectNode"
     }
 }
diff --git a/...rc/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/InactiveProjectNodeFolderStatJob.kt b/...rc/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/InactiveProjectNodeFolderStatJob.kt
@@ -27,6 +27,7 @@
 
 package com.tencent.bkrepo.job.batch.task.stat
 
+import com.tencent.bkrepo.common.api.constant.StringPool
 import com.tencent.bkrepo.job.DELETED_DATE
 import com.tencent.bkrepo.job.FOLDER
 import com.tencent.bkrepo.job.IGNORE_PROJECT_PREFIX_LIST
@@ -36,6 +37,7 @@ import com.tencent.bkrepo.job.batch.base.ActiveProjectService
 import com.tencent.bkrepo.job.batch.base.DefaultContextMongoDbJob
 import com.tencent.bkrepo.job.batch.base.JobContext
 import com.tencent.bkrepo.job.batch.context.NodeFolderJobContext
+import com.tencent.bkrepo.job.batch.utils.FolderUtils
 import com.tencent.bkrepo.job.batch.utils.StatUtils.specialRepoRunCheck
 import com.tencent.bkrepo.job.config.properties.InactiveProjectNodeFolderStatJobProperties
 import org.slf4j.LoggerFactory
@@ -104,10 +106,17 @@ class InactiveProjectNodeFolderStatJob(
             folder = row.folder,
             size = row.size
         )
-        nodeFolderStat.collectNode(node, context, collectionName)
+        nodeFolderStat.collectNode(
+            node = node,
+            context = context,
+            useMemory = properties.userMemory,
+            keyPrefix = KEY_PREFIX,
+            collectionName = collectionName
+        )
     }
 
     override fun createJobContext(): NodeFolderJobContext {
+        beforeRunCollection()
         val temp = mutableMapOf<String, Boolean>()
         activeProjectService.getActiveProjects().forEach {
             temp[it] = true
@@ -117,12 +126,40 @@ class InactiveProjectNodeFolderStatJob(
         )
     }
 
+    private fun beforeRunCollection() {
+        if (properties.userMemory) return
+        // 每次任务启动前要将redis上对应的key清理， 避免干扰
+        collectionNames().forEach {
+            val key = KEY_PREFIX + FolderUtils.buildCacheKey(
+                collectionName = it, projectId = StringPool.EMPTY
+            )
+            nodeFolderStat.removeRedisKey(key)
+        }
+    }
+
     override fun onRunCollectionFinished(collectionName: String, context: JobContext) {
         super.onRunCollectionFinished(collectionName, context)
         require(context is NodeFolderJobContext)
         // 当表执行完成后，将属于该表的所有记录写入数据库
         logger.info("store memory cache to db withe table $collectionName")
         nodeFolderStat.storeMemoryCacheToDB(context, collectionName, runCollection = true)
+
+        if (!properties.userMemory) {
+            nodeFolderStat.updateRedisCache(
+                context = context,
+                force = true,
+                keyPrefix = KEY_PREFIX,
+                collectionName = collectionName,
+            )
+        }
+
+        if (properties.userMemory) {
+            logger.info("store memory cache to db withe table $collectionName")
+            nodeFolderStat.storeMemoryCacheToDB(context, collectionName, runCollection = true)
+        } else {
+            logger.info("store redis cache to db withe table $collectionName")
+            nodeFolderStat.storeRedisCacheToDB(context, KEY_PREFIX, collectionName, runCollection = true)
+        }
     }
 
     /**
@@ -169,5 +206,7 @@ class InactiveProjectNodeFolderStatJob(
     companion object {
         private val logger = LoggerFactory.getLogger(InactiveProjectNodeFolderStatJob::class.java)
         private const val COLLECTION_NAME_PREFIX = "node_"
+        private val KEY_PREFIX = "inactiveProjectNode"
+
     }
 }