Skip to content

Commit

Permalink
feat: 空目录清理支持使用redis TencentBlueKing#2313
Browse files Browse the repository at this point in the history
  • Loading branch information
@zacYL
zacYL committed Jul 5, 2024
1 parent 9479a0a commit d5ea46e
Show file tree
Hide file tree
Showing 3 changed files with 250 additions and 20 deletions.
55 changes: 46 additions & 9 deletions .../main/kotlin/com/tencent/bkrepo/job/batch/task/stat/ActiveProjectEmptyFolderCleanupJob.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,11 @@

package com.tencent.bkrepo.job.batch.task.stat

import com.tencent.bkrepo.common.api.constant.StringPool
import com.tencent.bkrepo.job.batch.base.ActiveProjectService
import com.tencent.bkrepo.job.batch.base.JobContext
import com.tencent.bkrepo.job.batch.context.EmptyFolderCleanupJobContext
import com.tencent.bkrepo.job.batch.utils.FolderUtils
import com.tencent.bkrepo.job.config.properties.ActiveProjectEmptyFolderCleanupJobProperties
import org.slf4j.LoggerFactory
import org.springframework.boot.context.properties.EnableConfigurationProperties
Expand All @@ -45,7 +47,7 @@ import java.time.Duration
@Component
@EnableConfigurationProperties(ActiveProjectEmptyFolderCleanupJobProperties::class)
class ActiveProjectEmptyFolderCleanupJob(
private val properties: ActiveProjectEmptyFolderCleanupJobProperties,
val properties: ActiveProjectEmptyFolderCleanupJobProperties,
executor: ThreadPoolTaskExecutor,
private val activeProjectService: ActiveProjectService,
private val mongoTemplate: MongoTemplate,
Expand All @@ -59,6 +61,14 @@ class ActiveProjectEmptyFolderCleanupJob(
logger.info("empty folder cleanup job for active projects finished")
}


override fun beforeRunProject(projectId: String) {
if (properties.userMemory) return
// 每次任务启动前要将redis上对应的key清理， 避免干扰
val key = KEY_PREFIX + FolderUtils.buildCacheKey(projectId = projectId, repoName = StringPool.EMPTY)
emptyFolderCleanup.removeRedisKey(key)
}

override fun runRow(row: StatNode, context: JobContext) {
require(context is EmptyFolderCleanupJobContext)
try {
Expand All @@ -71,7 +81,12 @@ class ActiveProjectEmptyFolderCleanupJob(
folder = row.folder,
size = row.size
)
emptyFolderCleanup.collectEmptyFolder(node, context)
emptyFolderCleanup.collectEmptyFolderWithMemory(
row = node,
context = context,
keyPrefix = KEY_PREFIX,
useMemory = properties.userMemory
)
} catch (e: Exception) {
logger.error("run empty folder clean for Node $row failed, ${e.message}")
}
Expand All @@ -94,16 +109,38 @@ class ActiveProjectEmptyFolderCleanupJob(
override fun onRunProjectFinished(collection: String, projectId: String, context: JobContext) {
require(context is EmptyFolderCleanupJobContext)
logger.info("will filter empty folder in project $projectId")
emptyFolderCleanup.emptyFolderHandler(
collection = collection,
context = context,
deletedEmptyFolder = properties.deletedEmptyFolder,
projectId = projectId,
deleteFolderRepos = properties.deleteFolderRepos
)

if (!properties.userMemory) {
emptyFolderCleanup.collectEmptyFolderWithRedis(
context = context,
force = true,
keyPrefix = KEY_PREFIX,
collectionName = null,
projectId = projectId
)
}
if (properties.userMemory) {
emptyFolderCleanup.emptyFolderHandlerWithMemory(
collection = collection,
context = context,
deletedEmptyFolder = properties.deletedEmptyFolder,
projectId = projectId,
deleteFolderRepos = properties.deleteFolderRepos
)
} else {
emptyFolderCleanup.emptyFolderHandlerWithRedis(
collection = collection,
keyPrefix = KEY_PREFIX,
context = context,
deletedEmptyFolder = properties.deletedEmptyFolder,
projectId = projectId,
deleteFolderRepos = properties.deleteFolderRepos
)
}
}

companion object {
private val logger = LoggerFactory.getLogger(ActiveProjectEmptyFolderCleanupJob::class.java)
private val KEY_PREFIX = "activeEmptyFolder"
}
}
161 changes: 158 additions & 3 deletions .../job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/EmptyFolderCleanup.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,26 +34,30 @@ import com.tencent.bkrepo.job.DELETED_DATE
import com.tencent.bkrepo.job.FOLDER
import com.tencent.bkrepo.job.FULL_PATH
import com.tencent.bkrepo.job.LAST_MODIFIED_DATE
import com.tencent.bkrepo.job.NODE_NUM
import com.tencent.bkrepo.job.PROJECT
import com.tencent.bkrepo.job.REPO
import com.tencent.bkrepo.job.SIZE
import com.tencent.bkrepo.job.batch.context.EmptyFolderCleanupJobContext
import com.tencent.bkrepo.job.batch.utils.FolderUtils
import com.tencent.bkrepo.job.batch.utils.FolderUtils.extractFolderInfoFromCacheKey
import com.tencent.bkrepo.job.pojo.FolderInfo
import org.bson.types.ObjectId
import org.slf4j.LoggerFactory
import org.springframework.data.mongodb.core.MongoTemplate
import org.springframework.data.mongodb.core.query.Criteria
import org.springframework.data.mongodb.core.query.Query
import org.springframework.data.mongodb.core.query.Update
import org.springframework.data.mongodb.core.query.isEqualTo
import org.springframework.data.redis.core.HashOperations
import org.springframework.data.redis.core.RedisTemplate
import org.springframework.data.redis.core.ScanOptions
import org.springframework.stereotype.Component
import java.time.LocalDateTime

@Component
class EmptyFolderCleanup(
private val mongoTemplate: MongoTemplate,
private val redisTemplate: RedisTemplate<String, String>,
) {

fun buildNode(
Expand All @@ -76,9 +80,15 @@ class EmptyFolderCleanup(
)
}

fun collectEmptyFolder(
fun removeRedisKey(key: String) {
redisTemplate.delete(key)
}

fun collectEmptyFolderWithMemory(
row: Node,
context: EmptyFolderCleanupJobContext,
useMemory: Boolean,
keyPrefix: String,
collectionName: String? = null
) {
if (row.folder) {
Expand All @@ -105,9 +115,64 @@ class EmptyFolderCleanup(
folderMetric.nodeNum.increment()
}
}
if (!useMemory) {
collectEmptyFolderWithRedis(
context = context,
keyPrefix = keyPrefix,
projectId = row.projectId,
collectionName = collectionName
)
}
}

fun emptyFolderHandler(
/**
* 将存储在内存中的临时记录更新到redis
*/
fun collectEmptyFolderWithRedis(
context: EmptyFolderCleanupJobContext,
force: Boolean = false,
keyPrefix: String,
projectId: String = StringPool.EMPTY,
collectionName: String? = null
) {
if (!force && context.folders.size < 50000) return
if (context.folders.isEmpty()) return
val movedToRedis: MutableList<String> = mutableListOf()
val storedFolderPrefix = if (collectionName.isNullOrEmpty()) {
FolderUtils.buildCacheKey(collectionName = collectionName, projectId = projectId)
} else {
FolderUtils.buildCacheKey(collectionName = collectionName, projectId = StringPool.EMPTY)
}
// 避免每次设置值都创建一个 Redis 连接
redisTemplate.execute { connection ->
val hashCommands = connection.hashCommands()
for (entry in context.folders) {
if (!entry.key.startsWith(storedFolderPrefix)) continue
val folderInfo = extractFolderInfoFromCacheKey(entry.key, collectionName != null) ?: continue
val nodeNumHKey = FolderUtils.buildCacheKey(
projectId = folderInfo.projectId, repoName = folderInfo.repoName,
fullPath = folderInfo.fullPath, tag = NODE_NUM
)

val key = keyPrefix + FolderUtils.buildCacheKey(collectionName = collectionName, projectId = projectId)
hashCommands.hIncrBy(key.toByteArray(), nodeNumHKey.toByteArray(), entry.value.nodeNum.toLong())
entry.value.id?.let {
val idHKey = FolderUtils.buildCacheKey(
projectId = folderInfo.projectId, repoName = folderInfo.repoName,
fullPath = folderInfo.fullPath, tag = NODE_ID
)
hashCommands.hSet(key.toByteArray(), idHKey.toByteArray(), entry.value.id!!.toByteArray())
}
movedToRedis.add(entry.key)
}
null
}
for (key in movedToRedis) {
context.folders.remove(key)
}
}

fun emptyFolderHandlerWithMemory(
collection: String,
context: EmptyFolderCleanupJobContext,
deletedEmptyFolder: Boolean,
Expand Down Expand Up @@ -147,6 +212,87 @@ class EmptyFolderCleanup(
clearContextCache(projectId, context, collection, runCollection)
}

fun emptyFolderHandlerWithRedis(
collection: String,
keyPrefix: String,
deletedEmptyFolder: Boolean,
deleteFolderRepos: List<String>,
context: EmptyFolderCleanupJobContext,
runCollection: Boolean = false,
projectId: String = StringPool.EMPTY,
) {

val keySuffix = if (runCollection) {
FolderUtils.buildCacheKey(collectionName = collection, projectId = projectId)
} else {
FolderUtils.buildCacheKey(collectionName = null, projectId = projectId)
}
val key = keyPrefix + keySuffix
val hashOps = redisTemplate.opsForHash<String, String>()
val options = ScanOptions.scanOptions().build()
redisTemplate.execute { connection ->
val hashCommands = connection.hashCommands()
val cursor = hashCommands.hScan(key.toByteArray(), options)
while (cursor.hasNext()) {
val entry: Map.Entry<ByteArray, ByteArray> = cursor.next()
val folderInfo = extractFolderInfoFromCacheKey(String(entry.key), runCollection) ?: continue
val statInfo = getFolderStatInfo(
key, entry, folderInfo, hashOps
)
if (statInfo.nodeNum > 0) continue
if (emptyFolderDoubleCheck(
projectId = folderInfo.projectId,
repoName = folderInfo.repoName,
path = folderInfo.fullPath,
collectionName = collection
)) {
val deletedFlag = deletedFolderFlag(
repoName = folderInfo.repoName,
deletedEmptyFolder = deletedEmptyFolder,
deleteFolderRepos = deleteFolderRepos
)
logger.info(
"will delete empty folder ${folderInfo.fullPath}" +
" in repo ${folderInfo.projectId}|${folderInfo.repoName} " +
"with config deletedFlag: $deletedFlag"
)
doEmptyFolderDelete(statInfo.id, collection, deletedFlag)
context.totalDeletedNum.increment()
}
}
}
redisTemplate.delete(key)
}

/**
* 从redis中获取对应目录的统计信息
*/
private fun getFolderStatInfo(
key: String,
entry: Map.Entry<ByteArray, ByteArray>,
folderInfo: FolderInfo,
hashOps: HashOperations<String, String, String>
): StatInfo {
val id: String
val nodeNum: Long
if (String(entry.key).endsWith(SIZE)) {
val nodeNumKey = FolderUtils.buildCacheKey(
projectId = folderInfo.projectId, repoName = folderInfo.repoName,
fullPath = folderInfo.fullPath, tag = NODE_NUM
)
id = String(entry.value)
nodeNum = hashOps.get(key, nodeNumKey)?.toLongOrNull() ?: 0
} else {
val idKey = FolderUtils.buildCacheKey(
projectId = folderInfo.projectId, repoName = folderInfo.repoName,
fullPath = folderInfo.fullPath, tag = NODE_ID
)
nodeNum = String(entry.value).toLongOrNull() ?: 0
id = hashOps.get(key, idKey) ?: StringPool.EMPTY
}
return StatInfo(id, nodeNum)
}

/**
* 只针对指定的generic仓库可以进行删除
* 即使全局配置的deletedEmptyFolder是true
Expand Down Expand Up @@ -237,8 +383,17 @@ class EmptyFolderCleanup(
val repoName: String,
)

data class StatInfo(
var id: String?,
var nodeNum: Long
)


companion object {
private val logger = LoggerFactory.getLogger(EmptyFolderCleanup::class.java)
const val FULL_PATH_IDX = "projectId_repoName_fullPath_idx"
private const val NODE_NUM = "nodeNum"
private const val NODE_ID = "nodeId"

}
}
Loading

0 comments on commit d5ea46e

Please sign in to comment.