diff --git a/src/backend/common/common-artifact/artifact-cache/src/main/kotlin/com/tencent/bkrepo/common/artifact/cache/service/impl/ArtifactAccessRecorder.kt b/src/backend/common/common-artifact/artifact-cache/src/main/kotlin/com/tencent/bkrepo/common/artifact/cache/service/impl/ArtifactAccessRecorder.kt index a4129d0a41..d0d10adfcf 100644 --- a/src/backend/common/common-artifact/artifact-cache/src/main/kotlin/com/tencent/bkrepo/common/artifact/cache/service/impl/ArtifactAccessRecorder.kt +++ b/src/backend/common/common-artifact/artifact-cache/src/main/kotlin/com/tencent/bkrepo/common/artifact/cache/service/impl/ArtifactAccessRecorder.kt @@ -47,13 +47,13 @@ import java.time.format.DateTimeFormatter /** * 记录制品访问时间，用于统计项目制品使用习惯 */ -class ArtifactAccessRecorder( +open class ArtifactAccessRecorder( private val preloadProperties: ArtifactPreloadProperties, private val artifactAccessRecordDao: ArtifactAccessRecordDao, ) { @Async - fun onArtifactAccess(node: NodeDetail, cacheMiss: Boolean) { + open fun onArtifactAccess(node: NodeDetail, cacheMiss: Boolean) { val validNode = !node.folder && node.size > preloadProperties.minSize.toBytes() val shouldNotRecord = preloadProperties.onlyRecordCacheMiss && !cacheMiss if (!preloadProperties.enabled || shouldNotRecord || !validNode) { @@ -117,7 +117,7 @@ class ArtifactAccessRecorder( @Async @EventListener(ArtifactResponseEvent::class) - fun listen(event: ArtifactResponseEvent) { + open fun listen(event: ArtifactResponseEvent) { safeRecordArtifactCacheAccess(event.artifactResource) } diff --git a/src/backend/common/common-artifact/artifact-cache/src/main/kotlin/com/tencent/bkrepo/common/artifact/cache/service/impl/CacheFileEventListener.kt b/src/backend/common/common-artifact/artifact-cache/src/main/kotlin/com/tencent/bkrepo/common/artifact/cache/service/impl/CacheFileEventListener.kt index 2ed23f6a18..9dba4d4537 100644 --- a/src/backend/common/common-artifact/artifact-cache/src/main/kotlin/com/tencent/bkrepo/common/artifact/cache/service/impl/CacheFileEventListener.kt +++ b/src/backend/common/common-artifact/artifact-cache/src/main/kotlin/com/tencent/bkrepo/common/artifact/cache/service/impl/CacheFileEventListener.kt @@ -37,7 +37,7 @@ import org.springframework.scheduling.annotation.Async /** * 缓存文件相关事件监听器 */ -class CacheFileEventListener( +open class CacheFileEventListener( private val properties: ArtifactPreloadProperties, private val preloadPlanService: ArtifactPreloadPlanService, ) { @@ -47,7 +47,7 @@ class CacheFileEventListener( */ @Async @EventListener(CacheFileDeletedEvent::class) - fun onCacheFileDeleted(event: CacheFileDeletedEvent) { + open fun onCacheFileDeleted(event: CacheFileDeletedEvent) { if (properties.enabled && event.data.size >= properties.minSize.toBytes()) { with(event.data) { logger.info("try generate preload plan for sha256[${sha256}], fullPath[$fullPath], size[$size") diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/usage/ProjectDailyAvgMetricsJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/usage/ProjectDailyAvgMetricsJob.kt index 5b23564e66..f12406f010 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/usage/ProjectDailyAvgMetricsJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/usage/ProjectDailyAvgMetricsJob.kt @@ -77,22 +77,21 @@ class ProjectDailyAvgMetricsJob( override fun getLockAtMostFor(): Duration = Duration.ofDays(1) private fun doStoreProjectDailyAvgRecord(currentDate: LocalDateTime) { - val criteria = Criteria.where(ProjectMetricsDailyRecord::createdDate.name).lte(currentDate) - .gte(currentDate.minusDays(1)) + val yesterday = currentDate.minusDays(1).format(DateTimeFormatter.ofPattern("yyyyMMdd")) + logger.info("start to store avg record for $yesterday") + val criteria = Criteria.where(ProjectMetricsDailyRecord::createdDay.name).isEqualTo(yesterday) val query = Query(criteria) val data = mongoTemplate.findDistinct( query, ProjectMetricsDailyRecord::projectId.name, COLLECTION_NAME_PROJECT_METRICS_DAILY_RECORD, String::class.java ) data.forEach { - val criteria = Criteria.where(PROJECT).isEqualTo(it) - .and(ProjectMetricsDailyRecord::createdDate.name).lte(currentDate) - .gte(currentDate.minusDays(1)) - val query = Query.query(criteria) + val projectCriteria = Criteria.where(PROJECT).isEqualTo(it).andOperator(criteria) + val projectQuery = Query.query(projectCriteria) var capSize = 0L var count = 0 mongoTemplate.find( - query, ProjectMetricsDailyRecord::class.java, + projectQuery, ProjectMetricsDailyRecord::class.java, COLLECTION_NAME_PROJECT_METRICS_DAILY_RECORD ).forEach { capSize += it.capSize @@ -126,6 +125,8 @@ class ProjectDailyAvgMetricsJob( val productId = projectInfo.metadata.firstOrNull { it.key == ProjectMetadata.KEY_PRODUCT_ID }?.value as? Int val bgId = projectInfo.metadata.firstOrNull { it.key == ProjectMetadata.KEY_BG_ID }?.value as? String ?: StringPool.EMPTY + val enabled = projectInfo.metadata.firstOrNull { it.key == ProjectMetadata.KEY_ENABLED }?.value as? Boolean + ?: false val dailyRecord = TProjectMetricsDailyAvgRecord( projectId = projectInfo.name, costDate = convertToCostDate(currentDate), @@ -133,7 +134,7 @@ class ProjectDailyAvgMetricsJob( usage = usage, bgName = projectInfo.metadata.firstOrNull { it.key == ProjectMetadata.KEY_BG_NAME }?.value as? String ?: StringPool.EMPTY, - flag = covertToFlag(bgId, productId), + flag = covertToFlag(projectInfo.name, bgId, productId, enabled), costDateDay = currentDate.minusDays(1).format( DateTimeFormatter.ofPattern("yyyyMMdd") ), @@ -157,13 +158,21 @@ class ProjectDailyAvgMetricsJob( } private fun covertToFlag( + projectId: String, bgId: String, - productId: Int? + productId: Int?, + enabled: Boolean ): Boolean { + val streamReport = if (properties.reportStream) { + true + } else { + !projectId.startsWith(GIT_PROJECT_PREFIX) + } return if (properties.bgIds.isEmpty()) { - bgId.isNotBlank() && productId != null + streamReport && bgId.isNotBlank() && productId != null && enabled } else { - properties.bgIds.contains(bgId) && bgId.isNotBlank() && productId != null + streamReport && properties.bgIds.contains(bgId) + && bgId.isNotBlank() && productId != null && enabled } } @@ -172,7 +181,7 @@ class ProjectDailyAvgMetricsJob( val minusMonth = if (day >= properties.monthStartDay) { -1 } else { - 1 + 0 } return currentDate.minusMonths(minusMonth.toLong()).format( DateTimeFormatter.ofPattern("yyyyMM") @@ -188,7 +197,7 @@ class ProjectDailyAvgMetricsJob( data class ProjectMetricsDailyRecord( var projectId: String, var capSize: Long, - val createdDate: LocalDateTime, + var createdDay: String? = null ) companion object { @@ -196,5 +205,6 @@ class ProjectDailyAvgMetricsJob( private const val COLLECTION_NAME_PROJECT = "project" private const val COLLECTION_NAME_PROJECT_METRICS_DAILY_RECORD = "project_metrics_daily_record" private const val COLLECTION_NAME_PROJECT_METRICS_DAILY_AVG_RECORD = "project_metrics_daily_avg_record" + private const val GIT_PROJECT_PREFIX = "git_" } } diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/usage/ProjectMetricsReport2BkbaseJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/usage/ProjectMetricsReport2BkbaseJob.kt index df5b5cff6f..8ef58e2343 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/usage/ProjectMetricsReport2BkbaseJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/usage/ProjectMetricsReport2BkbaseJob.kt @@ -53,6 +53,7 @@ import org.springframework.stereotype.Component import java.time.Duration import java.time.LocalDate import java.time.LocalDateTime +import java.time.format.DateTimeFormatter import kotlin.reflect.KClass /** @@ -166,6 +167,9 @@ class ProjectMetricsReport2BkbaseJob( update.set(TProjectMetricsDailyRecord::dockerRepoCapSize.name, projectMetrics.dockerRepoCapSize) update.set(TProjectMetricsDailyRecord::active.name, projectMetrics.active) update.set(TProjectMetricsDailyRecord::createdDate.name, projectMetrics.createdDate) + update.set(TProjectMetricsDailyRecord::createdDay.name, projectMetrics.createdDate.format( + DateTimeFormatter.ofPattern("yyyyMMdd") + )) projectMetrics.enabled?.let { update.set(TProjectMetricsDailyRecord::enabled.name, projectMetrics.enabled) } diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/ProjectDailyAvgMetricsJobProperties.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/ProjectDailyAvgMetricsJobProperties.kt index f5bcee7dce..fd3c3ff09a 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/ProjectDailyAvgMetricsJobProperties.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/ProjectDailyAvgMetricsJobProperties.kt @@ -40,5 +40,6 @@ class ProjectDailyAvgMetricsJobProperties( var monthStartDay: Int = 15, // 每月账单截止时间 var monthEndDay: Int = 14, - var bgIds: MutableList = mutableListOf() + var bgIds: MutableList = mutableListOf(), + var reportStream: Boolean = false ) : MongodbJobProperties(enabled) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/ProjectMonthMetricReportJobProperties.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/ProjectMonthMetricReportJobProperties.kt index b186309f35..ffd37dfc7f 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/ProjectMonthMetricReportJobProperties.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/ProjectMonthMetricReportJobProperties.kt @@ -41,5 +41,5 @@ class ProjectMonthMetricReportJobProperties( var reportHost: String = "", var reportUrl: String = "", var reportPlatformKey: String = "", - var batchUploadSize: Int = 150 + var batchUploadSize: Int = 50 ) : MongodbJobProperties(enabled) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/pojo/project/TProjectMetricsDailyRecord.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/pojo/project/TProjectMetricsDailyRecord.kt index d1366ee691..13c57e6546 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/pojo/project/TProjectMetricsDailyRecord.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/pojo/project/TProjectMetricsDailyRecord.kt @@ -42,8 +42,13 @@ import java.time.LocalDateTime def = "{'projectId': 1,'createdDate': 1}", background = true, unique = true + ), + CompoundIndex( + name = "created_day_idx", + def = "{'createdDay': 1}", + background = true + ), ) -) data class TProjectMetricsDailyRecord( var projectId: String, var nodeNum: Long, @@ -55,4 +60,5 @@ data class TProjectMetricsDailyRecord( val active: Boolean = true, var enabled: Boolean?, val createdDate: LocalDateTime, + var createdDay: String? = null )