feat: 已禁用项目数据不计费 TencentBlueKing#2232

* feat: 已禁用项目数据不计费 TencentBlueKing#2232 * feat: 修复启动报错问题 TencentBlueKing#2232
scplsy · Jun 4, 2024 · 27fdd3c · 27fdd3c
1 parent 8c1e5c5
commit 27fdd3c
Show file tree

Hide file tree

Showing 7 changed files with 42 additions and 21 deletions.
diff --git a/...in/kotlin/com/tencent/bkrepo/common/artifact/cache/service/impl/ArtifactAccessRecorder.kt b/...in/kotlin/com/tencent/bkrepo/common/artifact/cache/service/impl/ArtifactAccessRecorder.kt
@@ -47,13 +47,13 @@ import java.time.format.DateTimeFormatter
 /**
  * 记录制品访问时间，用于统计项目制品使用习惯
  */
-class ArtifactAccessRecorder(
+open class ArtifactAccessRecorder(
     private val preloadProperties: ArtifactPreloadProperties,
     private val artifactAccessRecordDao: ArtifactAccessRecordDao,
 ) {
 
     @Async
-    fun onArtifactAccess(node: NodeDetail, cacheMiss: Boolean) {
+    open fun onArtifactAccess(node: NodeDetail, cacheMiss: Boolean) {
         val validNode = !node.folder && node.size > preloadProperties.minSize.toBytes()
         val shouldNotRecord = preloadProperties.onlyRecordCacheMiss && !cacheMiss
         if (!preloadProperties.enabled || shouldNotRecord || !validNode) {
@@ -117,7 +117,7 @@ class ArtifactAccessRecorder(
 
     @Async
     @EventListener(ArtifactResponseEvent::class)
-    fun listen(event: ArtifactResponseEvent) {
+    open fun listen(event: ArtifactResponseEvent) {
         safeRecordArtifactCacheAccess(event.artifactResource)
     }
 

diff --git a/...in/kotlin/com/tencent/bkrepo/common/artifact/cache/service/impl/CacheFileEventListener.kt b/...in/kotlin/com/tencent/bkrepo/common/artifact/cache/service/impl/CacheFileEventListener.kt
@@ -37,7 +37,7 @@ import org.springframework.scheduling.annotation.Async
 /**
  * 缓存文件相关事件监听器
  */
-class CacheFileEventListener(
+open class CacheFileEventListener(
     private val properties: ArtifactPreloadProperties,
     private val preloadPlanService: ArtifactPreloadPlanService,
 ) {
@@ -47,7 +47,7 @@ class CacheFileEventListener(
      */
     @Async
     @EventListener(CacheFileDeletedEvent::class)
-    fun onCacheFileDeleted(event: CacheFileDeletedEvent) {
+    open fun onCacheFileDeleted(event: CacheFileDeletedEvent) {
         if (properties.enabled && event.data.size >= properties.minSize.toBytes()) {
             with(event.data) {
                 logger.info("try generate preload plan for sha256[${sha256}], fullPath[$fullPath], size[$size")

diff --git a/...-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/usage/ProjectDailyAvgMetricsJob.kt b/...-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/usage/ProjectDailyAvgMetricsJob.kt
@@ -77,22 +77,21 @@ class ProjectDailyAvgMetricsJob(
     override fun getLockAtMostFor(): Duration = Duration.ofDays(1)
 
     private fun doStoreProjectDailyAvgRecord(currentDate: LocalDateTime) {
-        val criteria = Criteria.where(ProjectMetricsDailyRecord::createdDate.name).lte(currentDate)
-            .gte(currentDate.minusDays(1))
+        val yesterday = currentDate.minusDays(1).format(DateTimeFormatter.ofPattern("yyyyMMdd"))
+        logger.info("start to store avg record for $yesterday")
+        val criteria = Criteria.where(ProjectMetricsDailyRecord::createdDay.name).isEqualTo(yesterday)
         val query = Query(criteria)
         val data = mongoTemplate.findDistinct(
             query, ProjectMetricsDailyRecord::projectId.name,
             COLLECTION_NAME_PROJECT_METRICS_DAILY_RECORD, String::class.java
         )
         data.forEach {
-            val criteria = Criteria.where(PROJECT).isEqualTo(it)
-                .and(ProjectMetricsDailyRecord::createdDate.name).lte(currentDate)
-                .gte(currentDate.minusDays(1))
-            val query = Query.query(criteria)
+            val projectCriteria = Criteria.where(PROJECT).isEqualTo(it).andOperator(criteria)
+            val projectQuery = Query.query(projectCriteria)
             var capSize = 0L
             var count = 0
             mongoTemplate.find(
-                query, ProjectMetricsDailyRecord::class.java,
+                projectQuery, ProjectMetricsDailyRecord::class.java,
                 COLLECTION_NAME_PROJECT_METRICS_DAILY_RECORD
             ).forEach {
                 capSize += it.capSize
@@ -126,14 +125,16 @@ class ProjectDailyAvgMetricsJob(
         val productId = projectInfo.metadata.firstOrNull { it.key == ProjectMetadata.KEY_PRODUCT_ID }?.value as? Int
         val bgId = projectInfo.metadata.firstOrNull { it.key == ProjectMetadata.KEY_BG_ID }?.value as? String
             ?: StringPool.EMPTY
+        val enabled = projectInfo.metadata.firstOrNull { it.key == ProjectMetadata.KEY_ENABLED }?.value as? Boolean
+            ?: false
         val dailyRecord = TProjectMetricsDailyAvgRecord(
             projectId = projectInfo.name,
             costDate = convertToCostDate(currentDate),
             name = projectInfo.displayName,
             usage = usage,
             bgName = projectInfo.metadata.firstOrNull { it.key == ProjectMetadata.KEY_BG_NAME }?.value as? String
                 ?: StringPool.EMPTY,
-            flag = covertToFlag(bgId, productId),
+            flag = covertToFlag(projectInfo.name, bgId, productId, enabled),
             costDateDay = currentDate.minusDays(1).format(
                 DateTimeFormatter.ofPattern("yyyyMMdd")
             ),
@@ -157,13 +158,21 @@ class ProjectDailyAvgMetricsJob(
     }
 
     private fun covertToFlag(
+        projectId: String,
         bgId: String,
-        productId: Int?
+        productId: Int?,
+        enabled: Boolean
     ): Boolean {
+        val streamReport = if (properties.reportStream) {
+            true
+        } else {
+            !projectId.startsWith(GIT_PROJECT_PREFIX)
+        }
         return if (properties.bgIds.isEmpty()) {
-            bgId.isNotBlank() && productId != null
+            streamReport && bgId.isNotBlank() && productId != null && enabled
         } else {
-            properties.bgIds.contains(bgId) && bgId.isNotBlank() && productId != null
+            streamReport && properties.bgIds.contains(bgId)
+                && bgId.isNotBlank() && productId != null && enabled
         }
     }
 
@@ -172,7 +181,7 @@ class ProjectDailyAvgMetricsJob(
         val minusMonth = if (day >= properties.monthStartDay) {
             -1
         } else {
-            1
+            0
         }
         return currentDate.minusMonths(minusMonth.toLong()).format(
             DateTimeFormatter.ofPattern("yyyyMM")
@@ -188,13 +197,14 @@ class ProjectDailyAvgMetricsJob(
     data class ProjectMetricsDailyRecord(
         var projectId: String,
         var capSize: Long,
-        val createdDate: LocalDateTime,
+        var createdDay: String? = null
     )
 
     companion object {
         private val logger = LoggerFactory.getLogger(ProjectDailyAvgMetricsJob::class.java)
         private const val COLLECTION_NAME_PROJECT = "project"
         private const val COLLECTION_NAME_PROJECT_METRICS_DAILY_RECORD = "project_metrics_daily_record"
         private const val COLLECTION_NAME_PROJECT_METRICS_DAILY_AVG_RECORD = "project_metrics_daily_avg_record"
+        private const val GIT_PROJECT_PREFIX = "git_"
     }
 }
diff --git a/...src/main/kotlin/com/tencent/bkrepo/job/batch/task/usage/ProjectMetricsReport2BkbaseJob.kt b/...src/main/kotlin/com/tencent/bkrepo/job/batch/task/usage/ProjectMetricsReport2BkbaseJob.kt
@@ -53,6 +53,7 @@ import org.springframework.stereotype.Component
 import java.time.Duration
 import java.time.LocalDate
 import java.time.LocalDateTime
+import java.time.format.DateTimeFormatter
 import kotlin.reflect.KClass
 
 /**
@@ -166,6 +167,9 @@ class ProjectMetricsReport2BkbaseJob(
         update.set(TProjectMetricsDailyRecord::dockerRepoCapSize.name, projectMetrics.dockerRepoCapSize)
         update.set(TProjectMetricsDailyRecord::active.name, projectMetrics.active)
         update.set(TProjectMetricsDailyRecord::createdDate.name, projectMetrics.createdDate)
+        update.set(TProjectMetricsDailyRecord::createdDay.name, projectMetrics.createdDate.format(
+            DateTimeFormatter.ofPattern("yyyyMMdd")
+        ))
         projectMetrics.enabled?.let {
             update.set(TProjectMetricsDailyRecord::enabled.name, projectMetrics.enabled)
         }

diff --git a/...in/kotlin/com/tencent/bkrepo/job/config/properties/ProjectDailyAvgMetricsJobProperties.kt b/...in/kotlin/com/tencent/bkrepo/job/config/properties/ProjectDailyAvgMetricsJobProperties.kt
@@ -40,5 +40,6 @@ class ProjectDailyAvgMetricsJobProperties(
     var monthStartDay: Int = 15,
     // 每月账单截止时间
     var monthEndDay: Int = 14,
-    var bgIds: MutableList<String> = mutableListOf<String>()
+    var bgIds: MutableList<String> = mutableListOf<String>(),
+    var reportStream: Boolean = false
 ) : MongodbJobProperties(enabled)
diff --git a/.../kotlin/com/tencent/bkrepo/job/config/properties/ProjectMonthMetricReportJobProperties.kt b/.../kotlin/com/tencent/bkrepo/job/config/properties/ProjectMonthMetricReportJobProperties.kt
@@ -41,5 +41,5 @@ class ProjectMonthMetricReportJobProperties(
     var reportHost: String = "",
     var reportUrl: String = "",
     var reportPlatformKey: String = "",
-    var batchUploadSize: Int = 150
+    var batchUploadSize: Int = 50
 ) : MongodbJobProperties(enabled)
diff --git a/...biz-job/src/main/kotlin/com/tencent/bkrepo/job/pojo/project/TProjectMetricsDailyRecord.kt b/...biz-job/src/main/kotlin/com/tencent/bkrepo/job/pojo/project/TProjectMetricsDailyRecord.kt
@@ -42,8 +42,13 @@ import java.time.LocalDateTime
         def = "{'projectId': 1,'createdDate': 1}",
         background = true,
         unique = true
+    ),
+    CompoundIndex(
+        name = "created_day_idx",
+        def = "{'createdDay': 1}",
+        background = true
+    ),
     )
-)
 data class TProjectMetricsDailyRecord(
     var projectId: String,
     var nodeNum: Long,
@@ -55,4 +60,5 @@ data class TProjectMetricsDailyRecord(
     val active: Boolean = true,
     var enabled: Boolean?,
     val createdDate: LocalDateTime,
+    var createdDay: String? = null
 )