feat: 支持bkbase拉取活跃用户与项目 TencentBlueKing#1551 (TencentBlueKing#1596)

src/backend/job/api-job/src/main/kotlin/com/tencent/bkrepo/job/api/JobClient.kt

@@ -49,4 +49,7 @@ interface JobClient {
 
     @GetMapping("/activeProjects")
     fun activeProjects(): Response<MutableSet<String>>
+
+    @GetMapping("/activeUsers")
+    fun activeUsers(): Response<Set<String>>
 }

src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/base/ActiveProjectService.kt

@@ -28,14 +28,15 @@
 package com.tencent.bkrepo.job.batch.base
 
 import com.tencent.bkrepo.common.artifact.event.base.EventType
+import com.tencent.bkrepo.common.operate.service.model.TOperateLog
 import com.tencent.bkrepo.job.IGNORE_PROJECT_PREFIX_LIST
-import com.tencent.bkrepo.job.PROJECT
-import com.tencent.bkrepo.job.TYPE
-import com.tencent.bkrepo.repository.pojo.project.ProjectInfo
+import org.slf4j.LoggerFactory
 import org.springframework.data.mongodb.core.MongoTemplate
 import org.springframework.data.mongodb.core.query.Criteria
 import org.springframework.data.mongodb.core.query.Query
-import org.springframework.data.mongodb.core.query.where
+import org.springframework.data.mongodb.core.query.inValues
+import org.springframework.data.mongodb.core.query.ne
+import org.springframework.data.mongodb.core.query.not
 import org.springframework.scheduling.annotation.Scheduled
 import org.springframework.stereotype.Component
 import java.time.LocalDate
@@ -52,6 +53,7 @@ class ActiveProjectService(
 
     private var uploadActiveProjects = mutableSetOf<String>()
 
+    private var activeUsers = mutableSetOf<String>()
 
     fun getActiveProjects(): MutableSet<String> {
         return activeProjects
@@ -65,31 +67,22 @@ class ActiveProjectService(
         return uploadActiveProjects
     }
 
-
-    private fun getActiveProjectsFromOpLog() {
-        downloadActiveProjects = getActiveProjectsFromOplog(DOWNLOAD_EVENTS)
-        uploadActiveProjects = getActiveProjectsFromOplog(UPLOAD_EVENTS)
-        activeProjects = getActiveProjectsFromOplog()
+    fun getActiveUsers(): Set<String> {
+        return activeUsers
     }
 
-    private fun getActiveProjectsFromOplog(types: List<String> = emptyList()): MutableSet<String> {
-        val tempList = mutableSetOf<String>()
+    private fun findDistinct(field: String, criteria: Criteria): MutableSet<String> {
+        val tempList = HashSet<String>()
         val months = listOf(
             LocalDate.now().format(DateTimeFormatter.ofPattern(YEAR_MONTH_FORMAT)),
             LocalDate.now().minusMonths(1)
                 .format(DateTimeFormatter.ofPattern(YEAR_MONTH_FORMAT))
         )
 
         months.forEach {
-            val collectionName = COLLECTION_NAME_PREFIX +it
-            val criteria = Criteria().andOperator(
-                IGNORE_PROJECT_PREFIX_LIST.map { where(ProjectInfo::name).not().regex(it) }
-            )
-            if (types.isNotEmpty()) {
-                criteria.and(TYPE).`in`(types)
-            }
+            val collectionName = COLLECTION_NAME_PREFIX + it
             val query = Query(criteria)
-            val data = mongoTemplate.findDistinct(query, PROJECT, collectionName, String::class.java)
+            val data = mongoTemplate.findDistinct(query, field, collectionName, String::class.java)
             tempList.addAll(data)
         }
         return tempList
@@ -100,10 +93,36 @@ class ActiveProjectService(
      */
     @Scheduled(fixedDelay = FIXED_DELAY, initialDelay = INITIAL_DELAY, timeUnit = TimeUnit.MINUTES)
     fun refreshActiveProjects() {
-        getActiveProjectsFromOpLog()
+        val criteriaList = IGNORE_PROJECT_PREFIX_LIST.mapTo(ArrayList()) { prefix ->
+            TOperateLog::projectId.not().regex("^$prefix")
+        }
+        criteriaList.add(TOperateLog::projectId.ne(""))
+        fun buildTypesCriteriaList(types: List<String>): List<Criteria> {
+            return ArrayList(criteriaList).apply { add(TOperateLog::type.inValues(types)) }
+        }
+
+        // all
+        activeProjects = findDistinct(TOperateLog::projectId.name, Criteria().andOperator(criteriaList))
+
+        // download event
+        downloadActiveProjects = findDistinct(
+            TOperateLog::projectId.name,
+            Criteria().andOperator(buildTypesCriteriaList(DOWNLOAD_EVENTS))
+        )
+
+        // upload event
+        uploadActiveProjects = findDistinct(
+            TOperateLog::projectId.name,
+            Criteria().andOperator(buildTypesCriteriaList(UPLOAD_EVENTS))
+        )
+
+        // active users
+        activeUsers = findDistinct(TOperateLog::userId.name, TOperateLog::userId.ne(""))
+        logger.info("refresh active projects and users success")
     }
 
     companion object {
+        private val logger = LoggerFactory.getLogger(ActiveProjectService::class.java)
         private const val INITIAL_DELAY = 2L
         private const val FIXED_DELAY = 60L
         private const val COLLECTION_NAME_PREFIX = "artifact_oplog_"

src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/controller/service/JobController.kt

@@ -55,4 +55,8 @@ class JobController(
     override fun activeProjects(): Response<MutableSet<String>> {
         return ResponseBuilder.success(activeProjectService.getActiveProjects())
     }
+
+    override fun activeUsers(): Response<Set<String>> {
+        return ResponseBuilder.success(activeProjectService.getActiveUsers())
+    }
 }

src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/config/OpProperties.kt

@@ -32,5 +32,9 @@ import org.springframework.boot.context.properties.ConfigurationProperties
 @ConfigurationProperties("op.security")
 data class OpProperties(
     var adminUsername: String = "",
-    var adminPassword: String = ""
+    var adminPassword: String = "",
+    /**
+     * bkbase调用制品库接口拉取数据时使用的token
+     */
+    var bkBaseToken: String = ""
 )

src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/controller/BkBaseExportController.kt

@@ -0,0 +1,69 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2023 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.opdata.controller
+
+import com.google.common.net.HttpHeaders
+import com.tencent.bkrepo.common.api.util.JsonUtils
+import com.tencent.bkrepo.common.security.exception.PermissionException
+import com.tencent.bkrepo.job.api.JobClient
+import com.tencent.bkrepo.opdata.config.OpProperties
+import org.springframework.http.MediaType
+import org.springframework.http.ResponseEntity
+import org.springframework.web.bind.annotation.GetMapping
+import org.springframework.web.bind.annotation.RequestMapping
+import org.springframework.web.bind.annotation.RequestParam
+import org.springframework.web.bind.annotation.RestController
+
+@RestController
+@RequestMapping("/api/bkbase")
+class BkBaseExportController(
+    private val opProperties: OpProperties,
+    private val jobClient: JobClient
+) {
+
+    /**
+     * 获取活跃用户及项目
+     */
+    @GetMapping("/active")
+    fun getActiveProjectAndUsers(@RequestParam token: String): ResponseEntity<Any> {
+        if (opProperties.bkBaseToken.isEmpty() || opProperties.bkBaseToken != token) {
+            throw PermissionException("invalid token")
+        }
+        val result = ActiveProjectsAndUsers(
+            jobClient.activeProjects().data ?: emptySet(),
+            jobClient.activeUsers().data ?: emptySet()
+        )
+        val response = JsonUtils.objectMapper.writeValueAsString(result)
+        return ResponseEntity.ok().header(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE).body(response)
+    }
+
+    data class ActiveProjectsAndUsers(
+        val activeProjects: Set<String>,
+        val activeUsers: Set<String>
+    )
+}

src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/controller/ProjectController.kt

@@ -82,34 +82,36 @@ class ProjectController(
      * 获取项目的统计数据
      */
     @GetMapping("/list/project/capSize")
+    @Principal(PrincipalType.ADMIN)
     fun getProjectCapSizeMetrics(
         metricsRequest: ProjectMetricsRequest
     ): Response<List<ProjectMetrics>> {
         return ResponseBuilder.success(projectMetricsService.list(metricsRequest))
     }
+
     /**
      * 获取项目的统计数据
      */
     @GetMapping("/list/project/capSize/download")
-    fun downloadProjectCapSizeMetrics(
-        metricsRequest: ProjectMetricsRequest
-    ) {
+    @Principal(PrincipalType.ADMIN)
+    fun downloadProjectCapSizeMetrics(metricsRequest: ProjectMetricsRequest) {
         projectMetricsService.download(metricsRequest)
     }
 
     /**
      * 获取活跃项目列表
      */
     @GetMapping("/list/activeProjects")
+    @Principal(PrincipalType.ADMIN)
     fun getActiveProjects(): Response<MutableSet<String>> {
-
         return ResponseBuilder.success(projectMetricsService.getActiveProjects())
     }
 
     /**
      * 获取下载活跃项目列表
      */
     @GetMapping("/list/downloadActiveProjects")
+    @Principal(PrincipalType.ADMIN)
     fun getDownloadActiveProjects(): Response<MutableSet<String>> {
         return ResponseBuilder.success(projectMetricsService.getDownloadActiveProjects())
     }
@@ -118,6 +120,7 @@ class ProjectController(
      * 获取上传活跃项目列表
      */
     @GetMapping("/list/uploadActiveProjects")
+    @Principal(PrincipalType.ADMIN)
     fun getUploadActiveProjects(): Response<MutableSet<String>> {
         return ResponseBuilder.success(projectMetricsService.getUploadActiveProjects())
     }