AI大模型融入 #10825

scripts/bkenv.properties

@@ -470,6 +470,10 @@ BK_CI_API_TOKEN_EXPIRED_MILLISECOND=86400000
 BK_CI_DISPATCH_KUBERNETES_NS=default
 # BK_CI_DISPATCH_THIRD_AGENT_WORKER_ERROR_TEMPLATE dispatch服务发送worker启动失败的模板名称，无需修改
 BK_CI_DISPATCH_THIRD_AGENT_WORKER_ERROR_TEMPLATE=THIRD_AGENT_WORKER_ERROR
+# BK_CI_GPT_GATEWAY 大模型网关https地址
+BK_CI_GPT_GATEWAY=
+# BK_CI_GPT_HEADERS 大模型网关请求附加header 比如{'Authorization': 'authorization-value'}
+BK_CI_GPT_HEADERS={}
 ##########
 # 5-api port
 ##########

src/backend/ci/core/log/api-log/src/main/kotlin/com/tencent/devops/common/log/utils/BuildLogPrinter.kt

@@ -130,6 +130,28 @@ class BuildLogPrinter(
         stepId = stepId
     )
 
+    fun addAIErrorLine(
+        buildId: String,
+        message: String,
+        tag: String,
+        containerHashId: String? = null,
+        executeCount: Int,
+        subTag: String? = null,
+        jobId: String?,
+        stepId: String?
+    ) {
+        addErrorLine(
+            buildId = buildId,
+            message = "$LOG_AI_FLAG$message",
+            tag = tag,
+            containerHashId = containerHashId,
+            executeCount = executeCount,
+            subTag = subTag,
+            jobId = jobId,
+            stepId = stepId
+        )
+    }
+
     fun addErrorLine(
         buildId: String,
         message: String,
@@ -383,5 +405,7 @@ class BuildLogPrinter(
         private const val LOG_ERROR_FLAG = "##[error]"
 
         private const val LOG_WARN_FLAG = "##[warning]"
+
+        private const val LOG_AI_FLAG = "##[ai]"
     }
 }

src/backend/ci/core/log/api-log/src/main/kotlin/com/tencent/devops/log/api/ServiceLogResource.kt

@@ -100,7 +100,10 @@ interface ServiceLogResource {
         stepId: String?,
         @Parameter(description = "是否查询归档数据", required = false)
         @QueryParam("archiveFlag")
-        archiveFlag: Boolean? = false
+        archiveFlag: Boolean? = false,
+        @Parameter(description = "查询结果是否倒序，默认false", required = false)
+        @QueryParam("reverse")
+        reverse: Boolean? = false
     ): Result<QueryLogs>
 
     @Operation(summary = "获取更多日志")

src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/lucene/LuceneClient.kt

@@ -27,11 +27,15 @@
 
 package com.tencent.devops.log.lucene
 
+import com.tencent.devops.common.log.constant.Constants
 import com.tencent.devops.common.log.pojo.LogLine
 import com.tencent.devops.common.log.pojo.enums.LogType
 import com.tencent.devops.common.redis.RedisOperation
 import com.tencent.devops.log.service.IndexService
-import com.tencent.devops.common.log.constant.Constants
+import java.io.File
+import java.sql.Date
+import java.text.SimpleDateFormat
+import javax.ws.rs.core.StreamingOutput
 import org.apache.lucene.document.Document
 import org.apache.lucene.document.IntPoint
 import org.apache.lucene.document.NumericDocValuesField
@@ -50,10 +54,6 @@ import org.apache.lucene.search.TermQuery
 import org.apache.lucene.store.Directory
 import org.apache.lucene.store.FSDirectory
 import org.slf4j.LoggerFactory
-import java.io.File
-import java.sql.Date
-import java.text.SimpleDateFormat
-import javax.ws.rs.core.StreamingOutput
 
 @Suppress("LongParameterList", "TooManyFunctions", "MagicNumber")
 class LuceneClient constructor(
@@ -88,7 +88,8 @@ class LuceneClient constructor(
         executeCount: Int?,
         size: Int? = null,
         jobId: String?,
-        stepId: String?
+        stepId: String?,
+        reverse: Boolean?
     ): MutableList<LogLine> {
         val lineNum = size ?: Constants.SCROLL_MAX_LINES
         val query = prepareQueryBuilder(
@@ -103,7 +104,7 @@ class LuceneClient constructor(
             stepId = stepId
         ).build()
         logger.info("[$buildId] fetchInitLogs with query: $query")
-        return doQueryLogsInSize(buildId, query, lineNum)
+        return doQueryLogsInSize(buildId = buildId, query = query, size = lineNum, reverse = reverse)
     }
 
     fun fetchLogs(
@@ -137,7 +138,7 @@ class LuceneClient constructor(
             .add(NumericDocValuesField.newSlowRangeQuery("lineNo", lower, upper), BooleanClause.Occur.MUST)
             .build()
         logger.info("[$buildId] fetchLogsInRange with query: $query")
-        return doQueryLogsInSize(buildId, query, logSize)
+        return doQueryLogsInSize(buildId = buildId, query = query, size = logSize, reverse = false)
     }
 
     fun fetchLogsCount(
@@ -298,10 +299,15 @@ class LuceneClient constructor(
         }
     }
 
-    private fun doQueryLogsInSize(buildId: String, query: BooleanQuery, size: Int): MutableList<LogLine> {
+    private fun doQueryLogsInSize(
+        buildId: String,
+        query: BooleanQuery,
+        size: Int,
+        reverse: Boolean?
+    ): MutableList<LogLine> {
         val searcher = prepareSearcher(buildId)
         try {
-            val topDocs = searcher.search(query, size, getQuerySort())
+            val topDocs = searcher.search(query, size, getQuerySort(reverse))
             return topDocs.scoreDocs.map {
                 val hit = searcher.doc(it.doc)
                 genLogLine(hit)
@@ -411,8 +417,8 @@ class LuceneClient constructor(
         )
     }
 
-    private fun getQuerySort(): Sort {
-        return Sort(SortedNumericSortField("timestamp", SortField.Type.LONG, false))
+    private fun getQuerySort(reverse: Boolean? = null): Sort {
+        return Sort(SortedNumericSortField("timestamp", SortField.Type.LONG, reverse ?: false))
     }
 
     companion object {

src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/resources/BuildLogPrintResourceImpl.kt

@@ -226,7 +226,8 @@ class BuildLogPrintResourceImpl @Autowired constructor(
             containerHashId = jobId,
             executeCount = executeCount,
             jobId = null,
-            stepId = null
+            stepId = null,
+            reverse = false
         )
         recordMultiLogCount(initLogs.data?.logs?.size ?: 0)
         return initLogs

src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/resources/ServiceLogResourceImpl.kt

@@ -64,7 +64,8 @@ class ServiceLogResourceImpl @Autowired constructor(
         subTag: String?,
         jobId: String?,
         stepId: String?,
-        archiveFlag: Boolean?
+        archiveFlag: Boolean?,
+        reverse: Boolean?
     ): Result<QueryLogs> {
         return buildLogQueryService.getInitLogs(
             userId = userId,
@@ -79,7 +80,8 @@ class ServiceLogResourceImpl @Autowired constructor(
             subTag = subTag,
             jobId = jobId,
             stepId = stepId,
-            archiveFlag = archiveFlag
+            archiveFlag = archiveFlag,
+            reverse = reverse ?: false
         )
     }
 

src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/resources/UserLogResourceImpl.kt

@@ -85,7 +85,8 @@ class UserLogResourceImpl @Autowired constructor(
             executeCount = executeCount,
             jobId = null,
             stepId = null,
-            archiveFlag = archiveFlag
+            archiveFlag = archiveFlag,
+            reverse = false
         )
         recordListLogCount(initLogs.data?.logs?.size ?: 0)
         return initLogs

src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/service/BuildLogQueryService.kt

@@ -66,7 +66,8 @@ class BuildLogQueryService @Autowired constructor(
         subTag: String? = null,
         jobId: String?,
         stepId: String?,
-        archiveFlag: Boolean? = null
+        archiveFlag: Boolean? = null,
+        reverse: Boolean?
     ): Result<QueryLogs> {
         validateAuth(
             userId = userId,
@@ -88,7 +89,8 @@ class BuildLogQueryService @Autowired constructor(
                 containerHashId = containerHashId,
                 executeCount = executeCount,
                 jobId = jobId,
-                stepId = stepId
+                stepId = stepId,
+                reverse = reverse
             )
             result.timeUsed = System.currentTimeMillis() - startEpoch
             success = logStatusSuccess(result.status)

src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/service/LogService.kt

@@ -54,7 +54,8 @@ interface LogService {
         containerHashId: String?,
         executeCount: Int?,
         jobId: String?,
-        stepId: String?
+        stepId: String?,
+        reverse: Boolean?
     ): QueryLogs
 
     fun queryLogsBetweenLines(

src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/service/impl/LogServiceESImpl.kt

@@ -194,7 +194,8 @@ class LogServiceESImpl(
         containerHashId: String?,
         executeCount: Int?,
         jobId: String?,
-        stepId: String?
+        stepId: String?,
+        reverse: Boolean?
     ): QueryLogs {
         return doQueryInitLogs(
             buildId = buildId,
@@ -205,7 +206,8 @@ class LogServiceESImpl(
             containerHashId = containerHashId,
             executeCount = executeCount,
             jobId = jobId,
-            stepId = stepId
+            stepId = stepId,
+            reverse = reverse
         )
     }
 
@@ -751,7 +753,8 @@ class LogServiceESImpl(
         containerHashId: String? = null,
         executeCount: Int?,
         jobId: String?,
-        stepId: String?
+        stepId: String?,
+        reverse: Boolean?
     ): QueryLogs {
         val (queryLogs, index) = getQueryLogs(
             buildId = buildId,
@@ -794,6 +797,7 @@ class LogServiceESImpl(
                 "[$index|$buildId|$tag|$subTag|$containerHashId|$executeCount] " +
                         "doQueryInitLogs get the query builder: $boolQueryBuilder"
             )
+            val sortOrder = if (reverse == true) SortOrder.DESC else SortOrder.ASC
 
             val searchRequest = SearchRequest(index)
                 .source(
@@ -802,8 +806,8 @@ class LogServiceESImpl(
                         .docValueField("lineNo")
                         .docValueField("timestamp")
                         .size(Constants.NORMAL_MAX_LINES)
-                        .sort("timestamp", SortOrder.ASC)
-                        .sort("lineNo", SortOrder.ASC)
+                        .sort("timestamp", sortOrder)
+                        .sort("lineNo", sortOrder)
                         .timeout(TimeValue.timeValueSeconds(SEARCH_TIMEOUT_SECONDS))
                 )
             queryLogs.logs = searchByClient(buildId, searchRequest)

src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/service/impl/LogServiceLuceneImpl.kt

@@ -139,7 +139,8 @@ class LogServiceLuceneImpl constructor(
         containerHashId: String?,
         executeCount: Int?,
         jobId: String?,
-        stepId: String?
+        stepId: String?,
+        reverse: Boolean?
     ): QueryLogs {
         return doQueryInitLogs(
             buildId = buildId,
@@ -150,7 +151,8 @@ class LogServiceLuceneImpl constructor(
             containerHashId = containerHashId,
             executeCount = executeCount,
             jobId = jobId,
-            stepId = stepId
+            stepId = stepId,
+            reverse = reverse
         )
     }
 
@@ -534,7 +536,8 @@ class LogServiceLuceneImpl constructor(
         containerHashId: String? = null,
         executeCount: Int?,
         jobId: String?,
-        stepId: String?
+        stepId: String?,
+        reverse: Boolean?
     ): QueryLogs {
         val startTime = System.currentTimeMillis()
         val (queryLogs, index) = getQueryLogs(
@@ -569,7 +572,8 @@ class LogServiceLuceneImpl constructor(
                 containerHashId = containerHashId,
                 executeCount = executeCount,
                 jobId = jobId,
-                stepId = stepId
+                stepId = stepId,
+                reverse = reverse
             )
             logger.info("logs query time cost: ${System.currentTimeMillis() - startTime}")
             queryLogs.logs.addAll(logs)

src/backend/ci/core/misc/api-gpt/build.gradle.kts

@@ -0,0 +1,36 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2019 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+dependencies {
+    api(project(":core:common:common-api"))
+    api(project(":core:common:common-web"))
+    api(project(":core:common:common-event"))
+}
+
+plugins {
+    `task-deploy-to-maven`
+}

src/backend/ci/core/misc/api-gpt/src/main/kotlin/com/tencent/devops/gpt/api/GPTConfigResource.kt

@@ -0,0 +1,57 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2019 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.devops.gpt.api
+
+import com.tencent.devops.common.api.auth.AUTH_HEADER_USER_ID
+import com.tencent.devops.common.api.auth.AUTH_HEADER_USER_ID_DEFAULT_VALUE
+import com.tencent.devops.common.api.pojo.Result
+import io.swagger.v3.oas.annotations.Operation
+import io.swagger.v3.oas.annotations.Parameter
+import io.swagger.v3.oas.annotations.tags.Tag
+import javax.ws.rs.Consumes
+import javax.ws.rs.GET
+import javax.ws.rs.HeaderParam
+import javax.ws.rs.Path
+import javax.ws.rs.Produces
+import javax.ws.rs.core.MediaType
+
+@Tag(name = "USER_GPT", description = "ai服务-配置")
+@Path("/user/gpt_config")
+@Produces(MediaType.APPLICATION_JSON)
+@Consumes(MediaType.APPLICATION_JSON)
+interface GPTConfigResource {
+
+    @Operation(summary = "是否已实装大模型")
+    @GET
+    @Path("/is_ok")
+    fun gptCheck(
+        @Parameter(description = "用户ID", required = true, example = AUTH_HEADER_USER_ID_DEFAULT_VALUE)
+        @HeaderParam(AUTH_HEADER_USER_ID)
+        userId: String
+    ): Result<Boolean>
+}