diff --git a/scripts/bkenv.properties b/scripts/bkenv.properties index a73ba0b7251..628d2e37252 100644 --- a/scripts/bkenv.properties +++ b/scripts/bkenv.properties @@ -470,6 +470,10 @@ BK_CI_API_TOKEN_EXPIRED_MILLISECOND=86400000 BK_CI_DISPATCH_KUBERNETES_NS=default # BK_CI_DISPATCH_THIRD_AGENT_WORKER_ERROR_TEMPLATE dispatch服务发送worker启动失败的模板名称，无需修改 BK_CI_DISPATCH_THIRD_AGENT_WORKER_ERROR_TEMPLATE=THIRD_AGENT_WORKER_ERROR +# BK_CI_GPT_GATEWAY 大模型网关https地址 +BK_CI_GPT_GATEWAY= +# BK_CI_GPT_HEADERS 大模型网关请求附加header 比如{'Authorization': 'authorization-value'} +BK_CI_GPT_HEADERS={} ########## # 5-api port ########## diff --git a/src/backend/ci/core/log/api-log/src/main/kotlin/com/tencent/devops/common/log/utils/BuildLogPrinter.kt b/src/backend/ci/core/log/api-log/src/main/kotlin/com/tencent/devops/common/log/utils/BuildLogPrinter.kt index 7683b15384a..24c2c06fc80 100644 --- a/src/backend/ci/core/log/api-log/src/main/kotlin/com/tencent/devops/common/log/utils/BuildLogPrinter.kt +++ b/src/backend/ci/core/log/api-log/src/main/kotlin/com/tencent/devops/common/log/utils/BuildLogPrinter.kt @@ -130,6 +130,28 @@ class BuildLogPrinter( stepId = stepId ) + fun addAIErrorLine( + buildId: String, + message: String, + tag: String, + containerHashId: String? = null, + executeCount: Int, + subTag: String? = null, + jobId: String?, + stepId: String? + ) { + addErrorLine( + buildId = buildId, + message = "$LOG_AI_FLAG$message", + tag = tag, + containerHashId = containerHashId, + executeCount = executeCount, + subTag = subTag, + jobId = jobId, + stepId = stepId + ) + } + fun addErrorLine( buildId: String, message: String, @@ -383,5 +405,7 @@ class BuildLogPrinter( private const val LOG_ERROR_FLAG = "##[error]" private const val LOG_WARN_FLAG = "##[warning]" + + private const val LOG_AI_FLAG = "##[ai]" } } diff --git a/src/backend/ci/core/log/api-log/src/main/kotlin/com/tencent/devops/log/api/ServiceLogResource.kt b/src/backend/ci/core/log/api-log/src/main/kotlin/com/tencent/devops/log/api/ServiceLogResource.kt index fceee1806f5..56eb34be5b7 100644 --- a/src/backend/ci/core/log/api-log/src/main/kotlin/com/tencent/devops/log/api/ServiceLogResource.kt +++ b/src/backend/ci/core/log/api-log/src/main/kotlin/com/tencent/devops/log/api/ServiceLogResource.kt @@ -100,7 +100,10 @@ interface ServiceLogResource { stepId: String?, @Parameter(description = "是否查询归档数据", required = false) @QueryParam("archiveFlag") - archiveFlag: Boolean? = false + archiveFlag: Boolean? = false, + @Parameter(description = "查询结果是否倒序，默认false", required = false) + @QueryParam("reverse") + reverse: Boolean? = false ): Result @Operation(summary = "获取更多日志") diff --git a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/lucene/LuceneClient.kt b/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/lucene/LuceneClient.kt index 1257f194655..a3cad576448 100644 --- a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/lucene/LuceneClient.kt +++ b/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/lucene/LuceneClient.kt @@ -27,11 +27,15 @@ package com.tencent.devops.log.lucene +import com.tencent.devops.common.log.constant.Constants import com.tencent.devops.common.log.pojo.LogLine import com.tencent.devops.common.log.pojo.enums.LogType import com.tencent.devops.common.redis.RedisOperation import com.tencent.devops.log.service.IndexService -import com.tencent.devops.common.log.constant.Constants +import java.io.File +import java.sql.Date +import java.text.SimpleDateFormat +import javax.ws.rs.core.StreamingOutput import org.apache.lucene.document.Document import org.apache.lucene.document.IntPoint import org.apache.lucene.document.NumericDocValuesField @@ -50,10 +54,6 @@ import org.apache.lucene.search.TermQuery import org.apache.lucene.store.Directory import org.apache.lucene.store.FSDirectory import org.slf4j.LoggerFactory -import java.io.File -import java.sql.Date -import java.text.SimpleDateFormat -import javax.ws.rs.core.StreamingOutput @Suppress("LongParameterList", "TooManyFunctions", "MagicNumber") class LuceneClient constructor( @@ -88,7 +88,8 @@ class LuceneClient constructor( executeCount: Int?, size: Int? = null, jobId: String?, - stepId: String? + stepId: String?, + reverse: Boolean? ): MutableList { val lineNum = size ?: Constants.SCROLL_MAX_LINES val query = prepareQueryBuilder( @@ -103,7 +104,7 @@ class LuceneClient constructor( stepId = stepId ).build() logger.info("[$buildId] fetchInitLogs with query: $query") - return doQueryLogsInSize(buildId, query, lineNum) + return doQueryLogsInSize(buildId = buildId, query = query, size = lineNum, reverse = reverse) } fun fetchLogs( @@ -137,7 +138,7 @@ class LuceneClient constructor( .add(NumericDocValuesField.newSlowRangeQuery("lineNo", lower, upper), BooleanClause.Occur.MUST) .build() logger.info("[$buildId] fetchLogsInRange with query: $query") - return doQueryLogsInSize(buildId, query, logSize) + return doQueryLogsInSize(buildId = buildId, query = query, size = logSize, reverse = false) } fun fetchLogsCount( @@ -298,10 +299,15 @@ class LuceneClient constructor( } } - private fun doQueryLogsInSize(buildId: String, query: BooleanQuery, size: Int): MutableList { + private fun doQueryLogsInSize( + buildId: String, + query: BooleanQuery, + size: Int, + reverse: Boolean? + ): MutableList { val searcher = prepareSearcher(buildId) try { - val topDocs = searcher.search(query, size, getQuerySort()) + val topDocs = searcher.search(query, size, getQuerySort(reverse)) return topDocs.scoreDocs.map { val hit = searcher.doc(it.doc) genLogLine(hit) @@ -411,8 +417,8 @@ class LuceneClient constructor( ) } - private fun getQuerySort(): Sort { - return Sort(SortedNumericSortField("timestamp", SortField.Type.LONG, false)) + private fun getQuerySort(reverse: Boolean? = null): Sort { + return Sort(SortedNumericSortField("timestamp", SortField.Type.LONG, reverse ?: false)) } companion object { diff --git a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/resources/BuildLogPrintResourceImpl.kt b/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/resources/BuildLogPrintResourceImpl.kt index b6e6e851521..0291ad4fe93 100644 --- a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/resources/BuildLogPrintResourceImpl.kt +++ b/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/resources/BuildLogPrintResourceImpl.kt @@ -226,7 +226,8 @@ class BuildLogPrintResourceImpl @Autowired constructor( containerHashId = jobId, executeCount = executeCount, jobId = null, - stepId = null + stepId = null, + reverse = false ) recordMultiLogCount(initLogs.data?.logs?.size ?: 0) return initLogs diff --git a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/resources/ServiceLogResourceImpl.kt b/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/resources/ServiceLogResourceImpl.kt index 74eb5b423e3..3a148710221 100644 --- a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/resources/ServiceLogResourceImpl.kt +++ b/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/resources/ServiceLogResourceImpl.kt @@ -64,7 +64,8 @@ class ServiceLogResourceImpl @Autowired constructor( subTag: String?, jobId: String?, stepId: String?, - archiveFlag: Boolean? + archiveFlag: Boolean?, + reverse: Boolean? ): Result { return buildLogQueryService.getInitLogs( userId = userId, @@ -79,7 +80,8 @@ class ServiceLogResourceImpl @Autowired constructor( subTag = subTag, jobId = jobId, stepId = stepId, - archiveFlag = archiveFlag + archiveFlag = archiveFlag, + reverse = reverse ?: false ) } diff --git a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/resources/UserLogResourceImpl.kt b/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/resources/UserLogResourceImpl.kt index af519afdd47..bcb1099be26 100644 --- a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/resources/UserLogResourceImpl.kt +++ b/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/resources/UserLogResourceImpl.kt @@ -85,7 +85,8 @@ class UserLogResourceImpl @Autowired constructor( executeCount = executeCount, jobId = null, stepId = null, - archiveFlag = archiveFlag + archiveFlag = archiveFlag, + reverse = false ) recordListLogCount(initLogs.data?.logs?.size ?: 0) return initLogs diff --git a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/service/BuildLogQueryService.kt b/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/service/BuildLogQueryService.kt index 0018de44f8c..54f05636ca2 100644 --- a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/service/BuildLogQueryService.kt +++ b/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/service/BuildLogQueryService.kt @@ -66,7 +66,8 @@ class BuildLogQueryService @Autowired constructor( subTag: String? = null, jobId: String?, stepId: String?, - archiveFlag: Boolean? = null + archiveFlag: Boolean? = null, + reverse: Boolean? ): Result { validateAuth( userId = userId, @@ -88,7 +89,8 @@ class BuildLogQueryService @Autowired constructor( containerHashId = containerHashId, executeCount = executeCount, jobId = jobId, - stepId = stepId + stepId = stepId, + reverse = reverse ) result.timeUsed = System.currentTimeMillis() - startEpoch success = logStatusSuccess(result.status) diff --git a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/service/LogService.kt b/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/service/LogService.kt index a10e6362472..88b769107fe 100644 --- a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/service/LogService.kt +++ b/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/service/LogService.kt @@ -54,7 +54,8 @@ interface LogService { containerHashId: String?, executeCount: Int?, jobId: String?, - stepId: String? + stepId: String?, + reverse: Boolean? ): QueryLogs fun queryLogsBetweenLines( diff --git a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/service/impl/LogServiceESImpl.kt b/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/service/impl/LogServiceESImpl.kt index e2f4a831e30..995c89f612d 100644 --- a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/service/impl/LogServiceESImpl.kt +++ b/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/service/impl/LogServiceESImpl.kt @@ -194,7 +194,8 @@ class LogServiceESImpl( containerHashId: String?, executeCount: Int?, jobId: String?, - stepId: String? + stepId: String?, + reverse: Boolean? ): QueryLogs { return doQueryInitLogs( buildId = buildId, @@ -205,7 +206,8 @@ class LogServiceESImpl( containerHashId = containerHashId, executeCount = executeCount, jobId = jobId, - stepId = stepId + stepId = stepId, + reverse = reverse ) } @@ -751,7 +753,8 @@ class LogServiceESImpl( containerHashId: String? = null, executeCount: Int?, jobId: String?, - stepId: String? + stepId: String?, + reverse: Boolean? ): QueryLogs { val (queryLogs, index) = getQueryLogs( buildId = buildId, @@ -794,6 +797,7 @@ class LogServiceESImpl( "[$index|$buildId|$tag|$subTag|$containerHashId|$executeCount] " + "doQueryInitLogs get the query builder: $boolQueryBuilder" ) + val sortOrder = if (reverse == true) SortOrder.DESC else SortOrder.ASC val searchRequest = SearchRequest(index) .source( @@ -802,8 +806,8 @@ class LogServiceESImpl( .docValueField("lineNo") .docValueField("timestamp") .size(Constants.NORMAL_MAX_LINES) - .sort("timestamp", SortOrder.ASC) - .sort("lineNo", SortOrder.ASC) + .sort("timestamp", sortOrder) + .sort("lineNo", sortOrder) .timeout(TimeValue.timeValueSeconds(SEARCH_TIMEOUT_SECONDS)) ) queryLogs.logs = searchByClient(buildId, searchRequest) diff --git a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/service/impl/LogServiceLuceneImpl.kt b/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/service/impl/LogServiceLuceneImpl.kt index 5f8b9edd622..d2fd656ad91 100644 --- a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/service/impl/LogServiceLuceneImpl.kt +++ b/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/service/impl/LogServiceLuceneImpl.kt @@ -139,7 +139,8 @@ class LogServiceLuceneImpl constructor( containerHashId: String?, executeCount: Int?, jobId: String?, - stepId: String? + stepId: String?, + reverse: Boolean? ): QueryLogs { return doQueryInitLogs( buildId = buildId, @@ -150,7 +151,8 @@ class LogServiceLuceneImpl constructor( containerHashId = containerHashId, executeCount = executeCount, jobId = jobId, - stepId = stepId + stepId = stepId, + reverse = reverse ) } @@ -534,7 +536,8 @@ class LogServiceLuceneImpl constructor( containerHashId: String? = null, executeCount: Int?, jobId: String?, - stepId: String? + stepId: String?, + reverse: Boolean? ): QueryLogs { val startTime = System.currentTimeMillis() val (queryLogs, index) = getQueryLogs( @@ -569,7 +572,8 @@ class LogServiceLuceneImpl constructor( containerHashId = containerHashId, executeCount = executeCount, jobId = jobId, - stepId = stepId + stepId = stepId, + reverse = reverse ) logger.info("logs query time cost: ${System.currentTimeMillis() - startTime}") queryLogs.logs.addAll(logs) diff --git a/src/backend/ci/core/misc/api-gpt/build.gradle.kts b/src/backend/ci/core/misc/api-gpt/build.gradle.kts new file mode 100644 index 00000000000..f6744a8bcc1 --- /dev/null +++ b/src/backend/ci/core/misc/api-gpt/build.gradle.kts @@ -0,0 +1,36 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +dependencies { + api(project(":core:common:common-api")) + api(project(":core:common:common-web")) + api(project(":core:common:common-event")) +} + +plugins { + `task-deploy-to-maven` +} diff --git a/src/backend/ci/core/misc/api-gpt/src/main/kotlin/com/tencent/devops/gpt/api/GPTConfigResource.kt b/src/backend/ci/core/misc/api-gpt/src/main/kotlin/com/tencent/devops/gpt/api/GPTConfigResource.kt new file mode 100644 index 00000000000..d2154c60a52 --- /dev/null +++ b/src/backend/ci/core/misc/api-gpt/src/main/kotlin/com/tencent/devops/gpt/api/GPTConfigResource.kt @@ -0,0 +1,57 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.gpt.api + +import com.tencent.devops.common.api.auth.AUTH_HEADER_USER_ID +import com.tencent.devops.common.api.auth.AUTH_HEADER_USER_ID_DEFAULT_VALUE +import com.tencent.devops.common.api.pojo.Result +import io.swagger.v3.oas.annotations.Operation +import io.swagger.v3.oas.annotations.Parameter +import io.swagger.v3.oas.annotations.tags.Tag +import javax.ws.rs.Consumes +import javax.ws.rs.GET +import javax.ws.rs.HeaderParam +import javax.ws.rs.Path +import javax.ws.rs.Produces +import javax.ws.rs.core.MediaType + +@Tag(name = "USER_GPT", description = "ai服务-配置") +@Path("/user/gpt_config") +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +interface GPTConfigResource { + + @Operation(summary = "是否已实装大模型") + @GET + @Path("/is_ok") + fun gptCheck( + @Parameter(description = "用户ID", required = true, example = AUTH_HEADER_USER_ID_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_USER_ID) + userId: String + ): Result +} diff --git a/src/backend/ci/core/misc/api-gpt/src/main/kotlin/com/tencent/devops/gpt/api/LLMResource.kt b/src/backend/ci/core/misc/api-gpt/src/main/kotlin/com/tencent/devops/gpt/api/LLMResource.kt new file mode 100644 index 00000000000..be56b289289 --- /dev/null +++ b/src/backend/ci/core/misc/api-gpt/src/main/kotlin/com/tencent/devops/gpt/api/LLMResource.kt @@ -0,0 +1,161 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.gpt.api + +import com.tencent.devops.common.api.auth.AUTH_HEADER_USER_ID +import com.tencent.devops.common.api.auth.AUTH_HEADER_USER_ID_DEFAULT_VALUE +import com.tencent.devops.common.api.pojo.Result +import com.tencent.devops.gpt.pojo.AIScoreRes +import io.swagger.v3.oas.annotations.Operation +import io.swagger.v3.oas.annotations.Parameter +import io.swagger.v3.oas.annotations.tags.Tag +import javax.ws.rs.Consumes +import javax.ws.rs.DELETE +import javax.ws.rs.DefaultValue +import javax.ws.rs.GET +import javax.ws.rs.HeaderParam +import javax.ws.rs.POST +import javax.ws.rs.Path +import javax.ws.rs.PathParam +import javax.ws.rs.Produces +import javax.ws.rs.QueryParam +import javax.ws.rs.core.MediaType +import org.glassfish.jersey.server.ChunkedOutput + +@Tag(name = "USER_GPT", description = "ai服务") +@Path("/user/gpt") +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +interface LLMResource { + + @Operation(summary = "脚本执行报错AI分析") + @POST + @Path("/script_error_analysis/{projectId}/{pipelineId}/{buildId}") + fun scriptErrorAnalysis( + @Parameter(description = "用户ID", required = true, example = AUTH_HEADER_USER_ID_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_USER_ID) + userId: String, + @Parameter(description = "项目ID", required = true) + @PathParam("projectId") + projectId: String, + @Parameter(description = "流水线ID", required = true) + @PathParam("pipelineId") + pipelineId: String, + @Parameter(description = "构建ID", required = true) + @PathParam("buildId") + buildId: String, + @Parameter(description = "插件id", required = false) + @QueryParam("taskId") + taskId: String, + @Parameter(description = "执行次数", required = false) + @QueryParam("executeCount") + @DefaultValue("1") + executeCount: Int, + @Parameter(description = "是否刷新", required = false) + @QueryParam("refresh") + refresh: Boolean? + ): ChunkedOutput + + @Operation(summary = "脚本执行报错AI分析-评分") + @POST + @Path("/script_error_analysis_score/{projectId}/{pipelineId}/{buildId}") + fun scriptErrorAnalysisScore( + @Parameter(description = "用户ID", required = true, example = AUTH_HEADER_USER_ID_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_USER_ID) + userId: String, + @Parameter(description = "项目ID", required = true) + @PathParam("projectId") + projectId: String, + @Parameter(description = "流水线ID", required = true) + @PathParam("pipelineId") + pipelineId: String, + @Parameter(description = "构建ID", required = true) + @PathParam("buildId") + buildId: String, + @Parameter(description = "插件id", required = false) + @QueryParam("taskId") + taskId: String, + @Parameter(description = "执行次数", required = false) + @QueryParam("executeCount") + @DefaultValue("1") + executeCount: Int, + @Parameter(description = "好or不好", required = false) + @QueryParam("score") + score: Boolean + ): Result + + @Operation(summary = "脚本执行报错AI分析-获取评分") + @GET + @Path("/script_error_analysis_score/{projectId}/{pipelineId}/{buildId}") + fun scriptErrorAnalysisScoreGet( + @Parameter(description = "用户ID", required = true, example = AUTH_HEADER_USER_ID_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_USER_ID) + userId: String, + @Parameter(description = "项目ID", required = true) + @PathParam("projectId") + projectId: String, + @Parameter(description = "流水线ID", required = true) + @PathParam("pipelineId") + pipelineId: String, + @Parameter(description = "构建ID", required = true) + @PathParam("buildId") + buildId: String, + @Parameter(description = "插件id", required = false) + @QueryParam("taskId") + taskId: String, + @Parameter(description = "执行次数", required = false) + @QueryParam("executeCount") + @DefaultValue("1") + executeCount: Int + ): Result + + @Operation(summary = "脚本执行报错AI分析-取消评分") + @DELETE + @Path("/script_error_analysis_score/{projectId}/{pipelineId}/{buildId}") + fun scriptErrorAnalysisScoreDel( + @Parameter(description = "用户ID", required = true, example = AUTH_HEADER_USER_ID_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_USER_ID) + userId: String, + @Parameter(description = "项目ID", required = true) + @PathParam("projectId") + projectId: String, + @Parameter(description = "流水线ID", required = true) + @PathParam("pipelineId") + pipelineId: String, + @Parameter(description = "构建ID", required = true) + @PathParam("buildId") + buildId: String, + @Parameter(description = "插件id", required = false) + @QueryParam("taskId") + taskId: String, + @Parameter(description = "执行次数", required = false) + @QueryParam("executeCount") + @DefaultValue("1") + executeCount: Int + ): Result +} diff --git a/src/backend/ci/core/misc/api-gpt/src/main/kotlin/com/tencent/devops/gpt/constant/GptMessageCode.kt b/src/backend/ci/core/misc/api-gpt/src/main/kotlin/com/tencent/devops/gpt/constant/GptMessageCode.kt new file mode 100644 index 00000000000..0940f20c90c --- /dev/null +++ b/src/backend/ci/core/misc/api-gpt/src/main/kotlin/com/tencent/devops/gpt/constant/GptMessageCode.kt @@ -0,0 +1,69 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.gpt.constant + +/** + * 流水线微服务模块请求返回状态码 + * 返回码制定规则（0代表成功，为了兼容历史接口的成功状态都是返回0）： + * 1、返回码总长度为7位， + * 2、前2位数字代表系统名称（如21代表平台） + * 3、第3位和第4位数字代表微服务模块（00：common-公共模块 01：process-流水线 02：artifactory-版本仓库 03:dispatch-分发 04：dockerhost-docker机器 + * 05:environment-环境 06：experience-版本体验 07：image-镜像 08：log-日志 09：measure-度量 10：monitoring-监控 11：notify-通知 + * 12：openapi-开放api接口 13：plugin-插件 14：quality-质量红线 15：repository-代码库 16：scm-软件配置管理 17：support-支撑服务 + * 18：ticket-证书凭据 19：project-项目管理 20：store-商店 21： auth-权限 22:sign-签名服务 23:metrics-度量服务 24：external-外部 + * 25：prebuild-预建 26: dispatcher-kubernetes 27：buildless 28: lambda 29: stream 30: worker 31: dispatcher-docker + * 32: remotedev 35：misc-杂项） + * 4、最后3位数字代表具体微服务模块下返回给客户端的业务逻辑含义（如001代表系统服务繁忙，建议一个模块一类的返回码按照一定的规则制定） + * 5、系统公共的返回码写在CommonMessageCode这个类里面，具体微服务模块的返回码写在相应模块的常量类里面 + * + * @since: 2023-3-20 + * @version: $Revision$ $Date$ $LastChangedBy$ + * + */ +object GptMessageCode { + // 发生错误！插件可分析内容并未找到。 + const val SCRIPT_ERROR_ANALYSIS_CHAT_TASK_NOT_FIND = "scriptErrorAnalysisChatTaskNotFind" + + // 发生错误！插件{0}结构损坏。 + const val SCRIPT_ERROR_ANALYSIS_CHAT_TASK_STRUCTURAL_DAMAGE = "scriptErrorAnalysisChatTaskStructuralDamage" + + // 请等待插件执行失败后再分析错误。 + const val SCRIPT_ERROR_ANALYSIS_CHAT_TASK_NOT_FAILED = "scriptErrorAnalysisChatTaskNotFailed" + + // 发生错误！暂未支持分析该插件执行错误。 + const val SCRIPT_ERROR_ANALYSIS_CHAT_TASK_NOT_SUPPORT = "scriptErrorAnalysisChatTaskNotSupport" + + // 发生错误！插件日志未入库或已清理。 + const val SCRIPT_ERROR_ANALYSIS_CHAT_TASK_LOGS_EMPTY = "scriptErrorAnalysisChatTaskLogsEmpty" + + // 当前模型忙，请稍后重试 + const val GPT_BUSY = "gptBusy" + + // 未开启GPT服务 + const val GPT_DISABLE = "gptDisable" +} diff --git a/src/backend/ci/core/misc/api-gpt/src/main/kotlin/com/tencent/devops/gpt/pojo/AIScoreRes.kt b/src/backend/ci/core/misc/api-gpt/src/main/kotlin/com/tencent/devops/gpt/pojo/AIScoreRes.kt new file mode 100644 index 00000000000..75718f7ffa6 --- /dev/null +++ b/src/backend/ci/core/misc/api-gpt/src/main/kotlin/com/tencent/devops/gpt/pojo/AIScoreRes.kt @@ -0,0 +1,6 @@ +package com.tencent.devops.gpt.pojo + +data class AIScoreRes( + val goodUsers: Set, + val badUsers: Set +) diff --git a/src/backend/ci/core/misc/biz-gpt/build.gradle.kts b/src/backend/ci/core/misc/biz-gpt/build.gradle.kts new file mode 100644 index 00000000000..66349831d81 --- /dev/null +++ b/src/backend/ci/core/misc/biz-gpt/build.gradle.kts @@ -0,0 +1,39 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C)) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software")), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +dependencies { + api(project(":core:misc:model-plugin")) + api(project(":core:common:common-api")) + api(project(":core:common:common-web")) + api(project(":core:common:common-service")) + api(project(":core:common:common-client")) + api(project(":core:misc:api-gpt")) + api(project(":core:process:api-process")) + api(project(":core:common:common-pipeline")) + implementation("dev.langchain4j:langchain4j:0.33.0") + implementation("dev.langchain4j:langchain4j-open-ai:0.33.0") +} diff --git a/src/backend/ci/core/misc/biz-gpt/src/main/kotlin/com/tencent/devops/gpt/dao/AIScoreDao.kt b/src/backend/ci/core/misc/biz-gpt/src/main/kotlin/com/tencent/devops/gpt/dao/AIScoreDao.kt new file mode 100644 index 00000000000..f7902e4ecb7 --- /dev/null +++ b/src/backend/ci/core/misc/biz-gpt/src/main/kotlin/com/tencent/devops/gpt/dao/AIScoreDao.kt @@ -0,0 +1,125 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.tencent.devops.gpt.dao + +import com.tencent.devops.common.service.utils.ByteUtils +import com.tencent.devops.gpt.service.config.GptGatewayCondition +import com.tencent.devops.model.plugin.tables.TAiScore +import com.tencent.devops.model.plugin.tables.records.TAiScoreRecord +import org.jooq.DSLContext +import org.springframework.context.annotation.Conditional +import org.springframework.stereotype.Repository + +@Repository +@Conditional(GptGatewayCondition::class) +class AIScoreDao { + + fun create( + dslContext: DSLContext, + label: String, + aiMsg: String, + systemMsg: String, + userMsg: String, + goodUserIds: Set, + badUserIds: Set + ) { + with(TAiScore.T_AI_SCORE) { + dslContext.insertInto( + this, + LABEL, + GOOD_USERS, + BAD_USERS, + AI_MSG, + SYSTEM_MSG, + USER_MSG + ).values( + label, + goodUserIds.joinToString(","), + badUserIds.joinToString(","), + aiMsg, + systemMsg, + userMsg + ).onDuplicateKeyUpdate() + .set(GOOD_USERS, goodUserIds.joinToString(",")) + .set(BAD_USERS, badUserIds.joinToString(",")) + .execute() + } + } + + fun fetchAny( + dslContext: DSLContext, + label: String + ): TAiScoreRecord? { + with(TAiScore.T_AI_SCORE) { + return dslContext.selectFrom(this) + .where(LABEL.eq(label)) + .and(ARCHIVE.eq(ByteUtils.bool2Byte(false))) + .orderBy(CREATE_TIME.desc()) + .fetchAny() + } + } + + fun updateUsers( + dslContext: DSLContext, + id: Long, + goodUserIds: Set, + badUserIds: Set + ) { + with(TAiScore.T_AI_SCORE) { + dslContext.update(this) + .set(GOOD_USERS, goodUserIds.joinToString(",")) + .set(BAD_USERS, badUserIds.joinToString(",")) + .where(ID.eq(id)) + .execute() + } + } + + fun updateMsg( + dslContext: DSLContext, + id: Long, + aiMsg: String, + systemMsg: String, + userMsg: String + ) { + with(TAiScore.T_AI_SCORE) { + dslContext.update(this) + .set(AI_MSG, aiMsg) + .set(SYSTEM_MSG, systemMsg) + .set(USER_MSG, userMsg) + .where(ID.eq(id)) + .execute() + } + } + + fun archive(dslContext: DSLContext, label: String) { + with(TAiScore.T_AI_SCORE) { + dslContext.update(this) + .set(ARCHIVE, ByteUtils.bool2Byte(true)) + .where(LABEL.eq(label)).execute() + } + } +} diff --git a/src/backend/ci/core/misc/biz-gpt/src/main/kotlin/com/tencent/devops/gpt/resources/GPTConfigResourceImpl.kt b/src/backend/ci/core/misc/biz-gpt/src/main/kotlin/com/tencent/devops/gpt/resources/GPTConfigResourceImpl.kt new file mode 100644 index 00000000000..43d85b0864c --- /dev/null +++ b/src/backend/ci/core/misc/biz-gpt/src/main/kotlin/com/tencent/devops/gpt/resources/GPTConfigResourceImpl.kt @@ -0,0 +1,55 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.gpt.resources + +import com.tencent.devops.common.api.pojo.Result +import com.tencent.devops.common.service.utils.SpringContextUtil +import com.tencent.devops.common.web.RestResource +import com.tencent.devops.common.web.utils.I18nUtil +import com.tencent.devops.gpt.api.GPTConfigResource +import com.tencent.devops.gpt.constant.GptMessageCode.GPT_DISABLE +import com.tencent.devops.gpt.service.LLMService + +@RestResource +class GPTConfigResourceImpl : GPTConfigResource { + + var check: Boolean? = null + + override fun gptCheck(userId: String): Result { + if (check == null) { + kotlin.runCatching { + SpringContextUtil.getBean(LLMService::class.java) + }.onFailure { + check = false + }.onSuccess { + check = true + } + } + return if (check == false) Result(I18nUtil.getCodeLanMessage(GPT_DISABLE), false) else Result(true) + } +} diff --git a/src/backend/ci/core/misc/biz-gpt/src/main/kotlin/com/tencent/devops/gpt/resources/LLMResourceImpl.kt b/src/backend/ci/core/misc/biz-gpt/src/main/kotlin/com/tencent/devops/gpt/resources/LLMResourceImpl.kt new file mode 100644 index 00000000000..2b707f03bfe --- /dev/null +++ b/src/backend/ci/core/misc/biz-gpt/src/main/kotlin/com/tencent/devops/gpt/resources/LLMResourceImpl.kt @@ -0,0 +1,149 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.gpt.resources + +import com.tencent.devops.common.api.pojo.Result +import com.tencent.devops.common.web.RestResource +import com.tencent.devops.gpt.api.LLMResource +import com.tencent.devops.gpt.pojo.AIScoreRes +import com.tencent.devops.gpt.service.LLMService +import com.tencent.devops.gpt.service.config.GptGatewayCondition +import java.util.concurrent.Executors +import org.glassfish.jersey.server.ChunkedOutput +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.context.annotation.Conditional + +@RestResource +@Conditional(GptGatewayCondition::class) +class LLMResourceImpl @Autowired constructor( + private val llmService: LLMService +) : LLMResource { + + companion object { + val logger = LoggerFactory.getLogger(LLMResourceImpl::class.java)!! + } + + private val executor = Executors.newCachedThreadPool() + + override fun scriptErrorAnalysis( + userId: String, + projectId: String, + pipelineId: String, + buildId: String, + taskId: String, + executeCount: Int, + refresh: Boolean? + ): ChunkedOutput { + /* http/2 streaming + * 由于jersey 设置了缓冲区ServerProperties.OUTBOUND_CONTENT_LENGTH_BUFFER + * 所以不能使用 StreamingOutput + * 而改用 ChunkedOutput + * */ + val output: ChunkedOutput = ChunkedOutput(String::class.java) + executor.execute { + try { + output.use { out -> + llmService.scriptErrorAnalysisChat( + userId = userId, + projectId = projectId, + pipelineId = pipelineId, + buildId = buildId, + taskId = taskId, + executeCount = executeCount, + refresh = refresh, + output = out + ) + } + } catch (ex: Exception) { + logger.warn("scriptErrorAnalysis Chunked output error!") + } + } + return output + } + + override fun scriptErrorAnalysisScore( + userId: String, + projectId: String, + pipelineId: String, + buildId: String, + taskId: String, + executeCount: Int, + score: Boolean + ): Result { + llmService.scriptErrorAnalysisScore( + userId = userId, + projectId = projectId, + pipelineId = pipelineId, + buildId = buildId, + taskId = taskId, + executeCount = executeCount, + score = score + ) + return Result(true) + } + + override fun scriptErrorAnalysisScoreGet( + userId: String, + projectId: String, + pipelineId: String, + buildId: String, + taskId: String, + executeCount: Int + ): Result { + return Result( + llmService.scriptErrorAnalysisScoreGet( + userId = userId, + projectId = projectId, + pipelineId = pipelineId, + buildId = buildId, + taskId = taskId, + executeCount = executeCount + ) + ) + } + + override fun scriptErrorAnalysisScoreDel( + userId: String, + projectId: String, + pipelineId: String, + buildId: String, + taskId: String, + executeCount: Int + ): Result { + llmService.scriptErrorAnalysisScoreDel( + userId = userId, + projectId = projectId, + pipelineId = pipelineId, + buildId = buildId, + taskId = taskId, + executeCount = executeCount + ) + return Result(true) + } +} diff --git a/src/backend/ci/core/misc/biz-gpt/src/main/kotlin/com/tencent/devops/gpt/service/LLMModelService.kt b/src/backend/ci/core/misc/biz-gpt/src/main/kotlin/com/tencent/devops/gpt/service/LLMModelService.kt new file mode 100644 index 00000000000..b3f8761eddb --- /dev/null +++ b/src/backend/ci/core/misc/biz-gpt/src/main/kotlin/com/tencent/devops/gpt/service/LLMModelService.kt @@ -0,0 +1,35 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.gpt.service + +import com.tencent.devops.gpt.service.processor.ScriptErrorAnalysisProcessor + +interface LLMModelService { + + fun scriptErrorAnalysisChat(script: List, errorLog: List, output: ScriptErrorAnalysisProcessor) +} diff --git a/src/backend/ci/core/misc/biz-gpt/src/main/kotlin/com/tencent/devops/gpt/service/LLMService.kt b/src/backend/ci/core/misc/biz-gpt/src/main/kotlin/com/tencent/devops/gpt/service/LLMService.kt new file mode 100644 index 00000000000..066643e9801 --- /dev/null +++ b/src/backend/ci/core/misc/biz-gpt/src/main/kotlin/com/tencent/devops/gpt/service/LLMService.kt @@ -0,0 +1,384 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.gpt.service + +import com.tencent.devops.common.api.util.JsonUtil +import com.tencent.devops.common.client.Client +import com.tencent.devops.common.log.pojo.enums.LogStatus +import com.tencent.devops.common.log.pojo.enums.LogType +import com.tencent.devops.common.pipeline.enums.BuildStatus +import com.tencent.devops.common.pipeline.pojo.element.Element +import com.tencent.devops.common.redis.RedisLock +import com.tencent.devops.common.redis.RedisOperation +import com.tencent.devops.common.web.utils.I18nUtil +import com.tencent.devops.gpt.constant.GptMessageCode.GPT_BUSY +import com.tencent.devops.gpt.constant.GptMessageCode.SCRIPT_ERROR_ANALYSIS_CHAT_TASK_LOGS_EMPTY +import com.tencent.devops.gpt.constant.GptMessageCode.SCRIPT_ERROR_ANALYSIS_CHAT_TASK_NOT_FAILED +import com.tencent.devops.gpt.constant.GptMessageCode.SCRIPT_ERROR_ANALYSIS_CHAT_TASK_NOT_FIND +import com.tencent.devops.gpt.dao.AIScoreDao +import com.tencent.devops.gpt.pojo.AIScoreRes +import com.tencent.devops.gpt.service.config.GptGatewayCondition +import com.tencent.devops.gpt.service.processor.ScriptErrorAnalysisProcessor +import com.tencent.devops.log.api.ServiceLogResource +import com.tencent.devops.process.api.service.ServicePipelineTaskResource +import java.util.concurrent.TimeUnit +import org.glassfish.jersey.server.ChunkedOutput +import org.jooq.DSLContext +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.context.annotation.Conditional +import org.springframework.stereotype.Service + +@Service +@Conditional(GptGatewayCondition::class) +class LLMService @Autowired constructor( + private val client: Client, + private val dslContext: DSLContext, + private val llmModelService: LLMModelService, + private val redisOperation: RedisOperation, + private val aIScoreDao: AIScoreDao +) { + companion object { + private val logger = LoggerFactory.getLogger(LLMService::class.java) + private const val CHUNK = 500 + private const val REDIS_KEY = "llm_out_cache:" + + fun label4scriptErrorAnalysisScore( + projectId: String, + pipelineId: String, + buildId: String, + taskId: String, + executeCount: Int + ) = "scriptErrorAnalysis.$projectId.$pipelineId.$buildId.$taskId.$executeCount" + } + + fun llmRedisCacheKey( + label: String + ) = "$REDIS_KEY$label" + + fun scriptErrorAnalysisChat( + userId: String, + projectId: String, + pipelineId: String, + buildId: String, + taskId: String, + executeCount: Int, + refresh: Boolean?, + output: ChunkedOutput + ) { + logger.info("scriptErrorAnalysisChat|$userId|$projectId|$pipelineId|$buildId|$taskId|$executeCount|$refresh") + // 拿插件执行信息 + val task = client.get(ServicePipelineTaskResource::class).getTaskBuildDetail( + projectId = projectId, buildId = buildId, taskId = taskId, stepId = null, executeCount = executeCount + ).data ?: run { + output.write(I18nUtil.getCodeLanMessage(SCRIPT_ERROR_ANALYSIS_CHAT_TASK_NOT_FIND)) + return + } + // 校验插件状态 + if (task.status != BuildStatus.FAILED) { + output.write(I18nUtil.getCodeLanMessage(SCRIPT_ERROR_ANALYSIS_CHAT_TASK_NOT_FAILED)) + return + } + val label = label4scriptErrorAnalysisScore( + projectId = projectId, + pipelineId = pipelineId, + buildId = buildId, + taskId = taskId, + executeCount = executeCount + ) + val cache = redisOperation.get(llmRedisCacheKey(label)) + if (refresh != true && cache != null) { + logger.info("read form cache") + output.write(cache) + return + } + + aIScoreDao.archive(dslContext, label) + val processor = ScriptErrorAnalysisProcessor(output) + // 拿脚本内容 + val ele = JsonUtil.mapTo(task.taskParams, Element::class.java) + val script = processor.getTaskScript(ele) ?: return + + // 第一阶段：通过脚本插件落库的错误信息（提取自错误流，有长度限制） + if (!task.errorMsg.isNullOrBlank()) { + llmModelService.scriptErrorAnalysisChat(script, task.errorMsg!!.lines(), processor) + } + if (cacheInSucceed(label, processor)) return + // 第二阶段：通过拿error log日志 + scriptErrorAnalysisChatByLog( + userId = userId, + projectId = projectId, + pipelineId = pipelineId, + buildId = buildId, + taskId = taskId, + executeCount = executeCount, + output = output, + script = script, + processor = processor, + logType = LogType.ERROR + ) + if (cacheInSucceed(label, processor)) return + // 第三阶段：拿全部日志 + + scriptErrorAnalysisChatByLog( + userId = userId, + projectId = projectId, + pipelineId = pipelineId, + buildId = buildId, + taskId = taskId, + executeCount = executeCount, + output = output, + script = script, + processor = processor + ) + if (!cacheInSucceed(label, processor)) { + output.write(I18nUtil.getCodeLanMessage(GPT_BUSY)) + logger.info("scriptErrorAnalysisChat GPT_BUSY") + return + } + } + + private fun scriptErrorAnalysisChatByLog( + userId: String, + projectId: String, + pipelineId: String, + buildId: String, + taskId: String, + executeCount: Int, + output: ChunkedOutput, + script: List, + processor: ScriptErrorAnalysisProcessor, + logType: LogType? = null + ) { + val logsData = client.get(ServiceLogResource::class).getInitLogs( + userId = userId, + projectId = projectId, + pipelineId = pipelineId, + buildId = buildId, + tag = taskId, + logType = logType, + containerHashId = null, + executeCount = executeCount, + jobId = null, + stepId = null, + reverse = true + ).data + + if (logsData?.status != LogStatus.SUCCEED.status || logsData.logs.isEmpty()) { + output.write(I18nUtil.getCodeLanMessage(SCRIPT_ERROR_ANALYSIS_CHAT_TASK_LOGS_EMPTY)) + return + } + llmModelService.scriptErrorAnalysisChat( + script, + logsData.logs.takeLast(CHUNK).reversed().map { it.message }.dropLast(1), + processor + ) + } + + fun scriptErrorAnalysisScore( + userId: String, + projectId: String, + pipelineId: String, + buildId: String, + taskId: String, + executeCount: Int, + score: Boolean + ) { + logger.info("scriptErrorAnalysisScore|$userId|$projectId|$pipelineId|$buildId|$executeCount|$score") + val label = label4scriptErrorAnalysisScore( + projectId = projectId, + pipelineId = pipelineId, + buildId = buildId, + taskId = taskId, + executeCount = executeCount + ) + val redisLock = RedisLock(redisOperation, label, 10) + redisLock.use { + redisLock.lock() + val record = aIScoreDao.fetchAny( + dslContext = dslContext, + label = label + ) + val recordGood = record?.goodUsers?.ifBlank { null } + ?.split(",")?.toMutableSet() ?: mutableSetOf() + val recordBad = record?.badUsers?.ifBlank { null } + ?.split(",")?.toMutableSet() ?: mutableSetOf() + when { + score -> recordGood.add(userId) + !score -> recordBad.add(userId) + } + when { + score && userId in recordBad -> recordBad.remove(userId) + !score && userId in recordGood -> recordGood.remove(userId) + } + if (record != null) { + /* update */ + aIScoreDao.updateUsers( + dslContext = dslContext, + id = record.id, + goodUserIds = recordGood, + badUserIds = recordBad + ) + } else { + /* create */ + val cacheKey = llmRedisCacheKey(label) + val cacheAiMsg = redisOperation.get(cacheKey) + val cachePushSystemMsg = redisOperation.get("$cacheKey:system") + val cachePushUserMsg = redisOperation.get("$cacheKey:user") + aIScoreDao.create( + dslContext = dslContext, + label = label, + aiMsg = cacheAiMsg ?: "", + systemMsg = cachePushSystemMsg ?: "", + userMsg = cachePushUserMsg ?: "", + goodUserIds = recordGood, + badUserIds = recordBad + ) + } + return + } + } + + fun scriptErrorAnalysisScoreGet( + userId: String, + projectId: String, + pipelineId: String, + buildId: String, + taskId: String, + executeCount: Int + ): AIScoreRes { + logger.info("scriptErrorAnalysisScoreGet|$userId|$projectId|$pipelineId|$buildId|$taskId|$executeCount") + val label = label4scriptErrorAnalysisScore( + projectId = projectId, + pipelineId = pipelineId, + buildId = buildId, + taskId = taskId, + executeCount = executeCount + ) + return aIScoreDao.fetchAny( + dslContext = dslContext, + label = label + )?.let { + AIScoreRes( + goodUsers = it.goodUsers.ifBlank { null }?.split(",")?.toSet() ?: emptySet(), + badUsers = it.badUsers.ifBlank { null }?.split(",")?.toSet() ?: emptySet() + ) + } ?: AIScoreRes( + goodUsers = emptySet(), + badUsers = emptySet() + ) + } + + fun scriptErrorAnalysisScoreDel( + userId: String, + projectId: String, + pipelineId: String, + buildId: String, + taskId: String, + executeCount: Int + ) { + logger.info("scriptErrorAnalysisScoreDel|$userId|$projectId|$pipelineId|$buildId|$taskId|$executeCount") + val redisLock = RedisLock( + redisOperation, label4scriptErrorAnalysisScore( + projectId = projectId, + pipelineId = pipelineId, + buildId = buildId, + taskId = taskId, + executeCount = executeCount + ), 10 + ) + val label = label4scriptErrorAnalysisScore( + projectId = projectId, + pipelineId = pipelineId, + buildId = buildId, + taskId = taskId, + executeCount = executeCount + ) + redisLock.use { + redisLock.lock() + val record = aIScoreDao.fetchAny( + dslContext = dslContext, + label = label + ) ?: return + val recordGood = record.goodUsers?.ifBlank { null } + ?.split(",")?.toMutableSet() ?: mutableSetOf() + val recordBad = record.badUsers?.ifBlank { null } + ?.split(",")?.toMutableSet() ?: mutableSetOf() + recordGood.remove(userId) + recordBad.remove(userId) + aIScoreDao.updateUsers( + dslContext = dslContext, + id = record.id, + goodUserIds = recordGood, + badUserIds = recordBad + ) + } + } + + fun cacheInSucceed( + label: String, + processor: ScriptErrorAnalysisProcessor + ): Boolean { + if (processor.checkSucceed()) { + val record = aIScoreDao.fetchAny( + dslContext = dslContext, + label = label + ) + if (record != null) { + aIScoreDao.updateMsg( + dslContext = dslContext, + id = record.id, + aiMsg = processor.aiMsg.toString(), + systemMsg = processor.getPushSystemMsg(), + userMsg = processor.getPushUserMsg() + ) + } + + redisOperation.set( + "${llmRedisCacheKey(label)}:system", + processor.getPushSystemMsg(), + expiredInSecond = TimeUnit.HOURS.toSeconds(1), + expired = true + ) + redisOperation.set( + "${llmRedisCacheKey(label)}:user", + processor.getPushUserMsg(), + expiredInSecond = TimeUnit.HOURS.toSeconds(1), + expired = true + ) + redisOperation.set( + llmRedisCacheKey(label), + processor.aiMsg.toString(), + expiredInSecond = TimeUnit.HOURS.toSeconds(1), + expired = true + ) + return true + } + return false + } +} diff --git a/src/backend/ci/core/misc/biz-gpt/src/main/kotlin/com/tencent/devops/gpt/service/config/GptGatewayCondition.kt b/src/backend/ci/core/misc/biz-gpt/src/main/kotlin/com/tencent/devops/gpt/service/config/GptGatewayCondition.kt new file mode 100644 index 00000000000..a1aaaea0c7d --- /dev/null +++ b/src/backend/ci/core/misc/biz-gpt/src/main/kotlin/com/tencent/devops/gpt/service/config/GptGatewayCondition.kt @@ -0,0 +1,13 @@ +package com.tencent.devops.gpt.service.config + +import org.springframework.context.annotation.Condition +import org.springframework.context.annotation.ConditionContext +import org.springframework.core.type.AnnotatedTypeMetadata + +class GptGatewayCondition : Condition { + override fun matches(context: ConditionContext, metadata: AnnotatedTypeMetadata): Boolean { + val environment = context.environment + val gptGateway = environment.getProperty("gpt.gateway") + return !gptGateway.isNullOrBlank() + } +} diff --git a/src/backend/ci/core/misc/biz-gpt/src/main/kotlin/com/tencent/devops/gpt/service/config/HunYuanConfig.kt b/src/backend/ci/core/misc/biz-gpt/src/main/kotlin/com/tencent/devops/gpt/service/config/HunYuanConfig.kt new file mode 100644 index 00000000000..c01ab09c7bd --- /dev/null +++ b/src/backend/ci/core/misc/biz-gpt/src/main/kotlin/com/tencent/devops/gpt/service/config/HunYuanConfig.kt @@ -0,0 +1,15 @@ +package com.tencent.devops.gpt.service.config + +import org.springframework.beans.factory.annotation.Value +import org.springframework.context.annotation.Conditional +import org.springframework.stereotype.Component + +@Component +@Conditional(GptGatewayCondition::class) +class HunYuanConfig { + @Value("\${gpt.gateway:}") + val url = "" + + @Value("#{\${gpt.headers:{}}}") + val headers: Map = emptyMap() +} diff --git a/src/backend/ci/core/misc/biz-gpt/src/main/kotlin/com/tencent/devops/gpt/service/hunyuan/HunYuanChatHelper.kt b/src/backend/ci/core/misc/biz-gpt/src/main/kotlin/com/tencent/devops/gpt/service/hunyuan/HunYuanChatHelper.kt new file mode 100644 index 00000000000..6cd7485d0bf --- /dev/null +++ b/src/backend/ci/core/misc/biz-gpt/src/main/kotlin/com/tencent/devops/gpt/service/hunyuan/HunYuanChatHelper.kt @@ -0,0 +1,44 @@ +package com.tencent.devops.gpt.service.hunyuan + +import dev.ai4j.openai4j.chat.ChatCompletionRequest +import dev.langchain4j.agent.tool.ToolSpecification +import dev.langchain4j.data.message.AiMessage +import dev.langchain4j.data.message.ChatMessage +import dev.langchain4j.model.chat.listener.ChatModelRequest +import dev.langchain4j.model.chat.listener.ChatModelResponse +import dev.langchain4j.model.output.Response + +object HunYuanChatHelper { + fun createModelListenerRequest( + request: ChatCompletionRequest, + messages: List, + toolSpecifications: List? + ): ChatModelRequest { + return ChatModelRequest.builder() + .model(request.model()) + .temperature(request.temperature()) + .topP(request.topP()) + .maxTokens(request.maxTokens()) + .messages(messages) + .toolSpecifications(toolSpecifications) + .build() + } + + fun createModelListenerResponse( + responseId: String, + responseModel: String, + response: Response? + ): ChatModelResponse? { + if (response == null) { + return null + } + + return ChatModelResponse.builder() + .id(responseId) + .model(responseModel) + .tokenUsage(response.tokenUsage()) + .finishReason(response.finishReason()) + .aiMessage(response.content()) + .build() + } +} diff --git a/src/backend/ci/core/misc/biz-gpt/src/main/kotlin/com/tencent/devops/gpt/service/hunyuan/HunYuanChatModel.kt b/src/backend/ci/core/misc/biz-gpt/src/main/kotlin/com/tencent/devops/gpt/service/hunyuan/HunYuanChatModel.kt new file mode 100644 index 00000000000..66e8a59f973 --- /dev/null +++ b/src/backend/ci/core/misc/biz-gpt/src/main/kotlin/com/tencent/devops/gpt/service/hunyuan/HunYuanChatModel.kt @@ -0,0 +1,202 @@ +package com.tencent.devops.gpt.service.hunyuan + +import com.tencent.devops.gpt.service.config.HunYuanConfig +import dev.ai4j.openai4j.OpenAiClient +import dev.ai4j.openai4j.OpenAiHttpException +import dev.ai4j.openai4j.chat.ChatCompletionChoice +import dev.ai4j.openai4j.chat.ChatCompletionRequest +import dev.ai4j.openai4j.chat.ChatCompletionResponse +import dev.langchain4j.agent.tool.ToolSpecification +import dev.langchain4j.data.message.AiMessage +import dev.langchain4j.data.message.ChatMessage +import dev.langchain4j.internal.RetryUtils +import dev.langchain4j.model.Tokenizer +import dev.langchain4j.model.chat.ChatLanguageModel +import dev.langchain4j.model.chat.TokenCountEstimator +import dev.langchain4j.model.chat.listener.ChatModelErrorContext +import dev.langchain4j.model.chat.listener.ChatModelListener +import dev.langchain4j.model.chat.listener.ChatModelRequestContext +import dev.langchain4j.model.chat.listener.ChatModelResponseContext +import dev.langchain4j.model.openai.InternalOpenAiHelper +import dev.langchain4j.model.openai.OpenAiTokenizer +import dev.langchain4j.model.output.Response +import java.net.Proxy +import java.time.Duration +import java.util.concurrent.ConcurrentHashMap +import java.util.function.Consumer +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +class HunYuanChatModel( + private val client: OpenAiClient, + private val modelName: String?, + private val temperature: Double, + private val topP: Double?, + private val stop: List?, + private val maxTokens: Int?, + private val presencePenalty: Double?, + private val frequencyPenalty: Double?, + private val logitBias: Map?, + private val responseFormat: String?, + private val seed: Int?, + private val user: String?, + private val maxRetries: Int, + private val tokenizer: Tokenizer, + private val listeners: List +) : ChatLanguageModel, TokenCountEstimator { + + constructor(hunYuanConfig: HunYuanConfig) : this( + url = hunYuanConfig.url, + customHeaders = hunYuanConfig.headers + ) + + constructor( + url: String, + temperature: Double? = null, + topP: Double? = null, + stop: List? = null, + maxTokens: Int? = null, + presencePenalty: Double? = null, + frequencyPenalty: Double? = null, + logitBias: Map? = null, + responseFormat: String? = null, + seed: Int? = null, + user: String? = null, + timeout: Duration? = null, + maxRetries: Int? = null, + proxy: Proxy? = null, + logRequests: Boolean? = null, + logResponses: Boolean? = null, + tokenizer: Tokenizer? = null, + customHeaders: Map? = null, + listeners: List? = null + ) : this( + client = OpenAiClient.builder() + .baseUrl(url) + .openAiApiKey("*") + .callTimeout(timeout ?: Duration.ofSeconds(60L)) + .connectTimeout(timeout ?: Duration.ofSeconds(60L)) + .readTimeout(timeout ?: Duration.ofSeconds(60L)) + .writeTimeout(timeout ?: Duration.ofSeconds(60L)).proxy(proxy) + .logRequests(logRequests).logResponses(logResponses).userAgent("langchain4j-openai") + .customHeaders(customHeaders).build(), + modelName = "hunyuan", + temperature = temperature ?: 0.7, + topP = topP, + stop = stop, + maxTokens = maxTokens, + presencePenalty = presencePenalty, + frequencyPenalty = frequencyPenalty, + logitBias = logitBias, + responseFormat = responseFormat, + seed = seed, + user = user, + maxRetries = maxRetries ?: 3, + tokenizer = tokenizer ?: OpenAiTokenizer(), + listeners = listeners ?: emptyList() + ) + + fun modelName(): String? { + return this.modelName + } + + override fun generate(messages: List): Response { + return this.generate(messages, null, null) + } + + override fun generate( + messages: List, + toolSpecifications: List + ): Response { + return this.generate(messages, toolSpecifications, null) + } + + override fun generate(messages: List, toolSpecification: ToolSpecification): Response { + return this.generate(messages, listOf(toolSpecification), toolSpecification) + } + + private fun generate( + messages: List, + toolSpecifications: List?, + toolThatMustBeExecuted: ToolSpecification? + ): Response { + val requestBuilder = ChatCompletionRequest.builder().model(this.modelName) + .messages(InternalOpenAiHelper.toOpenAiMessages(messages)).temperature(this.temperature).topP(this.topP) + .stop(this.stop).maxTokens(this.maxTokens).presencePenalty(this.presencePenalty) + .frequencyPenalty(this.frequencyPenalty).logitBias(this.logitBias).responseFormat(this.responseFormat) + .seed(this.seed).user(this.user) + if (!toolSpecifications.isNullOrEmpty()) { + requestBuilder.tools(InternalOpenAiHelper.toTools(toolSpecifications)) + } + + if (toolThatMustBeExecuted != null) { + requestBuilder.toolChoice(toolThatMustBeExecuted.name()) + } + + val request = requestBuilder.build() + val modelListenerRequest = HunYuanChatHelper.createModelListenerRequest(request, messages, toolSpecifications) + val attributes: Map = ConcurrentHashMap() + val requestContext = ChatModelRequestContext(modelListenerRequest, attributes) + listeners.forEach(Consumer { listener: ChatModelListener -> + try { + listener.onRequest(requestContext) + } catch (ignore: Exception) { + log.warn("Exception while calling model listener", ignore) + } + }) + + try { + val chatCompletionResponse = RetryUtils.withRetry( + { client.chatCompletion(request).execute() as ChatCompletionResponse }, + maxRetries + ) as ChatCompletionResponse + val response = Response.from( + InternalOpenAiHelper.aiMessageFrom(chatCompletionResponse), + InternalOpenAiHelper.tokenUsageFrom(chatCompletionResponse.usage()), + InternalOpenAiHelper.finishReasonFrom( + (chatCompletionResponse.choices()[0] as ChatCompletionChoice).finishReason() + ) + ) + val modelListenerResponse = HunYuanChatHelper.createModelListenerResponse( + chatCompletionResponse.id(), + chatCompletionResponse.model(), + response + ) + val responseContext = ChatModelResponseContext(modelListenerResponse, modelListenerRequest, attributes) + listeners.forEach(Consumer { listener: ChatModelListener -> + try { + listener.onResponse(responseContext) + } catch (ignore: Exception) { + log.warn("Exception while calling model listener", ignore) + } + }) + return response + } catch (e: RuntimeException) { + val error = if (e.cause is OpenAiHttpException) { + e.cause + } else { + e + } + + val errorContext = ChatModelErrorContext( + error, modelListenerRequest, null, attributes + ) + listeners.forEach(Consumer { listener: ChatModelListener -> + try { + listener.onError(errorContext) + } catch (ignore: Exception) { + log.warn("Exception while calling model listener", ignore) + } + }) + throw e + } + } + + override fun estimateTokenCount(messages: List): Int { + return tokenizer.estimateTokenCountInMessages(messages) + } + + companion object { + private val log: Logger = LoggerFactory.getLogger(HunYuanChatModel::class.java) + } +} diff --git a/src/backend/ci/core/misc/biz-gpt/src/main/kotlin/com/tencent/devops/gpt/service/hunyuan/HunYuanStreamingChatModel.kt b/src/backend/ci/core/misc/biz-gpt/src/main/kotlin/com/tencent/devops/gpt/service/hunyuan/HunYuanStreamingChatModel.kt new file mode 100644 index 00000000000..cdea8554b53 --- /dev/null +++ b/src/backend/ci/core/misc/biz-gpt/src/main/kotlin/com/tencent/devops/gpt/service/hunyuan/HunYuanStreamingChatModel.kt @@ -0,0 +1,237 @@ +package com.tencent.devops.gpt.service.hunyuan + +import com.tencent.devops.gpt.service.config.HunYuanConfig +import dev.ai4j.openai4j.OpenAiClient +import dev.ai4j.openai4j.chat.ChatCompletionChoice +import dev.ai4j.openai4j.chat.ChatCompletionRequest +import dev.ai4j.openai4j.chat.ChatCompletionResponse +import dev.langchain4j.agent.tool.ToolSpecification +import dev.langchain4j.data.message.AiMessage +import dev.langchain4j.data.message.ChatMessage +import dev.langchain4j.internal.Utils +import dev.langchain4j.model.StreamingResponseHandler +import dev.langchain4j.model.Tokenizer +import dev.langchain4j.model.chat.StreamingChatLanguageModel +import dev.langchain4j.model.chat.TokenCountEstimator +import dev.langchain4j.model.chat.listener.ChatModelErrorContext +import dev.langchain4j.model.chat.listener.ChatModelListener +import dev.langchain4j.model.chat.listener.ChatModelRequestContext +import dev.langchain4j.model.chat.listener.ChatModelResponseContext +import dev.langchain4j.model.openai.InternalOpenAiHelper +import dev.langchain4j.model.openai.OpenAiStreamingResponseBuilder +import dev.langchain4j.model.openai.OpenAiTokenizer +import dev.langchain4j.model.output.Response +import java.net.Proxy +import java.time.Duration +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicReference +import java.util.function.Consumer +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +class HunYuanStreamingChatModel( + private val client: OpenAiClient, + private val modelName: String, + private val temperature: Double, + private val topP: Double?, + private val stop: List?, + private val maxTokens: Int?, + private val presencePenalty: Double?, + private val frequencyPenalty: Double?, + private val logitBias: Map?, + private val responseFormat: String?, + private val seed: Int?, + private var user: String?, + private val tokenizer: Tokenizer?, + private val listeners: List +) : StreamingChatLanguageModel, TokenCountEstimator { + + constructor(hunYuanConfig: HunYuanConfig) : this( + url = hunYuanConfig.url, + customHeaders = hunYuanConfig.headers + ) + + constructor( + url: String, + temperature: Double? = null, + topP: Double? = null, + stop: List? = null, + maxTokens: Int? = null, + presencePenalty: Double? = null, + frequencyPenalty: Double? = null, + logitBias: Map? = null, + responseFormat: String? = null, + seed: Int? = null, + user: String? = null, + timeout: Duration? = null, + proxy: Proxy? = null, + logRequests: Boolean? = null, + logResponses: Boolean? = null, + tokenizer: Tokenizer? = null, + customHeaders: Map? = null, + listeners: List? = null + ) : this( + client = OpenAiClient.builder() + .baseUrl(url) + .openAiApiKey("*") + .callTimeout(timeout ?: Duration.ofSeconds(60L)) + .connectTimeout(timeout ?: Duration.ofSeconds(60L)) + .readTimeout(timeout ?: Duration.ofSeconds(60L)) + .writeTimeout(timeout ?: Duration.ofSeconds(60L)) + .proxy(proxy) + .logRequests(logRequests) + .logStreamingResponses(logResponses) + .userAgent("langchain4j-openai") + .customHeaders(customHeaders).build(), + modelName = "hunyuan", + temperature = temperature ?: 0.7, + topP = topP, + stop = stop, + maxTokens = maxTokens, + presencePenalty = presencePenalty, + frequencyPenalty = frequencyPenalty, + logitBias = logitBias, + responseFormat = responseFormat, + seed = seed, + user = user, + tokenizer = tokenizer ?: OpenAiTokenizer(), + listeners = listeners ?: emptyList() + ) + + fun modelName(): String { + return this.modelName + } + + override fun generate(messages: List, handler: StreamingResponseHandler) { + this.generate(messages, null, null, handler) + } + + override fun generate( + messages: List, + toolSpecifications: List, + handler: StreamingResponseHandler + ) { + this.generate(messages, toolSpecifications, null, handler) + } + + override fun generate( + messages: List, + toolSpecification: ToolSpecification, + handler: StreamingResponseHandler + ) { + this.generate(messages, null, toolSpecification, handler) + } + + private fun generate( + messages: List, + toolSpecifications: List?, + toolThatMustBeExecuted: ToolSpecification?, + handler: StreamingResponseHandler + ) { + val requestBuilder = ChatCompletionRequest.builder().stream(true).model(this.modelName) + .messages(InternalOpenAiHelper.toOpenAiMessages(messages)).temperature(this.temperature).topP(this.topP) + .stop(this.stop).maxTokens(this.maxTokens).presencePenalty(this.presencePenalty) + .frequencyPenalty(this.frequencyPenalty).logitBias(this.logitBias).responseFormat(this.responseFormat) + .seed(this.seed).user(this.user) + if (toolThatMustBeExecuted != null) { + requestBuilder.tools(InternalOpenAiHelper.toTools(listOf(toolThatMustBeExecuted))) + requestBuilder.toolChoice(toolThatMustBeExecuted.name()) + } else if (!Utils.isNullOrEmpty(toolSpecifications)) { + requestBuilder.tools(InternalOpenAiHelper.toTools(toolSpecifications)) + } + + val request = requestBuilder.build() + val modelListenerRequest = HunYuanChatHelper.createModelListenerRequest(request, messages, toolSpecifications) + val attributes: Map = ConcurrentHashMap() + val requestContext = ChatModelRequestContext(modelListenerRequest, attributes) + listeners.forEach(Consumer { listener: ChatModelListener -> + try { + listener.onRequest(requestContext) + } catch (e: Exception) { + log.warn("Exception while calling model listener", e) + } + }) + val inputTokenCount = this.countInputTokens(messages, toolSpecifications, toolThatMustBeExecuted) + val responseBuilder = OpenAiStreamingResponseBuilder(inputTokenCount) + val responseId: AtomicReference = AtomicReference() + val responseModel: AtomicReference = AtomicReference() + client.chatCompletion(request).onPartialResponse { partialResponse: ChatCompletionResponse -> + responseBuilder.append(partialResponse) + handle(partialResponse, handler) + if (!Utils.isNullOrBlank(partialResponse.id())) { + responseId.set(partialResponse.id()) + } + if (!Utils.isNullOrBlank(partialResponse.model())) { + responseModel.set(partialResponse.model()) + } + }.onComplete { + val response = this.createResponse(responseBuilder, toolThatMustBeExecuted) + val modelListenerResponse = + HunYuanChatHelper.createModelListenerResponse(responseId.get(), responseModel.get(), response) + val responseContext = ChatModelResponseContext(modelListenerResponse, modelListenerRequest, attributes) + listeners.forEach(Consumer { listener: ChatModelListener -> + try { + listener.onResponse(responseContext) + } catch (e: Exception) { + log.warn("Exception while calling model listener", e) + } + }) + handler.onComplete(response) + }.onError { error: Throwable -> + val response = this.createResponse(responseBuilder, toolThatMustBeExecuted) + val modelListenerPartialResponse = + HunYuanChatHelper.createModelListenerResponse(responseId.get(), responseModel.get(), response) + val errorContext = + ChatModelErrorContext(error, modelListenerRequest, modelListenerPartialResponse, attributes) + listeners.forEach(Consumer { listener: ChatModelListener -> + try { + listener.onError(errorContext) + } catch (e: Exception) { + log.warn("Exception while calling model listener", e) + } + }) + handler.onError(error) + }.execute() + } + + private fun createResponse( + responseBuilder: OpenAiStreamingResponseBuilder, + toolThatMustBeExecuted: ToolSpecification? + ): Response { + val response = responseBuilder.build(this.tokenizer, toolThatMustBeExecuted != null) + return response + } + + private fun countInputTokens( + messages: List, + toolSpecifications: List?, + toolThatMustBeExecuted: ToolSpecification? + ): Int { + var inputTokenCount = tokenizer!!.estimateTokenCountInMessages(messages) + if (toolThatMustBeExecuted != null) { + inputTokenCount += tokenizer.estimateTokenCountInForcefulToolSpecification(toolThatMustBeExecuted) + } else if (!Utils.isNullOrEmpty(toolSpecifications)) { + inputTokenCount += tokenizer.estimateTokenCountInToolSpecifications(toolSpecifications) + } + + return inputTokenCount + } + + override fun estimateTokenCount(messages: List): Int { + return tokenizer!!.estimateTokenCountInMessages(messages) + } + + companion object { + private val log: Logger = LoggerFactory.getLogger(HunYuanStreamingChatModel::class.java) + private fun handle(partialResponse: ChatCompletionResponse, handler: StreamingResponseHandler) { + val choices = partialResponse.choices() + if (choices != null && choices.isNotEmpty()) { + val delta = (choices[0] as ChatCompletionChoice).delta() + val content = delta.content() + if (content != null) { + handler.onNext(content) + } + } + } + } +} diff --git a/src/backend/ci/core/misc/biz-gpt/src/main/kotlin/com/tencent/devops/gpt/service/hunyuan/LLMHunYuanServiceImpl.kt b/src/backend/ci/core/misc/biz-gpt/src/main/kotlin/com/tencent/devops/gpt/service/hunyuan/LLMHunYuanServiceImpl.kt new file mode 100644 index 00000000000..7ee4c5380ba --- /dev/null +++ b/src/backend/ci/core/misc/biz-gpt/src/main/kotlin/com/tencent/devops/gpt/service/hunyuan/LLMHunYuanServiceImpl.kt @@ -0,0 +1,75 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.gpt.service.hunyuan + +import com.tencent.devops.gpt.service.LLMModelService +import com.tencent.devops.gpt.service.config.GptGatewayCondition +import com.tencent.devops.gpt.service.config.HunYuanConfig +import com.tencent.devops.gpt.service.processor.ScriptErrorAnalysisProcessor +import dev.langchain4j.service.AiServices +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.context.annotation.Conditional +import org.springframework.stereotype.Service + +@Service +@Conditional(GptGatewayCondition::class) +class LLMHunYuanServiceImpl @Autowired constructor(private val config: HunYuanConfig) : LLMModelService { + companion object { + private val logger = LoggerFactory.getLogger(LLMHunYuanServiceImpl::class.java) + } + + override fun scriptErrorAnalysisChat( + script: List, + errorLog: List, + output: ScriptErrorAnalysisProcessor + ) { + output.init() + val streamingModel = HunYuanStreamingChatModel(config) + val model = HunYuanChatModel(config) + val ai = AiServices.builder(ScriptErrorAnalysisProcessor.Prompt::class.java) + .streamingChatLanguageModel(streamingModel) + .chatLanguageModel(model) + .chatMemoryProvider { output.pushMsg } + .build() + + val latch = CountDownLatch(1) + val tokenStream = ai.ask("1", script.joinToString("\n"), errorLog.joinToString("\n")) + tokenStream.onNext { x: String -> + output.next(x) + }.onComplete { + latch.countDown() + }.onError { obj: Throwable -> + obj.printStackTrace() + latch.countDown() + }.start() + latch.await(5, TimeUnit.MINUTES) + } +} diff --git a/src/backend/ci/core/misc/biz-gpt/src/main/kotlin/com/tencent/devops/gpt/service/processor/ScriptErrorAnalysisProcessor.kt b/src/backend/ci/core/misc/biz-gpt/src/main/kotlin/com/tencent/devops/gpt/service/processor/ScriptErrorAnalysisProcessor.kt new file mode 100644 index 00000000000..7949f08b04c --- /dev/null +++ b/src/backend/ci/core/misc/biz-gpt/src/main/kotlin/com/tencent/devops/gpt/service/processor/ScriptErrorAnalysisProcessor.kt @@ -0,0 +1,117 @@ +package com.tencent.devops.gpt.service.processor + +import com.tencent.devops.common.pipeline.pojo.element.Element +import com.tencent.devops.common.pipeline.pojo.element.agent.LinuxScriptElement +import com.tencent.devops.common.pipeline.pojo.element.agent.WindowsScriptElement +import com.tencent.devops.common.pipeline.pojo.element.market.MarketBuildAtomElement +import com.tencent.devops.common.web.utils.I18nUtil +import com.tencent.devops.gpt.constant.GptMessageCode.SCRIPT_ERROR_ANALYSIS_CHAT_TASK_NOT_SUPPORT +import com.tencent.devops.gpt.constant.GptMessageCode.SCRIPT_ERROR_ANALYSIS_CHAT_TASK_STRUCTURAL_DAMAGE +import dev.langchain4j.memory.chat.MessageWindowChatMemory +import dev.langchain4j.service.MemoryId +import dev.langchain4j.service.SystemMessage +import dev.langchain4j.service.TokenStream +import dev.langchain4j.service.UserMessage +import dev.langchain4j.service.V +import java.net.URLDecoder +import org.glassfish.jersey.server.ChunkedOutput +import dev.langchain4j.data.message.SystemMessage as SM +import dev.langchain4j.data.message.UserMessage as UM + +class ScriptErrorAnalysisProcessor(private val output: ChunkedOutput) { + private val lineOne = StringBuilder() + val aiMsg = StringBuilder() + private var valid: Boolean? = null + val pushMsg: MessageWindowChatMemory = MessageWindowChatMemory.withMaxMessages(10) + fun init() { + lineOne.clear() + aiMsg.clear() + pushMsg.clear() + valid = null + } + + fun getPushSystemMsg(): String { + return pushMsg.messages().find { it is SM }?.text() ?: "" + } + + fun getPushUserMsg(): String { + val msg = pushMsg.messages().filterIsInstance().firstOrNull() ?: return "" + return msg.contents().joinToString { it.toString() } + } + + fun checkSucceed() = valid == true + + fun next(input: String): Boolean { + if (valid == null) { + lineOne.append(input) + if (input.contains("\n") && lineOne.contains("yes")) { + valid = true + } + if (input.contains("\n") && lineOne.contains("no")) { + valid = false + } + } + when (valid) { + true -> { + aiMsg.append(input) + output.write(input) + return true + } + + false -> return false + null -> return true + } + } + + fun getTaskScript(ele: Element): List? { + if (ele is LinuxScriptElement) { + val script = URLDecoder.decode(ele.script, "UTF-8") + return script.lines().filterNot { it.startsWith("# ") || it.isBlank() } + } + if (ele is WindowsScriptElement) { + val script = URLDecoder.decode(ele.script, "UTF-8") + return script.lines().filterNot { it.startsWith("REM ") || it.isBlank() } + } + if (ele is MarketBuildAtomElement && ele.getAtomCode() == "run") { + val input = ele.data["input"] as Map? ?: run { + output.write( + I18nUtil.getCodeLanMessage( + SCRIPT_ERROR_ANALYSIS_CHAT_TASK_STRUCTURAL_DAMAGE, + params = arrayOf("input") + ) + ) + return null + } + val script = input["script"] as String? ?: run { + output.write( + I18nUtil.getCodeLanMessage( + SCRIPT_ERROR_ANALYSIS_CHAT_TASK_STRUCTURAL_DAMAGE, + params = arrayOf("input.script") + ) + ) + return null + } + return script.lines().filterNot { it.isBlank() } + } + output.write(I18nUtil.getCodeLanMessage(SCRIPT_ERROR_ANALYSIS_CHAT_TASK_NOT_SUPPORT)) + return null + } + + interface Prompt { + @SystemMessage("You are a professional script engineer.") + @UserMessage( + "Please help me find out the reason for the script execution error. " + + "Please answer me in standard markdown format. " + + "Please use the format below to provide a detailed answer to the specific cause of the error: " + + "是否找到错误原因: [yes or no]. 如果为yes, 继续输出。\n" + + "错误原因: ...\n" + + "解决办法: ...\n" + + "The script content is: '{{script}}' and the error log is: '{{errorLog}}'" + ) + fun ask( + @MemoryId memoryId: String, + @V("script") script: String, + @V("errorLog") errorLog: String + ): TokenStream + } +} diff --git a/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/config/JooqConfiguration.kt b/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/config/JooqConfiguration.kt index e31c778b06e..5fd78517fda 100644 --- a/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/config/JooqConfiguration.kt +++ b/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/config/JooqConfiguration.kt @@ -59,7 +59,7 @@ import javax.sql.DataSource class JooqConfiguration { @Value("\${spring.datasource.misc.pkgRegex:}") - private val pkgRegex = "\\.(process|project|repository|dispatch|plugin|quality|artifactory|environment)" + private val pkgRegex = "\\.(process|project|repository|dispatch|plugin|quality|artifactory|environment|gpt)" companion object { private val LOG = LoggerFactory.getLogger(JooqConfiguration::class.java) @@ -136,6 +136,15 @@ class JooqConfiguration { return generateDefaultConfiguration(pluginDataSource, executeListenerProviders) } + @Bean + fun gptJooqConfiguration( + @Qualifier("pluginDataSource") + pluginDataSource: DataSource, + executeListenerProviders: ObjectProvider + ): DefaultConfiguration { + return generateDefaultConfiguration(pluginDataSource, executeListenerProviders) + } + @Bean fun qualityJooqConfiguration( @Qualifier("qualityDataSource") diff --git a/src/backend/ci/core/misc/boot-misc/build.gradle.kts b/src/backend/ci/core/misc/boot-misc/build.gradle.kts index 8f8becc4aa7..bfa1705767a 100644 --- a/src/backend/ci/core/misc/boot-misc/build.gradle.kts +++ b/src/backend/ci/core/misc/boot-misc/build.gradle.kts @@ -30,5 +30,6 @@ dependencies { api(project(":core:misc:biz-image")) api(project(":core:misc:biz-monitoring")) api(project(":core:misc:biz-plugin")) + api(project(":core:misc:biz-gpt")) api(project(":core:common:common-auth:common-auth-provider")) } diff --git a/src/backend/ci/core/misc/boot-misc/src/main/kotlin/com/tencent/devops/misc/MiscApplication.kt b/src/backend/ci/core/misc/boot-misc/src/main/kotlin/com/tencent/devops/misc/MiscApplication.kt index dcb30f08962..024c21aaa1e 100644 --- a/src/backend/ci/core/misc/boot-misc/src/main/kotlin/com/tencent/devops/misc/MiscApplication.kt +++ b/src/backend/ci/core/misc/boot-misc/src/main/kotlin/com/tencent/devops/misc/MiscApplication.kt @@ -36,7 +36,8 @@ import org.springframework.context.annotation.ComponentScan "com.tencent.devops.misc", "com.tencent.devops.image", "com.tencent.devops.monitoring", - "com.tencent.devops.plugin" + "com.tencent.devops.plugin", + "com.tencent.devops.gpt" ) class MiscApplication diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v4/ApigwArtifactoryFileTaskResourceV4Impl.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v4/ApigwArtifactoryFileTaskResourceV4Impl.kt index ffb973aca63..85896c081f4 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v4/ApigwArtifactoryFileTaskResourceV4Impl.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v4/ApigwArtifactoryFileTaskResourceV4Impl.kt @@ -74,7 +74,7 @@ class ApigwArtifactoryFileTaskResourceV4Impl @Autowired constructor( logger.info("OPENAPI_ARTIFACTORY_FILE_TASK_V4|$userId|get status|$projectId|$pipelineId|$buildId|$taskId") val realTaskId = if (stepId != null) { client.get(ServicePipelineTaskResource::class).getTaskBuildDetail( - projectId, buildId, taskId, stepId + projectId, buildId, taskId, stepId, null ).data?.taskId } else taskId if (realTaskId == null) { @@ -102,7 +102,7 @@ class ApigwArtifactoryFileTaskResourceV4Impl @Autowired constructor( logger.info("OPENAPI_ARTIFACTORY_FILE_TASK_V4|$userId|clear file task|$projectId|$pipelineId|$buildId|$taskId") val realTaskId = if (stepId != null) { client.get(ServicePipelineTaskResource::class).getTaskBuildDetail( - projectId, buildId, taskId, stepId + projectId, buildId, taskId, stepId, null ).data?.taskId } else taskId if (realTaskId == null) { diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/service/ServicePipelineTaskResource.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/service/ServicePipelineTaskResource.kt index 937c6553ac1..7db2ea24c75 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/service/ServicePipelineTaskResource.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/service/ServicePipelineTaskResource.kt @@ -127,7 +127,10 @@ interface ServicePipelineTaskResource { taskId: String?, @Parameter(description = "任务ID", required = false) @QueryParam("stepId") - stepId: String? + stepId: String?, + @Parameter(description = "执行次数", required = false) + @QueryParam("executeCount") + executeCount: Int? ): Result @Operation(summary = "获取流水线指定Job的构建状态") diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineBuildTaskDao.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineBuildTaskDao.kt index c6e655df7d6..853556d02f2 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineBuildTaskDao.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineBuildTaskDao.kt @@ -233,7 +233,8 @@ class PipelineBuildTaskDao { projectId: String, buildId: String, taskId: String?, - stepId: String? + stepId: String?, + executeCount: Int? ): PipelineBuildTask? { return with(T_PIPELINE_BUILD_TASK) { @@ -244,6 +245,9 @@ class PipelineBuildTaskDao { if (stepId != null) { where.and(STEP_ID.eq(stepId)) } + if (executeCount != null) { + where.and(EXECUTE_COUNT.eq(executeCount)) + } where.fetchAny(mapper) } } diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineTaskService.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineTaskService.kt index b54b8922ba5..3a781897d5c 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineTaskService.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineTaskService.kt @@ -222,14 +222,16 @@ class PipelineTaskService @Autowired constructor( projectId: String, buildId: String, taskId: String?, - stepId: String? = null + stepId: String? = null, + executeCount: Int? = null ): PipelineBuildTask? { return pipelineBuildTaskDao.get( dslContext = transactionContext ?: dslContext, projectId = projectId, buildId = buildId, taskId = taskId, - stepId = stepId + stepId = stepId, + executeCount = executeCount ) } @@ -300,7 +302,8 @@ class PipelineTaskService @Autowired constructor( projectId = projectId, buildId = buildId, taskId = taskId, - stepId = stepId + stepId = stepId, + executeCount = null ) } @@ -330,7 +333,8 @@ class PipelineTaskService @Autowired constructor( projectId = updateTaskInfo.projectId, buildId = updateTaskInfo.buildId, taskId = updateTaskInfo.taskId, - stepId = null + stepId = null, + executeCount = null ) } if (updateTaskInfo.taskStatus.isFinish()) { diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/vmbuild/EngineVMBuildService.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/vmbuild/EngineVMBuildService.kt index 4a22a6d3941..fabf2abe690 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/vmbuild/EngineVMBuildService.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/vmbuild/EngineVMBuildService.kt @@ -53,6 +53,8 @@ import com.tencent.devops.common.pipeline.enums.BuildTaskStatus import com.tencent.devops.common.pipeline.pojo.BuildParameters import com.tencent.devops.common.pipeline.pojo.JobHeartbeatRequest import com.tencent.devops.common.pipeline.pojo.element.RunCondition +import com.tencent.devops.common.pipeline.pojo.element.agent.LinuxScriptElement +import com.tencent.devops.common.pipeline.pojo.element.agent.WindowsScriptElement import com.tencent.devops.common.redis.RedisOperation import com.tencent.devops.common.web.utils.AtomRuntimeUtil import com.tencent.devops.common.web.utils.I18nUtil @@ -1055,19 +1057,36 @@ class EngineVMBuildService @Autowired(required = false) constructor( ErrorType.PLUGIN -> "Please contact the plugin developer." ErrorType.SYSTEM -> "Please contact platform." } - buildLogPrinter.addRedLine( - buildId = buildId, - message = errMsg, - tag = taskId, - containerHashId = containerHashId, - executeCount = executeCount ?: 1, - jobId = null, - stepId = stepId - ) + if (showAI(task)) { + buildLogPrinter.addAIErrorLine( + buildId = buildId, + message = errMsg, + tag = taskId, + containerHashId = containerHashId, + executeCount = executeCount ?: 1, + jobId = null, + stepId = stepId + ) + } else { + buildLogPrinter.addRedLine( + buildId = buildId, + message = errMsg, + tag = taskId, + containerHashId = containerHashId, + executeCount = executeCount ?: 1, + jobId = null, + stepId = stepId + ) + } } } } + private fun showAI(task: PipelineBuildTask): Boolean = + task.atomCode == "run" || + task.atomCode == LinuxScriptElement.classType || + task.atomCode == WindowsScriptElement.classType + /** * #5046 提交构建机出错失败信息，并结束构建 */ diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/ServicePipelineTaskResourceImpl.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/ServicePipelineTaskResourceImpl.kt index 55e43a79672..58c853bf704 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/ServicePipelineTaskResourceImpl.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/ServicePipelineTaskResourceImpl.kt @@ -87,14 +87,16 @@ class ServicePipelineTaskResourceImpl @Autowired constructor( projectId: String, buildId: String, taskId: String?, - stepId: String? + stepId: String?, + executeCount: Int? ): Result { if (taskId != null) { return Result( pipelineTaskService.getByTaskId( projectId = projectId, buildId = buildId, - taskId = taskId + taskId = taskId, + executeCount = executeCount ) ) } @@ -105,7 +107,8 @@ class ServicePipelineTaskResourceImpl @Autowired constructor( projectId = projectId, buildId = buildId, taskId = null, - stepId = stepId + stepId = stepId, + executeCount = executeCount ) ) } diff --git a/src/backend/ci/settings.gradle.kts b/src/backend/ci/settings.gradle.kts index 9d7fabb70e8..ef39a2d0d9e 100644 --- a/src/backend/ci/settings.gradle.kts +++ b/src/backend/ci/settings.gradle.kts @@ -108,11 +108,13 @@ include(":core:misc:api-misc") include(":core:misc:api-plugin") include(":core:misc:api-image") include(":core:misc:api-monitoring") +include(":core:misc:api-gpt") include(":core:misc:biz-misc") include(":core:misc:biz-plugin") include(":core:misc:biz-monitoring") include(":core:misc:biz-image") include(":core:misc:biz-misc-sample") +include(":core:misc:biz-gpt") include(":core:misc:boot-misc") include(":core:misc:model-misc") include(":core:misc:model-image") diff --git a/support-files/i18n/misc/message_en_US.properties b/support-files/i18n/misc/message_en_US.properties index b0d953c5f4c..a12694e3e71 100644 --- a/support-files/i18n/misc/message_en_US.properties +++ b/support-files/i18n/misc/message_en_US.properties @@ -46,3 +46,10 @@ bkBuildIdNotFound=Server internal exception. Construction of buildId= {0} was no bkCiPipeline=Bk-ci pipeline bkPipelIneidNotFound=Server internal exception. Construction of pipelineId= {0} was not found. bkProjectManager=Project administrator +scriptErrorAnalysisChatTaskNotFind=An error occurred! The plug-in can analyze the content and find it automatically. +scriptErrorAnalysisChatTaskStructuralDamage=An error occurred! The plugin {0} structure is damaged. +scriptErrorAnalysisChatTaskNotFailed=Please wait for the plug-in execution to fail before analyzing the error. +scriptErrorAnalysisChatTaskNotSupport=An error occurred! Analysis of execution errors of this plug-in is not currently supported. +scriptErrorAnalysisChatTaskLogsEmpty=An error occurred! The plug-in logs are not stored in the database or have been cleaned. +gptBusy=The current model is busy, please try again later. +gptDisable=The AI ​​service has not been deployed yet. If you need to use it, please contact the administrator for deployment. diff --git a/support-files/i18n/misc/message_zh_CN.properties b/support-files/i18n/misc/message_zh_CN.properties index 300262da6ac..1f5469bef4e 100644 --- a/support-files/i18n/misc/message_zh_CN.properties +++ b/support-files/i18n/misc/message_zh_CN.properties @@ -46,4 +46,10 @@ bkBuildIdNotFound=服务端内部异常，buildId={0}的构建未查到 bkCiPipeline=蓝盾流水线 bkPipelineIdNotFound=服务端内部异常，pipelineId={0}的构建未查到 bkProjectManager=项目管理员 - +scriptErrorAnalysisChatTaskNotFind=发生错误！插件可分析内容并未找到。 +scriptErrorAnalysisChatTaskStructuralDamage=发生错误！插件{0}结构损坏。 +scriptErrorAnalysisChatTaskNotFailed=请等待插件执行失败后再分析错误。 +scriptErrorAnalysisChatTaskNotSupport=发生错误！暂未支持分析该插件执行错误。 +scriptErrorAnalysisChatTaskLogsEmpty=发生错误！插件日志未入库或已清理。 +gptBusy=当前模型忙，请稍后重试。 +gptDisable=AI服务暂未部署完成。如需使用，请联系管理员部署。 diff --git a/support-files/sql/1001_ci_plugin_ddl_mysql.sql b/support-files/sql/1001_ci_plugin_ddl_mysql.sql index 4e118fd05c2..0cf372da18f 100644 --- a/support-files/sql/1001_ci_plugin_ddl_mysql.sql +++ b/support-files/sql/1001_ci_plugin_ddl_mysql.sql @@ -41,4 +41,27 @@ CREATE TABLE IF NOT EXISTS `T_PLUGIN_GIT_CHECK` ( KEY `PIPELINE_ID_REPO_ID_COMMIT_ID` (`PIPELINE_ID`,`COMMIT_ID`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT=''; +-- ---------------------------- +-- Table structure for T_AI_SCORE +-- ---------------------------- + +CREATE TABLE IF NOT EXISTS `T_AI_SCORE` +( + `ID` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键ID', + `LABEL` varchar(256) NOT NULL COMMENT '任务ID', + `ARCHIVE` boolean NOT NULL DEFAULT 0 COMMENT '是否已归档', + `CREATE_TIME` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `UPDATE_TIME` datetime NOT NULL ON UPDATE CURRENT_TIMESTAMP DEFAULT CURRENT_TIMESTAMP COMMENT '更新时间', + `GOOD_USERS` text COMMENT '赞的人', + `BAD_USERS` text COMMENT '踩的人', + `AI_MSG` text COMMENT '大模型生成的内容', + `SYSTEM_MSG` text COMMENT 'Prompt for system', + `USER_MSG` text COMMENT 'Prompt for user', + PRIMARY KEY (`ID`), + INDEX IDX_LABEL (`LABEL`), + INDEX IDX_CREATE_TIME (`CREATE_TIME`), + INDEX IDX_ARCHIVE (`ARCHIVE`) +) ENGINE = InnoDB + DEFAULT CHARSET = utf8mb4 COMMENT ='脚本执行报错AI分析-评分'; + SET FOREIGN_KEY_CHECKS = 1; diff --git a/support-files/templates/#etc#ci#application-misc.yml b/support-files/templates/#etc#ci#application-misc.yml index b617d9d3096..55e9f9bfb0d 100644 --- a/support-files/templates/#etc#ci#application-misc.yml +++ b/support-files/templates/#etc#ci#application-misc.yml @@ -34,7 +34,7 @@ spring: username: __BK_CI_MYSQL_USER__ password: __BK_CI_MYSQL_PASSWORD__ misc: - pkgRegex: "\\.(process|project|repository|dispatch|plugin|quality|artifactory|environment|image)" + pkgRegex: "\\.(process|project|repository|dispatch|plugin|quality|artifactory|environment|image|gpt)" # 数据源配置（勿随便变更配置项的顺序） dataSourceConfigs: - index: 0 @@ -101,3 +101,7 @@ plugin: path: __BK_CODECC_DATA_DIR__/tools covFile: build_dev.py toolFile: build_tool_dev.py + +gpt: + gateway: __BK_CI_GPT_GATEWAY__ + headers: __BK_CI_GPT_HEADERS__