From 4025657ae92c3def92d39e4a80df7dbb9854a495 Mon Sep 17 00:00:00 2001 From: royalhuang Date: Thu, 23 Dec 2021 23:07:56 +0800 Subject: [PATCH 1/4] =?UTF-8?q?feat:=20=E6=B5=81=E6=B0=B4=E7=BA=BF?= =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=9E=84=E5=BB=BA=E7=9F=A9=E9=98=B5=20#4518?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../process/service/PipelineContextService.kt | 42 +++++++++++++------ 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/PipelineContextService.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/PipelineContextService.kt index 7a2d29c6d5b..bec04de4b1c 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/PipelineContextService.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/PipelineContextService.kt @@ -46,7 +46,7 @@ import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service -@Suppress("ComplexMethod", "TooManyFunctions", "NestedBlockDepth") +@Suppress("ComplexMethod", "TooManyFunctions", "NestedBlockDepth", "LongParameterList") @Service class PipelineContextService @Autowired constructor( private val pipelineBuildDetailService: PipelineBuildDetailService @@ -63,9 +63,23 @@ class PipelineContextService @Autowired constructor( try { modelDetail.model.stages.forEach { stage -> stage.containers.forEach { container -> - buildJobContext(stage, container, containerId, contextMap, variables) + buildJobContext( + stage = stage, + c = container, + containerId = containerId, + contextMap = contextMap, + variables = variables, + inMatrix = false + ) container.fetchGroupContainers()?.forEach { c -> - buildJobContext(stage, c, containerId, contextMap, variables) + buildJobContext( + stage = stage, + c = c, + containerId = containerId, + contextMap = contextMap, + variables = variables, + inMatrix = true + ) } } } @@ -112,17 +126,9 @@ class PipelineContextService @Autowired constructor( c: Container, containerId: String?, contextMap: MutableMap, - variables: Map + variables: Map, + inMatrix: Boolean ) { - // TODO 兼容逻辑，暂时把该job所有插件的output都去掉前缀，在后的覆盖前者 - // 后续需要改为只有本job才去掉前缀 - variables.forEach { (key, value) -> - val prefix = "jobs.${c.jobId ?: containerId}." - if (key.startsWith(prefix)) { - contextMap[key.removePrefix(prefix)] = value - } - } - // current job if (c.id?.let { it == containerId } == true) { contextMap["job.id"] = c.jobId ?: "" @@ -146,6 +152,16 @@ class PipelineContextService @Autowired constructor( // all element buildStepContext(c, contextMap) + + // TODO 兼容逻辑，暂时把该job所有插件的output都去掉前缀，在后的覆盖前者 + // 后续需要改为只有本job才去掉前缀 + if (inMatrix && c.id?.let { it == containerId } != true) return + variables.forEach { (key, value) -> + val prefix = "jobs.${c.jobId ?: containerId}." + if (key.startsWith(prefix)) { + contextMap[key.removePrefix(prefix)] = value + } + } } private fun buildStepContext( From 8ce62758fd2a10731930fae697da81eed85ac31d Mon Sep 17 00:00:00 2001 From: royalhuang Date: Thu, 23 Dec 2021 23:48:22 +0800 Subject: [PATCH 2/4] =?UTF-8?q?bug:=20=E7=9F=A9=E9=98=B5=E4=B8=8A=E4=B8=8B?= =?UTF-8?q?=E6=96=87=E8=A6=86=E7=9B=96=E9=97=AE=E9=A2=98=20#5881?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tencent/devops/common/pipeline/container/Stage.kt | 9 +++++++-- .../devops/common/pipeline/pojo/StagePauseCheck.kt | 1 + .../process/engine/extend/DefaultModelCheckPlugin.kt | 2 +- .../process/engine/service/PipelineBuildDetailService.kt | 2 +- .../process/engine/service/PipelineRepositoryService.kt | 2 +- .../process/engine/service/PipelineRuntimeService.kt | 5 ++--- .../process/engine/service/PipelineStageService.kt | 7 ------- .../devops/process/service/PipelineInfoFacadeService.kt | 2 +- .../process/service/builds/PipelineBuildFacadeService.kt | 2 -- 9 files changed, 14 insertions(+), 18 deletions(-) diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/container/Stage.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/container/Stage.kt index 63e26f07509..8435bd32acf 100644 --- a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/container/Stage.kt +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/container/Stage.kt @@ -64,9 +64,14 @@ data class Stage( var checkOut: StagePauseCheck? = null // stage准出配置 ) { /** - * 兼容性逻辑 - 将原有的审核配置刷新到审核流中，并且补充审核组ID + * 刷新stage的所有配置，如果是初始化则重置所有历史数据 */ - fun refreshCheckOption(init: Boolean? = false) { + fun resetBuildOption(init: Boolean? = false) { + if (init == true) { + status = null + startEpoch = null + elapsed = null + } checkIn?.fixReviewGroups(init == true) checkOut?.fixReviewGroups(init == true) if (stageControlOption?.manualTrigger == true) { diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/StagePauseCheck.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/StagePauseCheck.kt index 7ba7536b7c5..3d6f935f9d7 100644 --- a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/StagePauseCheck.kt +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/StagePauseCheck.kt @@ -125,6 +125,7 @@ data class StagePauseCheck( group.reviewTime = null group.operator = null group.params = null + group.suggest = null } } if (init) { diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/extend/DefaultModelCheckPlugin.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/extend/DefaultModelCheckPlugin.kt index c0229c729df..e0f600c7669 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/extend/DefaultModelCheckPlugin.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/extend/DefaultModelCheckPlugin.kt @@ -168,7 +168,7 @@ open class DefaultModelCheckPlugin constructor( } private fun checkStageReviewers(stage: Stage) { - stage.refreshCheckOption() + stage.resetBuildOption() if (stage.checkIn?.reviewGroups.isNullOrEmpty()) { throw ErrorCodeException( defaultMessage = "Stage(${stage.name})准入配置不正确", diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineBuildDetailService.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineBuildDetailService.kt index 2ef185a1f13..629308331e8 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineBuildDetailService.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineBuildDetailService.kt @@ -134,7 +134,7 @@ class PipelineBuildDetailService @Autowired constructor( // #4531 兼容历史构建的页面显示 model.stages.forEach { stage -> - stage.refreshCheckOption() + stage.resetBuildOption() // #4518 兼容历史构建的containerId作为日志JobId，发布后新产生的groupContainers无需校准 stage.containers.forEach { container -> container.containerHashId = container.containerHashId ?: container.containerId diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineRepositoryService.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineRepositoryService.kt index 3e2a5c072fb..fa67136101f 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineRepositoryService.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineRepositoryService.kt @@ -205,7 +205,7 @@ class PipelineRepositoryService constructor( model.stages.forEachIndexed { index, s -> s.id = VMUtils.genStageId(index + 1) // #4531 对存量的stage审核数据做兼容处理 - s.refreshCheckOption(true) + s.resetBuildOption(true) if (index == 0) { // 在流程模型中初始化触发类容器 initTriggerContainer( stage = s, diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineRuntimeService.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineRuntimeService.kt index 1791a050c18..b8140aee0e8 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineRuntimeService.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineRuntimeService.kt @@ -774,8 +774,7 @@ class PipelineRuntimeService @Autowired constructor( 构建矩阵特殊处理，即使重试也要重新计算执行策略 */ if (container.matrixGroupFlag == true) { - if (container is VMBuildContainer) container.retryFreshMatrixOption() - else if (container is NormalContainer) container.retryFreshMatrixOption() + container.retryFreshMatrixOption() pipelineContainerService.cleanContainersInMatrixGroup( transactionContext = dslContext, projectId = pipelineInfo.projectId, @@ -819,7 +818,7 @@ class PipelineRuntimeService @Autowired constructor( if (stage.tag == null) stage.tag = listOf(defaultStageTagId) } - stage.refreshCheckOption(true) + stage.resetBuildOption(true) if (lastTimeBuildStageRecords.isNotEmpty()) { if (needUpdateStage) { diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineStageService.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineStageService.kt index 467e8ee34db..938ce8a5651 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineStageService.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineStageService.kt @@ -551,11 +551,4 @@ class PipelineStageService @Autowired constructor( BuildStatus.QUALITY_CHECK_FAIL } } - - fun retryRefreshStage(model: Model) { - model.stages.forEach { stage -> - stage.checkIn?.retryRefresh() - stage.checkOut?.retryRefresh() - } - } } diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/PipelineInfoFacadeService.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/PipelineInfoFacadeService.kt index 94ffd371d8c..55e635a0605 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/PipelineInfoFacadeService.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/PipelineInfoFacadeService.kt @@ -730,7 +730,7 @@ class PipelineInfoFacadeService @Autowired constructor( model.stages.forEach { if (it.name.isNullOrBlank()) it.name = it.id if (it.tag == null) it.tag = defaultTagIds - it.refreshCheckOption() + it.resetBuildOption() } // 部分老的模板实例没有templateId，需要手动加上 diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/builds/PipelineBuildFacadeService.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/builds/PipelineBuildFacadeService.kt index 650444ecc3d..f04a112aaa2 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/builds/PipelineBuildFacadeService.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/builds/PipelineBuildFacadeService.kt @@ -425,8 +425,6 @@ class PipelineBuildFacadeService( } ) } - // #4531 重试完整构建时将所有stage的审核状态恢复 - pipelineStageService.retryRefreshStage(model) } catch (ignored: Exception) { logger.warn("ENGINE|$buildId|Fail to get the startup param: $ignored") } From 703eddd99d6501e0dc9cad443f0683c5a0151dec Mon Sep 17 00:00:00 2001 From: royalhuang Date: Fri, 24 Dec 2021 00:07:34 +0800 Subject: [PATCH 3/4] =?UTF-8?q?bug:=20=E7=9F=A9=E9=98=B5=E4=B8=8A=E4=B8=8B?= =?UTF-8?q?=E6=96=87=E8=A6=86=E7=9B=96=E9=97=AE=E9=A2=98=20#5881?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../devops/process/engine/service/PipelineStageService.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineStageService.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineStageService.kt index 938ce8a5651..4101bd36bc1 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineStageService.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineStageService.kt @@ -34,7 +34,6 @@ import com.tencent.devops.common.event.dispatcher.pipeline.PipelineEventDispatch import com.tencent.devops.common.event.enums.ActionType import com.tencent.devops.common.event.pojo.pipeline.PipelineBuildQualityCheckBroadCastEvent import com.tencent.devops.common.event.pojo.pipeline.PipelineBuildReviewBroadCastEvent -import com.tencent.devops.common.pipeline.Model import com.tencent.devops.common.pipeline.enums.BuildStatus import com.tencent.devops.common.pipeline.enums.ManualReviewAction import com.tencent.devops.common.pipeline.pojo.StagePauseCheck From ceef49392c160bb345f13488240a859beec5fc3a Mon Sep 17 00:00:00 2001 From: royalhuang Date: Fri, 24 Dec 2021 10:18:24 +0800 Subject: [PATCH 4/4] =?UTF-8?q?bug:=20=E7=9F=A9=E9=98=B5=E4=B8=8A=E4=B8=8B?= =?UTF-8?q?=E6=96=87=E8=A6=86=E7=9B=96=E9=97=AE=E9=A2=98=20#5881?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../common/event/dispatcher/pipeline/mq/MQ.kt | 3 --- .../init/BuildEngineCoreContainerConfiguration.kt | 15 --------------- 2 files changed, 18 deletions(-) diff --git a/src/backend/ci/core/common/common-event/src/main/kotlin/com/tencent/devops/common/event/dispatcher/pipeline/mq/MQ.kt b/src/backend/ci/core/common/common-event/src/main/kotlin/com/tencent/devops/common/event/dispatcher/pipeline/mq/MQ.kt index 90a5098bf59..58a65e1ca72 100644 --- a/src/backend/ci/core/common/common-event/src/main/kotlin/com/tencent/devops/common/event/dispatcher/pipeline/mq/MQ.kt +++ b/src/backend/ci/core/common/common-event/src/main/kotlin/com/tencent/devops/common/event/dispatcher/pipeline/mq/MQ.kt @@ -58,9 +58,6 @@ object MQ { const val ROUTE_PIPELINE_BUILD_CONTAINER = "r.engine.pipeline.build.container" const val QUEUE_PIPELINE_BUILD_CONTAINER = "q.engine.pipeline.build.container" - const val ROUTE_PIPELINE_BUILD_MATRIX_GROUP = "r.engine.pipeline.matrix.group" - const val QUEUE_PIPELINE_BUILD_MATRIX_GROUP = "q.engine.pipeline.matrix.group" - const val ROUTE_PIPELINE_BUILD_CANCEL = "r.engine.pipeline.build.cancel" const val QUEUE_PIPELINE_BUILD_CANCEL = "q.engine.pipeline.build.cancel" const val ROUTE_PIPELINE_BUILD_FINISH = "r.engine.pipeline.build.finish" diff --git a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/init/BuildEngineCoreContainerConfiguration.kt b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/init/BuildEngineCoreContainerConfiguration.kt index bd4d845280c..b59ca78702b 100644 --- a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/init/BuildEngineCoreContainerConfiguration.kt +++ b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/init/BuildEngineCoreContainerConfiguration.kt @@ -67,21 +67,6 @@ class BuildEngineCoreContainerConfiguration { .to(pipelineCoreExchange).with(MQ.ROUTE_PIPELINE_BUILD_CONTAINER) } - /** - * Job构建矩阵队列---- 并发一般 - */ - @Bean - fun pipelineBuildMatrixGroupQueue() = Queue(MQ.QUEUE_PIPELINE_BUILD_MATRIX_GROUP) - - @Bean - fun pipelineBuildMatrixGroupQueueBind( - @Autowired pipelineBuildContainerQueue: Queue, - @Autowired pipelineCoreExchange: DirectExchange - ): Binding { - return BindingBuilder.bind(pipelineBuildContainerQueue) - .to(pipelineCoreExchange).with(MQ.ROUTE_PIPELINE_BUILD_MATRIX_GROUP) - } - @Bean fun pipelineContainerBuildListenerContainer( @Autowired connectionFactory: ConnectionFactory,