Merge pull request #5882 from royalhuang/issue_5881

bug: 矩阵上下文覆盖问题 #5881
TencentBlueKing · Dec 24, 2021 · 00f8815 · 00f8815
2 parents 9e14ffe + ceef493
commit 00f8815
Show file tree

Hide file tree

Showing 12 changed files with 43 additions and 50 deletions.
diff --git a/...common-event/src/main/kotlin/com/tencent/devops/common/event/dispatcher/pipeline/mq/MQ.kt b/...common-event/src/main/kotlin/com/tencent/devops/common/event/dispatcher/pipeline/mq/MQ.kt
@@ -58,9 +58,6 @@ object MQ {
     const val ROUTE_PIPELINE_BUILD_CONTAINER = "r.engine.pipeline.build.container"
     const val QUEUE_PIPELINE_BUILD_CONTAINER = "q.engine.pipeline.build.container"
 
-    const val ROUTE_PIPELINE_BUILD_MATRIX_GROUP = "r.engine.pipeline.matrix.group"
-    const val QUEUE_PIPELINE_BUILD_MATRIX_GROUP = "q.engine.pipeline.matrix.group"
-
     const val ROUTE_PIPELINE_BUILD_CANCEL = "r.engine.pipeline.build.cancel"
     const val QUEUE_PIPELINE_BUILD_CANCEL = "q.engine.pipeline.build.cancel"
     const val ROUTE_PIPELINE_BUILD_FINISH = "r.engine.pipeline.build.finish"

diff --git a/...mon/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/container/Stage.kt b/...mon/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/container/Stage.kt
@@ -64,9 +64,14 @@ data class Stage(
     var checkOut: StagePauseCheck? = null // stage准出配置
 ) {
     /**
-     * 兼容性逻辑 - 将原有的审核配置刷新到审核流中，并且补充审核组ID
+     * 刷新stage的所有配置，如果是初始化则重置所有历史数据
      */
-    fun refreshCheckOption(init: Boolean? = false) {
+    fun resetBuildOption(init: Boolean? = false) {
+        if (init == true) {
+            status = null
+            startEpoch = null
+            elapsed = null
+        }
         checkIn?.fixReviewGroups(init == true)
         checkOut?.fixReviewGroups(init == true)
         if (stageControlOption?.manualTrigger == true) {

diff --git a/...ommon-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/StagePauseCheck.kt b/...ommon-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/StagePauseCheck.kt
@@ -125,6 +125,7 @@ data class StagePauseCheck(
                 group.reviewTime = null
                 group.operator = null
                 group.params = null
+                group.suggest = null
             }
         }
         if (init) {

diff --git a/...-base/src/main/kotlin/com/tencent/devops/process/engine/extend/DefaultModelCheckPlugin.kt b/...-base/src/main/kotlin/com/tencent/devops/process/engine/extend/DefaultModelCheckPlugin.kt
@@ -168,7 +168,7 @@ open class DefaultModelCheckPlugin constructor(
     }
 
     private fun checkStageReviewers(stage: Stage) {
-        stage.refreshCheckOption()
+        stage.resetBuildOption()
         if (stage.checkIn?.reviewGroups.isNullOrEmpty()) {
             throw ErrorCodeException(
                 defaultMessage = "Stage(${stage.name})准入配置不正确",

diff --git a/...e/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineBuildDetailService.kt b/...e/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineBuildDetailService.kt
@@ -134,7 +134,7 @@ class PipelineBuildDetailService @Autowired constructor(
 
         // #4531 兼容历史构建的页面显示
         model.stages.forEach { stage ->
-            stage.refreshCheckOption()
+            stage.resetBuildOption()
             // #4518 兼容历史构建的containerId作为日志JobId，发布后新产生的groupContainers无需校准
             stage.containers.forEach { container ->
                 container.containerHashId = container.containerHashId ?: container.containerId

diff --git a/...se/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineRepositoryService.kt b/...se/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineRepositoryService.kt
@@ -205,7 +205,7 @@ class PipelineRepositoryService constructor(
         model.stages.forEachIndexed { index, s ->
             s.id = VMUtils.genStageId(index + 1)
             // #4531 对存量的stage审核数据做兼容处理
-            s.refreshCheckOption(true)
+            s.resetBuildOption(true)
             if (index == 0) { // 在流程模型中初始化触发类容器
                 initTriggerContainer(
                     stage = s,

diff --git a/...-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineRuntimeService.kt b/...-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineRuntimeService.kt
@@ -774,8 +774,7 @@ class PipelineRuntimeService @Autowired constructor(
                     构建矩阵特殊处理，即使重试也要重新计算执行策略
                 */
                 if (container.matrixGroupFlag == true) {
-                    if (container is VMBuildContainer) container.retryFreshMatrixOption()
-                    else if (container is NormalContainer) container.retryFreshMatrixOption()
+                    container.retryFreshMatrixOption()
                     pipelineContainerService.cleanContainersInMatrixGroup(
                         transactionContext = dslContext,
                         projectId = pipelineInfo.projectId,
@@ -819,7 +818,7 @@ class PipelineRuntimeService @Autowired constructor(
                 if (stage.tag == null) stage.tag = listOf(defaultStageTagId)
             }
 
-            stage.refreshCheckOption(true)
+            stage.resetBuildOption(true)
 
             if (lastTimeBuildStageRecords.isNotEmpty()) {
                 if (needUpdateStage) {

diff --git a/...iz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineStageService.kt b/...iz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineStageService.kt
@@ -34,7 +34,6 @@ import com.tencent.devops.common.event.dispatcher.pipeline.PipelineEventDispatch
 import com.tencent.devops.common.event.enums.ActionType
 import com.tencent.devops.common.event.pojo.pipeline.PipelineBuildQualityCheckBroadCastEvent
 import com.tencent.devops.common.event.pojo.pipeline.PipelineBuildReviewBroadCastEvent
-import com.tencent.devops.common.pipeline.Model
 import com.tencent.devops.common.pipeline.enums.BuildStatus
 import com.tencent.devops.common.pipeline.enums.ManualReviewAction
 import com.tencent.devops.common.pipeline.pojo.StagePauseCheck
@@ -551,11 +550,4 @@ class PipelineStageService @Autowired constructor(
             BuildStatus.QUALITY_CHECK_FAIL
         }
     }
-
-    fun retryRefreshStage(model: Model) {
-        model.stages.forEach { stage ->
-            stage.checkIn?.retryRefresh()
-            stage.checkOut?.retryRefresh()
-        }
-    }
 }
diff --git a/...ess/biz-base/src/main/kotlin/com/tencent/devops/process/service/PipelineContextService.kt b/...ess/biz-base/src/main/kotlin/com/tencent/devops/process/service/PipelineContextService.kt
@@ -46,7 +46,7 @@ import org.slf4j.LoggerFactory
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.stereotype.Service
 
-@Suppress("ComplexMethod", "TooManyFunctions", "NestedBlockDepth")
+@Suppress("ComplexMethod", "TooManyFunctions", "NestedBlockDepth", "LongParameterList")
 @Service
 class PipelineContextService @Autowired constructor(
     private val pipelineBuildDetailService: PipelineBuildDetailService
@@ -63,9 +63,23 @@ class PipelineContextService @Autowired constructor(
         try {
             modelDetail.model.stages.forEach { stage ->
                 stage.containers.forEach { container ->
-                    buildJobContext(stage, container, containerId, contextMap, variables)
+                    buildJobContext(
+                        stage = stage,
+                        c = container,
+                        containerId = containerId,
+                        contextMap = contextMap,
+                        variables = variables,
+                        inMatrix = false
+                    )
                     container.fetchGroupContainers()?.forEach { c ->
-                        buildJobContext(stage, c, containerId, contextMap, variables)
+                        buildJobContext(
+                            stage = stage,
+                            c = c,
+                            containerId = containerId,
+                            contextMap = contextMap,
+                            variables = variables,
+                            inMatrix = true
+                        )
                     }
                 }
             }
@@ -112,17 +126,9 @@ class PipelineContextService @Autowired constructor(
         c: Container,
         containerId: String?,
         contextMap: MutableMap<String, String>,
-        variables: Map<String, String>
+        variables: Map<String, String>,
+        inMatrix: Boolean
     ) {
-        // TODO 兼容逻辑，暂时把该job所有插件的output都去掉前缀，在后的覆盖前者
-        // 后续需要改为只有本job才去掉前缀
-        variables.forEach { (key, value) ->
-            val prefix = "jobs.${c.jobId ?: containerId}."
-            if (key.startsWith(prefix)) {
-                contextMap[key.removePrefix(prefix)] = value
-            }
-        }
-
         // current job
         if (c.id?.let { it == containerId } == true) {
             contextMap["job.id"] = c.jobId ?: ""
@@ -146,6 +152,16 @@ class PipelineContextService @Autowired constructor(
 
         // all element
         buildStepContext(c, contextMap)
+
+        // TODO 兼容逻辑，暂时把该job所有插件的output都去掉前缀，在后的覆盖前者
+        // 后续需要改为只有本job才去掉前缀
+        if (inMatrix && c.id?.let { it == containerId } != true) return
+        variables.forEach { (key, value) ->
+            val prefix = "jobs.${c.jobId ?: containerId}."
+            if (key.startsWith(prefix)) {
+                contextMap[key.removePrefix(prefix)] = value
+            }
+        }
     }
 
     private fun buildStepContext(

diff --git a/...in/kotlin/com/tencent/devops/process/engine/init/BuildEngineCoreContainerConfiguration.kt b/...in/kotlin/com/tencent/devops/process/engine/init/BuildEngineCoreContainerConfiguration.kt
@@ -67,21 +67,6 @@ class BuildEngineCoreContainerConfiguration {
             .to(pipelineCoreExchange).with(MQ.ROUTE_PIPELINE_BUILD_CONTAINER)
     }
 
-    /**
-     * Job构建矩阵队列---- 并发一般
-     */
-    @Bean
-    fun pipelineBuildMatrixGroupQueue() = Queue(MQ.QUEUE_PIPELINE_BUILD_MATRIX_GROUP)
-
-    @Bean
-    fun pipelineBuildMatrixGroupQueueBind(
-        @Autowired pipelineBuildContainerQueue: Queue,
-        @Autowired pipelineCoreExchange: DirectExchange
-    ): Binding {
-        return BindingBuilder.bind(pipelineBuildContainerQueue)
-            .to(pipelineCoreExchange).with(MQ.ROUTE_PIPELINE_BUILD_MATRIX_GROUP)
-    }
-
     @Bean
     fun pipelineContainerBuildListenerContainer(
         @Autowired connectionFactory: ConnectionFactory,

diff --git a/...z-process/src/main/kotlin/com/tencent/devops/process/service/PipelineInfoFacadeService.kt b/...z-process/src/main/kotlin/com/tencent/devops/process/service/PipelineInfoFacadeService.kt
@@ -730,7 +730,7 @@ class PipelineInfoFacadeService @Autowired constructor(
             model.stages.forEach {
                 if (it.name.isNullOrBlank()) it.name = it.id
                 if (it.tag == null) it.tag = defaultTagIds
-                it.refreshCheckOption()
+                it.resetBuildOption()
             }
 
             // 部分老的模板实例没有templateId，需要手动加上

diff --git a/...s/src/main/kotlin/com/tencent/devops/process/service/builds/PipelineBuildFacadeService.kt b/...s/src/main/kotlin/com/tencent/devops/process/service/builds/PipelineBuildFacadeService.kt
@@ -425,8 +425,6 @@ class PipelineBuildFacadeService(
                             }
                         )
                     }
-                    // #4531 重试完整构建时将所有stage的审核状态恢复
-                    pipelineStageService.retryRefreshStage(model)
                 } catch (ignored: Exception) {
                     logger.warn("ENGINE|$buildId|Fail to get the startup param: $ignored")
                 }