Skip to content

Commit

Permalink
feat：流水线变量语法支持两种风格 #10576
Browse files Browse the repository at this point in the history
  • Loading branch information
@mingshewhe
mingshewhe committed Sep 23, 2024
1 parent f31c3fb commit 4e466e4
Show file tree
Hide file tree
Showing 12 changed files with 109 additions and 157 deletions.
2 changes: 1 addition & 1 deletion ...mon-api/src/test/kotlin/com/tencent/devops/common/api/util/ObjectReplaceEnvVarUtilTest.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,7 @@ class ObjectReplaceEnvVarUtilTest {

assertEquals(
toJson(ObjectReplaceEnvVarUtil.replaceEnvVar(toJson(originDataMapObj), envMap)),
toJson(ObjectReplaceEnvVarUtil.replaceEnvVar(originDataMapObj, envMap)),
toJson(ObjectReplaceEnvVarUtil.replaceEnvVar(originDataMapObj, envMap))
)
}

Expand Down
23 changes: 23 additions & 0 deletions ...ipeline/src/main/kotlin/com/tencent/devops/common/pipeline/dialect/PipelineDialectType.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
package com.tencent.devops.common.pipeline.dialect

import com.tencent.devops.common.api.pojo.PipelineAsCodeSettings
import com.tencent.devops.common.pipeline.enums.ChannelCode

/**
* 流水线语法风格
Expand Down Expand Up @@ -70,6 +71,28 @@ enum class PipelineDialectType(val dialect: IPipelineDialect) {
}
}

fun getPipelineDialectType(
channelCode: ChannelCode,
asCodeSettings: PipelineAsCodeSettings?
): PipelineDialectType {
return when {
asCodeSettings == null -> CLASSIC
// stream并且开启pac需要使用制约模式
channelCode == ChannelCode.GIT && asCodeSettings.enable ->
CONSTRAINED

else -> {
with(asCodeSettings) {
getPipelineDialectType(
inheritedDialect = inheritedDialect,
projectDialect = projectDialect,
pipelineDialect = pipelineDialect
)
}
}
}
}

fun getPipelineDialect(
inheritedDialect: Boolean?,
projectDialect: String?,
Expand Down
1 change: 0 additions & 1 deletion ...ommon-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/StagePauseCheck.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@ package com.tencent.devops.common.pipeline.pojo
import com.tencent.devops.common.api.util.UUIDUtil
import com.tencent.devops.common.api.util.timestampmilli
import com.tencent.devops.common.pipeline.EnvReplacementParser
import com.tencent.devops.common.pipeline.dialect.IPipelineDialect
import com.tencent.devops.common.pipeline.enums.BuildStatus
import com.tencent.devops.common.pipeline.enums.ManualReviewAction
import com.tencent.devops.common.pipeline.option.StageControlOption
Expand Down
179 changes: 71 additions & 108 deletions ...ipeline/src/test/kotlin/com/tencent/devops/common/pipeline/EnvReplaceAndExpressionTest.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import com.tencent.devops.common.api.util.JsonUtil
import com.tencent.devops.common.api.util.JsonUtil.toJson
import com.tencent.devops.common.api.util.ObjectReplaceEnvVarUtil
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Assertions.assertNotEquals
import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.api.Test

Expand Down Expand Up @@ -71,34 +72,23 @@ class EnvReplaceAndExpressionTest {
dataMapObj["dataMapKey"] = "变量替换测试_\${specStrEnvVar}"
dataMapObj["testBean"] = testBean
originDataListObj.add(dataMapObj)
val convertDataObj = ObjectReplaceEnvVarUtil.replaceEnvVar(originDataListObj, envMap) as List<*>
val convertDataObj2 = JsonUtil.to(
EnvReplacementParser.parse(value = toJson(originDataListObj), envMap),
List::class.java
)

assertEquals(convertDataObj[0], convertDataObj2[0])
assertEquals(convertDataObj[1], convertDataObj2[1])
assertEquals(convertDataObj[2], convertDataObj2[2])
assertEquals(convertDataObj[3], convertDataObj2[3])
assertEquals(convertDataObj[4], convertDataObj2[4])

val convertTestBean = convertDataObj[5] as TestBean
assertEquals("bean变量替换测试_${envMap["specStrEnvVar"]}", convertTestBean.testBeanKey)
assertEquals(jsonExcept, convertTestBean.testBeanValue)
val convertDataObj = toJson(ObjectReplaceEnvVarUtil.replaceEnvVar(originDataListObj, envMap))
val convertDataObj2 = EnvReplacementParser.parse(value = toJson(originDataListObj), envMap)
assertEquals(convertDataObj, convertDataObj2)
}

@Test
fun replaceIllegalJson() {
val objectJson = "{\"abc:\"变量替换测试_\${normalStrEnvVar}\""
val convertDataObj1 = ObjectReplaceEnvVarUtil.replaceEnvVar(objectJson, envMap)
println(convertDataObj1)
assertEquals("{\"abc:\"变量替换测试_${envMap["normalStrEnvVar"]}\"", convertDataObj1)
val convertDataObj11 = EnvReplacementParser.parse(objectJson, envMap)

assertEquals(convertDataObj1, convertDataObj11)

val arrayJson = "[1, \"变量替换测试_\${normalStrEnvVar}\""
val convertDataObj2 = ObjectReplaceEnvVarUtil.replaceEnvVar(arrayJson, envMap)
println(convertDataObj2)
assertEquals("[1, \"变量替换测试_${envMap["normalStrEnvVar"]}\"", convertDataObj2)
val convertDataObj21 = EnvReplacementParser.parse(arrayJson, envMap)
assertEquals(convertDataObj2, convertDataObj21)
}

@Test
Expand All @@ -121,53 +111,9 @@ class EnvReplaceAndExpressionTest {
setDataMapObj["dataMapKey"] = "变量替换测试_\${specStrEnvVar}"
setDataMapObj["testBean"] = testBean
originDataSetObj.add(setDataMapObj)
val convertDataObj = (ObjectReplaceEnvVarUtil.replaceEnvVar(originDataSetObj, envMap) as Set<*>)

convertDataObj.forEach { member ->
when {
member is Map<*, *> -> {
member.forEach { sm ->
when {
sm.key.toString() == "testBean" -> {
assertEquals(
"bean变量替换测试_${envMap["specStrEnvVar"]}",
(sm.value as TestBean).testBeanKey
)
assertEquals(jsonExcept, (sm.value as TestBean).testBeanValue)
}
sm.key.toString() == "dataMapKey" -> {
assertEquals("变量替换测试_${envMap["specStrEnvVar"]}", sm.value)
}
else -> {
assertEquals(member.toString(), "setDataMapObj")
}
}
}
}
member is TestBean -> {
assertEquals("bean变量替换测试_${envMap["specStrEnvVar"]}", member.testBeanKey)
assertEquals(jsonExcept, member.testBeanValue)
}
member.toString().startsWith("1") -> {
assertEquals("1变量替换测试_${envMap["normalStrEnvVar"]}", member)
}
member.toString().startsWith("2") -> {
assertEquals("2变量替换测试_${envMap["specStrEnvVar"]}", member)
}
member.toString().startsWith("3") -> {
assertEquals("3变量替换测试_${envMap["jsonStrEnvVar"]}", member)
}
member.toString().startsWith("{") -> {
assertEquals(jsonExcept, member)
}
member.toString().startsWith("[") -> {
assertEquals(arrayJsonExcept, member)
}
else -> {
assertEquals(member.toString(), "convertDataObj")
}
}
}
val convertDataObj = toJson(ObjectReplaceEnvVarUtil.replaceEnvVar(originDataSetObj, envMap))
val convertDataObj2 = EnvReplacementParser.parse(toJson(originDataSetObj), envMap)
assertEquals(convertDataObj, convertDataObj2)
}

@Test
Expand All @@ -191,20 +137,25 @@ class EnvReplaceAndExpressionTest {
)
originSubDataMapObj["testBean"] = testBean
originDataMapObj["originSubDataMapObj"] = originSubDataMapObj

val cpb = ObjectReplaceEnvVarUtil.replaceEnvVar(originDataMapObj, envMap)
val testBeanMap = ((cpb as Map<String, Any>)["originSubDataMapObj"] as Map<String, Any>)["testBean"] as TestBean
assertEquals("变量替换测试_${envMap["specStrEnvVar"]}", testBeanMap.testBeanKey)
assertEquals(jsonExcept, testBeanMap.testBeanValue)
// 判断map中jsonStrEnvVarKey3对应的值进行变量替换后能否正常转换为json串
assertEquals(envMap["jsonStrEnvVar"], (cpb as Map<String?, Any?>)["jsonStrEnvVarKey3"]!!)
originSubDataMapObj = cpb["originSubDataMapObj"] as MutableMap<String?, Any?>?
// 判断嵌套的map中jsonStrEnvVarKey2对应的值进行变量替换后能否正常转换为json串
assertEquals(envMap["jsonStrEnvVar"], originSubDataMapObj!!["jsonStrEnvVarKey2"]!!)

assertEquals(
toJson(ObjectReplaceEnvVarUtil.replaceEnvVar(toJson(originDataMapObj), envMap)),
toJson(ObjectReplaceEnvVarUtil.replaceEnvVar(originDataMapObj, envMap)),
val cpb2 = JsonUtil.to(EnvReplacementParser.parse(toJson(originDataMapObj), envMap), Map::class.java)
val testBeanMap2 = JsonUtil.to(
toJson(JsonUtil.to(toJson(cpb2["originSubDataMapObj"]!!), Map::class.java)["testBean"]!!),
TestBean::class.java
)
assertEquals(testBeanMap.testBeanKey, testBeanMap2.testBeanKey)
assertEquals(testBeanMap.testBeanValue, testBeanMap2.testBeanValue)
// EnvReplacementParser对格式进行了压缩
assertNotEquals(
(cpb as Map<String?, Any?>)["jsonStrEnvVarKey3"]!!,
(cpb2 as Map<String?, Any?>)["jsonStrEnvVarKey3"]!!
)
originSubDataMapObj = cpb["originSubDataMapObj"] as MutableMap<String?, Any?>?
val originSubDataMapObj2 = cpb2["originSubDataMapObj"] as MutableMap<String?, Any?>?
assertNotEquals(originSubDataMapObj!!["jsonStrEnvVarKey2"]!!, originSubDataMapObj2!!["jsonStrEnvVarKey2"]!!)
}

@Test
Expand Down Expand Up @@ -251,66 +202,78 @@ class EnvReplaceAndExpressionTest {
testComplexBean.dataSet = dataSet

// start to test
var convertDataObj = ObjectReplaceEnvVarUtil.replaceEnvVar(testComplexBean, envMap)
val convertBean = convertDataObj as TestComplexBean
assertEquals("变量替换测试_${envMap["specStrEnvVar"]}", convertBean.testBeanKey)

assertEquals("变量替换测试_${envMap["normalStrEnvVar"]}", convertBean.dataList!![0])
assertEquals("变量替换测试_${envMap["specStrEnvVar"]}", convertBean.dataList!![1])
assertEquals("变量替换测试_${envMap["jsonStrEnvVar"]}", convertBean.dataList!![2])
assertEquals(jsonExcept, convertBean.dataList!![3])
assertEquals("[ \"变量替换测试_{\\\"abc\\\":\\\"123\\\"}\" ]", convertBean.dataList!![4])

assertEquals(" 变量替换测试_${envMap["normalStrEnvVar"]} ", convertBean.dataMap!!["normalStrEnvVarKey"])
assertEquals("变量替换测试_${envMap["specStrEnvVar"]}", convertBean.dataMap!!["specStrEnvVarKey"])
assertEquals("变量替换测试_${envMap["jsonStrEnvVar"]}", convertBean.dataMap!!["jsonStrEnvVarKey1"])
assertEquals(jsonExcept, convertBean.dataMap!!["jsonStrEnvVarKey2"])
assertEquals(arrayJsonExcept, convertBean.dataMap!!["jsonStrEnvVarKey3"])
var convertDataObj1 = ObjectReplaceEnvVarUtil.replaceEnvVar(testComplexBean, envMap)
val convertBean1 = convertDataObj1 as TestComplexBean
val convertBean2 = JsonUtil.to(EnvReplacementParser.parse(toJson(testComplexBean), envMap), TestComplexBean::class.java)

assertEquals(convertBean1.testBeanKey, convertBean2.testBeanKey)

assertEquals(convertBean1.dataList!![0], convertBean2.dataList!![0])
assertEquals(convertBean1.dataList!![1], convertBean2.dataList!![1])
assertEquals(convertBean1.dataList!![2], convertBean2.dataList!![2])
assertEquals(convertBean1.dataList!![3], convertBean2.dataList!![3])
assertEquals(convertBean1.dataList!![4], convertBean2.dataList!![4])

assertEquals(convertBean1.dataMap!!["normalStrEnvVarKey"], convertBean2.dataMap!!["normalStrEnvVarKey"])
assertEquals(convertBean1.dataMap!!["specStrEnvVarKey"], convertBean2.dataMap!!["specStrEnvVarKey"])
assertEquals(convertBean1.dataMap!!["jsonStrEnvVarKey1"], convertBean2.dataMap!!["jsonStrEnvVarKey1"])
assertEquals(convertBean1.dataMap!!["jsonStrEnvVarKey2"], convertBean2.dataMap!!["jsonStrEnvVarKey2"])
assertEquals(convertBean1.dataMap!!["jsonStrEnvVarKey3"], convertBean2.dataMap!!["jsonStrEnvVarKey3"])

// 替换包含null的对象
dataMap = HashMap()
dataMap["key1"] = "变量"
dataMap["key2"] = arrayOf<Any?>(null, "哈哈")

convertDataObj = ObjectReplaceEnvVarUtil.replaceEnvVar(dataMap, envMap) as Map<*, *>
assertEquals(dataMap["key1"], convertDataObj["key1"])
assertEquals(toJson(dataMap["key2"]!!), convertDataObj["key2"])
println("convertDataObj=$convertDataObj")
convertDataObj1 = ObjectReplaceEnvVarUtil.replaceEnvVar(dataMap, envMap) as Map<*, *>
var convertDataObj2 = JsonUtil.to(EnvReplacementParser.parse(toJson(dataMap), envMap), Map::class.java)
assertEquals(convertDataObj1["key1"], convertDataObj2["key1"])
assertEquals(convertDataObj1["key2"], convertDataObj2["key2"])
}

@Test
fun replaceEnvVar() {

// 对普通字符串进行普通字符串变量替换
var originDataObj: Any = "变量替换测试_\${normalStrEnvVar}"
var convertDataObj = ObjectReplaceEnvVarUtil.replaceEnvVar(originDataObj, envMap)
assertEquals("变量替换测试_123", toJson(convertDataObj))
var convertDataObj = toJson(ObjectReplaceEnvVarUtil.replaceEnvVar(originDataObj, envMap))
var convertDataObj2 = EnvReplacementParser.parse(toJson(originDataObj), envMap)
assertEquals(convertDataObj, convertDataObj2)

// 对普通字符串进行带特殊字符字符串变量替换
originDataObj = "变量替换测试_\${specStrEnvVar}"
convertDataObj = ObjectReplaceEnvVarUtil.replaceEnvVar(originDataObj, envMap)
assertEquals("变量替换测试_D:\\tmp\\hha", toJson(convertDataObj))
convertDataObj = toJson(ObjectReplaceEnvVarUtil.replaceEnvVar(originDataObj, envMap))
convertDataObj2 = EnvReplacementParser.parse(toJson(originDataObj), envMap)
assertEquals(convertDataObj, convertDataObj2)

// 对普通字符串进行json字符串变量替换
originDataObj = "变量替换测试_\${jsonStrEnvVar}"
convertDataObj = ObjectReplaceEnvVarUtil.replaceEnvVar(originDataObj, envMap)
assertEquals("变量替换测试_{\"abc\":\"123\"}", toJson(convertDataObj))
convertDataObj = toJson(ObjectReplaceEnvVarUtil.replaceEnvVar(originDataObj, envMap))
convertDataObj2 = EnvReplacementParser.parse(toJson(originDataObj), envMap)
assertEquals(convertDataObj, convertDataObj2)

// number类型变量替换
originDataObj = "[1,2,3]"
convertDataObj = ObjectReplaceEnvVarUtil.replaceEnvVar(originDataObj, envMap)
println(toJson(convertDataObj))
assertEquals(toJson(JsonUtil.to(originDataObj, List::class.java)), toJson(convertDataObj))
convertDataObj = toJson(ObjectReplaceEnvVarUtil.replaceEnvVar(originDataObj, envMap))
convertDataObj2 = EnvReplacementParser.parse(toJson(originDataObj), envMap)
assertEquals(convertDataObj, convertDataObj2)

// 魔法数字符创测试
convertDataObj = ObjectReplaceEnvVarUtil.replaceEnvVar("12E2", envMap)
assertEquals("12E2", toJson(convertDataObj))
convertDataObj = toJson(ObjectReplaceEnvVarUtil.replaceEnvVar("12E2", envMap))
convertDataObj2 = EnvReplacementParser.parse("12E2", envMap)
assertEquals(convertDataObj, convertDataObj2)
// 替换”[133]-[sid-${normalStrEnvVar}]-[sid-zhiliang-test1]“带多个[]的字符串
convertDataObj = ObjectReplaceEnvVarUtil.replaceEnvVar(
convertDataObj = toJson(
ObjectReplaceEnvVarUtil.replaceEnvVar(
"[133]-[sid-\${normalStrEnvVar}]-[sid-zhiliang-test1]",
envMap
)
)
convertDataObj2 = EnvReplacementParser.parse(
"[133]-[sid-\${normalStrEnvVar}]-[sid-zhiliang-test1]",
envMap
)
assertEquals("[133]-[sid-123]-[sid-zhiliang-test1]", toJson(convertDataObj))
assertEquals(convertDataObj, convertDataObj2)
}

internal data class TestBean(
Expand Down
2 changes: 1 addition & 1 deletion ...n-pipeline/src/test/kotlin/com/tencent/devops/common/pipeline/EnvReplacementParserTest.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -373,7 +373,7 @@ internal class EnvReplacementParserTest {
)
Assertions.assertEquals(
"hello\$variables.abc}}",
EnvReplacementParser.parse(command7, data, true),
EnvReplacementParser.parse(command7, data, true)
)
Assertions.assertEquals(
"echo hahahahaha",
Expand Down
25 changes: 2 additions & 23 deletions ...-base/src/main/kotlin/com/tencent/devops/process/service/pipeline/PipelineBuildService.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,6 @@ import com.tencent.devops.common.auth.api.ActionId
import com.tencent.devops.common.auth.api.ResourceTypeId
import com.tencent.devops.common.pipeline.Model
import com.tencent.devops.common.pipeline.container.TriggerContainer
import com.tencent.devops.common.pipeline.dialect.IPipelineDialect
import com.tencent.devops.common.pipeline.dialect.PipelineDialectType
import com.tencent.devops.common.pipeline.enums.BuildFormPropertyType
import com.tencent.devops.common.pipeline.enums.ChannelCode
Expand Down Expand Up @@ -85,7 +84,6 @@ import com.tencent.devops.process.utils.PIPELINE_START_USER_ID
import com.tencent.devops.process.utils.PIPELINE_START_USER_NAME
import com.tencent.devops.process.utils.PIPELINE_START_WEBHOOK_USER_ID
import com.tencent.devops.process.utils.PIPELINE_UPDATE_USER
import com.tencent.devops.process.utils.PIPELINE_VARIABLES_STRING_LENGTH_MAX
import com.tencent.devops.process.utils.PIPELINE_VERSION
import com.tencent.devops.process.utils.PROJECT_NAME
import com.tencent.devops.process.utils.PROJECT_NAME_CHINESE
Expand Down Expand Up @@ -192,12 +190,8 @@ class PipelineBuildService(
projectId = pipeline.projectId,
asCodeSettings = setting.pipelineAsCodeSettings
)
val pipelineDialectType = PipelineDialectType.getPipelineDialectType(asCodeSettings = asCodeSettings)
// 校验流水线启动变量长度
checkBuildVariablesLength(
pipelineDialect = pipelineDialectType.dialect,
startValues = startValues
)
val pipelineDialectType =
PipelineDialectType.getPipelineDialectType(channelCode = channelCode, asCodeSettings = asCodeSettings)

// 如果指定了版本号，则设置指定的版本号
pipeline.version = signPipelineVersion ?: pipeline.version
Expand Down Expand Up @@ -429,19 +423,4 @@ class PipelineBuildService(
}
// return originStartParams
}

private fun checkBuildVariablesLength(
pipelineDialect: IPipelineDialect,
startValues: Map<String, String>?
) {
val longVarNames = startValues?.filter {
it.value.length >= PIPELINE_VARIABLES_STRING_LENGTH_MAX
}?.map { it.key }
if (!longVarNames.isNullOrEmpty() && !pipelineDialect.supportLongVarValue()) {
throw ErrorCodeException(
errorCode = ProcessMessageCode.ERROR_PIPELINE_VARIABLES_OUT_OF_LENGTH,
params = arrayOf(longVarNames.toString())
)
}
}
}
Loading

0 comments on commit 4e466e4

Please sign in to comment.