Skip to content

Commit

Permalink
feat：流水线变量语法支持两种风格 #10576
Browse files Browse the repository at this point in the history
  • Loading branch information
@mingshewhe
mingshewhe committed Sep 21, 2024
1 parent 61eae42 commit e1d95ee
Show file tree
Hide file tree
Showing 8 changed files with 51 additions and 118 deletions.
2 changes: 1 addition & 1 deletion ...mmon-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/EnvReplacementContext.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ data class EnvReplacementContext(
value = value,
contextMap = contextMap,
useSingleCurlyBraces = dialect.supportUseSingleCurlyBracesVar(),
contextNotNull = dialect.supportMissingVar(),
contextNotNull = !dialect.supportMissingVar(),
contextPair = contextPair,
functions = functions,
output = output
Expand Down
42 changes: 0 additions & 42 deletions ...ommon-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/EnvReplacementParser.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,48 +64,6 @@ object EnvReplacementParser {
private val logger = LoggerFactory.getLogger(EnvReplacementParser::class.java)
private val expressionPattern = Pattern.compile("\\$[{]{2}([^$^{}]+)[}]{2}")

/**
* 根据环境变量map进行object处理并保持原类型
* @param value 等待进行环境变量替换的对象，可以是任意类型
* @param contextMap 环境变量map值
* @param contextPair 自定义表达式计算上下文（如果指定则不使用表达式替换或默认替换逻辑）
* @param onlyExpression 只进行表达式替换（若指定了自定义替换逻辑此字段无效，为false）
* @param functions 用户自定义的拓展用函数
* @param output 表达式计算时输出
*/
fun parse(
value: String?,
contextMap: Map<String, String>,
onlyExpression: Boolean? = false,
contextPair: Pair<ExecutionContext, List<NamedValueInfo>>? = null,
functions: Iterable<IFunctionInfo>? = null,
output: ExpressionOutput? = null
): String {
if (value.isNullOrBlank()) return ""
return if (onlyExpression == true) {
try {
val (context, nameValues) = contextPair
?: getCustomExecutionContextByMap(contextMap)
?: return value
parseExpression(
value = value,
context = context,
nameValues = nameValues,
functions = functions,
output = output,
contextNotNull = false
)
} catch (ignore: Throwable) {
logger.warn("[$value]|EnvReplacementParser expression invalid: ", ignore)
value
}
} else {
ObjectReplaceEnvVarUtil.replaceEnvVar(value, contextMap).let {
JsonUtil.toJson(it, false)
}
}
}

/**
* 根据环境变量map进行object处理并保持原类型
* 根据方言的配置判断是否能够使用${}或者变量值是否存在
Expand Down
2 changes: 0 additions & 2 deletions ...line/src/main/kotlin/com/tencent/devops/common/pipeline/dialect/ClassicPipelineDialect.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,6 @@ package com.tencent.devops.common.pipeline.dialect
class ClassicPipelineDialect : IPipelineDialect {
override fun supportUseSingleCurlyBracesVar() = true

override fun supportDirectAccessVar() = true

override fun supportLongVarValue() = true

override fun supportChineseVarName() = true
Expand Down
2 changes: 0 additions & 2 deletions .../src/main/kotlin/com/tencent/devops/common/pipeline/dialect/ConstrainedPipelineDialect.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,6 @@ package com.tencent.devops.common.pipeline.dialect
class ConstrainedPipelineDialect : IPipelineDialect {
override fun supportUseSingleCurlyBracesVar() = false

override fun supportDirectAccessVar() = false

override fun supportLongVarValue() = false

override fun supportChineseVarName() = false
Expand Down
7 changes: 0 additions & 7 deletions ...n-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/dialect/IPipelineDialect.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,13 +37,6 @@ interface IPipelineDialect {
*/
fun supportUseSingleCurlyBracesVar(): Boolean

/**
* 是否支持直接访问变量
*
* 是否可以通过${{aaa}}方式访问变量,还是必须通过上下文才能访问
*/
fun supportDirectAccessVar(): Boolean

/**
* 是否支持长变量
*/
Expand Down
104 changes: 44 additions & 60 deletions ...n-pipeline/src/test/kotlin/com/tencent/devops/common/pipeline/EnvReplacementParserTest.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -186,22 +186,19 @@ internal class EnvReplacementParserTest {
parseAndEquals(
data = map,
template = "\${{ (variables.TXT == 'txt') }}",
expect = true.toString(),
onlyExpression = true
expect = true.toString()
)

parseAndEquals(
data = map,
template = "\${{ (variables.TXT2 == 'txt2') }}",
expect = true.toString(),
onlyExpression = true
expect = true.toString()
)

parseAndEquals(
data = map,
template = "\${{ ((variables.TXT == 'txt') && (variables.TXT2 == 'txt2')) }}",
expect = true.toString(),
onlyExpression = true
expect = true.toString()
)

parseAndEquals(
Expand All @@ -213,8 +210,7 @@ internal class EnvReplacementParserTest {
expect = "echo \" 这可是来自master的改动 \"\n" +
" echo \" true \"\n" +
" echo \" true \"\n" +
" echo \" true \"",
onlyExpression = true
" echo \" true \""
)

parseAndEquals(
Expand All @@ -226,15 +222,13 @@ internal class EnvReplacementParserTest {
expect = "echo \" 这可是来自master的改动 \"\n" +
"echo \" false \"\n" +
"echo \" false \"\n" +
"echo \" false \"",
onlyExpression = true
"echo \" false \""
)

parseAndEquals(
data = map,
template = "{\"GDP\": \"\${{GDP}}亿\", \"People\": \${{People}} \"Country\": \"\${{twice}}\"}",
expect = "{\"GDP\": \"${map["GDP"]}亿\", \"People\": ${map["People"]} \"Country\": \"${map["Country"]}\"}",
onlyExpression = true
expect = "{\"GDP\": \"${map["GDP"]}亿\", \"People\": ${map["People"]} \"Country\": \"${map["Country"]}\"}"
)

val data = HashMap<String, String>()
Expand All @@ -243,55 +237,49 @@ internal class EnvReplacementParserTest {
data["t.cd"] = "\${{ab.cd}}"

val template2 = "abcd_\$abc}_ffs_\${{\${{ce}}_\${{ab.c}_ end"
val buff = EnvReplacementParser.parse(template2, data, true)
val buff = EnvReplacementParser.parse(template2, data)
Assertions.assertEquals(template2, buff)

parseAndEquals(
data = data,
template = "中国\$abc}_ffs_\${{\${{ce}}_\${{ab.c}_ end",
expect = "中国\$abc}_ffs_\${{\${{ce}}_\${{ab.c}_ end",
onlyExpression = true
expect = "中国\$abc}_ffs_\${{\${{ce}}_\${{ab.c}_ end"
)

parseAndEquals(
data = data,
template = "abcd_\${abc}_ffs_\${{ce}}_\${{t.cd}}_ end结束%\n # 这是注释行a1\$ab_^%!#@",
expect = "abcd_\${abc}_ffs_twice_${data["ab.cd"]}_ end结束%\n # 这是注释行a1\$ab_^%!#@",
contextMap = mapOf("ce" to "twice"),
onlyExpression = true
contextMap = mapOf("ce" to "twice")
)

data["c_e"] = "\${none}"
parseAndEquals(
data = data,
template = "abcd_\${abc}_ffs_\${{c_e}}_\${{t.cd}}_ end",
expect = "abcd_\${abc}_ffs_\${none}_${data["ab.cd"]}_ end",
onlyExpression = true
expect = "abcd_\${abc}_ffs_\${none}_${data["ab.cd"]}_ end"
)

data["center中"] = "中国"
// 新表达式中中文不支持作为key
parseAndEquals(
data = data,
template = "abcd_\${{center中}}_ffs",
expect = "abcd_\${{center中}}_ffs",
onlyExpression = true
expect = "abcd_\${{center中}}_ffs"
)

data["blank"] = ""
parseAndEquals(
data = data,
template = "\${{blank}}",
expect = "",
onlyExpression = true
expect = ""
)

data["all"] = "hello"
parseAndEquals(
data = data,
template = "\${{all}}",
expect = "hello",
onlyExpression = true
expect = "hello"
)
}

Expand Down Expand Up @@ -328,8 +316,7 @@ internal class EnvReplacementParserTest {
"echo envs.env_e=e, env_e=\$env_e\n" +
"echo envs.a=, a=\$a\n" +
"echo settings.sensitive.password=\${{ settings.sensitive.password }}\n" +
"echo ::set-output name=a::i am a at step_1",
onlyExpression = true
"echo ::set-output name=a::i am a at step_1"
)
}

Expand All @@ -356,47 +343,46 @@ internal class EnvReplacementParserTest {
)
// 与EnvUtils的差异点：不支持传可空对象
// Assertions.assertEquals("", EnvReplacementParser.parse(null, data))
Assertions.assertEquals("", EnvReplacementParser.parse("", data, true))
Assertions.assertEquals("", EnvReplacementParser.parse(null, data, true))
Assertions.assertEquals("", EnvReplacementParser.parse("", data))
Assertions.assertEquals("", EnvReplacementParser.parse(null, data))

Assertions.assertEquals(
"hello variables.value world",
EnvReplacementParser.parse(command1, data, true)
EnvReplacementParser.parse(command1, data)
)
Assertions.assertEquals(
"variables.valueworld",
EnvReplacementParser.parse(command2, data, true)
EnvReplacementParser.parse(command2, data)
)
Assertions.assertEquals(
"hellovariables.value",
EnvReplacementParser.parse(command3, data, true)
EnvReplacementParser.parse(command3, data)
)
Assertions.assertEquals(
"hello\${{variables.abc",
EnvReplacementParser.parse(command4, data, true)
EnvReplacementParser.parse(command4, data)
)
Assertions.assertEquals(
"hello\${{variables.abc}",
EnvReplacementParser.parse(command5, data, true)
EnvReplacementParser.parse(command5, data)
)
Assertions.assertEquals(
command6,
EnvReplacementParser.parse(command6, data, true)
EnvReplacementParser.parse(command6, data)
)
Assertions.assertEquals(
"hello\$variables.abc}}",
EnvReplacementParser.parse(command7, data, true)
EnvReplacementParser.parse(command7, data)
)
Assertions.assertEquals(
"echo hahahahaha",
EnvReplacementParser.parse(command8, data, true)
EnvReplacementParser.parse(command8, data)
)
Assertions.assertEquals(
"echo /data/landun/workspace",
EnvReplacementParser.parse(
value = command9,
contextMap = map.plus("ci.workspace" to "/data/landun/workspace"),
onlyExpression = true
contextMap = map.plus("ci.workspace" to "/data/landun/workspace")
)
)
}
Expand All @@ -410,7 +396,7 @@ internal class EnvReplacementParserTest {
println("parseEnvTestData $command")
Assertions.assertEquals(
"{\"age\": ${map["age"]} , \"sex\": \"boy\", \"name\": ${map["name"]}}",
EnvReplacementParser.parse(command, map, true)
EnvReplacementParser.parse(command, map)
)

val command1 = "hello \${{variables.abc}} world"
Expand All @@ -436,27 +422,26 @@ internal class EnvReplacementParserTest {
"{variables.abc" to "jacky"
)

Assertions.assertEquals("hello variables.value world", EnvReplacementParser.parse(command1, data, true))
Assertions.assertEquals("variables.valueworld", EnvReplacementParser.parse(command2, data, true))
Assertions.assertEquals("hellovariables.value", EnvReplacementParser.parse(command3, data, true))
Assertions.assertEquals("hello\${{variables.abc", EnvReplacementParser.parse(command4, data, true))
Assertions.assertEquals("hello\${{variables.abc}", EnvReplacementParser.parse(command5, data, true))
Assertions.assertEquals("hello\${variables.abc}}", EnvReplacementParser.parse(command6, data, true))
Assertions.assertEquals("hello\$variables.abc}}", EnvReplacementParser.parse(command7, data, true))
Assertions.assertEquals("echo hahahahaha", EnvReplacementParser.parse(command8, data, true))
Assertions.assertEquals("hello variables.value world", EnvReplacementParser.parse(command1, data))
Assertions.assertEquals("variables.valueworld", EnvReplacementParser.parse(command2, data))
Assertions.assertEquals("hellovariables.value", EnvReplacementParser.parse(command3, data))
Assertions.assertEquals("hello\${{variables.abc", EnvReplacementParser.parse(command4, data))
Assertions.assertEquals("hello\${{variables.abc}", EnvReplacementParser.parse(command5, data))
Assertions.assertEquals("hello\${variables.abc}}", EnvReplacementParser.parse(command6, data))
Assertions.assertEquals("hello\$variables.abc}}", EnvReplacementParser.parse(command7, data))
Assertions.assertEquals("echo hahahahaha", EnvReplacementParser.parse(command8, data))
Assertions.assertEquals(
"echo /data/landun/workspace || hahahahaha",
EnvReplacementParser.parse(
value = command9,
contextMap = data.plus("ci.workspace" to "/data/landun/workspace"),
onlyExpression = true
contextMap = data.plus("ci.workspace" to "/data/landun/workspace")
)
)
Assertions.assertEquals("echo \${{ ci.xyz == 'zzzz' }}", EnvReplacementParser.parse(command10, data, true))
Assertions.assertEquals("echo true", EnvReplacementParser.parse(command11, data, true))
Assertions.assertEquals("echo false", EnvReplacementParser.parse(command12, data, true))
Assertions.assertEquals("echo true", EnvReplacementParser.parse(command13, data, true))
Assertions.assertEquals("true", EnvReplacementParser.parse(command14, data, true))
Assertions.assertEquals("echo \${{ ci.xyz == 'zzzz' }}", EnvReplacementParser.parse(command10, data))
Assertions.assertEquals("echo true", EnvReplacementParser.parse(command11, data))
Assertions.assertEquals("echo false", EnvReplacementParser.parse(command12, data))
Assertions.assertEquals("echo true", EnvReplacementParser.parse(command13, data))
Assertions.assertEquals("true", EnvReplacementParser.parse(command14, data))
}

@Test
Expand Down Expand Up @@ -498,7 +483,7 @@ console.log("全局配置", variables)
let branch = master
let branch1 =
let branchs = branch.split("/")"""
Assertions.assertEquals(result, EnvReplacementParser.parse(command1, data, true))
Assertions.assertEquals(result, EnvReplacementParser.parse(command1, data))
Assertions.assertEquals(result, EnvReplacementParser.parse(command1, data, ClassicPipelineDialect()))
Assertions.assertThrows(VariableNotFoundException::class.java) {
EnvReplacementParser.parse(command1, data, ConstrainedPipelineDialect())
Expand Down Expand Up @@ -555,7 +540,7 @@ echo true"""
"variables.is_build" to "false",
"ci.branch" to "master"
)
Assertions.assertEquals(result, EnvReplacementParser.parse(command1, data, true))
Assertions.assertEquals(result, EnvReplacementParser.parse(command1, data))
Assertions.assertEquals(result, EnvReplacementParser.parse(command1, data, ClassicPipelineDialect()))
Assertions.assertThrows(VariableNotFoundException::class.java) {
EnvReplacementParser.parse(command1, mapOf(), ConstrainedPipelineDialect())
Expand All @@ -566,10 +551,9 @@ echo true"""
data: Map<String, String>,
template: String,
expect: String,
contextMap: Map<String, String> = emptyMap(),
onlyExpression: Boolean? = false
contextMap: Map<String, String> = emptyMap()
) {
val buff = EnvReplacementParser.parse(template, contextMap.plus(data), onlyExpression)
val buff = EnvReplacementParser.parse(template, contextMap.plus(data))
println("template=$template\nreplaced=$buff\n")
Assertions.assertEquals(expect, buff)
}
Expand Down
9 changes: 6 additions & 3 deletions ...kotlin/com/tencent/devops/stream/trigger/actions/streamActions/StreamRepoTriggerAction.kt
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package com.tencent.devops.stream.trigger.actions.streamActions

import com.tencent.devops.common.client.Client
import com.tencent.devops.common.pipeline.EnvReplacementContext
import com.tencent.devops.common.pipeline.EnvReplacementParser
import com.tencent.devops.common.pipeline.enums.StartType
import com.tencent.devops.common.pipeline.utils.PIPELINE_GIT_REPO_CREATE_TIME
Expand Down Expand Up @@ -331,9 +332,11 @@ class StreamRepoTriggerAction(
it.forEach { condition ->
// 进行表达式计算
if (EnvReplacementParser.parse(
value = "\${{ $condition }}",
contextMap = supportVar,
onlyExpression = true
EnvReplacementContext(
value = "\${{ $condition }}",
contextMap = supportVar,
useSingleCurlyBraces = false
)
).contains("true")
) {
return TriggerBody().triggerFail(
Expand Down
1 change: 0 additions & 1 deletion ...worker-common/src/test/kotlin/com/tencent/devops/worker/common/task/MarketAtomTaskTest.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,6 @@ internal class MarketAtomTaskTest {
EnvReplacementParser.parse(
value = "\${{ settings.a.password }}",
contextMap = variables,
onlyExpression = true,
contextPair = r,
functions = SpecialFunctions.functions
)
Expand Down

0 comments on commit e1d95ee

Please sign in to comment.