From e1d95ee016000e8c71618af8ce50b7d94871df74 Mon Sep 17 00:00:00 2001 From: mingshewhe Date: Sat, 21 Sep 2024 15:56:57 +0800 Subject: [PATCH] =?UTF-8?q?feat=EF=BC=9A=E6=B5=81=E6=B0=B4=E7=BA=BF?= =?UTF-8?q?=E5=8F=98=E9=87=8F=E8=AF=AD=E6=B3=95=E6=94=AF=E6=8C=81=E4=B8=A4?= =?UTF-8?q?=E7=A7=8D=E9=A3=8E=E6=A0=BC=20#10576?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../common/pipeline/EnvReplacementContext.kt | 2 +- .../common/pipeline/EnvReplacementParser.kt | 42 ------- .../dialect/ClassicPipelineDialect.kt | 2 - .../dialect/ConstrainedPipelineDialect.kt | 2 - .../pipeline/dialect/IPipelineDialect.kt | 7 -- .../pipeline/EnvReplacementParserTest.kt | 104 ++++++++---------- .../streamActions/StreamRepoTriggerAction.kt | 9 +- .../worker/common/task/MarketAtomTaskTest.kt | 1 - 8 files changed, 51 insertions(+), 118 deletions(-) diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/EnvReplacementContext.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/EnvReplacementContext.kt index bd943773162e..ed073fadc98b 100644 --- a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/EnvReplacementContext.kt +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/EnvReplacementContext.kt @@ -63,7 +63,7 @@ data class EnvReplacementContext( value = value, contextMap = contextMap, useSingleCurlyBraces = dialect.supportUseSingleCurlyBracesVar(), - contextNotNull = dialect.supportMissingVar(), + contextNotNull = !dialect.supportMissingVar(), contextPair = contextPair, functions = functions, output = output diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/EnvReplacementParser.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/EnvReplacementParser.kt index 2b216b447ed6..d606e6606cdb 100644 --- a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/EnvReplacementParser.kt +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/EnvReplacementParser.kt @@ -64,48 +64,6 @@ object EnvReplacementParser { private val logger = LoggerFactory.getLogger(EnvReplacementParser::class.java) private val expressionPattern = Pattern.compile("\\$[{]{2}([^$^{}]+)[}]{2}") - /** - * 根据环境变量map进行object处理并保持原类型 - * @param value 等待进行环境变量替换的对象，可以是任意类型 - * @param contextMap 环境变量map值 - * @param contextPair 自定义表达式计算上下文（如果指定则不使用表达式替换或默认替换逻辑） - * @param onlyExpression 只进行表达式替换（若指定了自定义替换逻辑此字段无效，为false） - * @param functions 用户自定义的拓展用函数 - * @param output 表达式计算时输出 - */ - fun parse( - value: String?, - contextMap: Map, - onlyExpression: Boolean? = false, - contextPair: Pair>? = null, - functions: Iterable? = null, - output: ExpressionOutput? = null - ): String { - if (value.isNullOrBlank()) return "" - return if (onlyExpression == true) { - try { - val (context, nameValues) = contextPair - ?: getCustomExecutionContextByMap(contextMap) - ?: return value - parseExpression( - value = value, - context = context, - nameValues = nameValues, - functions = functions, - output = output, - contextNotNull = false - ) - } catch (ignore: Throwable) { - logger.warn("[$value]|EnvReplacementParser expression invalid: ", ignore) - value - } - } else { - ObjectReplaceEnvVarUtil.replaceEnvVar(value, contextMap).let { - JsonUtil.toJson(it, false) - } - } - } - /** * 根据环境变量map进行object处理并保持原类型 * 根据方言的配置判断是否能够使用${}或者变量值是否存在 diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/dialect/ClassicPipelineDialect.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/dialect/ClassicPipelineDialect.kt index 84afb86df447..ef4ad0ec3b9f 100644 --- a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/dialect/ClassicPipelineDialect.kt +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/dialect/ClassicPipelineDialect.kt @@ -33,8 +33,6 @@ package com.tencent.devops.common.pipeline.dialect class ClassicPipelineDialect : IPipelineDialect { override fun supportUseSingleCurlyBracesVar() = true - override fun supportDirectAccessVar() = true - override fun supportLongVarValue() = true override fun supportChineseVarName() = true diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/dialect/ConstrainedPipelineDialect.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/dialect/ConstrainedPipelineDialect.kt index 2cddb3de51b7..e4b9ef4f378f 100644 --- a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/dialect/ConstrainedPipelineDialect.kt +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/dialect/ConstrainedPipelineDialect.kt @@ -33,8 +33,6 @@ package com.tencent.devops.common.pipeline.dialect class ConstrainedPipelineDialect : IPipelineDialect { override fun supportUseSingleCurlyBracesVar() = false - override fun supportDirectAccessVar() = false - override fun supportLongVarValue() = false override fun supportChineseVarName() = false diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/dialect/IPipelineDialect.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/dialect/IPipelineDialect.kt index ed69eea7e11f..46903d806a50 100644 --- a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/dialect/IPipelineDialect.kt +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/dialect/IPipelineDialect.kt @@ -37,13 +37,6 @@ interface IPipelineDialect { */ fun supportUseSingleCurlyBracesVar(): Boolean - /** - * 是否支持直接访问变量 - * - * 是否可以通过${{aaa}}方式访问变量,还是必须通过上下文才能访问 - */ - fun supportDirectAccessVar(): Boolean - /** * 是否支持长变量 */ diff --git a/src/backend/ci/core/common/common-pipeline/src/test/kotlin/com/tencent/devops/common/pipeline/EnvReplacementParserTest.kt b/src/backend/ci/core/common/common-pipeline/src/test/kotlin/com/tencent/devops/common/pipeline/EnvReplacementParserTest.kt index 1424fe4848ca..c739fc4e475f 100644 --- a/src/backend/ci/core/common/common-pipeline/src/test/kotlin/com/tencent/devops/common/pipeline/EnvReplacementParserTest.kt +++ b/src/backend/ci/core/common/common-pipeline/src/test/kotlin/com/tencent/devops/common/pipeline/EnvReplacementParserTest.kt @@ -186,22 +186,19 @@ internal class EnvReplacementParserTest { parseAndEquals( data = map, template = "\${{ (variables.TXT == 'txt') }}", - expect = true.toString(), - onlyExpression = true + expect = true.toString() ) parseAndEquals( data = map, template = "\${{ (variables.TXT2 == 'txt2') }}", - expect = true.toString(), - onlyExpression = true + expect = true.toString() ) parseAndEquals( data = map, template = "\${{ ((variables.TXT == 'txt') && (variables.TXT2 == 'txt2')) }}", - expect = true.toString(), - onlyExpression = true + expect = true.toString() ) parseAndEquals( @@ -213,8 +210,7 @@ internal class EnvReplacementParserTest { expect = "echo \" 这可是来自master的改动 \"\n" + " echo \" true \"\n" + " echo \" true \"\n" + - " echo \" true \"", - onlyExpression = true + " echo \" true \"" ) parseAndEquals( @@ -226,15 +222,13 @@ internal class EnvReplacementParserTest { expect = "echo \" 这可是来自master的改动 \"\n" + "echo \" false \"\n" + "echo \" false \"\n" + - "echo \" false \"", - onlyExpression = true + "echo \" false \"" ) parseAndEquals( data = map, template = "{\"GDP\": \"\${{GDP}}亿\", \"People\": \${{People}} \"Country\": \"\${{twice}}\"}", - expect = "{\"GDP\": \"${map["GDP"]}亿\", \"People\": ${map["People"]} \"Country\": \"${map["Country"]}\"}", - onlyExpression = true + expect = "{\"GDP\": \"${map["GDP"]}亿\", \"People\": ${map["People"]} \"Country\": \"${map["Country"]}\"}" ) val data = HashMap() @@ -243,30 +237,27 @@ internal class EnvReplacementParserTest { data["t.cd"] = "\${{ab.cd}}" val template2 = "abcd_\$abc}_ffs_\${{\${{ce}}_\${{ab.c}_ end" - val buff = EnvReplacementParser.parse(template2, data, true) + val buff = EnvReplacementParser.parse(template2, data) Assertions.assertEquals(template2, buff) parseAndEquals( data = data, template = "中国\$abc}_ffs_\${{\${{ce}}_\${{ab.c}_ end", - expect = "中国\$abc}_ffs_\${{\${{ce}}_\${{ab.c}_ end", - onlyExpression = true + expect = "中国\$abc}_ffs_\${{\${{ce}}_\${{ab.c}_ end" ) parseAndEquals( data = data, template = "abcd_\${abc}_ffs_\${{ce}}_\${{t.cd}}_ end结束%\n # 这是注释行a1\$ab_^%!#@", expect = "abcd_\${abc}_ffs_twice_${data["ab.cd"]}_ end结束%\n # 这是注释行a1\$ab_^%!#@", - contextMap = mapOf("ce" to "twice"), - onlyExpression = true + contextMap = mapOf("ce" to "twice") ) data["c_e"] = "\${none}" parseAndEquals( data = data, template = "abcd_\${abc}_ffs_\${{c_e}}_\${{t.cd}}_ end", - expect = "abcd_\${abc}_ffs_\${none}_${data["ab.cd"]}_ end", - onlyExpression = true + expect = "abcd_\${abc}_ffs_\${none}_${data["ab.cd"]}_ end" ) data["center中"] = "中国" @@ -274,24 +265,21 @@ internal class EnvReplacementParserTest { parseAndEquals( data = data, template = "abcd_\${{center中}}_ffs", - expect = "abcd_\${{center中}}_ffs", - onlyExpression = true + expect = "abcd_\${{center中}}_ffs" ) data["blank"] = "" parseAndEquals( data = data, template = "\${{blank}}", - expect = "", - onlyExpression = true + expect = "" ) data["all"] = "hello" parseAndEquals( data = data, template = "\${{all}}", - expect = "hello", - onlyExpression = true + expect = "hello" ) } @@ -328,8 +316,7 @@ internal class EnvReplacementParserTest { "echo envs.env_e=e, env_e=\$env_e\n" + "echo envs.a=, a=\$a\n" + "echo settings.sensitive.password=\${{ settings.sensitive.password }}\n" + - "echo ::set-output name=a::i am a at step_1", - onlyExpression = true + "echo ::set-output name=a::i am a at step_1" ) } @@ -356,47 +343,46 @@ internal class EnvReplacementParserTest { ) // 与EnvUtils的差异点：不支持传可空对象 // Assertions.assertEquals("", EnvReplacementParser.parse(null, data)) - Assertions.assertEquals("", EnvReplacementParser.parse("", data, true)) - Assertions.assertEquals("", EnvReplacementParser.parse(null, data, true)) + Assertions.assertEquals("", EnvReplacementParser.parse("", data)) + Assertions.assertEquals("", EnvReplacementParser.parse(null, data)) Assertions.assertEquals( "hello variables.value world", - EnvReplacementParser.parse(command1, data, true) + EnvReplacementParser.parse(command1, data) ) Assertions.assertEquals( "variables.valueworld", - EnvReplacementParser.parse(command2, data, true) + EnvReplacementParser.parse(command2, data) ) Assertions.assertEquals( "hellovariables.value", - EnvReplacementParser.parse(command3, data, true) + EnvReplacementParser.parse(command3, data) ) Assertions.assertEquals( "hello\${{variables.abc", - EnvReplacementParser.parse(command4, data, true) + EnvReplacementParser.parse(command4, data) ) Assertions.assertEquals( "hello\${{variables.abc}", - EnvReplacementParser.parse(command5, data, true) + EnvReplacementParser.parse(command5, data) ) Assertions.assertEquals( command6, - EnvReplacementParser.parse(command6, data, true) + EnvReplacementParser.parse(command6, data) ) Assertions.assertEquals( "hello\$variables.abc}}", - EnvReplacementParser.parse(command7, data, true) + EnvReplacementParser.parse(command7, data) ) Assertions.assertEquals( "echo hahahahaha", - EnvReplacementParser.parse(command8, data, true) + EnvReplacementParser.parse(command8, data) ) Assertions.assertEquals( "echo /data/landun/workspace", EnvReplacementParser.parse( value = command9, - contextMap = map.plus("ci.workspace" to "/data/landun/workspace"), - onlyExpression = true + contextMap = map.plus("ci.workspace" to "/data/landun/workspace") ) ) } @@ -410,7 +396,7 @@ internal class EnvReplacementParserTest { println("parseEnvTestData $command") Assertions.assertEquals( "{\"age\": ${map["age"]} , \"sex\": \"boy\", \"name\": ${map["name"]}}", - EnvReplacementParser.parse(command, map, true) + EnvReplacementParser.parse(command, map) ) val command1 = "hello \${{variables.abc}} world" @@ -436,27 +422,26 @@ internal class EnvReplacementParserTest { "{variables.abc" to "jacky" ) - Assertions.assertEquals("hello variables.value world", EnvReplacementParser.parse(command1, data, true)) - Assertions.assertEquals("variables.valueworld", EnvReplacementParser.parse(command2, data, true)) - Assertions.assertEquals("hellovariables.value", EnvReplacementParser.parse(command3, data, true)) - Assertions.assertEquals("hello\${{variables.abc", EnvReplacementParser.parse(command4, data, true)) - Assertions.assertEquals("hello\${{variables.abc}", EnvReplacementParser.parse(command5, data, true)) - Assertions.assertEquals("hello\${variables.abc}}", EnvReplacementParser.parse(command6, data, true)) - Assertions.assertEquals("hello\$variables.abc}}", EnvReplacementParser.parse(command7, data, true)) - Assertions.assertEquals("echo hahahahaha", EnvReplacementParser.parse(command8, data, true)) + Assertions.assertEquals("hello variables.value world", EnvReplacementParser.parse(command1, data)) + Assertions.assertEquals("variables.valueworld", EnvReplacementParser.parse(command2, data)) + Assertions.assertEquals("hellovariables.value", EnvReplacementParser.parse(command3, data)) + Assertions.assertEquals("hello\${{variables.abc", EnvReplacementParser.parse(command4, data)) + Assertions.assertEquals("hello\${{variables.abc}", EnvReplacementParser.parse(command5, data)) + Assertions.assertEquals("hello\${variables.abc}}", EnvReplacementParser.parse(command6, data)) + Assertions.assertEquals("hello\$variables.abc}}", EnvReplacementParser.parse(command7, data)) + Assertions.assertEquals("echo hahahahaha", EnvReplacementParser.parse(command8, data)) Assertions.assertEquals( "echo /data/landun/workspace || hahahahaha", EnvReplacementParser.parse( value = command9, - contextMap = data.plus("ci.workspace" to "/data/landun/workspace"), - onlyExpression = true + contextMap = data.plus("ci.workspace" to "/data/landun/workspace") ) ) - Assertions.assertEquals("echo \${{ ci.xyz == 'zzzz' }}", EnvReplacementParser.parse(command10, data, true)) - Assertions.assertEquals("echo true", EnvReplacementParser.parse(command11, data, true)) - Assertions.assertEquals("echo false", EnvReplacementParser.parse(command12, data, true)) - Assertions.assertEquals("echo true", EnvReplacementParser.parse(command13, data, true)) - Assertions.assertEquals("true", EnvReplacementParser.parse(command14, data, true)) + Assertions.assertEquals("echo \${{ ci.xyz == 'zzzz' }}", EnvReplacementParser.parse(command10, data)) + Assertions.assertEquals("echo true", EnvReplacementParser.parse(command11, data)) + Assertions.assertEquals("echo false", EnvReplacementParser.parse(command12, data)) + Assertions.assertEquals("echo true", EnvReplacementParser.parse(command13, data)) + Assertions.assertEquals("true", EnvReplacementParser.parse(command14, data)) } @Test @@ -498,7 +483,7 @@ console.log("全局配置", variables) let branch = master let branch1 = let branchs = branch.split("/")""" - Assertions.assertEquals(result, EnvReplacementParser.parse(command1, data, true)) + Assertions.assertEquals(result, EnvReplacementParser.parse(command1, data)) Assertions.assertEquals(result, EnvReplacementParser.parse(command1, data, ClassicPipelineDialect())) Assertions.assertThrows(VariableNotFoundException::class.java) { EnvReplacementParser.parse(command1, data, ConstrainedPipelineDialect()) @@ -555,7 +540,7 @@ echo true""" "variables.is_build" to "false", "ci.branch" to "master" ) - Assertions.assertEquals(result, EnvReplacementParser.parse(command1, data, true)) + Assertions.assertEquals(result, EnvReplacementParser.parse(command1, data)) Assertions.assertEquals(result, EnvReplacementParser.parse(command1, data, ClassicPipelineDialect())) Assertions.assertThrows(VariableNotFoundException::class.java) { EnvReplacementParser.parse(command1, mapOf(), ConstrainedPipelineDialect()) @@ -566,10 +551,9 @@ echo true""" data: Map, template: String, expect: String, - contextMap: Map = emptyMap(), - onlyExpression: Boolean? = false + contextMap: Map = emptyMap() ) { - val buff = EnvReplacementParser.parse(template, contextMap.plus(data), onlyExpression) + val buff = EnvReplacementParser.parse(template, contextMap.plus(data)) println("template=$template\nreplaced=$buff\n") Assertions.assertEquals(expect, buff) } diff --git a/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/trigger/actions/streamActions/StreamRepoTriggerAction.kt b/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/trigger/actions/streamActions/StreamRepoTriggerAction.kt index bf3cb3e91c83..69381e503617 100644 --- a/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/trigger/actions/streamActions/StreamRepoTriggerAction.kt +++ b/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/trigger/actions/streamActions/StreamRepoTriggerAction.kt @@ -1,6 +1,7 @@ package com.tencent.devops.stream.trigger.actions.streamActions import com.tencent.devops.common.client.Client +import com.tencent.devops.common.pipeline.EnvReplacementContext import com.tencent.devops.common.pipeline.EnvReplacementParser import com.tencent.devops.common.pipeline.enums.StartType import com.tencent.devops.common.pipeline.utils.PIPELINE_GIT_REPO_CREATE_TIME @@ -331,9 +332,11 @@ class StreamRepoTriggerAction( it.forEach { condition -> // 进行表达式计算 if (EnvReplacementParser.parse( - value = "\${{ $condition }}", - contextMap = supportVar, - onlyExpression = true + EnvReplacementContext( + value = "\${{ $condition }}", + contextMap = supportVar, + useSingleCurlyBraces = false + ) ).contains("true") ) { return TriggerBody().triggerFail( diff --git a/src/backend/ci/core/worker/worker-common/src/test/kotlin/com/tencent/devops/worker/common/task/MarketAtomTaskTest.kt b/src/backend/ci/core/worker/worker-common/src/test/kotlin/com/tencent/devops/worker/common/task/MarketAtomTaskTest.kt index 9c374a3f5f1c..4690675b5113 100644 --- a/src/backend/ci/core/worker/worker-common/src/test/kotlin/com/tencent/devops/worker/common/task/MarketAtomTaskTest.kt +++ b/src/backend/ci/core/worker/worker-common/src/test/kotlin/com/tencent/devops/worker/common/task/MarketAtomTaskTest.kt @@ -95,7 +95,6 @@ internal class MarketAtomTaskTest { EnvReplacementParser.parse( value = "\${{ settings.a.password }}", contextMap = variables, - onlyExpression = true, contextPair = r, functions = SpecialFunctions.functions )