From 0d3641ca0b27e5a1947f5c40d79ee562eef48875 Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Mon, 21 Oct 2024 17:14:04 +0200 Subject: [PATCH 01/15] Start with python match statement --- .../cpg/graph/statements/SwitchStatement.kt | 4 +- .../cpg/frontends/python/ExpressionHandler.kt | 36 ++++--- .../cpg/frontends/python/StatementHandler.kt | 93 ++++++++++++++++++- 3 files changed, 118 insertions(+), 15 deletions(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/SwitchStatement.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/SwitchStatement.kt index e677cc8911..9d955a5057 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/SwitchStatement.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/SwitchStatement.kt @@ -36,7 +36,7 @@ import org.neo4j.ogm.annotation.Relationship /** * Represents a Java or C++ switch statement of the `switch (selector) {...}` that can include case - * and default statements. Break statements break out of the switch and labeled breaks in JAva are + * and default statements. Break statements break out of the switch and labeled breaks in Java are * handled properly. */ class SwitchStatement : Statement(), BranchingNode { @@ -51,7 +51,7 @@ class SwitchStatement : Statement(), BranchingNode { @Relationship(value = "SELECTOR_DECLARATION") var selectorDeclarationEdge = astOptionalEdgeOf() - /** C++ allows to use a declaration instead of a expression as selector */ + /** C++ allows to use a declaration instead of an expression as selector */ var selectorDeclaration by unwrapping(SwitchStatement::selectorDeclarationEdge) @Relationship(value = "STATEMENT") var statementEdge = astOptionalEdgeOf() diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt index 951d01cf62..bfd039336b 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt @@ -159,6 +159,29 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : } } + fun joinListWithBinOp( + operatorCode: String, + nodes: List, + rawNode: Python.AST.AST? = null, + isImplicit: Boolean = true + ): BinaryOperator { + val lastTwo = newBinaryOperator(operatorCode, rawNode = rawNode) + lastTwo.rhs = nodes.last() + lastTwo.lhs = nodes[nodes.size - 2] + return nodes.subList(0, nodes.size - 2).foldRight(lastTwo) { newVal, start -> + val nextValue = newBinaryOperator(operatorCode) + if (isImplicit && rawNode != null) + nextValue.implicit( + code = frontend.codeOf(rawNode), + location = frontend.locationOf(rawNode) + ) + else if (isImplicit) nextValue.implicit() + nextValue.rhs = start + nextValue.lhs = newVal + nextValue + } + } + private fun handleStarred(node: Python.AST.Starred): Expression { val unaryOp = newUnaryOperator("*", postfix = false, prefix = false, rawNode = node) unaryOp.input = handle(node.value) @@ -203,18 +226,7 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : rawNode = node ) } else { - // Start with the last two operands, then keep prepending the previous ones until the - // list is finished. - val lastTwo = newBinaryOperator(op, rawNode = node) - lastTwo.rhs = handle(node.values.last()) - lastTwo.lhs = handle(node.values[node.values.size - 2]) - return node.values.subList(0, node.values.size - 2).foldRight(lastTwo) { newVal, start - -> - val nextValue = newBinaryOperator(op, rawNode = node) - nextValue.rhs = start - nextValue.lhs = handle(newVal) - nextValue - } + joinListWithBinOp(op, node.values.map(::handle), node, true) } } diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt index 709bde56a9..2632731b39 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt @@ -77,7 +77,7 @@ class StatementHandler(frontend: PythonLanguageFrontend) : is Python.AST.Global -> handleGlobal(node) is Python.AST.Nonlocal -> handleNonLocal(node) is Python.AST.Raise -> handleRaise(node) - is Python.AST.Match, + is Python.AST.Match -> handleMatch(node) is Python.AST.TryStar -> newProblemExpression( "The statement of class ${node.javaClass} is not supported yet", @@ -86,6 +86,97 @@ class StatementHandler(frontend: PythonLanguageFrontend) : } } + /** + * Translates a pattern which can be used by a `match_case`. There are various options available + * and all of them are translated to traditional comparisons and logical expressions which could + * also be seen in the condition of an if-statement. + */ + fun handlePattern(node: Python.AST.BasePattern, selector: String): Expression { + return when (node) { + is Python.AST.MatchValue -> + newBinaryOperator("==", node).implicit().apply { + this.lhs = newReference(selector) + this.rhs = frontend.expressionHandler.handle(node.value) + } + is Python.AST.MatchSingleton -> + newBinaryOperator("===", node).implicit().apply { + this.lhs = newReference(selector) + this.rhs = + when (val value = node.value) { + is Python.AST.BaseExpr -> frontend.expressionHandler.handle(value) + else -> + newProblemExpression( + "Can't handle ${value::class} in value of Python.AST.MatchSingleton yet" + ) + } + } + is Python.AST.MatchOr -> + frontend.expressionHandler.joinListWithBinOp( + "or", + node.patterns.map { handlePattern(it, selector) }, + node + ) + is Python.AST.MatchSequence, + is Python.AST.MatchMapping, + is Python.AST.MatchClass, + is Python.AST.MatchStar, + is Python.AST.MatchAs -> + newProblemExpression("Cannot handle of type ${node::class} yet") + else -> newProblemExpression("Cannot handle of type ${node::class} yet") + } + } + + /** + * Translates a [`match_case`](https://docs.python.org/3/library/ast.html#ast.match_case) to a + * [Block] which holds the [CaseStatement] and then all other statements of the + * [Python.AST.match_case.body]. + * + * The [CaseStatement] is generated by the [Python.AST.match_case.pattern] and, if available, + * [Python.AST.match_case.guard]. If there's a `guard` present, we model the + * [CaseStatement.caseExpression] as an `AND` BinaryOperator, where the `lhs` is the normal + * pattern and the `rhs` is the guard. This is in line with + * [PEP 634](https://peps.python.org/pep-0634/). + */ + fun handleCase(node: Python.AST.match_case, selector: String): List { + val statements = mutableListOf() + // First, we add the caseStatement + statements += + newCaseStatement(node).apply { + this.caseExpression = + node.guard?.let { + newBinaryOperator("and") + .implicit( + code = frontend.codeOf(node), + location = frontend.locationOf(node) + ) + .apply { + this.lhs = handlePattern(node.pattern, selector) + this.rhs = frontend.expressionHandler.handle(it) + } + } ?: handlePattern(node.pattern, selector) + } + // Now, we add the remaining body. + statements += node.body.map(::handle) + return statements + } + + /** + * Translates a Python [`Match`](https://docs.python.org/3/library/ast.html#ast.Match) into a + * [SwitchStatement]. + */ + fun handleMatch(node: Python.AST.Match): Statement { + return newSwitchStatement(node).apply { + val selector = frontend.expressionHandler.handle(node.subject) + this.selector = selector + + this.statement = + node.cases.fold(newBlock().implicit()) { block, case -> + block.statements += handleCase(case, selector.name.localName) + block + } + } + } + /** * Translates a Python [`Raise`](https://docs.python.org/3/library/ast.html#ast.Raise) into a * [ThrowStatement]. From de9260dea856d47935461f0d7a99742c38441d8c Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Thu, 24 Oct 2024 10:58:58 +0200 Subject: [PATCH 02/15] fix bug, add test --- .../aisec/cpg/frontends/python/Python.kt | 2 +- .../cpg/frontends/python/StatementHandler.kt | 4 ++++ .../statementHandler/StatementHandlerTest.kt | 14 ++++++++++++ .../src/test/resources/python/match.py | 22 +++++++++++++++++++ 4 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 cpg-language-python/src/test/resources/python/match.py diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/Python.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/Python.kt index 774bd91c0b..5eda378859 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/Python.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/Python.kt @@ -1153,7 +1153,7 @@ interface Python { * ``` */ class MatchSingleton(pyObject: PyObject) : BasePattern(pyObject) { - val value: Any by lazy { "value" of pyObject } + val value: Any? by lazy { "value" of pyObject } } /** diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt index 80733f47d0..b6b9d34d94 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt @@ -104,6 +104,10 @@ class StatementHandler(frontend: PythonLanguageFrontend) : this.rhs = when (val value = node.value) { is Python.AST.BaseExpr -> frontend.expressionHandler.handle(value) + null -> + newProblemExpression( + "Can't handle value 'None'/'null' in value of Python.AST.MatchSingleton yet" + ) else -> newProblemExpression( "Can't handle ${value::class} in value of Python.AST.MatchSingleton yet" diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt index 5a396dd72c..4781b3b458 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt @@ -59,6 +59,20 @@ class StatementHandlerTest : BaseTest() { assertNotNull(result) } + @Test + fun testMatch() { + analyzeFile("match.py") + + val func = result.functions["matcher"] + assertNotNull(func) + + val switchStatement = func.switches.singleOrNull() + assertNotNull(switchStatement) + + assertLocalName("x", switchStatement.selector) + assertIs(switchStatement.selector) + } + @Test fun testTry() { val tu = diff --git a/cpg-language-python/src/test/resources/python/match.py b/cpg-language-python/src/test/resources/python/match.py new file mode 100644 index 0000000000..729866de9d --- /dev/null +++ b/cpg-language-python/src/test/resources/python/match.py @@ -0,0 +1,22 @@ +def matcher(x): + match x: + case None: + print("singleton" + x) + case "value": + print("value" + x) + case [x] if x>0: + print(x) + case [1, 2]: + print("sequence" + x) + case [1, 2, *rest]: + print("star" + x) + case [*_]: + print("star2" + x) + case {1: _, 2: _}: + print("mapping" + x) + case Point2D(0, 0): + print("class" + x) + case [x] as y: + print("as" + y) + case [x] | [y]: + print("or" + x) \ No newline at end of file From c4a343e6a674379a9450913957e62c49153fdfb9 Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Thu, 24 Oct 2024 14:23:17 +0200 Subject: [PATCH 03/15] More testing --- .../statementHandler/StatementHandlerTest.kt | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt index 4781b3b458..7212b1520f 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt @@ -71,6 +71,55 @@ class StatementHandlerTest : BaseTest() { assertLocalName("x", switchStatement.selector) assertIs(switchStatement.selector) + val paramX = func.parameters.singleOrNull() + assertNotNull(paramX) + assertRefersTo(switchStatement.selector, paramX) + + val statementBlock = switchStatement.statement as? Block + assertNotNull(statementBlock) + val caseSingleton = statementBlock[0] + assertIs(caseSingleton) + val singletonCheck = caseSingleton.caseExpression + assertIs(singletonCheck) + assertNotNull(singletonCheck) + assertEquals("===", singletonCheck.operatorCode) + assertRefersTo(singletonCheck.lhs, paramX) + assertIs(singletonCheck.rhs) + + val caseValue = statementBlock[2] + assertIs(caseValue) + val valueCheck = caseValue.caseExpression + assertIs(valueCheck) + assertNotNull(valueCheck) + assertEquals("==", valueCheck.operatorCode) + assertRefersTo(valueCheck.lhs, paramX) + assertLiteralValue("value", valueCheck.rhs) + + val caseAnd = statementBlock[4] + assertIs(caseAnd) + val andExpr = caseAnd.caseExpression + assertIs(andExpr) + assertEquals("and", andExpr.operatorCode) + val andRhs = andExpr.rhs + assertIs(andRhs) + assertEquals(">", andRhs.operatorCode) + assertRefersTo(andRhs.lhs, paramX) + assertLiteralValue(0L, andRhs.rhs) + + assertIs(statementBlock[4]) + assertIs(statementBlock[6]) + assertIs(statementBlock[8]) + assertIs(statementBlock[10]) + assertIs(statementBlock[12]) + assertIs(statementBlock[14]) + assertIs(statementBlock[16]) + + val caseOr = statementBlock[18] + assertIs(caseOr) + val orExpr = caseOr.caseExpression + assertIs(orExpr) + assertNotNull(orExpr) + assertEquals("or", orExpr.operatorCode) } @Test From 3279e722f302a67bbeb8551ababb963b7de656be Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Thu, 24 Oct 2024 15:41:27 +0200 Subject: [PATCH 04/15] Add implicit break --- .../cpg/frontends/python/StatementHandler.kt | 5 ++++ .../statementHandler/StatementHandlerTest.kt | 29 ++++++++++++------- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt index b6b9d34d94..8b6abefb42 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt @@ -161,6 +161,11 @@ class StatementHandler(frontend: PythonLanguageFrontend) : } // Now, we add the remaining body. statements += node.body.map(::handle) + // Currently, the EOG pass requires a break statement to work as expected. For this reason, + // we insert an implicit break statement at the end of the block. + statements += + newBreakStatement() + .implicit(code = frontend.codeOf(node), location = frontend.locationOf(node)) return statements } diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt index 7212b1520f..4d99ff6e6e 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt @@ -85,8 +85,9 @@ class StatementHandlerTest : BaseTest() { assertEquals("===", singletonCheck.operatorCode) assertRefersTo(singletonCheck.lhs, paramX) assertIs(singletonCheck.rhs) + assertIs(statementBlock[2]) - val caseValue = statementBlock[2] + val caseValue = statementBlock[3] assertIs(caseValue) val valueCheck = caseValue.caseExpression assertIs(valueCheck) @@ -94,8 +95,9 @@ class StatementHandlerTest : BaseTest() { assertEquals("==", valueCheck.operatorCode) assertRefersTo(valueCheck.lhs, paramX) assertLiteralValue("value", valueCheck.rhs) + assertIs(statementBlock[5]) - val caseAnd = statementBlock[4] + val caseAnd = statementBlock[6] assertIs(caseAnd) val andExpr = caseAnd.caseExpression assertIs(andExpr) @@ -105,21 +107,28 @@ class StatementHandlerTest : BaseTest() { assertEquals(">", andRhs.operatorCode) assertRefersTo(andRhs.lhs, paramX) assertLiteralValue(0L, andRhs.rhs) + assertIs(statementBlock[8]) - assertIs(statementBlock[4]) - assertIs(statementBlock[6]) - assertIs(statementBlock[8]) - assertIs(statementBlock[10]) + assertIs(statementBlock[9]) + assertIs(statementBlock[11]) assertIs(statementBlock[12]) - assertIs(statementBlock[14]) - assertIs(statementBlock[16]) - - val caseOr = statementBlock[18] + assertIs(statementBlock[14]) + assertIs(statementBlock[15]) + assertIs(statementBlock[17]) + assertIs(statementBlock[18]) + assertIs(statementBlock[20]) + assertIs(statementBlock[21]) + assertIs(statementBlock[23]) + assertIs(statementBlock[24]) + assertIs(statementBlock[26]) + + val caseOr = statementBlock[27] assertIs(caseOr) val orExpr = caseOr.caseExpression assertIs(orExpr) assertNotNull(orExpr) assertEquals("or", orExpr.operatorCode) + assertIs(statementBlock[29]) } @Test From c174d1a13f845e853b44950dc9fcd9529c479ae9 Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Wed, 30 Oct 2024 16:27:36 +0100 Subject: [PATCH 05/15] Review feedback --- .../aisec/cpg/frontends/python/ExpressionHandler.kt | 12 +++++++++++- .../aisec/cpg/frontends/python/StatementHandler.kt | 6 +++--- .../python/statementHandler/StatementHandlerTest.kt | 4 ++-- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt index bfd039336b..d1dba6aa60 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt @@ -159,6 +159,11 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : } } + /** + * Joins the [nodes] with a [BinaryOperator] with the [operatorCode]. Nests the whole thing, + * where the first element in [nodes] is the lhs of the root of the tree of binary operators. + * The last operands are further down the tree. + */ fun joinListWithBinOp( operatorCode: String, nodes: List, @@ -226,7 +231,12 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : rawNode = node ) } else { - joinListWithBinOp(op, node.values.map(::handle), node, true) + joinListWithBinOp( + operatorCode = op, + nodes = node.values.map(::handle), + rawNode = node, + isImplicit = true + ) } } diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt index 8b6abefb42..f72496eb04 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt @@ -91,7 +91,7 @@ class StatementHandler(frontend: PythonLanguageFrontend) : * and all of them are translated to traditional comparisons and logical expressions which could * also be seen in the condition of an if-statement. */ - fun handlePattern(node: Python.AST.BasePattern, selector: String): Expression { + private fun handlePattern(node: Python.AST.BasePattern, selector: String): Expression { return when (node) { is Python.AST.MatchValue -> newBinaryOperator("==", node).implicit().apply { @@ -141,7 +141,7 @@ class StatementHandler(frontend: PythonLanguageFrontend) : * pattern and the `rhs` is the guard. This is in line with * [PEP 634](https://peps.python.org/pep-0634/). */ - fun handleCase(node: Python.AST.match_case, selector: String): List { + private fun handleCase(node: Python.AST.match_case, selector: String): List { val statements = mutableListOf() // First, we add the caseStatement statements += @@ -173,7 +173,7 @@ class StatementHandler(frontend: PythonLanguageFrontend) : * Translates a Python [`Match`](https://docs.python.org/3/library/ast.html#ast.Match) into a * [SwitchStatement]. */ - fun handleMatch(node: Python.AST.Match): Statement { + private fun handleMatch(node: Python.AST.Match): Statement { return newSwitchStatement(node).apply { val selector = frontend.expressionHandler.handle(node.subject) this.selector = selector diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt index 4d99ff6e6e..538d155564 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt @@ -75,8 +75,8 @@ class StatementHandlerTest : BaseTest() { assertNotNull(paramX) assertRefersTo(switchStatement.selector, paramX) - val statementBlock = switchStatement.statement as? Block - assertNotNull(statementBlock) + val statementBlock = switchStatement.statement + assertIs(statementBlock) val caseSingleton = statementBlock[0] assertIs(caseSingleton) val singletonCheck = caseSingleton.caseExpression From 57f1caaece22aa76379b4785fdcee0217b18e94f Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Thu, 31 Oct 2024 11:05:32 +0100 Subject: [PATCH 06/15] nullable MatchSingleton: comment and handling --- .../de/fraunhofer/aisec/cpg/frontends/python/Python.kt | 5 +++++ .../aisec/cpg/frontends/python/StatementHandler.kt | 5 +---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/Python.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/Python.kt index 5eda378859..93eafe9a1e 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/Python.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/Python.kt @@ -1153,6 +1153,11 @@ interface Python { * ``` */ class MatchSingleton(pyObject: PyObject) : BasePattern(pyObject) { + /** + * [value] is not optional. We have to make it nullable though because the value will be + * set to `null` if the case matches on `None`. This is known behavior of jep (similar + * to literals/constants). + */ val value: Any? by lazy { "value" of pyObject } } diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt index f72496eb04..5620a8a2e2 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt @@ -104,10 +104,7 @@ class StatementHandler(frontend: PythonLanguageFrontend) : this.rhs = when (val value = node.value) { is Python.AST.BaseExpr -> frontend.expressionHandler.handle(value) - null -> - newProblemExpression( - "Can't handle value 'None'/'null' in value of Python.AST.MatchSingleton yet" - ) + null -> newLiteral(value = null, rawNode = node) else -> newProblemExpression( "Can't handle ${value::class} in value of Python.AST.MatchSingleton yet" From 1445fc46869cae56179e720d099ede8adcd8aec5 Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Thu, 31 Oct 2024 11:41:47 +0100 Subject: [PATCH 07/15] Adapt test --- .../frontends/python/statementHandler/StatementHandlerTest.kt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt index 538d155564..f6131eee18 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt @@ -84,7 +84,9 @@ class StatementHandlerTest : BaseTest() { assertNotNull(singletonCheck) assertEquals("===", singletonCheck.operatorCode) assertRefersTo(singletonCheck.lhs, paramX) - assertIs(singletonCheck.rhs) + val singletonRhs = singletonCheck.rhs + assertIs>(singletonRhs) + assertNull(singletonRhs.value) assertIs(statementBlock[2]) val caseValue = statementBlock[3] From c0a8165042e565bdfbc47902984103f1e8c29392 Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Fri, 1 Nov 2024 08:41:14 +0100 Subject: [PATCH 08/15] Review --- .../cpg/frontends/python/ExpressionHandler.kt | 32 ++----------- .../cpg/frontends/python/StatementHandler.kt | 45 ++++++++++++------- .../statementHandler/StatementHandlerTest.kt | 7 +-- .../src/test/resources/python/match.py | 6 ++- 4 files changed, 39 insertions(+), 51 deletions(-) diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt index 2d2b0afc61..f765abe2cb 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt @@ -237,28 +237,7 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : * where the first element in [nodes] is the lhs of the root of the tree of binary operators. * The last operands are further down the tree. */ - private fun joinListWithBinOp( - operatorCode: String, - nodes: List, - rawNode: Python.AST.AST? = null - ): BinaryOperator { - val lastTwo = newBinaryOperator(operatorCode, rawNode = rawNode) - lastTwo.rhs = nodes.last() - lastTwo.lhs = nodes[nodes.size - 2] - return nodes.subList(0, nodes.size - 2).foldRight(lastTwo) { newVal, start -> - val nextValue = newBinaryOperator(operatorCode) - nextValue.rhs = start - nextValue.lhs = newVal - nextValue - } - } - - /** - * Joins the [nodes] with a [BinaryOperator] with the [operatorCode]. Nests the whole thing, - * where the first element in [nodes] is the lhs of the root of the tree of binary operators. - * The last operands are further down the tree. - */ - fun joinListWithBinOp( + internal fun joinListWithBinOp( operatorCode: String, nodes: List, rawNode: Python.AST.AST? = null, @@ -268,13 +247,8 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : lastTwo.rhs = nodes.last() lastTwo.lhs = nodes[nodes.size - 2] return nodes.subList(0, nodes.size - 2).foldRight(lastTwo) { newVal, start -> - val nextValue = newBinaryOperator(operatorCode) - if (isImplicit && rawNode != null) - nextValue.implicit( - code = frontend.codeOf(rawNode), - location = frontend.locationOf(rawNode) - ) - else if (isImplicit) nextValue.implicit() + val nextValue = newBinaryOperator(operatorCode, rawNode = rawNode) + if (isImplicit) nextValue.implicit() nextValue.rhs = start nextValue.lhs = newVal nextValue diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt index 5620a8a2e2..653d4ea9bb 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt @@ -113,9 +113,10 @@ class StatementHandler(frontend: PythonLanguageFrontend) : } is Python.AST.MatchOr -> frontend.expressionHandler.joinListWithBinOp( - "or", - node.patterns.map { handlePattern(it, selector) }, - node + operatorCode = "or", + nodes = node.patterns.map { handlePattern(it, selector) }, + rawNode = node, + isImplicit = false ) is Python.AST.MatchSequence, is Python.AST.MatchMapping, @@ -140,21 +141,31 @@ class StatementHandler(frontend: PythonLanguageFrontend) : */ private fun handleCase(node: Python.AST.match_case, selector: String): List { val statements = mutableListOf() - // First, we add the caseStatement + // First, we add the CaseStatement. If the pattern is a `MatchAs` without a pattern, then + // it's a default statement. + // We have to handle this here since we do not want to generate the CaseStatement in this + // case. statements += - newCaseStatement(node).apply { - this.caseExpression = - node.guard?.let { - newBinaryOperator("and") - .implicit( - code = frontend.codeOf(node), - location = frontend.locationOf(node) - ) - .apply { - this.lhs = handlePattern(node.pattern, selector) - this.rhs = frontend.expressionHandler.handle(it) - } - } ?: handlePattern(node.pattern, selector) + if ( + node.pattern is Python.AST.MatchAs && + (node.pattern as Python.AST.MatchAs).pattern == null + ) { + newDefaultStatement(rawNode = node.pattern) + } else { + newCaseStatement(node).apply { + this.caseExpression = + node.guard?.let { + newBinaryOperator("and") + .implicit( + code = frontend.codeOf(node), + location = frontend.locationOf(node) + ) + .apply { + this.lhs = handlePattern(node.pattern, selector) + this.rhs = frontend.expressionHandler.handle(it) + } + } ?: handlePattern(node.pattern, selector) + } } // Now, we add the remaining body. statements += node.body.map(::handle) diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt index f6131eee18..13eee247a5 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt @@ -81,7 +81,6 @@ class StatementHandlerTest : BaseTest() { assertIs(caseSingleton) val singletonCheck = caseSingleton.caseExpression assertIs(singletonCheck) - assertNotNull(singletonCheck) assertEquals("===", singletonCheck.operatorCode) assertRefersTo(singletonCheck.lhs, paramX) val singletonRhs = singletonCheck.rhs @@ -93,7 +92,6 @@ class StatementHandlerTest : BaseTest() { assertIs(caseValue) val valueCheck = caseValue.caseExpression assertIs(valueCheck) - assertNotNull(valueCheck) assertEquals("==", valueCheck.operatorCode) assertRefersTo(valueCheck.lhs, paramX) assertLiteralValue("value", valueCheck.rhs) @@ -128,9 +126,12 @@ class StatementHandlerTest : BaseTest() { assertIs(caseOr) val orExpr = caseOr.caseExpression assertIs(orExpr) - assertNotNull(orExpr) assertEquals("or", orExpr.operatorCode) assertIs(statementBlock[29]) + + val caseDefault = statementBlock[30] + assertIs(caseDefault) + assertIs(statementBlock[32]) } @Test diff --git a/cpg-language-python/src/test/resources/python/match.py b/cpg-language-python/src/test/resources/python/match.py index 729866de9d..c6acf87223 100644 --- a/cpg-language-python/src/test/resources/python/match.py +++ b/cpg-language-python/src/test/resources/python/match.py @@ -18,5 +18,7 @@ def matcher(x): print("class" + x) case [x] as y: print("as" + y) - case [x] | [y]: - print("or" + x) \ No newline at end of file + case [x] | (y): + print("or" + x) + case _: + print("Default match") \ No newline at end of file From 11c27d3d69b32bdc04f6acd9ab180852847820a8 Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Tue, 5 Nov 2024 19:42:05 +0100 Subject: [PATCH 09/15] Add another test which we do not handle yet --- .../cpg/frontends/python/ExpressionHandler.kt | 19 +++--- .../cpg/frontends/python/StatementHandler.kt | 67 +++++++++++-------- .../statementHandler/StatementHandlerTest.kt | 19 ++++++ .../src/test/resources/python/match.py | 7 +- 4 files changed, 75 insertions(+), 37 deletions(-) diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt index f765abe2cb..cfe11c5a3b 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt @@ -243,15 +243,18 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : rawNode: Python.AST.AST? = null, isImplicit: Boolean = true ): BinaryOperator { - val lastTwo = newBinaryOperator(operatorCode, rawNode = rawNode) - lastTwo.rhs = nodes.last() - lastTwo.lhs = nodes[nodes.size - 2] + val lastTwo = + newBinaryOperator(operatorCode = operatorCode, rawNode = rawNode).apply { + rhs = nodes.last() + lhs = nodes[nodes.size - 2] + this.isImplicit = isImplicit + } return nodes.subList(0, nodes.size - 2).foldRight(lastTwo) { newVal, start -> - val nextValue = newBinaryOperator(operatorCode, rawNode = rawNode) - if (isImplicit) nextValue.implicit() - nextValue.rhs = start - nextValue.lhs = newVal - nextValue + newBinaryOperator(operatorCode = operatorCode, rawNode = rawNode).apply { + rhs = start + lhs = newVal + this.isImplicit = isImplicit + } } } diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt index 653d4ea9bb..a9b5813c3e 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt @@ -80,7 +80,7 @@ class StatementHandler(frontend: PythonLanguageFrontend) : is Python.AST.Match -> handleMatch(node) is Python.AST.TryStar -> newProblemExpression( - "The statement of class ${node.javaClass} is not supported yet", + problem = "The statement of class ${node.javaClass} is not supported yet", rawNode = node ) } @@ -91,30 +91,31 @@ class StatementHandler(frontend: PythonLanguageFrontend) : * and all of them are translated to traditional comparisons and logical expressions which could * also be seen in the condition of an if-statement. */ - private fun handlePattern(node: Python.AST.BasePattern, selector: String): Expression { + private fun handlePattern(node: Python.AST.BasePattern, subject: String): Expression { return when (node) { is Python.AST.MatchValue -> - newBinaryOperator("==", node).implicit().apply { - this.lhs = newReference(selector) - this.rhs = frontend.expressionHandler.handle(node.value) + newBinaryOperator(operatorCode = "==", rawNode = node).implicit().apply { + this.lhs = newReference(name = subject) + this.rhs = frontend.expressionHandler.handle(ctx = node.value) } is Python.AST.MatchSingleton -> - newBinaryOperator("===", node).implicit().apply { - this.lhs = newReference(selector) + newBinaryOperator(operatorCode = "===", rawNode = node).implicit().apply { + this.lhs = newReference(name = subject) this.rhs = when (val value = node.value) { - is Python.AST.BaseExpr -> frontend.expressionHandler.handle(value) + is Python.AST.BaseExpr -> frontend.expressionHandler.handle(ctx = value) null -> newLiteral(value = null, rawNode = node) else -> newProblemExpression( - "Can't handle ${value::class} in value of Python.AST.MatchSingleton yet" + problem = + "Can't handle ${value::class} in value of Python.AST.MatchSingleton yet" ) } } is Python.AST.MatchOr -> frontend.expressionHandler.joinListWithBinOp( operatorCode = "or", - nodes = node.patterns.map { handlePattern(it, selector) }, + nodes = node.patterns.map { handlePattern(node = it, subject = subject) }, rawNode = node, isImplicit = false ) @@ -123,8 +124,15 @@ class StatementHandler(frontend: PythonLanguageFrontend) : is Python.AST.MatchClass, is Python.AST.MatchStar, is Python.AST.MatchAs -> - newProblemExpression("Cannot handle of type ${node::class} yet") - else -> newProblemExpression("Cannot handle of type ${node::class} yet") + newProblemExpression( + problem = "Cannot handle of type ${node::class} yet", + rawNode = node + ) + else -> + newProblemExpression( + problem = "Cannot handle of type ${node::class} yet", + rawNode = node + ) } } @@ -139,7 +147,7 @@ class StatementHandler(frontend: PythonLanguageFrontend) : * pattern and the `rhs` is the guard. This is in line with * [PEP 634](https://peps.python.org/pep-0634/). */ - private fun handleCase(node: Python.AST.match_case, selector: String): List { + private fun handleMatchCase(node: Python.AST.match_case, subject: String): List { val statements = mutableListOf() // First, we add the CaseStatement. If the pattern is a `MatchAs` without a pattern, then // it's a default statement. @@ -152,19 +160,19 @@ class StatementHandler(frontend: PythonLanguageFrontend) : ) { newDefaultStatement(rawNode = node.pattern) } else { - newCaseStatement(node).apply { + newCaseStatement(rawNode = node).apply { this.caseExpression = node.guard?.let { - newBinaryOperator("and") + newBinaryOperator(operatorCode = "and") .implicit( - code = frontend.codeOf(node), - location = frontend.locationOf(node) + code = frontend.codeOf(astNode = node), + location = frontend.locationOf(astNode = node) ) .apply { - this.lhs = handlePattern(node.pattern, selector) - this.rhs = frontend.expressionHandler.handle(it) + this.lhs = handlePattern(node = node.pattern, subject = subject) + this.rhs = frontend.expressionHandler.handle(ctx = it) } - } ?: handlePattern(node.pattern, selector) + } ?: handlePattern(node = node.pattern, subject = subject) } } // Now, we add the remaining body. @@ -173,7 +181,10 @@ class StatementHandler(frontend: PythonLanguageFrontend) : // we insert an implicit break statement at the end of the block. statements += newBreakStatement() - .implicit(code = frontend.codeOf(node), location = frontend.locationOf(node)) + .implicit( + code = frontend.codeOf(astNode = node), + location = frontend.locationOf(astNode = node) + ) return statements } @@ -181,18 +192,18 @@ class StatementHandler(frontend: PythonLanguageFrontend) : * Translates a Python [`Match`](https://docs.python.org/3/library/ast.html#ast.Match) into a * [SwitchStatement]. */ - private fun handleMatch(node: Python.AST.Match): Statement { - return newSwitchStatement(node).apply { - val selector = frontend.expressionHandler.handle(node.subject) - this.selector = selector + private fun handleMatch(node: Python.AST.Match) = + newSwitchStatement(rawNode = node).apply { + val subject = frontend.expressionHandler.handle(ctx = node.subject) + this.selector = subject this.statement = - node.cases.fold(newBlock().implicit()) { block, case -> - block.statements += handleCase(case, selector.name.localName) + node.cases.fold(initial = newBlock().implicit()) { block, case -> + block.statements += + handleMatchCase(node = case, subject = subject.name.localName) block } } - } /** * Translates a Python [`Raise`](https://docs.python.org/3/library/ast.html#ast.Raise) into a diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt index 13eee247a5..c1029af396 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt @@ -59,6 +59,25 @@ class StatementHandlerTest : BaseTest() { assertNotNull(result) } + @Test + fun testMatch2() { + analyzeFile("match.py") + + val func = result.functions["match_weird"] + assertNotNull(func) + + val switchStatement = func.switches.singleOrNull() + assertNotNull(switchStatement) + + assertIs(switchStatement.selector) + + val statementBlock = switchStatement.statement + assertIs(statementBlock) + val case = statementBlock[0] + assertIs(case) + assertIs(case.caseExpression) + } + @Test fun testMatch() { analyzeFile("match.py") diff --git a/cpg-language-python/src/test/resources/python/match.py b/cpg-language-python/src/test/resources/python/match.py index c6acf87223..8336639061 100644 --- a/cpg-language-python/src/test/resources/python/match.py +++ b/cpg-language-python/src/test/resources/python/match.py @@ -21,4 +21,9 @@ def matcher(x): case [x] | (y): print("or" + x) case _: - print("Default match") \ No newline at end of file + print("Default match") + +def match_weird(): + match command.split(): + case ["go", ("north" | "south" | "east" | "west") as direction]: + current_room = current_room.neighbor(direction) \ No newline at end of file From 8ac550654259e39112d66bc1a789ace345b7476d Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Tue, 5 Nov 2024 19:46:08 +0100 Subject: [PATCH 10/15] Update test --- .../frontends/python/statementHandler/StatementHandlerTest.kt | 2 ++ cpg-language-python/src/test/resources/python/match.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt index c1029af396..73dbc884b0 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt @@ -146,6 +146,8 @@ class StatementHandlerTest : BaseTest() { val orExpr = caseOr.caseExpression assertIs(orExpr) assertEquals("or", orExpr.operatorCode) + assertIs(orExpr.lhs) + assertIs(orExpr.rhs) assertIs(statementBlock[29]) val caseDefault = statementBlock[30] diff --git a/cpg-language-python/src/test/resources/python/match.py b/cpg-language-python/src/test/resources/python/match.py index 8336639061..234a58391e 100644 --- a/cpg-language-python/src/test/resources/python/match.py +++ b/cpg-language-python/src/test/resources/python/match.py @@ -18,7 +18,7 @@ def matcher(x): print("class" + x) case [x] as y: print("as" + y) - case [x] | (y): + case "xyz" | "abc": print("or" + x) case _: print("Default match") From a55e26b9c6077341448aaebb26951285e0b28eae Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Wed, 6 Nov 2024 18:21:47 +0100 Subject: [PATCH 11/15] Extract match test to own file and add simplified versions --- .../python/statementHandler/MatchTest.kt | 295 ++++++++++++++++++ .../statementHandler/StatementHandlerTest.kt | 96 ------ .../src/test/resources/python/match.py | 25 ++ 3 files changed, 320 insertions(+), 96 deletions(-) create mode 100644 cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/MatchTest.kt diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/MatchTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/MatchTest.kt new file mode 100644 index 0000000000..4db7662e54 --- /dev/null +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/MatchTest.kt @@ -0,0 +1,295 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.frontends.python.statementHandler + +import de.fraunhofer.aisec.cpg.TranslationResult +import de.fraunhofer.aisec.cpg.frontends.python.PythonLanguage +import de.fraunhofer.aisec.cpg.graph.functions +import de.fraunhofer.aisec.cpg.graph.get +import de.fraunhofer.aisec.cpg.graph.statements.BreakStatement +import de.fraunhofer.aisec.cpg.graph.statements.CaseStatement +import de.fraunhofer.aisec.cpg.graph.statements.DefaultStatement +import de.fraunhofer.aisec.cpg.graph.statements.expressions.BinaryOperator +import de.fraunhofer.aisec.cpg.graph.statements.expressions.Block +import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression +import de.fraunhofer.aisec.cpg.graph.statements.expressions.Literal +import de.fraunhofer.aisec.cpg.graph.statements.expressions.ProblemExpression +import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference +import de.fraunhofer.aisec.cpg.graph.switches +import de.fraunhofer.aisec.cpg.test.analyze +import de.fraunhofer.aisec.cpg.test.assertLiteralValue +import de.fraunhofer.aisec.cpg.test.assertLocalName +import de.fraunhofer.aisec.cpg.test.assertRefersTo +import java.nio.file.Path +import kotlin.test.assertEquals +import kotlin.test.assertIs +import kotlin.test.assertNotNull +import kotlin.test.assertNull +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInstance + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class MatchTest { + private lateinit var topLevel: Path + private lateinit var result: TranslationResult + + @BeforeAll + fun setup() { + topLevel = Path.of("src", "test", "resources", "python") + result = + analyze(listOf(topLevel.resolve("match.py").toFile()), topLevel, true) { + it.registerLanguage() + } + assertNotNull(result) + } + + @Test + fun testMatchSingleton() { + val func = result.functions["matchSingleton"] + assertNotNull(func) + + val switchStatement = func.switches.singleOrNull() + assertNotNull(switchStatement) + + assertLocalName("x", switchStatement.selector) + assertIs(switchStatement.selector) + val paramX = func.parameters.singleOrNull() + assertNotNull(paramX) + assertRefersTo(switchStatement.selector, paramX) + + val statementBlock = switchStatement.statement + assertIs(statementBlock) + assertEquals(3, statementBlock.statements.size) + val caseSingleton = statementBlock[0] + assertIs(caseSingleton) + val singletonCheck = caseSingleton.caseExpression + assertIs(singletonCheck) + assertEquals("===", singletonCheck.operatorCode) + assertRefersTo(singletonCheck.lhs, paramX) + val singletonRhs = singletonCheck.rhs + assertIs>(singletonRhs) + assertNull(singletonRhs.value) + assertIs(statementBlock[2]) + } + + @Test + fun testMatchValue() { + val func = result.functions["matchValue"] + assertNotNull(func) + + val switchStatement = func.switches.singleOrNull() + assertNotNull(switchStatement) + + assertLocalName("x", switchStatement.selector) + assertIs(switchStatement.selector) + val paramX = func.parameters.singleOrNull() + assertNotNull(paramX) + assertRefersTo(switchStatement.selector, paramX) + + val statementBlock = switchStatement.statement + assertIs(statementBlock) + assertEquals(3, statementBlock.statements.size) + val caseValue = statementBlock[0] + assertIs(caseValue) + val valueCheck = caseValue.caseExpression + assertIs(valueCheck) + assertEquals("==", valueCheck.operatorCode) + assertRefersTo(valueCheck.lhs, paramX) + assertLiteralValue("value", valueCheck.rhs) + assertIs(statementBlock[2]) + } + + @Test + fun testMatchOr() { + val func = result.functions["matchOr"] + assertNotNull(func) + + val switchStatement = func.switches.singleOrNull() + assertNotNull(switchStatement) + + assertLocalName("x", switchStatement.selector) + assertIs(switchStatement.selector) + val paramX = func.parameters.singleOrNull() + assertNotNull(paramX) + assertRefersTo(switchStatement.selector, paramX) + + val statementBlock = switchStatement.statement + assertIs(statementBlock) + assertEquals(3, statementBlock.statements.size) + val caseOr = statementBlock[0] + assertIs(caseOr) + val orExpr = caseOr.caseExpression + assertIs(orExpr) + assertEquals("or", orExpr.operatorCode) + assertIs(orExpr.lhs) + assertIs(orExpr.rhs) + assertIs(statementBlock[2]) + } + + @Test + fun testMatchDefault() { + val func = result.functions["matchDefault"] + assertNotNull(func) + + val switchStatement = func.switches.singleOrNull() + assertNotNull(switchStatement) + + assertLocalName("x", switchStatement.selector) + assertIs(switchStatement.selector) + val paramX = func.parameters.singleOrNull() + assertNotNull(paramX) + assertRefersTo(switchStatement.selector, paramX) + + val statementBlock = switchStatement.statement + assertIs(statementBlock) + assertEquals(3, statementBlock.statements.size) + val caseDefault = statementBlock[0] + assertIs(caseDefault) + assertIs(statementBlock[2]) + } + + @Test + fun testMatchAnd() { + val func = result.functions["matchAnd"] + assertNotNull(func) + + val switchStatement = func.switches.singleOrNull() + assertNotNull(switchStatement) + + assertLocalName("x", switchStatement.selector) + assertIs(switchStatement.selector) + val paramX = func.parameters.singleOrNull() + assertNotNull(paramX) + assertRefersTo(switchStatement.selector, paramX) + + val statementBlock = switchStatement.statement + assertIs(statementBlock) + val caseAnd = statementBlock[0] + assertIs(caseAnd) + val andExpr = caseAnd.caseExpression + assertIs(andExpr) + assertEquals("and", andExpr.operatorCode) + val andRhs = andExpr.rhs + assertIs(andRhs) + assertEquals(">", andRhs.operatorCode) + assertRefersTo(andRhs.lhs, paramX) + assertLiteralValue(0L, andRhs.rhs) + assertIs(statementBlock[2]) + } + + @Test + fun testMatchCombined() { + val func = result.functions["matcher"] + assertNotNull(func) + + val switchStatement = func.switches.singleOrNull() + assertNotNull(switchStatement) + + assertLocalName("x", switchStatement.selector) + assertIs(switchStatement.selector) + val paramX = func.parameters.singleOrNull() + assertNotNull(paramX) + assertRefersTo(switchStatement.selector, paramX) + + val statementBlock = switchStatement.statement + assertIs(statementBlock) + val caseSingleton = statementBlock[0] + assertIs(caseSingleton) + val singletonCheck = caseSingleton.caseExpression + assertIs(singletonCheck) + assertEquals("===", singletonCheck.operatorCode) + assertRefersTo(singletonCheck.lhs, paramX) + val singletonRhs = singletonCheck.rhs + assertIs>(singletonRhs) + assertNull(singletonRhs.value) + assertIs(statementBlock[2]) + + val caseValue = statementBlock[3] + assertIs(caseValue) + val valueCheck = caseValue.caseExpression + assertIs(valueCheck) + assertEquals("==", valueCheck.operatorCode) + assertRefersTo(valueCheck.lhs, paramX) + assertLiteralValue("value", valueCheck.rhs) + assertIs(statementBlock[5]) + + val caseAnd = statementBlock[6] + assertIs(caseAnd) + val andExpr = caseAnd.caseExpression + assertIs(andExpr) + assertEquals("and", andExpr.operatorCode) + val andRhs = andExpr.rhs + assertIs(andRhs) + assertEquals(">", andRhs.operatorCode) + assertRefersTo(andRhs.lhs, paramX) + assertLiteralValue(0L, andRhs.rhs) + assertIs(statementBlock[8]) + + assertIs(statementBlock[9]) + assertIs(statementBlock[11]) + assertIs(statementBlock[12]) + assertIs(statementBlock[14]) + assertIs(statementBlock[15]) + assertIs(statementBlock[17]) + assertIs(statementBlock[18]) + assertIs(statementBlock[20]) + assertIs(statementBlock[21]) + assertIs(statementBlock[23]) + assertIs(statementBlock[24]) + assertIs(statementBlock[26]) + + val caseOr = statementBlock[27] + assertIs(caseOr) + val orExpr = caseOr.caseExpression + assertIs(orExpr) + assertEquals("or", orExpr.operatorCode) + assertIs(orExpr.lhs) + assertIs(orExpr.rhs) + assertIs(statementBlock[29]) + + val caseDefault = statementBlock[30] + assertIs(caseDefault) + assertIs(statementBlock[32]) + } + + @Test + fun testMatch2() { + val func = result.functions["match_weird"] + assertNotNull(func) + + val switchStatement = func.switches.singleOrNull() + assertNotNull(switchStatement) + + assertIs(switchStatement.selector) + + val statementBlock = switchStatement.statement + assertIs(statementBlock) + val case = statementBlock[0] + assertIs(case) + assertIs(case.caseExpression) + } +} diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt index 73dbc884b0..5a396dd72c 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt @@ -59,102 +59,6 @@ class StatementHandlerTest : BaseTest() { assertNotNull(result) } - @Test - fun testMatch2() { - analyzeFile("match.py") - - val func = result.functions["match_weird"] - assertNotNull(func) - - val switchStatement = func.switches.singleOrNull() - assertNotNull(switchStatement) - - assertIs(switchStatement.selector) - - val statementBlock = switchStatement.statement - assertIs(statementBlock) - val case = statementBlock[0] - assertIs(case) - assertIs(case.caseExpression) - } - - @Test - fun testMatch() { - analyzeFile("match.py") - - val func = result.functions["matcher"] - assertNotNull(func) - - val switchStatement = func.switches.singleOrNull() - assertNotNull(switchStatement) - - assertLocalName("x", switchStatement.selector) - assertIs(switchStatement.selector) - val paramX = func.parameters.singleOrNull() - assertNotNull(paramX) - assertRefersTo(switchStatement.selector, paramX) - - val statementBlock = switchStatement.statement - assertIs(statementBlock) - val caseSingleton = statementBlock[0] - assertIs(caseSingleton) - val singletonCheck = caseSingleton.caseExpression - assertIs(singletonCheck) - assertEquals("===", singletonCheck.operatorCode) - assertRefersTo(singletonCheck.lhs, paramX) - val singletonRhs = singletonCheck.rhs - assertIs>(singletonRhs) - assertNull(singletonRhs.value) - assertIs(statementBlock[2]) - - val caseValue = statementBlock[3] - assertIs(caseValue) - val valueCheck = caseValue.caseExpression - assertIs(valueCheck) - assertEquals("==", valueCheck.operatorCode) - assertRefersTo(valueCheck.lhs, paramX) - assertLiteralValue("value", valueCheck.rhs) - assertIs(statementBlock[5]) - - val caseAnd = statementBlock[6] - assertIs(caseAnd) - val andExpr = caseAnd.caseExpression - assertIs(andExpr) - assertEquals("and", andExpr.operatorCode) - val andRhs = andExpr.rhs - assertIs(andRhs) - assertEquals(">", andRhs.operatorCode) - assertRefersTo(andRhs.lhs, paramX) - assertLiteralValue(0L, andRhs.rhs) - assertIs(statementBlock[8]) - - assertIs(statementBlock[9]) - assertIs(statementBlock[11]) - assertIs(statementBlock[12]) - assertIs(statementBlock[14]) - assertIs(statementBlock[15]) - assertIs(statementBlock[17]) - assertIs(statementBlock[18]) - assertIs(statementBlock[20]) - assertIs(statementBlock[21]) - assertIs(statementBlock[23]) - assertIs(statementBlock[24]) - assertIs(statementBlock[26]) - - val caseOr = statementBlock[27] - assertIs(caseOr) - val orExpr = caseOr.caseExpression - assertIs(orExpr) - assertEquals("or", orExpr.operatorCode) - assertIs(orExpr.lhs) - assertIs(orExpr.rhs) - assertIs(statementBlock[29]) - - val caseDefault = statementBlock[30] - assertIs(caseDefault) - assertIs(statementBlock[32]) - } - @Test fun testTry() { val tu = diff --git a/cpg-language-python/src/test/resources/python/match.py b/cpg-language-python/src/test/resources/python/match.py index 234a58391e..21e7884ba1 100644 --- a/cpg-language-python/src/test/resources/python/match.py +++ b/cpg-language-python/src/test/resources/python/match.py @@ -23,6 +23,31 @@ def matcher(x): case _: print("Default match") +def matchSingleton(x): + match x: + case None: + print("singleton" + x) + +def matchValue(x): + match x: + case "value": + print("value" + x) + +def matchOr(x): + match x: + case "xyz" | "abc": + print("or" + x) + +def matchAnd(x): + match x: + case [x] if x>0: + print(x) + +def matchDefault(x): + match x: + case _: + print("Default match") + def match_weird(): match command.split(): case ["go", ("north" | "south" | "east" | "west") as direction]: From 432dfcdf43afc68a305e2a6adceb1b52f56bcedd Mon Sep 17 00:00:00 2001 From: KuechA <31155350+KuechA@users.noreply.github.com> Date: Thu, 14 Nov 2024 10:26:32 +0100 Subject: [PATCH 12/15] Update cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt Co-authored-by: Maximilian Kaul --- .../aisec/cpg/frontends/python/StatementHandler.kt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt index 33bd0ff032..6967cfc071 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt @@ -153,12 +153,13 @@ class StatementHandler(frontend: PythonLanguageFrontend) : // it's a default statement. // We have to handle this here since we do not want to generate the CaseStatement in this // case. + val pattern = node.pattern statements += if ( - node.pattern is Python.AST.MatchAs && - (node.pattern as Python.AST.MatchAs).pattern == null + pattern is Python.AST.MatchAs && + pattern.pattern == null ) { - newDefaultStatement(rawNode = node.pattern) + newDefaultStatement(rawNode = pattern) } else { newCaseStatement(rawNode = node).apply { this.caseExpression = From 06348e19d7c36e432f56aa003e0a33f5f25aa575 Mon Sep 17 00:00:00 2001 From: KuechA <31155350+KuechA@users.noreply.github.com> Date: Thu, 14 Nov 2024 13:34:54 +0100 Subject: [PATCH 13/15] Apply suggestions from code review Co-authored-by: Maximilian Kaul --- .../fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt index 6967cfc071..96f30aab4b 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt @@ -149,7 +149,7 @@ class StatementHandler(frontend: PythonLanguageFrontend) : */ private fun handleMatchCase(node: Python.AST.match_case, subject: String): List { val statements = mutableListOf() - // First, we add the CaseStatement. If the pattern is a `MatchAs` without a pattern, then + // First, we add the CaseStatement. A `MatchAs` without a `pattern` implies // it's a default statement. // We have to handle this here since we do not want to generate the CaseStatement in this // case. @@ -193,7 +193,7 @@ class StatementHandler(frontend: PythonLanguageFrontend) : * Translates a Python [`Match`](https://docs.python.org/3/library/ast.html#ast.Match) into a * [SwitchStatement]. */ - private fun handleMatch(node: Python.AST.Match) = + private fun handleMatch(node: Python.AST.Match): SwitchStatement = newSwitchStatement(rawNode = node).apply { val subject = frontend.expressionHandler.handle(ctx = node.subject) this.selector = subject From 39cd2cefc945d8edd41f05d49d89300bf97163ec Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Mon, 18 Nov 2024 16:14:02 +0100 Subject: [PATCH 14/15] formatting --- .../aisec/cpg/frontends/python/StatementHandler.kt | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt index 96f30aab4b..9161e97b99 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt @@ -155,10 +155,7 @@ class StatementHandler(frontend: PythonLanguageFrontend) : // case. val pattern = node.pattern statements += - if ( - pattern is Python.AST.MatchAs && - pattern.pattern == null - ) { + if (pattern is Python.AST.MatchAs && pattern.pattern == null) { newDefaultStatement(rawNode = pattern) } else { newCaseStatement(rawNode = node).apply { From f6721713b637902261c0bf3bcd3f85765ed95bc2 Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Mon, 18 Nov 2024 16:22:34 +0100 Subject: [PATCH 15/15] Some more refactoring --- .../cpg/frontends/python/StatementHandler.kt | 34 ++++++++++--------- .../python/statementHandler/MatchTest.kt | 2 +- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt index 9161e97b99..dfb16d943b 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt @@ -142,10 +142,9 @@ class StatementHandler(frontend: PythonLanguageFrontend) : * [Python.AST.match_case.body]. * * The [CaseStatement] is generated by the [Python.AST.match_case.pattern] and, if available, - * [Python.AST.match_case.guard]. If there's a `guard` present, we model the - * [CaseStatement.caseExpression] as an `AND` BinaryOperator, where the `lhs` is the normal - * pattern and the `rhs` is the guard. This is in line with - * [PEP 634](https://peps.python.org/pep-0634/). + * [Python.AST.match_case.guard]. A `guard` is modeled with an `AND` BinaryOperator in the + * [CaseStatement.caseExpression]. Its `lhs` is the normal pattern and the `rhs` is the guard. + * This is in line with [PEP 634](https://peps.python.org/pep-0634/). */ private fun handleMatchCase(node: Python.AST.match_case, subject: String): List { val statements = mutableListOf() @@ -154,23 +153,26 @@ class StatementHandler(frontend: PythonLanguageFrontend) : // We have to handle this here since we do not want to generate the CaseStatement in this // case. val pattern = node.pattern + val guard = node.guard statements += if (pattern is Python.AST.MatchAs && pattern.pattern == null) { newDefaultStatement(rawNode = pattern) - } else { + } else if (guard != null) { newCaseStatement(rawNode = node).apply { this.caseExpression = - node.guard?.let { - newBinaryOperator(operatorCode = "and") - .implicit( - code = frontend.codeOf(astNode = node), - location = frontend.locationOf(astNode = node) - ) - .apply { - this.lhs = handlePattern(node = node.pattern, subject = subject) - this.rhs = frontend.expressionHandler.handle(ctx = it) - } - } ?: handlePattern(node = node.pattern, subject = subject) + newBinaryOperator(operatorCode = "and") + .implicit( + code = frontend.codeOf(astNode = node), + location = frontend.locationOf(astNode = node) + ) + .apply { + this.lhs = handlePattern(node = node.pattern, subject = subject) + this.rhs = frontend.expressionHandler.handle(ctx = guard) + } + } + } else { + newCaseStatement(rawNode = node).apply { + this.caseExpression = handlePattern(node = node.pattern, subject = subject) } } // Now, we add the remaining body. diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/MatchTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/MatchTest.kt index 4db7662e54..aca1ef4b65 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/MatchTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/MatchTest.kt @@ -173,7 +173,7 @@ class MatchTest { } @Test - fun testMatchAnd() { + fun testMatchGuard() { val func = result.functions["matchAnd"] assertNotNull(func)