From 537efe740f56fbe482752b06b6dad26757c78fa1 Mon Sep 17 00:00:00 2001 From: Leutrim Shala Date: Wed, 20 Nov 2024 14:53:14 +0100 Subject: [PATCH 1/9] WIP --- .../cpg/frontends/python/ExpressionHandler.kt | 8 +++---- .../frontends/python/PythonFrontendTest.kt | 21 +++++++++++++++++++ .../src/test/resources/python/datatypes.py | 3 ++- 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt index ac844775a2..4673169481 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt @@ -183,11 +183,9 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : } private fun handleFormattedValue(node: Python.AST.FormattedValue): Expression { - if (node.format_spec != null) { - return newProblemExpression( - "Cannot handle formatted value with format_spec ${node.format_spec} yet", - rawNode = node - ) + val formatSpec = node.format_spec?.let { handle(it) } + if (formatSpec is Literal<*>) { + return formatSpec } return when (node.conversion) { formattedValConversionNoFormatting -> { diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt index 4f8f908227..04980d895e 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt @@ -1589,6 +1589,27 @@ class PythonFrontendTest : BaseTest() { } } + @Test + fun testTest() { + val directoryPath = Path.of("/home", "lshala", "repos", "nova") + val tr = + analyze(fileExtension = ".py", directoryPath, usePasses = true) { + it.registerLanguage() + } + + val problemsList = tr.components.flatMap { it.translationUnits }.flatMap { it.problems } + + val msg = + (problemsList) + .groupBy { it.problem } + .toList() + .sortedBy { it.second.size } + .reversed() + .map { "${it.second.size}: ${it.first}" } + + msg.forEach(System.out::println) + } + class PythonValueEvaluator : ValueEvaluator() { override fun computeBinaryOpEffect( lhsValue: Any?, diff --git a/cpg-language-python/src/test/resources/python/datatypes.py b/cpg-language-python/src/test/resources/python/datatypes.py index 6e35b7b7d2..2bed43ac9e 100644 --- a/cpg-language-python/src/test/resources/python/datatypes.py +++ b/cpg-language-python/src/test/resources/python/datatypes.py @@ -6,5 +6,6 @@ "c": "d", "e": "f" } +aa = f"sin({a}) is {sin(a):.3}" e = f'Values of a: {a} and b: {b!s}' -f = a[1:3:2] \ No newline at end of file +f = a[1:3:2] From a79689ce1dadb851958a7465f84f3c80ff7a3916 Mon Sep 17 00:00:00 2001 From: Leutrim Shala Date: Thu, 21 Nov 2024 16:09:08 +0100 Subject: [PATCH 2/9] Add format spec handling --- .../cpg/frontends/python/ExpressionHandler.kt | 88 +++++++++++++------ .../frontends/python/PythonFrontendTest.kt | 88 +++++++++++++++++++ .../src/test/resources/python/datatypes.py | 7 +- 3 files changed, 153 insertions(+), 30 deletions(-) diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt index 4673169481..0f012ada58 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt @@ -184,39 +184,69 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : private fun handleFormattedValue(node: Python.AST.FormattedValue): Expression { val formatSpec = node.format_spec?.let { handle(it) } - if (formatSpec is Literal<*>) { - return formatSpec - } - return when (node.conversion) { - formattedValConversionNoFormatting -> { - // No formatting, just return the value. - handle(node.value) - } - formattedValConversionString -> { - // String representation. wrap in str() call. - val strCall = - newCallExpression(newReference("str", rawNode = node), "str", rawNode = node) - strCall.addArgument(handle(node.value)) - strCall - } - formattedValConversionRepr -> { - newProblemExpression( - "Cannot handle conversion '114: !r repr formatting', yet.", - rawNode = node - ) - } - formattedValConversionASCII -> { - newProblemExpression( - "Cannot handle conversion '97: !a ascii formatting', yet.", - rawNode = node - ) + + val valueExpression = handle(node.value) + val conversion = + when (node.conversion) { + formattedValConversionNoFormatting -> { + // No formatting, just return the value. + valueExpression + } + formattedValConversionString -> { + // String representation. wrap in str() call. + val strCall = + newCallExpression( + newReference("str", rawNode = node), + "str", + rawNode = node + ) + .implicit() + strCall.addArgument(valueExpression) + strCall + } + formattedValConversionRepr -> { + // String representation. wrap in repr() call. + val reprCall = + newCallExpression( + newReference("repr", rawNode = node), + "repr", + rawNode = node + ) + .implicit() + reprCall.addArgument(valueExpression) + reprCall + } + formattedValConversionASCII -> { + // String representation. wrap in ascii() call. + val asciiCall = + newCallExpression( + newReference("ascii", rawNode = node), + "ascii", + rawNode = node + ) + .implicit() + asciiCall.addArgument(handle(node.value)) + asciiCall + } + else -> + newProblemExpression( + "Cannot handle formatted value with conversion ${node.conversion} yet", + rawNode = node + ) } - else -> - newProblemExpression( - "Cannot handle formatted value with conversion ${node.conversion} yet", + if (formatSpec != null) { + return newCallExpression( + newReference("format", rawNode = node), + "format", rawNode = node ) + .implicit() + .apply { + addArgument(conversion) + addArgument(formatSpec) + } } + return conversion } private fun handleJoinedStr(node: Python.AST.JoinedStr): Expression { diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt index 04980d895e..d7786c868f 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt @@ -38,6 +38,7 @@ import de.fraunhofer.aisec.cpg.graph.types.ObjectType import de.fraunhofer.aisec.cpg.graph.types.SetType import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker import de.fraunhofer.aisec.cpg.passes.ControlDependenceGraphPass +import de.fraunhofer.aisec.cpg.query.value import de.fraunhofer.aisec.cpg.sarif.Region import de.fraunhofer.aisec.cpg.test.* import java.nio.file.Path @@ -1376,6 +1377,93 @@ class PythonFrontendTest : BaseTest() { assertEquals(2L, fStmtRhsThird.value) } + @Test + fun testFormattedValues() { + val topLevel = Path.of("src", "test", "resources", "python") + val tu = + analyzeAndGetFirstTU( + listOf(topLevel.resolve("datatypes.py").toFile()), + topLevel, + true + ) { + it.registerLanguage() + } + assertNotNull(tu) + val namespace = tu.namespaces.singleOrNull() + assertNotNull(namespace) + + // Test for g = f'Number: {42:.2f}' + val gStmt = namespace.statements[6] + assertIs(gStmt) + val gStmtRhs = gStmt.rhs.singleOrNull() + assertIs(gStmtRhs) + val gFormatCall = gStmtRhs.rhs + assertIs(gFormatCall) + assertEquals("format", gFormatCall.name.localName) + val gArguments = gFormatCall.arguments + assertEquals(2, gArguments.size) + assertIs>(gArguments[0]) + assertEquals(42.toLong(), gArguments[0].value.value) + assertIs>(gArguments[1]) + assertEquals(".2f", gArguments[1].value.value) + + // Test for h = f'Hexadecimal: {255:#x}' + val hStmt = namespace.statements[7] + assertIs(hStmt) + val hStmtRhs = hStmt.rhs.singleOrNull() + assertIs(hStmtRhs) + val hFormatCall = hStmtRhs.rhs + assertIs(hFormatCall) + assertEquals("format", hFormatCall.name.localName) + val hArguments = hFormatCall.arguments + assertEquals(2, hArguments.size) + assertIs>(hArguments[0]) + assertEquals(255L.toLong(), hArguments[0].value.value) + assertIs>(hArguments[1]) + assertEquals("#x", hArguments[1].value.value) + + // Test for i = f'String with conversion: {c!r}' + val iStmt = namespace.statements[8] + assertIs(iStmt) + val iStmtRhs = iStmt.rhs.singleOrNull() + assertIs(iStmtRhs) + val iConversionCall = iStmtRhs.rhs + assertIs(iConversionCall) + assertEquals("repr", iConversionCall.name.localName) + val iArg = iConversionCall.arguments.singleOrNull() + assertNotNull(iArg) + assertEquals("c", iArg.name.localName) + + // Test for j = f'ASCII representation: {d!a}' + val jStmt = namespace.statements[9] + assertIs(jStmt) + val jStmtRhs = jStmt.rhs.singleOrNull() + assertIs(jStmtRhs) + val jConversionCall = jStmtRhs.rhs + assertIs(jConversionCall) + assertEquals("ascii", jConversionCall.name.localName) + val jArg = jConversionCall.arguments.singleOrNull() + assertNotNull(jArg) + assertEquals("d", jArg.name.localName) + + // Test for k = f'Combined: {b!s:10}' + val kStmt = namespace.statements[10] + assertIs(kStmt) + val kStmtRhs = kStmt.rhs.singleOrNull() + assertIs(kStmtRhs) + val kFormatCall = kStmtRhs.rhs + assertIs(kFormatCall) + assertEquals("format", kFormatCall.name.localName) + val kArguments = kFormatCall.arguments + assertEquals(2, kArguments.size) + val kConversionCall = kArguments[0] + assertIs(kConversionCall) + assertEquals("str", kConversionCall.name.localName) + assertEquals("b", kConversionCall.arguments.singleOrNull()?.name?.localName) + assertIs>(kArguments[1]) + assertEquals("10", kArguments[1].value.value) + } + @Test fun testSimpleImport() { val topLevel = Path.of("src", "test", "resources", "python") diff --git a/cpg-language-python/src/test/resources/python/datatypes.py b/cpg-language-python/src/test/resources/python/datatypes.py index 2bed43ac9e..c84c162db7 100644 --- a/cpg-language-python/src/test/resources/python/datatypes.py +++ b/cpg-language-python/src/test/resources/python/datatypes.py @@ -6,6 +6,11 @@ "c": "d", "e": "f" } -aa = f"sin({a}) is {sin(a):.3}" e = f'Values of a: {a} and b: {b!s}' f = a[1:3:2] + +g = f'Number: {42:.2f}' +h = f'Hexadecimal: {255:#x}' +i = f'String with conversion: {c!r}' +j = f'ASCII representation: {d!a}' +k = f'Combined: {b!s:10}' \ No newline at end of file From 8d2d81e400caf7896337e6180faa8bd75b5baaa5 Mon Sep 17 00:00:00 2001 From: Leutrim Shala Date: Fri, 22 Nov 2024 14:46:05 +0100 Subject: [PATCH 3/9] spotless --- .../aisec/cpg/frontends/python/ExpressionHandler.kt | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt index 03f025a205..8e9930d848 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt @@ -182,9 +182,15 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : return assignExpression } + /** + * Translates a Python + * [`FormattedValue`](https://docs.python.org/3/library/ast.html#ast.FormattedValue) into a + * [Expression]. + * + * We are handling the format handling, following [PEP 3101](https://peps.python.org/pep-3101). + */ private fun handleFormattedValue(node: Python.AST.FormattedValue): Expression { val formatSpec = node.format_spec?.let { handle(it) } - val valueExpression = handle(node.value) val conversion = when (node.conversion) { @@ -249,6 +255,10 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : return conversion } + /** + * Translates a Python [`JoinedStr`](https://docs.python.org/3/library/ast.html#ast.JoinedStr) + * into a [Expression]. + */ private fun handleJoinedStr(node: Python.AST.JoinedStr): Expression { val values = node.values.map(::handle) return if (values.isEmpty()) { From 3b56fc45a6d18d3a84bbe08d607361dab2e79185 Mon Sep 17 00:00:00 2001 From: Leutrim Shala Date: Fri, 22 Nov 2024 14:48:43 +0100 Subject: [PATCH 4/9] Delete not related test --- .../frontends/python/PythonFrontendTest.kt | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt index d7786c868f..5bcba20b44 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt @@ -1677,27 +1677,6 @@ class PythonFrontendTest : BaseTest() { } } - @Test - fun testTest() { - val directoryPath = Path.of("/home", "lshala", "repos", "nova") - val tr = - analyze(fileExtension = ".py", directoryPath, usePasses = true) { - it.registerLanguage() - } - - val problemsList = tr.components.flatMap { it.translationUnits }.flatMap { it.problems } - - val msg = - (problemsList) - .groupBy { it.problem } - .toList() - .sortedBy { it.second.size } - .reversed() - .map { "${it.second.size}: ${it.first}" } - - msg.forEach(System.out::println) - } - class PythonValueEvaluator : ValueEvaluator() { override fun computeBinaryOpEffect( lhsValue: Any?, From 8eb1875aee312421950f35c3415da1fb17f3fdd2 Mon Sep 17 00:00:00 2001 From: Maximilian Kaul Date: Tue, 26 Nov 2024 13:51:23 +0100 Subject: [PATCH 5/9] pretty format --- .../cpg/frontends/python/ExpressionHandler.kt | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt index 8e9930d848..2923fc91bb 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt @@ -29,19 +29,18 @@ import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.ImportDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.MethodDeclaration import de.fraunhofer.aisec.cpg.graph.statements.expressions.* -import de.fraunhofer.aisec.cpg.graph.statements.expressions.CollectionComprehension import jep.python.PyObject class ExpressionHandler(frontend: PythonLanguageFrontend) : PythonHandler(::ProblemExpression, frontend) { /* - Magic numbers (https://docs.python.org/3/library/ast.html#ast.FormattedValue): - conversion is an integer: - -1: no formatting - 115: !s string formatting - 114: !r repr formatting - 97: !a ascii formatting + * Magic numbers (https://docs.python.org/3/library/ast.html#ast.FormattedValue): conversion is + * an integer: + * - `-1`: no formatting + * - `115`: `!s` string formatting + * - `114`: `!r` repr formatting + * - `97`: `!a` ascii formatting */ private val formattedValConversionNoFormatting = -1L private val formattedValConversionString = 115L @@ -184,7 +183,7 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : /** * Translates a Python - * [`FormattedValue`](https://docs.python.org/3/library/ast.html#ast.FormattedValue) into a + * [`FormattedValue`](https://docs.python.org/3/library/ast.html#ast.FormattedValue) into an * [Expression]. * * We are handling the format handling, following [PEP 3101](https://peps.python.org/pep-3101). @@ -199,11 +198,11 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : valueExpression } formattedValConversionString -> { - // String representation. wrap in str() call. + // String representation: wrap in `str()` call. val strCall = newCallExpression( - newReference("str", rawNode = node), - "str", + callee = newReference(name = "str", rawNode = node), + fqn = "str", rawNode = node ) .implicit() @@ -211,11 +210,11 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : strCall } formattedValConversionRepr -> { - // String representation. wrap in repr() call. + // Repr-String representation: wrap in `repr()` call. val reprCall = newCallExpression( - newReference("repr", rawNode = node), - "repr", + callee = newReference(name = "repr", rawNode = node), + fqn = "repr", rawNode = node ) .implicit() @@ -223,7 +222,7 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : reprCall } formattedValConversionASCII -> { - // String representation. wrap in ascii() call. + // ASCII-String representation: wrap in `ascii()` call. val asciiCall = newCallExpression( newReference("ascii", rawNode = node), @@ -236,14 +235,15 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : } else -> newProblemExpression( - "Cannot handle formatted value with conversion ${node.conversion} yet", + problem = + "Cannot handle formatted value with conversion code ${node.conversion} yet", rawNode = node ) } if (formatSpec != null) { return newCallExpression( - newReference("format", rawNode = node), - "format", + callee = newReference(name = "format", rawNode = node), + fqn = "format", rawNode = node ) .implicit() From cdab949b77da7d541408eb574367c68e29228399 Mon Sep 17 00:00:00 2001 From: Leutrim Shala Date: Wed, 27 Nov 2024 13:27:37 +0100 Subject: [PATCH 6/9] spotless --- .../cpg/frontends/python/ExpressionHandler.kt | 26 ++++++++--------- .../FormattedValueHandlerTest.kt | 28 +++++++++++++++++++ .../test/resources/python/formatted_values.py | 0 3 files changed, 41 insertions(+), 13 deletions(-) create mode 100644 cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/expressionHandler/FormattedValueHandlerTest.kt create mode 100644 cpg-language-python/src/test/resources/python/formatted_values.py diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt index 2923fc91bb..e164a0a8b4 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt @@ -34,19 +34,6 @@ import jep.python.PyObject class ExpressionHandler(frontend: PythonLanguageFrontend) : PythonHandler(::ProblemExpression, frontend) { - /* - * Magic numbers (https://docs.python.org/3/library/ast.html#ast.FormattedValue): conversion is - * an integer: - * - `-1`: no formatting - * - `115`: `!s` string formatting - * - `114`: `!r` repr formatting - * - `97`: `!a` ascii formatting - */ - private val formattedValConversionNoFormatting = -1L - private val formattedValConversionString = 115L - private val formattedValConversionRepr = 114L - private val formattedValConversionASCII = 97L - override fun handleNode(node: Python.AST.BaseExpr): Expression { return when (node) { is Python.AST.Name -> handleName(node) @@ -189,6 +176,19 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : * We are handling the format handling, following [PEP 3101](https://peps.python.org/pep-3101). */ private fun handleFormattedValue(node: Python.AST.FormattedValue): Expression { + /* + Magic numbers (https://docs.python.org/3/library/ast.html#ast.FormattedValue): + conversion is an integer: + -1: no formatting + 115: !s string formatting + 114: !r repr formatting + 97: !a ascii formatting + */ + val formattedValConversionNoFormatting = -1L + val formattedValConversionString = 115L + val formattedValConversionRepr = 114L + val formattedValConversionASCII = 97L + val formatSpec = node.format_spec?.let { handle(it) } val valueExpression = handle(node.value) val conversion = diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/expressionHandler/FormattedValueHandlerTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/expressionHandler/FormattedValueHandlerTest.kt new file mode 100644 index 0000000000..694b13dd3e --- /dev/null +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/expressionHandler/FormattedValueHandlerTest.kt @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.frontends.python.expressionHandler + +class FormattedValueHandlerTest {} diff --git a/cpg-language-python/src/test/resources/python/formatted_values.py b/cpg-language-python/src/test/resources/python/formatted_values.py new file mode 100644 index 0000000000..e69de29bb2 From f1c2cc5183cce2ed3f09fe859a59ccede4e4fbfb Mon Sep 17 00:00:00 2001 From: Leutrim Shala Date: Wed, 27 Nov 2024 13:23:25 +0100 Subject: [PATCH 7/9] Refactor --- .../frontends/python/PythonFrontendTest.kt | 87 ------------- .../FormattedValueHandlerTest.kt | 122 +++++++++++++++++- .../src/test/resources/python/datatypes.py | 5 - .../test/resources/python/formatted_values.py | 5 + 4 files changed, 126 insertions(+), 93 deletions(-) diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt index 5bcba20b44..e95489f9b4 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt @@ -1377,93 +1377,6 @@ class PythonFrontendTest : BaseTest() { assertEquals(2L, fStmtRhsThird.value) } - @Test - fun testFormattedValues() { - val topLevel = Path.of("src", "test", "resources", "python") - val tu = - analyzeAndGetFirstTU( - listOf(topLevel.resolve("datatypes.py").toFile()), - topLevel, - true - ) { - it.registerLanguage() - } - assertNotNull(tu) - val namespace = tu.namespaces.singleOrNull() - assertNotNull(namespace) - - // Test for g = f'Number: {42:.2f}' - val gStmt = namespace.statements[6] - assertIs(gStmt) - val gStmtRhs = gStmt.rhs.singleOrNull() - assertIs(gStmtRhs) - val gFormatCall = gStmtRhs.rhs - assertIs(gFormatCall) - assertEquals("format", gFormatCall.name.localName) - val gArguments = gFormatCall.arguments - assertEquals(2, gArguments.size) - assertIs>(gArguments[0]) - assertEquals(42.toLong(), gArguments[0].value.value) - assertIs>(gArguments[1]) - assertEquals(".2f", gArguments[1].value.value) - - // Test for h = f'Hexadecimal: {255:#x}' - val hStmt = namespace.statements[7] - assertIs(hStmt) - val hStmtRhs = hStmt.rhs.singleOrNull() - assertIs(hStmtRhs) - val hFormatCall = hStmtRhs.rhs - assertIs(hFormatCall) - assertEquals("format", hFormatCall.name.localName) - val hArguments = hFormatCall.arguments - assertEquals(2, hArguments.size) - assertIs>(hArguments[0]) - assertEquals(255L.toLong(), hArguments[0].value.value) - assertIs>(hArguments[1]) - assertEquals("#x", hArguments[1].value.value) - - // Test for i = f'String with conversion: {c!r}' - val iStmt = namespace.statements[8] - assertIs(iStmt) - val iStmtRhs = iStmt.rhs.singleOrNull() - assertIs(iStmtRhs) - val iConversionCall = iStmtRhs.rhs - assertIs(iConversionCall) - assertEquals("repr", iConversionCall.name.localName) - val iArg = iConversionCall.arguments.singleOrNull() - assertNotNull(iArg) - assertEquals("c", iArg.name.localName) - - // Test for j = f'ASCII representation: {d!a}' - val jStmt = namespace.statements[9] - assertIs(jStmt) - val jStmtRhs = jStmt.rhs.singleOrNull() - assertIs(jStmtRhs) - val jConversionCall = jStmtRhs.rhs - assertIs(jConversionCall) - assertEquals("ascii", jConversionCall.name.localName) - val jArg = jConversionCall.arguments.singleOrNull() - assertNotNull(jArg) - assertEquals("d", jArg.name.localName) - - // Test for k = f'Combined: {b!s:10}' - val kStmt = namespace.statements[10] - assertIs(kStmt) - val kStmtRhs = kStmt.rhs.singleOrNull() - assertIs(kStmtRhs) - val kFormatCall = kStmtRhs.rhs - assertIs(kFormatCall) - assertEquals("format", kFormatCall.name.localName) - val kArguments = kFormatCall.arguments - assertEquals(2, kArguments.size) - val kConversionCall = kArguments[0] - assertIs(kConversionCall) - assertEquals("str", kConversionCall.name.localName) - assertEquals("b", kConversionCall.arguments.singleOrNull()?.name?.localName) - assertIs>(kArguments[1]) - assertEquals("10", kArguments[1].value.value) - } - @Test fun testSimpleImport() { val topLevel = Path.of("src", "test", "resources", "python") diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/expressionHandler/FormattedValueHandlerTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/expressionHandler/FormattedValueHandlerTest.kt index 694b13dd3e..8efa19a88e 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/expressionHandler/FormattedValueHandlerTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/expressionHandler/FormattedValueHandlerTest.kt @@ -25,4 +25,124 @@ */ package de.fraunhofer.aisec.cpg.frontends.python.expressionHandler -class FormattedValueHandlerTest {} +import de.fraunhofer.aisec.cpg.frontends.python.PythonLanguage +import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration +import de.fraunhofer.aisec.cpg.graph.statements.expressions.AssignExpression +import de.fraunhofer.aisec.cpg.graph.statements.expressions.BinaryOperator +import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression +import de.fraunhofer.aisec.cpg.graph.statements.expressions.Literal +import de.fraunhofer.aisec.cpg.graph.variables +import de.fraunhofer.aisec.cpg.test.analyzeAndGetFirstTU +import de.fraunhofer.aisec.cpg.test.assertLiteralValue +import de.fraunhofer.aisec.cpg.test.assertLocalName +import java.nio.file.Path +import kotlin.test.assertEquals +import kotlin.test.assertIs +import kotlin.test.assertNotNull +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInstance + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class FormattedValueHandlerTest { + + private lateinit var topLevel: Path + private lateinit var result: TranslationUnitDeclaration + + @BeforeAll + fun setup() { + topLevel = Path.of("src", "test", "resources", "python") + analyzeFile() + } + + fun analyzeFile() { + result = + analyzeAndGetFirstTU( + listOf(topLevel.resolve("formatted_values.py").toFile()), + topLevel, + true + ) { + it.registerLanguage() + } + assertNotNull(result) + } + + @Test + fun testFormattedValues() { + // Test for a = f'Number: {42:.2f}' + val aAssExpression = + result.variables.find { variable -> variable.name.localName == "a" }?.astParent + assertIs(aAssExpression) + val aExprRhs = aAssExpression.rhs.singleOrNull() + assertIs(aExprRhs) + val aFormatCall = aExprRhs.rhs + assertIs(aFormatCall) + assertLocalName("format", aFormatCall) + val aArguments = aFormatCall.arguments + assertEquals(2, aArguments.size) + assertIs>(aArguments[0]) + assertLiteralValue(42.toLong(), aArguments[0]) + assertIs>(aArguments[1]) + assertLiteralValue(".2f", aArguments[1]) + + // Test for b = f'Hexadecimal: {255:#x}' + val bAssExpression = + result.variables.find { variable -> variable.name.localName == "b" }?.astParent + assertIs(bAssExpression) + val bExprRhs = bAssExpression.rhs.singleOrNull() + assertIs(bExprRhs) + val bFormatCall = bExprRhs.rhs + assertIs(bFormatCall) + assertLocalName("format", bFormatCall) + val bArguments = bFormatCall.arguments + assertEquals(2, bArguments.size) + assertIs>(bArguments[0]) + assertLiteralValue(255L.toLong(), bArguments[0]) + assertIs>(bArguments[1]) + assertLiteralValue("#x", bArguments[1]) + + // Test for c = f'String with conversion: {"Hello, world!"!r}' + val cAssExpression = + result.variables.find { variable -> variable.name.localName == "c" }?.astParent + assertIs(cAssExpression) + val cExprRhs = cAssExpression.rhs.singleOrNull() + assertIs(cExprRhs) + val cConversionCall = cExprRhs.rhs + assertIs(cConversionCall) + assertLocalName("repr", cConversionCall) + val cArguments = cConversionCall.arguments.singleOrNull() + assertNotNull(cArguments) + assertLocalName("c", cArguments) + + // Test for d = f'ASCII representation: {"50$"!a}' + val dAssExpression = + result.variables.find { variable -> variable.name.localName == "d" }?.astParent + assertIs(dAssExpression) + val dExprRhs = dAssExpression.rhs.singleOrNull() + assertIs(dExprRhs) + val dConversionCall = dExprRhs.rhs + assertIs(dConversionCall) + assertLocalName("ascii", dConversionCall) + val dArguments = dConversionCall.arguments.singleOrNull() + assertNotNull(dArguments) + assertLocalName("d", dArguments) + + // Test for e = f'Combined: {42!s:10}' + val eAssExpression = + result.variables.find { variable -> variable.name.localName == "e" }?.astParent + assertIs(eAssExpression) + val eExprRhs = eAssExpression.rhs.singleOrNull() + assertIs(eExprRhs) + val eFormatCall = eExprRhs.rhs + assertIs(eFormatCall) + assertLocalName("format", eFormatCall) + val eArguments = eFormatCall.arguments + assertEquals(2, eArguments.size) + val kConversionCall = eArguments[0] + assertIs(kConversionCall) + assertLocalName("str", kConversionCall) + assertLocalName("b", kConversionCall.arguments.singleOrNull()) + assertIs>(eArguments[1]) + assertLiteralValue("10", eArguments[1]) + } +} diff --git a/cpg-language-python/src/test/resources/python/datatypes.py b/cpg-language-python/src/test/resources/python/datatypes.py index c84c162db7..9ad09dcef6 100644 --- a/cpg-language-python/src/test/resources/python/datatypes.py +++ b/cpg-language-python/src/test/resources/python/datatypes.py @@ -9,8 +9,3 @@ e = f'Values of a: {a} and b: {b!s}' f = a[1:3:2] -g = f'Number: {42:.2f}' -h = f'Hexadecimal: {255:#x}' -i = f'String with conversion: {c!r}' -j = f'ASCII representation: {d!a}' -k = f'Combined: {b!s:10}' \ No newline at end of file diff --git a/cpg-language-python/src/test/resources/python/formatted_values.py b/cpg-language-python/src/test/resources/python/formatted_values.py index e69de29bb2..1efd3bc4dc 100644 --- a/cpg-language-python/src/test/resources/python/formatted_values.py +++ b/cpg-language-python/src/test/resources/python/formatted_values.py @@ -0,0 +1,5 @@ +a = f'Number: {42:.2f}' +b = f'Hexadecimal: {255:#x}' +c = f'String with conversion: {"Hello, world!"!r}' +d = f'ASCII representation: {"50$"!a}' +e = f'Combined: {42!s:10}' \ No newline at end of file From 5c997a1df26e7a1c26a98e262a5054be0ce714f4 Mon Sep 17 00:00:00 2001 From: Leutrim Shala Date: Wed, 27 Nov 2024 14:56:48 +0100 Subject: [PATCH 8/9] spotless --- .../cpg/frontends/python/ExpressionHandler.kt | 19 +++++++++++ .../FormattedValueHandlerTest.kt | 32 ++++++++----------- 2 files changed, 32 insertions(+), 19 deletions(-) diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt index e164a0a8b4..7e9ce3081c 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt @@ -174,6 +174,25 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : * [Expression]. * * We are handling the format handling, following [PEP 3101](https://peps.python.org/pep-3101). + * + * The following example + * + * ```python + * f"{value:.2f}" + * ``` + * + * is modeled: + * 1. The value `value` is wrapped in a `format()` call. + * 2. The `format()` call has two arguments: + * - The value to format (`value`). + * - The format specification (`".2f"`). + * + * CPG Representation: + * - `CallExpression` node: + * - `callee`: `Reference` to `format`. + * - `arguments`: + * 1. A node representing `value`. + * 2. A node representing the string `".2f"`. */ private fun handleFormattedValue(node: Python.AST.FormattedValue): Expression { /* diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/expressionHandler/FormattedValueHandlerTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/expressionHandler/FormattedValueHandlerTest.kt index 8efa19a88e..799257509e 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/expressionHandler/FormattedValueHandlerTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/expressionHandler/FormattedValueHandlerTest.kt @@ -26,12 +26,12 @@ package de.fraunhofer.aisec.cpg.frontends.python.expressionHandler import de.fraunhofer.aisec.cpg.frontends.python.PythonLanguage +import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration import de.fraunhofer.aisec.cpg.graph.statements.expressions.AssignExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.BinaryOperator import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.Literal -import de.fraunhofer.aisec.cpg.graph.variables import de.fraunhofer.aisec.cpg.test.analyzeAndGetFirstTU import de.fraunhofer.aisec.cpg.test.assertLiteralValue import de.fraunhofer.aisec.cpg.test.assertLocalName @@ -70,8 +70,7 @@ class FormattedValueHandlerTest { @Test fun testFormattedValues() { // Test for a = f'Number: {42:.2f}' - val aAssExpression = - result.variables.find { variable -> variable.name.localName == "a" }?.astParent + val aAssExpression = result.variables["a"]?.astParent assertIs(aAssExpression) val aExprRhs = aAssExpression.rhs.singleOrNull() assertIs(aExprRhs) @@ -86,8 +85,7 @@ class FormattedValueHandlerTest { assertLiteralValue(".2f", aArguments[1]) // Test for b = f'Hexadecimal: {255:#x}' - val bAssExpression = - result.variables.find { variable -> variable.name.localName == "b" }?.astParent + val bAssExpression = result.variables["b"]?.astParent assertIs(bAssExpression) val bExprRhs = bAssExpression.rhs.singleOrNull() assertIs(bExprRhs) @@ -98,12 +96,11 @@ class FormattedValueHandlerTest { assertEquals(2, bArguments.size) assertIs>(bArguments[0]) assertLiteralValue(255L.toLong(), bArguments[0]) - assertIs>(bArguments[1]) + // assertIs>(bArguments[1]) assertLiteralValue("#x", bArguments[1]) // Test for c = f'String with conversion: {"Hello, world!"!r}' - val cAssExpression = - result.variables.find { variable -> variable.name.localName == "c" }?.astParent + val cAssExpression = result.variables["c"]?.astParent assertIs(cAssExpression) val cExprRhs = cAssExpression.rhs.singleOrNull() assertIs(cExprRhs) @@ -112,11 +109,10 @@ class FormattedValueHandlerTest { assertLocalName("repr", cConversionCall) val cArguments = cConversionCall.arguments.singleOrNull() assertNotNull(cArguments) - assertLocalName("c", cArguments) + assertLiteralValue("Hello, world!", cArguments) // Test for d = f'ASCII representation: {"50$"!a}' - val dAssExpression = - result.variables.find { variable -> variable.name.localName == "d" }?.astParent + val dAssExpression = result.variables["d"]?.astParent assertIs(dAssExpression) val dExprRhs = dAssExpression.rhs.singleOrNull() assertIs(dExprRhs) @@ -125,11 +121,10 @@ class FormattedValueHandlerTest { assertLocalName("ascii", dConversionCall) val dArguments = dConversionCall.arguments.singleOrNull() assertNotNull(dArguments) - assertLocalName("d", dArguments) + assertLiteralValue("50$", dArguments) // Test for e = f'Combined: {42!s:10}' - val eAssExpression = - result.variables.find { variable -> variable.name.localName == "e" }?.astParent + val eAssExpression = result.variables["e"]?.astParent assertIs(eAssExpression) val eExprRhs = eAssExpression.rhs.singleOrNull() assertIs(eExprRhs) @@ -138,11 +133,10 @@ class FormattedValueHandlerTest { assertLocalName("format", eFormatCall) val eArguments = eFormatCall.arguments assertEquals(2, eArguments.size) - val kConversionCall = eArguments[0] - assertIs(kConversionCall) - assertLocalName("str", kConversionCall) - assertLocalName("b", kConversionCall.arguments.singleOrNull()) - assertIs>(eArguments[1]) + val eConversionCall = eArguments[0] + assertIs(eConversionCall) + assertLocalName("str", eConversionCall) + assertLiteralValue("42".toLong(), eConversionCall.arguments.singleOrNull()) assertLiteralValue("10", eArguments[1]) } } From 890cb16e5221a5602436bec4a50872185b47e21e Mon Sep 17 00:00:00 2001 From: Leutrim Shala Date: Wed, 27 Nov 2024 14:58:25 +0100 Subject: [PATCH 9/9] Add suggested comment --- .../python/expressionHandler/FormattedValueHandlerTest.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/expressionHandler/FormattedValueHandlerTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/expressionHandler/FormattedValueHandlerTest.kt index 799257509e..fe26f34a77 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/expressionHandler/FormattedValueHandlerTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/expressionHandler/FormattedValueHandlerTest.kt @@ -124,6 +124,7 @@ class FormattedValueHandlerTest { assertLiteralValue("50$", dArguments) // Test for e = f'Combined: {42!s:10}' + // This is translated to `'Combined: ' + format(str(b), "10")` val eAssExpression = result.variables["e"]?.astParent assertIs(eAssExpression) val eExprRhs = eAssExpression.rhs.singleOrNull()