diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/CommentMatcher.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/CommentMatcher.kt index b07fc55742..df7bfcae12 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/CommentMatcher.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/CommentMatcher.kt @@ -99,7 +99,7 @@ class CommentMatcher { .filter { artifactLocation == null || artifactLocation == it.location?.artifactLocation } - .toMutableList() + .toMutableSet() // Because we sometimes wrap all elements into a NamespaceDeclaration we have to extract the // children with a location @@ -109,6 +109,19 @@ class CommentMatcher { } ) + // When a child has no location we can not properly consider it for comment matching, + // however, it might have + // a child with a location that we want to consider, this can overlap with namespaces but + // nodes are considered + // only once in the set + children.addAll( + children + .filter { node -> node.location == null || node.location?.region == Region() } + .flatMap { locationLess -> + SubgraphWalker.getAstChildren(locationLess).filter { it !in children } + } + ) + // Searching for the closest successor to our comment amongst the children of the smallest // enclosing nodes val successors = diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt index 240e9cbaf1..a7d6ceabd0 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt @@ -81,12 +81,13 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : private fun handleDict(node: Python.ASTDict): Expression { val lst = mutableListOf() for (i in node.values.indices) { // TODO: keys longer than values possible? + // Here we can not use node as raw node as it spans all keys and values lst += newKeyValueExpression( - key = node.keys[i]?.let { handle(it) }, - value = handle(node.values[i]), - rawNode = node - ) + key = node.keys[i]?.let { handle(it) }, + value = handle(node.values[i]), + ) + .codeAndLocationFromChildren(node) } val ile = newInitializerListExpression(rawNode = node) ile.initializers = lst.toList() diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/JepSingleton.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/JepSingleton.kt index 3af8e4461c..f7675d4879 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/JepSingleton.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/JepSingleton.kt @@ -60,7 +60,7 @@ object JepSingleton { val virtualEnvName = System.getenv("CPG_PYTHON_VIRTUALENV") ?: "cpg" val virtualEnvPath = Paths.get(System.getProperty("user.home"), ".virtualenvs", virtualEnvName) - val pythonVersions = listOf("3.9", "3.10", "3.11", "3.12", "3.13") + val pythonVersions = listOf("3.8", "3.9", "3.10", "3.11", "3.12", "3.13") val wellKnownPaths = mutableListOf() pythonVersions.forEach { version -> // Linux diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/Python.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/Python.kt index 70623353a8..22609fe702 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/Python.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/Python.kt @@ -1197,7 +1197,7 @@ interface Python { * | arg(identifier arg, expr? annotation, string? type_comment) * ``` */ - class ASTarg(pyObject: PyObject) : AST(pyObject) { + class ASTarg(pyObject: PyObject) : AST(pyObject), WithPythonLocation { val arg: String by lazy { "arg" of pyObject } val annotation: ASTBASEexpr? by lazy { "annotation" of pyObject } val type_comment: String? by lazy { "type_comment" of pyObject } diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguageFrontend.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguageFrontend.kt index 4fef493171..0607ae7870 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguageFrontend.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguageFrontend.kt @@ -33,6 +33,7 @@ import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration import de.fraunhofer.aisec.cpg.graph.types.AutoType import de.fraunhofer.aisec.cpg.graph.types.Type +import de.fraunhofer.aisec.cpg.helpers.CommentMatcher import de.fraunhofer.aisec.cpg.passes.PythonAddDeclarationsPass import de.fraunhofer.aisec.cpg.passes.order.RegisterExtraPass import de.fraunhofer.aisec.cpg.sarif.PhysicalLocation @@ -74,7 +75,48 @@ class PythonLanguageFrontend(language: Language, ctx: Tr it.exec("parsed = ast.parse(content, filename=filename, type_comments=True)") val pyAST = it.getValue("parsed") as PyObject - return pythonASTtoCPG(pyAST, file.name) + val tud = pythonASTtoCPG(pyAST, file.name) + + if (config.matchCommentsToNodes) { + it.exec("import tokenize") + it.exec("reader = tokenize.open(filename).readline") + it.exec("tokens = tokenize.generate_tokens(reader)") + it.exec("tokenList = list(tokens)") + val pyTokens = (it.getValue("tokenList") as? ArrayList<*>) ?: TODO() + addCommentsToCPG(tud, pyTokens) + } + return tud + } + } + + private fun addCommentsToCPG(tud: TranslationUnitDeclaration, pyTokens: ArrayList<*>) { + val commentMatcher = CommentMatcher() + for (token in pyTokens) { + if (token !is List<*> || token.size != 5) { + TODO() + } else { + if (token[0] as Long != 60.toLong() && !(token[1].toString()).startsWith("#")) { + continue + } else { + val start = token[2] as List<*> + val end = token[3] as List<*> + val startLine = start[0] as Long + val startCol = start[1] as Long + val endLine = end[0] as Long + val endCol = end[1] as Long + + commentMatcher.matchCommentToNode( + token[1] as String, + Region( + startLine.toInt(), + (startCol + 1).toInt(), + endLine.toInt(), + (endCol + 1).toInt() + ), + tud + ) + } + } } } @@ -151,18 +193,10 @@ class PythonLanguageFrontend(language: Language, ctx: Tr lines: MutableList ): MutableList { for (idx in lines.indices) { - val prefixLength = min(location.region.startColumn, lines[idx].length) + // -1 to equalize for +1 in sarif + val prefixLength = min(location.region.startColumn - 1, lines[idx].length) if (idx == 0) { lines[idx] = lines[idx].substring(prefixLength) - } else { - - for (j in 0..prefixLength - 1) { - if (lines[idx][0] == ' ' || lines[idx][0] == '\t') { - lines[idx] = lines[idx].substring(1) - } else { - break - } - } } } return lines @@ -175,7 +209,7 @@ class PythonLanguageFrontend(language: Language, ctx: Tr val lastLineIdx = lines.lastIndex val lastLineLength = lines[lastLineIdx].length val locationEndColumn = location.region.endColumn - val toRemove = lastLineLength - locationEndColumn + val toRemove = lastLineLength - locationEndColumn + 1 if (toRemove > 0) { lines[lastLineIdx] = lines[lastLineIdx].dropLast(toRemove) } @@ -189,8 +223,8 @@ class PythonLanguageFrontend(language: Language, ctx: Tr Region( startLine = astNode.lineno, endLine = astNode.end_lineno, - startColumn = astNode.col_offset, - endColumn = astNode.end_col_offset, + startColumn = astNode.col_offset + 1, + endColumn = astNode.end_col_offset + 1, ) ) } else { @@ -205,7 +239,7 @@ class PythonLanguageFrontend(language: Language, ctx: Tr private fun pythonASTtoCPG(pyAST: PyObject, path: String): TranslationUnitDeclaration { val pythonASTModule = fromPython(pyAST) as? Python.ASTModule - ?: TODO() // could be one of ast.{Module,Interactive,Expression,FunctionType} + ?: TODO() // could be one of "ast.{Module,Interactive,Expression,FunctionType} val tud = newTranslationUnitDeclaration(path, rawNode = pythonASTModule) scopeManager.resetToGlobal(tud) @@ -236,133 +270,135 @@ fun fromPython(pyObject: Any?): Python.AST { if (pyObject !is PyObject) { TODO("Expected a PyObject") } else { - - return when (pyObject.getAttr("__class__").toString()) { - "" -> Python.ASTModule(pyObject) + var objectname = + pyObject.getAttr("__class__").toString().substringAfter("'").substringBeforeLast("'") + objectname = if (objectname.startsWith("_")) objectname.substringAfter("_") else objectname + return when (objectname) { + "ast.Module" -> Python.ASTModule(pyObject) // statements - "" -> Python.ASTFunctionDef(pyObject) - "" -> Python.ASTAsyncFunctionDef(pyObject) - "" -> Python.ASTClassDef(pyObject) - "" -> Python.ASTReturn(pyObject) - "" -> Python.ASTDelete(pyObject) - "" -> Python.ASTAssign(pyObject) - "" -> Python.ASTAugAssign(pyObject) - "" -> Python.ASTAnnAssign(pyObject) - "" -> Python.ASTFor(pyObject) - "" -> Python.ASTAsyncFor(pyObject) - "" -> Python.ASTWhile(pyObject) - "" -> Python.ASTIf(pyObject) - "" -> Python.ASTWith(pyObject) - "" -> Python.ASTAsyncWith(pyObject) - "" -> Python.ASTMatch(pyObject) - "" -> Python.ASTRaise(pyObject) - "" -> Python.ASTTry(pyObject) - "" -> Python.ASTTryStar(pyObject) - "" -> Python.ASTAssert(pyObject) - "" -> Python.ASTImport(pyObject) - "" -> Python.ASTImportFrom(pyObject) - "" -> Python.ASTGlobal(pyObject) - "" -> Python.ASTNonlocal(pyObject) - "" -> Python.ASTExpr(pyObject) - "" -> Python.ASTPass(pyObject) - "" -> Python.ASTBreak(pyObject) - "" -> Python.ASTContinue(pyObject) - - // `ast.expr` - "" -> Python.ASTBoolOp(pyObject) - "" -> Python.ASTNamedExpr(pyObject) - "" -> Python.ASTBinOp(pyObject) - "" -> Python.ASTUnaryOp(pyObject) - "" -> Python.ASTLambda(pyObject) - "" -> Python.ASTIfExp(pyObject) - "" -> Python.ASTDict(pyObject) - "" -> Python.ASTSet(pyObject) - "" -> Python.ASTListComp(pyObject) - "" -> Python.ASTSetComp(pyObject) - "" -> Python.ASTDictComp(pyObject) - "" -> Python.ASTGeneratorExp(pyObject) - "" -> Python.ASTAwait(pyObject) - "" -> Python.ASTYield(pyObject) - "" -> Python.ASTYieldFrom(pyObject) - "" -> Python.ASTCompare(pyObject) - "" -> Python.ASTCall(pyObject) - "" -> Python.ASTFormattedValue(pyObject) - "" -> Python.ASTJoinedStr(pyObject) - "" -> Python.ASTConstant(pyObject) - "" -> Python.ASTAttribute(pyObject) - "" -> Python.ASTSubscript(pyObject) - "" -> Python.ASTStarred(pyObject) - "" -> Python.ASTName(pyObject) - "" -> Python.ASTList(pyObject) - "" -> Python.ASTTuple(pyObject) - "" -> Python.ASTSlice(pyObject) - - // `ast.boolop` - "" -> Python.ASTAnd(pyObject) - "" -> Python.ASTOr(pyObject) - - // `ast.cmpop` - "" -> Python.ASTEq(pyObject) - "" -> Python.ASTNotEq(pyObject) - "" -> Python.ASTLt(pyObject) - "" -> Python.ASTLtE(pyObject) - "" -> Python.ASTGt(pyObject) - "" -> Python.ASTGtE(pyObject) - "" -> Python.ASTIs(pyObject) - "" -> Python.ASTIsNot(pyObject) - "" -> Python.ASTIn(pyObject) - "" -> Python.ASTNotIn(pyObject) - - // `ast.expr_context` - "" -> Python.ASTLoad(pyObject) - "" -> Python.ASTStore(pyObject) - "" -> Python.ASTDel(pyObject) - - // `ast.operator` - "" -> Python.ASTAdd(pyObject) - "" -> Python.ASTSub(pyObject) - "" -> Python.ASTMult(pyObject) - "" -> Python.ASTMatMult(pyObject) - "" -> Python.ASTDiv(pyObject) - "" -> Python.ASTMod(pyObject) - "" -> Python.ASTPow(pyObject) - "" -> Python.ASTLShift(pyObject) - "" -> Python.ASTRShift(pyObject) - "" -> Python.ASTBitOr(pyObject) - "" -> Python.ASTBitXor(pyObject) - "" -> Python.ASTBitAnd(pyObject) - "" -> Python.ASTFloorDiv(pyObject) - - // `ast.pattern` - "" -> Python.ASTMatchValue(pyObject) - "" -> Python.ASTMatchSingleton(pyObject) - "" -> Python.ASTMatchSequence(pyObject) - "" -> Python.ASTMatchMapping(pyObject) - "" -> Python.ASTMatchClass(pyObject) - "" -> Python.ASTMatchStar(pyObject) - "" -> Python.ASTMatchAs(pyObject) - "" -> Python.ASTMatchOr(pyObject) - - // `ast.unaryop` - "" -> Python.ASTInvert(pyObject) - "" -> Python.ASTNot(pyObject) - "" -> Python.ASTUAdd(pyObject) - "" -> Python.ASTUSub(pyObject) + "ast.FunctionDef" -> Python.ASTFunctionDef(pyObject) + "ast.AsyncFunctionDef" -> Python.ASTAsyncFunctionDef(pyObject) + "ast.ClassDef" -> Python.ASTClassDef(pyObject) + "ast.Return" -> Python.ASTReturn(pyObject) + "ast.Delete" -> Python.ASTDelete(pyObject) + "ast.Assign" -> Python.ASTAssign(pyObject) + "ast.AugAssign" -> Python.ASTAugAssign(pyObject) + "ast.AnnAssign" -> Python.ASTAnnAssign(pyObject) + "ast.For" -> Python.ASTFor(pyObject) + "ast.AsyncFor" -> Python.ASTAsyncFor(pyObject) + "ast.While" -> Python.ASTWhile(pyObject) + "ast.If" -> Python.ASTIf(pyObject) + "ast.With" -> Python.ASTWith(pyObject) + "ast.AsyncWith" -> Python.ASTAsyncWith(pyObject) + "ast.Match" -> Python.ASTMatch(pyObject) + "ast.Raise" -> Python.ASTRaise(pyObject) + "ast.Try" -> Python.ASTTry(pyObject) + "ast.TryStar" -> Python.ASTTryStar(pyObject) + "ast.Assert" -> Python.ASTAssert(pyObject) + "ast.Import" -> Python.ASTImport(pyObject) + "ast.ImportFrom" -> Python.ASTImportFrom(pyObject) + "ast.Global" -> Python.ASTGlobal(pyObject) + "ast.Nonlocal" -> Python.ASTNonlocal(pyObject) + "ast.Expr" -> Python.ASTExpr(pyObject) + "ast.Pass" -> Python.ASTPass(pyObject) + "ast.Break" -> Python.ASTBreak(pyObject) + "ast.Continue" -> Python.ASTContinue(pyObject) + + // `"ast.expr` + "ast.BoolOp" -> Python.ASTBoolOp(pyObject) + "ast.NamedExpr" -> Python.ASTNamedExpr(pyObject) + "ast.BinOp" -> Python.ASTBinOp(pyObject) + "ast.UnaryOp" -> Python.ASTUnaryOp(pyObject) + "ast.Lambda" -> Python.ASTLambda(pyObject) + "ast.IfExp" -> Python.ASTIfExp(pyObject) + "ast.Dict" -> Python.ASTDict(pyObject) + "ast.Set" -> Python.ASTSet(pyObject) + "ast.ListComp" -> Python.ASTListComp(pyObject) + "ast.SetComp" -> Python.ASTSetComp(pyObject) + "ast.DictComp" -> Python.ASTDictComp(pyObject) + "ast.GeneratorExp" -> Python.ASTGeneratorExp(pyObject) + "ast.Await" -> Python.ASTAwait(pyObject) + "ast.Yield" -> Python.ASTYield(pyObject) + "ast.YieldFrom" -> Python.ASTYieldFrom(pyObject) + "ast.Compare" -> Python.ASTCompare(pyObject) + "ast.Call" -> Python.ASTCall(pyObject) + "ast.FormattedValue" -> Python.ASTFormattedValue(pyObject) + "ast.JoinedStr" -> Python.ASTJoinedStr(pyObject) + "ast.Constant" -> Python.ASTConstant(pyObject) + "ast.Attribute" -> Python.ASTAttribute(pyObject) + "ast.Subscript" -> Python.ASTSubscript(pyObject) + "ast.Starred" -> Python.ASTStarred(pyObject) + "ast.Name" -> Python.ASTName(pyObject) + "ast.List" -> Python.ASTList(pyObject) + "ast.Tuple" -> Python.ASTTuple(pyObject) + "ast.Slice" -> Python.ASTSlice(pyObject) + + // `"ast.boolop` + "ast.And" -> Python.ASTAnd(pyObject) + "ast.Or" -> Python.ASTOr(pyObject) + + // `"ast.cmpop` + "ast.Eq" -> Python.ASTEq(pyObject) + "ast.NotEq" -> Python.ASTNotEq(pyObject) + "ast.Lt" -> Python.ASTLt(pyObject) + "ast.LtE" -> Python.ASTLtE(pyObject) + "ast.Gt" -> Python.ASTGt(pyObject) + "ast.GtE" -> Python.ASTGtE(pyObject) + "ast.Is" -> Python.ASTIs(pyObject) + "ast.IsNot" -> Python.ASTIsNot(pyObject) + "ast.In" -> Python.ASTIn(pyObject) + "ast.NotInt" -> Python.ASTNotIn(pyObject) + + // `"ast.expr_context` + "ast.Load" -> Python.ASTLoad(pyObject) + "ast.Store" -> Python.ASTStore(pyObject) + "ast.Del" -> Python.ASTDel(pyObject) + + // `"ast.operator` + "ast.Add" -> Python.ASTAdd(pyObject) + "ast.Sub" -> Python.ASTSub(pyObject) + "ast.Mult" -> Python.ASTMult(pyObject) + "ast.MatMult" -> Python.ASTMatMult(pyObject) + "ast.Div" -> Python.ASTDiv(pyObject) + "ast.Mod" -> Python.ASTMod(pyObject) + "ast.Pow" -> Python.ASTPow(pyObject) + "ast.LShift" -> Python.ASTLShift(pyObject) + "ast.RShift" -> Python.ASTRShift(pyObject) + "ast.BitOr" -> Python.ASTBitOr(pyObject) + "ast.BitXor" -> Python.ASTBitXor(pyObject) + "ast.BitAnd" -> Python.ASTBitAnd(pyObject) + "ast.FloorDiv" -> Python.ASTFloorDiv(pyObject) + + // `"ast.pattern` + "ast.MatchValue" -> Python.ASTMatchValue(pyObject) + "ast.MatchSingleton" -> Python.ASTMatchSingleton(pyObject) + "ast.MatchSequence" -> Python.ASTMatchSequence(pyObject) + "ast.MatchMapping" -> Python.ASTMatchMapping(pyObject) + "ast.MatchClass" -> Python.ASTMatchClass(pyObject) + "ast.MatchStar" -> Python.ASTMatchStar(pyObject) + "ast.MatchAs" -> Python.ASTMatchAs(pyObject) + "ast.MatchOr" -> Python.ASTMatchOr(pyObject) + + // `"ast.unaryop` + "ast.Invert" -> Python.ASTInvert(pyObject) + "ast.Not" -> Python.ASTNot(pyObject) + "ast.UAdd" -> Python.ASTUAdd(pyObject) + "ast.USub" -> Python.ASTUSub(pyObject) // misc - "" -> Python.ASTalias(pyObject) - "" -> Python.ASTarg(pyObject) - "" -> Python.ASTarguments(pyObject) - "" -> Python.ASTcomprehension(pyObject) - "" -> Python.ASTexcepthandler(pyObject) - "" -> Python.ASTkeyword(pyObject) - "" -> Python.ASTmatch_case(pyObject) - "" -> Python.ASTtype_ignore(pyObject) - "" -> Python.ASTwithitem(pyObject) + "ast.alias" -> Python.ASTalias(pyObject) + "ast.arg" -> Python.ASTarg(pyObject) + "ast.arguments" -> Python.ASTarguments(pyObject) + "ast.comprehension" -> Python.ASTcomprehension(pyObject) + "ast.excepthandler" -> Python.ASTexcepthandler(pyObject) + "ast.keyword" -> Python.ASTkeyword(pyObject) + "ast.match_case" -> Python.ASTmatch_case(pyObject) + "ast.type_ignore" -> Python.ASTtype_ignore(pyObject) + "ast.withitem" -> Python.ASTwithitem(pyObject) // complex numbers - "" -> TODO() + "complex" -> TODO() else -> { TODO("Implement for ${pyObject.getAttr("__class__")}") } diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt index b871376e90..1e2ddb9be6 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt @@ -77,7 +77,7 @@ class StatementHandler(frontend: PythonLanguageFrontend) : private fun handleWhile(node: Python.ASTWhile): Statement { val ret = newWhileStatement(rawNode = node) ret.condition = frontend.expressionHandler.handle(node.test) - ret.statement = makeBlock(node.body) + ret.statement = makeBlock(node.body).codeAndLocationFromChildren(node) node.orelse.firstOrNull()?.let { TODO("Not supported") } return ret } @@ -86,7 +86,7 @@ class StatementHandler(frontend: PythonLanguageFrontend) : val ret = newForEachStatement(rawNode = node) ret.iterable = frontend.expressionHandler.handle(node.iter) ret.variable = frontend.expressionHandler.handle(node.target) - ret.statement = makeBlock(node.body) + ret.statement = makeBlock(node.body).codeAndLocationFromChildren(node) node.orelse.firstOrNull()?.let { TODO("Not supported") } return ret } @@ -114,13 +114,13 @@ class StatementHandler(frontend: PythonLanguageFrontend) : ret.condition = frontend.expressionHandler.handle(node.test) ret.thenStatement = if (node.body.isNotEmpty()) { - makeBlock(node.body) + makeBlock(node.body).codeAndLocationFromChildren(node) } else { null } ret.elseStatement = if (node.orelse.isNotEmpty()) { - makeBlock(node.orelse) + makeBlock(node.orelse).codeAndLocationFromChildren(node) } else { null } @@ -330,7 +330,7 @@ class StatementHandler(frontend: PythonLanguageFrontend) : // END HANDLE ARGUMENTS if (s.body.isNotEmpty()) { - result.body = makeBlock(s.body) + result.body = makeBlock(s.body).codeAndLocationFromChildren(s) } frontend.scopeManager.leaveScope(result) diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt index da06608c93..a0e9c238f2 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt @@ -726,13 +726,13 @@ class PythonFrontendTest : BaseTest() { val p = tu.namespaces["literal"] assertNotNull(p) - assertEquals(Region(1, 0, 1, 8), (p.statements[0]).location?.region) - assertEquals(Region(1, 4, 1, 8), (p.variables["b"])?.firstAssignment?.location?.region) - assertEquals(Region(2, 0, 2, 6), (p.statements[1]).location?.region) - assertEquals(Region(3, 0, 3, 7), (p.statements[2]).location?.region) - assertEquals(Region(4, 0, 4, 10), (p.statements[3]).location?.region) - assertEquals(Region(5, 0, 5, 11), (p.statements[4]).location?.region) - assertEquals(Region(6, 0, 6, 8), (p.statements[5]).location?.region) + assertEquals(Region(1, 1, 1, 9), (p.statements[0]).location?.region) + assertEquals(Region(1, 5, 1, 9), (p.variables["b"])?.firstAssignment?.location?.region) + assertEquals(Region(2, 1, 2, 7), (p.statements[1]).location?.region) + assertEquals(Region(3, 1, 3, 8), (p.statements[2]).location?.region) + assertEquals(Region(4, 1, 4, 11), (p.statements[3]).location?.region) + assertEquals(Region(5, 1, 5, 12), (p.statements[4]).location?.region) + assertEquals(Region(6, 1, 6, 9), (p.statements[5]).location?.region) } @Test @@ -935,7 +935,6 @@ class PythonFrontendTest : BaseTest() { } @Test - @Ignore // TODO fun testCommentMatching() { val topLevel = Path.of("src", "test", "resources", "python") val tu = @@ -950,7 +949,7 @@ class PythonFrontendTest : BaseTest() { val commentedNodes = SubgraphWalker.flattenAST(tu).filter { it.comment != null } - assertEquals(10, commentedNodes.size) + assertEquals(9, commentedNodes.size) val functions = commentedNodes.filterIsInstance() assertEquals(1, functions.size) @@ -968,9 +967,10 @@ class PythonFrontendTest : BaseTest() { assertEquals("# a parameter", params.first { it.name.localName == "i" }.comment) assertEquals("# another parameter", params.first { it.name.localName == "j" }.comment) - val variable = commentedNodes.filterIsInstance() - assertEquals(1, variable.size) - assertEquals("# A comment", variable.first().comment) + val assignment = commentedNodes.filterIsInstance() + assertEquals(2, assignment.size) + assertEquals("# A comment# a number", assignment.first().comment) + assertEquals("# comment end", assignment.last().comment) val block = commentedNodes.filterIsInstance() assertEquals(1, block.size) @@ -980,14 +980,6 @@ class PythonFrontendTest : BaseTest() { assertEquals(2, kvs.size) assertEquals("# a entry", kvs.first { it.code?.contains("a") ?: false }.comment) assertEquals("# b entry", kvs.first { it.code?.contains("b") ?: false }.comment) - - val declStmts = commentedNodes.filterIsInstance() - assertEquals(2, declStmts.size) - assertEquals("# a number", declStmts.first { it.location?.region?.startLine == 3 }.comment) - assertEquals( - "# comment end", - declStmts.first { it.location?.region?.startLine == 18 }.comment - ) } @Test