From ebd0364f986a238045b4dfd7d948ed98e752e182 Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Fri, 24 Nov 2023 10:19:18 +0100 Subject: [PATCH] Performance improvements for CXX frontend (#1363) * Performance improvements for CXX frontend * Avoid using `rawSignature` directly when creating nodes (#1362) Previously, in the CXX frontend, we were supplying the `rawSignature` as code(override) to almost all functions that created nodes. The issue with that is, that this does not respect the `codeInNodes` config option, which might be turned on to save memory. This PR consequently uses the `rawNode` parameter, which forwards setting code and location to the language frontend, which DOES respect the `codeInNodes` option. * Added complexity and mermaid chart * More cleanup of old typedef remnants --- .../de/fraunhofer/aisec/cpg/ScopeManager.kt | 36 +-- .../de/fraunhofer/aisec/cpg/TypeManager.kt | 5 +- .../aisec/cpg/frontends/LanguageFrontend.kt | 4 - .../de/fraunhofer/aisec/cpg/graph/Node.kt | 7 - .../graph/declarations/FunctionDeclaration.kt | 15 ++ .../cpg/frontends/cxx/DeclarationHandler.kt | 9 +- .../cpg/frontends/cxx/DeclaratorHandler.kt | 33 ++- .../cpg/frontends/cxx/ExpressionHandler.kt | 76 +++--- .../cpg/frontends/cxx/InitializerHandler.kt | 4 +- .../cxx/ParameterDeclarationHandler.kt | 2 +- .../cpg/frontends/cxx/StatementHandler.kt | 36 +-- .../cpg/enhancements/types/TypedefTest.kt | 249 +++++++++++------- .../frontends/cxx/CXXLanguageFrontendTest.kt | 31 +-- .../src/test/resources/typedefs/typedefs.cpp | 2 +- 14 files changed, 275 insertions(+), 234 deletions(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt index 07755b54be..497e876e85 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt @@ -94,12 +94,6 @@ class ScopeManager : ScopeProvider { */ private val aliases = mutableMapOf>() - /** - * The language frontend tied to the scope manager. Can be used to implement language specific - * scope resolution or lookup. - */ - var lang: LanguageFrontend<*, *>? = null - /** True, if the scope manager is currently in a [BlockScope]. */ val isInBlock: Boolean get() = this.firstScopeOrNull { it is BlockScope } != null @@ -568,10 +562,7 @@ class ScopeManager : ScopeProvider { } } - /** - * Only used by the [de.fraunhofer.aisec.cpg.graph.TypeManager], adds typedefs to the current - * [ValueDeclarationScope]. - */ + /** Only used by the [TypeManager], adds typedefs to the current [ValueDeclarationScope]. */ fun addTypedef(typedef: TypedefDeclaration) { val scope = this.firstScopeIsInstanceOrNull() if (scope == null) { @@ -580,12 +571,6 @@ class ScopeManager : ScopeProvider { } scope.addTypedef(typedef) - - if (scope.astNode == null) { - lang?.currentTU?.addTypedef(typedef) - } else { - scope.astNode?.addTypedef(typedef) - } } private fun getCurrentTypedefs(searchScope: Scope?): Collection { @@ -878,6 +863,25 @@ class ScopeManager : ScopeProvider { list += Alias(from, to) } + fun typedefFor(alias: Type): Type? { + var current = currentScope + + // We need to build a path from the current scope to the top most one. This ensures us that + // a local definition overwrites / shadows one that was there on a higher scope. + while (current != null) { + if (current is ValueDeclarationScope) { + val decl = current.typedefs[alias] + if (decl != null) { + return decl.type + } + } + + current = current.parent + } + + return null + } + /** Returns the current scope for the [ScopeProvider] interface. */ override val scope: Scope? get() = currentScope diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TypeManager.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TypeManager.kt index dab639a511..276b50bc72 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TypeManager.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TypeManager.kt @@ -252,10 +252,7 @@ class TypeManager { fun resolvePossibleTypedef(alias: Type, scopeManager: ScopeManager): Type { val finalToCheck = alias.root - val applicable = - scopeManager.currentTypedefs - .firstOrNull { t: TypedefDeclaration -> t.alias.root == finalToCheck } - ?.type + val applicable = scopeManager.typedefFor(finalToCheck) return applicable ?: alias } } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/LanguageFrontend.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/LanguageFrontend.kt index 6ecb61fcf7..75965712a3 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/LanguageFrontend.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/LanguageFrontend.kt @@ -66,10 +66,6 @@ abstract class LanguageFrontend( val typeManager: TypeManager = ctx.typeManager val config: TranslationConfiguration = ctx.config - init { - this.scopeManager.lang = this - } - var currentTU: TranslationUnitDeclaration? = null @Throws(TranslationException::class) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt index 4466ada290..129f9e0392 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt @@ -34,7 +34,6 @@ import de.fraunhofer.aisec.cpg.frontends.Language import de.fraunhofer.aisec.cpg.graph.declarations.MethodDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.RecordDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration -import de.fraunhofer.aisec.cpg.graph.declarations.TypedefDeclaration import de.fraunhofer.aisec.cpg.graph.edge.* import de.fraunhofer.aisec.cpg.graph.edge.Properties import de.fraunhofer.aisec.cpg.graph.scopes.GlobalScope @@ -203,8 +202,6 @@ open class Node : IVisitable, Persistable, LanguageProvider, ScopeProvider /** Virtual property for accessing the parents of the Program Dependence Graph (PDG). */ var prevPDG: MutableSet by PropertyEdgeSetDelegate(Node::prevPDGEdges, false) - var typedefs: MutableSet = HashSet() - /** * If a node is marked as being inferred, it means that it was created artificially and does not * necessarily have a real counterpart in the scanned source code. However, the nodes @@ -333,10 +330,6 @@ open class Node : IVisitable, Persistable, LanguageProvider, ScopeProvider } } - fun addTypedef(typedef: TypedefDeclaration) { - typedefs.add(typedef) - } - fun addAnnotations(annotations: Collection) { this.annotations.addAll(annotations) } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.kt index b27d2dd259..61359c40ef 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.kt @@ -280,6 +280,21 @@ val Statement.cyclomaticComplexity: Int // add one for each branch (and include the children) stmt.caseExpression?.let { i += it.cyclomaticComplexity } } + is DoStatement -> { + // add one for the do statement (and include the children) + i += (stmt.statement?.cyclomaticComplexity ?: 0) + 1 + } + is WhileStatement -> { + // add one for the while statement (and include the children) + i += (stmt.statement?.cyclomaticComplexity ?: 0) + 1 + } + is GotoStatement -> { + // add one + i++ + } + is StatementHolder -> { + i += stmt.cyclomaticComplexity + } } } diff --git a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/DeclarationHandler.kt b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/DeclarationHandler.kt index bdb686ca4b..6e54810e29 100644 --- a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/DeclarationHandler.kt +++ b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/DeclarationHandler.kt @@ -81,7 +81,7 @@ class DeclarationHandler(lang: CXXLanguageFrontend) : * done yet. */ private fun handleUsingDirective(using: CPPASTUsingDirective): Declaration { - return newUsingDeclaration(using.rawSignature, using.qualifiedName.toString()) + return newUsingDeclaration(qualifiedName = using.qualifiedName.toString(), rawNode = using) } /** @@ -704,11 +704,7 @@ class DeclarationHandler(lang: CXXLanguageFrontend) : fun handleTranslationUnit(translationUnit: IASTTranslationUnit): TranslationUnitDeclaration { val node = - newTranslationUnitDeclaration( - translationUnit.filePath, - translationUnit.rawSignature, - translationUnit - ) + newTranslationUnitDeclaration(translationUnit.filePath, rawNode = translationUnit) // There might have been errors in the previous translation unit and in any case // we need to reset the scope manager scope to global, to avoid spilling scope errors into @@ -716,6 +712,7 @@ class DeclarationHandler(lang: CXXLanguageFrontend) : frontend.scopeManager.resetToGlobal(node) frontend.currentTU = node val problematicIncludes = HashMap>() + for (declaration in translationUnit.declarations) { if (declaration is CPPASTLinkageSpecification) { continue // do not care about these for now diff --git a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/DeclaratorHandler.kt b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/DeclaratorHandler.kt index e66c277079..9d1335c176 100644 --- a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/DeclaratorHandler.kt +++ b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/DeclaratorHandler.kt @@ -117,8 +117,8 @@ class DeclaratorHandler(lang: CXXLanguageFrontend) : ctx.name.toString(), unknownType(), // Type will be filled out later by // handleSimpleDeclaration - ctx.rawSignature, - implicitInitializerAllowed, + implicitInitializerAllowed = implicitInitializerAllowed, + rawNode = ctx ) // Add this declaration to the current scope @@ -142,10 +142,10 @@ class DeclaratorHandler(lang: CXXLanguageFrontend) : name.localName, unknownType(), emptyList(), - ctx.rawSignature, - frontend.locationOf(ctx), - initializer, - true + location = frontend.locationOf(ctx), + initializer = initializer, + implicitInitializerAllowed = true, + rawNode = ctx ) frontend.scopeManager.addDeclaration(declaration) @@ -401,19 +401,24 @@ class DeclaratorHandler(lang: CXXLanguageFrontend) : val recordDeclaration = frontend.scopeManager.currentRecord if (recordDeclaration == null) { // variable - result = newVariableDeclaration(name, unknownType(), ctx.rawSignature, true) + result = + newVariableDeclaration( + name, + unknownType(), + implicitInitializerAllowed = true, + rawNode = ctx + ) } else { // field - val code = ctx.rawSignature result = newFieldDeclaration( name, unknownType(), emptyList(), - code, - frontend.locationOf(ctx), - null, - false, + location = frontend.locationOf(ctx), + initializer = null, + implicitInitializerAllowed = false, + rawNode = ctx ) } @@ -435,7 +440,7 @@ class DeclaratorHandler(lang: CXXLanguageFrontend) : newRecordDeclaration( ctx.name.toString(), kind, - ctx.rawSignature, + rawNode = ctx, ) // Handle C++ classes @@ -483,7 +488,7 @@ class DeclaratorHandler(lang: CXXLanguageFrontend) : private fun handleTemplateTypeParameter( ctx: CPPASTSimpleTypeTemplateParameter ): TypeParameterDeclaration { - return newTypeParameterDeclaration(ctx.rawSignature, ctx.rawSignature, ctx) + return newTypeParameterDeclaration(ctx.rawSignature, rawNode = ctx) } private fun processMembers(ctx: IASTCompositeTypeSpecifier) { diff --git a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/ExpressionHandler.kt b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/ExpressionHandler.kt index 06f5b1ae2f..2af62aaa41 100644 --- a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/ExpressionHandler.kt +++ b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/ExpressionHandler.kt @@ -161,18 +161,17 @@ class ExpressionHandler(lang: CXXLanguageFrontend) : } val referencedType = frontend.typeOf(ctx.typeId) - return newTypeIdExpression(operatorCode, type, referencedType, ctx.rawSignature) + return newTypeIdExpression(operatorCode, type, referencedType, rawNode = ctx) } private fun handleArraySubscriptExpression(ctx: IASTArraySubscriptExpression): Expression { - val arraySubsExpression = newSubscriptExpression(ctx.rawSignature) + val arraySubsExpression = newSubscriptExpression(rawNode = ctx) handle(ctx.arrayExpression)?.let { arraySubsExpression.arrayExpression = it } handle(ctx.argument)?.let { arraySubsExpression.subscriptExpression = it } return arraySubsExpression } private fun handleNewExpression(ctx: CPPASTNewExpression): Expression { - val code = ctx.rawSignature val t = frontend.typeOf(ctx.typeId) val init = ctx.initializer @@ -180,7 +179,7 @@ class ExpressionHandler(lang: CXXLanguageFrontend) : return if (ctx.isArrayAllocation) { t.array() val arrayMods = (ctx.typeId.abstractDeclarator as IASTArrayDeclarator).arrayModifiers - val arrayCreate = newNewArrayExpression(code) + val arrayCreate = newNewArrayExpression(rawNode = ctx) arrayCreate.type = t for (arrayMod in arrayMods) { val constant = handle(arrayMod.constantExpression) @@ -192,7 +191,7 @@ class ExpressionHandler(lang: CXXLanguageFrontend) : arrayCreate } else { // new returns a pointer, so we need to reference the type by pointer - val newExpression = newNewExpression(code, t.pointer(), ctx) + val newExpression = newNewExpression(type = t.pointer(), rawNode = ctx) val declSpecifier = ctx.typeId.declSpecifier as? IASTNamedTypeSpecifier // Resolve possible templates if (declSpecifier?.name is CPPASTTemplateId) { @@ -270,7 +269,7 @@ class ExpressionHandler(lang: CXXLanguageFrontend) : } private fun handleDeleteExpression(ctx: CPPASTDeleteExpression): DeleteExpression { - val deleteExpression = newDeleteExpression(ctx.rawSignature) + val deleteExpression = newDeleteExpression(rawNode = ctx) for (name in ctx.implicitDestructorNames) { log.debug("Implicit constructor name {}", name) } @@ -279,7 +278,7 @@ class ExpressionHandler(lang: CXXLanguageFrontend) : } private fun handleCastExpression(ctx: IASTCastExpression): Expression { - val castExpression = newCastExpression(ctx.rawSignature) + val castExpression = newCastExpression(rawNode = ctx) castExpression.expression = handle(ctx.operand) ?: ProblemExpression("could not parse inner expression") castExpression.setCastOperator(ctx.operator) @@ -314,7 +313,7 @@ class ExpressionHandler(lang: CXXLanguageFrontend) : base, unknownType(), if (ctx.isPointerDereference) "->" else ".", - ctx.rawSignature + rawNode = ctx ) } @@ -371,7 +370,7 @@ class ExpressionHandler(lang: CXXLanguageFrontend) : operatorCode, ctx.isPostfixOperator, !ctx.isPostfixOperator, - ctx.rawSignature + rawNode = ctx ) if (input != null) { unaryOperator.input = input @@ -386,7 +385,7 @@ class ExpressionHandler(lang: CXXLanguageFrontend) : reference is MemberExpression -> { val baseType = reference.base.type.root assert(baseType !is SecondOrderType) - callExpression = newMemberCallExpression(reference, code = ctx.rawSignature) + callExpression = newMemberCallExpression(reference, rawNode = ctx) if ( (ctx.functionNameExpression as? IASTFieldReference)?.fieldName is CPPASTTemplateId @@ -408,10 +407,10 @@ class ExpressionHandler(lang: CXXLanguageFrontend) : reference is BinaryOperator && (reference.operatorCode == ".*" || reference.operatorCode == "->*") -> { // This is a function pointer call to a class method. We keep this as a binary - // operator - // with the .* or ->* operator code, so that we can resolve this later in the + // operator with the .* or ->* operator code, so that we can resolve this later in + // the // FunctionPointerCallResolver - callExpression = newMemberCallExpression(reference, code = ctx.rawSignature) + callExpression = newMemberCallExpression(reference, rawNode = ctx) } reference is UnaryOperator && reference.operatorCode == "*" -> { // Classic C-style function pointer call -> let's extract the target @@ -424,7 +423,7 @@ class ExpressionHandler(lang: CXXLanguageFrontend) : .templateName .toString() val ref = newReference(name) - callExpression = newCallExpression(ref, name, ctx.rawSignature, true) + callExpression = newCallExpression(ref, name, template = true, rawNode = ctx) getTemplateArguments( (ctx.functionNameExpression as IASTIdExpression).name as CPPASTTemplateId ) @@ -440,7 +439,7 @@ class ExpressionHandler(lang: CXXLanguageFrontend) : } else -> { callExpression = - newCallExpression(reference, reference?.name, ctx.rawSignature, false) + newCallExpression(reference, reference?.name, template = false, rawNode = ctx) } } @@ -462,11 +461,11 @@ class ExpressionHandler(lang: CXXLanguageFrontend) : // this expression could actually be a field / member expression, but somehow CDT only // recognizes them as a member expression if it has an explicit 'this' // TODO: handle this? convert the declared reference expression into a member expression? - return newReference(ctx.name.toString(), unknownType(), ctx.rawSignature) + return newReference(ctx.name.toString(), unknownType(), rawNode = ctx) } private fun handleExpressionList(exprList: IASTExpressionList): ExpressionList { - val expressionList = newExpressionList(exprList.rawSignature) + val expressionList = newExpressionList(rawNode = exprList) for (expr in exprList.expressions) { handle(expr)?.let { expressionList.addExpression(it) } } @@ -494,7 +493,7 @@ class ExpressionHandler(lang: CXXLanguageFrontend) : else -> String(ASTStringUtil.getBinaryOperatorString(ctx)) } - val binaryOperator = newBinaryOperator(operatorCode, ctx.rawSignature) + val binaryOperator = newBinaryOperator(operatorCode, rawNode = ctx) val lhs = handle(ctx.operand1) ?: newProblemExpression("could not parse lhs") val rhs = if (ctx.operand2 != null) { @@ -506,13 +505,7 @@ class ExpressionHandler(lang: CXXLanguageFrontend) : if (lhs is CastExpression && frontend.config.inferenceConfiguration.guessCastExpressions) { // this really is a combination of a cast and a unary operator - val op = - newUnaryOperator( - operatorCode, - postfix = true, - prefix = false, - code = ctx.rawSignature - ) + val op = newUnaryOperator(operatorCode, postfix = true, prefix = false, rawNode = ctx) op.input = rhs op.location = frontend.locationOf(ctx.operand2) lhs.expression = op @@ -548,18 +541,18 @@ class ExpressionHandler(lang: CXXLanguageFrontend) : return when (ctx.kind) { lk_integer_constant -> handleIntegerLiteral(ctx) lk_float_constant -> handleFloatLiteral(ctx) - lk_char_constant -> newLiteral(ctx.value[1], primitiveType("char"), ctx.rawSignature) + lk_char_constant -> newLiteral(ctx.value[1], primitiveType("char"), rawNode = ctx) lk_string_literal -> newLiteral( String(ctx.value.slice(IntRange(1, ctx.value.size - 2)).toCharArray()), primitiveType("char").array(), - ctx.rawSignature + rawNode = ctx ) lk_this -> handleThisLiteral(ctx) - lk_true -> newLiteral(true, primitiveType("bool"), ctx.rawSignature) - lk_false -> newLiteral(false, primitiveType("bool"), ctx.rawSignature) - lk_nullptr -> newLiteral(null, objectType("nullptr_t"), ctx.rawSignature) - else -> newLiteral(String(ctx.value), unknownType(), ctx.rawSignature) + lk_true -> newLiteral(true, primitiveType("bool"), rawNode = ctx) + lk_false -> newLiteral(false, primitiveType("bool"), rawNode = ctx) + lk_nullptr -> newLiteral(null, objectType("nullptr_t"), rawNode = ctx) + else -> newLiteral(String(ctx.value), unknownType(), rawNode = ctx) } } @@ -578,8 +571,7 @@ class ExpressionHandler(lang: CXXLanguageFrontend) : oneLhs = handle(des.subscriptExpression) } is CPPASTFieldDesignator -> { - oneLhs = - newReference(des.name.toString(), unknownType(), des.getRawSignature()) + oneLhs = newReference(des.name.toString(), unknownType(), rawNode = des) } is CPPASTArrayRangeDesignator -> { oneLhs = @@ -606,7 +598,7 @@ class ExpressionHandler(lang: CXXLanguageFrontend) : } } - val die = newDesignatedInitializerExpression(ctx.rawSignature) + val die = newDesignatedInitializerExpression(rawNode = ctx) die.lhs = lhs die.rhs = rhs @@ -628,8 +620,7 @@ class ExpressionHandler(lang: CXXLanguageFrontend) : oneLhs = handle(des.subscriptExpression) } is CPPASTFieldDesignator -> { - oneLhs = - newReference(des.name.toString(), unknownType(), des.getRawSignature()) + oneLhs = newReference(des.name.toString(), unknownType(), rawNode = des) } is CPPASTArrayRangeDesignator -> { oneLhs = @@ -656,7 +647,7 @@ class ExpressionHandler(lang: CXXLanguageFrontend) : } } - val die = newDesignatedInitializerExpression(ctx.rawSignature) + val die = newDesignatedInitializerExpression(rawNode = ctx) die.lhs = lhs die.rhs = rhs @@ -754,7 +745,7 @@ class ExpressionHandler(lang: CXXLanguageFrontend) : } ) - return newLiteral(numberValue, type, ctx.rawSignature) + return newLiteral(numberValue, type, rawNode = ctx) } catch (ex: NumberFormatException) { // It could be that we cannot parse the literal, in this case we return an error return ProblemExpression("could not parse literal: ${ex.message}") @@ -766,15 +757,14 @@ class ExpressionHandler(lang: CXXLanguageFrontend) : return try { when (suffix) { - "f" -> newLiteral(strippedValue.toFloat(), primitiveType("float"), ctx.rawSignature) + "f" -> newLiteral(strippedValue.toFloat(), primitiveType("float"), rawNode = ctx) "l" -> newLiteral( strippedValue.toBigDecimal(), primitiveType("long double"), - ctx.rawSignature + rawNode = ctx ) - else -> - newLiteral(strippedValue.toDouble(), primitiveType("double"), ctx.rawSignature) + else -> newLiteral(strippedValue.toDouble(), primitiveType("double"), rawNode = ctx) } } catch (ex: NumberFormatException) { // It could be that we cannot parse the literal, in this case we return an error @@ -794,7 +784,7 @@ class ExpressionHandler(lang: CXXLanguageFrontend) : // We do want to make sure that the type of the expression is at least a pointer. val pointerType = recordType.pointer() - return newReference("this", pointerType, ctx.rawSignature, ctx) + return newReference("this", pointerType, rawNode = ctx) } private val IASTLiteralExpression.valueWithSuffix: Pair diff --git a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/InitializerHandler.kt b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/InitializerHandler.kt index dd7be1da52..d2c4cdcb8f 100644 --- a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/InitializerHandler.kt +++ b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/InitializerHandler.kt @@ -57,7 +57,7 @@ class InitializerHandler(lang: CXXLanguageFrontend) : } private fun handleConstructorInitializer(ctx: CPPASTConstructorInitializer): Expression { - val constructExpression = newConstructExpression(ctx.rawSignature) + val constructExpression = newConstructExpression(rawNode = ctx) constructExpression.type = (frontend.declaratorHandler.lastNode as? VariableDeclaration)?.type ?: unknownType() @@ -79,7 +79,7 @@ class InitializerHandler(lang: CXXLanguageFrontend) : val targetType = (frontend.declaratorHandler.lastNode as? ValueDeclaration)?.type ?: unknownType() - val expression = newInitializerListExpression(targetType, ctx.rawSignature) + val expression = newInitializerListExpression(targetType, rawNode = ctx) for (clause in ctx.clauses) { frontend.expressionHandler.handle(clause)?.let { diff --git a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/ParameterDeclarationHandler.kt b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/ParameterDeclarationHandler.kt index 79e0c240f2..536e094743 100644 --- a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/ParameterDeclarationHandler.kt +++ b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/ParameterDeclarationHandler.kt @@ -52,7 +52,7 @@ class ParameterDeclarationHandler(lang: CXXLanguageFrontend) : val type = frontend.typeOf(ctx.declarator, ctx.declSpecifier) val paramVariableDeclaration = - newParameterDeclaration(ctx.declarator.name.toString(), type, false, ctx.rawSignature) + newParameterDeclaration(ctx.declarator.name.toString(), type, false, rawNode = ctx) // Add default values if (ctx.declarator.initializer != null) { diff --git a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/StatementHandler.kt b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/StatementHandler.kt index dc759a1f01..b98a0c3d9f 100644 --- a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/StatementHandler.kt +++ b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/StatementHandler.kt @@ -88,7 +88,7 @@ class StatementHandler(lang: CXXLanguageFrontend) : } private fun handleEmptyStatement(nullStatement: IASTNullStatement): EmptyStatement { - return newEmptyStatement(nullStatement.rawSignature) + return newEmptyStatement(rawNode = nullStatement) } private fun handleTryBlockStatement(tryBlockStatement: CPPASTTryBlockStatement): TryStatement { @@ -106,7 +106,7 @@ class StatementHandler(lang: CXXLanguageFrontend) : } private fun handleCatchHandler(catchHandler: ICPPASTCatchHandler): CatchClause { - val catchClause = newCatchClause(catchHandler.rawSignature) + val catchClause = newCatchClause(rawNode = catchHandler) frontend.scopeManager.enterScope(catchClause) val body = frontend.statementHandler.handle(catchHandler.catchBody) @@ -127,7 +127,7 @@ class StatementHandler(lang: CXXLanguageFrontend) : } private fun handleIfStatement(ctx: IASTIfStatement): IfStatement { - val statement = newIfStatement(ctx.rawSignature) + val statement = newIfStatement(rawNode = ctx) frontend.scopeManager.enterScope(statement) @@ -157,14 +157,14 @@ class StatementHandler(lang: CXXLanguageFrontend) : } private fun handleLabelStatement(ctx: IASTLabelStatement): LabelStatement { - val statement = newLabelStatement(ctx.rawSignature) + val statement = newLabelStatement(rawNode = ctx) statement.subStatement = handle(ctx.nestedStatement) statement.label = ctx.name.toString() return statement } private fun handleGotoStatement(ctx: IASTGotoStatement): GotoStatement { - val statement = newGotoStatement(ctx.rawSignature) + val statement = newGotoStatement(rawNode = ctx) val assigneeTargetLabel = BiConsumer { _: Any, to: Node -> statement.targetLabel = to as LabelStatement } @@ -189,7 +189,7 @@ class StatementHandler(lang: CXXLanguageFrontend) : } private fun handleWhileStatement(ctx: IASTWhileStatement): WhileStatement { - val statement = newWhileStatement(ctx.rawSignature) + val statement = newWhileStatement(rawNode = ctx) frontend.scopeManager.enterScope(statement) @@ -211,7 +211,7 @@ class StatementHandler(lang: CXXLanguageFrontend) : } private fun handleDoStatement(ctx: IASTDoStatement): DoStatement { - val statement = newDoStatement(ctx.rawSignature) + val statement = newDoStatement(rawNode = ctx) frontend.scopeManager.enterScope(statement) statement.condition = frontend.expressionHandler.handle(ctx.condition) statement.statement = handle(ctx.body) @@ -220,7 +220,7 @@ class StatementHandler(lang: CXXLanguageFrontend) : } private fun handleForStatement(ctx: IASTForStatement): ForStatement { - val statement = newForStatement(ctx.rawSignature) + val statement = newForStatement(rawNode = ctx) frontend.scopeManager.enterScope(statement) @@ -256,7 +256,7 @@ class StatementHandler(lang: CXXLanguageFrontend) : } private fun handleForEachStatement(ctx: CPPASTRangeBasedForStatement): ForEachStatement { - val statement = newForEachStatement(ctx.rawSignature) + val statement = newForEachStatement(rawNode = ctx) frontend.scopeManager.enterScope(statement) val decl = frontend.declarationHandler.handle(ctx.declaration) val `var` = newDeclarationStatement(decl?.code) @@ -270,12 +270,12 @@ class StatementHandler(lang: CXXLanguageFrontend) : } private fun handleBreakStatement(ctx: IASTBreakStatement): BreakStatement { - return newBreakStatement(ctx.rawSignature) + return newBreakStatement(rawNode = ctx) // C++ has no labeled break } private fun handleContinueStatement(ctx: IASTContinueStatement): ContinueStatement { - return newContinueStatement(ctx.rawSignature) + return newContinueStatement(rawNode = ctx) // C++ has no labeled continue } @@ -292,9 +292,9 @@ class StatementHandler(lang: CXXLanguageFrontend) : private fun handleDeclarationStatement(ctx: IASTDeclarationStatement): DeclarationStatement { return if (ctx.declaration is IASTASMDeclaration) { - newASMDeclarationStatement(ctx.rawSignature) + newASMDeclarationStatement(rawNode = ctx) } else { - val declarationStatement = newDeclarationStatement(ctx.rawSignature) + val declarationStatement = newDeclarationStatement(rawNode = ctx) val declaration = frontend.declarationHandler.handle(ctx.declaration) if (declaration is DeclarationSequence) { declarationStatement.declarations = declaration.asList() @@ -306,7 +306,7 @@ class StatementHandler(lang: CXXLanguageFrontend) : } private fun handleReturnStatement(ctx: IASTReturnStatement): ReturnStatement { - val returnStatement = newReturnStatement(ctx.rawSignature) + val returnStatement = newReturnStatement(rawNode = ctx) // Parse the return value if (ctx.returnValue != null) { @@ -317,7 +317,7 @@ class StatementHandler(lang: CXXLanguageFrontend) : } private fun handleCompoundStatement(ctx: IASTCompoundStatement): Block { - val block = newBlock(ctx.rawSignature) + val block = newBlock(rawNode = ctx) frontend.scopeManager.enterScope(block) @@ -334,7 +334,7 @@ class StatementHandler(lang: CXXLanguageFrontend) : } private fun handleSwitchStatement(ctx: IASTSwitchStatement): SwitchStatement { - val switchStatement = newSwitchStatement(ctx.rawSignature) + val switchStatement = newSwitchStatement(rawNode = ctx) frontend.scopeManager.enterScope(switchStatement) @@ -361,12 +361,12 @@ class StatementHandler(lang: CXXLanguageFrontend) : } private fun handleCaseStatement(ctx: IASTCaseStatement): CaseStatement { - val caseStatement = newCaseStatement(ctx.rawSignature) + val caseStatement = newCaseStatement(rawNode = ctx) caseStatement.caseExpression = frontend.expressionHandler.handle(ctx.expression) return caseStatement } private fun handleDefaultStatement(ctx: IASTDefaultStatement): DefaultStatement { - return newDefaultStatement(ctx.rawSignature) + return newDefaultStatement(rawNode = ctx) } } diff --git a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/types/TypedefTest.kt b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/types/TypedefTest.kt index 6416d95136..e98a7c0eee 100644 --- a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/types/TypedefTest.kt +++ b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/types/TypedefTest.kt @@ -26,14 +26,17 @@ package de.fraunhofer.aisec.cpg.enhancements.types import de.fraunhofer.aisec.cpg.BaseTest -import de.fraunhofer.aisec.cpg.TestUtils.analyze +import de.fraunhofer.aisec.cpg.TestUtils.analyzeAndGetFirstTU import de.fraunhofer.aisec.cpg.TestUtils.findByUniqueName -import de.fraunhofer.aisec.cpg.TestUtils.findByUniquePredicate +import de.fraunhofer.aisec.cpg.assertLocalName import de.fraunhofer.aisec.cpg.frontends.cxx.CPPLanguage +import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.ValueDeclaration -import de.fraunhofer.aisec.cpg.graph.records +import de.fraunhofer.aisec.cpg.graph.objectType import de.fraunhofer.aisec.cpg.graph.types.FunctionPointerType +import de.fraunhofer.aisec.cpg.graph.types.IntegerType import de.fraunhofer.aisec.cpg.graph.types.NumericType +import de.fraunhofer.aisec.cpg.graph.types.ObjectType import de.fraunhofer.aisec.cpg.graph.variables import java.nio.file.Path import kotlin.test.* @@ -44,141 +47,185 @@ internal class TypedefTest : BaseTest() { @Test @Throws(Exception::class) fun testSingle() { - val result = analyze("cpp", topLevel, true) { it.registerLanguage() } - val variables = result.variables - - // normal type - val l1 = findByUniqueName(variables, "l1") - val l2 = findByUniqueName(variables, "l2") - assertEquals(l1.type, l2.type) - - // pointer - val longptr1 = findByUniqueName(variables, "longptr1") - val longptr2 = findByUniqueName(variables, "longptr2") - assertEquals(longptr1.type, longptr2.type) - - // array - val arr1 = findByUniqueName(variables, "arr1") - val arr2 = findByUniqueName(variables, "arr2") - assertEquals(arr1.type, arr2.type) - - // function pointer - val uintfp1 = findByUniqueName(variables, "uintfp1") - val uintfp2 = findByUniqueName(variables, "uintfp2") - - val fpType = uintfp1.type as? FunctionPointerType - assertNotNull(fpType) - - val returnType = fpType.returnType as? NumericType - assertNotNull(returnType) - assertEquals(NumericType.Modifier.UNSIGNED, returnType.modifier) - assertEquals(uintfp1.type, uintfp2.type) - - val typedefs = result.finalCtx.scopeManager.currentTypedefs - val def = - typedefs.stream().filter { it.alias.name.localName == "test" }.findAny().orElse(null) - assertNotNull(def) + val tu = + analyzeAndGetFirstTU( + listOf(topLevel.resolve("typedefs.cpp").toFile()), + topLevel, + true + ) { + it.registerLanguage() + } + with(tu) { + // normal type + val l1 = tu.variables["l1"] + val l2 = tu.variables["l2"] + assertEquals(l1?.type, l2?.type) + + // pointer + val longptr1 = tu.variables["longptr1"] + val longptr2 = tu.variables["longptr2"] + assertEquals(longptr1?.type, longptr2?.type) + + // array + val arr1 = tu.variables["arr1"] + val arr2 = tu.variables["arr2"] + assertEquals(arr1?.type, arr2?.type) + + // function pointer + val uintfp1 = tu.variables["uintfp1"] + val uintfp2 = tu.variables["uintfp2"] + + val fpType = uintfp1?.type as? FunctionPointerType + assertNotNull(fpType) + + val returnType = fpType.returnType as? NumericType + assertNotNull(returnType) + assertEquals(NumericType.Modifier.UNSIGNED, returnType.modifier) + assertEquals(uintfp1.type, uintfp2?.type) + + val type = tu.ctx?.scopeManager?.typedefFor(objectType("test")) + assertIs(type) + assertLocalName("uint8_t", type) + } } @Test @Throws(Exception::class) fun testWithModifier() { - val result = analyze("cpp", topLevel, true) { it.registerLanguage() } - val variables = result.variables + val tu = + analyzeAndGetFirstTU( + listOf(topLevel.resolve("typedefs.cpp").toFile()), + topLevel, + true + ) { + it.registerLanguage() + } // pointer - val l1ptr = findByUniqueName(variables, "l1ptr") - val l2ptr = findByUniqueName(variables, "l2ptr") - val l3ptr = findByUniqueName(variables, "l3ptr") - val l4ptr = findByUniqueName(variables, "l4ptr") - assertEquals(l1ptr.type, l2ptr.type) - assertEquals(l1ptr.type, l3ptr.type) - assertEquals(l1ptr.type, l4ptr.type) + val l1ptr = tu.variables["l1ptr"] + val l2ptr = tu.variables["l2ptr"] + val l3ptr = tu.variables["l3ptr"] + val l4ptr = tu.variables["l4ptr"] + assertEquals(l1ptr?.type, l2ptr?.type) + assertEquals(l1ptr?.type, l3ptr?.type) + assertEquals(l1ptr?.type, l4ptr?.type) } @Test @Throws(Exception::class) fun testChained() { - val result = analyze("cpp", topLevel, true) { it.registerLanguage() } - val variables = result.variables - val l1 = findByUniqueName(variables, "l1") - val l3 = findByUniqueName(variables, "l3") - val l4 = findByUniqueName(variables, "l4") - assertEquals(l1.type, l3.type) - assertEquals(l1.type, l4.type) + val tu = + analyzeAndGetFirstTU( + listOf(topLevel.resolve("typedefs.cpp").toFile()), + topLevel, + true + ) { + it.registerLanguage() + } + + val l1 = tu.variables["l1"] + val l3 = tu.variables["l3"] + val l4 = tu.variables["l4"] + assertEquals(l1?.type, l3?.type) + assertEquals(l1?.type, l4?.type) } @Test @Throws(Exception::class) fun testMultiple() { - val result = analyze("cpp", topLevel, true) { it.registerLanguage() } - val variables = result.variables - - // simple type - val i1 = findByUniqueName(variables, "i1") - val i2 = findByUniqueName(variables, "i2") - assertEquals(i1.type, i2.type) - - // array - val a1 = findByUniqueName(variables, "a1") - val a2 = findByUniqueName(variables, "a2") - assertEquals(a1.type, a2.type) - - // pointer - val intPtr1 = findByUniqueName(variables, "intPtr1") - val intPtr2 = findByUniqueName(variables, "intPtr2") - assertEquals(intPtr1.type, intPtr2.type) - - // function pointer - val fPtr1 = findByUniqueName(variables, "intFptr1") - val fPtr2 = findByUniqueName(variables, "intFptr2") - assertEquals(fPtr1.type, fPtr2.type) - - // template, not to be confused with multiple typedef - val template = - findByUniquePredicate(result.translationUnits.firstOrNull()?.typedefs ?: listOf()) { - it.type.typeName == "template_class_A" + val tu = + analyzeAndGetFirstTU( + listOf(topLevel.resolve("typedefs.cpp").toFile()), + topLevel, + true + ) { + it.registerLanguage() } - assertEquals(template.alias.typeName, "type_B") + with(tu) { + // simple type + val i1 = tu.variables["i1"] + val i2 = tu.variables["i2"] + assertEquals(i1?.type, i2?.type) + + // array + val a1 = tu.variables["a1"] + val a2 = tu.variables["a2"] + assertEquals(a1?.type, a2?.type) + + // pointer + val intPtr1 = tu.variables["intPtr1"] + val intPtr2 = tu.variables["intPtr2"] + assertEquals(intPtr1?.type, intPtr2?.type) + + // function pointer + val fPtr1 = tu.variables["intFptr1"] + val fPtr2 = tu.variables["intFptr2"] + assertEquals(fPtr1?.type, fPtr2?.type) + + val type = tu.ctx?.scopeManager?.typedefFor(objectType("type_B")) + assertLocalName("template_class_A", type) + assertIs(type) + assertEquals(listOf(primitiveType("int"), primitiveType("int")), type.generics) + } } @Test @Throws(Exception::class) fun testStructs() { - val result = analyze("cpp", topLevel, true) { it.registerLanguage() } - val variables = result.variables - val ps1 = findByUniqueName(variables, "ps1") - val ps2 = findByUniqueName(variables, "ps2") - assertEquals(ps1.type, ps2.type) + val tu = + analyzeAndGetFirstTU( + listOf(topLevel.resolve("typedefs.cpp").toFile()), + topLevel, + true + ) { + it.registerLanguage() + } + + val ps1 = tu.variables["ps1"] + val ps2 = tu.variables["ps2"] + assertEquals(ps1?.type, ps2?.type) } @Test @Throws(Exception::class) fun testArbitraryTypedefLocation() { - val result = analyze("cpp", topLevel, true) { it.registerLanguage() } - val variables = result.variables - val ullong1 = findByUniqueName(variables, "someUllong1") - val ullong2 = findByUniqueName(variables, "someUllong2") - assertEquals(ullong1.type, ullong2.type) + val tu = + analyzeAndGetFirstTU( + listOf(topLevel.resolve("typedefs.cpp").toFile()), + topLevel, + true + ) { + it.registerLanguage() + } + + val ullong1 = tu.variables["someUllong1"] + val ullong2 = tu.variables["someUllong2"] + assertEquals(ullong1?.type, ullong2?.type) } @Test @Throws(Exception::class) fun testMemberTypeDef() { - val result = analyze("cpp", topLevel, true) { it.registerLanguage() } - val variables = result.variables - val records = result.records - val addConst = findByUniqueName(records, "add_const") + val tu = + analyzeAndGetFirstTU( + listOf(topLevel.resolve("typedefs.cpp").toFile()), + topLevel, + true + ) { + it.registerLanguage() + } + + val addConst = tu.records["add_const"] val typeMember1: ValueDeclaration = findByUniqueName(addConst.fields, "typeMember1") val typeMember2: ValueDeclaration = findByUniqueName(addConst.fields, "typeMember2") assertEquals(typeMember1.type, typeMember2.type) - val typeMemberOutside = findByUniqueName(variables, "typeMemberOutside") - assertNotEquals(typeMemberOutside.type, typeMember2.type) + val typeMemberOutside = tu.variables["typeMemberOutside"] + assertNotEquals(typeMemberOutside?.type, typeMember2.type) - val cptr1 = findByUniqueName(variables, "cptr1") - val cptr2 = findByUniqueName(variables, "cptr2") - assertEquals(cptr1.type, cptr2.type) - assertNotEquals(typeMemberOutside.type, cptr2.type) + val cptr1 = tu.variables["cptr1"] + val cptr2 = tu.variables["cptr2"] + assertEquals(cptr1?.type, cptr2?.type) + assertNotEquals(typeMemberOutside?.type, cptr2?.type) } } diff --git a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXLanguageFrontendTest.kt b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXLanguageFrontendTest.kt index ca7769e965..693673a7c2 100644 --- a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXLanguageFrontendTest.kt +++ b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXLanguageFrontendTest.kt @@ -1452,28 +1452,25 @@ internal class CXXLanguageFrontendTest : BaseTest() { @Throws(Exception::class) fun testTypedef() { val file = File("src/test/resources/c/typedef_in_header/main.c") - val result = - analyze(listOf(file), file.parentFile.toPath(), true) { + val tu = + analyzeAndGetFirstTU(listOf(file), file.parentFile.toPath(), true) { it.registerLanguage() } + with(tu) { + val typedefs = tu.ctx?.scopeManager?.typedefFor(objectType("MyStruct")) + assertLocalName("__myStruct", typedefs) - val typedefs = result.finalCtx.scopeManager.currentTypedefs - assertNotNull(typedefs) - assertTrue(typedefs.isNotEmpty()) - - val tu = result.translationUnits.firstOrNull() - assertNotNull(tu) - - val main = tu.byNameOrNull("main") - assertNotNull(main) + val main = tu.byNameOrNull("main") + assertNotNull(main) - val call = main.bodyOrNull() - assertNotNull(call) - assertTrue(call.invokes.isNotEmpty()) + val call = main.bodyOrNull() + assertNotNull(call) + assertTrue(call.invokes.isNotEmpty()) - val func = call.invokes.firstOrNull() - assertNotNull(func) - assertFalse(func.isInferred) + val func = call.invokes.firstOrNull() + assertNotNull(func) + assertFalse(func.isInferred) + } } @Test diff --git a/cpg-language-cxx/src/test/resources/typedefs/typedefs.cpp b/cpg-language-cxx/src/test/resources/typedefs/typedefs.cpp index ce7f569b78..f118811c43 100644 --- a/cpg-language-cxx/src/test/resources/typedefs/typedefs.cpp +++ b/cpg-language-cxx/src/test/resources/typedefs/typedefs.cpp @@ -75,7 +75,7 @@ typedef long type; type typeMemberOutside; // sample typedef with tabs -typedef uint8 test; +typedef uint8_t test; struct add_const { typedef const int type;