From 3273ed30e48ea57368322761c3dd6471aec8c0fb Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Wed, 30 Aug 2023 19:22:24 +0200 Subject: [PATCH] Symbol resolver with EOG power Co-Authored-By: KuechA <31155350+KuechA@users.noreply.github.com> --- .../cpg/analysis/fsm/DFAOrderEvaluator.kt | 4 +- .../de/fraunhofer/aisec/cpg/ScopeManager.kt | 49 +- .../aisec/cpg/TranslationConfiguration.kt | 6 +- .../aisec/cpg/frontends/LanguageTraits.kt | 9 +- .../aisec/cpg/graph/ExpressionBuilder.kt | 10 + .../de/fraunhofer/aisec/cpg/graph/Node.kt | 7 +- .../aisec/cpg/graph/builder/Fluent.kt | 33 +- .../cpg/graph/declarations/EnumDeclaration.kt | 10 +- .../graph/declarations/FunctionDeclaration.kt | 38 +- .../graph/declarations/MethodDeclaration.kt | 9 +- .../declarations/NamespaceDeclaration.kt | 16 +- .../graph/declarations/RecordDeclaration.kt | 15 +- .../TranslationUnitDeclaration.kt | 19 +- .../graph/declarations/ValueDeclaration.kt | 6 +- .../cpg/graph/statements/TryStatement.kt | 6 +- .../statements/expressions/CallExpression.kt | 11 +- .../expressions/ConstructExpression.kt | 8 +- .../expressions/LambdaExpression.kt | 3 +- .../graph/statements/expressions/Reference.kt | 6 +- .../fraunhofer/aisec/cpg/graph/types/Type.kt | 12 + .../aisec/cpg/helpers/SubgraphWalker.kt | 26 +- .../aisec/cpg/passes/CXXCallResolverHelper.kt | 63 +- .../de/fraunhofer/aisec/cpg/passes/DFGPass.kt | 3 +- .../cpg/passes/EvaluationOrderGraphPass.kt | 55 +- .../{CallResolver.kt => SymbolResolver.kt} | 603 +++++++++++++----- .../cpg/passes/TemplateCallResolverHelper.kt | 16 +- .../aisec/cpg/passes/VariableUsageResolver.kt | 460 ------------- .../aisec/cpg/passes/inference/Inference.kt | 2 +- .../de/fraunhofer/aisec/cpg/GraphExamples.kt | 146 +++-- .../fraunhofer/aisec/cpg/graph/FluentTest.kt | 7 +- .../cpg/graph/types/TypePropagationTest.kt | 45 +- .../de/fraunhofer/aisec/cpg/passes/DFGTest.kt | 7 +- .../aisec/cpg/passes/SymbolResolverTest.kt | 74 +++ .../de/fraunhofer/aisec/cpg/TestUtils.kt | 2 +- .../aisec/cpg/frontends/cxx/CLanguage.kt | 7 +- .../aisec/cpg/frontends/cxx/CPPLanguage.kt | 18 +- .../cpg/frontends/cxx/CXXLanguageFrontend.kt | 7 +- .../cpg/frontends/cxx/ExpressionHandler.kt | 7 +- .../aisec/cpg/passes/CXXExtraPass.kt | 97 +++ .../cpg/passes/FunctionPointerCallResolver.kt | 2 +- .../aisec/cpg/enhancements/EOGTest.kt | 4 +- .../frontends/cxx/CXXLanguageFrontendTest.kt | 8 +- .../aisec/cpg/frontends/cxx/LambdaTest.kt | 8 +- .../cpp/scope_variables.cpp | 6 +- .../frontends/golang/DeclarationHandler.kt | 5 +- .../cpg/frontends/golang/ExpressionHandler.kt | 11 +- .../aisec/cpg/passes/GoExtraPass.kt | 87 +-- .../golang/GoLanguageFrontendTest.kt | 136 +++- .../src/test/resources/golang/call.go | 9 +- .../resources/golang/complex_resolution.go | 21 + .../src/test/resources/golang/if.go | 12 +- .../src/test/resources/golang/struct.go | 6 +- .../cpg/frontends/java/DeclarationHandler.kt | 2 +- .../aisec/cpg/frontends/java/JavaLanguage.kt | 4 +- .../cpg/passes/JavaCallResolverHelper.kt | 33 +- .../cpg/frontends/llvm/ExpressionHandler.kt | 6 +- .../frontends/llvm/LLVMIRLanguageFrontend.kt | 4 +- .../src/main/python/CPGPython/_expressions.py | 2 +- .../frontends/python/PythonFrontendTest.kt | 4 + .../src/test/resources/python/class_fields.py | 2 +- .../aisec/cpg_vis_neo4j/Application.kt | 3 +- 61 files changed, 1242 insertions(+), 1055 deletions(-) rename cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/{CallResolver.kt => SymbolResolver.kt} (51%) delete mode 100644 cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/VariableUsageResolver.kt create mode 100644 cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/SymbolResolverTest.kt create mode 100644 cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/CXXExtraPass.kt create mode 100644 cpg-language-go/src/test/resources/golang/complex_resolution.go diff --git a/cpg-analysis/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/fsm/DFAOrderEvaluator.kt b/cpg-analysis/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/fsm/DFAOrderEvaluator.kt index 4bd44abb63f..5ca64973950 100644 --- a/cpg-analysis/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/fsm/DFAOrderEvaluator.kt +++ b/cpg-analysis/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/fsm/DFAOrderEvaluator.kt @@ -26,7 +26,6 @@ package de.fraunhofer.aisec.cpg.analysis.fsm import de.fraunhofer.aisec.cpg.graph.Node -import de.fraunhofer.aisec.cpg.graph.declarations.Declaration import de.fraunhofer.aisec.cpg.graph.declarations.ParameterDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration import de.fraunhofer.aisec.cpg.graph.edge.Properties @@ -291,7 +290,8 @@ open class DFAOrderEvaluator( if ( node is MemberCallExpression && node.base is Reference && - consideredBases.contains((node.base as Reference).refersTo as Declaration) + (node.base as Reference).refersTo != null && + consideredBases.contains((node.base as Reference).refersTo!!) ) { allUsedBases.add((node.base as Reference).refersTo) } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt index ec6488749c0..30448d57d52 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt @@ -615,20 +615,7 @@ class ScopeManager : ScopeProvider { */ @JvmOverloads fun resolveReference(ref: Reference, startScope: Scope? = currentScope): ValueDeclaration? { - // Unfortunately, we still have an issue about duplicate declarations because header files - // are included multiple times, so we need to exclude the C++ frontend (for now). - val language = ref.language - val (scope, name) = - if ( - language?.name?.localName != "CLanguage" && - (language?.name?.localName != "CPPLanguage") - ) { - // For all other languages, we can extract the scope information out of the name and - // start our search at the dedicated scope. - extractScope(ref, startScope) - } else { - Pair(scope, ref.name) - } + val (scope, name) = extractScope(ref, startScope) // Try to resolve value declarations according to our criteria return resolve(scope) { @@ -686,13 +673,17 @@ class ScopeManager : ScopeProvider { * This function extracts a possible scope out of a [Name], e.g. if the name is fully qualified. * This also resolves possible name aliases (e.g. because of imports). It returns a pair of a * scope (if found) as well as the name, which is possibly adjusted for the aliases. + * + * Note: Currently only *fully* qualified names are properly resolved. This function will + * probably return imprecise results for partially qualified names, e.g. if a name `A` inside + * `B` points to `A::B`, rather than to `A`. */ fun extractScope(node: Node, scope: Scope? = currentScope): Pair { var name: Name = node.name var s = scope // First, we need to check, whether we have some kind of scoping. - if (node.name.parent != null) { + if (node.name.isQualified()) { // extract the scope name, it is usually a name space, but could probably be something // else as well in other languages var scopeName = node.name.parent @@ -714,9 +705,9 @@ class ScopeManager : ScopeProvider { Util.errorWithFileLocation( node, LOGGER, - "Could not find the scope $scopeName needed to resolve the call ${node.name}. Falling back to the default (current) scope" + "Could not find the scope $scopeName needed to resolve the call ${node.name}" ) - s + scope } else { scopes[0] } @@ -729,7 +720,7 @@ class ScopeManager : ScopeProvider { * Directly jumps to a given scope. Returns the previous scope. Do not forget to set the scope * back to the old scope after performing the actions inside this scope. * - * Handle with care, here be dragons. Should not be exposed outside of the cpg-core module. + * Handle with care, here be dragons. Should not be exposed outside the cpg-core module. */ @PleaseBeCareful internal fun jumpTo(scope: Scope?): Scope? { @@ -774,24 +765,35 @@ class ScopeManager : ScopeProvider { searchScope: Scope?, stopIfFound: Boolean = false, noinline predicate: (T) -> Boolean + ): List { + return resolve(T::class.java, searchScope, stopIfFound, predicate) + } + + fun resolve( + klass: Class, + searchScope: Scope?, + stopIfFound: Boolean = false, + predicate: (T) -> Boolean ): List { var scope = searchScope val declarations = mutableListOf() while (scope != null) { if (scope is ValueDeclarationScope) { - declarations.addAll(scope.valueDeclarations.filterIsInstance().filter(predicate)) + declarations.addAll( + scope.valueDeclarations.filterIsInstance(klass).filter(predicate) + ) } if (scope is StructureDeclarationScope) { - var list = scope.structureDeclarations.filterIsInstance().filter(predicate) + var list = scope.structureDeclarations.filterIsInstance(klass).filter(predicate) // this was taken over from the old resolveStructureDeclaration. // TODO(oxisto): why is this only when the list is empty? if (list.isEmpty()) { for (declaration in scope.structureDeclarations) { if (declaration is RecordDeclaration) { - list = declaration.templates.filterIsInstance().filter(predicate) + list = declaration.templates.filterIsInstance(klass).filter(predicate) } } } @@ -832,11 +834,12 @@ class ScopeManager : ScopeProvider { /** * Retrieves the [RecordDeclaration] for the given name in the given scope. * - * @param scope the scope * @param name the name + * * @param scope the scope. Default is [currentScope] + * * @return the declaration, or null if it does not exist */ - fun getRecordForName(scope: Scope, name: Name): RecordDeclaration? { + fun getRecordForName(name: Name, scope: Scope? = currentScope): RecordDeclaration? { return resolve(scope, true) { it.name.lastPartsMatch(name) } .firstOrNull() } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt index b7104344875..6c393987ae9 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt @@ -444,8 +444,7 @@ private constructor( * This will register * - [TypeHierarchyResolver] * - [ImportResolver] - * - [VariableUsageResolver] - * - [CallResolver] + * - [SymbolResolver] * - [DFGPass] * - [EvaluationOrderGraphPass] * - [TypeResolver] @@ -457,8 +456,7 @@ private constructor( fun defaultPasses(): Builder { registerPass() registerPass() - registerPass() - registerPass() // creates CG + registerPass() registerPass() registerPass() // creates EOG registerPass() diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/LanguageTraits.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/LanguageTraits.kt index 47b427bca37..e62bae14074 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/LanguageTraits.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/LanguageTraits.kt @@ -27,15 +27,13 @@ package de.fraunhofer.aisec.cpg.frontends import de.fraunhofer.aisec.cpg.ScopeManager import de.fraunhofer.aisec.cpg.TranslationContext -import de.fraunhofer.aisec.cpg.graph.Name import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.RecordDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.MemberExpression import de.fraunhofer.aisec.cpg.graph.types.Type -import de.fraunhofer.aisec.cpg.passes.CallResolver -import java.util.regex.Pattern +import de.fraunhofer.aisec.cpg.passes.SymbolResolver /** * A language trait is a feature or trait that is common to a group of programming languages. Any @@ -116,7 +114,7 @@ interface HasComplexCallResolution : LanguageTrait { call: CallExpression, ctx: TranslationContext, currentTU: TranslationUnitDeclaration, - callResolver: CallResolver + callResolver: SymbolResolver ): List /** @@ -131,7 +129,7 @@ interface HasComplexCallResolution : LanguageTrait { fun refineInvocationCandidatesFromRecord( recordDeclaration: RecordDeclaration, call: CallExpression, - namePattern: Pattern, + name: String, ctx: TranslationContext ): List } @@ -170,7 +168,6 @@ interface HasSuperClasses : LanguageTrait { callee: MemberExpression, curClass: RecordDeclaration, scopeManager: ScopeManager, - recordMap: Map ): Boolean } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/ExpressionBuilder.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/ExpressionBuilder.kt index b9edf83c4c4..1e80728150a 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/ExpressionBuilder.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/ExpressionBuilder.kt @@ -282,6 +282,11 @@ fun MetadataProvider.newCallExpression( val node = CallExpression() node.applyMetadata(this, fqn, rawNode, code, true) + // Set the call expression as resolution helper for the callee + if (callee is Reference) { + callee.resolutionHelper = node + } + node.callee = callee node.template = template @@ -331,6 +336,11 @@ fun MetadataProvider.newMemberCallExpression( code, ) + // Set the call expression as resolution helper for the callee + if (callee is Reference) { + callee.resolutionHelper = node + } + node.callee = callee node.isStatic = isStatic diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt index 35e12fc560a..4466ada2906 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt @@ -406,7 +406,12 @@ open class Node : IVisitable, Persistable, LanguageProvider, ScopeProvider code == other.code && comment == other.comment && location == other.location && - file == other.file && + // We need to exclude "file" here, because in C++ the same header node can be + // imported in two different files and in this case, the "file" property will be + // different. Since want to squash those equal nodes, we will only consider all the + // other attributes, including "location" (which contains the *original* file + // location in the header file), but not "file". + // file == other.file && isImplicit == other.isImplicit } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/builder/Fluent.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/builder/Fluent.kt index d2c3c51a1cb..b760a9e50e7 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/builder/Fluent.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/builder/Fluent.kt @@ -438,12 +438,12 @@ context(Holder) fun LanguageFrontend<*, *>.memberCall( localName: CharSequence, - member: Expression, + base: Expression, isStatic: Boolean = false, init: (MemberCallExpression.() -> Unit)? = null ): MemberCallExpression { // Try to parse the name - val node = newMemberCallExpression(newMemberExpression(localName, member), isStatic) + val node = newMemberCallExpression(newMemberExpression(localName, base), isStatic) if (init != null) { init(node) } @@ -684,9 +684,9 @@ fun LanguageFrontend<*, *>.whileCondition(init: WhileStatement.() -> Expression) } /** - * <<<<<<< HEAD Configures the [DoStatement.condition] in the Fluent Node DSL of the nearest - * enclosing [DoStatement]. The [init] block can be used to create further sub-nodes as well as - * configuring the created node itself. + * Configures the [DoStatement.condition] in the Fluent Node DSL of the nearest enclosing + * [DoStatement]. The [init] block can be used to create further sub-nodes as well as configuring + * the created node itself. */ context(DoStatement) @@ -695,13 +695,9 @@ fun LanguageFrontend<*, *>.whileCondition(init: DoStatement.() -> Expression): E } /** - * Creates a new [CompoundStatement] in the Fluent Node DSL and sets it to the - * [IfStatement.thenStatement] of the nearest enclosing [IfStatement]. The [init] block can be used - * to create further sub-nodes as well as configuring the created node itself. - * ======= * Creates a new [Block] in the Fluent Node DSL and sets it to the [IfStatement.thenStatement] of * the nearest enclosing [IfStatement]. The [init] block can be used to create further sub-nodes as - * well as configuring the created node itself. >>>>>>> main + * well as configuring the created node itself. */ context(IfStatement) @@ -748,9 +744,9 @@ fun LanguageFrontend<*, *>.loopBody(init: Block.() -> Unit): Block { } /** - * Creates a new [CompoundStatement] in the Fluent Node DSL and sets it to the - * [DoStatement.statement] of the nearest enclosing [DoStatement]. The [init] block can be used to - * create further sub-nodes as well as configuring the created node itself. + * Creates a new [Block] in the Fluent Node DSL and sets it to the [DoStatement.statement] of the + * nearest enclosing [WhileStatement]. The [init] block can be used to create further sub-nodes as + * well as configuring the created node itself. */ context(DoStatement) @@ -1439,3 +1435,14 @@ private fun LanguageFrontend<*, *>.scopeIfNecessary( scopeManager.leaveScope(node) } } + +context(MethodDeclaration) + +fun LanguageFrontend<*, *>.receiver(name: String, type: Type): VariableDeclaration { + val node = newVariableDeclaration(name, type) + + this@MethodDeclaration.receiver = node + scopeManager.addDeclaration(node) + + return node +} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/EnumDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/EnumDeclaration.kt index e57ead2bb32..de9e48eff1c 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/EnumDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/EnumDeclaration.kt @@ -28,24 +28,16 @@ package de.fraunhofer.aisec.cpg.graph.declarations import de.fraunhofer.aisec.cpg.graph.AST import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdgeDelegate -import de.fraunhofer.aisec.cpg.graph.types.Type import org.apache.commons.lang3.builder.ToStringBuilder import org.neo4j.ogm.annotation.Relationship -class EnumDeclaration : Declaration() { +class EnumDeclaration : RecordDeclaration() { @Relationship(value = "ENTRIES", direction = Relationship.Direction.OUTGOING) @AST var entryEdges: MutableList> = ArrayList() - @Relationship(value = "SUPER_TYPES", direction = Relationship.Direction.OUTGOING) - var superTypeEdges: MutableList> = ArrayList() - - @Relationship var superTypeDeclarations: Set = HashSet() - var entries by PropertyEdgeDelegate(EnumDeclaration::entryEdges) - var superTypes by PropertyEdgeDelegate(EnumDeclaration::superTypeEdges) - override fun toString(): String { return ToStringBuilder(this, TO_STRING_STYLE) .appendSuper(super.toString()) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.kt index 4f87a4fc63d..880014f0be2 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.kt @@ -35,13 +35,13 @@ import de.fraunhofer.aisec.cpg.graph.statements.expressions.Block import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression import de.fraunhofer.aisec.cpg.graph.types.Type import de.fraunhofer.aisec.cpg.isDerivedFrom +import de.fraunhofer.aisec.cpg.passes.ResolutionStartHolder import java.util.* -import java.util.stream.Collectors import org.apache.commons.lang3.builder.ToStringBuilder import org.neo4j.ogm.annotation.Relationship /** Represents the declaration or definition of a function. */ -open class FunctionDeclaration : ValueDeclaration(), DeclarationHolder { +open class FunctionDeclaration : ValueDeclaration(), DeclarationHolder, ResolutionStartHolder { /** The function body. Usually a [Block]. */ @AST var body: Statement? = null @@ -111,13 +111,13 @@ open class FunctionDeclaration : ValueDeclaration(), DeclarationHolder { targetFunctionDeclaration.signatureTypes == signatureTypes } + // TODO: Documentation required. It's not completely clear what this method is supposed to do. fun hasSignature(targetSignature: List): Boolean { - val signature = - parameters - .stream() - .sorted(Comparator.comparingInt(ParameterDeclaration::argumentIndex)) - .collect(Collectors.toList()) + val signature = parameters.sortedBy { it.argumentIndex } + // TODO: Why do we have to sort it here while we don't sort the list in signatureTypes? return if (signature.all { !it.isVariadic } && targetSignature.size < signature.size) { + // TODO: So we don't consider arguments with default values (among others) but then, the + // SymbolResolver (or CXXCallResolverHelper) has a bunch of functions to consider it. false } else { // signature is a collection of positional arguments, so the order must be preserved @@ -125,9 +125,9 @@ open class FunctionDeclaration : ValueDeclaration(), DeclarationHolder { val declared = signature[i] if (declared.isVariadic) { // Everything that follows is collected by this param, so the signature is - // fulfilled no matter what comes now (potential FIXME: in Java, we could have - // overloading with different vararg types, in C++ we can't, as vararg types are - // not defined here anyways) + // fulfilled no matter what comes now + // FIXME: in Java, we could have overloading with different vararg types, in + // C++ we can't, as vararg types are not defined here anyways) return true } val provided = targetSignature[i] @@ -189,18 +189,15 @@ open class FunctionDeclaration : ValueDeclaration(), DeclarationHolder { return parameters.map { it.default } } - val defaultParameterSignature: List - get() { - val signature: MutableList = ArrayList() - for (paramVariableDeclaration in parameters) { - if (paramVariableDeclaration.default != null) { - signature.add(paramVariableDeclaration.type) + val defaultParameterSignature: List // TODO: What's this property? + get() = + parameters.map { + if (it.default != null) { + it.type } else { - signature.add(unknownType()) + unknownType() } } - return signature - } val signatureTypes: List get() = parameters.map { it.type } @@ -222,6 +219,9 @@ open class FunctionDeclaration : ValueDeclaration(), DeclarationHolder { .toString() } + override val resolutionStartNodes: List + get() = listOfNotNull(this) + override fun equals(other: Any?): Boolean { if (this === other) { return true diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/MethodDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/MethodDeclaration.kt index edfe3e7a6c2..11578253b86 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/MethodDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/MethodDeclaration.kt @@ -27,8 +27,7 @@ package de.fraunhofer.aisec.cpg.graph.declarations import de.fraunhofer.aisec.cpg.graph.AST import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference -import de.fraunhofer.aisec.cpg.passes.CallResolver -import de.fraunhofer.aisec.cpg.passes.VariableUsageResolver +import de.fraunhofer.aisec.cpg.passes.SymbolResolver /** * A method declaration is a [FunctionDeclaration] that is part of to a specific [RecordDeclaration] @@ -53,8 +52,8 @@ open class MethodDeclaration : FunctionDeclaration() { * [receiver] property of the method, since the scope manager cannot do this. If the name of the * receiver, e.g., `this`, is used anywhere in the method body, a [Reference] must be created by * the language frontend, and its [Reference.refersTo] property must point to this [receiver]. - * The latter is done automatically by the [VariableUsageResolver], which treats it like any - * other regular variable. + * The latter is done automatically by the [SymbolResolver], which treats it like any other + * regular variable. * * Some languages (for example Python) denote the first argument in a method declaration as the * receiver (e.g., in `def foo(self, arg1)`, `self` is the receiver). In this case, extra care @@ -68,7 +67,7 @@ open class MethodDeclaration : FunctionDeclaration() { * superclass of the current class. In this case, a [Reference] will also be created (with the * name `super`) and it will also refer to this receiver, even though the receiver's name is * `this`. This is one of the very few exceptions where the reference and its declaration do not - * share the same name. The [CallResolver] will recognize this and treat the scoping aspect of + * share the same name. The [SymbolResolver] will recognize this and treat the scoping aspect of * the super-call accordingly. */ @AST var receiver: VariableDeclaration? = null diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/NamespaceDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/NamespaceDeclaration.kt index 9fed1238d12..3c380e3a158 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/NamespaceDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/NamespaceDeclaration.kt @@ -27,10 +27,12 @@ package de.fraunhofer.aisec.cpg.graph.declarations import de.fraunhofer.aisec.cpg.graph.AST import de.fraunhofer.aisec.cpg.graph.DeclarationHolder +import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.StatementHolder import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdgeDelegate import de.fraunhofer.aisec.cpg.graph.statements.Statement +import de.fraunhofer.aisec.cpg.passes.ResolutionStartHolder import java.util.Objects import org.neo4j.ogm.annotation.Relationship @@ -45,7 +47,8 @@ import org.neo4j.ogm.annotation.Relationship * * The name property of this node need to be a FQN for property resolution. */ -class NamespaceDeclaration : Declaration(), DeclarationHolder, StatementHolder { +class NamespaceDeclaration : + Declaration(), DeclarationHolder, StatementHolder, ResolutionStartHolder { /** * Edges to nested namespaces, records, functions, fields etc. contained in the current * namespace. @@ -93,4 +96,15 @@ class NamespaceDeclaration : Declaration(), DeclarationHolder, StatementHolder { override var statements: List by PropertyEdgeDelegate(NamespaceDeclaration::statementEdges) + + override val resolutionStartNodes: List + get() { + val list = mutableListOf() + // Add all top-level declarations + list += declarations + // Add all top-level statements + list += statements + + return list + } } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/RecordDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/RecordDeclaration.kt index 3e1afe69cac..10b334ac0dc 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/RecordDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/RecordDeclaration.kt @@ -32,12 +32,14 @@ import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdgeDelegate import de.fraunhofer.aisec.cpg.graph.statements.Statement import de.fraunhofer.aisec.cpg.graph.types.ObjectType import de.fraunhofer.aisec.cpg.graph.types.Type +import de.fraunhofer.aisec.cpg.passes.ResolutionStartHolder import org.apache.commons.lang3.builder.ToStringBuilder import org.neo4j.ogm.annotation.Relationship import org.neo4j.ogm.annotation.Transient /** Represents a C++ union/struct/class or Java class */ -class RecordDeclaration : Declaration(), DeclarationHolder, StatementHolder { +open class RecordDeclaration : + Declaration(), DeclarationHolder, StatementHolder, ResolutionStartHolder { /** The kind, i.e. struct, class, union or enum. */ var kind: String? = null @@ -172,6 +174,17 @@ class RecordDeclaration : Declaration(), DeclarationHolder, StatementHolder { .toString() } + override val resolutionStartNodes: List + get() { + val list = mutableListOf() + + list += fields + list += methods + list += constructors + + return list + } + override fun equals(other: Any?): Boolean { if (this === other) return true if (other !is RecordDeclaration) return false diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TranslationUnitDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TranslationUnitDeclaration.kt index 7a7ad3f1b1b..f214af8abd8 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TranslationUnitDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TranslationUnitDeclaration.kt @@ -25,21 +25,21 @@ */ package de.fraunhofer.aisec.cpg.graph.declarations -import de.fraunhofer.aisec.cpg.graph.AST -import de.fraunhofer.aisec.cpg.graph.DeclarationHolder -import de.fraunhofer.aisec.cpg.graph.StatementHolder +import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge.Companion.propertyEqualsList import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge.Companion.unwrap import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdgeDelegate import de.fraunhofer.aisec.cpg.graph.statements.Statement import de.fraunhofer.aisec.cpg.passes.PassTarget +import de.fraunhofer.aisec.cpg.passes.ResolutionStartHolder import java.util.Objects import org.apache.commons.lang3.builder.ToStringBuilder import org.neo4j.ogm.annotation.Relationship /** The top most declaration, representing a translation unit, for example a file. */ -class TranslationUnitDeclaration : Declaration(), DeclarationHolder, StatementHolder, PassTarget { +class TranslationUnitDeclaration : + Declaration(), DeclarationHolder, StatementHolder, PassTarget, ResolutionStartHolder { /** A list of declarations within this unit. */ @Relationship(value = "DECLARATIONS", direction = Relationship.Direction.OUTGOING) @AST @@ -127,6 +127,17 @@ class TranslationUnitDeclaration : Declaration(), DeclarationHolder, StatementHo .toString() } + override val resolutionStartNodes: List + get() { + val list = mutableListOf() + // Add all top-level declarations + list += declarations + // Add all top-level statements + list += statements + + return list + } + override fun equals(other: Any?): Boolean { if (this === other) return true if (other !is TranslationUnitDeclaration) return false diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/ValueDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/ValueDeclaration.kt index 3204d7cfd5a..0835ebd7b14 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/ValueDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/ValueDeclaration.kt @@ -33,7 +33,7 @@ import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge.Companion.unwrap import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference import de.fraunhofer.aisec.cpg.graph.types.* import de.fraunhofer.aisec.cpg.helpers.identitySetOf -import de.fraunhofer.aisec.cpg.passes.VariableUsageResolver +import de.fraunhofer.aisec.cpg.passes.SymbolResolver import java.util.stream.Collectors import org.apache.commons.lang3.builder.ToStringBuilder import org.neo4j.ogm.annotation.Relationship @@ -75,12 +75,12 @@ abstract class ValueDeclaration : Declaration(), HasType { * Links to all the [Reference]s accessing the variable and the respective access value (read, * write, readwrite). */ - @PopulatedByPass(VariableUsageResolver::class) + @PopulatedByPass(SymbolResolver::class) @Relationship(value = "USAGE") var usageEdges: MutableList> = ArrayList() /** All usages of the variable/field. */ - @PopulatedByPass(VariableUsageResolver::class) + @PopulatedByPass(SymbolResolver::class) var usages: List get() = unwrap(usageEdges, true) /** Set all usages of the variable/field and assembles the access properties. */ diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/TryStatement.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/TryStatement.kt index d92011554d3..9c015837729 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/TryStatement.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/TryStatement.kt @@ -26,15 +26,17 @@ package de.fraunhofer.aisec.cpg.graph.statements import de.fraunhofer.aisec.cpg.graph.AST +import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge.Companion.propertyEqualsList import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdgeDelegate import de.fraunhofer.aisec.cpg.graph.statements.expressions.Block +import de.fraunhofer.aisec.cpg.passes.ResolutionStartHolder import java.util.* import org.neo4j.ogm.annotation.Relationship /** A [Statement] which represents a try/catch block, primarily used for exception handling. */ -class TryStatement : Statement() { +class TryStatement : Statement(), ResolutionStartHolder { @Relationship(value = "RESOURCES", direction = Relationship.Direction.OUTGOING) @AST var resourceEdges = mutableListOf>() @@ -50,6 +52,8 @@ class TryStatement : Statement() { var catchClauseEdges = mutableListOf>() var catchClauses by PropertyEdgeDelegate(TryStatement::catchClauseEdges) + override val resolutionStartNodes: List + get() = catchClauses override fun equals(other: Any?): Boolean { if (this === other) return true diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CallExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CallExpression.kt index c98fc1d3a82..0cb49764d94 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CallExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CallExpression.kt @@ -37,8 +37,7 @@ import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge.Companion.propertyEqualsL import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge.Companion.unwrap import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge.Companion.wrap import de.fraunhofer.aisec.cpg.graph.types.* -import de.fraunhofer.aisec.cpg.passes.CallResolver -import de.fraunhofer.aisec.cpg.passes.VariableUsageResolver +import de.fraunhofer.aisec.cpg.passes.SymbolResolver import java.util.* import org.apache.commons.lang3.builder.ToStringBuilder import org.neo4j.ogm.annotation.Relationship @@ -52,7 +51,7 @@ open class CallExpression : Expression(), HasType.TypeObserver, ArgumentHolder { * Connection to its [FunctionDeclaration]. This will be populated by the [CallResolver]. This * will have an effect on the [type] */ - @PopulatedByPass(CallResolver::class) + @PopulatedByPass(SymbolResolver::class) @Relationship(value = "INVOKES", direction = Relationship.Direction.OUTGOING) var invokeEdges = mutableListOf>() protected set @@ -61,7 +60,7 @@ open class CallExpression : Expression(), HasType.TypeObserver, ArgumentHolder { * A virtual property to quickly access the list of declarations that this call invokes without * property edges. */ - @PopulatedByPass(CallResolver::class) + @PopulatedByPass(SymbolResolver::class) var invokes: List get(): List { val targets: MutableList = ArrayList() @@ -90,9 +89,9 @@ open class CallExpression : Expression(), HasType.TypeObserver, ArgumentHolder { var arguments by PropertyEdgeDelegate(CallExpression::argumentEdges) /** - * The expression that is being "called". This is currently not yet used in the [CallResolver] + * The expression that is being "called". This is currently not yet used in the [SymbolResolver] * but will be in the future. In most cases, this is a [Reference] and its [Reference.refersTo] - * is intentionally left empty. It is not filled by the [VariableUsageResolver]. + * is intentionally left empty. It is not filled by the [SymbolResolver]. */ @AST var callee: Expression? = null diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ConstructExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ConstructExpression.kt index e69ab80afd4..a934139350c 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ConstructExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ConstructExpression.kt @@ -29,7 +29,7 @@ import de.fraunhofer.aisec.cpg.PopulatedByPass import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.* import de.fraunhofer.aisec.cpg.graph.types.UnknownType -import de.fraunhofer.aisec.cpg.passes.CallResolver +import de.fraunhofer.aisec.cpg.passes.SymbolResolver import java.util.* import org.apache.commons.lang3.builder.ToStringBuilder @@ -43,9 +43,9 @@ import org.apache.commons.lang3.builder.ToStringBuilder class ConstructExpression : CallExpression() { /** * The link to the [ConstructorDeclaration]. This is populated by the - * [de.fraunhofer.aisec.cpg.passes.CallResolver] later. + * [de.fraunhofer.aisec.cpg.passes.SymbolResolver] later. */ - @PopulatedByPass(CallResolver::class) + @PopulatedByPass(SymbolResolver::class) var constructor: ConstructorDeclaration? = null get() = if (anoymousClass != null) { @@ -65,7 +65,7 @@ class ConstructExpression : CallExpression() { @AST var anoymousClass: RecordDeclaration? = null /** The [Declaration] of the type this expression instantiates. */ - @PopulatedByPass(CallResolver::class) + @PopulatedByPass(SymbolResolver::class) var instantiates: Declaration? = null get() = if (anoymousClass != null) { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/LambdaExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/LambdaExpression.kt index a737cf20f11..eb43f87fecd 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/LambdaExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/LambdaExpression.kt @@ -25,10 +25,9 @@ */ package de.fraunhofer.aisec.cpg.graph.statements.expressions -import de.fraunhofer.aisec.cpg.graph.AST +import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.ValueDeclaration -import de.fraunhofer.aisec.cpg.graph.pointer import de.fraunhofer.aisec.cpg.graph.types.FunctionType import de.fraunhofer.aisec.cpg.graph.types.HasType import de.fraunhofer.aisec.cpg.graph.types.Type diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/Reference.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/Reference.kt index 27f25e11e37..11d47bb3a8e 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/Reference.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/Reference.kt @@ -34,7 +34,7 @@ import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration import de.fraunhofer.aisec.cpg.graph.edge.Properties import de.fraunhofer.aisec.cpg.graph.types.HasType import de.fraunhofer.aisec.cpg.graph.types.Type -import de.fraunhofer.aisec.cpg.passes.VariableUsageResolver +import de.fraunhofer.aisec.cpg.passes.SymbolResolver import java.util.* import org.apache.commons.lang3.builder.ToStringBuilder import org.neo4j.ogm.annotation.Relationship @@ -49,7 +49,7 @@ open class Reference : Expression(), HasType.TypeObserver { * The [Declaration]s this expression might refer to. This will influence the [declaredType] of * this expression. */ - @PopulatedByPass(VariableUsageResolver::class) + @PopulatedByPass(SymbolResolver::class) @Relationship(value = "REFERS_TO") var refersTo: Declaration? = null set(value) { @@ -103,7 +103,7 @@ open class Reference : Expression(), HasType.TypeObserver { override fun toString(): String { return ToStringBuilder(this, TO_STRING_STYLE) - .append(super.toString()) + .appendSuper(super.toString()) .append("refersTo", refersTo) .toString() } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/Type.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/Type.kt index 055cc0ef37b..af7f3b6188f 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/Type.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/Type.kt @@ -30,6 +30,7 @@ import de.fraunhofer.aisec.cpg.TypeManager import de.fraunhofer.aisec.cpg.frontends.Language import de.fraunhofer.aisec.cpg.graph.Name import de.fraunhofer.aisec.cpg.graph.Node +import de.fraunhofer.aisec.cpg.graph.declarations.RecordDeclaration import de.fraunhofer.aisec.cpg.graph.parseName import de.fraunhofer.aisec.cpg.graph.types.PointerType.PointerOrigin import de.fraunhofer.aisec.cpg.passes.TypeHierarchyResolver @@ -221,3 +222,14 @@ abstract class Type : Node { } } } + +/** A shortcut to return [ObjectType.recordDeclaration], if this is a [ObjectType]. */ +var Type.recordDeclaration: RecordDeclaration? + get() { + return (this as? ObjectType)?.recordDeclaration + } + set(value) { + if (this is ObjectType) { + this.recordDeclaration = value + } + } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/SubgraphWalker.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/SubgraphWalker.kt index 49644cfa7db..282d6360e1f 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/SubgraphWalker.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/SubgraphWalker.kt @@ -39,8 +39,6 @@ import java.lang.reflect.Field import java.util.* import java.util.function.BiConsumer import java.util.function.Consumer -import java.util.function.Predicate -import java.util.stream.Collectors import org.neo4j.ogm.annotation.Relationship import org.slf4j.LoggerFactory @@ -258,6 +256,8 @@ object SubgraphWalker { var backlog: Deque? = null private set + var strategy: (Node) -> Iterator = Strategy::AST_FORWARD + /** * This callback is triggered whenever a new node is visited for the first time. This is the * place where usual graph manipulation will happen. The current node is the single argument @@ -316,10 +316,8 @@ object SubgraphWalker { Consumer { c: BiConsumer -> c.accept(current, parent) } ) val unseenChildren = - getAstChildren(current) - .stream() - .filter(Predicate.not { o: Node -> seen.contains(o) }) - .collect(Collectors.toList()) + strategy(current).asSequence().filter { it !in seen }.toMutableList() + seen.addAll(unseenChildren) unseenChildren.asReversed().forEach { child: Node -> (todo as ArrayDeque>).push(Pair(child, current)) @@ -359,6 +357,7 @@ object SubgraphWalker { * resolving declarations or other scope-related tasks. */ class ScopedWalker { + lateinit var strategy: (Node) -> Iterator private var walker: IterativeGraphWalker? = null private val scopeManager: ScopeManager @@ -366,8 +365,12 @@ object SubgraphWalker { scopeManager = lang.scopeManager } - constructor(scopeManager: ScopeManager) { + constructor( + scopeManager: ScopeManager, + strategy: (Node) -> Iterator = Strategy::AST_FORWARD + ) { this.scopeManager = scopeManager + this.strategy = strategy } /** @@ -392,6 +395,14 @@ object SubgraphWalker { ) } + fun registerHandler(handler: Consumer) { + handlers.add( + TriConsumer { _: RecordDeclaration?, _: Node?, currNode: Node? -> + handler.accept(currNode) + } + ) + } + /** * Wraps [IterativeGraphWalker] to handle declaration scopes. * @@ -399,6 +410,7 @@ object SubgraphWalker { */ fun iterate(root: Node) { walker = IterativeGraphWalker() + walker!!.strategy = this.strategy handlers.forEach { h -> walker?.registerOnNodeVisit { n -> handleNode(n, h) } } walker?.iterate(root) } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/CXXCallResolverHelper.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/CXXCallResolverHelper.kt index 449416d5b4c..19825f857a3 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/CXXCallResolverHelper.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/CXXCallResolverHelper.kt @@ -240,64 +240,6 @@ fun shouldContinueSearchInParent(recordDeclaration: RecordDeclaration?, name: St return invocationCandidate.isEmpty() } -/** - * @param constructExpression we want to find an invocation target for - * @param recordDeclaration associated with the Object the ConstructExpression constructs - * @return a ConstructDeclaration that matches the signature of the ConstructExpression by applying - * one or more implicit casts to the primitive type arguments of the ConstructExpressions. The - * arguments are proxied through a CastExpression to the type required by the - * ConstructDeclaration. - */ -fun resolveConstructorWithImplicitCast( - constructExpression: ConstructExpression, - recordDeclaration: RecordDeclaration -): ConstructorDeclaration? { - for (constructorDeclaration in recordDeclaration.constructors) { - val workingSignature = mutableListOf(*constructExpression.signature.toTypedArray()) - val defaultParameterSignature = constructorDeclaration.defaultParameterSignature - if (constructExpression.arguments.size <= defaultParameterSignature.size) { - workingSignature.addAll( - defaultParameterSignature.subList( - constructExpression.arguments.size, - defaultParameterSignature.size - ) - ) - } - if ( - compatibleSignatures( - constructExpression.signature, - constructorDeclaration.signatureTypes, - ) - ) { - val implicitCasts = - signatureWithImplicitCastTransformation( - constructExpression, - constructExpression.signature, - constructExpression.arguments, - constructorDeclaration.signatureTypes, - ) - applyImplicitCastToArguments(constructExpression, implicitCasts) - return constructorDeclaration - } else if ( - compatibleSignatures( - workingSignature, - constructorDeclaration.signatureTypes, - ) - ) { - val implicitCasts = - signatureWithImplicitCastTransformation( - constructExpression, - getCallSignatureWithDefaults(constructExpression, constructorDeclaration), - constructExpression.arguments, - constructorDeclaration.signatureTypes, - ) - applyImplicitCastToArguments(constructExpression, implicitCasts) - return constructorDeclaration - } - } - return null -} - /** * Performs all necessary steps to make a CallExpression instantiate a template: 1. Set * TemplateInstantiation Edge from CallExpression to Template 2. Set Invokes Edge to all @@ -679,7 +621,10 @@ fun checkArgumentValidity( ) // Use provided arguments callArguments.addAll( functionDeclaration.defaultParameters - .subList(callArguments.size, functionDeclaration.defaultParameters.size) + .subList( + callArguments.size, + functionDeclaration.defaultParameters.size + ) // TODO: Could be replaced with functionDeclaration.parameters.size .filterNotNull() ) // Extend by defaults for (i in callArguments.indices) { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt index eb67dd0a3fd..0fb65f7f349 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt @@ -39,8 +39,7 @@ import de.fraunhofer.aisec.cpg.helpers.Util import de.fraunhofer.aisec.cpg.passes.order.DependsOn /** Adds the DFG edges for various types of nodes. */ -@DependsOn(VariableUsageResolver::class) -@DependsOn(CallResolver::class) +@DependsOn(SymbolResolver::class) class DFGPass(ctx: TranslationContext) : ComponentPass(ctx) { override fun accept(component: Component) { val inferDfgForUnresolvedCalls = config.inferenceConfiguration.inferDfgForUnresolvedSymbols diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/EvaluationOrderGraphPass.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/EvaluationOrderGraphPass.kt index fa4f4c6ec91..2be2585abb6 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/EvaluationOrderGraphPass.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/EvaluationOrderGraphPass.kt @@ -41,7 +41,6 @@ import de.fraunhofer.aisec.cpg.helpers.IdentitySet import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker import de.fraunhofer.aisec.cpg.helpers.Util import de.fraunhofer.aisec.cpg.isDerivedFrom -import de.fraunhofer.aisec.cpg.passes.order.DependsOn import de.fraunhofer.aisec.cpg.passes.order.ReplacePass import java.util.* import org.slf4j.LoggerFactory @@ -71,7 +70,6 @@ import org.slf4j.LoggerFactory * this pass and fine-tune it. */ @Suppress("MemberVisibilityCanBePrivate") -@DependsOn(CallResolver::class) open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPass(ctx) { protected val map = mutableMapOf, (Node) -> Unit>() protected var currentPredecessors = mutableListOf() @@ -176,7 +174,7 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa /** * Removes EOG edges by first building the negative set of nodes that cannot be visited and then - * remove there outgoing edges. This also removes cycles. + * remove their outgoing edges. This also removes cycles. */ protected fun removeUnreachableEOGEdges(tu: TranslationUnitDeclaration) { // All nodes which have an eog edge @@ -191,15 +189,19 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa eogNodes .filter { node -> node is FunctionDeclaration || + node is VariableDeclaration || node is RecordDeclaration || node is NamespaceDeclaration || - node is TranslationUnitDeclaration + node is TranslationUnitDeclaration || + node is CatchClause } .toSet() // Remove all nodes from eogNodes which are reachable from validStarts and transitively. + val alreadySeen = IdentitySet() while (validStarts.isNotEmpty()) { eogNodes.removeAll(validStarts) - validStarts = validStarts.flatMap { it.nextEOG }.filter { it in eogNodes }.toSet() + validStarts = validStarts.flatMap { it.nextEOG }.filter { it !in alreadySeen }.toSet() + alreadySeen.addAll(validStarts) } // The remaining nodes are unreachable from the entry points. We delete their outgoing EOG // edges. @@ -208,7 +210,7 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa next.end.removePrevEOGEntry(unvisitedNode) } - unvisitedNode.nextEOGEdges.clear() + unvisitedNode.clearNextEOG() } } @@ -233,16 +235,15 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa } protected fun handleVariableDeclaration(node: VariableDeclaration) { + pushToEOG(node) // analyze the initializer createEOG(node.initializer) - pushToEOG(node) } protected fun handleTupleDeclaration(node: TupleDeclaration) { + pushToEOG(node) // analyze the initializer createEOG(node.initializer) - node.elements.forEach { createEOG(it) } - pushToEOG(node) } protected open fun handleRecordDeclaration(node: RecordDeclaration) { @@ -255,6 +256,9 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa for (method in node.methods) { createEOG(method) } + for (fields in node.fields) { + createEOG(fields) + } for (records in node.records) { createEOG(records) } @@ -516,8 +520,13 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa // Handle left hand side(s) first node.lhs.forEach { createEOG(it) } - // Then the right side(s) - node.rhs.forEach { createEOG(it) } + // Then the right side(s). Avoid creating the EOG twice if it's already part of the + // initializer of a declaration + node.rhs.forEach { + if (it !in node.declarations.map { decl -> decl.initializer }) { + createEOG(it) + } + } pushToEOG(node) } @@ -613,9 +622,11 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa } } toRemove.forEach { catchesOrRelays?.remove(it) } + pushToEOG(catchClause) createEOG(catchClause.body) tmpEOGNodes.addAll(currentPredecessors) } + val canTerminateExceptionfree = tmpEOGNodes.any { reachableFromValidEOGRoot(it) } currentPredecessors.clear() currentPredecessors.addAll(tmpEOGNodes) @@ -733,6 +744,28 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa createEOG(arg) } pushToEOG(node) + + if (node.anoymousClass != null) { + // Generate the EOG inside the anonymous class. It's not linked to the EOG of the outer + // part. + val tmpCurrentEOG = currentPredecessors.toMutableList() + val tmpCurrentProperties = nextEdgeProperties.toMutableMap() + val tmpIntermediateNodes = intermediateNodes.toMutableList() + + nextEdgeProperties.clear() + currentPredecessors.clear() + intermediateNodes.clear() + + createEOG(node.anoymousClass) + + nextEdgeProperties.clear() + currentPredecessors.clear() + intermediateNodes.clear() + + nextEdgeProperties.putAll(tmpCurrentProperties) + currentPredecessors.addAll(tmpCurrentEOG) + intermediateNodes.addAll(tmpIntermediateNodes) + } } /** diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/CallResolver.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/SymbolResolver.kt similarity index 51% rename from cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/CallResolver.kt rename to cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/SymbolResolver.kt index f62186db48b..e61cc21efaf 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/CallResolver.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/SymbolResolver.kt @@ -29,22 +29,36 @@ import de.fraunhofer.aisec.cpg.TranslationContext import de.fraunhofer.aisec.cpg.frontends.* import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.* -import de.fraunhofer.aisec.cpg.graph.declarations.TemplateDeclaration.TemplateInitialization +import de.fraunhofer.aisec.cpg.graph.scopes.NameScope import de.fraunhofer.aisec.cpg.graph.statements.expressions.* import de.fraunhofer.aisec.cpg.graph.types.* import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker.ScopedWalker import de.fraunhofer.aisec.cpg.helpers.Util +import de.fraunhofer.aisec.cpg.passes.inference.Inference.TypeInferenceObserver import de.fraunhofer.aisec.cpg.passes.inference.inferFunction import de.fraunhofer.aisec.cpg.passes.inference.inferMethod import de.fraunhofer.aisec.cpg.passes.inference.startInference import de.fraunhofer.aisec.cpg.passes.order.DependsOn import de.fraunhofer.aisec.cpg.processing.strategy.Strategy -import java.util.* -import java.util.regex.Pattern import org.slf4j.Logger import org.slf4j.LoggerFactory /** + * Creates new connections between the place where a variable is declared and where it is used. + * + * A field access is modeled with a [MemberExpression]. After AST building, its base and member + * references are set to [Reference] stubs. This pass resolves those references and makes the member + * point to the appropriate [FieldDeclaration] and the base to the "this" [FieldDeclaration] of the + * containing class. It is also capable of resolving references to fields that are inherited from a + * superclass and thus not declared in the actual base class. When base or member declarations are + * not found in the graph, a new "inferred" [FieldDeclaration] is being created that is then used to + * collect all usages to the same unknown declaration. [Reference] stubs are removed from the graph + * after being resolved. + * + * Accessing a local variable is modeled directly with a [Reference]. This step of the pass doesn't + * remove the [Reference] nodes like in the field usage case but rather makes their "refersTo" point + * to the appropriate [ValueDeclaration]. + * * Resolves [CallExpression] and [NewExpression] targets. * * A [CallExpression] specifies the method that wants to be called via [CallExpression.name]. The @@ -59,105 +73,389 @@ import org.slf4j.LoggerFactory * * This pass should NOT use any DFG edges because they are computed / adjusted in a later stage. */ -@DependsOn(VariableUsageResolver::class) -open class CallResolver(ctx: TranslationContext) : SymbolResolverPass(ctx) { - /** - * This seems to be a map between function declarations (more likely method declarations) and - * their parent record (more accurately their type). Seems to be only used by - * [getOverridingCandidates] and should probably be replaced through a scope manager call. - */ - protected val containingType = mutableMapOf() +@DependsOn(TypeResolver::class) +@DependsOn(TypeHierarchyResolver::class) +@DependsOn(EvaluationOrderGraphPass::class) +open class SymbolResolver(ctx: TranslationContext) : ComponentPass(ctx) { - override fun cleanup() { - containingType.clear() - } + protected lateinit var walker: ScopedWalker + lateinit var currentTU: TranslationUnitDeclaration + + protected val templateList = mutableListOf() override fun accept(component: Component) { walker = ScopedWalker(scopeManager) - walker.registerHandler { node, _ -> findRecords(node) } - walker.registerHandler { node, _ -> findTemplates(node) } - walker.registerHandler { currentClass, _, currentNode -> - registerMethods(currentClass, currentNode) + + walker.registerHandler(::findTemplates) + for (tu in component.translationUnits) { + walker.iterate(tu) } + /*walker.registerHandler(::resolveFieldUsages) for (tu in component.translationUnits) { + currentTU = tu walker.iterate(tu) } + walker.clearCallbacks() - walker.registerHandler { node, _ -> fixInitializers(node) } + walker.registerHandler(::resolveReference) for (tu in component.translationUnits) { + currentTU = tu walker.iterate(tu) } + walker.clearCallbacks() - walker.registerHandler { node, _ -> handleNode(node) } + walker.registerHandler(::resolveCalls) for (tu in component.translationUnits) { walker.iterate(tu) + }*/ + + walker.strategy = Strategy::EOG_FORWARD + walker.clearCallbacks() + walker.registerHandler(this::handle) + for (tu in component.translationUnits) { + currentTU = tu + // gather all resolution start holders and their start nodes + val nodes = + tu.allChildren().flatMap { it.resolutionStartNodes }.toSet() + + for (node in nodes) { + walker.iterate(node) + } } } - protected fun registerMethods(currentClass: RecordDeclaration?, currentNode: Node?) { - if (currentNode is MethodDeclaration && currentClass != null) { - containingType[currentNode] = currentNode.objectType(currentClass.name) + override fun cleanup() { + templateList.clear() + } + + /** Caches all TemplateDeclarations in [templateList] */ + protected fun findTemplates(node: Node?) { + if (node is TemplateDeclaration) { + templateList.add(node) } } - protected fun fixInitializers(node: Node?) { - if (node is VariableDeclaration) { - // check if we have the corresponding class for this type - val typeString = node.type.root.name - if (typeString in recordMap) { - val currInitializer = node.initializer - if (currInitializer == null && node.isImplicitInitializerAllowed) { - val initializer = node.newConstructExpression(typeString, "$typeString()") - initializer.type = node.type - initializer.isImplicit = true - node.initializer = initializer - node.templateParameters?.let { - addImplicitTemplateParametersToCall(it, initializer) - } - } else if ( - currInitializer !is ConstructExpression && - currInitializer is CallExpression && - currInitializer.name.localName == node.type.root.name.localName - ) { - // This should actually be a construct expression, not a call! - val arguments = currInitializer.arguments - val signature = arguments.map(Node::code).joinToString(", ") - val initializer = - node.newConstructExpression(typeString, "$typeString($signature)") - initializer.type = node.type - initializer.arguments = mutableListOf(*arguments.toTypedArray()) - initializer.isImplicit = true - node.initializer = initializer - currInitializer.disconnectFromGraph() - } + /** Checks if the function has the given [name], [returnType] and [signature] */ + protected fun FunctionDeclaration.matches( + name: Name, + returnType: Type, + signature: List + ): Boolean { + val thisReturnType = + if (this.returnTypes.isEmpty()) { + IncompleteType() + } else { + // TODO(oxisto): support multiple return types + this.returnTypes[0] } + return this.name.lastPartsMatch(name) && + thisReturnType == returnType && + this.hasSignature(signature) + } + + /** + * Determines if the [reference] refers to the super class and we have to start searching there. + */ + protected fun isSuperclassReference(reference: Reference): Boolean { + val language = reference.language + return language is HasSuperClasses && reference.name.endsWith(language.superClassKeyword) + } + + /** This function seems to resolve function pointers pointing to a [MethodDeclaration]. */ + protected fun resolveMethodFunctionPointer( + reference: Reference, + type: FunctionPointerType + ): ValueDeclaration { + var target = scopeManager.resolveReference(reference) + + // If we didn't find anything, we create a new function or method declaration + if (target == null) { + // Determine the scope where we want to start our inference + var (scope, _) = scopeManager.extractScope(reference) + + if (scope !is NameScope) { + scope = null + } + + target = + (scope?.astNode ?: currentTU) + .startInference(ctx) + .createInferredFunctionDeclaration( + reference.name, + null, + false, + type.parameters, + type.returnType + ) } + + return target } - protected fun handleNode(node: Node?) { - when (node) { - is TranslationUnitDeclaration -> { - currentTU = node + protected fun handleReference(currentClass: RecordDeclaration?, current: Node?) { + val language = current?.language + + if (current !is Reference || current is MemberExpression) return + + // For now, we need to ignore reference expressions that are directly embedded into call + // expressions, because they are the "callee" property. In the future, we will use this + // property to actually resolve the function call. However, there is a special case that + // we want to catch already, that is if we are "calling" a reference to a variable. This + // can be done in several languages, e.g., in C/C++ as function pointers or in Go as + // function references. In this case, we want to resolve the declared reference expression + // of this call expression back to its original variable declaration. In the future, we want + // to extend this particular code to resolve all callee references to their declarations, + // i.e., their function definitions and get rid of the separate CallResolver. + var wouldResolveTo: Declaration? = null + if (current.resolutionHelper is CallExpression) { + // Peek into the declaration, and if it is a variable, we can proceed normally, as we + // are running into the special case explained above. Otherwise, we abort here (for + // now). + wouldResolveTo = scopeManager.resolveReference(current, current.scope) + if (wouldResolveTo !is VariableDeclaration) { + return } - is ConstructorCallExpression -> { - handleConstructorCallExpression(node) + } + + // Only consider resolving, if the language frontend did not specify a resolution. If we + // already have populated the wouldResolveTo variable, we can re-use this instead of + // resolving again + var refersTo = + current.refersTo + ?: wouldResolveTo ?: scopeManager.resolveReference(current, current.scope) + + var recordDeclType: Type? = null + if (currentClass != null) { + recordDeclType = currentClass.toType() + } + + val helperType = current.resolutionHelper?.type + if (helperType is FunctionPointerType && refersTo == null) { + refersTo = resolveMethodFunctionPointer(current, helperType) + } + + // only add new nodes for non-static unknown + if ( + refersTo == null && + !current.isStaticAccess && + recordDeclType != null && + recordDeclType.recordDeclaration != null + ) { + // Maybe we are referring to a field instead of a local var + if (language != null && language.namespaceDelimiter in current.name.toString()) { + recordDeclType = getEnclosingTypeOf(current) } - is ConstructExpression -> { - // We might have call expressions inside our arguments, so in order to correctly - // resolve this call's signature, we need to make sure any call expression arguments - // are fully resolved - handleArguments(node) - handleConstructExpression(node) + val field = resolveMember(recordDeclType, current) + if (field != null) { + refersTo = field } - is CallExpression -> { - // We might have call expressions inside our arguments, so in order to correctly - // resolve this call's signature, we need to make sure any call expression arguments - // are fully resolved - handleArguments(node) - handleCallExpression(scopeManager.currentRecord, node) + } + + // TODO: we need to do proper scoping (and merge it with the code above), but for now + // this just enables CXX static fields + if ( + refersTo == null && + language != null && + language.namespaceDelimiter in current.name.toString() + ) { + recordDeclType = getEnclosingTypeOf(current) + val field = resolveMember(recordDeclType, current) + if (field != null) { + refersTo = field } } + if (refersTo != null) { + current.refersTo = refersTo + } else { + Util.warnWithFileLocation( + current, + log, + "Did not find a declaration for ${current.name}" + ) + } + } + + /** + * We get the type of the "scope" this node is in. (e.g. for a field, we drop the field's name + * and have the class) + */ + protected fun getEnclosingTypeOf(current: Node): Type { + val language = current.language + + return if (language != null && language.namespaceDelimiter.isNotEmpty()) { + val parentName = (current.name.parent ?: current.name).toString() + current.objectType(parentName) + } else { + current.unknownType() + } + } + + protected fun handleMemberExpression(curClass: RecordDeclaration?, current: Node?) { + if (current !is MemberExpression) { + return + } + + // For legacy reasons, method and field resolving is split between the VariableUsageResolver + // and the CallResolver. Since we are trying to merge these two, the first step was to have + // the callee/member field of a MemberCallExpression set to a MemberExpression. This means + // however, that these will show up in this callback function. To not mess with legacy code + // (yet), we are ignoring all MemberExpressions whose parents are MemberCallExpressions in + // this function for now. + if (current.resolutionHelper is MemberCallExpression) { + return + } + + var baseTarget: Declaration? = null + if (current.base is Reference) { + val base = current.base as Reference + if ( + current.language is HasSuperClasses && + base.name.toString() == (current.language as HasSuperClasses).superClassKeyword + ) { + if (curClass != null && curClass.superClasses.isNotEmpty()) { + val superType = curClass.superClasses[0] + val superRecord = superType.recordDeclaration + if (superRecord == null) { + log.error( + "Could not find referring super type ${superType.typeName} for ${curClass.name} in the record map. Will set the super type to java.lang.Object" + ) + // TODO: Should be more generic! + base.type = current.objectType(Any::class.java.name) + } else { + // We need to connect this super reference to the receiver of this + // method + val func = scopeManager.currentFunction + if (func is MethodDeclaration) { + baseTarget = func.receiver + } + if (baseTarget != null) { + base.refersTo = baseTarget + // Explicitly set the type of the call's base to the super type + base.type = superType + (base.refersTo as? HasType)?.type = superType + // And set the assigned subtypes, to ensure, that really only our + // super type is in there + base.assignedTypes = mutableSetOf(superType) + (base.refersTo as? ValueDeclaration)?.assignedTypes = + mutableSetOf(superType) + } + } + } else { + // no explicit super type -> java.lang.Object + // TODO: Should be more generic + val objectType = current.objectType(Any::class.java.name) + base.type = objectType + } + } else { + // The base should have been resolved by now. Maybe we have some other clue about + // this base from the type system, so we can set the declaration accordingly. + if (base.refersTo == null) { + base.refersTo = base.type.recordDeclaration + } + } + } + + val baseType = current.base.type.root + current.refersTo = resolveMember(baseType, current) + } + + protected fun resolveMember(containingClass: Type, reference: Reference): ValueDeclaration? { + if (isSuperclassReference(reference)) { + // if we have a "super" on the member side, this is a member call. We need to resolve + // this in the call resolver instead + return null + } + var member: FieldDeclaration? = null + val record = containingClass.recordDeclaration + if (record != null) { + member = + record.fields + .filter { it.name.lastPartsMatch(reference.name) } + .map { it.definition } + .firstOrNull() + } + if (member == null) { + member = + containingClass.superTypes + .flatMap { it.fields } + .filter { it.name.lastPartsMatch(reference.name) } + .map { it.definition } + .firstOrNull() + } + return member ?: handleUnknownField(containingClass, reference) + } + + // TODO(oxisto): Move to inference class + protected fun handleUnknownField(base: Type, ref: Reference): FieldDeclaration? { + val name = ref.name + + // unwrap a potential pointer-type + if (base is PointerType) { + return handleUnknownField(base.elementType, ref) + } + + var record = base.recordDeclaration + if (record == null) { + // No matching record in the map? If we should infer it, we do so, otherwise we stop. + if (!config.inferenceConfiguration.inferRecords) return null + + // We access an unknown field of an unknown record. so we need to handle that + val kind = + if (base.language is HasStructs) { + "struct" + } else { + "class" + } + record = base.startInference(ctx).inferRecordDeclaration(base, currentTU, kind) + // update the type's record + if (record != null) { + base.recordDeclaration = record + } + } + + if (record == null) { + log.error( + "There is no matching record in the record map. Can't identify which field is used." + ) + return null + } + + val target = record.fields.firstOrNull { it.name.lastPartsMatch(name) } + + return if (target != null) { + target + } else { + val declaration = + record.newFieldDeclaration( + name.localName, + // we will set the type later through the type inference observer + unknownType(), + listOf(), + "", + null, + null, + false, + ) + record.addField(declaration) + declaration.isInferred = true + + // We might be able to resolve the type later (or better), if a type is + // assigned to our reference later + ref.registerTypeObserver(TypeInferenceObserver(declaration)) + + declaration + } + } + + protected fun handle(node: Node?, currClass: RecordDeclaration?) { + when (node) { + is MemberExpression -> handleMemberExpression(currClass, node) + is Reference -> handleReference(currClass, node) + is ConstructorCallExpression -> handleConstructorCallExpression(node) + is ConstructExpression -> handleConstructExpression(node) + is CallExpression -> handleCallExpression(scopeManager.currentRecord, node) + } } protected fun handleCallExpression(curClass: RecordDeclaration?, call: CallExpression) { @@ -190,8 +488,15 @@ open class CallResolver(ctx: TranslationContext) : SymbolResolverPass(ctx) { // If we do not have any candidates at this point, we will infer one. if (candidates.isEmpty()) { - // We need to see, whether we have any suitable base (e.g. a class) or not - val suitableBases = getPossibleContainingTypes(call) + // We need to see, whether we have any suitable base (e.g. a class) or not; but only if + // the call itself is not already scoped (e.g. to a namespace) + val suitableBases = + if (callee is MemberExpression || callee?.name?.isQualified() == false) { + getPossibleContainingTypes(call) + } else { + setOf() + } + candidates = if (suitableBases.isEmpty()) { // This is not really the most ideal place, but for now this will do. While this @@ -261,25 +566,6 @@ open class CallResolver(ctx: TranslationContext) : SymbolResolverPass(ctx) { } } - protected fun handleArguments(call: CallExpression) { - val worklist: Deque = ArrayDeque() - call.arguments.forEach { worklist.push(it) } - while (worklist.isNotEmpty()) { - val curr = worklist.pop() - if (curr is CallExpression) { - handleNode(curr) - } else { - val it = Strategy.AST_FORWARD(curr) - while (it.hasNext()) { - val astChild = it.next() - if (astChild !is RecordDeclaration) { - worklist.push(astChild) - } - } - } - } - } - /** * Resolves a [CallExpression.callee] of type [Reference] to a possible list of * [FunctionDeclaration] nodes. @@ -291,7 +577,7 @@ open class CallResolver(ctx: TranslationContext) : SymbolResolverPass(ctx) { ): List { val language = call.language - return if (curClass == null) { + return if (curClass == null || callee.name.isQualified()) { // Handle function (not method) calls. C++ allows function overloading. Make sure we // have at least the same number of arguments val candidates = @@ -305,32 +591,23 @@ open class CallResolver(ctx: TranslationContext) : SymbolResolverPass(ctx) { candidates } else { - resolveMemberCallee(callee, curClass, call) + resolveCalleeByName(callee.name.localName, curClass, call) } } /** * Resolves a [CallExpression.callee] of type [MemberExpression] to a possible list of * [FunctionDeclaration] nodes. - * - * TODO: Change callee to MemberExpression, but we can't since resolveReferenceCallee somehow - * delegates resolving of regular function calls within classes to this function (meh!) */ fun resolveMemberCallee( - callee: Reference, + callee: MemberExpression, curClass: RecordDeclaration?, call: CallExpression ): List { - // We need to resolve the base calls first. This might be done duplicate now - if (callee is MemberExpression && callee.base is CallExpression) { - handleCallExpression(curClass, callee.base as CallExpression) - } - // We need to adjust certain types of the base in case of a super call and we delegate this. - // If that is successful, we can continue with regular resolving + // If that is successful, we can continue with regular resolving. if ( curClass != null && - callee is MemberExpression && callee.base is Reference && isSuperclassReference(callee.base as Reference) ) { @@ -338,26 +615,46 @@ open class CallResolver(ctx: TranslationContext) : SymbolResolverPass(ctx) { callee, curClass, scopeManager, - recordMap ) } + return resolveCalleeByName(callee.name.localName, curClass, call) + } + + protected fun resolveCalleeByName( + localName: String, + curClass: RecordDeclaration?, + call: CallExpression + ): List { val possibleContainingTypes = getPossibleContainingTypes(call) - // Find function targets + // Find function targets. If our languages has a complex call resolution, we need to take + // this into account var invocationCandidates = - retrieveInvocationCandidatesFromCall(call, curClass, possibleContainingTypes) + if (call.language is HasComplexCallResolution) { + (call.language as HasComplexCallResolution) + .refineMethodCallResolution( + curClass, + possibleContainingTypes, + call, + ctx, + currentTU, + this + ) + .toMutableList() + } else { + scopeManager.resolveFunction(call).toMutableList() + } // Find invokes by supertypes if ( invocationCandidates.isEmpty() && - callee.name.localName.isNotEmpty() && - (!callee.language.isCPP || shouldSearchForInvokesInParent(call)) + localName.isNotEmpty() && + (!call.language.isCPP || shouldSearchForInvokesInParent(call)) ) { - val records = possibleContainingTypes.mapNotNull { recordMap[it.root.name] }.toSet() + val records = possibleContainingTypes.mapNotNull { it.root.recordDeclaration }.toSet() invocationCandidates = - getInvocationCandidatesFromParents(callee.name.localName, call, records) - .toMutableList() + getInvocationCandidatesFromParents(localName, call, records).toMutableList() } // Add overridden invokes @@ -371,27 +668,6 @@ open class CallResolver(ctx: TranslationContext) : SymbolResolverPass(ctx) { return invocationCandidates } - protected fun retrieveInvocationCandidatesFromCall( - call: CallExpression, - curClass: RecordDeclaration?, - possibleContainingTypes: Set - ): MutableList { - return if (call.language is HasComplexCallResolution) { - (call.language as HasComplexCallResolution) - .refineMethodCallResolution( - curClass, - possibleContainingTypes, - call, - ctx, - currentTU, - this - ) - .toMutableList() - } else { - scopeManager.resolveFunction(call).toMutableList() - } - } - /** * Creates an inferred element for each RecordDeclaration * @@ -404,11 +680,12 @@ open class CallResolver(ctx: TranslationContext) : SymbolResolverPass(ctx) { ): List { return possibleContainingTypes .mapNotNull { - var record = recordMap[it.root.name] - if (record == null && config.inferenceConfiguration.inferRecords == true) { + val root = it.root as? ObjectType + var record = root?.recordDeclaration + if (root != null && record == null && config.inferenceConfiguration.inferRecords) { record = it.startInference(ctx).inferRecordDeclaration(it, currentTU) - // update the record map - if (record != null) it.root.name.let { name -> recordMap[name] = record } + // update the record declaration + root.recordDeclaration = record } record } @@ -429,8 +706,7 @@ open class CallResolver(ctx: TranslationContext) : SymbolResolverPass(ctx) { protected fun handleConstructExpression(constructExpression: ConstructExpression) { if (constructExpression.instantiates != null && constructExpression.constructor != null) return - val typeName = constructExpression.type.name - val recordDeclaration = recordMap[typeName] + val recordDeclaration = constructExpression.type.root.recordDeclaration constructExpression.instantiates = recordDeclaration for (template in templateList) { if ( @@ -443,7 +719,7 @@ open class CallResolver(ctx: TranslationContext) : SymbolResolverPass(ctx) { template.parameters.size - constructExpression.templateParameters.size if (defaultDifference <= template.parameterDefaults.size) { // Check if predefined template value is used as default in next value - addRecursiveDefaultTemplateArgs(constructExpression, template) + addRecursiveDefaultTemplateArgs(constructExpression, template, scopeManager) // Add missing defaults val missingNewParams: List = @@ -455,7 +731,7 @@ open class CallResolver(ctx: TranslationContext) : SymbolResolverPass(ctx) { if (missingParam != null) { constructExpression.addTemplateParameter( missingParam, - TemplateInitialization.DEFAULT + TemplateDeclaration.TemplateInitialization.DEFAULT ) } } @@ -474,7 +750,8 @@ open class CallResolver(ctx: TranslationContext) : SymbolResolverPass(ctx) { constructorCallExpression: ConstructorCallExpression ) { constructorCallExpression.containingClass?.let { containingClass -> - val recordDeclaration = recordMap[constructorCallExpression.parseName(containingClass)] + val recordDeclaration = + constructorCallExpression.objectType(containingClass).recordDeclaration val signature = constructorCallExpression.arguments.map { it.type } if (recordDeclaration != null) { val constructor = @@ -504,35 +781,30 @@ open class CallResolver(ctx: TranslationContext) : SymbolResolverPass(ctx) { fun getInvocationCandidatesFromRecord( recordDeclaration: RecordDeclaration?, - name: String?, + name: String, call: CallExpression ): List { if (recordDeclaration == null) return listOf() - val namePattern = - Pattern.compile( - "(" + - Pattern.quote(recordDeclaration.name.toString()) + - Regex.escape(recordDeclaration.language?.namespaceDelimiter ?: "") + - ")?" + - Pattern.quote(name) - ) return if (call.language is HasComplexCallResolution) { (call.language as HasComplexCallResolution).refineInvocationCandidatesFromRecord( recordDeclaration, call, - namePattern, + name, ctx ) } else { - recordDeclaration.methods.filter { - namePattern.matcher(it.name).matches() && it.hasSignature(call.signature) - } + val scope = this.scopeManager.lookupScope(recordDeclaration) + val list = + this.scopeManager.resolve(scope) { + it.name.lastPartsMatch(name) && it.hasSignature(call.signature) + } + return list } } protected fun getInvocationCandidatesFromParents( - name: String?, + name: String, call: CallExpression, possibleTypes: Set ): List { @@ -571,7 +843,9 @@ open class CallResolver(ctx: TranslationContext) : SymbolResolverPass(ctx) { declaration: FunctionDeclaration ): Set { return declaration.overriddenBy - .filter { f -> containingType[f] in possibleSubTypes } + .filter { f -> + f is MethodDeclaration && f.recordDeclaration?.toType() in possibleSubTypes + } .toSet() } @@ -615,7 +889,8 @@ open class CallResolver(ctx: TranslationContext) : SymbolResolverPass(ctx) { // If we don't find any candidate and our current language is c/c++ we check if there is // a candidate with an implicit cast constructorCandidate = - resolveConstructorWithImplicitCast(constructExpression, recordDeclaration) + resolveWithImplicitCast(constructExpression, recordDeclaration.constructors) + .firstOrNull() as ConstructorDeclaration? } return constructorCandidate @@ -633,14 +908,14 @@ open class CallResolver(ctx: TranslationContext) : SymbolResolverPass(ctx) { } companion object { - val LOGGER: Logger = LoggerFactory.getLogger(CallResolver::class.java) + val LOGGER: Logger = LoggerFactory.getLogger(SymbolResolver::class.java) /** * Adds implicit duplicates of the TemplateParams to the implicit ConstructExpression * - * @param templateParams of the VariableDeclaration/NewExpressionp + * @param templateParams of the [VariableDeclaration]/[NewExpression] * @param constructExpression duplicate TemplateParameters (implicit) to preserve AST, as - * ConstructExpression uses AST as well as the VariableDeclaration/NewExpression + * [ConstructExpression] uses AST as well as the [VariableDeclaration]/[NewExpression] */ fun addImplicitTemplateParametersToCall( templateParams: List, @@ -656,3 +931,7 @@ open class CallResolver(ctx: TranslationContext) : SymbolResolverPass(ctx) { } } } + +interface ResolutionStartHolder { + val resolutionStartNodes: List +} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/TemplateCallResolverHelper.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/TemplateCallResolverHelper.kt index d0887d7cf2c..9830ab508d0 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/TemplateCallResolverHelper.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/TemplateCallResolverHelper.kt @@ -25,6 +25,7 @@ */ package de.fraunhofer.aisec.cpg.passes +import de.fraunhofer.aisec.cpg.ScopeManager import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.declarations.ParameterDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.RecordTemplateDeclaration @@ -47,7 +48,8 @@ import de.fraunhofer.aisec.cpg.graph.types.Type */ fun addRecursiveDefaultTemplateArgs( constructExpression: ConstructExpression, - template: RecordTemplateDeclaration + template: RecordTemplateDeclaration, + scopeManager: ScopeManager ) { var templateParameters: Int do { @@ -69,7 +71,8 @@ fun addRecursiveDefaultTemplateArgs( template, constructExpression, templateParametersExplicitInitialization, - templateParameterRealDefaultInitialization + templateParameterRealDefaultInitialization, + scopeManager ) } while (templateParameters != constructExpression.templateParameters.size) } @@ -112,7 +115,8 @@ fun applyMissingParams( template: RecordTemplateDeclaration, constructExpression: ConstructExpression, templateParametersExplicitInitialization: Map, - templateParameterRealDefaultInitialization: Map + templateParameterRealDefaultInitialization: Map, + scopeManager: ScopeManager ) { with(constructExpression) { val missingParams: List = @@ -123,6 +127,12 @@ fun applyMissingParams( for (m in missingParams) { var missingParam = m if (missingParam is Reference) { + if (missingParam.refersTo == null) { + val currentScope = scopeManager.currentScope + scopeManager.jumpTo(missingParam.scope) + missingParam.refersTo = scopeManager.resolveReference(missingParam) + scopeManager.jumpTo(currentScope) + } missingParam = missingParam.refersTo } if (missingParam in templateParametersExplicitInitialization) { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/VariableUsageResolver.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/VariableUsageResolver.kt deleted file mode 100644 index edb95d011e8..00000000000 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/VariableUsageResolver.kt +++ /dev/null @@ -1,460 +0,0 @@ -/* - * Copyright (c) 2019, Fraunhofer AISEC. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * $$$$$$\ $$$$$$$\ $$$$$$\ - * $$ __$$\ $$ __$$\ $$ __$$\ - * $$ / \__|$$ | $$ |$$ / \__| - * $$ | $$$$$$$ |$$ |$$$$\ - * $$ | $$ ____/ $$ |\_$$ | - * $$ | $$\ $$ | $$ | $$ | - * \$$$$$ |$$ | \$$$$$ | - * \______/ \__| \______/ - * - */ -package de.fraunhofer.aisec.cpg.passes - -import de.fraunhofer.aisec.cpg.TranslationContext -import de.fraunhofer.aisec.cpg.frontends.HasAnonymousIdentifier -import de.fraunhofer.aisec.cpg.frontends.HasStructs -import de.fraunhofer.aisec.cpg.frontends.HasSuperClasses -import de.fraunhofer.aisec.cpg.graph.* -import de.fraunhofer.aisec.cpg.graph.declarations.* -import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression -import de.fraunhofer.aisec.cpg.graph.statements.expressions.MemberCallExpression -import de.fraunhofer.aisec.cpg.graph.statements.expressions.MemberExpression -import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference -import de.fraunhofer.aisec.cpg.graph.types.* -import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker.ScopedWalker -import de.fraunhofer.aisec.cpg.helpers.Util -import de.fraunhofer.aisec.cpg.passes.inference.Inference.TypeInferenceObserver -import de.fraunhofer.aisec.cpg.passes.inference.startInference -import de.fraunhofer.aisec.cpg.passes.order.DependsOn -import org.slf4j.LoggerFactory - -/** - * Creates new connections between the place where a variable is declared and where it is used. - * - * A field access is modeled with a [MemberExpression]. After AST building, its base and member - * references are set to [Reference] stubs. This pass resolves those references and makes the member - * point to the appropriate [FieldDeclaration] and the base to the "this" [FieldDeclaration] of the - * containing class. It is also capable of resolving references to fields that are inherited from a - * superclass and thus not declared in the actual base class. When base or member declarations are - * not found in the graph, a new "inferred" [FieldDeclaration] is being created that is then used to - * collect all usages to the same unknown declaration. [Reference] stubs are removed from the graph - * after being resolved. - * - * Accessing a local variable is modeled directly with a [Reference]. This step of the pass doesn't - * remove the [Reference] nodes like in the field usage case but rather makes their "refersTo" point - * to the appropriate [ValueDeclaration]. - */ -@DependsOn(TypeHierarchyResolver::class) -open class VariableUsageResolver(ctx: TranslationContext) : SymbolResolverPass(ctx) { - - override fun accept(component: Component) { - walker = ScopedWalker(scopeManager) - for (tu in component.translationUnits) { - currentTU = tu - walker.clearCallbacks() - walker.registerHandler { node, _ -> findRecords(node) } - walker.registerHandler { node, _ -> findEnums(node) } - walker.iterate(currentTU) - } - - collectSupertypes() - - for (tu in component.translationUnits) { - walker.clearCallbacks() - walker.registerHandler { curClass, parent, node -> - resolveFieldUsages(curClass, parent, node) - } - walker.iterate(tu) - } - for (tu in component.translationUnits) { - walker.clearCallbacks() - walker.registerHandler(::resolveLocalVarUsage) - walker.iterate(tu) - } - } - - /** This function seems to resolve function pointers pointing to a [MethodDeclaration]. */ - protected fun resolveMethodFunctionPointer( - reference: Reference, - type: FunctionPointerType - ): ValueDeclaration { - val parent = reference.name.parent - - return handleUnknownFunction( - if (parent != null) { - recordMap[parent] - } else { - null - }, - reference.name, - type - ) - } - - protected fun resolveLocalVarUsage( - currentClass: RecordDeclaration?, - parent: Node?, - current: Node? - ) { - val language = current?.language - - if (current !is Reference || current is MemberExpression) return - - // We can safely ignore references to the anonymous identifier, e.g. _ in Go. - if ( - language is HasAnonymousIdentifier && - current.name.localName == language.anonymousIdentifier - ) { - return - } - - // For now, we need to ignore reference expressions that are directly embedded into call - // expressions, because they are the "callee" property. In the future, we will use this - // property to actually resolve the function call. However, there is a special case that - // we want to catch already, that is if we are "calling" a reference to a variable. This - // can be done in several languages, e.g., in C/C++ as function pointers or in Go as - // function references. In this case, we want to resolve the declared reference expression - // of this call expression back to its original variable declaration. In the future, we want - // to extend this particular code to resolve all callee references to their declarations, - // i.e., their function definitions and get rid of the separate CallResolver. - var wouldResolveTo: Declaration? = null - if (parent is CallExpression && parent.callee === current) { - // Peek into the declaration, and if it is a variable, we can proceed normally, as we - // are running into the special case explained above. Otherwise, we abort here (for - // now). - wouldResolveTo = scopeManager.resolveReference(current, current.scope) - if (wouldResolveTo !is VariableDeclaration) { - return - } - } - - // Only consider resolving, if the language frontend did not specify a resolution. If we - // already have populated the wouldResolveTo variable, we can re-use this instead of - // resolving again - var refersTo = - current.refersTo - ?: wouldResolveTo ?: scopeManager.resolveReference(current, current.scope) - - var recordDeclType: Type? = null - if (currentClass != null) { - recordDeclType = currentClass.toType() - } - - val helperType = current.resolutionHelper?.type - if (helperType is FunctionPointerType && refersTo == null) { - refersTo = resolveMethodFunctionPointer(current, helperType) - } - - // only add new nodes for non-static unknown - if ( - refersTo == null && - !current.isStaticAccess && - recordDeclType != null && - recordDeclType.name in recordMap - ) { - // Maybe we are referring to a field instead of a local var - if (language != null && language.namespaceDelimiter in current.name.toString()) { - recordDeclType = getEnclosingTypeOf(current) - } - val field = resolveMember(recordDeclType, current) - if (field != null) { - refersTo = field - } - } - - // TODO: we need to do proper scoping (and merge it with the code above), but for now - // this just enables CXX static fields - if ( - refersTo == null && - language != null && - language.namespaceDelimiter in current.name.toString() - ) { - recordDeclType = getEnclosingTypeOf(current) - val field = resolveMember(recordDeclType, current) - if (field != null) { - refersTo = field - } - } - if (refersTo != null) { - current.refersTo = refersTo - } else { - Util.warnWithFileLocation( - current, - log, - "Did not find a declaration for ${current.name}" - ) - } - } - - /** - * We get the type of the "scope" this node is in. (e.g. for a field, we drop the field's name - * and have the class) - */ - protected fun getEnclosingTypeOf(current: Node): Type { - val language = current.language - - return if (language != null && language.namespaceDelimiter.isNotEmpty()) { - val parentName = (current.name.parent ?: current.name).toString() - current.objectType(parentName) - } else { - current.unknownType() - } - } - - protected fun resolveFieldUsages(curClass: RecordDeclaration?, parent: Node?, current: Node?) { - if (current !is MemberExpression) { - return - } - - // For legacy reasons, method and field resolving is split between the VariableUsageResolver - // and the CallResolver. Since we are trying to merge these two, the first step was to have - // the callee/member field of a MemberCallExpression set to a MemberExpression. This means - // however, that these will show up in this callback function. To not mess with legacy code - // (yet), we are ignoring all MemberExpressions whose parents are MemberCallExpressions in - // this function for now. - if (parent is MemberCallExpression && parent.callee == current) { - return - } - - var baseTarget: Declaration? = null - if (current.base is Reference) { - val base = current.base as Reference - if ( - current.language is HasSuperClasses && - base.name.toString() == (current.language as HasSuperClasses).superClassKeyword - ) { - if (curClass != null && curClass.superClasses.isNotEmpty()) { - val superType = curClass.superClasses[0] - val superRecord = recordMap[superType.name] - if (superRecord == null) { - log.error( - "Could not find referring super type ${superType.typeName} for ${curClass.name} in the record map. Will set the super type to java.lang.Object" - ) - // TODO: Should be more generic! - base.type = current.objectType(Any::class.java.name) - } else { - // We need to connect this super reference to the receiver of this - // method - val func = scopeManager.currentFunction - if (func is MethodDeclaration) { - baseTarget = func.receiver - } - if (baseTarget != null) { - base.refersTo = baseTarget - // Explicitly set the type of the call's base to the super type - base.type = superType - (base.refersTo as? HasType)?.type = superType - // And set the assigned subtypes, to ensure, that really only our - // super type is in there - base.assignedTypes = mutableSetOf(superType) - (base.refersTo as? ValueDeclaration)?.assignedTypes = - mutableSetOf(superType) - } - } - } else { - // no explicit super type -> java.lang.Object - // TODO: Should be more generic - val objectType = current.objectType(Any::class.java.name) - base.type = objectType - } - } else { - baseTarget = resolveBase(current.base as Reference) - base.refersTo = baseTarget - } - if (baseTarget is EnumDeclaration) { - val memberTarget = - baseTarget.entries.firstOrNull { it.name.lastPartsMatch(current.name) } - if (memberTarget != null) { - current.refersTo = memberTarget - return - } - } else if (baseTarget is RecordDeclaration) { - var baseType = baseTarget.toType() - if (baseType.name !in recordMap) { - val containingT = baseType - val fqnResolvedType = - recordMap.keys.firstOrNull { it.lastPartsMatch(containingT.name) } - if (fqnResolvedType != null) { - baseType = baseTarget.objectType(fqnResolvedType) - } - } - current.refersTo = resolveMember(baseType, current) - return - } - } - var baseType = current.base.type - if (baseType.name !in recordMap) { - val fqnResolvedType = recordMap.keys.firstOrNull { it.lastPartsMatch(baseType.name) } - if (fqnResolvedType != null) { - baseType = current.base.objectType(fqnResolvedType) - } - } - current.refersTo = resolveMember(baseType, current) - } - - protected fun resolveBase(reference: Reference): Declaration? { - // We need to check, whether we first need to resolve our own base - if (reference is MemberExpression) { - val base = reference.base - if (base is Reference && base.refersTo == null) { - base.refersTo = resolveBase(base) - } - } - - val declaration = scopeManager.resolveReference(reference) - if (declaration != null) { - return declaration - } - - // check if this refers to an enum - return if (reference.type in enumMap) { - enumMap[reference.type] - } else if (reference.type.name in recordMap) { - recordMap[reference.type.name] - } else { - null - } - } - - protected fun resolveMember(containingClass: Type, reference: Reference): ValueDeclaration? { - if (isSuperclassReference(reference)) { - // if we have a "super" on the member side, this is a member call. We need to resolve - // this in the call resolver instead - return null - } - var member: FieldDeclaration? = null - if (containingClass !is UnknownType && containingClass.name in recordMap) { - member = - recordMap[containingClass.name] - .fields - .filter { it.name.lastPartsMatch(reference.name) } - .map { it.definition } - .firstOrNull() - } - if (member == null) { - member = - superTypesMap - .getOrDefault(containingClass.name, listOf()) - .mapNotNull { recordMap[it.name] } - .flatMap { it.fields } - .filter { it.name.lastPartsMatch(reference.name) } - .map { it.definition } - .firstOrNull() - } - return member ?: handleUnknownField(containingClass, reference) - } - - // TODO(oxisto): Move to inference class - protected fun handleUnknownField(base: Type, ref: Reference): FieldDeclaration? { - val name = ref.name - - // unwrap a potential pointer-type - if (base is PointerType) { - return handleUnknownField(base.elementType, ref) - } - - if (base.name !in recordMap) { - // No matching record in the map? If we should infer it, we do so, otherwise we stop. - if (!config.inferenceConfiguration.inferRecords) return null - - // We access an unknown field of an unknown record. so we need to handle that - val kind = - if (base.language is HasStructs) { - "struct" - } else { - "class" - } - val record = base.startInference(ctx).inferRecordDeclaration(base, currentTU, kind) - // update the record map - if (record != null) recordMap[base.name] = record - } - - val recordDeclaration = recordMap[base.name] - if (recordDeclaration == null) { - log.error( - "There is no matching record in the record map. Can't identify which field is used." - ) - return null - } - - val target = recordDeclaration.fields.firstOrNull { it.name.lastPartsMatch(name) } - - return if (target != null) { - target - } else { - val declaration = - recordDeclaration.newFieldDeclaration( - name.localName, - // we will set the type later through the type inference observer - unknownType(), - listOf(), - "", - null, - null, - false, - ) - recordDeclaration.addField(declaration) - declaration.isInferred = true - - // We might be able to resolve the type later (or better), if a type is - // assigned to our reference later - ref.registerTypeObserver(TypeInferenceObserver(declaration)) - - declaration - } - } - - /** - * Generates a [MethodDeclaration] if the [declarationHolder] is a [RecordDeclaration] or a - * [FunctionDeclaration] if the [declarationHolder] is a [TranslationUnitDeclaration]. The - * resulting function/method has the signature and return type specified in [fctPtrType] and the - * specified [name]. - */ - protected fun handleUnknownFunction( - declarationHolder: RecordDeclaration?, - name: Name, - fctPtrType: FunctionPointerType - ): FunctionDeclaration { - // Try to find the function or method in the list of existing functions. - val target = - if (declarationHolder != null) { - declarationHolder.methods.firstOrNull { f -> - f.matches(name, fctPtrType.returnType, fctPtrType.parameters) - } - } else { - currentTU.functions.firstOrNull { f -> - f.matches(name, fctPtrType.returnType, fctPtrType.parameters) - } - } - // If we didn't find anything, we create a new function or method declaration - return target - ?: (declarationHolder ?: currentTU) - .startInference(ctx) - .createInferredFunctionDeclaration( - name, - null, - false, - fctPtrType.parameters, - fctPtrType.returnType - ) - } - - companion object { - protected val log = LoggerFactory.getLogger(VariableUsageResolver::class.java) - } -} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/Inference.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/Inference.kt index d8783edbe6d..161fd6cf26f 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/Inference.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/Inference.kt @@ -341,7 +341,7 @@ class Inference(val start: Node, override val ctx: TranslationContext) : // delegate further operations to the scope manager. We also save the old scope so we can // restore it. return inferInScopeOf(start) { - Companion.log.debug( + log.debug( "Inferring a new namespace declaration {} {}", name, if (path != null) { diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/GraphExamples.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/GraphExamples.kt index adf48e0eb29..7e250c1b647 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/GraphExamples.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/GraphExamples.kt @@ -28,6 +28,7 @@ package de.fraunhofer.aisec.cpg import de.fraunhofer.aisec.cpg.frontends.StructTestLanguage import de.fraunhofer.aisec.cpg.frontends.TestLanguage import de.fraunhofer.aisec.cpg.frontends.TestLanguageFrontend +import de.fraunhofer.aisec.cpg.graph.autoType import de.fraunhofer.aisec.cpg.graph.builder.* import de.fraunhofer.aisec.cpg.graph.newInitializerListExpression import de.fraunhofer.aisec.cpg.graph.newVariableDeclaration @@ -136,67 +137,41 @@ class GraphExamples { ) = testFrontend(config).build { translationResult { - translationUnit("initializerListExprDFG.cpp") { - function("foo", t("int")) { body { returnStmt { literal(0, t("int")) } } } - function("main", t("int")) { - body { - declare { - variable("i", t("int")) { - val initList = newInitializerListExpression() - initList.initializers = listOf(call("foo")) - initializer = initList + translationUnit("Variables.java") { + record("Variables") { + field("field", t("int")) { + literal(42, t("int")) + modifiers = listOf("private") + } + method("getField", t("int")) { + receiver = newVariableDeclaration("this", t("Variables")) + body { returnStmt { member("field") } } + } + method("getLocal", t("int")) { + receiver = newVariableDeclaration("this", t("Variables")) + body { + declare { + variable("local", t("int")) { literal(42, t("int")) } } + returnStmt { ref("local") } } - returnStmt { ref("i") } - - translationUnit("Variables.java") { - record("Variables") { - field("field", t("int")) { - literal(42, t("int")) - modifiers = listOf("private") - } - method("getField", t("int")) { - receiver = - newVariableDeclaration("this", t("Variables")) - body { returnStmt { member("field") } } - } - method("getLocal", t("int")) { - receiver = - newVariableDeclaration("this", t("Variables")) - body { - declare { - variable("local", t("int")) { - literal(42, t("int")) - } - } - returnStmt { ref("local") } - } - } - method("getShadow", t("int")) { - receiver = - newVariableDeclaration("this", t("Variables")) - body { - declare { - variable("field", t("int")) { - literal(43, t("int")) - } - } - returnStmt { ref("field") } - } - } - method("getNoShadow", t("int")) { - receiver = - newVariableDeclaration("this", t("Variables")) - body { - declare { - variable("field", t("int")) { - literal(43, t("int")) - } - } - returnStmt { member("field", ref("this")) } - } - } + } + method("getShadow", t("int")) { + receiver = newVariableDeclaration("this", t("Variables")) + body { + declare { + variable("field", t("int")) { literal(43, t("int")) } + } + returnStmt { ref("field") } + } + } + method("getNoShadow", t("int")) { + receiver = newVariableDeclaration("this", t("Variables")) + body { + declare { + variable("field", t("int")) { literal(43, t("int")) } } + returnStmt { member("field", ref("this")) } } } } @@ -1002,5 +977,60 @@ class GraphExamples { } } } + + /** + * This roughly represents the following Java Code: + * ```java + * public class TestClass { + * public TestClass(int i) { + * + * }; + * + * public TestClass method1() { + * return new TestClass(4); + * } + * + * public void method2() { + * var variable = this.method1(); + * variable.method2(); + * return; + * } + * } + * ``` + */ + fun getCombinedVariableAndCallTest( + config: TranslationConfiguration = + TranslationConfiguration.builder() + .defaultPasses() + .registerLanguage(TestLanguage(".")) + .build() + ) = + testFrontend(config).build { + translationResult { + translationUnit("CombinedVariableAndCall.java") { + record("TestClass") { + constructor { param("i", t("int")) } + method("method1", t("TestClass")) { + body { + returnStmt { construct("TestClass") { literal(4, t("int")) } } + } + } + + method("method2") { + receiver("this", t("TestClass")) + body { + declare { + variable("variable", autoType()) { + memberCall("method1", ref("this")) + } + } + + memberCall("method2", ref("variable")) + } + } + } + } + } + } } } diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/FluentTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/FluentTest.kt index f9be5aa68a1..96409f76f13 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/FluentTest.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/FluentTest.kt @@ -37,7 +37,8 @@ import de.fraunhofer.aisec.cpg.graph.statements.DeclarationStatement import de.fraunhofer.aisec.cpg.graph.statements.IfStatement import de.fraunhofer.aisec.cpg.graph.statements.ReturnStatement import de.fraunhofer.aisec.cpg.graph.statements.expressions.* -import de.fraunhofer.aisec.cpg.passes.VariableUsageResolver +import de.fraunhofer.aisec.cpg.passes.EvaluationOrderGraphPass +import de.fraunhofer.aisec.cpg.passes.SymbolResolver import kotlin.test.* class FluentTest { @@ -167,7 +168,9 @@ class FluentTest { assertNotNull(lit2.scope) assertEquals(2, lit2.value) - VariableUsageResolver(result.finalCtx).accept(result.components.first()) + EvaluationOrderGraphPass(result.finalCtx) + .accept(result.components.first().translationUnits.first()) + SymbolResolver(result.finalCtx).accept(result.components.first()) // Now the reference should be resolved and the MCE name set assertRefersTo(ref, variable) diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/types/TypePropagationTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/types/TypePropagationTest.kt index 45946b8b880..bfdae5d934c 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/types/TypePropagationTest.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/types/TypePropagationTest.kt @@ -34,16 +34,23 @@ import de.fraunhofer.aisec.cpg.graph.statements.ReturnStatement import de.fraunhofer.aisec.cpg.graph.statements.expressions.AssignExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.Block import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference -import de.fraunhofer.aisec.cpg.passes.ControlFlowSensitiveDFGPass -import de.fraunhofer.aisec.cpg.passes.EvaluationOrderGraphPass -import de.fraunhofer.aisec.cpg.passes.VariableUsageResolver import kotlin.test.* class TypePropagationTest { @Test fun testBinopTypePropagation() { + val frontend = + TestLanguageFrontend( + ctx = + TranslationContext( + TranslationConfiguration.builder().defaultPasses().build(), + ScopeManager(), + TypeManager() + ) + ) + val result = - TestLanguageFrontend().build { + frontend.build { translationResult { translationUnit("test") { function("main", t("int")) { @@ -62,8 +69,6 @@ class TypePropagationTest { } } - VariableUsageResolver(result.finalCtx).accept(result.components.first()) - val intVar = result.variables["intVar"] assertNotNull(intVar) assertLocalName("int", intVar.type) @@ -85,7 +90,15 @@ class TypePropagationTest { @Test fun testAssignTypePropagation() { - val frontend = TestLanguageFrontend() + val frontend = + TestLanguageFrontend( + ctx = + TranslationContext( + TranslationConfiguration.builder().defaultPasses().build(), + ScopeManager(), + TypeManager() + ) + ) /** * This roughly represents the following program in C: @@ -117,12 +130,6 @@ class TypePropagationTest { } } - VariableUsageResolver(result.finalCtx).accept(result.components.first()) - EvaluationOrderGraphPass(result.finalCtx) - .accept(result.components.first().translationUnits.first()) - ControlFlowSensitiveDFGPass(result.finalCtx) - .accept(result.components.first().translationUnits.first()) - with(frontend) { val main = result.functions["main"] assertNotNull(main) @@ -165,7 +172,15 @@ class TypePropagationTest { @Test fun testNewPropagation() { - val frontend = TestLanguageFrontend() + val frontend = + TestLanguageFrontend( + ctx = + TranslationContext( + TranslationConfiguration.builder().defaultPasses().build(), + ScopeManager(), + TypeManager() + ) + ) /** * This roughly represents the following C++ code: @@ -197,8 +212,6 @@ class TypePropagationTest { } } - VariableUsageResolver(result.finalCtx).accept(result.components.first()) - with(frontend) { val main = result.functions["main"] assertNotNull(main) diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/DFGTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/DFGTest.kt index 5b3228ed421..441ded7cbb2 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/DFGTest.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/DFGTest.kt @@ -368,9 +368,10 @@ class DFGTest { val b = result.variables["b"] assertNotNull(b) - val ab = b.prevEOG[0] as Reference + val ab = b.nextEOG[0] as Reference val literal4 = result.literals[{ it.value == 4 }] assertNotNull(literal4) + val a4 = ab.prevDFG.first { it is Reference } assertTrue(literal4.nextDFG.contains(a4)) assertEquals(1, ab.prevDFG.size) @@ -419,7 +420,7 @@ class DFGTest { assertEquals(1, aPrintln.nextEOG.size) assertEquals(println, aPrintln.nextEOG[0]) - val ab = b.prevEOG[0] as Reference + val ab = b.nextEOG[0] as Reference assertTrue(refersTo.nextDFG.contains(ab)) assertTrue(a2.nextDFG.contains(ab)) } @@ -435,7 +436,7 @@ class DFGTest { val b = result.variables["b"] assertNotNull(b) - val ab = b.prevEOG[0] as Reference + val ab = b.nextEOG[0] as Reference val a10 = result.refs[{ TestUtils.compareLineFromLocationIfExists(it, true, 8) }] val a11 = result.refs[{ TestUtils.compareLineFromLocationIfExists(it, true, 11) }] val a12 = result.refs[{ TestUtils.compareLineFromLocationIfExists(it, true, 14) }] diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/SymbolResolverTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/SymbolResolverTest.kt new file mode 100644 index 00000000000..0f141ca244f --- /dev/null +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/SymbolResolverTest.kt @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2023, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.passes + +import de.fraunhofer.aisec.cpg.GraphExamples +import de.fraunhofer.aisec.cpg.TestUtils.assertInvokes +import de.fraunhofer.aisec.cpg.TestUtils.assertRefersTo +import de.fraunhofer.aisec.cpg.graph.* +import de.fraunhofer.aisec.cpg.graph.statements.expressions.ConstructExpression +import de.fraunhofer.aisec.cpg.graph.statements.expressions.MemberCallExpression +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertIs +import kotlin.test.assertNotNull + +class SymbolResolverTest { + @Test + fun testCombinedVariableAndCallResolution() { + val result = GraphExamples.getCombinedVariableAndCallTest() + + val type = result.records["TestClass"]?.toType() + assertNotNull(type) + + val method1 = result.methods["method1"] + assertNotNull(method1) + + val method2 = result.methods["method2"] + assertNotNull(method2) + + val constructor = result.methods["TestClass"] + assertNotNull(constructor) + + val variable = method2.variables["variable"] + assertEquals(type, variable?.type) + + val ref = method2.refs["variable"] + assertEquals(type, ref?.type) + + val callmethod1 = method2.calls["method1"] + assertIs(callmethod1) + assertRefersTo(callmethod1.base, method2.receiver) + assertInvokes(callmethod1, method1) + + val callmethod2 = method2.calls["method2"] + assertInvokes(callmethod2, method2) + + val construct = method1.calls { it is ConstructExpression }.firstOrNull() + assertNotNull(construct) + assertInvokes(construct, constructor) + } +} diff --git a/cpg-core/src/testFixtures/kotlin/de/fraunhofer/aisec/cpg/TestUtils.kt b/cpg-core/src/testFixtures/kotlin/de/fraunhofer/aisec/cpg/TestUtils.kt index 1f5ca2cab3f..a53fadb5e83 100644 --- a/cpg-core/src/testFixtures/kotlin/de/fraunhofer/aisec/cpg/TestUtils.kt +++ b/cpg-core/src/testFixtures/kotlin/de/fraunhofer/aisec/cpg/TestUtils.kt @@ -265,7 +265,7 @@ object TestUtils { * @param usedMember * - THe expected member that is used */ - fun assertUsageOfMemberAndBase(usingNode: Node?, usedBase: Node?, usedMember: Node?) { + fun assertUsageOfMemberAndBase(usingNode: Node?, usedBase: Node?, usedMember: Declaration?) { assertNotNull(usingNode) if (usingNode !is MemberExpression && !ENFORCE_MEMBER_EXPRESSION) { // Assumtion here is that the target of the member portion of the expression and not the diff --git a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CLanguage.kt b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CLanguage.kt index 5d7f0cc4cb9..041f884d3e7 100644 --- a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CLanguage.kt +++ b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CLanguage.kt @@ -33,9 +33,8 @@ import de.fraunhofer.aisec.cpg.graph.declarations.RecordDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression import de.fraunhofer.aisec.cpg.graph.types.* -import de.fraunhofer.aisec.cpg.passes.CallResolver +import de.fraunhofer.aisec.cpg.passes.SymbolResolver import de.fraunhofer.aisec.cpg.passes.resolveWithImplicitCast -import java.util.regex.Pattern import kotlin.reflect.KClass import org.neo4j.ogm.annotation.Transient @@ -130,13 +129,13 @@ open class CLanguage : call: CallExpression, ctx: TranslationContext, currentTU: TranslationUnitDeclaration, - callResolver: CallResolver + callResolver: SymbolResolver ): List = emptyList() override fun refineInvocationCandidatesFromRecord( recordDeclaration: RecordDeclaration, call: CallExpression, - namePattern: Pattern, + name: String, ctx: TranslationContext ): List = emptyList() diff --git a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CPPLanguage.kt b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CPPLanguage.kt index 8a39561a659..6cd6312b0d7 100644 --- a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CPPLanguage.kt +++ b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CPPLanguage.kt @@ -35,7 +35,6 @@ import de.fraunhofer.aisec.cpg.graph.statements.expressions.MemberCallExpression import de.fraunhofer.aisec.cpg.graph.types.* import de.fraunhofer.aisec.cpg.passes.* import de.fraunhofer.aisec.cpg.passes.inference.startInference -import java.util.regex.Pattern import org.neo4j.ogm.annotation.Transient /** The C++ language. */ @@ -117,11 +116,10 @@ class CPPLanguage : call: CallExpression, ctx: TranslationContext, currentTU: TranslationUnitDeclaration, - callResolver: CallResolver + callResolver: SymbolResolver ): List { var invocationCandidates = mutableListOf() - val records = - possibleContainingTypes.mapNotNull { callResolver.recordMap[it.root.name] }.toSet() + val records = possibleContainingTypes.mapNotNull { it.root.recordDeclaration }.toSet() for (record in records) { invocationCandidates.addAll( callResolver.getInvocationCandidatesFromRecord(record, call.name.localName, call) @@ -158,15 +156,13 @@ class CPPLanguage : override fun refineInvocationCandidatesFromRecord( recordDeclaration: RecordDeclaration, call: CallExpression, - namePattern: Pattern, + name: String, ctx: TranslationContext ): List { val invocationCandidate = mutableListOf( *recordDeclaration.methods - .filter { m -> - namePattern.matcher(m.name).matches() && m.hasSignature(call.signature) - } + .filter { m -> m.name.lastPartsMatch(name) && m.hasSignature(call.signature) } .toTypedArray() ) if (invocationCandidate.isEmpty()) { @@ -175,8 +171,8 @@ class CPPLanguage : resolveWithDefaultArgs( call, recordDeclaration.methods.filter { m -> - (namePattern.matcher(m.name).matches() /*&& !m.isImplicit()*/ && - call.signature.size < m.signatureTypes.size) + m.name.lastPartsMatch(name) /*&& !m.isImplicit()*/ && + call.signature.size < m.signatureTypes.size } ) ) @@ -187,7 +183,7 @@ class CPPLanguage : resolveWithImplicitCast( call, recordDeclaration.methods.filter { m -> - namePattern.matcher(m.name).matches() /*&& !m.isImplicit()*/ + m.name.lastPartsMatch(name) /*&& !m.isImplicit()*/ } ) ) diff --git a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXLanguageFrontend.kt b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXLanguageFrontend.kt index ab0713f3d30..f7a5b3df517 100644 --- a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXLanguageFrontend.kt +++ b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXLanguageFrontend.kt @@ -37,6 +37,7 @@ import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression import de.fraunhofer.aisec.cpg.graph.types.* import de.fraunhofer.aisec.cpg.helpers.Benchmark import de.fraunhofer.aisec.cpg.helpers.Util +import de.fraunhofer.aisec.cpg.passes.CXXExtraPass import de.fraunhofer.aisec.cpg.passes.FunctionPointerCallResolver import de.fraunhofer.aisec.cpg.passes.order.RegisterExtraPass import de.fraunhofer.aisec.cpg.sarif.PhysicalLocation @@ -78,6 +79,7 @@ import org.slf4j.LoggerFactory * and C++ code. */ @RegisterExtraPass(FunctionPointerCallResolver::class) +@RegisterExtraPass(CXXExtraPass::class) class CXXLanguageFrontend(language: Language, ctx: TranslationContext) : LanguageFrontend(language, ctx) { @@ -498,10 +500,7 @@ class CXXLanguageFrontend(language: Language, ctx: Translat // translation unit resolveAlias = true - val decl = - scopeManager.currentScope?.let { - scopeManager.getRecordForName(it, Name(name)) - } + val decl = scopeManager.getRecordForName(Name(name)) // We found a symbol, so we can use its name if (decl != null) { diff --git a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/ExpressionHandler.kt b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/ExpressionHandler.kt index babe8116ab2..ac785bda0b4 100644 --- a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/ExpressionHandler.kt +++ b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/ExpressionHandler.kt @@ -31,7 +31,7 @@ import de.fraunhofer.aisec.cpg.graph.declarations.MethodDeclaration import de.fraunhofer.aisec.cpg.graph.statements.expressions.* import de.fraunhofer.aisec.cpg.graph.types.* import de.fraunhofer.aisec.cpg.helpers.Util -import de.fraunhofer.aisec.cpg.passes.CallResolver +import de.fraunhofer.aisec.cpg.passes.SymbolResolver.Companion.addImplicitTemplateParametersToCall import java.math.BigInteger import java.util.* import java.util.function.Supplier @@ -217,10 +217,7 @@ class ExpressionHandler(lang: CXXLanguageFrontend) : // expression since the construct expression will do the actual template instantiation if (newExpression.templateParameters?.isNotEmpty() == true) { newExpression.templateParameters?.let { - CallResolver.addImplicitTemplateParametersToCall( - it, - initializer as ConstructExpression - ) + addImplicitTemplateParametersToCall(it, initializer as ConstructExpression) } } diff --git a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/CXXExtraPass.kt b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/CXXExtraPass.kt new file mode 100644 index 00000000000..a1ab2689869 --- /dev/null +++ b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/CXXExtraPass.kt @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2023, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.passes + +import de.fraunhofer.aisec.cpg.TranslationContext +import de.fraunhofer.aisec.cpg.graph.Component +import de.fraunhofer.aisec.cpg.graph.Node +import de.fraunhofer.aisec.cpg.graph.declarations.RecordDeclaration +import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration +import de.fraunhofer.aisec.cpg.graph.newConstructExpression +import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression +import de.fraunhofer.aisec.cpg.graph.statements.expressions.ConstructExpression +import de.fraunhofer.aisec.cpg.graph.types.recordDeclaration +import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker +import de.fraunhofer.aisec.cpg.passes.order.DependsOn +import de.fraunhofer.aisec.cpg.passes.order.ExecuteBefore + +/** + * This [Pass] executes certain C++ specific conversions on initializers, that are only possible + * once we know all the types. It may be extended in the future with other things that we currently + * still do in the frontend, but might be more accurate to do once we parsed all files and have all + * type information. + */ +@ExecuteBefore(EvaluationOrderGraphPass::class) +@DependsOn(TypeResolver::class) +class CXXExtraPass(ctx: TranslationContext) : ComponentPass(ctx) { + override fun accept(component: Component) { + val walker = SubgraphWalker.ScopedWalker(ctx.scopeManager) + + walker.registerHandler(::fixInitializers) + for (tu in component.translationUnits) { + walker.iterate(tu) + } + } + + protected fun fixInitializers(node: Node?, currClass: RecordDeclaration?) { + if (node is VariableDeclaration) { + // check if we have the corresponding class for this type + val record = node.type.root.recordDeclaration + val typeString = node.type.root.name + if (record != null) { + val currInitializer = node.initializer + if (currInitializer == null && node.isImplicitInitializerAllowed) { + val initializer = node.newConstructExpression(typeString, "$typeString()") + initializer.type = node.type + initializer.isImplicit = true + node.initializer = initializer + node.templateParameters?.let { + SymbolResolver.addImplicitTemplateParametersToCall(it, initializer) + } + } else if ( + currInitializer !is ConstructExpression && + currInitializer is CallExpression && + currInitializer.name.localName == node.type.root.name.localName + ) { + // This should actually be a construct expression, not a call! + val arguments = currInitializer.arguments + val signature = arguments.map(Node::code).joinToString(", ") + val initializer = + node.newConstructExpression(typeString, "$typeString($signature)") + initializer.type = node.type + initializer.arguments = mutableListOf(*arguments.toTypedArray()) + initializer.isImplicit = true + node.initializer = initializer + currInitializer.disconnectFromGraph() + } + } + } + } + + override fun cleanup() { + // Nothing to do + } +} diff --git a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/FunctionPointerCallResolver.kt b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/FunctionPointerCallResolver.kt index 76c565bf269..a630903eba7 100644 --- a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/FunctionPointerCallResolver.kt +++ b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/FunctionPointerCallResolver.kt @@ -52,7 +52,7 @@ import java.util.function.Consumer * extend it to other languages that support some kind of function reference/pointer calling, such * as Go. */ -@DependsOn(CallResolver::class) +@DependsOn(SymbolResolver::class) @DependsOn(DFGPass::class) @RequiredFrontend(CXXLanguageFrontend::class) class FunctionPointerCallResolver(ctx: TranslationContext) : ComponentPass(ctx) { diff --git a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/EOGTest.kt b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/EOGTest.kt index 3d304f2556b..d5202124e5d 100644 --- a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/EOGTest.kt +++ b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/EOGTest.kt @@ -930,10 +930,10 @@ internal class EOGTest : BaseTest() { assertNotNull(lambda) // The "outer" EOG is assembled correctly. - assertTrue(lambda in lambdaVar.prevEOG) + assertTrue(lambda in lambdaVar.nextEOG) val printFunctionCall = function.calls["print_function"] assertNotNull(printFunctionCall) - assertTrue(printFunctionCall in lambda.prevEOG) + assertTrue(printFunctionCall in lambdaVar.prevEOG) // The "inner" EOG is assembled correctly. val body = (lambda.function?.body as? Block) diff --git a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXLanguageFrontendTest.kt b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXLanguageFrontendTest.kt index f481972eead..19f2b3d5d3b 100644 --- a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXLanguageFrontendTest.kt +++ b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXLanguageFrontendTest.kt @@ -1531,8 +1531,8 @@ internal class CXXLanguageFrontendTest : BaseTest() { it.registerLanguage() it.registerPass() it.registerPass() - it.registerPass() - it.registerPass() // creates CG + it.registerPass() + it.registerPass() it.registerPass() it.registerPass() // creates EOG it.registerPass() @@ -1577,8 +1577,8 @@ internal class CXXLanguageFrontendTest : BaseTest() { it.registerLanguage() it.registerPass() it.registerPass() - it.registerPass() - it.registerPass() // creates CG + it.registerPass() + it.registerPass() it.registerPass() it.registerPass() // creates EOG it.registerPass() diff --git a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/LambdaTest.kt b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/LambdaTest.kt index f2234553973..919142cedbb 100644 --- a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/LambdaTest.kt +++ b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/LambdaTest.kt @@ -56,11 +56,11 @@ class CPPLambdaTest { val lambda = lambdaVar.initializer as? LambdaExpression assertNotNull(lambda) - assertTrue(lambda in lambdaVar.prevEOG) + assertTrue(lambda in lambdaVar.nextEOG) val printFunctionCall = function.calls["print_function"] assertNotNull(printFunctionCall) - assertTrue(printFunctionCall in lambda.prevEOG) + assertTrue(printFunctionCall in lambdaVar.prevEOG) val lambdaCall = function.calls["this_is_a_lambda"] assertEquals(1, lambdaCall?.invokes?.size) @@ -95,10 +95,10 @@ class CPPLambdaTest { assertNotNull(numberRef) assertEquals(lambda.function?.parameters?.firstOrNull(), numberRef.refersTo) - assertTrue(lambda in lambdaVar.prevEOG) + assertTrue(lambda in lambdaVar.nextEOG) val printFunctionCall = function.calls["print_function"] assertNotNull(printFunctionCall) - assertTrue(printFunctionCall in lambda.prevEOG) + assertTrue(printFunctionCall in lambdaVar.prevEOG) } @Test diff --git a/cpg-language-cxx/src/test/resources/variables_extended/cpp/scope_variables.cpp b/cpg-language-cxx/src/test/resources/variables_extended/cpp/scope_variables.cpp index 0d5e395b66f..2546c166a42 100644 --- a/cpg-language-cxx/src/test/resources/variables_extended/cpp/scope_variables.cpp +++ b/cpg-language-cxx/src/test/resources/variables_extended/cpp/scope_variables.cpp @@ -12,6 +12,8 @@ void printLog(string logId, string message){ cout << logId << ": " << message << endl; } +class error; + class ScopeVariables{ public: string varName = "instance_field"; @@ -44,8 +46,8 @@ class ScopeVariables{ } try { - throw string("exception_string"); - } catch (const string& varName) { + throw new error(); + } catch (const error& varName) { printLog("func2_catch_varName", varName); }; ScopeVariables scopeVariables; diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/DeclarationHandler.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/DeclarationHandler.kt index 28d450bd651..d3c43fd3e09 100644 --- a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/DeclarationHandler.kt +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/DeclarationHandler.kt @@ -76,10 +76,7 @@ class DeclarationHandler(frontend: GoLanguageFrontend) : // TODO: this will only find methods within the current translation unit. // this is a limitation that we have for C++ as well - val record = - frontend.scopeManager.currentScope?.let { - frontend.scopeManager.getRecordForName(it, recordName) - } + val record = frontend.scopeManager.getRecordForName(recordName) // now this gets a little bit hacky, we will add it to the record declaration // this is strictly speaking not 100 % true, since the method property edge is diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/ExpressionHandler.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/ExpressionHandler.kt index 6055a14c7a6..488bedb528f 100644 --- a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/ExpressionHandler.kt +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/ExpressionHandler.kt @@ -149,16 +149,7 @@ class ExpressionHandler(frontend: GoLanguageFrontend) : return literal } - val ref = newReference(ident.name, rawNode = ident) - - // Check, if this refers to a package import - val import = frontend.currentTU?.getIncludeByName(ident.name) - // Then set the refersTo, because our regular CPG passes will not resolve them - if (import != null) { - ref.refersTo = import - } - - return ref + return newReference(ident.name, rawNode = ident) } private fun handleIndexExpr(indexExpr: GoStandardLibrary.Ast.IndexExpr): SubscriptExpression { diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/GoExtraPass.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/GoExtraPass.kt index ea3e154cf21..521f146f7ba 100644 --- a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/GoExtraPass.kt +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/GoExtraPass.kt @@ -107,8 +107,8 @@ import de.fraunhofer.aisec.cpg.passes.order.ExecuteBefore * that the expression is creating. This way we can resolve the static references to the field to * the actual field. */ -@ExecuteBefore(VariableUsageResolver::class) -@ExecuteBefore(CallResolver::class) +@ExecuteBefore(SymbolResolver::class) +@ExecuteBefore(EvaluationOrderGraphPass::class) @ExecuteBefore(DFGPass::class) class GoExtraPass(ctx: TranslationContext) : ComponentPass(ctx), ScopeProvider { @@ -139,48 +139,51 @@ class GoExtraPass(ctx: TranslationContext) : ComponentPass(ctx), ScopeProvider { private fun addBuiltIn(): TranslationUnitDeclaration { val builtin = newTranslationUnitDeclaration("builtin.go") + builtin.language = GoLanguage() scopeManager.resetToGlobal(builtin) - val len = newFunctionDeclaration("len", localNameOnly = true) - len.parameters = listOf(newParameterDeclaration("v", GoLanguage().autoType())) - len.type = - typeManager.registerType( - FunctionType(funcTypeName(len.signatureTypes, len.returnTypes)) - ) - scopeManager.addDeclaration(len) - - /** - * ```go - * func append(slice []Type, elems ...Type) []Type - * ``` - */ - val append = newFunctionDeclaration("append", localNameOnly = true) - append.parameters = - listOf( - newParameterDeclaration("slice", GoLanguage().autoType().array()), - newParameterDeclaration("elems", GoLanguage().autoType(), variadic = true), - ) - append.returnTypes = listOf(GoLanguage().autoType().array()) - append.type = - typeManager.registerType( - FunctionType(funcTypeName(append.signatureTypes, append.returnTypes)) - ) - scopeManager.addDeclaration(append) - - val error = newRecordDeclaration("error", "interface") - scopeManager.enterScope(error) - - val errorFunc = newMethodDeclaration("Error", recordDeclaration = error) - errorFunc.returnTypes = listOf(GoLanguage().primitiveType("string")) - errorFunc.type = - typeManager.registerType( - FunctionType(funcTypeName(errorFunc.signatureTypes, errorFunc.returnTypes)) - ) - scopeManager.addDeclaration(errorFunc) - - scopeManager.leaveScope(error) - - return builtin + return with(builtin) { + val len = newFunctionDeclaration("len", localNameOnly = true) + len.parameters = listOf(newParameterDeclaration("v", autoType())) + len.returnTypes = listOf(primitiveType("int")) + len.type = + typeManager.registerType( + FunctionType(funcTypeName(len.signatureTypes, len.returnTypes)) + ) + scopeManager.addDeclaration(len) + + /** + * ```go + * func append(slice []Type, elems ...Type) []Type + * ``` + */ + val append = newFunctionDeclaration("append", localNameOnly = true) + append.parameters = + listOf( + newParameterDeclaration("slice", autoType().array()), + newParameterDeclaration("elems", autoType(), variadic = true), + ) + append.returnTypes = listOf(autoType().array()) + append.type = + typeManager.registerType( + FunctionType(funcTypeName(append.signatureTypes, append.returnTypes)) + ) + scopeManager.addDeclaration(append) + + val error = newRecordDeclaration("error", "interface") + scopeManager.enterScope(error) + + val errorFunc = newMethodDeclaration("Error", recordDeclaration = error) + errorFunc.returnTypes = listOf(primitiveType("string")) + errorFunc.type = + typeManager.registerType( + FunctionType(funcTypeName(errorFunc.signatureTypes, errorFunc.returnTypes)) + ) + scopeManager.addDeclaration(errorFunc) + + scopeManager.leaveScope(error) + builtin + } } /** diff --git a/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontendTest.kt b/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontendTest.kt index 66ae52b06b0..bbcbcab012a 100644 --- a/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontendTest.kt +++ b/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontendTest.kt @@ -452,14 +452,13 @@ class GoLanguageFrontendTest : BaseTest() { @Test fun testMemberCalls() { val topLevel = Path.of("src", "test", "resources", "golang") - val tu = - analyzeAndGetFirstTU(listOf(topLevel.resolve("struct.go").toFile()), topLevel, true) { + val result = + analyze(listOf(topLevel.resolve("struct.go").toFile()), topLevel, true) { it.registerLanguage() } + assertNotNull(result) - assertNotNull(tu) - - val p = tu.namespaces["p"] + val p = result.namespaces["p"] assertNotNull(p) val myStruct = p.records["MyStruct"] @@ -471,16 +470,14 @@ class GoLanguageFrontendTest : BaseTest() { assertLocalName("MyFunc", myFunc) val body = myFunc.body as? Block - assertNotNull(body) - val printf = body.statements.first() as? CallExpression - - assertNotNull(printf) - assertLocalName("Printf", printf) - assertFullName("fmt.Printf", printf) + val printfCall = body.statements.first() as? CallExpression + assertNotNull(printfCall) + assertLocalName("Printf", printfCall) + assertFullName("fmt.Printf", printfCall) - val arg1 = printf.arguments[0] as? MemberCallExpression + val arg1 = printfCall.arguments[0] as? MemberCallExpression assertNotNull(arg1) assertLocalName("myOtherFunc", arg1) @@ -489,6 +486,60 @@ class GoLanguageFrontendTest : BaseTest() { assertEquals(myFunc.receiver, (arg1.base as? Reference)?.refersTo) } + @Test + fun testCorrectInference() { + val topLevel = Path.of("src", "test", "resources", "golang") + val result = + analyze(listOf(topLevel.resolve("struct.go").toFile()), topLevel, true) { + it.registerLanguage() + } + assertNotNull(result) + + // Make sure, that we inferred the Printf function at the correct namespace + val fmt = result.namespaces["fmt"] + assertNotNull(fmt) + + val printf = fmt.functions["Printf"] + assertNotNull(printf) + assertTrue(printf.isInferred) + + val printfCall = result.calls["fmt.Printf"] + assertNotNull(printfCall) + assertLocalName("Printf", printfCall) + assertFullName("fmt.Printf", printfCall) + assertInvokes(printfCall, printf) + } + + @Test + fun testQualifiedCallInMethod() { + val stdLib = Path.of("src", "test", "resources", "golang-std") + val topLevel = Path.of("src", "test", "resources", "golang") + val result = + analyze( + listOf( + topLevel.resolve("struct.go").toFile(), + stdLib.resolve("fmt").toFile(), + ), + topLevel, + true + ) { + it.registerLanguage() + it.includePath(stdLib) + } + assertNotNull(result) + + val fmt = result.namespaces["fmt"] + assertNotNull(fmt) + + val printf = fmt.functions["Printf"] + assertNotNull(printf) + assertFalse(printf.isInferred) + + val printfCall = result.calls["fmt.Printf"] + assertNotNull(printfCall) + assertInvokes(printfCall, printf) + } + @Test fun testField() { val topLevel = Path.of("src", "test", "resources", "golang") @@ -627,6 +678,21 @@ class GoLanguageFrontendTest : BaseTest() { assertNotNull(app) val tus = app.translationUnits + + // fetch the function declaration from the struct TU + val tu2 = tus[1] + + val p2 = tu2.namespaces["p"] + assertNotNull(p2) + + val myOtherFunc = p2.methods["myOtherFunc"] + assertNotNull(myOtherFunc) + assertFalse(myOtherFunc.isImplicit) + + val newMyStruct = p2.functions["NewMyStruct"] + assertNotNull(newMyStruct) + + // and compare it with the call TU val tu = tus[0] val p = tu.namespaces["p"] @@ -646,23 +712,19 @@ class GoLanguageFrontendTest : BaseTest() { assertEquals(objectType("p.MyStruct").pointer(), c.type) } - val newMyStruct = assertIs(c.firstAssignment) - - // fetch the function declaration from the other TU - val tu2 = tus[1] - - val p2 = tu2.namespaces["p"] - assertNotNull(p2) - - val newMyStructDef = p2.functions["NewMyStruct"] - assertTrue(newMyStruct.invokes.contains(newMyStructDef)) + val newMyStructCall = assertIs(c.firstAssignment) + assertInvokes(newMyStructCall, newMyStruct) - val call = body.statements[1] as? MemberCallExpression + val call = tu.calls["myOtherFunc"] as? MemberCallExpression assertNotNull(call) val base = call.base as? Reference assertNotNull(base) - assertEquals(c, base.refersTo) + assertRefersTo(base, c) + + val myOtherFuncCall = tu.calls["myOtherFunc"] + assertNotNull(myOtherFuncCall) + assertInvokes(myOtherFuncCall, myOtherFunc) val go = main.calls["go"] assertNotNull(go) @@ -1025,4 +1087,30 @@ class GoLanguageFrontendTest : BaseTest() { val call = tu.calls["Elem"] assertInvokes(call, elem) } + + @Test + fun testComplexResolution() { + val topLevel = Path.of("src", "test", "resources", "golang") + val result = + analyze( + listOf( + topLevel.resolve("complex_resolution.go").toFile(), + ), + topLevel, + true + ) { + it.registerLanguage() + } + + assertNotNull(result) + + val calls = result.calls + calls.forEach { + assertTrue(it.invokes.isNotEmpty()) + it.invokes.forEach { func -> assertFalse(func.isInferred) } + } + + val refs = result.refs + refs.forEach { assertNotNull(it.refersTo) } + } } diff --git a/cpg-language-go/src/test/resources/golang/call.go b/cpg-language-go/src/test/resources/golang/call.go index 9ecadbeb096..4f78d7d39dc 100644 --- a/cpg-language-go/src/test/resources/golang/call.go +++ b/cpg-language-go/src/test/resources/golang/call.go @@ -1,11 +1,10 @@ package p -import ("http") - func main() { - c := NewMyStruct() - c.myOtherFunc() + c := NewMyStruct() + if i := len("a"); i < 1 { + c.myOtherFunc(i) + } go c.MyFunc() - go c.MyFunc() } diff --git a/cpg-language-go/src/test/resources/golang/complex_resolution.go b/cpg-language-go/src/test/resources/golang/complex_resolution.go new file mode 100644 index 00000000000..e213cea0efb --- /dev/null +++ b/cpg-language-go/src/test/resources/golang/complex_resolution.go @@ -0,0 +1,21 @@ +package p + +type Something struct { + self *Something +} + +func (s *Something) Do() { + Func() +} + +func NewSomething() *Something { + return &Something{} +} + +func main() { + var some = NewSomething() + + some.self.self.self.Do() +} + +func Func(args ...int) {} diff --git a/cpg-language-go/src/test/resources/golang/if.go b/cpg-language-go/src/test/resources/golang/if.go index 1377e57c82e..d6740d5255e 100644 --- a/cpg-language-go/src/test/resources/golang/if.go +++ b/cpg-language-go/src/test/resources/golang/if.go @@ -1,11 +1,9 @@ package p -import ("fmt") - func main() { - var b bool = true + var b bool = true - if b { - b = false - } -} \ No newline at end of file + if b { + b = false + } +} diff --git a/cpg-language-go/src/test/resources/golang/struct.go b/cpg-language-go/src/test/resources/golang/struct.go index 18e466eea84..25e160472eb 100644 --- a/cpg-language-go/src/test/resources/golang/struct.go +++ b/cpg-language-go/src/test/resources/golang/struct.go @@ -21,13 +21,13 @@ type MyInterface interface { } func (s MyStruct) MyFunc() string { - fmt.Printf(s.myOtherFunc(), s.MyField) + fmt.Printf(s.myOtherFunc(1), s.MyField) return "s" } -func (s MyStruct) myOtherFunc() string { - return "%d" +func (s MyStruct) myOtherFunc(i int) string { + return fmt.Sprintf("%d", i) } func NewMyStruct() *MyStruct { diff --git a/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/java/DeclarationHandler.kt b/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/java/DeclarationHandler.kt index 285cacf5b96..51c4eb80e69 100644 --- a/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/java/DeclarationHandler.kt +++ b/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/java/DeclarationHandler.kt @@ -376,7 +376,7 @@ open class DeclarationHandler(lang: JavaLanguageFrontend) : entries.forEach { it.type = this.objectType(enumDeclaration.name) } enumDeclaration.entries = entries val superTypes = enumDecl.implementedTypes.map { frontend.getTypeAsGoodAsPossible(it) } - enumDeclaration.superTypes = superTypes + enumDeclaration.superClasses.addAll(superTypes) return enumDeclaration } diff --git a/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/java/JavaLanguage.kt b/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/java/JavaLanguage.kt index c1666370daf..59f37dc5378 100644 --- a/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/java/JavaLanguage.kt +++ b/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/java/JavaLanguage.kt @@ -28,7 +28,6 @@ package de.fraunhofer.aisec.cpg.frontends.java import com.fasterxml.jackson.annotation.JsonIgnore import de.fraunhofer.aisec.cpg.ScopeManager import de.fraunhofer.aisec.cpg.frontends.* -import de.fraunhofer.aisec.cpg.graph.Name import de.fraunhofer.aisec.cpg.graph.declarations.RecordDeclaration import de.fraunhofer.aisec.cpg.graph.statements.expressions.BinaryOperator import de.fraunhofer.aisec.cpg.graph.statements.expressions.MemberExpression @@ -112,8 +111,7 @@ open class JavaLanguage : callee: MemberExpression, curClass: RecordDeclaration, scopeManager: ScopeManager, - recordMap: Map - ) = JavaCallResolverHelper.handleSuperCall(callee, curClass, scopeManager, recordMap) + ) = JavaCallResolverHelper.handleSuperCall(callee, curClass, scopeManager) override val startCharacter = '<' override val endCharacter = '>' diff --git a/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/JavaCallResolverHelper.kt b/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/JavaCallResolverHelper.kt index e556311e66e..1067931ad3f 100644 --- a/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/JavaCallResolverHelper.kt +++ b/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/JavaCallResolverHelper.kt @@ -27,7 +27,6 @@ package de.fraunhofer.aisec.cpg.passes import de.fraunhofer.aisec.cpg.ScopeManager import de.fraunhofer.aisec.cpg.frontends.java.JavaLanguage -import de.fraunhofer.aisec.cpg.graph.Name import de.fraunhofer.aisec.cpg.graph.declarations.MethodDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.RecordDeclaration import de.fraunhofer.aisec.cpg.graph.objectType @@ -35,8 +34,9 @@ import de.fraunhofer.aisec.cpg.graph.statements.expressions.MemberCallExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.MemberExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference import de.fraunhofer.aisec.cpg.graph.types.HasType +import de.fraunhofer.aisec.cpg.graph.types.recordDeclaration import de.fraunhofer.aisec.cpg.helpers.Util -import de.fraunhofer.aisec.cpg.passes.CallResolver.Companion.LOGGER +import de.fraunhofer.aisec.cpg.passes.SymbolResolver.Companion.LOGGER class JavaCallResolverHelper { @@ -46,8 +46,8 @@ class JavaCallResolverHelper { * JLS13 ยง15.12.1. * * This function basically sets the correct type of the [Reference] containing the "super" - * keyword. Afterwards, we can use the regular [CallResolver.resolveMemberCallee] to resolve - * the [MemberCallExpression]. + * keyword. Afterwards, we can use the regular [SymbolResolver.resolveMemberCallee] to + * resolve the [MemberCallExpression]. * * @param callee The callee of the call expression that needs to be adjusted * @param curClass The class containing the call @@ -55,12 +55,10 @@ class JavaCallResolverHelper { fun handleSuperCall( callee: MemberExpression, curClass: RecordDeclaration, - scopeManager: ScopeManager, - recordMap: Map + scopeManager: ScopeManager ): Boolean { // Because the "super" keyword still refers to "this" (but casted to another class), we - // still - // need to connect the super reference to the receiver of this method. + // still need to connect the super reference to the receiver of this method. val func = scopeManager.currentFunction if (func is MethodDeclaration) { (callee.base as Reference?)?.refersTo = func.receiver @@ -70,11 +68,10 @@ class JavaCallResolverHelper { var target: RecordDeclaration? = null // In case the reference is just called "super", this is a direct superclass, either - // defined - // explicitly or java.lang.Object by default + // defined explicitly or java.lang.Object by default if (callee.base.name.toString() == JavaLanguage().superClassKeyword) { if (curClass.superClasses.isNotEmpty()) { - target = recordMap[curClass.superClasses[0].root.name] + target = curClass.superClasses[0].root.recordDeclaration } else { Util.warnWithFileLocation( callee, @@ -86,7 +83,7 @@ class JavaCallResolverHelper { // BaseName.super.call(), might either be in order to specify an enclosing class or // an // interface that is implemented - target = handleSpecificSupertype(callee, curClass, recordMap) + target = handleSpecificSupertype(callee, curClass) } if (target != null) { @@ -112,20 +109,20 @@ class JavaCallResolverHelper { fun handleSpecificSupertype( callee: MemberExpression, - curClass: RecordDeclaration, - recordMap: Map + curClass: RecordDeclaration ): RecordDeclaration? { val baseName = callee.base.name.parent ?: return null - if (curClass.objectType(baseName) in curClass.implementedInterfaces) { + val type = curClass.objectType(baseName) + if (type in curClass.implementedInterfaces) { // Basename is an interface -> BaseName.super refers to BaseName itself - return recordMap[baseName] + return type.recordDeclaration } else { // BaseName refers to an enclosing class -> BaseName.super is BaseName's superclass - val base = recordMap[baseName] + val base = type.recordDeclaration if (base != null) { if (base.superClasses.isNotEmpty()) { - return recordMap[base.superClasses[0].root.name] + return base.superClasses[0].root.recordDeclaration } else { Util.warnWithFileLocation( callee, diff --git a/cpg-language-llvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/llvm/ExpressionHandler.kt b/cpg-language-llvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/llvm/ExpressionHandler.kt index a6a6e728172..2a23f71a5a7 100644 --- a/cpg-language-llvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/llvm/ExpressionHandler.kt +++ b/cpg-language-llvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/llvm/ExpressionHandler.kt @@ -33,7 +33,7 @@ import de.fraunhofer.aisec.cpg.graph.statements.expressions.* import de.fraunhofer.aisec.cpg.graph.types.ObjectType import de.fraunhofer.aisec.cpg.graph.types.PointerType import de.fraunhofer.aisec.cpg.graph.types.Type -import de.fraunhofer.aisec.cpg.passes.VariableUsageResolver +import de.fraunhofer.aisec.cpg.passes.SymbolResolver import org.bytedeco.javacpp.IntPointer import org.bytedeco.javacpp.SizeTPointer import org.bytedeco.llvm.LLVM.LLVMValueRef @@ -138,8 +138,8 @@ class ExpressionHandler(lang: LLVMIRLanguageFrontend) : * can either be a reference to a global or local one, depending on the prefix. * * This function will also take care of actually resolving the reference. This is a) faster and - * b) needed because the [VariableUsageResolver] is not familiar with the prefix system, - * determining the scope of the variable. + * b) needed because the [SymbolResolver] is not familiar with the prefix system, determining + * the scope of the variable. */ private fun handleReference(valueRef: LLVMValueRef): Expression { val namePair = frontend.getNameOf(valueRef) diff --git a/cpg-language-llvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/llvm/LLVMIRLanguageFrontend.kt b/cpg-language-llvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/llvm/LLVMIRLanguageFrontend.kt index b4208108253..471347afc01 100644 --- a/cpg-language-llvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/llvm/LLVMIRLanguageFrontend.kt +++ b/cpg-language-llvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/llvm/LLVMIRLanguageFrontend.kt @@ -39,7 +39,7 @@ import de.fraunhofer.aisec.cpg.graph.types.* import de.fraunhofer.aisec.cpg.helpers.Benchmark import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker import de.fraunhofer.aisec.cpg.passes.CompressLLVMPass -import de.fraunhofer.aisec.cpg.passes.VariableUsageResolver +import de.fraunhofer.aisec.cpg.passes.SymbolResolver import de.fraunhofer.aisec.cpg.passes.order.RegisterExtraPass import de.fraunhofer.aisec.cpg.sarif.PhysicalLocation import java.io.File @@ -71,7 +71,7 @@ class LLVMIRLanguageFrontend(language: Language, ctx: Tr * This contains a cache binding between an LLVMValueRef (representing a variable) and its * [Declaration] in the graph. We need this, because this way we can look up and connect a * [Reference] to its [Declaration] already in the language frontend. This in turn is needed - * because of the local/global system we cannot rely on the [VariableUsageResolver]. + * because of the local/global system we cannot rely on the [SymbolResolver]. */ var bindingsCache = mutableMapOf() diff --git a/cpg-language-python/src/main/python/CPGPython/_expressions.py b/cpg-language-python/src/main/python/CPGPython/_expressions.py index f4a31e8ab4e..98df5c2678b 100644 --- a/cpg-language-python/src/main/python/CPGPython/_expressions.py +++ b/cpg-language-python/src/main/python/CPGPython/_expressions.py @@ -233,7 +233,7 @@ def handle_expression_impl(self, expr): # try to see, whether this refers to a known class and thus is a # constructor. record = self.scopemanager.getRecordForName( - self.scopemanager.getCurrentScope(), refname) + refname, self.scopemanager.getCurrentScope()) if record is not None: self.log_with_loc("Received a record: %s" % record) call = ExpressionBuilderKt.newConstructExpression( diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt index 2ca7c4fb665..8a3ef47966f 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt @@ -351,6 +351,10 @@ class PythonFrontendTest : BaseTest() { assertEquals(4, recordFoo.fields.size) assertEquals(1, recordFoo.methods.size) + // TODO: When developing the new python frontend, remove the type specifier from the field + // again and check if the field still occurs. It's absolutely not clear to me who would be + // responsible for adding it but IMHO it should be the frontend. This, however, is + // currently not the case. val fieldX = recordFoo.fields["x"] assertNotNull(fieldX) diff --git a/cpg-language-python/src/test/resources/python/class_fields.py b/cpg-language-python/src/test/resources/python/class_fields.py index 3df4ec440b9..ed833025105 100644 --- a/cpg-language-python/src/test/resources/python/class_fields.py +++ b/cpg-language-python/src/test/resources/python/class_fields.py @@ -1,5 +1,5 @@ class Foo: - x + x: int y = 123 def bar(self): diff --git a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt index f65e8c3cb68..022309fd2aa 100644 --- a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt +++ b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt @@ -219,8 +219,7 @@ class Application : Callable { listOf( TypeHierarchyResolver::class, ImportResolver::class, - VariableUsageResolver::class, - CallResolver::class, + SymbolResolver::class, DFGPass::class, EvaluationOrderGraphPass::class, TypeResolver::class,