From b8ef18c2c496c8c5e1ed26042a3ec1f2d92e2a3c Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Mon, 16 Oct 2023 20:21:12 +0200 Subject: [PATCH 1/2] Caching symbols on symbol resolution This PR caches symbol resolutions. This can can significantly speed up the resolution process. --- .../de/fraunhofer/aisec/cpg/ScopeManager.kt | 72 ++++++++++++------- .../aisec/cpg/graph/scopes/Scope.kt | 17 +++++ .../graph/statements/expressions/Reference.kt | 12 ++++ 3 files changed, 77 insertions(+), 24 deletions(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt index 5e7c57d990..830bdf7f1e 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt @@ -34,6 +34,7 @@ import de.fraunhofer.aisec.cpg.graph.statements.* import de.fraunhofer.aisec.cpg.graph.statements.expressions.Block import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference +import de.fraunhofer.aisec.cpg.graph.statements.expressions.ReferenceTag import de.fraunhofer.aisec.cpg.graph.types.FunctionPointerType import de.fraunhofer.aisec.cpg.graph.types.IncompleteType import de.fraunhofer.aisec.cpg.graph.types.Type @@ -74,6 +75,12 @@ class ScopeManager : ScopeProvider { /** Represents an alias with the name [to] for the particular name [from]. */ data class Alias(var from: Name, var to: Name) + /** + * A cache map of unique tags (computed with [Reference.buildUniqueTag]) and their respective + * [ValueDeclaration]. This is used by [resolveReference] as a caching mechanism. + */ + private val symbolTable = mutableMapOf() + /** * In some languages, we can define aliases for names. An example is renaming package imports in * Go, e.g., to avoid name conflicts. @@ -615,37 +622,54 @@ class ScopeManager : ScopeProvider { */ @JvmOverloads fun resolveReference(ref: Reference, startScope: Scope? = currentScope): ValueDeclaration? { + // Retrieve a unique tag for the particular reference based on the current scope + val tag = ref.buildUniqueTag(startScope) + + // If we find a match in our symbol table, we can immediately return the declaration + var decl = symbolTable[tag] + if (decl != null) { + return decl + } + val (scope, name) = extractScope(ref, startScope) // Try to resolve value declarations according to our criteria - return resolve(scope) { - if (it.name.lastPartsMatch(name)) { - val helper = ref.resolutionHelper - return@resolve when { - // If the reference seems to point to a function (using a function pointer) - // the entire signature is checked for equality - helper?.type is FunctionPointerType && it is FunctionDeclaration -> { - val fptrType = helper.type as FunctionPointerType - // TODO(oxisto): Support multiple return values - val returnType = it.returnTypes.firstOrNull() ?: IncompleteType() - returnType == fptrType.returnType && - it.hasSignature(fptrType.parameters) - } - // If our language has first-class functions, we can safely return them as a - // reference - ref.language is HasFirstClassFunctions -> { - true - } - // Otherwise, we are not looking for functions here - else -> { - it !is FunctionDeclaration + decl = + resolve(scope) { + if (it.name.lastPartsMatch(name)) { + val helper = ref.resolutionHelper + return@resolve when { + // If the reference seems to point to a function (using a function + // pointer) the entire signature is checked for equality + helper?.type is FunctionPointerType && it is FunctionDeclaration -> { + val fptrType = helper.type as FunctionPointerType + // TODO(oxisto): Support multiple return values + val returnType = it.returnTypes.firstOrNull() ?: IncompleteType() + returnType == fptrType.returnType && + it.hasSignature(fptrType.parameters) + } + // If our language has first-class functions, we can safely return them + // as a reference + ref.language is HasFirstClassFunctions -> { + true + } + // Otherwise, we are not looking for functions here + else -> { + it !is FunctionDeclaration + } } } + + return@resolve false } + .firstOrNull() - return@resolve false - } - .firstOrNull() + // Update the symbol cache, if we found a declaration for the tag + if (decl != null) { + symbolTable[tag] = decl + } + + return decl } /** diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/Scope.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/Scope.kt index a7255d0165..02610fc6a1 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/Scope.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/Scope.kt @@ -82,6 +82,23 @@ abstract class Scope( return this is LoopScope } + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as Scope + + if (astNode != other.astNode) return false + return name == other.name + } + + override fun hashCode(): Int { + var result = astNode?.hashCode() ?: 0 + result = 31 * result + (parent?.hashCode() ?: 0) + result = 31 * result + (name?.hashCode() ?: 0) + return result + } + /** Returns the [GlobalScope] of this scope by traversing its parents upwards. */ val globalScope: Scope? get() { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/Reference.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/Reference.kt index 11d47bb3a8..0c7c1922a4 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/Reference.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/Reference.kt @@ -32,6 +32,7 @@ import de.fraunhofer.aisec.cpg.graph.declarations.Declaration import de.fraunhofer.aisec.cpg.graph.declarations.ValueDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration import de.fraunhofer.aisec.cpg.graph.edge.Properties +import de.fraunhofer.aisec.cpg.graph.scopes.Scope import de.fraunhofer.aisec.cpg.graph.types.HasType import de.fraunhofer.aisec.cpg.graph.types.Type import de.fraunhofer.aisec.cpg.passes.SymbolResolver @@ -153,4 +154,15 @@ open class Reference : Expression(), HasType.TypeObserver { prev.registerTypeObserver(this) } } + + /** + * This function builds a unique tag for the particular reference, based on the [startScope]. + * Its purpose is to cache symbol resolutions, similar to LLVMs system of Unified Symbol + * Resolution (USR). + */ + fun buildUniqueTag(startScope: Scope?): ReferenceTag { + return Objects.hash(this.name, this.resolutionHelper, startScope) + } } + +typealias ReferenceTag = Int From ee1c7aa8f5279d2c674b106187398359f6a29a03 Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Wed, 18 Oct 2023 09:21:39 +0200 Subject: [PATCH 2/2] =?UTF-8?q?Support=20for=20`<<`,=20`>>`=20and=20`|`,?= =?UTF-8?q?=20`&`,=20`^`=C2=A0in=20`ValueEvaluator`=20and=20`BinaryOperati?= =?UTF-8?q?on`=20(#1333)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Support for `<<`, `>>` and `|` in `ValueEvaluator` and `BinaryOperation` --- .../de/fraunhofer/aisec/cpg/ScopeManager.kt | 8 +++---- .../graph/statements/expressions/Reference.kt | 8 +++---- .../aisec/cpg/passes/SymbolResolver.kt | 6 ++---- .../aisec/cpg/passes/SymbolResolverTest.kt | 21 +++++++++++++++++++ 4 files changed, 31 insertions(+), 12 deletions(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt index 830bdf7f1e..49fff98607 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt @@ -614,16 +614,16 @@ class ScopeManager : ScopeProvider { * Resolves only references to Values in the current scope, static references to other visible * records are not resolved over the ScopeManager. * - * @param scope * @param ref * @return * * TODO: We should merge this function with [.resolveFunction] */ - @JvmOverloads - fun resolveReference(ref: Reference, startScope: Scope? = currentScope): ValueDeclaration? { + fun resolveReference(ref: Reference): ValueDeclaration? { + val startScope = ref.scope + // Retrieve a unique tag for the particular reference based on the current scope - val tag = ref.buildUniqueTag(startScope) + val tag = ref.uniqueTag // If we find a match in our symbol table, we can immediately return the declaration var decl = symbolTable[tag] diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/Reference.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/Reference.kt index 0c7c1922a4..5eddd269f1 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/Reference.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/Reference.kt @@ -32,7 +32,6 @@ import de.fraunhofer.aisec.cpg.graph.declarations.Declaration import de.fraunhofer.aisec.cpg.graph.declarations.ValueDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration import de.fraunhofer.aisec.cpg.graph.edge.Properties -import de.fraunhofer.aisec.cpg.graph.scopes.Scope import de.fraunhofer.aisec.cpg.graph.types.HasType import de.fraunhofer.aisec.cpg.graph.types.Type import de.fraunhofer.aisec.cpg.passes.SymbolResolver @@ -160,9 +159,10 @@ open class Reference : Expression(), HasType.TypeObserver { * Its purpose is to cache symbol resolutions, similar to LLVMs system of Unified Symbol * Resolution (USR). */ - fun buildUniqueTag(startScope: Scope?): ReferenceTag { - return Objects.hash(this.name, this.resolutionHelper, startScope) - } + val uniqueTag: ReferenceTag + get() { + return Objects.hash(this.name, this.resolutionHelper, this.scope) + } } typealias ReferenceTag = Int diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/SymbolResolver.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/SymbolResolver.kt index 5e3ec966b7..827ca9425c 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/SymbolResolver.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/SymbolResolver.kt @@ -205,7 +205,7 @@ open class SymbolResolver(ctx: TranslationContext) : ComponentPass(ctx) { // Peek into the declaration, and if it is a variable, we can proceed normally, as we // are running into the special case explained above. Otherwise, we abort here (for // now). - wouldResolveTo = scopeManager.resolveReference(current, current.scope) + wouldResolveTo = scopeManager.resolveReference(current) if (wouldResolveTo !is VariableDeclaration && wouldResolveTo !is ParameterDeclaration) { return } @@ -214,9 +214,7 @@ open class SymbolResolver(ctx: TranslationContext) : ComponentPass(ctx) { // Only consider resolving, if the language frontend did not specify a resolution. If we // already have populated the wouldResolveTo variable, we can re-use this instead of // resolving again - var refersTo = - current.refersTo - ?: wouldResolveTo ?: scopeManager.resolveReference(current, current.scope) + var refersTo = current.refersTo ?: wouldResolveTo ?: scopeManager.resolveReference(current) var recordDeclType: Type? = null if (currentClass != null) { diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/SymbolResolverTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/SymbolResolverTest.kt index 0f141ca244..24415bf6ee 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/SymbolResolverTest.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/SymbolResolverTest.kt @@ -31,6 +31,8 @@ import de.fraunhofer.aisec.cpg.TestUtils.assertRefersTo import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.statements.expressions.ConstructExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.MemberCallExpression +import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference +import de.fraunhofer.aisec.cpg.graph.statements.expressions.ReferenceTag import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertIs @@ -71,4 +73,23 @@ class SymbolResolverTest { assertNotNull(construct) assertInvokes(construct, constructor) } + + @Test + fun testUniqueTags() { + val result = GraphExamples.getConditionalExpression() + + val map = mutableMapOf>() + + val refs = result.refs + refs.forEach { + // Build a unique tag based on the scope of the reference is in (since this is usually + // the start scope) + val list = map.computeIfAbsent(it.uniqueTag) { mutableListOf() } + list += it + + // All elements in the list must have the same scope and name + assertEquals(1, list.map { ref -> ref.scope }.toSet().size) + assertEquals(1, list.map { ref -> ref.name }.toSet().size) + } + } }