diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 785475de5c..31eb168719 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -38,7 +38,7 @@ jobs: go-version: 1.21 - name: Setup neo4j run: | - docker run -d --env NEO4J_AUTH=neo4j/password -p7474:7474 -p7687:7687 neo4j || true + docker run -d --env NEO4J_AUTH=neo4j/password -p7474:7474 -p7687:7687 -e NEO4JLABS_PLUGINS='["apoc"]' neo4j:5 || true - name: Determine Version run: | # determine version from tag diff --git a/README.md b/README.md index 3a9ef5911a..98c262f3b9 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,14 @@ Instead of manually generating or editing the `gradle.properties` file, you can ### For Visualization Purposes -In order to get familiar with the graph itself, you can use the subproject [cpg-neo4j](https://github.com/Fraunhofer-AISEC/cpg/tree/master/cpg-neo4j). It uses this library to generate the CPG for a set of user-provided code files. The graph is then persisted to a [Neo4j](https://neo4j.com/) graph database. The advantage this has for the user, is that Neo4j's visualization software [Neo4j Browser](https://neo4j.com/developer/neo4j-browser/) can be used to graphically look at the CPG nodes and edges, instead of their Java representations. +In order to get familiar with the graph itself, you can use the subproject [cpg-neo4j](./cpg-neo4j). It uses this library to generate the CPG for a set of user-provided code files. The graph is then persisted to a [Neo4j](https://neo4j.com/) graph database. The advantage this has for the user, is that Neo4j's visualization software [Neo4j Browser](https://neo4j.com/developer/neo4j-browser/) can be used to graphically look at the CPG nodes and edges, instead of their Java representations. + +Please make sure, that the [APOC](https://neo4j.com/labs/apoc/) plugin is enabled on your neo4j server. It is used in mass-creating nodes and relationships. + +For example using docker: +``` +docker run -p 7474:7474 -p 7687:7687 -d -e NEO4J_AUTH=neo4j/password -e NEO4JLABS_PLUGINS='["apoc"]' neo4j:5 +``` ### As Library diff --git a/build.gradle.kts b/build.gradle.kts index 67e352fed8..8873eda1e0 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -44,7 +44,7 @@ allprojects { val dokkaPlugin by configurations dependencies { - dokkaPlugin("org.jetbrains.dokka:versioning-plugin:1.9.0") + dokkaPlugin("org.jetbrains.dokka:versioning-plugin:2.0.0") } } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt index 01530f6db2..c14dd1b970 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt @@ -509,7 +509,14 @@ class ScopeManager : ScopeProvider { return pair.second } - var (scope, name) = extractScope(ref, startScope) + val extractedScope = extractScope(ref, startScope) + // If the scope extraction fails, we can only return here directly without any result + if (extractedScope == null) { + return null + } + + var scope = extractedScope.scope + val name = extractedScope.adjustedName if (scope == null) { scope = startScope } @@ -555,13 +562,21 @@ class ScopeManager : ScopeProvider { } /** - * This function extracts a scope for the [Name], e.g. if the name is fully qualified. `null` is - * returned, if no scope can be extracted. + * This class represents the result of the [extractScope] operation. It contains a [scope] + * object, if a scope was found and the [adjustedName] that is normalized if any aliases were + * found during scope extraction. + */ + data class ScopeExtraction(val scope: Scope?, val adjustedName: Name) + + /** + * This function extracts a scope for the [Name], e.g. if the name is fully qualified (wrapped + * in a [ScopeExtraction] object. `null` is returned if a scope was specified, but does not + * exist as a [Scope] object. * - * The pair returns the extracted scope and a name that is adjusted by possible import aliases. - * The extracted scope is "responsible" for the name (e.g. declares the parent namespace) and - * the returned name only differs from the provided name if aliasing was involved at the node - * location (e.g. because of imports). + * The returned object contains the extracted scope and a name that is adjusted by possible + * import aliases. The extracted scope is "responsible" for the name (e.g. declares the parent + * namespace) and the returned name only differs from the provided name if aliasing was involved + * at the node location (e.g. because of imports). * * Note: Currently only *fully* qualified names are properly resolved. This function will * probably return imprecise results for partially qualified names, e.g. if a name `A` inside @@ -569,9 +584,9 @@ class ScopeManager : ScopeProvider { * * @param node the nodes name references a namespace constituted by a scope * @param scope the current scope relevant for the name resolution, e.g. parent of node - * @return a pair with the scope of node.name and the alias-adjusted name + * @return a [ScopeExtraction] object with the scope of node.name and the alias-adjusted name */ - fun extractScope(node: HasNameAndLocation, scope: Scope? = currentScope): Pair { + fun extractScope(node: HasNameAndLocation, scope: Scope? = currentScope): ScopeExtraction? { return extractScope(node.name, node.location, scope) } @@ -596,7 +611,7 @@ class ScopeManager : ScopeProvider { name: Name, location: PhysicalLocation? = null, scope: Scope? = currentScope, - ): Pair { + ): ScopeExtraction? { var n = name var s: Scope? = null @@ -611,20 +626,18 @@ class ScopeManager : ScopeProvider { // this is a scoped call. we need to explicitly jump to that particular scope val scopes = filterScopes { (it is NameScope && it.name == scopeName) } - s = - if (scopes.isEmpty()) { - Util.warnWithFileLocation( - location, - LOGGER, - "Could not find the scope $scopeName needed to resolve $n" - ) - null - } else { - scopes[0] - } + if (scopes.isEmpty()) { + Util.warnWithFileLocation( + location, + LOGGER, + "Could not find the scope $scopeName needed to resolve $n" + ) + return null + } + s = scopes[0] } - return Pair(s, n) + return ScopeExtraction(s, n) } /** @@ -818,7 +831,7 @@ class ScopeManager : ScopeProvider { // This process has several steps: // First, do a quick local lookup, to see if we have a typedef our current scope // (only do this if the name is not qualified) - if (!alias.isQualified() && current == currentScope) { + if (!alias.isQualified() && current == scope) { val decl = current.typedefs[alias] if (decl != null) { return decl.type @@ -892,7 +905,16 @@ class ScopeManager : ScopeProvider { startScope: Scope? = currentScope, predicate: ((Declaration) -> Boolean)? = null, ): List { - val (scope, n) = extractScope(name, location, startScope) + val extractedScope = extractScope(name, location, startScope) + val scope: Scope? + val n: Name + if (extractedScope == null) { + // the scope does not exist at all + return listOf() + } else { + scope = extractedScope.scope + n = extractedScope.adjustedName + } // We need to differentiate between a qualified and unqualified lookup. We have a qualified // lookup, if the scope is not null. In this case we need to stay within the specified scope diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt index 2e28229be9..b4221681ff 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt @@ -36,6 +36,7 @@ import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.passes.* import de.fraunhofer.aisec.cpg.passes.configuration.* import de.fraunhofer.aisec.cpg.passes.inference.DFGFunctionSummaries +import de.fraunhofer.aisec.cpg.persistence.DoNotPersist import java.io.File import java.nio.file.Path import java.util.* @@ -51,6 +52,7 @@ import org.slf4j.LoggerFactory * The configuration for the [TranslationManager] holds all information that is used during the * translation. */ +@DoNotPersist class TranslationConfiguration private constructor( /** Definition of additional symbols, mostly useful for C++. */ @@ -540,6 +542,7 @@ private constructor( // registerPass() registerPass() registerPass() + registerPass() useDefaultPasses = true return this } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationContext.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationContext.kt index 3d7636ee12..1524fe7050 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationContext.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationContext.kt @@ -26,11 +26,13 @@ package de.fraunhofer.aisec.cpg import de.fraunhofer.aisec.cpg.graph.Component +import de.fraunhofer.aisec.cpg.persistence.DoNotPersist /** * The translation context holds all necessary managers and configurations needed during the * translation process. */ +@DoNotPersist class TranslationContext( /** The configuration for this translation. */ val config: TranslationConfiguration, diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationResult.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationResult.kt index 05ace1a4dc..7b4b3ddfb6 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationResult.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationResult.kt @@ -35,6 +35,7 @@ import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import de.fraunhofer.aisec.cpg.helpers.MeasurementHolder import de.fraunhofer.aisec.cpg.helpers.StatisticsHolder import de.fraunhofer.aisec.cpg.passes.Pass +import de.fraunhofer.aisec.cpg.persistence.DoNotPersist import java.util.* import java.util.concurrent.ConcurrentHashMap import org.neo4j.ogm.annotation.Relationship @@ -92,6 +93,7 @@ class TranslationResult( * @return the list of all translation units. */ @Deprecated(message = "translation units of individual components should be accessed instead") + @DoNotPersist val translationUnits: List get() { if (components.size == 1) { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/LanguageTraits.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/LanguageTraits.kt index b701827023..2310db7865 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/LanguageTraits.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/LanguageTraits.kt @@ -212,10 +212,20 @@ interface HasGlobalVariables : LanguageTrait */ interface HasGlobalFunctions : LanguageTrait +/** + * A common trait for classes, in which supposed member expressions (and thus also member calls) in + * the form of "a.b" have an ambiguity between a real field/method access (when "a" is an object) + * and a qualified call because of an import, if "a" is an import / namespace. + * + * We can only resolve this after we have dealt with imports and know all symbols. Therefore, we + * invoke the [ResolveMemberExpressionAmbiguityPass]. + */ +interface HasMemberExpressionAmbiguity : LanguageTrait + /** * A common super-class for all language traits that arise because they are an ambiguity of a * function call, e.g., function-style casts. This means that we cannot differentiate between a - * [CallExpression] and other expressions during the frontend and we need to invoke the + * [CallExpression] and other expressions during the frontend, and we need to invoke the * [ResolveCallExpressionAmbiguityPass] to resolve this. */ sealed interface HasCallExpressionAmbiguity : LanguageTrait diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Assignment.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Assignment.kt index d9a32dbe22..d20a0c82e5 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Assignment.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Assignment.kt @@ -45,4 +45,6 @@ class Assignment( /** The holder of this assignment */ @JsonIgnore val holder: AssignmentHolder -) : Edge(value, target as Node) +) : Edge(value, target as Node) { + override var labels = setOf("ASSIGMENT") +} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Extensions.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Extensions.kt index b50df60822..6bb5e8f909 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Extensions.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Extensions.kt @@ -1023,13 +1023,46 @@ fun Expression?.unwrapReference(): Reference? { /** Returns the [TranslationUnitDeclaration] where this node is located in. */ val Node.translationUnit: TranslationUnitDeclaration? get() { - var node: Node? = this - while (node != null) { - if (node is TranslationUnitDeclaration) { - return node - } - node = node.astParent + return firstParentOrNull { it is TranslationUnitDeclaration } as? TranslationUnitDeclaration + } + +/** + * This helper function be used to find out if a particular expression (usually a [CallExpression] + * or a [Reference]) is imported through a [ImportDeclaration]. + * + * It returns a [Pair], with the [Pair.first] being a boolean value whether it was imported and + * [Pair.second] the [ImportDeclaration] if applicable. + */ +val Expression.importedFrom: List + get() { + if (this is CallExpression) { + return this.callee.importedFrom + } else if (this is MemberExpression) { + return this.base.importedFrom + } else if (this is Reference) { + val imports = this.translationUnit.imports + + return if (name.parent == null) { + // If the name does not have a parent, this reference could directly be the name + // of an import, let's check + imports.filter { it.name.lastPartsMatch(name) } + } else { + // Otherwise, the parent name could be the import + imports.filter { it.name == this.name.parent } + } ?: listOf() } - return null + return listOf() + } + +/** + * Determines whether the expression is imported from another source. + * + * This property evaluates to `true` if the expression originates from an external or supplemental + * source by checking if the [importedFrom] property contains any entries. Otherwise, it evaluates + * to `false`. + */ +val Expression.isImported: Boolean + get() { + return this.importedFrom.isNotEmpty() } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/MermaidPrinter.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/MermaidPrinter.kt index e83fc25114..8b23d81ddd 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/MermaidPrinter.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/MermaidPrinter.kt @@ -110,7 +110,7 @@ fun > Node.printGraph( private fun Edge.label(): String { val builder = StringBuilder() builder.append("\"") - builder.append(this.label) + builder.append(this.labels.joinToString(",")) if (this is Dataflow) { var granularity = this.granularity diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt index 9670b101ad..268d341144 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt @@ -45,9 +45,11 @@ import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker import de.fraunhofer.aisec.cpg.helpers.neo4j.LocationConverter import de.fraunhofer.aisec.cpg.helpers.neo4j.NameConverter import de.fraunhofer.aisec.cpg.passes.* +import de.fraunhofer.aisec.cpg.persistence.DoNotPersist import de.fraunhofer.aisec.cpg.processing.IVisitable import de.fraunhofer.aisec.cpg.sarif.PhysicalLocation import java.util.* +import kotlin.uuid.Uuid import org.apache.commons.lang3.builder.ToStringBuilder import org.apache.commons.lang3.builder.ToStringStyle import org.neo4j.ogm.annotation.* @@ -166,7 +168,7 @@ abstract class Node : var astChildren: List = listOf() get() = SubgraphWalker.getAstChildren(this) - @Transient var astParent: Node? = null + @DoNotPersist @Transient var astParent: Node? = null /** Virtual property for accessing [prevEOGEdges] without property edges. */ @PopulatedByPass(EvaluationOrderGraphPass::class) var prevEOG by unwrapping(Node::prevEOGEdges) @@ -189,7 +191,8 @@ abstract class Node : * Virtual property for accessing [nextDFGEdges] that have a * [de.fraunhofer.aisec.cpg.graph.edges.flows.FullDataflowGranularity]. */ - @PopulatedByPass(DFGPass::class, PointsToPass::class) + @DoNotPersist + @PopulatedByPass(DFGPass::class, PointsToPass::class, ControlFlowSensitiveDFGPass::class) val prevFullDFG: List get() { return prevDFGEdges @@ -212,7 +215,8 @@ abstract class Node : * Virtual property for accessing [nextDFGEdges] that have a * [de.fraunhofer.aisec.cpg.graph.edges.flows.FullDataflowGranularity]. */ - @PopulatedByPass(DFGPass::class, PointsToPass::class) + @DoNotPersist + @PopulatedByPass(DFGPass::class, PointsToPass::class, ControlFlowSensitiveDFGPass::class) val nextFullDFG: List get() { return nextDFGEdges.filter { it.granularity is FullDataflowGranularity }.map { it.end } @@ -252,7 +256,10 @@ abstract class Node : var isImplicit = false /** Required field for object graph mapping. It contains the node id. */ - @Id @GeneratedValue var id: Long? = null + @DoNotPersist @Id @GeneratedValue var legacyId: Long? = null + + /** Will replace [legacyId] */ + var id: Uuid = Uuid.random() /** Index of the argument if this node is used in a function call or parameter list. */ var argumentIndex = 0 @@ -277,6 +284,9 @@ abstract class Node : * further children that have no alternative connection paths to the rest of the graph. */ fun disconnectFromGraph() { + // Disconnect all AST children first + this.astChildren.forEach { it.disconnectFromGraph() } + nextDFGEdges.clear() prevDFGEdges.clear() prevCDGEdges.clear() @@ -285,6 +295,8 @@ abstract class Node : nextPDGEdges.clear() nextEOGEdges.clear() prevEOGEdges.clear() + + astParent = null } override fun toString(): String { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Persistable.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Persistable.kt index 01e0b33c36..604bbb7196 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Persistable.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Persistable.kt @@ -25,4 +25,5 @@ */ package de.fraunhofer.aisec.cpg.graph +/** This interface represents all objects that can be persisted in a graph database. */ interface Persistable diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/Declaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/Declaration.kt index c98eeaac6a..068c956eeb 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/Declaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/Declaration.kt @@ -28,6 +28,7 @@ package de.fraunhofer.aisec.cpg.graph.declarations import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.scopes.Symbol import de.fraunhofer.aisec.cpg.graph.statements.expressions.MemoryAddress +import de.fraunhofer.aisec.cpg.persistence.DoNotPersist import org.neo4j.ogm.annotation.NodeEntity /** @@ -41,6 +42,7 @@ import org.neo4j.ogm.annotation.NodeEntity */ @NodeEntity abstract class Declaration : Node() { + @DoNotPersist val symbol: Symbol get() { return this.name.localName diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FieldDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FieldDeclaration.kt index 6986ec21e2..bcfbd1a9ce 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FieldDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FieldDeclaration.kt @@ -36,7 +36,7 @@ import org.neo4j.ogm.annotation.Relationship */ class FieldDeclaration : VariableDeclaration() { /** Specifies, whether this field declaration is also a definition, i.e. has an initializer. */ - private var isDefinition = false + var isDefinition = false /** If this is only a declaration, this provides a link to the definition of the field. */ @Relationship(value = "DEFINES") diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.kt index ab7a98ea91..f841e4de4c 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.kt @@ -34,6 +34,7 @@ import de.fraunhofer.aisec.cpg.graph.statements.* import de.fraunhofer.aisec.cpg.graph.statements.expressions.Block import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression import de.fraunhofer.aisec.cpg.graph.types.Type +import de.fraunhofer.aisec.cpg.persistence.DoNotPersist import java.util.* import org.apache.commons.lang3.builder.ToStringBuilder import org.neo4j.ogm.annotation.Relationship @@ -41,7 +42,7 @@ import org.neo4j.ogm.annotation.Relationship /** Represents the declaration or definition of a function. */ open class FunctionDeclaration : ValueDeclaration(), DeclarationHolder, EOGStarterHolder { @Relationship("BODY") var bodyEdge = astOptionalEdgeOf() - /** The function body. Usually a [Block]. */ + /** The function body. Usualfly a [Block]. */ var body by unwrapping(FunctionDeclaration::bodyEdge) /** The list of function parameters. */ @@ -143,16 +144,6 @@ open class FunctionDeclaration : ValueDeclaration(), DeclarationHolder, EOGStart return parameters.map { it.default } } - val defaultParameterSignature: List // TODO: What's this property? - get() = - parameters.map { - if (it.default != null) { - it.type - } else { - unknownType() - } - } - val signatureTypes: List get() = parameters.map { it.type } @@ -163,6 +154,7 @@ open class FunctionDeclaration : ValueDeclaration(), DeclarationHolder, EOGStart .toString() } + @DoNotPersist override val eogStarters: List get() = listOfNotNull(this) @@ -188,11 +180,11 @@ open class FunctionDeclaration : ValueDeclaration(), DeclarationHolder, EOGStart } } + @DoNotPersist override val declarations: List get() { val list = ArrayList() list.addAll(parameters) - list.addAll(records) return list } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/NamespaceDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/NamespaceDeclaration.kt index c2e9927eb5..20bebae19d 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/NamespaceDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/NamespaceDeclaration.kt @@ -29,6 +29,7 @@ import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgesOf import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import de.fraunhofer.aisec.cpg.graph.statements.Statement +import de.fraunhofer.aisec.cpg.persistence.DoNotPersist import java.util.Objects import org.neo4j.ogm.annotation.Relationship @@ -75,6 +76,7 @@ class NamespaceDeclaration : Declaration(), DeclarationHolder, StatementHolder, override var statements by unwrapping(NamespaceDeclaration::statementEdges) + @DoNotPersist override val eogStarters: List get() { val list = mutableListOf() diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/RecordDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/RecordDeclaration.kt index 86a9ceef33..23440b37c2 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/RecordDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/RecordDeclaration.kt @@ -33,6 +33,7 @@ import de.fraunhofer.aisec.cpg.graph.statements.Statement import de.fraunhofer.aisec.cpg.graph.types.DeclaresType import de.fraunhofer.aisec.cpg.graph.types.ObjectType import de.fraunhofer.aisec.cpg.graph.types.Type +import de.fraunhofer.aisec.cpg.persistence.DoNotPersist import org.apache.commons.lang3.builder.ToStringBuilder import org.neo4j.ogm.annotation.Relationship import org.neo4j.ogm.annotation.Transient @@ -120,6 +121,7 @@ open class RecordDeclaration : templateEdges.removeIf { it.end == templateDeclaration } } + @DoNotPersist override val declarations: List get() { val list = ArrayList() @@ -162,6 +164,7 @@ open class RecordDeclaration : .toString() } + @DoNotPersist override val eogStarters: List get() { val list = mutableListOf() diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TemplateDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TemplateDeclaration.kt index 2b53dc4efa..f49c43fda3 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TemplateDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TemplateDeclaration.kt @@ -30,6 +30,7 @@ import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.edges.Edge.Companion.propertyEqualsList import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgesOf import de.fraunhofer.aisec.cpg.graph.edges.unwrapping +import de.fraunhofer.aisec.cpg.persistence.DoNotPersist import org.neo4j.ogm.annotation.NodeEntity import org.neo4j.ogm.annotation.Relationship @@ -81,6 +82,7 @@ abstract class TemplateDeclaration : Declaration(), DeclarationHolder { return defaults } + @DoNotPersist override val declarations: List get() { val list = ArrayList() diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TranslationUnitDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TranslationUnitDeclaration.kt index 99b6f5ea16..4b479bcd36 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TranslationUnitDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TranslationUnitDeclaration.kt @@ -30,6 +30,7 @@ import de.fraunhofer.aisec.cpg.graph.edges.Edge.Companion.propertyEqualsList import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgesOf import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import de.fraunhofer.aisec.cpg.graph.statements.Statement +import de.fraunhofer.aisec.cpg.persistence.DoNotPersist import java.util.* import org.apache.commons.lang3.builder.ToStringBuilder import org.neo4j.ogm.annotation.Relationship @@ -74,6 +75,7 @@ class TranslationUnitDeclaration : .toString() } + @DoNotPersist override val eogStarters: List get() { val list = mutableListOf() diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TupleDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TupleDeclaration.kt index acd7fe230f..2755ad7a83 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TupleDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TupleDeclaration.kt @@ -64,10 +64,9 @@ import de.fraunhofer.aisec.cpg.graph.types.TupleType class TupleDeclaration : VariableDeclaration() { /** The list of elements in this tuple. */ var elementEdges = - astEdgesOf( - onAdd = { registerTypeObserver(it.end) }, - onRemove = { unregisterTypeObserver(it.end) } - ) + astEdgesOf(onAdd = { registerTypeObserver(it.end) }) { + unregisterTypeObserver(it.end) + } var elements by unwrapping(TupleDeclaration::elementEdges) override var name: Name diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/VariableDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/VariableDeclaration.kt index b28ab774b1..f8b8dd901a 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/VariableDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/VariableDeclaration.kt @@ -73,7 +73,7 @@ open class VariableDeclaration : ValueDeclaration(), HasInitializer, HasType.Typ if (value is Reference) { value.resolutionHelper = this } - } + }, ) /** The (optional) initializer of the declaration. */ override var initializer by unwrapping(VariableDeclaration::initializerEdge) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/Edge.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/Edge.kt index 1bb365dd63..064b0caa29 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/Edge.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/Edge.kt @@ -71,7 +71,7 @@ abstract class Edge : Persistable, Cloneable { end = edge.end } - @Transient open val label: String = "EDGE" + abstract var labels: Set /** * The index of this node, if it is stored in an diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/AstEdge.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/AstEdge.kt index ebdc6b69e5..e8bb941de2 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/AstEdge.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/AstEdge.kt @@ -37,6 +37,8 @@ open class AstEdge(start: Node, end: T) : Edge(start, end) { init { end.astParent = start } + + override var labels: Set = setOf("AST") } /** Creates an [AstEdges] container starting from this node. */ diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/TemplateArgument.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/TemplateArgument.kt index b67f7711ea..9e4518158a 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/TemplateArgument.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/TemplateArgument.kt @@ -38,4 +38,7 @@ class TemplateArgument( /** A container for [TemplateArgument] edges. */ class TemplateArguments(thisRef: Node) : - AstEdges>(thisRef, init = ::TemplateArgument) + AstEdges>( + thisRef, + init = { start, end -> TemplateArgument(start, end) } + ) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/ControlDependence.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/ControlDependence.kt index e374adfcef..5f3badedad 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/ControlDependence.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/ControlDependence.kt @@ -49,6 +49,8 @@ class ControlDependence( dependence = DependenceType.CONTROL } + override var labels = setOf("CDG") + override fun equals(other: Any?): Boolean { if (this === other) return true if (other !is ControlDependence) return false diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Dataflow.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Dataflow.kt index 8bc4f27bbc..1fb3d21fb5 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Dataflow.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Dataflow.kt @@ -117,7 +117,7 @@ open class Dataflow( @JsonIgnore var granularity: Granularity = default() ) : Edge(start, end) { - override val label: String = "DFG" + override var labels = setOf("DFG") override fun equals(other: Any?): Boolean { if (this === other) return true @@ -158,7 +158,7 @@ class ContextSensitiveDataflow( val callingContext: CallingContext ) : Dataflow(start, end, granularity) { - override val label: String = "DFG" + override var labels = setOf("DFG") override fun equals(other: Any?): Boolean { if (this === other) return true diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/EvaluationOrder.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/EvaluationOrder.kt index af99e4dcaf..7ac7126b3b 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/EvaluationOrder.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/EvaluationOrder.kt @@ -68,6 +68,8 @@ class EvaluationOrder( result = 31 * result + branch.hashCode() return result } + + override var labels = setOf("EOG") } /** diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Invoke.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Invoke.kt index 261b4957e4..9bbaffad82 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Invoke.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Invoke.kt @@ -43,6 +43,8 @@ class Invoke( */ var dynamicInvoke: Boolean = false, ) : Edge(start, end) { + override var labels = setOf("INVOKES") + override fun equals(other: Any?): Boolean { if (this === other) return true if (other !is Invoke) return false diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Usage.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Usage.kt index f5c7a863d7..0178456adf 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Usage.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Usage.kt @@ -52,6 +52,8 @@ class Usage( result = 31 * result + access.hashCode() return result } + + override var labels = setOf("USAGE") } /** A container for [Usage] edges. [NodeType] is necessary because of the Neo4J OGM. */ diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/NameScope.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/NameScope.kt index 4280e774b9..b70f98b05d 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/NameScope.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/NameScope.kt @@ -25,6 +25,7 @@ */ package de.fraunhofer.aisec.cpg.graph.scopes +import de.fraunhofer.aisec.cpg.graph.Name import de.fraunhofer.aisec.cpg.graph.Node /** @@ -37,6 +38,6 @@ sealed class NameScope(node: Node?) : StructureDeclarationScope(node) { init { astNode = node // Set the name so that we can use it as a namespace later - name = node?.name + name = node?.name ?: Name(EMPTY_NAME) } } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/Scope.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/Scope.kt index cf322f7ff1..d393e137a5 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/Scope.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/Scope.kt @@ -27,7 +27,6 @@ package de.fraunhofer.aisec.cpg.graph.scopes import com.fasterxml.jackson.annotation.JsonBackReference import de.fraunhofer.aisec.cpg.frontends.Language -import de.fraunhofer.aisec.cpg.graph.Name import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.Node.Companion.TO_STRING_STYLE import de.fraunhofer.aisec.cpg.graph.declarations.Declaration @@ -35,13 +34,9 @@ import de.fraunhofer.aisec.cpg.graph.declarations.ImportDeclaration import de.fraunhofer.aisec.cpg.graph.statements.LabelStatement import de.fraunhofer.aisec.cpg.graph.statements.LookupScopeStatement import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference -import de.fraunhofer.aisec.cpg.helpers.neo4j.NameConverter import org.apache.commons.lang3.builder.ToStringBuilder -import org.neo4j.ogm.annotation.GeneratedValue -import org.neo4j.ogm.annotation.Id import org.neo4j.ogm.annotation.NodeEntity import org.neo4j.ogm.annotation.Relationship -import org.neo4j.ogm.annotation.typeconversion.Convert /** * A symbol is a simple, local name. It is valid within the scope that declares it and all of its @@ -60,17 +55,11 @@ sealed class Scope( @Relationship(value = "SCOPE", direction = Relationship.Direction.INCOMING) @JsonBackReference open var astNode: Node? -) { - - /** Required field for object graph mapping. It contains the scope id. */ - @Id @GeneratedValue var id: Long? = null +) : Node() { /** FQN Name currently valid */ var scopedName: String? = null - /** The real new name */ - @Convert(NameConverter::class) var name: Name? = null - /** * Scopes are nested and therefore have a parent child relationship, this two members will help * navigate through the scopes,e.g. when looking up variables. diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/CatchClause.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/CatchClause.kt index 5f317eff4c..7e93010f70 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/CatchClause.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/CatchClause.kt @@ -32,6 +32,7 @@ import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration import de.fraunhofer.aisec.cpg.graph.edges.ast.astOptionalEdgeOf import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import de.fraunhofer.aisec.cpg.graph.statements.expressions.Block +import de.fraunhofer.aisec.cpg.persistence.DoNotPersist import java.util.Objects import org.neo4j.ogm.annotation.Relationship @@ -43,9 +44,11 @@ class CatchClause : Statement(), BranchingNode, EOGStarterHolder { @Relationship(value = "BODY") var bodyEdge = astOptionalEdgeOf() var body by unwrapping(CatchClause::bodyEdge) + @DoNotPersist override val branchedBy: Node? get() = parameter + @DoNotPersist override val eogStarters: List get() = listOf(this) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/ForEachStatement.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/ForEachStatement.kt index d0917b194d..4ed2991fa6 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/ForEachStatement.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/ForEachStatement.kt @@ -70,10 +70,10 @@ class ForEachStatement : LoopStatement(), BranchingNode, StatementHolder { override var statementEdges: AstEdges> get() { val statements = astEdgesOf() - variable?.let { statements.add(AstEdge(this, it)) } - iterable?.let { statements.add(AstEdge(this, it)) } - statement?.let { statements.add(AstEdge(this, it)) } - elseStatement?.let { statements.add(AstEdge(this, it)) } + statements += variableEdge + statements += iterableEdge + statements += statementEdge + statements += elseStatementEdge return statements } set(_) { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/AssignExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/AssignExpression.kt index ec59fb81a5..38f90de77f 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/AssignExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/AssignExpression.kt @@ -110,13 +110,11 @@ class AssignExpression : var lhs by unwrapping(AssignExpression::lhsEdges) @Relationship("RHS") - /** The expressions on the right-hand side. */ var rhsEdges = - astEdgesOf( - onAdd = { it.end.registerTypeObserver(this) }, - onRemove = { it.end.unregisterTypeObserver(this) }, - ) + astEdgesOf(onAdd = { it.end.registerTypeObserver(this) }) { + it.end.unregisterTypeObserver(this) + } var rhs by unwrapping(AssignExpression::rhsEdges) /** diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CollectionComprehension.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CollectionComprehension.kt index e51e306f3d..2e99cb4dea 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CollectionComprehension.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CollectionComprehension.kt @@ -60,7 +60,7 @@ class CollectionComprehension : Expression(), ArgumentHolder { @Relationship("STATEMENT") var statementEdge = astEdgeOf( - ProblemExpression("No statement provided but is required in ${this::class}") + ProblemExpression("No statement provided but is required in ${this::class}"), ) /** * This field contains the statement which is applied to each element of the input for which the diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/InitializerListExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/InitializerListExpression.kt index 62fa90ca8e..2f70d1026f 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/InitializerListExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/InitializerListExpression.kt @@ -49,8 +49,9 @@ class InitializerListExpression : Expression(), ArgumentHolder, HasType.TypeObse var initializerEdges = astEdgesOf( onAdd = { it.end.registerTypeObserver(this) }, - onRemove = { it.end.unregisterTypeObserver(this) }, - ) + ) { + it.end.unregisterTypeObserver(this) + } /** Virtual property to access [initializerEdges] without property edges. */ var initializers by unwrapping(InitializerListExpression::initializerEdges) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/MeasurementHolder.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/MeasurementHolder.kt index f16b284cdc..2072ccae8f 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/MeasurementHolder.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/MeasurementHolder.kt @@ -28,6 +28,7 @@ package de.fraunhofer.aisec.cpg.helpers import com.fasterxml.jackson.databind.SerializationFeature import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import de.fraunhofer.aisec.cpg.TranslationConfiguration +import de.fraunhofer.aisec.cpg.persistence.DoNotPersist import java.io.File import java.nio.file.Path import java.time.Duration @@ -36,6 +37,7 @@ import java.util.* import org.slf4j.Logger import org.slf4j.LoggerFactory +@DoNotPersist class BenchmarkResults(val entries: List>) { val json: String @@ -175,6 +177,7 @@ constructor( } } +@DoNotPersist /** Represents some kind of measurements, e.g., on the performance or problems. */ open class MeasurementHolder @JvmOverloads diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/SubgraphWalker.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/SubgraphWalker.kt index 6aded16eb7..17246c21e1 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/SubgraphWalker.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/SubgraphWalker.kt @@ -34,7 +34,11 @@ import de.fraunhofer.aisec.cpg.graph.StatementHolder import de.fraunhofer.aisec.cpg.graph.declarations.RecordDeclaration import de.fraunhofer.aisec.cpg.graph.edges.ast.AstEdge import de.fraunhofer.aisec.cpg.graph.edges.collections.EdgeCollection +import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression +import de.fraunhofer.aisec.cpg.graph.statements.expressions.MemberCallExpression +import de.fraunhofer.aisec.cpg.graph.statements.expressions.MemberExpression +import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference import de.fraunhofer.aisec.cpg.graph.types.HasType import de.fraunhofer.aisec.cpg.passes.Pass import de.fraunhofer.aisec.cpg.processing.strategy.Strategy @@ -350,14 +354,17 @@ object SubgraphWalker { /** * Tries to replace the [old] expression with a [new] one, given the [parent]. * - * There are three things to consider: + * There are different things to consider: * - First, this only works if [parent] is either an [ArgumentHolder] or [StatementHolder]. * Otherwise, we cannot instruct the parent to exchange the node * - Second, since exchanging the node has influence on their edges (such as EOG, DFG, etc.), we * only support a replacement very early in the pass system. To be specific, we only allow - * replacement before any DFG edges are set. We are re-wiring EOG edges, but nothing else. If one + * replacement BEFORE any DFG edges are set. We are re-wiring EOG edges, but nothing else. If one * tries to replace a node with existing [Node.nextDFG] or [Node.prevDFG], we fail. * - We also migrate [HasType.typeObservers] from the [old] to the [new] node. + * - Lastly, if the [new] node is a [CallExpression.callee] of a [CallExpression] parent, and the + * [old] and [new] expressions are of different types (e.g., exchanging a simple [Reference] for a + * [MemberExpression]), we also replace the [CallExpression] with a [MemberCallExpression]. */ context(ContextProvider) fun SubgraphWalker.ScopedWalker.replace(parent: Node?, old: Expression, new: Expression): Boolean { @@ -370,6 +377,25 @@ fun SubgraphWalker.ScopedWalker.replace(parent: Node?, old: Expression, new: Exp val success = when (parent) { + is CallExpression -> { + if (parent.callee == old) { + // Now we are running into a problem. If the previous callee and the new callee + // are of different types (ref/vs. member expression). We also need to replace + // the whole call expression instead. + if (parent is MemberCallExpression && new is Reference) { + val newCall = parent.toCallExpression(new) + return replace(parent.astParent, parent, newCall) + } else if (new is MemberExpression) { + val newCall = parent.toMemberCallExpression(new) + return replace(parent.astParent, parent, newCall) + } else { + parent.callee = new + true + } + } else { + parent.replace(old, new) + } + } is ArgumentHolder -> parent.replace(old, new) is StatementHolder -> parent.replace(old, new) else -> { @@ -399,9 +425,45 @@ fun SubgraphWalker.ScopedWalker.replace(parent: Node?, old: Expression, new: Exp new.registerTypeObserver(it) } + old.astParent = null + new.astParent = parent + // Make sure to inform the walker about our change this.registerReplacement(old, new) } return success } + +private fun CallExpression.duplicateTo(call: CallExpression, callee: Reference) { + call.ctx = this.ctx + call.language = this.language + call.scope = this.scope + call.arguments = this.arguments + call.type = this.type + call.assignedTypes = this.assignedTypes + call.code = this.code + call.location = this.location + call.argumentIndex = this.argumentIndex + call.annotations = this.annotations + call.comment = this.comment + call.file = this.file + call.callee = callee + callee.resolutionHelper = call + call.isImplicit = this.isImplicit + call.isInferred = this.isInferred +} + +fun MemberCallExpression.toCallExpression(callee: Reference): CallExpression { + val call = CallExpression() + duplicateTo(call, callee) + + return call +} + +fun CallExpression.toMemberCallExpression(callee: MemberExpression): MemberCallExpression { + val call = MemberCallExpression() + duplicateTo(call, callee) + + return call +} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ResolveMemberExpressionAmbiguityPass.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ResolveMemberExpressionAmbiguityPass.kt new file mode 100644 index 0000000000..622245f034 --- /dev/null +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ResolveMemberExpressionAmbiguityPass.kt @@ -0,0 +1,135 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.passes + +import de.fraunhofer.aisec.cpg.TranslationContext +import de.fraunhofer.aisec.cpg.frontends.HasCallExpressionAmbiguity +import de.fraunhofer.aisec.cpg.frontends.HasMemberExpressionAmbiguity +import de.fraunhofer.aisec.cpg.graph.Name +import de.fraunhofer.aisec.cpg.graph.codeAndLocationFrom +import de.fraunhofer.aisec.cpg.graph.declarations.NamespaceDeclaration +import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration +import de.fraunhofer.aisec.cpg.graph.fqn +import de.fraunhofer.aisec.cpg.graph.imports +import de.fraunhofer.aisec.cpg.graph.newReference +import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression +import de.fraunhofer.aisec.cpg.graph.statements.expressions.MemberExpression +import de.fraunhofer.aisec.cpg.graph.translationUnit +import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker +import de.fraunhofer.aisec.cpg.helpers.replace +import de.fraunhofer.aisec.cpg.passes.configuration.DependsOn +import de.fraunhofer.aisec.cpg.passes.configuration.ExecuteBefore +import de.fraunhofer.aisec.cpg.passes.configuration.RequiresLanguageTrait + +/** + * A translation unit pass that resolves ambiguities in member expressions within a translation + * unit. This pass checks whether the base or member name in a member expression refers to an import + * and, if so, replaces the member expression with a reference using the fully qualified name. + * + * This pass is dependent on the [ImportResolver] pass and requires the language trait + * [HasCallExpressionAmbiguity]. It is executed before the [EvaluationOrderGraphPass]. + * + * @constructor Initializes the pass with the provided translation context. + */ +@ExecuteBefore(EvaluationOrderGraphPass::class) +@DependsOn(ImportResolver::class) +@RequiresLanguageTrait(HasMemberExpressionAmbiguity::class) +class ResolveMemberExpressionAmbiguityPass(ctx: TranslationContext) : TranslationUnitPass(ctx) { + + lateinit var walker: SubgraphWalker.ScopedWalker + + override fun accept(tu: TranslationUnitDeclaration) { + walker = SubgraphWalker.ScopedWalker(ctx.scopeManager) + walker.registerHandler { _, _, node -> + when (node) { + is MemberExpression -> resolveAmbiguity(node) + } + } + + walker.iterate(tu) + } + + /** + * Resolves ambiguities in a given member expression. Checks whether the base or member name of + * the member expression refers to an import, and if so, replaces the member expression with a + * reference that uses the fully qualified name. + * + * @param me The member expression to disambiguate and potentially replace. + */ + private fun resolveAmbiguity(me: MemberExpression) { + // We need to check, if our "base" (or our expression) is really a name that refers to an + // import, because in this case we do not have a member expression, but a reference with a + // qualified name + val baseName = me.base.reconstructedImportName + var isImportedNamespace = isImportedNamespace(baseName, me) + + if (isImportedNamespace) { + with(me) { + val ref = newReference(baseName.fqn(me.name.localName)).codeAndLocationFrom(this) + walker.replace(me.astParent, me, ref) + } + } + } + + private fun isImportedNamespace(name: Name, hint: Expression): Boolean { + val resolved = + scopeManager.lookupSymbolByName( + name, + language = hint.language, + location = hint.location, + startScope = hint.scope + ) + var isImportedNamespace = resolved.singleOrNull() is NamespaceDeclaration + if (!isImportedNamespace) { + // It still could be an imported namespace of an imported package that we do not know. + // The problem is that we do not really know at this point whether we import a + // (sub)module or a global variable of the namespace. We tend to assume that this is a + // namespace + val imports = hint.translationUnit.imports + isImportedNamespace = + imports.any { it.name.lastPartsMatch(name) || it.name.startsWith(name) } + } + return isImportedNamespace + } + + override fun cleanup() { + // Nothing to do + } +} + +/** + * This utility function tries to reconstruct the name as if the expression was part of an imported + * symbol. This is needed because the [MemberExpression.name] includes the [MemberExpression.base]'s + * type instead of the name, and thus it might be "UNKNOWN". + */ +val Expression.reconstructedImportName: Name + get() { + return if (this is MemberExpression) { + this.base.reconstructedImportName.fqn(this.name.localName) + } else { + this.name + } + } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/SymbolResolver.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/SymbolResolver.kt index 8d85ae954c..1cf4fe3a36 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/SymbolResolver.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/SymbolResolver.kt @@ -106,7 +106,7 @@ open class SymbolResolver(ctx: TranslationContext) : ComponentPass(ctx) { // Gather all resolution EOG starters; and make sure they really do not have a // predecessor, otherwise we might analyze a node multiple times - val nodes = it.allEOGStarters.filter { it.prevEOGEdges.isEmpty() } + val nodes = it.allEOGStarters.filter { it.prevEOGEdges.isEmpty } for (node in nodes) { walker.iterate(node) @@ -143,7 +143,12 @@ open class SymbolResolver(ctx: TranslationContext) : ComponentPass(ctx) { // If we didn't find anything, we create a new function or method declaration if (target == null) { // Determine the scope where we want to start our inference - var (scope, _) = scopeManager.extractScope(reference) + val extractedScope = scopeManager.extractScope(reference) + if (extractedScope == null) { + return null + } + + var scope = extractedScope.scope if (scope !is NameScope) { scope = null } @@ -528,18 +533,26 @@ open class SymbolResolver(ctx: TranslationContext) : ComponentPass(ctx) { setOf(), mapOf(), setOf(), - CallResolutionResult.SuccessKind.UNRESOLVED, + UNRESOLVED, source.scope, ) val language = source.language if (language == null) { - result.success = CallResolutionResult.SuccessKind.PROBLEMATIC + result.success = PROBLEMATIC return result } // Set the start scope. This can either be the call's scope or a scope specified in an FQN - val (scope, _) = ctx.scopeManager.extractScope(source, source.scope) + val extractedScope = ctx.scopeManager.extractScope(source, source.scope) + + // If we could not extract the scope (even though one was specified), we can only return an + // empty result + if (extractedScope == null) { + return result + } + + val scope = extractedScope.scope result.actualStartScope = scope ?: source.scope // If the function does not allow function overloading, and we have multiple candidate @@ -569,7 +582,7 @@ open class SymbolResolver(ctx: TranslationContext) : ComponentPass(ctx) { // If we have a "problematic" result, we can stop here. In this case we cannot really // determine anything more. - if (result.success == CallResolutionResult.SuccessKind.PROBLEMATIC) { + if (result.success == PROBLEMATIC) { result.bestViable = result.viableFunctions return result } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/PassHelper.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/PassHelper.kt index f88a8fc3dc..37014c17c0 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/PassHelper.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/PassHelper.kt @@ -64,7 +64,8 @@ import kotlin.collections.forEach */ fun Pass<*>.tryNamespaceInference(name: Name, locationHint: Node?): NamespaceDeclaration? { // Determine the scope where we want to start our inference - var (scope, _) = scopeManager.extractScope(name, location = locationHint?.location) + val extractedScope = scopeManager.extractScope(name, location = locationHint?.location) + var scope = extractedScope?.scope if (scope !is NameScope) { scope = null @@ -99,7 +100,9 @@ internal fun Pass<*>.tryRecordInference( "class" } // Determine the scope where we want to start our inference - var (scope, _) = scopeManager.extractScope(type, scope = type.scope) + val extractedScope = + scopeManager.extractScope(type.name, location = locationHint?.location, scope = type.scope) + var scope = extractedScope?.scope if (scope !is NameScope) { scope = null @@ -173,8 +176,8 @@ internal fun Pass<*>.tryVariableInference( } else if (ref.name.isQualified()) { // For now, we only infer globals at the top-most global level, i.e., no globals in // namespaces - val (scope, _) = scopeManager.extractScope(ref, null) - when (scope) { + val extractedScope = scopeManager.extractScope(ref, null) + when (val scope = extractedScope?.scope) { is NameScope -> { log.warn( "We should infer a namespace variable ${ref.name} at this point, but this is not yet implemented." @@ -218,7 +221,8 @@ internal fun Pass<*>.tryFieldInference( ): VariableDeclaration? { // We only want to infer fields here, this can either happen if we have a reference with an // implicit receiver or if we have a scoped reference and the scope points to a record - val (scope, _) = scopeManager.extractScope(ref) + val extractedScope = scopeManager.extractScope(ref) + val scope = extractedScope?.scope if (scope != null && scope !is RecordScope) { return null } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Common.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Common.kt new file mode 100644 index 0000000000..27a25541ae --- /dev/null +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Common.kt @@ -0,0 +1,352 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.persistence + +import de.fraunhofer.aisec.cpg.graph.Name +import de.fraunhofer.aisec.cpg.graph.Node +import de.fraunhofer.aisec.cpg.graph.Persistable +import de.fraunhofer.aisec.cpg.graph.edges.collections.EdgeCollection +import de.fraunhofer.aisec.cpg.graph.edges.collections.EdgeList +import de.fraunhofer.aisec.cpg.helpers.neo4j.NameConverter +import java.math.BigInteger +import kotlin.reflect.KClass +import kotlin.reflect.KProperty1 +import kotlin.reflect.KTypeProjection +import kotlin.reflect.KVariance +import kotlin.reflect.KVisibility +import kotlin.reflect.full.createInstance +import kotlin.reflect.full.createType +import kotlin.reflect.full.hasAnnotation +import kotlin.reflect.full.isSubtypeOf +import kotlin.reflect.full.memberProperties +import kotlin.reflect.full.superclasses +import kotlin.reflect.full.withNullability +import kotlin.reflect.jvm.javaField +import kotlin.reflect.jvm.javaType +import kotlin.uuid.Uuid +import org.neo4j.ogm.annotation.Relationship +import org.neo4j.ogm.annotation.Relationship.Direction.INCOMING +import org.neo4j.ogm.annotation.typeconversion.Convert +import org.neo4j.ogm.typeconversion.AttributeConverter +import org.neo4j.ogm.typeconversion.CompositeAttributeConverter + +/** + * A cache used to store and retrieve sets of labels associated with specific Kotlin class types. + * + * This mutable map uses a Kotlin class type as the key and a set of strings representing associated + * labels as the value. The [labelCache] provides efficient lookup and prevents redundant + * re-computation of labels for the same class type. + */ +val labelCache: MutableMap, Set> = mutableMapOf() + +/** + * A cache mapping classes of type [Persistable] to their respective properties. + * + * This mutable map stores metadata about [Persistable] objects. For each specific class that + * implements the [Persistable] interface, it caches a mapping between property names and their + * corresponding [KProperty1] references. This allows efficient reflection-based access to class + * properties without repeatedly inspecting the class at runtime. + * + * The key in the map is the [KClass] of a subclass of [Persistable]. The value is a [Map] where the + * keys are strings representing the property names, and the values are [KProperty1] references + * pointing to those properties. This can be used for dynamic property access or serialization + * processes. + */ +val schemaPropertiesCache: + MutableMap, Map>> = + mutableMapOf() + +/** A cache mapping classes of type [Persistable] to their respective properties. */ +val schemaRelationshipCache: + MutableMap, Map>> = + mutableMapOf() + +/** + * Returns the [Persistable]'s properties. This DOES NOT include relationships, but only properties + * directly attached to the node/edge. + */ +fun Persistable.properties(): Map { + val properties = mutableMapOf() + for (entry in this::class.schemaProperties) { + val value = entry.value.call(this) + + if (value == null) { + continue + } + + value.convert(entry, properties) + } + + return properties +} + +/** + * Runs any conversions that are necessary by [CompositeAttributeConverter] and + * [AttributeConverter]. Since both of these classes are Neo4J OGM classes, we need to find new base + * types at some point. + */ +fun Any.convert( + entry: Map.Entry>, + properties: MutableMap +) { + val originalKey = entry.key + + val annotation = entry.value.javaField?.getAnnotation(Convert::class.java) + @Suppress("UNCHECKED_CAST") + if (annotation != null) { + val converter = annotation.value.createInstance() + if (converter is CompositeAttributeConverter<*>) { + properties += (converter as CompositeAttributeConverter).toGraphProperties(this) + } else if (converter is AttributeConverter<*, *>) { + properties.put( + originalKey, + (converter as AttributeConverter).toGraphProperty(this) + ) + } + } else if (this is Name && originalKey == "name") { + // needs to be extra because of the way annotations work, this will be re-designed once OGM + // is completely gone + properties += NameConverter().toGraphProperties(this) + } else if (this is Enum<*>) { + properties.put(originalKey, this.name) + } else if (this is Uuid) { + properties.put(originalKey, this.toString()) + } else if (this is BigInteger) { + properties.put(originalKey, this.toString()) + } else { + properties.put(originalKey, this) + } +} + +/** + * Represents a computed property for obtaining a set of labels associated with a Kotlin class. + * + * Recursively collects labels from the class hierarchy, including superclass labels, and adds the + * simple name of the current class to the set of labels. + * + * Interfaces and the Kotlin base class `Any` are excluded from the labels. The results are cached + * to improve performance. + */ +val KClass<*>.labels: Set + get() { + // Ignore interfaces and the Kotlin base class + if (this.java.isInterface || this == Any::class) { + return setOf() + } + + val cacheKey = this + + // Note: we cannot use computeIfAbsent here, because we are calling our function + // recursively and this would result in a ConcurrentModificationException + if (labelCache.containsKey(cacheKey)) { + return labelCache[cacheKey] ?: setOf() + } + + val labels = mutableSetOf() + labels.addAll(this.superclasses.flatMap { it.labels }) + this.simpleName?.let { labels.add(it) } + + // update the cache + labelCache[cacheKey] = labels + return labels + } + +internal val kClassType = KClass::class.createType(listOf(KTypeProjection.STAR)) +internal val nodeType = Node::class.createType() +internal val collectionType = Collection::class.createType(listOf(KTypeProjection.STAR)) +internal val collectionOfNodeType = + Collection::class.createType(listOf(KTypeProjection(variance = KVariance.OUT, type = nodeType))) +internal val edgeCollectionType = + EdgeCollection::class.createType(listOf(KTypeProjection.STAR, KTypeProjection.STAR)) +internal val mapType = Map::class.createType(listOf(KTypeProjection.STAR, KTypeProjection.STAR)) + +/** + * Retrieves a map of schema properties (not relationships!) for the given class implementing + * [Persistable]. + * + * This property computes a map that associates property names (as strings) to their corresponding + * [KProperty1] objects, which represent the properties defined in the class. Only properties for + * which [isSimpleProperty] returns true, are included. + * + * The computed map is cached to optimize subsequent lookups for properties of the same class. + */ +val KClass.schemaProperties: Map> + get() { + // Check, if we already computed the properties for this node's class + return schemaPropertiesCache.computeIfAbsent(this) { + val schema = mutableMapOf>() + val properties = it.memberProperties + for (property in properties) { + if (isSimpleProperty(property)) { + schema.put(property.name, property) + } + } + schema + } + } + +/** + * Provides a property that computes and returns a map of "relationships" for a given class + * implementing the [Persistable] interface. + * + * The relationships are represented as a `Map` where: + * - The key is the name of the relationship as a `String`. + * - The value is a reference to the property representing the relationship, encapsulated as a + * [KProperty1]. + * + * A "relationship" is determined based on a specific set of criteria defined by the + * [isRelationship] function. These criteria evaluate properties that are associated with other + * nodes in a graph model, excluding fields explicitly marked to skip persistence. + * + * The computed relationships are cached for performance optimization to ensure that repeated + * lookups do not re-evaluate the relationships for the same class. + * + * This property enhances schema introspection, allowing retrieval of relational data connections + * within classes modeled as entities in a graph database. + */ +val KClass.schemaRelationships: Map> + get() { + // Check, if we already computed the relationship for this node's class + return schemaRelationshipCache.computeIfAbsent(this) { + val schema = mutableMapOf>() + val properties = it.memberProperties + for (property in properties) { + if (isRelationship(property)) { + val name = property.relationshipName + schema.put(name, property) + } + } + schema + } + } + +/** + * Evaluates whether a given property qualifies as a "simple" property based on its characteristics. + * + * This evaluates to true, when + * - The property is not a list (see [collectionType]) + * - The property is not a map (see [mapType]) + * - The property is not a [KClass] + * - The property is not referring to a [Node] + * - The property is not an interface + * - The property does not have the annotation [DoNotPersist] and its return type does not have the + * annotation [DoNotPersist] + * + * @param property the property to be evaluated, belonging to a class implementing the Persistable + * interface + * @return true if the property satisfies the conditions of being a "simple" property, false + * otherwise + */ +private fun isSimpleProperty(property: KProperty1): Boolean { + val returnType = property.returnType.withNullability(false) + + return when { + property.visibility == KVisibility.PRIVATE -> false + property.hasAnnotation() -> false + returnType.hasAnnotation() -> false + (returnType.javaType as? Class<*>)?.getAnnotation(DoNotPersist::class.java) != null -> false + returnType.isSubtypeOf(kClassType) -> false + returnType.isSubtypeOf(collectionType) -> false + returnType.isSubtypeOf(mapType) -> false + returnType.isSubtypeOf(nodeType) -> false + (returnType.javaType as? Class<*>)?.isInterface == true -> false + else -> true + } +} + +/** + * Evaluates whether a given property qualifies as a "relationship" based on its characteristics. + * + * This evaluates to true, when + * - The property is not a delegate (Note: this might change in the future, once we re-design node + * properties if Neo4J OGM is completely removed) + * - The property is an [EdgeList] + * - The property is referring to a [Collection] of [Node] objects + * - The property is referring to a [Node] + * - The property does not have the annotation [DoNotPersist] + * - The property does not have the annotation [org.neo4j.ogm.annotation.Relationship] with an + * incoming direction (Note: We will replace this with our own annotation at some point) + * + * @param property the property to be evaluated, belonging to a class implementing the Persistable + * interface + * @return true if the property satisfies the conditions of being a "relationship", false otherwise + */ +private fun isRelationship(property: KProperty1): Boolean { + val returnType = property.returnType.withNullability(false) + + return when { + property.hasAnnotation() -> false + property.javaField?.type?.simpleName?.contains("Delegate") == true -> false + property.javaField?.getAnnotation(Relationship::class.java)?.direction == INCOMING -> false + property.visibility == KVisibility.PRIVATE -> false + returnType.isSubtypeOf(edgeCollectionType) -> true + returnType.isSubtypeOf(collectionOfNodeType) -> true + returnType.isSubtypeOf(nodeType) -> true + else -> false + } +} + +/** + * Retrieves the relational name associated with a property in the context of the raph schema. + * + * The `relationshipName` is determined based on the following rules: + * - If the property is annotated with the `@Relationship` annotation, the value of the annotation + * is used as the relationship name, provided it is non-null and not an empty string. + * - If the property name ends with "Edge", this suffix is removed. This adjustment is made to + * account for cases where two variables represent an edge, one named with "Edge" and another as + * the delegate without the suffix. The desired name is the one without "Edge". + * - The resulting name is converted to UPPER_SNAKE_CASE for standardization. + */ +val KProperty1.relationshipName: String + get() { + // If we have a (legacy) Neo4J annotation for our relationship, we take this one + // Note: We will replace this with something else in the future + val value = this.javaField?.getAnnotation(Relationship::class.java)?.value + if (value != null && value != "") { + return value + } + + // If the name ends with "Edge", we cut that of, since we always have two + // variables for real edges, one called "exampleEdge" and one just called + // "example" (which is only a delegate), but the name we want is "example". + // + // Replace camel case with UPPER_CASE + return this.name.substringBeforeLast("Edge").toUpperSnakeCase() + } + +/** + * Converts the current string to UPPER_SNAKE_CASE format. + * + * Each word boundary in camelCase or PascalCase naming convention is replaced with an underscore, + * and all characters are converted to uppercase. This is commonly used for representing constants + * or environment-style variable names. + * + * @return A string converted to UPPER_SNAKE_CASE. + */ +fun String.toUpperSnakeCase(): String { + val pattern = "(?<=.)[A-Z]".toRegex() + return this.replace(pattern, "_$0").uppercase() +} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/DoNotPersist.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/DoNotPersist.kt new file mode 100644 index 0000000000..3843d71344 --- /dev/null +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/DoNotPersist.kt @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.persistence + +/** This annotation is used to denote that this property or class should not be persisted */ +@Target(AnnotationTarget.PROPERTY, AnnotationTarget.CLASS) annotation class DoNotPersist() diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/persistence/TestCommon.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/persistence/TestCommon.kt new file mode 100644 index 0000000000..8d68203300 --- /dev/null +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/persistence/TestCommon.kt @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.persistence + +import de.fraunhofer.aisec.cpg.TranslationResult +import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration +import kotlin.test.Test +import kotlin.test.assertEquals + +class TestCommon { + @Test + fun testSchemaProperties() { + val properties = FunctionDeclaration::class.schemaProperties + assertEquals( + setOf( + "complexity", + "isDefinition", + "signature", + "argumentIndex", + "code", + "comment", + "file", + "id", + "isImplicit", + "isInferred", + "location", + "name" + ), + properties.keys + ) + } + + @Test + fun testSchemaRelationships() { + var relationships = FunctionDeclaration::class.schemaRelationships + assertEquals( + listOf( + "ANNOTATIONS", + "ASSIGNED_TYPES", + "AST", + "BODY", + "CDG", + "DEFINES", + "DFG", + "EOG", + "LANGUAGE", + "OVERRIDES", + "PARAMETERS", + "PDG", + "RETURN_TYPES", + "SCOPE", + "SIGNATURE_TYPES", + "THROWS_TYPES", + "TYPE", + "USAGE", + ), + relationships.keys.sorted() + ) + + relationships = TranslationResult::class.schemaRelationships + assertEquals( + listOf( + "ADDITIONAL_NODES", + "ANNOTATIONS", + "AST", + "CDG", + "COMPONENTS", + "DFG", + "EOG", + "LANGUAGE", + "PDG", + "SCOPE", + ), + relationships.keys.sorted() + ) + } +} diff --git a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/DeclarationHandler.kt b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/DeclarationHandler.kt index 210aa97d68..0656e7d21e 100644 --- a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/DeclarationHandler.kt +++ b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/DeclarationHandler.kt @@ -242,18 +242,20 @@ class DeclarationHandler(lang: CXXLanguageFrontend) : */ private val IASTSimpleDeclaration.isTypedef: Boolean get() { - return if (this.rawSignature.contains("typedef")) { - if (this.declSpecifier is CPPASTCompositeTypeSpecifier) { - // we need to make a difference between structs that have typedefs and structs - // that are typedefs themselves - this.declSpecifier.toString() == "struct" && - this.rawSignature.trim().startsWith("typedef") + return if (this.declSpecifier is IASTCompositeTypeSpecifier) { + if (this.declSpecifier.rawSignature.contains("typedef")) { + // This is very stupid. For composite type specifiers, we need to make sure that + // we do not match simply because our declarations contain a typedef. + // The problem is that we cannot correctly detect the case where both our "main" + // declaration and our sub declarations contain a typedef :( + (this.declSpecifier as IASTCompositeTypeSpecifier).getDeclarations(true).none { + it.rawSignature.contains("typedef") + } } else { - - true + false } } else { - false + this.declSpecifier.rawSignature.contains("typedef") } } diff --git a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/types/TypedefTest.kt b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/types/TypedefTest.kt index 319cb8b077..0a85008599 100644 --- a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/types/TypedefTest.kt +++ b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/types/TypedefTest.kt @@ -25,6 +25,8 @@ */ package de.fraunhofer.aisec.cpg.enhancements.types +import de.fraunhofer.aisec.cpg.InferenceConfiguration.Companion.builder +import de.fraunhofer.aisec.cpg.frontends.cxx.CLanguage import de.fraunhofer.aisec.cpg.frontends.cxx.CPPLanguage import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.ValueDeclaration @@ -34,6 +36,7 @@ import de.fraunhofer.aisec.cpg.graph.types.NumericType import de.fraunhofer.aisec.cpg.graph.types.ObjectType import de.fraunhofer.aisec.cpg.graph.variables import de.fraunhofer.aisec.cpg.test.* +import java.io.File import java.nio.file.Path import kotlin.test.* @@ -187,7 +190,7 @@ internal class TypedefTest : BaseTest() { fun testArbitraryTypedefLocation() { val tu = analyzeAndGetFirstTU( - listOf(topLevel.resolve("typedefs.cpp").toFile()), + listOf(topLevel.resolve("weird_typedefs.cpp").toFile()), topLevel, true ) { @@ -195,8 +198,15 @@ internal class TypedefTest : BaseTest() { } val ullong1 = tu.variables["someUllong1"] + assertNotNull(ullong1) + val ullong2 = tu.variables["someUllong2"] - assertEquals(ullong1?.type, ullong2?.type) + assertNotNull(ullong2) + assertEquals(ullong1.type, ullong2.type) + + val records = tu.records + assertEquals(2, records.size) + assertEquals(listOf("bar", "foo"), records.map { it.name.localName }) } @Test @@ -243,4 +253,40 @@ internal class TypedefTest : BaseTest() { assertNotNull(size) assertRefersTo(size, sizeField) } + + @Test + fun testTypedefStructCPP() { + val file = File("src/test/resources/cxx/typedef_struct.cpp") + val tu = + analyzeAndGetFirstTU(listOf(file), file.parentFile.toPath(), true) { + it.registerLanguage() + it.inferenceConfiguration(builder().enabled(false).build()) + } + with(tu) { + val me = tu.memberExpressions + me.forEach { assertNotNull(it.refersTo) } + + val test = tu.records.singleOrNull() + assertNotNull(test) + assertLocalName("test", test) + } + } + + @Test + fun testTypedefStructC() { + val file = File("src/test/resources/c/typedef_struct.c") + val tu = + analyzeAndGetFirstTU(listOf(file), file.parentFile.toPath(), true) { + it.registerLanguage() + it.inferenceConfiguration(builder().enabled(false).build()) + } + with(tu) { + val me = tu.memberExpressions + me.forEach { assertNotNull(it.refersTo) } + + val test = tu.records.singleOrNull() + assertNotNull(test) + assertLocalName("test", test) + } + } } diff --git a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXInferenceTest.kt b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXInferenceTest.kt index e5e991e2f7..3a368e86bb 100644 --- a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXInferenceTest.kt +++ b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXInferenceTest.kt @@ -32,13 +32,7 @@ import de.fraunhofer.aisec.cpg.graph.scopes.GlobalScope import de.fraunhofer.aisec.cpg.graph.types.BooleanType import de.fraunhofer.aisec.cpg.test.* import java.io.File -import kotlin.test.Test -import kotlin.test.assertContains -import kotlin.test.assertEquals -import kotlin.test.assertIs -import kotlin.test.assertIsNot -import kotlin.test.assertNotNull -import kotlin.test.assertTrue +import kotlin.test.* class CXXInferenceTest { @Test @@ -199,4 +193,32 @@ class CXXInferenceTest { assertEquals(pairType, pair.returnTypes.singleOrNull()) } } + + @Test + fun testInferParentClassInNamespace() { + val file = File("src/test/resources/cxx/inference/parent_inference.cpp") + val tu = + analyzeAndGetFirstTU(listOf(file), file.parentFile.toPath(), true) { + it.registerLanguage() + it.loadIncludes(false) + it.addIncludesToGraph(false) + } + assertNotNull(tu) + + val util = tu.namespaces["ABC"] + assertNotNull(util) + + val recordABCA = util.records["A"] + assertNotNull(recordABCA) + assertTrue(recordABCA.isInferred) + + val recordA = tu.records["A"] + assertNotNull(recordA) + val funcFoo = recordA.functions["foo"] + assertNotNull(funcFoo) + assertTrue(funcFoo.isInferred) + val funcBar = recordA.functions["bar"] + assertNotNull(funcBar) + assertFalse(funcBar.isInferred) + } } diff --git a/cpg-language-cxx/src/test/resources/c/typedef_struct.c b/cpg-language-cxx/src/test/resources/c/typedef_struct.c new file mode 100644 index 0000000000..788a6e8f6d --- /dev/null +++ b/cpg-language-cxx/src/test/resources/c/typedef_struct.c @@ -0,0 +1,13 @@ +typedef struct test { + int a; + int b; +} S; + +int structs() { + S s; + S t; + S* p=&s; + s.a=1; + s.b=2; + printf("%d %d\n", s.a, s.b); + } \ No newline at end of file diff --git a/cpg-language-cxx/src/test/resources/cxx/inference/parent_inference.cpp b/cpg-language-cxx/src/test/resources/cxx/inference/parent_inference.cpp new file mode 100644 index 0000000000..b4f9913b8d --- /dev/null +++ b/cpg-language-cxx/src/test/resources/cxx/inference/parent_inference.cpp @@ -0,0 +1,18 @@ +/* +namespace ABC { +struct A { + A(); + void foo(); +}; +} +*/ + +struct A : ABC::A { + A() { + foo(); + bar(); + } + void bar() { + + } +}; diff --git a/cpg-language-cxx/src/test/resources/cxx/typedef_struct.cpp b/cpg-language-cxx/src/test/resources/cxx/typedef_struct.cpp new file mode 100644 index 0000000000..cdfb0f4031 --- /dev/null +++ b/cpg-language-cxx/src/test/resources/cxx/typedef_struct.cpp @@ -0,0 +1,15 @@ +typedef struct test { + int a; + int b; +} S; + +int structs() { + S s; + S t; + S* p=&s; + s.a=1; + s.b=2; + printf("%d %d\n", s.a, s.b); + } + + long typedef bla; \ No newline at end of file diff --git a/cpg-language-cxx/src/test/resources/typedefs/weird_typedefs.cpp b/cpg-language-cxx/src/test/resources/typedefs/weird_typedefs.cpp new file mode 100644 index 0000000000..b5826bbb33 --- /dev/null +++ b/cpg-language-cxx/src/test/resources/typedefs/weird_typedefs.cpp @@ -0,0 +1,19 @@ +// typedef can be used anywhere in the decl-specifier-seq +// more conventionally spelled "typedef unsigned long long int ullong;" +unsigned long typedef long int ullong; + +// usage of type that is identical to typedef +unsigned long long int someUllong1; +// usage of typedef +ullong someUllong2; + +// also possible with structs +struct bar { + int a; + int b; +} typedef baz; + +// just some type that contains a typedef for more confusion +struct foo { + typedef const int a; +}; \ No newline at end of file diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt index 897dfd617d..2b0a74547d 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt @@ -26,7 +26,6 @@ package de.fraunhofer.aisec.cpg.frontends.python import de.fraunhofer.aisec.cpg.graph.* -import de.fraunhofer.aisec.cpg.graph.declarations.ImportDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.MethodDeclaration import de.fraunhofer.aisec.cpg.graph.statements.expressions.* import jep.python.PyObject @@ -473,19 +472,7 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : private fun handleAttribute(node: Python.AST.Attribute): Expression { var base = handle(node.value) - // We do a quick check, if this refers to an import. This is faster than doing - // this in a pass and most likely valid, since we are under the assumption that - // our current file is (more or less) complete, but we might miss some - // additional dependencies - var ref = - if (isImport(base.name)) { - // Yes, it's an import, so we need to construct a reference with an FQN - newReference(base.name.fqn(node.attr), rawNode = node) - } else { - newMemberExpression(name = node.attr, base = base, rawNode = node) - } - - return ref + return newMemberExpression(name = node.attr, base = base, rawNode = node) } private fun handleConstant(node: Python.AST.Constant): Expression { @@ -553,14 +540,6 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : return ret } - private fun isImport(name: Name): Boolean { - val decl = - frontend.scopeManager.currentScope - ?.lookupSymbol(name.localName, replaceImports = false) - ?.filterIsInstance() - return decl?.isNotEmpty() == true - } - private fun handleName(node: Python.AST.Name): Expression { val r = newReference(name = node.id, rawNode = node) diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguage.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguage.kt index 9537028190..9d27a34b89 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguage.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguage.kt @@ -42,7 +42,8 @@ class PythonLanguage : Language(), HasShortCircuitOperators, HasOperatorOverloading, - HasFunctionStyleConstruction { + HasFunctionStyleConstruction, + HasMemberExpressionAmbiguity { override val fileExtensions = listOf("py", "pyi") override val namespaceDelimiter = "." @Transient diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PythonAddDeclarationsPass.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PythonAddDeclarationsPass.kt index 1b19b1ab3d..3931fa671e 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PythonAddDeclarationsPass.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PythonAddDeclarationsPass.kt @@ -47,12 +47,15 @@ import de.fraunhofer.aisec.cpg.passes.configuration.RequiredFrontend @ExecuteBefore(SymbolResolver::class) @RequiredFrontend(PythonLanguageFrontend::class) class PythonAddDeclarationsPass(ctx: TranslationContext) : ComponentPass(ctx), LanguageProvider { + + lateinit var walker: SubgraphWalker.ScopedWalker + override fun cleanup() { // nothing to do } override fun accept(p0: Component) { - val walker = SubgraphWalker.ScopedWalker(ctx.scopeManager) + walker = SubgraphWalker.ScopedWalker(ctx.scopeManager) walker.registerHandler { _, _, currNode -> handle(currNode) } for (tu in p0.translationUnits) { diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt index e95489f9b4..616cbedde4 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt @@ -29,7 +29,10 @@ import de.fraunhofer.aisec.cpg.InferenceConfiguration import de.fraunhofer.aisec.cpg.analysis.ValueEvaluator import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.Annotation -import de.fraunhofer.aisec.cpg.graph.declarations.* +import de.fraunhofer.aisec.cpg.graph.declarations.FieldDeclaration +import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration +import de.fraunhofer.aisec.cpg.graph.declarations.ParameterDeclaration +import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration import de.fraunhofer.aisec.cpg.graph.statements.* import de.fraunhofer.aisec.cpg.graph.statements.expressions.* import de.fraunhofer.aisec.cpg.graph.types.ListType @@ -38,7 +41,6 @@ import de.fraunhofer.aisec.cpg.graph.types.ObjectType import de.fraunhofer.aisec.cpg.graph.types.SetType import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker import de.fraunhofer.aisec.cpg.passes.ControlDependenceGraphPass -import de.fraunhofer.aisec.cpg.query.value import de.fraunhofer.aisec.cpg.sarif.Region import de.fraunhofer.aisec.cpg.test.* import java.nio.file.Path @@ -1426,6 +1428,8 @@ class PythonFrontendTest : BaseTest() { assertNotNull(call) assertInvokes(call, aFunc) + assertTrue(call.isImported) + call = result.calls["a_func"] assertNotNull(call) assertInvokes(call, aFunc) @@ -1445,6 +1449,37 @@ class PythonFrontendTest : BaseTest() { call = result.calls["different.completely_different_func"] assertNotNull(call) assertInvokes(call, cCompletelyDifferentFunc) + assertTrue(call.isImported) + } + + @Test + fun testImportsWithoutDependencySource() { + val topLevel = Path.of("src", "test", "resources", "python") + val tu = + analyzeAndGetFirstTU( + listOf(topLevel.resolve("import_no_src.py").toFile()), + topLevel, + true + ) { + it.registerLanguage() + } + assertNotNull(tu) + + val barCall = tu.calls["bar"] + assertIs(barCall) + assertTrue(barCall.isImported) + + val bazCall = tu.calls["baz"] + assertIs(bazCall) + assertTrue(bazCall.isImported) + + val fooCall = tu.calls["foo"] + assertIs(fooCall) + assertTrue(fooCall.isImported) + + val foo3Call = tu.calls["foo3"] + assertIs(foo3Call) + assertTrue(foo3Call.isImported) } @Test @@ -1590,6 +1625,44 @@ class PythonFrontendTest : BaseTest() { } } + @Test + fun testImportTest() { + val topLevel = Path.of("src", "test", "resources", "python") + val tu = + analyzeAndGetFirstTU( + listOf(topLevel.resolve("import_test.py").toFile()), + topLevel, + true + ) { + it.registerLanguage() + } + assertNotNull(tu) + + val refs = tu.refs + refs.forEach { assertIsNot(it, "{${it.name}} is a member expression") } + assertEquals( + setOf("a", "b", "pkg.module.foo", "another_module.foo"), + refs.map { it.name.toString() }.toSet() + ) + } + + @Test + fun testImportVsMember() { + val topLevel = Path.of("src", "test", "resources", "python") + val tu = + analyzeAndGetFirstTU( + listOf(topLevel.resolve("import_vs_member.py").toFile()), + topLevel, + true + ) { + it.registerLanguage() + } + assertNotNull(tu) + + val refs = tu.refs + refs.forEach { assertIsNot(it) } + } + class PythonValueEvaluator : ValueEvaluator() { override fun computeBinaryOpEffect( lhsValue: Any?, diff --git a/cpg-language-python/src/test/resources/python/import_no_src.py b/cpg-language-python/src/test/resources/python/import_no_src.py new file mode 100644 index 0000000000..1f63f88cc3 --- /dev/null +++ b/cpg-language-python/src/test/resources/python/import_no_src.py @@ -0,0 +1,8 @@ +from foo import bar +from foo import bar as baz +import foo + +bar() +baz() +foo.foo() +foo.foo2.foo3() diff --git a/cpg-language-python/src/test/resources/python/import_test.py b/cpg-language-python/src/test/resources/python/import_test.py new file mode 100644 index 0000000000..b8ba5b8148 --- /dev/null +++ b/cpg-language-python/src/test/resources/python/import_test.py @@ -0,0 +1,5 @@ +import pkg.module +from pkg import another_module + +a = pkg.module.foo +b = another_module.foo diff --git a/cpg-language-python/src/test/resources/python/import_vs_member.py b/cpg-language-python/src/test/resources/python/import_vs_member.py new file mode 100644 index 0000000000..0b2e8e4cd1 --- /dev/null +++ b/cpg-language-python/src/test/resources/python/import_vs_member.py @@ -0,0 +1,8 @@ +from pkg import some_variable, function +import another_pkg +import another_pkg as alias + +a = pkg.some_variable.field +b = pkg.function() +c = another_pkg.function() +d = alias.function() \ No newline at end of file diff --git a/cpg-language-python/src/test/resources/python/modules/a.py b/cpg-language-python/src/test/resources/python/modules/a.py index 7a1891dde7..78fb505eff 100644 --- a/cpg-language-python/src/test/resources/python/modules/a.py +++ b/cpg-language-python/src/test/resources/python/modules/a.py @@ -6,3 +6,11 @@ def func(): def other_func(): print("I am other_func of a") return + + +class Foobar: + def bar(self): + pass + + +foobar = Foobar() diff --git a/cpg-language-python/src/test/resources/python/modules/main.py b/cpg-language-python/src/test/resources/python/modules/main.py index e79c07fe92..240f9dfad1 100644 --- a/cpg-language-python/src/test/resources/python/modules/main.py +++ b/cpg-language-python/src/test/resources/python/modules/main.py @@ -31,3 +31,6 @@ # these calls should resolve to module "c" completely_different_func() different.completely_different_func() + +# these should be a nested member call with an "inner" call to a qualified "a.foobar" +a.foobar.bar() diff --git a/cpg-neo4j/README.md b/cpg-neo4j/README.md index b379d89493..5ef4c51032 100644 --- a/cpg-neo4j/README.md +++ b/cpg-neo4j/README.md @@ -6,6 +6,13 @@ A simple tool to export a *code property graph* to a neo4j database. The application requires Java 17 or higher. +Please make sure, that the [APOC](https://neo4j.com/labs/apoc/) plugin is enabled on your neo4j server. It is used in mass-creating nodes and relationships. + +For example using docker: +``` +docker run -p 7474:7474 -p 7687:7687 -d -e NEO4J_AUTH=neo4j/password -e NEO4JLABS_PLUGINS='["apoc"]' neo4j:5 +``` + ## Build Build (and install) a distribution using Gradle @@ -19,16 +26,19 @@ Please remember to adjust the `gradle.properties` before building the project. ## Usage ``` -./build/install/cpg-neo4j/bin/cpg-neo4j [--infer-nodes] [--load-includes] [--no-default-passes] +./build/install/cpg-neo4j/bin/cpg-neo4j [--infer-nodes] [--load-includes] [--no-default-passes] [--no-neo4j] [--no-purge-db] [--print-benchmark] - [--use-unity-build] [--benchmark-json=] + [--schema-json] [--schema-markdown] [--use-unity-build] + [--benchmark-json=] [--custom-pass-list=] [--export-json=] [--host=] [--includes-file=] + [--max-complexity-cf-dfg=] [--password=] [--port=] [--save-depth=] [--top-level=] - [--user=] ([...] | -S= - [-S=]... | + [--user=] + [--exclusion-patterns=]... ([...] + | -S= [-S=]... | --json-compilation-database= | --list-passes) [...] The paths to analyze. If module support is @@ -37,11 +47,14 @@ Please remember to adjust the `gradle.properties` before building the project. --benchmark-json= Save benchmark results to json file --custom-pass-list= - Add custom list of passes (includes - --no-default-passes) which is passed as a - comma-separated list; give either pass name if - pass is in list, or its FQDN (e.g. + Add custom list of passes (might be used + additional to --no-default-passes) which is + passed as a comma-separated list; give either + pass name if pass is in list, or its FQDN (e.g. --custom-pass-list=DFGPass,CallResolver) + --exclusion-patterns= + Configures an exclusion pattern for files or + directories that should not be parsed --export-json= Export cpg as json --host= Set the host of the neo4j Database (default: @@ -53,6 +66,11 @@ Please remember to adjust the `gradle.properties` before building the project. The path to an optional a JSON compilation database --list-passes Prints the list available passes --load-includes Enable TranslationConfiguration option loadIncludes + --max-complexity-cf-dfg= + Performance optimisation: Limit the + ControlFlowSensitiveDFGPass to functions with a + complexity less than what is specified here. -1 + (default) means no limit is used. --no-default-passes Do not register default passes [used for debugging] --no-neo4j Do not push cpg into neo4j [used for debugging] --no-purge-db Do no purge neo4j database before pushing the cpg @@ -69,6 +87,8 @@ Please remember to adjust the `gradle.properties` before building the project. --save-depth= Performance optimisation: Limit recursion depth form neo4j OGM when leaving the AST. -1 (default) means no limit is used. + --schema-json Print the CPGs nodes and edges that they can have. + --schema-markdown Print the CPGs nodes and edges that they can have. --top-level= Set top level directory of project structure. Default: Largest common path of all source files --use-unity-build Enable unity build mode for C++ (requires @@ -96,13 +116,4 @@ $ build/install/cpg-neo4j/bin/cpg-neo4j --export-json cpg-export.json --no-neo4j ``` To export the cpg from a neo4j database, you can use the neo4j `apoc` plugin. -There it's also possible to export only parts of the graph. - -## Known issues: - -- While importing sufficiently large projects with the parameter --save-depth=-1 - a java.lang.StackOverflowError may occur. - - This error could be solved by increasing the stack size with the JavaVM option: -Xss4m - - Otherwise the depth must be limited (e.g. 3 or 5) - -- While pushing a constant value larger than 2^63 - 1 a java.lang.IllegalArgumentException occurs. +There it's also possible to export only parts of the graph. \ No newline at end of file diff --git a/cpg-neo4j/build.gradle.kts b/cpg-neo4j/build.gradle.kts index 496f37df4e..8a62618803 100644 --- a/cpg-neo4j/build.gradle.kts +++ b/cpg-neo4j/build.gradle.kts @@ -50,6 +50,7 @@ publishing { dependencies { // neo4j implementation(libs.bundles.neo4j) + implementation(libs.neo4j.driver) // Command line interface support implementation(libs.picocli) diff --git a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Neo4J.kt b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Neo4J.kt new file mode 100644 index 0000000000..2e25a93e9f --- /dev/null +++ b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Neo4J.kt @@ -0,0 +1,295 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.persistence + +import de.fraunhofer.aisec.cpg.TranslationResult +import de.fraunhofer.aisec.cpg.graph.Node +import de.fraunhofer.aisec.cpg.graph.Persistable +import de.fraunhofer.aisec.cpg.graph.edges.Edge +import de.fraunhofer.aisec.cpg.graph.edges.collections.EdgeCollection +import de.fraunhofer.aisec.cpg.graph.nodes +import de.fraunhofer.aisec.cpg.helpers.Benchmark +import de.fraunhofer.aisec.cpg.helpers.IdentitySet +import de.fraunhofer.aisec.cpg.helpers.identitySetOf +import org.neo4j.driver.Session +import org.slf4j.LoggerFactory + +/** + * Defines the number of edges to be processed in a single batch operation during persistence. + * + * This constant is used for chunking collections of edges into smaller groups to optimize write + * performance and reduce memory usage when interacting with the Neo4j database. Specifically, it + * determines the maximum size of each chunk of edges to be persisted in one batch operation. + */ +const val edgeChunkSize = 10000 + +/** + * Specifies the maximum number of nodes to be processed in a single chunk during persistence + * operations. + * + * This constant is used to control the size of batches when persisting a list of nodes to the + * database. Breaking the list into chunks of this size helps improve performance and memory + * efficiency during database writes. Each chunk is handled individually, ensuring that operations + * remain manageable even for large data sets. + */ +const val nodeChunkSize = 10000 + +internal val log = LoggerFactory.getLogger("Persistence") + +internal typealias Relationship = Map + +/** + * Persists the current [TranslationResult] into a graph database. + * + * This method performs the following actions: + * - Logs information about the number and categories of nodes (e.g., AST nodes, scopes, types, + * languages) and edges that are being persisted. + * - Collects nodes that include AST nodes, scopes, types, and languages, as well as all associated + * edges. + * - Persists the collected nodes and edges. + * - Persists additional relationships between nodes, such as those related to types, scopes, and + * languages. + * - Utilizes a benchmarking mechanism to measure and log the time taken to complete the persistence + * operation. + * + * This method relies on the following context and properties: + * - The [TranslationResult.finalCtx] property for accessing the scope manager, type manager, and + * configuration. + * - A [Session] context to perform persistence actions. + */ +context(Session) +fun TranslationResult.persist() { + val b = Benchmark(Persistable::class.java, "Persisting translation result") + + val astNodes = this@persist.nodes + val connected = astNodes.flatMap { it.connectedNodes }.toSet() + val nodes = (astNodes + connected).distinct() + + log.info( + "Persisting {} nodes: AST nodes ({}), other nodes ({})", + nodes.size, + astNodes.size, + connected.size + ) + nodes.persist() + + val relationships = nodes.collectRelationships() + + log.info("Persisting {} relationships", relationships.size) + relationships.persist() + + b.stop() +} + +/** + * Persists a list of nodes into a database in chunks for efficient processing. + * + * This function utilizes the surrounding [Session] context to execute the database write + * operations. Nodes are processed in chunks of size determined by [nodeChunkSize], and each chunk + * is persisted using Cypher queries. The process is benchmarked using the [Benchmark] utility. + * + * The function generates a list of properties for the nodes, which includes their labels and other + * properties. These properties are used to construct Cypher queries that create nodes in the + * database with the given labels and properties. + * + * The function uses the APOC library for creating nodes in the database. For each node in the list, + * it extracts the labels and properties and executes the Cypher query to persist the node. + */ +context(Session) +private fun List.persist() { + this.chunked(nodeChunkSize).map { chunk -> + val b = Benchmark(Persistable::class.java, "Persisting chunk of ${chunk.size} nodes") + val params = + mapOf("props" to chunk.map { mapOf("labels" to it::class.labels) + it.properties() }) + this@Session.executeWrite { tx -> + tx.run( + """ + UNWIND ${"$"}props AS map + WITH map, apoc.map.removeKeys(map, ['labels']) AS properties + CALL apoc.create.node(map.labels, properties) YIELD node + RETURN node + """, + params + ) + .consume() + } + b.stop() + } +} + +/** + * Persists a collection of edges into a Neo4j graph database within the context of a [Session]. + * + * This method ensures that the required index for node IDs is created before proceeding with + * relationship creation. The edges are subdivided into chunks, and for each chunk, the + * relationships are created in the database. Neo4j does not support multiple labels on edges, so + * each edge is duplicated for each assigned label. The created relationships are associated with + * their respective nodes and additional properties derived from the edges. + * + * Constraints: + * - The session context is required to execute write transactions. + * - Edges should define their labels and properties for appropriate persistence. + * + * Mechanisms: + * - An index for [Node] IDs is created (if not already existing) to optimize matching operations. + * - Edges are chunked to avoid overloading transactional operations. + * - Relationship properties and labels are mapped before using database utilities for creation. + */ +context(Session) +private fun Collection.persist() { + // Create an index for the "id" field of node, because we are "MATCH"ing on it in the edge + // creation. We need to wait for this to be finished + this@Session.executeWrite { tx -> + tx.run("CREATE INDEX IF NOT EXISTS FOR (n:Node) ON (n.id)").consume() + } + + this.chunked(edgeChunkSize).map { chunk -> createRelationships(chunk) } +} + +context(Session) +private fun Collection>.persistEdgesOld() { + // Create an index for the "id" field of node, because we are "MATCH"ing on it in the edge + // creation. We need to wait for this to be finished + this@Session.executeWrite { tx -> + tx.run("CREATE INDEX IF NOT EXISTS FOR (n:Node) ON (n.id)").consume() + } + + this.chunked(edgeChunkSize).map { chunk -> + createRelationships( + chunk.flatMap { edge -> + // Since Neo4J does not support multiple labels on edges, but we do internally, we + // duplicate the edge for each label + edge.labels.map { label -> + mapOf( + "startId" to edge.start.id.toString(), + "endId" to edge.end.id.toString(), + "type" to label + ) + edge.properties() + } + } + ) + } +} + +/** + * Creates relationships in a graph database based on provided properties. + * + * @param props A list of maps, where each map represents properties of a relationship including + * keys such as `startId`, `endId`, and `type`. The `startId` and `endId` identify the nodes to + * connect, while `type` defines the type of the relationship. Additional properties for the + * relationship can also be included in the map. + */ +private fun Session.createRelationships( + props: List, +) { + val b = Benchmark(Persistable::class.java, "Persisting chunk of ${props.size} relationships") + val params = mapOf("props" to props) + executeWrite { tx -> + tx.run( + """ + UNWIND ${'$'}props AS map + MATCH (s:Node {id: map.startId}) + MATCH (e:Node {id: map.endId}) + WITH s, e, map, apoc.map.removeKeys(map, ['startId', 'endId', 'type']) AS properties + CALL apoc.create.relationship(s, map.type, properties, e) YIELD rel + RETURN rel + """ + .trimIndent(), + params + ) + .consume() + } + b.stop() +} + +/** + * Returns all [Node] objects that are connected with this node with some kind of relationship + * defined in [schemaRelationships]. + */ +val Persistable.connectedNodes: IdentitySet + get() { + val nodes = identitySetOf() + + for (entry in this::class.schemaRelationships) { + val value = entry.value.call(this) + if (value is EdgeCollection<*, *>) { + nodes += value.toNodeCollection() + } else if (value is List<*>) { + nodes += value.filterIsInstance() + } else if (value is Node) { + nodes += value + } + } + + return nodes + } + +private fun List.collectRelationships(): List { + val relationships = mutableListOf() + + for (node in this) { + for (entry in node::class.schemaRelationships) { + val value = entry.value.call(node) + if (value is EdgeCollection<*, *>) { + relationships += + value.map { edge -> + mapOf( + "startId" to edge.start.id.toString(), + "endId" to edge.end.id.toString(), + "type" to entry.key + ) + edge.properties() + } + } else if (value is List<*>) { + relationships += + value.filterIsInstance().map { end -> + mapOf( + "startId" to node.id.toString(), + "endId" to end.id.toString(), + "type" to entry.key + ) + } + } else if (value is Node) { + relationships += + mapOf( + "startId" to node.id.toString(), + "endId" to value.id.toString(), + "type" to entry.key + ) + } + } + } + + // Since Neo4J does not support multiple labels on edges, but we do internally, we + // duplicate the edge for each label + /*edge.labels.map { label -> + mapOf( + "startId" to edge.start.id.toString(), + "endId" to edge.end.id.toString(), + "type" to label + ) + edge.properties() + }*/ + return relationships +} diff --git a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt index ea2b225751..54fab47fb7 100644 --- a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt +++ b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt @@ -30,37 +30,30 @@ import de.fraunhofer.aisec.cpg.* import de.fraunhofer.aisec.cpg.frontends.CompilationDatabase.Companion.fromFile import de.fraunhofer.aisec.cpg.helpers.Benchmark import de.fraunhofer.aisec.cpg.passes.* +import de.fraunhofer.aisec.cpg.persistence.persist import java.io.File import java.net.ConnectException import java.nio.file.Paths import java.util.concurrent.Callable import kotlin.reflect.KClass import kotlin.system.exitProcess -import org.neo4j.driver.exceptions.AuthenticationException -import org.neo4j.ogm.config.Configuration +import org.neo4j.driver.GraphDatabase import org.neo4j.ogm.context.EntityGraphMapper import org.neo4j.ogm.context.MappingContext import org.neo4j.ogm.cypher.compiler.MultiStatementCypherCompiler import org.neo4j.ogm.cypher.compiler.builders.node.DefaultNodeBuilder import org.neo4j.ogm.cypher.compiler.builders.node.DefaultRelationshipBuilder -import org.neo4j.ogm.exception.ConnectionException import org.neo4j.ogm.metadata.MetaData -import org.neo4j.ogm.session.Session -import org.neo4j.ogm.session.SessionFactory import org.slf4j.Logger import org.slf4j.LoggerFactory import picocli.CommandLine import picocli.CommandLine.ArgGroup private const val S_TO_MS_FACTOR = 1000 -private const val TIME_BETWEEN_CONNECTION_TRIES: Long = 2000 -private const val MAX_COUNT_OF_FAILS = 10 private const val EXIT_SUCCESS = 0 private const val EXIT_FAILURE = 1 -private const val VERIFY_CONNECTION = true private const val DEBUG_PARSER = true -private const val AUTO_INDEX = "none" -private const val PROTOCOL = "bolt://" +private const val PROTOCOL = "neo4j://" private const val DEFAULT_HOST = "localhost" private const val DEFAULT_PORT = 7687 @@ -84,6 +77,14 @@ data class JsonGraph(val nodes: List, val edges: List) /** * An application to export the cpg to a neo4j database. + * + * Please make sure, that the [APOC](https://neo4j.com/labs/apoc/) plugin is enabled on your neo4j + * server. It is used in mass-creating nodes and relationships. + * + * For example using docker: + * ``` + * docker run -p 7474:7474 -p 7687:7687 -d -e NEO4J_AUTH=neo4j/password -e NEO4JLABS_PLUGINS='["apoc"]' neo4j:5 + * ``` */ class Application : Callable { @@ -251,6 +252,13 @@ class Application : Callable { ) private var topLevel: File? = null + @CommandLine.Option( + names = ["--exclusion-patterns"], + description = + ["Configures an exclusion pattern for files or directories that should not be parsed"] + ) + private var exclusionPatterns: List = listOf() + @CommandLine.Option( names = ["--benchmark-json"], description = ["Save benchmark results to json file"] @@ -381,33 +389,14 @@ class Application : Callable { * Pushes the whole translationResult to the neo4j db. * * @param translationResult, not null - * @throws InterruptedException, if the thread is interrupted while it try´s to connect to the - * neo4j db. - * @throws ConnectException, if there is no connection to bolt://localhost:7687 possible */ - @Throws(InterruptedException::class, ConnectException::class) fun pushToNeo4j(translationResult: TranslationResult) { - val bench = Benchmark(this.javaClass, "Push cpg to neo4j", false, translationResult) - log.info("Using import depth: $depth") - log.info( - "Count base nodes to save: " + - translationResult.components.size + - translationResult.additionalNodes.size - ) - - val sessionAndSessionFactoryPair = connect() - - val session = sessionAndSessionFactoryPair.first - session.beginTransaction().use { transaction -> - if (!noPurgeDb) session.purgeDatabase() - session.save(translationResult.components, depth) - session.save(translationResult.additionalNodes, depth) - transaction.commit() + val session = connect() + with(session) { + if (!noPurgeDb) executeWrite { tx -> tx.run("MATCH (n) DETACH DELETE n").consume() } + translationResult.persist() } - - session.clear() - sessionAndSessionFactoryPair.second.close() - bench.addMeasurement() + session.close() } /** @@ -420,41 +409,14 @@ class Application : Callable { * @throws ConnectException, if there is no connection to bolt://localhost:7687 possible */ @Throws(InterruptedException::class, ConnectException::class) - fun connect(): Pair { - var fails = 0 - var sessionFactory: SessionFactory? = null - var session: Session? = null - while (session == null && fails < MAX_COUNT_OF_FAILS) { - try { - val configuration = - Configuration.Builder() - .uri("$PROTOCOL$host:$port") - .credentials(neo4jUsername, neo4jPassword) - .verifyConnection(VERIFY_CONNECTION) - .build() - sessionFactory = SessionFactory(configuration, *packages) - - session = sessionFactory.openSession() - } catch (ex: ConnectionException) { - sessionFactory = null - fails++ - log.error( - "Unable to connect to localhost:7687, " + - "ensure the database is running and that " + - "there is a working network connection to it." - ) - Thread.sleep(TIME_BETWEEN_CONNECTION_TRIES) - } catch (ex: AuthenticationException) { - log.error("Unable to connect to localhost:7687, wrong username/password!") - exitProcess(EXIT_FAILURE) - } - } - if (session == null || sessionFactory == null) { - log.error("Unable to connect to localhost:7687") - exitProcess(EXIT_FAILURE) - } - assert(fails <= MAX_COUNT_OF_FAILS) - return Pair(session, sessionFactory) + fun connect(): org.neo4j.driver.Session { + val driver = + GraphDatabase.driver( + "$PROTOCOL$host:$port", + org.neo4j.driver.AuthTokens.basic(neo4jUsername, neo4jPassword) + ) + driver.verifyConnectivity() + return driver.session() } /** @@ -495,6 +457,7 @@ class Application : Callable { .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.jvm.JVMLanguage") .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.ini.IniFileLanguage") .loadIncludes(loadIncludes) + .exclusionPatterns(*exclusionPatterns.toTypedArray()) .addIncludesToGraph(loadIncludes) .debugParser(DEBUG_PARSER) .useUnityBuild(useUnityBuild) diff --git a/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/ApplicationTest.kt b/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/ApplicationTest.kt index bf40cdb2a8..e121a98116 100644 --- a/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/ApplicationTest.kt +++ b/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/ApplicationTest.kt @@ -43,9 +43,9 @@ import kotlin.test.assertNotNull import org.neo4j.ogm.annotation.Relationship import picocli.CommandLine -fun createTranslationResult(): Pair { +fun createTranslationResult(file: String = "client.cpp"): Pair { val topLevel = Paths.get("src").resolve("test").resolve("resources").toAbsolutePath() - val path = topLevel.resolve("client.cpp").toAbsolutePath() + val path = topLevel.resolve(file).toAbsolutePath() val cmd = CommandLine(Application::class.java) cmd.parseArgs(path.toString()) diff --git a/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Neo4JTest.kt b/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Neo4JTest.kt index 96f477d58b..91d2d937d3 100644 --- a/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Neo4JTest.kt +++ b/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Neo4JTest.kt @@ -25,20 +25,17 @@ */ package de.fraunhofer.aisec.cpg_vis_neo4j -import de.fraunhofer.aisec.cpg.frontends.TestLanguageFrontend -import de.fraunhofer.aisec.cpg.graph.Name -import de.fraunhofer.aisec.cpg.graph.builder.translationResult -import de.fraunhofer.aisec.cpg.graph.declarations.ImportDeclaration -import de.fraunhofer.aisec.cpg.graph.functions +import de.fraunhofer.aisec.cpg.graph.* +import de.fraunhofer.aisec.cpg.graph.statements.expressions.Literal +import java.math.BigInteger import kotlin.test.Test import kotlin.test.assertEquals -import kotlin.test.assertNotNull +import kotlin.test.assertIs import org.junit.jupiter.api.Tag @Tag("integration") class Neo4JTest { @Test - @Throws(InterruptedException::class) fun testPush() { val (application, translationResult) = createTranslationResult() @@ -49,32 +46,15 @@ class Neo4JTest { } @Test - fun testSimpleNameConverter() { - val result = - with(TestLanguageFrontend()) { - translationResult { - val import = ImportDeclaration() - import.name = Name("myname") - import.alias = Name("myname", Name("myparent"), "::") - additionalNodes += import - } - } + fun testPushVeryLong() { + val (application, translationResult) = createTranslationResult("very_long.cpp") - val app = Application() - app.pushToNeo4j(result) + assertEquals(1, translationResult.variables.size) - val sessionAndSessionFactoryPair = app.connect() + val lit = translationResult.variables["l"]?.initializer + assertIs>(lit) + assertEquals(BigInteger("10958011617037158669"), lit.value) - val session = sessionAndSessionFactoryPair.first - session.beginTransaction().use { transaction -> - val imports = session.loadAll(ImportDeclaration::class.java) - assertNotNull(imports) - - var loadedImport = imports.singleOrNull() - assertNotNull(loadedImport) - assertEquals("myname", loadedImport.alias?.localName) - - transaction.commit() - } + application.pushToNeo4j(translationResult) } } diff --git a/cpg-neo4j/src/test/resources/very_long.cpp b/cpg-neo4j/src/test/resources/very_long.cpp new file mode 100644 index 0000000000..406c77bf2a --- /dev/null +++ b/cpg-neo4j/src/test/resources/very_long.cpp @@ -0,0 +1 @@ +unsigned long long l = 10958011617037158669ull; \ No newline at end of file diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index ccefbaf875..caea156e20 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -2,6 +2,7 @@ kotlin = "2.0.20" kotlin19 = "1.9.10" neo4j = "4.0.10" +neo4j5 = "5.27.0" log4j = "2.24.0" spotless = "6.25.0" nexus-publish = "2.0.0" @@ -28,6 +29,7 @@ slf4j-api = { module = "org.slf4j:slf4j-api", version.ref = "slf4j"} apache-commons-lang3 = { module = "org.apache.commons:commons-lang3", version = "3.17.0"} neo4j-ogm-core = { module = "org.neo4j:neo4j-ogm-core", version.ref = "neo4j"} neo4j-ogm-bolt-driver = { module = "org.neo4j:neo4j-ogm-bolt-driver", version.ref = "neo4j"} +neo4j-driver = { module = "org.neo4j.driver:neo4j-java-driver", version.ref = "neo4j5"} javaparser = { module = "com.github.javaparser:javaparser-symbol-solver-core", version = "3.26.0"} jackson = { module = "com.fasterxml.jackson.module:jackson-module-kotlin", version = "2.18.0"} @@ -43,7 +45,7 @@ picocli-codegen = { module = "info.picocli:picocli-codegen", version = "4.7.0"} jep = { module = "black.ninia:jep", version = "4.2.0" } # build.yml uses grep to extract the jep verison number for CI/CD purposes llvm = { module = "org.bytedeco:llvm-platform", version = "16.0.4-1.5.9"} jruby = { module = "org.jruby:jruby-core", version = "9.4.3.0" } -jline = { module = "org.jline:jline", version = "3.27.0" } +jline = { module = "org.jline:jline", version = "3.28.0" } antlr-runtime = { module = "org.antlr:antlr4-runtime", version = "4.8-1" } # we cannot upgrade until ki-shell upgrades this! ini4j = { module = "org.ini4j:ini4j", version = "0.5.4" } @@ -53,9 +55,9 @@ mockito = { module = "org.mockito:mockito-core", version = "5.14.0"} # plugins needed for build.gradle.kts in buildSrc kotlin-gradle = { module = "org.jetbrains.kotlin:kotlin-gradle-plugin", version.ref = "kotlin" } -dokka-gradle = { module = "org.jetbrains.dokka:dokka-gradle-plugin", version = "1.9.0" } # the dokka plugin is slightly behind the main Kotlin release cycle -dokka-versioning = { module = "org.jetbrains.dokka:versioning-plugin", version = "1.9.0"} -kover-gradle = { module = "org.jetbrains.kotlinx:kover-gradle-plugin", version = "0.8.0" } +dokka-gradle = { module = "org.jetbrains.dokka:dokka-gradle-plugin", version = "2.0.0" } # the dokka plugin is slightly behind the main Kotlin release cycle +dokka-versioning = { module = "org.jetbrains.dokka:versioning-plugin", version = "2.0.0"} +kover-gradle = { module = "org.jetbrains.kotlinx:kover-gradle-plugin", version = "0.9.0" } spotless-gradle = { module = "com.diffplug.spotless:spotless-plugin-gradle", version.ref = "spotless" } nexus-publish-gradle = { module = "io.github.gradle-nexus:publish-plugin", version.ref = "nexus-publish" }