Merge branch 'main' into small-dfg-fix

.github/workflows/build.yml

@@ -38,7 +38,7 @@ jobs:
           go-version: 1.21
       - name: Setup neo4j
         run: |
-          docker run -d --env NEO4J_AUTH=neo4j/password -p7474:7474 -p7687:7687 neo4j || true
+          docker run -d --env NEO4J_AUTH=neo4j/password -p7474:7474 -p7687:7687 -e NEO4JLABS_PLUGINS='["apoc"]' neo4j:5 || true
       - name: Determine Version
         run: |
           # determine version from tag

README.md

@@ -29,7 +29,14 @@ Instead of manually generating or editing the `gradle.properties` file, you can
 
 ### For Visualization Purposes
 
-In order to get familiar with the graph itself, you can use the subproject [cpg-neo4j](https://github.com/Fraunhofer-AISEC/cpg/tree/master/cpg-neo4j). It uses this library to generate the CPG for a set of user-provided code files. The graph is then persisted to a [Neo4j](https://neo4j.com/) graph database. The advantage this has for the user, is that Neo4j's visualization software [Neo4j Browser](https://neo4j.com/developer/neo4j-browser/) can be used to graphically look at the CPG nodes and edges, instead of their Java representations.
+In order to get familiar with the graph itself, you can use the subproject [cpg-neo4j](./cpg-neo4j). It uses this library to generate the CPG for a set of user-provided code files. The graph is then persisted to a [Neo4j](https://neo4j.com/) graph database. The advantage this has for the user, is that Neo4j's visualization software [Neo4j Browser](https://neo4j.com/developer/neo4j-browser/) can be used to graphically look at the CPG nodes and edges, instead of their Java representations.
+
+Please make sure, that the [APOC](https://neo4j.com/labs/apoc/) plugin is enabled on your neo4j server. It is used in mass-creating nodes and relationships.
+
+For example using docker:
+```
+docker run -p 7474:7474 -p 7687:7687 -d -e NEO4J_AUTH=neo4j/password -e NEO4JLABS_PLUGINS='["apoc"]' neo4j:5
+```
 
 ### As Library
 

build.gradle.kts

@@ -44,7 +44,7 @@ allprojects {
 
     val dokkaPlugin by configurations
     dependencies {
-        dokkaPlugin("org.jetbrains.dokka:versioning-plugin:1.9.0")
+        dokkaPlugin("org.jetbrains.dokka:versioning-plugin:2.0.0")
     }
 }
 

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt

@@ -509,7 +509,14 @@ class ScopeManager : ScopeProvider {
             return pair.second
         }
 
-        var (scope, name) = extractScope(ref, startScope)
+        val extractedScope = extractScope(ref, startScope)
+        // If the scope extraction fails, we can only return here directly without any result
+        if (extractedScope == null) {
+            return null
+        }
+
+        var scope = extractedScope.scope
+        val name = extractedScope.adjustedName
         if (scope == null) {
             scope = startScope
         }
@@ -555,23 +562,31 @@ class ScopeManager : ScopeProvider {
     }
 
     /**
-     * This function extracts a scope for the [Name], e.g. if the name is fully qualified. `null` is
-     * returned, if no scope can be extracted.
+     * This class represents the result of the [extractScope] operation. It contains a [scope]
+     * object, if a scope was found and the [adjustedName] that is normalized if any aliases were
+     * found during scope extraction.
+     */
+    data class ScopeExtraction(val scope: Scope?, val adjustedName: Name)
+
+    /**
+     * This function extracts a scope for the [Name], e.g. if the name is fully qualified (wrapped
+     * in a [ScopeExtraction] object. `null` is returned if a scope was specified, but does not
+     * exist as a [Scope] object.
      *
-     * The pair returns the extracted scope and a name that is adjusted by possible import aliases.
-     * The extracted scope is "responsible" for the name (e.g. declares the parent namespace) and
-     * the returned name only differs from the provided name if aliasing was involved at the node
-     * location (e.g. because of imports).
+     * The returned object contains the extracted scope and a name that is adjusted by possible
+     * import aliases. The extracted scope is "responsible" for the name (e.g. declares the parent
+     * namespace) and the returned name only differs from the provided name if aliasing was involved
+     * at the node location (e.g. because of imports).
      *
      * Note: Currently only *fully* qualified names are properly resolved. This function will
      * probably return imprecise results for partially qualified names, e.g. if a name `A` inside
      * `B` points to `A::B`, rather than to `A`.
      *
      * @param node the nodes name references a namespace constituted by a scope
      * @param scope the current scope relevant for the name resolution, e.g. parent of node
-     * @return a pair with the scope of node.name and the alias-adjusted name
+     * @return a [ScopeExtraction] object with the scope of node.name and the alias-adjusted name
      */
-    fun extractScope(node: HasNameAndLocation, scope: Scope? = currentScope): Pair<Scope?, Name> {
+    fun extractScope(node: HasNameAndLocation, scope: Scope? = currentScope): ScopeExtraction? {
         return extractScope(node.name, node.location, scope)
     }
 
@@ -596,7 +611,7 @@ class ScopeManager : ScopeProvider {
         name: Name,
         location: PhysicalLocation? = null,
         scope: Scope? = currentScope,
-    ): Pair<Scope?, Name> {
+    ): ScopeExtraction? {
         var n = name
         var s: Scope? = null
 
@@ -611,20 +626,18 @@ class ScopeManager : ScopeProvider {
 
             // this is a scoped call. we need to explicitly jump to that particular scope
             val scopes = filterScopes { (it is NameScope && it.name == scopeName) }
-            s =
-                if (scopes.isEmpty()) {
-                    Util.warnWithFileLocation(
-                        location,
-                        LOGGER,
-                        "Could not find the scope $scopeName needed to resolve $n"
-                    )
-                    null
-                } else {
-                    scopes[0]
-                }
+            if (scopes.isEmpty()) {
+                Util.warnWithFileLocation(
+                    location,
+                    LOGGER,
+                    "Could not find the scope $scopeName needed to resolve $n"
+                )
+                return null
+            }
+            s = scopes[0]
         }
 
-        return Pair(s, n)
+        return ScopeExtraction(s, n)
     }
 
     /**
@@ -818,7 +831,7 @@ class ScopeManager : ScopeProvider {
                 // This process has several steps:
                 // First, do a quick local lookup, to see if we have a typedef our current scope
                 // (only do this if the name is not qualified)
-                if (!alias.isQualified() && current == currentScope) {
+                if (!alias.isQualified() && current == scope) {
                     val decl = current.typedefs[alias]
                     if (decl != null) {
                         return decl.type
@@ -892,7 +905,16 @@ class ScopeManager : ScopeProvider {
         startScope: Scope? = currentScope,
         predicate: ((Declaration) -> Boolean)? = null,
     ): List<Declaration> {
-        val (scope, n) = extractScope(name, location, startScope)
+        val extractedScope = extractScope(name, location, startScope)
+        val scope: Scope?
+        val n: Name
+        if (extractedScope == null) {
+            // the scope does not exist at all
+            return listOf()
+        } else {
+            scope = extractedScope.scope
+            n = extractedScope.adjustedName
+        }
 
         // We need to differentiate between a qualified and unqualified lookup. We have a qualified
         // lookup, if the scope is not null. In this case we need to stay within the specified scope

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt

@@ -36,6 +36,7 @@ import de.fraunhofer.aisec.cpg.graph.Node
 import de.fraunhofer.aisec.cpg.passes.*
 import de.fraunhofer.aisec.cpg.passes.configuration.*
 import de.fraunhofer.aisec.cpg.passes.inference.DFGFunctionSummaries
+import de.fraunhofer.aisec.cpg.persistence.DoNotPersist
 import java.io.File
 import java.nio.file.Path
 import java.util.*
@@ -51,6 +52,7 @@ import org.slf4j.LoggerFactory
  * The configuration for the [TranslationManager] holds all information that is used during the
  * translation.
  */
+@DoNotPersist
 class TranslationConfiguration
 private constructor(
     /** Definition of additional symbols, mostly useful for C++. */
@@ -540,6 +542,7 @@ private constructor(
             //            registerPass<ControlFlowSensitiveDFGPass>()
             registerPass<FilenameMapper>()
             registerPass<ResolveCallExpressionAmbiguityPass>()
+            registerPass<ResolveMemberExpressionAmbiguityPass>()
             useDefaultPasses = true
             return this
         }

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationContext.kt

@@ -26,11 +26,13 @@
 package de.fraunhofer.aisec.cpg
 
 import de.fraunhofer.aisec.cpg.graph.Component
+import de.fraunhofer.aisec.cpg.persistence.DoNotPersist
 
 /**
  * The translation context holds all necessary managers and configurations needed during the
  * translation process.
  */
+@DoNotPersist
 class TranslationContext(
     /** The configuration for this translation. */
     val config: TranslationConfiguration,

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationResult.kt

@@ -35,6 +35,7 @@ import de.fraunhofer.aisec.cpg.graph.edges.unwrapping
 import de.fraunhofer.aisec.cpg.helpers.MeasurementHolder
 import de.fraunhofer.aisec.cpg.helpers.StatisticsHolder
 import de.fraunhofer.aisec.cpg.passes.Pass
+import de.fraunhofer.aisec.cpg.persistence.DoNotPersist
 import java.util.*
 import java.util.concurrent.ConcurrentHashMap
 import org.neo4j.ogm.annotation.Relationship
@@ -92,6 +93,7 @@ class TranslationResult(
      * @return the list of all translation units.
      */
     @Deprecated(message = "translation units of individual components should be accessed instead")
+    @DoNotPersist
     val translationUnits: List<TranslationUnitDeclaration>
         get() {
             if (components.size == 1) {

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/LanguageTraits.kt

@@ -212,10 +212,20 @@ interface HasGlobalVariables : LanguageTrait
  */
 interface HasGlobalFunctions : LanguageTrait
 
+/**
+ * A common trait for classes, in which supposed member expressions (and thus also member calls) in
+ * the form of "a.b" have an ambiguity between a real field/method access (when "a" is an object)
+ * and a qualified call because of an import, if "a" is an import / namespace.
+ *
+ * We can only resolve this after we have dealt with imports and know all symbols. Therefore, we
+ * invoke the [ResolveMemberExpressionAmbiguityPass].
+ */
+interface HasMemberExpressionAmbiguity : LanguageTrait
+
 /**
  * A common super-class for all language traits that arise because they are an ambiguity of a
  * function call, e.g., function-style casts. This means that we cannot differentiate between a
- * [CallExpression] and other expressions during the frontend and we need to invoke the
+ * [CallExpression] and other expressions during the frontend, and we need to invoke the
  * [ResolveCallExpressionAmbiguityPass] to resolve this.
  */
 sealed interface HasCallExpressionAmbiguity : LanguageTrait

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Assignment.kt

@@ -45,4 +45,6 @@ class Assignment(
 
     /** The holder of this assignment */
     @JsonIgnore val holder: AssignmentHolder
-) : Edge<Node>(value, target as Node)
+) : Edge<Node>(value, target as Node) {
+    override var labels = setOf("ASSIGMENT")
+}

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Extensions.kt

@@ -1023,13 +1023,46 @@ fun Expression?.unwrapReference(): Reference? {
 /** Returns the [TranslationUnitDeclaration] where this node is located in. */
 val Node.translationUnit: TranslationUnitDeclaration?
     get() {
-        var node: Node? = this
-        while (node != null) {
-            if (node is TranslationUnitDeclaration) {
-                return node
-            }
-            node = node.astParent
+        return firstParentOrNull { it is TranslationUnitDeclaration } as? TranslationUnitDeclaration
+    }
+
+/**
+ * This helper function be used to find out if a particular expression (usually a [CallExpression]
+ * or a [Reference]) is imported through a [ImportDeclaration].
+ *
+ * It returns a [Pair], with the [Pair.first] being a boolean value whether it was imported and
+ * [Pair.second] the [ImportDeclaration] if applicable.
+ */
+val Expression.importedFrom: List<ImportDeclaration>
+    get() {
+        if (this is CallExpression) {
+            return this.callee.importedFrom
+        } else if (this is MemberExpression) {
+            return this.base.importedFrom
+        } else if (this is Reference) {
+            val imports = this.translationUnit.imports
+
+            return if (name.parent == null) {
+                // If the name does not have a parent, this reference could directly be the name
+                // of an import, let's check
+                imports.filter { it.name.lastPartsMatch(name) }
+            } else {
+                // Otherwise, the parent name could be the import
+                imports.filter { it.name == this.name.parent }
+            } ?: listOf<ImportDeclaration>()
         }
 
-        return null
+        return listOf<ImportDeclaration>()
+    }
+
+/**
+ * Determines whether the expression is imported from another source.
+ *
+ * This property evaluates to `true` if the expression originates from an external or supplemental
+ * source by checking if the [importedFrom] property contains any entries. Otherwise, it evaluates
+ * to `false`.
+ */
+val Expression.isImported: Boolean
+    get() {
+        return this.importedFrom.isNotEmpty()
     }

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/MermaidPrinter.kt

@@ -110,7 +110,7 @@ fun <T : Edge<Node>> Node.printGraph(
 private fun Edge<Node>.label(): String {
     val builder = StringBuilder()
     builder.append("\"")
-    builder.append(this.label)
+    builder.append(this.labels.joinToString(","))
 
     if (this is Dataflow) {
         var granularity = this.granularity

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt

@@ -45,9 +45,11 @@ import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker
 import de.fraunhofer.aisec.cpg.helpers.neo4j.LocationConverter
 import de.fraunhofer.aisec.cpg.helpers.neo4j.NameConverter
 import de.fraunhofer.aisec.cpg.passes.*
+import de.fraunhofer.aisec.cpg.persistence.DoNotPersist
 import de.fraunhofer.aisec.cpg.processing.IVisitable
 import de.fraunhofer.aisec.cpg.sarif.PhysicalLocation
 import java.util.*
+import kotlin.uuid.Uuid
 import org.apache.commons.lang3.builder.ToStringBuilder
 import org.apache.commons.lang3.builder.ToStringStyle
 import org.neo4j.ogm.annotation.*
@@ -166,7 +168,7 @@ abstract class Node :
     var astChildren: List<Node> = listOf()
         get() = SubgraphWalker.getAstChildren(this)
 
-    @Transient var astParent: Node? = null
+    @DoNotPersist @Transient var astParent: Node? = null
 
     /** Virtual property for accessing [prevEOGEdges] without property edges. */
     @PopulatedByPass(EvaluationOrderGraphPass::class) var prevEOG by unwrapping(Node::prevEOGEdges)
@@ -189,7 +191,8 @@ abstract class Node :
      * Virtual property for accessing [nextDFGEdges] that have a
      * [de.fraunhofer.aisec.cpg.graph.edges.flows.FullDataflowGranularity].
      */
-    @PopulatedByPass(DFGPass::class, PointsToPass::class)
+    @DoNotPersist
+    @PopulatedByPass(DFGPass::class, PointsToPass::class, ControlFlowSensitiveDFGPass::class)
     val prevFullDFG: List<Node>
         get() {
             return prevDFGEdges
@@ -212,7 +215,8 @@ abstract class Node :
      * Virtual property for accessing [nextDFGEdges] that have a
      * [de.fraunhofer.aisec.cpg.graph.edges.flows.FullDataflowGranularity].
      */
-    @PopulatedByPass(DFGPass::class, PointsToPass::class)
+    @DoNotPersist
+    @PopulatedByPass(DFGPass::class, PointsToPass::class, ControlFlowSensitiveDFGPass::class)
     val nextFullDFG: List<Node>
         get() {
             return nextDFGEdges.filter { it.granularity is FullDataflowGranularity }.map { it.end }
@@ -252,7 +256,10 @@ abstract class Node :
     var isImplicit = false
 
     /** Required field for object graph mapping. It contains the node id. */
-    @Id @GeneratedValue var id: Long? = null
+    @DoNotPersist @Id @GeneratedValue var legacyId: Long? = null
+
+    /** Will replace [legacyId] */
+    var id: Uuid = Uuid.random()
 
     /** Index of the argument if this node is used in a function call or parameter list. */
     var argumentIndex = 0
@@ -277,6 +284,9 @@ abstract class Node :
      * further children that have no alternative connection paths to the rest of the graph.
      */
     fun disconnectFromGraph() {
+        // Disconnect all AST children first
+        this.astChildren.forEach { it.disconnectFromGraph() }
+
         nextDFGEdges.clear()
         prevDFGEdges.clear()
         prevCDGEdges.clear()
@@ -285,6 +295,8 @@ abstract class Node :
         nextPDGEdges.clear()
         nextEOGEdges.clear()
         prevEOGEdges.clear()
+
+        astParent = null
     }
 
     override fun toString(): String {

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Persistable.kt

@@ -25,4 +25,5 @@
  */
 package de.fraunhofer.aisec.cpg.graph
 
+/** This interface represents all objects that can be persisted in a graph database. */
 interface Persistable

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/Declaration.kt

@@ -28,6 +28,7 @@ package de.fraunhofer.aisec.cpg.graph.declarations
 import de.fraunhofer.aisec.cpg.graph.Node
 import de.fraunhofer.aisec.cpg.graph.scopes.Symbol
 import de.fraunhofer.aisec.cpg.graph.statements.expressions.MemoryAddress
+import de.fraunhofer.aisec.cpg.persistence.DoNotPersist
 import org.neo4j.ogm.annotation.NodeEntity
 
 /**
@@ -41,6 +42,7 @@ import org.neo4j.ogm.annotation.NodeEntity
  */
 @NodeEntity
 abstract class Declaration : Node() {
+    @DoNotPersist
     val symbol: Symbol
         get() {
             return this.name.localName