Merge branch 'main' into mk/pythonRewrite

README.md

@@ -34,7 +34,7 @@ In order to get familiar with the graph itself, you can use the subproject [cpg-
 ### As Library
 
 The most recent version is being published to Maven central and can be used as a simple dependency, either using Maven or Gradle. Since Eclipse CDT is not published on maven central, it is necessary to add a repository with a custom layout to find the released CDT files. For example, using Gradle's Kotlin syntax:
-```
+```kotlin
 repositories {
     ivy {
         setUrl("https://download.eclipse.org/tools/cdt/releases/11.0/cdt-11.0.0/plugins")
@@ -121,7 +121,7 @@ Instead of manually editing the `gradle.properties` file, you can also use the `
 
 #### Golang
 
-In the case of Golang, the necessary native code can be found in the `src/main/golang` folder of the `cpg-language-go` submodule. Gradle should automatically find JNI headers and stores the finished library in the `src/main/golang` folder. This currently only works for Linux and macOS. In order to use it in an external project, the resulting library needs to be placed somewhere in `java.library.path`.
+In the case of Golang, the necessary native code can be found in the `src/main/golang` folder of the `cpg-language-go` submodule. Gradle should automatically store the finished library in the `src/main/golang` folder. This currently only works for Linux and macOS.
 
 #### Python
 
@@ -145,7 +145,6 @@ Through the `JepSingleton`, the CPG library will look for well known paths on Li
 
 For parsing TypeScript, the necessary NodeJS-based code can be found in the `src/main/nodejs` directory of the `cpg-language-typescript` submodule. Gradle should build the script automatically, provided NodeJS (>=16) is installed. The bundles script will be placed inside the jar's resources and should work out of the box.
 
-
 ### Code Style
 
 We use [Google Java Style](https://github.com/google/google-java-format) as a formatting. Please install the appropriate plugin for your IDE, such as the [google-java-format IntelliJ plugin](https://plugins.jetbrains.com/plugin/8527-google-java-format) or [google-java-format Eclipse plugin](https://github.com/google/google-java-format/releases/download/google-java-format-1.6/google-java-format-eclipse-plugin_1.6.0.jar).
@@ -183,8 +182,7 @@ The following authors have contributed to this project (in alphabetical order):
 
 ## Contributing
 
-We are currently discussing the implementation of a Contributor License Agreement (CLA). Unfortunately,
-we cannot merge external pull requests until this issue is resolved.
+Before accepting external contributions, you need to sign our [CLA](https://cla-assistant.io/Fraunhofer-AISEC/cpg). Our CLA assistent will check, whether you already signed the CLA when you open your first pull request.
 
 ## Further reading
 

cpg-analysis/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/ValueEvaluator.kt

@@ -163,6 +163,11 @@ open class ValueEvaluator(
             "/=" -> handleDiv(lhsValue, rhsValue, expr)
             "*",
             "*=" -> handleTimes(lhsValue, rhsValue, expr)
+            "<<" -> handleShiftLeft(lhsValue, rhsValue, expr)
+            ">>" -> handleShiftRight(lhsValue, rhsValue, expr)
+            "&" -> handleBitwiseAnd(lhsValue, rhsValue, expr)
+            "|" -> handleBitwiseOr(lhsValue, rhsValue, expr)
+            "^" -> handleBitwiseXor(lhsValue, rhsValue, expr)
             ">" -> handleGreater(lhsValue, rhsValue, expr)
             ">=" -> handleGEq(lhsValue, rhsValue, expr)
             "<" -> handleLess(lhsValue, rhsValue, expr)
@@ -202,6 +207,51 @@ open class ValueEvaluator(
         }
     }
 
+    private fun handleShiftLeft(lhsValue: Any?, rhsValue: Any?, expr: Expression?): Any? {
+        return when {
+            // right side must always be an int
+            lhsValue is Int && rhsValue is Int -> lhsValue shl rhsValue
+            lhsValue is Long && rhsValue is Int -> lhsValue shl rhsValue
+            else -> cannotEvaluate(expr, this)
+        }
+    }
+
+    private fun handleShiftRight(lhsValue: Any?, rhsValue: Any?, expr: Expression?): Any? {
+        return when {
+            // right side must always be an int
+            lhsValue is Int && rhsValue is Int -> lhsValue shr rhsValue
+            lhsValue is Long && rhsValue is Int -> lhsValue shr rhsValue
+            else -> cannotEvaluate(expr, this)
+        }
+    }
+
+    private fun handleBitwiseAnd(lhsValue: Any?, rhsValue: Any?, expr: Expression?): Any? {
+        return when {
+            // left and right must be equal and only long and int are supported
+            lhsValue is Int && rhsValue is Int -> lhsValue and rhsValue
+            lhsValue is Long && rhsValue is Long -> lhsValue and rhsValue
+            else -> cannotEvaluate(expr, this)
+        }
+    }
+
+    private fun handleBitwiseOr(lhsValue: Any?, rhsValue: Any?, expr: Expression?): Any? {
+        return when {
+            // left and right must be equal and only long and int are supported
+            lhsValue is Int && rhsValue is Int -> lhsValue or rhsValue
+            lhsValue is Long && rhsValue is Long -> lhsValue or rhsValue
+            else -> cannotEvaluate(expr, this)
+        }
+    }
+
+    private fun handleBitwiseXor(lhsValue: Any?, rhsValue: Any?, expr: Expression?): Any? {
+        return when {
+            // left and right must be equal and only long and int are supported
+            lhsValue is Int && rhsValue is Int -> lhsValue xor rhsValue
+            lhsValue is Long && rhsValue is Long -> lhsValue xor rhsValue
+            else -> cannotEvaluate(expr, this)
+        }
+    }
+
     private fun handleGreater(lhsValue: Any?, rhsValue: Any?, expr: Expression?): Any? {
         return if (lhsValue is Number && rhsValue is Number) {
             lhsValue.compareTo(rhsValue) > 0

cpg-analysis/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/ValueEvaluatorTest.kt

@@ -639,6 +639,106 @@ class ValueEvaluatorTest {
         }
     }
 
+    @Test
+    fun testHandleShiftLeft() {
+        with(TestHandler(TestLanguageFrontend())) {
+            val binOp = newBinaryOperator("<<")
+            // Int.plus
+            binOp.lhs = newLiteral(3, primitiveType("int"))
+            binOp.rhs = newLiteral(2, primitiveType("int"))
+            assertEquals(12, ValueEvaluator().evaluate(binOp))
+
+            // Long.plus
+            binOp.lhs = newLiteral(3L, primitiveType("long"))
+            binOp.rhs = newLiteral(2, primitiveType("int"))
+            assertEquals(12L, ValueEvaluator().evaluate(binOp))
+
+            binOp.lhs = newLiteral("Hello", primitiveType("string"))
+            binOp.rhs = newLiteral(" world", primitiveType("string"))
+            assertEquals("{<<}", ValueEvaluator().evaluate(binOp))
+        }
+    }
+
+    @Test
+    fun testHandleShiftRight() {
+        with(TestHandler(TestLanguageFrontend())) {
+            val binOp = newBinaryOperator(">>")
+            // Int.plus
+            binOp.lhs = newLiteral(3, primitiveType("int"))
+            binOp.rhs = newLiteral(2, primitiveType("int"))
+            assertEquals(0, ValueEvaluator().evaluate(binOp))
+
+            // Long.plus
+            binOp.lhs = newLiteral(3L, primitiveType("long"))
+            binOp.rhs = newLiteral(2, primitiveType("int"))
+            assertEquals(0L, ValueEvaluator().evaluate(binOp))
+
+            binOp.lhs = newLiteral("Hello", primitiveType("string"))
+            binOp.rhs = newLiteral(" world", primitiveType("string"))
+            assertEquals("{>>}", ValueEvaluator().evaluate(binOp))
+        }
+    }
+
+    @Test
+    fun testHandleBitwiseAnd() {
+        with(TestHandler(TestLanguageFrontend())) {
+            val binOp = newBinaryOperator("&")
+            // Int.plus
+            binOp.lhs = newLiteral(3, primitiveType("int"))
+            binOp.rhs = newLiteral(2, primitiveType("int"))
+            assertEquals(2, ValueEvaluator().evaluate(binOp))
+
+            // Long.plus
+            binOp.lhs = newLiteral(3L, primitiveType("long"))
+            binOp.rhs = newLiteral(2L, primitiveType("long"))
+            assertEquals(2L, ValueEvaluator().evaluate(binOp))
+
+            binOp.lhs = newLiteral("Hello", primitiveType("string"))
+            binOp.rhs = newLiteral(" world", primitiveType("string"))
+            assertEquals("{&}", ValueEvaluator().evaluate(binOp))
+        }
+    }
+
+    @Test
+    fun testHandleBitwiseOr() {
+        with(TestHandler(TestLanguageFrontend())) {
+            val binOp = newBinaryOperator("|")
+            // Int.plus
+            binOp.lhs = newLiteral(3, primitiveType("int"))
+            binOp.rhs = newLiteral(2, primitiveType("int"))
+            assertEquals(3, ValueEvaluator().evaluate(binOp))
+
+            // Long.plus
+            binOp.lhs = newLiteral(3L, primitiveType("long"))
+            binOp.rhs = newLiteral(2L, primitiveType("long"))
+            assertEquals(3L, ValueEvaluator().evaluate(binOp))
+
+            binOp.lhs = newLiteral("Hello", primitiveType("string"))
+            binOp.rhs = newLiteral(" world", primitiveType("string"))
+            assertEquals("{|}", ValueEvaluator().evaluate(binOp))
+        }
+    }
+
+    @Test
+    fun testHandleBitwiseXor() {
+        with(TestHandler(TestLanguageFrontend())) {
+            val binOp = newBinaryOperator("^")
+            // Int.plus
+            binOp.lhs = newLiteral(3, primitiveType("int"))
+            binOp.rhs = newLiteral(2, primitiveType("int"))
+            assertEquals(1, ValueEvaluator().evaluate(binOp))
+
+            // Long.plus
+            binOp.lhs = newLiteral(3L, primitiveType("long"))
+            binOp.rhs = newLiteral(2L, primitiveType("long"))
+            assertEquals(1L, ValueEvaluator().evaluate(binOp))
+
+            binOp.lhs = newLiteral("Hello", primitiveType("string"))
+            binOp.rhs = newLiteral(" world", primitiveType("string"))
+            assertEquals("{^}", ValueEvaluator().evaluate(binOp))
+        }
+    }
+
     @Test
     fun testHandleUnary() {
         with(TestHandler(TestLanguageFrontend())) {

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt

@@ -34,6 +34,7 @@ import de.fraunhofer.aisec.cpg.graph.statements.*
 import de.fraunhofer.aisec.cpg.graph.statements.expressions.Block
 import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression
 import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference
+import de.fraunhofer.aisec.cpg.graph.statements.expressions.ReferenceTag
 import de.fraunhofer.aisec.cpg.graph.types.FunctionPointerType
 import de.fraunhofer.aisec.cpg.graph.types.IncompleteType
 import de.fraunhofer.aisec.cpg.graph.types.Type
@@ -74,6 +75,12 @@ class ScopeManager : ScopeProvider {
     /** Represents an alias with the name [to] for the particular name [from]. */
     data class Alias(var from: Name, var to: Name)
 
+    /**
+     * A cache map of unique tags (computed with [Reference.buildUniqueTag]) and their respective
+     * [ValueDeclaration]. This is used by [resolveReference] as a caching mechanism.
+     */
+    private val symbolTable = mutableMapOf<ReferenceTag, ValueDeclaration>()
+
     /**
      * In some languages, we can define aliases for names. An example is renaming package imports in
      * Go, e.g., to avoid name conflicts.
@@ -607,45 +614,62 @@ class ScopeManager : ScopeProvider {
      * Resolves only references to Values in the current scope, static references to other visible
      * records are not resolved over the ScopeManager.
      *
-     * @param scope
      * @param ref
      * @return
      *
      * TODO: We should merge this function with [.resolveFunction]
      */
-    @JvmOverloads
-    fun resolveReference(ref: Reference, startScope: Scope? = currentScope): ValueDeclaration? {
+    fun resolveReference(ref: Reference): ValueDeclaration? {
+        val startScope = ref.scope
+
+        // Retrieve a unique tag for the particular reference based on the current scope
+        val tag = ref.uniqueTag
+
+        // If we find a match in our symbol table, we can immediately return the declaration
+        var decl = symbolTable[tag]
+        if (decl != null) {
+            return decl
+        }
+
         val (scope, name) = extractScope(ref, startScope)
 
         // Try to resolve value declarations according to our criteria
-        return resolve<ValueDeclaration>(scope) {
-                if (it.name.lastPartsMatch(name)) {
-                    val helper = ref.resolutionHelper
-                    return@resolve when {
-                        // If the reference seems to point to a function (using a function pointer)
-                        // the entire signature is checked for equality
-                        helper?.type is FunctionPointerType && it is FunctionDeclaration -> {
-                            val fptrType = helper.type as FunctionPointerType
-                            // TODO(oxisto): Support multiple return values
-                            val returnType = it.returnTypes.firstOrNull() ?: IncompleteType()
-                            returnType == fptrType.returnType &&
-                                it.hasSignature(fptrType.parameters)
-                        }
-                        // If our language has first-class functions, we can safely return them as a
-                        // reference
-                        ref.language is HasFirstClassFunctions -> {
-                            true
-                        }
-                        // Otherwise, we are not looking for functions here
-                        else -> {
-                            it !is FunctionDeclaration
+        decl =
+            resolve<ValueDeclaration>(scope) {
+                    if (it.name.lastPartsMatch(name)) {
+                        val helper = ref.resolutionHelper
+                        return@resolve when {
+                            // If the reference seems to point to a function (using a function
+                            // pointer) the entire signature is checked for equality
+                            helper?.type is FunctionPointerType && it is FunctionDeclaration -> {
+                                val fptrType = helper.type as FunctionPointerType
+                                // TODO(oxisto): Support multiple return values
+                                val returnType = it.returnTypes.firstOrNull() ?: IncompleteType()
+                                returnType == fptrType.returnType &&
+                                    it.hasSignature(fptrType.parameters)
+                            }
+                            // If our language has first-class functions, we can safely return them
+                            // as a reference
+                            ref.language is HasFirstClassFunctions -> {
+                                true
+                            }
+                            // Otherwise, we are not looking for functions here
+                            else -> {
+                                it !is FunctionDeclaration
+                            }
                         }
                     }
+
+                    return@resolve false
                 }
+                .firstOrNull()
 
-                return@resolve false
-            }
-            .firstOrNull()
+        // Update the symbol cache, if we found a declaration for the tag
+        if (decl != null) {
+            symbolTable[tag] = decl
+        }
+
+        return decl
     }
 
     /**

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/Language.kt

@@ -164,6 +164,9 @@ abstract class Language<T : LanguageFrontend<*, *>> : Node() {
             "-",
             "*",
             "/" -> arithmeticOpTypePropagation(operation.lhs.type, operation.rhs.type)
+            "&",
+            "|",
+            "^",
             "<<",
             ">>" ->
                 if (operation.lhs.type.isPrimitive && operation.rhs.type.isPrimitive) {

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/Scope.kt

@@ -82,6 +82,23 @@ abstract class Scope(
         return this is LoopScope
     }
 
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (javaClass != other?.javaClass) return false
+
+        other as Scope
+
+        if (astNode != other.astNode) return false
+        return name == other.name
+    }
+
+    override fun hashCode(): Int {
+        var result = astNode?.hashCode() ?: 0
+        result = 31 * result + (parent?.hashCode() ?: 0)
+        result = 31 * result + (name?.hashCode() ?: 0)
+        return result
+    }
+
     /** Returns the [GlobalScope] of this scope by traversing its parents upwards. */
     val globalScope: Scope?
         get() {

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/Reference.kt

@@ -153,4 +153,16 @@ open class Reference : Expression(), HasType.TypeObserver {
             prev.registerTypeObserver(this)
         }
     }
+
+    /**
+     * This function builds a unique tag for the particular reference, based on the [startScope].
+     * Its purpose is to cache symbol resolutions, similar to LLVMs system of Unified Symbol
+     * Resolution (USR).
+     */
+    val uniqueTag: ReferenceTag
+        get() {
+            return Objects.hash(this.name, this.resolutionHelper, this.scope)
+        }
 }
+
+typealias ReferenceTag = Int

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/SymbolResolver.kt

@@ -205,7 +205,7 @@ open class SymbolResolver(ctx: TranslationContext) : ComponentPass(ctx) {
             // Peek into the declaration, and if it is a variable, we can proceed normally, as we
             // are running into the special case explained above. Otherwise, we abort here (for
             // now).
-            wouldResolveTo = scopeManager.resolveReference(current, current.scope)
+            wouldResolveTo = scopeManager.resolveReference(current)
             if (wouldResolveTo !is VariableDeclaration && wouldResolveTo !is ParameterDeclaration) {
                 return
             }
@@ -214,9 +214,7 @@ open class SymbolResolver(ctx: TranslationContext) : ComponentPass(ctx) {
         // Only consider resolving, if the language frontend did not specify a resolution. If we
         // already have populated the wouldResolveTo variable, we can re-use this instead of
         // resolving again
-        var refersTo =
-            current.refersTo
-                ?: wouldResolveTo ?: scopeManager.resolveReference(current, current.scope)
+        var refersTo = current.refersTo ?: wouldResolveTo ?: scopeManager.resolveReference(current)
 
         var recordDeclType: Type? = null
         if (currentClass != null) {