Refactor of cpg-language-python (#1282)

.github/workflows/build.yml

@@ -93,12 +93,6 @@ jobs:
           if [ -d "/opt/hostedtoolcache/Python" ]; then
             find /opt/hostedtoolcache/Python/ -name libjep.so -exec sudo cp '{}' /usr/lib/ \;
           fi
-      - name: Install pycodestyle
-        run: |
-          pip3 install pycodestyle
-      - name: Run pycodestyle
-        run: |
-          find cpg-language-python/src/main/python -iname "*.py" -print0 | xargs -n 1 -0 pycodestyle
       - uses: actions/download-artifact@v3
         with:
           name: libcpgo-arm64.dylib

README.md

@@ -127,7 +127,7 @@ In the case of Golang, the necessary native code can be found in the `src/main/g
 
 You need to install [jep](https://github.com/ninia/jep/). This can either be system-wide or in a virtual environment. Your jep version has to match the version used by the CPG (see [version catalog](./gradle/libs.versions.toml)).
 
-Currently, only Python 3.{9,10,11,12} is supported.
+Currently, only Python 3.{9,10,11,12,13} is supported.
 
 ##### System Wide
 

buildSrc/src/main/kotlin/cpg.formatting-conventions.gradle.kts

@@ -122,21 +122,6 @@ spotless {
         licenseHeader(headerWithStars).yearSeparator(" - ")
     }
 
-    python {
-        targetExclude(
-            fileTree(project.projectDir) {
-                include("**/node_modules")
-            }
-        )
-        target("src/main/**/*.py")
-        targetExclude(
-            fileTree(project.projectDir) {
-                include("src/main/nodejs/node_modules")
-            }
-        )
-        licenseHeader(headerWithHashes, "from").yearSeparator(" - ")
-    }
-
     format("golang") {
         target("src/main/golang/**/*.go")
         licenseHeader(headerWithSlashes, "package").yearSeparator(" - ")

cpg-analysis/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/ValueEvaluator.kt

@@ -56,7 +56,7 @@ open class ValueEvaluator(
      * Contains a reference to a function that gets called if the value cannot be resolved by the
      * standard behaviour.
      */
-    val cannotEvaluate: (Node?, ValueEvaluator) -> Any? = { node: Node?, _: ValueEvaluator ->
+    open val cannotEvaluate: (Node?, ValueEvaluator) -> Any? = { node: Node?, _: ValueEvaluator ->
         // end of the line, lets just keep the expression name
         if (node != null) {
             "{${node.name}}"
@@ -148,7 +148,7 @@ open class ValueEvaluator(
      * Note: this is both used by a [BinaryOperator] with basic arithmetic operations as well as
      * [AssignExpression], if [AssignExpression.isCompoundAssignment] is true.
      */
-    protected fun computeBinaryOpEffect(
+    protected open fun computeBinaryOpEffect(
         lhsValue: Any?,
         rhsValue: Any?,
         has: HasOperatorCode?,

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/AssignExpression.kt

@@ -113,7 +113,7 @@ class AssignExpression :
      * we need to later resolve this in an additional pass. The declarations are then stored in
      * [declarations].
      */
-    override var declarations = mutableListOf<VariableDeclaration>()
+    @AST override var declarations = mutableListOf<VariableDeclaration>()
 
     /** Finds the value (of [rhs]) that is assigned to the particular [lhs] expression. */
     fun findValue(lhsExpression: HasType): Expression? {

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/HasType.kt

@@ -33,6 +33,7 @@ import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression
 import de.fraunhofer.aisec.cpg.graph.statements.expressions.Literal
 import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference
 import de.fraunhofer.aisec.cpg.graph.statements.expressions.UnaryOperator
+import de.fraunhofer.aisec.cpg.graph.unknownType
 
 /**
  * This interfaces denotes that the given [Node] has a "type". Currently, we only have two known
@@ -182,3 +183,24 @@ interface HasType : ContextProvider, LanguageProvider {
         typeObservers -= typeObserver
     }
 }
+
+/**
+ * A special [HasType.TypeObserver] that can be used in cases where we cannot directly use an
+ * initializer but still want to depend on the type of the variable in [decl]. Most cases include
+ * languages that have implicit declarations that are later computed in a pass, such sa Go or
+ * Python.
+ */
+class InitializerTypePropagation(private var decl: HasType, private var tupleIdx: Int = -1) :
+    HasType.TypeObserver {
+    override fun typeChanged(newType: Type, src: HasType) {
+        if (newType is TupleType && tupleIdx != -1) {
+            decl.type = newType.types.getOrElse(tupleIdx) { decl.unknownType() }
+        } else {
+            decl.type = newType
+        }
+    }
+
+    override fun assignedTypeChanged(assignedTypes: Set<Type>, src: HasType) {
+        // TODO
+    }
+}

cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/GoExtraPass.kt

@@ -470,19 +470,4 @@ class GoExtraPass(ctx: TranslationContext) : ComponentPass(ctx) {
     override fun cleanup() {
         // Nothing to do
     }
-
-    class InitializerTypePropagation(private var decl: HasType, private var tupleIdx: Int = -1) :
-        HasType.TypeObserver {
-        override fun typeChanged(newType: Type, src: HasType) {
-            if (newType is TupleType && tupleIdx != -1) {
-                decl.type = newType.types.getOrElse(tupleIdx) { decl.unknownType() }
-            } else {
-                decl.type = newType
-            }
-        }
-
-        override fun assignedTypeChanged(assignedTypes: Set<Type>, src: HasType) {
-            // TODO
-        }
-    }
 }

cpg-language-python/build.gradle.kts

@@ -39,15 +39,10 @@ publishing {
     }
 }
 
-// add python source code to resources
-tasks {
-    processResources {
-        from("src/main/python/")
-        include("CPGPython/*.py", "cpg.py")
-    }
-}
-
 dependencies {
     // jep for python support
     api(libs.jep)
+
+    // to evaluate some test cases
+    testImplementation(project(":cpg-analysis"))
 }