Improvements to the Go language

* Added parsing of more expressions

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationManager.kt

@@ -38,14 +38,11 @@ import java.io.File
 import java.io.PrintWriter
 import java.lang.reflect.InvocationTargetException
 import java.nio.file.Files
-import java.nio.file.Path
-import java.nio.file.attribute.BasicFileAttributes
 import java.util.*
 import java.util.concurrent.CompletableFuture
 import java.util.concurrent.CompletionException
 import java.util.concurrent.ExecutionException
 import java.util.concurrent.atomic.AtomicBoolean
-import java.util.stream.Collectors
 import kotlin.reflect.full.findAnnotation
 import org.slf4j.LoggerFactory
 
@@ -148,15 +145,13 @@ private constructor(
             val list =
                 sourceLocations.flatMap { file ->
                     if (file.isDirectory) {
-                        Files.find(
-                                file.toPath(),
-                                999,
-                                { _: Path?, fileAttr: BasicFileAttributes ->
-                                    fileAttr.isRegularFile
-                                }
-                            )
-                            .map { it.toFile() }
-                            .collect(Collectors.toList())
+                        val files =
+                            file
+                                .walkTopDown()
+                                .onEnter { !it.name.startsWith(".") }
+                                .filter { it.isFile && !it.name.startsWith(".") }
+                                .toList()
+                        files
                     } else {
                         val frontendClass = file.language?.frontend
                         val supportsParallelParsing =
@@ -277,7 +272,7 @@ private constructor(
                 Thread.currentThread().interrupt()
             } catch (e: ExecutionException) {
                 log.error("Error parsing ${futureToFile[future]}", e)
-                Thread.currentThread().interrupt()
+                // Thread.currentThread().interrupt()
             }
         }
 

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/Language.kt

@@ -34,11 +34,9 @@ import de.fraunhofer.aisec.cpg.TranslationContext
 import de.fraunhofer.aisec.cpg.graph.Name
 import de.fraunhofer.aisec.cpg.graph.Node
 import de.fraunhofer.aisec.cpg.graph.statements.expressions.BinaryOperator
-import de.fraunhofer.aisec.cpg.graph.statements.expressions.CastExpression
 import de.fraunhofer.aisec.cpg.graph.types.*
 import de.fraunhofer.aisec.cpg.graph.types.Type
 import de.fraunhofer.aisec.cpg.graph.unknownType
-import de.fraunhofer.aisec.cpg.helpers.Util
 import java.io.File
 import kotlin.reflect.KClass
 import kotlin.reflect.full.primaryConstructor

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Name.kt

@@ -131,6 +131,10 @@ fun LanguageProvider?.parseName(fqn: CharSequence) =
 
 /** Tries to parse the given fully qualified name using the specified [delimiter] into a [Name]. */
 internal fun parseName(fqn: CharSequence, delimiter: String, vararg splitDelimiters: String): Name {
+    if (fqn is Name) {
+        return fqn
+    }
+
     val parts = fqn.split(delimiter, *splitDelimiters)
 
     var name: Name? = null

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/GotoStatement.kt

@@ -38,9 +38,8 @@ class GotoStatement : Statement() {
         if (other !is GotoStatement) {
             return false
         }
-        return super.equals(other) &&
-            labelName == other.labelName &&
-            targetLabel == other.targetLabel
+        return super.equals(other) && labelName == other.labelName /*&&
+            targetLabel == other.targetLabel*/
     }
 
     override fun hashCode() = Objects.hash(super.hashCode(), labelName, targetLabel)

cpg-language-go/src/main/golang/lib/cpg/main.go

@@ -286,6 +286,12 @@ func GetCallExprArg(ptr unsafe.Pointer, i int) unsafe.Pointer {
 	})
 }
 
+//export GetEllipsisElt
+func GetEllipsisElt(ptr unsafe.Pointer) unsafe.Pointer {
+	expr := restore[*ast.Ellipsis](ptr)
+	return save(expr.Elt)
+}
+
 //export GetForStmtInit
 func GetForStmtInit(ptr unsafe.Pointer) unsafe.Pointer {
 	stmt := restore[*ast.ForStmt](ptr)
@@ -466,6 +472,26 @@ func GetIndexExprIndex(ptr unsafe.Pointer) unsafe.Pointer {
 	return save(expr.Index)
 }
 
+//export GetIndexListExprX
+func GetIndexListExprX(ptr unsafe.Pointer) unsafe.Pointer {
+	expr := restore[*ast.IndexListExpr](ptr)
+	return save(expr.X)
+}
+
+//export GetNumIndexListExprIndices
+func GetNumIndexListExprIndices(ptr unsafe.Pointer) C.int {
+	return num[*ast.IndexListExpr](ptr, func(t *ast.IndexListExpr) []ast.Expr {
+		return t.Indices
+	})
+}
+
+//export GetIndexListExprIndex
+func GetIndexListExprIndex(ptr unsafe.Pointer, i int) unsafe.Pointer {
+	return item[*ast.IndexListExpr](ptr, i, func(t *ast.IndexListExpr) []ast.Expr {
+		return t.Indices
+	})
+}
+
 //export GetKeyValueExprKey
 func GetKeyValueExprKey(ptr unsafe.Pointer) unsafe.Pointer {
 	kv := restore[*ast.KeyValueExpr](ptr)
@@ -478,6 +504,12 @@ func GetKeyValueExprValue(ptr unsafe.Pointer) unsafe.Pointer {
 	return save(kv.Value)
 }
 
+//export GetParenExprX
+func GetParenExprX(ptr unsafe.Pointer) unsafe.Pointer {
+	p := restore[*ast.ParenExpr](ptr)
+	return save(p.X)
+}
+
 //export GetSelectorExprX
 func GetSelectorExprX(ptr unsafe.Pointer) unsafe.Pointer {
 	sel := restore[*ast.SelectorExpr](ptr)

cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/DeclarationHandler.kt

@@ -162,14 +162,7 @@ class DeclarationHandler(frontend: GoLanguageFrontend) :
 
             // Check for varargs. In this case we want to parse the element type
             // (and make it an array afterwards)
-            var variadic = false
-            val type =
-                if (param.type is GoStandardLibrary.Ast.Ellipsis) {
-                    variadic = true
-                    frontend.typeOf((param.type as GoStandardLibrary.Ast.Ellipsis).elt).array()
-                } else {
-                    frontend.typeOf(param.type)
-                }
+            val (type, variadic) = frontend.fieldTypeOf(param)
 
             val p = newParamVariableDeclaration(name, type, variadic, rawNode = param)
 

cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/ExpressionHandler.kt

@@ -45,8 +45,12 @@ class ExpressionHandler(frontend: GoLanguageFrontend) :
             is GoStandardLibrary.Ast.IndexExpr -> handleIndexExpr(expr)
             is GoStandardLibrary.Ast.CallExpr -> handleCallExpr(expr)
             is GoStandardLibrary.Ast.KeyValueExpr -> handleKeyValueExpr(expr)
+            is GoStandardLibrary.Ast.ParenExpr -> {
+                return handle(expr.x)
+            }
             is GoStandardLibrary.Ast.SelectorExpr -> handleSelectorExpr(expr)
             is GoStandardLibrary.Ast.SliceExpr -> handleSliceExpr(expr)
+            is GoStandardLibrary.Ast.StarExpr -> handleStarExpr(expr)
             is GoStandardLibrary.Ast.TypeAssertExpr -> handleTypeAssertExpr(expr)
             is GoStandardLibrary.Ast.UnaryExpr -> handleUnaryExpr(expr)
             else -> {
@@ -69,7 +73,12 @@ class ExpressionHandler(frontend: GoLanguageFrontend) :
                 type = primitiveType("string")
             }
             INT -> {
-                value = rawValue.toInt()
+                value =
+                    if (rawValue.startsWith("0x")) {
+                        rawValue.substring(2).toInt(16)
+                    } else {
+                        rawValue.toInt()
+                    }
                 type = primitiveType("int")
             }
             FLOAT -> {
@@ -154,10 +163,25 @@ class ExpressionHandler(frontend: GoLanguageFrontend) :
             }
         }
 
-        // Parse the Fun field, to see which kind of expression it is
+        val typeConstraints = mutableListOf<Node>()
+
         val callee =
-            this.handle(callExpr.`fun`)
-                ?: return ProblemExpression("Could not parse call expr without fun")
+            when (val `fun` = callExpr.`fun`) {
+                // If "fun" is either an index or an index list expression, this is a call with type
+                // constraints. We do not fully support that yet, but we can at least try to set
+                // some of the parameters as template parameters
+                is GoStandardLibrary.Ast.IndexExpr -> {
+                    typeConstraints += frontend.typeOf(`fun`.index)
+                    this.handle(`fun`.x)
+                }
+                is GoStandardLibrary.Ast.IndexListExpr -> {
+                    typeConstraints.addAll(`fun`.indices.map { frontend.typeOf(it) })
+                    this.handle(`fun`.x)
+                }
+                else -> {
+                    this.handle(callExpr.`fun`)
+                }
+            }
 
         // Handle special functions, such as make and new in a special way
         val name = callee.name.localName
@@ -175,9 +199,18 @@ class ExpressionHandler(frontend: GoLanguageFrontend) :
                 newCallExpression(callee, name, rawNode = callExpr)
             }
 
+        // TODO(oxisto) Add type constraints
+        if (typeConstraints.isNotEmpty()) {
+            log.debug(
+                "Call {} has type constraints ({}), but we cannot add them to the call expression yet",
+                call.name,
+                typeConstraints.joinToString(", ") { it.name }
+            )
+        }
+
         // Parse and add call arguments
         for (arg in callExpr.args) {
-            handle(arg)?.let { call += it }
+            call += handle(arg)
         }
 
         return call
@@ -304,6 +337,13 @@ class ExpressionHandler(frontend: GoLanguageFrontend) :
         return ase
     }
 
+    private fun handleStarExpr(starExpr: GoStandardLibrary.Ast.StarExpr): UnaryOperator {
+        val op = newUnaryOperator("*", postfix = false, prefix = false, rawNode = starExpr)
+        op.input = handle(starExpr.x)
+
+        return op
+    }
+
     private fun handleTypeAssertExpr(
         typeAssertExpr: GoStandardLibrary.Ast.TypeAssertExpr
     ): CastExpression {
@@ -328,7 +368,7 @@ class ExpressionHandler(frontend: GoLanguageFrontend) :
                 prefix = false,
                 rawNode = unaryExpr
             )
-        handle(unaryExpr.x)?.let { op.input = it }
+        op.input = handle(unaryExpr.x)
 
         return op
     }
@@ -341,13 +381,16 @@ class ExpressionHandler(frontend: GoLanguageFrontend) :
     private fun handleCompositeLit(
         compositeLit: GoStandardLibrary.Ast.CompositeLit
     ): ConstructExpression {
-        // Parse the type field, to see which kind of expression it is
-        val type = frontend.typeOf(compositeLit.type)
+        // Parse the type field, to see which kind of expression it is. The type of a composite
+        // literal can be omitted if this is an "inner" composite literal, so we need to set it from
+        // the "outer" one. See below
+        val type = compositeLit.type?.let { frontend.typeOf(it) } ?: unknownType()
 
         val construct = newConstructExpression(type.name, rawNode = compositeLit)
         construct.type = type
 
         val list = newInitializerListExpression(type, rawNode = compositeLit)
+        list.type = type
         construct += list
 
         // Normally, the construct expression would not have DFG edge, but in this case we are
@@ -357,7 +400,34 @@ class ExpressionHandler(frontend: GoLanguageFrontend) :
 
         val expressions = mutableListOf<Expression>()
         for (elem in compositeLit.elts) {
-            handle(elem)?.let { expressions += it }
+            var expression = handle(elem)
+
+            // The type of a "inner" composite literal can be omitted if the outer one is creating
+            // an array type. In this case, we need to set the type manually because the type for
+            // the "inner" one is empty.
+            // Example code:
+            // ```go
+            // var a = []*MyObject{
+            //   {
+            //      Name: "a",
+            //   },
+            //   {
+            //      Name: "b",
+            //   }
+            // }
+            if (
+                type is PointerType &&
+                    type.isArray &&
+                    elem is GoStandardLibrary.Ast.CompositeLit &&
+                    expression is ConstructExpression
+            ) {
+                val elementType = type.dereference()
+                // Set the type of the inner composite to be the deref'd type of the outer
+                expression.type = type
+                (expression as? ConstructExpression)?.arguments?.firstOrNull()?.type = elementType
+            }
+
+            expressions += expression
         }
 
         list.initializers = expressions