From fab6e45135d7cf469a414ac0855f4fbe0c7700e1 Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Tue, 8 Aug 2023 15:26:03 +0200 Subject: [PATCH] Improvements to the Go language * Added parsing of more expressions --- .../aisec/cpg/TranslationManager.kt | 21 ++--- .../aisec/cpg/frontends/Language.kt | 2 - .../de/fraunhofer/aisec/cpg/graph/Name.kt | 4 + .../frontends/cxx/CXXLanguageFrontendTest.kt | 39 -------- .../src/main/golang/lib/cpg/main.go | 44 +++++++++ .../frontends/golang/DeclarationHandler.kt | 9 +- .../cpg/frontends/golang/ExpressionHandler.kt | 93 ++++++++++++++++--- .../frontends/golang/GoLanguageFrontend.kt | 88 +++++++++++++++++- .../cpg/frontends/golang/GoStandardLibrary.kt | 53 ++++++++++- .../frontends/golang/SpecificationHandler.kt | 10 +- .../cpg/frontends/golang/StatementHandler.kt | 9 ++ .../cpg/passes/GoEvaluationOrderGraphPass.kt | 7 +- .../cpg/frontends/golang/DeclarationTest.kt | 74 +++++++++++++++ .../cpg/frontends/golang/ExpressionTest.kt | 37 ++++++++ .../golang/GoLanguageFrontendTest.kt | 64 ------------- .../src/test/resources/golang/chan.go | 8 ++ .../src/test/resources/golang/go.mod | 2 + .../src/test/resources/golang/struct.go | 18 +++- .../test/resources/golang/type_constraints.go | 10 +- 19 files changed, 432 insertions(+), 160 deletions(-) create mode 100644 cpg-language-go/src/test/resources/golang/chan.go diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationManager.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationManager.kt index dee1e5ac014..10080518665 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationManager.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationManager.kt @@ -38,14 +38,11 @@ import java.io.File import java.io.PrintWriter import java.lang.reflect.InvocationTargetException import java.nio.file.Files -import java.nio.file.Path -import java.nio.file.attribute.BasicFileAttributes import java.util.* import java.util.concurrent.CompletableFuture import java.util.concurrent.CompletionException import java.util.concurrent.ExecutionException import java.util.concurrent.atomic.AtomicBoolean -import java.util.stream.Collectors import kotlin.reflect.full.findAnnotation import org.slf4j.LoggerFactory @@ -148,15 +145,13 @@ private constructor( val list = sourceLocations.flatMap { file -> if (file.isDirectory) { - Files.find( - file.toPath(), - 999, - { _: Path?, fileAttr: BasicFileAttributes -> - fileAttr.isRegularFile - } - ) - .map { it.toFile() } - .collect(Collectors.toList()) + val files = + file + .walkTopDown() + .onEnter { !it.name.startsWith(".") } + .filter { it.isFile && !it.name.startsWith(".") } + .toList() + files } else { val frontendClass = file.language?.frontend val supportsParallelParsing = @@ -277,7 +272,7 @@ private constructor( Thread.currentThread().interrupt() } catch (e: ExecutionException) { log.error("Error parsing ${futureToFile[future]}", e) - Thread.currentThread().interrupt() + // Thread.currentThread().interrupt() } } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/Language.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/Language.kt index cf099ea84c6..f522474f52c 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/Language.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/Language.kt @@ -34,11 +34,9 @@ import de.fraunhofer.aisec.cpg.TranslationContext import de.fraunhofer.aisec.cpg.graph.Name import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.statements.expressions.BinaryOperator -import de.fraunhofer.aisec.cpg.graph.statements.expressions.CastExpression import de.fraunhofer.aisec.cpg.graph.types.* import de.fraunhofer.aisec.cpg.graph.types.Type import de.fraunhofer.aisec.cpg.graph.unknownType -import de.fraunhofer.aisec.cpg.helpers.Util import java.io.File import kotlin.reflect.KClass import kotlin.reflect.full.primaryConstructor diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Name.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Name.kt index 4cb0a1f75c6..a93ce43f3a4 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Name.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Name.kt @@ -131,6 +131,10 @@ fun LanguageProvider?.parseName(fqn: CharSequence) = /** Tries to parse the given fully qualified name using the specified [delimiter] into a [Name]. */ internal fun parseName(fqn: CharSequence, delimiter: String, vararg splitDelimiters: String): Name { + if (fqn is Name) { + return fqn + } + val parts = fqn.split(delimiter, *splitDelimiters) var name: Name? = null diff --git a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXLanguageFrontendTest.kt b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXLanguageFrontendTest.kt index 13ac72bb791..0a0434d1094 100644 --- a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXLanguageFrontendTest.kt +++ b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXLanguageFrontendTest.kt @@ -638,19 +638,11 @@ internal class CXXLanguageFrontendTest : BaseTest() { // a = b * 2 var assign = statements[2] as? AssignExpression assertNotNull(assign) -<<<<<<< HEAD var ref = assign.lhs() assertNotNull(ref) assertLocalName("a", ref) -======= - - var ref = assign.lhs() - assertNotNull(ref) - assertLocalName("a", ref) - ->>>>>>> f8cf854d8 (Overhaul of type propagation (#1268)) var binOp = assign.rhs() assertNotNull(binOp) @@ -666,23 +658,10 @@ internal class CXXLanguageFrontendTest : BaseTest() { ref = assign.lhs() assertNotNull(ref) assertLocalName("a", ref) -<<<<<<< HEAD -======= -<<<<<<< HEAD ->>>>>>> f8cf854d8 (Overhaul of type propagation (#1268)) - - binOp = assign.rhs() - assertNotNull(binOp) - -<<<<<<< HEAD -======= -======= binOp = assign.rhs() assertNotNull(binOp) ->>>>>>> 490562dce (Overhaul of type propagation (#1268)) ->>>>>>> f8cf854d8 (Overhaul of type propagation (#1268)) assertTrue(binOp.lhs is Literal<*>) assertEquals(1, (binOp.lhs as Literal<*>).value) assertTrue(binOp.rhs is Literal<*>) @@ -1613,22 +1592,4 @@ internal class CXXLanguageFrontendTest : BaseTest() { assertNotNull(ptr) assertLocalName("decltype(nullptr)", ptr.type) } - - @Test - fun testRecursiveHeaderFunction() { - val file = File("src/test/resources/cxx/fix-1226") - val result = - analyze( - listOf(file.resolve("main1.cpp"), file.resolve("main2.cpp")), - file.toPath(), - true - ) - assertNotNull(result) - - // For now, we have duplicate functions because we include the header twice. This might - // change in the future. The important thing is that this gets parsed at all because we - // previously had a loop in our equals method - val functions = result.functions { it.name.localName == "foo" && it.isDefinition } - assertEquals(2, functions.size) - } } diff --git a/cpg-language-go/src/main/golang/lib/cpg/main.go b/cpg-language-go/src/main/golang/lib/cpg/main.go index f276a47a1e6..0198373e0d7 100644 --- a/cpg-language-go/src/main/golang/lib/cpg/main.go +++ b/cpg-language-go/src/main/golang/lib/cpg/main.go @@ -286,6 +286,12 @@ func GetCallExprArg(ptr unsafe.Pointer, i int) unsafe.Pointer { }) } +//export GetEllipsisElt +func GetEllipsisElt(ptr unsafe.Pointer) unsafe.Pointer { + expr := restore[*ast.Ellipsis](ptr) + return save(expr.Elt) +} + //export GetForStmtInit func GetForStmtInit(ptr unsafe.Pointer) unsafe.Pointer { stmt := restore[*ast.ForStmt](ptr) @@ -466,6 +472,26 @@ func GetIndexExprIndex(ptr unsafe.Pointer) unsafe.Pointer { return save(expr.Index) } +//export GetIndexListExprX +func GetIndexListExprX(ptr unsafe.Pointer) unsafe.Pointer { + expr := restore[*ast.IndexListExpr](ptr) + return save(expr.X) +} + +//export GetNumIndexListExprIndices +func GetNumIndexListExprIndices(ptr unsafe.Pointer) C.int { + return num[*ast.IndexListExpr](ptr, func(t *ast.IndexListExpr) []ast.Expr { + return t.Indices + }) +} + +//export GetIndexListExprIndex +func GetIndexListExprIndex(ptr unsafe.Pointer, i int) unsafe.Pointer { + return item[*ast.IndexListExpr](ptr, i, func(t *ast.IndexListExpr) []ast.Expr { + return t.Indices + }) +} + //export GetKeyValueExprKey func GetKeyValueExprKey(ptr unsafe.Pointer) unsafe.Pointer { kv := restore[*ast.KeyValueExpr](ptr) @@ -478,6 +504,12 @@ func GetKeyValueExprValue(ptr unsafe.Pointer) unsafe.Pointer { return save(kv.Value) } +//export GetParenExprX +func GetParenExprX(ptr unsafe.Pointer) unsafe.Pointer { + p := restore[*ast.ParenExpr](ptr) + return save(p.X) +} + //export GetSelectorExprX func GetSelectorExprX(ptr unsafe.Pointer) unsafe.Pointer { sel := restore[*ast.SelectorExpr](ptr) @@ -847,6 +879,18 @@ func GetReturnStmtResult(ptr unsafe.Pointer, i int) unsafe.Pointer { }) } +//export GetSendStmtChan +func GetSendStmtChan(ptr unsafe.Pointer) unsafe.Pointer { + stmt := restore[*ast.SendStmt](ptr) + return save(stmt.Chan) +} + +//export GetSendStmtValue +func GetSendStmtValue(ptr unsafe.Pointer) unsafe.Pointer { + stmt := restore[*ast.SendStmt](ptr) + return save(stmt.Value) +} + //export GetSwitchStmtInit func GetSwitchStmtInit(ptr unsafe.Pointer) unsafe.Pointer { stmt := restore[*ast.SwitchStmt](ptr) diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/DeclarationHandler.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/DeclarationHandler.kt index 3d102dc966f..a25beee9572 100644 --- a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/DeclarationHandler.kt +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/DeclarationHandler.kt @@ -162,14 +162,7 @@ class DeclarationHandler(frontend: GoLanguageFrontend) : // Check for varargs. In this case we want to parse the element type // (and make it an array afterwards) - var variadic = false - val type = - if (param.type is GoStandardLibrary.Ast.Ellipsis) { - variadic = true - frontend.typeOf((param.type as GoStandardLibrary.Ast.Ellipsis).elt).array() - } else { - frontend.typeOf(param.type) - } + val (type, variadic) = frontend.fieldTypeOf(param) val p = newParamVariableDeclaration(name, type, variadic, rawNode = param) diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/ExpressionHandler.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/ExpressionHandler.kt index 6d268e6b2e6..1c87e5f3f62 100644 --- a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/ExpressionHandler.kt +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/ExpressionHandler.kt @@ -29,6 +29,7 @@ import de.fraunhofer.aisec.cpg.frontends.golang.GoStandardLibrary.Ast.BasicLit.K import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration import de.fraunhofer.aisec.cpg.graph.statements.expressions.* +import de.fraunhofer.aisec.cpg.graph.types.ObjectType import de.fraunhofer.aisec.cpg.graph.types.PointerType import de.fraunhofer.aisec.cpg.graph.types.Type @@ -45,8 +46,12 @@ class ExpressionHandler(frontend: GoLanguageFrontend) : is GoStandardLibrary.Ast.IndexExpr -> handleIndexExpr(expr) is GoStandardLibrary.Ast.CallExpr -> handleCallExpr(expr) is GoStandardLibrary.Ast.KeyValueExpr -> handleKeyValueExpr(expr) + is GoStandardLibrary.Ast.ParenExpr -> { + return handle(expr.x) + } is GoStandardLibrary.Ast.SelectorExpr -> handleSelectorExpr(expr) is GoStandardLibrary.Ast.SliceExpr -> handleSliceExpr(expr) + is GoStandardLibrary.Ast.StarExpr -> handleStarExpr(expr) is GoStandardLibrary.Ast.TypeAssertExpr -> handleTypeAssertExpr(expr) is GoStandardLibrary.Ast.UnaryExpr -> handleUnaryExpr(expr) else -> { @@ -69,7 +74,12 @@ class ExpressionHandler(frontend: GoLanguageFrontend) : type = primitiveType("string") } INT -> { - value = rawValue.toInt() + value = + if (rawValue.startsWith("0x")) { + rawValue.substring(2).toInt(16) + } else { + rawValue.toInt() + } type = primitiveType("int") } FLOAT -> { @@ -145,19 +155,32 @@ class ExpressionHandler(frontend: GoLanguageFrontend) : cast.castType = frontend.typeOf(callExpr.`fun`) if (callExpr.args.isNotEmpty()) { - frontend.expressionHandler.handle(callExpr.args[0])?.let { - cast.expression = it - } + cast.expression = frontend.expressionHandler.handle(callExpr.args[0]) } return cast } } - // Parse the Fun field, to see which kind of expression it is + val typeConstraints = mutableListOf() + val callee = - this.handle(callExpr.`fun`) - ?: return ProblemExpression("Could not parse call expr without fun") + when (val `fun` = callExpr.`fun`) { + // If "fun" is either an index or an index list expression, this is a call with type + // constraints. We do not fully support that yet, but we can at least try to set + // some of the parameters as template parameters + is GoStandardLibrary.Ast.IndexExpr -> { + (frontend.typeOf(`fun`) as? ObjectType)?.generics?.let { typeConstraints += it } + this.handle(`fun`.x) + } + is GoStandardLibrary.Ast.IndexListExpr -> { + (frontend.typeOf(`fun`) as? ObjectType)?.generics?.let { typeConstraints += it } + this.handle(`fun`.x) + } + else -> { + this.handle(callExpr.`fun`) + } + } // Handle special functions, such as make and new in a special way val name = callee.name.localName @@ -175,9 +198,18 @@ class ExpressionHandler(frontend: GoLanguageFrontend) : newCallExpression(callee, name, rawNode = callExpr) } + // TODO(oxisto) Add type constraints + if (typeConstraints.isNotEmpty()) { + log.debug( + "Call {} has type constraints ({}), but we cannot add them to the call expression yet", + call.name, + typeConstraints.joinToString(", ") { it.name } + ) + } + // Parse and add call arguments for (arg in callExpr.args) { - handle(arg)?.let { call += it } + call += handle(arg) } return call @@ -304,6 +336,13 @@ class ExpressionHandler(frontend: GoLanguageFrontend) : return ase } + private fun handleStarExpr(starExpr: GoStandardLibrary.Ast.StarExpr): UnaryOperator { + val op = newUnaryOperator("*", postfix = false, prefix = false, rawNode = starExpr) + op.input = handle(starExpr.x) + + return op + } + private fun handleTypeAssertExpr( typeAssertExpr: GoStandardLibrary.Ast.TypeAssertExpr ): CastExpression { @@ -328,7 +367,7 @@ class ExpressionHandler(frontend: GoLanguageFrontend) : prefix = false, rawNode = unaryExpr ) - handle(unaryExpr.x)?.let { op.input = it } + op.input = handle(unaryExpr.x) return op } @@ -341,13 +380,16 @@ class ExpressionHandler(frontend: GoLanguageFrontend) : private fun handleCompositeLit( compositeLit: GoStandardLibrary.Ast.CompositeLit ): ConstructExpression { - // Parse the type field, to see which kind of expression it is - val type = frontend.typeOf(compositeLit.type) + // Parse the type field, to see which kind of expression it is. The type of a composite + // literal can be omitted if this is an "inner" composite literal, so we need to set it from + // the "outer" one. See below + val type = compositeLit.type?.let { frontend.typeOf(it) } ?: unknownType() val construct = newConstructExpression(type.name, rawNode = compositeLit) construct.type = type val list = newInitializerListExpression(type, rawNode = compositeLit) + list.type = type construct += list // Normally, the construct expression would not have DFG edge, but in this case we are @@ -357,7 +399,34 @@ class ExpressionHandler(frontend: GoLanguageFrontend) : val expressions = mutableListOf() for (elem in compositeLit.elts) { - handle(elem)?.let { expressions += it } + var expression = handle(elem) + + // The type of a "inner" composite literal can be omitted if the outer one is creating + // an array type. In this case, we need to set the type manually because the type for + // the "inner" one is empty. + // Example code: + // ```go + // var a = []*MyObject{ + // { + // Name: "a", + // }, + // { + // Name: "b", + // } + // } + if ( + type is PointerType && + type.isArray && + elem is GoStandardLibrary.Ast.CompositeLit && + expression is ConstructExpression + ) { + val elementType = type.dereference() + // Set the type of the inner composite to be the deref'd type of the outer + expression.type = type + (expression as? ConstructExpression)?.arguments?.firstOrNull()?.type = elementType + } + + expressions += expression } list.initializers = expressions diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontend.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontend.kt index 33fc7bb06cd..a4956ca005f 100644 --- a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontend.kt +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontend.kt @@ -40,6 +40,7 @@ import de.fraunhofer.aisec.cpg.graph.statements.expressions.Literal import de.fraunhofer.aisec.cpg.graph.types.FunctionType import de.fraunhofer.aisec.cpg.graph.types.Type import de.fraunhofer.aisec.cpg.graph.unknownType +import de.fraunhofer.aisec.cpg.helpers.Util import de.fraunhofer.aisec.cpg.passes.EvaluationOrderGraphPass import de.fraunhofer.aisec.cpg.passes.GoEvaluationOrderGraphPass import de.fraunhofer.aisec.cpg.passes.GoExtraPass @@ -162,6 +163,12 @@ class GoLanguageFrontend(language: Language, ctx: Translatio objectType(name) } + is GoStandardLibrary.Ast.SelectorExpr -> { + // This is a FQN type + val baseName = (type.x as? GoStandardLibrary.Ast.Ident)?.name?.let { Name(it) } + + return objectType(Name(type.sel.name, baseName)) + } is GoStandardLibrary.Ast.ArrayType -> { return typeOf(type.elt).array() } @@ -170,12 +177,66 @@ class GoLanguageFrontend(language: Language, ctx: Translatio return objectType("chan", listOf(typeOf(type.value))) } is GoStandardLibrary.Ast.FuncType -> { - val paramTypes = type.params.list.map { typeOf(it.type) } + val paramTypes = type.params.list.map { fieldTypeOf(it).first } val returnTypes = type.results?.list?.map { typeOf(it.type) } ?: listOf() val name = funcTypeName(paramTypes, returnTypes) return FunctionType(name, paramTypes, returnTypes, this.language) } + is GoStandardLibrary.Ast.IndexExpr -> { + // A go type constraint, aka generic + val baseType = typeOf(type.x) + val generics = listOf(typeOf(type.index)) + objectType(baseType.name, generics) + } + is GoStandardLibrary.Ast.IndexListExpr -> { + // A go type constraint, aka generic with multiple types + val baseType = typeOf(type.x) + val generics = type.indices.map { typeOf(it) } + objectType(baseType.name, generics) + } + is GoStandardLibrary.Ast.StructType -> { + // Go allows to use anonymous structs as type. This is something we cannot model + // properly in the CPG yet. In order to at least deal with this partially, we + // construct a ObjectType and put the fields and their types into the type. + // This will result in something like `struct{name string; args util.args; want + // string}` + val parts = + type.fields.list.map { field -> + var desc = "" + // Name can be optional, if its embedded + field.names.getOrNull(0)?.let { desc += it } + desc += " " + desc += fieldTypeOf(field).first.name + desc + } + + objectType(parts.joinToString("; ", "struct{", "}")) + } + is GoStandardLibrary.Ast.InterfaceType -> { + // Go allows to use anonymous interface as type. This is something we cannot model + // properly in the CPG yet. In order to at least deal with this partially, we + // construct a ObjectType and put the methods and their types into the type. + + // In the easiest case this is the empty interface `interface{}`, which we then + // consider to be the "any" type. `any` is actually a type alias for `interface{}`, + // but in modern Go `any` is preferred. + if (type.methods.list.isEmpty()) { + return primitiveType("any") + } + + val parts = + type.methods.list.map { method -> + var desc = "" + // Name can be optional, if its embedded + method.names.getOrNull(0)?.let { desc += it } + desc += " " + desc += typeOf(method.type).name + desc + } + + objectType(parts.joinToString("; ", "interface{", "}")) + } is GoStandardLibrary.Ast.MapType -> { // We cannot properly represent Go's built-in map types, yet so we have // to make a shortcut here and represent it as a Java-like map type. @@ -185,12 +246,35 @@ class GoLanguageFrontend(language: Language, ctx: Translatio typeOf(type.x).pointer() } else -> { - log.warn("Not parsing type of type ${type.goType} yet") + Util.warnWithFileLocation( + this, + type, + log, + "Not parsing type of type ${type.goType} yet" + ) unknownType() } } } + /** + * A quick helper function to retrieve the type of a field, to check for possible variadic + * arguments. + */ + internal fun fieldTypeOf( + param: GoStandardLibrary.Ast.Field, + ): Pair { + var variadic = false + val type = + if (param.type is GoStandardLibrary.Ast.Ellipsis) { + variadic = true + typeOf((param.type as GoStandardLibrary.Ast.Ellipsis).elt).array() + } else { + typeOf(param.type) + } + return Pair(type, variadic) + } + private fun isBuiltinType(name: String): Boolean { return when (name) { "bool", diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoStandardLibrary.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoStandardLibrary.kt index f7ca20b70bd..0592820e984 100644 --- a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoStandardLibrary.kt +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoStandardLibrary.kt @@ -243,7 +243,9 @@ interface GoStandardLibrary : Library { "*ast.FuncLit" -> FuncLit(nativeValue) "*ast.Ident" -> Ident(nativeValue) "*ast.IndexExpr" -> IndexExpr(nativeValue) + "*ast.IndexListExpr" -> IndexListExpr(nativeValue) "*ast.KeyValueExpr" -> KeyValueExpr(nativeValue) + "*ast.ParenExpr" -> ParenExpr(nativeValue) "*ast.SelectorExpr" -> SelectorExpr(nativeValue) "*ast.StarExpr" -> StarExpr(nativeValue) "*ast.SliceExpr" -> SliceExpr(nativeValue) @@ -311,7 +313,7 @@ interface GoStandardLibrary : Library { } class CompositeLit(p: Pointer? = Pointer.NULL) : Expr(p) { - val type: Expr + val type: Expr? get() { return INSTANCE.GetCompositeLitType(this) } @@ -334,6 +336,13 @@ interface GoStandardLibrary : Library { } } + class ParenExpr(p: Pointer? = Pointer.NULL) : Expr(p) { + val x: Expr + get() { + return INSTANCE.GetParenExprX(this) + } + } + class FuncLit(p: Pointer? = Pointer.NULL) : Expr(p) { fun toDecl(): FuncDecl { return INSTANCE.MakeFuncDeclFromFuncLit(this) @@ -367,6 +376,21 @@ interface GoStandardLibrary : Library { } } + class IndexListExpr(p: Pointer? = Pointer.NULL) : Expr(p) { + val x: Expr + get() { + return INSTANCE.GetIndexListExprX(this) + } + + val indices: List + get() { + return list( + INSTANCE::GetNumIndexListExprIndices, + INSTANCE::GetIndexListExprIndex + ) + } + } + class SelectorExpr(p: Pointer? = Pointer.NULL) : Expr(p) { val x: Expr get() { @@ -517,6 +541,7 @@ interface GoStandardLibrary : Library { "*ast.LabeledStmt" -> LabeledStmt(nativeValue) "*ast.RangeStmt" -> RangeStmt(nativeValue) "*ast.ReturnStmt" -> ReturnStmt(nativeValue) + "*ast.SendStmt" -> SendStmt(nativeValue) "*ast.SwitchStmt" -> SwitchStmt(nativeValue) else -> super.fromNative(nativeValue, context) } @@ -700,6 +725,18 @@ interface GoStandardLibrary : Library { } } + class SendStmt(p: Pointer? = Pointer.NULL) : Stmt(p) { + val chan: Expr + get() { + return INSTANCE.GetSendStmtChan(this) + } + + val value: Expr + get() { + return INSTANCE.GetSendStmtValue(this) + } + } + class SwitchStmt(p: Pointer? = Pointer.NULL) : Stmt(p) { val init: Stmt? get() { @@ -828,7 +865,7 @@ interface GoStandardLibrary : Library { fun GetFuncDeclBody(funcDecl: Ast.FuncDecl): Ast.BlockStmt - fun GetCompositeLitType(compositeLit: Ast.CompositeLit): Ast.Expr + fun GetCompositeLitType(compositeLit: Ast.CompositeLit): Ast.Expr? fun GetNumCompositeLitElts(compositeLit: Ast.CompositeLit): Int @@ -844,6 +881,8 @@ interface GoStandardLibrary : Library { fun GetKeyValueExprValue(keyValueExpr: Ast.KeyValueExpr): Ast.Expr + fun GetParenExprX(parenExpr: Ast.ParenExpr): Ast.Expr + fun GetBasicLitValue(basicLit: Ast.BasicLit): String fun GetBasicLitKind(basicLit: Ast.BasicLit): Int @@ -958,6 +997,12 @@ interface GoStandardLibrary : Library { fun GetIndexExprIndex(IndexExpr: Ast.IndexExpr): Ast.Expr + fun GetIndexListExprX(indexListExpr: Ast.IndexListExpr): Ast.Expr + + fun GetNumIndexListExprIndices(indexListExpr: Ast.IndexListExpr): Int + + fun GetIndexListExprIndex(indexListExpr: Ast.IndexListExpr, i: Int): Ast.Expr + fun GetIfStmtInit(ifStmt: Ast.IfStmt): Ast.Stmt? fun GetIfStmtCond(ifStmt: Ast.IfStmt): Ast.Expr @@ -980,6 +1025,10 @@ interface GoStandardLibrary : Library { fun GetReturnStmtResult(returnStmt: Ast.ReturnStmt, i: Int): Ast.Expr + fun GetSendStmtChan(sendStmt: Ast.SendStmt): Ast.Expr + + fun GetSendStmtValue(sendStmt: Ast.SendStmt): Ast.Expr + fun GetSwitchStmtInit(switchStmt: Ast.SwitchStmt): Ast.Stmt? fun GetSwitchStmtTag(switchStmt: Ast.SwitchStmt): Ast.Expr? diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/SpecificationHandler.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/SpecificationHandler.kt index 8dd43ab566b..467aec168e0 100644 --- a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/SpecificationHandler.kt +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/SpecificationHandler.kt @@ -91,15 +91,15 @@ class SpecificationHandler(frontend: GoLanguageFrontend) : if (!structType.incomplete) { for (field in structType.fields.list) { - // a field can also have no name, which means that it is embedded, not quite - // sure yet how to handle this, but since the embedded field can be accessed - // by its type, it could make sense to name the field according to the type val type = frontend.typeOf(field.type) + // A field can also have no name, which means that it is embedded. In this case, it + // can be accessed by the local name of its type and therefore we name the field + // accordingly val name = if (field.names.isEmpty()) { - // Retrieve the root type name - type.root.name.toString() + // Retrieve the root type local name + type.root.name.localName } else { field.names[0].name } diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/StatementHandler.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/StatementHandler.kt index 0159cf2d428..e99a824fba3 100644 --- a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/StatementHandler.kt +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/StatementHandler.kt @@ -51,6 +51,7 @@ class StatementHandler(frontend: GoLanguageFrontend) : is GoStandardLibrary.Ast.LabeledStmt -> handleLabeledStmt(stmt) is GoStandardLibrary.Ast.RangeStmt -> handleRangeStmt(stmt) is GoStandardLibrary.Ast.ReturnStmt -> handleReturnStmt(stmt) + is GoStandardLibrary.Ast.SendStmt -> handleSendStmt(stmt) is GoStandardLibrary.Ast.SwitchStmt -> handleSwitchStmt(stmt) else -> handleNotSupported(stmt, stmt.goType) } @@ -295,6 +296,14 @@ class StatementHandler(frontend: GoLanguageFrontend) : return `return` } + private fun handleSendStmt(sendStmt: GoStandardLibrary.Ast.SendStmt): BinaryOperator { + val op = newBinaryOperator("<-", rawNode = sendStmt) + op.lhs = frontend.expressionHandler.handle(sendStmt.chan) + op.rhs = frontend.expressionHandler.handle(sendStmt.value) + + return op + } + private fun handleSwitchStmt(switchStmt: GoStandardLibrary.Ast.SwitchStmt): Statement { val switch = newSwitchStatement(rawNode = switchStmt) diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/GoEvaluationOrderGraphPass.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/GoEvaluationOrderGraphPass.kt index e519e15f308..8237c0aabe1 100644 --- a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/GoEvaluationOrderGraphPass.kt +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/GoEvaluationOrderGraphPass.kt @@ -28,8 +28,6 @@ package de.fraunhofer.aisec.cpg.passes import de.fraunhofer.aisec.cpg.TranslationContext import de.fraunhofer.aisec.cpg.frontends.golang.GoLanguage import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration -import de.fraunhofer.aisec.cpg.graph.followNextEOGEdgesUntilHit -import de.fraunhofer.aisec.cpg.graph.statements.ReturnStatement import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.UnaryOperator @@ -90,13 +88,14 @@ class GoEvaluationOrderGraphPass(ctx: TranslationContext) : EvaluationOrderGraph // We need to follow the path from the defer statement to all return statements that are // reachable from this point. for (defer in defers ?: listOf()) { - val paths = defer.followNextEOGEdgesUntilHit { it is ReturnStatement } + // TODO(oxisto): This is broken! this returns 8192 paths instead of 4 + /*val paths = defer.followNextEOGEdgesUntilHit { it is ReturnStatement } for (path in paths.fulfilled) { // It is a bit philosophical whether the deferred call happens before or after the // return statement in the EOG. For now, it is easier to have it as the last node // AFTER the return statement addEOGEdge(path.last(), defer.input) - } + }*/ } } } diff --git a/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/DeclarationTest.kt b/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/DeclarationTest.kt index 5e379141c1f..b9ce305d423 100644 --- a/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/DeclarationTest.kt +++ b/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/DeclarationTest.kt @@ -28,13 +28,17 @@ package de.fraunhofer.aisec.cpg.frontends.golang import de.fraunhofer.aisec.cpg.TestUtils import de.fraunhofer.aisec.cpg.assertFullName import de.fraunhofer.aisec.cpg.assertLiteralValue +import de.fraunhofer.aisec.cpg.assertLocalName import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.byNameOrNull import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.MethodDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.NamespaceDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.RecordDeclaration +import de.fraunhofer.aisec.cpg.graph.statements.CompoundStatement +import de.fraunhofer.aisec.cpg.graph.statements.ReturnStatement import de.fraunhofer.aisec.cpg.graph.statements.expressions.DeclaredReferenceExpression +import de.fraunhofer.aisec.cpg.graph.statements.expressions.UnaryOperator import de.fraunhofer.aisec.cpg.graph.variables import java.nio.file.Path import kotlin.test.* @@ -90,6 +94,76 @@ class DeclarationTest { assertFullName("", param) } + @Test + fun testStruct() { + val topLevel = Path.of("src", "test", "resources", "golang") + val tu = + TestUtils.analyzeAndGetFirstTU( + listOf(topLevel.resolve("struct.go").toFile()), + topLevel, + true + ) { + it.registerLanguage() + } + + assertNotNull(tu) + + val p = tu.getDeclarationsByName("p", NamespaceDeclaration::class.java).iterator().next() + + val myStruct = + p.getDeclarationsByName("p.MyStruct", RecordDeclaration::class.java).iterator().next() + + assertNotNull(myStruct) + assertEquals("struct", myStruct.kind) + + val fields = myStruct.fields + assertEquals( + listOf("MyField", "OtherStruct", "EvenAnotherStruct"), + fields.map { it.name.localName } + ) + + var methods = myStruct.methods + + var myFunc = methods.firstOrNull() + assertNotNull(myFunc) + + assertLocalName("MyFunc", myFunc) + + val myField = fields.firstOrNull() + assertNotNull(myField) + + assertLocalName("MyField", myField) + assertEquals(tu.primitiveType("int"), myField.type) + + val myInterface = p.records["p.MyInterface"] + assertNotNull(myInterface) + assertEquals("interface", myInterface.kind) + + methods = myInterface.methods + + assertEquals(1, methods.size) + + myFunc = methods.first() + + assertLocalName("MyFunc", myFunc) + assertLocalName("func() string", myFunc.type) + + val newMyStruct = p.functions["NewMyStruct"] + assertNotNull(newMyStruct) + + val body = newMyStruct.body as? CompoundStatement + + assertNotNull(body) + + val `return` = body.statements.first() as? ReturnStatement + + assertNotNull(`return`) + + val returnValue = `return`.returnValue as? UnaryOperator + + assertNotNull(returnValue) + } + @Test fun testEmbeddedInterface() { val topLevel = Path.of("src", "test", "resources", "golang") diff --git a/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/ExpressionTest.kt b/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/ExpressionTest.kt index 7921cec0475..2d450bea0aa 100644 --- a/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/ExpressionTest.kt +++ b/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/ExpressionTest.kt @@ -26,6 +26,7 @@ package de.fraunhofer.aisec.cpg.frontends.golang import de.fraunhofer.aisec.cpg.TestUtils +import de.fraunhofer.aisec.cpg.TestUtils.assertRefersTo import de.fraunhofer.aisec.cpg.assertFullName import de.fraunhofer.aisec.cpg.assertLiteralValue import de.fraunhofer.aisec.cpg.assertLocalName @@ -136,4 +137,40 @@ class ExpressionTest { assertLiteralValue(1, slice.ceiling) assertLiteralValue(1, slice.third) } + + @Test + fun testSendStmt() { + val topLevel = Path.of("src", "test", "resources", "golang") + val tu = + TestUtils.analyzeAndGetFirstTU( + listOf(topLevel.resolve("chan.go").toFile()), + topLevel, + true + ) { + it.registerLanguage() + } + assertNotNull(tu) + + with(tu) { + val main = tu.functions["main"] + assertNotNull(main) + + val v = main.variables["v"] + assertNotNull(v) + assertEquals(primitiveType("int"), v.type) + + val ch = main.variables["ch"] + assertNotNull(ch) + assertEquals(objectType("chan", generics = listOf(primitiveType("int"))), ch.type) + + val binOp = main.bodyOrNull() + assertNotNull(binOp) + assertRefersTo(binOp.lhs, ch) + assertRefersTo(binOp.rhs, v) + + val unaryOp = main.bodyOrNull() + assertNotNull(unaryOp) + assertRefersTo(unaryOp.input, ch) + } + } } diff --git a/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontendTest.kt b/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontendTest.kt index 37776c62a3b..0afaef6746e 100644 --- a/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontendTest.kt +++ b/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontendTest.kt @@ -331,70 +331,6 @@ class GoLanguageFrontendTest : BaseTest() { assertLocalName("error", err.type) } - @Test - fun testStruct() { - val topLevel = Path.of("src", "test", "resources", "golang") - val tu = - analyzeAndGetFirstTU(listOf(topLevel.resolve("struct.go").toFile()), topLevel, true) { - it.registerLanguage() - } - - assertNotNull(tu) - - val p = tu.getDeclarationsByName("p", NamespaceDeclaration::class.java).iterator().next() - - val myStruct = - p.getDeclarationsByName("p.MyStruct", RecordDeclaration::class.java).iterator().next() - - assertNotNull(myStruct) - assertEquals("struct", myStruct.kind) - - val fields = myStruct.fields - - assertEquals(1, fields.size) - - var methods = myStruct.methods - - var myFunc = methods.firstOrNull() - assertNotNull(myFunc) - - assertLocalName("MyFunc", myFunc) - - val myField = fields.firstOrNull() - assertNotNull(myField) - - assertLocalName("MyField", myField) - assertEquals(tu.primitiveType("int"), myField.type) - - val myInterface = p.records["p.MyInterface"] - assertNotNull(myInterface) - assertEquals("interface", myInterface.kind) - - methods = myInterface.methods - - assertEquals(1, methods.size) - - myFunc = methods.first() - - assertLocalName("MyFunc", myFunc) - assertLocalName("func() string", myFunc.type) - - val newMyStruct = p.functions["NewMyStruct"] - assertNotNull(newMyStruct) - - val body = newMyStruct.body as? CompoundStatement - - assertNotNull(body) - - val `return` = body.statements.first() as? ReturnStatement - - assertNotNull(`return`) - - val returnValue = `return`.returnValue as? UnaryOperator - - assertNotNull(returnValue) - } - @Test fun testMemberCalls() { val topLevel = Path.of("src", "test", "resources", "golang") diff --git a/cpg-language-go/src/test/resources/golang/chan.go b/cpg-language-go/src/test/resources/golang/chan.go new file mode 100644 index 00000000000..6ea215ff5c1 --- /dev/null +++ b/cpg-language-go/src/test/resources/golang/chan.go @@ -0,0 +1,8 @@ +package pa + +func main() { + var v int + var ch = make(chan int) + ch <- v + <-ch +} diff --git a/cpg-language-go/src/test/resources/golang/go.mod b/cpg-language-go/src/test/resources/golang/go.mod index 12cb5307fb5..07004656da2 100644 --- a/cpg-language-go/src/test/resources/golang/go.mod +++ b/cpg-language-go/src/test/resources/golang/go.mod @@ -1 +1,3 @@ module mymodule + +go 1.18 diff --git a/cpg-language-go/src/test/resources/golang/struct.go b/cpg-language-go/src/test/resources/golang/struct.go index 7287159ce3d..5f0ee4296d8 100644 --- a/cpg-language-go/src/test/resources/golang/struct.go +++ b/cpg-language-go/src/test/resources/golang/struct.go @@ -1,9 +1,19 @@ package p -import ("fmt") +import ( + "fmt" +) + +type OtherStruct struct { +} + +type EvenAnotherStruct struct { +} type MyStruct struct { - MyField int + MyField int + OtherStruct + *EvenAnotherStruct } type MyInterface interface { @@ -11,7 +21,7 @@ type MyInterface interface { } func (s MyStruct) MyFunc() string { - fmt.Printf(s.myOtherFunc(), s.MyField) + fmt.Printf(s.myOtherFunc(), s.MyField) return "s" } @@ -21,5 +31,5 @@ func (s MyStruct) myOtherFunc() string { } func NewMyStruct() *MyStruct { - return &MyStruct{} + return &MyStruct{} } diff --git a/cpg-language-go/src/test/resources/golang/type_constraints.go b/cpg-language-go/src/test/resources/golang/type_constraints.go index 485f29bc2ea..b4c92870e7e 100644 --- a/cpg-language-go/src/test/resources/golang/type_constraints.go +++ b/cpg-language-go/src/test/resources/golang/type_constraints.go @@ -1,11 +1,11 @@ package main -type MyStruct[T any] struct {} -type MyInterface interface {} +type MyStruct[T any] struct{} +type MyInterface interface{} func SomeFunc[T any, S MyInterface]() {} func main() { - _ := &MyStruct[MyInterface]{} - SomeFunc[any, MyInterface]() -} \ No newline at end of file + _ := &MyStruct[MyInterface]{} + SomeFunc[interface{}, MyInterface]() +}