From 9134847b43ba937f205632094a1ea771c62bdd8c Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Mon, 28 Aug 2023 21:52:40 +0200 Subject: [PATCH] Improvements to the Go language * Added parsing of more expressions --- .../de/fraunhofer/aisec/cpg/TypeManager.kt | 23 +- .../aisec/cpg/frontends/Language.kt | 10 + .../graph/declarations/IncludeDeclaration.kt | 11 + .../DeclaredReferenceExpression.kt | 8 +- .../expressions/InitializerListExpression.kt | 9 +- .../expressions/KeyValueExpression.kt | 23 +- .../expressions/MemberExpression.kt | 16 +- .../statements/expressions/UnaryOperator.kt | 16 +- .../cpg/passes/ControlFlowSensitiveDFGPass.kt | 16 +- .../cpg/passes/EvaluationOrderGraphPass.kt | 5 + .../aisec/cpg/passes/TypeResolver.kt | 3 +- .../aisec/cpg/passes/VariableUsageResolver.kt | 9 + .../de/fraunhofer/aisec/cpg/TestUtils.kt | 8 +- .../frontends/cxx/CXXLanguageFrontendTest.kt | 3 + cpg-language-go/build.gradle.kts | 1 + .../src/main/golang/lib/cpg/main.go | 83 +++- .../cpg/frontends/golang/BuildConstraints.kt | 136 ++++++ .../frontends/golang/DeclarationHandler.kt | 156 ++++--- .../cpg/frontends/golang/ExpressionHandler.kt | 226 +++++++--- .../aisec/cpg/frontends/golang/GoHandler.kt | 37 +- .../aisec/cpg/frontends/golang/GoLanguage.kt | 36 +- .../frontends/golang/GoLanguageFrontend.kt | 327 ++++++++++++--- .../cpg/frontends/golang/GoStandardLibrary.kt | 108 ++++- .../aisec/cpg/frontends/golang/GoUtils.kt | 188 +++++++++ .../frontends/golang/SpecificationHandler.kt | 253 ++++++++--- .../cpg/frontends/golang/StatementHandler.kt | 135 +++++- .../cpg/passes/GoEvaluationOrderGraphPass.kt | 7 +- .../aisec/cpg/passes/GoExtraPass.kt | 129 +++++- .../frontends/golang/BuildConstraintsTest.kt | 61 +++ .../cpg/frontends/golang/DeclarationTest.kt | 225 +++++++++- .../cpg/frontends/golang/ExpressionTest.kt | 48 ++- .../golang/GoLanguageFrontendTest.kt | 396 ++++++++++++++---- .../cpg/frontends/golang/StatementTest.kt | 40 ++ .../test/resources/golang-std/fmt/print.go | 6 + .../resources/golang/buildtags/func_darwin.go | 10 + .../golang/buildtags/func_darwin_arm64.go | 3 + .../resources/golang/buildtags/func_ios.go | 8 + .../golang/buildtags/func_linux_arm64.go | 5 + .../src/test/resources/golang/chan.go | 8 + .../src/test/resources/golang/const.go | 24 ++ .../src/test/resources/golang/declare.go | 21 +- .../src/test/resources/golang/function.go | 14 +- .../src/test/resources/golang/go.mod | 2 + .../src/test/resources/golang/importalias.go | 16 + .../src/test/resources/golang/literal.go | 33 ++ .../src/test/resources/golang/options/srv.go | 9 + .../resources/golang/options/srv_option.go | 9 + .../src/test/resources/golang/struct.go | 33 +- .../test/resources/golang/submodule/const.go | 3 + .../src/test/resources/golang/type_assert.go | 38 +- .../test/resources/golang/type_constraints.go | 10 +- .../src/test/resources/golang/types.go | 43 ++ 52 files changed, 2573 insertions(+), 474 deletions(-) create mode 100644 cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/BuildConstraints.kt create mode 100644 cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoUtils.kt create mode 100644 cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/BuildConstraintsTest.kt create mode 100644 cpg-language-go/src/test/resources/golang-std/fmt/print.go create mode 100644 cpg-language-go/src/test/resources/golang/buildtags/func_darwin.go create mode 100644 cpg-language-go/src/test/resources/golang/buildtags/func_darwin_arm64.go create mode 100644 cpg-language-go/src/test/resources/golang/buildtags/func_ios.go create mode 100644 cpg-language-go/src/test/resources/golang/buildtags/func_linux_arm64.go create mode 100644 cpg-language-go/src/test/resources/golang/chan.go create mode 100644 cpg-language-go/src/test/resources/golang/const.go create mode 100644 cpg-language-go/src/test/resources/golang/importalias.go create mode 100644 cpg-language-go/src/test/resources/golang/options/srv.go create mode 100644 cpg-language-go/src/test/resources/golang/options/srv_option.go create mode 100644 cpg-language-go/src/test/resources/golang/submodule/const.go create mode 100644 cpg-language-go/src/test/resources/golang/types.go diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TypeManager.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TypeManager.kt index 72bc8aa5103..0fc51ab8d4c 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TypeManager.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TypeManager.kt @@ -50,12 +50,11 @@ class TypeManager { * Generics) to the ParameterizedType to be able to resolve the Type of the fields, since * ParameterizedTypes are unique to the RecordDeclaration and are not merged. */ - private val recordToTypeParameters = - Collections.synchronizedMap(mutableMapOf>()) - private val templateToTypeParameters = - Collections.synchronizedMap( - mutableMapOf>() - ) + private val recordToTypeParameters: MutableMap> = + ConcurrentHashMap() + private val templateToTypeParameters: + MutableMap> = + ConcurrentHashMap() val firstOrderTypes: MutableSet = ConcurrentHashMap.newKeySet() val secondOrderTypes: MutableSet = ConcurrentHashMap.newKeySet() @@ -288,14 +287,12 @@ internal fun Type.getAncestors(depth: Int): Set { return types } -/** Checks, if this [Type] is either derived from or equals to [superType]. */ +/** + * Checks, if this [Type] is either derived from or equals to [superType]. This is forwarded to the + * [Language] of the [Type] and can be overridden by the individual languages. + */ fun Type.isDerivedFrom(superType: Type): Boolean { - // Retrieve all ancestor types of our type (more concretely of the root type) - val root = this.root - val superTypes = root.ancestors.map { it.type } - - // Check, if super type (or its root) is in the list - return superType.root in superTypes + return this.language?.isDerivedFrom(this, superType) ?: false } /** diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/Language.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/Language.kt index f522474f52c..3cb4e01103d 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/Language.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/Language.kt @@ -31,6 +31,7 @@ import com.fasterxml.jackson.databind.SerializerProvider import com.fasterxml.jackson.databind.annotation.JsonSerialize import com.fasterxml.jackson.databind.ser.std.StdSerializer import de.fraunhofer.aisec.cpg.TranslationContext +import de.fraunhofer.aisec.cpg.ancestors import de.fraunhofer.aisec.cpg.graph.Name import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.statements.expressions.BinaryOperator @@ -199,6 +200,15 @@ abstract class Language> : Node() { return true } + + open fun isDerivedFrom(type: Type, superType: Type): Boolean { + // Retrieve all ancestor types of our type (more concretely of the root type) + val root = type.root + val superTypes = root.ancestors.map { it.type } + + // Check, if super type (or its root) is in the list + return superType.root in superTypes + } } /** diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/IncludeDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/IncludeDeclaration.kt index 4d9650ee828..b4af3043c6b 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/IncludeDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/IncludeDeclaration.kt @@ -34,6 +34,7 @@ import java.util.Objects import org.apache.commons.lang3.builder.ToStringBuilder import org.neo4j.ogm.annotation.Relationship +/** This declaration represents either an include or an import, depending on the language. */ class IncludeDeclaration : Declaration() { @Relationship(value = "INCLUDES", direction = Relationship.Direction.OUTGOING) @AST @@ -43,8 +44,18 @@ class IncludeDeclaration : Declaration() { @AST private val problemEdges: MutableList> = ArrayList() + /** + * This property refers to the file or directory or path. For example, in C this refers to an + * include header file. In Go, this refers to the package path (e.g., github.com/a/b) + */ var filename: String? = null + /** + * Some languages allow to name the imported "thing" (e.g., the package) differently in the + * local scope to avoid conflicts. + */ + var alias: String? = null + val includes: List by PropertyEdgeDelegate(IncludeDeclaration::includeEdges) val problems: List by PropertyEdgeDelegate(IncludeDeclaration::problemEdges) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/DeclaredReferenceExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/DeclaredReferenceExpression.kt index 0a7b589939f..30a0038b282 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/DeclaredReferenceExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/DeclaredReferenceExpression.kt @@ -137,7 +137,11 @@ open class DeclaredReferenceExpression : Expression(), HasType.TypeObserver { if (other !is DeclaredReferenceExpression) { return false } - return super.equals(other) && refersTo == other.refersTo + return super.equals(other) + } + + override fun hashCode(): Int { + return super.hashCode() } override fun addPrevDFG(prev: Node, properties: MutableMap) { @@ -150,6 +154,4 @@ open class DeclaredReferenceExpression : Expression(), HasType.TypeObserver { prev.registerTypeObserver(this) } } - - override fun hashCode(): Int = Objects.hash(super.hashCode(), refersTo) } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/InitializerListExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/InitializerListExpression.kt index 9a03dd70a3e..696b851be8a 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/InitializerListExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/InitializerListExpression.kt @@ -68,7 +68,14 @@ class InitializerListExpression : Expression(), ArgumentHolder, HasType.TypeObse } override fun replaceArgument(old: Expression, new: Expression): Boolean { - // Not supported, too complex + val idx = initializerEdges.indexOfFirst { it.end == old } + if (idx != -1) { + old.unregisterTypeObserver(this) + initializerEdges[idx].end = new + new.registerTypeObserver(this) + return true + } + return false } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/KeyValueExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/KeyValueExpression.kt index 7ca1b0d42d4..c5bda8735e9 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/KeyValueExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/KeyValueExpression.kt @@ -26,6 +26,7 @@ package de.fraunhofer.aisec.cpg.graph.statements.expressions import de.fraunhofer.aisec.cpg.graph.AST +import de.fraunhofer.aisec.cpg.graph.ArgumentHolder import java.util.* /** @@ -35,7 +36,7 @@ import java.util.* * Most often used in combination with an [InitializerListExpression] to represent the creation of * an array. */ -class KeyValueExpression : Expression() { +class KeyValueExpression : Expression(), ArgumentHolder { /** * The key of this pair. It is usually a literal, but some languages even allow references to @@ -46,6 +47,26 @@ class KeyValueExpression : Expression() { /** The value of this pair. It can be any expression */ @AST var value: Expression? = null + override fun addArgument(expression: Expression) { + if (key == null) { + key = expression + } else if (value == null) { + value = expression + } + } + + override fun replaceArgument(old: Expression, new: Expression): Boolean { + if (key == old) { + key = new + return true + } else if (value == old) { + value = new + return true + } + + return false + } + override fun equals(other: Any?): Boolean { if (this === other) return true if (other !is KeyValueExpression) return false diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/MemberExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/MemberExpression.kt index 59c95880642..29644a6b2c5 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/MemberExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/MemberExpression.kt @@ -26,6 +26,7 @@ package de.fraunhofer.aisec.cpg.graph.statements.expressions import de.fraunhofer.aisec.cpg.graph.AST +import de.fraunhofer.aisec.cpg.graph.ArgumentHolder import de.fraunhofer.aisec.cpg.graph.HasBase import de.fraunhofer.aisec.cpg.graph.declarations.RecordDeclaration import de.fraunhofer.aisec.cpg.graph.fqn @@ -39,7 +40,7 @@ import org.apache.commons.lang3.builder.ToStringBuilder * use-case is access of a member function (method) as part of the [MemberCallExpression.callee] * property of a [MemberCallExpression]. */ -class MemberExpression : DeclaredReferenceExpression(), HasBase { +class MemberExpression : DeclaredReferenceExpression(), ArgumentHolder, HasBase { @AST override var base: Expression = ProblemExpression("could not parse base expression") set(value) { @@ -58,6 +59,19 @@ class MemberExpression : DeclaredReferenceExpression(), HasBase { .toString() } + override fun addArgument(expression: Expression) { + this.base = expression + } + + override fun replaceArgument(old: Expression, new: Expression): Boolean { + if (old == base) { + base = new + return true + } + + return false + } + override fun equals(other: Any?): Boolean { if (this === other) return true if (other !is MemberExpression) return false diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/UnaryOperator.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/UnaryOperator.kt index fca3d2deda1..4ea22698011 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/UnaryOperator.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/UnaryOperator.kt @@ -27,13 +27,14 @@ package de.fraunhofer.aisec.cpg.graph.statements.expressions import de.fraunhofer.aisec.cpg.graph.AST import de.fraunhofer.aisec.cpg.graph.AccessValues +import de.fraunhofer.aisec.cpg.graph.ArgumentHolder import de.fraunhofer.aisec.cpg.graph.pointer import de.fraunhofer.aisec.cpg.graph.types.HasType import de.fraunhofer.aisec.cpg.graph.types.Type import org.apache.commons.lang3.builder.ToStringBuilder /** A unary operator expression, involving one expression and an operator, such as `a++`. */ -class UnaryOperator : Expression(), HasType.TypeObserver { +class UnaryOperator : Expression(), ArgumentHolder, HasType.TypeObserver { /** The expression on which the operation is applied. */ @AST var input: Expression = ProblemExpression("could not parse input") @@ -112,6 +113,19 @@ class UnaryOperator : Expression(), HasType.TypeObserver { ) } + override fun addArgument(expression: Expression) { + this.input = expression + } + + override fun replaceArgument(old: Expression, new: Expression): Boolean { + if (this.input == old) { + this.input = new + return true + } + + return false + } + override fun equals(other: Any?): Boolean { if (this === other) { return true diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ControlFlowSensitiveDFGPass.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ControlFlowSensitiveDFGPass.kt index ab0e1535475..09a31e4ade6 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ControlFlowSensitiveDFGPass.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ControlFlowSensitiveDFGPass.kt @@ -202,11 +202,19 @@ open class ControlFlowSensitiveDFGPass(ctx: TranslationContext) : TranslationUni // the "normal" case won't work. We handle this case separately here... // This is what we write to the declaration val iterable = currentNode.iterable as? Expression - val writtenTo = - when (currentNode.variable) { - is DeclarationStatement -> - (currentNode.variable as DeclarationStatement).singleDeclaration + when (val variable = currentNode.variable) { + is DeclarationStatement -> { + if (variable.isSingleDeclaration()) { + variable.singleDeclaration + } else if (variable.variables.size == 2) { + // If there are two variables, we just blindly assume that the order is + // (key, value), so we return the second one + variable.declarations[1] + } else { + null + } + } else -> currentNode.variable } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/EvaluationOrderGraphPass.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/EvaluationOrderGraphPass.kt index 527a64a4b4f..32ce1708495 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/EvaluationOrderGraphPass.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/EvaluationOrderGraphPass.kt @@ -132,6 +132,7 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa map[DoStatement::class.java] = { handleDoStatement(it as DoStatement) } map[ForStatement::class.java] = { handleForStatement(it as ForStatement) } map[ForEachStatement::class.java] = { handleForEachStatement(it as ForEachStatement) } + map[TypeExpression::class.java] = { handleTypeExpression(it as TypeExpression) } map[TryStatement::class.java] = { handleTryStatement(it as TryStatement) } map[ContinueStatement::class.java] = { handleContinueStatement(it as ContinueStatement) } map[DeleteExpression::class.java] = { handleDeleteExpression(it as DeleteExpression) } @@ -594,6 +595,10 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa pushToEOG(node) } + protected fun handleTypeExpression(node: TypeExpression) { + pushToEOG(node) + } + protected fun handleTryStatement(node: TryStatement) { scopeManager.enterScope(node) val tryScope = scopeManager.currentScope as TryScope? diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/TypeResolver.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/TypeResolver.kt index 4891b229c08..3def0992f55 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/TypeResolver.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/TypeResolver.kt @@ -29,7 +29,8 @@ import de.fraunhofer.aisec.cpg.TranslationContext import de.fraunhofer.aisec.cpg.graph.Component import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.declarations.RecordDeclaration -import de.fraunhofer.aisec.cpg.graph.types.* +import de.fraunhofer.aisec.cpg.graph.types.ObjectType +import de.fraunhofer.aisec.cpg.graph.types.Type import de.fraunhofer.aisec.cpg.processing.IVisitor import de.fraunhofer.aisec.cpg.processing.strategy.Strategy diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/VariableUsageResolver.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/VariableUsageResolver.kt index 2a91d557d3b..71529b14f69 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/VariableUsageResolver.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/VariableUsageResolver.kt @@ -198,6 +198,7 @@ open class VariableUsageResolver(ctx: TranslationContext) : SymbolResolverPass(c log, "Did not find a declaration for ${current.name}" ) + ctx.unresolvedDeclarations++ } } @@ -308,6 +309,14 @@ open class VariableUsageResolver(ctx: TranslationContext) : SymbolResolverPass(c } protected fun resolveBase(reference: DeclaredReferenceExpression): Declaration? { + // We need to check, whether we first need to resolve our own base + if (reference is MemberExpression) { + val base = reference.base + if (base is DeclaredReferenceExpression && base.refersTo == null) { + base.refersTo = resolveBase(base) + } + } + val declaration = scopeManager.resolveReference(reference) if (declaration != null) { return declaration diff --git a/cpg-core/src/testFixtures/kotlin/de/fraunhofer/aisec/cpg/TestUtils.kt b/cpg-core/src/testFixtures/kotlin/de/fraunhofer/aisec/cpg/TestUtils.kt index f7f12b58117..1e5b5d0d937 100644 --- a/cpg-core/src/testFixtures/kotlin/de/fraunhofer/aisec/cpg/TestUtils.kt +++ b/cpg-core/src/testFixtures/kotlin/de/fraunhofer/aisec/cpg/TestUtils.kt @@ -26,7 +26,7 @@ package de.fraunhofer.aisec.cpg import de.fraunhofer.aisec.cpg.frontends.CompilationDatabase -import de.fraunhofer.aisec.cpg.graph.Node +import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.Declaration import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration @@ -155,7 +155,11 @@ object TestUtils { ): List { val config = builder.build() val analyzer = TranslationManager.builder().config(config).build() - return analyzer.analyze().get().translationUnits + val result = analyzer.analyze().get() + + println(result.finalCtx.unresolvedDeclarations) + + return result.components["application"]?.translationUnits ?: listOf() } @JvmOverloads diff --git a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXLanguageFrontendTest.kt b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXLanguageFrontendTest.kt index b618ca00f92..ac8a3c37d81 100644 --- a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXLanguageFrontendTest.kt +++ b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXLanguageFrontendTest.kt @@ -662,6 +662,9 @@ internal class CXXLanguageFrontendTest : BaseTest() { binOp = assign.rhs() assertNotNull(binOp) + binOp = assign.rhs() + assertNotNull(binOp) + assertTrue(binOp.lhs is Literal<*>) assertEquals(1, (binOp.lhs as Literal<*>).value) assertTrue(binOp.rhs is Literal<*>) diff --git a/cpg-language-go/build.gradle.kts b/cpg-language-go/build.gradle.kts index fc19473bca5..984f246c5d8 100644 --- a/cpg-language-go/build.gradle.kts +++ b/cpg-language-go/build.gradle.kts @@ -41,6 +41,7 @@ publishing { dependencies { implementation("net.java.dev.jna:jna:5.13.0") + testImplementation(project(":cpg-analysis")) } if (!project.hasProperty("skipGoBuild")) { diff --git a/cpg-language-go/src/main/golang/lib/cpg/main.go b/cpg-language-go/src/main/golang/lib/cpg/main.go index f276a47a1e6..351f9e04504 100644 --- a/cpg-language-go/src/main/golang/lib/cpg/main.go +++ b/cpg-language-go/src/main/golang/lib/cpg/main.go @@ -60,10 +60,10 @@ func NewFileSet() unsafe.Pointer { } //export goParserParseFile -func goParserParseFile(fset unsafe.Pointer, path *C.char, src *C.char) unsafe.Pointer { +func goParserParseFile(fset unsafe.Pointer, path *C.char) unsafe.Pointer { f, err := parser.ParseFile( pointer.Restore(fset).(*token.FileSet), - C.GoString(path), C.GoString(src), parser.ParseComments) + C.GoString(path), nil, parser.ParseComments|parser.SkipObjectResolution) if err != nil { panic(err) } @@ -175,9 +175,10 @@ func GetCommentMapNodeComment(ptr1 unsafe.Pointer, ptr2 unsafe.Pointer) *C.char } for _, c := range comments { - text := strings.TrimRight(c.Text(), "\n") - comment += text + comment += c.Text() } + // Remove last \n + comment = strings.TrimRight(comment, "\n") return C.CString(comment) } @@ -286,6 +287,12 @@ func GetCallExprArg(ptr unsafe.Pointer, i int) unsafe.Pointer { }) } +//export GetEllipsisElt +func GetEllipsisElt(ptr unsafe.Pointer) unsafe.Pointer { + expr := restore[*ast.Ellipsis](ptr) + return save(expr.Elt) +} + //export GetForStmtInit func GetForStmtInit(ptr unsafe.Pointer) unsafe.Pointer { stmt := restore[*ast.ForStmt](ptr) @@ -466,6 +473,26 @@ func GetIndexExprIndex(ptr unsafe.Pointer) unsafe.Pointer { return save(expr.Index) } +//export GetIndexListExprX +func GetIndexListExprX(ptr unsafe.Pointer) unsafe.Pointer { + expr := restore[*ast.IndexListExpr](ptr) + return save(expr.X) +} + +//export GetNumIndexListExprIndices +func GetNumIndexListExprIndices(ptr unsafe.Pointer) C.int { + return num[*ast.IndexListExpr](ptr, func(t *ast.IndexListExpr) []ast.Expr { + return t.Indices + }) +} + +//export GetIndexListExprIndex +func GetIndexListExprIndex(ptr unsafe.Pointer, i int) unsafe.Pointer { + return item[*ast.IndexListExpr](ptr, i, func(t *ast.IndexListExpr) []ast.Expr { + return t.Indices + }) +} + //export GetKeyValueExprKey func GetKeyValueExprKey(ptr unsafe.Pointer) unsafe.Pointer { kv := restore[*ast.KeyValueExpr](ptr) @@ -478,6 +505,12 @@ func GetKeyValueExprValue(ptr unsafe.Pointer) unsafe.Pointer { return save(kv.Value) } +//export GetParenExprX +func GetParenExprX(ptr unsafe.Pointer) unsafe.Pointer { + p := restore[*ast.ParenExpr](ptr) + return save(p.X) +} + //export GetSelectorExprX func GetSelectorExprX(ptr unsafe.Pointer) unsafe.Pointer { sel := restore[*ast.SelectorExpr](ptr) @@ -643,6 +676,12 @@ func GetGenDeclSpec(ptr unsafe.Pointer, i int) unsafe.Pointer { }) } +//export GetGenDeclTok +func GetGenDeclTok(ptr unsafe.Pointer) int { + decl := restore[*ast.GenDecl](ptr) + return int(decl.Tok) +} + //export GetInterfaceTypeMethods func GetInterfaceTypeMethods(ptr unsafe.Pointer) unsafe.Pointer { i := restore[*ast.InterfaceType](ptr) @@ -739,6 +778,12 @@ func GetTypeSpecName(ptr unsafe.Pointer) unsafe.Pointer { return save(spec.Name) } +//export GetTypeSpecAssign +func GetTypeSpecAssign(ptr unsafe.Pointer) int { + spec := restore[*ast.TypeSpec](ptr) + return int(spec.Assign) +} + //export GetTypeSpecType func GetTypeSpecType(ptr unsafe.Pointer) unsafe.Pointer { spec := restore[*ast.TypeSpec](ptr) @@ -847,6 +892,18 @@ func GetReturnStmtResult(ptr unsafe.Pointer, i int) unsafe.Pointer { }) } +//export GetSendStmtChan +func GetSendStmtChan(ptr unsafe.Pointer) unsafe.Pointer { + stmt := restore[*ast.SendStmt](ptr) + return save(stmt.Chan) +} + +//export GetSendStmtValue +func GetSendStmtValue(ptr unsafe.Pointer) unsafe.Pointer { + stmt := restore[*ast.SendStmt](ptr) + return save(stmt.Value) +} + //export GetSwitchStmtInit func GetSwitchStmtInit(ptr unsafe.Pointer) unsafe.Pointer { stmt := restore[*ast.SwitchStmt](ptr) @@ -865,6 +922,24 @@ func GetSwitchStmtBody(ptr unsafe.Pointer) unsafe.Pointer { return save(stmt.Body) } +//export GetTypeSwitchStmtInit +func GetTypeSwitchStmtInit(ptr unsafe.Pointer) unsafe.Pointer { + stmt := restore[*ast.TypeSwitchStmt](ptr) + return save(stmt.Init) +} + +//export GetTypeSwitchStmtAssign +func GetTypeSwitchStmtAssign(ptr unsafe.Pointer) unsafe.Pointer { + stmt := restore[*ast.TypeSwitchStmt](ptr) + return save(stmt.Assign) +} + +//export GetTypeSwitchStmtBody +func GetTypeSwitchStmtBody(ptr unsafe.Pointer) unsafe.Pointer { + stmt := restore[*ast.TypeSwitchStmt](ptr) + return save(stmt.Body) +} + func restore[T any](ptr unsafe.Pointer) T { return pointer.Restore(ptr).(T) } diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/BuildConstraints.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/BuildConstraints.kt new file mode 100644 index 00000000000..83e21ae7439 --- /dev/null +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/BuildConstraints.kt @@ -0,0 +1,136 @@ +/* + * Copyright (c) 2023, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.frontends.golang + +import java.util.StringTokenizer + +fun interface BuildConstraintExpression { + fun evaluate(tags: Set): Boolean + + companion object { + fun fromString(str: String): BuildConstraintExpression? { + val tokenizer = StringTokenizer(str.replace(" ", ""), "!&|()", true) + + return tokenizer.readExpression() + } + + private fun StringTokenizer.readExpression(): BuildConstraintExpression? { + if (!this.hasMoreTokens()) { + return null + } + + when (val token = nextToken()) { + !in listOf("!", "(", ")", "&", "|") -> { + // A tag + val tag = BuildConstraintTag(token) + + // We need to check, if there is more + if (hasMoreTokens()) { + // If there is more, there needs to be an operator code + val op = readOperatorCode() ?: return null + + // There needs to be another expression + val rhs = readExpression() ?: return null + + return BuildConstraintBinaryExpression(op, tag, rhs) + } else { + return tag + } + } + "!" -> { + // A unary expression + val expr = readExpression() ?: return null + return BuildConstraintUnaryExpression("!", expr) + } + "(" -> { + val inner = this.nextToken(")") + val expr = fromString(inner) ?: return null + + return BuildConstraintParenthesis(expr) + } + else -> return null + } + } + + private fun StringTokenizer.readOperatorCode(): String? { + if (!this.hasMoreTokens()) { + return null + } + + val char1 = this.nextToken() + if (char1 != "|" && char1 != "&") { + return null + } + + if (!this.hasMoreTokens()) { + return null + } + + val char2 = this.nextToken() + if (char2 != char1) { + return null + } + + return char1 + char2 + } + } +} + +class BuildConstraintUnaryExpression( + val operatorCode: String, + val expr: BuildConstraintExpression +) : BuildConstraintExpression { + override fun evaluate(tags: Set): Boolean { + if (operatorCode == "!") { + return !expr.evaluate(tags) + } else { + TODO() + } + } +} + +class BuildConstraintBinaryExpression( + val operatorCode: String, + val lhs: BuildConstraintExpression, + val rhs: BuildConstraintExpression +) : BuildConstraintExpression { + override fun evaluate(tags: Set): Boolean { + return when (operatorCode) { + "&&" -> lhs.evaluate(tags) && rhs.evaluate(tags) + "||" -> lhs.evaluate(tags) || rhs.evaluate(tags) + else -> TODO() + } + } +} + +class BuildConstraintParenthesis(val expr: BuildConstraintExpression) : + BuildConstraintExpression by expr + +class BuildConstraintTag(val tag: String) : BuildConstraintExpression { + override fun evaluate(tags: Set): Boolean { + return tag in tags + } +} diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/DeclarationHandler.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/DeclarationHandler.kt index 3d102dc966f..06e3a48147e 100644 --- a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/DeclarationHandler.kt +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/DeclarationHandler.kt @@ -25,8 +25,6 @@ */ package de.fraunhofer.aisec.cpg.frontends.golang -import de.fraunhofer.aisec.cpg.frontends.Handler -import de.fraunhofer.aisec.cpg.frontends.HandlerInterface import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.* import de.fraunhofer.aisec.cpg.graph.statements.CompoundStatement @@ -35,36 +33,31 @@ import de.fraunhofer.aisec.cpg.graph.types.Type import de.fraunhofer.aisec.cpg.graph.types.UnknownType class DeclarationHandler(frontend: GoLanguageFrontend) : - Handler( - ::ProblemDeclaration, - frontend - ) { - - init { - map[GoStandardLibrary.Ast.FuncDecl::class.java] = HandlerInterface { - handleFuncDecl(it as GoStandardLibrary.Ast.FuncDecl) - } - map[GoStandardLibrary.Ast.GenDecl::class.java] = HandlerInterface { - handleGenDecl(it as GoStandardLibrary.Ast.GenDecl) + GoHandler(::ProblemDeclaration, frontend) { + + override fun handleNode(node: GoStandardLibrary.Ast.Decl): Declaration? { + return when (node) { + is GoStandardLibrary.Ast.FuncDecl -> handleFuncDecl(node) + is GoStandardLibrary.Ast.GenDecl -> handleGenDecl(node) + else -> { + return handleNotSupported(node, node.goType) + } } - map.put(GoStandardLibrary.Ast.Decl::class.java, ::handleNode) - } - - private fun handleNode(decl: GoStandardLibrary.Ast.Decl): Declaration { - val message = "Not parsing declaration of type ${decl.goType} yet" - log.error(message) - - return newProblemDeclaration(message) } private fun handleFuncDecl(funcDecl: GoStandardLibrary.Ast.FuncDecl): FunctionDeclaration { val recv = funcDecl.recv val func = if (recv != null) { - val method = newMethodDeclaration(funcDecl.name.name, rawNode = funcDecl) val recvField = recv.list.firstOrNull() val recordType = recvField?.type?.let { frontend.typeOf(it) } ?: unknownType() + val method = + newMethodDeclaration( + Name(funcDecl.name.name, recordType.root.name), + rawNode = funcDecl + ) + // The name of the Go receiver is optional. In fact, if the name is not // specified we probably do not need any receiver variable at all, // because the syntax is only there to ensure that this method is part @@ -79,7 +72,7 @@ class DeclarationHandler(frontend: GoLanguageFrontend) : } if (recordType !is UnknownType) { - val recordName = recordType.name + val recordName = recordType.root.name // TODO: this will only find methods within the current translation unit. // this is a limitation that we have for C++ as well @@ -93,9 +86,14 @@ class DeclarationHandler(frontend: GoLanguageFrontend) : // marked as AST and in Go a method is not part of the struct's AST but is // declared outside. In the future, we need to differentiate between just the // associated members of the class and the pure AST nodes declared in the - // struct - // itself - record?.addMethod(method) + // struct itself + if (record != null) { + method.recordDeclaration = record + record.addMethod(method) + + // Enter scope of record + frontend.scopeManager.enterScope(record) + } } method } else { @@ -143,69 +141,93 @@ class DeclarationHandler(frontend: GoLanguageFrontend) : // Parse parameters for (param in funcDecl.type.params.list) { - var name = "" - - // Somehow parameters end up having no name sometimes, have not fully understood why. - if (param.names.isNotEmpty()) { - name = param.names[0].name - - // If the name is an underscore, it means that the parameter is - // unnamed. In order to avoid confusing and some compatibility with - // other languages, we are just setting the name to an empty string - // in this case. - if (name == "_") { - name = "" - } + // We need to differentiate between three cases: + // - an empty list of names, which means that the parameter is unnamed; and we also give + // it an empty name + // - a single entry in the list of names, which is one regular parameter + // - multiple entries in the list of names, which specifies multiple parameters with the + // same type + // + // We can treat the last two cases together, by just gathering all the names and + // creating one param per name with the same type + val names = mutableListOf() + if (param.names.isEmpty()) { + names += "" } else { - log.warn("Some param has no name, which is a bit weird: $param") + names += + param.names.map { + // If the name is an underscore, it means that the parameter is + // unused (but not unnamed). + // + // But, but order to avoid confusion in resolving and to add + // some compatibility with other languages, we are just setting + // the name to an empty string in this case as well. + val name = it.name + if (name == "_") { + return@map "" + } + name + } } - // Check for varargs. In this case we want to parse the element type - // (and make it an array afterwards) - var variadic = false - val type = - if (param.type is GoStandardLibrary.Ast.Ellipsis) { - variadic = true - frontend.typeOf((param.type as GoStandardLibrary.Ast.Ellipsis).elt).array() - } else { - frontend.typeOf(param.type) - } - - val p = newParamVariableDeclaration(name, type, variadic, rawNode = param) + // Create one param variable per name + for (name in names) { + // Check for varargs. In this case we want to parse the element type + // (and make it an array afterwards) + val (type, variadic) = frontend.fieldTypeOf(param.type) - frontend.scopeManager.addDeclaration(p) + val p = newParamVariableDeclaration(name, type, variadic, rawNode = param) - frontend.setComment(p, param) + frontend.scopeManager.addDeclaration(p) + frontend.setComment(p, param) + } } - // Check, if the last statement is a return statement, otherwise we insert an implicit one - val body = frontend.statementHandler.handle(funcDecl.body) - if (body is CompoundStatement) { - val last = body.statements.lastOrNull() - if (last !is ReturnStatement) { - val ret = newReturnStatement() - ret.isImplicit = true - body += ret + // Only parse function body in non-dependencies + if (!frontend.isDependency) { + // Check, if the last statement is a return statement, otherwise we insert an implicit + // one + val body = funcDecl.body?.let { frontend.statementHandler.handle(it) } + if (body is CompoundStatement) { + val last = body.statements.lastOrNull() + if (last !is ReturnStatement) { + val ret = newReturnStatement() + ret.isImplicit = true + body += ret + } } + func.body = body } - func.body = body - frontend.scopeManager.leaveScope(func) + // Leave scope of record, if applicable + (func as? MethodDeclaration)?.recordDeclaration?.let { + frontend.scopeManager.leaveScope(it) + } + return func } private fun handleGenDecl(genDecl: GoStandardLibrary.Ast.GenDecl): DeclarationSequence { + // Reset the iota value. We need to start with -1 because we immediately increment in + // handleValueSpec + frontend.declCtx.iotaValue = -1 + // Set ourselves as the current gendecl + frontend.declCtx.currentDecl = genDecl + // Reset the initializers + frontend.declCtx.constInitializers.clear() + val sequence = DeclarationSequence() for (spec in genDecl.specs) { - frontend.specificationHandler.handle(spec)?.let { - sequence += it + val declaration = frontend.specificationHandler.handle(spec) + if (declaration != null) { + sequence += declaration // Go associates the comment to the genDecl, so we need to explicitly launch // setComment here. - frontend.setComment(it, genDecl) + frontend.setComment(declaration, genDecl) } } diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/ExpressionHandler.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/ExpressionHandler.kt index 6d268e6b2e6..5bd7869b5de 100644 --- a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/ExpressionHandler.kt +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/ExpressionHandler.kt @@ -29,35 +29,41 @@ import de.fraunhofer.aisec.cpg.frontends.golang.GoStandardLibrary.Ast.BasicLit.K import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration import de.fraunhofer.aisec.cpg.graph.statements.expressions.* +import de.fraunhofer.aisec.cpg.graph.types.ObjectType import de.fraunhofer.aisec.cpg.graph.types.PointerType import de.fraunhofer.aisec.cpg.graph.types.Type +import java.math.BigInteger class ExpressionHandler(frontend: GoLanguageFrontend) : GoHandler(::ProblemExpression, frontend) { - override fun handleNode(expr: GoStandardLibrary.Ast.Expr): Expression { - return when (expr) { - is GoStandardLibrary.Ast.BasicLit -> handleBasicLit(expr) - is GoStandardLibrary.Ast.BinaryExpr -> handleBinaryExpr(expr) - is GoStandardLibrary.Ast.CompositeLit -> handleCompositeLit(expr) - is GoStandardLibrary.Ast.FuncLit -> handleFuncLit(expr) - is GoStandardLibrary.Ast.Ident -> handleIdent(expr) - is GoStandardLibrary.Ast.IndexExpr -> handleIndexExpr(expr) - is GoStandardLibrary.Ast.CallExpr -> handleCallExpr(expr) - is GoStandardLibrary.Ast.KeyValueExpr -> handleKeyValueExpr(expr) - is GoStandardLibrary.Ast.SelectorExpr -> handleSelectorExpr(expr) - is GoStandardLibrary.Ast.SliceExpr -> handleSliceExpr(expr) - is GoStandardLibrary.Ast.TypeAssertExpr -> handleTypeAssertExpr(expr) - is GoStandardLibrary.Ast.UnaryExpr -> handleUnaryExpr(expr) + override fun handleNode(node: GoStandardLibrary.Ast.Expr): Expression { + return when (node) { + is GoStandardLibrary.Ast.BasicLit -> handleBasicLit(node) + is GoStandardLibrary.Ast.BinaryExpr -> handleBinaryExpr(node) + is GoStandardLibrary.Ast.CompositeLit -> handleCompositeLit(node) + is GoStandardLibrary.Ast.FuncLit -> handleFuncLit(node) + is GoStandardLibrary.Ast.Ident -> handleIdent(node) + is GoStandardLibrary.Ast.IndexExpr -> handleIndexExpr(node) + is GoStandardLibrary.Ast.CallExpr -> handleCallExpr(node) + is GoStandardLibrary.Ast.KeyValueExpr -> handleKeyValueExpr(node) + is GoStandardLibrary.Ast.ParenExpr -> { + return handle(node.x) + } + is GoStandardLibrary.Ast.SelectorExpr -> handleSelectorExpr(node) + is GoStandardLibrary.Ast.SliceExpr -> handleSliceExpr(node) + is GoStandardLibrary.Ast.StarExpr -> handleStarExpr(node) + is GoStandardLibrary.Ast.TypeAssertExpr -> handleTypeAssertExpr(node) + is GoStandardLibrary.Ast.UnaryExpr -> handleUnaryExpr(node) else -> { - return handleNotSupported(expr, expr.goType) + return handleNotSupported(node, node.goType) } } } private fun handleBasicLit(basicLit: GoStandardLibrary.Ast.BasicLit): Literal<*> { - val rawValue = basicLit.value - val value: Any? + var rawValue = basicLit.value + var value: Any? val type: Type when (basicLit.kind) { STRING -> { @@ -69,7 +75,31 @@ class ExpressionHandler(frontend: GoLanguageFrontend) : type = primitiveType("string") } INT -> { - value = rawValue.toInt() + // Get rid of all underscores + rawValue = rawValue.replace("_", "") + val prefix = rawValue.substring(0, 2.coerceAtMost(rawValue.length)) + val postfix = rawValue.substring(2.coerceAtMost(rawValue.length), rawValue.length) + + value = + when (prefix) { + "0x" -> BigInteger(postfix, 16) + "0o" -> BigInteger(postfix, 10) + "0b" -> BigInteger(postfix, 2) + else -> BigInteger(rawValue, 10) + } + + value = + when { + value > BigInteger.valueOf(Long.MAX_VALUE) -> { + value + } + value.toLong() > Int.MAX_VALUE -> { + value.toLong() + } + else -> { + value.toInt() + } + } type = primitiveType("int") } FLOAT -> { @@ -94,16 +124,26 @@ class ExpressionHandler(frontend: GoLanguageFrontend) : private fun handleBinaryExpr(binaryExpr: GoStandardLibrary.Ast.BinaryExpr): BinaryOperator { val binOp = newBinaryOperator(binaryExpr.opString, rawNode = binaryExpr) - binOp.lhs = handle(binaryExpr.x) ?: newProblemExpression("missing LHS") - binOp.rhs = handle(binaryExpr.y) ?: newProblemExpression("missing RHS") + binOp.lhs = handle(binaryExpr.x) + binOp.rhs = handle(binaryExpr.y) return binOp } private fun handleIdent(ident: GoStandardLibrary.Ast.Ident): Expression { - // Check, if this is 'nil', because then we handle it as a literal in the graph - if (ident.name == "nil") { - val literal = newLiteral(null, rawNode = ident) + val builtinLiterals = + mapOf( + "nil" to Pair(unknownType(), null), + "true" to Pair(primitiveType("bool"), true), + "false" to Pair(primitiveType("bool"), false), + "iota" to Pair(primitiveType("int"), frontend.declCtx.iotaValue) + ) + + // Check, if this is one of the builtinLiterals and handle them as a literal + val literalPair = builtinLiterals[ident.name] + if (literalPair != null) { + val (type, value) = literalPair + val literal = newLiteral(value, type, rawNode = ident) literal.name = parseName(ident.name) return literal @@ -134,7 +174,7 @@ class ExpressionHandler(frontend: GoLanguageFrontend) : private fun handleCallExpr(callExpr: GoStandardLibrary.Ast.CallExpr): Expression { // In Go, regular cast expressions (not type asserts are modelled as calls). // In this case, the Fun contains a type expression. - when (callExpr.`fun`) { + when (val unwrapped = unwrap(callExpr.`fun`)) { is GoStandardLibrary.Ast.ArrayType, is GoStandardLibrary.Ast.ChanType, is GoStandardLibrary.Ast.FuncType, @@ -142,22 +182,35 @@ class ExpressionHandler(frontend: GoLanguageFrontend) : is GoStandardLibrary.Ast.StructType, is GoStandardLibrary.Ast.MapType, -> { val cast = newCastExpression(rawNode = callExpr) - cast.castType = frontend.typeOf(callExpr.`fun`) + cast.castType = frontend.typeOf(unwrapped) if (callExpr.args.isNotEmpty()) { - frontend.expressionHandler.handle(callExpr.args[0])?.let { - cast.expression = it - } + cast.expression = frontend.expressionHandler.handle(callExpr.args[0]) } return cast } } - // Parse the Fun field, to see which kind of expression it is + val typeConstraints = mutableListOf() + val callee = - this.handle(callExpr.`fun`) - ?: return ProblemExpression("Could not parse call expr without fun") + when (val `fun` = callExpr.`fun`) { + // If "fun" is either an index or an index list expression, this is a call with type + // constraints. We do not fully support that yet, but we can at least try to set + // some of the parameters as template parameters + is GoStandardLibrary.Ast.IndexExpr -> { + (frontend.typeOf(`fun`) as? ObjectType)?.generics?.let { typeConstraints += it } + this.handle(`fun`.x) + } + is GoStandardLibrary.Ast.IndexListExpr -> { + (frontend.typeOf(`fun`) as? ObjectType)?.generics?.let { typeConstraints += it } + this.handle(`fun`.x) + } + else -> { + this.handle(callExpr.`fun`) + } + } // Handle special functions, such as make and new in a special way val name = callee.name.localName @@ -175,19 +228,41 @@ class ExpressionHandler(frontend: GoLanguageFrontend) : newCallExpression(callee, name, rawNode = callExpr) } + // TODO(oxisto) Add type constraints + if (typeConstraints.isNotEmpty()) { + log.debug( + "Call {} has type constraints ({}), but we cannot add them to the call expression yet", + call.name, + typeConstraints.joinToString(", ") { it.name } + ) + } + // Parse and add call arguments for (arg in callExpr.args) { - handle(arg)?.let { call += it } + call += handle(arg) } return call } + /** + * Unwrap is a small helper that unwraps an AST element that is wrapped into parenthesis. This + * is needed because we sometimes need to "peek" into the AST and this is not possible if the + * expression we are looking for is wrapped in parenthesis. + */ + private fun unwrap(expr: GoStandardLibrary.Ast.Expr): GoStandardLibrary.Ast.Expr { + return if (expr is GoStandardLibrary.Ast.ParenExpr) { + unwrap(expr.x) + } else { + expr + } + } + private fun handleKeyValueExpr( keyValueExpr: GoStandardLibrary.Ast.KeyValueExpr ): KeyValueExpression { - val key = handle(keyValueExpr.key) ?: newProblemExpression("could not parse key") - val value = handle(keyValueExpr.value) ?: newProblemExpression("could not parse value") + val key = handle(keyValueExpr.key) + val value = handle(keyValueExpr.value) return newKeyValueExpression(key, value, rawNode = keyValueExpr) } @@ -228,7 +303,7 @@ class ExpressionHandler(frontend: GoLanguageFrontend) : // second argument is a dimension (if this is an array), usually a literal if (args.size > 1) { - handle(args[1])?.let { array.addDimension(it) } + array.addDimension(handle(args[1])) } array } else { @@ -253,13 +328,16 @@ class ExpressionHandler(frontend: GoLanguageFrontend) : private fun handleSelectorExpr( selectorExpr: GoStandardLibrary.Ast.SelectorExpr ): DeclaredReferenceExpression { - val base = handle(selectorExpr.x) ?: newProblemExpression("missing base") + val base = handle(selectorExpr.x) // Check, if this just a regular reference to a variable with a package scope and not a // member expression var isMemberExpression = true for (imp in frontend.currentFile?.imports ?: listOf()) { - if (base.name.localName == frontend.getImportName(imp)) { + // If we have an alias, we need to check it instead of the import name + val name = imp.name?.name ?: imp.importName + + if (base.name.localName == name) { // found a package name, so this is NOT a member expression isMemberExpression = false } @@ -289,9 +367,7 @@ class ExpressionHandler(frontend: GoLanguageFrontend) : sliceExpr: GoStandardLibrary.Ast.SliceExpr ): ArraySubscriptionExpression { val ase = newArraySubscriptionExpression(rawNode = sliceExpr) - ase.arrayExpression = - frontend.expressionHandler.handle(sliceExpr.x) - ?: newProblemExpression("missing array expression") + ase.arrayExpression = frontend.expressionHandler.handle(sliceExpr.x) // Build the slice expression val range = newRangeExpression(rawNode = sliceExpr) @@ -304,20 +380,39 @@ class ExpressionHandler(frontend: GoLanguageFrontend) : return ase } - private fun handleTypeAssertExpr( - typeAssertExpr: GoStandardLibrary.Ast.TypeAssertExpr - ): CastExpression { - val cast = newCastExpression(rawNode = typeAssertExpr) - - // Parse the inner expression - cast.expression = - handle(typeAssertExpr.x) ?: newProblemExpression("missing inner expression") + private fun handleStarExpr(starExpr: GoStandardLibrary.Ast.StarExpr): UnaryOperator { + val op = newUnaryOperator("*", postfix = false, prefix = false, rawNode = starExpr) + op.input = handle(starExpr.x) - // The type can be null, but only in certain circumstances, i.e, a type switch (which we do - // not support yet) - typeAssertExpr.type?.let { cast.castType = frontend.typeOf(it) } + return op + } - return cast + private fun handleTypeAssertExpr( + typeAssertExpr: GoStandardLibrary.Ast.TypeAssertExpr + ): Expression { + // This can either be a regular type assertion, which we handle as a cast expression or the + // "special" type assertion `.(type)`, which is used in a type switch to retrieve the type + // of the variable. In this case we treat it as a special unary operator. + return if (typeAssertExpr.type == null) { + val op = + newUnaryOperator( + ".(type)", + postfix = true, + prefix = false, + rawNode = typeAssertExpr + ) + op.input = handle(typeAssertExpr.x) + op + } else { + val cast = newCastExpression(rawNode = typeAssertExpr) + + // Parse the inner expression + cast.expression = handle(typeAssertExpr.x) + + // The type can be null, but only in certain circumstances, i.e, a type switch + typeAssertExpr.type?.let { cast.castType = frontend.typeOf(it) } + cast + } } private fun handleUnaryExpr(unaryExpr: GoStandardLibrary.Ast.UnaryExpr): UnaryOperator { @@ -328,7 +423,7 @@ class ExpressionHandler(frontend: GoLanguageFrontend) : prefix = false, rawNode = unaryExpr ) - handle(unaryExpr.x)?.let { op.input = it } + op.input = handle(unaryExpr.x) return op } @@ -338,31 +433,24 @@ class ExpressionHandler(frontend: GoLanguageFrontend) : * of a ConstructExpression and a list of KeyValueExpressions. The problem is that we need to * add the list as a first argument of the construct expression. */ - private fun handleCompositeLit( - compositeLit: GoStandardLibrary.Ast.CompositeLit - ): ConstructExpression { - // Parse the type field, to see which kind of expression it is - val type = frontend.typeOf(compositeLit.type) - - val construct = newConstructExpression(type.name, rawNode = compositeLit) - construct.type = type + private fun handleCompositeLit(compositeLit: GoStandardLibrary.Ast.CompositeLit): Expression { + // Parse the type field, to see which kind of expression it is. The type of a composite + // literal can be omitted if this is an "inner" composite literal, so we need to set it from + // the "outer" one. See below + val type = compositeLit.type?.let { frontend.typeOf(it) } ?: unknownType() val list = newInitializerListExpression(type, rawNode = compositeLit) - construct += list - - // Normally, the construct expression would not have DFG edge, but in this case we are - // mis-using it to simulate an object literal, so we need to add a DFG here, otherwise a - // declaration is disconnected from its initialization. - construct.addPrevDFG(list) + list.type = type val expressions = mutableListOf() for (elem in compositeLit.elts) { - handle(elem)?.let { expressions += it } + val expression = handle(elem) + expressions += expression } list.initializers = expressions - return construct + return list } /* diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoHandler.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoHandler.kt index dc21160c7a6..eeebac42a43 100644 --- a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoHandler.kt +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoHandler.kt @@ -28,21 +28,26 @@ package de.fraunhofer.aisec.cpg.frontends.golang import de.fraunhofer.aisec.cpg.frontends.Handler import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.ProblemNode +import de.fraunhofer.aisec.cpg.graph.statements.expressions.Literal import de.fraunhofer.aisec.cpg.helpers.Util import java.util.function.Supplier -abstract class GoHandler( - configConstructor: Supplier, +abstract class GoHandler( + configConstructor: Supplier, lang: GoLanguageFrontend -) : Handler(configConstructor, lang) { +) : Handler(configConstructor, lang) { /** * We intentionally override the logic of [Handler.handle] because we do not want the map-based * logic, but rather want to make use of the Kotlin-when syntax. * - * We also want non-nullable result handlers + * We also want to support nullable results for some handlers, e.g., when ignoring private + * declarations in dependencies. */ override fun handle(ctx: HandlerNode): ResultNode { val node = handleNode(ctx) + if (node == null) { + return node + } // The language frontend might set a location, which we should respect. Otherwise, we will // set the location here. @@ -77,6 +82,28 @@ abstract class GoHandler + val paths = (path?.value as? String)?.split("/") ?: listOf() + + // Return the last name in the path as the import name. However, if the last name is a + // module version (e.g., v5), then we need to return the second-to-last + var last = paths.lastOrNull() + last = + if (last?.startsWith("v") == true) { + paths.getOrNull(paths.size - 2) + } else { + last + } + + return last ?: "" + } } diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguage.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguage.kt index eaa4dc68634..61b73e8305f 100644 --- a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguage.kt +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguage.kt @@ -25,16 +25,18 @@ */ package de.fraunhofer.aisec.cpg.frontends.golang -import de.fraunhofer.aisec.cpg.frontends.HasGenerics -import de.fraunhofer.aisec.cpg.frontends.HasShortCircuitOperators -import de.fraunhofer.aisec.cpg.frontends.HasStructs -import de.fraunhofer.aisec.cpg.frontends.Language +import de.fraunhofer.aisec.cpg.frontends.* import de.fraunhofer.aisec.cpg.graph.types.* import org.neo4j.ogm.annotation.Transient /** The Go language. */ class GoLanguage : - Language(), HasShortCircuitOperators, HasGenerics, HasStructs { + Language(), + HasShortCircuitOperators, + HasGenerics, + HasStructs, + HasFirstClassFunctions, + HasAnonymousIdentifier { override val fileExtensions = listOf("go") override val namespaceDelimiter = "." @Transient override val frontend = GoLanguageFrontend::class @@ -108,4 +110,28 @@ class GoLanguage : // https://pkg.go.dev/builtin#string "string" to StringType("string", this) ) + + override fun isDerivedFrom(type: Type, superType: Type): Boolean { + if (type == superType) { + return true + } + + // We additionally want to emulate the behaviour of Go's interface system here + if (superType is ObjectType && superType.recordDeclaration?.kind == "interface") { + var b = true + val target = (type.root as? ObjectType)?.recordDeclaration + + // Our target struct type needs to implement all the functions of the interface + // TODO(oxisto): Differentiate on the receiver (pointer vs non-pointer) + for (method in superType.recordDeclaration?.methods ?: listOf()) { + if (target?.methods?.firstOrNull { it.signature == method.signature } != null) { + b = false + } + } + + return b + } + + return false + } } diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontend.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontend.kt index 33fc7bb06cd..1e721c563cc 100644 --- a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontend.kt +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontend.kt @@ -36,10 +36,11 @@ import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.DeclarationSequence import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration import de.fraunhofer.aisec.cpg.graph.newNamespaceDeclaration -import de.fraunhofer.aisec.cpg.graph.statements.expressions.Literal import de.fraunhofer.aisec.cpg.graph.types.FunctionType +import de.fraunhofer.aisec.cpg.graph.types.ObjectType import de.fraunhofer.aisec.cpg.graph.types.Type import de.fraunhofer.aisec.cpg.graph.unknownType +import de.fraunhofer.aisec.cpg.helpers.Util import de.fraunhofer.aisec.cpg.passes.EvaluationOrderGraphPass import de.fraunhofer.aisec.cpg.passes.GoEvaluationOrderGraphPass import de.fraunhofer.aisec.cpg.passes.GoExtraPass @@ -71,19 +72,79 @@ class GoLanguageFrontend(language: Language, ctx: Translatio private var commentMap: GoStandardLibrary.Ast.CommentMap? = null var currentFile: GoStandardLibrary.Ast.File? = null + var isDependency: Boolean = false + val declarationHandler = DeclarationHandler(this) val specificationHandler = SpecificationHandler(this) var statementHandler = StatementHandler(this) var expressionHandler = ExpressionHandler(this) + /** + * This helper class contains values needed to properly decide in which state const declaration + * / specifications are in. + */ + class DeclarationContext { + /** + * The current value of `iota`. This needs to be reset for each + * [GoStandardLibrary.Ast.GenDecl] and incremented for each observed + * [GoStandardLibrary.Ast.ValueSpec]. + */ + var iotaValue = -1 + + /** + * The current initializers in a list representing the different "columns". For example in + * the following code: + * ```go + * const ( + * a, b = 1, 2 + * c, d + * e, f = 4, 5 + * ) + * ``` + * + * The current list of initializers would first be (`1`,`2`) until a new set of initializers + * is declared in the last spec. The key corresponds to the "column" of the variable + * (a=0,b=1). + */ + var constInitializers = mutableMapOf() + + /** The current [GoStandardLibrary.Ast.GenDecl] that is being processed. */ + var currentDecl: GoStandardLibrary.Ast.GenDecl? = null + } + + /** + * The current [DeclarationContext]. This is somewhat of a workaround since we cannot properly + * communicate state between different handlers. However, because *within* a [LanguageFrontend], + * everything is parsed sequentially according to AST order, we can safely use this context + * here. + */ + var declCtx = DeclarationContext() + @Throws(TranslationException::class) override fun parse(file: File): TranslationUnitDeclaration { + if (!shouldBuild(file, ctx.config.symbols)) { + log.debug( + "Ignoring the contents of {} because of missing build tags or different GOOS/GOARCH.", + file + ) + return newTranslationUnitDeclaration(file.name) + } + + val dependency = + ctx.config.includePaths.firstOrNull { + file.absolutePath.contains(it.toAbsolutePath().toString()) + } + // Make sure, that our top level is set either way val topLevel = - if (config.topLevel != null) { - config.topLevel - } else { - file.parentFile + // If this file is part of an include, we set the top level to the root of the include + when { + dependency != null -> { + isDependency = true + dependency.toFile() + } + config.topLevel != null -> config.topLevel + else -> file.parentFile }!! val std = GoStandardLibrary.INSTANCE @@ -95,7 +156,7 @@ class GoLanguageFrontend(language: Language, ctx: Translatio } val fset = std.NewFileSet() - val f = Parser.parseFile(fset, file.absolutePath, file.readText()) + val f = Parser.parseFile(fset, file.absolutePath) this.commentMap = std.NewCommentMap(fset, f, f.comments) @@ -119,8 +180,12 @@ class GoLanguageFrontend(language: Language, ctx: Translatio // module path as well as the current directory in relation to the topLevel var packagePath = file.parentFile.relativeTo(topLevel) - // If we are in a module, we need to prepend the module path to it - currentModule?.let { packagePath = File(it.module.mod.path).resolve(packagePath) } + // If we are in a module, we need to prepend the module path to it. There is an + // exception if we are in the "std" module, which represents the standard library + val modulePath = currentModule?.module?.mod?.path + if (modulePath != null && modulePath != "std") { + packagePath = File(modulePath).resolve(packagePath) + } p.path = packagePath.path } catch (ex: IllegalArgumentException) { @@ -149,46 +214,157 @@ class GoLanguageFrontend(language: Language, ctx: Translatio } override fun typeOf(type: GoStandardLibrary.Ast.Expr): Type { - return when (type) { - is GoStandardLibrary.Ast.Ident -> { - val name: String = - if (isBuiltinType(type.name)) { - // Definitely not an FQN type - type.name - } else { - // FQN'ize this name (with the current file) - "${currentFile?.name?.name}.${type.name}" // this.File.Name.Name + val type = + when (type) { + is GoStandardLibrary.Ast.Ident -> { + val name: String = + if (isBuiltinType(type.name)) { + // Definitely not an FQN type + type.name + } else { + // FQN'ize this name (with the current file) + "${currentFile?.name?.name}.${type.name}" // this.File.Name.Name + } + + objectType(name) + } + is GoStandardLibrary.Ast.SelectorExpr -> { + // This is a FQN type + val baseName = (type.x as? GoStandardLibrary.Ast.Ident)?.name?.let { Name(it) } + + return objectType(Name(type.sel.name, baseName)) + } + is GoStandardLibrary.Ast.ArrayType -> { + return typeOf(type.elt).array() + } + is GoStandardLibrary.Ast.ChanType -> { + // Handle them similar to a map type (see below) + return objectType("chan", listOf(typeOf(type.value))) + } + is GoStandardLibrary.Ast.FuncType -> { + val paramTypes = + type.params.list + .flatMap { field -> + // Because we can have unnamed parameters or multiple parameters + // declared at once, we need to expand the list of types according + // to the list of names + if (field.names.isEmpty()) { + listOf(field.type) + } else { + field.names.map { field.type } + } + } + .map { fieldTypeOf(it).first } + val returnTypes = type.results?.list?.map { typeOf(it.type) } ?: listOf() + val name = funcTypeName(paramTypes, returnTypes) + + FunctionType(name, paramTypes, returnTypes, this.language) + } + is GoStandardLibrary.Ast.IndexExpr -> { + // A go type constraint, aka generic + val baseType = typeOf(type.x) + val generics = listOf(typeOf(type.index)) + objectType(baseType.name, generics) + } + is GoStandardLibrary.Ast.IndexListExpr -> { + // A go type constraint, aka generic with multiple types + val baseType = typeOf(type.x) + val generics = type.indices.map { typeOf(it) } + objectType(baseType.name, generics) + } + is GoStandardLibrary.Ast.StructType -> { + // Go allows to use anonymous structs as type. This is something we cannot model + // properly in the CPG yet. In order to at least deal with this partially, we + // construct a ObjectType and put the fields and their types into the type. + // This will result in something like `struct{name string; args util.args; want + // string}` + val parts = + type.fields.list.map { field -> + var desc = "" + // Name can be optional, if its embedded + field.names.getOrNull(0)?.let { desc += it } + desc += " " + desc += fieldTypeOf(field.type).first.name + desc + } + + val name = parts.joinToString("; ", "struct{", "}") + + // Create an anonymous struct, this will add it to the scope manager. This is + // somewhat duplicate, but the easiest for now. We need to create it in the + // global + // scope to avoid namespace issues + var record = + scopeManager.withScope(scopeManager.globalScope) { + specificationHandler.buildRecordDeclaration(type, name) + } + + record.toType() + } + is GoStandardLibrary.Ast.InterfaceType -> { + // Go allows to use anonymous interface as type. This is something we cannot + // model + // properly in the CPG yet. In order to at least deal with this partially, we + // construct a ObjectType and put the methods and their types into the type. + + // In the easiest case this is the empty interface `interface{}`, which we then + // consider to be the "any" type. `any` is actually a type alias for + // `interface{}`, + // but in modern Go `any` is preferred. + if (type.methods.list.isEmpty()) { + return primitiveType("any") } - objectType(name) - } - is GoStandardLibrary.Ast.ArrayType -> { - return typeOf(type.elt).array() - } - is GoStandardLibrary.Ast.ChanType -> { - // Handle them similar to a map type (see below) - return objectType("chan", listOf(typeOf(type.value))) + val parts = + type.methods.list.map { method -> + var desc = "" + // Name can be optional, if its embedded + method.names.getOrNull(0)?.let { desc += it } + // the function type has a weird "func" prefix, which we do not want + desc += typeOf(method.type).name.toString().removePrefix("func") + desc + } + + objectType(parts.joinToString("; ", "interface{", "}")) + } + is GoStandardLibrary.Ast.MapType -> { + // We cannot properly represent Go's built-in map types, yet so we have + // to make a shortcut here and represent it as a Java-like map type. + return objectType("map", listOf(typeOf(type.key), typeOf(type.value))) + } + is GoStandardLibrary.Ast.StarExpr -> { + typeOf(type.x).pointer() + } + else -> { + Util.warnWithFileLocation( + this, + type, + log, + "Not parsing type of type ${type.goType} yet" + ) + unknownType() + } } - is GoStandardLibrary.Ast.FuncType -> { - val paramTypes = type.params.list.map { typeOf(it.type) } - val returnTypes = type.results?.list?.map { typeOf(it.type) } ?: listOf() - val name = funcTypeName(paramTypes, returnTypes) - return FunctionType(name, paramTypes, returnTypes, this.language) - } - is GoStandardLibrary.Ast.MapType -> { - // We cannot properly represent Go's built-in map types, yet so we have - // to make a shortcut here and represent it as a Java-like map type. - return objectType("map", listOf(typeOf(type.key), typeOf(type.value))) - } - is GoStandardLibrary.Ast.StarExpr -> { - typeOf(type.x).pointer() - } - else -> { - log.warn("Not parsing type of type ${type.goType} yet") - unknownType() + return typeManager.registerType(typeManager.resolvePossibleTypedef(type, scopeManager)) + } + + /** + * A quick helper function to retrieve the type of a field, to check for possible variadic + * arguments. + */ + internal fun fieldTypeOf( + paramType: GoStandardLibrary.Ast.Expr, + ): Pair { + var variadic = false + val type = + if (paramType is GoStandardLibrary.Ast.Ellipsis) { + variadic = true + typeOf(paramType.elt).array() + } else { + typeOf(paramType) } - } + return Pair(type, variadic) } private fun isBuiltinType(name: String): Boolean { @@ -267,15 +443,62 @@ class GoLanguageFrontend(language: Language, ctx: Translatio return pn.joinToString(", ", prefix = "func(", postfix = ")$rs") } - fun getImportName(spec: GoStandardLibrary.Ast.ImportSpec): String { - val name = spec.name - if (name != null) { - return name.name - } - - val path = expressionHandler.handle(spec.path) as? Literal<*> - val paths = (path?.value as? String)?.split("/") ?: listOf() + companion object { + /** + * All possible goos values. See + * https://github.com/golang/go/blob/release-branch.go1.21/src/go/build/syslist.go#L11 + */ + val goosValues = + listOf( + "aix", + "android", + "darwin", + "dragonfly", + "freebsd", + "hurd", + "illumos", + "ios", + "js", + "linux", + "nacl", + "netbsd", + "openbsd", + "plan9", + "solaris", + "wasip1", + "windows", + "zos" + ) - return paths.lastOrNull() ?: "" + /** + * All possible architecture values. See + * https://github.com/golang/go/blob/release-branch.go1.21/src/go/build/syslist.go#L54 + */ + val goarchValues = + listOf( + "386", + "amd64", + "arm", + "arm64", + "loong64", + "mips", + "mips64", + "mips64le", + "mipsle", + "ppc64", + "ppc64le", + "riscv64", + "s390x" + ) } } + +val Type?.underlyingType: Type? + get() { + return (this as? ObjectType)?.recordDeclaration?.superClasses?.singleOrNull() + } + +val Type?.isOverlay: Boolean + get() { + return this is ObjectType && this.recordDeclaration?.kind == "overlay" + } diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoStandardLibrary.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoStandardLibrary.kt index f7ca20b70bd..0fc6480d579 100644 --- a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoStandardLibrary.kt +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoStandardLibrary.kt @@ -52,8 +52,8 @@ interface GoStandardLibrary : Library { * this package. */ object Parser { - fun parseFile(fileSet: Ast.FileSet, path: String, contents: String): Ast.File { - return INSTANCE.goParserParseFile(fileSet, path, contents) + fun parseFile(fileSet: Ast.FileSet, path: String): Ast.File { + return INSTANCE.goParserParseFile(fileSet, path) } } @@ -153,6 +153,11 @@ interface GoStandardLibrary : Library { get() { return list(INSTANCE::GetNumGenDeclSpecs, INSTANCE::GetGenDeclSpec) } + + val tok: Int + get() { + return INSTANCE.GetGenDeclTok(this) + } } class FuncDecl(p: Pointer? = Pointer.NULL) : Decl(p) { @@ -168,7 +173,7 @@ interface GoStandardLibrary : Library { return INSTANCE.GetFuncDeclName(this) } - val body: BlockStmt + val body: BlockStmt? get() { return INSTANCE.GetFuncDeclBody(this) } @@ -195,6 +200,11 @@ interface GoStandardLibrary : Library { return INSTANCE.GetTypeSpecName(this) } + val assign: Int + get() { + return INSTANCE.GetTypeSpecAssign(this) + } + val type: Expr get() { return INSTANCE.GetTypeSpecType(this) @@ -243,7 +253,9 @@ interface GoStandardLibrary : Library { "*ast.FuncLit" -> FuncLit(nativeValue) "*ast.Ident" -> Ident(nativeValue) "*ast.IndexExpr" -> IndexExpr(nativeValue) + "*ast.IndexListExpr" -> IndexListExpr(nativeValue) "*ast.KeyValueExpr" -> KeyValueExpr(nativeValue) + "*ast.ParenExpr" -> ParenExpr(nativeValue) "*ast.SelectorExpr" -> SelectorExpr(nativeValue) "*ast.StarExpr" -> StarExpr(nativeValue) "*ast.SliceExpr" -> SliceExpr(nativeValue) @@ -311,7 +323,7 @@ interface GoStandardLibrary : Library { } class CompositeLit(p: Pointer? = Pointer.NULL) : Expr(p) { - val type: Expr + val type: Expr? get() { return INSTANCE.GetCompositeLitType(this) } @@ -334,6 +346,13 @@ interface GoStandardLibrary : Library { } } + class ParenExpr(p: Pointer? = Pointer.NULL) : Expr(p) { + val x: Expr + get() { + return INSTANCE.GetParenExprX(this) + } + } + class FuncLit(p: Pointer? = Pointer.NULL) : Expr(p) { fun toDecl(): FuncDecl { return INSTANCE.MakeFuncDeclFromFuncLit(this) @@ -345,6 +364,11 @@ interface GoStandardLibrary : Library { } class Ident(p: Pointer? = Pointer.NULL) : Expr(p) { + val isUnexported: Boolean + get() { + return name.isNotEmpty() && name[0].isLowerCase() + } + val name: String get() { return INSTANCE.GetIdentName(this) @@ -367,6 +391,21 @@ interface GoStandardLibrary : Library { } } + class IndexListExpr(p: Pointer? = Pointer.NULL) : Expr(p) { + val x: Expr + get() { + return INSTANCE.GetIndexListExprX(this) + } + + val indices: List + get() { + return list( + INSTANCE::GetNumIndexListExprIndices, + INSTANCE::GetIndexListExprIndex + ) + } + } + class SelectorExpr(p: Pointer? = Pointer.NULL) : Expr(p) { val x: Expr get() { @@ -517,7 +556,9 @@ interface GoStandardLibrary : Library { "*ast.LabeledStmt" -> LabeledStmt(nativeValue) "*ast.RangeStmt" -> RangeStmt(nativeValue) "*ast.ReturnStmt" -> ReturnStmt(nativeValue) + "*ast.SendStmt" -> SendStmt(nativeValue) "*ast.SwitchStmt" -> SwitchStmt(nativeValue) + "*ast.TypeSwitchStmt" -> TypeSwitchStmt(nativeValue) else -> super.fromNative(nativeValue, context) } } @@ -700,6 +741,18 @@ interface GoStandardLibrary : Library { } } + class SendStmt(p: Pointer? = Pointer.NULL) : Stmt(p) { + val chan: Expr + get() { + return INSTANCE.GetSendStmtChan(this) + } + + val value: Expr + get() { + return INSTANCE.GetSendStmtValue(this) + } + } + class SwitchStmt(p: Pointer? = Pointer.NULL) : Stmt(p) { val init: Stmt? get() { @@ -717,6 +770,23 @@ interface GoStandardLibrary : Library { } } + class TypeSwitchStmt(p: Pointer? = Pointer.NULL) : Stmt(p) { + val init: Stmt? + get() { + return INSTANCE.GetTypeSwitchStmtInit(this) + } + + val assign: Stmt + get() { + return INSTANCE.GetTypeSwitchStmtAssign(this) + } + + val body: BlockStmt + get() { + return INSTANCE.GetTypeSwitchStmtBody(this) + } + } + class Position(p: Pointer? = Pointer.NULL) : GoObject(p) { val line: Int get() { @@ -774,7 +844,7 @@ interface GoStandardLibrary : Library { // go/parser package - fun goParserParseFile(fileSet: Ast.FileSet, path: String, src: String): Ast.File + fun goParserParseFile(fileSet: Ast.FileSet, path: String): Ast.File fun GetType(obj: Pointer): String @@ -826,9 +896,9 @@ interface GoStandardLibrary : Library { fun GetFuncDeclName(funcDecl: Ast.FuncDecl): Ast.Ident - fun GetFuncDeclBody(funcDecl: Ast.FuncDecl): Ast.BlockStmt + fun GetFuncDeclBody(funcDecl: Ast.FuncDecl): Ast.BlockStmt? - fun GetCompositeLitType(compositeLit: Ast.CompositeLit): Ast.Expr + fun GetCompositeLitType(compositeLit: Ast.CompositeLit): Ast.Expr? fun GetNumCompositeLitElts(compositeLit: Ast.CompositeLit): Int @@ -844,6 +914,8 @@ interface GoStandardLibrary : Library { fun GetKeyValueExprValue(keyValueExpr: Ast.KeyValueExpr): Ast.Expr + fun GetParenExprX(parenExpr: Ast.ParenExpr): Ast.Expr + fun GetBasicLitValue(basicLit: Ast.BasicLit): String fun GetBasicLitKind(basicLit: Ast.BasicLit): Int @@ -958,6 +1030,12 @@ interface GoStandardLibrary : Library { fun GetIndexExprIndex(IndexExpr: Ast.IndexExpr): Ast.Expr + fun GetIndexListExprX(indexListExpr: Ast.IndexListExpr): Ast.Expr + + fun GetNumIndexListExprIndices(indexListExpr: Ast.IndexListExpr): Int + + fun GetIndexListExprIndex(indexListExpr: Ast.IndexListExpr, i: Int): Ast.Expr + fun GetIfStmtInit(ifStmt: Ast.IfStmt): Ast.Stmt? fun GetIfStmtCond(ifStmt: Ast.IfStmt): Ast.Expr @@ -980,16 +1058,28 @@ interface GoStandardLibrary : Library { fun GetReturnStmtResult(returnStmt: Ast.ReturnStmt, i: Int): Ast.Expr + fun GetSendStmtChan(sendStmt: Ast.SendStmt): Ast.Expr + + fun GetSendStmtValue(sendStmt: Ast.SendStmt): Ast.Expr + fun GetSwitchStmtInit(switchStmt: Ast.SwitchStmt): Ast.Stmt? fun GetSwitchStmtTag(switchStmt: Ast.SwitchStmt): Ast.Expr? fun GetSwitchStmtBody(stmt: Ast.SwitchStmt): Ast.BlockStmt + fun GetTypeSwitchStmtInit(typeSwitchStmt: Ast.TypeSwitchStmt): Ast.Stmt? + + fun GetTypeSwitchStmtAssign(typeSwitchStmt: Ast.TypeSwitchStmt): Ast.Stmt + + fun GetTypeSwitchStmtBody(typeSwitchStmt: Ast.TypeSwitchStmt): Ast.BlockStmt + fun GetNumGenDeclSpecs(genDecl: Ast.GenDecl): Int fun GetGenDeclSpec(genDecl: Ast.GenDecl, i: Int): Ast.Spec + fun GetGenDeclTok(genDecl: Ast.GenDecl): Int + fun GetImportSpecName(importSpec: Ast.ImportSpec): Ast.Ident? fun GetImportSpecPath(importSpec: Ast.ImportSpec): Ast.BasicLit @@ -998,7 +1088,7 @@ interface GoStandardLibrary : Library { fun GetValueSpecName(valueSpec: Ast.ValueSpec, i: Int): Ast.Ident - fun GetValueSpecType(valueSpec: Ast.ValueSpec): Ast.Expr + fun GetValueSpecType(valueSpec: Ast.ValueSpec): Ast.Expr? fun GetNumValueSpecValues(valueSpec: Ast.ValueSpec): Int @@ -1006,6 +1096,8 @@ interface GoStandardLibrary : Library { fun GetTypeSpecName(typeSpec: Ast.TypeSpec): Ast.Ident + fun GetTypeSpecAssign(typeSpec: Ast.TypeSpec): Int + fun GetTypeSpecType(typeSpec: Ast.TypeSpec): Ast.Expr companion object { diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoUtils.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoUtils.kt new file mode 100644 index 00000000000..5685293ba6b --- /dev/null +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoUtils.kt @@ -0,0 +1,188 @@ +/* + * Copyright (c) 2023, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.frontends.golang + +import java.io.File +import java.util.concurrent.TimeUnit + +fun shouldBuild(file: File, symbols: Map): Boolean { + // First, we need to check, whether the filename ends with a possible _$GOOS.go, $_GOARCH.go + // or $_GOOS_$GOARCH.go + val parts = file.nameWithoutExtension.split("_") + + // If the last part is a possible GOOS value, then we need to check, if this is equal to our + // current GOOS + if ( + parts.lastOrNull() in GoLanguageFrontend.goosValues && parts.lastOrNull() != symbols["GOOS"] + ) { + // Skip the contents + return false + } + + // If the last part is a possible GOARCH value, then we need to check, if this is equal to + // our current GOARCH + if ( + parts.lastOrNull() in GoLanguageFrontend.goarchValues && + parts.lastOrNull() != symbols["GOARCH"] + ) { + return false + } + + val sub = parts.subList((parts.size - 2).coerceAtLeast(0), parts.size.coerceAtLeast(0)) + if ( + sub.size == 2 && + ((sub[0] in GoLanguageFrontend.goosValues && sub[0] != symbols["GOOS"]) || + (sub[1] in GoLanguageFrontend.goarchValues && sub[1] != symbols["GOARCH"])) + ) { + return false + } + + // Next, we need to peek into the file, to see whether any build tags are present. The + // fastest way + // to do that is to read the file and look for a go:build line + val goBuildLine = + file + .bufferedReader() + .useLines { lines -> lines.take(50).toList() } + .firstOrNull() { it.startsWith("//go:build") } + ?: return true + + val constraint = BuildConstraintExpression.fromString(goBuildLine.substringAfter("//go:build ")) + + return constraint?.evaluate(symbols.buildTags) == true +} + +private val Map.buildTags: Set + get() { + val tags = mutableSetOf() + val goos = this["GOOS"] + val goarch = this["GOARCH"] + + // Add GOOS and GOARCH + goos?.let { tags += it } + goarch?.let { tags += it } + + // We need to derive some more build tags based on the GOOS. + // See + // https://github.com/golang/go/blob/release-branch.go1.21/src/go/build/syslist.go#L39 + if ( + goos in + listOf( + "aix", + "android", + "darwin", + "dragonfly", + "freebsd", + "hurd", + "illumos", + "ios", + "linux", + "netbsd", + "openbsd", + "solaris" + ) + ) { + tags += "unix" + } + + // Additional "derived" operating systems + when (goos) { + "android" -> tags += "linux" + "illumos" -> tags += "solaris" + "ios" -> tags += "darwin" + } + + // Add remaining tags + this["-tags"]?.split(" ")?.let { tags += it } + + return tags + } + +fun gatherGoFiles(root: File, includeSubDir: Boolean = true): List { + return root + .walkTopDown() + .onEnter { (it == root || includeSubDir) && !it.name.contains(".go") } + .filter { + // skip tests for now + it.extension == "go" && !it.name.endsWith("_test.go") + } + .toList() +} + +class Project { + var symbols: Map = mutableMapOf() + + var components: MutableMap> = mutableMapOf() + + var includePaths: List = mutableListOf() +} + +fun buildProject( + modulePath: String, + goos: String? = null, + goarch: String? = null, + tags: List = listOf() +): Project { + val project = Project() + val symbols = mutableMapOf() + var files = mutableListOf() + + val topLevel = File(modulePath) + + var proc = + ProcessBuilder("go", "list", "all") + .directory(topLevel) + .redirectOutput(ProcessBuilder.Redirect.PIPE) + .start() + proc.waitFor(5, TimeUnit.MINUTES) + + // For now, we only support deps in the standard library + val deps = proc.inputStream.bufferedReader().readLines().filter { !it.contains(".") } + + proc = + ProcessBuilder("go", "env", "GOROOT").redirectOutput(ProcessBuilder.Redirect.PIPE).start() + proc.waitFor(5, TimeUnit.MINUTES) + + val stdLib = File(proc.inputStream.bufferedReader().readLine()).resolve("src") + + files += deps.flatMap { gatherGoFiles(stdLib.resolve(it), false) } + files += gatherGoFiles(topLevel) + + goos?.let { symbols["GOOS"] = it } + goarch?.let { symbols["GOARCH"] = it } + tags.let { symbols["-tags"] = tags.joinToString { " " } } + + // Pre-filter any files we are not building anyway based on our symbols + files = files.filter { shouldBuild(it, symbols) }.toMutableList() + + // TODO(oxisto): look for binaries + project.components["app"] = files + project.symbols = symbols + // TODO(oxisto): support vendor includes + project.includePaths = listOf(stdLib) + + return project +} diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/SpecificationHandler.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/SpecificationHandler.kt index 8dd43ab566b..d43623b2c10 100644 --- a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/SpecificationHandler.kt +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/SpecificationHandler.kt @@ -25,40 +25,29 @@ */ package de.fraunhofer.aisec.cpg.frontends.golang -import de.fraunhofer.aisec.cpg.frontends.Handler -import de.fraunhofer.aisec.cpg.frontends.HandlerInterface import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.* +import de.fraunhofer.aisec.cpg.graph.scopes.NameScope +import de.fraunhofer.aisec.cpg.helpers.Util class SpecificationHandler(frontend: GoLanguageFrontend) : - Handler( - ::ProblemDeclaration, - frontend - ) { - - init { - map[GoStandardLibrary.Ast.ImportSpec::class.java] = HandlerInterface { - handleImportSpec(it as GoStandardLibrary.Ast.ImportSpec) - } - map[GoStandardLibrary.Ast.TypeSpec::class.java] = HandlerInterface { - handleTypeSpec(it as GoStandardLibrary.Ast.TypeSpec) - } - map[GoStandardLibrary.Ast.ValueSpec::class.java] = HandlerInterface { - handleValueSpec(it as GoStandardLibrary.Ast.ValueSpec) - } - map.put(GoStandardLibrary.Ast.Spec::class.java, ::handleNode) - } + GoHandler(::ProblemDeclaration, frontend) { - private fun handleNode(spec: GoStandardLibrary.Ast.Spec): Declaration { - val message = "Not parsing specification of type ${spec.goType} yet" - log.error(message) - - return newProblemDeclaration(message) + override fun handleNode(node: GoStandardLibrary.Ast.Spec): Declaration? { + return when (node) { + is GoStandardLibrary.Ast.ImportSpec -> handleImportSpec(node) + is GoStandardLibrary.Ast.TypeSpec -> handleTypeSpec(node) + is GoStandardLibrary.Ast.ValueSpec -> handleValueSpec(node) + else -> { + return handleNotSupported(node, node.goType) + } + } } private fun handleImportSpec(importSpec: GoStandardLibrary.Ast.ImportSpec): IncludeDeclaration { // We set the name of the include declaration to the imported name, i.e., the package name - val name = frontend.getImportName(importSpec) + val name = importSpec.importName + // We set the filename of the include declaration to the package path, i.e., its full path // including any module identifiers. This way we can match the include declaration back to // the namespace's path and name @@ -66,6 +55,17 @@ class SpecificationHandler(frontend: GoLanguageFrontend) : val include = newIncludeDeclaration(filename, rawNode = importSpec) include.name = parseName(name) + var alias = importSpec.name?.name + if (alias != null && alias != name) { + var location = frontend.locationOf(importSpec) + + // If the name differs from the import name, we have an alias + // Add an alias for the package on the current file + location?.artifactLocation?.let { + frontend.scopeManager.addAlias(it, Name(name), Name(alias)) + } + } + return include } @@ -75,7 +75,14 @@ class SpecificationHandler(frontend: GoLanguageFrontend) : when (type) { is GoStandardLibrary.Ast.StructType -> handleStructTypeSpec(spec, type) is GoStandardLibrary.Ast.InterfaceType -> handleInterfaceTypeSpec(spec, type) - else -> return ProblemDeclaration("not parsing type of type ${spec.goType} yet") + is GoStandardLibrary.Ast.FuncType -> handleFuncTypeSpec(spec, type) + is GoStandardLibrary.Ast.Ident, + is GoStandardLibrary.Ast.SelectorExpr, + is GoStandardLibrary.Ast.MapType, + is GoStandardLibrary.Ast.ArrayType, + is GoStandardLibrary.Ast.StarExpr, + is GoStandardLibrary.Ast.ChanType -> handleTypeDef(spec, type) + else -> return ProblemDeclaration("not parsing type of type ${type.goType} yet") } return decl @@ -85,26 +92,39 @@ class SpecificationHandler(frontend: GoLanguageFrontend) : typeSpec: GoStandardLibrary.Ast.TypeSpec, structType: GoStandardLibrary.Ast.StructType ): RecordDeclaration { - val record = newRecordDeclaration(typeSpec.name.name, "struct", rawNode = typeSpec) + val record = buildRecordDeclaration(structType, typeSpec.name.name, typeSpec) + + // Make sure to register the type + frontend.typeManager.registerType(record.toType()) + + return record + } + + fun buildRecordDeclaration( + structType: GoStandardLibrary.Ast.StructType, + name: CharSequence, + typeSpec: GoStandardLibrary.Ast.TypeSpec? = null, + ): RecordDeclaration { + val record = newRecordDeclaration(name, "struct", rawNode = typeSpec) frontend.scopeManager.enterScope(record) if (!structType.incomplete) { for (field in structType.fields.list) { - // a field can also have no name, which means that it is embedded, not quite - // sure yet how to handle this, but since the embedded field can be accessed - // by its type, it could make sense to name the field according to the type val type = frontend.typeOf(field.type) - val name = + // A field can also have no name, which means that it is embedded. In this case, it + // can be accessed by the local name of its type and therefore we name the field + // accordingly + val fieldName = if (field.names.isEmpty()) { - // Retrieve the root type name - type.root.name.toString() + // Retrieve the root type local name + type.root.name.localName } else { field.names[0].name } - val decl = newFieldDeclaration(name, type, rawNode = field) + val decl = newFieldDeclaration(fieldName, type, rawNode = field) frontend.scopeManager.addDeclaration(decl) } } @@ -120,6 +140,9 @@ class SpecificationHandler(frontend: GoLanguageFrontend) : ): Declaration { val record = newRecordDeclaration(typeSpec.name.name, "interface", rawNode = typeSpec) + // Make sure to register the type + frontend.typeManager.registerType(record.toType()) + frontend.scopeManager.enterScope(record) if (!interfaceType.incomplete) { @@ -151,39 +174,155 @@ class SpecificationHandler(frontend: GoLanguageFrontend) : } /** - * // handleValueSpec handles parsing of an ast.ValueSpec, which is a variable // declaration. - * Since this can potentially declare multiple variables with one // "spec", this returns a + * // handleValueSpec handles parsing of an ast.ValueSpec, which is a variable declaration. + * Since this can potentially declare multiple variables with one "spec", this returns a * [DeclarationSequence]. */ - private fun handleValueSpec(valueSpec: GoStandardLibrary.Ast.ValueSpec): DeclarationSequence { - val sequence = DeclarationSequence() + private fun handleValueSpec( + valueSpec: GoStandardLibrary.Ast.ValueSpec, + ): Declaration { + // Increment iota value + frontend.declCtx.iotaValue++ - for ((idx, ident) in valueSpec.names.withIndex()) { - val decl = newVariableDeclaration(ident.name, rawNode = valueSpec) + // If we only have one initializer on the right side and multiple ones on the left side, + // we are deconstructing a tuple + val lenValues = valueSpec.values.size + if (lenValues == 1 && lenValues != valueSpec.names.size) { + // We need to construct a "tuple" declaration on the left side that holds all the + // variables + val tuple = TupleDeclaration() + tuple.type = autoType() - if (valueSpec.type != null) { - decl.type = frontend.typeOf(valueSpec.type!!) - } else { - decl.type = autoType() - } + for (ident in valueSpec.names) { + // We want to make sure that top-level declarations, i.e, the ones that are directly + // in a namespace are FQNs. Otherwise we cannot resolve them properly when we access + // them outside of the package. + val fqn = + if (frontend.scopeManager.currentScope is NameScope) { + fqn(ident.name) + } else { + ident.name + } + val decl = newVariableDeclaration(fqn, rawNode = valueSpec) + + if (valueSpec.type != null) { + decl.type = frontend.typeOf(valueSpec.type!!) + } else { + decl.type = autoType() + } + + if (valueSpec.values.isNotEmpty()) { + tuple.initializer = frontend.expressionHandler.handle(valueSpec.values[0]) + } - // There could either be no initializers, otherwise the amount of values - // must match the names - val lenValues = valueSpec.values.size - if (lenValues != 0 && lenValues != valueSpec.names.size) { - log.error( - "Number of initializers does not match number of names. Initializers might be incomplete" - ) + // We need to manually add the variables to the scope manager + frontend.scopeManager.addDeclaration(decl) + + tuple += decl } + return tuple + } else { + val sequence = DeclarationSequence() + + for ((nameIdx, ident) in valueSpec.names.withIndex()) { + // We want to make sure that top-level declarations, i.e, the ones that are directly + // in a namespace are FQNs. Otherwise we cannot resolve them properly when we access + // them outside of the package. + val fqn = + if (frontend.scopeManager.currentScope is NameScope) { + fqn(ident.name) + } else { + ident.name + } + val decl = newVariableDeclaration(fqn, rawNode = valueSpec) + + if (valueSpec.type != null) { + decl.type = frontend.typeOf(valueSpec.type!!) + } else { + decl.type = autoType() + } + + if (valueSpec.values.size > nameIdx) { + // the initializer is in the "Values" slice with the respective index + decl.initializer = frontend.expressionHandler.handle(valueSpec.values[nameIdx]) + } + + // If we are in a const declaration, we need to do something rather unusual. + // If we have an initializer, we need to set this as the current const initializer, + // because following specs will "inherit" the one from the previous line. + // + // Note: we cannot just take the already parsed initializer, but instead we need to + // reparse the raw AST expression, so that `iota` gets evaluated differently for + // each spec + if (frontend.declCtx.currentDecl?.tok == 64) { + var initializerExpr = valueSpec.values.getOrNull(nameIdx) + if (initializerExpr != null) { + // Set the current initializer + frontend.declCtx.constInitializers[nameIdx] = initializerExpr + } else { + // Fetch expr from existing initializers + initializerExpr = frontend.declCtx.constInitializers[nameIdx] + if (initializerExpr == null) { + Util.errorWithFileLocation( + decl, + log, + "Const declaration is missing its initializer" + ) + } else { + decl.initializer = frontend.expressionHandler.handle(initializerExpr) + } + } + } - // The initializer is in the "Values" slice with the respective index - if (valueSpec.values.size > idx) { - decl.initializer = frontend.expressionHandler.handle(valueSpec.values[idx]) + sequence += decl } - sequence += decl + return sequence } + } + + private fun handleFuncTypeSpec( + spec: GoStandardLibrary.Ast.TypeSpec, + type: GoStandardLibrary.Ast.FuncType + ): Declaration { + // We model function types as typedef's, so that we can resolve it later + val funcType = frontend.typeOf(type) + val typedef = newTypedefDeclaration(funcType, frontend.typeOf(spec.name), rawNode = spec) + + frontend.scopeManager.addTypedef(typedef) - return sequence + return typedef + } + + private fun handleTypeDef( + spec: GoStandardLibrary.Ast.TypeSpec, + type: GoStandardLibrary.Ast.Expr + ): Declaration { + val targetType = frontend.typeOf(type) + + // We need to return either a type alias or a new type. See + // https://go.dev/ref/spec#Type_identity + return when { + // When we have an assignment, we have *identical* types. We handle them as a typedef / + // alias. + spec.assign != 0 -> { + val aliasType = frontend.typeOf(spec.name) + val typedef = newTypedefDeclaration(targetType, aliasType, rawNode = spec) + + frontend.scopeManager.addTypedef(typedef) + typedef + } + // Otherwise, we are creating a new type, which is *different*. Since Go allows to add + // methods to these kind of types, we need to create them as a record declaration. We + // use the special kind "overlay" to identity such types and put the target type in the + // list of superclasses. + else -> { + val record = newRecordDeclaration(spec.name.name, "overlay") + + // We add the underlying type as the single super class + record.superClasses = mutableListOf(targetType) + record + } + } } } diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/StatementHandler.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/StatementHandler.kt index 0159cf2d428..a0b2165d8ac 100644 --- a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/StatementHandler.kt +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/StatementHandler.kt @@ -29,6 +29,8 @@ import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.DeclarationSequence import de.fraunhofer.aisec.cpg.graph.statements.* import de.fraunhofer.aisec.cpg.graph.statements.expressions.* +import de.fraunhofer.aisec.cpg.graph.types.HasType +import de.fraunhofer.aisec.cpg.graph.types.Type class StatementHandler(frontend: GoLanguageFrontend) : GoHandler(::ProblemExpression, frontend) { @@ -51,7 +53,9 @@ class StatementHandler(frontend: GoLanguageFrontend) : is GoStandardLibrary.Ast.LabeledStmt -> handleLabeledStmt(stmt) is GoStandardLibrary.Ast.RangeStmt -> handleRangeStmt(stmt) is GoStandardLibrary.Ast.ReturnStmt -> handleReturnStmt(stmt) + is GoStandardLibrary.Ast.SendStmt -> handleSendStmt(stmt) is GoStandardLibrary.Ast.SwitchStmt -> handleSwitchStmt(stmt) + is GoStandardLibrary.Ast.TypeSwitchStmt -> handleTypeSwitchStmt(stmt) else -> handleNotSupported(stmt, stmt.goType) } } @@ -114,13 +118,26 @@ class StatementHandler(frontend: GoLanguageFrontend) : return compound } - private fun handleCaseClause(caseClause: GoStandardLibrary.Ast.CaseClause): Statement { + private fun handleCaseClause( + caseClause: GoStandardLibrary.Ast.CaseClause, + typeSwitchLhs: Node? = null, + typeSwitchRhs: Expression? = null, + ): Statement { + val isTypeSwitch = typeSwitchRhs != null + val case = if (caseClause.list.isEmpty()) { newDefaultStatement(rawNode = caseClause) } else { val case = newCaseStatement(rawNode = caseClause) - case.caseExpression = frontend.expressionHandler.handle(caseClause.list[0]) + if (isTypeSwitch) { + // If this case is within a type switch, we want to wrap the case expression in + // a TypeExpression + val type = frontend.typeOf(caseClause.list[0]) + case.caseExpression = newTypeExpression(type.name, type) + } else { + case.caseExpression = frontend.expressionHandler.handle(caseClause.list[0]) + } case } @@ -135,10 +152,65 @@ class StatementHandler(frontend: GoLanguageFrontend) : // Add the case statement block += case + // Wrap everything inside the case in a block statement + var compound = + if (isTypeSwitch) { + newCompoundStatement() + } else { + null + } + + compound?.let { frontend.scopeManager.enterScope(it) } + + // TODO(oxisto): This variable is not yet resolvable + if (isTypeSwitch && typeSwitchRhs != null && typeSwitchLhs != null) { + val stmt = newDeclarationStatement() + stmt.isImplicit = true + + val decl = newVariableDeclaration(typeSwitchLhs.name) + if (case is CaseStatement) { + decl.type = (case.caseExpression as? TypeExpression)?.type ?: unknownType() + } else { + // We need to work with type listeners here because they might not have their type + // yet + typeSwitchRhs.registerTypeObserver( + object : HasType.TypeObserver { + override fun typeChanged(newType: Type, src: HasType) { + decl.type = newType + } + + override fun assignedTypeChanged(assignedTypes: Set, src: HasType) { + // Nothing to do + } + } + ) + } + decl.initializer = typeSwitchRhs + + // Add the variable to the declaration statement as well as to the current scope (aka + // our block wrapper) + stmt.addToPropertyEdgeDeclaration(decl) + frontend.scopeManager.addDeclaration(decl) + + if (compound != null) { + compound += stmt + } + } + for (s in caseClause.body) { - block += handle(s) + if (compound != null) { + compound += handle(s) + } else { + block += handle(s) + } } + if (compound != null) { + block += compound + } + + compound?.let { frontend.scopeManager.leaveScope(it) } + // this is a little trick, to not add the case statement in handleStmt because we added it // already. otherwise, the order is screwed up. return case @@ -285,9 +357,7 @@ class StatementHandler(frontend: GoLanguageFrontend) : val expr = frontend.expressionHandler.handle(results[0]) // TODO: parse more than one result expression - if (expr != null) { - `return`.returnValue = expr - } + `return`.returnValue = expr } else { // TODO: connect result statement to result variables } @@ -295,6 +365,14 @@ class StatementHandler(frontend: GoLanguageFrontend) : return `return` } + private fun handleSendStmt(sendStmt: GoStandardLibrary.Ast.SendStmt): BinaryOperator { + val op = newBinaryOperator("<-", rawNode = sendStmt) + op.lhs = frontend.expressionHandler.handle(sendStmt.chan) + op.rhs = frontend.expressionHandler.handle(sendStmt.value) + + return op + } + private fun handleSwitchStmt(switchStmt: GoStandardLibrary.Ast.SwitchStmt): Statement { val switch = newSwitchStatement(rawNode = switchStmt) @@ -307,19 +385,46 @@ class StatementHandler(frontend: GoLanguageFrontend) : handle(switchStmt.body) as? CompoundStatement ?: return newProblemExpression("missing switch body") - // Because of the way we parse the statements, the case statement turns out to be the last - // statement. However, we need it to be the first statement, so we need to switch first and - // last items - /*val statements = block.statements.toMutableList() - val tmp = statements.first() - statements[0] = block.statements.last() - statements[(statements.size - 1).coerceAtLeast(0)] = tmp - block.statements = statements*/ - switch.statement = block frontend.scopeManager.leaveScope(switch) return switch } + + private fun handleTypeSwitchStmt( + typeSwitchStmt: GoStandardLibrary.Ast.TypeSwitchStmt + ): SwitchStatement { + val switch = newSwitchStatement(rawNode = typeSwitchStmt) + + frontend.scopeManager.enterScope(switch) + + typeSwitchStmt.init?.let { switch.initializerStatement = handle(it) } + + val assign = frontend.statementHandler.handle(typeSwitchStmt.assign) + val (lhs, rhs) = + if (assign is AssignExpression) { + val rhs = assign.rhs.singleOrNull() + switch.selector = rhs + Pair(assign.lhs.singleOrNull(), (rhs as? UnaryOperator)?.input) + } else { + Pair(null, null) + } + + val body = newCompoundStatement(rawNode = typeSwitchStmt.body) + + frontend.scopeManager.enterScope(body) + + for (c in typeSwitchStmt.body.list.filterIsInstance()) { + handleCaseClause(c, lhs, rhs) + } + + frontend.scopeManager.leaveScope(body) + + switch.statement = body + + frontend.scopeManager.leaveScope(switch) + + return switch + } } diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/GoEvaluationOrderGraphPass.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/GoEvaluationOrderGraphPass.kt index e519e15f308..8237c0aabe1 100644 --- a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/GoEvaluationOrderGraphPass.kt +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/GoEvaluationOrderGraphPass.kt @@ -28,8 +28,6 @@ package de.fraunhofer.aisec.cpg.passes import de.fraunhofer.aisec.cpg.TranslationContext import de.fraunhofer.aisec.cpg.frontends.golang.GoLanguage import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration -import de.fraunhofer.aisec.cpg.graph.followNextEOGEdgesUntilHit -import de.fraunhofer.aisec.cpg.graph.statements.ReturnStatement import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.UnaryOperator @@ -90,13 +88,14 @@ class GoEvaluationOrderGraphPass(ctx: TranslationContext) : EvaluationOrderGraph // We need to follow the path from the defer statement to all return statements that are // reachable from this point. for (defer in defers ?: listOf()) { - val paths = defer.followNextEOGEdgesUntilHit { it is ReturnStatement } + // TODO(oxisto): This is broken! this returns 8192 paths instead of 4 + /*val paths = defer.followNextEOGEdgesUntilHit { it is ReturnStatement } for (path in paths.fulfilled) { // It is a bit philosophical whether the deferred call happens before or after the // return statement in the EOG. For now, it is easier to have it as the last node // AFTER the return statement addEOGEdge(path.last(), defer.input) - } + }*/ } } } diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/GoExtraPass.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/GoExtraPass.kt index de3dd2c9b6f..d7d6387f6f2 100644 --- a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/GoExtraPass.kt +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/GoExtraPass.kt @@ -27,21 +27,15 @@ package de.fraunhofer.aisec.cpg.passes import de.fraunhofer.aisec.cpg.TranslationContext import de.fraunhofer.aisec.cpg.frontends.golang.GoLanguage +import de.fraunhofer.aisec.cpg.frontends.golang.isOverlay +import de.fraunhofer.aisec.cpg.frontends.golang.underlyingType import de.fraunhofer.aisec.cpg.graph.* -import de.fraunhofer.aisec.cpg.graph.declarations.IncludeDeclaration -import de.fraunhofer.aisec.cpg.graph.declarations.NamespaceDeclaration -import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration +import de.fraunhofer.aisec.cpg.graph.declarations.* import de.fraunhofer.aisec.cpg.graph.scopes.Scope import de.fraunhofer.aisec.cpg.graph.statements.DeclarationStatement import de.fraunhofer.aisec.cpg.graph.statements.ForEachStatement -import de.fraunhofer.aisec.cpg.graph.statements.expressions.AssignExpression -import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression -import de.fraunhofer.aisec.cpg.graph.statements.expressions.CastExpression -import de.fraunhofer.aisec.cpg.graph.statements.expressions.DeclaredReferenceExpression -import de.fraunhofer.aisec.cpg.graph.types.HasType -import de.fraunhofer.aisec.cpg.graph.types.PointerType -import de.fraunhofer.aisec.cpg.graph.types.Type -import de.fraunhofer.aisec.cpg.graph.types.UnknownType +import de.fraunhofer.aisec.cpg.graph.statements.expressions.* +import de.fraunhofer.aisec.cpg.graph.types.* import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker import de.fraunhofer.aisec.cpg.passes.inference.startInference import de.fraunhofer.aisec.cpg.passes.order.ExecuteBefore @@ -101,6 +95,13 @@ import de.fraunhofer.aisec.cpg.passes.order.ExecuteBefore * they are compatible. Because types in the same package can be defined in multiple files, we * cannot decide during the frontend run. Therefore, we need to execute this pass before the * [CallResolver] and convert certain [CallExpression] nodes into a [CastExpression]. + * + * ## Adjust Names of Keys in Key Value Expressions to FQN + * + * This pass also adjusts the names of keys in a [KeyValueExpression], which is part of an + * [InitializerListExpression] to a fully-qualified name that contains the name of the [ObjectType] + * that the expression is creating. This way we can resolve the static references to the field to + * the actual field. */ @ExecuteBefore(VariableUsageResolver::class) @ExecuteBefore(CallResolver::class) @@ -118,6 +119,7 @@ class GoExtraPass(ctx: TranslationContext) : ComponentPass(ctx), ScopeProvider { is IncludeDeclaration -> handleInclude(node) is AssignExpression -> handleAssign(node) is ForEachStatement -> handleForEachStatement(node) + is InitializerListExpression -> handleInitializerListExpression(node) } } @@ -126,6 +128,60 @@ class GoExtraPass(ctx: TranslationContext) : ComponentPass(ctx), ScopeProvider { } } + /** + * handleInitializerListExpression changes the references of keys in a [KeyValueExpression] to + * include the object it is creating as a parent name. + */ + private fun handleInitializerListExpression(node: InitializerListExpression) { + var type: Type? = node.type + + // If our type is an "overlay", we need to look for the underlying type + type = + if (type.isOverlay) { + type.underlyingType + } else { + type + } + + // The type of a "inner" composite literal can be omitted if the outer one is creating + // an array type. In this case, we need to set the type manually because the type for + // the "inner" one is empty. + // Example code: + // ```go + // var a = []*MyObject{ + // { + // Name: "a", + // }, + // { + // Name: "b", + // } + // } + if (type is PointerType && type.isArray) { + for (init in node.initializers) { + if (init is InitializerListExpression) { + init.type = type.elementType + } + } + } + + // We are not interested in arrays and maps, but only the "inner" single-object expressions + if ( + type is UnknownType || + (type is PointerType && type.isArray) || + node.type.name.localName == "map" + ) { + return + } + + for (keyValue in node.initializers.filterIsInstance()) { + val key = keyValue.key + if (key is DeclaredReferenceExpression) { + key.name = Name(key.name.localName, node.type.root.name) + key.isStaticAccess = true + } + } + } + /** * handleForEachStatement adds a [HasType.TypeObserver] to the [ForEachStatement.iterable] of an * [ForEachStatement] in order to determine the types used in [ForEachStatement.variable] (index @@ -173,16 +229,23 @@ class GoExtraPass(ctx: TranslationContext) : ComponentPass(ctx), ScopeProvider { } // Loop through the target variables (left-hand side) - for (expr in assign.lhs) { + for ((idx, expr) in assign.lhs.withIndex()) { if (expr is DeclaredReferenceExpression) { // And try to resolve it val ref = scopeManager.resolveReference(expr) if (ref == null) { - // We need to implicitly declare it, if its not declared before. + // We need to implicitly declare it, if it's not declared before. val decl = newVariableDeclaration(expr.name, expr.autoType()) decl.location = expr.location decl.isImplicit = true - decl.initializer = assign.findValue(expr) + + // We cannot assign an initializer here because this will lead to duplicate + // DFG edges, but we need to propagate the type information + if (assign.rhs.size < assign.lhs.size && assign.rhs.size == 1) { + assign.rhs[0].registerTypeObserver(InitializerTypePropagation(decl, idx)) + } else { + assign.rhs[idx].registerTypeObserver(InitializerTypePropagation(decl)) + } assign.declarations += decl @@ -230,13 +293,20 @@ class GoExtraPass(ctx: TranslationContext) : ComponentPass(ctx), ScopeProvider { // We need to check, whether the "callee" refers to a type and if yes, convert it into a // cast expression. And this is only really necessary, if the function call has a single // argument. - val callee = call.callee - if (parent != null && callee is DeclaredReferenceExpression && call.arguments.size == 1) { + var callee = call.callee + if (parent != null && callee != null && call.arguments.size == 1) { val language = parent.language ?: GoLanguage() + var pointer = false + // If the argument is a UnaryOperator, unwrap them + if (callee is UnaryOperator && callee.operatorCode == "*") { + pointer = true + callee = callee.input + } + // First, check if this is a built-in type if (language.builtInTypes.contains(callee.name.toString())) { - replaceCallWithCast(callee.name.toString(), parent, call) + replaceCallWithCast(callee.name.toString(), parent, call, false) } else { // If not, then this could still refer to an existing type. We need to make sure // that we take the current namespace into account @@ -248,7 +318,7 @@ class GoExtraPass(ctx: TranslationContext) : ComponentPass(ctx), ScopeProvider { } if (typeManager.typeExists(fqn.toString())) { - replaceCallWithCast(fqn, parent, call) + replaceCallWithCast(fqn, parent, call, pointer) } } } @@ -258,10 +328,16 @@ class GoExtraPass(ctx: TranslationContext) : ComponentPass(ctx), ScopeProvider { typeName: CharSequence, parent: Node, call: CallExpression, + pointer: Boolean, ) { val cast = parent.newCastExpression(call.code) cast.location = call.location - cast.castType = call.objectType(typeName) + cast.castType = + if (pointer) { + call.objectType(typeName).pointer() + } else { + call.objectType(typeName) + } cast.expression = call.arguments.single() if (parent !is ArgumentHolder) { @@ -284,4 +360,19 @@ class GoExtraPass(ctx: TranslationContext) : ComponentPass(ctx), ScopeProvider { override fun cleanup() { // Nothing to do } + + class InitializerTypePropagation(private var decl: HasType, private var tupleIdx: Int = -1) : + HasType.TypeObserver { + override fun typeChanged(newType: Type, src: HasType) { + if (newType is TupleType && tupleIdx != -1) { + decl.type = newType.types.getOrElse(tupleIdx) { decl.unknownType() } + } else { + decl.type = newType + } + } + + override fun assignedTypeChanged(assignedTypes: Set, src: HasType) { + // TODO + } + } } diff --git a/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/BuildConstraintsTest.kt b/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/BuildConstraintsTest.kt new file mode 100644 index 00000000000..814456b487e --- /dev/null +++ b/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/BuildConstraintsTest.kt @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2023, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.frontends.golang + +import kotlin.test.* + +class BuildConstraintsTest { + + @Test + fun testFromString() { + val string = "darwin && !amd64" + + val root = BuildConstraintExpression.fromString(string) + assertNotNull(root) + + var expr = root + assertIs(expr) + assertEquals("&&", expr.operatorCode) + + val lhs = expr.lhs + assertIs(lhs) + assertEquals("darwin", lhs.tag) + + val rhs = expr.rhs + assertIs(rhs) + assertEquals("!", rhs.operatorCode) + + expr = rhs.expr + assertIs(expr) + assertEquals("amd64", expr.tag) + + val darwinAmd64 = setOf("darwin", "amd64") + assertFalse(root.evaluate(darwinAmd64)) + + val darwinArm64 = setOf("darwin", "arm64") + assertTrue(root.evaluate(darwinArm64)) + } +} diff --git a/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/DeclarationTest.kt b/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/DeclarationTest.kt index 5e379141c1f..f0842837ab0 100644 --- a/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/DeclarationTest.kt +++ b/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/DeclarationTest.kt @@ -26,15 +26,18 @@ package de.fraunhofer.aisec.cpg.frontends.golang import de.fraunhofer.aisec.cpg.TestUtils +import de.fraunhofer.aisec.cpg.TestUtils.assertInvokes +import de.fraunhofer.aisec.cpg.TestUtils.assertRefersTo import de.fraunhofer.aisec.cpg.assertFullName import de.fraunhofer.aisec.cpg.assertLiteralValue +import de.fraunhofer.aisec.cpg.assertLocalName import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.byNameOrNull -import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration -import de.fraunhofer.aisec.cpg.graph.declarations.MethodDeclaration -import de.fraunhofer.aisec.cpg.graph.declarations.NamespaceDeclaration -import de.fraunhofer.aisec.cpg.graph.declarations.RecordDeclaration -import de.fraunhofer.aisec.cpg.graph.statements.expressions.DeclaredReferenceExpression +import de.fraunhofer.aisec.cpg.graph.declarations.* +import de.fraunhofer.aisec.cpg.graph.statements.CompoundStatement +import de.fraunhofer.aisec.cpg.graph.statements.ReturnStatement +import de.fraunhofer.aisec.cpg.graph.statements.expressions.* +import de.fraunhofer.aisec.cpg.graph.types.ObjectType import de.fraunhofer.aisec.cpg.graph.variables import java.nio.file.Path import kotlin.test.* @@ -90,6 +93,89 @@ class DeclarationTest { assertFullName("", param) } + @Test + fun testStruct() { + val topLevel = Path.of("src", "test", "resources", "golang") + val tu = + TestUtils.analyzeAndGetFirstTU( + listOf(topLevel.resolve("struct.go").toFile()), + topLevel, + true + ) { + it.registerLanguage() + } + assertNotNull(tu) + + val p = tu.namespaces["p"] + assertNotNull(p) + + val myStruct = p.records["MyStruct"] + assertNotNull(myStruct) + assertEquals("struct", myStruct.kind) + + val fields = myStruct.fields + assertEquals( + listOf("MyField", "OtherStruct", "EvenAnotherStruct"), + fields.map { it.name.localName } + ) + + var methods = myStruct.methods + + var myFunc = methods.firstOrNull() + assertNotNull(myFunc) + assertFullName("p.MyStruct.MyFunc", myFunc) + + val myField = fields.firstOrNull() + assertNotNull(myField) + + assertLocalName("MyField", myField) + assertEquals(tu.primitiveType("int"), myField.type) + + val myInterface = p.records["p.MyInterface"] + assertNotNull(myInterface) + assertEquals("interface", myInterface.kind) + + methods = myInterface.methods + + assertEquals(1, methods.size) + + myFunc = methods.first() + + assertLocalName("MyFunc", myFunc) + assertLocalName("func() string", myFunc.type) + + val newMyStruct = p.functions["NewMyStruct"] + assertNotNull(newMyStruct) + + val body = newMyStruct.body as? CompoundStatement + assertNotNull(body) + + val `return` = body.statements.first() as? ReturnStatement + assertNotNull(`return`) + + val returnValue = `return`.returnValue as? UnaryOperator + assertNotNull(returnValue) + + val s = p.variables["p.s"] + assertNotNull(s) + + val type = s.type + assertIs(type) + + val record = type.recordDeclaration + assertNotNull(record) + + val init = s.initializer + assertIs(init) + + val keyValue = init.initializers(0) + assertNotNull(keyValue) + + val key = keyValue.key + assertNotNull(key) + assertRefersTo(key, record.fields["field"]) + } + @Test fun testEmbeddedInterface() { val topLevel = Path.of("src", "test", "resources", "golang") @@ -134,8 +220,8 @@ class DeclarationTest { val main = tu.functions["main.main"] assertNotNull(main) - // We should have 7 variables (a, b, c, d, e, f, g) - assertEquals(7, tu.variables.size) + // We should have 10 variables (a, b, c, d, (e,f), e, f, g, h, i) + assertEquals(10, main.variables.size) // Four should have (literal) initializers val a = main.variables["a"] @@ -150,13 +236,24 @@ class DeclarationTest { val d = main.variables["d"] assertLiteralValue(4, d?.initializer) + val e = main.variables["e"] + assertNotNull(e) + // e does not have a direct initializer, since it is initialized through the tuple + // declaration (e,f) + assertNull(e.initializer) + + // The tuple (e,f) does have an initializer + val ef = main.allChildren { it.name.toString() == "(e,f)" }.firstOrNull() + assertNotNull(ef) + assertIs(ef.initializer) + // The next two variables are using a short assignment, therefore they do not have an // initializer, but we can use the firstAssignment function - val e = main.variables["e"] - assertLiteralValue(5, e?.firstAssignment) + val g = main.variables["g"] + assertLiteralValue(5, g?.firstAssignment) - val f = main.variables["f"] - assertLiteralValue(6, f?.firstAssignment) + val h = main.variables["h"] + assertLiteralValue(6, h?.firstAssignment) // And they should all be connected to the arguments of the Printf call val printf = main.calls["Printf"] @@ -167,8 +264,8 @@ class DeclarationTest { assertNotNull(ref.refersTo) } - // We have eight assignments in total (6 initializers + 2 assign expressions) - assertEquals(8, tu.assignments.size) + // We have eight assignments in total (7 initializers + 2 assign expressions) + assertEquals(9, tu.assignments.size) } @Test @@ -190,4 +287,106 @@ class DeclarationTest { val myInterface = tu.records["MyInterface"] assertNotNull(myInterface) } + + @Test + fun testConst() { + val topLevel = Path.of("src", "test", "resources", "golang") + val tu = + TestUtils.analyzeAndGetFirstTU( + listOf(topLevel.resolve("const.go").toFile()), + topLevel, + true + ) { + it.registerLanguage() + } + assertNotNull(tu) + + val values = + mapOf( + "one" to 1, + "oneAsWell" to 1, + "two" to 2, + "three" to 3, + "four" to 4, + "tenAsWell" to 10, + "five" to 5, + "fiveAsWell" to 5, + "six" to 6, + "fivehundred" to 500, + "sixhundred" to 600, + "onehundredandfive" to 105 + ) + values.forEach { + val variable = tu.variables[it.key] + assertNotNull(variable, "variable \"${it.key}\" not found") + assertEquals(it.value, variable.evaluate(), "${it.key} does not match") + } + } + + @Test + fun testImportAlias() { + val stdLib = Path.of("src", "test", "resources", "golang-std") + val topLevel = Path.of("src", "test", "resources", "golang") + val result = + TestUtils.analyze( + listOf( + topLevel.resolve("importalias.go").toFile(), + stdLib.resolve("fmt").toFile(), + ), + topLevel, + true + ) { + it.registerLanguage() + it.includePath(stdLib) + } + assertNotNull(result) + + val printf = result.functions["fmt.Printf"] + assertNotNull(printf) + + val callPrintf = result.calls["fmtother.Printf"] + assertNotNull(callPrintf) + assertInvokes(callPrintf, printf) + + val expr = result.allChildren().firstOrNull() + assertNotNull(expr) + + val fmt = result.variables["fmt"] + assertNotNull(fmt) + + val base = expr.base + assertIs(base) + assertRefersTo(base, fmt) + } + + @Test + fun testFuncOptions() { + val topLevel = Path.of("src", "test", "resources", "golang", "options") + val result = + TestUtils.analyze( + listOf( + topLevel.resolve("srv_option.go").toFile(), + topLevel.resolve("srv.go").toFile(), + ), + topLevel, + true + ) { + it.registerLanguage() + } + assertNotNull(result) + + val inner = result.records["inner"] + assertNotNull(inner) + + val field = inner.fields["field"] + assertNotNull(field) + + val assign = result.assignments.firstOrNull() + assertNotNull(assign) + + val mce = assign.target + assertNotNull(mce) + assertIs(mce) + assertRefersTo(mce, field) + } } diff --git a/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/ExpressionTest.kt b/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/ExpressionTest.kt index 7921cec0475..0aee2c2de48 100644 --- a/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/ExpressionTest.kt +++ b/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/ExpressionTest.kt @@ -26,6 +26,7 @@ package de.fraunhofer.aisec.cpg.frontends.golang import de.fraunhofer.aisec.cpg.TestUtils +import de.fraunhofer.aisec.cpg.TestUtils.assertRefersTo import de.fraunhofer.aisec.cpg.assertFullName import de.fraunhofer.aisec.cpg.assertLiteralValue import de.fraunhofer.aisec.cpg.assertLocalName @@ -100,7 +101,7 @@ class ExpressionTest { // [:1] var slice = assertIs( - assertIs(b.initializer).subscriptExpression + assertIs(b.firstAssignment).subscriptExpression ) assertNull(slice.floor) assertLiteralValue(1, slice.ceiling) @@ -111,7 +112,8 @@ class ExpressionTest { assertLocalName("int[]", c.type) // [1:] - slice = assertIs(assertIs(c.initializer).subscriptExpression) + slice = + assertIs(assertIs(c.firstAssignment).subscriptExpression) assertLiteralValue(1, slice.floor) assertNull(slice.ceiling) assertNull(slice.third) @@ -121,7 +123,8 @@ class ExpressionTest { assertLocalName("int[]", d.type) // [0:1] - slice = assertIs(assertIs(d.initializer).subscriptExpression) + slice = + assertIs(assertIs(d.firstAssignment).subscriptExpression) assertLiteralValue(0, slice.floor) assertLiteralValue(1, slice.ceiling) assertNull(slice.third) @@ -131,9 +134,46 @@ class ExpressionTest { assertLocalName("int[]", e.type) // [0:1:1] - slice = assertIs(assertIs(e.initializer).subscriptExpression) + slice = + assertIs(assertIs(e.firstAssignment).subscriptExpression) assertLiteralValue(0, slice.floor) assertLiteralValue(1, slice.ceiling) assertLiteralValue(1, slice.third) } + + @Test + fun testSendStmt() { + val topLevel = Path.of("src", "test", "resources", "golang") + val tu = + TestUtils.analyzeAndGetFirstTU( + listOf(topLevel.resolve("chan.go").toFile()), + topLevel, + true + ) { + it.registerLanguage() + } + assertNotNull(tu) + + with(tu) { + val main = tu.functions["main"] + assertNotNull(main) + + val v = main.variables["v"] + assertNotNull(v) + assertEquals(primitiveType("int"), v.type) + + val ch = main.variables["ch"] + assertNotNull(ch) + assertEquals(objectType("chan", generics = listOf(primitiveType("int"))), ch.type) + + val binOp = main.bodyOrNull() + assertNotNull(binOp) + assertRefersTo(binOp.lhs, ch) + assertRefersTo(binOp.rhs, v) + + val unaryOp = main.bodyOrNull() + assertNotNull(unaryOp) + assertRefersTo(unaryOp.input, ch) + } + } } diff --git a/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontendTest.kt b/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontendTest.kt index 37776c62a3b..29a21698ad4 100644 --- a/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontendTest.kt +++ b/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontendTest.kt @@ -28,7 +28,10 @@ package de.fraunhofer.aisec.cpg.frontends.golang import de.fraunhofer.aisec.cpg.BaseTest import de.fraunhofer.aisec.cpg.TestUtils.analyze import de.fraunhofer.aisec.cpg.TestUtils.analyzeAndGetFirstTU +import de.fraunhofer.aisec.cpg.TestUtils.assertInvokes +import de.fraunhofer.aisec.cpg.TestUtils.assertRefersTo import de.fraunhofer.aisec.cpg.assertFullName +import de.fraunhofer.aisec.cpg.assertLiteralValue import de.fraunhofer.aisec.cpg.assertLocalName import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration @@ -38,6 +41,9 @@ import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration import de.fraunhofer.aisec.cpg.graph.statements.* import de.fraunhofer.aisec.cpg.graph.statements.expressions.* import de.fraunhofer.aisec.cpg.graph.types.FunctionType +import de.fraunhofer.aisec.cpg.graph.types.ObjectType +import de.fraunhofer.aisec.cpg.graph.types.PointerType +import java.io.File import java.nio.file.Path import kotlin.test.* @@ -61,16 +67,13 @@ class GoLanguageFrontendTest : BaseTest() { val message = main.variables["message"] assertNotNull(message) - val map = - assertIs( - assertIs(message.firstAssignment).arguments.firstOrNull() - ) + val map = assertIs(message.firstAssignment) assertNotNull(map) val nameEntry = map.initializers.firstOrNull() as? KeyValueExpression assertNotNull(nameEntry) - assertLocalName("string[]", (nameEntry.value as? ConstructExpression)?.type) + assertLocalName("string[]", (nameEntry.value as? InitializerListExpression)?.type) } @Test @@ -96,7 +99,7 @@ class GoLanguageFrontendTest : BaseTest() { val path = data.firstAssignment?.followPrevDFG { it is KeyValueExpression } assertNotNull(path) - assertEquals(3, path.size) + assertEquals(2, path.size) } @Test @@ -178,11 +181,20 @@ class GoLanguageFrontendTest : BaseTest() { @Test fun testLiteral() { val topLevel = Path.of("src", "test", "resources", "golang") - val tu = - analyzeAndGetFirstTU(listOf(topLevel.resolve("literal.go").toFile()), topLevel, true) { + val result = + analyze( + listOf( + topLevel.resolve("literal.go").toFile(), + topLevel.resolve("submodule/const.go").toFile(), + ), + topLevel, + true + ) { it.registerLanguage() } + assertNotNull(result) + val tu = result.components["application"]?.translationUnits?.firstOrNull() assertNotNull(tu) val p = tu.namespaces["p"] @@ -230,6 +242,82 @@ class GoLanguageFrontendTest : BaseTest() { assertFullName("", func) assertEquals(1, func.parameters.size) assertEquals(1, func.returnTypes.size) + + val o = p.variables["o"] + assertNotNull(o) + assertLocalName("MyStruct[]", o.type) + + val myStruct = tu.records["MyStruct"] + assertNotNull(myStruct) + + val field = myStruct.fields["Field"] + assertNotNull(field) + + var composite = + (o.initializer as? InitializerListExpression)?.initializers( + 0 + ) + assertNotNull(composite) + assertIs(composite) + assertIs(composite.type) + assertLocalName("MyStruct", composite.type) + + var keyValue = composite.initializers(0) + assertNotNull(keyValue) + assertLocalName("Field", keyValue.key) + assertRefersTo(keyValue.key, field) + assertLiteralValue(10, keyValue.value) + + val o3 = p.variables["o3"] + assertNotNull(o3) + assertLocalName("MyStruct[]", o.type) + + composite = + (o.initializer as? InitializerListExpression)?.initializers( + 0 + ) + assertNotNull(composite) + assertIs(composite) + assertIs(composite.type) + assertLocalName("MyStruct", composite.type) + + keyValue = composite.initializers(0) + assertNotNull(keyValue) + assertLocalName("Field", keyValue.key) + assertRefersTo(keyValue.key, field) + assertLiteralValue(10, keyValue.value) + + val rr = tu.variables["rr"] + assertNotNull(rr) + + var init = rr.initializer + assertIs(init) + + keyValue = init.initializers(0) + assertNotNull(keyValue) + + var key = keyValue.key + assertNotNull(key) + + var zero = result.variables["submodule.Zero"] + assertNotNull(zero) + assertRefersTo(key, zero) + + val mapr = tu.variables["mapr"] + assertNotNull(mapr) + + init = mapr.initializer + assertIs(init) + + keyValue = init.initializers(0) + assertNotNull(keyValue) + + key = keyValue.key + assertNotNull(key) + + zero = result.variables["submodule.Zero"] + assertNotNull(zero) + assertRefersTo(key, zero) } @Test @@ -248,22 +336,24 @@ class GoLanguageFrontendTest : BaseTest() { val main = p.functions["main"] assertNotNull(main) - var type = main.type as? FunctionType + var type = main.type + assertIs(type) assertNotNull(type) assertLocalName("func()", type) assertEquals(0, type.parameters.size) assertEquals(0, type.returnTypes.size) - val myTest = p.functions["myTest"] - assertNotNull(myTest) - assertEquals(1, myTest.parameters.size) - assertEquals(2, myTest.returnTypes.size) + val funcA = p.functions["funcA"] + assertNotNull(funcA) + assertEquals(1, funcA.parameters.size) + assertEquals(2, funcA.returnTypes.size) - type = myTest.type as? FunctionType + type = funcA.type + assertIs(type) assertNotNull(type) assertLocalName("func(string) (int, error)", type) - assertEquals(myTest.parameters.size, type.parameters.size) - assertEquals(myTest.returnTypes.size, type.returnTypes.size) + assertEquals(funcA.parameters.size, type.parameters.size) + assertEquals(funcA.returnTypes.size, type.returnTypes.size) assertEquals(listOf("int", "error"), type.returnTypes.map { it.name.localName }) var body = main.body as? CompoundStatement @@ -272,17 +362,17 @@ class GoLanguageFrontendTest : BaseTest() { var callExpression = body.calls.firstOrNull() assertNotNull(callExpression) - assertLocalName("myTest", callExpression) - assertEquals(myTest, callExpression.invokes.iterator().next()) + assertLocalName("funcA", callExpression) + assertEquals(funcA, callExpression.invokes.iterator().next()) - val s = myTest.parameters.first() + val s = funcA.parameters.first() assertNotNull(s) assertLocalName("s", s) assertEquals(tu.primitiveType("string"), s.type) - assertLocalName("myTest", myTest) + assertLocalName("funcA", funcA) - body = myTest.body as? CompoundStatement + body = funcA.body as? CompoundStatement assertNotNull(body) callExpression = body.statements.first() as? CallExpression @@ -329,70 +419,34 @@ class GoLanguageFrontendTest : BaseTest() { assertNotNull(err) assertLocalName("error", err.type) - } - - @Test - fun testStruct() { - val topLevel = Path.of("src", "test", "resources", "golang") - val tu = - analyzeAndGetFirstTU(listOf(topLevel.resolve("struct.go").toFile()), topLevel, true) { - it.registerLanguage() - } - - assertNotNull(tu) - - val p = tu.getDeclarationsByName("p", NamespaceDeclaration::class.java).iterator().next() - - val myStruct = - p.getDeclarationsByName("p.MyStruct", RecordDeclaration::class.java).iterator().next() - - assertNotNull(myStruct) - assertEquals("struct", myStruct.kind) - - val fields = myStruct.fields - - assertEquals(1, fields.size) - - var methods = myStruct.methods - - var myFunc = methods.firstOrNull() - assertNotNull(myFunc) - - assertLocalName("MyFunc", myFunc) - - val myField = fields.firstOrNull() - assertNotNull(myField) - - assertLocalName("MyField", myField) - assertEquals(tu.primitiveType("int"), myField.type) - - val myInterface = p.records["p.MyInterface"] - assertNotNull(myInterface) - assertEquals("interface", myInterface.kind) - - methods = myInterface.methods - - assertEquals(1, methods.size) - - myFunc = methods.first() - - assertLocalName("MyFunc", myFunc) - assertLocalName("func() string", myFunc.type) - - val newMyStruct = p.functions["NewMyStruct"] - assertNotNull(newMyStruct) - - val body = newMyStruct.body as? CompoundStatement - - assertNotNull(body) - val `return` = body.statements.first() as? ReturnStatement + val funcB = tu.functions["funcB"] + assertNotNull(funcB) + assertEquals(3, funcB.parameters.size) + assertEquals( + listOf("uint8[]", "uint8[]", "int"), + funcB.parameters.map { it.type.name.toString() } + ) - assertNotNull(`return`) + type = funcB.type + assertIs(type) + assertNotNull(type) + assertEquals(3, type.parameters.size) + assertEquals( + listOf("uint8[]", "uint8[]", "int"), + type.parameters.map { it.name.toString() } + ) - val returnValue = `return`.returnValue as? UnaryOperator + val funcC = tu.functions["funcC"] + assertNotNull(funcC) + assertEquals(1, funcC.parameters.size) + assertEquals(listOf("string"), funcC.parameters.map { it.type.name.toString() }) - assertNotNull(returnValue) + type = funcC.type + assertIs(type) + assertNotNull(type) + assertEquals(1, type.parameters.size) + assertEquals(listOf("string"), type.parameters.map { it.name.toString() }) } @Test @@ -491,12 +545,10 @@ class GoLanguageFrontendTest : BaseTest() { assertLocalName("b", b) assertEquals(tu.primitiveType("bool"), b.type) - // true, false are builtin variables, NOT literals in Golang - // we might need to parse this special case differently - val initializer = b.initializer as? DeclaredReferenceExpression - - assertNotNull(initializer) - assertLocalName("true", initializer) + // Technically, "true" and "false" are builtin variables, NOT literals in Golang, + // however, we parse them as literals to have compatibility with other languages + // also enable all features, such as value resolution based on literal values. + assertLiteralValue(true, b.initializer) val `if` = body.statements[1] as? IfStatement @@ -572,7 +624,10 @@ class GoLanguageFrontendTest : BaseTest() { } assertNotNull(result) - val tus = result.translationUnits + val app = result.components["application"] + assertNotNull(app) + + val tus = app.translationUnits val tu = tus[0] val p = tu.namespaces["p"] @@ -670,9 +725,10 @@ class GoLanguageFrontendTest : BaseTest() { it.registerLanguage() } - assertNotNull(result) - val tus = result.translationUnits + val app = result.components["application"] + assertNotNull(app) + val tus = app.translationUnits val tu0 = tus[0] assertNotNull(tu0) @@ -713,7 +769,6 @@ class GoLanguageFrontendTest : BaseTest() { analyzeAndGetFirstTU(listOf(topLevel.resolve("comment.go").toFile()), topLevel, true) { it.registerLanguage() } - assertNotNull(tu) val mainNamespace = tu.namespaces["main"] @@ -787,6 +842,163 @@ class GoLanguageFrontendTest : BaseTest() { assertNotNull(assign) val call = assertIs(assign.value) - assertLocalName("myTest", call) + assertLocalName("funcA", call) + } + + @Test + fun testTypes() { + val topLevel = Path.of("src", "test", "resources", "golang") + val tu = + analyzeAndGetFirstTU(listOf(topLevel.resolve("types.go").toFile()), topLevel, true) { + it.registerLanguage() + } + assertNotNull(tu) + + val s = tu.variables["s"] + assertNotNull(s) + assertIs(s.type) + assertLocalName("struct{Field int}", s.type) + + val i = tu.variables["i"] + assertNotNull(i) + assertIs(i.type) + assertLocalName("interface{MyMethod(int) error}", i.type) + + val m = tu.variables["m"] + assertNotNull(m) + var type = m.type + assertIs(type) + assertEquals(listOf("int", "string"), type.generics.map { it.name.toString() }) + + val a = tu.variables["a"] + assertNotNull(a) + type = a.type + assertIs(type) + assertEquals(PointerType.PointerOrigin.ARRAY, type.pointerOrigin) + assertLocalName("int", type.elementType) + + val f = tu.variables["f"] + assertNotNull(f) + assertIs(f.type) + + val g = tu.variables["g"] + assertNotNull(g) + assertLocalName("string", g.type) + + val h = tu.variables["h"] + assertNotNull(h) + + type = h.type + assertIs(type) + assertLocalName("newType", type) + + assertEquals(1, type.recordDeclaration?.methods?.size) + } + + @Test + fun testResolveStdLibImport() { + val stdLib = Path.of("src", "test", "resources", "golang-std") + val topLevel = Path.of("src", "test", "resources", "golang") + val tu = + analyzeAndGetFirstTU( + listOf( + topLevel.resolve("function.go").toFile(), + stdLib.resolve("fmt").toFile(), + ), + topLevel, + true + ) { + it.registerLanguage() + it.includePath(stdLib) + } + + assertNotNull(tu) + + val p = tu.namespaces["p"] + assertNotNull(p) + + val main = p.functions["main"] + assertNotNull(main) + + val printfCall = main.calls["fmt.Printf"] + assertNotNull(printfCall) + + val printf = printfCall.invokes.firstOrNull() + assertNotNull(printf) + assertEquals("print.go", File(printf.location?.artifactLocation?.uri?.path.toString()).name) + } + + @Test + fun testBuildTags() { + val stdLib = Path.of("src", "test", "resources", "golang-std") + val topLevel = Path.of("src", "test", "resources", "golang", "buildtags") + val result = + analyze( + listOf( + topLevel.resolve("").toFile(), + stdLib.resolve("fmt").toFile(), + ), + topLevel, + true + ) { + it.registerLanguage() + it.includePath(stdLib) + it.symbols(mapOf("GOOS" to "darwin", "GOARCH" to "arm64")) + } + + assertNotNull(result) + + val funcOS = result.functions("OS") + assertEquals(1, funcOS.size) + + val specific = result.functions["someSpecific"] + assertNotNull(specific) + } + + @Test + fun testInterfaceDeriveFrom() { + val topLevel = Path.of("src", "test", "resources", "golang") + val tu = + analyzeAndGetFirstTU(listOf(topLevel.resolve("struct.go").toFile()), topLevel, true) { + it.registerLanguage() + } + assertNotNull(tu) + + val doInterface = tu.functions["DoInterface"] + assertNotNull(doInterface) + + val call = tu.calls["DoInterface"] + assertNotNull(call) + assertInvokes(call, doInterface) + } + + @Test + fun testFuncOptions() { + val topLevel = Path.of("src", "test", "resources", "golang", "options") + val result = + analyze( + listOf( + topLevel.resolve("srv.go").toFile(), + topLevel.resolve("srv_option.go").toFile() + ), + topLevel, + true + ) { + it.registerLanguage() + } + assertNotNull(result) + + val inner = result.records["inner"] + assertNotNull(inner) + + val field = inner.fields["field"] + assertNotNull(field) + + val assign = result.assignments.firstOrNull() + assertNotNull(assign) + + val mce = assign.target + assertIs(mce) + assertRefersTo(mce, field) } } diff --git a/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/StatementTest.kt b/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/StatementTest.kt index 3e698bc9054..bd232a92016 100644 --- a/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/StatementTest.kt +++ b/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/StatementTest.kt @@ -26,6 +26,7 @@ package de.fraunhofer.aisec.cpg.frontends.golang import de.fraunhofer.aisec.cpg.TestUtils.analyzeAndGetFirstTU +import de.fraunhofer.aisec.cpg.assertLocalName import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.statements.* import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression @@ -110,4 +111,43 @@ class StatementTest { // Its call expression should connect to the return statement op.input.prevEOG.all { it is ReturnStatement } } + + @Test + fun testTypeSwitch() { + val topLevel = Path.of("src", "test", "resources", "golang") + val tu = + analyzeAndGetFirstTU( + listOf(topLevel.resolve("type_assert.go").toFile()), + topLevel, + true + ) { + it.registerLanguage() + } + assertNotNull(tu) + + val main = tu.functions["main"] + assertNotNull(main) + + val body = main.body + assertIs(body) + assertNotNull(body) + + val switch = body.statements(6) + assertNotNull(switch) + + val block = switch.statement + assertIs(block) + assertNotNull(block) + + val vs = main.variables("v") + assertNotNull(vs) + assertEquals( + listOf("main.MyStruct", "main.MyStruct*", "main.MyInterface"), + vs.map { it.type.name.toString() } + ) + vs.forEach { + assertLocalName("v", it) + assertLocalName("f", it.initializer) + } + } } diff --git a/cpg-language-go/src/test/resources/golang-std/fmt/print.go b/cpg-language-go/src/test/resources/golang-std/fmt/print.go new file mode 100644 index 00000000000..b9b025641d6 --- /dev/null +++ b/cpg-language-go/src/test/resources/golang-std/fmt/print.go @@ -0,0 +1,6 @@ +package fmt + +func Printf(format string, a ...any) (n int, err error) { + // Not a real implementation, and we are ignoring it anyway + return 0, nil +} diff --git a/cpg-language-go/src/test/resources/golang/buildtags/func_darwin.go b/cpg-language-go/src/test/resources/golang/buildtags/func_darwin.go new file mode 100644 index 00000000000..4fa975686fd --- /dev/null +++ b/cpg-language-go/src/test/resources/golang/buildtags/func_darwin.go @@ -0,0 +1,10 @@ +// Some other header + +//go:build darwin + +// Package buildtags is awesome +package buildtags + +func OS() string { + return "darwin" +} diff --git a/cpg-language-go/src/test/resources/golang/buildtags/func_darwin_arm64.go b/cpg-language-go/src/test/resources/golang/buildtags/func_darwin_arm64.go new file mode 100644 index 00000000000..fb81f4123bc --- /dev/null +++ b/cpg-language-go/src/test/resources/golang/buildtags/func_darwin_arm64.go @@ -0,0 +1,3 @@ +package buildtags + +func someSpecific() {} diff --git a/cpg-language-go/src/test/resources/golang/buildtags/func_ios.go b/cpg-language-go/src/test/resources/golang/buildtags/func_ios.go new file mode 100644 index 00000000000..96a511d32e1 --- /dev/null +++ b/cpg-language-go/src/test/resources/golang/buildtags/func_ios.go @@ -0,0 +1,8 @@ +// Some other header + +// Package buildtags is awesome +package buildtags + +func OS() string { + return "ios" +} diff --git a/cpg-language-go/src/test/resources/golang/buildtags/func_linux_arm64.go b/cpg-language-go/src/test/resources/golang/buildtags/func_linux_arm64.go new file mode 100644 index 00000000000..93ed03405d9 --- /dev/null +++ b/cpg-language-go/src/test/resources/golang/buildtags/func_linux_arm64.go @@ -0,0 +1,5 @@ +package buildtags + +func OS() string { + return "linux" +} diff --git a/cpg-language-go/src/test/resources/golang/chan.go b/cpg-language-go/src/test/resources/golang/chan.go new file mode 100644 index 00000000000..6ea215ff5c1 --- /dev/null +++ b/cpg-language-go/src/test/resources/golang/chan.go @@ -0,0 +1,8 @@ +package pa + +func main() { + var v int + var ch = make(chan int) + ch <- v + <-ch +} diff --git a/cpg-language-go/src/test/resources/golang/const.go b/cpg-language-go/src/test/resources/golang/const.go new file mode 100644 index 00000000000..3bfe9256401 --- /dev/null +++ b/cpg-language-go/src/test/resources/golang/const.go @@ -0,0 +1,24 @@ +package p + +const ( + one = 1 + oneAsWell + ten = 10 + tenAsWell +) + +const ( + two = 2 + iota + three + four +) + +const ( + five, fivehundred = 5 + iota, (5 + iota) * 100 + six, sixhundred +) + +const ( + fiveAsWell = 5 + iota*100 + onehundredandfive +) diff --git a/cpg-language-go/src/test/resources/golang/declare.go b/cpg-language-go/src/test/resources/golang/declare.go index 15da35ffa9a..238f27a7faf 100644 --- a/cpg-language-go/src/test/resources/golang/declare.go +++ b/cpg-language-go/src/test/resources/golang/declare.go @@ -23,24 +23,21 @@ func main() { // VariableDeclaration nodes var c, d = 3, 4 + var e, f = test() + // Short assignment using an assignment, where all variables were not // defined before. This is an AssignStmt which has DEFINE as its token. - // - // We need to split this up into several nodes. First, we translate this - // into one (implicit) DeclarationStatement with two VariableDeclaration - // nodes. Afterwards we are parsing it as a regular assignment. - e, f := 5, 6 + g, h := 5, 6 // Short assignment using an assignment, where one variable (f) was defined // before in the local scope. This is an AssignStmt which has DEFINE as its // token. From the AST we cannot differentiate this from the previous // example and we need to do a (local) variable lookup here. - // - // Finally, We need to split this up into several nodes. First, we translate - // this into one (implicit) DeclarationStatement with one - // VariableDeclaration node. Afterwards we are parsing it as a regular - // assignment. - f, g := 7, 8 + h, i := test() + + fmt.Printf("%d %d %d %d %d %d %d\n", a, b, c, d, e, f, g, h, i) +} - fmt.Printf("%d %d %d %d %d %d %d\n", a, b, c, d, e, f, g) +func test() (i int, err error) { + return 0, nil } diff --git a/cpg-language-go/src/test/resources/golang/function.go b/cpg-language-go/src/test/resources/golang/function.go index e8984d0a400..e2c5ef52cce 100644 --- a/cpg-language-go/src/test/resources/golang/function.go +++ b/cpg-language-go/src/test/resources/golang/function.go @@ -6,14 +6,14 @@ func main() { var i int var err error - i, err = myTest("some string") + i, err = funcA("some string") if err == nil { fmt.Printf("%d", i) } } -func myTest(s string) (a int, err error) { +func funcA(s string) (a int, err error) { fmt.Printf("%s", s) a = 1 + 2 @@ -22,3 +22,13 @@ func myTest(s string) (a int, err error) { return } + +// funcB is a function that show-cases different ways of declaring used and unused +// parameters in Go. +func funcB(a, b []byte, _ int) (byte, byte) { + return a[0], b[0] +} + +func funcC(string) { + return +} diff --git a/cpg-language-go/src/test/resources/golang/go.mod b/cpg-language-go/src/test/resources/golang/go.mod index 12cb5307fb5..07004656da2 100644 --- a/cpg-language-go/src/test/resources/golang/go.mod +++ b/cpg-language-go/src/test/resources/golang/go.mod @@ -1 +1,3 @@ module mymodule + +go 1.18 diff --git a/cpg-language-go/src/test/resources/golang/importalias.go b/cpg-language-go/src/test/resources/golang/importalias.go new file mode 100644 index 00000000000..750ff62022f --- /dev/null +++ b/cpg-language-go/src/test/resources/golang/importalias.go @@ -0,0 +1,16 @@ +package p + +import ( + fmtother "fmt" +) + +type formatter struct { + field int +} + +func main() { + fmt := formatter{} + fmt.field = 1 + + fmtother.Printf("%d", 1) +} diff --git a/cpg-language-go/src/test/resources/golang/literal.go b/cpg-language-go/src/test/resources/golang/literal.go index 4b7207b8c0c..dbefc5bda1b 100644 --- a/cpg-language-go/src/test/resources/golang/literal.go +++ b/cpg-language-go/src/test/resources/golang/literal.go @@ -1,6 +1,19 @@ package p +import ( + "mymodule/submodule" +) + +type MyStruct struct { + Field int +} + const a = 1 + +const ( + b int = iota +) + const s = "test" const f = 1.0 const f32 float32 = 1.00 @@ -10,3 +23,23 @@ var n *int = nil var fn = func(_ int) int { return 1 } + +type structArray []MyStruct + +// o is a composite literal. It also demonstrates that we can omit the +// type specifier of an "inner" composite literal, if the outer one is an array type +var o = []MyStruct{{Field: 10}} + +// o is similar to o2, but with a pointer type +var o2 = []*MyStruct{{Field: 10}} + +// o3 is similar to o3, but with a new type that uses []MyStruct as underlying type +var o3 = structArray{{Field: 10}} + +var rr = []int{ + submodule.Zero: 1, +} + +var mapr = map[int][]byte{ + submodule.Zero: {1, 2, 3}, +} diff --git a/cpg-language-go/src/test/resources/golang/options/srv.go b/cpg-language-go/src/test/resources/golang/options/srv.go new file mode 100644 index 00000000000..8971b5e6cb8 --- /dev/null +++ b/cpg-language-go/src/test/resources/golang/options/srv.go @@ -0,0 +1,9 @@ +package options + +type srv struct { + inner *inner +} + +type inner struct { + field int +} diff --git a/cpg-language-go/src/test/resources/golang/options/srv_option.go b/cpg-language-go/src/test/resources/golang/options/srv_option.go new file mode 100644 index 00000000000..d0a7360853b --- /dev/null +++ b/cpg-language-go/src/test/resources/golang/options/srv_option.go @@ -0,0 +1,9 @@ +package options + +type Option func(*srv) + +func WithField(a int) Option { + return func(s *srv) { + s.inner.field = a + } +} diff --git a/cpg-language-go/src/test/resources/golang/struct.go b/cpg-language-go/src/test/resources/golang/struct.go index 7287159ce3d..18e466eea84 100644 --- a/cpg-language-go/src/test/resources/golang/struct.go +++ b/cpg-language-go/src/test/resources/golang/struct.go @@ -1,9 +1,19 @@ package p -import ("fmt") +import ( + "fmt" +) + +type OtherStruct struct { +} + +type EvenAnotherStruct struct { +} type MyStruct struct { - MyField int + MyField int + OtherStruct + *EvenAnotherStruct } type MyInterface interface { @@ -11,7 +21,7 @@ type MyInterface interface { } func (s MyStruct) MyFunc() string { - fmt.Printf(s.myOtherFunc(), s.MyField) + fmt.Printf(s.myOtherFunc(), s.MyField) return "s" } @@ -21,5 +31,20 @@ func (s MyStruct) myOtherFunc() string { } func NewMyStruct() *MyStruct { - return &MyStruct{} + return &MyStruct{} +} + +var s = struct { + field int +}{ + field: 1, +} + +func DoInterface(i MyInterface) string { + return i.MyFunc() +} + +func main() { + var myStruct = NewMyStruct() + DoInterface(myStruct) } diff --git a/cpg-language-go/src/test/resources/golang/submodule/const.go b/cpg-language-go/src/test/resources/golang/submodule/const.go new file mode 100644 index 00000000000..9d6d1ade54e --- /dev/null +++ b/cpg-language-go/src/test/resources/golang/submodule/const.go @@ -0,0 +1,3 @@ +package submodule + +const Zero = 0 diff --git a/cpg-language-go/src/test/resources/golang/type_assert.go b/cpg-language-go/src/test/resources/golang/type_assert.go index 4f42ac27989..4d25a380ea4 100644 --- a/cpg-language-go/src/test/resources/golang/type_assert.go +++ b/cpg-language-go/src/test/resources/golang/type_assert.go @@ -2,19 +2,37 @@ package main import "fmt" -type MyStruct struct {} +type MyStruct struct{} + type MyInterface interface { - MyFunc() + MyFunc() } + func (MyStruct) MyFunc() {} -func main () { - var f MyInterface = MyStruct{} - var s = f.(MyStruct) +func main() { + var f MyInterface = MyStruct{} + var s = f.(MyStruct) + + fmt.Printf("%+v", s) + + var _ = MyInterface(s) + var _ = interface{}(s) + var _ = any(s) + + switch v := f.(type) { + case MyStruct: + var s2 = v + fmt.Printf("%+v", s2) + case *MyStruct: + var p2 = v + fmt.Printf("%+v", p2) + default: + var v2 = v + fmt.Printf("%+v", v2) + } +} - fmt.Printf("%+v", s) +type myStruct struct{} - var _ = MyInterface(s) - var _ = interface{}(s) - var _ = any(s) -} \ No newline at end of file +var test = (*myStruct)(nil) diff --git a/cpg-language-go/src/test/resources/golang/type_constraints.go b/cpg-language-go/src/test/resources/golang/type_constraints.go index 485f29bc2ea..b4c92870e7e 100644 --- a/cpg-language-go/src/test/resources/golang/type_constraints.go +++ b/cpg-language-go/src/test/resources/golang/type_constraints.go @@ -1,11 +1,11 @@ package main -type MyStruct[T any] struct {} -type MyInterface interface {} +type MyStruct[T any] struct{} +type MyInterface interface{} func SomeFunc[T any, S MyInterface]() {} func main() { - _ := &MyStruct[MyInterface]{} - SomeFunc[any, MyInterface]() -} \ No newline at end of file + _ := &MyStruct[MyInterface]{} + SomeFunc[interface{}, MyInterface]() +} diff --git a/cpg-language-go/src/test/resources/golang/types.go b/cpg-language-go/src/test/resources/golang/types.go new file mode 100644 index 00000000000..cb9f5b13d3d --- /dev/null +++ b/cpg-language-go/src/test/resources/golang/types.go @@ -0,0 +1,43 @@ +package p + +// Funcy is a function type +type Funcy func() error + +// newType is a new distinct type, it has string as the "underlying" type. +type newType string + +func (n *newType) SomeFunc() {} + +var _ = newType("test") + +// alias is really a type alias +type alias = string + +// s is a variable with a struct type. +var s struct{ Field int } + +// i is a variable with an interface type. +var i interface{ MyMethod(int) error } + +// m is a variable with an map type. +var m map[int]string + +// a is a variable with an array type. +var a []int + +// f is of Funcy type +var f Funcy + +var g alias + +var h newType + +func init() { + _ = s + _ = i + _ = m + _ = a + _ = f + _ = g + _ = h +}