From 7d2b175f2c443ac778a6872ced685d2bcf1d4bbf Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Wed, 8 Mar 2023 12:32:50 +0100 Subject: [PATCH] Multiple assignments in Go --- .../aisec/cpg/graph/TypeManager.java | 6 +- .../aisec/cpg/graph/types/FunctionType.kt | 6 +- .../de/fraunhofer/aisec/cpg/ScopeManager.kt | 60 +- .../aisec/cpg/frontends/cpp/CPPLanguage.kt | 3 +- .../aisec/cpg/graph/DeclarationBuilder.kt | 5 +- .../aisec/cpg/graph/ExpressionBuilder.kt | 19 + .../aisec/cpg/graph/StatementHolder.kt | 12 + .../declarations/NamespaceDeclaration.kt | 6 + .../graph/declarations/VariableDeclaration.kt | 5 + .../ArraySubscriptionExpression.kt | 16 +- .../expressions/AssignExpression.kt | 4 +- .../statements/expressions/BinaryOperator.kt | 2 +- .../expressions/InitializerListExpression.kt | 16 +- .../aisec/cpg/passes/CallResolver.kt | 29 +- .../aisec/cpg/passes/VariableUsageResolver.kt | 5 +- .../aisec/cpg/passes/inference/Inference.kt | 167 +++-- .../fraunhofer/aisec/cpg/graph/FluentTest.kt | 3 +- .../aisec/cpg/passes/CallResolverTest.kt | 1 + .../de/fraunhofer/aisec/cpg/TestUtils.kt | 17 +- .../src/main/golang/basic_types.go | 13 +- .../src/main/golang/declarations.go | 12 + .../src/main/golang/expressions.go | 54 +- .../golang/frontend/declaration_builder.go | 8 +- .../golang/frontend/expression_builder.go | 22 + .../src/main/golang/frontend/frontend.go | 19 +- .../src/main/golang/frontend/handler.go | 570 ++++++++++++------ .../main/golang/frontend/statement_builder.go | 4 + .../src/main/golang/lib/cpg/main.go | 3 + cpg-language-go/src/main/golang/statements.go | 17 + cpg-language-go/src/main/golang/types.go | 5 +- .../aisec/cpg/frontends/golang/GoLanguage.kt | 4 +- .../frontends/golang/GoLanguageFrontend.kt | 5 +- .../cpg/frontends/golang/DeclarationTest.kt | 57 ++ .../cpg/frontends/golang/ExpressionTest.kt | 83 ++- .../golang/GoLanguageFrontendTest.kt | 199 +++--- .../src/test/resources/golang/call.go | 3 + .../src/test/resources/golang/dfg.go | 2 + .../src/test/resources/golang/for.go | 17 +- .../src/test/resources/golang/function.go | 17 +- .../src/test/resources/golang/go.mod | 2 +- .../src/test/resources/golang/literal.go | 4 + .../src/test/resources/golang/type_assert.go | 4 + cpg-language-go/src/test/resources/log4j2.xml | 2 +- .../aisec/cpg_vis_neo4j/ApplicationTest.kt | 5 +- 44 files changed, 1126 insertions(+), 387 deletions(-) diff --git a/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/graph/TypeManager.java b/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/graph/TypeManager.java index 8b5e2fac85d..bd9284af563 100644 --- a/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/graph/TypeManager.java +++ b/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/graph/TypeManager.java @@ -31,10 +31,7 @@ import de.fraunhofer.aisec.cpg.frontends.Language; import de.fraunhofer.aisec.cpg.frontends.LanguageFrontend; import de.fraunhofer.aisec.cpg.frontends.cpp.CLanguage; -import de.fraunhofer.aisec.cpg.graph.declarations.Declaration; -import de.fraunhofer.aisec.cpg.graph.declarations.RecordDeclaration; -import de.fraunhofer.aisec.cpg.graph.declarations.TemplateDeclaration; -import de.fraunhofer.aisec.cpg.graph.declarations.TypedefDeclaration; +import de.fraunhofer.aisec.cpg.graph.declarations.*; import de.fraunhofer.aisec.cpg.graph.scopes.NameScope; import de.fraunhofer.aisec.cpg.graph.scopes.RecordScope; import de.fraunhofer.aisec.cpg.graph.scopes.Scope; @@ -113,6 +110,7 @@ public ParameterizedType getTypeParameter(RecordDeclaration recordDeclaration, S } } } + return null; } diff --git a/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/graph/types/FunctionType.kt b/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/graph/types/FunctionType.kt index 7cc18683fa9..67525667e75 100644 --- a/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/graph/types/FunctionType.kt +++ b/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/graph/types/FunctionType.kt @@ -57,7 +57,11 @@ class FunctionType : Type { override fun reference(pointer: PointerType.PointerOrigin?): Type { // TODO(oxisto): In the future, we actually could just remove the FunctionPointerType // and just have a regular PointerType here - return FunctionPointerType(parameters.toList(), returnTypes.first(), language) + return FunctionPointerType( + parameters.toList(), + returnTypes.firstOrNull() ?: UnknownType.getUnknownType(), + language + ) } override fun dereference(): Type { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt index cb8991f74be..45ba1fdbe87 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt @@ -441,7 +441,9 @@ class ScopeManager : ScopeProvider { /** This function returns the [Scope] associated with a node. */ fun lookupScope(node: Node): Scope? { - return scopeMap[node] + return if (node is TranslationUnitDeclaration) { + globalScope + } else scopeMap[node] } /** This function looks up scope by its FQN. This only works for [NameScope]s */ @@ -635,13 +637,19 @@ class ScopeManager : ScopeProvider { call: CallExpression, scope: Scope? = currentScope ): List { + val s = extractScope(call, scope) + + return resolve(s) { it.name.lastPartsMatch(call.name) && it.hasSignature(call.signature) } + } + + fun extractScope(node: Node, scope: Scope? = currentScope): Scope? { var s = scope // First, we need to check, whether we have some kind of scoping. - if (call.language != null && call.name.parent != null) { + if (node.name.parent != null) { // extract the scope name, it is usually a name space, but could probably be something // else as well in other languages - val scopeName = call.name.parent + val scopeName = node.name.parent // TODO: proper scope selection @@ -649,18 +657,52 @@ class ScopeManager : ScopeProvider { val scopes = filterScopes { (it is NameScope && it.name == scopeName) } s = if (scopes.isEmpty()) { - LOGGER.error( - "Could not find the scope {} needed to resolve the call {}. Falling back to the current scope", - scopeName, - call.name + Util.errorWithFileLocation( + node, + LOGGER, + "Could not find the scope $scopeName needed to resolve the call ${node.name}. Falling back to the default (current) scope" ) - currentScope + s } else { scopes[0] } } - return resolve(s) { it.name.lastPartsMatch(call.name) && it.hasSignature(call.signature) } + return s + } + + /** + * Directly jumps to a given scope. + * + * Handle with care, here be dragons. Should not be exposed outside of the cpg-core module. + */ + @PleaseBeCareful + internal fun jumpTo(scope: Scope?): Scope? { + val oldScope = currentScope + currentScope = scope + return oldScope + } + + /** + * Directly jumps to the scope a given node defines (if it exists). + * + * Handle with care, here be dragons. Should not be exposed outside of the cpg-core module. + */ + @PleaseBeCareful + internal fun jumpTo(node: Node): Scope? { + return jumpTo(lookupScope(node)) + } + + /** + * This function can be used to wrap multiple statements contained in [init] into the scope of + * [node]. It will execute [enterScope] before calling [init] and [leaveScope] afterwards. + */ + fun withScope(node: T, init: (T.() -> Unit)) { + enterScope(node) + + init(node) + + leaveScope(node) } fun resolveFunctionStopScopeTraversalOnDefinition( diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cpp/CPPLanguage.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cpp/CPPLanguage.kt index 1d8c477cf48..d817ee0d79d 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cpp/CPPLanguage.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cpp/CPPLanguage.kt @@ -48,6 +48,7 @@ class CPPLanguage : HasDefaultArguments, HasTemplates, HasComplexCallResolution, + HasStructs, HasClasses, HasUnknownType { override val fileExtensions = listOf("cpp", "cc", "cxx", "hpp", "hh") @@ -300,7 +301,7 @@ class CPPLanguage : // If we want to use an inferred functionTemplateDeclaration, this needs to be provided. // Otherwise, we could not resolve to a template and no modifications are made val functionTemplateDeclaration = - holder.startInference().createInferredFunctionTemplate(templateCall) + holder.startInference(scopeManager).createInferredFunctionTemplate(templateCall) templateCall.templateInstantiation = functionTemplateDeclaration val edges = templateCall.templateParameterEdges // Set instantiation propertyEdges diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/DeclarationBuilder.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/DeclarationBuilder.kt index 6fc62696c63..8a33128933e 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/DeclarationBuilder.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/DeclarationBuilder.kt @@ -66,10 +66,11 @@ fun MetadataProvider.newTranslationUnitDeclaration( fun MetadataProvider.newFunctionDeclaration( name: CharSequence?, code: String? = null, - rawNode: Any? = null + rawNode: Any? = null, + localNameOnly: Boolean = false ): FunctionDeclaration { val node = FunctionDeclaration() - node.applyMetadata(this, name, rawNode, code) + node.applyMetadata(this, name, rawNode, code, localNameOnly) log(node) return node diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/ExpressionBuilder.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/ExpressionBuilder.kt index 298a02d75ed..2dde9455f1e 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/ExpressionBuilder.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/ExpressionBuilder.kt @@ -30,6 +30,7 @@ import de.fraunhofer.aisec.cpg.frontends.LanguageFrontend import de.fraunhofer.aisec.cpg.graph.Node.Companion.EMPTY_NAME import de.fraunhofer.aisec.cpg.graph.NodeBuilder.log import de.fraunhofer.aisec.cpg.graph.statements.expressions.* +import de.fraunhofer.aisec.cpg.graph.statements.expressions.AssignExpression import de.fraunhofer.aisec.cpg.graph.types.Type import de.fraunhofer.aisec.cpg.graph.types.UnknownType @@ -412,6 +413,24 @@ fun MetadataProvider.newArraySubscriptionExpression( return node } +/** + * Creates a new [SliceExpression]. The [MetadataProvider] receiver will be used to fill different + * meta-data using [Node.applyMetadata]. Calling this extension function outside of Kotlin requires + * an appropriate [MetadataProvider], such as a [LanguageFrontend] as an additional prepended + * argument. + */ +@JvmOverloads +fun MetadataProvider.newSliceExpression( + code: String? = null, + rawNode: Any? = null +): SliceExpression { + val node = SliceExpression() + node.applyMetadata(this, EMPTY_NAME, rawNode, code, true) + + log(node) + return node +} + /** * Creates a new [ArrayCreationExpression]. The [MetadataProvider] receiver will be used to fill * different meta-data using [Node.applyMetadata]. Calling this extension function outside of Kotlin diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/StatementHolder.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/StatementHolder.kt index d2a8a2b0ff1..2ad18b1ea7a 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/StatementHolder.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/StatementHolder.kt @@ -66,6 +66,18 @@ interface StatementHolder : Holder { statementEdges.add(propertyEdge) } + /** Inserts the statement [s] before the statement specified in [before]. */ + fun insertStatementBefore(s: Statement, before: Statement) { + val statements = this.statements + val idx = statements.indexOf(before) + if (idx != -1) { + val before = statements.subList(0, idx) + val after = statements.subList(idx, statements.size) + + this.statements = listOf(*before.toTypedArray(), s, *after.toTypedArray()) + } + } + override operator fun plusAssign(node: Statement) { addStatement(node) } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/NamespaceDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/NamespaceDeclaration.kt index bb0326760fc..9fed1238d12 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/NamespaceDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/NamespaceDeclaration.kt @@ -57,6 +57,12 @@ class NamespaceDeclaration : Declaration(), DeclarationHolder, StatementHolder { @AST override var statementEdges: MutableList> = ArrayList() + /** + * In some languages, there is a relationship between paths / directories and the package + * structure. Therefore, we need to be aware of the path this namespace / package is in. + */ + var path: String? = null + /** * Returns a non-null, possibly empty `Set` of the declaration of a specified type and clazz. * diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/VariableDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/VariableDeclaration.kt index fe207e99569..3b61ae3786b 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/VariableDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/VariableDeclaration.kt @@ -128,6 +128,11 @@ class VariableDeclaration : ValueDeclaration(), HasType.TypeListener, HasInitial .toString() } + override val assignments: List + get() { + return initializer?.let { listOf(Assignment(it, this, this)) } ?: listOf() + } + override fun equals(other: Any?): Boolean { if (this === other) { return true diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ArraySubscriptionExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ArraySubscriptionExpression.kt index 5fd45de90b0..aedab5335c4 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ArraySubscriptionExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ArraySubscriptionExpression.kt @@ -50,8 +50,8 @@ class ArraySubscriptionExpression : Expression(), HasType.TypeListener, HasBase /** * The expression which represents the "subscription" or index on which the array is accessed. - * This can for example be a reference to another variable ([DeclaredReferenceExpression]) or a - * [Literal]. + * This can for example be a reference to another variable ([DeclaredReferenceExpression]), a + * [Literal] or a [SliceExpression]. */ @AST var subscriptExpression: Expression = ProblemExpression("could not parse index expression") @@ -61,8 +61,18 @@ class ArraySubscriptionExpression : Expression(), HasType.TypeListener, HasBase override val operatorCode: String get() = "[]" + /** + * This helper function returns the subscript type of the [arrayType]. We have to differentiate + * here between to types of subscripts: + * * Slices (in the form of a [SliceExpression] return the same type as the array + * * Everything else (for example a [Literal] or any other [Expression] that is being evaluated) + * returns the de-referenced type + */ private fun getSubscriptType(arrayType: Type): Type { - return arrayType.dereference() + return when (subscriptExpression) { + is SliceExpression -> arrayType + else -> arrayType.dereference() + } } override fun typeChanged(src: HasType, root: MutableList, oldType: Type) { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/AssignExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/AssignExpression.kt index 1c462f2bd9f..3938e91ece0 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/AssignExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/AssignExpression.kt @@ -51,9 +51,9 @@ class AssignExpression : Expression(), AssignmentHolder, HasType.TypeListener { var operatorCode: String = "=" - @field:SubGraph("AST") var lhs: List = listOf() + @AST var lhs: List = listOf() - @field:SubGraph("AST") + @AST var rhs: List = listOf() set(value) { // Unregister any old type listeners diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/BinaryOperator.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/BinaryOperator.kt index 4690750ee4d..f7a06a3234c 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/BinaryOperator.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/BinaryOperator.kt @@ -61,7 +61,7 @@ class BinaryOperator : override var operatorCode: String? = null set(value) { field = value - if (value?.contains("=") == true) { + if (compoundOperators.contains(operatorCode) || operatorCode == "=") { NodeBuilder.LOGGER.warn( "Creating a BinaryOperator with an assignment operator code is deprecated. The class AssignExpression should be used instead." ) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/InitializerListExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/InitializerListExpression.kt index 6cba776b9e0..08494653b01 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/InitializerListExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/InitializerListExpression.kt @@ -28,7 +28,6 @@ package de.fraunhofer.aisec.cpg.graph.statements.expressions import de.fraunhofer.aisec.cpg.graph.AST import de.fraunhofer.aisec.cpg.graph.HasType import de.fraunhofer.aisec.cpg.graph.TypeManager -import de.fraunhofer.aisec.cpg.graph.edge.Properties import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge.Companion.propertyEqualsList import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdgeDelegate @@ -60,14 +59,6 @@ class InitializerListExpression : Expression(), HasType.TypeListener { /** Virtual property to access [initializerEdges] without property edges. */ var initializers by PropertyEdgeDelegate(InitializerListExpression::initializerEdges) - fun addInitializer(initializer: Expression) { - val edge = PropertyEdge(this, initializer) - edge.addProperty(Properties.INDEX, initializerEdges.size) - initializer.registerTypeListener(this) - addPrevDFG(initializer) - initializerEdges.add(edge) - } - override fun typeChanged(src: HasType, root: MutableList, oldType: Type) { if (!TypeManager.isTypeSystemActive()) { return @@ -129,5 +120,10 @@ class InitializerListExpression : Expression(), HasType.TypeListener { propertyEqualsList(initializerEdges, other.initializerEdges) } - override fun hashCode() = Objects.hash(super.hashCode(), initializers) + override fun hashCode(): Int { + // Including initializerEdges directly is a HUGE performance loss in the calculation of each + // hash code. Therefore, we only include the array's size, which should hopefully be sort of + // unique to avoid too many hash collisions. + return Objects.hash(super.hashCode(), initializerEdges.size) + } } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/CallResolver.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/CallResolver.kt index 28582650360..59861766a69 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/CallResolver.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/CallResolver.kt @@ -201,7 +201,26 @@ open class CallResolver : SymbolResolverPass() { val suitableBases = getPossibleContainingTypes(call) candidates = if (suitableBases.isEmpty()) { - listOf(currentTU.inferFunction(call)) + // This is not really the most ideal place, but for now this will do. While this + // is definitely a function, it could still be a function inside a namespace. In + // this case, we want to start inference in that particular namespace and not in + // the TU. It is also a little bit redundant, since ScopeManager.resolveFunction + // (which gets called before) already extracts the scope, but this information + // gets lost. + val scope = scopeManager.extractScope(call, scopeManager.globalScope) + + // We have two possible start points, a namespace declaration or a translation + // unit. Nothing else is allowed (fow now) + val func = + when (val start = scope?.astNode) { + is TranslationUnitDeclaration -> + start.inferFunction(call, scopeManager = scopeManager) + is NamespaceDeclaration -> + start.inferFunction(call, scopeManager = scopeManager) + else -> null + } + + listOfNotNull(func) } else { createMethodDummies(suitableBases, call) } @@ -390,13 +409,13 @@ open class CallResolver : SymbolResolverPass() { .mapNotNull { var record = recordMap[it.root.name] if (record == null && config?.inferenceConfiguration?.inferRecords == true) { - record = it.startInference().inferRecordDeclaration(it, currentTU) + record = it.startInference(scopeManager).inferRecordDeclaration(it, currentTU) // update the record map if (record != null) recordMap[it.root.name] = record } record } - .map { record -> record.inferMethod(call) } + .map { record -> record.inferMethod(call, scopeManager = scopeManager) } } /** @@ -597,7 +616,7 @@ open class CallResolver : SymbolResolverPass() { return constructorCandidate ?: recordDeclaration - .startInference() + .startInference(scopeManager) .createInferredConstructor(constructExpression.signature) } @@ -606,7 +625,7 @@ open class CallResolver : SymbolResolverPass() { recordDeclaration: RecordDeclaration ): ConstructorDeclaration { return recordDeclaration.constructors.firstOrNull { it.hasSignature(signature) } - ?: recordDeclaration.startInference().createInferredConstructor(signature) + ?: recordDeclaration.startInference(scopeManager).createInferredConstructor(signature) } companion object { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/VariableUsageResolver.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/VariableUsageResolver.kt index 25ab6b1e2a6..20fece39b60 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/VariableUsageResolver.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/VariableUsageResolver.kt @@ -345,7 +345,8 @@ open class VariableUsageResolver : SymbolResolverPass() { } else { "class" } - val record = base.startInference().inferRecordDeclaration(base, currentTU, kind) + val record = + base.startInference(scopeManager).inferRecordDeclaration(base, currentTU, kind) // update the record map if (record != null) recordMap[base.name] = record } @@ -404,7 +405,7 @@ open class VariableUsageResolver : SymbolResolverPass() { // If we didn't find anything, we create a new function or method declaration return target ?: (declarationHolder ?: currentTU) - .startInference() + .startInference(scopeManager) .createInferredFunctionDeclaration( name, null, diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/Inference.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/Inference.kt index bfd414284af..8f0381f53e5 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/Inference.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/Inference.kt @@ -25,11 +25,13 @@ */ package de.fraunhofer.aisec.cpg.passes.inference +import de.fraunhofer.aisec.cpg.ScopeManager import de.fraunhofer.aisec.cpg.frontends.HasClasses import de.fraunhofer.aisec.cpg.frontends.Language import de.fraunhofer.aisec.cpg.frontends.LanguageFrontend import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.* +import de.fraunhofer.aisec.cpg.graph.scopes.Scope import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression import de.fraunhofer.aisec.cpg.graph.statements.expressions.TypeExpression @@ -48,12 +50,19 @@ import org.slf4j.LoggerFactory * Since this class implements [IsInferredProvider], all nodes that are created using the node * builder functions, will automatically have [Node.isInferred] set to true. */ -class Inference(val start: Node) : LanguageProvider, IsInferredProvider { +class Inference(val start: Node, val scopeManager: ScopeManager) : + LanguageProvider, ScopeProvider, IsInferredProvider { val log: Logger = LoggerFactory.getLogger(Inference::class.java) override val language: Language? get() = start.language + override val isInferred: Boolean + get() = true + + override val scope: Scope? + get() = scopeManager.currentScope + fun createInferredFunctionDeclaration( name: CharSequence?, code: String?, @@ -61,82 +70,101 @@ class Inference(val start: Node) : LanguageProvider, IsInferredProvider { signature: List, returnType: Type?, ): FunctionDeclaration { - // We assume that the start is either a record or the translation unit + // We assume that the start is either a record, a namespace or the translation unit val record = start as? RecordDeclaration + val namespace = start as? NamespaceDeclaration val tu = start as? TranslationUnitDeclaration - // If both are null, we have the wrong type - if (record == null && tu == null) { + // If all are null, we have the wrong type + if (record == null && namespace == null && tu == null) { throw UnsupportedOperationException( "Starting inference with the wrong type of start node" ) } + // Here be dragons. Jump to the scope that the node defines directly, so that we can + // delegate further operations to the scope manager. We also save the old scope so we can + // restore it. + val oldScope = scopeManager.jumpTo(start) + log.debug( "Inferring a new function declaration $name with parameter types ${signature.map { it?.name }}" ) + // "upgrade" our struct to a class, if it was inferred by us, since we are calling + // methods on it. But only if the language supports classes in the first place. if ( record?.isInferred == true && record.kind == "struct" && record.language is HasClasses ) { - // "upgrade" our struct to a class, if it was inferred by us, since we are calling - // methods on it record.kind = "class" } - val declarationHolder = (record ?: tu) - val parameters = createInferredParameters(signature) val inferred: FunctionDeclaration = if (record != null) { newMethodDeclaration(name ?: "", code, isStatic, record) } else { newFunctionDeclaration(name ?: "", code) } - inferred.parameters = parameters - // TODO: Once, we used inferred.type = returnType and once the two following statements: - // Why? What's the "right way"? - returnType?.let { - inferred.returnTypes = listOf(it) - inferred.type = returnType - } + createInferredParameters(inferred, signature) + + // Set the type and return type(s) + returnType?.let { inferred.returnTypes = listOf(it) } + inferred.type = FunctionType.computeType(inferred) + + // Add it to the scope + scopeManager.addDeclaration(inferred) - // TODO: Handle multiple return values? - if (declarationHolder is RecordDeclaration) { - declarationHolder.addMethod(inferred as MethodDeclaration) + // Some magic that adds it to static imports. Not sure if this really needed + if (record != null) { if (isStatic) { - declarationHolder.staticImports.add(inferred) + record.staticImports.add(inferred) } - } else { - declarationHolder?.addDeclaration(inferred) } + // Revert back to the old scope, so that whoever called us can continue safely + scopeManager.jumpTo(oldScope) + return inferred } fun createInferredConstructor(signature: List): ConstructorDeclaration { + // Here be dragons. Jump to the scope that the node defines directly, so that we can + // delegate further operations to the scope manager. We also save the old scope so we can + // restore it. + val oldScope = scopeManager.jumpTo(start) + val inferred = newConstructorDeclaration( start.name.localName, "", start as? RecordDeclaration, ) - inferred.parameters = createInferredParameters(signature) + createInferredParameters(inferred, signature) + + scopeManager.addDeclaration(inferred) + + // Revert back to the old scope, so that whoever called us can continue safely + scopeManager.jumpTo(oldScope) - (start as? RecordDeclaration)?.addConstructor(inferred) return inferred } - fun createInferredParameters(signature: List): List { - val params: MutableList = ArrayList() - for (i in signature.indices) { - val targetType = signature[i] - val paramName = generateParamName(i, targetType!!) - val param = newParamVariableDeclaration(paramName, targetType, false, "") - param.argumentIndex = i - params.add(param) + private fun createInferredParameters(function: FunctionDeclaration, signature: List) { + // To save some unnecessary scopes, we only want to "enter" the function if it is necessary, + // e.g., if we need to create parameters + if (signature.isNotEmpty()) { + scopeManager.withScope(function) { + for (i in signature.indices) { + val targetType = signature[i] + val paramName = generateParamName(i, targetType!!) + val param = newParamVariableDeclaration(paramName, targetType, false, "") + param.argumentIndex = i + + scopeManager.addDeclaration(param) + } + } } - return params } /** Generates a name for an inferred function parameter based on the type. */ @@ -183,7 +211,7 @@ class Inference(val start: Node) : LanguageProvider, IsInferredProvider { return paramName.toString() } - fun inferNonTypeTemplateParameter(name: String): ParamVariableDeclaration { + private fun inferNonTypeTemplateParameter(name: String): ParamVariableDeclaration { val expr = start as? Expression ?: throw UnsupportedOperationException( @@ -194,7 +222,7 @@ class Inference(val start: Node) : LanguageProvider, IsInferredProvider { return newParamVariableDeclaration(name, expr.type, false, name) } - fun inferTemplateParameter( + private fun inferTemplateParameter( name: String, ): TypeParamDeclaration { val parameterizedType = ParameterizedType(name, language) @@ -211,7 +239,6 @@ class Inference(val start: Node) : LanguageProvider, IsInferredProvider { * Create an inferred FunctionTemplateDeclaration if a call to an FunctionTemplate could not be * resolved * - * @param containingRecord * @param call * @return inferred FunctionTemplateDeclaration which can be invoked by the call */ @@ -235,10 +262,10 @@ class Inference(val start: Node) : LanguageProvider, IsInferredProvider { val inferredRealization: FunctionDeclaration = if (record != null) { record.addDeclaration(inferred) - record.inferMethod(call) + record.inferMethod(call, scopeManager = scopeManager) } else { tu!!.addDeclaration(inferred) - tu.inferFunction(call) + tu.inferFunction(call, scopeManager = scopeManager) } inferred.addRealization(inferredRealization) @@ -250,13 +277,17 @@ class Inference(val start: Node) : LanguageProvider, IsInferredProvider { // Template Parameter val inferredTypeIdentifier = "T$typeCounter" val typeParamDeclaration = - inferred.startInference().inferTemplateParameter(inferredTypeIdentifier) + inferred + .startInference(scopeManager) + .inferTemplateParameter(inferredTypeIdentifier) typeCounter++ inferred.addParameter(typeParamDeclaration) } else if (node is Expression) { val inferredNonTypeIdentifier = "N$nonTypeCounter" val paramVariableDeclaration = - node.startInference().inferNonTypeTemplateParameter(inferredNonTypeIdentifier) + node + .startInference(scopeManager) + .inferNonTypeTemplateParameter(inferredNonTypeIdentifier) paramVariableDeclaration.addPrevDFG(node) node.addNextDFG(paramVariableDeclaration) @@ -269,8 +300,7 @@ class Inference(val start: Node) : LanguageProvider, IsInferredProvider { /** * Infers a record declaration for the given type. [type] is the object type representing a - * record that we want to infer, the [recordToUpdate] is either the type's name or the type's - * root name. The [kind] specifies if we create a class or a struct. + * record that we want to infer. The [kind] specifies if we create a class or a struct. */ fun inferRecordDeclaration( type: Type, @@ -300,8 +330,30 @@ class Inference(val start: Node) : LanguageProvider, IsInferredProvider { return declaration } - override val isInferred: Boolean - get() = true + fun createInferredNamespaceDeclaration(name: Name, path: String?): NamespaceDeclaration { + // Here be dragons. Jump to the scope that the node defines directly, so that we can + // delegate further operations to the scope manager. We also save the old scope so we can + // restore it. + val oldScope = scopeManager.jumpTo(start) + + log.debug( + "Inferring a new namespace declaration $name ${if(path != null) {"with path '$path'"} else {""}}" + ) + + val inferred = newNamespaceDeclaration(name) + inferred.path = path + + scopeManager.addDeclaration(inferred) + + // We need to "enter" the scope to make it known to the scope map of the ScopeManager + scopeManager.enterScope(inferred) + scopeManager.leaveScope(inferred) + + // Revert back to the old scope, so that whoever called us can continue safely + scopeManager.jumpTo(oldScope) + + return inferred + } } /** Provides information about the inference status of a node. */ @@ -310,14 +362,15 @@ interface IsInferredProvider : MetadataProvider { } /** Returns a new [Inference] object starting from this node. */ -fun Node.startInference() = Inference(this) +fun Node.startInference(scopeManager: ScopeManager) = Inference(this, scopeManager) /** Tries to infer a [FunctionDeclaration] from a [CallExpression]. */ fun TranslationUnitDeclaration.inferFunction( call: CallExpression, - isStatic: Boolean = false + isStatic: Boolean = false, + scopeManager: ScopeManager, ): FunctionDeclaration { - return Inference(this) + return Inference(this, scopeManager) .createInferredFunctionDeclaration( call.name.localName, call.code, @@ -328,12 +381,30 @@ fun TranslationUnitDeclaration.inferFunction( ) } +/** Tries to infer a [FunctionDeclaration] from a [CallExpression]. */ +fun NamespaceDeclaration.inferFunction( + call: CallExpression, + isStatic: Boolean = false, + scopeManager: ScopeManager, +): FunctionDeclaration { + return Inference(this, scopeManager) + .createInferredFunctionDeclaration( + call.name, + call.code, + isStatic, + call.signature, + // TODO: Is the call's type the return value's type? + call.type + ) +} + /** Tries to infer a [MethodDeclaration] from a [CallExpression]. */ fun RecordDeclaration.inferMethod( call: CallExpression, - isStatic: Boolean = false + isStatic: Boolean = false, + scopeManager: ScopeManager ): MethodDeclaration { - return Inference(this) + return Inference(this, scopeManager) .createInferredFunctionDeclaration( call.name.localName, call.code, diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/FluentTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/FluentTest.kt index d7852eb94d9..f73cb9fcace 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/FluentTest.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/FluentTest.kt @@ -139,7 +139,8 @@ class FluentTest { assertNotNull(lit1) assertEquals(1, lit1.value) - // Third line is the CallExpression (containing another MemberCallExpression as argument) + // Third line is th + // e CallExpression (containing another MemberCallExpression as argument) val call = main[2] as? CallExpression assertNotNull(call) assertLocalName("do", call) diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/CallResolverTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/CallResolverTest.kt index 495dc17549d..13faeffb4aa 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/CallResolverTest.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/CallResolverTest.kt @@ -117,6 +117,7 @@ class CallResolverTest : BaseTest() { val inferenceSignature = listOf(intType, intType, intType) for (inferredCall in calls.filter { c: CallExpression -> c.signature == inferenceSignature }) { + val inferredTarget = findByUniquePredicate(methods) { m: FunctionDeclaration -> m.hasSignature(inferenceSignature) diff --git a/cpg-core/src/testFixtures/kotlin/de/fraunhofer/aisec/cpg/TestUtils.kt b/cpg-core/src/testFixtures/kotlin/de/fraunhofer/aisec/cpg/TestUtils.kt index 16f28fa9770..0e2907a6065 100644 --- a/cpg-core/src/testFixtures/kotlin/de/fraunhofer/aisec/cpg/TestUtils.kt +++ b/cpg-core/src/testFixtures/kotlin/de/fraunhofer/aisec/cpg/TestUtils.kt @@ -31,10 +31,7 @@ import de.fraunhofer.aisec.cpg.graph.TypeManager import de.fraunhofer.aisec.cpg.graph.declarations.Declaration import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration -import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression -import de.fraunhofer.aisec.cpg.graph.statements.expressions.DeclaredReferenceExpression -import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression -import de.fraunhofer.aisec.cpg.graph.statements.expressions.MemberExpression +import de.fraunhofer.aisec.cpg.graph.statements.expressions.* import java.io.File import java.nio.file.Files import java.nio.file.Path @@ -297,10 +294,18 @@ object TestUtils { fun assertFullName(fqn: String, node: Node?, message: String? = null) { assertNotNull(node) - asserter.assertEquals(message, fqn, node.name.toString()) + assertEquals(fqn, node.name.toString(), message) } fun assertLocalName(localName: String, node: Node?, message: String? = null) { assertNotNull(node) - asserter.assertEquals(message, localName, node.name.localName) + assertEquals(localName, node.name.localName, message) +} + +/** + * Asserts that a) the expression in [expr] is a [Literal] and b) that it's value is equal to + * [expected]. + */ +fun assertLiteralValue(expected: T, expr: Expression?, message: String? = null) { + assertEquals(expected, assertIs>(expr).value, message) } diff --git a/cpg-language-go/src/main/golang/basic_types.go b/cpg-language-go/src/main/golang/basic_types.go index c789e338bab..33252605722 100644 --- a/cpg-language-go/src/main/golang/basic_types.go +++ b/cpg-language-go/src/main/golang/basic_types.go @@ -32,13 +32,16 @@ import ( ) func NewString(s string) *jnigi.ObjectRef { - o, err := env.NewObject("java/lang/String", []byte(s)) - if err != nil { - log.Fatal(err) + if s != "" { + o, err := env.NewObject("java/lang/String", []byte(s)) + if err != nil { + log.Fatal(err) + } + return o + } else { + return jnigi.NewObjectRef("java/lang/String") } - - return o } func NewCharSequence(s string) *jnigi.ObjectRef { diff --git a/cpg-language-go/src/main/golang/declarations.go b/cpg-language-go/src/main/golang/declarations.go index 74d376e09e1..caadcced292 100644 --- a/cpg-language-go/src/main/golang/declarations.go +++ b/cpg-language-go/src/main/golang/declarations.go @@ -84,6 +84,12 @@ func (f *FunctionDeclaration) SetBody(s *Statement) (err error) { return } +func (n *NamespaceDeclaration) SetPath(path string) (err error) { + err = (*jnigi.ObjectRef)(n).CallMethod(env, "setPath", nil, NewString(path)) + + return +} + func (m *MethodDeclaration) SetType(t *Type) { (*HasType)(m).SetType(t) } @@ -108,6 +114,12 @@ func (p *ParamVariableDeclaration) SetType(t *Type) { (*HasType)(p).SetType(t) } +func (p *ParamVariableDeclaration) SetVariadic(b bool) (err error) { + err = (*jnigi.ObjectRef)(p).CallMethod(env, "setVariadic", nil, b) + + return +} + func (f *FieldDeclaration) SetType(t *Type) { (*HasType)(f).SetType(t) } diff --git a/cpg-language-go/src/main/golang/expressions.go b/cpg-language-go/src/main/golang/expressions.go index 51851bff0dd..6cdcc3491b3 100644 --- a/cpg-language-go/src/main/golang/expressions.go +++ b/cpg-language-go/src/main/golang/expressions.go @@ -59,15 +59,19 @@ type CastExpression Expression type NewExpression Expression type ArrayCreationExpression Expression type ArraySubscriptionExpression Expression +type SliceExpression Expression type ConstructExpression Expression type InitializerListExpression Expression type MemberCallExpression CallExpression type MemberExpression Expression type BinaryOperator Expression +type AssignExpression Expression type UnaryOperator Expression type Literal Expression type DeclaredReferenceExpression Expression type KeyValueExpression Expression +type LambdaExpression Expression +type ProblemExpression Expression func (e *Expression) SetType(t *Type) { (*HasType)(e).SetType(t) @@ -143,6 +147,28 @@ func (b *BinaryOperator) SetOperatorCode(s string) (err error) { return (*jnigi.ObjectRef)(b).SetField(env, "operatorCode", NewString(s)) } +func (a *AssignExpression) SetLHS(e []*Expression) { + list, err := ListOf(e) + if err != nil { + panic(err) + } + + (*jnigi.ObjectRef)(a).CallMethod(env, "setLhs", nil, list.Cast("java/util/List")) +} + +func (a *AssignExpression) SetRHS(e []*Expression) { + list, err := ListOf(e) + if err != nil { + panic(err) + } + + (*jnigi.ObjectRef)(a).CallMethod(env, "setRhs", nil, list.Cast("java/util/List")) +} + +func (a *AssignExpression) SetOperatorCode(op string) { + (*jnigi.ObjectRef)(a).CallMethod(env, "setOperatorCode", nil, NewString(op)) +} + func (u *UnaryOperator) SetInput(e *Expression) { (*jnigi.ObjectRef)(u).CallMethod(env, "setInput", nil, (*jnigi.ObjectRef)(e).Cast(ExpressionClass)) } @@ -184,6 +210,18 @@ func (r *ArraySubscriptionExpression) SetSubscriptExpression(e *Expression) { (*jnigi.ObjectRef)(r).CallMethod(env, "setSubscriptExpression", nil, (*jnigi.ObjectRef)(e).Cast(ExpressionClass)) } +func (s *SliceExpression) SetLowerBound(e *Expression) { + (*jnigi.ObjectRef)(s).CallMethod(env, "setLowerBound", nil, (*jnigi.ObjectRef)(e).Cast(ExpressionClass)) +} + +func (s *SliceExpression) SetUpperBound(e *Expression) { + (*jnigi.ObjectRef)(s).CallMethod(env, "setUpperBound", nil, (*jnigi.ObjectRef)(e).Cast(ExpressionClass)) +} + +func (s *SliceExpression) SetThird(e *Expression) { + (*jnigi.ObjectRef)(s).CallMethod(env, "setThird", nil, (*jnigi.ObjectRef)(e).Cast(ExpressionClass)) +} + func (c *ConstructExpression) AddArgument(e *Expression) { (*jnigi.ObjectRef)(c).CallMethod(env, "addArgument", nil, (*jnigi.ObjectRef)(e).Cast(ExpressionClass)) } @@ -198,8 +236,13 @@ func (n *NewExpression) SetInitializer(e *Expression) (err error) { return } -func (c *InitializerListExpression) AddInitializer(e *Expression) { - (*jnigi.ObjectRef)(c).CallMethod(env, "addInitializer", nil, (*jnigi.ObjectRef)(e).Cast(ExpressionClass)) +func (c *InitializerListExpression) SetInitializers(e []*Expression) { + l, err := ListOf(e) + if err != nil { + panic(err) + } + + (*jnigi.ObjectRef)(c).CallMethod(env, "setInitializers", nil, l.Cast("java/util/List")) } func (k *KeyValueExpression) SetKey(e *Expression) { @@ -209,3 +252,10 @@ func (k *KeyValueExpression) SetKey(e *Expression) { func (k *KeyValueExpression) SetValue(e *Expression) { (*jnigi.ObjectRef)(k).CallMethod(env, "setValue", nil, (*jnigi.ObjectRef)(e).Cast(ExpressionClass)) } + +func (l *LambdaExpression) SetFunction(f *FunctionDeclaration) { + err := (*jnigi.ObjectRef)(l).CallMethod(env, "setFunction", nil, (*jnigi.ObjectRef)(f).Cast(FunctionDeclarationClass)) + if err != nil { + panic(err) + } +} diff --git a/cpg-language-go/src/main/golang/frontend/declaration_builder.go b/cpg-language-go/src/main/golang/frontend/declaration_builder.go index 4965cacb36c..817ce7faf93 100644 --- a/cpg-language-go/src/main/golang/frontend/declaration_builder.go +++ b/cpg-language-go/src/main/golang/frontend/declaration_builder.go @@ -46,8 +46,12 @@ func (frontend *GoLanguageFrontend) NewIncludeDeclaration(fset *token.FileSet, a return (*cpg.IncludeDeclaration)(frontend.NewDeclaration("IncludeDeclaration", fset, astNode, name)) } -func (frontend *GoLanguageFrontend) NewFunctionDeclaration(fset *token.FileSet, astNode ast.Node, name string) *cpg.FunctionDeclaration { - return (*cpg.FunctionDeclaration)(frontend.NewDeclaration("FunctionDeclaration", fset, astNode, name)) +func (frontend *GoLanguageFrontend) NewFunctionDeclaration(fset *token.FileSet, astNode ast.Node, name string, code string, localNameOnly bool) *cpg.FunctionDeclaration { + return (*cpg.FunctionDeclaration)(frontend.NewDeclaration("FunctionDeclaration", fset, astNode, name, + cpg.NewString(code), + jnigi.NewObjectRef("java/lang/Object"), + localNameOnly, + )) } func (frontend *GoLanguageFrontend) NewMethodDeclaration(fset *token.FileSet, astNode ast.Node, name string) *cpg.MethodDeclaration { diff --git a/cpg-language-go/src/main/golang/frontend/expression_builder.go b/cpg-language-go/src/main/golang/frontend/expression_builder.go index 06df6661b7f..e998abe53da 100644 --- a/cpg-language-go/src/main/golang/frontend/expression_builder.go +++ b/cpg-language-go/src/main/golang/frontend/expression_builder.go @@ -70,6 +70,10 @@ func (frontend *GoLanguageFrontend) NewArraySubscriptionExpression(fset *token.F return (*cpg.ArraySubscriptionExpression)(frontend.NewExpression("ArraySubscriptionExpression", fset, astNode)) } +func (frontend *GoLanguageFrontend) NewSliceExpression(fset *token.FileSet, astNode ast.Node) *cpg.SliceExpression { + return (*cpg.SliceExpression)(frontend.NewExpression("SliceExpression", fset, astNode)) +} + func (frontend *GoLanguageFrontend) NewConstructExpression(fset *token.FileSet, astNode ast.Node) *cpg.ConstructExpression { return (*cpg.ConstructExpression)(frontend.NewExpression("ConstructExpression", fset, astNode)) } @@ -84,6 +88,12 @@ func (frontend *GoLanguageFrontend) NewBinaryOperator(fset *token.FileSet, astNo )) } +func (frontend *GoLanguageFrontend) NewAssignExpression(fset *token.FileSet, astNode ast.Node, opCode string) *cpg.AssignExpression { + return (*cpg.AssignExpression)(frontend.NewExpression("AssignExpression", fset, astNode, + cpg.NewString(opCode), + )) +} + func (frontend *GoLanguageFrontend) NewUnaryOperator(fset *token.FileSet, astNode ast.Node, opCode string, postfix bool, prefix bool) *cpg.UnaryOperator { return (*cpg.UnaryOperator)(frontend.NewExpression("UnaryOperator", fset, astNode, cpg.NewString(opCode), @@ -98,6 +108,10 @@ func (frontend *GoLanguageFrontend) NewLiteral(fset *token.FileSet, astNode ast. value = value.Cast("java/lang/Object") } + if typ == nil { + panic("typ is nil") + } + return (*cpg.Literal)(frontend.NewExpression("Literal", fset, astNode, value, typ.Cast(cpg.TypeClass))) } @@ -109,6 +123,14 @@ func (frontend *GoLanguageFrontend) NewKeyValueExpression(fset *token.FileSet, a return (*cpg.KeyValueExpression)(frontend.NewExpression("KeyValueExpression", fset, astNode)) } +func (frontend *GoLanguageFrontend) NewLambdaExpression(fset *token.FileSet, astNode ast.Node) *cpg.LambdaExpression { + return (*cpg.LambdaExpression)(frontend.NewExpression("LambdaExpression", fset, astNode)) +} + +func (frontend *GoLanguageFrontend) NewProblemExpression(fset *token.FileSet, astNode ast.Node, problem string) *cpg.ProblemExpression { + return (*cpg.ProblemExpression)(frontend.NewExpression("ProblemExpression", fset, astNode, cpg.NewString(problem))) +} + func (frontend *GoLanguageFrontend) NewExpression(typ string, fset *token.FileSet, astNode ast.Node, args ...any) *jnigi.ObjectRef { var node = jnigi.NewObjectRef(fmt.Sprintf("%s/%s", cpg.ExpressionsPackage, typ)) diff --git a/cpg-language-go/src/main/golang/frontend/frontend.go b/cpg-language-go/src/main/golang/frontend/frontend.go index b70c070a5bc..4dbe97468ff 100644 --- a/cpg-language-go/src/main/golang/frontend/frontend.go +++ b/cpg-language-go/src/main/golang/frontend/frontend.go @@ -45,6 +45,7 @@ type GoLanguageFrontend struct { File *ast.File Module *modfile.File CommentMap ast.CommentMap + TopLevel string CurrentTU *cpg.TranslationUnitDeclaration } @@ -101,6 +102,18 @@ func (g *GoLanguageFrontend) LogDebug(format string, args ...interface{}) (err e return } +func (g *GoLanguageFrontend) LogTrace(format string, args ...interface{}) (err error) { + var logger *jnigi.ObjectRef + + if logger, err = g.getLog(); err != nil { + return + } + + err = logger.CallMethod(env, "trace", nil, cpg.NewString(fmt.Sprintf(format, args...))) + + return +} + func (g *GoLanguageFrontend) LogError(format string, args ...interface{}) (err error) { var logger *jnigi.ObjectRef @@ -121,10 +134,14 @@ func (g *GoLanguageFrontend) GetLanguage() (l *cpg.Language, err error) { } func updateCode(fset *token.FileSet, node *cpg.Node, astNode ast.Node) { + node.SetCode(code(fset, astNode)) +} + +func code(fset *token.FileSet, astNode ast.Node) string { var codeBuf bytes.Buffer _ = printer.Fprint(&codeBuf, fset, astNode) - node.SetCode(codeBuf.String()) + return codeBuf.String() } func updateLocation(fset *token.FileSet, node *cpg.Node, astNode ast.Node) { diff --git a/cpg-language-go/src/main/golang/frontend/handler.go b/cpg-language-go/src/main/golang/frontend/handler.go index f7cd9dedb08..22213c3a828 100644 --- a/cpg-language-go/src/main/golang/frontend/handler.go +++ b/cpg-language-go/src/main/golang/frontend/handler.go @@ -34,6 +34,7 @@ import ( "log" "os" "path" + "path/filepath" "strconv" "strings" @@ -104,22 +105,43 @@ func (this *GoLanguageFrontend) HandleFile(fset *token.FileSet, file *ast.File, } } - // create a new namespace declaration, representing the package + // Create a new namespace declaration, representing the package p := this.NewNamespaceDeclaration(fset, nil, file.Name.Name) + // we need to construct the package "path" (e.g. "encoding/json") out of the + // module path as well as the current directory in relation to the topLevel + packagePath := filepath.Dir(path) + + // Construct a relative path starting from the top level + packagePath, err = filepath.Rel(this.TopLevel, packagePath) + if err == nil { + // If we are in a module, we need to prepend the module path to it + if this.Module != nil { + packagePath = filepath.Join(this.Module.Module.Mod.Path, packagePath) + } + + p.SetPath(packagePath) + } else { + this.LogError("Could not relativize package path to top level. Cannot set package path: %v", err) + } + // enter scope scope.EnterScope((*cpg.Node)(p)) for _, decl := range file.Decls { - var d *cpg.Declaration - - d = this.handleDecl(fset, decl) - - if d != nil { - err = scope.AddDeclaration((*cpg.Declaration)(d)) - if err != nil { - log.Fatal(err) + // Retrieve all top level declarations. One "Decl" could potentially + // contain multiple CPG declarations. + decls := this.handleDecl(fset, decl) + + for _, d := range decls { + if d != nil { + // Add declaration to current scope. This will also add it to the + // respective AST scope holder + err = scope.AddDeclaration((*cpg.Declaration)(d)) + if err != nil { + log.Fatal(err) + } } } } @@ -135,7 +157,7 @@ func (this *GoLanguageFrontend) HandleFile(fset *token.FileSet, file *ast.File, // handleComments maps comments from ast.Node to a cpg.Node by using ast.CommentMap. func (this *GoLanguageFrontend) handleComments(node *cpg.Node, astNode ast.Node) { - this.LogDebug("Handling comments for %+v", astNode) + this.LogTrace("Handling comments for %+v", astNode) var comment = "" @@ -155,33 +177,43 @@ func (this *GoLanguageFrontend) handleComments(node *cpg.Node, astNode ast.Node) if comment != "" { node.SetComment(comment) - this.LogDebug("Comments: %+v", comment) + this.LogTrace("Comments: %+v", comment) } } -func (this *GoLanguageFrontend) handleDecl(fset *token.FileSet, decl ast.Decl) (d *cpg.Declaration) { - this.LogDebug("Handling declaration (%T): %+v", decl, decl) +// handleDecl parses an [ast.Decl]. Note, that in a "Decl", one or more actual +// declarations can be found. Therefore, this function returns a slice of +// [cpg.Declaration]. +func (this *GoLanguageFrontend) handleDecl(fset *token.FileSet, decl ast.Decl) (decls []*cpg.Declaration) { + this.LogTrace("Handling declaration (%T): %+v", decl, decl) + + decls = []*cpg.Declaration{} switch v := decl.(type) { case *ast.FuncDecl: - d = (*cpg.Declaration)(this.handleFuncDecl(fset, v)) + // There can be only a single function declaration + decls = append(decls, (*cpg.Declaration)(this.handleFuncDecl(fset, v))) case *ast.GenDecl: - d = (*cpg.Declaration)(this.handleGenDecl(fset, v)) + // GenDecl can hold multiple declarations + decls = this.handleGenDecl(fset, v) default: this.LogError("Not parsing declaration of type %T yet: %+v", v, v) - // no match - d = nil + // TODO: Return a ProblemDeclaration } - if d != nil { - this.handleComments((*cpg.Node)(d), decl) + // Handle comments for all declarations + for _, d := range decls { + // TODO: This is problematic because we are assigning it the wrong node + if d != nil { + this.handleComments((*cpg.Node)(d), decl) + } } return } -func (this *GoLanguageFrontend) handleFuncDecl(fset *token.FileSet, funcDecl *ast.FuncDecl) *jnigi.ObjectRef { - this.LogDebug("Handling func Decl: %+v", *funcDecl) +func (this *GoLanguageFrontend) handleFuncDecl(fset *token.FileSet, funcDecl *ast.FuncDecl) *cpg.FunctionDeclaration { + this.LogTrace("Handling func Decl: %+v", *funcDecl) var scope = this.GetScopeManager() var receiver *cpg.VariableDeclaration @@ -193,7 +225,7 @@ func (this *GoLanguageFrontend) handleFuncDecl(fset *token.FileSet, funcDecl *as // TODO: why is this a list? var recv = funcDecl.Recv.List[0] - var recordType = this.handleType(recv.Type) + var recordType = this.handleType(fset, recv.Type) // The name of the Go receiver is optional. In fact, if the name is not // specified we probably do not need any receiver variable at all, @@ -230,7 +262,7 @@ func (this *GoLanguageFrontend) handleFuncDecl(fset *token.FileSet, funcDecl *as // marked as AST and in Go a method is not part of the struct's AST but is declared // outside. In the future, we need to differentiate between just the associated members // of the class and the pure AST nodes declared in the struct itself - this.LogDebug("Record: %+v", record) + this.LogTrace("Record: %+v", record) err = record.AddMethod(m) if err != nil { @@ -242,31 +274,37 @@ func (this *GoLanguageFrontend) handleFuncDecl(fset *token.FileSet, funcDecl *as f = (*cpg.FunctionDeclaration)(m) } else { - f = this.NewFunctionDeclaration(fset, funcDecl, funcDecl.Name.Name) + // We do not want to prefix the package for an empty (lambda) function name + var localNameOnly bool = false + if funcDecl.Name.Name == "" { + localNameOnly = true + } + + f = this.NewFunctionDeclaration(fset, funcDecl, funcDecl.Name.Name, "", localNameOnly) } // enter scope for function scope.EnterScope((*cpg.Node)(f)) if receiver != nil { - this.LogDebug("Adding receiver %s", (*cpg.Node)(receiver).GetName()) + this.LogTrace("Adding receiver %s", (*cpg.Node)(receiver).GetName()) // add the receiver do the scope manager, so we can resolve the receiver value this.GetScopeManager().AddDeclaration((*cpg.Declaration)(receiver)) } - var t *cpg.Type = this.handleType(funcDecl.Type) + var t *cpg.Type = this.handleType(fset, funcDecl.Type) var returnTypes []*cpg.Type = []*cpg.Type{} if funcDecl.Type.Results != nil { for _, returnVariable := range funcDecl.Type.Results.List { - returnTypes = append(returnTypes, this.handleType(returnVariable.Type)) + returnTypes = append(returnTypes, this.handleType(fset, returnVariable.Type)) // if the function has named return variables, be sure to declare them as well if returnVariable.Names != nil { p := this.NewVariableDeclaration(fset, returnVariable, returnVariable.Names[0].Name) - p.SetType(this.handleType(returnVariable.Type)) + p.SetType(this.handleType(fset, returnVariable.Type)) // add parameter to scope this.GetScopeManager().AddDeclaration((*cpg.Declaration)(p)) @@ -274,7 +312,7 @@ func (this *GoLanguageFrontend) handleFuncDecl(fset *token.FileSet, funcDecl *as } } - this.LogDebug("Function has type %s", t.GetName()) + this.LogTrace("Function has type %s", t.GetName()) f.SetType(t) f.SetReturnTypes(returnTypes) @@ -284,7 +322,7 @@ func (this *GoLanguageFrontend) handleFuncDecl(fset *token.FileSet, funcDecl *as // go, since we do not have a 'this', but rather a named receiver for _, param := range funcDecl.Type.Params.List { - this.LogDebug("Parsing param: %+v", param) + this.LogTrace("Parsing param: %+v", param) var name string // Somehow parameters end up having no name sometimes, have not fully understood why. @@ -305,7 +343,22 @@ func (this *GoLanguageFrontend) handleFuncDecl(fset *token.FileSet, funcDecl *as p := this.NewParamVariableDeclaration(fset, param, name) - p.SetType(this.handleType(param.Type)) + // Check for varargs. In this case we want to parse the element type + // (and make it an array afterwards) + if ell, ok := param.Type.(*ast.Ellipsis); ok { + p.SetVariadic(true) + var t = this.handleType(fset, ell.Elt) + + var i = jnigi.NewObjectRef(cpg.PointerOriginClass) + err := env.GetStaticField(cpg.PointerOriginClass, "ARRAY", i) + if err != nil { + log.Fatal(err) + } + + p.SetType(t.Reference(i)) + } else { + p.SetType(this.handleType(fset, param.Type)) + } // add parameter to scope this.GetScopeManager().AddDeclaration((*cpg.Declaration)(p)) @@ -313,7 +366,7 @@ func (this *GoLanguageFrontend) handleFuncDecl(fset *token.FileSet, funcDecl *as this.handleComments((*cpg.Node)(p), param) } - this.LogDebug("Parsing function body of %s", (*cpg.Node)(f).GetName()) + this.LogTrace("Parsing function body of %s", (*cpg.Node)(f).GetName()) // parse body s := this.handleBlockStmt(fset, funcDecl.Body) @@ -331,55 +384,70 @@ func (this *GoLanguageFrontend) handleFuncDecl(fset *token.FileSet, funcDecl *as } - return (*jnigi.ObjectRef)(f) + return f } -func (this *GoLanguageFrontend) handleGenDecl(fset *token.FileSet, genDecl *ast.GenDecl) *jnigi.ObjectRef { - // TODO: Handle multiple declarations +func (this *GoLanguageFrontend) handleGenDecl(fset *token.FileSet, genDecl *ast.GenDecl) (decls []*cpg.Declaration) { + decls = []*cpg.Declaration{} + for _, spec := range genDecl.Specs { switch v := spec.(type) { case *ast.ValueSpec: - return (*jnigi.ObjectRef)(this.handleValueSpec(fset, v)) + decls = append(decls, this.handleValueSpec(fset, v)...) case *ast.TypeSpec: - return (*jnigi.ObjectRef)(this.handleTypeSpec(fset, v)) + decls = append(decls, this.handleTypeSpec(fset, v)) case *ast.ImportSpec: - // somehow these end up duplicate in the AST, so do not handle them here - return nil - /*return (*jnigi.ObjectRef)(this.handleImportSpec(fset, v))*/ + // Somehow these end up duplicate in the AST, so do not handle them here default: - this.LogError("Not parsing specication of type %T yet: %+v", v, v) + this.LogError("Not parsing specification of type %T yet: %+v", v, v) } } - return nil + return } -func (this *GoLanguageFrontend) handleValueSpec(fset *token.FileSet, valueDecl *ast.ValueSpec) *cpg.Declaration { - // TODO: more names - var ident = valueDecl.Names[0] +// handleValueSpec handles parsing of an [ast.ValueSpec], which is a variable +// declaration. Since this can potentially declare multiple variables with one +// "spec", this returns a slice of [cpg.Declaration]. +func (this *GoLanguageFrontend) handleValueSpec(fset *token.FileSet, valueDecl *ast.ValueSpec) (decls []*cpg.Declaration) { + decls = []*cpg.Declaration{} - d := this.NewVariableDeclaration(fset, valueDecl, ident.Name) + // We need to declare one variable for each name + for idx, ident := range valueDecl.Names { + d := this.NewVariableDeclaration(fset, valueDecl, ident.Name) - if valueDecl.Type != nil { - t := this.handleType(valueDecl.Type) + // Handle the type (if its there) + if valueDecl.Type != nil { + t := this.handleType(fset, valueDecl.Type) - d.SetType(t) - } + d.SetType(t) + } - // add an initializer - if len(valueDecl.Values) > 0 { - // TODO: How to deal with multiple values - var expr = this.handleExpr(fset, valueDecl.Values[0]) + // There could either be no initializers, otherwise the amount of values + // must match the names + lenValues := len(valueDecl.Values) + if lenValues != 0 && lenValues != len(valueDecl.Names) { + this.LogError("Number of initializers does not match number of names. Initializers might be incomplete") + } - err := d.SetInitializer(expr) - if err != nil { - log.Fatal(err) + // The initializer is in the "Values" slice with the respective index + if len(valueDecl.Values) > idx { + var expr = this.handleExpr(fset, valueDecl.Values[idx]) + + err := d.SetInitializer(expr) + if err != nil { + log.Fatal(err) + } } + + decls = append(decls, d.Declaration()) } - return (*cpg.Declaration)(d) + return decls } +// handleTypeSpec handles an [ast.TypeSec], which defines either a struct or an +// interface. It returns a single [cpg.Declaration]. func (this *GoLanguageFrontend) handleTypeSpec(fset *token.FileSet, typeDecl *ast.TypeSpec) *cpg.Declaration { err := this.LogInfo("Type specifier with name %s and type (%T, %+v)", typeDecl.Name.Name, typeDecl.Type, typeDecl.Type) if err != nil { @@ -397,7 +465,7 @@ func (this *GoLanguageFrontend) handleTypeSpec(fset *token.FileSet, typeDecl *as } func (this *GoLanguageFrontend) handleImportSpec(fset *token.FileSet, importSpec *ast.ImportSpec) *cpg.Declaration { - this.LogInfo("Import specifier with: %+v)", *importSpec) + this.LogTrace("Import specifier with: %+v)", *importSpec) i := this.NewIncludeDeclaration(fset, importSpec, getImportName(importSpec)) @@ -432,17 +500,17 @@ func (this *GoLanguageFrontend) handleStructTypeSpec(fset *token.FileSet, typeDe // by its type, it could make sense to name the field according to the type var name string - t := this.handleType(field.Type) + t := this.handleType(fset, field.Type) if field.Names == nil { // retrieve the root type name var typeName = t.GetRoot().GetName().ToString() - this.LogDebug("Handling embedded field of type %s", typeName) + this.LogTrace("Handling embedded field of type %s", typeName) name = typeName } else { - this.LogDebug("Handling field %s", field.Names[0].Name) + this.LogTrace("Handling field %s", field.Names[0].Name) // TODO: Multiple names? name = field.Names[0].Name @@ -470,7 +538,7 @@ func (this *GoLanguageFrontend) handleInterfaceTypeSpec(fset *token.FileSet, typ if !interfaceType.Incomplete { for _, method := range interfaceType.Methods.List { - t := this.handleType(method.Type) + t := this.handleType(fset, method.Type) // Even though this list is called "Methods", it contains all kinds // of things, so we need to proceed with caution. Only if the @@ -482,7 +550,7 @@ func (this *GoLanguageFrontend) handleInterfaceTypeSpec(fset *token.FileSet, typ scope.AddDeclaration((*cpg.Declaration)(m)) } else { - this.LogDebug("Adding %s as super class of interface %s", t.GetName(), (*cpg.Node)(r).GetName()) + this.LogTrace("Adding %s as super class of interface %s", t.GetName(), (*cpg.Node)(r).GetName()) // Otherwise, it contains either types or interfaces. For now we // hope that it only has interfaces. We consider embedded // interfaces as sort of super types for this interface. @@ -497,7 +565,7 @@ func (this *GoLanguageFrontend) handleInterfaceTypeSpec(fset *token.FileSet, typ } func (this *GoLanguageFrontend) handleBlockStmt(fset *token.FileSet, blockStmt *ast.BlockStmt) *cpg.CompoundStatement { - this.LogDebug("Handling block statement: %+v", *blockStmt) + this.LogTrace("Handling block statement: %+v", *blockStmt) c := this.NewCompoundStatement(fset, blockStmt) @@ -507,7 +575,7 @@ func (this *GoLanguageFrontend) handleBlockStmt(fset *token.FileSet, blockStmt * for _, stmt := range blockStmt.List { var s *cpg.Statement - s = this.handleStmt(fset, stmt) + s = this.handleStmt(fset, stmt, blockStmt) if s != nil { // add statement @@ -522,7 +590,7 @@ func (this *GoLanguageFrontend) handleBlockStmt(fset *token.FileSet, blockStmt * } func (this *GoLanguageFrontend) handleForStmt(fset *token.FileSet, forStmt *ast.ForStmt) *cpg.ForStatement { - this.LogDebug("Handling for statement: %+v", *forStmt) + this.LogTrace("Handling for statement: %+v", *forStmt) f := this.NewForStatement(fset, forStmt) @@ -530,19 +598,22 @@ func (this *GoLanguageFrontend) handleForStmt(fset *token.FileSet, forStmt *ast. scope.EnterScope((*cpg.Node)(f)) - if initStatement := this.handleStmt(fset, forStmt.Init); initStatement != nil { + if forStmt.Init != nil { + initStatement := this.handleStmt(fset, forStmt.Init, forStmt) f.SetInitializerStatement(initStatement) } - if condition := this.handleExpr(fset, forStmt.Cond); condition != nil { + if forStmt.Cond != nil { + condition := this.handleExpr(fset, forStmt.Cond) f.SetCondition(condition) } - if iter := this.handleStmt(fset, forStmt.Post); iter != nil { + if forStmt.Post != nil { + iter := this.handleStmt(fset, forStmt.Post, forStmt) f.SetIterationStatement(iter) } - if body := this.handleStmt(fset, forStmt.Body); body != nil { + if body := this.handleStmt(fset, forStmt.Body, forStmt); body != nil { f.SetStatement(body) } @@ -551,8 +622,53 @@ func (this *GoLanguageFrontend) handleForStmt(fset *token.FileSet, forStmt *ast. return f } +func (this *GoLanguageFrontend) handleRangeStmt(fset *token.FileSet, rangeStmt *ast.RangeStmt) *cpg.ForEachStatement { + this.LogTrace("Handling range statement: %+v", *rangeStmt) + + f := this.NewForEachStatement(fset, rangeStmt) + + var scope = this.GetScopeManager() + + scope.EnterScope((*cpg.Node)(f)) + + // TODO: Support other use cases that do not use DEFINE + if rangeStmt.Tok == token.DEFINE { + stmt := this.NewDeclarationStatement(fset, rangeStmt) + + // TODO: not really the best way to deal with this + // TODO: key type is always int. we could set this + var keyName = rangeStmt.Key.(*ast.Ident).Name + + key := this.NewVariableDeclaration(fset, rangeStmt.Key, keyName) + this.GetScopeManager().AddDeclaration((*cpg.Declaration)(key)) + stmt.AddToPropertyEdgeDeclaration((*cpg.Declaration)(key)) + + if rangeStmt.Value != nil { + // TODO: not really the best way to deal with this + // TODO: key type is always int. we could set this + var valueName = rangeStmt.Value.(*ast.Ident).Name + + value := this.NewVariableDeclaration(fset, rangeStmt.Key, valueName) + this.GetScopeManager().AddDeclaration((*cpg.Declaration)(value)) + stmt.AddToPropertyEdgeDeclaration((*cpg.Declaration)(value)) + } + + f.SetVariable((*cpg.Statement)(stmt)) + } + + iterable := (*cpg.Statement)(this.handleExpr(fset, rangeStmt.X)) + f.SetIterable(iterable) + + body := this.handleStmt(fset, rangeStmt.Body, rangeStmt) + f.SetStatement(body) + + scope.LeaveScope((*cpg.Node)(f)) + + return f +} + func (this *GoLanguageFrontend) handleReturnStmt(fset *token.FileSet, returnStmt *ast.ReturnStmt) *cpg.ReturnStatement { - this.LogDebug("Handling return statement: %+v", *returnStmt) + this.LogTrace("Handling return statement: %+v", *returnStmt) r := this.NewReturnStatement(fset, returnStmt) @@ -572,7 +688,7 @@ func (this *GoLanguageFrontend) handleReturnStmt(fset *token.FileSet, returnStmt } func (this *GoLanguageFrontend) handleIncDecStmt(fset *token.FileSet, incDecStmt *ast.IncDecStmt) *cpg.UnaryOperator { - this.LogDebug("Handling decimal increment statement: %+v", *incDecStmt) + this.LogTrace("Handling decimal increment statement: %+v", *incDecStmt) var opCode string if incDecStmt.Tok == token.INC { @@ -592,8 +708,8 @@ func (this *GoLanguageFrontend) handleIncDecStmt(fset *token.FileSet, incDecStmt return u } -func (this *GoLanguageFrontend) handleStmt(fset *token.FileSet, stmt ast.Stmt) (s *cpg.Statement) { - this.LogDebug("Handling statement (%T): %+v", stmt, stmt) +func (this *GoLanguageFrontend) handleStmt(fset *token.FileSet, stmt ast.Stmt, parent ast.Stmt) (s *cpg.Statement) { + this.LogTrace("Handling statement (%T): %+v", stmt, stmt) switch v := stmt.(type) { case *ast.ExprStmt: @@ -601,9 +717,11 @@ func (this *GoLanguageFrontend) handleStmt(fset *token.FileSet, stmt ast.Stmt) ( // so we do not need an expression statement wrapper s = (*cpg.Statement)(this.handleExpr(fset, v.X)) case *ast.AssignStmt: - s = (*cpg.Statement)(this.handleAssignStmt(fset, v)) + s = (*cpg.Statement)(this.handleAssignStmt(fset, v, parent)) case *ast.DeclStmt: s = (*cpg.Statement)(this.handleDeclStmt(fset, v)) + case *ast.GoStmt: + s = (*cpg.Statement)(this.handleGoStmt(fset, v)) case *ast.IfStmt: s = (*cpg.Statement)(this.handleIfStmt(fset, v)) case *ast.SwitchStmt: @@ -614,13 +732,16 @@ func (this *GoLanguageFrontend) handleStmt(fset *token.FileSet, stmt ast.Stmt) ( s = (*cpg.Statement)(this.handleBlockStmt(fset, v)) case *ast.ForStmt: s = (*cpg.Statement)(this.handleForStmt(fset, v)) + case *ast.RangeStmt: + s = (*cpg.Statement)(this.handleRangeStmt(fset, v)) case *ast.ReturnStmt: s = (*cpg.Statement)(this.handleReturnStmt(fset, v)) case *ast.IncDecStmt: s = (*cpg.Statement)(this.handleIncDecStmt(fset, v)) default: - this.LogError("Not parsing statement of type %T yet: %+v", v, v) - s = nil + msg := fmt.Sprintf("Not parsing statement of type %T yet: %s", v, code(fset, v)) + this.LogError(msg) + s = (*cpg.Statement)(this.NewProblemExpression(fset, v, msg)) } if s != nil { @@ -631,7 +752,7 @@ func (this *GoLanguageFrontend) handleStmt(fset *token.FileSet, stmt ast.Stmt) ( } func (this *GoLanguageFrontend) handleExpr(fset *token.FileSet, expr ast.Expr) (e *cpg.Expression) { - this.LogDebug("Handling expression (%T): %+v", expr, expr) + this.LogTrace("Handling expression (%T): %+v", expr, expr) switch v := expr.(type) { case *ast.CallExpr: @@ -646,12 +767,16 @@ func (this *GoLanguageFrontend) handleExpr(fset *token.FileSet, expr ast.Expr) ( e = (*cpg.Expression)(this.handleStarExpr(fset, v)) case *ast.SelectorExpr: e = (*cpg.Expression)(this.handleSelectorExpr(fset, v)) + case *ast.SliceExpr: + e = (*cpg.Expression)(this.handleSliceExpr(fset, v)) case *ast.KeyValueExpr: e = (*cpg.Expression)(this.handleKeyValueExpr(fset, v)) case *ast.BasicLit: e = (*cpg.Expression)(this.handleBasicLit(fset, v)) case *ast.CompositeLit: e = (*cpg.Expression)(this.handleCompositeLit(fset, v)) + case *ast.FuncLit: + e = (*cpg.Expression)(this.handleFuncLit(fset, v)) case *ast.Ident: e = (*cpg.Expression)(this.handleIdent(fset, v)) case *ast.TypeAssertExpr: @@ -659,9 +784,9 @@ func (this *GoLanguageFrontend) handleExpr(fset *token.FileSet, expr ast.Expr) ( case *ast.ParenExpr: e = this.handleExpr(fset, v.X) default: - this.LogError("Could not parse expression of type %T: %+v", v, v) - // TODO: return an error instead? - e = nil + msg := fmt.Sprintf("Not parsing expression of type %T yet: %s", v, code(fset, v)) + this.LogError(msg) + e = (*cpg.Expression)(this.NewProblemExpression(fset, v, msg)) } if e != nil { @@ -671,67 +796,72 @@ func (this *GoLanguageFrontend) handleExpr(fset *token.FileSet, expr ast.Expr) ( return } -func (this *GoLanguageFrontend) handleAssignStmt(fset *token.FileSet, assignStmt *ast.AssignStmt) (expr *cpg.Expression) { - this.LogDebug("Handling assignment statement: %+v", assignStmt) +func (this *GoLanguageFrontend) handleAssignStmt(fset *token.FileSet, assignStmt *ast.AssignStmt, parent ast.Stmt) (expr *cpg.Expression) { + this.LogTrace("Handling assignment statement: %+v", assignStmt) - // TODO: more than one Rhs?! - rhs := this.handleExpr(fset, assignStmt.Rhs[0]) + this.LogDebug("Parent: %#v", parent) - if assignStmt.Tok == token.DEFINE { - // lets create a variable declaration (wrapped with a declaration stmt) with this, because we define the variable here - stmt := this.NewDeclarationStatement(fset, assignStmt) + var rhs = []*cpg.Expression{} + var lhs = []*cpg.Expression{} + for _, expr := range assignStmt.Lhs { + lhs = append(lhs, this.handleExpr(fset, expr)) + } - var name = assignStmt.Lhs[0].(*ast.Ident).Name + for _, expr := range assignStmt.Rhs { + rhs = append(rhs, this.handleExpr(fset, expr)) + } - // TODO: assignment of multiple values - d := this.NewVariableDeclaration(fset, assignStmt, name) + a := this.NewAssignExpression(fset, assignStmt, "=") - if rhs != nil { - d.SetInitializer(rhs) - } + a.SetLHS(lhs) + a.SetRHS(rhs) - this.GetScopeManager().AddDeclaration((*cpg.Declaration)(d)) + // We need to explicitly set the operator code on this assignment as + // something which potentially declares a variable, so we can resolve this + // in our extra pass. + if assignStmt.Tok == token.DEFINE { + a.SetOperatorCode(":=") + } - stmt.SetSingleDeclaration((*cpg.Declaration)(d)) + expr = (*cpg.Expression)(a) - expr = (*cpg.Expression)(stmt) - } else { - lhs := this.handleExpr(fset, assignStmt.Lhs[0]) + return +} - b := this.NewBinaryOperator(fset, assignStmt, "=") +func (this *GoLanguageFrontend) handleDeclStmt(fset *token.FileSet, declStmt *ast.DeclStmt) (expr *cpg.Expression) { + this.LogTrace("Handling declaration statement: %+v", *declStmt) - if lhs != nil { - b.SetLHS(lhs) - } + // Lets create a variable declaration (wrapped with a declaration stmt) with + // this, because we define the variable here + stmt := this.NewDeclarationStatement(fset, declStmt) - if rhs != nil { - b.SetRHS(rhs) - } + decls := this.handleDecl(fset, declStmt.Decl) - expr = (*cpg.Expression)(b) + // Loop over the declarations and add them to the scope as well as the statement. + for _, d := range decls { + stmt.AddToPropertyEdgeDeclaration(d) + this.GetScopeManager().AddDeclaration(d) } - return + return (*cpg.Expression)(stmt) } -func (this *GoLanguageFrontend) handleDeclStmt(fset *token.FileSet, declStmt *ast.DeclStmt) (expr *cpg.Expression) { - this.LogDebug("Handling declaration statement: %+v", *declStmt) - - // lets create a variable declaration (wrapped with a declaration stmt) with this, - // because we define the variable here - stmt := this.NewDeclarationStatement(fset, declStmt) - - d := this.handleDecl(fset, declStmt.Decl) +// handleGoStmt handles the `go` statement, which is a special keyword in go +// that starts the supplied call expression in a separate Go routine. We cannot +// model this 1:1, so we basically we create a call expression to a built-in call. +func (this *GoLanguageFrontend) handleGoStmt(fset *token.FileSet, goStmt *ast.GoStmt) (expr *cpg.Expression) { + this.LogTrace("Handling go statement: %+v", *goStmt) - stmt.SetSingleDeclaration((*cpg.Declaration)(d)) + ref := (*cpg.Expression)(this.NewDeclaredReferenceExpression(fset, nil, "go")) - this.GetScopeManager().AddDeclaration(d) + call := this.NewCallExpression(fset, goStmt, ref, "go") + call.AddArgument(this.handleCallExpr(fset, goStmt.Call)) - return (*cpg.Expression)(stmt) + return (*cpg.Expression)(call) } func (this *GoLanguageFrontend) handleIfStmt(fset *token.FileSet, ifStmt *ast.IfStmt) (expr *cpg.Expression) { - this.LogDebug("Handling if statement: %+v", *ifStmt) + this.LogTrace("Handling if statement: %+v", *ifStmt) stmt := this.NewIfStatement(fset, ifStmt) @@ -739,23 +869,22 @@ func (this *GoLanguageFrontend) handleIfStmt(fset *token.FileSet, ifStmt *ast.If scope.EnterScope((*cpg.Node)(stmt)) - init := this.handleStmt(fset, ifStmt.Init) - if init != nil { + if ifStmt.Init != nil { + init := this.handleStmt(fset, ifStmt.Init, ifStmt) stmt.SetInitializerStatement(init) } cond := this.handleExpr(fset, ifStmt.Cond) - if cond != nil { - stmt.SetCondition(cond) - } else { - this.LogError("If statement should really have a condition. It is either missing or could not be parsed.") - } + stmt.SetCondition(cond) then := this.handleBlockStmt(fset, ifStmt.Body) - stmt.SetThenStatement((*cpg.Statement)(then)) + // Somehow this can be nil-ish? + if !then.IsNil() { + stmt.SetThenStatement((*cpg.Statement)(then)) + } - els := this.handleStmt(fset, ifStmt.Else) - if els != nil { + if ifStmt.Else != nil { + els := this.handleStmt(fset, ifStmt.Else, ifStmt) stmt.SetElseStatement((*cpg.Statement)(els)) } @@ -765,12 +894,12 @@ func (this *GoLanguageFrontend) handleIfStmt(fset *token.FileSet, ifStmt *ast.If } func (this *GoLanguageFrontend) handleSwitchStmt(fset *token.FileSet, switchStmt *ast.SwitchStmt) (expr *cpg.Expression) { - this.LogDebug("Handling switch statement: %+v", *switchStmt) + this.LogTrace("Handling switch statement: %+v", *switchStmt) s := this.NewSwitchStatement(fset, switchStmt) if switchStmt.Init != nil { - s.SetInitializerStatement(this.handleStmt(fset, switchStmt.Init)) + s.SetInitializerStatement(this.handleStmt(fset, switchStmt.Init, switchStmt)) } if switchStmt.Tag != nil { @@ -783,7 +912,7 @@ func (this *GoLanguageFrontend) handleSwitchStmt(fset *token.FileSet, switchStmt } func (this *GoLanguageFrontend) handleCaseClause(fset *token.FileSet, caseClause *ast.CaseClause) (expr *cpg.Expression) { - this.LogDebug("Handling case clause: %+v", *caseClause) + this.LogTrace("Handling case clause: %+v", *caseClause) var s *cpg.Statement @@ -805,7 +934,7 @@ func (this *GoLanguageFrontend) handleCaseClause(fset *token.FileSet, caseClause } for _, stmt := range caseClause.Body { - s = this.handleStmt(fset, stmt) + s = this.handleStmt(fset, stmt, caseClause) if s != nil && block != nil && !block.IsNil() { // add statement @@ -820,6 +949,28 @@ func (this *GoLanguageFrontend) handleCaseClause(fset *token.FileSet, caseClause func (this *GoLanguageFrontend) handleCallExpr(fset *token.FileSet, callExpr *ast.CallExpr) *cpg.Expression { var c *cpg.CallExpression + + // In Go, regular cast expressions (not type asserts are modelled as calls). + // In this case, the Fun contains a type expression. + switch v := callExpr.Fun.(type) { + case *ast.ArrayType, + *ast.StructType, + *ast.FuncType, + *ast.InterfaceType, + *ast.MapType, + *ast.ChanType: + this.LogDebug("Handling cast expression: %#v", callExpr) + + cast := this.NewCastExpression(fset, callExpr) + cast.SetCastType(this.handleType(fset, v)) + + if len(callExpr.Args) > 1 { + cast.SetExpression(this.handleExpr(fset, callExpr.Args[0])) + } + + return (*cpg.Expression)(cast) + } + // parse the Fun field, to see which kind of expression it is var reference = this.handleExpr(fset, callExpr.Fun) @@ -842,13 +993,13 @@ func (this *GoLanguageFrontend) handleCallExpr(fset *token.FileSet, callExpr *as } if isMemberExpression { - this.LogDebug("Fun is a member call to %s", name) + this.LogTrace("Fun is a member call to %s", name) m := this.NewMemberCallExpression(fset, callExpr, reference) c = (*cpg.CallExpression)(m) } else { - this.LogDebug("Handling regular call expression to %s", name) + this.LogTrace("Handling regular call expression to %s", name) c = this.NewCallExpression(fset, callExpr, reference, name) } @@ -861,25 +1012,50 @@ func (this *GoLanguageFrontend) handleCallExpr(fset *token.FileSet, callExpr *as } } - // reference.disconnectFromGraph() - return (*cpg.Expression)(c) } -func (this *GoLanguageFrontend) handleIndexExpr(fset *token.FileSet, indexExpr *ast.IndexExpr) *cpg.Expression { +func (this *GoLanguageFrontend) handleIndexExpr(fset *token.FileSet, indexExpr *ast.IndexExpr) *cpg.ArraySubscriptionExpression { a := this.NewArraySubscriptionExpression(fset, indexExpr) a.SetArrayExpression(this.handleExpr(fset, indexExpr.X)) a.SetSubscriptExpression(this.handleExpr(fset, indexExpr.Index)) - return (*cpg.Expression)(a) + return a +} + +// handleSliceExpr handles a [ast.SliceExpr], which is an extended version of +// [ast.IndexExpr]. We are modelling this as a combination of a +// [cpg.ArraySubscriptionExpression] that contains a [cpg.SliceExpression] as +// its subscriptExpression to share some code between this and an index +// expression. +func (this *GoLanguageFrontend) handleSliceExpr(fset *token.FileSet, sliceExpr *ast.SliceExpr) *cpg.ArraySubscriptionExpression { + a := this.NewArraySubscriptionExpression(fset, sliceExpr) + + a.SetArrayExpression(this.handleExpr(fset, sliceExpr.X)) + + // Build the slice expression + s := this.NewSliceExpression(fset, sliceExpr) + if sliceExpr.Low != nil { + s.SetLowerBound(this.handleExpr(fset, sliceExpr.Low)) + } + if sliceExpr.High != nil { + s.SetUpperBound(this.handleExpr(fset, sliceExpr.High)) + } + if sliceExpr.Max != nil { + s.SetThird(this.handleExpr(fset, sliceExpr.Max)) + } + + a.SetSubscriptExpression((*cpg.Expression)(s)) + + return a } func (this *GoLanguageFrontend) handleNewExpr(fset *token.FileSet, callExpr *ast.CallExpr) *cpg.Expression { n := this.NewNewExpression(fset, callExpr) // first argument is type - t := this.handleType(callExpr.Args[0]) + t := this.handleType(fset, callExpr.Args[0]) // new is a pointer, so need to reference the type with a pointer var pointer = jnigi.NewObjectRef(cpg.PointerOriginClass) @@ -907,7 +1083,7 @@ func (this *GoLanguageFrontend) handleMakeExpr(fset *token.FileSet, callExpr *as } // first argument is always the type, handle it - t := this.handleType(callExpr.Args[0]) + t := this.handleType(fset, callExpr.Args[0]) // actually make() can make more than just arrays, i.e. channels and maps if _, isArray := callExpr.Args[0].(*ast.ArrayType); isArray { @@ -949,13 +1125,8 @@ func (this *GoLanguageFrontend) handleBinaryExpr(fset *token.FileSet, binaryExpr lhs := this.handleExpr(fset, binaryExpr.X) rhs := this.handleExpr(fset, binaryExpr.Y) - if lhs != nil { - b.SetLHS(lhs) - } - - if rhs != nil { - b.SetRHS(rhs) - } + b.SetLHS(lhs) + b.SetRHS(rhs) return b } @@ -1003,7 +1174,7 @@ func (this *GoLanguageFrontend) handleSelectorExpr(fset *token.FileSet, selector // we need to set the name to a FQN-style, including the package scope. the call resolver will then resolve this fqn := fmt.Sprintf("%s.%s", base.GetName(), selectorExpr.Sel.Name) - this.LogDebug("Trying to parse the fqn '%s'", fqn) + this.LogTrace("Trying to parse the fqn '%s'", fqn) name := this.ParseName(fqn) @@ -1032,7 +1203,7 @@ func (this *GoLanguageFrontend) handleSelectorExpr(fset *token.FileSet, selector } func (this *GoLanguageFrontend) handleKeyValueExpr(fset *token.FileSet, expr *ast.KeyValueExpr) *cpg.KeyValueExpression { - this.LogDebug("Handling key value expression %+v", *expr) + this.LogTrace("Handling key value expression %+v", *expr) k := this.NewKeyValueExpression(fset, expr) @@ -1050,7 +1221,7 @@ func (this *GoLanguageFrontend) handleKeyValueExpr(fset *token.FileSet, expr *as } func (this *GoLanguageFrontend) handleBasicLit(fset *token.FileSet, lit *ast.BasicLit) *cpg.Literal { - this.LogDebug("Handling literal %+v", *lit) + this.LogTrace("Handling literal %+v", *lit) var value cpg.Castable var t *cpg.Type @@ -1075,8 +1246,11 @@ func (this *GoLanguageFrontend) handleBasicLit(fset *token.FileSet, lit *ast.Bas value = cpg.NewDouble(f) t = cpg.TypeParser_createFrom("float64", lang) case token.IMAG: + // TODO + t = &cpg.UnknownType_getUnknown(lang).Type case token.CHAR: value = cpg.NewString(lit.Value) + t = cpg.TypeParser_createFrom("rune", lang) break } @@ -1089,12 +1263,12 @@ func (this *GoLanguageFrontend) handleBasicLit(fset *token.FileSet, lit *ast.Bas // ConstructExpression and a list of KeyValueExpressions. The problem is that we need to add the list // as a first argument of the construct expression. func (this *GoLanguageFrontend) handleCompositeLit(fset *token.FileSet, lit *ast.CompositeLit) *cpg.ConstructExpression { - this.LogDebug("Handling composite literal %+v", *lit) + this.LogTrace("Handling composite literal %+v", *lit) c := this.NewConstructExpression(fset, lit) // parse the type field, to see which kind of expression it is - var typ = this.handleType(lit.Type) + var typ = this.handleType(fset, lit.Type) if typ != nil { (*cpg.Node)(c).SetName(typ.GetName()) @@ -1110,17 +1284,37 @@ func (this *GoLanguageFrontend) handleCompositeLit(fset *token.FileSet, lit *ast // from its initialization. c.AddPrevDFG((*cpg.Node)(l)) + var exprs = []*cpg.Expression{} for _, elem := range lit.Elts { expr := this.handleExpr(fset, elem) if expr != nil { - l.AddInitializer(expr) + exprs = append(exprs, expr) } } + l.SetInitializers(exprs) + return c } +// handleFuncLit handles a function literal, which we need to translate into a combination of a +// LambdaExpression and a function declaration. +func (this *GoLanguageFrontend) handleFuncLit(fset *token.FileSet, lit *ast.FuncLit) *cpg.LambdaExpression { + this.LogTrace("Handling function literal %#v", *lit) + + l := this.NewLambdaExpression(fset, lit) + + // Parse the expression as a function declaration with a little trick + funcDecl := this.handleFuncDecl(fset, &ast.FuncDecl{Type: lit.Type, Body: lit.Body, Name: ast.NewIdent("")}) + + this.LogTrace("Function of literal is: %#v", funcDecl) + + l.SetFunction(funcDecl) + + return l +} + func (this *GoLanguageFrontend) handleIdent(fset *token.FileSet, ident *ast.Ident) *cpg.Expression { lang, err := this.GetLanguage() if err != nil { @@ -1158,18 +1352,21 @@ func (this *GoLanguageFrontend) handleTypeAssertExpr(fset *token.FileSet, assert expr := this.handleExpr(fset, assert.X) // Parse the type - typ := this.handleType(assert.Type) + typ := this.handleType(fset, assert.Type) cast.SetExpression(expr) - cast.SetCastType(typ) + + if typ != nil { + cast.SetCastType(typ) + } return cast } -func (this *GoLanguageFrontend) handleType(typeExpr ast.Expr) *cpg.Type { +func (this *GoLanguageFrontend) handleType(fset *token.FileSet, typeExpr ast.Expr) *cpg.Type { var err error - this.LogDebug("Parsing type %T: %+v", typeExpr, typeExpr) + this.LogTrace("Parsing type %T: %s", typeExpr, code(fset, typeExpr)) lang, err := this.GetLanguage() if err != nil { @@ -1181,20 +1378,20 @@ func (this *GoLanguageFrontend) handleType(typeExpr ast.Expr) *cpg.Type { var name string if this.isBuiltinType(v.Name) { name = v.Name - this.LogDebug("non-fqn type: %s", name) + this.LogTrace("non-fqn type: %s", name) } else { name = fmt.Sprintf("%s.%s", this.File.Name.Name, v.Name) - this.LogDebug("fqn type: %s", name) + this.LogTrace("fqn type: %s", name) } return cpg.TypeParser_createFrom(name, lang) case *ast.SelectorExpr: // small shortcut fqn := fmt.Sprintf("%s.%s", v.X.(*ast.Ident).Name, v.Sel.Name) - this.LogDebug("FQN type: %s", fqn) + this.LogTrace("FQN type: %s", fqn) return cpg.TypeParser_createFrom(fqn, lang) case *ast.StarExpr: - t := this.handleType(v.X) + t := this.handleType(fset, v.X) var i = jnigi.NewObjectRef(cpg.PointerOriginClass) err = env.GetStaticField(cpg.PointerOriginClass, "POINTER", i) @@ -1202,11 +1399,11 @@ func (this *GoLanguageFrontend) handleType(typeExpr ast.Expr) *cpg.Type { log.Fatal(err) } - this.LogDebug("Pointer to %s", t.GetName()) + this.LogTrace("Pointer to %s", t.GetName()) return t.Reference(i) case *ast.ArrayType: - t := this.handleType(v.Elt) + t := this.handleType(fset, v.Elt) var i = jnigi.NewObjectRef(cpg.PointerOriginClass) err = env.GetStaticField(cpg.PointerOriginClass, "ARRAY", i) @@ -1214,27 +1411,27 @@ func (this *GoLanguageFrontend) handleType(typeExpr ast.Expr) *cpg.Type { log.Fatal(err) } - this.LogDebug("Array of %s", t.GetName()) + this.LogTrace("Array of %s", t.GetName()) return t.Reference(i) case *ast.MapType: // we cannot properly represent Golangs built-in map types, yet so we have // to make a shortcut here and represent it as a Java-like map type. t := cpg.TypeParser_createFrom("map", lang) - keyType := this.handleType(v.Key) - valueType := this.handleType(v.Value) + keyType := this.handleType(fset, v.Key) + valueType := this.handleType(fset, v.Value) // TODO(oxisto): Find a better way to represent casts - (&(cpg.ObjectType{Type: *t})).AddGeneric(keyType) - (&(cpg.ObjectType{Type: *t})).AddGeneric(valueType) + (*cpg.ObjectType)(t).AddGeneric(keyType) + (*cpg.ObjectType)(t).AddGeneric(valueType) return t case *ast.ChanType: // handle them similar to maps t := cpg.TypeParser_createFrom("chan", lang) - chanType := this.handleType(v.Value) + chanType := this.handleType(fset, v.Value) - (&(cpg.ObjectType{Type: *t})).AddGeneric(chanType) + (*cpg.ObjectType)(t).AddGeneric(chanType) return t case *ast.FuncType: @@ -1243,7 +1440,7 @@ func (this *GoLanguageFrontend) handleType(typeExpr ast.Expr) *cpg.Type { var returnTypes = []*cpg.Type{} for _, param := range v.Params.List { - parameterTypes = append(parameterTypes, this.handleType(param.Type)) + parameterTypes = append(parameterTypes, this.handleType(fset, param.Type)) } parametersTypesList, err = cpg.ListOf(parameterTypes) @@ -1253,7 +1450,7 @@ func (this *GoLanguageFrontend) handleType(typeExpr ast.Expr) *cpg.Type { if v.Results != nil { for _, ret := range v.Results.List { - returnTypes = append(returnTypes, this.handleType(ret.Type)) + returnTypes = append(returnTypes, this.handleType(fset, ret.Type)) } } @@ -1277,6 +1474,29 @@ func (this *GoLanguageFrontend) handleType(typeExpr ast.Expr) *cpg.Type { } return &cpg.Type{ObjectRef: t} + case *ast.InterfaceType: + var name = "interface{" + // We do not really support dedicated interfaces types, so all we can for now + // is parse it as an object type with a pseudo-name + for _, method := range v.Methods.List { + name += this.handleType(fset, method.Type).GetName().ToString() + } + + name += "}" + + return cpg.TypeParser_createFrom(name, lang) + case *ast.IndexExpr: + // This is a type with one type parameter. First we need to parse the "X" expression as a type + var t = this.handleType(fset, v.X) + + // Then we parse the "Index" as a type parameter + var genericType = this.handleType(fset, v.Index) + + (*cpg.ObjectType)(t).AddGeneric(genericType) + + return t + default: + this.LogError("Not parsing type of type %T yet. Defaulting to unknown type", v) } return &cpg.UnknownType_getUnknown(lang).Type diff --git a/cpg-language-go/src/main/golang/frontend/statement_builder.go b/cpg-language-go/src/main/golang/frontend/statement_builder.go index f483536e5ee..f951d413926 100644 --- a/cpg-language-go/src/main/golang/frontend/statement_builder.go +++ b/cpg-language-go/src/main/golang/frontend/statement_builder.go @@ -54,6 +54,10 @@ func (frontend *GoLanguageFrontend) NewForStatement(fset *token.FileSet, astNode return (*cpg.ForStatement)(frontend.NewStatement("ForStatement", fset, astNode)) } +func (frontend *GoLanguageFrontend) NewForEachStatement(fset *token.FileSet, astNode ast.Node) *cpg.ForEachStatement { + return (*cpg.ForEachStatement)(frontend.NewStatement("ForEachStatement", fset, astNode)) +} + func (frontend *GoLanguageFrontend) NewSwitchStatement(fset *token.FileSet, astNode ast.Node) *cpg.SwitchStatement { return (*cpg.SwitchStatement)(frontend.NewStatement("SwitchStatement", fset, astNode)) } diff --git a/cpg-language-go/src/main/golang/lib/cpg/main.go b/cpg-language-go/src/main/golang/lib/cpg/main.go index 9ca8326023b..eb28bdee51f 100644 --- a/cpg-language-go/src/main/golang/lib/cpg/main.go +++ b/cpg-language-go/src/main/golang/lib/cpg/main.go @@ -58,6 +58,7 @@ func Java_de_fraunhofer_aisec_cpg_frontends_golang_GoLanguageFrontend_parseInter nil, nil, ast.CommentMap{}, + "", nil, } @@ -86,6 +87,8 @@ func Java_de_fraunhofer_aisec_cpg_frontends_golang_GoLanguageFrontend_parseInter log.Fatal(err) } + goFrontend.TopLevel = string(topLevel) + fset := token.NewFileSet() file, err := parser.ParseFile(fset, string(path), string(src), parser.ParseComments) if err != nil { diff --git a/cpg-language-go/src/main/golang/statements.go b/cpg-language-go/src/main/golang/statements.go index 481a2fe5886..35bfcb001a6 100644 --- a/cpg-language-go/src/main/golang/statements.go +++ b/cpg-language-go/src/main/golang/statements.go @@ -38,6 +38,7 @@ type SwitchStatement Statement type CaseStatement Statement type DefaultStatement Statement type ForStatement Statement +type ForEachStatement Statement const StatementsPackage = GraphPackage + "/statements" const StatementClass = StatementsPackage + "/Statement" @@ -51,6 +52,10 @@ func (f *DeclarationStatement) SetSingleDeclaration(d *Declaration) { (*jnigi.ObjectRef)(f).CallMethod(env, "setSingleDeclaration", nil, (*jnigi.ObjectRef)(d).Cast(DeclarationClass)) } +func (f *DeclarationStatement) AddToPropertyEdgeDeclaration(d *Declaration) { + (*jnigi.ObjectRef)(f).CallMethod(env, "addToPropertyEdgeDeclaration", nil, (*jnigi.ObjectRef)(d).Cast(DeclarationClass)) +} + func (m *IfStatement) SetThenStatement(s *Statement) { (*jnigi.ObjectRef)(m).SetField(env, "thenStatement", (*jnigi.ObjectRef)(s).Cast(StatementClass)) } @@ -91,6 +96,18 @@ func (fw *ForStatement) SetStatement(s *Statement) { (*jnigi.ObjectRef)(fw).SetField(env, "statement", (*jnigi.ObjectRef)(s).Cast(StatementClass)) } +func (fw *ForEachStatement) SetStatement(s *Statement) { + (*jnigi.ObjectRef)(fw).SetField(env, "statement", (*jnigi.ObjectRef)(s).Cast(StatementClass)) +} + +func (fw *ForEachStatement) SetIterable(s *Statement) { + (*jnigi.ObjectRef)(fw).CallMethod(env, "setIterable", nil, (*jnigi.ObjectRef)(s).Cast(StatementClass)) +} + +func (fw *ForEachStatement) SetVariable(s *Statement) { + (*jnigi.ObjectRef)(fw).SetField(env, "variable", (*jnigi.ObjectRef)(s).Cast(StatementClass)) +} + func (fw *ForStatement) SetIterationStatement(s *Statement) { (*jnigi.ObjectRef)(fw).SetField(env, "iterationStatement", (*jnigi.ObjectRef)(s).Cast(StatementClass)) } diff --git a/cpg-language-go/src/main/golang/types.go b/cpg-language-go/src/main/golang/types.go index 73e6bd48cf2..0c03ef5abdf 100644 --- a/cpg-language-go/src/main/golang/types.go +++ b/cpg-language-go/src/main/golang/types.go @@ -37,6 +37,7 @@ import ( var env *jnigi.Env type Type struct{ *jnigi.ObjectRef } +type ObjectType Type const TypesPackage = GraphPackage + "/types" const TypeClass = TypesPackage + "/Type" @@ -64,10 +65,6 @@ func (*Type) IsArray() bool { return false } -type ObjectType struct { - Type -} - func (*ObjectType) GetClassName() string { return ObjectTypeClass } diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguage.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguage.kt index 7e25efa95f2..62a1e748ca8 100644 --- a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguage.kt +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguage.kt @@ -29,12 +29,14 @@ import de.fraunhofer.aisec.cpg.ScopeManager import de.fraunhofer.aisec.cpg.TranslationConfiguration import de.fraunhofer.aisec.cpg.frontends.HasGenerics import de.fraunhofer.aisec.cpg.frontends.HasShortCircuitOperators +import de.fraunhofer.aisec.cpg.frontends.HasStructs import de.fraunhofer.aisec.cpg.frontends.Language import de.fraunhofer.aisec.cpg.graph.types.* import org.neo4j.ogm.annotation.Transient /** The Go language. */ -class GoLanguage : Language(), HasShortCircuitOperators, HasGenerics { +class GoLanguage : + Language(), HasShortCircuitOperators, HasGenerics, HasStructs { override val fileExtensions = listOf("go") override val namespaceDelimiter = "." @Transient override val frontend = GoLanguageFrontend::class diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontend.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontend.kt index 1c10b07a190..82e8ebafb37 100644 --- a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontend.kt +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontend.kt @@ -32,11 +32,14 @@ import de.fraunhofer.aisec.cpg.frontends.LanguageFrontend import de.fraunhofer.aisec.cpg.frontends.SupportsParallelParsing import de.fraunhofer.aisec.cpg.frontends.TranslationException import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration +import de.fraunhofer.aisec.cpg.passes.GoExtraPass +import de.fraunhofer.aisec.cpg.passes.order.RegisterExtraPass import de.fraunhofer.aisec.cpg.sarif.PhysicalLocation import java.io.File import java.io.FileOutputStream @SupportsParallelParsing(false) +@RegisterExtraPass(GoExtraPass::class) class GoLanguageFrontend( language: Language, config: TranslationConfiguration, @@ -84,7 +87,7 @@ class GoLanguageFrontend( override fun parse(file: File): TranslationUnitDeclaration { return parseInternal( file.readText(Charsets.UTF_8), - file.path, + file.absolutePath, config.topLevel?.absolutePath ?: file.parent ) } diff --git a/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/DeclarationTest.kt b/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/DeclarationTest.kt index 569c429b1b4..1b6a3783895 100644 --- a/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/DeclarationTest.kt +++ b/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/DeclarationTest.kt @@ -27,11 +27,15 @@ package de.fraunhofer.aisec.cpg.frontends.golang import de.fraunhofer.aisec.cpg.TestUtils import de.fraunhofer.aisec.cpg.assertFullName +import de.fraunhofer.aisec.cpg.assertLiteralValue +import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.byNameOrNull import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.MethodDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.NamespaceDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.RecordDeclaration +import de.fraunhofer.aisec.cpg.graph.statements.expressions.DeclaredReferenceExpression +import de.fraunhofer.aisec.cpg.graph.variables import java.nio.file.Path import kotlin.test.* @@ -113,4 +117,57 @@ class DeclarationTest { assertContains(myInterface.superTypeDeclarations, myOtherInterface) assertTrue(myInterface.superClasses.any { it.name == myOtherInterface.name }) } + + @Test + fun testMultipleDeclarations() { + val topLevel = Path.of("src", "test", "resources", "golang") + val tu = + TestUtils.analyzeAndGetFirstTU( + listOf(topLevel.resolve("declare.go").toFile()), + topLevel, + true + ) { + it.registerLanguage() + } + assertNotNull(tu) + + val main = tu.functions["main.main"] + assertNotNull(main) + + // We should have 7 variables (a, b, c, d, e, f, g) + assertEquals(7, tu.variables.size) + + // Four should have (literal) initializers + val a = main.variables["a"] + assertLiteralValue(1, a?.initializer) + + val b = main.variables["b"] + assertLiteralValue(2, b?.initializer) + + val c = main.variables["c"] + assertLiteralValue(3, c?.initializer) + + val d = main.variables["d"] + assertLiteralValue(4, d?.initializer) + + // The next two variables are using a short assignment, therefore they do not have an + // initializer, but we can use the firstAssignment function + val e = main.variables["e"] + assertLiteralValue(5, e?.firstAssignment) + + val f = main.variables["f"] + assertLiteralValue(6, f?.firstAssignment) + + // And they should all be connected to the arguments of the Printf call + val printf = main.calls["Printf"] + assertNotNull(printf) + + printf.arguments.drop(1).forEach { + val ref = assertIs(it) + assertNotNull(ref.refersTo) + } + + // We have eight assignments in total (6 initializers + 2 assign expressions) + assertEquals(8, tu.assignments.size) + } } diff --git a/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/ExpressionTest.kt b/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/ExpressionTest.kt index 033439f2746..f319d87a4fe 100644 --- a/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/ExpressionTest.kt +++ b/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/ExpressionTest.kt @@ -27,22 +27,19 @@ package de.fraunhofer.aisec.cpg.frontends.golang import de.fraunhofer.aisec.cpg.TestUtils import de.fraunhofer.aisec.cpg.assertFullName +import de.fraunhofer.aisec.cpg.assertLiteralValue +import de.fraunhofer.aisec.cpg.assertLocalName +import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.bodyOrNull -import de.fraunhofer.aisec.cpg.graph.byNameOrNull -import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration -import de.fraunhofer.aisec.cpg.graph.declarations.NamespaceDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration import de.fraunhofer.aisec.cpg.graph.statements.DeclarationStatement -import de.fraunhofer.aisec.cpg.graph.statements.expressions.CastExpression -import de.fraunhofer.aisec.cpg.graph.statements.expressions.DeclaredReferenceExpression +import de.fraunhofer.aisec.cpg.graph.statements.expressions.* import java.nio.file.Path -import kotlin.test.assertNotNull -import kotlin.test.assertSame -import org.junit.jupiter.api.Test +import kotlin.test.* class ExpressionTest { @Test - fun testTypeAssert() { + fun testCastExpression() { val topLevel = Path.of("src", "test", "resources", "golang") val tu = TestUtils.analyzeAndGetFirstTU( @@ -54,10 +51,10 @@ class ExpressionTest { } assertNotNull(tu) - val main = tu.byNameOrNull("main") + val main = tu.namespaces["main"] assertNotNull(main) - val mainFunc = main.byNameOrNull("main") + val mainFunc = main.functions["main"] assertNotNull(mainFunc) val f = @@ -74,5 +71,69 @@ class ExpressionTest { assertNotNull(cast) assertFullName("main.MyStruct", cast.castType) assertSame(f, (cast.expression as? DeclaredReferenceExpression)?.refersTo) + + val ignored = main.variables("_") + ignored.forEach { assertIs(it.initializer) } + } + + @Test + fun testSliceExpression() { + val topLevel = Path.of("src", "test", "resources", "golang") + val tu = + TestUtils.analyzeAndGetFirstTU( + listOf(topLevel.resolve("slices.go").toFile()), + topLevel, + true + ) { + it.registerLanguage() + } + assertNotNull(tu) + + val a = tu.variables["a"] + assertNotNull(a) + assertLocalName("int[]", a.type) + + val b = tu.variables["b"] + assertNotNull(b) + assertLocalName("int[]", b.type) + + // [:1] + var slice = + assertIs( + assertIs(b.initializer).subscriptExpression + ) + assertNull(slice.lowerBound) + assertLiteralValue(1, slice.upperBound) + assertNull(slice.third) + + val c = tu.variables["c"] + assertNotNull(c) + assertLocalName("int[]", c.type) + + // [1:] + slice = assertIs(assertIs(c.initializer).subscriptExpression) + assertLiteralValue(1, slice.lowerBound) + assertNull(slice.upperBound) + assertNull(slice.third) + + val d = tu.variables["d"] + assertNotNull(d) + assertLocalName("int[]", d.type) + + // [0:1] + slice = assertIs(assertIs(d.initializer).subscriptExpression) + assertLiteralValue(0, slice.lowerBound) + assertLiteralValue(1, slice.upperBound) + assertNull(slice.third) + + val e = tu.variables["e"] + assertNotNull(e) + assertLocalName("int[]", e.type) + + // [0:1:1] + slice = assertIs(assertIs(e.initializer).subscriptExpression) + assertLiteralValue(0, slice.lowerBound) + assertLiteralValue(1, slice.upperBound) + assertLiteralValue(1, slice.third) } } diff --git a/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontendTest.kt b/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontendTest.kt index 2ffdb45db91..80f557a2dd8 100644 --- a/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontendTest.kt +++ b/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontendTest.kt @@ -31,16 +31,16 @@ import de.fraunhofer.aisec.cpg.TestUtils.analyzeAndGetFirstTU import de.fraunhofer.aisec.cpg.assertFullName import de.fraunhofer.aisec.cpg.assertLocalName import de.fraunhofer.aisec.cpg.graph.* -import de.fraunhofer.aisec.cpg.graph.declarations.* +import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration +import de.fraunhofer.aisec.cpg.graph.declarations.NamespaceDeclaration +import de.fraunhofer.aisec.cpg.graph.declarations.RecordDeclaration +import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration import de.fraunhofer.aisec.cpg.graph.statements.* import de.fraunhofer.aisec.cpg.graph.statements.expressions.* import de.fraunhofer.aisec.cpg.graph.types.FunctionType import de.fraunhofer.aisec.cpg.graph.types.TypeParser import java.nio.file.Path -import kotlin.test.Test -import kotlin.test.assertEquals -import kotlin.test.assertNotNull -import kotlin.test.assertTrue +import kotlin.test.* class GoLanguageFrontendTest : BaseTest() { @@ -53,19 +53,19 @@ class GoLanguageFrontendTest : BaseTest() { } assertNotNull(tu) - val p = tu.byNameOrNull("p") + val p = tu.namespaces["p"] assertNotNull(p) - val main = p.byNameOrNull("main") + val main = p.functions["main"] assertNotNull(main) - val message = - main.bodyOrNull(2)?.singleDeclaration as? VariableDeclaration + val message = main.variables["message"] assertNotNull(message) val map = - ((message.initializer as? ConstructExpression)?.arguments?.firstOrNull() - as? InitializerListExpression) + assertIs( + assertIs(message.firstAssignment).arguments.firstOrNull() + ) assertNotNull(map) val nameEntry = map.initializers.firstOrNull() as? KeyValueExpression @@ -83,18 +83,18 @@ class GoLanguageFrontendTest : BaseTest() { } assertNotNull(tu) - val p = tu.byNameOrNull("p") + val p = tu.namespaces["p"] assertNotNull(p) - val main = p.byNameOrNull("main") + val main = p.functions["main"] assertNotNull(main) - val data = main.bodyOrNull(0)?.singleDeclaration + val data = main.variables["data"] assertNotNull(data) // We should be able to follow the DFG backwards from the declaration to the individual // key/value expressions - val path = data.followPrevDFG { it is KeyValueExpression } + val path = data.usages.firstOrNull()?.followPrevDFG { it is KeyValueExpression } assertNotNull(path) assertEquals(4, path.size) @@ -113,26 +113,22 @@ class GoLanguageFrontendTest : BaseTest() { } assertNotNull(tu) - val p = tu.byNameOrNull("p") + val p = tu.namespaces["p"] assertNotNull(p) - val myStruct = p.byNameOrNull("p.MyStruct") + val myStruct = p.records["p.MyStruct"] assertNotNull(myStruct) - val main = p.byNameOrNull("main") + val main = p.functions["main"] assertNotNull(main) val body = main.body as? CompoundStatement assertNotNull(body) - var stmt = main.body(0) - assertNotNull(stmt) - - var decl = stmt.singleDeclaration as? VariableDeclaration + var decl = main.variables["o"] assertNotNull(decl) - val new = decl.initializer as? NewExpression - assertNotNull(new) + val new = assertIs(decl.firstAssignment) assertEquals(TypeParser.createFrom("p.MyStruct*", GoLanguage()), new.type) val construct = new.initializer as? ConstructExpression @@ -141,13 +137,10 @@ class GoLanguageFrontendTest : BaseTest() { // make array - stmt = main.body(1) - assertNotNull(stmt) - - decl = stmt.singleDeclaration as? VariableDeclaration + decl = main.variables["a"] assertNotNull(decl) - var make = decl.initializer + var make = assertIs(decl.firstAssignment) assertNotNull(make) assertEquals(TypeParser.createFrom("int[]", GoLanguage()), make.type) @@ -158,29 +151,23 @@ class GoLanguageFrontendTest : BaseTest() { assertEquals(5, dimension.value) // make map - stmt = main.body(2) - assertNotNull(stmt) - decl = stmt.singleDeclaration as? VariableDeclaration + decl = main.variables["m"] assertNotNull(decl) - make = decl.initializer + make = assertIs(decl.firstAssignment) assertNotNull(make) assertTrue(make is ConstructExpression) // TODO: Maps can have dedicated types and parsing them as a generic here is only a - // temporary solution. - // This should be fixed in the future. + // temporary solution. This should be fixed in the future. assertEquals(TypeParser.createFrom("map[string,string]", GoLanguage()), make.type) // make channel - stmt = main.body(3) - assertNotNull(stmt) - - decl = stmt.singleDeclaration as? VariableDeclaration + decl = main.variables["ch"] assertNotNull(decl) - make = decl.initializer + make = assertIs(decl.firstAssignment) assertNotNull(make) assertTrue(make is ConstructExpression) assertEquals(TypeParser.createFrom("chan[int]", GoLanguage()), make.type) @@ -196,32 +183,32 @@ class GoLanguageFrontendTest : BaseTest() { assertNotNull(tu) - val p = tu.byNameOrNull("p") + val p = tu.namespaces["p"] assertNotNull(p) - val a = p.byNameOrNull("a") + val a = p.variables["a"] assertNotNull(a) assertNotNull(a.location) assertLocalName("a", a) assertEquals(TypeParser.createFrom("int", GoLanguage()), a.type) - val s = p.byNameOrNull("s") + val s = p.variables["s"] assertNotNull(s) assertLocalName("s", s) assertEquals(TypeParser.createFrom("string", GoLanguage()), s.type) - val f = p.byNameOrNull("f") + val f = p.variables["f"] assertNotNull(f) assertLocalName("f", f) assertEquals(TypeParser.createFrom("float64", GoLanguage()), f.type) - val f32 = p.byNameOrNull("f32") + val f32 = p.variables["f32"] assertNotNull(f32) assertLocalName("f32", f32) assertEquals(TypeParser.createFrom("float32", GoLanguage()), f32.type) - val n = p.byNameOrNull("n") + val n = p.variables["n"] assertNotNull(n) assertEquals(TypeParser.createFrom("int*", GoLanguage()), n.type) @@ -229,6 +216,18 @@ class GoLanguageFrontendTest : BaseTest() { assertNotNull(nil) assertLocalName("nil", nil) assertEquals(null, nil.value) + + val fn = p.variables["fn"] + assertNotNull(fn) + + val lambda = assertIs(fn.initializer) + assertNotNull(lambda) + + val func = lambda.function + assertNotNull(func) + assertFullName("", func) + assertEquals(1, func.parameters.size) + assertEquals(1, func.returnTypes.size) } @Test @@ -268,7 +267,7 @@ class GoLanguageFrontendTest : BaseTest() { var body = main.body as? CompoundStatement assertNotNull(body) - var callExpression = body.statements.first() as? CallExpression + var callExpression = body.calls.firstOrNull() assertNotNull(callExpression) assertLocalName("myTest", callExpression) @@ -302,17 +301,15 @@ class GoLanguageFrontendTest : BaseTest() { assertLocalName("s", ref) assertEquals(s, ref.refersTo) - val stmt = body.statements[1] as? BinaryOperator + val stmt = body.statements[1] as? AssignExpression assertNotNull(stmt) - val a = stmt.lhs as? DeclaredReferenceExpression + val a = stmt.lhs.firstOrNull() as? DeclaredReferenceExpression assertNotNull(a) assertLocalName("a", a) - val op = stmt.rhs as? BinaryOperator - assertNotNull(op) - + val op = assertIs(stmt.rhs.firstOrNull()) assertEquals("+", op.operatorCode) val lhs = op.lhs as? Literal<*> @@ -325,14 +322,11 @@ class GoLanguageFrontendTest : BaseTest() { assertEquals(2, rhs.value) - val binOp = body.statements[2] as? BinaryOperator - - assertNotNull(binOp) - - val err = binOp.lhs + val binOp = assertIs(body.statements[2]) + val err = binOp.lhs.firstOrNull() assertNotNull(err) - assertEquals(TypeParser.createFrom("error", GoLanguage()), err.type) + assertLocalName("error", err.type) } @Test @@ -460,16 +454,16 @@ class GoLanguageFrontendTest : BaseTest() { val body = myFunc.body as? CompoundStatement assertNotNull(body) - val binOp = body.statements.first() as? BinaryOperator - assertNotNull(binOp) + val assign = body.statements.first() as? AssignExpression + assertNotNull(assign) - val lhs = binOp.lhs as? MemberExpression + val lhs = assign.lhs.firstOrNull() as? MemberExpression assertNotNull(lhs) assertEquals(myFunc.receiver, (lhs.base as? DeclaredReferenceExpression)?.refersTo) assertLocalName("Field", lhs) assertEquals(TypeParser.createFrom("int", GoLanguage()), lhs.type) - val rhs = binOp.rhs as? DeclaredReferenceExpression + val rhs = assign.rhs.firstOrNull() as? DeclaredReferenceExpression assertNotNull(rhs) assertFullName("otherPackage.OtherField", rhs) } @@ -590,15 +584,13 @@ class GoLanguageFrontendTest : BaseTest() { val body = main.body as? CompoundStatement assertNotNull(body) - val c = - (body.statements[0] as? DeclarationStatement)?.singleDeclaration as? VariableDeclaration + val c = body.variables["c"] assertNotNull(c) // type will be inferred from the function declaration assertEquals(TypeParser.createFrom("p.MyStruct*", GoLanguage()), c.type) - val newMyStruct = c.initializer as? CallExpression - assertNotNull(newMyStruct) + val newMyStruct = assertIs(c.firstAssignment) // fetch the function declaration from the other TU val tu2 = tus[1] @@ -609,12 +601,15 @@ class GoLanguageFrontendTest : BaseTest() { val newMyStructDef = p2.functions["NewMyStruct"] assertTrue(newMyStruct.invokes.contains(newMyStructDef)) - val call = body.statements[1] as? MemberCallExpression + val call = body.statements[2] as? MemberCallExpression assertNotNull(call) val base = call.base as? DeclaredReferenceExpression assertNotNull(base) assertEquals(c, base.refersTo) + + val go = main.calls["go"] + assertNotNull(go) } @Test @@ -631,15 +626,30 @@ class GoLanguageFrontendTest : BaseTest() { it.registerLanguage() } - val main = tu.functions["p.main"] + val main = tu.functions["main.main"] assertNotNull(main) val f = main.bodyOrNull() assertNotNull(f) assertTrue(f.condition is BinaryOperator) assertTrue(f.statement is CompoundStatement) - assertTrue(f.initializerStatement is DeclarationStatement) + assertTrue(f.initializerStatement is AssignExpression) assertTrue(f.iterationStatement is UnaryOperator) + + val each = main.bodyOrNull() + assertNotNull(each) + + val bytes = assertIs(each.iterable) + assertLocalName("bytes", bytes) + assertNotNull(bytes.refersTo) + + val idx = assertIs(each.variable).variables["idx"] + assertNotNull(idx) + assertLocalName("int", idx.type) + + val b = assertIs(each.variable).variables["b"] + assertNotNull(b) + assertLocalName("uint8", b.type) } @Test @@ -673,12 +683,21 @@ class GoLanguageFrontendTest : BaseTest() { val main = tu1.functions["main.main"] assertNotNull(main) - val a = main.getBodyStatementAs(0, DeclarationStatement::class.java) + val a = main.variables["a"] assertNotNull(a) - val call = (a.singleDeclaration as? VariableDeclaration)?.initializer as? CallExpression + val call = a.firstAssignment as? CallExpression assertNotNull(call) assertTrue(call.invokes.contains(newAwesome)) + + val util = result.namespaces["util"] + assertNotNull(util) + + // Check, if we correctly inferred this function in the namespace + val doSomethingElse = util.functions["DoSomethingElse"] + assertNotNull(doSomethingElse) + assertTrue(doSomethingElse.isInferred) + assertSame(util, doSomethingElse.scope?.astNode) } @Test @@ -691,10 +710,10 @@ class GoLanguageFrontendTest : BaseTest() { assertNotNull(tu) - val mainNamespace = tu.byNameOrNull("main") + val mainNamespace = tu.namespaces["main"] assertNotNull(mainNamespace) - val main = mainNamespace.byNameOrNull("main") + val main = mainNamespace.functions["main"] assertNotNull(main) assertEquals("comment before function", main.comment) @@ -706,11 +725,11 @@ class GoLanguageFrontendTest : BaseTest() { assertNotNull(j) assertEquals("comment before parameter2", j.comment) - var declStmt = main.bodyOrNull() - assertNotNull(declStmt) - assertEquals("comment before assignment", declStmt.comment) + val assign = main.bodyOrNull() + assertNotNull(assign) + assertEquals("comment before assignment", assign.comment) - declStmt = main.bodyOrNull(1) + val declStmt = main.bodyOrNull() assertNotNull(declStmt) assertEquals("comment before declaration", declStmt.comment) @@ -737,9 +756,31 @@ class GoLanguageFrontendTest : BaseTest() { val main = mainPackage.byNameOrNull("main") assertNotNull(main) - val binOp = main.bodyOrNull() - assertNotNull(binOp) + val assign = main.bodyOrNull() + assertNotNull(assign) + assertEquals(1, assign.rhs.size) + + assertNotNull(tu) + } + @Test + fun testAssign() { + val topLevel = Path.of("src", "test", "resources", "golang") + val tu = + analyzeAndGetFirstTU(listOf(topLevel.resolve("function.go").toFile()), topLevel, true) { + it.registerLanguage() + } assertNotNull(tu) + + val i = tu.variables["i"] + + val assign = + tu.functions["main"].assignments.firstOrNull { + (it.target as? DeclaredReferenceExpression)?.refersTo == i + } + assertNotNull(assign) + + val call = assertIs(assign.value) + assertLocalName("myTest", call) } } diff --git a/cpg-language-go/src/test/resources/golang/call.go b/cpg-language-go/src/test/resources/golang/call.go index b4fc3284942..9ecadbeb096 100644 --- a/cpg-language-go/src/test/resources/golang/call.go +++ b/cpg-language-go/src/test/resources/golang/call.go @@ -5,4 +5,7 @@ import ("http") func main() { c := NewMyStruct() c.myOtherFunc() + + go c.MyFunc() + go c.MyFunc() } diff --git a/cpg-language-go/src/test/resources/golang/dfg.go b/cpg-language-go/src/test/resources/golang/dfg.go index 67d8a787fde..2d282d64722 100644 --- a/cpg-language-go/src/test/resources/golang/dfg.go +++ b/cpg-language-go/src/test/resources/golang/dfg.go @@ -1,5 +1,7 @@ package p +import "db" + func main() int { data := &Data{Name: name} db.Create(data) diff --git a/cpg-language-go/src/test/resources/golang/for.go b/cpg-language-go/src/test/resources/golang/for.go index 67ec298a6cd..e3f0829b407 100644 --- a/cpg-language-go/src/test/resources/golang/for.go +++ b/cpg-language-go/src/test/resources/golang/for.go @@ -1,7 +1,18 @@ -package p +package main + +import "fmt" func main() { - for i := 0; i < 5; i++ { - do() + var bytes = []byte{1,2,3,4} + + // Regular old-school for loop + for i := 0; i < 4; i++ { + fmt.Printf("bytes[%d]=%d\n", i, bytes[i]) + } + + // For-each style loop with range expression with key and value. idx and b are created using + // the short assignment syntax. Its scope is limited to the for-block. + for idx, b := range bytes { + fmt.Printf("bytes[%d]=%d; idx=%T b=%T\n", idx, b, idx, b) } } \ No newline at end of file diff --git a/cpg-language-go/src/test/resources/golang/function.go b/cpg-language-go/src/test/resources/golang/function.go index 0720767f829..e8984d0a400 100644 --- a/cpg-language-go/src/test/resources/golang/function.go +++ b/cpg-language-go/src/test/resources/golang/function.go @@ -3,15 +3,22 @@ package p import "fmt" func main() { - myTest("some string") + var i int + var err error + + i, err = myTest("some string") + + if err == nil { + fmt.Printf("%d", i) + } } func myTest(s string) (a int, err error) { - fmt.Printf("%s", s) + fmt.Printf("%s", s) - a = 1 + 2 + a = 1 + 2 - err = nil + err = nil - return + return } diff --git a/cpg-language-go/src/test/resources/golang/go.mod b/cpg-language-go/src/test/resources/golang/go.mod index 745649b879a..12cb5307fb5 100644 --- a/cpg-language-go/src/test/resources/golang/go.mod +++ b/cpg-language-go/src/test/resources/golang/go.mod @@ -1 +1 @@ -module p +module mymodule diff --git a/cpg-language-go/src/test/resources/golang/literal.go b/cpg-language-go/src/test/resources/golang/literal.go index 63cdc1c7e59..0450e78c4bf 100644 --- a/cpg-language-go/src/test/resources/golang/literal.go +++ b/cpg-language-go/src/test/resources/golang/literal.go @@ -5,3 +5,7 @@ const s = "test" const f = 1.0 const f32 float32 = 1.00 var n *int = nil + +var fn = func(_ int) int { + return 1 +} \ No newline at end of file diff --git a/cpg-language-go/src/test/resources/golang/type_assert.go b/cpg-language-go/src/test/resources/golang/type_assert.go index cbee1cb06be..4f42ac27989 100644 --- a/cpg-language-go/src/test/resources/golang/type_assert.go +++ b/cpg-language-go/src/test/resources/golang/type_assert.go @@ -13,4 +13,8 @@ func main () { var s = f.(MyStruct) fmt.Printf("%+v", s) + + var _ = MyInterface(s) + var _ = interface{}(s) + var _ = any(s) } \ No newline at end of file diff --git a/cpg-language-go/src/test/resources/log4j2.xml b/cpg-language-go/src/test/resources/log4j2.xml index 359d8071bf3..747860628a4 100644 --- a/cpg-language-go/src/test/resources/log4j2.xml +++ b/cpg-language-go/src/test/resources/log4j2.xml @@ -2,7 +2,7 @@ - + diff --git a/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/ApplicationTest.kt b/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/ApplicationTest.kt index 0f9a88e3b4e..2d1d4428671 100644 --- a/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/ApplicationTest.kt +++ b/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/ApplicationTest.kt @@ -29,6 +29,7 @@ import de.fraunhofer.aisec.cpg.TranslationConfiguration import de.fraunhofer.aisec.cpg.TranslationManager import de.fraunhofer.aisec.cpg.TranslationResult import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration +import de.fraunhofer.aisec.cpg.graph.functions import java.io.File import java.nio.file.Paths import kotlin.test.Test @@ -60,6 +61,8 @@ class ApplicationTest { TranslationManager.builder().config(translationConfiguration).build() translationResult = translationManager.analyze().get() + assertEquals(31, translationResult.functions.size) + val application = Application() application.pushToNeo4j(translationResult!!) @@ -71,7 +74,7 @@ class ApplicationTest { val functions = session.loadAll(FunctionDeclaration::class.java) assertNotNull(functions) - assertEquals(38, functions.size) + assertEquals(31, functions.size) transaction.commit() }