Skip to content

Commit

Permalink
Improvements to the Go language
Browse files Browse the repository at this point in the history
* Added parsing of more expressions
  • Loading branch information
oxisto committed Aug 10, 2023
1 parent c2982d0 commit d4c4c01
Show file tree
Hide file tree
Showing 12 changed files with 305 additions and 84 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -38,14 +38,11 @@ import java.io.File
import java.io.PrintWriter
import java.lang.reflect.InvocationTargetException
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.attribute.BasicFileAttributes
import java.util.*
import java.util.concurrent.CompletableFuture
import java.util.concurrent.CompletionException
import java.util.concurrent.ExecutionException
import java.util.concurrent.atomic.AtomicBoolean
import java.util.stream.Collectors
import kotlin.reflect.full.findAnnotation
import org.slf4j.LoggerFactory

Expand Down Expand Up @@ -148,15 +145,13 @@ private constructor(
val list =
sourceLocations.flatMap { file ->
if (file.isDirectory) {
Files.find(
file.toPath(),
999,
{ _: Path?, fileAttr: BasicFileAttributes ->
fileAttr.isRegularFile
}
)
.map { it.toFile() }
.collect(Collectors.toList())
val files =
file
.walkTopDown()
.onEnter { !it.name.startsWith(".") }
.filter { it.isFile && !it.name.startsWith(".") }
.toList()
files
} else {
val frontendClass = file.language?.frontend
val supportsParallelParsing =
Expand Down Expand Up @@ -277,7 +272,7 @@ private constructor(
Thread.currentThread().interrupt()
} catch (e: ExecutionException) {
log.error("Error parsing ${futureToFile[future]}", e)
Thread.currentThread().interrupt()
// Thread.currentThread().interrupt()
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,11 +34,9 @@ import de.fraunhofer.aisec.cpg.TranslationContext
import de.fraunhofer.aisec.cpg.graph.Name
import de.fraunhofer.aisec.cpg.graph.Node
import de.fraunhofer.aisec.cpg.graph.statements.expressions.BinaryOperator
import de.fraunhofer.aisec.cpg.graph.statements.expressions.CastExpression
import de.fraunhofer.aisec.cpg.graph.types.*
import de.fraunhofer.aisec.cpg.graph.types.Type
import de.fraunhofer.aisec.cpg.graph.unknownType
import de.fraunhofer.aisec.cpg.helpers.Util
import java.io.File
import kotlin.reflect.KClass
import kotlin.reflect.full.primaryConstructor
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,10 @@ fun LanguageProvider?.parseName(fqn: CharSequence) =

/** Tries to parse the given fully qualified name using the specified [delimiter] into a [Name]. */
internal fun parseName(fqn: CharSequence, delimiter: String, vararg splitDelimiters: String): Name {
if (fqn is Name) {
return fqn
}

val parts = fqn.split(delimiter, *splitDelimiters)

var name: Name? = null
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -638,19 +638,11 @@ internal class CXXLanguageFrontendTest : BaseTest() {
// a = b * 2
var assign = statements[2] as? AssignExpression
assertNotNull(assign)
<<<<<<< HEAD

var ref = assign.lhs<DeclaredReferenceExpression>()
assertNotNull(ref)
assertLocalName("a", ref)

=======

var ref = assign.lhs<DeclaredReferenceExpression>()
assertNotNull(ref)
assertLocalName("a", ref)

>>>>>>> f8cf854d8 (Overhaul of type propagation (#1268))
var binOp = assign.rhs<BinaryOperator>()
assertNotNull(binOp)

Expand All @@ -666,23 +658,10 @@ internal class CXXLanguageFrontendTest : BaseTest() {
ref = assign.lhs<DeclaredReferenceExpression>()
assertNotNull(ref)
assertLocalName("a", ref)
<<<<<<< HEAD
=======
<<<<<<< HEAD
>>>>>>> f8cf854d8 (Overhaul of type propagation (#1268))

binOp = assign.rhs<BinaryOperator>()
assertNotNull(binOp)

<<<<<<< HEAD
=======
=======

binOp = assign.rhs<BinaryOperator>()
assertNotNull(binOp)

>>>>>>> 490562dce (Overhaul of type propagation (#1268))
>>>>>>> f8cf854d8 (Overhaul of type propagation (#1268))
assertTrue(binOp.lhs is Literal<*>)
assertEquals(1, (binOp.lhs as Literal<*>).value)
assertTrue(binOp.rhs is Literal<*>)
Expand Down Expand Up @@ -1613,22 +1592,4 @@ internal class CXXLanguageFrontendTest : BaseTest() {
assertNotNull(ptr)
assertLocalName("decltype(nullptr)", ptr.type)
}

@Test
fun testRecursiveHeaderFunction() {
val file = File("src/test/resources/cxx/fix-1226")
val result =
analyze(
listOf(file.resolve("main1.cpp"), file.resolve("main2.cpp")),
file.toPath(),
true
)
assertNotNull(result)

// For now, we have duplicate functions because we include the header twice. This might
// change in the future. The important thing is that this gets parsed at all because we
// previously had a loop in our equals method
val functions = result.functions { it.name.localName == "foo" && it.isDefinition }
assertEquals(2, functions.size)
}
}
32 changes: 32 additions & 0 deletions cpg-language-go/src/main/golang/lib/cpg/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -286,6 +286,12 @@ func GetCallExprArg(ptr unsafe.Pointer, i int) unsafe.Pointer {
})
}

//export GetEllipsisElt
func GetEllipsisElt(ptr unsafe.Pointer) unsafe.Pointer {
expr := restore[*ast.Ellipsis](ptr)
return save(expr.Elt)
}

//export GetForStmtInit
func GetForStmtInit(ptr unsafe.Pointer) unsafe.Pointer {
stmt := restore[*ast.ForStmt](ptr)
Expand Down Expand Up @@ -466,6 +472,26 @@ func GetIndexExprIndex(ptr unsafe.Pointer) unsafe.Pointer {
return save(expr.Index)
}

//export GetIndexListExprX
func GetIndexListExprX(ptr unsafe.Pointer) unsafe.Pointer {
expr := restore[*ast.IndexListExpr](ptr)
return save(expr.X)
}

//export GetNumIndexListExprIndices
func GetNumIndexListExprIndices(ptr unsafe.Pointer) C.int {
return num[*ast.IndexListExpr](ptr, func(t *ast.IndexListExpr) []ast.Expr {
return t.Indices
})
}

//export GetIndexListExprIndex
func GetIndexListExprIndex(ptr unsafe.Pointer, i int) unsafe.Pointer {
return item[*ast.IndexListExpr](ptr, i, func(t *ast.IndexListExpr) []ast.Expr {
return t.Indices
})
}

//export GetKeyValueExprKey
func GetKeyValueExprKey(ptr unsafe.Pointer) unsafe.Pointer {
kv := restore[*ast.KeyValueExpr](ptr)
Expand All @@ -478,6 +504,12 @@ func GetKeyValueExprValue(ptr unsafe.Pointer) unsafe.Pointer {
return save(kv.Value)
}

//export GetParenExprX
func GetParenExprX(ptr unsafe.Pointer) unsafe.Pointer {
p := restore[*ast.ParenExpr](ptr)
return save(p.X)
}

//export GetSelectorExprX
func GetSelectorExprX(ptr unsafe.Pointer) unsafe.Pointer {
sel := restore[*ast.SelectorExpr](ptr)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -162,14 +162,7 @@ class DeclarationHandler(frontend: GoLanguageFrontend) :

// Check for varargs. In this case we want to parse the element type
// (and make it an array afterwards)
var variadic = false
val type =
if (param.type is GoStandardLibrary.Ast.Ellipsis) {
variadic = true
frontend.typeOf((param.type as GoStandardLibrary.Ast.Ellipsis).elt).array()
} else {
frontend.typeOf(param.type)
}
val (type, variadic) = frontend.fieldTypeOf(param)

val p = newParamVariableDeclaration(name, type, variadic, rawNode = param)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,12 @@ class ExpressionHandler(frontend: GoLanguageFrontend) :
is GoStandardLibrary.Ast.IndexExpr -> handleIndexExpr(expr)
is GoStandardLibrary.Ast.CallExpr -> handleCallExpr(expr)
is GoStandardLibrary.Ast.KeyValueExpr -> handleKeyValueExpr(expr)
is GoStandardLibrary.Ast.ParenExpr -> {
return handle(expr.x)
}
is GoStandardLibrary.Ast.SelectorExpr -> handleSelectorExpr(expr)
is GoStandardLibrary.Ast.SliceExpr -> handleSliceExpr(expr)
is GoStandardLibrary.Ast.StarExpr -> handleStarExpr(expr)
is GoStandardLibrary.Ast.TypeAssertExpr -> handleTypeAssertExpr(expr)
is GoStandardLibrary.Ast.UnaryExpr -> handleUnaryExpr(expr)
else -> {
Expand All @@ -69,7 +73,12 @@ class ExpressionHandler(frontend: GoLanguageFrontend) :
type = primitiveType("string")
}
INT -> {
value = rawValue.toInt()
value =
if (rawValue.startsWith("0x")) {
rawValue.substring(2).toInt(16)
} else {
rawValue.toInt()
}
type = primitiveType("int")
}
FLOAT -> {
Expand Down Expand Up @@ -154,10 +163,25 @@ class ExpressionHandler(frontend: GoLanguageFrontend) :
}
}

// Parse the Fun field, to see which kind of expression it is
val typeConstraints = mutableListOf<Node>()

val callee =
this.handle(callExpr.`fun`)
?: return ProblemExpression("Could not parse call expr without fun")
when (val `fun` = callExpr.`fun`) {
// If "fun" is either an index or an index list expression, this is a call with type
// constraints. We do not fully support that yet, but we can at least try to set
// some of the parameters as template parameters
is GoStandardLibrary.Ast.IndexExpr -> {
typeConstraints += frontend.typeOf(`fun`.index)
this.handle(`fun`.x)
}
is GoStandardLibrary.Ast.IndexListExpr -> {
typeConstraints.addAll(`fun`.indices.map { frontend.typeOf(it) })
this.handle(`fun`.x)
}
else -> {
this.handle(callExpr.`fun`)
}
}

// Handle special functions, such as make and new in a special way
val name = callee.name.localName
Expand All @@ -175,9 +199,18 @@ class ExpressionHandler(frontend: GoLanguageFrontend) :
newCallExpression(callee, name, rawNode = callExpr)
}

// TODO(oxisto) Add type constraints
if (typeConstraints.isNotEmpty()) {
log.debug(
"Call {} has type constraints ({}), but we cannot add them to the call expression yet",
call.name,
typeConstraints.joinToString(", ") { it.name }
)
}

// Parse and add call arguments
for (arg in callExpr.args) {
handle(arg)?.let { call += it }
call += handle(arg)
}

return call
Expand Down Expand Up @@ -304,6 +337,13 @@ class ExpressionHandler(frontend: GoLanguageFrontend) :
return ase
}

private fun handleStarExpr(starExpr: GoStandardLibrary.Ast.StarExpr): UnaryOperator {
val op = newUnaryOperator("*", postfix = false, prefix = false, rawNode = starExpr)
op.input = handle(starExpr.x)

return op
}

private fun handleTypeAssertExpr(
typeAssertExpr: GoStandardLibrary.Ast.TypeAssertExpr
): CastExpression {
Expand All @@ -328,7 +368,7 @@ class ExpressionHandler(frontend: GoLanguageFrontend) :
prefix = false,
rawNode = unaryExpr
)
handle(unaryExpr.x)?.let { op.input = it }
op.input = handle(unaryExpr.x)

return op
}
Expand All @@ -341,13 +381,16 @@ class ExpressionHandler(frontend: GoLanguageFrontend) :
private fun handleCompositeLit(
compositeLit: GoStandardLibrary.Ast.CompositeLit
): ConstructExpression {
// Parse the type field, to see which kind of expression it is
val type = frontend.typeOf(compositeLit.type)
// Parse the type field, to see which kind of expression it is. The type of a composite
// literal can be omitted if this is an "inner" composite literal, so we need to set it from
// the "outer" one. See below
val type = compositeLit.type?.let { frontend.typeOf(it) } ?: unknownType()

val construct = newConstructExpression(type.name, rawNode = compositeLit)
construct.type = type

val list = newInitializerListExpression(type, rawNode = compositeLit)
list.type = type
construct += list

// Normally, the construct expression would not have DFG edge, but in this case we are
Expand All @@ -357,7 +400,34 @@ class ExpressionHandler(frontend: GoLanguageFrontend) :

val expressions = mutableListOf<Expression>()
for (elem in compositeLit.elts) {
handle(elem)?.let { expressions += it }
var expression = handle(elem)

// The type of a "inner" composite literal can be omitted if the outer one is creating
// an array type. In this case, we need to set the type manually because the type for
// the "inner" one is empty.
// Example code:
// ```go
// var a = []*MyObject{
// {
// Name: "a",
// },
// {
// Name: "b",
// }
// }
if (
type is PointerType &&
type.isArray &&
elem is GoStandardLibrary.Ast.CompositeLit &&
expression is ConstructExpression
) {
val elementType = type.dereference()
// Set the type of the inner composite to be the deref'd type of the outer
expression.type = type
(expression as? ConstructExpression)?.arguments?.firstOrNull()?.type = elementType
}

expressions += expression
}

list.initializers = expressions
Expand Down
Loading

0 comments on commit d4c4c01

Please sign in to comment.