From e86261cc1f5a3478553b866f0ab67224be455f37 Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Sun, 20 Aug 2023 01:19:45 +0200 Subject: [PATCH] Very hacky way of parsing depdenencies --- .../aisec/cpg/TranslationConfiguration.kt | 24 +++++++++++- .../aisec/cpg/TranslationContext.kt | 11 +++++- .../aisec/cpg/TranslationManager.kt | 33 +++++++++++++--- .../aisec/cpg/frontends/LanguageFrontend.kt | 2 + .../frontends/golang/GoLanguageFrontend.kt | 39 +++++++++++++++++-- .../golang/GoLanguageFrontendTest.kt | 28 +++++++++++++ .../test/resources/golang-std/fmt/print.go | 6 +++ .../src/test/resources/golang-std/go.mod | 3 ++ 8 files changed, 134 insertions(+), 12 deletions(-) create mode 100644 cpg-language-go/src/test/resources/golang-std/fmt/print.go create mode 100644 cpg-language-go/src/test/resources/golang-std/go.mod diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt index 2f051ba1932..7bc9e4680dd 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt @@ -53,7 +53,7 @@ private constructor( val symbols: Map, /** Source code files to parse. */ val softwareComponents: Map>, - val topLevel: File?, + var topLevel: File?, /** Set to true to generate debug output for the parser. */ val debugParser: Boolean, /** @@ -106,6 +106,7 @@ private constructor( useUnityBuild: Boolean, useParallelFrontends: Boolean, inferenceConfiguration: InferenceConfiguration, + frontendConfiguration: Map>, LanguageFrontend.Configuration>, compilationDatabase: CompilationDatabase?, matchCommentsToNodes: Boolean, addIncludesToGraph: Boolean @@ -165,6 +166,9 @@ private constructor( /** This sub configuration object holds all information about inference and smart-guessing. */ val inferenceConfiguration: InferenceConfiguration + val frontendConfiguration: + Map>, LanguageFrontend.Configuration> + init { registeredPasses = passes this.languages = languages @@ -175,6 +179,7 @@ private constructor( this.useUnityBuild = useUnityBuild this.useParallelFrontends = useParallelFrontends this.inferenceConfiguration = inferenceConfiguration + this.frontendConfiguration = frontendConfiguration this.compilationDatabase = compilationDatabase this.matchCommentsToNodes = matchCommentsToNodes this.addIncludesToGraph = addIncludesToGraph @@ -224,6 +229,8 @@ private constructor( private var useUnityBuild = false private var useParallelFrontends = false private var inferenceConfiguration = InferenceConfiguration.Builder().build() + private var frontendConfiguration = + mutableMapOf>, LanguageFrontend.Configuration>() private var compilationDatabase: CompilationDatabase? = null private var matchCommentsToNodes = false private var addIncludesToGraph = true @@ -583,6 +590,20 @@ private constructor( return this } + inline fun > configureFrontend( + configuration: LanguageFrontend.Configuration + ): Builder { + return configureFrontend(T::class, configuration) + } + + fun > configureFrontend( + clazz: KClass, + configuration: LanguageFrontend.Configuration + ): Builder { + frontendConfiguration[clazz] = configuration + return this + } + @Throws(ConfigurationException::class) fun build(): TranslationConfiguration { registerExtraFrontendPasses() @@ -606,6 +627,7 @@ private constructor( useUnityBuild, useParallelFrontends, inferenceConfiguration, + frontendConfiguration, compilationDatabase, matchCommentsToNodes, addIncludesToGraph diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationContext.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationContext.kt index c309225e624..8b8fecda4db 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationContext.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationContext.kt @@ -25,6 +25,10 @@ */ package de.fraunhofer.aisec.cpg +import de.fraunhofer.aisec.cpg.graph.Component +import java.io.File +import java.util.concurrent.ConcurrentHashMap + /** * The translation context holds all necessary managers and configurations needed during the * translation process. @@ -45,5 +49,8 @@ class TranslationContext( * The type manager is responsible for managing type information. Currently, we have one * instance of a [TypeManager] for the overall [TranslationResult]. */ - val typeManager: TypeManager -) + val typeManager: TypeManager, + val additionalLocations: MutableMap> = ConcurrentHashMap(), +) { + var component: Component? = null +} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationManager.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationManager.kt index 7ce143eef72..062779c6461 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationManager.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationManager.kt @@ -132,6 +132,8 @@ private constructor( result: TranslationResult ): Set> { val usedFrontends = mutableSetOf>() + var useParallelFrontends = ctx.config.useParallelFrontends + for (sc in ctx.config.softwareComponents.keys) { val component = Component() component.name = Name(sc) @@ -139,8 +141,6 @@ private constructor( var sourceLocations: List = ctx.config.softwareComponents[sc] ?: listOf() - var useParallelFrontends = ctx.config.useParallelFrontends - val list = sourceLocations.flatMap { file -> if (file.isDirectory) { @@ -180,7 +180,7 @@ private constructor( val cxxExtensions = listOf(".c", ".cpp", ".cc", ".cxx") if (cxxExtensions.contains(Util.getExtension(it))) { if (ctx.config.topLevel != null) { - val topLevel = ctx.config.topLevel.toPath() + val topLevel = ctx.config.topLevel!!.toPath() writer.write( """ #include "${topLevel.relativize(it.toPath())}" @@ -220,6 +220,25 @@ private constructor( parseSequentially(component, result, ctx, sourceLocations) } ) + + // Parse any additional dependencies that were gathered during the first parsing + usedFrontends.addAll( + if (useParallelFrontends) { + parseParallel( + component, + result, + ctx, + ctx.additionalLocations[component] ?: listOf() + ) + } else { + parseSequentially( + component, + result, + ctx, + ctx.additionalLocations[component] ?: listOf() + ) + } + ) } return usedFrontends @@ -293,9 +312,7 @@ private constructor( val usedFrontends = mutableSetOf>() for (sourceLocation in sourceLocations) { - log.info("Parsing {}", sourceLocation.absolutePath) - - var f = parse(component, ctx, sourceLocation) + val f = parse(component, ctx, sourceLocation) if (f != null) { handleCompletion(result, usedFrontends, sourceLocation, f) } @@ -340,6 +357,10 @@ private constructor( } return null } + + log.info("Parsing {}", sourceLocation.absolutePath) + + ctx.component = component component.translationUnits.add(frontend.parse(sourceLocation)) } catch (ex: TranslationException) { log.error("An error occurred during parsing of ${sourceLocation.name}: ${ex.message}") diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/LanguageFrontend.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/LanguageFrontend.kt index 6ecb61fcf7e..b2f1a87486f 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/LanguageFrontend.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/LanguageFrontend.kt @@ -66,6 +66,8 @@ abstract class LanguageFrontend( val typeManager: TypeManager = ctx.typeManager val config: TranslationConfiguration = ctx.config + abstract class Configuration {} + init { this.scopeManager.lang = this } diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontend.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontend.kt index ee93f364308..ecc596e0d41 100644 --- a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontend.kt +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontend.kt @@ -77,16 +77,25 @@ class GoLanguageFrontend(language: Language, ctx: Translatio var statementHandler = StatementHandler(this) var expressionHandler = ExpressionHandler(this) + class Configuration(val stdLibFile: File? = null) : LanguageFrontend.Configuration() {} + @Throws(TranslationException::class) override fun parse(file: File): TranslationUnitDeclaration { + var stdLibFile = + (ctx.config.frontendConfiguration[this::class] as? Configuration)?.stdLibFile + // Make sure, that our top level is set either way - val topLevel = + var topLevel = if (config.topLevel != null) { config.topLevel } else { file.parentFile }!! + // HACK: We should find out where the "top level" for the dependency is instead + if (stdLibFile != null && file.absolutePath.contains(stdLibFile.path)) { + topLevel = stdLibFile + } val std = GoStandardLibrary.INSTANCE // Try to parse a possible go.mod @@ -110,6 +119,26 @@ class GoLanguageFrontend(language: Language, ctx: Translatio for (spec in f.imports) { val import = specificationHandler.handle(spec) scopeManager.addDeclaration(import) + + // If the name is unqualified, we assume that this is a standard library import, which + // we can only process if we know where it is + if ( + import != null && + import.name.parent == null && + ctx.component != null && + stdLibFile != null + ) { + // Add the corresponding folder to the list of additional source locations + var files = + ctx.additionalLocations.computeIfAbsent(ctx.component!!) { mutableListOf() } + files += + stdLibFile + .resolve(import.name.localName) + .walkTopDown() + .onEnter { !it.name.startsWith(".") } + .filter { it.isFile && !it.name.startsWith(".") } + .toList() + } } val p = newNamespaceDeclaration(f.name.name) @@ -120,8 +149,12 @@ class GoLanguageFrontend(language: Language, ctx: Translatio // module path as well as the current directory in relation to the topLevel var packagePath = file.parentFile.relativeTo(topLevel) - // If we are in a module, we need to prepend the module path to it - currentModule?.let { packagePath = File(it.module.mod.path).resolve(packagePath) } + // If we are in a module, we need to prepend the module path to it. There is an + // exception if we are in the "std" module, which represents the standard library + val modulePath = currentModule?.module?.mod?.path + if (modulePath != null && modulePath != "std") { + packagePath = File(modulePath).resolve(packagePath) + } p.path = packagePath.path } catch (ex: IllegalArgumentException) { diff --git a/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontendTest.kt b/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontendTest.kt index a6afc3c4754..530177bf9c5 100644 --- a/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontendTest.kt +++ b/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontendTest.kt @@ -41,6 +41,7 @@ import de.fraunhofer.aisec.cpg.graph.statements.expressions.* import de.fraunhofer.aisec.cpg.graph.types.FunctionType import de.fraunhofer.aisec.cpg.graph.types.ObjectType import de.fraunhofer.aisec.cpg.graph.types.PointerType +import java.io.File import java.nio.file.Path import kotlin.test.* @@ -790,4 +791,31 @@ class GoLanguageFrontendTest : BaseTest() { assertNotNull(g) assertLocalName("string", g.type) } + + @Test + fun testResolveStdLibImport() { + val topLevel = Path.of("src", "test", "resources", "golang") + val tu = + analyzeAndGetFirstTU(listOf(topLevel.resolve("function.go").toFile()), topLevel, true) { + it.registerLanguage() + it.configureFrontend( + GoLanguageFrontend.Configuration(File("src/test/resources/golang-std")) + ) + } + + assertNotNull(tu) + + val p = tu.namespaces["p"] + assertNotNull(p) + + val main = p.functions["main"] + assertNotNull(main) + + val printfCall = main.calls["fmt.Printf"] + assertNotNull(printfCall) + + val printf = printfCall.invokes.firstOrNull() + assertNotNull(printf) + assertEquals("print.go", File(printf.location?.artifactLocation?.uri?.path.toString()).name) + } } diff --git a/cpg-language-go/src/test/resources/golang-std/fmt/print.go b/cpg-language-go/src/test/resources/golang-std/fmt/print.go new file mode 100644 index 00000000000..b9b025641d6 --- /dev/null +++ b/cpg-language-go/src/test/resources/golang-std/fmt/print.go @@ -0,0 +1,6 @@ +package fmt + +func Printf(format string, a ...any) (n int, err error) { + // Not a real implementation, and we are ignoring it anyway + return 0, nil +} diff --git a/cpg-language-go/src/test/resources/golang-std/go.mod b/cpg-language-go/src/test/resources/golang-std/go.mod new file mode 100644 index 00000000000..e299934a599 --- /dev/null +++ b/cpg-language-go/src/test/resources/golang-std/go.mod @@ -0,0 +1,3 @@ +module std + +go 1.21 \ No newline at end of file