Very hacky way of parsing depdenencies

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt

@@ -53,7 +53,7 @@ private constructor(
     val symbols: Map<String, String>,
     /** Source code files to parse. */
     val softwareComponents: Map<String, List<File>>,
-    val topLevel: File?,
+    var topLevel: File?,
     /** Set to true to generate debug output for the parser. */
     val debugParser: Boolean,
     /**
@@ -106,6 +106,7 @@ private constructor(
     useUnityBuild: Boolean,
     useParallelFrontends: Boolean,
     inferenceConfiguration: InferenceConfiguration,
+    frontendConfiguration: Map<KClass<out LanguageFrontend<*, *>>, LanguageFrontend.Configuration>,
     compilationDatabase: CompilationDatabase?,
     matchCommentsToNodes: Boolean,
     addIncludesToGraph: Boolean
@@ -165,6 +166,9 @@ private constructor(
     /** This sub configuration object holds all information about inference and smart-guessing. */
     val inferenceConfiguration: InferenceConfiguration
 
+    val frontendConfiguration:
+        Map<KClass<out LanguageFrontend<*, *>>, LanguageFrontend.Configuration>
+
     init {
         registeredPasses = passes
         this.languages = languages
@@ -175,6 +179,7 @@ private constructor(
         this.useUnityBuild = useUnityBuild
         this.useParallelFrontends = useParallelFrontends
         this.inferenceConfiguration = inferenceConfiguration
+        this.frontendConfiguration = frontendConfiguration
         this.compilationDatabase = compilationDatabase
         this.matchCommentsToNodes = matchCommentsToNodes
         this.addIncludesToGraph = addIncludesToGraph
@@ -224,6 +229,8 @@ private constructor(
         private var useUnityBuild = false
         private var useParallelFrontends = false
         private var inferenceConfiguration = InferenceConfiguration.Builder().build()
+        private var frontendConfiguration =
+            mutableMapOf<KClass<out LanguageFrontend<*, *>>, LanguageFrontend.Configuration>()
         private var compilationDatabase: CompilationDatabase? = null
         private var matchCommentsToNodes = false
         private var addIncludesToGraph = true
@@ -583,6 +590,20 @@ private constructor(
             return this
         }
 
+        inline fun <reified T : LanguageFrontend<*, *>> configureFrontend(
+            configuration: LanguageFrontend.Configuration
+        ): Builder {
+            return configureFrontend(T::class, configuration)
+        }
+
+        fun <T : LanguageFrontend<*, *>> configureFrontend(
+            clazz: KClass<T>,
+            configuration: LanguageFrontend.Configuration
+        ): Builder {
+            frontendConfiguration[clazz] = configuration
+            return this
+        }
+
         @Throws(ConfigurationException::class)
         fun build(): TranslationConfiguration {
             registerExtraFrontendPasses()
@@ -606,6 +627,7 @@ private constructor(
                 useUnityBuild,
                 useParallelFrontends,
                 inferenceConfiguration,
+                frontendConfiguration,
                 compilationDatabase,
                 matchCommentsToNodes,
                 addIncludesToGraph

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationContext.kt

@@ -25,6 +25,10 @@
  */
 package de.fraunhofer.aisec.cpg
 
+import de.fraunhofer.aisec.cpg.graph.Component
+import java.io.File
+import java.util.concurrent.ConcurrentHashMap
+
 /**
  * The translation context holds all necessary managers and configurations needed during the
  * translation process.
@@ -45,5 +49,8 @@ class TranslationContext(
      * The type manager is responsible for managing type information. Currently, we have one
      * instance of a [TypeManager] for the overall [TranslationResult].
      */
-    val typeManager: TypeManager
-)
+    val typeManager: TypeManager,
+    val additionalLocations: MutableMap<Component, MutableList<File>> = ConcurrentHashMap(),
+) {
+    var component: Component? = null
+}

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationManager.kt

@@ -132,15 +132,15 @@ private constructor(
         result: TranslationResult
     ): Set<LanguageFrontend<*, *>> {
         val usedFrontends = mutableSetOf<LanguageFrontend<*, *>>()
+        var useParallelFrontends = ctx.config.useParallelFrontends
+
         for (sc in ctx.config.softwareComponents.keys) {
             val component = Component()
             component.name = Name(sc)
             result.addComponent(component)
 
             var sourceLocations: List<File> = ctx.config.softwareComponents[sc] ?: listOf()
 
-            var useParallelFrontends = ctx.config.useParallelFrontends
-
             val list =
                 sourceLocations.flatMap { file ->
                     if (file.isDirectory) {
@@ -180,7 +180,7 @@ private constructor(
                         val cxxExtensions = listOf(".c", ".cpp", ".cc", ".cxx")
                         if (cxxExtensions.contains(Util.getExtension(it))) {
                             if (ctx.config.topLevel != null) {
-                                val topLevel = ctx.config.topLevel.toPath()
+                                val topLevel = ctx.config.topLevel!!.toPath()
                                 writer.write(
                                     """
 #include "${topLevel.relativize(it.toPath())}"
@@ -220,6 +220,25 @@ private constructor(
                     parseSequentially(component, result, ctx, sourceLocations)
                 }
             )
+
+            // Parse any additional dependencies that were gathered during the first parsing
+            usedFrontends.addAll(
+                if (useParallelFrontends) {
+                    parseParallel(
+                        component,
+                        result,
+                        ctx,
+                        ctx.additionalLocations[component] ?: listOf()
+                    )
+                } else {
+                    parseSequentially(
+                        component,
+                        result,
+                        ctx,
+                        ctx.additionalLocations[component] ?: listOf()
+                    )
+                }
+            )
         }
 
         return usedFrontends
@@ -293,9 +312,7 @@ private constructor(
         val usedFrontends = mutableSetOf<LanguageFrontend<*, *>>()
 
         for (sourceLocation in sourceLocations) {
-            log.info("Parsing {}", sourceLocation.absolutePath)
-
-            var f = parse(component, ctx, sourceLocation)
+            val f = parse(component, ctx, sourceLocation)
             if (f != null) {
                 handleCompletion(result, usedFrontends, sourceLocation, f)
             }
@@ -340,6 +357,10 @@ private constructor(
                 }
                 return null
             }
+
+            log.info("Parsing {}", sourceLocation.absolutePath)
+
+            ctx.component = component
             component.translationUnits.add(frontend.parse(sourceLocation))
         } catch (ex: TranslationException) {
             log.error("An error occurred during parsing of ${sourceLocation.name}: ${ex.message}")

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/LanguageFrontend.kt

@@ -66,6 +66,8 @@ abstract class LanguageFrontend<in AstNode, TypeNode>(
     val typeManager: TypeManager = ctx.typeManager
     val config: TranslationConfiguration = ctx.config
 
+    abstract class Configuration {}
+
     init {
         this.scopeManager.lang = this
     }

cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontend.kt

@@ -77,16 +77,25 @@ class GoLanguageFrontend(language: Language<GoLanguageFrontend>, ctx: Translatio
     var statementHandler = StatementHandler(this)
     var expressionHandler = ExpressionHandler(this)
 
+    class Configuration(val stdLibFile: File? = null) : LanguageFrontend.Configuration() {}
+
     @Throws(TranslationException::class)
     override fun parse(file: File): TranslationUnitDeclaration {
+        var stdLibFile =
+            (ctx.config.frontendConfiguration[this::class] as? Configuration)?.stdLibFile
+
         // Make sure, that our top level is set either way
-        val topLevel =
+        var topLevel =
             if (config.topLevel != null) {
                 config.topLevel
             } else {
                 file.parentFile
             }!!
 
+        // HACK: We should find out where the "top level" for the dependency is instead
+        if (stdLibFile != null && file.absolutePath.contains(stdLibFile.path)) {
+            topLevel = stdLibFile
+        }
         val std = GoStandardLibrary.INSTANCE
 
         // Try to parse a possible go.mod
@@ -110,6 +119,26 @@ class GoLanguageFrontend(language: Language<GoLanguageFrontend>, ctx: Translatio
         for (spec in f.imports) {
             val import = specificationHandler.handle(spec)
             scopeManager.addDeclaration(import)
+
+            // If the name is unqualified, we assume that this is a standard library import, which
+            // we can only process if we know where it is
+            if (
+                import != null &&
+                    import.name.parent == null &&
+                    ctx.component != null &&
+                    stdLibFile != null
+            ) {
+                // Add the corresponding folder to the list of additional source locations
+                var files =
+                    ctx.additionalLocations.computeIfAbsent(ctx.component!!) { mutableListOf() }
+                files +=
+                    stdLibFile
+                        .resolve(import.name.localName)
+                        .walkTopDown()
+                        .onEnter { !it.name.startsWith(".") }
+                        .filter { it.isFile && !it.name.startsWith(".") }
+                        .toList()
+            }
         }
 
         val p = newNamespaceDeclaration(f.name.name)
@@ -120,8 +149,12 @@ class GoLanguageFrontend(language: Language<GoLanguageFrontend>, ctx: Translatio
             // module path as well as the current directory in relation to the topLevel
             var packagePath = file.parentFile.relativeTo(topLevel)
 
-            // If we are in a module, we need to prepend the module path to it
-            currentModule?.let { packagePath = File(it.module.mod.path).resolve(packagePath) }
+            // If we are in a module, we need to prepend the module path to it. There is an
+            // exception if we are in the "std" module, which represents the standard library
+            val modulePath = currentModule?.module?.mod?.path
+            if (modulePath != null && modulePath != "std") {
+                packagePath = File(modulePath).resolve(packagePath)
+            }
 
             p.path = packagePath.path
         } catch (ex: IllegalArgumentException) {

cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontendTest.kt

@@ -41,6 +41,7 @@ import de.fraunhofer.aisec.cpg.graph.statements.expressions.*
 import de.fraunhofer.aisec.cpg.graph.types.FunctionType
 import de.fraunhofer.aisec.cpg.graph.types.ObjectType
 import de.fraunhofer.aisec.cpg.graph.types.PointerType
+import java.io.File
 import java.nio.file.Path
 import kotlin.test.*
 
@@ -790,4 +791,31 @@ class GoLanguageFrontendTest : BaseTest() {
         assertNotNull(g)
         assertLocalName("string", g.type)
     }
+
+    @Test
+    fun testResolveStdLibImport() {
+        val topLevel = Path.of("src", "test", "resources", "golang")
+        val tu =
+            analyzeAndGetFirstTU(listOf(topLevel.resolve("function.go").toFile()), topLevel, true) {
+                it.registerLanguage<GoLanguage>()
+                it.configureFrontend<GoLanguageFrontend>(
+                    GoLanguageFrontend.Configuration(File("src/test/resources/golang-std"))
+                )
+            }
+
+        assertNotNull(tu)
+
+        val p = tu.namespaces["p"]
+        assertNotNull(p)
+
+        val main = p.functions["main"]
+        assertNotNull(main)
+
+        val printfCall = main.calls["fmt.Printf"]
+        assertNotNull(printfCall)
+
+        val printf = printfCall.invokes.firstOrNull()
+        assertNotNull(printf)
+        assertEquals("print.go", File(printf.location?.artifactLocation?.uri?.path.toString()).name)
+    }
 }

cpg-language-go/src/test/resources/golang-std/fmt/print.go

@@ -0,0 +1,6 @@
+package fmt
+
+func Printf(format string, a ...any) (n int, err error) {
+	// Not a real implementation, and we are ignoring it anyway
+	return 0, nil
+}

cpg-language-go/src/test/resources/golang-std/go.mod

@@ -0,0 +1,3 @@
+module std
+
+go 1.21