From 7a767b6a75bedc56eb62f036f55140a0bbbfb99d Mon Sep 17 00:00:00 2001 From: KuechA <31155350+KuechA@users.noreply.github.com> Date: Tue, 26 Apr 2022 14:12:45 +0200 Subject: [PATCH] Ensure backwards compatibility to cpg version < 4.4.0 (#769) Co-authored-by: Christian Banse --- .../aisec/cpg/TranslationConfiguration.java | 17 +++++-- .../aisec/cpg/TranslationManager.kt | 18 +++---- .../aisec/cpg/TranslationResult.java | 47 +++++++++++++++++-- .../cpg/frontends/cpp/CXXLanguageFrontend.kt | 6 +-- .../frontends/java/JavaLanguageFrontend.kt | 6 +-- .../de/fraunhofer/aisec/cpg/graph/Graph.kt | 4 +- .../{Benchmark.kt => MeasurementHolder.kt} | 19 +++++--- .../cpg/passes/StatisticsCollectionPass.kt | 5 +- .../aisec/cpg/PerformanceRegressionTest.kt | 4 +- .../frontends/llvm/LLVMIRLanguageFrontend.kt | 6 +-- .../aisec/cpg_vis_neo4j/Application.kt | 4 +- 11 files changed, 96 insertions(+), 40 deletions(-) rename cpg-core/src/main/java/de/fraunhofer/aisec/cpg/helpers/{Benchmark.kt => MeasurementHolder.kt} (94%) diff --git a/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/TranslationConfiguration.java b/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/TranslationConfiguration.java index 8060d059eb..65a87e170d 100644 --- a/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/TranslationConfiguration.java +++ b/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/TranslationConfiguration.java @@ -212,6 +212,15 @@ public Map getSymbols() { return this.symbols; } + /** Returns a list of all analyzed files. */ + public List getSourceLocations() { + List sourceLocations = new ArrayList<>(); + for (var entry : softwareComponents.entrySet()) { + sourceLocations.addAll(entry.getValue()); + } + return sourceLocations; + } + public Map> getSoftwareComponents() { return this.softwareComponents; } @@ -283,25 +292,25 @@ public Builder symbols(Map symbols) { /** * Files or directories containing the source code to analyze. Generates a dummy software - * component called "SWC". + * component called "application". * * @param sourceLocations The files with the source code * @return this */ public Builder sourceLocations(File... sourceLocations) { - this.softwareComponents.put("SWC", Arrays.asList(sourceLocations)); + this.softwareComponents.put("application", Arrays.asList(sourceLocations)); return this; } /** * Files or directories containing the source code to analyze. Generates a dummy software - * component called "SWC". + * component called "application". * * @param sourceLocations The files with the source code * @return this */ public Builder sourceLocations(List sourceLocations) { - this.softwareComponents.put("SWC", sourceLocations); + this.softwareComponents.put("application", sourceLocations); return this; } diff --git a/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/TranslationManager.kt b/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/TranslationManager.kt index fe30319d58..0d010fb837 100644 --- a/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/TranslationManager.kt +++ b/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/TranslationManager.kt @@ -31,8 +31,8 @@ import de.fraunhofer.aisec.cpg.frontends.cpp.CXXLanguageFrontend import de.fraunhofer.aisec.cpg.graph.Component import de.fraunhofer.aisec.cpg.graph.TypeManager import de.fraunhofer.aisec.cpg.helpers.Benchmark +import de.fraunhofer.aisec.cpg.helpers.MeasurementHolder import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker -import de.fraunhofer.aisec.cpg.helpers.TimeBenchmark import de.fraunhofer.aisec.cpg.helpers.Util import de.fraunhofer.aisec.cpg.passes.Pass import de.fraunhofer.aisec.cpg.passes.scopes.ScopeManager @@ -77,7 +77,7 @@ private constructor( return CompletableFuture.supplyAsync { val scopesBuildForAnalysis = ScopeManager() val outerBench = - TimeBenchmark( + Benchmark( TranslationManager::class.java, "Translation into full graph", false, @@ -88,8 +88,7 @@ private constructor( try { // Parse Java/C/CPP files - var bench = - TimeBenchmark(this.javaClass, "Executing Language Frontend", false, result) + var bench = Benchmark(this.javaClass, "Executing Language Frontend", false, result) frontendsNeedCleanup = runFrontends(result, config, scopesBuildForAnalysis) bench.addMeasurement() @@ -99,7 +98,7 @@ private constructor( // Apply passes for (pass in config.registeredPasses) { passesNeedCleanup.add(pass) - bench = TimeBenchmark(pass.javaClass, "Executing Pass", false, result) + bench = Benchmark(pass.javaClass, "Executing Pass", false, result) pass.accept(result) bench.addMeasurement() if (result.isCancelled) { @@ -227,14 +226,17 @@ private constructor( result.components.forEach { s -> s.translationUnits.forEach { val bench = - Benchmark(this.javaClass, "Activating types for ${it.name}", true) + MeasurementHolder( + this.javaClass, + "Activating types for ${it.name}", + true + ) SubgraphWalker.activateTypes(it, scopeManager) bench.addMeasurement() } } result.translationUnits.forEach { - val bench = - TimeBenchmark(this.javaClass, "Activating types for ${it.name}", true) + val bench = Benchmark(this.javaClass, "Activating types for ${it.name}", true) SubgraphWalker.activateTypes(it, scopeManager) bench.addMeasurement() } diff --git a/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/TranslationResult.java b/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/TranslationResult.java index 8f54e5796f..7fb0aeebbf 100644 --- a/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/TranslationResult.java +++ b/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/TranslationResult.java @@ -29,8 +29,8 @@ import de.fraunhofer.aisec.cpg.graph.Node; import de.fraunhofer.aisec.cpg.graph.SubGraph; import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration; -import de.fraunhofer.aisec.cpg.helpers.Benchmark; import de.fraunhofer.aisec.cpg.helpers.BenchmarkResults; +import de.fraunhofer.aisec.cpg.helpers.MeasurementHolder; import de.fraunhofer.aisec.cpg.helpers.StatisticsHolder; import java.util.*; import java.util.concurrent.ConcurrentHashMap; @@ -61,7 +61,7 @@ public class TranslationResult extends Node implements StatisticsHolder { */ private final Set additionalNodes = new HashSet<>(); - private final List benchmarks = new ArrayList<>(); + private final List benchmarks = new ArrayList<>(); public TranslationResult(TranslationManager translationManager) { this.translationManager = translationManager; @@ -88,6 +88,45 @@ public List getTranslationUnits() { return result; } + /** + * If no component exists, it generates a [Component] called "application" and adds [tu]. If a + * component already exists, adds the tu to this component. + * + * @param tu The translation unit to add. + * @deprecated This should not be used anymore. Instead, the corresponding component should be + * selected and the translation unit should be added there. + */ + @Deprecated(since = "4.4.1") + public synchronized void addTranslationUnit(TranslationUnitDeclaration tu) { + Component swc = null; + if (components.size() == 1) { + // Only one component exists, so we take this one + swc = components.get(0); + } else if (components.isEmpty()) { + // No component exists, so we create the new dummy component. + swc = new Component(); + swc.setName("application"); + components.add(swc); + } else { + // Multiple components exist. As we don't know where to put the tu, we check if we have the + // component we created and add it there or create a new one. + for (var component : components) { + if (component.getName().equals("application")) { + swc = component; + break; + } + } + + if (swc == null) { + swc = new Component(); + swc.setName("application"); + components.add(swc); + } + } + + swc.getTranslationUnits().add(tu); + } + /** * List of software components. Note that this list is immutable. Use {@link * #addComponent(Component)} if you wish to modify it. @@ -127,12 +166,12 @@ public TranslationManager getTranslationManager() { } @Override - public void addBenchmark(@NotNull Benchmark b) { + public void addBenchmark(@NotNull MeasurementHolder b) { this.benchmarks.add(b); } @NotNull - public List getBenchmarks() { + public List getBenchmarks() { return benchmarks; } diff --git a/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/frontends/cpp/CXXLanguageFrontend.kt b/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/frontends/cpp/CXXLanguageFrontend.kt index 93334e0343..8a656e06c8 100644 --- a/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/frontends/cpp/CXXLanguageFrontend.kt +++ b/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/frontends/cpp/CXXLanguageFrontend.kt @@ -36,7 +36,7 @@ import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression import de.fraunhofer.aisec.cpg.graph.types.TypeParser import de.fraunhofer.aisec.cpg.graph.types.UnknownType -import de.fraunhofer.aisec.cpg.helpers.TimeBenchmark +import de.fraunhofer.aisec.cpg.helpers.Benchmark import de.fraunhofer.aisec.cpg.passes.scopes.ScopeManager import de.fraunhofer.aisec.cpg.sarif.PhysicalLocation import de.fraunhofer.aisec.cpg.sarif.Region @@ -196,7 +196,7 @@ class CXXLanguageFrontend(config: TranslationConfiguration, scopeManager: ScopeM val log = DefaultLogService() val opts = ILanguage.OPTION_PARSE_INACTIVE_CODE // | ILanguage.OPTION_ADD_COMMENTS; return try { - var bench = TimeBenchmark(this.javaClass, "Parsing sourcefile") + var bench = Benchmark(this.javaClass, "Parsing sourcefile") val translationUnit = GPPLanguage.getDefault() .getASTTranslationUnit( @@ -211,7 +211,7 @@ class CXXLanguageFrontend(config: TranslationConfiguration, scopeManager: ScopeM val length = translationUnit.length LOGGER.info("Parsed {} bytes corresponding roughly to {} LoC", length, length / 50) bench.addMeasurement() - bench = TimeBenchmark(this.javaClass, "Transform to CPG") + bench = Benchmark(this.javaClass, "Transform to CPG") if (config.debugParser) { explore(translationUnit, 0) } diff --git a/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/frontends/java/JavaLanguageFrontend.kt b/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/frontends/java/JavaLanguageFrontend.kt index d9be87eefc..f6e7ae40de 100644 --- a/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/frontends/java/JavaLanguageFrontend.kt +++ b/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/frontends/java/JavaLanguageFrontend.kt @@ -61,8 +61,8 @@ import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression import de.fraunhofer.aisec.cpg.graph.types.TypeParser import de.fraunhofer.aisec.cpg.graph.types.UnknownType +import de.fraunhofer.aisec.cpg.helpers.Benchmark import de.fraunhofer.aisec.cpg.helpers.CommonPath -import de.fraunhofer.aisec.cpg.helpers.TimeBenchmark import de.fraunhofer.aisec.cpg.passes.scopes.Scope import de.fraunhofer.aisec.cpg.passes.scopes.ScopeManager import de.fraunhofer.aisec.cpg.sarif.PhysicalLocation @@ -105,11 +105,11 @@ open class JavaLanguageFrontend(config: TranslationConfiguration, scopeManager: val parser = JavaParser(parserConfiguration) // parse the file - var bench = TimeBenchmark(this.javaClass, "Parsing source file") + var bench = Benchmark(this.javaClass, "Parsing source file") context = parse(file, parser) bench.addMeasurement() - bench = TimeBenchmark(this.javaClass, "Transform to CPG") + bench = Benchmark(this.javaClass, "Transform to CPG") context!!.setData(Node.SYMBOL_RESOLVER_KEY, javaSymbolResolver) // starting point is always a translation declaration diff --git a/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/graph/Graph.kt b/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/graph/Graph.kt index 2bbef02a6a..5271cc0e75 100644 --- a/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/graph/Graph.kt +++ b/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/graph/Graph.kt @@ -28,8 +28,8 @@ package de.fraunhofer.aisec.cpg.graph import de.fraunhofer.aisec.cpg.ExperimentalGraph import de.fraunhofer.aisec.cpg.TranslationResult import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge +import de.fraunhofer.aisec.cpg.helpers.Benchmark import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker -import de.fraunhofer.aisec.cpg.helpers.TimeBenchmark import java.io.Closeable import java.util.* import java.util.function.Predicate @@ -45,7 +45,7 @@ import scala.Option @ExperimentalGraph class QueryBenchmark constructor(db: Graph, query: Query) : - TimeBenchmark(db.javaClass, "totalNodes: " + db.size() + " query: " + query.toString()), + Benchmark(db.javaClass, "totalNodes: " + db.size() + " query: " + query.toString()), AutoCloseable, Closeable { override fun close() { diff --git a/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/helpers/Benchmark.kt b/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/helpers/MeasurementHolder.kt similarity index 94% rename from cpg-core/src/main/java/de/fraunhofer/aisec/cpg/helpers/Benchmark.kt rename to cpg-core/src/main/java/de/fraunhofer/aisec/cpg/helpers/MeasurementHolder.kt index bb517aa04f..99825dbb7c 100644 --- a/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/helpers/Benchmark.kt +++ b/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/helpers/MeasurementHolder.kt @@ -55,10 +55,10 @@ class BenchmarkResults(val entries: List>) { /** Interface definition to hold different statistics about the translation process. */ interface StatisticsHolder { val translatedFiles: List - val benchmarks: List + val benchmarks: List val config: TranslationConfiguration - fun addBenchmark(b: Benchmark) + fun addBenchmark(b: MeasurementHolder) val benchmarkResults: BenchmarkResults get() { @@ -129,15 +129,19 @@ fun relativeOrAbsolute(path: Path, topLevel: File?): Path { } /** Measures the time between creating the object to calling its stop() method. */ -open class TimeBenchmark( +open class Benchmark( c: Class<*>, message: String, debug: Boolean = false, holder: StatisticsHolder? = null -) : Benchmark(c, message, debug, holder) { +) : MeasurementHolder(c, message, debug, holder) { private val start: Instant + fun stop() { + addMeasurement() + } + /** Stops the time and computes the difference between */ override fun addMeasurement(measurementKey: String?, measurementValue: String?): Any? { val duration = Duration.between(start, Instant.now()).toMillis() @@ -152,7 +156,7 @@ open class TimeBenchmark( } companion object { - val log: Logger = LoggerFactory.getLogger(Benchmark::class.java) + val log: Logger = LoggerFactory.getLogger(MeasurementHolder::class.java) } init { @@ -162,7 +166,7 @@ open class TimeBenchmark( } /** Represents some kind of measurements, e.g., on the performance or problems. */ -open class Benchmark +open class MeasurementHolder @JvmOverloads constructor( /** The class which called this benchmark. */ @@ -198,6 +202,7 @@ constructor( } /** Adds a measurement for the respective benchmark and saves it to the map. */ + @JvmOverloads open fun addMeasurement( measurementKey: String? = null, measurementValue: String? = null @@ -213,7 +218,7 @@ constructor( } companion object { - val log: Logger = LoggerFactory.getLogger(Benchmark::class.java) + val log: Logger = LoggerFactory.getLogger(MeasurementHolder::class.java) } init { diff --git a/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/passes/StatisticsCollectionPass.kt b/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/passes/StatisticsCollectionPass.kt index 15f5597f39..7677583cd7 100644 --- a/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/passes/StatisticsCollectionPass.kt +++ b/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/passes/StatisticsCollectionPass.kt @@ -29,7 +29,7 @@ import de.fraunhofer.aisec.cpg.TranslationResult import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.ProblemNode import de.fraunhofer.aisec.cpg.graph.declarations.RecordDeclaration -import de.fraunhofer.aisec.cpg.helpers.Benchmark +import de.fraunhofer.aisec.cpg.helpers.MeasurementHolder import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker.ScopedWalker /** @@ -54,7 +54,8 @@ class StatisticsCollectionPass : Pass() { walker.iterate(tu) } - val nodeMeasurement = Benchmark(this.javaClass, "Measuring Nodes", false, translationResult) + val nodeMeasurement = + MeasurementHolder(this.javaClass, "Measuring Nodes", false, translationResult) nodeMeasurement.addMeasurement("Graph nodes", nodes.toString()) nodeMeasurement.addMeasurement("Problem nodes", problemNodes.toString()) } diff --git a/cpg-core/src/test/java/de/fraunhofer/aisec/cpg/PerformanceRegressionTest.kt b/cpg-core/src/test/java/de/fraunhofer/aisec/cpg/PerformanceRegressionTest.kt index df67b86bed..c152fabc50 100644 --- a/cpg-core/src/test/java/de/fraunhofer/aisec/cpg/PerformanceRegressionTest.kt +++ b/cpg-core/src/test/java/de/fraunhofer/aisec/cpg/PerformanceRegressionTest.kt @@ -35,8 +35,8 @@ import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge import de.fraunhofer.aisec.cpg.graph.statements.expressions.InitializerListExpression import de.fraunhofer.aisec.cpg.graph.types.ObjectType import de.fraunhofer.aisec.cpg.graph.types.Type +import de.fraunhofer.aisec.cpg.helpers.Benchmark import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker -import de.fraunhofer.aisec.cpg.helpers.TimeBenchmark import java.time.Duration import java.time.temporal.ChronoUnit import kotlin.io.path.writeText @@ -103,7 +103,7 @@ class PerformanceRegressionTest { // Even on a slow machine, this should not exceed 1 second (it should be more like // 200-300ms) assertTimeout(Duration.of(1, ChronoUnit.SECONDS)) { - val b = TimeBenchmark(PerformanceRegressionTest::class.java, "getAstChildren") + val b = Benchmark(PerformanceRegressionTest::class.java, "getAstChildren") doNothing(tu) b.addMeasurement() } diff --git a/cpg-language-llvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/llvm/LLVMIRLanguageFrontend.kt b/cpg-language-llvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/llvm/LLVMIRLanguageFrontend.kt index 7ac0b07bdc..66fc67e442 100644 --- a/cpg-language-llvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/llvm/LLVMIRLanguageFrontend.kt +++ b/cpg-language-llvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/llvm/LLVMIRLanguageFrontend.kt @@ -35,7 +35,7 @@ import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration import de.fraunhofer.aisec.cpg.graph.statements.expressions.DeclaredReferenceExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression import de.fraunhofer.aisec.cpg.graph.types.* -import de.fraunhofer.aisec.cpg.helpers.TimeBenchmark +import de.fraunhofer.aisec.cpg.helpers.Benchmark import de.fraunhofer.aisec.cpg.passes.VariableUsageResolver import de.fraunhofer.aisec.cpg.passes.scopes.ScopeManager import de.fraunhofer.aisec.cpg.sarif.PhysicalLocation @@ -71,7 +71,7 @@ class LLVMIRLanguageFrontend(config: TranslationConfiguration, scopeManager: Sco } override fun parse(file: File): TranslationUnitDeclaration { - var bench = TimeBenchmark(this.javaClass, "Parsing sourcefile") + var bench = Benchmark(this.javaClass, "Parsing sourcefile") // clear the bindings cache, because it is just valid within one module bindingsCache.clear() @@ -109,7 +109,7 @@ class LLVMIRLanguageFrontend(config: TranslationConfiguration, scopeManager: Sco throw TranslationException("Could not parse IR: $errorMsg") } bench.addMeasurement() - bench = TimeBenchmark(this.javaClass, "Transform to CPG") + bench = Benchmark(this.javaClass, "Transform to CPG") val tu = TranslationUnitDeclaration() diff --git a/cpg-neo4j/src/main/java/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt b/cpg-neo4j/src/main/java/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt index 45ffebea4e..8ba4e0bfc1 100644 --- a/cpg-neo4j/src/main/java/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt +++ b/cpg-neo4j/src/main/java/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt @@ -31,7 +31,7 @@ import de.fraunhofer.aisec.cpg.frontends.golang.GoLanguageFrontend import de.fraunhofer.aisec.cpg.frontends.llvm.LLVMIRLanguageFrontend import de.fraunhofer.aisec.cpg.frontends.python.PythonLanguageFrontend import de.fraunhofer.aisec.cpg.frontends.typescript.TypeScriptLanguageFrontend -import de.fraunhofer.aisec.cpg.helpers.TimeBenchmark +import de.fraunhofer.aisec.cpg.helpers.Benchmark import java.io.File import java.net.ConnectException import java.nio.file.Paths @@ -214,7 +214,7 @@ class Application : Callable { */ @Throws(InterruptedException::class, ConnectException::class) fun pushToNeo4j(translationResult: TranslationResult) { - val bench = TimeBenchmark(this.javaClass, "Push cpg to neo4j", false, translationResult) + val bench = Benchmark(this.javaClass, "Push cpg to neo4j", false, translationResult) log.info("Using import depth: $depth") log.info( "Count base nodes to save: " +