Added pretty printing of benchmarks (#680)

* Added pretty printing of benchmarks * Fixed minor code smells Co-authored-by: Konrad Weiss <[email protected]>
Fraunhofer-AISEC · Feb 1, 2022 · ddcd1ba · ddcd1ba
1 parent eaa8271
commit ddcd1ba
Show file tree

Hide file tree

Showing 6 changed files with 195 additions and 7 deletions.
diff --git a/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/InferenceConfiguration.kt b/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/InferenceConfiguration.kt
@@ -26,6 +26,8 @@
 package de.fraunhofer.aisec.cpg
 
 import de.fraunhofer.aisec.cpg.frontends.cpp.CXXLanguageFrontend
+import org.apache.commons.lang3.builder.ToStringBuilder
+import org.apache.commons.lang3.builder.ToStringStyle
 
 /**
  * This class holds configuration options for the inference of certain constructs and auto-guessing
@@ -51,4 +53,11 @@ private constructor(
             return Builder()
         }
     }
+
+    override fun toString(): String {
+        return ToStringBuilder(this, ToStringStyle.JSON_STYLE)
+            .append("guessCastExpressions", guessCastExpressions)
+            .append("inferRecords", inferRecords)
+            .toString()
+    }
 }
diff --git a/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/TranslationConfiguration.java b/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/TranslationConfiguration.java
@@ -38,6 +38,8 @@
 import java.io.File;
 import java.lang.reflect.InvocationTargetException;
 import java.util.*;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
 import org.checkerframework.checker.nullness.qual.NonNull;
 import org.checkerframework.checker.nullness.qual.Nullable;
 import org.slf4j.Logger;
@@ -552,4 +554,28 @@ public TranslationConfiguration build() {
           compilationDatabase);
     }
   }
+
+  @Override
+  public String toString() {
+    return new ToStringBuilder(this, ToStringStyle.JSON_STYLE)
+        .append("debugParser", debugParser)
+        .append("loadIncludes", loadIncludes)
+        .append("includePaths", includePaths)
+        .append("includeWhitelist", includeWhitelist)
+        .append("includeBlacklist", includeBlacklist)
+        .append("frontends", frontends)
+        .append("disableCleanup", disableCleanup)
+        .append("codeInNodes", codeInNodes)
+        .append("processAnnotations", processAnnotations)
+        .append("failOnError", failOnError)
+        .append("symbols", symbols)
+        .append("sourceLocations", sourceLocations)
+        .append("topLevel", topLevel)
+        .append("useUnityBuild", useUnityBuild)
+        .append("useParallelFrontends", useParallelFrontends)
+        .append("typeSystemActiveInFrontend", typeSystemActiveInFrontend)
+        .append("passes", passes)
+        .append("inferenceConfiguration", inferenceConfiguration)
+        .toString();
+  }
 }
diff --git a/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/TranslationManager.kt b/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/TranslationManager.kt
@@ -76,13 +76,18 @@ private constructor(
         return CompletableFuture.supplyAsync {
             val scopesBuildForAnalysis = ScopeManager()
             val outerBench =
-                Benchmark(TranslationManager::class.java, "Translation into full graph")
+                Benchmark(
+                    TranslationManager::class.java,
+                    "Translation into full graph",
+                    false,
+                    result
+                )
             val passesNeedCleanup = mutableSetOf<Pass>()
             var frontendsNeedCleanup: Set<LanguageFrontend>? = null
 
             try {
                 // Parse Java/C/CPP files
-                var bench = Benchmark(this.javaClass, "Frontend")
+                var bench = Benchmark(this.javaClass, "Executing Language Frontend", false, result)
                 frontendsNeedCleanup = runFrontends(result, config, scopesBuildForAnalysis)
                 bench.stop()
 
@@ -92,7 +97,7 @@ private constructor(
                 // Apply passes
                 for (pass in config.registeredPasses) {
                     passesNeedCleanup.add(pass)
-                    bench = Benchmark(pass.javaClass, "Executing Pass")
+                    bench = Benchmark(pass.javaClass, "Executing Pass", false, result)
                     pass.accept(result)
                     bench.stop()
                     if (result.isCancelled) {

diff --git a/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/TranslationResult.java b/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/TranslationResult.java
@@ -28,18 +28,22 @@
 import de.fraunhofer.aisec.cpg.graph.Node;
 import de.fraunhofer.aisec.cpg.graph.SubGraph;
 import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration;
+import de.fraunhofer.aisec.cpg.helpers.Benchmark;
+import de.fraunhofer.aisec.cpg.helpers.StatisticsHolder;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+import org.jetbrains.annotations.NotNull;
 
 /**
  * The global (intermediate) result of the translation. A {@link
  * de.fraunhofer.aisec.cpg.frontends.LanguageFrontend} will initially populate it and a {@link
  * de.fraunhofer.aisec.cpg.passes.Pass} can extend it.
  */
-public class TranslationResult extends Node {
+public class TranslationResult extends Node implements StatisticsHolder {
   public static final String SOURCE_LOCATIONS_TO_FRONTEND = "sourceLocationsToFrontend";
   private final TranslationManager translationManager;
 
@@ -50,6 +54,8 @@ public class TranslationResult extends Node {
   /** A free-for-use HashMap where passes can store whatever they want. */
   private final Map<String, Object> scratch = new ConcurrentHashMap<>();
 
+  private final List<Benchmark> benchmarks = new ArrayList<>();
+
   public TranslationResult(TranslationManager translationManager) {
     this.translationManager = translationManager;
   }
@@ -95,4 +101,32 @@ public Map<String, Object> getScratch() {
   public TranslationManager getTranslationManager() {
     return translationManager;
   }
+
+  @Override
+  public void addBenchmark(@NotNull Benchmark b) {
+    this.benchmarks.add(b);
+  }
+
+  public List<Benchmark> getBenchmarks() {
+    return benchmarks;
+  }
+
+  @Override
+  public void printBenchmark() {
+    StatisticsHolder.DefaultImpls.printBenchmark(this);
+  }
+
+  @NotNull
+  @Override
+  public List<String> getTranslatedFiles() {
+    return translationUnits.stream()
+        .map(TranslationUnitDeclaration::getName)
+        .collect(Collectors.toList());
+  }
+
+  @NotNull
+  @Override
+  public TranslationConfiguration getConfig() {
+    return translationManager.getConfig();
+  }
 }
diff --git a/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/helpers/Benchmark.kt b/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/helpers/Benchmark.kt
@@ -25,15 +25,104 @@
  */
 package de.fraunhofer.aisec.cpg.helpers
 
+import de.fraunhofer.aisec.cpg.TranslationConfiguration
+import java.nio.file.Path
 import java.time.Duration
 import java.time.Instant
+import java.util.*
+import kotlin.IllegalArgumentException
 import org.slf4j.LoggerFactory
 
+/** Interface definition to hold different statistics about the translation process. */
+interface StatisticsHolder {
+    val translatedFiles: List<String>
+    val benchmarks: List<Benchmark>
+    val config: TranslationConfiguration
+
+    fun addBenchmark(b: Benchmark)
+
+    /** Pretty-prints benchmark results for easy copying to GitHub issues. */
+    fun printBenchmark() {
+        println("# Benchmark run ${UUID.randomUUID()}")
+        printMarkdown(
+            listOf(
+                listOf(
+                    "Translation config",
+                    "`${config.toString().replace(",", ", ").replace(":", ": ")}`"
+                ),
+                listOf("Number of files translated", translatedFiles.size),
+                listOf(
+                    "Translated file(s)",
+                    translatedFiles.map {
+                        relativeOrAbsolute(Path.of(it), config.topLevel.toPath())
+                    }
+                ),
+                *benchmarks
+                    .map { listOf("${it.caller}: ${it.message}", "${it.duration} ms") }
+                    .toTypedArray(),
+            ),
+            listOf("Metric", "Value")
+        )
+    }
+}
+
+/**
+ * Prints a table of values and headers in markdown format. Table columns are automatically adjusted
+ * to the longest column.
+ */
+fun printMarkdown(table: List<List<Any>>, headers: List<String>) {
+    val lengths = IntArray(headers.size)
+
+    // first, we need to calculate the longest column per line
+    for (row in table) {
+        for (i in row.indices) {
+            val value = row[i].toString()
+            if (value.length > lengths[i]) {
+                lengths[i] = value.length
+            }
+        }
+    }
+
+    // table header
+    val dash = lengths.joinToString(" | ", "| ", " |") { ("-".repeat(it)) }
+    var i = 0
+    val header = headers.joinToString(" | ", "| ", " |") { it.padEnd(lengths[i++]) }
+
+    println()
+    println(header)
+    println(dash)
+
+    for (row in table) {
+        var rowIndex = 0
+        val line = row.joinToString(" | ", "| ", " |") { it.toString().padEnd(lengths[rowIndex++]) }
+        println(line)
+    }
+
+    println()
+}
+
+/**
+ * This function will shorten / relativize the [path], if it is relative to [topLevel]. Otherwise,
+ * the full path will be returned.
+ */
+fun relativeOrAbsolute(path: Path, topLevel: Path): Path {
+    return try {
+        topLevel.toAbsolutePath().relativize(path)
+    } catch (ex: IllegalArgumentException) {
+        path
+    }
+}
+
 open class Benchmark
 @JvmOverloads
-constructor(c: Class<*>, private val message: String, private var debug: Boolean = false) {
+constructor(
+    c: Class<*>,
+    val message: String,
+    private var debug: Boolean = false,
+    private var holder: StatisticsHolder? = null
+) {
 
-    private val caller: String
+    val caller: String
     private val start: Instant
 
     var duration: Long
@@ -50,6 +139,9 @@ constructor(c: Class<*>, private val message: String, private var debug: Boolean
             log.info(msg)
         }
 
+        // update our holder, if we have any
+        holder?.addBenchmark(this)
+
         return duration
     }
 

diff --git a/cpg-core/src/testFixtures/java/de/fraunhofer/aisec/cpg/TestUtils.kt b/cpg-core/src/testFixtures/java/de/fraunhofer/aisec/cpg/TestUtils.kt
@@ -119,6 +119,28 @@ object TestUtils {
         usePasses: Boolean,
         configModifier: Consumer<TranslationConfiguration.Builder>? = null
     ): List<TranslationUnitDeclaration> {
+        return analyzeWithResult(files, topLevel, usePasses, configModifier).translationUnits
+    }
+
+    /**
+     * Default way of parsing a list of files into a full CPG. All default passes are applied
+     *
+     * @param topLevel The directory to traverse while looking for files to parse
+     * @param usePasses Whether the analysis should run passes after the initial phase
+     * @param configModifier An optional modifier for the config
+     *
+     * @return A list of [TranslationUnitDeclaration] nodes, representing the CPG roots
+     * @throws Exception Any exception thrown during the parsing process
+     */
+    @JvmOverloads
+    @JvmStatic
+    @Throws(Exception::class)
+    fun analyzeWithResult(
+        files: List<File>?,
+        topLevel: Path,
+        usePasses: Boolean,
+        configModifier: Consumer<TranslationConfiguration.Builder>? = null
+    ): TranslationResult {
         val builder =
             TranslationConfiguration.builder()
                 .sourceLocations(files)
@@ -136,7 +158,7 @@ object TestUtils {
         configModifier?.accept(builder)
         val config = builder.build()
         val analyzer = TranslationManager.builder().config(config).build()
-        return analyzer.analyze().get().translationUnits
+        return analyzer.analyze().get()
     }
 
     /**