implement Google C++ Benchmark Framework support

benchmark-action · Nov 29, 2019 · b62c24f · b62c24f
1 parent abd6588
commit b62c24f
Show file tree

Hide file tree

Showing 7 changed files with 136 additions and 5 deletions.
diff --git a/src/config.ts b/src/config.ts
@@ -3,7 +3,7 @@ import { promises as fs } from 'fs';
 import * as os from 'os';
 import * as path from 'path';
 
-export type ToolType = 'cargo' | 'go' | 'benchmarkjs' | 'pytest';
+export type ToolType = 'cargo' | 'go' | 'benchmarkjs' | 'pytest' | 'googlecpp';
 export interface Config {
     name: string;
     tool: ToolType;
@@ -21,7 +21,7 @@ export interface Config {
     maxItemsInChart: number | null;
 }
 
-export const VALID_TOOLS: ToolType[] = ['cargo', 'go', 'benchmarkjs', 'pytest'];
+export const VALID_TOOLS: ToolType[] = ['cargo', 'go', 'benchmarkjs', 'pytest', 'googlecpp'];
 const RE_UINT = /^\d+$/;
 
 function validateToolType(tool: string): asserts tool is ToolType {

diff --git a/src/default_index_html.ts b/src/default_index_html.ts
@@ -101,7 +101,8 @@ export const DEFAULT_INDEX_HTML = String.raw`<!DOCTYPE html>
           cargo: '#dea584',
           go: '#00add8',
           benchmarkjs: '#f1e05a',
-          pytest: '#3572a5'
+          pytest: '#3572a5',
+          googlecpp: '#f34b7d'
         };
 
         const data = window.BENCHMARK_DATA;

diff --git a/src/extract.ts b/src/extract.ts
@@ -34,6 +34,32 @@ export interface Benchmark {
     benches: BenchmarkResult[];
 }
 
+export interface GoogleCppBenchmarkJson {
+    context: {
+        date: string;
+        host_name: string;
+        executable: string;
+        num_cpus: number;
+        mhz_per_cpu: number;
+        cpu_scaling_enabled: boolean;
+        caches: unknown;
+        load_avg: number[];
+        library_build_type: 'release' | 'debug';
+    };
+    benchmarks: Array<{
+        name: string;
+        run_name: string;
+        run_type: string;
+        repetitions: number;
+        repetition_index: number;
+        threads: number;
+        iterations: number;
+        real_time: number;
+        cpu_time: number;
+        time_unit: string;
+    }>;
+}
+
 export interface PytestBenchmarkJson {
     machine_info: {
         node: string;
@@ -223,6 +249,24 @@ function extractPytestResult(output: string): BenchmarkResult[] {
     }
 }
 
+function extractGoogleCppResult(output: string): BenchmarkResult[] {
+    let json: GoogleCppBenchmarkJson;
+    try {
+        json = JSON.parse(output);
+    } catch (err) {
+        throw new Error(
+            `Output file for 'googlecpp' must be JSON file generated by --benchmark_format=json option: ${err.message}`,
+        );
+    }
+    return json.benchmarks.map(b => {
+        const name = b.name;
+        const value = b.real_time;
+        const unit = b.time_unit + '/iter';
+        const extra = `iterations: ${b.iterations}\ncpu: ${b.cpu_time} ${b.time_unit}\nthreads: ${b.threads}`;
+        return { name, value, unit, extra };
+    });
+}
+
 export async function extractResult(config: Config): Promise<Benchmark> {
     const output = await fs.readFile(config.outputFilePath, 'utf8');
     const { tool } = config;
@@ -241,6 +285,9 @@ export async function extractResult(config: Config): Promise<Benchmark> {
         case 'pytest':
             benches = extractPytestResult(output);
             break;
+        case 'googlecpp':
+            benches = extractGoogleCppResult(output);
+            break;
         default:
             throw new Error(`FATAL: Unexpected tool: '${tool}'`);
     }

diff --git a/src/write.ts b/src/write.ts
@@ -87,6 +87,8 @@ function biggerIsBetter(tool: ToolType): boolean {
             return true;
         case 'pytest':
             return true;
+        case 'googlecpp':
+            return false;
     }
 }
 

diff --git a/test/data/extract/googlecpp_output.txt b/test/data/extract/googlecpp_output.txt
@@ -0,0 +1,64 @@
+{
+  "context": {
+    "date": "2019-11-29 21:26:59",
+    "host_name": "Corgi.local",
+    "executable": "./a.out",
+    "num_cpus": 4,
+    "mhz_per_cpu": 2700,
+    "cpu_scaling_enabled": false,
+    "caches": [
+      {
+        "type": "Data",
+        "level": 1,
+        "size": 32768,
+        "num_sharing": 2
+      },
+      {
+        "type": "Instruction",
+        "level": 1,
+        "size": 32768,
+        "num_sharing": 2
+      },
+      {
+        "type": "Unified",
+        "level": 2,
+        "size": 262144,
+        "num_sharing": 2
+      },
+      {
+        "type": "Unified",
+        "level": 3,
+        "size": 3145728,
+        "num_sharing": 4
+      }
+    ],
+    "load_avg": [1.68408,1.73779,2.02783],
+    "library_build_type": "release"
+  },
+  "benchmarks": [
+    {
+      "name": "fib_10",
+      "run_name": "fib_10",
+      "run_type": "iteration",
+      "repetitions": 0,
+      "repetition_index": 0,
+      "threads": 1,
+      "iterations": 3070566,
+      "real_time": 2.1498980114547953e+02,
+      "cpu_time": 2.1365507206163295e+02,
+      "time_unit": "ns"
+    },
+    {
+      "name": "fib_20",
+      "run_name": "fib_20",
+      "run_type": "iteration",
+      "repetitions": 0,
+      "repetition_index": 0,
+      "threads": 1,
+      "iterations": 23968,
+      "real_time": 2.7455600415007055e+04,
+      "cpu_time": 2.7364903204272359e+04,
+      "time_unit": "ns"
+    }
+  ]
+}
diff --git a/test/extract.ts b/test/extract.ts
@@ -93,6 +93,23 @@ describe('extractResult()', function() {
                 },
             ],
         },
+        {
+            tool: 'googlecpp',
+            expected: [
+                {
+                    extra: 'iterations: 3070566\ncpu: 213.65507206163295 ns\nthreads: 1',
+                    name: 'fib_10',
+                    unit: 'ns/iter',
+                    value: 214.98980114547953,
+                },
+                {
+                    extra: 'iterations: 23968\ncpu: 27364.90320427236 ns\nthreads: 1',
+                    name: 'fib_20',
+                    unit: 'ns/iter',
+                    value: 27455.600415007055,
+                },
+            ],
+        },
     ] as Array<{
         tool: string;
         expected: BenchmarkResult[];

diff --git a/test/write.ts b/test/write.ts
@@ -419,7 +419,7 @@ describe('writeBenchmark()', function() {
                             {
                                 commit: commit('prev commit id'),
                                 date: lastUpdate - 1000,
-                                tool: 'cargo',
+                                tool: 'googlecpp',
                                 benches: [bench('bench_fib_10', 100)],
                             },
                         ],
@@ -428,7 +428,7 @@ describe('writeBenchmark()', function() {
                 added: {
                     commit: commit('current commit id'),
                     date: lastUpdate,
-                    tool: 'cargo',
+                    tool: 'googlecpp',
                     benches: [bench('bench_fib_10', 210)], // Exceeds 2.0 threshold
                 },
                 error: [