Add support for catch2 benchmarks

benchmark-action · Jan 15, 2020 · 88404b7 · 88404b7
1 parent 116283d
commit 88404b7
Show file tree

Hide file tree

Showing 6 changed files with 176 additions and 2 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -160,6 +160,41 @@ jobs:
           skip-fetch-gh-pages: true
           fail-on-alert: true
       - run: node ./scripts/ci_validate_modification.js before_data.js 'C++ Benchmark'
+  catch2-framework:
+    name: Run Catch2 C++ Benchmark Framework example
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v1
+      - uses: actions/cache@v1
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('package-lock.json') }}
+      - run: npm ci
+      - run: npm run build
+      - name: Save previous data.js
+        run: |
+          git fetch origin gh-pages
+          git checkout gh-pages
+          cp ./dev/bench/data.js before_data.js
+          git checkout -
+      - name: Run benchmark
+        run: |
+          cd examples/catch2
+          mkdir build && cd build
+          cmake -DCMAKE_BUILD_TYPE=Release ..
+          cmake --build . --config Release
+          ./Catch2_bench > ../benchmark_result.txt
+      - name: Store benchmark result
+        uses: ./
+        with:
+          name: Catch2 Benchmark
+          tool: "catch2"
+          output-file-path: examples/catch2/benchmark_result.txt
+          skip-fetch-gh-pages: true
+          fail-on-alert: true
+      - run: node ./scripts/ci_validate_modification.js before_data.js 'Catch2 Benchmark'
+
   only-alert-with-cache:
     name: Run alert check with actions/cache
     runs-on: ubuntu-latest

diff --git a/src/config.ts b/src/config.ts
@@ -3,7 +3,7 @@ import { promises as fs } from 'fs';
 import * as os from 'os';
 import * as path from 'path';
 
-export type ToolType = 'cargo' | 'go' | 'benchmarkjs' | 'pytest' | 'googlecpp';
+export type ToolType = 'cargo' | 'go' | 'benchmarkjs' | 'pytest' | 'googlecpp' | 'catch2';
 export interface Config {
     name: string;
     tool: ToolType;
@@ -22,7 +22,7 @@ export interface Config {
     maxItemsInChart: number | null;
 }
 
-export const VALID_TOOLS: ToolType[] = ['cargo', 'go', 'benchmarkjs', 'pytest', 'googlecpp'];
+export const VALID_TOOLS: ToolType[] = ['cargo', 'go', 'benchmarkjs', 'pytest', 'googlecpp', 'catch2'];
 const RE_UINT = /^\d+$/;
 
 function validateToolType(tool: string): asserts tool is ToolType {

diff --git a/src/extract.ts b/src/extract.ts
@@ -304,6 +304,82 @@ function extractGoogleCppResult(output: string): BenchmarkResult[] {
     });
 }
 
+function extractCatch2Result(output: string): BenchmarkResult[] {
+    const lines = output.split('\n');
+
+    const ret = [];
+    // Example:
+
+    // benchmark name samples       iterations    estimated <-- Start benchmark section
+    //                mean          low mean      high mean <-- Ignored
+    //                std dev       low std dev   high std dev <-- Ignored
+    // ----------------------------------------------------- <-- Ignored
+    // Fibonacci 20   100           2             8.4318 ms <-- Start actual benchmark
+    //                43.186 us     41.402 us     46.246 us <-- Actual benchmark data
+    //                11.719 us      7.847 us     17.747 us <-- Ignored
+
+    const reTestCaseStart = /^benchmark name +samples +iterations +estimated/;
+    const reBenchmarkStart = /^([a-zA-Z\d ]+) +(\d+) +(\d+) +(\d+(\.\d+)?) (ns|ms|us|s)/;
+    const reBenchmarkValues = /^ +(\d+(?:\.\d+)?) (ns|us|ms|s) +(\d+(?:\.\d+)?) (ns|us|ms|s) +(\d+(?:\.\d+)?) (ns|us|ms|s)/;
+
+    let benchmarkNr = -1;
+    let testCaseNr = -1;
+
+    let linesSinceBenchmarkStart = -1;
+
+    for (const line of lines) {
+        const m = line.match(reTestCaseStart);
+        if (m !== null) {
+            testCaseNr++;
+        }
+        // no benchmark section found so far, ignore
+        if (testCaseNr < 0) {
+            continue;
+        }
+
+        if (benchmarkNr >= 0) {
+            linesSinceBenchmarkStart++;
+        }
+
+        const benchmarkValueMatch = line.match(reBenchmarkValues);
+        if (benchmarkValueMatch === null && linesSinceBenchmarkStart === 1) {
+            throw new Error(
+                'Retrieved a catch2 benchmark but no values for it\nCatch2 result file is possibly mangled\n\n' + line,
+            );
+        }
+        if (linesSinceBenchmarkStart === 1 && benchmarkValueMatch !== null) {
+            ret[benchmarkNr].value = parseFloat(benchmarkValueMatch[1]);
+            ret[benchmarkNr].unit = benchmarkValueMatch[2];
+        }
+        if (linesSinceBenchmarkStart === 2 && benchmarkValueMatch !== null) {
+            ret[benchmarkNr].range = '+/- ' + benchmarkValueMatch[1].trim();
+        }
+
+        const benchmarkMatch = line.match(reBenchmarkStart);
+        if (benchmarkMatch !== null) {
+            linesSinceBenchmarkStart = 0;
+            benchmarkNr++;
+            ret.push({
+                name: benchmarkMatch[1].trim(),
+                value: 0,
+                range: '',
+                unit: '',
+                extra: benchmarkMatch[2] + ' samples',
+            });
+        }
+    }
+
+    if (
+        ret.every(function(r) {
+            return r.range === '' && r.unit === '';
+        })
+    ) {
+        throw new Error(`Invalid range or unit for catch2 benchmark`);
+    }
+
+    return ret;
+}
+
 export async function extractResult(config: Config): Promise<Benchmark> {
     const output = await fs.readFile(config.outputFilePath, 'utf8');
     const { tool } = config;
@@ -325,6 +401,9 @@ export async function extractResult(config: Config): Promise<Benchmark> {
         case 'googlecpp':
             benches = extractGoogleCppResult(output);
             break;
+        case 'catch2':
+            benches = extractCatch2Result(output);
+            break;
         default:
             throw new Error(`FATAL: Unexpected tool: '${tool}'`);
     }

diff --git a/src/write.ts b/src/write.ts
@@ -69,6 +69,8 @@ function biggerIsBetter(tool: ToolType): boolean {
             return true;
         case 'googlecpp':
             return false;
+        case 'catch2':
+            return false;
     }
 }
 

diff --git a/test/data/extract/catch2_output.txt b/test/data/extract/catch2_output.txt
@@ -0,0 +1,32 @@
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Catch2_bench is a Catch v2.11.0 host application.
+Run with -? for options
+
+-------------------------------------------------------------------------------
+Fibonacci
+-------------------------------------------------------------------------------
+/home/doeme/Code/github-action-benchmark/examples/catch2/catch2_bench.cpp:5
+...............................................................................
+
+benchmark name                                  samples       iterations    estimated
+                                                mean          low mean      high mean
+                                                std dev       low std dev   high std dev
+-------------------------------------------------------------------------------
+Fibonacci 20                                              100             2     8.4318 ms 
+                                                    43.186 us     41.402 us     46.246 us 
+                                                    11.719 us      7.847 us     17.747 us 
+
+Fibonacci 25                                              100             1    45.6213 ms 
+                                                   451.183 us    441.654 us    469.296 us 
+                                                    65.064 us     36.524 us    106.192 us 
+
+Fibonacci Integer                                         100             1         45 ms 
+                                                        123 s         441 s    296.123 ns 
+                                                          2 s     36.524 us    106.192 us                                                     
+
+
+===============================================================================
+test cases: 1 | 1 passed
+assertions: - none -
+
diff --git a/test/extract.ts b/test/extract.ts
@@ -54,6 +54,32 @@ describe('extractResult()', function() {
                 },
             ],
         },
+        {
+            tool: 'catch2',
+            expected: [
+                {
+                    name: 'Fibonacci 20',
+                    range: '+/- 11.719',
+                    unit: 'us',
+                    value: 43.186,
+                    extra: '100 samples',
+                },
+                {
+                    name: 'Fibonacci 25',
+                    range: '+/- 65.064',
+                    unit: 'us',
+                    value: 451.183,
+                    extra: '100 samples',
+                },
+                {
+                    name: 'Fibonacci Integer',
+                    range: '+/- 2',
+                    unit: 's',
+                    value: 123,
+                    extra: '100 samples',
+                },
+            ],
+        },
         {
             tool: 'go',
             expected: [