From 88404b780c651ed5e61c09bd0a5161678d1d835f Mon Sep 17 00:00:00 2001 From: Dominik Berner Date: Thu, 26 Dec 2019 23:48:14 +0100 Subject: [PATCH] Add support for catch2 benchmarks --- .github/workflows/ci.yml | 35 +++++++++++++ src/config.ts | 4 +- src/extract.ts | 79 +++++++++++++++++++++++++++++ src/write.ts | 2 + test/data/extract/catch2_output.txt | 32 ++++++++++++ test/extract.ts | 26 ++++++++++ 6 files changed, 176 insertions(+), 2 deletions(-) create mode 100644 test/data/extract/catch2_output.txt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c01221f28..fbc055e30 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -160,6 +160,41 @@ jobs: skip-fetch-gh-pages: true fail-on-alert: true - run: node ./scripts/ci_validate_modification.js before_data.js 'C++ Benchmark' + catch2-framework: + name: Run Catch2 C++ Benchmark Framework example + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-node@v1 + - uses: actions/cache@v1 + with: + path: ~/.npm + key: ${{ runner.os }}-node-${{ hashFiles('package-lock.json') }} + - run: npm ci + - run: npm run build + - name: Save previous data.js + run: | + git fetch origin gh-pages + git checkout gh-pages + cp ./dev/bench/data.js before_data.js + git checkout - + - name: Run benchmark + run: | + cd examples/catch2 + mkdir build && cd build + cmake -DCMAKE_BUILD_TYPE=Release .. + cmake --build . --config Release + ./Catch2_bench > ../benchmark_result.txt + - name: Store benchmark result + uses: ./ + with: + name: Catch2 Benchmark + tool: "catch2" + output-file-path: examples/catch2/benchmark_result.txt + skip-fetch-gh-pages: true + fail-on-alert: true + - run: node ./scripts/ci_validate_modification.js before_data.js 'Catch2 Benchmark' + only-alert-with-cache: name: Run alert check with actions/cache runs-on: ubuntu-latest diff --git a/src/config.ts b/src/config.ts index a07e30285..64de1ff93 100644 --- a/src/config.ts +++ b/src/config.ts @@ -3,7 +3,7 @@ import { promises as fs } from 'fs'; import * as os from 'os'; import * as path from 'path'; -export type ToolType = 'cargo' | 'go' | 'benchmarkjs' | 'pytest' | 'googlecpp'; +export type ToolType = 'cargo' | 'go' | 'benchmarkjs' | 'pytest' | 'googlecpp' | 'catch2'; export interface Config { name: string; tool: ToolType; @@ -22,7 +22,7 @@ export interface Config { maxItemsInChart: number | null; } -export const VALID_TOOLS: ToolType[] = ['cargo', 'go', 'benchmarkjs', 'pytest', 'googlecpp']; +export const VALID_TOOLS: ToolType[] = ['cargo', 'go', 'benchmarkjs', 'pytest', 'googlecpp', 'catch2']; const RE_UINT = /^\d+$/; function validateToolType(tool: string): asserts tool is ToolType { diff --git a/src/extract.ts b/src/extract.ts index 5fb077e07..018ddeac8 100644 --- a/src/extract.ts +++ b/src/extract.ts @@ -304,6 +304,82 @@ function extractGoogleCppResult(output: string): BenchmarkResult[] { }); } +function extractCatch2Result(output: string): BenchmarkResult[] { + const lines = output.split('\n'); + + const ret = []; + // Example: + + // benchmark name samples iterations estimated <-- Start benchmark section + // mean low mean high mean <-- Ignored + // std dev low std dev high std dev <-- Ignored + // ----------------------------------------------------- <-- Ignored + // Fibonacci 20 100 2 8.4318 ms <-- Start actual benchmark + // 43.186 us 41.402 us 46.246 us <-- Actual benchmark data + // 11.719 us 7.847 us 17.747 us <-- Ignored + + const reTestCaseStart = /^benchmark name +samples +iterations +estimated/; + const reBenchmarkStart = /^([a-zA-Z\d ]+) +(\d+) +(\d+) +(\d+(\.\d+)?) (ns|ms|us|s)/; + const reBenchmarkValues = /^ +(\d+(?:\.\d+)?) (ns|us|ms|s) +(\d+(?:\.\d+)?) (ns|us|ms|s) +(\d+(?:\.\d+)?) (ns|us|ms|s)/; + + let benchmarkNr = -1; + let testCaseNr = -1; + + let linesSinceBenchmarkStart = -1; + + for (const line of lines) { + const m = line.match(reTestCaseStart); + if (m !== null) { + testCaseNr++; + } + // no benchmark section found so far, ignore + if (testCaseNr < 0) { + continue; + } + + if (benchmarkNr >= 0) { + linesSinceBenchmarkStart++; + } + + const benchmarkValueMatch = line.match(reBenchmarkValues); + if (benchmarkValueMatch === null && linesSinceBenchmarkStart === 1) { + throw new Error( + 'Retrieved a catch2 benchmark but no values for it\nCatch2 result file is possibly mangled\n\n' + line, + ); + } + if (linesSinceBenchmarkStart === 1 && benchmarkValueMatch !== null) { + ret[benchmarkNr].value = parseFloat(benchmarkValueMatch[1]); + ret[benchmarkNr].unit = benchmarkValueMatch[2]; + } + if (linesSinceBenchmarkStart === 2 && benchmarkValueMatch !== null) { + ret[benchmarkNr].range = '+/- ' + benchmarkValueMatch[1].trim(); + } + + const benchmarkMatch = line.match(reBenchmarkStart); + if (benchmarkMatch !== null) { + linesSinceBenchmarkStart = 0; + benchmarkNr++; + ret.push({ + name: benchmarkMatch[1].trim(), + value: 0, + range: '', + unit: '', + extra: benchmarkMatch[2] + ' samples', + }); + } + } + + if ( + ret.every(function(r) { + return r.range === '' && r.unit === ''; + }) + ) { + throw new Error(`Invalid range or unit for catch2 benchmark`); + } + + return ret; +} + export async function extractResult(config: Config): Promise { const output = await fs.readFile(config.outputFilePath, 'utf8'); const { tool } = config; @@ -325,6 +401,9 @@ export async function extractResult(config: Config): Promise { case 'googlecpp': benches = extractGoogleCppResult(output); break; + case 'catch2': + benches = extractCatch2Result(output); + break; default: throw new Error(`FATAL: Unexpected tool: '${tool}'`); } diff --git a/src/write.ts b/src/write.ts index 647277c1e..c76cb1f0c 100644 --- a/src/write.ts +++ b/src/write.ts @@ -69,6 +69,8 @@ function biggerIsBetter(tool: ToolType): boolean { return true; case 'googlecpp': return false; + case 'catch2': + return false; } } diff --git a/test/data/extract/catch2_output.txt b/test/data/extract/catch2_output.txt new file mode 100644 index 000000000..29037ef30 --- /dev/null +++ b/test/data/extract/catch2_output.txt @@ -0,0 +1,32 @@ + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Catch2_bench is a Catch v2.11.0 host application. +Run with -? for options + +------------------------------------------------------------------------------- +Fibonacci +------------------------------------------------------------------------------- +/home/doeme/Code/github-action-benchmark/examples/catch2/catch2_bench.cpp:5 +............................................................................... + +benchmark name samples iterations estimated + mean low mean high mean + std dev low std dev high std dev +------------------------------------------------------------------------------- +Fibonacci 20 100 2 8.4318 ms + 43.186 us 41.402 us 46.246 us + 11.719 us 7.847 us 17.747 us + +Fibonacci 25 100 1 45.6213 ms + 451.183 us 441.654 us 469.296 us + 65.064 us 36.524 us 106.192 us + +Fibonacci Integer 100 1 45 ms + 123 s 441 s 296.123 ns + 2 s 36.524 us 106.192 us + + +=============================================================================== +test cases: 1 | 1 passed +assertions: - none - + diff --git a/test/extract.ts b/test/extract.ts index ae5ff5d35..d896dcb2d 100644 --- a/test/extract.ts +++ b/test/extract.ts @@ -54,6 +54,32 @@ describe('extractResult()', function() { }, ], }, + { + tool: 'catch2', + expected: [ + { + name: 'Fibonacci 20', + range: '+/- 11.719', + unit: 'us', + value: 43.186, + extra: '100 samples', + }, + { + name: 'Fibonacci 25', + range: '+/- 65.064', + unit: 'us', + value: 451.183, + extra: '100 samples', + }, + { + name: 'Fibonacci Integer', + range: '+/- 2', + unit: 's', + value: 123, + extra: '100 samples', + }, + ], + }, { tool: 'go', expected: [