From e32befd66b86b5a24c0c9811464f9838f0432942 Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Sun, 19 May 2024 12:48:33 +0200 Subject: [PATCH] fix: Catch2 v.3.5.0 changed output format [#241] * add test case for catch2 v3 * modify extract header regexp to work with v3 --- src/extract.ts | 2 +- test/__snapshots__/extract.spec.ts.snap | 51 ++++++++++++++++++- ...catch2_output.txt => catch2_output_v2.txt} | 0 test/data/extract/catch2_output_v3.txt | 47 +++++++++++++++++ test/extract.spec.ts | 6 ++- 5 files changed, 103 insertions(+), 3 deletions(-) rename test/data/extract/{catch2_output.txt => catch2_output_v2.txt} (100%) create mode 100644 test/data/extract/catch2_output_v3.txt diff --git a/src/extract.ts b/src/extract.ts index a582e8907..014517667 100644 --- a/src/extract.ts +++ b/src/extract.ts @@ -473,7 +473,7 @@ function extractCatch2Result(output: string): BenchmarkResult[] { // 43.186 us 41.402 us 46.246 us <-- Actual benchmark data // 11.719 us 7.847 us 17.747 us <-- Ignored - const reTestCaseStart = /^benchmark name +samples +iterations +estimated/; + const reTestCaseStart = /^benchmark name +samples +iterations +(estimated|est run time)/; const reBenchmarkStart = /(\d+) +(\d+) +(?:\d+(\.\d+)?) (?:ns|ms|us|s)\s*$/; const reBenchmarkValues = /^ +(\d+(?:\.\d+)?) (ns|us|ms|s) +(?:\d+(?:\.\d+)?) (?:ns|us|ms|s) +(?:\d+(?:\.\d+)?) (?:ns|us|ms|s)/; diff --git a/test/__snapshots__/extract.spec.ts.snap b/test/__snapshots__/extract.spec.ts.snap index 19f473cf9..36b02e2ba 100644 --- a/test/__snapshots__/extract.spec.ts.snap +++ b/test/__snapshots__/extract.spec.ts.snap @@ -245,7 +245,56 @@ exports[`extractResult() extracts benchmark output from cargo - criterion_output } `; -exports[`extractResult() extracts benchmark output from catch2 - catch2_output.txt 1`] = ` +exports[`extractResult() extracts benchmark output from catch2 - catch2_output_v2.txt 1`] = ` +{ + "benches": [ + { + "extra": "100 samples +208 iterations", + "name": "Fibonacci 10", + "range": "± 19", + "unit": "ns", + "value": 344, + }, + { + "extra": "100 samples +2 iterations", + "name": "Fibonacci 20", + "range": "± 3.256", + "unit": "us", + "value": 41.731, + }, + { + "extra": "100 samples +1961 iterations", + "name": "Fibonacci~ 5!", + "range": "± 4", + "unit": "ns", + "value": 36, + }, + { + "extra": "100 samples +20 iterations", + "name": "Fibonacci-15_bench", + "range": "± 362", + "unit": "us", + "value": 3.789, + }, + ], + "commit": { + "author": null, + "committer": null, + "id": "123456789abcdef", + "message": "this is dummy", + "timestamp": "dummy timestamp", + "url": "https://github.com/dummy/repo", + }, + "date": 1712131503296, + "tool": "catch2", +} +`; + +exports[`extractResult() extracts benchmark output from catch2 - catch2_output_v3.txt 1`] = ` { "benches": [ { diff --git a/test/data/extract/catch2_output.txt b/test/data/extract/catch2_output_v2.txt similarity index 100% rename from test/data/extract/catch2_output.txt rename to test/data/extract/catch2_output_v2.txt diff --git a/test/data/extract/catch2_output_v3.txt b/test/data/extract/catch2_output_v3.txt new file mode 100644 index 000000000..5f05be73f --- /dev/null +++ b/test/data/extract/catch2_output_v3.txt @@ -0,0 +1,47 @@ + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Catch2_bench is a Catch v2.11.0 host application. +Run with -? for options + +------------------------------------------------------------------------------- +Fibonacci +------------------------------------------------------------------------------- +/Users/rhayasd/Develop/github.com/benchmark-action/github-action-benchmark/examples/catch2/catch2_bench.cpp:5 +............................................................................... + +benchmark name samples iterations est run time + mean low mean high mean + std dev low std dev high std dev +------------------------------------------------------------------------------- +Fibonacci 10 100 208 7.1968 ms + 344 ns 341 ns 349 ns + 19 ns 11 ns 29 ns + +Fibonacci 20 100 2 8.3712 ms + 41.731 us 41.25 us 42.622 us + 3.256 us 2.163 us 5.353 us + + +------------------------------------------------------------------------------- +More Fibonacci +------------------------------------------------------------------------------- +/Users/rhayasd/Develop/github.com/benchmark-action/github-action-benchmark/examples/catch2/catch2_bench.cpp:13 +............................................................................... + +benchmark name samples iterations estimated + mean low mean high mean + std dev low std dev high std dev +------------------------------------------------------------------------------- +Fibonacci~ 5! 100 1961 7.0596 ms + 36 ns 35 ns 37 ns + 4 ns 3 ns 6 ns + +Fibonacci-15_bench 100 20 7.48 ms + 3.789 us 3.734 us 3.888 us + 362 ns 234 ns 539 ns + + +=============================================================================== +test cases: 2 | 2 passed +assertions: - none - + diff --git a/test/extract.spec.ts b/test/extract.spec.ts index b49567bec..911e442b3 100644 --- a/test/extract.spec.ts +++ b/test/extract.spec.ts @@ -77,7 +77,11 @@ describe('extractResult()', function () { }, { tool: 'catch2', - file: 'catch2_output.txt', + file: 'catch2_output_v2.txt', + }, + { + tool: 'catch2', + file: 'catch2_output_v3.txt', }, { tool: 'go',