From fc8cd6898e596bd56e4b183aa108039c80b7195a Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Sun, 19 May 2024 12:06:34 +0200 Subject: [PATCH 01/10] fix: Catch2 v.3.5.0 changed output format [#241] * update catch2 version --- examples/catch2/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/catch2/CMakeLists.txt b/examples/catch2/CMakeLists.txt index 6ed2214c2..0af808cb6 100644 --- a/examples/catch2/CMakeLists.txt +++ b/examples/catch2/CMakeLists.txt @@ -6,7 +6,7 @@ include(FetchContent) FetchContent_Declare( catch2 GIT_REPOSITORY https://github.com/catchorg/Catch2.git - GIT_TAG v2.11.0) + GIT_TAG v3.6.0) FetchContent_GetProperties(catch2) if(NOT catch2_POPULATED) From b481d474689d2af0268cf911144c4c6999936750 Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Sun, 19 May 2024 12:17:33 +0200 Subject: [PATCH 02/10] fix: Catch2 v.3.5.0 changed output format [#241] * fix catch2_bench.cpp file --- examples/catch2/CMakeLists.txt | 2 +- examples/catch2/catch2_bench.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/catch2/CMakeLists.txt b/examples/catch2/CMakeLists.txt index 0af808cb6..844f5324e 100644 --- a/examples/catch2/CMakeLists.txt +++ b/examples/catch2/CMakeLists.txt @@ -16,7 +16,7 @@ endif() add_executable(${PROJECT_NAME}) target_sources(${PROJECT_NAME} PRIVATE catch2_bench.cpp) -target_link_libraries(${PROJECT_NAME} Catch2::Catch2) +target_link_libraries(${PROJECT_NAME} Catch2::Catch2WithMain) target_compile_options( ${PROJECT_NAME} diff --git a/examples/catch2/catch2_bench.cpp b/examples/catch2/catch2_bench.cpp index 4c2880410..b0f58cb45 100644 --- a/examples/catch2/catch2_bench.cpp +++ b/examples/catch2/catch2_bench.cpp @@ -1,6 +1,6 @@ #include "fib.hpp" -#define CATCH_CONFIG_MAIN -#include +#include +#include TEST_CASE("Fibonacci") { From e32befd66b86b5a24c0c9811464f9838f0432942 Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Sun, 19 May 2024 12:48:33 +0200 Subject: [PATCH 03/10] fix: Catch2 v.3.5.0 changed output format [#241] * add test case for catch2 v3 * modify extract header regexp to work with v3 --- src/extract.ts | 2 +- test/__snapshots__/extract.spec.ts.snap | 51 ++++++++++++++++++- ...catch2_output.txt => catch2_output_v2.txt} | 0 test/data/extract/catch2_output_v3.txt | 47 +++++++++++++++++ test/extract.spec.ts | 6 ++- 5 files changed, 103 insertions(+), 3 deletions(-) rename test/data/extract/{catch2_output.txt => catch2_output_v2.txt} (100%) create mode 100644 test/data/extract/catch2_output_v3.txt diff --git a/src/extract.ts b/src/extract.ts index a582e8907..014517667 100644 --- a/src/extract.ts +++ b/src/extract.ts @@ -473,7 +473,7 @@ function extractCatch2Result(output: string): BenchmarkResult[] { // 43.186 us 41.402 us 46.246 us <-- Actual benchmark data // 11.719 us 7.847 us 17.747 us <-- Ignored - const reTestCaseStart = /^benchmark name +samples +iterations +estimated/; + const reTestCaseStart = /^benchmark name +samples +iterations +(estimated|est run time)/; const reBenchmarkStart = /(\d+) +(\d+) +(?:\d+(\.\d+)?) (?:ns|ms|us|s)\s*$/; const reBenchmarkValues = /^ +(\d+(?:\.\d+)?) (ns|us|ms|s) +(?:\d+(?:\.\d+)?) (?:ns|us|ms|s) +(?:\d+(?:\.\d+)?) (?:ns|us|ms|s)/; diff --git a/test/__snapshots__/extract.spec.ts.snap b/test/__snapshots__/extract.spec.ts.snap index 19f473cf9..36b02e2ba 100644 --- a/test/__snapshots__/extract.spec.ts.snap +++ b/test/__snapshots__/extract.spec.ts.snap @@ -245,7 +245,56 @@ exports[`extractResult() extracts benchmark output from cargo - criterion_output } `; -exports[`extractResult() extracts benchmark output from catch2 - catch2_output.txt 1`] = ` +exports[`extractResult() extracts benchmark output from catch2 - catch2_output_v2.txt 1`] = ` +{ + "benches": [ + { + "extra": "100 samples +208 iterations", + "name": "Fibonacci 10", + "range": "± 19", + "unit": "ns", + "value": 344, + }, + { + "extra": "100 samples +2 iterations", + "name": "Fibonacci 20", + "range": "± 3.256", + "unit": "us", + "value": 41.731, + }, + { + "extra": "100 samples +1961 iterations", + "name": "Fibonacci~ 5!", + "range": "± 4", + "unit": "ns", + "value": 36, + }, + { + "extra": "100 samples +20 iterations", + "name": "Fibonacci-15_bench", + "range": "± 362", + "unit": "us", + "value": 3.789, + }, + ], + "commit": { + "author": null, + "committer": null, + "id": "123456789abcdef", + "message": "this is dummy", + "timestamp": "dummy timestamp", + "url": "https://github.com/dummy/repo", + }, + "date": 1712131503296, + "tool": "catch2", +} +`; + +exports[`extractResult() extracts benchmark output from catch2 - catch2_output_v3.txt 1`] = ` { "benches": [ { diff --git a/test/data/extract/catch2_output.txt b/test/data/extract/catch2_output_v2.txt similarity index 100% rename from test/data/extract/catch2_output.txt rename to test/data/extract/catch2_output_v2.txt diff --git a/test/data/extract/catch2_output_v3.txt b/test/data/extract/catch2_output_v3.txt new file mode 100644 index 000000000..5f05be73f --- /dev/null +++ b/test/data/extract/catch2_output_v3.txt @@ -0,0 +1,47 @@ + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Catch2_bench is a Catch v2.11.0 host application. +Run with -? for options + +------------------------------------------------------------------------------- +Fibonacci +------------------------------------------------------------------------------- +/Users/rhayasd/Develop/github.com/benchmark-action/github-action-benchmark/examples/catch2/catch2_bench.cpp:5 +............................................................................... + +benchmark name samples iterations est run time + mean low mean high mean + std dev low std dev high std dev +------------------------------------------------------------------------------- +Fibonacci 10 100 208 7.1968 ms + 344 ns 341 ns 349 ns + 19 ns 11 ns 29 ns + +Fibonacci 20 100 2 8.3712 ms + 41.731 us 41.25 us 42.622 us + 3.256 us 2.163 us 5.353 us + + +------------------------------------------------------------------------------- +More Fibonacci +------------------------------------------------------------------------------- +/Users/rhayasd/Develop/github.com/benchmark-action/github-action-benchmark/examples/catch2/catch2_bench.cpp:13 +............................................................................... + +benchmark name samples iterations estimated + mean low mean high mean + std dev low std dev high std dev +------------------------------------------------------------------------------- +Fibonacci~ 5! 100 1961 7.0596 ms + 36 ns 35 ns 37 ns + 4 ns 3 ns 6 ns + +Fibonacci-15_bench 100 20 7.48 ms + 3.789 us 3.734 us 3.888 us + 362 ns 234 ns 539 ns + + +=============================================================================== +test cases: 2 | 2 passed +assertions: - none - + diff --git a/test/extract.spec.ts b/test/extract.spec.ts index b49567bec..911e442b3 100644 --- a/test/extract.spec.ts +++ b/test/extract.spec.ts @@ -77,7 +77,11 @@ describe('extractResult()', function () { }, { tool: 'catch2', - file: 'catch2_output.txt', + file: 'catch2_output_v2.txt', + }, + { + tool: 'catch2', + file: 'catch2_output_v3.txt', }, { tool: 'go', From 91e03035d0a31c6399b219b7220019a47a844fd3 Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Sun, 19 May 2024 12:51:21 +0200 Subject: [PATCH 04/10] fix: Catch2 v.3.5.0 changed output format [#241] * revert changes to old catch2 example * add new catch2 v3 example --- examples/catch2/CMakeLists.txt | 4 +- examples/catch2/catch2_bench.cpp | 4 +- examples/catch2_v3/.gitignore | 1 + examples/catch2_v3/CMakeLists.txt | 26 +++++++++++ examples/catch2_v3/README.md | 68 +++++++++++++++++++++++++++++ examples/catch2_v3/catch2_bench.cpp | 11 +++++ examples/catch2_v3/fib.hpp | 11 +++++ 7 files changed, 121 insertions(+), 4 deletions(-) create mode 100644 examples/catch2_v3/.gitignore create mode 100644 examples/catch2_v3/CMakeLists.txt create mode 100644 examples/catch2_v3/README.md create mode 100644 examples/catch2_v3/catch2_bench.cpp create mode 100644 examples/catch2_v3/fib.hpp diff --git a/examples/catch2/CMakeLists.txt b/examples/catch2/CMakeLists.txt index 844f5324e..6ed2214c2 100644 --- a/examples/catch2/CMakeLists.txt +++ b/examples/catch2/CMakeLists.txt @@ -6,7 +6,7 @@ include(FetchContent) FetchContent_Declare( catch2 GIT_REPOSITORY https://github.com/catchorg/Catch2.git - GIT_TAG v3.6.0) + GIT_TAG v2.11.0) FetchContent_GetProperties(catch2) if(NOT catch2_POPULATED) @@ -16,7 +16,7 @@ endif() add_executable(${PROJECT_NAME}) target_sources(${PROJECT_NAME} PRIVATE catch2_bench.cpp) -target_link_libraries(${PROJECT_NAME} Catch2::Catch2WithMain) +target_link_libraries(${PROJECT_NAME} Catch2::Catch2) target_compile_options( ${PROJECT_NAME} diff --git a/examples/catch2/catch2_bench.cpp b/examples/catch2/catch2_bench.cpp index b0f58cb45..4c2880410 100644 --- a/examples/catch2/catch2_bench.cpp +++ b/examples/catch2/catch2_bench.cpp @@ -1,6 +1,6 @@ #include "fib.hpp" -#include -#include +#define CATCH_CONFIG_MAIN +#include TEST_CASE("Fibonacci") { diff --git a/examples/catch2_v3/.gitignore b/examples/catch2_v3/.gitignore new file mode 100644 index 000000000..567609b12 --- /dev/null +++ b/examples/catch2_v3/.gitignore @@ -0,0 +1 @@ +build/ diff --git a/examples/catch2_v3/CMakeLists.txt b/examples/catch2_v3/CMakeLists.txt new file mode 100644 index 000000000..844f5324e --- /dev/null +++ b/examples/catch2_v3/CMakeLists.txt @@ -0,0 +1,26 @@ +cmake_minimum_required(VERSION 3.12) +project("Catch2_bench") + +include(FetchContent) + +FetchContent_Declare( + catch2 + GIT_REPOSITORY https://github.com/catchorg/Catch2.git + GIT_TAG v3.6.0) + +FetchContent_GetProperties(catch2) +if(NOT catch2_POPULATED) + FetchContent_Populate(catch2) + add_subdirectory(${catch2_SOURCE_DIR} ${catch2_BINARY_DIR}) +endif() + +add_executable(${PROJECT_NAME}) +target_sources(${PROJECT_NAME} PRIVATE catch2_bench.cpp) +target_link_libraries(${PROJECT_NAME} Catch2::Catch2WithMain) + +target_compile_options( + ${PROJECT_NAME} + PRIVATE + $<$:/DCATCH_CONFIG_ENABLE_BENCHMARKING> + $<$,$,$>:-DCATCH_CONFIG_ENABLE_BENCHMARKING> +) diff --git a/examples/catch2_v3/README.md b/examples/catch2_v3/README.md new file mode 100644 index 000000000..b22944b36 --- /dev/null +++ b/examples/catch2_v3/README.md @@ -0,0 +1,68 @@ +C++ example for benchmarking with [Catch2 Framework][tool] +==================================================================== + +- [Workflow for this example](../../.github/workflows/catch2.yml) +- [Action log of this example](https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22Catch2+C%2B%2B+Example%22) +- [Benchmark results on GitHub pages](https://benchmark-action.github.io/github-action-benchmark/dev/bench/) + +This directory shows how to use [`github-action-benchmark`][action] with [Catch2 Framework][tool]. + + + +## Run benchmarks + +Official documentation for usage of Catch2 Framework can be found in its repository: + +https://github.com/catchorg/Catch2 + +Since Catch2 is a header-only test framework, you don't need to build it in advance. +Download and put the headers in your `include` directory and write your benchmarks. + +```cpp +#define CATCH_CONFIG_MAIN +#include + +TEST_CASE("Fibonacci") { + // now let's benchmark: + BENCHMARK("Some benchmark") { + // Your benchmark goes here + }; +} +``` + +Build the source with C++ compiler and run the built executable to get the benchmark output. +Ensure to use `console` reporter for this. `xml` reporter may be supported in the future. + + + +## Process benchmark results + +Store the benchmark results with step using the action. Please set `catch2` to `tool` input. + +```yaml +- name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + tool: 'catch2' + output-file-path: benchmark_result.json +``` + +Please read ['How to use' section](https://github.com/benchmark-action/github-action-benchmark#how-to-use) for common usage. + + + +## Run this example + +To try this example, please use [cmake](./CMakeLists.txt) and `clang++`. + +```sh +$ mkdir build +$ cd build +$ cmake -DCMAKE_BUILD_TYPE=Release .. +$ cmake --build . --config Release +``` + +This will create `Catch2_bench` executable. The results are output to stdout. + +[tool]: https://github.com/catchorg/Catch2 +[action]: https://github.com/benchmark-action/github-action-benchmark diff --git a/examples/catch2_v3/catch2_bench.cpp b/examples/catch2_v3/catch2_bench.cpp new file mode 100644 index 000000000..b0f58cb45 --- /dev/null +++ b/examples/catch2_v3/catch2_bench.cpp @@ -0,0 +1,11 @@ +#include "fib.hpp" +#include +#include + +TEST_CASE("Fibonacci") { + + // now let's benchmark: + BENCHMARK("Fibonacci 10") { return fib(10); }; + + BENCHMARK("Fibonacci 20") { return fib(20); }; +} diff --git a/examples/catch2_v3/fib.hpp b/examples/catch2_v3/fib.hpp new file mode 100644 index 000000000..559e886c8 --- /dev/null +++ b/examples/catch2_v3/fib.hpp @@ -0,0 +1,11 @@ +#if !defined FIB_HPP_INCLUDED +#define FIB_HPP_INCLUDED + +int fib(int const i) { + if (i <= 1) { + return 1; + } + return fib(i - 2) + fib(i - 1); +} + +#endif // FIB_HPP_INCLUDED From e970805ac7e68edb45a3d011db08469da7f87808 Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Sun, 19 May 2024 12:53:51 +0200 Subject: [PATCH 05/10] rename examples/catch2 directory to catch2_v2 --- examples/{catch2 => catch2_v2}/.gitignore | 0 examples/{catch2 => catch2_v2}/CMakeLists.txt | 0 examples/{catch2 => catch2_v2}/README.md | 0 examples/{catch2 => catch2_v2}/catch2_bench.cpp | 0 examples/{catch2 => catch2_v2}/fib.hpp | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename examples/{catch2 => catch2_v2}/.gitignore (100%) rename examples/{catch2 => catch2_v2}/CMakeLists.txt (100%) rename examples/{catch2 => catch2_v2}/README.md (100%) rename examples/{catch2 => catch2_v2}/catch2_bench.cpp (100%) rename examples/{catch2 => catch2_v2}/fib.hpp (100%) diff --git a/examples/catch2/.gitignore b/examples/catch2_v2/.gitignore similarity index 100% rename from examples/catch2/.gitignore rename to examples/catch2_v2/.gitignore diff --git a/examples/catch2/CMakeLists.txt b/examples/catch2_v2/CMakeLists.txt similarity index 100% rename from examples/catch2/CMakeLists.txt rename to examples/catch2_v2/CMakeLists.txt diff --git a/examples/catch2/README.md b/examples/catch2_v2/README.md similarity index 100% rename from examples/catch2/README.md rename to examples/catch2_v2/README.md diff --git a/examples/catch2/catch2_bench.cpp b/examples/catch2_v2/catch2_bench.cpp similarity index 100% rename from examples/catch2/catch2_bench.cpp rename to examples/catch2_v2/catch2_bench.cpp diff --git a/examples/catch2/fib.hpp b/examples/catch2_v2/fib.hpp similarity index 100% rename from examples/catch2/fib.hpp rename to examples/catch2_v2/fib.hpp From 8cd4821cec19a2c6a2b22f85f0a30fe7a17a5338 Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Sun, 19 May 2024 12:54:17 +0200 Subject: [PATCH 06/10] rename examples/catch2_v3 directory to catch2 --- examples/{catch2_v3 => catch2}/.gitignore | 0 examples/{catch2_v3 => catch2}/CMakeLists.txt | 0 examples/{catch2_v3 => catch2}/README.md | 0 examples/{catch2_v3 => catch2}/catch2_bench.cpp | 0 examples/{catch2_v3 => catch2}/fib.hpp | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename examples/{catch2_v3 => catch2}/.gitignore (100%) rename examples/{catch2_v3 => catch2}/CMakeLists.txt (100%) rename examples/{catch2_v3 => catch2}/README.md (100%) rename examples/{catch2_v3 => catch2}/catch2_bench.cpp (100%) rename examples/{catch2_v3 => catch2}/fib.hpp (100%) diff --git a/examples/catch2_v3/.gitignore b/examples/catch2/.gitignore similarity index 100% rename from examples/catch2_v3/.gitignore rename to examples/catch2/.gitignore diff --git a/examples/catch2_v3/CMakeLists.txt b/examples/catch2/CMakeLists.txt similarity index 100% rename from examples/catch2_v3/CMakeLists.txt rename to examples/catch2/CMakeLists.txt diff --git a/examples/catch2_v3/README.md b/examples/catch2/README.md similarity index 100% rename from examples/catch2_v3/README.md rename to examples/catch2/README.md diff --git a/examples/catch2_v3/catch2_bench.cpp b/examples/catch2/catch2_bench.cpp similarity index 100% rename from examples/catch2_v3/catch2_bench.cpp rename to examples/catch2/catch2_bench.cpp diff --git a/examples/catch2_v3/fib.hpp b/examples/catch2/fib.hpp similarity index 100% rename from examples/catch2_v3/fib.hpp rename to examples/catch2/fib.hpp From c8b06710c4ea1759e70eca64caf3f2c85e4a8191 Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Sun, 19 May 2024 12:55:22 +0200 Subject: [PATCH 07/10] update catch2 v2 example to lastest v2 version --- examples/catch2_v2/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/catch2_v2/CMakeLists.txt b/examples/catch2_v2/CMakeLists.txt index 6ed2214c2..5d0ba3153 100644 --- a/examples/catch2_v2/CMakeLists.txt +++ b/examples/catch2_v2/CMakeLists.txt @@ -6,7 +6,7 @@ include(FetchContent) FetchContent_Declare( catch2 GIT_REPOSITORY https://github.com/catchorg/Catch2.git - GIT_TAG v2.11.0) + GIT_TAG v2.13.10) FetchContent_GetProperties(catch2) if(NOT catch2_POPULATED) From 3ee9b5e8df64d43aac977044e7a6ac41d4244505 Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Sun, 19 May 2024 13:05:02 +0200 Subject: [PATCH 08/10] add Catch2 v2 and v3 to CI workflows --- .github/workflows/catch2.yml | 10 +++--- .github/workflows/catch2_v2.yml | 52 +++++++++++++++++++++++++++ .github/workflows/ci-results-repo.yml | 45 +++++++++++++++++++++-- .github/workflows/ci.yml | 45 ++++++++++++++++++++--- 4 files changed, 140 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/catch2_v2.yml diff --git a/.github/workflows/catch2.yml b/.github/workflows/catch2.yml index 4c108fbca..41bbe7b4d 100644 --- a/.github/workflows/catch2.yml +++ b/.github/workflows/catch2.yml @@ -10,7 +10,7 @@ permissions: jobs: benchmark: - name: Run C++ benchmark example + name: Run Catch2 C++ Benchmark Framework example (v3.x) runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v4 @@ -25,7 +25,7 @@ jobs: - name: Store benchmark result uses: benchmark-action/github-action-benchmark@v1 with: - name: Catch2 Benchmark + name: Catch2 Benchmark (v3) tool: 'catch2' output-file-path: examples/catch2/benchmark_result.txt github-token: ${{ secrets.GITHUB_TOKEN }} @@ -34,12 +34,12 @@ jobs: alert-threshold: '200%' comment-on-alert: true fail-on-alert: true - alert-comment-cc-users: '@bernedom,@ktrz' + alert-comment-cc-users: '@ktrz' - name: Store benchmark result - separate results repo uses: benchmark-action/github-action-benchmark@v1 with: - name: Catch2 Benchmark + name: Catch2 Benchmark (v3) tool: 'catch2' output-file-path: examples/catch2/benchmark_result.txt github-token: ${{ secrets.BENCHMARK_ACTION_BOT_TOKEN }} @@ -48,5 +48,5 @@ jobs: alert-threshold: '200%' comment-on-alert: true fail-on-alert: true - alert-comment-cc-users: '@bernedom,@ktrz' + alert-comment-cc-users: '@ktrz' gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' diff --git a/.github/workflows/catch2_v2.yml b/.github/workflows/catch2_v2.yml new file mode 100644 index 000000000..d28b86704 --- /dev/null +++ b/.github/workflows/catch2_v2.yml @@ -0,0 +1,52 @@ +name: Catch2 C++ Example +on: + push: + branches: + - master + +permissions: + contents: write + deployments: write + +jobs: + benchmark: + name: Run Catch2 C++ Benchmark Framework example (v2.x) + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + - name: Build and run benchmarks with Catch2 + run: | + cd examples/catch2_v2 + mkdir build && cd build + cmake -DCMAKE_BUILD_TYPE=Release .. + cmake --build . --config Release + ./Catch2_bench | tee ../benchmark_result.txt + + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + name: Catch2 Benchmark + tool: 'catch2' + output-file-path: examples/catch2_v2/benchmark_result.txt + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: true + # Show alert with commit comment on detecting possible performance regression + alert-threshold: '200%' + comment-on-alert: true + fail-on-alert: true + alert-comment-cc-users: '@ktrz' + + - name: Store benchmark result - separate results repo + uses: benchmark-action/github-action-benchmark@v1 + with: + name: Catch2 Benchmark + tool: 'catch2' + output-file-path: examples/catch2_v2/benchmark_result.txt + github-token: ${{ secrets.BENCHMARK_ACTION_BOT_TOKEN }} + auto-push: true + # Show alert with commit comment on detecting possible performance regression + alert-threshold: '200%' + comment-on-alert: true + fail-on-alert: true + alert-comment-cc-users: '@ktrz' + gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' diff --git a/.github/workflows/ci-results-repo.yml b/.github/workflows/ci-results-repo.yml index 5a1ca41de..ee453506f 100644 --- a/.github/workflows/ci-results-repo.yml +++ b/.github/workflows/ci-results-repo.yml @@ -79,7 +79,7 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.js Benchmark' './benchmark-data-repository' - catch2-framework: + catch2-v2-framework: name: Run Catch2 C++ Benchmark Framework example - github.com/benchmark-action/github-action-benchmark-results runs-on: ubuntu-20.04 steps: @@ -92,7 +92,7 @@ jobs: - run: npm run build - name: Run benchmark run: | - cd examples/catch2 + cd examples/catch2_v2 mkdir build && cd build cmake -DCMAKE_BUILD_TYPE=Release .. cmake --build . --config Release @@ -112,12 +112,51 @@ jobs: with: name: Catch2 Benchmark tool: 'catch2' - output-file-path: examples/catch2/benchmark_result.txt + output-file-path: examples/catch2_v2/benchmark_result.txt fail-on-alert: true gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' github-token: ${{ secrets.GITHUB_TOKEN }} - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Catch2 Benchmark' './benchmark-data-repository' + catch2-v3-framework: + name: Run Catch2 C++ Benchmark Framework example - github.com/benchmark-action/github-action-benchmark-results + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + - run: npm ci + - run: npm run build + - name: Run benchmark + run: | + cd examples/catch2 + mkdir build && cd build + cmake -DCMAKE_BUILD_TYPE=Release .. + cmake --build . --config Release + ./Catch2_bench > ../benchmark_result.txt + + - uses: actions/checkout@v4 + with: + repository: benchmark-action/github-action-benchmark-results + ref: 'gh-pages' + path: 'dist/other-repo' + - name: Save previous data.js + run: | + cp ./dist/other-repo/dev/bench/data.js before_data.js + + - name: Store benchmark result + uses: ./ + with: + name: Catch2 Benchmark (v3) + tool: 'catch2' + output-file-path: examples/catch2/benchmark_result.txt + fail-on-alert: true + gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' + github-token: ${{ secrets.GITHUB_TOKEN }} + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Catch2 Benchmark (v3)' './benchmark-data-repository' + cpp-framework: name: Run Google C++ Benchmark Framework example - github.com/benchmark-action/github-action-benchmark-results runs-on: ubuntu-20.04 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 071287f70..274cf633b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -75,8 +75,8 @@ jobs: comment-on-alert: true - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.js Benchmark' - catch2-framework: - name: Run Catch2 C++ Benchmark Framework example + catch2-v2-framework: + name: Run Catch2 C++ Benchmark Framework example (v2.x) runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v4 @@ -88,7 +88,7 @@ jobs: - run: npm run build - name: Run benchmark run: | - cd examples/catch2 + cd examples/catch2_v2 mkdir build && cd build cmake -DCMAKE_BUILD_TYPE=Release .. cmake --build . --config Release @@ -104,7 +104,7 @@ jobs: with: name: Catch2 Benchmark tool: 'catch2' - output-file-path: examples/catch2/benchmark_result.txt + output-file-path: examples/catch2_v2/benchmark_result.txt skip-fetch-gh-pages: true fail-on-alert: true summary-always: true @@ -112,6 +112,43 @@ jobs: comment-on-alert: true - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Catch2 Benchmark' + catch2-v3-framework: + name: Run Catch2 C++ Benchmark Framework example (v3.x) + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + - run: npm ci + - run: npm run build + - name: Run benchmark + run: | + cd examples/catch2 + mkdir build && cd build + cmake -DCMAKE_BUILD_TYPE=Release .. + cmake --build . --config Release + ./Catch2_bench > ../benchmark_result.txt + - name: Save previous data.js + run: | + git fetch origin gh-pages + git checkout gh-pages + cp ./dev/bench/data.js before_data.js + git checkout - + - name: Store benchmark result + uses: ./ + with: + name: Catch2 Benchmark (v3) + tool: 'catch2' + output-file-path: examples/catch2/benchmark_result.txt + skip-fetch-gh-pages: true + fail-on-alert: true + summary-always: true + github-token: ${{ secrets.GITHUB_TOKEN }} + comment-on-alert: true + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Catch2 Benchmark (v3)' + cpp-framework: name: Run Google C++ Benchmark Framework example runs-on: ubuntu-20.04 From 207e19cad065fb8ce7789ec594f710d01a90d033 Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Sun, 19 May 2024 13:06:43 +0200 Subject: [PATCH 09/10] update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bdd79dc65..348f0b2c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ ## Unreleased +- **fix** Catch2 v.3.5.0 changed output format [#241] # [v1.20.2](https://github.com/benchmark-action/github-action-benchmark/releases/tag/v1.20.2) - 19 May 2024 From 9895d74177f7a107abe73eed0feab817863680f8 Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Sun, 19 May 2024 13:10:00 +0200 Subject: [PATCH 10/10] update ci-results-repo.yml --- .github/workflows/ci-results-repo.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-results-repo.yml b/.github/workflows/ci-results-repo.yml index ee453506f..59356f26f 100644 --- a/.github/workflows/ci-results-repo.yml +++ b/.github/workflows/ci-results-repo.yml @@ -80,7 +80,7 @@ jobs: - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.js Benchmark' './benchmark-data-repository' catch2-v2-framework: - name: Run Catch2 C++ Benchmark Framework example - github.com/benchmark-action/github-action-benchmark-results + name: Run Catch2 C++ Benchmark Framework example (v2.x) - github.com/benchmark-action/github-action-benchmark-results runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v4 @@ -119,7 +119,7 @@ jobs: - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Catch2 Benchmark' './benchmark-data-repository' catch2-v3-framework: - name: Run Catch2 C++ Benchmark Framework example - github.com/benchmark-action/github-action-benchmark-results + name: Run Catch2 C++ Benchmark Framework example (v3.x) - github.com/benchmark-action/github-action-benchmark-results runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v4