From 437c71feb6861453bdadea71abacaa0a829bb243 Mon Sep 17 00:00:00 2001 From: raar1 Date: Sun, 1 Dec 2024 00:00:07 +0100 Subject: [PATCH 01/17] Add benchmarking suite for the handwritten poisson example --- .github/workflows/benchmarks.yml | 47 +++++++++++++++++++ .../benchmarks/run_benchmarks.jl | 34 ++++++++++++++ 2 files changed, 81 insertions(+) create mode 100644 .github/workflows/benchmarks.yml create mode 100644 GalerkinToolkitExamples/benchmarks/run_benchmarks.jl diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml new file mode 100644 index 00000000..94e21608 --- /dev/null +++ b/.github/workflows/benchmarks.yml @@ -0,0 +1,47 @@ +name: Performance Regression Tests +on: + push: + branches: + - main + +permissions: + contents: write + deployments: write + +jobs: + benchmark: + name: Run julia benchmark example + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + version: + - '1.10' + os: + - ubuntu-latest + arch: + - x64 + steps: + - uses: actions/checkout@v4 + - uses: julia-actions/setup-julia@v1 + with: + version: ${{ matrix.version }} + arch: ${{ matrix.arch }} + - uses: actions/cache@v4 + env: + cache-name: cache-artifacts + with: + path: ~/.julia/artifacts + key: runner.os−test−env.cache−name−{{ hashFiles('**/Project.toml') }} + restore-keys: | + runner.os−test− + ${{ env.cache-name }}- + ${{ runner.os }}-test- + ${{ runner.os }}- + - name: Run benchmark + run: | + cd performance_tests/ + julia --project --color=yes -e ' + using Pkg; + Pkg.instantiate(); + include("poisson.jl")' diff --git a/GalerkinToolkitExamples/benchmarks/run_benchmarks.jl b/GalerkinToolkitExamples/benchmarks/run_benchmarks.jl new file mode 100644 index 00000000..956e8725 --- /dev/null +++ b/GalerkinToolkitExamples/benchmarks/run_benchmarks.jl @@ -0,0 +1,34 @@ +module GalerkinToolkitBenchmarkTests + +using BenchmarkTools + +import GalerkinToolkit as GT +using GalerkinToolkitExamples: Poisson + + +function handwritten_poisson(n) + mesh = GT.cartesian_mesh((0,2,0,2),(n,n)) + + params = Dict{Symbol,Any}() + params[:implementation] = :hand_written + params[:mesh] = mesh + params[:dirichlet_tags] = ["1-face-1","1-face-2","1-face-3","1-face-4"] + params[:discretization_method] = :continuous_galerkin + params[:dirichlet_method] = :strong + params[:integration_degree] = 2 + + Poisson.main(params) +end + + +suite = BenchmarkGroup() +suite["poisson"] = BenchmarkGroup(["tag1", "tag2"]) +suite["poisson"][2] = @benchmarkable handwritten_poisson(2) +suite["poisson"][3] = @benchmarkable handwritten_poisson(3) + +tune!(suite) +results = run(suite, verbose = true) + +BenchmarkTools.save("output.json", median(results)) + +end # module From cd77e6b465cc57fa0453b24fee56ceafe1ca6ce2 Mon Sep 17 00:00:00 2001 From: raar1 Date: Sun, 1 Dec 2024 14:27:31 +0100 Subject: [PATCH 02/17] Trigger benchmarks on PRs temporarily for testing purposes --- .github/workflows/benchmarks.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 94e21608..29a14d80 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -3,6 +3,7 @@ on: push: branches: - main + pull_request: permissions: contents: write From 683e692720fbc4b7ad0177da6a0ab68016f4b250 Mon Sep 17 00:00:00 2001 From: raar1 Date: Sun, 1 Dec 2024 14:29:44 +0100 Subject: [PATCH 03/17] Fix wrong path --- .github/workflows/benchmarks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 29a14d80..ac0e3a62 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -41,7 +41,7 @@ jobs: ${{ runner.os }}- - name: Run benchmark run: | - cd performance_tests/ + cd GalerkinToolkitExamples/benchmarks/ julia --project --color=yes -e ' using Pkg; Pkg.instantiate(); From 2176b3de507a489f8c1d77715ac5cee3e1b8a54c Mon Sep 17 00:00:00 2001 From: raar1 Date: Sun, 1 Dec 2024 14:33:15 +0100 Subject: [PATCH 04/17] Run benchmarks module --- .github/workflows/benchmarks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index ac0e3a62..6f759b06 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -45,4 +45,4 @@ jobs: julia --project --color=yes -e ' using Pkg; Pkg.instantiate(); - include("poisson.jl")' + include("run_benchmarks.jl")' From b9189020c5309cba87eb40a78790e95db2a6cb6d Mon Sep 17 00:00:00 2001 From: raar1 Date: Sun, 1 Dec 2024 18:55:43 +0100 Subject: [PATCH 05/17] Use benchmarktools in the examples subpackage --- .github/workflows/benchmarks.yml | 9 ++++----- GalerkinToolkitExamples/Project.toml | 1 + 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 6f759b06..a25db12d 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -41,8 +41,7 @@ jobs: ${{ runner.os }}- - name: Run benchmark run: | - cd GalerkinToolkitExamples/benchmarks/ - julia --project --color=yes -e ' - using Pkg; - Pkg.instantiate(); - include("run_benchmarks.jl")' + julia --project=GalerkinToolkitExamples -e ' + using Pkg + Pkg.develop(path=".") + include("GalerkinToolkitExamples/benchmarks/run_benchmarks.jl")' diff --git a/GalerkinToolkitExamples/Project.toml b/GalerkinToolkitExamples/Project.toml index 758d6c66..9a58e0a4 100644 --- a/GalerkinToolkitExamples/Project.toml +++ b/GalerkinToolkitExamples/Project.toml @@ -20,6 +20,7 @@ SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" TimerOutputs = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f" WriteVTK = "64499a7a-5c06-52f2-abe2-ccb03c286192" +BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" [extras] Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" From 2c11e638a52f19e5541907eef025e7419d51e430 Mon Sep 17 00:00:00 2001 From: raar1 Date: Sun, 1 Dec 2024 19:09:22 +0100 Subject: [PATCH 06/17] Try storing the result --- .github/workflows/benchmarks.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index a25db12d..cae4c2ad 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -45,3 +45,17 @@ jobs: using Pkg Pkg.develop(path=".") include("GalerkinToolkitExamples/benchmarks/run_benchmarks.jl")' + + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + name: Julia benchmark result + tool: 'julia' + output-file-path: output.json + # Use personal access token instead of GITHUB_TOKEN due to https://github.community/t/github-action-not-triggering-gh-pages-upon-push/16096 + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: true + # Show alert with commit comment on detecting possible performance regression + alert-threshold: '200%' + comment-on-alert: true + fail-on-alert: true From 48be109d3b3bbe30605f8f02b60940b9d937fa16 Mon Sep 17 00:00:00 2001 From: raar1 Date: Sun, 1 Dec 2024 20:23:24 +0100 Subject: [PATCH 07/17] Pick a larger problem for the poisson benchmark --- GalerkinToolkitExamples/benchmarks/run_benchmarks.jl | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/GalerkinToolkitExamples/benchmarks/run_benchmarks.jl b/GalerkinToolkitExamples/benchmarks/run_benchmarks.jl index 956e8725..89dd791d 100644 --- a/GalerkinToolkitExamples/benchmarks/run_benchmarks.jl +++ b/GalerkinToolkitExamples/benchmarks/run_benchmarks.jl @@ -20,13 +20,12 @@ function handwritten_poisson(n) Poisson.main(params) end - suite = BenchmarkGroup() -suite["poisson"] = BenchmarkGroup(["tag1", "tag2"]) -suite["poisson"][2] = @benchmarkable handwritten_poisson(2) -suite["poisson"][3] = @benchmarkable handwritten_poisson(3) +suite["poisson"] = BenchmarkGroup(["Poisson", "handwritten"]) +suite["poisson"]["n=200"] = @benchmarkable handwritten_poisson(200) tune!(suite) + results = run(suite, verbose = true) BenchmarkTools.save("output.json", median(results)) From 7fa1c6af93ef1e62702ab1eec107346d86eb13c4 Mon Sep 17 00:00:00 2001 From: raar1 Date: Sun, 1 Dec 2024 20:41:18 +0100 Subject: [PATCH 08/17] Try slightly less big benchmark for now so that things run a bit faster --- GalerkinToolkitExamples/benchmarks/run_benchmarks.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GalerkinToolkitExamples/benchmarks/run_benchmarks.jl b/GalerkinToolkitExamples/benchmarks/run_benchmarks.jl index 89dd791d..8cd6c530 100644 --- a/GalerkinToolkitExamples/benchmarks/run_benchmarks.jl +++ b/GalerkinToolkitExamples/benchmarks/run_benchmarks.jl @@ -22,7 +22,7 @@ end suite = BenchmarkGroup() suite["poisson"] = BenchmarkGroup(["Poisson", "handwritten"]) -suite["poisson"]["n=200"] = @benchmarkable handwritten_poisson(200) +suite["poisson"]["n=150"] = @benchmarkable handwritten_poisson(150) tune!(suite) From 6c26082fab11bfa95aca0b07db2a17835e0fa320 Mon Sep 17 00:00:00 2001 From: raar1 Date: Sun, 1 Dec 2024 20:51:18 +0100 Subject: [PATCH 09/17] Use a 3D mesh as benchmark system instead --- GalerkinToolkitExamples/benchmarks/run_benchmarks.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/GalerkinToolkitExamples/benchmarks/run_benchmarks.jl b/GalerkinToolkitExamples/benchmarks/run_benchmarks.jl index 8cd6c530..3ad83801 100644 --- a/GalerkinToolkitExamples/benchmarks/run_benchmarks.jl +++ b/GalerkinToolkitExamples/benchmarks/run_benchmarks.jl @@ -7,7 +7,7 @@ using GalerkinToolkitExamples: Poisson function handwritten_poisson(n) - mesh = GT.cartesian_mesh((0,2,0,2),(n,n)) + mesh = GT.cartesian_mesh((0,2,0,2,0,2),(n,n,n)) params = Dict{Symbol,Any}() params[:implementation] = :hand_written @@ -22,7 +22,7 @@ end suite = BenchmarkGroup() suite["poisson"] = BenchmarkGroup(["Poisson", "handwritten"]) -suite["poisson"]["n=150"] = @benchmarkable handwritten_poisson(150) +suite["poisson"]["n=150"] = @benchmarkable handwritten_poisson(10) tune!(suite) From c065eeb379808e8d43ec5271414cf6d0f592ed18 Mon Sep 17 00:00:00 2001 From: raar1 Date: Sun, 1 Dec 2024 22:28:50 +0100 Subject: [PATCH 10/17] Try adding the automatic poisson as a benchmark too --- .../benchmarks/run_benchmarks.jl | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/GalerkinToolkitExamples/benchmarks/run_benchmarks.jl b/GalerkinToolkitExamples/benchmarks/run_benchmarks.jl index 3ad83801..a061846e 100644 --- a/GalerkinToolkitExamples/benchmarks/run_benchmarks.jl +++ b/GalerkinToolkitExamples/benchmarks/run_benchmarks.jl @@ -7,7 +7,7 @@ using GalerkinToolkitExamples: Poisson function handwritten_poisson(n) - mesh = GT.cartesian_mesh((0,2,0,2,0,2),(n,n,n)) + mesh = GT.cartesian_mesh((0,2,0,2,0,2), (n,n,n)) params = Dict{Symbol,Any}() params[:implementation] = :hand_written @@ -20,9 +20,26 @@ function handwritten_poisson(n) Poisson.main(params) end +function automatic_poisson(n) + mesh = GT.cartesian_mesh((0,2,0,2,0,2), (n,n,n)) + + params = Dict{Symbol,Any}() + params[:implementation] = :automatic + params[:mesh] = mesh + params[:dirichlet_tags] = ["1-face-1","1-face-2","1-face-3","1-face-4"] + params[:discretization_method] = :continuous_galerkin + params[:dirichlet_method] = :strong + params[:integration_degree] = 2 + + Poisson.main(params) +end + suite = BenchmarkGroup() -suite["poisson"] = BenchmarkGroup(["Poisson", "handwritten"]) -suite["poisson"]["n=150"] = @benchmarkable handwritten_poisson(10) +suite["poisson-hand"] = BenchmarkGroup(["Poisson", "handwritten"]) +suite["poisson-hand"]["n=10"] = @benchmarkable handwritten_poisson(10) + +suite["poisson-auto"] = BenchmarkGroup(["Poisson", "automatic"]) +suite["poisson-auto"]["n=10"] = @benchmarkable automatic_poisson(10) tune!(suite) From 3f1546f904df417deef4673fbb738820bf08eb0f Mon Sep 17 00:00:00 2001 From: raar1 Date: Sun, 1 Dec 2024 22:57:00 +0100 Subject: [PATCH 11/17] Remove automatic example --- .../benchmarks/run_benchmarks.jl | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/GalerkinToolkitExamples/benchmarks/run_benchmarks.jl b/GalerkinToolkitExamples/benchmarks/run_benchmarks.jl index a061846e..08357edb 100644 --- a/GalerkinToolkitExamples/benchmarks/run_benchmarks.jl +++ b/GalerkinToolkitExamples/benchmarks/run_benchmarks.jl @@ -20,27 +20,10 @@ function handwritten_poisson(n) Poisson.main(params) end -function automatic_poisson(n) - mesh = GT.cartesian_mesh((0,2,0,2,0,2), (n,n,n)) - - params = Dict{Symbol,Any}() - params[:implementation] = :automatic - params[:mesh] = mesh - params[:dirichlet_tags] = ["1-face-1","1-face-2","1-face-3","1-face-4"] - params[:discretization_method] = :continuous_galerkin - params[:dirichlet_method] = :strong - params[:integration_degree] = 2 - - Poisson.main(params) -end - suite = BenchmarkGroup() suite["poisson-hand"] = BenchmarkGroup(["Poisson", "handwritten"]) suite["poisson-hand"]["n=10"] = @benchmarkable handwritten_poisson(10) -suite["poisson-auto"] = BenchmarkGroup(["Poisson", "automatic"]) -suite["poisson-auto"]["n=10"] = @benchmarkable automatic_poisson(10) - tune!(suite) results = run(suite, verbose = true) From 7000bc960bbe7d9b843985241169eccffe9d1c57 Mon Sep 17 00:00:00 2001 From: raar1 Date: Sun, 1 Dec 2024 23:37:18 +0100 Subject: [PATCH 12/17] Add some comments --- GalerkinToolkitExamples/benchmarks/run_benchmarks.jl | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/GalerkinToolkitExamples/benchmarks/run_benchmarks.jl b/GalerkinToolkitExamples/benchmarks/run_benchmarks.jl index 08357edb..134a84f8 100644 --- a/GalerkinToolkitExamples/benchmarks/run_benchmarks.jl +++ b/GalerkinToolkitExamples/benchmarks/run_benchmarks.jl @@ -7,6 +7,11 @@ using GalerkinToolkitExamples: Poisson function handwritten_poisson(n) + """ + Runs the hand-written Poisson example code, for a 3D + mesh of dimensions n x n x n. + """ + mesh = GT.cartesian_mesh((0,2,0,2,0,2), (n,n,n)) params = Dict{Symbol,Any}() @@ -20,14 +25,17 @@ function handwritten_poisson(n) Poisson.main(params) end + +# Build a benchmark suite for the Poisson example suite = BenchmarkGroup() suite["poisson-hand"] = BenchmarkGroup(["Poisson", "handwritten"]) suite["poisson-hand"]["n=10"] = @benchmarkable handwritten_poisson(10) +# Run all benchmarks tune!(suite) - results = run(suite, verbose = true) +# Save benchmark results for tracking and visualization BenchmarkTools.save("output.json", median(results)) end # module From 5942af9ef3a848c4d5d2862e0299cee180315f8b Mon Sep 17 00:00:00 2001 From: raar1 Date: Mon, 2 Dec 2024 00:07:36 +0100 Subject: [PATCH 13/17] Add dev documentation on the benchmarking --- docs/src/developers_guide.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/src/developers_guide.md b/docs/src/developers_guide.md index 2e07d702..9f8991d2 100644 --- a/docs/src/developers_guide.md +++ b/docs/src/developers_guide.md @@ -1,3 +1,8 @@ # Developers guide +## Performance Benchmarks +There is a benchmark suite defined in `GalerkinToolkitExamples/benchmarks`. This uses [BenchmarkTools.jl](https://github.com/JuliaCI/BenchmarkTools.jl) to perform the timings +and [github-action-benchmark](https://github.com/benchmark-action/github-action-benchmark) to collect the results and store them in the `gh-pages` branch. Graphs of performance +changes over time (per commit hash) can be viewed here: . +More benchmarks can be added (or existing ones modified) in `GalerkinToolkitExamples/benchmarks/run_benchmarks.jl`. From 80e0e6b566fd01dcb1558ef6a91bb4734e7f2ded Mon Sep 17 00:00:00 2001 From: raar1 Date: Mon, 2 Dec 2024 00:14:41 +0100 Subject: [PATCH 14/17] Add docs about failing due to large performance change --- docs/src/developers_guide.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/src/developers_guide.md b/docs/src/developers_guide.md index 9f8991d2..c20da3e8 100644 --- a/docs/src/developers_guide.md +++ b/docs/src/developers_guide.md @@ -5,4 +5,6 @@ There is a benchmark suite defined in `GalerkinToolkitExamples/benchmarks`. This and [github-action-benchmark](https://github.com/benchmark-action/github-action-benchmark) to collect the results and store them in the `gh-pages` branch. Graphs of performance changes over time (per commit hash) can be viewed here: . +The github action can be configured (in `.github/workflows/benchmarks.yml`) to fail if the performance change is beyond a given threshold. Look for the `alert-threshold:` and `fail-on-alert:` keys. + More benchmarks can be added (or existing ones modified) in `GalerkinToolkitExamples/benchmarks/run_benchmarks.jl`. From 6ce8516a0e10aed32f5a88105dd3ba2a6cdfd38b Mon Sep 17 00:00:00 2001 From: raar1 Date: Mon, 2 Dec 2024 00:32:32 +0100 Subject: [PATCH 15/17] Remove PR trigger for benchmarking. Only run for merges to main. --- .github/workflows/benchmarks.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index cae4c2ad..c7e89499 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -3,7 +3,6 @@ on: push: branches: - main - pull_request: permissions: contents: write From 2184063fca7b236b7cdd7c2e5985161cdb616dcd Mon Sep 17 00:00:00 2001 From: raar1 Date: Mon, 2 Dec 2024 10:53:19 +0100 Subject: [PATCH 16/17] Reactivate benchmarks on PR to test deletion of benchmarks --- .github/workflows/benchmarks.yml | 1 + GalerkinToolkitExamples/benchmarks/run_benchmarks.jl | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index c7e89499..cae4c2ad 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -3,6 +3,7 @@ on: push: branches: - main + pull_request: permissions: contents: write diff --git a/GalerkinToolkitExamples/benchmarks/run_benchmarks.jl b/GalerkinToolkitExamples/benchmarks/run_benchmarks.jl index 134a84f8..373d41bf 100644 --- a/GalerkinToolkitExamples/benchmarks/run_benchmarks.jl +++ b/GalerkinToolkitExamples/benchmarks/run_benchmarks.jl @@ -28,7 +28,7 @@ end # Build a benchmark suite for the Poisson example suite = BenchmarkGroup() -suite["poisson-hand"] = BenchmarkGroup(["Poisson", "handwritten"]) +suite["poisson-hand"] = BenchmarkGroup(["Poisson", "handwritten", "3D"]) suite["poisson-hand"]["n=10"] = @benchmarkable handwritten_poisson(10) # Run all benchmarks From 28177e8b0e54d08848a42deb4326af7205f2eeb7 Mon Sep 17 00:00:00 2001 From: raar1 Date: Mon, 2 Dec 2024 11:11:25 +0100 Subject: [PATCH 17/17] Deletion of benchmarks worked. Removing testing on PR trigger --- .github/workflows/benchmarks.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index cae4c2ad..c7e89499 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -3,7 +3,6 @@ on: push: branches: - main - pull_request: permissions: contents: write