From 2887989ca858eafb0f87b4d2e163c28c4209a357 Mon Sep 17 00:00:00 2001 From: Joris Dral Date: Wed, 4 Sep 2024 16:25:11 +0200 Subject: [PATCH 1/2] WIP: run mempool-bench in a separate job --- .github/workflows/ci.yml | 90 ++++++++++++++++++++++++++-------------- 1 file changed, 59 insertions(+), 31 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 737295f8e8..ba799e11f0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -146,10 +146,67 @@ jobs: if: matrix.test-set == 'no-thunks-safe' run: cabal test ouroboros-consensus:consensus-test ouroboros-consensus:doctest ouroboros-consensus:infra-test ouroboros-consensus:storage-test ouroboros-consensus-cardano:byron-test ouroboros-consensus-cardano:shelley-test ouroboros-consensus-diffusion:infra-test ouroboros-consensus-protocol:protocol-test -j --test-show-details=streaming + - name: Identify benchmark executables + run: | + cp $(cabal list-bin mempool-bench) mempool-bench + + - name: Upload benchmark executables + uses: actions/upload-artifact@v4 + with: + name: benchmark-exes-${{ runner.os }}-${{ matrix.ghc }} + path: mempool-bench + retention-days: 1 + + # NB: build the haddocks at the end to avoid unecessary recompilations. + # We build the haddocks only for one GHC version. + - name: Build Haddock documentation + if: | + github.event_name == 'push' + && github.ref == 'refs/heads/main' + && matrix.ghc=='9.6.6' + run: | + # need for latex, dvisvgm and standalone + sudo apt install texlive-latex-extra texlive-latex-base + # cabal-docspec doesn't work with XDG https://github.com/phadej/cabal-extras/issues/136 + sed -i 's_-- store-dir:_store-dir: /home/runner/.local/state/cabal/store_g' ~/.config/cabal/config + export CABAL_CONFIG=~/.config/cabal/config + + ./scripts/docs/haddocks.sh + tar vzcf haddocks.tgz ./docs/website/static/haddocks + + - name: Upload haddocks as an artifact + if: | + github.event_name == 'push' + && github.ref == 'refs/heads/main' + && matrix.ghc=='9.6.6' + uses: actions/upload-artifact@v4 + with: + name: haddocks + path: haddocks.tgz + retention-days: 1 + + benchmarks: + name: Run benchmarks + needs: build-test-bench-haddocks + + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + ghc: ["8.10.7", "9.6.6", "9.10.1"] + + steps: + - uses: actions/checkout@v4 + + - name: Download benchmark executables + uses: actions/download-artifact@v4 + with: + name: benchmark-exes-${{ runner.os }}-${{ matrix.ghc }} + - name: Create baseline-benchmark if: github.event_name == 'push' && github.ref == 'refs/heads/main' run: | - cabal new-run ouroboros-consensus:mempool-bench -- \ + ./mempool-bench \ --timeout=60 --csv mempool-benchmarks.csv \ +RTS -T @@ -163,7 +220,6 @@ jobs: # then we will save the same results we just restored. - name: Cache benchmark baseline results uses: actions/cache@v4 - if: matrix.variant == 'default' with: path: baseline-mempool-benchmarks.csv key: baseline-mempool-benchmarks-${{ runner.os }}-${{ matrix.ghc }}-${{ github.run_id }} @@ -188,7 +244,7 @@ jobs: if: ${{ github.event_name == 'pull_request' }} run: | if [ -f baseline-mempool-benchmarks.csv ]; then - cabal new-run ouroboros-consensus:mempool-bench -- \ + ./mempool-bench \ --timeout=60 --baseline baseline-mempool-benchmarks.csv \ --fail-if-slower 100 \ +RTS -T @@ -197,34 +253,6 @@ jobs: echo "Benchmarks comparison skipped." fi - # NB: build the haddocks at the end to avoid unecessary recompilations. - # We build the haddocks only for one GHC version. - - name: Build Haddock documentation - if: | - github.event_name == 'push' - && github.ref == 'refs/heads/main' - && matrix.ghc=='9.6.6' - run: | - # need for latex, dvisvgm and standalone - sudo apt install texlive-latex-extra texlive-latex-base - # cabal-docspec doesn't work with XDG https://github.com/phadej/cabal-extras/issues/136 - sed -i 's_-- store-dir:_store-dir: /home/runner/.local/state/cabal/store_g' ~/.config/cabal/config - export CABAL_CONFIG=~/.config/cabal/config - - ./scripts/docs/haddocks.sh - tar vzcf haddocks.tgz ./docs/website/static/haddocks - - - name: Upload haddocks as an artifact - if: | - github.event_name == 'push' - && github.ref == 'refs/heads/main' - && matrix.ghc=='9.6.6' - uses: actions/upload-artifact@v4 - with: - name: haddocks - path: haddocks.tgz - retention-days: 1 - deploy-documentation: name: Deploy documentation to GitHub Pages if: github.event_name == 'push' && github.ref == 'refs/heads/main' From 4b32aa06d005c15b2cb8f659fb04aa719697ecde Mon Sep 17 00:00:00 2001 From: Joris Dral Date: Wed, 4 Sep 2024 17:55:00 +0200 Subject: [PATCH 2/2] WIP: accept benchmark result based on a PR label --- .github/workflows/ci.yml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ba799e11f0..74b6ea9982 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -195,6 +195,10 @@ jobs: matrix: ghc: ["8.10.7", "9.6.6", "9.10.1"] + env: + ACCEPT_BENCHMARKS: ${{ contains(github.event.pull_request.labels.*.name, 'acceptable benchmarks') }} + BENCH_OPTIONS: "" + steps: - uses: actions/checkout@v4 @@ -230,6 +234,13 @@ jobs: if: github.event_name == 'push' && github.ref == 'refs/heads/main' run: cp mempool-benchmarks.csv baseline-mempool-benchmarks.csv + # The `fail-if-slower` value is determined ad-hoc based on the variability + # we observed in our benchmarks. + - name: Set benchmark options (accept) + if: ${{ !env.ACCEPT_BENCHMARKS }} + run: | + echo "BENCH_OPTIONS='--fail-if-slower 100'" >> "GITHUB_ENV" + # TODO: this will be necessary when we publish the benchmarks results. # - name: Upload mempool benchmark baseline results # if: github.event_name == 'push' && github.ref == 'refs/heads/main' @@ -238,15 +249,13 @@ jobs: # name: baseline-mempool-benchmarks # path: baseline-mempool-benchmarks.csv - # The `fail-if-slower` value is determined ad-hoc based on the variability - # we observed in our benchmarks. - name: Run mempool benchmarks on pull requests if: ${{ github.event_name == 'pull_request' }} run: | if [ -f baseline-mempool-benchmarks.csv ]; then ./mempool-bench \ --timeout=60 --baseline baseline-mempool-benchmarks.csv \ - --fail-if-slower 100 \ + ${{ env.BENCH_OPTIONS }} \ +RTS -T else echo "No baseline benchmarks found. This likely happened when adding a new GHC version to the build matrix."