benchmark-testing.yaml

name: "Benchmark testing"

on:
  workflow_dispatch:
  pull_request:

permissions:
  contents: read

jobs:

  Benchmark-Test:
    name: "Benchmark tests"
    runs-on: ubuntu-20.04
    # note: we want benchmarks to run on pull_request events in order to publish results to a sticky comment, and
    # we also want to run on push such that merges to main are recorded to the cache. For this reason we don't filter
    # the job by event.
    steps:
      - name: Checkout code
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

      - name: Bootstrap environment
        uses: ./.github/actions/bootstrap

      - name: Restore base benchmark result
        uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a #v4.1.2
        with:
          path: test/results/benchmark-main.txt
          # use base sha for PR or new commit hash for main push in benchmark result key
          key: ${{ runner.os }}-bench-${{ (github.event.pull_request.base.sha != github.event.after) && github.event.pull_request.base.sha || github.event.after }}

      - name: Run benchmark tests
        id: benchmark
        run: |
          REF_NAME=${GITHUB_REF##*/} make benchmark
          OUTPUT=$(make show-benchstat)
          OUTPUT="${OUTPUT//'%'/'%25'}"   # URL encode all '%' characters
          OUTPUT="${OUTPUT//$'\n'/'%0A'}" # URL encode all '\n' characters
          OUTPUT="${OUTPUT//$'\r'/'%0D'}" # URL encode all '\r' characters
          echo "result=$OUTPUT" >> $GITHUB_OUTPUT

      - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
        with:
          name: benchmark-test-results
          path: test/results/**/*

      - name: Update PR benchmark results comment
        uses: marocchino/sticky-pull-request-comment@331f8f5b4215f0445d3c07b4967662a32a2d3e31 #v2.9.0
        continue-on-error: true
        with:
          header: benchmark
          message: |
            ### Benchmark Test Results

            <details>
              <summary>Benchmark results from the latest changes vs base branch</summary>

            ```
            ${{ steps.benchmark.outputs.result }}
            ```

            </details>