benchmark.yaml

name: Benchmark Performance
on:
  push:
    branches: [main]
  pull_request:
    branches: [main, "release/*", "dev"]

jobs:
  benchmark:
    permissions: write-all
    name: Run benchmark
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: 3.11

      - name: Install packages
        run: |
          pip install -e .[tests,benchmarks]
          pip install sample_dataset_builder/

      - name: Run benchmark
        run: |
          pytest benchmark/run.py

      - name: Store benchmark result
        uses: benchmark-action/github-action-benchmark@v1
        with:
          name: Benchmark
          tool: "customSmallerIsBetter"
          output-file-path: benchmark/outputs/output_256_512_5.json
          # Use personal access token instead of GITHUB_TOKEN due to https://github.community/t/github-action-not-triggering-gh-pages-upon-push/16096
          github-token: ${{ secrets.GITHUB_TOKEN }}
          auto-push: true
          # Show alert with commit comment on detecting possible performance regression
          alert-threshold: "150%"
          comment-on-alert: true
          fail-on-alert: false