bench.yml

# SPDX-License-Identifier: Apache-2.0

name: Bench
on:
  workflow_dispatch:
  push:
    branches: ["main"]
  pull_request:
    branches: ["main"]
    types: [ "labeled" ]

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  bench:
    permissions:
      contents: write
      pull-requests: write
    name: ${{ matrix.target.name }}
    strategy:
      fail-fast: true
      matrix:
       target:
        - system: rpi4
          name: Arm Cortex-A72 (Raspberry Pi 4) benchmarks
          bench_pmu: PMU
          archflags: -mcpu=cortex-a72 -DSYS_AARCH64_SLOW_BARREL_SHIFTER
          cflags: "-flto -DFORCE_AARCH64"
          bench_extra_args: ""
        - system: rpi5
          name: Arm Cortex-A76 (Raspberry Pi 5) benchmarks
          bench_pmu: PERF
          archflags: "-mcpu=cortex-a76 -march=armv8.2-a"
          cflags: "-flto -DFORCE_AARCH64"
          bench_extra_args: ""
        - system: a55
          name: Arm Cortex-A55 (Snapdragon 888) benchmarks
          bench_pmu: PERF
          archflags: "-mcpu=cortex-a55 -march=armv8.2-a"
          cflags: "-flto -static -DFORCE_AARCH64 -DFIPS202_NATIVE_PROFILE=\"aarch64/profiles/cortex_a55.h\""
          bench_extra_args: -w exec-on-a55
    if: github.repository_owner == 'pq-code-package' && (github.event.label.name == 'benchmark' || github.ref == 'refs/heads/main')
    runs-on: self-hosted-${{ matrix.target.system }}
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
      - uses: ./.github/actions/bench
        with:
          name: ${{ matrix.target.name }}
          cflags: ${{ matrix.target.cflags }}
          archflags: ${{ matrix.target.archflags }}
          perf: ${{ matrix.target.bench_pmu }}
          store_results: ${{ github.repository_owner == 'pq-code-package' && github.ref == 'refs/heads/main' }}
          bench_extra_args: ${{ matrix.target.bench_extra_args }}
          gh_token: ${{ secrets.AWS_GITHUB_TOKEN }}
  ec2_all:
    name: ${{ matrix.target.name }} ${{ matrix.opt.name }}
    permissions:
      contents: write
      pull-requests: write
      id-token: write
    strategy:
      fail-fast: false
      matrix:
        opt:
          - name: opt
            value: true
          - name: no_opt
            value: false
        target:
          - name: Graviton2
            ec2_instance_type: t4g.small
            ec2_ami: ubuntu-latest (aarch64)
            archflags: -mcpu=cortex-a76 -march=armv8.2-a
            cflags: "-flto -DFORCE_AARCH64"
            perf: PERF
            alert_threshold: 102% # NOTE: set alert_threshold to default
          - name: Graviton3
            ec2_instance_type: c7g.medium
            ec2_ami: ubuntu-latest (aarch64)
            archflags: -march=armv8.4-a+sha3
            cflags: "-flto -DFORCE_AARCH64"
            perf: PERF
            alert_threshold: 102% # NOTE: set alert_threshold to default
          - name: Graviton4
            ec2_instance_type: c8g.medium
            ec2_ami: ubuntu-latest (aarch64)
            archflags: -march=armv9-a+sha3
            cflags: "-flto -DFORCE_AARCH64"
            perf: PERF
            alert_threshold: 102% # NOTE: set alert_threshold to default
          - name: AMD EPYC 4th gen (c7a)
            ec2_instance_type: c7a.medium
            ec2_ami: ubuntu-latest (x86_64)
            archflags: -mavx2 -mbmi2 -mpopcnt -maes
            cflags: "-flto -DFORCE_X86_64"
            perf: PMU
            alert_threshold: 102% # NOTE: set alert_threshold to default
          - name: Intel Xeon 4th gen (c7i)
            ec2_instance_type: c7i.large
            ec2_ami: ubuntu-latest (x86_64)
            archflags: -mavx2 -mbmi2 -mpopcnt -maes
            cflags: "-flto -DFORCE_X86_64"
            perf: PMU
            alert_threshold: 115% # NOTE: set alert_threshold to 115% for intel
          - name: AMD EPYC 3rd gen (c6a)
            ec2_instance_type: c7a.medium
            ec2_ami: ubuntu-latest (x86_64)
            archflags: -mavx2 -mbmi2 -mpopcnt -maes
            cflags: "-flto -DFORCE_X86_64"
            perf: PMU
            alert_threshold: 102% # NOTE: set alert_threshold to default
          - name: Intel Xeon 3rd gen (c6i)
            ec2_instance_type: c7i.large
            ec2_ami: ubuntu-latest (x86_64)
            archflags: -mavx2 -mbmi2 -mpopcnt -maes
            cflags: "-flto -DFORCE_X86_64"
            perf: PMU
            alert_threshold: 115% # NOTE: set alert_threshold to 115% for intel
    uses: ./.github/workflows/bench_ec2_reusable.yml
    if: github.repository_owner == 'pq-code-package' && (github.event.label.name == 'benchmark' || github.ref == 'refs/heads/main')
    with:
      ec2_instance_type: ${{ matrix.target.ec2_instance_type }}
      ec2_ami: ${{ matrix.target.ec2_ami }}
      archflags: ${{ matrix.target.archflags }}
      cflags: ${{ matrix.target.cflags }}
      opt: ${{ matrix.opt.value }}
      store_results: ${{ github.repository_owner == 'pq-code-package' && github.ref == 'refs/heads/main' }} # Only store optimized results
      name: "${{ matrix.target.name }}${{ (!matrix.opt.value && ' (no-opt)') || ''}}"
      perf: ${{ matrix.target.perf }}
      alert_threshold: ${{ matrix.target.alert_threshold }}
    secrets: inherit