ci: make alert threshold configurable

Set default alert threshold to 102%, and 115% for intel machines for now, as the current benchmark result for intels are not yet stabled Signed-off-by: Thing-han, Lim <[email protected]>
pq-code-package · Nov 21, 2024 · d91221a · d91221a
1 parent 400ee2b
commit d91221a
Show file tree

Hide file tree

Showing 3 changed files with 18 additions and 1 deletion.
diff --git a/.github/actions/bench/action.yml b/.github/actions/bench/action.yml
@@ -44,6 +44,9 @@ inputs:
   cross_prefix:
     description: "Binary prefix for cross-compilation builds"
     default: ""
+  alert_threshold:
+    description: "Set alert threshold in percentage for benchmark result"
+    default: "102%"
 runs:
   using: composite
   steps:
@@ -93,4 +96,4 @@ runs:
         auto-push: ${{ inputs.store_results == 'true' }}
         comment-on-alert: true
         summary-always: true
-        alert-threshold: 103%
+        alert-threshold: ${{ inputs.alert_threshold }}
diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
@@ -75,42 +75,49 @@ jobs:
             archflags: -mcpu=cortex-a76 -march=armv8.2-a
             cflags: "-flto -DFORCE_AARCH64"
             perf: PERF
+            alert_threshold: 102% # NOTE: set alert_threshold to default
           - name: Graviton3
             ec2_instance_type: c7g.medium
             ec2_ami: ubuntu-latest (aarch64)
             archflags: -march=armv8.4-a+sha3
             cflags: "-flto -DFORCE_AARCH64"
             perf: PERF
+            alert_threshold: 102% # NOTE: set alert_threshold to default
           - name: Graviton4
             ec2_instance_type: c8g.medium
             ec2_ami: ubuntu-latest (aarch64)
             archflags: -march=armv9-a+sha3
             cflags: "-flto -DFORCE_AARCH64"
             perf: PERF
+            alert_threshold: 102% # NOTE: set alert_threshold to default
           - name: AMD EPYC 4th gen (c7a)
             ec2_instance_type: c7a.medium
             ec2_ami: ubuntu-latest (x86_64)
             archflags: -mavx2 -mbmi2 -mpopcnt -maes
             cflags: "-flto -DFORCE_X86_64"
             perf: PMU
+            alert_threshold: 102% # NOTE: set alert_threshold to default
           - name: Intel Xeon 4th gen (c7i)
             ec2_instance_type: c7i.large
             ec2_ami: ubuntu-latest (x86_64)
             archflags: -mavx2 -mbmi2 -mpopcnt -maes
             cflags: "-flto -DFORCE_X86_64"
             perf: PMU
+            alert_threshold: 115% # NOTE: set alert_threshold to 115% for intel
           - name: AMD EPYC 3rd gen (c6a)
             ec2_instance_type: c6a.large
             ec2_ami: ubuntu-latest (x86_64)
             archflags: -mavx2 -mbmi2 -mpopcnt -maes
             cflags: "-flto -DFORCE_X86_64"
             perf: PMU
+            alert_threshold: 102% # NOTE: set alert_threshold to default
           - name: Intel Xeon 3rd gen (c6i)
             ec2_instance_type: c6i.large
             ec2_ami: ubuntu-latest (x86_64)
             archflags: -mavx2 -mbmi2 -mpopcnt -maes
             cflags: "-flto -DFORCE_X86_64"
             perf: PMU
+            alert_threshold: 115% # NOTE: set alert_threshold to 115% for intel
     uses: ./.github/workflows/bench_ec2_reusable.yml
     if: github.repository_owner == 'pq-code-package' && (github.event.label.name == 'benchmark' || github.ref == 'refs/heads/main')
     with:
@@ -122,4 +129,5 @@ jobs:
       store_results: ${{ github.repository_owner == 'pq-code-package' && github.ref == 'refs/heads/main' }} # Only store optimized results
       name: "${{ matrix.target.name }}${{ (!matrix.opt.value && ' (no-opt)') || ''}}"
       perf: ${{ matrix.target.perf }}
+      alert_threshold: ${{ matrix.target.alert_threshold }}
     secrets: inherit
diff --git a/.github/workflows/bench_ec2_reusable.yml b/.github/workflows/bench_ec2_reusable.yml
@@ -58,6 +58,10 @@ on:
         type: string
         description: Additional packages to install when custom compiler is used.
         default: ''
+      alert_threshold:
+        type: string
+        description: "Set alert threshold in percentage for benchmark result"
+        default: "102%"
 env:
   AWS_ROLE: arn:aws:iam::559050233797:role/mlkem-c-aarch64-gh-action
   AWS_REGION: us-east-1
@@ -129,6 +133,7 @@ jobs:
           store_results: ${{ inputs.store_results }}
           bench_extra_args: ${{ inputs.bench_extra_args }}
           gh_token: ${{ secrets.AWS_GITHUB_TOKEN }}
+          alert_threshold: ${{ inputs.alert_threshold }}
   bench_custom:
     name: Bench (custom compiler)
     runs-on: ${{ needs.start-ec2-runner.outputs.label }}
@@ -156,6 +161,7 @@ jobs:
           store_results: ${{ inputs.store_results }}
           bench_extra_args: ${{ inputs.bench_extra_args }}
           gh_token: ${{ secrets.AWS_GITHUB_TOKEN }}
+          alert_threshold: ${{ inputs.alert_threshold }}
   stop-ec2-runner:
     name: Stop ${{ inputs.name }} (${{ inputs.ec2_instance_type }})
     permissions: