Push EC2 benchmark results to GH pages (#101)

* Push EC2 benchmark results to GH pages Signed-off-by: Hanno Becker <[email protected]> * Switch Gv2 benchmark to t4g.small Also, use archflags instead of cflags for -mcpu setting Signed-off-by: Hanno Becker <[email protected]> --------- Signed-off-by: Hanno Becker <[email protected]>
pq-code-package · Sep 5, 2024 · 16a4c69 · 16a4c69
1 parent bd2edd6
commit 16a4c69
Show file tree

Hide file tree

Showing 3 changed files with 45 additions and 16 deletions.
diff --git a/.github/workflows/bench_ec2_all.yml b/.github/workflows/bench_ec2_all.yml
@@ -1,4 +1,4 @@
-name: bench-ec2-all
+name: Bench EC2
 on:
   workflow_dispatch:
   push:
@@ -15,10 +15,11 @@ jobs:
     uses: ./.github/workflows/bench_ec2_reusable.yml
     if: github.repository_owner == 'pq-code-package' && (github.event.label.name == 'benchmark' || github.ref == 'refs/heads/main')
     with:
-      ec2_instance_type: c6g.medium
+      ec2_instance_type: t4g.small
       ec2_ami_id: ami-096ea6a12ea24a797
-      cflags: -mcpu=cortex-a76
-      archflags: -march=armv8.2-a
+      archflags: -mcpu=cortex-a76 -march=armv8.2-a
+      store_results: 'true'
+      name: Graviton2
     secrets: inherit
   bench-gv3:
     name: Graviton3
@@ -30,6 +31,7 @@ jobs:
     with:
       ec2_instance_type: c7g.medium
       ec2_ami_id: ami-096ea6a12ea24a797
-      cflags: -mcpu=neoverse-v1
-      archflags: -march=armv8.4-a
+      archflags: -mcpu=neoverse-v1 -march=armv8.4-a
+      store_results: 'true'
+      name: Graviton3
     secrets: inherit
diff --git a/.github/workflows/bench_ec2_any.yml b/.github/workflows/bench_ec2_any.yml
@@ -2,18 +2,24 @@ name: bench-ec2-any
 on:
   workflow_dispatch:
     inputs:
+      name:
+        description: Alternative name of instance
+        default: Graviton2
       ec2_instance_type:
         description: Type if EC2 instance to benchmark on
-        default: c6g.medium
+        default: t4g.small
       ec2_ami_id:
         description: AMI ID
         default: ami-096ea6a12ea24a797
       cflags:
         description: Custom CFLAGS for compilation
-        default: -mcpu=cortex-a76
+        default:
       archflags:
         description: Custom ARCH flags for compilation
-        default: -march=armv8.2-a
+        default: -mcpu=cortex-a76 -march=armv8.2-a
+      store_results:
+        description: Indicates if results should be pushed to github pages
+        default: false
 jobs:
   bench-ec2-any:
     name: Ad-hoc benchmark on $${{ github.event.inputs.ec2_instance_type }}
@@ -23,4 +29,6 @@ jobs:
       ec2_ami_id: ${{ github.event.inputs.ec2_ami_id }}
       cflags: ${{ github.event.inputs.cflags }}
       archflags: ${{ github.event.inputs.archflags }}
+      name: ${{ github.event.inputs.name }}
+      store_results: ${{ github.event.inputs.store_results }}
     secrets: inherit
diff --git a/.github/workflows/bench_ec2_reusable.yml b/.github/workflows/bench_ec2_reusable.yml
@@ -2,28 +2,36 @@ name: bench-ec2-reusable
 on:
   workflow_call:
     inputs:
+      name:
+        type: string
+        description: Alternative name of instance
+        default: Graviton2
       ec2_instance_type:
         type: string
         description: Type if EC2 instance to benchmark on
-        default: c6g.medium
+        default: t4g.small
       ec2_ami_id:
         type: string
         description: AMI ID
         default: ami-096ea6a12ea24a797
       cflags:
         type: string
         description: Custom CFLAGS for compilation
-        default: -mcpu=cortex-a76
+        default:
       archflags:
         type: string
         description: Custom ARCH flags for compilation
-        default: -march=armv8.2-a
+        default: -mcpu=neoverse-n1 -march=armv8.2-a
+      store_results:
+        type: string
+        description: Indicates if results should be pushed to github pages
+        default: 'false'
 env:
   AWS_ROLE: arn:aws:iam::559050233797:role/mlkem-c-aarch64-gh-action
   AWS_REGION: us-east-1
 jobs:
   start-ec2-runner:
-    name: Start ${{ github.event.inputs.ec2_instance_type }} instance
+    name: Start ${{ github.event.inputs.name }} (${{ github.event.inputs.ec2_instance_type }})
     permissions:
       contents: 'read'
       id-token: 'write'
@@ -48,7 +56,7 @@ jobs:
           subnet-id: subnet-07b2729e5e065962f
           security-group-id: sg-0ab2e297196c8c381
   bench:
-    name: Bench on ${{ inputs.ec2_instance_type }}
+    name: Bench ${{ inputs.name }}
     needs: start-ec2-runner # required to start the main job when the runner is ready
     runs-on: ${{ needs.start-ec2-runner.outputs.label }} # run the job on the newly created runner
     steps:
@@ -74,12 +82,23 @@ jobs:
       - name: Run benchmark
         shell: nix develop .#ci -c bash -e {0}
         run: |
-          tests bench -c PERF --cflags ${{ inputs.cflags }} --arch-flags ${{ inputs.archflags }} -v --output output.json
+          tests bench -c PERF --cflags "${{ inputs.cflags }}" --arch-flags "${{ inputs.archflags }}" -v --output output.json
       - name: Dump benchmark
+        if: ${{ inputs.store_results != 'true' }}
         run: |
           cat output.json
+      - name: Store benchmark result
+        if: ${{ inputs.store_results == 'true' }}
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: ${{ inputs.name }}
+          tool: 'customSmallerIsBetter'
+          output-file-path: output.json
+          github-token: ${{ secrets.AWS_GITHUB_TOKEN }}
+          auto-push: true
+
   stop-ec2-runner:
-    name: Stop ${{ inputs.ec2_instance_type }} instance
+    name: Stop ${{ github.event.inputs.name }} (${{ github.event.inputs.ec2_instance_type }})
     permissions:
       contents: 'read'
       id-token: 'write'