Modify benchmark actions and workflows to allow custom compiler

.github/actions/bench/action.yml

@@ -69,12 +69,12 @@ runs:
     - name: Run benchmark
       shell: ${{ env.SHELL }}
       run: |
-        tests bench -c ${{ inputs.perf }} --cross-prefix="${{ inputs.cross_prefix }}" \
+        ./scripts/tests bench -c ${{ inputs.perf }} --cross-prefix="${{ inputs.cross_prefix }}" \
               --cflags="${{ inputs.cflags }}" --arch-flags="${{ inputs.archflags }}" \
               $([[ ${{ inputs.opt }} == "false" ]] && echo "--no-opt")  \
               -v --output=output.json ${{ inputs.bench_extra_args }}
 
-        tests bench --components -c ${{ inputs.perf }} --cross-prefix="${{ inputs.cross_prefix }}" \
+        ./scripts/tests bench --components -c ${{ inputs.perf }} --cross-prefix="${{ inputs.cross_prefix }}" \
               --cflags="${{ inputs.cflags }}" --arch-flags="${{ inputs.archflags }}" \
               $([[ ${{ inputs.opt }} == "false" ]] && echo "--no-opt")  \
               -v --output=output.json ${{ inputs.bench_extra_args }}

.github/actions/setup-shell/action.yml

@@ -4,11 +4,8 @@ name: Set Shell
 description: Setup nix or custom shell for workflows
 
 inputs:
-  use-nix:
-    description: Indicate whether to use nix or not
-    default: 'true'
   nix-shell:
-    description: Run in the specified Nix environment if exists
+    description: Run in the specified Nix environment if exists. If empty, custom shell will be used instead of nix.
     default: 'ci'
   nix-cache:
     description: Determine whether to enable nix cache
@@ -28,15 +25,15 @@ runs:
   steps:
     - name: Setup nix
       uses: ./.github/actions/setup-nix
-      if: ${{ inputs.use-nix == 'true' && (env.SHELL == '' || env.Nix_SHELL != inputs.nix-shell) }}
+      if: ${{ inputs.nix-shell != '' }}
       with:
         devShell: ${{ inputs.nix-shell }}
         verbose: ${{ inputs.nix-verbose }}
         cache: ${{ inputs.nix-cache }}
         script: ${{ inputs.script }}
     - name: Set custom shell
       shell: bash
-      if: ${{ inputs.use-nix != 'true' && env.SHELL == '' }}
+      if: ${{ inputs.nix-shell == '' }}
       run: |
           echo SHELL="${{ inputs.custom_shell }}" >> $GITHUB_ENV
 

.github/actions/setup-ubuntu/action.yml

@@ -0,0 +1,35 @@
+# SPDX-License-Identifier: Apache-2.0
+
+name: Setup ubuntu
+description: Setup ubuntu
+
+inputs:
+  packages:
+    description: Space-separated list of additional packages to install
+    required: false
+    default: ''
+
+runs:
+  using: composite
+  steps:
+    - name: Update package repository
+      shell: bash
+      run: |
+        sudo apt-get update
+    - name: Install base packages
+      shell: bash
+      run: |
+        sudo apt-get install python3-venv python3-pip make -y
+    - name: Install additional packages
+      if: ${{ inputs.packages != ''}}
+      shell: bash
+      run: |
+        sudo apt-get install ${{ inputs.packages }} -y
+    - name: Setup Python venv
+      shell: bash
+      run: |
+        python3 -m venv venv
+        source venv/bin/activate
+        python3 -m pip install -r requirements.txt
+        deactivate
+        echo "$(pwd)/venv/bin/" >> "$GITHUB_PATH"

.github/workflows/bench_ec2_any.yml

@@ -32,15 +32,14 @@ on:
         description: Run with optimized code if enabled
         type: boolean
         default: true
-      verbose:
-        description: Determine for the log verbosity
-        type: boolean
-        default: false
       bench_extra_args:
         description: Additional command line to be appended to `tests bench` script
         default: ''
-      cross_prefix:
-        description: "Binary prefix for cross-compilation builds"
+      compiler:
+        description: Compiler to use. When unset, default nix shell is used.
+        default: ''
+      additional_packages:
+        description: Additional packages to install when custom compiler is used.
         default: ''
 jobs:
   bench-ec2-any:
@@ -55,7 +54,7 @@ jobs:
       opt: ${{ inputs.opt }}
       name: ${{ inputs.name }}
       store_results: false
-      verbose: ${{ inputs.verbose }}
       bench_extra_args: ${{ inputs.bench_extra_args }}
-      cross_prefix: ${{ inputs.cross_prefix }}
+      compiler: ${{ inputs.compiler }}
+      additional_packages: ${{ inputs.additional_packages }}
     secrets: inherit

.github/workflows/bench_ec2_reusable.yml

@@ -44,9 +44,13 @@ on:
         type: string
         description: Additional command line to be appended to `bench` script
         default: ''
-      cross_prefix:
+      compiler:
         type: string
-        description: "Binary prefix for cross-compilation builds"
+        description: Compiler to use. When unset, default nix shell is used.
+        default: ''
+      additional_packages:
+        type: string
+        description: Additional packages to install when custom compiler is used.
         default: ''
 env:
   AWS_ROLE: arn:aws:iam::559050233797:role/mlkem-c-aarch64-gh-action
@@ -98,10 +102,11 @@ jobs:
           ec2-instance-type: ${{ inputs.ec2_instance_type }}
           subnet-id: subnet-07b2729e5e065962f
           security-group-id: sg-0ab2e297196c8c381
-  bench:
-    name: Bench ${{ inputs.name }}
+  bench_nix:
+    name: Bench (nix)
     runs-on: ${{ needs.start-ec2-runner.outputs.label }}
     needs: start-ec2-runner # required to start the main job when the runner is ready
+    if: ${{ inputs.compiler == '' }}
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/bench
@@ -117,15 +122,42 @@ jobs:
           store_results: ${{ inputs.store_results }}
           bench_extra_args: ${{ inputs.bench_extra_args }}
           gh_token: ${{ secrets.AWS_GITHUB_TOKEN }}
-          cross_prefix: ${{ inputs.cross_prefix }}
+  bench_custom:
+    name: Bench (custom compiler)
+    runs-on: ${{ needs.start-ec2-runner.outputs.label }}
+    needs: start-ec2-runner # required to start the main job when the runner is ready
+    if: ${{ inputs.compiler != '' }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup-ubuntu
+        with:
+          packages: ${{ inputs.additional_packages }}
+      - name: Set compiler
+        run: |
+          echo "CC=${{ inputs.compiler }}" >> "$GITHUB_ENV"
+      - uses: ./.github/actions/bench
+        with:
+          nix-shell: ''
+          custom-shell: 'bash'
+          nix-cache: false
+          nix-verbose: ${{ inputs.verbose }}
+          name: ${{ inputs.name }} (${{ inputs.compiler }})
+          cflags: ${{ inputs.cflags }}
+          archflags: ${{ inputs.archflags }}
+          opt: ${{ inputs.opt }}
+          perf: PERF
+          store_results: ${{ inputs.store_results }}
+          bench_extra_args: ${{ inputs.bench_extra_args }}
+          gh_token: ${{ secrets.AWS_GITHUB_TOKEN }}
   stop-ec2-runner:
     name: Stop ${{ inputs.name }} (${{ inputs.ec2_instance_type }})
     permissions:
       contents: 'read'
       id-token: 'write'
     needs:
       - start-ec2-runner
-      - bench # required to wait when the main job is done
+      - bench_nix # required to wait when the main job is done
+      - bench_custom # required to wait when the main job is done
     runs-on: ubuntu-latest
     if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
     steps:

mk/config.mk

@@ -33,6 +33,7 @@ CFLAGS += \
 	-Wshadow \
 	-Wpointer-arith \
 	-Wno-unknown-pragmas \
+        -Wno-unused-command-line-argument \
 	-O3 \
 	-fomit-frame-pointer \
         -std=c99 \