Add benchmark workflow (RPi4 / Arm Cortex-A72)

.github/actions/setup-nix/action.yml

@@ -7,17 +7,20 @@ inputs:
   script:
     description: The script to be run in the nix shell
     required: false
+  devShell:
+    description: The name of the devShell
+    required: true
 
 runs:
   using: composite
   steps:
     - uses: DeterminateSystems/nix-installer-action@v12
     - uses: DeterminateSystems/magic-nix-cache-action@main
     - name: Prepare nix dev shell
-      shell: nix develop .#ci -c bash -e {0}
+      shell: nix develop .#${{ inputs.devShell }} -c bash -e {0}
       run: |
     - name: Dependency check
-      shell: nix develop .#ci -c bash -e {0}
+      shell: nix develop .#${{ inputs.devShell }} -c bash -e {0}
       if: inputs.script != ''
       env:
         INPUT_SCRIPT: ${{ inputs.script }}

.github/workflows/bench.yml

@@ -0,0 +1,36 @@
+# SPDX-License-Identifier: Apache-2.0
+
+name: Bench
+on:
+  push:
+    branches: ["main"]
+  pull_request:
+    branches: ["main"]
+jobs:
+  bench:
+    runs-on: self-hosted-rpi4
+    if: github.repository_owner == 'pq-code-package'
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup nix
+        uses: ./.github/actions/setup-nix
+        with:
+          devShell: ci
+          script: |
+            ARCH=$(uname -m)
+            cat >> $GITHUB_STEP_SUMMARY <<-EOF
+              ## Setup
+              Architecture: $ARCH
+              - $(uname -a)
+              - $(nix --version)
+              - $(astyle --version)
+              - $(${{ matrix.cross_prefix }}gcc --version | grep -m1 "")
+              - $(bash --version | grep -m1 "")
+
+              ## CPU Info
+              $(cat /proc/cpuinfo)
+            EOF
+      - name: Run benchmark
+        shell: nix develop .#ci -c bash -e {0}
+        run: |
+          tests bench -c PMU --cpu cortex-a72 -v

.github/workflows/ci.yml

@@ -11,21 +11,20 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        include:
-        - system: macos-latest
-          expect_arch: arm64
-        - system: pqcp-arm64
-          expect_arch: aarch64
-        - system: ubuntu-latest
-          cross_prefix: aarch64-none-linux-gnu-
-          expect_arch: x86_64
+        system: [ macos-latest, pqcp-arm64, ubuntu-latest ]
+        exclude:
+          - system: ${{ github.repository_owner != 'pq-code-package' && 'pqcp-arm64' }}
     name: build_kat (${{ matrix.system }})
     runs-on: ${{ matrix.system }}
+    env:
+      CROSS_PREFIX: "${{ (matrix.system == 'ubuntu-latest' && 'aarch64-none-linux-gnu-') || ' ' }}"
+      EXPECT_ARCH: "${{ (matrix.system == 'macos-latest' && 'arm64') || (matrix.system == 'pqcp-arm64' && 'aarch64') || (matrix.system == 'ubuntu-latest' && 'x86_64') }}"
     steps:
       - uses: actions/checkout@v4
       - name: Setup nix
         uses: ./.github/actions/setup-nix
         with:
+          devShell: ci
           script: |
             ARCH=$(uname -m)
             cat >> $GITHUB_STEP_SUMMARY <<-EOF
@@ -34,12 +33,12 @@ jobs:
               - $(uname -a)
               - $(nix --version)
               - $(astyle --version)
-              - $(${{ matrix.cross_prefix }}gcc --version | grep -m1 "")
+              - $(${CROSS_PREFIX}gcc --version | grep -m1 "")
               - $(bash --version | grep -m1 "")
             EOF
 
-            if [[ "$ARCH" != ${{ matrix.expect_arch }} ]]; then
-              echo ":x: Expecting to run on ${{ matrix.expect_arch }}, but instead running on $ARCH" >> $GITHUB_STEP_SUMMARY
+            if [[ "$ARCH" != $EXPECT_ARCH ]]; then
+              echo ":x: Expecting to run on $EXPECT_ARCH, but instead running on $ARCH" >> $GITHUB_STEP_SUMMARY
               exit 1
             fi
       - name: Run functional tests
@@ -69,23 +68,26 @@ jobs:
       matrix:
         system: [ubuntu-latest]
     runs-on: ${{ matrix.system }}
+    env:
+      CROSS_PREFIX: "${{ (matrix.system == 'ubuntu-latest' && 'aarch64-none-linux-gnu-') || '' }}"
     steps:
       - uses: actions/checkout@v4
       - name: Setup nix
         uses: ./.github/actions/setup-nix
         with:
+          devShell: ci-linter
           script: |
             cat >> $GITHUB_STEP_SUMMARY << EOF
               ## Setup
               Architecture: $(uname -m)
               - $(uname -a)
               - $(nix --version)
               - $(astyle --version)
-              - $(${{ matrix.cross_prefix }}gcc --version | grep -m1 "")
+              - $(${CROSS_PREFIX}gcc --version | grep -m1 "")
               - $(bash --version | grep -m1 "")
             EOF
       - name: Lint
-        shell: nix develop .#ci -c bash -e {0}
+        shell: nix develop .#ci-linter -c bash -e {0}
         run: |
           echo "## Lint & Checks" >> $GITHUB_STEP_SUMMARY
           lint
@@ -99,6 +101,7 @@ jobs:
       - name: Setup nix
         uses: ./.github/actions/setup-nix
         with:
+          devShell: ci-cbmc
           script: |
             cat >> $GITHUB_STEP_SUMMARY << EOF
               ## Setup
@@ -111,7 +114,7 @@ jobs:
               - $(bash --version | grep -m1 "")
             EOF
       - name: Run CBMC proofs
-        shell: nix develop .#ci -c bash -e {0}
+        shell: nix develop .#ci-cbmc -c bash -e {0}
         run: |
           cd cbmc/proofs;
           KYBER_K=2 ./run-cbmc-proofs.py --summarize;

Makefile

@@ -9,23 +9,23 @@ INCLUDE_FIPS202 = -I fips202
 INCLUDE_MLKEM = -I mlkem
 INCLUDE_RANDOM = -I randombytes
 INCLUDE_NISTRANDOM = -I test/nistrng
-CFLAGS += -Wall -Wextra -Wpedantic -Werror -Wmissing-prototypes -Wredundant-decls \
+override CFLAGS += -Wall -Wextra -Wpedantic -Werror -Wmissing-prototypes -Wredundant-decls \
   -Wshadow -Wpointer-arith -Wno-unknown-pragmas -O3 -fomit-frame-pointer -pedantic \
    ${INCLUDE_MLKEM} ${INCLUDE_FIPS202}
 
 HOST_PLATFORM := $(shell uname -s)-$(shell uname -m)
 ifeq ($(HOST_PLATFORM),Linux-x86_64)
-	CFLAGS += -static
+	override CFLAGS += -static
 endif
 
 CYCLES ?= NO
 
 ifeq ($(CYCLES),PMU)
-	CFLAGS += -DPMU_CYCLES
+	override CFLAGS += -DPMU_CYCLES
 endif
 
 ifeq ($(CYCLES),PERF)
-	CFLAGS += -DPERF_CYCLES
+	override CFLAGS += -DPERF_CYCLES
 endif
 
 CFLAGS_RANDOMBYTES = ${CFLAGS} ${INCLUDE_RANDOM}

flake.nix

@@ -37,25 +37,34 @@
             };
           });
 
-          core = builtins.attrValues
+          cbmcpkg = builtins.attrValues
             {
+              cbmc = cbmc;
               litani = litani; # 1.29.0
               cbmc-viewer = cbmc-viewer; # 3.8
-              astyle = astyle;
-              cbmc = cbmc;
+            };
+
+          linters = builtins.attrValues {
+            astyle = astyle;
 
+            inherit (pkgs)
+              nixpkgs-fmt
+              shfmt;
+
+            inherit (pkgs.python3Packages)
+              black;
+          };
+
+          core = builtins.attrValues
+            {
               inherit (pkgs)
                 yq
                 ninja# 1.11.1
                 qemu# 8.2.4
-                # formatter & linters
-                cadical
-                nixpkgs-fmt
-                shfmt;
+                cadical;
 
               inherit (pkgs.python3Packages)
                 python
-                black
                 click;
             }
           ++ {
@@ -68,7 +77,7 @@
         in
         {
           devShells.default = pkgs.mkShellNoCC {
-            packages = core ++ builtins.attrValues {
+            packages = core ++ linters ++ cbmcpkg ++ builtins.attrValues {
               inherit (pkgs)
                 direnv
                 nix-direnv;
@@ -86,6 +95,20 @@
             '';
           };
 
+          devShells.ci-linter = pkgs.mkShellNoCC {
+            packages = linters;
+            shellHook = ''
+              export PATH=$PWD/scripts:$PWD/scripts/ci:$PATH
+            '';
+          };
+
+          devShells.ci-cbmc = pkgs.mkShellNoCC {
+            packages = core ++ cbmcpkg;
+            shellHook = ''
+              export PATH=$PWD/scripts:$PWD/scripts/ci:$PATH
+            '';
+          };
+
         };
       flake = {
         # The usual flake attributes can be defined here, including system-

scripts/tests

@@ -25,16 +25,16 @@ def sha256sum(result):
     return m.hexdigest()
 
 
-def base_run(bin, force_qemu, verbose, cycles="NO"):
+def base_run(bin, force_qemu, verbose, extra_make_args=[]):
     if force_qemu or (platform.system() == "Linux" and platform.machine() == "x86_64"):
         logging.debug(f"Emulating {bin} with QEMU")
 
         args = [
             "make",
             "CROSS_PREFIX=aarch64-none-linux-gnu-",
-            f"CYCLES={cycles}",
             f"{bin}",
-        ]
+        ] + extra_make_args
+
         logging.info(" ".join(args))
 
         p = subprocess.run(
@@ -54,7 +54,7 @@ def base_run(bin, force_qemu, verbose, cycles="NO"):
     else:
         logging.debug(f"Running {bin} natively")
 
-        args = ["make", f"CYCLES={cycles}", f"{bin}"]
+        args = ["make", f"{bin}"] + extra_make_args
         logging.info(" ".join(args))
 
         p = subprocess.run(
@@ -103,7 +103,13 @@ def parse_meta(scheme, field):
 
 
 def test_schemes(
-    title, scheme2file, actual_proc, expect_proc, force_qemu, verbose, cycles="NO"
+    title,
+    scheme2file,
+    actual_proc,
+    expect_proc,
+    force_qemu,
+    verbose,
+    extra_make_args=[],
 ):
     logging.info(f"{title}")
 
@@ -127,7 +133,7 @@ def test_schemes(
     results = {}
     for scheme in SCHEME:
         bin = scheme2file(scheme)
-        result = base_run(bin, force_qemu, verbose, cycles)
+        result = base_run(bin, force_qemu, verbose, extra_make_args)
         results[scheme] = result
 
         actual = actual_proc(result)
@@ -279,7 +285,12 @@ def kat(force_qemu, verbose):
     default="NO",
     help="Method for counting clock cycles. PMU requires (user-space) access to the Arm Performance Monitor Unit (PMU). PERF requires a kernel with perf support.",
 )
-def bench(force_qemu, verbose, cycles):
+@click.option(
+    "--cpu",
+    nargs=1,
+    help="Benchmark on specified CPU",
+)
+def bench(force_qemu, verbose, cycles, cpu):
     config_logger(verbose)
 
     results = test_schemes(
@@ -289,7 +300,7 @@ def bench(force_qemu, verbose, cycles):
         lambda _: True,
         force_qemu,
         verbose,
-        cycles=cycles,
+        [f"CYCLES={cycles}", f"CFLAGS=-mcpu={cpu}"],
     )
     for scheme, result in results.items():
         print(scheme)