From 1aad15000b548a5ee66c7120b064fb4a902ca95f Mon Sep 17 00:00:00 2001
From: "Thing-han, Lim" <15379156+potsrevennil@users.noreply.github.com>
Date: Wed, 19 Jun 2024 13:32:55 +0800
Subject: [PATCH 01/16] parametrized nix shell name

Signed-off-by: Thing-han, Lim <15379156+potsrevennil@users.noreply.github.com>
---
 .github/actions/setup-nix/action.yml | 7 +++++--
 .github/workflows/ci.yml             | 3 +++
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/.github/actions/setup-nix/action.yml b/.github/actions/setup-nix/action.yml
index 83de271cc..b9c41b431 100644
--- a/.github/actions/setup-nix/action.yml
+++ b/.github/actions/setup-nix/action.yml
@@ -7,6 +7,9 @@ inputs:
   script:
     description: The script to be run in the nix shell
     required: false
+  devShell:
+    description: The name of the devShell
+    required: true
 
 runs:
   using: composite
@@ -14,10 +17,10 @@ runs:
     - uses: DeterminateSystems/nix-installer-action@v12
     - uses: DeterminateSystems/magic-nix-cache-action@main
     - name: Prepare nix dev shell
-      shell: nix develop .#ci -c bash -e {0}
+      shell: nix develop .#${{ inputs.devShell }} -c bash -e {0}
       run: |
     - name: Dependency check
-      shell: nix develop .#ci -c bash -e {0}
+      shell: nix develop .#${{ inputs.devShell }} -c bash -e {0}
       if: inputs.script != ''
       env:
         INPUT_SCRIPT: ${{ inputs.script }}
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 8f9ef7c34..b8b843581 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -26,6 +26,7 @@ jobs:
       - name: Setup nix
         uses: ./.github/actions/setup-nix
         with:
+          devShell: ci
           script: |
             ARCH=$(uname -m)
             cat >> $GITHUB_STEP_SUMMARY <<-EOF
@@ -74,6 +75,7 @@ jobs:
       - name: Setup nix
         uses: ./.github/actions/setup-nix
         with:
+          devShell: ci
           script: |
             cat >> $GITHUB_STEP_SUMMARY << EOF
               ## Setup
@@ -99,6 +101,7 @@ jobs:
       - name: Setup nix
         uses: ./.github/actions/setup-nix
         with:
+          devShell: ci
           script: |
             cat >> $GITHUB_STEP_SUMMARY << EOF
               ## Setup

From 3cac8e83aa8f383507df3bb59cd3c59aaf584e8e Mon Sep 17 00:00:00 2001
From: "Thing-han, Lim" <15379156+potsrevennil@users.noreply.github.com>
Date: Wed, 19 Jun 2024 13:33:27 +0800
Subject: [PATCH 02/16] use environment variables instead of using matrix for
 some configuration

Signed-off-by: Thing-han, Lim <15379156+potsrevennil@users.noreply.github.com>
---
 .github/workflows/ci.yml | 22 ++++++++++------------
 1 file changed, 10 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index b8b843581..85396762d 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -11,16 +11,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        include:
-        - system: macos-latest
-          expect_arch: arm64
-        - system: pqcp-arm64
-          expect_arch: aarch64
-        - system: ubuntu-latest
-          cross_prefix: aarch64-none-linux-gnu-
-          expect_arch: x86_64
+        system: [ macos-latest, pqcp-arm64, ubuntu-latest ]
     name: build_kat (${{ matrix.system }})
     runs-on: ${{ matrix.system }}
+    env:
+      CROSS_PREFIX: "${{ (matrix.system == 'ubuntu-latest' && 'aarch64-none-linux-gnu-') || '' }}"
+      EXPECT_ARCH: "${{ (matrix.system == 'macos-latest' && 'arm64') || (matrix.system == 'pqcp-arm64' && 'aarch64') || (matrix.system == 'ubuntu-latest' && 'x86_64') }}"
     steps:
       - uses: actions/checkout@v4
       - name: Setup nix
@@ -35,12 +31,12 @@ jobs:
               - $(uname -a)
               - $(nix --version)
               - $(astyle --version)
-              - $(${{ matrix.cross_prefix }}gcc --version | grep -m1 "")
+              - $(${CROSS_PREFIX}gcc --version | grep -m1 "")
               - $(bash --version | grep -m1 "")
             EOF
 
-            if [[ "$ARCH" != ${{ matrix.expect_arch }} ]]; then
-              echo ":x: Expecting to run on ${{ matrix.expect_arch }}, but instead running on $ARCH" >> $GITHUB_STEP_SUMMARY
+            if [[ "$ARCH" != $EXPECT_ARCH ]]; then
+              echo ":x: Expecting to run on $EXPECT_ARCH, but instead running on $ARCH" >> $GITHUB_STEP_SUMMARY
               exit 1
             fi
       - name: Run functional tests
@@ -70,6 +66,8 @@ jobs:
       matrix:
         system: [ubuntu-latest]
     runs-on: ${{ matrix.system }}
+    env:
+      CROSS_PREFIX: "${{ (matrix.system == 'ubuntu-latest' && 'aarch64-none-linux-gnu-') || '' }}"
     steps:
       - uses: actions/checkout@v4
       - name: Setup nix
@@ -83,7 +81,7 @@ jobs:
               - $(uname -a)
               - $(nix --version)
               - $(astyle --version)
-              - $(${{ matrix.cross_prefix }}gcc --version | grep -m1 "")
+              - $(${CROSS_PREFIX}gcc --version | grep -m1 "")
               - $(bash --version | grep -m1 "")
             EOF
       - name: Lint

From 511c3100ee6731f2a5a1cd10f94743224a1e822f Mon Sep 17 00:00:00 2001
From: "Thing-han, Lim" <15379156+potsrevennil@users.noreply.github.com>
Date: Wed, 19 Jun 2024 14:31:30 +0800
Subject: [PATCH 03/16] conditionally exclude pqcp-arm64

Signed-off-by: Thing-han, Lim <15379156+potsrevennil@users.noreply.github.com>
---
 .github/workflows/ci.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 85396762d..901a4ec17 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -12,10 +12,13 @@ jobs:
       fail-fast: false
       matrix:
         system: [ macos-latest, pqcp-arm64, ubuntu-latest ]
+        exclude:
+          - system: ${{ github.event_name != 'pull_request' && 'pqcp-arm64' }}
+          - system: ${{ (github.ref != 'refs/heads/main' || github.repository_owner != 'pq-code-package') && 'pqcp-arm64' }}
     name: build_kat (${{ matrix.system }})
     runs-on: ${{ matrix.system }}
     env:
-      CROSS_PREFIX: "${{ (matrix.system == 'ubuntu-latest' && 'aarch64-none-linux-gnu-') || '' }}"
+      CROSS_PREFIX: "${{ (matrix.system == 'ubuntu-latest' && 'aarch64-none-linux-gnu-') || ' ' }}"
       EXPECT_ARCH: "${{ (matrix.system == 'macos-latest' && 'arm64') || (matrix.system == 'pqcp-arm64' && 'aarch64') || (matrix.system == 'ubuntu-latest' && 'x86_64') }}"
     steps:
       - uses: actions/checkout@v4

From fd22f4606536cf0f2277c47870b1339f2c5ef948 Mon Sep 17 00:00:00 2001
From: "Thing-han, Lim" <15379156+potsrevennil@users.noreply.github.com>
Date: Wed, 19 Jun 2024 15:01:52 +0800
Subject: [PATCH 04/16] separate cbmc dependency for speeding up ci

Signed-off-by: Thing-han, Lim <15379156+potsrevennil@users.noreply.github.com>
---
 .github/workflows/ci.yml |  4 ++--
 flake.nix                | 17 ++++++++++++++---
 2 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 901a4ec17..4d548ac12 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -102,7 +102,7 @@ jobs:
       - name: Setup nix
         uses: ./.github/actions/setup-nix
         with:
-          devShell: ci
+          devShell: ci-cbmc
           script: |
             cat >> $GITHUB_STEP_SUMMARY << EOF
               ## Setup
@@ -115,7 +115,7 @@ jobs:
               - $(bash --version | grep -m1 "")
             EOF
       - name: Run CBMC proofs
-        shell: nix develop .#ci -c bash -e {0}
+        shell: nix develop .#ci-cbmc -c bash -e {0}
         run: |
           cd cbmc/proofs;
           KYBER_K=2 ./run-cbmc-proofs.py --summarize;
diff --git a/flake.nix b/flake.nix
index 93143fba2..72b9ce20a 100644
--- a/flake.nix
+++ b/flake.nix
@@ -37,12 +37,16 @@
             };
           });
 
-          core = builtins.attrValues
+          cbmcpkg = builtins.attrValues
             {
+              cbmc = cbmc;
               litani = litani; # 1.29.0
               cbmc-viewer = cbmc-viewer; # 3.8
+            };
+
+          core = builtins.attrValues
+            {
               astyle = astyle;
-              cbmc = cbmc;
 
               inherit (pkgs)
                 yq
@@ -68,7 +72,7 @@
         in
         {
           devShells.default = pkgs.mkShellNoCC {
-            packages = core ++ builtins.attrValues {
+            packages = core ++ cbmcpkg ++ builtins.attrValues {
               inherit (pkgs)
                 direnv
                 nix-direnv;
@@ -86,6 +90,13 @@
             '';
           };
 
+          devShells.ci-cbmc = pkgs.mkShellNoCC {
+            packages = core ++ cbmcpkg;
+            shellHook = ''
+              export PATH=$PWD/scripts:$PWD/scripts/ci:$PATH
+            '';
+          };
+
         };
       flake = {
         # The usual flake attributes can be defined here, including system-

From e1c671f177b74ebae16006409d9a39bcf6524680 Mon Sep 17 00:00:00 2001
From: "Thing-han, Lim" <15379156+potsrevennil@users.noreply.github.com>
Date: Wed, 19 Jun 2024 15:12:02 +0800
Subject: [PATCH 05/16] separate lint dependency for speeding up ci

Signed-off-by: Thing-han, Lim <15379156+potsrevennil@users.noreply.github.com>
---
 .github/workflows/ci.yml |  4 ++--
 flake.nix                | 28 ++++++++++++++++++++--------
 2 files changed, 22 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 4d548ac12..6465b0355 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -76,7 +76,7 @@ jobs:
       - name: Setup nix
         uses: ./.github/actions/setup-nix
         with:
-          devShell: ci
+          devShell: ci-linter
           script: |
             cat >> $GITHUB_STEP_SUMMARY << EOF
               ## Setup
@@ -88,7 +88,7 @@ jobs:
               - $(bash --version | grep -m1 "")
             EOF
       - name: Lint
-        shell: nix develop .#ci -c bash -e {0}
+        shell: nix develop .#ci-linter -c bash -e {0}
         run: |
           echo "## Lint & Checks" >> $GITHUB_STEP_SUMMARY
           lint
diff --git a/flake.nix b/flake.nix
index 72b9ce20a..7d55db95a 100644
--- a/flake.nix
+++ b/flake.nix
@@ -44,22 +44,27 @@
               cbmc-viewer = cbmc-viewer; # 3.8
             };
 
+          linters = builtins.attrValues {
+            astyle = astyle;
+
+            inherit (pkgs)
+              nixpkgs-fmt
+              shfmt;
+
+            inherit (pkgs.python3Packages)
+              black;
+          };
+
           core = builtins.attrValues
             {
-              astyle = astyle;
-
               inherit (pkgs)
                 yq
                 ninja# 1.11.1
                 qemu# 8.2.4
-                # formatter & linters
-                cadical
-                nixpkgs-fmt
-                shfmt;
+                cadical;
 
               inherit (pkgs.python3Packages)
                 python
-                black
                 click;
             }
           ++ {
@@ -72,7 +77,7 @@
         in
         {
           devShells.default = pkgs.mkShellNoCC {
-            packages = core ++ cbmcpkg ++ builtins.attrValues {
+            packages = core ++ linters ++ cbmcpkg ++ builtins.attrValues {
               inherit (pkgs)
                 direnv
                 nix-direnv;
@@ -90,6 +95,13 @@
             '';
           };
 
+          devShells.ci-linter = pkgs.mkShellNoCC {
+            packages = linters;
+            shellHook = ''
+              export PATH=$PWD/scripts:$PWD/scripts/ci:$PATH
+            '';
+          };
+
           devShells.ci-cbmc = pkgs.mkShellNoCC {
             packages = core ++ cbmcpkg;
             shellHook = ''

From 1a53dd471777ee083fa7ef8e07c8c729f3564efd Mon Sep 17 00:00:00 2001
From: "Thing-han, Lim" <15379156+potsrevennil@users.noreply.github.com>
Date: Wed, 19 Jun 2024 15:31:06 +0800
Subject: [PATCH 06/16] benchmark workflow

Signed-off-by: Thing-han, Lim <15379156+potsrevennil@users.noreply.github.com>
---
 .github/workflows/bench.yml | 51 +++++++++++++++++++++++++++++++++++++
 1 file changed, 51 insertions(+)
 create mode 100644 .github/workflows/bench.yml

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
new file mode 100644
index 000000000..e455f0f73
--- /dev/null
+++ b/.github/workflows/bench.yml
@@ -0,0 +1,51 @@
+# SPDX-License-Identifier: Apache-2.0
+
+name: Bench
+on:
+  push:
+    branches: ["main"]
+  pull_request:
+    branches: ["main"]
+jobs:
+  bench:
+    runs-on: self-hosted-rpi4
+    if: github.repository_owner == 'pq-code-package'
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup nix
+        uses: ./.github/actions/setup-nix
+        with:
+          devShell: ci
+          script: |
+            ARCH=$(uname -m)
+            cat >> $GITHUB_STEP_SUMMARY <<-EOF
+              ## Setup
+              Architecture: $ARCH
+              - $(uname -a)
+              - $(nix --version)
+              - $(astyle --version)
+              - $(${{ matrix.cross_prefix }}gcc --version | grep -m1 "")
+              - $(bash --version | grep -m1 "")
+            EOF
+      - name: Run functional tests
+        id: func_test
+        shell: nix develop .#ci -c bash -e {0}
+        run: |
+          tests func -v
+      - name: Run KAT tests
+        id: kat_test
+        if: |
+          success()
+          || steps.func_test.conclusion == 'failure'
+        shell: nix develop .#ci -c bash -e {0}
+        run: |
+          tests kat -v
+      - name: Run Nistkat tests
+        id: nistkat_test
+        if: |
+          success()
+          || steps.func_test.conclusion == 'failure'
+          || steps.kat_test.conclusion == 'failure'
+        shell: nix develop .#ci -c bash -e {0}
+        run: |
+          tests nistkat -v

From 83cdb8e716dbc610de636ee9dbf9d3b5afa76f86 Mon Sep 17 00:00:00 2001
From: "Thing-han, Lim" <15379156+potsrevennil@users.noreply.github.com>
Date: Wed, 19 Jun 2024 16:00:26 +0800
Subject: [PATCH 07/16] run benchmark in bench workflow

Signed-off-by: Thing-han, Lim <15379156+potsrevennil@users.noreply.github.com>
---
 .github/workflows/bench.yml | 22 ++--------------------
 .github/workflows/ci.yml    |  3 +--
 2 files changed, 3 insertions(+), 22 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index e455f0f73..8b41bb188 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -27,25 +27,7 @@ jobs:
               - $(${{ matrix.cross_prefix }}gcc --version | grep -m1 "")
               - $(bash --version | grep -m1 "")
             EOF
-      - name: Run functional tests
-        id: func_test
+      - name: Run benchmark
         shell: nix develop .#ci -c bash -e {0}
         run: |
-          tests func -v
-      - name: Run KAT tests
-        id: kat_test
-        if: |
-          success()
-          || steps.func_test.conclusion == 'failure'
-        shell: nix develop .#ci -c bash -e {0}
-        run: |
-          tests kat -v
-      - name: Run Nistkat tests
-        id: nistkat_test
-        if: |
-          success()
-          || steps.func_test.conclusion == 'failure'
-          || steps.kat_test.conclusion == 'failure'
-        shell: nix develop .#ci -c bash -e {0}
-        run: |
-          tests nistkat -v
+          tests bench -c PMU -v
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 6465b0355..1d9bbc756 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -13,8 +13,7 @@ jobs:
       matrix:
         system: [ macos-latest, pqcp-arm64, ubuntu-latest ]
         exclude:
-          - system: ${{ github.event_name != 'pull_request' && 'pqcp-arm64' }}
-          - system: ${{ (github.ref != 'refs/heads/main' || github.repository_owner != 'pq-code-package') && 'pqcp-arm64' }}
+          - system: ${{ github.repository_owner != 'pq-code-package' && 'pqcp-arm64' }}
     name: build_kat (${{ matrix.system }})
     runs-on: ${{ matrix.system }}
     env:

From 6f6e3e8e94bc533868d37bda5459f8ab6fe29a12 Mon Sep 17 00:00:00 2001
From: "Thing-han, Lim" <15379156+potsrevennil@users.noreply.github.com>
Date: Wed, 19 Jun 2024 16:51:41 +0800
Subject: [PATCH 08/16] dump cpu info

Signed-off-by: Thing-han, Lim <15379156+potsrevennil@users.noreply.github.com>
---
 .github/workflows/bench.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 8b41bb188..cf42b6f2f 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -26,6 +26,9 @@ jobs:
               - $(astyle --version)
               - $(${{ matrix.cross_prefix }}gcc --version | grep -m1 "")
               - $(bash --version | grep -m1 "")
+
+              ## CPU Info
+              $(cat /proc/cpuinfo)
             EOF
       - name: Run benchmark
         shell: nix develop .#ci -c bash -e {0}

From fe53db2440a5de29f8f4e61540a00d8ff017227d Mon Sep 17 00:00:00 2001
From: "Thing-han, Lim" <15379156+potsrevennil@users.noreply.github.com>
Date: Wed, 19 Jun 2024 17:14:10 +0800
Subject: [PATCH 09/16] refactor tests script for arbitrary make args

Signed-off-by: Thing-han, Lim <15379156+potsrevennil@users.noreply.github.com>
---
 scripts/tests | 27 +++++++++++++++++++--------
 1 file changed, 19 insertions(+), 8 deletions(-)

diff --git a/scripts/tests b/scripts/tests
index 88eab2bd0..8bbc0d91d 100755
--- a/scripts/tests
+++ b/scripts/tests
@@ -25,16 +25,16 @@ def sha256sum(result):
     return m.hexdigest()
 
 
-def base_run(bin, force_qemu, verbose, cycles="NO"):
+def base_run(bin, force_qemu, verbose, extra_make_args=[]):
     if force_qemu or (platform.system() == "Linux" and platform.machine() == "x86_64"):
         logging.debug(f"Emulating {bin} with QEMU")
 
         args = [
             "make",
             "CROSS_PREFIX=aarch64-none-linux-gnu-",
-            f"CYCLES={cycles}",
             f"{bin}",
-        ]
+        ] + extra_make_args
+
         logging.info(" ".join(args))
 
         p = subprocess.run(
@@ -54,7 +54,7 @@ def base_run(bin, force_qemu, verbose, cycles="NO"):
     else:
         logging.debug(f"Running {bin} natively")
 
-        args = ["make", f"CYCLES={cycles}", f"{bin}"]
+        args = ["make", f"{bin}"] + extra_make_args
         logging.info(" ".join(args))
 
         p = subprocess.run(
@@ -103,7 +103,13 @@ def parse_meta(scheme, field):
 
 
 def test_schemes(
-    title, scheme2file, actual_proc, expect_proc, force_qemu, verbose, cycles="NO"
+    title,
+    scheme2file,
+    actual_proc,
+    expect_proc,
+    force_qemu,
+    verbose,
+    extra_make_args=[],
 ):
     logging.info(f"{title}")
 
@@ -127,7 +133,7 @@ def test_schemes(
     results = {}
     for scheme in SCHEME:
         bin = scheme2file(scheme)
-        result = base_run(bin, force_qemu, verbose, cycles)
+        result = base_run(bin, force_qemu, verbose, extra_make_args)
         results[scheme] = result
 
         actual = actual_proc(result)
@@ -279,7 +285,12 @@ def kat(force_qemu, verbose):
     default="NO",
     help="Method for counting clock cycles. PMU requires (user-space) access to the Arm Performance Monitor Unit (PMU). PERF requires a kernel with perf support.",
 )
-def bench(force_qemu, verbose, cycles):
+@click.option(
+    "--cpu",
+    nargs=1,
+    help="Benchmark on specified CPU",
+)
+def bench(force_qemu, verbose, cycles, cpu):
     config_logger(verbose)
 
     results = test_schemes(
@@ -289,7 +300,7 @@ def bench(force_qemu, verbose, cycles):
         lambda _: True,
         force_qemu,
         verbose,
-        cycles=cycles,
+        f"CYCLES={cycles}",
     )
     for scheme, result in results.items():
         print(scheme)

From e9583903279d4eb1687d2b268c8837b19ed83284 Mon Sep 17 00:00:00 2001
From: "Thing-han, Lim" <15379156+potsrevennil@users.noreply.github.com>
Date: Wed, 19 Jun 2024 17:35:37 +0800
Subject: [PATCH 10/16] run benchmark for cpu cortex-a72 only

Signed-off-by: Thing-han, Lim <15379156+potsrevennil@users.noreply.github.com>
---
 .github/workflows/bench.yml | 2 +-
 Makefile                    | 8 ++++----
 scripts/tests               | 2 +-
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index cf42b6f2f..44e257df0 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -33,4 +33,4 @@ jobs:
       - name: Run benchmark
         shell: nix develop .#ci -c bash -e {0}
         run: |
-          tests bench -c PMU -v
+          tests bench -c PMU --cpu cortex-a72 -v
diff --git a/Makefile b/Makefile
index f24454db8..7913708b1 100644
--- a/Makefile
+++ b/Makefile
@@ -9,23 +9,23 @@ INCLUDE_FIPS202 = -I fips202
 INCLUDE_MLKEM = -I mlkem
 INCLUDE_RANDOM = -I randombytes
 INCLUDE_NISTRANDOM = -I test/nistrng
-CFLAGS += -Wall -Wextra -Wpedantic -Werror -Wmissing-prototypes -Wredundant-decls \
+override CFLAGS += -Wall -Wextra -Wpedantic -Werror -Wmissing-prototypes -Wredundant-decls \
   -Wshadow -Wpointer-arith -Wno-unknown-pragmas -O3 -fomit-frame-pointer -pedantic \
    ${INCLUDE_MLKEM} ${INCLUDE_FIPS202}
 
 HOST_PLATFORM := $(shell uname -s)-$(shell uname -m)
 ifeq ($(HOST_PLATFORM),Linux-x86_64)
-	CFLAGS += -static
+	override CFLAGS += -static
 endif
 
 CYCLES ?= NO
 
 ifeq ($(CYCLES),PMU)
-	CFLAGS += -DPMU_CYCLES
+	override CFLAGS += -DPMU_CYCLES
 endif
 
 ifeq ($(CYCLES),PERF)
-	CFLAGS += -DPERF_CYCLES
+	override CFLAGS += -DPERF_CYCLES
 endif
 
 CFLAGS_RANDOMBYTES = ${CFLAGS} ${INCLUDE_RANDOM}
diff --git a/scripts/tests b/scripts/tests
index 8bbc0d91d..a7477cd11 100755
--- a/scripts/tests
+++ b/scripts/tests
@@ -300,7 +300,7 @@ def bench(force_qemu, verbose, cycles, cpu):
         lambda _: True,
         force_qemu,
         verbose,
-        f"CYCLES={cycles}",
+        [f"CYCLES={cycles}", f"CFLAGS=-mcpu={cpu}"],
     )
     for scheme, result in results.items():
         print(scheme)

From 0f38480d945b00b6d672c944032839b463826c10 Mon Sep 17 00:00:00 2001
From: "Thing-han, Lim" <15379156+potsrevennil@users.noreply.github.com>
Date: Wed, 19 Jun 2024 18:21:09 +0800
Subject: [PATCH 11/16] print benchmark summary

Signed-off-by: Thing-han, Lim <15379156+potsrevennil@users.noreply.github.com>
---
 scripts/tests | 49 +++++++++++++++++++++++++++++++------------------
 1 file changed, 31 insertions(+), 18 deletions(-)

diff --git a/scripts/tests b/scripts/tests
index a7477cd11..856737248 100755
--- a/scripts/tests
+++ b/scripts/tests
@@ -107,28 +107,19 @@ def test_schemes(
     scheme2file,
     actual_proc,
     expect_proc,
+    process_result,
     force_qemu,
     verbose,
     extra_make_args=[],
 ):
+    """
+    :param process_result: process result and return summary
+    """
     logging.info(f"{title}")
 
     summary_file = os.environ.get("GITHUB_STEP_SUMMARY")
     summary = f"## {title}\n"
 
-    def check(scheme, expect, actual):
-        if actual != expect:
-            logging.error(f"{scheme} failed, expecting {expect}, but getting {actual}")
-            summary = f":x: {scheme}, expecting {expect}, but getting {actual}\n"
-
-            fail = True
-        else:
-            logging.info(f"{scheme} passed")
-            summary = f":white_check_mark: {scheme}\n"
-            fail = False
-
-        return (fail, summary)
-
     fail = False
     results = {}
     for scheme in SCHEME:
@@ -139,9 +130,10 @@ def test_schemes(
         actual = actual_proc(result)
         expect = expect_proc(scheme)
 
-        (f, s) = check(scheme, expect, actual)
-        fail = fail or f
+        f = actual != expect
+        s = process_result(f, scheme, result, expect, actual)
         summary += s
+        fail = fail or f
 
     if summary_file is not None:
         with open(summary_file, "a") as f:
@@ -153,6 +145,26 @@ def test_schemes(
     return results
 
 
+def process_test(fail, scheme, result, expect, actual):
+    if fail:
+        logging.error(f"{scheme} failed, expecting {expect}, but getting {actual}")
+        summary = f":x: {scheme}, expecting {expect}, but getting {actual}\n"
+    else:
+        logging.info(f"{scheme} passed")
+        summary = f":white_check_mark: {scheme}\n"
+
+    return summary
+
+
+def process_bench(fail, scheme, result, expect, actual):
+    logging.info(f"{scheme}")
+    logging.info(f"\n{result.decode()}")
+
+    summary = f"{scheme}\n{result.decode()}\n"
+
+    return summary
+
+
 def validate_force_qemu(ctx, _, v):
     if platform.system() == "Darwin" and v:
         config_logger(False)
@@ -230,6 +242,7 @@ def func(force_qemu, verbose):
         lambda scheme: scheme.name.replace("MLKEM", "test/bin/test_kyber"),
         lambda result: str(result, encoding="utf-8"),
         expect,
+        process_test,
         force_qemu,
         verbose,
     )
@@ -248,6 +261,7 @@ def nistkat(force_qemu, verbose):
         lambda scheme: scheme.name.replace("MLKEM", "test/bin/gen_NISTKAT"),
         sha256sum,
         lambda scheme: parse_meta(scheme, "nistkat-sha256"),
+        process_test,
         force_qemu,
         verbose,
     )
@@ -266,6 +280,7 @@ def kat(force_qemu, verbose):
         lambda scheme: scheme.name.replace("MLKEM", "test/bin/gen_KAT"),
         sha256sum,
         lambda scheme: parse_meta(scheme, "kat-sha256"),
+        process_test,
         force_qemu,
         verbose,
     )
@@ -298,13 +313,11 @@ def bench(force_qemu, verbose, cycles, cpu):
         lambda scheme: scheme.name.replace("MLKEM", "test/bin/bench_kyber"),
         lambda _: True,
         lambda _: True,
+        process_bench,
         force_qemu,
         verbose,
         [f"CYCLES={cycles}", f"CFLAGS=-mcpu={cpu}"],
     )
-    for scheme, result in results.items():
-        print(scheme)
-        print(result.decode())
 
 
 @click.group(invoke_without_command=True)

From 880e5a64491549efa62180e2fbe70c8c38783616 Mon Sep 17 00:00:00 2001
From: "Thing-han, Lim" <15379156+potsrevennil@users.noreply.github.com>
Date: Wed, 19 Jun 2024 18:44:15 +0800
Subject: [PATCH 12/16] run benchmark if there is benchmark label or on main
 branch

Signed-off-by: Thing-han, Lim <15379156+potsrevennil@users.noreply.github.com>
---
 .github/workflows/bench.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 44e257df0..a472ad4c8 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -9,7 +9,7 @@ on:
 jobs:
   bench:
     runs-on: self-hosted-rpi4
-    if: github.repository_owner == 'pq-code-package'
+    if: github.repository_owner == 'pq-code-package' && (contains(github.event.pull_request.labels.*.name, 'benchmark') || github.ref == 'refs/heads/main')
     steps:
       - uses: actions/checkout@v4
       - name: Setup nix

From 129808c1c977d55c4be3002e940b9edd55db20c2 Mon Sep 17 00:00:00 2001
From: "Thing-han, Lim" <15379156+potsrevennil@users.noreply.github.com>
Date: Thu, 20 Jun 2024 13:46:45 +0800
Subject: [PATCH 13/16] fix if cpu not provided for `tests bench`

Signed-off-by: Thing-han, Lim <15379156+potsrevennil@users.noreply.github.com>
---
 scripts/tests | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/tests b/scripts/tests
index 856737248..7c3671566 100755
--- a/scripts/tests
+++ b/scripts/tests
@@ -316,7 +316,7 @@ def bench(force_qemu, verbose, cycles, cpu):
         process_bench,
         force_qemu,
         verbose,
-        [f"CYCLES={cycles}", f"CFLAGS=-mcpu={cpu}"],
+        [f"CYCLES={cycles}", *([f"CFLAGS=-mcpu={cpu}"] if cpu is not None else [])],
     )
 
 

From 1d13369d906bb9e201817a76aac164df825bc736 Mon Sep 17 00:00:00 2001
From: "Thing-han, Lim" <15379156+potsrevennil@users.noreply.github.com>
Date: Thu, 20 Jun 2024 14:03:44 +0800
Subject: [PATCH 14/16] fail the tests script if running the binary failed

Signed-off-by: Thing-han, Lim <15379156+potsrevennil@users.noreply.github.com>
---
 scripts/tests | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/scripts/tests b/scripts/tests
index 7c3671566..a69599082 100755
--- a/scripts/tests
+++ b/scripts/tests
@@ -42,7 +42,7 @@ def base_run(bin, force_qemu, verbose, extra_make_args=[]):
             stdout=subprocess.DEVNULL if not verbose else None,
         )
         if p.returncode != 0:
-            logging.error("make failed")
+            logging.error(f"make failed: {p.returncode}")
             sys.exit(1)
 
         result = subprocess.run(
@@ -51,6 +51,12 @@ def base_run(bin, force_qemu, verbose, extra_make_args=[]):
             universal_newlines=False,
         )
 
+        if result.returncode != 0:
+            logging.error(
+                f"Emulating {bin} failed: {result.returncode} {result.stderr.decode()}"
+            )
+            sys.exit(1)
+
     else:
         logging.debug(f"Running {bin} natively")
 
@@ -63,7 +69,7 @@ def base_run(bin, force_qemu, verbose, extra_make_args=[]):
         )
 
         if p.returncode != 0:
-            logging.error("make failed")
+            logging.error(f"make failed: {p.returncode}")
             sys.exit(1)
 
         result = subprocess.run(
@@ -72,6 +78,12 @@ def base_run(bin, force_qemu, verbose, extra_make_args=[]):
             universal_newlines=False,
         )
 
+        if result.returncode != 0:
+            logging.error(
+                f"Running {bin} natively failed: {result.returncode} {result.stderr.decode()}"
+            )
+            sys.exit(1)
+
     return result.stdout
 
 

From 635babfa7766ea97e8a1003b7fc1df326778bb4d Mon Sep 17 00:00:00 2001
From: "Thing-han, Lim" <15379156+potsrevennil@users.noreply.github.com>
Date: Thu, 20 Jun 2024 14:53:54 +0800
Subject: [PATCH 15/16] use env vars for setting makefile cflags from cli

Signed-off-by: Thing-han, Lim <15379156+potsrevennil@users.noreply.github.com>
---
 Makefile      |  8 ++++----
 scripts/tests | 20 +++++++++++++++-----
 2 files changed, 19 insertions(+), 9 deletions(-)

diff --git a/Makefile b/Makefile
index 7913708b1..f24454db8 100644
--- a/Makefile
+++ b/Makefile
@@ -9,23 +9,23 @@ INCLUDE_FIPS202 = -I fips202
 INCLUDE_MLKEM = -I mlkem
 INCLUDE_RANDOM = -I randombytes
 INCLUDE_NISTRANDOM = -I test/nistrng
-override CFLAGS += -Wall -Wextra -Wpedantic -Werror -Wmissing-prototypes -Wredundant-decls \
+CFLAGS += -Wall -Wextra -Wpedantic -Werror -Wmissing-prototypes -Wredundant-decls \
   -Wshadow -Wpointer-arith -Wno-unknown-pragmas -O3 -fomit-frame-pointer -pedantic \
    ${INCLUDE_MLKEM} ${INCLUDE_FIPS202}
 
 HOST_PLATFORM := $(shell uname -s)-$(shell uname -m)
 ifeq ($(HOST_PLATFORM),Linux-x86_64)
-	override CFLAGS += -static
+	CFLAGS += -static
 endif
 
 CYCLES ?= NO
 
 ifeq ($(CYCLES),PMU)
-	override CFLAGS += -DPMU_CYCLES
+	CFLAGS += -DPMU_CYCLES
 endif
 
 ifeq ($(CYCLES),PERF)
-	override CFLAGS += -DPERF_CYCLES
+	CFLAGS += -DPERF_CYCLES
 endif
 
 CFLAGS_RANDOMBYTES = ${CFLAGS} ${INCLUDE_RANDOM}
diff --git a/scripts/tests b/scripts/tests
index a69599082..4f69fb689 100755
--- a/scripts/tests
+++ b/scripts/tests
@@ -25,7 +25,13 @@ def sha256sum(result):
     return m.hexdigest()
 
 
-def base_run(bin, force_qemu, verbose, extra_make_args=[]):
+def base_run(bin, force_qemu, verbose, extra_make_envs={}, extra_make_args=[]):
+    def dict2str(dict):
+        s = ""
+        for k, v in dict.items():
+            s += f"{k}={v} "
+        return s
+
     if force_qemu or (platform.system() == "Linux" and platform.machine() == "x86_64"):
         logging.debug(f"Emulating {bin} with QEMU")
 
@@ -35,11 +41,12 @@ def base_run(bin, force_qemu, verbose, extra_make_args=[]):
             f"{bin}",
         ] + extra_make_args
 
-        logging.info(" ".join(args))
+        logging.info(dict2str(extra_make_envs) + " ".join(args))
 
         p = subprocess.run(
             args,
             stdout=subprocess.DEVNULL if not verbose else None,
+            env=os.environ.copy() | extra_make_envs,
         )
         if p.returncode != 0:
             logging.error(f"make failed: {p.returncode}")
@@ -61,11 +68,12 @@ def base_run(bin, force_qemu, verbose, extra_make_args=[]):
         logging.debug(f"Running {bin} natively")
 
         args = ["make", f"{bin}"] + extra_make_args
-        logging.info(" ".join(args))
+        logging.info(dict2str(extra_make_envs) + " ".join(args))
 
         p = subprocess.run(
             args,
             stdout=subprocess.DEVNULL if not verbose else None,
+            env=os.environ.copy() | extra_make_envs,
         )
 
         if p.returncode != 0:
@@ -122,6 +130,7 @@ def test_schemes(
     process_result,
     force_qemu,
     verbose,
+    extra_make_envs={},
     extra_make_args=[],
 ):
     """
@@ -136,7 +145,7 @@ def test_schemes(
     results = {}
     for scheme in SCHEME:
         bin = scheme2file(scheme)
-        result = base_run(bin, force_qemu, verbose, extra_make_args)
+        result = base_run(bin, force_qemu, verbose, extra_make_envs, extra_make_args)
         results[scheme] = result
 
         actual = actual_proc(result)
@@ -328,7 +337,8 @@ def bench(force_qemu, verbose, cycles, cpu):
         process_bench,
         force_qemu,
         verbose,
-        [f"CYCLES={cycles}", *([f"CFLAGS=-mcpu={cpu}"] if cpu is not None else [])],
+        ({"CFLAGS": f"-mcpu={cpu}"} if cpu is not None else {}),
+        [f"CYCLES={cycles}"],
     )
 
 

From 4fb247ee773a9b2b55993d657bb42b9a8ef93e04 Mon Sep 17 00:00:00 2001
From: "Thing-han, Lim" <15379156+potsrevennil@users.noreply.github.com>
Date: Thu, 20 Jun 2024 17:16:05 +0800
Subject: [PATCH 16/16] allow user to pass in cflags and arch flags for tests
 command

Signed-off-by: Thing-han, Lim <15379156+potsrevennil@users.noreply.github.com>
---
 .github/workflows/bench.yml |  2 +-
 Makefile                    |  2 +-
 scripts/tests               | 43 ++++++++++++++++++++++++++-----------
 3 files changed, 33 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index a472ad4c8..fff40d1b7 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -33,4 +33,4 @@ jobs:
       - name: Run benchmark
         shell: nix develop .#ci -c bash -e {0}
         run: |
-          tests bench -c PMU --cpu cortex-a72 -v
+          tests bench -c PMU --cflags -mcpu=cortex-a72 -v
diff --git a/Makefile b/Makefile
index f24454db8..33cae997c 100644
--- a/Makefile
+++ b/Makefile
@@ -11,7 +11,7 @@ INCLUDE_RANDOM = -I randombytes
 INCLUDE_NISTRANDOM = -I test/nistrng
 CFLAGS += -Wall -Wextra -Wpedantic -Werror -Wmissing-prototypes -Wredundant-decls \
   -Wshadow -Wpointer-arith -Wno-unknown-pragmas -O3 -fomit-frame-pointer -pedantic \
-   ${INCLUDE_MLKEM} ${INCLUDE_FIPS202}
+   ${INCLUDE_MLKEM} ${INCLUDE_FIPS202} $(ARCH_FLAGS)
 
 HOST_PLATFORM := $(shell uname -s)-$(shell uname -m)
 ifeq ($(HOST_PLATFORM),Linux-x86_64)
diff --git a/scripts/tests b/scripts/tests
index 4f69fb689..23cee2837 100755
--- a/scripts/tests
+++ b/scripts/tests
@@ -193,6 +193,12 @@ def validate_force_qemu(ctx, _, v):
         ctx.exit(1)
 
 
+def process_make_envs(cflags, arch_flags):
+    return ({"CFLAGS": f"{cflags}"} if cflags is not None else {}) | (
+        {"ARCH_FLAGS": f"{arch_flags}"} if arch_flags is not None else {}
+    )
+
+
 _shared_options = [
     click.option(
         "--force-qemu",
@@ -214,6 +220,16 @@ _shared_options = [
         type=bool,
         help="Show verbose output or not",
     ),
+    click.option(
+        "--cflags",
+        nargs=1,
+        help="Extra cflags to passed in (e.g. '-mcpu=cortex-a72')",
+    ),
+    click.option(
+        "--arch-flags",
+        nargs=1,
+        help="Extra arch flags to passed in (e.g. '-march=armv8')",
+    ),
 ]
 
 
@@ -232,10 +248,15 @@ def add_options(options):
     type=click.Path(),
     help="The binary file that you wanted to test.",
 )
-def run(bin, force_qemu, verbose):
+def run(bin, force_qemu, verbose, cflags, arch_flags):
     config_logger(verbose)
 
-    result = base_run(bin, force_qemu, verbose)
+    result = base_run(
+        bin,
+        force_qemu,
+        verbose,
+        process_make_envs(cflags, arch_flags),
+    )
     logging.info(str(result, encoding="utf-8"))
 
 
@@ -244,7 +265,7 @@ def run(bin, force_qemu, verbose):
     context_settings={"show_default": True},
 )
 @add_options(_shared_options)
-def func(force_qemu, verbose):
+def func(force_qemu, verbose, cflags, arch_flags):
     config_logger(verbose)
 
     def expect(scheme):
@@ -266,6 +287,7 @@ def func(force_qemu, verbose):
         process_test,
         force_qemu,
         verbose,
+        process_make_envs(cflags, arch_flags),
     )
 
 
@@ -274,7 +296,7 @@ def func(force_qemu, verbose):
     context_settings={"show_default": True},
 )
 @add_options(_shared_options)
-def nistkat(force_qemu, verbose):
+def nistkat(force_qemu, verbose, cflags, arch_flags):
     config_logger(verbose)
 
     test_schemes(
@@ -285,6 +307,7 @@ def nistkat(force_qemu, verbose):
         process_test,
         force_qemu,
         verbose,
+        process_make_envs(cflags, arch_flags),
     )
 
 
@@ -293,7 +316,7 @@ def nistkat(force_qemu, verbose):
     context_settings={"show_default": True},
 )
 @add_options(_shared_options)
-def kat(force_qemu, verbose):
+def kat(force_qemu, verbose, cflags, arch_flags):
     config_logger(verbose)
 
     test_schemes(
@@ -304,6 +327,7 @@ def kat(force_qemu, verbose):
         process_test,
         force_qemu,
         verbose,
+        process_make_envs(cflags, arch_flags),
     )
 
 
@@ -321,12 +345,7 @@ def kat(force_qemu, verbose):
     default="NO",
     help="Method for counting clock cycles. PMU requires (user-space) access to the Arm Performance Monitor Unit (PMU). PERF requires a kernel with perf support.",
 )
-@click.option(
-    "--cpu",
-    nargs=1,
-    help="Benchmark on specified CPU",
-)
-def bench(force_qemu, verbose, cycles, cpu):
+def bench(force_qemu, verbose, cycles, cflags, arch_flags):
     config_logger(verbose)
 
     results = test_schemes(
@@ -337,7 +356,7 @@ def bench(force_qemu, verbose, cycles, cpu):
         process_bench,
         force_qemu,
         verbose,
-        ({"CFLAGS": f"-mcpu={cpu}"} if cpu is not None else {}),
+        process_make_envs(cflags, arch_flags),
         [f"CYCLES={cycles}"],
     )