Merge branch 'main' into pin-clang-format-version

Signed-off-by: Hanno Becker <[email protected]>

.github/workflows/ci.yml

@@ -35,11 +35,15 @@ jobs:
     runs-on: ${{ matrix.target.runner }}
     steps:
       - uses: actions/checkout@v4
-      - name: quickcheck
+      - name: make quickcheck
         run: |
           OPT=0 make quickcheck >/dev/null
           make clean            >/dev/null
           OPT=1 make quickcheck >/dev/null
+      - uses: ./.github/actions/setup-ubuntu
+      - name: tests func
+        run: |
+          ./scripts/tests func
   build_kat:
     needs: quickcheck
     strategy:

Makefile

@@ -7,7 +7,7 @@ buildall:
 	$(Q)$(MAKE) mlkem
 	$(Q)$(MAKE) nistkat
 	$(Q)$(MAKE) kat
-	$(Q)echo "  ALL GOOD!"
+	$(Q)echo "  Everything builds fine!"
 
 include mk/config.mk
 -include mk/$(MAKECMDGOALS).mk
@@ -16,19 +16,16 @@ include mk/schemes.mk
 include mk/rules.mk
 
 quickcheck:
+        # Check that everything builds
 	$(Q)$(MAKE) mlkem
+	$(Q)$(MAKE) nistkat
+	$(Q)$(MAKE) kat
+	$(Q)echo "  Everything builds fine!"
+        # Run basic functionality checks
 	$(MLKEM512_DIR)/bin/test_kyber512
 	$(MLKEM768_DIR)/bin/test_kyber768
 	$(MLKEM1024_DIR)/bin/test_kyber1024
-	$(Q)$(MAKE) nistkat
-	$(MLKEM512_DIR)/bin/gen_NISTKAT512
-	$(MLKEM768_DIR)/bin/gen_NISTKAT768
-	$(MLKEM1024_DIR)/bin/gen_NISTKAT1024
-	$(Q)$(MAKE) kat
-	$(MLKEM512_DIR)/bin/gen_KAT512
-	$(MLKEM768_DIR)/bin/gen_KAT768
-	$(MLKEM1024_DIR)/bin/gen_KAT1024
-	$(Q)echo "  ALL GOOD!"
+	$(Q)echo "  Functionality tests passed!"
 
 mlkem: \
   $(MLKEM512_DIR)/bin/test_kyber512 \

README.md

@@ -1,78 +1,97 @@
 [//]: # (SPDX-License-Identifier: CC-BY-4.0)
 
-**mlkem-c-aarch64** is a collection of [MLKEM](https://doi.org/10.6028/NIST.FIPS.203.ipd) implementations for CPUs based on the Armv8-A and Armv9-A architectures.
+# mlkem-c-aarch64
 
-## Goals of mlkem-c-aarch64
+**mlkem-c-aarch64** is a C99 implementation of [ML-KEM](https://doi.org/10.6028/NIST.FIPS.203.ipd) targeting
+PC, mobile and server platforms. It is a fork of the ML-KEM [reference
+implementation](https://github.com/pq-crystals/kyber/tree/main/ref) and remains very close to it, adding a simple
+interface for native code (e.g. assembler) as well as implementations of that interface in C, AArch64, and x86_64.
 
-The primary goals of this project are as follows:
-- _Assurance:_ Offer code that's extensively tested and amenable for audit and verification
-- _Ease of use:_ Permissive licensing, modularity, adaptability, few dependencies
-- _Performance:_ Competitive performance for most Armv8-A/Armv9-A platforms
+If you need an ML-KEM implementation suitable for embedded systems, see
+[**mlkem-c-embedded**](https://github.com/pq-code-package/mlkem-c-embedded/).
 
-There are tensions between these goals:
-- Optimal code is target-specific, but a large variety of CPU-specific implementations makes a library harder to both use and maintain.
-- Optimal code is complex (e.g. relying on handwritten assembly and extensive interleaving), impeding maintainenance and
-  amenability for audit, and potentially hardening verification efforts.
+**Note on naming:** **mlkem-c-aarch64** is a misnomer reflecting the original focus on AArch64. The project is likely to be renamed to
+reflect its broadened scope.
 
-In doubt, **mlkem-c-aarch64** chooses assurance and ease of use over performance: We only include implementations into **mlkem-c-aarch64** which are manually auditable or (ideally _and_) for which we see a path towards formal verification. All assembly should be as readable as possible and micro-optimization ideally deferred to automated tooling such as [SLOTHY](https://slothy-optimizer.github.io/slothy/). Ultimately, **mlkem-c-aarch64** strives for constant-time implementations for which the C-code is, at minimum, verified to be free of undefined behaviour, and where all assembly is functionally verified.
+### Goals
 
-**mlkem-c-aarch64** aims to provide a portfolio of implementations jointly providing competitive performance for most Armv8-A/Armv9-A microarchitectures. For some specific microarchitectures of particular interest, **mlkem-c-aarch64** may also provide CPU-specific implementations. Initially, our benchmarking platforms are:
-- Arm Cortex-A55
-- Arm Cortex-A72 (as used in the Raspberry Pi4)
-- Arm Cortex-A76 (as used in the Raspberry Pi5) / Neoverse N1 (as used in AWS Graviton2/c6g instances)
-- Arm Neoverse V1 (as used in the AWS Graviton3/c7g instances)
-- Apple M1
+**mlkem-c-aarch64** aims for _assurance_, _ease of use_, and _performance_. We only include implementations into
+**mlkem-c-aarch64** which are manually auditable or for which we see a path towards formal verification. All assembly aims
+to be readable and micro-optimization deferred to automated tooling such as
+[SLOTHY](https://slothy-optimizer.github.io/slothy/). Ultimately, **mlkem-c-aarch64** strives for constant-time
+implementations for which the C-code is verified to be free of undefined behaviour, and where all assembly is
+functionally verified.
 
-Please reach out to the **mlkem-c-aarch64** maintainers or open an issue if you would like to see benchmarking on other microarchitectures.
+### Current state
 
-## Non-goals
+**mlkem-c-aarch64** is work in progress. **WE DO NOT CURRENTLY RECOMMEND RELYING ON THIS LIBRARY IN A PRODUCTION
+ENVIRONMENT OR TO PROTECT ANY SENSITIVE DATA.** Once we have the first stable version, this notice will be removed.
 
-At this point, we do not provide implementations optimized for memory usage (code / RAM). If you need a memory-optimized implementation and the implementation provided by MLKEM-C-Generic is not of sufficient performance to your application, please contact us.
+#### Performance
 
-## Relation to MLKEM-C-Generic
+**mlkem-c-aarch64** has complete AArch64 and AVX2 backends of competitive performance (see
+[benchmarks](https://pq-code-package.github.io/mlkem-c-aarch64/dev/bench/)).
 
-Eventually, we aim to unify the (shared) C-part of the implementations provided by **mlkem-c-aarch64** with the implementations in [mlkem-c-generic](https://github.com/pq-code-package/mlkem-c-generic). Initially, however, we will allow some divergence, e.g. to explore interfaces to 2-/4-/8-way parallel Keccak implementations which are essential for high-performance implementations of MLKEM.
+#### Verification
 
+Mostly TODO. We apply CBMC to verify the absence of UB in a few basic C routines, but the bulk of the C verification
+is outstanding. No formal verification has yet been applied to the backends.
 
-## Current state
+### Getting started
 
-**mlkem-c-aarch64** is currently a work in progress and we do not recommend relying on it at this point.
-**WE DO NOT CURRENTLY RECOMMEND RELYING ON THIS LIBRARY IN A PRODUCTION ENVIRONMENT OR TO PROTECT ANY SENSITIVE DATA.**
-Once we have the first stable version, this notice will be removed.
+### Nix setup
 
-The current code is compatible with the [`standard` branch of the official MLKEM repository](https://github.com/pq-crystals/kyber/tree/standard).
+All the development and build dependencies are specified in [flake.nix](flake.nix). We recommend installing them using
+[nix](https://nixos.org/download/).
 
+To execute a bash shell with the development environment specified in [flake.nix](flake.nix), run
+```bash
+nix develop --experimental-features 'nix-command flakes'
+```
+
+### Native setup
 
-## Development
+To build **mlkem-c-aarch64**, you need `make` and a C99 compiler. To use the test scripts, you need Python3 with
+dependencies as specified in [requirements.txt](requirements.txt). We recommend using a virtual environment, e.g.:
 
-### Environment Setup
+```bash
+python3 -m venv venv
+./venv/bin/python3 -m pip install -r requirements.txt
+source venv/bin/activate
+```
 
-All the development and build dependencies are specified in [flake.nix](flake.nix). We recommend installing them using [nix](https://nixos.org/download/).
+### Using `make`
 
-- **Setup with nix**
-    - Running `nix develop` will execute a bash shell with the development environment specified in [flake.nix](flake.nix).
-    - Alternatively, you can enable `direnv` by using `direnv allow`, allowing it to handle the environment setup for you.
+You can build tests and benchmarks using the following `make` targets:
 
-    - As flake is still an experimental feature of nix, `--experimental-features 'nix-command flakes'` is needed when running the nix command. Alternatively, add the following to your `~/.config/nix/nix.conf` or `/etc/nix/nix.conf`:
-```
-experimental-features = nix-command flakes
+```bash
+make mlkem
+make bench
+make bench_components
+make nistkat
+make kat
 ```
 
-- If you are not using nix, please ensure you have installed the same versions as specified in [flake.nix](flake.nix).
+The resulting binaries can be found in [test/build](test/build).
 
-### Development scripts
-After running `nix develop` you should automatically have a number of support scripts in your PATH:
+### Using `tests` script
+
+We recommend compiling and running tests and benchmarks using the [`./scripts/tests`](scripts/tests) script. For
+example,
+
+```bash
+./scripts/tests func
+```
 
-- [`format`](scripts/format) formats all files. The format is enforced by our CI, so you should run this script prior to committing.
-- [`tests`](scripts/tests) run functional, kat tests natively or emulate them using `QEMU`. For information on how to use the script, please refer to the `--help` option.
+will compile and run functionality tests. For detailed information on how to use the script, please refer to the
+`--help` option.
 
-## Call for contributors
+### Call for contributors
 
-We are actively seeking contributors who can help us build **mlkem-c-aarch64**.
-If you are interested, please contact us, or volunteer for any of the open issues.
+We are actively seeking contributors who can help us build **mlkem-c-aarch64**. If you are interested, please contact us,
+or volunteer for any of the open issues.
 
-## Call for potential consumers
+### Call for potential consumers
 
-If you are a potential consumer of **mlkem-c-aarch64**, please reach out to us.
-We're interested in hearing the way you are considering using **mlkem-c-aarch64** and could benefit from additional features.
-If you have specific feature requests, please open an issue.
+If you are a potential consumer of **mlkem-c-aarch64**, please reach out: We're interested in hearing the way you want to
+use **mlkem-c-aarch64**. If you have specific feature requests, please open an issue.

flake.nix

@@ -72,7 +72,7 @@
                 qemu; # 8.2.4
 
               inherit (pkgs.python3Packages)
-                yq
+                pyyaml
                 python
                 click;
             };

requirements.txt

@@ -1,4 +1,3 @@
 # SPDX-License-Identifier: Apache-2.0
 click
-yq==3.4.3
-jq==1.8.0
+pyyaml==6.0.2

scripts/tests

@@ -8,6 +8,7 @@ import subprocess
 import logging
 import click
 import hashlib
+import yaml
 import asyncio
 from functools import reduce
 from enum import IntEnum
@@ -100,21 +101,9 @@ class SCHEME(IntEnum):
 
 
 def parse_meta(scheme: SCHEME, field: str) -> str:
-    result = subprocess.run(
-        [
-            "yq",
-            "-r",
-            "--arg",
-            "scheme",
-            str(scheme),
-            f'.implementations.[] | select(.name == $scheme) | ."{field}"',
-            "./META.yml",
-        ],
-        capture_output=True,
-        encoding="utf-8",
-        universal_newlines=False,
-    )
-    return result.stdout.strip()
+    with open("META.yml", "r") as f:
+        meta = yaml.safe_load(f)
+    return meta["implementations"][int(scheme) - 1][field]
 
 
 def github_summary(title: str, test: TEST_TYPES, results: TypedDict):