diff --git a/.github/actions/functest/action.yml b/.github/actions/functest/action.yml index 48db89aa0..f44fd85dd 100644 --- a/.github/actions/functest/action.yml +++ b/.github/actions/functest/action.yml @@ -37,6 +37,9 @@ inputs: nistkat: description: Determine whether to run nistkat test or not default: "true" + acvp: + description: Determine whether to run acvp test or not + default: "true" runs: using: composite steps: @@ -55,6 +58,7 @@ runs: echo FUNC="${{ inputs.func == 'true' && 'func' || 'no-func' }}" >> $GITHUB_ENV echo KAT="${{ inputs.kat == 'true' && 'kat' || 'no-kat' }}" >> $GITHUB_ENV echo NISTKAT="${{ inputs.nistkat == 'true' && 'nistkat' || 'no-nistkat' }}" >> $GITHUB_ENV + echo ACVP="${{ inputs.acvp == 'true' && 'acvp' || 'no-acvp' }}" >> $GITHUB_ENV - name: Setup nix uses: ./.github/actions/setup-shell with: @@ -86,7 +90,7 @@ runs: - name: ${{ inputs.mode }} ${{ inputs.opt }} tests (${{ env.FUNC }}, ${{ env.KAT }}, ${{ env.NISTKAT }}) shell: ${{ env.SHELL }} run: | - tests all --cross-prefix="${{ env._CROSS_PREFIX }}" --cflags="${{ inputs.cflags }}" --opt=${{ inputs.opt }} --${{ env.FUNC }} --${{ env.KAT }} --${{ env.NISTKAT }} -v + tests all --cross-prefix="${{ env._CROSS_PREFIX }}" --cflags="${{ inputs.cflags }}" --opt=${{ inputs.opt }} --${{ env.FUNC }} --${{ env.KAT }} --${{ env.NISTKAT }} --${{ env.ACVP }} -v - name: Check namespacing ${{ inputs.mode }} ${{ inputs.opt }} tests (${{ env.FUNC }}, ${{ env.KAT }}, ${{ env.NISTKAT }}) shell: ${{ env.SHELL }} run: | diff --git a/.github/actions/multi-functest/action.yml b/.github/actions/multi-functest/action.yml index ca90ce99b..36e93ff91 100644 --- a/.github/actions/multi-functest/action.yml +++ b/.github/actions/multi-functest/action.yml @@ -37,6 +37,9 @@ inputs: nistkat: description: Determine whether to run nistkat test or not default: "true" + acvp: + description: Determine whether to run acvp test or not + default: "true" runs: using: composite steps: @@ -55,6 +58,7 @@ runs: func: ${{ inputs.func }} kat: ${{ inputs.kat }} nistkat: ${{ inputs.nistkat }} + acvp: ${{ inputs.acvp }} - name: Cross Tests if: ${{ (inputs.compile_mode == 'all' || inputs.compile_mode == 'cross') && (success() || failure()) }} uses: ./.github/actions/functest @@ -70,3 +74,4 @@ runs: func: ${{ inputs.func }} kat: ${{ inputs.kat }} nistkat: ${{ inputs.nistkat }} + acvp: ${{ inputs.acvp }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 25c711721..e3edc515b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -83,9 +83,6 @@ jobs: with: gh_token: ${{ secrets.GITHUB_TOKEN }} compile_mode: native - func: true - nistkat: true - kat: true - name: native tests (+debug+memsan+ubsan) uses: ./.github/actions/multi-functest with: @@ -101,9 +98,6 @@ jobs: gh_token: ${{ secrets.GITHUB_TOKEN }} compile_mode: cross opt: opt - func: true - nistkat: true - kat: true compiler_tests: name: Compiler tests (${{ matrix.target.name }}) strategy: @@ -133,6 +127,7 @@ jobs: func: true nistkat: false kat: false + acvp: false nix-shell: "ci_gcc48" - name: native build+functest (gcc-4.9) uses: ./.github/actions/multi-functest @@ -142,6 +137,7 @@ jobs: func: true nistkat: false kat: false + acvp: false nix-shell: "ci_gcc49" - name: native build+functest (gcc-7) uses: ./.github/actions/multi-functest @@ -151,6 +147,7 @@ jobs: func: true nistkat: false kat: false + acvp: false nix-shell: "ci_gcc7" - name: native build+functest (gcc-11) uses: ./.github/actions/multi-functest @@ -160,6 +157,7 @@ jobs: func: true nistkat: false kat: false + acvp: false nix-shell: "ci_gcc11" - name: native build+functest (clang-18) uses: ./.github/actions/multi-functest @@ -169,6 +167,7 @@ jobs: func: true nistkat: false kat: false + acvp: false nix-shell: "ci_clang18" lint: strategy: @@ -230,6 +229,7 @@ jobs: functest: true kattest: true nistkattest: true + acvptest: true lint: false verbose: true secrets: inherit @@ -252,6 +252,7 @@ jobs: functest: true kattest: false nistkattest: false + acvptest: false cbmc: true cbmc_mlkem_k: 2 secrets: inherit @@ -274,6 +275,7 @@ jobs: functest: true kattest: false nistkattest: false + acvptest: false cbmc: true cbmc_mlkem_k: 3 secrets: inherit @@ -296,6 +298,7 @@ jobs: functest: true kattest: false nistkattest: false + acvptest: false cbmc: true cbmc_mlkem_k: 4 secrets: inherit diff --git a/.github/workflows/ci_ec2_any.yml b/.github/workflows/ci_ec2_any.yml index 8e7f7f60e..2d5a3e678 100644 --- a/.github/workflows/ci_ec2_any.yml +++ b/.github/workflows/ci_ec2_any.yml @@ -66,6 +66,7 @@ jobs: functest: ${{ inputs.compile_mode != 'none' }} kattest: ${{ inputs.compile_mode != 'none' }} nistkattest: ${{ inputs.compile_mode != 'none' }} + acvptest: ${{ inputs.compile_mode != 'none' }} lint: true cbmc: ${{ inputs.cbmc }} verbose: ${{ inputs.verbose }} diff --git a/.github/workflows/ci_ec2_reusable.yml b/.github/workflows/ci_ec2_reusable.yml index 98e15a100..44f6628eb 100644 --- a/.github/workflows/ci_ec2_reusable.yml +++ b/.github/workflows/ci_ec2_reusable.yml @@ -47,6 +47,9 @@ on: nistkattest: type: boolean default: true + acvptest: + type: boolean + default: true lint: type: boolean default: true @@ -136,6 +139,7 @@ jobs: func: ${{ inputs.functest }} kat: ${{ inputs.kattest }} nistkat: ${{ inputs.nistkattest }} + acvp: ${{ inputs.acvptest }} - name: CBMC if: ${{ inputs.cbmc && (success() || failure()) }} uses: ./.github/actions/cbmc diff --git a/Makefile b/Makefile index 8b779a8e3..ff2feb4ff 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ quickcheck: buildall $(MLKEM512_DIR)/bin/test_mlkem512 $(MLKEM768_DIR)/bin/test_mlkem768 $(MLKEM1024_DIR)/bin/test_mlkem1024 - python3 ./test/acvp_client.py + ./scripts/acvp $(Q)echo " Functionality and ACVP tests passed!" mlkem: \ diff --git a/scripts/acvp b/scripts/acvp new file mode 100755 index 000000000..16c2737fb --- /dev/null +++ b/scripts/acvp @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 + +import os +import sys +import re + +sys.path.append(f"{os.path.join(os.path.dirname(__file__), 'lib')}") +from mlkem_test import * +from util import ( + path, + config_logger, +) + +opts = Options() +opts.compile = False + +config_logger(opts.verbose) + +build_config = path("test/build/config.mk") +acvp_test_data = path("test/acvp_data/") + +if not os.path.isfile(build_config): + logging.error(f"{build_config} not found") + exit(1) + + +def get_opt() -> bool: + with open(build_config, "r") as file: + for line in file: + # Use regex to match lines in the format "OPT := value" + match = re.match(r"^OPT\s*:=\s*(.*)$", line) + if match: + value = match.group(1).strip() + return value == "1" + logging.error(f"OPT is not defined in {build_config}") + exit(1) + + +opts.opt = "opt" if get_opt() else "no_opt" + +Tests(opts).acvp("test/acvp_data/") diff --git a/scripts/lib/mlkem_test.py b/scripts/lib/mlkem_test.py index 590dabac9..da6fdf9cb 100644 --- a/scripts/lib/mlkem_test.py +++ b/scripts/lib/mlkem_test.py @@ -6,7 +6,7 @@ import io import logging import subprocess -from functools import reduce +from functools import reduce, partial from typing import Optional, Callable, TypedDict from util import ( TEST_TYPES, @@ -17,6 +17,7 @@ github_summary, logger, ) +import json gh_env = os.environ.get("GITHUB_ENV") @@ -32,6 +33,9 @@ def __init__( self.auto = auto self.verbose = verbose + def compile_mode(self) -> str: + return "Cross" if self.cross_prefix else "Native" + class Options(object): def __init__(self): @@ -55,8 +59,9 @@ def __init__(self, test_type: TEST_TYPES, copts: CompileOptions, opt: bool): self.auto = copts.auto self.verbose = copts.verbose self.opt = opt - self.compile_mode = "Cross" if self.cross_prefix else "Native" + self.compile_mode = copts.compile_mode() self.opt_label = "opt" if self.opt else "no_opt" + self.i = 0 def compile_schemes( self, @@ -117,19 +122,20 @@ def run_scheme( self, scheme: SCHEME, actual_proc: Callable[[bytes], str] = None, - expect_proc: Callable[[SCHEME], str] = None, - run_as_root=False, - exec_wrapper=None, + expect_proc: Callable[[SCHEME, str], tuple[bool, str]] = None, + cmd_prefix: [str] = [], + extra_args: [str] = [], ): """Run the binary in all different ways""" - log = logger(self.test_type, scheme, self.cross_prefix, self.opt) + log = logger(self.test_type, scheme, self.cross_prefix, self.opt, self.i) + self.i += 1 bin = self.test_type.bin_path(scheme) if not os.path.isfile(bin): log.error(f"{bin} does not exists") sys.exit(1) - cmd = [f"./{bin}"] + cmd = [f"{bin}"] if self.cross_prefix and platform.system() != "Darwin": log.info(f"Emulating {bin} with QEMU") if "x86_64" in self.cross_prefix: @@ -141,16 +147,7 @@ def run_scheme( f"Emulation for {self.cross_prefix} on {platform.system()} not supported", ) - if run_as_root: - log.info( - f"Running {bin} as root -- you may need to enter your root password.", - ) - cmd = ["sudo"] + cmd - - if exec_wrapper: - log.info(f"Running {bin} with customized wrapper.") - exec_wrapper = exec_wrapper.split(" ") - cmd = exec_wrapper + cmd + cmd = cmd_prefix + cmd + extra_args log.debug(" ".join(cmd)) @@ -168,10 +165,9 @@ def run_scheme( ) elif actual_proc is not None and expect_proc is not None: actual = actual_proc(p.stdout) - expect = expect_proc(scheme) - result = actual != expect + result, err = expect_proc(scheme, actual) if result: - log.error(f"failed, expecting {expect}, but getting {actual}") + log.error(f"{err}") else: log.info(f"passed") else: @@ -187,6 +183,7 @@ def run_scheme( class Test_Implementations: def __init__(self, test_type: TEST_TYPES, copts: CompileOptions): self.test_type = test_type + self.compile_mode = copts.compile_mode() self.ts = {} self.ts["opt"] = Base(test_type, copts, True) self.ts["no_opt"] = Base(test_type, copts, False) @@ -202,20 +199,39 @@ def compile( extra_make_args, ) + def run_scheme( + self, + opt: bool, + scheme: SCHEME, + actual_proc: Callable[[bytes], str] = None, + expect_proc: Callable[[SCHEME, str], tuple[bool, str]] = None, + prefix: [str] = [], + extra_args: [str] = [], + ) -> TypedDict: + k = "opt" if opt else "no_opt" + + results = {} + results[k] = {} + results[k][scheme] = self.ts[k].run_scheme( + scheme, actual_proc, expect_proc, prefix, extra_args + ) + + return results + def run_schemes( self, opt: bool, actual_proc: Callable[[bytes], str] = None, - expect_proc: Callable[[SCHEME], str] = None, - run_as_root=False, - exec_wrapper=None, + expect_proc: Callable[[SCHEME, str], tuple[bool, str]] = None, + cmd_prefix: [str] = [], + extra_args: [str] = [], ) -> TypedDict: results = {} k = "opt" if opt else "no_opt" if gh_env is not None: - print(f"::group::run {self.ts[k].compile_mode} {k} {self.test_type.desc()}") + print(f"::group::run {self.compile_mode} {k} {self.test_type.desc()}") results[k] = {} for scheme in SCHEME: @@ -223,14 +239,14 @@ def run_schemes( scheme, actual_proc, expect_proc, - run_as_root, - exec_wrapper, + cmd_prefix, + extra_args, ) results[k][scheme] = result - title = "## " + (self.ts[k].compile_mode) + " " + (k.capitalize()) + " Tests" - github_summary(title, self.test_type, results[k]) + title = "## " + (self.compile_mode) + " " + (k.capitalize()) + " Tests" + github_summary(title, self.test_type.desc(), results[k]) if gh_env is not None: print(f"::endgroup::") @@ -262,28 +278,34 @@ def __init__(self, opts: Options): self._func = Test_Implementations(TEST_TYPES.MLKEM, copts) self._nistkat = Test_Implementations(TEST_TYPES.NISTKAT, copts) self._kat = Test_Implementations(TEST_TYPES.KAT, copts) + self._acvp = Test_Implementations(TEST_TYPES.ACVP, copts) self._bench = Test_Implementations(TEST_TYPES.BENCH, copts) self._bench_components = Test_Implementations( TEST_TYPES.BENCH_COMPONENTS, copts ) - self.compile_mode = "Cross" if opts.cross_prefix else "Native" + self.compile_mode = copts.compile_mode() self.compile = opts.compile self.run = opts.run def _run_func(self, opt: bool): """Underlying function for functional test""" - def expect(scheme: SCHEME) -> str: + def expect(scheme: SCHEME, actual: str) -> tuple[bool, str]: sk_bytes = parse_meta(scheme, "length-secret-key") pk_bytes = parse_meta(scheme, "length-public-key") ct_bytes = parse_meta(scheme, "length-ciphertext") - return ( + expect = ( f"CRYPTO_SECRETKEYBYTES: {sk_bytes}\n" + f"CRYPTO_PUBLICKEYBYTES: {pk_bytes}\n" + f"CRYPTO_CIPHERTEXTBYTES: {ct_bytes}\n" ) + fail = expect != actual + return ( + fail, + f"Failed, expecting {expect}, but getting {actual}" if fail else "", + ) return self._func.run_schemes( opt, @@ -311,10 +333,19 @@ def _func(opt: bool): exit(1) def _run_nistkat(self, opt: bool): + def expect_proc(scheme: SCHEME, actual: str) -> tuple[bool, str]: + expect = parse_meta(scheme, "nistkat-sha256") + fail = expect != actual + + return ( + fail, + f"Failed, expecting {expect}, but getting {actual}" if fail else "", + ) + return self._nistkat.run_schemes( opt, actual_proc=sha256sum, - expect_proc=lambda scheme: parse_meta(scheme, "nistkat-sha256"), + expect_proc=expect_proc, ) def nistkat(self): @@ -336,10 +367,19 @@ def _nistkat(opt: bool): exit(1) def _run_kat(self, opt: bool): + def expect_proc(scheme: SCHEME, actual: str) -> tuple[bool, str]: + expect = parse_meta(scheme, "kat-sha256") + fail = expect != actual + + return ( + fail, + f"Failed, expecting {expect}, but getting {actual}" if fail else "", + ) + return self._kat.run_schemes( opt, actual_proc=sha256sum, - expect_proc=lambda scheme: parse_meta(scheme, "kat-sha256"), + expect_proc=expect_proc, ) def kat(self): @@ -361,14 +401,174 @@ def _kat(opt: bool): if fail: exit(1) + def _run_acvp(self, opt: bool, acvp_dir: str = "test/acvp_data"): + acvp_keygen_json = f"{acvp_dir}/acvp_keygen_internalProjection.json" + acvp_encapDecap_json = f"{acvp_dir}/acvp_encapDecap_internalProjection.json" + + with open(acvp_keygen_json, "r") as f: + acvp_keygen_data = json.load(f) + + with open(acvp_encapDecap_json, "r") as f: + acvp_encapDecap_data = json.load(f) + + def actual_proc(bs: bytes) -> str: + return bs.decode("utf-8") + + def _expect_proc( + tc: TypedDict, scheme: SCHEME, actual: str + ) -> tuple[bool, str]: + fail = False + err = "" + for l in actual.splitlines(): + (k, v) = l.split("=") + if v != tc[k]: + fail = True + err = ( + err + + f"Failed, Mismatching result for {k}: expect {tc[k]}, but got {v}\n" + ) + return (fail, err) + + opt_label = "opt" if opt else "no_opt" + + def init_results() -> TypedDict: + results = {} + results[opt_label] = {} + for s in SCHEME: + results[opt_label][s] = False + return results + + fail = False + results = init_results() + # encapDecap + if gh_env is not None: + print( + f"::group::run {self.compile_mode} {opt_label} {TEST_TYPES.ACVP.desc()} encapDecap" + ) + + for i, tg in enumerate(acvp_encapDecap_data["testGroups"]): + scheme = SCHEME.from_str(tg["parameterSet"]) + + for tc in tg["tests"]: + if tg["function"] == "encapsulation": + extra_args = [ + "encapDecap", + "AFT", + "encapsulation", + f"ek={tc['ek']}", + f"m={tc['m']}", + ] + + elif tg["function"] == "decapsulation": + extra_args = [ + "encapDecap", + "VAL", + "decapsulation", + f"dk={tg['dk']}", + f"c={tc['c']}", + ] + + rs = self._acvp.run_scheme( + opt, + scheme, + extra_args=extra_args, + actual_proc=actual_proc, + expect_proc=partial(_expect_proc, tc), + ) + for k, r in rs.items(): + results[k][scheme] = results[k][scheme] or r[scheme] + + if gh_env is not None: + print(f"::endgroup::") + + for k, result in results.items(): + title = ( + "## " + (self._acvp.compile_mode) + " " + (k.capitalize()) + " Tests" + ) + github_summary(title, f"{TEST_TYPES.ACVP.desc()} encapDecap", result) + + fail = reduce(lambda acc, c: acc or c, result.values(), fail) + + results = init_results() + + if gh_env is not None: + print( + f"::group::run {self.compile_mode} {opt_label} {TEST_TYPES.ACVP.desc()} keyGen" + ) + + for i, tg in enumerate(acvp_keygen_data["testGroups"]): + scheme = SCHEME.from_str(tg["parameterSet"]) + + for tc in tg["tests"]: + extra_args = [ + "keyGen", + "AFT", + f"z={tc['z']}", + f"d={tc['d']}", + ] + + rs = self._acvp.run_scheme( + opt, + scheme, + extra_args=extra_args, + actual_proc=actual_proc, + expect_proc=partial(_expect_proc, tc), + ) + for k, r in rs.items(): + results[k][scheme] = results[k][scheme] or r[scheme] + + if gh_env is not None: + print(f"::endgroup::") + + for k, result in results.items(): + title = ( + "## " + + (self._acvp.ts[k].compile_mode) + + " " + + (k.capitalize()) + + " Tests" + ) + github_summary(title, f"{TEST_TYPES.ACVP.desc()} keyGen", result) + + fail = reduce(lambda acc, c: acc or c, result.values(), fail) + + return fail + + def acvp(self, acvp_dir: str): + config_logger(self.verbose) + + def _acvp(opt: bool): + if self.compile: + self._acvp.compile(opt) + if self.run: + return self._run_acvp(opt, acvp_dir) + + fail = False + + if self.opt.lower() == "all" or self.opt.lower() == "no_opt": + fail = fail or _acvp(False) + if self.opt.lower() == "all" or self.opt.lower() == "opt": + fail = fail or _acvp(True) + + if fail: + exit(1) + def _run_bench( self, t: Test_Implementations, opt: bool, run_as_root: bool, exec_wrapper: str ) -> TypedDict: - return t.run_schemes( - opt, - run_as_root=run_as_root, - exec_wrapper=exec_wrapper, - ) + cmd_prefix = [] + if run_as_root: + logging.info( + f"Running {bin} as root -- you may need to enter your root password.", + ) + cmd_prefix.append("sudo") + + if exec_wrapper: + logging.info(f"Running with customized wrapper.") + exec_wrapper = exec_wrapper.split(" ") + cmd_prefix = cmd_prefix + exec_wrapper + + return t.run_schemes(opt, cmd_prefix=cmd_prefix) def bench( self, @@ -444,7 +644,7 @@ def bench( ) f.write(json.dumps(v)) - def all(self, func: bool, kat: bool, nistkat: bool): + def all(self, func: bool, kat: bool, nistkat: bool, acvp: bool): config_logger(self.verbose) def all(opt: bool): @@ -454,6 +654,7 @@ def all(opt: bool): *([self._func.compile] if func else []), *([self._nistkat.compile] if nistkat else []), *([self._kat.compile] if kat else []), + *([self._acvp.compile] if kat else []), ] for f in compiles: @@ -469,6 +670,7 @@ def all(opt: bool): *([self._run_func] if func else []), *([self._run_nistkat] if nistkat else []), *([self._run_kat] if kat else []), + *([self._run_acvp] if acvp else []), ] for f in runs: diff --git a/scripts/lib/util.py b/scripts/lib/util.py index d92efe4af..c4dbcd17a 100644 --- a/scripts/lib/util.py +++ b/scripts/lib/util.py @@ -9,6 +9,13 @@ from functools import reduce import yaml +CWD = os.getcwd() +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) + + +def path(p: str): + return os.path.relpath(os.path.join(ROOT, p), CWD) + def sha256sum(result: bytes) -> str: m = hashlib.sha256() @@ -39,6 +46,16 @@ def __next__(self): def suffix(self): return self.name.removeprefix("MLKEM") + @classmethod + def from_str(cls, s: str): + # Iterate through all enum members to find a match for the given string + for m in cls: + if str(m) == s: + return m + raise ValueError( + f"'{s}' is not a valid string representation for {cls.__name__}" + ) + class TEST_TYPES(IntEnum): MLKEM = 1 @@ -46,6 +63,7 @@ class TEST_TYPES(IntEnum): NISTKAT = 3 KAT = 4 BENCH_COMPONENTS = 5 + ACVP = 6 def __str__(self): return self.name.lower() @@ -62,6 +80,8 @@ def desc(self): return "Nistkat Test" case TEST_TYPES.KAT: return "Kat Test" + case TEST_TYPES.ACVP: + return "ACVP Test" def bin(self): match self: @@ -75,9 +95,13 @@ def bin(self): return "gen_NISTKAT" case TEST_TYPES.KAT: return "gen_KAT" + case TEST_TYPES.ACVP: + return "acvp_mlkem" def bin_path(self, scheme: SCHEME): - return f"test/build/{scheme.name.lower()}/bin/{self.bin()}{scheme.suffix()}" + return path( + f"test/build/{scheme.name.lower()}/bin/{self.bin()}{scheme.suffix()}" + ) def parse_meta(scheme: SCHEME, field: str) -> str: @@ -86,7 +110,7 @@ def parse_meta(scheme: SCHEME, field: str) -> str: return meta["implementations"][int(scheme) - 1][field] -def github_summary(title: str, test: TEST_TYPES, results: TypedDict): +def github_summary(title: str, test_label: str, results: TypedDict): summary_file = os.environ.get("GITHUB_STEP_SUMMARY") res = list(results.values()) @@ -103,7 +127,7 @@ def github_summary(title: str, test: TEST_TYPES, results: TypedDict): ), ) ) - summaries = [f"| {test.desc()} |" + summaries[0]] + [ + summaries = [f"| {test_label} |" + summaries[0]] + [ "| |" + x for x in summaries[1:] ] else: @@ -111,7 +135,7 @@ def github_summary(title: str, test: TEST_TYPES, results: TypedDict): reduce( lambda acc, b: f"{acc} " + (":x: |" if b else ":white_check_mark: |"), res, - f"| {test.desc()} |", + f"| {test_label} |", ) ] @@ -171,10 +195,21 @@ def config_logger(verbose): logger.setLevel(logging.INFO) -def logger(test_type: TEST_TYPES, scheme: SCHEME, cross_prefix: str, opt: bool): +def logger( + test_type: TEST_TYPES, scheme: SCHEME, cross_prefix: str, opt: bool, i: int = None +): compile_mode = "cross" if cross_prefix else "native" implementation = "opt" if opt else "no_opt" return logging.getLogger( - f"{test_type.desc():<15} {str(scheme):<11} ({compile_mode:<6}, {implementation:>6})" + "{:<18} {:<11} ({:<6}, {:>6})".format( + ( + test_type.desc() + if (i is None or test_type is not TEST_TYPES.ACVP) + else f"{test_type.desc():<15} {i}" + ), + str(scheme), + compile_mode, + implementation, + ) ) diff --git a/scripts/tests b/scripts/tests index 22fde1215..1c60a811a 100755 --- a/scripts/tests +++ b/scripts/tests @@ -9,6 +9,7 @@ from functools import reduce sys.path.append(f"{os.path.join(os.path.dirname(__file__), 'lib')}") from mlkem_test import * +from util import path """ Click interface configuration @@ -190,6 +191,29 @@ def kat(opts: Options): Tests(opts).kat() +@cli.command( + short_help="Run ACVP client", + context_settings={"show_default": True}, +) +@add_options(_shared_options) +@add_options( + [ + click.option( + "-d", + "--acvp_dir", + nargs=1, + show_default=True, + type=click.Path(), + default=path("test/acvp_data"), + help="Path to acvp directory", + ), + ] +) +@click.make_pass_decorator(Options, ensure=True) +def acvp(opts: Options, acvp_dir: str): + Tests(opts).acvp(acvp_dir) + + @cli.command( short_help="Run the benchmarks for all parameter sets", context_settings={"show_default": True}, @@ -244,6 +268,13 @@ def bench( default=True, help="Determine whether to run nistkat test or not", ), + click.option( + "--acvp/--no-acvp", + is_flag=True, + show_default=True, + default=True, + help="Determine whether to run acvp test or not", + ), ] ) @click.make_pass_decorator(Options, ensure=True) @@ -252,8 +283,9 @@ def all( func: bool, kat: bool, nistkat: bool, + acvp: bool, ): - Tests(opts).all(func, kat, nistkat) + Tests(opts).all(func, kat, nistkat, acvp) if __name__ == "__main__": diff --git a/test/acvp_client.py b/test/acvp_client.py deleted file mode 100644 index bc7e6c885..000000000 --- a/test/acvp_client.py +++ /dev/null @@ -1,97 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 - -# ACVP client for ML-KEM -# -# Processes 'internalProjection.json' files from -# https://github.com/usnistgov/ACVP-Server/blob/master/gen-val/json-files -# -# Invokes `acvp_mlkem{lvl}` under the hood. - -import json -import subprocess - -acvp_dir = "test/acvp_data" -acvp_keygen_json = f"{acvp_dir}/acvp_keygen_internalProjection.json" -acvp_encapDecap_json = f"{acvp_dir}/acvp_encapDecap_internalProjection.json" - -with open(acvp_keygen_json, 'r') as f: - acvp_keygen_data = json.load(f) - -with open(acvp_encapDecap_json, 'r') as f: - acvp_encapDecap_data = json.load(f) - -def get_acvp_binary(tg): - """Convert JSON dict for ACVP test group to suitable ACVP binary.""" - parameterSetToLevel = { - "ML-KEM-512": 512, - "ML-KEM-768": 768, - "ML-KEM-1024": 1024, - } - level = parameterSetToLevel[tg["parameterSet"]] - basedir = f"./test/build/mlkem{level}/bin" - acvp_bin = f"acvp_mlkem{level}" - return f"{basedir}/{acvp_bin}" - -def run_encapDecap_test(tg, tc): - print(f"Running encapDecap test case {tc['tcId']} ({tg['function']}) ... ", end='') - if tg["function"] == "encapsulation": - acvp_bin = get_acvp_binary(tg) - acvp_call = [ acvp_bin, "encapDecap", "AFT", "encapsulation", f"ek={tc['ek']}", f"m={tc['m']}" ] - result = subprocess.run(acvp_call, encoding="utf-8", capture_output=True) - if result.returncode != 0: - print("FAIL!") - print(f"{acvp_call} failed with error code {result.returncode}") - print(result.stderr) - exit(1) - # Extract results and compare to expected data - for l in result.stdout.splitlines(): - (k,v) = l.split("=") - if v != tc[k]: - print("FAIL!") - print(f"Mismatching result for {k}: expected {tc[k]}, got {v}") - exit(1) - print("OK") - elif tg["function"] == "decapsulation": - acvp_bin = get_acvp_binary(tg) - acvp_call = [ acvp_bin, "encapDecap", "VAL", "decapsulation", f"dk={tg['dk']}", f"c={tc['c']}" ] - result = subprocess.run(acvp_call, encoding="utf-8", capture_output=True) - if result.returncode != 0: - print("FAIL!") - print(f"{acvp_call} failed with error code {result.returncode}") - print(result.stderr) - exit(1) - # Extract results and compare to expected data - for l in result.stdout.splitlines(): - (k,v) = l.split("=") - if v != tc[k]: - print("FAIL!") - print(f"Mismatching result for {k}: expected {tc[k]}, got {v}") - exit(1) - print("OK") - -def run_keyGen_test(tg, tc): - print(f"Running keyGen test case {tc['tcId']} ... ", end='') - acvp_bin = get_acvp_binary(tg) - acvp_call = [ acvp_bin, "keyGen", "AFT", f"z={tc['z']}", f"d={tc['d']}" ] - result = subprocess.run(acvp_call, encoding="utf-8", capture_output=True) - if result.returncode != 0: - print("FAIL!") - print(f"{acvp_call} failed with error code {result.returncode}") - print(result.stderr) - exit(1) - # Extract results and compare to expected data - for l in result.stdout.splitlines(): - (k,v) = l.split("=") - if v != tc[k]: - print("FAIL!") - print(f"Mismatching result for {k}: expected {tc[k]}, got {v}") - exit(1) - print("OK") - -for tg in acvp_encapDecap_data["testGroups"]: - for tc in tg["tests"]: - run_encapDecap_test(tg,tc) - -for tg in acvp_keygen_data["testGroups"]: - for tc in tg["tests"]: - run_keyGen_test(tg,tc)