From a462f0260041a214441617b44623e1f59e72a4a1 Mon Sep 17 00:00:00 2001 From: Hanno Becker Date: Mon, 16 Dec 2024 05:56:50 +0000 Subject: [PATCH] WIP: Introduce global config and reorganize backends Signed-off-by: Hanno Becker --- fips202/fips202.h | 3 +- fips202/fips202x4.h | 2 +- fips202/keccakf1600.c | 2 +- fips202/keccakf1600.h | 5 +- fips202/namespace.h | 30 -------- .../native/aarch64/fips202_native_aarch64.h | 6 +- .../aarch64/keccak_f1600_x1_scalar_asm_opt.S | 6 +- .../aarch64/keccak_f1600_x1_v84a_asm_clean.S | 5 +- .../aarch64/keccak_f1600_x2_v84a_asm_clean.S | 5 +- .../keccak_f1600_x2_v8a_v84a_asm_hybrid.S | 6 +- ...ccak_f1600_x4_scalar_v84a_asm_hybrid_opt.S | 6 +- ...eccak_f1600_x4_v8a_scalar_hybrid_asm_opt.S | 6 +- ..._f1600_x4_v8a_v84a_scalar_hybrid_asm_opt.S | 5 +- .../{cortex_a55.h => cortex_a55_impl.h} | 8 +- .../aarch64/profiles/cortex_a55_metadata.h | 24 ++++++ .../profiles/{default.h => default_impl.h} | 4 +- .../aarch64/profiles/default_metadata.h | 24 ++++++ fips202/native/default.h | 22 ++++++ fips202/native/fips202_impl.h | 15 ++++ fips202/native/fips202_native.h | 50 +------------ fips202/native/x86_64/fips202_native_x86_64.h | 20 ----- .../x86_64/profiles/{xkcp.h => xkcp_impl.h} | 12 +-- .../native/x86_64/profiles/xkcp_metadata.h | 24 ++++++ .../x86_64/xkcp/KeccakP-1600-times4-SIMD256.c | 7 +- .../x86_64/xkcp/KeccakP-1600-times4-SnP.h | 2 +- mlkem/api.h | 62 ---------------- mlkem/cbd.c | 1 - mlkem/cbd.h | 2 +- mlkem/common.h | 69 ++++-------------- mlkem/config.h | 34 +++++++++ mlkem/debug/debug.c | 1 - mlkem/debug/debug.h | 5 ++ mlkem/indcpa.c | 1 - mlkem/indcpa.h | 3 +- mlkem/kem.c | 1 - mlkem/namespace.h | 57 +++++++++++++++ mlkem/native/aarch64/aarch64_zetas.c | 12 ++- mlkem/native/aarch64/arith_native_aarch64.h | 7 +- mlkem/native/aarch64/consts.h | 2 +- mlkem/native/aarch64/intt_clean.S | 4 +- mlkem/native/aarch64/intt_opt.S | 4 +- mlkem/native/aarch64/ntt_clean.S | 4 +- mlkem/native/aarch64/ntt_opt.S | 4 +- mlkem/native/aarch64/poly_clean.S | 4 +- mlkem/native/aarch64/poly_opt.S | 4 +- mlkem/native/aarch64/polyvec_clean.S | 4 +- mlkem/native/aarch64/polyvec_opt.S | 8 +- .../profiles/{clean.h => clean_impl.h} | 11 +-- .../native/aarch64/profiles/clean_metadata.h | 24 ++++++ .../aarch64/profiles/{opt.h => opt_impl.h} | 10 +-- mlkem/native/aarch64/profiles/opt_metadata.h | 24 ++++++ mlkem/native/aarch64/rej_uniform_asm_clean.S | 6 +- mlkem/native/arith_impl.h | 15 ++++ mlkem/native/arith_native.h | 9 +-- mlkem/native/default.h | 32 ++++++++ mlkem/native/profile.h | 73 ------------------- mlkem/native/x86_64/arith_native_x86_64.h | 10 +-- mlkem/native/x86_64/basemul.S | 2 +- mlkem/native/x86_64/basemul.c | 8 +- mlkem/native/x86_64/consts.c | 6 +- mlkem/native/x86_64/consts.h | 2 +- mlkem/native/x86_64/fq.S | 5 +- mlkem/native/x86_64/intt.S | 5 +- mlkem/native/x86_64/ntt.S | 2 +- .../profiles/{default.h => default_impl.h} | 6 +- .../native/x86_64/profiles/default_metadata.h | 24 ++++++ mlkem/native/x86_64/rej_uniform_avx2.c | 10 +-- mlkem/native/x86_64/shuffle.S | 4 +- mlkem/ntt.c | 6 +- mlkem/ntt.h | 3 +- mlkem/params.h | 39 +--------- mlkem/poly.c | 9 +-- mlkem/poly.h | 2 +- mlkem/polyvec.c | 1 - mlkem/polyvec.h | 2 +- mlkem/reduce.h | 2 +- mlkem/rej_uniform.c | 3 +- mlkem/rej_uniform.h | 2 +- mlkem/symmetric.h | 6 +- mlkem/{sys/cpucap.h => sys.h} | 57 ++++++++++++++- mlkem/sys/config.h | 22 ------ mlkem/verify.c | 1 - scripts/autogenerate_files.py | 12 ++- 83 files changed, 526 insertions(+), 521 deletions(-) delete mode 100644 fips202/namespace.h rename fips202/native/aarch64/profiles/{cortex_a55.h => cortex_a55_impl.h} (78%) create mode 100644 fips202/native/aarch64/profiles/cortex_a55_metadata.h rename fips202/native/aarch64/profiles/{default.h => default_impl.h} (97%) create mode 100644 fips202/native/aarch64/profiles/default_metadata.h create mode 100644 fips202/native/default.h create mode 100644 fips202/native/fips202_impl.h delete mode 100644 fips202/native/x86_64/fips202_native_x86_64.h rename fips202/native/x86_64/profiles/{xkcp.h => xkcp_impl.h} (60%) create mode 100644 fips202/native/x86_64/profiles/xkcp_metadata.h delete mode 100644 mlkem/api.h create mode 100644 mlkem/config.h create mode 100644 mlkem/namespace.h rename mlkem/native/aarch64/profiles/{clean.h => clean_impl.h} (90%) create mode 100644 mlkem/native/aarch64/profiles/clean_metadata.h rename mlkem/native/aarch64/profiles/{opt.h => opt_impl.h} (92%) create mode 100644 mlkem/native/aarch64/profiles/opt_metadata.h create mode 100644 mlkem/native/arith_impl.h create mode 100644 mlkem/native/default.h delete mode 100644 mlkem/native/profile.h rename mlkem/native/x86_64/profiles/{default.h => default_impl.h} (95%) create mode 100644 mlkem/native/x86_64/profiles/default_metadata.h rename mlkem/{sys/cpucap.h => sys.h} (53%) delete mode 100644 mlkem/sys/config.h diff --git a/fips202/fips202.h b/fips202/fips202.h index 48429c271..333a959d7 100644 --- a/fips202/fips202.h +++ b/fips202/fips202.h @@ -7,9 +7,8 @@ #include #include -#include "namespace.h" - #include "cbmc.h" +#include "common.h" #define SHAKE128_RATE 168 #define SHAKE256_RATE 136 diff --git a/fips202/fips202x4.h b/fips202/fips202x4.h index 33a732a88..1bb0d08c7 100644 --- a/fips202/fips202x4.h +++ b/fips202/fips202x4.h @@ -7,9 +7,9 @@ #include #include +#include "common.h" #include "fips202.h" #include "keccakf1600.h" -#include "namespace.h" #include "cbmc.h" diff --git a/fips202/keccakf1600.c b/fips202/keccakf1600.c index 7b653fca9..5a88a4244 100644 --- a/fips202/keccakf1600.c +++ b/fips202/keccakf1600.c @@ -15,7 +15,7 @@ #include #include "config.h" -#include "fips202_native.h" +#include "fips202_impl.h" #include "cbmc.h" diff --git a/fips202/keccakf1600.h b/fips202/keccakf1600.h index 7341df0bf..4b3a88253 100644 --- a/fips202/keccakf1600.h +++ b/fips202/keccakf1600.h @@ -6,10 +6,9 @@ #define KECCAKF1600_H #include -#include "fips202_native.h" -#include "namespace.h" - #include "cbmc.h" +#include "common.h" +#include "fips202_native.h" #define KECCAK_LANES 25 /* diff --git a/fips202/namespace.h b/fips202/namespace.h deleted file mode 100644 index fab249afa..000000000 --- a/fips202/namespace.h +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (c) 2024 The mlkem-native project authors - * SPDX-License-Identifier: Apache-2.0 - */ -#ifndef NAMESPACE_H -#define NAMESPACE_H - -#include "config.h" - -#if defined(MLKEM_USE_NATIVE_X86_64) -#define MLKEM_NATIVE_BACKEND X86_64 -#elif defined(MLKEM_USE_NATIVE_AARCH64) -#define MLKEM_NATIVE_BACKEND AARCH64 -#else -#define MLKEM_NATIVE_BACKEND C -#endif - -#define ___FIPS202_NAMESPACE(x1, x2, x3) x1##_##x2##_##x3 -#define __FIPS202_NAMESPACE(x1, x2, x3) ___FIPS202_NAMESPACE(x1, x2, x3) - -/* - * NAMESPACE is PQCP_MLKEM_NATIVE_FIPS202__ - * e.g., PQCP_MLKEM_NATIVE_FIPS202_AARCH64_ - */ -#define FIPS202_NAMESPACE(s) \ - __FIPS202_NAMESPACE(PQCP_MLKEM_NATIVE_FIPS202, MLKEM_NATIVE_BACKEND, s) -#define _FIPS202_NAMESPACE(s) \ - __FIPS202_NAMESPACE(_PQCP_MLKEM_NATIVE_FIPS202, MLKEM_NATIVE_BACKEND, s) - -#endif diff --git a/fips202/native/aarch64/fips202_native_aarch64.h b/fips202/native/aarch64/fips202_native_aarch64.h index 72e7c9710..d62b2c958 100644 --- a/fips202/native/aarch64/fips202_native_aarch64.h +++ b/fips202/native/aarch64/fips202_native_aarch64.h @@ -6,11 +6,8 @@ #define FIPS202_AARCH64_NATIVE_H #include -#include "config.h" -#include "namespace.h" -#include "params.h" +#include "common.h" -#ifdef MLKEM_USE_NATIVE_AARCH64 #define keccak_f1600_x1_scalar_asm_opt \ FIPS202_NAMESPACE(keccak_f1600_x1_scalar_asm_opt) void keccak_f1600_x1_scalar_asm_opt(uint64_t *state); @@ -38,6 +35,5 @@ void keccak_f1600_x4_scalar_v84a_asm_hybrid_opt(uint64_t *state); #define keccak_f1600_x4_scalar_v8a_v84a_hybrid_asm_opt \ FIPS202_NAMESPACE(keccak_f1600_x4_scalar_v8a_v84a_hybrid_asm_opt) void keccak_f1600_x4_scalar_v8a_v84a_hybrid_asm_opt(uint64_t *state); -#endif /* MLKEM_USE_NATIVE_AARCH64 */ #endif /* FIPS202_AARCH64_NATIVE_H */ diff --git a/fips202/native/aarch64/keccak_f1600_x1_scalar_asm_opt.S b/fips202/native/aarch64/keccak_f1600_x1_scalar_asm_opt.S index 3db9e2046..98e522806 100644 --- a/fips202/native/aarch64/keccak_f1600_x1_scalar_asm_opt.S +++ b/fips202/native/aarch64/keccak_f1600_x1_scalar_asm_opt.S @@ -28,12 +28,12 @@ // Author: Hanno Becker // Author: Matthias Kannwischer -#include "config.h" -#if defined(MLKEM_USE_NATIVE_AARCH64) +#include "common.h" +#if defined(MLKEM_NATIVE_FIPS202_BACKEND_AARCH64_DEFAULT) || \ + defined(MLKEM_NATIVE_FIPS202_BACKEND_AARCH64_A55) // Needed to provide ASM_LOAD directive #include "common.i" -#include "namespace.h" /********************** CONSTANTS *************************/ .data diff --git a/fips202/native/aarch64/keccak_f1600_x1_v84a_asm_clean.S b/fips202/native/aarch64/keccak_f1600_x1_v84a_asm_clean.S index 0613a8a75..c60e4cad4 100644 --- a/fips202/native/aarch64/keccak_f1600_x1_v84a_asm_clean.S +++ b/fips202/native/aarch64/keccak_f1600_x1_v84a_asm_clean.S @@ -37,8 +37,9 @@ // during load and store, so that the caller need not do this. // -#include "config.h" -#if defined(MLKEM_USE_NATIVE_AARCH64) +#include "common.h" +#if defined(MLKEM_NATIVE_FIPS202_BACKEND_AARCH64_DEFAULT) || \ + defined(MLKEM_NATIVE_FIPS202_BACKEND_AARCH64_A55) // Needed to provide ASM_LOAD directive #include "common.i" diff --git a/fips202/native/aarch64/keccak_f1600_x2_v84a_asm_clean.S b/fips202/native/aarch64/keccak_f1600_x2_v84a_asm_clean.S index f74856a5f..45304aa25 100644 --- a/fips202/native/aarch64/keccak_f1600_x2_v84a_asm_clean.S +++ b/fips202/native/aarch64/keccak_f1600_x2_v84a_asm_clean.S @@ -37,8 +37,9 @@ // during load and store, so that the caller need not do this. // -#include "config.h" -#if defined(MLKEM_USE_NATIVE_AARCH64) +#include "common.h" +#if defined(MLKEM_NATIVE_FIPS202_BACKEND_AARCH64_DEFAULT) || \ + defined(MLKEM_NATIVE_FIPS202_BACKEND_AARCH64_A55) // Needed to provide ASM_LOAD directive #include "common.i" diff --git a/fips202/native/aarch64/keccak_f1600_x2_v8a_v84a_asm_hybrid.S b/fips202/native/aarch64/keccak_f1600_x2_v8a_v84a_asm_hybrid.S index b7cc6f261..e4fecb11d 100644 --- a/fips202/native/aarch64/keccak_f1600_x2_v8a_v84a_asm_hybrid.S +++ b/fips202/native/aarch64/keccak_f1600_x2_v8a_v84a_asm_hybrid.S @@ -37,12 +37,12 @@ // during load and store, so that the caller need not do this. // -#include "config.h" -#if defined(MLKEM_USE_NATIVE_AARCH64) +#include "common.h" +#if defined(MLKEM_NATIVE_FIPS202_BACKEND_AARCH64_DEFAULT) || \ + defined(MLKEM_NATIVE_FIPS202_BACKEND_AARCH64_A55) // Needed to provide ASM_LOAD directive #include "common.i" -#include "namespace.h" #if defined(__ARM_FEATURE_SHA3) diff --git a/fips202/native/aarch64/keccak_f1600_x4_scalar_v84a_asm_hybrid_opt.S b/fips202/native/aarch64/keccak_f1600_x4_scalar_v84a_asm_hybrid_opt.S index 19a93b9e8..abf1af497 100644 --- a/fips202/native/aarch64/keccak_f1600_x4_scalar_v84a_asm_hybrid_opt.S +++ b/fips202/native/aarch64/keccak_f1600_x4_scalar_v84a_asm_hybrid_opt.S @@ -24,12 +24,12 @@ * */ -#include "config.h" -#if defined(MLKEM_USE_NATIVE_AARCH64) +#include "common.h" +#if defined(MLKEM_NATIVE_FIPS202_BACKEND_AARCH64_DEFAULT) || \ + defined(MLKEM_NATIVE_FIPS202_BACKEND_AARCH64_A55) // Needed to provide ASM_LOAD directive #include "common.i" -#include "namespace.h" #if defined(__ARM_FEATURE_SHA3) diff --git a/fips202/native/aarch64/keccak_f1600_x4_v8a_scalar_hybrid_asm_opt.S b/fips202/native/aarch64/keccak_f1600_x4_v8a_scalar_hybrid_asm_opt.S index a32375f07..f001d3eaf 100644 --- a/fips202/native/aarch64/keccak_f1600_x4_v8a_scalar_hybrid_asm_opt.S +++ b/fips202/native/aarch64/keccak_f1600_x4_v8a_scalar_hybrid_asm_opt.S @@ -24,12 +24,12 @@ * */ -#include "config.h" -#if defined(MLKEM_USE_NATIVE_AARCH64) +#include "common.h" +#if defined(MLKEM_NATIVE_FIPS202_BACKEND_AARCH64_DEFAULT) || \ + defined(MLKEM_NATIVE_FIPS202_BACKEND_AARCH64_A55) // Needed to provide ASM_LOAD directive #include "common.i" -#include "namespace.h" #define KECCAK_F1600_ROUNDS 24 diff --git a/fips202/native/aarch64/keccak_f1600_x4_v8a_v84a_scalar_hybrid_asm_opt.S b/fips202/native/aarch64/keccak_f1600_x4_v8a_v84a_scalar_hybrid_asm_opt.S index 889ba2983..67a0adccd 100644 --- a/fips202/native/aarch64/keccak_f1600_x4_v8a_v84a_scalar_hybrid_asm_opt.S +++ b/fips202/native/aarch64/keccak_f1600_x4_v8a_v84a_scalar_hybrid_asm_opt.S @@ -24,8 +24,9 @@ * */ -#include "config.h" -#if defined(MLKEM_USE_NATIVE_AARCH64) +#include "common.h" +#if defined(MLKEM_NATIVE_FIPS202_BACKEND_AARCH64_DEFAULT) || \ + defined(MLKEM_NATIVE_FIPS202_BACKEND_AARCH64_A55) // Needed to provide ASM_LOAD directive #include "common.i" diff --git a/fips202/native/aarch64/profiles/cortex_a55.h b/fips202/native/aarch64/profiles/cortex_a55_impl.h similarity index 78% rename from fips202/native/aarch64/profiles/cortex_a55.h rename to fips202/native/aarch64/profiles/cortex_a55_impl.h index 3ca536bf4..304429d11 100644 --- a/fips202/native/aarch64/profiles/cortex_a55.h +++ b/fips202/native/aarch64/profiles/cortex_a55_impl.h @@ -5,12 +5,10 @@ /* FIPS202 assembly profile targeting Cortex-A55 */ -#ifdef FIPS202_NATIVE_PROFILE_H +#ifdef FIPS202_NATIVE_PROFILE_IMPL_H #error Only one FIPS202 assembly profile can be defined -- did you include multiple profiles? #else -#define FIPS202_NATIVE_PROFILE_H - -#include "../fips202_native_aarch64.h" +#define FIPS202_NATIVE_PROFILE_IMPL_H /* * On Cortex-A55, we use lazy rotation assembly for Keccak-x1, @@ -22,4 +20,4 @@ static INLINE void keccak_f1600_x1_native(uint64_t *state) keccak_f1600_x1_scalar_asm_opt(state); } -#endif /* FIPS202_NATIVE_PROFILE_H */ +#endif /* FIPS202_NATIVE_PROFILE_IMPL_H */ diff --git a/fips202/native/aarch64/profiles/cortex_a55_metadata.h b/fips202/native/aarch64/profiles/cortex_a55_metadata.h new file mode 100644 index 000000000..0e2cfc8cf --- /dev/null +++ b/fips202/native/aarch64/profiles/cortex_a55_metadata.h @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2024 The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 + */ + +/* FIPS202 assembly profile targeting Cortex-A55 */ + +#ifdef FIPS202_NATIVE_PROFILE_H +#error Only one FIPS202 assembly profile can be defined -- did you include multiple profiles? +#else +#define FIPS202_NATIVE_PROFILE_H + +/* Identifier for this backend so that source and assembly files + * in the build can be appropriately guarded. */ +#define MLKEM_NATIVE_FIPS202_BACKEND_AARCH64_A55 + +#define MLKEM_NATIVE_FIPS202_BACKEND_NAME AARCH64_A55 + +/* Filename of the C backend implementation. + * This is not inlined here because this header is included in assembly + * files as well. */ +#define MLKEM_NATIVE_FIPS202_BACKEND_IMPL "aarch64/profiles/cortex_a55_impl.h" + +#endif /* FIPS202_NATIVE_PROFILE_H */ diff --git a/fips202/native/aarch64/profiles/default.h b/fips202/native/aarch64/profiles/default_impl.h similarity index 97% rename from fips202/native/aarch64/profiles/default.h rename to fips202/native/aarch64/profiles/default_impl.h index 3a9da28b7..5155b33f8 100644 --- a/fips202/native/aarch64/profiles/default.h +++ b/fips202/native/aarch64/profiles/default_impl.h @@ -5,10 +5,10 @@ /* Default FIPS202 assembly profile for AArch64 systems */ -#ifdef FIPS202_NATIVE_PROFILE_H +#ifdef FIPS202_NATIVE_PROFILE_IMPL_H #error Only one FIPS202 assembly profile can be defined -- did you include multiple profiles? #else -#define FIPS202_NATIVE_PROFILE_H +#define FIPS202_NATIVE_PROFILE_IMPL_H #include "../fips202_native_aarch64.h" diff --git a/fips202/native/aarch64/profiles/default_metadata.h b/fips202/native/aarch64/profiles/default_metadata.h new file mode 100644 index 000000000..eafc01535 --- /dev/null +++ b/fips202/native/aarch64/profiles/default_metadata.h @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2024 The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 + */ + +/* Default FIPS202 assembly profile for AArch64 systems */ + +#ifdef FIPS202_NATIVE_PROFILE_H +#error Only one FIPS202 assembly profile can be defined -- did you include multiple profiles? +#else +#define FIPS202_NATIVE_PROFILE_H + +/* Identifier for this backend so that source and assembly files + * in the build can be appropriately guarded. */ +#define MLKEM_NATIVE_FIPS202_BACKEND_AARCH64_DEFAULT + +#define MLKEM_NATIVE_FIPS202_BACKEND_NAME AARCH64_DEFAULT + +/* Filename of the C backend implementation. + * This is not inlined here because this header is included in assembly + * files as well. */ +#define MLKEM_NATIVE_FIPS202_BACKEND_IMPL "aarch64/profiles/default_impl.h" + +#endif /* FIPS202_NATIVE_PROFILE_H */ diff --git a/fips202/native/default.h b/fips202/native/default.h new file mode 100644 index 000000000..367fb5af4 --- /dev/null +++ b/fips202/native/default.h @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2024 The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef MLKEM_NATIVE_FIPS202_BACKEND_DEFAULT_H +#define MLKEM_NATIVE_FIPS202_BACKEND_DEFAULT_H + +/* + * Default FIPS202 backend + */ +#include "common.h" + +#if defined(SYS_AARCH64) +#include "aarch64/profiles/default_metadata.h" +#endif + +#if defined(SYS_X86_64) && defined(SYS_X86_64_AVX2) +#include "x86_64/profiles/xkcp_metadata.h" +#endif + +#endif /* MLKEM_NATIVE_FIPS202_BACKEND_DEFAULT_H */ diff --git a/fips202/native/fips202_impl.h b/fips202/native/fips202_impl.h new file mode 100644 index 000000000..e1b8a3363 --- /dev/null +++ b/fips202/native/fips202_impl.h @@ -0,0 +1,15 @@ +/* + * Copyright (c) 2024 The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifdef MLKEM_NATIVE_FIPS202_IMPL_H +#error Only one FIPS202 assembly profile can be defined -- did you include multiple profiles? +#else +#define MLKEM_NATIVE_FIPS202_IMPL_H + +#if defined(MLKEM_NATIVE_FIPS202_BACKEND_IMPL) +#include MLKEM_NATIVE_FIPS202_BACKEND_IMPL +#endif + +#endif /* MLKEM_NATIVE_FIPS202_IMPL_H */ diff --git a/fips202/native/fips202_native.h b/fips202/native/fips202_native.h index ccddc32a9..685b8bfc5 100644 --- a/fips202/native/fips202_native.h +++ b/fips202/native/fips202_native.h @@ -6,54 +6,7 @@ #define FIPS202_NATIVE_H #include -#include "config.h" -#include "params.h" - -#if defined(MLKEM_USE_NATIVE) - -/* - * FIPS202 native profile - * - * The profile decides which implementation(s) of FIPS202 to use. - * - * If you don't change anything, the default profile will be used. This profile - * picks implementations based on characteristics of your system visible to - * the compiler. - * - * The default logic is not perfect, and you may want to pick a specific - * profile for your target. There are three ways to do so, in descending - * order of convenience to the user: - * 1. Pick one of the profiles shipped with this repository. - * 2. Provide your own profile and register it via FIPS202_ASM_PROFILE - * (which must be the profile's path relative to this directoru). - * 3. Set FIPS202_NATIVE_MANUAL and use an adhoc profile specified via CFLAGS. - */ - -/* Option 2: Manually written profile */ -#if defined(FIPS202_NATIVE_PROFILE) - -#define STRINGIFY_(x) #x -#define STRINGIFY(x) STRINGIFY_(x) -#include STRINGIFY(FIPS202_NATIVE_PROFILE) - -/* Option 1: Choose from shipped list of profiles */ -#elif !defined(FIPS202_NATIVE_MANUAL) - -#ifdef SYS_AARCH64 -/* Pick exactly one profile from the following list */ -#include "aarch64/profiles/default.h" -/* #include "aarch64/profiles/cortex_a55.h" */ -#endif - -#if defined(SYS_X86_64) && defined(SYS_X86_64_AVX2) -#include "x86_64/profiles/xkcp.h" -#endif - -#else /* !FIPS202_NATIVE_PROFILE && FIPS202_NATIVE_MANUAL */ - -/* Option 3: Build your own profile here, or via CFLAGS */ - -#endif /* !FIPS202_NATIVE_PROFILE && !FIPS202_NATIVE_MANUAL */ +#include "common.h" /* * FIPS202 native interface @@ -77,5 +30,4 @@ static INLINE void keccak_f1600_x2_native(uint64_t *state); static INLINE void keccak_f1600_x4_native(uint64_t *state); #endif -#endif /* MLKEM_USE_NATIVE */ #endif /* FIPS202_NATIVE_H */ diff --git a/fips202/native/x86_64/fips202_native_x86_64.h b/fips202/native/x86_64/fips202_native_x86_64.h deleted file mode 100644 index cbc0bb97d..000000000 --- a/fips202/native/x86_64/fips202_native_x86_64.h +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright (c) 2024 The mlkem-native project authors - * SPDX-License-Identifier: Apache-2.0 - */ -#ifndef FIPS202_X86_64_NATIVE_H -#define FIPS202_X86_64_NATIVE_H - -#include "config.h" - -#ifdef MLKEM_USE_NATIVE_X86_64 - -#include -#include -#include - -#include "xkcp/KeccakP-1600-times4-SnP.h" - -#endif /* MLKEM_USE_NATIVE_X86_64 */ - -#endif /* FIPS202_X86_64_NATIVE_H */ diff --git a/fips202/native/x86_64/profiles/xkcp.h b/fips202/native/x86_64/profiles/xkcp_impl.h similarity index 60% rename from fips202/native/x86_64/profiles/xkcp.h rename to fips202/native/x86_64/profiles/xkcp_impl.h index 4f320e994..265404179 100644 --- a/fips202/native/x86_64/profiles/xkcp.h +++ b/fips202/native/x86_64/profiles/xkcp_impl.h @@ -5,14 +5,12 @@ /* Default FIPS202 assembly profile for AArch64 systems */ -#ifdef FIPS202_NATIVE_PROFILE_H +#ifdef MLKEM_NATIVE_FIPS202_PROFILE_IMPL_H #error Only one FIPS202 assembly profile can be defined -- did you include multiple profiles? #else -#define FIPS202_NATIVE_PROFILE_H +#define MLKEM_NATIVE_FIPS202_PROFILE_IMPL_H -#include "../fips202_native_x86_64.h" - -#if defined(MLKEM_USE_NATIVE_X86_64) && defined(SYS_X86_64_AVX2) +#include "../xkcp/KeccakP-1600-times4-SnP.h" #define MLKEM_USE_FIPS202_X4_NATIVE static INLINE void keccak_f1600_x4_native(uint64_t *state) @@ -20,6 +18,4 @@ static INLINE void keccak_f1600_x4_native(uint64_t *state) KeccakP1600times4_PermuteAll_24rounds(state); } -#endif /* MLKEM_USE_NATIVE_X86_64 && SYS_X86_64_AVX2 */ - -#endif /* FIPS202_NATIVE_PROFILE_H */ +#endif /* MLKEM_NATIVE_FIPS202_PROFILE_IMPL_H */ diff --git a/fips202/native/x86_64/profiles/xkcp_metadata.h b/fips202/native/x86_64/profiles/xkcp_metadata.h new file mode 100644 index 000000000..ab7e20b8d --- /dev/null +++ b/fips202/native/x86_64/profiles/xkcp_metadata.h @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2024 The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 + */ + +/* Default FIPS202 assembly profile for AArch64 systems */ + +#ifdef MLKEM_NATIVE_FIPS202_PROFILE_H +#error Only one FIPS202 assembly profile can be defined -- did you include multiple profiles? +#else +#define MLKEM_NATIVE_FIPS202_PROFILE_H + +/* Identifier for this backend so that source and assembly files + * in the build can be appropriately guarded. */ +#define MLKEM_NATIVE_FIPS202_BACKEND_X86_64_XKCP + +#define MLKEM_NATIVE_FIPS202_BACKEND_NAME X86_64_XKCP + +/* Filename of the C backend implementation. + * This is not inlined here because this header is included in assembly + * files as well. */ +#define MLKEM_NATIVE_FIPS202_BACKEND_IMPL "x86_64/profiles/xkcp_impl.h" + +#endif /* MLKEM_NATIVE_FIPS202_PROFILE_H */ diff --git a/fips202/native/x86_64/xkcp/KeccakP-1600-times4-SIMD256.c b/fips202/native/x86_64/xkcp/KeccakP-1600-times4-SIMD256.c index 9a36f17aa..6cf34f76d 100644 --- a/fips202/native/x86_64/xkcp/KeccakP-1600-times4-SIMD256.c +++ b/fips202/native/x86_64/xkcp/KeccakP-1600-times4-SIMD256.c @@ -24,8 +24,8 @@ and related or neighboring rights to the source code in this file. * Keccak states in memory. */ -#include "config.h" -#if defined(MLKEM_USE_NATIVE_X86_64) && defined(SYS_X86_64_AVX2) +#include "common.h" +#if defined(MLKEM_NATIVE_FIPS202_BACKEND_X86_64_XKCP) #include #include @@ -445,10 +445,9 @@ void KeccakP1600times4_PermuteAll_24rounds(void *states) } #else -#include "params.h" /* Dummy constant to keep compiler happy despite empty CU */ -#define empty_cu_avx2_keccakx4 MLKEM_NAMESPACE(empty_cu_avx2_keccakx4) +#define empty_cu_avx2_keccakx4 FIPS202_NAMESPACE(empty_cu_avx2_keccakx4) int empty_cu_avx2_keccakx4; #endif /* MLKEM_USE_NATIVE_X86_64 && SYS_X86_64_AVX2 */ diff --git a/fips202/native/x86_64/xkcp/KeccakP-1600-times4-SnP.h b/fips202/native/x86_64/xkcp/KeccakP-1600-times4-SnP.h index 6d1951787..0738c9488 100644 --- a/fips202/native/x86_64/xkcp/KeccakP-1600-times4-SnP.h +++ b/fips202/native/x86_64/xkcp/KeccakP-1600-times4-SnP.h @@ -26,7 +26,7 @@ and related or neighboring rights to the source code in this file. #include #include "KeccakP-SIMD256-config.h" -#include "namespace.h" +#include "common.h" #define KeccakP1600times4_statesAlignment 32 diff --git a/mlkem/api.h b/mlkem/api.h deleted file mode 100644 index 94597323f..000000000 --- a/mlkem/api.h +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright (c) 2024 The mlkem-native project authors - * SPDX-License-Identifier: Apache-2.0 - */ -#ifndef API_H -#define API_H - -#include - -#define PQCP_MLKEM_NATIVE_MLKEM512_SECRETKEYBYTES 1632 -#define PQCP_MLKEM_NATIVE_MLKEM512_PUBLICKEYBYTES 800 -#define PQCP_MLKEM_NATIVE_MLKEM512_CIPHERTEXTBYTES 768 -#define PQCP_MLKEM_NATIVE_MLKEM512_KEYPAIRCOINBYTES 64 -#define PQCP_MLKEM_NATIVE_MLKEM512_ENCCOINBYTES 32 -#define PQCP_MLKEM_NATIVE_MLKEM512_BYTES 32 - -int PQCP_MLKEM_NATIVE_MLKEM512_keypair_derand(uint8_t *pk, uint8_t *sk, - const uint8_t *coins); -int PQCP_MLKEM_NATIVE_MLKEM512_keypair(uint8_t *pk, uint8_t *sk); -int PQCP_MLKEM_NATIVE_MLKEM512_enc_derand(uint8_t *ct, uint8_t *ss, - const uint8_t *pk, - const uint8_t *coins); -int PQCP_MLKEM_NATIVE_MLKEM512_enc(uint8_t *ct, uint8_t *ss, const uint8_t *pk); -int PQCP_MLKEM_NATIVE_MLKEM512_dec(uint8_t *ss, const uint8_t *ct, - const uint8_t *sk); - -#define PQCP_MLKEM_NATIVE_MLKEM768_SECRETKEYBYTES 2400 -#define PQCP_MLKEM_NATIVE_MLKEM768_PUBLICKEYBYTES 1184 -#define PQCP_MLKEM_NATIVE_MLKEM768_CIPHERTEXTBYTES 1088 -#define PQCP_MLKEM_NATIVE_MLKEM768_KEYPAIRCOINBYTES 64 -#define PQCP_MLKEM_NATIVE_MLKEM768_ENCCOINBYTES 32 -#define PQCP_MLKEM_NATIVE_MLKEM768_BYTES 32 - -int PQCP_MLKEM_NATIVE_MLKEM768_keypair_derand(uint8_t *pk, uint8_t *sk, - const uint8_t *coins); -int PQCP_MLKEM_NATIVE_MLKEM768_keypair(uint8_t *pk, uint8_t *sk); -int PQCP_MLKEM_NATIVE_MLKEM768_enc_derand(uint8_t *ct, uint8_t *ss, - const uint8_t *pk, - const uint8_t *coins); -int PQCP_MLKEM_NATIVE_MLKEM768_enc(uint8_t *ct, uint8_t *ss, const uint8_t *pk); -int PQCP_MLKEM_NATIVE_MLKEM768_dec(uint8_t *ss, const uint8_t *ct, - const uint8_t *sk); - -#define PQCP_MLKEM_NATIVE_MLKEM1024_SECRETKEYBYTES 3168 -#define PQCP_MLKEM_NATIVE_MLKEM1024_PUBLICKEYBYTES 1568 -#define PQCP_MLKEM_NATIVE_MLKEM1024_CIPHERTEXTBYTES 1568 -#define PQCP_MLKEM_NATIVE_MLKEM1024_KEYPAIRCOINBYTES 64 -#define PQCP_MLKEM_NATIVE_MLKEM1024_ENCCOINBYTES 32 -#define PQCP_MLKEM_NATIVE_MLKEM1024_BYTES 32 - -int PQCP_MLKEM_NATIVE_MLKEM1024_keypair_derand(uint8_t *pk, uint8_t *sk, - const uint8_t *coins); -int PQCP_MLKEM_NATIVE_MLKEM1024_keypair(uint8_t *pk, uint8_t *sk); -int PQCP_MLKEM_NATIVE_MLKEM1024_enc_derand(uint8_t *ct, uint8_t *ss, - const uint8_t *pk, - const uint8_t *coins); -int PQCP_MLKEM_NATIVE_MLKEM1024_enc(uint8_t *ct, uint8_t *ss, - const uint8_t *pk); -int PQCP_MLKEM_NATIVE_MLKEM1024_dec(uint8_t *ss, const uint8_t *ct, - const uint8_t *sk); - -#endif diff --git a/mlkem/cbd.c b/mlkem/cbd.c index 073f3c81d..2e0fac38a 100644 --- a/mlkem/cbd.c +++ b/mlkem/cbd.c @@ -4,7 +4,6 @@ */ #include "cbd.h" #include -#include "params.h" /************************************************* * Name: load32_littleendian diff --git a/mlkem/cbd.h b/mlkem/cbd.h index 4dc8635bb..31c9649e3 100644 --- a/mlkem/cbd.h +++ b/mlkem/cbd.h @@ -6,7 +6,7 @@ #define CBD_H #include -#include "params.h" +#include "common.h" #include "poly.h" #define poly_cbd_eta1 MLKEM_NAMESPACE(poly_cbd_eta1) diff --git a/mlkem/common.h b/mlkem/common.h index 94c29ed92..0b379d9bd 100644 --- a/mlkem/common.h +++ b/mlkem/common.h @@ -2,64 +2,21 @@ * Copyright (c) 2024 The mlkem-native project authors * SPDX-License-Identifier: Apache-2.0 */ -#ifndef COMMON_H -#define COMMON_H +#ifndef MLKEM_NATIVE_COMMON_H +#define MLKEM_NATIVE_COMMON_H +#include "config.h" +#include "params.h" +#include "sys.h" -/* - * C90 does not have the inline compiler directive yet. - * We don't use it in C90 builds. - * However, in that case the compiler warns about some inline functions in - * header files not being used in every compilation unit that includes that - * header. To work around it we silence that warning in that case using - * __attribute__((unused)). - */ - -/* Do not use inline for C90 builds*/ -#if !defined(inline) -#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L -#define INLINE inline -#define ALWAYS_INLINE __attribute__((always_inline)) -#elif defined(_MSC_VER) -#define INLINE __inline -#define ALWAYS_INLINE __forceinline -#else -#define INLINE __attribute__((unused)) -#define ALWAYS_INLINE -#endif - -#else -#define INLINE inline -#define ALWAYS_INLINE __attribute__((always_inline)) -#endif - - -/* - * C90 does not have the restrict compiler directive yet. - * We don't use it in C90 builds. - */ -#if !defined(restrict) -#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L -#define RESTRICT restrict -#else -#define RESTRICT -#endif - -#else - -#define RESTRICT restrict -#endif - -#define DEFAULT_ALIGN 32 -#if defined(_WIN32) -#define ALIGN __declspec(align(DEFAULT_ALIGN)) -#define asm __asm -#else -#define asm __asm__ -#define ALIGN __attribute__((aligned(DEFAULT_ALIGN))) +/* Include backend metadata */ +#if defined(MLKEM_USE_NATIVE) +#include MLKEM_NATIVE_ARITH_BACKEND +#include MLKEM_NATIVE_FIPS202_BACKEND #endif -#define MLKEM_CONCAT_(left, right) left##right -#define MLKEM_CONCAT(left, right) MLKEM_CONCAT_(left, right) +/* This must come after the inclusion of the backend metadata + * since the backend choice may be part of the namespace. */ +#include "namespace.h" -#endif +#endif /* MLKEM_NATIVE_COMMON_H */ diff --git a/mlkem/config.h b/mlkem/config.h new file mode 100644 index 000000000..b4de97188 --- /dev/null +++ b/mlkem/config.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2024 The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef MLKEM_NATIVE_CONFIG_H +#define MLKEM_NATIVE_CONFIG_H + +#ifndef MLKEM_K +#define MLKEM_K 3 /* Change this for different security strengths */ +#endif + +/* Change this to use a custom namespaces for your mlkem-native build */ +#define MLKEM_NAMESPACE(sym) MLKEM_DEFAULT_NAMESPACE(sym) +#define _MLKEM_NAMESPACE(sym) _MLKEM_DEFAULT_NAMESPACE(sym) +#define FIPS202_NAMESPACE(sym) FIPS202_DEFAULT_NAMESPACE(sym) +#define _FIPS202_NAMESPACE(sym) _FIPS202_DEFAULT_NAMESPACE(sym) + +/* If you don't want to use a native backend, unset this */ +#if !defined(MLKEM_USE_NATIVE) +/* #define MLKEM_USE_NATIVE */ +#endif + +/* Arithmetic backend */ +#if !defined(MLKEM_NATIVE_ARITH_BACKEND) +#define MLKEM_NATIVE_ARITH_BACKEND "native/default.h" +#endif /* MLKEM_NATIVE_ARITH_BACKEND */ + +/* FIPS202 backend */ +#if !defined(MLKEM_NATIVE_FIPS202_BACKEND) +#define MLKEM_NATIVE_FIPS202_BACKEND "../fips202/native/default.h" +#endif /* MLKEM_NATIVE_FIPS202_BACKEND */ + +#endif /* MLkEM_NATIVE_CONFIG_H */ diff --git a/mlkem/debug/debug.c b/mlkem/debug/debug.c index 0dc2e8822..690977908 100644 --- a/mlkem/debug/debug.c +++ b/mlkem/debug/debug.c @@ -52,7 +52,6 @@ void mlkem_debug_print_error(const char *file, int line, const char *msg) } #else /* MLKEM_DEBUG */ -#include "params.h" #define empty_cu_debug MLKEM_NAMESPACE(empty_cu_debug) int empty_cu_debug; diff --git a/mlkem/debug/debug.h b/mlkem/debug/debug.h index 65208771d..5838ae4bf 100644 --- a/mlkem/debug/debug.h +++ b/mlkem/debug/debug.h @@ -5,6 +5,8 @@ #ifndef MLKEM_DEBUG_H #define MLKEM_DEBUG_H +#include "common.h" + #if defined(MLKEM_DEBUG) #include #include @@ -153,6 +155,9 @@ void mlkem_debug_print_error(const char *file, int line, const char *msg); "polyvec unsigned bound for " #ptr ".vec[i]"); \ } while (0) +#define MLKEM_CONCAT_(left, right) left##right +#define MLKEM_CONCAT(left, right) MLKEM_CONCAT_(left, right) + /* Following AWS-LC to define a C99-compliant static assert */ #define MLKEM_STATIC_ASSERT_DEFINE(cond, msg) \ typedef struct \ diff --git a/mlkem/indcpa.c b/mlkem/indcpa.c index bde832c18..5b5d6c6fa 100644 --- a/mlkem/indcpa.c +++ b/mlkem/indcpa.c @@ -10,7 +10,6 @@ #include "fips202x4.h" #include "indcpa.h" #include "ntt.h" -#include "params.h" #include "poly.h" #include "polyvec.h" #include "randombytes.h" diff --git a/mlkem/indcpa.h b/mlkem/indcpa.h index 0e6485312..7e2a0b247 100644 --- a/mlkem/indcpa.h +++ b/mlkem/indcpa.h @@ -7,10 +7,9 @@ #include #include "cbmc.h" -#include "params.h" +#include "common.h" #include "polyvec.h" - #define gen_matrix MLKEM_NAMESPACE(gen_matrix) /************************************************* * Name: gen_matrix diff --git a/mlkem/kem.c b/mlkem/kem.c index 1abce65e4..03e997af3 100644 --- a/mlkem/kem.c +++ b/mlkem/kem.c @@ -7,7 +7,6 @@ #include #include #include "indcpa.h" -#include "params.h" #include "randombytes.h" #include "symmetric.h" #include "verify.h" diff --git a/mlkem/namespace.h b/mlkem/namespace.h new file mode 100644 index 000000000..8c409fb0c --- /dev/null +++ b/mlkem/namespace.h @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2024 The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 + */ +#ifndef MLKEM_NATIVE_NAMESPACE_H +#define MLKEM_NATIVE_NAMESPACE_H + +#if !defined(MLKEM_NATIVE_ARITH_BACKEND_NAME) +#define MLKEM_NATIVE_ARITH_BACKEND_NAME C +#endif + +/* Don't change parameters below this line */ +#if (MLKEM_K == 2) +#define MLKEM_PARAM_NAME MLKEM512 +#elif (MLKEM_K == 3) +#define MLKEM_PARAM_NAME MLKEM768 +#elif (MLKEM_K == 4) +#define MLKEM_PARAM_NAME MLKEM1024 +#else +#error "MLKEM_K must be in {2,3,4}" +#endif + +#define ___MLKEM_DEFAULT_NAMESPACE(x1, x2, x3, x4) x1##_##x2##_##x3##_##x4 +#define __MLKEM_DEFAULT_NAMESPACE(x1, x2, x3, x4) \ + ___MLKEM_DEFAULT_NAMESPACE(x1, x2, x3, x4) + +/* + * NAMESPACE is PQCP_MLKEM_NATIVE___ + * e.g., PQCP_MLKEM_NATIVE_MLKEM512_AARCH64_OPT_ + */ +#define MLKEM_DEFAULT_NAMESPACE(s) \ + __MLKEM_DEFAULT_NAMESPACE(PQCP_MLKEM_NATIVE, MLKEM_PARAM_NAME, \ + MLKEM_NATIVE_ARITH_BACKEND_NAME, s) +#define _MLKEM_DEFAULT_NAMESPACE(s) \ + __MLKEM_DEFAULT_NAMESPACE(_PQCP_MLKEM_NATIVE, MLKEM_PARAM_NAME, \ + MLKEM_NATIVE_ARITH_BACKEND_NAME, s) + +#if !defined(MLKEM_NATIVE_FIPS202_BACKEND_NAME) +#define MLKEM_NATIVE_FIPS202_BACKEND_NAME C +#endif + +#define ___FIPS202_DEFAULT_NAMESPACE(x1, x2, x3) x1##_##x2##_##x3 +#define __FIPS202_DEFAULT_NAMESPACE(x1, x2, x3) \ + ___FIPS202_DEFAULT_NAMESPACE(x1, x2, x3) + +/* + * NAMESPACE is PQCP_MLKEM_NATIVE_FIPS202__ + * e.g., PQCP_MLKEM_NATIVE_FIPS202_X86_64_XKCP_ + */ +#define FIPS202_DEFAULT_NAMESPACE(s) \ + __FIPS202_DEFAULT_NAMESPACE(PQCP_MLKEM_NATIVE_FIPS202, \ + MLKEM_NATIVE_FIPS202_BACKEND_NAME, s) +#define _FIPS202_DEFAULT_NAMESPACE(s) \ + __FIPS202_DEFAULT_NAMESPACE(_PQCP_MLKEM_NATIVE_FIPS202, \ + MLKEM_NATIVE_FIPS202_BACKEND_NAME, s) + +#endif /* MLKEM_NATIVE_NAMESPACE_H */ diff --git a/mlkem/native/aarch64/aarch64_zetas.c b/mlkem/native/aarch64/aarch64_zetas.c index f0a0d5546..ecf1b529a 100644 --- a/mlkem/native/aarch64/aarch64_zetas.c +++ b/mlkem/native/aarch64/aarch64_zetas.c @@ -8,9 +8,13 @@ * Do not modify it directly. */ -#include "arith_native_aarch64.h" +#include "common.h" + +#if defined(MLKEM_NATIVE_ARITH_BACKEND_AARCH64_CLEAN) || \ + defined(MLKEM_NATIVE_ARITH_BACKEND_AARCH64_OPT) -#ifdef MLKEM_USE_NATIVE_AARCH64 +#include +#include "arith_native_aarch64.h" /* * Table of zeta values used in the AArch64 forward NTT @@ -163,10 +167,10 @@ const int16_t aarch64_zetas_mulcache_twisted_native[] = { -11566, 11566, }; -#else /* MLKEM_USE_NATIVE_AARCH64 */ +#else #include "params.h" /* Dummy declaration for compilers disliking empty compilation units */ #define empty_cu_aarch64_zetas MLKEM_NAMESPACE(empty_cu_aarch64_zetas) int empty_cu_aarch64_zetas; -#endif /* MLKEM_USE_NATIVE_AARCH64 */ +#endif diff --git a/mlkem/native/aarch64/arith_native_aarch64.h b/mlkem/native/aarch64/arith_native_aarch64.h index f39eb9ac8..e921cc8de 100644 --- a/mlkem/native/aarch64/arith_native_aarch64.h +++ b/mlkem/native/aarch64/arith_native_aarch64.h @@ -6,10 +6,7 @@ #define MLKEM_AARCH64_NATIVE_H #include -#include "config.h" -#include "params.h" - -#ifdef MLKEM_USE_NATIVE_AARCH64 +#include "common.h" #define aarch64_ntt_zetas_layer01234 \ MLKEM_NAMESPACE(aarch64_ntt_zetas_layer01234) @@ -85,6 +82,4 @@ void polyvec_basemul_acc_montgomery_cached_asm_opt(int16_t *r, const int16_t *a, const int16_t *b, const int16_t *b_cache); - -#endif /* MLKEM_USE_NATIVE_AARCH64 */ #endif /* MLKEM_AARCH64_NATIVE_H */ diff --git a/mlkem/native/aarch64/consts.h b/mlkem/native/aarch64/consts.h index 0e834bed4..c40947299 100644 --- a/mlkem/native/aarch64/consts.h +++ b/mlkem/native/aarch64/consts.h @@ -7,7 +7,7 @@ #define MLKEM_NATIVE_AARCH64_CONSTS #include -#include "params.h" +#include "common.h" #define zetas_mulcache_native MLKEM_NAMESPACE(zetas_mulcache_native) extern const int16_t zetas_mulcache_native[256]; diff --git a/mlkem/native/aarch64/intt_clean.S b/mlkem/native/aarch64/intt_clean.S index 2ac4e0f15..40f407138 100644 --- a/mlkem/native/aarch64/intt_clean.S +++ b/mlkem/native/aarch64/intt_clean.S @@ -23,8 +23,8 @@ /// SOFTWARE. /// -#include "config.h" -#if defined(MLKEM_USE_NATIVE_AARCH64) +#include "common.h" +#if defined(MLKEM_NATIVE_ARITH_BACKEND_AARCH64_CLEAN) // Needed to provide ASM_LOAD directive #include "common.i" diff --git a/mlkem/native/aarch64/intt_opt.S b/mlkem/native/aarch64/intt_opt.S index 48c412f36..c4a2a8a72 100644 --- a/mlkem/native/aarch64/intt_opt.S +++ b/mlkem/native/aarch64/intt_opt.S @@ -23,8 +23,8 @@ /// SOFTWARE. /// -#include "config.h" -#if defined(MLKEM_USE_NATIVE_AARCH64) +#include "common.h" +#if defined(MLKEM_NATIVE_ARITH_BACKEND_AARCH64_OPT) // Needed to provide ASM_LOAD directive #include "common.i" diff --git a/mlkem/native/aarch64/ntt_clean.S b/mlkem/native/aarch64/ntt_clean.S index 4243b9431..f2e1de37e 100644 --- a/mlkem/native/aarch64/ntt_clean.S +++ b/mlkem/native/aarch64/ntt_clean.S @@ -24,8 +24,8 @@ /// SOFTWARE. /// -#include "config.h" -#if defined(MLKEM_USE_NATIVE_AARCH64) +#include "common.h" +#if defined(MLKEM_NATIVE_ARITH_BACKEND_AARCH64_CLEAN) // Needed to provide ASM_LOAD directive #include "common.i" diff --git a/mlkem/native/aarch64/ntt_opt.S b/mlkem/native/aarch64/ntt_opt.S index 71779afe9..31c21ce5f 100644 --- a/mlkem/native/aarch64/ntt_opt.S +++ b/mlkem/native/aarch64/ntt_opt.S @@ -24,8 +24,8 @@ /// SOFTWARE. /// -#include "config.h" -#if defined(MLKEM_USE_NATIVE_AARCH64) +#include "common.h" +#if defined(MLKEM_NATIVE_ARITH_BACKEND_AARCH64_OPT) // Needed to provide ASM_LOAD directive #include "common.i" diff --git a/mlkem/native/aarch64/poly_clean.S b/mlkem/native/aarch64/poly_clean.S index 86e4e494a..470013a8f 100644 --- a/mlkem/native/aarch64/poly_clean.S +++ b/mlkem/native/aarch64/poly_clean.S @@ -3,8 +3,8 @@ * SPDX-License-Identifier: Apache-2.0 */ -#include "config.h" -#if defined(MLKEM_USE_NATIVE_AARCH64) +#include "common.h" +#if defined(MLKEM_NATIVE_ARITH_BACKEND_AARCH64_CLEAN) // Needed to provide ASM_LOAD directive #include "common.i" diff --git a/mlkem/native/aarch64/poly_opt.S b/mlkem/native/aarch64/poly_opt.S index d86595059..c47c5453a 100644 --- a/mlkem/native/aarch64/poly_opt.S +++ b/mlkem/native/aarch64/poly_opt.S @@ -3,8 +3,8 @@ * SPDX-License-Identifier: Apache-2.0 */ -#include "config.h" -#if defined(MLKEM_USE_NATIVE_AARCH64) +#include "common.h" +#if defined(MLKEM_NATIVE_ARITH_BACKEND_AARCH64_OPT) // Needed to provide ASM_LOAD directive #include "common.i" diff --git a/mlkem/native/aarch64/polyvec_clean.S b/mlkem/native/aarch64/polyvec_clean.S index 59f57d77a..f14f28b80 100644 --- a/mlkem/native/aarch64/polyvec_clean.S +++ b/mlkem/native/aarch64/polyvec_clean.S @@ -9,8 +9,8 @@ // https://eprint.iacr.org/2021/986 // https://github.com/neon-ntt/neon-ntt -#include "config.h" -#if defined(MLKEM_USE_NATIVE_AARCH64) +#include "common.h" +#if defined(MLKEM_NATIVE_ARITH_BACKEND_AARCH64_CLEAN) #include "params.h" diff --git a/mlkem/native/aarch64/polyvec_opt.S b/mlkem/native/aarch64/polyvec_opt.S index 9ae035639..c91096ce4 100644 --- a/mlkem/native/aarch64/polyvec_opt.S +++ b/mlkem/native/aarch64/polyvec_opt.S @@ -6,11 +6,11 @@ // AArch64 re-implementation of the asymmetric base multiplication from: // Neon NTT: Faster Dilithium, Kyber, and Saber on Cortex-A72 and Apple M1 -// https: // eprint.iacr.org/2021/986 -// https: // github.com/neon-ntt/neon-ntt +// https://eprint.iacr.org/2021/986 +// https://github.com/neon-ntt/neon-ntt -#include "config.h" -#if defined(MLKEM_USE_NATIVE_AARCH64) +#include "common.h" +#if defined(MLKEM_NATIVE_ARITH_BACKEND_AARCH64_OPT) #include "params.h" diff --git a/mlkem/native/aarch64/profiles/clean.h b/mlkem/native/aarch64/profiles/clean_impl.h similarity index 90% rename from mlkem/native/aarch64/profiles/clean.h rename to mlkem/native/aarch64/profiles/clean_impl.h index e7f71bd53..e59e5485c 100644 --- a/mlkem/native/aarch64/profiles/clean.h +++ b/mlkem/native/aarch64/profiles/clean_impl.h @@ -5,14 +5,12 @@ /* ML-KEM arithmetic native profile for clean assembly */ -#ifdef MLKEM_ARITH_NATIVE_PROFILE_H +#ifdef MLKEM_NATIVE_ARITH_PROFILE_IMPL_H #error Only one MLKEM_ARITH assembly profile can be defined -- did you include multiple profiles? #else -#define MLKEM_ARITH_NATIVE_PROFILE_H - -#include "../../arith_native.h" -#include "../arith_native_aarch64.h" +#define MLKEM_NATIVE_ARITH_PROFILE_IMPL_H +/* Set of primitives that this backend replaces */ #define MLKEM_USE_NATIVE_NTT #define MLKEM_USE_NATIVE_INTT #define MLKEM_USE_NATIVE_POLY_REDUCE @@ -22,7 +20,6 @@ #define MLKEM_USE_NATIVE_POLY_TOBYTES #define MLKEM_USE_NATIVE_REJ_UNIFORM -#define NTT_BOUND_NATIVE (6 * MLKEM_Q) static INLINE void ntt_native(poly *data) { ntt_asm_clean(data->coeffs, aarch64_ntt_zetas_layer01234, @@ -75,4 +72,4 @@ static INLINE int rej_uniform_native(int16_t *r, unsigned int len, return (int)rej_uniform_asm_clean(r, buf, buflen); } -#endif /* MLKEM_ARITH_NATIVE_PROFILE_H */ +#endif /* MLKEM_NATIVE_ARITH_PROFILE_IMPL_H */ diff --git a/mlkem/native/aarch64/profiles/clean_metadata.h b/mlkem/native/aarch64/profiles/clean_metadata.h new file mode 100644 index 000000000..cde9d50e8 --- /dev/null +++ b/mlkem/native/aarch64/profiles/clean_metadata.h @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2024 The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 + */ + +/* ML-KEM arithmetic native profile for clean assembly */ + +#ifdef MLKEM_NATIVE_ARITH_PROFILE_H +#error Only one MLKEM_ARITH assembly profile can be defined -- did you include multiple profiles? +#else +#define MLKEM_NATIVE_ARITH_PROFILE_H + +/* Identifier for this backend so that source and assembly files + * in the build can be appropriately guarded. */ +#define MLKEM_NATIVE_ARITH_BACKEND_AARCH64_CLEAN + +#define MLKEM_NATIVE_ARITH_BACKEND_NAME AARCH64_CLEAN + +/* Filename of the C backend implementation. + * This is not inlined here because this header is included in assembly + * files as well. */ +#define MLKEM_NATIVE_ARITH_BACKEND_IMPL "aarch64/profiles/clean_impl.h" + +#endif /* MLKEM_NATIVE_ARITH_PROFILE_H */ diff --git a/mlkem/native/aarch64/profiles/opt.h b/mlkem/native/aarch64/profiles/opt_impl.h similarity index 92% rename from mlkem/native/aarch64/profiles/opt.h rename to mlkem/native/aarch64/profiles/opt_impl.h index 2414700bf..599f44e19 100644 --- a/mlkem/native/aarch64/profiles/opt.h +++ b/mlkem/native/aarch64/profiles/opt_impl.h @@ -5,14 +5,12 @@ /* ML-KEM arithmetic native profile for clean assembly */ -#ifdef MLKEM_ARITH_NATIVE_PROFILE_H +#ifdef MLKEM_NATIVE_ARITH_PROFILE_IMPL_H #error Only one MLKEM_ARITH assembly profile can be defined -- did you include multiple profiles? #else -#define MLKEM_ARITH_NATIVE_PROFILE_H - -#include "../../arith_native.h" -#include "../arith_native_aarch64.h" +#define MLKEM_NATIVE_ARITH_PROFILE_IMPL_H +/* Set of primitives that this backend replaces */ #define MLKEM_USE_NATIVE_NTT #define MLKEM_USE_NATIVE_INTT #define MLKEM_USE_NATIVE_POLY_REDUCE @@ -75,4 +73,4 @@ static INLINE int rej_uniform_native(int16_t *r, unsigned int len, return (int)rej_uniform_asm_clean(r, buf, buflen); } -#endif /* MLKEM_ARITH_NATIVE_PROFILE_H */ +#endif /* MLKEM_NATIVE_ARITH_PROFILE_IMPL_H */ diff --git a/mlkem/native/aarch64/profiles/opt_metadata.h b/mlkem/native/aarch64/profiles/opt_metadata.h new file mode 100644 index 000000000..40a644fc2 --- /dev/null +++ b/mlkem/native/aarch64/profiles/opt_metadata.h @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2024 The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 + */ + +/* ML-KEM arithmetic native profile for clean assembly */ + +#ifdef MLKEM_NATIVE_ARITH_PROFILE_H +#error Only one MLKEM_ARITH assembly profile can be defined -- did you include multiple profiles? +#else +#define MLKEM_NATIVE_ARITH_PROFILE_H + +/* Identifier for this backend so that source and assembly files + * in the build can be appropriately guarded. */ +#define MLKEM_NATIVE_ARITH_BACKEND_AARCH64_OPT + +#define MLKEM_NATIVE_ARITH_BACKEND_NAME AARCH64_OPT + +/* Filename of the C backend implementation. + * This is not inlined here because this header is included in assembly + * files as well. */ +#define MLKEM_NATIVE_ARITH_BACKEND_IMPL "aarch64/profiles/opt_impl.h" + +#endif /* MLKEM_NATIVE_ARITH_PROFILE_H */ diff --git a/mlkem/native/aarch64/rej_uniform_asm_clean.S b/mlkem/native/aarch64/rej_uniform_asm_clean.S index 00332ec65..421509dc5 100644 --- a/mlkem/native/aarch64/rej_uniform_asm_clean.S +++ b/mlkem/native/aarch64/rej_uniform_asm_clean.S @@ -18,9 +18,9 @@ * * Returns number of sampled 16-bit integers (at most MLKEM_N). **************************************************/ -#include "config.h" -#include "params.h" -#if defined(MLKEM_USE_NATIVE_AARCH64) +#include "common.h" +#if defined(MLKEM_NATIVE_ARITH_BACKEND_AARCH64_CLEAN) || \ + defined(MLKEM_NATIVE_ARITH_BACKEND_AARCH64_OPT) // Needed to provide ASM_LOAD directive #include "common.i" diff --git a/mlkem/native/arith_impl.h b/mlkem/native/arith_impl.h new file mode 100644 index 000000000..128a3b513 --- /dev/null +++ b/mlkem/native/arith_impl.h @@ -0,0 +1,15 @@ +/* + * Copyright (c) 2024 The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifdef MLKEM_NATIVE_ARITH_IMPL_H +#error Only one ARITH assembly profile can be defined -- did you include multiple profiles? +#else +#define MLKEM_NATIVE_ARITH_IMPL_H + +#if defined(MLKEM_NATIVE_ARITH_BACKEND_IMPL) +#include MLKEM_NATIVE_ARITH_BACKEND_IMPL +#endif + +#endif /* MLKEM_NATIVE_ARITH_IMPL_H */ diff --git a/mlkem/native/arith_native.h b/mlkem/native/arith_native.h index b7e921323..5a2d92b1d 100644 --- a/mlkem/native/arith_native.h +++ b/mlkem/native/arith_native.h @@ -6,14 +6,12 @@ #define MLKEM_ARITH_NATIVE_H #include -#include "config.h" -#include "params.h" - -#if defined(MLKEM_USE_NATIVE) +#include "cbmc.h" #include "poly.h" #include "polyvec.h" -#include "profile.h" + +#include "common.h" /* * MLKEM native arithmetic interface @@ -246,5 +244,4 @@ static INLINE int rej_uniform_native(int16_t *r, unsigned int len, const uint8_t *buf, unsigned int buflen); #endif /* MLKEM_USE_NATIVE_REJ_UNIFORM */ -#endif /* MLKEM_USE_NATIVE */ #endif /* MLKEM_ARITH_NATIVE_H */ diff --git a/mlkem/native/default.h b/mlkem/native/default.h new file mode 100644 index 000000000..34104c818 --- /dev/null +++ b/mlkem/native/default.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2024 The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 + */ +#ifndef MLKEM_NATIVE_ARITH_BACKEND_DEFAULT_H +#define MLKEM_NATIVE_ARITH_BACKEND_DEFAULT_H + +/* + * Default arithmetic backend + */ +#include "common.h" + +#ifdef SYS_AARCH64 +/* + * For AArch64, we currently we have one clean and one opt profile. + * We default to the opt profile. + * + * In the future, this may branch further depending on the microarchitecture. + */ +#include "aarch64/profiles/opt_metadata.h" +#endif /* SYS_AARCH64 */ + +#ifdef SYS_X86_64_AVX2 +/* + * For now, there's only one x86_64 profile, based on + * the AVX2 code from the Kyber repository. + * https://github.com/pq-crystals/kyber + */ +#include "x86_64/profiles/default_metadata.h" +#endif /* SYS_X86_64 */ + +#endif /* MLKEM_NATIVE_ARITH_BACKEND_DEFAULT_H */ diff --git a/mlkem/native/profile.h b/mlkem/native/profile.h deleted file mode 100644 index ab9d93aa1..000000000 --- a/mlkem/native/profile.h +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright (c) 2024 The mlkem-native project authors - * SPDX-License-Identifier: Apache-2.0 - */ -#ifndef MLKEM_ARITH_NATIVE_PROFILE_CHOICE_H -#define MLKEM_ARITH_NATIVE_PROFILE_CHOICE_H - -#include -#include "config.h" -#include "params.h" - -#if defined(MLKEM_USE_NATIVE) -#include "poly.h" -#include "polyvec.h" - -/* - * MLKEM native arithmetic profile - * - * The profile decides which implementation(s) of the arithmetic backend to use. - * - * If you don't change anything, the default profile will be used. This profile - * picks implementations based on characteristics of your system visible to - * the compiler. - * - * If you want to pick a specific profile for your target, there are three ways - * to do so, in descending order of convenience to the user: - * 1. Pick one of the profiles shipped with this repository. - * 2. Provide your own profile and register it via MLKEM_ARITH_NATIVE_PROFILE - * (which must be the profile's path relative to this directoru). - * 3. Set MLKEM_ARITH_NATIVE_MANUAL and use an adhoc profile specified via - * CFLAGS. - */ - -/* Option 2: Manually written profile */ -#if defined(MLKEM_ARITH_NATIVE_PROFILE) - -#define STRINGIFY_(x) #x -#define STRINGIFY(x) STRINGIFY_(x) -#include STRINGIFY(MLKEM_ARITH_NATIVE_PROFILE) - -/* Option 1: Choose from shipped list of profiles */ -#elif !defined(MLKEM_ARITH_NATIVE_MANUAL) - -#ifdef SYS_AARCH64 -/* - * For now, we only have clean and opt profiles. - * In the future, this is likely to branch further depending - * on the microarchitecture. - */ -#if defined(MLKEM_USE_NATIVE_AARCH64_CLEAN) -#include "aarch64/profiles/clean.h" -#else /* MLKEM_USE_NATIVE_AARCH64_CLEAN */ -#include "aarch64/profiles/opt.h" -#endif /* !MLKEM_USE_NATIVE_AARCH64_CLEAN */ -#endif /* SYS_AARCH64 */ - -#ifdef SYS_X86_64_AVX2 -/* - * For now, there's only one x86_64 profile, which is essentially - * the AVX2 code from the Kyber repository - * https://github.com/pq-crystals/kyber - */ -#include "x86_64/profiles/default.h" -#endif /* SYS_X86_64 */ - -#else /* !MLKEM_ARITH_NATIVE_PROFILE && MLKEM_ARITH_NATIVE_MANUAL */ - -/* Option 3: Build your own profile here, or via CFLAGS */ - -#endif /* !MLKEM_ARITH_NATIVE_PROFILE && !MLKEM_ARITH_NATIVE_MANUAL */ - -#endif /* MLKEM_USE_NATIVE */ -#endif /* MLKEM_ARITH_NATIVE_PROFILE_CHOICE_H */ diff --git a/mlkem/native/x86_64/arith_native_x86_64.h b/mlkem/native/x86_64/arith_native_x86_64.h index b346e1eb2..76e7bc072 100644 --- a/mlkem/native/x86_64/arith_native_x86_64.h +++ b/mlkem/native/x86_64/arith_native_x86_64.h @@ -5,16 +5,14 @@ #ifndef MLKEM_X86_64_NATIVE_H #define MLKEM_X86_64_NATIVE_H -#include -#include "config.h" -#include "fips202.h" -#include "params.h" -#include "polyvec.h" +#include "common.h" -#if defined(MLKEM_USE_NATIVE_X86_64) && defined(SYS_X86_64_AVX2) +#if defined(MLKEM_NATIVE_ARITH_BACKEND_X86_64_DEFAULT) #include #include +#include "fips202.h" +#include "polyvec.h" #define REJ_UNIFORM_AVX_NBLOCKS \ ((12 * MLKEM_N / 8 * (1 << 12) / MLKEM_Q + SHAKE128_RATE) / SHAKE128_RATE) diff --git a/mlkem/native/x86_64/basemul.S b/mlkem/native/x86_64/basemul.S index 797b0f46a..e093f13c1 100644 --- a/mlkem/native/x86_64/basemul.S +++ b/mlkem/native/x86_64/basemul.S @@ -8,7 +8,7 @@ #include "config.h" -#if defined(MLKEM_USE_NATIVE_X86_64) && defined(SYS_X86_64_AVX2) +#if defined(MLKEM_NATIVE_ARITH_BACKEND_X86_64_DEFAULT) #include "consts.h" #include "params.h" diff --git a/mlkem/native/x86_64/basemul.c b/mlkem/native/x86_64/basemul.c index c9dba813f..2b4a5854b 100644 --- a/mlkem/native/x86_64/basemul.c +++ b/mlkem/native/x86_64/basemul.c @@ -3,15 +3,16 @@ * SPDX-License-Identifier: Apache-2.0 */ -#include "config.h" +#include "common.h" -#if defined(MLKEM_USE_NATIVE_X86_64) && defined(SYS_X86_64_AVX2) +#if defined(MLKEM_NATIVE_ARITH_BACKEND_X86_64_DEFAULT) -#include "arith_native_x86_64.h" #include "consts.h" #include "poly.h" #include "polyvec.h" +#include "arith_native_x86_64.h" + static void poly_basemul_montgomery_avx2(poly *r, const poly *a, const poly *b) { basemul_avx2((__m256i *)r->coeffs, (const __m256i *)a->coeffs, @@ -58,7 +59,6 @@ void polyvec_basemul_acc_montgomery_cached_avx2(poly *r, const polyvec *a, } #else -#include "params.h" /* Dummy constant to keep compiler happy despite empty CU */ diff --git a/mlkem/native/x86_64/consts.c b/mlkem/native/x86_64/consts.c index 573eabb64..685d5cc1c 100644 --- a/mlkem/native/x86_64/consts.c +++ b/mlkem/native/x86_64/consts.c @@ -8,13 +8,12 @@ * https://github.com/pq-crystals/kyber/blob/main/avx2/consts.c */ -#include "config.h" +#include "common.h" -#if defined(MLKEM_USE_NATIVE_X86_64) && defined(SYS_X86_64_AVX2) +#if defined(MLKEM_NATIVE_ARITH_BACKEND_X86_64_DEFAULT) #include "align.h" #include "consts.h" -#include "params.h" #define Q MLKEM_Q #define MONT -1044 /* 2^16 mod q */ @@ -87,7 +86,6 @@ const qdata_t qdata = {{ SHIFT, SHIFT, SHIFT, SHIFT}}; #else /* MLKEM_USE_NATIVE_X86_64 && SYS_X86_64_AVX2 */ -#include "params.h" /* Dummy declaration for compilers disliking empty compilation units */ #define empty_cu_consts MLKEM_NAMESPACE(empty_cu_consts) diff --git a/mlkem/native/x86_64/consts.h b/mlkem/native/x86_64/consts.h index 8ca98bbf8..a9f0c484f 100644 --- a/mlkem/native/x86_64/consts.h +++ b/mlkem/native/x86_64/consts.h @@ -11,7 +11,7 @@ #ifndef CONSTS_H #define CONSTS_H -#include "params.h" +#include "common.h" #define _16XQ 0 #define _16XQINV 16 diff --git a/mlkem/native/x86_64/fq.S b/mlkem/native/x86_64/fq.S index 00fb093c5..35d4669cc 100644 --- a/mlkem/native/x86_64/fq.S +++ b/mlkem/native/x86_64/fq.S @@ -11,10 +11,9 @@ // in [0,1,...,q-1] rather than [0,1,...,q], matching the // semantics of poly_reduce(). -#include "config.h" -#include "params.h" +#include "common.h" -#if defined(MLKEM_USE_NATIVE_X86_64) && defined(SYS_X86_64_AVX2) +#if defined(MLKEM_NATIVE_ARITH_BACKEND_X86_64_DEFAULT) #include "consts.h" .include "fq.inc" diff --git a/mlkem/native/x86_64/intt.S b/mlkem/native/x86_64/intt.S index 8418cc471..ab875e08f 100644 --- a/mlkem/native/x86_64/intt.S +++ b/mlkem/native/x86_64/intt.S @@ -8,9 +8,10 @@ * * Changes to placement of modular reductions have * been made to simplify reasoning of non-overflow */ -#include "config.h" -#if defined(MLKEM_USE_NATIVE_X86_64) && defined(SYS_X86_64_AVX2) +#include "common.h" + +#if defined(MLKEM_NATIVE_ARITH_BACKEND_X86_64_DEFAULT) #include "consts.h" #include "params.h" diff --git a/mlkem/native/x86_64/ntt.S b/mlkem/native/x86_64/ntt.S index f159faf4f..de990484a 100644 --- a/mlkem/native/x86_64/ntt.S +++ b/mlkem/native/x86_64/ntt.S @@ -8,7 +8,7 @@ #include "config.h" -#if defined(MLKEM_USE_NATIVE_X86_64) && defined(SYS_X86_64_AVX2) +#if defined(MLKEM_NATIVE_ARITH_BACKEND_X86_64_DEFAULT) #include "consts.h" #include "params.h" diff --git a/mlkem/native/x86_64/profiles/default.h b/mlkem/native/x86_64/profiles/default_impl.h similarity index 95% rename from mlkem/native/x86_64/profiles/default.h rename to mlkem/native/x86_64/profiles/default_impl.h index 55eb0ad01..357350617 100644 --- a/mlkem/native/x86_64/profiles/default.h +++ b/mlkem/native/x86_64/profiles/default_impl.h @@ -5,10 +5,10 @@ /* ML-KEM arithmetic native profile for clean assembly */ -#ifdef MLKEM_ARITH_NATIVE_PROFILE_H +#ifdef MLKEM_NATIVE_ARITH_PROFILE_H #error Only one MLKEM_ARITH assembly profile can be defined -- did you include multiple profiles? #else -#define MLKEM_ARITH_NATIVE_PROFILE_H +#define MLKEM_NATIVE_ARITH_PROFILE_H #include @@ -96,4 +96,4 @@ static INLINE void poly_frombytes_native(poly *r, nttfrombytes_avx2((__m256i *)r->coeffs, a, qdata.vec); } -#endif /* MLKEM_ARITH_NATIVE_PROFILE_H */ +#endif /* MLKEM_NATIVE_ARITH_PROFILE_H */ diff --git a/mlkem/native/x86_64/profiles/default_metadata.h b/mlkem/native/x86_64/profiles/default_metadata.h new file mode 100644 index 000000000..05519eebe --- /dev/null +++ b/mlkem/native/x86_64/profiles/default_metadata.h @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2024 The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 + */ + +/* ML-KEM arithmetic native profile for clean assembly */ + +#ifdef MLKEM_NATIVE_ARITH_PROFILE_H +#error Only one MLKEM_ARITH assembly profile can be defined -- did you include multiple profiles? +#else +#define MLKEM_NATIVE_ARITH_PROFILE_H + +/* Identifier for this backend so that source and assembly files + * in the build can be appropriately guarded. */ +#define MLKEM_NATIVE_ARITH_BACKEND_X86_64_DEFAULT + +#define MLKEM_NATIVE_ARITH_BACKEND_NAME X86_64_DEFAULT + +/* Filename of the C backend implementation. + * This is not inlined here because this header is included in assembly + * files as well. */ +#define MLKEM_NATIVE_ARITH_BACKEND_IMPL "x86_64/profiles/default_impl.h" + +#endif /* MLKEM_NATIVE_ARITH_PROFILE_H */ diff --git a/mlkem/native/x86_64/rej_uniform_avx2.c b/mlkem/native/x86_64/rej_uniform_avx2.c index c6e663f7d..82e1f57a0 100644 --- a/mlkem/native/x86_64/rej_uniform_avx2.c +++ b/mlkem/native/x86_64/rej_uniform_avx2.c @@ -8,18 +8,15 @@ * https://github.com/pq-crystals/kyber/blob/main/avx2 */ -#include "config.h" +#include "common.h" -#if defined(MLKEM_USE_NATIVE_X86_64) && defined(SYS_X86_64_AVX2) - -#include "arith_native_x86_64.h" +#if defined(MLKEM_NATIVE_ARITH_BACKEND_X86_64_DEFAULT) #include #include #include +#include "arith_native_x86_64.h" #include "consts.h" -#include "params.h" - /* #define BMI */ #ifndef BMI @@ -291,7 +288,6 @@ unsigned int rej_uniform_avx2(int16_t *RESTRICT r, const uint8_t *buf) } #else /* MLKEM_USE_NATIVE_X86_64 && SYS_X86_64_AVX2 */ -#include "params.h" /* Dummy declaration for compilers disliking empty compilation units */ #define empty_cu_rej_uniform_avx2 MLKEM_NAMESPACE(empty_cu_rej_uniform_avx2) diff --git a/mlkem/native/x86_64/shuffle.S b/mlkem/native/x86_64/shuffle.S index dd1243121..794b4ccaf 100644 --- a/mlkem/native/x86_64/shuffle.S +++ b/mlkem/native/x86_64/shuffle.S @@ -6,9 +6,9 @@ // Implementation from Kyber reference repository // https://github.com/pq-crystals/kyber/blob/main/avx2 -#include "config.h" +#include "common.h" -#if defined(MLKEM_USE_NATIVE_X86_64) && defined(SYS_X86_64_AVX2) +#if defined(MLKEM_NATIVE_ARITH_BACKEND_X86_64_DEFAULT) #include "consts.h" #include "params.h" diff --git a/mlkem/ntt.c b/mlkem/ntt.c index 5b17ad0ce..f77a2d6f3 100644 --- a/mlkem/ntt.c +++ b/mlkem/ntt.c @@ -2,14 +2,12 @@ * Copyright (c) 2024 The mlkem-native project authors * SPDX-License-Identifier: Apache-2.0 */ -#include "ntt.h" #include -#include "params.h" -#include "reduce.h" -#include "arith_native.h" +#include "arith_impl.h" #include "debug/debug.h" #include "ntt.h" +#include "reduce.h" #if !defined(MLKEM_USE_NATIVE_NTT) /* diff --git a/mlkem/ntt.h b/mlkem/ntt.h index 68c9e6724..efa38ecc9 100644 --- a/mlkem/ntt.h +++ b/mlkem/ntt.h @@ -6,9 +6,8 @@ #define NTT_H #include -#include "arith_native.h" #include "cbmc.h" -#include "params.h" +#include "common.h" #include "poly.h" #include "reduce.h" diff --git a/mlkem/params.h b/mlkem/params.h index 85797f067..586c31d33 100644 --- a/mlkem/params.h +++ b/mlkem/params.h @@ -5,47 +5,12 @@ #ifndef PARAMS_H #define PARAMS_H -#include "common.h" #include "config.h" -#include "cpucap.h" -#ifndef MLKEM_K -#define MLKEM_K 3 /* Change this for different security strengths */ +#if !defined(MLKEM_K) +#error MLKEM_K is not defined #endif -#if defined(MLKEM_USE_NATIVE_X86_64) -#define MLKEM_NATIVE_BACKEND X86_64 -#elif defined(MLKEM_USE_NATIVE_AARCH64) -#define MLKEM_NATIVE_BACKEND AARCH64 -#else -#define MLKEM_NATIVE_BACKEND C -#endif - -/* Don't change parameters below this line */ -#if (MLKEM_K == 2) -#define MLKEM_PARAM_NAME MLKEM512 -#elif (MLKEM_K == 3) -#define MLKEM_PARAM_NAME MLKEM768 -#elif (MLKEM_K == 4) -#define MLKEM_PARAM_NAME MLKEM1024 -#else -#error "MLKEM_K must be in {2,3,4}" -#endif - -#define ___MLKEM_NAMESPACE(x1, x2, x3, x4) x1##_##x2##_##x3##_##x4 -#define __MLKEM_NAMESPACE(x1, x2, x3, x4) ___MLKEM_NAMESPACE(x1, x2, x3, x4) - -/* - * NAMESPACE is PQCP_MLKEM_NATIVE___ - * e.g., PQCP_MLKEM_NATIVE_MLKEM512_AARCH64_ - */ -#define MLKEM_NAMESPACE(s) \ - __MLKEM_NAMESPACE(PQCP_MLKEM_NATIVE, MLKEM_PARAM_NAME, MLKEM_NATIVE_BACKEND, \ - s) -#define _MLKEM_NAMESPACE(s) \ - __MLKEM_NAMESPACE(_PQCP_MLKEM_NATIVE, MLKEM_PARAM_NAME, \ - MLKEM_NATIVE_BACKEND, s) - #define MLKEM_N 256 #define MLKEM_Q 3329 #define UINT12_MAX 4095 diff --git a/mlkem/poly.c b/mlkem/poly.c index a3a545cd1..4e43e310e 100644 --- a/mlkem/poly.c +++ b/mlkem/poly.c @@ -2,21 +2,20 @@ * Copyright (c) 2024 The mlkem-native project authors * SPDX-License-Identifier: Apache-2.0 */ -#include "poly.h" #include #include + +#include "arith_impl.h" #include "cbd.h" #include "cbmc.h" +#include "debug/debug.h" #include "fips202x4.h" #include "ntt.h" -#include "params.h" +#include "poly.h" #include "reduce.h" #include "symmetric.h" #include "verify.h" -#include "arith_native.h" -#include "debug/debug.h" - void poly_compress_du(uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_DU], const poly *a) { int j; diff --git a/mlkem/poly.h b/mlkem/poly.h index 772afefe0..fd171e4ba 100644 --- a/mlkem/poly.h +++ b/mlkem/poly.h @@ -8,7 +8,7 @@ #include #include #include "cbmc.h" -#include "params.h" +#include "common.h" #include "reduce.h" #include "verify.h" diff --git a/mlkem/polyvec.c b/mlkem/polyvec.c index 7f268ee50..f6e559c8e 100644 --- a/mlkem/polyvec.c +++ b/mlkem/polyvec.c @@ -7,7 +7,6 @@ #include "arith_native.h" #include "config.h" #include "ntt.h" -#include "params.h" #include "poly.h" #include "debug/debug.h" diff --git a/mlkem/polyvec.h b/mlkem/polyvec.h index 785cf613d..11ac14684 100644 --- a/mlkem/polyvec.h +++ b/mlkem/polyvec.h @@ -6,7 +6,7 @@ #define POLYVEC_H #include -#include "params.h" +#include "common.h" #include "poly.h" typedef struct diff --git a/mlkem/reduce.h b/mlkem/reduce.h index 39e85588a..515f706fa 100644 --- a/mlkem/reduce.h +++ b/mlkem/reduce.h @@ -7,8 +7,8 @@ #include #include "cbmc.h" +#include "common.h" #include "debug/debug.h" -#include "params.h" #define HALF_Q ((MLKEM_Q + 1) / 2) /* 1665 */ diff --git a/mlkem/rej_uniform.c b/mlkem/rej_uniform.c index 4e8a5ce9b..bfaef5c73 100644 --- a/mlkem/rej_uniform.c +++ b/mlkem/rej_uniform.c @@ -2,10 +2,9 @@ * Copyright (c) 2024 The mlkem-native project authors * SPDX-License-Identifier: Apache-2.0 */ -#include "params.h" -#include "arith_native.h" #include "rej_uniform.h" +#include "arith_native.h" /************************************************* * Name: rej_uniform_scalar diff --git a/mlkem/rej_uniform.h b/mlkem/rej_uniform.h index aeb9cc3eb..da462398b 100644 --- a/mlkem/rej_uniform.h +++ b/mlkem/rej_uniform.h @@ -8,7 +8,7 @@ #include #include #include "cbmc.h" -#include "params.h" +#include "common.h" #define rej_uniform MLKEM_NAMESPACE(rej_uniform) /************************************************* diff --git a/mlkem/symmetric.h b/mlkem/symmetric.h index 4eb6aaba6..55ebbbd53 100644 --- a/mlkem/symmetric.h +++ b/mlkem/symmetric.h @@ -7,11 +7,9 @@ #include #include -#include "params.h" - -#include "fips202.h" - #include "cbmc.h" +#include "common.h" +#include "fips202.h" /* Macros denoting FIPS-203 specific Hash functions */ diff --git a/mlkem/sys/cpucap.h b/mlkem/sys.h similarity index 53% rename from mlkem/sys/cpucap.h rename to mlkem/sys.h index 7c3a1cdec..be3070dc2 100644 --- a/mlkem/sys/cpucap.h +++ b/mlkem/sys.h @@ -2,9 +2,8 @@ * Copyright (c) 2024 The mlkem-native project authors * SPDX-License-Identifier: Apache-2.0 */ - -#ifndef CPUCAP_H -#define CPUCAP_H +#ifndef MLKEM_NATIVE_SYS_H +#define MLKEM_NATIVE_SYS_H /* Check if we're running on an AArch64 little endian system. _M_ARM64 is set by * MSVC. */ @@ -52,4 +51,56 @@ #error "FORCE_X86_64 is set, but we don't seem to be on an X86_64 system." #endif +/* + * C90 does not have the inline compiler directive yet. + * We don't use it in C90 builds. + * However, in that case the compiler warns about some inline functions in + * header files not being used in every compilation unit that includes that + * header. To work around it we silence that warning in that case using + * __attribute__((unused)). + */ + +/* Do not use inline for C90 builds*/ +#if !defined(inline) +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L +#define INLINE inline +#define ALWAYS_INLINE __attribute__((always_inline)) +#elif defined(_MSC_VER) +#define INLINE __inline +#define ALWAYS_INLINE __forceinline +#else +#define INLINE __attribute__((unused)) +#define ALWAYS_INLINE +#endif + +#else +#define INLINE inline +#define ALWAYS_INLINE __attribute__((always_inline)) +#endif + +/* + * C90 does not have the restrict compiler directive yet. + * We don't use it in C90 builds. + */ +#if !defined(restrict) +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L +#define RESTRICT restrict +#else +#define RESTRICT +#endif + +#else + +#define RESTRICT restrict +#endif + +#define DEFAULT_ALIGN 32 +#if defined(_WIN32) +#define ALIGN __declspec(align(DEFAULT_ALIGN)) +#define asm __asm +#else +#define asm __asm__ +#define ALIGN __attribute__((aligned(DEFAULT_ALIGN))) #endif + +#endif /* MLKEM_NATIVE_SYS_H */ diff --git a/mlkem/sys/config.h b/mlkem/sys/config.h deleted file mode 100644 index 370a141a6..000000000 --- a/mlkem/sys/config.h +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright (c) 2024 The mlkem-native project authors - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef CONFIG_H -#define CONFIG_H - -#include "cpucap.h" - -#if defined(MLKEM_USE_NATIVE) - -#if defined(SYS_AARCH64) -#define MLKEM_USE_NATIVE_AARCH64 -#endif /* SYS_AARCH64 */ - -#if defined(SYS_X86_64) -#define MLKEM_USE_NATIVE_X86_64 -#endif /* SYS_X86_64 */ - -#endif /* MLKEM_USE_NATIVE */ -#endif /* CONFIG_H */ diff --git a/mlkem/verify.c b/mlkem/verify.c index e616a8bab..b7078fcc1 100644 --- a/mlkem/verify.c +++ b/mlkem/verify.c @@ -13,7 +13,6 @@ volatile uint64_t ct_opt_blocker_u64 = 0; #else /* MLKEM_USE_ASM_VALUE_BARRIER */ -#include "params.h" #define empty_cu_verify MLKEM_NAMESPACE(empty_cu_verify) int empty_cu_verify; diff --git a/scripts/autogenerate_files.py b/scripts/autogenerate_files.py index a33d3cdfe..f8c3c3acf 100644 --- a/scripts/autogenerate_files.py +++ b/scripts/autogenerate_files.py @@ -295,9 +295,13 @@ def gen_aarch64_mulcache_twiddles_twisted(): def gen_aarch64_fwd_ntt_zeta_file(dry_run=False): def gen(): yield from gen_header() - yield '#include "arith_native_aarch64.h"' + yield '#include "common.h"' + yield "" + yield "#if defined(MLKEM_NATIVE_ARITH_BACKEND_AARCH64_CLEAN) || \\" + yield " defined(MLKEM_NATIVE_ARITH_BACKEND_AARCH64_OPT)" yield "" - yield "#ifdef MLKEM_USE_NATIVE_AARCH64" + yield "#include " + yield '#include "arith_native_aarch64.h"' yield "" yield "/*" yield " * Table of zeta values used in the AArch64 forward NTT" @@ -327,13 +331,13 @@ def gen(): yield from map(lambda t: str(t) + ",", gen_aarch64_mulcache_twiddles_twisted()) yield "};" yield "" - yield "#else /* MLKEM_USE_NATIVE_AARCH64 */" + yield "#else" yield '#include "params.h"' yield "" yield "/* Dummy declaration for compilers disliking empty compilation units */" yield "#define empty_cu_aarch64_zetas MLKEM_NAMESPACE(empty_cu_aarch64_zetas)" yield "int empty_cu_aarch64_zetas;" - yield "#endif /* MLKEM_USE_NATIVE_AARCH64 */" + yield "#endif" yield "" update_file(