diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 5aa9a53c8..4dd9c7b12 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -43,7 +43,7 @@ jobs: name: Arm Cortex-A55 (Snapdragon 888) benchmarks bench_pmu: PERF archflags: "-mcpu=cortex-a55 -march=armv8.2-a" - cflags: "-flto -static -DFORCE_AARCH64 -DFIPS202_NATIVE_PROFILE=\"aarch64/profiles/cortex_a55.h\"" + cflags: "-flto -static -DFORCE_AARCH64 -DMLKEM_NATIVE_FIPS202_BACKEND=\\\\\\\"aarch64/cortex_a55.h\\\\\\\"" bench_extra_args: -w exec-on-a55 - system: bpi name: Bananapi bpi-f3 benchmarks diff --git a/examples/bring_your_own_fips202/Makefile b/examples/bring_your_own_fips202/Makefile index 1a80951e9..e4152b7b0 100644 --- a/examples/bring_your_own_fips202/Makefile +++ b/examples/bring_your_own_fips202/Makefile @@ -19,7 +19,6 @@ INC= INC+=-Imlkem_native/mlkem INC+=-Imlkem_native/mlkem INC+=-Imlkem_native/mlkem/native -INC+=-Imlkem_native/mlkem/sys # Part B: # diff --git a/examples/bring_your_own_fips202/custom_fips202/fips202.h b/examples/bring_your_own_fips202/custom_fips202/fips202.h index 5ffe60fae..4bebebeb0 100644 --- a/examples/bring_your_own_fips202/custom_fips202/fips202.h +++ b/examples/bring_your_own_fips202/custom_fips202/fips202.h @@ -11,7 +11,7 @@ #ifndef FIPS202_H #define FIPS202_H -#include "namespace.h" +#include "common.h" #include "tiny_sha3/sha3.h" #define SHAKE128_RATE 168 diff --git a/examples/bring_your_own_fips202/custom_fips202/namespace.h b/examples/bring_your_own_fips202/custom_fips202/namespace.h deleted file mode 100644 index fab249afa..000000000 --- a/examples/bring_your_own_fips202/custom_fips202/namespace.h +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (c) 2024 The mlkem-native project authors - * SPDX-License-Identifier: Apache-2.0 - */ -#ifndef NAMESPACE_H -#define NAMESPACE_H - -#include "config.h" - -#if defined(MLKEM_USE_NATIVE_X86_64) -#define MLKEM_NATIVE_BACKEND X86_64 -#elif defined(MLKEM_USE_NATIVE_AARCH64) -#define MLKEM_NATIVE_BACKEND AARCH64 -#else -#define MLKEM_NATIVE_BACKEND C -#endif - -#define ___FIPS202_NAMESPACE(x1, x2, x3) x1##_##x2##_##x3 -#define __FIPS202_NAMESPACE(x1, x2, x3) ___FIPS202_NAMESPACE(x1, x2, x3) - -/* - * NAMESPACE is PQCP_MLKEM_NATIVE_FIPS202__ - * e.g., PQCP_MLKEM_NATIVE_FIPS202_AARCH64_ - */ -#define FIPS202_NAMESPACE(s) \ - __FIPS202_NAMESPACE(PQCP_MLKEM_NATIVE_FIPS202, MLKEM_NATIVE_BACKEND, s) -#define _FIPS202_NAMESPACE(s) \ - __FIPS202_NAMESPACE(_PQCP_MLKEM_NATIVE_FIPS202, MLKEM_NATIVE_BACKEND, s) - -#endif diff --git a/examples/mlkem_native_as_code_package/Makefile b/examples/mlkem_native_as_code_package/Makefile index 1a6aed46a..127afd7d2 100644 --- a/examples/mlkem_native_as_code_package/Makefile +++ b/examples/mlkem_native_as_code_package/Makefile @@ -21,7 +21,6 @@ INC+=-Imlkem_native/mlkem INC+=-Imlkem_native/mlkem/native INC+=-Imlkem_native/fips202 INC+=-Imlkem_native/fips202/native -INC+=-Imlkem_native/mlkem/sys # Part B: # diff --git a/fips202/fips202.h b/fips202/fips202.h index 48429c271..333a959d7 100644 --- a/fips202/fips202.h +++ b/fips202/fips202.h @@ -7,9 +7,8 @@ #include #include -#include "namespace.h" - #include "cbmc.h" +#include "common.h" #define SHAKE128_RATE 168 #define SHAKE256_RATE 136 diff --git a/fips202/fips202_backend.h b/fips202/fips202_backend.h new file mode 100644 index 000000000..24e6593b2 --- /dev/null +++ b/fips202/fips202_backend.h @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2024 The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifdef MLKEM_NATIVE_FIPS202_IMPL_H +#error Only one FIPS202 assembly profile can be defined -- did you include multiple profiles? +#else +#define MLKEM_NATIVE_FIPS202_IMPL_H + +/* Include to enforce consistency of API and implementation */ +#include "native/api.h" + +#if defined(MLKEM_NATIVE_FIPS202_BACKEND_IMPL) +#include MLKEM_NATIVE_FIPS202_BACKEND_IMPL +#endif + +#endif /* MLKEM_NATIVE_FIPS202_IMPL_H */ diff --git a/fips202/fips202x4.h b/fips202/fips202x4.h index 33a732a88..1bb0d08c7 100644 --- a/fips202/fips202x4.h +++ b/fips202/fips202x4.h @@ -7,9 +7,9 @@ #include #include +#include "common.h" #include "fips202.h" #include "keccakf1600.h" -#include "namespace.h" #include "cbmc.h" diff --git a/fips202/keccakf1600.c b/fips202/keccakf1600.c index 7b653fca9..5049999dd 100644 --- a/fips202/keccakf1600.c +++ b/fips202/keccakf1600.c @@ -15,7 +15,7 @@ #include #include "config.h" -#include "fips202_native.h" +#include "fips202_backend.h" #include "cbmc.h" diff --git a/fips202/keccakf1600.h b/fips202/keccakf1600.h index 7341df0bf..3d59e0b19 100644 --- a/fips202/keccakf1600.h +++ b/fips202/keccakf1600.h @@ -6,10 +6,8 @@ #define KECCAKF1600_H #include -#include "fips202_native.h" -#include "namespace.h" - #include "cbmc.h" +#include "common.h" #define KECCAK_LANES 25 /* diff --git a/fips202/namespace.h b/fips202/namespace.h deleted file mode 100644 index fab249afa..000000000 --- a/fips202/namespace.h +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (c) 2024 The mlkem-native project authors - * SPDX-License-Identifier: Apache-2.0 - */ -#ifndef NAMESPACE_H -#define NAMESPACE_H - -#include "config.h" - -#if defined(MLKEM_USE_NATIVE_X86_64) -#define MLKEM_NATIVE_BACKEND X86_64 -#elif defined(MLKEM_USE_NATIVE_AARCH64) -#define MLKEM_NATIVE_BACKEND AARCH64 -#else -#define MLKEM_NATIVE_BACKEND C -#endif - -#define ___FIPS202_NAMESPACE(x1, x2, x3) x1##_##x2##_##x3 -#define __FIPS202_NAMESPACE(x1, x2, x3) ___FIPS202_NAMESPACE(x1, x2, x3) - -/* - * NAMESPACE is PQCP_MLKEM_NATIVE_FIPS202__ - * e.g., PQCP_MLKEM_NATIVE_FIPS202_AARCH64_ - */ -#define FIPS202_NAMESPACE(s) \ - __FIPS202_NAMESPACE(PQCP_MLKEM_NATIVE_FIPS202, MLKEM_NATIVE_BACKEND, s) -#define _FIPS202_NAMESPACE(s) \ - __FIPS202_NAMESPACE(_PQCP_MLKEM_NATIVE_FIPS202, MLKEM_NATIVE_BACKEND, s) - -#endif diff --git a/fips202/native/aarch64/cortex_a55.h b/fips202/native/aarch64/cortex_a55.h new file mode 100644 index 000000000..3d9d92879 --- /dev/null +++ b/fips202/native/aarch64/cortex_a55.h @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2024 The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 + */ + +/* FIPS202 assembly profile targeting Cortex-A55 */ + +#ifdef FIPS202_NATIVE_PROFILE_H +#error Only one FIPS202 assembly profile can be defined -- did you include multiple profiles? +#else +#define FIPS202_NATIVE_PROFILE_H + +/* Identifier for this backend so that source and assembly files + * in the build can be appropriately guarded. */ +#define MLKEM_NATIVE_FIPS202_BACKEND_AARCH64_A55 + +#define MLKEM_NATIVE_FIPS202_BACKEND_NAME AARCH64_A55 + +/* Filename of the C backend implementation. + * This is not inlined here because this header is included in assembly + * files as well. */ +#define MLKEM_NATIVE_FIPS202_BACKEND_IMPL "aarch64/src/cortex_a55_impl.h" + +#endif /* FIPS202_NATIVE_PROFILE_H */ diff --git a/fips202/native/aarch64/default.h b/fips202/native/aarch64/default.h new file mode 100644 index 000000000..1386c6440 --- /dev/null +++ b/fips202/native/aarch64/default.h @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2024 The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 + */ + +/* Default FIPS202 assembly profile for AArch64 systems */ + +#ifdef FIPS202_NATIVE_PROFILE_H +#error Only one FIPS202 assembly profile can be defined -- did you include multiple profiles? +#else +#define FIPS202_NATIVE_PROFILE_H + +/* Identifier for this backend so that source and assembly files + * in the build can be appropriately guarded. */ +#define MLKEM_NATIVE_FIPS202_BACKEND_AARCH64_DEFAULT + +#define MLKEM_NATIVE_FIPS202_BACKEND_NAME AARCH64_DEFAULT + +/* Filename of the C backend implementation. + * This is not inlined here because this header is included in assembly + * files as well. */ +#define MLKEM_NATIVE_FIPS202_BACKEND_IMPL "aarch64/src/default_impl.h" + +#endif /* FIPS202_NATIVE_PROFILE_H */ diff --git a/fips202/native/aarch64/common.i b/fips202/native/aarch64/src/common.i similarity index 100% rename from fips202/native/aarch64/common.i rename to fips202/native/aarch64/src/common.i diff --git a/fips202/native/aarch64/profiles/cortex_a55.h b/fips202/native/aarch64/src/cortex_a55_impl.h similarity index 76% rename from fips202/native/aarch64/profiles/cortex_a55.h rename to fips202/native/aarch64/src/cortex_a55_impl.h index 3ca536bf4..90b857f66 100644 --- a/fips202/native/aarch64/profiles/cortex_a55.h +++ b/fips202/native/aarch64/src/cortex_a55_impl.h @@ -5,12 +5,12 @@ /* FIPS202 assembly profile targeting Cortex-A55 */ -#ifdef FIPS202_NATIVE_PROFILE_H +#ifdef FIPS202_NATIVE_PROFILE_IMPL_H #error Only one FIPS202 assembly profile can be defined -- did you include multiple profiles? #else -#define FIPS202_NATIVE_PROFILE_H +#define FIPS202_NATIVE_PROFILE_IMPL_H -#include "../fips202_native_aarch64.h" +#include "fips202_native_aarch64.h" /* * On Cortex-A55, we use lazy rotation assembly for Keccak-x1, @@ -22,4 +22,4 @@ static INLINE void keccak_f1600_x1_native(uint64_t *state) keccak_f1600_x1_scalar_asm_opt(state); } -#endif /* FIPS202_NATIVE_PROFILE_H */ +#endif /* FIPS202_NATIVE_PROFILE_IMPL_H */ diff --git a/fips202/native/aarch64/profiles/default.h b/fips202/native/aarch64/src/default_impl.h similarity index 95% rename from fips202/native/aarch64/profiles/default.h rename to fips202/native/aarch64/src/default_impl.h index 3a9da28b7..4d28e35bf 100644 --- a/fips202/native/aarch64/profiles/default.h +++ b/fips202/native/aarch64/src/default_impl.h @@ -5,12 +5,12 @@ /* Default FIPS202 assembly profile for AArch64 systems */ -#ifdef FIPS202_NATIVE_PROFILE_H +#ifdef FIPS202_NATIVE_PROFILE_IMPL_H #error Only one FIPS202 assembly profile can be defined -- did you include multiple profiles? #else -#define FIPS202_NATIVE_PROFILE_H +#define FIPS202_NATIVE_PROFILE_IMPL_H -#include "../fips202_native_aarch64.h" +#include "fips202_native_aarch64.h" /* * Default logic to decide which implementation to use. diff --git a/fips202/native/aarch64/fips202_native_aarch64.h b/fips202/native/aarch64/src/fips202_native_aarch64.h similarity index 91% rename from fips202/native/aarch64/fips202_native_aarch64.h rename to fips202/native/aarch64/src/fips202_native_aarch64.h index 72e7c9710..d62b2c958 100644 --- a/fips202/native/aarch64/fips202_native_aarch64.h +++ b/fips202/native/aarch64/src/fips202_native_aarch64.h @@ -6,11 +6,8 @@ #define FIPS202_AARCH64_NATIVE_H #include -#include "config.h" -#include "namespace.h" -#include "params.h" +#include "common.h" -#ifdef MLKEM_USE_NATIVE_AARCH64 #define keccak_f1600_x1_scalar_asm_opt \ FIPS202_NAMESPACE(keccak_f1600_x1_scalar_asm_opt) void keccak_f1600_x1_scalar_asm_opt(uint64_t *state); @@ -38,6 +35,5 @@ void keccak_f1600_x4_scalar_v84a_asm_hybrid_opt(uint64_t *state); #define keccak_f1600_x4_scalar_v8a_v84a_hybrid_asm_opt \ FIPS202_NAMESPACE(keccak_f1600_x4_scalar_v8a_v84a_hybrid_asm_opt) void keccak_f1600_x4_scalar_v8a_v84a_hybrid_asm_opt(uint64_t *state); -#endif /* MLKEM_USE_NATIVE_AARCH64 */ #endif /* FIPS202_AARCH64_NATIVE_H */ diff --git a/fips202/native/aarch64/keccak_f1600_x1_scalar_asm_opt.S b/fips202/native/aarch64/src/keccak_f1600_x1_scalar_asm_opt.S similarity index 99% rename from fips202/native/aarch64/keccak_f1600_x1_scalar_asm_opt.S rename to fips202/native/aarch64/src/keccak_f1600_x1_scalar_asm_opt.S index 3db9e2046..3655a219d 100644 --- a/fips202/native/aarch64/keccak_f1600_x1_scalar_asm_opt.S +++ b/fips202/native/aarch64/src/keccak_f1600_x1_scalar_asm_opt.S @@ -28,12 +28,12 @@ // Author: Hanno Becker // Author: Matthias Kannwischer -#include "config.h" -#if defined(MLKEM_USE_NATIVE_AARCH64) +#include "common.h" +#if defined(MLKEM_NATIVE_FIPS202_BACKEND_AARCH64_DEFAULT) || \ + defined(MLKEM_NATIVE_FIPS202_BACKEND_AARCH64_A55) // Needed to provide ASM_LOAD directive #include "common.i" -#include "namespace.h" /********************** CONSTANTS *************************/ .data @@ -457,4 +457,5 @@ initial: free_stack ret -#endif /* MLKEM_USE_NATIVE_AARCH64 */ +#endif /* defined(MLKEM_NATIVE_FIPS202_BACKEND_AARCH64_DEFAULT) || + defined(MLKEM_NATIVE_FIPS202_BACKEND_AARCH64_A55) */ diff --git a/fips202/native/aarch64/keccak_f1600_x1_v84a_asm_clean.S b/fips202/native/aarch64/src/keccak_f1600_x1_v84a_asm_clean.S similarity index 98% rename from fips202/native/aarch64/keccak_f1600_x1_v84a_asm_clean.S rename to fips202/native/aarch64/src/keccak_f1600_x1_v84a_asm_clean.S index 0613a8a75..c60e4cad4 100644 --- a/fips202/native/aarch64/keccak_f1600_x1_v84a_asm_clean.S +++ b/fips202/native/aarch64/src/keccak_f1600_x1_v84a_asm_clean.S @@ -37,8 +37,9 @@ // during load and store, so that the caller need not do this. // -#include "config.h" -#if defined(MLKEM_USE_NATIVE_AARCH64) +#include "common.h" +#if defined(MLKEM_NATIVE_FIPS202_BACKEND_AARCH64_DEFAULT) || \ + defined(MLKEM_NATIVE_FIPS202_BACKEND_AARCH64_A55) // Needed to provide ASM_LOAD directive #include "common.i" diff --git a/fips202/native/aarch64/keccak_f1600_x2_v84a_asm_clean.S b/fips202/native/aarch64/src/keccak_f1600_x2_v84a_asm_clean.S similarity index 98% rename from fips202/native/aarch64/keccak_f1600_x2_v84a_asm_clean.S rename to fips202/native/aarch64/src/keccak_f1600_x2_v84a_asm_clean.S index f74856a5f..45304aa25 100644 --- a/fips202/native/aarch64/keccak_f1600_x2_v84a_asm_clean.S +++ b/fips202/native/aarch64/src/keccak_f1600_x2_v84a_asm_clean.S @@ -37,8 +37,9 @@ // during load and store, so that the caller need not do this. // -#include "config.h" -#if defined(MLKEM_USE_NATIVE_AARCH64) +#include "common.h" +#if defined(MLKEM_NATIVE_FIPS202_BACKEND_AARCH64_DEFAULT) || \ + defined(MLKEM_NATIVE_FIPS202_BACKEND_AARCH64_A55) // Needed to provide ASM_LOAD directive #include "common.i" diff --git a/fips202/native/aarch64/keccak_f1600_x2_v8a_v84a_asm_hybrid.S b/fips202/native/aarch64/src/keccak_f1600_x2_v8a_v84a_asm_hybrid.S similarity index 98% rename from fips202/native/aarch64/keccak_f1600_x2_v8a_v84a_asm_hybrid.S rename to fips202/native/aarch64/src/keccak_f1600_x2_v8a_v84a_asm_hybrid.S index b7cc6f261..e4fecb11d 100644 --- a/fips202/native/aarch64/keccak_f1600_x2_v8a_v84a_asm_hybrid.S +++ b/fips202/native/aarch64/src/keccak_f1600_x2_v8a_v84a_asm_hybrid.S @@ -37,12 +37,12 @@ // during load and store, so that the caller need not do this. // -#include "config.h" -#if defined(MLKEM_USE_NATIVE_AARCH64) +#include "common.h" +#if defined(MLKEM_NATIVE_FIPS202_BACKEND_AARCH64_DEFAULT) || \ + defined(MLKEM_NATIVE_FIPS202_BACKEND_AARCH64_A55) // Needed to provide ASM_LOAD directive #include "common.i" -#include "namespace.h" #if defined(__ARM_FEATURE_SHA3) diff --git a/fips202/native/aarch64/keccak_f1600_x4_scalar_v84a_asm_hybrid_opt.S b/fips202/native/aarch64/src/keccak_f1600_x4_scalar_v84a_asm_hybrid_opt.S similarity index 99% rename from fips202/native/aarch64/keccak_f1600_x4_scalar_v84a_asm_hybrid_opt.S rename to fips202/native/aarch64/src/keccak_f1600_x4_scalar_v84a_asm_hybrid_opt.S index 19a93b9e8..abf1af497 100644 --- a/fips202/native/aarch64/keccak_f1600_x4_scalar_v84a_asm_hybrid_opt.S +++ b/fips202/native/aarch64/src/keccak_f1600_x4_scalar_v84a_asm_hybrid_opt.S @@ -24,12 +24,12 @@ * */ -#include "config.h" -#if defined(MLKEM_USE_NATIVE_AARCH64) +#include "common.h" +#if defined(MLKEM_NATIVE_FIPS202_BACKEND_AARCH64_DEFAULT) || \ + defined(MLKEM_NATIVE_FIPS202_BACKEND_AARCH64_A55) // Needed to provide ASM_LOAD directive #include "common.i" -#include "namespace.h" #if defined(__ARM_FEATURE_SHA3) diff --git a/fips202/native/aarch64/keccak_f1600_x4_v8a_scalar_hybrid_asm_opt.S b/fips202/native/aarch64/src/keccak_f1600_x4_v8a_scalar_hybrid_asm_opt.S similarity index 99% rename from fips202/native/aarch64/keccak_f1600_x4_v8a_scalar_hybrid_asm_opt.S rename to fips202/native/aarch64/src/keccak_f1600_x4_v8a_scalar_hybrid_asm_opt.S index a32375f07..f001d3eaf 100644 --- a/fips202/native/aarch64/keccak_f1600_x4_v8a_scalar_hybrid_asm_opt.S +++ b/fips202/native/aarch64/src/keccak_f1600_x4_v8a_scalar_hybrid_asm_opt.S @@ -24,12 +24,12 @@ * */ -#include "config.h" -#if defined(MLKEM_USE_NATIVE_AARCH64) +#include "common.h" +#if defined(MLKEM_NATIVE_FIPS202_BACKEND_AARCH64_DEFAULT) || \ + defined(MLKEM_NATIVE_FIPS202_BACKEND_AARCH64_A55) // Needed to provide ASM_LOAD directive #include "common.i" -#include "namespace.h" #define KECCAK_F1600_ROUNDS 24 diff --git a/fips202/native/aarch64/keccak_f1600_x4_v8a_v84a_scalar_hybrid_asm_opt.S b/fips202/native/aarch64/src/keccak_f1600_x4_v8a_v84a_scalar_hybrid_asm_opt.S similarity index 99% rename from fips202/native/aarch64/keccak_f1600_x4_v8a_v84a_scalar_hybrid_asm_opt.S rename to fips202/native/aarch64/src/keccak_f1600_x4_v8a_v84a_scalar_hybrid_asm_opt.S index 889ba2983..67a0adccd 100644 --- a/fips202/native/aarch64/keccak_f1600_x4_v8a_v84a_scalar_hybrid_asm_opt.S +++ b/fips202/native/aarch64/src/keccak_f1600_x4_v8a_v84a_scalar_hybrid_asm_opt.S @@ -24,8 +24,9 @@ * */ -#include "config.h" -#if defined(MLKEM_USE_NATIVE_AARCH64) +#include "common.h" +#if defined(MLKEM_NATIVE_FIPS202_BACKEND_AARCH64_DEFAULT) || \ + defined(MLKEM_NATIVE_FIPS202_BACKEND_AARCH64_A55) // Needed to provide ASM_LOAD directive #include "common.i" diff --git a/fips202/native/api.h b/fips202/native/api.h new file mode 100644 index 000000000..685b8bfc5 --- /dev/null +++ b/fips202/native/api.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2024 The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 + */ +#ifndef FIPS202_NATIVE_H +#define FIPS202_NATIVE_H + +#include +#include "common.h" + +/* + * FIPS202 native interface + */ + +/* + * Those functions are meant to be trivial wrappers around + * the chosen native implementation. The are static inline + * to avoid unnecessary calls. + * The macro before each declaration controls whether a native + * implementation is present. + */ + +#if defined(MLKEM_USE_FIPS202_X1_NATIVE) +static INLINE void keccak_f1600_x1_native(uint64_t *state); +#endif +#if defined(MLKEM_USE_FIPS202_X2_NATIVE) +static INLINE void keccak_f1600_x2_native(uint64_t *state); +#endif +#if defined(MLKEM_USE_FIPS202_X4_NATIVE) +static INLINE void keccak_f1600_x4_native(uint64_t *state); +#endif + +#endif /* FIPS202_NATIVE_H */ diff --git a/fips202/native/default.h b/fips202/native/default.h new file mode 100644 index 000000000..41c068910 --- /dev/null +++ b/fips202/native/default.h @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2024 The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef MLKEM_NATIVE_FIPS202_BACKEND_DEFAULT_H +#define MLKEM_NATIVE_FIPS202_BACKEND_DEFAULT_H + +/* + * Default FIPS202 backend + */ +#include "sys.h" + +#if defined(SYS_AARCH64) +#include "aarch64/default.h" +#endif + +#if defined(SYS_X86_64) && defined(SYS_X86_64_AVX2) +#include "x86_64/xkcp.h" +#endif + +#endif /* MLKEM_NATIVE_FIPS202_BACKEND_DEFAULT_H */ diff --git a/fips202/native/fips202_native.h b/fips202/native/fips202_native.h deleted file mode 100644 index ccddc32a9..000000000 --- a/fips202/native/fips202_native.h +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright (c) 2024 The mlkem-native project authors - * SPDX-License-Identifier: Apache-2.0 - */ -#ifndef FIPS202_NATIVE_H -#define FIPS202_NATIVE_H - -#include -#include "config.h" -#include "params.h" - -#if defined(MLKEM_USE_NATIVE) - -/* - * FIPS202 native profile - * - * The profile decides which implementation(s) of FIPS202 to use. - * - * If you don't change anything, the default profile will be used. This profile - * picks implementations based on characteristics of your system visible to - * the compiler. - * - * The default logic is not perfect, and you may want to pick a specific - * profile for your target. There are three ways to do so, in descending - * order of convenience to the user: - * 1. Pick one of the profiles shipped with this repository. - * 2. Provide your own profile and register it via FIPS202_ASM_PROFILE - * (which must be the profile's path relative to this directoru). - * 3. Set FIPS202_NATIVE_MANUAL and use an adhoc profile specified via CFLAGS. - */ - -/* Option 2: Manually written profile */ -#if defined(FIPS202_NATIVE_PROFILE) - -#define STRINGIFY_(x) #x -#define STRINGIFY(x) STRINGIFY_(x) -#include STRINGIFY(FIPS202_NATIVE_PROFILE) - -/* Option 1: Choose from shipped list of profiles */ -#elif !defined(FIPS202_NATIVE_MANUAL) - -#ifdef SYS_AARCH64 -/* Pick exactly one profile from the following list */ -#include "aarch64/profiles/default.h" -/* #include "aarch64/profiles/cortex_a55.h" */ -#endif - -#if defined(SYS_X86_64) && defined(SYS_X86_64_AVX2) -#include "x86_64/profiles/xkcp.h" -#endif - -#else /* !FIPS202_NATIVE_PROFILE && FIPS202_NATIVE_MANUAL */ - -/* Option 3: Build your own profile here, or via CFLAGS */ - -#endif /* !FIPS202_NATIVE_PROFILE && !FIPS202_NATIVE_MANUAL */ - -/* - * FIPS202 native interface - */ - -/* - * Those functions are meant to be trivial wrappers around - * the chosen native implementation. The are static inline - * to avoid unnecessary calls. - * The macro before each declaration controls whether a native - * implementation is present. - */ - -#if defined(MLKEM_USE_FIPS202_X1_NATIVE) -static INLINE void keccak_f1600_x1_native(uint64_t *state); -#endif -#if defined(MLKEM_USE_FIPS202_X2_NATIVE) -static INLINE void keccak_f1600_x2_native(uint64_t *state); -#endif -#if defined(MLKEM_USE_FIPS202_X4_NATIVE) -static INLINE void keccak_f1600_x4_native(uint64_t *state); -#endif - -#endif /* MLKEM_USE_NATIVE */ -#endif /* FIPS202_NATIVE_H */ diff --git a/fips202/native/x86_64/fips202_native_x86_64.h b/fips202/native/x86_64/fips202_native_x86_64.h deleted file mode 100644 index cbc0bb97d..000000000 --- a/fips202/native/x86_64/fips202_native_x86_64.h +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright (c) 2024 The mlkem-native project authors - * SPDX-License-Identifier: Apache-2.0 - */ -#ifndef FIPS202_X86_64_NATIVE_H -#define FIPS202_X86_64_NATIVE_H - -#include "config.h" - -#ifdef MLKEM_USE_NATIVE_X86_64 - -#include -#include -#include - -#include "xkcp/KeccakP-1600-times4-SnP.h" - -#endif /* MLKEM_USE_NATIVE_X86_64 */ - -#endif /* FIPS202_X86_64_NATIVE_H */ diff --git a/fips202/native/x86_64/xkcp/KeccakP-1600-times4-SIMD256.c b/fips202/native/x86_64/src/KeccakP-1600-times4-SIMD256.c similarity index 99% rename from fips202/native/x86_64/xkcp/KeccakP-1600-times4-SIMD256.c rename to fips202/native/x86_64/src/KeccakP-1600-times4-SIMD256.c index 9a36f17aa..d56823ab3 100644 --- a/fips202/native/x86_64/xkcp/KeccakP-1600-times4-SIMD256.c +++ b/fips202/native/x86_64/src/KeccakP-1600-times4-SIMD256.c @@ -24,8 +24,8 @@ and related or neighboring rights to the source code in this file. * Keccak states in memory. */ -#include "config.h" -#if defined(MLKEM_USE_NATIVE_X86_64) && defined(SYS_X86_64_AVX2) +#include "common.h" +#if defined(MLKEM_NATIVE_FIPS202_BACKEND_X86_64_XKCP) #include #include @@ -445,10 +445,9 @@ void KeccakP1600times4_PermuteAll_24rounds(void *states) } #else -#include "params.h" /* Dummy constant to keep compiler happy despite empty CU */ -#define empty_cu_avx2_keccakx4 MLKEM_NAMESPACE(empty_cu_avx2_keccakx4) +#define empty_cu_avx2_keccakx4 FIPS202_NAMESPACE(empty_cu_avx2_keccakx4) int empty_cu_avx2_keccakx4; -#endif /* MLKEM_USE_NATIVE_X86_64 && SYS_X86_64_AVX2 */ +#endif /* MLKEM_NATIVE_FIPS202_BACKEND_X86_64_XKCP */ diff --git a/fips202/native/x86_64/xkcp/KeccakP-1600-times4-SnP.h b/fips202/native/x86_64/src/KeccakP-1600-times4-SnP.h similarity index 97% rename from fips202/native/x86_64/xkcp/KeccakP-1600-times4-SnP.h rename to fips202/native/x86_64/src/KeccakP-1600-times4-SnP.h index 6d1951787..0738c9488 100644 --- a/fips202/native/x86_64/xkcp/KeccakP-1600-times4-SnP.h +++ b/fips202/native/x86_64/src/KeccakP-1600-times4-SnP.h @@ -26,7 +26,7 @@ and related or neighboring rights to the source code in this file. #include #include "KeccakP-SIMD256-config.h" -#include "namespace.h" +#include "common.h" #define KeccakP1600times4_statesAlignment 32 diff --git a/fips202/native/x86_64/xkcp/KeccakP-1600-unrolling.macros b/fips202/native/x86_64/src/KeccakP-1600-unrolling.macros similarity index 100% rename from fips202/native/x86_64/xkcp/KeccakP-1600-unrolling.macros rename to fips202/native/x86_64/src/KeccakP-1600-unrolling.macros diff --git a/fips202/native/x86_64/xkcp/KeccakP-SIMD256-config.h b/fips202/native/x86_64/src/KeccakP-SIMD256-config.h similarity index 100% rename from fips202/native/x86_64/xkcp/KeccakP-SIMD256-config.h rename to fips202/native/x86_64/src/KeccakP-SIMD256-config.h diff --git a/fips202/native/x86_64/xkcp/KeccakP-align.h b/fips202/native/x86_64/src/KeccakP-align.h similarity index 100% rename from fips202/native/x86_64/xkcp/KeccakP-align.h rename to fips202/native/x86_64/src/KeccakP-align.h diff --git a/fips202/native/x86_64/xkcp/KeccakP-brg_endian.h b/fips202/native/x86_64/src/KeccakP-brg_endian.h similarity index 100% rename from fips202/native/x86_64/xkcp/KeccakP-brg_endian.h rename to fips202/native/x86_64/src/KeccakP-brg_endian.h diff --git a/fips202/native/x86_64/profiles/xkcp.h b/fips202/native/x86_64/src/xkcp_impl.h similarity index 60% rename from fips202/native/x86_64/profiles/xkcp.h rename to fips202/native/x86_64/src/xkcp_impl.h index 4f320e994..c0ad76850 100644 --- a/fips202/native/x86_64/profiles/xkcp.h +++ b/fips202/native/x86_64/src/xkcp_impl.h @@ -5,14 +5,12 @@ /* Default FIPS202 assembly profile for AArch64 systems */ -#ifdef FIPS202_NATIVE_PROFILE_H +#ifdef MLKEM_NATIVE_FIPS202_PROFILE_IMPL_H #error Only one FIPS202 assembly profile can be defined -- did you include multiple profiles? #else -#define FIPS202_NATIVE_PROFILE_H +#define MLKEM_NATIVE_FIPS202_PROFILE_IMPL_H -#include "../fips202_native_x86_64.h" - -#if defined(MLKEM_USE_NATIVE_X86_64) && defined(SYS_X86_64_AVX2) +#include "KeccakP-1600-times4-SnP.h" #define MLKEM_USE_FIPS202_X4_NATIVE static INLINE void keccak_f1600_x4_native(uint64_t *state) @@ -20,6 +18,4 @@ static INLINE void keccak_f1600_x4_native(uint64_t *state) KeccakP1600times4_PermuteAll_24rounds(state); } -#endif /* MLKEM_USE_NATIVE_X86_64 && SYS_X86_64_AVX2 */ - -#endif /* FIPS202_NATIVE_PROFILE_H */ +#endif /* MLKEM_NATIVE_FIPS202_PROFILE_IMPL_H */ diff --git a/fips202/native/x86_64/xkcp.h b/fips202/native/x86_64/xkcp.h new file mode 100644 index 000000000..1cfcee9fd --- /dev/null +++ b/fips202/native/x86_64/xkcp.h @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2024 The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 + */ + +/* Default FIPS202 assembly profile for AArch64 systems */ + +#ifdef MLKEM_NATIVE_FIPS202_PROFILE_H +#error Only one FIPS202 assembly profile can be defined -- did you include multiple profiles? +#else +#define MLKEM_NATIVE_FIPS202_PROFILE_H + +/* Identifier for this backend so that source and assembly files + * in the build can be appropriately guarded. */ +#define MLKEM_NATIVE_FIPS202_BACKEND_X86_64_XKCP + +#define MLKEM_NATIVE_FIPS202_BACKEND_NAME X86_64_XKCP + +/* Filename of the C backend implementation. + * This is not inlined here because this header is included in assembly + * files as well. */ +#define MLKEM_NATIVE_FIPS202_BACKEND_IMPL "x86_64/src/xkcp_impl.h" + +#endif /* MLKEM_NATIVE_FIPS202_PROFILE_H */ diff --git a/mk/crypto.mk b/mk/crypto.mk index faf0eb047..9cb04227d 100644 --- a/mk/crypto.mk +++ b/mk/crypto.mk @@ -2,7 +2,7 @@ CPPFLAGS += -Ifips202 -Ifips202/native FIPS202_SRCS = $(wildcard fips202/*.c) ifeq ($(OPT),1) - FIPS202_SRCS += $(wildcard fips202/native/aarch64/*.S) $(wildcard fips202/native/x86_64/xkcp/*.c) + FIPS202_SRCS += $(wildcard fips202/native/aarch64/src/*.S) $(wildcard fips202/native/x86_64/src/*.c) endif $(BUILD_DIR)/libmlkem.a: $(call OBJS, $(FIPS202_SRCS)) diff --git a/mk/schemes.mk b/mk/schemes.mk index deabcdd86..ede00a635 100644 --- a/mk/schemes.mk +++ b/mk/schemes.mk @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 SOURCES += $(wildcard mlkem/*.c) $(wildcard mlkem/debug/*.c) ifeq ($(OPT),1) - SOURCES += $(wildcard mlkem/native/aarch64/*.[csS]) $(wildcard mlkem/native/x86_64/*.[csS]) + SOURCES += $(wildcard mlkem/native/aarch64/src/*.[csS]) $(wildcard mlkem/native/x86_64/src/*.[csS]) CPPFLAGS += -DMLKEM_USE_NATIVE endif diff --git a/mlkem/api.h b/mlkem/api.h deleted file mode 100644 index 94597323f..000000000 --- a/mlkem/api.h +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright (c) 2024 The mlkem-native project authors - * SPDX-License-Identifier: Apache-2.0 - */ -#ifndef API_H -#define API_H - -#include - -#define PQCP_MLKEM_NATIVE_MLKEM512_SECRETKEYBYTES 1632 -#define PQCP_MLKEM_NATIVE_MLKEM512_PUBLICKEYBYTES 800 -#define PQCP_MLKEM_NATIVE_MLKEM512_CIPHERTEXTBYTES 768 -#define PQCP_MLKEM_NATIVE_MLKEM512_KEYPAIRCOINBYTES 64 -#define PQCP_MLKEM_NATIVE_MLKEM512_ENCCOINBYTES 32 -#define PQCP_MLKEM_NATIVE_MLKEM512_BYTES 32 - -int PQCP_MLKEM_NATIVE_MLKEM512_keypair_derand(uint8_t *pk, uint8_t *sk, - const uint8_t *coins); -int PQCP_MLKEM_NATIVE_MLKEM512_keypair(uint8_t *pk, uint8_t *sk); -int PQCP_MLKEM_NATIVE_MLKEM512_enc_derand(uint8_t *ct, uint8_t *ss, - const uint8_t *pk, - const uint8_t *coins); -int PQCP_MLKEM_NATIVE_MLKEM512_enc(uint8_t *ct, uint8_t *ss, const uint8_t *pk); -int PQCP_MLKEM_NATIVE_MLKEM512_dec(uint8_t *ss, const uint8_t *ct, - const uint8_t *sk); - -#define PQCP_MLKEM_NATIVE_MLKEM768_SECRETKEYBYTES 2400 -#define PQCP_MLKEM_NATIVE_MLKEM768_PUBLICKEYBYTES 1184 -#define PQCP_MLKEM_NATIVE_MLKEM768_CIPHERTEXTBYTES 1088 -#define PQCP_MLKEM_NATIVE_MLKEM768_KEYPAIRCOINBYTES 64 -#define PQCP_MLKEM_NATIVE_MLKEM768_ENCCOINBYTES 32 -#define PQCP_MLKEM_NATIVE_MLKEM768_BYTES 32 - -int PQCP_MLKEM_NATIVE_MLKEM768_keypair_derand(uint8_t *pk, uint8_t *sk, - const uint8_t *coins); -int PQCP_MLKEM_NATIVE_MLKEM768_keypair(uint8_t *pk, uint8_t *sk); -int PQCP_MLKEM_NATIVE_MLKEM768_enc_derand(uint8_t *ct, uint8_t *ss, - const uint8_t *pk, - const uint8_t *coins); -int PQCP_MLKEM_NATIVE_MLKEM768_enc(uint8_t *ct, uint8_t *ss, const uint8_t *pk); -int PQCP_MLKEM_NATIVE_MLKEM768_dec(uint8_t *ss, const uint8_t *ct, - const uint8_t *sk); - -#define PQCP_MLKEM_NATIVE_MLKEM1024_SECRETKEYBYTES 3168 -#define PQCP_MLKEM_NATIVE_MLKEM1024_PUBLICKEYBYTES 1568 -#define PQCP_MLKEM_NATIVE_MLKEM1024_CIPHERTEXTBYTES 1568 -#define PQCP_MLKEM_NATIVE_MLKEM1024_KEYPAIRCOINBYTES 64 -#define PQCP_MLKEM_NATIVE_MLKEM1024_ENCCOINBYTES 32 -#define PQCP_MLKEM_NATIVE_MLKEM1024_BYTES 32 - -int PQCP_MLKEM_NATIVE_MLKEM1024_keypair_derand(uint8_t *pk, uint8_t *sk, - const uint8_t *coins); -int PQCP_MLKEM_NATIVE_MLKEM1024_keypair(uint8_t *pk, uint8_t *sk); -int PQCP_MLKEM_NATIVE_MLKEM1024_enc_derand(uint8_t *ct, uint8_t *ss, - const uint8_t *pk, - const uint8_t *coins); -int PQCP_MLKEM_NATIVE_MLKEM1024_enc(uint8_t *ct, uint8_t *ss, - const uint8_t *pk); -int PQCP_MLKEM_NATIVE_MLKEM1024_dec(uint8_t *ss, const uint8_t *ct, - const uint8_t *sk); - -#endif diff --git a/mlkem/arith_backend.h b/mlkem/arith_backend.h new file mode 100644 index 000000000..7fb3df84d --- /dev/null +++ b/mlkem/arith_backend.h @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2024 The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifdef MLKEM_NATIVE_ARITH_IMPL_H +#error Only one ARITH assembly profile can be defined -- did you include multiple profiles? +#else +#define MLKEM_NATIVE_ARITH_IMPL_H + +/* Include to enforce consistency of API and implementation */ +#include "native/api.h" + +#if defined(MLKEM_NATIVE_ARITH_BACKEND_IMPL) +#include MLKEM_NATIVE_ARITH_BACKEND_IMPL +#endif + +#endif /* MLKEM_NATIVE_ARITH_IMPL_H */ diff --git a/mlkem/cbd.c b/mlkem/cbd.c index 073f3c81d..2e0fac38a 100644 --- a/mlkem/cbd.c +++ b/mlkem/cbd.c @@ -4,7 +4,6 @@ */ #include "cbd.h" #include -#include "params.h" /************************************************* * Name: load32_littleendian diff --git a/mlkem/cbd.h b/mlkem/cbd.h index 4dc8635bb..31c9649e3 100644 --- a/mlkem/cbd.h +++ b/mlkem/cbd.h @@ -6,7 +6,7 @@ #define CBD_H #include -#include "params.h" +#include "common.h" #include "poly.h" #define poly_cbd_eta1 MLKEM_NAMESPACE(poly_cbd_eta1) diff --git a/mlkem/common.h b/mlkem/common.h index 94c29ed92..0b379d9bd 100644 --- a/mlkem/common.h +++ b/mlkem/common.h @@ -2,64 +2,21 @@ * Copyright (c) 2024 The mlkem-native project authors * SPDX-License-Identifier: Apache-2.0 */ -#ifndef COMMON_H -#define COMMON_H +#ifndef MLKEM_NATIVE_COMMON_H +#define MLKEM_NATIVE_COMMON_H +#include "config.h" +#include "params.h" +#include "sys.h" -/* - * C90 does not have the inline compiler directive yet. - * We don't use it in C90 builds. - * However, in that case the compiler warns about some inline functions in - * header files not being used in every compilation unit that includes that - * header. To work around it we silence that warning in that case using - * __attribute__((unused)). - */ - -/* Do not use inline for C90 builds*/ -#if !defined(inline) -#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L -#define INLINE inline -#define ALWAYS_INLINE __attribute__((always_inline)) -#elif defined(_MSC_VER) -#define INLINE __inline -#define ALWAYS_INLINE __forceinline -#else -#define INLINE __attribute__((unused)) -#define ALWAYS_INLINE -#endif - -#else -#define INLINE inline -#define ALWAYS_INLINE __attribute__((always_inline)) -#endif - - -/* - * C90 does not have the restrict compiler directive yet. - * We don't use it in C90 builds. - */ -#if !defined(restrict) -#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L -#define RESTRICT restrict -#else -#define RESTRICT -#endif - -#else - -#define RESTRICT restrict -#endif - -#define DEFAULT_ALIGN 32 -#if defined(_WIN32) -#define ALIGN __declspec(align(DEFAULT_ALIGN)) -#define asm __asm -#else -#define asm __asm__ -#define ALIGN __attribute__((aligned(DEFAULT_ALIGN))) +/* Include backend metadata */ +#if defined(MLKEM_USE_NATIVE) +#include MLKEM_NATIVE_ARITH_BACKEND +#include MLKEM_NATIVE_FIPS202_BACKEND #endif -#define MLKEM_CONCAT_(left, right) left##right -#define MLKEM_CONCAT(left, right) MLKEM_CONCAT_(left, right) +/* This must come after the inclusion of the backend metadata + * since the backend choice may be part of the namespace. */ +#include "namespace.h" -#endif +#endif /* MLKEM_NATIVE_COMMON_H */ diff --git a/mlkem/config.h b/mlkem/config.h new file mode 100644 index 000000000..198b08977 --- /dev/null +++ b/mlkem/config.h @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2024 The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef MLKEM_NATIVE_CONFIG_H +#define MLKEM_NATIVE_CONFIG_H + +/****************************************************************************** + * Name: MLKEM_K + * + * Description: Determines the security level for ML-KEM + * - MLKEM_K=2 corresponds to ML-KEM-512 + * - MLKEM_K=3 corresponds to ML-KEM-768 + * - MLKEM_K=4 corresponds to ML-KEM-1024 + * + * This can also be set using CFLAGS. + * + *****************************************************************************/ +#ifndef MLKEM_K +#define MLKEM_K 3 /* Change this for different security strengths */ +#endif + +/****************************************************************************** + * Name: MLKEM_NAMESPACE + * _MLKEM_NAMESPACE + * + * Description: The macros to use to namespace global symbols + * from mlkem/. + *****************************************************************************/ +#define MLKEM_NAMESPACE(sym) MLKEM_DEFAULT_NAMESPACE(sym) +#define _MLKEM_NAMESPACE(sym) _MLKEM_DEFAULT_NAMESPACE(sym) + +/****************************************************************************** + * Name: FIPS202_NAMESPACE + * _FIPS202_NAMESPACE + * + * Description: The macros to use to namespace global symbols + * from fips202/. + *****************************************************************************/ +#define FIPS202_NAMESPACE(sym) FIPS202_DEFAULT_NAMESPACE(sym) +#define _FIPS202_NAMESPACE(sym) _FIPS202_DEFAULT_NAMESPACE(sym) + +/****************************************************************************** + * Name: MLKEM_USE_NATIVE + * + * Description: Determines whether a native backend should + * be used, if available. + * + * This can also be set using CFLAGS. + * + *****************************************************************************/ +#if !defined(MLKEM_USE_NATIVE) +/* #define MLKEM_USE_NATIVE */ +#endif + +/****************************************************************************** + * Name: MLKEM_NATIVE_ARITH_BACKEND + * + * Description: The arithmetic backend to use. + * + * This must be the filename of an arithmetic + * backend. The backend is expected to define + * + * - MLKEM_NATIVE_ARITH_BACKEND_NAME + * + * The name of the backend as used in the default namespace. + * + * - MLKEM_NATIVE_ARITH_BACKEND_IMPL + * + * The filename of the implementation of the arithmetic backend. + * + * See the existing backends for more information. + * + *****************************************************************************/ +#if !defined(MLKEM_NATIVE_ARITH_BACKEND) +#define MLKEM_NATIVE_ARITH_BACKEND "native/default.h" +#endif /* MLKEM_NATIVE_ARITH_BACKEND */ + +/****************************************************************************** + * Name: MLKEM_NATIVE_FIPS202_BACKEND + * + * Description: The FIPS-202 backend to use. + * + * This must be the filename of an FIPS-202 + * backend. The backend is expected to define + * + * - MLKEM_NATIVE_FIPS202_BACKEND_NAME + * + * The name of the backend as used in the default namespace. + * + * - MLKEM_NATIVE_FIPS202_BACKEND_IMPL + * + * The filename of the implementation of the FIPS-202 backend. + * + * See the existing backends for more information. + * + *****************************************************************************/ +#if !defined(MLKEM_NATIVE_FIPS202_BACKEND) +#define MLKEM_NATIVE_FIPS202_BACKEND "../fips202/native/default.h" +#endif /* MLKEM_NATIVE_FIPS202_BACKEND */ + +#endif /* MLkEM_NATIVE_CONFIG_H */ diff --git a/mlkem/debug/debug.c b/mlkem/debug/debug.c index 0dc2e8822..690977908 100644 --- a/mlkem/debug/debug.c +++ b/mlkem/debug/debug.c @@ -52,7 +52,6 @@ void mlkem_debug_print_error(const char *file, int line, const char *msg) } #else /* MLKEM_DEBUG */ -#include "params.h" #define empty_cu_debug MLKEM_NAMESPACE(empty_cu_debug) int empty_cu_debug; diff --git a/mlkem/debug/debug.h b/mlkem/debug/debug.h index 65208771d..5838ae4bf 100644 --- a/mlkem/debug/debug.h +++ b/mlkem/debug/debug.h @@ -5,6 +5,8 @@ #ifndef MLKEM_DEBUG_H #define MLKEM_DEBUG_H +#include "common.h" + #if defined(MLKEM_DEBUG) #include #include @@ -153,6 +155,9 @@ void mlkem_debug_print_error(const char *file, int line, const char *msg); "polyvec unsigned bound for " #ptr ".vec[i]"); \ } while (0) +#define MLKEM_CONCAT_(left, right) left##right +#define MLKEM_CONCAT(left, right) MLKEM_CONCAT_(left, right) + /* Following AWS-LC to define a C99-compliant static assert */ #define MLKEM_STATIC_ASSERT_DEFINE(cond, msg) \ typedef struct \ diff --git a/mlkem/indcpa.c b/mlkem/indcpa.c index bde832c18..ee2f140a0 100644 --- a/mlkem/indcpa.c +++ b/mlkem/indcpa.c @@ -10,14 +10,13 @@ #include "fips202x4.h" #include "indcpa.h" #include "ntt.h" -#include "params.h" #include "poly.h" #include "polyvec.h" #include "randombytes.h" #include "rej_uniform.h" #include "symmetric.h" -#include "arith_native.h" +#include "arith_backend.h" #include "debug/debug.h" #include "cbmc.h" diff --git a/mlkem/indcpa.h b/mlkem/indcpa.h index 0e6485312..7e2a0b247 100644 --- a/mlkem/indcpa.h +++ b/mlkem/indcpa.h @@ -7,10 +7,9 @@ #include #include "cbmc.h" -#include "params.h" +#include "common.h" #include "polyvec.h" - #define gen_matrix MLKEM_NAMESPACE(gen_matrix) /************************************************* * Name: gen_matrix diff --git a/mlkem/kem.c b/mlkem/kem.c index 1abce65e4..03e997af3 100644 --- a/mlkem/kem.c +++ b/mlkem/kem.c @@ -7,7 +7,6 @@ #include #include #include "indcpa.h" -#include "params.h" #include "randombytes.h" #include "symmetric.h" #include "verify.h" diff --git a/mlkem/namespace.h b/mlkem/namespace.h new file mode 100644 index 000000000..8c409fb0c --- /dev/null +++ b/mlkem/namespace.h @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2024 The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 + */ +#ifndef MLKEM_NATIVE_NAMESPACE_H +#define MLKEM_NATIVE_NAMESPACE_H + +#if !defined(MLKEM_NATIVE_ARITH_BACKEND_NAME) +#define MLKEM_NATIVE_ARITH_BACKEND_NAME C +#endif + +/* Don't change parameters below this line */ +#if (MLKEM_K == 2) +#define MLKEM_PARAM_NAME MLKEM512 +#elif (MLKEM_K == 3) +#define MLKEM_PARAM_NAME MLKEM768 +#elif (MLKEM_K == 4) +#define MLKEM_PARAM_NAME MLKEM1024 +#else +#error "MLKEM_K must be in {2,3,4}" +#endif + +#define ___MLKEM_DEFAULT_NAMESPACE(x1, x2, x3, x4) x1##_##x2##_##x3##_##x4 +#define __MLKEM_DEFAULT_NAMESPACE(x1, x2, x3, x4) \ + ___MLKEM_DEFAULT_NAMESPACE(x1, x2, x3, x4) + +/* + * NAMESPACE is PQCP_MLKEM_NATIVE___ + * e.g., PQCP_MLKEM_NATIVE_MLKEM512_AARCH64_OPT_ + */ +#define MLKEM_DEFAULT_NAMESPACE(s) \ + __MLKEM_DEFAULT_NAMESPACE(PQCP_MLKEM_NATIVE, MLKEM_PARAM_NAME, \ + MLKEM_NATIVE_ARITH_BACKEND_NAME, s) +#define _MLKEM_DEFAULT_NAMESPACE(s) \ + __MLKEM_DEFAULT_NAMESPACE(_PQCP_MLKEM_NATIVE, MLKEM_PARAM_NAME, \ + MLKEM_NATIVE_ARITH_BACKEND_NAME, s) + +#if !defined(MLKEM_NATIVE_FIPS202_BACKEND_NAME) +#define MLKEM_NATIVE_FIPS202_BACKEND_NAME C +#endif + +#define ___FIPS202_DEFAULT_NAMESPACE(x1, x2, x3) x1##_##x2##_##x3 +#define __FIPS202_DEFAULT_NAMESPACE(x1, x2, x3) \ + ___FIPS202_DEFAULT_NAMESPACE(x1, x2, x3) + +/* + * NAMESPACE is PQCP_MLKEM_NATIVE_FIPS202__ + * e.g., PQCP_MLKEM_NATIVE_FIPS202_X86_64_XKCP_ + */ +#define FIPS202_DEFAULT_NAMESPACE(s) \ + __FIPS202_DEFAULT_NAMESPACE(PQCP_MLKEM_NATIVE_FIPS202, \ + MLKEM_NATIVE_FIPS202_BACKEND_NAME, s) +#define _FIPS202_DEFAULT_NAMESPACE(s) \ + __FIPS202_DEFAULT_NAMESPACE(_PQCP_MLKEM_NATIVE_FIPS202, \ + MLKEM_NATIVE_FIPS202_BACKEND_NAME, s) + +#endif /* MLKEM_NATIVE_NAMESPACE_H */ diff --git a/mlkem/native/aarch64/README.md b/mlkem/native/aarch64/README.md index c21e27f36..e499a4a22 100644 --- a/mlkem/native/aarch64/README.md +++ b/mlkem/native/aarch64/README.md @@ -16,4 +16,4 @@ This backend comes with two profiles: "clean" and optimized. The "clean" backend read and modify; for example, is heavily leverages register aliases and assembly macros. The optimized profile is automatically generated from the clean profile via [SLOTHY](https://github.com/slothy-optimizer/slothy). Currently, the target architecture is Cortex-A55, but you can easily re-optimize the code for a different microarchitecture supported -by SLOTHY, by adjusting the parameters in [optimize.sh](optimize.sh). +by SLOTHY, by adjusting the parameters in [optimize.sh](src/optimize.sh). diff --git a/mlkem/native/aarch64/clean.h b/mlkem/native/aarch64/clean.h new file mode 100644 index 000000000..43a401dfc --- /dev/null +++ b/mlkem/native/aarch64/clean.h @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2024 The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 + */ + +/* ML-KEM arithmetic native profile for clean assembly */ + +#ifdef MLKEM_NATIVE_ARITH_PROFILE_H +#error Only one MLKEM_ARITH assembly profile can be defined -- did you include multiple profiles? +#else +#define MLKEM_NATIVE_ARITH_PROFILE_H + +/* Identifier for this backend so that source and assembly files + * in the build can be appropriately guarded. */ +#define MLKEM_NATIVE_ARITH_BACKEND_AARCH64_CLEAN + +#define MLKEM_NATIVE_ARITH_BACKEND_NAME AARCH64_CLEAN + +/* Filename of the C backend implementation. + * This is not inlined here because this header is included in assembly + * files as well. */ +#define MLKEM_NATIVE_ARITH_BACKEND_IMPL "aarch64/src/clean_impl.h" + +#endif /* MLKEM_NATIVE_ARITH_PROFILE_H */ diff --git a/mlkem/native/aarch64/opt.h b/mlkem/native/aarch64/opt.h new file mode 100644 index 000000000..04323c3e7 --- /dev/null +++ b/mlkem/native/aarch64/opt.h @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2024 The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 + */ + +/* ML-KEM arithmetic native profile for clean assembly */ + +#ifdef MLKEM_NATIVE_ARITH_PROFILE_H +#error Only one MLKEM_ARITH assembly profile can be defined -- did you include multiple profiles? +#else +#define MLKEM_NATIVE_ARITH_PROFILE_H + +/* Identifier for this backend so that source and assembly files + * in the build can be appropriately guarded. */ +#define MLKEM_NATIVE_ARITH_BACKEND_AARCH64_OPT + +#define MLKEM_NATIVE_ARITH_BACKEND_NAME AARCH64_OPT + +/* Filename of the C backend implementation. + * This is not inlined here because this header is included in assembly + * files as well. */ +#define MLKEM_NATIVE_ARITH_BACKEND_IMPL "aarch64/src/opt_impl.h" + +#endif /* MLKEM_NATIVE_ARITH_PROFILE_H */ diff --git a/mlkem/native/aarch64/aarch64_zetas.c b/mlkem/native/aarch64/src/aarch64_zetas.c similarity index 98% rename from mlkem/native/aarch64/aarch64_zetas.c rename to mlkem/native/aarch64/src/aarch64_zetas.c index f0a0d5546..ecf1b529a 100644 --- a/mlkem/native/aarch64/aarch64_zetas.c +++ b/mlkem/native/aarch64/src/aarch64_zetas.c @@ -8,9 +8,13 @@ * Do not modify it directly. */ -#include "arith_native_aarch64.h" +#include "common.h" + +#if defined(MLKEM_NATIVE_ARITH_BACKEND_AARCH64_CLEAN) || \ + defined(MLKEM_NATIVE_ARITH_BACKEND_AARCH64_OPT) -#ifdef MLKEM_USE_NATIVE_AARCH64 +#include +#include "arith_native_aarch64.h" /* * Table of zeta values used in the AArch64 forward NTT @@ -163,10 +167,10 @@ const int16_t aarch64_zetas_mulcache_twisted_native[] = { -11566, 11566, }; -#else /* MLKEM_USE_NATIVE_AARCH64 */ +#else #include "params.h" /* Dummy declaration for compilers disliking empty compilation units */ #define empty_cu_aarch64_zetas MLKEM_NAMESPACE(empty_cu_aarch64_zetas) int empty_cu_aarch64_zetas; -#endif /* MLKEM_USE_NATIVE_AARCH64 */ +#endif diff --git a/mlkem/native/aarch64/arith_native_aarch64.h b/mlkem/native/aarch64/src/arith_native_aarch64.h similarity index 96% rename from mlkem/native/aarch64/arith_native_aarch64.h rename to mlkem/native/aarch64/src/arith_native_aarch64.h index f39eb9ac8..2f3b0ef4f 100644 --- a/mlkem/native/aarch64/arith_native_aarch64.h +++ b/mlkem/native/aarch64/src/arith_native_aarch64.h @@ -6,10 +6,7 @@ #define MLKEM_AARCH64_NATIVE_H #include -#include "config.h" -#include "params.h" - -#ifdef MLKEM_USE_NATIVE_AARCH64 +#include "common.h" #define aarch64_ntt_zetas_layer01234 \ MLKEM_NAMESPACE(aarch64_ntt_zetas_layer01234) @@ -72,6 +69,9 @@ void poly_mulcache_compute_asm_opt(int16_t *, const int16_t *, const int16_t *, #define poly_tobytes_asm_clean MLKEM_NAMESPACE(poly_tobytes_asm_clean) void poly_tobytes_asm_clean(uint8_t *r, const int16_t *a); +#define poly_tobytes_asm_opt MLKEM_NAMESPACE(poly_tobytes_asm_opt) +void poly_tobytes_asm_opt(uint8_t *r, const int16_t *a); + #define polyvec_basemul_acc_montgomery_cached_asm_clean \ MLKEM_NAMESPACE(polyvec_basemul_acc_montgomery_cached_asm_clean) void polyvec_basemul_acc_montgomery_cached_asm_clean(int16_t *r, @@ -85,6 +85,4 @@ void polyvec_basemul_acc_montgomery_cached_asm_opt(int16_t *r, const int16_t *a, const int16_t *b, const int16_t *b_cache); - -#endif /* MLKEM_USE_NATIVE_AARCH64 */ #endif /* MLKEM_AARCH64_NATIVE_H */ diff --git a/mlkem/native/aarch64/profiles/clean.h b/mlkem/native/aarch64/src/clean_impl.h similarity index 90% rename from mlkem/native/aarch64/profiles/clean.h rename to mlkem/native/aarch64/src/clean_impl.h index e7f71bd53..0a40199e7 100644 --- a/mlkem/native/aarch64/profiles/clean.h +++ b/mlkem/native/aarch64/src/clean_impl.h @@ -5,14 +5,14 @@ /* ML-KEM arithmetic native profile for clean assembly */ -#ifdef MLKEM_ARITH_NATIVE_PROFILE_H +#ifdef MLKEM_NATIVE_ARITH_PROFILE_IMPL_H #error Only one MLKEM_ARITH assembly profile can be defined -- did you include multiple profiles? #else -#define MLKEM_ARITH_NATIVE_PROFILE_H +#define MLKEM_NATIVE_ARITH_PROFILE_IMPL_H -#include "../../arith_native.h" -#include "../arith_native_aarch64.h" +#include "arith_native_aarch64.h" +/* Set of primitives that this backend replaces */ #define MLKEM_USE_NATIVE_NTT #define MLKEM_USE_NATIVE_INTT #define MLKEM_USE_NATIVE_POLY_REDUCE @@ -22,7 +22,6 @@ #define MLKEM_USE_NATIVE_POLY_TOBYTES #define MLKEM_USE_NATIVE_REJ_UNIFORM -#define NTT_BOUND_NATIVE (6 * MLKEM_Q) static INLINE void ntt_native(poly *data) { ntt_asm_clean(data->coeffs, aarch64_ntt_zetas_layer01234, @@ -75,4 +74,4 @@ static INLINE int rej_uniform_native(int16_t *r, unsigned int len, return (int)rej_uniform_asm_clean(r, buf, buflen); } -#endif /* MLKEM_ARITH_NATIVE_PROFILE_H */ +#endif /* MLKEM_NATIVE_ARITH_PROFILE_IMPL_H */ diff --git a/mlkem/native/aarch64/common.i b/mlkem/native/aarch64/src/common.i similarity index 100% rename from mlkem/native/aarch64/common.i rename to mlkem/native/aarch64/src/common.i diff --git a/mlkem/native/aarch64/consts.h b/mlkem/native/aarch64/src/consts.h similarity index 96% rename from mlkem/native/aarch64/consts.h rename to mlkem/native/aarch64/src/consts.h index 0e834bed4..c40947299 100644 --- a/mlkem/native/aarch64/consts.h +++ b/mlkem/native/aarch64/src/consts.h @@ -7,7 +7,7 @@ #define MLKEM_NATIVE_AARCH64_CONSTS #include -#include "params.h" +#include "common.h" #define zetas_mulcache_native MLKEM_NAMESPACE(zetas_mulcache_native) extern const int16_t zetas_mulcache_native[256]; diff --git a/mlkem/native/aarch64/intt_clean.S b/mlkem/native/aarch64/src/intt_clean.S similarity index 98% rename from mlkem/native/aarch64/intt_clean.S rename to mlkem/native/aarch64/src/intt_clean.S index 2ac4e0f15..2f05d8cca 100644 --- a/mlkem/native/aarch64/intt_clean.S +++ b/mlkem/native/aarch64/src/intt_clean.S @@ -23,8 +23,8 @@ /// SOFTWARE. /// -#include "config.h" -#if defined(MLKEM_USE_NATIVE_AARCH64) +#include "common.h" +#if defined(MLKEM_NATIVE_ARITH_BACKEND_AARCH64_CLEAN) // Needed to provide ASM_LOAD directive #include "common.i" @@ -371,4 +371,4 @@ layer012_start: pop_stack ret -#endif /* MLKEM_USE_NATIVE_AARCH64 */ +#endif /* MLKEM_NATIVE_ARITH_BACKEND_AARCH64_CLEAN */ diff --git a/mlkem/native/aarch64/intt_opt.S b/mlkem/native/aarch64/src/intt_opt.S similarity index 99% rename from mlkem/native/aarch64/intt_opt.S rename to mlkem/native/aarch64/src/intt_opt.S index 48c412f36..fc720e504 100644 --- a/mlkem/native/aarch64/intt_opt.S +++ b/mlkem/native/aarch64/src/intt_opt.S @@ -23,8 +23,8 @@ /// SOFTWARE. /// -#include "config.h" -#if defined(MLKEM_USE_NATIVE_AARCH64) +#include "common.h" +#if defined(MLKEM_NATIVE_ARITH_BACKEND_AARCH64_OPT) // Needed to provide ASM_LOAD directive #include "common.i" @@ -1027,4 +1027,4 @@ layer012_start: pop_stack ret -#endif /* MLKEM_USE_NATIVE_AARCH64 */ +#endif /* MLKEM_NATIVE_ARITH_BACKEND_AARCH64_OPT */ diff --git a/mlkem/native/aarch64/ntt_clean.S b/mlkem/native/aarch64/src/ntt_clean.S similarity index 98% rename from mlkem/native/aarch64/ntt_clean.S rename to mlkem/native/aarch64/src/ntt_clean.S index 4243b9431..ad9086e27 100644 --- a/mlkem/native/aarch64/ntt_clean.S +++ b/mlkem/native/aarch64/src/ntt_clean.S @@ -24,8 +24,8 @@ /// SOFTWARE. /// -#include "config.h" -#if defined(MLKEM_USE_NATIVE_AARCH64) +#include "common.h" +#if defined(MLKEM_NATIVE_ARITH_BACKEND_AARCH64_CLEAN) // Needed to provide ASM_LOAD directive #include "common.i" @@ -295,4 +295,4 @@ layer3456_start: pop_stack ret -#endif /* MLKEM_USE_NATIVE_AARCH64 */ +#endif /* MLKEM_NATIVE_ARITH_BACKEND_AARCH64_CLEAN */ diff --git a/mlkem/native/aarch64/ntt_opt.S b/mlkem/native/aarch64/src/ntt_opt.S similarity index 99% rename from mlkem/native/aarch64/ntt_opt.S rename to mlkem/native/aarch64/src/ntt_opt.S index 71779afe9..f4cbc928f 100644 --- a/mlkem/native/aarch64/ntt_opt.S +++ b/mlkem/native/aarch64/src/ntt_opt.S @@ -24,8 +24,8 @@ /// SOFTWARE. /// -#include "config.h" -#if defined(MLKEM_USE_NATIVE_AARCH64) +#include "common.h" +#if defined(MLKEM_NATIVE_ARITH_BACKEND_AARCH64_OPT) // Needed to provide ASM_LOAD directive #include "common.i" @@ -892,4 +892,4 @@ layer3456_start: pop_stack ret -#endif /* MLKEM_USE_NATIVE_AARCH64 */ +#endif /* MLKEM_NATIVE_ARITH_BACKEND_AARCH64_OPT */ diff --git a/mlkem/native/aarch64/profiles/opt.h b/mlkem/native/aarch64/src/opt_impl.h similarity index 89% rename from mlkem/native/aarch64/profiles/opt.h rename to mlkem/native/aarch64/src/opt_impl.h index 2414700bf..bc9af0dee 100644 --- a/mlkem/native/aarch64/profiles/opt.h +++ b/mlkem/native/aarch64/src/opt_impl.h @@ -5,14 +5,14 @@ /* ML-KEM arithmetic native profile for clean assembly */ -#ifdef MLKEM_ARITH_NATIVE_PROFILE_H +#ifdef MLKEM_NATIVE_ARITH_PROFILE_IMPL_H #error Only one MLKEM_ARITH assembly profile can be defined -- did you include multiple profiles? #else -#define MLKEM_ARITH_NATIVE_PROFILE_H +#define MLKEM_NATIVE_ARITH_PROFILE_IMPL_H -#include "../../arith_native.h" -#include "../arith_native_aarch64.h" +#include "arith_native_aarch64.h" +/* Set of primitives that this backend replaces */ #define MLKEM_USE_NATIVE_NTT #define MLKEM_USE_NATIVE_INTT #define MLKEM_USE_NATIVE_POLY_REDUCE @@ -62,7 +62,7 @@ static INLINE void polyvec_basemul_acc_montgomery_cached_native( static INLINE void poly_tobytes_native(uint8_t r[MLKEM_POLYBYTES], const poly *a) { - poly_tobytes_asm_clean(r, a->coeffs); + poly_tobytes_asm_opt(r, a->coeffs); } static INLINE int rej_uniform_native(int16_t *r, unsigned int len, @@ -75,4 +75,4 @@ static INLINE int rej_uniform_native(int16_t *r, unsigned int len, return (int)rej_uniform_asm_clean(r, buf, buflen); } -#endif /* MLKEM_ARITH_NATIVE_PROFILE_H */ +#endif /* MLKEM_NATIVE_ARITH_PROFILE_IMPL_H */ diff --git a/mlkem/native/aarch64/optimize.sh b/mlkem/native/aarch64/src/optimize.sh similarity index 100% rename from mlkem/native/aarch64/optimize.sh rename to mlkem/native/aarch64/src/optimize.sh diff --git a/mlkem/native/aarch64/poly_clean.S b/mlkem/native/aarch64/src/poly_clean.S similarity index 98% rename from mlkem/native/aarch64/poly_clean.S rename to mlkem/native/aarch64/src/poly_clean.S index 86e4e494a..3e1bc5cf4 100644 --- a/mlkem/native/aarch64/poly_clean.S +++ b/mlkem/native/aarch64/src/poly_clean.S @@ -3,8 +3,8 @@ * SPDX-License-Identifier: Apache-2.0 */ -#include "config.h" -#if defined(MLKEM_USE_NATIVE_AARCH64) +#include "common.h" +#if defined(MLKEM_NATIVE_ARITH_BACKEND_AARCH64_CLEAN) // Needed to provide ASM_LOAD directive #include "common.i" @@ -285,4 +285,4 @@ poly_tomont_asm_loop: ret -#endif /* MLKEM_USE_NATIVE_AARCH64 */ +#endif /* MLKEM_NATIVE_ARITH_BACKEND_AARCH64_CLEAN */ diff --git a/mlkem/native/aarch64/poly_opt.S b/mlkem/native/aarch64/src/poly_opt.S similarity index 99% rename from mlkem/native/aarch64/poly_opt.S rename to mlkem/native/aarch64/src/poly_opt.S index d86595059..df3b21008 100644 --- a/mlkem/native/aarch64/poly_opt.S +++ b/mlkem/native/aarch64/src/poly_opt.S @@ -3,8 +3,8 @@ * SPDX-License-Identifier: Apache-2.0 */ -#include "config.h" -#if defined(MLKEM_USE_NATIVE_AARCH64) +#include "common.h" +#if defined(MLKEM_NATIVE_ARITH_BACKEND_AARCH64_OPT) // Needed to provide ASM_LOAD directive #include "common.i" @@ -649,4 +649,4 @@ poly_tomont_asm_loop: ret -#endif /* MLKEM_USE_NATIVE_AARCH64 */ +#endif /* MLKEM_NATIVE_ARITH_BACKEND_AARCH64_OPT */ diff --git a/mlkem/native/aarch64/polyvec_clean.S b/mlkem/native/aarch64/src/polyvec_clean.S similarity index 98% rename from mlkem/native/aarch64/polyvec_clean.S rename to mlkem/native/aarch64/src/polyvec_clean.S index 59f57d77a..bfd1d2b8a 100644 --- a/mlkem/native/aarch64/polyvec_clean.S +++ b/mlkem/native/aarch64/src/polyvec_clean.S @@ -9,8 +9,8 @@ // https://eprint.iacr.org/2021/986 // https://github.com/neon-ntt/neon-ntt -#include "config.h" -#if defined(MLKEM_USE_NATIVE_AARCH64) +#include "common.h" +#if defined(MLKEM_NATIVE_ARITH_BACKEND_AARCH64_CLEAN) #include "params.h" @@ -279,4 +279,4 @@ k4_loop_start: ret #endif /* MLKEM_K == 4 */ -#endif /* MLKEM_USE_NATIVE_AARCH64 */ +#endif /* MLKEM_NATIVE_ARITH_BACKEND_AARCH64_CLEAN */ diff --git a/mlkem/native/aarch64/polyvec_opt.S b/mlkem/native/aarch64/src/polyvec_opt.S similarity index 99% rename from mlkem/native/aarch64/polyvec_opt.S rename to mlkem/native/aarch64/src/polyvec_opt.S index 9ae035639..07dc98efd 100644 --- a/mlkem/native/aarch64/polyvec_opt.S +++ b/mlkem/native/aarch64/src/polyvec_opt.S @@ -6,12 +6,11 @@ // AArch64 re-implementation of the asymmetric base multiplication from: // Neon NTT: Faster Dilithium, Kyber, and Saber on Cortex-A72 and Apple M1 -// https: // eprint.iacr.org/2021/986 -// https: // github.com/neon-ntt/neon-ntt - -#include "config.h" -#if defined(MLKEM_USE_NATIVE_AARCH64) +// https://eprint.iacr.org/2021/986 +// https://github.com/neon-ntt/neon-ntt +#include "common.h" +#if defined(MLKEM_NATIVE_ARITH_BACKEND_AARCH64_OPT) #include "params.h" // Needed to provide ASM_LOAD directive @@ -1122,4 +1121,4 @@ k4_loop_start: ret #endif /* MLKEM_K == 4 */ -#endif /* MLKEM_USE_NATIVE_AARCH64 */ +#endif /* MLKEM_NATIVE_ARITH_BACKEND_AARCH64_OPT */ diff --git a/mlkem/native/aarch64/rej_uniform_asm_clean.S b/mlkem/native/aarch64/src/rej_uniform_asm_clean.S similarity index 99% rename from mlkem/native/aarch64/rej_uniform_asm_clean.S rename to mlkem/native/aarch64/src/rej_uniform_asm_clean.S index 00332ec65..c51e53188 100644 --- a/mlkem/native/aarch64/rej_uniform_asm_clean.S +++ b/mlkem/native/aarch64/src/rej_uniform_asm_clean.S @@ -18,9 +18,9 @@ * * Returns number of sampled 16-bit integers (at most MLKEM_N). **************************************************/ -#include "config.h" -#include "params.h" -#if defined(MLKEM_USE_NATIVE_AARCH64) +#include "common.h" +#if defined(MLKEM_NATIVE_ARITH_BACKEND_AARCH64_CLEAN) || \ + defined(MLKEM_NATIVE_ARITH_BACKEND_AARCH64_OPT) // Needed to provide ASM_LOAD directive #include "common.i" @@ -605,4 +605,5 @@ return: pop_stack ret -#endif /* MLKEM_USE_NATIVE_AARCH64 */ +#endif /* defined(MLKEM_NATIVE_ARITH_BACKEND_AARCH64_CLEAN) || + defined(MLKEM_NATIVE_ARITH_BACKEND_AARCH64_OPT) */ diff --git a/mlkem/native/arith_native.h b/mlkem/native/api.h similarity index 98% rename from mlkem/native/arith_native.h rename to mlkem/native/api.h index b7e921323..5a2d92b1d 100644 --- a/mlkem/native/arith_native.h +++ b/mlkem/native/api.h @@ -6,14 +6,12 @@ #define MLKEM_ARITH_NATIVE_H #include -#include "config.h" -#include "params.h" - -#if defined(MLKEM_USE_NATIVE) +#include "cbmc.h" #include "poly.h" #include "polyvec.h" -#include "profile.h" + +#include "common.h" /* * MLKEM native arithmetic interface @@ -246,5 +244,4 @@ static INLINE int rej_uniform_native(int16_t *r, unsigned int len, const uint8_t *buf, unsigned int buflen); #endif /* MLKEM_USE_NATIVE_REJ_UNIFORM */ -#endif /* MLKEM_USE_NATIVE */ #endif /* MLKEM_ARITH_NATIVE_H */ diff --git a/mlkem/native/default.h b/mlkem/native/default.h new file mode 100644 index 000000000..d1e41c52e --- /dev/null +++ b/mlkem/native/default.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2024 The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 + */ +#ifndef MLKEM_NATIVE_ARITH_BACKEND_DEFAULT_H +#define MLKEM_NATIVE_ARITH_BACKEND_DEFAULT_H + +/* + * Default arithmetic backend + */ +#include "sys.h" + +#ifdef SYS_AARCH64 +/* + * For AArch64, we currently we have one clean and one opt profile. + * We default to the opt profile. + * + * In the future, this may branch further depending on the microarchitecture. + */ +#include "aarch64/opt.h" +#endif /* SYS_AARCH64 */ + +#ifdef SYS_X86_64_AVX2 +/* + * For now, there's only one x86_64 profile, based on + * the AVX2 code from the Kyber repository. + * https://github.com/pq-crystals/kyber + */ +#include "x86_64/default.h" +#endif /* SYS_X86_64 */ + +#endif /* MLKEM_NATIVE_ARITH_BACKEND_DEFAULT_H */ diff --git a/mlkem/native/profile.h b/mlkem/native/profile.h deleted file mode 100644 index ab9d93aa1..000000000 --- a/mlkem/native/profile.h +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright (c) 2024 The mlkem-native project authors - * SPDX-License-Identifier: Apache-2.0 - */ -#ifndef MLKEM_ARITH_NATIVE_PROFILE_CHOICE_H -#define MLKEM_ARITH_NATIVE_PROFILE_CHOICE_H - -#include -#include "config.h" -#include "params.h" - -#if defined(MLKEM_USE_NATIVE) -#include "poly.h" -#include "polyvec.h" - -/* - * MLKEM native arithmetic profile - * - * The profile decides which implementation(s) of the arithmetic backend to use. - * - * If you don't change anything, the default profile will be used. This profile - * picks implementations based on characteristics of your system visible to - * the compiler. - * - * If you want to pick a specific profile for your target, there are three ways - * to do so, in descending order of convenience to the user: - * 1. Pick one of the profiles shipped with this repository. - * 2. Provide your own profile and register it via MLKEM_ARITH_NATIVE_PROFILE - * (which must be the profile's path relative to this directoru). - * 3. Set MLKEM_ARITH_NATIVE_MANUAL and use an adhoc profile specified via - * CFLAGS. - */ - -/* Option 2: Manually written profile */ -#if defined(MLKEM_ARITH_NATIVE_PROFILE) - -#define STRINGIFY_(x) #x -#define STRINGIFY(x) STRINGIFY_(x) -#include STRINGIFY(MLKEM_ARITH_NATIVE_PROFILE) - -/* Option 1: Choose from shipped list of profiles */ -#elif !defined(MLKEM_ARITH_NATIVE_MANUAL) - -#ifdef SYS_AARCH64 -/* - * For now, we only have clean and opt profiles. - * In the future, this is likely to branch further depending - * on the microarchitecture. - */ -#if defined(MLKEM_USE_NATIVE_AARCH64_CLEAN) -#include "aarch64/profiles/clean.h" -#else /* MLKEM_USE_NATIVE_AARCH64_CLEAN */ -#include "aarch64/profiles/opt.h" -#endif /* !MLKEM_USE_NATIVE_AARCH64_CLEAN */ -#endif /* SYS_AARCH64 */ - -#ifdef SYS_X86_64_AVX2 -/* - * For now, there's only one x86_64 profile, which is essentially - * the AVX2 code from the Kyber repository - * https://github.com/pq-crystals/kyber - */ -#include "x86_64/profiles/default.h" -#endif /* SYS_X86_64 */ - -#else /* !MLKEM_ARITH_NATIVE_PROFILE && MLKEM_ARITH_NATIVE_MANUAL */ - -/* Option 3: Build your own profile here, or via CFLAGS */ - -#endif /* !MLKEM_ARITH_NATIVE_PROFILE && !MLKEM_ARITH_NATIVE_MANUAL */ - -#endif /* MLKEM_USE_NATIVE */ -#endif /* MLKEM_ARITH_NATIVE_PROFILE_CHOICE_H */ diff --git a/mlkem/native/x86_64/default.h b/mlkem/native/x86_64/default.h new file mode 100644 index 000000000..592e8996d --- /dev/null +++ b/mlkem/native/x86_64/default.h @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2024 The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 + */ + +/* ML-KEM arithmetic native profile for clean assembly */ + +#ifdef MLKEM_NATIVE_ARITH_PROFILE_H +#error Only one MLKEM_ARITH assembly profile can be defined -- did you include multiple profiles? +#else +#define MLKEM_NATIVE_ARITH_PROFILE_H + +/* Identifier for this backend so that source and assembly files + * in the build can be appropriately guarded. */ +#define MLKEM_NATIVE_ARITH_BACKEND_X86_64_DEFAULT + +#define MLKEM_NATIVE_ARITH_BACKEND_NAME X86_64_DEFAULT + +/* Filename of the C backend implementation. + * This is not inlined here because this header is included in assembly + * files as well. */ +#define MLKEM_NATIVE_ARITH_BACKEND_IMPL "x86_64/src/default_impl.h" + +#endif /* MLKEM_NATIVE_ARITH_PROFILE_H */ diff --git a/mlkem/native/x86_64/align.h b/mlkem/native/x86_64/src/align.h similarity index 100% rename from mlkem/native/x86_64/align.h rename to mlkem/native/x86_64/src/align.h diff --git a/mlkem/native/x86_64/arith_native_x86_64.h b/mlkem/native/x86_64/src/arith_native_x86_64.h similarity index 91% rename from mlkem/native/x86_64/arith_native_x86_64.h rename to mlkem/native/x86_64/src/arith_native_x86_64.h index b346e1eb2..4b78c004a 100644 --- a/mlkem/native/x86_64/arith_native_x86_64.h +++ b/mlkem/native/x86_64/src/arith_native_x86_64.h @@ -5,16 +5,13 @@ #ifndef MLKEM_X86_64_NATIVE_H #define MLKEM_X86_64_NATIVE_H -#include -#include "config.h" -#include "fips202.h" -#include "params.h" -#include "polyvec.h" - -#if defined(MLKEM_USE_NATIVE_X86_64) && defined(SYS_X86_64_AVX2) +#include "common.h" #include #include +#include "consts.h" +#include "fips202.h" +#include "polyvec.h" #define REJ_UNIFORM_AVX_NBLOCKS \ ((12 * MLKEM_N / 8 * (1 << 12) / MLKEM_Q + SHAKE128_RATE) / SHAKE128_RATE) @@ -57,6 +54,4 @@ void nttfrombytes_avx2(__m256i *r, const uint8_t *a, const __m256i *qdata); #define tomont_avx2 MLKEM_NAMESPACE(tomont_avx2) void tomont_avx2(__m256i *r, const __m256i *qdata); -#endif /* MLKEM_USE_NATIVE_X86_64 && SYS_X86_64_AVX2 */ - #endif /* MLKEM_X86_64_NATIVE_H */ diff --git a/mlkem/native/x86_64/basemul.S b/mlkem/native/x86_64/src/basemul.S similarity index 96% rename from mlkem/native/x86_64/basemul.S rename to mlkem/native/x86_64/src/basemul.S index 797b0f46a..503fbeb51 100644 --- a/mlkem/native/x86_64/basemul.S +++ b/mlkem/native/x86_64/src/basemul.S @@ -6,9 +6,8 @@ // Implementation from Kyber reference repository // https://github.com/pq-crystals/kyber/blob/main/avx2 -#include "config.h" - -#if defined(MLKEM_USE_NATIVE_X86_64) && defined(SYS_X86_64_AVX2) +#include "common.h" +#if defined(MLKEM_NATIVE_ARITH_BACKEND_X86_64_DEFAULT) #include "consts.h" #include "params.h" @@ -135,4 +134,4 @@ schoolbook 3 mov %r8,%rsp ret -#endif /* MLKEM_USE_NATIVE_X86_64 && SYS_X86_64_AVX2 */ +#endif /* MLKEM_NATIVE_ARITH_BACKEND_X86_64_DEFAULT */ diff --git a/mlkem/native/x86_64/basemul.c b/mlkem/native/x86_64/src/basemul.c similarity index 90% rename from mlkem/native/x86_64/basemul.c rename to mlkem/native/x86_64/src/basemul.c index c9dba813f..3f1653ed3 100644 --- a/mlkem/native/x86_64/basemul.c +++ b/mlkem/native/x86_64/src/basemul.c @@ -3,15 +3,16 @@ * SPDX-License-Identifier: Apache-2.0 */ -#include "config.h" +#include "common.h" -#if defined(MLKEM_USE_NATIVE_X86_64) && defined(SYS_X86_64_AVX2) +#if defined(MLKEM_NATIVE_ARITH_BACKEND_X86_64_DEFAULT) -#include "arith_native_x86_64.h" #include "consts.h" #include "poly.h" #include "polyvec.h" +#include "arith_native_x86_64.h" + static void poly_basemul_montgomery_avx2(poly *r, const poly *a, const poly *b) { basemul_avx2((__m256i *)r->coeffs, (const __m256i *)a->coeffs, @@ -57,12 +58,11 @@ void polyvec_basemul_acc_montgomery_cached_avx2(poly *r, const polyvec *a, } } -#else -#include "params.h" +#else /* MLKEM_NATIVE_ARITH_BACKEND_X86_64_DEFAULT */ /* Dummy constant to keep compiler happy despite empty CU */ #define empty_cu_avx2_basemul MLKEM_NAMESPACE(empty_cu_avx2_basemul) int empty_cu_avx2_basemul; -#endif /* MLKEM_USE_NATIVE_X86_64 && SYS_X86_64_AVX2 */ +#endif /* MLKEM_NATIVE_ARITH_BACKEND_X86_64_DEFAULT */ diff --git a/mlkem/native/x86_64/consts.c b/mlkem/native/x86_64/src/consts.c similarity index 92% rename from mlkem/native/x86_64/consts.c rename to mlkem/native/x86_64/src/consts.c index 573eabb64..9b6d0993e 100644 --- a/mlkem/native/x86_64/consts.c +++ b/mlkem/native/x86_64/src/consts.c @@ -8,13 +8,12 @@ * https://github.com/pq-crystals/kyber/blob/main/avx2/consts.c */ -#include "config.h" +#include "common.h" -#if defined(MLKEM_USE_NATIVE_X86_64) && defined(SYS_X86_64_AVX2) +#if defined(MLKEM_NATIVE_ARITH_BACKEND_X86_64_DEFAULT) #include "align.h" #include "consts.h" -#include "params.h" #define Q MLKEM_Q #define MONT -1044 /* 2^16 mod q */ @@ -86,10 +85,9 @@ const qdata_t qdata = {{ SHIFT, SHIFT, SHIFT, SHIFT, SHIFT, SHIFT, SHIFT, SHIFT, SHIFT, SHIFT}}; -#else /* MLKEM_USE_NATIVE_X86_64 && SYS_X86_64_AVX2 */ -#include "params.h" +#else /* MLKEM_NATIVE_ARITH_BACKEND_X86_64_DEFAULT */ /* Dummy declaration for compilers disliking empty compilation units */ #define empty_cu_consts MLKEM_NAMESPACE(empty_cu_consts) int empty_cu_consts; -#endif /* MLKEM_USE_NATIVE_X86_64 && SYS_X86_64_AVX2 */ +#endif /* MLKEM_NATIVE_ARITH_BACKEND_X86_64_DEFAULT */ diff --git a/mlkem/native/x86_64/consts.h b/mlkem/native/x86_64/src/consts.h similarity index 97% rename from mlkem/native/x86_64/consts.h rename to mlkem/native/x86_64/src/consts.h index 8ca98bbf8..a9f0c484f 100644 --- a/mlkem/native/x86_64/consts.h +++ b/mlkem/native/x86_64/src/consts.h @@ -11,7 +11,7 @@ #ifndef CONSTS_H #define CONSTS_H -#include "params.h" +#include "common.h" #define _16XQ 0 #define _16XQINV 16 diff --git a/mlkem/native/x86_64/profiles/default.h b/mlkem/native/x86_64/src/default_impl.h similarity index 92% rename from mlkem/native/x86_64/profiles/default.h rename to mlkem/native/x86_64/src/default_impl.h index 55eb0ad01..4f3cd2c9a 100644 --- a/mlkem/native/x86_64/profiles/default.h +++ b/mlkem/native/x86_64/src/default_impl.h @@ -5,17 +5,14 @@ /* ML-KEM arithmetic native profile for clean assembly */ -#ifdef MLKEM_ARITH_NATIVE_PROFILE_H +#ifdef MLKEM_NATIVE_ARITH_PROFILE_IMPL_H #error Only one MLKEM_ARITH assembly profile can be defined -- did you include multiple profiles? #else -#define MLKEM_ARITH_NATIVE_PROFILE_H +#define MLKEM_NATIVE_ARITH_PROFILE_IMPL_H #include -#include "../../arith_native.h" -#include "../arith_native_x86_64.h" -#include "../consts.h" - +#include "arith_native_x86_64.h" #include "poly.h" #define MLKEM_USE_NATIVE_NTT_CUSTOM_ORDER @@ -96,4 +93,4 @@ static INLINE void poly_frombytes_native(poly *r, nttfrombytes_avx2((__m256i *)r->coeffs, a, qdata.vec); } -#endif /* MLKEM_ARITH_NATIVE_PROFILE_H */ +#endif /* MLKEM_NATIVE_ARITH_PROFILE_IMPL_H */ diff --git a/mlkem/native/x86_64/fq.S b/mlkem/native/x86_64/src/fq.S similarity index 91% rename from mlkem/native/x86_64/fq.S rename to mlkem/native/x86_64/src/fq.S index 00fb093c5..50ef190b7 100644 --- a/mlkem/native/x86_64/fq.S +++ b/mlkem/native/x86_64/src/fq.S @@ -11,13 +11,12 @@ // in [0,1,...,q-1] rather than [0,1,...,q], matching the // semantics of poly_reduce(). -#include "config.h" -#include "params.h" +#include "common.h" -#if defined(MLKEM_USE_NATIVE_X86_64) && defined(SYS_X86_64_AVX2) +#if defined(MLKEM_NATIVE_ARITH_BACKEND_X86_64_DEFAULT) #include "consts.h" -.include "fq.inc" +#include "fq.inc" .text reduce128_avx2: @@ -115,4 +114,4 @@ add $256,%rdi call tomont128_avx2 ret -#endif /* MLKEM_USE_NATIVE_X86_64 && SYS_X86_64_AVX2 */ +#endif /* MLKEM_NATIVE_ARITH_BACKEND_X86_64_DEFAULT */ diff --git a/mlkem/native/x86_64/fq.inc b/mlkem/native/x86_64/src/fq.inc similarity index 100% rename from mlkem/native/x86_64/fq.inc rename to mlkem/native/x86_64/src/fq.inc diff --git a/mlkem/native/x86_64/intt.S b/mlkem/native/x86_64/src/intt.S similarity index 97% rename from mlkem/native/x86_64/intt.S rename to mlkem/native/x86_64/src/intt.S index 8418cc471..4860985ed 100644 --- a/mlkem/native/x86_64/intt.S +++ b/mlkem/native/x86_64/src/intt.S @@ -8,14 +8,15 @@ * * Changes to placement of modular reductions have * been made to simplify reasoning of non-overflow */ -#include "config.h" -#if defined(MLKEM_USE_NATIVE_X86_64) && defined(SYS_X86_64_AVX2) +#include "common.h" + +#if defined(MLKEM_NATIVE_ARITH_BACKEND_X86_64_DEFAULT) #include "consts.h" #include "params.h" -.include "shuffle.inc" -.include "fq.inc" +#include "shuffle.inc" +#include "fq.inc" /* Compute four GS butterflies between rh{0,1,2,3} and rl{0,1,2,3}. * Butterflies 0,1 use root zh0 and twisted root zl0, and butterflies @@ -252,4 +253,4 @@ intt_level6 0 intt_level6 1 ret -#endif /* MLKEM_USE_NATIVE_X86_64 && SYS_X86_64_AVX2 */ +#endif /* MLKEM_NATIVE_ARITH_BACKEND_X86_64_DEFAULT */ diff --git a/mlkem/native/x86_64/ntt.S b/mlkem/native/x86_64/src/ntt.S similarity index 97% rename from mlkem/native/x86_64/ntt.S rename to mlkem/native/x86_64/src/ntt.S index f159faf4f..a0b6f734c 100644 --- a/mlkem/native/x86_64/ntt.S +++ b/mlkem/native/x86_64/src/ntt.S @@ -6,14 +6,13 @@ // Implementation from Kyber reference repository // https://github.com/pq-crystals/kyber/blob/main/avx2 -#include "config.h" - -#if defined(MLKEM_USE_NATIVE_X86_64) && defined(SYS_X86_64_AVX2) +#include "common.h" +#if defined(MLKEM_NATIVE_ARITH_BACKEND_X86_64_DEFAULT) #include "consts.h" #include "params.h" -.include "shuffle.inc" +#include "shuffle.inc" /* Compute steps 1,2 / 3 of Montgomery multiplication */ .macro mul rh0,rh1,rh2,rh3,zl0=15,zl1=15,zh0=2,zh1=2 @@ -219,4 +218,4 @@ levels1t6 1 ret -#endif /* MLKEM_USE_NATIVE_X86_64 && SYS_X86_64_AVX2 */ +#endif /* MLKEM_NATIVE_ARITH_BACKEND_X86_64_DEFAULT */ diff --git a/mlkem/native/x86_64/rej_uniform_avx2.c b/mlkem/native/x86_64/src/rej_uniform_avx2.c similarity index 98% rename from mlkem/native/x86_64/rej_uniform_avx2.c rename to mlkem/native/x86_64/src/rej_uniform_avx2.c index c6e663f7d..c3c8b8104 100644 --- a/mlkem/native/x86_64/rej_uniform_avx2.c +++ b/mlkem/native/x86_64/src/rej_uniform_avx2.c @@ -8,18 +8,15 @@ * https://github.com/pq-crystals/kyber/blob/main/avx2 */ -#include "config.h" +#include "common.h" -#if defined(MLKEM_USE_NATIVE_X86_64) && defined(SYS_X86_64_AVX2) - -#include "arith_native_x86_64.h" +#if defined(MLKEM_NATIVE_ARITH_BACKEND_X86_64_DEFAULT) #include #include #include +#include "arith_native_x86_64.h" #include "consts.h" -#include "params.h" - /* #define BMI */ #ifndef BMI @@ -290,10 +287,9 @@ unsigned int rej_uniform_avx2(int16_t *RESTRICT r, const uint8_t *buf) return ctr; } -#else /* MLKEM_USE_NATIVE_X86_64 && SYS_X86_64_AVX2 */ -#include "params.h" +#else /* MLKEM_NATIVE_ARITH_BACKEND_X86_64_DEFAULT */ /* Dummy declaration for compilers disliking empty compilation units */ #define empty_cu_rej_uniform_avx2 MLKEM_NAMESPACE(empty_cu_rej_uniform_avx2) int empty_cu_rej_uniform_avx2; -#endif /* MLKEM_USE_NATIVE_X86_64 && SYS_X86_64_AVX2 */ +#endif /* MLKEM_NATIVE_ARITH_BACKEND_X86_64_DEFAULT */ diff --git a/mlkem/native/x86_64/shuffle.S b/mlkem/native/x86_64/src/shuffle.S similarity index 96% rename from mlkem/native/x86_64/shuffle.S rename to mlkem/native/x86_64/src/shuffle.S index dd1243121..34f6b30b0 100644 --- a/mlkem/native/x86_64/shuffle.S +++ b/mlkem/native/x86_64/src/shuffle.S @@ -6,14 +6,14 @@ // Implementation from Kyber reference repository // https://github.com/pq-crystals/kyber/blob/main/avx2 -#include "config.h" +#include "common.h" -#if defined(MLKEM_USE_NATIVE_X86_64) && defined(SYS_X86_64_AVX2) +#if defined(MLKEM_NATIVE_ARITH_BACKEND_X86_64_DEFAULT) #include "consts.h" #include "params.h" -.include "fq.inc" -.include "shuffle.inc" +#include "fq.inc" +#include "shuffle.inc" .global MLKEM_NAMESPACE(nttpack_avx2) MLKEM_NAMESPACE(nttpack_avx2): @@ -255,4 +255,4 @@ add $192,%rsi call nttfrombytes128_avx ret -#endif /* MLKEM_USE_NATIVE_X86_64 && SYS_X86_64_AVX2 */ +#endif /* MLKEM_NATIVE_ARITH_BACKEND_X86_64_DEFAULT */ diff --git a/mlkem/native/x86_64/shuffle.inc b/mlkem/native/x86_64/src/shuffle.inc similarity index 100% rename from mlkem/native/x86_64/shuffle.inc rename to mlkem/native/x86_64/src/shuffle.inc diff --git a/mlkem/native/x86_64/x86_64_zetas.i b/mlkem/native/x86_64/src/x86_64_zetas.i similarity index 100% rename from mlkem/native/x86_64/x86_64_zetas.i rename to mlkem/native/x86_64/src/x86_64_zetas.i diff --git a/mlkem/ntt.c b/mlkem/ntt.c index 8de1c5446..178e8467c 100644 --- a/mlkem/ntt.c +++ b/mlkem/ntt.c @@ -2,14 +2,12 @@ * Copyright (c) 2024 The mlkem-native project authors * SPDX-License-Identifier: Apache-2.0 */ -#include "ntt.h" #include -#include "params.h" -#include "reduce.h" -#include "arith_native.h" +#include "arith_backend.h" #include "debug/debug.h" #include "ntt.h" +#include "reduce.h" #if !defined(MLKEM_USE_NATIVE_NTT) /* diff --git a/mlkem/ntt.h b/mlkem/ntt.h index 68c9e6724..efa38ecc9 100644 --- a/mlkem/ntt.h +++ b/mlkem/ntt.h @@ -6,9 +6,8 @@ #define NTT_H #include -#include "arith_native.h" #include "cbmc.h" -#include "params.h" +#include "common.h" #include "poly.h" #include "reduce.h" diff --git a/mlkem/params.h b/mlkem/params.h index 85797f067..586c31d33 100644 --- a/mlkem/params.h +++ b/mlkem/params.h @@ -5,47 +5,12 @@ #ifndef PARAMS_H #define PARAMS_H -#include "common.h" #include "config.h" -#include "cpucap.h" -#ifndef MLKEM_K -#define MLKEM_K 3 /* Change this for different security strengths */ +#if !defined(MLKEM_K) +#error MLKEM_K is not defined #endif -#if defined(MLKEM_USE_NATIVE_X86_64) -#define MLKEM_NATIVE_BACKEND X86_64 -#elif defined(MLKEM_USE_NATIVE_AARCH64) -#define MLKEM_NATIVE_BACKEND AARCH64 -#else -#define MLKEM_NATIVE_BACKEND C -#endif - -/* Don't change parameters below this line */ -#if (MLKEM_K == 2) -#define MLKEM_PARAM_NAME MLKEM512 -#elif (MLKEM_K == 3) -#define MLKEM_PARAM_NAME MLKEM768 -#elif (MLKEM_K == 4) -#define MLKEM_PARAM_NAME MLKEM1024 -#else -#error "MLKEM_K must be in {2,3,4}" -#endif - -#define ___MLKEM_NAMESPACE(x1, x2, x3, x4) x1##_##x2##_##x3##_##x4 -#define __MLKEM_NAMESPACE(x1, x2, x3, x4) ___MLKEM_NAMESPACE(x1, x2, x3, x4) - -/* - * NAMESPACE is PQCP_MLKEM_NATIVE___ - * e.g., PQCP_MLKEM_NATIVE_MLKEM512_AARCH64_ - */ -#define MLKEM_NAMESPACE(s) \ - __MLKEM_NAMESPACE(PQCP_MLKEM_NATIVE, MLKEM_PARAM_NAME, MLKEM_NATIVE_BACKEND, \ - s) -#define _MLKEM_NAMESPACE(s) \ - __MLKEM_NAMESPACE(_PQCP_MLKEM_NATIVE, MLKEM_PARAM_NAME, \ - MLKEM_NATIVE_BACKEND, s) - #define MLKEM_N 256 #define MLKEM_Q 3329 #define UINT12_MAX 4095 diff --git a/mlkem/poly.c b/mlkem/poly.c index fdc0ef4af..db7d64ebf 100644 --- a/mlkem/poly.c +++ b/mlkem/poly.c @@ -2,21 +2,20 @@ * Copyright (c) 2024 The mlkem-native project authors * SPDX-License-Identifier: Apache-2.0 */ -#include "poly.h" #include #include + +#include "arith_backend.h" #include "cbd.h" #include "cbmc.h" +#include "debug/debug.h" #include "fips202x4.h" #include "ntt.h" -#include "params.h" +#include "poly.h" #include "reduce.h" #include "symmetric.h" #include "verify.h" -#include "arith_native.h" -#include "debug/debug.h" - void poly_compress_du(uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_DU], const poly *a) { int j; diff --git a/mlkem/poly.h b/mlkem/poly.h index 6d119539d..19cf7b96b 100644 --- a/mlkem/poly.h +++ b/mlkem/poly.h @@ -8,7 +8,7 @@ #include #include #include "cbmc.h" -#include "params.h" +#include "common.h" #include "reduce.h" #include "verify.h" diff --git a/mlkem/polyvec.c b/mlkem/polyvec.c index 7f268ee50..72277a626 100644 --- a/mlkem/polyvec.c +++ b/mlkem/polyvec.c @@ -4,10 +4,9 @@ */ #include "polyvec.h" #include -#include "arith_native.h" +#include "arith_backend.h" #include "config.h" #include "ntt.h" -#include "params.h" #include "poly.h" #include "debug/debug.h" diff --git a/mlkem/polyvec.h b/mlkem/polyvec.h index a4a03bf6d..cd90734fa 100644 --- a/mlkem/polyvec.h +++ b/mlkem/polyvec.h @@ -6,7 +6,7 @@ #define POLYVEC_H #include -#include "params.h" +#include "common.h" #include "poly.h" typedef struct diff --git a/mlkem/reduce.h b/mlkem/reduce.h index 39e85588a..515f706fa 100644 --- a/mlkem/reduce.h +++ b/mlkem/reduce.h @@ -7,8 +7,8 @@ #include #include "cbmc.h" +#include "common.h" #include "debug/debug.h" -#include "params.h" #define HALF_Q ((MLKEM_Q + 1) / 2) /* 1665 */ diff --git a/mlkem/rej_uniform.c b/mlkem/rej_uniform.c index 4e8a5ce9b..88102b4d7 100644 --- a/mlkem/rej_uniform.c +++ b/mlkem/rej_uniform.c @@ -2,10 +2,9 @@ * Copyright (c) 2024 The mlkem-native project authors * SPDX-License-Identifier: Apache-2.0 */ -#include "params.h" -#include "arith_native.h" #include "rej_uniform.h" +#include "arith_backend.h" /************************************************* * Name: rej_uniform_scalar diff --git a/mlkem/rej_uniform.h b/mlkem/rej_uniform.h index 7b86fa63e..e422f73cf 100644 --- a/mlkem/rej_uniform.h +++ b/mlkem/rej_uniform.h @@ -8,7 +8,7 @@ #include #include #include "cbmc.h" -#include "params.h" +#include "common.h" #define rej_uniform MLKEM_NAMESPACE(rej_uniform) /************************************************* diff --git a/mlkem/symmetric.h b/mlkem/symmetric.h index 4eb6aaba6..55ebbbd53 100644 --- a/mlkem/symmetric.h +++ b/mlkem/symmetric.h @@ -7,11 +7,9 @@ #include #include -#include "params.h" - -#include "fips202.h" - #include "cbmc.h" +#include "common.h" +#include "fips202.h" /* Macros denoting FIPS-203 specific Hash functions */ diff --git a/mlkem/sys/cpucap.h b/mlkem/sys.h similarity index 53% rename from mlkem/sys/cpucap.h rename to mlkem/sys.h index 7c3a1cdec..be3070dc2 100644 --- a/mlkem/sys/cpucap.h +++ b/mlkem/sys.h @@ -2,9 +2,8 @@ * Copyright (c) 2024 The mlkem-native project authors * SPDX-License-Identifier: Apache-2.0 */ - -#ifndef CPUCAP_H -#define CPUCAP_H +#ifndef MLKEM_NATIVE_SYS_H +#define MLKEM_NATIVE_SYS_H /* Check if we're running on an AArch64 little endian system. _M_ARM64 is set by * MSVC. */ @@ -52,4 +51,56 @@ #error "FORCE_X86_64 is set, but we don't seem to be on an X86_64 system." #endif +/* + * C90 does not have the inline compiler directive yet. + * We don't use it in C90 builds. + * However, in that case the compiler warns about some inline functions in + * header files not being used in every compilation unit that includes that + * header. To work around it we silence that warning in that case using + * __attribute__((unused)). + */ + +/* Do not use inline for C90 builds*/ +#if !defined(inline) +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L +#define INLINE inline +#define ALWAYS_INLINE __attribute__((always_inline)) +#elif defined(_MSC_VER) +#define INLINE __inline +#define ALWAYS_INLINE __forceinline +#else +#define INLINE __attribute__((unused)) +#define ALWAYS_INLINE +#endif + +#else +#define INLINE inline +#define ALWAYS_INLINE __attribute__((always_inline)) +#endif + +/* + * C90 does not have the restrict compiler directive yet. + * We don't use it in C90 builds. + */ +#if !defined(restrict) +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L +#define RESTRICT restrict +#else +#define RESTRICT +#endif + +#else + +#define RESTRICT restrict +#endif + +#define DEFAULT_ALIGN 32 +#if defined(_WIN32) +#define ALIGN __declspec(align(DEFAULT_ALIGN)) +#define asm __asm +#else +#define asm __asm__ +#define ALIGN __attribute__((aligned(DEFAULT_ALIGN))) #endif + +#endif /* MLKEM_NATIVE_SYS_H */ diff --git a/mlkem/sys/config.h b/mlkem/sys/config.h deleted file mode 100644 index 370a141a6..000000000 --- a/mlkem/sys/config.h +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright (c) 2024 The mlkem-native project authors - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef CONFIG_H -#define CONFIG_H - -#include "cpucap.h" - -#if defined(MLKEM_USE_NATIVE) - -#if defined(SYS_AARCH64) -#define MLKEM_USE_NATIVE_AARCH64 -#endif /* SYS_AARCH64 */ - -#if defined(SYS_X86_64) -#define MLKEM_USE_NATIVE_X86_64 -#endif /* SYS_X86_64 */ - -#endif /* MLKEM_USE_NATIVE */ -#endif /* CONFIG_H */ diff --git a/mlkem/verify.c b/mlkem/verify.c index e616a8bab..b7078fcc1 100644 --- a/mlkem/verify.c +++ b/mlkem/verify.c @@ -13,7 +13,6 @@ volatile uint64_t ct_opt_blocker_u64 = 0; #else /* MLKEM_USE_ASM_VALUE_BARRIER */ -#include "params.h" #define empty_cu_verify MLKEM_NAMESPACE(empty_cu_verify) int empty_cu_verify; diff --git a/scripts/autogenerate_files.py b/scripts/autogenerate_files.py index a33d3cdfe..70d37d905 100644 --- a/scripts/autogenerate_files.py +++ b/scripts/autogenerate_files.py @@ -295,9 +295,13 @@ def gen_aarch64_mulcache_twiddles_twisted(): def gen_aarch64_fwd_ntt_zeta_file(dry_run=False): def gen(): yield from gen_header() - yield '#include "arith_native_aarch64.h"' + yield '#include "common.h"' + yield "" + yield "#if defined(MLKEM_NATIVE_ARITH_BACKEND_AARCH64_CLEAN) || \\" + yield " defined(MLKEM_NATIVE_ARITH_BACKEND_AARCH64_OPT)" yield "" - yield "#ifdef MLKEM_USE_NATIVE_AARCH64" + yield "#include " + yield '#include "arith_native_aarch64.h"' yield "" yield "/*" yield " * Table of zeta values used in the AArch64 forward NTT" @@ -327,17 +331,17 @@ def gen(): yield from map(lambda t: str(t) + ",", gen_aarch64_mulcache_twiddles_twisted()) yield "};" yield "" - yield "#else /* MLKEM_USE_NATIVE_AARCH64 */" + yield "#else" yield '#include "params.h"' yield "" yield "/* Dummy declaration for compilers disliking empty compilation units */" yield "#define empty_cu_aarch64_zetas MLKEM_NAMESPACE(empty_cu_aarch64_zetas)" yield "int empty_cu_aarch64_zetas;" - yield "#endif /* MLKEM_USE_NATIVE_AARCH64 */" + yield "#endif" yield "" update_file( - "mlkem/native/aarch64/aarch64_zetas.c", "\n".join(gen()), dry_run=dry_run + "mlkem/native/aarch64/src/aarch64_zetas.c", "\n".join(gen()), dry_run=dry_run ) @@ -414,7 +418,9 @@ def gen(): yield from map(lambda t: str(t) + ",", gen_avx2_fwd_ntt_zetas()) yield "" - update_file("mlkem/native/x86_64/x86_64_zetas.i", "\n".join(gen()), dry_run=dry_run) + update_file( + "mlkem/native/x86_64/src/x86_64_zetas.i", "\n".join(gen()), dry_run=dry_run + ) def _main(): diff --git a/scripts/lib/mlkem_test.py b/scripts/lib/mlkem_test.py index 796c4953a..73d24122a 100644 --- a/scripts/lib/mlkem_test.py +++ b/scripts/lib/mlkem_test.py @@ -98,9 +98,7 @@ def dict2str(dict): f"{self.test_type}", ] + extra_make_args - make_envs = ( - {"CFLAGS": f"{self.cflags}"} if self.cflags is not None else {} - ) | ( + make_envs = ({"CFLAGS": self.cflags} if self.cflags is not None else {}) | ( {"ARCH_FLAGS": f"{self.arch_flags}"} if self.arch_flags is not None else {} ) extra_make_envs.update(make_envs) diff --git a/test/bench_components_mlkem.c b/test/bench_components_mlkem.c index c922cb76d..b3956171c 100644 --- a/test/bench_components_mlkem.c +++ b/test/bench_components_mlkem.c @@ -12,7 +12,7 @@ #include "randombytes.h" #include "rej_uniform.h" -#include "../mlkem/native/arith_native.h" +#include "../mlkem/arith_backend.h" #include "fips202.h" #include "indcpa.h" #include "keccakf1600.h" @@ -184,7 +184,7 @@ static int bench(void) BENCH("gen_matrix", gen_matrix((polyvec *)data0, (uint8_t *)data1, 0)) -#if defined(MLKEM_USE_NATIVE_AARCH64) +#if defined(MLKEM_NATIVE_ARITH_BACKEND_AARCH64_CLEAN) BENCH("ntt-clean", ntt_asm_clean((int16_t *)data0, (int16_t *)data1, (int16_t *)data2)); BENCH("intt-clean", @@ -200,7 +200,9 @@ static int bench(void) polyvec_basemul_acc_montgomery_cached_asm_clean( (int16_t *)data0, (int16_t *)data1, (int16_t *)data2, (int16_t *)data3)); +#endif /* MLKEM_NATIVE_ARITH_BACKEND_AARCH64_CLEAN */ +#if defined(MLKEM_NATIVE_ARITH_BACKEND_AARCH64_OPT) BENCH("ntt-opt", ntt_asm_opt((int16_t *)data0, (int16_t *)data1, (int16_t *)data2)); BENCH("intt-opt", @@ -214,7 +216,7 @@ static int bench(void) polyvec_basemul_acc_montgomery_cached_asm_opt( (int16_t *)data0, (int16_t *)data1, (int16_t *)data2, (int16_t *)data3)); -#endif +#endif /* MLKEM_NATIVE_ARITH_BACKEND_AARCH64_OPT */ return 0; }