Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Enable rej_uniform native implementation in x86_64 #409

Merged
merged 3 commits into from
Nov 16, 2024
Merged

Enable rej_uniform native implementation in x86_64 #409

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
40b64c0
Enable rej_uniform native implementation
mkannwischer Nov 15, 2024
24b58de
rej_uniform: pad buffer
mkannwischer Nov 15, 2024
cb54991
avx2: Fix buffer over-read in rej_uniform native code
hanno-becker Nov 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions mlkem/native/x86_64/rej_uniform_avx2.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -171,9 +171,10 @@ unsigned int rej_uniform_avx2(int16_t *restrict r, const uint8_t *buf) {
ctr = pos = 0;
while (ctr <= MLKEM_N - 32 && pos <= REJ_UNIFORM_AVX_BUFLEN - 48) {
f0 = _mm256_loadu_si256((__m256i *)&buf[pos]);
f1 = _mm256_loadu_si256((__m256i *)&buf[pos + 24]);
f0 = _mm256_permute4x64_epi64(f0, 0x94);
f1 = _mm256_permute4x64_epi64(f1, 0x94);
// Don't load from offset 24, as this would over-read the buffer
f1 = _mm256_loadu_si256((__m256i *)&buf[pos + 16]);
f0 = _mm256_permute4x64_epi64(f0, 0x94 /* 0b10010100 ~= (2,1,1,0) */);
f1 = _mm256_permute4x64_epi64(f1, 0xe9 /* 0x11101001 ~= (3,2,2,1) */);
f0 = _mm256_shuffle_epi8(f0, idx8);
f1 = _mm256_shuffle_epi8(f1, idx8);
g0 = _mm256_srli_epi16(f0, 4);
Expand Down Expand Up @@ -237,7 +238,7 @@ unsigned int rej_uniform_avx2(int16_t *restrict r, const uint8_t *buf) {
ctr += _mm_popcnt_u32((good >> 24) & 0xFF);
}

while (ctr <= MLKEM_N - 8 && pos <= REJ_UNIFORM_AVX_BUFLEN - 12) {
while (ctr <= MLKEM_N - 8 && pos <= REJ_UNIFORM_AVX_BUFLEN - 24) {
f = _mm_loadu_si128((__m128i *)&buf[pos]);
f = _mm_shuffle_epi8(f, _mm256_castsi256_si128(idx8));
t = _mm_srli_epi16(f, 4);
Expand Down
6 changes: 3 additions & 3 deletions mlkem/rej_uniform.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,12 +61,12 @@ static unsigned int rej_uniform_scalar(int16_t *r, unsigned int target,
return ctr;
}

#if !defined(MLKEM_USE_NATIVE_AARCH64)
#if !defined(MLKEM_USE_NATIVE_REJ_UNIFORM)
unsigned int rej_uniform(int16_t *r, unsigned int target, unsigned int offset,
const uint8_t *buf, unsigned int buflen) {
return rej_uniform_scalar(r, target, offset, buf, buflen);
}
#else /* MLKEM_USE_NATIVE_AARCH64 */
#else /* MLKEM_USE_NATIVE_REJ_UNIFORM */

unsigned int rej_uniform(int16_t *r, unsigned int target, unsigned int offset,
const uint8_t *buf, unsigned int buflen) {
Expand All @@ -79,4 +79,4 @@ unsigned int rej_uniform(int16_t *r, unsigned int target, unsigned int offset,

return rej_uniform_scalar(r, target, offset, buf, buflen);
}
#endif /* MLKEM_USE_NATIVE_AARCH64 */
#endif /* MLKEM_USE_NATIVE_REJ_UNIFORM */