diff --git a/crates/core/machine/include/bb31_t.hpp b/crates/core/machine/include/bb31_t.hpp index 387456d42..c9b387ee3 100644 --- a/crates/core/machine/include/bb31_t.hpp +++ b/crates/core/machine/include/bb31_t.hpp @@ -45,7 +45,7 @@ class bb31_t { inline bb31_t() {} - inline bb31_t(const uint32_t a) { val = a; } + inline constexpr bb31_t(const uint32_t a) { val = a; } inline bb31_t(const uint32_t* p) { val = *p; } @@ -57,7 +57,7 @@ class bb31_t { static inline const bb31_t two() { return from_canonical_u32(2); } - static inline uint32_t to_monty(uint32_t x) { + static inline constexpr uint32_t to_monty(uint32_t x) { return (((uint64_t)x << MONTY_BITS) % MOD); } @@ -71,6 +71,10 @@ class bb31_t { return x_sub_u_hi + corr; } + static inline uint32_t from_monty(uint32_t x) { + return monty_reduce((uint64_t)x); + } + static inline bb31_t from_canonical_u32(uint32_t x) { return bb31_t(to_monty(x)); } @@ -483,9 +487,9 @@ class bb31_t { static const uint32_t NBITS = 31; static const uint32_t MOD = 0x78000001; - inline bb31_t() {} + inline constexpr bb31_t() {} - inline bb31_t(uint32_t a) : val(a) {} + inline constexpr bb31_t(uint32_t a) : val(a) {} inline constexpr bb31_t(int a) : val(((uint64_t)a << 32) % MOD) {} @@ -495,7 +499,7 @@ class bb31_t { static inline const bb31_t two() { return bb31_t(to_monty(2)); } - static inline uint32_t to_monty(uint32_t x) { + static inline constexpr uint32_t to_monty(uint32_t x) { return (((uint64_t)x << MONTY_BITS) % MOD); } diff --git a/crates/cuda/src/lib.rs b/crates/cuda/src/lib.rs index 745bed162..e79311ddd 100644 --- a/crates/cuda/src/lib.rs +++ b/crates/cuda/src/lib.rs @@ -107,7 +107,7 @@ impl SP1CudaProver { pub fn new() -> Result> { let container_name = "sp1-gpu"; let image_name = std::env::var("SP1_GPU_IMAGE") - .unwrap_or_else(|_| "public.ecr.aws/succinct-labs/moongate:v4.0.0-rc.1".to_string()); + .unwrap_or_else(|_| "public.ecr.aws/succinct-labs/moongate:v4.0.0-rc2".to_string()); let cleaned_up = Arc::new(AtomicBool::new(false)); let cleanup_name = container_name; diff --git a/crates/recursion/core/build.rs b/crates/recursion/core/build.rs index 7a819dd54..b18855604 100644 --- a/crates/recursion/core/build.rs +++ b/crates/recursion/core/build.rs @@ -137,6 +137,7 @@ mod sys { .include_item("Poseidon2Instr") .include_item("Poseidon2PreprocessedColsSkinny") .include_item("Poseidon2PreprocessedColsWide") + .include_item("OUTPUT_ROUND_IDX") .with_namespace("sp1_recursion_core_sys") .with_crate(crate_dir) .generate() diff --git a/crates/recursion/core/cpp/extern.cpp b/crates/recursion/core/cpp/extern.cpp index c1408944e..12354c4dd 100644 --- a/crates/recursion/core/cpp/extern.cpp +++ b/crates/recursion/core/cpp/extern.cpp @@ -117,20 +117,14 @@ extern void poseidon2_skinny_instr_to_row_babybear( *reinterpret_cast*>(cols)); } -extern "C" void poseidon2_wide_event_to_row_babybear( - const BabyBearP3* input, BabyBearP3* external_rounds_state, - BabyBearP3* internal_rounds_state, BabyBearP3* internal_rounds_s0, - BabyBearP3* external_sbox, BabyBearP3* internal_sbox, - BabyBearP3* output_state) { - poseidon2_wide::event_to_row( - reinterpret_cast(input), - reinterpret_cast(external_rounds_state), - reinterpret_cast(internal_rounds_state), - reinterpret_cast(internal_rounds_s0), - reinterpret_cast(external_sbox), - reinterpret_cast(internal_sbox), - reinterpret_cast(output_state)); +extern "C" void poseidon2_wide_event_to_row_babybear(const BabyBearP3* input, + BabyBearP3* input_row, + bool sbox_state) { + poseidon2_wide::event_to_row(reinterpret_cast(input), + reinterpret_cast(input_row), 0, + 1, sbox_state); } + extern void poseidon2_wide_instr_to_row_babybear( const Poseidon2SkinnyInstr* instr, Poseidon2PreprocessedColsWide* cols) { diff --git a/crates/recursion/core/include/poseidon2.hpp b/crates/recursion/core/include/poseidon2.hpp index 7bbb6e808..2234d916b 100644 --- a/crates/recursion/core/include/poseidon2.hpp +++ b/crates/recursion/core/include/poseidon2.hpp @@ -1,610 +1,74 @@ #pragma once +#include "poseidon2_constants.hpp" #include "prelude.hpp" namespace sp1_recursion_core_sys::poseidon2 { -constexpr size_t OUTPUT_ROUND_IDX = NUM_EXTERNAL_ROUNDS + 2; +using namespace constants; + constexpr size_t INPUT_ROUND_IDX = 0; constexpr size_t INTERNAL_ROUND_IDX = NUM_EXTERNAL_ROUNDS / 2 + 1; -constexpr uint32_t RC_16_30_U32[30][16] = { - { - 2110014213U, - 3964964605U, - 2190662774U, - 2732996483U, - 640767983U, - 3403899136U, - 1716033721U, - 1606702601U, - 3759873288U, - 1466015491U, - 1498308946U, - 2844375094U, - 3042463841U, - 1969905919U, - 4109944726U, - 3925048366U, - }, - { - 3706859504U, - 759122502U, - 3167665446U, - 1131812921U, - 1080754908U, - 4080114493U, - 893583089U, - 2019677373U, - 3128604556U, - 580640471U, - 3277620260U, - 842931656U, - 548879852U, - 3608554714U, - 3575647916U, - 81826002U, - }, - { - 4289086263U, - 1563933798U, - 1440025885U, - 184445025U, - 2598651360U, - 1396647410U, - 1575877922U, - 3303853401U, - 137125468U, - 765010148U, - 633675867U, - 2037803363U, - 2573389828U, - 1895729703U, - 541515871U, - 1783382863U, - }, - { - 2641856484U, - 3035743342U, - 3672796326U, - 245668751U, - 2025460432U, - 201609705U, - 286217151U, - 4093475563U, - 2519572182U, - 3080699870U, - 2762001832U, - 1244250808U, - 606038199U, - 3182740831U, - 73007766U, - 2572204153U, - }, - { - 1196780786U, - 3447394443U, - 747167305U, - 2968073607U, - 1053214930U, - 1074411832U, - 4016794508U, - 1570312929U, - 113576933U, - 4042581186U, - 3634515733U, - 1032701597U, - 2364839308U, - 3840286918U, - 888378655U, - 2520191583U, - }, - { - 36046858U, - 2927525953U, - 3912129105U, - 4004832531U, - 193772436U, - 1590247392U, - 4125818172U, - 2516251696U, - 4050945750U, - 269498914U, - 1973292656U, - 891403491U, - 1845429189U, - 2611996363U, - 2310542653U, - 4071195740U, - }, - { - 3505307391U, - 786445290U, - 3815313971U, - 1111591756U, - 4233279834U, - 2775453034U, - 1991257625U, - 2940505809U, - 2751316206U, - 1028870679U, - 1282466273U, - 1059053371U, - 834521354U, - 138721483U, - 3100410803U, - 3843128331U, - }, - { - 3878220780U, - 4058162439U, - 1478942487U, - 799012923U, - 496734827U, - 3521261236U, - 755421082U, - 1361409515U, - 392099473U, - 3178453393U, - 4068463721U, - 7935614U, - 4140885645U, - 2150748066U, - 1685210312U, - 3852983224U, - }, - { - 2896943075U, - 3087590927U, - 992175959U, - 970216228U, - 3473630090U, - 3899670400U, - 3603388822U, - 2633488197U, - 2479406964U, - 2420952999U, - 1852516800U, - 4253075697U, - 979699862U, - 1163403191U, - 1608599874U, - 3056104448U, - }, - { - 3779109343U, - 536205958U, - 4183458361U, - 1649720295U, - 1444912244U, - 3122230878U, - 384301396U, - 4228198516U, - 1662916865U, - 4082161114U, - 2121897314U, - 1706239958U, - 4166959388U, - 1626054781U, - 3005858978U, - 1431907253U, - }, - { - 1418914503U, - 1365856753U, - 3942715745U, - 1429155552U, - 3545642795U, - 3772474257U, - 1621094396U, - 2154399145U, - 826697382U, - 1700781391U, - 3539164324U, - 652815039U, - 442484755U, - 2055299391U, - 1064289978U, - 1152335780U, - }, - { - 3417648695U, - 186040114U, - 3475580573U, - 2113941250U, - 1779573826U, - 1573808590U, - 3235694804U, - 2922195281U, - 1119462702U, - 3688305521U, - 1849567013U, - 667446787U, - 753897224U, - 1896396780U, - 3143026334U, - 3829603876U, - }, - { - 859661334U, - 3898844357U, - 180258337U, - 2321867017U, - 3599002504U, - 2886782421U, - 3038299378U, - 1035366250U, - 2038912197U, - 2920174523U, - 1277696101U, - 2785700290U, - 3806504335U, - 3518858933U, - 654843672U, - 2127120275U, - }, - { - 1548195514U, - 2378056027U, - 390914568U, - 1472049779U, - 1552596765U, - 1905886441U, - 1611959354U, - 3653263304U, - 3423946386U, - 340857935U, - 2208879480U, - 139364268U, - 3447281773U, - 3777813707U, - 55640413U, - 4101901741U, - }, - {104929687U, 1459980974U, 1831234737U, 457139004U, 2581487628U, 2112044563U, - 3567013861U, 2792004347U, 576325418U, 41126132U, 2713562324U, 151213722U, - 2891185935U, 546846420U, 2939794919U, 2543469905U}, - { - 2191909784U, - 3315138460U, - 530414574U, - 1242280418U, - 1211740715U, - 3993672165U, - 2505083323U, - 3845798801U, - 538768466U, - 2063567560U, - 3366148274U, - 1449831887U, - 2408012466U, - 294726285U, - 3943435493U, - 924016661U, - }, - { - 3633138367U, - 3222789372U, - 809116305U, - 30100013U, - 2655172876U, - 2564247117U, - 2478649732U, - 4113689151U, - 4120146082U, - 2512308515U, - 650406041U, - 4240012393U, - 2683508708U, - 951073977U, - 3460081988U, - 339124269U, - }, - { - 130182653U, - 2755946749U, - 542600513U, - 2816103022U, - 1931786340U, - 2044470840U, - 1709908013U, - 2938369043U, - 3640399693U, - 1374470239U, - 2191149676U, - 2637495682U, - 4236394040U, - 2289358846U, - 3833368530U, - 974546524U, - }, - { - 3306659113U, - 2234814261U, - 1188782305U, - 223782844U, - 2248980567U, - 2309786141U, - 2023401627U, - 3278877413U, - 2022138149U, - 575851471U, - 1612560780U, - 3926656936U, - 3318548977U, - 2591863678U, - 188109355U, - 4217723909U, - }, - { - 1564209905U, - 2154197895U, - 2459687029U, - 2870634489U, - 1375012945U, - 1529454825U, - 306140690U, - 2855578299U, - 1246997295U, - 3024298763U, - 1915270363U, - 1218245412U, - 2479314020U, - 2989827755U, - 814378556U, - 4039775921U, - }, - { - 1165280628U, - 1203983801U, - 3814740033U, - 1919627044U, - 600240215U, - 773269071U, - 486685186U, - 4254048810U, - 1415023565U, - 502840102U, - 4225648358U, - 510217063U, - 166444818U, - 1430745893U, - 1376516190U, - 1775891321U, - }, - { - 1170945922U, - 1105391877U, - 261536467U, - 1401687994U, - 1022529847U, - 2476446456U, - 2603844878U, - 3706336043U, - 3463053714U, - 1509644517U, - 588552318U, - 65252581U, - 3696502656U, - 2183330763U, - 3664021233U, - 1643809916U, - }, - { - 2922875898U, - 3740690643U, - 3932461140U, - 161156271U, - 2619943483U, - 4077039509U, - 2921201703U, - 2085619718U, - 2065264646U, - 2615693812U, - 3116555433U, - 246100007U, - 4281387154U, - 4046141001U, - 4027749321U, - 111611860U, - }, - { - 2066954820U, - 2502099969U, - 2915053115U, - 2362518586U, - 366091708U, - 2083204932U, - 4138385632U, - 3195157567U, - 1318086382U, - 521723799U, - 702443405U, - 2507670985U, - 1760347557U, - 2631999893U, - 1672737554U, - 1060867760U, - }, - { - 2359801781U, - 2800231467U, - 3010357035U, - 1035997899U, - 1210110952U, - 1018506770U, - 2799468177U, - 1479380761U, - 1536021911U, - 358993854U, - 579904113U, - 3432144800U, - 3625515809U, - 199241497U, - 4058304109U, - 2590164234U, - }, - { - 1688530738U, - 1580733335U, - 2443981517U, - 2206270565U, - 2780074229U, - 2628739677U, - 2940123659U, - 4145206827U, - 3572278009U, - 2779607509U, - 1098718697U, - 1424913749U, - 2224415875U, - 1108922178U, - 3646272562U, - 3935186184U, - }, - { - 820046587U, - 1393386250U, - 2665818575U, - 2231782019U, - 672377010U, - 1920315467U, - 1913164407U, - 2029526876U, - 2629271820U, - 384320012U, - 4112320585U, - 3131824773U, - 2347818197U, - 2220997386U, - 1772368609U, - 2579960095U, - }, - { - 3544930873U, - 225847443U, - 3070082278U, - 95643305U, - 3438572042U, - 3312856509U, - 615850007U, - 1863868773U, - 803582265U, - 3461976859U, - 2903025799U, - 1482092434U, - 3902972499U, - 3872341868U, - 1530411808U, - 2214923584U, - }, - { - 3118792481U, - 2241076515U, - 3983669831U, - 3180915147U, - 3838626501U, - 1921630011U, - 3415351771U, - 2249953859U, - 3755081630U, - 486327260U, - 1227575720U, - 3643869379U, - 2982026073U, - 2466043731U, - 1982634375U, - 3769609014U, - }, - { - 2195455495U, - 2596863283U, - 4244994973U, - 1983609348U, - 4019674395U, - 3469982031U, - 1458697570U, - 1593516217U, - 1963896497U, - 3115309118U, - 1659132465U, - 2536770756U, - 3059294171U, - 2618031334U, - 2040903247U, - 3799795076U, - }}; +constexpr size_t NUM_ROUNDS = OUTPUT_ROUND_IDX + 1; + +constexpr size_t PERMUTATION_NO_SBOX = + (WIDTH * NUM_EXTERNAL_ROUNDS) + WIDTH + (NUM_INTERNAL_ROUNDS - 1) + WIDTH; +constexpr size_t PERMUTATION_SBOX = + PERMUTATION_NO_SBOX + (WIDTH * NUM_EXTERNAL_ROUNDS) + NUM_INTERNAL_ROUNDS; -static const bb31_t POSEIDON2_INTERNAL_MATRIX_DIAG_16_BABYBEAR_MONTY[16] = { - bb31_t::from_canonical_u32(0x78000001u - 2), // BabyBear::ORDER_U32 - 2 - bb31_t::from_canonical_u32(1), // 1 - bb31_t::from_canonical_u32(1 << 1), // 1 << 1 - bb31_t::from_canonical_u32(1 << 2), // 1 << 2 - bb31_t::from_canonical_u32(1 << 3), // 1 << 3 - bb31_t::from_canonical_u32(1 << 4), // 1 << 4 - bb31_t::from_canonical_u32(1 << 5), // 1 << 5 - bb31_t::from_canonical_u32(1 << 6), // 1 << 6 - bb31_t::from_canonical_u32(1 << 7), // 1 << 7 - bb31_t::from_canonical_u32(1 << 8), // 1 << 8 - bb31_t::from_canonical_u32(1 << 9), // 1 << 9 - bb31_t::from_canonical_u32(1 << 10), // 1 << 10 - bb31_t::from_canonical_u32(1 << 11), // 1 << 11 - bb31_t::from_canonical_u32(1 << 12), // 1 << 12 - bb31_t::from_canonical_u32(1 << 13), // 1 << 13 - bb31_t::from_canonical_u32(1 << 15), // 1 << 15 -}; +constexpr size_t POSEIDON2_WIDTH = 16; template -__SP1_HOSTDEV__ __SP1_INLINE__ void external_linear_layer(F* state_var) { - for (size_t j = 0; j < WIDTH; j += 4) { - F t01 = state_var[j + 0] + state_var[j + 1]; - F t23 = state_var[j + 2] + state_var[j + 3]; - F t0123 = t01 + t23; - F t01123 = t0123 + state_var[j + 1]; - F t01233 = t0123 + state_var[j + 3]; +__SP1_HOSTDEV__ __SP1_INLINE__ void mdsLightPermutation4x4(F state[4]) { + F t01 = state[0] + state[1]; + F t23 = state[2] + state[3]; + F t0123 = t01 + t23; + F t01123 = t0123 + state[1]; + F t01233 = t0123 + state[3]; + state[3] = t01233 + operator<<(state[0], 1); + state[1] = t01123 + operator<<(state[2], 1); + state[0] = t01123 + t01; + state[2] = t01233 + t23; +} - // The order here is important. Need to overwrite x[0] and x[2] after x[1] and x[3]. - state_var[j + 3] = - t01233 + - (state_var[j + 0] + state_var[j + 0]); // 3*x[0] + x[1] + x[2] + 2*x[3] - state_var[j + 1] = - t01123 + - (state_var[j + 2] + state_var[j + 2]); // x[0] + 2*x[1] + 3*x[2] + x[3] - state_var[j + 0] = t01123 + t01; // 2*x[0] + 3*x[1] + x[2] + x[3] - state_var[j + 2] = t01233 + t23; // x[0] + x[1] + 2*x[2] + 3*x[3] +template +__SP1_HOSTDEV__ __SP1_INLINE__ void external_linear_layer(F state_var[POSEIDON2_WIDTH]) { + for (int i = 0; i < POSEIDON2_WIDTH; i += 4) { + mdsLightPermutation4x4(state_var + i); } F sums[4] = {F::zero(), F::zero(), F::zero(), F::zero()}; for (size_t k = 0; k < 4; k++) { - for (size_t j = 0; j < WIDTH; j += 4) { + for (size_t j = 0; j < POSEIDON2_WIDTH; j += 4) { sums[k] = sums[k] + state_var[j + k]; } } - for (size_t j = 0; j < WIDTH; j++) { + for (size_t j = 0; j < POSEIDON2_WIDTH; j++) { state_var[j] = state_var[j] + sums[j % 4]; } } template -__SP1_HOSTDEV__ __SP1_INLINE__ void internal_linear_layer(F* state) { - F matmul_constants[WIDTH]; - for (size_t i = 0; i < WIDTH; i++) { +__SP1_HOSTDEV__ __SP1_INLINE__ void internal_linear_layer(F state[POSEIDON2_WIDTH]) { + F matmul_constants[POSEIDON2_WIDTH]; + for (size_t i = 0; i < POSEIDON2_WIDTH; i++) { matmul_constants[i] = F(F::to_monty(F::from_monty( - POSEIDON2_INTERNAL_MATRIX_DIAG_16_BABYBEAR_MONTY[i].val))); + constants::POSEIDON2_INTERNAL_MATRIX_DIAG_16_BABYBEAR_MONTY[i].val))); } F sum = F::zero(); - for (size_t i = 0; i < WIDTH; i++) { + for (size_t i = 0; i < POSEIDON2_WIDTH; i++) { sum = sum + state[i]; } - for (size_t i = 0; i < WIDTH; i++) { + for (size_t i = 0; i < POSEIDON2_WIDTH; i++) { state[i] = state[i] * matmul_constants[i]; state[i] = state[i] + sum; } F monty_inverse = F(F::to_monty(F::from_monty(1))); - for (size_t i = 0; i < WIDTH; i++) { + for (size_t i = 0; i < POSEIDON2_WIDTH; i++) { state[i] = state[i] * monty_inverse; } } diff --git a/crates/recursion/core/include/poseidon2_constants.hpp b/crates/recursion/core/include/poseidon2_constants.hpp new file mode 100644 index 000000000..c076d6b64 --- /dev/null +++ b/crates/recursion/core/include/poseidon2_constants.hpp @@ -0,0 +1,1100 @@ +#pragma once + +namespace sp1_recursion_core_sys::constants { +#ifdef __CUDA_ARCH__ +__constant__ constexpr const uint32_t RC_16_30_U32[30][16] = { + { + 2110014213U, + 3964964605U, + 2190662774U, + 2732996483U, + 640767983U, + 3403899136U, + 1716033721U, + 1606702601U, + 3759873288U, + 1466015491U, + 1498308946U, + 2844375094U, + 3042463841U, + 1969905919U, + 4109944726U, + 3925048366U, + }, + { + 3706859504U, + 759122502U, + 3167665446U, + 1131812921U, + 1080754908U, + 4080114493U, + 893583089U, + 2019677373U, + 3128604556U, + 580640471U, + 3277620260U, + 842931656U, + 548879852U, + 3608554714U, + 3575647916U, + 81826002U, + }, + { + 4289086263U, + 1563933798U, + 1440025885U, + 184445025U, + 2598651360U, + 1396647410U, + 1575877922U, + 3303853401U, + 137125468U, + 765010148U, + 633675867U, + 2037803363U, + 2573389828U, + 1895729703U, + 541515871U, + 1783382863U, + }, + { + 2641856484U, + 3035743342U, + 3672796326U, + 245668751U, + 2025460432U, + 201609705U, + 286217151U, + 4093475563U, + 2519572182U, + 3080699870U, + 2762001832U, + 1244250808U, + 606038199U, + 3182740831U, + 73007766U, + 2572204153U, + }, + { + 1196780786U, + 3447394443U, + 747167305U, + 2968073607U, + 1053214930U, + 1074411832U, + 4016794508U, + 1570312929U, + 113576933U, + 4042581186U, + 3634515733U, + 1032701597U, + 2364839308U, + 3840286918U, + 888378655U, + 2520191583U, + }, + { + 36046858U, + 2927525953U, + 3912129105U, + 4004832531U, + 193772436U, + 1590247392U, + 4125818172U, + 2516251696U, + 4050945750U, + 269498914U, + 1973292656U, + 891403491U, + 1845429189U, + 2611996363U, + 2310542653U, + 4071195740U, + }, + { + 3505307391U, + 786445290U, + 3815313971U, + 1111591756U, + 4233279834U, + 2775453034U, + 1991257625U, + 2940505809U, + 2751316206U, + 1028870679U, + 1282466273U, + 1059053371U, + 834521354U, + 138721483U, + 3100410803U, + 3843128331U, + }, + { + 3878220780U, + 4058162439U, + 1478942487U, + 799012923U, + 496734827U, + 3521261236U, + 755421082U, + 1361409515U, + 392099473U, + 3178453393U, + 4068463721U, + 7935614U, + 4140885645U, + 2150748066U, + 1685210312U, + 3852983224U, + }, + { + 2896943075U, + 3087590927U, + 992175959U, + 970216228U, + 3473630090U, + 3899670400U, + 3603388822U, + 2633488197U, + 2479406964U, + 2420952999U, + 1852516800U, + 4253075697U, + 979699862U, + 1163403191U, + 1608599874U, + 3056104448U, + }, + { + 3779109343U, + 536205958U, + 4183458361U, + 1649720295U, + 1444912244U, + 3122230878U, + 384301396U, + 4228198516U, + 1662916865U, + 4082161114U, + 2121897314U, + 1706239958U, + 4166959388U, + 1626054781U, + 3005858978U, + 1431907253U, + }, + { + 1418914503U, + 1365856753U, + 3942715745U, + 1429155552U, + 3545642795U, + 3772474257U, + 1621094396U, + 2154399145U, + 826697382U, + 1700781391U, + 3539164324U, + 652815039U, + 442484755U, + 2055299391U, + 1064289978U, + 1152335780U, + }, + { + 3417648695U, + 186040114U, + 3475580573U, + 2113941250U, + 1779573826U, + 1573808590U, + 3235694804U, + 2922195281U, + 1119462702U, + 3688305521U, + 1849567013U, + 667446787U, + 753897224U, + 1896396780U, + 3143026334U, + 3829603876U, + }, + { + 859661334U, + 3898844357U, + 180258337U, + 2321867017U, + 3599002504U, + 2886782421U, + 3038299378U, + 1035366250U, + 2038912197U, + 2920174523U, + 1277696101U, + 2785700290U, + 3806504335U, + 3518858933U, + 654843672U, + 2127120275U, + }, + { + 1548195514U, + 2378056027U, + 390914568U, + 1472049779U, + 1552596765U, + 1905886441U, + 1611959354U, + 3653263304U, + 3423946386U, + 340857935U, + 2208879480U, + 139364268U, + 3447281773U, + 3777813707U, + 55640413U, + 4101901741U, + }, + {104929687U, 1459980974U, 1831234737U, 457139004U, 2581487628U, 2112044563U, + 3567013861U, 2792004347U, 576325418U, 41126132U, 2713562324U, 151213722U, + 2891185935U, 546846420U, 2939794919U, 2543469905U}, + { + 2191909784U, + 3315138460U, + 530414574U, + 1242280418U, + 1211740715U, + 3993672165U, + 2505083323U, + 3845798801U, + 538768466U, + 2063567560U, + 3366148274U, + 1449831887U, + 2408012466U, + 294726285U, + 3943435493U, + 924016661U, + }, + { + 3633138367U, + 3222789372U, + 809116305U, + 30100013U, + 2655172876U, + 2564247117U, + 2478649732U, + 4113689151U, + 4120146082U, + 2512308515U, + 650406041U, + 4240012393U, + 2683508708U, + 951073977U, + 3460081988U, + 339124269U, + }, + { + 130182653U, + 2755946749U, + 542600513U, + 2816103022U, + 1931786340U, + 2044470840U, + 1709908013U, + 2938369043U, + 3640399693U, + 1374470239U, + 2191149676U, + 2637495682U, + 4236394040U, + 2289358846U, + 3833368530U, + 974546524U, + }, + { + 3306659113U, + 2234814261U, + 1188782305U, + 223782844U, + 2248980567U, + 2309786141U, + 2023401627U, + 3278877413U, + 2022138149U, + 575851471U, + 1612560780U, + 3926656936U, + 3318548977U, + 2591863678U, + 188109355U, + 4217723909U, + }, + { + 1564209905U, + 2154197895U, + 2459687029U, + 2870634489U, + 1375012945U, + 1529454825U, + 306140690U, + 2855578299U, + 1246997295U, + 3024298763U, + 1915270363U, + 1218245412U, + 2479314020U, + 2989827755U, + 814378556U, + 4039775921U, + }, + { + 1165280628U, + 1203983801U, + 3814740033U, + 1919627044U, + 600240215U, + 773269071U, + 486685186U, + 4254048810U, + 1415023565U, + 502840102U, + 4225648358U, + 510217063U, + 166444818U, + 1430745893U, + 1376516190U, + 1775891321U, + }, + { + 1170945922U, + 1105391877U, + 261536467U, + 1401687994U, + 1022529847U, + 2476446456U, + 2603844878U, + 3706336043U, + 3463053714U, + 1509644517U, + 588552318U, + 65252581U, + 3696502656U, + 2183330763U, + 3664021233U, + 1643809916U, + }, + { + 2922875898U, + 3740690643U, + 3932461140U, + 161156271U, + 2619943483U, + 4077039509U, + 2921201703U, + 2085619718U, + 2065264646U, + 2615693812U, + 3116555433U, + 246100007U, + 4281387154U, + 4046141001U, + 4027749321U, + 111611860U, + }, + { + 2066954820U, + 2502099969U, + 2915053115U, + 2362518586U, + 366091708U, + 2083204932U, + 4138385632U, + 3195157567U, + 1318086382U, + 521723799U, + 702443405U, + 2507670985U, + 1760347557U, + 2631999893U, + 1672737554U, + 1060867760U, + }, + { + 2359801781U, + 2800231467U, + 3010357035U, + 1035997899U, + 1210110952U, + 1018506770U, + 2799468177U, + 1479380761U, + 1536021911U, + 358993854U, + 579904113U, + 3432144800U, + 3625515809U, + 199241497U, + 4058304109U, + 2590164234U, + }, + { + 1688530738U, + 1580733335U, + 2443981517U, + 2206270565U, + 2780074229U, + 2628739677U, + 2940123659U, + 4145206827U, + 3572278009U, + 2779607509U, + 1098718697U, + 1424913749U, + 2224415875U, + 1108922178U, + 3646272562U, + 3935186184U, + }, + { + 820046587U, + 1393386250U, + 2665818575U, + 2231782019U, + 672377010U, + 1920315467U, + 1913164407U, + 2029526876U, + 2629271820U, + 384320012U, + 4112320585U, + 3131824773U, + 2347818197U, + 2220997386U, + 1772368609U, + 2579960095U, + }, + { + 3544930873U, + 225847443U, + 3070082278U, + 95643305U, + 3438572042U, + 3312856509U, + 615850007U, + 1863868773U, + 803582265U, + 3461976859U, + 2903025799U, + 1482092434U, + 3902972499U, + 3872341868U, + 1530411808U, + 2214923584U, + }, + { + 3118792481U, + 2241076515U, + 3983669831U, + 3180915147U, + 3838626501U, + 1921630011U, + 3415351771U, + 2249953859U, + 3755081630U, + 486327260U, + 1227575720U, + 3643869379U, + 2982026073U, + 2466043731U, + 1982634375U, + 3769609014U, + }, + { + 2195455495U, + 2596863283U, + 4244994973U, + 1983609348U, + 4019674395U, + 3469982031U, + 1458697570U, + 1593516217U, + 1963896497U, + 3115309118U, + 1659132465U, + 2536770756U, + 3059294171U, + 2618031334U, + 2040903247U, + 3799795076U, + }}; +#else +constexpr const uint32_t RC_16_30_U32[30][16] = { + { + 2110014213U, + 3964964605U, + 2190662774U, + 2732996483U, + 640767983U, + 3403899136U, + 1716033721U, + 1606702601U, + 3759873288U, + 1466015491U, + 1498308946U, + 2844375094U, + 3042463841U, + 1969905919U, + 4109944726U, + 3925048366U, + }, + { + 3706859504U, + 759122502U, + 3167665446U, + 1131812921U, + 1080754908U, + 4080114493U, + 893583089U, + 2019677373U, + 3128604556U, + 580640471U, + 3277620260U, + 842931656U, + 548879852U, + 3608554714U, + 3575647916U, + 81826002U, + }, + { + 4289086263U, + 1563933798U, + 1440025885U, + 184445025U, + 2598651360U, + 1396647410U, + 1575877922U, + 3303853401U, + 137125468U, + 765010148U, + 633675867U, + 2037803363U, + 2573389828U, + 1895729703U, + 541515871U, + 1783382863U, + }, + { + 2641856484U, + 3035743342U, + 3672796326U, + 245668751U, + 2025460432U, + 201609705U, + 286217151U, + 4093475563U, + 2519572182U, + 3080699870U, + 2762001832U, + 1244250808U, + 606038199U, + 3182740831U, + 73007766U, + 2572204153U, + }, + { + 1196780786U, + 3447394443U, + 747167305U, + 2968073607U, + 1053214930U, + 1074411832U, + 4016794508U, + 1570312929U, + 113576933U, + 4042581186U, + 3634515733U, + 1032701597U, + 2364839308U, + 3840286918U, + 888378655U, + 2520191583U, + }, + { + 36046858U, + 2927525953U, + 3912129105U, + 4004832531U, + 193772436U, + 1590247392U, + 4125818172U, + 2516251696U, + 4050945750U, + 269498914U, + 1973292656U, + 891403491U, + 1845429189U, + 2611996363U, + 2310542653U, + 4071195740U, + }, + { + 3505307391U, + 786445290U, + 3815313971U, + 1111591756U, + 4233279834U, + 2775453034U, + 1991257625U, + 2940505809U, + 2751316206U, + 1028870679U, + 1282466273U, + 1059053371U, + 834521354U, + 138721483U, + 3100410803U, + 3843128331U, + }, + { + 3878220780U, + 4058162439U, + 1478942487U, + 799012923U, + 496734827U, + 3521261236U, + 755421082U, + 1361409515U, + 392099473U, + 3178453393U, + 4068463721U, + 7935614U, + 4140885645U, + 2150748066U, + 1685210312U, + 3852983224U, + }, + { + 2896943075U, + 3087590927U, + 992175959U, + 970216228U, + 3473630090U, + 3899670400U, + 3603388822U, + 2633488197U, + 2479406964U, + 2420952999U, + 1852516800U, + 4253075697U, + 979699862U, + 1163403191U, + 1608599874U, + 3056104448U, + }, + { + 3779109343U, + 536205958U, + 4183458361U, + 1649720295U, + 1444912244U, + 3122230878U, + 384301396U, + 4228198516U, + 1662916865U, + 4082161114U, + 2121897314U, + 1706239958U, + 4166959388U, + 1626054781U, + 3005858978U, + 1431907253U, + }, + { + 1418914503U, + 1365856753U, + 3942715745U, + 1429155552U, + 3545642795U, + 3772474257U, + 1621094396U, + 2154399145U, + 826697382U, + 1700781391U, + 3539164324U, + 652815039U, + 442484755U, + 2055299391U, + 1064289978U, + 1152335780U, + }, + { + 3417648695U, + 186040114U, + 3475580573U, + 2113941250U, + 1779573826U, + 1573808590U, + 3235694804U, + 2922195281U, + 1119462702U, + 3688305521U, + 1849567013U, + 667446787U, + 753897224U, + 1896396780U, + 3143026334U, + 3829603876U, + }, + { + 859661334U, + 3898844357U, + 180258337U, + 2321867017U, + 3599002504U, + 2886782421U, + 3038299378U, + 1035366250U, + 2038912197U, + 2920174523U, + 1277696101U, + 2785700290U, + 3806504335U, + 3518858933U, + 654843672U, + 2127120275U, + }, + { + 1548195514U, + 2378056027U, + 390914568U, + 1472049779U, + 1552596765U, + 1905886441U, + 1611959354U, + 3653263304U, + 3423946386U, + 340857935U, + 2208879480U, + 139364268U, + 3447281773U, + 3777813707U, + 55640413U, + 4101901741U, + }, + {104929687U, 1459980974U, 1831234737U, 457139004U, 2581487628U, 2112044563U, + 3567013861U, 2792004347U, 576325418U, 41126132U, 2713562324U, 151213722U, + 2891185935U, 546846420U, 2939794919U, 2543469905U}, + { + 2191909784U, + 3315138460U, + 530414574U, + 1242280418U, + 1211740715U, + 3993672165U, + 2505083323U, + 3845798801U, + 538768466U, + 2063567560U, + 3366148274U, + 1449831887U, + 2408012466U, + 294726285U, + 3943435493U, + 924016661U, + }, + { + 3633138367U, + 3222789372U, + 809116305U, + 30100013U, + 2655172876U, + 2564247117U, + 2478649732U, + 4113689151U, + 4120146082U, + 2512308515U, + 650406041U, + 4240012393U, + 2683508708U, + 951073977U, + 3460081988U, + 339124269U, + }, + { + 130182653U, + 2755946749U, + 542600513U, + 2816103022U, + 1931786340U, + 2044470840U, + 1709908013U, + 2938369043U, + 3640399693U, + 1374470239U, + 2191149676U, + 2637495682U, + 4236394040U, + 2289358846U, + 3833368530U, + 974546524U, + }, + { + 3306659113U, + 2234814261U, + 1188782305U, + 223782844U, + 2248980567U, + 2309786141U, + 2023401627U, + 3278877413U, + 2022138149U, + 575851471U, + 1612560780U, + 3926656936U, + 3318548977U, + 2591863678U, + 188109355U, + 4217723909U, + }, + { + 1564209905U, + 2154197895U, + 2459687029U, + 2870634489U, + 1375012945U, + 1529454825U, + 306140690U, + 2855578299U, + 1246997295U, + 3024298763U, + 1915270363U, + 1218245412U, + 2479314020U, + 2989827755U, + 814378556U, + 4039775921U, + }, + { + 1165280628U, + 1203983801U, + 3814740033U, + 1919627044U, + 600240215U, + 773269071U, + 486685186U, + 4254048810U, + 1415023565U, + 502840102U, + 4225648358U, + 510217063U, + 166444818U, + 1430745893U, + 1376516190U, + 1775891321U, + }, + { + 1170945922U, + 1105391877U, + 261536467U, + 1401687994U, + 1022529847U, + 2476446456U, + 2603844878U, + 3706336043U, + 3463053714U, + 1509644517U, + 588552318U, + 65252581U, + 3696502656U, + 2183330763U, + 3664021233U, + 1643809916U, + }, + { + 2922875898U, + 3740690643U, + 3932461140U, + 161156271U, + 2619943483U, + 4077039509U, + 2921201703U, + 2085619718U, + 2065264646U, + 2615693812U, + 3116555433U, + 246100007U, + 4281387154U, + 4046141001U, + 4027749321U, + 111611860U, + }, + { + 2066954820U, + 2502099969U, + 2915053115U, + 2362518586U, + 366091708U, + 2083204932U, + 4138385632U, + 3195157567U, + 1318086382U, + 521723799U, + 702443405U, + 2507670985U, + 1760347557U, + 2631999893U, + 1672737554U, + 1060867760U, + }, + { + 2359801781U, + 2800231467U, + 3010357035U, + 1035997899U, + 1210110952U, + 1018506770U, + 2799468177U, + 1479380761U, + 1536021911U, + 358993854U, + 579904113U, + 3432144800U, + 3625515809U, + 199241497U, + 4058304109U, + 2590164234U, + }, + { + 1688530738U, + 1580733335U, + 2443981517U, + 2206270565U, + 2780074229U, + 2628739677U, + 2940123659U, + 4145206827U, + 3572278009U, + 2779607509U, + 1098718697U, + 1424913749U, + 2224415875U, + 1108922178U, + 3646272562U, + 3935186184U, + }, + { + 820046587U, + 1393386250U, + 2665818575U, + 2231782019U, + 672377010U, + 1920315467U, + 1913164407U, + 2029526876U, + 2629271820U, + 384320012U, + 4112320585U, + 3131824773U, + 2347818197U, + 2220997386U, + 1772368609U, + 2579960095U, + }, + { + 3544930873U, + 225847443U, + 3070082278U, + 95643305U, + 3438572042U, + 3312856509U, + 615850007U, + 1863868773U, + 803582265U, + 3461976859U, + 2903025799U, + 1482092434U, + 3902972499U, + 3872341868U, + 1530411808U, + 2214923584U, + }, + { + 3118792481U, + 2241076515U, + 3983669831U, + 3180915147U, + 3838626501U, + 1921630011U, + 3415351771U, + 2249953859U, + 3755081630U, + 486327260U, + 1227575720U, + 3643869379U, + 2982026073U, + 2466043731U, + 1982634375U, + 3769609014U, + }, + { + 2195455495U, + 2596863283U, + 4244994973U, + 1983609348U, + 4019674395U, + 3469982031U, + 1458697570U, + 1593516217U, + 1963896497U, + 3115309118U, + 1659132465U, + 2536770756U, + 3059294171U, + 2618031334U, + 2040903247U, + 3799795076U, + }}; +#endif + +#ifdef __CUDA_ARCH__ +__constant__ constexpr const bb31_t + POSEIDON2_INTERNAL_MATRIX_DIAG_16_BABYBEAR_MONTY[16] = { + bb31_t(bb31_t::to_monty(0x78000001u - 2)), // BabyBear::ORDER_U32 - 2 + bb31_t(bb31_t::to_monty(1)), // 1 + bb31_t(bb31_t::to_monty(1 << 1)), // 1 << 1 + bb31_t(bb31_t::to_monty(1 << 2)), // 1 << 2 + bb31_t(bb31_t::to_monty(1 << 3)), // 1 << 3 + bb31_t(bb31_t::to_monty(1 << 4)), // 1 << 4 + bb31_t(bb31_t::to_monty(1 << 5)), // 1 << 5 + bb31_t(bb31_t::to_monty(1 << 6)), // 1 << 6 + bb31_t(bb31_t::to_monty(1 << 7)), // 1 << 7 + bb31_t(bb31_t::to_monty(1 << 8)), // 1 << 8 + bb31_t(bb31_t::to_monty(1 << 9)), // 1 << 9 + bb31_t(bb31_t::to_monty(1 << 10)), // 1 << 10 + bb31_t(bb31_t::to_monty(1 << 11)), // 1 << 11 + bb31_t(bb31_t::to_monty(1 << 12)), // 1 << 12 + bb31_t(bb31_t::to_monty(1 << 13)), // 1 << 13 + bb31_t(bb31_t::to_monty(1 << 15)), // 1 << 15 +}; +#else +constexpr const bb31_t POSEIDON2_INTERNAL_MATRIX_DIAG_16_BABYBEAR_MONTY[16] = { + bb31_t(bb31_t::to_monty(0x78000001u - 2)), // BabyBear::ORDER_U32 - 2 + bb31_t(bb31_t::to_monty(1)), // 1 + bb31_t(bb31_t::to_monty(1 << 1)), // 1 << 1 + bb31_t(bb31_t::to_monty(1 << 2)), // 1 << 2 + bb31_t(bb31_t::to_monty(1 << 3)), // 1 << 3 + bb31_t(bb31_t::to_monty(1 << 4)), // 1 << 4 + bb31_t(bb31_t::to_monty(1 << 5)), // 1 << 5 + bb31_t(bb31_t::to_monty(1 << 6)), // 1 << 6 + bb31_t(bb31_t::to_monty(1 << 7)), // 1 << 7 + bb31_t(bb31_t::to_monty(1 << 8)), // 1 << 8 + bb31_t(bb31_t::to_monty(1 << 9)), // 1 << 9 + bb31_t(bb31_t::to_monty(1 << 10)), // 1 << 10 + bb31_t(bb31_t::to_monty(1 << 11)), // 1 << 11 + bb31_t(bb31_t::to_monty(1 << 12)), // 1 << 12 + bb31_t(bb31_t::to_monty(1 << 13)), // 1 << 13 + bb31_t(bb31_t::to_monty(1 << 15)), // 1 << 15 +}; +#endif +} // namespace sp1_recursion_core_sys::constants \ No newline at end of file diff --git a/crates/recursion/core/include/poseidon2_skinny.hpp b/crates/recursion/core/include/poseidon2_skinny.hpp index 9e75bcce9..cc911a6a1 100644 --- a/crates/recursion/core/include/poseidon2_skinny.hpp +++ b/crates/recursion/core/include/poseidon2_skinny.hpp @@ -4,12 +4,12 @@ #include "prelude.hpp" namespace sp1_recursion_core_sys::poseidon2_skinny { +using namespace constants; using namespace poseidon2; template -__SP1_HOSTDEV__ __SP1_INLINE__ void populate_external_round(F* round_state, - size_t r, - F* next_state_var) { +__SP1_HOSTDEV__ __SP1_INLINE__ void populate_external_round( + F round_state[WIDTH], size_t r, F next_state_var[WIDTH]) { size_t round = (r < NUM_EXTERNAL_ROUNDS / 2) ? r : r + NUM_INTERNAL_ROUNDS - 1; @@ -25,7 +25,8 @@ __SP1_HOSTDEV__ __SP1_INLINE__ void populate_external_round(F* round_state, template __SP1_HOSTDEV__ __SP1_INLINE__ void populate_internal_rounds( - F* state, F* internal_rounds_s0, F* next_state_var) { + F state[WIDTH], F internal_rounds_s0[NUM_INTERNAL_ROUNDS_S0], + F next_state_var[WIDTH]) { for (size_t i = 0; i < WIDTH; i++) { next_state_var[i] = state[i]; } @@ -48,7 +49,7 @@ __SP1_HOSTDEV__ __SP1_INLINE__ void populate_internal_rounds( template __SP1_HOSTDEV__ void event_to_row(const Poseidon2Event& event, - Poseidon2* cols) { + Poseidon2 cols[OUTPUT_ROUND_IDX + 1]) { Poseidon2& first_row = cols[0]; for (size_t i = 0; i < 16; i++) { first_row.state_var[i] = event.input[i]; @@ -112,4 +113,4 @@ __SP1_HOSTDEV__ void instr_to_row(const Poseidon2Instr& instr, size_t i, } } } -} // namespace sp1_recursion_core_sys::poseidon2_skinny +} // namespace sp1_recursion_core_sys::poseidon2_skinny \ No newline at end of file diff --git a/crates/recursion/core/include/poseidon2_wide.hpp b/crates/recursion/core/include/poseidon2_wide.hpp index 9d82401d6..8badbfe38 100644 --- a/crates/recursion/core/include/poseidon2_wide.hpp +++ b/crates/recursion/core/include/poseidon2_wide.hpp @@ -4,11 +4,13 @@ #include "prelude.hpp" namespace sp1_recursion_core_sys::poseidon2_wide { +using namespace constants; using namespace poseidon2; template __SP1_HOSTDEV__ __SP1_INLINE__ void populate_external_round( - const F* external_rounds_state, F* sbox, size_t r, F* next_state) { + const F external_rounds_state[WIDTH * NUM_EXTERNAL_ROUNDS], + F sbox[WIDTH * NUM_EXTERNAL_ROUNDS], size_t r, F next_state[WIDTH]) { F round_state[WIDTH]; if (r == 0) { // external_linear_layer_immut @@ -39,10 +41,8 @@ __SP1_HOSTDEV__ __SP1_INLINE__ void populate_external_round( sbox_deg_7[i] = sbox_deg_3[i] * sbox_deg_3[i] * add_rc[i]; } - if (sbox != nullptr) { - for (size_t i = 0; i < WIDTH; i++) { - sbox[r * WIDTH + i] = sbox_deg_3[i]; - } + for (size_t i = 0; i < WIDTH; i++) { + sbox[r * WIDTH + i] = sbox_deg_3[i]; } for (size_t i = 0; i < WIDTH; i++) { @@ -53,8 +53,9 @@ __SP1_HOSTDEV__ __SP1_INLINE__ void populate_external_round( template __SP1_HOSTDEV__ __SP1_INLINE__ void populate_internal_rounds( - const F* internal_rounds_state, F* internal_rounds_s0, F* sbox, - F* ret_state) { + const F internal_rounds_state[WIDTH], + F internal_rounds_s0[NUM_INTERNAL_ROUNDS - 1], F sbox[NUM_INTERNAL_ROUNDS], + F ret_state[WIDTH]) { F state[WIDTH]; for (size_t i = 0; i < WIDTH; i++) { state[i] = internal_rounds_state[i]; @@ -81,18 +82,18 @@ __SP1_HOSTDEV__ __SP1_INLINE__ void populate_internal_rounds( } // Store sbox values if pointer is not null - if (sbox != nullptr) { - for (size_t r = 0; r < NUM_INTERNAL_ROUNDS; r++) { - sbox[r] = sbox_deg_3[r]; - } + for (size_t r = 0; r < NUM_INTERNAL_ROUNDS; r++) { + sbox[r] = sbox_deg_3[r]; } } template -__SP1_HOSTDEV__ void event_to_row(const F* input, F* external_rounds_state, - F* internal_rounds_state, - F* internal_rounds_s0, F* external_sbox, - F* internal_sbox, F* output_state) { +__SP1_HOSTDEV__ __SP1_INLINE__ void populate_perm( + const F input[WIDTH], F external_rounds_state[WIDTH * NUM_EXTERNAL_ROUNDS], + F internal_rounds_state[WIDTH], + F internal_rounds_s0[NUM_INTERNAL_ROUNDS - 1], + F external_sbox[WIDTH * NUM_EXTERNAL_ROUNDS], + F internal_sbox[NUM_INTERNAL_ROUNDS], F output_state[WIDTH]) { for (size_t i = 0; i < WIDTH; i++) { external_rounds_state[i] = input[i]; } @@ -136,6 +137,54 @@ __SP1_HOSTDEV__ void event_to_row(const F* input, F* external_rounds_state, } } +template +__SP1_HOSTDEV__ void event_to_row(const F input[WIDTH], F* input_row, + size_t start, size_t stride, + bool sbox_state) { + F external_rounds_state[WIDTH * NUM_EXTERNAL_ROUNDS]; + F internal_rounds_state[WIDTH]; + F internal_rounds_s0[NUM_INTERNAL_ROUNDS - 1]; + F output_state[WIDTH]; + F external_sbox[WIDTH * NUM_EXTERNAL_ROUNDS]; + F internal_sbox[NUM_INTERNAL_ROUNDS]; + + populate_perm(input, external_rounds_state, internal_rounds_state, + internal_rounds_s0, external_sbox, internal_sbox, + output_state); + + size_t cursor = 0; + for (size_t i = 0; i < (WIDTH * NUM_EXTERNAL_ROUNDS); i++) { + input_row[start + (cursor + i) * stride] = external_rounds_state[i]; + } + + cursor += WIDTH * NUM_EXTERNAL_ROUNDS; + for (size_t i = 0; i < WIDTH; i++) { + input_row[start + (cursor + i) * stride] = internal_rounds_state[i]; + } + + cursor += WIDTH; + for (size_t i = 0; i < (NUM_INTERNAL_ROUNDS - 1); i++) { + input_row[start + (cursor + i) * stride] = internal_rounds_s0[i]; + } + + cursor += NUM_INTERNAL_ROUNDS - 1; + for (size_t i = 0; i < WIDTH; i++) { + input_row[start + (cursor + i) * stride] = output_state[i]; + } + + if (sbox_state) { + cursor += WIDTH; + for (size_t i = 0; i < (WIDTH * NUM_EXTERNAL_ROUNDS); i++) { + input_row[start + (cursor + i) * stride] = external_sbox[i]; + } + + cursor += WIDTH * NUM_EXTERNAL_ROUNDS; + for (size_t i = 0; i < NUM_INTERNAL_ROUNDS; i++) { + input_row[start + (cursor + i) * stride] = internal_sbox[i]; + } + } +} + template __SP1_HOSTDEV__ void instr_to_row(const Poseidon2SkinnyInstr& instr, Poseidon2PreprocessedColsWide& cols) { @@ -146,4 +195,4 @@ __SP1_HOSTDEV__ void instr_to_row(const Poseidon2SkinnyInstr& instr, } cols.is_real_neg = F::zero() - F::one(); } -} // namespace sp1_recursion_core_sys::poseidon2_wide +} // namespace sp1_recursion_core_sys::poseidon2_wide \ No newline at end of file diff --git a/crates/recursion/core/include/sys.hpp b/crates/recursion/core/include/sys.hpp index 36510ab04..8e96d4227 100644 --- a/crates/recursion/core/include/sys.hpp +++ b/crates/recursion/core/include/sys.hpp @@ -7,6 +7,8 @@ #include "fri_fold.hpp" #include "public_values.hpp" #include "select.hpp" +#include "poseidon2_constants.hpp" +#include "poseidon2.hpp" #include "poseidon2_skinny.hpp" #include "poseidon2_wide.hpp" #include "sp1-recursion-core-sys-cbindgen.hpp" diff --git a/crates/recursion/core/src/chips/poseidon2_skinny/trace.rs b/crates/recursion/core/src/chips/poseidon2_skinny/trace.rs index 51619fa23..794fc00d7 100644 --- a/crates/recursion/core/src/chips/poseidon2_skinny/trace.rs +++ b/crates/recursion/core/src/chips/poseidon2_skinny/trace.rs @@ -30,7 +30,7 @@ const PREPROCESSED_POSEIDON2_WIDTH: usize = size_of:: MachineAir for Poseidon2SkinnyChip { type Record = ExecutionRecord; diff --git a/crates/recursion/core/src/lib.rs b/crates/recursion/core/src/lib.rs index 2a7281f27..0fc2cd021 100644 --- a/crates/recursion/core/src/lib.rs +++ b/crates/recursion/core/src/lib.rs @@ -225,7 +225,7 @@ pub struct FriFoldIo { } /// The extension-field-valued single inputs to the FRI fold operation. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Copy, Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[repr(C)] pub struct FriFoldExtSingleIo { pub z: V, @@ -233,7 +233,7 @@ pub struct FriFoldExtSingleIo { } /// The extension-field-valued vector inputs to the FRI fold operation. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Copy, Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[repr(C)] pub struct FriFoldExtVecIo { pub mat_opening: V, @@ -245,7 +245,7 @@ pub struct FriFoldExtVecIo { } /// The base-field-valued inputs to the FRI fold operation. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Copy, Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[repr(C)] pub struct FriFoldBaseIo { pub x: V, @@ -325,7 +325,7 @@ impl<'a, F> From<&'a Box>> for FriFoldInstrFFI<'a, F> { /// The event encoding the data of a single iteration within the FRI fold operation. /// For any given event, we are accessing a single element of the `Vec` inputs, so that the event /// is not a type alias for `FriFoldIo` like many of the other events. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Copy, Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[repr(C)] pub struct FriFoldEvent { pub base_single: FriFoldBaseIo, @@ -341,14 +341,14 @@ pub struct BatchFRIIo { } /// The extension-field-valued single inputs to the batch FRI operation. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Copy, Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[repr(C)] pub struct BatchFRIExtSingleIo { pub acc: V, } /// The extension-field-valued vector inputs to the batch FRI operation. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Copy, Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[repr(C)] pub struct BatchFRIExtVecIo { pub p_at_z: V, @@ -356,7 +356,7 @@ pub struct BatchFRIExtVecIo { } /// The base-field-valued vector inputs to the batch FRI operation. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Copy, Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[repr(C)] pub struct BatchFRIBaseVecIo { pub p_at_x: V, @@ -415,7 +415,7 @@ impl<'a, 'b: 'a, F> From<&'b &'b Box>> for BatchFRIInstrFFI<'a, /// The event encoding the data of a single iteration within the batch FRI operation. /// For any given event, we are accessing a single element of the `Vec` inputs, so that the event /// is not a type alias for `BatchFRIIo` like many of the other events. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Copy, Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[repr(C)] pub struct BatchFRIEvent { pub base_vec: BatchFRIBaseVecIo, @@ -432,7 +432,7 @@ pub struct CommitPublicValuesInstr { } /// The event for committing to the public values. -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Copy, Clone, Debug, Serialize, Deserialize)] #[repr(C)] pub struct CommitPublicValuesEvent { pub public_values: RecursionPublicValues,