From 85c01e4496c9ac214caf99243aae4a87fc945162 Mon Sep 17 00:00:00 2001 From: ImmanuelSegol <3ditds@gmail.com> Date: Tue, 20 Feb 2024 23:01:08 -0400 Subject: [PATCH 1/8] refactor --- docs/icicle/image.png | Bin 0 -> 35743 bytes docs/icicle/multi-gpu.md | 58 +++++++++++++++++++++++++ docs/icicle/rust-bindings/multi-gpu.md | 2 + sidebars.js | 20 ++++++++- 4 files changed, 78 insertions(+), 2 deletions(-) create mode 100644 docs/icicle/image.png create mode 100644 docs/icicle/multi-gpu.md create mode 100644 docs/icicle/rust-bindings/multi-gpu.md diff --git a/docs/icicle/image.png b/docs/icicle/image.png new file mode 100644 index 0000000000000000000000000000000000000000..9e6aecaaf38977e47e7983116107b0d821bdfe11 GIT binary patch literal 35743 zcmeEuWl&zr(N#gl_e@WBpYEQS9|THE3L(K`!-Ii=A&Cfokp%;TPzF9^u+TtD^_Li2 zFfc@OV}5>V5q^GrX&XxeV>5j)FyX)$B^YJ7evDML$cTua5U|2<&2WTFUePG3V7me+ zF%gicq9Jg+-C;Bobq;U3gGz$PYGMp}OQ?dHX))eL+sk0$P+0OoG#pYcvoAAWJ+j(v zac_=nEWPsDf+_hD8DIh)K}!?S69Dz8Cnx8Va0(c>JP$Zo+2*Mps-l_MTQ7R2r^cp6 zxZ(_TTUo0I+sD?R*IM}N;9fuYll$XNP6@cc!CF#Dzhi*$$nQ|+z7>A!^t!7-_8op# z;ae89Kt~Fz98^54GLaBNem5vECW<)9VQ8Lpu1N-X=h+CMb%rh)ofct4uPxYb56*lv zcsc_6A5YIn$*rKe9DZs-dXDEznMv$5WWtt;L$LI>85|rU#^Km!dS=Sp z>B!{>X*loU%0G2wtTpYB#c(l*y&V9(JH)Un$n%PVEeG#HfU1aqj2SlA{oxctTSX%_ zV32>lT|W-q3a^E{Sua9`{)Cg^m9^H(7~~sBjv|L&L-Ceah`$KDypoEUKvtijLAWpM zL)G_zb@UT8-uJX90Z`;OMFQ?8GmrZ_lQSu z5WqJeMmmMn)6IgmT}ANbkFM8Efv}eK4%+tlflsDckC7{Us-tjO=Z8-#o>A-dOG7a} zT{V~#vu!wkqqmX>syuLjM@gh8i`2vBDql(E$M+xR9{iH8k*u(}Z0?l@S(Cb&Qx^D1M zH`KgU?uv*cA@CdeM){O zTGbnB<+*?v?mff9qHaxc$Qj?u?ApP=hqL-=6*?ou$}lre`IbLgmxp}=Sz(Is>`ry% zbHv0eyInGjr`nH9AGY&UaB<|76J)Uvgd@rdzi1Sm>iFx z%XSXpU{zD#EX#dSVhAy8>ed8po^TRQL@HhC-MBegr;EF1+ zSM_hj!5P}rr6C5|=*zH+UM00DmZ9x~!|-sjKuCI5uAtUISa`WwUDC&s{SC=lxG%npFh^vN9{%jNci%rUL^um$RY0f&nDG{Te-Qp?gIe!Z zMc^1AB#e0qedDV`hCU@AA&p~4H1I{mh$!(}m{8?dztI~uJPmKfbcG#@s@I%>ntYb& z2RmX_n6Kj5a>5gICEuax>V8tjj`b5MS5YQc3{%jqRz`ChnJIH9!nyl~X3aLzma_8t zb#Ii_2QI8MKiZy|RrV9Mdqxi=Pv5H*EjUK+Pwn4V_*YPRP%Gf%V2OQgIvhF(IpeaYk+i`{`5)uH$lz>-oIUk9p-bMpQr=*G8C{9V|eNGKAI+`o$$JwjC2RHRX) zS42q!_Xo1cZxaKP5|a!QV-uZfoobtE4U^uMtg)V$) z5kJJglBpuEdCM(INE|>PC(E0kot&|d+oYhRR4aX@7%qpIKPpoyy`1Zz`SAT#sNnS! zXdov?ZgY}&5_b}9l6|t45qsDlLs%>e{b%gY0^?jGWMe$W9qbY8_ed>0W}%{?Y@vnN zrHuWF(}`5WaEStmn~5+?w5ATG->c24eyOS#tEk4PhN{+9xm54YWmV}`omFF+9*wu8 zFXr=)myf}Xt?muY%of;+Zo}bbsg2LteQ^Kaj`M)n%paF$S~xg1JVrgSm^EK8pOu@n zT@qKstWYX@l&9Iqm1CJTIyO3?Rp80t+%BKlpU_>B`ed zGPn$d4tEZ3ZmA8a4c}3jQAbh*QD3TZl>Jbqr>>+@Qu(ZQshpq^TArw_pld&qXf|Ah zp;}tbW%#hvGfmQM0Fr2y$Snp@_tx~)^jx;!4%4(*RA|I3+Ph|QPH@e5_&lJ)OtyFO z?nv)i&Azcew|mQKVP%HH`)-w!k0aBhi<-0mZ7~05e$Vdr$`;#b+j86Hv%>F+A14H9 z@SH6+lrWLwIO5D{t^{{Og5D``6c|t9D8VSDnr83WGbgI$&70Li*UlV<9YS1%9@cX1 ze9+~Qbglo6FOgpng7Z6*H;K5aXme;=Z$5MGe_p9rK1|fIt*`iV+$Y?UTV~{N32}*xJtRJCx2q z6|6J->CX-}`>QNE>SpTt3zgS0Ck$(QtUNcKgY;@(BqF`j5fYD9A6v#Kz#a@Mom zxQkj11Vvw#f9xH3?@B&JrVu?BAt)*)IiEVsPNTQ*+ZuQG!!GeIbMJgu8rx$BWEPw= zTA1Wya$kxQolnH~$a2bc24;V??t|eCy^_)W)pBQ3C)yj(y}~^WDDA7?igT>ibWqWV zS?hkxtVRczc72&z7e+B!^9bDNZ1foyGl`>+g{P+I(vn;CeaO9Fc^He5n@i;GkNl`P zqOeF*kvqM#{zB3#;g8w#`rpi&iZXaGJtk}W4mP;^@z8HE6)`;GBRMu#s*$z_aDQ*>Q&Bj)4j~>ROeKy=yA6QtZZ$)o> zTT1q5syWb+>R_e+Y27@K%JQVqck(@gan?OSRyjg1xZK39RUAX3WSq9V0!Wrc>{rBi51 zT4av2%T@jrf-O1Cgaw(}#Zz;Ql>Yv$(bE_^ zoz8TcShBd=Sh^3lG$Fkw->6^uxK!c(HO*}~_yOzA^20ZdpKcM(WtU2OS~GZ=cofC_ zzjLN8CK2}Lab$7kus>oaCAlTN#&zIoTK{(QeIUz4DDFen`y|hzC#SKKCA3JiMbb(s zF5}c+Zby^9l#k$YA2~kuL8T#tVkThj(K>4un4TE1o+tHM z2B>nX?a{buSk$e#XXl&S&6gK~D@kc0v^ZR{ZQo|&j4{tykSsR2Q$8r(nLi$kusb_y zE>kznJ6evocRY;4s3H&GVRIMVEy7%yS|L6gyG=O{k zwrz!{vI5Mh@)ejW1(@BhTA3bl*LHB;=!ZQph$lG8yKe9k$aD;k)?WM0(j+yo{dwql z+P+da*C}Oeu)rlSds`fAitqaM^%V7aWl{tj`$wJ#uAj$6!CI*Hj*bd{A3ar|LnFBM zqUq=(7oZUCV}0)R=pPz|d=iX)*mY!mbb<+jU1$%#@&pGPi>PXl(}n}qAF=w%A_fu? zU=+YLEEp6xD%dOF3LN+a1IGq~`s*4DOcWgF&$TQ#**`Q8U|<2pV37aNr~=>5KM}wO z;Qr_PRir-{3~+}Ee0)UDL zJDHnV*s?is68=TO23$Wk(-PwUMPg^lNvJF#jn8jsqmTcYhK`1gkP99kAD_cU&wx$# zi{L-(z#S){k)53t8!fG)qa%$Y1C6DPA?+ttR#sX%dRlsVYJh^;*4e^N+lku3_T%53 zy!7)$-&WVg*vih>(gOdvUu_*rdpk}-!smhh{QNDazLW94BU#w~Gc90(w9ijyKhe<9 z{&#G4#s>czw&y2*v;8%%zlY;^?u<>rR^Nu-(%f9%!j9`-jdT1p(*N}EUpfC~ls0zK zH&gy%3~<^4bK?3;NB0lce|_@ba;p5x$;3qepPc{kwKaR@-U)m^1uFf0_d_pfe{ewE*EjeX z!s$*CkhE`SRu137-U{(qQ+F>`OLGY0zoqtF`>Ep>$@2{iPKNJWbDw9+-O|GR{5+>K z>CxdO!*NApB|~Lp+G5IG*h9{&w_mjw(rXA5FEB`aFmPC&^skVKm{i25-v8W#L7*JN z;+HwS`Rgfg=fwkp!!LtJjNbUu0tON6`26H=mbdX>5QS|#BL8*@u*3Wz{{QU%Uv;oN zf4R&5UAN(p08JGB+UOBiGwTu8Slje0S{$9aI9ejiC41$^pabCAksW-g-s8vqn{i@`vV+(cH72 zC>8U5!BL?nPA%yu6vg@3C8c^j1&pE}mhfcL4L{vT-wdP?kVhkfD{1bRxrp)KNZ)Rw zQiAN)cW$awV;*tWgNTXJArv~qGJ_$QT#x0ODhvhs28h#7!OImIwlQ%)0^q6*aY2+Z zFyQbLL!-P=Y8LOC@ojndc+!Jl_sdkpDznU|I?&^Is30-c3hHRzST|$B`yp|b_f_x#B1=I5^bTZKYHellTnnAsD1^Lf)DtuabNWeFVlLoz}rBf zXIdxT$e|b!$;Lp+$NzS1!%`!mGRQ=rg@B}bCT)RMnD>q`C77!fvz^GbOc?{@A} zpe(DP{t{3muj^m5!1#_LgD2mY17mxF%)MY1BMh|}HE!1bYFT6fLC&(NAex9;$4tl((n0dU=ApRH zW2D!^7qcK+3_FOE==1j>1SkohI$cNksl>ctJ{T(@JSt6Q(75*~BdNPHQ!SN8GcSlO zB;n~GE{yP6T75NRy7{na<62Tk7soP%@;Pam60UG{zAD2k&mU%l(+N^uaHn}><6Txl zp!DH(q3lREAaTymvy9@=o$m6oAEdEj z&a3(;A;(er4@Mop@mb&l{exBAthdB><`OuN?3)NULiJr(o<_T(dQ`}PARdx28N~!@ z8-wdra{8~4vdGs0p09QFzGYhq4H>faDQ__AR@u z;M8#7pgW!?2fF|dsak;>>5>2i8KOS=%?ZA2Q)3YuD8>Tc^n@;w9ui-A1d={b#KUgj zX7@WYtUL**08KgO=3O7t-{}erV(JSRNI^m>$`$mI!Ki?IHL&(60uDAhwax2;+hca_ z2s!~6s1D_uKHQ*qVO`VGNt09$TK8GK?F=B2^BX`Uic%pSi4KLgL2)?B#Cz3yl9#pooF|Qqpx=WPfOoLqgkG%8;uNr#(j>iZC83D8+70 zVtPc-dsw<}Oz0SjdvXaYufSp7#sSH;&P@dOpY#iQhsyJ{`#8gP@K2cgmm&`gV%?A? z-%CAG07&$afSM?FjlV1X6OAD$yn((5KcFS}zG(8lA>rAF#Q0oBvDQo!^-o|7B0*sX zDU3-45Amn-Kq^;O^5Ba=Ki3J6;E;-t@0tjn%Zk4W2VJ0Wz!nR0dJ&BW9~hAm0&xi? zPaeX(;aElKcjd=rfb#++ov0+$us`LGwn+eW>7U~FLI16;baJ?ml$ttnS}pq-!LJI7 zA2y82+Scrne8WL6PqJXH`g(=pW@R7u`CU3~~S0mrTg zDpt+@qUummVy9))_%x;Snfu2@UtnQ(FlJIX_ibMj(=BzS)hb`*qQiWW{fx#;mC;xS znt#dxa?g0KgonPV{1Ges&FTpG(9LU zM^SL5{Pt5o>GFm`7;1fb2)wzRw;VnatV(rFtnSb?~<)^w^KkWGhPtsPKz?{Rd zWPRe4reRSv8w`A|rT7yO2FL?gw>Epm&W7WS#|{Sg6$?J`E0$<#kjwnc;;-oYo>zah zK;q>Oa z!CDvGqg>#F#bQH3#@)WI!;&**<(x?&@qIPJa1fOeyJm^5ZKYm6YGr?q&(QhqL{!J_ zN(TZhJq)f}**5ph%#KBe=HO&`0*7-2R?}<_`EK8OFzu7+T$Qn|M_yY#`*kjZRoVr+8}HcJfA@?b5L)iGU8p9QXysNVeF-bwDJ(8#lHxe2 ztG|d<3uy^3OZhTQS=waca`}7HJ?&O9E7>0=k0Y|r<}d)yqwGZM^C;(C(anD4XtLvg z!_LnPgE$RG1|Mjo(*b-;p3Ml;;Y@Wa`HJPRM~m-RK(%57Nu9xDCYyF5-N`;+oMzE zd`V;CyhSsUtU9^2pTd&ElCjX)w(TgB?e@q;Nv`L)&kAik6D+5=I{AK9_@f5SsPVpX zbQD;O>-iDV>Yf|+qk@WtPpuBn1L=1x|cRtx-*;;=9#o*=7~F+ zqMhzY-2L@>xH+-4*x3mC5z(4Mb(Cgumbw+uuUH(AzgCaUAl6NPq8Dk28PZhs8T(B1OG2djQ8AKmv({z~C zuZFY7E<39CuV7|O<{*abRDXJWXy!d++itZ`R=&rVK7=GHS-8rNO&z3d9a8sPw0Aaa zbiaG8iM{K=!?+|SIm5X(#K2Ia>2DVv%IR0@*DKMd=sB zW0us|&&fMo9tTn)z2n?3;~xfmQW+(ru(YKiPI5AY2Z_V}Fldl=w7#*u`@(X{KHY+U zoAKjVeT(~D{bF=+1O5g!A1TZULQY;u57Pk^-}XTJNcbRv8teui&$5@6y7B#5H>Q>z zBl!;$Ne1{$Jp+OvWAU8AV@_R|QRW;0_V|(pl2Mk2bKhFe&vcu75D4u0U)q>A%0p}f z6e~#5Ra{H(?>6JKhE^5T^|9yEyOdOoi7RK8_=hP9i_F;UlzZInPlU*s<~xzOJM5|} zA2u?3zD`ZnnhhtjPH&ZNS|^rES27RC|qn5@#POT|tf?+&jzFxK$wF$+xPoTUewZKDywAm?oe zz1k>pbHD3qGL|g_!oekXzUG`XIyPaOX>`Q1y+K!s4`p8Z1s~==m%Fc7T}gsA>4)#Q z0)k0=svN!@>0gqFx8)3=Rh>ZIU6`IX(+y+$ZS0 z{M;0aX?0tPiDEUSvx#u-gw}aDva3bM(m}of#Y$x2vW2T6uxJGG_+zqHK-Em|1RL7# z9qS^)(3GsIC&{_FQ3aLIA+n+Fe!-NZ+Z)CvZY}iQ`N!;&*Mr1hY;jZUM@*g^)Xav# zWWhVa&@%Rms)->u(&U}Uvzk5S_4KO+*HYGp^NFF#QI06P5nMNRha%Flp*ZB*qZoalF!AgIGsPFdaQIP*|ih@^F?m>;?X6 z=y-M8m)QLAs`bhF@z8Y|sOZQjIFjPMO21bJ+IDQ(65pq$R0XxEWG&2e6_q_2N3ZtwpQ)-xu>6rZ_9E zsatKw&9*veK6*2z!1*06Ho`;+vpM*+;1tH#RXnCE6izQKjStnn$H3)=Wh4h5Ua%$h zQ4IrNWL_)bE5l2&?lo6{E*{EPJe z0CAQ>=37CMOl3IH19R!{2?*cq_)Fy!n$0d#!?4ra6uhJ}#qK_#g$>F=DNk72gkNg$%G|DVR{jac)1-- z9`ONj;)AiPitP_RJqk{P#+Tu1gZ)+f(P}+3NpI{bAaJ*CFVx6aai>yt51j&A;i%sf zTI32ew-pQ;uMh>Gkm*9G?l06E3WmUh>WZ>u_^sBbUnmyb!G6_=9JOFEULzJaBGSv2t}e{$Erf-_^EIZOzmFIp zwJcu={&#FJcIywu87)tD+dSOsL&p-5-&YfCAl(C}T9De2?yr(9D}dZ zXYgJ8)U^H350>E#t~ec-_lj#h2(d!v;cXAttFD@|jfqqz{Z^S9={g=!zSMV(vtH9r}qh|SXP{VOC;qunA{MAuHz3!2HtUjA7c=`|Nh%14~=?! zOJ?;psIi7`PmY2<&6L|mHHF2RwFdbEP%)Iilo97`K0dF#clWC5KVi@_rB^p(CS!0P z&Xj7cRlyI>TeJ>`xSVwRYPGNf8dKr>>Sw=A{;)fdLrk*{8OAYrnuFylV=>GGR@O_?k$=Q78P+{<3@=?jw4S%`TLpMksp$dJzrOqaUzOa;(&p>iKxnH)LMBcnK7+ z4id>}Z1x59dn4>ay5XF$)$yWV!hYRcXN99e6l&Z#WMt%sr$!z-E@yMdOX7N5Yk#Mn zfm2AuN$fv}x~b=hhc~UKk^$@k*-L@V{O1d}{7v5SKM3#l4%~{)7 zj_%LU#3!qB{GvOdBS|5cbb5<7^YaF&8`X{Amp!>hC||i;9h&E;e7{PQdeY^pD3oeC z9~aA5P1$}U^nMkKXR4fJHg77-rhQCX%>sxfo0nhcH+rWomz>r^`y%cR8?|%{A|U$d zv@%krA0x|Uw`0|4n((J0Ib9Fx0}x`T6m@%RmRxXXiy-9|i`onf@^eT9>hdrcJ*}9R zKBr`iCEm8*K4L{aGX4tSVMVn-P?2dK3+NPWF^ zRZNFGauZDmQs5iJXqu^v-QrqTMz&QcA}s4KYJ7R_%~!{}>WcFcf@U>GnIw&^ETT=v zGE4LgMXHDc^75|v)B9oTDzt}wefsYFt5a!AM#&{(5K7~ELr(IO`|Y{NMut*i_beKy zVxfM9rhcMQX;BXYDa5gODjGorf=Ia^?x(kc%|>Wzx@og4ys9_hP`>k0Ant6OOgp>Y z*)Rk4dqlCpn-qh%ig$I>Jd}AR;&CG>!=jX>$$ECus})#!hvUI>q5yUeJxL2GA_8tl`-|&zEY&9V&7prKxM1H!SY&J=G0*iG)rZjKa^&w(FMG>7IMcM=2mP?q zLsS@9bZkicJ<9#G?>B`U@Th(;eOC3<$5ME4x!}tPF=%g`=O6CFll>+lKZY##MAD`L zjWkIPm(7muW2*BsCFYXLnFNDw4`O8|(!_i$l4^nA)`x|;AKJ!`hmNfbYG&mEzsQfU z>F-uzQ{j=K{c?MvG#ww13|87efj=wgN-?Mm6}@c;K>jK8AAwT05++i@p^8K~+H@jE zVr}wXz#b}}V_MVI{D{`N!eg7#v}858GApi7rp(vgwony=#$rpwJ#OcFj3E}MxoMKW z9!%~h$(oWdj^AXJcZaS-(AAWNeMxf~Dlv?_!*p_O4+XQ2p* zRWWro69Whfg^6~!ukod+Ac8f_i8dfg?$mgR$~N?1mG^w1XW&y948vxL=_b=@1Up+L zf9;6Lsg(6$h-ub;g`q2`R*qd#n^!cf78g>0#gP?Tk{g@@!QkqJaoxuMn6^Q}iZ^}Q zAj6DIu$K3Devhxj32iqT>M}7*6~B1edg9ERFbu*23yA!f{_2%F1o1@1{`=i)e|v}& zPpiGy`26ZM6N9fBd`f-x`|&3Bobh4ss#BR$E;T_KllHwS4vh4@D~_U4rpc_<*|w4t z`8f0iV(SlAOJ^nv6zB6QS$+t`b;*V^@=<#_Nbefv4jpU!OF>uKlGXa#CX1o0;qdd9 zyCE54@UD7L{U!b|Mo1lWq(12dKdNf7Tc$A&zQTMn6?tEMeY@D?d^Tj9n_@yD8FDD9 zLX?tNIP7j(I@~haNPXs4MN9K^PMk-SDO1}f{dezRAWz?Nxn(FjA=+|)7cU~7AEKUPkurKuBSm& zQ|h`wR?s&0HpLJUuGM3u{hi~YeeJBtUa_3iwVS-xNF|fp6GY|VIjYqD{Cmb0*0 zv)F;}CFV&Io6Du!ve4ix>C z8Rl4W-89PE8S26vUA$PRWe3=Gq68~lA*YRNL6lz%?!x9RrIMq_PH|L}4$j+Q@aSBR znw1pSMB(bE_BtUI8|)v{N$wSsQqpQDe3@eTip`=7Y2nb7;I}P@RkyG9`Y0zg+{Og9 zf@!f$jwr4)W@*Kyke?1Lw*q_dkV7e@YVjS)@WVsefU^lW@T(CTwN@`1G zrK=xLd~JOX*Hr*j_#Tf(_pIvvbPQq-1art->5-rx#{DRj1B5KWb#lf=;m zxX7QZ*NAA7v>_;y5(^rIroJdBf9~n0i7|UHvw)e>wC>)_5ZP&QST*a4O zXQkh43x+$=w(WkHP7JUL4?lg8i|-r*s4;DrSZM`t?O->q?Y?#3iY+FI!)zcN2+l!!U-SdtN^BIcYgjz- zn!=H=5G8zyTUQ35&J@gr2B>P?`)6cmHpC*XyNP%dYU1sJBWlnXRjWM;i)$!;p}(#Y z+xwDz^>DRJTC%QnCZ1X%EMXX&lcV`{bApa>l+;C%WB11w5BTzrgD0*B)p2prhlVXF zZ~T0Ch~HRs!ZbJ>m`Ji2E4Npa%=UKrr@T*0Rc%@bF{l+ZQEt0@v-hSZ53ey|bVjfq zMeu#A4FM2Z<4C{jb1l5phvGgf8hZ7mQp6zu*O75s%PNCepPlM{r*?_{Y-{*JQULwO z2#=w%+wgsq?HV1GI0<*)hpUg{t)8BvI(9iMv#G7ukz!~JnfH~^6`@jfr=-#crnI+e z;jh`Qzpn4sZ=UH*ntNPvUL`8rmZByM_2PN%r`Q_cbwCKtcG&a;E;{03Fh}8S@{#Z* zTZ#gebEL#_Iv`1`i7}7Oa2NuVtvq4Of%|x5`l;FwOV?|V_D0WBmE0LJ$$U& zr(p118R(F)756|XzIF2(V^1_iM(*(5i}afG!~qENB83E%X!Is;YS z_SJYECffGGIMeAd&IX6!b^X%UlP^4tAKz@K)}H3)Q%F2`^|i#<_CD=OrCvDB@NL~! z&K98W&u*vIzFM7;fSfTC1a?_syq+GWP`u+FPbv(hDB|v9ve%zCSq_`8TDlrMAEyYs zjr$pc_DCjKs9J6LX|A|eyrKu<_SNgV*D4bN_obNYr0P+KI;`tajB37WH#7Cv^vk5) z)LEaqF&L;`heQ)lPQ?>YZgZE>k5XX}xy(2WH*IRE>(CQAbt28UGHi|yLysdi0xO*< zvz8kSIPJ~xpHxlDrOmPI;lHXh=?q zD`EQHJGNsESSSoT%$vtmq^3I^G>%HIH@xz>f4n^IymX!GI(zI-RIn-urq0saSC&Lm zZ@E44Y~FNIoyVxLH}CE#PNbWsabG`Lvpca}l%>%VGHXitd2&qJ!du*8WKE6Z1`FR1 z+(tn*GES0%txIYu%($bsC-`cj_3l!Qp*JW;)8V%Rttp<6DRPZ@67c@bnPDhvkM*c> z$Kib5|Ivr#^jmbn{A#h8W`MlKcuI9!%FozDvWckKQ+{yrl-bXQo(TK%2ea=?L2UNF zz$WzrC{Mo(V zrNt(Yx$nQW;IdCMkj)jlGP5h1vQ~D$Q~H)*<(+&x#BtsD9x!~fow6M%I`?|f5YLot zQ*?Bcwv2JTPAFi~?fRqXH&83pNt)>p*CwBV32f0&2t0ZwTcRzzKvu{2nAg=!4T`*W z0UWF3iqQjYD5Z^VaEFB$wMa!WSMMWvDw=ZReudbNwsBMjRApXv8B|j4CsopMM!`Xb z&99Ud!v`%5P*ZE!ZW!bF>|bAlf{6dJWZksE%68N+SAXS`Au;`V#q{$%PDhqx@+ zoppwIXWU^;WI4CTyr%4FG9ju=-+MSqg3EXp2a`69dXtaq{F|B)5NP<&sI6Tnx^kv) zj^ugg<}h?rO~qs1B$5)v!B;k*@0gmGj`01bI4ywxSM6+pgyQ1QvO6X=_)7EK~cI@1fohQZQ#d?-Kv=2A5 zz>q-{5p|XegDs^zu)rw|72CqwE1J^eHdWKo>}9*!;!f>v>;gKrGy!&Pnbd zB#KqAn@Xlj*$W4_3z3_i{)P?qkIytg1NzrPx!mFv-|pY1ehrePMUmj)Icnm4>(yZ4-hmWmtT z=hZ;^lr>B7h{I(#7i+;=X-{sQlhf0Q2T#{RwtX1^E+-0i-Mj#ZLW+l4^;T}LtcSMX z6dgG{l4wDc zo3x)AC~k@s4XlgtP#^bL`H>Y-v^rWKL!iKtrUz0#@}8|`9Q%Pm-a{I-pH6(m`xlnz z1tbPA6cGrBO~J1nR1KyB2MG}XcmrP(DDY2QmKLZX3F9Imkp2P~ybS}0t*Auc{;)d% zR86pdkPGQ;z&#YSsLh`iFiodMzgwR}n&HbdyF9F&KU@Cz6c^d1gq9yob~ zd;Je80t^x!z?LXj{UCyU!AUFtIEl4KXsf4}&fg{j;;aaZpg>4SeZ3d&A^iLxhYte) zC6b>h@6YS6exbf&Ft;{4O2G*Lz#G!=fT3iix3FJ2=n4XW7K-6XNq}bl0!pO6g)9*L zI(8rvq~veIoSiEe8n8+V!#~)y6 zIiRsnVj+${eWUP#fzZLBEn(#uK%eAPL~|3ko z!4p=)wb}M(qV?rmP{!pVyJ%Mr-XtYuE8_^#ZNMqSq}pTzLtEO)yrmaaEH^^)7 zCYpo)9yLzxj$z45qD#WC;I34^vb}Y%m1yEw9b^j^pc#etEi;MCxF}!=uP}j%#o5=~ z_pV|I&%T!(uBIDdY9gP2s9x|@A5ni*in{32qsQ)x?-j-mD^Corz_O;fOEK+JtO_Fj zN+H}U`3ywu^mZ-ROgE8Y*5Z}3aLdb|=KL;Or4ut>Ny?1^i~7@X1^c7-Hp@sF&!vys zX{%z?k8}@xlLHCw)ZkbwaFYTg0=yA)2+oTpyfAZJqc@*cHw*dT&* z-lWqZk*WoOAFV3&U_wX;d7Y*+?(2Rm^R*2*pn0!vh&isWoER5e82yY3-Pz_zKpFOD zm=fTju2B^YqiG-KoKc3O3YbDKU=uWBr7$b}4z@yd zhrG-?kGKwi*B0X-mV!=Lx(tb1U%?hGq@@-*$?OiKXvp=FO**kvAHN;YS5@bj-=piK zRg>Msd<|0Y81;RR4)Q`9n~Fqjwb@9S3*u8qr(K~87@((UfX_Ua|b zy#?`1&9_rPFhsrp*uWr+fdDzaw)zK$htihCkw6=KT;Lt~_m)ZXy zm5zF3G$fD~TJZA?ue2OzYB0$F0}S{}hpDgjp(xJg@*~X5Fjg+fLyL@PifyiNx8$2aIqqFeu;9 zL1w)svXWiDt5*!D!67k9dEr4{dFym?H)ejA-syp|A@fplzIMHWD97h9b)@>hz&r~J z(chXJD(pozC9ooBTF$w1sY*GuH1)cW_)a7`pCs4^404Shf4)31?Arhwg>D%1ya}mS zTfrr1a_55|dMq!&W+VawJ)42aqAVK?q8b3qY=DDQD~0~0U*%0RMElNP7gqiwXtFc| z`zX?NAgImjkUFv>|0^PzERD3ZTWB`aV1_mVsDKnjl>k9yF~c)mJp;Y)w-%B`St{^% z8A=3!URsO?D6-18uf*IoQJ3)I+<_q~;C!n!p(u?TI=`|Kw63s7-K;(%T9t##=2DGP zud+JmGzt~yZqz$r)@>01vy*_*Wyb8#=>ha^!GPPyM#Blwgi3=+qmnch?H#ALlALI!1`r9&0&e5?Pb6aj;< z26mEz=Fzr2UJ^JN8&Cm^y|MjMBCP=%>gctkBM;BxJ-@orc7PjXe%$&~pLK--@3nBkE;B&C4`uFLDgeHRhzN002scocX;@Z z;s4jtmPexKF!t}t7L@sBXQtS%ek0r*ZMW8_hOvC~HRS)~JY=XK?0M4hdUFP?-;eHT z%#A4Dz=1IlKiIU?){UGp{C-i|4>y!>1C4d`dvM1z9_f-w>d z)7*RR*YTd{3QKDmW-Xr{hHVC^m7rOJ4KYZ>_!-=M^8Amzpij1d=R64>_#^11_&CQ;hK4p8*6Ddh}Qel zU=`11`zV^KNOYFcrd!oWt&G>t_}?VTlDY)|s@97Yr4GRRhBX|QbrrSTFG958q?aHg z`k_3yqDUm@18^OT2&YH^JYdt=p=*xeXeu1HgQ(C-r4%=lVY0Q$n)fJ&ZljT=M|U1d zCvTe+4|juQ^F_|hZf=^mL@;)#_he>{AAlMzelRtht1g_cF_{{us-i}u-d`Ti78||? z4tS5n(rckr--=dbBI5H(TY zcB<_2QwVzNOFQiS>0r^%s!4q%SP=0i_eJ1;KsOKv}vxuB>P})UbUb&zsB6~X|C$TB=0xE z41%Uw(#q3iUwCQPd|ebV5zj4^l-b+>3*r5xW1Qy}ypC>UqP5VMLIUmJlQrknZl5 z?rv!gDV;|^KtM#K4=G4@cY}0DHv-b#eR%d!`F#I=*0Y}9yWV%b{w4eDy=Ug0nS1V; z`?{{(*cm%|KieFAFt?Paz`ejRw#H+WjqsMHG}UJX#rbp5^Jk-XJ4W|xHA_y?W?yk> z6h8`dhpym!A;+li3dcukLVd^32HngAb8n=%8>)ISH=TaDl}(Hmm-h0rz$gRIQ=zB$ z4BB6&-T4jbCu@B8F{`O4ur!ap*3ymdHVOIOK*~(JsTaaOOq80?c(3zT9=HSPlbV_3 zNH9lR-=zmg7GA3kW-Kqpxq@Fxpqd3YrSEk---*dDS*hSF)Lh{0i7M~KnzVNJgl$iu zNMt_7hQ4d&pt;SNPYZI}vOi+E{iBOef(+aiCVWnpLoC`q9Kje17VDo9*K3y66C9ea zs865Y=m~HTHUpV5i>z2Ju7)|bHdple22#^8~F94(9!mUHOrV6LMCC74!woIgLg_UyuG7oz6$W< zzJ@QOKZ`7_Rxr+g`uAlcS;(Vdc9Lzr#O{_uGGI8jr&W; zmP(47AU(}xf1XkHji}jiD=Yq0Qw34T$qE{?c7qIAf*f-L1D$7PS#TP3LpZy{j#uaA zq9u|eA#eGq1M~#_WJ1g6`CYC907kJm_$Jv&*qhRb;C% zy=t~(m+~k5%oTQa%TvnyR5&W-)@#WKYWL8`1AIHhrsvb9N)ALd{M>%G7xvs`2EGxT zI5)G2mN~AGgh!R`jOfL&oEvA{(<2af9)`+vxiJ0C$AeZl@nqoJpSbBZo@0>_sHqCO zBK@*SgBdr-&NI*4Opb2-hIRNY>uRe?gW}MN%hC9q+B;(eT#;Q)z9SrZ_V~yp>^SJ< z{w{{@7!8uR&y(VJo{UX5lK@zD+v*t$S`&`l<;~2`@|Op-V~TE1=FVi6c7TP~f~jJK zk*8nBiQjeK3|L&IBz*P;0JwZIzfY~_e%;2`+YXHVx*L1ug>F)Bfh~xM$(^5Cjv{BK zRFl=>)fB%b`R5Z|&=b4HGG1VLRnxt{J!(InSj)M6Vu?;)Sif*9E?(TS{3Kcd^mf|y z-O<>A6cb})n$K68m!LDw(x6zhlXqyu{Qg-;HW?F2-e(g2Sus!ao7+j20-eXc`J9T#TTe#TwDe878lAsQhUXq>nuKD$ z;9(gcZb79;XX;4bndiMZQZuR0IeR=mGb9w}Axl#{?@H#q`*Y+#sk-4-ks_mVh&XjpsTn?H>-B9X1+V)m8;Vf{pZK#Cz#?=~cFn+jq z+c?d7=8y=Qwr=|g+#Xu}%w}o@vWwJfNzT@5<$J=$B1~;)MCaE#(G6(^D=jq(O|^Yg z?^}-r8-V%S=00AUv=^CK$Qv@K_%oz{3=C%X6W{Lj2KqXZ_vm}HDr8m|g`teR%{c_m zbRz}a9RJ=A4ukhHy{F8)qy_3V8pP2oRtp9%DC&+%W@>*EzXxTG1b7cpS zls!7vyGy(PjD8)60=_yQp>>_Geyt`=3or8D-5^W^bikZ0vT%h8Ip| z{UR(YC0mXZx*o}O<&LgZrJsF-^kVa?691wN47|2m#|g3+=!pVmePhRu1S*_reuvp* zqz*+s)+~X;kfVcwcJG<2c%p?Rubm)p4S?3M%=LT>=9}3qcsKq{Jj6BtU*MDO-mBC= z#o;dDP)d)~)ni~Ib1azID;$LBb6}75&33+~I*)H4Z^A6?n|DZx>`^q~6_7@16zc-B z+45aH#+-K;sxIP%U{_3~dVgoE4Osom;lZf|@P{@-qzq8!>}pbky=87}s_I9J8_o_p zdc|JSeh~yV0v!?4xtyEx!uK3Xr4n&Q4n$c*Papzj=cuWh9=dLa93k_$?gfS7+~s1tsMXx`p!D4l^{;Y!!~L}{ zs>4^#4Y8!?V1Lu}a~Y$e^eokTU`%nU9S`xV?OoCCGSB8NyqU42Vvb@99=6a_59nB%7?vg9pS5eF!OZwCA_khip ztbewXKBN)1^4l|uTGh8QvE{&CU_mdmJ#8A%?S)0c&57p*O(=cP&!-D41YF4()zy>9 z0=C-Pj>Ae9KAR>Xp7drPG%^6Rub{(QWygW7ZZh9wRW4f5GLPQ};0tQZD`v0TBk-~A z&vq}I06b-I>(rkeraPuA`GG?x_=>!hBLS=wnD-N#{2_}_i>CCG-O z{EHK_aP^xja*d*=$0LNap-D(&^2JnAMQL11klY?g{VR5gmnk8nUT&!V#28v^mCMUr??@N`!fYkt>4P`&^EJ1Q&(w3P?Ji~MOil~0iUyS zRa*DkwtMwih_WSIjAsnf6j+P<V}eoU&fE$&e`7}F^AP-WlAK#Tb;Wq8!HeBQ+V(mXv0(W1x2{3P8Tnz z(x6IYMU}MZ?gsW-*KwRB?^mw&1r)jN3(WNaXI007B{~$QjJNEzKBD68Etiz8$&%Ww z;Y`gutZtT>5nQs^0_;VITP#yKN@*+yuY^$&fiU3ZySsb>+Y9l8n|=+gET3^*ab_N4 zJts~fUYrt9)&4^v*Z6b(TCbu`IR0{p z+i>a(+Ip^!)Ky5RK^31(A%5{Yln>7_Cl@NK(K;?$lC?1U|f4sxwA@921%1Mrhw81U-NO; zp{8p}w3@rMBeH0}CgYCcHB{q@U~L?bQ`OJuag&khZi>O+|_YYRCI$n# zT!>mT2prW+0~6ZbUNg11Whu~F0)0o5Fsvy|`w<099X&OZ(V~*Pz8*67X^I5((Xu!s zo5JYBx_c;6qkXe6k**ZoW~1vx;ZOAqvvw-m#zg*-qFviw!?Y#qX^D|MISq5C1T{pK z>9y^4$tuSUPVXy*6RbDd^?A6g_1ST_T(Q9!5Ux8PN#WY?^4`m^Y&TxG#R(0Nqk)>T zgJ7YB=KA`fnkrjY>n98S_a3~PYgHf7g)inMiuBvJN9CAgPzX*vI%aoN>C>E5R-4-K zK^iacwt-D#Qg~`P`YHq`H;U8*K0Hw*;)9OV&-M?W%X7$vZRY`WVglDRZB8)Sr&8Gu zPs%|8>IEVfzgMX8%;TBzncS;Bo4v~)N9Y;>K#w5&=~!~=X%JJjc_ww=>CV*f>E6_o zBApIJo{#FQA3#Gt#pbKy-N50)i z;KSeF9mKa;0r1F3g%9yMA5Dt}Q*4u;{JKHZR8UaZw58Jt!K6>vuYN3%S^vnz7e(;? zQ=jc|j9o&h3{DE4TOSwSa$k(d_*Whk1C_S9k2X=7JPJ~wM8a8|vK`5v=ALxZ$sz~U zQHTeB@XWb%-AeLdYwjUOUBSKresiRPuWMjqhCrKMR*gmW1Q#n;HNY|J{hBJEwE&X&u|XFnWI%p+0Zq0o2e~5;)Z$< znNpNtx!6%ol8hHZ(pz@+|cALCy`*GbcI)Rk2 zssMH}PJT5|#Z{4V|GV2gH}nb@MN@?^s#5AXbCCc=t}w?~1@ZyM?0u{AU8C~)GM89p zu)7Wmt8a+Oei8RlpRTdpv$sz<7zWYz-kuO`{C3x^1|3=iM#P<>@KuQjo^NJOzUo_2 zrQ;x?r}ED#7w@1P)$j<|vRVI~my>ESZvOn4Hmdg^r&f{yu+_oaQAo?B%sHmoRR6rm zOpvn!L~VC<5-8{e#Xa&X+sV$sb2^D?pBR4Lb8T=({!Ct4_sF(@ooH37G94mp6M+h? z&&w)G?mDQ&n8Pe}aNgW?G56cXQLA0IVb(t%>#*EI=*Xg%(6Ex2P>+kyet+f5|I0hG z)%ZA9H{W)`jJ>EPkZ~7?X~<(*c?+iyjiBELT|WvF07 z(f>>+{-h03x#Bi7`o@lFio(_!Z~InLQPIl8_~4o?p!HP;%aP%gw^lB0qotQ zr`Uf#STM&jVBVm8xbEhwayNaG^5G}IM6l+NvZB&60R#avQbaNw1XHYzOSUe+_wLzs zBYqn*Jh3P>ODdKbBIkQ%aTEhw@jp0vzX#1jCqH%iy7d;xbG6=1MdJAC?m}t#cT@ro ziFeFwuh)|tWQh$9+Mm0jl-VZJ#cdP$I*o^iqS|pCA*+!hK;osj`d$(kG-RU|0%!N; zv30L+IKq5(ow}v7zu{x}9*SOU)0aGHYAhq5O`SB2R^Ubl)^cJVmU*F1Hufef3Kfsd z18wf_&Z`(DZ)iGWaiI3|?xCj(ZzoYApYqo)?btFNT(lU`n9D;lY)WtBGKsR)?%SC9 zajE?@dW>9t23dG}3lTeIRk{kN`5c$Ip>1;vy&U)%fG3qze#@~g{kT5RzISvJ-yA$W z$i37KPMFrx(&o!SA!P$&xp*B&KcRm2p-8!OLQ^j>SBA2UgL~1#ROt=kohySHHrv$k zz(?#vfuS{lSOJPO^V%y#er&D%N?_Vz0rQAVVPWnF`cbe}PWPHR4SPH`SI zDWMHG*hY{Z#P@{DNzeJ%DVq}^?X+kz-N@~Jr_o;nanI{e)mm6-gkWOWUzjI8j0PAg=*%3Uuxgq5~+D zE(Z}vd&<2t$NMJqrMW66l{D%t@_5=0GH7Ya)ZTrwUFg*{cq!rD=Q@#lQ6__-Zy<0b z&2?Reb}cZN27#VKw?Sj+rZ*?~D~{oo)%miVTXh9ULH%OeXs-`fG{^CZC$uRy%)oSP z!dZiWsrVR~L$9>d&3ete!Gl*bJOXTu@9RjEp4!`6yEdt3m-tLxOrnaNyNGA&`+*4l z@0#hiCoxXwbgwkXd2`G`nl}si`y+GP(gt#hzgNan@T+rexU6%teY#@7C`G;?+u$G4 zK1Wvsp{&^3E86|2BCchd-IN1@Hv{b7==+4PeyhnCykaY3nXfjpPL|-yne^{xDk+E% zKK&JF$o-b3F;%WUK#))<$rK%Dl3d?4&+7egkeh{NcA4GGce#W+kb z)IS|V8_ig8A-}_A*T^=lFz>WM44f&JEpH@~=*q-is&ZQmr#1np$t=(UR@8Yl(szY9 zE<+)`$x4nv2sis|v0YH61y?zdglVe8ShuJ#VyZtX!Sj6Yog(h0{ic+nvjyAu{wS-X zNW@XEz@4J_St^3s@9w!<-J(e>+k8G=o$NAgoN%`QUrOmx@OpYxhEXgO_qkhu=OEU6 z62{%e$q@IPLyNB_Sp!d>xmpiMw?Z@af*CD??3)Vn-OE#5GM_{Ly8T&@YCV(282@!b zW&}cP*A^Hf8djnlI?w-muR*G*@u>bzQAuPO1c*fW_!Jx!IjaC2uq*~?wcWza z`ur@}%G|iu4fHdWLU?>e`QMPbwVquQeg6kAc+CYoRfZBR^Z-C8d)dm+J}vmp9sq&= zMN7Pep>L8{w#5G-U$FQA(8B)@-;qmuj`-#;VtqyQ*9NcL;5e@w4)0DHwhOq41L zOOIeIiqHepM1-6fhAcY7e*UkM9wC1R6bG}|+K~Rc@EbT&iX{-DeCRefe|6v)b?VCp zoE4mZBLI>*ZkINDD2Du@HvhXo05sI_@*{vg`>!hGrK*2K5C2^_22N!9(8!`5z(emv z=zwSFaD5&?F7HKP*nZFDmw=AsKbX@S;QU|KJm5cFN35Q3z*NjW5)Z1%Rf*Ss8q_0V+&!TSc_g^)r|G8Ml(n_= zxGnb4^PGYN5e2*@;u?U}01R3xY8G3^Bpjote$jLRn9@P&aFG7Pd{k}&#vR`6VGrNK zU|o{|P+Y0QS+?)GMhVPMUiTpX`9&QI80Cn9h1orH6#A?#17mU-?_0;ZHIZ^(04ceZ?l^cP#^CFmW63GAS8Qe$p z$j?*fFcRv?YVAz=&eweN@mQjzm&YvV#Bs!_b`s?CW#Ei$Ms$^=yMr7Ng zj>x&-(M)2G49NeX^T>nV8(C45s~$>tb&44`S?LwtGQW9+uJ~X&EZNq(&y?k2N8dd( z?-;~7HTu}QAOCfoBd_5BlA1x6IkoZxJepJN6(Tvp-?2bW0Q@qLJS1OeV~M@Jyu{-r zY5@;30r?05!nIr8?zLUoP;BD85Z;hrnil773bAm-&~C?v>%C2<@bczP1@Fp!KHC?M zHTMV^c7Lf4?Yubusin9dkvC38#9DkxmLv;yr`r;wDVdOI^%b}0H(MiO2MC_j#NV=l zpYEDVwh#bX&aMC<p6~En{xn!0p~~s|MwcaC`r;qP~|2sBIvRF{KlWjnsA{iy_& z1AL;FdnFqxQ?kezN+AP*XVGu}-sMP~e^^|RWt5TrF`jHdNV&YNHKD-vH8xDB7BJ(N zRJfgMsguy0*0&MgG1O6>j+f@%ae(w;Zs7 zGK;P}yco_O4w&X!* z*D)Td{+bnNN~pgx??Zj^(*Qv2bsDStf0VpXplz~Gll%Yk)^|V~ChkAgiT`~@BopOR zTJPK82>*~)o2Q&B57Qmap98qj`j;c>f8h~Fu!w>wB{s~Nh^XOOjR6A1Q=UA6d*g%o%UtY?EidpgRCQpE(VY-~kGsV>j@&Bj%fW?|W*ULWoV zqj$n${asRC?x+4%fWGo(6pOXm*Z{8ZPl}zPH&}yJMUu|=65)i*3Uhp~C|-)gq z0VP64g~(Jvg{fi#yB*Dil`BT~FjpymlmZbB&xg-L^ z7v`G!PLfI^c{j}YlJeFK^D?<40Jg^B`gAMw6wrw>H3YAsAEbg}ie@`$0p=nQDZ_%# z@Q`S%TzhyNR)bkC?(HWh4LTJTiCRX!6OGVW&>wW-N@pm5O!PYmaM_t0d|F*3;l0}& zQWKl=gDG!Lkx_BzY;$D(%{&h`Akljgj7Oho=<_rCF6=~c#vTT}ixj+InX|2kunPsS zAMwi3ACp+Qbvgkl;XB%+*kSsT+WC2({-QGy#Vy#r}~?Em#w!*Z6QyN1iD1_fum~9)gKWB z{K&Wbo_)`TrA<5ju%F7Qeiay55C;K&h-Bd^$EWKi=xfd0UY!gAIZdnTMVtQ06*`Y4 zK$$UVs{JLHMsu9*`;q25qFO(w=SE^QfP$fST9eGbmloUq6^OwBnBKcxZ?)sqynfq6 zwn9>8-LIhpR#O4yDMUhn5>sw!d4mWQ`#SUZGsV#7FkY`nHG)2Gter`h15x#Gr zo8U5OqQ_Zf(%kxB-^~5pAOb1st`kfV@a8w9K_Ky+>>@N32!KU3RO46JsvAbolXZc# zKV>wruwSG5P-t%Hdqv~V_d-C{$LacXZ%0yL3I{)q%MU0Dmd#kXyREw+#Mvy4E4aQ% z<*@$xoX^GKdr2cp3N|JFjL} zw#)+}F`(skMAWx>dGiZiAu{*1>_DnR_ShYXh^53{iuZ>xKn4Uf^KD^K?l%cSWOWpO zV1m-rcX5Ot9Clztyzq|MW4yoKv1QJ6k(V>#p2dw6PPN0)2_j~FE3x2u>Tu`ydXVqa zJGmV@{_j7vLUT_xhUIo|+%>BKOL1OL13Rp8VCCzB#}Or0Mn~X~%2&9n#2RTL%=M!y1EEgrU=!p=R@rCvl8=RJR#!92gAIN!-l(#%H@T^w^wz0r2cI+{;k2!Z8=y8)| zUv1q!Jow@kfULYkssg#C@2a~VEDBur)=?q>E!3L3nh@gA1t4!c* zNyGVU&y7m>gz%D2|IjUQ9F`Vp<8dE-P9+PV+ie3JTGt$ne${Z__@U>FsiMfJ;kUdJ znRALE^!rP0E(*Vf{qA@GqU&m5JLPF*u-sa?RV$my77zWt$<`_ous{P4-0hPExRLAR1SM}UJpOa1WC z07*IA$9;fr9CC|6+gY*)U~Vgp+sRbxmN(q}x;F0iS8MU^4YYHJlgS@8wjC^GAv>p= ziydeXB_LPRUoQm>p3gEv!Gx`#-w-w_pLPY%AmoR?uaL{~-@BOib^krM;J6x*tyRL; zpoIDx(C#WWH89wPfJ#p4j{!j)i)-BzMEjW+C|J+o_AKJ(qCkrb_50aOVs%S}uCAeE z=we1ch-h8y0j;!#g&4c|YtcQX(_yuY4!44XfYX{US+BGY%B)ms6VD;|XOgc_=}}3b zz!ZDY761-asU7DO|oQ~^S(pQI0chW-_BGyp9F*QE00-tr`c z3;VgQ1YyMJFSm{LIdfMIi-Lw>$Zeyi^B`~-y&mw&ZlZoYvLqUiNs~ie72X`rvwy0| z#i(6Xg?6Wcd!zdj8y26WvjWjV#5buI$iOa-`&?Fs^y=0*idT4h(-ddJVo(o)rFE>% zZOA{@_xnQ}xS&{cBhKS-YGKtG#L;AhOnz|!^2sqk{)iJ}dw#j&Wdrk+t~aB5VkeiZ zF{7S?ufZdNi6qb)3DlXNVv{CJvaf@Kx?t3#_dvMQuM`Nd_9!0JbcV~*Se31@07vS2 zYfpj-Ci;qKH!=CS-?o{733JS6t3W#!inCK0!zBD}@;F+$+=sQMkusn|^>8_)h2Oo) zAZ(_cibK)~$xQ-x-8hHLr?|$7wiznKc6zaSg?-vEF6D|gzCOw z6$3pR+hR~o81$Og;y4Od+wVrDm*SJB_4sRctJR1m`~rZbGXQ*cKzjXb%1~|q>uK;* z*^_aHx(Uri+M>GM-9#yr%v8Z`3#{O*wOf6neI8pmG)+RhTR<`3e?E-mk!7|qKmdqx zFMHS2^)6`8Vn9`0Owkyi7PN_2vg`snzX(liq=3d5@8TshVzQ;)=xqgM2PHqTw#i0v zS_NN=)lbTe*z%&hJbFr!juRI~jd+QnpSxB?DJTkv8(ikL*DRUNgqG;-4du+6f?h%BAm* z0K(-H|9l+nSpQ^wvGhs=K*O@yE~8trF6%9%q1C#4y4sw5EYFU8Y!Soj!eIh$3jtz5 zY%^)pygwq2$^Hk4k;wbV2`^b=6P_uQ$YTwDx(L*E-NxiWvX+eJJ;g?Vhw&%&76s1b z$gw|DD4Qeb&&l0jpi|=n)Lc3@?a?MGI0mi+nz$5Cu?;F%(ihL_L`{?_99LT+5w{u) zNCD#IfTF(fY?~=c&Q5X5A^_ID3RQy?5kAwlo2`^dzBsrQsA?0^Od_)6ETH(Hd!{=< zoSw>K{k9{tb@7Zx;@&#hCto*2s}-!;QrBr!LW2cOe!y`puSEV&61JPM#E@sa|E zMa+%xHz_{8psKdA7aCH5TP zM-=?JHf6Kc-6S|Gvi?*S&6g|A%!@uPO^V$0LQ!uQKFc#T-nWHV?A9zL*4&?c_PmUe z9?$^a70iH2nxZ>npA7Uvd=hHk+3D;QSJ;!E1$Q(Y)YChqGYy*Wy%X-AWl{u(Polk) z2B>&6H0@XJoqdCppR{GV$&vv_LgkP09N)XRFNtYH;b(vb`IlFZ{ohfQDhFrE8-xQ) zIw*zAi>s4c00)yGY~PZ?dz_m&z}I5btp{?123%RJdVD_27)scp zs+!KP%R1L})jG*pEM*SgGf}KlcVCB3TH>wl>g#vYZoPISQWthCQs3EU>KjuMHm@u_ zUGzTK^YrvQHs1}rSmFji?1^Zp9o&Uo7Mh7yy9Fhmj#qeR$CP!+<21>nIYD za8*C>>4YTvde$P9VV7`G={9ZWUm^!F;HR4E+V|~!!h`nuOGy^=)ur5(KZrntQwxM{ zO%=m@V|t`kz)ofGn%LI3!;Nz(@@BE|9_kHfX&s=4e?6SD*s*K$E$hJ01QhUra$z3< z(g~m>M4(G-72|)bv?7g04kY2b);n>PUQh4Ynllz&Y>jox4eLZG#d|1pij8coAPf)I zREy3{uVGXXp4`{C3$wn{bA8d8qQD)css5~N&hcZ|$B*M*=~U_*KYFFpRMV(3@ouD6 zR=$zk7FMQ5=B|Gco(ty$hoL})iqvP#>r5{pJNNOs;D;EF%GZO!ao60P(I!iUmOmWa zDonu)3P5Ilc=e)e?5g4E+j_bhKAejkntLGR`fJt3NEiH`-O zZ<+v_!OHJmGV?0)(&LI@K_do6#;k7sZkCIj^7GlXL%3^Ak4C!k7$bapPcZK}R@kvw5G5>aT-WdwEc`a)RdE zRjhY^+yd=+DcK0Up498gn^M9%XuMNAg5DgQ1%p140IsV>+8QWH#LxsB;C6+L@a|2pe#w;MfQJj zy(sdRz$6pWzrE&eVKnrB)~x`aOTjH-oV{L+*9zNI(p1p0pU3#Fk(h>nj76wr++DS^ zb}y;R*42@!r5UmWZ^(JH%)Fo>(5O^gC~&)*-MX;$W}Wrd`V(=DGv&!XR@@bNt%YYK zt<3R{Jv8GHO=mnbO#~EkAVX=wTFJuqhGm+cGhIssyfq@4{hAt2&xhY77tGVnvhq}G z6f*DQ_?%gm-~*t>?%4@4on!wh^zi=3pG>&^#b|}i-WA${dW#xtX1dMb zW{*?8@W%(wxwnZydB4YlKSj zghq`PSl&q*6`5fAK{svuYQu&fYH0?u&DMj7`;}N(WXopSj=kce&|FM9%aD$AjJ!fe&jq~U4(jtT*gVdFU*K_&i zNiD4Klk3Dc2!HQPH-(4S>~^bN$07gosw8G|W4q=urN4h6@KKW+yV{0;m|%rqB?PxZ zsjxGKq7+j;d;TftJsj{cdU(yp92&4)hdK@hs^5 z+UtK!K$h15`j{nsx_#(3?((lIh*5y9u(F(f z_RruX@F{588#cjr4&6)>sDCbgVOG9KNWQJA0ize8YTv|5ac|6<-&M|nwLuhr3}0IT zpNZR>vLKYE__x2Y5_kb`x9*mRPW`Xjl&7*R?lUe~Y%pCEdpOrrLOxvmt+`0CNCEsL zv@6Bb^LCqgs#|wR?9UJB1@PAl8+%uph=u8d82>izkC0RcH}HEu=0Kz*cJD0`xL%j%e|w4{?ic(@E4%zplYZnthAE@xTmIPa&efm;yxniT#{5;LG!`}P&k4*$=n z;|qe*Dqp1r?G0K{gVt^~!iAHYS-IXYo24{0c_0+s>?cMJ^LK*QGEryC-Lwp;6V_X8}=%;hcj0=(C^Ygfs*|4BrzW3}+sWb{-7v_r;W zCrFyKzfn9v%&o1#B+Of`+;mnUm0tB|X8`G*R#UTNf@6UJ*2>KQ5xvJ|W5QPX%LGy> zc|>luoip@zCBePl&5If(2nRnkYeyT>$$Q4YhD2*28V3qa#mGAU&tj+4D}FRLy|ReC zy;ikt$v(Ft4|+R`p*dBQV=PHVAyRuo^&BBGG|9v zDKE}T6w$}vUPH0{L8kAi$H8B9XTr;WLs!iKuOUB?xt?6M?PPW@-P%W83IAsRu#or{ zU3aK{gbYFR^}^q4ddeJg5-|EAd&AQs3A{p*`V zfH3O)``E4=wugK1uu4^NsUD^s!=p#^;Y#Lzha3PAAUDpUmVKC`I)JmmPwRWQS6!r4 zR^d-qzzQlyKqzDoiv2rVAHnAVr9fU&{Ph9IX27h}@u9>0=M#X~fu2)E?BC?-0bF Date: Wed, 21 Feb 2024 00:10:50 -0400 Subject: [PATCH 2/8] refactor --- docs/icicle/multi-gpu.md | 6 + docs/icicle/rust-bindings/multi-gpu.md | 150 +++++++++++++++++++++++++ 2 files changed, 156 insertions(+) diff --git a/docs/icicle/multi-gpu.md b/docs/icicle/multi-gpu.md index e9fc4ae..13a0802 100644 --- a/docs/icicle/multi-gpu.md +++ b/docs/icicle/multi-gpu.md @@ -43,6 +43,12 @@ To dive deeper and learn about the API checkout the docs for our different ICICL - C++ Multi GPU APIs +## Best practices + +- Never hardcode device IDs, if you want your software to take advantage of all GPUs on a machine use methods such as `get_device_count` to support arbitrary number of GPUs. + +- Launch one thread per GPU, to avoid nasty errors and hard to read code we suggest that for every GPU task you wish to launch you create a dedicated thread. This will make your code way more manageable, easy to read and performant. + ## ZKContainer support for multi GPUs Multi GPU support should work with ZK-Containers by simple defining which devices the docker container should interact with: diff --git a/docs/icicle/rust-bindings/multi-gpu.md b/docs/icicle/rust-bindings/multi-gpu.md index b8c72f6..9b51163 100644 --- a/docs/icicle/rust-bindings/multi-gpu.md +++ b/docs/icicle/rust-bindings/multi-gpu.md @@ -1,2 +1,152 @@ # Multi GPU APIs +To learn more about the theory of Multi GPU programming refer to [this part](../multi-gpu.md) of documentation. + +## Device management API + +To stream line device management we offer as part of `icicle-cuda-runtime` package methods for dealing with devices. + +#### [`set_device`](https://github.com/vhnatyk/icicle/blob/275eaa99040ab06b088154d64cfa50b25fbad2df/wrappers/rust/icicle-cuda-runtime/src/device.rs#L6) + +Sets the current CUDA device by its ID, when calling `set_device` it will set the current thread to a CUDA device. + +**Parameters:** + +- `device_id: usize`: The ID of the device to set as the current device. Device IDs start from 0. + +**Returns:** + +- `CudaResult<()>`: An empty result indicating success if the device is set successfully. In case of failure, returns a `CudaError`. + +**Errors:** + +- Returns a `CudaError` if the specified device ID is invalid or if a CUDA-related error occurs during the operation. + +**Example:** + +```rust +let device_id = 0; // Device ID to set +match set_device(device_id) { + Ok(()) => println!("Device set successfully."), + Err(e) => eprintln!("Failed to set device: {:?}", e), +} +``` + +#### [`get_device_count`](https://github.com/vhnatyk/icicle/blob/275eaa99040ab06b088154d64cfa50b25fbad2df/wrappers/rust/icicle-cuda-runtime/src/device.rs#L10) + +Retrieves the number of CUDA devices available on the machine. + +**Returns:** + +- `CudaResult`: The number of available CUDA devices. On success, contains the count of CUDA devices. On failure, returns a `CudaError`. + +**Errors:** + +- Returns a `CudaError` if a CUDA-related error occurs during the retrieval of the device count. + +**Example:** + +```rust +match get_device_count() { + Ok(count) => println!("Number of devices available: {}", count), + Err(e) => eprintln!("Failed to get device count: {:?}", e), +} +``` + +#### [`get_device`](https://github.com/vhnatyk/icicle/blob/275eaa99040ab06b088154d64cfa50b25fbad2df/wrappers/rust/icicle-cuda-runtime/src/device.rs#L15) + +Retrieves the ID of the current CUDA device. + +**Returns:** + +- `CudaResult`: The ID of the current CUDA device. On success, contains the device ID. On failure, returns a `CudaError`. + +**Errors:** + +- Returns a `CudaError` if a CUDA-related error occurs during the retrieval of the current device ID. + +**Example:** + +```rust +match get_device() { + Ok(device_id) => println!("Current device ID: {}", device_id), + Err(e) => eprintln!("Failed to get current device: {:?}", e), +} +``` + +## Device context API + +The `DeviceContext` is embedded into `NTTConfig`, `MSMConfig` and `PoseidonConfig`, meaning you can simple pass a `device_id` to your existing config an the same computation will be triggered on a different device automatically. + +#### [`DeviceContext`](https://github.com/vhnatyk/icicle/blob/eef6876b037a6b0797464e7cdcf9c1ecfcf41808/wrappers/rust/icicle-cuda-runtime/src/device_context.rs#L11) + +Represents the configuration a CUDA device, encapsulating the device's stream, ID, and memory pool. The default device is always `0`, unless configured otherwise. + +```rust +pub struct DeviceContext<'a> { + pub stream: &'a CudaStream, + pub device_id: usize, + pub mempool: CudaMemPool, +} +``` + +##### Fields + +- **`stream: &'a CudaStream`** + + A reference to a `CudaStream`. This stream is used for executing CUDA operations. By default, it points to a null stream CUDA's default execution stream. + +- **`device_id: usize`** + + The index of the GPU currently in use. The default value is `0`, indicating the first GPU in the system. + +- **`mempool: CudaMemPool`** + + Represents the memory pool used for CUDA memory allocations. The default is set to a null pointer, which signifies the use of the default CUDA memory pool. + +##### Implementation Notes + +- The `DeviceContext` structure is cloneable and can be debugged, facilitating easier logging and duplication of contexts when needed. + + +#### [`DeviceContext::default_for_device(device_id: usize) -> DeviceContext<'static>`](https://github.com/vhnatyk/icicle/blob/eef6876b037a6b0797464e7cdcf9c1ecfcf41808/wrappers/rust/icicle-cuda-runtime/src/device_context.rs#L30C12-L30C30) + +Provides a default `DeviceContext` with system-wide defaults, ideal for straightforward setups. + +#### Returns + +A `DeviceContext` instance configured with: +- The default stream (`null_mut()`). +- The default device ID (`0`). +- The default memory pool (`null_mut()`). + +#### Parameters + +- **`device_id: usize`**: The ID of the device for which to create the context. + +#### Returns + +A `DeviceContext` instance with the provided `device_id` and default settings for the stream and memory pool. + + +#### [`check_device(device_id: i32)`](https://github.com/vhnatyk/icicle/blob/eef6876b037a6b0797464e7cdcf9c1ecfcf41808/wrappers/rust/icicle-cuda-runtime/src/device_context.rs#L42) + +Validates that the specified `device_id` matches the ID of the currently active device, ensuring operations are targeted correctly. + +#### Parameters + +- **`device_id: i32`**: The device ID to verify against the currently active device. + +#### Behavior + +- **Panics** if the `device_id` does not match the active device's ID, preventing cross-device operation errors. + +#### Example + +```rust +let device_id: i32 = 0; // Example device ID +check_device(device_id); +// Ensures that the current context is correctly set for the specified device ID. +``` + + From 090151be0c3436b8eb0a7f534b6b517483836b3e Mon Sep 17 00:00:00 2001 From: ImmanuelSegol <3ditds@gmail.com> Date: Wed, 21 Feb 2024 00:50:23 -0400 Subject: [PATCH 3/8] Refactor --- docs/icicle/rust-bindings/multi-gpu.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/icicle/rust-bindings/multi-gpu.md b/docs/icicle/rust-bindings/multi-gpu.md index 9b51163..a62f3de 100644 --- a/docs/icicle/rust-bindings/multi-gpu.md +++ b/docs/icicle/rust-bindings/multi-gpu.md @@ -150,3 +150,8 @@ check_device(device_id); ``` +## A Multi GPU example + +todo + + From b228cc64f2d5498ec6637191242ed082417f4cb3 Mon Sep 17 00:00:00 2001 From: ImmanuelSegol <3ditds@gmail.com> Date: Wed, 21 Feb 2024 07:21:27 -0500 Subject: [PATCH 4/8] Update docs/icicle/multi-gpu.md Co-authored-by: Jeremy Felder --- docs/icicle/multi-gpu.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/icicle/multi-gpu.md b/docs/icicle/multi-gpu.md index 13a0802..a25c4a0 100644 --- a/docs/icicle/multi-gpu.md +++ b/docs/icicle/multi-gpu.md @@ -51,7 +51,7 @@ To dive deeper and learn about the API checkout the docs for our different ICICL ## ZKContainer support for multi GPUs -Multi GPU support should work with ZK-Containers by simple defining which devices the docker container should interact with: +Multi GPU support should work with ZK-Containers by simply defining which devices the docker container should interact with: ```sh docker run -it --gpus '"device=0,2"' zk-container-image From b56ee6a00dc9d23347d6db808fed4a4c662b8bc2 Mon Sep 17 00:00:00 2001 From: ImmanuelSegol <3ditds@gmail.com> Date: Wed, 21 Feb 2024 07:21:58 -0500 Subject: [PATCH 5/8] Apply suggestions from code review Co-authored-by: Jeremy Felder --- docs/icicle/multi-gpu.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/icicle/multi-gpu.md b/docs/icicle/multi-gpu.md index a25c4a0..5adf265 100644 --- a/docs/icicle/multi-gpu.md +++ b/docs/icicle/multi-gpu.md @@ -13,7 +13,7 @@ Multi GPU is an approach at developing software to run on multiple GPU devices. ## Approaches to Multi GPU programming -There are many [different strategies](https://github.com/NVIDIA/multi-gpu-programming-models) available for implementing multi GPU, whoever they really split into two catagories. +There are many [different strategies](https://github.com/NVIDIA/multi-gpu-programming-models) available for implementing multi GPU, however, it can be split into two categories. ### GPU Server approach @@ -23,19 +23,19 @@ This approach usually involves a single or multiple CPUs opening threads to read This approach wont let us tackle larger computation sizes but it will allow us to compute multiple computations which we wouldn't be able to load onto a single GPU. -For example lets say that you had to compute two MSM of size 2^20 on a 16GB VRAM GPU you would normally have preform them asynchronously. How ever if you double the number of GPUs in your system you can now run them in parallel. +For example lets say that you had to compute two MSMs of size 2^20 on a 16GB VRAM GPU you would normally have to perform them asynchronously. However, if you double the number of GPUs in your system you can now run them in parallel. ### Inter GPU approach -This approach involves a more sphisticated approch to multi GPU computation. Using technologies such as [GPUDirect, NCCL, NVSHMEM](https://www.nvidia.com/en-us/on-demand/session/gtcspring21-cwes1084/) and NVLink its possible to combine multiple GPUs and split a computation amongst the different devices. +This approach involves a more sophisticated approach to multi GPU computation. Using technologies such as [GPUDirect, NCCL, NVSHMEM](https://www.nvidia.com/en-us/on-demand/session/gtcspring21-cwes1084/) and NVLink its possible to combine multiple GPUs and split a computation among different devices. -This approach requires redesigning the algorithm at the software level to be compatible with splitting amongst devices. In some cases to lower latency to a minimum special inter GPU connections would be installed on a server to allow GPU direct communication with each other. +This approach requires redesigning the algorithm at the software level to be compatible with splitting amongst devices. In some cases, to lower latency to a minimum, special inter GPU connections would be installed on a server to allow direct communication between multiple GPUs. # Writing ICICLE Code for Multi GPUs -The approach we have taken for the moment is a GPU Server approach, we assume you have a machine with multiple GPUs and you wish to run some computation on each GPU. +The approach we have taken for the moment is a GPU Server approach; we assume you have a machine with multiple GPUs and you wish to run some computation on each GPU. To dive deeper and learn about the API checkout the docs for our different ICICLE API From 963cb369379cd31a51db008a61a32543e04b17a1 Mon Sep 17 00:00:00 2001 From: ImmanuelSegol <3ditds@gmail.com> Date: Wed, 21 Feb 2024 10:57:57 -0400 Subject: [PATCH 6/8] Refactor --- docs/icicle/rust-bindings/multi-gpu.md | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/docs/icicle/rust-bindings/multi-gpu.md b/docs/icicle/rust-bindings/multi-gpu.md index a62f3de..20567e0 100644 --- a/docs/icicle/rust-bindings/multi-gpu.md +++ b/docs/icicle/rust-bindings/multi-gpu.md @@ -2,6 +2,11 @@ To learn more about the theory of Multi GPU programming refer to [this part](../multi-gpu.md) of documentation. + +## A Multi GPU example + + + ## Device management API To stream line device management we offer as part of `icicle-cuda-runtime` package methods for dealing with devices. @@ -148,10 +153,3 @@ let device_id: i32 = 0; // Example device ID check_device(device_id); // Ensures that the current context is correctly set for the specified device ID. ``` - - -## A Multi GPU example - -todo - - From 7c543f3f0bc4c79100e365e67c069fdaa42d3a20 Mon Sep 17 00:00:00 2001 From: ImmanuelSegol <3ditds@gmail.com> Date: Thu, 22 Feb 2024 05:53:23 +0200 Subject: [PATCH 7/8] Update docs/icicle/rust-bindings/multi-gpu.md Co-authored-by: Jeremy Felder --- docs/icicle/rust-bindings/multi-gpu.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/icicle/rust-bindings/multi-gpu.md b/docs/icicle/rust-bindings/multi-gpu.md index 20567e0..2c6822e 100644 --- a/docs/icicle/rust-bindings/multi-gpu.md +++ b/docs/icicle/rust-bindings/multi-gpu.md @@ -9,7 +9,7 @@ To learn more about the theory of Multi GPU programming refer to [this part](../ ## Device management API -To stream line device management we offer as part of `icicle-cuda-runtime` package methods for dealing with devices. +To streamline device management we offer as part of `icicle-cuda-runtime` package methods for dealing with devices. #### [`set_device`](https://github.com/vhnatyk/icicle/blob/275eaa99040ab06b088154d64cfa50b25fbad2df/wrappers/rust/icicle-cuda-runtime/src/device.rs#L6) From 3a6ed0f88dee772c0c32889dbacc0a81b66ce271 Mon Sep 17 00:00:00 2001 From: ImmanuelSegol <3ditds@gmail.com> Date: Thu, 22 Feb 2024 00:54:26 -0400 Subject: [PATCH 8/8] refactor --- docs/icicle/multi-gpu.md | 4 +- docs/icicle/rust-bindings/multi-gpu.md | 52 ++++++++++++++++++++++++-- 2 files changed, 50 insertions(+), 6 deletions(-) diff --git a/docs/icicle/multi-gpu.md b/docs/icicle/multi-gpu.md index 5adf265..fd7e2fd 100644 --- a/docs/icicle/multi-gpu.md +++ b/docs/icicle/multi-gpu.md @@ -6,9 +6,9 @@ If you are looking for the Multi GPU API documentation refer here for [Rust](./r ::: -One of the givens with ZK compute can be the large input sizes, its not rare to se today circuits exceeding 2^25 MSM, such input sizes push even high end GPUs to their limits. To scale and run such large circuits we will need to combine multiple GPUs. +One common challenge with Zero-Knowledge computation is often managing the large input sizes. It's not uncommon to encounter circuits surpassing 2^25 constraints, such large inputs push the capabilities of even advanced GPUs to their limits. To effectively scale and process such large circuits, leveraging multiple GPUs in tandem becomes a necessity. -Multi GPU is an approach at developing software to run on multiple GPU devices. There are many ways to approach writing such software. This documentation will cover how you can develop with ICICLE for multiple GPUs. +Multi-GPU programming involves developing software to operate across multiple GPU devices. Lets first explore different approaches to Multi-GPU programming then we will cover how ICICLE allows you to easily develop youR ZK computations to run across many GPUs. ## Approaches to Multi GPU programming diff --git a/docs/icicle/rust-bindings/multi-gpu.md b/docs/icicle/rust-bindings/multi-gpu.md index 20567e0..b57f6b1 100644 --- a/docs/icicle/rust-bindings/multi-gpu.md +++ b/docs/icicle/rust-bindings/multi-gpu.md @@ -2,10 +2,7 @@ To learn more about the theory of Multi GPU programming refer to [this part](../multi-gpu.md) of documentation. - -## A Multi GPU example - - +Here we will cover the core multi GPU apis and a [example](#a-multi-gpu-example) ## Device management API @@ -153,3 +150,50 @@ let device_id: i32 = 0; // Example device ID check_device(device_id); // Ensures that the current context is correctly set for the specified device ID. ``` + + +## A Multi GPU example + +In this example we will display how you can + +1. Fetch the number of devices installed on a machine +2. For every GPU launch a thread and set a active device per thread. +3. Execute a MSM on each GPU + + + +```rust + +... + +let device_count = get_device_count().unwrap(); + +(0..device_count) + .into_par_iter() + .for_each(move |device_id| { + set_device(device_id).unwrap(); + + // you can allocate points and scalars_d here + + let mut cfg = MSMConfig::default_for_device(device_id); + cfg.ctx.stream = &stream; + cfg.is_async = true; + cfg.are_scalars_montgomery_form = true; + msm(&scalars_d, &HostOrDeviceSlice::on_host(points), &cfg, &mut msm_results).unwrap(); + + // collect and process results + }) + +... +``` + + +We use `get_device_count` to fetch the number of connected devices, device IDs will be `0...device_count-1` + +[`into_par_iter`](https://docs.rs/rayon/latest/rayon/iter/trait.IntoParallelIterator.html#tymethod.into_par_iter) is a parallel iterator, you should expect it to launch a thread for every iteration. + +We then call `set_device(device_id).unwrap();` it should set the context of that thread to the selected `device_id`. + +Any data you now allocate from the context of this thread will be linked to the `device_id`. We create our `MSMConfig` with the selected device ID `let mut cfg = MSMConfig::default_for_device(device_id);`, behind the scene this will create for us a `DeviceContext` configured for that specific GPU. + +We finally call our `msm` method.