From 27acf6f2d51516a10e70cfceb37990f490ff74e6 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Wed, 11 Dec 2024 21:30:27 +0900 Subject: [PATCH 01/23] build: Use signed char in RISC-V 64bit (#9524) * build: riscv64: Add a fundamental architecture cmake file for riscv64 Signed-off-by: Hiroshi Hatake * build: Add riscv64 architecture target for internal tests Signed-off-by: Hiroshi Hatake * build: Restrict to use gcc only This is because this workflow is not injected and used clang packages Signed-off-by: Hiroshi Hatake * build: Use gcc-9 and g++-9 on qemu workflow Signed-off-by: Hiroshi Hatake * build: Use ubuntu-22.04 distro due to try to use gcc-12 for atomic operations on riscv64 Signed-off-by: Hiroshi Hatake * build: riscv64: Remove commented out lines Signed-off-by: Hiroshi Hatake --------- Signed-off-by: Hiroshi Hatake --- .github/workflows/unit-tests.yaml | 11 +++++------ CMakeLists.txt | 5 +++++ cmake/riscv64.cmake | 8 ++++++++ 3 files changed, 18 insertions(+), 6 deletions(-) create mode 100644 cmake/riscv64.cmake diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index 3d4a7564ce8..24d93fc61ba 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -179,6 +179,7 @@ jobs: matrix: arch: - s390x + - riscv64 steps: - name: Checkout Fluent Bit code uses: actions/checkout@v4 @@ -188,19 +189,17 @@ jobs: id: build-and-test-on-qemu with: arch: ${{ matrix.arch }} - distro: ubuntu20.04 + distro: ubuntu22.04 shell: /bin/bash dockerRunArgs: | --volume "/var/lib/dbus/machine-id:/var/lib/dbus/machine-id" --volume "/etc/machine-id:/etc/machine-id" install: | apt-get update - apt-get install -y gcc-7 g++-7 clang-6.0 libyaml-dev cmake flex bison libssl-dev libbpf-dev linux-tools-common#libsystemd-dev - ln -s /usr/bin/llvm-symbolizer-6.0 /usr/bin/llvm-symbolizer || true + apt-get install -y gcc-12 g++-12 libyaml-dev cmake flex bison libssl-dev libbpf-dev linux-tools-common - update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 90 - update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 90 - update-alternatives --install /usr/bin/clang clang /usr/bin/clang-6.0 90 + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 90 + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 90 run: | cd build export nparallel=$(( $(getconf _NPROCESSORS_ONLN) > 8 ? 8 : $(getconf _NPROCESSORS_ONLN) )) diff --git a/CMakeLists.txt b/CMakeLists.txt index 08ecfb7ad94..0e6340bbfc9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -321,6 +321,11 @@ if (FLB_SYSTEM_LINUX) include(cmake/s390x.cmake) endif () +# Build for Linux - riscv64 arch +if (FLB_SYSTEM_LINUX) + include(cmake/riscv64.cmake) +endif () + # Enable signed char support on Linux AARCH64 if specified if (FLB_LINUX_ON_AARCH64) set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsigned-char") diff --git a/cmake/riscv64.cmake b/cmake/riscv64.cmake new file mode 100644 index 00000000000..c4a8fb15751 --- /dev/null +++ b/cmake/riscv64.cmake @@ -0,0 +1,8 @@ +if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(riscv64)") + message(STATUS "Forcing characters to be signed, as on x86_64.") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsigned-char") + if(FLB_LUAJIT) + message(WARNING "LuaJIT is disabled, this platform does not support built-in LuaJIT and system provided one neither.") + set(FLB_LUAJIT OFF) + endif() +endif () From 295c52ad8b32b531a58845bf75819bca0d76f26b Mon Sep 17 00:00:00 2001 From: Erik Bledsoe Date: Wed, 11 Dec 2024 10:38:13 -0500 Subject: [PATCH 02/23] update fluentbit_users.png (#9710) Added WSO2 by their request https://github.com/fluent/fluent-bit/issues/9704 Signed-off-by: Erik Bledsoe --- documentation/fluentbit_users.png | Bin 123627 -> 128823 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/documentation/fluentbit_users.png b/documentation/fluentbit_users.png index 9b3b985b76489cef886d32a0a3d35cbb341d5569..403a6f169a3392209f70aed5a3c9d4ffdc96fdc9 100644 GIT binary patch delta 15925 zcmc(`Wl$Vn^fri-;1US#!QGt@90DXHSa1*SE{!Beu;372NP-0$++BiOATYQ~u)!Ie zZGQiEYpeFlTl-;mZ`A-Z(@l5ZbM86M^PF?{eq+|dF)IeJQ1rwIoJ1%Jf5zT(5quI5Mx0 zI5{ZLDj(#a*8d%*=`WhN%lea00g(S+V?*l)-7)%#UTxU0SV^VOuU9kKt*voV6_zqj zM}JDd&V|$_>J?QcvY26`^Y1J}YzW0fybhOsCl4g)*E!67hNvdEh-dH5)g~3aE2s#% zyT+EHCj6f<5E9z6osj}j+0B#*HY2$2!8Ps+68qXyd>O+a1(=i<6Sge15iOtn8Xe$F zLT-n1OdKoZeZ-*^P9B=CebJ;Z;6x+!g=B~aPUmW9NQGE*$9Mct>tQm9fgYvsKR17$ zlAL!TsrGQGTfzAXz7s}BvEjLYj@x&#@9+DW7y64pv4~Cr=_avy#z!(crLw5;;;LN+ zzq%MO`T!*3TEVAk8~orjZA-49tLLiE@6F}&`U4~an{;QL>P^09Jd2%~(JN5T92`hu zLFUCl&HXAit*}94sc&i>BK2?Ik%z(mV&8g)R!zS>NXnqz$?T!+j){rcw0f4+pa7@V z4p#6sw^v?CX|l-OW-x`r`$r)s12}$Xfzy%zkY_4SS8&n^p{H1>=e|4ru=xW)pJ3}nVO|V&J^5NHp%3HHxlq%#0?ir{*XQTHU_XcaAyh zdH0#1s(08EGs-d3#=_tfy)L_5*7}RLl0Thw*iyRgLWF+3LEG5PdX|`^NRT0<5r*5{ zQ5E1Fp{%ZMwVTZLI-CDs#v>io9=PuwaCC{;L+EpB6UUq=vGd`r6amg z0w(2TT=ezfwnUoU_g~aXXZ}j%EHk+5k zoZk`?m5`1+orB#;9|DWsO9M7JfxGXV$uMw#et)a~i~4CoCq^pzv2BF_;eAwCAWKVF zZ~Jg&AJ6I5@xtlY>E#T|EhOyG7|LdB$^j5j;RDpkF5~({O>XmG=KQdyHRNy)LR5Q& zEsmF2i)ltBlfD-kc!gq%o~QWC9LkT*hrZQv{la0duyF{et4+cpVG<^~u-G@*p|Ia% z`HWBWaGa_;>)XwOgr@N^+SMn_8PlI&28C|O-x>2}qMlEqcZE#+#n>segjbAoYq@zh z#$n}9L>E^X4u(kh)@BF+HVSfIQLqcrS7e=UowN}+%y+0J(Axe%Ql6l#okIO)FRta+ z57Rkn&j|?u&>aMqH%r_!M1^UQI+iL_NrBXSX^iGqxWo`eFDp!Jn~Gcx(gtz(Q8Vj zr@c}cvynUF#qE(+R$Nx0szYyj{V@lrau)7IRS)nm?M1a(N>`?p-oRG&Z_^PA!msUBViD~&6leFe=U>_3iVCrz~$k(R)hH@4CrKT1wsAC)i zcx^V`w_Ig;V{BbEKPdGMKrx+KlDn<)O=~t<*OEQ^MX}0#D4DTSU6lQ3@F3CorCseh zN7fZp-J!0wK?#Mhz|B7NY|=XI9N!dbRO#?Dc?bLUi}x7e37!aksymsE(nazTdEX}- zVcKF8+fzJ`A6H=dZB_3Z#pE$c1KVz*RE9ym{@}`Q*eee28ss}VL5e1#?=V|ITS1C> zw<)bDTPR@|VjldO4o?(iYUo3N(1f`iaOlou)RY_)Sk7^T_%96in1HK_Ws48S%r3jy z>08%)SjGBt$gl9TUX#0w=G_scOzCoOCoxNFT)w5{ao^~jN8q{|1`u$GYhiP{D}C{% z`UE}}m7#%GU`cYp56)a3nScSFw=){1sYJ><(RK%epiel5^*nL8Jy6Jk}Iwz_)(p9{~4dW}_J#fK!7(H*avtT_U46O&m3Bw z&k3WM2U{A3tWIKw0h1vv-w~(d$3eO)muK!blLth>7QC0Lmp8&;yv;V+$1RQS zvPy#!sTjwd`L>ie90vnn6rtA($v5T*%69B+7_D@KHjlC|lT|w*s^hGLo5D*aNSjf5 zv<-aAI595$CYZJ!c75q+Nl;3h!?dhi(IY#FsW4#9bA7>Hg`-A4_yk$8qil@Ppex6+ zkxm~gfq*>3ya`Y1B?_g8ljPp*f2PDzm0dffH;|TK{8KBYIy}HsM;m9+IlbuX75H!7 z*LZV)L*zvMFs{Ybf=8NmD!y%m(nWpJ59i~bNRGX!6`&hxwwGW5e`ox*v8LJCJX~M? zITOQ!@Xf^m*BO~cVG=QmhKne3xJ6~9r8Pk6pkVU0e|VUQ?|7J1^Hgz(s#g_zv&UhvlC(;2!|K0|f{7;At0z+(D)|WZ!-#w_~`A+IVftE}%OA?=Q zgyAb;FYJ|-)e$?^rk8(b9WlJU>_*hfGLZ;KpY`V%TnilDn>ol(Xzry+uV3jySKL}B zAb&}i2UR%T*gjPG@WoOJTXq4>hbtc9ckr}t-v+a0t9>tjMEv_x#^GrP`{_5u{3e){g&buY7DtG!vXsUiV%ccz2d;t>UUxr0GhWuTjp)Fo6s|D6zwI&%{V%aW!JtX&ApFGT?nh$FL2&c+mk-)c zD`v^gj_Yriz1#V{rcsdYC7jv*j6K9-7LMh*zF0Q&IcHBEIe(vO^_E%k)0cck9h2Rbb1silUEi7*rY6dkS)NB`EZLF*a{m)q2dGxz3UDrQO z`~nD_+t01rQ1%Kz&zHVCBc_-x?@Kpu+RC(@oN4hDI6gkk%g;~9${PL=hN)j^j_%xg zg^^vpue{A60YR8H_cxjgH+SC6D$m4MK4`YzSdY;DmCzcsl2Ls zJq+@yE}*N=fF6ANc5*eI~pq#!{7v9xHZ!&Sq zBQYbw%=Gkrp+Kltn)>7hIAwY@AKyX)ak}Y1+8}+i|1L2x(aq7DIikCBJzgsHaHZb% z+k#K`OL*fR_uAU6$#DZ)H!1r^f31pDMbQHaiIK zL})iL>&%Kngn^(9groxyw9keXZeB;WMc(XcGFDmo9)o-rP;d^x4epUv0f01+>DPm>lX0l z&pWV!gcT@kO^irySItX4lnszWwFQ143m2z6IkV+pcwWxZ8)vmMk#FpIzB{!YE3!OX zt2tX`WlOUNEEKlAFVd>9oBFb(Yofj!bk}e)$N@tABU=$LpezNfpLc0WKaZEXX3Z4w zUxq;&FFq*7kY(7k!=?N#|E6#mqzXIEjco{Ncf|GN&a1rycz3;h6033G9 z&(HsFDpx}0?lO0*a;|od*QcL=#^;D0-gHCbcd-j>o=Uw|Q863l5;;BYyT|O5zmF|t z=c*VKa!CXIH`w<=`^uPdnjFJADF`XOO!CSr*1~|#d^+2c(_$G5q&e^^aw#;NvBQ_6 z9JHaqrelrX!z@R!y)(i=-qV$1XLqIg#+wqE$VD4ebkbZ;|6Qc-ohAULBYZRG(ItlE z5`Z8Sqwkcg8_mH5L0-S_+de#0+4!Fw%veEeJyYVNB`-RC(7E6@ZUmqGB`i|wg$`tD zm%VJ?!sW&JWdOwysd2!0d0I>o$-YXxpqyC~dG=H$fzMwXx-Io}*KedL$dcfP*PVB? zA^X-y_R!6tZ#fyaE|lu@IcJCU&<$X7w_^iA|E^OuCwaPG&3R?P){KB#T;*_FnO#t| z-8o@k2O0E0rTl$&cWj>EoJpEiaYi4n_hu*It}p|4Lc}aVtJ=D?AB3c@&II-^T9!XG ztj9?pf1EBQ=V?15;?RMbBbEX=joL!>mmfOnmq%9Ot%hb>y=f6LN8dFc(g2fyol?rp zjDi7hf%MV>x2xJFKg<=ZhqHcfl^I-}l&G{^&)K)!jY`~JPLAy_v_?eGTo@HeUR3t! z)v_b}>-&Fq#L(=s<-IVSKek_@o*=4lyvR%C&BP6*gU!FSo;PUyHa88CqHPS!M|6{J zr{MdFJi%ey(>W9)ca?aO4oId4Yx(L|g4&DhC((jTBD>jTt-w?H-da6cCn+>y*VDT)Lf+ z*{_?;Ys`|YU)k#MEWt%mc~KArYco1P;xOH0d!Jrq>btNl}+U+){;Xrzq32|ErVWQMKRWgd<;-f2#@1^Rb& zcL&ZXC@P+u35*@Bsqy$ufnbAwz4^AhXJ}}y*1mE5V!mY;r`vRir~OTx1Kf8;bMJ4X zfLdC)>A8u$@7V)DlWR4U`svfB%Nh=&Cc*NSYj%98+b7|x`fbmN-6Ly(yOeD)ul6(%9Id+(6K8n3o8ZtH+J>qaZNq@1WF!JcU8Lt8&3(y zqS%?hIH20Nh_0%tO0U+=3UUW;-KNQDvR!^%=yf12 zR&&9N6V`m#m?kT-xS=ic05j3Acgi+#C%qD>Qq}G&cKzJ!FfWM8?`~pjZ*Uz~=sYW~ zr~Sc)+8bi*iT#x#-tmT~F}^m4A|1k7R=tkeI99EST?ID|uU>8@(CPKhsx=|L1pF-a z-E)FtlEg0Jqr^6r{t|x=A~(GHKm55x23 z$sN4=t8LoyYeGh)t@KwvSYsPKV|0D5s;m@*FaEt{oUOEI)?GC6UhWPKL5<$0vcbZ} z&Q%%%=fEBY9jOi6|C4*$W*Mcscz$>DxMbSLu6AtNeB)_>{fApPz;oif;gG^*(jb66 zf1O1eOc^n4o;9<_otd`0IoazFl;n-KDn#rqRUY^|0`lr5uXRbwX)3BHPkZLDQvKR~ zK<`+#mAT0YMM$|&SKAtDPSC)>Qph1IF|5F&G4KFM{463dr8DEHnoM|AgBwl+>P_{^ z*;YVyUsGrwvkTcTK<7C`9BJ%#cXZTI0E%Sl%S|VUzl)-uc(|EM-$A{rb~yxxnm?V= znM}D^xXf0sU0PG>im9l`ce9eG_!I=6g~;2+=jZl^~Y1_S{aBJl|3mgtlN8kbqCwBz1P z=j>B=5y!dVlB}#OL#Fk%yXnl~ObJ*CR4xLq<>I<=p6akhBU{`z1%H1z$9`UadF-U9 z8PRrg>M7m4sPTY|+Om6pd4~vyr9W}DP{cpR+rQ6tAAEXKHY9bSFlC?<&2Al@J+MHlGeBCfM4M1wvqUXP%3ZCrz$NpzLFT_5d zuS1=fUJ`3RB-quCQJV6CcMPfSo!A_#RDa#w*B`jp-vW{7j~y%ZnF_P8G1tdQO<1_R zqO5)P{zU45alaCIhEliNxEbXlT-()pz6KAeP^I@Q-*Qx6;brvP%$|0y4i^^Vc$}{w z^7aK;m5x)Ns_M`&m7*RmGH!|ARGH|vu7dTQ+Jr4VN%eR*!~`8r3dmEe10PkMTK zVEATJmw}PdeMj5lRd-j#tV4Ri4s;0QqpYmIIxM(N+OoiG>pLa%as2Df@(Alu`qkU~ zFNi<^KE7?;kxuVTkCbBVqIb<+ZDYQRo`D%c?gh;bnm0Sqc=ta>DZ! z)@ckE#Zd40=+1GmY}^5AiZ|&X%12o~2)6jklCJK=%wLtKyPOQ?=@{w0mH4v;`;G=R z?9RHb=R6v7nyE$G_3+BizYM~O*+;7tZbq=H#^y;F!7?(kpFqP8Y=8&VsN}nTd%B;0^fqrhLjFBqU_``W|yIg$FR+0;Thc*_K0r&re;f!F2jB zJjA$Me%?WtSsn8F{rmSJeIxM#s0%hwQb{f;sq8RJJfA6T==&)CActSx1x@F>zK*7a zbd`o!0m2m$7DAPdcOs^~G`wOG(#d;zI%OP{sy=Z0izyQy5f^K~e5ei?qTMK_^O+ev zJJ15^yHe2*B)sNx_n8p+^ zoyc!<8~t-15t0P1{!ZRj`(mRyHz2z#+LRx{Kv5JNz<(3XU&UcP?5dTMqwywwi|p-- zPS&PhWhB~(U+r~B+?=c$uwoI^2Y4DEpmK9ofey)4_rQd z`qW{*F1FSG;_KJ+m)45mnwhQsh%2PcS`;CgyWltv+)XFH7I%x zO*_tudK}R`eag}z{2NvGYvAJYUdygqGo+?QaBV%_e=3FhUHI3pUjxQU-WMr@`9;*V zi{$9-S3PcY>s))f7OK34paRFe0-rK??ZeIht%HKj@8T)G!{eV3<|fy|D<7upsg`54 zdu!c*JXIo{j|f};a62Ow~#_;o}udc7&`hgXYD=QEA&m`EIj z?bw?#5zo3))I?e8kd(!dty~SddSKXBxVR&3jPV48 z-%PCzM+TQtXz)CUgo0u)xoEP1y^RIrYUbTQIPPQRqfJ`3CHLjvl70{839OX|8Y}>cSiHH_I%HDV*l?rAmDzs5G*!{ zm*|u+2nbYHyhHCuNKtV6;*GZzT5$OU8<&wu@R`PCyIB#4v}gAf5e3eQ!+tR{TzZ?7 zjncGB|D0^g)(v~4_MJ*>jsN_%0eCZ(P6xT5cz?Kh9Dhx z9ajiEwL{kUj#j#Wc05~G_Hn9sO26FVB)RB)(T3{P@gJp!cVG1$LHMZZ4)bWz(RKvU zOwmfYoV5)(1{sDG&0)NDSyuD?sUwSz=N;eAS19QdWfa2s83c{MKdy9{+zO(=_o?m_@LF4=MGUp5S`TJ`uq?=vM*4;b(EBlnPXzeNDC$(i6Pl8?fF)Wtd z#>6u5Pv!*6e~AJhGG@yQ@pX{MRrJ@g%hM;F#79YSaeVb0(yjZ{P zTXGB#bJkQ49E|ixF+lxsOuWsEOnTX7a**pJNn}zm#rzewicSit-X?`Pl{-3>Q{Q??Y#m9aUDTNJV-IbHysIamG+gol-nc-(vgiZ)0AR3o8`-nKc0dP zyLScMr$bzdHK?H>q0&k0xciY2Bktdi0nelEKmzQ)J+)sNm_+fq^F|=to_o`|3eluN zxXa&Vd5{Ry@w$!l@_>;ax!Wnc>a7F>c4#e)JMj} zS^ujryP~lF8%b!4yf!}m2I7D38;(Z_*$#K*(1luV#gL0q#od~8kQZy0kM4`fOw`pP zI41F>kB}a3)u!AyC_W`MwRftu{G^!UT>;KR;JG#iUP}nk3%9W$X~cHU9Tw62zSpbJ z)>Hf(N#BV>G3&I1jJJ6uSqi|t`Y@?n<-@M(XA7!X`HtjHXjXG!ytm)XVPzJVlIg=S zA+J$83;J*a_4*|Z!nH)}UFqN--#D}RTC(*OJ6&EeUP0RDi3cOdEGr(HOtjMGXeOF& zj^FATk=u-(id-P!y{Yg%)!1fVI+ObQ5g)rP6stjpobn};Iy3-lqzI=B0 zx=*uVi~3pmY52e+VPKeLTkJtMP3D&EPlw2BG-PX8_z_Jsad%oVKIuKDHDA4+rz7C) z3VT{&lEN?I@qHt+e#PgP%%=z!-o=5|k~p{Zv1YoC-X88LA6aDc1U0QN4BV?_CEA<6 z%aVT2?59hQ{j<IQF~TCAlrXKH;~Fq+8W6`y?Ra@sPk_L zJY~M#`P4s4Q~Ex#!GUrGq}=@5g*oIxb{-I9^9~!vwgogyOk#5Vi4tyl`X4VjLX(o- zDXSkd@o^1r0budLx+DSeN|C58RzPy|$PJp(bB`6(DE0qariX2fqbR`~7 zOO5}aodaJmCcDJRG(VcN=UKn{j?2ODti+A{3I3J`tA&v{a}JfV)nN$nC(JpY5;+n- z-ROp|Y4i-+ot}X=Cl?ZjM=PlKK2wPHHri-Xo&u*uKO5@xvihYnc0}VAg}B$gUf$_2 zC=o_fJ1vSY1#UZfFIHOg{rl4@VNmbL!pV8q(hlhB>)&3?I~#Y-X=F=GHhJzbWZU@o zh;9z1A&R5H0&h=`ylM)2N_l16@7~0$tbmJfhUfGgRqWQ;&>1hT6R|BxMw?2R;pyB; z$E!bxmoF>4U(OZgqB1%~G=DU?;Et0ezLPFLI2>5&)^4M6X+?L1Eh3IqF%MC{W}yc% zcLU`JlougM8^DzUfr}(wT#qc%3ewMx3?!)nf068?rJJB_!H28D98s@bTwPkJS)Un;r-CvA;fiD+hX?`No9= z-16+>;Y&4;jpEP<_Lpl{?!n3t?B2lPQmgW|E+Cq9$4xeln{mC0L&wLyDB6y4wkNj-9xO&g2|NgnF)YF!=-eDXC!h4-Q- z%Sscl$iFomprQJhdcQE1F-i9^^XQL+G@#}9q3V8z9-3;;{qn6J85r?%zBa( zXPe=?(-hq*iwKs;>VNgiKDz@fzIU-AZU;8sm|2K4K+ImU`vBNac{M*EqNM|`kM8}6 z7dG?qm`HCLLnWHaSppBVxs9&7md6ynB+lm1&V+UB4755k?7;!rfm26f~8cnAlEY8?A$R9>4TuFbKRerLhflY%P0dm z6t8I<2;Y#ywg7jHBMHozotLg+GeF1pWhTAnhrV(@13qI7XPvj^gmsS7hb>ZuwF#J^ z{P=Oz-}gVQJE@?KH3Kk@u(>S%ew)uK#UH5|q=yzB`x$UFo0`{rwz`i6A5d{uR#mgP zgUCG>RK6WrHn_C=ql$!KNTYs$b&uI9&#lYa=4YEaKwdGBNN+D@}^D^2LH~8=6}z2Z-P;d;6W4CcxEa!tMx?kq!4h zzusCD05{Yy_j9`urKFB;^6k0Vf+O@P-3n>FbT8>I*55eU;V69T#Iid9rekNiz(ilIO;- z9wLO$x%wn?ILagIYa5s;OnDbZ^UhyCnD)Bvm7}#S5oGBjk*O#>Ne>{_kyj|iQm13Z zVccQuUBz8F(r~tzKK#6Xjc}uGMoj76pna)l`EYhkaV^B_QvpzW6d@-|lN@=js3@5A zol`atXQ^OR>vU&wnEsYM9N`^G;Tik+JG$s?Zh;9*!g2RKByfuy3531rQY;YxqmRLu zq4CYvORDBv@6pUxT#GG3)J#DErkZxCV|V9$642d_kOpGIrHWMDyDe0u8P`iPm7<*) zN8zjj6-ysh09NA^S{7XX`B26y)XKoi!p!U&DdCUK_p$5>CE`=9bF+@nmC_L}Cu14! zZ_F~#tBAb69yr}3dOQaFSp`Du@1!S^BCgIPk)*S3FjG{O5;u>G4nCTh3E(hvzp~Y8 z)rv#jOpbZ2DyGsfKI^g}AME9uIz}fYtY^jl12`q}& z?oKs=kUrr0tohz=$!{{L%?w0h_ z)niQ}xkLT^rf0trO0Vp>ri%TSBSOQ%qT-x7!CC;=e+8=fDIB^JP$P+)V`@Cip)@XF zJw^9^DD>*|RI1(ousH_=IG+M-}zH2KyHll?< zKbarzR7fG~P>0Llb=;#3+3a&0HS_xpz6V_(Aa2ugH55IDV&}^Eq0Xmrxg`F-{*jlm zYuL>2JR6ZjMiz}N3MLJevG1ozhbu1PSMB#w4Tx_VJ=w3fmg3v3ego4|Vi!B(A33d5 zH`;!v8B1rkXYl|AJMT=I&7UF8c6xAi<)43CxE25Qwdal&*n8(JfP>-bFo4|+eZkui zXX|%g;TJgls=e=Uu12~%4!(^is(G`{W1QU>e3|zwr>^xU<;g&+9JUJy!}j#Ax4E-T zRvgZ4$4$C-M`m*R6)pgMtv&3xnD(ZJ*jUy)*uH$Hd;CXtLzO$rU^2HH*mnYo*jC^( zIxIH*UWA{?5f7ccsFm$E-09+DRUFk z1~b~zkXZE)_uW3Izuf7gB^R?}a-6HlH@rHrw6dzS-_{_#yPz}yq@rSCaLOy^YHYhv z@wc2`RQLbxaS%tkIYIEvFTcI4{IDDoc<(dpCq>T+IP>b0@G$PJs?R)^j1Qb*`Rc`h zsM9uB-M zYPwWkE~mu5c?c|D=E;VI2O<8GtB}#7c*wAaN|H0rI5qhtwu@f&O3|usk2(4yHxyej z7N>H39^vipcO%fQLL;Q_6vN$2!@TjzS8U`5&@g6PzLEANY3aAvVp3kvtR;-uXjKQ0 ze}I(*U4&p2xn(FDw>Qc>WwZT+hVH^IlnnbbkMARnzS3nC z04mn0NhKeI#kXM`T5rZAZ#XkW9=gG%uSL%YJeoYU?a_Uil~%V}utqLK*R$C90kG4zkrS znL1o(46}g*&6V>}2!8BuG1(@jGCcCH?QhT4F5}Yq8&F#1i7lgJsa?un%SU-8?Z2Ld z7QO~{cxEm3n$I^=F>X1>d-r5{ z9tQ1_%Nb?j1B7CChNdT)oILjxvaHP?Dv5hoUL++#qDHIQ#xm)Gq{dgXn8n2Y!P=yZjSFwf@J&-4}=C3cxA@E;E~eH zgT2stbEUs1evr8%XuHs#JxV7jnbjEjeRiLd(PXT?@*lFS*C0dJF&; zvx7mqi&HojWz7Ly6%$xJ5B77eEvN+iXf_h=;M8_SYV3O;q?+7NakZc@DpYMoTWaSR z4LC^l{X48=U?*|6eLnd0$4}I({l>hP^T%HUX zg&ZHBJtKHo(A^o{%10HW_Wf1S6C5Bj#FPCz^z-M*nkTqe!>(DL{&*)#n(fu5cXa{n z;t*FmtLnEMS$(q^slTHq0W3iFvSAqTDUWcoH^uMS+RKZqN5F>F9eY2oL&YR!_4NBi z@4saRJbiV-nX%*oHhh5R;Ik#b8HG~V;b`!OL-p)0-E!&2w)alC@bSE;ZCSnTQKL)u z-#Ie}Ud6DZ2d*fv8h(PfJ`Xtlz5O~{G6T%#*8#MepiV>u8e@dZ!YLmt)CJ|pZTuwt zK*3S-PdK@DP=vdxa
    FIJGeb*<1tBRpMQU3>ca5+DFvRpQ-GdC|9TH(MsQH+0R- z8K}h#Wx-!7e)AT?=>}sHYL7N#^8VsuB%i2p=Xz|q%>C%?r!YsW`=&B5epb#q7|W_h zOsebV3MnxVvbNOK{cdx&BzNY2Y~kGuf?}=w*Gw9d;*99w(IFvb9mUi)${Is- ztU!cz0J_k|dDLy=NQ!P_fF&AkaWV@l9a3S+Dc0c2=4>gY^;zG`bpLMgYNR2O_3F(q z>D2VseTub3@0*w+G_y9A;tNm-NU1(`?_~7toc?9i{ifqtB2emX|I=Q31ktGA=FYq2 z^5Ys-tev^D9Hayq;g@0lp#J=_=tlw(JbABd7$ynQlhzQ zXP`9sZ*{ReuC7AeyQRrOV(nbj)4QC#*_uP%$|x1f)iYb#oaC;PvS$4@PmvPueT%LG z5&m<85K=NXVQ;dM;v~@UL7{zg@$h%Ne*N}$%|KZgarv@;KQ&$3;oPXCU!ZsUbzt+p zGtHg3PJ$zZ#G60k+vX6j^MX`>C<33F17H-@i&wTQT@Ju zTJ4`BdVkh0dP#shbnSEY*GJXWkmwmxTO`@+tE#&>;e3<%0;Ie=n-yMlhgq$cU`*F2 zC^)_9lDYc0q-9+`*I$xbUf5Hle(>Qhbk{qW%#gAjSyW?FWyGx^bb`JyuSPU;zr^=& zXOb;G)n*u2|0@JBp3RhUwl6vwr@F~v^KYJPOi}wBangg^>a!MZ45k$=e*V_B9O3Sc z8qN!<*lkFBLG{DTQ|W7~jL;sCez+|eE|>ERr8O^2kcm5x9R?nCqC*n?yK(l>wxP6J z`;BBg6go+!^!|!B46o!COw2lE-*XxzcIwsvZuLWMqKjhk2&&jISfd4T5|Rr<)t1oI zJGR6^ixNxg zGAp~Ou|>vE!4NNrQHK(@17XSxJQjb8^{ewkE-<(weRof4LNbIEu+8IfS%PWR!}o;R zuioMHXDvds83$kE_({R1_)oWhYp)SjN zUDGV^`oiK|51a&4F5Mm9Zv4`QmuT8<08yk zdxlnwUA|_QwPfL)+FTTTP5%DO+_y~TKBoei>jaCVa}2ErnloaJn3oH?c{LmTw`~U6%>G|alW^uKjz2%%I+m})Y8zvqX;T(| z3aA}qp6zMGCP)=lO!8M!3bUuh=9#5nc{ne11h&ZK&fy&Vom*sP!JA~_6pa)o8JfuR zd+}ka2y&zF*hTOxWSNwE1JwNs!Ok~e=HMa#QO_A&adhO)EAR*@8~>8&&=et$GD}VJ zjamypxVo-D2j~$jJEoUKUdNgck;?v7hqMi0fFwp5GH?!kAfh50&;NcWKsBm>@vNa8 zjM}f2bjrZ;bSwH!Nqd4k6HkHF5)~1dXeC{sCqS6c@@(rKWv)Tv(K7$D|1nSRf7LG^ zKLR(N{D1fEzw7^hdBOkN13q5=|HsQe&j0`O%YUJrRBai5N00!N|7T^m0~z`u2Sb|V zWYvpj4b^ZShQx?hizaXWt4V*MpR6J^`6m|H-=1WpuZATQJlWQu%*?=$#*~>+3qU)v zAALgd+3x)-v#=h&FE7Bz{QSrGN=MYoE_6tEsX$BvZnS(WsNUu_gcT`T$NvTn3pk=CQ#|`<6>6< zCh7FgI>44FOEc2zdjg_oym-+((dN<&Zci&_?8U2_NPoXs4GG!fM0b;CqMYP+=H6$Q zdAzj%lBv7=aj~fC*ERt97ljR+r~2u4*b>UhZ#w1`>2(;R`|}4WXaO3s1}Li{7FJC> zM#;G5qQb_Swf;2GWX4o2>wSkfmniR%BhFcUqOz+^C9G%n!!u{Ltb&{9NF;?IfRA&C z{futs&1C8(6{b$j=eNb; i|6e(h_W!*6{sXe7kGp4+J4q-KI8+og_YhexFM79f<|Hb6jnlOmyaLhl4r1f+KY z0fN#YgdRc*oS^Ub&7AY=%$f6LCOx+4$8?ZabW>cKqpB_`x8#ew_n`a zcsG*xAXDYe+Lf!Ek^7?97T`t5C)N93;cfh(~Tz77!a`_%)D^!8r1eR`?g~R}JkB#&m zl@$ygi-MX)bF$=E3WttJ^*;^gD6LniH*x1HkVmrm=1jmP>J{d5b2(;ZUcHmF42;ER z)ZL=Ke6fQ&C|kvawl`=+x%;to+T)iGo7)Bh^y)Y=JU48>m&R+gGUW*Zak-aMv|MB| zO2F5qI-(%k$vGx+xxtSj06AS>yLK z!}~u^^?NQ*OI1&M1a%HkxNbFyJBwQI^bfqQX5A=#>}MG6>dFkjmqiG*2dQ^au-}EB z885xD`mFJQ=AR44elb074C76}cbKTgVPUfr2;Z8|3QMkTci{LYF@DkeNxAvo&W_s- zYWE3={oxz&e6V~o5Txyki&9gUXz z@_65m{5t?=>e%+^DOA(j*k$h4p?#ft8|PAj=eNJJNs2Ga&-2cse4g#ztEwq9EWBIE zQG5M*4D4vpusT%a`tS#z+8*z+XnzrqOtYKJXop9NP3rZ(bH5%%#x#UC)Un2K7@LT3 z?*=uGm!6v$nZ}LX|EVqLU1YjpHs>-6l)WwQpIF7yt-3oZ#Rj3*(MG~r6I|%@J}s;N z*bwaDZce+yq>{jq^1$)#^h(IxEVUl+8Y}KyIQ9-KBk%WC-(Ks0xA^`NGkcBfL4|IW z7g%w6IWD&@>ui1cv61@Cm96{e_|2Hn*M}RRKcOYAeBqa>Mm)fAL{64oH8)E~^GK-D zaqzjo@#|M3`+K{YXS;-d8(H*yvq&x!uU>)h_+j?B@0H+H*;PF|Vv8;DGRkrlEjBd(B zJaQX0Dgh3d5;$gOOCqB3zLbi!cxtEPSCT{sG_=m+t(3_eJVHoC70Ukvl$_1Ofcqum zWI~dFs$^V7mbr4gwP*?lEi3h$m|N^1#u!t#-rTI@s2mvg{Sua8M#yC9Wq*7SQ zl)r<$C00DCa8)u^I3*X3Yp-#$gBC^90MgnWbBu4QzrVd)={S<>(75d0JgcUyZRn~M zRSGJRmseQY0&HcCJjI!My1R8$R8$(5J%}SzRbxkg#{kdX$ji%H*MiWNL5+=#J^lTO zQ_i4B1`%sfy<^r3v)vEaIpDkiS;qi-x3K#9RCW=x1t`saAdj1!n3Z&w1~l2FEHj6m zr42rgr$FAS`>c6vh}S|R-=0BJA)~icM_T6UE<%RM$`JCmS|zEatPVJVSkt6l|Jvxb z6i&aac?P?`NPtPDHMzdj_JyRZQXQGVxM1 zWeWX%weAP==(*C3k%$P8!$643I{mD^3ub&|9T6Is?_|uCJ*iF~lC^z5Z9QaZtM&Sh z6uofhVKTq?CHGBn|HY54tw#@u#>%*#6OPYLws4(kZoIt3kXXWv_MzDJP!Y(kb${AP zN?d$=1hBE<)13G@CiBOKD@3of$~Dr8r8d(uc?8?cQtH`NB|Q%-l+cgmZ?TqtIP@dbc;;mU92m>sPxqQm zGQv1-PG}R))$w*a)&Al_G#?L7(NEcwloUxOfUcm{3F(L$OCBeN261wBV!%-M2VP%N zJ3ACp*v9u|JwKA;vg?_IOrC%Om6CkR-J6eV6Vp@5>{?G$o9{$nW?^E*E(Dbu_DK0S z8msy>gwT6~dfP}VL6H_cDJ-zbGCP40R*makIB2C~d8R^{?xqyYP^3Uknnm^SH` zBaTff3DK-_a5Tby4RAvkJ3lj{N$$-{81% z3v)q)KCCa@xEMovaR^>c0h6j`iW;?!Y9|Yvs#Dh|cvpup9IDuL#q|g&7nuTDN+=$i9J;g*(`){6PaS65)b zqkRi~eJ^lhZUnGCR>T+Do+T6DgGsJ+3WA0j8yhzru!1hMUNbgS*NK>C;NSV92uMU)Crt;Mnd9UR@ zvqvlVwr=r^qBq~n!&^49oRc|<3_iAP+!c+wyaGfhUZ-1+LP=M8i5VL+3kCtu|M`0=gK{WA?^7SLH7FqHXdJmdT@E`v4L=WguA=gh z1(?4!UP@mY$`UvM`nSr60&LU7Jl2L5rJ(cwRWWMwF&84OMGK4XXpp#jA0YdB|;YfHH zP1Q?2F1_oFJMY#&CnN!Rp9Hz(+3aiUjNH}}X(H5Po%ccuC1 z%Ph&}vUqQU*&`fg)Rf2sWwC>8+wb7_9%ug8Gm~)t#dCT7-9vN$EIY^aq^3U=ry5oc zmgM2#VUY5#@vW~)VD%q7^Nup9b2iA9`2PL77UM10zoKclAMt=TulV7j*YKLRBjFfa zz~qbka4i_M94J~p`R4FSe0Q!C*G%4Tq~+FV>tPpbt;1+Obs7DIBX4?Ms1T8Z6uOHU z87b(W1RJG92l5YACQuJ9pnP6VY4<%Eb6u3YURx%en#WYH94lHPsaO?%PaOG$+jqN* z6*#Pkrk4T%c#W2N-SVJ40kM-7i)w#xaKFrEXC(>|kgPq;?5j02^s1eAg8`tscdxLO zaC}tbGSg5wd3J4Q=h$IM4OZtAN;#aDY+*9*)QUeN4@MD>?6(L0mo6-PS$j2%FC`Yd zi}`hCNQ+)to?=S@yFoo_e17)CtCookYNT%t&7otMm#m!YaiDMT1 zu!Ucp+&$^&)$jwv8L9+LeJ#%W8yVf|Qqd?c0fn9R-G+QgNik;IYZZ1WzAIq1IyzBi zDn{__NuWqM*KhLO+T1%H`u4u#fH2=HNKrcr#>?b2Wpr2#$7Cr4p`d4L1-ktg5Dbt! zv&nqu!P%R>Ek`E%R6nUM(mJe|T^$qRx%F`myLiCuw#h5eYAlEe)~Tu~74*t65k4|r z4AV{wK;D)>V&h_C^%g}m-jY_Gfv=*kIW1JzVEy2Zhbbb(Hor;ezc$*e3@|f@dVkbT z>$e@vW`#;Gy>g=MV7bR#J;fj0wE(zyi8Jlu@q&BlP7|fWL?k(?QBFD2h763@ z4}HZqrX(dG)3N^uJ#I)fL3jPGpCzMW;41k=xStsinU*yaA;pa; zlAs6-Y$&CBsxzb~#{GK6n_ePHJ}8nYG~LPPXQQq18r%bH*FE3NbNUSccX6l+f{VD+ zhw~yy)7f?P^hEp@RT^hq-O^w%SnJ^~AF()?b2>dv>kpQn-@TH4GgP6yoH2i6!x`q( zmoB1S>p0GfG*MsQBK7??=k}aP1H+}fe-@NM5&jf{-`@HI$ZQ*R<1)||sn3TW7lyv6 zCS~*Btd7KT4#qA3Jy2Wt>4{H55lFLJ-=nBZrzGE5A<)T6wraDG+S6Y`fTzws>&updzw93wiq3BtpD@0sHh0l=v82a-!gPGDQ1E|f{%Z6vT)O6r}pgd zHy@;aQ`6AccSzeA?pmL$e4Vnw&n?9ET*_y2?018BUVc7s$P~`C6uOZHom(6Q%@7tB z7r7;l8njJ~{2H7^O^cAU3=G-7e*K%5yNJx@;`jrw={JbA=-Ds!_5|FavqX|Jdn}rMu+>4Aft1>6!H)W@d}!W>b-}oiEyOrl z@*4q}^iO2R1f7t_8>?l`=e~16CUxFL_}Xsk5wAhHa$@FN!XeBzLuLKWW%|>k3^B;^ z=BR4`BkVmtZpw(5o0OEec>c&VxxwQIa8+w-$3X>t9GOQGX3IE+v~fKJO#Vk1o* zuFi6tI?N#(AXiTkFzmhC58nPt>NXv)g(%5n5ZNDD?9O|1RN=L96Ni~LB%+Tj)~DH= z&N8r)LChh0d;4lie=B$(npvtC;)I`gt6qH{cQ)`i#afh7(PG35J`GQ}onU$0 z%d|=FPBQCImRWs>a`0-(0J!5Ht6!(PSl_uK-@yzuH8o(jP7|<6wX(7zHWP)q{osYq zLb9Zlv#!X|LFt^>93|N9*9I_V!r|NL5&AN+Cr4+Y^Y}Q;Z9H$ZLl%qIN-w-CU{Y*( ze#V)>$Hk>=c(dVqfq=BMnV?oi>8p0QUL->Y(#G%SY}OxlVI9UUE2Z|Quu z%LJhX0ss8fGQ#Cm0{{8^-SMZ7m)umS#lz7MnV#7e%g!@D9tj04RaaHprSsnn?=D%p z-70z$rMc8tnrL?80f(wbqbs6;Q&kZc|CXv{JszL^0Vu!*>p1s!DeA>-B5bhQ;QBEVJjj=cwlSw4PmbqK?Nk z#FXxN{7H9faiFhH*9Vk@2SaY2|8X%h`xZ6qxsuThdcy0)IXR`>d&2W^Hh^|-QWbL? z%vhV%XO z$Z^y~N|ETZ?X5Lus(scJ7J+kWpLFh;jmT$2WLwvia*>f8dP}WVaCbVcM)oq=yUsnl z^5dC^=QQ?wq`_k$2t_@=AYyg6O^pqoZGlt|Vq}TXxq93)-=+uT7n;_)%DU`yMLDd- zI3=c~1@u%Bp(y}=$f7T{U0lK&7MVLt#t>CU8l}4d!gYHc4=hIeu>aT4<j-Aj*dU|>jo$t&uWI@7v zcVNg@uk6ksH~)m5Sg%g-eQrI{K8j^8gClXsW=nx&4O;;MR=W&X8m2HapOO*XW3*XZ zblf{@w|mNQ_8TvU+t~%j`D&-8BV zxbgK9q!Cek2`dj+T#cp1V~G@e!>8HP)3e1TTK2TUwQ*^Jda%-YO&5!*9W8K~N8pqE zr2lx67#!0OM>uIZ3s=_ya8tk6?JDh+&00Xt(*mjtejZXH;gs+jJ zz=oS$9Y-~C5)N@2J^@9@Kh+d?rpV4As}MI*-6F@Tt|}!y#f62-ve;5?H@?(WOava*HuaHuC~>JF`yvXt;O?m;;8lE zoMz};jWHV=8yfJbo+>4>yz*$dGZlE-x%Uq7P(`eYY$22|!;WrnhO~VT!7xVXH@5NE z#;%lOEj$8M`07Pr-^2Lb^he{x#tPr+-Uydt(BBYa9KvQ$Q1uLpLf4b3TB~VSmoS;q z6FH;I{Mz>MBE^MKZGA0VyTVhO(_4AAOFx(04E%lIJ}g>8hL}40fx$K&slhCy>q!A{ zN-<`^jK<0bUm?3oZ*fU#f~@E!&#(_h_wBeOK*4gaGF~STx`w{G3hjk_0h}_R(v9NtU_eyf;*@t z{AyMTtZxL|9&%i>HC1h@Eq&$}w)`>5wMP>%tJ8FKfedPzu zwcdf@nQ=y9&|e$#HuU}Y_Jf9an0fh-Y}Hd7B^}v74Q`e!B_qCMf5S{*u1Vdpe6IK+ z^78MT(RfRsVTZx^dIF4lfN}0TxBGgMy=7pPC$pA|M2zL&n3bs0(jFeP8boSikUCrD zC7V{_<1+2nOMiuo%;CpJ#_)~vZT9HZ)mZim1yoC_o+yWI$p-Czj}R;(_7LH-te890 zFh@fVh!~UL;jTJvW4PVaZPK;gX|m##B1@0h2{0>hHgWDes4|jy) zFcT1k`MAxvR{!mP1`o8U_D6v85(be0XBpo=^^D>HhiquW>`_*1 z+cX%T$D~b=vu>VK}X{HrPK-IJ1sz) zk7!zt(O$#-=vS+ajdJ$CJF$}K=_dISrDT_K6vB8ML0t1Sua7%wCKk!mIaFQSUw#?- zcgw$ZszUSJe%kEY0SQd^Zblnan-qE~_`c^hJ$=#7pDL=lwxFa3PRK_)M$NNuPW9yV z`RjuE#1^C9aywWOysmF;Nz_{rM{U4)k>M8Fe+7H)(4Q)T*1f^?)D109IL3_@3cbF# zEpQ7UYF@Q|J62Q9q{1m0!mxP-`ZR%O!^6WpeSHZD=Y3ai+z{B2AznjhunwNSARO<5 zntiZO@q2^B+S&WyB(>l6Vo%vGGNx`VXQ_40Q}cQXQ_i9#Hoc&xWfb5fG9SV?2A}Pw z`tHBKT4XtK_JGK?IyKeK1j2fWmjG>U=u#@V(`T@O)I1{m(;mxY+#>qO_YdWerr2Dz z>ugivA;X`?pifI(?xmvS3f1flAjgjp!&T!qdQ$*hqW}d(@Y_~v7p{3W(}LJv4&PW` z?TMInbS^*51nj&vUYYOBNTH!F4C4mxw{JDdbk3>$yZEG=BVN^{?c|NH*Y2X_k4LRl zO|N<=2~|(p;GWo+e-*_F1%MI**PK=61nAv}gDx(o!QmHMZ^0SD&W0L~UQ$U43Fsx) zhv_0JQYBO4;LYd^K2-|J?d0AdPi8LoD2K2HUA=v&4j?1RlW*LvYstv1H4~euNC zoUlA_Vn$?5dW3&&XicBP+CwqKFR`$GK~)Dt6R$d~>!-&nblCrDxMf41*%U$@e-33S z_JNFuvNoZ`U$bZ{Cv$VQ88a#YTO-y;8LSLZ0U_Qwf!LKJN8sjCwZ4NEHa0dzNq!!J%}3IM1IaV8_6=BeD1cK6mYMbIGf+$3aKEwk}fc`qv!jJ$uXx0Z*l* ziH*qu5vc=*_RY)w*g9Yx7==L-IfBQpj%}QD6}jXzaW)-@3?gzgPZlU@lfiO3%aiTs zH;0?gW%j?;2B|%DMX%4{zJ^-rPyRMmT}yE^kur;Ts5wNWFcZ6F&EEHg6sI6j4^FmZ zl5!DUaZi26mHHw7wmfp0EtQ`jO-LrYkm#jE54@?dV|jZAf7r= z7gVukT3upIb~aH119^L6co7f(ct#r>bI4@Hd#l@GG!Fq^8?lH>NKE5O?qc_POMUBZ zwei|i+P$;0SokjRW$2M zQ*17Od0Y_ybR&0=G8vOorWLME_A!W`LV#h#To|e+>$#7Nzd3O6ZJcqnS&|5Z!ffMJ zS=xur4y`r)0^m0v=C5K|L3jSPK_tJ@eUJ58-6yz!%pR!`>sA1EEDoupv?WjEpx((Y)65upYOM`FH_D*y;zW$ZgHzc8!$f2;!HKv z(a)&bLls6Yjmw9(6qYe4hjb5G#WsJ1n?+3VhxV3Dey}DSp1{J@)e<>y7x!%5;FSXu z#IJ}Lg~FBBqaweu>x%C%@Rv*l_MCVGNK;git)kQ=xgUtDbkiF*?UfCBE8w*pryJn! zuyv<9K=aCv8V#2jUlm}^b@HKV(_`LqSxHeD;{xx#niPg9f>mv)q3K)E^Z4dZnh#Sn zALiLC*`H*+PObHPpL=?lB!_Vfq-I+OMYS4+biIsS*-6>OyW1@;u1CM`x&R$j-I8N) zI70hxKz^A$R|yQEDTQDTXjqF)1Z>^r?vRg0211mI-~jDyCL%$kiX#ecv~+dD4YJ9O zogW2DMzr$vv&H1}FSs#6*3Ro^6MI1Yv&Ne7xw(yGy6zSq=oNlku8q@4?IrQ~RO6Vv40bv)TR9Rk#N@XouOns1Yw<=zM($-J zloo-GJfSO?E+Z*1`2Fp9F&#)xP6jAXa+%FDSElwBmoQ*ykLg%6CF^kUvlcNOtMO?- ziNFh7U9tnwZB2V*>)_|=0&{hy%{RIH)L+w-@@$AoSAFg)X1$#~abvwuy9==2I+;bi zaIhv``pYVMK%|KnZMfHT+Y#B2@yHe3aF}-G??W75d$>T8N22d>Rx9LiJa45Rn(wYV{%%qa5T9)bgv=3pD5`% z8+Y!yagY?u3rR1yKmq8J9^s^w>u>aU2KN-_e@Xt`jihkmYS`b$L6XV9{#W-UO4p8` zV($HCM6rw@rsf8>K2ka_N0Z9#Ewm&^V9C9`Y9TxIFNpyBT2d; zWk2_z7_DC`2{%XrCaBw}U%) zI(@}GvAiOd(=sYfIT%D+!9cP?A3c4f4_WzjbSaaYb|;X>P{q00#IPdm-m_TBj(?6# zTloiRu85O_FC-Mq43ej{f#WEH^lxKG-B}*fU_7J(41VZ8heFdR-vLY{WPX%`hr64* z@fUWLW@iX4!Bsc*8$xfp~1usmpo|AbH6l5M-s<-dSoR`nKKi= zsHWMkh3Dx0C4@{KZp1&FDxILEmGOd) zjV*f5^6@AMipO7f2+RAf4#&+A}*HLJoBHNCiyi3{P*ikB;)tkg#MFT`mg_96%l{FjJ*Ay3QO`# pMCrem`TsST_wSDX+tQQgm-RR`J`G#wxnKONs;Kn>F8}Yl{{_)%3nBmj From 5262973e16e05b20d550ee888b414728f5f1705e Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 11 Dec 2024 12:18:33 -0600 Subject: [PATCH 03/23] docs: update deployment details Signed-off-by: Eduardo Silva --- README.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index be2e805cc50..e329da456eb 100644 --- a/README.md +++ b/README.md @@ -13,16 +13,14 @@ ![](documentation/fluentbit_ecosystem.png) -[Fluent Bit](http://fluentbit.io) is a fast Log Processor and Forwarder for Linux, Windows, Embedded Linux, MacOS and BSD family operating systems. It's part of the Graduated [Fluentd](http://fluentd.org) Ecosystem and a [CNCF](https://cncf.io) sub-project. +[Fluent Bit](http://fluentbit.io) is a fast Log, Metrics and Traces Processor and Forwarder for Linux, Windows, Embedded Linux, MacOS and BSD family operating systems. It's part of the Graduated [Fluentd](http://fluentd.org) Ecosystem and a [CNCF](https://cncf.io) sub-project. -Fluent Bit allows to collect log events or metrics from different sources, process them and deliver them to different backends such as [Fluentd](http://fluentd.org), Elasticsearch, Splunk, DataDog, Kafka, New Relic, Azure services, AWS services, Google services, NATS, InfluxDB or any custom HTTP end-point. +Fluent Bit allows to collect different signal types such as logs, metrics and traces from different sources, process them and deliver them to different backends such as [Fluentd](http://fluentd.org), Elasticsearch, Splunk, DataDog, Kafka, New Relic, Azure services, AWS services, Google services, NATS, InfluxDB or any custom HTTP end-point. Fluent Bit comes with full SQL [Stream Processing](https://docs.fluentbit.io/manual/stream-processing/introduction) capabilities: data manipulation and analytics using SQL queries. Fluent Bit runs on x86_64, x86, arm32v7, and arm64v8 architectures. - - ## Features - High Performance at low CPU and Memory footprint @@ -48,7 +46,7 @@ Fluent Bit runs on x86_64, x86, arm32v7, and arm64v8 architectures. ## Fluent Bit in Production -[Fluent Bit](https://fluentbit.io) is used widely in production environments. As of 2022, [Fluent Bit surpasses 3 Billion downloads](https://www.cncf.io/blog/2022/10/13/fluent-bit-surpasses-three-billion-downloads/) and continues to be deployed over **10 million times a day**. The following is a preview of who uses Fluent Bit heavily in production: +Fluent Bit is a widely adopted solution in production environments. As of 2024, Fluent Bit has surpassed 15 billion downloads and continues to be deployed over 10 million times daily. Below is a preview of some of the organizations that rely heavily on Fluent Bit in their production systems: > If your company uses Fluent Bit and is not listed, feel free to open a GitHub issue and we will add the logo. From 039e91f20b3974c9d4a99d86e1d6e698d18e8990 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Mon, 18 Nov 2024 19:10:48 +0900 Subject: [PATCH 04/23] in_forward: Recreate connection when resumed Signed-off-by: Hiroshi Hatake --- plugins/in_forward/fw.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/plugins/in_forward/fw.c b/plugins/in_forward/fw.c index e692f674cd1..8b98b56b010 100644 --- a/plugins/in_forward/fw.c +++ b/plugins/in_forward/fw.c @@ -383,8 +383,25 @@ static void in_fw_pause(void *data, struct flb_config *config) } static void in_fw_resume(void *data, struct flb_config *config) { + struct flb_connection *connection; + struct fw_conn *conn; struct flb_in_fw_config *ctx = data; if (config->is_running == FLB_TRUE) { + connection = flb_downstream_conn_get(ctx->downstream); + + if (connection == NULL) { + flb_plg_error(ctx->ins, "could not accept new connection"); + + return; + } + conn = fw_conn_add(connection, ctx); + + if (!conn) { + flb_plg_error(ctx->ins, "could not add connection"); + + return; + } + ctx->is_paused = FLB_FALSE; flb_input_collector_resume(ctx->coll_fd, ctx->ins); } From 500242d9c1ba35ea5de2a8e734531cdb55279a1a Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Tue, 19 Nov 2024 15:25:15 +0900 Subject: [PATCH 05/23] in_forward: Synchronize the status of pause/resume with mutex Signed-off-by: Hiroshi Hatake --- plugins/in_forward/fw.c | 48 ++++++++++++++++++++++++----------------- plugins/in_forward/fw.h | 2 ++ 2 files changed, 30 insertions(+), 20 deletions(-) diff --git a/plugins/in_forward/fw.c b/plugins/in_forward/fw.c index 8b98b56b010..faefc4d82a8 100644 --- a/plugins/in_forward/fw.c +++ b/plugins/in_forward/fw.c @@ -351,6 +351,8 @@ static int in_fw_init(struct flb_input_instance *ins, ctx->coll_fd = ret; + pthread_mutex_init(&ctx->conn_mutex, NULL); + return 0; } @@ -358,15 +360,18 @@ static void in_fw_pause(void *data, struct flb_config *config) { struct flb_in_fw_config *ctx = data; if (config->is_running == FLB_TRUE) { - /* - * This is the case when we are not in a shutdown phase, but - * backpressure built up, and the plugin needs to - * pause the ingestion. The plugin should close all the connections - * and wait for the ingestion to resume. - */ - flb_input_collector_pause(ctx->coll_fd, ctx->ins); - fw_conn_del_all(ctx); - ctx->is_paused = FLB_TRUE; + if (pthread_mutex_lock(&ctx->conn_mutex)) { + /* + * This is the case when we are not in a shutdown phase, but + * backpressure built up, and the plugin needs to + * pause the ingestion. The plugin should close all the connections + * and wait for the ingestion to resume. + */ + flb_input_collector_pause(ctx->coll_fd, ctx->ins); + fw_conn_del_all(ctx); + ctx->is_paused = FLB_TRUE; + } + pthread_mutex_unlock(&ctx->conn_mutex); } /* @@ -387,23 +392,26 @@ static void in_fw_resume(void *data, struct flb_config *config) { struct fw_conn *conn; struct flb_in_fw_config *ctx = data; if (config->is_running == FLB_TRUE) { - connection = flb_downstream_conn_get(ctx->downstream); + if (pthread_mutex_lock(&ctx->conn_mutex)) { + connection = flb_downstream_conn_get(ctx->downstream); + if (connection == NULL) { + flb_plg_error(ctx->ins, "could not accept new connection"); - if (connection == NULL) { - flb_plg_error(ctx->ins, "could not accept new connection"); + return; + } - return; - } - conn = fw_conn_add(connection, ctx); + conn = fw_conn_add(connection, ctx); + if (!conn) { + flb_plg_error(ctx->ins, "could not add connection"); - if (!conn) { - flb_plg_error(ctx->ins, "could not add connection"); + return; + } - return; + flb_input_collector_resume(ctx->coll_fd, ctx->ins); + ctx->is_paused = FLB_FALSE; } + pthread_mutex_unlock(&ctx->conn_mutex); - ctx->is_paused = FLB_FALSE; - flb_input_collector_resume(ctx->coll_fd, ctx->ins); } } diff --git a/plugins/in_forward/fw.h b/plugins/in_forward/fw.h index 032371b7d3a..d51e1c47925 100644 --- a/plugins/in_forward/fw.h +++ b/plugins/in_forward/fw.h @@ -73,6 +73,8 @@ struct flb_in_fw_config { struct flb_log_event_decoder *log_decoder; struct flb_log_event_encoder *log_encoder; + pthread_mutex_t conn_mutex; + /* Plugin is paused */ int is_paused; }; From 470bb186e90285f8aa284a5d7f11c1e406400810 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Mon, 9 Dec 2024 20:33:48 +0900 Subject: [PATCH 06/23] in_forward: Remove a needless add event for handling sockets Signed-off-by: Hiroshi Hatake --- plugins/in_forward/fw.c | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/plugins/in_forward/fw.c b/plugins/in_forward/fw.c index faefc4d82a8..6618d948323 100644 --- a/plugins/in_forward/fw.c +++ b/plugins/in_forward/fw.c @@ -388,30 +388,13 @@ static void in_fw_pause(void *data, struct flb_config *config) } static void in_fw_resume(void *data, struct flb_config *config) { - struct flb_connection *connection; - struct fw_conn *conn; struct flb_in_fw_config *ctx = data; if (config->is_running == FLB_TRUE) { if (pthread_mutex_lock(&ctx->conn_mutex)) { - connection = flb_downstream_conn_get(ctx->downstream); - if (connection == NULL) { - flb_plg_error(ctx->ins, "could not accept new connection"); - - return; - } - - conn = fw_conn_add(connection, ctx); - if (!conn) { - flb_plg_error(ctx->ins, "could not add connection"); - - return; - } - flb_input_collector_resume(ctx->coll_fd, ctx->ins); ctx->is_paused = FLB_FALSE; } pthread_mutex_unlock(&ctx->conn_mutex); - } } From dee6d1200ec60f8d4ef6adbabda3ceefff190c97 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 12 Dec 2024 14:40:19 +0900 Subject: [PATCH 07/23] in_forward: Synchronize is_paused state and deleting conns Signed-off-by: Hiroshi Hatake --- plugins/in_forward/fw.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/plugins/in_forward/fw.c b/plugins/in_forward/fw.c index 6618d948323..1e4d83089af 100644 --- a/plugins/in_forward/fw.c +++ b/plugins/in_forward/fw.c @@ -360,14 +360,14 @@ static void in_fw_pause(void *data, struct flb_config *config) { struct flb_in_fw_config *ctx = data; if (config->is_running == FLB_TRUE) { + /* + * This is the case when we are not in a shutdown phase, but + * backpressure built up, and the plugin needs to + * pause the ingestion. The plugin should close all the connections + * and wait for the ingestion to resume. + */ + flb_input_collector_pause(ctx->coll_fd, ctx->ins); if (pthread_mutex_lock(&ctx->conn_mutex)) { - /* - * This is the case when we are not in a shutdown phase, but - * backpressure built up, and the plugin needs to - * pause the ingestion. The plugin should close all the connections - * and wait for the ingestion to resume. - */ - flb_input_collector_pause(ctx->coll_fd, ctx->ins); fw_conn_del_all(ctx); ctx->is_paused = FLB_TRUE; } @@ -390,8 +390,8 @@ static void in_fw_pause(void *data, struct flb_config *config) static void in_fw_resume(void *data, struct flb_config *config) { struct flb_in_fw_config *ctx = data; if (config->is_running == FLB_TRUE) { + flb_input_collector_resume(ctx->coll_fd, ctx->ins); if (pthread_mutex_lock(&ctx->conn_mutex)) { - flb_input_collector_resume(ctx->coll_fd, ctx->ins); ctx->is_paused = FLB_FALSE; } pthread_mutex_unlock(&ctx->conn_mutex); From 7feb09a618e741a68f351615be03dfecf1f87803 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Tue, 3 Dec 2024 20:22:02 -0600 Subject: [PATCH 08/23] utf8: enhance handling of multibyte sequences This patch refactor a bit how UTF8 decoding works by replacing the old lookup table for special characters/codepoints with a new routine and optional lookup table based on the compiler type (GNU/Clang). It also supports proper encoding of multibyte sequences. Signed-off-by: Eduardo Silva --- include/fluent-bit/flb_utf8.h | 82 ++------------ src/flb_utf8.c | 201 ++++++++++++++++++++++++++++++++++ 2 files changed, 208 insertions(+), 75 deletions(-) create mode 100644 src/flb_utf8.c diff --git a/include/fluent-bit/flb_utf8.h b/include/fluent-bit/flb_utf8.h index 00cb08d066f..b883ff0f78c 100644 --- a/include/fluent-bit/flb_utf8.h +++ b/include/fluent-bit/flb_utf8.h @@ -20,84 +20,16 @@ #ifndef FLB_UTF8_H #define FLB_UTF8_H +#define FLB_UTF8_ACCEPT 0 +#define FLB_UTF8_REJECT 1 +#define FLB_UTF8_CONTINUE 2 + #include #include -/* is the start of a UTF-8 string ? */ -#define flb_utf8_check(c) (((c) & 0xC0) != 0x80) - -static const char trailingBytesForUTF8[256] = { - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, - 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5 -}; - /* returns length of next utf-8 sequence */ -static inline int flb_utf8_len(const char *s) -{ - return trailingBytesForUTF8[(unsigned int)(unsigned char)s[0]] + 1; -} - -/* - * UTF-8 Decoding routines are originally written by Bjoern Hoehrmann - * and taken from the following web site: - * - * http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ - * - * They have been siglhy renamed to follow Fluent Bit naming requirements. - */ - -#define FLB_UTF8_ACCEPT 0 -#define FLB_UTF8_REJECT 1 - -static const uint8_t utf8d[] = { - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 00..1f - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 20..3f - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 40..5f - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 60..7f - 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, // 80..9f - 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, // a0..bf - 8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, // c0..df - 0xa,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x4,0x3,0x3, // e0..ef - 0xb,0x6,0x6,0x6,0x5,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8, // f0..ff - 0x0,0x1,0x2,0x3,0x5,0x8,0x7,0x1,0x1,0x1,0x4,0x6,0x1,0x1,0x1,0x1, // s0..s0 - 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,0,1,0,1,1,1,1,1,1, // s1..s2 - 1,2,1,1,1,1,1,2,1,2,1,1,1,1,1,1,1,1,1,1,1,1,1,2,1,1,1,1,1,1,1,1, // s3..s4 - 1,2,1,1,1,1,1,1,1,2,1,1,1,1,1,1,1,1,1,1,1,1,1,3,1,3,1,1,1,1,1,1, // s5..s6 - 1,3,1,1,1,1,1,3,1,3,1,1,1,1,1,1,1,3,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // s7..s8 -}; - -static inline uint32_t flb_utf8_decode(uint32_t *state, uint32_t *codep, - uint32_t byte) -{ - uint32_t type = utf8d[byte]; - - *codep = (*state != FLB_UTF8_ACCEPT) ? - (byte & 0x3fu) | (*codep << 6) : - (0xff >> type) & (byte); - - *state = utf8d[256 + *state*16 + type]; - return *state; -} - - -static inline void flb_utf8_print(const uint8_t *s) { - uint32_t codepoint; - uint32_t state = 0; - - for (; *s; ++s) - if (!flb_utf8_decode(&state, &codepoint, *s)) { - printf("\\u%04x\n", codepoint); - } - - if (state != FLB_UTF8_ACCEPT) { - printf("The string is not well-formed\n"); - } -} +int flb_utf8_len(const char *s); +uint32_t flb_utf8_decode(uint32_t *state, uint32_t *codep, uint8_t byte); +void flb_utf8_print(char *input); #endif diff --git a/src/flb_utf8.c b/src/flb_utf8.c new file mode 100644 index 00000000000..ba8b4696415 --- /dev/null +++ b/src/flb_utf8.c @@ -0,0 +1,201 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include +#include +#include + +static const char trailing_bytes_for_utf8[256] = { + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, + 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5 +}; + +/* returns length of next utf-8 sequence */ +int flb_utf8_len(const char *s) +{ + return trailing_bytes_for_utf8[(unsigned int)(unsigned char)s[0]] + 1; +} + +#if defined(__GNUC__) || defined(__clang__) +/* + * if we are compiling with GNU or CLang compiler , we have the ranges + * functionality available, so we can tweak our decoder by using a lookup + * table. + * + * Lookup table for byte classification and state transitions: + * + * Format: {initial_state, bitmask, expected_continuation_bytes} + * ASCII: state 0, no continuation bytes + * Start of multi-byte sequence: state X, continuation byte count + * Invalid: reject state + */ +static const uint8_t utf8_lookup[256][3] = { + [0x00 ... 0x7F] = {0, 0x7F, 0}, /* ASCII */ + [0xC0 ... 0xDF] = {1, 0x1F, 1}, /* Start of 2-byte sequence */ + [0xE0 ... 0xEF] = {2, 0x0F, 2}, /* Start of 3-byte sequence */ + [0xF0 ... 0xF7] = {3, 0x07, 3}, /* Start of 4-byte sequence */ + [0x80 ... 0xBF] = {FLB_UTF8_REJECT, 0, 0}, /* Continuation bytes */ + [0xF8 ... 0xFF] = {FLB_UTF8_REJECT, 0, 0}, /* Invalid bytes */ +}; + +uint32_t flb_utf8_decode(uint32_t *state, uint32_t *codep, uint8_t byte) +{ + const uint8_t *entry = utf8_lookup[byte]; + + if (*state == FLB_UTF8_ACCEPT) { + /* starting a new character */ + *state = entry[0]; + if (*state == FLB_UTF8_REJECT) { + /* invalid start byte */ + return FLB_UTF8_REJECT; + } + *codep = byte & entry[1]; + } + else { + /* continuation byte */ + if ((byte & 0xC0) == 0x80) { + *codep = (*codep << 6) | (byte & 0x3F); + /* decrement continuation bytes */ + (*state)--; + } + else { + /* invalid continuation byte */ + *state = FLB_UTF8_REJECT; + return FLB_UTF8_REJECT; + } + } + + /* check if the sequence is complete */ + if (*state == 0) { + if (*codep >= 0xD800 && *codep <= 0xDFFF) { + /* surrogate pair (invalid UTF-8) */ + *state = FLB_UTF8_REJECT; + return FLB_UTF8_REJECT; + } + else if (*codep > 0x10FFFF) { + /* out of range codepoint */ + *state = FLB_UTF8_REJECT; + return FLB_UTF8_REJECT; + } + /* valid and complete sequence */ + return FLB_UTF8_ACCEPT; + } + + /* we are still processing the current sequence */ + return FLB_UTF8_CONTINUE; +} + +#else + +/* fallback decoder: no lookup table */ +uint32_t flb_utf8_decode(uint32_t *state, uint32_t *codep, uint8_t byte) +{ + /* Start of a new character */ + if (*state == 0) { + if (byte <= 0x7F) { + /* ASCII */ + *codep = byte; + return FLB_UTF8_ACCEPT; + } + else if ((byte & 0xE0) == 0xC0) { + /* start of a 2-byte sequence */ + *codep = byte & 0x1F; + *state = 1; + } + else if ((byte & 0xF0) == 0xE0) { + /* start of a 3-byte sequence */ + *codep = byte & 0x0F; + *state = 2; + } + else if ((byte & 0xF8) == 0xF0) { + /* start of a 4-byte sequence */ + *codep = byte & 0x07; + *state = 3; + } + else { + /* invalid first byte */ + *state = FLB_UTF8_REJECT; + return FLB_UTF8_REJECT; + } + } + else { + /* continuation byte */ + if ((byte & 0xC0) == 0x80) { + *codep = (*codep << 6) | (byte & 0x3F); + + /* reduce the expected continuation bytes */ + (*state)--; + } + else { + /* invalid continuation byte */ + *state = FLB_UTF8_REJECT; + return FLB_UTF8_REJECT; + } + } + + if (*state == 0) { + /* sequence complete */ + if (*codep >= 0xD800 && *codep <= 0xDFFF) { + /* invalid surrogate pair */ + *state = FLB_UTF8_REJECT; + return FLB_UTF8_REJECT; + } + else if (*codep > 0x10FFFF) { + /* codepoint is out of range */ + *state = FLB_UTF8_REJECT; + return FLB_UTF8_REJECT; + } + return FLB_UTF8_ACCEPT; + } + + /* we are still processing the current sequence */ + return FLB_UTF8_CONTINUE; +} + +#endif + +void flb_utf8_print(char *input) +{ + int i; + int ret; + int len; + uint32_t state = 0; + uint32_t codepoint = 0; + + len = strlen(input); + for (i = 0; i < len; i++) { + ret = flb_utf8_decode(&state, &codepoint, (uint8_t) input[i]); + if (ret == FLB_UTF8_ACCEPT) { + printf("Valid Codepoint: U+%04X\n", codepoint); + } + else if (ret == FLB_UTF8_REJECT) { + printf("Invalid UTF-8 sequence detected.\n"); + break; + } + } +} From 7b83b826d33c1b213a2795bdadfcef8a7d1e0e40 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Tue, 3 Dec 2024 20:23:48 -0600 Subject: [PATCH 09/23] build: add flb_utf8.c Signed-off-by: Eduardo Silva --- src/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b10a9fcd131..9ba91972ac0 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -12,6 +12,7 @@ set(src flb_env.c flb_file.c flb_uri.c + flb_utf8.c flb_hash_table.c flb_help.c flb_pack.c From 31257d909cf0a31e305f73afefc03858dcb6cc15 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Tue, 3 Dec 2024 20:24:35 -0600 Subject: [PATCH 10/23] utils: use new utf8 decoder API Signed-off-by: Eduardo Silva --- src/flb_utils.c | 83 +++++++++++++++++++++++++++++-------------------- 1 file changed, 50 insertions(+), 33 deletions(-) diff --git a/src/flb_utils.c b/src/flb_utils.c index 33fc32c2a35..b42b1048cc2 100644 --- a/src/flb_utils.c +++ b/src/flb_utils.c @@ -792,8 +792,10 @@ static const struct escape_seq json_escape_table[128] = { int flb_utils_write_str(char *buf, int *off, size_t size, const char *str, size_t str_len) { int i, b, ret, len, hex_bytes, utf_sequence_length, utf_sequence_number; + int processed_bytes = 0; int is_valid, copypos = 0, vlen; - uint32_t codepoint, state = 0; + uint32_t codepoint = 0; + uint32_t state = 0; char tmp[16]; size_t available; uint32_t c; @@ -906,31 +908,40 @@ int flb_utils_write_str(char *buf, int *off, size_t size, const char *str, size_ /* decode UTF-8 sequence */ state = FLB_UTF8_ACCEPT; codepoint = 0; + processed_bytes = 0; for (b = 0; b < hex_bytes; b++) { s = (unsigned char *) &str[i + b]; ret = flb_utf8_decode(&state, &codepoint, *s); - if (ret == 0) { + processed_bytes++; + + if (ret == FLB_UTF8_ACCEPT) { + /* check if all required bytes for the sequence are processed */ + if (processed_bytes == hex_bytes) { + break; + } + } + else if (ret == FLB_UTF8_REJECT) { + flb_warn("[pack] Invalid UTF-8 bytes found, skipping."); break; } } - if (state != FLB_UTF8_ACCEPT) { - flb_warn("[pack] Invalid UTF-8 bytes found, skipping."); - } - else { - + if (state == FLB_UTF8_ACCEPT) { len = snprintf(tmp, sizeof(tmp), "\\u%.4x", codepoint); if (available < len) { - return FLB_FALSE; // Not enough space + return FLB_FALSE; } memcpy(p, tmp, len); p += len; offset += len; available -= len; } + else { + flb_warn("[pack] Invalid UTF-8 bytes found, skipping."); + } - i += hex_bytes; + i += processed_bytes; } /* Handle sequences beyond 0xFFFF */ else if (c > 0xFFFF) { @@ -942,26 +953,25 @@ int flb_utils_write_str(char *buf, int *off, size_t size, const char *str, size_ break; } + state = FLB_UTF8_ACCEPT; + codepoint = 0; is_valid = FLB_TRUE; + + /* Decode the sequence */ for (utf_sequence_number = 0; utf_sequence_number < utf_sequence_length; utf_sequence_number++) { - /* Leading characters must start with bits 11 */ - if (utf_sequence_number == 0 && ((str[i] & 0xC0) != 0xC0)) { - /* Invalid unicode character. replace */ - flb_debug("[pack] unexpected UTF-8 leading byte, " - "substituting character with replacement character"); - tmp[utf_sequence_number] = str[i]; - i++; /* Consume invalid leading byte */ - utf_sequence_length = utf_sequence_number + 1; - is_valid = FLB_FALSE; - break; - } - /* Trailing characters must start with bits 10 */ - else if (utf_sequence_number > 0 && ((str[i] & 0xC0) != 0x80)) { - /* Invalid unicode character. replace */ - flb_debug("[pack] unexpected UTF-8 continuation byte, " - "substituting character with replacement character"); - /* This byte, i, is the start of the next unicode character */ - utf_sequence_length = utf_sequence_number; + ret = flb_utf8_decode(&state, &codepoint, (uint8_t) str[i]); + + if (ret == FLB_UTF8_REJECT) { + if (utf_sequence_number == 0 && ((str[i] & 0xC0) != 0xC0)) { + tmp[utf_sequence_number] = str[i]; + i++; /* Consume invalid leading byte */ + utf_sequence_length = utf_sequence_number + 1; + } + else if (utf_sequence_number > 0 && ((str[i] & 0xC0) != 0x80)) { + utf_sequence_length = utf_sequence_number; + } + flb_debug("[pack] invalid UTF-8 sequence detected, replacing with substitution character"); + //codepoint = 0xFFFD; // Replacement character is_valid = FLB_FALSE; break; } @@ -970,17 +980,23 @@ int flb_utils_write_str(char *buf, int *off, size_t size, const char *str, size_ ++i; } - --i; + //--i; if (is_valid) { if (available < utf_sequence_length) { - return FLB_FALSE; // Not enough space + /* not enough space */ + return FLB_FALSE; } - encoded_to_buf(p, tmp, utf_sequence_length); - p += utf_sequence_length; + len = snprintf(tmp, sizeof(tmp), "\\u%.4x", codepoint); + if (available < len) { + /* not enough space */ + return FLB_FALSE; + } + memcpy(p, tmp, len); + p += len; offset += utf_sequence_length; - available -= utf_sequence_length; + available -= len;//utf_sequence_length; } else { if (available < utf_sequence_length * 3) { @@ -1037,8 +1053,9 @@ int flb_utils_write_str(char *buf, int *off, size_t size, const char *str, size_ available -= 3; } } + } - else { + else { if (available < 1) { /* no space for a single byte */ return FLB_FALSE; From 64c307f99c49a995f8245494b21d1252e45908b8 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Tue, 3 Dec 2024 20:27:57 -0600 Subject: [PATCH 11/23] tests: internal: utils: adjusts expected utf8 encoded bytes Signed-off-by: Eduardo Silva --- tests/internal/utils.c | 53 ++++++++++++++++++++++++++++++------------ 1 file changed, 38 insertions(+), 15 deletions(-) diff --git a/tests/internal/utils.c b/tests/internal/utils.c index 68fdc6900d3..80389e6b208 100644 --- a/tests/internal/utils.c +++ b/tests/internal/utils.c @@ -127,7 +127,8 @@ static void write_str_test_cases(struct write_str_case *cases) { } /* test case loop for flb_utils_write_str */ -static void write_str_test_cases_w_buf_size(struct write_str_case *cases, int buf_size) { +static void write_str_test_cases_w_buf_size(struct write_str_case *cases, int buf_size) +{ char *buf = flb_calloc(buf_size + 1, sizeof(char)); int size = buf_size + 1; int off; @@ -170,11 +171,13 @@ static void write_str_test_cases_w_buf_size(struct write_str_case *cases, int bu void test_write_str() { - char buf[10]; - char japanese_a[4] = {0xe3, 0x81, 0x82}; - int size = sizeof(buf); int off; int ret; + char buf[10] = {0}; + int size = sizeof(buf); + + /* escaped Unicode representation of あ */ + char jp_expected_output[] = "\\u3042"; off = 0; ret = flb_utils_write_str(buf, &off, size, "a", 1); @@ -189,15 +192,16 @@ void test_write_str() off = 0; ret = flb_utils_write_str(buf, &off, size, "\xe3\x81\x82", 3); TEST_CHECK(ret == FLB_TRUE); - TEST_CHECK(memcmp(buf, japanese_a, off) == 0); + TEST_CHECK(memcmp(buf, jp_expected_output, off) == 0); - // Truncated bytes + /* Truncated bytes: 'buf' should not be touched and off == 0 */ off = 0; ret = flb_utils_write_str(buf, &off, size, "\xe3\x81\x82\xe3", 1); TEST_CHECK(ret == FLB_TRUE); - TEST_CHECK(memcmp(buf, japanese_a, off) == 0); + TEST_CHECK(off == 0); + TEST_CHECK(memcmp(buf, jp_expected_output, off) == 0); - // Error: buffer too small + /* Error: buffer too small */ off = 0; ret = flb_utils_write_str(buf, &off, size, "aaaaaaaaaaa", 11); TEST_CHECK(ret == FLB_FALSE); @@ -238,7 +242,7 @@ void test_write_str_invalid_leading_byte() */ { "\x00\x01\xe3\x81\x82""abc", 8, /* note that 0x01 is an invalid byte */ - "\\u0000\\u0001""\xe3\x81\x82""abc", /* escape hex */ + "\\u0000\\u0001\\u3042""abc", /* escape hex */ FLB_TRUE }, /* @@ -252,7 +256,7 @@ void test_write_str_invalid_leading_byte() "\xee\x83\xb3" /* f3 fragment */ /* replace invalid unicode */ "\xee\x82\x81" /* 81 fragment */ "\xee\x82\x81" /* 81 fragment */ - "\xe3\x81\x82""abc", /* valid unicode */ + "\\u3042""abc", /* valid unicode */ FLB_TRUE }, /* @@ -263,7 +267,25 @@ void test_write_str_invalid_leading_byte() "\xf3\x81\x01\xe3\x81\x82""abc", 9, /* note that 0x01 is an invalid byte */ "\xee\x83\xb3" /* f3 fragment */ /* replace invalid unicode */ "\xee\x82\x81" /* 81 fragment */ - "\\u0001""\xe3\x81\x82""abc", + "\\u0001""\\u3042""abc", + FLB_TRUE + }, + { 0 } + }; + + write_str_test_cases(cases); +} + +void test_write_str_special_bytes() +{ + + struct write_str_case cases[] = { + /* + * Escaped leading hex (two hex, one valid unicode) + */ + { + "你好世界", 12, + "\\u4f60\\u597d\\u4e16\\u754c", FLB_TRUE }, { 0 } @@ -342,10 +364,10 @@ void test_write_str_buffer_overrun() FLB_FALSE }, { - "\"" + "a" "\xe3\x81\x82", 4, /* valid unicode */ - "\\\"""\xe3\x81\x82", /* just enough space for valid unicode */ - FLB_TRUE + "a", /* just enough space for valid ascii */ + FLB_FALSE /* no space for \u3042 */ }, { "\x81" @@ -666,7 +688,7 @@ struct size_to_bytes_check size_to_bytes_checks[] = { {"9223372036.78G", -1}, }; -void test_size_to_bytes() +void test_size_to_bytes() { int i; int size; @@ -686,6 +708,7 @@ TEST_LIST = { /* JSON maps iteration */ { "url_split", test_url_split }, { "write_str", test_write_str }, + { "write_str_special_bytes", test_write_str_special_bytes }, { "test_write_str_invalid_trailing_bytes", test_write_str_invalid_trailing_bytes }, { "test_write_str_invalid_leading_byte", test_write_str_invalid_leading_byte }, { "test_write_str_edge_cases", test_write_str_edge_cases }, From 9643f53c6eb387c6e82a5e2a4b3197b9f69efaf6 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Thu, 5 Dec 2024 17:39:33 -0600 Subject: [PATCH 12/23] tests: internal: data: pack: generators: fix json encoding ascii utf8 Signed-off-by: Eduardo Silva --- tests/internal/data/pack/mixed.py | 26 +++++++------ tests/internal/data/pack/mixed_002.json | 2 +- tests/internal/data/pack/mixed_002.mp | Bin 37 -> 36 bytes tests/internal/data/pack/mixed_003.json | 2 +- tests/internal/data/pack/utf8_bell.json | 2 +- tests/internal/data/pack/utf8_copyright.json | 2 +- tests/internal/data/pack/utf8_gen.py | 37 +++++++++++-------- tests/internal/data/pack/utf8_hokke.json | 2 +- tests/internal/data/pack/utf8_relaxed.json | 2 +- 9 files changed, 42 insertions(+), 33 deletions(-) diff --git a/tests/internal/data/pack/mixed.py b/tests/internal/data/pack/mixed.py index 7ad0a6149b6..c8859231d98 100644 --- a/tests/internal/data/pack/mixed.py +++ b/tests/internal/data/pack/mixed.py @@ -6,22 +6,24 @@ import json import msgpack + def gen_json(f): - raw = open(f, 'r') - data = raw.read() - raw.close() + # Open the input file in text mode with UTF-8 encoding + with open(f, 'r', encoding='utf-8') as raw: + data = raw.read() - out_mp = f[:-4] + ".mp" - out_json = f[:-4] + ".json" + # Define output filenames + base_name = os.path.splitext(f)[0] + out_mp = base_name + ".mp" + out_json = base_name + ".json" - # Write messagepack - fmp = open(out_mp, 'w') - fmp.write(msgpack.packb(data)) - fmp.close() + # Write MessagePack-encoded data in binary mode + with open(out_mp, 'wb') as fmp: + fmp.write(msgpack.packb(data)) - fjson = open(out_json, 'w') - fjson.write(json.dumps(data)) - fjson.close() + # Write JSON-encoded data in text mode + with open(out_json, 'w', encoding='utf-8') as fjson: + fjson.write(json.dumps(data)) for fn in os.listdir('.'): if not os.path.isfile(fn): diff --git a/tests/internal/data/pack/mixed_002.json b/tests/internal/data/pack/mixed_002.json index e32314b0aa9..d1afe50ace3 100644 --- a/tests/internal/data/pack/mixed_002.json +++ b/tests/internal/data/pack/mixed_002.json @@ -1 +1 @@ -"mixed_002 =>\n\n áéíóú\n\n\n'\n\\t\n" \ No newline at end of file +"mixed_002 =>\n\n \u00e1\u00e9\u00ed\u00f3\u00fa\n\n\n'\n\\t\n" \ No newline at end of file diff --git a/tests/internal/data/pack/mixed_002.mp b/tests/internal/data/pack/mixed_002.mp index 1bf975535a1c3e87759e44027bf688f83f835be2..4938203d59a1b150b84400a5fa60eb0dcbff5798 100644 GIT binary patch delta 7 OcmY#YVZ1q!Nf`hKB>~z1 delta 8 PcmY#UWxB;Mkx>}{2vGss diff --git a/tests/internal/data/pack/mixed_003.json b/tests/internal/data/pack/mixed_003.json index 167c89b8a06..126945cb3c7 100644 --- a/tests/internal/data/pack/mixed_003.json +++ b/tests/internal/data/pack/mixed_003.json @@ -1 +1 @@ -"á\n" \ No newline at end of file +"\u00e1\n" \ No newline at end of file diff --git a/tests/internal/data/pack/utf8_bell.json b/tests/internal/data/pack/utf8_bell.json index ced4da0cfe5..d0730c4a7ce 100644 --- a/tests/internal/data/pack/utf8_bell.json +++ b/tests/internal/data/pack/utf8_bell.json @@ -1 +1 @@ -"🔔" \ No newline at end of file +"\ud83d\udd14" \ No newline at end of file diff --git a/tests/internal/data/pack/utf8_copyright.json b/tests/internal/data/pack/utf8_copyright.json index 4d52a66f3d0..92d937cf7bd 100644 --- a/tests/internal/data/pack/utf8_copyright.json +++ b/tests/internal/data/pack/utf8_copyright.json @@ -1 +1 @@ -"©" \ No newline at end of file +"\u00a9" \ No newline at end of file diff --git a/tests/internal/data/pack/utf8_gen.py b/tests/internal/data/pack/utf8_gen.py index 606e8cc2d31..9f1eef2b9c2 100644 --- a/tests/internal/data/pack/utf8_gen.py +++ b/tests/internal/data/pack/utf8_gen.py @@ -6,27 +6,34 @@ import msgpack def gen_json(f): + print(f) - print f - - with io.open(f, 'rb') as raw: + with open(f, 'rb') as raw: data = raw.read() - out_mp = f[:-4] + ".mp" - out_json = f[:-4] + ".json" + out_mp = f"{os.path.splitext(f)[0]}.mp" + out_json = f"{os.path.splitext(f)[0]}.json" + + # Decode input bytes to a string + try: + decoded_data = data.decode('utf-8') + except UnicodeDecodeError as e: + print(f"Error: Unable to decode file {f} as UTF-8: {e}") + return # Write messagepack - fmp = open(out_mp, 'w') - fmp.write(msgpack.packb(data)) - fmp.close() + with open(out_mp, 'wb') as fmp: + fmp.write(msgpack.packb(decoded_data)) - fjson = open(out_json, 'w') - fjson.write(json.dumps(data).encode('utf8')) - fjson.close() + # Write JSON with properly encoded Unicode escape sequences + with open(out_json, 'w', encoding='utf-8') as fjson: + # Use json.dumps with ensure_ascii=True for \uXXXX escape sequences + escaped_data = json.dumps(decoded_data, ensure_ascii=True) + fjson.write(escaped_data) for fn in os.listdir('.'): - if not os.path.isfile(fn): - continue + if not os.path.isfile(fn): + continue - if fn.startswith('utf8_') and fn.endswith('.txt'): - gen_json(fn) + if fn.startswith('utf8_') and fn.endswith('.txt'): + gen_json(fn) diff --git a/tests/internal/data/pack/utf8_hokke.json b/tests/internal/data/pack/utf8_hokke.json index d93624bf21f..37f460c4b35 100644 --- a/tests/internal/data/pack/utf8_hokke.json +++ b/tests/internal/data/pack/utf8_hokke.json @@ -1 +1 @@ -"𩸽" \ No newline at end of file +"\ud867\ude3d" \ No newline at end of file diff --git a/tests/internal/data/pack/utf8_relaxed.json b/tests/internal/data/pack/utf8_relaxed.json index 4526bf40faf..2402faf9df4 100644 --- a/tests/internal/data/pack/utf8_relaxed.json +++ b/tests/internal/data/pack/utf8_relaxed.json @@ -1 +1 @@ -"☺" \ No newline at end of file +"\u263a" \ No newline at end of file From bcac86c13eb81c2ad6891f4d0001f23a17732cdc Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Thu, 5 Dec 2024 17:40:41 -0600 Subject: [PATCH 13/23] utf8: remove lookup table Signed-off-by: Eduardo Silva --- src/flb_utf8.c | 74 -------------------------------------------------- 1 file changed, 74 deletions(-) diff --git a/src/flb_utf8.c b/src/flb_utf8.c index ba8b4696415..8e6ee71c68d 100644 --- a/src/flb_utf8.c +++ b/src/flb_utf8.c @@ -41,78 +41,6 @@ int flb_utf8_len(const char *s) return trailing_bytes_for_utf8[(unsigned int)(unsigned char)s[0]] + 1; } -#if defined(__GNUC__) || defined(__clang__) -/* - * if we are compiling with GNU or CLang compiler , we have the ranges - * functionality available, so we can tweak our decoder by using a lookup - * table. - * - * Lookup table for byte classification and state transitions: - * - * Format: {initial_state, bitmask, expected_continuation_bytes} - * ASCII: state 0, no continuation bytes - * Start of multi-byte sequence: state X, continuation byte count - * Invalid: reject state - */ -static const uint8_t utf8_lookup[256][3] = { - [0x00 ... 0x7F] = {0, 0x7F, 0}, /* ASCII */ - [0xC0 ... 0xDF] = {1, 0x1F, 1}, /* Start of 2-byte sequence */ - [0xE0 ... 0xEF] = {2, 0x0F, 2}, /* Start of 3-byte sequence */ - [0xF0 ... 0xF7] = {3, 0x07, 3}, /* Start of 4-byte sequence */ - [0x80 ... 0xBF] = {FLB_UTF8_REJECT, 0, 0}, /* Continuation bytes */ - [0xF8 ... 0xFF] = {FLB_UTF8_REJECT, 0, 0}, /* Invalid bytes */ -}; - -uint32_t flb_utf8_decode(uint32_t *state, uint32_t *codep, uint8_t byte) -{ - const uint8_t *entry = utf8_lookup[byte]; - - if (*state == FLB_UTF8_ACCEPT) { - /* starting a new character */ - *state = entry[0]; - if (*state == FLB_UTF8_REJECT) { - /* invalid start byte */ - return FLB_UTF8_REJECT; - } - *codep = byte & entry[1]; - } - else { - /* continuation byte */ - if ((byte & 0xC0) == 0x80) { - *codep = (*codep << 6) | (byte & 0x3F); - /* decrement continuation bytes */ - (*state)--; - } - else { - /* invalid continuation byte */ - *state = FLB_UTF8_REJECT; - return FLB_UTF8_REJECT; - } - } - - /* check if the sequence is complete */ - if (*state == 0) { - if (*codep >= 0xD800 && *codep <= 0xDFFF) { - /* surrogate pair (invalid UTF-8) */ - *state = FLB_UTF8_REJECT; - return FLB_UTF8_REJECT; - } - else if (*codep > 0x10FFFF) { - /* out of range codepoint */ - *state = FLB_UTF8_REJECT; - return FLB_UTF8_REJECT; - } - /* valid and complete sequence */ - return FLB_UTF8_ACCEPT; - } - - /* we are still processing the current sequence */ - return FLB_UTF8_CONTINUE; -} - -#else - -/* fallback decoder: no lookup table */ uint32_t flb_utf8_decode(uint32_t *state, uint32_t *codep, uint8_t byte) { /* Start of a new character */ @@ -177,8 +105,6 @@ uint32_t flb_utf8_decode(uint32_t *state, uint32_t *codep, uint8_t byte) return FLB_UTF8_CONTINUE; } -#endif - void flb_utf8_print(char *input) { int i; From 13206cfbdc56d8fd3737ee0e542f6e8dc2bd0c4b Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Thu, 5 Dec 2024 17:42:20 -0600 Subject: [PATCH 14/23] utils: add utf8 encoding for codepoints beyond BMP Signed-off-by: Eduardo Silva --- src/flb_utils.c | 47 +++++++++++++++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/src/flb_utils.c b/src/flb_utils.c index b42b1048cc2..55b5bf6cae8 100644 --- a/src/flb_utils.c +++ b/src/flb_utils.c @@ -788,20 +788,23 @@ static const struct escape_seq json_escape_table[128] = { * to escape special characters and convert utf-8 byte characters to string * representation. */ - int flb_utils_write_str(char *buf, int *off, size_t size, const char *str, size_t str_len) { int i, b, ret, len, hex_bytes, utf_sequence_length, utf_sequence_number; int processed_bytes = 0; int is_valid, copypos = 0, vlen; + uint32_t c; uint32_t codepoint = 0; uint32_t state = 0; - char tmp[16]; size_t available; - uint32_t c; - char *p; uint8_t *s; off_t offset = 0; + char tmp[16]; + char *p; + + /* to encode codepoints > 0xFFFF */ + uint16_t high; + uint16_t low; available = size - *off; @@ -959,19 +962,21 @@ int flb_utils_write_str(char *buf, int *off, size_t size, const char *str, size_ /* Decode the sequence */ for (utf_sequence_number = 0; utf_sequence_number < utf_sequence_length; utf_sequence_number++) { - ret = flb_utf8_decode(&state, &codepoint, (uint8_t) str[i]); + ret = flb_utf8_decode(&state, &codepoint, (uint8_t) str[i]); if (ret == FLB_UTF8_REJECT) { - if (utf_sequence_number == 0 && ((str[i] & 0xC0) != 0xC0)) { + /* Handle invalid leading byte */ + if (utf_sequence_number == 0) { + flb_debug("[pack] unexpected UTF-8 leading byte, substituting character"); tmp[utf_sequence_number] = str[i]; - i++; /* Consume invalid leading byte */ - utf_sequence_length = utf_sequence_number + 1; + utf_sequence_length = utf_sequence_number + 1; /* Process only this invalid byte */ + i++; /* Consume invalid byte */ } - else if (utf_sequence_number > 0 && ((str[i] & 0xC0) != 0x80)) { - utf_sequence_length = utf_sequence_number; + /* Handle invalid continuation byte */ + else { + flb_debug("[pack] unexpected UTF-8 continuation byte, substituting character"); + utf_sequence_length = utf_sequence_number; /* Adjust length */ } - flb_debug("[pack] invalid UTF-8 sequence detected, replacing with substitution character"); - //codepoint = 0xFFFD; // Replacement character is_valid = FLB_FALSE; break; } @@ -980,7 +985,7 @@ int flb_utils_write_str(char *buf, int *off, size_t size, const char *str, size_ ++i; } - //--i; + --i; if (is_valid) { if (available < utf_sequence_length) { @@ -988,15 +993,25 @@ int flb_utils_write_str(char *buf, int *off, size_t size, const char *str, size_ return FLB_FALSE; } - len = snprintf(tmp, sizeof(tmp), "\\u%.4x", codepoint); + /* Handle codepoints beyond BMP (requires surrogate pairs in UTF-16) */ + if (codepoint > 0xFFFF) { + high = 0xD800 + ((codepoint - 0x10000) >> 10); + low = 0xDC00 + ((codepoint - 0x10000) & 0x3FF); + + len = snprintf(tmp, sizeof(tmp), "\\u%.4x\\u%.4x", high, low); + } + else { + len = snprintf(tmp, sizeof(tmp), "\\u%.4x", codepoint); + } + if (available < len) { /* not enough space */ return FLB_FALSE; } memcpy(p, tmp, len); p += len; - offset += utf_sequence_length; - available -= len;//utf_sequence_length; + offset += len; + available -= len; } else { if (available < utf_sequence_length * 3) { From 1b6c9a187b0e3e845e800982d8b6d2dfee0e4aed Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Thu, 5 Dec 2024 17:49:26 -0600 Subject: [PATCH 15/23] sds: use new utf8 decoder for sds_cat_utf8 Signed-off-by: Eduardo Silva --- src/flb_sds.c | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/src/flb_sds.c b/src/flb_sds.c index 759be89d9ab..1d8263b3432 100644 --- a/src/flb_sds.c +++ b/src/flb_sds.c @@ -28,6 +28,7 @@ #include #include #include +#include #include #include @@ -279,13 +280,15 @@ flb_sds_t flb_sds_copy(flb_sds_t s, const char *str, int len) return s; } -flb_sds_t flb_sds_cat_utf8 (flb_sds_t *sds, const char *str, int str_len) +flb_sds_t flb_sds_cat_utf8(flb_sds_t *sds, const char *str, int str_len) { static const char int2hex[] = "0123456789abcdef"; int i; int b; int ret; int hex_bytes; + int offset; + size_t size; uint32_t cp; uint32_t state = 0; unsigned char c; @@ -297,6 +300,7 @@ flb_sds_t flb_sds_cat_utf8 (flb_sds_t *sds, const char *str, int str_len) s = *sds; head = FLB_SDS_HEADER(s); + /* make sure we have at least str_len extra bytes available */ if (flb_sds_avail(s) <= str_len) { tmp = flb_sds_increase(s, str_len); if (tmp == NULL) { @@ -306,6 +310,30 @@ flb_sds_t flb_sds_cat_utf8 (flb_sds_t *sds, const char *str, int str_len) head = FLB_SDS_HEADER(s); } + while (1) { + offset = head->len; + ret = flb_utils_write_str(s, &offset, flb_sds_alloc(s), str, str_len); + if (ret == FLB_FALSE) { + /* realloc */ + size = flb_sds_alloc(s) * 2; + tmp = flb_sds_increase(s, size); + if (tmp == NULL) { + return NULL; + } + *sds = s = tmp; + head = FLB_SDS_HEADER(s); + } + else { + break; + } + } + + flb_sds_len_set(s, offset); + s[head->len] = '\0'; + return s; + + + for (i = 0; i < str_len; i++) { if (flb_sds_avail(s) < 8) { tmp = flb_sds_increase(s, 8); From b030346a638bad63d22a2f0dcd450c8b869cd046 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Thu, 5 Dec 2024 17:50:20 -0600 Subject: [PATCH 16/23] tests: internal: sds: adjust unit test for utf8 Signed-off-by: Eduardo Silva --- tests/internal/sds.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/internal/sds.c b/tests/internal/sds.c index 30dc166ddb1..d9d5db1027f 100644 --- a/tests/internal/sds.c +++ b/tests/internal/sds.c @@ -44,7 +44,7 @@ static void test_sds_printf_7143_off_by_1() flb_sds_t test; flb_sds_t test2; int len; - + /* 66 char final string, not impacted by bug */ test = flb_sds_create_size(64); TEST_CHECK(test != NULL); @@ -69,13 +69,20 @@ static void test_sds_printf_7143_off_by_1() static void test_sds_cat_utf8() { + int ret; flb_sds_t s; char *utf8_str = "\xe8\x9f\xb9\xf0\x9f\xa6\x80"; + char *expected = "\\u87f9\\ud83e\\udd80"; s = flb_sds_create(""); flb_sds_cat_utf8(&s, utf8_str, strlen(utf8_str)); - TEST_CHECK(strcmp(s, "\\u87f9\\u1f980") == 0); + ret = strcmp(s, expected); + TEST_CHECK(ret == 0); + if (ret != 0) { + printf("Expected: %s\n", expected); + printf("Received: %s\n", s); + } flb_sds_destroy(s); } From 8aba382729f6a2eb5d87fa2aef3f839cdc32099f Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Thu, 5 Dec 2024 17:50:42 -0600 Subject: [PATCH 17/23] tests: internal: utils: add missing test Signed-off-by: Eduardo Silva --- tests/internal/utils.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/internal/utils.c b/tests/internal/utils.c index 80389e6b208..1222258f8c7 100644 --- a/tests/internal/utils.c +++ b/tests/internal/utils.c @@ -278,7 +278,6 @@ void test_write_str_invalid_leading_byte() void test_write_str_special_bytes() { - struct write_str_case cases[] = { /* * Escaped leading hex (two hex, one valid unicode) @@ -288,6 +287,11 @@ void test_write_str_special_bytes() "\\u4f60\\u597d\\u4e16\\u754c", FLB_TRUE }, + { + "\xC3\xA1\x0A", 3, /* UTF-8 encoding of á and newline */ + "\\u00e1\\n", /* Expected escaped output */ + FLB_TRUE + }, { 0 } }; From 362ca1f3e907d975d14073cc7d98c6fe68548ae6 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Thu, 12 Dec 2024 10:36:03 -0600 Subject: [PATCH 18/23] sds: utf8: remove dead code Signed-off-by: Eduardo Silva --- src/flb_sds.c | 100 -------------------------------------------------- 1 file changed, 100 deletions(-) diff --git a/src/flb_sds.c b/src/flb_sds.c index 1d8263b3432..4ac36ad22e8 100644 --- a/src/flb_sds.c +++ b/src/flb_sds.c @@ -331,106 +331,6 @@ flb_sds_t flb_sds_cat_utf8(flb_sds_t *sds, const char *str, int str_len) flb_sds_len_set(s, offset); s[head->len] = '\0'; return s; - - - - for (i = 0; i < str_len; i++) { - if (flb_sds_avail(s) < 8) { - tmp = flb_sds_increase(s, 8); - if (tmp == NULL) { - return NULL; - } - *sds = s = tmp; - head = FLB_SDS_HEADER(s); - } - - c = (unsigned char)str[i]; - if (c == '\\' || c == '"') { - s[head->len++] = '\\'; - s[head->len++] = c; - } - else if (c >= '\b' && c <= '\r') { - s[head->len++] = '\\'; - switch (c) { - case '\n': - s[head->len++] = 'n'; - break; - case '\t': - s[head->len++] = 't'; - break; - case '\b': - s[head->len++] = 'b'; - break; - case '\f': - s[head->len++] = 'f'; - break; - case '\r': - s[head->len++] = 'r'; - break; - case '\v': - s[head->len++] = 'u'; - s[head->len++] = '0'; - s[head->len++] = '0'; - s[head->len++] = '0'; - s[head->len++] = 'b'; - break; - } - } - else if (c < 32 || c == 0x7f) { - s[head->len++] = '\\'; - s[head->len++] = 'u'; - s[head->len++] = '0'; - s[head->len++] = '0'; - s[head->len++] = int2hex[ (unsigned char) ((c & 0xf0) >> 4)]; - s[head->len++] = int2hex[ (unsigned char) (c & 0x0f)]; - } - else if (c >= 0x80) { - hex_bytes = flb_utf8_len(str + i); - state = FLB_UTF8_ACCEPT; - cp = 0; - for (b = 0; b < hex_bytes; b++) { - p = (const unsigned char *) str + i + b; - if (p >= (unsigned char *) (str + str_len)) { - break; - } - ret = flb_utf8_decode(&state, &cp, *p); - if (ret == 0) { - break; - } - } - - if (state != FLB_UTF8_ACCEPT) { - /* Invalid UTF-8 hex, just skip utf-8 bytes */ - flb_warn("[pack] invalid UTF-8 bytes, skipping"); - break; - } - - s[head->len++] = '\\'; - s[head->len++] = 'u'; - if (cp > 0xFFFF) { - c = (unsigned char) ((cp & 0xf00000) >> 20); - if (c > 0) { - s[head->len++] = int2hex[c]; - } - c = (unsigned char) ((cp & 0x0f0000) >> 16); - if (c > 0) { - s[head->len++] = int2hex[c]; - } - } - s[head->len++] = int2hex[ (unsigned char) ((cp & 0xf000) >> 12)]; - s[head->len++] = int2hex[ (unsigned char) ((cp & 0x0f00) >> 8)]; - s[head->len++] = int2hex[ (unsigned char) ((cp & 0xf0) >> 4)]; - s[head->len++] = int2hex[ (unsigned char) (cp & 0x0f)]; - i += (hex_bytes - 1); - } - else { - s[head->len++] = c; - } - } - - s[head->len] = '\0'; - - return s; } flb_sds_t flb_sds_printf(flb_sds_t *sds, const char *fmt, ...) From 8916ac77ab93acc188873388ee1fd5f238077f1f Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 13 Dec 2024 22:59:27 -0600 Subject: [PATCH 19/23] out_opentelemetry: remove strict check on severityText value Signed-off-by: Eduardo Silva --- .../out_opentelemetry/opentelemetry_logs.c | 33 ++----------------- 1 file changed, 3 insertions(+), 30 deletions(-) diff --git a/plugins/out_opentelemetry/opentelemetry_logs.c b/plugins/out_opentelemetry/opentelemetry_logs.c index b908cf7cb3e..83464f2a392 100644 --- a/plugins/out_opentelemetry/opentelemetry_logs.c +++ b/plugins/out_opentelemetry/opentelemetry_logs.c @@ -27,8 +27,6 @@ #include #include - -//#include #include #include "opentelemetry.h" @@ -81,26 +79,6 @@ static int hex_to_id(char *str, int len, unsigned char *out_buf, int out_size) return 0; } -/* https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-severitynumber */ -static int is_valid_severity_text(const char *str, size_t str_len) -{ - if (str_len == 5) { - if (strncmp("TRACE", str, 5) == 0 || - strncmp("DEBUG", str, 5) == 0 || - strncmp("ERROR", str, 5) == 0 || - strncmp("FATAL", str, 5) == 0) { - return FLB_TRUE; - } - } - else if (str_len == 4) { - if (strncmp("INFO", str, 4) == 0|| - strncmp("WARN", str, 4) == 0) { - return FLB_TRUE; - } - } - return FLB_FALSE; -} - /* https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-severitynumber */ static int is_valid_severity_number(uint64_t val) { @@ -110,7 +88,6 @@ static int is_valid_severity_number(uint64_t val) return FLB_FALSE; } - /* * From a group record, extract it metadata and validate if it has a valid OTLP schema and check that * resource_id is set. On success it returns the resource_id, otherwise it returns -1. @@ -495,9 +472,7 @@ static int append_v1_logs_metadata_and_fields(struct opentelemetry_context *ctx, /* SeverityText */ ra_val = flb_ra_get_value_object(ctx->ra_log_meta_otlp_severity_text, *event->metadata); if (ra_val != NULL) { - if (ra_val->o.type == MSGPACK_OBJECT_STR && - is_valid_severity_text(ra_val->o.via.str.ptr, ra_val->o.via.str.size)) { - + if (ra_val->o.type == MSGPACK_OBJECT_STR) { log_record->severity_text = flb_calloc(1, ra_val->o.via.str.size + 1); if (log_record->severity_text) { strncpy(log_record->severity_text, ra_val->o.via.str.ptr, ra_val->o.via.str.size); @@ -510,8 +485,7 @@ static int append_v1_logs_metadata_and_fields(struct opentelemetry_context *ctx, if (!severity_text_set && ctx->ra_severity_text_metadata) { ra_val = flb_ra_get_value_object(ctx->ra_severity_text_metadata, *event->metadata); if (ra_val != NULL) { - if (ra_val->o.type == MSGPACK_OBJECT_STR && - is_valid_severity_text(ra_val->o.via.str.ptr, ra_val->o.via.str.size)) { + if (ra_val->o.type == MSGPACK_OBJECT_STR) { log_record->severity_text = flb_calloc(1, ra_val->o.via.str.size + 1); if (log_record->severity_text) { strncpy(log_record->severity_text, ra_val->o.via.str.ptr, ra_val->o.via.str.size); @@ -525,8 +499,7 @@ static int append_v1_logs_metadata_and_fields(struct opentelemetry_context *ctx, if (!severity_text_set && ctx->ra_severity_text_message) { ra_val = flb_ra_get_value_object(ctx->ra_severity_text_message, *event->body); if (ra_val != NULL) { - if (ra_val->o.type == MSGPACK_OBJECT_STR && - is_valid_severity_text(ra_val->o.via.str.ptr, ra_val->o.via.str.size)) { + if (ra_val->o.type == MSGPACK_OBJECT_STR) { log_record->severity_text = flb_calloc(1, ra_val->o.via.str.size + 1); if (log_record->severity_text) { strncpy(log_record->severity_text, ra_val->o.via.str.ptr, ra_val->o.via.str.size); From f73949c1322205d85ed37f7c8faa3d870af10835 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 13 Dec 2024 22:59:49 -0600 Subject: [PATCH 20/23] in_opentelemetry: logs: add support for metadata in JSON payload This change adds support to parse and register the resources and scope metadata coming from Logs which are ingested from a JSON payload. Signed-off-by: Eduardo Silva --- plugins/in_opentelemetry/opentelemetry_prot.c | 232 +++++++++++++++--- 1 file changed, 203 insertions(+), 29 deletions(-) diff --git a/plugins/in_opentelemetry/opentelemetry_prot.c b/plugins/in_opentelemetry/opentelemetry_prot.c index 7b89902bf29..acd1e8c6949 100644 --- a/plugins/in_opentelemetry/opentelemetry_prot.c +++ b/plugins/in_opentelemetry/opentelemetry_prot.c @@ -611,7 +611,7 @@ static int binary_payload_to_msgpack(struct flb_opentelemetry *ctx, msgpack_pack_uint64(&mp_pck, resource->dropped_attributes_count); } - flb_mp_map_header_end(&mh_tmp); + if (resource_log->schema_url) { flb_mp_map_header_append(&mh); @@ -781,13 +781,23 @@ static int find_map_entry_by_key(msgpack_object_map *map, size_t match_index, int case_insensitive) { - size_t match_count; int result; int index; + int key_len; + size_t match_count; + + if (!key) { + return -1; + } + key_len = strlen(key); match_count = 0; for (index = 0 ; index < (int) map->size ; index++) { + if (key_len != map->ptr[index].key.via.str.size) { + continue; + } + if (map->ptr[index].key.type == MSGPACK_OBJECT_STR) { if (case_insensitive) { result = strncasecmp(map->ptr[index].key.via.str.ptr, @@ -1229,6 +1239,8 @@ static int process_json_payload_log_records_entry( int body_type; struct flb_time timestamp; int result; + msgpack_object *severity_number = NULL; + msgpack_object *severity_text = NULL; if (log_records_object->type != MSGPACK_OBJECT_MAP) { flb_plg_error(ctx->ins, "unexpected logRecords entry type"); @@ -1288,12 +1300,28 @@ static int process_json_payload_log_records_entry( flb_time_from_uint64(×tamp, timestamp_uint64); } + /* severityNumber */ + result = find_map_entry_by_key(log_records_entry, "severityNumber", 0, FLB_TRUE); + if (result == -1) { + result = find_map_entry_by_key(log_records_entry, "severity_number", 0, FLB_TRUE); + } + if (result >= 0) { + severity_number = &log_records_entry->ptr[result].val; + } + + /* severityText */ + result = find_map_entry_by_key(log_records_entry, "severityText", 0, FLB_TRUE); + if (result == -1) { + result = find_map_entry_by_key(log_records_entry, "severity_text", 0, FLB_TRUE); + } + if (result >= 0) { + severity_text = &log_records_entry->ptr[result].val; + } + result = find_map_entry_by_key(log_records_entry, "attributes", 0, FLB_TRUE); - if (result == -1) { flb_plg_debug(ctx->ins, "attributes missing"); - metadata_object = NULL; } else { @@ -1329,15 +1357,33 @@ static int process_json_payload_log_records_entry( result = flb_log_event_encoder_set_timestamp(encoder, ×tamp); } - if (result == FLB_EVENT_ENCODER_SUCCESS && - metadata_object != NULL) { - flb_log_event_encoder_dynamic_field_reset(&encoder->metadata); + flb_log_event_encoder_dynamic_field_reset(&encoder->metadata); + result = flb_log_event_encoder_begin_map(encoder, FLB_LOG_EVENT_METADATA); + if (result == FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_append_string(encoder, FLB_LOG_EVENT_METADATA, ctx->logs_metadata_key, flb_sds_len(ctx->logs_metadata_key)); + flb_log_event_encoder_begin_map(encoder, FLB_LOG_EVENT_METADATA); + + if (severity_number != NULL) { + flb_log_event_encoder_append_metadata_values(encoder, + FLB_LOG_EVENT_STRING_VALUE("severity_number", 15), + FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(severity_number)); + } + + if (severity_text != NULL && severity_text->type == MSGPACK_OBJECT_STR) { + flb_log_event_encoder_append_metadata_values(encoder, + FLB_LOG_EVENT_STRING_VALUE("severity_text", 13), + FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(severity_text)); + } + + if (metadata_object != NULL) { + flb_log_event_encoder_append_string(encoder, FLB_LOG_EVENT_METADATA, "attributes", 10); + result = json_payload_append_converted_kvlist(encoder, FLB_LOG_EVENT_METADATA, metadata_object); + } + + flb_log_event_encoder_commit_map(encoder, FLB_LOG_EVENT_METADATA); - result = json_payload_append_converted_kvlist( - encoder, - FLB_LOG_EVENT_METADATA, - metadata_object); } + flb_log_event_encoder_commit_map(encoder, FLB_LOG_EVENT_METADATA); if (result == FLB_EVENT_ENCODER_SUCCESS && body_object != NULL) { @@ -1400,7 +1446,6 @@ static int process_json_payload_scope_logs_entry( if (result == -1) { flb_plg_error(ctx->ins, "scopeLogs missing"); - return -3; } } @@ -1425,31 +1470,48 @@ static int process_json_payload_scope_logs_entry( return result; } - static int process_json_payload_resource_logs_entry( - struct flb_opentelemetry *ctx, - struct flb_log_event_encoder *encoder, - msgpack_object *resource_logs_object) + struct flb_opentelemetry *ctx, + struct flb_log_event_encoder *encoder, + size_t resource_logs_index, + msgpack_object *resource_logs_object) { - msgpack_object_map *resource_logs_entry; + int ret; + int result; + size_t index; + msgpack_object obj; + msgpack_object_map *resource = NULL; + msgpack_object *resource_attr = NULL; + msgpack_object_map *resource_logs_entry = NULL; + msgpack_object *scope = NULL; msgpack_object_array *scope_logs; - int result; - size_t index; - if (resource_logs_object->type != MSGPACK_OBJECT_MAP) { flb_plg_error(ctx->ins, "unexpected resourceLogs entry type"); - return -2; } - resource_logs_entry = &resource_logs_object->via.map; + /* get 'resource' and resource['attributes'] */ + result = find_map_entry_by_key(&resource_logs_object->via.map, "resource", 0, FLB_TRUE); + if (result >= 0) { + obj = resource_logs_object->via.map.ptr[result].val; + if (obj.type == MSGPACK_OBJECT_MAP) { + resource = &obj.via.map; + result = find_map_entry_by_key(resource, "attributes", 0, FLB_TRUE); + if (result >= 0) { + obj = resource->ptr[result].val; + if (obj.type == MSGPACK_OBJECT_ARRAY) { + resource_attr = &obj; + } + } + } + } + resource_logs_entry = &resource_logs_object->via.map; result = find_map_entry_by_key(resource_logs_entry, "scopeLogs", 0, FLB_TRUE); if (result == -1) { result = find_map_entry_by_key(resource_logs_entry, "scope_logs", 0, FLB_TRUE); - if (result == -1) { flb_plg_error(ctx->ins, "scopeLogs missing"); @@ -1459,19 +1521,130 @@ static int process_json_payload_resource_logs_entry( if (resource_logs_entry->ptr[result].val.type != MSGPACK_OBJECT_ARRAY) { flb_plg_error(ctx->ins, "unexpected scopeLogs type"); - return -2; } scope_logs = &resource_logs_entry->ptr[result].val.via.array; - result = 0; - for (index = 0 ; index < scope_logs->size ; index++) { + /* + * Add the information about OTLP metadata, we do this by registering + * a group-type record. + */ + flb_log_event_encoder_group_init(encoder); + + /* pack internal schema */ + ret = flb_log_event_encoder_append_metadata_values(encoder, + FLB_LOG_EVENT_STRING_VALUE("schema", 6), + FLB_LOG_EVENT_STRING_VALUE("otlp", 4), + FLB_LOG_EVENT_STRING_VALUE("resource_id", 11), + FLB_LOG_EVENT_INT64_VALUE(resource_logs_index), + FLB_LOG_EVENT_STRING_VALUE("scope_id", 8), + FLB_LOG_EVENT_INT64_VALUE(index)); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_plg_error(ctx->ins, "could not set group content metadata"); + return -2; + } + + /* Resource key */ + flb_log_event_encoder_append_body_string(encoder, "resource", 8); + + /* start resource value (map) */ + flb_log_event_encoder_body_begin_map(encoder); + + /* Check if we have OTel resource attributes */ + if (resource_attr) { + flb_log_event_encoder_append_body_string(encoder, "attributes", 10); + result = json_payload_append_converted_kvlist(encoder, + FLB_LOG_EVENT_BODY, + resource_attr); + } + + /* resource dropped_attributers_count */ + result = find_map_entry_by_key(resource, "droppedAttributesCount", 0, FLB_TRUE); + if (result >= 0) { + obj = resource->ptr[result].val; + flb_log_event_encoder_append_body_values(encoder, + FLB_LOG_EVENT_CSTRING_VALUE("dropped_attributes_count"), + FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&obj)); + } + + /* close resource map */ + flb_log_event_encoder_body_commit_map(encoder); + + /* scope metadata */ + scope = NULL; + obj = scope_logs->ptr[index]; + if (obj.type == MSGPACK_OBJECT_MAP) { + result = find_map_entry_by_key(&obj.via.map, "scope", 0, FLB_TRUE); + if (result >= 0) { + if (obj.via.map.ptr[result].val.type == MSGPACK_OBJECT_MAP) { + scope = &obj.via.map.ptr[result].val; + } + } + } + + if (scope) { + /* + * if the scope is found, process every expected key one by one to avoid + * wrongly ingested items. + */ + + /* append scope key */ + flb_log_event_encoder_append_body_string(encoder, "scope", 5); + + /* scope map value */ + flb_log_event_encoder_body_begin_map(encoder); + + /* scope name */ + result = find_map_entry_by_key(&scope->via.map, "name", 0, FLB_TRUE); + if (result >= 0) { + obj = scope->via.map.ptr[result].val; + if (obj.type == MSGPACK_OBJECT_STR) { + flb_log_event_encoder_append_body_values(encoder, + FLB_LOG_EVENT_CSTRING_VALUE("name"), + FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&obj)); + } + } + + /* scope version */ + result = find_map_entry_by_key(&scope->via.map, "version", 0, FLB_TRUE); + if (result >= 0) { + obj = scope->via.map.ptr[result].val; + if (obj.type == MSGPACK_OBJECT_STR) { + flb_log_event_encoder_append_body_values(encoder, + FLB_LOG_EVENT_CSTRING_VALUE("version"), + FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&obj)); + } + } + + /* scope attributes */ + result = find_map_entry_by_key(&scope->via.map, "attributes", 0, FLB_TRUE); + if (result >= 0) { + obj = scope->via.map.ptr[result].val; + if (obj.type == MSGPACK_OBJECT_ARRAY) { + flb_log_event_encoder_append_body_string(encoder, "attributes", 10); + result = json_payload_append_converted_kvlist(encoder, + FLB_LOG_EVENT_BODY, + &obj); + if (result != 0) { + return -2; + } + } + } + + flb_log_event_encoder_commit_map(encoder, FLB_LOG_EVENT_BODY); + } + + flb_log_event_encoder_commit_map(encoder, FLB_LOG_EVENT_BODY); + + flb_log_event_encoder_group_header_end(encoder); + result = process_json_payload_scope_logs_entry( - ctx, - encoder, - &scope_logs->ptr[index]); + ctx, + encoder, + &scope_logs->ptr[index]); + flb_log_event_encoder_group_end(encoder); } return result; @@ -1520,6 +1693,7 @@ static int process_json_payload_root(struct flb_opentelemetry *ctx, result = process_json_payload_resource_logs_entry( ctx, encoder, + index, &resource_logs->ptr[index]); } From b588925a6aa8a42bc60e9885448ffd28cebea829 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 13 Dec 2024 23:40:47 -0600 Subject: [PATCH 21/23] in_opentelemetry: json logs: process spanID, traceID and observedTimeUnixNano Signed-off-by: Eduardo Silva --- plugins/in_opentelemetry/opentelemetry_prot.c | 79 +++++++++++++++++-- 1 file changed, 71 insertions(+), 8 deletions(-) diff --git a/plugins/in_opentelemetry/opentelemetry_prot.c b/plugins/in_opentelemetry/opentelemetry_prot.c index acd1e8c6949..a99ca12fb89 100644 --- a/plugins/in_opentelemetry/opentelemetry_prot.c +++ b/plugins/in_opentelemetry/opentelemetry_prot.c @@ -1225,22 +1225,25 @@ static int json_payload_append_converted_value( return result; } -static int process_json_payload_log_records_entry( - struct flb_opentelemetry *ctx, - struct flb_log_event_encoder *encoder, - msgpack_object *log_records_object) +static int process_json_payload_log_records_entry(struct flb_opentelemetry *ctx, + struct flb_log_event_encoder *encoder, + msgpack_object *log_records_object) { - msgpack_object_map *log_records_entry; + int result; + int body_type; char timestamp_str[32]; + msgpack_object_map *log_records_entry; msgpack_object *timestamp_object; uint64_t timestamp_uint64; msgpack_object *metadata_object; msgpack_object *body_object; - int body_type; - struct flb_time timestamp; - int result; + msgpack_object *observed_time_unix_nano = NULL; msgpack_object *severity_number = NULL; msgpack_object *severity_text = NULL; + msgpack_object *trace_id = NULL; + msgpack_object *span_id = NULL; + struct flb_time timestamp; + if (log_records_object->type != MSGPACK_OBJECT_MAP) { flb_plg_error(ctx->ins, "unexpected logRecords entry type"); @@ -1300,6 +1303,15 @@ static int process_json_payload_log_records_entry( flb_time_from_uint64(×tamp, timestamp_uint64); } + /* observedTimeUnixNano (yes, we do it again) */ + result = find_map_entry_by_key(log_records_entry, "observedTimeUnixNano", 0, FLB_TRUE); + if (result == -1) { + result = find_map_entry_by_key(log_records_entry, "observed_time_unix_nano", 0, FLB_TRUE); + } + else if (result >= 0) { + observed_time_unix_nano = &log_records_entry->ptr[result].val; + } + /* severityNumber */ result = find_map_entry_by_key(log_records_entry, "severityNumber", 0, FLB_TRUE); if (result == -1) { @@ -1334,6 +1346,24 @@ static int process_json_payload_log_records_entry( metadata_object = &log_records_entry->ptr[result].val; } + /* traceId */ + result = find_map_entry_by_key(log_records_entry, "traceId", 0, FLB_TRUE); + if (result == -1) { + result = find_map_entry_by_key(log_records_entry, "trace_id", 0, FLB_TRUE); + } + if (result >= 0) { + trace_id = &log_records_entry->ptr[result].val; + } + + /* spanId */ + result = find_map_entry_by_key(log_records_entry, "spanId", 0, FLB_TRUE); + if (result == -1) { + result = find_map_entry_by_key(log_records_entry, "span_id", 0, FLB_TRUE); + } + if (result >= 0) { + span_id = &log_records_entry->ptr[result].val; + } + result = find_map_entry_by_key(log_records_entry, "body", 0, FLB_TRUE); if (result == -1) { @@ -1363,6 +1393,27 @@ static int process_json_payload_log_records_entry( flb_log_event_encoder_append_string(encoder, FLB_LOG_EVENT_METADATA, ctx->logs_metadata_key, flb_sds_len(ctx->logs_metadata_key)); flb_log_event_encoder_begin_map(encoder, FLB_LOG_EVENT_METADATA); + if (observed_time_unix_nano != NULL && observed_time_unix_nano->type == MSGPACK_OBJECT_STR) { + memset(timestamp_str, 0, sizeof(timestamp_str)); + + if (timestamp_object->via.str.size < sizeof(timestamp_str)) { + strncpy(timestamp_str, + timestamp_object->via.str.ptr, + timestamp_object->via.str.size); + } + else { + strncpy(timestamp_str, + timestamp_object->via.str.ptr, + sizeof(timestamp_str) - 1); + } + + timestamp_uint64 = strtoul(timestamp_str, NULL, 10); + + flb_log_event_encoder_append_metadata_values(encoder, + FLB_LOG_EVENT_STRING_VALUE("observed_timestamp", 18), + FLB_LOG_EVENT_INT64_VALUE(timestamp_uint64)); + } + if (severity_number != NULL) { flb_log_event_encoder_append_metadata_values(encoder, FLB_LOG_EVENT_STRING_VALUE("severity_number", 15), @@ -1380,6 +1431,18 @@ static int process_json_payload_log_records_entry( result = json_payload_append_converted_kvlist(encoder, FLB_LOG_EVENT_METADATA, metadata_object); } + if (trace_id != NULL && (trace_id->type == MSGPACK_OBJECT_STR || trace_id->type == MSGPACK_OBJECT_BIN)) { + flb_log_event_encoder_append_metadata_values(encoder, + FLB_LOG_EVENT_STRING_VALUE("trace_id", 8), + FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(trace_id)); + } + + if (span_id != NULL && (span_id->type == MSGPACK_OBJECT_STR || span_id->type == MSGPACK_OBJECT_BIN)) { + flb_log_event_encoder_append_metadata_values(encoder, + FLB_LOG_EVENT_STRING_VALUE("span_id", 7), + FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(span_id)); + } + flb_log_event_encoder_commit_map(encoder, FLB_LOG_EVENT_METADATA); } From 615bdd4e277acaa92c9c367907e1ea456b37936d Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Sun, 15 Dec 2024 18:58:10 -0600 Subject: [PATCH 22/23] out_lib: add new option 'data_mode' the new configuration option 'data_mode' allows to control what type of data the plugin pass to the callback set by the caller. By default it was always passing one log record at a time in the configured 'format'; now with 'data_mode' the user can set 'single_record' (default) or 'chunk'. When the option 'chunk' is passed, the whole binary chunk is passed to the callback as a reference (it's up to the caller to unpack and validate the content) Signed-off-by: Eduardo Silva --- plugins/out_lib/out_lib.c | 49 +++++++++++++++++++++++++++++++++++---- plugins/out_lib/out_lib.h | 5 ++++ 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/plugins/out_lib/out_lib.c b/plugins/out_lib/out_lib.c index 79aa4fa45c0..a654fba31e3 100644 --- a/plugins/out_lib/out_lib.c +++ b/plugins/out_lib/out_lib.c @@ -48,12 +48,15 @@ static int configure(struct flb_out_lib_config *ctx, } } - tmp = flb_output_get_property("max_records", ins); - if (tmp) { - ctx->max_records = atoi(tmp); + if (strcasecmp(ctx->data_mode_str, "single_record") == 0) { + ctx->data_mode = FLB_DATA_MODE_SINGLE_RECORD; + } + else if (strcasecmp(ctx->data_mode_str, "chunk") == 0) { + ctx->data_mode = FLB_DATA_MODE_CHUNK; } else { - ctx->max_records = 0; + flb_plg_error(ctx->ins, "Invalid data_mode: %s", ctx->data_mode_str); + return -1; } return 0; @@ -85,6 +88,8 @@ static int out_lib_init(struct flb_output_instance *ins, } ctx->ins = ins; + flb_output_config_map_set(ins, (void *) ctx); + if (cb_data) { /* Set user callback and data */ ctx->cb_func = cb_data->cb; @@ -125,6 +130,16 @@ static void out_lib_flush(struct flb_event_chunk *event_chunk, (void) i_ins; (void) config; + /* + * if the plugin is configured with data_mode = 'chunk', we pass the chunk + * as a reference to the callback function. + */ + if (ctx->data_mode == FLB_DATA_MODE_CHUNK) { + ctx->cb_func(event_chunk->data, event_chunk->size, ctx->cb_data); + FLB_OUTPUT_RETURN(FLB_OK); + } + + /* Everything else here is for data_mode = 'single_record' */ msgpack_unpacked_init(&result); while (msgpack_unpack_next(&result, event_chunk->data, @@ -187,6 +202,7 @@ static void out_lib_flush(struct flb_event_chunk *event_chunk, flb_free(buf); data_for_user = out_buf; data_size = len; + #ifdef FLB_HAVE_METRICS } #endif @@ -211,6 +227,30 @@ static int out_lib_exit(void *data, struct flb_config *config) return 0; } +/* Configuration properties map */ +static struct flb_config_map config_map[] = { + { + FLB_CONFIG_MAP_STR, "format", NULL, + 0, FLB_FALSE, 0, + "Specifies the data format to be printed. Supported formats are " + "'msgpack' or 'json', json_lines and json_stream." + }, + + { + FLB_CONFIG_MAP_INT, "max_records", NULL, + 0, FLB_TRUE, offsetof(struct flb_out_lib_config, max_records), + "Specifies the maximum number of log records to be printed." + }, + + { + FLB_CONFIG_MAP_STR, "data_mode", "single_record", + 0, FLB_TRUE, offsetof(struct flb_out_lib_config, data_mode_str), + }, + + /* EOF */ + {0} +}; + struct flb_output_plugin out_lib_plugin = { .name = "lib", .description = "Library mode Output", @@ -219,4 +259,5 @@ struct flb_output_plugin out_lib_plugin = { .cb_exit = out_lib_exit, .event_type = FLB_OUTPUT_LOGS | FLB_OUTPUT_METRICS, .flags = 0, + .config_map = config_map }; diff --git a/plugins/out_lib/out_lib.h b/plugins/out_lib/out_lib.h index 0575149c064..4d82bf5c415 100644 --- a/plugins/out_lib/out_lib.h +++ b/plugins/out_lib/out_lib.h @@ -31,9 +31,14 @@ enum { #define FLB_FMT_STR_MSGPACK "msgpack" #define FLB_FMT_STR_JSON "json" +#define FLB_DATA_MODE_SINGLE_RECORD 0 /* "single_record" */ +#define FLB_DATA_MODE_CHUNK 1 /* "chunk" */ + struct flb_out_lib_config { int format; int max_records; + int data_mode; + flb_sds_t data_mode_str; int (*cb_func)(void *record, size_t size, void *data); void *cb_data; struct flb_output_instance *ins; From 4d715c07d91ae8087a2bf1e6a185b3e95ac18914 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Sun, 15 Dec 2024 19:01:01 -0600 Subject: [PATCH 23/23] tests: internal: in_opentelemetry: extend tests for group metadata and body Signed-off-by: Eduardo Silva --- tests/runtime/in_opentelemetry.c | 362 +++++++++++++++++++++++++++---- 1 file changed, 317 insertions(+), 45 deletions(-) diff --git a/tests/runtime/in_opentelemetry.c b/tests/runtime/in_opentelemetry.c index c28c58dfb8c..7dc0a2c2435 100644 --- a/tests/runtime/in_opentelemetry.c +++ b/tests/runtime/in_opentelemetry.c @@ -18,21 +18,18 @@ * limitations under the License. */ -#include #include #include #include #include #include -#include -#include "flb_tests_runtime.h" +#include +#include "flb_tests_runtime.h" #define JSON_CONTENT_TYPE "application/json" #define PORT_OTEL 4318 -#define TEST_MSG_OTEL_LOGS "{\"resourceLogs\":[{\"resource\":{},\"scopeLogs\":[{\"scope\":{},\"logRecords\":[{\"timeUnixNano\":\"1660296023390371588\",\"body\":{\"stringValue\":\"{\\\"message\\\":\\\"test\\\"}\"}}]}]}]}" -#define TEST_CB_MSG_OTEL_LOGS "[1660296024.698112,{\"log\":\"{\\\"message\\\":\\\"test\\\"}\"}]" - +#define TEST_MSG_OTEL_LOGS "{\"resourceLogs\":[{\"resource\":{\"attributes\":[{\"key\":\"my-resource-attribute\",\"value\":{\"stringValue\":\"my-attribute\"}}]},\"scopeLogs\":[{\"scope\":{\"name\":\"my.library\",\"version\":\"1.0.0\",\"attributes\":[{\"key\":\"a-scope-attribute\",\"value\":{\"stringValue\":\"my-scope-attribute\"}}]},\"logRecords\":[{\"timeUnixNano\":\"1660296023390371588\",\"body\":{\"stringValue\":\"{\\\"message\\\":\\\"test\\\"}\"}}]}]}]}" #define V1_ENDPOINT_LOGS "/v1/logs" struct http_client_ctx { @@ -75,33 +72,78 @@ static void clear_output_num() set_output_num(0); } -/* Callback to check expected results */ -static int cb_check_result_json(void *record, size_t size, void *data) +/* + * Records generated by OTel in Fluent Bit are packaged with groups inside the chunk: + * + * - Group Header + * - Record(s) + * - Group Footer + */ + +/* retrieve the group header metadata, contains internal Fluent Bit packaging info */ +static char *get_group_metadata(void *chunk, size_t size) { - char *p; - char *expected; - char *result; - int num = get_output_num(); + int ret; + char *json; + struct flb_log_event log_event; + struct flb_log_event_decoder log_decoder; + + ret = flb_log_event_decoder_init(&log_decoder, chunk, size); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + + ret = flb_log_event_decoder_next(&log_decoder, &log_event); + if (ret != FLB_EVENT_DECODER_SUCCESS) { + return NULL; + } - set_output_num(num+1); + json = flb_msgpack_to_json_str(1024, log_event.metadata); + flb_log_event_decoder_destroy(&log_decoder); + return json; +} - expected = (char *) data; - result = (char *) record; +/* retrieve the group header record content (body), which must contain the OTLP metadata (resource, scope, etc) */ +static char *get_group_body(void *chunk, size_t size) +{ + int ret; + char *json; + struct flb_log_event log_event; + struct flb_log_event_decoder log_decoder; - p = strstr(result, expected); - TEST_CHECK(p != NULL); + ret = flb_log_event_decoder_init(&log_decoder, chunk, size); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); - if (p==NULL) { - flb_error("Expected to find: '%s' in result '%s'", - expected, result); + ret = flb_log_event_decoder_next(&log_decoder, &log_event); + if (ret != FLB_EVENT_DECODER_SUCCESS) { + return NULL; } - /* - * If you want to debug your test - * - * printf("Expect: '%s' in result '%s'", expected, result); - */ - flb_free(record); - return 0; + + json = flb_msgpack_to_json_str(1024, log_event.body); + flb_log_event_decoder_destroy(&log_decoder); + return json; +} + +/* from the second entry in the chunk, return the record body */ +static char *get_log_body(void *chunk, size_t size) +{ + int ret; + char *json; + struct flb_log_event log_event; + struct flb_log_event_decoder log_decoder; + + ret = flb_log_event_decoder_init(&log_decoder, chunk, size); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + + /* 0: group header */ + flb_log_event_decoder_next(&log_decoder, &log_event); + + /* 1: record */ + flb_log_event_decoder_next(&log_decoder, &log_event); + + /* convert log body to json */ + json = flb_msgpack_to_json_str(1024, log_event.body); + + flb_log_event_decoder_destroy(&log_decoder); + return json; } struct http_client_ctx* http_client_ctx_create() @@ -181,6 +223,10 @@ static struct test_ctx *test_ctx_create(struct flb_lib_out_cb *data) o_ffd = flb_output(ctx->flb, (char *) "lib", (void *) data); ctx->o_ffd = o_ffd; + /* Set output properties */ + flb_output_set(ctx->flb, o_ffd, + "data_mode", "chunk", + NULL); return ctx; } @@ -215,8 +261,30 @@ static void test_ctx_destroy(struct test_ctx *ctx) flb_destroy(ctx->flb); flb_free(ctx); } +static int cb_check_log_group_metadata(void *chunk, size_t size, void *data) +{ + int ret; + int num = get_output_num(); + char *json; -void flb_test_otel_logs() + json = get_group_metadata(chunk, size); + TEST_CHECK(json != NULL); + + ret = strcmp(json, (char *) data); + TEST_CHECK(ret == 0); + if (ret != 0) { + flb_error("Record mismatch:"); + flb_error(" - Expected: '%s'", (char *) data); + flb_error(" - Received: '%s'", json); + } + + set_output_num(num + 1); + + flb_free(json); + return 0; +} + +static void flb_test_log_group_metadata() { struct flb_lib_out_cb cb_data; struct test_ctx *ctx; @@ -229,8 +297,8 @@ void flb_test_otel_logs() clear_output_num(); - cb_data.cb = cb_check_result_json; - cb_data.data = TEST_CB_MSG_OTEL_LOGS; + cb_data.cb = cb_check_log_group_metadata; + cb_data.data = "{\"schema\":\"otlp\",\"resource_id\":0,\"scope_id\":0}"; ctx = test_ctx_create(&cb_data); if (!TEST_CHECK(ctx != NULL)) { @@ -284,7 +352,30 @@ void flb_test_otel_logs() test_ctx_destroy(ctx); } -void flb_test_otel_successful_response_code(char *response_code) +static int cb_check_log_group_body(void *chunk, size_t size, void *data) +{ + int ret; + char *json; + int num = get_output_num(); + + json = get_group_body(chunk, size); + TEST_CHECK(json != NULL); + + ret = strcmp(json, (char *) data); + TEST_CHECK(ret == 0); + if (ret != 0) { + flb_error("Record mismatch:"); + flb_error(" - Expected: '%s'", (char *) data); + flb_error(" - Received: '%s'", json); + } + + set_output_num(num + 1); + + flb_free(json); + return 0; +} + +static void flb_test_log_group_body() { struct flb_lib_out_cb cb_data; struct test_ctx *ctx; @@ -297,8 +388,179 @@ void flb_test_otel_successful_response_code(char *response_code) clear_output_num(); - cb_data.cb = cb_check_result_json; - cb_data.data = TEST_CB_MSG_OTEL_LOGS; + cb_data.cb = cb_check_log_group_body; + cb_data.data = "{\"resource\":{\"attributes\":{\"my-resource-attribute\":\"my-attribute\"}},\"scope\":{\"name\":\"my.library\",\"version\":\"1.0.0\",\"attributes\":{\"a-scope-attribute\":\"my-scope-attribute\"}}}"; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + ctx->httpc = http_client_ctx_create(); + TEST_CHECK(ctx->httpc != NULL); + + c = flb_http_client(ctx->httpc->u_conn, FLB_HTTP_POST, V1_ENDPOINT_LOGS, buf, strlen(buf), + "127.0.0.1", PORT_OTEL, NULL, 0); + ret = flb_http_add_header(c, FLB_HTTP_HEADER_CONTENT_TYPE, strlen(FLB_HTTP_HEADER_CONTENT_TYPE), + JSON_CONTENT_TYPE, strlen(JSON_CONTENT_TYPE)); + TEST_CHECK(ret == 0); + if (!TEST_CHECK(c != NULL)) { + TEST_MSG("http_client failed"); + exit(EXIT_FAILURE); + } + + ret = flb_http_do(c, &b_sent); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("ret error. ret=%d\n", ret); + } + else if (!TEST_CHECK(b_sent > 0)){ + TEST_MSG("b_sent size error. b_sent = %lu\n", b_sent); + } + else if (!TEST_CHECK(c->resp.status == 201)) { + TEST_MSG("http response code error. expect: 201, got: %d\n", c->resp.status); + } + + /* waiting to flush */ + flb_time_msleep(1500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + flb_http_client_destroy(c); + flb_upstream_conn_release(ctx->httpc->u_conn); + test_ctx_destroy(ctx); +} + +static int cb_check_log_body(void *chunk, size_t size, void *data) +{ + int ret; + int num = get_output_num(); + char *json; + + json = get_log_body(chunk, size); + TEST_CHECK(json != NULL); + + ret = strcmp(json, (char *) data); + TEST_CHECK(ret == 0); + if (ret != 0) { + flb_error("Record mismatch:"); + flb_error(" - Expected: '%s'", (char *) data); + flb_error(" - Received: '%s'", json); + } + + set_output_num(num + 1); + + flb_free(json); + return 0; +} + +static int cb_no_check(void *chunk, size_t size, void *data) +{ + int num; + (void) chunk; + (void) data; + (void) size; + + num = get_output_num(); + set_output_num(num + 1); + return 0; +} + +static void flb_test_log_body() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + struct flb_http_client *c; + int ret; + int num; + size_t b_sent; + + char *buf = TEST_MSG_OTEL_LOGS; + + clear_output_num(); + + cb_data.cb = cb_check_log_body; + cb_data.data = "{\"log\":\"{\\\"message\\\":\\\"test\\\"}\"}"; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + ctx->httpc = http_client_ctx_create(); + TEST_CHECK(ctx->httpc != NULL); + + c = flb_http_client(ctx->httpc->u_conn, FLB_HTTP_POST, V1_ENDPOINT_LOGS, buf, strlen(buf), + "127.0.0.1", PORT_OTEL, NULL, 0); + ret = flb_http_add_header(c, FLB_HTTP_HEADER_CONTENT_TYPE, strlen(FLB_HTTP_HEADER_CONTENT_TYPE), + JSON_CONTENT_TYPE, strlen(JSON_CONTENT_TYPE)); + TEST_CHECK(ret == 0); + if (!TEST_CHECK(c != NULL)) { + TEST_MSG("http_client failed"); + exit(EXIT_FAILURE); + } + + ret = flb_http_do(c, &b_sent); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("ret error. ret=%d\n", ret); + } + else if (!TEST_CHECK(b_sent > 0)){ + TEST_MSG("b_sent size error. b_sent = %lu\n", b_sent); + } + else if (!TEST_CHECK(c->resp.status == 201)) { + TEST_MSG("http response code error. expect: 201, got: %d\n", c->resp.status); + } + + /* waiting to flush */ + flb_time_msleep(1500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + flb_http_client_destroy(c); + flb_upstream_conn_release(ctx->httpc->u_conn); + test_ctx_destroy(ctx); +} + +void flb_test_successful_response_code(char *response_code) +{ + struct flb_lib_out_cb cb_data = {0}; + struct test_ctx *ctx; + struct flb_http_client *c; + int ret; + int num; + size_t b_sent; + + char *buf = TEST_MSG_OTEL_LOGS; + + clear_output_num(); + + cb_data.cb = cb_no_check; + cb_data.data = NULL; ctx = test_ctx_create(&cb_data); if (!TEST_CHECK(ctx != NULL)) { @@ -357,17 +619,26 @@ void flb_test_otel_successful_response_code(char *response_code) test_ctx_destroy(ctx); } -void flb_test_otel_successful_response_code_200() +void flb_test_successful_response_code_200() +{ + flb_test_successful_response_code("200"); +} + +void flb_test_successful_response_code_204() { - flb_test_otel_successful_response_code("200"); + flb_test_successful_response_code("204"); } -void flb_test_otel_successful_response_code_204() +static int cb_check_tag_from_uri_false(void *record, size_t size, void *data) { - flb_test_otel_successful_response_code("204"); + int num = get_output_num(); + + /* just make sure we received some records so routing worked */ + set_output_num(num + 1); + return 0; } -void flb_test_otel_tag_from_uri_false() +void flb_test_tag_from_uri_false() { struct flb_lib_out_cb cb_data; struct test_ctx *ctx; @@ -380,8 +651,8 @@ void flb_test_otel_tag_from_uri_false() clear_output_num(); - cb_data.cb = cb_check_result_json; - cb_data.data = TEST_CB_MSG_OTEL_LOGS; + cb_data.cb = cb_check_tag_from_uri_false; + cb_data.data = NULL; ctx = test_ctx_create(&cb_data); if (!TEST_CHECK(ctx != NULL)) { @@ -442,10 +713,11 @@ void flb_test_otel_tag_from_uri_false() } TEST_LIST = { - {"otel_logs", flb_test_otel_logs}, - {"successful_response_code_200", flb_test_otel_successful_response_code_200}, - {"successful_response_code_204", flb_test_otel_successful_response_code_204}, - {"tag_from_uri_false", flb_test_otel_tag_from_uri_false}, + {"log_group_metadata" , flb_test_log_group_metadata}, + {"log_group_body" , flb_test_log_group_body}, + {"log_body" , flb_test_log_body}, + {"successful_response_code_200", flb_test_successful_response_code_200}, + {"successful_response_code_204", flb_test_successful_response_code_204}, + {"tag_from_uri_false" , flb_test_tag_from_uri_false}, {NULL, NULL} }; -