From d2c47a31c212fc33b0c5ec8a278bcad0bb0242a6 Mon Sep 17 00:00:00 2001 From: Pat Nadolny Date: Tue, 3 Oct 2023 08:36:15 -0400 Subject: [PATCH] Adds loaders target-hdfs (automattic) (#1536) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Edgar Ramírez Mondragón --- _data/default_variants.yml | 1 + .../loaders/target-hdfs/automattic.yml | 60 ++++++++++++++++++ static/assets/logos/loaders/hdfs.png | Bin 0 -> 8512 bytes 3 files changed, 61 insertions(+) create mode 100644 _data/meltano/loaders/target-hdfs/automattic.yml create mode 100644 static/assets/logos/loaders/hdfs.png diff --git a/_data/default_variants.yml b/_data/default_variants.yml index 0a8b2fe9f..8c4b0a166 100644 --- a/_data/default_variants.yml +++ b/_data/default_variants.yml @@ -605,6 +605,7 @@ loaders: target-elasticsearch: dtmirizzi target-gcs: datateer target-gsheet: singer-io + target-hdfs: automattic target-hubspot: meltanolabs target-iomete: iomete target-jsonl: andyh1203 diff --git a/_data/meltano/loaders/target-hdfs/automattic.yml b/_data/meltano/loaders/target-hdfs/automattic.yml new file mode 100644 index 000000000..4f5179fe1 --- /dev/null +++ b/_data/meltano/loaders/target-hdfs/automattic.yml @@ -0,0 +1,60 @@ +capabilities: [] +description: Hadoop File System - Parquet +domain_url: https://arrow.apache.org/docs/python/generated/pyarrow.fs.HadoopFileSystem.html +executable: target-hdfs +keywords: [] +label: HDFS Parquet +logo_url: /assets/logos/loaders/hdfs.png +maintenance_status: active +name: target-hdfs +namespace: target_hdfs +next_steps: '' +pip_url: git+https://github.com/Automattic/target-hdfs.git +quality: silver +repo: https://github.com/Automattic/target-hdfs +settings: +- description: The compression extension to include in the file names. + kind: string + label: Compression Extension + name: compression_extension +- description: The compression method to use. + kind: string + label: Compression Method + name: compression_method +- description: The prefix to add to each output file. + kind: string + label: File Prefix + name: file_prefix +- description: The size of the output files. + kind: integer + label: File Size Mb + name: file_size_mb +- description: The separator to use in the file name. + kind: string + label: Filename Separator + name: filename_separator +- description: The destination path in HDFS. + kind: string + label: Hdfs Destination Path + name: hdfs_destination_path +- description: The max size of the queue. + kind: integer + label: Max Queue Size + name: max_queue_size +- description: A partition value to use for separating output files. + kind: string + label: Partitions + name: partitions +- description: The number of rows per file. + kind: integer + label: Rows Per File + name: rows_per_file +- description: Whether to put stream data in separate folders or not. + kind: boolean + label: Streams In Separate Folder + name: streams_in_separate_folder +settings_group_validation: +- [] +settings_preamble: '' +usage: '' +variant: automattic diff --git a/static/assets/logos/loaders/hdfs.png b/static/assets/logos/loaders/hdfs.png new file mode 100644 index 0000000000000000000000000000000000000000..669f5d5713cf2bab527c6d081141f3758168dac9 GIT binary patch literal 8512 zcmV-GA-~>c>C0MO6?($WC`|IXCZ0RR8d|Nqnf|IqsS0QmR->goXS@BrQ2 z0OR8T^YZ|!s{q>C0KdNg;^F|8mjI@w0J*sUE-nDDumJJ#@t~jp=;-J-Hvo5c0J*uj zRaF2>O8{|k0BC3ckBfdF4$0PXGVK0W{r4gkf)#Z**Oii(OOBLLRc);c->u&}To zARreP7ez$?78U?}d;oraewvz^Pfq|fH8n^`NGvP>prD{EEG%efXoG`;9UTB>W@b7% zE_r!jn3#G0|9wwSPd`6D|No_0T1cm-hX4P;x3`kv;k5t%JpccR|Nmal(5C> zFD6)z%p?E+AH7LLK~#90)F8Dotgjt2exj7j@3az2s3-W_dVMv&~^IO&9{1G<#ExT?sr39?@~Nl(&a1@)02Jj z|MvdP%{1sO-SyMw=Ld-6;Qa0Bw7tYDd}!>W^Dg1xEkf(^H|(9hMKZiP-COE~J{+E- zQ^LS&1b_I^UcZvAUkJqIzrwz_g1di$&`nqJ;{Nk60vJwYAAC}nqUlQe9HAY_OEL-g z8`}2$?5ex;%U&qX_gC7lC7R&b>7jd}hadmrKmY5$6AUH2v32Bl@y%1#kKMPR5Rg~5 z?Rk{or+<9=^2=WmZse<*_B#U9=~KWZ`Oine`ngL0-hay^fB9oVhHT4{hUZUK6xu6o zgxpTS)WyK}Z>h|0f07cwpr47i2Kp91J>Ix&V%KR2e*K!tWRialVE}+HgoSvlHQW4gvbatdbVAYm$agMHWKW3OpO>~YW7OTp|MBnpZ)eu|FsGwvJvD42tT zgr*4daQ>76N$w!u`Y1@Ivomv$Qxv1aZOqdNB)LiWmiAGQOncKNF<@dvZ@he3J|owM zpPhbz*cCJRNCJwYXqYDzUOxdNIo;<$!uw2`Ad`VmDV4sx%}{@N7GK%Pfuaee@Ux4; zx+7ik;lqc!yZ7IJ|8(;|lYU40oTjpRcWTHz;DjeCND-@gT{Uf{R5i6kjXdUX)yDy%<0 zM6a(NCAJVbvv;@2$ly3MuO5l4rq_Zc!P3dgHy+$0KzHeA3KEIG-iHI?HtG%b+x(uh zGmF1u<8g_ESCopCBAF?V;zJ&5jD#ESb-ZyY<)w;(px|p}9#7ap>`0=3^^VL$PLUgL zmPf&i^=JKZ3P`cJtD~G-8hN>8S`YYYKk6p&lD70QIc{|t?WL)k~Hdn;0 z`e;14M!iv0QPzU(MYh#ED#T_0f+$sIj)Joabwz*Dhc~RT^7ZQ_QDAR`%rD{^bM?(L zQ87f5QPohDV#t!}2-Rj_AOV7)wdj|>wJ(I}|Kyb;JRlaqa(zU( z6m7P!p{WY4gn3OEp(c*63K+7AqAJ($I}Z8vJYzjBW+I{KqY*KnRr>FT@+Q0MC$Bsj z_(LX{6brQ|7ytnT`LNYg5ZcA&j1(ji28kq@ZMbB&gi%n1^kWs6Pz@>w4-s5-_pgv2 z#5J-^0_7j%bXNriun{iiBpz;5W~LxfB;m9%u>^tZ~QT zNGC0QnA;{=pbClLmIf3=sC%N>Mp{IGAwx;iHLc7wg6Q?y1SirMBhbI5qR{NyLd~PJ zt&ZYTDw(nZ;w{+TpcF?2P!J{M6Ercx6lMVM)_7RMm4WLzh)Kw*#wOaLBw=F+KU`e7 zcJD;Cct-y_NEM1(nTJgYzx_y$LqP(evNm(3xysu!2FK2;AxqC``X*|!?^Ct9wN(~? z{Tt-PuXkT>zp=mGrjoXEM6Rl%gA_K1GQeR@0Ywo_^clsVGTODSVq^=#1tvE@)yXF0 z-)AV8zOV?(&^|?O!16Phbmo>*u$o-8JOd&IV?(Wi!y2s%U_q#+%p?Y_v&q=z;N^(_ zrLU9M*3@bbAurUh-#P>0(UYVR!fySby0V9F}h%PBZzTk3JTsnW(r_Ve;PHSeL=+B^r;7E>b zqb*$_9Grm|#Bh8{J#9wcprnq7B>r$*Ruz6)&t`3xr_t=V{fvrY+*cv&+YEfLn#u$PUzv7*pQis{{nirNRj7A7Az~0u-B)}* zqFEo36*bF2VpMEoQmITzT-%6pW>o!Q8BvNf2R*lk(6%#kcd4|ee4J)h@kc7UeEm$tW zyKfm^3qdN$v@*4d4uzO7!6=YTU=~Td>VC_R8YfDBAH@3`74zVXG=vLR3TkFp~7=LdVtK9oa<~LC(3GsgHbTR zXAZW&hGNiqy|34|>;7kZ>SL)bg2R9Mw*d(MdqxkV;1|d5JObyC?g0aPw?e%yzs{^< z35JUT^g{OvlLsXYvWCHdcSX@@K;g|HoH{vZUrNS~tfJYSObC*h&-BIjoF3;TL5#rU zx8U2${c-4W04DOFp%L3qG<-y46&fdOv~-ta=tB`}G@s*ca&3QfI1FtT3kFf}RVD<< zG_#7In780cBsY#Axx;_E*DCbRBJD)e%fYC=+1u@_o)4NVScIVAC1}M)FsWGe8-}j) zrgJygV?9Ng;>WF|mx!o9W)j|U&ETq@4(pnT6iwB8J=@K2xh5a?Qa~{2_-q4BCvR5m z)Imq~WfXa=vd~c&2J+`Ry0mY%FtB@43Vf}V!Xz+)_^TOLRjCQS0lR4>?175*b5c2M8 z7w~r3AcFgDRS4V5jxO6gW7L8_HiU1q6~O^ZA%N*?lel5e7W*#S2d$w)PAF5Hf)9Re zNZjS${>&uM>N<4U#YCd!j1jzdknG|{h(ajTjDn%)L$EffBoVevr68j$&vJ#th;FbS zbV4gsgaf)pa1d_JS5^+KDlEI(yM;Xc(~)Ah38G*%{pAmE&wI|7J#$Gp$eILR4Y5(> zkRS@yOLi~*o=K)s@6dKxm;1>bM41`US_*)8RpsEXRTQ1p9Zx|QPN9?DMZrcEz6o}N zWh!{u^v+8`I`!AT@GC%;Rdi)e0C^miJ|w@Z6O}NkC|ZkHqj%^G8v)t&7i_^zbSzdG z<{tnDG;$=$@cB82IrkV@H#JltcF_^U6k-;U$I9P56lA`L*qHe+q6OL8YiaogQoz2A z${$1htor2>>T5yj(`3Ox$;z81bc8~vC6`27<<>Vcz_&LP{K&6N&=%MaxbQ4hDK; za;>DdP)85mQQY6gwqJzNZpilu4n7hG8MZ>(b^vx^w-x(TEeHYbv) zB;UkN@buSZmGImd;b5Z0$TLqmTCnQGT<4aNMN^9z3OT?(C^Nm^z9dWMyZ$5m$bfk< zF&n}-X!}h<`isj8#4csw?XxW6@RAbY;2KA#mXwhmB26v+XU)F%HPn*77eMx>N?q%7 zm^9nx8j*^@F`J4wb8^5k2ERIdql~A4?uAn!+C>!(HKI%a0bVt{2oF%ykgv%J3Xko3 ze~dz0@REZAh>!Mc^Jh}YrTGVNmJZvNk8F>fq>`Q+O|w$(EsnggT+}@Vv)WQ46@n2f z89lbm;oxP+2l9f0*DO3>TLFGBrgls*z~Py4Cy8zmTjwWNzj!+cn@)~09)+0DqKd1u z5FM=$;x%JT?7wagLf&V(IN14UbI^j<2nWzJ)t%DdnJL6~76twT3YZuO57GOw8Cg#O zqte%-6hbau-OZ7b3W=N@9MHY6vtzr66)FZQ1|P9acE`R5n1#-lozZaQ6a!x)X7mk) zNaM(e$c-iPvD%U<_Eso_c}R&h+b#eH`?eocq?0j}AaIXfclSIFqJ#p)`G&N&-Pnv) z_z;_1>!Iywu~BvWH6*JH&bQ?s00(>aUcgz{3N)C298z1PO{}{tnXx!?zQNaeuf({0 zkosywC5q$0Axxvqus2$LTQqMXGO1t!N5Fwf?<~aCf&#JGWn(65@P<$P4H@q&&RQHr z2622{o2}O%8XF-26U>Gxto%+B%j>+Ql`gnQD4^BIgB3?rk>6$MNgecyxC`l1h}pWU18EQH;bvw>ed$<@#XN_pG<@bzSA;!I2rW9vu zI*CFhbZdbNpdE=l?*JHqfkE5WKJw+C*(>xpKNQ&MK+So%CHM|i3EjU71~HB9yt0J@ zh+Z_ive#@JS-yd9uQOC_4!j|Ae1N0bQfF>Vz6T7*$qx@ z0UT5gK2Q!az8AAG)g1i7TBJ|CNMKu#DsvQ9 zxI5G^gykF;47;n_-;f0H^GH`zjW*U(pP9KonZRkYnq?TgZ>fi5w%vf8xnacY)LH~= zb>Q#8py5H4_6-ptDj+RiS7mt-8&s)7ZN=+A;iK4>q`}_PLKCkG1$an&!a>rD1BJHT zvTo?QhHJDIC3k)tpn`CiN_%KBQGA~Vl*%<$qg$941n>@6mFbnA0$yZkj)!ADNlyd9 z@Ut1{J3nRI)wNtmtLKAa6?v**jx~R{w0a*^gWfI!=OAX^;(cLS!9q~K@-x>+Mkd+; z%xg4-uB-6IsjUveXNvL)?FDo!8>1N%B0zAk z6i*ROP+|4f$nHeCDZmR3_Kz4!yzkY@VVWqc!Xty=V6owI0Khp=>8wX&r!F1PfOsh? z7Rq9AUKiKv7oo*Y88`<^F%0w|9X4kf*|mNmRAR7?$U*iOJNdas|Ji4KL9;+`u#lW$ z1YoMrxDIlV+;}cHh&-Z%c;8&dz-}N8c38bv3c*2ev&zB%p@Ke&(1-%()cuQA2Ru2) zT+b}+#+b=2CYZl{4Wk_oZyagT7_;LyyOA6~4EZBoYNP;R*nkPb5km0;1rG4CUeL+c z*v2t8XKX=BiLB)_#{34n?VrR;3^>2S2}&-F!zQ3?W_I^rBW=VQeFfz8`sP_@jv3$t zyIb6f#SjC^0VGE~46&%Dmxe5s8+bp^f--|3bEM>|; zq=V2zeLdpDfIu(`_<3Tnrurxc=@@2#LgQ&Cgl0Lvg@Kmq=z$24H3pODjd`62+E&2r zSIqN~s7_49J=YO9HkxyZ0Zk|NSn?`reFkc2*+gMRSDd0iDd74r=8!L9wo#HUmZGAt zeDoaEBA{+F9tZUv=zDxl|3Ek!C2EFSsgoK2B1Ew^oKQ@eAoGU6o zfu@UR8^s>zd&N13T|uL^adyK^Tb*GRK8{`B?M7lTn+-G4M&#Bo6M{x_Rti{K;w&6M zVJ@`j?}u6K^l3`A#MD!D|dp#L4*^I2X=ues#T53nwihJsK}Pgf<_4-GJwS?qq9A% zfp&0zPRG^d)xr}p%Qqk5~Va0;w&0p_iagX3u5)T`qQ$D5;ixz!+O zl{?477tR;2W}R}Uwa3FuTpx)R4P$u9rnQ&12Q|!Z8>hf3I2LIBzZMi4wSqYa!SmS< z{$OP{&!A!m*i`n*h*4}y299c&9~)=GN<8GSN{#tQ&cfKcyiV(Q62ZXLTt}bKkA92w z05u9$%-#)&YqQp|dBcW}09OWu_;t*$?aO>`@A|Ga%@}!q=n{c>ALm8K`97!PpZd&1 z^hF>E;wd`Y$5{6fpt$C~Xo|ARlGVPsX2ZM0q0Yt)WH@}aGyhYO+4#!$OspIV1FK{9 zcK5%j8|Q_okOSZ>~PxoS)x(yn3^lipW63=li6>V{$HZmFkl(u@w<=dRx4zL$fR%8DlUTd)ND z@SEGn5{nZ&HuX*pSjJ#nbu&P=eNpb|(u@bNz& zIk?&CIZt0$6E|+qFjN--g%B&k-l>Wc(3-Zp;IjGfwFc{`Soqo5qF48mygxPg(#4gH zJGf|?#cEBsX_K;(0j!1*kYLXku@m8Bhgk(#ua`)?Ln?m@KO@#U88o)D%wftc^=?Pg zYyxm4D#$MJy->5iLf8$D3Fzpo+<6easM){HPefGyfR5slYC~uBb-LVH^71Rps?K&Ny?eY#G zrfaKcf@|aEOSIh_*9o^O>j7Ex18&)&FvnCOrDv~bm1Jv$W~Otz^zuRp&Pnp5m1a> z9m#Ta!0k+1%oWkpeTfLia8z!UYc=u~g^J3ewE?G~XV9~xexIu+Kd-}xfE^MldT|zc z%5!zV^FNo?a8;A$ldJUL)InWU^#(h@e!)h8Q}D8?vJ*Gw98bv?zz3wz3Nf^x3kAl3 z2m@deQyk309p}86HAv{qNm}sJ6gUOjc{soxZ1Lac;~)V!y}X&v&r>1Q-5wIX&4Ps8 z2LHxvL!=28{2T#(Snjq7^YL{4?O<*W=0?`x9rT!1T(J)AAs099v>L6^#L3VCA^bxK z;xv9EmN{v}z|{e5ZeePXUR>LMpf$m(1zW`(2L%wo(rW$eT^08Q^A6~=X_F0Gth;ZK z1B-s#xp;sNE9Svwpm$-xy_3;C0y8Hx4A`vDO)t9ayU$$&9Nzz^Y!7x^6tJmah|&TM zd6XcI`p-vp9p$)Xq+5Xx+I7W6?!6HQx*nb^{^2@ezS+7)vMpKA61$N&xa?&wFnVsO z$jT#7!Q9pO9%E?*c{0KK@UW`u!R1?;&Tb^vY6cux;Igd_nP21m<5g>3~8z+>HxoR-4DrEf1# zKZ!P1#4`<9J-jC?i?vat9Ap(cR%ozxHF{x7c{(R^$lYgXz+r7$v3-P`)(kZa6-Dnw zN|b3^g%`X{Y5Wk*kQA(};_19M_~9V2oW+vb3={PtJo_Q?OB_~Vi?^%M?YNw0F240E z1n^gwX^0awm})O@exn49LB=qW1K}WnRj4`~#w9(pm}RUy$JMW4RfRl7F9rui^@Y40 zr?jSHHac(}`5kO4fOpvB{VA*k_HFb676IlYxjW?Gm_2$QVoc~%w27}nF_$k^6wHI+ zdh~PyyY)0!<6D7q@G<)O7XcP9^jz0O`>{9T*+*SFBJauqa;3a!u2po=7{pe#8QHd# zx5Yqg1QYL>bm4MR%QH!a>8(HC5$#G_iP9)pQsnzlb!lhAjp|LwuV4 z`g=b~C?Z9RC3`c|2mzY=7P+c4>>*=HI zLr>_)?~AI?!|Wp>A``Qk2w?lVd9*PY49+%QT-c3>UiTwF3^WK8$5vs6aj@`WS`c_K z=@%TXM(j-rnAQ^b)qwit+s#N}}y zj+aMUu8s*dt`wU8SmPYL-FYj2sOP{;6vI*x1&3Wcw#7hX5SGV*O~O&t76p~|U~Y#| z0D??8;EsK}-R(YWuiN=>T03tA``l~TR*2WC;|B0~yd`pA*D}zS)i%K9!9tz9yf}!> zaq!YJI}NbkmycF>C`jRpT9_*bQI5-G2?M$KZ z%AlR|dKHU{Cy1=7@3g~Zp}W4nzt5NmT1Ca-+&Bkmujs!cN)AOqfyxrt z{F$eoy)byJye_uq^ja4v;QCsSTleZUL*+=)5F1`|7P)Kx4L4O zxm;;YaZ_MekMDj=yB7l@4d5ri_Uu_uBePyt^s~EAeRJ9ZW)ely_08i-dJKqr)$peLP+`{z43*C`cI6-DwqfWbf!B_f)mgC^baxKDG@$F= zV0gUMAFvty;GpJ>(2I5{S&SPhelpox@y^IWM_WZi!^5L&>IcO8ad!uIu-8?muun_D!MB(ET u@eUV{b!t@(UWq4X{y)XD?qlaeT=+l1Ecu_Xhb%P!0000