From ff99933a395d466db2ba524527e865d36e14f240 Mon Sep 17 00:00:00 2001 From: Scott Haug Date: Mon, 23 Dec 2024 11:46:38 -0800 Subject: [PATCH] Support quoting columns in inferred schemas If `infer_schema` is set to `quote`, then the columns will be quoted. This is useful in scenarios e.g., where the schemas are generated from nested JSON objects. --- .../plugins/snowflake/snowflake_external.yml | 60 ++++++ .../parquet_quoted/section=a/users_a.parquet | Bin 0 -> 5418 bytes .../parquet_quoted/section=b/users_b.parquet | Bin 0 -> 5459 bytes .../parquet_quoted/section=c/users_c.parquet | Bin 0 -> 5420 bytes .../parquet_quoted/section=d/users_d.parquet | Bin 0 -> 5517 bytes integration_tests/seeds/users.csv | 201 ++++++++++++++++++ .../snowflake/create_external_table.sql | 7 +- sample_sources/snowflake.yml | 18 ++ 8 files changed, 283 insertions(+), 3 deletions(-) create mode 100644 integration_tests/public_data/parquet_quoted/section=a/users_a.parquet create mode 100644 integration_tests/public_data/parquet_quoted/section=b/users_b.parquet create mode 100644 integration_tests/public_data/parquet_quoted/section=c/users_c.parquet create mode 100644 integration_tests/public_data/parquet_quoted/section=d/users_d.parquet create mode 100644 integration_tests/seeds/users.csv diff --git a/integration_tests/models/plugins/snowflake/snowflake_external.yml b/integration_tests/models/plugins/snowflake/snowflake_external.yml index b24e6046..5ce39360 100644 --- a/integration_tests/models/plugins/snowflake/snowflake_external.yml +++ b/integration_tests/models/plugins/snowflake/snowflake_external.yml @@ -29,6 +29,28 @@ sources: - last_name - email + - name: users_csv_unpartitioned + external: &csv-users + location: '@{{ target.schema }}.dbt_external_tables_testing/csv' + file_format: '( type = csv skip_header = 1 )' + columns: &cols-of-the-users + - name: user.id + data_type: int + - name: user.first_name + data_type: varchar(64) + - name: user.last_name + data_type: varchar(64) + - name: user.email + data_type: varchar(64) + tests: &equal-to-the-users + - dbt_utils.equality: + compare_model: ref('users') + compare_columns: + - user.id + - user.first_name + - user.last_name + - user.email + - name: people_csv_partitioned external: <<: *csv-people @@ -40,6 +62,17 @@ sources: columns: *cols-of-the-people tests: *equal-to-the-people + - name: users_csv_partitioned + external: + <<: *csv-users + auto_refresh: false # make sure this templates right + partitions: &parts-of-the-users + - name: section + data_type: varchar + expression: "substr(split_part(metadata$filename, 'section=', 2), 1, 1)" + columns: *cols-of-the-users + tests: *equal-to-the-users + - name: people_json_unpartitioned external: &json-people location: '@{{ target.schema }}.dbt_external_tables_testing/json' @@ -47,6 +80,13 @@ sources: columns: *cols-of-the-people tests: *equal-to-the-people + - name: users_json_unpartitioned + external: &json-users + location: '@{{ target.schema }}.dbt_external_tables_testing/json' + file_format: '( type = json )' + columns: *cols-of-the-users + tests: *equal-to-the-users + - name: people_json_partitioned external: <<: *json-people @@ -123,6 +163,13 @@ sources: columns: *cols-of-the-people tests: *equal-to-the-people + - name: users_parquet_column_list_unpartitioned + external: &parquet-users + location: '@{{ target.schema }}.dbt_external_tables_testing/parquet/' + file_format: '{{ target.schema }}.dbt_external_tables_testing_parquet' + columns: *cols-of-the-users + tests: *equal-to-the-users + - name: people_parquet_column_list_partitioned external: <<: *parquet-people @@ -136,6 +183,12 @@ sources: infer_schema: true tests: *equal-to-the-people + - name: users_parquet_infer_schema_unpartitioned_quoted + external: + <<: *parquet-users + infer_schema: 'quote' + tests: *equal-to-the-users + - name: people_parquet_infer_schema_partitioned external: <<: *parquet-people @@ -143,6 +196,13 @@ sources: infer_schema: true tests: *equal-to-the-people + - name: users_parquet_infer_schema_partitioned + external: + <<: *parquet-users + partitions: *parts-of-the-users + infer_schema: 'quote' + tests: *equal-to-the-users + - name: people_parquet_infer_schema_partitioned_and_column_desc external: <<: *parquet-people diff --git a/integration_tests/public_data/parquet_quoted/section=a/users_a.parquet b/integration_tests/public_data/parquet_quoted/section=a/users_a.parquet new file mode 100644 index 0000000000000000000000000000000000000000..c35034c5cac56381b79e52be7e040f9954a898cf GIT binary patch literal 5418 zcmcIoU2Gf25x%uViIVAX^RF80vJVrzVxB9 zM@p7#IY^L}@OQl3*`1l4`DT{Zl8}pq+rvNT2;XR%56fZh2*+J(Ye!rc;tq6mwsUQV z{9HT02j~EF0{nn3KsVqZAOJW7cmQx15Crr9jsT7VLVz%!7jO&^0rUYL1Uv-j2OI}{ z4)A%v0ALU>1o#5r1mGlK81OJ)1aJ!Q2;fn`DBv_83OEB8JLI2^am`P>^&RHmbmdWa z1j8NSj{b;$TN3f3)|y^!WS?0O#@{>uPZ@O_<_7|^xIcY;bpEf?OD7gq$rI~8)-L_y z>X$~oDJw6RZ~R>S;0-gt^g&eL=?LHU&4<^$sJ_}}w!?H5OVv*JejM9n>gpZvnj#dd zGU^V^7`mZ0ku0;W|o9Zb5qUG6#+TEQB)f=+Q+B>9VkqFhL@$|`YD_h{OvV=A|SkAmjw z3C$ah0zNJJR~6l-2D4aKB;T#TDxuW({YaJ+ixjqf$2W$up@O2SaOAHtrE&Io^!FzGQMIGUFtd>TE(tv zP!x`&s8TG#0pqIz)gj-^`HR$68Z45t*hLS93REeTZD7RXM1@%UQ9fdbif$P?3iLyU zY~V%%>46l5<+CB6L<4t0wg|P^nnMF=LUrH`EIQYpGV}&k;q^>#7T16oVgQI{cKxU; zk5&UYM`@r}BM@cao{An0&11@RoDVKR;#_oTAZN(njspq?f-}@Gs=no50hSZrU)l>8 zJ&Ps+3zB5`-U%)kw1j=fgP^`fxsc8<`J175PwFDZmd zWZ!gqQH*pn!wgIBP59{Zze^krWsa_G9D7lD=qL8A&S&0BzgWC-U4G=oqX%INOZY#J zg#UARyGy>-b=wKTe5LHAe2h0mD+=d+5az0NFZ(f`G$27NO_EBisAOmlfV4z`5pfMyGSUB`Hk5j1?&D6n6+cdrih}g4dODv}Dvmu$0!lAWiWuE*l1D z1s{SaGE()3hxwuuM~`t~^jLzn7%1|xVqI~3@2)amHLb-)iMOz3IeH0=qrY=<*L6EN}rB{fSy=Uv0Gov27cgTG9uUR4-;P0_JI zF{%w-Okhwexg-^<^PDU#8pDa)r0bjtyO*cT z74+496>>-+GG_w_y*>h@wt3C9Oe~!|!S$n;PW_E{c!|OurAB}W5EyIl8%r$qwgzSA z)_9TP8nG0coa`HXVX30AD%9(|AO0#-nF&>p7a0Nt(?4ZjD z5dztW#d)~7szg=>mo9ugK+Cx9i-*Sfl4h3ByQnrTiC&S%CU~5P>cmEW;;1IJ$0pBz z#G%-zrD2Fc>@2THSI`H-pCQVu55&aTUuq36&s&hkAskl;8a+b?7E1x%qxD=gA)MWGdcA)a{8ttD?4c^vZqx zee=Uyf4IGGlIsZe?HCFl=#_h4_V@nGH~eaHil0 zefJFg(BJz;=kS{chkxPgzvu4$czt)B*Hd@P-}_$Y@b#m^{|YqK?Y$d5Ug54NdCG2w zdY?HmeEY=kA3{vor(wXohI+3>d*xw&@7u%BkNZX=9VYw1XGLD^cydUAD}3v+WT=j& zTSMmuzWC%2T$?hy4N4Kw~?vB6^D@3xHYZ|ekAKMs~KRE<9 zks&L3iS1omojaWvy1L7mysQE`Jo=K0DzBWREvH)4QBuvAQ}6KD7?r z)Ox3Q?qhcEg1gLrx^V38!A^g#}wK;op1@@*UV`8dqWs+G97xLET+?2CXh^cIysOK#)33~Jy@C$6mrpCznj3>+J zc?TC3Yn!Pxh4}(&Z7Ho5<+NJ&^d_^l4b4WKO-)tBRIa#lreroI#q;jGrdGXJ?&m+` zL!R9jYsESyuFp)akLM;OI1j{E0l&r;EBpHH<)aAkh1}ia6Zqmnp0<72H?G_M@3XeP z0bJNvU)&wXn5d~0F`nDz#XY`?@$8k&!eZ+zF;z~@fsR~xCx_`VE0f-p^PTu?l|QMk zVrs4-Z*zn3bHA8=HcoteTnn)>F4R3P-9O)Xd!wLQ_s#kL!oNJkGMTMNS{}GO<+2z9 z4=h~Bx*O}+Nsr6=%p|!Z@@mj`7X)Yy%gj z98X6j<18o>jf_6Vx=4yWq|_YrkhuoDvK}T_?2+>Njkn0_H|beuatRPznDZ9RZ1*p= zn0*YuOshVc@q#8*Vy#hMf(Ki^yykT_Xl88SJ=tTHnMyZnZ#N%6vds23?%JRAcJA8G zbhq~J;r=o^&vLV0!Xcm;gXs+cTwh=+JfWuxmXy60@6g_PoAYV*C9f~4k}_#-di(1_ z60EiQF%!?+@kDrxiwu~U!e7 BHOl}1 literal 0 HcmV?d00001 diff --git a/integration_tests/public_data/parquet_quoted/section=b/users_b.parquet b/integration_tests/public_data/parquet_quoted/section=b/users_b.parquet new file mode 100644 index 0000000000000000000000000000000000000000..ac1e17bc52791a5419844d9178a38ecc30995974 GIT binary patch literal 5459 zcmcIoUu+yl8Q)2~&N;obm*mbl6F>Nj?KqNi^{(yIZe6h+1>Ho zo_BXP`^V>V8X*c)P?gdbc&L=LLa2xW6{=JO0{VbZB~T@#KJ=jv2uj-keL&)&s(3(r zvv+pv#EDuJPX6|G=9~Hc{=RRZfR=ztKxLq1pcSB1pvQqO0$l?7GSFq9Cx9@JawIgVa?Ov{ z`i^nnUv&r`kyv}Iy*nP-=EOrItu>w8;6Ag!6n}RBo*HU@Kphu!}x=Ia#4oCCUt@;=pj${aRDtt7YAr9_Pqbckd z)J2a>%GjY6v#_K)72VSLk!X=B#PQHmaoO^yrBn1&xaccjayXCGb#yX*4trInVWH_r z9=p^+-Jv3N`C1QHWm%05j^hU+#L+Fn4~Bh1C;ViDSQhb4huUCWcO*>|2lM@5;<^Mq z(lc%7TUa4p6}=getqL*tp2)PR8x8(=o1q+eIXZrsRFBN@#i$x@bOlnu#uAOM_)!8=s*rK$-<=mNba7PnO%8;f)_^`3p<;%`z0O zaNNHDS$_9T{`$>p;rKIOTzTQt3#ET-XZii(u}>n~4e?(ce~yAEsLf`Y1tQh_3Ne#4 z`vlC_D$SG&Ym(_3o^BhBq)N?BU_m8{-vkT@6{$w4Y2YN}?Km(TCqRTym0Vh(J`&K0 zLaiQvL(nCvT1kt5_&B;c=roby3N;B8yDh4_Nuv2^B&(SLa)fnBQB=^F^w-geaTn0% zGz3E;#P*>$a5@J1JQpA(n37GX4c(yg`J`&~GTR}f3}dcfNf!2W)@dq)RvZk@s^~O& zY~HCc@VNq&Tn~FB>1`%`H^GOo0U-V5pe@l2;+#6xkQ0l2 z4Z)XO)32C5|5c>VS8x;5!iKbF;0@}yNt2-0+ces5&ex@SRrg4h`YuUU=*CeHq#k)a zyeZiV|5W0S5(a0W1R6p^suRVcTEBb&&OUC#fQk^dB-hpn^q-_g_}BWsJ}M4No}=4X z+t6L=pbrwAlUQLzeA&1_{#Pyyk6ce??5h-yj?qOry z!P;p7{Ux*x`2KyLI4ae#VPfdBByl2J1Kexkm}EoO^wu1mSX%Bnx-nQ`ZQR6`cvcec zKfp!g7f081t0F!m`6}Il^z*Mx)~c)~U2$AepjMz!#1Y}vO>Rh>kgx&81J--fe?vD5 zH7}@I5lg=7IRu+YZzFNFt!B$F4vUIJZGZ&z9hD>wL;}2aoQ$dxb+DTI0sq0c17$91 zQeB7q5l!b+)Wy|K|F}&g$?qc6HLj_x`WM%v3Z?q_j;_IG8xSj!Pszz}QgObJsD^5^ z+OH2m6?i@ai*Hl@@8~U#ppPR&)TJtVug!B42))$<9Ui)cE7s6j1dceY@h2xEkT6ic z3Bg^LOhZ8rrdgq4$cE>MhUDp1!`3hMvFs~0I}j5$(EDWOug3+4pxZY(8o1~vBbc4EpcYnY$AS8FN4hJldsvE5xomy9SS7Ctb zjd)H9B9bl_rH~!Q1tGzm$m7ABLRrsdm_CDP6 z_|;zM5*obqIv6fgy+b{{EZ;C;`Gx@_=!1?2!{Oka+VDEod?)N(xONE`O@WsK>7*rh zub4HR7+QdrTcVqGUG^F_)7R@;a23^>Cite^*kMqmMpTcPW)cr;*u${=YA;-psRk{M z?OnPsb7s8v+Ad@ETG87=HxshQl--Qm?1BDzb~Fgc9&2`{cNxQCb!`Z2YB}AMdaHcy zqIU0s+th!yaO`ivPLbZlfIa;GgZ`is+(q{e@AK;oAP7C*qfUSfLx=wwoCX7i-GF-q ztuq_o8sfqQhRq7&Ng+Ah3n}YuY2U>ZlVWnD_Zk-u%f;gP^25zO8GknXAp4BU)3Pit z0I|_YR*wf-?7ShczJOe2KKPSs|8TP?edY%I&E zVOT%)P|h^`)l6x0S)AKYvL#QkX7%&>ltLPL2bagzb2VDYS^27BYDP}4S90c9RVgpA zHNIBfbPFk2S4WCf)mnf#x!xLYsHxo9TzYAHY&6F+6U&QCMoX75VJl|QAeqt@(^DRC zb;X?VRu^G!W=v2rbvKvFo48zZS7#>tm9k*4b&6GTl@!Rarob-HEldoP<*9&|vr0ZL z&uy$`mULzdto78av8H8>dLTEI-&iqiV`XHa$uF0NSIWhzmYHTgK|D(Sia4>{Zh+luU7L$EQ?YP{mTCU>fT|Ecde?@}Z?y zhFh@~l;x?h<&ol;3jRQRb+BuAuC}l3K|aWP7brfZZ5kk=R5J)gbUE8vsKB= zY-!urVEEiGrk@WJ_l|2>sN!-xz|!6GUGi4ShI`kX|1az-K`c}GnrfB+%M%S2W59un z%lXF2a(*npvNbgZ?+s-{4eZs-QbQGu4Mm4~OXW7tr$#pmHCbmE+7(A}bveI<%M*Se zqn7h$_0g@IHO#t5hCS5G4D^thEqG-;Om^8L6ZD&4QP6L)X=rlez?hy17ENvUFSeL{ zD9}`^KAQ1@CRJgr(VB+`TfVRqbT(*aY!E!zBh6H0o3*za55QSq`?qe}pAUBK+RtRS z_V2;|0`q5~*)PEfXvUy=MF!JnnTUWH$bu!+;3aT6@V7aiWnbFzoT2Jt_G+-dE~h|S ys~9jh!n#@y@8WeI__$dRm7_PxE-`Vpry!Kdycs{15~`r~Vg9K|9p| literal 0 HcmV?d00001 diff --git a/integration_tests/public_data/parquet_quoted/section=c/users_c.parquet b/integration_tests/public_data/parquet_quoted/section=c/users_c.parquet new file mode 100644 index 0000000000000000000000000000000000000000..44a1c98616b72e33500edf6ea808bc09835c3bca GIT binary patch literal 5420 zcmcIoTWlOx8J?4PopritHp#BLCXQ;G#7%><-L=<=-KJ$`?RYO)JHEubwwo$FGjqIi zc4y9P=Hm5iY9OJ7QbI5EB2kI90tuw5RDe)bA;e3oN=P6cK%hSGK#__>DXLT+ctD8% z%&r|faiUg*RsNZC&VSB-{r>-qGcC;~qixYw+M}=UEJW3)uwM|K-q}Wk4kRph>}nIr zdxC-nM1d+mRUjS60AfHUPz~rJkOgG#3AQ=>w98Mgz)Brx703f>09^ul1ZWNDOF&-+ z`U=oxpesOM1$q?dF`%yjeI4j=pl<*@0rX9vt3clZdJ^aw_H+tIw^;?|;M+UksiOA1VozuscW1vhQ25KK z>BE!f$U_T1FwXyV?ct+Os`Mr8)z|e8epw4~c@WpXw@3dOC`6b1xW2LTkzKGnNfiwf zJpjWw4cA!^24}FN>4Y#8jpV6ecpZD9VPa86P-kQgTUcvHUkqW(p@F-@6BcGw^gwr> zFh-Y&g^m`ogjrr2$eFN+x)a`#;h|V~&LJxViIH=}A~r_5Ba(rcijmZPh8i-lbPvSm zO^q=abQh^rri@}V6_gkYq#`A%>y)7Vp*ezC$8W>B3eu4p9>*(GMdRTjRU9+$R7k>> z8F;TdO;`mRl%e}V1q$l^+&!jQ)OIKf$iX~FcqOcv2DZ^%=wmb$$-*gJMS5gTqn1tr zKa7lXhir6bxL~@}K5{rTi7gr!jAV4vqB!tukYX0-2%jN37`{I|rR6b@9wY6* z7&FZpv3R0psX_w55fA(6pHx(nVTeX3Hfze1*=8Wt?dMsCcxJ*SqAME8gyxv1c4UI% zE(;tCPZAH3f2w_jVV*hBloGmMrsheHqGK>z|M8kSs0 zOkg+90^|r;VM)b?jSfXJ05)u(-Qldu9MbVt(4`FscG@z{YM?&^QL{TP#m-Px-Yp1# z4V_|R&u?NGlQpnTjvNY|trHed!;_|I5p*>M7_0+u(Ol%5t5m_@E1|O-$BA$r8w9`* z1OWCPLU(0L1g{XQ1I&~x!*(DmW%RGmnB_LmvDjRVFa=H@i%gYW4+`ZNL`t=2+3k2N z1jf|^`N%YJh-C-v4}wqV{*fuDGgG&DvE@1Ec9_^1#ASq7wlG!mksfhQ(mP+KIfm5LT--j*~%A!B*K+OkF36L!p3&2fBe^XH*Q+x2>Q-pqlK#zBAOO3I;L_p%T^O&?4($CnF{E!OaN34U3jIk z%zF!9ZbURN)#H|X2OQAq9Dbvs4#j*N;DgHq#2rA>F&pqmEPUvF^h^&{UB+aQ$pJ6B zmwRJyd@7=%D)F+pg03#u(5!rV z^76iyk#V?cVXrK%0`ky(=!I;h>cd(SD_ECZBhH9}X2q4|Cd8=-eaDL{3i>#OF$X)V zh$^LJ6Q$8Pp^nys>b_-kG-ZPdr~qATs9eI)Biab&XfQxWP%AaDPKc~WrEXc~HDg~q^cFM+LD1_^IwrHg-Wp~# z({{i~^m0nBf=9Lp7_bRzpa*+@E-J8<*A;AcL>YG8eL=WWs!W%gMb>a3LQdVZsqMv8)!kL-7rLWuUBg0qxNAdEbVsM!`BJd+$ASJo?CcMInop{( z0k}Mc=nerMVR#7wR}g9`xZZYO(VM}}w|Djb`nLWz1Kl^?;I|fdox{Go{{%Z95BC55 zK>xo(>+&|;2Dg^CDN4SqUqm|pwy*!WBmIv?xvWo;`5P5__dF`Y3ej)eCSdxCXQP?(KQ-QXe!96-Ir~v&*D!pr?;#85WqU>o9`?u^gBt z{IOYs;c#08@m%vLMwhu&L#AQ|h&%xry4X8?ICT>B6zS1skP#69cyJ{}1~8`oD?p4c=$h z>qF2YaM4;mGIT0i*z!0qY94f9v(9aR+slRz1-_abi6`TUK1f+#OZz6~V0)VC(Y zLelK)+0wnu?ijlox`+Q#(rHPOW`Owbj3lL`bXl5^q_Q+t_Se$sE&MuPgflIEW@fxO zho>*a=jSA;F2Ol?`$9%w%IA_amVo`~lQ~0omomlF!r;`3oGm&so1|xHS|(oJ!lmJA zu40yREU(Fis^@6EoHK?sxiruBxN2$DE{sW38Jg7;HUn!?y)|D^#&RcenK@$SeRglDe9i02#B| zMh>$HJD1&*^NskdYM<0rIWy5v*RjFzxm`>@8zycY*HTi$rMi!$o9DaeES7ZpraAv# z*jI#Dj^!(gQ3NcHdOXH}0~?p}-r_=j*vB$U55w4#hZNsl)hK$(puQqgsJF4)>e;c> zYM~-gj-gF)99&w+H*jgx^<`9Y?j%h$axB5SNQOU@%mnn1i3SXL50h;E$oT!n-{kk3 z>^L;J5n!C2@HeH``xoELe|Ke-7my?YS_By66d&5s~z_*nV!}q($lcwGvoETgcqPlsEtsOC?zzsffib5T1tQtpp;Td`+$U0sUVPA1X6hb-ul3wfOBWp zj-5DBtHR1>cJ4j*p7Wh^zI$f8(^8&~HbozAj$Yh35mlqyevbRx&ZbAW7Q}tAWmgmT z=uJWHOCaY!z6|mh$a#>jfIJTJ1jq%DuY!CH&X38-PnV+8k}}j0M*g!kt2>n7?d2gh>Kdn8DbC%4Mx(0VnYqA1PyEjIzpJz8tMp3mP$$BSR{$5 zjA`JxP!220=<~5@O*ag^g7!pCXgW1W%aBN2&7&-sf=(zB>Sc`_g3$0 zy}0c=4n8sR@RLtJrtO9;tg;t(MK3o`MDO)0`|F)9^Oz^P%0fvITJ|6;R_!F#4N^n9 zJ11?3&kAs;KT#u&UrPdIS8EVh5xdK}<)B-}j}gQ0izrdkaj9*UTe`!Bffa&cozvKL z33cqi7>kO9@o%+A{lJz zhT8%+9y5hrj*VkWCx(NpuDq>Mz3h=SG#$#a{L~^!fId028L({ls1#W99>gabZcSmGGU5NTbFoUbzv*_O7G`QP?l0iV)dly;? zjq7*~{W3U7YPSs>NmE?&38kpE1ti{v15GSnF}9th(}Iw$VL`Hg&zmE_g4Vf(0}qhj zzUIBX>w!OHzR>4gc>3}S&+dUOj0NxQkA8OFdf&X<@}7?NfkLA|1fGawRW|H$e5uk7 zlSZk*3ce(|p6l9*?z%Xxy9YptWuGQ|Su7K#(1{aQY!jL>M@^qNye1OOuCC8olmLr( zT`b#n*&uQ1MPTjHa-CBAlBn69L*iw7h0Ry0b%OEBq6dYy;zacrrsZWuH{K9s&w*3q zI8{QHnTrlkV!Td4-V|M;XbSZ#MPuq#P+L_X2k?@&#ENDsIIgTQxNaSoy$(#V7ywdf zP}kxsqT^ZXVZeC?9`mQySC?oMR;!A=%2TmKb>K{i5S1-w3b_h(;~mik!XD~98Ddbn z#V$YswC7zBwQ&wQFtQhxni?Dd-V>|(Dr)7T-5t%6=eALC^7 z*oY$xi86up3W-B!Zu!R`z;}d%h$~pp2s(#uZ_+e$?p7)ciB_miBR~m4M7-0kyg;_ z==lLhg{~vmfDBT42l{xzGT}^8p(2(HU0Lq!;yR?i&z4z5PKydK$c1hP-!4{MltbsH zm&`_o5~v90U-jxV-};sUPptjbI=dS2u%KfdH#nUT*){F#eZ8)5oTvxeh2u6GTQAwwcd&&X|_i_`R;Lg|G?CapQUCtK#*|{VUk^V6gAC*1o@ln7ON;GdDK4DNDYwPet3# zxAtA=>-$@T8T-IAxYk?SGx0XHFW7drADVLeK&-g}1jkP7+2;FubfEg`DPXu~T29ZM zU3cH#17|~pyB>G}E0dmqt{zr$n6Nysx ztuos?H=8;%(sO1Ln5`&!8U$xXwm{jexC)PMxrY;eLbjmUnBD}2w#8=okvJb8?17T?G`w$83UMJm)N_W5g`|A`c=679tBgGux}ALz(osp0 zW$8$nm0%xy`$k4!$i|X1 zHVEsJcVtb&TTB-c^TO1MoGG}nHK`xhlQOC0C@u~!XP4}9*2-zJsTx_mTF#oonp~V? zYdp2M>P(DDx-yj46l(_Nq-tZlqKst^XUFH($3}ge8l6AEY&3isqZQfA8zfy=V}6Q5 zjxL)icku-5O%L;Oy6R-dawaYooW<0rw@~B_wobMRjywi-tR&z9-~8wxnNRwYs@dciiUIDn*T5G6(E;mxh2Ic0GCH=EXTkD2N!d- zh56jD&t)q)4EHs8Nb&KiW}&7C#)_=Nc^k{F9v@4rPAo|}czy zZ&FXJWvxNhMbhk|q*KsCQfqL_dYI&}OWN-@{vyBMWX7S%jez24%3qXR?_X>&``94K z#`&n{3z}4!wMJ_iE^PVCoZs1?nX!R?WtVa0DpNmuoB05m8Mc4zs{J{C=cfJ4c4Pk* z?$5AzX6pSCf`DcWt`{VLKFLh@O5YYN(fk`fXg}Wie1?6g^HYYR4_6lb{Z(lUyfykU wQzx$=k?!JRT@_5>hbi}vQmGvBA2MByhe%ib@zT!Tnm7KqHTXGU8h(iVFC4sKJ^%m! literal 0 HcmV?d00001 diff --git a/integration_tests/seeds/users.csv b/integration_tests/seeds/users.csv new file mode 100644 index 00000000..fadcb122 --- /dev/null +++ b/integration_tests/seeds/users.csv @@ -0,0 +1,201 @@ +user.id,user.first_name,user.last_name,user.email +1,Jack,Hunter,jhunter0@pbs.org +2,Kathryn,Walker,kwalker1@ezinearticles.com +3,Gerald,Ryan,gryan2@com.com +4,Bonnie,Spencer,bspencer3@ameblo.jp +5,Harold,Taylor,htaylor4@people.com.cn +6,Jacqueline,Griffin,jgriffin5@t.co +7,Wanda,Arnold,warnold6@google.nl +8,Craig,Ortiz,cortiz7@sciencedaily.com +9,Gary,Day,gday8@nih.gov +10,Rose,Wright,rwright9@yahoo.co.jp +11,Raymond,Kelley,rkelleya@fc2.com +12,Gerald,Robinson,grobinsonb@disqus.com +13,Mildred,Martinez,mmartinezc@samsung.com +14,Dennis,Arnold,darnoldd@google.com +15,Judy,Gray,jgraye@opensource.org +16,Theresa,Garza,tgarzaf@epa.gov +17,Gerald,Robertson,grobertsong@csmonitor.com +18,Philip,Hernandez,phernandezh@adobe.com +19,Julia,Gonzalez,jgonzalezi@cam.ac.uk +20,Andrew,Davis,adavisj@patch.com +21,Kimberly,Harper,kharperk@foxnews.com +22,Mark,Martin,mmartinl@marketwatch.com +23,Cynthia,Ruiz,cruizm@google.fr +24,Samuel,Carroll,scarrolln@youtu.be +25,Jennifer,Larson,jlarsono@vinaora.com +26,Ashley,Perry,aperryp@rakuten.co.jp +27,Howard,Rodriguez,hrodriguezq@shutterfly.com +28,Amy,Brooks,abrooksr@theatlantic.com +29,Louise,Warren,lwarrens@adobe.com +30,Tina,Watson,twatsont@myspace.com +31,Janice,Kelley,jkelleyu@creativecommons.org +32,Terry,Mccoy,tmccoyv@bravesites.com +33,Jeffrey,Morgan,jmorganw@surveymonkey.com +34,Louis,Harvey,lharveyx@sina.com.cn +35,Philip,Miller,pmillery@samsung.com +36,Willie,Marshall,wmarshallz@ow.ly +37,Patrick,Lopez,plopez10@redcross.org +38,Adam,Jenkins,ajenkins11@harvard.edu +39,Benjamin,Cruz,bcruz12@linkedin.com +40,Ruby,Hawkins,rhawkins13@gmpg.org +41,Carlos,Barnes,cbarnes14@a8.net +42,Ruby,Griffin,rgriffin15@bravesites.com +43,Sean,Mason,smason16@icq.com +44,Anthony,Payne,apayne17@utexas.edu +45,Steve,Cruz,scruz18@pcworld.com +46,Anthony,Garcia,agarcia19@flavors.me +47,Doris,Lopez,dlopez1a@sphinn.com +48,Susan,Nichols,snichols1b@freewebs.com +49,Wanda,Ferguson,wferguson1c@yahoo.co.jp +50,Andrea,Pierce,apierce1d@google.co.uk +51,Lawrence,Phillips,lphillips1e@jugem.jp +52,Judy,Gilbert,jgilbert1f@multiply.com +53,Eric,Williams,ewilliams1g@joomla.org +54,Ralph,Romero,rromero1h@sogou.com +55,Jean,Wilson,jwilson1i@ocn.ne.jp +56,Lori,Reynolds,lreynolds1j@illinois.edu +57,Donald,Moreno,dmoreno1k@bbc.co.uk +58,Steven,Berry,sberry1l@eepurl.com +59,Theresa,Shaw,tshaw1m@people.com.cn +60,John,Stephens,jstephens1n@nationalgeographic.com +61,Richard,Jacobs,rjacobs1o@state.tx.us +62,Andrew,Lawson,alawson1p@over-blog.com +63,Peter,Morgan,pmorgan1q@rambler.ru +64,Nicole,Garrett,ngarrett1r@zimbio.com +65,Joshua,Kim,jkim1s@edublogs.org +66,Ralph,Roberts,rroberts1t@people.com.cn +67,George,Montgomery,gmontgomery1u@smugmug.com +68,Gerald,Alvarez,galvarez1v@flavors.me +69,Donald,Olson,dolson1w@whitehouse.gov +70,Carlos,Morgan,cmorgan1x@pbs.org +71,Aaron,Stanley,astanley1y@webnode.com +72,Virginia,Long,vlong1z@spiegel.de +73,Robert,Berry,rberry20@tripadvisor.com +74,Antonio,Brooks,abrooks21@unesco.org +75,Ruby,Garcia,rgarcia22@ovh.net +76,Jack,Hanson,jhanson23@blogtalkradio.com +77,Kathryn,Nelson,knelson24@walmart.com +78,Jason,Reed,jreed25@printfriendly.com +79,George,Coleman,gcoleman26@people.com.cn +80,Rose,King,rking27@ucoz.com +81,Johnny,Holmes,jholmes28@boston.com +82,Katherine,Gilbert,kgilbert29@altervista.org +83,Joshua,Thomas,jthomas2a@ustream.tv +84,Julie,Perry,jperry2b@opensource.org +85,Richard,Perry,rperry2c@oracle.com +86,Kenneth,Ruiz,kruiz2d@wikimedia.org +87,Jose,Morgan,jmorgan2e@webnode.com +88,Donald,Campbell,dcampbell2f@goo.ne.jp +89,Debra,Collins,dcollins2g@uol.com.br +90,Jesse,Johnson,jjohnson2h@stumbleupon.com +91,Elizabeth,Stone,estone2i@histats.com +92,Angela,Rogers,arogers2j@goodreads.com +93,Emily,Dixon,edixon2k@mlb.com +94,Albert,Scott,ascott2l@tinypic.com +95,Barbara,Peterson,bpeterson2m@ow.ly +96,Adam,Greene,agreene2n@fastcompany.com +97,Earl,Sanders,esanders2o@hc360.com +98,Angela,Brooks,abrooks2p@mtv.com +99,Harold,Foster,hfoster2q@privacy.gov.au +100,Carl,Meyer,cmeyer2r@disqus.com +101,Michael,Perez,mperez0@chronoengine.com +102,Shawn,Mccoy,smccoy1@reddit.com +103,Kathleen,Payne,kpayne2@cargocollective.com +104,Jimmy,Cooper,jcooper3@cargocollective.com +105,Katherine,Rice,krice4@typepad.com +106,Sarah,Ryan,sryan5@gnu.org +107,Martin,Mcdonald,mmcdonald6@opera.com +108,Frank,Robinson,frobinson7@wunderground.com +109,Jennifer,Franklin,jfranklin8@mail.ru +110,Henry,Welch,hwelch9@list-manage.com +111,Fred,Snyder,fsnydera@reddit.com +112,Amy,Dunn,adunnb@nba.com +113,Kathleen,Meyer,kmeyerc@cdc.gov +114,Steve,Ferguson,sfergusond@reverbnation.com +115,Teresa,Hill,thille@dion.ne.jp +116,Amanda,Harper,aharperf@mail.ru +117,Kimberly,Ray,krayg@xing.com +118,Johnny,Knight,jknighth@jalbum.net +119,Virginia,Freeman,vfreemani@tiny.cc +120,Anna,Austin,aaustinj@diigo.com +121,Willie,Hill,whillk@mail.ru +122,Sean,Harris,sharrisl@zdnet.com +123,Mildred,Adams,madamsm@usatoday.com +124,David,Graham,dgrahamn@zimbio.com +125,Victor,Hunter,vhuntero@ehow.com +126,Aaron,Ruiz,aruizp@weebly.com +127,Benjamin,Brooks,bbrooksq@jalbum.net +128,Lisa,Wilson,lwilsonr@japanpost.jp +129,Benjamin,King,bkings@comsenz.com +130,Christina,Williamson,cwilliamsont@boston.com +131,Jane,Gonzalez,jgonzalezu@networksolutions.com +132,Thomas,Owens,towensv@psu.edu +133,Katherine,Moore,kmoorew@naver.com +134,Jennifer,Stewart,jstewartx@yahoo.com +135,Sara,Tucker,stuckery@topsy.com +136,Harold,Ortiz,hortizz@vkontakte.ru +137,Shirley,James,sjames10@yelp.com +138,Dennis,Johnson,djohnson11@slate.com +139,Louise,Weaver,lweaver12@china.com.cn +140,Maria,Armstrong,marmstrong13@prweb.com +141,Gloria,Cruz,gcruz14@odnoklassniki.ru +142,Diana,Spencer,dspencer15@ifeng.com +143,Kelly,Nguyen,knguyen16@altervista.org +144,Jane,Rodriguez,jrodriguez17@biblegateway.com +145,Scott,Brown,sbrown18@geocities.jp +146,Norma,Cruz,ncruz19@si.edu +147,Marie,Peters,mpeters1a@mlb.com +148,Lillian,Carr,lcarr1b@typepad.com +149,Judy,Nichols,jnichols1c@t-online.de +150,Billy,Long,blong1d@yahoo.com +151,Howard,Reid,hreid1e@exblog.jp +152,Laura,Ferguson,lferguson1f@tuttocitta.it +153,Anne,Bailey,abailey1g@geocities.com +154,Rose,Morgan,rmorgan1h@ehow.com +155,Nicholas,Reyes,nreyes1i@google.ru +156,Joshua,Kennedy,jkennedy1j@house.gov +157,Paul,Watkins,pwatkins1k@upenn.edu +158,Kathryn,Kelly,kkelly1l@businessweek.com +159,Adam,Armstrong,aarmstrong1m@techcrunch.com +160,Norma,Wallace,nwallace1n@phoca.cz +161,Timothy,Reyes,treyes1o@google.cn +162,Elizabeth,Patterson,epatterson1p@sun.com +163,Edward,Gomez,egomez1q@google.fr +164,David,Cox,dcox1r@friendfeed.com +165,Brenda,Wood,bwood1s@over-blog.com +166,Adam,Walker,awalker1t@blogs.com +167,Michael,Hart,mhart1u@wix.com +168,Jesse,Ellis,jellis1v@google.co.uk +169,Janet,Powell,jpowell1w@un.org +170,Helen,Ford,hford1x@creativecommons.org +171,Gerald,Carpenter,gcarpenter1y@about.me +172,Kathryn,Oliver,koliver1z@army.mil +173,Alan,Berry,aberry20@gov.uk +174,Harry,Andrews,handrews21@ameblo.jp +175,Andrea,Hall,ahall22@hp.com +176,Barbara,Wells,bwells23@behance.net +177,Anne,Wells,awells24@apache.org +178,Harry,Harper,hharper25@rediff.com +179,Jack,Ray,jray26@wufoo.com +180,Phillip,Hamilton,phamilton27@joomla.org +181,Shirley,Hunter,shunter28@newsvine.com +182,Arthur,Daniels,adaniels29@reuters.com +183,Virginia,Rodriguez,vrodriguez2a@walmart.com +184,Christina,Ryan,cryan2b@hibu.com +185,Theresa,Mendoza,tmendoza2c@vinaora.com +186,Jason,Cole,jcole2d@ycombinator.com +187,Phillip,Bryant,pbryant2e@rediff.com +188,Adam,Torres,atorres2f@sun.com +189,Margaret,Johnston,mjohnston2g@ucsd.edu +190,Paul,Payne,ppayne2h@hhs.gov +191,Todd,Willis,twillis2i@businessweek.com +192,Willie,Oliver,woliver2j@noaa.gov +193,Frances,Robertson,frobertson2k@go.com +194,Gregory,Hawkins,ghawkins2l@joomla.org +195,Lisa,Perkins,lperkins2m@si.edu +196,Jacqueline,Anderson,janderson2n@cargocollective.com +197,Shirley,Diaz,sdiaz2o@ucla.edu +198,Nicole,Meyer,nmeyer2p@flickr.com +199,Mary,Gray,mgray2q@constantcontact.com +200,Jean,Mcdonald,jmcdonald2r@baidu.com diff --git a/macros/plugins/snowflake/create_external_table.sql b/macros/plugins/snowflake/create_external_table.sql index 1a783270..1742ab3a 100644 --- a/macros/plugins/snowflake/create_external_table.sql +++ b/macros/plugins/snowflake/create_external_table.sql @@ -56,15 +56,16 @@ {% endfor %} {% else %} {%- for column in columns_infer %} + {%- set column_quoted = adapter.quote(column[0]) if infer_schema == 'quote' else column[0] %} {%- set col_expression -%} {%- if ignore_case -%} - {%- set col_id = 'GET_IGNORE_CASE($1, ' ~ "'"~ column[0] ~"'"~ ')' -%} + {%- set col_id = 'GET_IGNORE_CASE($1, ' ~ "'"~ column_quoted ~"'"~ ')' -%} {%- else -%} - {%- set col_id = 'value:' ~ column[0] -%} + {%- set col_id = 'value:' ~ column_quoted -%} {%- endif -%} (case when is_null_value({{col_id}}) or lower({{col_id}}) = 'null' then null else {{col_id}} end) {%- endset %} - {{column[0]}} {{column[1]}} as ({{col_expression}}::{{column[1]}}) + {{column_quoted}} {{column[1]}} as ({{col_expression}}::{{column[1]}}) {{- ',' if not loop.last -}} {% endfor %} {%- endif -%} diff --git a/sample_sources/snowflake.yml b/sample_sources/snowflake.yml index 5498c319..dfcefb3c 100644 --- a/sample_sources/snowflake.yml +++ b/sample_sources/snowflake.yml @@ -98,6 +98,24 @@ sources: - name: name description: and this is a name + - name: parquet_with_inferred_schema_and_quoted_columns + description: "External table using Parquet, inferring the schema, quoting the columns" + external: + location: "@stage" # reference an existing external stage + file_format: "my_file_format" # we need a named file format for infer to work + infer_schema: "quote" # infer the table schema and quote the columns + partitions: + - name: section # we can define partitions on top of the schema columns + data_type: varchar(64) + expression: "substr(split_part(metadata$filename, 'section=', 2), 1, 1)" + columns: # columns can still be listed for documentation/testing purpose + - name: user.id + description: this is a user id + quote: true + - name: user.name + description: and this is a user name + quote: true + - name: aws_sns_refresh_tbl description: "External table using AWS SNS for auto-refresh" external: