From 5b429543db8146d8bb3fc2f906664baefc608216 Mon Sep 17 00:00:00 2001 From: Cole MacKenzie Date: Thu, 18 May 2023 16:36:31 -0700 Subject: [PATCH] test case for #1374 --- .../_delta_log/00000000000000000000.json | 3 + .../00000000000000000001.checkpoint.parquet | Bin 0 -> 20791 bytes .../_delta_log/00000000000000000001.json | 3 + .../_delta_log/00000000000000000002.json | 3 + .../issue_1374/_delta_log/_last_checkpoint | 1 + ...4212-ac17-8c9dbe362c04-c000.snappy.parquet | Bin 0 -> 925 bytes ...42b2-a97e-9961470a3bb8-c000.snappy.parquet | Bin 0 -> 925 bytes ...4deb-86c6-d713d6721dd6-c000.snappy.parquet | Bin 0 -> 925 bytes ...44d7-a2b0-1e08e64476fe-c000.snappy.parquet | Bin 0 -> 925 bytes rust/tests/datafusion_test.rs | 106 ++++++++++++++---- 10 files changed, 97 insertions(+), 19 deletions(-) create mode 100644 rust/tests/data/issue_1374/_delta_log/00000000000000000000.json create mode 100644 rust/tests/data/issue_1374/_delta_log/00000000000000000001.checkpoint.parquet create mode 100644 rust/tests/data/issue_1374/_delta_log/00000000000000000001.json create mode 100644 rust/tests/data/issue_1374/_delta_log/00000000000000000002.json create mode 100644 rust/tests/data/issue_1374/_delta_log/_last_checkpoint create mode 100644 rust/tests/data/issue_1374/date=2023-05-17/part-00000-9579982b-cbd4-4212-ac17-8c9dbe362c04-c000.snappy.parquet create mode 100644 rust/tests/data/issue_1374/date=2023-05-18/part-00000-9ab6bf58-cac5-42b2-a97e-9961470a3bb8-c000.snappy.parquet create mode 100644 rust/tests/data/issue_1374/date=2023-05-19/part-00000-1e58be07-a5c9-4deb-86c6-d713d6721dd6-c000.snappy.parquet create mode 100644 rust/tests/data/issue_1374/date=2023-05-20/part-00000-5f3e10b7-416f-44d7-a2b0-1e08e64476fe-c000.snappy.parquet diff --git a/rust/tests/data/issue_1374/_delta_log/00000000000000000000.json b/rust/tests/data/issue_1374/_delta_log/00000000000000000000.json new file mode 100644 index 0000000000..c17a48f9e0 --- /dev/null +++ b/rust/tests/data/issue_1374/_delta_log/00000000000000000000.json @@ -0,0 +1,3 @@ +{"protocol":{"minReaderVersion":1,"minWriterVersion":1}} +{"metaData":{"id":"3431dac3-0ec4-4c02-9bd7-cbf3f1b98523","name":null,"description":null,"format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"timestamp\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{}},{\"name\":\"temperature\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"date\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["date"],"createdTime":1684452961889,"configuration":{}}} +{"commitInfo":{"timestamp":1684452961890,"operation":"CREATE TABLE","operationParameters":{"mode":"ErrorIfExists","location":"file:///Users/cole/github.com/cmackenzie1/delta-rs/rust/tests/data/issue_1374","metadata":"{\"configuration\":{},\"created_time\":1684452961889,\"description\":null,\"format\":{\"options\":{},\"provider\":\"parquet\"},\"id\":\"3431dac3-0ec4-4c02-9bd7-cbf3f1b98523\",\"name\":null,\"partition_columns\":[\"date\"],\"schema\":{\"fields\":[{\"metadata\":{},\"name\":\"timestamp\",\"nullable\":true,\"type\":\"timestamp\"},{\"metadata\":{},\"name\":\"temperature\",\"nullable\":true,\"type\":\"integer\"},{\"metadata\":{},\"name\":\"date\",\"nullable\":true,\"type\":\"string\"}],\"type\":\"struct\"}}","protocol":"{\"minReaderVersion\":1,\"minWriterVersion\":1}"},"clientVersion":"delta-rs.0.11.0"}} \ No newline at end of file diff --git a/rust/tests/data/issue_1374/_delta_log/00000000000000000001.checkpoint.parquet b/rust/tests/data/issue_1374/_delta_log/00000000000000000001.checkpoint.parquet new file mode 100644 index 0000000000000000000000000000000000000000..e8aff42338e5b4886a971a4a176c3e29dc04fffb GIT binary patch literal 20791 zcmeG^eP|n3o+C$b+$8O#p)<&>+oohqR`}lo-y*DooC_@g>Dh9+rzgQy*y9MDHK@eQ+E=Mw!aJAS| ziT0NEgx%TVi6=W-67f;js3Y!icQ{?5Anq1-?-T39U1F_R+pxd3>6vQQ*{YfaL~OyI zIzgxrch+|ZwY%z@>awX~%oi)hYSKwjkm{sQYT%z-l3V%ZcCHz*Ctm#y*#)`tDS#fAayNTv#je0r*wp3HH$f8HwpcB=+=d5gHb zMN{7jNbP82aegY*c%ZRR%ugqZjr$r$)2U3d(0JenM;nno7%HZ-sX{T9oq|zywy`TW zoyo-F8JGzmQu`WF@=1_<n=z(|u$Q z5Jw>tYtb&2b32Y(C_tAcqdINO##<(9vX&;59f^Dj0#R}pA|n~oNsDySA}>BBfBtBL z4n%KBwM`tf-bC_fz5$>oImA`sYzV@V+n9LdU3GsNR4G=71p8;$&hnvn!j<4W(ir z-%u)F(8`@TX-+=(oP4{ffx%#(;Cm*xAnHw~B%Dtd%PCoEmQOaXM~PVr4~Y+z**&-S zrY*&}+}7CCR3EEo`=x&QlNR~i7EQj@z|uk)!rC@tkR3w1)+&G0szJM6BVMoRN4tco z_7JE8BhdGpb|;kj9W9Q|HjFSWc6?eq9i1MJ+Zl(dKM9)vr^DG2OE@}P+zC%Io^tJX zChYAk37FYh$N>ZMt*TFzgLRw0#^U?qqaE&+L@dz(VB-MR)0t}Vc=kKmJMA%7JPu>! zV8OB;5g*}_Y}jvYI%KAPQ_$E-fNqLNG`l}1Shn!E-n1o_Oj@U6#c>w7Yo*0ndD$sn zYx^7H4a~`q?|y-C6rQq3U~Q_?#%#O=6DjVV69lWs?HC19NG}-E@RCP9?twI14nkT5 zFpvcaS2Ol`wNhBnD~3{Ys?)}7b=qv5JXD;KzFpbLWO_86pk&BPhVvHbyhXnGy!_$w zKx;WXq_w=4gr#mU1-6;CHdzE422L&>QK;W5)Yg;LYzM=Sb&bbzV@yOnQjfg&;tSvs z>MG#8kEWw3!Fx9HYF^>(l|J>HtWxglIMijAfkXgiQ6L*<6Uo zZ%4Gs)ePULD-);R2d>vbU|3>iC z6@CN7unRVsNS!ujtK+l5go$cq7%I(V7$%w@;5EVnydEIp-?d2ZTIBPu$}9f_#FryL z;>&x9c*8#J5hmi&Ld=T&Hu{8yMkqL)x@BC8&K-snOO!513-ViW`42JJwdrtyKEt-P(C!I+RsIT0J90YY`nt>`!wpFpI@28QoH)rLQnaVvl*?aV8JdP+3h4Fh z1(nuamCn-!$5z2@&S}KA;Vhl^syDqj>#R3y@Zl(cSxQw7 z!1s}0EZUY*1AwNcZ40r<_mSKZX`=-0$98MHd!wt=q8o9kkBO8uhwoh)>&xrwn9(2= z#bt{Gywx{Im#v~xZT@*HZ~kSw_=jz3_u^w>Qf)3jF2>d7S5JyBs?94;iQiY7m!1}X zq&BbaQB&hiOo=foG(3Ag5X^XgIFeCHTz!tulHuHK!o ze13BFRd)4A+%9y~*!UZJaTnCcduwcPKXI?n-vH{SuXHES*kQcWXY4m!Xwn9(0MK|h z1u>T01cA$&wf!#u!VasAZiR?747YYWM1=-x&G$H9VNL%&wopH}6s@>gu;JYfjSggt z6{$Kb^BT(?3tx+C6tY+C8cQfz z?S(nbgflyFx2iOJGi#hkJ196lChR3I|Da5PZo~eM!C)^3G-1@(N;gdRCKTGD+#3kf zVV_vXxHBA7qEPfRKolafuOGGSeWd^G-3@3yVCtwB> z0y0Uu`D2Utq9A|x1h>IWyX+}_L>N9eZV_7r`B!@?U@#W~dh%c8LZ|dwg7|xY2!3<2 z@h%`5&bG{VHl-T`^NoKcm=~VmFz@;9rF4fNp9jbX1oxQ-c&Pj z@atc8l&%u^&lvbGTewKpzSk)IiI&}lWnfCj+jLi3=L)0r<~dZs*?*&jD;)-WcxUYI z;4+Ht9En>juNM{9*&_ z^sB$bCS9l1zx;Pn_KkkCvcJ1{^EY*C4-)K6vg}Ks&QEIv`O9Ih&vh=O@Nm4=B^v3( ze_>=eahd|%wQ#k-4S~7Z+b+^MLiKAv)g#C&5i_dQUL4{7j?izkNFNf6I{*VB;DsNo z3By36`O=DXnV_8eDM2~^L+;C8-|Z9Wb3*wkpoGfh#E&@2wJv4o8%D}48@)Ut-6Gs? zoFUx5i}Pf*7H;Dfk+4^HAw*g{kM{I2;DY*TDYYh_+8(zepLc(sLwc8R|B3PW*7%xy zYPc)%x&9pu=_28J?`Op4<%zZU~(+yz8K1U6qWkWLfK6$bOv>t!%kO9$qA z1k(QynB#CV0qNlCvGu@|(4$`#kUk|q?-3yBmV)o~8*Dq3p`n+<2iTdl?dOec__${W z6h#Tb4?Zs`%B<2grX0jB_^l{j>W8oJT~Ye*8+2*E;oD*CDs%yaQ3VzRg1w5;bp)HV zbilWONN|!5Kkx}U6&lCh9s~=>V1B#O4HN`nnRgZ`n#Q>}p!f#dKx%U!(AGAG-xZ`8 zA8&V*{l-rOq`bTU_?m;B(Y7`q1Rg?-jll?hGScnXqWNB*9q4m=0CW#_^7-98oL=A$ z$Gs?}-spI|H754Oy#9vq=!i5_=o&7lW7V0?}{Kjfj}Ja6F!dL7bYmg+8{M)KJm97=`qvm zHPY)OdQqQVoxZ*aC(uJm=JY6sIX=*h1IB46<_vWx$`M=-{ANaz*`7i&OiKeuO2XeW zpY&(!v9M=4L}Q@^rDp=y6Muf3gio-K7Wt7s(iHWomPqnih>c8<4wODxZ>BF8(dp~e z$D$l!{HFfeP}%C2fKr3wp^cH@0hO70*=p4!c6ax$jH!m&^bOH z&nAHmUjglnpm@nXVNQ1!+0#o4AU_X+XAjfl5PpgJLxt$b&{W*naX1!sWD>5S@%RW$ zoAk`AX^#;cCxP^OfqsUQQ1>3p#KRsZ+SUllZ-K@%d3b%pK_mm=lq|ClFo-@>61;L3=yGGuUSk>6;<@B!)SDqrK&Ed=|$^o=l%! zG=1-EWaOYd#$*#FUl|!3Mf*uOLug=%2jT%Sq5JFLAo(`J^R*)_!xT@5f1+OrfRg4H zTtI&UVQ_m6I^6i3LJczontu@IJA62wnDEmi_)zV$J1|kfJ_|6x3wGw1jgA}~Pxgl9 zqrt9jq_?7dbPu;zun%Gle5Dv283Y@3kP))@(cy^u=X`NjXgZ1X3`|tu2jVL}$hv`_ zS>&gg_^-%M9{HK4mH46hTP1#!vF7r4`4X3p;!Od&Z2*JfkOqQ4;hW=k_h6a;p=uau1l;cf`85yV{L1oO=m7nZ5zuntT8%}?O`4w7d{yfx&5J{qUcXNdpqX24|5uZkhRcG;0$a%IeqK^C7d zpNzQle9Pk9(0J6v!ihJK2YnH#Jie6G1Az&UXLADao9$_}dSC(M;puCJ_+_ToR1dV{ zcxgSbKLDNP{A#KPB1rGL^#F#VO7U+%nI?WoADmxfJbjJeqiVe9Uv{(+=uiM|Z^ zagfHzzM`-ua(Hku67I-h{_qh$00UD4d%Vgx;`a@pJc$zOXF-NO{f?QaKa-7x2O%$I zX5tz2cS5fmS{0ts*9QJu3Xkb~a4H49iBGCrzNY#mhT|-sK>irQL=6wclkq5&P_NN? z@&_(IUzom=A5)P+L9=TN`CC|(-mZa)^dhLaZRrM;V~JVQPXVl1HfT^Qzkw4W$q ze(%F>unb_^g|W;_r3he0`hAQ zbe8Hvsa$V+N#3If-y9E2_8Q0WS<=5=Z+k)KYt`Eai9Zt68ud1Xy12_rwRU3GcLeEQ zJ)ZdvRl+8*N|F(k^_@`gKgl7MX<>d`&E3bz9m#t z1e3ohcAOs}`4SNBpT_uh7{|-xg-}iIdi^=@GJPlCBKvX>y}ojN1`YW#K=w!aio%`*Y9p-ch<@XKDG2_O z3_GSNri1+hP==o*iHN(qSbiHrhXEoA%zv1x;{;YTK|eGn64?;M=g^T5)|f#rZm-v{ z4{rg-$>@3qCt^@HMM8u2L^d-WW#Nd%&GI(p%dP~j@4~r$hzrg^h&RkO!T({TWPfRt z&j5`R7x8Ql(2A^pd?5KkCy3mwGB(DP&-P!s{V*e=MIZoJLI6l=u`ssLUcFFf4~-9D m{$udJC)n!MKV;I9FYHnOn#vxRyA}TAfv}}X5CZTg6aOELM7_TN literal 0 HcmV?d00001 diff --git a/rust/tests/data/issue_1374/_delta_log/00000000000000000001.json b/rust/tests/data/issue_1374/_delta_log/00000000000000000001.json new file mode 100644 index 0000000000..dc3be51a5b --- /dev/null +++ b/rust/tests/data/issue_1374/_delta_log/00000000000000000001.json @@ -0,0 +1,3 @@ +{"add":{"path":"date=2023-05-17/part-00000-9579982b-cbd4-4212-ac17-8c9dbe362c04-c000.snappy.parquet","size":925,"partitionValues":{"date":"2023-05-17"},"modificationTime":1684452961913,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"temperature\":20,\"timestamp\":\"2023-05-17T17:00:00.000Z\"},\"maxValues\":{\"temperature\":20,\"timestamp\":\"2023-05-17T17:00:00.000Z\"},\"nullCount\":{\"temperature\":0,\"timestamp\":0}}","tags":null}} +{"add":{"path":"date=2023-05-18/part-00000-9ab6bf58-cac5-42b2-a97e-9961470a3bb8-c000.snappy.parquet","size":925,"partitionValues":{"date":"2023-05-18"},"modificationTime":1684452961914,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"temperature\":20,\"timestamp\":\"2023-05-18T18:00:00.000Z\"},\"maxValues\":{\"temperature\":20,\"timestamp\":\"2023-05-18T18:00:00.000Z\"},\"nullCount\":{\"timestamp\":0,\"temperature\":0}}","tags":null}} +{"commitInfo":{"timestamp":1684452961914,"clientVersion":"delta-rs.0.11.0"}} \ No newline at end of file diff --git a/rust/tests/data/issue_1374/_delta_log/00000000000000000002.json b/rust/tests/data/issue_1374/_delta_log/00000000000000000002.json new file mode 100644 index 0000000000..26fc4652f2 --- /dev/null +++ b/rust/tests/data/issue_1374/_delta_log/00000000000000000002.json @@ -0,0 +1,3 @@ +{"add":{"path":"date=2023-05-20/part-00000-5f3e10b7-416f-44d7-a2b0-1e08e64476fe-c000.snappy.parquet","size":925,"partitionValues":{"date":"2023-05-20"},"modificationTime":1684452961921,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"temperature\":20,\"timestamp\":\"2023-05-20T20:00:00.000Z\"},\"maxValues\":{\"timestamp\":\"2023-05-20T20:00:00.000Z\",\"temperature\":20},\"nullCount\":{\"timestamp\":0,\"temperature\":0}}","tags":null}} +{"add":{"path":"date=2023-05-19/part-00000-1e58be07-a5c9-4deb-86c6-d713d6721dd6-c000.snappy.parquet","size":925,"partitionValues":{"date":"2023-05-19"},"modificationTime":1684452961922,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"temperature\":20,\"timestamp\":\"2023-05-19T19:00:00.000Z\"},\"maxValues\":{\"timestamp\":\"2023-05-19T19:00:00.000Z\",\"temperature\":20},\"nullCount\":{\"temperature\":0,\"timestamp\":0}}","tags":null}} +{"commitInfo":{"timestamp":1684452961922,"clientVersion":"delta-rs.0.11.0"}} \ No newline at end of file diff --git a/rust/tests/data/issue_1374/_delta_log/_last_checkpoint b/rust/tests/data/issue_1374/_delta_log/_last_checkpoint new file mode 100644 index 0000000000..4276238a21 --- /dev/null +++ b/rust/tests/data/issue_1374/_delta_log/_last_checkpoint @@ -0,0 +1 @@ +{"parts":null,"size":20791,"version":1} \ No newline at end of file diff --git a/rust/tests/data/issue_1374/date=2023-05-17/part-00000-9579982b-cbd4-4212-ac17-8c9dbe362c04-c000.snappy.parquet b/rust/tests/data/issue_1374/date=2023-05-17/part-00000-9579982b-cbd4-4212-ac17-8c9dbe362c04-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..a72bbeb8a28e8f67751f8a1cde58c9c8dcd7e5c4 GIT binary patch literal 925 zcmb_b&2G~`5FRg16hsc8%B;MS4?VO#g^-FLSqLF*O4IZY8Vl5f3vyEnD#bK7E|z!% z!~r2r9D4476OYpe=u5zXnROB;;=+MB%-^@)?6+h2PzM`mV-4&3C=gZvDlV*bF0Ox- z0D=1r>esN0Ww&3MAuP4K|Q5>R8Pg@rYD_XDb ze_nALbehi-v?v;+P_Alt-!1olZdZOQVO?I@_yoKkH(ZjLf4Ie~|9M~XvV>Az+t>v8 z+meD7;8b-GJU@DrzuEaB?y~tkO=cX$J*_jn`9|+@i=x(ThRJ$VYrVrMwPukjPTMc1 zeXS2(@X!I*<`5^nogoL}ADFpW4$X1B#jxJY$aRC!albtp#+}wle|N-}b(o~@>!(?M zkmeY@UBhA$+tyw&kLv;7x6LuPC6wN$^~v@qebs-~8XRqyA>gQSn7lua(mTmy^+S|Q VPS4)0-t(KSW(ywD4|PaC<{x_r#D@R? literal 0 HcmV?d00001 diff --git a/rust/tests/data/issue_1374/date=2023-05-18/part-00000-9ab6bf58-cac5-42b2-a97e-9961470a3bb8-c000.snappy.parquet b/rust/tests/data/issue_1374/date=2023-05-18/part-00000-9ab6bf58-cac5-42b2-a97e-9961470a3bb8-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..bee7f512878fa8dea566474cc244acc51e464f50 GIT binary patch literal 925 zcmb_b&2G~`5FRg16hsc8%B;MS553gl5JD;nvMPkMDQ(j~Xe>|@F33$Ss1&!sak0cB zAP%VF#1r(u1sA`4p$=Bj#v0c5Q6MYBNS4|PnNv2Jm0Ih1+mEd z*|4GqK%y#GorJ?^<|2g`3lI?=K8e zcZDjX%%uoQ0vOm3RL-8XUFqX7;h;h;Kk8;{$}Tkc);d&HJNb~_q5LR##_D1Es9#V87Avet@RG4)S5=BIBmb0 z^tC>C$wT{Gn?sy*cLp4Ye_-ZjIW)udCc}CoBi9Lr$G!G&5O1|kdb>luY>P>HO+U-> zgEYtJ?HCr5*f#fyd0hATzHN@VEui#1txvXx>FeI}R{v<#3;{=t!sPu$l-^Iq%O9d- We0qMi{LpWA`4kqOq(!8$e$P#~-TR5dj9r}|A9 z5V+r@ejO`V(f#TSapxH@L68?E5a_5!P^l~Tc6t^^lQfLSD5Q{|OY&RE^S!Drh(+el zhZQ{l5>>_8Oq5e=&>epXa=7bSzbr&dh1^M_cpN2RdYMF4N+Lgc*TVZHbWY2?zcfhQ z6{?W3kRm7vU|~y8IeXgH(#I3RL1}p{FyIu0deW_hNpg|BnGD}Wafl8*ZA~n!XsvPZ zWkomXG@mDEQ8Y-Qv}$@kO8S3pSAH*JLtZ)f6uh6eG|9|A+~U>$yf1lGMk#+g*aCS| zl7bfC)=Ut*INr$L?0yyZ+5Da%vyS4vF`3zXYxcNB(U=ayWId{l+2xd4vq%-EombPo zF^4aC=zwc`h?Cy#kOT1#?c6Mf=D68n*lcFxy20qA-x&?#?e=MZZ^V~vGs!$O&$9d= z&9QpBmc=Btt^Hyi*8{$9hhuI_D80{^)16WJy8paAI9|6yz*XZgd4CzD_mau#hbWnx UU7W8z@LTOx8y?dSbwoerAN+#0(EtDd literal 0 HcmV?d00001 diff --git a/rust/tests/data/issue_1374/date=2023-05-20/part-00000-5f3e10b7-416f-44d7-a2b0-1e08e64476fe-c000.snappy.parquet b/rust/tests/data/issue_1374/date=2023-05-20/part-00000-5f3e10b7-416f-44d7-a2b0-1e08e64476fe-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..26950c986c1cc9e3b0025cb6fa0ee311b1863185 GIT binary patch literal 925 zcmb_b&2G~`5FRg16hsc8%B;MS552^v5L`i#g({>?X`B8*V}Y7*L2hb6rML}_OC)#z zB*XzBPCP*moOql*08h~aGwUQy#DxQMn7?no*>A`4kq+0;#v0ZSP#~-TR9x^seQey6 z0D=2W>esN0Ww%$EA=wwlTn;T*<~6VDXIM6T?_A*aC2JjgQY?0 zu26-Pg%m+a00SF>%GuMlD+4?s9F!}s1%{lWP#c|Ul%^Mx*WEU zFDq`7PV;$!7Da;;%2iG8hvojy?aJ>ZtjjAKpMdxCmP<1854U*rKkrLkl~BsxHnu?C zSW?geoT?7P=f{upH#=X&eYT*h$&918uk}Q4zR`Q!qNsJ7VX`jOTJLg7ty!ds)Aq}0 zU+cpcJhacXImAhKcff)8hi2|1hvvB6VpwlZ$aTWuNv}N|B-{RJZ*Rz#Z8J$f)K4e* zL7HRqb_|P2Y+L)qJg)nE-wwyzmQZ@1)~7qe>{aiX-#=b6L%>m^D1CPsXZOiaky VpIw} Result<()> { } #[tokio::test] -async fn test_issue_1291_datafusion_sql_partitioned_data() -> Result<()> { +async fn test_issue_1374() -> Result<()> { + let _ = std::fs::remove_dir_all("./tests/data/issue_1374"); // cleanup previous runs + + let mut table = DeltaOps::try_from_uri("./tests/data/issue_1374") + .await? + .create() + .with_columns(vec![ + SchemaField::new( + "timestamp".to_string(), + SchemaDataType::primitive("timestamp".to_string()), + true, + HashMap::new(), + ), + SchemaField::new( + "temperature".to_string(), + SchemaDataType::primitive("integer".to_string()), + true, + HashMap::new(), + ), + SchemaField::new( + "date".to_string(), + SchemaDataType::primitive("string".to_string()), + true, + HashMap::new(), + ), + ]) + .with_partition_columns(vec!["date"]) + .await?; + + let mut writer = JsonWriter::for_table(&table)?; + writer + .write(vec![ + json!({ + "date": "2023-05-17", + "timestamp": "2023-05-17T17:00:00Z", + "temperature": 20 + }), + json!({ + "date": "2023-05-18", + "timestamp": "2023-05-18T18:00:00Z", + "temperature": 20 + }), + ]) + .await?; + writer.flush_and_commit(&mut table).await?; + create_checkpoint(&table).await.unwrap(); + + // Add some more data past the checkpoint + // For some reason, commenting this write and flush out + // allows the query to succeed + writer + .write(vec![ + json!({ + "date": "2023-05-19", + "timestamp": "2023-05-19T19:00:00Z", + "temperature": 20 + }), + json!({ + "date": "2023-05-20", + "timestamp": "2023-05-20T20:00:00Z", + "temperature": 20 + }), + ]) + .await?; + writer.flush_and_commit(&mut table).await?; + let ctx = SessionContext::new(); - let table = deltalake::open_table("./tests/data/http_requests") + let table = deltalake::open_table("./tests/data/issue_1374") .await .unwrap(); - ctx.register_table("http_requests", Arc::new(table))?; + ctx.register_table("t", Arc::new(table))?; let batches = ctx - .sql( - "SELECT \"ClientRequestURI\", date FROM http_requests WHERE date > '2023-04-13' LIMIT 5", - ) - .await? + .sql(r#"SELECT timestamp, temperature, date FROM t WHERE timestamp < '2023-05-20T00:00:00Z'"#) + .await + .unwrap() .collect() - .await?; + .await + .unwrap(); let expected = vec![ - "+------------------+------------+", - "| ClientRequestURI | date |", - "+------------------+------------+", - "| / | 2023-04-14 |", - "| / | 2023-04-14 |", - "| / | 2023-04-14 |", - "| / | 2023-04-14 |", - "| / | 2023-04-14 |", - "+------------------+------------+", + "+---------------------+-------------+------------+", + "| timestamp | temperature | date |", + "+---------------------+-------------+------------+", + "| 2023-05-17T17:00:00 | 20 | 2023-05-17 |", + "| 2023-05-18T18:00:00 | 20 | 2023-05-18 |", + "| 2023-05-19T19:00:00 | 20 | 2023-05-19 |", + "+---------------------+-------------+------------+", ]; assert_batches_sorted_eq!(&expected, &batches);