From f69d498d6d856abe42b89e0d07256cae22f77847 Mon Sep 17 00:00:00 2001 From: Arieh Schneier <15041913+AriehSchneier@users.noreply.github.com> Date: Fri, 30 Aug 2024 09:11:02 +1000 Subject: [PATCH] Handle comments between concatenated string literals (#940) --- .../bundles/assets/import_proto_cli.arraiz | Bin 38965 -> 39017 bytes pkg/importer/proto/proto_parser.arrai | 17 +++++++++------ pkg/importer/proto/proto_parser_test.arrai | 20 ++++++++++++++---- 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/internal/bundles/assets/import_proto_cli.arraiz b/internal/bundles/assets/import_proto_cli.arraiz index 2bdf3a81fbae50f7436bd25459e9c3e40620bd70..61d30c7ed126b98c2bb331c54a083a14c8283aa9 100644 GIT binary patch delta 4547 zcmV;!5j^g-umb6@0{c-}x&#a4IBah4SoN-POS>+q1LDRqB%4)MR$*sw9m~ zQ;^LKUqe7nT(bWARpUi~lw`*ZVQ0GHtIQ=a9me>uLq8%IYOfsaGIw*lCymDZ`dVS0EbEL< zTDD>>t1_m>zkkgef0q3*xc;PYp`112Fog}BO!QY$q}5G=QlLSw$_ob0-|I8TM!@eY zk$wd(N->R;c;Nw>H2aJac!0V==HUTE-MmLi|H@YI0IY1p{U?zMReVVn`NArM&;LV&k9E0*v8Ekjz^a$rkR z7$YECqA;onE^RC9v9ctu$vNbeK&3=vrfB{pE!OOBa$TC$Wtjq5pz?fq_aG7d9z~r7 zfe~U)PP&^IAQ_CL@SdtrZa< zT(m4Lr9m`n@j(j?y@s?*tlNr{9-!)WA)}~owtFas7$d_e{@>Y@A;%$AGvSe>MjU~k z20kUjH1R-dzb^&VO12m;BkuT` zBbW|BUz3AiCx0DSpQ}K5VSg1OyNUAA9Zn+qjo0qiSU0`Du7ki?_IK#e$#OQJ_$-xa zLxB95z-M-^%x_H8isX!76xzAw6BjkpVZfZW7gG zxyY6*%ga<#QfI<6Z`oHF3Y~E?WwM!l_xe8<^W|uJ8rUh|jGxR-UjN4ehZP!c#mOc* z&*PJ?v-vW8y;|u15IC@Ma-rwTsP2WCN}1s&_vOkE&-> zgImh>ES#vwv3q$8&z~Wp$weGH*k#6h3d<3-kw%@QP@lTU4m*rSH1AJPz_|nHGJh-6 z>=fqNx@}n{KI)CPLmoonQkw96#5!SnxVkf#-T#Ws$T(FWtp2&urbvLNikU%b`(Vd3!d}}B0 zRLSx#xtN|%F5t^8YZ)XLXD80&FHe~&sng9SNP#3y zp>j?)TXdC;Cobh2iLFo)QzG=`>H*4Xr32mi34&dz$cEhZPulXIB~ zhc<@&t=sQE%|e}}{M7RDDtfuux9&Dh)Sw!06MVeAI>^c+c(1-O*MGUp|5%r}8RJi6 zJ+o$%>4_9F)KD*@fsN_6DZgx5w#vUoWHKSDbTRoxw0Suk9Y2OvkaY2Dt8GK3x_DqAPMX{F4994*J38t+ga)vt3&&@k{53#~%JG-wUX|GStnQkg zIk4LAyn0aWI}TQ0j(_4z1!)Mj2MMTi3ngREZVsJe8b^`152v;sjC@jTwTn=WK8|tg z<{Dx|z}kNvW!f%}UnP6};~3@aY2oDWH?GrZ@~SC(~%v*5z)VR7n+`#PJwJUK9tRzKeUi=sybaNgNMN z>nC%P>T}wcwLILkqLyErs>4uY^ip6p+Il{=LwPoc4qU4{R@aE7A@}j_Q>4S%8++Y3 zeWIJ}j_h#IV1FMm(%z-$VQcLHKhEtyoo~#g@nLm7$8{8+dsuutN@@{XU{g(dnu?wr zBYQj#3K3VW=8jz#AVx;+Fn%8_Kstx{_~Wy~u)fWVsc&3)^>`e0uR9axXr~ug>$P|s zmv=SlA?s}UpAm1jy|*%QTT4DPa7=q>)%!5Z z`%jBne)Vvx%T}`smBuBz5_@&3t=tjjfKxTYxp~D6=wq~La7&&3?$KvnS9lcat*`TR z;|WCilXg3(7VHzKe~$6)#f1f5hYG)Tm4tR5V(Y`_o1D2S?xe3f`r@~hqFUQjj?Z7& z+^KfSUw?OC{KrZ=#QOQJI}c^_Y2fDSzv%O%#T-tex&q<0*r`;+ci&d}Ktt-4k57wmbiyQ213*Nd$aclC>l%z4VWivuY&_qpfr zGQrFFgHZ{A09}14KjV0KVbIT`yOC)a>X>@zmupWn_uj1CSyT^tD(L9d0rw!i8iu+N zet$rV2HGtZ_3ZAjge3@dLg8Qu1=2g&*$&+HZC>=Z_VWHpy|td2I$3=u@`<#6o6ucg z_9F4ruI%4#+UzAtAwAMujma8E!{LB)wPsucvU**K7^d(Yyn5xaHdsV8d0yQVELPb4 zi?+ywttg7SE6rpuQ%>sGM7p^n8N-H-|9=W$F9yao11LO4@b=yRet1Knu8Mo8D`rV= zSPQGXKpVNO*ToXjl3w@(x#%aj$dPIpED=?{H-;|tXkG9zXnA7GussGT9{k_%fL)4^Lx%`Q za3w!5BwQ);=8(emyc3m*}D;Gq*P6gwT3&<-`Lt5eV zB+W6wvLXqT@#>XyoKTxzEvD`|_kY(pAMtx$sGoIQuUCJ~Ti~Db=1Plaj^uGK)T~rU zGtn*TezYPNjr7+T2ttyv_tf{4PIIIOQFcQfO@5Wz=E(`3RFKYmqJH1nsE`vV|5xe80Jk~aVFo2d8_>w6T8sr?tN(OutmF#Enb&N8Bi#M>K9L;9MB{@ zQMHajey4tStFkXaRMU7UINjZoOTX6v{$ow#QJDruunoE_EB8mL`F{~z@0>~xq_c(D zAxU?&0kNpFw0Nf@g#01nWQTgL*VVFM%S$!s$12IspgvH)eVN*jB!&#ADR4k5XzkRN}N-reLo4A^drTO zVlT0w@>2s-ke{kjJ;jgYLSE09galJ~dG&H52evP99=u^8^vj^N#@vjF+S zVd-{YPC1r!6@LmygJ2t@PZ_g9M%SUPPBAXNd-O;yyVfk8{ zwQb!-FT=M#{QSdv3RzlJH45CGDuH&MFy4u1_!4r*Gk?zmj418w-u{ENYUu(~(>Oi@ zNt1b~ykkXoy5i%lH25h*_KMu(8}n?-`-|rG1Tpnwb%EM>|akiUxbbV z$H(s4V@>`Zja*W1vV|VXSN3S0&6Jf|VY7{c(?Fe_QRduSkKqV@$$5TVFcs7p7kPD~ z0IlUJ${F;oD~N;t3jhHB|4>T@2;y4H9!L=Y0DnM}afKYSkZy=43gTMJ9!L=Y0DnM} zB9?;!s&11o9VU}MSr3yyT?UiQZVi*~mMQ`df|D>ECX+u|50g-rGyK#LxeAD9pZ4Uzx=003?w-UR>v delta 4453 zcmV-r5t{DlumZKP0rV<7%2_iGbJ)L_V|hqgr)+Qx+MCFMx~gZ81Wr#WX~zx`~$J|D+X@ zGRT^mi*!Vo17(mzB=Q)Ek(vVAF+?OGx{ZQ&6{KggdR@T-RE%?_P0qScMTRZ*MX)+Q z3jzrPBg(Oxe@Fz99&G%j$?wks+ttHM8q^5FI3r|3AhITe0Iv&H=I{U=Lt0oqvLz{u z36L#O7}W%qwiWhRS&|pz0*YFoQX+DxX#OQF*X(a{U79sjl>%9!@_cy@AQAl@MV$qK z5#m5jx|#9x78g$YsZNEy~ zdLfLln6cbcWriXd3UBSD4gm3={37HBZyV46%im@ zbS$l;L9}b}P74mbhICA|t>ff#+KQ(x=qhIPv6WQTGi?)aJ` zm>$7UlZ0R=e?8cct3Y|-a1|oEiSp4MP9pn_*Y4L?KYh5agTPt#cj(Z`ayFp&ELCYs zfc%-jM|Q8wZ%WjP+PUT<7d6vi#Hm@$86ib2zEgEVQHy{7H`!YEo$J1r_r2Wo z1~C33tKtjGHKc4C1nvNLM{b&?RK21r>RWU$8-IIRe`N&I-2|d8t#6r5Ru2$`B(&-Z zNf@h{mF8~;neyaL_{xBas#H_bWWqG>*cTZJopCc|wz>TF#eXgr`DA+**eT$Qzr1|;;y;!+tk8HX&Nk6S z5x@L$xyaKOtEK)AfdeZi7kZJOMi+FEEb`NffA|8T3(BNa7x`&?0S~Z9V3D7W`->Wj zkvgprvyrxkS#)bGZ&qo^83JkDxIpL-z~|BBuVgu)At7N5*d@(N4TQ@HC1FTn?274v zpTTSvVkB0&bkWJ6n@zNuRZx>Yask=I23coN&r@Ch&F+Cva-lzE4Qaq}%r4W{j}GK; ze-IrAT}xUT_+(xxGUd=4!787~B{>WFp&hFRe%++S4YL^}6Tmx3^Y@sGU{vw-|P#2 z@<)hh@+yuU>@v$ehUJLb=%P+fs83yFhaE;ES_~&B;KBiPnU$$^4vTEvb*vWefAwbD zArB#ODNXo3Vx3SuUfmf?il>mQm1wo*2-mO#npQLweK;_MI<44J9y_6acv}_%_tR6$ zQQS{72MwA>hq-Ue(DUaKBYg8`3jP5X4*4>SDO~zJ%K)26p3u$?8T>1g@y$3dGyJ5sv(pVpQhtgcGnaciwb0M zv#aETmsHPy4Q3h|l5h%Fly9dy!N2XnG5=5!l6~M?+oR0WMu3iGQZYZ6f28oZNT7H! zE)sDxh)$j$wnfRJ>&0TRxQ-LPzKO0E*HcfIS}}3#`3YzE?yC|@Lyk1y zyyO#RyQl~8&C6fblrI)dGeBbi4k6LSg~f3*z@f(v(joJ-#UN{NE!8qv*F{rD3M6r= zgA2OZqE&A^aVfh^Y*~<)f4n@Ds|P4cgdTM3lP9yip-z4zwg8Y+oSzSHTTCvRM<U3-xa;AMWIeNH zl9L*|!wWFby&*$fQFXtO_~89Iapu%-*A=brpEK#a=qm*rk**88mPnx8wc z+V8v~OcnbF%N|E@rff3=+Z{XV+(OCNvztTbJj78XZg{CJ!Xlp(TVf)VqmR>bG&9E# zBPzGt)&}Jl?X$n?e~5L3a2S(PRUey`tf?z_-Pz;Yr?hcM=})YlDtLPwrkw3})9$&- zE_qmz4`&L8v-94*MjADS`1P`TXHMjvDmi_;<{2z}oelaCNT13bpQ`}}(;w+9c24|H zF;`FKKgn~m9VwufvZgo;F(=b#)iu>_o>WN{y^P~2h@vbHe?om554Ns<6yle0JT|SL z%#o$f=}^}46w!)WesQW!9gWdTf!S#5`PdHS*&sS{t^WL4BbJ6d#Jf+C4r@Q!d&T)9 z-DG!Uhl2)t#7GCXkB6;w0DPR=kve}gx3h=U`4rbteC}cKc$Cy4w!o&E_B0heIz|q7 z9u*?4TFqs)e=b0bjND=TK3IVC4)gwpCx>BuZ5UJExQhCC9Chy%6X!f-5LlbFIF8G^ zgY=N~w*1eCx7)$4_?>f8eptjiqsc+@TRkB^?~7e%Fm^Hg7_234^c~Y4TlGH7@}kkA zmR|$h>Q>Y2LZxxZuEgF^YAbhyIpS2!aIQFU1A2@$e+_P_)87O7%rgd$LcR5M-bg%x zNPp692i1ao1oclb-o3c6;OkN0*RGP#?n7*S_;iyqce1_ob;nTrwo+7Uo67O&E1S#9 zKKbho%>TA{IQNQ)QMPvC=Lh|=#c|U^G{NWp#L-@M3-|Tm7HjsVSW>ygIe|V(C z^-4!$f252h>S@SRCFchEW0L!S45)in{Fe!RU!#!^qLOF0(FOGvqVgIjN#aKxAr<=7 ztYqHH=lZcGg^hQnt9lcCw2R8@MXJZl`$k!3ZK5!wRpW#YJxZZ!a`%|jh+^xVB-#s= z9TeJ2iotQ5?oaw-IYZ|_wCXCuUvS{L?uy!De_b!OPTbcoE;8rM5|$v;35BC26v*IYXFGD+*F`zp+RGaz^{9Dn>SXnu$hXM;Z9;!*IEch^ zf4j1OrD(I4sD$*$&v&e2rpD25IN)4s8P|ZUS=SLO6(lvCRMq&k4MK^S|$Yq|nsmJv245q_?bt zRZ*gi+%@YmhcwS2h3t-H9{~lG1Ve7+f2D4p8x(!3q7y0-679H@9MSHEqvq18=7iv~2q?zaz z^}i00i$?lu3Irj^RC37OTXbnff2qjsr1~QSOU#Yd1-)Ft`8jkfkswhL$6g5Q(Wo=h zZ!{(<3B%Znl2#hLw(gfgS#SZ(O8-&>VYw8je#b3^j%p~NV=ZnsY9W88lOQ??el1~k z;7OEj#oZ7DyI_qqm}qbd)0Cf>Aqgk__$95{l0ma>*MbHounA58fV3ZPe`gAeb?TQn zGr_pvv#hCZikjuKHD{el7X*uz6D<;bM6lUT`z~dfey65^!X`vaSK_{93SEe$lLOiB zz1{hCf8%!c>WeU_E`Bh`G5u6IGD2hz)>*&#`HwE?kgvb21oBZU0L;B1F_Zq{{P zviwTUoyj>@TE5O>u!6tE5moDnI}dk+J@EyV?@{6Ta}eE{^-~e?yR=nB4@Q5C^K*s7 zM!FN2%^IGFjptpOC-N4MEB1H%nJZ*#8XHNQXMO=y~vAOHz^2-Y}ItC7gNm* zE9zSew;AK6hCVU*@gS5r@s)o#=Sus25;EyWiXX*ZVMFDo7Um#7Ri%1{AIU3uJ!29Q z%;DMfvyB|sKEsW3u0Ciu3%A>68<~=>4=1n+H_7hCIC>fDHBn6_t!cKeOlEVXr#Je2 zQWJNq=u8xFCB}P^aZE~brUB>}Uzb8bs5WM?v@C7aVbPHDg9>J$Q25-c zA%EMu*;2inzg@djg?f@CA}z}0c&~N>Z&tkp$Y1D6w*zy^kvDZHAQggDt#EU2s%(Ol zoPF-Ae)rEf=6hv3-)R4q))J!+BLej`ejaEmuzdW+)BQ6G&_h7x?;9nf$V`gmU2tFN zMbbBMvPmF8|nly-J+|G`?dbb+~PoSuWE$-EBUv7$R&@$p(3{2U^CMQ-wq zdA8;Ko;^xl#T1fPTN^WCXu6zrH}{b2Ur#$(~O!6cGRb=s%M|iyX7WZipue>(~O!6cGRb=s%NImV*LzZj&$_CX*nQ50em=Jp$W- rlQ117lOUB3lVE}hla7}v0&|6vFdZh7;FmrD50f95L /{\d+(?:\.\d+?)? (?:e[-+]?\d+)? | \. \d+ (?:e[-+]?\d+)? | inf boolLit -> /{true | false}; -strLit -> ('"' s=([^"]*:'""') '"' | "'" s=([^']*:"''") "'"):\s; //'; +strLit -> (c* (s=(["]\s* [^"]*:'""') ["] | s=([']\s* [^']*:"''") ['])):\s; // "; object -> "{" (c* objectField comma? c*)* "}"; objectField -> objectKey ":"? objectValue; @@ -137,12 +137,15 @@ let concatStrLit = \strLit //seq.concat(//seq.concat( (strLit.s >> .'') => \i ( @:i.@, - @item: i.@item >> \v - let s=(v rank (:.@)); - cond strLit.@choice(i.@) { - 0: cond s {'""':'"', _:s}, - 1: cond s {"''":"'", _:s}, - } + @item: i.@item => \v + let st=(v.@item rank (:.@)); + let s=cond {v.@=0:st where .@>0 rank (:.@), _:st}; + (@:v.@, @item: + cond strLit.@choice(i.@) { + 0: cond s {'""':'"', _:s}, + 1: cond s {"''":"'", _:s}, + } + ) ) )); diff --git a/pkg/importer/proto/proto_parser_test.arrai b/pkg/importer/proto/proto_parser_test.arrai index e81f3384..217e1c6d 100644 --- a/pkg/importer/proto/proto_parser_test.arrai +++ b/pkg/importer/proto/proto_parser_test.arrai @@ -3,7 +3,7 @@ let (:evalConstant, :parseFile, ...) = //{./proto_parser}; ( constant: ( string: - let constant = (strLit: ('': [153\'"', 231\'"'], @choice: [0], s: [('': [154\'test'])])); + let constant = (strLit: ('': [231\'"'], @choice: [0], s: [('': [153\'"', 154\'test'])])); let expected = (s: "test"); let actual = evalConstant(constant); //test.assert.equal(expected, actual), @@ -23,12 +23,12 @@ let (:evalConstant, :parseFile, ...) = //{./proto_parser}; let actual = evalConstant(constant); //test.assert.equal(expected, actual), object: - let constant = (object: (objectField: [('': 1390\':', objectKey: (constant: (fullIdent: (ident: [('': [1387\'g', 1388\'et'])]))), objectValue: (constant: (strLit: ('': [1392\'"', 1415\'"'], @choice: [0], s: [('': [1393\'/api/accounts'])]))))])); + let constant = (object: (objectField: [('': 1390\':', objectKey: (constant: (fullIdent: (ident: [('': [1387\'g', 1388\'et'])]))), objectValue: (constant: (strLit: ('': [1415\'"'], @choice: [0], s: [('': [1392\'"', 1393\'/api/accounts'])]))))])); let expected = (fields: {"get": (s: "/api/accounts")}); let actual = evalConstant(constant); //test.assert.equal(expected, actual), array: - let constant = (array: ('': [1497\'[', 1543\']'], arrayItem: [(constant: (strLit: ('': [1498\'"', 1542\'"'], @choice: [0], s: [('': [1499\'string'])]))), (constant: (intLit: (decimalLit: ('': 235\'2'))))])); + let constant = (array: ('': [1497\'[', 1543\']'], arrayItem: [(constant: (strLit: ('': [1542\'"'], @choice: [0], s: [('': [1498\'"', 1499\'string'])]))), (constant: (intLit: (decimalLit: ('': 235\'2'))))])); let expected = [(s: "string"), (d: 2)]; let actual = evalConstant(constant); //test.assert.equal(expected, actual), @@ -87,10 +87,22 @@ let (:evalConstant, :parseFile, ...) = //{./proto_parser}; // Comment ] // Comment + foo2: + // Comment + "two" + // Comment + " halves" + // Comment + foo3: + // Comment + 'two' + // Comment + ' halves' + // Comment }; // Comment `); - let expected = {'(foo.java_package).bar': (fields: {'foo': [(s: 'bar')]})}; + let expected = {'(foo.java_package).bar': (fields: {'foo': [(s: 'bar')], 'foo2': (s: 'two halves'), 'foo3': (s: "two halves")})}; let actual = file.options; //test.assert.equal(expected, actual), ),