From b28c5264f51014df7f937049fdb321790b30faaf Mon Sep 17 00:00:00 2001 From: andyzhangx Date: Sun, 8 Dec 2024 05:35:23 +0000 Subject: [PATCH 1/2] feat: add volume resize support fix fix test fix --- charts/README.md | 3 + charts/latest/csi-driver-nfs-v0.0.0.tgz | Bin 11351 -> 11440 bytes .../templates/csi-nfs-controller.yaml | 24 +++++ charts/latest/csi-driver-nfs/values.yaml | 10 ++ deploy/csi-nfs-controller.yaml | 24 +++++ hack/verify-helm-chart.sh | 7 +- pkg/nfs/controllerserver.go | 15 ++- pkg/nfs/controllerserver_test.go | 61 +++++++++++ pkg/nfs/nfs.go | 1 + test/e2e/dynamic_provisioning_test.go | 51 ++++++--- ...ically_provisioned_resize_volume_tester.go | 99 ++++++++++++++++++ test/external-e2e/testdriver.yaml | 2 + 12 files changed, 278 insertions(+), 19 deletions(-) create mode 100644 test/e2e/testsuites/dynamically_provisioned_resize_volume_tester.go diff --git a/charts/README.md b/charts/README.md index 9298d57b2..a032ece32 100644 --- a/charts/README.md +++ b/charts/README.md @@ -79,6 +79,9 @@ The following table lists the configurable parameters of the latest NFS CSI Driv | `controller.resources.csiProvisioner.limits.memory` | csi-provisioner memory limits | 100Mi | | `controller.resources.csiProvisioner.requests.cpu` | csi-provisioner cpu requests limits | 10m | | `controller.resources.csiProvisioner.requests.memory` | csi-provisioner memory requests limits | 20Mi | +| `controller.resources.csiResizer.limits.memory` | csi-resizer memory limits | 400Mi | +| `controller.resources.csiResizer.requests.cpu` | csi-resizer cpu requests | 10m | +| `controller.resources.csiResizer.requests.memory` | csi-resizer memory requests | 20Mi | | `controller.resources.livenessProbe.limits.memory` | liveness-probe memory limits | 100Mi | | `controller.resources.livenessProbe.requests.cpu` | liveness-probe cpu requests limits | 10m | | `controller.resources.livenessProbe.requests.memory` | liveness-probe memory requests limits | 20Mi | diff --git a/charts/latest/csi-driver-nfs-v0.0.0.tgz b/charts/latest/csi-driver-nfs-v0.0.0.tgz index 6e5cfe418c4acfa4de928d4964a4e88ec89fdb2d..2a3eca635846cd345e2392c5762f59e067fe3852 100644 GIT binary patch delta 11414 zcmV;HENRo%Sg=`;Jb&$S+c>i5{MJ+8%6lrwtSQTmlkCy0)TxY}nf2X?TVH-#wQaJ^Y6MFw^i@l19@;t-UAne(H@!{ws4vjKVvLG$W{a*PKZtJdlynEcIw*DtL&OmIL0nA_jhlj_9 zGwc6p_i(rVZ-3(nGa*^>Gaf)Jdg$|)7D@03dM$(y8ejpJkg^`)Fd{Kq;~iw0#87~ynjTElA&iUO2#MxKM3MQrJbf=?1S#qmb7mCsZVGAcQX<8^1O zC4f&z2!F4_kY!XVOhOJ=LJvuv0op@<9Rt;W?;~3lVN+P3YZAw30Jch`)qp9>I;am4 zNrr|v7J$f5y+qfT>X(@S^_wGhO*^W+q6v!7Vnfuo7+7=_IH!5W z74WEKX1|1{r*I88Ek3|ueMl5^t==e|rez4D=YPKJgo2d(F;Va@fG0#KCL?;N+iDE~ z%Z%&Q0Td5n__6;JXIYw0k!{C(JEJk7@Seq)YCsQZ3!#*=6pyfkKQkfUs7Bsu)lB3? zT36WsVvwhV_t3!v^MjZS4(!WT%Ng|o`PtELX_Aadh(!+_wh&U(FJUx&H%+03F2Rh9 zDu0&IxAYAbiku51^~)0j?IOn1!QaCs#(-lvzF=JT(9yp?e|DsbvC+>kfw+gBwGg^y z{EE<#?yl*VOwtxY5f%B+oy3{K;YY%S{40@TH9Y4a5!yp|f=R4K6dCQNhfZOLvsmso z)vA6_4VemW>8Tn;$QzCHFMk{6+dkqTv456zcsL}K$f=gHhE)3yLns-yuToAJCvtic zVN$&{b5&^OijO3ElG>Ix zuvFDnO}AfvB0`G%0}O`{${uQ;vwk>+D2t)J6nslPshaN_Zzb4LGbeUHUqun4;(slT zr|ZJxAXvsju(RP;QDTx1>8MQ!B%0?Rce`&$E0+cT$UwNI!Zhol!)`KD?I^D`tB2fR zOCAZ<)Y{R8t$CG-rp698Y|K$;Z=ydfEu3bUu6lc$tSrHwexU^%!`-Xgb_o{!^j7$p zcL~q%y-L$X*GkD(n*>5fNbmfG0>NR}Lt`mZ(K|S(%HN2E;y_|? zC6p*0l2OJXNLi#Q0uu`SK*0+%ups4Rf+ge|sOdp#Z_ij5F0wf88-#l3!>_G96j&N- z+o}qRLAkB_)i-+}Gh8{HAbR^A_A)i2KsXkwbjhGbtQj^w~dZ{!ww@N0x=p-5e10su1%MU)EDe0&R#;%l+knz zsjkvWU}sUo3zz}85|b|m7=Htyk=`e$UEo=gKt!-D;C3$`(Ll}31h{y4+&TP4?b0>- zuY@GkcK?I(x9?u}JMxpFa?7z41pm$@adv%CFBal*Yg$UA^ zO(9Z3LmO{OQS*+Wa17y9y>`kXfrP{|6Z#8PMfEe-2Qj)Mh|!IuspMtc`;5i<0H2W1ikSB%e?_D1iZu^xp=xSUa$^N;8HyDfbb`iG zfSJFQ%Gk_XdZ_K=F5Wr`rg#KVFqqbA<;LfxidJ#@{2Aq|(0?7oPGh~o!LC~~WBaP` zFYwi2L;{Xpm_i!rZPYat{;mO>8|JAtT$kex9NB|tceG!5YRf)gr% zpC}lm*?+hauy#fx5-NtKs9b3)h0%`)WXx!eJxnsCIY`C{H!hhiC>1zsM$H45U?En!Hs5G>25@V96 ziK_I!uM1ziKdE$INKWXevG)aP72Bqm(`o&;E|tSokKgFZ27iZ8Ds9GuKd7CpTfJN( z^9Nw!gWfOzwNgtC#$}G8>1#qLNwNe9@|*6nHC%J}M@D!~uj6UNm9nU&+aU#K!spSj zrhoQ6`K?aG_4kV~e^;9tpOM>|>mfpesi#oW=uc)r%?`TzbxD-Zo6)Tjj%39$s+TMV z+!j>~iQ2VG)Mklw(1kIAP$mJ%F_wy-k)dKsO3P+LT~d)D4rPYpc4_G`(xQqC zm>kzo@c9%h{*ob}BI9L5m9CFDpp+4_5Pz^lfWt9jatyqKtPK^t`KA9(u^)pgM8?Zw z0(tPL#&(FX{conU0n2FQ#j<%WS6VuS*fd@ZHC4Kg0il5;uDeIEKZ&n^=46=B2zWeo z8Hp8^){)iREmvq)I&m`l!So<9fint!2VAbtbY`@G3e;Dl24Gf2I5grl97BS8Wq%|C zeRXm6{%ODR8dZm{MpHx}tYjj8R4iM)y9T1j#Of-Fda8q5z((b8GsWOb*~5|sTH?em2}?L`u0e5-)?G+8Njett5-l|N6at-tr%D~tf!&^ z80r4doZfXv8E~O=oh*)&sG0y*vVSIIBT9ZNU~8738nt2dRhU|jp*S{v{e8Wp6Ffy6 zG*`)}1JA~2qc=)TA#}(}U)_kvT!ju56478C^HTR zO8?cUQ=_pb(OzLfc~rr5W*M4CE0A+$KF;`MUqE^zNz2sDF=sO_qyt z3eq10`P+EiLRfh8 zw}Ov5vu7}906G9|DX;G)BL+fgc*Vv=ad6Kz&POj?6y6SJ=QJqxhkrfvq20Lh+rKWT*+>dAH5)c)3pnp1&lXWr8kybbdy0AnbJ70N z2S&H3G>3tYjl?jNh>}<>Vq#56W)EissBvgjcey1MX2QSVU!f71KoPg4n;+u|pd^bW zN#hc-LAct0Upg>a)=GhEue|ZdcZ`Xp#`g+_W!ANCS>S5B^?#5yW9KNvquQbx#%#jE zd`M;zUFV~Rg@m+5A9eCG7j?FYVkaI-!tf(>3QE_stlzF-=iJ6!? zwG+a25}g*b)?NXsXLSHTQG)r^j1Fbc&ilD<;}iQ8&0~m3tRatpf;RLhCaDy~4tkxW z(l>Kf$0lZmaeo}MYa8$-3x-uwqtHB7RIr80+rTPM9_4g~W0~#p9QD)`aP)ekt!JDo zS)r6mo)iWX?Aafh=Y&zEAE@0xCak4pTI@3iAg|SAENejV?mdJWSCNGw2rOjeiQ{eDlj9PCdW&5)9kh(Q-&# z3mXxB70qi}I~#gnmRyZMw2z1^Shpy7x4(@s)=S0%R4?Lk*5j!eQe(Q9tKdvQcF_6T zcdvbH`yE4g!bpTtCInI21p#5kiJT%LLxM;iMo~1SI3Xbtb(`{RUr>kz>^oe%kZMB`#}Fy~ z)V5h5<2fr@K2}l!YUfRSRGn)V_8FORYB#-F)U~TJs_lU}y1Y0+A&%n?`cVr?uVrse zFE+(f+L*@n&qT-^Q!y9Se79g?A&|(@lyS8tq<>ZwW?70cHW{|!0bKBzV&=3Bn$0>P zGkQhsk=W{EB%ZPH`YCP>@dM|KFQ9r|CMWk83&ja^qT0$!gSr}$qYvykjBOb4(9qGn3DJ&yA9TB9_{V3VTOn^_SEGs+bOUgM4#ykz60ZoIHmvb#4I)4`m zN}oiK`$nO=CJj1^FzF)N?|24nkDT=aMG0UV3ZgnlD2TetVxCl;jmB|@qKJB!CG5G2 zrAGAynEHNpK|qI}?u|ey$UdJaH^5`=O7Q8Eo2ZeR#JoBWptkDS$H7#BSYC&8)`17? zdOBpW4m=PSBDruC_!Xcg1ybiyF@IEHeCCZ4=_7eP3zl9SpNbC9IDe~yG(>{+#Av&d zSsC^Cof@h{uM$V4Emo>i-|*4G3YD)=A7fgy-PRqLw=PpGC2-n9|M&5K{_*$V$$vb4 z{Gl8C`>%g|{GT2D$M;YE%K_Q;>?auOwQWN0zE{n##%=2cHrB6qWdNr)Q(3N zW;Ju(eis_)j}rVQLn6QVP{>^f(WX z8ITvqgZhfoiTVUd8B#_BijyxB1suMIUR|8Iv0rAob3hzIAQL6_t}&MTZk3gN@UpGNQuQLY#<~yjp+Ky?IKB!>=vV zGDPq5lyvIlE~nvkavFxl$av$K4a*3bru2ro#DW}$7F$Dmo=k^f(Ms_P=BGc*YuFr! z_U4|I6CgUV7lw0I;w(MVsZ8WK9|3dntR9>Cy&#vBl3w7h>KAi4+9 z88tb$O9JN6rK_uO&RkRrs-60hT*haPtn!?-cufs#Gv`5lqk|SRpoz78sV{kK>ltgC zYiZhbC9Wux?|;0eZTFrig2NTk&8AU}+ZCj;Qy{fe@w)x*N`ap5-+#7t8g_LEE~-W)i9h=bnB_Sf@xspF~Ox>;o@}`{I`6lt>+XM|vu`Fk+Yrzu zA=(|(+5F}>*>#clBLRdx;A+9%w|&R`pRfv-aQzt{Ty z?wG>^(di$&7QehdX6J!<7#{DcE%Z4VFIcW#9u3Zf$FVG%9%;-pL zk|d$oskJ`N%LKvKP3w6~yd|sUx^xLceH-kY9<9(BbF6OD9&LZt6OF^1?wr@kJS@Mh z9idtbnMrkl+KdJf5m#-WYo|CeF@JsD14~8Sxv=w#?p!#LPL)?#y=V5=yI`xv;s1n2TJ#Y(~#BA^{&Lm5)v|xOZ$|wD^IUpr)BuE4ycH@_; zT5;PpXclSIeDhPtxbgW!p40EfPNzEVjcXSBI$?#6ymYW|eumi{fQmn36{^aGWOF)* zS(_+ovWuc7+bEhJ`tGxlqUG(Rb2lEIKCjM5>}f2UqsxB-vPczy9rOxsT}glFCy*C$ zHX>9hm5-m82=k#g<+NUokv-MkJU?^Bb?rnW`N^<~ncy+w<<9hp2jPZgj2NFuAEd0(`-aSDQ z#hllO0aJwIbc}5-j8)kBdYNMuWrk_WO6%!VW1d=cUgX#`9JmkzgpT%`n)b9{Pq^kf zx;{;u`^dF&7Lnl)bY>0#I_;iO$&F;eK9C{{$NTOf*@2SK8b}B_;erfiPZzI0FPGq1 zl5>A@DqX^`#=pGgFrT8ZhJmb}_otsi*jbb|)O`#gCq{41|64rITPv zh9G3ewgFsxIQq3o9Z4nP=dsB)*j9zw2D!pK>I#w)NnxoI7^heBL@5GFIe|D1^1c5xo~nOQ$S2OGg~eRXrPd-<;M>!;J%gDds1eO`Wethq zeI{|(eANOOvuN%k3CEa@ioAuxOlBO`OZ6~oDb62{#kkHm?#$-8WasE|`3@hsp0>H2 zo0mhBO@L1bTpwIBensdgP^1~yluHL%{v7P-f7b4c7GTSEs3PrdvRA4daj<_@r*hZA z@^b7^KM!T9)vakp1&?i5OhljrM^L8VxnWXP1FF-@h10;s^#qtk>Ti?sG@PzAf*DzA z`TlZtdOHN3k+s%p`TzYg_}|0e$J{J-qbRo9=vnO0gAM5*J4|Nf#IbPt2> zp$B5O+tdF;zq~tX&N8QQZD4P1*jf49*~O`DE;YMN)v4JYT3)pV3BP~4!Sk!i!G`bm z{hc+T%J&7mw~v#H@TGW{W3b9{LHG-N~0*J^8Xb z@p7L}yzH{K+)(xwOEMefZ!s{omceDkQ%Mi#EYdD(ig|XaB_8&fqKABn=#Es*U5cG0 zPxfqlj%N#}cD8axXN40vbM?5d_yEqv$89!0TyxKl(roYG%-4T=OlG44F`FEHd6 zHayO9pKrij_Z-8vPc3YGM&WBYk#Gy=4>mt-u+6gto1P@t*13V*DS`SafsM`sY$fu4 zvl#zPqxmgUCeit*piM{>l>iQ(QfVtXs` z*}MC=AMAbH4>Ur0<9O&zqo40btn)p+M*Q>V00E6q@a0SEyIXofK?1X!13DB35|UI$ zK^>?MbSQ%I){D**C-FLMbi3W|^W$UvZ@1g6{d@TQ*^7VfcZbI>j-DMIy?B27;=Arq z_u12D-=XgIyF2@2LSp`1cjLC|oqHfp5Cry8lv7A!Hcf!a+D$5{IG7w(u0}CYqn|&c z&U+L7<`9XtQ-Ar=at(XB=q1E}1$0Ui-j^?}&z}*YVVp&P+H@#7F&;oH+Ngtm*QRSK z(UB^bhgyGeNMjOW(bRhW-C!UfhEV3)sRYY#{IhN#U)MNI7xzO*jwOt2(3D{+{yKTd zchl6?&DWE~pcum?`u*HMSOL*$M4I9zWLVT72_C^k7RP-EIY?1LuYb|lN%l9K#Lcw} zH2+ir%K2CEV$6g*2YJo-l?m8IXwSJu2jN&SL;-&y?I=mqb_UU*aDrGOk_*X*3LCrD zFb+k5vVFzxNeHh(y;C}`Q*HIL1ItimU{ohhQ+TLv402k^-&Dq~Hb^tWLsd-(&GX-683N!tQQ3;=9Ib{MuwcEJh zY)Gn#o8cCh8CCo5QxlXvw~La+VDwgJg*q=+BCygo=;}y@dn@@~20ZbWo#` zfu?`L=;9LPm8LJ71h5APe3BdohW`e+#CoZ?x2NzBu*^6FBc(IG{4=ABU%bDBFvfo* zG2FPRrdkjzlP?dCy4^~;6rhv((Ah&QW@zehHfoM~Na?HU{~f`?tNP?C!Q~Mi-g!-n zvehY_*c7@Ml`~VTC6_soqj0EUXs}m!fecE4h8#Dc)@Opc(5>h-wqOli^gE)K1son# z5pLvG$Xf5-(s+8wm@ISmI;U##C&Yg#9*~%LW6C*8>+b^e>gS)m%!Y&7RWHkEE=}pJ zGZ`;j?_$qOuFqUHY->5DvMl@XGN0REDUSkkacW>zP(WxVAkcQ~mwMOz=x!H(KKAP@ z{5eZoJj>!Zl2%+`{16XEp=QS==h3&5GIiP2kS+($* zD|=flF0-zy3%H5eO-Vf)#R7XOW(tg@OoRNS<;$3bI36=0d;U#s74tFRSdK3kmsO+< zJ}fHcfwrVwr_Whhn?SaR%V*l2Ap~Tzcz3uP?K?vy^f&Au)_9 z*#nlD864?rS^9tLe|&ZN=IngtPL{<}r@_QAyS~Vgcr8L4V;l2P#b)suQv=bovKm#0 z23EjEKf?rKztE>F0usIa!{hU}r>{Sro%*zkg6sU@@k#&eY-Y<7!`1BN>cP`>p2t;Si3N0uqIe{hd&)ye-hOK&2~=moe*~ z>fvF0I@7$j=a8gwdP=wsBe-k7Z~iRw|C^754%rR)|GV91-C6&C_vz06e;ZE_1QmDx zOfn)H9dOb=vw6XIVplUZ-P7faADT^=reDf@b|9SfSxVHWeUmU$S+O7ZFc%1kj3<;xsSR?#L)5y&R{+NtV z{Yof~QWOXs{h~kR0zJW>exZ1RNvsz1ESlBtxhi4(^e%tEb>fF=194N)Kec8aK-|wB zVw{Zh`Cp~Yg^W_8S=yu413&}Lu7K*m2$o3~1`;5hch=GC@r z17FmD$7+A=sj{w({03TIZt+%E$Epk3np29o)L!Tm%kj%UJU+cVd;j|K}tt9O5XvW!jCSJ7{wY&PuZly${s$NX&ShH3RLzx?p}qc+E$zk2if)(o;s_yjx? zeXCrltX!Gq`5@e)>vA|x&+j%wh7QG7;*{FC8JmAtBwFNSGYZJJ;a%JLt!~?IwUT;! z!yc>6{M9!si|*c8)?jf?ZGHGN6H<5oZCcy%R(qmoMrkRgGa;k(yUyL=02v>?L;#@one-%iSpH5eO~HV&&}u42Iu+x-g0wX|NgiiH|h6BtBXySY6~sW z^)`PY`}rMNK->mnwM(6uAGCNWt=e>L2`r+JW(=H_=d7cfw*Sxd|Kwrb8y*5*;QxL0 z^!T{u|2=%Z^Z(w?Q(L9*$y|NnhEE&Us7Xs;FWdRst2aNRA&CKrF}oIM%&w7SDAGsB ztBF=h(rM}1-cN`mX{=MeiXulUXEBNyOLc#yUN^f68fMfAS|X>N)*d=@^J)_+B-WWv zuqe`#>NLbyfSZOqQhbXHGLtQeB1{oZ6?@`Dt=K&$wXi}VWJ$`XMzO$_(2)U~Cp@oU zrZcu!bX3V}BFC*g^d3_Z$8JXSeQjzFERY0WfzX+!$`r~4DxKDLPVD<8`er}OqrnGN-#_E}9h?`vWm}q?s=(Lk_R<&3pVYmkg^8{i zeWX*MCzxNEg=6w@*?d2fLv(U^YOYktO6RGDNhsQT);+Gk4qVtIjuCt!d4hM$^-V4> zI?qs^TTiF+(ZE8i5E&90RoWH#tzmzvo6b@nY1vvN3Bn3ULa4s&vE0Z!o5|tO6*swh4Wzg-ZL3aP)xW#ImAMGiy>Jk(llfr28`jtDZ{;Ze3XFAu zEzehdtG2y`4a{2Od{vgR{oQQiTR6X0J5kqkQhu}3d-J#7{vok7<`~fzn|yz~3$nok zbOR>&yU3;W5TC~VA0OZ_u$GV;jsNQ&KRrI0iT``H+yC9lQ&R@&DUnPTCbfv_fx<;( zaKBPz{;p$g#-;6UJ8lBsGA?Y}m^#8l*DGg4twVPnY&dqD2?_j?#Zc*`E{!ueW}N(H zk`j-eJk^r3Ctl#W#T&`yg;Z)@bW zg(koUy1)ozdmpu9BJ{u4YA2-rFvFt$O7mCdB$fuSM$0;H&C<=Am{ea|)hK#j%eF+Q zRMRCp>}$t=hp?|fJG8b!-tIoCeYgWu%*{@#j(RcaH-XE6$@1BN(h`3d_?psp`>a9R z%XD~QKSMzj_gZ^;eK0oy7&68qLR}u^YeVzZn;A6kvNXF`UwgkgmN$p}{$TD9y9q8_ zOVJkC>&ljooj3O?SvKsIeh95DrI(X_N?KSy{;bGaU;dzC4(AN>PWxZ_nXmtE#{sm+ z|MTL-)4KoX#qsX^&#iwvEBk*ge|U9bTtB&Q!edeHwDxw+o^3gMHr~bDp0CF*BDW+P z=X4|B88t~v@5}DwjyE97>z^Aco||q!?#phY?DgV(huve|#SUcq4c?1=S5tRq+gx5= zo#q@LqT;J!%$n(Kd3WQOZ*F0eFqaXoVx(DjgRscQcoX}BCAfe8a(e@3hHgnX83#`8 zzuB_LZ4zbM*k2cGKd1MS^OY;pa?W4t1ABWkf1~4j6KiCK_SYf$nO?88F>c)dxdzIc z_5U0m9X_l3e~x$gA8zNV`+v-=>xhZm>)>&wG-ZkDJJlC>R@{cl2!&-s$|`|b$cC6y z0x>EL%mj9!iZg%QadaQM->pb@j8OBE-ONWJ#T+LffolxPUkyImKVQa(VSVBDZ02*NiBZc{ax=EZRpu820QPbnWx78J7b6wGNO#b_S#sO|Lf>^ zjsG7XJv-j<|J!)>&;?d%1=Y7MnVC1Y%?>gWD^7q?9A1Cn5r|G6{axJqD8>-SXcV)7 ziB2VSw2wH%Sl^hNVmWr+V;Z&g5CwB&3O!DB5L$jg<^OupL2qe1MU3j23MG^RM=_z$ zX?0HfAC+){)*d=xNx~?4f6_-0;iAcyyrtlm8SGdQbuy z;P5I-wM|Y_w7&0%>$LTKXMnF--*;q^w!Z&=+Kk5u%LF<*eJxs@lrw#&V24D24@^1E k{@&_LM93mI*fFx*vwL>W19<*_00030|Ejxp!2oCh0F*XQH2?qr delta 11324 zcmV-CEW^{VS=U&QJb!<4+c=Ww{H;%cC-2oHv!*0FPO_(0sjG~gnf2R=v>(SNlQkzTAG2SNK5^q(J?)V<%$mPAn2=G=N5Tqr1@v1qmWfroaO_ z65Z1Y=CU)xN&L<3o?frldvf|H@%~i-tqHq zP;ZAgG(MS-n19pTx~+QW9>{}J@(#EljP}v=sD;zC_!(mfLbf6hAt$NS?@kBj=;DU~ zI@2VBMvSAsWJBN-B#0Kp3G`8wq*~Jge6Q2%^je?0$NR^9YU_WB;|#=(8NlN8e{|G) z@~po8kDu+=|9_o4VJ0L?e!@eDMIU|q)FKHUL%)R(LPIRz3R2cb9L7XQKI>fnNpuL) zU+GAgV(Q_LGC?HcvpyP!yhB*``cEPd5|0TT2ZD?PAsNSG=+co$<7`Z5QCZ?~ABC7= zJ}cg(SsY)on1r)FIv-sydC5TlmAXJE$R%e}qWB7MKYtcAl)Ekpis;i4Koybc)6P++ zw=yCFic>LRQf`WfC;`+E`P1r%#AFH-gt+8v2%h zc*8+afO5XXb5Qe$9j}TQe2~B?j<=eve5QhsQOQ{xZ#rWw0enhAco~K)qf%iKa=;S$ zNb(HO0e|}I1gQS|5ZST_o5BL!kT^y|uvH>0gl8=4paDoE85-eO03svx65U{`UuFW- zZ;sdv?Wp#OE*ODLN5bh=ATT;W7dQdLMrdF$u;?mqPVAFe9UiW%LbwjfEoT0!jVy)Iht8 zF@JUN_ppfx;8;#B8JB%@{O`}69;;$({1Z$e?xUwIgl-tWCUmU3Yx*UVw1rSaMLu+= zai(zifp8)JO5{Wh&jmp z--h{ih&V{BB@G^p2qki+Wt<_^0mKkW#((XrloQ5@oSnv4hzmV6VkRUcL1@qkLQW(J zaoj>kvKTlvQ?G|HK-Vzqqqe4c&SL0X7kaVcBZ;1*w&e{hRkc;q?N=X&kRty8qY;F% zkJ=Y(5KbVYExn+Ojc6$-9mI zw6w^WV}-PPz%`8d{pD;))>M3_ePFKW~@3UzJQXt_k1NjOC3 zBP1s)dK!K&Pwymn57?j((UjwriM@+*`K4&pg>%vx^w;zBG?viyO)n>pyp-@T)a5x9PMqjU;Itt?GzKXZ2aF(%*$g5j zAhr3Z6svC;3MUX=*MDoLED}gaEHiP4(ipG+#53A&wGdLmBa|@;N`Cg@<44rF0$r|i zk-tTsJ}DdwupkgtiGI<>X<8RSR7gxcfY7H;Z4?AISca3&J^+v?B?u|knGoZ_T{1~7}Z4m&q=}2@`u{H|IFG_Dg9TR1s*J$UR@i?lI zZ>GRGi2(hswhuH!&z|T%NixVrBk}>YgAy3fNdIk6i?stwt8WvClXB05ym*=72AKI< zsmIN{rH|S^?tkK~lVFO+5Cy|otyXUSZmMXN#E&0Qz6#wz>@?OZ9PGL^Gqx`aKfbM| zMx`SrT|HTqqP8L@>q10&u#sec!%6Hf@jD_k zQro^`J^Tion$6n8e(sg=bv%vyKjcy1x+k}Y30TPgkAF^{J$urt^8e%B(X-y3|KG)9 z*Rta4`D*o;zdF{;v#?B~rKqT_mUR#WEjQ>0r$hmxdjFnpfTu^TYeJ(wI&Fa2Ne7)p2CCWRFe7mUyuvT1z0RMkyB4P;z^U4ILsVt2ebmmaK4T&$*{~C` z#4!M=MSs%Ghr@N|YnxB^*|Vp{bnmgbJNka z-?E=Y9x-92>8k!(YOn2(u^2G*;y6MWQMf5_qX3bp`o|BEoDhMmNm003SQ&(%`e$r_ zmw)e0(TKAI-AqV0F?9>@H&~!YjOVu1$Axj#)7KQjj)B=x+W{gZP-$r4G{z)R6IJQ| zKo`DzcUtMbketwQWA6*pDt274pws$qT`GsG9>0N^4gU_IRN9OQz)(9|w|coo77xJG z2fbkcYNb{jjH?_))7OMhl4J=I`vV>+hFg{O+bW(Gf$zW(VxzPnjduc>xw8}G^1N39Lb7hR4-W!xGky} z618iWsLc}Vpi5%}p-cjj6D$=!BO}F@l$Om#x}+jQ9LfyG@yv-#(i1I=r9~APGJiR# zq2TkMSo|eJKt;yOm@Zu(b3iF0W+7mS0EZLAh{qx|hVm}5~h>S8+ zt<$c#TdvTqbmDaWgXuwJLU$DY4u7~@pXtnL0Trk(#|^-&ig0AaYdC=f_sduc`ttJp z-IGD(HL4C@ji!h|Sjj~GZLw_i?iz?96RWEz>ZuNL0UMRW%@l(#Wrt;Ahlufl@Fi14 zI0TIx*~DG}mjoO~u2$S~RnmD|>)W@g`*u@n%m9YHTD<}yJ7R_zXvM&?VShap6~IXM zhvxLIgW7-#rR!vIq(s#exRNy?8&mRI0b8>K)u;`tufo)N48^gDVK~%FI>9r9VxC5tNR72BCM8cPXf+!qL>81TG}}Ja`wFcarjhYIpHp;-6Npc-fLosc zHPRwb1H0N37mvCQe4#I zZ8=Lib3hS#tBH<4tO=bYNhWnz5+2EdV=~$)`!NTjxhwi>lltu|T2V)s99n5FxU}sJ=r*bPG@wsxyV731Q*U-wHnN z%$~uZ0q794rM$kIj2H-|;T0Pf#lbz>I3MAlX5#S|Z(soV)3{ApflwOjHV_JK=ubtM zuf8k8bbe3g>s+ZH%uig?6xpq116(t2dfJMB5GAgvs(>Q6@_)da+js-mEhp1_G$S?l zkx_UvnxE64*dO-MFYU&a-~M$;%|=p~tJ$zYTf%w2^=uI}rIGoKzNe^XFqiEwePDEp zN^=v70!p%2k~A(M z8-%M3_@x7*Wq+*{xc15$k9^0NSZaK)U|42d`<4Z+wp$P1Gj@(rJgP0KVaz53&WB_s z(M>*jSO^vpEmBKl5wX0}9pBYv-+^~SH3}_aMFm@^ybY}KwlSPYOR>PO2sVm7!6^h2rCn&i8+Idt=gzSE;hd`WkL|OT@VmvoX8m>LS!HgQMibty?>alN92C8_5j{Bp`*2leGQgX^`|W% zaJOhq&2ou7(Z_Zj>sf<=v2IvI=M+UViW3qdQMW12_XUMmz@fv%3#m3DaSV~tPi>n8 zGM>|_4{kDP~UFpxL|=GNaej z9?GpgM&cP8ub<);5I=Cv_!6quWpZ+lu~3{qN99NMQ^8(FS zls4A@l!al&MLrGHMO~Paeok2-nCzM|v)8mOEfn?`@CIh61cWG3MAV!y zNKE3w155Q$aNs%3HEg4sC{dAJu`aEY%`t4=)u|_DO}3FVM37rDu^Ug zaJ5g52tvy(GSVx8<_1hI7C&ZMZ?JHO&2Lm#&O4T3y}D2k|udBR?}SZY*X zfT{0imjrb9>D~yWg6#8&asxaTt^}Vxxs4jBNzAMB0BWnAy&ujbh}CsSXMY`dz;32P z7VE$RaVe5ZSAkyvYEmF|HWxz`#%JC*kv@{wvta4P@u}zljf*!rNJAu8PmH!ZnUztG z->IQW^eS;w+G3?T^$i~_tWfz1^)aSp+il%}Me8!fQUa%a^nV}y=O2F$9{3le?0y@|9|nv$B!R9`sFXLe|&rS>R0mk_g`q1T$^9NKl%k;{kkHM z$B+Nx4|Aw|2%PXvwc}$vFzdcZ664H{PfX6+%K|+_t;SkL?{?tZeWLp&8`h437iKk+ zTVNj=>5mfpDnlZ_`cTMy2;?1wK<+g3ad#n)y9{;wl0qCGY-r;UBm_3`jR%u36B-mnyRlzpx^qArLLgHm_HHnz+OKBD-ej@1c_0eTu2uRRrCR;^nrFj@v#KZYPm% zJBoAr!lT-D62rEA#M<`pX*-BU+ePfz{f;!-CZ=p3J$9d?$951awv))P&m$$4qp*dL z*fyf;3ynN)o#Z+y78$q>gywpBAYFE)w`OO`ik1kzZg$w4QT2SrOSL8B2 zb7Yn0Y{YA7V4FD)8Wp}2Zet${ zo{#Bm?NEXvJ(nDVbB?w10mEgdiqF3m#itbNFmR7u{7)4W7~nK4k%`6Y z^B!0#>du9|Uv%%-$>9L3BSk$HrJ2kX0R1x zQ~4}3_pG8wd0_6ufab$>M<%s(hhYUHn+N2M%x)gYs~h5eU_1|hL)NL6)JByUp6`KM z*d*qIe{m*Rf~5uHi&Q=xoG$<=fg?d82(cT#T-A!(wn4KUX8p%(FRm=pB7%z9Gk39%CEMvs@T>3!8Oa7h-S!Pbnc&(wZ;TcB8yk^ld z7Bi?7&zX3l{;nQ}xtTC0tiUr)`I&63{i_+Vx!L~p&;R~`#9&szzyA3@itGIApa1=R z%);w`{qui+R!;0)jTo`a(IEG&Fc>>$WU4dlgILUYjTkURI8GUn<# zDTJM6X+zz|5OQK%c*!GDJYvfCt`C+e|c=|o9n2L!cAe+m&jhx9MU zzcylA9WN!v;R8B0TXZ7SC_T0-Qp~8tA5=@B(l(k#RwsOcB(W=DD&)MDae8IBcO1jALbEbiBg>YEAn55 z`p170|90_TcIc|>&)`fetqP*l@uPo#-V1t1LGQ=|vDfSC|Dm7Xo;GKh)3`RUH#h98 zd_LR7sctScyG_-p*&bS6wFU`)zrVrri^{=<@Av(kHKF7YLHj#Azrs5_S55P_FaX-u zBO4OEVP-e4L2qq}o=dm3lu$kcJ+F`CPN{jTym5^l)2%Ye`OM|p=a%zoimPUuTdY0V z9Eg3?TT8K8=FeZ`%O}lFe1~WFMsjI%$!5dYyv8R=xEsHynkVByvvV$giT~zUsuZ>{ z2O-vQdhIVB-JghA`$WtRPr^L(6EOReFL!$KWq;!3KA(8mXK%Ts>@AjLw#wgPVC*b| z%bKT>9?)5&ebyB7>{ClT>@!6V`4rJ-QaSf2c2+#uv-LTi9h}*FXcqS z9h^Vd{Zb&@IuEdu$p7tP{I`wf-#Tu8ClUJF#Nuxgb-zt~ z{XUX@?au8Z>GzTJ`Wk!(S>K)!|Mu+e3@xD*@c<&^Jd)tWZoy2GF@8f>3_i;ba2B%=oC(n z@&rL(FGV?nG-k5|sI1+jl8Wy1sB$%mi5mU*5p~{~@HdA@w4M6Xr(TVX8V$nt&^t(1)Q;CjMxjfW=ibEQc5R0bP^X~=&0WpL! z-%cf1hLfLk1NpkfX}Y{0LUJr&Y=fo@Q}NfyOTL|@wr;+jEC$6GuF&si2ErPMRwL3B zHzA{<21)Q3F0(iuK*&Lg5_U>+iWd_mn#`@nsVd*l@}hBf2h4JhRmVEz zgmEHgWvD{~8O;@R$tleACq^ZFkmZyK2-R-mlCu%1DsF~bTxC@4zt2oi`obDAB(!Vza30p z9Jl9Q25KU36e*4P#UCEMJUhF3H5feh6pR5!zynSHgwf?C$}3G@Gznl25cnWD4h;Vd za*6dqac@uI5n!2d2*yfheDP;S8NYmY1!0VTNn*HhQ%$uXSSDW_9rt>bbSXfm^`Wze zSj^DW<80I%^^ww7)&Dz!h1d1TSAxqUJiPOo7G0u4ECLaomPb)j3)Yiz+9y6AUAEekk0t|HvXt&p|ey`k~!iZNN{?sZPpRw@y3*Mme$_|=;cp8d6|uyBWN{dGbn59oY^$O_AY$4^ybWFM{TagY8uwq3vsM=Y<<6CT*Z6> zIF^%3#$^?019|%5c&)`(g}sRg-gtd%u%Ij0J`f2^M`D$QNk?M0%W)3i{E|>bHe7m} zK(8;eg|n1$G9@vLD<_UunHe1Es~iS@8@zvc_WJx{?jDEbQ>VeiF}u0Uk$5G4LL6i3 zC$3_%e2uArXj)l~DntV-VB?=)3b9}4t6e&YUi{(F#hbHN@6XSC+C{;2{_yB@aQ^<) z#o6VX^NY8>qQ*L{sMy8Eo+z@$FHbeS;)4`;LWDN4duki5P!{n+%45Y0#E9^QQL*JT ze>x-F56m7+4;VFv7wmn_4IpiQ={M9II4h=ZB}K^&zfV#6fius!Htjz2o>Xu{$F-4B zn_}j!C&*iNr_`^fWt4pAce_mjtioZd=!`IlGY)Sj9K?jhQ6D|6d{;v4N09YCXu*aG z8CUz78p&XU+HdubjLRl#(W*jGUoej2Gl=nQU_y%e=tEU5~HrKqj=a8gwc1E~%J>G`@XQ}_sd?a+lZpr`G>pkuD>i)lz zXV3Tkzq@#XAgH+e=8}ho!{fWtVw?UZoRAfa^=(UBAXcNIwjIi@@{M_@m?Xz_KD?z{U$~Vsv|NoO`C$;r|^!)gL>9hU%zl+E5|6@oP z9l!?ee@!E|>-aYrVFndfyb}6^j(^sla^ald4?j~p#Uxe>dLGT{_gpQoemWQ6Ch-Hc zfw(R2f32AZ;P*d3jFYiG|EDywkWp&1M0?b72x!RJHBjvsWtntgAOX@zK70F-0BU?! zbsg)u_#94S-SB6BiA3!bJ$qRL9t*Um%BJ@2TWEc?#amq+>n><(8Y$*ddz4cwColf+=PRu!9n9rLrP8>ZF2{QUi^_u902 z@$&VnJ2S4XVBhyl^qq30vT|jb7lUw%Zpz_2J>Qhi?atY@VuSbG!L)Z7HnB*w%wJ{{ zkgvm2w)a8ZwGV17_4by%PuuyPZ&?<7e#cjX#W}V0;m=G+-TgOdZOdEjiJ}>$rI^ly zjMncuc83FhWZ2EicIL`9gAD4|Y%K)ZiROAb$1K+qDm%lMj_1@I4jRtN4IVN zU+DkIL$H{|v+PFr|v1EJM#@}AP{t1mp z3`k7ajX)E2gCs+dK0;bev`~^xOV{>(LL^CJo$^r>ITAUGQOsDXGj+Pz70@W7R?reT z>$DEgxtmv)P$99-e1Jufo=~SD#RA+k#F64#WSE(3F%)5naH`l7Cu+qWIH^Sy3L#5U zMm36m1-68a4cI*4aRoD-vB9FFO5PATX&s< zg-zlZ!3UBjc*b1c?KbUF_WEW`?t5us6~U6J1!q?4AEOMR?mYmp=bD z`nI=nBlC3b40n^`c#!cilvz9Ku4HpTlHXO_xtGJ8@P2?h;q#GEn!zLA=R- z$>+sxSYNllmZQ9DFxDluJYV&l+V++xa7uIxwdkW? z+SX0!NTpxeX%Xpdjl8zd6!=gV7=vsdqIOJ#{`W@hgw!8qSkzx>{>q%H(f~GpXj$j2 zSh;x`$8b$AG*_H^EYPw>FeeL+~5cM@^ht^ie+dV|J51#=QbF<63qh3toQuqHnKi;4JxszvY|IgL;FHeo@C-+TwEXtkM!QR=k zD`(HvyO_K4_4q~Pj%4G4ZUj7|CW+yF*`0jm4an;H=a!1+wi}T9vfC(sd%b+$VfUDy zV+XSP2JgkbtEs#5ZLTh_&T@_qQSntVX3g|=yt{GCH+Qf}SjY(1G16?hL0INvyp8?A z3fzCSy@4}BcO;yQ11I<2>{#SBiLzbnuZy*x(|gJJ%9Uw3=dbmFy*rw})$zTJH8My0 zn-Kk6uQ%ElH}3yj1Lf_1`hSj&kDk{3KPUVA4|ns_{Xb^bb;LyOb?`V-nzFUF=sf3OX5r-J-8#7ZZC(e6Jqt*eUV2(_oN2v}%%P*+=UynQJ z4UK1rQC(A^gi_#tC?*s-th8w*)lA;dPd3o2F=e*AX{q>$}bnU$?&N$Rurj_y4pRj}w*&bbj_q yv^ptg`cAP9i2!#^InMsx>P$t*BIxcJ+5Xu-`{w~X|33f=0RR8d;eq`CWB~x!HY|hy diff --git a/charts/latest/csi-driver-nfs/templates/csi-nfs-controller.yaml b/charts/latest/csi-driver-nfs/templates/csi-nfs-controller.yaml index 82332a0a9..afea01ea7 100644 --- a/charts/latest/csi-driver-nfs/templates/csi-nfs-controller.yaml +++ b/charts/latest/csi-driver-nfs/templates/csi-nfs-controller.yaml @@ -75,6 +75,30 @@ spec: capabilities: drop: - ALL + - name: csi-resizer +{{- if hasPrefix "/" .Values.image.csiResizer.repository }} + image: "{{ .Values.image.baseRepo }}{{ .Values.image.csiResizer.repository }}:{{ .Values.image.csiResizer.tag }}" +{{- else }} + image: "{{ .Values.image.csiResizer.repository }}:{{ .Values.image.csiResizer.tag }}" +{{- end }} + args: + - "-csi-address=$(ADDRESS)" + - "-v=2" + - "-leader-election" + - "--leader-election-namespace={{ .Release.Namespace }}" + - '-handle-volume-inuse-error=false' + env: + - name: ADDRESS + value: /csi/csi.sock + imagePullPolicy: {{ .Values.image.csiResizer.pullPolicy }} + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: {{- toYaml .Values.controller.resources.csiResizer | nindent 12 }} + securityContext: + capabilities: + drop: + - ALL - name: csi-snapshotter {{- if hasPrefix "/" .Values.image.csiSnapshotter.repository }} image: "{{ .Values.image.baseRepo }}{{ .Values.image.csiSnapshotter.repository }}:{{ .Values.image.csiSnapshotter.tag }}" diff --git a/charts/latest/csi-driver-nfs/values.yaml b/charts/latest/csi-driver-nfs/values.yaml index f47b11f9b..518706d18 100755 --- a/charts/latest/csi-driver-nfs/values.yaml +++ b/charts/latest/csi-driver-nfs/values.yaml @@ -9,6 +9,10 @@ image: repository: registry.k8s.io/sig-storage/csi-provisioner tag: v5.1.0 pullPolicy: IfNotPresent + csiResizer: + repository: registry.k8s.io/sig-storage/csi-resizer + tag: v1.12.0 + pullPolicy: IfNotPresent csiSnapshotter: repository: registry.k8s.io/sig-storage/csi-snapshotter tag: v8.1.0 @@ -81,6 +85,12 @@ controller: requests: cpu: 10m memory: 20Mi + csiResizer: + limits: + memory: 400Mi + requests: + cpu: 10m + memory: 20Mi csiSnapshotter: limits: memory: 200Mi diff --git a/deploy/csi-nfs-controller.yaml b/deploy/csi-nfs-controller.yaml index 4ed471c80..c4f8833d9 100644 --- a/deploy/csi-nfs-controller.yaml +++ b/deploy/csi-nfs-controller.yaml @@ -63,6 +63,30 @@ spec: capabilities: drop: - ALL + - name: csi-resizer + image: registry.k8s.io/sig-storage/csi-resizer:v1.12.0 + args: + - "-csi-address=$(ADDRESS)" + - "-v=2" + - "-leader-election" + - "--leader-election-namespace=kube-system" + - '-handle-volume-inuse-error=false' + env: + - name: ADDRESS + value: /csi/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: + limits: + memory: 400Mi + requests: + cpu: 10m + memory: 20Mi + securityContext: + capabilities: + drop: + - ALL - name: csi-snapshotter image: registry.k8s.io/sig-storage/csi-snapshotter:v8.1.0 args: diff --git a/hack/verify-helm-chart.sh b/hack/verify-helm-chart.sh index 642eec226..45504bf5e 100755 --- a/hack/verify-helm-chart.sh +++ b/hack/verify-helm-chart.sh @@ -62,9 +62,10 @@ pip install yq --ignore-installed PyYAML # Extract images from csi-nfs-controller.yaml expected_csi_provisioner_image="$(cat ${PKG_ROOT}/deploy/csi-nfs-controller.yaml | yq -r .spec.template.spec.containers[0].image | head -n 1)" -expected_csi_snapshotter_image="$(cat ${PKG_ROOT}/deploy/csi-nfs-controller.yaml | yq -r .spec.template.spec.containers[1].image | head -n 1)" -expected_liveness_probe_image="$(cat ${PKG_ROOT}/deploy/csi-nfs-controller.yaml | yq -r .spec.template.spec.containers[2].image | head -n 1)" -expected_nfs_image="$(cat ${PKG_ROOT}/deploy/csi-nfs-controller.yaml | yq -r .spec.template.spec.containers[3].image | head -n 1)" +expected_csi_resizer_image="$(cat ${PKG_ROOT}/deploy/csi-nfs-controller.yaml | yq -r .spec.template.spec.containers[1].image | head -n 1)" +expected_csi_snapshotter_image="$(cat ${PKG_ROOT}/deploy/csi-nfs-controller.yaml | yq -r .spec.template.spec.containers[2].image | head -n 1)" +expected_liveness_probe_image="$(cat ${PKG_ROOT}/deploy/csi-nfs-controller.yaml | yq -r .spec.template.spec.containers[3].image | head -n 1)" +expected_nfs_image="$(cat ${PKG_ROOT}/deploy/csi-nfs-controller.yaml | yq -r .spec.template.spec.containers[4].image | head -n 1)" csi_provisioner_image="$(get_image_from_helm_chart "csiProvisioner")" validate_image "${expected_csi_provisioner_image}" "${csi_provisioner_image}" diff --git a/pkg/nfs/controllerserver.go b/pkg/nfs/controllerserver.go index f160c6450..adb5049d4 100644 --- a/pkg/nfs/controllerserver.go +++ b/pkg/nfs/controllerserver.go @@ -476,8 +476,19 @@ func (cs *ControllerServer) ListSnapshots(_ context.Context, _ *csi.ListSnapshot return nil, status.Error(codes.Unimplemented, "") } -func (cs *ControllerServer) ControllerExpandVolume(_ context.Context, _ *csi.ControllerExpandVolumeRequest) (*csi.ControllerExpandVolumeResponse, error) { - return nil, status.Error(codes.Unimplemented, "") +func (cs *ControllerServer) ControllerExpandVolume(_ context.Context, req *csi.ControllerExpandVolumeRequest) (*csi.ControllerExpandVolumeResponse, error) { + if len(req.GetVolumeId()) == 0 { + return nil, status.Error(codes.InvalidArgument, "Volume ID missing in request") + } + + if req.GetCapacityRange() == nil { + return nil, status.Error(codes.InvalidArgument, "Capacity Range missing in request") + } + + volSizeBytes := int64(req.GetCapacityRange().GetRequiredBytes()) + klog.V(2).Infof("ControllerExpandVolume(%s) successfully, currentQuota: %d bytes", req.VolumeId, volSizeBytes) + + return &csi.ControllerExpandVolumeResponse{CapacityBytes: req.GetCapacityRange().GetRequiredBytes()}, nil } // Mount nfs server at base-dir diff --git a/pkg/nfs/controllerserver_test.go b/pkg/nfs/controllerserver_test.go index 57a817610..196a5aa5c 100644 --- a/pkg/nfs/controllerserver_test.go +++ b/pkg/nfs/controllerserver_test.go @@ -372,6 +372,13 @@ func TestControllerGetCapabilities(t *testing.T) { }, }, }, + { + Type: &csi.ControllerServiceCapability_Rpc{ + Rpc: &csi.ControllerServiceCapability_RPC{ + Type: csi.ControllerServiceCapability_RPC_EXPAND_VOLUME, + }, + }, + }, }, }, expectedErr: nil, @@ -1057,6 +1064,60 @@ func TestDeleteSnapshot(t *testing.T) { } } +func TestControllerExpandVolume(t *testing.T) { + testCases := []struct { + name string + testFunc func(t *testing.T) + }{ + { + name: "volume ID missing", + testFunc: func(t *testing.T) { + d := initTestController(t) + req := &csi.ControllerExpandVolumeRequest{} + _, err := d.ControllerExpandVolume(context.Background(), req) + expectedErr := status.Error(codes.InvalidArgument, "Volume ID missing in request") + if !reflect.DeepEqual(err, expectedErr) { + t.Errorf("actualErr: (%v), expectedErr: (%v)", err, expectedErr) + } + }, + }, + { + name: "Capacity Range missing", + testFunc: func(t *testing.T) { + d := initTestController(t) + req := &csi.ControllerExpandVolumeRequest{ + VolumeId: "unit-test", + } + _, err := d.ControllerExpandVolume(context.Background(), req) + expectedErr := status.Error(codes.InvalidArgument, "Capacity Range missing in request") + if !reflect.DeepEqual(err, expectedErr) { + t.Errorf("actualErr: (%v), expectedErr: (%v)", err, expectedErr) + } + }, + }, + { + name: "Error = nil", + testFunc: func(t *testing.T) { + d := initTestController(t) + req := &csi.ControllerExpandVolumeRequest{ + VolumeId: "unit-test", + CapacityRange: &csi.CapacityRange{ + RequiredBytes: 10000, + }, + } + _, err := d.ControllerExpandVolume(context.Background(), req) + if !reflect.DeepEqual(err, nil) { + t.Errorf("actualErr: (%v), expectedErr: (%v)", err, nil) + } + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, tc.testFunc) + } +} + func matchCreateSnapshotResponse(e, r *csi.CreateSnapshotResponse) error { if e == nil && r == nil { return nil diff --git a/pkg/nfs/nfs.go b/pkg/nfs/nfs.go index 5a0c2c2ec..5d36e5ecc 100644 --- a/pkg/nfs/nfs.go +++ b/pkg/nfs/nfs.go @@ -103,6 +103,7 @@ func NewDriver(options *DriverOptions) *Driver { csi.ControllerServiceCapability_RPC_SINGLE_NODE_MULTI_WRITER, csi.ControllerServiceCapability_RPC_CLONE_VOLUME, csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT, + csi.ControllerServiceCapability_RPC_EXPAND_VOLUME, }) n.AddNodeServiceCapabilities([]csi.NodeServiceCapability_RPC_Type{ diff --git a/test/e2e/dynamic_provisioning_test.go b/test/e2e/dynamic_provisioning_test.go index 7f9df668a..542e449d1 100644 --- a/test/e2e/dynamic_provisioning_test.go +++ b/test/e2e/dynamic_provisioning_test.go @@ -52,7 +52,7 @@ var _ = ginkgo.Describe("Dynamic Provisioning", func() { }) testDriver = driver.InitNFSDriver() - ginkgo.It("should create a volume on demand with mount options [nfs.csi.k8s.io]", func(ctx ginkgo.SpecContext) { + ginkgo.It("should create a volume on demand with mount options", func(ctx ginkgo.SpecContext) { pods := []testsuites.PodDetails{ { Cmd: "echo 'hello world' > /mnt/test-1/data && grep 'hello world' /mnt/test-1/data", @@ -75,7 +75,7 @@ var _ = ginkgo.Describe("Dynamic Provisioning", func() { test.Run(ctx, cs, ns) }) - ginkgo.It("should create a volume on demand with zero mountPermissions [nfs.csi.k8s.io]", func(ctx ginkgo.SpecContext) { + ginkgo.It("should create a volume on demand with zero mountPermissions", func(ctx ginkgo.SpecContext) { pods := []testsuites.PodDetails{ { Cmd: "echo 'hello world' > /mnt/test-1/data && grep 'hello world' /mnt/test-1/data", @@ -98,7 +98,7 @@ var _ = ginkgo.Describe("Dynamic Provisioning", func() { test.Run(ctx, cs, ns) }) - ginkgo.It("should create multiple PV objects, bind to PVCs and attach all to different pods on the same node [nfs.csi.k8s.io]", func(ctx ginkgo.SpecContext) { + ginkgo.It("should create multiple PV objects, bind to PVCs and attach all to different pods on the same node", func(ctx ginkgo.SpecContext) { pods := []testsuites.PodDetails{ { Cmd: "while true; do echo $(date -u) >> /mnt/test-1/data; sleep 100; done", @@ -135,7 +135,7 @@ var _ = ginkgo.Describe("Dynamic Provisioning", func() { }) // Track issue https://github.com/kubernetes/kubernetes/issues/70505 - ginkgo.It("should create a volume on demand and mount it as readOnly in a pod [nfs.csi.k8s.io]", func(ctx ginkgo.SpecContext) { + ginkgo.It("should create a volume on demand and mount it as readOnly in a pod", func(ctx ginkgo.SpecContext) { pods := []testsuites.PodDetails{ { Cmd: "touch /mnt/test-1/data", @@ -159,7 +159,7 @@ var _ = ginkgo.Describe("Dynamic Provisioning", func() { test.Run(ctx, cs, ns) }) - ginkgo.It("should create a deployment object, write and read to it, delete the pod and write and read to it again [nfs.csi.k8s.io]", func(ctx ginkgo.SpecContext) { + ginkgo.It("should create a deployment object, write and read to it, delete the pod and write and read to it again", func(ctx ginkgo.SpecContext) { pod := testsuites.PodDetails{ Cmd: "echo 'hello world' >> /mnt/test-1/data && while true; do sleep 100; done", Volumes: []testsuites.VolumeDetails{ @@ -188,7 +188,7 @@ var _ = ginkgo.Describe("Dynamic Provisioning", func() { test.Run(ctx, cs, ns) }) - ginkgo.It("[subDir]should create a deployment object, write and read to it, delete the pod and write and read to it again [nfs.csi.k8s.io]", func(ctx ginkgo.SpecContext) { + ginkgo.It("[subDir]should create a deployment object, write and read to it, delete the pod and write and read to it again", func(ctx ginkgo.SpecContext) { pod := testsuites.PodDetails{ Cmd: "echo 'hello world' >> /mnt/test-1/data && while true; do sleep 100; done", Volumes: []testsuites.VolumeDetails{ @@ -217,7 +217,7 @@ var _ = ginkgo.Describe("Dynamic Provisioning", func() { test.Run(ctx, cs, ns) }) - ginkgo.It(fmt.Sprintf("should delete PV with reclaimPolicy %q [nfs.csi.k8s.io]", v1.PersistentVolumeReclaimDelete), func(ctx ginkgo.SpecContext) { + ginkgo.It(fmt.Sprintf("should delete PV with reclaimPolicy %q", v1.PersistentVolumeReclaimDelete), func(ctx ginkgo.SpecContext) { reclaimPolicy := v1.PersistentVolumeReclaimDelete volumes := []testsuites.VolumeDetails{ { @@ -234,7 +234,7 @@ var _ = ginkgo.Describe("Dynamic Provisioning", func() { test.Run(ctx, cs, ns) }) - ginkgo.It(fmt.Sprintf("should retain PV with reclaimPolicy %q [nfs.csi.k8s.io]", v1.PersistentVolumeReclaimRetain), func(ctx ginkgo.SpecContext) { + ginkgo.It(fmt.Sprintf("should retain PV with reclaimPolicy %q", v1.PersistentVolumeReclaimRetain), func(ctx ginkgo.SpecContext) { reclaimPolicy := v1.PersistentVolumeReclaimRetain volumes := []testsuites.VolumeDetails{ { @@ -251,7 +251,7 @@ var _ = ginkgo.Describe("Dynamic Provisioning", func() { test.Run(ctx, cs, ns) }) - ginkgo.It("should create a pod with multiple volumes [nfs.csi.k8s.io]", func(ctx ginkgo.SpecContext) { + ginkgo.It("should create a pod with multiple volumes", func(ctx ginkgo.SpecContext) { volumes := []testsuites.VolumeDetails{} for i := 1; i <= 6; i++ { volume := testsuites.VolumeDetails{ @@ -278,7 +278,7 @@ var _ = ginkgo.Describe("Dynamic Provisioning", func() { test.Run(ctx, cs, ns) }) - ginkgo.It("should create a pod with volume mount subpath [nfs.csi.k8s.io]", func(ctx ginkgo.SpecContext) { + ginkgo.It("should create a pod with volume mount subpath", func(ctx ginkgo.SpecContext) { pods := []testsuites.PodDetails{ { Cmd: convertToPowershellCommandIfNecessary("echo 'hello world' > /mnt/test-1/data && grep 'hello world' /mnt/test-1/data"), @@ -301,7 +301,7 @@ var _ = ginkgo.Describe("Dynamic Provisioning", func() { test.Run(ctx, cs, ns) }) - ginkgo.It("should create a CSI inline volume [nfs.csi.k8s.io]", func(ctx ginkgo.SpecContext) { + ginkgo.It("should create a CSI inline volume", func(ctx ginkgo.SpecContext) { pods := []testsuites.PodDetails{ { Cmd: convertToPowershellCommandIfNecessary("echo 'hello world' > /mnt/test-1/data && grep 'hello world' /mnt/test-1/data"), @@ -328,7 +328,7 @@ var _ = ginkgo.Describe("Dynamic Provisioning", func() { test.Run(ctx, cs, ns) }) - ginkgo.It("should create a volume on demand with retaining subdir on delete [nfs.csi.k8s.io]", func(ctx ginkgo.SpecContext) { + ginkgo.It("should create a volume on demand with retaining subdir on delete", func(ctx ginkgo.SpecContext) { pods := []testsuites.PodDetails{ { Cmd: "echo 'hello world' > /mnt/test-1/data && grep 'hello world' /mnt/test-1/data", @@ -351,7 +351,7 @@ var _ = ginkgo.Describe("Dynamic Provisioning", func() { test.Run(ctx, cs, ns) }) - ginkgo.It("should create a volume on demand with archive on delete [nfs.csi.k8s.io]", func(ctx ginkgo.SpecContext) { + ginkgo.It("should create a volume on demand with archive on delete", func(ctx ginkgo.SpecContext) { pods := []testsuites.PodDetails{ { Cmd: "echo 'hello world' > /mnt/test-1/data && grep 'hello world' /mnt/test-1/data", @@ -374,7 +374,7 @@ var _ = ginkgo.Describe("Dynamic Provisioning", func() { test.Run(ctx, cs, ns) }) - ginkgo.It("should create a volume on demand with archive subdir on delete [nfs.csi.k8s.io]", func(ctx ginkgo.SpecContext) { + ginkgo.It("should create a volume on demand with archive subdir on delete", func(ctx ginkgo.SpecContext) { pods := []testsuites.PodDetails{ { Cmd: "echo 'hello world' > /mnt/test-1/data && grep 'hello world' /mnt/test-1/data", @@ -396,4 +396,27 @@ var _ = ginkgo.Describe("Dynamic Provisioning", func() { } test.Run(ctx, cs, ns) }) + + ginkgo.It("should create a volume on demand and resize it", func(ctx ginkgo.SpecContext) { + pods := []testsuites.PodDetails{ + { + Cmd: "echo 'hello world' > /mnt/test-1/data && grep 'hello world' /mnt/test-1/data", + Volumes: []testsuites.VolumeDetails{ + { + ClaimSize: "10Gi", + VolumeMount: testsuites.VolumeMountDetails{ + NameGenerate: "test-volume-", + MountPathGenerate: "/mnt/test-", + }, + }, + }, + }, + } + test := testsuites.DynamicallyProvisionedResizeVolumeTest{ + CSIDriver: testDriver, + Pods: pods, + StorageClassParameters: archiveSubDirStorageClassParameters, + } + test.Run(ctx, cs, ns) + }) }) diff --git a/test/e2e/testsuites/dynamically_provisioned_resize_volume_tester.go b/test/e2e/testsuites/dynamically_provisioned_resize_volume_tester.go new file mode 100644 index 000000000..c311f0528 --- /dev/null +++ b/test/e2e/testsuites/dynamically_provisioned_resize_volume_tester.go @@ -0,0 +1,99 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testsuites + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/kubernetes-csi/csi-driver-nfs/test/e2e/driver" + "github.com/onsi/ginkgo/v2" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/test/e2e/framework" +) + +// DynamicallyProvisionedResizeVolumeTest will provision required StorageClass(es), PVC(s) and Pod(s) +// Waiting for the PV provisioner to resize the PV +// Testing if the PV is resized successfully. +type DynamicallyProvisionedResizeVolumeTest struct { + CSIDriver driver.DynamicPVTestDriver + Pods []PodDetails + StorageClassParameters map[string]string +} + +func (t *DynamicallyProvisionedResizeVolumeTest) Run(ctx context.Context, client clientset.Interface, namespace *v1.Namespace) { + for _, pod := range t.Pods { + tpod, cleanup := pod.SetupWithDynamicVolumes(ctx, client, namespace, t.CSIDriver, t.StorageClassParameters) + // defer must be called here for resources not get removed before using them + for i := range cleanup { + defer cleanup[i](ctx) + } + + ginkgo.By("deploying the pod") + tpod.Create(ctx) + defer tpod.Cleanup(ctx) + ginkgo.By("checking that the pods command exits with no error") + tpod.WaitForSuccess(ctx) + + pvcName := tpod.pod.Spec.Volumes[0].VolumeSource.PersistentVolumeClaim.ClaimName + pvc, err := client.CoreV1().PersistentVolumeClaims(namespace.Name).Get(ctx, pvcName, metav1.GetOptions{}) + if err != nil { + framework.ExpectNoError(err, fmt.Sprintf("fail to get original pvc(%s): %v", pvcName, err)) + } + + originalSize := pvc.Spec.Resources.Requests["storage"] + delta := resource.Quantity{} + delta.Set(1024 * 1024 * 1024) + originalSize.Add(delta) + pvc.Spec.Resources.Requests["storage"] = originalSize + + ginkgo.By("resizing the pvc") + updatedPvc, err := client.CoreV1().PersistentVolumeClaims(namespace.Name).Update(ctx, pvc, metav1.UpdateOptions{}) + if err != nil { + framework.ExpectNoError(err, fmt.Sprintf("fail to resize pvc(%s): %v", pvcName, err)) + } + updatedSize := updatedPvc.Spec.Resources.Requests["storage"] + + ginkgo.By("sleep 30s waiting for resize complete") + time.Sleep(30 * time.Second) + + ginkgo.By("checking the resizing result") + newPvc, err := client.CoreV1().PersistentVolumeClaims(namespace.Name).Get(ctx, tpod.pod.Spec.Volumes[0].VolumeSource.PersistentVolumeClaim.ClaimName, metav1.GetOptions{}) + if err != nil { + framework.ExpectNoError(err, fmt.Sprintf("fail to get new pvc(%s): %v", pvcName, err)) + } + newSize := newPvc.Spec.Resources.Requests["storage"] + if !newSize.Equal(updatedSize) { + framework.Failf("newSize(%+v) is not equal to updatedSize(%+v)", newSize, updatedSize) + } + + ginkgo.By("checking the resizing PV result") + newPv, _ := client.CoreV1().PersistentVolumes().Get(ctx, updatedPvc.Spec.VolumeName, metav1.GetOptions{}) + newPvSize := newPv.Spec.Capacity["storage"] + newPvSizeStr := newPvSize.String() + "Gi" + + if !strings.Contains(newPvSizeStr, newSize.String()) { + framework.Failf("newPVCSize(%+v) is not equal to newPVSize(%+v)", newSize.String(), newPvSizeStr) + } + } +} diff --git a/test/external-e2e/testdriver.yaml b/test/external-e2e/testdriver.yaml index f31623cd8..4e907eb60 100644 --- a/test/external-e2e/testdriver.yaml +++ b/test/external-e2e/testdriver.yaml @@ -16,6 +16,8 @@ DriverInfo: fsGroup: true pvcDataSource: true snapshotDataSource: true + controllerExpansion: true + nodeExpansion: true InlineVolumes: - Attributes: server: nfs-server.default.svc.cluster.local From 9211889e27957f31892c56d6e1eb19d303178d1d Mon Sep 17 00:00:00 2001 From: andyzhangx Date: Sun, 8 Dec 2024 08:07:03 +0000 Subject: [PATCH 2/2] chore: add rbac roles for resize volume feature fix fix --- charts/latest/csi-driver-nfs-v0.0.0.tgz | Bin 11440 -> 11473 bytes .../templates/rbac-csi-nfs.yaml | 36 ++++++++++++++++++ deploy/rbac-csi-nfs.yaml | 36 ++++++++++++++++++ 3 files changed, 72 insertions(+) diff --git a/charts/latest/csi-driver-nfs-v0.0.0.tgz b/charts/latest/csi-driver-nfs-v0.0.0.tgz index 2a3eca635846cd345e2392c5762f59e067fe3852..7036ac3cd98cab7c6160e7202a21c26fbb500218 100644 GIT binary patch delta 10094 zcmV-!Cz05&SBCL~LK#si2&4}Jd9A_*QruZ0jo11#VYQr1HpMnp(H?OgpybO_U5 z=}?$r>fw+wK_uhT9vX$bL)gL9pF|)e9uYbU1Q`WFGLA=ZK!+lYvk{?1Wr;^U6k>|` zw0M_haeTpIe-ci6=xlh-|P7{aDmcK5$V`M4y%bs)$UUbq+h- zr4hLVL4I2o32rLa5IF1{9<6{upW;-EnUw2dAxZ!>O#Za|P{d>c6ok0oYyhj{kylSu z^%@pOox|g0(V#2>vp8Lv*6^Bxq5$QRk>{Xh5nFnqf8dh@PI0`^Y~?c*gp5kg;&|N| zYYE^J62hx6WEqtTlaK?J&_j}EfcDT|$3XSp`^c6>*c2A%n#3_0fUOc~HDJoJ4(fwM zlA$4v1t2n1FVQuo`ei0S{pN^W(~fGd=z>uh=ukM_3Is-b=o}}2*bwzC1{Pff&S{=; z1w3k*f7vgg=_yna;S4Dyul9y*v{eh`zvfqmI(Iip@6KRfy@e@&7x39;y*!xln{`X!8}@1`mA&?T6W zQN=R)mcGG4k#m8hetBY`UBs9=_g2{7ATved65_apkYq7%Y^GihV}Pz;+Cyzk^_<1fxhgbsB}WoHNo^|| zSgLBPrrWPS5g|qX0fs{eWe>H_Sw9>@f0V`0UJAaYo>a~EjkgkPshJZyps%8cQSp|> z({*8T5G>;%*xB%_C^1QhbkwE<5-szOyWKaWm9GW=$UwNI!Zhol!)`KD?I^D`tB2fR zOP&?1skNgGTk|RvO^qFH*qEcx-b8;|dU2W&y6Ww1in0WM`h^y7jC8L`+a*Nwf74r$ zXI>Vp@!9QCNV9TzheY{w6QXQTON|N=mFu=4TEG6e(mYockh-2RGrMk=XFFW)ZmqMm z(cD8PT5UlStyC4gK_{1|LaU^DGkj@rI0a4MB}Ep4qE%#s_Q((krSWPl*=TC{IM#nF zZ3w7Dm`3$4YSc6eb-mi?(?ps{57CVYtZRxNi{Zp%1^d_E2DHtZl0* zCXH+u|sA6a5XO^$rnn4qW)qy`UgTIf4xsoyTG#~frwyR!0lc> zqJf&532^c9xO4c8+NEpuUkOR7?fwVnZ{NM{cjPCzc^ixTKTnTe)a<{{o*h5k*?({2 zLGQ*;_v#RVMj)k<3K66+n?j@(4Q;$BmzsABg<}Y>>a|lA2_z(znb=2Z3|Ii-8SS-N z2q}RY$`}PDe~-WX{26sFL6_^C=Wo%MFA4_(EC_^EEMK;9n%2cK6%tbqAoS%+8wJ5N zmf`rWhxy(rvbY!}zD`)}#A09oe`+uGvA3fjk|7|>5 zYonn2qSkWMF;NDZM?3G0$5EAhI|0r~1n76AQqvGUf2#i^Nk1D7$tTnfN?<@E{kK6a z)($MK(u^Ta%6$a#;$?~(VCHY7GB(SW9%}ofi?>dKDIP%-45qbOx$(KFqE(zee?~bM zx`Wthf2>zH*mY}WY+n_Ad|OS8N=J+j^kh|v+Rg;!o4n@ldUK|var=*L)c)AEu+(Uh zVKH^C3lZtTMw0yvC$Yc8?}*SyZH12X@EdSyHfs}m*PQWfJdN@{CD(Iq^_YY@f7bLdw@jn;q^PafItYT68+3$IqJUAo zr_DEolf%{(p-~T=G{9^nAaR5x)^gk3#_Mfn=|&s@)#7rHkvIZgk(ZNh=TFvIic=Ec zRP99}s*2YhYUh@a5s~9;&N#vi@3XpXq?H7%=tXI7ApxxGr&{0FkJA zr}vQ@6M?LmO}JWE83LgCXY67Z?@!Q>vjkm_NjNrj3-H%ipvR2o_QofL!O_##MHIr0 zf!R^p0U{(&X=vdj#w1Y_Rq20U7ruCZQt7^soX}BY?+er_woNgo)B0~+Du=5cztNQq z{tls3+KdT*P&-?h(np-IG#FjL3*Nvv9zcn1185c6ns7fi@#(DsK|I3QKjo+4k%^BFankc za5zRxj)8ZOwV|Rnzx3ZJ@ndj>$ar~7AP*kZ#10X*|IL&(U>S|PST;@1!q29D+uPY$Bh4O9GB_Q!8${D(Sqf_3e@BzTMOsGk_7VRTZ;tMs12*H!qj>Uf5ox!>+frp zPVf|Q&{8F%4m=y9jov6Vh0q}@eRU%ua}_#dNs>__rzT{E3^K{MIEY{Zv5qOkJRB1V zq0BfODE(KXPL0N%M0e>{yeks51NOiGzj&}ueFh%6{wXt8~w_Z3<ez~9rL=#c{e{F48ETyTzQR|#g ztA|$q%jKdT?aEownE{H>J56*1Vom5INiwNJb?{IY5|hzR*^e0*&0W!7o78V#;aV>n z>pC5;Dc4@02wnIfHEP&aVJ&we4N#rowu00RgU}*N@_6!CMe?SMIE#>vyWW+!y4X@a^C<*S_#`y>bH4~4&c!L1QpT=#<5`@xN zw}DV_Lw_oZ`SQCmOlS9WzRs2U!R*8}O_AMNHo!IWrl+k4l2FT4RTWSKR~~qC8*c!+ z;bfYRW~635G74{pvvV30`@Uylu?4QSTMub7c8+p+R9jRd zm`zxi56MiTf9rhoun;UHdPyyjMa0TZcYK$deFxq(kz+GW4&)qBJu7NVbX1%iD`lQ8 zOJXMGPVI!SokXVvt+iKx>RBBCP?TVPHKRirwDW%M+xWzOMe`V95^KmKpr8#sN=Pb2 zv4dVGsr1d9)v<}0VI0To+6H{dLSWU@C^SzL6=I?Ce>SknlSet7kyvKCJV!k>1suKJ zXzLm0YOPSpB~J>23HIy{&2z%2(ht;bAQRToGA;I*1CZBhGL|)=e}#<*zl!EHt(^@$FiWXMAlgSn7NT2}vfJOr7;BdC0M$%f&U!pGLuyPHa}}H^ z$PPMx`|h<*Y`lOPg_9XZqb~Y9tQLT2=e+9R>m$EI(UEusLh~}2Gdl}o*o$Q)mxX7oW%G8B9>6erh3zOxO77;Ojd}&wN9K+^aoqA$68Yxy* zixnxXPSnmBCKL&vf=Ds}SNrrVL1?i>MtYZ^nE{jYmmf2&*I2m2<~J%VmK_VRo@06_ zBOew>n6jXwf-Mn<;Tu9vXohd`Y+fqou);Enr?UiUbU#Y=F%#fhRhC^l=dYA=6pVQq zKm(cvDX-*zT4L#3xKR351i5b%x@*#)vj~$eqWz9%(DtmeUZ5xeY(qg*2MGmHcUjC^ zRcE7d+@UC<9%c!9F0<6Az5r9-&n^h)@YB6nkP5QTC&~@*n9B)1eR2~uQj?fh=K<7K zJ^MJAN)XHIkPaVsz^Ily|M>pNe?0j;|MABsPaZ%1@Rv6~y}Nk*D|zz! z2bv{+SLWC6k3YccUzY^(DMQLGh(M8&EMY0#_t2}0GdK3jOm_~5LkMJ| zmc47tsrIXxu{T*A6zG&uVVA=axU-XR6Bd8X0(swDJgdzSM*nKy^iaFu$j3nrW;WKd z#QJ~hM19Q=YL(6Q_wE8tzoIbH?F5$YLP?uLNw*X{x(gRwCS3Fu0z~guNa)uV0J>?Y z=Y~O@8-{Og9H{wlLo&Aze7Sj;<;DS(HyAp3C-I7#Mkj6~7IE{)!;Rw%H;pRXVhn%b z4MYrX8XvfMG~l-s`?uw2znhBh`-&s_Hjm%iLbTpiV)HhR#CxdZcAsMEwiQ9QrFgl` zqvJM@h1*Kx+m_!tiftt_ z>~2^_9nOZ6jK1YqWp3VIIS}p5Ju4?bbYg!mjO47u zS)97B7Db$#f&`vu7ht|?X|kx~+Y&kEEE`orZ+({1(ll+ZO0&0-5Gkzn#uufa3wlCU z^cTXYxf-I{U*9KBzwd7{E<0X@v!}dk#H=9LV;ynLui8$cG-F|g*omc>ar3;&+X)vO zh*ztXFun|H+1FNQDU8vO&{=<1K<=ya00tK{=71=p;q%%7t_0 zqFPYx)R&YpK67MM=By=aYG9is59%8ow3q=+tnEvE$zxm3Sle7n)2=IVMWKBE<27x& z_e3f5((zP~5XVi2gBfi$jcVMkAeEg0sihjPHPML5;>K$NJAeEE5I}#kr0!(0SNv38 zUglS7E+)w0n${*lEN(oR2^zP(u_g3~jWLa4EjDJ-(s`(^DbQn=LGQkXM7fuExy4;gn0ap2{nj24)@;oSLn3jqvj2xYnElXvm!K$TB@RdN&3Eu9#ve{zs;- zg>xb<*4wUm^)3Cu)h>UxaL#4flP&i~=lxc7-CcC{%|vG#0@@@*TSM4Zoc20@5u3=mIgd5ozY-2C*RcQbBTHoItb9f*+{e#!ym-ol) zJTMQ#17qa;Ex2GDQ&Xc8#*|s)hOsC&`(etz7PwyuoHlg8Jh)FZ3$?TMAxl!V@>ZPY zCE65er~VwYRy$FzhHzxX&EmbdZ~1M|P6`3$YAJK0Ntd@2ud-v;NBFm}ZErlff6_n8 z*Or`$Y#53TvGpixrEHT;7h!*&KktF1qV8PS`9*iGoqMQ2Ncedsu$g`=HiIp(HkHp@bI&r0ln3Tk3}`-Fw`5XVbr_Z~vUxym$?WEV zyu2aqN5=CoWSx3RZB&Vo`5w51O=33q7iW?sSXwZ?Nad6M*&L7(I1+y(f)Km$%T=wo zZ5uR98a3bi6f$mnK9T42yRp-$j(g*p#lB8h;Ug~{ES#TVb_by1&sc@3@Jqzz%wax2~i=^b^R7I2#eF zl*-3XOoaK+n{rySV`P6%wKvbtoN-+{(MWzWtYRj3%y_voed0m5fsGO4GwB1BEctsT zWSMCVJC?*oI<2XFH=pe!@Bg2**?o!gPt^*wRA`Ru-$wp=5+f8ICS$e?T;=bWF-|b*G2!yiPg_(b2TpBlLgPU$*L|eU3 zZ}U6Ys+76p#cK_Pjm$7>%o`Rh5;6T+@mz=}>hJ1tnCmff!U{a&l%L7g+P|6+o15)l z|NQSCNeqSy{`JrQQBvn$|NQSCVisQg>!1I#bYkyn#E2D+2DwMVVCmH|RX`%O)ITCgWva~)lu zCeD52S~-iza0ogxhX9>+^PWx+m>A`8d+?jhNMTA?+N5Ol%?8O)w8UVmOL!NZbs zaw=WIuqKlY8W?|O4Y#9M#6@`lJ+jd;_C&dFS4JYjAv9hMH(}xcOg0MddBD|F-F<_E zPPnz4{jpi=)6^Z=#5NAPW1Ob2ymsP&yso~|WNgcy3jMvhIID-k*{SsI78kDEx~JL- zy14B}XG7B%FLPoNFjG<3?hv6u0*>aWNfZH1yg3iGJDkd#Oo;{hk55%}+rKz5s?^F%#SBAuue*#SXq(w}09ojucZW@h8<%sI-lyk<|&~G&fI_BA}ELh~psN`(NX! z8ijn~Y+6{%klQU$&}ecLmbDS{f&JXh9`DA|8!5{J!KEs!yb=01{ejOnPzTR6;Q z#$mlw53`oy{P9?f>w@FXY_3aojxLw)@R93jo7=g0IYik6_=Ldq!8PMogpLA5nt@HZ zbfDLtgFXGv+I`Ur*s>j}tadlqE7guTSgTXHYhn3y>`^}tWvbP!X-0*NZD1xMP=X^U zQ}BP>Fe$46)#>HJX<*}e0!$7TiVd>^_qrv2SrqbucLD-0tI=I?4YwyKQ; zY&NIDeP5bE&@YsH@~&-^2?&`^)~Fx?4OxF=l9OVJ2$j!|JLoOd_4o6Lu|z3O{}uVK zL;d5wiGMr)FFSNq{xdk!N~?k>b^P$(Uvz`+VbDGFK`^^ zE1x^NIMvOiX1A$2HQPhWtJbi>?{4t?s&cU5`+a|BZBg0zd1yr?bKhsv|Ph~o-TR` zF<=3m(uDWrOY8GzL}(ai5ui35icX9N5Q{eIpx?FWno4w}%H^R}9MYJCSTwbse>WHi zh#{2ub}GR#9RI8v$oU$l>EeC}$+3iy4Vp4S#a|~c`EHupx;Z~t42m&aqTkOAgcT62 zMx-feLWV^RlHd_sWO3Ywkb@L|CG`3ijh$qF!%5s+yFl|#C7@h>6)(n2$a9d_j9;06 zU4-_WYjhBf1w#}d(vFfuZD$Z23MYsqBKaa2QDI}(8pfe0P`0n+Jqh7esCP=|b*im? zc3>IG42PUQuSotWPjF{ZFYleyJ6RplF6o;U99fLZCW z>{zFqFizyO40UKAqnUy(Ifa@2#HfT%vYavjq1tU+a5f}W#mz{I%Z#f1_o)d=pW8*r zVlaBE!=cX0l?bf#4Z1pil9AqOeJ=x^c+1Z87>f%IL-Gl=5874s?}UO}lr@yBWr9e? zr>>;3@}!5_&iI=811#VYQii^KnfW#kc5hK>iAU(mm$n&zSiq9_Y_Cms(6u<=BT+Z^ zw}Z*cqxQ_pKurXWBBc?({KMl{r>B>%`~4@Lf-&F-c%bQ@FuJ&ZM0us@%ccd`0|Y)v zjsqiqgM7t$sie23@DQ-fI0Pf5Grs&Yql{m?zl1QxBr(#ssitBGmdTfgN8N5ET?)`i zedz2V7Be*UI2$!bJ*4zi_5Y4w;Z=R|)xzajJiPOo7Gb!$AR|nSemsv0v(4_oKUI{(S7$Vf;CNtgW7Axv(DNTU6I&`t9+u zo9}Bme(skww$9Uyu)VzImR_H?x76lxJicYkVLiQIv8=dHOS(JaO`N?R6FQLRFyf6! zja%UEm`s3z5NdBQfMu9FuW_L&Y?rs^fSzTuYT-3Gds{6o!`D>?ZlZQmQqM-gU{A$N zfw7cnke{?bd>OM4$73dB&%ep7Vm<~O%kc%{vWm2UJb8Jv67yAI-}hCug6#v*rs+^D z!==u;K}iC+HU z@%h`+*B{SLecDAKb^h@9q<{AD_4(<=+q3g`zM{s{q^Q`%#-0wehHOtY&GA7BJRw56 z%RIG>R49x1A?0zO24X~b!>HJDT0Wf;?gwU*OazRYgGl!7^#+i(^jm5Uj}}w6f0UwR zhu^0t{m7Z;jH(U=@SZwykAY~1fi|noTs4xn>`tj)Ps=Fz)H^t6TEHq2wu;UWlQ`q> zZp=Z9SseAyv&wh1sQm=8-Uq#~p<<1zeND~EU>3FC>K_@GO_twf7ezveBse~W7*G2q zJg|qJR$cver7k`nRzIh}35)!de;2QeDT_o&!Og3q)6J`Fps3D}@__>2t;Si3M^-4c ze{hd&)ye-hO>~&W7a>_!^8M=rg?AAAxY)*lyDtJ zaMym{{8{M#Hy;TdvK#XMce~HJGyebM7sosQ|7|=$5LDd#Gs%c-bihgff6V4Nn;QsF zUCIl$J(d!O=2Qi${m{5Di zofQoMdYz6T0gmHV)vq}h(vSV0ILlJAHtXU;;k@Y~rQKHm-u3)#`Yhysr5voy`Q}*= z|MT?uagF~EUmQJuy5s-blR70J3k!OJKm9`S1d~`XlVv57e~x*p+_5S{TXRY=m)Z-R zVmW^KhsURvXYXHMe!P7B)5reXlfQhtc=hhjPnNNX`YQS@l+A`6owBaj?3kZT-7u~G z<(D5`f7IsK^H*#xqHk3ym6a>gJRgKxbX^JO>G|D;$k3tqN}N(VH)E5C zM2mcEMgjRYf4pluztwH~tyWTRZ`fnCnZNpm*P^?3mNi(MQ(GVY%!Jh4f1B2}yw#p4 zno(Mc=}gFI{jPI&I6#I6dD+fP*=CSI{hGbGKs(V)PiKVXYD@X*u0Ahytmo$QX@m3p zes8%su77{rkDK)Sqt(TxOSOd->3W-x{rrwBAZ~-Pf7+$a%nw@3N~<=i zuipHOh9m|g#_U?4F}p^Rp-3MguO?b4NvEZ2dp{wPq_IxRs>Bl~ievYj z)WQmdkR>Uj8pQ%zLPrK{p76YanacW_?x zmThTnssdlD+e=?Ce^U3F7!zGH`beihPcXkSjFTxSLj?}2kM!D;aVSRtf|I!@F*FR> zU;?@Ull)!e(t3zbwd@&RM_v~o5|GSN+rVP|mBAF~qiiwjQ zDH{iiiLO`9h~kq!DI|Y|x0@kICv;l6Q#^}}xDF6biH@NbJ@lb%-IUI%^r4*=k>1wG zYYR<)4|IVM$o4*J$3*CVuhmXS{b7bh{gvjg%tcsrd(UBi>CAwUYyEHF~z!gISXHDsc#{u52!+#3@W| zILa*RugAxznD~D=V|ypl<7#{MZp3eCmS&XQa_ND&faY~dpEXX+@7!ZMpfiF-3WL_O%mh#q9Wh1 z0phl_){PAi_r=;!_ImM-$J&_g!UD1V2JeO4W%>Qt^6P)qX)f_0D!wYgtcBi|ts9Q{ z_7?Q^xq|REc2;9QbW4kqap30u+bu7-O-tD}>JU9v+6cfMkp;CQd$YjL^i~%5{Ol4U?#8=Rf1uhK$m0pyIs;9 zyU@I3SMyOwF~x z`o1HRwDtY}(`GzQSSHZf>1)yIq@3wHMLQ${d|=9P_V-q2B0?6y!A_9vp53#19>DYe Q0{{U3|F$J&GXQJ>0OKa19RL6T delta 10064 zcmV-WC$HGiS+H4oq{%_+6Ga*^>Gaf)Jdg$|)7D@03dM$(y8ejpJkg^`)Fd{Kq;~iw0#87~ynjTElA&iUO2#MxKM3MQrJbe}YdEIK}Zsvz5aOkY!XVOhOJ=LJvuv0op@<9Rt;W?;~3lVN+P3YZAw30Jch`)qp9>I;am4 zNrr|v7J$f5y+qfT>X(@S^_wGhO*^W+q6v!7Vnfuo7+7=_IH!5W z74WEKe`ddgrl)WXI4wTFVSPvxbgkYfou*|7qvyWtgo2d(F;Va@fG0#KCL?;N+iDE~ z%Z%&Q0Td5n__6;JXIYw0k!{C(JEJk7@Seq)YCsQZ3!#*=6pyfkKQkfUs7Bsu)lB3? zT36WsVvwhV_t3!v^MjZS4(!WT%Ng|o`PtELe`%78Nr*)c9kvis)GuK)eK$>^hc3a4 zj4GDVxAYAbiku51^~)0j?IOn1!QaCs#(-lvzF=JT(9yp?e|DsbvC+>kfw+gBwGg^y z{EE<#?yl*VOwtxY5f%B+oy3{K;YY%S{40@TH9Y4a5!yp|f=R4K6dCQNhfZOLvsmso zf7PmfPz{+1Z|SKTMaUbC^e=xK=G#8vAhDKpcsL}K$f=gHhE)3yLns-yuToAJCvtic zVN$&{b5&^OijO3ElG>Ix zuvFDnO}AfvB0`G%0}O`{${uQ;vwk>+e<+Kgy%c;)J*k@S8*e4pQZpxZKwm`>qv9=% zr|ZJxAXvsju(RP;QDTx1>8MQ!B%0?Rce`&$E0+cT$UwNI!Zhol!)`KD?I^D`tB2fR zOCAZ<)Y{R8t$CG-rp698Y|K$;Z=ydfEu3bUu6lc$tSrHwexU^%!`-Xgb_o{!fAm)P znRf}+*4gb+NV9TzheY{w6QXQTON|N=m2%qs0)Yj4k37%s9n?i+-9=)R^ zA_A)i2KsXkwbjhGe|60ibDV$#UP1z{wg|dG^LNKq?}$m*$>%I)YF*eWW6Yt44)UMb zKxw=7*X)j>-EG@%YU^P!&$o?^eZvkTA_6fQPZ0%(?5<6hiqz~U&R#;%l+knzsjkvW zU}sUo3zz}85}q1@NO7pZ*da3jxSAJ|-uY@GkcK?I(x9?u}JMxpFa?7z41 zpm$@adv%CFBal*Yg$UA^O(9Z3LmO{OQS*+Wa17y9y>`kXfrP{|6ZikSB%e?_D1iZu^xp=x zSUa$^N;8HyDfbb`iGfSJFQ%Gk_XdZ_K=F5Wr`rg#KVFqqbA<;LfxidJ#@{2Aq| z&>h51e`CGE!LC~~WBaP`FYwi2L;{Xpm_i!rZPYat{;mO>8|JAtT$ zkex9NB|tceG!5YRf)gr%pC}lm*|-w0e|APA5-NtKs9b3)h0%`)WXx!eJxnsCIY`C< zVT)SF+7kaR7`?!9+(R9+x#{TI@7OOQk5@3$bXk8bwa;|GSPYnYaU3FyC|sAgQGiHP zz0>{H!r25@V96iK_I!uM1ziKdE$INKWXevG)aP72Bqm(`o&;E|tSokKgFZ z27iZ8Ds9GuKd7CpTfJP9APQoC#WJdwEC$>bRSb#RwM*1yiFMG0F@jJg0m(6zil338 zVoOTPW*HG~J6fFLdA)q4TWki*(k2#=}5wj4m zM1aFFVsZ?;gRBh|z4@j8PO%?@D@4Z2V*+{bsK$1Pu>EhQv;oU#x{m>&fh4ZGN3lPNuYl%cn9&G$Jarj~6_(bK)!Z#tXjeLMGW)^wAToh73V#P& zuFrI4w15iKSEB}CRz)~8;*(Ab8GmFVe^e}6y}JgY$i(U@ih8PpT);-dg9z$F34xv3SmT$Oa**828Hb>D7kjTykOSF2Y*WJk;}1FaZX zHms+j0vPH3(45|NNEvXUbe$}Yl&G2jSF$E#BT9ZNU~8738nt2dRhU|jp?^3we*JyD zq!TjG)*vi9O#QqI@o`*3+quhMzC!0ntQMe}7vW7E5VraFm=A zN_uGZzZ4hsXjjgX&J0k5-f5yE5NkpwNs>t&s)L8J;Fyed%6`niXzq&s+N6H_3fFqs zSl8)zO}X|0Md-o@sZqnW3TwF&X@Kesw`HVm7=#{KwWCoCM2IXgs_#G%-5gYf>P+Ei zLRfh8w}Ov5vu7}90Dn3FZ7HwsCL;zyX?VrPMR9P?HqJ*lsF`^D#Tytv{xoh=mLQbI zx($Sa8~Rfb=F9KOFrD4g`8rqX2eT8`G(~o6*#Otfo1V5JNJ5FLsw$uet~~JOHr@bs z!^t!s%}C9BWE9>GXXi91_J=+6q20Lh+rKWT*+>dAH5)c)3x7E8BhMC5QyQ7w=zEHK z26NH=(g#Mjs5FOxkB!7Il!%g8En;F#NoEgc1gLRnRd=~16=uS};9sE;nLrV@rJEn) z37{m4B}wBFvO&1ofL}T=TGmQ|Yp=ZV$ajp1rN;LPhGo{ZZ&~1KyY-MZW9KNvquQbx z#%#jEd`M;zU4Q4JhlOAv(IT}(77@!k-SJ&+_8oZFM2^igIgoQi^{l8d(NS@7tdx1W zEQy(zJGB$Sb`qTywANk$s%LcoKv9DE)r<~h(9Zk0Z{rjD70qLaNvt7{fPyykC?=^C z#SVI%q|!HYR>vl0hH)ITYa8$-3x-uwqtHB7RIr80+ke0+PafrThGUuS@*MTl6max< zqpfG0D_NnGOP&-46YSX^n&*U3r5~u>KqjoEWm@br2OzK2WGrhy@$Nl@8ds5pAqX+d zV)S@0Rl6AyVXO9Fxtf`#){5DyRLn9@&;W*turgtqm@&B6s*MWdeDlj9PCdW&5)9kh z(Q-x69CeihAYT00whV3u5sK(vpDELgWFdAGlfG1g1Q15_{Ka@OOi8B$}qn5*DS zL3Ysj+jp;hZ2KKUc*018QYHjZ+XVq(#)+IFB18saABFQs+KcIGMD8bR58!PRI$D|7 zS72FHf7${9cZ=rKEEm`leQejUo;4U4>xM;iMt@N>r8prW5_OyMY+q1_1?)RqypU=` z62}lJ{nWNuAmceJT0T}%0&3??d{mul7xo#MacVccTGX|xGOF!?Il86erhg2}Fv77;Oj>(WZu9K+^aoqA$68mXnM zgcT{QPSnmBCKL&vf=Ds}SNrscAhg&bBfTPMX29fp@nfd-8Vh&W{6>YvykjBOb4(9q zGn3DJ&yA9TB9_{V3VTOn^_SEGs+bOUgM4#ykz6 z0ZoIHmvb$DAvzZdN}oiK`$nO=CJj1^FzF)N?|24nkDT=aMG0UV3ZgnlD2TetVxCl; zjmB|@qKJB!CG5G2rAGAynEHNpK|qI}?u|ey$UdJaH^5`=O7Q8Eo2ZeR#JoBWptkDS z$H7#BSYC&8)`17?dOBpW4m=PSBDruC_!Xcg1ybjKQZZCveCCZ4=_7eP3zl9SpNbC9 zIDe~yG(>{+#Av&dSsC^Cof@h{uM$V4Emo>i-|*4G3YD)=A7fgy-PRqLw=PpGC2-n9 z|M&5K{_*$V$$vb4{Gl8C`>%g|{GT2D$M;YE%K_Q;>?auOwQWN0zE{n##%=2 zcHrB6qWdNr)Q(3NW;Ju(eis_)j}rVQLn6QVP{>^f(WX8ITvqgZhfoiTVUd8B#_q1d0@A2}|z2hhANrxv^hnx^qArLLd_* z_O3Cf+OKBD-eh%9pi@SL6^A8oN0V_A7Jt12^1iuvR+}S?{?)+gp?1TOkAob{Y^-OA z_5aq1`kEosDx2-^-36R}MPa7f2`t@(k~W8uZYg+l7cRO?xacheh~BS|(6223bkk7J z4TCy24By;1Q1ju2WNsn&a`Q0DjRPugFm&=x;uSZIPTWQ;;^vWu8^;-L8dbQ(7=OYW zh#1^7K5+ABz;7w`Z_CksHx=La6-V@K9>2GRXuYk(=4~8__fX00KE>2+D}ruI@p79- z$88=9x0T4ZEycNg0iDBD3Vr}#Iv@JxVZ6o&Ven*;Z5>vK|9=p%cV_S$7+e&2E z-AIYWC~P1kHjU`|%Hy}T5v{c~T7TRy5^Hlx+&Ic=OEFcOMo?`UFSUz~+E#Q_ej9}o z9_756A0Jg(A#fN%Dj_jZ%nGMSbnWprHy2OGUhZb8yd!9^(VbMzQ3+AUk%xl;ji1y~5l@lO3v40nab5`Ok zPTg0FB2G?00#CFHFyFN_Syb|Ei5zp5jjExyK1*q7nl@LZ+1p5n6xMpi{?~|wB_cs}r9k0UKQ{FXVBnb9cM_lu(wv#B$SePMpVku_aJg@S0!UYH7 zm9!GZmtigY+UhKYF&Yv&>wgN!eRUqd;DW{+5M{KyBOtm5(HS*4xJv@&(WR@aaL!y* z3#y&^l3d1Tj;!*WwRlYpY%}LUeWQaGGoXpJeW@>bZ0i|on`>#>btSGSl<$AMrfv70 zD1}}+o(dA;xan{(qs^vKjoTHZvQr?nRO7WK8c|u?c#UD_k3RqcXn&T}olN$MpX$rY z{7TKm1X*0u+C+%OjVCifA}&vF$i$k6ie|xGJP$a z6LGQLcFn7A=?|`UxqpRoF3X;5xi>oRx2o&zqO)%%I@=J?CL!7y!nWeH*V&6~a|g2Z zeaHRWZR}#f^D({Y9mQ?%@KtsV+uA4G$j)FJdx5V)`@h%v{_dE=1JUUpycWN_KW68F zc^DoT!{=|o1>=~S8l5nv%py08MY-7zQ~tHU{Zinxp#$c@rA}F>owW~Hk}AntahjKC zQ>2~xbI@AtM7#LP zL)?#y=V8b?^^)4C62tR7a0{EnZ16A6BulWgV0@9vC;hWIASG}le@Fx&cH@_;T5;Pp zXclSIeDhPtxbgW!p40EfPNzEVjcXSBI$?#6ymYW|eumi{fQmn36{^aGWOF)*S(_+o zvWuc7+bEhJ`tGxlqUG(Rb2lEIKCjM5>}f2Uqss%bNELw{^a^iXNq^`kkQZ?_B2+1r zkDr(b^PxB8v|f&pe?8USJU?^Bb?rnW`N^<~ncy+w<<9hp2jPZgj2NFuAEqm0e6f0hTD9%gWbRo%4Qd4e~xi!+?)+=x``2O^+LVP z?_jG^=8_k$H54{H!^oI7ELy~3`nBRY6HnCN)#EVNW9Ebvc*ZF|ldZLXH6u1R+rR$# z-#?NV%qsZTKmSK@oqzrFzki5Xc=fM;{?F2hy{i!;mN^>a9tnf7b4I2*!#;?`oY#l} zQ-tGmjBPHAV^!GtdYNMuWrk_WO6%!VW1d=cUgX#`9JmkzgpT%`n)b9{Pq^kfx;{;u z`^dF&7Lnl)bY>0#I_;iO$&F;eK9C{{$NTOf*@2SK8b}B_;erfiPZzI0FPGq1l5=t@ zUBa-&lMfmge@2GeQ7q!3ynr6r=oovV+_x(u5#bOTuZEj2aR4S8h4(z*YO3zOK|&|o zTF(C1to3Q?j%;EZ2i-AFQ&?U*@jzZzUuiP741|64rITPvh9G3ewgFsxIQq3o9ZI#w)NnxoI7^heBL@5GFIe|D1^1c5xo~lvE zC(fpY#azy%)*@Bl+tar_gP9_z5zTXD4T<7?eH2o0mhBO@L1bTpwIBensdgP^1~yluHL% z{v7P-f7b4c7GTSEs3PrdvRA4daj;gWa@WH0a_mt*4`r&=t!YLDk8M~?M4$vmP^RFy ze_>Kq1FF-@h10;s^#qtk>Ti?sG@PzAf*DzA`TlZtdOHN3k+s%p`TzYg_} z|0e$J{J-qbRo9=vnO0gAM5*J4|Nf#IbPt2>p$B5O+tdF;zq~tX&N8QQZD4P1*jf49 z*~O`DE;YMN)v4JYT3)pV3BS9+^Q+3ihVS?Noi(B45kb2?MbSQ%I){D**C-FLMbi3W| z^W$UvZ@1g6{d@TQ*^BOXhsQ6Do*f;%cz*oiyY5l<+0$p=q3-s(JNslpV*XusgEFMln&NpFg9{dlUZV5Q(-^fBDjK4S##O=q1E} z1$0Ui-j^?}&z}*YVVp&P+H@#7F&;oH+Ngtm*QRSK(UB^bhgxw+V-jM~)O!BiU?3od zQ0CjI1j}&zvu+??*EmfV_d`gIC5&v)lwm6VI(f-=)6~|@*OSGd7{ev{{oFuU0nuti zn&Kv8Skxd19>GNx$9)JnNPkg6uYb|lN%l9K#Lcw}H2+ir%K2CEV$6g*2YJo-l?m8I zXwSJu2jN&SL;)i0C`r_I2GOB#f>VHUvdn@@~20ZbWo#`} zCNGcLGcN-*5jcvJM*Q**k6)dhUcT=4pLhz!fFt06rhmfd;(rq5m8LJ71h5APe3Bdo zhW`e+#CoZ?x2NzBu*^6FBc(IG{4=ABU%bDBFvcV?+_8tAh9l^q@`s6FYh-wqOli^gE)K1son#5pLvG$Xf5-(s+8wm@ISmI;U## zC&VcpkeGO5$~jBx?*jDd=bydIhJ)HwFUx2yP3f&O882M#V$Vyi&s;WaYdNN}Ec@^> zpW9$5j{J}fHcf|O znT1J*V!O+62H@F0usIa z!{hU}r>{Sro%*zkg6sU@@k#&en7_&I)p=XuvN~rw=vfc+R*ia$kYF|?$8H`Z-t^Sd5*<|@$RwxomB*F11#CX~_ z;ekE$wCd`&lDhbOSpA#=CoJ+;UVkhZQx=Jmf}5qI)6J`Fps3D}@__>2t;Si3N0uqI ze{hd&)ye-hOCXRu8&41f6?gwkG9nutaMC}sd4JC41_D$? zdEvIlQsQuYS=@1#N86b7`!g9rI1OWcL0yqD@Is*^?1fS_Kcsa#bb#VP3}-YZ)ZTGt zSwnzcr(;Nfh3j7_2Wzi<^DKz} zdHVdgw*C)a9KAT&t^eD2oRdB!9t#3J!JmGic!Ei+7L#WslYegUR#(TW3)-4fin-KY z=oHKG%Rf9my*zvW`tsxD>z_XM-=6&CZ=q~9?C6wr#b(F+Z0d$- z^)J8t@cN@R$DY4>^ZM2dvP<{`JQIDZT&b*FndbQ*+@kApI8V>-HbjOF#aH5#+PN8< zSR`8HV>1fKw}0VX+xe|-+i$g!dV9kjtIhn?H!O?p-dWaQaZYW0_%jnycmHi#+wxX> zqG(2GDW)?aqxHMa-QfTk9^_>^Gi94W2K8(9<^t_RGd-PQmaB>K)m?pF>R8Xs=hFt~ z`TgE&-MT0Vcr`a0$+N64#*R!Y)o>Dt~;h$LyOQ@)BKM=EDAiWy6Frd~I@ z3L0kA3V&K6r=8XwI&<@C6DlOunNP4N(v#{m#8`lvhCEVyiwrW8Es7#c5l$6*;zX_3 zJtwuWLLp>H%BV)Mz?RUF0h=d0uVAJ#wpesj$!j9Vtv&P}QxeB+M)ZAcY7Z=s1Yd#B znWxGW$^|N&)^<+p`zHEkJb1|Cz?#?kj7z{**flg}(NHZ~wW4VHc#)&Q2UXuc&yU3;W5TC~VA0OZ_u$GV;jsNQ&KRrI0iT``H+yC9lQ&R@&DUnPTCbfu@ zA1NCL!bI0AXGE=&K`A7EnYWuENGEh!x>G!h4Z98yPKl187CrQ#ZQYcPRQk|Pi%4&4 zguR%MswnE)($XZ|ipkfZ^4D(L=U;3G^|8K_u zw8;PS;>FXt|L4Vj@$UT3tvoCHe=dJ`bz)pUxo^T_QSP+%cFvw{IeRwV#oV5+$1ftc zBpc^+Bj6b|Nlfp{?&OX)Aj|8Y8!DcgZb0tKZlmn=;(dqRW8TFMWcv-?i+xv9cW2vN zUS6H%93P_Mt76QW>1}y;AG_bJNOz1-^OD`nM=vkH*zhTjwS(quhxKl|je%t+B?-*_9*)0Y;@1B{b#{WBGh!ZlRjKlWYSeXCo z=y{F*A0ItC-tqt2c=pf*R%!*+w=S8PH@D3WG7>9)PJmJzUf~glP9FVT-1{iT5XWc~ zvw?|DC3LipIK)`rn44lbcHUzewe}DNb7TrVPIVAkenI8`deT8}X*@-Y>Y55AlmbUF zq0nh{PWvB~aDmnyI$=q|D0+X=M-kzo)fo|ap#N*oZ*>O0@dN$e{Ka^5p#GEp6cc(- z0vg~`@G48SO;fbK?}+QP^?hf6uUg-CWRkYN|9{$y#|g^>Iy-$WTAh?LeWzfDM1T)W mInMsx>P$q)A~@JFvfZV diff --git a/charts/latest/csi-driver-nfs/templates/rbac-csi-nfs.yaml b/charts/latest/csi-driver-nfs/templates/rbac-csi-nfs.yaml index 0fc047cf0..9bff94bb7 100644 --- a/charts/latest/csi-driver-nfs/templates/rbac-csi-nfs.yaml +++ b/charts/latest/csi-driver-nfs/templates/rbac-csi-nfs.yaml @@ -57,6 +57,42 @@ rules: resources: ["secrets"] verbs: ["get"] --- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.rbac.name }}-external-resizer-role +{{ include "nfs.labels" . | indent 2 }} +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "update", "patch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.rbac.name }}-csi-resizer-role +{{ include "nfs.labels" . | indent 2 }} +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccount.controller }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ .Values.rbac.name }}-external-resizer-role + apiGroup: rbac.authorization.k8s.io +--- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: diff --git a/deploy/rbac-csi-nfs.yaml b/deploy/rbac-csi-nfs.yaml index 6d21b7b20..21e36effe 100644 --- a/deploy/rbac-csi-nfs.yaml +++ b/deploy/rbac-csi-nfs.yaml @@ -64,3 +64,39 @@ roleRef: kind: ClusterRole name: nfs-external-provisioner-role apiGroup: rbac.authorization.k8s.io +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: nfs-external-resizer-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "update", "patch"] +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: nfs-csi-resizer-role +subjects: + - kind: ServiceAccount + name: csi-nfs-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: nfs-external-resizer-role + apiGroup: rbac.authorization.k8s.io