From 9bdeff23e1b297313c72fb1db6ca0c3b0c735ce5 Mon Sep 17 00:00:00 2001 From: umagnus Date: Wed, 29 May 2024 08:55:24 +0000 Subject: [PATCH] add modify volume --- .../latest/azuredisk-csi-driver-v1.30.2.tgz | Bin 14733 -> 14750 bytes .../rbac-csi-azuredisk-controller.yaml | 6 + deploy/example/modifyvolume/README.md | 83 +++++++++ .../storageclass-azuredisk-csi-premiumv2.yaml | 14 ++ .../modifyvolume/volumeattributesclass.yaml | 9 + deploy/rbac-csi-azuredisk-controller.yaml | 6 + hack/verify-yamllint.sh | 2 +- pkg/azuredisk/azure_managedDiskController.go | 73 ++++++++ .../azure_managedDiskController_test.go | 169 ++++++++++++++++++ pkg/azuredisk/azuredisk.go | 1 + pkg/azuredisk/azuredisk_v2.go | 1 + pkg/azuredisk/controllerserver.go | 67 ++++++- pkg/azuredisk/controllerserver_test.go | 119 +++++++++++- pkg/azuredisk/controllerserver_v2.go | 67 ++++++- pkg/azuredisk/fake_azuredisk.go | 1 + pkg/azuredisk/fake_azuredisk_v2.go | 1 + pkg/azureutils/azure_disk_utils.go | 6 + pkg/azureutils/azure_disk_utils_test.go | 32 +++- test/sanity/run-test.sh | 2 +- 19 files changed, 645 insertions(+), 14 deletions(-) create mode 100644 deploy/example/modifyvolume/README.md create mode 100644 deploy/example/modifyvolume/storageclass-azuredisk-csi-premiumv2.yaml create mode 100644 deploy/example/modifyvolume/volumeattributesclass.yaml diff --git a/charts/latest/azuredisk-csi-driver-v1.30.2.tgz b/charts/latest/azuredisk-csi-driver-v1.30.2.tgz index 3f73abf7151c664ed71defad219fa037bbcb2969..4a07fd9360e03540d1623e04e3bbc51acc807846 100644 GIT binary patch delta 14652 zcmV-CIm5<{bDndMO#xAnPA`8d*=FrdTdV>Sw?rb5_+!$e&@IxJ08Om#CDFy zkcsXT2L2DrczV5F@9^M2{oCvHivRZZ_Ip3<9vmLLI^2JC_-g-$-tMctSNlJJ-cm|$ zcp@$!`=PgRS;fYEB@YboN5nX$qz@*$TQCeWKfC+A?%tM%xXW-T)XRS($uB@M3^=Al zKuiz=13?)aA#g?nVngU6@ZxxI_A+^Y=|>|>wg?PRAJ{l!qvHRtFd93q)0{sZ_ zr3rxg`M>+Rceq!a|F8BAHuHZS4-Vi6^|t`Hg&bX>koG~~vTlG~MmZgdu1kY0kWN~e z6aa=o%CVqq+6Oz7^PPXY=oT@81mcp9c5+;$atJs|C%1eNM!tVZee6#A;B0tK#U(== z5wQh;%kd?n6D%bHV(m(l+#9BuGG)H%_PV<*iaZhmy5m{OgC<9&u!HVltHQ1j$G^`~ z7SkDWeYv~aT$Ya~h#<}{8NEgAii@*^GE1hwz3%RQx7So4p&oxaQDWo@jW8Ds&d^Gk z*$5$r#o6;!*lnu97DsG?U3BESG$KM4?lJ@g>I1-|01$zv@>mPsIQk?H&7xF%k zqAx&#VVCO^d;fXZJFrjFF@nArr^#eq7Lg0`9&(W_S-mtvcCbqJvs4WnOFYUQ=~XQmfx?h z-<^>Gaw&h2UmYo?d^&oECdluDSF$r71cOIx#QWga-?jjt0{Pbrx|ob~^L1}~X*G2k z!+GaTJfc&dK7^DB#7>D9QcOf2bjCu2{hb}3y3ij}F8T-iyRT)-1@#ewQp@GK1`fDG z(>~~^`NgP@x~atKV!9Kkap((7j*%DnsMCtR zX-_DJbmgTu3l$?f@*IhK%I*>=E5?`uj)cHu1Y8>Vo|F-WBRwfX(*Xk>MH~<+fQLdv zJU|I}KRAoCOYVWU$mf0V`HN&iLZ~o9rQd|qdzVcSQtzmgNOr*h`N&l*)0BooOt6@m zpA3I-8Zj3c4zMXAa;^qg@c5UU(1BXiUiEtKv3bkTKO@AmsoXG<(&91n}VlfqHTCQ6H3-nHKBn{?=Bk@AS7U z>rSeUSk$5|0FoV46|=Wde0iG9Y?Y#Q!_yXu>Vd?XtX_wkARt+R!Y z!2k)wl4@b%G87;ppx+12J95T!o`Yu~nG-;pa{54j+Xfv~iw=09Ux3>w@X!!OzEE2o zZQi?#ZhhzfAg1clujKqfn}r)=HiJ_u1h3?EEdbIvrem zxH>)sZ?7&sT!I&8L!}Jx5V?4Wk++@jRK2^ zWy;swR7=2-Y)b$|A3VQ9)7>`{=Qkvl)3SzlShep}fU8dNJr_$J#1#>w%m>^GW zXqhKfFF{oXfC(3nxCn@8h%y8D!r7I*Qj+RMZFeIsXz+m}b~HjtgO&Y``BgD9YcBcX`KbIjgD3WkgZ`lnt&6w@Q<$;l*DhqY}aWQwEs)mFTsV1~Sth+#4^nX zCA|q|B7(k?&Va(&x>ohO($M$>H2y{QFcle=H>GhOu#jXIN99g+kpzk6Uyhcm~4N<&{GjZujZPn2aAZI z<|6m&id?lAQUz@(6M{ML*Bb(*;csm{dj_u15HX~bNeU=IfHDwJh5%Q_25t@nB91(8 zKSo6D8s*ZjHp^U+U})IaDqVkPCqJt^>hc`{`5oVJl`SA*$O);Zc8yBl?MQl#BIbk) zPoO~YdC_!d>)C&^E%oU#@_lXn&{pPWz%d$l$7GdMbd@_>TT)^1DfG@fM1+|zu8$EB zXD3F(dv42+0$`~2ukwpI{a_B{9FL?b9;hP$4`4%AsyA|Xlij)q zL?97z%OYR7e#EsUKx$5iNX^M4IXeOOSd3M&*-AA8 zMZ~v3=p%o~5nvJNZyCkY6+Unr5*w%AeEtl&S4gGnCMIj}<%?1?6*0%*MiF__fnius z$Rwo_5%A?p2RP2X)KfO%EB`di|4(D&hlug62VyB-`SN9JOC^6heF!1(lBoebBK97;7G*ggLj2SO^=$zDJXN20mc4n>$M?CB8WVZt6otsZ>&Qr7fUCI-m})#d}qWSWuB8gxSIkbpI502G=(UlRsCrDYJFeO_763~MJ_6rs-eSV!H z4M&J5d3GCN-$N`X&?dd^j|Kn=L+rpXWORbOy!K8h8DTMwZo4iGif7)k1&n_{ zKYR49%nCb)hp##@Xc1F#jRV9548lGjk?$v`c;-aTbRtAV_T3oR|-oj3OVhVzw;+xJyIS z2Z=83Z2@3LLs4-uC!?1P6M@*Vq>+CbPjl`gXT-yo^1u5Okl` zjUeRlJjiuhA8DC&jc1jj#Ga`J{HbAgD^ym0J89Gq*Te%k(lN__e8IM;kGn8wxFTc1=7`g%~1vT4^eTK%n3 z#dPIU07XzDYYXV@KaeY=Dxskx>XbT@J)I-<9Wb#`ujqh6s+OH`6<1NWHNZq2w{!fv zYO}6DVpG@rI-Yt=>T7biGpv7*8SpwDR@^?ozZaRf_`a#;*M*_6z*AzAF!I3d)RL&G z_a|ji>gwhhN6=&%6;mc1ZjwrYM9kd9a_>UB;8L3*2o(s3F%(js;bCG( z9jcU)4WTO{==;-zs2OOkO&&7IEfwQJ6%<~^UVlLWB0OU0Nr=i183KRmSV|Qt0D|rq zP%%cV3;ts2^8Ul%T8bZCSLA_^Val}g0UQ;?4(6u))tq)qBjQTeYu3Sh9)H zRV4T1J2`WDmUK6j0zO?kj1b!flqHNWs7%6;Qz?;^qdg7OzIHK%VI_Eew=mV8pN*isCWi5I_?1yYfttyG%Y-AQOL=k_lpRt#Rpy;NKJ4 zN+ifa)jmfQXevF1KwqCiY6x5ar+^_PRU(q1XPhO$d#R>iGNkFT)(DGOJ$7jjL4z5#Nog9o}?-$l`o(_TRFieJs=|NQLk6Px`gb8Upz}$NnDTN{evE zkU>`+=WIx28+4d0Wb7i!i zZoW6{azS;7Vxsbiey&>#r^U!mE>{nw{+9rtiH9DeLAOK{fNRBc82O6PaS%j8LG$2H zBodR7ZrY9#jq0YTuT|=|Epefhm3cD_FDTbmq5xdlC{=3MMqy2NA{A6K-E9PE+TmT@ zvvNlx1qgrVQJ_`dTV#&3D~PIt?&PQ^KI<+lbg>-IjGnGR1<_li4A;foWa3X0Z(O7V zw`}8Vj038f$DX`G0K`w_Hl+olROYQ<6j;%o++yB-R|ZLWPiJdfsvndGu4;(P*0O@G z9yc{?l=xFKcCerdNQV0*6>XWqE66T5nEIpcsnUN(O3=m723U&shkfvCr*h_Zerw2C z$qJ>M6&th$$@|>0MO2)I%Nu=5QcGtx?JsR)REbh^xV4E99}NW{*q4(S8&i_r!|4T7 zpDg8fxhWS$+&d{)Mo`%*83}M7_a5Y&x>zl#ITV1Fo!S0vH~JZR z2C)Q1mU3KCH7c@C6o5>Or7}<2vX~MRY9|EEAZAKXT6>PD8r55b2nZm%E9p?W+HpI# z^`X>@FZ`%><1CbWP^1Xe+f0(GJ&5eo&R zrBFsijda4s-^m?tWM~um}LMoRmNDL;>vOm<%F(pzzkh_718%xVH*=OD& zaW4m>tpUZl_u#T$dE~l?^I_zJ7q?Tno53D5Y7Z2%nQ028=)FqLEb|iFqM>B0i0gkM zN(L8`wNinsH@`GV>iXO*1h%!IX_sa#Y?#|sG^=UtY-pib@Z`cG5#2nM-S#rZP)!*N zQ8kIvQTM01OSS2uR>7Gf(FNxh*QYkIeN6$JQ0#$_a*pM8fg?9!8nwwI#J7Q4$I{lC z&c@_+wzd#nHlu@?nSBP4l|N5wVDNuz(wLh?gFVqEb~Dzq3Ik)&un5iw@TLR?*af_3 zQ!cj!xR9gmj1*5+wITL>?Vf1LOe$0IvCTdSwzm@(@kutWlh|iO#E9ASs->=zU!yt} znxm`BW8gyH?}DGyLTR<^)wE*KddhUFuJ14xaZPz`RpZ@)1|-B@;~Ze2RE2-CNYTb7 zEmkaq6FHNNoU%dF@*!kI?uZHFlYe3#Q4b1a=eQc?M~qR{!1XFk%EOEOVBR`ZYrJx`RrE=<{3Gy?H$zR|u>e|R=A?IoOmy(o+ zTg%y}Y2$X+}w~Ch_R-CByq0DIHqYB+Zj#OX}B@L#b8a>r9)~$zf z_)*3R>)g{Ueuq#fnUyJ7pahH5<|>F%*NqsDhao?y6LZpskR}V0nNxc7sAXT*K;JoqT*c z8X@CsjuWdBF=GZX0UQyI1)iX+_GwvyK(j?gSeKwuhl%>-M-S^g$i25FluvAhLm&AYMGO zs9RObXq;?UB&dfnW7kfW0@o*Giu>6HK!=_0^@5bJeLPTEfJg02u<4VFsF8}rtU3<@ z9oe!^w^M<5dmS<}4=iNo(;<_2V4>Jp$;Me=SAeP%NEQE4bA^9CGp`&-8_TOvF#M7^ zQ&9ko^9u#i-~m(vqwG$4W)$%|g{opTOEM~LGE+18wH!^XQ1J}438tp)wrW7#ybPfb zh>EVtEG2|jV zk;~paWJH~-nLr8(cP2@PlyI{gro=N#x2Z5So`yVkx*1bVR8Xn%E0h`s0aQ~UZdgPQK;vFpw0#1n+pRqzg|e@62O;>!z>pDRIU&@`6zhBMbU}N zz#=Y=JX{!OxG1V{i5S8aAO;u32QH2V{GPCXOGf*x3g7n)Bl;G{?=1nXw-jvN!brTY zmEE2arfyjXx+US|7DvY|j)hwa@@+{tw{INPwiJI1+v13|#qnuNK%*@Kd-k-EW{Y6T zHt4aZj2>G8R%|KAu*X>?nyawDBC#l<>l?>!Ed#B!G+tZ~iM2QtNR;uo6QYQC~WF?5qsl3%Dkv|*gX#5X)nZd{lx zxM2oYp`~IDLvfOQ)r2NFK?1Qr9q+|Ek}7LSyyXyMMx#+4R%^3@reIZ?El>rQJ_A(3a`1d~n4$xxx$eLD_4Mb|aTz7E>Cs8`W zIf~118=T$cO2qX!g|XTzi?ZO_Ac#rX%pae_SqP&nX~+;Dl%x^(y(fUYD6Dat0ACh8 zxo}Nr^;qN+%3XL%DPuD==4H-YvZg?`TJnH_)-RJ0P|Vu7LUKRf(YKQ7%TYT;1ucI` z6z^@UX6sm*YNG)M;dsii3;im`UrMUYo~mwlC=}hKQmqOv6j51~=E4gCJAUlo21KHu zcz~3-Xf}PRJF*o+CYPL6F=BG9NC^;bT4PeE9vegA`ATe*Le5#Zt_sAliH`SF;ZEhd zjP;XMxT=him8OF{#3o(3)llA{siS|#F@T}o78e*VUXBaJWeBOO4B{m{4gC$S$B3h> zn4%&4XQXaN%b+Rd+lpEBE%hO*T`s|grLEV>lhJvLB@iccE8=QCCr>ovktV4#+vTIn@K4B$0gJtXmz6tIBN$dL?7lwbYM5q7i zwfOdDEt}Jo$3I=sGJFlkD;YCVlK$0jw35_m!NZlWE<8fPN8Y7D zC|A|oVRu?>0@PuAO|@q8gqK}7GNPn;pKV)OD>RQpIAt}ES`=t|EATA4apiiyge`fc z%l+}-EM7NaE;8^D1&y^RVyS=Jp4F*awPiu`Nlmx_rAOys;7PS_kx$8pFB06K`oL=b zI=2C@7tQ7|mff&byCz+sp}In~rbRO}M!Ul;+M>l;qUuo7oby7Ouj@?K^iTo&r9)JJ z4keC<`CZ3$Y?HV;)X&c+V1+0i9Bj@!H^(+laEsUArJi$APbZVclh%LQgbJP2>}Fy& z08}H`>&#oFb~Kx~TC7d!Q)|j;qexk3F2y)ztLu`?WwUle3j>;kHiYr<;H-wEKn!r~As1$G$9b)|V+z!h)T_R+ z$)!x6=Xe&cb!~%_4Yz;dk@>a?G+|>e1@eu3kFy<+JFA$5tKvnnI*dcv-U+H~o}kL^ z3F;y1Hv1=NZv(B}cv#vzyC5-MR3^5Tg=7-q0cNU~oW!O42i_ua=|>|>q*D3frACGi zt)ZgTbo7ijdiD6!^y}OKLSm1uoSEPSW$DiJrKQ3Z(-=@z3j2R2WywDxF472TFk>aG zkiK*w{^GvZaTX{Af&}MMtsTgxc2F-xwo{nr|#Xq|Mh=b2jtG9L5y(Jm3z*0jWfP2`4P5JOvb#x43Gfmhhu1B zSj-~U=ZpB#6w#$BnyjT!mGNc4c@i5_OW*&f0I6nVY%9)3U zL!=^VaHPWJX_Z{BEYL=hNA7q#>)d)PS7=4VMJiB&aLQi6^N-6FvP{XESV&iBSdd>< zbC^w0n8QF;jMp;=k=tzw_nCb(BL{j7H@%o(o~(qP8{~`e1h?(Vu!k9P^{GZS(BUo8 zQ6#K>e6xS4y4$)EDo|EC`(w4%r?}fQ0ckST9m6n0?X?pN<$3j$Dq~yxHmzYPh3rSi@Z9HIjG(z^fAAs2=srXJWrAWte zMb3YaASPH(vP73E{p;Rul{L=ymt4ou1K87Bbj)Qhy)-kDPl3LNt^u#qndY7#Dv9$D`R?TQ-igi&OYKd(u|hxVqS%bb{Cf zqx+qE%I+{3Ig&LU6Dny(tv@@@)IW2#MJ-@$8tTQJ z2+uWFE4RE=LHsIEV!(zJ165)FoU&SOIof%oD}k$(zwi8h(Br{b){t+jZppO2xg&I@ z*lCGjq{952kD`{hksuR=DLKAR%^=_d!4KB8jZltUs)8&^iio(>(?LZc0hoxxgD$ur zD*twj7z?E0^mosGjp?8N7yn=9e>Z0$b}bZpy}tT? z@ZtKnIy#)HYlV2r!p_p?(JnmA#y~UMR25R~Ym0|gvBGa|-u$Lwr@^Ov?`C09vMfQH zyEnhXyEog0XR8H3+gfBG(Q`&*vl{ef=IBzGv_?jG1b&{ql1t_0h2p(co|u0w6amg= zE?+z*oK;h78zrt@dor;L`=B?IV%2n>Uw4*|s}J!rGQ-xBjp&m3y0LnWkLBX7&qd`p z=@T?F<^tPqd?iw08!^QB42NEu3qv;^F*AF_EYTC@YxjWJc)qOF^JU}l@{~PZHj!CY z6q&`a%tEnQG>xT2XPMC}=_`NoMcPC}(a$Dq#MkXp^fh}CJ(9|~37^x_vu9yHo+Wtg zEX7A>29KOtJ?q1^K@i#(#fNH2=c5{iPuE z7s28$g1TP>U%x@p&)m2Tl754v-(co9nE4H6{^{NS-E1QJZtnokA1W?_T3;BWz9=I7 zE5)CGb7=F;?c3ibf_$ZSZ_kJh-`v0bl-Ki!1?lJ>xa!#|G?iPhCIyg9G7E9OQaovf>43DcI3-UlH^*B z?i2=o+cJ8+UhnYWK>ge6^$P#)@AeK4e%L)YJa~1u|LXA7{tvyq{oPlue*nGJHeEjv z7m)qXTez%Z* z<;&LR&j1tGk30lA36FJsc#C}A0bTHivbqxi_H=4Tufdjbbsj@gW)$>qFz{0f2a|vz zDSv*anidOC7>3P#AJ1MxB^kTc{e7tE%uu9$Wc%?jNqJA%oll{6CW&G(?P}2C*-4JP ztVoiyi%r^{h@(E}2&p&c=(iZakb3X33}MD9)4(5RV26-;M@5cM6)$i_b19&Jbu4rj z4&Vq~M!r8lE<++yRPjr$ysF&!YLq%qzke80F3ypV-Cs4xnIvgGsR3XY9l2`nd|ng@ z`Og_@DFs43G{~{LTKIEDXUhA)$0T~F9iS|w)+1iQve}s#kqh!3a)DSTy)4XO)JOR{ zrgj3AFwt2uo!!2$WBvQmhXl1KGCn&9W%}BZLxw5CVw&PpjGF4W`er7_Wmsfx`hVfb zjZ_9o1bPtZ_;BRP{oo~|L!4KwwUT%h5rG4AqTL2+hfPo)vEro4ylgqCY9;wl-8MGO zOj`41h8b&ytt>dhbh1@R&-73^t6b|9RBBc!QjTpI<`_eM$)cJ2AM}Jo+q~Ob!(ziO={if6cj=+~M9o=<4M=k5KyfX8X z8UV;fym(I3ahxH7B4WrHN!|9%H4UjxN7FZQmD&E863Q+=ULn_qIM9blZ}RI;LPu%E zo+o!-Z~pY+=;Y+;bTD{nNjQ14XL&292#v&>{T{b`^%3+CbCg8Glr*PEuYc2hQ>ANJ zh)98sm?OpvL65(&3;zs*{#B*F*2(Q8(Su;n(Eu_h);p^DP;O6M;fU-YSdvMZ=fDs< z{|xz?SG~Fvw-Mvw&2BH32$4yV5sWXF9s!lfW~Y3rC-!`EH#4Y@KYbbE`1jUJ z`ISz+^p)I7*SfUEDd%6u7c6VJGtW9F)-fHQJNnjrM^6j~lZYcn0VUq3W1Z)v4Eg=k z6>{kWG3_{T`Vc~OC4cx&5c?5BO202sNY#_QgYRUj8xDHhY3}0*B8YP*Wb_ubndxzk zOPOK0lf8y?c@E|i&pG9nRbrcopEnj%F{nm-7++E*602CLi%lsir4v#nDs=ls%74YK)PyH*_MYG@&NXJe z6SOtD;X|2pdUQfYAF}vt10*dDWwHEbvtAA3 ziCk|Omdh?Q3X>ow6@T}^_3`EE*`hFo;wh2XtJ_HV)VRG=3130BENjz*b4vC_OKNFH zo>YlBI(mmD$gePu#&Y;iFOCOipH9zDE-%i`uWfeY011X&o~#?auKK?UxouKgxZ9BP zFqfjiffJ5|L#T(Uj4&qU(F9^&GvaiB#JS92Cp`%qF%3}#xqnG;Jq;@i&>CwZxip*E zgg7uq%QoA)Pb1|lGsGi@tMeD9NF-AOmHb$x)`vCcGgD60=$7PbtnND0mlJWs95_Nm zSlX`BeBg*3$rN@%DFuf|!y$gK5Q<_F1C_!~2>L#~cTf!VvZjX$ECinP5~_ECsLv@X|%R_>lD8?*1Oc3xq|Y&I@DY1>oG%oOnt8pUgf?^?f)$j zMV%qkdT?B$g*F))LedAx)}mO}oLyM8-H0h>D{t96V1fmP{t5Eo)X0)P*w4d=%|0vn zyqo_VB8I7F&s=St_@56~r=O0mPEStHug{L&efsI_-G8agR`C;~mWW7N^1g@RFKB8F zbzyAqAX*{!iFP$1$05@5$#5P5X)oEZ=CT!%JiWLl)bZ)m`jnn26wq{?1PI zX16>+&5EU*PcP*@0dzX$nW-!A^Y!)R;M3*R#ee_JUVeEc%+PHk!5V7Vrh>EAZzi_Ttmvy|;>(Du1z?El2)6&|tmu0+_Q@-syi|UR+(D zUVXZ_es|d*0os*(`k+D$veP)SWg8wV*&J&2c43k%Ph%x8j$-Bl!&nKasyc4*y3vKX zoAY9T{9?^G_jg=WzTbf%=v@$hYFPm4IXY8Wh1rx_Nx>-S43S3}TyIq~#GUZ2IEcjp z!hbuw<3dNUh>IPi4!fds`Od5zGlyX@Vcf?!9nf>a0#vjzyV@_FU}kPE$mg(fwNwtv zF_XgPxuTJ!Co1$6Wz!3$efzeuy+8A0v2$Nz9m}j(jVYdGLM*L1)r{XrtYH>p8v58i zE1PF|&rJSbMm?Lw_v?p7|F2%})xloD|9@+5@9<#b|Fw=M>+V&GpueTP!;S}M=5mf& zN3gu=CYiW}5%X73CfikKS`fE+`?RE`Q>@ zK_b-8|J_%I`-g@3|LXPYz0Lez$77fs5)Zsc9QX7H$Ggy%;6+B4i9bdh{VY`q%>l=W zr;D0hFQou5uB8wKlnl@uo~3b79zS2FV`*IM;$EdOrKP%*>PsC4fbymg>jpQmVT>(8Bjp zEv5y>%lTamm*uh5$;2s7$iW+SPke#sWXB67q(aKv;=JLlKn=)?8T7k^i0*MGaY z8T@T3gSv{dEtk6+3n;UkEz6wntyPp> zY(s$RRNABWolIjV#*mK0=g4ie#sqSo3Wr2NsgL5cCs4o> z%%0a0rrg*v}eQtmSI@qZ|(=p%zh&Dw7rW5BJ7 zzA-XzaqKxmMgs>2c>u2}c%%X1XD4!fI(vUIm?j7_FcZ>0V4|aQ-^WSwm?aqHQi`C}*cXR%`Ise_9|8CBIS8)Ej z+*91u1gwE(^!&J#DLNE*=k46 zb(~Bc_mwHbYR_oi1LP~etaFb?j3HC($IW5%;~hp<9&y;>BCh;|xx$d3w*glfI+|OAlv6CvSbbl2N=7;qT=KDQ66RcLkJ*7kR zZ{mo%ISg6ML$z_pI@V4uPUMnaR7uaGR>j$Kyxc@2n|Of*Zga5z-37whoW*a>;x}jU z?at!kgLMa!A=PK=+TLY55|0nq$&kwRF)>|zEeG~9283$bGg?gtZQ?7X?^hmFsqiru zvncyz6Mv}ai9sc*)#wfIVFP^F03SBMhYj!ne3iRk6O~3SIkg1rwhMD#-^0`yawo(g za!S)VL+0kDdm|IwWU^r&nWLXV&TDq|ygv-6p!0F$+l>lqMZ#F1L8Ie-i$ z*OaT%Cl3rM11=3hD6ozytZuKLK4%C-ybx>Y8GoR}pT?KutAoukj4wWyuWpfmV4|gJ zeE0qF;H(s2yngT`l?6tyoV|V+W%|8&#+LyPcwv7!`!V4jnsM~ zx1Pi^mR6k6GtA~0ODa$8rA_N4P3c*t^CE+qr#0KWo^N>bRA!yv^A5DV)EaYfT|uoJ z0Dn^_xcnA3ALFf5CA%&$e{E(e+^Tmh+!Hm2D0Q_fg}-Z1Z|dq<(sioJS}!DZ^Q`fi z8UH(T^tVR*@BV)8pn(6~eZBj7ga2K}Q`kf6NR?*SybKJl34OU<1g|c1ar~~P^Btmh z%cHhh>~6U{{S+-cl->ChdS}XpET&!8$A5^3vy&WcS&JlT7n`&@5l4N{$vZkW5B?IM z!U>@j0fh^llvsBR#xpId_{D~7E#=NvqttDwmXXBrDUECP5{V>>M|f<0%A;22!#(S~ zZE?w}UR5Kt8ced0C96Utr&WK7c;t%SSSlv{X`-Fm#E{=$ouBCao5chzuG5_9@PE2| z$7Q{}P0yMQig<$}o{{g<3#UuT7Q@cQ99*-Fcmlv-OX4R38g2r6yqXa-Jl6%;^MHg) zYF=vN+A%h+ok|bf;!YL|~-ZMev@8Q(hZE@*>+-CWkW z!M$25x52%Bqqx@y^#2VnQ_ClX#)US}?3HH$_bJJA%EVh$d&&{QE z8++@fnb8oA)IOpQ{(?OPA5jaFwF)Ya8n4w|pqeM-p@}uKha}H7)Phg>&VOyhceq?2 zb&jGR3W<+V9P_;eN!N$GqD_*J`n+5bwYd}aiQfrZG$1u$X)SQqmZ-GZ?R9x&yaAF{ zBW⁣#@VTkTnZ;R03n1uUeI+$oh&cbWLuJ#B3m+f-zg>}^r5 zMG&_sEia(w63&}Dh)cxU-+w{8fut_yX@UQYFI#momzK{20&Tee_w{aXzZn0ux4XIj zb1hF{1DRZ`8}A#P6a_XnU zlj97!=$Y}L?Z!{J3ne!;gH&9AdrF=QaEm|y*&Xr#;D7k)M7tP8DXLY` z!~{Hh)jP=1cCxgw?*sIJ1%N|=7=VmM!~?IpujD?R5|7JLuu#NYC@V6=#LJb-DG+1K zwdDcuF^zmrR(GUyC`M`i6wDk$ZnMSn@Pq{?ZlX?F_BWTWqX zRC2hr0&A;AssZJeAD|qcuccmrIk+D~0X#Is1aYlw#KR}GAqd8l-gUuWu^7`x=mb6{ zNI6s+Qvaa0l9Eda#~xzH10iH0c9&*`&s5^h82Le$k9Q>5P9KDn3kMS}pzm+#U7q#) z(6q!uK2mti8h@s-^+Z0Oq!dC110;-G#l&SuZXuxGo@h#vPZyAlk zLmv7{0 zu>Y`I=sI~P;MuJ}_4c3LgS~^-1^drI@Ac;XpS3)N<$qR=ch^&2l-GJM5)4Xd!|Xx}~ig z!*eD>ZHdb)NQ&;J5wn!%Qp!9`vJ>1^mQp{fXeV3{ccYm-Ic`WfB<2gXB`8awzhL5F zGFpW&GGTHOIdp4^noSh(==LAT6-FBA`VVrZBY)LP<$XrpyyTY5N^NZ9Qgj+vURi$6 zFN{IL6*Zk*HDW7WH?*t46J9i`#rM*skizh`6`q(3>HPAprS#FG6U?$yEMsJOi{Gu_ED59;WaoNCCqo?n{`8gAS;gl9 z29AYFm|sq;Yq5DmdDO07ske?zK(hvHDbPASS1eB#n4sT)AkDS@v4p4`hRPvIIw$tA zS5=)wHrA+mzg4tm2|PDjqvDnP>9f=vgMZbIuRec{&mJH9nB<#%Wt%6+z?pUrYU)bKdq64Z&T4huZuV#J824Y>y%93)+W-*!n zX)FI1aRO?P{|EcKuZsRZulII0@_!wVk^hi*)jmLL0f$sD+3f4Ve0GlwiesrJkAIC- ze7#oj4r5FaF!#-Ru9bvwyJvYIkq%_3Qo*{he34`>%cg{gss7 z@I+id_CtT^vWkuSN*);EkBD(h$pB1uwqO`$es=fz-Q6t@ahKsxsF#0-l3##i7;r?1 zfS4c#hJrFUM&OhP#759X;KkAK^kwq?!jHz7Y!Mis0kCo0*0fmYWB1GV@>u8p6#5b3 zD-!_q^MB`c|6sQ`|6lDKZ07$%JUD=3G}r>*26A+XLOK9}%enz}8Rc{&x-Jd2Kssq< zQUDkVDaV4c*#Pt?=e>X1=ms%@1mcp9dO5CAIRqT1lUu$BBj3NEK6YmVa5_4p;({TL zh}Z(a<@kcpDV7oev34a&?hVsSnKEB>``w)uMIH(P-N`)VL6aj>*nan*RbiKi@-zjiz7D0E;@8w8WAB2cNu~L4S-+~0?)u-Cy1!Ow}D9upia!e9rk^2 zgG?3=j3@&$8g;=C2_PsKL7yXlNAe}OgG9ZIIFi2^pm(Gz%cDdv>idWdfMS$%IwW5l z4#6{U1_K1>2n>HsJxrw~(M%;MlSO!@cZ)Ptr!7Q{6BzOd6-R~o#_yM`?iRa&Zn6Jm zBl3E{)>fv!lFwEBCdv6U@{TAO;qef;42iRpprbx&QQ%dl<{@;^07w>e_>2prgq-iw zyOSUkv!gW6Ko9vA0EoaFA6+px8e#W6BycSIEB-2S?=OES69e%2^^ZSpDbDw?3;6&@ z(H9`Wu*-Fd-Tyr3@7t&81VLX+(qu9(i^w^754p&etX`TSyzn7G=_e0!S?9wk#6G;i zJ{Gef5@*yyJWVtWu4W+`fKF!%fQ$_ND`mHcT}>E*9v^_6zMOcm%qxHFD^i>v$?sQJ z?@q}Oxs-p%uZ|Q`J{`Y9Q{)f8E7_U%g26*J<^%BSZ(9IRf&425T};Nh`MS5gw3<4N z;k>gZ9?_{!?nBB1Vkg84DJEh7IujwnL9gdi7y1**#bAGL=e2CPpgv+yYPnq3zyY^t zHUJ$pzZmsVH^70IVuCr~NC-^Ez@?GzNf}`{(vu=I9WdZg!~vlKcql}~ z1C)UG!_zpsM>$0n)7cPBs|?5f^`Qa{s@euP2eBcMJtQ#{72ZQ*jmr1ND(d zp%lsgLrJO+RIIz^>sT*Xr^i1r8l1jA9;Qn|0PjB#s24XM4M2IBX|b;EZEeN+&S1;3 z?xgC7MJ?I_AlX4xF?$=um#5iGF0JtXm~38ZmF`^8M>%b(O%3b~o4Xbkby8(&>cM}a zhE$ODHhEu`_!Tn0B-5*>>E2TE;G>!q$r$*M^GZd=rr}PxtG;>2N3sENA0Nx#I$H=C z43I!9sTL+KLjfWJ`UCL1BWFzKId}$=IRUgOX8`oKZO~D*=ztgc1-O|34~<~t3$@kJ z=DmyP#>XyDvvC_h;w9Ov$%g{Ew>p3I%cTEwr@v)13Z-gkt<=eXpAN51&)$N|li~S? z%cB$U_VWD01$c2fQpx}ik&8zddD{tZB^xv@D3fXg0b+sxmm8C~P%xII0?2{+of<7) zrhLs!wFDf?wggZN!1G%)+j%pE{?7I-n(fNJcAsk&^gJww1^OAT+ZrJ#B8Gobx$)Cb zwKyI;CBn9}+)TGNvLaoV#aXm~a7!i-4GgC^L{RoE_OKC8=)Ib~oaJ1|K+LhhwBPXsPf;@$X?Y8sYl^ z=$sFS&II{EC$<*7rNiT2-X(w1xPk>bofm{kq0L`zgX5FGe0aMJPS1Wi-v)m@ygWNS zd%F!zE-%k7rIIN`C-*=ZrePlE_wz$hyqGD5^AWn zfLbhlBnF^2g{MPs0}5;FgR0+^hQ=qL@h`KF*}B8Tqj%gUyCRN$rd(Vw>LSjQ1GtPar&_u> zcbL>lxaUWM>nUW{KEAm&FQ2br0zQz4lg(Yd9jIVDp?pz`$wq$+JryzZYN4sRzl<1a zE^@D~$aRY$RnV3)A(#Vyy&+H<{?^vBXW$Zz5JO6tq<|6xC<6gy2ykU=;O0Oe;>ZJc z6GYUmQ7-*zv&PWx?*wB^gjojU2w=Mz^ zNQB(7$XBi(acv2ZniC>Yb23RzkHH-l6P0YfQcV%3$5MaB8CqIZR{V6tkV*;tpRkX# z4dy)z6AKeVo+FjB4Km}KdaU>lfr8M70`cD2`PIp=EAGWIWi;Wx_6~Lm_>Y6vul6_i zkB4}!Cb4^*4d+7I%UM;6Hj_zHDu&WG8?3AtatsJ%|Bt zzjX!8o^wPQm>@q$_XVJ)b~=+LX?A6DBUKVDvHf%aIyM1g&0~}>=Vn&O6$5ChG8#4Y z^Jjk$&&6_wb}MryXj)aK?&re<*=w@N9_sQ*PYqu#scDjvzKL`Gp-yU~NbdZ(13I5N zidK0bq+*@y1>|rfntSa~gsH-w4ly1j>`~O}!Iv*(OS43NtJjXJd3zv(Ryz zt*oB_3^BSFh$w4M_x49Eb9T0FG4TfANF5km8qs@gL)%pWNm(-s%MSU$ePTqg+qh(W(pJN1oEDAPcY*swV@A21n5K4isgTL5sEhG+m1 zUEJFOz>J2X;$%)nFBm2Qu_H+%HJ*Rw+(%A{hg~UIB-1JDM|`e}IbaBSv#jYzP`QK) z$`F92YF%Tk`Uj`v0*c81boKV4t8!n_4?MOr>S4Mpz7`7Xctfd=AhFhQ1Rx;jF0C6u z$m4mC>$X19GV2=8Dn*GsO|?&-Kwo@3(zfxt33eyuI4n8Rh8(=0EI!(P8JB-yYio$y zuBO?QdO79-5uthE$cH#M(?=b;^R@S`CKrKtFyO0sT%b4pde+_u4%OTTcwKW z%BKK|phVUd(3^iCS4dStLr2spbtZc{N9sFZYNKA!0fkg8JL590qHb$|i8^lQ_;=N2 zU4g`=uK9H`^O)4v&*Fxr^oAg?7P(HbW395D*h6q&&l;#E?2t zDJ2_1S47bFX9-a=&|I55WRe>yCWR^}yo|m6f&xT%#L|-xl^-$$)Ukh*DpUXj-3g## zf>;;)#nk2fhvAhJKf12S10lndY3BntE{Gk>P5Y}k?S@9gv(~bDEa#d!$+D>r0OU}q zHaeI%0tYBB{sg>5Bxb{ic!>G4tROLqCD5?N>SkpZ?OY>{${%zKJZR8WPL^D(t8`0R zK#uC+xPq8b5j1FyyAyvDz(IOaNDePfKkf~4uR*^13Ot25fXcxCJejs?Uxlz_6Qiq0 z?#Xv@=JYJ-ZYl+Qx^@^Lwhbst7++ACgdwL=A}dFM5HW%CXLK1gF?Vv8Yuj_#e6yp} zn?NmIjb4E~(_>l~D8;~tVKo%RW5f}FB<6SJnId@OO~GVD(_^hM7O{Hl(jbTk7BdaF z$2XCnjQ2b=MShRtu>+Yq!2-D=VyGu|TUB+UROD{pJxgpE&vSJxO+)=C1+r!Yqz zN@vxrC@;yEC&PcM6jvt)quBd}wVbCzAUh0`5n_4}mHMNWd2%M`>TdPe$TPLhq%%r+zDjR z6^O~X=W;z^lj)l*+J!#5nHdRT?Fh>Ot4oQCLcx1Rw)TItAV~H@{F!YJkv^A3>*?lu z!!8$8hbSg0pXleh#c*1T{N!@=Q0jjP0GfE{F&cDBL;<)`Oox%L7##;eBos6cjzl6c zDe0!|DAA~Hiuzime%lfkT3MMl)9`|FZ6yl8g^g0BhHVtqbSF|lHPhWjkft5p)jcbB zG*WYgfnqy&GRk8FUYcz-wmzji8Te&@G_oRzFl z%2}~NYmmIpEn7szX}G-6w{;C26h@vCNrxSDRQ0axW2B zdR~9qrUlHGyImlm?HuXyD7UCuFlz*rt&))dcX97Q&Z&#llA1#SXxW+V-*%&)p=S_F zP-H2`6;-1m`$PfA#6&9dq%DgnF`;%s&9 zFxnbWta}eG`;|woi#Q)eK6r66le-!0L8JCSF`t>HP>SBG5Lx;2v<3#xCXIi&Sv1%aZDKcLJ*zM<77dHwlmKr=V1Qk~i#FwQ zTYw8W+RjMvWK|ns-`DPmrp%-=B_G@DlVE#0aS@+n<2s3bMnsI5O|M$&I{7uKW1%^| zyf^|b^!+aQNiCFC%U(?@maV5ur|S9+a}n2+*H$&&EoeYO>^05-7D`npixhutY|>)I zLO78#$;c@iG%X)OM&y>5Fh2Pw_7U}GS|7@cMn0<04dh4#22s*r7OK%x9b?^kD2E?q ztgy~K&EmHRg_2pBk_AezNNuiyD0SV4@pu^WlR7aceF$l?Fqt`}N3UvInkei(f(w|T z0>pU|fS~x~)drI^XZQ7xfY?t?V2O(7gjmdT72D`4PLyny%uBP%#x;Me-qp#+r{gg) z&gM9=IuSEw5EH-=;aK1)%4(mMB?vTIWQ27IDs`BsUw-tk-a(%2HoH-wS#~tmy2kX7 zA|DzgOqwvGg0%o**t*aH&9E(=)k|f~RA{q!W|kmT-A{skia4^ZDs9)!`jxUq!Kl*! zDypd~WtChlmd?h7(zbshh;6OV-QfzIh3gP^ohufhj%A&-5(NQ*1_~k@hza7wBa6CK zwT#Bec141EC^L5LWGQfcLZ-N%Z2)xG`Cczb3ERg5l?8az&IFr2xr`dASj?*PAkdL5 z`*brCh_}}vGxNYgb}=0?nFkh%jg@Si1$G6fN`X}IA2nC#GxLASfwZx_8U@2InKKmy z&^S9+APpWsH89HVq-RDEzf-6xR#acGSE%AkTy0juQMXA(7ub6mkQBd>{zqN}-RdgFLPh>i8`|9KTv< z;|&r};~Sr8D^pg@KyiW?)R#C5D({dKQl>(dg$%i=bbo&a4lho#*e^ZYF(HOrgr{=Z zyMv6Vb2Sr4LE+9M>5vj`mcx{IX6ZH+rpD8d=T0|cs)-6JRept1;~;=)3dC*8)-zum zrsS`h&H!{OTt0d+nKITht@Zz-f{RyEC}h^#-^T--enT+R)c{L3P||8B>59Ok8@Om2 zxab1_M4x{aB=lPYfG!L5ToTl|Bz$vepyt;L$y@>Wa(S5L(tye}LMI;uuedBaaTQp^ z<&lR=;|!NY6|N9NxCX@FviQK|(SY9*_HV^#zjfjJzF|b)^7y?Kp!HUQ&089Y_qDRy zQ^M4(3PHCbyxj8WxaF~MD?z@k2%uZ3SqwRbbDaHqvYvOxXrK z_LR|ME5M4a1R3@?t3-1ZmRKZ~MRa}R_^nl-wN}Q9OCqtB=ftH^Rx83(EsLO97B96y zN39ASm0rk^z@sv+>hV#j6#}|03I+1cit+C{`ZO~RX{Ua=lxR#eT^SEFW3ZsvOZ zIc|fqyIhI5KBq8Ndu34;TpI*2DVzD@b2tlOlqC%r0)&z@0>AeJkQaqDZWG|kq9+%w z39TNBd_uVkZz*MLrpCO?SxD9t$W}`pFx2{GG6ITOJC{iA=R5jVQhhmUr>LMMiQ<2~ zjn!-&OH*w$;2@mLICi05<@ifUwb@hE?G}Zin^dY*;e{e9tI}L}L14#^9o&FO6ci7T zG8fIJFLg(@V#wr@(<(+xt`#W(!cA*T3e{s1NIYMOjZ(-t3)fYFI5yGoo+{j_e3!9) zvInC6ZexSFvjE75=Hzj`gc z{aMTAbmj3+SF{X&fa8^n8L9GsMW<;xVlh>&K4VG$YB*X+>a^tH%2yX2q2MF$(jb(p zYVNQ*tu_JbFutZ*^LfI{E*u+C(!9^MEv*%rM`Ah23_-J@qLsB3HIQEbWv$*5DR@^ZK>PhNV-`M0* zrq6Rci`Tlg!O4bO@yLICTLqf1v6lk*#=ghd4#=HV%)?djB3T{Ap=|F2RW?shW%mU2 zkae5=6STL1)^0p3ZJu9{7%wUlTgyT+3Go0k)k{v|QvL&Pk+|@qF(y)}eDP8v!$;Oo z(P}z+#v8qQd}{i2;Q%4AM_0~F@Pe{*XZq4o;fiStC@Y2ild^y09}yR6gfy735?1nQ zLG4?Al#;mpKP!`Z3%{j&hC)ya+%FBFB=vYf6$U`L+@Sywv908Xj%a_RA_orTmu>(F9w9EerVPaRue-lh zsRNRE(LD@8A8p46afu2R&mqipcO(>*z_VvN#zDjHRCDF>DD^~WEOuIzv@*Bpv|K3 z8HqcAWSqns97Q5xXt59tWlM4VSS-dx!SR1+HrJMoBkkf8KF^-C)i$m!_9vYpHpS?! zcSqSRCSymkrei`S_0;;)d#3(bxGib{YuljQYPZf-$#Qsap+?0=3+>mj=ffDLRLEP< zj1n1JF_|z20rXHBI%mzSEIU+%g>z?)!R$DYc=B%@s??n=)PoUjHGMyv9zP6&M|gjs zxmvm9tqS5-ff55Yq!_3Q`{$I^a?8=qBV7qxt^9rG_k$h{PqT)6TXjpO{mmVrbHz?e z3?mih?|c-syp05zC``%meQE{)9|*p;u5E;J6{o*@_G?W4{J;4BI{Ulny8M6q({(1fQaP6D_|AX6?mPV*r@v#N*zXV2|AP-# zN7d2cR9!2?TNZYfK96?cX*LF$*`}(HYF}GCw2BpebMxjm6*~<+?Rz&1i;`st+T6YQ z9p1g!HauG`0NU0f1BqTRBAeBqH#bL@!lX4a$|Lae{FPiOH!l_Mt@6ZlsR(~?Hgoy% zG2yJ5V%sQj_1cq(T{r;!xfH9W^ZdHAd|Z8qpOG21o@_*yEY^+Hb9^EfcYQ7@$4Q@{ znK2jGe&Z{V3fq_=&gVGv+FTgA@raq*BW8u3Fkia|%*ONOK|NnK9xqSX<7E?>WlfP; z49hGPn?=)DS#*{;y^_8nU!;FcL=^pO!bW`EK1E-%7ttfBoSX1DEj@dd_TyQB*Un0O zbms8Lsnz4YVF#S0-EEe4u6g3FG^=x*`F7o8mU4($#`WdvbaGkJo#iRtT)U_r!>YX& zmiAHjRy-0O!2e)*Z-Z6(7A)&YuqwZSjh8^tOJFH~fR!Nsm&5oki{^h{8n?d^g#I#E z{AE!0%i!xbNcy=Ow?Wcxkn|hO{01|>!OTCs`@fq_WZ&%_;Q2$vWl-x&W7L;Lq<^LO z^KTAqzPWw-+eDDB_3rIC(czo>x1ZAe+bh9vFN@e-2|jyskM>u4kM>uJkX{-Oy)63q zX<(h7==I;9KRW;s4>*5czHI%ldFmhdd&H23`K{w}>}ZM96DSB3h^r@GW|Ab=a&%`f z@Y|Nr@Avx$`}^wOe!pM%cWZr({=xpM-MzinulqmrclUN)?fd}x>utJzA}%2N zp}%xl#m0RlkDhABDD>$pKtvR-b_scJx|6%BMMGUbe+J!;8ncr;A`Ln(k&hrp-PG>* z<;&LR&j1tGk30lA36FJsc!PZ20bTHivbqxic6Dk;ufdjbbsj@gW)$>qFz{0f`;&tr zDSvsVnidOC7>3P#AJ1MxB^kTc{e7V6%u%F%Wc%?bNqI-vtxutMDv4q->uS*A>2Z#{ ztVoiyi%r^I#L)nBgw&gJ^ji#INWFJihA?B5Y2c4@utP|_!y-qhiWj(|xfD>qIu^PM z2XKroBHte(mm!fUs`w>WURCaVHA)?*Uw=#}7iUPw?ynl;Op-L8)Bv!H4qdf(J}ZiZ z{O1g{lmek18s^wtE&Ms7Gvx!|V-nrh4p5d->k+SD+3d`W$T@irxj-zFUKZvs>ZAM} zQ@ubXOmvn^XSXlxNdLa@AweyQjL!~2nZCBqoz8pzM08!85WtFet$S} zBb9*?f$l{*J{-AnKX}3D2gVH4CxtT?GMFI!HkT1h@qw~fs* zlh(YMW5$|cD+|sroorRoGd)z!D%W}im6}zGlw(_lIVO-_Ff_vVpwsK*XLr}-_#(|A z*AB|Dplp`qXd^BLpp%)9I{yvi=zkK06nyzo`c_AK(3D!hG5GSOqr1-MsAYXtS7v@v z0|42W7te`0jx$0~L<~7&soTD}q9OI^c=krFGTT2>LfOT~OXT_x2l_DSO@94J=qRn& z^W^U9&7WQz9v@$x42Lf*38!y%EpO!%p^6SbHs!p=vHPeW|B-z0O*K_5`P8q zt@bCASqp@9p^_XDj)${0i+}9Rjh~(3!k$wWO~S%l0Dk~tgwM`yej-##a{0J0gg4m7 zR!0X$!{R#!9KL&J6=YBd=uTQou|t!5Br$rAuokwyZ*=tq6)`&CYJ$ zCT$)@*tC*0F%LBhnw(77?OTLGK_=`!hD@lJ2rHKdF^4_AFvv!+tACXT3ECl_wGtW9F9%4E^cl53Mj-D6}CJ{%D0!qA5$2!kT8S?w7 zOXSiiV%l-wC)EKW{9kVo;q@jYeu6;PJqqe>NFr<>boOMN^yz5jt_; zfHyvMp)a!y>>7B!sS^Z!F}a{jBv!Ff7n@R4N++aDROt4NlxT}xsR>Wt>^{L+oNLU( zPUKpKKm^|S=!(J72)pkgfn&r3@!Uo+xt zh{T!9VJAHe95D+~1-XA|a5W1n4A2^DBDpl1*@QSSN6R+byH6wKEHlDmhpY1!r${7I z1C{(prPhZvXLD0d)##SwYpm`%)Rz-+#2h$AL|EFc(|q8F9m*7TLMa7@N23wGw-AbA z5<`{3P6+xwy>n0yidn7)s;qP}`H4(@ls3JJH8^)%=uVI$gzsY*7l2Am;rvLZy}EjL zN`}T=pkf{=W;PPo3A6~Vple+etz--ImReQvHKhXilDvyaRkz9{S2~5MvRA=C6Xf2i z&|;1s5yz#zuTB9h=OD{t96V1fmP{xS05 z%*c`f*vrF+%|0vnyp#VNB8I7F&s=St_@57#C!dZkPmWK{u1*i%efsJ2-HFXs@e`w# zh)7!UzK7v2Xl4y{VS(Q%e@*6yAqAX*{!iFPM=_al@5$#D%?64IEZ=CT!%JiWLl)bZ z)m`jnn26x}L9bW6*)2~{vtlXd(+jyz0G&>GX6g$3e06m({B&`7{@;JImtS59Gj!WX zu!b78so?zeTS@f@ZK~Ya`FvH2DYmOvej>a*|1>=Q<({JhU z_?LI7Ms4eWtk9C`28C3dsr$iIggtanxb zbC$|G`R|MK%d3;ie^2LE?=Bi7K)aGp?p3Hkb`nRnY{O$En?udsE=-c;X{-gtQOsOm z7%L%FRmUw}H@Y-;b6yOPU#uDDe$Pea`yCj9-Z}ASmIa`mqcfFNm`%Br6pVt-2zivj zhplRcxD(zM2eDW{c#C^3bOejI=qYvB6{X8}=IxjT42ucle?G?PfSwZ;prV!8)qe39 zGjnr6K8Ka7rE*w~nG`n96^$%CR-vybo1HW5+qaeN{kbQLo%e)QLUq3YZfA#yX_IC^ZU%R`n4mSQ@5AkH(y-E@E zx3qWI(eTtRdYe+;uj;(-^5 z@yCdxpQTEnIp8SqbWyYGr4#_hwG^U&k|A2avotQs<7W@)SQ;0*yjN*VX{9cu`cj9X zU{Ij3xq?7$B`_I(P@iHAFM#(S2-GKee!gerMDFIl$LgqY`&A59C-!;Jct*+^=sU$TX;h#c|{95LO}&e;_> z{BZU2`Q_==->$ERe;Zz%ypInlbH2wJIV)(TF+M@Q-$nNp?>@Km>I~gErve2){a1lU63Y*8E})ozqrvs{6te4UAKzSGOPS*% zaeW=CYcu%ub>)ua`WiStQRe0UB=l~%iWCy zf0S9ymSsWs);dZr@_QAp%AJmNT#Bq)feFjw`hHaBl6lOn!V)EusFODxyKb~BE zx;%ONX?TA0%cqOOtDkK&G{;*>T)g}6_Vi4VI6l2>;h9`{qTTwm*P@A;M#`xd&(7+} zS%Hc+cY|8G#9o^vq(f~D@;$mp*`Lfnf6cD1ttI^O;V&nj&d!gkPxS1d0Tr#T0xgi< zRG?H#HCr+_nu1pH-!{dnR)YFdkC-viBM%Gh$90YPs`0Wfv7c+LY?0!DR-Iq_&=0X z^pQcMX6?6*G2m83-xwRXIQE!>^FkG ziwDAzW|b(sRVzdr?B2JI-CKtuf10)ZPj?p6RKH<#;nCpw_IgE-!g@h!t?vAILR%pZ z04A&|_R)}pYjoy&nDE65pa0H*0;xa$-QDjW>=fhw?)P^$=f4l}v^)RJELNNI-_7~& z=KOba{<}H5d%isXR94OAL3-{XrN3PR(nSC9w1-&Rh@f0VhovLKW+}AAMY@_@`%G07jf+; z%oTXb zT+e`*Adb{x%>iU6xu#s5K6zk78E|P3LV^Vaq;)PgC&j2O>k4Y+0GK+#<+r%`7;mL2*>#D3oOi-SVid7rR?7Pd`Nq4`sJLh2E*MA&Xhp^)Vvi^f*Ub)*?yT#U^bp z;%ERmc}K_Qf5Be?R5&5DCZKSklM?HW!FZ-c6~EY!t)<-gYLvPy)iRP;KBaNZULuiX z@d%I2PkGepVz_6Ww=FJN)vIcx)`LklvSeL|ps>Q&mf9_*IxMNqK4g)p#1;k-e>ZV<5L^y@*u8mL}~ z*vO%wzybbUG4GqwB-JS|5F4Fm(yjD{)(x(9gR6bgxLQ3Eo+heR*R2^-yLtp`!$qFW z_|`>te?c1z?B=r04er%ixef018^yh@={nK6)x%ov`2n!yA|3$#YA^I@M*RvIo?;)3 zGdE48&vHAthq<}5ZewrVG&35)vD!xrz+bSZ;3H~bvQ|OmQRB6~3sm!jJT$dt_K@V+ zhFb6$@7+Ya$K?X4a}@nhNPLXqnC~q}x<2F;e{GV4)aTWTsLh?QPy9~Uq5-K1OKX9< zwnU}PZ?DTM;|-9s8ey||6&I>Og{)b+qY@b7V%4fNMb=krscUj;BxXaYMe$jMm#h)1 z4J9#3N|B z1CB0_wTn@dqFNPAOu(~O{rwzmCrcaqe?CC>SO7Q@hyloGL_F}i`%3Q9De<^01q(&Y zg|Z?eOuSsVoB}byTw5LhpU}wnWOc_%hhjiXAd&x(L;_4MW!6;wJSKG<#7~b^)aN7*b#06pxhS1C`fA%Ya zY1S2y#l@PDw1Hh82LrmmJgG!iRNq{mm)5I*T+CnAVBXwwgEJnwPD@4H^bzzBbJQHh zlvLf4R7h4h^0%omoz_O7`bFL3ticu4nY&x{<^$e*vE?{U5lNWT@uRYKB^8wN(W0bU zQsuPOv^xc5veCCcDmmO*fwfg5f7O6;%MVbF&sS2fz#QC7pa32kVS>0;Hsaxv+7JX2 zN^iU1uUJfIBy<8F6Qmrf4XJ<78%fEfgkuje!)GdSXM+5o%O^cawle@B z<-);)3+Ve>dY5NCKQt}zkdG8zvxaGGJ(15RDTR>100|>kF>x7^TL|d4eq#~O(DSf9j7in8;=76^ zcGt3g$0nDJ`Q;mV8tgyp2D(n(33z@hP`&+UXYcjyLBamB-#^&f*nb}4DJ-{gyt}U2 z^=4MpBfa;yq&~{6rYbdzf5e0`10-#`o~oQ?wQV^Ygh?M~51Mq97^V?QI(fYL$$~JI;h%m=!CeakJ8^ugKU+HYwDJMay=!Y!jLyA{OX5f76<6&$;&`4W96#Q7yh#E`=0^x2^ERWJDL2cP*ul z9-UyGrD7!uM5**17f?j2ti&Tq#zzs1$wd2ZrD9&xJYA6(FZp8qQqisjE~oxk^;l!U ze<)>XIemHMN?O`~%>pYGA=UD8+4@yT(~Z4Ss~ul`{v03GwYUGzfBrLn5ZDI%@6O&qG5+iB z!ES$Z{__w|f5Z9DgGwqDxM8)zX=!`gnhq!)No=YfUXQ@c2I71WQJHef`lREx|Fo6= z%Q%5F$p8JlomWNwkJme!^Zy5VjQoehtM&nT5O6>Rlg+;#EVi%SoZ_!^OTDqWKCIPs z^#!i>|hrh`~Uvy-Oc@P5Ar;k{7>!08_BIp84hf{3iho^1r{gSM>io v=x@${ALg0I{{L``LzNw(-BPiUT$^X}Y@W^2{QTbl009608X71?00062zYtQl diff --git a/charts/latest/azuredisk-csi-driver/templates/rbac-csi-azuredisk-controller.yaml b/charts/latest/azuredisk-csi-driver/templates/rbac-csi-azuredisk-controller.yaml index b78940fc66..9210eb5a73 100644 --- a/charts/latest/azuredisk-csi-driver/templates/rbac-csi-azuredisk-controller.yaml +++ b/charts/latest/azuredisk-csi-driver/templates/rbac-csi-azuredisk-controller.yaml @@ -72,6 +72,9 @@ rules: - apiGroups: ["storage.k8s.io"] resources: ["volumeattachments/status"] verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattributesclasses"] + verbs: ["get"] - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "watch", "list", "delete", "update", "create", "patch"] @@ -158,6 +161,9 @@ rules: - apiGroups: [""] resources: ["pods"] verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattributesclasses"] + verbs: ["get", "list", "watch"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 diff --git a/deploy/example/modifyvolume/README.md b/deploy/example/modifyvolume/README.md new file mode 100644 index 0000000000..eddbd5ff8a --- /dev/null +++ b/deploy/example/modifyvolume/README.md @@ -0,0 +1,83 @@ +# Volume Modification +## Prerequisites +Volume modification only work on a cluster with the `VolumeAttributesClass` feature enabled. To use this feature, it should: +- `VolumeAttributesClass` feature gate on `kube-apiserver` (consult your Kubernetes distro's documentation) +- `storage.k8s.io/v1alpha1` enabled in `kube-apiserver` via [`runtime-config`](https://kubernetes.io/docs/tasks/administer-cluster/enable-disable-api/) (consult your Kubernetes distro's documentation) +- `VolumeAttributesClass` feature gate on `kube-controller-manager` (consult your Kubernetes distro's documentation) +- `VolumeAttributesClass` feature gate on `csi-provisioner` container in csi-azuredisk-controller +- `VolumeAttributesClass` feature gate on `csi-resizer` container in csi-azuredisk-controller + +For more information, see the [Kubernetes documentation for the feature](https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/). + +## Parameters +Users can specify the following modification parameters: +- `skuName`: to update the disk type(skuName is not allowed to change from or to UltraSSD_LRS or PremiumV2_LRS disk type, more details on [Change the disk type of an Azure managed disk](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-convert-types?tabs=azure-powershell)) +- `DiskIOPSReadWrite`: to update the IOPS +- `DiskMBpsReadWrite`: to update the throughput + +## Usage + +### Create an example Pod, PVC and StorageClass +```console +kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/deploy/example/modifyvolume/storageclass-azuredisk-csi-premiumv2.yaml +kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/deploy/example/pvc-azuredisk-csi.yaml +kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/deploy/example/nginx-pod-azuredisk.yaml +``` + +### Wait for the PVC in Bound state and the pod in Running state +```console +kubectl get pvc pvc-azuredisk +NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS VOLUMEATTRIBUTESCLASS AGE +pvc-azuredisk Bound pvc-e2a5c302-0b48-49a5-bde7-5c0528c7a06f 10Gi RWO managed-csi 17m + +kubectl get pod nginx-azuredisk +NAME READY STATUS RESTARTS AGE +nginx-azuredisk 1/1 Running 0 20s +``` + +### Create VolumeAttributesClass +```console +kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/deploy/example/modifyvolume/volumeattributesclass.yaml +kubectl get volumeattributesclass premium2-disk-class +NAME DRIVERNAME AGE +premium2-disk-class disk.csi.azure.com 4s +``` + +### Edit the PVC to point to the VolumeAttributesClass +```console +kubectl patch pvc pvc-azuredisk --patch '{"spec": {"volumeAttributesClassName": "premium2-disk-class"}}' +``` + +### Wait for the VolumeAttributesClass to apply to the volume +```console +kubectl get pvc pvc-azuredisk +NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS VOLUMEATTRIBUTESCLASS AGE +pvc-azuredisk Bound pvc-e2a5c302-0b48-49a5-bde7-5c0528c7a06f 10Gi RWO managed-csi premium2-disk-class 20m + +kubectl describe pvc pvc-azuredisk +Name: pvc-azuredisk +Namespace: default +StorageClass: managed-csi-prev2 +Status: Bound +Volume: pvc-e2a5c302-0b48-49a5-bde7-5c0528c7a06f +Labels: +Annotations: pv.kubernetes.io/bind-completed: yes + pv.kubernetes.io/bound-by-controller: yes + volume.beta.kubernetes.io/storage-provisioner: disk.csi.azure.com + volume.kubernetes.io/selected-node: aks-agentpool-17390711-vmss000000 + volume.kubernetes.io/storage-provisioner: disk.csi.azure.com +Finalizers: [kubernetes.io/pvc-protection] +Capacity: 10Gi +Access Modes: RWO +VolumeMode: Filesystem +Used By: nginx-azuredisk +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal WaitForFirstConsumer 21m persistentvolume-controller waiting for first consumer to be created before binding + Normal Provisioning 21m disk.csi.azure.com_aks-agentpool-17390711-vmss000000_c36f4e97-171f-46c7-ba4a-c8567bb41452 External provisioner is provisioning volume for claim "default/pvc-azuredisk" + Normal ExternalProvisioning 21m (x2 over 21m) persistentvolume-controller Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. + Normal ProvisioningSucceeded 21m disk.csi.azure.com_aks-agentpool-17390711-vmss000000_c36f4e97-171f-46c7-ba4a-c8567bb41452 Successfully provisioned volume pvc-e2a5c302-0b48-49a5-bde7-5c0528c7a06f + Normal VolumeModify 18s external-resizer disk.csi.azure.com external resizer is modifying volume pvc-azuredisk with vac premium2-disk-class + Normal VolumeModifySuccessful 15s external-resizer disk.csi.azure.com external resizer modified volume pvc-azuredisk with vac premium2-disk-class successfully +``` \ No newline at end of file diff --git a/deploy/example/modifyvolume/storageclass-azuredisk-csi-premiumv2.yaml b/deploy/example/modifyvolume/storageclass-azuredisk-csi-premiumv2.yaml new file mode 100644 index 0000000000..0f881b011f --- /dev/null +++ b/deploy/example/modifyvolume/storageclass-azuredisk-csi-premiumv2.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: managed-csi +provisioner: disk.csi.azure.com +parameters: + cachingMode: None + skuName: PremiumV2_LRS + DiskIOPSReadWrite: "4000" + DiskMBpsReadWrite: "1000" +reclaimPolicy: Delete +volumeBindingMode: WaitForFirstConsumer +allowVolumeExpansion: true diff --git a/deploy/example/modifyvolume/volumeattributesclass.yaml b/deploy/example/modifyvolume/volumeattributesclass.yaml new file mode 100644 index 0000000000..64dc23481f --- /dev/null +++ b/deploy/example/modifyvolume/volumeattributesclass.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: storage.k8s.io/v1alpha1 +kind: VolumeAttributesClass +metadata: + name: premium2-disk-class +driverName: disk.csi.azure.com +parameters: + DiskIOPSReadWrite: "5000" + DiskMBpsReadWrite: "1200" diff --git a/deploy/rbac-csi-azuredisk-controller.yaml b/deploy/rbac-csi-azuredisk-controller.yaml index 0e59ad15b4..4d931c3fa3 100644 --- a/deploy/rbac-csi-azuredisk-controller.yaml +++ b/deploy/rbac-csi-azuredisk-controller.yaml @@ -75,6 +75,9 @@ rules: - apiGroups: ["storage.k8s.io"] resources: ["volumeattachments/status"] verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattributesclasses"] + verbs: ["get"] - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "watch", "list", "delete", "update", "create", "patch"] @@ -157,6 +160,9 @@ rules: - apiGroups: [""] resources: ["pods"] verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattributesclasses"] + verbs: ["get", "list", "watch"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 diff --git a/hack/verify-yamllint.sh b/hack/verify-yamllint.sh index 6fd42d8434..96ca1107d7 100755 --- a/hack/verify-yamllint.sh +++ b/hack/verify-yamllint.sh @@ -29,7 +29,7 @@ if [[ "${deployDirNum}" != "${helmDirNum}" ]]; then exit 1 fi -for path in "deploy/*.yaml" "deploy/example/*.yaml" "deploy/example/metrics/*.yaml" "deploy/example/snapshot/*.yaml" "deploy/example/cloning/*.yaml" "deploy/example/rawblock/*.yaml" "deploy/example/windows/*.yaml" "deploy/example/sharedisk/*.yaml" "docs/known-issues/node-shutdown-recovery/*.yaml" +for path in "deploy/*.yaml" "deploy/example/*.yaml" "deploy/example/metrics/*.yaml" "deploy/example/snapshot/*.yaml" "deploy/example/cloning/*.yaml" "deploy/example/rawblock/*.yaml" "deploy/example/windows/*.yaml" "deploy/example/sharedisk/*.yaml" "deploy/example/modifyvolume/*.yaml" "docs/known-issues/node-shutdown-recovery/*.yaml" do echo "checking yamllint under path: $path ..." yamllint -f parsable $path | grep -v "line too long" > $LOG diff --git a/pkg/azuredisk/azure_managedDiskController.go b/pkg/azuredisk/azure_managedDiskController.go index 11542050ac..02e81989d0 100644 --- a/pkg/azuredisk/azure_managedDiskController.go +++ b/pkg/azuredisk/azure_managedDiskController.go @@ -414,6 +414,79 @@ func (c *ManagedDiskController) ResizeDisk(ctx context.Context, diskURI string, return newSizeQuant, nil } +// ModifyDisk: modify disk +func (c *ManagedDiskController) ModifyDisk(ctx context.Context, options *ManagedDiskOptions) error { + klog.V(4).Infof("azureDisk - modifying managed Name:%s, StorageAccountType:%s, DiskIOPSReadWrite:%s, DiskMBpsReadWrite:%s", options.DiskName, options.StorageAccountType, options.DiskIOPSReadWrite, options.DiskMBpsReadWrite) + + rg, subsID, err := getInfoFromDiskURI(options.SourceResourceID) + if err != nil { + return err + } + + diskClient, err := c.clientFactory.GetDiskClientForSub(subsID) + if err != nil { + return err + } + + model := armcompute.DiskUpdate{} + result, err := diskClient.Get(ctx, rg, options.DiskName) + if err != nil { + return err + } + + if result.Properties == nil || result.SKU == nil || result.SKU.Name == nil { + return fmt.Errorf("DiskProperties or SKU of disk(%s) is nil", options.DiskName) + } + + diskSku := *result.SKU.Name + if options.StorageAccountType != "" && options.StorageAccountType != diskSku { + diskSku = options.StorageAccountType + model.SKU = &armcompute.DiskSKU{ + Name: to.Ptr(diskSku), + } + } + + diskProperties := armcompute.DiskUpdateProperties{} + + if diskSku == armcompute.DiskStorageAccountTypesUltraSSDLRS || diskSku == armcompute.DiskStorageAccountTypesPremiumV2LRS { + if options.DiskIOPSReadWrite != "" { + v, err := strconv.Atoi(options.DiskIOPSReadWrite) + if err != nil { + return fmt.Errorf("AzureDisk - failed to parse DiskIOPSReadWrite: %w", err) + } + diskIOPSReadWrite := int64(v) + diskProperties.DiskIOPSReadWrite = pointer.Int64(diskIOPSReadWrite) + } + + if options.DiskMBpsReadWrite != "" { + v, err := strconv.Atoi(options.DiskMBpsReadWrite) + if err != nil { + return fmt.Errorf("AzureDisk - failed to parse DiskMBpsReadWrite: %w", err) + } + diskMBpsReadWrite := int64(v) + diskProperties.DiskMBpsReadWrite = pointer.Int64(diskMBpsReadWrite) + } + + model.Properties = &diskProperties + } else { + if options.DiskIOPSReadWrite != "" { + return fmt.Errorf("AzureDisk - DiskIOPSReadWrite parameter is only applicable in UltraSSD_LRS or PremiumV2_LRS disk type") + } + if options.DiskMBpsReadWrite != "" { + return fmt.Errorf("AzureDisk - DiskMBpsReadWrite parameter is only applicable in UltraSSD_LRS or PremiumV2_LRS disk type") + } + } + + if model.SKU != nil || model.Properties != nil { + if _, err := diskClient.Patch(ctx, rg, options.DiskName, model); err != nil { + return err + } + } else { + klog.V(4).Infof("azureDisk - no modification needed for disk(%s)", options.DiskName) + } + return nil +} + // get resource group name, subs id from a managed disk URI, e.g. return {group-name}, {sub-id} according to // /subscriptions/{sub-id}/resourcegroups/{group-name}/providers/microsoft.compute/disks/{disk-id} // according to https://docs.microsoft.com/en-us/rest/api/compute/disks/get diff --git a/pkg/azuredisk/azure_managedDiskController_test.go b/pkg/azuredisk/azure_managedDiskController_test.go index 6d4e722786..1eeee8750f 100644 --- a/pkg/azuredisk/azure_managedDiskController_test.go +++ b/pkg/azuredisk/azure_managedDiskController_test.go @@ -609,3 +609,172 @@ func TestResizeDisk(t *testing.T) { assert.Equal(t, test.expectedQuantity.Value(), result.Value(), "TestCase[%d]: %s, expected Quantity: %v, return Quantity: %v", i, test.desc, test.expectedQuantity, result) } } + +func TestModifyDisk(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + diskName := disk1Name + fakeCreateDiskFailed := "fakeCreateDiskFailed" + storageAccountTypeUltraSSDLRS := armcompute.DiskStorageAccountTypesUltraSSDLRS + storageAccountTypePremiumLRS := armcompute.DiskStorageAccountTypesPremiumLRS + testCases := []struct { + desc string + diskName string + diskIOPSReadWrite string + diskMBpsReadWrite string + storageAccountType armcompute.DiskStorageAccountTypes + existedDisk *armcompute.Disk + expectedErr bool + expectedErrMsg error + }{ + { + desc: "new sku and no error shall be returned if everything is good", + diskName: diskName, + storageAccountType: armcompute.DiskStorageAccountTypesStandardLRS, + existedDisk: &armcompute.Disk{Name: pointer.String(disk1Name), SKU: &armcompute.DiskSKU{Name: &storageAccountTypePremiumLRS}, Properties: &armcompute.DiskProperties{DiskIOPSReadWrite: pointer.Int64(100)}}, + expectedErr: false, + }, + { + desc: "new diskIOPSReadWrite and no error shall be returned if everything is good", + diskName: diskName, + diskIOPSReadWrite: "200", + storageAccountType: armcompute.DiskStorageAccountTypesUltraSSDLRS, + existedDisk: &armcompute.Disk{Name: pointer.String(disk1Name), SKU: &armcompute.DiskSKU{Name: &storageAccountTypeUltraSSDLRS}, Properties: &armcompute.DiskProperties{DiskIOPSReadWrite: pointer.Int64(100)}}, + expectedErr: false, + }, + { + desc: "new diskMBpsReadWrite and no error shall be returned if everything is good", + diskName: diskName, + diskMBpsReadWrite: "200", + storageAccountType: armcompute.DiskStorageAccountTypesUltraSSDLRS, + existedDisk: &armcompute.Disk{Name: pointer.String(disk1Name), SKU: &armcompute.DiskSKU{Name: &storageAccountTypeUltraSSDLRS}, Properties: &armcompute.DiskProperties{DiskMBpsReadWrite: pointer.Int64(100)}}, + expectedErr: false, + }, + { + desc: "new diskIOPSReadWrite and diskMBpsReadWrite and no error shall be returned if everything is good", + diskName: diskName, + diskIOPSReadWrite: "200", + diskMBpsReadWrite: "200", + storageAccountType: armcompute.DiskStorageAccountTypesUltraSSDLRS, + existedDisk: &armcompute.Disk{Name: pointer.String(disk1Name), SKU: &armcompute.DiskSKU{Name: &storageAccountTypeUltraSSDLRS}, Properties: &armcompute.DiskProperties{DiskIOPSReadWrite: pointer.Int64(100), DiskMBpsReadWrite: pointer.Int64(100)}}, + expectedErr: false, + }, + { + desc: "nothing to modify and no error shall be returned if everything is good", + diskName: diskName, + storageAccountType: armcompute.DiskStorageAccountTypesPremiumLRS, + existedDisk: &armcompute.Disk{Name: pointer.String(disk1Name), SKU: &armcompute.DiskSKU{Name: &storageAccountTypePremiumLRS}, Properties: &armcompute.DiskProperties{DiskIOPSReadWrite: pointer.Int64(100)}}, + expectedErr: false, + }, + { + desc: "an error shall be returned when disk SKU is nil", + diskName: diskName, + diskIOPSReadWrite: "200", + diskMBpsReadWrite: "200", + storageAccountType: armcompute.DiskStorageAccountTypesUltraSSDLRS, + existedDisk: &armcompute.Disk{Name: pointer.String(diskName), Properties: &armcompute.DiskProperties{DiskIOPSReadWrite: pointer.Int64(100), DiskMBpsReadWrite: pointer.Int64(100)}}, + expectedErr: true, + expectedErrMsg: fmt.Errorf("DiskProperties or SKU of disk(disk1) is nil"), + }, + { + desc: "new diskIOPSReadWrite but wrong disk type error shall be returned", + diskName: diskName, + diskIOPSReadWrite: "200", + existedDisk: &armcompute.Disk{Name: pointer.String(disk1Name), SKU: &armcompute.DiskSKU{Name: &storageAccountTypePremiumLRS}, Properties: &armcompute.DiskProperties{DiskIOPSReadWrite: pointer.Int64(100)}}, + expectedErr: true, + expectedErrMsg: fmt.Errorf("AzureDisk - DiskIOPSReadWrite parameter is only applicable in UltraSSD_LRS or PremiumV2_LRS disk type"), + }, + { + desc: "new diskMBpsReadWrite but wrong disk type error shall be returned", + diskName: diskName, + diskMBpsReadWrite: "200", + existedDisk: &armcompute.Disk{Name: pointer.String(disk1Name), SKU: &armcompute.DiskSKU{Name: &storageAccountTypePremiumLRS}, Properties: &armcompute.DiskProperties{DiskMBpsReadWrite: pointer.Int64(100)}}, + expectedErr: true, + expectedErrMsg: fmt.Errorf("AzureDisk - DiskMBpsReadWrite parameter is only applicable in UltraSSD_LRS or PremiumV2_LRS disk type"), + }, + { + desc: "new diskIOPSReadWrite but failed to parse", + diskName: diskName, + diskIOPSReadWrite: "error", + storageAccountType: armcompute.DiskStorageAccountTypesUltraSSDLRS, + existedDisk: &armcompute.Disk{Name: pointer.String(disk1Name), SKU: &armcompute.DiskSKU{Name: &storageAccountTypeUltraSSDLRS}, Properties: &armcompute.DiskProperties{DiskIOPSReadWrite: pointer.Int64(100)}}, + expectedErr: true, + expectedErrMsg: fmt.Errorf("AzureDisk - failed to parse DiskIOPSReadWrite: strconv.Atoi: parsing \"error\": invalid syntax"), + }, + { + desc: "new diskMBpsReadWrite but failed to parse", + diskName: diskName, + diskMBpsReadWrite: "error", + storageAccountType: armcompute.DiskStorageAccountTypesUltraSSDLRS, + existedDisk: &armcompute.Disk{Name: pointer.String(disk1Name), SKU: &armcompute.DiskSKU{Name: &storageAccountTypeUltraSSDLRS}, Properties: &armcompute.DiskProperties{DiskMBpsReadWrite: pointer.Int64(100)}}, + expectedErr: true, + expectedErrMsg: fmt.Errorf("AzureDisk - failed to parse DiskMBpsReadWrite: strconv.Atoi: parsing \"error\": invalid syntax"), + }, + { + desc: "an error shall be returned if everything is good but get disk failed", + diskName: fakeGetDiskFailed, + diskIOPSReadWrite: "200", + diskMBpsReadWrite: "200", + storageAccountType: armcompute.DiskStorageAccountTypesUltraSSDLRS, + existedDisk: &armcompute.Disk{Name: pointer.String(fakeCreateDiskFailed), SKU: &armcompute.DiskSKU{Name: &storageAccountTypeUltraSSDLRS}, Properties: &armcompute.DiskProperties{DiskIOPSReadWrite: pointer.Int64(100), DiskMBpsReadWrite: pointer.Int64(100)}}, + expectedErr: true, + expectedErrMsg: fmt.Errorf("Get Disk failed"), + }, + { + desc: "an error shall be returned if everything is good but patch disk failed", + diskName: fakeCreateDiskFailed, + diskIOPSReadWrite: "200", + diskMBpsReadWrite: "200", + storageAccountType: armcompute.DiskStorageAccountTypesUltraSSDLRS, + existedDisk: &armcompute.Disk{Name: pointer.String(fakeCreateDiskFailed), SKU: &armcompute.DiskSKU{Name: &storageAccountTypeUltraSSDLRS}, Properties: &armcompute.DiskProperties{DiskIOPSReadWrite: pointer.Int64(100), DiskMBpsReadWrite: pointer.Int64(100)}}, + expectedErr: true, + expectedErrMsg: fmt.Errorf("Patch Disk failed"), + }, + } + + for i, test := range testCases { + testCloud := provider.GetTestCloud(ctrl) + managedDiskController := &ManagedDiskController{ + controllerCommon: &controllerCommon{ + cloud: testCloud, + lockMap: newLockMap(), + DisableDiskLunCheck: true, + clientFactory: testCloud.ComputeClientFactory, + }, + } + diskURI := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Compute/disks/%s", + testCloud.SubscriptionID, testCloud.ResourceGroup, *test.existedDisk.Name) + diskOptions := &ManagedDiskOptions{ + DiskName: test.diskName, + DiskIOPSReadWrite: test.diskIOPSReadWrite, + DiskMBpsReadWrite: test.diskMBpsReadWrite, + StorageAccountType: test.storageAccountType, + ResourceGroup: testCloud.ResourceGroup, + SubscriptionID: testCloud.SubscriptionID, + SourceResourceID: diskURI, + } + + mockDisksClient := mock_diskclient.NewMockInterface(ctrl) + managedDiskController.controllerCommon.clientFactory.(*mock_azclient.MockClientFactory).EXPECT().GetDiskClientForSub(testCloud.SubscriptionID).Return(mockDisksClient, nil).AnyTimes() + if test.diskName == fakeGetDiskFailed { + mockDisksClient.EXPECT().Get(gomock.Any(), testCloud.ResourceGroup, test.diskName).Return(test.existedDisk, fmt.Errorf("Get Disk failed")).AnyTimes() + } else { + mockDisksClient.EXPECT().Get(gomock.Any(), testCloud.ResourceGroup, test.diskName).Return(test.existedDisk, nil).AnyTimes() + } + if test.diskName == fakeCreateDiskFailed { + mockDisksClient.EXPECT().Patch(gomock.Any(), testCloud.ResourceGroup, test.diskName, gomock.Any()).Return(test.existedDisk, fmt.Errorf("Patch Disk failed")).AnyTimes() + } else { + mockDisksClient.EXPECT().Patch(gomock.Any(), testCloud.ResourceGroup, test.diskName, gomock.Any()).Return(test.existedDisk, nil).AnyTimes() + } + + err := managedDiskController.ModifyDisk(ctx, diskOptions) + assert.Equal(t, test.expectedErr, err != nil, "TestCase[%d]: %s, return error: %v", i, test.desc, err) + if test.expectedErr { + assert.EqualError(t, test.expectedErrMsg, err.Error(), "TestCase[%d]: %s, expected: %v, return: %v", i, test.desc, test.expectedErrMsg, err) + } + } +} diff --git a/pkg/azuredisk/azuredisk.go b/pkg/azuredisk/azuredisk.go index a625b87109..1511e0913d 100644 --- a/pkg/azuredisk/azuredisk.go +++ b/pkg/azuredisk/azuredisk.go @@ -273,6 +273,7 @@ func newDriverV1(options *DriverOptions) *Driver { csi.ControllerServiceCapability_RPC_CLONE_VOLUME, csi.ControllerServiceCapability_RPC_EXPAND_VOLUME, csi.ControllerServiceCapability_RPC_SINGLE_NODE_MULTI_WRITER, + csi.ControllerServiceCapability_RPC_MODIFY_VOLUME, } if driver.enableListVolumes { controllerCap = append(controllerCap, csi.ControllerServiceCapability_RPC_LIST_VOLUMES, csi.ControllerServiceCapability_RPC_LIST_VOLUMES_PUBLISHED_NODES) diff --git a/pkg/azuredisk/azuredisk_v2.go b/pkg/azuredisk/azuredisk_v2.go index fa951ab4a5..0aa018d0c0 100644 --- a/pkg/azuredisk/azuredisk_v2.go +++ b/pkg/azuredisk/azuredisk_v2.go @@ -158,6 +158,7 @@ func newDriverV2(options *DriverOptions) *DriverV2 { csi.ControllerServiceCapability_RPC_LIST_VOLUMES, csi.ControllerServiceCapability_RPC_LIST_VOLUMES_PUBLISHED_NODES, csi.ControllerServiceCapability_RPC_SINGLE_NODE_MULTI_WRITER, + csi.ControllerServiceCapability_RPC_MODIFY_VOLUME, }) driver.AddVolumeCapabilityAccessModes( []csi.VolumeCapability_AccessMode_Mode{ diff --git a/pkg/azuredisk/controllerserver.go b/pkg/azuredisk/controllerserver.go index b6ed97e31d..199544471b 100644 --- a/pkg/azuredisk/controllerserver.go +++ b/pkg/azuredisk/controllerserver.go @@ -382,8 +382,71 @@ func (d *Driver) ControllerGetVolume(context.Context, *csi.ControllerGetVolumeRe } // ControllerModifyVolume modify volume -func (d *Driver) ControllerModifyVolume(context.Context, *csi.ControllerModifyVolumeRequest) (*csi.ControllerModifyVolumeResponse, error) { - return nil, status.Error(codes.Unimplemented, "") +func (d *Driver) ControllerModifyVolume(ctx context.Context, req *csi.ControllerModifyVolumeRequest) (*csi.ControllerModifyVolumeResponse, error) { + volumeID := req.GetVolumeId() + if len(volumeID) == 0 { + return nil, status.Error(codes.InvalidArgument, "Volume ID missing in the request") + } + + if err := d.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_MODIFY_VOLUME); err != nil { + return nil, status.Errorf(codes.Internal, "invalid modify volume req: %v", req) + } + diskURI := volumeID + + diskName, err := azureutils.GetDiskName(diskURI) + if err != nil { + return nil, status.Errorf(codes.Internal, err.Error()) + } + + if _, err := d.checkDiskExists(ctx, diskURI); err != nil { + return nil, status.Error(codes.NotFound, fmt.Sprintf("Volume not found, failed with error: %v", err)) + } + + diskParams, err := azureutils.ParseDiskParameters(req.GetMutableParameters()) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "Failed parsing disk parameters: %v", err) + } + + // normalize values + skuName, err := azureutils.NormalizeStorageAccountType(diskParams.AccountType, d.cloud.Config.Cloud, d.cloud.Config.DisableAzureStackCloud) + if err != nil { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + if diskParams.AccountType == "" { + skuName = "" + } + + klog.V(2).Infof("begin to modify azure disk(%s) account type(%s) rg(%s) location(%s)", + diskParams.DiskName, skuName, diskParams.ResourceGroup, diskParams.Location) + + volumeOptions := &ManagedDiskOptions{ + DiskIOPSReadWrite: diskParams.DiskIOPSReadWrite, + DiskMBpsReadWrite: diskParams.DiskMBPSReadWrite, + DiskName: diskName, + ResourceGroup: diskParams.ResourceGroup, + SubscriptionID: diskParams.SubscriptionID, + StorageAccountType: skuName, + SourceResourceID: diskURI, + SourceType: consts.SourceVolume, + } + + mc := metrics.NewMetricContext(consts.AzureDiskCSIDriverName, "controller_modify_volume", d.cloud.ResourceGroup, d.cloud.SubscriptionID, d.Name) + isOperationSucceeded := false + defer func() { + mc.ObserveOperationWithResult(isOperationSucceeded, consts.VolumeID, diskURI) + }() + + if err = d.diskController.ModifyDisk(ctx, volumeOptions); err != nil { + if strings.Contains(err.Error(), consts.NotFound) { + return nil, status.Error(codes.NotFound, err.Error()) + } + return nil, status.Errorf(codes.Internal, err.Error()) + } + + isOperationSucceeded = true + klog.V(2).Infof("modify azure disk(%s) account type(%s) rg(%s) location(%s) successfully", diskParams.DiskName, skuName, diskParams.ResourceGroup, diskParams.Location) + + return &csi.ControllerModifyVolumeResponse{}, err } // ControllerPublishVolume attach an azure disk to a required node diff --git a/pkg/azuredisk/controllerserver_test.go b/pkg/azuredisk/controllerserver_test.go index 147c2a5582..4902303fde 100644 --- a/pkg/azuredisk/controllerserver_test.go +++ b/pkg/azuredisk/controllerserver_test.go @@ -227,8 +227,8 @@ func TestCreateVolume(t *testing.T) { mp[consts.LocationField] = "ut" mp[consts.StorageAccountTypeField] = "ut" mp[consts.ResourceGroupField] = "ut" - mp[consts.DiskIOPSReadWriteField] = "ut" - mp[consts.DiskMBPSReadWriteField] = "ut" + mp[consts.DiskIOPSReadWriteField] = "1" + mp[consts.DiskMBPSReadWriteField] = "1" mp[consts.DiskNameField] = "ut" mp[consts.DesIDField] = "ut" mp[consts.WriteAcceleratorEnabled] = "ut" @@ -622,6 +622,121 @@ func TestControllerGetVolume(t *testing.T) { } } +func TestControllerModifyVolume(t *testing.T) { + cntl := gomock.NewController(t) + defer cntl.Finish() + d, err := NewFakeDriver(cntl) + if err != nil { + t.Fatalf("Error getting driver: %v", err) + } + storageAccountTypeUltraSSDLRS := armcompute.DiskStorageAccountTypesUltraSSDLRS + + tests := []struct { + desc string + req *csi.ControllerModifyVolumeRequest + oldSKU *armcompute.DiskStorageAccountTypes + expectedResp *csi.ControllerModifyVolumeResponse + expectedErrCode codes.Code + expectedErrmsg string + }{ + { + desc: "success standard", + req: &csi.ControllerModifyVolumeRequest{ + VolumeId: testVolumeID, + MutableParameters: map[string]string{ + consts.DiskIOPSReadWriteField: "100", + consts.DiskMBPSReadWriteField: "100", + }, + }, + oldSKU: &storageAccountTypeUltraSSDLRS, + expectedResp: &csi.ControllerModifyVolumeResponse{}, + }, + { + desc: "fail with no volume id", + req: &csi.ControllerModifyVolumeRequest{ + VolumeId: "", + }, + expectedResp: nil, + expectedErrCode: codes.InvalidArgument, + }, + { + desc: "fail with the invalid diskURI", + req: &csi.ControllerModifyVolumeRequest{ + VolumeId: "123", + }, + expectedResp: nil, + expectedErrCode: codes.Internal, + }, + { + desc: "fail with wrong disk name", + req: &csi.ControllerModifyVolumeRequest{ + VolumeId: "/subscriptions/123", + }, + expectedResp: nil, + expectedErrCode: codes.Internal, + }, + { + desc: "fail with wrong sku name", + req: &csi.ControllerModifyVolumeRequest{ + VolumeId: testVolumeID, + MutableParameters: map[string]string{ + consts.SkuNameField: "ut", + }, + }, + expectedResp: nil, + expectedErrCode: codes.InvalidArgument, + }, + { + desc: "fail with error parse parameter", + req: &csi.ControllerModifyVolumeRequest{ + VolumeId: testVolumeID, + MutableParameters: map[string]string{ + consts.DiskIOPSReadWriteField: "ut", + }, + }, + expectedResp: nil, + expectedErrCode: codes.InvalidArgument, + }, + { + desc: "fail with unsupported sku", + req: &csi.ControllerModifyVolumeRequest{ + VolumeId: testVolumeID, + MutableParameters: map[string]string{ + consts.SkuNameField: "Premium_LRS", + consts.DiskIOPSReadWriteField: "100", + }, + }, + expectedResp: nil, + expectedErrCode: codes.Internal, + }, + } + + for _, test := range tests { + ctx, cancel := context.WithCancel(context.TODO()) + defer cancel() + id := test.req.VolumeId + disk := &armcompute.Disk{ + ID: &id, + SKU: &armcompute.DiskSKU{ + Name: test.oldSKU, + }, + Properties: &armcompute.DiskProperties{}, + } + diskClient := mock_diskclient.NewMockInterface(cntl) + d.getClientFactory().(*mock_azclient.MockClientFactory).EXPECT().GetDiskClientForSub(gomock.Any()).Return(diskClient, nil).AnyTimes() + diskClient.EXPECT().Get(gomock.Eq(ctx), gomock.Any(), gomock.Any()).Return(disk, nil).AnyTimes() + diskClient.EXPECT().Patch(gomock.Eq(ctx), gomock.Any(), gomock.Any(), gomock.Any()).Return(disk, nil).AnyTimes() + + result, err := d.ControllerModifyVolume(ctx, test.req) + if err != nil { + checkTestError(t, test.expectedErrCode, err) + } + if !reflect.DeepEqual(result, test.expectedResp) { + t.Errorf("input request: %v, ControllerModifyVolume result: %v, expected: %v", test.req, result, test.expectedResp) + } + } +} + func TestGetSnapshotInfo(t *testing.T) { cntl := gomock.NewController(t) defer cntl.Finish() diff --git a/pkg/azuredisk/controllerserver_v2.go b/pkg/azuredisk/controllerserver_v2.go index dd8d12b90f..fc81942fd1 100644 --- a/pkg/azuredisk/controllerserver_v2.go +++ b/pkg/azuredisk/controllerserver_v2.go @@ -299,8 +299,71 @@ func (d *DriverV2) ControllerGetVolume(context.Context, *csi.ControllerGetVolume } // ControllerModifyVolume modify volume -func (d *DriverV2) ControllerModifyVolume(context.Context, *csi.ControllerModifyVolumeRequest) (*csi.ControllerModifyVolumeResponse, error) { - return nil, status.Error(codes.Unimplemented, "") +func (d *DriverV2) ControllerModifyVolume(ctx context.Context, req *csi.ControllerModifyVolumeRequest) (*csi.ControllerModifyVolumeResponse, error) { + volumeID := req.GetVolumeId() + if len(volumeID) == 0 { + return nil, status.Error(codes.InvalidArgument, "Volume ID missing in the request") + } + + if err := d.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_MODIFY_VOLUME); err != nil { + return nil, status.Errorf(codes.Internal, "invalid modify volume req: %v", req) + } + diskURI := volumeID + + diskName, err := azureutils.GetDiskName(diskURI) + if err != nil { + return nil, status.Errorf(codes.Internal, err.Error()) + } + + if _, err := d.checkDiskExists(ctx, diskURI); err != nil { + return nil, status.Error(codes.NotFound, fmt.Sprintf("Volume not found, failed with error: %v", err)) + } + + diskParams, err := azureutils.ParseDiskParameters(req.GetMutableParameters()) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "Failed parsing disk parameters: %v", err) + } + + // normalize values + skuName, err := azureutils.NormalizeStorageAccountType(diskParams.AccountType, d.cloud.Config.Cloud, d.cloud.Config.DisableAzureStackCloud) + if err != nil { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + if diskParams.AccountType == "" { + skuName = "" + } + + klog.V(2).Infof("begin to modify azure disk(%s) account type(%s) rg(%s) location(%s)", + diskParams.DiskName, skuName, diskParams.ResourceGroup, diskParams.Location) + + volumeOptions := &ManagedDiskOptions{ + DiskIOPSReadWrite: diskParams.DiskIOPSReadWrite, + DiskMBpsReadWrite: diskParams.DiskMBPSReadWrite, + DiskName: diskName, + ResourceGroup: diskParams.ResourceGroup, + SubscriptionID: diskParams.SubscriptionID, + StorageAccountType: skuName, + SourceResourceID: diskURI, + SourceType: consts.SourceVolume, + } + + mc := metrics.NewMetricContext(consts.AzureDiskCSIDriverName, "controller_modify_volume", d.cloud.ResourceGroup, d.cloud.SubscriptionID, d.Name) + isOperationSucceeded := false + defer func() { + mc.ObserveOperationWithResult(isOperationSucceeded, consts.VolumeID, diskURI) + }() + + if err = d.diskController.ModifyDisk(ctx, volumeOptions); err != nil { + if strings.Contains(err.Error(), consts.NotFound) { + return nil, status.Error(codes.NotFound, err.Error()) + } + return nil, status.Errorf(codes.Internal, err.Error()) + } + + isOperationSucceeded = true + klog.V(2).Infof("modify azure disk(%s) account type(%s) rg(%s) location(%s) successfully", diskParams.DiskName, skuName, diskParams.ResourceGroup, diskParams.Location) + + return &csi.ControllerModifyVolumeResponse{}, err } // ControllerPublishVolume attach an azure disk to a required node diff --git a/pkg/azuredisk/fake_azuredisk.go b/pkg/azuredisk/fake_azuredisk.go index fa88806f9f..a88cde2b3e 100644 --- a/pkg/azuredisk/fake_azuredisk.go +++ b/pkg/azuredisk/fake_azuredisk.go @@ -152,6 +152,7 @@ func newFakeDriverV1(ctrl *gomock.Controller) (*fakeDriverV1, error) { csi.ControllerServiceCapability_RPC_EXPAND_VOLUME, csi.ControllerServiceCapability_RPC_LIST_VOLUMES, csi.ControllerServiceCapability_RPC_LIST_VOLUMES_PUBLISHED_NODES, + csi.ControllerServiceCapability_RPC_MODIFY_VOLUME, }) driver.AddVolumeCapabilityAccessModes([]csi.VolumeCapability_AccessMode_Mode{csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER}) driver.AddNodeServiceCapabilities([]csi.NodeServiceCapability_RPC_Type{ diff --git a/pkg/azuredisk/fake_azuredisk_v2.go b/pkg/azuredisk/fake_azuredisk_v2.go index 4facb5a421..edd32ed5da 100644 --- a/pkg/azuredisk/fake_azuredisk_v2.go +++ b/pkg/azuredisk/fake_azuredisk_v2.go @@ -93,6 +93,7 @@ func newFakeDriverV2(ctrl *gomock.Controller) (*fakeDriverV2, error) { csi.ControllerServiceCapability_RPC_EXPAND_VOLUME, csi.ControllerServiceCapability_RPC_LIST_VOLUMES, csi.ControllerServiceCapability_RPC_LIST_VOLUMES_PUBLISHED_NODES, + csi.ControllerServiceCapability_RPC_MODIFY_VOLUME, }) driver.AddVolumeCapabilityAccessModes([]csi.VolumeCapability_AccessMode_Mode{csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER}) driver.AddNodeServiceCapabilities([]csi.NodeServiceCapability_RPC_Type{ diff --git a/pkg/azureutils/azure_disk_utils.go b/pkg/azureutils/azure_disk_utils.go index 90f510363b..e1be1be914 100644 --- a/pkg/azureutils/azure_disk_utils.go +++ b/pkg/azureutils/azure_disk_utils.go @@ -597,8 +597,14 @@ func ParseDiskParameters(parameters map[string]string) (ManagedDiskParameters, e case consts.ResourceGroupField: diskParams.ResourceGroup = v case consts.DiskIOPSReadWriteField: + if _, err = strconv.Atoi(v); err != nil { + return diskParams, fmt.Errorf("parse %s:%s failed with error: %v", consts.DiskIOPSReadWriteField, v, err) + } diskParams.DiskIOPSReadWrite = v case consts.DiskMBPSReadWriteField: + if _, err = strconv.Atoi(v); err != nil { + return diskParams, fmt.Errorf("parse %s:%s failed with error: %v", consts.DiskMBPSReadWriteField, v, err) + } diskParams.DiskMBPSReadWrite = v case consts.LogicalSectorSizeField: diskParams.LogicalSectorSize, err = strconv.Atoi(v) diff --git a/pkg/azureutils/azure_disk_utils_test.go b/pkg/azureutils/azure_disk_utils_test.go index 1cc945c362..2dc4aac71e 100644 --- a/pkg/azureutils/azure_disk_utils_test.go +++ b/pkg/azureutils/azure_disk_utils_test.go @@ -1451,6 +1451,26 @@ func TestParseDiskParameters(t *testing.T) { }, expectedError: fmt.Errorf("cachingMode ReadOnly is not supported for PremiumV2_LRS"), }, + { + name: "invalid DiskIOPSReadWriteField value in parameters", + inputParams: map[string]string{consts.DiskIOPSReadWriteField: "diskIOPSReadWrite"}, + expectedOutput: ManagedDiskParameters{ + Tags: make(map[string]string), + VolumeContext: map[string]string{consts.DiskIOPSReadWriteField: "diskIOPSReadWrite"}, + DeviceSettings: make(map[string]string), + }, + expectedError: fmt.Errorf("parse diskiopsreadwrite:diskIOPSReadWrite failed with error: strconv.Atoi: parsing \"diskIOPSReadWrite\": invalid syntax"), + }, + { + name: "invalid DiskMBPSReadWriteField value in parameters", + inputParams: map[string]string{consts.DiskMBPSReadWriteField: "diskMBPSReadWrite"}, + expectedOutput: ManagedDiskParameters{ + Tags: make(map[string]string), + VolumeContext: map[string]string{consts.DiskMBPSReadWriteField: "diskMBPSReadWrite"}, + DeviceSettings: make(map[string]string), + }, + expectedError: fmt.Errorf("parse diskmbpsreadwrite:diskMBPSReadWrite failed with error: strconv.Atoi: parsing \"diskMBPSReadWrite\": invalid syntax"), + }, { name: "valid parameters input", inputParams: map[string]string{ @@ -1458,8 +1478,8 @@ func TestParseDiskParameters(t *testing.T) { consts.LocationField: "location", consts.CachingModeField: "cachingMode", consts.ResourceGroupField: "resourceGroup", - consts.DiskIOPSReadWriteField: "diskIOPSReadWrite", - consts.DiskMBPSReadWriteField: "diskMBPSReadWrite", + consts.DiskIOPSReadWriteField: "4000", + consts.DiskMBPSReadWriteField: "1000", consts.LogicalSectorSizeField: "1", consts.DiskNameField: "diskName", consts.DesIDField: "diskEncyptionSetID", @@ -1484,8 +1504,8 @@ func TestParseDiskParameters(t *testing.T) { Location: "location", CachingMode: v1.AzureDataDiskCachingMode("cachingMode"), ResourceGroup: "resourceGroup", - DiskIOPSReadWrite: "diskIOPSReadWrite", - DiskMBPSReadWrite: "diskMBPSReadWrite", + DiskIOPSReadWrite: "4000", + DiskMBPSReadWrite: "1000", DiskName: "diskName", DiskEncryptionSetID: "diskEncyptionSetID", Tags: map[string]string{ @@ -1507,8 +1527,8 @@ func TestParseDiskParameters(t *testing.T) { consts.LocationField: "location", consts.CachingModeField: "cachingMode", consts.ResourceGroupField: "resourceGroup", - consts.DiskIOPSReadWriteField: "diskIOPSReadWrite", - consts.DiskMBPSReadWriteField: "diskMBPSReadWrite", + consts.DiskIOPSReadWriteField: "4000", + consts.DiskMBPSReadWriteField: "1000", consts.LogicalSectorSizeField: "1", consts.DiskNameField: "diskName", consts.DesIDField: "diskEncyptionSetID", diff --git a/test/sanity/run-test.sh b/test/sanity/run-test.sh index 37de8abe25..8b3a2ce58b 100755 --- a/test/sanity/run-test.sh +++ b/test/sanity/run-test.sh @@ -47,4 +47,4 @@ sleep 1 echo 'Begin to run sanity test...' readonly CSI_SANITY_BIN='csi-sanity' -"$CSI_SANITY_BIN" --ginkgo.v --csi.endpoint="$endpoint" --ginkgo.skip='should work|should fail when volume does not exist on the specified path|should be idempotent|pagination should detect volumes added between pages and accept tokens when the last volume from a page is deleted|should remove target path' +"$CSI_SANITY_BIN" --ginkgo.v --csi.endpoint="$endpoint" --ginkgo.skip='should work|should fail when volume does not exist on the specified path|should be idempotent|pagination should detect volumes added between pages and accept tokens when the last volume from a page is deleted|should remove target path|should return appropriate capabilities'