From 9603e5ea794aba8f88935741de192eefc3109601 Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Wed, 20 Dec 2023 16:04:09 -0500 Subject: [PATCH 1/3] [TT-765] Moves Client Compatability Tests to Nightly Run (#11610) * Moves Client Compatability Tests to Nightly Run * midnight 30 * adds Slack notification * Start independent action * parameterize slack notify action * Actually checkout repo * Fix action * Better Debutg * Link and docs * Fix comma * Finish Cleanup * Freedom --- .../notify-slack-jobs-result/README.md | 37 +++ .../notify-slack-jobs-result/action.yml | 110 +++++++ .../notify-slack-jobs-result/image.png | Bin 0 -> 47298 bytes .../workflows/client-compatibility-tests.yml | 269 ++++++++++++++++++ .github/workflows/integration-tests.yml | 67 +---- .github/workflows/live-testnet-tests.yml | 93 +----- 6 files changed, 438 insertions(+), 138 deletions(-) create mode 100644 .github/actions/notify-slack-jobs-result/README.md create mode 100644 .github/actions/notify-slack-jobs-result/action.yml create mode 100644 .github/actions/notify-slack-jobs-result/image.png create mode 100644 .github/workflows/client-compatibility-tests.yml diff --git a/.github/actions/notify-slack-jobs-result/README.md b/.github/actions/notify-slack-jobs-result/README.md new file mode 100644 index 00000000000..298930c0d95 --- /dev/null +++ b/.github/actions/notify-slack-jobs-result/README.md @@ -0,0 +1,37 @@ +# Notify Slack Jobs Result + +Sends a Slack message to a specified channel detailing the results of one to many GHA job results using a regex. The job results will be grouped by the `github_job_name_regex` and displayed underneath the `message_title`, with the regex matching group displayed as an individual result. This is primarily designed for when you have test groups running in a matrix, and would like condensed reporting on their status by group. It's often accompanied by posting a Slack message before to start a thread, then attaching all the results to that thread like we do in the reporting section of the [live-testnet-test.yml workflow](../../workflows/live-testnet-tests.yml). Check out the example below, where we post an initial summary message, then use this action to thread together specific results: + +```yaml +message_title: Optimism Goerli +github_job_name_regex: ^Optimism Goerli (?.*?) Tests$ # Note that the regex MUST have a capturing group named "cap" +``` + +![example](image.png) + +## Inputs + +```yaml +inputs: + github_token: + description: "The GitHub token to use for authentication (usually ${{ github.token }})" + required: true + github_repository: + description: "The GitHub owner/repository to use for authentication (usually ${{ github.repository }}))" + required: true + workflow_run_id: + description: "The workflow run ID to get the results from (usually ${{ github.run_id }})" + required: true + github_job_name_regex: + description: "The regex to use to match 1..many job name(s) to collect results from. Should include a capture group named 'cap' for the part of the job name you want to display in the Slack message (e.g. ^Client Compatability Test (?.*?)$)" + required: true + message_title: + description: "The title of the Slack message" + required: true + slack_channel_id: + description: "The Slack channel ID to post the message to" + required: true + slack_thread_ts: + description: "The Slack thread timestamp to post the message to, handy for keeping multiple related results in a single thread" + required: false +``` diff --git a/.github/actions/notify-slack-jobs-result/action.yml b/.github/actions/notify-slack-jobs-result/action.yml new file mode 100644 index 00000000000..63840cfa393 --- /dev/null +++ b/.github/actions/notify-slack-jobs-result/action.yml @@ -0,0 +1,110 @@ +name: Notify Slack Jobs Result +description: Will send a notification in Slack for the result of a GitHub action run, typically for test results +inputs: + github_token: + description: "The GitHub token to use for authentication (usually github.token)" + required: true + github_repository: + description: "The GitHub owner/repository to use for authentication (usually github.repository))" + required: true + workflow_run_id: + description: "The workflow run ID to get the results from (usually github.run_id)" + required: true + github_job_name_regex: + description: "The regex to use to match 1..many job name(s) to collect results from. Should include a capture group named 'cap' for the part of the job name you want to display in the Slack message (e.g. ^Client Compatability Test (?.*?)$)" + required: true + message_title: + description: "The title of the Slack message" + required: true + slack_channel_id: + description: "The Slack channel ID to post the message to" + required: true + slack_bot_token: + description: "The Slack bot token to use for authentication which needs permission and an installed app in the channel" + required: true + slack_thread_ts: + description: "The Slack thread timestamp to post the message to, handy for keeping multiple related results in a single thread" + required: false + +runs: + using: composite + steps: + - name: Get Results + shell: bash + id: test-results + run: | + # I feel like there's some clever, fully jq way to do this, but I ain't got the motivation to figure it out + echo "Querying test results at https://api.github.com/repos/${{inputs.github_repository}}/actions/runs/${{ inputs.workflow_run_id }}/jobs" + + PARSED_RESULTS=$(curl \ + -H "Authorization: Bearer ${{ inputs.github_token }}" \ + 'https://api.github.com/repos/${{inputs.github_repository}}/actions/runs/${{ inputs.workflow_run_id }}/jobs' \ + | jq -r --arg pattern "${{ inputs.github_job_name_regex }}" '.jobs[] + | select(.name | test($pattern)) as $job + | $job.steps[] + | select(.name == "Run Tests") + | { conclusion: (if .conclusion == "success" then ":white_check_mark:" else ":x:" end), cap: ("*" + ($job.name | capture($pattern).cap) + "*"), html_url: $job.html_url }') + + echo "Parsed Results:" + echo $PARSED_RESULTS + + ALL_SUCCESS=true + echo "Checking for failures" + echo "$PARSED_RESULTS" | jq -s | jq -r '.[] | select(.conclusion != ":white_check_mark:")' + for row in $(echo "$PARSED_RESULTS" | jq -s | jq -r '.[] | select(.conclusion != ":white_check_mark:")'); do + ALL_SUCCESS=false + break + done + echo "Success: $ALL_SUCCESS" + + echo all_success=$ALL_SUCCESS >> $GITHUB_OUTPUT + + FORMATTED_RESULTS=$(echo $PARSED_RESULTS | jq -s '[.[] + | { + conclusion: .conclusion, + cap: .cap, + html_url: .html_url + } + ] + | map("{\"type\": \"section\", \"text\": {\"type\": \"mrkdwn\", \"text\": \"<\(.html_url)|\(.cap)>: \(.conclusion)\"}}") + | join(",")') + + echo "Formatted Results:" + echo $FORMATTED_RESULTS + + # Cleans out backslashes and quotes from jq + CLEAN_RESULTS=$(echo "$FORMATTED_RESULTS" | sed 's/\\\"/"/g' | sed 's/^"//;s/"$//') + + echo "Clean Results" + echo $CLEAN_RESULTS + + echo results=$CLEAN_RESULTS >> $GITHUB_OUTPUT + - name: Post Results + uses: slackapi/slack-github-action@e28cf165c92ffef168d23c5c9000cffc8a25e117 # v1.24.0 + env: + SLACK_BOT_TOKEN: ${{ inputs.slack_bot_token }} + with: + channel-id: ${{ inputs.slack_channel_id }} + payload: | + { + "thread_ts": "${{ inputs.slack_thread_ts }}", + "attachments": [ + { + "color": "${{ steps.test-results.outputs.all_success == 'true' && '#2E7D32' || '#C62828' }}", + "blocks": [ + { + "type": "header", + "text": { + "type": "plain_text", + "text": "${{ inputs.message_title }} ${{ steps.test-results.outputs.all_success == 'true' && ':white_check_mark:' || ':x:'}}", + "emoji": true + } + }, + { + "type": "divider" + }, + ${{ steps.test-results.outputs.results }} + ] + } + ] + } diff --git a/.github/actions/notify-slack-jobs-result/image.png b/.github/actions/notify-slack-jobs-result/image.png new file mode 100644 index 0000000000000000000000000000000000000000..3bd398101fbdc8e4a0eb4c5addd7332f130190ab GIT binary patch literal 47298 zcmd3ORajh2*CiI*HCS*9?i$=05AN=6!QBboxQAdNKyY`baY=%EwTV_yl0iczLWY8ZLIcQ3szO1*&_F>!3n0Qn?ttD<@FhhWJ*uq)YmfURDF}i%elXDudyXtnbijvsOq?fiz1H(l0gL!*)vuo zfm(wgII+iJq&d?&q<%sB9P&-xE0c*?VssAfAt9|IeuR_ywhLX{Kh*!*$VV4H8!-@B%5L}KP^fS28) zs#t951~6YOT5OQcsh|wn+QVIu&xQNVG<@4h_T0<9^Gpr zi!DEkF`dM0U0`XJ3ioT3<)w>18!OYIF}w*jye~TuL5+&hN@6+Ez&>OFq^Y?#IiB1Bv#8c^~1i8RJMM zl4IET=@}Q6A`|Ud7wi>V7u@|?KU(W$xPO6aQn!5>KHQsT6EAtCI-C05dupvg)s7nA zZC(Cp3dz!$>L)vZ6#xvqBcV(Nvk_7hk^|{LYUZ)#(*knn+(T)UZswOQ(?7kTsZ{fM zW7=c(JjDiItjsgk>n8`~@_8}mC+8>aMpEY-4F&uiwPM<&HS^nkyY0JETM^kZyALn^ z%ypf7?KRq+t2Wx@)kS?l2!IVdT+(dfKI9tHMD}lW((9MG{GenmT+FOtZ&FwIRfk*i z0fuZ8%D=^{4bci-?+JVh5&_zQZh>U9fm^F!;f)EmeAj%JwV+{xbrx46S0fiPw|p7f zpWdfjb|+PZHOM~bgjI6f3Znvu5K@LawE2V`Uzd0U$bVR@3LbPj zw-SyG~C5$W5>l{kB7Ss9h==`Vae(fmwW~@zaeM8p4=ROd4%M`ha<2^p=cJ~+h z!`8hvlIarkhIH#~sA^vhzFsH5%QL}}cd>OJHd5eg2P2w3L5eeh9>^^wKOz66zt#oFi=pDHc)W? z(?+M>=<9&&k8glR1rFA2b{4bsN9Y z!+Z{+{DSzmIm4h@zmL3}JHmsO62(%1!~N5+A<{EUBZ&R$s|s8^B3w!e3D&PK;s4X) zPeViVfcl?Dbz=nw0fAIsiRv-`^K?uC6u9v$-ei=?Qz}xyocEvXeqmM3G4?VtzqnkW zd%B*dhl{|?pWpSd4D^ErmC#?X&m!#zt*L@atFyC{85uPzQ(U@~Q&_uR~ns;Lc_k-iH!t@E#SS`!A14GUfLOkuzO^JdP8h}zR7a3T-$5*%mWp#!it zCSuJcWO;_)i}E+x?OpvI3=5>RG&YK1T{Y!vd<+GiQJ~In>#_j_55pZ&Micltxk+ z86R&?)2|PIM~0Cdz+#8K*Z4l1^{aef=Tak9^@m?epR{bw5;^<5@8jA;f#cr|=d;A3 zPf77RTNRvX(6c&=BDvJUd_yb*<4(n^Q=Md=_*s7n8~Ogtz&=%b6y_u>+|sIhj+GE5 zp0sPUg*uxHhtHVy{@SapmMnOs?W9$Lh}VXMy*IwgbC}d~8IN5^7fMh^j=I*wn`wWh zxGu+A+NtbgVT8G*CFp3OZaapxGex6ZQ$}96t{xm^a%q2&~Q>A_;Run5sC4rJ5FN1)&d+iXUJ{g1D+l-KOyPz zDAy=Yb6vTflPms=r>HubUX|E(+SycX9)l}g@!=h7y`6g9y~WwORgvS0zy<%=%hBv) z?(9HqrAmQ#4YsnHZ9!};-#B(CR6cCRf(F!WGid|)zY%ZbN~eG(&gJ!c(y+DZfW z#asgBm4W_-dbJh$(UCjUpdCWuN7I+)NjFq8l0hjZ{YjB303(90@DV$Kh|akzFZg+{ z4!}p-i<^y(}t_Mm~+n>tn3dh$;2c#^WjAmGM!keRJ7uGKtu!+0gcP1%oi^ zFTlIS20;Kz+f!}S9LY(LYOx%$;FwtGNfO8QNSJwgPL6%GoPq)>ztOx{01439!IW~` zg_1_@?p#q`ULxR@NSwoRtm@(kg%xC1)$vigRtlZ?>4*~|b(3$ZLN8?H(penh<m zLW5t|l`O0sRaI5R&HTz0<| ze(Z~oE3Z%-wOa!VJgCqY(W>|_L@%M^gOeB~?pXV(Ms5b8>FByWH=X0?mcn$&DUq5v z(C64bEK$}JOp%4FN6t;ry&*;%mo|!xIx)K#tDF3a`ts!1xi8JXU({IZb4}#E-V>=H z(3t#%|NWx!)>gd4E{viI+{W;8h_JuHo>hJzgONYCxLgVoMXBdLW#H4fOj!BnQ!dcr z2hSB6j1co3^IaQ)&$=OlaS|@`?KXAG{*&Yk(Z0Etb0o>j(GJQf&3si){C17)_d82c z1hn$hRn>qICL3w;>&)I566{`)!tD@|*7_@oH>twY5uF$123Vf`*L+?hN|6(GM z@e}(aom=QVu$g;6(4f?Vd7;j;YP$Y-rH%FNI~%L+pqGn<-TWT?jCTy5^R)hoVP(J|^&s{D6l(c!)UHFIT_{e1Pb zJAS+l6;c4!ZjM`Hz2h%A>|~(%%hUbEb(B-l3h21V^E8k4I9XX3@Lfwe*=1{Jv>yp4 z^xmakejG6zJ(`R0Xt733Iw+@^hOj@vo6BJa*PD8{ws=%=sP5AxK2F>V;ckw98kg<- zaO@KSQwA$p;6>TdYRhP@f^)uZ^viQq%3T=Tz4g3It^LAkBV+ zpvqaZYwC#Ws`oAtr>v!P+E^~Ha_8%dAYd(@!Yc1m$<@KHzWz_nh=x(Q*W^eBt?o&V z25lL$4bwfsck8ex^c(@3aG7k5A^ho@RA{$8jXx{ZWIT=+nK>;dQj!X#gBKpaVT}&O zvo@vThwAbkCyNXsuR^M{g_k8dd%ZC%Tyt9Av08c6LrWXQ%amCPNqAue~RWh zgzuY;q}XtH9BJ41H>#MqY{l^o=kU8(9JhXQozSr&teiYerW+?OOTwCdPr6HqH;3eq+^G{q$j?vrMS}7ErcJpX1`=8y`@_Sla!~K9{ipucuQ^F-Fnm?BaeysfJPpVp7`w$)td(N>VrF^ z&j1n|S9xw!E>=>(0wx584xd|Kuogz5zPk?!K2r+*9+$OY%jpY(1-o?JoS|=}yHI7Q z{Zgkvqr(!*2VeK}IAUHRZr`$?eL5E5RqHp{qA<(i;*woSi*LgXxxbLj@M{dl6Vu_) zy|DP+9WI>k*K8)3sbq_?kZtqFo%#DpPbJ*P0EnxRW!? zG#u`iY_;yLJ|f@MdTnijZcy(&+%VHlLau|{yOFeRR zFUDq-iJ1`uRj})572}iD1l37ef|_^D9hT(Nti}CjJbE)mFczKihwE4X?+nIihs`W4 zhyA>B;xB4z`mEtAn>YSJGqcZ&2noFPvy)-?;ge;V8~6;#Y`aTMA`8pA!$#8bpD;CL z?keg(oWpq^WLm|Q#3~K-m!s#A3P0Y52T0Huqp!5hy&kppC4UJnke->&p@tf=Pf3@Y z#ByVq6}gZN*Gb4KkR@tuzT*=jX7Ez>h~%T^jH4D(^Mn6}D8?zmk=(Mw4;^p-jy;~Gx<>~np9{L0b+#`b<#=;0un&7S1j zQjn0HM_NMx_Mq>_FF7UiIOxj1LTil#k_D>f`CZr)@K}I02UMZK+m*bAoc%PFw%j_A z)9rc>SNxNhKxVzh;i&?N6|pxrL$QRR@t^XqPJXF5HlGOSpur5p&oh(o&mM6&EV_H| zZ)=U=YCSlrmupJ-+#FGycHMS@hk`953M5w1X*mOyy=>;cKyP}cXL4GBMr9)}&iXKd z;doF6iJdh9ljpqF@83HY9#E~4=JqKU;qIy!#;|K* zL$*^VD6A>GB%-K@o69c6D3I9hBwvyimm5caeYbgw2&81)Bs;SsPBrt6h=-TK2EG>v z|5aLXI3zsAV$v~iR~gMv%R1}3Ru#*miLMSb!p7I(SdrHhkrIWZohi(10+iWQfGdfs zs1f-P*)VB3^(C;ApiijrT|b-5CQMj#(ZiG5BxhKWP!uQkheDZz;7J{V#PkjhiZA>X z`o+NL+ZP0~;q9~1#dtyq>e6JBue!+NNmSGXE3KcB7s+MdxQEgiPXq4?D2S-F#747x`@M%1)1j8fecGWF?EbBQvmgqWWEa#)MyB%fZy$;H+7e(IJq1g5^ z@zbAW$shaHJPw&ToYy~zci|%Yv-06Yap1uP6&1N8(~5VcaNGSxw5qSSU0JlY{>DuH zksgIkgO{P@_~x~b9{U>B^c*}p+1*Aj#$ILU#_&)Ep@?I5u!C~-M-q~0sp{OzAPpi_l zkp<|u2c$r})e(k0@=1X@-h+nd*!yeVf=fp(8(5CMC)Vb5KYN_(~oHG5>882;g?Vic7F#D*W=% z1%;d1Uv#GH?t*b6w#*ANz3sGlMH19)Uz;x-mqNPa*{%PU^>)>t;K?Llh*L=M$HAHr zEwq&_i;~cWLh9 zivCpJVt^P9;qpjx=vwH=_*oQGiLdDud3E@-=j>o5|0bjw_C>^Fd4=Q z2VWGOH?vNdmO?)#z3S)x-A~XpT1ew!q2&iwq z0oB5I({zs|86EsC4=r2r6g~V1s&ML!Z~eB(!m3Dt>k&ejNf?pRKlFG=soT0%cmlAT z4BD@PID%}i1{U>n%EIUF{qi<>g^Q^2y)#8zLeozW- zhCyTB&lZy{cnSSQFR+WJ#an{uXflTvp|DUE7N^hWU0}SlR5{z%THK7 zgbg@Qv!NyNb$B--EDV^G$1s{`#N3V`C%63Vpp(-1iVCxHHx#`O z#^8&DU7lz|UNRJG0`)>@qb&?B(yfS)o+AmLexq5nYKfm4U=77n8$))qGk5b`jQMzp znefp=Hlztx`s(eqmOYJNqZfEWPdH8NRyoy(wlG)>e&LK0G$C$fg20D;IRHuf=_YH1 z!Oj=rl$ia{4xT=Xs_Ntk-1_oTe2Nf`c2!>0j;uBUb^pfJMs5^;eEj$Qc}QY7&P@%( z@r%m|#^QF^`02yPcd0brn`eHSZ?5`Mr^&I9-yWG0A>5(}6e>Qp zjE#ignIrp1GVV0LPPU*b4@R{5rQ-P5wS-FHeRxbkZu7i7J1Ok;HOziXXyY&>>384N zx@=zr+BgfT8Dk2o(~VY@;WivWYjH`4T{CL)1CjUEc$0$uLuz#Z6(A;UT+yJaXN)z= z>5i9FGJ`YS`sKbftSGiWXa_edG9a|i3A)F@#Xr}%lgMH;Jr&xE_uKf{wkeFr%QsMw z!XRp$+F}oQrlMsm>?XANc(p;^FyNXy6T?BJX|_%7@#gqB3AT$mK3(6nrvAeQoWyWl z?K5~fe-(EhJ@CDN|737P%qapvLSbM$LeXz5phx~_`T&2`U-Q$7g;>teGrwTtc5#A^ zUZu0(Qu}5koWL+US_g}BLv=>G;tq#~A%Fm1r4ZIs2yw)|Om~*CN!XpGQs`o{kji4~ zAY8Rm)s9Phs-{0ZWT7aeOS^>R{#AOO*`cWpyCHfU>1A^T&Y^k|~zYH%Q^5EvSfCX82ariRaKXy2%Qs-`5^ z0NeW=w3N|MO~Ymuv8!i{tKcfSk8p{pNH&4HsXSfwv2!Rk-qG6HrY`V!JKdOI&4vU( zcyoTX@wVr2?w($)HYLaBfC14CTOritVzl=S%6Pq=t@{sX3Lhvn(}iTVTj5FIq`2jpGD|~*DkDv*bR{T(;B z!@*b<>s1Od?~@h+#h>YhJ~Xd1JM$kL`0T?ZoH@9;rC2)mU`REs!fRcfb zyR<&WUQ6^)%+`)3+d`EyA>y!z%9rEBJqZsWM?iCzE@5zi_Un}+ECsi|RabCknXUZe zlvDk2%1Pj+$A30*Oj)bg!I%aqRhfU5f$wTl5Po#6RIgap^;^XTrrMcMq7!{*Qt(~w zQeZjNlW^>+iAz!LnYnm7?LqomEzkE;$gYOB9#Ob_BxPmSzkwBW? zN+4!i{vIs=G3D6Zx#LvW42JK=v`dNFwl48$;2Fj60*P5yemI^nKh6C?IsBIz(u8Y@ z&XgFIBeNeEx0f5$CM8M2=!-#Wq8YS`8MrLE!ERmMMzmlBWyjq_-zOHRs z?jSzv?2KhbZojOt&t?-&#=){9T7@{zFxm?eb|U78`!86eXSGN1<^rrtQWP(Lzna{P z+hia~9rp^J8c(+dXu%~vqLc)csg-7avexZIu%=Q#xCbCn1zt@Lv5g;xhzt|X;M z)nvSo_X$6I>sBZ+-n{ql`cSH81Rbd;%o_N@FaNCv#7Z(i_TzMw&zn8LS{19L@RDs@ zpqP5Bfc~}t7GZ?dOd7>1qQw4m%bdANCs!sjCr@1k`yBf)PC*H z5=SE~JxqUZFq1*Zzw}*wU$y%=@B(RdLwGlbvDEVfj&6mzy-;Mdu~KgZ1hHKZX@f-! zGq4O;S=1n>e)^1Nxzw%SH!vfs&WCY#UeE?^XwvMS$hLoyt99d~VX;ff@ssm!0e@|D zwIx0o-?>KiB+B{T1uM_tdB5d=dV_`1SkC z{?PDnh!_GxP|7+~oT2XcBA+l9Fa(Q=xc8}0iYKa!9QSb@lThhUbPZD=H3G#=PI_0A zH7dh*1($AYVP+v2z2fILFu{cD@BktwaMtwYE^;fw5*~=Xc;AQQ;#?9IfxL0^lO*b?UUQ_{tDYzb;2u=jv@rh#OjbluGNsvs(b;9MY0czDRS#R5_ZH`wtUALEfw0OAZ&5 zew`#kMx7!_2fYB9ra%7R6(y#o+U7F)5?vGcflnUZirsf`kgIcM_pfNE$+v$s*@+J{cGC1(JG$lBoCS)6Su;|~3N z#dNUdbDEoY({qrH#7 zU20JAj@uWs4H$!WD5D@M6VBFN9$E2=J9%=nIod#IeX$8g{Z9$hjc03eJ2@mEoznM4 z2*fLYp|3xno(u{MdNRGrDkjT18!j6%Fo~ohg%EX!e+C+(R5}C&JTuu}qV>Vj#hk|M z8KYwHv(12EnD;t8$|yRMkU1^aT6YKD{{4D_X+Nk`)CO5ZOx%>hyyGBq0uupn$D_lj#>rPT5xePK?`K zRz(Z`?tKlL%+{v4+D~Rs6xE6UZq7eQ*gyL-Eb~Q06t3l`vdCy3D zRj`lI_k%BWBt!2SH?&eE2#}!>`fQ&wsx+2({m7KbmRwfy06?ljh&K6rrTFnLdh{30 z3=U+4G4Yq9?1hFAyXaov#E%Lef}K@Vr6F{FX0w^=4A%E>5!?@P>IjYr3P0irU|6^# zH-dL16ybc6PnSzI^!6M0~vI6MDg!e8_U za^N6(dJD1omX%yM5xUMbDi_rX3mYde<%$VqF)_k*q6HEjPZ|NUYvhHl$UrNMfatP2 zY+JMYgHbfshlTuM0yQaVg{GoIexs4y{E8l4eqC8VeIX{n$Ns(sMUEz8{_BOA-FA46 zv(K8qOhtGY6GLsU@iERpmeH?&&dA>d(8CM~!;=`xE6DBL6BYy687%Ez?w0y=AEUlm(_?2=LtMBK9Zev~n z`5k(xof81Od~lTlTRs1^{UeNdl;jj zgAsSKnYcue))BPdui8YP1$P#~JpjQ$)zqW=$5OhyPK+o7O{tM5#B`$s%!wDjbd5i0 zVUXx;3=<_Jcy0)?t~RUt0r(_<$vEaqtqU{NI$1)Oy$z~uC9*btEeYcqISpiC;hf&h zok&7v6anQ}rO3OL-li~DGLR#LE(o*lFhkRsEwjjVOn%MCgUKo!KU5MFR;YqhPmpoO zTK;OtShuU3bcN(g<1HR7&N?gl7o3lRC=VFj;K9go3<7btFg1Tp8j5$7@u|=pNkLBP z{eV;)?CtWTTyO`)p~bMgNKEP!$zgbZXlzl`aX1z|m6+@8X>WS47AWxKlv2wmpB694 zJSR8$XWe=_%w~_%KDhTBFQ%4|otdnJG~0PT@((WYWgc_*dmfF;2M7M`lnK=;1p`q1 zdBJeTTO?hy_BLwj*Z_T7SO%jX^|8^f0FK5|`5C{A(0?|W9xk}Ls6RbBbTXxn@RvV@ zIzb*^yhrW8#6-U^{(G~nj z5PjG4vYA>W;fY9G?7A;_!2(V+dAPMeZ? zzlxf%<5}$&-&gC3oh;@AVc$G1M_Sd9MTT={<7;pV9AN6XpPXik#Ewf!*f}~$ zt(6sCW0=cevI!WGCW_r}jiHpmwDOc=+(^y}$5YExB5`M)#_z!EVxbl;2G|FHoV#}T z_WqBFe9vuwY(Q3Qvc;2=WdnB(f4l1_Gq3y-jxt-O(9@n7htotm456Vbz=(hMsBY|@ zz|c=8ApgoYm@o1{a>3PN8xk7Gwt3KHhVNtq@Q}n0DQD34iQKDu9M5}yt3%B;6jHF8 zLv0@kBb8Hi9UbQdEQulz^K5m4kh&i%j05Py;9}NHk!6Hd_A&0)^vJ(E;?R;da9TV^ zTwly8Q(0{G1GLMU@BH(=tCtg1#CWdyi-e{0`#7CEGF6jV$xG8J{NTy?y?=n$m^$1x z5=y`?=v$Seb|S}LXXu1;Hghm%(U&*i9HgAOY){?mr1I^r&cu8Z9AwDD^_#%Rf7>ax z8&?*4nA(qR35?u+UNgRt^VSeB0Mi2hsvxJ&@ou8B88P6;PQs&*6(w6d4w>G2`MY1n zB}4U@2$vk*z~XfdHTg|tr^69u-ZlEBwU@xb$ju;Z0aOCgqmFGn zBoyjRJ*<|6{rOGS^6D;B>theV*Q%6dqesK!{&@ql?NJ(u@c@tC!bM{FxHL50cL#Y8 zJuE3se)QIQwcVQo{Co?}Do0SU2)5%N;F)ZKjTs7+>?S5NdJbnsc%**EGjUfrnKqD= zRDi$ZQzB{*jj`)6pY?2}>=%ns0Xt46em|t$9-$Da7CMj;Y{9Pz?n@9(8yjOe+;B-Mb~kT3v~rRzl&|%U_5X0c6(|d7QqzDy&wy*pE*n+S#tq{~9gH40=pQ>t#ga zB(Pk~oRoPh{I!T0gL_2{4mVi#k6wv=8a7LSzyjRGc1=pwDY6o>G&wub?DM8c-&CH( z(zUQLUUtawx!(Qw_QK{DhEdQyfgPJuJrHK*Huzz5NZHPhW2dR61T`2@rk|FALaanF z-*E&O#jU}|!Hm`go27=|-RQKt!`;qp`>T5_^ukiNe%3}OLsO?t8m&yHuB526baW)O z-2Ac80qZo5%2lTGWrtX&%cFx|ztO(MZPA{X%bvW9fuYuUn$m6gy9Q&O5*(7j!;3p4 z1d^LAQ7l%-&~Ynexz^vMZcZSgZgl;GQGiW4mWDIe>Y7jc8FUTJ@->i zk1+%(r){RcZLu8BG6r5PxQvE~`b@NF&sgL@xCC;)zH7vKTY`WzwHBB7&Q=NP<*FJ= zFiBsEdLzGiqwooeL9$aJsh=-poA@(NBW=uk-RoQj-j_jr^|CxQq?OKHYzQp^8;=b2 z=U7ENUJ@`Nrj=Qg<4+BHnXkP(*Wy}?v(_x@QR`onTEy23A{y>=3&u8O!Qu#$`^`PG z)ZSOCFafvskE>PiC%lOL?bQA2 zV$Td?q`Ul~@ZKb}KZ1eWGMxxLpL&X&8q-{2R;+cS-azO$n6|%)w1*Y;n+ZF?7aUj0 z%~&k1Ac-}9<#z#l?7L%Fb;Xk)4cLe53uj!T56p@`+C0AS2SCt63(6-U`Z5~T?G~$i z?ia+aVEv>N^&h%AkTj`YvDotiEqb8q^j+Y~efq^Vff4*F6xJuK|QZ89V#O8aG! zL-dHTM>kYJW!UUoqyA0B@V#b*F>oO)-uM^vd0?&7{_sF-=}Uf_nQjX1(4Jv7 zXp9v?Rg|yP<(9yH@CMTpiUhzY@F>GgYJFD+yz{=JaPwQa=G}ZpmE5;36`WhCcX2j6 zM%|N^anMwKfkt`}4}nloCi)E&edY{33+*Sb(ru)a%oLA;SA9=8H%{^&t`G2h zob*}gv028Ng65};LnO7?TD&UNm6|z$c0JSc@{PT>1bP9-#3r}gexg8Nvboxhw72)Y z+Kbi_Cid&4#i|PpO#tS#7R1{^mV&bX>g;`2FjOp4yPa$T~`?B>hS!$3GB$!m146r{?BzE8`wS8_5 z2(Qq*w|~uTPi6*Yok?bdwto6x;M0f8=5SCy)$da8wLu9%5?(VGc-MLp|ZLiRO=$1P|f%AaXfFV-L)-#KAqxZY{oOE*DDki4~u>we(qbL!PQ}cAgco)3k&I?pIM|lt(nJ->&*R zI@AUFq)#BY=kIKa2bd~0v2elT+In<8*oLt;`Q7zY8;_6$3h6^=0qDIMluL~ckvsc^ z#@h9{5N-g^B68`wsB1gvn0iH@>-frm!c9t~Llx>Z%0D>?R_)K715(g&)hhPDirJw3^H#WwJ83YitI9mYljMuBt%8 zBfv;9f3D_$;htryYGV@*Q$^1=Cv}!_2NBIK0;fd~x)m1*Pz(M2m&xrwmOj7D{{A*5Hb#Q-j@(OXHSe+BopnWRF(+S-NRhT746ZSy zBp0Ma>_X!c1o z4ApZ@8BQ?Hz&>Pvy!Txb<(DjAXodHZ)#x{)fTU5@Hr&P7#<|GO{j& zByuu~#cG<{ZZ5pY?HP@CM4q*S{z|6>CV&J&4UIesIUNjMwUXU5K{r)Q_geE@dsc07 z{J>u<;mcp>=_G*(QJOVRVKzhP5hk@!KDLLAmAz71O5&uma}g8$1O{G7erGQT>}#I& z7u!4VBoZHqhUgeyLx z8dXnc&dlE4exr=A`SodII8s8@l)hXVk}wJm{N1C z$jJkKYDEKSa=v6Vo|-Exk!pxcVpM6@u9Rdl9ewBtzKkM#l=bxhot6AD?eGXTH+kGX@nEgFF-Wvrh z(9mMgomwwB-`;eGVhn`I%PyiPeJOzh_j0K18E5s3{m8a7pqdq&VBgMab_US7eOT$J~;uCPX+F>bo zKCdIlV?XSsveMw2;Jgwic+!r7Qg}Z%Oz-T1rQ9CREzLI?d6-o# z0VWf?nOpOck3c2F=Hbq&+%wxyDDAje4BpETQvLaen(dX5JUvsqzvWxu)rPJXSg!tq zX*`=d^;z2qdZ4ix^J^ax-WBVpU)%kjTvO0boVd|lua^s{@G8)Lfv8l9%iS+)1EOj&YBmX8+ zh6_A%=cr_t*$wlK8m4|zf4|G7fL>&jF)=mWNvylkpUiTwAENfJ8QTIR{?_9D`+~x| z4l0g>J>pGXhdZgw4rlkchMFC2LngP0{rpHZk~qXt9`Y5<oo3u-hamwrM*z3vJLD`)oFgii8XE_}hM%b@Z z8rY?2_>8<>CbI4y{-&(E4kaU+A9MCegSIFt<56lrnsiJ@=In{%&c9Inb zdAEP(c%dP+KYNA$bwdUYvIFr^wo~H&?(-S)P=EOU0y?5ZMFAkJk7&Axf0zBwUsS(N z{=4u9lD@`*lo|0;eiuV2_|If=a#w_Z*ZGg0p@M;w8Oca;TT}e!*`k1GVc7r4KSLU4 zN%Aj6;HYo#%whkvYKCd6zfbt5s_Fli+z2oxF6~tXRR8 zdk--eAVuC4>mTp6Iz#*fo~X)H8*DSwFO)Q@bPQNp>Z?#&PYof21gq^l-xDCD_D0@M z+|RJ6ul^P^)K{c2nnrsKA4fF8{Mxe3wp|4JHw1(b85B4i=xjrxrlnP(510@rZ-UhP z03boU`D%_g!l%I9TYjU5n-1BR2fO&OTtQ`z(^Y0jM4mdz(xEQkzDJ%Qa&sqEY?Kls zpZ=bPIg6Khgofx(EcN$Cpyz=I1UR(StyBq;D=^jR3h1~xT7CW^3pm_b3i7*3B`^w* z`h&{@XSIyt5;-hJ!*XU}|0M?NYq1bn(K?>hDzch9x#&UYGv69iya3}Q0BkTL4Fo+} zO$R!CkGZu*!vI|bsRuAs z1uA6oXh3S5n6jJflj2KrcRpi1Iznt}vFgBQ4(|&xb^}Q=NKsgT?9I^v6$FbiBAG7(S0FE_Ebq6ev}@U8Ke zIvi)z_-;Kj-%6uSEbgFif3Y3X;<(bN2>b%w^6>rTVx+ukw7zUsVSbO@4E*dFDdz-! z{gqBN_IvjdG^;FPHo{@ENv#9J{3rDQN?{0$LThv(lpRJOLx+?vJb{YX9Uk28pJq3D z+FfhwXapGGPy?~YNOo>eg$Z-w7|SE&nB zs<5xpmE=lJ9h&d2n-}O<-F`o4r%K7KG4ov`9SkK`#o_D$`71Kqg8zJ=5?5n zv>eO85|7`R%y;@BFJI2`>w5`64tXYsvy!SaN8f7B{`P2rQEzY$3`ey(Gck5mcx&SgC>%F?{G;uR1bapb~_Hba_H<>PJXXh=8;MBAOF zcaRIE%ozqs{0P(BEbD83^aPpopfAO{}Jm-7;ij)<5sXLicQ!-KZ%# zR+72>PiIL{f4R>%4&(CwQm`s;l$2kJs42OK{>$hNL_^AJHzoBhRsYe#5M>-31(C~W z#5nkW>$T6KE|8H$eAB<%|6?!7DIw;XQoIoQUnd%3>2n|>8IKqF7N;kO`uu ztpCeGPTh-wh`*!41$!qKoKCY;K~rf^B}y7D<)kBjhU8&<3^EkE-LhsY@jlnPdGv z-M|&r%PYLQ2D@#3xU<@J7Wb2#-DwNpnwz6|vxO*kr0o!C7Ib&@!+@F}&*9fc+Sjj| zTbOeSG4D3{e-z#cx6t|ZrXEJ^_LlbZUfCvH`BU1&_1n%!9o0xkxz?iSn_ahR9On?I!ciQ->ogc47^h8oaV{STt>Q zYF-VgZepFJJnTne&|pkIk4M3_41Fld$`Q{o+bcDf^_T$!OlZ45r2G>Rrt_ZFQHC8+ZJt}H$-$6v7Y}Yi$CoOK zRNN1*GJzVyUKs$K646HZM41$I4eGg_2Z7ZWv7t|>iqz#dA8MaE#|PPt;ltLhT)maN z`EBmM^PWV!+X_!CFzYDsMk?0=mX6v6blG){M-pSXimV6N7z@int#pok1|QC@hv?Mh zcvCt_TS(RjpVvZ)Y^NoZHtRG#vt&~*T)Cc)E_Q?3C+12UOg11=&ioOdhxL3WgdP%f zKS%9i7&^7lU3d->*Sc6^cmM5C65%P~%yOV?UW<}#8kh&XbMBDxjGzv^SZuod{tJC-Kmp;#so%mS zSJJ}#3b{?^lw_v)^)eIFv3oUj6r1U8v_Wa1cwi%7@Z=%JcLw7s(<^(aK^vde@vU9J zW5TNcX(^WTf@&%}3S`%R?208hW^SnPDt(G@5s@fF;XU~(#1zuu<^~@OJL_~Ks1l7? z>m~n%)dm^i)HZ18W+)INUgMlbntA4Yz^f(eyB06sFRXGChBXFwZ^6?qN&OBp3ElJ} z_Qjs`9Fplx2iAv&3eI_=>x1dR^Ua8%XJx#e>H&{09n2Qb^3dh4RODEeVHXEDV%Rx# z!5TDrlmKU(unyzjyYe8&g5q>n5HFHH;82^||6x#?E+K_wGr`liyUeRy(pg+L^DC}k zMOJo-xYE5muR0^fj{W>i6{O#7=6j9C*Yt>sDH7I_R5Cy`Zg;~8W~5OHBh5yH63+J3 zA`HMtVUa*I-FZAh`2xBg2G#en0s^n+l{yLCMz6E)XMi0FNFM1zQ)OJpkgr+YgW%lg zHiOj6C$x!krPCdkkr1{vcYxg*7~N0;3NumM5z6wY?ygg5I5!w(y{A-F#`vZNN5V!k z+n#w38)Q4g%cip7SUCtEJJMzBMowWZi`J$kv&H+1&I zPvvR^S8^Z>BC|$reo6s96iUiU&!310hz|M0n$kQbJR7ym@9rSbX2L+^~;iH@r8*ttexI{w2@2LzBTOeZH$ z4;_f6)iX6-{D$soY^VZiYoT5siRZIv${66h3V>YkEqTCX!gOq%Ij2g67C#5*(hg-R z9d?6Y6&!|U%Tl^1h#zt{g`3=~mZ(M%vXfqjT!w?H0yO@R(!~d=#0r^-qg*ckz9fF6 z_QCfAn5)$bPI~G21zKtU8l7q!Y~9yEH~aN-om^QOG2fYFGeA?IO`53HQniGsv(^S@ zx;!WbEYA?-3>!{Q zzOAouSiO^{rq2OKOvdH&!*FBo%-~~@4->$emq}^Wl|#7C6zwqOabOx zpsO3(LztYh?~f={rxYcJ6uAI=5@c6zAB-}&jP2*LDAD_0IGaU3EG=b5JLx-$irl|RzzSIoWj$SL#u)#?vP6!?qBkQkE4#__By8ke>m$~es>i_; zx5jb}j}TCKtMgcEi;T}7AMa^c&n(v#e&OKQ9X%}?=Tm=C_nJygY2dg-9P;832Fzxgd;+HY0N3qymOkiDb@AKCcKD*Bc8_x4VJQeuz z`WmA{VRz9!{JH0Sci=veh#UvOChSKa@y6!_X)uc+a&z4GcR;WDQ#ynXkb;i(ZMSQdT;sFJ?0M&N!RCCF0 zf{|S$K!K*kf&$FRa9DR^rL1bLi&Z3Hh#kee7%7p(Zad%m&f>0Mok>!~;VvominIrC zsl73bjB*0x5~|w8BsoZ%Ix$^k9EC&|kd?%&RyCE4!p{Fa4`cW$P`|Z;(&HTWD$_e_ zX-B7^f9I}gN>x92y#sG*i?-m*eK3DbrobcWh|lOK&p`gn`mHhk!q9|&R;J#5pG@ht zLz{PueFlbI%Xz|NFmacfog&*3>UF9p(_6$>fm zvwiC$LJ5v#PHsj79Fa*t1VFGa4;E$8MN1;f&%t9*?)|Nn%O5$EjJJ#-^t>H*v`Tn zA>yPm<67H6KVp9`{6-mMUQ#U%y*0aS^ao6qahM`y8nPZIzzuQmGdhyb@+T3)0%<-P z=b6UaE{NRg+CGS*IBFu+GEq1JG{k#D#G+)hTzqc={GC@?Q*aNvMlY4W`%iFJ_uNlB zEmwc-9y2A>$PLCy?LWm*%m+=!dThtW2f5Rm+a@=+e{pTq1~bRR-B%a-XbS6GpLq=d z6bVjbcQ{36-7WgdxKj?j+`+MmU#(Sb{7fPD(|wNyg3|@MszXBXDVNsi7CMQVL#$$V zVfU9_wz!)t8ezprZ9h3Wf&u>DL{;o2A96`Vi{HA<)iw@SN!Tt@h+rooZ-(}=0$Nkb zOiMLZ^-oJ@=1TT%$n~oy(#UTuS^JMXyR_0NzmU$H4~bgNsPcB$V1BbC0n{E{&g5}B z`U(VTT)_OiVZUJRRI+)@)+zolnLK6he#o|4>X7%oY`3kb<1?tkb=}<#B?n!v7L4`{ zd?@u(`fOJ}>C?S=?}!Iae&SP|Wy~1Mh3feu62W@M@anc1fpV{4c#3+YSw!*!6C5== z&ME;;uR1bRsjtQ!@B&tk5_Oz#Vfi7HU1F$LD2aj*?s#za4Av)8w<-KX6B=CwGj5!< zMa+*a99Eu+#sUAq=rdz)@J>7CzVbJ>_1RkH!HLKXW&<;md+)^w!TV1~o}mwa9#2~^ zS2{+}?3>&&9?01>&$>a5e~b041vf6FHl;g>{|J02aF$6` z{pPRl$IJa)k~@WGk%L3%DBN$U%%rNbT^x4}A%BoqWef4xp6q9mOVr2NrQW zB8i_l5L%ho17a>?!Xrn-Zxn;V1=AE^7_s-vmjZuGe27({BO&Jv3KSIEyxZ#*f5lY7 zcf@X@@Hic~%udszZ8+(Zrv8fnIbi&8qaWc-QPq2a&(KEeloRSRDRR`prlr_^)LjbA zr9NDBm{UlUW^p;vBz8UV{Tp8qW?TZZAQ2FVD#4?#Zc1?i!X`4pWu3H_(=Ri4ZSL2C zn&%CTdB}wO@I{;oZId$wYF4lq8Iq1YfiXM%z!&ZtCzZ>}fnlvKX=j5Eo+%H8pyj+C zM&;YMm|wQ!PqriGClTY;E{J=kR8qi+qKn*&pDYL;zh+8b#s@yirhR5j6Bj}Yd8JS4 z&eCTTmkdOWildNXIlvqf$0`m|drc4`JPy5x0TxBeit;op2pVlZ;0eGUeWQG+16}t* z?XeRqOpJILv`G+G#i4l-oHpF_zj@b$narZsYHM9e8#QuYdB%B7)7pJPsp!a3s-hkn zc!OB{@Lci$HDoeHr!IMZxEQq0aWg&}vwz%V;12@pQGLsKW`5DljPp3BEh*6)$`rzp zOV38R)>QHoK)E-HPc@e1j~?+&S!EE6{q;y|Fa0#DFB^-uIX87RSrM{~%>Uii|3UY; zV%OX8+xebNtrXxG==Hiq?JzFmIlblDsvF|>5in#sMNPpCiJA2<2SMJ(A$=YyF<>ta zED_l(KphBPSjZEH&(~cjMb;zt$a^O~R6HgSP&v)Yuq7JihGisfr)Uv?fWo-fIL>1_ zCUE<^>FT|q+l$Vne+s40uJAOrtlGm3}-6 zF^epH9WGSwO!&}vkMCHORfWwX#4Vgu`weA^zy!(0+v5dJp1nl(AS}mWdw7hKqyWJ@ zh%J9t2hAWRJ>A}R1B67;++7mGhB#glU6i+ZmDd_L>)25v0vhbfcz|r{Z8;HwGM^2b z#uJ0><;Eza=Y8lP$kAM)``*IAu0i`=U_c(>J(VJTNJbEsBWTTB!BX}$*PmJXRg6AE zih#|+zZXyZ1n0TIjW=vrq=jUYZZJ`37DFNgukD4+G&4d_%d2LDpjYw+R%Pe@ncJ{O zp%;F*@W7szgtWf%LJ;Lqwl90oWBFzqblu~>f-!!WLlQVnaLtH*;_clp>Q8ClY z6=HQ-Vt(WG@jITP!~rvaa$`&njU2JMjP=)M%f}v8yOoBuZE!E|@xQO8({dcF@!ZPJIHQ-lCpn_I23tCeJRb z$1{x-KEyKx8uZ6B4aM1;v}=71aYc-2j-)_rGXJnIxG~?@eHhct#sxOUwsQDhm}3ce z>nLf!t5?Tt>QMrs{q`A5@hDw(hRed*NhYhC-ByI52*ZhAu}xeLqPP~xVfk4vf@({smi^^9{BhYvHAL)2t4G5iOkj1OPNcv~9jJ!j(O93k2M_0uE7@?{dbg=!;|x`dz#cX{6s zT{$yTn4Rve;1A`{cRy8>GIxFtW>7OsvA6|7#kH9g6lG}*sFum>>m_5l`4lO6H0~zL zmf#PU+y}_-7OK|a@*NoBfoGXx6AKh80)ekln^y9BB%r&`K;c2I{rXQ6XtsmVqQwxRjFyt9i z*<~|C9KJV3ShbPQGFpynaN$%kn;GVZtwt<=70N~Ugb!6N&UO{$iGWWSd-xe2Jw3%M z<8pi-haY4nE;|h}1g69@vpPD#WpQ2qapN_)KqoW-6k^Ww#w(e+ z7Fk{_;$Di#uqwUi@$|0lEcp1(jJuX*1?>1ZJS8xcl9DFdV*AfX=>4;esO*BI?3C8b zwOr&zL`nf$*mK!w%t!n<*4gqAW`x#g%4`XwgU=BvjWx@)LLvBB{G?=%kVkw_-q^Q0 zxfXnp)9O9r?sy@ZNX*Ens*Ny&UT%DZ2zdMKEYwotC}`1wxiE<6Jcz_Qw3&ex1J+-4Gcv!2O>g%8fd|W18NM0@Bbh?u3DFpQ z|)w0C*2+XgVMKz)7@}$MOu9 zoF&Lo)*Dnv>Cv@0D9x2)y)`E#;hDz8`@ zmNf)sM@xlgc+n4${LFN?Q=Hb&vM4yu2N647z!E?NQY}T%k_S48q0p*G;y$I%tv*p8ak4e@yHtbACty0;pn(5mZq0~tn+OS( zbp111+((9tFyH6G1ibMu$EBTT%<&k@-^~MgB45Yt2dhp{Z10yxjL6v1_pdVwtT^s7 zle!x;LJ6dt`kYP9P&}4ZXkczU-416(;6#ST({RE|e_y)zDWJX@z1+c=a$>CcE~WBV zMFaay3AYa$6uQXDE{{A<<0T)A#G_?Cmz^Np74YU%dwA^Sr85$U~t8qEM1*OO{WwOA7Mb z;jD5Y=X)Q;y5j>}nC}mGJu(Xuq2TUOQl40U4vpN0>~FyL(rv22jQf7WnqvRF?_k|9 zNLG#IiV?h}YJ-KS#F%q|#UDlb6k0%IAl+&^v82 z8~S=kuCFGZb=w`CtAd3n3BMlITN(_;^SzFtB{FhFr8~zX-u7_Mj(cu9np;qQbgRk> zf@7oG<%@uEAhz9`daE3I-}x@ixrxnyh+Vrzru3s#{~Y% zQ(H(FxG>i@-f41@UqbTq!V==+a1XaWSg6m$vgVo5g90|9X1RVfxeE`Oe8cTXbCSE! z1pZ&irtyTDwG5fnxbK`>!~;<=tln>%YM5Oad75ouM+fq>b;sWjKJpNgyg>7suAR|* z(;v5b3}9o>AFMry4!l|RI;6Zb^nTTQpPPz0%$XTkN|12&yInQryLI!ez<}e}= zCj>;liq(XEdPVn~;~B?0>!--{~_mc?pk<8 zfA8(GV!CO7ZA;EKLc(#i3{5#UBnpjiZ`{z45{`M~^~lXmT<(`g z@sR)z8B?c9QeI}+vKYALQ}Q`vrBG<@KRKG2wYRIc8@@m^-$&SBza&XIll6Gke5rC2 z4+zl|%0x)|wr$sCG9tx3`nqHUJY4OpzH(zt$9^tV1PukVFL>Bpe{2fc4>b1AS4K6Z z+)G$JrZ2339WkI}INKzPS3wt$H`P zd%FTp_;XQAY;2Mn>uLlWLJ)3jiLJr!3jm7J362`hTV(!=VK63988^eA!#vs=3p+6> z99g<5(xg?rY0>zeY>9mmJ7wL(IKBmlWtHR>dez?pW`a~I-81oj&3fIc&JPNvv3 zn{1TlHqo{eGa#dk!G$Jc>gqJ<75Am~W3yfa{Mxef6ZPJ_3*#RAwms>$=|3QEcXaA} zxc>dfik7b2O%Tcj^fTOZy%2#|v2MAWv7smF@_yIgUlxPH`d=2q1E=?-&M6T@!>k{C zmNs`KbM_g3>L+|cinuv{Bxoak#cCs;?I&+;>r`r@A;w)4lY9lN)t&`XIWYtA?(+)6 z4O6ZDYP86e>^~cfZ0`#hHaGxA)fd8%HZRjg{;iFxxb26s4?#I;-; zgXwAou7n8aCjo9Kc?Lc0Pvd{yi((HbzM}^mFrA_=N`g^c@V$a2*|(59U9MJ6YZOU6 zN}xw~V==8$Uu%wYka-BEVLw;g=iRRm-!%_G;7wWYUe-I|SPw@4G;KUuaqsBpiN=PR zwV1(_S-nRXHgW0cmNO$&+}JgKs|+3yF>XIV02{m9NLC>4rg<|5R8T5i5Fmn=3mOOTexc z@*|RdsZfkw+I!}Bb#F~G(O6_#{8&SnoM_k2+!h7n#7v_mG-NyN=ccL#+}|(5=0X+g z_~c9?kB73$7pjKSPpd_PDWfA+j(9-)aaEjwp*jktSYz~uhA#MeMa$LTVYmkDe7o7} zbWd`C+b6L+J{mD8hTvvtX~Pdv`Gi`~9M&tLLWD=1e}y^0B#&;O;8qiOkrrn(KDRaY zMl&ya+9&OvmHCHD0#?kPCi`I2`rpU@kIC`Q3np>AJ@HIq{EsH`0nB5}i}^2C=kH@+ zE+!#nLH}gVx8Hheph@ok$w`EF$HjEB3Q2!C&^N$A!+f*T`zyNfUv^0}KXh~$T@Uw( zXM7!VFS(F}UY1$dKi>HHP(cm_7VYY3I=IjN!#(L{4F=Opj4Fihod02{_~m2$C3#v- zn791X-FgJ`0QV&B^ds)P8ZT=?f;X{^0z~VaQ~WWKhCfIEBFwt1|}yGR(>ts zQ~TFRR_N{Dzn#2dj4I^+|5h=;v^&D2`J$lzJ<0d?#~w~)E(QNEQVEm5d=+l`ayi7x ze@z~LP4~wt7yqSpf|Vxe(ZR$}6}1d(t$#X)>;5lS^mM*-Bkmtss;vr`oU>a}!NFAj zuj!NwEh#aPR#VslESa=5$rK}DF<(X{m(C6rs2Rn97nq}I*Np)S2AVg!OJwVL7s z9|J|QqKbX zqI5|V=~Td&Yne62NI_%yDV>r87BUw6Q&voL?a40mKQhjvDOzYmqBWRsxkv8!0+vWm z`VD4ridxPVMV&6UNd%+5_h)9a?o7SgT=0I#FKPVzVk1RE2@TFm6TZ-Khr>dL5h5`x z{vq@!k(!(q6&hVs4A4OyPD?Kr3(oLE6GlcOh4|jo;c$M=m@)ASo(f@)aD5!iLv=Cf z)HHeHJ6QYX3Jl_3sxg47P_ry*ZK<_Af8mbwc0mFQqub}ZnJdG?Ly(a|2l}E7!$Inb zNr{R=^S@uqhz3gJ*yRVOp`2~`1uV5V6xT7b(xHpzqw{~#l2VXyNWPIB2*<*{Eely9sc2F@yS&upM9 zDs-xPZ~ycr8e|hOL^U8jlh`7zSL{--4Kz}IowO+TMV9*I4Mbw$bz?Gt=CKREP@$Qh zd@z|EV;~Z&-y?uHZ*QSR{{~=_m~iO8|5`>y%>!F&C!vxCBEe>~UacQYRNxouU50K* z?~_z5SEW#Lf;~Dq3N1F*vY?M9;@j_Zi%xzb9032FXpn#V%5ROzRwR!yvX^Cw?}5EY z;XMu7&$Rj@iMOX``wb&Gc`qHM3iLYh#WCZm&(;S|qu z#Ha#m5Z>P2$848(5KHYp8L(M&{b^|3*b@Usz-erNqeJfM-OJp*w9z#jc>qtCzQ0MmX2 zul+UbqI;R@-W~(ubr82_n6vDju7BF59xSHS^~-E~3mI$vGa#6R1=T6ZN0ar2dh_e- zF&Gb`YiiM2FE%``u8OnEh&dPWO=f04Sh{yGM_lULAh{qc+$ZQAFo3ZB6DWTtfp*Nx zbMMQ}3ArB$@3T7;(;3Bx`1ei!i9Z)uKhL|wW(rvanc^38bjow>|2exZxYg>uxrVI2OBnQTthQ5; z=262}{Z!+#nUCbdBbr5*XP z@tgBsQ#T?6I+bp>WbOYVs0sY;T_&65jDJmALB63FEcyi+|C%Oft6^1BMp`R0#=+N7s9uVWO6#zeE-E^z4`_8(8rH6WM6<4HPSQ68Es<#>84_%-R4Bqd_kYM_0o zKgX5S#4;KfdG!?Wkt3<}gy5JpOvFm>X~zV9RJUi%vhTrV_31I>eH~|Gwrrn5A%3AQ zQ!CwW=nX=|nXY~>*Im@J4(ic=$Fizjnwse7j3(~2)@l+TibljzP?J+tZy5=a-|aHgYbU4 zgL?R;?V?bws(BzmEcEWpLXqB%r}kVpxt%MuBkmeZ{Py%Smkb7-m{^{;7)Yx^%RXxW z4GT-J_V)IcHd^q6IC+Vnei20|Ej|QuOwP&nK`)>hH3f%B>k|wi zBbzntf0lE2l5`q%2+#gvzMXx?G4s?7mM{@NdMTpn8 zT5Xj3x>6$qg&PHHwN{Tu;`4lTcSf60xx}DUB#TG*&3R2TqVH<}Y-iXt)~L}nVfn^YDm46rE&P^Rt;-#l5IH9i8Gca1g0)))SME;YJfD6$ zbTp;$cew9>m0?BgrNf)(zcc?0#pY6I-thRL_nQ*7xcN7(15#uV|0z$9!w;tl1;Vg0 z*zg|Qm)pdB3mm3SM&|m1?6@&at5m7jX_c|Xd~3-BS|URnnsYbW`z*eVJ0&ENxDMLC zTSJZvpzEc5Nf0LV*K#=naFn;<49I*cCJafqtx4TOvb?NbVBS3|(P`p9_z) z^YsU+2ql$4eoor`Yl z)(zX>^XMKL!rAl&F>%RmPtdDZgl=@W8ywF~wW|uv4ClGlJK!WTsAHS=JAk5z9$#aY zC1vCW`cMTT;Yh5i8|S&2IUHXe5iks_Lod9_xTtyXFCslx8$Kp8>I4_cq?G-}ppr(5 zkf)U(OA+Pix+CB)e0yYOzAx0l#>e&dImGlf;F zhK6}F#V!i&IgQ<2@66P|-~Qf7y5Ch_p&&o+<)bEf0Ak3ZyqRwLx|sNeK^;6Fisb?Q z;G#ftZ(Q#|I*(K<>mk2Y4(|K{ep623dquk5METKTVg?LQ#{b5oYzL53ospOo6Wd4N zusHL1ekBn(^ZX3-!rcI}3q7A7r_cDDf=|2-Cidghl%LaxA8r(@qKx7)Uh zQs<7>E3g#02Yy$sXGM4Shp%0~)}I;Yzg`3gP5Hlu*Vg=Ap{h5^QlJ>aP-si{7M)J#2kt#zo7TXya!i@Tj!E$`EW*Wlb4gZJI>v?up8sj{MwE@(GU5Op@esXV5}AD0&v&%5hd{-QT>&N|0VE%F*aU<0+)c z#5Xa{J2^=y#3E})*Qa@bqs1ROj=%H>oyz|}MV{-um03S06|FH9NFKsk?TG2A_ZTV< zp~(%|Qw(y}0{8&Ra!5oxgy8I(A~+q@$kz*x!{_XwLqAS@WP;dBvJ4)lbZK&A6qLcA z*S{|(8f}5SW()NQ;MB-qmd=;&^A7K9z<@AUkNzLhDwizUZ@f3R%C{c~@>@AwJRixZ zBp=CMo=*Z|Wj8{4Y>?!T{gu&c|#FX2OqL-lwib zI{Yhz5g3%k3fY6Ev;0%_lU@ek1LcwS_sdHW)a|4cyiTz+B` zq(42`^D8cR<{kc-aZMMN9f~GW7VedNXI|D&;lDr3Pt>nacauI{*A$FAr;iHFFY z6(WhXxV2MDhl9yNs21N*DJ88E2@H#>!cC5Wc7F5Krfr+r1c^qc{mN{|X;r#@q_ScMZtC&yBoowr zpCbq+@we&#PYjSUZOhjzMvbo^)H1k(OQ1;U;yCtrzFr0xm{B>gzdkse$v*_W+?1$w zz@~Fp1aerfmcSk6_*Da!58QL4_eRUon0_FC8WDkvFG65Uml#;bW*bjMjq$hE`%DS z5u-6e)6WDXTF;lwODf2ylb}G@%q0v)zu_K$$X zUtes0vV0evNT)rNR5XcEm9G>X*`_ZTyi@}a>x6lCeWJ68`diig@7|uoiOMTYcI3KR!nq@?~daog1nDh&Yp3JLQwZY z1L>xMlHvNu23#>1+ng_YYV(IO&kBk&Vr|a}QAqgxDeNTL-ElGrnkT^ZQB&oxD|DJr z6nNjbIKG=j@Tt#BCQjwsrj^&4xn)MDc^F;YUFO$pP3H!vV%=}N@R>$qKzFLVVZVOU z>@=hTV?{E3Ft@$n@|cM^W*S`($q7y2V*5vRrk4akG|L)+&G>pZ$ON1-tmEnf9Ugb0 zX}^EkJ;uJ`N8){x_j)HWTa4ySzLKP}nU|UB^1bkW)O8XtjhV<t_1Mv?<^n92n!?rnv9^x=Qs zW_Woz$hPp3f4hWoI!R_auFHb;F%s(KQ{$3SLwvH0&o_CwG&S<;muj5?gF+!ZOE(-` zaKf1KT3*(p+?X@0g(GZ=vU_gEDKvT3eF#-XL!Am8YW$+;*Ds%Pr;|)z`J5JyRcV6~WQ)8#z)HGRtT2pr0$6*cYZe8~bA zUP@-;ewYSoBq?ui3~RcF+HBMhbUYR@IxJqK9-19zeJbmWDOamjYvHYm3JIQ0WxLgh zifZsM`y+>!_(Mcj_RAY*zkwJ(XVnlRH~YYPVoWxdUw9Jz&Q@*!r9OkfWP8i#>^<6^ zVXcG>TiDAIMQv5v_ZtKR)1-$0zPNT0`PBVb**{_^g+YJbCK-f=$f$J}l(*A4h*3gh zAI=1P|I8H9S<&WEB5Kot@k~|-76nfM3~2uQ9<3}IQ_=M0l;6BBc42^i8$@%23syrDAx3MXRZP-;v+W$+9U&>ln8XYNLxq|rUdahI z;L?VXqvdv$16?&S;PQXrWy)rLQ4Sns2`;{8TS6caX{Kos8^Zd#WjQy!A-a*&N&p3K z@6Cz^NdPR0ivRA8LQx)Ag7r_|tAuXW01{<%$8uQ0y#w%$P~MQ?J7p%n*c|V}=xA7N ztWkB(?5E(=m$)oWJ(|wWjS981^!}6BW4=Bh!JE}mvyj^;mjX>!WY>F$k&#_AFKZz! zZ@ukIUPXxT<_^+zsO#vwzTJdDll{-pGGZa>>rPv^gghcwpKfH*@bg5 z0{E-xbjHoESKJ1L`1F(Rxsy)H!)*1~6n{o=if%oPpT$Y%(>+0C&XpL~Lm`qA0Wu3= zRX+q`S=am94^@Y~%b`;N{Z14XSk7AtRv_iNbK>xic2U+OfgYW(p$-8d1Ep>irybhIO>d2yN|i^mW*h-Be=d}*?2ZOFA#qcT7A z!&^%_-_tIpBU8FzkAILwy++m4TAc^ln~2kTRNF&zz9()Y0_Le3$GMaDyH}&fjGW$n z?q&ArJy+-J8yptC-DZ#2XaH~xlb9ul4}F%i&Q_^-foNBq46 zhmWm_d4vKVo>FiQBF6!*H0%0aDui9u8ER#mum|=pPm6>Y827y4U0}JB*D@BcM#OQ| zk5a=z*}TA^LtGiutjW2olp8D!SzG%J$LFNii!Nscq<7&xU3RbM+uZi((c|1vZdkqV z!du{j`Fs>W!uMxxzf|!lGLTiFd8~vJ^n7A-h~r9SpQc_=qg*Gup4egCw|6sw*^l}Y zMaFEf%hs)6>A>AlT_;(U*5brjmZE2n7`suHFcjjzTsy|@VPk~kX=m9KpawGB$dqjj z64q0moxOavtj@)XTJ52@l?QDe(4%XtqSkHm!pe(RiUAn!Tbm;=szB=xo8_0H#nD7F zB?eG?PTM_eoloa&jC^VJjqAmyd^;ibF8DTqT`UB2aNC&ITg?Qh5Y@#YZolv^b}R;S z^ChJE?Lxy7?DZ&XM|B0Tkze!=3eJXR5e_De-u<8^TRk4m ziqDr~O4o3)%VSeH}lSxP+2|+{qELwzm9JrPfb$&M3nGpYdY~3HDkp z?-o0aJiE4R6(xJn_V{+HHrE&|z{Sh9K_uQP0*tPL5|ea5pL~Q*`=7wj!`3AEmR;%- z+ZBye?Bx%uk|yLJOV=9qHkWz-889tkX9`7`mrHOjDV6c)v&pGgKK0)8nZVsl1q!W9 zpgq<20Tx{X8B)p-pw*k@P^cpdt{YUDvnRFy-e!v9paP9Bx+Vu zn?z_eVufBx?@4@`zjWDWomVr?xC~%^xJW-jI$ywO$Ub zbB$vjlEi{B)ZX@N{Ak4=%BG3b}oeuSA;1ka6@-0l&4 zQ3mobrqB@?_H|XPdWQEY>2%85(#P1Tpzm7w=HE_!VlbWJ0o@79fGC5+*BZZkI_+VF zx1y=LTf~*$InRnRDIyZ9yPm||7`;js7pd_31`^O(>Aoe1|D7PI5L~gSstiRZ97OwA z`-Fu-tOLIs|4`RQk9nF|1XK&a=zmdB`xwbjDQVWhou&KdV9W-KIwm4$a4V-7DT2Tm z^QO&QWp;(xjGh`gu)9)$lv6a|dT>1x9$1O07_A5fOk|>wzQl^u>>M%cbN0x$Xfw ziw5b!3lWnSH~?V@#W4|Jptrs8Iq_Z^$~$dDB#)}wp6FcMigS{-7FBU@j2GaS)=tJN zn4&AuH=^9fF?xGI#3eS2^=fF-dshvDXs&V?2PLtS=QMf|hR!kF+IJeM-(Tdmeggk5 zq<>Ls@D_v=LQcw*+eBKKIHK^&nlHdhWU47854BMVrJ^&Mu^D5%Zhuh{!MqUubgmI^L`{bO zYv%7bumJZF2EStQ<#e}3SXAv7FD!`CDUvdPfOL0vcOxAVl0%ntcMHyF$zR#YQtt|ZrF`!l{06ajaF z$ZnohWiFZE*fq(=p`%O0?9~qH^@dO66W>>Nt{e#%Ec{TcxJP^Fr=ngnG(P{Z&CARu z5t!c3Cduy?UR$<(BH?=G){HeNTPpC`MUUkudDm2?<^#?De0rLE=X#5cTsmKV;D>_k zaOG%*P!>16@DMRg8S<=3OV*<)`MoOZO%#NV_)MRP&=@1X);FXI`aI%kzGKD5;+Efq zj@Uo?QjS0qy6u>0E}-K$_Kz1D)AHO~da#sQp3Xx;S`-^KknxsP)dI?X%gKW2=3@i@nB;E>rz0S*>{By;flASpe#BT-JHQQ%S1k93c)xSEp)sQb6 zIKC%+^8p?0oya?0{yVk~vi65N7`_Y^RIabP0^R)u*TBek5u4|HzMkEh zIa{Z_=yI}nRy)&f&o}d;kG?tYXly5vAP?rtEN>PqR%yx~6EhM=kHHL+jU0I>M7@d7 z%C^j^mS~tr$+XXvxC)5VPWbth>RE#LY9$)Y$)~Vde_*WU$h$si_j%~-W@*iu6uu469Tlg&GLBa7k!Yeyt;k-#+*CM3h_F~gT2>&i`id16 zj4?Yzbb2i;FiY_iV0pSMKX-oV89BSp{Xv~_AzDXod@s$YKpjBm_{I&Ets^V8eOJtF zQ%K~R;+2gp9H5s-_!JMs2=FY%5rveYk$k35lhh|q_yykavp~I+m=!b`=PxQ4h|e)V zwGjFQKByuyZB_zSabPq(UBdBUWk9^1Auz>{-*<4v+qYTdN&&iLMdq4NP3~PDY8$;c zF2OU_66+)N!{hyu2@#5M272fg$@F1V15x!u2p5RBlrf~z0#KVbJ^q%sp zoC5Jo5yfCV?-?Z)^kYD`dZawb!oFk)$v8T$t2PlhJXOo~&!>5L7{ekTB z+IVLyq1PKgN-o&e?6}>fDshn1ZxX`oh6%Kp?pbQGx7l{wb!$tvL#Ox}7#qlXiVD{W z`(gzU8xf^%e|fpwa@k6Zj){j?yl{GaESNYNm8t%t{hEn`Ac}XYQPPO#OCF+BBAx6+ z6ZL8*z(KD!yzNj{KS^-)w2X4t74a>5O>>s~6pM^=P4w$}tpkGa0!2*9R0hw(&_p?f z$JWk}6_buwf2HYomZf{0q0$mLhwFPnvH;jF!)&puHl#lR?;Gj2<1aU-^c|TtQMytr zX2)P^O0?TsZ&wYM=1nCtF-|V+8JL6;5sfr@UKfH}{e-M(#Cxg@!az;9w84Wm3yp#! zofS))#KQ}z3vdgmo9Bb^n~{x(nT{>c<22}J2;ijM(Bp2_WKglk?U~Ri)(C_3{w;^| z;ZgzuXR1-d{d5$+;5!vXo`;?|iT2de`m;RkC# z!pj>9)nZc6!6feL>T02O)$8Z&4|stnUeav}N1Bne%__ARq=$>zxe9&T&R>Ps`y#3= z=j}+z7kfk9+fwN=xEvAI$z?>+xatzfJ0Uj4qegqnEs4s}xp{f(iF&4nKOgSTgv6O2 z`9wJaLNPEHibl7UBPxn!@6fI%>>12Yc|x}S1}zVZ12XjoDq9G;4E8efmxNx&aRr2H)^1i8xWS49j6h$IE5!w}F-MY9sA?rpC_)1~4H0^rugC3$uRSY^QC2r}P=6n| zuCkILo>~iw7;|lPDWT2%vRDBlC;?!0LL8*DSAZ^!2opDWb>?eGn^XH=JED6010b)6 z9S&z~Ne|&73Aqm!Ec35{9&1^SB_E5!pGkH+RQM9!B_Ax*A$pBnr$AH%0kOhl=uG@r ziZHKem>5y|#mLqnl2bitkYbtQZ;^PMh8b zB9C&FGq1RC?lFjq6}4BN$!@))QX{WKhIe6ly}>i%**KI$aiZo)p#-ya5v z-~tPvW+H%Q?T|cH{W+ky55F=oEx`J{=ng_vNByq&qqR(pUaKCOmclZ3UK|q|fjnj^ z;zXWhJ?N*q(L-37Rzq>NA}JN;eyzo&^~n=tRbv7!2Z0J*UW&6m$e8ukXj*@842dn* z1553F+1nZ{N6Lb}*;$Z8a<8Z82#ZQ^xYXy^_kw=*{7m=RdD^&QY>A-w(|ryI2Nr>`E51hAk#lnX z-E&J#_F4$|3igVG?;i6F;Z4Q;UO5c>&lKVpiiTT6UcW??bTH*SY1#-nra`EL0cuvD z&f2l|o(Eo7eaXBODsG9Zm14Jv=*n;SEQu4N3q81Jk^N zMr5jWwx)ykQ|*}^%NYf}{*h0}q&}{NR0DJ%Q5wiUeZSMkZ*=WsbBL=K7CR7=Cb%H| z1N7dPivD^lZk4vzqM@12Khg)aj)BQ7s~bgeqX$24-04gn*rYu;R++g0AvGn?Ojs5a z%AtCYhpngJyRlK^0?4J_^3q%ju5^4InnqqcC5s3RRa)i1BBfDjzdv-hTo_)|tl8+V zEYwHQs-Jr%hq&-FxFL|l??hXzLKk4|2FRN!ZNKd_&cDc+k0fxyTBvS#Ut=4poLkcu2;h&?qK!z|qU(DJ;N zqBlXW@PMIa_6|DXJ<08NOZNTe_A><1&mN1`&ShaDE%}f@pKk2#9s2@Jn9WeLkI}Nw zdt`2K=2$p5g%H#QCi$;~{7@EON)-(^{ZRXIcmnhei`3-Pg zE%fjof%#S02W?hG7PkHiA0~t13K7rWu>RQE3vxO*>=xcY2C(Ebs;P(leDL&i(UeNk zUaEAkxK>77krXAP^7e#y6tiSilEDK!iF8EYP|SVR{+d#$#1$eJ|6nT9t-eE(Emd(1 zi!I~#-iOk_wW^6Z{__0S?EAFnzjh)xC~~MV{*ttkACb7TM?#8t(%mTm{+JHshCYxi zka`%u6``KLFh9`B+PNG7wSiwG?b$3xcluMd!?L&}7sZi`>}WJZjQTkeCsc$nY@#u5 za&FtZ(8TXE5BCxR5jh{SZr_7js0~FH>3Df%Q}XJgIJTk)+NufZ8~~}72y>88#q_#1 zo_dKw7Bry8S-S&c*cudfH<4NbG7-n`(cpIYt?|m2l-fAY2Jm|qEiUGAq6~(UoM(dK z+~x-rS0Y+G?$6n&*vW`jOHZhpD#j+N5gHU^)n05|rPLeIsF9Z`rN|X=z3k!>HIS`a z%ld3>hGu;E1!?}Y`2=Z~qBcha*{WD}eg}f`;*Su&QAj`pz6y|sB*jStAZ%7Uz<*JD zeEw2;`4DX0ncjuzF?)IK$~x4H(E6O4kJxW-1EdvKzxb-I2z|#HG~cI_7+}Oe8L}tc zRu9UnxgVu!H73YgJY(S=UOarMXNvhupjM`Lr);l#1DoY2Sssy!2p{u+1MT zB{r($=cw1LaWR#e4A$dJiMv)XuyA+S&K8ebF*swgrtM3XxR2@r^+kFW&DG-~J-z3Y zxj#!A<{lBaO1@g;F4*T7pd82Nb9w&@;{dy(eOA2se!-Ej$1ip0+BJ}tD&1wIcD6l1 zkl}e#v}>DPrsvRgnuDNjtEj$-y*b7@!uei;zJzFS{8WY_{kzDFB zPHW{e+8_l=dDQ7>38GNcFf5bpbKi96(T0Hr2R!IC3pUHc@MmKJVZ|{b%Zs1Oj7PPKgr9!GMHbs#X;nF&p13fs0{0z{ zFdtUm5s*z*9}`)}QPSsG%>GnBEil&?E7PnStz|}$KA5kOPY?y1H1JjHuMz>b4%I{t zHOJ0J*!aa(*D(7Xv&54XR!i_|M@ zrWgz+@z#5`2qrECUO3LCIR3->P`8W^cLN27z2=`<(oPC1_N^xN!|2=XB464aeIu^5 zGV&slPv=2Tr9Ocm(yuOE+iRX{r?9=Bgt2L7R0yzfihB|qTD#Q*)*Uq2?Xs`20y@!? zN_;rx2tmhsv&1|Qr_*XC4|*^g#QeZ=Ev)K%gzu-K;U9pBZ=ZZK&%!#kUOb$0*o9i# zqSr=kw!qRNE{zZ2D+S}M6rfGWvdcQ{g-&Vu%baw^sTq1_T6)qL47ypHIf<2k zDjaU7RA%%kf_b~$KJ#4S^?43#!|843Dxdl(%Lzz8u|-AwFs}!so0;6#e9szad(rh& zquSw<+uUraqO7Zj)4@WSSiU2H!@kKuqIgUiqtSC+x1E=F!fhUr3J{toGRQ|MzqEs; zCh6nrq&{r}ZqGZ8n^*xOddx61EK=>L7NU-$wV_XL*OdaRRhpv;96=G!;J4zz*n3e0 zbBoH<)@y`hq;or61)a*QHi0YXeYn*&(F==vT6$F-s$If%B_84x+EKw=dk>wKQg_Lt;O?*)rAMX?7A$Efk-lOy(?Ckv4i=*>X${52(Gp%M5=R z%KlQmfKk##_dOzg#cJ5CsrHq@*A%? zswu>4vs8@DH833F4AqF~J8ej+4ygo1L{Fb9X?Z^F2^){RO?GYG_#W*nETcYovl&H#xCB+Tw~Z4!+B2k#nj31pJx-DuDjs zJNHBDCd@x;k5;&MaIDt8@8{KDW%VP$p#^w!nzSlfQyc!k%Rc{)U-X$n&S5hQb2Nqh ze5Fi}iPPrsK0H2uoWw-g{uE@X(x2g}i%5PS18()kYbf}y8?UwBYyOAYf93>SM4YB% ztpC25HYs3Q!d{h{dR!c+$F~$1eRLmjQWsJF?^~ghHNmMT{{o?Yz06KdKtyu%t)%SV zs}Lf>emqaKakBQ$4FOzk_JIA!ZdA(rAJft#0h~t%Nyz_q38^mth|gY4>3_VOqL1gH zm$T6RyM$C=1yXBblmA^O-J_K%44n7=-z984dZJPjrZJ)azRWHLI8WQ3TK1o@i<4Y` z*j2UYQ^tRIbR+7S*i%bR9vp(18qKoGz7%gOv8$e&?_lYoKQZnfZLBcUhzKBAz*Scj>6I&w{rZW zdXu3Wj{g5(btRIv8Q(VYJdTc$eS1YBwk4N>|59O%y>Qx_=+0pTR-jjMWear^*{;lh zI}u7y(1*kAjN(D36^2^9+`BjS++ZbB@F|PM0_IC<6@kD3|9jXABOn!;YjMutT6na@ z0AsvnT6+z2-&@aOd@_OcZeX6j@1HWF2y~hL3*3X2oljNcMg`QNWsQ@B%jp<@-|3W?rH$q|It^oH@m}E- z=(xBcNl9ja&&E$gn#)|t9YBYBx&a)kBk^|iE&wr-646vPd!(p))_R4M zC2L-m^A5IRwt}rH*Zw!Olge>mAmOyD;7)Pr8kMqeHSoYb#sDEALbzZGTLG_cJ{MBa z`|}N`KcZ5+{rzjqWx-&Ekf?`3FL~axgGKzfH?nG*0D_?4`)(}V1BcuC)o2EP#h|dT zFyPUVw*1;YGEwbu(u=LPCb5J*la|;kGlVq){^48cf%VW*qCo{tv>v3rAZX^zTP_?6$HpvYWn2~KF4}W7oBN?1T zABoO$elLCpuS49V5vf4if(n%c0!mL(P%`YL{-KzJd>7G(Eg!~4+%F*Ea zo*vS><9^G!$i(FcVF3UYQ|%2v3@D-uQUiSQq6arE>f}&@e);nhPDauNmgssoGM)_Y z6r6PtUfSYX#*;wyD8b!vS;m3g{GEiyI|U5a<6Z~lFR@6AkyrYT2b#$#0p0Si7#Ljv zL=m)ywcfDq*;#D&i$g)^%@2)Qc6Ll(%D!fXmZ+748uo^E_4mWxUi?tze0_LcqxEIy z98k3S6j$3{TC>uMbCTzXM~JlC{upYe3_&^Zp|Fum(u2X8d}6k@ci2@KyMCbQZztiv z##>6NWbV%*IyG-=3#S&B9s-= z@?Ft8Y(DZ9phHG&;?g*pw>qy(%bh3DwCAPo|v0 zdv|kHix=XHmC^T=F~6u#gF^QMpDQ|{4oCA=D7yrauk-^?nj{@imIvxe^Ie!- z=5l*pn5>abuL*%%?v*+y!j)w-n4j!(`<>nQ8@EB+_w&Vh{kbz|;Yc5@R)dN+eJa{Q zJG}+0*yrWvYXpTLCi!>heyEXY3w~L!D+-H`rC1H#sCdXY21lGHUYv83>OdKr)Iy#^ zTq&V!=DQB@|6~9}WAHf8e!k-0uXSxsP@WbWd+u57mRpMBoHjax_5p|1ZlP8I+hNP4 zZalpn=zMQhyaKeW*rFy2SpHtE_eY6q;+?w?uivz`n{L5iD4e= zkTL_NGoZKFIh+~@mVtF^;dwB>Cmk0S?0g zqpl%a;LEwvFBj^sfCr=-ww+rOYCxb5&&>^E`Sw;(rQk^F6B8pxO0+lNhK&g_Vv1GX ze_C+-26y9Jqt*pY0ixzY!D)>p zu-t$&{YCGJ$1OTOeo6h*!yH~T1!b;N&WUh5$AltIQ0p6N`m!vaNnM4Uy2t8>5&vuE z3R`X7u0i+IFySb*K}{9KTgvPnIpIH%ue0p8!~M9hSruhxA&R8b;yX|DDKiggxUNyn ztNMv4FWmf8SDm}NJF{)H7n`=)dRUY;8R^!SkVa5(-dEhK_#6z{@q86{TBTV$i$M5g zi;mE$gg2QFlgYthU-RrFEm`U%vDQn57bjOgMJ;9$sDx1Ycj1T(M#=en#(#qKoV!BcuI6U|S!pjdKQ$+>n&dD$#KH`q& z;HB1xFp8=?u6WsxInn=~eQ|-?`!p|cBKOhsPM|DMl2n|Nkl?`3omup~j_7)ai<`(` zI_tn{Q=`ONp5xJo;>Dt9NIVfMQ!)!%baNv9C+F-e6A=&@D05)tc@<`>_hi^&dzp2h zl;*PwBoYPNG${SK+Nqf)uhkR_{xG+Pfp$i>EB{Ce|LEwp?W z5~Q-&EJS1W;O zw#=^VSN3UrOd@m_j*z&KKf9|uoXhd;)Nw{@O(^3GtP{^ z{&&bb4)ElYBYgV1xe&pe07qi)a_MjXCDW-u{Nhmj9R4qH&Uj2@$vx)(78g*&-+)BD zCq(E`efoPzeeNDpcmAEB_Wx3806s(~lfz{CpCaJ#$hD;2#lK9N1OD@#LMp|(Hw|z} zVV?|2(oj5X3AZDjpJ%V9?uUJyHLBAzh+v0W3_l7v9~m%F`m;vX%0NzTJ%#Ap@~XZ_ z_xatzb*AQw2?e$;=((S6NNh=a!Q6465&ldhw-=I@MTO{KEanx>d5eymPUN4&zWgy2 zw}5#wV=vdKJ&E)c_S|f;k!%o)D$GF(&n{oWVLZm%KoGK!tEAwHiGtYmI-^NK`m?uf zfs2ft=gZvu$M5~^0w{JiE}GODmScADjEQP38IzE4=aL(7xj2-P%D=yvSk#xT5in)y z%*?TZev%V~j=q@#ZtW!oeqL6WI?+#m-SiEA;7U>zV!38T1qg-LUK2ePS3>DBuklqG zP?dpA8Ab5x^=3pjQs8WS9hsNW*%|jVCix?D*b1~q{Dz6%v+UB}WPClaNK5sJhc~D^ z3`A%fo>w{cbffAV4b>m>WQ zpxQs{Gmp_YX!Lr~>c{522dY#wsyAmq){Kj_ZBWptT)!z zpVst3z6Udmiy!>nUBZDY!#b{JL(jul{U8jHs`G2(G1iRTV`^~S*{~(Z zfB0er&k&US@ON_@T_X3#mDH5KfDJgG#dPm0$TFS5NHVKpo)lS6k4VmXH)>1S$h=`( zzAS_Nclup}cih~CCOgYmG8jKaXA8`9*I zSJf<`L~aY~2*wBqEmim#C{4E(8*RF+Q7Q^-MEzP-pkrY=Mnl-bgAeTXY}?Dd zSsQ_zKx8x!x(|R)HqH!sR()u)n^*swQwNw}YW8;k50}f${8n^wYO?XNEWQ2@Bn!9% zX~Vn^=ol}T$^u{LzSZLhzgdVTC;yw3s&ZXlk@lpoz@^DQDPvx=IAN3Or`Ta00 zDZ(K62zHU!TrwjRestXroz|J}gKtf@*3iJwbvu4mN+Ngy_)4>^Qob~PxMp%XT-sd9 zm(Soco6dy#VlxMrICz-!JDD_TSFP$;coz^Iy_L0_GQS(2l=c8W$C6_wjoFc)N!?;H#Vh zwr3Q%3|>FPE&4($j?#%nTO)_VJ2)=K4@r?(EGQrze-z}2;uY_wJ(sQmC6LQrsKJbeNcFzAg*YcuK^$GDgWrvfP^g zUbVpLc`bF^OCu^dp|YceV0r!kKJIki`BW*^MwD4HN8#IsH;TFgu|%4GG9@4G2q2Nw zE;lPtD^vNr`Bn6%;aK`|BX|{hfqz^M`6^GG=|X4BjIG-X!4{{>lJm{MkG9Xw>-a3VM$JTJ~te)3Xx%f1`d+`&O zY4J-?TTC<l6+gaN$O?k+qUOk_pGo(i4{PfmDUB%aeTk=)RVW_p<;$8buU1m zNV*Nt>xULX@V9S*s^m;e80SBVwwAt5$@3P$3?==H>`rF2h<}35^$t!m+ANvdnGTpZ zTef>Md3?5W3q6{~UGRm~dddIGSoi8&_m%z{spU%Tzs9*gyV4Do6 zWB}8nyw;ehTB4?kdR({nr8^OV{p@o|rSYCx-GvYAzOK(++r@qag;FB};2kf7fp3UQ zmRd7ibh?^q4^6Bjcd~O;Qo@i|8B2>JmoIX~BDEBc>?wM0CQ+DdhY!?aVk#=}2wC7v zKiZyE$trr|pABLWyHXrhi_NmC8`h9l?S=rk=9_HpHD}*r!lv~Z@kRdkrU#(#wLP9B zt4rO98idNiP!uzJhUJC%fMm%mg5 z84d1Y+|8TET zmmB8Pm1Si=e@t|FUQ;nUXwOw91d4mO*AsJz0r&KQh>0B!ny@^fpjg;~;c?##o`P|< zCMv7+n~6!#v;1M)rIARPCP%g#)?F>lstIDUVsNw%30iB`Wy1?Oqz+6bCs5Jc&0$^T z!{hzp=B2Z#!07)G2F|2FQsVGdSV(C5o+bhu=r%8N{mwieQKv!9DxqW|wr8;PZAZxm zv~A^P0VCNeCv+*^3*6Gj!Zi-sRWubAx7;2K*3UMuKkLvWu>sn z)bGVxN_aKXqpm7tcZRc0{41M)<87>a81v^dBL503$LYLxYVvE{#CFza(eK=U=Z=l6 z@Df*n<6EnzwooT|6dN|H5^`RN*X!qdv_#=?>1bXA!}fu)KBi0&K$9jaO4WuY!L;ECvi z_&4*xC%e4yEWhwd{@pF1U=H0BAv&7ey-V&!}Zpu`jH*zD6HFS>}g#UQod|m-l+R6#=7Aw zoDnvKhE3V{%psllCU>yyLPN{Jd<(Ws)GN{DU|E;rfcD~si5p~mNfu;v!@;)pTRS+{ zh-7vP-|Zx`OI4}hLR%>fBMQYX1V15W(n;NgNv#mx!%wxZZ8^UaCS}2djR^Huz@sHn z+Wf%e_y@p%VbR84h6JyX_gYyQ==d=c{Jh2m#nLg)G#)?W_wstDaMj0$SG0A}D97bE zl7WABlAUFd$G&R9PQGp(oj!uGhOupgPc!R<5IsttvWbdWKhwV$W9qP3XJ z@9OE00XkOME)9D+ZVqfSsFjvNCgI}s3U5u`>0`PRSqST*+w65YXteD3T|ErpoYbex z)4Vzws#j}}L8GW>g=^cfM2<8!Jlg`vXdD+6b=0k{#l~O^tzzMMmxZkO-&k#@m8`%m zvb$D>|EoJ%Dg8Kp%c5KgOH$IgYsi}KSFB;e>hN&sq#BFn3y_#O?B7z)+V!!FxR!Rf1!exL^r-%Cl^)@CRWs&qT3|drOh;DsVV3GF2=w?0AOtw zr2Sdznk77CprtEgkL4rriD=s&;z#Hk+v{(=&Yt*bKCg=Z)K~zWEsoYmA`+4iM@jJi zw?HhRXOlWgP?ALIyz4~@e|-rQdCZ)Tu9&d{{V8-j*I{R literal 0 HcmV?d00001 diff --git a/.github/workflows/client-compatibility-tests.yml b/.github/workflows/client-compatibility-tests.yml new file mode 100644 index 00000000000..8688e51e1df --- /dev/null +++ b/.github/workflows/client-compatibility-tests.yml @@ -0,0 +1,269 @@ +name: Client Compatibility Tests +on: + schedule: + - cron: "30 5 * * *" # Run every night at midnight + 30min EST + push: + tags: + - "*" + workflow_dispatch: + +jobs: + build-chainlink: + environment: integration + permissions: + id-token: write + contents: read + name: Build Chainlink Image + runs-on: ubuntu-latest + steps: + - name: Collect Metrics + id: collect-gha-metrics + uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2 + with: + basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }} + hostname: ${{ secrets.GRAFANA_CLOUD_HOST }} + this-job-name: Build Chainlink Image + continue-on-error: true + - name: Checkout the repo + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }} + - name: Build Chainlink Image + uses: ./.github/actions/build-chainlink-image + with: + tag_suffix: "" + dockerfile: core/chainlink.Dockerfile + git_commit_sha: ${{ github.sha }} + GRAFANA_CLOUD_BASIC_AUTH: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }} + GRAFANA_CLOUD_HOST: ${{ secrets.GRAFANA_CLOUD_HOST }} + AWS_REGION: ${{ secrets.QA_AWS_REGION }} + AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} + + client-compatibility-matrix: + environment: integration + permissions: + checks: write + pull-requests: write + id-token: write + contents: read + needs: [build-chainlink] + env: + SELECTED_NETWORKS: SIMULATED,SIMULATED_1,SIMULATED_2 + CHAINLINK_COMMIT_SHA: ${{ github.sha }} + CHAINLINK_ENV_USER: ${{ github.actor }} + TEST_LOG_LEVEL: debug + strategy: + fail-fast: false + matrix: + product: + - name: ocr-geth + os: ubuntu-latest + run: -run TestOCRBasic + file: ocr + client: geth + pyroscope_env: ci-smoke-ocr-evm-simulated + # Uncomment, when https://smartcontract-it.atlassian.net/browse/TT-753 is DONE + # - name: ocr-nethermind + # run: -run TestOCRBasic + # file: ocr + # client: nethermind + # pyroscope_env: ci-smoke-ocr-evm-simulated + - name: ocr-besu + run: -run TestOCRBasic + file: ocr + client: besu + pyroscope_env: ci-smoke-ocr-evm-simulated + - name: ocr-erigon + run: -run TestOCRBasic + file: ocr + client: erigon + pyroscope_env: ci-smoke-ocr-evm-simulated + - name: ocr2-geth + run: -run TestOCRv2Basic + file: ocr2 + client: geth + pyroscope_env: ci-smoke-ocr2-evm-simulated + # Uncomment, when https://smartcontract-it.atlassian.net/browse/TT-753 is DONE + # - name: ocr2-nethermind + # run: -run TestOCRv2Basic + # file: ocr2 + # client: nethermind + # pyroscope_env: ci-smoke-ocr2-evm-simulated + - name: ocr2-besu + run: -run TestOCRv2Basic + file: ocr2 + client: besu + pyroscope_env: ci-smoke-ocr2-evm-simulated + - name: ocr2-erigon + run: -run TestOCRv2Basic + file: ocr2 + client: erigon + pyroscope_env: ci-smoke-ocr2-evm-simulated + runs-on: ubuntu-latest + name: Client Compatibility Test ${{ matrix.product.name }} + steps: + - name: Checkout the repo + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }} + - name: Build Go Test Command + id: build-go-test-command + run: | + # if the matrix.product.run is set, use it for a different command + if [ "${{ matrix.product.run }}" != "" ]; then + echo "run_command=${{ matrix.product.run }} ./smoke/${{ matrix.product.file }}_test.go" >> "$GITHUB_OUTPUT" + else + echo "run_command=./smoke/${{ matrix.product.name }}_test.go" >> "$GITHUB_OUTPUT" + fi + - name: Run Tests + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@e865e376b8c2d594028c8d645dd6c47169b72974 # v2.2.16 + env: + PYROSCOPE_SERVER: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725 + PYROSCOPE_ENVIRONMENT: ${{ matrix.product.pyroscope_env }} + PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }} + ETH2_EL_CLIENT: ${{matrix.product.client}} + LOKI_TENANT_ID: ${{ vars.LOKI_TENANT_ID }} + LOKI_URL: ${{ secrets.LOKI_URL }} + LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }} + LOGSTREAM_LOG_TARGETS: ${{ vars.LOGSTREAM_LOG_TARGETS }} + GRAFANA_URL: ${{ vars.GRAFANA_URL }} + GRAFANA_DATASOURCE: ${{ vars.GRAFANA_DATASOURCE }} + RUN_ID: ${{ github.run_id }} + with: + test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestfmt + test_download_vendor_packages_command: cd ./integration-tests && go mod download + cl_repo: ${{ env.CHAINLINK_IMAGE }} + cl_image_tag: ${{ github.sha }}${{ matrix.product.tag_suffix }} + aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} + artifacts_name: ${{ matrix.product.name }}-test-logs + artifacts_location: ./integration-tests/smoke/logs/ + publish_check_name: ${{ matrix.product.name }} + token: ${{ secrets.GITHUB_TOKEN }} + go_mod_path: ./integration-tests/go.mod + cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }} + cache_restore_only: "true" + QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} + QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} + QA_KUBECONFIG: "" + - name: Collect Metrics + if: always() + id: collect-gha-metrics + uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2 + with: + basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }} + hostname: ${{ secrets.GRAFANA_CLOUD_HOST }} + this-job-name: ETH Smoke Tests ${{ matrix.product.name }}${{ matrix.product.tag_suffix }} + test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}' + continue-on-error: true + - name: Print failed test summary + if: always() + run: | + directory="./integration-tests/smoke/.test_summary" + files=("$directory"/*) + if [ -d "$directory" ]; then + echo "Test summary folder found" + if [ ${#files[@]} -gt 0 ]; then + first_file="${files[0]}" + echo "Name of the first test summary file: $(basename "$first_file")" + echo "### Failed Test Execution Logs Dashboard (over VPN):" >> $GITHUB_STEP_SUMMARY + cat "$first_file" | jq -r '.loki[] | "* [\(.test_name)](\(.value))"' >> $GITHUB_STEP_SUMMARY + if [ ${#files[@]} -gt 1 ]; then + echo "Found more than one test summary file. This is incorrect, there should be only one file" + fi + else + echo "Test summary directory is empty. This should not happen" + fi + else + echo "No test summary folder found. If no test failed or log collection wasn't explicitly requested this is correct. Exiting" + fi + + start-slack-thread: + name: Start Slack Thread + if: ${{ always() && needs.*.result != 'skipped' && needs.*.result != 'cancelled' }} + environment: integration + outputs: + thread_ts: ${{ steps.slack.outputs.thread_ts }} + permissions: + checks: write + pull-requests: write + id-token: write + contents: read + runs-on: ubuntu-latest + needs: [client-compatibility-matrix] + steps: + - name: Debug Result + run: echo ${{ join(needs.*.result, ',') }} + - name: Main Slack Notification + uses: slackapi/slack-github-action@e28cf165c92ffef168d23c5c9000cffc8a25e117 # v1.24.0 + id: slack + with: + channel-id: ${{ secrets.QA_SLACK_CHANNEL }} + payload: | + { + "attachments": [ + { + "color": "${{ contains(join(needs.*.result, ','), 'failure') && '#C62828' || '#2E7D32' }}", + "blocks": [ + { + "type": "header", + "text": { + "type": "plain_text", + "text": "Live Smoke Test Results ${{ contains(join(needs.*.result, ','), 'failure') && ':x:' || ':white_check_mark:'}}", + "emoji": true + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "${{ contains(join(needs.*.result, ','), 'failure') && 'Some tests failed, notifying <@U01Q4N37KFG> and <@U060CGGPY8H>' || 'All Good!' }}" + } + }, + { + "type": "divider" + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "<${{ github.server_url }}/${{ github.repository }}/releases/tag/${{ github.ref_name }}|${{ github.ref_name }}> | <${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|${{ github.sha }}> | <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Run>" + } + } + ] + } + ] + } + env: + SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }} + + post-test-results-to-slack: + name: Post Test Results for ${{matrix.product}} + if: ${{ always() && needs.*.result != 'skipped' && needs.*.result != 'cancelled' }} + environment: integration + permissions: + checks: write + pull-requests: write + id-token: write + contents: read + runs-on: ubuntu-latest + needs: start-slack-thread + strategy: + fail-fast: false + matrix: + product: [ocr, ocr2] + steps: + - name: Checkout the repo + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }} + - name: Post Test Results to Slack + uses: ./.github/actions/notify-slack-jobs-result + with: + github_token: ${{ github.token }} + github_repository: ${{ github.repository }} + workflow_run_id: ${{ github.run_id }} + github_job_name_regex: ^Client Compatability Test ${{ matrix.product }}-(?.*?)$ + message_title: ${{ matrix.product }} + slack_channel_id: ${{ secrets.QA_SLACK_CHANNEL }} + slack_bot_token: ${{ secrets.QA_SLACK_API_KEY }} + slack_thread_ts: ${{ needs.start-slack-thread.outputs.thread_ts }} diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index e8243c8cdfa..0fa9370b60d 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -327,75 +327,23 @@ jobs: os: ubuntu-latest run: -run TestOCRJobReplacement file: ocr - pyroscope_env: ci-smoke-ocr-evm-simulated - - name: ocr-geth - nodes: 1 - os: ubuntu-latest - run: -run TestOCRBasic - file: ocr - client: geth - pyroscope_env: ci-smoke-ocr-evm-simulated - # Uncomment, when https://smartcontract-it.atlassian.net/browse/TT-753 is DONE - # - name: ocr-nethermind - # nodes: 1 - # os: ubuntu20.04-8cores-32GB - # run: -run TestOCRBasic - # file: ocr - # client: nethermind - # pyroscope_env: ci-smoke-ocr-evm-simulated - - name: ocr-besu - nodes: 1 - os: ubuntu20.04-8cores-32GB - run: -run TestOCRBasic - file: ocr - client: besu - pyroscope_env: ci-smoke-ocr-evm-simulated - - name: ocr-erigon - nodes: 1 - os: ubuntu20.04-8cores-32GB - run: -run TestOCRBasic - file: ocr - client: erigon - pyroscope_env: ci-smoke-ocr-evm-simulated + pyroscope_env: ci-smoke-ocr-evm-simulated - name: ocr2 nodes: 1 - os: ubuntu20.04-8cores-32GB + os: ubuntu-latest run: -run TestOCRv2JobReplacement file: ocr2 pyroscope_env: ci-smoke-ocr2-evm-simulated - - name: ocr2-geth - nodes: 1 - os: ubuntu20.04-8cores-32GB - run: -run TestOCRv2Basic - file: ocr2 - client: geth - pyroscope_env: ci-smoke-ocr2-evm-simulated - # Uncomment, when https://smartcontract-it.atlassian.net/browse/TT-753 is DONE - # - name: ocr2-nethermind - # nodes: 1 - # os: ubuntu20.04-8cores-32GB - # run: -run TestOCRv2Basic - # file: ocr2 - # client: nethermind - # pyroscope_env: ci-smoke-ocr2-evm-simulated - - name: ocr2-besu - nodes: 1 - os: ubuntu20.04-8cores-32GB - run: -run TestOCRv2Basic - file: ocr2 - client: besu - pyroscope_env: ci-smoke-ocr2-evm-simulated - - name: ocr2-erigon + - name: ocr2 nodes: 1 - os: ubuntu20.04-8cores-32GB + os: ubuntu-latest run: -run TestOCRv2Basic file: ocr2 - client: erigon - pyroscope_env: ci-smoke-ocr2-evm-simulated + pyroscope_env: ci-smoke-ocr2-evm-simulated - name: ocr2 nodes: 1 - os: ubuntu20.04-8cores-32GB - pyroscope_env: ci-smoke-ocr2-evm-simulated + os: ubuntu-latest + pyroscope_env: ci-smoke-ocr2-plugins-evm-simulated tag_suffix: "-plugins" - name: runlog nodes: 1 @@ -497,7 +445,6 @@ jobs: PYROSCOPE_SERVER: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725 PYROSCOPE_ENVIRONMENT: ${{ matrix.product.pyroscope_env }} PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }} - ETH2_EL_CLIENT: ${{matrix.product.client}} LOKI_TENANT_ID: ${{ vars.LOKI_TENANT_ID }} LOKI_URL: ${{ secrets.LOKI_URL }} LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }} diff --git a/.github/workflows/live-testnet-tests.yml b/.github/workflows/live-testnet-tests.yml index 7a12ab6d6a0..e68dd410c10 100644 --- a/.github/workflows/live-testnet-tests.yml +++ b/.github/workflows/live-testnet-tests.yml @@ -10,7 +10,7 @@ name: Live Testnet Tests on: schedule: - - cron: "0 5 * * *" # Run every Sunday at midnight EST + - cron: "0 5 * * *" # Run every night at midnight EST push: tags: - "*" @@ -207,84 +207,21 @@ jobs: matrix: network: [Sepolia, Optimism Sepolia, Arbitrum Sepolia, Base Goerli, Base Sepolia, Polygon Mumbai, Avalanche Fuji, Fantom Testnet, Celo Alfajores, Linea Goerli, BSC Testnet] steps: - - name: Get Results - id: test-results - run: | - # I feel like there's some clever, fully jq way to do this, but I ain't got the motivation to figure it out - echo "Querying test results" - - PARSED_RESULTS=$(curl \ - -H "Authorization: Bearer ${{ github.token }}" \ - 'https://api.github.com/repos/${{github.repository}}/actions/runs/${{ github.run_id }}/jobs' \ - | jq -r --arg pattern "^${{ matrix.network }} (?.*?) Tests$" '.jobs[] - | select(.name | test($pattern)) as $job - | $job.steps[] - | select(.name == "Run Tests") - | { conclusion: (if .conclusion == "success" then ":white_check_mark:" else ":x:" end), product: ("*" + ($job.name | capture($pattern).product) + "*") }') - - echo "Parsed Results:" - echo $PARSED_RESULTS - - ALL_SUCCESS=true - echo "Checking for failures" - echo "$PARSED_RESULTS" | jq -s | jq -r '.[] | select(.conclusion != ":white_check_mark:")' - for row in $(echo "$PARSED_RESULTS" | jq -s | jq -r '.[] | select(.conclusion != ":white_check_mark:")'); do - ALL_SUCCESS=false - break - done - echo "Success: $ALL_SUCCESS" - - echo all_success=$ALL_SUCCESS >> $GITHUB_OUTPUT - - FORMATTED_RESULTS=$(echo $PARSED_RESULTS | jq -s '[.[] - | { - conclusion: .conclusion, - product: .product - } - ] - | map("{\"type\": \"section\", \"text\": {\"type\": \"mrkdwn\", \"text\": \"\(.product): \(.conclusion)\"}}") - | join(",")') - - echo "Formatted Results:" - echo $FORMATTED_RESULTS - - # Cleans out backslashes and quotes from jq - CLEAN_RESULTS=$(echo "$FORMATTED_RESULTS" | sed 's/\\\"/"/g' | sed 's/^"//;s/"$//') - - echo "Clean Results" - echo $CLEAN_RESULTS - - echo results=$CLEAN_RESULTS >> $GITHUB_OUTPUT - - - name: Test Details - uses: slackapi/slack-github-action@e28cf165c92ffef168d23c5c9000cffc8a25e117 # v1.24.0 + - name: Checkout the repo + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: - channel-id: ${{ secrets.QA_SLACK_CHANNEL }} - payload: | - { - "thread_ts": "${{ needs.start-slack-thread.outputs.thread_ts }}", - "attachments": [ - { - "color": "${{ steps.test-results.outputs.all_success == 'true' && '#2E7D32' || '#C62828' }}", - "blocks": [ - { - "type": "header", - "text": { - "type": "plain_text", - "text": "${{ matrix.network }} ${{ steps.test-results.outputs.all_success == 'true' && ':white_check_mark:' || ':x:'}}", - "emoji": true - } - }, - { - "type": "divider" - }, - ${{ steps.test-results.outputs.results }} - ] - } - ] - } - env: - SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }} + ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }} + - name: Post Test Results + uses: ./.github/actions/notify-slack-jobs-result + with: + github_token: ${{ github.token }} + github_repository: ${{ github.repository }} + workflow_run_id: ${{ github.run_id }} + github_job_name_regex: ^${{ matrix.network }} (?.*?) Tests$ + message_title: ${{ matrix.network }} + slack_channel_id: ${{ secrets.QA_SLACK_CHANNEL }} + slack_bot_token: ${{ secrets.QA_SLACK_API_KEY }} + slack_thread_ts: ${{ needs.start-slack-thread.outputs.thread_ts }} # End Reporting Jobs From be5351965c2986c128a574f773108986dd239ed6 Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Wed, 20 Dec 2023 16:20:23 -0500 Subject: [PATCH 2/3] Temporarily Disables BSC in Live Tests (#11642) --- .github/workflows/live-testnet-tests.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/live-testnet-tests.yml b/.github/workflows/live-testnet-tests.yml index e68dd410c10..2b5ecb2d5da 100644 --- a/.github/workflows/live-testnet-tests.yml +++ b/.github/workflows/live-testnet-tests.yml @@ -144,7 +144,7 @@ jobs: id-token: write contents: read runs-on: ubuntu-latest - needs: [sepolia-smoke-tests, bsc-testnet-tests, optimism-sepolia-smoke-tests, arbitrum-sepolia-smoke-tests, base-goerli-smoke-tests, base-sepolia-smoke-tests, polygon-mumbai-smoke-tests, avalanche-fuji-smoke-tests, fantom-testnet-smoke-tests, celo-alfajores-smoke-tests, linea-goerli-smoke-tests] + needs: [sepolia-smoke-tests, optimism-sepolia-smoke-tests, arbitrum-sepolia-smoke-tests, base-goerli-smoke-tests, base-sepolia-smoke-tests, polygon-mumbai-smoke-tests, avalanche-fuji-smoke-tests, fantom-testnet-smoke-tests, celo-alfajores-smoke-tests, linea-goerli-smoke-tests] steps: - name: Debug Result run: echo ${{ join(needs.*.result, ',') }} @@ -205,7 +205,7 @@ jobs: strategy: fail-fast: false matrix: - network: [Sepolia, Optimism Sepolia, Arbitrum Sepolia, Base Goerli, Base Sepolia, Polygon Mumbai, Avalanche Fuji, Fantom Testnet, Celo Alfajores, Linea Goerli, BSC Testnet] + network: [Sepolia, Optimism Sepolia, Arbitrum Sepolia, Base Goerli, Base Sepolia, Polygon Mumbai, Avalanche Fuji, Fantom Testnet, Celo Alfajores, Linea Goerli] steps: - name: Checkout the repo uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 @@ -276,6 +276,8 @@ jobs: QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} bsc-testnet-tests: + # TODO: BSC RPCs are all in a bad state right now, so we're skipping these tests until they're fixed + if: false environment: integration permissions: checks: write From c5aa49bf0e2a19550886f0dfcc06a92b07058518 Mon Sep 17 00:00:00 2001 From: Sam Date: Thu, 21 Dec 2023 09:51:26 -0500 Subject: [PATCH 3/3] Increase disablement of cache if LatestReportTTL=0 (#11636) * Increase disablement of cache if LatestReportTTL=0 * Also reset transport on consecutive LatestReport request failures * Optimize case where caching disabled * Fix comment * Fix tests --- .../evm/mercury/wsrpc/cache/cache_set.go | 6 ++- .../evm/mercury/wsrpc/cache/cache_set_test.go | 13 +++++- .../relay/evm/mercury/wsrpc/client.go | 45 ++++++++++--------- .../relay/evm/mercury/wsrpc/client_test.go | 15 +++---- 4 files changed, 48 insertions(+), 31 deletions(-) diff --git a/core/services/relay/evm/mercury/wsrpc/cache/cache_set.go b/core/services/relay/evm/mercury/wsrpc/cache/cache_set.go index 98388442d9a..01d47743950 100644 --- a/core/services/relay/evm/mercury/wsrpc/cache/cache_set.go +++ b/core/services/relay/evm/mercury/wsrpc/cache/cache_set.go @@ -46,7 +46,7 @@ func newCacheSet(lggr logger.Logger, cfg Config) *cacheSet { func (cs *cacheSet) Start(context.Context) error { return cs.StartOnce("CacheSet", func() error { - cs.lggr.Debugw("CacheSet starting", "config", cs.cfg) + cs.lggr.Debugw("CacheSet starting", "config", cs.cfg, "cachingEnabled", cs.cfg.LatestReportTTL > 0) return nil }) } @@ -65,6 +65,10 @@ func (cs *cacheSet) Close() error { } func (cs *cacheSet) Get(ctx context.Context, client Client) (f Fetcher, err error) { + if cs.cfg.LatestReportTTL == 0 { + // caching disabled + return client, nil + } ok := cs.IfStarted(func() { f, err = cs.get(ctx, client) }) diff --git a/core/services/relay/evm/mercury/wsrpc/cache/cache_set_test.go b/core/services/relay/evm/mercury/wsrpc/cache/cache_set_test.go index 7d754b8326e..59be76ed265 100644 --- a/core/services/relay/evm/mercury/wsrpc/cache/cache_set_test.go +++ b/core/services/relay/evm/mercury/wsrpc/cache/cache_set_test.go @@ -13,7 +13,8 @@ import ( func Test_CacheSet(t *testing.T) { lggr := logger.TestLogger(t) - cs := newCacheSet(lggr, Config{}) + cs := newCacheSet(lggr, Config{LatestReportTTL: 1}) + disabledCs := newCacheSet(lggr, Config{LatestReportTTL: 0}) ctx := testutils.Context(t) servicetest.Run(t, cs) @@ -22,6 +23,16 @@ func Test_CacheSet(t *testing.T) { var err error var f Fetcher + t.Run("with caching disabled, returns the passed client", func(t *testing.T) { + assert.Len(t, disabledCs.caches, 0) + + f, err = disabledCs.Get(ctx, c) + require.NoError(t, err) + + assert.Same(t, c, f) + assert.Len(t, disabledCs.caches, 0) + }) + t.Run("with virgin cacheset, makes new entry and returns it", func(t *testing.T) { assert.Len(t, cs.caches, 0) diff --git a/core/services/relay/evm/mercury/wsrpc/client.go b/core/services/relay/evm/mercury/wsrpc/client.go index 5b6bfa1a9b0..c9533717757 100644 --- a/core/services/relay/evm/mercury/wsrpc/client.go +++ b/core/services/relay/evm/mercury/wsrpc/client.go @@ -24,9 +24,9 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/utils" ) -// MaxConsecutiveTransmitFailures controls how many consecutive requests are +// MaxConsecutiveRequestFailures controls how many consecutive requests are // allowed to time out before we reset the connection -const MaxConsecutiveTransmitFailures = 5 +const MaxConsecutiveRequestFailures = 10 var ( timeoutCount = promauto.NewCounterVec(prometheus.CounterOpts{ @@ -55,7 +55,7 @@ var ( ) connectionResetCount = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "mercury_connection_reset_count", - Help: fmt.Sprintf("Running count of times connection to mercury server has been reset (connection reset happens automatically after %d consecutive transmit failures)", MaxConsecutiveTransmitFailures), + Help: fmt.Sprintf("Running count of times connection to mercury server has been reset (connection reset happens automatically after %d consecutive request failures)", MaxConsecutiveRequestFailures), }, []string{"serverURL"}, ) @@ -256,13 +256,26 @@ func (w *client) Transmit(ctx context.Context, req *pb.TransmitRequest) (resp *p return nil, errors.Wrap(err, "Transmit call failed") } resp, err = w.rawClient.Transmit(ctx, req) + w.handleTimeout(err) + if err != nil { + w.logger.Warnw("Transmit call failed due to networking error", "err", err, "resp", resp) + incRequestStatusMetric(statusFailed) + } else { + w.logger.Tracew("Transmit call succeeded", "resp", resp) + incRequestStatusMetric(statusSuccess) + setRequestLatencyMetric(float64(time.Since(start).Milliseconds())) + } + return +} + +func (w *client) handleTimeout(err error) { if errors.Is(err, context.DeadlineExceeded) { w.timeoutCountMetric.Inc() cnt := w.consecutiveTimeoutCnt.Add(1) - if cnt == MaxConsecutiveTransmitFailures { + if cnt == MaxConsecutiveRequestFailures { w.logger.Errorf("Timed out on %d consecutive transmits, resetting transport", cnt) - // NOTE: If we get 5+ request timeouts in a row, close and re-open - // the websocket connection. + // NOTE: If we get at least MaxConsecutiveRequestFailures request + // timeouts in a row, close and re-open the websocket connection. // // This *shouldn't* be necessary in theory (ideally, wsrpc would // handle it for us) but it acts as a "belts and braces" approach @@ -271,11 +284,11 @@ func (w *client) Transmit(ctx context.Context, req *pb.TransmitRequest) (resp *p select { case w.chResetTransport <- struct{}{}: default: - // This can happen if we had 5 consecutive timeouts, already - // sent a reset signal, then the connection started working - // again (resetting the count) then we got 5 additional - // failures before the runloop was able to close the bad - // connection. + // This can happen if we had MaxConsecutiveRequestFailures + // consecutive timeouts, already sent a reset signal, then the + // connection started working again (resetting the count) then + // we got MaxConsecutiveRequestFailures additional failures + // before the runloop was able to close the bad connection. // // It should be safe to just ignore in this case. // @@ -286,15 +299,6 @@ func (w *client) Transmit(ctx context.Context, req *pb.TransmitRequest) (resp *p } else { w.consecutiveTimeoutCnt.Store(0) } - if err != nil { - w.logger.Warnw("Transmit call failed due to networking error", "err", err, "resp", resp) - incRequestStatusMetric(statusFailed) - } else { - w.logger.Tracew("Transmit call succeeded", "resp", resp) - incRequestStatusMetric(statusSuccess) - setRequestLatencyMetric(float64(time.Since(start).Milliseconds())) - } - return } func (w *client) LatestReport(ctx context.Context, req *pb.LatestReportRequest) (resp *pb.LatestReportResponse, err error) { @@ -306,6 +310,7 @@ func (w *client) LatestReport(ctx context.Context, req *pb.LatestReportRequest) var cached bool if w.cache == nil { resp, err = w.rawClient.LatestReport(ctx, req) + w.handleTimeout(err) } else { cached = true resp, err = w.cache.LatestReport(ctx, req) diff --git a/core/services/relay/evm/mercury/wsrpc/client_test.go b/core/services/relay/evm/mercury/wsrpc/client_test.go index f265d54879c..10712461ae1 100644 --- a/core/services/relay/evm/mercury/wsrpc/client_test.go +++ b/core/services/relay/evm/mercury/wsrpc/client_test.go @@ -54,7 +54,7 @@ func Test_Client_Transmit(t *testing.T) { noopCacheSet := newNoopCacheSet() - t.Run("sends on reset channel after MaxConsecutiveTransmitFailures timed out transmits", func(t *testing.T) { + t.Run("sends on reset channel after MaxConsecutiveRequestFailures timed out transmits", func(t *testing.T) { calls := 0 transmitErr := context.DeadlineExceeded wsrpcClient := &mocks.MockWSRPCClient{ @@ -70,11 +70,11 @@ func Test_Client_Transmit(t *testing.T) { c.conn = conn c.rawClient = wsrpcClient require.NoError(t, c.StartOnce("Mock WSRPC Client", func() error { return nil })) - for i := 1; i < MaxConsecutiveTransmitFailures; i++ { + for i := 1; i < MaxConsecutiveRequestFailures; i++ { _, err := c.Transmit(ctx, req) require.EqualError(t, err, "context deadline exceeded") } - assert.Equal(t, 4, calls) + assert.Equal(t, MaxConsecutiveRequestFailures-1, calls) select { case <-c.chResetTransport: t.Fatal("unexpected send on chResetTransport") @@ -82,7 +82,7 @@ func Test_Client_Transmit(t *testing.T) { } _, err := c.Transmit(ctx, req) require.EqualError(t, err, "context deadline exceeded") - assert.Equal(t, 5, calls) + assert.Equal(t, MaxConsecutiveRequestFailures, calls) select { case <-c.chResetTransport: default: @@ -94,14 +94,14 @@ func Test_Client_Transmit(t *testing.T) { // working transmit to reset counter _, err = c.Transmit(ctx, req) require.NoError(t, err) - assert.Equal(t, 6, calls) + assert.Equal(t, MaxConsecutiveRequestFailures+1, calls) assert.Equal(t, 0, int(c.consecutiveTimeoutCnt.Load())) }) t.Run("doesn't block in case channel is full", func(t *testing.T) { transmitErr = context.DeadlineExceeded c.chResetTransport = nil // simulate full channel - for i := 0; i < MaxConsecutiveTransmitFailures; i++ { + for i := 0; i < MaxConsecutiveRequestFailures; i++ { _, err := c.Transmit(ctx, req) require.EqualError(t, err, "context deadline exceeded") } @@ -162,10 +162,7 @@ func Test_Client_LatestReport(t *testing.T) { // simulate start without dialling require.NoError(t, c.StartOnce("Mock WSRPC Client", func() error { return nil })) - var err error servicetest.Run(t, cacheSet) - c.cache, err = cacheSet.Get(ctx, c) - require.NoError(t, err) for i := 0; i < 5; i++ { r, err := c.LatestReport(ctx, req)