From 80bd4d946cf6e7253620df423ceb63f6f9534692 Mon Sep 17 00:00:00 2001 From: Phil Wang Date: Tue, 9 May 2023 09:40:47 -0700 Subject: [PATCH] add ResiDual paper --- README.md | 37 ++++++++++++++++++++++++++++++- images/resi_dual.png | Bin 0 -> 55719 bytes setup.py | 2 +- x_transformers/x_transformers.py | 26 ++++++++++++++++------ 4 files changed, 56 insertions(+), 9 deletions(-) create mode 100644 images/resi_dual.png diff --git a/README.md b/README.md index 928323ec..535cf4df 100644 --- a/README.md +++ b/README.md @@ -1045,6 +1045,31 @@ x = torch.randint(0, 20000, (1, 1024)) model(x) ``` +### ResiDual + + + +This Microsoft paper proposes yet another normalization configuration, combining both pre and post layernorm. They claim this hybridization reduces representation collapse (known to be an issue with pre-layernorm with increasing depth), while maintaining stability and reducing vanishing gradients (issues with post-layernorm). Initial experiments on my end show it to work no worse than pre-layernorm or sandwich norm. More study needed by the public to see if this is actually a winning technique. + +```python +import torch +from x_transformers import TransformerWrapper, Decoder + +model = TransformerWrapper( + num_tokens = 20000, + max_seq_len = 1024, + attn_layers = Decoder( + dim = 512, + depth = 6, + heads = 8, + resi_dual = True # set this to True + ) +) + +x = torch.randint(0, 20000, (1, 1024)) +model(x) +``` + ### Normformer @@ -1831,10 +1856,20 @@ generated = model.generate(start_emb, 17) # (17, 777) } ``` +```bibtex +@article{Xie2023ResiDualTW, + title = {ResiDual: Transformer with Dual Residual Connections}, + author = {Shufang Xie and Huishuai Zhang and Junliang Guo and Xu Tan and Jiang Bian and Hany Hassan Awadalla and Arul Menezes and Tao Qin and Rui Yan}, + journal = {ArXiv}, + year = {2023}, + volume = {abs/2304.14802} +} +``` + ```bibtex @inproceedings{Dehghani2023ScalingVT, title = {Scaling Vision Transformers to 22 Billion Parameters}, - author={Mostafa Dehghani and Josip Djolonga and Basil Mustafa and Piotr Padlewski and Jonathan Heek and Justin Gilmer and Andreas Steiner and Mathilde Caron and Robert Geirhos and Ibrahim M. Alabdulmohsin and Rodolphe Jenatton and Lucas Beyer and Michael Tschannen and Anurag Arnab and Xiao Wang and Carlos Riquelme and Matthias Minderer and Joan Puigcerver and Utku Evci and Manoj Kumar and Sjoerd van Steenkiste and Gamaleldin F. Elsayed and Aravindh Mahendran and Fisher Yu and Avital Oliver and Fantine Huot and Jasmijn Bastings and Mark Collier and Alexey A. Gritsenko and Vighnesh Birodkar and Cristina Nader Vasconcelos and Yi Tay and Thomas Mensink and Alexander Kolesnikov and Filip Paveti'c and Dustin Tran and Thomas Kipf and Mario Luvci'c and Xiaohua Zhai and Daniel Keysers and Jeremiah Harmsen and Neil Houlsby}, + author = {Mostafa Dehghani and Josip Djolonga and Basil Mustafa and Piotr Padlewski and Jonathan Heek and Justin Gilmer and Andreas Steiner and Mathilde Caron and Robert Geirhos and Ibrahim M. Alabdulmohsin and Rodolphe Jenatton and Lucas Beyer and Michael Tschannen and Anurag Arnab and Xiao Wang and Carlos Riquelme and Matthias Minderer and Joan Puigcerver and Utku Evci and Manoj Kumar and Sjoerd van Steenkiste and Gamaleldin F. Elsayed and Aravindh Mahendran and Fisher Yu and Avital Oliver and Fantine Huot and Jasmijn Bastings and Mark Collier and Alexey A. Gritsenko and Vighnesh Birodkar and Cristina Nader Vasconcelos and Yi Tay and Thomas Mensink and Alexander Kolesnikov and Filip Paveti'c and Dustin Tran and Thomas Kipf and Mario Luvci'c and Xiaohua Zhai and Daniel Keysers and Jeremiah Harmsen and Neil Houlsby}, year = {2023} } ``` diff --git a/images/resi_dual.png b/images/resi_dual.png new file mode 100644 index 0000000000000000000000000000000000000000..0f0ad55683da93f0019fb0eaa1f11014cba2dcda GIT binary patch literal 55719 zcmZs?1ymK^7Y2%=fFRx74bl>qP*OlZQc95S?nXj7q`Nz$yIZ=uySw`xe*YKiz4ct{ zx-N5P&di*%&)(nu_O}DRNsA%EHTxQMQ%0(6@Eavett#Ft;$%W3l(_I` z1;?a=MTejAn@>Hb4r3IHF31wNzO3+YUY=xLG}BI9zu*Mkb~T~Os&hff{!W9F{$Soz zuGeduR<7G;_zR0?XNlfuY~$gH<|6Q$Vd7-e@qWoisg(0yDM{|311z~Wy(noG6~Dm1PU4%)9Lku zYB4a06;H09qYHh@9WrceOd^!2E$xe>rj?e?sjtPEplc)}gWH?Hz4nAM5JS)C3U41r z3f|NsxIezuHJ$V6Qg)jf<`Wb&a@c29MXZB~PE15cgI{&V2=X0Bbtlv7MZ2iuHh5Go zo1))$(c^e7Y^c*5_t!)%PfKgtq{Of8I%8NtYAR>;L2 zuxQZ|QdSuijzk#nDOaVRxW}#Mt=1xu(Rv2iMmP_Sw@u+aq^3T##y%LDdkA z;S?1%+L;R2l9its&R1lmb|$v`OUlN^dH)6}w^v@BArp%3uM35}LxU57XSxDVxHouCx5~f}DP37AscR4if`|C`oAH>daac$noglcfHRKB^0 z(OFrgkDIZhJGQpCAA>z-l(t<{w9CSqbzt)oIoGFi7~OKewhb;Wwi)?aB4R}b9F{am zh|8f_LBo#Kdw5J-zE`B}?^|0L%f=}z*m;~Yc|&PyZK=9Shas|Ba(m8I=fsA(SdS`T zOM{nulr%gU8)a!Z%i32_{>AmdN*hM}lvY^2_mdDl%lGdU4pR&Z?B*AZ+egkp(ooT= zQv1zrySD2gQQ0}+50kM-U+H{IxbGgXqbMj}1~mIyo3WIE-pWK5D0x%|8`1#lVlbE? zTH^3iZdIbZ+H`eeYf$)dPozDBb4tzjwIj@%5uFJdS8{Uhn^7?ZeFF${J_=(HM#{&C zGA+Kl1p=OjeI8zDX#9%=fAdAS{W;&tDwF)a!73X;QgsAmV6!kX z^dj2FMOi$AQM3Dvun*^+9bI%_xKHTL`8hKN)*NlJ zrD7K>(km1K;^m#`-}VRo+8^az*=MK9d}?XFtQtWpIypf(`uVW$UTUmyG+?iV^U)qGbri1AiFPEnH8rybW3ro zNR1Mv58dRIMorm*mAXXfq)JUCD;~!(q|)bb@7k1Ae$5)1UYH=}4vE$2BaE`kInH^y z{3J<9*TNuzOzlkL_4VZq^podh7ID&a3cR)Z+WJlXtD=@{rlwk)_50*h0S#1C`!7`L zDxZ2n(`YaV1(tTEv9cyKwxQP{GdwXU*l%fh%_y8tegd*dMF0fF{{1C_D(4 zfSbnRuq1&pny>!2XQwwi80?-N2wtF8Q4xi~F~8u4 zx;NJ_9}=m3xFW$TqB%TX?mRqjH9BMF6coTTYG98VgDIijM=t`jZt=x9hzPkStp;lOFD#%q6DeytoWazBmB zB;7nZU`Rfq^P0SeLHh7RLbi`(?U>;6r~7WE4ho+UmTISOv@*efw<=@7z$!ft=~d9xN13Q5WNBjR zYEh(TKRN;)U-2_3H!QNH^|mzFc0Y)nqT)V#ClRW#+i<`@Hy8}1Sq>Aql9Bm06*2uX zn@>AG6O_expPjuyO&M9{2I&s826F{_N-p~E1d+6e#5b2vA!%S^q}>TUkl6xop<`sEq@wyA7#LXJW8U&;kHx6E(WxL<`%y%;7VNs4%W2Jh zq9A{N5=KVEWa;fr-S)}HrkZo$oG{v}vwFruHQaAE*#f0KyMnMKOL9?^lxxfvaYd*< zNlPQ0Zw=bWk&U8(`z9;ezE-yx)^`N?gaiLrDWHX^|R=km!Q&pA3{k!i@I*?x(j^`kfl9ImXt@>eNWJC*b zII`7eCsoCuaJsmWzJq=qedF8p)h)qNropV z+z`6Dg{uoA`8|W~bT}(y@&hdXK>}J@XHJn z=x_E#tAE_VBXWJhG!Ww!3Tu9>OE)1X{9Y2%KK}Oc;e2@K(m6-QBY{-tjt4S?TV4+D zXfTDnvAqq^p6u`3W4u z0_~VJ3_o+l%l7|Zh-e+oRe`NB^0%js#3}3@Yy>c}(V~3jZ?+F_D4%a2x#c^w;xbz` z4k>F5Lb)z!1{Axc{^@Z}9dVDmc&uwg+O; z`v(Q-Jlvd8P*BWle?p`Z-foM8k9n#q; z{2Q70D2?dbJojS9Y_1HkdEF+~*!Z~9?PhFp`GTBxuglk&xUV131Z$ZQ%dQ8=o-S7# zA5UB2t{w3WVC#1jl$6TdADvlQS@VjDI~R6JY0;+@Sb*}1J5=!$bNl(PEJYb zn=hZM8vT3>507AkiJ@O{wZTVaD)rw6mbXpDVL7yV!r=}*hIx+7iG|5v`Di-R+C%Iy zus2yax(~3CjI8mox5Z+8^xB$s;T)^E+7Nl>cUkN1;!DCtr{hq0%?8xx&Ia{!{qf7z!yEiE$N#ka-l$NOu8vyEO| z1A}vq=W7os^XbR4h8z7Z6l3X2$0wdc4*eW^gFQco@Ew@6cDAYdw<{jEo7nDmyVM#U zk9J^1_K5^6SD;o`SCy(B9iU;}m#V0FP#?2s(Z8&*Diq=&QBrRnBIil(N50tQ!vkK% zY0hQW4!-M!4$4^d_IkU0I3y(de^!_xt4-HS&?^y=AGCl&eUHZu2fP3{RY!5&T+0jx z#r_Vf{MSL4Z8)y*^sgq|tctN2g;t| zVG}Hj(97;h5U_y{5~H+~ud!d4C?=fNyn+(={;1Pe_VE`aHzczn1pcj6bwUDOit`>V z8XDS6sg`#K2!9+VQz#f1ojFp8GgT(o-mu8#SPf_FQ@gJx4B5+nw!H??Lrg;>r9uao zBUkW?e@@`&Y7$s?*s%HgV*>W+VIFtcgokhFAu|j@H6yYUtLdJemgZTr?4H;Pu=2(J zXeMmS!}pj1c$mJCzzkLOQMO|TPlECo{$*Kw8<|M^%uXKfLiF#@jxJE(`XIX~3#Y|%i|v{B^5k=$ zL}2nA5nKG%tlZql_;_(Ar@C?L886XHQZh2H+FGvHuU{t-=E}Z=HLij4VS!-~9%@Zm zDJyqpvQ=EQOIUDc8xA?!+uLjJ&Er|D_%SgtKPLY&qX)*7CX>qFc(JCgRPux=ph;{X z24E?x%$D`ZC-`Chdj??ZNZdyi!~Fdt=|Z@6o7uKs4n`T4bo6ZyJq~9AL0m)Tze$s4 z)z#Cp0)7w#=jP_-N|R|MO-)U9JgJHs0b`n=E-#Ie+`K&hNNSk~@bq85z6T)_1b!Zf zhW87tVj1^`Tr6YMFpkXt$%#5tbuuR7lamOydvmi=Ri;v5-RAPM{@z5s&+p%=lS+~X ztnmq(#9+sAx<8zXy+FzC5d7Z3d`l3LT;1H_S#)65yF+YdSW23%RHuqmI_51}q;jH< zS@M3EObibX7kpHp!xZ{UG=Oi>Xw^*x1@bF+u~355ozcvjo12je0~s%G?;iy-ys_10 z^Kw6aNLgF60+DWS}j;;Z2gg);;76?yHMr;N9QQX+LAQGKJLrxA2 zKsYc)Qq4|L5uN_^(I$cTl3-Q3PiZNMYE3A`hYtxmJIuU%9J9nd<>Map{&dIA%f_E1LNSS>OT16M*Q{Q?{1`!#Oa6+2zU<3astBZ#6zK z^dZ?F7#T&1l~Nr@B90-!ptSHTiL`N78$7u?HWAdba=H%Bi2X1e963wW#1eY0t3 z(#|f!la;mw$Mp~>0E3E;PU%Y;BVInhP7Fj$07pF3@J3G7iYic>lp^c2;mOg(Wo>u& zv(}TY<}ANu@6{KP@q*mNrOE&kV5jcX*w|48so5s_m14(&4a zlNeVw1}w=RknQZle>%icT0fTkROI6ZlB_H&ck^^L7L&Y1HpQM}`L}!}-DqdgJsTv_ ztpboR0_TjMmLT)ZDzdIg( z;$0_dfR~Ame}p)!;E|i)i#qcTe=eD>uo{;+m~99HIInPizRH9eDl)g(ZYi5k_!0R2 zU$0(W?Ag8Z{)^||;PP1WQC-~1>fJu4=|*dt&<8fdmb#g`O#8JC6N|5H^YcCWhXo3B z?HwIE4zqd}`wlAJSN7&d?Y+!2J(k3GTj1g9p7?A09aa}5Wo<(PItC7$(hwRw1I=p^Mm|EOlHW}8Bu`?p z3>a8gryT{81NSTB8XlLJ&KlZ@vpz%Rwzu$0ZWcr^-mfeyEFN6liutEMQ2B~dEzsf#B1Prg?^q#j?zezqQ_$`CP5h1U)BKf|p?KH(G&!58B+@F* z;`-}c#SS$)ak^}7bDlkl7W3PXZK`HrUEMcf{0}H@&YH%NOQBBZO9pG50U&*k16kr7 zt)2i?WPm`mxz3XxvAmpIyNiwfXyG>HQ0OFurpGzqag#u*^t*SwXCu(IM+?1_F>Qv@ z`ok%MS>h#@2g(wQucxasrtxP=t;kaul>L)YWgLHXh>P1E zj`_};H~C7sTvi$@Sx^7{0U{~=vfH&LJ{9KRuwH2}S7$>F;<&oo<>V!QZV_)PUxP>M^mO9Yp<1$o{hbjd z}5x!ZK)Ra&;KEgwY|a;{$r)| zAqpK{8CrUJgR>|jVKK4OCl{NY=^DKSZ4>cIRYvF%g4V`58&DQpsp8D}nP)zCLtr2w z#W!?+E|9fN$>(UWH4ogqO5l)w_xQMXqDB252mR&{KtG&K)Ev}sRuD$TM4#@g%h+t} z$fh{miElNsdDFwHk7~!XqyFs}91Iys<~coJlVP(sYPfN{KN+YPt}wYKG7#$+=8UR7 z{E*q$7@^a{${hdolK;Cb#@Tt0xYS^0e}C%D)g!)pTKTf$<*LK1#Wxajt`goEbmVB? zK)aH&lLu&avz4Izi1zcSNt0HazRsYcJ>o4T8B=H>>P^Y6SU`{kKX#D;ipO%J(ly>*c z?(Xc3BkXw7{iR_QVzR-v?k#Z{*6VM$fm9K)YzF6h!Q8wHqp;E0KB*cm!Q8OwCD2`X za*iqGCeuV-US2z+O^Fqm{xCbsB}_+0M-O!-H5HlcbEysXb`|7S9*RmZQYi+Lc{(x8L+-qxs3AgNQTD_bCLSGBv~ZFYHBtNIk^LMoIp+3 zfg4O+4o#lI*sBAH3A!rDkOoL`J^~iPHrHc=wXe;f>BkDYdHcJ|!=SLR!WDs#;%P*Ex@d^D)_V-TON{ zs?(F8xJr8<13&B>_mv7HGd%!A%hqtjRQ+wb!YiDIgL|*reX$#iV-<^t_Ks9FBJd0z z>NWK1J)-#r2d3*K2d17-qN-k&@^b01OfiST`Q7DbYySHKHZpQ@vv7%8JP!|#6_4$r zTE{P|Dq*|h%Ud8t)Y&m3-x(`Jrc3fLsm$r_On7$hf=1^3Bq&Jjd|`R`cET5>I{O=h zVCCWjs!3xQ z;KO245uDa=i@SKr9W>@$3Rg2<&oMt#t~SjFXvzQl4)A7R`>hMtus{A^T7XMr{_O;< z8VF=ar8x-_xv-#Kcle=GL|A`(BC|ZhEBxP%J|M^6_)QKK6&6LgHTj}!f0vbY!DnM& z?dugE!g8h?-whV@5Aw!Hsa9x8^d%QN`Ee#-FThN=QTXT4=_vDY0`c%W-7K#b+_Cu- zgeMzCFv;(aP=&>IUD)uFZ1z^GdI~OUk_9sU9+7->$wo1LszJjmQlDB9P-^?)`zvFe zn&wuFG5`O?+m#II)5)Zel=OW>j|D;lrT0$M{}_1gh03&PNVZQ@w`5#nb#gh?SYjh% z?a}I@-jU)7!pn$_cG?VH;x)UST40+3!kJ~yN3Cw?jCkqEkr;bapI-34Dv1_*=Mr0x zEJGARHA=glimh3l-Q2q2Gux4cgQX8jhE~>zEs4l<0ID6f>FyQav#Dx)#Be-Xu$P-} zK8Q(`6NRE=tb%ixrE>jnUk*3j@bBv5Vo>^6ir~%haYa7skRCH|xlU0~kox3>VWc2^ zcWEQaz6%tY*A;C*X%{g9mQqh4XSg1rsfZ7^HEhk(w$4enV#E$#nMr*NnF;xF57)Mhw(n;5T`R7RmrV?> zXXxCW2UvOmu`dHaUT(ZOSt0wlx}g77KDP#2f({j_&!adnZ;;`Rg;L4Ok2k_F5z6of zXVAQP`<5FLh>{TxaZCOp&a<&WVIavDO-n-3+AHpZ5=v1sx6fjK@EDAL&5pdpV$qcKbiRJl(b?I0`iBEI1k7eUsCy76^Vn@Iilg6{Y%4WBJ|q*q^L!|Ze5iKk zs9AExZ)Y#2xa{ zY)*MH1|?&2+>pIUxy;KT#p(B;z~4??R+Y(=F740Ra_Qw~-nFqcoH{+BdV3R`#l~X) z)h~+tB(m2=#faBO#fphJT0rFsY%=ub6U zu7{vjZL0P3;Iy%^!3Cy)(TnmKuI2Fp9p51v%xedz#2XYU6=yU8&{t=(oLN~p(Hj8l z^AsfH9yMJC!zqWI6oTr!?{BHOA`--+E4*jIF;eWaXyD~@rHk*IfhY!0NUZ_?T~lp3 z+iuZfmVXZlxB`DPWJnKMOYdlTe0)ZZn&Ssr?&UHv=Eut1*ije7&NpTEyA3(9fmx2e z@SknoutScfdlR%{`+%aLty$tZ2tE!GC70p9^@$vV(;~hzCifs5)?G}6Rk_$teYoCE zE}E&26+j{)Z^)Yg?`L`7iSptPNyTtNzK119ftFNy)>5=UHVkEm4wdh&q6U&=8t)x~yqmdj8MK zy1%c?urhC9F`qqNyl+7NaOkmUp*k$%Oahidjv02L)ny->&EqQf;o6sUM*#~yloGNs zb43m6e?_WQfjc`mfEFFj4C9AyU23|IgIv2Q{eAfC%;$^pL2Ie;I-XhUHMMjWDh7sd z7!lviehG9i5nsafa_atpqo8t^vV6x}GVDxjLTGLnlhx_F%HyOjrVWm&Rc11Yp~*WV zRS>N|qZo<`XB|^~x*OQD0?LWzw#e4d6K_O)*0Ns*i{2AYztZKVgzS#zB6BXe{ z9l(6b9cR%Wz!424P~lwt0!NxapE>%k(dm3xb%PI-5y9QXK)Eb3;++iL7$ht=HAjaczYN4pKJ)JQLVMi_pW*Zw+O;R3-0f*y#aTe);q@cf8k}P|#oh zSa7=F{KvMB;ZX#u^*=f~Dt6Yuw2@K-e$DM8Y10NI0vY~+;rlSleazSu`@bPtm1a8! zO|h_=DnfrU^qdsc?1vfyi;z>+nq!q@iyUos3-RMOoKiw?657lIuE=WUUKC+gvSrE` zS2?;XN2}tN%EI27xs$taL80mDnq8+!Ud}1m>Ek*!PBw<`jk|=;9<09JZtl5VVgZ{=@VI%4PyN^CU*`o^qC zkq%R%*?l86Dvd^>$+u8#;mcNYMQCzn%U0Ix>%E`9r5`=1c^anqdTeH>H{u2IukY`O zZ1ljs{ne#S#q^AG>mM9iAzZ2Y|nN^V)h?OR&`Po_( z4c7@*nIVSFeQRq=)YQ=gO_h3O_eezsAv7{JwqJ^Ku`^xB&va_*=5%-_E15dgHG)Q$*e(dbwY3*%-pi064I&<;U@FAjFg}-_iRGmHT?rlPX|Vbo0e9M8K>{$;(~;V zDhY{D)n;7^nkC*iFE1~nG|gW}%!8DMD|D)?DSP@8`|Rn#`T?XIJ4m}tfEh)=L%{Z^ z>e{jZaWX18#+sKtI(Wg>nd*X!ioHAENz?UIBqB00p3mhqp9>Xx5ZSOHJB!8Bb>sDt z%X?f_pU>Vf5Qtnwc%p+(B(BS+x4(Y5-`wAyOhl5u0+^JRHlIY`7*52uC%|Mp4C?r9yhg6H1eeT!`C=HSFO5JmoMc$vBo`UYwfnyKT+EL; zPjoY#K0RBkj)!cv^ zE_YXa(M)oCJJfdP)!eC-A{vti{SSblFF0duf4%n%(JVJ&4I5Yt`D{gg@E3Q$-Hzv& zZ-Fqm&&&NBpZDR2U$xu%Bm~5Z5Dcv3w1R@TI3-j^7fw@#AHu@S?#=ugy2PEjWAohU zhT>Pb2Ze+!v(*!^;PgWD0k#h_e;3p!&*xdf;jC?hg zAqeUZgxY&5J=KnECA>VT6p?RL7yBA4mZfVGnZ7sic$XbEep8XSwd>1ltax5~XkdDI z_zgm#+(}R^99fypQ=rSp5R%0}cl9eo8yMI~#c<#B zxu3D7zkdB9fY?XP%qa2RZlEL4eDY;fRKiT3jzu9N@)8|CDcjc2AbRumR$e_yRBEtm zE{t3~Ks9?U;;Ia%u&{TSe|BXWw-9kdFO_1X$8SpO()2G1S6p2$rZUD`%10Mb;S^`4 zw;F>FnbB2Y*v~_q=0m9`yYF8#eY2(+b;wo+M|P?XHX|*EdsdkG8JDhY33x=F*g6s9|#E5oD3FZ?Y=O8FPZ{Ooy56 zhJ;vxS{fg2XH~EdZrOJ(vxp&HsT{xpMXz5f7t{Kvsd2SOfOpdr!i1BG(lTSO`uckk~S z@wQXeG9g+E)k$KaB^evREr<}u^CiOEW_$9d0EfD*!A~X`niKDTTLTl}`LS-Wkkb5= z+)M;Nx{$9x2V=kgZ>tGDzHU|_CKeS9HK67=KKTn(=ASh$-oUf-k-SOu`9k{7%}df> zkLG|L?NxuijJVX}1$bIKzHVXlMgXhmq6@?&J@cfKGw-jD4Z3Rj%?OK!>fi@Jz4~=z z@5IK(MW&V@3#IqzqObB+r;SQj*s37%0*&6noKTo4Wrv9i&3VZbvl}-16DAv~RO$iU zQiB|P5G!WJAikcle_!~N?pOO4p@Gylq}RV!8RAO8!xjFNJioqAxj&^U1TTZn)#O+{ zI#Vf7Aet$T^%DzmEB$_%l_Wv*gS+e9>$^V5l9Sno;e&7_Pz-Hr`)s$|)7{N#wT2Lk z!{T*=B*pZ8v3-u2q-Zez_`R-2miGWjJ<`D-N%F#3`P-rgbkWePAkVk|N%~xBX@CaS`wh zD%Y0}x5Uiao&HqENMr>4^Y<5ZisGL>t#>=c#7>(}?ydhA_CX*`;7%45@{2pUGbltz z4DS{B2sCnV@S!gpT|q(jAl6WSVe)%ajEOm-QDFoOES%rcg_$IrEr88n#6s}Bee3Ar zjJ`5|GKr1X0qXs_y=Ys*&z>0>+DFaz<_(z1cn*vy8mhM!rb18dvU!tuXVyV@ZswcY z*sGhE{(gR`gy=w-mHZ-6C7MF5nhU9{Jb;l&%PU|FkCgW;vB2S93fZT{#AX5+R?p0v z3q?hY4Gvd(XLI`x-FwD7U3S&;O3{Gz#iUo7sah%8Y&&6!`BSs6@;7S!;N$vM4k>R2 z`X**H21XS7ja_^^KvEpSqV{DvJfy;sKw1{EW2;dacqpP6odO8S{Ct+w>)YsP=TIUk zvEr$bdb?EpGJ`5h!b6=obTkw}dws08PH(plljSgc7Y@|VY^(#20&*1N9;*i~uCAJY zwi%VNxAam^4OCkgk+q|rT)~s^@!flM?P0VCjPr5*_wr^6RqDbQ6J)dRKi9Ilnj`*d&GOGUpr8Ldyr#JU+BcuQHp6cW) zM14lwk$1``^E|#UCI`&B*EgDkXKOG4k|_rL+4&IJcs!d2&rVYtj&RWL+X+EP^ZI z&)2;_jl=3rzzCLRjTE?HN?IOHqJA95#FsK+foIW}2A^sB^YoMs6UlCta#GqPLZocCg z7M9RM)R1D3g6o%>+Q=j$U%2QWa3t;M_`0iWS3v=*{LGKoXu0`CJlHkO8RhATvoLXH zAPhHZY28(d9xdV9qDk@O{-;RgkraXdkx7Q{mR0so*-$8zDoL`ESYqDCPLb}@iANa# zj{1NiEQxSG3FbXctLH1|B6EYIg=@#l$?vnZR_H)l`0)$XkVv5wH3NIfN?J9;!?2L; zVZy)fnv44f%7Sp%lsdeJ13#Hh4l0JVKPCeCM194&KbAR=uN^G6qylwdncKa+K~GsC zsF-R1VEUB`oy}m#);=kyJ%TbwTwGkMD{vDi-CBXH5aojeybRD(Y`OjvyhTf4ElJbtA538@VU`kWld6?DBNlv zSSWJuw$_W?=l#t}Wd@*B1=>Sl$!bt$e!4U9SiEMenyx4)*bCMe0um#je%S03W?bG~ z8XC`<23(cG3n>tcJ3gH7sSHHgv9YnFs4tZYff+zW+cE6?P{04Q>BD}-q1T521ieF! z2Y`-@(2#+!Bn}1>LavH>{1EZCeAoryzQ3g4IPYPaZ`Kf?U=Z=%e7e6nd=n%7@oxL%Do`h2h|vAa`KMecOsJMH1PD_ zpISksNXp0v_vz`$$4~5=x_UdvZ`w0Nf|3YF^|6>V241*ZQBf*ySWU#%twDiPAbOGi z_U+|fk?7S7z%noQIZqz1gMezZr@udFW=0vjxzqEblk5Fq%}^)}U?IzsB9h+|^Am(@ z9Xv1n{egZ9EMgBU0=r!mWK9-Jt`mD~0%FGOcwpNFL5^@jGiVsKe;orAOUk+1Bv8Hb zKR?(wY0kERC$pZa1BnzapL6Ad2g3i}lzW=Er~5(O|M#6i+pUbbxjB-Aw#QksbrC`` z=2ohTOie{@1vC9y zI@iP+0;qQ&a7^)VCo2Zqk1o=eF#SDTzVUC-Cm&0_o6iV67l-QBL1 zXE!#WqWBVoN9|lTx_>#4@`#l9(X8E&9cYZ~?LSgdQYxjYE%Ea&*zcRY7=@Qkl<$AN zIhQwJ!`H2c&cB1dqUPtHD!^?k*7i2S#CP3;NJh@*v*6x>!kTi6JDsv7;`*c0BNxZ#QY|%y5t>Z{PcF991*!nM z3w4a{Py9;2(!aOomn66oLUYB(_puEJlk@R~^hV~Y-@FwB5^GR40Qn#HRW*4?52|N2hpK1H()xm5BcqS+Nc0-Nii*QgADN|-x zu$va%8cgi58RiQsu{f{mh9^CcJhGYq&zCIe;$WmCLfxw9?F_)1?yEa23bu+XhuP-tS}h3%DJK zZv;B-f#gu;!d?A&BT&0K-tFebaUQ$f6MbQq%;7_D+{wAV?N?~OJFT%kCzt$JGjp4T zj*V>dmij6;+_oIeB}FkXqttkk4hFG4@a)VQLL~0browrrbq>?yq+Dp8!1GxHA|yFDLCtqd{m&rW9Q>R9Y>*LvT=d_Idj6NMnl zcQ?lW;kV^=j2&St=t(p#M?PJzHZ?vjn*M*%6FCl=ywk%GZ7bUCBrdSy}F^y}T zmHzx9(At;nya3fT7=#3UH!>MWN|@`yl`Eax_Y2x%lJ99FQil722k3irXG?>^yR^0T zpNg||a{&MLxv+NX=Iza7GD2X+&yjuF0Vaw|T{@LpmG^a~aG#$(U%k^EAZ=)Ulj7~D zcgRD8WZ|@(1sLd`|91+rbo&353B${X$ld_&aph*Aj{d+^X2B~b2Hl3cP z1GU$Ce0zG%1xx5xuR;e&d`heEs)s%NMuTjI8mpRD*^U-1_e`YT%lv)x%6ChA>aptd zN2Q>qPQG+3(cnv5(3c0IQb5qxt^UTZyPQ%3%xYK>+g!VuKxuN+0EU;4kbiIh&;E7s zOO|#dOVoF2oz>dHXY~!6T#hjrMpR35;pW?LN||11^@R4NLWb zIVDg_4}hYAJ&IlK;-vrm&8DE|{Qox-lE@}wVfk}&bH$+9{1BYFXx-0PX}&m@a1)Y2 z13x(_3Sz`er4jl5RIyOz=*dyTvDGl&H5Y>)O8Zo$`qKgZgJn&A^$Gh^S0Xu9g!Q73 z>&+*zO3#d|G)3xSMKN56J;N{`=c~M?YT7`r)Kub3u`qkXQG<_wHM^joq&#hdJ-=P? zz6CmV(5^xs5FBi)jI_$N%e_k=B_Q$b3$tu-29x9V(0q^I$J0Zu7+as^>!+cSCm}QE z8ktQ@aY1d9P;>52{7;ofQz4y(%UGPapy>l*iD1&3&4o0Q`K7XN@KoivU3zydcqgV)xvT z&@ryMx;oeAM-L}(o4TjFedP-MVvJNqP>?#{Z^SQ_o=(d+XP04;Y0o6<<&N*_ZHxr$ z)C@zUT00IFgQAt&j9Zc2ao5eJNiApECZdjsk?(~_uGxIG>q(p6!F)|B=nY6$Dx7U} zs*o=S;1mYAIU$<<`zR3h6S*A7x<5ZUX$l$0DHtk8`SwZE+W;O8)g!2o zua5zc1$Ly?;n_jwhdsJ%l3<ULRukCvz z4O)7UK!jfHIM!Wl9|n~YyE{=_Xjr5ypgjg=Agi*Qw`uD`9;F!bs){czB-`Ewc%#g} zAqa(}PQCMuTvlZ0nvT9QHCU<63j<}FY|?yBJer3MTth5%ljaWp(`pl1X2r$fEIAo` z1OjX%uDRPq_j|&|({6mw0TT>*1=?TG>C{C#^~4nY4sgq1Yz-Knw{j( zZ2L-^_v!h$&hA(?&_|g5(=7mWHK)_-L{a&93xfg{j)U7|;^&zDZhN`9nM`vP7SwZf7qcq@Nj1(#Q=H|;ZAkTFJz@)#hk@AZ zwAwWPV02vGlU8V;Cg*{K8hQbJvgIJh^vO^WlX8VK!qd~zQ0N=bl}Bo1G|PKFzy>to zT#%u+w|954ii(>DIPJe5+)PR;V4Bj!*Le{Yog4X@Mr%1Tg2y_ohwsN#8rIFR^zh5l zy07bdT`wHho~L%-6Aa>w?}%<&QOY?^>FkW8ztr?@&$lBJ6HOG=z|=K1@&fgDk#W!9 zU?dP!Fl)8jyn%&Pn~Wjfv8}S8jO?{)2&*lS5%}@x`iKwo4HAm#R4JN8$HV~5{K-}I z-+%rg=7Z>A{B7RI{JM&eb|+xnpjid+1ptq4NlnDz*{hZEcXf3+I)XMQqaVj#O-)_8 zLlXbFT`w9aLP9gfjaXD}J*X36^G4AK%3haOVZ> zgf#_(BCDZ1-Je|~YX;ft-Y0zRb$gh5?`$4jwe`SR32wqk>yZG&jd zXyWwH_~`#WfPzyXcBM4DJ~ZO@zrmqbX+ryU;z(bVI0W1UrJ7erVv0G)1kchkRPh%Q@U%V}7)Gzj0t5Yj zrZZkCmO<^wOe-?{{2Yk!>J@%D9oY(y2?u(5rg>F*MumSOX}V^OclYpkQ%=IDM->_0 zm}b|P}b0R38G#)*ty5G^NsHjK> ziyAVhJsB*ZVhpX+AX}OX)o!+az3bxZ@@&F1`rm!&!xy7#2$C{he6Ht2@_EY!QGM$ zHW3s7UX8C=$64V{5YrrDmk=S!vn5|$tlx=+eDufbfAB3~IbNjR&5ID~Mauc=?nhos zac!I2iT+OBj-zW{|Ge|T#^Zr8uAp#4%~U%0a4lGL-3i+;sEf>)??<%m|H0Z@a8>n% z@7nyNK|(<3Zb|9xZYk;R?hZxi?gr`ZZV`~~?(Xh}Gx?u$#(6)$>(BvXyY|{^ubA_R z`@Uv&Gf$?jshq(}Q~!@15$E`f1cfIcnGi8B*x-!xH3805xiE=BhW^|F5?k0wiec{4 ze+vc^O2w`(7307W)D?_Q?Hp-o$s5V}g+#D2^2T~`vP0ZA3qmYD46EBC>;D+|+ld8j}!V+Q+;aV+b8qaQD=i z>V9RpJH<4T$)5?P;meH2yUj?^vVs!D+qDQm(No&@z^6-559ow1}rM{Za#Nsi+Da(oqWpQR!7E;{CW6nUqyYrN_+VXriNqh`Fh_a zaWomcW9O;=w{EXqrb^(kmxY)?2y*eRNIJOTENx|OF=QM`CpDJ59`ZCk_mIuNNc*F) z4{~JWv8nOl8d)0)C$kPptEuCKtVle54D1Y?LE{v}$&Pmygwduk47xV%uM9G5@cpVI zrwlr>!na2uTD0d-%SaV9XD3o7GR4-% z!K?^PgR@fERJQ5o4%t*VG&Bow&4X)?uZR02VLm^T+V${6FDbOI{%MTAsQwdu{wLg( zq1XQuh<$Skd4STB#huO^Na?p8_6X$r8J#xZ2)!|axC!RlMR282~-XKcWL1PbPg zK)bv>2LaU*6}9gBs7c$}5|SLdV>fo!5Sq%AQbp*~l}dc*{@_oi)wbukgY10N{i0eD z4~jpP4%?VgZ6XReUMhH^;(wBzy(@Q;-;;Z+KZd_T z!hkQpZ`#`PR$o&vVR(RjCP?aTO~tfZsYDh=$n2;N62`aLy&1r7{bEh`yh5-w4Ht# zk{G?2*)j6BD|*R2^#>hu{<-xl3iik2#=%B1$+u+2&ls8&2ML`*26gs}AS%w#wgr!6 zKYXh-7oxZdv*`v&%#i8hm^x?knxxK1Q|PPBQ_882&cAC=vE86FA0}Aui5UrKrLWa< z@C_Kw7;JM7O|HFT%&+PHK^_bpDMP-w!Z3Gt-SG6Z9{zMcWW?9+WJM_NFJwP&ya-$% z5?tCrJZGOw=$>GEE-9<-+?#qCI2-00*JYOEsb$uJ6x1UkIoEJsn}wH|5A zY;aM)2?;kDg;OfrZt|CpQYGY3V?%!TqaXF|iSGA-I@m&GjJb7r|9hxlyW)bhF=1g# zkH`B|L#lW*S@6?<9M5By55+xncC7H9<`Eqr{)uZ7;K zmTR=Vmp=;P zyoF&dJ?+opiqSYfcb9ul1=~K)Bf;w*h(NX)L>sA-(Mt8pJJZr$&K_yuKgqc@H@rSj zA)%>6aP@9@~}$gVX@{%!-c*DDJopLzb~-o8g%_-$4Siz9zQ^9eFmXJ( z`?A5^?MTAzZ~fSS_6bYQ%8ix)2gNjSUzJQU`*&-|FzTuXOvv zplxj)Z&5*a*2R5`k4^AQ@tb&OVFW{BNP2%Bi3Ifkg@%nIn);iK&Sh*xoY1@b>He~> zzrSkm<>9uYzW?#fv(IcYw3VP~Qzb0oM}r<*pvSK6uZ|}p*=@v@rC1`lHNIZp6L_irW1h+Rls_R$agn_W!Q!S~!e zJU-kC8f`Ecws^U&*abie5B8KIhXgX8 z5V<4r4Wh{wD4d9>XrHnj-9N`m2`z}A;%5qk&nvftPrF3(A_yH)NTT@?EX%q(bAGL| z4z|K|7My@5N*shofG5dPg58pWZb5z(tjk(}|Dda7E(u>{|AFrFJ`_l8{sOnf!~~jY zAPGmZ1dl&Ol^Pwmp>!jDH%8si8F1>l5c8pCuO~Wqe$Ls5z!1LI2+V08e)E5n3A+le@f)=^+ zRX?*;2^At5T%kCnq;SPNp;C~zw*(3hW?_UuJ|l|m}jkw9+KFAc^~hNB`d~a5e}K{C~vI73So9Q-u>d5p|j%r z2zO~YH|$5$SaMP908y)h#9zQdfP;YmT`-|YJxpEaYL*xm74;_=o$~JG(HWF!xZVk&LG<2$I>8ja4h_ zBN%1I$PoPBS(1uNYL1feOZig;AZ5i3Mw7v(4hLwhRIV`9H=k;ip3}jZrnkYU4kRp?`uybr_Z^o%pq_g10?He6Lhxpl! zQ`S%}xHz+CS>Sy>9WtE*wN9&Bv5;@*T}Qxh8ZiASB}|O=6Jq*;Yrc{UrT8z`A3#jp zoLlp)VnrtX;WH?s8peu;(+F)gt?$$X=l#gu#_2=OeT=ef`bsC3i+Do}g9$H-_6Z}R z-zc}aSpRY^dDsMa3gJ-@FqN!W6dHZ4SBI)&HehTubri#d=qbCu(PmM@nclnI|Gpkm z)X?%HW3_^Fm9)3+7&-f6C;rRj>zmRERG&a5o>HdS;IJ7k-0iGvhYiNX6fz20m^>j# z8&%|b7@2Vp68|KV>=v(g)N?xN7cL{=qWXRjv(bz*^iWt#RZyIrc2}u4K1C0=3m$#2 zS(k#6u%!JnB?mipIL0<6>K5*=%#qO_A6_Cm_11pCo|q0M=wf{__$7=kmnY_5ph})1 z2Hk@8c4aN%Q%;UO6mn)kX5!wn@J=$OZ*;hzG&jyA`uOb9CL^@2W&Z>24~;CML#NLa zIqM;(>{EOFlKkPSW%-WEYRn>3aOLFfvOGR3K3hgcU)AlmIXUS}n9V6C1^h}DbdT>K zMggl}?corXT^OkV*6^E`bmz-1wo!q}O~4EV;)x_#hEQxBlBIM@AJl-9l%y=5WO(~d z_9?&)7^TTk@>3>OerIg{=WMwnu^d>S(er)eTtbz`oVu5qhQ4`EblIfltD>5K#(+J> zpFbn#h4I<-N~kY#^of%dhjnfiJe=Mn@lR>3p~!4tthuOmkR!{}W5L>6ZbISHp)<8$ z0+z$-8U&oFchh)M(lmvoQ#q5YrMk^)J|alXV@6ELVK(Q>V;*Ll4hM{sF)`;?7k>vA z5%Uy-hb=^P%Lpu|_fBr#VrH#u&qRF9K8OhG>VmX>gy}xCWuauPfGcgjyA{>T{ z5eG^6o2p)&+S%^!&<3iwF!nKu{dV>BJAnLfbH_gom!foPc=589kHo?Q1;S_!-HQQLOR{9A%;9ziySKK}tTT0NPCuL)ani~_a zis@Dw7fde$Z*mw!RJl1L7i6z}XRMcG}%c9YX-%|umK1Y~h7TLZU+H!Mnd)n}} zvc+c_zIzl1d08*Ns3-+WaWc4aS&G;!)nZPMD>&_z=8?LguCAYFXZf<{PQbPaL)UPcTdSzLl|oho&eh| z0dDc^-3sH;v$eR7c%Q%Iz%{2l_SSZ81USReJWjb5gSw`{XvRVwQzEUp5CDP1l2-Hgn~+hc5Y5l11g*8AE%Q}Z6+Z_{E#>B6)r2WTlVt=~zYF9ZFE9WY+cCOhre{~My zuw>_K2qtpm8ql`{YKbI{b4k;{iO*>~?lAz^GR6A>`ZLs2eXa%~erpd_Y`C|}t_;BX z26Ef}`E{05^%|3z=Z@;?nA=s~)!=asG06on3n?Xd^GOuK4w_NwGGz}mY;KIaiP$CK z!Af;mmyqthQ{7VXeyDcCgv#So6!nXHigx!C91WA*U-=(`3)WD9N(=1uzdHDF;9yL{ zi+=RnNiG;IiT;ut*7|Tz0_r}fTC3pWblF*f(t}w;w9M4NfCQJ#isQ5in|Z~dL_-79$zJ9r%h5` z)4DjCh9Bsp#H6P~Pu)aw-JYmdb^ww_wP7#vZx;f7{*Hy<`)?IGbQZkOS<&EJ0H|37 zZf;_d+jm```=0)vjWc=Ickv=HLPi41V zkN>7)Je)FZqjadQl!5SlhC91+alpu-R>!)KT2?fMp80FXJ(Q(s`S7(!BlFLIX_RfwY3!$ z7r!H7U`PZH`P(d?A05+ZquRc}DQw~iH)F#^D;v_4auvS6(mS~ASI6pIlZz!N@q_>yUko*|2Cua*?s)5 zj!tIMLSYGT_y$evoQ7ZR8|Z_HQy@$4^iXU#>LsS!5N?bau>cc#NzlZE0^IR(5o6C(7$Jx;BaY4oQ<7xqeq#cbvp%vdE^p>W?9JZQpIY>$;7S21 zS_-G*PNQ|1sgTfL^4-O&^K*bysZJ(!01E!7T(MVt@(~-?Cd-l-x!6z4j%H*I{W2Gx zqZc#9uI?TSVp7&jEVFIh7l{v9&@9ioMm?=xx@j)v>aU#hebR(hi(>Z+>^dwP3PB=Q z+}M}|WLoGz2?oXp2!*04nUpImbP66ZDXEZHQVb};#%e&XyV%Y2j2hfNnyksMC@|RjZ>pve3aM+TfyP%m)`n>6O)uwKB^a!{5 z5l?^Rd6abC3Y9Tw{*+(d^?IJ&!~R;yl7s6+7EL}5eeT|3;hsi+zz?Q(W+E6zzOx*3 zHV!E0_&q7z>+6RnCu2dsg@cYw5cSu?!FWzDX9-67s!ATPf|2Yy*)M#LfKXO3VR6$; z9Tsv{L0 z=aueJLA$pw@Ux=n6;hL3UPbn8NiAHtp(U5{Q7e;owBKMLq9l@1=V0+)!#A66DO@4G zX4EE;g?jUa(!7m{jM;DdbR|kzSdIk|gluH*nE!2l?HSxYeYi9Ev zcbl$AsEK$Drih&f%ybkzlo!DuX*aQ>h0@4#n16& z8-HzU<|{Q)>s9^9Bwg-DJ|g&=>0oGOkV%4GoJD}3}r^h&F zoGHs&0P%23_y?%?KtU-7p9BqRmK05Xod*^fZ*9f%l$eCe?~$+t#ed>nR3wJ%=5%Eg zk)ic_W+p#qs=2yIxER;WgVv`_Oa!6n9Y4XJ-im#Gn;QkKq)}JhihN*FzdoGh=Ef5= zi`vcbOaN0FlhK{5h?sb75bn%@1uuX3@*hB=l#t;GN}MI1x%&jgzVlQnWcKw<>+>0lIVhz|9Ql*yOO@PnN9v$#EaJuP2Gb7h|(0iZ_)swi6F3ecOw_EB{Xho0O+mG4rG1` zzSGmw08whNvACdb%W9R$9?44i)O7x1{Cx*oO#anGSa2kCHz#?m4j519^)e}_aaEwn zm0_;+hnR1fZbnMZ;@Pb`=zix7W#QbkrE=8ku`qZ!{pvtTt;ACwa%2xTcAj{BsO$gi zt9NtEaA6AB(p>jzM^ey?rA2F?n3{fSli7?VlJ5b<<`{b{mr%}L3cf7;ZMdkkbL%uB z&~8L2GQ+i9al>SIW=DB^pZsBj2_rXJMbWYIGA1jiH8)d@%lHFjxM@bMs$x_|xX#lD zsH_#E9Yfm3UQ$|a1o)Xj>+I zyGaZ9Ujfmst)ZH9uj|K=8mq-vetHZsp99vFboE4F%`gwkPGvl7e$*i*kDAmTv3=adZCL9!4ls>CvDPbMtpgpRF>Tt7cd9rsa3}RMGg& zc^4^I<&D5LesG-TZ@Vdvs-JI=CjgzE$U)2V*yF5qDHui+twBzf)m?(1Zj)JfqC*jAQ`j&pW9EP z>{FALoQqg}jc}8Ii`XX!K0Yo=aT2%@VhbMsOCU3i*d9z|)*s4Q@_Lk|+LBd|pW`Ew zG#DlH?3Wqb(tW*ct?8<20TFQwsNm$;wh76Z;PCnUH-ZfReZ$YIU9&1|#rpvj9ewl2 z4%`_$yu3|-lXh=1%-*-D`6+?&A&30y%k%wS zj&bY7z+fx;E-y$C-*m!&mYEDBJ#+I}bLl{O2SnjANsK?HPZ{1G@i+Avcu%B8cVs;g zOE-u&K@fPV>N}g=9QH;41HNvBS`xUj1W+zR3?t;v03ujL4UL6dL>F~+>{(s@Ae9o; z$vo*qik_|AUEme4nBdc>2HF&00~Zn%g%-;>AHdc62Zk7FJw2j|_6LjDo6Yt0A7Gk- zgr42n-0TYZn3*$sSX5qaGtmz4dLuxM3bF>-*qa>X&p<1K&h8*oLIH-^kvWEN`<3R?OkfAo#x=%oa)B!l1_M&1*E$~--Jl_>D7&e z#Eq9$_r-qsb=AW%0G;#$)GZO1`iK+gwE++jG~g{~Km&e~3+Mtb0SRcDSEv4n^y2a| z9V?oskqtmMfSdY&;Pt%s^k}K6EBFJv-yyi&^<05q3Dn1TTX)|us6QZK(=fgsGP+-E zT|R83$WY|VByV&FL3P%(U5wmu|_s?!`Vb=h{iG+b+G&u|2PkI8N_MUGRA%Grb7RiQ|p5CN4LN^^d zWsN9iSnRwiC++S+8ktT@VB^UhPrE*>v`q7hiYf;;PXjTUJZv;s@Wy~egvpizSI12R z=ehDHN(Y2-Tv@q@5qofCYAr5ww(Xqhps5Q9AfqIFcWc5-OcSEI@~}YWO#p}ohNmmu zBBG*`Wty%oCr{x(|C8>Ih{?sx(*kx~d_Yp|7%^5AOK3ZsBqO&!!r)~)YT2!gZw6AzYp)vdovri6Ew#x>hQP|x`DfX zV5#X+V+up|#-w;4m&6+~{avgYmo?);@vyRqb*BN9gNqq*Y=%|J_)f0$HRUmqGn|_C5B$2|;1TzG*a0Obqs9!fc*q%~ZLLt&6mFgVe&mGbg6WpEc@9$UQbA6l}6ZMTaGGGNw18C2h) ze*DylL#W3i%JHK6{nJSUO8>!4nwv5ICRH6YA5c`Hl77aqK4ZPB z!~H3Ayt`#~EL=S=@o=Ry!Cor9c_{sNEDOr@==;u4hyz=w|3#@)_3bI?b1xzFTuY>m zl5P(-N%aTu*v-@Au7=B0)9t?jyk;g%lBm+k){ugEZV)e7x5qW^e*2Yc<)qmON+>uJ75)9P-d!lj`q!Yx6i* z<8xMh1~Vt~$6t>*-PjBU1f47F!}i;Q_JdWdbZ>dp<@tnb@nk$_FAY=v+Y(&jwBdNU zdH7BjWoFTk6ZTi}roaT?u|>6d>7o1srpePGKrctW4AVN>Fvw<>pk{dBhLm!-et$VV zHCuzr_2DEmR6aJjiV)z-xLkuNX=y9z_kS&14h#;)&dsrrk=6ECusVg`ojCDn%gCO8 zkc#m26WbKG`b*^}R9+;_m34!6dG?;9c32>LQ=CXR`)o{Th!t8cta%>+4JE4n1uRkE zk~bOj0h27<7ZZz6KJt9?%Qw}F*_fVi8q8w3Fv3N|IvKy^)f&D|$^bNYo-iLYrH z3#i!!fj@?Y`p@OH*{BoAg15koIJ3XUXr~;8=_*jdl6(+%ROvYJ&8agcIwuprh^58u3CO{`* z32Uu(2%$->&()N!HR#y|KzRTg-9J*$hzCMs`>nnYXz1wa_)@Xkm}G`E>OsNgKZU+_ z48}J!B#ey-3=CF}A4=onWqDSqS(6!WuNPgD5OL({EcO{O36W+ie2&NK?{l{MHOxRm z6AG$m{(Rnm_>K`+I57WXSW8lEO8`SE(!)CE9FBa)zp0>%UiP7a}7DtTT>6)~=8 z?}y*5LbAK7qSq(kLS+<|gEM0InkzT-|5j1MWQz35%cAP)^08N~Muv@c0z#1~RDjGARozkacqRYnPA z{b68CiYc}zwg^KGt91W}v2_WAUxsX{?)&7oY3Sc{D{<@o5thy7J?`p;`3jgCF_BM0 zx8P$W8^+`DA_&+$A|tJ+7LBhFq6_y&cZG8|^2D{AU<-ZM5_;Ig{Y3lvWAhU{+KHZ* z?=AMFJqTcHf8xX@lFB;o+e!pVGY_rc3HESO*k;ucz}~;PXlY&Bu_F z<(A8h&=dD6n|>i7p=4%L`rb&w0ssVgivz8IWb&`j>_Mb4lZs&#*7>=)8+w68+21bV zpY03Y1-|ybvYF`^bix}G(wFo5dNgf8@&yFx%##HjlBX+e^G%Mh6|EPa0W!(O#mvRD zW=gMoq-&5eyR!zZ@1q4zU1!pOJSP=cqGxx9Sc&TF%2vx)%G!=GKLml6HT+PTfWrW< z0lMV31Lf%}Y^ zq~9-FGG-DlF*7Bo3@S2eLdatN3vX*yAu}#pXXaX(_gkEkn~5PCuQRoM(qa);DpEFM zA#ggF3I@-`Se`UyoYdFav@%fUeR{g22nh|HZ?GlwygkXPtVGQjS96UWi}C;UWcxYs zpIu;+bZV*$g>X!+n73;T=)|M`asFxBUGPJjRO1~`pNQpjuC8WUmhX#Z@_EA{B09RL zAv@nN&`O4n;;;am46hSWn?0BPCM@Wzs&|l}e&<954&B*> z3dx`z>;7=8VTvVt^f#I0)D+g)6bv^J({0}6lpZMEBn>0kbbi5kUn^@i%1avX;ltkH zZ#ZaAMYDGT`G1UPEvIh8ggK?GpJB4A||m&~T6L&|hHj$+Na zmZ0E|eHfh5;BGTphLSU!kYQ0A z4iB#vHC>t6nS0(WE@LX)o>LZb$Oy0h-HcA)=K&l(TmfTuY9^w--Zs@sdsC(+v~XJ< zm2K?0x=y6hvhx=hGjGhoUnpZAk6y7@0i5S*>YokT7f;eJU76mVyGlW7diN-naUq@33)y+ZA!H!XQDYUx|%(#52me1%PaykV>?Y~6-_|K-{ePzof zatCtBw19d4IE?kTv<^auinBw!4+&k=EI0<@5xG3fKU+Iqif>&${rrKR%9eGGLbMTp zi{qnbNMef=MqK+m+a6W{H1KS73A#c-2qS=oBNVJE))SN58(<7)U>}r zHyrtcz8Py-j%^?ji-?^99svOt6Vt`P@>WKsu8gShNBwneg(9triY#Uzh$xQ;{bRru zn;NZJkwR3`BjmC`y@kNJZ16XgSgJFF=VyiBF}nEI6(5V@%Y!;ucXOtX#zk%o*uSI> zo1gVIPg>D4=Qv_~pQ5wl;()BiiqURwkl{;Q{J8T(Z1~}7QP_I%3x068nXpq}`_lE_ zf+vnwmnF}#?91TpNx3Zt>}T$O*>U}S9LYk)^LdX$b5?8@j@49s{UmeZ;)!Ony!OzY z(OVAC_V$nvoc;agaq%iT7i~ZkJv8K1U4wXb4o}Kcs&ol`Aolmkdk+I!%-9_}xj+th zs2T2cgk_QTDPz9iSnMsATWkBeb~?X>a@@ROGTWji3!}<$>5W@Rfn@5`pE(3{X5^-t z>+k-BTkMz}OP4E-AI2w+2Lw9al~hQqG=4DL_MgQ{9TpQ*MDt9f%ORl&rI+Us?3%vX*nLXV-A^l;IRoTC2>{)_Qi0(%q#L zszuyuz?M@{oL!m=7dkH^@}1v%vB9~&NE|Aw$<52DwZ9ynoxK;M|E6Cz zrI0gP(0T<0D((a?>2OCp|21WwW5?zsF7R**cg34MK{mR~SVhq3%8gVk_H)`nWCp>t z3JXP1f5f8sOg%c(t)(3k8_H)ip?zjUv9aS^Gc>=qLirUg`l)rw%d zPFkG>i0o|c&E3Vb{>o*4#jl={?8!2+3z480@2HHm{=x^M7}AEusOpWsm*s{C$ze%R zM9pHMNl7s8!r=--{2XDW73 z$kYoLQ(|{cdqRlk{>;GO!Sl>gMJgUuw~ZNsOh|D%Mod=nu`Uv&9iIzzrGtMR5+>Vh z%WHXf`)H|$&K=o0+KubGB5{GV6GwXXU#gPwb+3zP`uhj@YGR#~}H zqc!rf22+jaTKs!l?^^4qwDpMJWxW_VeB}EoJTAHt*2I-+=zF{~O@4Z@`VmoKKl`f( z)Z%eIJ#$HYSO#6q&CcOrD4AqjV&dkhZpVUKW|A5g|AX1G;kPuz#@t+}bh+7VO!yDh zUy;cY`{B-Jn4aS?=POS%i$gC(C(OkoY>GcUxP9=WUJUPR0); z%N3Bau?6tovUr6g4UQMa?MWlNy*0n>46l=K1+56(pYv#MW+YQbQyI^*?OC0DM?9q2*;h)<1s_*@VcD+ z>EeXtZkHMBE}Ly94IEfBlm+Jw(AkA1dJB1dmu+Uf5$WUsm>Ddp1Np`Ua%qb$Dk{?N z3dr1Bs6&j7{@}8B+5}=bAVZ`CZvildK)f7sPzTupY9A(S&_8u{;YcJ$8)e?u+>{vY z``w!TatS@t3;KPBFC74!5P@$wr)B=P2c+jp%33mBUad94=9PDi91#T47GpfYe_thH zxG@gx?!@5g;X(G;g&Zu4P^#5b zeR7U#+q$pF+9}kP-Zl;~S9gd~Z@X&J;O%=h&LlNBkaTt??6SDq<8-9fLh{wXAbU~Q zmYIpEZ*&yc=Ltr|X#ZPC6WMV8H>PRx&!ncx<2n&+4Hh?;Z@x~v%)kZ4h2@*olrpt@ zXbARImhS#&mL;%hHds(3lik|Ig^J$%@y7aSxdl|jfY(7V5O2_;?&2P^WX!{ZNl(Bx z5U*>cwXMzFeW0VizkhqXsN$=(_VSiD-^-ohF7HZ%EdwsQjn##Zf5QGLI}QY?c!br( zFp#ErwL5$~WPj>D?y}9%?P!->SdDjccOM4}d4NQA9xM|Q3%gvW3@qB9@qxw9&krgD z_cW5h-Irkm%Pn8F{^8zP9J6~}U$f6y)l-~aTA}NKp6fxz;Ly-k1x#rSQQ9at%zjgv zJNHSGljck04Fkh8e8rzgfYf}9j^^gt?|Jle>-P)@3UR1Rq{1O8e4M`wu*X=!EeI?(Y19a(0poYtZVbXpLbMT3;t|xB`6XR@Zax^+4S}^itd2g`*1=P(G4~JHkCMGB5 z+B`8EBC-0xU@!t4K6eJwILH(fF^QSI!1u~}kEQxHW7F2#EQ$fl|7LIadU63}%hu{n z!|IFP&Y~j~Jw52l`|-7Te89&3p;1I#DNnM{(qN4~AP-pHBncNrcQk3-75yW&d4&S*5a^1;#sbL8r@Xv87xQ_i zPi3moU?D&-TJ;S2YZwZUWhAO7D~qVRyT{%#yN=-TJ?xQ2;PHltk>)E|0HFjJaNyzW zI;OvpNrIYmriJ(9<3=6Dn@L-@TwQqLeA8vQ;ypGPwd;saKvy*?Q}_G4!@e;iLxF3B zjuyWgBN$?99F2seuYelmeCBH3YPmif@EKuj84_vEZZEr5JGRS2Y-;8Ui;C=<1jRvN z3k4BDNZRF8BxDC&dq(|Uyw2L0cymvGe*6l-)etgHK!I;dq%bO`MeDq(_JxI31dnwk;5r{Vf|*nuBVF zKV>_pqaKM<#5Or8CDWj-x*s~+G)O-ULwlkHn=*4OVt!3)_0!{bXwq65Vr4#0Z=g5z zZRuXss-%o|V}rE6K=6AGzAV1)-*&KvTn-76Lht+_vN8W&#L=Hke3%mzSu}Ac}FT?UvZuS4~WgvtBdtXS} zp|s%9#I%oqk03%DH6fDI5v=657oL6UUdx^lO*8Z(4eGAJ4%$I@ds&B-e|&1CIml7Q z4(ja)ziv{)-B6B{>;O^0|7Ra3IMAIh-R{#*BH^ZoBqLw`_fdCi=lq|eE=vENXS`ntLx)8!~F3{#@z3cA3bd40HH5y0!5+&{43gYG0Sis=h@bBF+Ci3yR# z|8xi6eg%Dq|9KHAOC7Yi{-@(8X!Z%T`TqB|DgL1S`hVW_|L>;w17i18_)k8?4+FMk ze8aYjk%}fQSDG>^@ z#7N0RyqD+LOmi@0R?yK3cv0NW6^(D9iDOXuNZwUctq+w7KPXr{}f9lo@!-Oy8i@(VlwXiD*sfhy<^gl?@) zrxV@{&uUt`5u2LtNAWwa0a~^=Nlf3i2UvpO2$Q4YjnEFA8n9ZVHC+3-?9D4(+Cc>& zW%V=ZlY(QgH=>K9O;b%CouSAT(h#ppaBeAiTuV*4Lc0YUk96;t1kCW#&-K=CSEb&S zH0zPRh8=29lr>*#UyT2aE|133XW}gC4mIz(`_MLtIGE1;t4Sw)`4%AT2ZwMaUT{sE zJ?h=sP?5bqZsPkl{p-&p=RPP_P4{NQRL6*q-z#Q9|Hug^-JNF)u53+b#>d^eqB1#^ z^SV&L4NH+_ARDels#lyZMq#>B$+>Sg&}_Qh<)z+PA6U6)Q*hQGd4=45wg?6Lbu(En)f6-C<-k7Bb7dQK!igwJ4>{k&=1$ zd*QylC95yf<2L^L=rafg6t2EF&JM?Y#jM=h!;a_#A35xm)?d*H;W8e3 zvM#BqpUQdMIBI6wMxT$BoqG@2y}uowPsrKx*fXE5lBUMXr;SjVlzrHg@ z!|6)6Wxf11&}ggaTkAY}Ao;p!T7SeZTBIT_)1iqy&d_`kuwwZ~_(|9Y@v_4w!}1iR z<4*g>LSSW~S@uZ#+|Ab=zDvjw-KFk`kXOs2G{LT;m+EU=4r_)uU-y%8cGzOyg$kK7uIjf?6ruiH5r!+yPOZVB&ez4T`!&OmwG8{6lxPQ(nN_Pv`CMv5k89^kE5WxZ`$4PED(TlEEx5U5!3!+ z%s#v2HmB`i=-jI-C?tZ{n0mgN_P~gX4Rw%lBTh%C5)f%6@?o9_EO*#b?bYX z89NOhuUxPANkL|ZAI-A~oF5GuYX-7D>1`^*JeECk7gwQWW_9jmPg~nQ-;V6e$x`ph zwX~aq0*XzF@0qV_NAb-PTP+fQ`Afo+$2Tsiy=xeefTzhJAaC67Z&Oz+h(q)$Zi3P2MD(ZKbBJXS$+GbG#;` z>&1QZ5)58J9KY+6Xp=3*7r{E%X07D9?efiW9mu=gZ0Dz!-bh*>CkXr#w?s0;xq+9h zP4m#XIPazdUwlY{#?TY!#ru7E$T@_$b|;uqo6(9E!MKb1nyu^GIaQ+RQ+8ogt*F9t zryv}k*K48!8??ldK7`DIIu;How9n*u3 zJK4x1?sE59P&j$IGx1Ts$A9qTGhCd=a?@MigN=1#ibG*Fg4Uc56-vs$b^+#_y?)>?U-czB`th6vTi!s|R zl5Zw1uQ5y8Qa=^s(Dt~@lQpw^4^^kvFZPEK3eP9@C)QVrcfd~igv^fOmC$g~7~Jy{ zR~Nlp!Yz!v;V58N_XuF>flk{WzJKBV$m5=v1FOh!~PEGBv; z${GJ^ct0fWW_pn~cvLzxU0S_79B@p?BtqKd;KUgzFHONs?>8R9&&8 zvUSmcy?o#$h(mo6$8Zk%ET1#0s&l9 z335KyuLyDJ1g8CmxQ#b;l0*3I>AN^**YM#^(GIZR^J!l1b_Fb6y|^BhxMNB_l^HzR zGqdYxKaNnub2o)E351+TW(F--ThA}fyL1QCFLSMU*?1qzgH*@cE8Wcf!3 zyaAEg@7k%hCC`(~XF)gzNGN>M_DjvdZH>U1!)4p$n%ntZaoSaO)C5I=N3Negl}fS_ zAWR%Kvu5>%em81SU!01{@N~c^3l)(K4beqi98o>ct0Ln@xWfY0Z3Bl?0$rd16DI`8 zb#{IC-}ffUBsxW&yo!a~xGM#eUJI6qI4bY64eKhas_v)gbHR^aWb`oK@wR;zA}^t5 z^}1Hj=U5F9cf$LBAau>H&t;}Z(Ws~lSR#(K_UO8mK|^3)MDhjCc9@F6RkQR9Zgy@g z9_iw1d%!{{(#*|yHIr0T2au}-g}>$+s!y~S(bDQ=T)A2+V`P)>R91ZnOjdWu;wj|U zd*`59KiuroFRiyVpw6M_`8j~j-0uVG^6V2iKl!RhyVr z8QjP_@qv1m>mSWhJFAMGGKd9YP|DJjjwiM#m-AE~C1aVfm_AgX9X-EacQUx zvU7+bq69;~hfjXFFNI84e*CUa&j~@ccF|}t=S5eB_<>K1RMi3rHBw962*jB@|6RbKGj@DC=`LXYi>tNZ3J>4o*h#B9b1kiE0amr+3s|TTGqmnh z6VUR0k|(OG`w`EEOqjz8vGsik9{Mtw^laX-`?uIetfwr*%6gRwknH#Y~@3Ccv7vyjlSOoVooyzA0kKAoZQ@H)K`(1iHSBk#on8-5lG*J ziI>P>o3kS9))REr2+5A;|1a*|u}QOFi5Bi@+qP}n_Oxx=wr$(yv~An&o~Ny8o43y! z=foTLFWmgRbE7IMcV*UEE32r0zz&@RQ>it7gyTs>VFIw?v51Mk*@w#?^+7c4Qk*6t z)#vg_7D^F=#9?aHVm(ZOS)l+P&EFWZJ8zhhl7i>Y_WOF>{TDf8irjM;va!oh{~$hq z&g$|Rop#7x`e24@=}?$f|FP+KyMrYFUArdFd!E|hK6}Skj5Bt7;{<8E3sc%q1TSyH z9-nh~IQ(PUWoD30tq*`kqb?8*y`xN>qpn0ErhlQr7x4Qy%b!E(j$QpMp=IK#h!(65 zpO5&uyGWA1PMYr?-di#QROln>dE1V?K+iS+!A~1ztX3&m3Xfbzn zXBQTUH|#*pLGKG9T4aN}JyKD#I6&BqJ8u~ImWbi+{R`92H%0KI#~Z9xbUx`ernpBIc=D zCe!Pj>_4TRnG_ZbGILC|;!eT&KN#le{A;HpJ;NVea+Z`jzU&E4JhO^9S@4?`Jt*}d z!?fZ`;!uY(4KC`iC}T9AMSfiOe+BcCq%MZy9JLfsGtiR(J0fqTRqTPm$U!I)Ok(2T zY7-*T8lPEr5m9`rGJa}uhR6C<@=4KXx(2dfviM!A@cFX?iX)|}pt>z3lJ2Z9Y^<+VcXsdq|R zIjy*4uIDRVJvS>q5FO!zLP|OU0j3|E0fVV&$=EQ9Q;n^S{d1UUG-v41fuv~jG#CZ0 zHhPVL^oslYTJ#HKI0q{}iX>IT-`;g`0wyf-5jXnJ0ciWq7Yv+B&Lkl*@!cmOeTzRG z?SBmXKD&KlP$9nR- zUw@%%9womO?Rk_`dBnJTe1?Y~n+YYq4^bFc{n*_W)=Rhkhb^r>N(cZZ9P?8&CnCa6 zvU+{6_4i9fOGs@f zQaa@@5wke7H4e+|T7YC4(xd$0UOf6_Mf|+h=3oLgjALR%qSnx;?6Lt;qCbeyKGhF? z?fKPual4f0>rY3E3H(>TYt#H;vJ3dDyuKI9z-w^-fi_5Q7!yZ9L+z75GRpL%pA z27AEng$<-MRH28g7zd!E~v|Fn(s(ah)8~ z)6H`(t3{B#;e84`rFJKToBw#3K{4FGQD0c0VMpib^Shge9xqf|X0Bq36O8QoaqRe| zC7F!R+I_KK;LInuk+KCmC7-|8QWah!`OSStdy9u)XKGW3d?qkTd9HNDKURP@O(D>lA4T$> zy*~qu;`WYU@>RtTKVz+s_)h%_tnI|=^Z6BjEUEAjK$$F8db66 z2)n8SCCw1dK(q>yoW(A(a|?k8LU?t{M2oY9>Uo4LIdfO+_D+Ag*a4!&Q5UNf%VRVg z)YDFo7u%P|7RcG=g^P_&A9gN)_m$Fa0%l8DSC1EMb|h0M-?y>ugQ%K{cJUKZFn&rMyb+`2p$3|}=2l{`$@Y}Pf={?cg_Ey`P1;>E zvj18vVPpEais9Q`5dkTidqKx&&}-_Gs^m2~$=%HDFoVUh}$sTLq?r1F1|VrNx>VTfVucG#I#k zmoP!gEf2~+;Q{PuU&{u~BRF|lGa<>t=^a-VeLuIRw<}z=D(+Z!=B?*bJ{x;-l5>W? z(ItPX0!VO`7L$^;={l}F!NC)$irPC==?9z1G5FYY+Ri6g(-Nd0#rWMepFUV#E6zTh z@zio$qc?Vjrjh+^IPbGiPf9MtVM(qKyR>N{u$x{vJsAlIZsN;cDO)J#EZ2&g3OS^~ z^>{|A105?j9}WB=P>Qw>Mc)aftT5f&al+MS#Wb(Ha@_9Fn&x^!oOPYOix!Z<&03UP zf!v?yMU=a&a`s=zYvDuo4$k4pC~Kd+ldjG&-_sF0BL&81T!yRs5p{djU-~c9@k>wp zM$M6`r<@yh>T1ptlg1!WNu*;>9!-1~R{WNco@SN91a_h*DvQ2HjJt2?gCM-oOeZYY zDu^rZ`lgFHcCnvq^`qww?^j;%AAc5hV#jjVavdXEAZ&mN@#`f+Sq0;R`GfOspt9HB z)ty+F zm2&I%wAKm%tbW-{7UN#LC@~C7WSgO&W@TnC+suoEY=N3z+^5^aNJQ_WMOz=%mU-ur zxjBDRco6QeYeXTCxnxn11&O6B_e$pDGnh)?Q3uK!{jQT`)huASWaUf(mQl;h!gy%d zn8bR;@c>rOzuqODhwa;x1Z)!rCnkRV@FhM^IMSv}!%pbIz+_P+Nvt(ehyDuHHT>ou zP5ZZ&@Tf~$MyXMaWre}|?IL9a9Lj4MryN2S>|3g*;p>GEwzWNS&F2M}8+*%NH-q;y z^41jZk7g@oG#Zmp$?JvcW19(QCAU@oHN-(H%3zI#jKSp^^C_c9NjRqkx1qqjR~@+b z1~~1TK%m-{CJ&~~R%Gk2^Ie!a%q0dg$~C3~m@MD9@cO=5L;7oP48X#&kv0wkMr zQKTB%J%13@N6dYA-}0a76bQZTm~Q?kp8Getv7b9%7l11~tehjzK=4H-VJxXzUBc50 z+0hH5J81rJWtLW!au$8P1c|GqLp-U+wNI~p)7&(^*;y#mdL zc>}IXB@9+gq2E**nDqqjk+)X8{swmwlebXS4GGHqZqbE7eRclDUhh73T_^Xq$B&|} zTV2RV6|s)btJn>nDyaT?_-Touuwos`cq;`MHinM=hWhE+D~1>Zhj7rAPkwk!=sJt2 z-GdJj3Rwo#>0jJ#%e@e?3(A(e2l0FfZYQi&xDmDPJxch=*iOUBJ$}#;m}1v~;6qg| zMpAm7M$0GnU<)+o>S;`GDzT$QHALPRa#Rl4!VhUor4?qd18&Qas)}$k_B<|%>j8qs zv|sF!-vh6HP#DqpJf;)RyGRv_M{L~$O5lo;zQzZPR zXqeH3>3R5#FCEEd4QPnsQIKoO4!g-7@PUN(0Qx-F(?f3^f7zTxG%z#Wp#AmOF-klVer<@@s~hHW zF?tT{3+!iSo$%jGJkHZ(AeIWj3ucSWDYOw+H;sOoc(!#v8dR-N5G$8|(UXM|4UdP) z!-Buw7iSiVjaAosIwJbqbp%74tz*vBu#PQ_ih<)f;swj0t{`bmqh9;oid@2}PHnQ* zgF>izc{<^%cL<31^+90s)^SvAVmL*JFxA`a$FROF=Ic;#50>mcByqB5ZY{2rF7I!K ztz9YWzW5NDaxWDtz4Tsm?uR!xZXtXAKslRx_qS>(#Jo0Ant1JpZ2!eQV=!KqSLHrh zs?GUYuY>D~=F5&h_|Y7qbWRZq59IqjMCA#b3d!4t0bvJDNJ|P{CU#L^Jf}u!tXL*K zd3s+7FT~mO56A~pSx`(P)jDW5R`+2Hp{JKisQxl!+`PTPA3-k#H;|mkyYcZwSM;zd z@`~iGC{4uZ`(ZY6QE;}z$F|`5?N}-F`FVD~;N7`B<`TH+z~6YT#l69n6*ak2Jpe5Q zD_h1_LE6~up`cWB0J~2Q0ORlf_Wr(feZ@5EVmrI%gIodCJ>5cbv7l!)d5j~HlwW`M zFK>WW%^Tc)%(Ak(FDJId6*OSv^wF@BRH_3Xvp;Q{ady=CNe$Kolsr$Q_^8GE7dh@t~W+i|Zt#W4ES-)3z zH;5Te=HH|p5u(hZou1 zH!F??0-CW$3T8|C}5ZL14*~=GS;4!7EYH>ZCvXBsGQ7TS=^4g)p`2hI-~~>qMz$ z@R5ZB>)l(3h6n4s7i>?9@s^Gk(>}1Xz$HsF^!6$Z6Pq;%YiOBu76laD9O6+*)7Nl#nY*W%a2+< zU*WUTyWihrsxhS?@za^QIh@>|bCSeYF8VYp7B`Vh`WTlJ zIG(;!>N2TLO8;jG>O!jE7f-qOpzX-*Q!i+`jwVl^9Gu42dS`2aDNrRDyQj38l=(+Ed;yok9ImY{clwP> z?u>IhH!;iv<2#{xbJgrUbvK3x>!K<0-+e|IZOg^oXVocV?+#ypJUJQX)izI9SSSlQ zdltYRp&`8(D_Y^k#K8h{$aooiUo_&2D+*?nKs#o$((d(?%)G~~JM@nu;tk3c(_ zZVgUKiru7wUn4h(4b&pZ&TFZIbQsTbP7H^XXrwy(4e>#vXEJiOX>0+S+{@9|GQ zXr$D#@=H_Yz3fUGni5pluDNbI9P5VMe?_N?(=hykM{r^sVQ zw_yr&KAt&N2;OQG#+OJtd!d=@RyMDYi=8uCZKxSO;~-A{^2vFh8%t8TznWu<*zWPr z*h7e1_I(W+PDaH%=Ea_Cxd6QbmY&)lgJPuUG=5sHgMKT_RzKBWIqD}u5lP}Tm2ax z7AwW``M52SB)3fcyvhA2U|c8aGYQXTim``R7Slb~&h-q3e3mKg#m>E7r=xR<8{T)|O=$$jyWyphXxH1R#Wx4av=w(5xxc46oI~z1 z|1J%y6M!2RUMk26tlKnnNQn=c1^IC_rhxj9V2<)v9ggQ(*N;3)Ie0b*$LKT6Z z74ib?7gt96=)NISXt; zUPB+NV-Y>T&GCmxz9Rs)`W`R;1jak2ggU~ zDGid>gKl_1Y*)X3;Zf6%vaU_GbGY%NEBp(JgCh&5)1@5b@$lRoc{f=}(kk}I<#u3d z{V*-K?%1Lqxayb#aNF{E!O=!rTg&z~)QM&^p`NooppYyh!Q^K*uxOeF6eQqWpw!HlqJLE3i%JQ+BT;o zv(dJgo;966n!O0$Qu)FkUxMIl)9N(bPUs7u0^8orbb6dI3^zT7X_C*AB#DyAuX^N~ zMX_pVX9CtWhOJsWTn{1XrDP0DCKOspm{=x41b{V4==}B`NS-+C1G5o^5{l(vR9q!Z zLBv0jYa;k}KnR%vd7G?CBgIW((8o_e%VREQSjmTMgw1TBWS|1#ks_9X#W|qQF}=O? z%LA{G5VmRNk&kMV#;jt+o7@T#^Q_=5Izh#>TE$|;l;_(QMPjo1AeXq?BtU}kQj3N6 z@+#U~z_;PM#h(kokoKR!&vA*?`*uKJvR#FP-ie0Uvm*z&ed~!z8vZm&Mb#^AB|%GxqCPDD9S; zWZN2DYA9y{g7g4*;_r?}148*HM6%u`nk$WihcM&^XbMTP$|`>g8h}r-E&!B?7-G`! zs0|lljw?L;St^n*s6Hf5H$XR zk#`-q8-;2=uN8eq>iNx0)0G;^7!qP>pw$T{9#8X#VB;enD#o^4V*%!FFTm?ZQy`Z3 zK&-;&6;r+}tXD}+O$G+;+2Mz|Z2=b6WjhwVO6H)kg4luMvfjGXVwVEPWsgPbRS`3{!eM8lacxy{k~I*XX=8 zt&m&#lX2*Wn}#A`v^scI(d<=oUmzlQvW^~Wet*;>q4C+d|O%hio>f1D@d zsVS~j#IS8;N^}bfbbpu8ZVcyh{v5DXg8x!#EZ-n+MkCcP^SsC8s{p$>oe@VAx@&MS zxP?VqJ&?UNFmFXyFkuuY7EPVeYK+x=0XzI>_Y>fB1r*jvYP-mCo}cq@n)itDNcJ_# z=0O4Uio+_D2|MGjW|HA(B350L6bxktu{)M)@T&#JB)lB`hMyC5^TosDp@Gr36F$7r zLBKRHlElo=aqWD)!sw*+in3y4H; z72$pfEZ*wz>|Nf5%k`aK+%cUVL}h{NqR+%!JFEe%yaiIq8S_T?@uznP|w>84$CZXe9n=&C6A zF1plU`=yH+kQ2qBn_|CcAqB?NPpVpdy)~xByqSN=`N_KEM%>~U5?tBaH>ub@M*P9n z3g4X0VF1v_Z^%Cj^!=16A>d3Nkvae;bq+}`M%9zpQJ$p&0kRNr1iYk9!xLGL=Po!K zC?CqU#}9?)UvN$_{Bn=a6$>Z(KkN23vi|2W{kCson1F$*?lwj00-g{BcjPIpJ z4`g-z+UPv|&O&)A+id5|P?2@VLQ`dI-dpmBGW3Wu5Q5#_s|zO+kemr=(X>pl6<5#X z{fJ44ob8S6c3e#alHJUj*@# z$#E8DwC=Rou>nZnd0`rhC2lxNd>5S#v?eRf{V#D2oyofNB$j~f2_*RQa0h`q*4SAq z0iDstYCwZc7o>XfsP_+adsR;QzGY}Kx33`Xl8N1} zBkK%-*5c|r@~Wq8!0`pwg~2*qDx6G&9zuKE*w^vdaCJ{0oy&a;yzMt7iW_oqr`*8C z4XG4%qhWDpI@Yqn?Q@YYWV3b5nK3qTEZf;_rDn1rIb(hJSI^V{=@&P4LgVLM3z8nlwvK3rN(c3JY)KvEw}e6_9KkdccsD1= z-shXs>X-PK;mAwMVjSbU--k0W6f-E7d)^c?k;sG9+3om1sPww9tJnhDo!L(xi{lZP zlG5tJTq7sWq~Afz6S<#2^;?A%+)f~Ne7zxkn;QGe&5auW(lej$`V`%=5Pcw(3#1>V zy{g!f)KAaet0+Fmoh#QM$cAYOe-)~bohrbtU|R@529Z}Hk=1=J;?c~!8J@R0Vse;K za;rPaE=91WgItn2-uRxvkOgS;DYrifLk*7*hDGkIAO{o#CTu&0!fdwELo``O&zn!; z^TsI(JpzcHrgNh1qm;0agQ$sqM|{X)Z_Zb1FgYTx?oFO1UMn45cVYUwQAKFWlp7o* zFV8AQN612Y4K zh!_82J%`216-xc7f~HXOO^ASRnv8qa?D%0oB@9$PIn?BB;vQq8EvNJk4&o-gD29ky z7rr8biV!sau0|#Vnzs_jhd~40E;nB0LQ(mX1|*CsycbG*c|XRQrpoBMM(rZw91;_N z(N$h(-iO6`YEPQQ_^S#Om^$^V*r!fphrA|s#vFwd~356cns}MM#L}@Th~J0LV+%ymyk+Bznk?Lj_Igt2xx@XfvgTN{1Ug-sK5FJv=TXG z8IoxqOQMvzMX>mMN#NFy48b}ciL2oAA}j0I6e(uT z8`0#rj;?#vHweu}?|E+*G~2@Uq!+WXfHZ2x_JM>B zEI=ZZMD{NrFJ7?tRa>yfFd1DW*G0|f{|B#W&4Ys9SE+AkDIi|ZhLm`Dd50g!igjKP z#Pxi#GEiKkLYW>r9_8$WlB97YA@@8nJ`OOYvsvtIi}ONz5#2k*dvp5e4XWchlp@e= zn5+-KwK~K&FV_FN4KJTx3CR<`{940!{w0r+P>kDmLJ^Ap~T=jh;2Xl(%;Zc`_ZI{4yKgTy~ zl9Wdt9uX^Z>S2q&*yH;zGki@_w34KSRx8X&hwkCU9DDRwU;Fe73ihHNb+Jg(*|zID z-UOwWe3cYWW(W{uWC$|~&T;K_91KG^xm^qz|IPR~b#t0FvwkGC4U;L3`V;M_k1CJD zTb*gKPE9XkaNkKeE#G%aL+C*;0x`Sg|}qtOJVd!9F!>!XoycS_VP<~z3+XJ^@8cqXxX(CoI{ z`Ms6YD|`Zt4|*%vrZ%#aiIS8|Q&U^AGUpWkhhZgp9W?qZ}FG8oNl$5?{*l@4`I-Lh!PcvnN0IKhuU}6`qf@;Ab8s? z7-_b~AtR~oRc%Mze%|M&OS{cVGrJ5|21;4#%f%2vxBFiN_{uZUN>fX>Z(E8UVg9w$ z`0h7n6u0$+52xRwqqN<-WBjveDO!hyXWai@r)C8nX8Ftc>gl$&k-)!Jy64+ z&-&A9xt-RH6nd2sKWr`qu6qsp*D7BZeX(&4naV|h2fGQgKb>N(L6tn5gSW~!%96x<^88ZM>x4kq+^JORK%HzDAQM4EQHX5 zRwGb77NW4w6Ovzan2?m?ZyA{Huehq9f99{@Y$d`Ows10OzldqlBHk{;8?mXrv4L5oJ? zjC1*OYAh|_)Pkee43pVu_fW7rN}pEo|(GD~qxl zJ|@VtFsR{OjpFItTDdw9+{r?qH_$%;J5DUeFyAUy2@N^GKvNI(&)lYr0G32_M1rIS z&pO3<4`Orz&J@$*KEgU@&z-uBy5MwII%L{mm~&pthE8i?tf*uO8t!GX*34-}YM}>K ziI|?bL8uYaV07(eRlvOllHn#XyvNqkhlwjH{-rC3!MlV zKj_FHzrrkoz>-?TsJ4IrXo$5z<`&jSG?&m=+<8oK?9h7v;d~{d)5*AE^L*unRxEge z_3dqoKoDyj<-Scktb&9NC~>LV%OQ3#622+4cUKH$GBnYzfrKNvkV^*^VOd1I?%zX= z8h#nWq9S(I$z0zy-M>g2iL)n560lw^ke>Mwg)bS{QE8bh^%p9e{;}=R!Mkl|5~w9u_K}T_l@on6y4%?{2rz{Akdx9RW~WP)_0_D zV!B->cCeLD;ob)A9g`TKaz)1Y2BJ~p7ptGhHK=dvju`)D*qq1FXnBC1l*f-3f7)Zk z`xtxi$8Gw-FG`xdq-1dIQ!cuGJi%7}J6>#$4(0PLv$@pS$W>g@4y^#SQ+!cQg zFBmOS5f?$!PX4O(%L6qp&aZpDzWL@f`$0D*PWNY63=94g5=fiJkLzPUMNF=hcvL)m zO$k~Fdc!b~kU2TiMPI$>7Rvm7ALIw2{0|ZN3&>(mZ{*mu%Ylm#8~I|eVz-2jyRGu$ ztU1S@Y9r_W2TMkr9_#0EME?(Q`Jp{7x{npEuIGNk{CXp}!n57|Hx}Ux9(RfO?Nt?b zw+my;zE6sKNnx$!8$!jy`JT92z~rGs{})oPo(HPYUqUHc)(`t_p|ZHO_wWdBR~&b} zuQqe-cP1c?D^QbDUZupEWVP~#t@Nj2fjZpX=pG6?fbHGCO^fb!AJCu10^w_2F@6vd zq8L>idhHGjFFecdprYMA0b9?XNO?elIC1Vm`p$KIrrw#c6q-c801Wv1f&Lj^_dx;@ z&Gq%z_bwaTeQaMQgg;Yvx2q>>r-M8jTek0_Vx)xX>;RqB@fC4aexS5dJvgKq%y{=# zLZe3@1j*IPXu1*-Z*AX&<58dFm~ym>U92gSB9?q>Fnda4w;cOU{WR#Yx3l-PaA1n1 znyJQ(3Dxxz3}e4{4-a`W0%D!55-SR=I=Zr)~ zy9pehYH3&Hae6n0nh7w4Wzyo;ml_HJzP-FJEfa7A;yE?Y-VV1^1=oCx&(w&Msi|_0 zgWLtQ~#EZw1*i|(`hwNU>?+)dnC z9Y260+X#S{&e1@&`8C79Y4-k&514V)^UX5&7&TcuOQC;Y~XAE3ZZ#P{+Zyapic$ubEfo-z4jlsxC&GSuH4&Ty!lF+9XZ0vdmfP+B%; zd{GhfdW?vOR58v41hd{`hFrJxxO~w%-yTLsaE>(Af;&$AbN|5nN=jdSFs5R`GL1!F zm=j!wf0e(yARudEVo1KF&Yk&;K2Qit-!%O*H@m^Y5YahfO=w>=B&5o}>w!oUs`Cpd zjdG`*Dyqjf;YtfD+uq+S{~TcTK+u9$-?De7o`D6uW*dg>$?Q>4qXJzyF!w1Dy5BO?l~)8)+_b<{};LTbcKF zHdt~T84sRThC_IXnd`6n2@&(NpL=m%cGlzSFhUF`A7z3Fw} zk*3f+^zQRsFfx=9t!V((w^yl4O|f04WCBmLfw4gUtZ*sD>E_N|93#*ZJvJATwK4Ug4ZxwjcT`ua(XXe zR6kHIdrAi454;Mc;i*Cx^>em6f2se5L%Rn?aAO$m+@Wwh9^ipYqU7y50{N!M<2=90 zJkKH^Eu}7WfTu$I1PxdFj8x5j+FxwqT7qF|v4&;Zrhj$Oep`mN=KBIBnGVqcwA zp*N3<>}#rW?RY)RUei$`+@kZf-faT4O5h1Ac2-4cMiPDpK!3PqM3M+%O{fm~OOQ^m zB2&VTAQK$0gwx>l&Xm)28C!rWrbu2-hOCz}8s^I5@YDf2LW3G2Hf)T$*^XCUmp4#2 z)K);Pf<)#-UL|xnHMQP&Op7c{=YpwM1ze#)jDZRi($AvNGy^fx)e^e@qC~syfT%Zs?ojy`wz_0?)a&2i;*wxj zB%k5{tJK-UB(mb0w^8yIl*dk2B#!$>;jB6~SgngEC>=3A{y^D33dhG99{9aDk-{kY z@ydYTuVs?AD`f4pJFw47>9;Cq;bY!&!2)q2*VJl~_V{lnT&pLLYINF>a_{u}m5aYh zrH{iroxlbmbhcWdv*GcFZkL`B)5}?4x5TwMFDlSgoB11;p@CDai&<7n& z|7|hDO6$rEh$knYPH~cTQDX{h|EW*UR!a-HxndpH_JLezt^A&+l~>3eeXFTwpa%8M zA5LpIAyRQ3ZjzH}H0j~-K`T{k$)Gd2jc;u9SC{2Qhnz&dc`}8CR&@@{g>PZ4P!1dsfVABd(O2Im&}8y3+vLZ0>-;7=>9Z1<}1#@)=#jhtX$ROw%X zq#DUgg7ixKy|OL$O~SiIidLI;*yXB_?uiBVRf6(1yV2d^aDd+=jJ@gW-+i0(j2M&Q z-F=Y1jT3Mb_TC4_u{<7?6TypMHLfr4c#?8AZ_D8A`Q0(^QRgp>c7_^~KpEV{I*vFR zT*uO{+=+hgAwv3Zs+&_W0!?w!#n%%?93>1@1@AZOo#+^Bsel_BnlxI?@F4{gO63+# z7ryVr6Pj@9ks77Xf`q{MoyI#oJJ^hMc(TL%nXoq|x9?u`F4s+UT)mv#xEb&GzY*_L z9zDqg0WwJI)Je-?^|7T@`Y7=EEL{!98i>+qDnrgkMzmWlz+dcWiaiaWy%OJ}lc_ik zr-`7Y)+v&ymwuL8Z?+N`;>LOMzS8|e)1x$ykP7C`-1`b@HmBM}??M*5*IyfnE#CIW z%4Ad_d;2+FaiWV#UF3g>)nO~-aVp`uF5}6is`$fhXVUVTnX0EZ+JdmK(I+Y*9@h5R zN^VATNqAz5-={dOgzX*7VsluldBWWf_~)&sJXZgtE|iG7%VV>}>{Owd2ZOm+E-D|# zG3U+kqBrUap-fIS12QJ=+wJa|pQx1YTmJ))PW6Z}?NueLhy%i~kj9qxQZlck-}wUbMcbl! z3WzU&RkyghGr`jHv*nV^3+7%l$(iUC z8x!R$WlA2#6`|p z6&Xb?WI-17ZkAf?pB%aNMFla_Q?NMqMw7gO_g8w_9H#S{cqz#B33@VY$Hw|3;bAuU zN&`8|*~v>*Bl`H>)r61!l{&Y|G;YvdrupyqJQR(NDe_(nNuuRs5^cG|RF~?y&c}(4 zTV8&}o})k{T~5G_^QiTp=i>?Zw4r)(-%hOIdTlMP=2WCo^q)&c8%~Ou&sY@g^r8-0 z3Y<^C%*%6IhpCG1rj(p!&xvRp_G<7-6x(0mYal*Xa=YjGEptHbuG6go->dglQA$>d zQ5RG4>sIHC&Sp8XTyxD4E=ybHO`KSm@ZPjvaG_a+W%8pOK{2yOOWT zOLp$1jqT`UQ~kZK1t!VC!3)zRU7OpajLLUr+io~BEpr}r&u9EA@P8u5(A=!i%p_mO zl)e-+QunPl4EJ%>gnnI!GGx!p49PiD`E<+?5A4y6l=I8hup6x%eR=8yF+U?G1NbPK zvq-%C(h*N8EN@>F2=xi6liTrEJE=aFTjG+HirMwfTt!iT#D&cIFcXILY;S)B?1nb0 zLVV4Py!PJjg8q4wgLR|HP+7Von(MC#a(HNI(z59_6WTV9aKB@3*sb!Pkb(A}kP)qV zKJQNQHrRghkg*pDRtlYF$B~tMQh#(@`l-3f3vuxU7>cp=h%WQaFPdf<*PU~!lWy~6 z3OXrUH4o+AI;$rRbzq>k$jM)I^*=tMjMdZa&i_$Nddt_H>Mfl+AQ7vnrOO41C+` znzvIbOSLxAi1Ebif5*Psavg(Yjhr6`p zc%Gy5z>5E)h5qO`mFpxhYScFZ;Y7vJp!U@rQzYs9#6`e|Vo8YQX+r30xdxMtv|yN( zA$yv8v<>TyVbn<0tu5WtKTUp*rGO!3eK74#g`J`MAS$di#mz;=at(1X+txolPgB)!RNDDksa65Qb=kK$pXIk3vOW2$^nb=e+-_>t zJL$j#a*e&Eym?7Y+`Q1%_$e2>_-i79-;`U=*Us7GXUWL1KhVuL8Y)JskHrI`OsPIp zPM4ZISd>wKD&r&-$B9lbc>7fa!cJb*=a;+T;!&Pa>W^&VN1{zBFA1f#+X&Nz_Y*KW zOtEM|unw7l$jc;kY?B5j7lD;rQH}PS158Id&EAI$9;jMT{JT62~a)V$k3k%Jta_xQKF_oFq0B40d+J+DjE01-Z(O7;4bgbGcphOz9{c13W=h%Wa) zZhP&(eI9H-I+2Ngyv2>6-gC&Qp4Tomz(>~TM(OGe3d{hq6H00&T$^bcQJHX6`%fjI z8q9@sn_8z>-djiQYv__5C!yah?z@2BD*6Q9vsdIqq4<)(Ax0L9C0amYV4?|Ctf@ss zAA%sj#1*8-kd)FWq9Z|)iivTUAd<=X(ol53fk0GjuyuAHayWVqL&pFQK<|o@-##2xqu5>Vn_X!e|o9ewy1bt!F^S3?drE~4ePJt|7 z7c1%L$C}*^m=g9F`-wN7kSyM2!5A3EEp=en*~E=xtkAV(E>v7#GD|+FYPJNXtzwVo zpa?|`iXjPwV{#xs3UItzuLCkT=8s%D+IJj5=zhWBMN?5g5C1&zul+i4oh4!dj?NHi zUy>uD{c%NtDp>}fLo??(YlClT(nPiG4Q%wczpM z^rC;k!Jr@hYgAV3B}CMwIVz=Yc7|x|c%z8AzD>|ewsSu3_d@6buNM?7@c?;wS&C?n zU~m%wuYvS}>XnlljFvw;5&`gwVyB7M?)t6a_7|)B&jE>Seh0Ed(3pY2RsDU>Nt0u2R` zke{_C<3(#d8_2@aBGOq<`2Q!e4o&f0$6%&^%ZmFsS$Zg2sZDilCg9Qyuf6&hRxjIx zUlY5*aI!8&e_qf0W&xsi1|wjP2IB+g;f*>G`1IRwhlePL-h+jKJK!=d0Q0BT!|6}H z#d#Z_*R}9P6G4Eiu@Ib!cIk!>_U?tp+izk~(DUfL^<{ju{TobbU#ntY`LWm;6o}om zUciU%PDP#Ybyyo|pH3#d9!au@mQ{2ZF*OBga&0^`ZU$!fmeV)?AQJDGkeizeyUW9> zpc9}&$B9!h;``ZHxB3`HEbfJvwWPVApT~&7)tS+Cj zOl54TY^$p3%p4<(5RsLU1sS!?@vMW7k0-QpG79nuSeQqxw+|XQX^`xst%P1cm2(@7 z$T7-bRt5|*;BFpQ*Te^&0+Cr@&*nca=(%7Ae)pJzHy16z@NQc%aou)&GqEG=2@g5( zI$~WPKGQYEUL6#0acBm`-uhfYT@Zw7ut+SL|nfI8`WyP;N1xL`?tNgRkobV6pMWaE>>R~ zODM*n7Jd_7!C?7MY*@Mm5iQ4{x4D2vJhu|#U;Pz+vjQ-(tp|k4RL7exVA;6W@muQ| z2priSZUPlWVpbNio&50V!;LJ6Yuhez;yIkl>xEfcPT|slWq3MkJEjI~L28ALM4VqX z2`}yU#q7Z6(bipnBkJiq8=@W!xcKk2Dt%eed3&Q#!CCAHy}Qy+jQ#@$qCC;mTZ1!u zwj;9ao5o8x5s_K)-%_!wrV{{YTTj56mDAA0a1d)h`5NaRUWH?8hN6L$q-0C}{gDMo zJbMngkNM;0vlnn6cmRw)&BpuNlI@iS-rfzMITMQC?y8Cw5xs9eA}tzTM#`OZ6gKUO zUd=4sRZ!|vwjy(??RIu=D`0&>s>`P=QyE(-w5blA{o109tBi9ej>Ay$s)Y1(14OS* z=-Sfqogw+o9i+hrJ$kmNgkE;$He0dd)C%p})CMk|#EI<-8-55A1IMzn-bkdD+$D&&{? z4y`d&AJ5TzR*}m3i!ud>3R#M6-ZSvgs8;4XdNQ)& z#dM?_BsA{a6)h?&+C3ZFvG`WZPj+6jROfnC)fl>;SiJCTGweC^z(styXcMj#06;FT z#oPv^Y-DDdl@LXUkRX*Ysw|oyQG`j9t-t2f9>a!pfc9`OCI@aoxZVJx{v^JhvmCKa z+5seF8K5kcem_@g0~wMG+%=7&2v9apo&YyDR|q+2NXZBClas6lkj85SroP~Vzd|C= zXl~|j2Cd`k^zR) zR2Tu|T>BT-vP{U&%Y|6dut@(RBwP{kl0LYMKx#({y+eJ<~i1oi*h6rag z?LP;r-|7vau&_-3#pEL5%vBWTq#@NvAUhfn*V2LfJmg9eWCi0LkYUKmw46LG%(JfJ5Da@zz|o*r5wPtPZe<&haQr*WUU?NwI(5eAu|45_ zdL>?(`UU>HdK*bs{=ka)TanVFHXyjd%~gYp)D#pTAIV7xI9COo03hejg&4o3F5a3q z3$teVW6E*WQ-E)iXub+GK*dj++5+xY^0>*1NnE5R4yRx^Qm(21pu-LlO#hl znaa$p4ACS3qRD)tLD`lL+rJugMuTDSf8{~s>Cmr`!4m^(A&qFA9MnJ=j7VWj`dXq&zt8+XQ=<5|Y7L&p&F z$aMHW?FPjhE#1MsZ?+&x0-T#YiFvF2@hA#WD9b=zBsRY{2P=*w!WvGkeUbV33@kaG zQTn2kjH}nv!Hxql&`+n3U%Du)G1WbuqxmcYEbs63_)BqqEitn_< zTm4ZB*4E7O5He)|W#pBM@>tn5lLWw`&s1(~<@hd(Qht(^u2~#jugb?9Dy%;?V+i@U zxOo9)uRMh0nsB0v-0^=S9ncdR1{!1%g;u3U>%)6LD10EZx9zA z&{L90cG=|GIeaE-^dQs4Nao`6V`Dbh{nH$Gr8)oy-x>};;FfQE_L7L(jGOi6> zG#+f$;~Bobax*`Cq9JvjojLBSGt5ie$9Ye-;RAhN?SekL`X${D(ao#tH-i!Z6^wXK*6IGpAGt* ziBeI!*yRK0KV zlG%E@g4iQm`dq6bTQ1(s$mD2-1q`N_R!grT3wak`3mtR-a5&jf`+r1MqsU;uk*{@r+B+CfhT?p=8&e&MD1>z`qfT`-xh7iUpf0% zm(I++!jP%`_+X!j4E|z0m(P5O;}-7at>T>)V%D&)&V{W9OyHC`i}~>flR0F}A|5S1 z*rO79St4T zj(;(Lzuz*E_T=cC3taU0gB&n(EqCq<<#+Fl=Y)0PG?Emfzu=RuPV{**%tG!ec9z4O z)}<5sKJ^?Y&t1%wpMAo)&v$3*-cuP8S(@`~<`)~e^q!38NIna{SaCg3@|XM@Jh)^O zo0)xZZ)6gq!xnKsV-&Z6lZ#B5jE{<9LbgQ7kiqIcaLAMq&#N)XG&*p?L1o$$ zlb007_zZi_3CLll!NTW^$&86hHCKsAH0YzL&vsN0Ba>p{m})LHlMD%r)*J3hZux09 zdF65((;X;BvuGE|M3Y1oq~GM_D>s>`pt>z>G>e zjENaHuQDdtXff$aIf;ypy~Df;Z{{&6HlAgtxbixQyt!uy-~aw3V=tfM@neViN7!%N z6|#y`hRx#sd|Sb5S$@f6qQnBG-?+>xH!~`K!jdV2H?PJp`R>VP?D7E=>W}93Yy%Ub zOMR^JecBwDTbJn z%-gnu=)NJ4ta%RrsHpreBj4D7A$|5>{H48krGA-WA~_vhYwvD!PyABl<6_pyq|B>?-RaEXX8W*^l z4q^V_iMTepFZ>#4ked>Z7(LK^%6ts3Q5T&6AR{|F7cgm%X~exK8C9tP$l2M*g$Y?% zM%cYioQjHyN)5usk(nwg|9=tccEPBzPr^H2kL%ZCk!o^@} zword+S5Z+>QBhG*QMoTTwlA!Tii(Pgii(PgN+nS*JXKUwR8&+{R8;N@>V>C@ii(Pg sii(QLeL=nOR8dh;QBhG*QMoVt533)P22GiL*#H0l07*qoM6N<$g7+k6wg3PC literal 0 HcmV?d00001 diff --git a/setup.py b/setup.py index d4b41155..fd79be4d 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name = 'x-transformers', packages = find_packages(exclude=['examples']), - version = '1.12.3', + version = '1.14.0', license='MIT', description = 'X-Transformers - Pytorch', author = 'Phil Wang', diff --git a/x_transformers/x_transformers.py b/x_transformers/x_transformers.py index 2060a84f..a999896f 100644 --- a/x_transformers/x_transformers.py +++ b/x_transformers/x_transformers.py @@ -895,6 +895,7 @@ def __init__( deepnorm = False, shift_tokens = 0, sandwich_norm = False, + resi_dual = False, zero_init_branch_output = False, layer_dropout = 0., cross_attn_tokens_dropout = 0., @@ -944,13 +945,16 @@ def __init__( if deepnorm: assert scale_residual_constant == 1, 'scale residual constant is being overridden by deep norm settings' - pre_norm = sandwich_norm = False + pre_norm = sandwich_norm = resi_dual = False scale_residual = True scale_residual_constant = (2 * depth) ** 0.25 + assert (int(sandwich_norm) + int(resi_dual)) <= 1, 'either sandwich norm or resiDual is selected, but not both' assert not (not pre_norm and sandwich_norm), 'sandwich norm cannot be used when not using prenorm' + assert not (not pre_norm and resi_dual), 'resiDualcannot be used when not using prenorm' self.pre_norm = pre_norm self.sandwich_norm = sandwich_norm + self.resi_dual = resi_dual self.residual_attn = residual_attn self.cross_residual_attn = cross_residual_attn @@ -1037,7 +1041,7 @@ def __init__( pre_branch_norm = norm_fn() if pre_norm else None post_branch_norm = norm_fn() if sandwich_norm else None - post_main_norm = norm_fn() if not pre_norm and not is_last_layer else None + post_main_norm = norm_fn() if (resi_dual or not pre_norm) and not is_last_layer else None norms = nn.ModuleList([ pre_branch_norm, @@ -1080,6 +1084,8 @@ def forward( max_rotary_emb_length = max(list(map(lambda m: (m.shape[1] if exists(m) else 0) + x.shape[1], mems))) rotary_pos_emb = self.rotary_pos_emb(max_rotary_emb_length, x.device) + outer_residual = x + for ind, (layer_type, (norm, block, residual_fn), layer_dropout) in enumerate(zip(self.layer_types, self.layers, self.layer_dropouts)): is_last = ind == (len(self.layers) - 1) @@ -1095,12 +1101,12 @@ def forward( if self.training and self.cross_attn_tokens_dropout > 0.: context, context_mask = dropout_seq(context, context_mask, self.cross_attn_tokens_dropout) - residual = x + inner_residual = x - pre_branch_norm, post_branch_norm, post_main_norm = norm + pre_norm, post_branch_norm, post_main_norm = norm - if exists(pre_branch_norm): - x = pre_branch_norm(x) + if exists(pre_norm) and not self.resi_dual: + x = pre_norm(x) if layer_type == 'a': out, inter = block(x, mask = mask, context_mask = self_attn_context_mask, attn_mask = attn_mask, rel_pos = self.rel_pos, rotary_pos_emb = rotary_pos_emb, prev_attn = prev_attn, mem = layer_mem) @@ -1109,10 +1115,13 @@ def forward( elif layer_type == 'f': out = block(x) + if self.resi_dual: + outer_residual = residual_fn(out, outer_residual) + if exists(post_branch_norm): out = post_branch_norm(out) - x = residual_fn(out, residual) + x = residual_fn(out, inner_residual) if layer_type in ('a', 'c') and return_hiddens: intermediates.append(inter) @@ -1125,6 +1134,9 @@ def forward( if exists(post_main_norm): x = post_main_norm(x) + if self.resi_dual: + x = x + pre_norm(outer_residual) + if return_hiddens: intermediates = LayerIntermediates( hiddens = hiddens,