From 00afc123429a7fe1be53cf8536ddae9bbf3761a9 Mon Sep 17 00:00:00 2001 From: dan_the_3rd <43445237+danthe3rd@users.noreply.github.com> Date: Mon, 6 Feb 2023 16:16:03 +0000 Subject: [PATCH] Documentation for fMHA + blockdiag (fairinternal/xformers#459) Co-authored-by: danthe3rd __original_commit__ = fairinternal/xformers@7475b63973cff86adb683fd3ce2b827d7243582b --- CHANGELOG.md | 9 ++- docs/source/_static/block_diag_bias.png | Bin 0 -> 39801 bytes docs/source/_static/block_diag_cat_split.png | Bin 0 -> 29841 bytes docs/source/components/ops.rst | 43 +++++++++++- tests/test_mem_eff_attention.py | 29 ++++++++ xformers/ops/__init__.py | 2 +- xformers/ops/fmha/__init__.py | 9 +-- xformers/ops/fmha/attn_bias.py | 70 ++++++++++++++++++- xformers/ops/fmha/common.py | 15 ++-- xformers/ops/fmha/cutlass.py | 2 + xformers/ops/fmha/flash.py | 7 +- xformers/ops/fmha/small_k.py | 2 + xformers/ops/fmha/triton.py | 8 +++ 13 files changed, 173 insertions(+), 23 deletions(-) create mode 100644 docs/source/_static/block_diag_bias.png create mode 100644 docs/source/_static/block_diag_cat_split.png diff --git a/CHANGELOG.md b/CHANGELOG.md index e25540ad00..dc63c5d573 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,10 +6,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## TBD ### Fixed +- fMHA: Fixed BW pass on Sm86/Sm89 GPUs when `K > 64` (RTX 3090, RTX 4090, A6000, ..) [facebookresearch/xformers#631] + ### Added -- Added tensor attn bias support to CUTLASS FlashAttention -- Added tensor attn bias grad support to CUTLASS FlashAttention -- Added dropout support to CUTLASS FlashAttention +- fMHA/CUTLASS: Added tensor attn bias support [facebookresearch/xformers#587] - contribution from [@jfc4050](https://github.com/jfc4050) +- fMHA/CUTLASS: Added tensor attn bias grad support [facebookresearch/xformers#587] - contribution from [@jfc4050](https://github.com/jfc4050) +- fMHA/CUTLASS: Added dropout support [facebookresearch/xformers#587] - contribution from [@jfc4050](https://github.com/jfc4050) +- fMHA: Added support for varying sequence lengths [facebookresearch/xformers#500] ## [0.0.16] - 2023-01-31 diff --git a/docs/source/_static/block_diag_bias.png b/docs/source/_static/block_diag_bias.png new file mode 100644 index 0000000000000000000000000000000000000000..d18054a30822b03940dffa230da1acadf2db9ff9 GIT binary patch literal 39801 zcmZ@=1z1&Ew*>@2kQR^-kPhi?1QF@(lJ0ZpZWIJ0q&uV=q#Hy)y1Tney5B~@tM~rT z?*or#@6|KL9CI%Hq$PzB;c?(0ARrJ$-n^29fPe-=KtNu>Jp{f7UwC~#1A zdX_LFUjA@Y2wy~HEEN3nV2a3Q!tkg9h!a`7>%(H_Yl4H!veW+i86N-UIHbJQw*$|(9tR)mpB>?}* z!N>=F#lgyF(EvLz@#qnvouD!*1h-!|CMg8}!0WBU7m`qDzDq$xF&>>*gdb}kWDX56 z(|&!UnH+oyhnncu`{683FY#Pr)_R1naQR>*rZ?dzY57&L{TR13yo zbg=xqNskVoH46|Nc}PvAx=%Zu&R(=nb$yctw@-{C0FS!fM`M2qov7rt4~9vi_iLaZ zR0tJ|Mx)qYO?MF4!d7`6S{3u%Q#~4&V!WB8TxyQ{0pA?tCFHYrP-1T(-!(pUjD-wo z?@Z)KZKRQiCSD*XF&K#XPP5pr0u|JbR{cKaJt+i}z1_~6TI}e$02IWAx05!w5Eq7U zLodcIYaoo!9}sRfydAWC9VCYVk${9(Okvu?YMlV_@QgGBQa%}a$OE%4DZlF>I|Npm z4grdXJRXc96w({$Y<_4(@3Ob(6c9phVaFbHyd-6ZHfdt8fGYK3nuDr@0(nd(V?FlL zrhk~|EwYCsE{O8Z`#TDZtk=-+LgMgR()~@Z1q3?t?zWIfqVtizfbFL0e<&i9w_I7#7ldrN&1Vobd>&&69JC zl{mcc6^&SPxE6HWkNJEvr6EY)e@v$C4+xFLmNAu3Q@m3$Q;t*5rk?IR(1b4xs%UgxNZp5W!LNU!70l<`;7!mbg|7FA z6WUn-!c*+~b2IWGl!-?baJuh|KjX;+uf2IpN`}gY+W8^yTj#gu3%Hb&loFI&*@Tq! zL$9+LvlnHR(qE_XhAxTtkg&-<77?F#%_*=!I89#nJgmK156g;~Bdj)1UAX>@+pB9r ziBGJb3`DHc+1@CVyEOB<7*3QeYur*wXQ=hh0bK){XYv?rdm+MQL==0hBHt_ z<_PP{gk=d0=?;N5(!6TDA_OW0$_3tkRe7(C+b?T0X-sb9S}GMFkHvtsk7=$k zRJqpk`NTf}L*t2~|9gK4|2qGq@HJ6<(RNX;2-OIDs?-R?2=8zuY6AKCyh|Mh+u~-O zo`tV1oj^5))bPLglm{&}2ls&TbuMXqX za;-RbJTkO_;S5U@pi9;IEM<~qb`1C8D|+; z*>G8uap5qi&n;Q7PkW7IV{3SQz_Bnl6E~$ss6yc+JMlBV7@bn#OYobdyxqK{0CQOaC97fVzHK}OY z`8X}?jC&z|fqv}H`IL)Z|7vbe)?%x~w(=3(k!QHc-JWYxr3 zaX=)(=yFjf`=Ce6H|`FXb(0k;D|0*7U4z~7p8V!#%R9@OYct!=cTrblcJ{x^4Kg*A zG+7UHPM8KN`LeZ6jyh-B^NB^YXb%^Yxth6-oT>jHJ@Yw@yx=`%Iz&G!qw|hBdofvN z_Fbz@Yi&cp4>;%2YRT(=DEAZlWHFxvbl3E0{#7q^zVSF0`Tt;rpN zYSCsKYIR*iE(f98-f28S_v`UX_oF)nU0Pk|lI*5+q@A=wwhMGd3|EH|7x_*7meupXPOyI#2tqWW)ZTL-|vGq=-;#YMhSizWu_d7nynfdC0lV zN=aRjotS#FT?}30u}q1ZF&Be4hd4d4J!LqiZ28Yr+SHZG@Kp&a>aD#;d@d1naqCz| zKtv$XnM2b=6-|+u$!^rR$peK0wv}MF)gui-PaDsyWcPsP$d29q*@K4N8Ivm8*^$Ec zQ(rfQ4205!k^(qdM&0ReQnq}-lM5BL;EHiK3s^hfx5BfMs|jVKR-{#=sWp+?CgT#b z8kHB@i3-jIfI2GfknUjYKBGOLHl56@1Gm-pF8eLlVH99UTqqoar*EzyP8t)uX<>1x z6{rKK_^!U+SSEMm=A^4uR3D-oOOK?Gv&^$-@G-dMaOJrGM-)ZuMciE0R2H>iwz;vP zuu;L*h`8Y6m&zavCa~rk88Mnt9)4v?#fcOY5X^VMalBfOUQf;`#VE;|IKWZX5Hp}Y z2359HDqc=jj#MF3b`*ciF+IN1H**=#PAA-B6ratWYhic6u~2nsJ=?RIn6!anS6dlk zpY~EMq{IX%lT{rqn8@1CR$^)kW-V#y8C)!ee>gHKARb+bgHy|{4)2b zuzQ+ynx?`GTsB=^Gd)vMx;pi2l?!X9dbZ9g*mn1Ip*vVgOL>;xUHoPb!hUZ3D*U1l zUymmme~SGbHkAv-MbY7>j?|6W$8M%L<=DJj3%phbmfshfWO~*PW|`b>xFc`gU8dQn zyVmYxw+u3&C!$v*(YUi7tDm}@3pEi~xo^2=-~1R#qII8%{}_+wOzL#BZM%<~&L%vg zTDKh!AMbQXdo+D`@fBH=z=V&H*Ur`dl$p2goN32;9Lt+-Tg$Zp$yMy?SmHZov7s8| zlg}0|+z^Gq5d1+<9$tqS-<0KY2gD~lLNNI*~lpWz@NeT^ZYflrXY4-W7H0r4Qz2LcxO8x8n*l?-+N6dIiT z;Qlk@)$NNf+B($dq? z6Y;_m5)yJ-=^1d!zIuJv9QYRxk&%s!1t%Syy}doHJrgb1%8-tMgM)*Po{^4`kp_5! z#@Z2NqwPQgvL^o3$$dYs^sRNRj4f=8!63rhezkSLwl+LOM7IO|{_|^|`VPkbj0Cd2 zTNbcDy4xpo47BugzxxK7a^HT-DQ)baZ>I9f*c{j%U<_UsdJgWN@Bin?KO_FpQu&{j z3=B*xe>VN&(Z8E2SnFE}g3W<3ZFv7Vn!Cn-KD=wlO?SKWKepo6KL7j{*l1pOZo1!x z#tTnD=8pgY!4Dzw>ZQB`P)H7u=x#^|iYOZ4@+8L$;&{kAk}Xirxa-OcOJU&CdlW6ntUgxDV#*hQ8=`LV>Z`7yL!ob8gm_~rDbLAFbF^97+YjTldFIl2Eu!#Ih9Dl(?Nlq@T5`zR5EYC$82ikL; z@ah>FQnw8U?a{AZPlha;c&qPb(vw#_r&fztpz>P^WO64$XSv(%u~*5j193W#a+969 zd&)#!ks|93Y9Tzq$Ie#`l%zXrXXVh{?jU%gR+W)6%+XGU8o~W>xIseE+h90Cda|mn zaz#ojFHDU^M%^Vn^ugHN(Ukz$DMZn?5F15A4Ks5&OIN~#FnTVmh(uUe(Dj& zOC#GI^naggvm=x*iu5*&39W+MbUH`EWV(KC295Pm$#?qlg4vYmI9Z!xkskBrS+$W( z{@2x#d?}8f%*zgXA~uzBbzGJO1Z_ao4Si(A4zF*hZq`d`_Gq~QNS1_7p$5cqo_ehAM-R6<8|_LI=ae`|vW zKLmV8?W4vx(XfBdkDt&2`ppb4Kc1BwI;wU;qK@;{d4xz}jy4_5urf~B=K z*W~U0;lo}x%%ya$AK!}?;`8%8b<^0`*dsjj+ZHAa=0?Lg5vrv|^Fw4PY=7?qKQ;PWvszRJe&3b zjEh3P>To^y36bF6TUnG2p*u77qa5YQ-<{uT&R!t|57>eH_YU)7jwO`J@%1F;jE%9E z@fOx~D1A8E$8c4;IxS8esvp$P!AY8{2{_GK>Bb#3X{yEc&sA4gt=Rn>9SzT09lD#N zad4t~b-5_OVlp6Ym}DQxQjoG+uu>B%!KGMut`?;}GH#ExFVhWpE7jvkt5MVUIIN&# z1g?0x!R84OQfekK1h-2eHCC{~rgcy0jFhB5A2?6MujXhv3f%~#$ zrB<+i$+SDI&CtnFLE5YCcrLXF9bMZd4s-c_maPwRozcCmIbv-A@;vpiG;#dBxN0+) zUUx#~$&VU+xvgvq8dPS=bjv?^jnq%0R30I+!d~q1cC}`U3>B13oLi}|Kqg0+cS-QidI z?rXP#>hcDGMc(tT4Ew9dLoRfWosO3;~((41U9wV|F=I+i-G4j?oK3@0=v4&#ZrNLCrwfUOpdITfCkBoc>PeEvosg251qV zoknXuJ@=e(arfdq-w;;t&;5KNT)&i+u`My3YIre~jVgG-MFp}iBC{;FJ#dn^Fiv<_ zcU+v&qn4Vr_B7Dth~wmW`OW!YJWTA~uwM76ap=rb9`V_pr-ex?*Lg96ZbhzJjt?%i zTk({>B>UQJStvsFP5Yd@4(g}OWF5`Z%Gl^W(HtBiAUE2lYgpf;tG6Z4xhO0|bQH`? z=odgYbWN8b2<{uC<4(c4Kf?5GARMe@ zV-rriAT^bc<|e4w_o1noc_)D(mT?DHAt*O*R5;k?)#VPM+(|~x$zZVQ-k#D_oY_Uh zQlL2)(ewE)ujVlbgGm>kyZ7OZ?bcMYiKw!2W%W_5h#a}gN>)Z4^W5`%t&6A#4=ee( z8-(LV&#UShdv~G8i<98beMR&MwvuWwZCUZM8B*7p+t=2f8zVRN$9<%{b6+l=PgJ_e?O57r ztxe^JC2rj)gQ8hXN2Q&de2&x7RzTKYzfQQ17@u|$9~#ZhNWz_DJyog-WusB|n^hGrhOQB{Nj}hD$iMii@J5? zt?QGF0;d;?n_H<3%tFJ|;+IghuY8V}O{=Y)X5@463poNB>;t3Zg|6@k=K3+o^P=BL z%&y{}z#SzGo0cB$+B&W6U+|PqI;Ly1A>{8dGBTE~tjasL}DcrwSSY}4WL zVx&mZb=&f2NL)-xN!+HC7}U9V!k2b#yU}q-^2M3sN8GzD>eswZaA7nf-(mkoZo6a8WK&{{(ft@!@*>ag;g2v`4 z+vQObJadv}it?jmD0|Z#Z#0k1H)u$MZQAe%r}}`m7dmqq_h2JrG+%G9j7>xTL>l2E z7`(x2TC~=U%a*GaTu{Be6L;iG9b>({%%)P84A0$efXfD+{!A6Y(~v~nqWI;rjR4WN zZeuhyNy@`o%Z>yh$$M>%=XY>;$H>IQv(VFZX;0r_1lwb?)Q{Vb=!P-@)e3r5%lY1i3 z!S`54MZli}-lR%;V68p;~TKKl6-DOhSrt zScWEBGqw77YacJ{+eev&A>r3}?~~}1nd-YEGzzSbh;nj*8(ex>;2h2BiQ|h+@K>3= z7iX;60!^mUK4nP8?|f*~a++6A$WngU%e(KjQhsVkl&nbPD-u+Vb zV05>7u|BiZaZr^;>qp7a`DUw<0 z3mb1nnkx!?GnmJG8ic*N;wik_?y9jPx)iNb9y3ENNkPtRo(DIoD@=#%GR)FBil1B2 z-C2My^oRvreUV&a=^a^C^Lujz9$wAn*Y%*Qi*oG)GS|~HWgZJ*I^l1--w3e}TK((; zqfvih;zm>`1JR9_g|2Dj_?0iUT#plnh3ZXDaMc^)xvWDZgp*eh1`erCFBd;?Mt>^s z-Q8I)uFz~IJ9j#_3O2XXVBj%(i1aMkcXwU4dq?Nt^)vb>_$rXM7b=R;8 zb?e0lr}+rO1JIZ4AKP`H{tkWjEZ5C|{3H>B%Lr{^E!o@>$qMr>&!=MtbSD*PhTA>W z%Eqt*L$xh_=ZA3&BdKPRMqp*5=-2h!>h?V69=!N`lFLM$>p7hTeajlx5*ZvRh>`)H zmI9G?$9B>Z&MlASTUuMIUZ;R}W8r&ZdYdzzjjiH5+0vLkob`3dTCT}q1FPr492h8< zxW>=c2k1N{9Izi7`gn^ZEyDQ^F!PzUkEymNiWL-cNhw}*RW-ON_!SnkxD%`sxy$RC zS2lg5VQ_zMZJvHHsV&Pj--4 z$18SaDoF7;#m(63FD58|X%|e`6$6kL%h*Wc>o^NIqGs$Oa5_d=cNY7t<@uHHtpuSr-%JHK~&k6c45KZ7us!-N7=BbqAg_#az z$()OJoOV6qvV(qw+Yv=qSklW|^O2>B`_HiDRKNoeSNZN}D$9smry;ysf6W#wKU^@` zr<r+enBNlhAR|I}RNmKSg}>!@Q%I!HTviOKeQgk1wbC<73EclVS~* zr**Gg=tUHA6c0LWz2}LBa}?y>yv~Rnwgr6#&*$iiX5ke+WZ7Upj_g(|elwh_WN&r} z9rWiYLJ0IL68*gC6KXUm8L}rN&hc@vv1W*oVhfGlEvwYn1)3PhGnVfoX<`fQ+qz{3 z!z7Q+SgQW);ispWc>+vUqWfle`1IWhBNIz{6~4Yj6yR;Fx)x=3q7$x%KhoLs=4)u^ zB9fYRomSfGwRZ;?)qF;MLVnSWN&pDUjy*HH%Td4K+w|=az+K z8aUdU=Hbc79xd{#PpqHP#ifR0SMx|@>#28t75vXW@b?2@z5g-g(aZ{ss`1w?g^4oV z+#E$QY2|3^Y`TpBxs~NKG#s3qGip07NjO;@YsflkCQ3l*=~ST zER>mi{t1ojPoeu69*@a*Fo3rlls^2I-#|G{x(%bLZ{G{TdfSEDHv5&(sBx;wI$x2D zgoM4k%>DW-a|Aq=<$irVS#17>FXVGyh;;sWbCqsG`nH&?!>e$jR7}oeUc5Kc>6JM$ z9X(xEX1G>gtV^9&@zV*uTj=3_(b$g*t#qv*3M0lF_Yw4E%&)WuiP#|a&vo@K4GtoS z7N`}wE=mf|JU~CJA7a+8s*g&T@~NY|u-V&tZaz^qU3{WlXqvI=?TgG+jR?y)U_+n1 z=Z^JehW&V3WDbsS?wKe5Q#vw~iMfOt5t?HRcAK@3a`V}g?HbuF+$9F0$vM#}J$-z| zjUAdc_JQjfrh$`jzVAdt+%6Q&5zJ$9-%yWdj~0;+6mk`5E?`yPyklTsP%Sa3`|<_; zMldx*&b*j|gydC!f4^$kC4s4(vKq9)SHiBR!7?&3s#OjVR#x$6FD0W*KVvafbq;8` z(P?jPTRS*7R1y&r1MJ3!4{$M~h3+rPkj)+uGn_QKNSV!+rM@$qShYB~GMkW5e(oq2 z{df|UpYRQI{8^`D$P`{IHa`@L$&hqMy`ag|=Q>@rGPkRT>AO-Rg2+5eW2`8ZJkSr3 z@$3z(+|WEQ(LG=Er8ej>-_8bI+nUd`>6nI1M( z1|Kcb9YjQIcQqj)PF+8MJsg2e?fe!JK=dm({W&hb#ci)a7vj{YMG=j}R~H+fsp*F4 z0^AjLGcU46b?ctRG`9EjjJQhi0wHgmKDCzDFJJXb!#x|(-j-+on)D<>m&RULeGKDE zYw|AXD^A;pN-mLA`>WuL8+thnPP;Fzmpc#=y(z-yJSiCN*9B`?MkfaKe@+GhN)s!2 zHE>iU8wC>DA1#{Ex|&n7uSaw$dJ%B9N}0#cj*gZd2_>vFyQbsPt;BMH6}_E~js#|} zc|Qi`+(D>2)6@EkT6V)6t2Gb68Oq7%9MnpcjZ%(h9 z2JJjvYn$7{cfpP9Q#mF&vM7g-dsQx5oydl`F>;{xP95$~>e5TCF|$3+cre2la3DdL zniZggkphj)HCB{bSLgwxEu})ux-E2R>{n>0_#o0(2b$-5^WqV~e(g2w!JX-wj*D#& zRew=#&{ZB%VQiL`mYm_wvSSe?tzR5OniFJOF1A)54v0OQdiHIH&-e!!6_u#=e2hO8!K(6V zUmyMZcki-9!pMTmA9zj*Hr57<3owm|m7vF(wEOGhKL(J!(&9rz9-zzjiEP&;E24iA zrqVc&260q@IIDyn{Mj&|s+ET{n&t9?hP|a5(MOATN{oitJC*w*vZL*Oe>U^rCqy_c zU6mkg?q`KYp-Aw+UOQP{eYjCij<0BBl(mp6Kb)KiTWOH212H$?XN3vnNms;Woaq+N zqflIIY2vEs98uW#0_^ENYW)08o0c#L9mABo^`TMr0vd;-j--3V=tc{^y&4wJnvXI) z-yoW%t$(#51W5WU+%2+8bY7rMumw4%laQM%nLao?Zj)1B;ca5WN>jXtNsT^G+^qV| zaRP7PpuPE{O}QyE$w6N8y#24CJ7fttjH7&zT*T=fU-5(M`CbyZ9;H&XDD+lTHA4OAC1Gkw!(ZrR(d`1R0B2%z1Iri>^2e%cR@ zX6UcY#Njb9>!`N3h=zwV!>i3s@m`xe`2pC2{@p&R{8vCe_tw+o5lcih`FJW!c6PhJ ze~?T>&M72H89llEGCch8c4Ql=vVSV<49r^Z9 za7iW=E30IFuW8{A44rKpD$9HJFBm$u!YKVwiIVYXP=1ao%}Z_V@~hz%VJP|E(C|(9 zmLt>IWbwvSMzMk9mFjLF-efC~ zyg|v(g3cMul2~Y^8&HS7JDM$uw>3)=|349{C(NgFtKnvZbB9Y|h;WY}|FBHB2xGL4 zB@XlI?}cCG=&s^p#{Gu&Ot?&FI8aS5<_sts#E$9JN4Tb76aRrhWD3seT6A?kR8L)a@p$%r&I}-X)y6$i0Q*9#UcJ4&UA5LF zTSJiBK>cEH@Chn@fmK0Tv0P_;>4TivEKeL3lcj%WL!LZ;+pyF<5(uOShKZNP#yUo- z9n*3E8WI0O`DJ4^l!}T<2YX0Hn%U~{rE&wPZnL;mbYfLBkFzYg_yZC?Z<=6?Z^8uh zRmloLLiLe+TdNcRQ~vRX(xo2Tp=|T>s2yEhhaWf(+}-7c+T{1fdf=fXG3j|gvi92} z*@2QaW=!=-2i^eA&!E+)W-mwdL^-WHnaN%0j)N7DGkS2ZS+^ZOA+%T;PlK|HY{|G2 z{l$U*E&Bh=1G@Tl8i7(q1-7P&lr-k<1y{SBSo@Xo1rHJaobJzwqliP5i_Mf8I~V7) zk{cc3(ou17sj0E%4@M;gjhNf>VTzx;FaO)k{_G?2c8nBlZJfgi;aHY!+%j_ox!dqG zfwoFEp<1)$Pi%32+Fp{4b`LP)xcX{^s3oH7L~HEK6oyS6!6TIDN54EVNNM`d?B?7d z)eMCIS>1L=E%=>S{b#9TZyN1_CC0z|NZ;Fxe}vML5v$0yH;!@iImDgj?fEwbJ>#;W z&}}FOEAK>{cYF>NgdOZy_@a(}-wXxFTtnYYV-Cx6YwN``(b?4~)3XI4j)GIl`w8-^ z0QQm+!L|T99P0V==P$=WIDx=vJ-wAW3jNvV!+hC+x-S)Pj(i7B$hlbEpoH)CU=^Nz zCD3~J*(c(E#OBwhKTYuchTScyu?jG;@y@#+@b0JiTY@JP;QU6VB0l{?Akm63pLY5YhuI8}y26x&kCs z2mr~hd}@GF1LG5DC};nQ)HK^N$PwhOL8gbiw+J zoR(DLT(Tkx)t{mc_;nG$7KgtI&_$)t_3WN3&(YlGgh%B0kw1CMor_GTmL4851_gEg zYwu148Z#eY5^y_6kE{6F(z_PY>70lqaEz5f9F8N*~I>1wp&UZEABX)Af8|Oy_`|aN< z^uFYYbgM|XJpXecW&Ih{Jnnc1ZGj;D>t z0oaD0zOg8?alYBN%wk@Ex)NQATV{WM|M)SpmOL)2S%~9szM4`XE?ZAUl)}9rr#F7` zWOA{78X6ijvD%HeIg&r9AElAO)Wa^Qq;C1XhxFZZV{xAdKB=5G`uGgR=I-hD0wNsR~FpH>0`Ab1a7>F$W#5f zPs%KRZW*9EEo ztmtqEh=bpwd`$;V25lCqGB*b{oi~2;XxKVNiJ41h_xX`e$h_EoZIXA77N(L;J_7sUY*7a6g@K9!wTaFs2 zN(dz&o)_X9L2Jo+O#3Wb{IJsY@VKUijS-0QURiM)fA zu;7-4Iej@7cH(RL-MX%ftAD&b4gldQJo=nAYi~)xkEdY?T;{BszM}C9X^b5foUNq> z=LzoA?D7T+O7R@YYgo0DGPaWsI{&WKsRuynqMDY8aHAm<3%Uu#)bMuVElh&`(l|53 z<5b?8fWuJQuo!2IMv-yiPl=^r^Qnh(^Tlw1h8nd}!O%iWK(NW|cEv2{$c2H6Oe!%? zGJ(_hbZR8AxlF*^jZb5RYNET#&gKySIWfx6{>-GP#dP>ELVc5}ddB zU@TLbxSW;Vq{5@kQ8n+YM7s&&0+D9|Hp2=^DowRF*B8Pt`k_3I&Rt6ybY?Z#di|e; zMmkAsUw?e69auQ{SqQ;2gTne*7(0**=iG2QV~bu2q(sDFE~;O-k-9oR%9n7zI#RmU z$h$Kv9#OZM1?=#Rd;9AQRp)dv=&RKu*W60(R5-wxW4tw1JQ7h@vr|0j?H9{WxqS4D1}q9m8f$GaEB$wF_0nC`bQhX=c`HX%-|XB z?pPKvCc~a^Y#Vk~Rzjj8D9r>1Il zcE>UoKfv_fcySAEE>t&eCw6j%PamwUu4+H~<*o{W@|%4s!}*Aqoo@(Ow53S&(~;A? z*;sBqJjZ5ZMNK$!MS#^)90fVQ5_=z0EVc&7M|K59v4gs4q{7@ zomvXK*w6kt@y+gT1K8al&nxj8w2MumIc;;QZ8r_-d>&(4by=5#rgIFFT#vIB2L=a^ zE)RM;d3e@jU>%H`ftU0$Ab1u(GTF3XC@BvHu-;kEr~-Z#I6Txh;^IS6d{=sP@rSu8D=ln1a#Pn=GB z61ei45HiJMtcDnFjc#sXea?c7J@3Yn4v60UP29V{n23di0J@PP?dHQdZ{&{4Vy8P> z+X9fHC=`i}opcTHaZ@kWT$36i@BBzlgUWkSo?y;Bc2+$D168?QR?`u2V0+5D2z1RS zN@bdT5cSbYNGyXMM9HDT@k54%u-@#%>9m$-%chHZ(lNPK#Dq>>e!MkrR-|2vQW;MU zcim(ztf<+8-iJx?s@QG}_4{K`>*23=Xw{#s;F^Y=VZXVLE@$D#cpaJCwkWnGC)U+q z2u4APV78>53X|auQUB<1+}2SKy^zP@|p@9!U~LQ8cfFFj=@m`o2Dwsi`q4N%R*vWar=onNMf#9kbY=|&e;h?FNF zr!eh8Jh;*@(Xo}YTYR2N=NsJ7`k*idm3&bsTbc@k7Z<8Jha-7-t4JzKh($N8ZS-j?J($HJuJIxXcK#F%i= z8A%a@rFPV%H82R@v!(3*h>>9*c!Az(BY!a;c1z>%nW`}1BM4ptve znC349I&O86B-dXZs{t~bo2Fn5But>Q_-BMOf>?;dL}sFOyL@@c)^$aJ-N`FtGUzHo z8})vO_+_Nqz01KSBRWaY0DccJV5*xlm%(R*ZQtG+bTS6J$0d2XUL2Pd7>4c^He6SC zL{P7Gi^*c$;jkx!Z#$Q||8Z{akk7eH!$gv!XQ(rxY5NT(dM3(DO_w_&RBIjfWdgBz zT`EC!M~bEwku+*U*>dEY==mJhE6SZaiEJxaEG1xoVjC$oIP-+XFTyu>+NoN#$|T}B z=wdnogtLPutVIT^tD@*gQChfKxu zIG0DTGrD(G-WXZ2RXZJRRIg{m%IWG`Ya#{R;jZBqQy+U>(j;PQHpiQ45buf@3UvE z>NkK17?=b%&sTI$jwE=4K<^3*rnx3vZXr<`O-9K1dB#BtKxxV4L6aWOiP*PsiBDKgA?3+|*0{(Rz#zF2OQgf>%C zG@q5Yh=c^qnhjtO`9mH>3#WInTai-2@F<$Q(IP#i=bx+q_qu)?8ZvI3VI+W#!S~DX zEp#RmE74T0+AOHi(*IGAhC z2hd8=?IJZnsm90b!_&EqS9mpk9WUZ6^}7}^RqPHF}11^aqHjBLjZt96qFHO!!? zF(;P}mvgYw(`c!yI)$#cUw6I*W)HrITb=7>XBCU0wPd>t5{yM7XZ+B`^R$Z6782#D;3q3mpjf>CM?FIKAFYRJ42#0i z!q5^#16}V{XaScqeIX(B?10^C{d)IWu!~+ma(L*X$jm!hggSH$%lKG^&OSaCOk(PMM zNjeS06E0h>vcmk9mD5Qyl7m_H7K#fnwl2CqYB(cr4dzIgJ`?*CbG2(mk#6j<{@egR zjxqR2qGc;~m4eOgjD{5NSz6=^6`do0OFmGvGe0GkcrQK^uQ!*aZVrQp0+U2l4AkgH z3~i^1h>Yl6{V-_8B2+6UC$m5% z>-h;PTW?*DvYLEj@?=Zk&JDDw;e!~ zm#3s{3v6#)L@me6J5kFpww<<2cvJLAR0`2XaYPWlE@KOKE8y!p)uslgV`ofn!&7mA zRR^(`ez#;NoHC~?AxiTxzOenw^NdCSbs_I^y)qT;RhdZ@};=iPNU|KuJt|D0SRkNJug zy|WEN%402c{MBpSTo>P#SuR)#i2AoDD^3LR<^N7=KB0ZPxg;^KJ?R`=qvYwRx1r3$ z7_$_}(fZiLER;7R+PGRGl#GQ%A;Y&5k!>nuat;98s+}vA)J!my&SoJ~4{D3vI-y3t zRPS~mHaK7YL@b;Go~<7_>Pn?GXiWr?{EqXY@~9u73{*jKoc22}2CdVJbY{GFmNG>m zD+5H%qrX_MN}iBy=5iIQNDmAyHZCeN=&IBKE?RM+{iD6EQ;y8vq@xy)5ZM1xw2rF# z@GZ+S5H9f=Y#Tg`x7(gjX{#bYZs$rbUblVAJI&^YC#UJMF9f85zn?b`cggB!upQfF zOQ(1ZKRi$#{rVo^Xh=?K9YA^IXrg46gVnaCV6D(|uUCyEHvonOmzA^666}0iP7?_W zkB&A8U-6kh`MbK zR$SHM+XAcYwsR^g*<;4D#e<(QA@#%DP3CR_wUmJk&S7u4a?rI5T7C*-5S0nP9YtF>EHx}v zW4OG-s_=0rF1S(9Ul?s@9&~K5GI@1R-ueTTD{|0CH7NXz^-$W{+9O>?Fha`810~72 z|6t(tsrY|n zh@Vxl|II_*(!A28fU*7TI`%tJ^bcbmo(()=TwY!H%R=5Z^0+Ho(WZkc$9#G)uDmTO zT59_d+V|(f-@hze0D3K`H@F0qg2}Uj&qc}=_E~Q`(CD zb>|1TPUA9hU~qGN)xjNJ;7U39j5;zQVts7v5jQ;ye#6c4;X*kjzZyq%D*9de{6BNj zkkH&PE{U}qMRY>zx$eicIo=~k&%~{T;ue!XJYwmC5nu56ok;w(VJO^Cyb?1&8JM6- z3Wj{)I7bl1bglk7q>NqLf7ed^+|C|Mpwh;0mhbaD-T&JYctT(SsmwXH=#}5K#rFr` zQF2=^5ME;Z;U1@N5r$Ngt5_RK+}6HG-g|Omr!bF^fI_tR~LomUne>GbwN>W8$|MEW!63kVSdjXHbj68b`V} zZM!8-mVnA+v(@-nnHoFLVADq%uJ@uZelJoAX3cR?PLt|xXFLbsOHEuoyQsl297m?W z)g1NZ&z}!K2H5v<0cQj-P(Z!7G2Son_9UiXu90GKZnei@Fp!;N_W$P&R?DrTJtjfA z;{iOsV4Uh~aXFdyp*z`qr$}(LoBV?4llwbq;C5R}XJ|otzaUa{ht1*D!}g127Kc;v z$MWiooI!VmVxADU7rqGDLJjWZ;RBQ)EN=$oY~Ualfs|Z8q#=wBx-t5%83F~T;=r8S zNEYt%O%w>8KxL}YlrQn&UYKBbLPGkom|G8Gs$+m6aGR@3!|-FHu`jQguE zBush$;`ktLSj$sqbZcdvny>^-^gO&5Cqfh$v$$H3>9D-260#~@l`9jrLWPBq zV9(U~-?#Z2Z!^weS}q;-NhZf1cbRp%(k^S1|D$)nO+U@Xk@h&c*80@Zhde2Uz8?o- z3@Z>%<6b`S79(V0l2a)d9|BUlOH)3pOYCfRM1OaINAz}bsavg(H&becprEOlPdi!1 zw>Ac@L=Q?tL_}0Xq`xOTEi}O>HC6Dd0E^=fy-oepBDlYEXmAv_@x^Tghg>~wraf^1XoF z9^$1`qg{L4|B$OXm2S0C*;lXhb~q8>*j17NS0{-y>tE+`_5=Q%9sBj`zI(a+X+;19 znu$6zO8A?5Cv{W-gi}l01)ef>lbpR24^RB$Yp(} zj03U}5fSN8noik$x?`UFme9c65`_Zwss+3b20$=8w4i~QybQ?HG^hszgi8ViChJq| z_j42_L_tIQY$cs@0vm7&;!#X2>k(k|Y32_(dZe@E_`~m`A$JI^+L!h}S6=J3Vq}%($Bw&|o z(4#~IfV3cWz!4Key9lFJY;HaCN|dj+IugOrQj{Uhu5pT>aM25y5MLH@@nKtKrt z=moF%?Y)KnLpO^Ofcw>x(`ZotV@HF4yflYS6WX5=^XnAf-ctJIHKfr0V;*>bnX~@p zMf%_4YqSHry_vlj+~3jM&*NezLUGgUzYqidBOcE1Pbt$}BMuE#e8bTZj&Fd=l(CHZ z{z(Pj_5Yao-5~?_Gyt!P2jpv2e32-49H&1~bDCw#nmL-!gy-3A#!o)<{RC@P2|%VC zxlGB%@?f8TZO`fJ+cG!+wVjcfmbekFDZD#$ikL!*mxHkHiRb%x2$*RMV0msl}Q(=5pK!t?%MszA4D-k;l zxIHlt-7CBvXPlI4wagxf$J_BTpHTjO33I0an~&GW@Vs3Kj(YWm)?*1^O;$H6gwJYK3}_l zA^NsP2(BKB$6eM;n@;8KBo&D-UnE?$l14KlH|HNKIjTF#4Yt{%C&pug-l)LGRVguI zMGIA*=Ch`$v}F0bY>B|2aoseVwMQf;77vdUwLZ%PBZlL5UnIiw#p?yhZQ-visN8v7 zpzrUwbAoyxU)Vc~9e|vNIzLJlKR-FGB&H;4*4w^M#3dw>jGak*7gB#&A{J1c?YG2c zc9bdPix+crEJg)WAcR~BtqY!dU0wYkHArn53GbL*mau@8O<0*BB4V4j;tHxv2mzt#~qL49SHzgWqgE=|cXt3=#>kDy^ zqk|AX7aJdMygc@;=UR#FLWj%Na6+f7sMhDtNS_q>t|IXTDbQm8MEV=+owC!LGY(Uq z4N8Nr+?pGJ=CXsz@9z@$-Jp)m+A$!fJEzHJ&e4HPYoJVumOCK-WRvQM@kFOCQy!xq zVz1BtLnWXH_)^qqh_0l7s_(u_QFQ=3M}V+jZe)Z9mFZ7>8e_dww&kel8}t=d#>~NE ziY4+3)u3&1SPGeps=aDELnawU>OtPdaRg^fI_~k&dg=g4)*(mRWkbpLbc#v_Tj%{+ zil0A2bd+m6NQ9Fe4|k@0B}pA9D-TH#V$DSD9ai4&1cU~q)u$?3MwkW zsy0V$I<%5#X0RZGltp!{JriWk=0Ku;z0bCk8D!Wh4%ecP_KKV!x$#+>|97aygBiqA zh4Pd~ejxz?b^wEH@a*$|0)P6fgOUsv0l&=0BRA>kr#uR220yXXhjPsZmE)(qBN{=L^E0fn0cjW%F@BR2qbsL?_YUqN%0qewqDaDc zNhP6#H^M%*4qT1L^M@NP=`6yB-hHTl73Aq*ZB0>C^_`eRp{3mn5C5FfledNv1>c-c zhgEwkRM6bHWS}m3zqysZ@iG)pHPnR7=YtC&65D`6qIxcz#&!EgVishmZW&}>&=(YK zAxcZP4iYxNaf*pu2(7LMPDe6e96052iil{Vz(>#;L^yv>@8xK@%bnAtOT#6+Tb32b z!;+v%Kxg<$&SmIV@{3stteF?85z5&k@xWrbZQRAUu{f+TJ3C9gVA7ZQC0-g95B_&F z-I|t!@tAcLSs5(!W!>1yriXTdEJhi~%-!h>R`~4#eRYA)>4l6P^Y(F}NF@?ybG|=@ zIipnGZrG;x=||W_Dgi@4es~#H$N@^9ygwVw&F8dk7Fr}FB`MF_jk;_-&2plw{{?@G z$pPo=T=Jl_9=P+Mm87>h3v2*Pt>e1$^C`YrN9;!c@ZIEOEu@lM7|466&zh=`(DL*L z@L2_mCwv5d4@piz$3WQ+QTSGbGAh!-VnX>{78M7|K%e1H7xn-|+m)p!VK$h*km&L< z>p2*&=O0!5#^wE=BTh@*PD^DWZk)ftws&t;hVtHZoG?Psn+X!JgKjBx0J{r7Q?l8^ zy`^4|JhFrdxE(qlj3N)BPy`iVM1}22D#%=SXBlh|J<`A}p$u3A36E-&%jwCCvuwB;SM?}ssF09Ve1}GZM6GcF?F8Yz7Q!zr$!guZ3H9J@V zDUWGiG|2b$#T3`qKSFWDu!L*9CESncaz~yX9~%}mOn|MtBPzU*`LT}Ifn+DWOUA+8 zW=Q#Uig5l-_tqB!6V=&}1A3SBMOEp2eGlLT@MoRdwfcF>Wq%|=z4X;bOG``2{xf!M zobk;a{^F5kL_>S?`!F<-L#U)_R3;C2Q-seVFkOZJx+P%ohDx4VLH>==4g-3T=DsiVTTv zZS+RD2_{p0Bc11L6^a4uS4S?V-St_=y@dkde5$2Jzian%pFyXcc9T=mI{*pUsZ_~I z1nuX6lO{N~+!q*Sk-x)r_}1WzNnian;BOdwzFD=QGL#JJn^Q(B84f6=XW2uF4KmJb zQs{1|GP$t6w3`b2*qP@rq0g$yp#d`PIfT??QEs2Lsn*}UE9Wkp$7cQ(Bu&`1P8nA4 zBL3^wMxzB$;F(DCYPsB(1$a#gVVC0Od+I(nG2fxRf4_c>X%q=1e7M|8TfsRm@0rA`GfsJwvcy_D&h|wBuW*?D8rxscs}P~W`}Z}LoyVH=I(hoMb|-Eh56-7 zklG0xQNF8vi-9BoEcP=W_tL7pKk@^YY&vpOokFvLAR$nnfxDYx>0%BcFa1>Vic;$4 zeBUBJK?>0T=UCERfu%QAU9tV=Wbk@#)QTci)8ZL;u0bZgjC+v;L_6%S)Cs@g{uq9E zu;*mTywG^VSLmyk*FV0UQBI5eN~x;t|LbT_O1++hXZ>4R>V}9tc}Z(7j3Oh@p~!Ocw(!|=&cEkY|yij^md~ECtU~8-1D^!;}Kpja_{{&yK`zfoH5V}m9;3PH&bY7!+6&%B-{{nwox8>LAa6+RLa;pMB% z6Q@%}dlubdLngv>Yd;j_55mJ%hq;C=>+mfrqQcw!v;=L7{AF5AWu;E$;N_))i|2mu1S zvA0x#0R0ylr7xCkk9#V!iyG7u(4FYT>j)g5r2+-Pyy|ONp;;UkX*TyC@UFQbIDYCi zMw5U-D}^)A-AVu2!;kMH+kJDU5#QM#YWPy`Ox_boUEjBzHmcdx;dWbWbsTGv(8?K@ z4a1}6PE}A~nyJ=|sMY{tUV*JE^eK(WvPtAb z#5ND1F?(7iNJJV5w>W~Qb=+9t4LAkFKg7Ir<)Mjl1R!XY=3cX;K6!32KU9ipje_>t zm#|=De$_U&W9o5uGy25k^)+KNZhn|;`?Q1%~iQ5@zJo8hJ}>z_W}HY0=tZS^Zxj2-*uMVM^<#I zgpz{`LOIe}Y`jM&Vm~*n*3x)2=||53;$)M7bOy+I$s)GY#uQI5O4?@>z^t#Xr5Rd| zpW6lPO5&&=y*`)+_b4W&%59l#hB&D0(q}R7*?NJHy5!Twp!}fzg*xt67nC#u(0y zzf4Au4|)Lfe}v@9-ZwtFMafn3xUQ3)b9`c;$k(;}v6-ND^!D6}<3Uit`Znh!6Hk*< z6R3bM%GGz9pK$Bvo1AGl#eE5NJ^#|pO{pI1LH$NEh33FPhNyudZ#_f?m7rZ474J}t zXR);+Xq3=9|Je%JiUBB&bA~yw2oGz z&|fyALg*e`mj7wD=IG#3#la`rg-2b|1v&kknrS8qG6Gu*KDXa0Qu?kVU5*`whcmD79pJ=GTRlx{D(PC_0& zMF84e9V%4N%IRoU6VejQU&s{^{^!CaW_QL?s|Net8*C=Fhl>%7`-+}dZ%l1@dmdPG zk7~zfm^d!Hk94l=?$iru)_WAs@FAcnM4?uIZNE|JmPw^lYa-Iq<)yw?ljs?r3GxS( zczsXCiWqb<9UuAZIOnF@!6zS{TnKNTR`$Ei^6&d}CkBlzlM<9u;WR(%-Js~nxzBi) z%KdSbf8n{gf>YjGpDORm3XPGIrTYX?ZnV=Sl42r=%(7FbNe2%}FVi0D+c)NCguEvK zCT$-oyx0?by?K&|IP$!}Kd;kJu*~iz0Xo%z`TZ&Ea#Fk&wGS?7bMW2ABlZdb#lOV@ z7_^HWrP6m8yH7m3Umrg%#Fw7?51g=+1*GP*&%_C!C1kl=%sDSvJ!kFGlFAcwDI9pR z)1WYY;x+8ShhKxDUlEi7iF&g&()l;||LQded(IBsymyic#XL523uCpxOB?c^MfW#NM^e_9ePCZ|7}4tHEdc(hc?Xa;B!KdChK^2SS})rr`zvhV1 zEfoG(PqM7B8n0X9;o;>W*JKqotvxh*;c-Jrs;EB*ALiEM2}D8q8fsZpXLzHMb%Hnx@m zvSA}}{SfZ<^w?K3AGNxdn5*3sdYDl^tA5&)3>&K3Y ziHh3NvSu$T#yqaJ2ipwh8rO{02!(w=Imsndnjw;o8@&r4NCD;k=Pz7Rc-t3x&nFW( z^`Rc`rM~`vsN8k5TtZ(HJ*ymlT@uHIfp1R+!@+(GOcL#|^uQk(VN__#01(JAb0+-1 z)CwvFf-0g%cE2HU7`+jzJjbV7IljLB4?Kcj76g*L4#1GA=b4-t@{YcXdDqD{zXljk z{*pt^V$_DQ%j%A!-pnI-=p6+uwo*+`GB&0}*1sB{fOEr^soZRUF;F|I4`N&~ZCV>rOdj zOBDXk$s|!}wKQ0PdLJ;7P2w3^1^HD~Rk>z}%^@?Elc4(7%0LPUF$oWb4-yj*rGc6Mlkt5srEkacny{Q4+P^6Rt!P)_*+IFbU5A|O}yv#E|UF9QG9pqN);(dYUE z90KHG!T?UHgc*~)_0jLW-AX*0}s4+Ib@`(n|D31Ujmbvt|Yy(-AoYS=6Kdc)F!yrYxe;ID7`@N~gUGIj(~&(g?de!vd+Sa&bvRYgNe`$PVD(G6 z?jNeU@chs#TWg1f-PzL6aug@qK&&}Oa|dmn?56nnf60n^&j_Z}sP>szPp4ku+ueF6e3Wu;6RR??FBH0oT8%QqAe*#LqKVErps!r zH;oE^OIf`N+7{qpLwlwpnbsOLKtHo|dbkWNB!KKKD3m`kRCxASb*NHtnKm)>Zy4pr z23LrhK2 zbGfS1ePbbnhdDxDWH6!rc+C+TF|}TYs#v%M);*W*OR=l=glRmC%s%*ffhv+p{y59FOu6;vdr6~Mm;OSsLX8ZAQz5>H5 zR+^G~ob`jABG?|wd5YC`KJ4Tzloh9Pq^W+Tfqt2cQ`jQF+D~w*-})=Jg@xmz1H#1> z3iFsMll=Xuv z*c&!Zwhl8UZ9UF@aOq=N+1@Yi@AK9ZMo+)v^CNtUWu;(?xZeoKvKXpKZ?jGn# z(*T^{s;8_vxrUzrN@LxlX25^T{0&O*8}C(GG{B6zlan;9V_S@Svsnga15D_BrR0rQ zMWVW-<1khMJT5WaGAP_YfV!x>i#wo<#k&7x2#I6iV}CD-E#$)Rxo~BrVZLP=&+~8I zm}0*PBVAPgoL}^Xo~Y=J(^o(|%mPI6t|VTU z-SzF^C0DW#P0Pm-tD4mNJFRX$8Lk^C9IwVI3zy3GEFGB5h-g6<1{`%$IDmx*UObyE zQ56wixj=w}g^hzpKr8lI?Ay7(kdV5Atx=j;nn#_sO{@i}9Peljm)}EL_@}D@%O24M z<2F&|S$jEI|H-D&%Hg)d>Leea6~<&|-w7#b=K&*mNJiKA^NPpOUqO$03JW`pak)>= zw+0j&5->zq%bOMq+pbhyV6U$JNJKH;KHr~%5H7G^8GqD7ky{N=moK&=lI zj4FZKM!zNeszHQ}@_aEOy&V9$rc=$RQ!vf?k-DnB&tX1PXlmOgBe+0}Dz6N;AOC0q z3A%-P6gkuiZ~TB^p6s<47Typ@;;D__gcllt%z$ZLx3U@`gYd(R%F~y+KGMPilXkko z4GlM&8#h2oieT1Ej_I-W81!r{pjb!JneT2bMbX)IkPDD$+dr2MpxKM?7ct6jvuYJ4 z6I?1?tLssLqUM@ApW8PWKR42I`FKm@B(aTn#B7voSkx5ll*O)HRa91d2oj3tlbUwb z|HWqd=I_sIBnS#bOl~~cW9s*(ju80qFOhK*DA|mJTl0fL7ClhTxl*>|lCjB=vL2tX zzAYd2v=gAPsoD!~kp%(%JW1MNMV)CH1*u$l0RvJEp-Zk?C3&9o8IB8$bw*l|>sgmk zvN(Zr(}H|@>elf=J)^?u@OphoRP{=R5U(v&dSgX0C@*WNkePk@_?!@Of9(@BjQsX9 zG3U*ks5Eb3VSyk*PQ6;M2L0Ok>?XFn>9Fho{mp+Ucv5*F0RsY4p?cTL;+;EUAFqHM zUh3xDO8ii3-qA5s00-@WP4XN{A5KB_i63QfYG>NJ&Mz04g=KXIo6SWOjCf~d?B~3+ zm@~&wzFY_r7TNybpY)c`-n=7xpq47UOhl-RM<~splTrg5oL&x`MWEX`%M}#g| zvNGZ|Hds|)D46Oae&MU5xx`HHV(YvIa*c=>A!|1_g9LqvnG$-P z|$dRxE&@%owjXv&(=JoFzXt_HM;hrY;%BFFp88dy*FBow!Z!^dR0K)d~h3^n)Iqa zT<+KM5#H?cEgSsE5E|u9m~kAPA_#t&8y$38MlB`KD@J&3!wyyycgHfwIdVgQ(AMTe$6M*h0YN zs0Z>O{Wnx+pQc!61lAubjO)3{gTeTYbvVqTl=YpK)qR$&TUGnJ8#DOFGlMm@;Q9!y z)<25M8|LYI<{m2KU>Pw7)X3*OwId^_A0eC#ngFG9$zW&NB_wuH70~@R)RN4@-g4S4 z4h;yTETSxP3TayB3dogJ)Nb}BaRdQDlf%Y#5M7%(bfWec9ko{ZXK&ahtA^o_b2ta5 zR{X>9-GbWaFfCf)>!>uBVQ>1dv01`uVBztjAD)|ZS=C>1+43^balNxC{48p}v-1W< zgiapc2Y2-_D`~`q7tR5lI@VCk-@x!Y%zj&(7ONpLtw)#H$;pBjMPz=Z4P{w^M@s$M zM{vkl{kD9ixdrZ!5u)@RTr}5iTh*r8PbC?|SX79tyk&P25wM+?BZi!NgB~z5AKRbT zggK7-c$~IzY!&2sapB(9R*m-F;$`H19YwptLj{0e7o0t0=p1>(;ID4Yz$x^Wq;peO6;3F%qqdZ?6Vw zWA=~y8n8lmTp{RPFCv>6$dl6C&7rAbcXQQqo(jlAllUaP_gK4Trj)B3r-6UmT+Yx_Z<45&_YN+n;9nN$*QWU ztW4Uiq{OjzP?TcxRqjGCYYJmmI)e9)$B?7NW(_V?17CUqe2ALZ6$O=q$Fd4(O)SA- zyWFEG1%21)q{vvki9@{qoW_qCN-8z7KG)4mFXJjGQn6%UNh0N0uy@#emBVfE4QWLN z1zKNEj7!{&XK)z3SDqTLbfp#3A5W60jLpikHWf2M+^YA`tY;SVmcf}vU2vzM^2FP% zL>>4C!q2oKqu9iSIgU;ps(GGkeRNuLL}3tdy{(GVq9@K|?$0T4@E(n9@R;$OilE1c)3P_CyeYZWzvzSMr2gDo<2}{a(wm# z^%1~FSTxN;|GXbn39f6s_BV7Khk1|jLno2+T=2*%Jk)irv%eDz;x?T@1oV`*YOx;i)ptWiR1qG0L6HzXN!+Q z>wX4H-+Kumaxxdh9%MaBqB>Z^to_BQ|2S1O+6;q{eJ6dj`Qkl;@edi)$|i-=mFyCd zbESem%LeWUo`@o5Lv=~&5ubTso3kV;`}+wt^HJO1X<}UVY_k?F4t6owohX%9GyzVB z=!1cyjjb%ko7`xIDV^Tw{kLFZS_R)Tr$tcM(QLq~yVK{dJ5QWJ?sfxAMaPxZ2efj( z;#~c0CGA4%tzswv0Y(MHjPO8|63>NALWlYl7r01nT09iq&~$dGH-Zc`)K$y6ZGJZ# z%%?3LHg%7^_NPg(#$6S;;NMfoKV8B8fkx97vfsMZ!aV$mkpE}~MhFz5H~OKD#^Gx< z$-lD4Cl&ZrOq*R*b91SGrR;($tbizg;~AssC}71QL4>eUIR!vibEkz|JnY@R72qUb*CJ&$X92bbxR>$l+VB zcUNJ{Mo&tdRC$TL^yZ%6-B0frSkTL;B*$G9aImT^N)X@eJC&_VVneYu$aX6DzA&TSnk=Bg&@8}OJL44OPgk;3?&L+uZmdaXA z6O^())zT^(DAFWm(JJPW0gnq<`TA^Oz4XVscA}e^_rSIUxt#&cBtATid03NPvPdpL zWuwTVnn?@#0JT5MAZ6F4G#dBK7aM-x4m1_A*(9+daksHBIcm&|OZiG{ldnr3W=0A+{d%^?NM!VMfm-~DKn0nG#sp~? zDJUrRP>>boMj=V275Y z^fbL6Rc}*w2 zjme=rMfRixtUBuD*Pmwl5HKrMppc^&0F#vGb1fu@gXTEfKMDbtPcJ~4CrKEDx6>~z zVkHq1>xf=Ti2zpTM!JX3b<~IT8RFtfXb9mNIt3 zdUtc7WFU^hyzsfQGD;iyaeo^NwMAlx3y(%}tFFXy>(_c+4H$OV-(LA7f^TMY_s0}y zFje!;?9UH82lgem%D%f9r8AE-LA#NSP1>?+x5}vIKs5j1Gp9lNrlC?98571VV@|?D zrZS}pb|==X=U^Ok>A51}Y;XHqeK205zmWZR;s^4ztQJ}5EIRa*jGj&euJcd&-Tw+2B7E}a z0{vBd0a$UvDbHRS271N$ek5Yh_th{1kQ0aXOk1J@O!30|S=*z*qkY5xxsXLe{qj=9 z-jcvi8Y(Vb0fYouLH)>AT+bZjnSy%w5C#y*^asGciY z_wgzDZbyRhf%&{2Ob(spE_+vaOjR|VS87&zG%Q|51=a(OmXs!>h6GY|j-eWKdDa6B z1yDHKqY>xS?DME+X(Plo$~Sb_$+mYhK4^gbWNrUQKcd&)@iGg*M7&ufAsAEFqb0Vg zKMW^#5nmc9yK&GoQEYnSp_gjfE@(2Mx!8Ul&~nhH$ZGnb`c}Mz zFSVM3fVrU{Xq0u`g}lUOrtMmZj$Vcx70Ox!z7yNE7^|YAscq5e%ZI*RbDM(4@+Q>g zZpHKdk4yY$WU_^4nsp0M=0}`j#r^)0I0#;Gu{OM8= z16WS-d0@?-E-G;<@b#QFC(sf z==Imc^?RRrWxbv0%L_Y2R@y#DOl-myeNJ!ju>jO-xa}zJ`Q2nn-lNS3;8hAzf^5iU zcv`XV>JK?CS&Sckva1Q%+R@c?+G+E@LV5SE@Mm>5D`|Cgl@B>?r8{=$)a|1fRbDm$ zhY%V2E@1QuvjQ9s;6DfOQrHgMz9_sp*6cPiW7 zVLL>WCN2E*h{%xdbZa zH6!kvMUM#8C9MaEUG>p;gA8g2Jg&D+HvRB9i+V1jtL|`Ha6{1W}ocF?13>r=@c5 z*qI~F5eu^k$j^2QsrfuAz72^Y*g9>l% z$O~t%H_h<>Um=`DA9;Mm!vBwED7cG!-5Fbdxd{MVfp6GrL(3f~==fSdGwpjuk` z>{h>9hN<0dNvDa2Ge!|Tw1G{Zd9V8VSkOR6fFfl3_O2h&XKrq8;FPw@PM&GSDCvQ5 z(>zAA`ZT>9GD_}EnyF-8vqX%M1;HnH9_~03G1ilbQi)~R4pU>uPOJ(uzjGwF${zybK*DdeLLQf>c>=uAEU!4E?!IYwYG zdxXd3i@}qENP;+Ky_4XbYeI%E8@|S6f%;;hnU8EIZ$-bTxrK6>{&^)H8I%3hn7b+P z&@e~c=#cEu&?NH8EIR1H%+F|jgd@%ObB;u=+nTP;rkx*7%|}j*h7*u9-7@(PGZm&$ z8O94|O&0#kTyTQiA=i!gc+4*RS~V+J_0ZRWKb<=~t{7m>IdzRa{&zFL^*4kH1O9sw zR5mUwt2GP_^qqh5Y^wQlmNmrTiGztcbYnke(Us5QuJz*3P7JmaP8cI9hd`e{W(l5` zODnpxR0y~iVIzK|VK+0u4sI`hzeHhre>MItIm7XTL#K!n(-VE-a+;8xFJMaQ+U1Q9 zYHDTHBo7da{(RnQoz#ZYfxIcaGa2x<$LxSkEcy|yd`Gk!^02+UY+cOc(h7*xDCXa^ z8~N3qLtBm_9LKFiD$i+luU1#%_fy8%HG`j*7h~$5H zt~~ZMsxi>amGfn5pcKO%^yiKHc01Lflh&K)WK+>EpgjP&H#*>GJN?&rxPv{!@8G-p zho})4x&ZCzp*b!53oSy60~aGWsQ;dXF>w+;pUn?h^Df9M52OB5=y22@p&)d!s4e%x zReuwW54C0(F`zlef;8+!Cj-Qy#c|KmJ6HjSp?apm*XSuF?_W}pn~wruP&dgtZL9m5 z+XrjvgG~p2=@RR$wsP^Mo)0i3WyRWp)?z-k?U917I!HKq`x`0D>iVmF{=U6e1(vk7pUsn7*)Q`|A;j^To6oEL?5-H`*Oy zS|AN}>`j54DOH`en@rO+y)wJT^Lb0u{aK=aJx0wf6jT#aVQU|F;AETT^4@O58wF0~ z4?lF z$sEAmW*wms=fp9IYE$eRhs$!#d+*8WB6YUqn?LjNx^6Zh^s3=@4>PERJy{hoc@i9Q z;`%*YPm1{>w19~{y$}fhOeI>po4wtwBicTV`s^2DoF> z9GE43yCs7WM|JJ7vf0kBgY?65LxoMXTG3WPnkNE) zxf9Xo(Kuh&`HrIJ7wE{S541#=TI<-&fMS!*44ZV4_&$PX5P=rht^}Z;y%+Uxaqs`* z6Mgy!*%}R^Q6F(nvfhUrQyW=X;uVK^b?76;!j7W_8KmntZ^=G^`95AJ8`E}x|d`(9!khBmQ&ZrBTJ8B&f9&{kpFw=%(Qi2v|#(rN{KS#Nue zM62OwIu$u)iyKgv;E>;i(4)^2M~X#BJCSK8Y#jd#EcFvq#*Y3G9CHLGB5w(jLID?) z%>MFkn5~E*F(qdHe>2rN~fNT*;Yt*=rezD^4w6{4JG#Y!)T8P<$N}oKwJ1Mx}9aMulq%X}__-s8gJ3 zIDQ$Vty@|tHBX(oc8A*2i62S%F1e#@R>E^1o2j7rd;{_=Y&Th&ceyHZ42j^z&wDpT@RGD;_EhtaWmV3Qr0W{mXpRd7X@ab z3qde?1Mbn#9Zbl}5dP{OxTxr41ZHE8;$7SRmsx=IQR{w5Esqcm$%fd|3DW>mBMBL5h~%uPou1{yY6RdK%ZE0(9c;jUx#yT z3UFhZd=$RYnSnO&Pi-|pa%c8(8u(H&E>QGH24F=k9xHvo!BH#o-P4-@w*bxl*jdoi z(x6L&n^iMBe`jsF*Qq@3yvDqlLK;c*9$MAX*!@TkHFB*Xr6LaZSS$_}xsuP)u&MhM zSVS8tim@G=T{!JB=rRqON8(?(O$*K{B((+wI%*V50o3thtIML5NvV|NVOMi>(vA#S zs%E4Q!;?^g4A2F3ABdDU=-kZV#Mxyay4Qb&Q~yc6Z42l^M+2yKrl#Ovk{sO!6GS$W z3di1DAQwV5bGE*i7Fqk;%B7mXEdv0kcg0S}t~(*IA?Ub}+Ur{O#=8{QFn%qetY^=y zk1`9A>SF**ZX8E7+clx;a882=jR+}9mB z2wGlei@SG{3xJAB3y>a7+0=x$08+@{5erHDt%92R6@J#+`@4vYIuu9?zFQe{dh(r~!Wxp8IO0(f59uW&+0^6I1S7ehsO{3-Kg&@BWSfK>*6;nzv{>Ed&XN zb%)woI)LPMQCByLi+EzK@dy~@p}DVK4Rq+qpr}XrI%{>HA!5#OAi-lqd#uVU-nwAY z``R{xufF5wd<8L`lB~--#Bpd2pkCf;Ir=pKX`-n>9xTP&e}9q2NlQuvxTQvrtv_5R zTsA*a0vS)-1T%nC_9uMt>!_-M2bHQ>luc=fF3l!HwTw6i^}zXOV*|zkLC|x(c-eVq zHAiKKF;4dPhectZ^m3dC`ZOXd@MQ$uDgGlDgS2-Js)=MDIIc}5`zm)kJ-tBUbA{u& zh9?8O=W)hGolc5G^+YL?Qg3dHp8AQK|GRqH%xsjc{)a|Hh*{EP&>HmS7-_{iOGyB} zW8$TvvT`lx3aLH#z!9b}FsE@LVMgHorJQBnMJIh=T5IWN?wi;k> zFOb%D%KE4;C@3jyaD4y4^~Yo&z5yL+U&I!0#(m~8f{wn+52Q>|E99fIFg_G52g&Mw zx+L$g>@XhVmblLXMSUG#YmI`Xi!H)9)&cI%O;%vE$+HGR`-+Nn$#9p(opzf5Hk&ud zwG=lGAF9D_reiYKZyPqL{OlMf1%0stOCLY>&rO6#?#A{y1HpEwLAKg?0b62_F)5Ql znqia!Vi^v|c=bFa#Vo&{{@q0?a&C(NW1mFx!XNKI8-C-SF3=+6>a4jR_!`NM9H`Zvu;XSe&fht;E4PNfJ$S>^#>1E zLn^xRxeZ#vRXY>%b*bkk51L^s%>KNB3#Bg9axqhLYW)J&Vr!r(!#*Or zGmoM`YZ6hO0mGOXK;BiRgXdorcC>De5ajJB<`s5auWzJrS#MW@B?^y#Y+Uwk94Jpg4s)`>Z`+T$D?n-wT-OOa zkepRqqsxjW)qGhL~-zLUuo(6-dH~CifjFP3f!9t z^s?SXR#W0g_+B^h4w?`lDK?hxwB+H|s$GCfDD@gcY!f?&H)*pv6>vJ=XG6L4-h5aS zQpY({w|zYK5Y8QU-@@VG)fn>^!PqYR3&2BvTOK-6H z%s2*J=>jR>Fy5pP!TC~`{naUiUVD}J)2C`zwMuN07D}m(&0=&RZn@iztZt)|!o=d} z5b1px;V*^mBuD5tIL~M-XsPT-*Ln#69l1*|nR>#@+bspZVMd2Nk`HL<%uk>?c@V}kZRot-{v*zXmXZQ23Gt(y|9^}%n7<1bVB`m6 zw61^rfi?sDVb#&z+6V3PU|dK>8x(4W#QhNU#~@|l0yJ{isoD!d zWaW6t8w2sJDpfYaoT;uJVB$862ZWaj_+~4df58cpS-_vCzT}gR!7)!H>!!v2nUfzd zFC0L&HlWTu#^evhIRzzk8U%W)#=@*anjwO0{rXgA`CC+O|{BuNFzK=BTFqd6>?yDv(o;h;R zgi7BOe7yy$vq1s4xWo>y1s8G^P-Og#C`HL%F00tx^SP?4`r!A>kD|G494lXAaTvBm zb1Omq+;iHtn^-0zTpwF8B%+gx06GNVvdJD5#9ynW2O-!0w@;wZGE0hgJ;!1Nbw)MU z(|0bkf4=XsFRJSxVrPx-q9I?-Rct+*{f=A)vsIP=s0^S`NkbbB=5wHZ5YMi84==#6 z%H)lnFZf*Lqz9yWIn{_B4YYtWf~1Xy?iJi)z-d+E8Op_8x|lA$1bVa(fK~*%>RpO& z(X!W8ibD7hN!;iUf51&nN%7M3)dXQ@2#-H0`NsYXZAYucRbR* z<(x$OcwOhcz?VeXj$S<4RYp>1l0pKh6eBKG9Y`|wJwwu2kjFwWv{Z~5(6;SjYmP2G`4!_Ri3{ai_%SV9kFM7uDWDv{?Lpc!4o?wS3BLONKJHXQZFOwXKa#q#LGmP3B Ro&x_p7MFRDe(%|v{{!S76J-DZ literal 0 HcmV?d00001 diff --git a/docs/source/_static/block_diag_cat_split.png b/docs/source/_static/block_diag_cat_split.png new file mode 100644 index 0000000000000000000000000000000000000000..ddec7fb1473fde618a93f570b465076d1d960053 GIT binary patch literal 29841 zcmeFYWl&vP(*_6x2o8bZ?(Xgo+}+*X-6aVSG`PFN!QI{6gS%@WSg_f7zwh3>HS=q# zW~%1LQ0LUyEZb}MTDw>G(@%d>l$St+!+`?>14ERO6jcTTgRlex1E+?821+R3ZQz4} z!Btp^h$u>lh!80{JD6M9nt_2yeoIV#uNuFM8hH6u9Qq{*1)MsU^O!y`VfvlqWgcoA zSnd}T7zlS$C#c2Xa2Oi+T4^OmwQvwrjiJuR#Jn8%+OU#SNhkg%6g|f|M4^4@_trFI`2P2jWmSc_5ws<1DjbI>_ zLh}hu)YCEq?ZDGHWY-)!D(e;&(OpFI0~p`82@G;Dp;_^>Uv%;isG)n&)v1AFm_(87 z?~3Q=*%*2y^>bn#U_NAhn+ki9$@tPW5E)dehx>(OJ9bJ(1PWr8;i@Yx z*pmg!Jl&7yHZV)HcSNUMhI4M>(JJU*nMimwR5p_wE}3A^kL2IMRdXQc12LwvE5@O@ zz%UDp2~h%7@u1WpkR%~Xg&@^GHyEN(fr%Nu|M6~An4AmJx`){jqCS{)3!)jqK5#V$ z6Cv1$2|DYu)D@Df2+Ehw!zfV7!SlrU!!WghSl`k1z#oNs6hD51Cl`W{M5y@6AtI&- ztBRy1>^40$ZFL0Uig*%qAwoJ0`3vVaWWFJ)>{m4-y&oUVyJdf1B7N29hIQo0fIsY8 z-C}9R5rAvz#@xbnWaL8-3@ug!BmbJ0%ceL;1wD$09vCh@kS|(tTk=Vjx(sI_%v!=N z_aZMuiBXYoMw1OnP69U{e|qDHiS2!6w0;bOcyhkY4E>SJQQOgnUu3^&Ln4)Wu#C=_ z>14?$!0T{S0;vb^2H8v)OuU&*Qam*2l0p{xR*XAplzxz%;n*_Yfi=Re2A%gr8y7or zWuQ*N9EQPn*Epp%L#zp`DXn>};jdv`ywiuPif-!m*~$M2;Y-l@(I7@JwCgkBkOG=1 zEDxm5C$J!yVPad#d6X5{CK!`1R(W_zF-MXJ6KgaIN*GC`PUNO`kvaHEC#m#r{+(MpT72LeKo=fFOo5tQMO#QqO|m@SBOx>WXAl7VW&6W zd^DMNA0C0O_t7i-Yq(r^M|gI^ku-tyh%|4ab|L{yej;Mx=L8K}Le);tvoW(n6IwCtcTR5KeswywW&21BFQoAqB$|I0r)Tk16kjw^Mg0d?ODgvg! zY`U2wIk*fRS5Y|XIb_*DOzmcnxXV~SqmZy}>3ZmDbVxLnI|?>aOvl+KS?rkzm~&Gf zQuk8p)7seO|s zr==yO%~}vH2e%2oWjUo^%v?0EwY3ukhCpjvX{>2{vU|c-6(`D1aauq!Ikax-xrkhs z@WFkOeL}kn;KAZ0t94{QZl4{iBU@;>!;QN*6zeRpwZG>Uvye z$5w2jG(tHCSC@Q>Jp^SE2aFbKDEw^w79Vx5$sa==lAZ+aSbw2CHZXopexzG%upKrS zGT7S00fW#4Z-XF!K7lHF|BCtr5e}gY#v9d^pq-BT-dhc+^}0LXI?`7gOc4$^U>8&-;Bx_f6}Fv0EP*^3d&s5CkL^+ za2wi0jUo_Vo3uH)&%%hNtgWc6-a z#xI*JEq$qt)rk-3t2b&loCh)fhqrnnL9Rh1IRO!UNu!rDn>SsT8`dpun~POn*LqLI z%*6`DvLm<$mI9dGa?e5?R(G1*9GaH>9pAf$8cJ-+A7(a?JCnDN=eH%DTQAFP>NP)| zXQ}yA0|k)c&{q4)jWNxhJ(dhxNBtu>0=J&kXNzY1L>W zXarw|-<)zr%gYM2n_7RN+$k>RQnGKe>j^UZm+^vpfh9^N4JPet=xInku{}LGQ9Ehk z>_*%P`IgTt;XvpjxVUJ!rnw+$L&Jj<9T6k=#C>7}j(dIiVtV5_VuVp*(lVoztK8B3iF>Ez*=2L`Ff026#l5}xyQ5LP zi`wDVz;MJp1C-uL=b6vr3HlT*W?PEhVHJ-)zstny%JFIBX$7aPef?T}=kjy;ZPmm& z$2xtJtwY0lW83;hUH#!2{vj{sMeAmVbBx=icvXOdf`R6yP=M^)6`04?@k_!}6@e*# z3c(uJ7i=0|s;BB-v7`AXn+X0kIE~l>ygLHUH%`O5JxY^DH=C^fZ+uB_U!DtGb^Y2e zN(bgx(X!B*vgrdj?sOk~e~a}HItQEul)hchXEOwBWJG4*`H*|xp1b|TE##Eg(C#?T zfXndy#c;d+>!}x6n$TL1MZn!J{DDoN<2UPt%QEI?#&ZL|E+jvhmpi#(j2a6a@Q-

1a#4cD`kvFIe=0K7iewV&nJ*RzbXdZxMK9bbE7wWL2u@790uyZ}f+@MY}87 z0_<0HGfgRTIXN&I;2H)DJk$yd61V~fzBs@a4D4N82-thz7Zv!5=0N;Y3t^e_?w@OL z>OT#IRYatufL|38XEQT<7fT0MVwOGuV5mhaRZUk-IawYP2RjBMQwL)+22VT3KV881 zJb8diJ2O`!B2PP8dlw#0ev<#R-~q1x6f=?#{ilhm4L^ydoFb8kgR>bC2LlrW6NvyE z5fKrev#B|cvZ(m~b_f3BC$V&Ob>v}W^ziUt@L*+daJFD%=H}*RWMW}tVW9_F(7Slq zyBc}Y+q;nd&mjMdBWmVi;%w#UYUN;0^k-ZnV+S`^eiD*DH~RPQf97fCY4!I`_AdYX zSil1^{;6SPW?*9ccWj_5-=9(*MJrD;TP;y5J3u_ZJp@>|Ir;w6{{K?*_m2P7Q}b_6 z4rY%3?)hIe|L2}+E@sXm4tBsTT?PKG%>Q=&@5=w}$jA8S$^T0d|0CxAlmc=Vfa7EQ zcg+Og`k0+6z`%sSq(p^PJ;Bd2-=};Wm>Y!Wa-I~f4GbrN{wOvQMq~)q|0R~lTo_6e ziuo|I1WQP1ehL&uC8a2g@=bbD%bD+@x$W_{-CNzoh26$-X6kxVmc^yJg*VT$_a`|w zH#cyRZ;FzEkpF)0L~?RWtb4=1gTem$v4l{a_9XfD-|vBaFp9!1nxWM^WI_=Ce4wHZ z|EmjEBp5nds2fbV$iEH#41)CkjX|(WzZmY1Cf2&0F8a#R=r&ig+5S*0Q!R~i$X>_2 zJ71-Q$7D!w-X9&YD$X?5a&vZeHhsL_ueDp{;J%$#DYRZFOWL1^l3~k=ueD6I-S_mWKC<%5xLFpUsOSwGe^CI6yU2c zzSkp^0*57U4LW(Eu)7&9lgvpBde&O_0WY^2y9*iyQM7{db%s5`PWvCu3~z(2O5TSD zHL6Gw;XsV|zvkNi)K^d!e2d2CHRroua|vPbd|1;-a5j&`V(uLtACH7CJ-1RB4K}A4 zs7~_I$2wT|G{qRw?klJvks|e(5^Q^4S}}G)RC&qv&__B|%DK zw@K1<9wCQ-U$JXir;tnMNL;S782eFU0_E^5`0^`0!);OP8&#TDoRBP}A%EvqFg)MT z&KkexRSun2t?0pcD(PXSN0#r+1m|p-8m&T_W~cAV?UM7dS+dmI>x&6D-}X>RXlNy& zICw&E($%%5kTt60;?2~zrMd6KB70vk^`}Xd`$7=o#iMXWTHQ|fZkG&_V@V}WEV{gz zUFC*~3xeHe#4tx>7&{WG&=#u<7=gt_yA@}+?0l-^yXt=TI4fEQF2W`s&wJUfQd?WQ z_ZeB>`$mV)>B;Tsl0pLcC;QGvb8H^h64^|i1*-*fzu6R_OdhwrIR(K9iLK#SQs><$ z&J({Ff!`(mEyKxF;fQVS=cu>Rcmp3sWWq7a1^j-)gm6-mp&$pD^1o%c^Ot@gv16iE zDT;K*?e!Z-0DlKE(In)@3H0-dhz`m8fC~i7W)@$4!Hu@ zJU=GUbb?#O19LxPH{2}bL!=6>mXnUa4c0)LMhhz=W$Z3>9i`Di-# zdqe%TcN9mJW~Bxj^ia<%@c1?r8IJ}{#JJ;`WGGPJyx9RG%~=WMv7K30dAk)E;7SeF z3k43FooN^sE}?Jc6sPhe5bT{5?M{=6@@+s#Q%FC3K@ZA&yeG=xEq6nb(q@2b66*&TrOvlwx-R-^cxAzga^=R+b44;b@@x#f? zsGWg`Q@bCpD>hZfCiBxp?)sI0D%&U?e7b$EZ*O@8NDaJp6)sZxL>i zr}`&gahINO*enf8az9Q9tTDYu!YjF7DA$;(!#7B1ka5s(=oDCW>J#HG&h~Guv6`cp z64*B|rW#Ax2u)2wxq%e(X0Pc9ynZkIG8jWxHMe6_?f2x~!9Sl3D9n>zTnyh+>{OAK zgWO$kVaXB$OLOpnS2N27eTl9zdx*eioN3CUoi26htCsNoL%D%xLgK1oJnZ;{xV?R) z(Luzi_+zH`9rbBKr$7Q@kp*ril>yGXCH~4DrF?O(yKS8I+eMvQ_i+k#sN*YYokkF| z!v>#A9dp2|Z%l9)Pd|z3v~ljBi4f89RF~mWeAW4!GRVq0Y7j;SB)_&z;zZV?B2 zr;TR6y#k6;{!*il_qFXI9WqY7TeV=^hITqYGl5brEuqSOG;|)&I-hgb;IGeBUKv81 z<)#l?(@=Jo3D1J^DCUpqt6iTW(q8OJQfx!g9?xK>Ocv3G9>;QoKfCH93q0h!RfhSi z2uHx6p+OND(pVlmVjp!S{^#MZ!RMB`RTTIWtCBo7O1FLO*1F|^-SP++eECle`Qm?$k;@b6;P%`&~io;zHKQZMe~@O?G|&M_Xg zQ0yXzB^xf`9Lr&J4c%DA(pre&5JcQ-yP*)FZEMGlPb4k+Zl7L>!6RUQJU)x4e9}{} zc9k($eU)q6c|6Rf^d~h2_nk%Qh5mq$uqu*c8oizr2Z!@9Uq~9o8$7z%v46-`Ui9Qn zJv^F#pN7!?A-8*(;KSrL9>Ui#orJsd7uL;tu?UPFno`o-!&PkK&p5`g&6sHDM4pyk z(YV+}P;ThIr>rs~rhbBuCkA`}Xt+dMH0g1^qPF3=i-?bmxKv5qJekm-$wZ2~M4R!P z`@>77`4rts2^z8-Q-53)^iAp9^p zKnsTbD|V)*RNr1uD53uk!Ed_(Px7}ezHYl!MCNAOkb_ZAV~kzDOvB8G9pemX7=l}O z&h@xC;eywVtT5lSVzb`8bf!q7m+a$hEb1YzmEC@@NFgr1PS1cniDCp zF8ng*`$a{zg#pp5PrR{Ip7tx>AVXf@(Nt-y;@kDYFE6640XWnPZ!sL*2&^Pn_X%UM zcT}dHK`#5={+484AzvkkVy6Vzoy0?1g{jH{ULPs9aHg?xd%v3#!Plc_7Es(BIci~1 ze==NwXKbGix3wD~J*Tr|>Ph?A3z!K-xh7&@52@P}2zIMF+#(QbIdVNei-`F9RX48i zIepk0G#Te{jzX&NXx)5SQ}inUBeV60gGyr97nx%ILlP9sGuVITUTr8ceu}=yTo3ED z^Nq5eG39=h2V`>~BMoc`FCuxgemaS_hfPAwchv~!&*~lMqL*yX@(}jK-;WLnxxTb! zU6IV!n#uUl`xvCF|NLAIX`K2Ay#EBxUVXBgtm0#RBCW>Tf@`Hc=XW_d9ffre};IFzVf%5yFx78CP zxrba5EOyY^l?*@NTDWOdzUC*yC46v6V=kH}F)Vyknu$jQnOE^h2PEu|Jsjo7_^w3d z)c=S%egqtWDrWmNE_@B>E@pa7dQj!P&ycRN!=*Q_ae9N@>Y^eBO=dC6C64uV&_?Jq zCQMi57ODf?r;)RhoRqfZ|xz9+h z1eT%KO|Ys6O|e3=uqj3y3AbY|6=_Q#?PHPmIqX$h{S3b_z6|sT==xaO@%wd@mAvvK zgVf!bO=xDsfg)m2>Q+w7G)XkY%~4jc-j1Nw~E*h9E!vcRS(lx3bHX*&Xj*F z@b>o!Py!|{cp+8;(`VcNDQYNCs5Uzww~}%nx2)t0t`5 z*AyL&T0c+hI@XCR$~3Tjb-G{oIQ1KSoOYqhBW`lo)R&MQWR^66`$#N$rr`ja#pkp` zBp-(@hHX{BCp=M`h`p@X$(`k_#3||^&W8mD%k>pF(^Hqy?*kxP`ne+)!2N$87yGJ- z^L>JB`@&*Q@LmkLW@g@2{$1>N4gOMb(-w|cg8!L_p{2v)@6#%`aaLBr=blf`y z)8CpjmI9Iwec+IBFV7{bae=Q#!}19!Vixl)(={3G7TncVFnt)>CO?#4zZ(=e(?X38rvArpcy z#oP#%h$sxOyQmciI;?Neoxi60*u+>=W7 z)yK01-aZsQv|V<_1J1J&4u|bg9peU}CO#M~*x6>60igaU3T2X`Ic27P$|N(Oi#}Ev zbam=~S+485gjXk49AiDM0M4jRiy+G`(+@8hydIS~a)z))J)-nz&Uzr!>;j8@ zR>{6K|C(pr{E*2)kXpIA)K@t4UWRTyz}o9Mf!?mHb42g!UDRo^M{mHW)}JBM{Edjq zF%0;#zwFoEv9hw7-5gA0?O9hSbDx6h__`hkmLW5)pT{OA7ng1aqi}Dnak!dDvM z2q5M#7DCT0@Q&(MtvBqQ%aeF7nnk+4E>`H=2k7{|`2ns6i}Rl3W)#M=GG%TF)QlK? z30;TJJw{>5H)ZKPzz>bRndDWN@12;fw^D(Zc2>ub=8o_5dpcnknR$Ir=&N{zaeI*N za1tJk%_{!-u#0S=me$~e0AAQGd4I892)L-=r}co#bi7cW+&yYmXoxS=Ak_K{IJzrw z{sZV5N9<054_oh2`MhdN-e+}}nH55}8qZ_U%t_bTij z?Et)zo{q>Jc?fs@w-5wu62|tcV0~7XAJs9Px16K?|*FzD@L3MGt~LA5y*dTon} z@ZmO5Ua!xO0{i*032qB&8jrRy`=N6iL>qqhddLdx{7|U3jtl3-Q1mF7zt+vQr=gJv z@K7M=W*s_ilVZeHq~TWp`>tea3>py!3-HFS=7p()2LU&^^uF|cw1p3yL58{TO=AXi z;SWVd<;S$b5O23tE8(%w!5Q2$-`DN--Wz0f$A<0}SV{mZOrDeHl>>mCUG61N z8AAxewrZ3V>?@-x)h_TVVRDht8Yq;9*RvDK{^)UJ2w`_<|0&+XROw|<+h9z`M-7Sk z@~nbk!B2;z@m0vU6D&1;7^2W=j)Rl$4e#_bMoB(%{09Mw=50#pKR7C*&BKHt9S(r) zCGV|X4kB`KZ@UyxKn*uL?e3-7R63ckYFfNU>y9k0>Uz}%G*yv-_55^LVFSAb<4z}7 zkFCr@xM7>~MPj4fsx$x{(b3OUj?3=bUT$>cQJ@T$DCH|3h|Mq*+0$|Y(xZBNfQ$#c z7U9cJs_&X2bg}{}m8Z85(q((TQ-Z9RUUn+Bfx{M)y>G6T<`0A`w~yC^nXKZAcUuj|A^{{ z2CgFP^K{+>)mM6jHXwrV^L4}P5dXz^p~gf4f2tY)S)4c#nR5+Wq>JeE+ckCA4Y@^R z>oR#gMgfo5^w-hM2nc#i+61EOH;_OA&n|~8W#I3vtgeEj>up&;7xqA5mE95o@6I&`(z@(_j z&~#jBMlRhK^dJL5Ut^=7YMv{Hyw$$vbz!icQKyR@1#H&m3iZjeysgGVLUgt&R;UAA zbWfEhkf+_9Z-r(}-fMKpkv%%XvbRjWc$Dj&rL7306*|!mAI`4$KoeNQBq~TaO(Nkl zb`TrbSh6-WjUwuh1Fb=4y8`|;Z&~n-HKFns1|^x=$FoCZ)Uf10xWTe~jFP|x?|P`Z z7ZtI!yXXAi1Gj@71E?PQZ3_%Z6e^TBYZ-ncPncYktHv*>Wts(l9DPc#rg#13ESLs& zRaIUc#@`xMQ4tV(C=r9V6`HAmZ2<=XLkSfi-#%{`jORS4;5k#tYgbko53m=j$vOL-GFCqxddTs8< z)wmb0e%-v%^Twsq<|RZ_}; z7|>X`Yj$Au%a}eUhFuYi8234&iV&@w9s`U6DjKP>MzRBh`0{hT3TAy(;~=H5g7D*%i9Iw*bj!1$LX9|N z%HWJ8Q*+@^)X5STb!8R=WbWOGDbPOBQyJMTf@bQdwv)dmQ*$ltLeZ*0#gmPDT3%(# zC6ZStH*{sJF=*yaq-0l%_O;@j)djryQ*x3Yv~piR7R&BM$3-x-4l8~^+y~i9btKGn zIRT*;vCQN&d^3=4%dBEeZzOOCf92EXCqmfwwJeF}Gv%T6plFjt*B81U10YpvGuD*k zG1#r&y|+97vL>UAI^`8*qYmZHu3n*yUdu50$&hN3U8Ld=mnAciH(u`maO5hU#I!Ig zY@u(YnZV1l1Agi%#B#7NA99=|p0OmZ#&>2VhzTguB^8W?X|22V1f>K==hU7Q#I!c1 zWNlU(iM+siFwxm*%2Q0MzBvvJwr)nVm{iclkOoP@_PglsrhT>f#MR?i(&}?^g||Jc z-2l{Al=0p;F$Q^77E9{b4Yk-wq{d!Xvt*i}vzax_ieSKitf31c8@C;w`|4fah>~Uw z&}jG<57U{9f(_RvO{Ho_zbvone#i0m`iKf64u&*>WlX18or8pjOQ%zF?MhG@xIal{1Sr}0i zebZnUOaHjg3t~R3d{#tY&my?_8B5^#aJ4NNubsqzZB|z3>Pnd67xad=QxAeyAt7}M zo8yyEJu7HlNsPSV!8UskkyK9|wOBk%+eax5P~duMN}Lj`XTv_!V;Q}=Y%onMHLv}x zt6(MA|Ly3<0jkVN5n~uMLAhp2wO0ub5o$m!Ikp)|^C1xUI+|k}Ps#rtD7CfbGIh%> z!6YU~AT{Wf8OidDO;#>D-YF8Oq{74-wL<^_AkNb#rWqb1RDE6V zN`qS#)qH#aki4V5GW80avF&IsbHKi$o`FW7HR7q)fqGGFv|SE+4uxo7R+}79J(6ht zm_Vz+pvcMXpYF`J$hw5`%%O!0R-Q{Vj~v2Pa{`;>?2?biVpU#s-ml3uXT_)n3-n+SI+q{cOv$UDERH=ry|!XCPALYQ>Q9)$eY< zzPEnBOhim?Tn?I!V~H^*NHC`fQ3KT)^+Cr%gxD@Oaxc=^3SJz=(oyR|9`1;*o}$`7 zu^|f93xBSEVY<4;O zm~%jrHt2RiI)Ran{Zu1{9eyF}xzUEI+yqbLbxZTG(vx{lzO0 z7a~fAGgt41ixO8!rE#e~>^Od!^IChQe)M!o$7N+i=;(&rz@6Qs{ zRymzEr!2Eki#e>33gk0m)lWx7N5mSd+%t@@Y8_vVZ?(v7qpN5uR+|zapx}O}XKTVuPK+dOnoq<|q&G_5 zxJ=C+s?a@hZKt%i^F3a5#1x7gx+`~xvs0>4=keh2cuF2T^KcrhG-B4fpSxvNF@LZX zdV6_2B&;=FkhXGP!FC>%iX!D2O_eH|Oe5YOOH5`>V{3v)hC&j(8=#@0+EZqxbvc?UZBITg}ole5`T34r(yXek-sR+O!d{ zY`-YPYrm(3$7Ma?$L2nRU2gV|Ph6(ptJ(}mr)=8rY(l_g>phIm^rPE0<1GvXf#|F@ z^CNx{y0(o!x@75kza(DYJ=DmT>Z{&9m~3ay9#-m@cK*(;j8lbQ$pnn*1un+)IP z>L|Br1diu@@2J)Kz(}p%-fVMXr~WH*W_K?^MIniEup_U+yjrNi!9Ujb=ke0)X&Qk! zi^N;N@$pi@F}9rP_`{MLztM=V@dNAZ?x@Tk&?36BEnxpuzg9LUr$y;izg2hA);~n{ z!5V~H?jV(E;CsVPU378m;-5lHE|c&byK6G;)Zvm6)8m9hjYh4pAX3yKE+5AylVUDHOY@-{%W--PKH^B1bJOMzIsW;W z(+1mmjd|?ZI+#HMwMlpG!eNIsN^z|n>v;E2v&P)vPL`;vueAX1#ecg16KGstdw|a0f8hkeX z!Su6u#Cd?6AGmklYQ~asnR+SB<_i~f$l4kA-Y%m+e3Kv^rGvmz_|r?5X}Xw3yAVrF@2@#xR%m#Mmi4)pFl%?}^$6%0}i z4_JqdM_u2??bmu-3h>Kr+=1j-9J>vdI~V*eSZivB;HM7kbgrq{-E0<;=vNyD)9hwr zY?f*p##7n!7%b-)j?H*0y5HCe$>tFR zWKa7utM~=A?K>}!>|Ra-o&p$!Dnf_TkGKiN7@v#bmdtGDbUC3vGC6tzc3e}=b z;Y4hfe^u^l(9rYlXf$ul`{1PRe0ZS;L0AvB|K?Yeww$OI0(s^Vp<1Oq;Z_O$%{X0^ zus$-fH!gnn^f^8Lrs|!Ala4t`t4&eL0wwql!93sNgYn-|;vM&yswGF2oUfjqrlzKn z_MI-w638tB@r*H1QQd;P?$X)i4zU2KAd4_!*NzdsMjpO>3>Aac)em(LqRiLqC zv;TQ&GxmUfvbLB-DWTP4O1~vw%hBs#fwI81g$1w*@dsumCAG_3rBYdZ@(ew(4^j5& z&E@uMqcw(+H(#leZ(Dz0qstl6!<8S>?xk${ou{-;Y0zxjU`ZLWr!;(pjI$Ol*KKG$ z+iV&t{{3`+ITL+%T!8lgR((7)GfwKP(RESlxKVX|_oa09s6fT8>>K}pda1hB-TBS! zhu|p`-yEtDaPxcH#)7)TW~yrZt(~T@S~K5b()hPQ>q8FWfvIB=@5Q zc5PO0x~p{t_?Vu(z0ftBa!C5-BGKl~z#^J%XUq6nHylk3go~XZVE<;D%oSX$)TY$X z_sz_3d0nQ#>poR(|I%ijM}j9ABd;;iUlnkLBK{*jUwV6FY~se&*Jj zKR}R@eSTq}!BKYtN4~PP*gj<<@;f)y4x8D_3SzCry?&`$TTu)3{u?JxXAqrc`|zAR zf)|5Mi|z@PAY1qo44IlRZU8cW#W_o4k@rFr~LMGb$6Mo_^Hd2 z858}R1c}l)Dr3b0CKt2!1L$3y%Ja&-FVidIpl&cMyKwQiO-5z4a)Jvte!Q(5u^6AE zz;>5DcJdNVt4EJjT`)vo(9P6LqaaVvGdHuqSx81K9uAjJMJ>9CcW=!w#>CR6H;t0> zw+)U{q`q&o^~qZEx~#~WvioemIN;H>gL*t)VZu_5>Y{~+gU|RiakaNDrmMhw>S9j4 z&O%MMwQQlhc{ZC_9fecRFEjG(%tWirDARwCN-}wmbw%$*JrW$b8AKU=GjDO)H> zD>06Xb4ZrqffcZio13(yDfggrQ%QWc*r3}|%W@arRIfm4o@cB%Z6~AV#fi}4w*G^W zvK-5@s^ccFI~-QR>z7QcR0vBjpMdQbKG!J}Zr6uu)v~RN5pdS7)C_yQqdEQ6`h1#& zqxz#cv}YrDa8Spw`~EnFBOMzdI2Gh^$;Z8SMmHgGh0-g!@17!ttRWs0*W~`J(1rNG z%3rHg$Qdw$WsoeAKK-@3KcE4&D>DrR*xaS?Tfg@7*1e&hkUg%0)BCamBOFHac_w>h1>> zc>wQ>qjO_8I5-$^ZAU{I8Ly_G}y54e(4FrTOA6N$XEU_l}I?8@nh-r}!bDq~*j=nZ2Z5lbADN)Oa=>L-@KT*Fv{SsK0_Z~vOP?aKYy}aPoZG|ECkOJO=2^?S zyBLugIcY}WOiNqy31^_2Mr$CvobFBd>=<+G6)(QR3; z$qqTNmGp!8vi=s)Q2>sXanAUEZKeOe)juxh|EDA|i34%ERMkX5dhj2`;)DW_zeAsa zfRxi-w}=!G2jIsW?+)gpO8#L>vsg@KKqhsA{%i;$76@96$^?Mz%Tn7fD+Ymq3g807 zvgT%p;8Yf@7L)Z9tp5y21tjwWmpX`$b`ifX4$@Of|EGWNJ^%%%mTX`Iu}x z5PmGX6%RcS^&c6#qvSwMAf3>m0n`4;=ZF9f`I!zNkGfuN=!JizCIL9dnFQJ?JZ@zs zC&PczKngH{!Dq|6GUv5vEagw4*eb_*zrvG~l8OMKTe?IVwb^j3V)@Jhg=_&9lfg(f zioPP?-1s3Ch{48tT&&vr_)+vdz8A&eSaZ#gFp#H&v8e_mVG_kQ}1zza@EN4p) zqCV$kD0nkY1~{9Xz3S>=V6LLrAtYr1;eP)J(N0JX#R?BA1XO}it(h_g8H!}wEru5Y zA}JvKBgmSi%o%eU%gLkni@gzG4q^IC4~dYH-Ri)w%i)3*@WsJNi|d!;gXtLcLTFGK zS|*F(z;0KA-SUUrexXwH`O$Ko7kseRDYcr`6AZmA8dT2@^}`VrSjUYRyg76$jSn=I zR!jSn4osVxnVS+IV%P$BplY_TICDuJxbJCvebt9e&AHc)L}35-dc{5tN*Axy3!=jW zLfTSxKgFKweQ!k12~KL%yIc!NcK~IOZLJ9yPe>B!!Cd(#Fz}zZyDJO3=w}!BMLi6r z-4%m@e-AVPs`gSUCELP*HEUz=@c;Zcc=9Y@j41X0Sg&zh5(}rImr@}xGDBWK>544`_P zfZ)z9f_~9w!)gD%>oC!ghfgzU3qdChsweG!Qd)6qF~P~$PD!gKA-fWU@HN>l1zyIj zj}SE*F^vwJ48Wn62oSD$3#qB~Vn`*TCC$x?>`egfdIG%;hoKKLJ`WjICbeFxYxj%% ziX?nmy!U6f8OVphQM4dnM^HoNbr1r9Ngro`OD(xO81*Obh{feVwwO%}gHfLjAR@WR z`hh^h0IiuE$Tvv_SW41BSSr}uZ&Axa3WyhczCdm;8B3zS29#H*wDGZk_B7qV@#{Fi zO{e%ntzatvnE%F|!rmNZj6f1!t;G}p5Nr>>H<#&5-=BFuftnFLY6>J6TrU{}?3bhc zZg)RlQEswd4?NWAd>7YjwAtxb>$v@iTB9-}!+pg{ite#1?@yM@!+g1h!pVH;%_fq+ z3|Ellg~ammGK=Ljk=N~l#y$}LpGm@@lw)W+e(BY@Y?gk%1?pVtma2eD14AKYGq_V)(qT_ ztv@84XxP!-Iq0XF6@}6_tc1{6K^* z&}=saV5{hK8v)FkEMf^((t22o_7<1JllJlRhZy(sA2lKlE7eAyw@eDR_EGFE=89w1 z{O;Gm>yXp~<`IW_se%O99xL=acnk&9?FyhKXyT~jrl2r&i2!GR3DdHq36w;yGg>5< zUI1jhJ!&J7UjoS_3y*uN*gYYZ8=9oU(NcJlV2e~4Hbq;5QSYbY$Yo3?Gq{}rK7ti_ z7eKNxk4t$463YJOUh!nTj};=jCv6}8)S?aWias*x>vA}3oPoa|W7?aQVO$vP0fbbQ z$V#O*CGZ4V1Be;kZ@^Iu>RKvM66wOyC|8&3e7=^kn$8L0`!y}1Tp;=3@#hd>L)_I> zA2K|M`>IYG$Z-=8n9l|nJ9-8>R5wwa8{dIcDoM)A#ftR9{e589CT8*bSkuza8vWV> zS|tKRfYbUv6f#}^$9)#}GtEDqU1wJep`bpG#|7wq*DH`s5yuH3p$mvjY7>5jtp2p? zdBzCPc)D4lFr$HBrzGJUKwhx$Y_-3NjH zN#|N*;jN-+BaYc*a2Uwloj5ZM|?o`;$a3pa7b{1dvJogYl00f z0TNt7aEC#HLxKjE0Kwh$Zq7O1`@MDlgIjg0sHv%Drl+TS_ugwgYyF;4;hsc`4SJ7o zX?=LxqC&ccH7MQmkPlIm{uoK55cR7CQWnE*gc0SP$SnIyE&8LsgE7s;JFku8oO{r$ z`#I0BShS<9gHn7_UpMb%CXjj^7!mr^2!-1AM@0^KQ3;sM7Xlvd!>i4S2c?PwSY$_C zm1qIVv!M16Aen?XPf_|{X<7UT`UUMCl&%6Wo#~a%>ln;*sExS<0f;2~4rctDkvOrC z%hw`qPdpNSyR=U3paup(rLH-GlFqx20_5jM_-@ec2BA}(w9RA7AeX5T>3}{*-y?V2 zk~GJ1m6OHD+5@HM6tL$ocm<(bN-G8*S3}iIH-vEQ4R;PmOKq<&=!74A5nMv(;HjG^ zOiW?Ip@v=?1p7BZhnq;#)Nc;jE)AR36BW-?7Xr9&gHENG>~H*|?Xia+dg7bC2c`*0 z?A{E<*V{t95n?zi36NRUk5R)}O}-01-kl;yr;`XcOpYNxqF`1nn4CO=_tya}r5E^q zpx1s7^s(i!`E05cG~r1%SFwv-t%PjSKjDe~{>H`A_&TGXJ=aj6KRNGSnp~r8TAGoY zO+BieH(B8P56AstBR~V*WBY*S%x82{^G@$d1RP$%!eMDy$o_iZAR5J>J>t?JgwMu& z@U)%*T~^QNHtK0{s!sgZ$y}aErxF#)JP_fAseWrPPQ&I{D1@_QHkS`^9UHu*v1Bj} zCJGlk4vrfG^TH1C67McT?=O6V%;L{XX@N&zHbXghv_`{MXY|Id#c~UOAHqb0`or-m zJbi5hDVpmfW^>kL6K@Ed%Q!l5mQQ)XmI$&)8$n6L&qn}S@>^M=0RpbmKjUSBZ8q z!xan~9UXiF z#1jF_0A_XbE7is>J17wCt;HG2^w1E{+Fc&t!chN``YsW&>8)R1>HoZoJAM3!LvoeS z#epeiZTnk;LGqTfbj@Bj!~&f|YCxm&ZuiBON7YTHO~^6NrT+bA)Ql+^`%$dHX;tz2 zuU6@u0~A>Jk1@ly%IH(QtY0LaSG-f^a*SNUdegGg<$@;-BVaXL&MQvzA~=Z?pPf-8 zZ2+auz)}6-J-7WF?>;X^0eTG&ST%iK^71&XNMujWXM~**jYdd#bvvP#n_=sGO8)%C zA4aMvBp-o=B)kCf`ICRYadEGHE-TcTBpRQdyOSA)!3G&3d>Da6t}$37tB(;cPzeR} z@|LbAmgrGny_DJ24}2Yt+|JcJ#8h}e&d24)`_Ecyx$&lvk)~L%3voxU*YA<>kU6=7 zc44D&4Wu};tW_whN;jOgA}s9gZ_b4kCp2R7lv_WnEcBquB99JOhd<9#%NWCvf$da` zvoQ&wf{~6ymJH@aN2JgKc4(?kUI1#-P9t5tKT3b{c?NZ}YEF{)g&kg{;W`8q-(-3P zQdC+jh*LIV(Pp1nILub)8!-({M;bXEQ z64?TKz$=7UH&Pf=k%wHR&{*fV>2J;i?5j4bwe(vYu^;8nccOH(JyE)zK}BE~Axfgw zdFx(CBRwJHs6^z3o(aJ&$U$95VW;2@pXRw3K2bx2sQeJ)iTZs?aZfnlMeu$f+n4}V zG<=;4AK3!&Z$ZPOQ@NK91=0+mPI(5eO@76MWg|*!?H>dLf72tbGSC?3{^mH~^~vba zqX@o&fq2!8$rkP)T6sS<5$sV{5S6b^PJ+1M2eI2do}o4Ju6x8wI7WYKq;?;y^zi># z7WADTp=yjdB~=HX0miXe*>k5YK&XI*gOc`qQWp@7b|{& z1n~}=@>q|$0Frdu)YkpZezzJHyR!neYci9xT|*->P$=e&Za(_?(XlV*7eyg4k{ygGH32c{m)IB^B%4AU(o|}LIeFn^ z=X*kKsl1%eMj^&Ef?SyxHpx;}^cXirOzOgHJEZcEp}?hYfC@ za*aK06T&kt=Bknh49p<&qZO#PxRoCJ!i#>G@RaXXA)sDPH>MrJS-TGrSm zJW*@HEz(`jHKBs>Y*-!IUm{;KY63wwBwW`Na#l-hLAOk4>8jXJT=X)DmCI)+S`??k zlTlX;+wi#dlBW*X<%++MU$G3Ivi)?fI!n_saut6ibc8}%Re0S3g;#iPO~Pt@6`H6s z-P!XqfD!6CiW>Fxff4DibnQL9;ysXfq(-29omQ}xc4?`{FkK-(jW}7K86gcb^H8u` zzCa(ix?M`UJrZB==V1%aItq$=29b2Ma^z7ier$y4dTA|9Hs*Ig7ibc)`>n&UgRxU+ z>Fv)InUAYJ=y6gjSm7p=S5?Jn>T^)BL=(oV4#t>Du}iK{q7oTz%uH08ahiNzMyMKI zkHNYS&?5Nrf6KvKY7{NSVc+zlFXBXE-?h>!XW(0>zORRBGC(AP(A@V!g-5q^ELK6UOyCJ*<5T zpB{A*5n*N2>pcvYY^419hQiVx!QjK@WRh}~_{D3MB`FyKAI%ZD|G@M8`^QAlCru`f zPw^%WXaJ8oUwZ{wfBE!9yV!oQk#w~kqpdgeeB;0Oz3)SnA|&)D9=0nZ`81bf*^3Jk z_rGin3&G#?jGMT7&njeIIC4D~(@wHJQBLl=iNuCa`{?!?zDAk~xo)*18qqS?)N&K# zcn9g(QQ!(%o6;9O25+gY1v7a!V?VqW1X9BPs7jQbRHHCzjRXEbFR1Il!qt5yxLxLk z>KEM!4|5ffE)9u1`Zm2WX~U@$NhEVr5TyD-3nlQ8kp7#oq?J-6v`P%03*93_<(Hdg zpbj#G^s0;sn-#(h|Na%FPLH~D8v2z*7Xw~*Y?^0aXEt&JPFd$Ax))Wz)mSG7ejYb= zAI2&lW#q4LoiJ?1#I%)=*CIz9A#@l+bx`g|i%=)Ks#Vo&*hg9j%%L1g@6d1vj+%fp zz}!B;bYUBRH&C#};q106zD>X+S-m&9W_iJknU)tk_(CCSV8&^K!;W6W9MJC{?X zv0g{?Ha;glLVcMoa39D}TN!wsX9cNKiO4&yjL49*H9od|Gt5b%(8MEfeW%^-EbLf2 zqR)w7q&tXS_aRv|JNd+_6%e^guU{@FA7*0m^@Ec_71DC)ma4Pw=9gb7PDpMryU1vGM!2ul+Gp!9$^zr=`A#L@i4s5&QADe zO2|i89D({P-UPa2{tn2$Nu%BbG@jgDnxzmA23lY$z6~qmVU4O*A=N?Xp@`;3#ObnukD&`+Wl<77`s`_JF@aOW zl%b=HtQ?Z39nJG3dd2$7H|T{1oGS@X+kK-!Udh`QKOMy(Fr#}CK0kpI4ph5O)|>>q zavz?ttf~e&mOsbs*KK(fL{atr-g2WsSIIjKX1ssgR z;LS(}W4b$(m2i0q-T>x1AZHsR0r6`uca*y%_n6d)7Ip!9J7lxJ(xgD1av%9;aEkl8 zgGj_TlCClWRk#oE){>_TUCTgC1|4=!U25C!*a>RS4g&h>a|n6++%Z^Q!P&?K64uu4 zy2$aSp=QuD2PGdd00~?i2iVNuR>c76);&S>(1o~|zwLJUdj%+}wO2yn79;B&QQ_)9 z*XHg1{A!-?1RweFTj4Mj2(A3#?rjUpTwxirNB5j z1V~{!n#93%&vpYV^&39W?f{H^WAaZ?c}qLj_b`# zooY1iSPh+6$q+x6el5whGWB}C_Ka?x^nZW*DdheUzPU+WpiF|NLEirTpzWzgy@ffsu*PZQ5 zDH{7#YfPH{b=fXRwJkFMrOLC*&jAm*5Eps{oCF{+s#dmL%r(Y@hGAaQ96kY%lU*Bg z3YSnMivIP1Bw40xZh#s->68W*7{uH%hMubkZ7P4OOuEl9va1+NWXCMl`xC0|X1Ud4 z3i>p|@vsbmGP7BRFS$Kl5aSE-F>m{yKz=}*mO;1#0!@FEW<#w?ZLV%~Qo5`>5XwVy z(^#9}w?Zs24uJ%v04xtqEeZ{3H-I9S;&a%hjY%pP?- zfWlLe%x6J`QIcyy)iN9y(I-(7dG zH%?=6BGeo)zh0 zKF@b}vRJ8|h(BDJJ<(kbT0nz23pjIbX0&d!WTcH(g3AIAbF3_;K+l(rW!?!Dao>vv zmQwF~_koX-shHiDQO15)Ak*Srv6ie#1m&-@0knbOiUoRT13G1vT726ngj{4QJ085f zR)Uax>u(=DcS?$C$6;hg<`ob>i|Dld5ArV^{1D;gU$!dpZWK^pFX!(%?m&V)hx;3V zWE30d+beHo8XS%M`iIkb?y=nV9UcFdIh&SzBjjrTWvy87P|msYPaYRuE&`k72T{%h zFaao%zS9XZ<_q5V3qrf({rQI7JeD$r&?!tUA;#b(1-z4pwb1r3AYN-d#+_q@v!}A^ zQUg#fiDy)h#)uz?0Zl7--?F?m)AR3!&(w@^TZj1%<{LPEI?ULIjF~a< zBG6@rU4Hp+;CDNxf-oX>!72m~D~c`n6&p^Cw8a?OrA8BudAyR&YokHhgdkS?wnPIM z_QaCE)B^${tJ+#?1nT{S?8$tVW@LJ7Jez*~)~bq~w#wc(pv3~+B555Dd5@gC8Pe{6 zQ0T1mdQo|B#+%rCv%XC1Uv5VRiP6S!Pc|}7t ziYj1`sR&tY02ke&{rCsT39yOT%~RF++J3WB))*5Zg;=6jS6WP*;@d{L`q)+aY>XWV}BX9tj!$&W%=b11#d;Rvqcsh(y2cL zE_VB_ z>u@%&y>rduNqmhVt~W!*?5=nE-KQjd1-e2;Mc|gd1JP{|6dnoyAtsE*sIFG5#AWqO zDu`6Xy}U`6g&v5$mv@+F8x70;NnpBBIqb3PZB2@A4ivWZQKzP$M0_g;7c>QzCtLO-7xV$yVtu`_n7?E;5*x&z1F+uIs-hF?39uQnDqdIL zZ`mm=J5!o5MuK*X1)RuM)h0MZtigJLDSD&4W)wHnDrh`DOWwf9|}lRwL?yH|@DX_{Q{yt0>Z< zP~EhSp^9iN1=3;U$+RNe8bb@f43F~laxzP1!s6lo?%Ge{*vCI9Z3}`LccM35!JBWwo>v`%GKF9PkSkYDs~grxbAjo`#(bz+Twoz~O# zs>haLh7pVy3GtA`%Xhr0L4RNX&FrE`q)!!GId?^wRmB*H(fPLR_dmy5ES5wUGY;6h;f06Rk-uOiI zYGb7FSgHK>{IJwlIOxIn@ZcZj=`0Eo#WlT0Ud^j2vM2^nzkUj|(7|*R1Dhd@85A5& zRUHw6et-XPd)Qn^bQ3G*4C-LX7tgq#6JI*E>wY5WR#zKd`HN`bI9;wgCpXm5`u6r? zp4Qb@NzOZzqU#Sy{PuHKtVxJN+d!YlbRha|m+W~TLHD{&^hWizg1c&?Ns_>1Q|m5#To5AX1>+1kCMTly9YJY2YG zu!-8Rxe^1E)5_hwzGy&@1FSJ0#vJ{rf@E;}<$?KHCcG(~1e#v?I-obyiplIAYnSC#*@@J5u&Q%&F{3vIf-5pO4 zPbkco4i#HnE9`xPB@)+bj?P2=qAR8|BTXgPM5UE8_fbfQGTVFUp?*BrGJv8A5f%hF zgfI%h{Lr0Xs&X~BFeDR>;q;`u@r>K59Gc)B>NRl)gWx$0Z8$-6x-Q}aO%*e%cBDSS zqrg=ipY_j9k=Mh^>N7eF+%iLZF&yoY#W|5D6y+WHY2T5W2G>mm>z) z7;-#Kg{h4*?VtJu=#NbjsdKpA8X-De=nwat*_YRGVZSqZZE#1Z zP?SlqYIImE<a=9aHKjDQzVE^3sG=rq z(XuN62uAEz)RHLqp$x`IHRdTzz-nw?G%mMko3cW`VYFdgN4PfAzBW|?_8c4iWxa{x zy^ePVuAuK#)hE7CPbJ6}3MFe7s5HJrGS#vW71PjJ#^w3Vn1ELmqFX&^H&jS=rTc81Vg_nGOZI6?uaG7M+N442D$H%q$=XluBtcxtK+aiUHqzt)JaUs z0cHUGJ%~{WY>86bUqfl+DM`V3=)zg^|8G;CJU49ktUR|9q1=CZ?LaFHxXIAuB5(iO zy+t_yI!@?_1!MkY_~8ULh0*lf|I@lh8*dI4l0@{Z-h_XU7zx3r<@x{piOklo5l7cR z^7May;JBErSCO5?|J50o4$e9)Xw3fa?y*0CO-`YWpa1i=K=?$Zr!K>w8o~d5A!NDE zgqVT=P;k;Heam!@G6TbKFg7%l1jsSD!Bh?oP}z4gCFq`J5)skA*qfyRl6eZKrH3gW z_Q1=1B29RPgeeb94zqOHB6n&5GdVCk{K1Xnsa^AMrNnl4U zNVR#uf>WbFAvv;=f`ML&%ByOx$yzu#IOH%77MXOL!U`l{JznU!x>RrU z2fZ`JA9Q2=>qIalTsr4;6baBauk~W{GWW(Nich4Ykc9;U?S|E#dEGmDgtPS%Z8P1V z>xk})Q~3J9hrczjGq>$+w*@i@Yc6|7znoG#%q|a?8suEKYiL~`srsh540_!U775s+ z_HW5ba-pVg>MocxN)0sUVS6=#S{F!lg>QLE)F08AwDRp(b$|OSX>CgCas}j1+D;mv zS~O0edR}#^=r(-F;1#P0+7~se=p428F<-1!u5{>5ITtbiJG}^=HE(3nzIh2K3-$_~ zTCFxtLI`5fA7q@&pTRrM8)-u6s3IGMC{|*9qRxqHVHij1dHH2-!NX&orcOCD}f7~Y!s>O2xb2}{1cwQE)=>#V1Xm=3im3b;qwTvzLDn#yBwe&J^9zL zMF*QqV4Ma1HEc7VCJQ_(FEDJ`X**xTvG4I}rNbL2175AHB`*0N4;^{gr=aV*_ALEB zN39K-ayNGl%r^cxXpIE0x3_zm^-=#B@|{!Qu4`(()$#hz+k)QG#wRFz90Wr=EZ~m0 zO#7ZqSyfz4#d=|~ZbZMuJH6cSI`uiKh|dvlxg+%W;eqxjD38> zl_y5ibDvgks-BJLKtYXzvo(M9!*Sg9pmyzy}7f#KED0DoojUz>< zV>DUrAVA97ZJ({hW1q{TW#q7CNoJN zpUF2G<Cw5*c*QXkc|wxO0HA^*SX7+$w(24v4iaKqp$Q_ zf6Zos{w#axs)%lU@On5`t(&sjO~G)niFmW#!XXY(Efz8w&T>8;Nk;`o^Y^8OG z2OUMT$5&DfLW_piqu)t&>rCrfFOM_lg175ItwQ|qB1XfyJ>hf5wffFvIh}~dVPeWV zj`)Fz!hTsY_7BQQ{ux}#;i5P3drLL?F9d{t@txO~<%1KIDSXYcJ@-csc-d|?v5ClU zlulU;(q4+b1>=IP8_|&A<+p8M18nquMhM5XiD-`1uo`mKzRf-sZ~5`hj(m1AhQ&lU zAJ^Z^;b$HXHHyY=JAM*eXAX_oc*Z33r~LF~+DxS${q8ht>W`wV)!CEhJ5=ug5DFDhSCMT4j` z3@j_Ayly(#sO76s%p6*;M*9~3c)O*j&1W_-J)+T|MfO>ErsADkz1{py&B%Z_vvw(K zhiVR$$3;tTUAybR<}N-PWu^P!?0sA(QiZ{FwAeL-{f&xynV&-fzyKA)qLUXj7OT!h zeYXP9DHh~L?Uoeg-few~7~fRZPbR2u#AiBL4ndB#+U72>m=SO3uAgN>0cgk01Xz?< zroDa71C|E?dT4%yciEQ_X5R;0o2j$+@Ot7XrHkCh8>4)}-8tWiZB0{+AJ&s>^!)f} zXGWw79ZF$Om5#tH+0Cn3xLlDC0%j8@D{WXqa(@>7{AsZnB4jeE%)$zHu^8ORQ~CT6 z^Fh7T{B_A&=q2&~%8qcpEmK~ug*pB-=RMi4;})5y#7{8U4#~gkTwBgs3cuj98_4AN z?VF9o7flEOF0B{^ua(K-=HD_uZ)Qdz|oauN-;~H?=O~50>6K6hHHrB?j z2LA;AI<#y%*!LRL`F?yjAT(;O-Wcq6=JWfu$iJ^|lZp`P=p6F4LPPdqfBEm5DLmzJC%;2fw@r$%I(%pNcm&$M`(AGg z2$zR9;5uC$;gC&W_}8dBzk)6aA6eSl3rjMkUcD`UE7=0G;`7V&u?beNi4A!MjzZWm zUECw6vjM-Rc9D{S*P1qf-Eiy8l=yRwMB%yx0oeW`oI9LE31CUlxHhM6K8Z#Hd2>LZ zz$9Xb?fymYquzM+qAV4xQ;{3|(N8Z=`2Yg^x+_SDHF42&n#m{<$i@l5pkl)i zd!jaE111m0=~=*ay1I+(VNU70jhFkYsXe$?UOTMBB9^V9NBzcpwMh(~xfw1y+eO8B zrvZyEhV51Emg~*l6X$i+#QN<%YpNfcs^0rXaB{J0R$wuI`c=c?@9i2so8H@l)=Mpl z+F9LM=E9{e-+4b=p}js3TO`8@ER=)`=nu~R<(<T=G!4|(If2_J?b*|4wORK*vi z95GpTyLm}~n&;uyBb-c1$H9w7`WUEETHx2Rry8;iJ?b&GvD+3d?L`kAW2h!3sK)&g zQ>#3rOXYnweh+vWG4ldhz13P8d#_}P>#da=-r zS!QFajpT0?DI#FnZ&vMg%6~qtDdEu6(qXWC`C^8*wXz!WQsl(B$#0ADejM5AmVv@o zW@Ha^ZSrLkNz0-!mPP!r+yB!Sm#W8*#ZfRQOJU5vYh6ex=G#9#8}HR5RQ=(?YJ4kp zRHIyGmZ-V+LAsh7YcTp6Z)4rHx$H{$fN1cj$o+rA?B=ZHrt~M*WV3WC9b1KzYR5G9 zzx6Mx9cJl;THZ}W?(Fr*=uaQUXX|?m*|5D3`RsUB@D$nh*UWmm+)n zm1&92ucqle$oJfI29$EED>ZpaL%7In8LA*pz&_aTH`lF=g!k!7z^qy=unO8ze>E$WG$zaa285_z?ux7|D=3*TZ#t%cYnA~K%CTZq15IleH2!L+-i*f`*`d`@Y(U6jY&_h z)rf*IYSbtZm`#}k)Dv6XZED3o`Lz=jtQ>nvPG%;{qumWiRCRWxzEX!L6z#UzFG?{% z)O(Wc`-ff@?zVYNi8AF9Zn=F~Clx8Z>5AV#n-;9TN%+lO&0Vvh^L_CDV4ER9qQLOf9Nx1U#6uEt_w)V}(*(@<`j zBT-?a*y^l#Khu2ofGmO5!!85-SfV2Eej?Ksmt(fx7)nhAr6R?FL^?7w_P4zNz@hCf zW(1odKspRp*!k@T>VO)J60#)*O%QzMbrppD{)AwSS~Y|cgZ2zFp@_2I55czuPEcx% z-d_9P*$w*9miSM3w)x@D!S>8sDvtLpVv7rlh`%ns3~sPQw0t0K5$8}mbA=rfYEW6B1dM&FWQz>N}{Qm$VCWl7= literal 0 HcmV?d00001 diff --git a/docs/source/components/ops.rst b/docs/source/components/ops.rst index af51f41e76..5f98fdcb52 100644 --- a/docs/source/components/ops.rst +++ b/docs/source/components/ops.rst @@ -1,8 +1,47 @@ -Operators -====================== +xFormers optimized operators +============================================================ + +Memory-efficient attention +--------------------------- .. automodule:: xformers.ops + :members: memory_efficient_attention, AttentionOpBase, AttentionBias + :show-inheritance: + :imported-members: + + +Available implementations +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: xformers.ops.fmha.cutlass + :members: FwOp, BwOp + :member-order: bysource + +.. automodule:: xformers.ops.fmha.flash + :members: FwOp, BwOp + :member-order: bysource + +.. automodule:: xformers.ops.fmha.triton + :members: FwOp, BwOp + :member-order: bysource + +.. automodule:: xformers.ops.fmha.small_k + :members: FwOp, BwOp + :member-order: bysource + +Attention biases +~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: xformers.ops.fmha.attn_bias :members: :show-inheritance: + :member-order: bysource + + +Non-autograd implementations +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. automodule:: xformers.ops.fmha + :members: memory_efficient_attention_forward, memory_efficient_attention_forward_requires_grad, memory_efficient_attention_backward + :show-inheritance: :imported-members: :member-order: bysource diff --git a/tests/test_mem_eff_attention.py b/tests/test_mem_eff_attention.py index b1057dd1c6..3a1be8e694 100644 --- a/tests/test_mem_eff_attention.py +++ b/tests/test_mem_eff_attention.py @@ -1311,3 +1311,32 @@ def test_attn_bias_from_seqlens() -> None: out = bias.split(torch.randn([1, 3 + 5 + 1, 16])) assert len(out) == 3 assert tuple(out[0].shape) == (1, 3, 16) + + +@cuda_only +def test_attn_bias_blockdiag_doc() -> None: + """IMPORTANT: + This is the example in the doc for `BlockDiagonalMask`. + If this example needs to be updated, please also update the doc + """ + import torch + + from xformers.ops import fmha + + K = 16 + dtype = torch.float16 + device = "cuda" + list_x = [ + torch.randn([1, 3, 1, K], dtype=dtype, device=device), + torch.randn([1, 6, 1, K], dtype=dtype, device=device), + torch.randn([1, 2, 1, K], dtype=dtype, device=device), + ] + attn_bias, x = fmha.BlockDiagonalMask.from_tensor_list(list_x) + + linear = torch.nn.Linear(K, K * 3).to(device=device, dtype=dtype) # type: ignore + + q, k, v = linear(x).reshape([1, -1, 1, 3, K]).unbind(-2) + out = fmha.memory_efficient_attention(q, k, v, attn_bias=attn_bias) + list_out = attn_bias.split(out) + print(list_out[0].shape) # [1, 3, 1, K] + assert tuple(list_out[0].shape) == (1, 3, 1, K) diff --git a/xformers/ops/__init__.py b/xformers/ops/__init__.py index a1943cb72e..f0169774d4 100644 --- a/xformers/ops/__init__.py +++ b/xformers/ops/__init__.py @@ -60,6 +60,7 @@ def masked_matmul(a, b, mask=None): __all__ = [ + "memory_efficient_attention", "AttentionBias", "AttentionMask", "AttentionOp", @@ -71,7 +72,6 @@ def masked_matmul(a, b, mask=None): "MemoryEfficientAttentionFlashAttentionOp", "MemoryEfficientAttentionOp", "MemoryEfficientAttentionTritonFwdFlashBwOp", - "memory_efficient_attention", "memory_efficient_attention_backward", "memory_efficient_attention_forward", "memory_efficient_attention_forward_requires_grad", diff --git a/xformers/ops/fmha/__init__.py b/xformers/ops/fmha/__init__.py index 762b14ea42..743fcfff82 100644 --- a/xformers/ops/fmha/__init__.py +++ b/xformers/ops/fmha/__init__.py @@ -176,17 +176,18 @@ def memory_efficient_attention( Raises: NotImplementedError: if there is no operator available to compute the MHA + ValueError: if inputs are invalid :parameter query: Tensor of shape ``[B, Mq, H, K]`` :parameter key: Tensor of shape ``[B, Mkv, H, K]`` :parameter value: Tensor of shape ``[B, Mkv, H, Kv]`` :parameter attn_bias: Bias to apply to the attention matrix - defaults to no masking. \ - For causal attention, use :attr:`xformers.ops.LowerTriangularMask`. \ - This can also be a :attr:`torch.Tensor` for an arbitrary mask. + For common biases implemented efficiently in xFormers, see :attr:`xformers.ops.fmha.attn_bias.AttentionBias`. \ + This can also be a :attr:`torch.Tensor` for an arbitrary mask (slower). :parameter p: Dropout probability. Disabled if set to ``0.0`` - :parameter scale: The scale to query_state weights. If set to ``None``, the default \ + :parameter scale: Scaling factor for ``Q @ K.transpose()``. If set to ``None``, the default \ scale (q.shape[-1]**-0.5) will be used. - :parameter op: The operator to use - see :attr:`xformers.ops.AttentionOpBase`. \ + :parameter op: The operators to use - see :attr:`xformers.ops.AttentionOpBase`. \ If set to ``None`` (recommended), xFormers \ will dispatch to the best available operator, depending on the inputs \ and options. diff --git a/xformers/ops/fmha/attn_bias.py b/xformers/ops/fmha/attn_bias.py index 4458185538..5e5939083f 100644 --- a/xformers/ops/fmha/attn_bias.py +++ b/xformers/ops/fmha/attn_bias.py @@ -129,7 +129,40 @@ def split( @dataclass class BlockDiagonalMask(AttentionBias): - """A block-diagonal mask - can be used to handle batch elements with different sequence length""" + """ + A block-diagonal mask that can be passed as ``attn_bias`` + argument to :attr:`xformers.ops.memory_efficient_attention`. + + .. figure:: /_static/block_diag_bias.png + + This bias can be used to handle a batch of sequences of + different lengths, via :attr:`BlockDiagonalMask.from_tensor_list` + + :Example: + + .. code-block:: python + + import torch + from xformers.ops import fmha + + K = 16 + dtype = torch.float16 + device = "cuda" + list_x = [ + torch.randn([1, 3, 1, K], dtype=dtype, device=device), + torch.randn([1, 6, 1, K], dtype=dtype, device=device), + torch.randn([1, 2, 1, K], dtype=dtype, device=device), + ] + attn_bias, x = fmha.BlockDiagonalMask.from_tensor_list(list_x) + linear = torch.nn.Linear(K, K * 3).to(device=device, dtype=dtype) + + q, k, v = linear(x).reshape([1, -1, 1, 3, K]).unbind(-2) + out = fmha.memory_efficient_attention(q, k, v, attn_bias=attn_bias) + list_out = attn_bias.split(out) + print(list_out[0].shape) # [1, 3, 1, K] + assert tuple(list_out[0].shape) == (1, 3, 1, K) + + """ q_seqinfo: _SeqLenInfo k_seqinfo: _SeqLenInfo @@ -153,6 +186,7 @@ def materialize( dtype: torch.dtype = torch.float32, device: Union[str, torch.device] = "cpu", ) -> torch.Tensor: + """Materialize the attention bias - for debugging & testing""" assert shape[-1] == self.k_seqinfo.cu_seqlen_py[-1] assert shape[-2] == self.q_seqinfo.cu_seqlen_py[-1] mask = torch.empty(shape[-2:], dtype=dtype, device=device) @@ -178,6 +212,15 @@ def from_seqlens( q_seqlen: Sequence[int], kv_seqlen: Optional[Sequence[int]] = None, ) -> "BlockDiagonalMask": + """Creates a :attr:`BlockDiagonalMask` from a list of tensors lengths for query and key/value. + + Args: + q_seqlen (Sequence[int]): List of sequence lengths for query tensors + kv_seqlen (Sequence[int], optional): List of sequence lengths for key/value. Defaults to ``q_seqlen``. + + Returns: + BlockDiagonalMask + """ assert kv_seqlen is None or len(q_seqlen) == len(kv_seqlen) q_seqinfo = _SeqLenInfo.from_seqlens(q_seqlen) if kv_seqlen is None or q_seqlen == kv_seqlen: @@ -191,6 +234,23 @@ def from_tensor_list( cls, tensors: Sequence[torch.Tensor], ) -> Tuple["BlockDiagonalMask", torch.Tensor]: + """Creates a :attr:`BlockDiagonalMask` from a list of tensors, and returns the tensors + concatenated on the sequence length dimension + + .. figure:: /_static/block_diag_cat_split.png + + See also :attr:`BlockDiagonalMask.split` to split the returned + :attr:`torch.Tensor` back to a list of tensors of varying sequence length + + Args: + tensors (Sequence[torch.Tensor]): A list of tensors of shape ``[B, M_i, *]``. + All tensors should have the same dimension and the same batch size ``B``, but + they can have different sequence length ``M``. + + Returns: + Tuple[BlockDiagonalMask, torch.Tensor]: The corresponding bias for the attention + along with `tensors` concatenated on the sequence length dimension, with shape ``[1, sum_i{M_i}, *]`` + """ batch_sizes = [tensor.shape[0] for tensor in tensors] seqlens = [] for x in tensors: @@ -236,6 +296,14 @@ def split_kv(self, tensor: torch.Tensor) -> Sequence[torch.Tensor]: return self.k_seqinfo.split(tensor, self._batch_sizes) def split(self, tensor: torch.Tensor) -> Sequence[torch.Tensor]: + """The inverse operation of :attr:`BlockDiagonalCausalMask.from_tensor_list` + + Args: + tensor (torch.Tensor): Tensor of tokens of shape ``[1, sum_i{M_i}, *]`` + + Returns: + Sequence[torch.Tensor]: A list of tokens with possibly different sequence lengths + """ assert self.q_seqinfo is self.k_seqinfo return self.q_seqinfo.split(tensor, self._batch_sizes) diff --git a/xformers/ops/fmha/common.py b/xformers/ops/fmha/common.py index 3b172e2211..1df234f5d2 100644 --- a/xformers/ops/fmha/common.py +++ b/xformers/ops/fmha/common.py @@ -132,13 +132,14 @@ class AttentionOpBase(BaseOperator): See: - - :attr:`xformers.ops.MemoryEfficientAttentionOp` - - - :attr:`xformers.ops.MemoryEfficientAttentionCutlassOp` - - - :attr:`xformers.ops.MemoryEfficientAttentionFlashAttentionOp` - - - :attr:`xformers.ops.MemoryEfficientAttentionCutlassFwdFlashBwOp` + - :attr:`xformers.ops.fmha.cutlass.FwOp` + - :attr:`xformers.ops.fmha.cutlass.BwOp` + - :attr:`xformers.ops.fmha.flash.FwOp` + - :attr:`xformers.ops.fmha.flash.BwOp` + - :attr:`xformers.ops.fmha.triton.FwOp` + - :attr:`xformers.ops.fmha.triton.BwOp` + - :attr:`xformers.ops.fmha.small_k.FwOp` + - :attr:`xformers.ops.fmha.small_k.BwOp` """ OPERATOR: Any diff --git a/xformers/ops/fmha/cutlass.py b/xformers/ops/fmha/cutlass.py index d5181ac60c..3d37f6d830 100644 --- a/xformers/ops/fmha/cutlass.py +++ b/xformers/ops/fmha/cutlass.py @@ -175,6 +175,8 @@ def not_supported_reasons(cls, d: Inputs) -> List[str]: @register_operator class BwOp(AttentionBwOpBase): + __doc__ = FwOp.__doc__ + OPERATOR = get_xformers_operator("efficient_attention_backward_cutlass") SUPPORTED_DEVICES = FwOp.SUPPORTED_DEVICES SUPPORTED_DTYPES = FwOp.SUPPORTED_DTYPES diff --git a/xformers/ops/fmha/flash.py b/xformers/ops/fmha/flash.py index acb8585ed4..c2c32e9945 100644 --- a/xformers/ops/fmha/flash.py +++ b/xformers/ops/fmha/flash.py @@ -184,11 +184,6 @@ class FwOp(AttentionFwOpBase): """Operator that computes memory-efficient attention using \ `Flash-Attention `_ \ implementation. - - - This is a wrapper to make FlashAttention compatible with xformers's API - Most of this code was taken from: - https://github.com/HazyResearch/flash-attention/blob/main/flash_attn/flash_attn_interface.py """ OPERATOR = get_operator("xformers_flash", "flash_fwd") @@ -261,6 +256,8 @@ def apply( @register_operator class BwOp(AttentionBwOpBase): + __doc__ = FwOp.__doc__ + OPERATOR = get_operator("xformers_flash", "flash_bwd") SUPPORTED_DEVICES = FwOp.SUPPORTED_DEVICES CUDA_MINIMUM_COMPUTE_CAPABILITY = FwOp.CUDA_MINIMUM_COMPUTE_CAPABILITY diff --git a/xformers/ops/fmha/small_k.py b/xformers/ops/fmha/small_k.py index 218ae85de8..a2192f1ba5 100644 --- a/xformers/ops/fmha/small_k.py +++ b/xformers/ops/fmha/small_k.py @@ -113,6 +113,8 @@ def apply( @register_operator class BwOp(AttentionBwOpBase): + __doc__ = FwOp.__doc__ + OPERATOR = get_xformers_operator("efficient_attention_backward_small_k") SUPPORTED_DEVICES = FwOp.SUPPORTED_DEVICES SUPPORTED_DTYPES = FwOp.SUPPORTED_DTYPES diff --git a/xformers/ops/fmha/triton.py b/xformers/ops/fmha/triton.py index e50a489eec..3b7ab54bcd 100644 --- a/xformers/ops/fmha/triton.py +++ b/xformers/ops/fmha/triton.py @@ -52,6 +52,12 @@ def _prepare_inputs(inp: Inputs) -> Inputs: @register_operator class FwOp(AttentionFwOpBase): + """Operator that computes memory-efficient attention using \ + `Tri Dao's `_ \ + implementation, based on + `Phil Tillet's code `_ + """ + OPERATOR = triton_flash_forward SUPPORTED_DEVICES = {"cuda"} CUDA_MINIMUM_COMPUTE_CAPABILITY = (8, 0) @@ -101,6 +107,8 @@ def apply( @register_operator class BwOp(AttentionBwOpBase): + __doc__ = FwOp.__doc__ + OPERATOR = triton_flash_backward SUPPORTED_DEVICES = FwOp.SUPPORTED_DEVICES CUDA_MINIMUM_COMPUTE_CAPABILITY = FwOp.CUDA_MINIMUM_COMPUTE_CAPABILITY