From 37f9151de3773bb95aba0d151dd16a177aa01b46 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Sat, 5 Oct 2019 16:23:43 +0300 Subject: [PATCH] Add traffic mirroring documentation --- .../flagger-canary-traffic-mirroring.png | Bin 0 -> 39546 bytes docs/gitbook/faq.md | 13 ++- docs/gitbook/usage/progressive-delivery.md | 101 ++++++++++++++++++ pkg/controller/scheduler.go | 8 +- 4 files changed, 113 insertions(+), 9 deletions(-) create mode 100644 docs/diagrams/flagger-canary-traffic-mirroring.png diff --git a/docs/diagrams/flagger-canary-traffic-mirroring.png b/docs/diagrams/flagger-canary-traffic-mirroring.png new file mode 100644 index 0000000000000000000000000000000000000000..dd3ab1c9a5da48fedee8baab4d468d3947cfd7a8 GIT binary patch literal 39546 zcmce;cRZEx|2HlwO5q$8vJbLCMwC5{z4wf=WhHwP>sN1IB_blauBrmnBO)S$ z5)qM%l92#+hE)Wpfd4K!$Z5(E5!ED+pFFt){C&YwPg$O*s*hn6_`^m=!$5IzI>p1pogVg#Yb70fepOD$LH2MJT?Jk8N>lZ1Nzrrx1VUX}s8- zY8(!}5Vn%?r#YealarR1W|Dmkh1Y@WR4D$f-lD&ba>PESzc0Abe+=DXR9W7Wt9=>jjhS7)xt|ym zXO${x@FG=qGFLrIl z44=~`=H-<;SnpdTxIp~qdb_=zOl(Y|Brej2SDsnln0XH7Y8-HB$}W4 zBcETVF-RG_eE0L`<59(OTaura?Gb7hGqT@H35Y<&1~*a2s=s6pq>#h}mw`@!bGm@RQ+?GQ23Bj~N*0Q8ZDJ@!~)qXQS?V6XeLH#_Pr ztW8WxHP8|UbIYH)vh4&4b#v5@sghv!l-_LgEiDuyZT4>TBDg28|CtJeh9c;z6>43; z^j6k#iWyH27!;FT%94g5@m!!M?g5GaWag*^6FxrwCHL8GF{Gd*DbDbP8R`NDWJ^=~ zXEsfK%}UFX8Fv;9HxTSpZr9S?7R~&=YHCIWVVF&UNjEJD$90EbaTRcD_vHRVIzVny zAPZesRlqFH9!fDKIw{xK6Sw^+XFYfA{l(~0u%Nd}>azd(0c#e3k4ov6l{m@Ykt=tQ zzHYVE83t?4Ij&;J{pqjXwrKkU($gao8a0%1lvhH9ktOp_)ybU)Fj`mAXh1%}r2b69 zr#iHf}8dXOPJA`6Z!K@)eNuQQ4@nj%{RhAQ|65ww6fmXe!KsuhW zr_VW^X`$xYS!(iisSsC0IaB%_n&<(c_8Sr?Ul2Z6wd5%BDkA&{!}>4FIUae zM_#pX67tp5ys>4UR=L>jfF3LM)9i>egXS=+fO>VP=m8hm;zp%90IN=%G9o~lvIj6Z z<)dSn)Au|BE-W~&Te{DHDv=taUn-pOq>4%?^_7`&O@$4b3#V@q8MR1zr~$SsTy`IQ zS~>!)6GC()Eev5Av|eyD_A)4Qv@*AhsY(h~^|OdvoRwzV*V7i@DT;d*00?0PcM6{neA4FY?fw`dNeGLA5q4Cw19k^2sdt#j@AV1 z@kkxdyRSN@hyjJ_>OU30IJhm!8!#K~Gv>Ar`gSyv^gvQ^WX~310ZE_eUoprIdGQKJ zo5{AL;9)IQ5n++<6tdXr>^XZ;@D$?Dw-*5!O$*dn>(#zzBvKnGWWi>>U{+s3DqdnX zSdr8st&vLGQ7{Y+TGwP#i93>)FXco%3T@$ zAnO``k1C;7lwTRYTD2F_aHT~$v@i25q^*RAUyT7IafjeM z2T`GL*ZTVVFx|#Kzeg=IG!wg5V|X9&fG%@_;(p{@dl<;XVg3B1C86Oa&>v0Dz}2rur)6D;y?`C=i1s=Vu5<%by*ie==D_5*b|PcH+LI8VV&rm*AN_jZm=7pK4smmaKJ6{{$P^{m~ut zYixmqqJPopvES|D+S4SAh^TeI~kYNSB*(}aDL zbkhmKWWjqVzGolbuH=?@! z=!}6M+xNJ?!^KM`p7qby->hILf^!hfcnkeb4u05T_$`BTt)%{wID9brrMj=7weJmo ziH`NxeLY}81*1*8LR;|_`n4vUNZ)8TyNAf~j(i+C+!zELkiKtSSSQZeaNb#{sq?Kh zOv(1Uvrv=FFXhrb$1Xot#7S8HE%0XJwvUQe;4QQ=TBLGW=E`(ae^Q+v&D2`cL_nZ6 zZyq$rOGV5If^r3hpX3VMK36jvu1S!Odek=>d(AjN==kn4xN`j{Q*mtwxURD}xdA_{ z#wi`vbHXk^O|a5|t8~k^q5?*1T`8bbid%azsaNw*z<_19iBieCIhEJ ztd1<}{m70&DBE?On1IpHgEQRdo;TH|l{pqzLC*;9RMbs#z_zFQMzGbCefucV8|}=| zqG)oKJX9e;SPu-lt+SH#P417{sZfS3x36yo`HB@q*k~d<1t@j+J~zg=X2@x{Tkl{7 zQiCgxCPf1^@u}}-+%@2|=w0at|54GhID^w@7~3lp8*rbwr=n(i)04>j3HuzS zdx3kqxyCSNfE7u0oWbpdt!4D>&HfLQt_cJT%sNb7tD<^LO^IvjR09M*OEA^1LGs z1CM!rdDe7K#6TdVAwsd*kl&{+DgdlLvc2lilEKsMGhYoafhjyiEY_FYcn23CU+>5L*49m83wzQWQ?iFN4QP-ET_Z>-h)?yW_|*6V#Nl)XggvR1dEq7|P3{B)i~)miFo^KIdqzfc*pc{Gs3*1Y*mx0su z6VDcd14iO&^Q0r3qldz3M5PALb@Z^c=^f|D!P0J^8%wD@S#xR>7#gBM>6COxh531A zj5}+48{B)X8-C!w|M5yOch7tOl)z96d-AtJNwGIzpH|^me@BWHBV~a{7UG6@&nP$C zsk@oivd8W1b(DIn$;$Pirrz4UEJ5W+m|(%t^X7M}2><${Rh^psTeFWz6*ZnW!+`{x zE3wQbmC3OvgS%l)-LD~yd!I(1iQPwfl9+^a3odwYFGQ!_@%3|eF%F;vr8~qWWsGm> zJ67&SeV~Te5A3oZNQ#hqmG^Xy4i=I>Jk_Q~*W+e?uZ&h(6;OiQN7 z^2^W-?|IEdeX;=kM13pn}*Bo?8_2I^ZwFtIGQ3%k}izw!L~{HZTky^9w?CzLrPr z37PD(@LFV(6{oV2fp#1NUIa3wGb3x?o^cYRpWYj^*G{l4?7I1my?^L&yj!WRmW72Y zRp4v;i02F5*}+t10*;PGkudHICaDCf3&=-Z7RR%@h_TWBpoS33Pk+vudPyXXf-$HK zVCx{7=zKfUF2j59@_WhN`hGc%yfjC;D7>KQ>NT{l*z~7CMp^ujlJEOd zfzwsEd5)>LUn74+2BYUPDz1tee=LBj_x~{s-=pQc^>R=T)#=?E!CvmF5buX{4A_;F zzqrEWG%z$e+beR~&xhRDZ+$&*I`=&*XJU?P7Ta|FneoJ8_Rm3hdPh-JZAXEKGZREf zvQo!$e0Wc7{84)I_y>zsXXEuNAfJCBbt7uRe^gcynV}Y3QYhO*)socGvxyc@p?kYEB*S{#h61b=WFVd~t1j`+ zqkry*7_bRnWPOoM_HOM3=?Z9ppKta$2BA?Wnzzsl{QBI?j zyvZ$rXDvoM#Xd8<230ZFKtLLxD=WADFmXLR?I!|o%URLZ2eWfCTSb4`JAm~r*crtK zRvu$Z`AN^*dmlm+DqaLHulVmuUPPYm%Z#nmPv9d3;EUT2Jv>_OjR{E9_-vp?5IeG6 zrLv1zUQCy?V7bdL(K}Qt`y=?wf+=gfGdsO51y7tj>gYXoVT5Whep))E>W5q9qvDA! z5^z){-Fh0?8z7?P_sgsU8^w5kSw!6#f;zZe_=>0b;;*lU&JuzTH{5HeAd?P>-QWYxH*KE%$+)48=&=eXKgPh+oQX7V!@Czo~g$H4EZQg5O)pqMf#zF zEnu#3)t}5Oa5dC&z_k>ADb|hA@I|KMz1p8CN(X!q?~?0pYz5c7ghGdQ7|sFPl@VB} zAI96d!%gg+AL{czO~TRn7`Q9X9J}t4=9rR$0<|0lcH}n!r4eoqS4KVaGn34^FQ%l3 zAQ{u|Mw{w2Y-SQXwNbygav2@Mj%)aoyJ5q47g2YElm7tswH8Fuvcks>NzY!+2hh8Y zT+B7^hbLNA{&7XoNEdNC8TkNC3vv(x>Tf|7o-gQuc^WfVA3_I!uFj?R>SsiQ#C74S zwKF>|-^w@1rD+Ln%lQT+Tt2d#uF2Nv`Y)UYY83mKZo->Zhks{Kgq~1*o@+$iiInaBSMIb^Bkge&N-z3K`Z2%15-R#!kRN6DD3P0 zB63lnAKRf-8wqO&U}CHGSfP!1;Re|K39mzZ5no%xwZtJVl<}+w4o8v&FcG73%^8pd zUJK7LS5ic0%&l;q@-mJZ02~CI3Nk>fL-42EUGVOq@LvNR|JxPUg*qM10P#Ni)m}~2 zT7shXyj+Q&FUj6$?Y?#}cTSZ$uu=D!Pq5?!Fa)Y69DCdA7mBLF#9ee$XLS;$srv!^ zg;c=|@Sfg_rUI(54DI4YOldlA)KxKT4sb^^@`8fyq4o$ci)m$Qm?M@a>P0#KY@qx2 zR?iN%npg>-JS(B`ni4Jv1XQoS6_@HCbrZd1{7$dfQ{4w*gdzh#jZ~0BJn-tmpw1-C zF#~X;sMQJ!GY?~x@eBpnV>rdU&Hx_;8YD5eTIx5oJoiXTMjz6C9^|tUPcH$$^sq5z zUH8Ji`!P2CT>=j)AsUKHXb;VIiCCxu>15{T#`w#rD&If;2ZAbY{XbbN?a|&Fuq29C zC8BY(Y+yp}qoSd{;?7=OSSZfO%kM=m@+ODoKXu7)xBCuW%A2UgcI@@JvsWK$$B-YJ z27Ni5^sbyo$+dDT3VWC}ZiZGGMT#suci3wfuzU+56wSZ*Hc=t;6QdBWEH`QQtAQi~ zNSX3C1`>ylVOW1Rx;UVAzQP4fbVuSTKU7|IE{-2k8qNjFUzDQ!8^7nX1tE03^?qC- z4lUVF3Oe`q1=cW{n&j=rt?|6cp&8FQ0|A-$EvAbLZrar6&yjM7;xTb;ul^2rl&{GE zd&};K>qFwrl7Fk+GHJcGpISBi!tA9~KmEG7hrS6x?Ei&AP5Iz?$Vzq+H6o+eXSk#V ziO&jAZ4`+1@EuR@i^5sVH7OA4k3rnZtNMLD&f(L0e?249dTH*SGNJGKgP4>(69J#r zo&)C%MnjQ;OfFBkSVA9A3}l77wLV>Xg^jxNBWVrHgiSVt&t)-Z(R593W&I`|qm;g1 z_vBzyCGI?1#Aw0XrWlf=*nv|`|K~zaMhnLzrIwA8i~LjF`|*4r+-X_iPpei}M5SI* zi_s}<&D16|^*Z+$Bw*%~6lJ*E0OQB&JrRRQ)bTIf5sJK#d79)Go=H!|I&eRy;wv&X zeplZ|J55_pZQyWD`OTaxKcTN!GWU(sJ; zuI#tLYhIECWT$0|DTllHtBqk+u0GD5kbSh-dAE!8fnd@3@S~ro41L54$4KanGI%`` zX+y!c|IW27;3h^Gxp8tF^%zsbs}Ta&o@f3qz2a1nY401NO^R5 z5|_|S7%S+dd%YC`bv|%Wb&soBZzWn@tB-L;u4`QF-HkrdlL}nRl<4XBJvI9n3M5RceC~l zBNUui0q9?#3O7EUB}cWsr)g80;#h0NpBbzS(fWLUVeY4^+HyM_ERXWEHH!k^F;zeS+vHU0 za8iBY+3DfHGwTp{lvZKDOyh%F3DgRdpnich3Kqbzqyii{aD=eYJM*hw@86_kLE3+Xu@cr_^=rDt$|Ixh? zgL$Y2>w;;&=Ow2ArmGC-_z1VXGt5}^!;s7D1roS7Z}B0H54S@Tf)4M$HKLwHgzg+1 z_~u*@Ta~%?eco($4>J~c5D^mE|13Y3YWw~2X-+fmWTL@H9@o6eMYi^pk91IE=%2xw+U->cNc1V@Y^DlPh5hw^%+~J?Pe&;%(#t#2A?vppL z2XL54>b9SudW>`0-Cw8&t#rO!2<7k8fB*UicYrE1QyqHO#?va8uk`r-1YL=FC>&Y0 zCzt5^tYX#Pt1a%weC{tkYOQDC$;4;Gn^|-%P8JOTGQzd2Z*0et)wx0Yb`$hg9K9R^J4Twhhb?(upM&GO7K$1di3e( z_OIn9`c?L@Zow^CMZXu1mbesj)?bAwQ;@}L%RGKMK%O5r?jK7eba&=q>@|1lmU*kG>yRYi zF`D?xZ3+hWYo9>;y-h}q)sPvt1{rk}+>%lB-?$OoH)!%mnqY!StWW5CFD&fz_bD&r_5tonSoNt;? zx5Ae&Si>@e_`+|{%dp)rv9b-K*FUnxc9;Ff#ZSUvX#_Y|p%=}w9XB+867YQBE%t#Y zx8wEnQREz&)}I zVx&jAo|b~m9$KHnFY!f@BnUBO+ungl-e`vDbm)QvZ?gVYj2t|2zCpt49@2Y#Qq}p+ zRSWNHGZ*O5?25aIum>41D&DHFW>qu27ndjUhS4q>XxH>Wda?04_PZpF*C6dXMMQZ- z5c&K@3CdqB8ExOW?z;T(bIi5hX)8cFiw4MfF{~Ooeu{ZJbNGtbzx#@yGU!WE%i|Vv z2s-VG*aaJCbm`d5cjaU6u{Rm+vesOKJ|sc;SEo(ZOIlIqNGC`O(=iE|^#O<!U18oN(IqBMhh18c*8 zTdus3=VG_HmH?7`=^<2qTPaj?%`NraQzkh6?a-J~ZemH*Y^HInD+ZxySs;3SMvlC2bZCGEs&#mrrKN_@;-ffil=pAzVM1|7d zVLM0W zux35Swhv8P-I#s@d(5GYy3x!I>AM3lirP9=H5;V3ZA(*x{r*BW<$+UySS~%BYzu0; z7~CNab>6SalV$sBiyR9R-RFZj-i5t0XDkS0PC9U-B&gJED!_*y)(m+?V_hq;pOPSNL znOm!sM|ATXT(*$*cP6Jb55$Z?ieA!ok+JCmCnTE875i&_zA+8vGmETZhwSGpGs^w^ zBO$HwvGpaA({efpYiq2C3TB6_Q$vXLR_JHsdy6fgk+z>{ppiUqSd@Y8U0Xp=$7$O0 zRrsbn+8H!w$6nvrkXeH>KBJw|c*IzBNGF8~R8M`q>MunwPl z8P{U5TO_;B1t&wOS&FOI=h?pWi$y36NWTm9Mnv-J9XOM>c8?ZU8?H`h(u8JkK>+KW zi?J@Vsk!OpQlJ)%NGp)AvmhuoMYxY2GOEu8^v2(|_e{*(&D`BL=44i7zx%MqTz~K%i`mQwmhw3ALSbJehy04eb__H|_Bh z1nCn-TPmzX_BV)K^N7xNN*D1U4QXiBvQ;3wPbqu0`~ucSEiBwSUP2A3kz5?64(~R* ztfa(vBk4!e13i(_!d%w@6_Vg^!FFlIJ(~B}y?1>V;q5oV52YbT5Gn`f6v_I#eTd(1 z6N|pKxq*Vc1VinM+Q{T;dk2j{b;yDRB)?tEC$h!fz&(2X+VJW2Rqx_I19|y%urJY& zJ39O*==;g>Qucl79=a<39D3b&_!h?{wASyuXF6*&E_Y&X)mVqcoAWl_cs=)l;>O3-asMqh&^5RVRQZZ5$Tbb&}Nn~ZKhKO%VPq4^)^IXW7U{5!Ps zREb5W!UT7?=!G`!67AhUmBF~y87wmv-u>CQd2p8Cltq@deYD@%5Rnw5L2e$HU;T1M zWRyDiBi|MTr{RUf;xSBct}srsIW^g<(V3phmDlCnQs zQoPus7)fS>unQ6Etk>Zr?#WDxsN8U|<4URbyj<^|yIK0)phVyoT}!80cUAve0TuMq z%G^F$c&#i{<>b-t?>R9Fh0_DB=DW~w9?%=6!|RKrnzN&1R59Qjk}9`ufx}B6UCTbM zJRm#%P-vXPIsiv5*+*x6gl(BkNwvIhrY|ym&uc2D&yq#;K(Pz`qK^Fficy`Z?<r`pAad`@a`y`-Os zC?9GiqdFNe+MAnZC^UzM4bPC~-prji>Ua_HK0vXl9wY3f{m4#-oa# z$G)liv=8)v2qeGk+mPUKI)wMTPZ1VzY3)pISaZ19QJb?Pw2yd@P4{ydtK$X$02~S( zIlc4V<`#dgWnvs9x%X#Z!ZX5Uck|_zIpBE1{w_3 zg}RhfJ7p^?0s4X$m|~3II5r^|;Ex{eO18F5RUgo?|19bPc3n0|Sw39?t5d$?$E?z( zQ(aaMI1-Mu+Sq0QPuN7F5~6qS3zjwSph>0W7iPI2^&`Lsu5VhhXm=YWXVzxu~`&uP*A6+kaS5?yq)d{zwoSfXm^HDx_Hjmm; zNPLYtt-y6<25y?x{j$e>W|b4livAIeoS+Uza<(k2Zyx`f`lpQUD* zeF{s^wYg&-%lBVnObB|NL)VrkvFrB*jl~3_2dgb^@RlGt^L>I&3h-j zk`gz;QUah;&+MU)-b`gcW?%HU7}veG46mDIXR`eIdYp10r;qvDP%t-0IvLgyrTH*f zB>K)wNru3jJQ}ZIAP%l(nNi4JBi*iGeX~iI4~^7;FdR6mU973V8l&T98N3Uyc`GgU zCZ`8pV(`gd&cb;}~ zYdxt~IIJfg_6HNBe$~7tF9}RQGMl2&>V*@t#US>w1AZ{&b24bJ=`UR?e<&9yrq7>z z*Lg|u1yUD*Se-t~YL<#)(UofTc|GnMRO##4*HA49c?;uM{>;G+#Cx|KcN6drH)IG= zesk!t0;xGTG}}z1o$BHOF(s2-k5?xsvo=MO4#y!kwa3!f@DxdhgPBgaC{)5M_(mu) z(OF5RK0z1qnH!WIC46_V`rQuMniBe~Xy+9s;58~?zaPO)!C-}v;6Pp<^dsU zz(K&fjLZgP!0!WU@6p!QuZORL@wM5|hc{7JBY=ymo=nF0rix%(uo_DOC{?oZ>U!_l zyqD2E%Px&SRNhFhybOI9j0H5mREuAD6le z&rT*o+h0VSraw$D(Lub@h9h+kvZ|4o8gcvxOTCu{hpvVM^RQB$n#yh^bX`OYoVgE# z_jBytroq7l?*cd6M~mju=94k%G1MQ4?z?4?P`p~?KDRbMA+VZiw{Sq8hN;iYiKfi_ zJ0GaWNCDT$X9CB;*4q zXJ-s#M4VB1Y9j4inxYW%eI-Hf~HoQ^a?~~}~=#zPcn&qcG>);evkf?Yuq&Ha( zm>?7!&nwZe8O4P>zFYlF|9%DbJr>Hu1~Zo`avLZ=I> z*P}O$%MZdIXoWrPWjkUS`~@DYR?8rS36w5ra@N^*apov~Dx8XBg2-7mhq=lw61p3j zHkBlD-Y^ms-|A*k{+xDg*MZoxmHAOsCLdoy`_-YNO|63|@Bj94USU5V0_o}K>LCO7 zvxP#6BB7h=(Ai7}Z9+Tg3@QShSI8G=q%(pW+pd!XcfB}Fv<9lJ1-jeoUw{0!HznG3 zleRq$ujj|3YC(UlaI`0Y1)nK&y!vN^RcWu^V$J>mSh$b$Q4t?g5efV(752m~J>{RL z-)xiaun3Z)Vx7pr9bBLe<~4hce@KZW@f4+zEXaBTJmgR{l&y(_DuB`5TsNnUrz{DD z*Z?v;VHJjIJ0llX!8EX-|9<)q(1zcL*O+Em0i?10Az9sls*(wpb&Hypm&n)y3?W5T z6hBsg<|AZ^R8BX$Uq(=JIgAYcCIzPIIm4hTybJ<5vO`N#l_vrZdaVLCap}uobz;y2 z^u=FLET?a}!6I+{jKkV2rU5@$pg;vkD=2%o-|W1Nk^-JMa=bXtqU&l_5@IbS9Iv^_ z=SI~(dd>Vb?PK_3AblCFPt4t?wayF>)0gnZ&9(K4L3Q@OXxT5DLehW{dncvp?AYBn zq|_X1?PnCr_t=NwaRf?7{URZ+sC4t&{$p!yD8`rMPYd7WAlFP|NLOX88~$3gUUhZ; zjXrZ={C+s5V>6ywUbmRh&8`Fv<^d5RrZy*;yI&Iw3&#?#6#Xr7+8$6k2`JUvED)p*xS zuX^|uQOfyS2Z`rdq@^Qb{#_nqhc+3iyW@oRW~2XTVCp8C@n@&skYDobba?d>+D(8Q zwB~hp!CxSOVnzztn2;CwalItY@S2R5Izy>@3vp+_qxN&kwU)D z#jhnQ=h^`vsBR}9ML{9_UxXLY;zF4yz;y)hp|mgH#{B4kuSm?tRNqq})Ayz<#qXAv2Nr zSg2gS?{_iZ>d`amsp|U^w}}NG-1uwKwR9>V#?QiAh6rxG*0OWKVKS(40=3gga`%;& zHH=WQEH}JKi?O6Ex-)f~7OA~om!>&3r5J|aZR`^;gjxT!z97$XwV>1`pNqQS;j2i< zIci*M6k6!PBu84*Wb(d_0et?(g*@Q5K|o@nkrDd7>+H8lYZL`3Nl>>sBf(!@f{xGk zpjRjkJ;QeS=e(r95{gms(KcKXz%lq79^Yqc@#(1Uff-L4+_(6zeWa&5b^LG1_+xXI z2|KkIJO7*M+ps0wyA|QISEK)dF427p#03|X6FD6&wRPSN(}8j6{evYZ9A&5P#uLaA z{aM7nYt5D#_5Q)7XTEX&6E;lst*G3Mks^^5bvtU*lJnZBuf+evww=iJ5}h_3Qw46i zGXF#H{6R^u_cF)YD=4!#w3Y@vyqx{%nziTBg97NYCy`?31Dw?(ub&-L8XJ7+xqSv| ziMy;NBkO-Gs;K^mjCgCW)yc_Ilaq8XdsX=yvGfA$z%U~xJbynehIUnA%5?!!K~O1r zu9Dgak^)0l+Suz#x@v2w>;^Z{6PzLJ*qO)zc0u1^d(GD-51!TnrxdmGX(`GDDAm8`Uv=CK|6*~oPa3dLefmmzV*H>dD?Trt zNU}ayTRQX|`M>bc%{M$Ep}q`8jr!ou*l=~m7!+T$bZEQB6T;rX7>O*j@upE$@uEr1 z=vvCQ=Vg%Ez6*26IqjcmG$yZ`dq z=dIa#P|1EW=s=(;JVu-NzSZX~_Y5}_zo(8@w`LVkByh#Tylv3BmK`HmnBfW`3{op} zZ)^)w_kq0#Stg!SUS+sY==a)jkiZokx7Fc3v3m z4}imgkd}*$sfX&B+su>KbtLUK8V9vZGI$>vBV);j004(0pHpxXLHT zfb19m1w{4P!A*?lT4?Qxzka7_Uy~KD{O2yHD@QuT@>DSjOJ|!CJ)9mSN??RMIgo5a zp6<48njlQ^UpxhUucLi@W9t>*7jJ9 zC~>4q679XpHY|6^Wx8qrlof%v4*q~WIi9{-Z-xMfzkSG|)lj4tt6NM1j|YftS01`g ze>=oC#(N3ZWx-SPsB@J0OC4snjL|`%`I{nv!3lzVa1(~}+gv;qIF@LrO1*xm0lt2V z)Y3XK8%)W|`}x!re+weYRU=@*?UCh>lkv=8%6<+1d{ZWMg1EOisCB_S$aT8E23Vs{iLkV{bdyrruz|qlbRx<+|A> ztNc!dM?LI1{%U2`8I=#ve*!Dnq^c&qXBHF4@?Dl80IzXvDodAeHmP!6xo3B9P~_8!&IbGe4nNtBDDWod#kZ*TR=8Wc!H z=fDojl~`;3Q%VM24=#iSv5u7D1mj8I;NL_iMK0!w zp~%HMnfpRc1^78qkZLC*R(tjSE=}M-Y7WufqC!~l+He$mIUhhd|GD#G)Bkm%EVFrN z&*rWSV$>CZ=a!Gu=cxDPcuJ8G&`Xi?bA6xc$usl3r4OZJ$sY{UgS$UStqR{42F3)< zpFn99C#ssX9pSYo7#916Sv0|4Z{d9?_bFRHzg4-b zHi76zKt+{ocqO{w-tp{Zj8;DWiWSTx?&=RQOE(L<9L+&x*f-}>?}{l0SDL&$;U83l z^p67#R1)f*I^zBTTYb=aeKZS-PtrJ*cJ+(6vH4`nCVlT$a$`$Chzw@mTw*M84FfD< zVrK|i+y!I-a{G2@QnB(I(N<*_il*kD>$Pj;#eqzRR?$N&3Ks)8rOx*Z#PZqlB+_+X zMg*J=rj(&v3PFGa$J8X7vq+KJm!2B0Sp0`24$M7(C2o%BJS!ehdB(c1{2u$n9Y%W{ zk#MuvwR+TYXnpxREL2^1y7nu!5lw%R}6y3NH zAka5)5Bj4Q3Tw;nzyf;>ZY{L0l0fbrL$VccMOFIm=WM>LYyNmQjw$Tr*&HTk;Mp-7 z^duPD;Ph1fA}eH_Fza*9mcJk?_gbG-N{H^d_xt^xSo>fM|7vqo9|?QPA<{tw+B$6Z z5|RH87r>i9IQ`%TY_2ih=4+wTne%big?U|{Od~{pRD4ow+Z-VfcH5A_`!WKf0ByX6 zqn-XV<$%s-g|x%CRV{SuT^*I6r+7p*O~CoCnqq?Hpk+CixuLkD)bgZd)3f@Qw!M=tg5!QEI{-a~Dc98t1LR zW#8ya8nB@Y(;xVy=C}0Mg-%o7Sj&e2(^1lIaXz{I0MlDc^B>c@4OwX78}HqvVHz+c zKe{P(b@njBjLz?`JAn75FfyV9<%OOsa*(ug&ZTU70(G$s*=cfc1rtvfO5=E~36s() zDG`+gJn`6*RI-eK1O9o8GCF1N-8bQ!J)JT3r_W7~I*m^Uwi6QeXz!R>tV$|G@?Bxv z=IGd2<^bhw!X8XGW&D7Vy*S}ghBp?Ibmc{v{!LOv27D~fQFlP~@P!DWVJ|#J--Sl* zYCI2x^-kLKpN=PTpA4SYd>ADLTjNG17aeSBR;xB9Z=l(sc2Na$uKGGJXjVV}SMpRI zLuK%2(l17FP2n2wg@yBDkJyXD+YwVcF2I+0LA6tj9I3O?CvM-8qOwWnI*}V!ZeZfy zwXgcu*WBf+61u6ioTBlM(M?ogp6RWuv8fkbfPNy*nW~q0`%0iEp>+N};a3zqCvpQ{ zf}ICLgt_H_))f|S;%N&ROE*|3Q2Q&J!L9$~jXG#yaAFS3ZJI&_QtX$o4hr)Yw!6xG zwu#-Dd7~{(n_u>pmX$zSCW2HpsZ1i@Sj%#bSE%d$e1A@#mM!vL<~H{|VOVb=Vp5Od z9M|#@JJw3Pb58ds!d5hOA-}{9u89g2(=9wOmEsqP0YeC_A4bT>#fG%0$AbjQwsBnZ z1;^A`b=5LPu4E2zAdYlzwocd0pujHDajm#I{jQhb`z4WtR036C>I=Tn!DT;dBB{JOVvWGqu25}USE`ucNgoC8RT zk_dB|?m0tG>p@`xjRRj)b?B(|PrpvlM+Z7T@>{71waOxhF#?(R=37`To((o1PsxB} zWk0)3oZ8&O?J>VgUcc9~1Is3*%JD9j5o_l?vr1I)c=V;2*MkAq5^4#35Z~cV)ZYn- z(&6X0JIqm$?N)9x^paWAG9+{P$crcX^9T1>`k%TBz949&qV`_mQ#zNihV%@G6^lnS zEWY6H6IiN;AE+V+SdjqNwzN5^2fVK)6TP+#8Fb2psMHL)!GdY|z}Pru{&zaxnlkcz z$-kq5E7Cy>|4u*7oDuiA7y@Ba!+<+)K2a4trnzBd_1Addje=V$e}D1YhU>vGEejZYtb;HJH!qGatodvz6qU0XWbApxRB+5L(6Q!;_uo;HcIf1m%qL;u(F z|Ml_zng3Vn|L61nA5~q8dPmqF(kL*)x!_E45}pLQR>>Afsl7iNCa`-_b)(F^tpUsS z2($)X(yhyW6`oe@uKm5}ntPEE*5vmC;z&u7S6b0m3spi1#q$ZdbPf0KU7LeLlhS=F z?01IYq_Rv7ZLi11EK1ScTF0R7k{Y};W%~qwV)O9*S@o#vYkUzQ;m5)=xZ!7peTDEA z$IoqYAGOcS$T-th|7eY5Y>R$Qv<-cjk+hNBi$(&}@qRg7k+9oCESrX-dd7g3G8}oNLl=qR zsL|N)tYK`aQVg|%PJ_cJxqD&qyJi94AO|G<7j|_mVYFGe zHXe+mxtXSz4E4owtd#kJFi*l&T&v@w|AFVzxewy1hd}|Ar5*_(ZsGFcPxSYoz_b_W z-4uecH+XZ7f+_AY%!;)7jlE|B6VZp*@=gY3>rNI|QikVFjw8*BI)%koyVp8n#Ca`& z!_`IgPZ8Vn#6fM(9v4h!Lw0toWH#%+HM;sQ4dq-&dC^9vwj##52FkC3>5>-gJ_dEq zM1fu&NsNX!+i^=!{~-%oOm|o8h=uH|TFLDGV)^5|MT60~y0SX06SuNN<>PKx4^v%F zoCXYxFtja$2Mj3rR+?fwyK4y7>AN4%&CO3lV-5?C=xs+SICXi)hCMs)&x@&5_q-Hp z&k&@x7YbJ~REXTfZ&zgVKKv)wVo*&MW~#Yci)xyyw#oeUEce4Wk-NG_`G*@q{t3{F zlRh@I^@Vq2Ylm!ms~>lp#D^jk+}tL@w~hPo@!U&FLq+*MDwS@Y9Om7S|3jiJLmI)- zmU0NkvQm^J@A_{28k0|IM29-wD5q0+Qd_n0(6@e~);Tq0>VeHzt|Wl#HX;s%FjR1a z&esY)fA&VChO4C94D3XHJtf^C;@7m|r{J&hk;(#&N@>bQF^Eur@J8R2T+R)BMjs?_ zSl5>p9b2N-JBhbb$^l~vLtYxX*-!L)@HbBCz*4G-fzunGo!uk2V(@Cz3 zqj+&951N5ahBQQm>`)E!46@T_B{O|gwe?D^{4=8zQFot;`rQ-Sq$McK<4km<{Ju2w zqp?DsxSBE8LJDJ2!HkmL?9sNYkYPD$wke-}_?L7X&IPAia3gzo9rH=mi53CBFBpif#DnDiR0TcC41GHLgCp-EOkhc^0NiEo=D zG1|YDr3(|Pv69}Y+^)UwDk*)#4}Zl3_X5Jy+vu>G66OJaRr#z82)SB;1DNvTmePwi zGH|g5#6B^)!H1xK>Q2J0tt}}jPTqm^(1(Qaci@8Eum17?byXdRufn{T3!CJ)fplhd zOV6p1uli_HAk)Z&K zN>bD6x}gnM)poXmO|ft+^%_c;vITe~grCR33sL*+XsjcylMWgT$O+I$EaMc7Q7*h? zW_D^b{$6RjzK!8TYuism@4Z22!*G3}{6cmhjAz z)w^Y)pVFc`O^QzFu6AP@VE!PlF0Tw*vuNZr#rc-Zx*==5+6OJC=EHB>_}LmCg7N!1 zMynfUsuWEISHTMrZY}%VUj{JLtfJAZXsv99`{T4fvMNl zq7#Y`fV_V_mg0Lw_opLiYgikpGEz;}9G4x59|J0;OirfJ1^%k8WR zr1PU*_bb0O`Kwa~2z`5*eR}hEHM*Cb<2EmR8v;Ov?B#$Wx6<9Y!yyfR2ZqR&+Ak<@; z3V%P0wj~OtBjZC5vbR43(+32^&~0UpN%mg-OxS!eY^bpR2sWKBf!lCwxa-n#`k2Wx zW@U0hElV+9vo7(Y1XoWVeKgFEBJfy|H{v?CWCo+^ju~wlzcPal4T}UQ2CDS8dt?UY z`zAa3Pa!)sRQ`yrq4gKN(S6RTBhzbN%X@fU!5aUh+ygw&vCUr+69U$FT!9#kXjZ&3 zZ_zA6DzFNW)+QER>M7dbQsYcSBZ@}&*lN`+dY=Ds%`q9btCSQpvKW|ue6748k-Daa zD|cAkwfHHv3deMoM4+p>)%MuPI;HDWUkG`PLh8U+iM^4FXU$W^ya;>q$r6FoDu^hM zM=56KF8k=V^soKd@IS$=v*&?15`oNWB-g6{sl2SC0S^zYa8e{bKO=}z8qoW+M8Ufl z5`)A8V0Xe?U04h^T2pNBdK#y3(Z#&AvFt7-O`{SIy+=5PJ&BGrqd*gWe!m<7a=fIg z`yE4jr-txqvV$uj51ho#?+>Y@2Ra#`s9KTIRF%t9? zrtdNuX~(!6lnB?w(-5)obZBN?%7D?M9l||Sr=V_L@;LZch~6Tpxu*eh@3QGmo>)z3 zqSf7!{;bml7CW)Zqx}>Df**q4D&L&}ZfTOhSTbBD$@#ci$+(au*=13=E5#>=!-ZGs zcHPupF{hgF*sKY#|2BD?R5SbHUefP~rsPgS_8e~s@S)1P&}PCsTKx`m+o=$uE}kWz zkQs)7fT>l;(vp5@uZU_``E|@m3)Mw@O5OX8Ddo(>nbXA{009u8JN^Zhqf%7~>R2_JAhB-;|HPx9@GqdyIrOUZwbIcOU?J{E3e2HAHv5n(- z6oKnU00^QO&SR}Gh8yj;S3fK{eDTcK{mfuiX3B&0u;|QGeQ9R#Igye3qQ4`T zpY!AZc1kw4C_!MF+d+v01v~;@%_1;4IQ?HkyLPxJ5@NM$;gS^$x2b7&-R{+-AWzUB z`pz-Hvx5x~b#R#FlosCo&#@i7RG-4Hmmu z8=q1jSyE$^c3!*oCp zCbeJ88;=*J;4kro5NsxFGMow8Ilm}YHIz!8iX352q{E1wO zai$Qsd=xIKv6!sGC|Vu2?7A6gooK_*z)lhX+P*JTwZrJvd5kqHA_+HOr-Z)!zJOAE z1%pksFL6-l4nIp7#$Wxm81c+hF<&roh5%e&&Md*zRF8?tcQOn*p`eD5EDKWj7-35{Y zxUG_*iWc_}KH-AP(XvO84GCj+ zZy=L)$ATNP)JNoB{0al(sN7)~g@= zOCQ8*u)lBhEhbG#7mI2b!33D49`3qX!R7q_u#K^>@LnEY=OTI1P7;lSSSwkeF1-A? z7%_SZ+B->r9#mHp)ZLgGeX%C)lQ!awRD+~zn`yc{9Cr6N`zlwZgOJO^UezvAY~QO( zsD#P>6D2^|APT;YgGCM~A#_Z=k|&jxtx4a++-KBH9bvzj-{Q=e{lUk#| zY@~djsbC_Hn!}2D#iMA!`c8ArD(%uGDp^y2ogx%+TM+Xl=AFneA>W5x-?FD^p+!vr! z1z(4~qR_mE&oV>8vfWcJwYkRwoP?C~4W#fOcs-gmOC**Etlf~KkXL6fc}AOA;Vg=9 zgEL#nxbaBVadGcbOsR<&6Q#Y}b&D7w+R%bt8@jussN=$>Gp^P(F>ulH4M}x5DzBM0 zex9wVHnw&WD8>DN>1AG3x|K{9kKZ@zK-r8wk1pgqQJV0cn<5WOMn1L~;8;O0*ml1C zt~8`%br$)$a8xHO+~#!vGAA(2>Z3(Qb4h49wL=XP0z)E+9*^d=4y@O9o9xu0q|r!D zuo_*82LKWBun_LK#d~uC5eVGNns#}H)ukbVS9t7N&C{W;{C3Ja`o|x4(oI1!`v;HR z15r`)pH=FtQ1dM$L^zvSkLOht(d>_yo%=h7G**cA+z5$T6z_3wQPqX^!f#)f>X7xw z8FH>ubvI$HusEYI?!)T1UMbfjqta)viFSBkU7}DyRvB~AB#AeaZ|+vq9}f~{rfB^B zmnRh-yc>hNZU%Bih{O=rP^_fi13L%DXU{h%%}ZWYtKkw^+bS3o!KUWMhKf=gbtS>v zN2Wyl(%{cHmGm*MCLo`#`&XGnf_rs7D-cv;>x8)r-E70_j$BkEi@!RI*?B>{@J z|H%9*Tf7lz11%yA-K>@W?ZbVvV=>`QkIDYlrlsZm7mx3#e(t5A8!N}y8Q``)h_L>? z?rg7L+Mv$8oV(li%iEr{mpJ&zp5d;W?_<$%d7+Ts1B60X;XkN5^`h0~#o#78>C_m> z4_Spy8T{bx+JjOko>{?Rg|K}|h%<#0dDtCtY zsoF4h(zuK z3x2mi1?Dtmg!*$|TkoJ`SuGRFUNwCaPHJm6jhrAVaytH10o@8q*=fc@Rm63LUB>k8 z`qlKN%p1X-p5MQn`yX$)lYTySe@qi=o_WM;F8z8A)d7h2dd~2 z%O3Uajp_EKlWNfyih+GKrt4F8fH>)Vpyr}KuHF^3Mjfdx(8t6?8D4I-a=!80gN$gi zlvzv~aS!}Ch@G{dmHEZknq%2fbwuah!Ib_GN!{KrLb}0FW?Gi{>}$XslKAtC4V}&>o>S);3}?JozAsn{uHS zpl)RAR_kUQAc5-WIC_^g^#!@&1SXyJ^{5njVTl{TloE4#cfhKrSoAdIu<8}XL1UPJ zu4ni1iGXPnHh`jtG1_Y6hkhm;HmF8BlejZXps$5EcLA5cyf(W@0RaBolC{DeF7^?E zvJ3Hh9tDSQ=EY#WJeAnC=d^5|`!(eG)4;)1;4PEOiL3H6B^&w1D2%VC~SN zD`gR|#^fMscJQT9JF2@sBC((I*o80x8>d3>XtlCB8r1JEUVLEn8Ss#Qv*9E7vjC7X zG)n@!2cf3U0hOnIfAR?Y#}XIy!m>Kryu+CZoaX55J&g!W?A<%D?RyRs0RdGkXiyX2YfM@$#kcI+$vN8vCP+&s!@WGOvBNiavO)c<8%55m)h)F2nqcq_*-niNC z$oHnVx^}+Q6tf$K_f$vwr+x<_->^ii&{Pu9wal?X7eU0D@-Dvny#j)_jNR~i)tdSm zWzVLHT5#W`va%OoY(O)!8t2csjbE>_a@d!ywzC9w+^rv5;tnJgGRRbQ?HL>pTAyOK^tdOm^BFbF zZGxn=Gdw}>By)KlFA*r+Ud@jRomg{nxbfKMCA64YjG_p@iO%*NLA?dCmRypQ`BEl8 zirj~48`^3{Cj+DQWKHU63aDYZnd|80!{vD)-F;)ixC=WWxmY0h3E}x(&C*O_(&Fcf z5=6B@AS7tdqM>5ADovo}pbljN)!ccAO)(PRp7PdOL zoc;tW(Z~dF%t++PKET~ww1tuF+#Nh4JoZfqHXaxi3=Ve{pABjCwfSgb^XyxJz(bHV)SP@JoY~lcXLe; zB8buOk!S%vyygYpva!anm^{)ORz4Q0h2-VhnN|Qpf|;}6<~Nzfuj0UsX6j9vF&_P*C*t*vB|<&U4B!< z2}(#+*rpq(4*ypQm5#=M8Ry^dh;hk}h^@AU>;qZwv3YMmN-V3t z?uZHa^_{v}2a^RSuMDPuHTLtxlBe!EIlHS*Cd|&2HO)R?xQUlzIw)8DSW>Zo#SlRP zaSi_)g4@95JHXxH8zQy|Q8m_8WmYT+2LyR4u54Mb$Kq{PZWOFp(E=Us%)9zRWQ_XdtQhr2)p z_1{o1*vW$RkSSxjsVVQk8c!0E?cN9MJk=(;wD_x_L(MH4`AW0*jW)q-upbanm!H@3 z`0F>g@a0IY^>KG@Cw0zq1MxZC+F_DmlAiNsc|ppC0fBKGmasSqA9F7nNLb!;X(PG` z&pfzLhMcl6AMbhA%Up$YucQl&GIR8exJA8OuDLNy`_tkLT2M`j;ns=Ppp2jf~ zRL*Z|(H4T!VDcI$p7)`uU2_A0nGA@m34;+0vC629Psro`0M1E=xQQ|19oY!ITCadt zHUft``Zy_(GIOWI>)*-TrYgkCvGL?;qwoHO-gof59%og@!L?#BnE;X)z5o5%Sou zIg9nm8OkVn%<_TbxcmF&aDkbtjRcfSDmRg&qW4@F=Ub|;wo+c}^;e2elq{&or#Wof zv%$kE_FZiPQ*b_%!ch6T2m?8jc#sJ8CFo+2bn|d$+&Ol|+n@N1IRB=@DjPJX;OLR1 z%gTy_orI=8l}V&z{IMLeeR4-A=bctK?ISN0YD6qSJE>Z2s^m)Cdz}o;?uy=mnAhaz z9(aeH&JQ@)F<{7iSHPg!^WJ0Z`qGM&BJJlrN9&?c2a51sj$>Rrap`_ZiVXtB`%Zse z^1`ES6&2w_8%W!li$ScNqYpS=su^fodh4q=;)JbM=6kWvig+|RD`paZs8^jYAsc8+=|P`VefW?etvv0w{<4*(9#XMGaN}N)KFjVQK!wi2p%aM z!4jlq7ibF(nHlIz7QQ^cyg2K9$gp$HXqlv&bbdS9@LDQ15+#}Ue)uO2gw;}&GExet z@*%$tKt_<5jdU_QS6A0|maVO=MO~JW->SrCaPWo#MBEA@%(ZZo9ddkP9OBvYc-htY zgwNytb!qU#)sN0m8Gg(Xe7i`kB@c@7(8{l%`F-Sr*`}_HDD%NPra-P*Qph z))xdOxVJyxigVYGm&beKSw~)r55#r`B}>lgtNU?RpOhjxf6Ptk>}|T#4VL*gc+SkQ zj;wF8oy7r$)t){hQH-kOlim+$aF;<=q);Bo5C@abyEQ0DU#)Nceyjw%Bv;#OWI0u5 zfR*@O-1|Z#Zze*(SYIjG?NmMrw^^RDK%Ro$ufF@%ysl({{MGR;fi&r;tq2cf&I)w|ygaq%+GfgqnTz=Gz8;AVy(s?m4= zuMGK!QsK@Uf}b*&7Szu$632}dK@rAlU&v2~jk>*+oh`kC_9M`f0g$W&PtVF+mwOk; zhlkcgGns+ZV$p!I-{f`PG1u?CecCp;N4a+`{0OM8$L|ME7yUc_j9Q$n-VdY-3niI# z+z-|Y0NyVHlnmo}PIQ#US=@8)0cgA;FSmXqaR9mIaaYDML9*$dY|%CeI1w;3xUY}j z{05co8zO%BX6AQ9pTOW);0bdQxD0wk+zOE9yq{<6@{<2U4|_IiHRNK)(FPXZVfY81w9 zJjw+z$dIx8NXR%Pm1BHr8Y2eyNAUjhe31d-VawNx(yV-B}7YkxSUxJnL4 zQ8Il$sg;wtu6^qUTF-HOof0EK(Ftp93{JEU-wO+3sA=_6j>_@?@xP?J<+n*q1MwVf z(+MIVKj$OBC&$yY_aA-30)IjThWuk=H6n$}*$?;^iSoikY z(a!a@KbrR~BKpYgvkYO6Cpr69VlCRMoi=Q53H2d7?S9yAJ2or}{k*Q=lH(*9>RrSB z6a`}OV>prrf$qX3E*p2Szos}d7R8_aR`Xu-KY zT2nIhxm>qNTY&w0X;MPlkCu^izwN$BlV+nnKGFXoks%&WUqMIxWd*&h0mC9RP_3yK z*Ef`~%VZIgDB(SG`EfGtF!9l^(v2<@v~BkCc1Cp?dai+6s<{B`ogoLi>~1FBc;fI+ z5c|U*{rf@<%^0I-1g~`bo;uEkr{A_3%#YAOw`N+H9V+F+=P;G?A(7e}_zS@?K1C`L zVjIf}Hq(6(PpLh4pB)B<6^nVvztlD<ep?R@)J_oHgBuu4=WVg$ZY|9Kxz4sNjH{xqkiiLg;qk%;d z+2>(84xP_6hMlev#X39Orvh2K*GUY8S1D}aagh3N16hx*IJNdye6l7 z+VP;dPbW_zI?SL6-sXqhn~V3-K?qYw5exh9*ugj^^nN-g*ZC(xQ8QR8trvq>n%i&q z>uiMuC5K3F8D)ZK#z%|YfiIH9Vykooax!&&ow2emyAO=ffqSRm*L5vxN{d$+%>xnL z50brwlBTa-Zv_-9=Q8!HzHRv=C?KK zx1dSec&jz8X{B2*-y#wE>IE_B#JmxZe~$o&YLKF5>?#un*_hwzbyRtL*qymqi|#0? zHGipP^|gcd3Ha}{N~?wxUc4wVC)XNqjT~a_&@Z&bIw*ZDsAt`S%G4FrN%;0vI7ti^ z+?-KQK(H`)2HuPZ>5z+RqNFUO^xq~i1*~D)-!7$XB#8*M%_VH226GlK zZ)_W@V1_^Wbuy4DcHDa+`_amQGsJHH;FvwxP*rKoex zaHFP5c<;8|<%@JC=SZVPJ&I@pPLFGd8BE%KuGprkQ1 ziCUxI`t(0zH43BDB1kkx4|Xx{jB4ojm0#p|f(Ig7WEnc5K~p3qrAl|^cxA$~#OHDc zw4!Al61ilZ4+t${x4%ToYjB3NkQxp;kXWDy7H45OI;nj08Pcy(P2YP=(a0bw-Ll&z zjN($x>(x=3;l_ArR&*&}i@s3>X~qrVaGAwM#))Pbl&Bd47J5f~jlc9230Ch3Po zi&#H2Ep&tMsh~-X`UI;+vY&9(B;tOodb{!H^Y$B<3T+F$L!AJ7?1NmL)|RgD+)RG6 zXhfTFTI$!VX@0bUhM`JD(kaOl<-omrA4xjb=cP9vb(0!$y>A>>?T4Zy#=?o08F}cG zmV2ndt+OTH(VRkIsxeEYjEW%OpKE-blOjX4#q!)Yic#%u11zz&xq zHk+5L=s$4_E7vY8<3m*F8Y>e!HDKeLa_ZgkG+_K&dBktfh&|~;Ix)!$tJm3X;}pN% zoWxVDDH1=K1nHB|#=^Yl9HB(O2 z&Y}ZPz4AfX;!$+3G2!P3CWnErKTSRSHQ5o1D}|W*BLB0a#Ok4GrW^e{SC%MLeZwAv z<{5TAx5{Mk&x!)g(CVA%TbhpG5PFkc!1AFr3MJzjT)(tYQe9&PRhNoe@PDF^=awo= z8mQk(X$QHMqx_J*I{o){dz5tx@)bK1hpfT0;Mv`=!)a-B+{s|Kh(-lO&w5iNm$d$M zLDPy7OzW10DuY&=;JSQfw@DNOO{fGSkiMv*EN#};fjT2BJerak zhL#N-ybg;554Gc3XG{KJcBFcW#tKKQ~?c!%z*gK(`=G)_g zAVcGVoedROSEqA{JYpFuF5#6A&JSwR?4-UrJ{kuig>b=Jc#*wXq~+F7!sTzVf|3kL zo0S`HpyDF1TIs-`O&4|j{ZqLxI`Byv0ux+GjpV*pgqd&g@+kb;37l5Y} zgOy>(Grzmtg$LsDTKv$Y18**dO>2|IyywpAQ4Vylt%oL&69@w9CzDb>I3ff8e85>~ zV+O0+5Q5BDNT!H>v7N+grO)e!@kyU2)!AvkRUT=~bdXW>5r0s$vJ>Wh(e+ zTq!o)U7}Gm;I}1o-}Gq3+$#mj%1fU~#tFM}B+2fKkrATJxraytO*W-`UKuXtDu~NY zlA}V$1sUT;tnu6uTAs9(2_rG`(&w;B=@aNlM=%yZ1w zxcx@> zsP{E&!$s@PzP8z#dLQMC_lUdSSemqUEZ8F7sGMO2N>=0<*5U{ite*~beX~g@Jr{r| zlppASomyp||MSbz(8y@*z?intMo3&jRa0EOnN^vBH-{8udiat#Mgr6EJ#uj;QT+nZ z9c*lMYoh66**}rXzpxHSq`swjjBSa(*b$@e&T=@s4ZutOC?*fb@~7}OE#e)cR$Gvc z4@b897Da3KKEutT5TK1~km_haaX}PbiwRc*uei8G&$0Ut-`4sbhD?$7ObFjaJ$X`}KIFGnFhZ>a zolm;&IMh-P_Ci!@lH`mX)HZ^uTU|0bpajncr7?SYv%S3#!MsS0E>tEfW5ZBqx73RV zwH%OSq1uf$?DV*j&4cJ;rERmNSKMcO@%`p~hCqn9BR||y{kmH^R{DHM-_ILMq`3*7 zSsN*qfKXy^E>jhu)L-1xyf-K#O zUxh@gKcszXVxqzUUZXq8__n1^k=nKthBUUx$*m!lzrZarrcF^F)3wsvc6vF3Ei)e( z7Y3R+!F{}Ec^RWr>PEMC*FTdgD;!zUAulAPFss7x0l&lBa6rSO*X5QbrT%>PL2DK{ zsx7H_$5(t0M-v+W74bB&C1vTm#wr0Gx&qiyMH_xmm!1p*;rQyc=9>`vfr@j>l7e5S zKE-*>3dL@}pzhy?zr~6uKYsMQQ8SZo0w_~-fyg_mqJ(^-L?w=XJ|K31lM^qVQ zx6)!~cZ>i0ST%qMRdJJ17vINyx8B}QyUavl%LMEB^K_bW!b3Is_CZ)L*wq)eNR1zJ zIkG}fF&h%{H45_xRrrP^dLy>6@N1sbT{nDpk?oW0z+Y{*p}l8(kI>`ob;3>GxpRl_ zqP8>Ga>uxQKO=HGmhpvV?L-&0hnJ0^SDU~+H&$Uqv5||U7*gO?Ly|*4nh5MSp-@6M zGOLUp+VIH#sTxrP!;zdb6Cfs)ZIGgFFrN{ZwoxeQ3K+I&=pzxUrbG@iDKh!|X8CeZ zZbf~8N`5W?0fz^&<~YdeQGp#!&i&+%OQS5o!1W)MB~M#&KgFH+Pl!;1 zi6Gja>In^4!`}%WKV$43AuEOHYAm!FA;N;}xCqg?H7-}fp(e-nYUhu&s7fbjz=ji$ z&)NfNvk_Ea*}0}9zh=cJeoesN&~w!I%-MuH3?S{sFo8Rrtcrnk;eA+doI0{(ce)T@8AQL#>eqS6clZN$#%=U)QP{ip3h zlSHpD`mlY-j>!j*?G&J%2~vc~4;kf>jm;4*j_Q5Ro=5bmSCIzg+(xV!QOO66vI83ACY6?)`X>abNZ(Y>`aMl?1JHDOaCTVweyrf@y*Z**!0IlK)N z@C*t?9xuytJTumyGOR7(UV5(yv<{cBO&k1Ym1U`g+e>N8=4cdh{?{Gq)ZXA1lhIV* z9%8?b|7mng9gwmmG2V$TAae;vYA)lR*0D4Y4#tL7NcohrP5%!;?@M_~Q$(RQ!hEO| zH~~b%|CjuZQ8Iz;E113S4)i~1{;3HXarlK#Sr!`{)t;ysTH1q_jv*PeVx9B+|D_gr z@GwpDEp&`IA!lml<(hOnC#_gp9IcopiMRuK{<~}JdT4pTDQ3c_1Bx(0^v>c-CU72o zL0#4`4@||w<#jLhpKEZrqX*Wp+KMDn7@3oEU-{$sXaqy}PF0e`&M4gwEs#W`q?p*% z7Z_^PFXEQ<&Dh3o(oH~~kzX+98TW=~x{seLaC-_J7C-&Ych_+kX#h#4+dTkW^($=_ zC;-9Jw3Iv}kiKXnLv{lr`*c+0ij#(qA<3+Psz8W{MgS$iULeas;1v`DJf0QraYcHGW<%C$9_^9V%5i2K&LF3(Com;%9e?b^g*4l z)!?CH5AR)XeT*P#g)y+oL2#Kb1fZrNpjY?Ur+180ZIvI@oTd4CsId_cBL{PNFducJ zO?BB08z?ZG#*puU0kS$&oUaT(&Oiew3$Dd~tO-iJ6Y#eZKy#G)KaD8TyKnmjwgmO# z`j0N)$huz-6;G}MKVy2PR_jYcJIB|kWRNOPxXX<~E))G<6^}^*R`eA7fC1I}&G(&C5nYV9 zuK&T~c~`0d4LK?Oo$$#2hY1S!2B?Y+KK`%jNw2;&H38^8Zc!y6j&5w|(JRbgW1-o9 z1jZpV9ou)(d#U}@o&f&ka6{EV^qvMr{TEj*k>5NgLN#MBGWU9Wyu^L(%LM)J5t_@2 zK#|WU{)dnpT`5pq4IH{H#RA!iz;JzwvVkDxGy~GV0ERbUX#Fxl{EJ#`@W!z%W@#N` zhGHXX3HZ+fK^gOL@q-jN0Z=1GiX1zi(=QWL&^0QNtp5*#aM<&yh3ZIN3Y%FFsxsKU z`&0W1%tsX4BR{ov;MEn&>DHLx7}oN!VJ98%VcVRk|GU?Lm8{^UMz!Yup8HF8+ZkE6 zmW%Hf@s3kB`mnRl)g!*(7~uWqNg?A;L6sa{WtzpMc4`_RTi{C42Tj%g*}+QqGGh4h zPxK4KkMb@O5eh26F>nj{H=Cx7!wvsL^I|Xx54{k_ zS|syK-hXq}C7$!u+!$Pkf%mSR9r7ilD3|T~zUc#RdOV{5!MM6>JyNn z4_cICwlUwN&Lcp8c-eU%ZJ*pTD1R2{NW}4mc3S$b@2rgs;S@bH_wjWxsgAI0PXFNsB`^IjoaA)b>^q^_NBf4UBF3Ggh zU-){h5E1{l^A9P4NuH8k>aMToDa!%H*lxEw5JHY098b z6EnygK$QALOQnu+Mup^*a$1R{dd|j{;fH2)^PmA?`OxFi*CU0co1_`bt9$h;k1%$_UR7TieHdFklhFLD=nfJg^#zGNNZi41AlfhW$bodc9jOy|zlR)g~8940me;e2?d&0?{*sBq( zKoljq_mNr-TWb&KB^qF%^6txyw+?{w{#A(#qw)e`$Zs`o7_!#n?9oa!??&$Q;wx)EZXfq1W4^zm%E_OyJE|e2m zQj^*4o#K>^P*B;cFJ@Ui!xm}-I+lU0_w9?Q2Vu(_iVJ$tmn5xc>4*O?5)k6|1t8|Z znmQw&fNd?Mq_V_ck0>Ecf{zp!gI!|5zy?=g-@ zd`8$yZmv|rGD$-*%*&a?2(%@jiM~9qN^BWyIvV(xwC^#bfb^HN{9tpJcFzCmFitw~ z8OrFQ;#V84FafYba@|l7zug2k(g;%~tDUpcv44~@45|v#cRgSS%<$%wB6dM3NG=v> zFyHhP;Ca4gmxaXT@ttTq%BU&5r7G+*DO*&ff0y zikR*P((})3-fR24pHv6yGU0s&=sb)hIpDO>(P_)u#xA}=k;0C@7uX%d*{RAXu9S4e z{WShNgi+ zq1XGez#uoiti(@OZ^<>c6?g8ItQ1uoPln;JWMk6N5~-1G(wxpsla#GF|BWsrCXh^)bX84D1BA z?3cuLqvuGVHv>u^O0@K+zvI;Q9~}+2#?Joe^J*=Df8k$Fe4h@pu@u#+=r%IqmOS-b zk%V7|ikd}W{!26Bzbeil&OK11Jql`CjBm?j362ui;!#9jhpc4k)2Vx6M3paYV)YnL zLOdf-m6u(qzz9?V?K`hLleEV6zaH0WYVdM-+FHQtx)ts|R9K>#B%npFmg|fg^5BC5H0h9lWi+wRiYw)d#(n; zd`~_+E#~B?G(OlGGwAptVek3oApPPk4nS7z%c=#E&43Rhk7;hYa!ML?ArjI4UyB|t zb>@s5w1-OUd%QltKI2gp1NNx5!0;E6HAbcDXV(p-chMzp3_Ja*6kCsUA{c3)?8nsY zSlqzC!!Qyv<85A#-z<;Mtcm|JJW9QgWx$c->e0|1iPFGKgjB}&mJ_#J`?bPE+bJ-* zbw~c+L>M*cp9}L(4$~Mgaiees zYMz-4!+`I5%pDo818AisCGS>9UnS&5(t635#?xZ^C5i-;n2$9x(9>qU%mHXD3d5+` zq^KHo%kik*WYI^x*Q(;|T~J1ZDIl!@^XKh#*7rS5Z`Lrd2Lmgkn`*-3R<7eS3s0?+ zgTn01!MZx{{#7cJQ=^0E6Z#ifcS!0Znk1vrqg62`vC;FQN4lx4gut!-)K4e!#8CC_ z0~-d;Q91@A0;sHsm7`Ic2LR=+YWCcFxpuu|b_K-vi(FrICc-C9#HLj^jKL}H!#VcI zXZo=SEr2fEO)7@wK+Kn!_6#0EUh4)%AVjS0x5@-J&i$YkU}Y%E94cQBn}o0{qkkvx z-5rB%OcDJIB{_W9pEXC7H_~fj%s|QBrXQ{y!@t3EV>j4oVr-E=(ya~1c- zATyY-47s50?;oEYi_HLUf&ut;836fC?(kYZ5b)9+Wt&qNAo+7%t2O&q7n ziy^VYUj4Hn_cOnfyu2Q08zMAfSB8|RySiC`<}dF&zAv5NnGlzPz%>Ua@L|xBV%lBf z4vN4MNY6mKl_!BDkWS0CqR2x(zC%53X5NE7bkijyp#W&mD5ZxmIj+=R<@EDKzwaTO zRvq=(E7k7)68)eAuJhEN<8Dk}o&@bK>zpL4`mKVnh1i(-eH5EIYx#5ba1kc2Xjq_xw@NG>ytgB#L7a_A?O*$uN-OFPKf zprIe1j}Jn`*K|Ict)xSdRs2U8AeqSCdJAW>lu=ami zyUwttvTm(|(ggwzO_UE91eK~t5eN#Q=>Rea2qaYL%^)=fu%Th7ii%3f&`|Y z>L)ymYkn%gaqxVW)w;gg9qM&+1=QXnVI=mj0GX$7Bl{39oQm;CBxr-U3d#jU;5DB? zNjBec*l8e;+cBWdmZTg^rT?D&H;R&ZpJI6;(X_<3 zw%PEt<}CizozqVtGIE-1GCw>+x-S`4sc9jWHs3nE<2Q1S1!`)!327(u0ji&I54xy6 zKsmji?}E0(}(9rCcH6tZoFjoF~a)C$cvrlZghcQ;RV1a1H2M~{ChRHX~e*RH(} z5VokM&itHaZQ@Gv&W}2!6_wqAkm7P@l@<|>F2$$Pj+-6w!Geo6Q;o0}`!QwbG{qvQS#l57-LB)I*-auqsYc``6Zi4Cd0*tIGABQ5I;|G>BhJDA=o6P=RoSxBmq=H+ zDO{UUz>2nC%6nBsG|MVt#^YN}sx|#idDW}&-6lr?>fH3?Y{8An7kTSD#3kpd_k+H; z@plw|Meg$VgITaBQ|GRVq|y^QK+V`WVAV1ib@UW006;3k1KMQ{S%>SE7YG#^rSF zk~=DXp-ZXK^&k&UW_Bf%DT5mL*&<>sf1fmG8sNsE55_G8S^qklRxg@rf2l(s|A-Sy zgnDC0WkH~ICaO?#3V)@v^4>~Kbipl1U@cZ_yU&x<@Jd9&f&e;?aCTI+na0xe*c}zl z=SUGz`ASpo9|ichbAFuKdX#{=VHfS@!IvMWkdv!LxJka|TODoW*8K*y-ipglGJ&qRaVpnnEpoa73%4+3`*@o`>MFsRwsKrujELgtTQMvPUUnmDxBaB+ut8 z{8HfBXwMnb?5FiT`h*z)*lHc>?+~#(h@iLo>00WyjBrwwH9$_CTo`(=eaCFc4*W5r zC94Mq592Zx7k}9K=UDI@JS!2^ z{2F=oP$q6diPAe;J+aPG{U(#sg91ds-Xu)aG({cFe@~Dpl6{?EpnNCLdwILYq! zBKOjUc~4xOsy%fior%(=0e38VKj`uINZi+vU}ypxs5c4j+?~CPrzTc&3b}YXF`J!V zGJR=D(-Aot+n$=4qJR_u&4!CLMEq4X(2X;n$)9LW@1s!lS*0-9MaS-aN%|{ZMNeI> z#rXu+mWKm>a}{)2B3LXtTwFQ(`7fd8(`}ebg`K%?g85o%v?m>LXxsVY$vuO9LC>4U zUISHjj~Tlgnnliz2!N86`v^VK9-3k^9HQ4-f(&~#+K48m-jizhC(6~o1a#qp;BM%v$SQa7JHW)uxrk?Tl3 z=HTUwtyljFBKaJA$Iwf2FV7tl!iDSUzCEXcQL z(07!SUfnaCczo{Xuuz5egj{4#2@SOg~e{r~{NqypCRuV}#s^kJk zDe$=KATg<&;tA-Kds$VSjbTjMxR{Q{_eXVC3PxxA)Lfgs#njVha)8Qz^DzX&o0Bul zcyI{aUqM51ak)v7yO3w19_~n^aadZ~g1Y#*FPQ))T9>x&k@qdf?pBGs@ioF=tM*zV zui`@Hsbn7sM22Y!>^N+ty97C0d``zG>5MhRAp1|=H4&peQ$eJ>aiWF>K5MuB(q96G zfMgXrd%vVCS5vBJ0(*F&brB=>YMT;p(~hg!PUjU37H)L9aVP_TEGF(^69Pt&V{66k z(>>g07l=>-qD|lR5{SDqr?$Q&LE+ZivK_eb{hurT&X|vQv^4cb|m_Goq_*1bZz6=-{5dDvhQ)$H}zvpLR z-&pBGh~^l_vOHf~U`$9Kl%v{aZL8%>sN?WBx}`lJ!Vhcmwcivec0(<>~Q95wNZZ6IS@81U>041WtB zpQ;K`Q_U|Df*VRyNwG3Gj%SN}MT=%kc~7h3_ey_2PQtMu5=#MG>5d-&5I=e6jg&ayu zaV^sB>dy@O23Fq%^JEX^Ai*xf-jX$)2QW7mvTA<65d7nB3KS8J1X^ximz@x}&p}pz z^UJpDE|p)TyFhSf7YroeYa&!zBydyEW81%aQ;sW4#Kfzi#h8Vun3~Kl5%-mLNI;dG z2)Pke70&TEfV)`+<&nNe8qX9FI1 z*Gh}vkgYIOa*79tKu{SdCI{2(hZ#^pl}V7{wn}fws+8=XB%6;Fi zCf8q^B@9Ri$d<@W0*S|Le7Wsi9gaILP)t@Sgjx6rGsg4$nxTEJGvRw^BnKioXnHjx zT%YU*82OXF!;p7Ugf?Z{eM?`YTqRjI$gJ9kBjoE_66@=yqS=_ydUf<>_mwY_ik4kG z4_S+I17Rp;gOB>jF@RH(-}+;=Q-H86z-#rLs>=rn>SFhda%&M|SmilA3qI4s@yKDX z8LXB^P{#*Ojz7|<-*_@vh!)b+Ye_95vU;@U>Q^=D=oX2R^tKhs-GZ3K_Y0J-;$XlA z&tkjU5l&{4_aOU*DY3IRIwedwrfvNi*yP#Y?*o87QJgP}t!bDpxlB8+pj|e0zSL}O zYtA5fW7-w~<{#eh-k46XrvtpSJ|{1F4;8Oj9%dOHz3e56txlktjgzT)f&j}>uX^6t z548OxT%)=RF!kIi+FB`HbF1;&s>!t7G7fA{31#XM={u2NpNrh}yohdv0`(b{qupPh#OBzVVi{@k%qVU&A{Pl(MSnydI@BP`2lSGh-u~9T{ASCkuCLxX zCBIuknowkcaTLbvy_nlbQ3i3F&3IX*O~6M+2#j6j7U*nn6mBN`T9#=BEP(4e6X^rC zW3v0LWAII*SW4JkO&Y|#gD^WR88X8ZQB%+c!8f|6Ox9UEwKLu2sAB4FcJEjae z_6Rsv@V}eCfD;&Ro;da&&HwBNp8i)qF#12}|Nq$hpJA@|9`Yn;rNYrryHemF$zztL L){qKg_b2}byG;fC literal 0 HcmV?d00001 diff --git a/docs/gitbook/faq.md b/docs/gitbook/faq.md index 9bf2e75b..10a7a6cc 100644 --- a/docs/gitbook/faq.md +++ b/docs/gitbook/faq.md @@ -105,9 +105,9 @@ If the analysis result is positive, Flagger will promote the canary (green versi **When can I use traffic mirroring?** Traffic Mirroring is a pre-stage in a Canary (progressive traffic shifting) or -Blue/Green deployment strategy. Traffic mirroring will copy each incoming -request, sending one request to the primary and one to the canary service. The -response from the primary is sent back to the user. The response from the canary +Blue/Green deployment strategy. Traffic mirroring will copy each incoming +request, sending one request to the primary and one to the canary service. +The response from the primary is sent back to the user. The response from the canary is discarded. Metrics are collected on both requests so that the deployment will only proceed if the canary metrics are healthy. @@ -115,12 +115,11 @@ Mirroring is supported by Istio only. In Istio, mirrored requests have `-shadow` appended to the `Host` (HTTP) or `Authority` (HTTP/2) header; for example requests to `podinfo.test` that are -mirrored will be reported in telemetry with a destination host -`podinfo.test-shadow`. +mirrored will be reported in telemetry with a destination host `podinfo.test-shadow`. Mirroring must only be used for requests that are **idempotent** or capable of -being processed twice (once by the primary and once by the canary). Reads are -idempotent. Before using mirroring on requests that may be writes, you should +being processed twice (once by the primary and once by the canary). Reads are +idempotent. Before using mirroring on requests that may be writes, you should consider what will happen if a write is duplicated and handled by the primary and canary. diff --git a/docs/gitbook/usage/progressive-delivery.md b/docs/gitbook/usage/progressive-delivery.md index d2ae025f..9d0bfcdc 100644 --- a/docs/gitbook/usage/progressive-delivery.md +++ b/docs/gitbook/usage/progressive-delivery.md @@ -2,8 +2,14 @@ This guide shows you how to use Istio and Flagger to automate canary deployments. +![Flagger Canary Stages](https://raw.githubusercontent.com/weaveworks/flagger/master/docs/diagrams/flagger-canary-steps.png) + ### Bootstrap +Flagger takes a Kubernetes deployment and optionally a horizontal pod autoscaler (HPA), +then creates a series of objects (Kubernetes deployments, ClusterIP services, Istio destination rules and virtual services). +These objects expose the application inside the mesh and drive the canary analysis and promotion. + Create a test namespace with Istio sidecar injection enabled: ```bash @@ -57,6 +63,11 @@ spec: # Istio virtual service host names (optional) hosts: - app.example.com + # Istio traffic policy (optional) + trafficPolicy: + tls: + # use ISTIO_MUTUAL when mTLS is enabled + mode: DISABLE canaryAnalysis: # schedule interval (default 60s) interval: 1m @@ -164,6 +175,11 @@ Events: **Note** that if you apply new changes to the deployment during the canary analysis, Flagger will restart the analysis. +A canary deployment is triggered by changes in any of the following objects: +* Deployment PodSpec (container image, command, ports, env, resources, etc) +* ConfigMaps mounted as volumes or mapped to environment variables +* Secrets mounted as volumes or mapped to environment variables + You can monitor all canaries with: ```bash @@ -229,3 +245,88 @@ Events: Warning Synced 1m flagger Rolling back podinfo.test failed checks threshold reached 10 Warning Synced 1m flagger Canary failed! Scaling down podinfo.test ``` + +### Traffic mirroring + +![Flagger Canary Traffic Shadowing](https://raw.githubusercontent.com/weaveworks/flagger/master/docs/diagrams/flagger-canary-traffic-mirroring.png) + +For applications that perform read operations, Flagger can be configured to drive a canary release with traffic mirroring. +Istio traffic mirroring will copy each incoming request, sending one request to the primary and one to the canary service. +The response from the primary is sent back to the user and the response from the canary is discarded. +Metrics are collected on both requests so that the deployment will only proceed if the canary metrics are healthy. + +Note that mirroring should be used for requests that are **idempotent** or capable of being processed twice +(once by the primary and once by the canary). + +You can enable mirroring by replacing `stepWeight/maxWeight` with `iterations` and +by setting `canaryAnalysis.mirror` to `true`: + +```yaml +apiVersion: flagger.app/v1alpha3 +kind: Canary +metadata: + name: podinfo + namespace: test +spec: + canaryAnalysis: + # schedule interval + interval: 1m + # max number of failed metric checks before rollback + threshold: 5 + # total number of iterations + iterations: 10 + # enable traffic shadowing + mirror: true + metrics: + - name: request-success-rate + threshold: 99 + interval: 1m + - name: request-duration + threshold: 500 + interval: 1m + webhooks: + - name: acceptance-test + type: pre-rollout + url: http://flagger-loadtester.test/ + timeout: 30s + metadata: + type: bash + cmd: "curl -sd 'test' http://podinfo-canary:9898/token | grep token" + - name: load-test + url: http://flagger-loadtester.test/ + timeout: 5s + metadata: + cmd: "hey -z 1m -q 10 -c 2 http://podinfo.test:9898/" +``` + +With the above configuration, Flagger will run a canary release with the following steps: +* detect new revision (deployment spec, secrets or configmaps changes) +* scale from zero the canary deployment +* wait for the HPA to set the canary minimum replicas +* check canary pods health +* run the acceptance tests +* abort the canary release if tests fail +* start the load tests +* mirror traffic from primary to canary +* check request success rate and request duration every minute +* abort the canary release if the metrics check failure threshold is reached +* stop traffic mirroring after the number of iterations is reached +* route live traffic to the canary pods +* promote the canary (update the primary secrets, configmaps and deployment spec) +* wait for the primary deployment rollout to finish +* wait for the HPA to set the primary minimum replicas +* check primary pods health +* switch live traffic back to primary +* scale to zero the canary +* send notification with the canary analysis result + +The above procedure can be extended with [custom metrics](https://docs.flagger.app/how-it-works#custom-metrics) checks, +[webhooks](https://docs.flagger.app/how-it-works#webhooks), +[manual promotion](https://docs.flagger.app/how-it-works#manual-gating) approval and +[Slack or MS Teams](https://docs.flagger.app/usage/alerting) notifications. + + + + + + diff --git a/pkg/controller/scheduler.go b/pkg/controller/scheduler.go index fa00b39c..7d9b909c 100644 --- a/pkg/controller/scheduler.go +++ b/pkg/controller/scheduler.go @@ -380,7 +380,7 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh c.recordEventWarningf(cd, "%v", err) } c.logger.With("canary", fmt.Sprintf("%s.%s", name, namespace)). - Infof("Enabling mirroring for Blue/Green") + Infof("Start traffic mirroring") } if err := c.deployer.SetStatusIterations(cd, cd.Status.Iterations+1); err != nil { c.recordEventWarningf(cd, "%v", err) @@ -399,7 +399,11 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh // route all traffic to canary - max iterations reached if cd.Spec.CanaryAnalysis.Iterations == cd.Status.Iterations { if provider != "kubernetes" { - c.recordEventInfof(cd, "Routing all traffic to canary") + if cd.Spec.CanaryAnalysis.Mirror { + c.recordEventInfof(cd, "Stop traffic mirroring and route all traffic to canary") + } else { + c.recordEventInfof(cd, "Routing all traffic to canary") + } if err := meshRouter.SetRoutes(cd, 0, 100, false); err != nil { c.recordEventWarningf(cd, "%v", err) return