From 3996bcfa67137200021a288758b582ebafd039c4 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Sun, 23 Jun 2019 13:35:36 +0300 Subject: [PATCH 1/5] Add canary provider field to docs --- README.md | 4 ++++ artifacts/canaries/canary.yaml | 4 ++++ docs/gitbook/how-it-works.md | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/README.md b/README.md index 7dc11978..574a6811 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,10 @@ metadata: name: podinfo namespace: test spec: + # service mesh provider (optional) + # can be: kubernetes, istio, appmesh, smi, nginx, gloo, supergloo + # use the kubernetes provider for Blue/Green style deployments + provider: istio # deployment reference targetRef: apiVersion: apps/v1 diff --git a/artifacts/canaries/canary.yaml b/artifacts/canaries/canary.yaml index ed0c8b01..e113a862 100644 --- a/artifacts/canaries/canary.yaml +++ b/artifacts/canaries/canary.yaml @@ -4,6 +4,10 @@ metadata: name: podinfo namespace: test spec: + # service mesh provider (default istio) + # can be: kubernetes, istio, appmesh, smi, nginx, gloo, supergloo + # use the kubernetes provider for Blue/Green style deployments + provider: istio # deployment reference targetRef: apiVersion: apps/v1 diff --git a/docs/gitbook/how-it-works.md b/docs/gitbook/how-it-works.md index 430b812e..51f9a4f1 100644 --- a/docs/gitbook/how-it-works.md +++ b/docs/gitbook/how-it-works.md @@ -17,6 +17,10 @@ metadata: name: podinfo namespace: test spec: + # service mesh provider (optional) + # can be: kubernetes, istio, appmesh, smi, nginx, gloo, supergloo + # use the kubernetes provider for Blue/Green style deployments + provider: istio # deployment reference targetRef: apiVersion: apps/v1 From df24ba86d04a3cb15157efb407ec3267e3863e63 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Sun, 23 Jun 2019 13:40:12 +0300 Subject: [PATCH 2/5] Add Blue/Green tutorial to docs --- README.md | 1 + docs/diagrams/flagger-bluegreen-steps.png | Bin 0 -> 31511 bytes docs/gitbook/SUMMARY.md | 1 + docs/gitbook/usage/blue-green.md | 354 ++++++++++++++++++++++ 4 files changed, 356 insertions(+) create mode 100644 docs/diagrams/flagger-bluegreen-steps.png create mode 100644 docs/gitbook/usage/blue-green.md diff --git a/README.md b/README.md index 574a6811..330f18e8 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,7 @@ Flagger documentation can be found at [docs.flagger.app](https://docs.flagger.ap * [App Mesh canary deployments](https://docs.flagger.app/usage/appmesh-progressive-delivery) * [NGINX ingress controller canary deployments](https://docs.flagger.app/usage/nginx-progressive-delivery) * [Gloo ingress controller canary deployments](https://docs.flagger.app/usage/gloo-progressive-delivery) + * [Blue/Green deployments](https://docs.flagger.app/usage/blue-green) * [Monitoring](https://docs.flagger.app/usage/monitoring) * [Alerting](https://docs.flagger.app/usage/alerting) * Tutorials diff --git a/docs/diagrams/flagger-bluegreen-steps.png b/docs/diagrams/flagger-bluegreen-steps.png new file mode 100644 index 0000000000000000000000000000000000000000..d1d2c9384e01c61cb40e7036382805277ce5e3db GIT binary patch literal 31511 zcmc$Fc{r5e_xDH<5mIR=LWoO2aeJ?Vy z$As+f===Nr`M%fn{`H$b=9xM7ea`(kpL6bW?s=|f!qin2sVNyLArJ`l^Jh;rArLYU z0wFEEOiGk+FYDrcj3g>30Psk~1>udc>%u~`*`$rnv27kRt>&a=Dad-E~>5g9ybUd<9YTr8SQxn=P z%aI>F3|&0DUNw#W+vf~{utJ_cmDPGVwVt{bmnD8(p^=iSP3A7ckCSrsZyLxoj3J|8(JiaGuZmq)p7#TSb@;#b=yNb>I z2C9voU#{jX<<_9l>wOk~wGg|2bVl`ku1EyrcT&gfkB7!RLa3ZOl!9Sm8Y6#S>v#%k zBbnZ53Mt-!oW5PnkiX#q^L9XYgzrso%vtBDmi$u7e=~d)GN&;s8_zdF6%eNvjwxaK zzAkY0M!|@tmD4OlvKN5-hG&)tGNe-M2;RQ`;nv%(=|;m`;*~6w1k%dw{)i2h8aCiq znUE)Whf**w~)9clG!0O=hJt&s7KtP12U#=aDB!ujQX7W{`B+ zyar!YvL1v&e!s8iGxFd_Y4zWkFvh>tHbh@3;=o5^vDLz)feeMgmwx;aQq;*7Oc!My z=NWp+r_EvQ53FB$?!+v6jivaqwJM;L;%4#!ku}04DNY*1-iu|o%34X#*Z;(+DUbAr zqu;sSaVZvj{;fD4;P!+^4!re$H(!??dQT*Ju%JczFCk_a9MHxW0&63Plp>5aLR0EO z5Xj}VwS1Ro$|dg=k}%}F@?^tKDn0&&jm{+b;Qk8bNHXbLX!rN?DVEV+;Cp+dS70e( zj~SD;hbCqo+gmZW8}l_aE)JYX+^hFlUmOF5i;Ba~=1*vhFMJIU#oBQ%njlt3wa8K_ zHW^#prD-fhqJM79NO(OtEqJ#+2X(BIs4$eL~Dnd<7bf%kP8l8~lpCkdhiUriIK5 z?liw`Qd}q~c?B(JzWDFC*So*-O(9^QH%r->-UG zXK;fz*_C$SFD0l)#(FG&bWmkc-b!1m{>Nrn*Io!}ccVWAFS0 zjvA{bd7CWQzbyTQBnL~3Ei2$W0M7F(v03BuZJP=(7SSyYdwGKS^y!xiFnQslwsPS0AV^nP5O#X_aBM#ABu2U`jSfAUJpPxhQpfap~WQti&}ke>*uqqRoy zBUA^?`${L88^GA?#KwdjYWgDMp_BjI-&6AbD@;QlI_|v%BtyRj=If{}Mq%$FkI zJN#G95Y`Hn=Lt?@g_mpCTjP+j!uyI=6=8UwoI0ngXZw5*scURutsnQY)e#7yv$XbVsW>Cc&&noE8&zl@ z+mO0F5ulwL(yP^PrQk##NLX{A#f3YSW6Y^PLF`l^bJ;A8qk%&(`wHERXpHrmuc`IM_OYR#lH`n`0tPj{Pc2Kx&7 zeyvb_fLwIbW5ixIQsE?K$s7toWu>s7l@xF3t*l_H2-G@3`n5RD%J}4b@MKPq#|h9> z8-)cjBjVcXw~J$wJH0hdTU%>ttT}8q;=q0@yx5xp=jfER>xlQ@d+;JF(I!5HJI5Pn zP1@w+KYlk^nGYZFnPsz>+VAf_Te2rpS*FqaeGfIMZ16Q6O^)%k9AKWuwnk?)xugs_ zrYYaqZV;?;5v_Vv(ls(Q3ME5yRJ5xC#MK|sPWMIxB9Z-(r`k3H;TQjgN&>Z5uc zRKbw?a40;+82V`PD+yQSKy#(Z}2tJ_UB{Z!LHXDojtm_yz6+>H=;#%Ya!`8EEr%Da8Pd#AA}!4f&7iW+?jnCCVNSt(3?<0LxzJL7Uq z(!p87$Hu{XuPXL9?Y&OM!7$8(gOIZQNz2mCu4$;Wev+#g+Ssy4Uv#`9cPOG9{f@X5 z%EzGs^uVgd^WHL+`@wh5?AQ=s-K91+pFCMbeW3xQzaB-~4(1Nn+}a;eef2z>erEw> zK2@rNM)={q9$6dsVWKH47w)GBpakh#e<_~c1$%eL5p2NDr$l+++nCQYHv&XC6 z&IFIYTG{HKUZ2vM{)k+@S@}{B(QY*cL@#ArB;mtZh2=8GFVG=p@xyg5~0) z%&m><<4KoS@Rjd$A9GOB6xKrfaCO6%FspsftYe^IKb5nfuK()P_ml2@p^O6XXZ9@< z>+M=4>s5KH*9Nm!Kjwc)4og8M+2lzh%RVBXMM~ZjT1m@hArHQ_my+OXdkI|;)5kly(a(7~!};r(8NAc3h}Yw^=kKuhtr z6E$kVVEVw0qaSJiQGc+os$$!)sw~pC)YjGxFV-k3`9X`siw>98JFE zZPra`A2pdr9TY^1INZUmp@`Y4Rps`rDwMNTmlH?wv`O-R=B@b4!C&Oqnx1qFM##$% zrUVI6%r6W_twW@HTyH89jlDTHHo!w4Z!2Kr0~3N-`aN+ZfB#W`Y~9)hU%OI#d%Ib_ zAwS(mTKvU(pVs%D1W2eDAZ8fL=1;%v>+Qc}ruJ^uD(M?bVR4tE_3%TuJ2kP(#TEmy zhd!5yeud`CU^rZ@2i;lbeQV-~8*j-?*Hat`ViovMHwE1f0^BmZGXZx`=y`9Y?u$tA zzrnFrsxLeXF?;!hX_{cm3am`1cWHtXHXLc4ubDp++qvOWb@IU2ZIe?jgmCjO%z=~c zi6q3b475jP!sunL`o^Yr$D_MJ%F=V+GBG{2yGNu<+j%wXFkpi8dw4@Wj6LoC|Cpj4 zzSJ8b*ozYTdZ@2b2}YTT0vC%+J{$7zLd%yG)$R8`;g6#S!d0-+5{ z+77kq!WQh72wrgt?el)rp5ZQTAB15AqG;(c$H_bwTu4L3ij%u~aHJwM!o&^x zq#Y@r|AGz^_#oUNzKBA;(dXE5%P9G_d?sfKbzIm5YfQU`+V1-4SwQ`pEE~TJI>oPI zqeAgY5vYrQMU%SU1tL{E0-9b(^yVeh8)YgBck>5MDxTBqrup>BcH zlM;bYGqCy1(S*wiMzX%yvkXU_G+<=N0jRfTI(e}j=FV*_pd7kRA+ZG%*gs>E%YGBFBI&0(|N?eTtRn!;`#=gm!lwwJcEe5QcZ- zpg%!{`+Lu%tO{47Qkg}S**7`$vLt_=(^uHdHMslP_APIiBUrQ)a^&DP7O3%3eU56ksC z=Q9fVHGoBjGp1FzL3hXN6uG9K5l?nt8YcOlwQt=1YuVqsyFObHK-DM15(JGVa?=(u z;&_lyr5NX6$XeE)e?fx()={Wb_O+t~Ea?TXPY})1-{&2_H6#4KNU!42yU9zB1?AE; z{<*^&CLka?oPf|TPOz3xDKIl>KdZeC>ujuT8DF!$YXgD)az6||CkjuO2Z75c=~`cM zIJQi)4IW)P$_Q)6_sZ#5%CHSKAI!e=-NmQs@?R`X~?axo4 zzC1y8d$-IP+9wmDH4HPNyZmipG9+eQ`NWhdyXzl){eJe~gd_LB_8l1#^R6NX=E0(0(_T-5LKe_7s%%je;$)N* z|8O)=>(nY>l;3d3wb!L0!z?GDoSc}KAF-r@_ZO}GZb)2?>X}Rg$-KxYss7EM(gADC z7~W-04r1B{WHQYaGRe@%2YDW|VitmKi3^6aJ?C-~i!NgF4hC(Hc5hsfYp+mpVFE}W zlU@pgrW4cZ>!oY=8U}mkx-j3-Aso4tJ}v;7Tv3?D2GCF>DZTyzEg}e%1B0CA7ngeQ z+v8O}7kRy58MiVfdoUdW7ZL_#?#c>B&sq^KH^{9Qzol>i{~6z%zGX$2Fcyxc?Y|oh zgx)bv?7W>CN{yjfr}|?c6@#@*nv{@%x^$_XBk?R0uJlMNbr@wDC6+1D1Cf~a~}(V-)A zM0m1D%-+%2iJEfj@?Rts0dgA&t(khhEnlknCN)Uw9kKG-NGl;C`l&3svP1=4x}HDC zw0q9Y)UKJV@GFqW(_DDAzxnZKG&D}(=2C3tpm@yb_NonZhUzVGu%&OY;}vv=AhZE~ zv&=MV^5y+SZLUsM_>h-KW#j6V=*Luh1pmXduT){txjS4{PXyc`7#Tx7V&M(kW0w*) z(Uif%!k}?1_9|ki>%-VUXio{T_D&gw-L~CBzJ)UZl9m9EI0}`_h}EzNg7#SZT_C1; zYW-R)>&*i$F5m|!o`4jDQMChYR6nbg*wN(G4`Lm1<_VtS`eq6qTrX@qdSa&dzXb(C zZECk=h|cONg?`AX6{JtjluQs~Uitc=babEc+jmlcmI&GR&$t}*4sK=S19D&j(@Y~s z-ws5zgGO)opSE7a5X&#ghHCI#z(N!3z#fm2BW(&FeG_$`m?-p}K&Reyh-&A)YSQ-H zjKkwD5s38A7k!YokIIrmY#=IG+n*0%ZlXEYK4(YDzu9~GYvKDPRx(iL9~C;>q11bI zH^5h1^pihT!!31lwR;Mjj+ju-i9SnO7vC*+$@V0228#H%XO5N0Vs2RiLz;b`Q!dGr ztY4p@`s0cU4$NgLA^Hy+CW+=j9#pyj%?Z7|)8nsMfX6A{X+%iH@-Y$bE_@I%IL~HY zz>*GjRF)(-_Pe|ic!BLV z9~MU5^cuq|SN}XsYj5%?te^)1cE9pVkpJbi+dfTwlPgc52_Y1U4;>b%k_B z_AuRF%1;bx)zf0gDsT)jHse@dl|UHb_ZK3^h)vy~7XxoZ&T)a=zWMz3TIMQ%-LGnr zF@&AB4vo0c_`=8%5#4z55yO%d856#Fu5>t!d=-m!{I@70HMPfhPn^0bS5}Q8w|Az_ zmN1Xf8uBpw4!>r4$*usK!9Fc^l72+O{v?*yf$b4}dES1GvHd?5#{__QOW-771+|yH zHH>q6+#acJ)TqtxF_s+0^uXF8E?tb!OVeOf6O>ToO6J4570Q-yy z%sDpG*a+EKb+)nr_}DtT%lkUeOVyX6Y8pJ| z>mjH;^=H1HBT-n6xv&6n11!8yrjaGkPNhhvYt@6EcoCUaS;K&Gs6PmxcvtNo^J$ZP z(*!N7@n}o}CZ+0cKJ~Yzx{0hz+SbZ~qi8vw;)Bmr8~hWa6!B%Vq zv{4>O{b_=2a0#nn2T=VjpZIoz&GMyOPj7SVkW_r(9bvfK%on%=pH{&fS_Ee;RIRrdw=)n!NhhxE zEu3+q-~*#`{EL{*OwV9j@7*Xbptyo>_vEQvM7IMcCSy_{Ke7LAAo7tA3fn%UutqFD z`a;hg=)h-ZS(t&|XZ_SCRD16gHosfzlut^rFe-i_e`G0qh+-jAK)GDEza6|S&a1c) z9fGFg#WX#w7|BpJ;aY7eoTcJb+*j5P240aF$Lr|^ygYGS)~TNeLaH}Lw4cG{F80#kz=fWbD>C^LF}(3QjHWwr;wT%s`OCk23RSD zxvh7zX5o6s$84W7&D&;Vh2c9c#;@(Occ=ear-Fn)se7QpO*4)S)5H1u9R)>OK`mp? z8haXKfiY^7b-Ys^zYVwuj=XgPJx-6aF z>?mk_!fu7z6{rgzWM`R$J`lf_?9wnK-VH(q4F#&^cAkI!SaT}Ytb~b1A%RLaF+zX! zgRCd*@6oo@%OU{QVJ1SL^#gSS)ZEDM({2WTeDcvu#ILE8G2AsF;MWvfW8Rb|roF4)ylxec0&9wRVBdVM7U0fW*#&^K^Pe zVJ-O*MCGrs00Xi7g)wJ4k8CjzRJ5zgb1y_}UlbGWzZE(YQSmL;#LaRAWHcF3J~f?< zsL+O0A9vD!>w*IL7k%mwk4lvB$VMq?1Uv zz4=HOJeXJS?aOdu*PT=6NivCR*vkvtd#CPerKN?Rr{9!h|9HHS7!WT~-aLzMJ#vo* zns;_^;r7>L-NODhVVN1~B6IDW+6EBw%W1gv%<4F zuGM7+Tok?4k?jDt&n&*~*`P3KKKC>Lnf~B}pUyh*p1?n^@%Q{S76NjD$A!>3FXpKX z&pzlQvsu(il-jN=eoNI`w_bO%zo(7H&3kV@(emKs=kXk5sJm+?ya?Laj6}aM3`+=a zMs{CN^N`+-9_}8W!24P{J5#+`(($3Wd-E)}Sde;n->FOH+sd$88S;ZjWg}0ulk%eyFisI`w3ms0Zm5=kJ@Y6+ zPS^CUv9)V=OK)j%)G_jGF@I|`{wjgHY8d>n>IjYA8_u=??cNVQihd@$7S|4Dl+hR0 zY%cyi&Mvav6e1K7c?!)X4S%+cG~VAjPFhNmQfwISug z_ot>|gjriQW)P`Az|4jUk}2jtXhsYkjy<+%W}Bq)$R2KRMHGPe)Xmb>V9eR1qFFDe zJV5&l>e7GMltK5W$@3!%HcLz@?BJuVw0 zQHc+#yb+vs7T`ni5{d7eBQ#(lBYJ63L4(>9^J~sY(xTq=e2X`$)2f(B7_oYaAq6OX zFZBB&8bNQe4cSlW*I~gR>u<4I-b?F}Z^jy@C8A{4icvv~q5j^%VB##t0$-9~=k}+O zh<4kHGC;;@%e!$)iD!ciW+Oq@j%uLoS(BB+PmLUrs{XuD{QLz5!0q=>0oo@&e9fvy zBgAc*UCJX4H>rxijG>6#%J5PYaygY_VJYBhlHjoAgG9U&-&=5y>g{*`xNb?C*=0CQ z{!7;Ki7`8H)RygZP~7^(f1d@|>CQ~F<;fnVDpP=w-W5%tBjPOce=~MvtoiP5&m0q2 z5g}?FHR;XcF4|%v)Luk~m7(PWV09V0)~y+yaT2U~AnG1v5!eu)Cij3p<4B*CIbvhw zyQciQ@td(E$>6cclpt1Y&jHIjaO&|_VXOx)!Sf*X;W;~ue7thjqb|^%bQG}OclF2M;hw-)UZY-<^>u+sxoq6 zt+N;!1<8P3{`$k27;`sa_a_IO!lP`0&Q^j`r>9(c^z0iZq4T>K=%Zo)c~$O z)DE%3KQMBe!}NLJZMZHuHbu$)9Vm5&$9-f(n@SxBb^E*)_Tv{o*P0++2oR{EP4(X2 zRL?)CZ7~Zk=Cme;A*ze!eK?oP2IKZ!er{SN(6JU-{TB~)vH0bCl*R$<@t~%2KjxYf zmYA*nZ>|(sD^MCN?=gMSaBiu44%toK9)ky3>j82AZ7I~4J#Z*^%=RK=N;j?$UrKHR z4DO|&o_|E2DK09SVX3e(oKuUyB6*2>9C>y8-O<5ks3!I^yRI!wyr+jC1u=^nm@{9$ zVk0ZT-3}gG!IMmB6iVE26Vu(wM}D05ME2t>JcxPpv}$YfX0^#vYJ+?<`P=57%{Q@r zW&OA0$!*BiZ8o2tDh!J2GiS6nSuMT5zSUQ#av%=O;KK?TpJ3Q8m9V4)kJSs4=1s;# zWJ2j&I*RY4V1fpzE@n^CL8Wa{2^d6TQy0~=ebF_t*#ncC1t?Fu%=sl#g7OLV0iK?py*(BIN(gw-c zSYnkykr+-IzqG;2jXphIj#W&eY{(c?00wD{p+o#OvqwKU4FNMW&C2Smnmli|1KXy* zF5U-wbT=bnSnGM@|iAQ-`6@pPt56#_VQzOBQv*9BBh|gpt0aQe1QBYC% z`)c=Ci!z59yTOR|T=+E2B^c?Z>F1FIfUq~)$|7NtDg>0XbD!|_&XO${U+c^q_e9oW zN+-?~0b298Pd+cZiL79$QYf8IF5TRo90=Ll_ffQsE;>s(ps^N41>K@4vyQF`7;JE? z`$#No!t8=GrFMMO(!x%Xf9m;$ufp-tHe?oNzP&We=Y`FMU8H4CdOvmVduUoBS3N8O zWW8E@zvyI8cJ1Q#H%xBkx4E-`Z&xKt`(ZR>T!0PP0xOT&aCe*Rng;=*OIe1E8+owx zX&H9`Mk?8dJR!=j1K$O7B)KEx{>p;(OBZd(B$C*Ov9F50Uabn@d#TRHz5K)Na&v+{ zjdJbT5LJi2hw_;nht8ll!}qpfptMxn7cDstqgx;!1AwK(%YC(2PyP4**z!C(Sxv7<|sADtEp#?%Hw1RcoD!&TakY&Z+ z8oT^Qb}jVu-w1+0l?eUa>Zl78C#!|^{b^v~J8uEB3rM%N$GrI$(4I4yJ<2|1Q2okF z!uKf`RgcwW*PhZpxwT|gXK;-Nuo9}$`?-?^MY^9N+;An^KNvY|QcnV23Nj_k^xaPA z40|E({w;!M&6HqjWw1v3QH)RNM)m8LD-1!(A~e5V1S}8bf89Ms9lrYRmUk1o(r^D- zRsd-39{ZcxO=E!QrFzAE*X>6vRVLrdjM09m7swGRv?t+3q^Bfv`Mq%hU5R?x!Xq}t z^#!9Q3PFa#hUxe|1ItEUUqs&%3S7e>b*x*9p zZFt*V|G0%%fLRG9T+BcIF=;dM5Fz<=_*uwq<#s54B3bGe#5BS4{gM~y>=b*`{?xMK2{oH$V3X}R@&YviVwLdibI*27b9SEA{ zWcT(Cf$u1&P<768A!3d?B;`bC{yv;HQBl%K6D>}=5ovjWVT$#Mn4CZ`;uoGIpu*N} z)F!lC;vrf0u^mwgc;e1YE$HgGiS=r6=-R^=U2HCmd?$d&2 zeRi3d-0=%65A&B^H5>7bFPl2+tk9nw_0Dc?Y&hyxx4sVG*PBY_A*NL8oY2lav|!ml zOF?0i1ckz>vU80`Rlypp!eFd;7^QNwmVZ<{dE|SN+OuC5vi`Kf{&l@&v8vZn5b;p} z+;4s_+_j=zZQd2Tk0z52sx?@Sl5xt;cv0=9Ek3(m)rQ{E_0@0S_-E5{eL$87&jyba zdZ7$fJy&r^TpI|yNOlXH5xg7K)TPsFsp`nqNdVD*6PpK0vjSGQ)R zU_JivmCxzBImmsU_p=8E)Nt|!9|+1P&VqwF0q)Ca1}s=|THWMC?!nrkbh|qKvvTt9 z+~!a6z*M6lNW2CkIJg+kwGMbY9mZ@;b=I(=EM9dFD_hhD;B@~6ntkS8)f?r~-TL8L zTz5KOb@`EWDjTu>zz7eE6TcXqW&3h`u!c$ij>G?Gd7m@BUdc)Is=EHNct^413}y%) zhWh7|g8W6}k3$tkSL@~3Eetr=TYMS@vB3g?pn~7rw@(fse;-kAHo1 znIFFMen38boCi0pUEgx~@1x^Y+l{n4D(J(?o3#vL@h`_7VM#k8ni9m)LtVsP z+(_ecBMY^8?jSw=CeP=4&%+Vlz8QB8%n@{p%N>V&bBQX?C-4SipJ*ECtv55|k5uF# z;XfdvAm$kgA4FOK}1FFVfh!gPL#eIA+?$>#9D3<@va1yPDkw#?5g-(6k4 zT>9Fp^tv|PT!%GQq9gA4e`eS7SpJMi%F`eAl&6v~lIU{L)Y(JQ^iMGdOGf0Dbk zren*JYA#6+_*vdVc-;c(lX0snxj&XT#!DM{P1(iMu@3ywuT*Uoo(ZtqqR1tB_&qHI zfLmP=v!&0*tB@zwnWK+0dnch5Ar>KSpf|VdW7J7;o0fW>f6rAPuAHTu}*+N8Bhv7t`{;r^EC_3S%#|0&K zfaN0NVGJ)g_wd&4-=3O*erO@oLILu8SmJwu{n(Z(%R1u0)*k76h2h><7-M_OQ=Yuu zlih&E^^BEcS(BI7fP(7#Y$}AnveHRr5fZE2hNR`G!8>!NoiGc| zmyqPlssxd6VrVHF2n)xtIQ!70EBlHyws%mc4$P>GHCi=M}0$T#U`!-)U!*}L- z8efQ=KHDgDErQNH!UiYPWg-KjmX4HAtGgVzEz@)u)(Rqxgr1u84170Z54vVHG?%S?m!cD;`U9Bc>Y%QC$ z;ud(b%(C_sSSX?dwct=8S|es6*ND!C7%^i{?@vv4A|`(!Pfnx(?i*!8U_va0dU-Trq_Yhtblv* z$t1%P@1ZXb*sRLwcMep+#X)#ZaslIp)l0vW(b+SdTq=S1N5u6?pcsZcJb#S%hi3aj?3c8o<9gjS;(ZuBlT8C#QxU={swB_GpaZjU7Ox%m7a%rG3o=!B zSEC8vG!ukM_m^3t6tzX39!)|;1k?%EPK;v*RZ23|Tv5Op($Ulde?okU%oj3Ng(|TO z(FG=0z7TUPhd+Xjr8My|mTHsax4cRd>dTzn!+PStuQ;I<@>PeTZALyd*vao z%f-vvEfk0yO?_a!B{|H5cuY5QeyT=*2+((Wdg!O=k0h*IkG|%$R~?>)Cusl$BBJih za#-}iQ?MPI7F|oe3;RTKlkXlYNQk@hlIP>Zy;KMXq^cnghb7A1(d53=JAgcr-T?nOMRzl@OKe`7(G zrfuCLhqBavh*5tsH?N;tknO)hy6piqN zpr`*tliL-T?_%v0@MrglUc`CtH5zpbem4vMy7js$%n})6IM9Q{2mOpR*%u;LLHe{# zKJ9}%&>xm)8_)V?iuS%d)4huX!V568a3Z7_;M9$S=|U?nkGM0r4R>b&{QXr%O)@vF zaScpdv3REfXvRTY<*&g{^UkZWo8$eeNWw8x8|L?%RE*H+n4NLq?C?GqkN_*)I`!cS zmg_h___R5c&$|gHTF+c6Sg*SPzq57fN=1YYi54=PQdwfcOGE3zd~uZhbl@Ek!Nptm zR0t5}r3lBtnNzKGT%mY>#$79yoU3nBh?XAOIP4|)TRv9ux`OIj5XXnKTHoeIh@R5e zR=&tIF;)RGx00J{(xyXN0 z(a+#HwC6ZnD>0xs54w5w;ZAX<@0;T@dsO#_+i6TOfGIKIk3ZozAkoRtZu46vMb5Xk zboZ<$R~nK&>J=D%yN=Da(@<&#d*O9#d7LQj$10eQB!5Wog%M5`8EuE7N zfgokstx@C<88+fJfk2*AogW7NtM~Ifiy^FW{}c+&8UO!%RcqZiqlQ4fSox?9LLhH! zcbZsjAdnw@ND1f5j1ZY}w~!1H$VFa+n*jn67_&rmgBrqWBPJ(L3b|}Bpgu%YNdzwb zM|Q6NfBEwEBXK^X1=)Y!XaR`KAn}dXI5w8Dg9`Gt(E2j;eE8oR@@(R9-F~7jlpkcX9FZn5mOiFuAw?=@VVU@{S|5xlgrm2knaz}=%5hDqYxP@;vXng0Xc8t zw_5-pdM*S(7eS&FgJJ!DTwJaK|1?P0h=0Y*!%(S`K_C%GXbROo>HG7G92-&pf8}Kv zYjQntsW6YHy(iv}9RkU?!f$^GA1g@sYXX7D1V@Lxw#8`MvPobU0(`VT=Gl6lh~X^1 zcwd0@vkmA-S_#Y#vQg|DFDpJn=JU$Rjh^alFG1Sg&^EITQ&LYacAk9O(NTYM>?eAA zyfRaUy~trV%q6!f5Q~ zA))0rLlbk#fIs5leZd9y=jnU{G`eB>EHt6+Y0D~IJUAQ;9`v^(AoLjjiXWe}%!N~E zI52v>Za>{2EJ&>}{A{tV*UU-iSY~@87`dcvY4^@{bV=ku+nzC37Ho04ayIKuku@=! zRBC$tG?l3Sc+-FAXFNltebZ74f6<<~kZxp+>;3&Z4~yV;Dh0L=mbWL@UtaB2HYk>} zV9v?YMgftC`HMsw`W;8j^WCqj9wwIaOwD*}@0M8F`folq@qY^UJ9YHy%VQq4!-%P8 zX`^=UmfjI#`0o5$0CWF6sQZg%h=bEBfOyYF^jA{y83pWv`#eiw#G<&RtY4pw)tHG8>qbU=H;wZ)$=bK>BuZf>v9@sHfbUjK*u3midU38rV~1=6)f%2Lsf(#J-g z<8MrD%&>=8yoCEJ^q(bn3Kb#GtpFJf>7(eDSdM*T|H=>oce&91Nk!j?xSU^YBA$QY zMcTMZ?)_V`M}2}SLs0L7o%)Lb7=gF?8tx*V46buyl(e z8?$YxsJ*(3iLhiHyxvKN&w|T1KH%W9wNY+?b%C0*5M}h=6ltlY+#P8HE2k&!8TFGY zEwA~$>?`suDA#yOFOtD~SXmn1PamkDg+FB7egNdMN-am>GPIGN%lAr}GF;Q}4Jlg-$2@))?Rg-0LBcfMX$%bnO* zJ}B3|p=Q!4Jc&H*iFiv}v8G|C;uk(V@$IZpCy>5dB%rSEb1vXkH+j8<4E*dRl6z*< zEndsGk+?^`&r&i0)orXh#p#IyC+{uCJCTY)0^XF7)b(4>GLglPa~1ps7}8Pae-0(l zMdcRlk1w#$mr=$96ef4+FkGQh+|S!KOGd5tsS%~WN8KsD zj`z*w3|ww5`dlf4khmeY;TBR+eA?52by7bH6UHIi8=2N%UcS{gy{YZ+f-Y z?WJf$2F3XaeQyGun>!mnYI%hkm9dNb3u8>TI0@J@jZgP^g|;jKX4HqB`l7zs{ndK9 zJ5yA?DeUWBG{Ijzcc)IUKv1@PX2x60qX!d%<_c3=-1dQwspUUZ8%Hu!7{m)#-7aIN zPn3LobH!xUFxjZ{^mWzn=SLRV+%I_d+whhAhD@a)v+(DBj8V9}X91gum#MjjYe`%R zYLUh4h`mbnckYtduOdnn!=h=dq*XlLetFK`izU}*FyVh;OzEzNuR1VR(ged#S5wHo z8w@z?u9_TLMs*^vCdSWtv_T zQR!(oxmAcN_WoZq_V${dvL>7Au-Vl_ksyhb|2_+_0``^0d|-))bWa8MIuO4fTMJ`x z9+Xjx{)m9X9gG!t(+=@9%>=wJLVqotW0CNC4im8p_%Bqe zs!l+YtQEv-4O1MeKAAU)CL#5InL{)bPtHWr9zQq!uu!$E3bG%=x0DAap!OQ{v~&$x z7QEGSf4njrJuHwy=~&Fj2frY+yqef|3xt6dPdLsA0J?BgmtQy{MBTcqkt;<%CXy34 zLBme;YzGdtTvY&XXv1b{c>xuVsrUCfmK(AKfj%6SwRV>k6-9BWS+PKKQUBY`4T6sSd5l*_FgqRxzKbk+DPuF&)nWdv!qe7GVPJYnDx3^ zuRl)rtd(!?(&^VME~l;*FYvc+-lt=^7t*q7H#lv z)Goa5?vL;v_7m=0*0_%tL+I50cz5slSNKZmRk+Pm@0QWTp0>6X&Z4fQtB~u#4|xr( zr!v)Kh@Zv%_`Bo6@~jDRrOjrUh)$ht0Q0DgPMFkrZKXg zW}RPj&3;;GEFs<|@m>D$dhGb-DvcEwCeE%kb$8w|=&gUhtDE24?I&_pFew~q6;kds zUfz1d15ABOMBlNdBxJOq>~ny2$$>4}Yakcck&_Nx{$pkheMau{Lpq-G^&J%0FO$&F z&Ceo2QdPexaAtmmiJM5IV%9Th4|lBF5O5~H$j$-eJqhK%e= z)(~N`i!n0UNkaB4S+dTIF^I85B1`stjpyV0dOfe_KY0A$hq<3~?z!i@@Akgub3fSa z4Cx(^`#c9zW>2L}t-`YF5MdLA2S98Qi|#d2zr{@GEC%g|@S&*Pm`D4eh#5JOgCBvhyco_H30rfFHiJHPu1?++k-^9dhhrKGy`4o}ZvP85 zv~F}1=nx>~cN25M^vHn>&Ip6#ulW$g0TJj}s?#=8``iuLBGsv#7VsN8Gt^Fu0>)Tx;qmi=V12U4nP~hd6Hd&nX9DGPvK>Wm7b3aOVk*|E#Iycv z7KAuCyZ0_BIech}=o9Nr6?^knDu&%j4rBqy2?^h6I#s4v=(c)qL@+ZftEd z-X?91??uuT*qHXE^^DfJ$0qsqLB^Zcz<+eoyVtxrO~~?u^lcsBP4lbF&-GKYvVkm% z(Nu|Z670$A@*Y`+K8T zHA$KEz=Hb(`U)`q?-F)SKKBEX`Bw=&*#~x7{Xem_|ybPAnQh^ zo71vk?S&!+ijykxso~wM-(v+oQv_m6?`|x$I zd+fdJ#hk<7XOtf_4HxNtT{F1chJ?*PV?S|sdzrhQoV&7+AKQUndON1g_e9T_=!JAs zJ@Xw-)!y|Mv(A_;b@$Lm?#-Y+#^ZC=k%u;8#I(e)TaR7y?}l5_!=`J=FNRYkE~1{P z-hy0>j(r#Vx(c?LerVk<61+0Th+E*ebXn*4WzX?HpOWuK!}nAd2z)#j`oM2*22Bv? zUTAzAKz8J9D}7SXNLPq3TDzzoMnbov;L8N6$aJq$_}71v7SZz)7*(8ESKv2=2X}V5 zko=9^kL^FsqPE{$+TnJM7~rBhyapx?-<&HUop<6rzg+i+t~mJEg#Lku>1UN+r;KHU zt5ArKRt|sqfzZ>iq>I~^_0q4{YbkJ3TF7eQPnLZ~nOM!vDHz`mYhi+W1)H-SRb=Y* zEy0Jj1*o?jU>x^Ox(ZRo+2*5aRozHyugQwl3$sm)RoF^@=9`4S@3DscOScqnQQDwB zW5io7uuO-YG`9}2wlcKIR>vmvEkuwti7}QTE8#DTT2YWKR7VVZ7z8aGS=faBknh9# zC`dV)pZO4qk2{w&>QW7!uv14HI-ZOleIrY=LcrQIYY39iSgLIvW`yb(`pf-NkEY@3 z-xqYd$U9*0oZsD)!~B>z)W%$sLKW5#PHn#vq;GWeiJ!i{XWc7RpxkumQyTG}$YC9{ zq2OyQnjT7O2(h#YO@X{a2iK!`=9hJ=Z8%Bl*XLV?GxveGIr+182c}kM(v@9YzK@)G z5n>_H_U2BIs=dXD%bu{i>Ly?69^7fRWfY^h9zntKJZcbcc?{ONhyx=8=IvKsS;3&) zfkGFuFi%#-v~bS6-(c0xI9QX+$Yf1#1LRpn-6Pn58!rdiUc@`-7gG)C7as_3&z;po zcOTIpksg14z=z-`POfGDiA(VKC79YB7g* zS1cVdTYA%&t(Bb7+A8w%j1cQbduvzd{b!HMZ+LfST7)ZwY(fgbk&?8ZyZ5%Kg-;&5Y*Tva zXB`>k6p>r*zT&@hM?VXLhbbktb_BEg2@@>4b-PVf6Y=8?pcNDXne(T*EHi5< zb~Sdu5!|xro5KhL+Q!F2bG?FPnL!XOc+P<`zDqTZcr>eiL?!hp2B{i%jK-|)a|anW z6r>eR6?7d3-mBH#(9iI!;9Ysyb;cEndwm~rQ8_h+pm9zQmry@!DzU53jn(LY`0`5M zx{fZe_%-)rgh$24u%h!MTa#0fJff2OT+#nM|M73=r0ZOu`FV5-M;`z-h?Jpf;EDy>*he?3?J9_f_*Z=SQwBb11;10q{VHf6b*)1Qp3hQ+ zP5y!CQD?-u)2_uHLGs;d42zyj&9C#N90`t312-E$s~gPO%CI}DTk-e>!98IYXyVb# z&x|RFgg3(|30hkMY`T|m>Bt}_`QcN0Lk1dL)Eq;P`Y0xqm2`Wr=rz7zN&k)q`1l1q z2|M;$3&N2``{Dc-3+Ndy&@4(yYYY=-0Toqb+$(y8w|o!Hpl=0}Ke!-ImHOL6ISR0q zaKt61Ac3b1v(WONBt^!Ho_Iv#Tg2)=o;TPFx>Nu{a{=!zDh0)-#5b8Fe8S#*LCpa3 zYk-2D{zADQ$PH=RrZMTtd#TE9`c5Tc!luhTRXh9)xP7r~M|6rRUyylsQm6K(uE|2F zm|N#`ieIRNJkVx<(gB70>wk#gmX@^1-&p!vAX{i}Gq&BH4|47SbF=Q60SJFz@^z

P$Z|v?mk-dcp-66oWR0;oNd6&Mtjzh;L47_G=OR@eZI8SVI$37X` zf*cR6YtieR9`z@^)yZ)7Tw^_A>sXP>F#7{>`;BGoA=*Mei*oAdY@IVneTKb&Qa=|I zioY^N3}9>4Of@?=yA*8;OjM)Sd6^LchVr#KD*rhX;hw1jvsi|du4`i*Kp=F7EI9I& z5(GAQpEVT!t6-3X94XTK3laa8>w~@fJ&Xf3oe}?DuRKc+bQ+4h0VFxzB8or+Lg9b} z2_ew7b*9TkKZ}ofbU4BQ`GC=-DQL7MawA4A^&tUhqzc$&*uzJrFnY79fYc+eCP^El z>5ytgkJp=pOuxu)k$^Uvfn6q$3Or8{yWB2fv|WMnIPj5V2U*f3P_=}B2^tY5=*uJQ zppSd@3TqfY|J(ix6TRB_2ibHHN~@^2VK7CqKC}aATFqW2e|QIWH|d!9P{H^aevK_Z zOTR=X{q3+Ybj?6fv3>+OVpL8TM=wUd1vguPpX<0+se(=JtV#_nKy`Qk9>kJUE zzX7w^y{NAbk?!DT25|9kc~7o9%UvLWCLEd z>PFJ#Vp5=VPz@p!pT9qHh{@Jb_5{*sCc0{k&80t@v?D&lb{iQ}X%yeCSF8?+DjFhxQX+bvc|JbT0kcl%#9xb1-)N3g-?aeUGK*L%#l`lwhZYW#WpY4q>qEHvDzuYAl}pfsre?vF#J z^#J`nkMWm^vfGN!Bf3Gf{ckS5%)9#({3s$!q%}TXGM4(!KcRx9qU>MIT0ujc57+UQ z^;xe;z(LL&x#Vy}{Xj}3nxvLMTJuo=>m1GyVH}UZa zFAGB+!$++NVA**U=%2yspPBCIEb`9{2x~-ml-$rr99U?$+T_*L>JX9;QO6M~EnHtb z^BFRy#oh2XxHWX|!xSNK4skkiD7J2USEQR{=ys*h)NC6VZgC49ae8Zt-BiQ7=8iqP z?r1q|WTephw{UUVLuO`(n>p#re&XRp?n!bAjpFj3x#`T(l946f=l&z_Ey5nM(aJNn zzSKCcAG+0#>b+7lDZ|_*w(FkieKOV3s%BuLP@Q&25A+R?yV7@0#t3*q%eZy9_hQ?$ z&evr%pQ;*@jzCaSQl)=Ix>)5%>l5(W066GOGj6F>PT(RLbZPST?X&+$pQZf$V=X>& zt~6IRS0fP!g%GHAz4?ye44d2yPMO>;ONk`@(+P&AnZ|qn+(+dgVHISnqeR79clskb z5bwL_oEZ1+zQF zw8seIxf}IE>$ATk-#t6QwDqGDdY}p)<@77(nDTNWLc!nKz-z1^%Sd>Yih9V^^T@>T zkW<^J^?XC`?yF}x0D3zA(-V6uS0l^ghIVv#wo=OjiD?%Z+MqKaLnh#MPvQ&iUiVGt z(d4(_le#J3zdxXINw54_0(eiP`Z}@iSW&SJ9JFz?tH^r92{gy$U0cJIBrDO2avdTi zwA~Ww5RHge@!jwC_lJ;=0NJ48-=WWM2sN6ceJpM9Mg|fC&y%)}`+Z(rTJ!29$N$4O zdfDk44rfcGe}LRj@tC#Lyk0+dazYT{Z?Lz1aOLpV!C&K_(8q^Ps21~)5sktM9p(c% zKAs<;e%Nj2^5WoWxmb7BclD)Rc1*=Ras5JmmOl?Z#p-fj3_P>Ovmvm*+W;4ZmpXh3 zi+OMoCv8ZeGAtx1JS)Hvkm??6lCoQj%~##8N3x5F1`Saxx<{sSFTH0aGD8H-Bo6Xee>PzVkT$^&4|KO7YyU!Y(Q3#?co1ELMKqmWzr6+B z!~LtU3Nw78%=H7Jl$e+NY@ZOQh_4D^FZkG;=y{@Y=~tU;$0u^w&Y7XRL#^yzZ5?eEoprB7g77JYxd z&PUooi?{qXzko{!PikK0QiMRf51%dlj!NekdzMMp_Szk00OwuT{bfA~LL-ylrSOAX z5i7w2IL#j2n>v|G(xTPv@p~1Yid`cmA7SD>7Iy!HEUMybwtfgcp0&!q>f7*o}-ZEI!Cb7o1@O7TP~6 zgNG%36yaGa4;k>zUpY(vu3Szu(m4dVW;V6npDXFP`QruB1m-YY@BW&>)I1mdw zQ|9mKVjhF%m}WuclDymA5+v5Kas7+?@n{y>$~Tjo-oDkId$i!%TOCjmF`t@ z(0MaCR&hogV-XpLs9YXKD&EOxbg5--o+v80`t-YQ3^=ZF(0o+;o^Q}Svx9VE(@ejW z(<)=DCewT5_MUm&UQs~dSA=H(luDG5HaxEtO5*REb8~g+5&zRpojLn?M&W?Yc2!0! zo6ecyaf=ZNZJTMVyw=~_CdrH&AtKUTz>xy!6rxA}Em3P}UVPVPhOTQw>?YhV^Coz; z8+q+Oa+HZuGa|yx4-HhD$k7Gu#OCnF^NGs!+{{8p=m@C_ITUvvGQUSUZ8kY-TLG3`-a^ zQ)ReeMCmIR{*>06$)yG)y8h(s>IRQMAH)Rt04wB%6Hm>|zmr=7+eL~g+12$JjSpyi zq&02)XF)r+K+Ob>3dplTPMCpChwx8oj%Y{xhRmwdGr<(8%}iw8K@8crr>m-HS`0KE zOnyowLIVG~=J$)w>f0RVdiuw}X{fTFVd-qn_k4{WQHkgSt5{^j4Y24_J7|Tq#8Yn4 zpd8L`;tDD_6Ae@Ph-~Vjhq0>Tzj7&SmKq526qxMY2Zp>}57G1Li5wc^hd_fyytT#d zeyE5!c$6#bZbJFQOR7A*oS#Ub|yliC%f8K3-q$|bS4{pLRH zB~7E43B>%#C*39d2?u~eH_zez3VvtGzk6q>Ojy{Bfvo$VxPl9-umSdTRUVR1f6;sI zl{!-sr}HO}8&O`ADx6AiAxY--`jxPb#!#e~SFM{nwhO?`q>O`kdgVM*2@M+gW$la2 z45S<^W&xslWVDtqo%zsUtWaF7VYA?hl)Ig+A&A;>fW~ZhGl@ZK!1*RZ77B%DKxyur z?+0?*@BVGl>qqXDDG08j78q)}A{(J&H<(-%nX(S7KoL9$Id~Ey!SX%Hu0##kcObSByL`312?_et_P)PAx4pI^q5HY-9!omG4#r9Gk3v^~t}s*k>P*|Dt3^i;VzXY*^wcI=sQ+`o<~v+3sAf8XCrG ziq~Z7uNmQ;g|)d|_*=&a1oE@Ta0Y8$dtv+P1Tvq)jEOcbXf^2NH5aRXc^mHZKRcuz z7pV~$zVu9WgRMr=fcP?=?&MQ_IX3U$$U%S35YJK~$Ll!8Km(dD%D5q_XFhY!L=a

;Q2Jk{Ycy9A9wEu=h1@?8|51mWzDKKF_U4I^@+NUg}dqeY^xo}@dNKMj5}5j zKRU#=>7o9gQAO{CXazN6?oz~R?}|q{B^IpO=mo=PRux78CdVpvZ|968l(|^4| zexIOaC0TLn!@kP#D{6PvS#Z0?zX6XEIW)<3BTo@ zHiq%-U;D1mq!08>qmhZ91=!o7NP)+ekpMgkej?{5>qzmhnd5;8)MC9*ra&`>-Q3R> zDzfGL_TjgSkyv=Qs~s&rpx9Xc0HvR&Qn}HLe6;BG7g7ReW{%gCT6+9J7u2f9o{CT8 zC}$KyBj3VBVNfq@0lZEZr?7v}KT13c;O!5BC~evCaNwq#svXp2%Q^O;SE}szwpflf zcVcPV6caEzUySo389FD}4d13}KM3sKfY)IE&QOrS$NC)HN_Y_Zd*j_3X6D~p>8M2R zz;yKN^L#3iH*P4o7oiOUX7{och@gx3bf0s=FG$2`+u-yR`OPvcF7efhO(g9WM3v}k%zu1ZMuNb*Yf2hPBu@101))O|TX=8ZQvY1?`Inxv?GG&`(9 z8~^(SuMyrsF^z;wu82d5<+#;G!f8w+8|@^W&6V43qmc>lKSiF=a3QU2(St$QgPr+Q z7Qj898Zjff8ZoBF^fUNe7RG#1u3u6vOsA=QnyQ0@qdG+O2r)JQ4yz>XI=S~%0UJ~ zf3f8?X>Y-67NQ#XWsLVN$<)GtpAPUj-+>>gCRZgcN2R%71Y?Xpnzw@<`Sw$ZZEc7q z2mspuUjtg)Svhslctpz@=T4X~7KSZzYe?AnbCFYr3t406KcLL2w`EA+i z0{^7D~=&=aN9mUfCz(a~e< z^IR240N(+Khu030h1P`+8vN^s*(_&3YaxrXrjrRATBpT81%!v<2af;zN?|xkRX){{ z9*~zhkR6iFRUwwOA_KN%SqnevLNaP-_fQ}Z_v6!V1hk3P7g2D|CiS!&V(^)qua)_g@0AaBe5@L1?9^ z-SSm5vII`LS7;b9_%cqz>mM=QdxPSEZ8q-zqyc3bzN_U@5H+6X zcuk@knk#YTu4V851G?rt?)mh{2n^`beG^kYaI^I$8X2R`!|ik4 z#(jRXX}^R9bb;@+hwG!6$}UO=Ep8-RcjGsdMfRZV?z5C; zH+MYi8xu-G529ctOR-`Ol)0K!Qvi2-{O^h$^N5Pt2eHGF&c>flpiLk9jvyXv`JT4r z!zC@VP-V>+KRpMa9M~}}*G`X0KaktrU;dW~gkhoQ$|$hs`J%_Bd8G{Net`Bn!Z(Uo zy<}wwEzmr-rz&CbBNQWy|t&?aN}lpPtS|Wz2#~e08ML~%t=R>8AX<$Ga-I`GY zw6`3z==Bg%+Dt(>>B!>OsW$3^>O!vT&MGnB^OI%Z;-jNtcrqg(A$2$US7mnl>jYsC zjxsxU1)$GpnmMAw4G>`QzI*NKchDUW#qQvfOx@AL-PLd3v@B0K1f1MEqa4*r^$+$` zjBC(S(CJUMj7fv#FDuo#(qGs@tM`kh{;T#K*ZOfV4epc@1eun{*8O<=xF^e=dfJRN za3NK(ublJ*;J4^Pq_WFopz#7SR#o)IeI~f9v$X$-6vv9{Fw;FLC!qy3xEjZp5V-da z41kHjRDFwAMbgFUa^7Ga6twbEDZtjTf`+1(I5=_4)nM2QQ(aUz#rx#NzYFI9rUz6u zag_tgLp=+7#@q}t@vnRJ>Xe)K1Xi&LzW37R!N}Od;(U_}DT=h>9 za`>hs-`fEc6(dN5;j&85tx8zNUK{yhjm%PIY5zx?z`=8H1t1H(3-reO;Hx&9}-4^|t zsri9i8*n_p3OX|S62E~SRCl`)5fPmJ-_w@h`9GfWulKLpy?|dAzioDA^ObGEpPTc8 z+8Q=KeBr(a%+ZIdXZzN42f*}zzg$}X>}{o9mMd+UWLcw6JbP9b*&M?+X4;4w@=4F% zUiL9i5jozZR4ZJ&=;|A2^$rX~n9O_szvbIZPD=PAQK2Xx(_c z;5#cDu^Yzf7vc)>G`wZ)1*n9NLF&>UKhD3(U$E~Kyi&#WCDETlc&xta&F%G~U&gsG z*I+k+Q}*lqwC91-J>6p2UB$@1;4*RWh0D@9TZ}s9CNA#W{X(M4Le~d+-84`6Xfcna zMY~kf^Nj=H65O`Z5W$;wFilzFPX@m|W_98Su1<5YAF#N58t($ZP0={;tmHo-ccrs1 z@#&AT>iie<>h`8e?!z4o&A(@d&;wY%p051@!9_@3>m{x<#8x|=zbbU>{* zJ|Ux|1t>R#DdaQMwiqJlY^9=EI3VZw44RjY85=WY1{^l3fl!_j847WtfAEMFZP1~Y zd;9mIv&(du%tI&3kI}XGN&Zu&H8TCZMmXZrYXVqEvRM$JeN*(@jr;Y_4HVcfArZz83!NAhlP!=9m1o%7gwYxE~Z9#?Emy#OdsVDJE(>#`%4| zkt68h%Fo~P-z2jfPT|gBIeYDDE8}aZtrFz3{+}G9Ik{M!I$~@9{gHP&Iq6~Q!UYs+ z-0GEUm(0th2*{b@k{ql)>n7UEixH4kwVLJXTmyq3L9}|@DrrjTqjz7iCW*1GItD`b z79#NP36}ObwrgP9s}R8iJxN$#M4FLDJqw;j@%`FB17T~Q$S7^v><_+b?7?QV6A+PZ z>``|CD79Lvuqm_To-Hu-pvTh$q?UWI@5)shoWNEfjpXmAuKA9L&9cKUSPC|Lm}FYB z?9gE~dw@Vpe06oWaQpe1J7{eHt{7);nJ>8r9WD(cgMtXP_atL~WqVXg4Ry)WpN?^X zsj_fPOCe!v4w^PV*O20Y&}pCbu{8I>E*~=OIpgpnf!2Z&b5LDEC>eCzL(PxA)pqFa zkh2MK0Jzcuj@C}Y{PKX=?U-Iab$8rr(hI*4pHZrThaosn?wQK~;`fWOxF2+iqMMt^ zkt}z>J*WuZ*X!VB6EG|8PHcK5jO!J`*ed_$PAhV4Kul3Q>=pRdK5<@)s|Oi^2i#?3 zSkxYfJ_|#j^PNW1p`jL)QjwDx!GqCIqLz-*YpYpN?f?qQjekfnDD5&Z%T+H;P>Z;}ayJt%nKxdq^auGF^N zH}7|0zdTa(y#Vxm$LSf1WjMhSlJkxnP%U_BL#el=tbvEQi?xfHoENr@1{hK}Z)E5o z4L7jjybt7PaqH*!JsEzgsV7jcUyQs06f+Az37go`>To3B2SxWn`*JzWMf2-lF=-T_ z7n9G*1ZpDR%|a)G_b|-qWnj9PfvN1?d&R&9OLG77wdO5<8_iY05%(}%Ec z0+dEC$GsmB4M;3MOb>r))EY=a z0yveC%b{OtOVX%*$aE5M;FRz@s{~CLqGG8?okm5MS1~<~0L+l8HJ!r?KIL6vKn- zFd5`)<>zcmsR@|XKdY}{IWU@n`#9OJbLuKF%k;bHdU57AqJS6sIXj^y)aimP3eg+h zwj##5aah%({8%PqLP+(ssdhbS|}ww z9ZN>>hz&HFO?+lgf`T07nYi;Rz(0UDgT>l-7+SQl+jZc~eCp9wYf!F@M6G=2(<941 z8i^FLrPS7o7{V1vWtQfkV&q}`x(-p^3<=dMF_s*tzz#DZA6C}3qay?}R$k%%ee=9` zce$s}Fbbd8hV|NK3Z8}hTkW15G=1eMPCg!X@Ww+gp1Guhhs>R^0^$`=L08(&c^KIa+^ufT1hexW#?>V!N z6tsH17{1%h4P!ZLKK%y*3sV63s|Za?y(7tCnO~3Rnp1usppjgnJ7z?!$~25?C|( zR8D&q>2A?CVaovj+Z<{!;%*ot5s_)vQSDT>(z2`i3HX*mnT}^W`fptvwX2=-TgmdM z{;Dy~vC9q+?FKx{w+Aq$J8O4CtWrrs)WHCD+v+DBUX<(?erIjcNQEQXb@Jv7?48fP zLD+FoH5z^VW_I@-KYS;9TpK(o*t31Yyqnf7^ivaN&_S5~5Ol4gwwjiWcUk}~MJu0h zudZqfshZtRxqktjudjZ2293NX`UTg>=738OOHaA_UTgkX_52scAnS!rq3jPuDWDXA zAk9&t#Kk*qY7jRqnp_J*Tj*g2H{xgx4kQqI^4CP7#mo(>!6_wr<>Q!bi%ROlfG>B? ziy@?q7{n9Fh8Q;R~ier7u~2AKSg@#l=A4 zhg!ZI!5HDd9n!TN$3hV@IaOxfRd7>JJSpf!J_0AqQ?9%AC29XJG)@s7<3MHl z67cNE8aN9ghm;}{MUht%r+K9*-KN%Z$zQp!oq^Uqq*tq-`WqV>T~`aoyj zHJx~gar&HX;J*aQtu;95Cx(FvEeBW9kcbQ9WugR3hhAoxvaqB{(8;zDH8^ z#=?Tvtw6t7g^Pd5juW@7aQngeU;~h?wNv%C# z`cgr05pse-d^dXw~1CQh$&2NI>#4yO(N+jN1AAW_bfUIJ2AuL7dY`{DF~c<5QJ@l zFIGNUxH;@YrS_E1DYU_K7;A@^W@|P}picjMD(Cx=Y%91rwV1eW%wCdP_jhDdGVl@t zRCvDPKGbqIJnfFrJ>9{ARsm88Timjs*{HbA$n8U>RLM4CB9>C6_o`I^OU_nFg|V(G z{ctLuyJKF&1e=Cn!8%C&J`^Xzb$`8hfL$!o9XTfAIL30)9-5Rt*}?KZ{2(EM^V={P z6Q*SQPGW}U-}4z_IQ*;oRbpJWU8HHlH&wcOT<%=2>U8{4lZK@fcu#BB1mpN|k)diH zZ=(zYBdWn`AsT=%ENAy~mU$(05;{OcynFeRxUp4tz3Sk3%JQO5xmf=p?;`OXT(>_D zSu=ec3F&&~tCF5yKZ^2QV85Y9w zc_{`y+{S@b)WJ~(=Twf}XNRI?YdM`~p!cJ4z;{5lJ9OBf%0hpbf%5B2QkR=KZzKXC ze$#>xcjNuKGJb68HRJ%6Q=q%hhV^d?-UCMULt<rQbwr!T*)^!Me;T<` zk0j@)u;YqObPaz;!Wn8-7sfF8@VrivWam+5i{Q$n0kF1)r`ruO5qbBE^nfMJ0J2h0 zRSICf_jbk1lWV|&>3+GoN=}Ovi8xw5J#9fNw+;%^UdSW-_HFZPQa`A`MJ{Qy5)H*> zV@R3wb#;71-_V@(Uw74>M?;BlrD3MM2l-z_>~5PAr9?<+7V->X&u`V@9^yK~ zW2;%dLeK;J(*nGwrd7a$K}v z-MEe0`kdD`gr(gi_7&XxQS7ZsKuo|@A++$zD!ortE23I{e?0!N`bjFukv=e86D4Ui zs{>bp&{|KG4alW)-TnONxoA2+blyPdy@vKz;-Z>#GWL~8?bT~3y38Dl-W`-`>}?HX zUx6Gsnj~0DEDnemA^x(eoqw9GCQ;{2u##haf{cLwsKcnyXnSJk;x4>IHrKmTD4)Zl zY{ox-0lx%fISTC9o0U`j(#dEp5UA<$CwcRZ*q5j-L|?%K;o?V-U_CJ@t}u}cJ}<2uuy%AC}J#4c3pYQnax^g>TU~um^UklfN6R0$SW# z_*s*`5wJLh{O!(%dx#(9-(5^+uG3b;%Nii)o5UOlouB*+x%`LwJi*Pc>amnLy6+?u zrj9tR_I?CHYty|9Yx%+Jw$k^d(&AZ%L6y}U3C#H&8k;y{FS&zluPeH#cisKNz!9K&&XPWmdkfp+E~aE5SQaZ<(>SsN z9)(Z}?C`QUgE?^OtGqHPASq9oerzHl9Jxr+9EAiw#$$B?VLqgefLZ0Y;JXp6PCpBQ zRD{1IO+Z_(5x#`XU&QEP4Xb6k`$n#bz}yjyB|Vo~fQDKC1D@Y&LDh>+PwPjQ_D5(J z_ske|k$&uv3=Yha?1G$9?^LNR)xsWoM-wkPASxzTbETNB(Te22`pGOZ-`2v!2 z;dat@k@&)zspY|n;EMo8 zeuN{F_^*oQ6r~GkAZ*JW4+Ioc9P2_+Z~DZ~-e$@8m!mr-DI4{bA}QEb#B>)cAMIHp ztZ%kCGqg}&ZJp)!dgN6Uaem+}g+C=!U1gU}(aF@ewy-{avB2_Mv5XTpvtSt93aoSA z)(nzYlpYpbP533v@#XQq-kqzf=2s(LiasdIzDg4)H{H)h%Ri?p`_@T8P}UBzc9pL9 z!EsqtR}A0Y-})X-+GGUS;9D5{I%|!vPSW5T6ArJZ-y96|>{-55y^ERmDt?rpmO#1i zLu#^0KKTzS&UuHr*OE;ZFZ(8^^F`LksERck$DC>WV?W?9HJblwP;5SAt_}8LD5#TZ zBW&p51n+=1gDe1@O$*D91{y;O{(OnKQM*2YBiM3_^Cd}tmKuSdL12=E zs8exg=-yiVh__8rE5P3aiEt{{d1r_7EP{<3$M(ghVXx|}jf`PM%t{D=Lk|(ya|LcZ e_<#EQ^(MH(;8ym(X$9ZY8Z}fND}R1u8~Q(;_o0md literal 0 HcmV?d00001 diff --git a/docs/gitbook/SUMMARY.md b/docs/gitbook/SUMMARY.md index 4a674a2d..f3ec961f 100644 --- a/docs/gitbook/SUMMARY.md +++ b/docs/gitbook/SUMMARY.md @@ -18,6 +18,7 @@ * [App Mesh Canary Deployments](usage/appmesh-progressive-delivery.md) * [NGINX Canary Deployments](usage/nginx-progressive-delivery.md) * [Gloo Canary Deployments](usage/gloo-progressive-delivery.md) +* [Blue/Green Deployments](usage/blue-green.md) * [Monitoring](usage/monitoring.md) * [Alerting](usage/alerting.md) diff --git a/docs/gitbook/usage/blue-green.md b/docs/gitbook/usage/blue-green.md new file mode 100644 index 00000000..899e6bbd --- /dev/null +++ b/docs/gitbook/usage/blue-green.md @@ -0,0 +1,354 @@ +# Blue/Green Deployments + +This guide shows you how to automate Blue/Green deployments with Flagger and Kubernetes. + +For applications that are not deployed on a service mesh, Flagger can orchestrate Blue/Green style deployments +with Kubernetes L4 networking. + +![Flagger Blue/Green Stages](https://raw.githubusercontent.com/weaveworks/flagger/master/docs/diagrams/flagger-bluegreen-steps.png) + +### Prerequisites + +Flagger requires a Kubernetes cluster **v1.11** or newer. + +Install Flagger and the Prometheus add-on: + +```bash +helm repo add flagger https://flagger.app + +helm upgrade -i flagger flagger/flagger \ +--namespace flagger \ +--set prometheus.install=true \ +--set meshProvider=kubernetes +``` + +If you already have a Prometheus instance running in your cluster, +you can point Flagger to the ClusterIP service with: + +```bash +helm upgrade -i flagger flagger/flagger \ +--namespace flagger \ +--set metricsServer=http://prometheus.monitoring:9090 +``` + +Optionally you can enable Slack notifications: + +```bash +helm upgrade -i flagger flagger/flagger \ +--reuse-values \ +--namespace flagger \ +--set slack.url=https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK \ +--set slack.channel=general \ +--set slack.user=flagger +``` + +### Bootstrap + +Flagger takes a Kubernetes deployment and optionally a horizontal pod autoscaler (HPA), +then creates a series of objects (Kubernetes deployment and ClusterIP services). +These objects expose the application inside the cluster and drive the canary analysis and Blue/Green promotion. + +Create a test namespace: + +```bash +kubectl create ns test +``` + +Create a deployment and a horizontal pod autoscaler: + +```bash +export REPO=https://raw.githubusercontent.com/weaveworks/flagger/master + +kubectl apply -f ${REPO}/artifacts/canary/deployment.yaml +kubectl apply -f ${REPO}/artifacts/canary/hpa.yaml +``` + +Deploy the load testing service to generate traffic during the analysis: + +```bash +kubectl -n test apply -f ${REPO}/artifacts/loadtester/deployment.yaml +kubectl -n test apply -f ${REPO}/artifacts/loadtester/service.yaml +``` + +Create a canary custom resource: + +```yaml +apiVersion: flagger.app/v1alpha3 +kind: Canary +metadata: + name: podinfo + namespace: test +spec: + # service mesh provider can be: kubernetes, istio, appmesh, nginx, gloo + # use the kubernetes provider for Blue/Green style deployments + provider: kubernetes + # deployment reference + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: podinfo + # the maximum time in seconds for the canary deployment + # to make progress before rollback (default 600s) + progressDeadlineSeconds: 60 + # HPA reference (optional) + autoscalerRef: + apiVersion: autoscaling/v2beta1 + kind: HorizontalPodAutoscaler + name: podinfo + service: + # container port + port: 9898 + canaryAnalysis: + # schedule interval (default 60s) + interval: 30s + # max number of failed checks before rollback + threshold: 2 + # number of checks to run before rollback + iterations: 10 + # Prometheus checks based on + # http_request_duration_seconds histogram + metrics: + - name: request-success-rate + # minimum req success rate (non 5xx responses) + # percentage (0-100) + threshold: 99 + interval: 1m + - name: request-duration + # maximum req duration P99 + # milliseconds + threshold: 500 + interval: 30s + # acceptance/load testing hooks + webhooks: + - name: smoke-test + type: pre-rollout + url: http://flagger-loadtester.test/ + timeout: 15s + metadata: + type: bash + cmd: "curl -sd 'anon' http://podinfo-canary.test:9898/token | grep token" + - name: load-test + url: http://flagger-loadtester.test/ + timeout: 5s + metadata: + type: cmd + cmd: "hey -z 1m -q 10 -c 2 http://podinfo-canary.test:9898/" +``` + +The above configuration will run an analysis for five minutes. + +Save the above resource as podinfo-canary.yaml and then apply it: + +```bash +kubectl apply -f ./podinfo-canary.yaml +``` + +After a couple of seconds Flagger will create the canary objects: + +```bash +# applied +deployment.apps/podinfo +horizontalpodautoscaler.autoscaling/podinfo +canary.flagger.app/podinfo + +# generated +deployment.apps/podinfo-primary +horizontalpodautoscaler.autoscaling/podinfo-primary +service/podinfo +service/podinfo-canary +service/podinfo-primary +``` + +Blue/Green scenario: +* on bootstrap, Flagger will create three ClusterIP services (`app-primary`,` app-canary`, `app`) and a shadow deployment named `app-primary` that represents the blue version +* when a new version is detected, Flagger would scale up the green version and run the conformance tests (the tests should target the `app-canary` ClusterIP service to reach the green version) +* if the conformance tests are passing, Flagger would start the load tests and validate them with custom Prometheus queries +* if the load test analysis is successful, Flagger will promote the new version to `app-primary` and scale down the green version + +### Automated Blue/Green promotion + +Trigger a deployment by updating the container image: + +```bash +kubectl -n test set image deployment/podinfo \ +podinfod=quay.io/stefanprodan/podinfo:1.4.1 +``` + +Flagger detects that the deployment revision changed and starts a new rollout: + +```text +kubectl -n test describe canary/podinfo + +Events: + +New revision detected podinfo.test +Waiting for podinfo.test rollout to finish: 0 of 1 updated replicas are available +Advance podinfo.test canary iteration 1/10 +Advance podinfo.test canary iteration 2/10 +Advance podinfo.test canary iteration 3/10 +Advance podinfo.test canary iteration 4/10 +Advance podinfo.test canary iteration 5/10 +Advance podinfo.test canary iteration 6/10 +Advance podinfo.test canary iteration 7/10 +Advance podinfo.test canary iteration 8/10 +Advance podinfo.test canary iteration 9/10 +Advance podinfo.test canary iteration 10/10 +Copying podinfo.test template spec to podinfo-primary.test +Waiting for podinfo-primary.test rollout to finish: 1 of 2 updated replicas are available +Promotion completed! Scaling down podinfo.test +``` + +**Note** that if you apply new changes to the deployment during the canary analysis, Flagger will restart the analysis. + +You can monitor all canaries with: + +```bash +watch kubectl get canaries --all-namespaces + +NAMESPACE NAME STATUS WEIGHT LASTTRANSITIONTIME +test podinfo Progressing 100 2019-06-16T14:05:07Z +prod frontend Succeeded 0 2019-06-15T16:15:07Z +prod backend Failed 0 2019-06-14T17:05:07Z +``` + +### Automated rollback + +During the analysis you can generate HTTP 500 errors and high latency to test Flagger's rollback. + +Exec into the load tester pod with: + +```bash +kubectl -n test exec -it flagger-loadtester-xx-xx sh +``` + +Generate HTTP 500 errors: + +```bash +watch curl http://podinfo-canary.test:9898/status/500 +``` + +Generate latency: + +```bash +watch curl http://podinfo-canary.test:9898/delay/1 +``` + +When the number of failed checks reaches the analysis threshold, +the green version is scaled to zero and the rollout is marked as failed. + +```text +kubectl -n test describe canary/podinfo + +Status: + Failed Checks: 2 + Phase: Failed +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal Synced 3m flagger New revision detected podinfo.test + Normal Synced 3m flagger Advance podinfo.test canary iteration 1/10 + Normal Synced 3m flagger Advance podinfo.test canary iteration 2/10 + Normal Synced 3m flagger Advance podinfo.test canary iteration 3/10 + Normal Synced 3m flagger Halt podinfo.test advancement success rate 69.17% < 99% + Normal Synced 2m flagger Halt podinfo.test advancement success rate 61.39% < 99% + Warning Synced 2m flagger Rolling back podinfo.test failed checks threshold reached 2 + Warning Synced 1m flagger Canary failed! Scaling down podinfo.test +``` + +### Custom metrics + +The analysis can be extended with Prometheus queries. The demo app is instrumented with Prometheus +so you can create a custom check that will use the HTTP request duration histogram to validate the canary (green version). + +Edit the canary analysis and add the following metric: + +```yaml + canaryAnalysis: + metrics: + - name: "404s percentage" + threshold: 5 + query: | + 100 - sum( + rate( + http_request_duration_seconds_count{ + kubernetes_namespace="test", + kubernetes_pod_name=~"podinfo-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)" + status!="404" + }[1m] + ) + ) + / + sum( + rate( + http_request_duration_seconds_count{ + kubernetes_namespace="test", + kubernetes_pod_name=~"podinfo-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)" + }[1m] + ) + ) * 100 +``` + +The above configuration validates the canary (green version) by checking if the HTTP 404 req/sec percentage is below 5 +percent of the total traffic. If the 404s rate reaches the 5% threshold, then the rollout is rolled back. + +Trigger a deployment by updating the container image: + +```bash +kubectl -n test set image deployment/podinfo \ +podinfod=quay.io/stefanprodan/podinfo:1.4.3 +``` + +Generate 404s: + +```bash +watch curl http://podinfo-canary.test:9898/status/400 +``` + +Watch Flagger logs: + +``` +kubectl -n flagger logs deployment/flagger -f | jq .msg + +New revision detected podinfo.test +Scaling up podinfo.test +Advance podinfo.test canary iteration 1/10 +Halt podinfo.test advancement 404s percentage 6.20 > 5 +Halt podinfo.test advancement 404s percentage 6.45 > 5 +Rolling back podinfo.test failed checks threshold reached 2 +Canary failed! Scaling down podinfo.test +``` + +If you have Slack configured, Flagger will send a notification with the reason why the canary failed. + +### Conformance Testing with Helm + +Flagger comes with a testing service that can run Helm tests when configured as a pre-rollout webhook. + +Deploy the Helm test runner in the `kube-system` namespace using the `tiller` service account: + +```bash +helm repo add flagger https://flagger.app + +helm upgrade -i flagger-helmtester flagger/loadtester \ +--namespace=kube-system \ +--set serviceAccountName=tiller +``` + +When deployed the Helm tester API will be available at `http://flagger-helmtester.kube-system/`. + +Add a helm test pre-rollout hook to your chart: + +```yaml + canaryAnalysis: + webhooks: + - name: "conformance testing" + type: pre-rollout + url: http://flagger-helmtester.kube-system/ + timeout: 3m + metadata: + type: "helm" + cmd: "test {{ .Release.Name }} --cleanup" +``` + +When the canary analysis starts, Flagger will call the pre-rollout webhooks. +If the helm test fails, Flagger will retry until the analysis threshold is reached and the canary is rolled back. From d076f0859e7c5ffd68a94f22309000de3409b71b Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Sun, 23 Jun 2019 13:48:39 +0300 Subject: [PATCH 3/5] Release v0.16.0 --- artifacts/flagger/deployment.yaml | 2 +- charts/flagger/Chart.yaml | 4 ++-- charts/flagger/values.yaml | 2 +- pkg/version/version.go | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/artifacts/flagger/deployment.yaml b/artifacts/flagger/deployment.yaml index b030920d..cf6183ed 100644 --- a/artifacts/flagger/deployment.yaml +++ b/artifacts/flagger/deployment.yaml @@ -22,7 +22,7 @@ spec: serviceAccountName: flagger containers: - name: flagger - image: weaveworks/flagger:0.15.0 + image: weaveworks/flagger:0.16.0 imagePullPolicy: IfNotPresent ports: - name: http diff --git a/charts/flagger/Chart.yaml b/charts/flagger/Chart.yaml index 890068ba..aecba758 100644 --- a/charts/flagger/Chart.yaml +++ b/charts/flagger/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 name: flagger -version: 0.15.0 -appVersion: 0.15.0 +version: 0.16.0 +appVersion: 0.16.0 kubeVersion: ">=1.11.0-0" engine: gotpl description: Flagger is a Kubernetes operator that automates the promotion of canary deployments using Istio, App Mesh or NGINX routing for traffic shifting and Prometheus metrics for canary analysis. diff --git a/charts/flagger/values.yaml b/charts/flagger/values.yaml index 9e34eaa5..8a55da93 100644 --- a/charts/flagger/values.yaml +++ b/charts/flagger/values.yaml @@ -2,7 +2,7 @@ image: repository: weaveworks/flagger - tag: 0.15.0 + tag: 0.16.0 pullPolicy: IfNotPresent metricsServer: "http://prometheus:9090" diff --git a/pkg/version/version.go b/pkg/version/version.go index 13ecc04a..c3168c4a 100644 --- a/pkg/version/version.go +++ b/pkg/version/version.go @@ -1,4 +1,4 @@ package version -var VERSION = "0.15.0" +var VERSION = "0.16.0" var REVISION = "unknown" From 191c3868abc419a525198d841b071e70ab75396b Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Sun, 23 Jun 2019 13:52:01 +0300 Subject: [PATCH 4/5] Update changelog for v0.16.0 --- CHANGELOG.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fbec31a4..f63b63f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,9 +2,9 @@ All notable changes to this project are documented in this file. -## vNext (TBA) +## 0.16.0 (2019-06-23) -Adds support for running blue/green deployments without a service mesh or ingress controller +Adds support for running [Blue/Green deployments](https://docs.flagger.app/usage/blue-green) without a service mesh or ingress controller #### Features @@ -16,6 +16,7 @@ Adds support for running blue/green deployments without a service mesh or ingres - Add [FAQ page](https://docs.flagger.app/faq) to docs website - Switch to go modules in CI [#218](https://github.com/weaveworks/flagger/pull/218) +- Update e2e testing to Kubernetes Kind 0.3.0 and Istio 1.2.0 #### Fixes From fb9709ae78a47e6d6155a9b01a18a19683f16541 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Sun, 23 Jun 2019 15:08:16 +0300 Subject: [PATCH 5/5] Add Blue/Green to FAQ --- docs/gitbook/faq.md | 65 +++++++++++++++++++++++++++----- docs/gitbook/usage/blue-green.md | 1 + 2 files changed, 57 insertions(+), 9 deletions(-) diff --git a/docs/gitbook/faq.md b/docs/gitbook/faq.md index 354da08b..edfc5332 100644 --- a/docs/gitbook/faq.md +++ b/docs/gitbook/faq.md @@ -1,8 +1,19 @@ # Frequently asked questions -### A/B Testing +### Deployment Strategies -When should I use A/B testing instead of progressive traffic shifting? +**Which deployment strategies are supported by Flagger?** + +Flagger can run automated application analysis, promotion and rollback for the following deployment strategies: +* Canary (progressive traffic shifting) +* A/B Testing (HTTP headers and cookies traffic routing) +* Blue/Green (traffic switch) + +For canary deployments you'll need a Layer 7 traffic management solution like a service mesh (Istio, App Mesh) or an ingress controller (NGINX, Gloo). +For A/B testing you'll need a Layer 7 traffic management solution that's capable of routing requests based on HTTP headers and cookies (Istio, NGINX). +For Blue/Green deployments no service mesh or ingress controller is required. + +**When should I use A/B testing instead of progressive traffic shifting?** For frontend applications that require session affinity you should use HTTP headers or cookies match conditions to ensure a set of users will stay on the same version for the whole duration of the canary analysis. @@ -53,9 +64,45 @@ curl -H 'X-Canary: insider' http://app.example.com curl -b 'canary=always' http://app.example.com ``` +**Can I use Flagger to manage applications that live outside of a service mesh?** + +For applications that are not deployed on a service mesh, Flagger can orchestrate Blue/Green style deployments +with Kubernetes L4 networking. + +Blue/Green example: + +```yaml +apiVersion: flagger.app/v1alpha3 +kind: Canary +spec: + provider: kubernetes + canaryAnalysis: + interval: 30s + threshold: 2 + iterations: 10 + metrics: + - name: request-success-rate + threshold: 99 + interval: 1m + - name: request-duration + threshold: 500 + interval: 30s + webhooks: + - name: load-test + url: http://flagger-loadtester.test/ + timeout: 5s + metadata: + type: cmd + cmd: "hey -z 1m -q 10 -c 2 http://podinfo-canary.test:9898/" +``` + +The above configuration will run an analysis for five minutes. +Flagger starts the load test for the canary service (green version) and checks the Prometheus metrics every 30 seconds. +If the analysis result is positive, Flagger will promote the canary (green version) to primary (blue version). + ### Kubernetes services -How is an application exposed inside the cluster? +**How is an application exposed inside the cluster?** Assuming the app name is podinfo you can define a canary like: @@ -139,7 +186,7 @@ canary analysis and can be used for conformance testing or load testing. ### Multiple ports -My application listens on multiple ports, how can I expose them inside the cluster? +**My application listens on multiple ports, how can I expose them inside the cluster?** If port discovery is enabled, Flagger scans the deployment spec and extracts the containers ports excluding the port specified in the canary service and Envoy sidecar ports. @@ -188,7 +235,7 @@ will point to the port specified in `spec.service.port`. ### Label selectors -What labels selectors are supported by Flagger? +**What labels selectors are supported by Flagger?** The target deployment must have a single label selector in the format `app: `: @@ -210,7 +257,7 @@ spec: Besides `app` Flagger supports `name` and `app.kubernetes.io/name` selectors. If you use a different convention you can specify your label with the `-selector-labels` flag. -Is pod affinity and anti affinity supported? +**Is pod affinity and anti affinity supported?** For pod affinity to work you need to use a different label than the `app`, `name` or `app.kubernetes.io/name`. @@ -245,7 +292,7 @@ spec: ### Istio Ingress Gateway -How can I expose multiple canaries on the same external domain? +**How can I expose multiple canaries on the same external domain?** Assuming you have two apps, one that servers the main website and one that serves the REST API. For each app you can define a canary object as: @@ -294,7 +341,7 @@ Note that host merging only works if the canaries are bounded to a ingress gatew ### Istio Mutual TLS -How can I enable mTLS for a canary? +**How can I enable mTLS for a canary?** When deploying Istio with global mTLS enabled, you have to set the TLS mode to `ISTIO_MUTUAL`: @@ -320,7 +367,7 @@ spec: mode: DISABLE ``` -If Flagger is outside of the mesh, how can it start the load test? +**If Flagger is outside of the mesh, how can it start the load test?** In order for Flagger to be able to call the load tester service from outside the mesh, you need to disable mTLS on port 80: diff --git a/docs/gitbook/usage/blue-green.md b/docs/gitbook/usage/blue-green.md index 899e6bbd..9eb12e11 100644 --- a/docs/gitbook/usage/blue-green.md +++ b/docs/gitbook/usage/blue-green.md @@ -98,6 +98,7 @@ spec: service: # container port port: 9898 + portDiscovery: true canaryAnalysis: # schedule interval (default 60s) interval: 30s