From 7993ae100ba3db2b7e7cf54583148548775f53df Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Sat, 29 Sep 2018 11:08:40 +0300 Subject: [PATCH] Refactor canary analysis - move CRD metrics and weight setting to canary analysis - add max weight to CRD spec --- artifacts/rollouts/podinfo.yaml | 34 ++++++++------ artifacts/steerer/crd.yaml | 43 +++++++++++------- artifacts/steerer/deployment.yaml | 2 +- artifacts/workloads/deployment-canary.yaml | 4 +- artifacts/workloads/deployment.yaml | 18 ++++---- artifacts/workloads/hpa-canary.yaml | 2 + artifacts/workloads/hpa.yaml | 19 ++++++++ artifacts/workloads/virtual-service.yaml | 12 ++--- chart/steerer/Chart.yaml | 2 +- chart/steerer/templates/crd.yaml | 42 ++++++++++------- chart/steerer/values.yaml | 2 +- docs/index.yaml | 8 ++-- docs/steerer-0.0.1.tgz | Bin 3084 -> 3127 bytes pkg/apis/rollout/v1beta1/types.go | 11 +++-- .../rollout/v1beta1/zz_generated.deepcopy.go | 27 +++++++++-- pkg/controller/controller.go | 7 ++- pkg/controller/deployment.go | 36 +++++++++------ pkg/version/version.go | 2 +- 18 files changed, 175 insertions(+), 96 deletions(-) create mode 100644 artifacts/workloads/hpa.yaml diff --git a/artifacts/rollouts/podinfo.yaml b/artifacts/rollouts/podinfo.yaml index a9f4e915..fd6473d2 100644 --- a/artifacts/rollouts/podinfo.yaml +++ b/artifacts/rollouts/podinfo.yaml @@ -2,6 +2,8 @@ # run tester: kubectl run -n test tester --image=quay.io/stefanprodan/podinfo:1.2.1 -- ./podinfo --port=9898 # generate latency: watch curl http://podinfo-canary:9898/delay/1 # generate errors: watch curl http://podinfo-canary:9898/status/500 +# run load test: kubectl run -n test -it --rm --restart=Never hey --image=stefanprodan/loadtest -- sh +# generate load: hey -z 2m -h2 -m POST -d '{test: 1}' -c 10 -q 5 http://podinfo:9898/api/echo apiVersion: apps.weave.works/v1beta1 kind: Rollout metadata: @@ -12,25 +14,29 @@ metadata: namespace: test spec: targetKind: Deployment + virtualService: + name: podinfo primary: name: podinfo host: podinfo canary: name: podinfo-canary host: podinfo-canary - virtualService: - name: podinfo + canaryAnalysis: + # max traffic percentage routed to canary + # percentage (0-100) + maxWeight: 100 # canary increment step # percentage (0-100) - weight: 10 - metrics: - - name: istio_requests_total - # minimum req success rate (non 5xx responses) - # percentage (0-100) - threshold: 99 - interval: 1m - - name: istio_request_duration_seconds_bucket - # maximum req duration P99 - # milliseconds - threshold: 500 - interval: 1m + stepWeight: 10 + metrics: + - name: istio_requests_total + # minimum req success rate (non 5xx responses) + # percentage (0-100) + threshold: 99 + interval: 1m + - name: istio_request_duration_seconds_bucket + # maximum req duration P99 + # milliseconds + threshold: 500 + interval: 30s diff --git a/artifacts/steerer/crd.yaml b/artifacts/steerer/crd.yaml index ef9ffe1a..f0126196 100644 --- a/artifacts/steerer/crd.yaml +++ b/artifacts/steerer/crd.yaml @@ -21,10 +21,18 @@ spec: properties: spec: required: + - targetKind + - virtualService - primary - canary - - virtualService + - canaryAnalysis properties: + targetKind: + type: string + virtualService: + properties: + name: + type: string primary: properties: name: @@ -37,22 +45,23 @@ spec: type: string host: type: string - virtualService: + canaryAnalysis: properties: - name: - type: string - weight: + maxWeight: type: number - metrics: - type: array - properties: - items: - type: object + stepWeight: + type: number + metrics: + type: array properties: - name: - type: string - interval: - type: string - pattern: "^[0-9]+(m)" - threshold: - type: number + items: + type: object + properties: + name: + type: string + interval: + type: string + pattern: "^[0-9]+(m)" + threshold: + type: number + diff --git a/artifacts/steerer/deployment.yaml b/artifacts/steerer/deployment.yaml index 04a95465..c3b76069 100644 --- a/artifacts/steerer/deployment.yaml +++ b/artifacts/steerer/deployment.yaml @@ -22,7 +22,7 @@ spec: serviceAccountName: steerer containers: - name: steerer - image: stefanprodan/steerer:0.0.1-beta.12 + image: stefanprodan/steerer:0.0.1-rc.1 imagePullPolicy: Always ports: - name: http diff --git a/artifacts/workloads/deployment-canary.yaml b/artifacts/workloads/deployment-canary.yaml index c9827df0..5ec7620b 100644 --- a/artifacts/workloads/deployment-canary.yaml +++ b/artifacts/workloads/deployment-canary.yaml @@ -57,7 +57,7 @@ spec: - http - localhost:9898/readyz failureThreshold: 3 - periodSeconds: 10 + periodSeconds: 3 successThreshold: 1 timeoutSeconds: 2 resources: @@ -65,7 +65,7 @@ spec: cpu: 1000m memory: 256Mi requests: - cpu: 10m + cpu: 100m memory: 16Mi volumeMounts: - mountPath: /data diff --git a/artifacts/workloads/deployment.yaml b/artifacts/workloads/deployment.yaml index b035f491..a16eaa2f 100644 --- a/artifacts/workloads/deployment.yaml +++ b/artifacts/workloads/deployment.yaml @@ -3,16 +3,13 @@ kind: Deployment metadata: name: podinfo namespace: test - annotations: - apps.weave.works/progressive: "true" labels: app: podinfo spec: - replicas: 2 + replicas: 1 strategy: rollingUpdate: - maxSurge: 25% - maxUnavailable: 1 + maxUnavailable: 0 type: RollingUpdate selector: matchLabels: @@ -48,6 +45,7 @@ spec: - check - http - localhost:9898/healthz + initialDelaySeconds: 5 failureThreshold: 3 periodSeconds: 10 successThreshold: 1 @@ -59,11 +57,15 @@ spec: - check - http - localhost:9898/readyz + initialDelaySeconds: 5 failureThreshold: 3 - periodSeconds: 10 + periodSeconds: 3 successThreshold: 1 timeoutSeconds: 1 resources: + limits: + cpu: 2000m + memory: 512Mi requests: - cpu: 1m - memory: 16Mi + cpu: 10m + memory: 64Mi diff --git a/artifacts/workloads/hpa-canary.yaml b/artifacts/workloads/hpa-canary.yaml index aeb57387..18bec4b7 100644 --- a/artifacts/workloads/hpa-canary.yaml +++ b/artifacts/workloads/hpa-canary.yaml @@ -14,6 +14,8 @@ spec: - type: Resource resource: name: cpu + # scale up if usage is above + # 99% of the requested CPU (100m) targetAverageUtilization: 99 - type: Resource resource: diff --git a/artifacts/workloads/hpa.yaml b/artifacts/workloads/hpa.yaml new file mode 100644 index 00000000..fa2b5a6f --- /dev/null +++ b/artifacts/workloads/hpa.yaml @@ -0,0 +1,19 @@ +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: podinfo + namespace: test +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: podinfo + minReplicas: 2 + maxReplicas: 4 + metrics: + - type: Resource + resource: + name: cpu + # scale up if usage is above + # 99% of the requested CPU (100m) + targetAverageUtilization: 99 diff --git a/artifacts/workloads/virtual-service.yaml b/artifacts/workloads/virtual-service.yaml index b36e74a8..bfcf7b72 100644 --- a/artifacts/workloads/virtual-service.yaml +++ b/artifacts/workloads/virtual-service.yaml @@ -1,13 +1,10 @@ apiVersion: networking.istio.io/v1alpha3 kind: VirtualService metadata: - annotations: - apps.weave.works/progressive-revision: "" - apps.weave.works/progressive-status: "" - labels: - app: podinfo name: podinfo namespace: test + labels: + app: podinfo spec: gateways: - public-gateway.istio-system.svc.cluster.local @@ -26,4 +23,7 @@ spec: port: number: 9898 weight: 0 - timeout: 30s + timeout: 10s + retries: + attempts: 3 + perTryTimeout: 2s diff --git a/chart/steerer/Chart.yaml b/chart/steerer/Chart.yaml index 65cf60f6..cac84709 100644 --- a/chart/steerer/Chart.yaml +++ b/chart/steerer/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v1 name: steerer version: 0.0.1 -appVersion: 0.0.1-beta.12 +appVersion: 0.0.1-rc.1 description: Steerer is a Kubernetes operator that automates the promotion of canary deployments using Istio routing for traffic shifting and Prometheus metrics for canary analysis. diff --git a/chart/steerer/templates/crd.yaml b/chart/steerer/templates/crd.yaml index 53427b46..153c2f83 100644 --- a/chart/steerer/templates/crd.yaml +++ b/chart/steerer/templates/crd.yaml @@ -22,10 +22,18 @@ spec: properties: spec: required: + - targetKind + - virtualService - primary - canary - - virtualService + - canaryAnalysis properties: + targetKind: + type: string + virtualService: + properties: + name: + type: string primary: properties: name: @@ -38,23 +46,23 @@ spec: type: string host: type: string - virtualService: + canaryAnalysis: properties: - name: - type: string - weight: + maxWeight: type: number - metrics: - type: array - properties: - items: - type: object + stepWeight: + type: number + metrics: + type: array properties: - name: - type: string - interval: - type: string - pattern: "^[0-9]+(m)" - threshold: - type: number + items: + type: object + properties: + name: + type: string + interval: + type: string + pattern: "^[0-9]+(m)" + threshold: + type: number {{- end }} diff --git a/chart/steerer/values.yaml b/chart/steerer/values.yaml index 37da5ada..0d9bb1b6 100644 --- a/chart/steerer/values.yaml +++ b/chart/steerer/values.yaml @@ -2,7 +2,7 @@ image: repository: stefanprodan/steerer - tag: 0.0.1-beta.12 + tag: 0.0.1-rc.1 pullPolicy: IfNotPresent controlLoopInterval: "10s" diff --git a/docs/index.yaml b/docs/index.yaml index 829ce00c..5f71a325 100755 --- a/docs/index.yaml +++ b/docs/index.yaml @@ -2,14 +2,14 @@ apiVersion: v1 entries: steerer: - apiVersion: v1 - appVersion: 0.0.1-beta.12 - created: 2018-09-28T16:49:49.90177054+03:00 + appVersion: 0.0.1-rc.1 + created: 2018-09-29T11:08:25.598356915+03:00 description: Steerer is a Kubernetes operator that automates the promotion of canary deployments using Istio routing for traffic shifting and Prometheus metrics for canary analysis. - digest: 792a2bf520cac415a2e2a4a3b2b7142c9fb09a4737ea7135146bd5796c5f9d94 + digest: af14826edae5afcda1b2afebf17e3b8007f1d2a35e65093ab32e786a6599b201 name: steerer urls: - https://stefanprodan.github.io/steerer/steerer-0.0.1.tgz version: 0.0.1 -generated: 2018-09-28T16:49:49.900919976+03:00 +generated: 2018-09-29T11:08:25.597473362+03:00 diff --git a/docs/steerer-0.0.1.tgz b/docs/steerer-0.0.1.tgz index 61148685b8fda98aa33a58b1a10c48d8d08b193f..09776f905bfe1f97464d941363c1bfed5d7b9acd 100644 GIT binary patch delta 3098 zcmV+#4CV8T7`GUZJb!;v8#|KuTc4uO?A9i1W<16L@?6JOl7AK;O?TT)Bi>TXHKB5Ks#!<1?p&S=IT-8)24 z6z%Ws`u|ZBE&h-C`@4^N{r+x$uixwM_Z~&Po!$NYM{xf=S$_e!u~a{bqHpE1IJn;l zP{rP(HcW~EOnZbXRewfd6!roghdq*@i8WK!m!B6)2ABZ~|HwzE1zI#96>4gw2Afg~ zG`BLNuH2>wO3O?-a*$()si1mB01{MO&N39%KyH|rz_GDRf|j{;Ut`Bi)A5+aU{W^r zg;XSPs%3^Y#edvDhE}uKI9viHYoLPinPDa*f@U~?a+Q;5Yi#643;#C&mHtmD&(VAz z18|%E??;RJ-;eq`8~uL>*n%S*)0|rj6Q#Q|8I zBO!W3YnI>uIvq03IbSLy8jZ|#jK*=`EMwOH0?07)g6Zt`c3!iNFN;;~ zM-)X_$A8Q3bB<<-pudElbqEnM!E@voTd4Tj6; zO8+fpiaT@NJ$du?<#}i??Y*tB+w?!$*;&y4XFGd)8~uL>s5f(AEae=N&8GN=f@S^x zB}J}Ko6xFzas_VD|D9-OasS`#?LFJ*|3kplReyIIrYswPK?`HX(asbeWKJw`iUZj0 z-rNv}MP6Pi=X*PM-tCRflXt&kA32f|FMtHcOdxbBC&J-&Aeh&BZzC)M&J)Kdd|x=9 z%^er6-QFx>*z5bBEIZG~V|EFhV9pFh;(r(YJM?=x(7Fw`i_hS5PB|MhOn@rodmfU1 zVt>Ki<6GB;8-;-p#xyqwMrPOyZTxu040&RJYAiN?Ou$O8DKqeRH1o1JIyrZ^E=(~B z<0m0G9)m`X)RaPVjt^zR6k;pNn1u_*IgAi;Rr=K(RGJAphR&Z&@MqIm;8wKz zHS28M@7DTjOsrFvGURlV>L<5cDFqg%BjjJ0V2{;VJDjGpi{EAN4%g0M=QKv>{Ivs} zUpikk&L3@*Ht=Auy#LcUmbtjkT;T2bzy9+6zw!Sc1QzmtpO{Dn;9_qUCdU|IiTo!oO8@D}~=@9Zx6|Ihl-#{Yj1Sa2HMGTEK>MrdheHV$)RWp-9($$yT}XQ-GID-4L1oXgyrkSb-u3#3yFFQooti1V8r z-bBk>xn>vY0j6_%RkU6gmG&km8=2QTFsP@PlqtWm$V%-kL|Nv@N*;>mn(}!^eW77u zl5?u-VqaPe{;W}AQmO69oLS)U34n>E!U6ayMPp1{-zjH_Pwf@RQlWTpdVl=Be;%ip z(Lvc)rKLh`S;f=0Q_89Wjh}O-F=>{$)-BZ&wEtitS*VyYZF9;?qqtD1G|P%}nr3Nz zc(YviqC9nJRIci^gL#kI zIsjxM=6SL*8~KG8aEWSd2)KlK{}Ev2KWS8)#ncSIIg7?>YH?C-vxO}>!@?_e)z!gl zags}IoW5!d^zK8wbp(}8R%gcZNpQehjebqDcMx6Evox)2z)9;uO@UFSa#1(BQ#s*n z%z|H>Yi4JMQdqoP*nhn>(&UZcvok5}-xjzQVk&BauRTyB0peaEZcSEg^PeosegiM_djS1XT^<-_J{%q zO1~-ap_Fb?w=Qk8={TFBKx0m|94$C5c-f9$)(p64{bGWPZc_k3@HuBI%dKW;4tlF< zScWpU2Rke3jAF@>!x%t$2gTnn@y@ZErEZFp+w|8~07et`#~EyW9p>r^to)f~yj~D> zC0qj&2?`6noquX!IG65%*>rwJ)VT06Up=jBFwHjRJfAmNmR{n@7OML)bMEyGns4OX zxSBI(-n_4Kcm6+M|6lh1^oYjyn*hAU|Lg7TJzLEGJ$tsh@&6tI7V>|7@>eGSeHG1Z zDmDAnFYPs1z~VCAndEralY={Xj=!!36+*Ui5s#6X}7@B@?XEl{A8v7XD?qI zy?z;H$#-buw*24z&Z7Uf)7!lNco^8Ko^ug$Pw%>#ue!)Ryy{wh(?zzn;JCQ2?nqrf z{D18?Lf*>iB75VKowqOUvvm=3Sg!oxfRN#E7)xQCC-5h58zJOsqG^H<1Qov$9m1~X zYN`|r1VO#KA2eU4IIe`eb=@eU{8BKcen2OM?^Z+9Zc) zDD&gvt=9JwFc|xJ;mp`_uBfINTGXbP$$yv%82KBio(>a8a_t0K4!Q7}ytLm5wocx8 zRn<+9qeWj`3%9WtR?4}g2?CYt3D(KNCDCcje_@=UH&=rN8t*Ygmz6I+@d1I&lR5{{H3J z`SF{RztvdtyyhI)m4Pafo!2mIzUUf~VIA~@)mx`-ta3-KWCDk$@2WRYjyinWo}JdStXNPlra48aY)Fy z!kCR`fHY3$OR#)G>-8@P3|jbvvje~0K0K{|ejEbTliYX1N*FfH-7ti|OMeY`Ni)Uq z={Fk2|01x~IF^+L?LoaN527#%cM9Eq!JPx>{YXB03KV{R@%oi_(F)sTyaZUO&5fsv z?vsw>w56J*rBhl?S>lXC>KHAUwBjAZ!wO&wjd!=j$o~f0+i#iL-X6fuGWT82O<`3l o6Y`-!^zm`y<4M8dNBa~T*b~48ejo6^0RRC1{~Ube^#DWw02sn56951J delta 3054 zcmVpQ@Jb!I-+cwhqj9;;j+)fiWq$nwlQ_bVf!=+YSiuB3Dq{9(VXAA@scD- zvc0wC|0hYZ{6Be=B=Hl_TS^s-UuY0Hew^24=4@b1%7Qzbf3RfmB*y#lawylgZ8RKfY7OQn4)~bHHt!kBznKKfRnHh_DY^G_P^1@iuF_$Uj{U4Gak`^JU&RpM9jnrZv ztS*ocJ*4R>)ESAFQ`DMe*oRh&j0(1Y(bwjSe75d*zx?4;7zty?(|8J+9oqx4AVaoD87_=~A9PLcuPVU5#POuM~ z?TZWIu*kDhLW)|;zf|*hzW#N%=#f!1xfEqn$a3d-4tVFpww-}9LK z6MqADk8fQYZWIQFn9{-^7@1=;uJPj?G31#6sxj>Rn1PjGC(OY8;mpfo@8HgBUI(Pp6t&}CF(M^6xj5!Ty8V9Nvo>~;{ z%f#=Wn3&1-v@=9YE3>g%7%TImGJi$3hdw>Uq*!A>wB%eC*2GjP6Q3ZRVtgX?dqbSx z?C{1~7RoidR1Ywn+pExe4Jz$TR5miNcg3KdVpit+N|2S>TZj_o$VwiH7n<^UM?KIm zF)ldOHP}PL;Ex&*Gm+XJ%$Y?Vp8%LtD(r)=QZ&WP^__B-`Mh30mI}qw!+-rZz2kI( zIqjEyRaz?4mQ_4`JEg3uqVYq)G-gehYh7uUhvSwev_6Ar0;f#dg7VT#HY(S(AK;$% zBJgr_SN!a(KmlCj*#~1a6XTWYi8S_;h{Evxc>P+G@E7mj39|9zBRv#FKJ@APc`4KH z%QOL>YE5UK&}n8dUjwe#$$#OmnA$bfAL9uh5e8f(5Z|flz~T2Y08rFg)S?fqf4@zl zAKra`KY!3#i)$wu%|!C-}moUldAz%MN{v$I_(ES@fHO&V$TQt;W4l=dgaF&yzOcx@$OA)cDo26Z1g zNs@FLmEq7DT5fQqUw`XC@%#`QE-v~j)s~Lo;-a<8>9BExy+SiT-)_nmKuK*EcCJbu z6KmC?pHd0$P`Z`>BzcmMW^JBRku7qH+pAKIAmVZyahxLGVPYf~>&nyih)b!WYWv%1 z3}?-8{Njirh)TyG@~*UQQZJm?FVb-~MS;c~YB^lmvGH`#e}BDXz@hb%F)sTF5k%34 zf~|#H&Cu+0*43~aWnp)^YwC<**^)QpD#{B8{&InL_}nbDCrG)S{I=kL(~SLf23y~S zI6DI?f2KLF7erkN=fFgU!a}E8Eez-5oG?3?pAj@}JIq(l!X1ca8*`pdnk-AVX>AMD zWr;cW`ufCYa(`~z%a}87Ue>ri|F5wBulj#_NYk550A1n#bsj%n$^Sgs>fHH%w^5ez zKYsGpCxCnvE$l>U_M2bYmt=v$O{^oy@w%DgYk7`;uj5TPAdRTu(Af%}BWUr5&Uy2t+pF0JAVn0{K9B;d+!`p< zz?s09#CylT99yZ8Y{0I}b1C4>?lELc8xoJ1ZTr8W`y?LzrrZ8+1(=N6?oah;rlLJ( zG^FYKLixK-hHS>>M3K#SNZ*sq*yf6C{)cS98-J>qEDY@LJu@U$TKcSI%rd0yfT!iJ zf5sEVO8<|ZJ>7fpEY7np(8g8yzwPd_|JUvG9^dKzZIq4bEe|0#^r)x#q=($Vlb+QF zJ!E48_QQQ;N9y|FZ$A<8N>&${8<(uSeQ}=)7x4z=${+Lz84L!g6vlZ1-+9=ait62cr}-?waV6xH>qdz3OTn1>^^_F8TMbc{uNZ)ZnkKus z4-tE&Z&N8oY+QsW^WFW8h3^MoF!l4onX%O` zcuii~uLZkI-bGc_&XA);UtJ5gu^86MxqqaIB30-y*2%&Z(P_-(6b)1}+mxBil@$H~ z88t?xjM{2bnXiE$evZ|;Etf(V-ELEIq1}<>T%P#R(jiCF5MwRpAzzSbc^$Tz+&wvc-peL>#y0lYOIBF#`*gbq* zJ%Dkvog`mR5}w6WRN<>h*z3Mv3zXpz&T>;88X8KLnkos_HdK@OS3CnUsFFkuPsj}0 z4!Hy}pWHf-0?DA6s;Y3y*vZ>4Pk&|%8w;lju7$IU0U0#XM2^ZWGPjOAwWZ9hX)CK_ zl8X`sZ{H0FIaZjm(F~BLllc-X-%on|iz0&--s5b?ueY}k>!0riK=rur-LMh{O>;Mh z;U}p9Pid|=KKx9>_%8yN8ppEIpxvog|4L!j{U(`OgYy^9uX zm+=x{r8YO7uDMS-l+y*(EG?bVa>_Di943y@l1U5q*l$+=BWS$43?u&$HaA}}wYk}c wpJm~@UYKB2D--gzLG 0 { + maxWeight = r.Spec.CanaryAnalysis.MaxWeight } // gate stage: check if canary deployment exists and is healthy @@ -44,6 +44,12 @@ func (c *Controller) advanceDeploymentRollout(name string, namespace string) { return } + // gate stage: check if primary deployment exists and is healthy + primary, ok := c.getDeployment(r, r.Spec.Primary.Name, r.Namespace) + if !ok { + return + } + // gate stage: check if virtual service exists // and if it contains weighted destination routes to the primary and canary services vs, primaryRoute, canaryRoute, ok := c.getVirtualService(r) @@ -67,15 +73,15 @@ func (c *Controller) advanceDeploymentRollout(name string, namespace string) { } // routing stage: increase canary traffic percentage - if canaryRoute.Weight != 100 { - primaryRoute.Weight -= r.Spec.VirtualService.Weight - if primaryRoute.Weight > 100 { - primaryRoute.Weight = 100 - } - canaryRoute.Weight += r.Spec.VirtualService.Weight + if canaryRoute.Weight < maxWeight { + primaryRoute.Weight -= r.Spec.CanaryAnalysis.StepWeight if primaryRoute.Weight < 0 { primaryRoute.Weight = 0 } + canaryRoute.Weight += r.Spec.CanaryAnalysis.StepWeight + if primaryRoute.Weight > 100 { + primaryRoute.Weight = 100 + } if ok := c.updateVirtualServiceRoutes(r, vs, primaryRoute, canaryRoute); !ok { return @@ -84,7 +90,7 @@ func (c *Controller) advanceDeploymentRollout(name string, namespace string) { c.recordEventInfof(r, "Advance rollout %s.%s weight %v", r.Name, r.Namespace, canaryRoute.Weight) // promotion stage: override primary.template.spec with the canary spec - if canaryRoute.Weight == 100 { + if canaryRoute.Weight == maxWeight { c.recordEventInfof(r, "Copying %s.%s template spec to %s.%s", canary.GetName(), canary.Namespace, primary.GetName(), primary.Namespace) @@ -175,7 +181,7 @@ func (c *Controller) getDeployment(r *rolloutv1.Rollout, name string, namespace } if msg, healthy := getDeploymentStatus(dep); !healthy { - c.logger.Infof("Halt rollout for %s.%s %s", dep.GetName(), dep.Namespace, msg) + c.recordEventWarningf(r, "Halt rollout %s.%s %s", dep.GetName(), dep.Namespace, msg) return nil, false } @@ -187,7 +193,7 @@ func (c *Controller) getDeployment(r *rolloutv1.Rollout, name string, namespace } func (c *Controller) checkDeploymentMetrics(r *rolloutv1.Rollout) bool { - for _, metric := range r.Spec.Metrics { + for _, metric := range r.Spec.CanaryAnalysis.Metrics { if metric.Name == "istio_requests_total" { val, err := c.getDeploymentCounter(r.Spec.Canary.Name, r.Namespace, metric.Name, metric.Interval) if err != nil { @@ -195,7 +201,7 @@ func (c *Controller) checkDeploymentMetrics(r *rolloutv1.Rollout) bool { return false } if float64(metric.Threshold) > val { - c.recordEventErrorf(r, "Halt rollout %s.%s success rate %.2f%% < %v%%", + c.recordEventWarningf(r, "Halt rollout %s.%s success rate %.2f%% < %v%%", r.Name, r.Namespace, val, metric.Threshold) return false } @@ -209,7 +215,7 @@ func (c *Controller) checkDeploymentMetrics(r *rolloutv1.Rollout) bool { } t := time.Duration(metric.Threshold) * time.Millisecond if val > t { - c.recordEventErrorf(r, "Halt rollout %s.%s request duration %v > %v", + c.recordEventWarningf(r, "Halt rollout %s.%s request duration %v > %v", r.Name, r.Namespace, val, t) return false } diff --git a/pkg/version/version.go b/pkg/version/version.go index b4ed4aba..addeb583 100644 --- a/pkg/version/version.go +++ b/pkg/version/version.go @@ -1,4 +1,4 @@ package version -var VERSION = "0.0.1-beta.12" +var VERSION = "0.0.1-rc.1" var REVISION = "unknown"