From c4981ef4dbe71ff61e0f3bb070a48b6ff318ec2a Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 16 Jan 2019 00:57:46 +0200 Subject: [PATCH 1/8] Add status and additional printer columns to CRD --- README.md | 6 +++--- artifacts/flagger/crd.yaml | 9 +++++++++ charts/flagger/templates/crd.yaml | 9 +++++++++ 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 3e727349..5766863e 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,8 @@ Flagger is a Kubernetes operator that automates the promotion of canary deployments using Istio routing for traffic shifting and Prometheus metrics for canary analysis. -The canary analysis can be extended with webhooks for running integration tests, load tests or any other custom -validation. +The canary analysis can be extended with webhooks for running integration tests, +load tests or any other custom validation. ### Install @@ -28,7 +28,7 @@ helm upgrade -i flagger flagger/flagger \ --set metricsServer=http://prometheus.istio-system:9090 ``` -Flagger is compatible with Kubernetes >1.10.0 and Istio >1.0.0. +Flagger is compatible with Kubernetes >1.11.0 and Istio >1.0.0. ### Usage diff --git a/artifacts/flagger/crd.yaml b/artifacts/flagger/crd.yaml index 9acfa09b..7983760b 100644 --- a/artifacts/flagger/crd.yaml +++ b/artifacts/flagger/crd.yaml @@ -20,6 +20,15 @@ spec: singular: canary kind: Canary scope: Namespaced + subresources: + status: {} + additionalPrinterColumns: + - name: Status + type: string + JSONPath: .status.state + - name: LastTransitionTime + type: string + JSONPath: .status.lastTransitionTime validation: openAPIV3Schema: properties: diff --git a/charts/flagger/templates/crd.yaml b/charts/flagger/templates/crd.yaml index c9789f0f..275be88c 100644 --- a/charts/flagger/templates/crd.yaml +++ b/charts/flagger/templates/crd.yaml @@ -21,6 +21,15 @@ spec: singular: canary kind: Canary scope: Namespaced + subresources: + status: {} + additionalPrinterColumns: + - name: Status + type: string + JSONPath: .status.state + - name: LastTransitionTime + type: string + JSONPath: .status.lastTransitionTime validation: openAPIV3Schema: properties: From ada446bbaa92cdfc324ccc4a93f6f34f7e4189df Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 16 Jan 2019 00:58:51 +0200 Subject: [PATCH 2/8] Drop compatibility with Kubernetes 1.10 --- charts/flagger/Chart.yaml | 4 ++-- charts/flagger/README.md | 2 +- docs/gitbook/README.md | 13 ++++++++++--- docs/gitbook/install/install-flagger.md | 2 +- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/charts/flagger/Chart.yaml b/charts/flagger/Chart.yaml index eb6ac91a..77788dd9 100644 --- a/charts/flagger/Chart.yaml +++ b/charts/flagger/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 name: flagger -version: 0.3.0 +version: 0.4.0 appVersion: 0.3.1-alpha.1 -kubeVersion: ">=1.9.0-0" +kubeVersion: ">=1.11.0-0" engine: gotpl description: Flagger is a Kubernetes operator that automates the promotion of canary deployments using Istio routing for traffic shifting and Prometheus metrics for canary analysis. home: https://docs.flagger.app diff --git a/charts/flagger/README.md b/charts/flagger/README.md index 96fd2d92..2a541a4b 100644 --- a/charts/flagger/README.md +++ b/charts/flagger/README.md @@ -8,7 +8,7 @@ Based on the KPIs analysis a canary is promoted or aborted and the analysis resu ## Prerequisites -* Kubernetes >= 1.9 +* Kubernetes >= 1.11 * Istio >= 1.0 * Prometheus >= 2.6 diff --git a/docs/gitbook/README.md b/docs/gitbook/README.md index 44828148..92dba1e3 100644 --- a/docs/gitbook/README.md +++ b/docs/gitbook/README.md @@ -4,13 +4,20 @@ description: Flagger is an Istio progressive delivery Kubernetes operator # Introduction -[Flagger](https://github.com/stefanprodan/flagger) is a **Kubernetes** operator that automates the promotion of canary deployments using **Istio** routing for traffic shifting and **Prometheus** metrics for canary analysis. +[Flagger](https://github.com/stefanprodan/flagger) is a **Kubernetes** operator that automates the promotion of canary +deployments using **Istio** routing for traffic shifting and **Prometheus** metrics for canary analysis. +The canary analysis can be extended with webhooks for running integration tests, +load tests or any other custom validation. -Flagger implements a control loop that gradually shifts traffic to the canary while measuring key performance indicators like HTTP requests success rate, requests average duration and pods health. Based on the **KPIs** analysis a canary is promoted or aborted and the analysis result is published to **Slack**. +Flagger implements a control loop that gradually shifts traffic to the canary while measuring key performance +indicators like HTTP requests success rate, requests average duration and pods health. +Based on the **KPIs** analysis a canary is promoted or aborted and the analysis result is published to **Slack**. ![Flagger overview diagram](https://raw.githubusercontent.com/stefanprodan/flagger/master/docs/diagrams/flagger-canary-overview.png) -Flagger can be configured with Kubernetes custom resources \(canaries.flagger.app kind\) and is compatible with any CI/CD solutions made for Kubernetes. Since Flagger is declarative and reacts to Kubernetes events, it can be used in **GitOps** pipelines together with Weave Flux or JenkinsX. +Flagger can be configured with Kubernetes custom resources \(canaries.flagger.app kind\) and is compatible with +any CI/CD solutions made for Kubernetes. Since Flagger is declarative and reacts to Kubernetes events, +it can be used in **GitOps** pipelines together with Weave Flux or JenkinsX. This project is sponsored by [Weaveworks](https://www.weave.works/) diff --git a/docs/gitbook/install/install-flagger.md b/docs/gitbook/install/install-flagger.md index 2b8c4d76..fd85d34d 100644 --- a/docs/gitbook/install/install-flagger.md +++ b/docs/gitbook/install/install-flagger.md @@ -6,7 +6,7 @@ If you are new to Istio you can follow this GKE guide **Prerequisites** -* Kubernetes >= 1.9 +* Kubernetes >= 1.11 * Istio >= 1.0 * Prometheus >= 2.6 From 4bf3b70048ba9d660e9e0af84f12e10743b4afec Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 16 Jan 2019 01:00:39 +0200 Subject: [PATCH 3/8] Use CRD UpdateStatus for Canary status updated - requires Kubernetes >=1.11 --- pkg/controller/deployer.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/controller/deployer.go b/pkg/controller/deployer.go index f590c781..62e2ff32 100644 --- a/pkg/controller/deployer.go +++ b/pkg/controller/deployer.go @@ -162,7 +162,7 @@ func (c *CanaryDeployer) SetFailedChecks(cd *flaggerv1.Canary, val int) error { cdCopy.Status.FailedChecks = val cdCopy.Status.LastTransitionTime = metav1.Now() - cd, err := c.flaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).Update(cdCopy) + cd, err := c.flaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy) if err != nil { return fmt.Errorf("canary %s.%s status update error %v", cdCopy.Name, cdCopy.Namespace, err) } @@ -175,7 +175,7 @@ func (c *CanaryDeployer) SetState(cd *flaggerv1.Canary, state flaggerv1.CanarySt cdCopy.Status.State = state cdCopy.Status.LastTransitionTime = metav1.Now() - cd, err := c.flaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).Update(cdCopy) + cd, err := c.flaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy) if err != nil { return fmt.Errorf("canary %s.%s status update error %v", cdCopy.Name, cdCopy.Namespace, err) } @@ -203,7 +203,7 @@ func (c *CanaryDeployer) SyncStatus(cd *flaggerv1.Canary, status flaggerv1.Canar cdCopy.Status.CanaryRevision = base64.StdEncoding.EncodeToString(specJson) cdCopy.Status.LastTransitionTime = metav1.Now() - cd, err = c.flaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).Update(cdCopy) + cd, err = c.flaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy) if err != nil { return fmt.Errorf("canary %s.%s status update error %v", cdCopy.Name, cdCopy.Namespace, err) } From 10c2bdec864a2e91c61b5edd8bda29f8266b67f7 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 16 Jan 2019 13:13:07 +0200 Subject: [PATCH 4/8] Use deep copy when updating the virtual service routes --- pkg/controller/router.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/controller/router.go b/pkg/controller/router.go index ce8f787b..f01594b9 100644 --- a/pkg/controller/router.go +++ b/pkg/controller/router.go @@ -276,13 +276,15 @@ func (c *CanaryRouter) SetRoutes( } return fmt.Errorf("VirtualService %s.%s query error %v", targetName, cd.Namespace, err) } - vs.Spec.Http = []istiov1alpha3.HTTPRoute{ + + vsCopy := vs.DeepCopy() + vsCopy.Spec.Http = []istiov1alpha3.HTTPRoute{ { Route: []istiov1alpha3.DestinationWeight{primary, canary}, }, } - vs, err = c.istioClient.NetworkingV1alpha3().VirtualServices(cd.Namespace).Update(vs) + vs, err = c.istioClient.NetworkingV1alpha3().VirtualServices(cd.Namespace).Update(vsCopy) if err != nil { return fmt.Errorf("VirtualService %s.%s update failed: %v", targetName, cd.Namespace, err) From 9872e6bc16e3c8bfab9bb91f27354ec23dfaaba9 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 16 Jan 2019 13:18:53 +0200 Subject: [PATCH 5/8] Skip readiness checks if canary analysis finished --- pkg/controller/deployer.go | 10 +++++++++- pkg/controller/scheduler.go | 9 ++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/pkg/controller/deployer.go b/pkg/controller/deployer.go index 62e2ff32..db6e866d 100644 --- a/pkg/controller/deployer.go +++ b/pkg/controller/deployer.go @@ -140,7 +140,7 @@ func (c *CanaryDeployer) IsNewSpec(cd *flaggerv1.Canary) (bool, error) { newSpec := &canary.Spec.Template.Spec oldSpecJson, err := base64.StdEncoding.DecodeString(cd.Status.CanaryRevision) if err != nil { - return false, err + return false, fmt.Errorf("%s.%s decode error %v", cd.Name, cd.Namespace, err) } oldSpec := &corev1.PodSpec{} err = json.Unmarshal(oldSpecJson, oldSpec) @@ -156,6 +156,14 @@ func (c *CanaryDeployer) IsNewSpec(cd *flaggerv1.Canary) (bool, error) { return false, nil } +// ShouldAdvance determines if the canary analysis can proceed +func (c *CanaryDeployer) ShouldAdvance(cd *flaggerv1.Canary) (bool, error) { + if cd.Status.CanaryRevision == "" || cd.Status.State == flaggerv1.CanaryRunning { + return true, nil + } + return c.IsNewSpec(cd) +} + // SetFailedChecks updates the canary failed checks counter func (c *CanaryDeployer) SetFailedChecks(cd *flaggerv1.Canary, val int) error { cdCopy := cd.DeepCopy() diff --git a/pkg/controller/scheduler.go b/pkg/controller/scheduler.go index 25a95892..e2e3672a 100644 --- a/pkg/controller/scheduler.go +++ b/pkg/controller/scheduler.go @@ -91,6 +91,13 @@ func (c *Controller) advanceCanary(name string, namespace string) { return } + if ok, err := c.deployer.ShouldAdvance(cd); !ok { + if err != nil { + c.recordEventWarningf(cd, "%v", err) + } + return + } + // set max weight default value to 100% maxWeight := 100 if cd.Spec.CanaryAnalysis.MaxWeight > 0 { @@ -249,7 +256,7 @@ func (c *Controller) advanceCanary(name string, namespace string) { func (c *Controller) checkCanaryStatus(cd *flaggerv1.Canary, deployer CanaryDeployer) bool { c.recorder.SetStatus(cd) - if cd.Status.State == "running" { + if cd.Status.State == flaggerv1.CanaryRunning { return true } From acdd2c46d572227afebee64eb62554162c664e72 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 16 Jan 2019 15:06:38 +0200 Subject: [PATCH 6/8] Refactor Canary status - add status phases (Initialized, Progressing, Succeeded, Failed) - rename status revision to LastAppliedSpec --- artifacts/flagger/crd.yaml | 2 +- charts/flagger/templates/crd.yaml | 2 +- pkg/apis/flagger/v1alpha3/types.go | 29 +++++++++++++++++++---------- pkg/controller/deployer.go | 16 ++++++++-------- pkg/controller/deployer_test.go | 12 ++++++------ pkg/controller/recorder.go | 4 ++-- pkg/controller/scheduler.go | 14 +++++++------- pkg/controller/scheduler_test.go | 6 +++--- 8 files changed, 47 insertions(+), 38 deletions(-) diff --git a/artifacts/flagger/crd.yaml b/artifacts/flagger/crd.yaml index 7983760b..ce7671ca 100644 --- a/artifacts/flagger/crd.yaml +++ b/artifacts/flagger/crd.yaml @@ -25,7 +25,7 @@ spec: additionalPrinterColumns: - name: Status type: string - JSONPath: .status.state + JSONPath: .status.phase - name: LastTransitionTime type: string JSONPath: .status.lastTransitionTime diff --git a/charts/flagger/templates/crd.yaml b/charts/flagger/templates/crd.yaml index 275be88c..6f58478d 100644 --- a/charts/flagger/templates/crd.yaml +++ b/charts/flagger/templates/crd.yaml @@ -26,7 +26,7 @@ spec: additionalPrinterColumns: - name: Status type: string - JSONPath: .status.state + JSONPath: .status.phase - name: LastTransitionTime type: string JSONPath: .status.lastTransitionTime diff --git a/pkg/apis/flagger/v1alpha3/types.go b/pkg/apis/flagger/v1alpha3/types.go index 944f9efa..32ea50ec 100755 --- a/pkg/apis/flagger/v1alpha3/types.go +++ b/pkg/apis/flagger/v1alpha3/types.go @@ -56,7 +56,7 @@ type CanarySpec struct { CanaryAnalysis CanaryAnalysis `json:"canaryAnalysis"` // the maximum time in seconds for a canary deployment to make progress - // before it is considered to be failed. Defaults to 60s. + // before it is considered to be failed. Defaults to ten minutes. ProgressDeadlineSeconds *int32 `json:"progressDeadlineSeconds,omitempty"` } @@ -70,21 +70,29 @@ type CanaryList struct { Items []Canary `json:"items"` } -// CanaryState used for status state op -type CanaryState string +// CanaryPhase is a label for the condition of a canary at the current time +type CanaryPhase string const ( - CanaryRunning CanaryState = "running" - CanaryFinished CanaryState = "finished" - CanaryFailed CanaryState = "failed" - CanaryInitialized CanaryState = "initialized" + // CanaryInitialized means the primary deployment, hpa and ClusterIP services + // have been created along with the Istio virtual service + CanaryInitialized CanaryPhase = "Initialized" + // CanaryProgressing means the canary analysis is underway + CanaryProgressing CanaryPhase = "Progressing" + // CanarySucceeded means the canary analysis has been successful + // and the canary deployment has been promoted + CanarySucceeded CanaryPhase = "Succeeded" + // CanaryFailed means the canary analysis failed + // and the canary deployment has been scaled to zero + CanaryFailed CanaryPhase = "Failed" ) // CanaryStatus is used for state persistence (read-only) type CanaryStatus struct { - State CanaryState `json:"state"` - CanaryRevision string `json:"canaryRevision"` - FailedChecks int `json:"failedChecks"` + Phase CanaryPhase `json:"phase"` + FailedChecks int `json:"failedChecks"` + // +optional + LastAppliedSpec string `json:"lastAppliedSpec,omitempty"` // +optional LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"` } @@ -139,6 +147,7 @@ func (c *Canary) GetProgressDeadlineSeconds() int { return ProgressDeadlineSeconds } +// GetAnalysisInterval returns the canary analysis interval (default 60s) func (c *Canary) GetAnalysisInterval() time.Duration { if c.Spec.CanaryAnalysis.Interval == "" { return AnalysisInterval diff --git a/pkg/controller/deployer.go b/pkg/controller/deployer.go index db6e866d..94cdc179 100644 --- a/pkg/controller/deployer.go +++ b/pkg/controller/deployer.go @@ -133,12 +133,12 @@ func (c *CanaryDeployer) IsNewSpec(cd *flaggerv1.Canary) (bool, error) { return false, fmt.Errorf("deployment %s.%s query error %v", targetName, cd.Namespace, err) } - if cd.Status.CanaryRevision == "" { + if cd.Status.LastAppliedSpec == "" { return true, nil } newSpec := &canary.Spec.Template.Spec - oldSpecJson, err := base64.StdEncoding.DecodeString(cd.Status.CanaryRevision) + oldSpecJson, err := base64.StdEncoding.DecodeString(cd.Status.LastAppliedSpec) if err != nil { return false, fmt.Errorf("%s.%s decode error %v", cd.Name, cd.Namespace, err) } @@ -158,7 +158,7 @@ func (c *CanaryDeployer) IsNewSpec(cd *flaggerv1.Canary) (bool, error) { // ShouldAdvance determines if the canary analysis can proceed func (c *CanaryDeployer) ShouldAdvance(cd *flaggerv1.Canary) (bool, error) { - if cd.Status.CanaryRevision == "" || cd.Status.State == flaggerv1.CanaryRunning { + if cd.Status.LastAppliedSpec == "" || cd.Status.Phase == flaggerv1.CanaryProgressing { return true, nil } return c.IsNewSpec(cd) @@ -178,9 +178,9 @@ func (c *CanaryDeployer) SetFailedChecks(cd *flaggerv1.Canary, val int) error { } // SetState updates the canary status state -func (c *CanaryDeployer) SetState(cd *flaggerv1.Canary, state flaggerv1.CanaryState) error { +func (c *CanaryDeployer) SetState(cd *flaggerv1.Canary, state flaggerv1.CanaryPhase) error { cdCopy := cd.DeepCopy() - cdCopy.Status.State = state + cdCopy.Status.Phase = state cdCopy.Status.LastTransitionTime = metav1.Now() cd, err := c.flaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy) @@ -206,9 +206,9 @@ func (c *CanaryDeployer) SyncStatus(cd *flaggerv1.Canary, status flaggerv1.Canar } cdCopy := cd.DeepCopy() - cdCopy.Status.State = status.State + cdCopy.Status.Phase = status.Phase cdCopy.Status.FailedChecks = status.FailedChecks - cdCopy.Status.CanaryRevision = base64.StdEncoding.EncodeToString(specJson) + cdCopy.Status.LastAppliedSpec = base64.StdEncoding.EncodeToString(specJson) cdCopy.Status.LastTransitionTime = metav1.Now() cd, err = c.flaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy) @@ -247,7 +247,7 @@ func (c *CanaryDeployer) Sync(cd *flaggerv1.Canary) error { return fmt.Errorf("creating deployment %s.%s failed: %v", primaryName, cd.Namespace, err) } - if cd.Status.State == "" { + if cd.Status.Phase == "" { c.logger.Infof("Scaling down %s.%s", cd.Spec.TargetRef.Name, cd.Namespace) if err := c.Scale(cd, 0); err != nil { return err diff --git a/pkg/controller/deployer_test.go b/pkg/controller/deployer_test.go index 2f9ea9ea..83740761 100644 --- a/pkg/controller/deployer_test.go +++ b/pkg/controller/deployer_test.go @@ -387,7 +387,7 @@ func TestCanaryDeployer_SetState(t *testing.T) { t.Fatal(err.Error()) } - err = deployer.SetState(canary, v1alpha3.CanaryRunning) + err = deployer.SetState(canary, v1alpha3.CanaryProgressing) if err != nil { t.Fatal(err.Error()) } @@ -397,8 +397,8 @@ func TestCanaryDeployer_SetState(t *testing.T) { t.Fatal(err.Error()) } - if res.Status.State != v1alpha3.CanaryRunning { - t.Errorf("Got %v wanted %v", res.Status.State, v1alpha3.CanaryRunning) + if res.Status.Phase != v1alpha3.CanaryProgressing { + t.Errorf("Got %v wanted %v", res.Status.Phase, v1alpha3.CanaryProgressing) } } @@ -424,7 +424,7 @@ func TestCanaryDeployer_SyncStatus(t *testing.T) { } status := v1alpha3.CanaryStatus{ - State: v1alpha3.CanaryRunning, + Phase: v1alpha3.CanaryProgressing, FailedChecks: 2, } err = deployer.SyncStatus(canary, status) @@ -437,8 +437,8 @@ func TestCanaryDeployer_SyncStatus(t *testing.T) { t.Fatal(err.Error()) } - if res.Status.State != status.State { - t.Errorf("Got state %v wanted %v", res.Status.State, status.State) + if res.Status.Phase != status.Phase { + t.Errorf("Got state %v wanted %v", res.Status.Phase, status.Phase) } if res.Status.FailedChecks != status.FailedChecks { diff --git a/pkg/controller/recorder.go b/pkg/controller/recorder.go index 31b2444e..47e23a6e 100644 --- a/pkg/controller/recorder.go +++ b/pkg/controller/recorder.go @@ -72,8 +72,8 @@ func (cr *CanaryRecorder) SetTotal(namespace string, total int) { // SetStatus sets the last known canary analysis status func (cr *CanaryRecorder) SetStatus(cd *flaggerv1.Canary) { status := 1 - switch cd.Status.State { - case flaggerv1.CanaryRunning: + switch cd.Status.Phase { + case flaggerv1.CanaryProgressing: status = 0 case flaggerv1.CanaryFailed: status = 2 diff --git a/pkg/controller/scheduler.go b/pkg/controller/scheduler.go index e2e3672a..3c4fc766 100644 --- a/pkg/controller/scheduler.go +++ b/pkg/controller/scheduler.go @@ -137,7 +137,7 @@ func (c *Controller) advanceCanary(name string, namespace string) { } // check if the number of failed checks reached the threshold - if cd.Status.State == flaggerv1.CanaryRunning && + if cd.Status.Phase == flaggerv1.CanaryProgressing && (!retriable || cd.Status.FailedChecks >= cd.Spec.CanaryAnalysis.Threshold) { if cd.Status.FailedChecks >= cd.Spec.CanaryAnalysis.Threshold { @@ -173,7 +173,7 @@ func (c *Controller) advanceCanary(name string, namespace string) { } // mark canary as failed - if err := c.deployer.SyncStatus(cd, flaggerv1.CanaryStatus{State: flaggerv1.CanaryFailed}); err != nil { + if err := c.deployer.SyncStatus(cd, flaggerv1.CanaryStatus{Phase: flaggerv1.CanaryFailed}); err != nil { c.logger.Errorf("%v", err) return } @@ -244,7 +244,7 @@ func (c *Controller) advanceCanary(name string, namespace string) { } // update status - if err := c.deployer.SetState(cd, flaggerv1.CanaryFinished); err != nil { + if err := c.deployer.SetState(cd, flaggerv1.CanarySucceeded); err != nil { c.recordEventWarningf(cd, "%v", err) return } @@ -256,12 +256,12 @@ func (c *Controller) advanceCanary(name string, namespace string) { func (c *Controller) checkCanaryStatus(cd *flaggerv1.Canary, deployer CanaryDeployer) bool { c.recorder.SetStatus(cd) - if cd.Status.State == flaggerv1.CanaryRunning { + if cd.Status.Phase == flaggerv1.CanaryProgressing { return true } - if cd.Status.State == "" { - if err := deployer.SyncStatus(cd, flaggerv1.CanaryStatus{State: flaggerv1.CanaryInitialized}); err != nil { + if cd.Status.Phase == "" { + if err := deployer.SyncStatus(cd, flaggerv1.CanaryStatus{Phase: flaggerv1.CanaryInitialized}); err != nil { c.logger.Errorf("%v", err) return false } @@ -280,7 +280,7 @@ func (c *Controller) checkCanaryStatus(cd *flaggerv1.Canary, deployer CanaryDepl c.recordEventErrorf(cd, "%v", err) return false } - if err := deployer.SyncStatus(cd, flaggerv1.CanaryStatus{State: flaggerv1.CanaryRunning}); err != nil { + if err := deployer.SyncStatus(cd, flaggerv1.CanaryStatus{Phase: flaggerv1.CanaryProgressing}); err != nil { c.logger.Errorf("%v", err) return false } diff --git a/pkg/controller/scheduler_test.go b/pkg/controller/scheduler_test.go index e764f9b4..b796cf5f 100644 --- a/pkg/controller/scheduler_test.go +++ b/pkg/controller/scheduler_test.go @@ -194,7 +194,7 @@ func TestScheduler_Rollback(t *testing.T) { ctrl.advanceCanary("podinfo", "default") // update failed checks to max - err := deployer.SyncStatus(canary, v1alpha3.CanaryStatus{State: v1alpha3.CanaryRunning, FailedChecks: 11}) + err := deployer.SyncStatus(canary, v1alpha3.CanaryStatus{Phase: v1alpha3.CanaryProgressing, FailedChecks: 11}) if err != nil { t.Fatal(err.Error()) } @@ -207,7 +207,7 @@ func TestScheduler_Rollback(t *testing.T) { t.Fatal(err.Error()) } - if c.Status.State != v1alpha3.CanaryFailed { - t.Errorf("Got canary state %v wanted %v", c.Status.State, v1alpha3.CanaryFailed) + if c.Status.Phase != v1alpha3.CanaryFailed { + t.Errorf("Got canary state %v wanted %v", c.Status.Phase, v1alpha3.CanaryFailed) } } From 1ef310f00daff584ba26038d99473dc31d91fa59 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 16 Jan 2019 16:29:59 +0200 Subject: [PATCH 7/8] Add traffic weight to canary status - show current weight on kubectl get canaries and kubectl get all --- README.md | 25 +++++++++++++------ artifacts/flagger/crd.yaml | 5 ++++ charts/flagger/templates/crd.yaml | 5 ++++ docs/gitbook/usage/progressive-delivery.md | 25 ++++++++++++------- pkg/apis/flagger/v1alpha3/types.go | 1 + pkg/controller/deployer.go | 28 ++++++++++++++++++---- pkg/controller/deployer_test.go | 4 ++-- pkg/controller/scheduler.go | 14 +++++++---- 8 files changed, 81 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 5766863e..bc2fce37 100644 --- a/README.md +++ b/README.md @@ -242,15 +242,16 @@ kubectl -n test set image deployment/podinfo \ podinfod=quay.io/stefanprodan/podinfo:1.2.1 ``` -Flagger detects that the deployment revision changed and starts a new rollout: +Flagger detects that the deployment revision changed and starts a new canary analysis: ``` kubectl -n test describe canary/podinfo Status: - Canary Revision: 19871136 - Failed Checks: 0 - State: finished + Canary Weight: 0 + Failed Checks: 0 + Last Transition Time: 2019-01-16T13:47:16Z + Phase: Succeeded Events: Type Reason Age From Message ---- ------ ---- ---- ------- @@ -272,6 +273,15 @@ Events: Normal Synced 5s flagger Promotion completed! Scaling down podinfo.test ``` +You can monitor all canaries with: + +```bash +watch kubectl get canaries --all-namespaces + +NAMESPACE NAME STATUS WEIGHT LASTTRANSITIONTIME +test podinfo Progressing 5 2019-01-16T14:05:07Z +``` + During the canary analysis you can generate HTTP 500 errors and high latency to test if Flagger pauses the rollout. Create a tester pod and exec into it: @@ -300,9 +310,10 @@ the canary is scaled to zero and the rollout is marked as failed. kubectl -n test describe canary/podinfo Status: - Canary Revision: 16695041 - Failed Checks: 10 - State: failed + Canary Weight: 0 + Failed Checks: 10 + Last Transition Time: 2019-01-16T13:47:16Z + Phase: Failed Events: Type Reason Age From Message ---- ------ ---- ---- ------- diff --git a/artifacts/flagger/crd.yaml b/artifacts/flagger/crd.yaml index ce7671ca..e674913c 100644 --- a/artifacts/flagger/crd.yaml +++ b/artifacts/flagger/crd.yaml @@ -19,6 +19,8 @@ spec: plural: canaries singular: canary kind: Canary + categories: + - all scope: Namespaced subresources: status: {} @@ -26,6 +28,9 @@ spec: - name: Status type: string JSONPath: .status.phase + - name: Weight + type: string + JSONPath: .status.canaryWeight - name: LastTransitionTime type: string JSONPath: .status.lastTransitionTime diff --git a/charts/flagger/templates/crd.yaml b/charts/flagger/templates/crd.yaml index 6f58478d..22e80bc9 100644 --- a/charts/flagger/templates/crd.yaml +++ b/charts/flagger/templates/crd.yaml @@ -20,6 +20,8 @@ spec: plural: canaries singular: canary kind: Canary + categories: + - all scope: Namespaced subresources: status: {} @@ -27,6 +29,9 @@ spec: - name: Status type: string JSONPath: .status.phase + - name: Weight + type: string + JSONPath: .status.canaryWeight - name: LastTransitionTime type: string JSONPath: .status.lastTransitionTime diff --git a/docs/gitbook/usage/progressive-delivery.md b/docs/gitbook/usage/progressive-delivery.md index 71455a08..d2805897 100644 --- a/docs/gitbook/usage/progressive-delivery.md +++ b/docs/gitbook/usage/progressive-delivery.md @@ -108,9 +108,9 @@ Flagger detects that the deployment revision changed and starts a new rollout: kubectl -n test describe canary/podinfo Status: - Canary Revision: 19871136 - Failed Checks: 0 - State: finished + Canary Weight: 0 + Failed Checks: 0 + Phase: Succeeded Events: Type Reason Age From Message ---- ------ ---- ---- ------- @@ -132,6 +132,17 @@ Events: Normal Synced 5s flagger Promotion completed! Scaling down podinfo.test ``` +You can monitor all canaries with: + +```bash +watch kubectl get canaries --all-namespaces + +NAMESPACE NAME STATUS WEIGHT LASTTRANSITIONTIME +test podinfo Progressing 15 2019-01-16T14:05:07Z +prod frontend Succeeded 0 2019-01-15T16:15:07Z +prod backend Failed 0 2019-01-14T17:05:07Z +``` + During the canary analysis you can generate HTTP 500 errors and high latency to test if Flagger pauses the rollout. Create a tester pod and exec into it: @@ -162,9 +173,9 @@ When the number of failed checks reaches the canary analysis threshold, the traf kubectl -n test describe canary/podinfo Status: - Canary Revision: 16695041 - Failed Checks: 10 - State: failed + Canary Weight: 0 + Failed Checks: 10 + Phase: Failed Events: Type Reason Age From Message ---- ------ ---- ---- ------- @@ -181,5 +192,3 @@ Events: Warning Synced 1m flagger Canary failed! Scaling down podinfo.test ``` -#### - diff --git a/pkg/apis/flagger/v1alpha3/types.go b/pkg/apis/flagger/v1alpha3/types.go index 32ea50ec..c356d8a0 100755 --- a/pkg/apis/flagger/v1alpha3/types.go +++ b/pkg/apis/flagger/v1alpha3/types.go @@ -91,6 +91,7 @@ const ( type CanaryStatus struct { Phase CanaryPhase `json:"phase"` FailedChecks int `json:"failedChecks"` + CanaryWeight int `json:"canaryWeight"` // +optional LastAppliedSpec string `json:"lastAppliedSpec,omitempty"` // +optional diff --git a/pkg/controller/deployer.go b/pkg/controller/deployer.go index 94cdc179..e2fac6a8 100644 --- a/pkg/controller/deployer.go +++ b/pkg/controller/deployer.go @@ -164,8 +164,8 @@ func (c *CanaryDeployer) ShouldAdvance(cd *flaggerv1.Canary) (bool, error) { return c.IsNewSpec(cd) } -// SetFailedChecks updates the canary failed checks counter -func (c *CanaryDeployer) SetFailedChecks(cd *flaggerv1.Canary, val int) error { +// SetStatusFailedChecks updates the canary failed checks counter +func (c *CanaryDeployer) SetStatusFailedChecks(cd *flaggerv1.Canary, val int) error { cdCopy := cd.DeepCopy() cdCopy.Status.FailedChecks = val cdCopy.Status.LastTransitionTime = metav1.Now() @@ -177,10 +177,10 @@ func (c *CanaryDeployer) SetFailedChecks(cd *flaggerv1.Canary, val int) error { return nil } -// SetState updates the canary status state -func (c *CanaryDeployer) SetState(cd *flaggerv1.Canary, state flaggerv1.CanaryPhase) error { +// SetStatusWeight updates the canary status weight value +func (c *CanaryDeployer) SetStatusWeight(cd *flaggerv1.Canary, val int) error { cdCopy := cd.DeepCopy() - cdCopy.Status.Phase = state + cdCopy.Status.CanaryWeight = val cdCopy.Status.LastTransitionTime = metav1.Now() cd, err := c.flaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy) @@ -190,6 +190,23 @@ func (c *CanaryDeployer) SetState(cd *flaggerv1.Canary, state flaggerv1.CanaryPh return nil } +// SetStatusPhase updates the canary status phase +func (c *CanaryDeployer) SetStatusPhase(cd *flaggerv1.Canary, phase flaggerv1.CanaryPhase) error { + cdCopy := cd.DeepCopy() + cdCopy.Status.Phase = phase + cdCopy.Status.LastTransitionTime = metav1.Now() + + if phase != flaggerv1.CanaryProgressing { + cdCopy.Status.CanaryWeight = 0 + } + + cd, err := c.flaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy) + if err != nil { + return fmt.Errorf("canary %s.%s status update error %v", cdCopy.Name, cdCopy.Namespace, err) + } + return nil +} + // SyncStatus encodes the canary pod spec and updates the canary status func (c *CanaryDeployer) SyncStatus(cd *flaggerv1.Canary, status flaggerv1.CanaryStatus) error { dep, err := c.kubeClient.AppsV1().Deployments(cd.Namespace).Get(cd.Spec.TargetRef.Name, metav1.GetOptions{}) @@ -207,6 +224,7 @@ func (c *CanaryDeployer) SyncStatus(cd *flaggerv1.Canary, status flaggerv1.Canar cdCopy := cd.DeepCopy() cdCopy.Status.Phase = status.Phase + cdCopy.Status.CanaryWeight = status.CanaryWeight cdCopy.Status.FailedChecks = status.FailedChecks cdCopy.Status.LastAppliedSpec = base64.StdEncoding.EncodeToString(specJson) cdCopy.Status.LastTransitionTime = metav1.Now() diff --git a/pkg/controller/deployer_test.go b/pkg/controller/deployer_test.go index 83740761..69910efc 100644 --- a/pkg/controller/deployer_test.go +++ b/pkg/controller/deployer_test.go @@ -351,7 +351,7 @@ func TestCanaryDeployer_SetFailedChecks(t *testing.T) { t.Fatal(err.Error()) } - err = deployer.SetFailedChecks(canary, 1) + err = deployer.SetStatusFailedChecks(canary, 1) if err != nil { t.Fatal(err.Error()) } @@ -387,7 +387,7 @@ func TestCanaryDeployer_SetState(t *testing.T) { t.Fatal(err.Error()) } - err = deployer.SetState(canary, v1alpha3.CanaryProgressing) + err = deployer.SetStatusPhase(canary, v1alpha3.CanaryProgressing) if err != nil { t.Fatal(err.Error()) } diff --git a/pkg/controller/scheduler.go b/pkg/controller/scheduler.go index 3c4fc766..17cf2e96 100644 --- a/pkg/controller/scheduler.go +++ b/pkg/controller/scheduler.go @@ -173,7 +173,7 @@ func (c *Controller) advanceCanary(name string, namespace string) { } // mark canary as failed - if err := c.deployer.SyncStatus(cd, flaggerv1.CanaryStatus{Phase: flaggerv1.CanaryFailed}); err != nil { + if err := c.deployer.SyncStatus(cd, flaggerv1.CanaryStatus{Phase: flaggerv1.CanaryFailed, CanaryWeight: 0}); err != nil { c.logger.Errorf("%v", err) return } @@ -188,7 +188,7 @@ func (c *Controller) advanceCanary(name string, namespace string) { c.recordEventInfof(cd, "Starting canary deployment for %s.%s", cd.Name, cd.Namespace) } else { if ok := c.analyseCanary(cd); !ok { - if err := c.deployer.SetFailedChecks(cd, cd.Status.FailedChecks+1); err != nil { + if err := c.deployer.SetStatusFailedChecks(cd, cd.Status.FailedChecks+1); err != nil { c.recordEventWarningf(cd, "%v", err) return } @@ -212,6 +212,12 @@ func (c *Controller) advanceCanary(name string, namespace string) { return } + // update weight status + if err := c.deployer.SetStatusWeight(cd, canaryRoute.Weight); err != nil { + c.recordEventWarningf(cd, "%v", err) + return + } + c.recorder.SetWeight(cd, primaryRoute.Weight, canaryRoute.Weight) c.recordEventInfof(cd, "Advance %s.%s canary weight %v", cd.Name, cd.Namespace, canaryRoute.Weight) @@ -243,8 +249,8 @@ func (c *Controller) advanceCanary(name string, namespace string) { return } - // update status - if err := c.deployer.SetState(cd, flaggerv1.CanarySucceeded); err != nil { + // update status phase + if err := c.deployer.SetStatusPhase(cd, flaggerv1.CanarySucceeded); err != nil { c.recordEventWarningf(cd, "%v", err) return } From 1d31b5ed90b841f91102a1d28032c5cd5cab004e Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Thu, 17 Jan 2019 13:58:10 +0200 Subject: [PATCH 8/8] Add canary name and namespace to controller logs - zap key-value: canary=name.namespace --- pkg/controller/controller.go | 6 +++--- pkg/controller/deployer.go | 6 +++--- pkg/controller/router.go | 8 ++++---- pkg/controller/scheduler.go | 8 ++++---- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index fd494f17..b6990df9 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -248,17 +248,17 @@ func checkCustomResourceType(obj interface{}, logger *zap.SugaredLogger) (flagge } func (c *Controller) recordEventInfof(r *flaggerv1.Canary, template string, args ...interface{}) { - c.logger.Infof(template, args...) + c.logger.With("canary", fmt.Sprintf("%s.%s", r.Name, r.Namespace)).Infof(template, args...) c.eventRecorder.Event(r, corev1.EventTypeNormal, "Synced", fmt.Sprintf(template, args...)) } func (c *Controller) recordEventErrorf(r *flaggerv1.Canary, template string, args ...interface{}) { - c.logger.Errorf(template, args...) + c.logger.With("canary", fmt.Sprintf("%s.%s", r.Name, r.Namespace)).Errorf(template, args...) c.eventRecorder.Event(r, corev1.EventTypeWarning, "Synced", fmt.Sprintf(template, args...)) } func (c *Controller) recordEventWarningf(r *flaggerv1.Canary, template string, args ...interface{}) { - c.logger.Infof(template, args...) + c.logger.With("canary", fmt.Sprintf("%s.%s", r.Name, r.Namespace)).Infof(template, args...) c.eventRecorder.Event(r, corev1.EventTypeWarning, "Synced", fmt.Sprintf(template, args...)) } diff --git a/pkg/controller/deployer.go b/pkg/controller/deployer.go index e2fac6a8..a5d65a3e 100644 --- a/pkg/controller/deployer.go +++ b/pkg/controller/deployer.go @@ -266,7 +266,7 @@ func (c *CanaryDeployer) Sync(cd *flaggerv1.Canary) error { } if cd.Status.Phase == "" { - c.logger.Infof("Scaling down %s.%s", cd.Spec.TargetRef.Name, cd.Namespace) + c.logger.With("canary", fmt.Sprintf("%s.%s", cd.Name, cd.Namespace)).Infof("Scaling down %s.%s", cd.Spec.TargetRef.Name, cd.Namespace) if err := c.Scale(cd, 0); err != nil { return err } @@ -333,7 +333,7 @@ func (c *CanaryDeployer) createPrimaryDeployment(cd *flaggerv1.Canary) error { return err } - c.logger.Infof("Deployment %s.%s created", primaryDep.GetName(), cd.Namespace) + c.logger.With("canary", fmt.Sprintf("%s.%s", cd.Name, cd.Namespace)).Infof("Deployment %s.%s created", primaryDep.GetName(), cd.Namespace) } return nil @@ -381,7 +381,7 @@ func (c *CanaryDeployer) createPrimaryHpa(cd *flaggerv1.Canary) error { if err != nil { return err } - c.logger.Infof("HorizontalPodAutoscaler %s.%s created", primaryHpa.GetName(), cd.Namespace) + c.logger.With("canary", fmt.Sprintf("%s.%s", cd.Name, cd.Namespace)).Infof("HorizontalPodAutoscaler %s.%s created", primaryHpa.GetName(), cd.Namespace) } return nil diff --git a/pkg/controller/router.go b/pkg/controller/router.go index f01594b9..2c5e130f 100644 --- a/pkg/controller/router.go +++ b/pkg/controller/router.go @@ -79,7 +79,7 @@ func (c *CanaryRouter) createServices(cd *flaggerv1.Canary) error { if err != nil { return err } - c.logger.Infof("Service %s.%s created", canaryService.GetName(), cd.Namespace) + c.logger.With("canary", fmt.Sprintf("%s.%s", cd.Name, cd.Namespace)).Infof("Service %s.%s created", canaryService.GetName(), cd.Namespace) } canaryTestServiceName := fmt.Sprintf("%s-canary", cd.Spec.TargetRef.Name) @@ -118,7 +118,7 @@ func (c *CanaryRouter) createServices(cd *flaggerv1.Canary) error { if err != nil { return err } - c.logger.Infof("Service %s.%s created", canaryTestService.GetName(), cd.Namespace) + c.logger.With("canary", fmt.Sprintf("%s.%s", cd.Name, cd.Namespace)).Infof("Service %s.%s created", canaryTestService.GetName(), cd.Namespace) } primaryService, err := c.kubeClient.CoreV1().Services(cd.Namespace).Get(primaryName, metav1.GetOptions{}) @@ -157,7 +157,7 @@ func (c *CanaryRouter) createServices(cd *flaggerv1.Canary) error { return err } - c.logger.Infof("Service %s.%s created", primaryService.GetName(), cd.Namespace) + c.logger.With("canary", fmt.Sprintf("%s.%s", cd.Name, cd.Namespace)).Infof("Service %s.%s created", primaryService.GetName(), cd.Namespace) } return nil @@ -218,7 +218,7 @@ func (c *CanaryRouter) createVirtualService(cd *flaggerv1.Canary) error { if err != nil { return fmt.Errorf("VirtualService %s.%s create error %v", targetName, cd.Namespace, err) } - c.logger.Infof("VirtualService %s.%s created", virtualService.GetName(), cd.Namespace) + c.logger.With("canary", fmt.Sprintf("%s.%s", cd.Name, cd.Namespace)).Infof("VirtualService %s.%s created", virtualService.GetName(), cd.Namespace) } return nil diff --git a/pkg/controller/scheduler.go b/pkg/controller/scheduler.go index 17cf2e96..f6975df3 100644 --- a/pkg/controller/scheduler.go +++ b/pkg/controller/scheduler.go @@ -75,7 +75,7 @@ func (c *Controller) advanceCanary(name string, namespace string) { // check if the canary exists cd, err := c.flaggerClient.FlaggerV1alpha3().Canaries(namespace).Get(name, v1.GetOptions{}) if err != nil { - c.logger.Errorf("Canary %s.%s not found", name, namespace) + c.logger.With("canary", fmt.Sprintf("%s.%s", name, namespace)).Errorf("Canary %s.%s not found", name, namespace) return } @@ -174,7 +174,7 @@ func (c *Controller) advanceCanary(name string, namespace string) { // mark canary as failed if err := c.deployer.SyncStatus(cd, flaggerv1.CanaryStatus{Phase: flaggerv1.CanaryFailed, CanaryWeight: 0}); err != nil { - c.logger.Errorf("%v", err) + c.logger.With("canary", fmt.Sprintf("%s.%s", cd.Name, cd.Namespace)).Errorf("%v", err) return } @@ -268,7 +268,7 @@ func (c *Controller) checkCanaryStatus(cd *flaggerv1.Canary, deployer CanaryDepl if cd.Status.Phase == "" { if err := deployer.SyncStatus(cd, flaggerv1.CanaryStatus{Phase: flaggerv1.CanaryInitialized}); err != nil { - c.logger.Errorf("%v", err) + c.logger.With("canary", fmt.Sprintf("%s.%s", cd.Name, cd.Namespace)).Errorf("%v", err) return false } c.recorder.SetStatus(cd) @@ -287,7 +287,7 @@ func (c *Controller) checkCanaryStatus(cd *flaggerv1.Canary, deployer CanaryDepl return false } if err := deployer.SyncStatus(cd, flaggerv1.CanaryStatus{Phase: flaggerv1.CanaryProgressing}); err != nil { - c.logger.Errorf("%v", err) + c.logger.With("canary", fmt.Sprintf("%s.%s", cd.Name, cd.Namespace)).Errorf("%v", err) return false } c.recorder.SetStatus(cd)