From b2ca0c4c16fd4c361863b835c0d54e203b39e233 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Mon, 29 Jul 2019 13:52:11 +0300 Subject: [PATCH] Implement finalising state Set the canary status to finalising after routing the traffic back to the primary. Run one final loop before scaling the canary to zero so that the canary has a chance to process all inflight requests. --- pkg/canary/status.go | 3 ++ pkg/controller/scheduler.go | 69 ++++++++++++++++++++----------------- 2 files changed, 40 insertions(+), 32 deletions(-) diff --git a/pkg/canary/status.go b/pkg/canary/status.go index a744d437..e68d911e 100644 --- a/pkg/canary/status.go +++ b/pkg/canary/status.go @@ -211,6 +211,9 @@ func (c *Deployer) MakeStatusConditions(canaryStatus flaggerv1.CanaryStatus, case flaggerv1.CanaryPhaseProgressing: status = corev1.ConditionUnknown message = "New revision detected, starting canary analysis." + case flaggerv1.CanaryPhaseFinalising: + status = corev1.ConditionUnknown + message = "Canary analysis completed, routing all traffic to primary." case flaggerv1.CanaryPhaseSucceeded: status = corev1.ConditionTrue message = "Canary analysis completed successfully, promotion finished." diff --git a/pkg/controller/scheduler.go b/pkg/controller/scheduler.go index cf9a595b..9454524f 100644 --- a/pkg/controller/scheduler.go +++ b/pkg/controller/scheduler.go @@ -214,6 +214,26 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh return } + // scale canary to zero if analysis has succeeded + if cd.Status.Phase == flaggerv1.CanaryPhaseFinalising { + if err := c.deployer.Scale(cd, 0); err != nil { + c.recordEventWarningf(cd, "%v", err) + return + } + + // set status to succeeded + if err := c.deployer.SetStatusPhase(cd, flaggerv1.CanaryPhaseSucceeded); err != nil { + c.recordEventWarningf(cd, "%v", err) + return + } + c.recorder.SetStatus(cd, flaggerv1.CanaryPhaseSucceeded) + c.runPostRolloutHooks(cd, flaggerv1.CanaryPhaseSucceeded) + c.recordEventInfof(cd, "Promotion completed! Scaling down %s.%s", cd.Spec.TargetRef.Name, cd.Namespace) + c.sendNotification(cd, "Canary analysis completed successfully, promotion finished.", + false, false) + return + } + // check if the number of failed checks reached the threshold if cd.Status.Phase == flaggerv1.CanaryPhaseProgressing && (!retriable || cd.Status.FailedChecks >= cd.Spec.CanaryAnalysis.Threshold) { @@ -319,31 +339,23 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh return } - // shutdown canary + // route all traffic to primary if cd.Spec.CanaryAnalysis.Iterations < cd.Status.Iterations { - // route all traffic to the primary - if err := meshRouter.SetRoutes(cd, 100, 0); err != nil { - c.recordEventWarningf(cd, "%v", err) - return - } - c.recorder.SetWeight(cd, 100, 0) - c.recordEventInfof(cd, "Promotion completed! Scaling down %s.%s", cd.Spec.TargetRef.Name, cd.Namespace) - - // canary scale to zero - if err := c.deployer.Scale(cd, 0); err != nil { + primaryWeight = 100 + canaryWeight = 0 + if err := meshRouter.SetRoutes(cd, primaryWeight, canaryWeight); err != nil { c.recordEventWarningf(cd, "%v", err) return } + c.recorder.SetWeight(cd, primaryWeight, canaryWeight) // update status phase - if err := c.deployer.SetStatusPhase(cd, flaggerv1.CanaryPhaseSucceeded); err != nil { + if err := c.deployer.SetStatusPhase(cd, flaggerv1.CanaryPhaseFinalising); err != nil { c.recordEventWarningf(cd, "%v", err) return } - c.recorder.SetStatus(cd, flaggerv1.CanaryPhaseSucceeded) - c.runPostRolloutHooks(cd, flaggerv1.CanaryPhaseSucceeded) - c.sendNotification(cd, "Canary analysis completed successfully, promotion finished.", - false, false) + + c.recordEventInfof(cd, "Routing all traffic to primary") return } @@ -385,32 +397,23 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh } } } else { - // route all traffic back to primary + // route all traffic to primary primaryWeight = 100 canaryWeight = 0 if err := meshRouter.SetRoutes(cd, primaryWeight, canaryWeight); err != nil { c.recordEventWarningf(cd, "%v", err) return } - c.recorder.SetWeight(cd, primaryWeight, canaryWeight) - c.recordEventInfof(cd, "Promotion completed! Scaling down %s.%s", cd.Spec.TargetRef.Name, cd.Namespace) - - // shutdown canary - if err := c.deployer.Scale(cd, 0); err != nil { - c.recordEventWarningf(cd, "%v", err) - return - } // update status phase - if err := c.deployer.SetStatusPhase(cd, flaggerv1.CanaryPhaseSucceeded); err != nil { + if err := c.deployer.SetStatusPhase(cd, flaggerv1.CanaryPhaseFinalising); err != nil { c.recordEventWarningf(cd, "%v", err) return } - c.recorder.SetStatus(cd, flaggerv1.CanaryPhaseSucceeded) - c.runPostRolloutHooks(cd, flaggerv1.CanaryPhaseSucceeded) - c.sendNotification(cd, "Canary analysis completed successfully, promotion finished.", - false, false) + + c.recordEventInfof(cd, "Routing all traffic to primary") + return } } @@ -462,7 +465,8 @@ func (c *Controller) shouldAdvance(cd *flaggerv1.Canary) (bool, error) { if cd.Status.LastAppliedSpec == "" || cd.Status.Phase == flaggerv1.CanaryPhaseInitializing || cd.Status.Phase == flaggerv1.CanaryPhaseProgressing || - cd.Status.Phase == flaggerv1.CanaryPhaseWaiting { + cd.Status.Phase == flaggerv1.CanaryPhaseWaiting || + cd.Status.Phase == flaggerv1.CanaryPhaseFinalising { return true, nil } @@ -485,7 +489,8 @@ func (c *Controller) shouldAdvance(cd *flaggerv1.Canary) (bool, error) { func (c *Controller) checkCanaryStatus(cd *flaggerv1.Canary, shouldAdvance bool) bool { c.recorder.SetStatus(cd, cd.Status.Phase) - if cd.Status.Phase == flaggerv1.CanaryPhaseProgressing { + if cd.Status.Phase == flaggerv1.CanaryPhaseProgressing || + cd.Status.Phase == flaggerv1.CanaryPhaseFinalising { return true }