From f3b444ab4939eaceec8404f168e3875ad380d5dc Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Fri, 15 May 2020 15:45:32 +0300 Subject: [PATCH 1/4] Add promotion step weight to Canary CRD --- artifacts/flagger/crd.yaml | 5 ++++- charts/flagger/crds/crd.yaml | 5 ++++- kustomize/base/flagger/crd.yaml | 5 ++++- pkg/apis/flagger/v1beta1/canary.go | 6 +++++- 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/artifacts/flagger/crd.yaml b/artifacts/flagger/crd.yaml index 68cabd96..d17a3589 100644 --- a/artifacts/flagger/crd.yaml +++ b/artifacts/flagger/crd.yaml @@ -562,7 +562,10 @@ spec: description: Max traffic percentage routed to canary type: number stepWeight: - description: Incremental traffic percentage step + description: Incremental traffic percentage step for the analysis phase + type: number + stepWeightPromotion: + description: Incremental traffic percentage step for the promotion phase type: number mirror: description: Mirror traffic to canary diff --git a/charts/flagger/crds/crd.yaml b/charts/flagger/crds/crd.yaml index 68cabd96..d17a3589 100644 --- a/charts/flagger/crds/crd.yaml +++ b/charts/flagger/crds/crd.yaml @@ -562,7 +562,10 @@ spec: description: Max traffic percentage routed to canary type: number stepWeight: - description: Incremental traffic percentage step + description: Incremental traffic percentage step for the analysis phase + type: number + stepWeightPromotion: + description: Incremental traffic percentage step for the promotion phase type: number mirror: description: Mirror traffic to canary diff --git a/kustomize/base/flagger/crd.yaml b/kustomize/base/flagger/crd.yaml index 68cabd96..d17a3589 100644 --- a/kustomize/base/flagger/crd.yaml +++ b/kustomize/base/flagger/crd.yaml @@ -562,7 +562,10 @@ spec: description: Max traffic percentage routed to canary type: number stepWeight: - description: Incremental traffic percentage step + description: Incremental traffic percentage step for the analysis phase + type: number + stepWeightPromotion: + description: Incremental traffic percentage step for the promotion phase type: number mirror: description: Mirror traffic to canary diff --git a/pkg/apis/flagger/v1beta1/canary.go b/pkg/apis/flagger/v1beta1/canary.go index 6106d143..3451e796 100644 --- a/pkg/apis/flagger/v1beta1/canary.go +++ b/pkg/apis/flagger/v1beta1/canary.go @@ -203,10 +203,14 @@ type CanaryAnalysis struct { // +optional MaxWeight int `json:"maxWeight,omitempty"` - // Incremental traffic percentage step + // Incremental traffic percentage step for analysis phase // +optional StepWeight int `json:"stepWeight,omitempty"` + // Incremental traffic percentage step for promotion phase + // +optional + StepWeightPromotion int `json:"stepWeightPromotion,omitempty"` + // Max number of failed checks before the canary is terminated Threshold int `json:"threshold"` From eaa5b14be6a598f9c00b35a2911e07b2dd059d23 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Fri, 15 May 2020 15:46:25 +0300 Subject: [PATCH 2/4] Implement progressive traffic shifting on promotion --- pkg/controller/scheduler.go | 79 +++++++++++++++++++++++++++++-------- 1 file changed, 62 insertions(+), 17 deletions(-) diff --git a/pkg/controller/scheduler.go b/pkg/controller/scheduler.go index e636c07d..83d49b98 100644 --- a/pkg/controller/scheduler.go +++ b/pkg/controller/scheduler.go @@ -251,23 +251,9 @@ func (c *Controller) advanceCanary(name string, namespace string) { } } - // route all traffic to primary if analysis has succeeded + // route traffic back to primary if analysis has succeeded if cd.Status.Phase == flaggerv1.CanaryPhasePromoting { - if provider != "kubernetes" { - c.recordEventInfof(cd, "Routing all traffic to primary") - if err := meshRouter.SetRoutes(cd, 100, 0, false); err != nil { - c.recordEventWarningf(cd, "%v", err) - return - } - c.recorder.SetWeight(cd, 100, 0) - } - - // update status phase - if err := canaryController.SetStatusPhase(cd, flaggerv1.CanaryPhaseFinalising); err != nil { - c.recordEventWarningf(cd, "%v", err) - return - } - + c.runPromotionTrafficShift(cd, canaryController, meshRouter, provider, canaryWeight, primaryWeight) return } @@ -332,7 +318,7 @@ func (c *Controller) advanceCanary(name string, namespace string) { } // use blue/green strategy for kubernetes provider - if provider == "kubernetes" { + if provider == flaggerv1.KubernetesProvider { if len(cd.GetAnalysis().Match) > 0 { c.recordEventWarningf(cd, "A/B testing is not supported when using the kubernetes provider") cd.GetAnalysis().Match = nil @@ -363,6 +349,65 @@ func (c *Controller) advanceCanary(name string, namespace string) { } +func (c *Controller) runPromotionTrafficShift(canary *flaggerv1.Canary, canaryController canary.Controller, + meshRouter router.Interface, provider string, canaryWeight int, primaryWeight int) { + // finalize promotion since no traffic shifting is possible for Kubernetes CNI + if provider == flaggerv1.KubernetesProvider { + if err := canaryController.SetStatusPhase(canary, flaggerv1.CanaryPhaseFinalising); err != nil { + c.recordEventWarningf(canary, "%v", err) + return + } + return + } + + // route all traffic to primary in one go when promotion step wight is not set + if canary.Spec.Analysis.StepWeightPromotion == 0 { + c.recordEventInfof(canary, "Routing all traffic to primary") + if err := meshRouter.SetRoutes(canary, 100, 0, false); err != nil { + c.recordEventWarningf(canary, "%v", err) + return + } + c.recorder.SetWeight(canary, 100, 0) + if err := canaryController.SetStatusPhase(canary, flaggerv1.CanaryPhaseFinalising); err != nil { + c.recordEventWarningf(canary, "%v", err) + return + } + return + } + + // increment the primary traffic weight until it reaches 100% + if canaryWeight > 0 { + primaryWeight += canary.GetAnalysis().StepWeightPromotion + if primaryWeight > 100 { + primaryWeight = 100 + } + canaryWeight -= canary.GetAnalysis().StepWeightPromotion + if canaryWeight < 0 { + canaryWeight = 0 + } + if err := meshRouter.SetRoutes(canary, primaryWeight, canaryWeight, false); err != nil { + c.recordEventWarningf(canary, "%v", err) + return + } + c.recorder.SetWeight(canary, primaryWeight, canaryWeight) + c.recordEventInfof(canary, "Advance %s.%s primary weight %v", canary.Name, canary.Namespace, primaryWeight) + + // finalize promotion + if primaryWeight == 100 { + if err := canaryController.SetStatusPhase(canary, flaggerv1.CanaryPhaseFinalising); err != nil { + c.recordEventWarningf(canary, "%v", err) + } + } else { + if err := canaryController.SetStatusWeight(canary, canaryWeight); err != nil { + c.recordEventWarningf(canary, "%v", err) + } + } + } + + return + +} + func (c *Controller) runCanary(canary *flaggerv1.Canary, canaryController canary.Controller, meshRouter router.Interface, mirrored bool, canaryWeight int, primaryWeight int, maxWeight int) { primaryName := fmt.Sprintf("%s-primary", canary.Spec.TargetRef.Name) From 2e75dbb170eeafcef6879486e452801b6ed3cfdc Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Mon, 18 May 2020 10:44:23 +0300 Subject: [PATCH 3/4] Add progressive promotion test --- pkg/controller/scheduler.go | 2 -- pkg/controller/scheduler_deployment_test.go | 11 ++++++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/pkg/controller/scheduler.go b/pkg/controller/scheduler.go index 83d49b98..63b6e52c 100644 --- a/pkg/controller/scheduler.go +++ b/pkg/controller/scheduler.go @@ -355,7 +355,6 @@ func (c *Controller) runPromotionTrafficShift(canary *flaggerv1.Canary, canaryCo if provider == flaggerv1.KubernetesProvider { if err := canaryController.SetStatusPhase(canary, flaggerv1.CanaryPhaseFinalising); err != nil { c.recordEventWarningf(canary, "%v", err) - return } return } @@ -370,7 +369,6 @@ func (c *Controller) runPromotionTrafficShift(canary *flaggerv1.Canary, canaryCo c.recorder.SetWeight(canary, 100, 0) if err := canaryController.SetStatusPhase(canary, flaggerv1.CanaryPhaseFinalising); err != nil { c.recordEventWarningf(canary, "%v", err) - return } return } diff --git a/pkg/controller/scheduler_deployment_test.go b/pkg/controller/scheduler_deployment_test.go index 44ac59d7..f030dd5f 100644 --- a/pkg/controller/scheduler_deployment_test.go +++ b/pkg/controller/scheduler_deployment_test.go @@ -134,8 +134,9 @@ func TestScheduler_DeploymentSkipAnalysis(t *testing.T) { func TestScheduler_DeploymentAnalysisPhases(t *testing.T) { cd := newDeploymentTestCanary() cd.Spec.Analysis = &flaggerv1.CanaryAnalysis{ - Interval: "1m", - StepWeight: 100, + Interval: "1m", + StepWeight: 100, + StepWeightPromotion: 50, } mocks := newDeploymentFixture(cd) @@ -163,7 +164,11 @@ func TestScheduler_DeploymentAnalysisPhases(t *testing.T) { mocks.ctrl.advanceCanary("podinfo", "default") require.NoError(t, assertPhase(mocks.flaggerClient, "podinfo", flaggerv1.CanaryPhaseProgressing)) - // promoting + // start promotion + mocks.ctrl.advanceCanary("podinfo", "default") + require.NoError(t, assertPhase(mocks.flaggerClient, "podinfo", flaggerv1.CanaryPhasePromoting)) + + // end promotion mocks.ctrl.advanceCanary("podinfo", "default") require.NoError(t, assertPhase(mocks.flaggerClient, "podinfo", flaggerv1.CanaryPhasePromoting)) From be96a1147965d748a5e6c158fe186adf383df28d Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Mon, 18 May 2020 11:07:40 +0300 Subject: [PATCH 4/4] Add promotion step weight to docs --- docs/gitbook/usage/deployment-strategies.md | 9 ++++++++- docs/gitbook/usage/how-it-works.md | 3 +++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/gitbook/usage/deployment-strategies.md b/docs/gitbook/usage/deployment-strategies.md index 05cc4fd5..e6781e52 100644 --- a/docs/gitbook/usage/deployment-strategies.md +++ b/docs/gitbook/usage/deployment-strategies.md @@ -43,13 +43,16 @@ Spec: # canary increment step # percentage (0-100) stepWeight: 2 + # promotion increment step (default 100) + # percentage (0-100) + stepWeightPromotion: 100 # deploy straight to production without # the metrics and webhook checks skipAnalysis: false ``` The above analysis, if it succeeds, will run for 25 minutes while validating the HTTP metrics and webhooks every minute. -You can determine the minimum time that it takes to validate and promote a canary deployment using this formula: +You can determine the minimum time it takes to validate and promote a canary deployment using this formula: ``` interval * (maxWeight / stepWeight) @@ -61,6 +64,10 @@ And the time it takes for a canary to be rollback when the metrics or webhook ch interval * threshold ``` +When `stepWeightPromotion` is specified, the promotion phase happens in stages, +the traffic is routed back to the primary pods in a progressive manner, +the primary weight is increased until it reaches 100%. + In emergency cases, you may want to skip the analysis phase and ship changes directly to production. At any time you can set the `spec.skipAnalysis: true`. When skip analysis is enabled, Flagger checks if the canary deployment is healthy and diff --git a/docs/gitbook/usage/how-it-works.md b/docs/gitbook/usage/how-it-works.md index 5d215c07..4aa1719d 100644 --- a/docs/gitbook/usage/how-it-works.md +++ b/docs/gitbook/usage/how-it-works.md @@ -311,6 +311,9 @@ Spec: # canary increment step # percentage (0-100) stepWeight: + # promotion increment step + # percentage (0-100) + stepWeightPromotion: # total number of iterations # used for A/B Testing and Blue/Green iterations: