From 9d856a4f96179be8c8f39803b983e5bfe32d5d66 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Sat, 21 Sep 2019 21:21:33 +0300 Subject: [PATCH 1/4] Implement B/G for service mesh providers Blue/Green steps: - scale up green - run conformance tests on green - run load tests and metric checks on green - route traffic to green - promote green spec over blue - wait for blue rollout - route traffic to blue --- pkg/controller/scheduler.go | 72 +++++++++++++++++++++++++++++++++++-- 1 file changed, 70 insertions(+), 2 deletions(-) diff --git a/pkg/controller/scheduler.go b/pkg/controller/scheduler.go index 6d7a35c7..7a184c74 100644 --- a/pkg/controller/scheduler.go +++ b/pkg/controller/scheduler.go @@ -283,7 +283,7 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh // check if the canary success rate is above the threshold // skip check if no traffic is routed to canary - if canaryWeight == 0 { + if canaryWeight == 0 && cd.Status.Iterations == 0 { c.recordEventInfof(cd, "Starting canary analysis for %s.%s", cd.Spec.TargetRef.Name, cd.Namespace) // run pre-rollout web hooks @@ -305,7 +305,7 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh } // canary fix routing: A/B testing - if len(cd.Spec.CanaryAnalysis.Match) > 0 || cd.Spec.CanaryAnalysis.Iterations > 0 { + if len(cd.Spec.CanaryAnalysis.Match) > 0 && cd.Spec.CanaryAnalysis.Iterations > 0 { // route traffic to canary and increment iterations if cd.Spec.CanaryAnalysis.Iterations > cd.Status.Iterations { if err := meshRouter.SetRoutes(cd, 0, 100); err != nil { @@ -362,6 +362,74 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh return } + // canary fix routing: B/G + if cd.Spec.CanaryAnalysis.Iterations > 0 { + // increment iterations + if cd.Spec.CanaryAnalysis.Iterations > cd.Status.Iterations { + if err := c.deployer.SetStatusIterations(cd, cd.Status.Iterations+1); err != nil { + c.recordEventWarningf(cd, "%v", err) + return + } + c.recordEventInfof(cd, "Advance %s.%s canary iteration %v/%v", + cd.Name, cd.Namespace, cd.Status.Iterations+1, cd.Spec.CanaryAnalysis.Iterations) + return + } + + // route all traffic to canary - max iterations reached + if cd.Spec.CanaryAnalysis.Iterations == cd.Status.Iterations { + c.recordEventInfof(cd, "Routing all traffic to canary") + if err := meshRouter.SetRoutes(cd, 0, 100); err != nil { + c.recordEventWarningf(cd, "%v", err) + return + } + c.recorder.SetWeight(cd, 0, 100) + + // increment iterations + if err := c.deployer.SetStatusIterations(cd, cd.Status.Iterations+1); err != nil { + c.recordEventWarningf(cd, "%v", err) + return + } + return + } + + // promote canary - max iterations reached + if cd.Spec.CanaryAnalysis.Iterations+1 == cd.Status.Iterations { + c.recordEventInfof(cd, "Copying %s.%s template spec to %s.%s", + cd.Spec.TargetRef.Name, cd.Namespace, primaryName, cd.Namespace) + if err := c.deployer.Promote(cd); err != nil { + c.recordEventWarningf(cd, "%v", err) + return + } + + // increment iterations + if err := c.deployer.SetStatusIterations(cd, cd.Status.Iterations+1); err != nil { + c.recordEventWarningf(cd, "%v", err) + return + } + return + } + + // route all traffic to primary + if cd.Spec.CanaryAnalysis.Iterations < cd.Status.Iterations { + if err := meshRouter.SetRoutes(cd, 100, 0); err != nil { + c.recordEventWarningf(cd, "%v", err) + return + } + c.recorder.SetWeight(cd, 100, 0) + + // update status phase + if err := c.deployer.SetStatusPhase(cd, flaggerv1.CanaryPhaseFinalising); err != nil { + c.recordEventWarningf(cd, "%v", err) + return + } + + c.recordEventInfof(cd, "Routing all traffic to primary") + return + } + + return + } + // canary incremental traffic weight if canaryWeight < maxWeight { primaryWeight -= cd.Spec.CanaryAnalysis.StepWeight From a6d86f2e817437313007db44d45db24757c2d13f Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Sun, 22 Sep 2019 00:48:42 +0300 Subject: [PATCH 2/4] Skip mesh routers for B/G when provider is kubernetes --- pkg/controller/scheduler.go | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/pkg/controller/scheduler.go b/pkg/controller/scheduler.go index 7a184c74..230eb9d0 100644 --- a/pkg/controller/scheduler.go +++ b/pkg/controller/scheduler.go @@ -377,12 +377,14 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh // route all traffic to canary - max iterations reached if cd.Spec.CanaryAnalysis.Iterations == cd.Status.Iterations { - c.recordEventInfof(cd, "Routing all traffic to canary") - if err := meshRouter.SetRoutes(cd, 0, 100); err != nil { - c.recordEventWarningf(cd, "%v", err) - return + if provider != "kubernetes" { + c.recordEventInfof(cd, "Routing all traffic to canary") + if err := meshRouter.SetRoutes(cd, 0, 100); err != nil { + c.recordEventWarningf(cd, "%v", err) + return + } + c.recorder.SetWeight(cd, 0, 100) } - c.recorder.SetWeight(cd, 0, 100) // increment iterations if err := c.deployer.SetStatusIterations(cd, cd.Status.Iterations+1); err != nil { @@ -411,11 +413,14 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh // route all traffic to primary if cd.Spec.CanaryAnalysis.Iterations < cd.Status.Iterations { - if err := meshRouter.SetRoutes(cd, 100, 0); err != nil { - c.recordEventWarningf(cd, "%v", err) - return + if provider != "kubernetes" { + c.recordEventInfof(cd, "Routing all traffic to primary") + if err := meshRouter.SetRoutes(cd, 100, 0); err != nil { + c.recordEventWarningf(cd, "%v", err) + return + } + c.recorder.SetWeight(cd, 100, 0) } - c.recorder.SetWeight(cd, 100, 0) // update status phase if err := c.deployer.SetStatusPhase(cd, flaggerv1.CanaryPhaseFinalising); err != nil { @@ -423,7 +428,6 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh return } - c.recordEventInfof(cd, "Routing all traffic to primary") return } From d908355ab3ef5ffadbef43acfe882e4e32dfc03b Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Sun, 22 Sep 2019 09:29:39 +0300 Subject: [PATCH 3/4] Add Blue/Green e2e tests --- test/e2e-tests.sh | 64 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 61 insertions(+), 3 deletions(-) diff --git a/test/e2e-tests.sh b/test/e2e-tests.sh index 3d654c15..5fb75fe1 100755 --- a/test/e2e-tests.sh +++ b/test/e2e-tests.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# This script runs e2e tests for Canary initialization, analysis and promotion +# This script runs e2e tests for Canary, B/G and A/B initialization, analysis and promotion # Prerequisites: Kubernetes Kind, Helm and Istio set -o errexit @@ -130,6 +130,64 @@ if [[ "$1" = "canary" ]]; then exit 0 fi +cat <>> Triggering B/G deployment' +kubectl -n test set image deployment/podinfo podinfod=quay.io/stefanprodan/podinfo:1.4.2 + +echo '>>> Waiting for B/G promotion' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test describe deployment/podinfo-primary | grep '1.4.2' && ok=true || ok=false + sleep 10 + kubectl -n istio-system logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n test describe deployment/podinfo + kubectl -n test describe deployment/podinfo-primary + kubectl -n istio-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ B/G promotion test passed' + cat <>> Triggering A/B testing' -kubectl -n test set image deployment/podinfo podinfod=quay.io/stefanprodan/podinfo:1.4.2 +kubectl -n test set image deployment/podinfo podinfod=quay.io/stefanprodan/podinfo:1.4.3 echo '>>> Waiting for A/B testing promotion' retries=50 count=0 ok=false until ${ok}; do - kubectl -n test describe deployment/podinfo-primary | grep '1.4.2' && ok=true || ok=false + kubectl -n test describe deployment/podinfo-primary | grep '1.4.3' && ok=true || ok=false sleep 10 kubectl -n istio-system logs deployment/flagger --tail 1 count=$(($count + 1)) From d19a070fafe82167232868761911e5013eeaebb5 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Sun, 22 Sep 2019 11:45:07 +0300 Subject: [PATCH 4/4] Add canary status checks to Istio e2e tests --- test/e2e-tests.sh | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/test/e2e-tests.sh b/test/e2e-tests.sh index 5fb75fe1..f0e04da7 100755 --- a/test/e2e-tests.sh +++ b/test/e2e-tests.sh @@ -124,6 +124,21 @@ until ${ok}; do fi done +echo '>>> Waiting for canary finalization' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n istio-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + echo '✔ Canary promotion test passed' if [[ "$1" = "canary" ]]; then @@ -186,6 +201,21 @@ until ${ok}; do fi done +echo '>>> Waiting for B/G finalization' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n istio-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + echo '✔ B/G promotion test passed' cat <