From 906103daa5eb5df73e98e80e450e743766c192d1 Mon Sep 17 00:00:00 2001 From: Robert Kwolek Date: Thu, 20 Aug 2020 20:55:47 +0200 Subject: [PATCH] feat: weighted deployments --- artifacts/examples/linkerd-canary-steps.yaml | 53 +++++ artifacts/flagger/crd.yaml | 27 ++- charts/flagger/crds/crd.yaml | 27 ++- kustomize/base/flagger/crd.yaml | 27 ++- pkg/apis/flagger/v1beta1/canary.go | 16 +- .../flagger/v1beta1/zz_generated.deepcopy.go | 5 + pkg/controller/events.go | 8 + pkg/controller/scheduler.go | 77 ++++-- test/e2e-linkerd-steps-tests.sh | 220 ++++++++++++++++++ 9 files changed, 417 insertions(+), 43 deletions(-) create mode 100644 artifacts/examples/linkerd-canary-steps.yaml create mode 100644 test/e2e-linkerd-steps-tests.sh diff --git a/artifacts/examples/linkerd-canary-steps.yaml b/artifacts/examples/linkerd-canary-steps.yaml new file mode 100644 index 00000000..0e1e81b9 --- /dev/null +++ b/artifacts/examples/linkerd-canary-steps.yaml @@ -0,0 +1,53 @@ +apiVersion: flagger.app/v1beta1 +kind: Canary +metadata: + name: podinfo + namespace: test +spec: + provider: linkerd + progressDeadlineSeconds: 600 + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: podinfo + autoscalerRef: + apiVersion: autoscaling/v2beta1 + kind: HorizontalPodAutoscaler + name: podinfo + service: + name: podinfo + port: 80 + targetPort: 9898 + portName: http + portDiscovery: true + skipAnalysis: false + analysis: + interval: 15s + threshold: 10 + fullWeight: 1000 + maxWeight: 500 + stepWeights: [50, 100, 150, 200, 250, 300, 350, 400, 450, 500, 550] + metrics: + - name: request-success-rate + thresholdRange: + min: 99 + interval: 1m + - name: request-duration + thresholdRange: + max: 500 + interval: 30s + webhooks: + - name: conformance-test + type: pre-rollout + url: http://flagger-loadtester.test/ + timeout: 15s + metadata: + type: "bash" + cmd: "curl -sd 'test' http://podinfo-canary.test/token | grep token" + - name: load-test + type: rollout + url: http://flagger-loadtester.test/ + timeout: 5s + metadata: + type: cmd + cmd: "hey -z 1m -q 10 -c 2 http://podinfo-canary.test/" diff --git a/artifacts/flagger/crd.yaml b/artifacts/flagger/crd.yaml index a56b9e1a..0701ca20 100644 --- a/artifacts/flagger/crd.yaml +++ b/artifacts/flagger/crd.yaml @@ -48,10 +48,18 @@ spec: type: boolean JSONPath: .spec.analysis.mirror priority: 1 + - name: FullWeight + type: string + JSONPath: .spec.analysis.fullWeight + priority: 1 - name: StepWeight type: string JSONPath: .spec.analysis.stepWeight priority: 1 + - name: StepWeights + type: string + JSONPath: .spec.analysis.stepWeights + priority: 1 - name: MaxWeight type: string JSONPath: .spec.analysis.maxWeight @@ -547,6 +555,7 @@ spec: oneOf: - required: ["interval", "threshold", "iterations"] - required: ["interval", "threshold", "stepWeight"] + - required: ["interval", "threshold", "stepWeights"] properties: interval: description: Schedule interval for this canary @@ -558,20 +567,28 @@ spec: threshold: description: Max number of failed checks before rollback type: number + fullWeight: + description: Sum of traffic step weights for canary and primary + type: number maxWeight: - description: Max traffic percentage routed to canary + description: Max traffic weight routed to canary type: number stepWeight: - description: Incremental traffic percentage step for the analysis phase + description: Incremental traffic step weight for the analysis phase type: number + stepWeights: + description: Incremental traffic step weights for the analysis phase + type: array + items: + type: number stepWeightPromotion: - description: Incremental traffic percentage step for the promotion phase + description: Incremental traffic step weight for the promotion phase type: number mirror: description: Mirror traffic to canary type: boolean mirrorWeight: - description: Percentage of traffic to be mirrored + description: Weight of traffic to be mirrored type: number match: description: A/B testing match conditions @@ -702,7 +719,7 @@ spec: - Terminating - Terminated canaryWeight: - description: Traffic weight percentage routed to canary + description: Traffic weight routed to canary type: number failedChecks: description: Failed check count of the current canary analysis diff --git a/charts/flagger/crds/crd.yaml b/charts/flagger/crds/crd.yaml index a56b9e1a..0701ca20 100644 --- a/charts/flagger/crds/crd.yaml +++ b/charts/flagger/crds/crd.yaml @@ -48,10 +48,18 @@ spec: type: boolean JSONPath: .spec.analysis.mirror priority: 1 + - name: FullWeight + type: string + JSONPath: .spec.analysis.fullWeight + priority: 1 - name: StepWeight type: string JSONPath: .spec.analysis.stepWeight priority: 1 + - name: StepWeights + type: string + JSONPath: .spec.analysis.stepWeights + priority: 1 - name: MaxWeight type: string JSONPath: .spec.analysis.maxWeight @@ -547,6 +555,7 @@ spec: oneOf: - required: ["interval", "threshold", "iterations"] - required: ["interval", "threshold", "stepWeight"] + - required: ["interval", "threshold", "stepWeights"] properties: interval: description: Schedule interval for this canary @@ -558,20 +567,28 @@ spec: threshold: description: Max number of failed checks before rollback type: number + fullWeight: + description: Sum of traffic step weights for canary and primary + type: number maxWeight: - description: Max traffic percentage routed to canary + description: Max traffic weight routed to canary type: number stepWeight: - description: Incremental traffic percentage step for the analysis phase + description: Incremental traffic step weight for the analysis phase type: number + stepWeights: + description: Incremental traffic step weights for the analysis phase + type: array + items: + type: number stepWeightPromotion: - description: Incremental traffic percentage step for the promotion phase + description: Incremental traffic step weight for the promotion phase type: number mirror: description: Mirror traffic to canary type: boolean mirrorWeight: - description: Percentage of traffic to be mirrored + description: Weight of traffic to be mirrored type: number match: description: A/B testing match conditions @@ -702,7 +719,7 @@ spec: - Terminating - Terminated canaryWeight: - description: Traffic weight percentage routed to canary + description: Traffic weight routed to canary type: number failedChecks: description: Failed check count of the current canary analysis diff --git a/kustomize/base/flagger/crd.yaml b/kustomize/base/flagger/crd.yaml index a56b9e1a..0701ca20 100644 --- a/kustomize/base/flagger/crd.yaml +++ b/kustomize/base/flagger/crd.yaml @@ -48,10 +48,18 @@ spec: type: boolean JSONPath: .spec.analysis.mirror priority: 1 + - name: FullWeight + type: string + JSONPath: .spec.analysis.fullWeight + priority: 1 - name: StepWeight type: string JSONPath: .spec.analysis.stepWeight priority: 1 + - name: StepWeights + type: string + JSONPath: .spec.analysis.stepWeights + priority: 1 - name: MaxWeight type: string JSONPath: .spec.analysis.maxWeight @@ -547,6 +555,7 @@ spec: oneOf: - required: ["interval", "threshold", "iterations"] - required: ["interval", "threshold", "stepWeight"] + - required: ["interval", "threshold", "stepWeights"] properties: interval: description: Schedule interval for this canary @@ -558,20 +567,28 @@ spec: threshold: description: Max number of failed checks before rollback type: number + fullWeight: + description: Sum of traffic step weights for canary and primary + type: number maxWeight: - description: Max traffic percentage routed to canary + description: Max traffic weight routed to canary type: number stepWeight: - description: Incremental traffic percentage step for the analysis phase + description: Incremental traffic step weight for the analysis phase type: number + stepWeights: + description: Incremental traffic step weights for the analysis phase + type: array + items: + type: number stepWeightPromotion: - description: Incremental traffic percentage step for the promotion phase + description: Incremental traffic step weight for the promotion phase type: number mirror: description: Mirror traffic to canary type: boolean mirrorWeight: - description: Percentage of traffic to be mirrored + description: Weight of traffic to be mirrored type: number match: description: A/B testing match conditions @@ -702,7 +719,7 @@ spec: - Terminating - Terminated canaryWeight: - description: Traffic weight percentage routed to canary + description: Traffic weight routed to canary type: number failedChecks: description: Failed check count of the current canary analysis diff --git a/pkg/apis/flagger/v1beta1/canary.go b/pkg/apis/flagger/v1beta1/canary.go index 3451e796..78e405ab 100644 --- a/pkg/apis/flagger/v1beta1/canary.go +++ b/pkg/apis/flagger/v1beta1/canary.go @@ -195,19 +195,27 @@ type CanaryAnalysis struct { // +optional Mirror bool `json:"mirror,omitempty"` - // Percentage of the traffic to be mirrored in the range of [0, 100]. + // Sum of weights of canary and primary. If not set default 100 is used. + // +optional + FullWeight int `json:"fullWeight,omitempty"` + + // Weight of the traffic to be mirrored in the range of [0, FullWeight]. // +optional MirrorWeight int `json:"mirrorWeight,omitempty"` - // Max traffic percentage routed to canary + // Max traffic weight routed to canary // +optional MaxWeight int `json:"maxWeight,omitempty"` - // Incremental traffic percentage step for analysis phase + // Incremental traffic weight step for analysis phase // +optional StepWeight int `json:"stepWeight,omitempty"` - // Incremental traffic percentage step for promotion phase + // Incremental traffic weight steps for analysis phase + // +optional + StepWeights []int `json:"stepWeights,omitempty"` + + // Incremental traffic weight step for promotion phase // +optional StepWeightPromotion int `json:"stepWeightPromotion,omitempty"` diff --git a/pkg/apis/flagger/v1beta1/zz_generated.deepcopy.go b/pkg/apis/flagger/v1beta1/zz_generated.deepcopy.go index 31ec068e..5a878808 100644 --- a/pkg/apis/flagger/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/flagger/v1beta1/zz_generated.deepcopy.go @@ -197,6 +197,11 @@ func (in *CanaryAlert) DeepCopy() *CanaryAlert { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CanaryAnalysis) DeepCopyInto(out *CanaryAnalysis) { *out = *in + if in.StepWeights != nil { + in, out := &in.StepWeights, &out.StepWeights + *out = make([]int, len(*in)) + copy(*out, *in) + } if in.Alerts != nil { in, out := &in.Alerts, &out.Alerts *out = make([]CanaryAlert, len(*in)) diff --git a/pkg/controller/events.go b/pkg/controller/events.go index d7a449e1..adbdbc6e 100644 --- a/pkg/controller/events.go +++ b/pkg/controller/events.go @@ -3,6 +3,7 @@ package controller import ( "context" "fmt" + "strings" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -172,6 +173,13 @@ func alertMetadata(canary *flaggerv1.Canary) []notifier.Field { canary.GetAnalysis().StepWeight, canary.GetAnalysis().MaxWeight), }) + } else if len(canary.GetAnalysis().StepWeights) > 0 { + fields = append(fields, notifier.Field{ + Name: "Traffic routing", + Value: fmt.Sprintf("Weight steps: %s max: %v", + strings.Trim(strings.Join(strings.Fields(fmt.Sprint(canary.GetAnalysis().StepWeights)), ","), "[]"), + canary.GetAnalysis().MaxWeight), + }) } else if len(canary.GetAnalysis().Match) > 0 { fields = append(fields, notifier.Field{ Name: "Traffic routing", diff --git a/pkg/controller/scheduler.go b/pkg/controller/scheduler.go index 11fac989..57d7507d 100644 --- a/pkg/controller/scheduler.go +++ b/pkg/controller/scheduler.go @@ -14,6 +14,33 @@ import ( "github.com/weaveworks/flagger/pkg/router" ) +func (c *Controller) fullWeight(canary *flaggerv1.Canary) int { + if canary.GetAnalysis().FullWeight > 0 { + return canary.GetAnalysis().FullWeight + } + // set max weight default value to 100% + return 100 +} + +func (c *Controller) nextStepWeight(canary *flaggerv1.Canary, canaryWeight int) int { + var stepWeightsLen = len(canary.GetAnalysis().StepWeights) + if canary.GetAnalysis().StepWeight > 0 || stepWeightsLen == 0 { + return canary.GetAnalysis().StepWeight + } + + if canaryWeight == 0 { + return canary.GetAnalysis().StepWeights[0] + } + + for i := 0; i < stepWeightsLen-1; i++ { + if canary.GetAnalysis().StepWeights[i] == canaryWeight { + return canary.GetAnalysis().StepWeights[i+1] - canaryWeight + } + } + + return c.fullWeight(canary) - canaryWeight +} + // scheduleCanaries synchronises the canary map with the jobs map, // for new canaries new jobs are created and started // for the removed canaries the jobs are stopped and deleted @@ -173,8 +200,7 @@ func (c *Controller) advanceCanary(name string, namespace string) { return } - // set max weight default value to 100% - maxWeight := 100 + maxWeight := c.fullWeight(cd) if cd.GetAnalysis().MaxWeight > 0 { maxWeight = cd.GetAnalysis().MaxWeight } @@ -207,7 +233,7 @@ func (c *Controller) advanceCanary(name string, namespace string) { cd.Spec.TargetRef.Name, cd.Namespace) // route all traffic back to primary - primaryWeight = 100 + primaryWeight = c.fullWeight(cd) canaryWeight = 0 if err := meshRouter.SetRoutes(cd, primaryWeight, canaryWeight, false); err != nil { c.recordEventWarningf(cd, "%v", err) @@ -343,7 +369,7 @@ func (c *Controller) advanceCanary(name string, namespace string) { } // strategy: Canary progressive traffic increase - if cd.GetAnalysis().StepWeight > 0 { + if c.nextStepWeight(cd, canaryWeight) > 0 { c.runCanary(cd, canaryController, meshRouter, mirrored, canaryWeight, primaryWeight, maxWeight) } @@ -362,22 +388,22 @@ func (c *Controller) runPromotionTrafficShift(canary *flaggerv1.Canary, canaryCo // route all traffic to primary in one go when promotion step wight is not set if canary.Spec.Analysis.StepWeightPromotion == 0 { c.recordEventInfof(canary, "Routing all traffic to primary") - if err := meshRouter.SetRoutes(canary, 100, 0, false); err != nil { + if err := meshRouter.SetRoutes(canary, c.fullWeight(canary), 0, false); err != nil { c.recordEventWarningf(canary, "%v", err) return } - c.recorder.SetWeight(canary, 100, 0) + c.recorder.SetWeight(canary, c.fullWeight(canary), 0) if err := canaryController.SetStatusPhase(canary, flaggerv1.CanaryPhaseFinalising); err != nil { c.recordEventWarningf(canary, "%v", err) } return } - // increment the primary traffic weight until it reaches 100% + // increment the primary traffic weight until it reaches 100%/full weight if canaryWeight > 0 { primaryWeight += canary.GetAnalysis().StepWeightPromotion - if primaryWeight > 100 { - primaryWeight = 100 + if primaryWeight > c.fullWeight(canary) { + primaryWeight = c.fullWeight(canary) } canaryWeight -= canary.GetAnalysis().StepWeightPromotion if canaryWeight < 0 { @@ -391,7 +417,7 @@ func (c *Controller) runPromotionTrafficShift(canary *flaggerv1.Canary, canaryCo c.recordEventInfof(canary, "Advance %s.%s primary weight %v", canary.Name, canary.Namespace, primaryWeight) // finalize promotion - if primaryWeight == 100 { + if primaryWeight == c.fullWeight(canary) { if err := canaryController.SetStatusPhase(canary, flaggerv1.CanaryPhaseFinalising); err != nil { c.recordEventWarningf(canary, "%v", err) } @@ -415,27 +441,30 @@ func (c *Controller) runCanary(canary *flaggerv1.Canary, canaryController canary // If in "mirror" mode, do one step of mirroring before shifting traffic to canary. // When mirroring, all requests go to primary and canary, but only responses from // primary go back to the user. + + var nextStepWeight int + nextStepWeight = c.nextStepWeight(canary, canaryWeight) if canary.GetAnalysis().Mirror && canaryWeight == 0 { if !mirrored { mirrored = true - primaryWeight = 100 + primaryWeight = c.fullWeight(canary) canaryWeight = 0 } else { mirrored = false - primaryWeight = 100 - canary.GetAnalysis().StepWeight - canaryWeight = canary.GetAnalysis().StepWeight + primaryWeight = c.fullWeight(canary) - nextStepWeight + canaryWeight = nextStepWeight } c.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)). Infof("Running mirror step %d/%d/%t", primaryWeight, canaryWeight, mirrored) } else { - primaryWeight -= canary.GetAnalysis().StepWeight + primaryWeight -= nextStepWeight if primaryWeight < 0 { primaryWeight = 0 } - canaryWeight += canary.GetAnalysis().StepWeight - if canaryWeight > 100 { - canaryWeight = 100 + canaryWeight += nextStepWeight + if canaryWeight > c.fullWeight(canary) { + canaryWeight = c.fullWeight(canary) } } @@ -483,11 +512,11 @@ func (c *Controller) runAB(canary *flaggerv1.Canary, canaryController canary.Con // route traffic to canary and increment iterations if canary.GetAnalysis().Iterations > canary.Status.Iterations { - if err := meshRouter.SetRoutes(canary, 0, 100, false); err != nil { + if err := meshRouter.SetRoutes(canary, 0, c.fullWeight(canary), false); err != nil { c.recordEventWarningf(canary, "%v", err) return } - c.recorder.SetWeight(canary, 0, 100) + c.recorder.SetWeight(canary, 0, c.fullWeight(canary)) if err := canaryController.SetStatusIterations(canary, canary.Status.Iterations+1); err != nil { c.recordEventWarningf(canary, "%v", err) @@ -529,7 +558,7 @@ func (c *Controller) runBlueGreen(canary *flaggerv1.Canary, canaryController can // If in "mirror" mode, mirror requests during the entire B/G canary test if provider != "kubernetes" && canary.GetAnalysis().Mirror && !mirrored { - if err := meshRouter.SetRoutes(canary, 100, 0, true); err != nil { + if err := meshRouter.SetRoutes(canary, c.fullWeight(canary), 0, true); err != nil { c.recordEventWarningf(canary, "%v", err) } c.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)). @@ -557,11 +586,11 @@ func (c *Controller) runBlueGreen(canary *flaggerv1.Canary, canaryController can } else { c.recordEventInfof(canary, "Routing all traffic to canary") } - if err := meshRouter.SetRoutes(canary, 0, 100, false); err != nil { + if err := meshRouter.SetRoutes(canary, 0, c.fullWeight(canary), false); err != nil { c.recordEventWarningf(canary, "%v", err) return } - c.recorder.SetWeight(canary, 0, 100) + c.recorder.SetWeight(canary, 0, c.fullWeight(canary)) } // increment iterations @@ -622,7 +651,7 @@ func (c *Controller) shouldSkipAnalysis(canary *flaggerv1.Canary, canaryControll } // route all traffic to primary - primaryWeight := 100 + primaryWeight := c.fullWeight(canary) canaryWeight := 0 if err := meshRouter.SetRoutes(canary, primaryWeight, canaryWeight, false); err != nil { c.recordEventWarningf(canary, "%v", err) @@ -756,7 +785,7 @@ func (c *Controller) rollback(canary *flaggerv1.Canary, canaryController canary. } // route all traffic back to primary - primaryWeight := 100 + primaryWeight := c.fullWeight(canary) canaryWeight := 0 if err := meshRouter.SetRoutes(canary, primaryWeight, canaryWeight, false); err != nil { c.recordEventWarningf(canary, "%v", err) diff --git a/test/e2e-linkerd-steps-tests.sh b/test/e2e-linkerd-steps-tests.sh new file mode 100644 index 00000000..4837cdc8 --- /dev/null +++ b/test/e2e-linkerd-steps-tests.sh @@ -0,0 +1,220 @@ +#!/usr/bin/env bash + +# This script runs Linkerd e2e tests for Canary initialization, analysis and promotion + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) + +echo '>>> Creating test namespace' +kubectl create namespace test +kubectl annotate namespace test linkerd.io/inject=enabled + +echo '>>> Installing the load tester' +kubectl apply -k ${REPO_ROOT}/kustomize/tester +kubectl -n test rollout status deployment/flagger-loadtester + +echo '>>> Initialising canary' +kubectl apply -f ${REPO_ROOT}/test/e2e-workload.yaml + +cat <>> Waiting for primary to be ready' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n linkerd logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary initialization test passed' + +echo '>>> Triggering canary deployment' +kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1 + +echo '>>> Waiting for canary promotion' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test describe deployment/podinfo-primary | grep '3.1.1' && ok=true || ok=false + sleep 10 + kubectl -n linkerd logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n linkerd logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '>>> Waiting for canary finalization' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n linkerd logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary promotion test passed' + +cat <>> Triggering canary deployment' +kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.2 + +echo '>>> Waiting for canary rollback' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Failed' && ok=true || ok=false + sleep 10 + kubectl -n linkerd logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n linkerd logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary rollback test passed' \ No newline at end of file