diff --git a/artifacts/examples/linkerd-canary-steps.yaml b/artifacts/examples/linkerd-canary-steps.yaml new file mode 100644 index 00000000..1e1a09af --- /dev/null +++ b/artifacts/examples/linkerd-canary-steps.yaml @@ -0,0 +1,51 @@ +apiVersion: flagger.app/v1beta1 +kind: Canary +metadata: + name: podinfo + namespace: test +spec: + provider: linkerd + progressDeadlineSeconds: 600 + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: podinfo + autoscalerRef: + apiVersion: autoscaling/v2beta1 + kind: HorizontalPodAutoscaler + name: podinfo + service: + name: podinfo + port: 80 + targetPort: 9898 + portName: http + portDiscovery: true + skipAnalysis: false + analysis: + interval: 15s + threshold: 10 + stepWeights: [5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55] + metrics: + - name: request-success-rate + thresholdRange: + min: 99 + interval: 1m + - name: request-duration + thresholdRange: + max: 500 + interval: 30s + webhooks: + - name: conformance-test + type: pre-rollout + url: http://flagger-loadtester.test/ + timeout: 15s + metadata: + type: "bash" + cmd: "curl -sd 'test' http://podinfo-canary.test/token | grep token" + - name: load-test + type: rollout + url: http://flagger-loadtester.test/ + timeout: 5s + metadata: + type: cmd + cmd: "hey -z 1m -q 10 -c 2 http://podinfo-canary.test/" diff --git a/artifacts/flagger/crd.yaml b/artifacts/flagger/crd.yaml index 6fd69629..1eebd4be 100644 --- a/artifacts/flagger/crd.yaml +++ b/artifacts/flagger/crd.yaml @@ -52,6 +52,10 @@ spec: type: string JSONPath: .spec.analysis.stepWeight priority: 1 + - name: StepWeights + type: string + JSONPath: .spec.analysis.stepWeights + priority: 1 - name: MaxWeight type: string JSONPath: .spec.analysis.maxWeight @@ -550,6 +554,7 @@ spec: oneOf: - required: ["interval", "threshold", "iterations"] - required: ["interval", "threshold", "stepWeight"] + - required: ["interval", "threshold", "stepWeights"] properties: interval: description: Schedule interval for this canary @@ -562,19 +567,24 @@ spec: description: Max number of failed checks before rollback type: number maxWeight: - description: Max traffic percentage routed to canary + description: Max traffic weight routed to canary type: number stepWeight: - description: Incremental traffic percentage step for the analysis phase + description: Incremental traffic step weight for the analysis phase type: number + stepWeights: + description: Incremental traffic step weights for the analysis phase + type: array + items: + type: number stepWeightPromotion: - description: Incremental traffic percentage step for the promotion phase + description: Incremental traffic step weight for the promotion phase type: number mirror: description: Mirror traffic to canary type: boolean mirrorWeight: - description: Percentage of traffic to be mirrored + description: Weight of traffic to be mirrored type: number match: description: A/B testing match conditions @@ -705,7 +715,7 @@ spec: - Terminating - Terminated canaryWeight: - description: Traffic weight percentage routed to canary + description: Traffic weight routed to canary type: number failedChecks: description: Failed check count of the current canary analysis diff --git a/charts/flagger/crds/crd.yaml b/charts/flagger/crds/crd.yaml index 6fd69629..1eebd4be 100644 --- a/charts/flagger/crds/crd.yaml +++ b/charts/flagger/crds/crd.yaml @@ -52,6 +52,10 @@ spec: type: string JSONPath: .spec.analysis.stepWeight priority: 1 + - name: StepWeights + type: string + JSONPath: .spec.analysis.stepWeights + priority: 1 - name: MaxWeight type: string JSONPath: .spec.analysis.maxWeight @@ -550,6 +554,7 @@ spec: oneOf: - required: ["interval", "threshold", "iterations"] - required: ["interval", "threshold", "stepWeight"] + - required: ["interval", "threshold", "stepWeights"] properties: interval: description: Schedule interval for this canary @@ -562,19 +567,24 @@ spec: description: Max number of failed checks before rollback type: number maxWeight: - description: Max traffic percentage routed to canary + description: Max traffic weight routed to canary type: number stepWeight: - description: Incremental traffic percentage step for the analysis phase + description: Incremental traffic step weight for the analysis phase type: number + stepWeights: + description: Incremental traffic step weights for the analysis phase + type: array + items: + type: number stepWeightPromotion: - description: Incremental traffic percentage step for the promotion phase + description: Incremental traffic step weight for the promotion phase type: number mirror: description: Mirror traffic to canary type: boolean mirrorWeight: - description: Percentage of traffic to be mirrored + description: Weight of traffic to be mirrored type: number match: description: A/B testing match conditions @@ -705,7 +715,7 @@ spec: - Terminating - Terminated canaryWeight: - description: Traffic weight percentage routed to canary + description: Traffic weight routed to canary type: number failedChecks: description: Failed check count of the current canary analysis diff --git a/docs/gitbook/SUMMARY.md b/docs/gitbook/SUMMARY.md index 240d1801..84301984 100644 --- a/docs/gitbook/SUMMARY.md +++ b/docs/gitbook/SUMMARY.md @@ -33,6 +33,7 @@ * [Canary analysis with Prometheus Operator](tutorials/prometheus-operator.md) * [Canaries with Helm charts and GitOps](tutorials/canary-helm-gitops.md) * [Zero downtime deployments](tutorials/zero-downtime-deployments.md) +* [Rollout Weights](tutorials/rollout-weights.md) ## Dev diff --git a/docs/gitbook/usage/deployment-strategies.md b/docs/gitbook/usage/deployment-strategies.md index 80af7348..fdbadf1d 100644 --- a/docs/gitbook/usage/deployment-strategies.md +++ b/docs/gitbook/usage/deployment-strategies.md @@ -116,6 +116,43 @@ Gated canary promotion stages: * send notification with the canary analysis result * wait for the canary deployment to be updated and start over +#### Rollout Weights + +By default Flagger uses linear weight values for the promotion, with the start value, the step and the maximum weight value in 0 to 100 range. + +Example: +```yaml +canary: + analysis: + promotion: + maxWeight: 50 + stepWeight: 20 +``` +This configuration performs analysis starting from 20, increasing by 20 until weight goes above 50. +We would have steps (canary weight : primary weight): +* 20 (20 : 80) +* 40 (40 : 60) +* 60 (60 : 40) +* promotion + +In order to enable non-linear promotion a new parameter was introduced: +* `stepWeights` - determines the ordered array of weights, which shall be used during canary promotion. + +Example: +```yaml +canary: + analysis: + promotion: + stepWeights: [1, 2, 10, 80] +``` +This configuration performs analysis starting from 1, going through `stepWeights` values till 80. +We would have steps (canary weight : primary weight): +* 1 (1 : 99) +* 2 (2 : 98) +* 10 (10 : 90) +* 80 (20 : 60) +* promotion + ### A/B Testing For frontend applications that require session affinity you should use HTTP headers or cookies match conditions diff --git a/kustomize/base/flagger/crd.yaml b/kustomize/base/flagger/crd.yaml index 6fd69629..1eebd4be 100644 --- a/kustomize/base/flagger/crd.yaml +++ b/kustomize/base/flagger/crd.yaml @@ -52,6 +52,10 @@ spec: type: string JSONPath: .spec.analysis.stepWeight priority: 1 + - name: StepWeights + type: string + JSONPath: .spec.analysis.stepWeights + priority: 1 - name: MaxWeight type: string JSONPath: .spec.analysis.maxWeight @@ -550,6 +554,7 @@ spec: oneOf: - required: ["interval", "threshold", "iterations"] - required: ["interval", "threshold", "stepWeight"] + - required: ["interval", "threshold", "stepWeights"] properties: interval: description: Schedule interval for this canary @@ -562,19 +567,24 @@ spec: description: Max number of failed checks before rollback type: number maxWeight: - description: Max traffic percentage routed to canary + description: Max traffic weight routed to canary type: number stepWeight: - description: Incremental traffic percentage step for the analysis phase + description: Incremental traffic step weight for the analysis phase type: number + stepWeights: + description: Incremental traffic step weights for the analysis phase + type: array + items: + type: number stepWeightPromotion: - description: Incremental traffic percentage step for the promotion phase + description: Incremental traffic step weight for the promotion phase type: number mirror: description: Mirror traffic to canary type: boolean mirrorWeight: - description: Percentage of traffic to be mirrored + description: Weight of traffic to be mirrored type: number match: description: A/B testing match conditions @@ -705,7 +715,7 @@ spec: - Terminating - Terminated canaryWeight: - description: Traffic weight percentage routed to canary + description: Traffic weight routed to canary type: number failedChecks: description: Failed check count of the current canary analysis diff --git a/pkg/apis/flagger/v1beta1/canary.go b/pkg/apis/flagger/v1beta1/canary.go index 12cd6052..38b90761 100644 --- a/pkg/apis/flagger/v1beta1/canary.go +++ b/pkg/apis/flagger/v1beta1/canary.go @@ -201,19 +201,23 @@ type CanaryAnalysis struct { // +optional Mirror bool `json:"mirror,omitempty"` - // Percentage of the traffic to be mirrored in the range of [0, 100]. + // Weight of the traffic to be mirrored in the range of [0, 100]. // +optional MirrorWeight int `json:"mirrorWeight,omitempty"` - // Max traffic percentage routed to canary + // Max traffic weight routed to canary // +optional MaxWeight int `json:"maxWeight,omitempty"` - // Incremental traffic percentage step for analysis phase + // Incremental traffic weight step for analysis phase // +optional StepWeight int `json:"stepWeight,omitempty"` - // Incremental traffic percentage step for promotion phase + // Incremental traffic weight steps for analysis phase + // +optional + StepWeights []int `json:"stepWeights,omitempty"` + + // Incremental traffic weight step for promotion phase // +optional StepWeightPromotion int `json:"stepWeightPromotion,omitempty"` diff --git a/pkg/apis/flagger/v1beta1/zz_generated.deepcopy.go b/pkg/apis/flagger/v1beta1/zz_generated.deepcopy.go index 31ec068e..5a878808 100644 --- a/pkg/apis/flagger/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/flagger/v1beta1/zz_generated.deepcopy.go @@ -197,6 +197,11 @@ func (in *CanaryAlert) DeepCopy() *CanaryAlert { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CanaryAnalysis) DeepCopyInto(out *CanaryAnalysis) { *out = *in + if in.StepWeights != nil { + in, out := &in.StepWeights, &out.StepWeights + *out = make([]int, len(*in)) + copy(*out, *in) + } if in.Alerts != nil { in, out := &in.Alerts, &out.Alerts *out = make([]CanaryAlert, len(*in)) diff --git a/pkg/controller/events.go b/pkg/controller/events.go index d7a449e1..adbdbc6e 100644 --- a/pkg/controller/events.go +++ b/pkg/controller/events.go @@ -3,6 +3,7 @@ package controller import ( "context" "fmt" + "strings" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -172,6 +173,13 @@ func alertMetadata(canary *flaggerv1.Canary) []notifier.Field { canary.GetAnalysis().StepWeight, canary.GetAnalysis().MaxWeight), }) + } else if len(canary.GetAnalysis().StepWeights) > 0 { + fields = append(fields, notifier.Field{ + Name: "Traffic routing", + Value: fmt.Sprintf("Weight steps: %s max: %v", + strings.Trim(strings.Join(strings.Fields(fmt.Sprint(canary.GetAnalysis().StepWeights)), ","), "[]"), + canary.GetAnalysis().MaxWeight), + }) } else if len(canary.GetAnalysis().Match) > 0 { fields = append(fields, notifier.Field{ Name: "Traffic routing", diff --git a/pkg/controller/scheduler.go b/pkg/controller/scheduler.go index 2de76f36..3008b8c2 100644 --- a/pkg/controller/scheduler.go +++ b/pkg/controller/scheduler.go @@ -14,6 +14,62 @@ import ( "github.com/weaveworks/flagger/pkg/router" ) +func (c *Controller) min(a int, b int) int { + if a < b { + return a + } + return b +} + +func (c *Controller) maxWeight(canary *flaggerv1.Canary) int { + var stepWeightsLen = len(canary.GetAnalysis().StepWeights) + if stepWeightsLen > 0 { + return c.min(c.totalWeight(canary), canary.GetAnalysis().StepWeights[stepWeightsLen-1]) + } + if canary.GetAnalysis().MaxWeight > 0 { + return canary.GetAnalysis().MaxWeight + } + // set max weight default value to total weight + return c.totalWeight(canary) +} + +func (c *Controller) totalWeight(canary *flaggerv1.Canary) int { + // set total weight default value to 100% + return 100 +} + +func (c *Controller) nextStepWeight(canary *flaggerv1.Canary, canaryWeight int) int { + var stepWeightsLen = len(canary.GetAnalysis().StepWeights) + if canary.GetAnalysis().StepWeight > 0 || stepWeightsLen == 0 { + return canary.GetAnalysis().StepWeight + } + + totalWeight := c.totalWeight(canary) + maxStep := totalWeight - canaryWeight + + // If maxStep is zero we need to promote, so any non zero step weight will move the canary to promotion. + // This is the same use case as the last step via StepWeight. + if maxStep == 0 { + return 1 + } + + // return min of maxStep and the calculated step to avoid going above totalWeight + + // initial step + if canaryWeight == 0 { + return c.min(maxStep, canary.GetAnalysis().StepWeights[0]) + } + + // find the current step and return the difference in weight + for i := 0; i < stepWeightsLen-1; i++ { + if canary.GetAnalysis().StepWeights[i] == canaryWeight { + return c.min(maxStep, canary.GetAnalysis().StepWeights[i+1]-canaryWeight) + } + } + + return maxStep +} + // scheduleCanaries synchronises the canary map with the jobs map, // for new canaries new jobs are created and started // for the removed canaries the jobs are stopped and deleted @@ -173,11 +229,7 @@ func (c *Controller) advanceCanary(name string, namespace string) { return } - // set max weight default value to 100% - maxWeight := 100 - if cd.GetAnalysis().MaxWeight > 0 { - maxWeight = cd.GetAnalysis().MaxWeight - } + maxWeight := c.maxWeight(cd) // check primary status if !cd.SkipAnalysis() { @@ -207,7 +259,7 @@ func (c *Controller) advanceCanary(name string, namespace string) { cd.Spec.TargetRef.Name, cd.Namespace) // route all traffic back to primary - primaryWeight = 100 + primaryWeight = c.totalWeight(cd) canaryWeight = 0 if err := meshRouter.SetRoutes(cd, primaryWeight, canaryWeight, false); err != nil { c.recordEventWarningf(cd, "%v", err) @@ -343,7 +395,7 @@ func (c *Controller) advanceCanary(name string, namespace string) { } // strategy: Canary progressive traffic increase - if cd.GetAnalysis().StepWeight > 0 { + if c.nextStepWeight(cd, canaryWeight) > 0 { c.runCanary(cd, canaryController, meshRouter, mirrored, canaryWeight, primaryWeight, maxWeight) } @@ -362,22 +414,22 @@ func (c *Controller) runPromotionTrafficShift(canary *flaggerv1.Canary, canaryCo // route all traffic to primary in one go when promotion step wight is not set if canary.Spec.Analysis.StepWeightPromotion == 0 { c.recordEventInfof(canary, "Routing all traffic to primary") - if err := meshRouter.SetRoutes(canary, 100, 0, false); err != nil { + if err := meshRouter.SetRoutes(canary, c.totalWeight(canary), 0, false); err != nil { c.recordEventWarningf(canary, "%v", err) return } - c.recorder.SetWeight(canary, 100, 0) + c.recorder.SetWeight(canary, c.totalWeight(canary), 0) if err := canaryController.SetStatusPhase(canary, flaggerv1.CanaryPhaseFinalising); err != nil { c.recordEventWarningf(canary, "%v", err) } return } - // increment the primary traffic weight until it reaches 100% + // increment the primary traffic weight until it reaches total weight if canaryWeight > 0 { primaryWeight += canary.GetAnalysis().StepWeightPromotion - if primaryWeight > 100 { - primaryWeight = 100 + if primaryWeight > c.totalWeight(canary) { + primaryWeight = c.totalWeight(canary) } canaryWeight -= canary.GetAnalysis().StepWeightPromotion if canaryWeight < 0 { @@ -391,7 +443,7 @@ func (c *Controller) runPromotionTrafficShift(canary *flaggerv1.Canary, canaryCo c.recordEventInfof(canary, "Advance %s.%s primary weight %v", canary.Name, canary.Namespace, primaryWeight) // finalize promotion - if primaryWeight == 100 { + if primaryWeight == c.totalWeight(canary) { if err := canaryController.SetStatusPhase(canary, flaggerv1.CanaryPhaseFinalising); err != nil { c.recordEventWarningf(canary, "%v", err) } @@ -415,27 +467,30 @@ func (c *Controller) runCanary(canary *flaggerv1.Canary, canaryController canary // If in "mirror" mode, do one step of mirroring before shifting traffic to canary. // When mirroring, all requests go to primary and canary, but only responses from // primary go back to the user. + + var nextStepWeight int + nextStepWeight = c.nextStepWeight(canary, canaryWeight) if canary.GetAnalysis().Mirror && canaryWeight == 0 { if !mirrored { mirrored = true - primaryWeight = 100 + primaryWeight = c.totalWeight(canary) canaryWeight = 0 } else { mirrored = false - primaryWeight = 100 - canary.GetAnalysis().StepWeight - canaryWeight = canary.GetAnalysis().StepWeight + primaryWeight = c.totalWeight(canary) - nextStepWeight + canaryWeight = nextStepWeight } c.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)). Infof("Running mirror step %d/%d/%t", primaryWeight, canaryWeight, mirrored) } else { - primaryWeight -= canary.GetAnalysis().StepWeight + primaryWeight -= nextStepWeight if primaryWeight < 0 { primaryWeight = 0 } - canaryWeight += canary.GetAnalysis().StepWeight - if canaryWeight > 100 { - canaryWeight = 100 + canaryWeight += nextStepWeight + if canaryWeight > c.totalWeight(canary) { + canaryWeight = c.totalWeight(canary) } } @@ -483,11 +538,11 @@ func (c *Controller) runAB(canary *flaggerv1.Canary, canaryController canary.Con // route traffic to canary and increment iterations if canary.GetAnalysis().Iterations > canary.Status.Iterations { - if err := meshRouter.SetRoutes(canary, 0, 100, false); err != nil { + if err := meshRouter.SetRoutes(canary, 0, c.totalWeight(canary), false); err != nil { c.recordEventWarningf(canary, "%v", err) return } - c.recorder.SetWeight(canary, 0, 100) + c.recorder.SetWeight(canary, 0, c.totalWeight(canary)) if err := canaryController.SetStatusIterations(canary, canary.Status.Iterations+1); err != nil { c.recordEventWarningf(canary, "%v", err) @@ -529,7 +584,7 @@ func (c *Controller) runBlueGreen(canary *flaggerv1.Canary, canaryController can // If in "mirror" mode, mirror requests during the entire B/G canary test if provider != "kubernetes" && canary.GetAnalysis().Mirror && !mirrored { - if err := meshRouter.SetRoutes(canary, 100, 0, true); err != nil { + if err := meshRouter.SetRoutes(canary, c.totalWeight(canary), 0, true); err != nil { c.recordEventWarningf(canary, "%v", err) } c.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)). @@ -557,11 +612,11 @@ func (c *Controller) runBlueGreen(canary *flaggerv1.Canary, canaryController can } else { c.recordEventInfof(canary, "Routing all traffic to canary") } - if err := meshRouter.SetRoutes(canary, 0, 100, false); err != nil { + if err := meshRouter.SetRoutes(canary, 0, c.totalWeight(canary), false); err != nil { c.recordEventWarningf(canary, "%v", err) return } - c.recorder.SetWeight(canary, 0, 100) + c.recorder.SetWeight(canary, 0, c.totalWeight(canary)) } // increment iterations @@ -631,7 +686,7 @@ func (c *Controller) shouldSkipAnalysis(canary *flaggerv1.Canary, canaryControll } // route all traffic to primary - primaryWeight := 100 + primaryWeight := c.totalWeight(canary) canaryWeight := 0 if err := meshRouter.SetRoutes(canary, primaryWeight, canaryWeight, false); err != nil { c.recordEventWarningf(canary, "%v", err) @@ -765,7 +820,7 @@ func (c *Controller) rollback(canary *flaggerv1.Canary, canaryController canary. } // route all traffic back to primary - primaryWeight := 100 + primaryWeight := c.totalWeight(canary) canaryWeight := 0 if err := meshRouter.SetRoutes(canary, primaryWeight, canaryWeight, false); err != nil { c.recordEventWarningf(canary, "%v", err) diff --git a/pkg/controller/scheduler_svc_test.go b/pkg/controller/scheduler_svc_test.go index 77b92ae5..ad1f608a 100644 --- a/pkg/controller/scheduler_svc_test.go +++ b/pkg/controller/scheduler_svc_test.go @@ -11,8 +11,28 @@ import ( flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1beta1" ) +const ( + totalWeight = 100 +) + func TestScheduler_ServicePromotion(t *testing.T) { - mocks := newDeploymentFixture(newTestServiceCanary()) + testServicePromotion(t, newTestServiceCanary(), []int{totalWeight, 80, 60, 40}) +} + +func TestScheduler_ServicePromotionMaxWeight(t *testing.T) { + testServicePromotion(t, newTestServiceCanaryMaxWeight(), []int{totalWeight, 50, 0}) +} + +func TestScheduler_ServicePromotionWithWeightsHappyCase(t *testing.T) { + testServicePromotion(t, newTestServiceCanaryWithWeightsHappyCase(), []int{totalWeight, 99, 98, 90, 20}) +} + +func TestScheduler_ServicePromotionWithWeightsOverflow(t *testing.T) { + testServicePromotion(t, newTestServiceCanaryWithWeightsOverflow(), []int{totalWeight, 99, 98, 90, 0}) +} + +func testServicePromotion(t *testing.T, canary *flaggerv1.Canary, expectedPrimaryWeigths []int) { + mocks := newDeploymentFixture(canary) // init mocks.ctrl.advanceCanary("podinfo", "default") @@ -27,19 +47,15 @@ func TestScheduler_ServicePromotion(t *testing.T) { _, err = mocks.kubeClient.CoreV1().Services("default").Update(context.TODO(), svc2, metav1.UpdateOptions{}) require.NoError(t, err) - // detect service spec changes - mocks.ctrl.advanceCanary("podinfo", "default") - - primaryWeight, canaryWeight, mirrored, err := mocks.router.GetRoutes(mocks.canary) - require.NoError(t, err) - - primaryWeight = 60 - canaryWeight = 40 - err = mocks.router.SetRoutes(mocks.canary, primaryWeight, canaryWeight, mirrored) - require.NoError(t, err) - - // advance - mocks.ctrl.advanceCanary("podinfo", "default") + for _, expectedPrimaryWeigth := range expectedPrimaryWeigths { + mocks.ctrl.advanceCanary("podinfo", "default") + expectedCanaryWeight := totalWeight - expectedPrimaryWeigth + primaryWeight, canaryWeight, mirrored, err := mocks.router.GetRoutes(mocks.canary) + require.NoError(t, err) + assert.Equal(t, expectedPrimaryWeigth, primaryWeight) + assert.Equal(t, expectedCanaryWeight, canaryWeight) + assert.False(t, mirrored) + } // check progressing status c, err = mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{}) @@ -57,9 +73,9 @@ func TestScheduler_ServicePromotion(t *testing.T) { // finalise mocks.ctrl.advanceCanary("podinfo", "default") - primaryWeight, canaryWeight, mirrored, err = mocks.router.GetRoutes(mocks.canary) + primaryWeight, canaryWeight, mirrored, err := mocks.router.GetRoutes(mocks.canary) require.NoError(t, err) - assert.Equal(t, 100, primaryWeight) + assert.Equal(t, totalWeight, primaryWeight) assert.Equal(t, 0, canaryWeight) assert.False(t, mirrored) @@ -101,7 +117,7 @@ func newTestServiceCanary() *flaggerv1.Canary { }, Analysis: &flaggerv1.CanaryAnalysis{ Threshold: 10, - StepWeight: 10, + StepWeight: 20, MaxWeight: 50, Metrics: []flaggerv1.CanaryMetric{ { @@ -120,3 +136,115 @@ func newTestServiceCanary() *flaggerv1.Canary { } return cd } + +func newTestServiceCanaryMaxWeight() *flaggerv1.Canary { + cd := &flaggerv1.Canary{ + TypeMeta: metav1.TypeMeta{APIVersion: flaggerv1.SchemeGroupVersion.String()}, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "podinfo", + }, + Spec: flaggerv1.CanarySpec{ + TargetRef: flaggerv1.CrossNamespaceObjectReference{ + Name: "podinfo", + APIVersion: "core/v1", + Kind: "Service", + }, + Service: flaggerv1.CanaryService{ + Port: 9898, + }, + Analysis: &flaggerv1.CanaryAnalysis{ + Threshold: 10, + StepWeight: 50, + MaxWeight: totalWeight, + Metrics: []flaggerv1.CanaryMetric{ + { + Name: "request-success-rate", + Threshold: 99, + Interval: "1m", + }, + { + Name: "request-duration", + Threshold: 500000, + Interval: "1m", + }, + }, + }, + }, + } + return cd +} + +func newTestServiceCanaryWithWeightsHappyCase() *flaggerv1.Canary { + cd := &flaggerv1.Canary{ + TypeMeta: metav1.TypeMeta{APIVersion: flaggerv1.SchemeGroupVersion.String()}, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "podinfo", + }, + Spec: flaggerv1.CanarySpec{ + TargetRef: flaggerv1.CrossNamespaceObjectReference{ + Name: "podinfo", + APIVersion: "core/v1", + Kind: "Service", + }, + Service: flaggerv1.CanaryService{ + Port: 9898, + }, + Analysis: &flaggerv1.CanaryAnalysis{ + Threshold: 10, + StepWeights: []int{1, 2, 10, 80}, + Metrics: []flaggerv1.CanaryMetric{ + { + Name: "request-success-rate", + Threshold: 99, + Interval: "1m", + }, + { + Name: "request-duration", + Threshold: 500000, + Interval: "1m", + }, + }, + }, + }, + } + return cd +} + +func newTestServiceCanaryWithWeightsOverflow() *flaggerv1.Canary { + cd := &flaggerv1.Canary{ + TypeMeta: metav1.TypeMeta{APIVersion: flaggerv1.SchemeGroupVersion.String()}, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "podinfo", + }, + Spec: flaggerv1.CanarySpec{ + TargetRef: flaggerv1.CrossNamespaceObjectReference{ + Name: "podinfo", + APIVersion: "core/v1", + Kind: "Service", + }, + Service: flaggerv1.CanaryService{ + Port: 9898, + }, + Analysis: &flaggerv1.CanaryAnalysis{ + Threshold: 10, + StepWeights: []int{1, 2, 10, totalWeight + 100}, + Metrics: []flaggerv1.CanaryMetric{ + { + Name: "request-success-rate", + Threshold: 99, + Interval: "1m", + }, + { + Name: "request-duration", + Threshold: 500000, + Interval: "1m", + }, + }, + }, + }, + } + return cd +} diff --git a/test/e2e-linkerd-steps-tests.sh b/test/e2e-linkerd-steps-tests.sh new file mode 100644 index 00000000..b8f6c53a --- /dev/null +++ b/test/e2e-linkerd-steps-tests.sh @@ -0,0 +1,218 @@ +#!/usr/bin/env bash + +# This script runs Linkerd e2e tests for Canary initialization, analysis and promotion + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) + +echo '>>> Creating test namespace' +kubectl create namespace test +kubectl annotate namespace test linkerd.io/inject=enabled + +echo '>>> Installing the load tester' +kubectl apply -k ${REPO_ROOT}/kustomize/tester +kubectl -n test rollout status deployment/flagger-loadtester + +echo '>>> Initialising canary' +kubectl apply -f ${REPO_ROOT}/test/e2e-workload.yaml + +cat <>> Waiting for primary to be ready' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n linkerd logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary initialization test passed' + +echo '>>> Triggering canary deployment' +kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1 + +echo '>>> Waiting for canary promotion' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test describe deployment/podinfo-primary | grep '3.1.1' && ok=true || ok=false + sleep 10 + kubectl -n linkerd logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n linkerd logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '>>> Waiting for canary finalization' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n linkerd logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary promotion test passed' + +cat <>> Triggering canary deployment' +kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.2 + +echo '>>> Waiting for canary rollback' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Failed' && ok=true || ok=false + sleep 10 + kubectl -n linkerd logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n linkerd logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary rollback test passed' \ No newline at end of file