github/flagger
1
0
Fork 0
mirror of https://github.com/fluxcd/flagger.git synced 2026-04-15 06:57:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: weighted deployments

Browse Source
This commit is contained in:
Robert Kwolek
2020-08-20 20:56:10 +02:00
parent 8a5a0538fd
commit 906103daa5
9 changed files with 417 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
artifacts/examples/linkerd-canary-steps.yaml
+53
View File
@@ -0,0 +1,53 @@
apiVersion: flagger.app/v1beta1
kind: Canary
metadata:
name: podinfo
namespace: test
spec:
provider: linkerd
progressDeadlineSeconds: 600
targetRef:
apiVersion: apps/v1
kind: Deployment
name: podinfo
autoscalerRef:
apiVersion: autoscaling/v2beta1
kind: HorizontalPodAutoscaler
name: podinfo
service:
name: podinfo
port: 80
targetPort: 9898
portName: http
portDiscovery: true
skipAnalysis: false
analysis:
interval: 15s
threshold: 10
fullWeight: 1000
maxWeight: 500
stepWeights: [50, 100, 150, 200, 250, 300, 350, 400, 450, 500, 550]
metrics:
- name: request-success-rate
thresholdRange:
min: 99
interval: 1m
- name: request-duration
thresholdRange:
max: 500
interval: 30s
webhooks:
- name: conformance-test
type: pre-rollout
url: http://flagger-loadtester.test/
timeout: 15s
metadata:
type: "bash"
cmd: "curl -sd 'test' http://podinfo-canary.test/token | grep token"
- name: load-test
type: rollout
url: http://flagger-loadtester.test/
timeout: 5s
metadata:
type: cmd
cmd: "hey -z 1m -q 10 -c 2 http://podinfo-canary.test/"
artifacts/flagger/crd.yaml
+22 -5
View File
@@ -48,10 +48,18 @@ spec:
type: boolean
JSONPath: .spec.analysis.mirror
priority: 1
- name: FullWeight
type: string
JSONPath: .spec.analysis.fullWeight
priority: 1
- name: StepWeight
type: string
JSONPath: .spec.analysis.stepWeight
priority: 1
- name: StepWeights
type: string
JSONPath: .spec.analysis.stepWeights
priority: 1
- name: MaxWeight
type: string
JSONPath: .spec.analysis.maxWeight
@@ -547,6 +555,7 @@ spec:
oneOf:
- required: ["interval", "threshold", "iterations"]
- required: ["interval", "threshold", "stepWeight"]
- required: ["interval", "threshold", "stepWeights"]
properties:
interval:
description: Schedule interval for this canary
@@ -558,20 +567,28 @@ spec:
threshold:
description: Max number of failed checks before rollback
type: number
fullWeight:
description: Sum of traffic step weights for canary and primary
type: number
maxWeight:
description: Max traffic percentage routed to canary
description: Max traffic weight routed to canary
type: number
stepWeight:
description: Incremental traffic percentage step for the analysis phase
description: Incremental traffic step weight for the analysis phase
type: number
stepWeights:
description: Incremental traffic step weights for the analysis phase
type: array
items:
type: number
stepWeightPromotion:
description: Incremental traffic percentage step for the promotion phase
description: Incremental traffic step weight for the promotion phase
type: number
mirror:
description: Mirror traffic to canary
type: boolean
mirrorWeight:
description: Percentage of traffic to be mirrored
description: Weight of traffic to be mirrored
type: number
match:
description: A/B testing match conditions
@@ -702,7 +719,7 @@ spec:
- Terminating
- Terminated
canaryWeight:
description: Traffic weight percentage routed to canary
description: Traffic weight routed to canary
type: number
failedChecks:
description: Failed check count of the current canary analysis
charts/flagger/crds/crd.yaml
+22 -5
View File
@@ -48,10 +48,18 @@ spec:
type: boolean
JSONPath: .spec.analysis.mirror
priority: 1
- name: FullWeight
type: string
JSONPath: .spec.analysis.fullWeight
priority: 1
- name: StepWeight
type: string
JSONPath: .spec.analysis.stepWeight
priority: 1
- name: StepWeights
type: string
JSONPath: .spec.analysis.stepWeights
priority: 1
- name: MaxWeight
type: string
JSONPath: .spec.analysis.maxWeight
@@ -547,6 +555,7 @@ spec:
oneOf:
- required: ["interval", "threshold", "iterations"]
- required: ["interval", "threshold", "stepWeight"]
- required: ["interval", "threshold", "stepWeights"]
properties:
interval:
description: Schedule interval for this canary
@@ -558,20 +567,28 @@ spec:
threshold:
description: Max number of failed checks before rollback
type: number
fullWeight:
description: Sum of traffic step weights for canary and primary
type: number
maxWeight:
description: Max traffic percentage routed to canary
description: Max traffic weight routed to canary
type: number
stepWeight:
description: Incremental traffic percentage step for the analysis phase
description: Incremental traffic step weight for the analysis phase
type: number
stepWeights:
description: Incremental traffic step weights for the analysis phase
type: array
items:
type: number
stepWeightPromotion:
description: Incremental traffic percentage step for the promotion phase
description: Incremental traffic step weight for the promotion phase
type: number
mirror:
description: Mirror traffic to canary
type: boolean
mirrorWeight:
description: Percentage of traffic to be mirrored
description: Weight of traffic to be mirrored
type: number
match:
description: A/B testing match conditions
@@ -702,7 +719,7 @@ spec:
- Terminating
- Terminated
canaryWeight:
description: Traffic weight percentage routed to canary
description: Traffic weight routed to canary
type: number
failedChecks:
description: Failed check count of the current canary analysis
kustomize/base/flagger/crd.yaml
+22 -5
View File
@@ -48,10 +48,18 @@ spec:
type: boolean
JSONPath: .spec.analysis.mirror
priority: 1
- name: FullWeight
type: string
JSONPath: .spec.analysis.fullWeight
priority: 1
- name: StepWeight
type: string
JSONPath: .spec.analysis.stepWeight
priority: 1
- name: StepWeights
type: string
JSONPath: .spec.analysis.stepWeights
priority: 1
- name: MaxWeight
type: string
JSONPath: .spec.analysis.maxWeight
@@ -547,6 +555,7 @@ spec:
oneOf:
- required: ["interval", "threshold", "iterations"]
- required: ["interval", "threshold", "stepWeight"]
- required: ["interval", "threshold", "stepWeights"]
properties:
interval:
description: Schedule interval for this canary
@@ -558,20 +567,28 @@ spec:
threshold:
description: Max number of failed checks before rollback
type: number
fullWeight:
description: Sum of traffic step weights for canary and primary
type: number
maxWeight:
description: Max traffic percentage routed to canary
description: Max traffic weight routed to canary
type: number
stepWeight:
description: Incremental traffic percentage step for the analysis phase
description: Incremental traffic step weight for the analysis phase
type: number
stepWeights:
description: Incremental traffic step weights for the analysis phase
type: array
items:
type: number
stepWeightPromotion:
description: Incremental traffic percentage step for the promotion phase
description: Incremental traffic step weight for the promotion phase
type: number
mirror:
description: Mirror traffic to canary
type: boolean
mirrorWeight:
description: Percentage of traffic to be mirrored
description: Weight of traffic to be mirrored
type: number
match:
description: A/B testing match conditions
@@ -702,7 +719,7 @@ spec:
- Terminating
- Terminated
canaryWeight:
description: Traffic weight percentage routed to canary
description: Traffic weight routed to canary
type: number
failedChecks:
description: Failed check count of the current canary analysis
pkg/apis/flagger/v1beta1/canary.go
+12 -4
View File
@@ -195,19 +195,27 @@ type CanaryAnalysis struct {
// +optional
Mirror bool `json:"mirror,omitempty"`
// Percentage of the traffic to be mirrored in the range of [0, 100].
// Sum of weights of canary and primary. If not set default 100 is used.
// +optional
FullWeight int `json:"fullWeight,omitempty"`
// Weight of the traffic to be mirrored in the range of [0, FullWeight].
// +optional
MirrorWeight int `json:"mirrorWeight,omitempty"`
// Max traffic percentage routed to canary
// Max traffic weight routed to canary
// +optional
MaxWeight int `json:"maxWeight,omitempty"`
// Incremental traffic percentage step for analysis phase
// Incremental traffic weight step for analysis phase
// +optional
StepWeight int `json:"stepWeight,omitempty"`
// Incremental traffic percentage step for promotion phase
// Incremental traffic weight steps for analysis phase
// +optional
StepWeights []int `json:"stepWeights,omitempty"`
// Incremental traffic weight step for promotion phase
// +optional
StepWeightPromotion int `json:"stepWeightPromotion,omitempty"`
pkg/apis/flagger/v1beta1/zz_generated.deepcopy.go
+5
View File
@@ -197,6 +197,11 @@ func (in *CanaryAlert) DeepCopy() *CanaryAlert {
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *CanaryAnalysis) DeepCopyInto(out *CanaryAnalysis) {
*out = *in
if in.StepWeights != nil {
in, out := &in.StepWeights, &out.StepWeights
*out = make([]int, len(*in))
copy(*out, *in)
}
if in.Alerts != nil {
in, out := &in.Alerts, &out.Alerts
*out = make([]CanaryAlert, len(*in))
pkg/controller/events.go
+8
View File
@@ -3,6 +3,7 @@ package controller
import (
"context"
"fmt"
"strings"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -172,6 +173,13 @@ func alertMetadata(canary *flaggerv1.Canary) []notifier.Field {
canary.GetAnalysis().StepWeight,
canary.GetAnalysis().MaxWeight),
})
} else if len(canary.GetAnalysis().StepWeights) > 0 {
fields = append(fields, notifier.Field{
Name: "Traffic routing",
Value: fmt.Sprintf("Weight steps: %s max: %v",
strings.Trim(strings.Join(strings.Fields(fmt.Sprint(canary.GetAnalysis().StepWeights)), ","), "[]"),
canary.GetAnalysis().MaxWeight),
})
} else if len(canary.GetAnalysis().Match) > 0 {
fields = append(fields, notifier.Field{
Name: "Traffic routing",
pkg/controller/scheduler.go
+53 -24
View File
@@ -14,6 +14,33 @@ import (
"github.com/weaveworks/flagger/pkg/router"
)
func (c *Controller) fullWeight(canary *flaggerv1.Canary) int {
if canary.GetAnalysis().FullWeight > 0 {
return canary.GetAnalysis().FullWeight
}
// set max weight default value to 100%
return 100
}
func (c *Controller) nextStepWeight(canary *flaggerv1.Canary, canaryWeight int) int {
var stepWeightsLen = len(canary.GetAnalysis().StepWeights)
if canary.GetAnalysis().StepWeight > 0 || stepWeightsLen == 0 {
return canary.GetAnalysis().StepWeight
}
if canaryWeight == 0 {
return canary.GetAnalysis().StepWeights[0]
}
for i := 0; i < stepWeightsLen-1; i++ {
if canary.GetAnalysis().StepWeights[i] == canaryWeight {
return canary.GetAnalysis().StepWeights[i+1] - canaryWeight
}
}
return c.fullWeight(canary) - canaryWeight
}
// scheduleCanaries synchronises the canary map with the jobs map,
// for new canaries new jobs are created and started
// for the removed canaries the jobs are stopped and deleted
@@ -173,8 +200,7 @@ func (c *Controller) advanceCanary(name string, namespace string) {
return
}
// set max weight default value to 100%
maxWeight := 100
maxWeight := c.fullWeight(cd)
if cd.GetAnalysis().MaxWeight > 0 {
maxWeight = cd.GetAnalysis().MaxWeight
}
@@ -207,7 +233,7 @@ func (c *Controller) advanceCanary(name string, namespace string) {
cd.Spec.TargetRef.Name, cd.Namespace)
// route all traffic back to primary
primaryWeight = 100
primaryWeight = c.fullWeight(cd)
canaryWeight = 0
if err := meshRouter.SetRoutes(cd, primaryWeight, canaryWeight, false); err != nil {
c.recordEventWarningf(cd, "%v", err)
@@ -343,7 +369,7 @@ func (c *Controller) advanceCanary(name string, namespace string) {
}
// strategy: Canary progressive traffic increase
if cd.GetAnalysis().StepWeight > 0 {
if c.nextStepWeight(cd, canaryWeight) > 0 {
c.runCanary(cd, canaryController, meshRouter, mirrored, canaryWeight, primaryWeight, maxWeight)
}
@@ -362,22 +388,22 @@ func (c *Controller) runPromotionTrafficShift(canary *flaggerv1.Canary, canaryCo
// route all traffic to primary in one go when promotion step wight is not set
if canary.Spec.Analysis.StepWeightPromotion == 0 {
c.recordEventInfof(canary, "Routing all traffic to primary")
if err := meshRouter.SetRoutes(canary, 100, 0, false); err != nil {
if err := meshRouter.SetRoutes(canary, c.fullWeight(canary), 0, false); err != nil {
c.recordEventWarningf(canary, "%v", err)
return
}
c.recorder.SetWeight(canary, 100, 0)
c.recorder.SetWeight(canary, c.fullWeight(canary), 0)
if err := canaryController.SetStatusPhase(canary, flaggerv1.CanaryPhaseFinalising); err != nil {
c.recordEventWarningf(canary, "%v", err)
}
return
}
// increment the primary traffic weight until it reaches 100%
// increment the primary traffic weight until it reaches 100%/full weight
if canaryWeight > 0 {
primaryWeight += canary.GetAnalysis().StepWeightPromotion
if primaryWeight > 100 {
primaryWeight = 100
if primaryWeight > c.fullWeight(canary) {
primaryWeight = c.fullWeight(canary)
}
canaryWeight -= canary.GetAnalysis().StepWeightPromotion
if canaryWeight < 0 {
@@ -391,7 +417,7 @@ func (c *Controller) runPromotionTrafficShift(canary *flaggerv1.Canary, canaryCo
c.recordEventInfof(canary, "Advance %s.%s primary weight %v", canary.Name, canary.Namespace, primaryWeight)
// finalize promotion
if primaryWeight == 100 {
if primaryWeight == c.fullWeight(canary) {
if err := canaryController.SetStatusPhase(canary, flaggerv1.CanaryPhaseFinalising); err != nil {
c.recordEventWarningf(canary, "%v", err)
}
@@ -415,27 +441,30 @@ func (c *Controller) runCanary(canary *flaggerv1.Canary, canaryController canary
// If in "mirror" mode, do one step of mirroring before shifting traffic to canary.
// When mirroring, all requests go to primary and canary, but only responses from
// primary go back to the user.
var nextStepWeight int
nextStepWeight = c.nextStepWeight(canary, canaryWeight)
if canary.GetAnalysis().Mirror && canaryWeight == 0 {
if !mirrored {
mirrored = true
primaryWeight = 100
primaryWeight = c.fullWeight(canary)
canaryWeight = 0
} else {
mirrored = false
primaryWeight = 100 - canary.GetAnalysis().StepWeight
canaryWeight = canary.GetAnalysis().StepWeight
primaryWeight = c.fullWeight(canary) - nextStepWeight
canaryWeight = nextStepWeight
}
c.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)).
Infof("Running mirror step %d/%d/%t", primaryWeight, canaryWeight, mirrored)
} else {
primaryWeight -= canary.GetAnalysis().StepWeight
primaryWeight -= nextStepWeight
if primaryWeight < 0 {
primaryWeight = 0
}
canaryWeight += canary.GetAnalysis().StepWeight
if canaryWeight > 100 {
canaryWeight = 100
canaryWeight += nextStepWeight
if canaryWeight > c.fullWeight(canary) {
canaryWeight = c.fullWeight(canary)
}
}
@@ -483,11 +512,11 @@ func (c *Controller) runAB(canary *flaggerv1.Canary, canaryController canary.Con
// route traffic to canary and increment iterations
if canary.GetAnalysis().Iterations > canary.Status.Iterations {
if err := meshRouter.SetRoutes(canary, 0, 100, false); err != nil {
if err := meshRouter.SetRoutes(canary, 0, c.fullWeight(canary), false); err != nil {
c.recordEventWarningf(canary, "%v", err)
return
}
c.recorder.SetWeight(canary, 0, 100)
c.recorder.SetWeight(canary, 0, c.fullWeight(canary))
if err := canaryController.SetStatusIterations(canary, canary.Status.Iterations+1); err != nil {
c.recordEventWarningf(canary, "%v", err)
@@ -529,7 +558,7 @@ func (c *Controller) runBlueGreen(canary *flaggerv1.Canary, canaryController can
// If in "mirror" mode, mirror requests during the entire B/G canary test
if provider != "kubernetes" &&
canary.GetAnalysis().Mirror && !mirrored {
if err := meshRouter.SetRoutes(canary, 100, 0, true); err != nil {
if err := meshRouter.SetRoutes(canary, c.fullWeight(canary), 0, true); err != nil {
c.recordEventWarningf(canary, "%v", err)
}
c.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)).
@@ -557,11 +586,11 @@ func (c *Controller) runBlueGreen(canary *flaggerv1.Canary, canaryController can
} else {
c.recordEventInfof(canary, "Routing all traffic to canary")
}
if err := meshRouter.SetRoutes(canary, 0, 100, false); err != nil {
if err := meshRouter.SetRoutes(canary, 0, c.fullWeight(canary), false); err != nil {
c.recordEventWarningf(canary, "%v", err)
return
}
c.recorder.SetWeight(canary, 0, 100)
c.recorder.SetWeight(canary, 0, c.fullWeight(canary))
}
// increment iterations
@@ -622,7 +651,7 @@ func (c *Controller) shouldSkipAnalysis(canary *flaggerv1.Canary, canaryControll
}
// route all traffic to primary
primaryWeight := 100
primaryWeight := c.fullWeight(canary)
canaryWeight := 0
if err := meshRouter.SetRoutes(canary, primaryWeight, canaryWeight, false); err != nil {
c.recordEventWarningf(canary, "%v", err)
@@ -756,7 +785,7 @@ func (c *Controller) rollback(canary *flaggerv1.Canary, canaryController canary.
}
// route all traffic back to primary
primaryWeight := 100
primaryWeight := c.fullWeight(canary)
canaryWeight := 0
if err := meshRouter.SetRoutes(canary, primaryWeight, canaryWeight, false); err != nil {
c.recordEventWarningf(canary, "%v", err)
test/e2e-linkerd-steps-tests.sh
+220
View File
@@ -0,0 +1,220 @@
#!/usr/bin/env bash
# This script runs Linkerd e2e tests for Canary initialization, analysis and promotion
set -o errexit
REPO_ROOT=$(git rev-parse --show-toplevel)
echo '>>> Creating test namespace'
kubectl create namespace test
kubectl annotate namespace test linkerd.io/inject=enabled
echo '>>> Installing the load tester'
kubectl apply -k ${REPO_ROOT}/kustomize/tester
kubectl -n test rollout status deployment/flagger-loadtester
echo '>>> Initialising canary'
kubectl apply -f ${REPO_ROOT}/test/e2e-workload.yaml
cat <<EOF | kubectl apply -f -
apiVersion: flagger.app/v1alpha1
kind: MetricTemplate
metadata:
name: latency
namespace: linkerd
spec:
provider:
type: prometheus
address: http://linkerd-prometheus.linkerd:9090
query: |
histogram_quantile(
0.99,
sum(
rate(
response_latency_ms_bucket{
namespace="{{ namespace }}",
deployment=~"{{ target }}",
direction="inbound"
}[{{ interval }}]
)
) by (le)
)
EOF
cat <<EOF | kubectl apply -f -
apiVersion: flagger.app/v1beta1
kind: Canary
metadata:
name: podinfo
namespace: test
spec:
targetRef:
apiVersion: apps/v1
kind: Deployment
name: podinfo
progressDeadlineSeconds: 60
service:
port: 80
targetPort: http
portDiscovery: true
analysis:
interval: 15s
threshold: 15
fullWeight: 1000
maxWeight: 500
stepWeights: [100, 200, 300, 400, 500, 600]
metrics:
- name: request-success-rate
threshold: 99
interval: 1m
- name: request-duration
threshold: 500
interval: 30s
- name: latency
templateRef:
name: latency
namespace: linkerd
threshold: 300
interval: 1m
webhooks:
- name: http-acceptance-test
type: pre-rollout
url: http://flagger-loadtester.test/
timeout: 30s
metadata:
type: bash
cmd: "curl -sd 'test' http://podinfo-canary/token | grep token"
- name: grpc-acceptance-test
type: pre-rollout
url: http://flagger-loadtester.test/
timeout: 5s
metadata:
type: bash
cmd: "grpc_health_probe -connect-timeout=1s -addr=podinfo-canary:9999"
- name: load-test
url: http://flagger-loadtester.test/
timeout: 5s
metadata:
type: cmd
cmd: "hey -z 10m -q 10 -c 2 http://podinfo.test/"
logCmdOutput: "true"
EOF
echo '>>> Waiting for primary to be ready'
retries=50
count=0
ok=false
until ${ok}; do
kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false
sleep 5
count=$(($count + 1))
if [[ ${count} -eq ${retries} ]]; then
kubectl -n linkerd logs deployment/flagger
echo "No more retries left"
exit 1
fi
done
echo '✔ Canary initialization test passed'
echo '>>> Triggering canary deployment'
kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1
echo '>>> Waiting for canary promotion'
retries=50
count=0
ok=false
until ${ok}; do
kubectl -n test describe deployment/podinfo-primary | grep '3.1.1' && ok=true || ok=false
sleep 10
kubectl -n linkerd logs deployment/flagger --tail 1
count=$(($count + 1))
if [[ ${count} -eq ${retries} ]]; then
kubectl -n linkerd logs deployment/flagger
echo "No more retries left"
exit 1
fi
done
echo '>>> Waiting for canary finalization'
retries=50
count=0
ok=false
until ${ok}; do
kubectl -n test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false
sleep 5
count=$(($count + 1))
if [[ ${count} -eq ${retries} ]]; then
kubectl -n linkerd logs deployment/flagger
echo "No more retries left"
exit 1
fi
done
echo '✔ Canary promotion test passed'
cat <<EOF | kubectl apply -f -
apiVersion: flagger.app/v1beta1
kind: Canary
metadata:
name: podinfo
namespace: test
spec:
targetRef:
apiVersion: apps/v1
kind: Deployment
name: podinfo
progressDeadlineSeconds: 60
service:
port: 80
targetPort: 9898
analysis:
interval: 15s
threshold: 3
fullWeight: 1000
maxWeight: 500
stepWeights: [100, 200, 300, 400, 500, 600]
metrics:
- name: request-success-rate
threshold: 99
interval: 1m
- name: request-duration
threshold: 500
interval: 30s
webhooks:
- name: http-acceptance-test
type: pre-rollout
url: http://flagger-loadtester.test/
timeout: 30s
metadata:
type: bash
cmd: "curl -sd 'test' http://podinfo-canary/token | grep token"
- name: load-test
url: http://flagger-loadtester.test/
timeout: 5s
metadata:
type: cmd
cmd: "hey -z 10m -q 10 -c 2 http://podinfo.test/status/500"
EOF
echo '>>> Triggering canary deployment'
kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.2
echo '>>> Waiting for canary rollback'
retries=50
count=0
ok=false
until ${ok}; do
kubectl -n test get canary/podinfo | grep 'Failed' && ok=true || ok=false
sleep 10
kubectl -n linkerd logs deployment/flagger --tail 1
count=$(($count + 1))
if [[ ${count} -eq ${retries} ]]; then
kubectl -n linkerd logs deployment/flagger
echo "No more retries left"
exit 1
fi
done
echo '✔ Canary rollback test passed'