github/flagger
1
0
Fork 0
mirror of https://github.com/fluxcd/flagger.git synced 2026-04-15 06:57:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #305 from weaveworks/service-mesh-blue-green

Browse Source 
Implement B/G for service mesh providers
This commit is contained in:
Stefan Prodan
2019-09-22 12:01:10 +03:00
committed by GitHub
parent a7112fafb0 d19a070faf
commit 37a1ff5c99
2 changed files with 165 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/controller/scheduler.go
+74 -2
View File
@@ -283,7 +283,7 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
// check if the canary success rate is above the threshold
// skip check if no traffic is routed to canary
if canaryWeight == 0 {
if canaryWeight == 0 && cd.Status.Iterations == 0 {
c.recordEventInfof(cd, "Starting canary analysis for %s.%s", cd.Spec.TargetRef.Name, cd.Namespace)
// run pre-rollout web hooks
@@ -305,7 +305,7 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
}
// canary fix routing: A/B testing
if len(cd.Spec.CanaryAnalysis.Match) > 0 || cd.Spec.CanaryAnalysis.Iterations > 0 {
if len(cd.Spec.CanaryAnalysis.Match) > 0 && cd.Spec.CanaryAnalysis.Iterations > 0 {
// route traffic to canary and increment iterations
if cd.Spec.CanaryAnalysis.Iterations > cd.Status.Iterations {
if err := meshRouter.SetRoutes(cd, 0, 100); err != nil {
@@ -362,6 +362,78 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
return
}
// canary fix routing: B/G
if cd.Spec.CanaryAnalysis.Iterations > 0 {
// increment iterations
if cd.Spec.CanaryAnalysis.Iterations > cd.Status.Iterations {
if err := c.deployer.SetStatusIterations(cd, cd.Status.Iterations+1); err != nil {
c.recordEventWarningf(cd, "%v", err)
return
}
c.recordEventInfof(cd, "Advance %s.%s canary iteration %v/%v",
cd.Name, cd.Namespace, cd.Status.Iterations+1, cd.Spec.CanaryAnalysis.Iterations)
return
}
// route all traffic to canary - max iterations reached
if cd.Spec.CanaryAnalysis.Iterations == cd.Status.Iterations {
if provider != "kubernetes" {
c.recordEventInfof(cd, "Routing all traffic to canary")
if err := meshRouter.SetRoutes(cd, 0, 100); err != nil {
c.recordEventWarningf(cd, "%v", err)
return
}
c.recorder.SetWeight(cd, 0, 100)
}
// increment iterations
if err := c.deployer.SetStatusIterations(cd, cd.Status.Iterations+1); err != nil {
c.recordEventWarningf(cd, "%v", err)
return
}
return
}
// promote canary - max iterations reached
if cd.Spec.CanaryAnalysis.Iterations+1 == cd.Status.Iterations {
c.recordEventInfof(cd, "Copying %s.%s template spec to %s.%s",
cd.Spec.TargetRef.Name, cd.Namespace, primaryName, cd.Namespace)
if err := c.deployer.Promote(cd); err != nil {
c.recordEventWarningf(cd, "%v", err)
return
}
// increment iterations
if err := c.deployer.SetStatusIterations(cd, cd.Status.Iterations+1); err != nil {
c.recordEventWarningf(cd, "%v", err)
return
}
return
}
// route all traffic to primary
if cd.Spec.CanaryAnalysis.Iterations < cd.Status.Iterations {
if provider != "kubernetes" {
c.recordEventInfof(cd, "Routing all traffic to primary")
if err := meshRouter.SetRoutes(cd, 100, 0); err != nil {
c.recordEventWarningf(cd, "%v", err)
return
}
c.recorder.SetWeight(cd, 100, 0)
}
// update status phase
if err := c.deployer.SetStatusPhase(cd, flaggerv1.CanaryPhaseFinalising); err != nil {
c.recordEventWarningf(cd, "%v", err)
return
}
return
}
return
}
// canary incremental traffic weight
if canaryWeight < maxWeight {
primaryWeight -= cd.Spec.CanaryAnalysis.StepWeight
test/e2e-tests.sh
+91 -3
View File
@@ -1,6 +1,6 @@
#!/usr/bin/env bash
# This script runs e2e tests for Canary initialization, analysis and promotion
# This script runs e2e tests for Canary, B/G and A/B initialization, analysis and promotion
# Prerequisites: Kubernetes Kind, Helm and Istio
set -o errexit
@@ -124,12 +124,100 @@ until ${ok}; do
fi
done
echo '>>> Waiting for canary finalization'
retries=50
count=0
ok=false
until ${ok}; do
kubectl -n test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false
sleep 5
count=$(($count + 1))
if [[ ${count} -eq ${retries} ]]; then
kubectl -n istio-system logs deployment/flagger
echo "No more retries left"
exit 1
fi
done
echo '✔ Canary promotion test passed'
if [[ "$1" = "canary" ]]; then
exit 0
fi
cat <<EOF | kubectl apply -f -
apiVersion: flagger.app/v1alpha3
kind: Canary
metadata:
name: podinfo
namespace: test
spec:
targetRef:
apiVersion: apps/v1
kind: Deployment
name: podinfo
progressDeadlineSeconds: 60
service:
portDiscovery: true
port: 9898
canaryAnalysis:
interval: 10s
threshold: 5
iterations: 5
metrics:
- name: request-success-rate
threshold: 99
interval: 1m
- name: request-duration
threshold: 500
interval: 30s
webhooks:
- name: load-test
url: http://flagger-loadtester.test/
timeout: 5s
metadata:
type: cmd
cmd: "hey -z 5m -q 10 -c 2 http://podinfo.test:9898/"
EOF
echo '>>> Triggering B/G deployment'
kubectl -n test set image deployment/podinfo podinfod=quay.io/stefanprodan/podinfo:1.4.2
echo '>>> Waiting for B/G promotion'
retries=50
count=0
ok=false
until ${ok}; do
kubectl -n test describe deployment/podinfo-primary | grep '1.4.2' && ok=true || ok=false
sleep 10
kubectl -n istio-system logs deployment/flagger --tail 1
count=$(($count + 1))
if [[ ${count} -eq ${retries} ]]; then
kubectl -n test describe deployment/podinfo
kubectl -n test describe deployment/podinfo-primary
kubectl -n istio-system logs deployment/flagger
echo "No more retries left"
exit 1
fi
done
echo '>>> Waiting for B/G finalization'
retries=50
count=0
ok=false
until ${ok}; do
kubectl -n test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false
sleep 5
count=$(($count + 1))
if [[ ${count} -eq ${retries} ]]; then
kubectl -n istio-system logs deployment/flagger
echo "No more retries left"
exit 1
fi
done
echo '✔ B/G promotion test passed'
cat <<EOF | kubectl apply -f -
apiVersion: flagger.app/v1alpha3
kind: Canary
@@ -180,14 +268,14 @@ spec:
EOF
echo '>>> Triggering A/B testing'
kubectl -n test set image deployment/podinfo podinfod=quay.io/stefanprodan/podinfo:1.4.2
kubectl -n test set image deployment/podinfo podinfod=quay.io/stefanprodan/podinfo:1.4.3
echo '>>> Waiting for A/B testing promotion'
retries=50
count=0
ok=false
until ${ok}; do
kubectl -n test describe deployment/podinfo-primary | grep '1.4.2' && ok=true || ok=false
kubectl -n test describe deployment/podinfo-primary | grep '1.4.3' && ok=true || ok=false
sleep 10
kubectl -n istio-system logs deployment/flagger --tail 1
count=$(($count + 1))