fix a/b testing logic and update e2e tests

Signed-off-by: Sanskar Jaiswal <jaiswalsanskar078@gmail.com>
This commit is contained in:
Sanskar Jaiswal
2022-03-10 16:54:36 +05:30
committed by Sanskar Jaiswal
parent ba4646cddb
commit 91126d102d
10 changed files with 385 additions and 258 deletions
+32 -6
View File
@@ -109,7 +109,7 @@ func (gwr *GatewayAPIRouter) Reconcile(canary *flaggerv1.Canary) error {
if len(canary.GetAnalysis().Match) > 0 {
analysisMatches, _ := gwr.mapRouteMatches(canary.GetAnalysis().Match)
// serviceMatches, _ := gwr.mapRouteMatches(canary.Spec.Service.Match)
httpRouteSpec.Rules[0].Matches = analysisMatches
httpRouteSpec.Rules[0].Matches = gwr.mergeMatchConditions(analysisMatches, matches)
httpRouteSpec.Rules = append(httpRouteSpec.Rules, v1alpha2.HTTPRouteRule{
Matches: matches,
BackendRefs: []v1alpha2.HTTPBackendRef{
@@ -165,16 +165,17 @@ func (gwr *GatewayAPIRouter) Reconcile(canary *flaggerv1.Canary) error {
}
if httpRoute != nil {
if diff := cmp.Diff(
diff := cmp.Diff(
httpRoute.Spec, httpRouteSpec,
cmpopts.IgnoreFields(v1alpha2.BackendRef{}, "Weight"),
); diff != "" {
)
if diff != "" && httpRoute.Name != "" {
hrClone := httpRoute.DeepCopy()
hrClone.Spec = httpRouteSpec
_, err := gwr.gatewayAPIClient.GatewayapiV1alpha2().HTTPRoutes(hrNamespace).
Update(context.TODO(), hrClone, metav1.UpdateOptions{})
if err != nil {
return fmt.Errorf("HTTPRoute %s.%s update error: %w", hrClone.GetName(), hrNamespace, err)
return fmt.Errorf("HTTPRoute %s.%s update error: %w while reconciling", hrClone.GetName(), hrNamespace, err)
}
gwr.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)).
Infof("HTTPProxy %s.%s updated", hrClone.GetName(), hrNamespace)
@@ -275,7 +276,7 @@ func (gwr *GatewayAPIRouter) SetRoutes(
// A/B testing
if len(canary.GetAnalysis().Match) > 0 {
analysisMatches, _ := gwr.mapRouteMatches(canary.GetAnalysis().Match)
hrClone.Spec.Rules[0].Matches = analysisMatches
hrClone.Spec.Rules[0].Matches = gwr.mergeMatchConditions(analysisMatches, matches)
hrClone.Spec.Rules = append(hrClone.Spec.Rules, v1alpha2.HTTPRouteRule{
Matches: matches,
BackendRefs: []v1alpha2.HTTPBackendRef{
@@ -288,8 +289,11 @@ func (gwr *GatewayAPIRouter) SetRoutes(
_, err = gwr.gatewayAPIClient.GatewayapiV1alpha2().HTTPRoutes(hrNamespace).Update(context.TODO(), hrClone, metav1.UpdateOptions{})
if err != nil {
return fmt.Errorf("HTTPRoute %s.%s update error: %w", hrClone.GetName(), hrNamespace, err)
return fmt.Errorf("HTTPRoute %s.%s update error: %w while setting weights", hrClone.GetName(), hrNamespace, err)
}
gwr.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)).
Infof("HTTPProxy %s.%s weights updated", hrClone.GetName(), hrNamespace)
return nil
}
@@ -386,3 +390,25 @@ func (gwr *GatewayAPIRouter) makeBackendRef(svcName string, weight, port int32)
Weight: &weight,
}
}
func (gwr *GatewayAPIRouter) mergeMatchConditions(analysis, service []v1alpha2.HTTPRouteMatch) []v1alpha2.HTTPRouteMatch {
if len(analysis) == 0 {
return service
}
merged := make([]v1alpha2.HTTPRouteMatch, len(service)*len(analysis))
num := 0
for _, a := range analysis {
for _, s := range service {
merged[num] = *s.DeepCopy()
if len(a.Headers) > 0 {
merged[num].Headers = a.Headers
}
if len(a.QueryParams) > 0 {
merged[num].QueryParams = a.QueryParams
}
num++
}
}
return merged
}
+2
View File
@@ -9,3 +9,5 @@ DIR="$(cd "$(dirname "$0")" && pwd)"
"$REPO_ROOT"/test/workloads/init.sh
"$DIR"/test-canary.sh
"$DIR"/test-bg.sh
"$DIR"/test-ab.sh
+125
View File
@@ -0,0 +1,125 @@
#!/usr/bin/env bash
# This script runs e2e tests for A/B traffic shifting, Canary analysis and promotion
# Prerequisites: Kubernetes Kind and Contour with GatewayAPI
set -o errexit
REPO_ROOT=$(git rev-parse --show-toplevel)
source ${REPO_ROOT}/test/gatewayapi/test-utils.sh
create_latency_metric_template
create_error_rate_metric_template
echo '>>> Deploy podinfo in ab-test namespace'
kubectl create ns ab-test
kubectl apply -f ${REPO_ROOT}/test/workloads/secret.yaml -n ab-test
kubectl apply -f ${REPO_ROOT}/test/workloads/deployment.yaml -n ab-test
cat <<EOF | kubectl apply -f -
apiVersion: flagger.app/v1beta1
kind: Canary
metadata:
name: podinfo
namespace: ab-test
spec:
progressDeadlineSeconds: 60
targetRef:
apiVersion: apps/v1
kind: Deployment
name: podinfo
service:
port: 9898
portName: http
hosts:
- localproject.contour.io
gatewayRefs:
- name: contour
namespace: projectcontour
analysis:
interval: 15s
threshold: 10
iterations: 10
match:
- headers:
x-canary:
exact: "insider"
metrics:
# - name: request-success-rate
# thresholdRange:
# min: 99
# interval: 1m
# - name: request-duration
# threshold: 500
# interval: 30s
- name: error-rate
templateRef:
name: error-rate
namespace: flagger-system
thresholdRange:
max: 1
interval: 1m
- name: latency
templateRef:
name: latency
namespace: flagger-system
thresholdRange:
max: 0.5
interval: 30s
webhooks:
- name: load-test
type: rollout
url: http://flagger-loadtester.test/
timeout: 5s
metadata:
cmd: "hey -z 2m -q 10 -c 2 -host localproject.contour.io -H 'X-Canary: insider' http://envoy.projectcontour/"
logCmdOutput: "true"
EOF
check_primary "ab-test"
display_httproute "ab-test"
echo '>>> Triggering A/B testing'
kubectl -n ab-test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1
echo '>>> Waiting for A/B testing promotion'
retries=50
count=0
ok=false
until ${ok}; do
kubectl -n ab-test describe deployment/podinfo-primary | grep '3.1.1' && ok=true || ok=false
sleep 10
kubectl -n flagger-system logs deployment/flagger --tail 1
count=$(($count + 1))
if [[ ${count} -eq ${retries} ]]; then
kubectl -n ab-test describe deployment/podinfo
kubectl -n ab-test describe deployment/podinfo-primary
kubectl -n flagger-system logs deployment/flagger
echo "No more retries left"
exit 1
fi
done
display_httproute "ab-test"
echo '>>> Waiting for A/B finalization'
retries=50
count=0
ok=false
until ${ok}; do
kubectl -n ab-test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false
sleep 5
count=$(($count + 1))
if [[ ${count} -eq ${retries} ]]; then
kubectl -n flagger-system logs deployment/flagger
echo "No more retries left"
exit 1
fi
done
echo '✔ A/B testing promotion test passed'
kubectl delete -n ab-test canary podinfo
+110
View File
@@ -0,0 +1,110 @@
#!/usr/bin/env bash
# This script runs e2e tests for Blue/Green traffic shifting, Canary analysis and promotion
# Prerequisites: Kubernetes Kind and Contour with GatewayAPI
set -o errexit
REPO_ROOT=$(git rev-parse --show-toplevel)
source ${REPO_ROOT}/test/gatewayapi/test-utils.sh
echo '>>> Deploy podinfo in bg-test namespace'
kubectl create ns bg-test
kubectl apply -f ${REPO_ROOT}/test/workloads/secret.yaml -n bg-test
kubectl apply -f ${REPO_ROOT}/test/workloads/deployment.yaml -n bg-test
cat <<EOF | kubectl apply -f -
apiVersion: flagger.app/v1beta1
kind: Canary
metadata:
name: podinfo
namespace: bg-test
spec:
targetRef:
apiVersion: apps/v1
kind: Deployment
name: podinfo
progressDeadlineSeconds: 60
service:
port: 9898
portName: http
hosts:
- localproject.contour.io
gatewayRefs:
- name: contour
namespace: projectcontour
analysis:
interval: 10s
threshold: 5
iterations: 5
metrics:
- name: request-success-rate
thresholdRange:
min: 99
interval: 1m
- name: request-duration
threshold: 500
interval: 30s
webhooks:
- name: smoke-test
type: pre-rollout
url: http://flagger-loadtester.test/
timeout: 15s
metadata:
type: bash
cmd: "curl -sd 'anon' http://podinfo-canary.bg-test:9898/token | grep token"
- name: load-test
type: rollout
url: http://flagger-loadtester.test/
timeout: 5s
metadata:
cmd: "hey -z 2m -q 10 -c 2 http://podinfo-canary.bg-test:9898"
logCmdOutput: "true"
EOF
check_primary "bg-test"
display_httproute "bg-test"
echo '>>> Triggering B/G deployment'
kubectl -n bg-test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1
echo '>>> Waiting for B/G promotion'
retries=50
count=0
ok=false
until ${ok}; do
kubectl -n bg-test describe deployment/podinfo-primary | grep '3.1.1' && ok=true || ok=false
sleep 10
kubectl -n flagger-system logs deployment/flagger --tail 1
count=$(($count + 1))
if [[ ${count} -eq ${retries} ]]; then
kubectl -n bg-test describe deployment/podinfo
kubectl -n bg-test describe deployment/podinfo-primary
kubectl -n flagger-system logs deployment/flagger
echo "No more retries left"
exit 1
fi
done
display_httproute "bg-test"
echo '>>> Waiting for B/G finalization'
retries=50
count=0
ok=false
until ${ok}; do
kubectl -n bg-test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false
sleep 5
count=$(($count + 1))
if [[ ${count} -eq ${retries} ]]; then
kubectl -n flagger-system logs deployment/flagger
echo "No more retries left"
exit 1
fi
done
echo '✔ B/G promotion test passed'
kubectl delete -n bg-test canary podinfo
+13 -246
View File
@@ -1,60 +1,16 @@
#!/usr/bin/env bash
# This script runs e2e tests for Canary, A/B initialization, analysis and promotion
# This script runs e2e tests for progressive traffic shifting, Canary analysis and promotion
# Prerequisites: Kubernetes Kind and Contour with GatewayAPI
set -o errexit
echo '>>> Create metric templates'
cat <<EOF | kubectl apply -f -
apiVersion: flagger.app/v1beta1
kind: MetricTemplate
metadata:
name: latency
namespace: flagger-system
spec:
provider:
type: prometheus
address: http://flagger-prometheus:9090
query: |
histogram_quantile(0.99,
sum(
rate(
envoy_cluster_upstream_rq_time_bucket{
envoy_cluster_name=~"{{ namespace }}_{{ target }}-canary_[0-9a-zA-Z-]+",
}[{{ interval }}]
)
) by (le)
)/1000
---
apiVersion: flagger.app/v1beta1
kind: MetricTemplate
metadata:
name: error-rate
namespace: flagger-system
spec:
provider:
type: prometheus
address: http://flagger-prometheus:9090
query: |
100 - sum(
rate(
envoy_cluster_upstream_rq{
envoy_cluster_name=~"{{ namespace }}_{{ target }}-canary_[0-9a-zA-Z-]+",
envoy_response_code!~"5.*"
}[{{ interval }}]
)
)
/
sum(
rate(
envoy_cluster_upstream_rq{
envoy_cluster_name=~"{{ namespace }}_{{ target }}-canary_[0-9a-zA-Z-]+",
}[{{ interval }}]
)
)
* 100
EOF
REPO_ROOT=$(git rev-parse --show-toplevel)
source ${REPO_ROOT}/test/gatewayapi/test-utils.sh
create_latency_metric_template
create_error_rate_metric_template
echo '>>> Installing Canary'
cat <<EOF | kubectl apply -f -
@@ -107,35 +63,9 @@ spec:
logCmdOutput: "true"
EOF
echo '>>> Waiting for primary to be ready'
retries=50
count=0
ok=false
until ${ok}; do
kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false
sleep 5
count=$(($count + 1))
if [[ ${count} -eq ${retries} ]]; then
kubectl -n flagger-system logs deployment/flagger
echo "No more retries left"
exit 1
fi
done
check_primary "test"
echo '✔ Canary initialization test passed'
passed=$(kubectl -n test get svc/podinfo -o jsonpath='{.spec.selector.app}' 2>&1 | { grep podinfo-primary || true; })
if [ -z "$passed" ]; then
echo -e '\u2716 podinfo selector test failed'
exit 1
fi
echo '✔ Canary service custom metadata test passed'
if ! kubectl -n test get httproute podinfo -oyaml; then
echo "Could not find HTTPRoute podinfo"
exit 1
fi
display_httproute "test"
echo '>>> Triggering canary deployment'
kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1
@@ -156,6 +86,8 @@ until ${ok}; do
fi
done
display_httproute "test"
echo '>>> Waiting for canary finalization'
retries=50
count=0
@@ -173,173 +105,6 @@ done
echo '✔ Canary promotion test passed'
if [[ "$1" = "canary" ]]; then
exit 0
fi
cat <<EOF | kubectl apply -f -
apiVersion: flagger.app/v1beta1
kind: Canary
metadata:
name: podinfo
namespace: test
spec:
targetRef:
apiVersion: apps/v1
kind: Deployment
name: podinfo
progressDeadlineSeconds: 60
service:
port: 9898
portName: http
hosts:
- localproject.contour.io
gatewayRefs:
- name: contour
namespace: projectcontour
analysis:
interval: 10s
threshold: 5
iterations: 5
metrics:
- name: error-rate
templateRef:
name: error-rate
namespace: flagger-system
thresholdRange:
max: 1
interval: 1m
- name: latency
templateRef:
name: latency
namespace: flagger-system
thresholdRange:
max: 0.5
interval: 30s
webhooks:
- name: load-test
type: rollout
url: http://flagger-loadtester.test/
timeout: 5s
metadata:
cmd: "hey -z 2m -q 10 -c 2 -host localproject.contour.io http://envoy.projectcontour/"
logCmdOutput: "true"
EOF
echo '>>> Triggering B/G deployment'
kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.2
echo '>>> Waiting for B/G promotion'
retries=50
count=0
ok=false
until ${ok}; do
kubectl -n test describe deployment/podinfo-primary | grep '3.1.2' && ok=true || ok=false
sleep 10
kubectl -n flagger-system logs deployment/flagger --tail 1
count=$(($count + 1))
if [[ ${count} -eq ${retries} ]]; then
kubectl -n test describe deployment/podinfo
kubectl -n test describe deployment/podinfo-primary
kubectl -n flagger-system logs deployment/flagger
echo "No more retries left"
exit 1
fi
done
echo '>>> Waiting for B/G finalization'
retries=50
count=0
ok=false
until ${ok}; do
kubectl -n test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false
sleep 5
count=$(($count + 1))
if [[ ${count} -eq ${retries} ]]; then
kubectl -n flagger-system logs deployment/flagger
echo "No more retries left"
exit 1
fi
done
echo '✔ B/G promotion test passed'
cat <<EOF | kubectl apply -f -
apiVersion: flagger.app/v1beta1
kind: Canary
metadata:
name: podinfo
namespace: test
spec:
progressDeadlineSeconds: 60
targetRef:
apiVersion: apps/v1
kind: Deployment
name: podinfo
service:
port: 9898
portName: http
hosts:
- localproject.contour.io
gatewayRefs:
- name: contour
namespace: projectcontour
analysis:
interval: 15s
threshold: 10
iterations: 10
match:
- headers:
x-canary:
exact: "insider"
metrics:
- name: error-rate
templateRef:
name: error-rate
namespace: flagger-system
thresholdRange:
max: 1
interval: 1m
- name: latency
templateRef:
name: latency
namespace: flagger-system
thresholdRange:
max: 0.5
interval: 30s
webhooks:
- name: load-test
type: rollout
url: http://flagger-loadtester.test/
timeout: 5s
metadata:
cmd: "hey -z 2m -q 10 -c 2 -host localproject.contour.io http://envoy.projectcontour/"
logCmdOutput: "true"
EOF
echo '>>> Triggering A/B testing'
kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.3
echo '>>> Waiting for A/B testing promotion'
retries=50
count=0
ok=false
until ${ok}; do
kubectl -n test describe deployment/podinfo-primary | grep '3.1.3' && ok=true || ok=false
sleep 10
kubectl -n flagger-system logs deployment/flagger --tail 1
count=$(($count + 1))
if [[ ${count} -eq ${retries} ]]; then
kubectl -n test describe deployment/podinfo
kubectl -n test describe deployment/podinfo-primary
kubectl -n flagger-system logs deployment/flagger
echo "No more retries left"
exit 1
fi
done
echo '✔ A/B testing promotion test passed'
cat <<EOF | kubectl apply -f -
apiVersion: flagger.app/v1beta1
kind: Canary
@@ -411,3 +176,5 @@ until ${ok}; do
done
echo '✔ Canary rollback test passed'
kubectl delete -n test canary podinfo
+100
View File
@@ -0,0 +1,100 @@
#!/usr/bin/env bash
check_primary() {
echo '>>> Waiting for primary to be ready'
namespace=$1
retries=50
count=0
ok=false
echo $namespace
until ${ok}; do
kubectl get -n ${namespace} canary/podinfo | grep 'Initialized' && ok=true || ok=false
sleep 5
count=$(($count + 1))
if [[ ${count} -eq ${retries} ]]; then
kubectl -n flagger-system logs deployment/flagger
echo "No more retries left"
exit 1
fi
done
echo '✔ Canary initialization test passed'
passed=$(kubectl -n $namespace get svc/podinfo -o jsonpath='{.spec.selector.app}' 2>&1 | { grep podinfo-primary || true; })
if [ -z "$passed" ]; then
echo -e '\u2716 podinfo selector test failed'
exit 1
fi
echo '✔ Canary service custom metadata test passed'
}
display_httproute() {
namespace=$1
if ! kubectl -n ${namespace} get httproute podinfo -oyaml; then
echo "Could not find HTTPRoute podinfo in ${namespace} namespace"
exit 1
fi
}
create_latency_metric_template() {
if ! kubectl -n flagger-system get metrictemplates latency; then
echo '>>> Create latency metric template'
cat <<EOF | kubectl apply -f -
apiVersion: flagger.app/v1beta1
kind: MetricTemplate
metadata:
name: latency
namespace: flagger-system
spec:
provider:
type: prometheus
address: http://flagger-prometheus:9090
query: |
histogram_quantile(0.99,
sum(
rate(
envoy_cluster_upstream_rq_time_bucket{
envoy_cluster_name=~"{{ namespace }}_{{ target }}-canary_[0-9a-zA-Z-]+",
}[{{ interval }}]
)
) by (le)
)/1000
EOF
fi
}
create_error_rate_metric_template() {
if ! kubectl -n flagger-system get metrictemplates error-rate; then
echo '>>> Create latency metric template'
cat <<EOF | kubectl apply -f -
apiVersion: flagger.app/v1beta1
kind: MetricTemplate
metadata:
name: error-rate
namespace: flagger-system
spec:
provider:
type: prometheus
address: http://flagger-prometheus:9090
query: |
100 - sum(
rate(
envoy_cluster_upstream_rq{
envoy_cluster_name=~"{{ namespace }}_{{ target }}-canary_[0-9a-zA-Z-]+",
envoy_response_code!~"5.*"
}[{{ interval }}]
)
)
/
sum(
rate(
envoy_cluster_upstream_rq{
envoy_cluster_name=~"{{ namespace }}_{{ target }}-canary_[0-9a-zA-Z-]+",
}[{{ interval }}]
)
)
* 100
EOF
fi
}
-1
View File
@@ -2,7 +2,6 @@ apiVersion: apps/v1
kind: DaemonSet
metadata:
name: podinfo-ds
namespace: test
labels:
app.kubernetes.io/name: podinfo-ds
spec:
-1
View File
@@ -2,7 +2,6 @@ apiVersion: apps/v1
kind: Deployment
metadata:
name: podinfo
namespace: test
labels:
app.kubernetes.io/name: podinfo
app.kubernetes.io/managed-by: test-label-prefix
+3 -3
View File
@@ -20,6 +20,6 @@ kubectl apply -k ${REPO_ROOT}/kustomize/tester
kubectl -n test rollout status deployment/flagger-loadtester
echo '>>> Deploy podinfo'
kubectl apply -f ${REPO_ROOT}/test/workloads/secret.yaml
kubectl apply -f ${REPO_ROOT}/test/workloads/deployment.yaml
kubectl apply -f ${REPO_ROOT}/test/workloads/daemonset.yaml
kubectl apply -f ${REPO_ROOT}/test/workloads/secret.yaml -n test
kubectl apply -f ${REPO_ROOT}/test/workloads/deployment.yaml -n test
kubectl apply -f ${REPO_ROOT}/test/workloads/daemonset.yaml -n test
-1
View File
@@ -2,6 +2,5 @@ apiVersion: v1
kind: Secret
metadata:
name: podinfo-secret
namespace: test
stringData:
value: s3cr3t