From 91126d102d44b7c7fcbefc2b57d29dc2b981db30 Mon Sep 17 00:00:00 2001 From: Sanskar Jaiswal Date: Wed, 23 Feb 2022 17:31:09 +0530 Subject: [PATCH] fix a/b testing logic and update e2e tests Signed-off-by: Sanskar Jaiswal --- pkg/router/gateway_api.go | 38 ++++- test/gatewayapi/run.sh | 2 + test/gatewayapi/test-ab.sh | 125 ++++++++++++++++ test/gatewayapi/test-bg.sh | 110 ++++++++++++++ test/gatewayapi/test-canary.sh | 259 ++------------------------------- test/gatewayapi/test-utils.sh | 100 +++++++++++++ test/workloads/daemonset.yaml | 1 - test/workloads/deployment.yaml | 1 - test/workloads/init.sh | 6 +- test/workloads/secret.yaml | 1 - 10 files changed, 385 insertions(+), 258 deletions(-) create mode 100755 test/gatewayapi/test-ab.sh create mode 100755 test/gatewayapi/test-bg.sh create mode 100755 test/gatewayapi/test-utils.sh diff --git a/pkg/router/gateway_api.go b/pkg/router/gateway_api.go index 39220ac3..79bc975c 100644 --- a/pkg/router/gateway_api.go +++ b/pkg/router/gateway_api.go @@ -109,7 +109,7 @@ func (gwr *GatewayAPIRouter) Reconcile(canary *flaggerv1.Canary) error { if len(canary.GetAnalysis().Match) > 0 { analysisMatches, _ := gwr.mapRouteMatches(canary.GetAnalysis().Match) // serviceMatches, _ := gwr.mapRouteMatches(canary.Spec.Service.Match) - httpRouteSpec.Rules[0].Matches = analysisMatches + httpRouteSpec.Rules[0].Matches = gwr.mergeMatchConditions(analysisMatches, matches) httpRouteSpec.Rules = append(httpRouteSpec.Rules, v1alpha2.HTTPRouteRule{ Matches: matches, BackendRefs: []v1alpha2.HTTPBackendRef{ @@ -165,16 +165,17 @@ func (gwr *GatewayAPIRouter) Reconcile(canary *flaggerv1.Canary) error { } if httpRoute != nil { - if diff := cmp.Diff( + diff := cmp.Diff( httpRoute.Spec, httpRouteSpec, cmpopts.IgnoreFields(v1alpha2.BackendRef{}, "Weight"), - ); diff != "" { + ) + if diff != "" && httpRoute.Name != "" { hrClone := httpRoute.DeepCopy() hrClone.Spec = httpRouteSpec _, err := gwr.gatewayAPIClient.GatewayapiV1alpha2().HTTPRoutes(hrNamespace). Update(context.TODO(), hrClone, metav1.UpdateOptions{}) if err != nil { - return fmt.Errorf("HTTPRoute %s.%s update error: %w", hrClone.GetName(), hrNamespace, err) + return fmt.Errorf("HTTPRoute %s.%s update error: %w while reconciling", hrClone.GetName(), hrNamespace, err) } gwr.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)). Infof("HTTPProxy %s.%s updated", hrClone.GetName(), hrNamespace) @@ -275,7 +276,7 @@ func (gwr *GatewayAPIRouter) SetRoutes( // A/B testing if len(canary.GetAnalysis().Match) > 0 { analysisMatches, _ := gwr.mapRouteMatches(canary.GetAnalysis().Match) - hrClone.Spec.Rules[0].Matches = analysisMatches + hrClone.Spec.Rules[0].Matches = gwr.mergeMatchConditions(analysisMatches, matches) hrClone.Spec.Rules = append(hrClone.Spec.Rules, v1alpha2.HTTPRouteRule{ Matches: matches, BackendRefs: []v1alpha2.HTTPBackendRef{ @@ -288,8 +289,11 @@ func (gwr *GatewayAPIRouter) SetRoutes( _, err = gwr.gatewayAPIClient.GatewayapiV1alpha2().HTTPRoutes(hrNamespace).Update(context.TODO(), hrClone, metav1.UpdateOptions{}) if err != nil { - return fmt.Errorf("HTTPRoute %s.%s update error: %w", hrClone.GetName(), hrNamespace, err) + return fmt.Errorf("HTTPRoute %s.%s update error: %w while setting weights", hrClone.GetName(), hrNamespace, err) } + gwr.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)). + Infof("HTTPProxy %s.%s weights updated", hrClone.GetName(), hrNamespace) + return nil } @@ -386,3 +390,25 @@ func (gwr *GatewayAPIRouter) makeBackendRef(svcName string, weight, port int32) Weight: &weight, } } + +func (gwr *GatewayAPIRouter) mergeMatchConditions(analysis, service []v1alpha2.HTTPRouteMatch) []v1alpha2.HTTPRouteMatch { + if len(analysis) == 0 { + return service + } + + merged := make([]v1alpha2.HTTPRouteMatch, len(service)*len(analysis)) + num := 0 + for _, a := range analysis { + for _, s := range service { + merged[num] = *s.DeepCopy() + if len(a.Headers) > 0 { + merged[num].Headers = a.Headers + } + if len(a.QueryParams) > 0 { + merged[num].QueryParams = a.QueryParams + } + num++ + } + } + return merged +} diff --git a/test/gatewayapi/run.sh b/test/gatewayapi/run.sh index 67153fa9..8c8571f3 100755 --- a/test/gatewayapi/run.sh +++ b/test/gatewayapi/run.sh @@ -9,3 +9,5 @@ DIR="$(cd "$(dirname "$0")" && pwd)" "$REPO_ROOT"/test/workloads/init.sh "$DIR"/test-canary.sh +"$DIR"/test-bg.sh +"$DIR"/test-ab.sh diff --git a/test/gatewayapi/test-ab.sh b/test/gatewayapi/test-ab.sh new file mode 100755 index 00000000..1db74869 --- /dev/null +++ b/test/gatewayapi/test-ab.sh @@ -0,0 +1,125 @@ +#!/usr/bin/env bash + +# This script runs e2e tests for A/B traffic shifting, Canary analysis and promotion +# Prerequisites: Kubernetes Kind and Contour with GatewayAPI + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) + +source ${REPO_ROOT}/test/gatewayapi/test-utils.sh + +create_latency_metric_template +create_error_rate_metric_template + +echo '>>> Deploy podinfo in ab-test namespace' +kubectl create ns ab-test +kubectl apply -f ${REPO_ROOT}/test/workloads/secret.yaml -n ab-test +kubectl apply -f ${REPO_ROOT}/test/workloads/deployment.yaml -n ab-test + +cat <>> Triggering A/B testing' +kubectl -n ab-test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1 + +echo '>>> Waiting for A/B testing promotion' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n ab-test describe deployment/podinfo-primary | grep '3.1.1' && ok=true || ok=false + sleep 10 + kubectl -n flagger-system logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n ab-test describe deployment/podinfo + kubectl -n ab-test describe deployment/podinfo-primary + kubectl -n flagger-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +display_httproute "ab-test" + +echo '>>> Waiting for A/B finalization' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n ab-test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n flagger-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ A/B testing promotion test passed' + +kubectl delete -n ab-test canary podinfo diff --git a/test/gatewayapi/test-bg.sh b/test/gatewayapi/test-bg.sh new file mode 100755 index 00000000..7f0bc6d0 --- /dev/null +++ b/test/gatewayapi/test-bg.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash + +# This script runs e2e tests for Blue/Green traffic shifting, Canary analysis and promotion +# Prerequisites: Kubernetes Kind and Contour with GatewayAPI + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) + +source ${REPO_ROOT}/test/gatewayapi/test-utils.sh + +echo '>>> Deploy podinfo in bg-test namespace' +kubectl create ns bg-test +kubectl apply -f ${REPO_ROOT}/test/workloads/secret.yaml -n bg-test +kubectl apply -f ${REPO_ROOT}/test/workloads/deployment.yaml -n bg-test + +cat <>> Triggering B/G deployment' +kubectl -n bg-test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1 + +echo '>>> Waiting for B/G promotion' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n bg-test describe deployment/podinfo-primary | grep '3.1.1' && ok=true || ok=false + sleep 10 + kubectl -n flagger-system logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n bg-test describe deployment/podinfo + kubectl -n bg-test describe deployment/podinfo-primary + kubectl -n flagger-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +display_httproute "bg-test" + +echo '>>> Waiting for B/G finalization' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n bg-test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n flagger-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ B/G promotion test passed' + +kubectl delete -n bg-test canary podinfo diff --git a/test/gatewayapi/test-canary.sh b/test/gatewayapi/test-canary.sh index d29c55bc..1055ee0f 100755 --- a/test/gatewayapi/test-canary.sh +++ b/test/gatewayapi/test-canary.sh @@ -1,60 +1,16 @@ #!/usr/bin/env bash -# This script runs e2e tests for Canary, A/B initialization, analysis and promotion +# This script runs e2e tests for progressive traffic shifting, Canary analysis and promotion # Prerequisites: Kubernetes Kind and Contour with GatewayAPI set -o errexit -echo '>>> Create metric templates' -cat <>> Installing Canary' cat <>> Waiting for primary to be ready' -retries=50 -count=0 -ok=false -until ${ok}; do - kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false - sleep 5 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n flagger-system logs deployment/flagger - echo "No more retries left" - exit 1 - fi -done +check_primary "test" -echo '✔ Canary initialization test passed' - -passed=$(kubectl -n test get svc/podinfo -o jsonpath='{.spec.selector.app}' 2>&1 | { grep podinfo-primary || true; }) -if [ -z "$passed" ]; then - echo -e '\u2716 podinfo selector test failed' - exit 1 -fi - -echo '✔ Canary service custom metadata test passed' - -if ! kubectl -n test get httproute podinfo -oyaml; then - echo "Could not find HTTPRoute podinfo" - exit 1 -fi +display_httproute "test" echo '>>> Triggering canary deployment' kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1 @@ -156,6 +86,8 @@ until ${ok}; do fi done +display_httproute "test" + echo '>>> Waiting for canary finalization' retries=50 count=0 @@ -173,173 +105,6 @@ done echo '✔ Canary promotion test passed' -if [[ "$1" = "canary" ]]; then - exit 0 -fi - -cat <>> Triggering B/G deployment' -kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.2 - -echo '>>> Waiting for B/G promotion' -retries=50 -count=0 -ok=false -until ${ok}; do - kubectl -n test describe deployment/podinfo-primary | grep '3.1.2' && ok=true || ok=false - sleep 10 - kubectl -n flagger-system logs deployment/flagger --tail 1 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n test describe deployment/podinfo - kubectl -n test describe deployment/podinfo-primary - kubectl -n flagger-system logs deployment/flagger - echo "No more retries left" - exit 1 - fi -done - -echo '>>> Waiting for B/G finalization' -retries=50 -count=0 -ok=false -until ${ok}; do - kubectl -n test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false - sleep 5 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n flagger-system logs deployment/flagger - echo "No more retries left" - exit 1 - fi -done - -echo '✔ B/G promotion test passed' - -cat <>> Triggering A/B testing' -kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.3 - -echo '>>> Waiting for A/B testing promotion' -retries=50 -count=0 -ok=false -until ${ok}; do - kubectl -n test describe deployment/podinfo-primary | grep '3.1.3' && ok=true || ok=false - sleep 10 - kubectl -n flagger-system logs deployment/flagger --tail 1 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n test describe deployment/podinfo - kubectl -n test describe deployment/podinfo-primary - kubectl -n flagger-system logs deployment/flagger - echo "No more retries left" - exit 1 - fi -done - -echo '✔ A/B testing promotion test passed' - cat <>> Waiting for primary to be ready' + namespace=$1 + retries=50 + count=0 + ok=false + echo $namespace + until ${ok}; do + kubectl get -n ${namespace} canary/podinfo | grep 'Initialized' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n flagger-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi + done + + echo '✔ Canary initialization test passed' + + passed=$(kubectl -n $namespace get svc/podinfo -o jsonpath='{.spec.selector.app}' 2>&1 | { grep podinfo-primary || true; }) + if [ -z "$passed" ]; then + echo -e '\u2716 podinfo selector test failed' + exit 1 + fi + + echo '✔ Canary service custom metadata test passed' +} + +display_httproute() { + namespace=$1 + if ! kubectl -n ${namespace} get httproute podinfo -oyaml; then + echo "Could not find HTTPRoute podinfo in ${namespace} namespace" + exit 1 + fi +} + +create_latency_metric_template() { + if ! kubectl -n flagger-system get metrictemplates latency; then + echo '>>> Create latency metric template' + cat <>> Create latency metric template' + cat <>> Deploy podinfo' -kubectl apply -f ${REPO_ROOT}/test/workloads/secret.yaml -kubectl apply -f ${REPO_ROOT}/test/workloads/deployment.yaml -kubectl apply -f ${REPO_ROOT}/test/workloads/daemonset.yaml +kubectl apply -f ${REPO_ROOT}/test/workloads/secret.yaml -n test +kubectl apply -f ${REPO_ROOT}/test/workloads/deployment.yaml -n test +kubectl apply -f ${REPO_ROOT}/test/workloads/daemonset.yaml -n test diff --git a/test/workloads/secret.yaml b/test/workloads/secret.yaml index cf8ec3d6..0351e20c 100644 --- a/test/workloads/secret.yaml +++ b/test/workloads/secret.yaml @@ -2,6 +2,5 @@ apiVersion: v1 kind: Secret metadata: name: podinfo-secret - namespace: test stringData: value: s3cr3t