diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 1e952573..cda3a344 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -33,6 +33,7 @@ jobs: - kubernetes - gatewayapi - keda + - apisix steps: - name: Checkout uses: actions/checkout@v3 diff --git a/pkg/router/apisix.go b/pkg/router/apisix.go index fc3bee57..6974c2a8 100644 --- a/pkg/router/apisix.go +++ b/pkg/router/apisix.go @@ -60,22 +60,9 @@ func (ar *ApisixRouter) Reconcile(canary *flaggerv1.Canary) error { } apexName, primaryName, canaryName := canary.GetServiceNames() - var targetHttpRoute *a6v2.ApisixRouteHTTP - var targetIndex int - for index, item := range apisixRouteClone.Spec.HTTP { - for _, backend := range item.Backends { - if backend.ServiceName == apexName { - targetHttpRoute = &item - targetIndex = index - goto found - } - } - } - -found: - if targetHttpRoute == nil { - return fmt.Errorf("Can not find %s backend on apisix route %s.%s ", - primaryName, canary.Spec.RouteRef.Name, canary.Namespace) + targetHttpRoute, targetIndex, err := ar.getTargetHttpRoute(canary, apisixRouteClone, apexName) + if err != nil { + return err } if len(targetHttpRoute.Backends) != 1 { return fmt.Errorf("APISIX route %s.%s's http route %s only one http backend is supported", @@ -154,6 +141,19 @@ found: return nil } +func (ar *ApisixRouter) getTargetHttpRoute(canary *flaggerv1.Canary, apisixRoute *a6v2.ApisixRoute, serviceName string) (*a6v2.ApisixRouteHTTP, int, error) { + for index, item := range apisixRoute.Spec.HTTP { + for _, backend := range item.Backends { + if backend.ServiceName == serviceName { + return &item, index, nil + } + } + } + + return nil, 0, fmt.Errorf("Can not find %s backend on apisix route %s.%s ", + serviceName, canary.Spec.RouteRef.Name, canary.Namespace) +} + // GetRoutes returns the destinations weight for primary and canary func (ar *ApisixRouter) GetRoutes(canary *flaggerv1.Canary) ( primaryWeight int, @@ -168,8 +168,12 @@ func (ar *ApisixRouter) GetRoutes(canary *flaggerv1.Canary) ( err = fmt.Errorf("apisix route %s.%s query error: %w", canaryApisixRouteName, canary.Namespace, err) return } + _, targetIndex, err := ar.getTargetHttpRoute(canary, apisixRoute, primaryName) + if err != nil { + return + } - for _, backend := range apisixRoute.Spec.HTTP[0].Backends { + for _, backend := range apisixRoute.Spec.HTTP[targetIndex].Backends { if backend.ServiceName == primaryName { primaryWeight = *backend.Weight canaryWeight = 100 - primaryWeight @@ -199,7 +203,11 @@ func (ar *ApisixRouter) SetRoutes( return fmt.Errorf("apisix route %s.%s query error: %w", canaryApisixRouteName, canary.Namespace, err) } - backends := apisixRoute.Spec.HTTP[0].Backends + _, targetIndex, err := ar.getTargetHttpRoute(canary, apisixRoute, primaryName) + if err != nil { + return err + } + backends := apisixRoute.Spec.HTTP[targetIndex].Backends for i, backend := range backends { if backend.ServiceName == primaryName { backends[i].Weight = &primaryWeight @@ -207,7 +215,7 @@ func (ar *ApisixRouter) SetRoutes( backends[i].Weight = &canaryWeight } } - apisixRoute.Spec.HTTP[0].Backends = backends + apisixRoute.Spec.HTTP[targetIndex].Backends = backends _, err = ar.apisixClient.ApisixV2().ApisixRoutes(canary.Namespace).Update(context.TODO(), apisixRoute, metav1.UpdateOptions{}) if err != nil { diff --git a/test/apisix/install.sh b/test/apisix/install.sh new file mode 100755 index 00000000..df772097 --- /dev/null +++ b/test/apisix/install.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +set -o errexit + +APISIX_CHART_VERSION="0.11.3" # apisix 2.15.1 +REPO_ROOT=$(git rev-parse --show-toplevel) + +mkdir -p ${REPO_ROOT}/bin + +echo '>>> Creating apisix namespace' +kubectl create ns apisix + +echo '>>> Installing APISIX' +helm repo add apisix https://charts.apiseven.com + +helm upgrade -i apisix apisix/apisix --version=${APISIX_CHART_VERSION} \ +--namespace apisix \ +--set apisix.podAnnotations."prometheus\.io/scrape"=true \ +--set apisix.podAnnotations."prometheus\.io/port"=9091 \ +--set apisix.podAnnotations."prometheus\.io/path"=/apisix/prometheus/metrics \ +--set pluginAttrs.prometheus.export_addr.ip=0.0.0.0 \ +--set pluginAttrs.prometheus.export_addr.port=9091 \ +--set pluginAttrs.prometheus.export_uri=/apisix/prometheus/metrics \ +--set pluginAttrs.prometheus.metric_prefix=apisix_ \ +--set ingress-controller.enabled=true \ +--set ingress-controller.config.apisix.serviceNamespace=apisix + +kubectl -n apisix rollout status deployment/apisix +kubectl -n apisix get all + +echo '>>> Installing Flagger' +helm upgrade -i flagger ${REPO_ROOT}/charts/flagger \ +--set crd.create=false \ +--namespace apisix \ +--set prometheus.install=true \ +--set meshProvider=apisix \ +--set image.repository=test\/flagger \ +--set image.tag=latest \ + +kubectl -n apisix get all diff --git a/test/apisix/run.sh b/test/apisix/run.sh new file mode 100755 index 00000000..67153fa9 --- /dev/null +++ b/test/apisix/run.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) +DIR="$(cd "$(dirname "$0")" && pwd)" + +"$DIR"/install.sh + +"$REPO_ROOT"/test/workloads/init.sh +"$DIR"/test-canary.sh diff --git a/test/apisix/test-canary.sh b/test/apisix/test-canary.sh new file mode 100755 index 00000000..fc6d7115 --- /dev/null +++ b/test/apisix/test-canary.sh @@ -0,0 +1,146 @@ +#!/usr/bin/env bash + +# This script runs e2e tests for Canary initialization, analysis and promotion +# Prerequisites: Kubernetes Kind, Helm and Apisix ingress controller + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) + +cat <>> Waiting for primary to be ready' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n apisix logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary initialization test passed' + +passed=$(kubectl -n test get svc/podinfo -o jsonpath='{.spec.selector.app}' 2>&1 | { grep podinfo-primary || true; }) +if [ -z "$passed" ]; then + echo -e '\u2716 podinfo selector test failed' + exit 1 +fi + +echo '✔ Canary service custom metadata test passed' + +echo '>>> Triggering canary deployment' +kubectl -n test set image deployment/podinfo podinfod=ghcr.io/stefanprodan/podinfo:6.0.1 + +echo '>>> Waiting for canary promotion' +retries=60 +count=0 +ok=false +until ${ok}; do + kubectl -n test describe deployment/podinfo-primary | grep '6.0.1' && ok=true || ok=false + sleep 10 + kubectl -n apisix logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n test describe deployment/podinfo + kubectl -n test describe deployment/podinfo-primary + kubectl -n test logs deployment/flagger-loadtester + kubectl -n apisix logs deployment/flagger + kubectl -n apisix get all + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary promotion test passed'