add e2e tests and helper functions for router

Signed-off-by: Gallardot <tttick@163.com>

.github/workflows/e2e.yaml

@@ -33,6 +33,7 @@ jobs:
           - kubernetes
           - gatewayapi
           - keda
+          - apisix
     steps:
       - name: Checkout
         uses: actions/checkout@v3

pkg/router/apisix.go

@@ -60,22 +60,9 @@ func (ar *ApisixRouter) Reconcile(canary *flaggerv1.Canary) error {
 	}
 
 	apexName, primaryName, canaryName := canary.GetServiceNames()
-	var targetHttpRoute *a6v2.ApisixRouteHTTP
-	var targetIndex int
-	for index, item := range apisixRouteClone.Spec.HTTP {
-		for _, backend := range item.Backends {
-			if backend.ServiceName == apexName {
-				targetHttpRoute = &item
-				targetIndex = index
-				goto found
-			}
-		}
-	}
-
-found:
-	if targetHttpRoute == nil {
-		return fmt.Errorf("Can not find %s backend on apisix route %s.%s ",
-			primaryName, canary.Spec.RouteRef.Name, canary.Namespace)
+	targetHttpRoute, targetIndex, err := ar.getTargetHttpRoute(canary, apisixRouteClone, apexName)
+	if err != nil {
+		return err
 	}
 	if len(targetHttpRoute.Backends) != 1 {
 		return fmt.Errorf("APISIX route %s.%s's http route %s only one http backend is supported",
@@ -154,6 +141,19 @@ found:
 	return nil
 }
 
+func (ar *ApisixRouter) getTargetHttpRoute(canary *flaggerv1.Canary, apisixRoute *a6v2.ApisixRoute, serviceName string) (*a6v2.ApisixRouteHTTP, int, error) {
+	for index, item := range apisixRoute.Spec.HTTP {
+		for _, backend := range item.Backends {
+			if backend.ServiceName == serviceName {
+				return &item, index, nil
+			}
+		}
+	}
+
+	return nil, 0, fmt.Errorf("Can not find %s backend on apisix route %s.%s ",
+		serviceName, canary.Spec.RouteRef.Name, canary.Namespace)
+}
+
 // GetRoutes returns the destinations weight for primary and canary
 func (ar *ApisixRouter) GetRoutes(canary *flaggerv1.Canary) (
 	primaryWeight int,
@@ -168,8 +168,12 @@ func (ar *ApisixRouter) GetRoutes(canary *flaggerv1.Canary) (
 		err = fmt.Errorf("apisix route %s.%s query error: %w", canaryApisixRouteName, canary.Namespace, err)
 		return
 	}
+	_, targetIndex, err := ar.getTargetHttpRoute(canary, apisixRoute, primaryName)
+	if err != nil {
+		return
+	}
 
-	for _, backend := range apisixRoute.Spec.HTTP[0].Backends {
+	for _, backend := range apisixRoute.Spec.HTTP[targetIndex].Backends {
 		if backend.ServiceName == primaryName {
 			primaryWeight = *backend.Weight
 			canaryWeight = 100 - primaryWeight
@@ -199,7 +203,11 @@ func (ar *ApisixRouter) SetRoutes(
 		return fmt.Errorf("apisix route %s.%s query error: %w", canaryApisixRouteName, canary.Namespace, err)
 	}
 
-	backends := apisixRoute.Spec.HTTP[0].Backends
+	_, targetIndex, err := ar.getTargetHttpRoute(canary, apisixRoute, primaryName)
+	if err != nil {
+		return err
+	}
+	backends := apisixRoute.Spec.HTTP[targetIndex].Backends
 	for i, backend := range backends {
 		if backend.ServiceName == primaryName {
 			backends[i].Weight = &primaryWeight
@@ -207,7 +215,7 @@ func (ar *ApisixRouter) SetRoutes(
 			backends[i].Weight = &canaryWeight
 		}
 	}
-	apisixRoute.Spec.HTTP[0].Backends = backends
+	apisixRoute.Spec.HTTP[targetIndex].Backends = backends
 
 	_, err = ar.apisixClient.ApisixV2().ApisixRoutes(canary.Namespace).Update(context.TODO(), apisixRoute, metav1.UpdateOptions{})
 	if err != nil {

test/apisix/install.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+set -o errexit
+
+APISIX_CHART_VERSION="0.11.3" # apisix 2.15.1
+REPO_ROOT=$(git rev-parse --show-toplevel)
+
+mkdir -p ${REPO_ROOT}/bin
+
+echo '>>> Creating apisix namespace'
+kubectl create ns apisix
+
+echo '>>> Installing APISIX'
+helm repo add apisix https://charts.apiseven.com
+
+helm upgrade -i apisix apisix/apisix --version=${APISIX_CHART_VERSION} \
+--namespace apisix \
+--set apisix.podAnnotations."prometheus\.io/scrape"=true \
+--set apisix.podAnnotations."prometheus\.io/port"=9091 \
+--set apisix.podAnnotations."prometheus\.io/path"=/apisix/prometheus/metrics \
+--set pluginAttrs.prometheus.export_addr.ip=0.0.0.0 \
+--set pluginAttrs.prometheus.export_addr.port=9091 \
+--set pluginAttrs.prometheus.export_uri=/apisix/prometheus/metrics \
+--set pluginAttrs.prometheus.metric_prefix=apisix_ \
+--set ingress-controller.enabled=true \
+--set ingress-controller.config.apisix.serviceNamespace=apisix
+
+kubectl -n apisix rollout status deployment/apisix
+kubectl -n apisix get all
+
+echo '>>> Installing Flagger'
+helm upgrade -i flagger ${REPO_ROOT}/charts/flagger \
+--set crd.create=false \
+--namespace apisix \
+--set prometheus.install=true \
+--set meshProvider=apisix \
+--set image.repository=test\/flagger \
+--set image.tag=latest \
+
+kubectl -n apisix get all

test/apisix/run.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -o errexit
+
+REPO_ROOT=$(git rev-parse --show-toplevel)
+DIR="$(cd "$(dirname "$0")" && pwd)"
+
+"$DIR"/install.sh
+
+"$REPO_ROOT"/test/workloads/init.sh
+"$DIR"/test-canary.sh

test/apisix/test-canary.sh

@@ -0,0 +1,146 @@
+#!/usr/bin/env bash
+
+# This script runs e2e tests for Canary initialization, analysis and promotion
+# Prerequisites: Kubernetes Kind, Helm and Apisix ingress controller
+
+set -o errexit
+
+REPO_ROOT=$(git rev-parse --show-toplevel)
+
+cat <<EOF | kubectl apply -f -
+apiVersion: apisix.apache.org/v2
+kind: ApisixRoute
+metadata:
+  name: podinfo
+  namespace: test
+spec:
+  http:
+    - backends:
+        - serviceName: podinfo
+          servicePort: 80
+      match:
+        hosts:
+          - foobar.com
+        methods:
+          - GET
+        paths:
+          - /*
+      name: method
+      plugins:
+        - name: prometheus
+          enable: true
+          config:
+            disable: false
+            prefer_name: true
+EOF
+
+cat <<EOF | kubectl apply -f -
+apiVersion: flagger.app/v1beta1
+kind: Canary
+metadata:
+  name: podinfo
+  namespace: test
+spec:
+  provider: apisix
+  targetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: podinfo
+  # apisix route reference
+  routeRef:
+    apiVersion: apisix.apache.org/v2
+    kind: ApisixRoute
+    name: podinfo
+  # the maximum time in seconds for the canary deployment
+  # to make progress before it is rollback (default 600s)
+  progressDeadlineSeconds: 60
+  service:
+    # ClusterIP port number
+    port: 80
+    # container port number or name
+    targetPort: 9898
+  analysis:
+    # schedule interval (default 60s)
+    interval: 10s
+    # max number of failed metric checks before rollback
+    threshold: 10
+    # max traffic percentage routed to canary
+    # percentage (0-100)
+    maxWeight: 50
+    # canary increment step
+    # percentage (0-100)
+    stepWeight: 10
+    # APISIX Prometheus checks
+    metrics:
+    - name: request-success-rate
+      # minimum req success rate (non 5xx responses)
+      # percentage (0-100)
+      thresholdRange:
+        min: 99
+      interval: 1m
+    - name: request-duration
+      # builtin Prometheus check
+      # maximum req duration P99
+      # milliseconds
+      thresholdRange:
+        max: 500
+      interval: 30s
+    webhooks:
+      - name: load-test
+        url: http://flagger-loadtester.test/
+        timeout: 5s
+        type: rollout
+        metadata:
+          cmd: |-
+              hey -z 1m -q 10 -c 2 -h2 -host foobar.com http://apisix-gateway.apisix/api/info
+EOF
+
+echo '>>> Waiting for primary to be ready'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false
+    sleep 5
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n apisix logs deployment/flagger
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+echo '✔ Canary initialization test passed'
+
+passed=$(kubectl -n test get svc/podinfo -o jsonpath='{.spec.selector.app}' 2>&1 | { grep podinfo-primary || true; })
+if [ -z "$passed" ]; then
+  echo -e '\u2716 podinfo selector test failed'
+  exit 1
+fi
+
+echo '✔ Canary service custom metadata test passed'
+
+echo '>>> Triggering canary deployment'
+kubectl -n test set image deployment/podinfo podinfod=ghcr.io/stefanprodan/podinfo:6.0.1
+
+echo '>>> Waiting for canary promotion'
+retries=60
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n test describe deployment/podinfo-primary | grep '6.0.1' && ok=true || ok=false
+    sleep 10
+    kubectl -n apisix logs deployment/flagger --tail 1
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n test describe deployment/podinfo
+        kubectl -n test describe deployment/podinfo-primary
+        kubectl -n test logs deployment/flagger-loadtester
+        kubectl -n apisix logs deployment/flagger
+        kubectl -n apisix get all
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+echo '✔ Canary promotion test passed'