From adeb585de1548c3411a2eb6c2c78d28ab6a4536e Mon Sep 17 00:00:00 2001
From: nmlc <public.nmlc@gmail.com>
Date: Wed, 25 Nov 2020 07:55:05 +0500
Subject: [PATCH] [traefik] add e2e test

---
 test/e2e-traefik-tests.sh | 156 ++++++++++++++++++++++++++++++++++++++
 test/e2e-traefik.sh       |  37 +++++++++
 2 files changed, 193 insertions(+)
 create mode 100755 test/e2e-traefik-tests.sh
 create mode 100755 test/e2e-traefik.sh

diff --git a/test/e2e-traefik-tests.sh b/test/e2e-traefik-tests.sh
new file mode 100755
index 00000000..e983d619
--- /dev/null
+++ b/test/e2e-traefik-tests.sh
@@ -0,0 +1,156 @@
+#!/usr/bin/env bash
+
+# This script runs e2e tests for Canary initialization, analysis and promotion
+# Prerequisites: Kubernetes Kind, Helm and Traefik ingress controller
+
+set -o errexit
+
+REPO_ROOT=$(git rev-parse --show-toplevel)
+
+echo '>>> Creating test namespace'
+kubectl create namespace test
+
+echo '>>> Installing load tester'
+kubectl apply -k ${REPO_ROOT}/kustomize/tester
+kubectl -n test rollout status deployment/flagger-loadtester
+
+echo '>>> Initialising canary'
+kubectl apply -f ${REPO_ROOT}/test/e2e-workload.yaml
+
+cat <<EOF | kubectl apply -f -
+apiVersion: traefik.containo.us/v1alpha1
+kind: IngressRoute
+metadata:
+  name: podinfo
+  namespace: test
+spec:
+  entryPoints:
+    - web
+  routes:
+    - match: Host(`app.example.com`)
+      kind: Rule
+      services:
+        - name: podinfo
+          kind: TraefikService
+          port: 80
+EOF
+
+cat <<EOF | kubectl apply -f -
+apiVersion: flagger.app/v1beta1
+kind: Canary
+metadata:
+  name: podinfo
+  namespace: test
+spec:
+  provider: traefik
+  targetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: podinfo
+  progressDeadlineSeconds: 60
+  service:
+    port: 80
+    targetPort: 9898
+    traefikService:
+      labels:
+        test: test-label
+      annotations:
+        test: test-annotation
+  analysis:
+    interval: 60s
+    threshold: 15
+    maxWeight: 60
+    stepWeight: 10
+    metrics:
+    - name: request-success-rate
+      threshold: 99
+      interval: 1m
+    - name: request-duration
+      threshold: 500
+      interval: 1m
+    webhooks:
+      - name: "gate"
+        type: confirm-rollout
+        url: http://flagger-loadtester.test/gate/approve
+      - name: acceptance-test
+        type: pre-rollout
+        url: http://flagger-loadtester.test/
+        timeout: 10s
+        metadata:
+          type: bash
+          cmd: "curl -sd 'test' http://podinfo-canary.test/token | grep token"
+      - name: traefik-acceptance-test
+        type: pre-rollout
+        url: http://flagger-loadtester.test/
+        timeout: 10s
+        metadata:
+          type: bash
+          cmd: "curl -sd 'test' -H 'Host: app.example.com' http://traefik.traefik/token | grep token"
+      - name: load-test
+        url: http://flagger-loadtester.test/
+        timeout: 5s
+        metadata:
+          type: cmd
+          cmd: "hey -z 2m -q 5 -c 2 -host app.example.com http://traefik.traefik"
+          logCmdOutput: "true"
+EOF
+
+echo '>>> Waiting for primary to be ready'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false
+    sleep 5
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n traefik logs deployment/flagger
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+echo '✔ Canary initialization test passed'
+
+passed=$(kubectl -n test get svc/podinfo -o jsonpath='{.spec.selector.app}' 2>&1 | { grep podinfo-primary || true; })
+if [ -z "$passed" ]; then
+  echo -e '\u2716 podinfo selector test failed'
+  exit 1
+fi
+passed=$(kubectl -n test get traefikservice/podinfo -o jsonpath='{.metadata.labels}' 2>&1 | { grep test-label || true; })
+if [ -z "$passed" ]; then
+  echo -e '\u2716 TraefikService does not have required labels'
+  exit 1
+fi
+passed=$(kubectl -n test get traefikservice/podinfo -o jsonpath='{.metadata.annotations}' 2>&1 | { grep test-annotation || true; })
+if [ -z "$passed" ]; then
+  echo -e '\u2716 TraefikService does not have required annotations'
+  exit 1
+fi
+
+echo '✔ Canary service custom metadata test passed'
+
+echo '>>> Triggering canary deployment'
+kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1
+
+echo '>>> Waiting for canary promotion'
+retries=60
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n test describe deployment/podinfo-primary | grep '3.1.1' && ok=true || ok=false
+    sleep 10
+    kubectl -n traefik logs deployment/flagger --tail 1
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n test describe deployment/podinfo
+        kubectl -n test describe deployment/podinfo-primary
+        kubectl -n test logs deployment/flagger-loadtester
+        kubectl -n traefik logs deployment/flagger
+        kubectl -n traefik get all
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+echo '✔ Canary promotion test passed'
diff --git a/test/e2e-traefik.sh b/test/e2e-traefik.sh
new file mode 100755
index 00000000..74b9ec9d
--- /dev/null
+++ b/test/e2e-traefik.sh
@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+set -o errexit
+
+REPO_ROOT=$(git rev-parse --show-toplevel)
+TRAEFIK_CHART_VERSION="9.11.0" # traefik 2.3.3
+
+echo '>>> Creating traefik namespace'
+kubectl create ns traefik
+
+echo '>>> Installing Traefik'
+# helm repo add traefik https://helm.traefik.io/traefik
+cat <<EOF | helm upgrade -i traefik traefik/traefik --version=${TRAEFIK_VERSION} --namespace traefik -f -
+additionalArguments:
+  - "--metrics.prometheus=true"
+deployment:
+  podAnnotations:
+    "prometheus.io/port": "9000"
+    "prometheus.io/scrape": "true"
+EOF
+
+kubectl -n traefik rollout status deployment/traefik
+kubectl -n traefik get all
+
+echo '>>> Loading Flagger image'
+kind load docker-image test/flagger:latest
+
+echo '>>> Installing Flagger'
+helm upgrade -i flagger ${REPO_ROOT}/charts/flagger \
+--set crd.create=false \
+--namespace traefik \
+--set prometheus.install=true \
+--set meshProvider=traefik \
+--set image.repository=test\/flagger \
+--set image.tag=latest \
+
+kubectl -n traefik rollout status deployment/flagger