From 344bd45a0e08bcd864da405e8c994de24941282b Mon Sep 17 00:00:00 2001
From: stefanprodan <stefan.prodan@gmail.com>
Date: Fri, 10 May 2019 10:24:35 +0300
Subject: [PATCH] Add nginx e2e tests

---
 .circleci/config.yml    |  20 ++++-
 test/e2e-ingress.yaml   |  17 ++++
 test/e2e-nginx-build.sh |  24 ++++++
 test/e2e-nginx-tests.sh | 185 ++++++++++++++++++++++++++++++++++++++++
 test/e2e-nginx.sh       |  29 +++++++
 5 files changed, 273 insertions(+), 2 deletions(-)
 create mode 100644 test/e2e-ingress.yaml
 create mode 100755 test/e2e-nginx-build.sh
 create mode 100755 test/e2e-nginx-tests.sh
 create mode 100755 test/e2e-nginx.sh

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 7c254c55..f9164539 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -1,6 +1,6 @@
 version: 2.1
 jobs:
-  e2e-testing:
+  e2e-istio-testing:
     machine: true
     steps:
       - checkout
@@ -18,11 +18,20 @@ jobs:
       - run: test/e2e-build.sh supergloo:test.supergloo-system
       - run: test/e2e-tests.sh canary
 
+  e2e-nginx-testing:
+    machine: true
+    steps:
+      - checkout
+      - run: test/e2e-kind.sh
+      - run: test/e2e-nginx.sh
+      - run: test/e2e-nginx-build.sh
+      - run: test/e2e-nginx-tests.sh
+
 workflows:
   version: 2
   build-and-test:
     jobs:
-      - e2e-testing:
+      - e2e-istio-testing:
           filters:
             branches:
               ignore:
@@ -36,3 +45,10 @@ workflows:
                 - /gh-pages.*/
                 - /docs-.*/
                 - /release-.*/
+      - e2e-nginx-testing:
+          filters:
+            branches:
+              ignore:
+                - /gh-pages.*/
+                - /docs-.*/
+                - /release-.*/
\ No newline at end of file
diff --git a/test/e2e-ingress.yaml b/test/e2e-ingress.yaml
new file mode 100644
index 00000000..c5a6fa62
--- /dev/null
+++ b/test/e2e-ingress.yaml
@@ -0,0 +1,17 @@
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: podinfo
+  namespace: test
+  labels:
+    app: podinfo
+  annotations:
+    kubernetes.io/ingress.class: "nginx"
+spec:
+  rules:
+    - host: app.example.com
+      http:
+        paths:
+          - backend:
+              serviceName: podinfo
+              servicePort: 9898
diff --git a/test/e2e-nginx-build.sh b/test/e2e-nginx-build.sh
new file mode 100755
index 00000000..6ae5449a
--- /dev/null
+++ b/test/e2e-nginx-build.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+set -o errexit
+
+REPO_ROOT=$(git rev-parse --show-toplevel)
+export KUBECONFIG="$(kind get kubeconfig-path --name="kind")"
+
+echo '>>> Building Flagger'
+cd ${REPO_ROOT} && docker build -t test/flagger:latest . -f Dockerfile
+
+echo '>>> Installing Flagger'
+kind load docker-image test/flagger:latest
+
+echo '>>> Installing Flagger'
+helm upgrade -i flagger ${REPO_ROOT}/charts/flagger \
+--wait \
+--namespace ingress-nginx \
+--set prometheus.install=true \
+--set meshProvider=nginx
+
+kubectl -n ingress-nginx set image deployment/flagger flagger=test/flagger:latest
+
+kubectl -n ingress-nginx rollout status deployment/flagger
+kubectl -n ingress-nginx rollout status deployment/flagger-prometheus
diff --git a/test/e2e-nginx-tests.sh b/test/e2e-nginx-tests.sh
new file mode 100755
index 00000000..a4064e79
--- /dev/null
+++ b/test/e2e-nginx-tests.sh
@@ -0,0 +1,185 @@
+#!/usr/bin/env bash
+
+# This script runs e2e tests for Canary initialization, analysis and promotion
+# Prerequisites: Kubernetes Kind, Helm and Istio
+
+set -o errexit
+
+REPO_ROOT=$(git rev-parse --show-toplevel)
+export KUBECONFIG="$(kind get kubeconfig-path --name="kind")"
+
+echo '>>> Creating test namespace'
+kubectl create namespace test
+
+echo '>>> Installing load tester'
+kubectl -n test apply -f ${REPO_ROOT}/artifacts/loadtester/
+kubectl -n test rollout status deployment/flagger-loadtester
+
+echo '>>> Initialising canary'
+kubectl apply -f ${REPO_ROOT}/test/e2e-workload.yaml
+
+cat <<EOF | kubectl apply -f -
+apiVersion: flagger.app/v1alpha3
+kind: Canary
+metadata:
+  name: podinfo
+  namespace: test
+spec:
+  targetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: podinfo
+  progressDeadlineSeconds: 60
+  service:
+    port: 9898
+  canaryAnalysis:
+    interval: 15s
+    threshold: 15
+    maxWeight: 30
+    stepWeight: 10
+    metrics:
+    - name: request-success-rate
+      threshold: 99
+      interval: 1m
+    - name: "latency"
+      threshold: 0.5
+      interval: 1m
+      query: |
+        histogram_quantile(0.99,
+          sum(
+            rate(
+              http_request_duration_seconds_bucket{
+                kubernetes_namespace="test",
+                kubernetes_pod_name=~"podinfo-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)"
+              }[1m]
+            )
+          ) by (le)
+        )
+    webhooks:
+      - name: load-test
+        url: http://flagger-loadtester.test/
+        timeout: 5s
+        metadata:
+          type: cmd
+          cmd: "hey -z 10m -q 10 -c 2 -host app.example.com http://nginx-ingress-controller.ingress-nginx"
+          logCmdOutput: "true"
+EOF
+
+echo '>>> Waiting for primary to be ready'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false
+    sleep 5
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n ingress-nginx logs deployment/flagger
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+echo '✔ Canary initialization test passed'
+
+echo '>>> Triggering canary deployment'
+kubectl -n test set image deployment/podinfo podinfod=quay.io/stefanprodan/podinfo:1.4.1
+
+echo '>>> Waiting for canary promotion'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n test describe deployment/podinfo-primary | grep '1.4.1' && ok=true || ok=false
+    sleep 10
+    kubectl -n ingress-nginx logs deployment/flagger --tail 1
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n test describe deployment/podinfo
+        kubectl -n test describe deployment/podinfo-primary
+        kubectl -n ingress-nginx logs deployment/flagger
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+echo '✔ Canary promotion test passed'
+
+if [ "$1" = "canary" ]; then
+  exit 0
+fi
+
+cat <<EOF | kubectl apply -f -
+apiVersion: flagger.app/v1alpha3
+kind: Canary
+metadata:
+  name: podinfo
+  namespace: test
+spec:
+  targetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: podinfo
+  progressDeadlineSeconds: 60
+  service:
+    port: 9898
+  canaryAnalysis:
+    interval: 10s
+    threshold: 5
+    iterations: 5
+    match:
+      - headers:
+          x-canary:
+            exact: "insider"
+      - headers:
+          cookie:
+            exact: "canary"
+    metrics:
+    - name: request-success-rate
+      threshold: 99
+      interval: 1m
+    webhooks:
+      - name: pre
+        type: pre-rollout
+        url: http://flagger-loadtester.test/
+        timeout: 5s
+        metadata:
+          type: cmd
+          cmd: "hey -z 10m -q 10 -c 2 -H 'X-Canary: insider' -host app.example.com http://nginx-ingress-controller.ingress-nginx"
+          logCmdOutput: "true"
+      - name: post
+        type: post-rollout
+        url: http://flagger-loadtester.test/
+        timeout: 15s
+        metadata:
+          type: cmd
+          cmd: "curl -sH 'Host: app.example.com' http://nginx-ingress-controller.ingress-nginx"
+          logCmdOutput: "true"
+EOF
+
+echo '>>> Triggering A/B testing'
+kubectl -n test set image deployment/podinfo podinfod=quay.io/stefanprodan/podinfo:1.4.2
+
+echo '>>> Waiting for A/B testing promotion'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n test describe deployment/podinfo-primary | grep '1.4.2' && ok=true || ok=false
+    sleep 10
+    kubectl -n ingress-nginx logs deployment/flagger --tail 1
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n test describe deployment/podinfo
+        kubectl -n test describe deployment/podinfo-primary
+        kubectl -n ingress-nginx logs deployment/flagger
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+echo '✔ A/B testing promotion test passed'
+
+kubectl -n ingress-nginx logs deployment/flagger
+
+echo '✔ All tests passed'
\ No newline at end of file
diff --git a/test/e2e-nginx.sh b/test/e2e-nginx.sh
new file mode 100755
index 00000000..3fa97bb7
--- /dev/null
+++ b/test/e2e-nginx.sh
@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+set -o errexit
+
+REPO_ROOT=$(git rev-parse --show-toplevel)
+export KUBECONFIG="$(kind get kubeconfig-path --name="kind")"
+
+echo ">>> Installing Helm"
+curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get | bash
+
+echo '>>> Installing Tiller'
+kubectl --namespace kube-system create sa tiller
+kubectl create clusterrolebinding tiller-cluster-rule --clusterrole=cluster-admin --serviceaccount=kube-system:tiller
+helm init --service-account tiller --upgrade --wait
+
+echo '>>> Installing NGINX Ingress'
+helm upgrade -i nginx-ingress stable/nginx-ingress \
+--wait \
+--namespace ingress-nginx \
+--set controller.stats.enabled=true \
+--set controller.metrics.enabled=true \
+--set controller.podAnnotations."prometheus\.io/scrape"=true \
+--set controller.podAnnotations."prometheus\.io/port"=10254 \
+--set controller.service.type=NodePort
+
+kubectl -n ingress-nginx rollout status deployment/nginx-ingress-controller
+kubectl -n ingress-nginx get all
+
+