Add Blue/Green e2e tests

.circleci/config.yml

@@ -78,6 +78,17 @@ jobs:
       - run: test/e2e-istio.sh
       - run: test/e2e-tests.sh
 
+  e2e-kubernetes-testing:
+    machine: true
+    steps:
+      - checkout
+      - attach_workspace:
+          at: /tmp/bin
+      - run: test/container-build.sh
+      - run: test/e2e-kind.sh
+      - run: test/e2e-kubernetes.sh
+      - run: test/e2e-kubernetes-tests.sh
+
   e2e-smi-istio-testing:
     machine: true
     steps:
@@ -145,7 +156,7 @@ workflows:
       - e2e-istio-testing:
           requires:
             - build-binary
-      - e2e-smi-istio-testing:
+      - e2e-kubernetes-testing:
           requires:
             - build-binary
 #      - e2e-supergloo-testing:
@@ -164,7 +175,7 @@ workflows:
           requires:
             - build-binary
             - e2e-istio-testing
-            - e2e-smi-istio-testing
+            - e2e-kubernetes-testing
             #- e2e-supergloo-testing
             - e2e-gloo-testing
             - e2e-nginx-testing

test/e2e-kubernetes-tests.sh

@@ -0,0 +1,98 @@
+#!/usr/bin/env bash
+
+# This script runs e2e tests for Blue/Green initialization, analysis and promotion
+# Prerequisites: Kubernetes Kind, Kustomize
+
+set -o errexit
+
+REPO_ROOT=$(git rev-parse --show-toplevel)
+export KUBECONFIG="$(kind get kubeconfig-path --name="kind")"
+
+echo '>>> Creating test namespace'
+kubectl create namespace test
+
+echo '>>> Installing the load tester'
+kubectl apply -k ${REPO_ROOT}/kustomize/tester
+kubectl -n test rollout status deployment/flagger-loadtester
+
+echo '>>> Initialising canary'
+kubectl apply -f ${REPO_ROOT}/test/e2e-workload.yaml
+
+cat <<EOF | kubectl apply -f -
+apiVersion: flagger.app/v1alpha3
+kind: Canary
+metadata:
+  name: podinfo
+  namespace: test
+spec:
+  provider: kubernetes
+  targetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: podinfo
+  progressDeadlineSeconds: 60
+  service:
+    port: 9898
+    portDiscovery: true
+  canaryAnalysis:
+    interval: 15s
+    threshold: 10
+    iterations: 5
+    metrics:
+    - name: request-success-rate
+      threshold: 99
+      interval: 1m
+    - name: request-duration
+      threshold: 500
+      interval: 30s
+    webhooks:
+      - name: load-test
+        url: http://flagger-loadtester.test/
+        timeout: 5s
+        metadata:
+          type: cmd
+          cmd: "hey -z 10m -q 10 -c 2 http://podinfo-canary.test:9898/"
+          logCmdOutput: "true"
+EOF
+
+echo '>>> Waiting for primary to be ready'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false
+    sleep 5
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n flagger-system logs deployment/flagger
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+echo '✔ Canary initialization test passed'
+
+echo '>>> Triggering canary deployment'
+kubectl -n test set image deployment/podinfo podinfod=quay.io/stefanprodan/podinfo:1.7.0
+
+echo '>>> Waiting for canary promotion'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n test describe deployment/podinfo-primary | grep '1.7.0' && ok=true || ok=false
+    sleep 10
+    kubectl -n flagger-system logs deployment/flagger --tail 1
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n test describe deployment/podinfo
+        kubectl -n test describe deployment/podinfo-primary
+        kubectl -n flagger-system logs deployment/flagger
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+echo '✔ Canary promotion test passed'
+
+kubectl -n flagger-system logs deployment/flagger

test/e2e-kubernetes.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -o errexit
+
+REPO_ROOT=$(git rev-parse --show-toplevel)
+export KUBECONFIG="$(kind get kubeconfig-path --name="kind")"
+
+echo '>>> Loading Flagger image'
+kind load docker-image test/flagger:latest
+
+echo '>>> Installing Flagger'
+kubectl apply -k ${REPO_ROOT}/kustomize/kubernetes
+
+kubectl -n flagger-system set image deployment/flagger flagger=test/flagger:latest
+
+kubectl -n flagger-system rollout status deployment/flagger
+kubectl -n flagger-system rollout status deployment/flagger-prometheus