From 013949a9f45576b63995bcd8a6afe80226e468f6 Mon Sep 17 00:00:00 2001
From: Daniel Albuquerque <dmartinsalbuqu@expediagroup.com>
Date: Fri, 18 Sep 2020 17:59:16 +0100
Subject: [PATCH] Add tests for when canary analysis is skipped

---
 .circleci/config.yml                  |   1 +
 pkg/apis/flagger/v1beta1/canary.go    |   4 +-
 test/e2e-istio-tests-skip-analysis.sh | 192 ++++++++++++++++++++++++++
 3 files changed, 196 insertions(+), 1 deletion(-)
 create mode 100755 test/e2e-istio-tests-skip-analysis.sh

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 09c6a598..1ab3bd28 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -95,6 +95,7 @@ jobs:
       - run: test/e2e-kind.sh v1.18.2
       - run: test/e2e-istio.sh
       - run: test/e2e-istio-tests.sh
+      - run: test/e2e-istio-tests-skip-analysis.sh
 
   e2e-gloo-testing:
     machine: true
diff --git a/pkg/apis/flagger/v1beta1/canary.go b/pkg/apis/flagger/v1beta1/canary.go
index 3451e796..52bc64a4 100644
--- a/pkg/apis/flagger/v1beta1/canary.go
+++ b/pkg/apis/flagger/v1beta1/canary.go
@@ -428,7 +428,9 @@ func (c *Canary) GetMetricInterval() string {
 // SkipAnalysis returns true if the analysis is nil
 // or if spec.SkipAnalysis is true
 func (c *Canary) SkipAnalysis() bool {
-	if c.Spec.Analysis == nil && c.Spec.CanaryAnalysis == nil {
+	// log.Printf("#1 SkipAnalysis, analysis=%v canaryanalysis=%v", c.Spec.Analysis, c.Spec.CanaryAnalysis)
+
+	if c.Spec.Analysis == nil || c.Spec.CanaryAnalysis == nil {
 		return true
 	}
 	return c.Spec.SkipAnalysis
diff --git a/test/e2e-istio-tests-skip-analysis.sh b/test/e2e-istio-tests-skip-analysis.sh
new file mode 100755
index 00000000..7ad419dd
--- /dev/null
+++ b/test/e2e-istio-tests-skip-analysis.sh
@@ -0,0 +1,192 @@
+#!/usr/bin/env bash
+
+# This script runs e2e tests for when the canary analysis is skipped
+# Prerequisites: Kubernetes Kind and Istio
+
+set -o errexit
+
+REPO_ROOT=$(git rev-parse --show-toplevel)
+
+echo '>>> Creating test namespace'
+kubectl create namespace test
+kubectl label namespace test istio-injection=enabled
+
+echo '>>> Installing the load tester'
+kubectl apply -k ${REPO_ROOT}/kustomize/tester
+kubectl -n test rollout status deployment/flagger-loadtester
+
+echo '>>> Deploy podinfo'
+kubectl apply -f ${REPO_ROOT}/test/e2e-workload.yaml
+
+echo '>>> Create latency metric template'
+cat <<EOF | kubectl apply -f -
+apiVersion: flagger.app/v1beta1
+kind: MetricTemplate
+metadata:
+  name: latency
+  namespace: istio-system
+spec:
+  provider:
+    type: prometheus
+    address: http://prometheus.istio-system:9090
+  query: |
+    histogram_quantile(
+        0.99,
+        sum(
+            rate(
+                istio_request_duration_milliseconds_bucket{
+                    reporter="destination",
+                    destination_workload_namespace="{{ namespace }}",
+                    destination_workload=~"{{ target }}"
+                }[{{ interval }}]
+            )
+        ) by (le)
+    )
+EOF
+
+echo '>>> Initialising canary'
+cat <<EOF | kubectl apply -f -
+apiVersion: flagger.app/v1beta1
+kind: Canary
+metadata:
+  name: podinfo
+  namespace: test
+spec:
+  targetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: podinfo
+  progressDeadlineSeconds: 60
+  service:
+    port: 9898
+    portDiscovery: true
+    apex:
+      annotations:
+        test: "annotations-test"
+      labels:
+        test: "labels-test"
+    headers:
+      request:
+        add:
+          x-envoy-upstream-rq-timeout-ms: "15000"
+          x-envoy-max-retries: "10"
+          x-envoy-retry-on: "gateway-error,connect-failure,refused-stream"
+  analysis:
+    skipAnalysis: true
+    interval: 15s
+    threshold: 15
+    maxWeight: 30
+    stepWeight: 10
+    webhooks:
+      - name: load-test
+        url: http://flagger-loadtester.test/
+        timeout: 5s
+        metadata:
+          type: cmd
+          cmd: "hey -z 10m -q 10 -c 2 http://podinfo.test:9898/"
+          logCmdOutput: "true"
+EOF
+
+echo '>>> Waiting for primary to be ready'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false
+    sleep 5
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n istio-system logs deployment/flagger
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+echo '✔ Canary initialization test passed'
+
+kubectl -n test get svc/podinfo -oyaml | grep annotations-test
+kubectl -n test get svc/podinfo -oyaml | grep labels-test
+
+echo '✔ Canary service custom metadata test passed'
+
+echo '>>> Triggering canary deployment'
+kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1
+
+echo '>>> Waiting for canary promotion'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n test describe deployment/podinfo-primary | grep '3.1.1' && ok=true || ok=false
+    sleep 10
+    kubectl -n istio-system logs deployment/flagger --tail 1
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n test describe deployment/podinfo
+        kubectl -n test describe deployment/podinfo-primary
+        kubectl -n istio-system logs deployment/flagger
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+echo '>>> Waiting for canary finalization'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false
+    sleep 5
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n istio-system logs deployment/flagger
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+echo '✔ Canary promotion test passed'
+
+if [[ "$1" = "canary" ]]; then
+  exit 0
+fi
+
+echo '>>> Triggering canary deployment with a bad release (unknown docker image)'
+kubectl -n test set image deployment/podinfo podinfod=stefanprodan/potato:1.0.0
+
+echo '>>> Waiting for canary to fail'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+    kubectl get canary/podinfo -n test -o=jsonpath='{.status.phase}' | grep 'Failed' && ok=true || ok=false
+    sleep 10
+    kubectl -n istio-system logs deployment/flagger --tail 1
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n test describe deployment/podinfo
+        kubectl -n test describe deployment/podinfo-primary
+        kubectl -n istio-system logs deployment/flagger
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+echo '>>> Confirm primary pod still running correct version'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n test describe deployment/podinfo-primary | grep '3.1.1' && ok=true || ok=false
+    sleep 5
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n istio-system logs deployment/flagger
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+kubectl -n istio-system logs deployment/flagger
+
+echo '✔ All tests passed'