From 7bef999c413fcf5d370beb4aabd7d0ad3370f85a Mon Sep 17 00:00:00 2001
From: Stefan Prodan <stefan.prodan@gmail.com>
Date: Sun, 20 Dec 2020 17:47:45 +0200
Subject: [PATCH] Add Linkerd e2e tests

Signed-off-by: Stefan Prodan <stefan.prodan@gmail.com>
---
 .github/workflows/e2e.yaml  |   1 +
 test/linkerd/install.sh     |  24 ++++
 test/linkerd/run.sh         |  13 ++
 test/linkerd/test-canary.sh | 242 ++++++++++++++++++++++++++++++++++++
 test/linkerd/test-init.sh   |  23 ++++
 test/linkerd/test-steps.sh  |  95 ++++++++++++++
 6 files changed, 398 insertions(+)
 create mode 100755 test/linkerd/install.sh
 create mode 100755 test/linkerd/run.sh
 create mode 100755 test/linkerd/test-canary.sh
 create mode 100755 test/linkerd/test-init.sh
 create mode 100755 test/linkerd/test-steps.sh

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index 41dfab38..5638abe6 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -14,6 +14,7 @@ jobs:
       matrix:
         provider:
           - istio
+          - linkerd
     steps:
       - name: Checkout
         uses: actions/checkout@v2
diff --git a/test/linkerd/install.sh b/test/linkerd/install.sh
new file mode 100755
index 00000000..7bfd15b0
--- /dev/null
+++ b/test/linkerd/install.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+set -o errexit
+
+LINKERD_VER="stable-2.8.1"
+REPO_ROOT=$(git rev-parse --show-toplevel)
+
+mkdir -p ${REPO_ROOT}/bin
+
+curl -SsL https://github.com/linkerd/linkerd2/releases/download/${LINKERD_VER}/linkerd2-cli-${LINKERD_VER}-linux > ${REPO_ROOT}/bin/linkerd
+chmod +x ${REPO_ROOT}/bin/linkerd
+
+echo ">>> Installing Linkerd ${LINKERD_VER}"
+${REPO_ROOT}/bin/linkerd install | kubectl apply -f -
+${REPO_ROOT}/bin/linkerd check
+
+kubectl -n linkerd rollout status deployment/linkerd-controller
+kubectl -n linkerd rollout status deployment/linkerd-proxy-injector
+
+echo '>>> Installing Flagger'
+kubectl apply -k ${REPO_ROOT}/kustomize/linkerd
+
+kubectl -n linkerd set image deployment/flagger flagger=test/flagger:latest
+kubectl -n linkerd rollout status deployment/flagger
diff --git a/test/linkerd/run.sh b/test/linkerd/run.sh
new file mode 100755
index 00000000..e3174d56
--- /dev/null
+++ b/test/linkerd/run.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+set -o errexit
+
+DIR="$(cd "$(dirname "$0")" && pwd)"
+
+"$DIR"/install.sh
+
+"$DIR"/test-init.sh
+"$DIR"/test-canary.sh
+
+"$DIR"/test-init.sh
+"$DIR"/test-steps.sh
diff --git a/test/linkerd/test-canary.sh b/test/linkerd/test-canary.sh
new file mode 100755
index 00000000..c7a85aff
--- /dev/null
+++ b/test/linkerd/test-canary.sh
@@ -0,0 +1,242 @@
+#!/usr/bin/env bash
+
+# This script runs Linkerd e2e tests for Canary initialization, analysis and promotion
+
+set -o errexit
+
+REPO_ROOT=$(git rev-parse --show-toplevel)
+
+cat <<EOF | kubectl apply -f -
+apiVersion: flagger.app/v1alpha1
+kind: MetricTemplate
+metadata:
+  name: latency
+  namespace: linkerd
+spec:
+  provider:
+    type: prometheus
+    address: http://linkerd-prometheus.linkerd:9090
+  query: |
+    histogram_quantile(
+        0.99,
+        sum(
+            rate(
+                response_latency_ms_bucket{
+                    namespace="{{ namespace }}",
+                    deployment=~"{{ target }}",
+                    direction="inbound"
+                    }[{{ interval }}]
+                )
+            ) by (le)
+        )
+EOF
+
+cat <<EOF | kubectl apply -f -
+apiVersion: flagger.app/v1beta1
+kind: Canary
+metadata:
+  name: podinfo
+  namespace: test
+spec:
+  targetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: podinfo
+  progressDeadlineSeconds: 60
+  service:
+    port: 80
+    targetPort: http
+    portDiscovery: true
+  analysis:
+    interval: 15s
+    threshold: 15
+    maxWeight: 50
+    stepWeight: 10
+    metrics:
+    - name: request-success-rate
+      threshold: 99
+      interval: 1m
+    - name: request-duration
+      threshold: 500
+      interval: 30s
+    - name: latency
+      templateRef:
+        name: latency
+        namespace: linkerd
+      threshold: 300
+      interval: 1m
+    webhooks:
+      - name: http-acceptance-test
+        type: pre-rollout
+        url: http://flagger-loadtester.test/
+        timeout: 30s
+        metadata:
+          type: bash
+          cmd: "curl -sd 'test' http://podinfo-canary/token | grep token"
+      - name: grpc-acceptance-test
+        type: pre-rollout
+        url: http://flagger-loadtester.test/
+        timeout: 5s
+        metadata:
+          type: bash
+          cmd: "grpc_health_probe -connect-timeout=1s -addr=podinfo-canary:9999"
+      - name: load-test
+        url: http://flagger-loadtester.test/
+        timeout: 5s
+        metadata:
+          type: cmd
+          cmd: "hey -z 10m -q 10 -c 2 http://podinfo.test/"
+          logCmdOutput: "true"
+EOF
+
+cat <<EOF | kubectl apply -f -
+apiVersion: flagger.app/v1beta1
+kind: Canary
+metadata:
+  name: podinfo-service
+  namespace: test
+spec:
+  targetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: podinfo-service
+  progressDeadlineSeconds: 60
+  service:
+    port: 9898
+    portDiscovery: true
+  analysis:
+    interval: 15s
+    threshold: 15
+    maxWeight: 30
+    stepWeight: 10
+EOF
+
+echo '>>> Waiting for primary to be ready'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false
+    sleep 5
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n linkerd logs deployment/flagger
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+echo '✔ Canary initialization test passed'
+
+passed=$(kubectl -n test get svc/podinfo -o jsonpath='{.spec.selector.app}' 2>&1 | { grep podinfo-primary || true; })
+if [ -z "$passed" ]; then
+  echo -e '\u2716 podinfo selector test failed'
+  exit 1
+fi
+passed=$(kubectl -n test get svc/podinfo-service-canary -o jsonpath='{.spec.selector.app}' 2>&1 | { grep podinfo || true; })
+if [ -z "$passed" ]; then
+  echo -e '\u2716 podinfo-service selector test failed'
+  exit 1
+fi
+
+echo '✔ Canary service custom metadata test passed'
+
+echo '>>> Triggering canary deployment'
+kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1
+
+echo '>>> Waiting for canary promotion'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n test describe deployment/podinfo-primary | grep '3.1.1' && ok=true || ok=false
+    sleep 10
+    kubectl -n linkerd logs deployment/flagger --tail 1
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n linkerd logs deployment/flagger
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+echo '>>> Waiting for canary finalization'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false
+    sleep 5
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n linkerd logs deployment/flagger
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+echo '✔ Canary promotion test passed'
+
+cat <<EOF | kubectl apply -f -
+apiVersion: flagger.app/v1beta1
+kind: Canary
+metadata:
+  name: podinfo
+  namespace: test
+spec:
+  targetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: podinfo
+  progressDeadlineSeconds: 60
+  service:
+    port: 80
+    targetPort: 9898
+  analysis:
+    interval: 15s
+    threshold: 3
+    maxWeight: 50
+    stepWeight: 10
+    metrics:
+    - name: request-success-rate
+      threshold: 99
+      interval: 1m
+    - name: request-duration
+      threshold: 500
+      interval: 30s
+    webhooks:
+      - name: http-acceptance-test
+        type: pre-rollout
+        url: http://flagger-loadtester.test/
+        timeout: 30s
+        metadata:
+          type: bash
+          cmd: "curl -sd 'test' http://podinfo-canary/token | grep token"
+      - name: load-test
+        url: http://flagger-loadtester.test/
+        timeout: 5s
+        metadata:
+          type: cmd
+          cmd: "hey -z 10m -q 10 -c 2 http://podinfo.test/status/500"
+EOF
+
+echo '>>> Triggering canary deployment'
+kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.2
+
+echo '>>> Waiting for canary rollback'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n test get canary/podinfo | grep 'Failed' && ok=true || ok=false
+    sleep 10
+    kubectl -n linkerd logs deployment/flagger --tail 1
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n linkerd logs deployment/flagger
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+echo '✔ Canary rollback test passed'
\ No newline at end of file
diff --git a/test/linkerd/test-init.sh b/test/linkerd/test-init.sh
new file mode 100755
index 00000000..4f0ab745
--- /dev/null
+++ b/test/linkerd/test-init.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+# This script setups the scenarios for istio tests by creating a Kubernetes namespace, installing the load tester and a test workload (podinfo)
+# Prerequisites: Kubernetes Kind and Istio
+
+set -o errexit
+
+REPO_ROOT=$(git rev-parse --show-toplevel)
+
+echo '>>> Delete test namespace'
+kubectl delete namespace test --ignore-not-found=true --wait=true
+
+echo '>>> Creating test namespace'
+kubectl create namespace test
+kubectl label namespace test istio-injection=enabled
+kubectl annotate namespace test linkerd.io/inject=enabled
+
+echo '>>> Installing the load tester'
+kubectl apply -k ${REPO_ROOT}/kustomize/tester
+kubectl -n test rollout status deployment/flagger-loadtester
+
+echo '>>> Deploy podinfo'
+kubectl apply -f ${REPO_ROOT}/test/e2e-workload.yaml
diff --git a/test/linkerd/test-steps.sh b/test/linkerd/test-steps.sh
new file mode 100755
index 00000000..ebb122fe
--- /dev/null
+++ b/test/linkerd/test-steps.sh
@@ -0,0 +1,95 @@
+#!/usr/bin/env bash
+
+# This script runs Linkerd e2e tests for Canary initialization, analysis and promotion
+
+set -o errexit
+
+cat <<EOF | kubectl apply -f -
+apiVersion: flagger.app/v1beta1
+kind: Canary
+metadata:
+  name: podinfo
+  namespace: test
+spec:
+  targetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: podinfo
+  progressDeadlineSeconds: 60
+  service:
+    port: 80
+    targetPort: http
+    portDiscovery: true
+  analysis:
+    interval: 15s
+    threshold: 15
+    maxWeight: 50
+    stepWeights: [10, 20, 30, 40, 50, 60]
+    metrics:
+    - name: request-success-rate
+      threshold: 99
+      interval: 1m
+    - name: request-duration
+      threshold: 500
+      interval: 30s
+    webhooks:
+      - name: load-test
+        url: http://flagger-loadtester.test/
+        timeout: 5s
+        metadata:
+          type: cmd
+          cmd: "hey -z 10m -q 10 -c 2 http://podinfo.test/"
+EOF
+
+echo '>>> Waiting for primary to be ready'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false
+    sleep 5
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n linkerd logs deployment/flagger
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+echo '✔ Canary initialization test passed'
+
+echo '>>> Triggering canary deployment'
+kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1
+
+echo '>>> Waiting for canary promotion'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n test describe deployment/podinfo-primary | grep '3.1.1' && ok=true || ok=false
+    sleep 10
+    kubectl -n linkerd logs deployment/flagger --tail 1
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n linkerd logs deployment/flagger
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+echo '>>> Waiting for canary finalization'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false
+    sleep 5
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n linkerd logs deployment/flagger
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+echo '✔ Canary promotion test passed'