From e81627a96d6d9ebab8fda2bf438e92515317ec4a Mon Sep 17 00:00:00 2001
From: John Harris <john.harris@konghq.com>
Date: Sat, 18 Dec 2021 14:08:21 -0800
Subject: [PATCH] Add tests

Signed-off-by: John Harris <john.harris@konghq.com>
---
 .github/workflows/e2e.yaml |   1 +
 test/kuma/install.sh       |  61 ++++++++++++
 test/kuma/run.sh           |  11 +++
 test/kuma/test-canary.sh   | 196 +++++++++++++++++++++++++++++++++++++
 test/workloads/init.sh     |   1 +
 5 files changed, 270 insertions(+)
 create mode 100755 test/kuma/install.sh
 create mode 100755 test/kuma/run.sh
 create mode 100755 test/kuma/test-canary.sh

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index da3560ea..1bf56ad0 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -23,6 +23,7 @@ jobs:
           - gloo
           - skipper
           - osm
+          - kuma
           - kubernetes
     steps:
       - name: Checkout
diff --git a/test/kuma/install.sh b/test/kuma/install.sh
new file mode 100755
index 00000000..fe121e62
--- /dev/null
+++ b/test/kuma/install.sh
@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+
+set -o errexit
+
+KUMA_VER="1.4.1"
+REPO_ROOT=$(git rev-parse --show-toplevel)
+mkdir -p ${REPO_ROOT}/bin
+
+echo ">>> Downloading Kuma ${KUMA_VER}"
+curl -SsL https://download.konghq.com/mesh-alpine/kuma-${KUMA_VER}-ubuntu-amd64.tar.gz -o kuma-${KUMA_VER}.tar.gz
+tar xvzf kuma-${KUMA_VER}.tar.gz
+cp kuma-${KUMA_VER}/bin/kumactl ${REPO_ROOT}/bin/kumactl
+chmod +x ${REPO_ROOT}/bin/kumactl
+
+echo ">>> Installing Kuma ${KUMA_VER}"
+${REPO_ROOT}/bin/kumactl install control-plane | kubectl apply -f -
+
+echo ">>> Installing Kuma Metrics"
+${REPO_ROOT}/bin/kumactl install metrics | kubectl apply -f -
+
+echo ">>> Waiting for Kuma Control Plane to be ready"
+kubectl wait --for=condition=ready pod -n kuma-system -l app=kuma-control-plane
+
+echo ">>> Configuring Default Kuma Mesh"
+cat <<EOF | kubectl apply -f -
+apiVersion: kuma.io/v1alpha1
+kind: Mesh
+metadata:
+  name: default
+spec:
+  metrics:
+    enabledBackend: prometheus-1
+    backends:
+      - name: prometheus-1
+        type: prometheus
+        conf:
+          skipMTLS: true
+          port: 5670
+          path: /metrics
+          tags:
+            kuma.io/service: dataplane-metrics
+  mtls:
+    enabledBackend: ca-1
+    backends:
+      - name: ca-1
+        type: builtin
+        mode: PERMISSIVE
+        dpCert:
+          rotation:
+            expiration: 1d
+        conf:
+          caCert:
+            RSAbits: 2048
+            expiration: 10y
+EOF
+
+echo '>>> Installing Flagger'
+kubectl apply -k ${REPO_ROOT}/kustomize/kuma
+
+kubectl -n kuma-system set image deployment/flagger flagger=test/flagger:latest
+kubectl -n kuma-system rollout status deployment/flagger
diff --git a/test/kuma/run.sh b/test/kuma/run.sh
new file mode 100755
index 00000000..fff250d6
--- /dev/null
+++ b/test/kuma/run.sh
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -o errexit
+
+REPO_ROOT=$(git rev-parse --show-toplevel)
+DIR="$(cd "$(dirname "$0")" && pwd)"
+
+"$DIR"/install.sh
+
+"$REPO_ROOT"/test/workloads/init.sh
+"$DIR"/test-canary.sh
\ No newline at end of file
diff --git a/test/kuma/test-canary.sh b/test/kuma/test-canary.sh
new file mode 100755
index 00000000..68ce67ca
--- /dev/null
+++ b/test/kuma/test-canary.sh
@@ -0,0 +1,196 @@
+#!/usr/bin/env bash
+
+# This script runs Kuma e2e tests for Canary initialization, analysis and promotion
+
+set -o errexit
+
+REPO_ROOT=$(git rev-parse --show-toplevel)
+
+cat <<EOF | kubectl apply -f -
+apiVersion: flagger.app/v1beta1
+kind: Canary
+metadata:
+  name: podinfo
+  namespace: test
+  annotations:
+    kuma.io/mesh: default
+spec:
+  targetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: podinfo
+  progressDeadlineSeconds: 60
+  service:
+    port: 9898
+    targetPort: 9898
+    apex:
+      annotations:
+        9898.service.kuma.io/protocol: "http"
+    canary:
+      annotations:
+        9898.service.kuma.io/protocol: "http"
+    primary:
+      annotations:
+        9898.service.kuma.io/protocol: "http"
+  analysis:
+    interval: 15s
+    threshold: 15
+    maxWeight: 50
+    stepWeight: 10
+    metrics:
+    - name: request-success-rate
+      threshold: 99
+      interval: 1m
+    - name: request-duration
+      threshold: 500
+      interval: 30s
+    webhooks:
+      - name: acceptance-test
+        type: pre-rollout
+        url: http://flagger-loadtester.test/
+        timeout: 30s
+        metadata:
+          type: bash
+          cmd: "curl -sd 'test' http://podinfo-canary.test:9898/token | grep token"
+      - name: load-test
+        type: rollout
+        url: http://flagger-loadtester.test/
+        metadata:
+          cmd: "hey -z 2m -q 10 -c 2 http://podinfo-canary.test:9898/"
+EOF
+
+echo '>>> Waiting for primary to be ready'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false
+    sleep 5
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n kuma-system logs deployment/flagger
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+echo '✔ Canary initialization test passed'
+
+passed=$(kubectl -n test get svc/podinfo -o jsonpath='{.spec.selector.app}' 2>&1 | { grep podinfo-primary || true; })
+if [ -z "$passed" ]; then
+  echo -e '\u2716 podinfo selector test failed'
+  exit 1
+fi
+
+echo '✔ Canary service custom metadata test passed'
+
+echo '>>> Triggering canary deployment'
+kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1
+
+echo '>>> Waiting for canary promotion'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n test describe deployment/podinfo-primary | grep '3.1.1' && ok=true || ok=false
+    sleep 10
+    kubectl -n kuma-system logs deployment/flagger --tail 1
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n kuma-system logs deployment/flagger
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+echo '>>> Waiting for canary finalization'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false
+    sleep 5
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n kuma-system logs deployment/flagger
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+echo '✔ Canary promotion test passed'
+
+cat <<EOF | kubectl apply -f -
+apiVersion: flagger.app/v1beta1
+kind: Canary
+metadata:
+  name: podinfo
+  namespace: test
+  annotations:
+    kuma.io/mesh: default
+spec:
+  targetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: podinfo
+  progressDeadlineSeconds: 60
+  service:
+    port: 9898
+    targetPort: 9898
+    apex:
+      annotations:
+        9898.service.kuma.io/protocol: "http"
+    canary:
+      annotations:
+        9898.service.kuma.io/protocol: "http"
+    primary:
+      annotations:
+        9898.service.kuma.io/protocol: "http"
+  analysis:
+    interval: 15s
+    threshold: 15
+    maxWeight: 50
+    stepWeight: 10
+    metrics:
+    - name: request-success-rate
+      threshold: 99
+      interval: 1m
+    - name: request-duration
+      threshold: 500
+      interval: 30s
+    webhooks:
+      - name: acceptance-test
+        type: pre-rollout
+        url: http://flagger-loadtester.test/
+        timeout: 30s
+        metadata:
+          type: bash
+          cmd: "curl -sd 'test' http://podinfo-canary.test:9898/token | grep token"
+      - name: load-test
+        url: http://flagger-loadtester.test/
+        timeout: 5s
+        metadata:
+          type: cmd
+          cmd: "hey -z 2m -q 10 -c 2 http://podinfo-canary.test:9898/status/500"
+EOF
+
+echo '>>> Triggering canary deployment rollback test'
+kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.2
+
+echo '>>> Waiting for canary rollback'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n test get canary/podinfo | grep 'Failed' && ok=true || ok=false
+    sleep 10
+    kubectl -n kuma-system logs deployment/flagger --tail 1
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n kuma-system logs deployment/flagger
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+echo '✔ Canary rollback test passed'
\ No newline at end of file
diff --git a/test/workloads/init.sh b/test/workloads/init.sh
index ff42fbe6..2b615a12 100755
--- a/test/workloads/init.sh
+++ b/test/workloads/init.sh
@@ -13,6 +13,7 @@ echo '>>> Creating test namespace'
 kubectl create namespace test
 kubectl label namespace test istio-injection=enabled
 kubectl annotate namespace test linkerd.io/inject=enabled
+kubectl annotate namespace test kuma.io/sidecar-injection=enabled
 
 echo '>>> Installing the load tester'
 kubectl apply -k ${REPO_ROOT}/kustomize/tester