From b01e4cf9ecffd665532c7373fc701848a2a77ed2 Mon Sep 17 00:00:00 2001
From: Sanskar Jaiswal <sanskar.jaiswal@weave.works>
Date: Fri, 10 Jun 2022 15:09:28 +0530
Subject: [PATCH] add e2e tests for KEDA ScaledObjects

Signed-off-by: Sanskar Jaiswal <sanskar.jaiswal@weave.works>
---
 .github/workflows/e2e.yaml       |   1 +
 kustomize/base/flagger/rbac.yaml |  13 ++
 test/keda/install.sh             |  22 ++++
 test/keda/run.sh                 |  11 ++
 test/keda/test-scaledobject.sh   | 208 +++++++++++++++++++++++++++++++
 test/kubernetes/test-hpa.sh      |   2 +
 6 files changed, 257 insertions(+)
 create mode 100755 test/keda/install.sh
 create mode 100755 test/keda/run.sh
 create mode 100755 test/keda/test-scaledobject.sh

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index 54695ecd..6ab45f9e 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -32,6 +32,7 @@ jobs:
           - skipper
           - kubernetes
           - gatewayapi
+          - keda
     steps:
       - name: Checkout
         uses: actions/checkout@v2
diff --git a/kustomize/base/flagger/rbac.yaml b/kustomize/base/flagger/rbac.yaml
index 168cf6d9..5fcaf008 100644
--- a/kustomize/base/flagger/rbac.yaml
+++ b/kustomize/base/flagger/rbac.yaml
@@ -215,6 +215,19 @@ rules:
       - update
       - patch
       - delete
+  - apiGroups:
+      - keda.sh
+    resources:
+      - scaledobjects
+      - scaledobjects/finalizers
+    verbs:
+      - get
+      - list
+      - watch
+      - create
+      - update
+      - patch
+      - delete
   - nonResourceURLs:
       - /version
     verbs:
diff --git a/test/keda/install.sh b/test/keda/install.sh
new file mode 100755
index 00000000..181e487a
--- /dev/null
+++ b/test/keda/install.sh
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+set -o errexit
+
+REPO_ROOT=$(git rev-parse --show-toplevel)
+
+mkdir -p ${REPO_ROOT}/bin
+
+echo '>>> Installing KEDA'
+helm repo add kedacore https://kedacore.github.io/charts
+kubectl create ns keda
+helm install keda kedacore/keda --namespace keda --wait
+
+kubectl -n keda get all
+
+echo '>>> Installing Flagger'
+kubectl apply -k ${REPO_ROOT}/kustomize/kubernetes
+
+kubectl -n flagger-system set image deployment/flagger flagger=test/flagger:latest
+
+kubectl -n flagger-system rollout status deployment/flagger
+kubectl -n flagger-system rollout status deployment/flagger-prometheus
diff --git a/test/keda/run.sh b/test/keda/run.sh
new file mode 100755
index 00000000..af6e543e
--- /dev/null
+++ b/test/keda/run.sh
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -o errexit
+
+REPO_ROOT=$(git rev-parse --show-toplevel)
+DIR="$(cd "$(dirname "$0")" && pwd)"
+
+"$DIR"/install.sh
+
+"$REPO_ROOT"/test/workloads/init.sh
+"$DIR"/test-scaledobject.sh
diff --git a/test/keda/test-scaledobject.sh b/test/keda/test-scaledobject.sh
new file mode 100755
index 00000000..5a28632d
--- /dev/null
+++ b/test/keda/test-scaledobject.sh
@@ -0,0 +1,208 @@
+#!/usr/bin/env bash
+
+# This script runs e2e tests targetting Flagger's integration with KEDA ScaledObjects.
+
+set -o errexit
+
+REPO_ROOT=$(git rev-parse --show-toplevel)
+
+cat <<EOF | kubectl apply -f -
+apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
+metadata:
+  name: podinfo-so
+  namespace: test
+spec:
+  scaleTargetRef:
+    name: podinfo
+  pollingInterval: 10
+  cooldownPeriod: 20
+  minReplicaCount: 1
+  maxReplicaCount: 3
+  triggers:
+  - type: prometheus
+    metadata:
+      serverAddress: http://flagger-prometheus.flagger-system:9090
+      metricName: http_requests_total
+      query: sum(rate(http_requests_total{ app="podinfo" }[30s]))
+      threshold: '5'
+EOF
+
+cat <<EOF | kubectl apply -f -
+apiVersion: v1
+kind: Service
+metadata:
+  name: podinfo-svc
+  namespace: test
+spec:
+  type: ClusterIP
+  selector:
+    app: podinfo
+  ports:
+    - name: http
+      port: 9898
+      protocol: TCP
+      targetPort: http
+---
+apiVersion: flagger.app/v1beta1
+kind: Canary
+metadata:
+  name: podinfo
+  namespace: test
+spec:
+  provider: kubernetes
+  targetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: podinfo
+  autoscalerRef:
+    apiVersion: keda.sh/v1alpha1
+    kind: ScaledObject
+    name: podinfo-so
+  progressDeadlineSeconds: 60
+  service:
+    port: 80
+    targetPort: 9898
+    name: podinfo-svc
+    portDiscovery: true
+  analysis:
+    interval: 15s
+    threshold: 10
+    iterations: 8
+    metrics:
+      - name: request-success-rate
+        interval: 1m
+        thresholdRange:
+          min: 99
+      - name: request-duration
+        interval: 30s
+        thresholdRange:
+          max: 500
+    webhooks:
+      - name: load-test
+        url: http://flagger-loadtester.test/
+        timeout: 5s
+        metadata:
+          type: cmd
+          cmd: "hey -z 2m -q 20 -c 2 http://podinfo-svc-canary.test/"
+EOF
+
+echo '>>> Waiting for primary to be ready'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false
+    sleep 5
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n flagger-system logs deployment/flagger
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+echo '✔ Canary initialization test passed'
+
+expectedQuery='sum(rate(http_requests_total{ app="podinfo-primary" }[30s]))'
+if kubectl -n test get scaledobjects podinfo-so-primary; then
+    query=$(kubectl -n test get scaledobjects podinfo-so-primary -o=jsonpath='{.spec.triggers[0].metadata.query}')
+    if [[ "$query" = "$expectedQuery" ]]; then
+        echo '✔ Primary ScaledObject successfully reconciled'
+    else
+        kubectl -n test get scaledobjects podinfo-so-primary -oyaml
+        echo '⨯ Primary ScaledObject query does not match expected query'
+        exit 1
+    fi
+else
+    echo '⨯ Primary ScaledObject not found'
+    exit 1
+fi
+
+val=$(kubectl -n test get scaledobject podinfo-so -o=jsonpath='{.metadata.annotations.autoscaling\.keda\.sh\/paused-replicas}' | xargs)
+if [[ "$val" = "0" ]]; then
+    echo '✔ Successfully paused autoscaling for target ScaledObject'
+else
+    echo '⨯ Could not pause autoscaling for target ScaledObject'
+fi
+
+echo '>>> Triggering canary deployment'
+kubectl -n test set image deployment/podinfo podinfod=ghcr.io/stefanprodan/podinfo:6.0.1
+
+echo '>>> Waiting for ScaledObject autoscaling to get unpaused'
+retries=20
+count=0
+ok=false
+until ${ok}; do
+    val=$(kubectl -n test get scaledobject podinfo-so -o=jsonpath='{.metadata.annotations.autoscaling\.keda\.sh\/paused-replicas}' | xargs)
+    if [[ "$val" = "" ]]; then
+        ok=true
+    fi
+    sleep 2
+    kubectl -n flagger-system logs deployment/flagger --tail 1
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n flagger-system logs deployment/flagger
+        kubectl -n test get scaledobject podinfo-so -oyaml
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+echo '>>> Waiting for canary deployment to be scaled up'
+retries=20
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n test get deployment/podinfo -oyaml | grep 'replicas: 3' && ok=true || ok=false
+    sleep 5
+    kubectl -n flagger-system logs deployment/flagger --tail 1
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n flagger-system logs deployment/flagger
+        kubectl -n test get deploy/podinfo -oyaml
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+echo '>>> Waiting for canary promotion'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n test describe deployment/podinfo-primary | grep '6.0.1' && ok=true || ok=false
+    sleep 10
+    kubectl -n flagger-system logs deployment/flagger --tail 1
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n flagger-system logs deployment/flagger
+        kubectl -n test get httpproxy podinfo -oyaml
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+echo '✔ Canary promotion test passed'
+
+echo '>>> Waiting for canary finalization'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false
+    sleep 5
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n flagger-system logs deployment/flagger
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+val=$(kubectl -n test get scaledobject podinfo-so -o=jsonpath='{.metadata.annotations.autoscaling\.keda\.sh\/paused-replicas}' | xargs)
+if [[ "$val" = "0" ]]; then
+    echo '✔ Successfully paused autoscaling for target ScaledObject'
+else
+    echo '⨯ Could not pause autoscaling for target ScaledObject'
+fi
diff --git a/test/kubernetes/test-hpa.sh b/test/kubernetes/test-hpa.sh
index 7b3773df..32faa003 100755
--- a/test/kubernetes/test-hpa.sh
+++ b/test/kubernetes/test-hpa.sh
@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+# This script runs E2E tests targetting Flagger's integration with HPAs.
+
 set -o errexit
 
 REPO_ROOT=$(git rev-parse --show-toplevel)