diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index aa4d7715..e6e0d1c8 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -16,6 +16,7 @@ jobs: - istio - linkerd - contour + - nginx steps: - name: Checkout uses: actions/checkout@v2 diff --git a/test/nginx/install.sh b/test/nginx/install.sh new file mode 100755 index 00000000..5c3cbbbd --- /dev/null +++ b/test/nginx/install.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +set -o errexit + +NGINX_HELM_VERSION=3.15.2 # ingress v0.41.2 +REPO_ROOT=$(git rev-parse --show-toplevel) + +mkdir -p ${REPO_ROOT}/bin + +echo '>>> Installing NGINX Ingress' +kubectl create ns ingress-nginx +helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx +helm upgrade -i ingress-nginx ingress-nginx/ingress-nginx --version=${NGINX_HELM_VERSION} \ +--wait \ +--namespace ingress-nginx \ +--set controller.metrics.enabled=true \ +--set controller.podAnnotations."prometheus\.io/scrape"=true \ +--set controller.podAnnotations."prometheus\.io/port"=10254 \ +--set controller.service.type=NodePort + +kubectl -n ingress-nginx get all + +echo '>>> Installing Flagger' +helm upgrade -i flagger ${REPO_ROOT}/charts/flagger \ +--set crd.create=false \ +--namespace ingress-nginx \ +--set prometheus.install=true \ +--set meshProvider=nginx + +kubectl -n ingress-nginx set image deployment/flagger flagger=test/flagger:latest + +kubectl -n ingress-nginx rollout status deployment/flagger +kubectl -n ingress-nginx rollout status deployment/flagger-prometheus diff --git a/test/nginx/run.sh b/test/nginx/run.sh new file mode 100755 index 00000000..96efa472 --- /dev/null +++ b/test/nginx/run.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -o errexit + +DIR="$(cd "$(dirname "$0")" && pwd)" + +"$DIR"/install.sh + +"$DIR"/test-init.sh +"$DIR"/test-canary.sh + diff --git a/test/nginx/test-canary.sh b/test/nginx/test-canary.sh new file mode 100755 index 00000000..ff185418 --- /dev/null +++ b/test/nginx/test-canary.sh @@ -0,0 +1,281 @@ +#!/usr/bin/env bash + +# This script runs e2e tests for Canary initialization, analysis and promotion +# Prerequisites: Kubernetes Kind, Helm and NGINX ingress controller + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) + +cat <>> Create metric templates' +cat <>> Waiting for primary to be ready' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n ingress-nginx logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary initialization test passed' + +echo '>>> Triggering canary deployment' +kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1 + +echo '>>> Waiting for canary promotion' +retries=50 +count=0 +ok=false +failed=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Failed' && failed=true || failed=false + if ${failed}; then + kubectl -n ingress-nginx logs deployment/flagger + echo "Canary failed!" + exit 1 + fi + kubectl -n test describe deployment/podinfo-primary | grep '3.1.1' && ok=true || ok=false + sleep 10 + kubectl -n ingress-nginx logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n test describe deployment/podinfo + kubectl -n test describe deployment/podinfo-primary + kubectl -n ingress-nginx logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '>>> Waiting for canary finalization' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n ingress-nginx logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary promotion test passed' + +cat <>> Triggering A/B testing' +kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.2 + +echo '>>> Waiting for A/B testing promotion' +retries=6 +count=0 +ok=false +until ${ok}; do + kubectl -n test describe deployment/podinfo-primary | grep '3.1.2' && ok=true || ok=false + sleep 30 + kubectl -n ingress-nginx logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n test describe deployment/podinfo + kubectl -n test describe deployment/podinfo-primary + kubectl -n ingress-nginx logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ A/B testing promotion test passed' + +kubectl -n ingress-nginx logs deployment/flagger + +echo '✔ All tests passed' diff --git a/test/nginx/test-init.sh b/test/nginx/test-init.sh new file mode 100755 index 00000000..14e3bce7 --- /dev/null +++ b/test/nginx/test-init.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +# This script setups the scenarios for tests by creating a +# Kubernetes namespace, installing the load tester +# and a test workload (podinfo) + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) + +echo '>>> Delete test namespace' +kubectl delete namespace test --ignore-not-found=true --wait=true + +echo '>>> Creating test namespace' +kubectl create namespace test +kubectl label namespace test istio-injection=enabled +kubectl annotate namespace test linkerd.io/inject=enabled + +echo '>>> Installing the load tester' +kubectl apply -k ${REPO_ROOT}/kustomize/tester +kubectl -n test rollout status deployment/flagger-loadtester + +echo '>>> Deploy podinfo' +kubectl apply -f ${REPO_ROOT}/test/e2e-workload.yaml