diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index d65441d1..c124734b 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -24,6 +24,10 @@ jobs: uses: actions/setup-go@v2 with: go-version: 1.15.x + - name: Download modules + run: | + go mod download + go install golang.org/x/tools/cmd/goimports - name: Run tests run: make test - name: Check if working tree is dirty diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml new file mode 100644 index 00000000..41dfab38 --- /dev/null +++ b/.github/workflows/e2e.yaml @@ -0,0 +1,28 @@ +name: e2e + +on: + pull_request: + push: + branches: + - main + - github-actions + +jobs: + kind: + runs-on: ubuntu-latest + strategy: + matrix: + provider: + - istio + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Setup Kubernetes + uses: engineerd/setup-kind@v0.5.0 + - name: Build container image + run: docker build -t test/flagger:latest . + - name: Load test image + run: kind load docker-image test/flagger:latest + - name: Run tests + run: | + ./test/${{ matrix['provider'] }}/run.sh diff --git a/pkg/loadtester/task_ngrinder.go b/pkg/loadtester/task_ngrinder.go index ab5ba5df..488cbf28 100644 --- a/pkg/loadtester/task_ngrinder.go +++ b/pkg/loadtester/task_ngrinder.go @@ -68,7 +68,7 @@ type NGrinderTask struct { } func (task *NGrinderTask) Hash() string { - return hash(task.canary + string(task.cloneId)) + return hash(task.canary + fmt.Sprint(task.cloneId)) } // nGrinder REST endpoints diff --git a/test/istio/install.sh b/test/istio/install.sh new file mode 100755 index 00000000..82e17a1d --- /dev/null +++ b/test/istio/install.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +set -o errexit + +ISTIO_VER="1.8.0" +REPO_ROOT=$(git rev-parse --show-toplevel) + +mkdir -p ${REPO_ROOT}/bin + +echo ">>> Downloading Istio ${ISTIO_VER}" +cd ${REPO_ROOT}/bin && \ +curl -L https://istio.io/downloadIstio | ISTIO_VERSION=${ISTIO_VER} sh - + +echo ">>> Installing Istio ${ISTIO_VER}" +${REPO_ROOT}/bin/istio-${ISTIO_VER}/bin/istioctl manifest install --set profile=default --skip-confirmation \ + --set values.pilot.resources.requests.cpu=100m \ + --set values.pilot.resources.requests.memory=100Mi + +kubectl apply -f https://raw.githubusercontent.com/istio/istio/release-1.8/samples/addons/prometheus.yaml +kubectl -n istio-system rollout status deployment/prometheus + +kubectl -n istio-system get all + +echo '>>> Installing Flagger' +kubectl apply -k ${REPO_ROOT}/kustomize/istio + +kubectl -n istio-system set image deployment/flagger flagger=test/flagger:latest +kubectl -n istio-system rollout status deployment/flagger diff --git a/test/istio/run.sh b/test/istio/run.sh new file mode 100755 index 00000000..6b20dae3 --- /dev/null +++ b/test/istio/run.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -o errexit + +DIR="$(cd "$(dirname "$0")" && pwd)" + +"$DIR"/install.sh + +"$DIR"/test-init.sh +"$DIR"/test-canary.sh + +"$DIR"/test-init.sh +"$DIR"/test-skip-analysis.sh + +"$DIR"/test-init.sh +"$DIR"/test-delegation.sh diff --git a/test/istio/test-canary.sh b/test/istio/test-canary.sh new file mode 100755 index 00000000..2954626e --- /dev/null +++ b/test/istio/test-canary.sh @@ -0,0 +1,608 @@ +#!/usr/bin/env bash + +# This script runs e2e tests for Canary, B/G and A/B initialization, analysis and promotion +# Prerequisites: Kubernetes Kind and Istio + +set -o errexit + +echo '>>> Create latency metric template' +cat <>> Initialising canaries' +cat <>> Waiting for primary to be ready' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false + kubectl -n test get canary/podinfo-service | grep 'Initialized' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n istio-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary initialization test passed' + +passed=$(kubectl -n test get svc/podinfo -oyaml 2>&1 | { grep annotations-test || true; }) +if [ -z "$passed" ]; then + echo -e '\u2716 podinfo annotations test failed' + exit 1 +fi +passed=$(kubectl -n test get svc/podinfo -oyaml 2>&1 | { grep labels-test || true; }) +if [ -z "$passed" ]; then + echo -e '\u2716 podinfo labels test failed' + exit 1 +fi +passed=$(kubectl -n test get svc/podinfo -o jsonpath='{.spec.selector.app}' 2>&1 | { grep podinfo-primary || true; }) +if [ -z "$passed" ]; then + echo -e '\u2716 podinfo selector test failed' + exit 1 +fi +passed=$(kubectl -n test get svc/podinfo-service-canary -o jsonpath='{.spec.selector.app}' 2>&1 | { grep podinfo || true; }) +if [ -z "$passed" ]; then + echo -e '\u2716 podinfo-service selector test failed' + exit 1 +fi + +echo '✔ Canary service custom metadata test passed' + +echo '>>> Triggering canary deployment' +kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1 + +echo '>>> Waiting for canary promotion' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test describe deployment/podinfo-primary | grep '3.1.1' && ok=true || ok=false + sleep 10 + kubectl -n istio-system logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n test describe deployment/podinfo + kubectl -n test describe deployment/podinfo-primary + kubectl -n istio-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '>>> Waiting for canary finalization' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n istio-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary promotion test passed' + +if [[ "$1" = "canary" ]]; then + exit 0 +fi + +cat <>> Triggering B/G deployment' +kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.2 + +echo '>>> Waiting for B/G promotion' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test describe deployment/podinfo-primary | grep '3.1.2' && ok=true || ok=false + sleep 10 + kubectl -n istio-system logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n test describe deployment/podinfo + kubectl -n test describe deployment/podinfo-primary + kubectl -n istio-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '>>> Waiting for B/G finalization' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n istio-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ B/G promotion test passed' + +cat <>> Triggering A/B testing' +kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.3 + +echo '>>> Waiting for A/B testing promotion' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test describe deployment/podinfo-primary | grep '3.1.3' && ok=true || ok=false + sleep 10 + kubectl -n istio-system logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n test describe deployment/podinfo + kubectl -n test describe deployment/podinfo-primary + kubectl -n istio-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ A/B testing promotion test passed' + +cat <>> Waiting for finalizers to be present' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl get canary podinfo -n test -o jsonpath='{.metadata.finalizers}' | grep "finalizer.flagger.app" && ok=true || ok=false + sleep 10 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n test describe canary/podinfo + echo "No more retries left" + exit 1 + fi +done + +kubectl delete canary podinfo -n test + +echo '>>> Waiting for primary to revert' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl get deployment podinfo -n test -o jsonpath='{.spec.replicas}' | grep 1 && ok=true || ok=false + sleep 10 + kubectl -n istio-system logs deployment/flagger --tail 10 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n test describe canary/podinfo + echo "No more retries left" + exit 1 + fi +done +echo '✔ Delete testing passed' + + + +cat <>> Waiting for canary to initialize' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n istio-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +kubectl delete canary podinfo -n test + +echo '>>> Waiting for revert' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl get svc/podinfo vs/podinfo -n test -o jsonpath="{range .items[*]}{.metadata.name}{'\n'}{end}" | wc -l | grep 2 && ok=true || ok=false + sleep 10 + kubectl -n istio-system logs deployment/flagger --tail 10 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n test describe canary/podinfo + kubectl -n test describe svc/podinfo + kubectl -n test describe vs/podinfo + echo "No more retries left" + exit 1 + fi +done +echo '✔ Revert testing passed' + + +kubectl -n istio-system logs deployment/flagger + +echo '✔ All tests passed' diff --git a/test/istio/test-delegation.sh b/test/istio/test-delegation.sh new file mode 100755 index 00000000..511abd0e --- /dev/null +++ b/test/istio/test-delegation.sh @@ -0,0 +1,149 @@ +#!/usr/bin/env bash + +# This script runs e2e tests for when the canary delegation is enabled +# Prerequisites: Kubernetes Kind and Istio + +set -o errexit + +echo '>>> Set pilot env to enable virtual service delegate' +kubectl -n istio-system set env deploy istiod PILOT_ENABLE_VIRTUAL_SERVICE_DELEGATE=true +kubectl -n istio-system rollout status deploy istiod + +echo '>>> Initialising Gateway' +cat <>> Initialising root virtual service' +cat <>> Initialising canary for delegate' +cat <>> Waiting for primary to be ready' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n istio-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary initialization test passed' + +echo '>>> Triggering canary deployment' +kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1 + +echo '>>> Waiting for canary promotion' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test describe deployment/podinfo-primary | grep '3.1.1' && ok=true || ok=false + sleep 10 + kubectl -n istio-system logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n test describe deployment/podinfo + kubectl -n test describe deployment/podinfo-primary + kubectl -n istio-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '>>> Waiting for canary finalization' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n istio-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '>>> Set pilot env to disable virtual service delegate' +kubectl -n istio-system set env deploy istiod PILOT_ENABLE_VIRTUAL_SERVICE_DELEGATE=false +kubectl -n istio-system rollout status deploy istiod + +echo '✔ Canary promotion test passed' + +if [[ "$1" = "canary" ]]; then + exit 0 +fi diff --git a/test/istio/test-init.sh b/test/istio/test-init.sh new file mode 100755 index 00000000..4f0ab745 --- /dev/null +++ b/test/istio/test-init.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +# This script setups the scenarios for istio tests by creating a Kubernetes namespace, installing the load tester and a test workload (podinfo) +# Prerequisites: Kubernetes Kind and Istio + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) + +echo '>>> Delete test namespace' +kubectl delete namespace test --ignore-not-found=true --wait=true + +echo '>>> Creating test namespace' +kubectl create namespace test +kubectl label namespace test istio-injection=enabled +kubectl annotate namespace test linkerd.io/inject=enabled + +echo '>>> Installing the load tester' +kubectl apply -k ${REPO_ROOT}/kustomize/tester +kubectl -n test rollout status deployment/flagger-loadtester + +echo '>>> Deploy podinfo' +kubectl apply -f ${REPO_ROOT}/test/e2e-workload.yaml diff --git a/test/istio/test-skip-analysis.sh b/test/istio/test-skip-analysis.sh new file mode 100755 index 00000000..621fe285 --- /dev/null +++ b/test/istio/test-skip-analysis.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash + +# This script runs e2e tests for when the canary analysis is skipped +# Prerequisites: Kubernetes Kind and Istio + +set -o errexit + +echo '>>> Initialising canary' +cat <>> Waiting for primary to be ready' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n istio-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary initialization test passed' + +echo '>>> Triggering canary deployment' +kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1 + +echo '>>> Waiting for canary promotion' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test describe deployment/podinfo-primary | grep '3.1.1' && ok=true || ok=false + sleep 10 + kubectl -n istio-system logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n test describe deployment/podinfo + kubectl -n test describe deployment/podinfo-primary + kubectl -n istio-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '>>> Waiting for canary finalization' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n istio-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary promotion test passed' + +if [[ "$1" = "canary" ]]; then + exit 0 +fi + +echo '>>> Triggering canary deployment with a bad release (non existent docker image)' +kubectl -n test set image deployment/podinfo podinfod=stefanprodan/potato:1.0.0 + +echo '>>> Waiting for canary to fail' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl get canary/podinfo -n test -o=jsonpath='{.status.phase}' | grep 'Failed' && ok=true || ok=false + sleep 10 + kubectl -n istio-system logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n test describe deployment/podinfo + kubectl -n test describe deployment/podinfo-primary + kubectl -n istio-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '>>> Confirm primary pod is still running and with correct version' +retries=50 +count=0 +ok=false +until ${okImage} && ${okRunning}; do + kubectl get deployment podinfo-primary -n test -o jsonpath='{.spec.replicas}' | grep 1 && okRunning=true || okRunning=false + kubectl -n test describe deployment/podinfo-primary | grep '3.1.3' && okImage=true || okImage=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n istio-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +kubectl -n istio-system logs deployment/flagger + +echo '✔ All tests passed'