From 2884a80d319e716f71516b429aa891a00456d040 Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Sun, 20 Dec 2020 14:13:56 +0200 Subject: [PATCH 01/19] Disable CircleCI Signed-off-by: Stefan Prodan --- .circleci/config.yml | 2 ++ .github/_main.workflow | 17 ----------------- 2 files changed, 2 insertions(+), 17 deletions(-) delete mode 100644 .github/_main.workflow diff --git a/.circleci/config.yml b/.circleci/config.yml index 39f30353..e328734b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -223,6 +223,8 @@ workflows: ignore: - gh-pages - /^user-.*/ + - github-actions + - main - e2e-kubernetes-testing: requires: - build-binary diff --git a/.github/_main.workflow b/.github/_main.workflow deleted file mode 100644 index 114e4ea8..00000000 --- a/.github/_main.workflow +++ /dev/null @@ -1,17 +0,0 @@ -workflow "Publish Helm charts" { - on = "push" - resolves = ["helm-push"] -} - -action "helm-lint" { - uses = "stefanprodan/gh-actions/helm@master" - args = ["lint charts/*"] -} - -action "helm-push" { - needs = ["helm-lint"] - uses = "stefanprodan/gh-actions/helm-gh-pages@master" - args = ["charts/*","https://flagger.app"] - secrets = ["GITHUB_TOKEN"] -} - From 3ba27628050179be846975edbaa89423e13d735f Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Sun, 20 Dec 2020 17:06:19 +0200 Subject: [PATCH 02/19] Add multi-arch Dockerfile Signed-off-by: Stefan Prodan --- Dockerfile | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 5f71bd6f..a45c9df1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,29 @@ +FROM golang:1.15-alpine as builder + +ARG TARGETPLATFORM + +WORKDIR /workspace + +# copy modules manifests +COPY go.mod go.mod +COPY go.sum go.sum + +# cache modules +RUN go mod download + +# copy source code +COPY cmd/ cmd/ +COPY pkg/ pkg/ + +# build +RUN CGO_ENABLED=0 go build -a -o flagger ./cmd/flagger + FROM alpine:3.12 RUN apk --no-cache add ca-certificates USER nobody -COPY --chown=nobody:nobody /bin/flagger . +COPY --from=builder --chown=nobody:nobody /workspace/flagger . ENTRYPOINT ["./flagger"] From b4af9e5f32b1fcc34a9d727d3f5a1daa3f3a6b4e Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Sun, 20 Dec 2020 17:07:18 +0200 Subject: [PATCH 03/19] Add build workflow Signed-off-by: Stefan Prodan --- .github/workflows/build.yaml | 37 ++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 .github/workflows/build.yaml diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 00000000..d65441d1 --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,37 @@ +name: build + +on: + pull_request: + push: + branches: + - main + - github-actions + +jobs: + container: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Restore Go cache + uses: actions/cache@v1 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + - name: Setup Go + uses: actions/setup-go@v2 + with: + go-version: 1.15.x + - name: Run tests + run: make test + - name: Check if working tree is dirty + run: | + if [[ $(git diff --stat) != '' ]]; then + git --no-pager diff + echo 'run make test and commit changes' + exit 1 + fi + - name: Build container image + run: docker build -t test/flagger:latest . From be9b03d99bebc307f29b38a1c31bd5519c866f1e Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Sun, 20 Dec 2020 17:07:32 +0200 Subject: [PATCH 04/19] Add release workflow Signed-off-by: Stefan Prodan --- .github/workflows/release.yml | 73 +++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..668ecc6c --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,73 @@ +name: release +on: + push: + tags: + - 'v*' + +jobs: + build-push: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Prepare + id: prep + run: | + VERSION=sha-${GITHUB_SHA::8} + if [[ $GITHUB_REF == refs/tags/* ]]; then + VERSION=${GITHUB_REF/refs\/tags\//} + fi + echo ::set-output name=BUILD_DATE::$(date -u +'%Y-%m-%dT%H:%M:%SZ') + echo ::set-output name=VERSION::${VERSION} + - name: Setup QEMU + uses: docker/setup-qemu-action@v1 + with: + platforms: all + - name: Setup Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v1 + with: + buildkitd-flags: "--debug" + - name: Login to GitHub Container Registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: fluxcdbot + password: ${{ secrets.GHCR_TOKEN }} + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: fluxcdbot + password: ${{ secrets.DOCKER_FLUXCD_PASSWORD }} + - name: Publish image + uses: docker/build-push-action@v2 + with: + push: true + builder: ${{ steps.buildx.outputs.name }} + context: . + file: ./Dockerfile + platforms: linux/amd64,linux/arm/v7,linux/arm64 + tags: | + ghcr.io/fluxcd/flagger:${{ steps.prep.outputs.VERSION }} + labels: | + org.opencontainers.image.title=${{ github.event.repository.name }} + org.opencontainers.image.description=${{ github.event.repository.description }} + org.opencontainers.image.url=${{ github.event.repository.html_url }} + org.opencontainers.image.source=${{ github.event.repository.html_url }} + org.opencontainers.image.revision=${{ github.sha }} + org.opencontainers.image.version=${{ steps.prep.outputs.VERSION }} + org.opencontainers.image.created=${{ steps.prep.outputs.BUILD_DATE }} + - name: Check images + run: | + docker buildx imagetools inspect docker.io/fluxcd/flagger:${{ steps.prep.outputs.VERSION }} + docker buildx imagetools inspect ghcr.io/fluxcd/flagger:${{ steps.prep.outputs.VERSION }} + - name: Create release + uses: actions/create-release@latest + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ github.ref }} + release_name: ${{ github.ref }} + draft: false + prerelease: false + body: | + [CHANGELOG](https://github.com/fluxcd/flagger/blob/main/CHANGELOG.md) From a2774d92da8cea75ad1dc0615338cef28143c6a0 Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Sun, 20 Dec 2020 17:07:45 +0200 Subject: [PATCH 05/19] Add Istio e2e tests Signed-off-by: Stefan Prodan --- .github/workflows/build.yaml | 4 + .github/workflows/e2e.yaml | 28 ++ pkg/loadtester/task_ngrinder.go | 2 +- test/istio/install.sh | 28 ++ test/istio/run.sh | 16 + test/istio/test-canary.sh | 608 +++++++++++++++++++++++++++++++ test/istio/test-delegation.sh | 149 ++++++++ test/istio/test-init.sh | 23 ++ test/istio/test-skip-analysis.sh | 138 +++++++ 9 files changed, 995 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/e2e.yaml create mode 100755 test/istio/install.sh create mode 100755 test/istio/run.sh create mode 100755 test/istio/test-canary.sh create mode 100755 test/istio/test-delegation.sh create mode 100755 test/istio/test-init.sh create mode 100755 test/istio/test-skip-analysis.sh diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index d65441d1..c124734b 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -24,6 +24,10 @@ jobs: uses: actions/setup-go@v2 with: go-version: 1.15.x + - name: Download modules + run: | + go mod download + go install golang.org/x/tools/cmd/goimports - name: Run tests run: make test - name: Check if working tree is dirty diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml new file mode 100644 index 00000000..41dfab38 --- /dev/null +++ b/.github/workflows/e2e.yaml @@ -0,0 +1,28 @@ +name: e2e + +on: + pull_request: + push: + branches: + - main + - github-actions + +jobs: + kind: + runs-on: ubuntu-latest + strategy: + matrix: + provider: + - istio + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Setup Kubernetes + uses: engineerd/setup-kind@v0.5.0 + - name: Build container image + run: docker build -t test/flagger:latest . + - name: Load test image + run: kind load docker-image test/flagger:latest + - name: Run tests + run: | + ./test/${{ matrix['provider'] }}/run.sh diff --git a/pkg/loadtester/task_ngrinder.go b/pkg/loadtester/task_ngrinder.go index ab5ba5df..488cbf28 100644 --- a/pkg/loadtester/task_ngrinder.go +++ b/pkg/loadtester/task_ngrinder.go @@ -68,7 +68,7 @@ type NGrinderTask struct { } func (task *NGrinderTask) Hash() string { - return hash(task.canary + string(task.cloneId)) + return hash(task.canary + fmt.Sprint(task.cloneId)) } // nGrinder REST endpoints diff --git a/test/istio/install.sh b/test/istio/install.sh new file mode 100755 index 00000000..82e17a1d --- /dev/null +++ b/test/istio/install.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +set -o errexit + +ISTIO_VER="1.8.0" +REPO_ROOT=$(git rev-parse --show-toplevel) + +mkdir -p ${REPO_ROOT}/bin + +echo ">>> Downloading Istio ${ISTIO_VER}" +cd ${REPO_ROOT}/bin && \ +curl -L https://istio.io/downloadIstio | ISTIO_VERSION=${ISTIO_VER} sh - + +echo ">>> Installing Istio ${ISTIO_VER}" +${REPO_ROOT}/bin/istio-${ISTIO_VER}/bin/istioctl manifest install --set profile=default --skip-confirmation \ + --set values.pilot.resources.requests.cpu=100m \ + --set values.pilot.resources.requests.memory=100Mi + +kubectl apply -f https://raw.githubusercontent.com/istio/istio/release-1.8/samples/addons/prometheus.yaml +kubectl -n istio-system rollout status deployment/prometheus + +kubectl -n istio-system get all + +echo '>>> Installing Flagger' +kubectl apply -k ${REPO_ROOT}/kustomize/istio + +kubectl -n istio-system set image deployment/flagger flagger=test/flagger:latest +kubectl -n istio-system rollout status deployment/flagger diff --git a/test/istio/run.sh b/test/istio/run.sh new file mode 100755 index 00000000..6b20dae3 --- /dev/null +++ b/test/istio/run.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -o errexit + +DIR="$(cd "$(dirname "$0")" && pwd)" + +"$DIR"/install.sh + +"$DIR"/test-init.sh +"$DIR"/test-canary.sh + +"$DIR"/test-init.sh +"$DIR"/test-skip-analysis.sh + +"$DIR"/test-init.sh +"$DIR"/test-delegation.sh diff --git a/test/istio/test-canary.sh b/test/istio/test-canary.sh new file mode 100755 index 00000000..2954626e --- /dev/null +++ b/test/istio/test-canary.sh @@ -0,0 +1,608 @@ +#!/usr/bin/env bash + +# This script runs e2e tests for Canary, B/G and A/B initialization, analysis and promotion +# Prerequisites: Kubernetes Kind and Istio + +set -o errexit + +echo '>>> Create latency metric template' +cat <>> Initialising canaries' +cat <>> Waiting for primary to be ready' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false + kubectl -n test get canary/podinfo-service | grep 'Initialized' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n istio-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary initialization test passed' + +passed=$(kubectl -n test get svc/podinfo -oyaml 2>&1 | { grep annotations-test || true; }) +if [ -z "$passed" ]; then + echo -e '\u2716 podinfo annotations test failed' + exit 1 +fi +passed=$(kubectl -n test get svc/podinfo -oyaml 2>&1 | { grep labels-test || true; }) +if [ -z "$passed" ]; then + echo -e '\u2716 podinfo labels test failed' + exit 1 +fi +passed=$(kubectl -n test get svc/podinfo -o jsonpath='{.spec.selector.app}' 2>&1 | { grep podinfo-primary || true; }) +if [ -z "$passed" ]; then + echo -e '\u2716 podinfo selector test failed' + exit 1 +fi +passed=$(kubectl -n test get svc/podinfo-service-canary -o jsonpath='{.spec.selector.app}' 2>&1 | { grep podinfo || true; }) +if [ -z "$passed" ]; then + echo -e '\u2716 podinfo-service selector test failed' + exit 1 +fi + +echo '✔ Canary service custom metadata test passed' + +echo '>>> Triggering canary deployment' +kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1 + +echo '>>> Waiting for canary promotion' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test describe deployment/podinfo-primary | grep '3.1.1' && ok=true || ok=false + sleep 10 + kubectl -n istio-system logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n test describe deployment/podinfo + kubectl -n test describe deployment/podinfo-primary + kubectl -n istio-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '>>> Waiting for canary finalization' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n istio-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary promotion test passed' + +if [[ "$1" = "canary" ]]; then + exit 0 +fi + +cat <>> Triggering B/G deployment' +kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.2 + +echo '>>> Waiting for B/G promotion' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test describe deployment/podinfo-primary | grep '3.1.2' && ok=true || ok=false + sleep 10 + kubectl -n istio-system logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n test describe deployment/podinfo + kubectl -n test describe deployment/podinfo-primary + kubectl -n istio-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '>>> Waiting for B/G finalization' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n istio-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ B/G promotion test passed' + +cat <>> Triggering A/B testing' +kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.3 + +echo '>>> Waiting for A/B testing promotion' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test describe deployment/podinfo-primary | grep '3.1.3' && ok=true || ok=false + sleep 10 + kubectl -n istio-system logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n test describe deployment/podinfo + kubectl -n test describe deployment/podinfo-primary + kubectl -n istio-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ A/B testing promotion test passed' + +cat <>> Waiting for finalizers to be present' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl get canary podinfo -n test -o jsonpath='{.metadata.finalizers}' | grep "finalizer.flagger.app" && ok=true || ok=false + sleep 10 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n test describe canary/podinfo + echo "No more retries left" + exit 1 + fi +done + +kubectl delete canary podinfo -n test + +echo '>>> Waiting for primary to revert' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl get deployment podinfo -n test -o jsonpath='{.spec.replicas}' | grep 1 && ok=true || ok=false + sleep 10 + kubectl -n istio-system logs deployment/flagger --tail 10 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n test describe canary/podinfo + echo "No more retries left" + exit 1 + fi +done +echo '✔ Delete testing passed' + + + +cat <>> Waiting for canary to initialize' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n istio-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +kubectl delete canary podinfo -n test + +echo '>>> Waiting for revert' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl get svc/podinfo vs/podinfo -n test -o jsonpath="{range .items[*]}{.metadata.name}{'\n'}{end}" | wc -l | grep 2 && ok=true || ok=false + sleep 10 + kubectl -n istio-system logs deployment/flagger --tail 10 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n test describe canary/podinfo + kubectl -n test describe svc/podinfo + kubectl -n test describe vs/podinfo + echo "No more retries left" + exit 1 + fi +done +echo '✔ Revert testing passed' + + +kubectl -n istio-system logs deployment/flagger + +echo '✔ All tests passed' diff --git a/test/istio/test-delegation.sh b/test/istio/test-delegation.sh new file mode 100755 index 00000000..511abd0e --- /dev/null +++ b/test/istio/test-delegation.sh @@ -0,0 +1,149 @@ +#!/usr/bin/env bash + +# This script runs e2e tests for when the canary delegation is enabled +# Prerequisites: Kubernetes Kind and Istio + +set -o errexit + +echo '>>> Set pilot env to enable virtual service delegate' +kubectl -n istio-system set env deploy istiod PILOT_ENABLE_VIRTUAL_SERVICE_DELEGATE=true +kubectl -n istio-system rollout status deploy istiod + +echo '>>> Initialising Gateway' +cat <>> Initialising root virtual service' +cat <>> Initialising canary for delegate' +cat <>> Waiting for primary to be ready' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n istio-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary initialization test passed' + +echo '>>> Triggering canary deployment' +kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1 + +echo '>>> Waiting for canary promotion' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test describe deployment/podinfo-primary | grep '3.1.1' && ok=true || ok=false + sleep 10 + kubectl -n istio-system logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n test describe deployment/podinfo + kubectl -n test describe deployment/podinfo-primary + kubectl -n istio-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '>>> Waiting for canary finalization' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n istio-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '>>> Set pilot env to disable virtual service delegate' +kubectl -n istio-system set env deploy istiod PILOT_ENABLE_VIRTUAL_SERVICE_DELEGATE=false +kubectl -n istio-system rollout status deploy istiod + +echo '✔ Canary promotion test passed' + +if [[ "$1" = "canary" ]]; then + exit 0 +fi diff --git a/test/istio/test-init.sh b/test/istio/test-init.sh new file mode 100755 index 00000000..4f0ab745 --- /dev/null +++ b/test/istio/test-init.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +# This script setups the scenarios for istio tests by creating a Kubernetes namespace, installing the load tester and a test workload (podinfo) +# Prerequisites: Kubernetes Kind and Istio + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) + +echo '>>> Delete test namespace' +kubectl delete namespace test --ignore-not-found=true --wait=true + +echo '>>> Creating test namespace' +kubectl create namespace test +kubectl label namespace test istio-injection=enabled +kubectl annotate namespace test linkerd.io/inject=enabled + +echo '>>> Installing the load tester' +kubectl apply -k ${REPO_ROOT}/kustomize/tester +kubectl -n test rollout status deployment/flagger-loadtester + +echo '>>> Deploy podinfo' +kubectl apply -f ${REPO_ROOT}/test/e2e-workload.yaml diff --git a/test/istio/test-skip-analysis.sh b/test/istio/test-skip-analysis.sh new file mode 100755 index 00000000..621fe285 --- /dev/null +++ b/test/istio/test-skip-analysis.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash + +# This script runs e2e tests for when the canary analysis is skipped +# Prerequisites: Kubernetes Kind and Istio + +set -o errexit + +echo '>>> Initialising canary' +cat <>> Waiting for primary to be ready' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n istio-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary initialization test passed' + +echo '>>> Triggering canary deployment' +kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1 + +echo '>>> Waiting for canary promotion' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test describe deployment/podinfo-primary | grep '3.1.1' && ok=true || ok=false + sleep 10 + kubectl -n istio-system logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n test describe deployment/podinfo + kubectl -n test describe deployment/podinfo-primary + kubectl -n istio-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '>>> Waiting for canary finalization' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n istio-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary promotion test passed' + +if [[ "$1" = "canary" ]]; then + exit 0 +fi + +echo '>>> Triggering canary deployment with a bad release (non existent docker image)' +kubectl -n test set image deployment/podinfo podinfod=stefanprodan/potato:1.0.0 + +echo '>>> Waiting for canary to fail' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl get canary/podinfo -n test -o=jsonpath='{.status.phase}' | grep 'Failed' && ok=true || ok=false + sleep 10 + kubectl -n istio-system logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n test describe deployment/podinfo + kubectl -n test describe deployment/podinfo-primary + kubectl -n istio-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '>>> Confirm primary pod is still running and with correct version' +retries=50 +count=0 +ok=false +until ${okImage} && ${okRunning}; do + kubectl get deployment podinfo-primary -n test -o jsonpath='{.spec.replicas}' | grep 1 && okRunning=true || okRunning=false + kubectl -n test describe deployment/podinfo-primary | grep '3.1.3' && okImage=true || okImage=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n istio-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +kubectl -n istio-system logs deployment/flagger + +echo '✔ All tests passed' From 7bef999c413fcf5d370beb4aabd7d0ad3370f85a Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Sun, 20 Dec 2020 17:47:45 +0200 Subject: [PATCH 06/19] Add Linkerd e2e tests Signed-off-by: Stefan Prodan --- .github/workflows/e2e.yaml | 1 + test/linkerd/install.sh | 24 ++++ test/linkerd/run.sh | 13 ++ test/linkerd/test-canary.sh | 242 ++++++++++++++++++++++++++++++++++++ test/linkerd/test-init.sh | 23 ++++ test/linkerd/test-steps.sh | 95 ++++++++++++++ 6 files changed, 398 insertions(+) create mode 100755 test/linkerd/install.sh create mode 100755 test/linkerd/run.sh create mode 100755 test/linkerd/test-canary.sh create mode 100755 test/linkerd/test-init.sh create mode 100755 test/linkerd/test-steps.sh diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 41dfab38..5638abe6 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -14,6 +14,7 @@ jobs: matrix: provider: - istio + - linkerd steps: - name: Checkout uses: actions/checkout@v2 diff --git a/test/linkerd/install.sh b/test/linkerd/install.sh new file mode 100755 index 00000000..7bfd15b0 --- /dev/null +++ b/test/linkerd/install.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -o errexit + +LINKERD_VER="stable-2.8.1" +REPO_ROOT=$(git rev-parse --show-toplevel) + +mkdir -p ${REPO_ROOT}/bin + +curl -SsL https://github.com/linkerd/linkerd2/releases/download/${LINKERD_VER}/linkerd2-cli-${LINKERD_VER}-linux > ${REPO_ROOT}/bin/linkerd +chmod +x ${REPO_ROOT}/bin/linkerd + +echo ">>> Installing Linkerd ${LINKERD_VER}" +${REPO_ROOT}/bin/linkerd install | kubectl apply -f - +${REPO_ROOT}/bin/linkerd check + +kubectl -n linkerd rollout status deployment/linkerd-controller +kubectl -n linkerd rollout status deployment/linkerd-proxy-injector + +echo '>>> Installing Flagger' +kubectl apply -k ${REPO_ROOT}/kustomize/linkerd + +kubectl -n linkerd set image deployment/flagger flagger=test/flagger:latest +kubectl -n linkerd rollout status deployment/flagger diff --git a/test/linkerd/run.sh b/test/linkerd/run.sh new file mode 100755 index 00000000..e3174d56 --- /dev/null +++ b/test/linkerd/run.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -o errexit + +DIR="$(cd "$(dirname "$0")" && pwd)" + +"$DIR"/install.sh + +"$DIR"/test-init.sh +"$DIR"/test-canary.sh + +"$DIR"/test-init.sh +"$DIR"/test-steps.sh diff --git a/test/linkerd/test-canary.sh b/test/linkerd/test-canary.sh new file mode 100755 index 00000000..c7a85aff --- /dev/null +++ b/test/linkerd/test-canary.sh @@ -0,0 +1,242 @@ +#!/usr/bin/env bash + +# This script runs Linkerd e2e tests for Canary initialization, analysis and promotion + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) + +cat <>> Waiting for primary to be ready' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n linkerd logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary initialization test passed' + +passed=$(kubectl -n test get svc/podinfo -o jsonpath='{.spec.selector.app}' 2>&1 | { grep podinfo-primary || true; }) +if [ -z "$passed" ]; then + echo -e '\u2716 podinfo selector test failed' + exit 1 +fi +passed=$(kubectl -n test get svc/podinfo-service-canary -o jsonpath='{.spec.selector.app}' 2>&1 | { grep podinfo || true; }) +if [ -z "$passed" ]; then + echo -e '\u2716 podinfo-service selector test failed' + exit 1 +fi + +echo '✔ Canary service custom metadata test passed' + +echo '>>> Triggering canary deployment' +kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1 + +echo '>>> Waiting for canary promotion' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test describe deployment/podinfo-primary | grep '3.1.1' && ok=true || ok=false + sleep 10 + kubectl -n linkerd logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n linkerd logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '>>> Waiting for canary finalization' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n linkerd logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary promotion test passed' + +cat <>> Triggering canary deployment' +kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.2 + +echo '>>> Waiting for canary rollback' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Failed' && ok=true || ok=false + sleep 10 + kubectl -n linkerd logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n linkerd logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary rollback test passed' \ No newline at end of file diff --git a/test/linkerd/test-init.sh b/test/linkerd/test-init.sh new file mode 100755 index 00000000..4f0ab745 --- /dev/null +++ b/test/linkerd/test-init.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +# This script setups the scenarios for istio tests by creating a Kubernetes namespace, installing the load tester and a test workload (podinfo) +# Prerequisites: Kubernetes Kind and Istio + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) + +echo '>>> Delete test namespace' +kubectl delete namespace test --ignore-not-found=true --wait=true + +echo '>>> Creating test namespace' +kubectl create namespace test +kubectl label namespace test istio-injection=enabled +kubectl annotate namespace test linkerd.io/inject=enabled + +echo '>>> Installing the load tester' +kubectl apply -k ${REPO_ROOT}/kustomize/tester +kubectl -n test rollout status deployment/flagger-loadtester + +echo '>>> Deploy podinfo' +kubectl apply -f ${REPO_ROOT}/test/e2e-workload.yaml diff --git a/test/linkerd/test-steps.sh b/test/linkerd/test-steps.sh new file mode 100755 index 00000000..ebb122fe --- /dev/null +++ b/test/linkerd/test-steps.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash + +# This script runs Linkerd e2e tests for Canary initialization, analysis and promotion + +set -o errexit + +cat <>> Waiting for primary to be ready' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n linkerd logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary initialization test passed' + +echo '>>> Triggering canary deployment' +kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1 + +echo '>>> Waiting for canary promotion' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test describe deployment/podinfo-primary | grep '3.1.1' && ok=true || ok=false + sleep 10 + kubectl -n linkerd logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n linkerd logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '>>> Waiting for canary finalization' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n linkerd logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary promotion test passed' From bb00f8cabd1dda36896a62368e2a2ca6ab0cd729 Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Sun, 20 Dec 2020 19:15:10 +0200 Subject: [PATCH 07/19] Add Contour e2e tests Signed-off-by: Stefan Prodan --- .github/workflows/e2e.yaml | 1 + test/contour/install.sh | 26 ++++ test/contour/run.sh | 11 ++ test/contour/test-canary.sh | 272 ++++++++++++++++++++++++++++++++++++ test/contour/test-init.sh | 24 ++++ 5 files changed, 334 insertions(+) create mode 100755 test/contour/install.sh create mode 100755 test/contour/run.sh create mode 100755 test/contour/test-canary.sh create mode 100755 test/contour/test-init.sh diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 5638abe6..aa4d7715 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -15,6 +15,7 @@ jobs: provider: - istio - linkerd + - contour steps: - name: Checkout uses: actions/checkout@v2 diff --git a/test/contour/install.sh b/test/contour/install.sh new file mode 100755 index 00000000..9058e0e6 --- /dev/null +++ b/test/contour/install.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +set -o errexit + +CONTOUR_VER="v1.7.0" +REPO_ROOT=$(git rev-parse --show-toplevel) + +mkdir -p ${REPO_ROOT}/bin + +echo '>>> Installing Contour' +kubectl apply -f https://projectcontour.io/quickstart/${CONTOUR_VER}/contour.yaml + +kubectl -n projectcontour rollout status deployment/contour +kubectl -n projectcontour get all + +echo '>>> Installing Flagger' +helm upgrade -i flagger ${REPO_ROOT}/charts/flagger \ +--namespace projectcontour \ +--set prometheus.install=true \ +--set meshProvider=contour \ +--set ingressClass=contour + +kubectl -n projectcontour set image deployment/flagger flagger=test/flagger:latest + +kubectl -n projectcontour rollout status deployment/flagger +kubectl -n projectcontour rollout status deployment/flagger-prometheus diff --git a/test/contour/run.sh b/test/contour/run.sh new file mode 100755 index 00000000..96efa472 --- /dev/null +++ b/test/contour/run.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -o errexit + +DIR="$(cd "$(dirname "$0")" && pwd)" + +"$DIR"/install.sh + +"$DIR"/test-init.sh +"$DIR"/test-canary.sh + diff --git a/test/contour/test-canary.sh b/test/contour/test-canary.sh new file mode 100755 index 00000000..f09aa464 --- /dev/null +++ b/test/contour/test-canary.sh @@ -0,0 +1,272 @@ +#!/usr/bin/env bash + +# This script runs e2e tests for Canary initialization, analysis and promotion +# Prerequisites: Kubernetes Kind and Contour ingress controller + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) + +cat <>> Waiting for primary to be ready' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n projectcontour logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +kubectl -n test get httpproxy podinfo -oyaml | grep 'projectcontour.io/ingress.class: contour' + +echo '✔ Canary initialization test passed' + +echo '>>> Triggering canary deployment' +kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1 + +echo '>>> Waiting for canary promotion' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test describe deployment/podinfo-primary | grep '3.1.1' && ok=true || ok=false + sleep 10 + kubectl -n projectcontour logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n projectcontour logs deployment/flagger + kubectl -n test get httpproxy podinfo -oyaml + echo "No more retries left" + exit 1 + fi +done + +echo '>>> Waiting for canary finalization' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n projectcontour logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary promotion test passed' + + +cat <>> Triggering canary deployment' +kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.2 + +echo '>>> Waiting for canary rollback' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Failed' && ok=true || ok=false + sleep 10 + kubectl -n projectcontour logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n projectcontour logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary rollback test passed' + +cat <>> Triggering A/B testing' +kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.3 + +echo '>>> Waiting for A/B testing promotion' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test describe deployment/podinfo-primary | grep '3.1.3' && ok=true || ok=false + sleep 10 + kubectl -n projectcontour logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n projectcontour logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ A/B testing promotion test passed' + +kubectl -n projectcontour logs deployment/flagger + +echo '✔ All tests passed' \ No newline at end of file diff --git a/test/contour/test-init.sh b/test/contour/test-init.sh new file mode 100755 index 00000000..14e3bce7 --- /dev/null +++ b/test/contour/test-init.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +# This script setups the scenarios for tests by creating a +# Kubernetes namespace, installing the load tester +# and a test workload (podinfo) + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) + +echo '>>> Delete test namespace' +kubectl delete namespace test --ignore-not-found=true --wait=true + +echo '>>> Creating test namespace' +kubectl create namespace test +kubectl label namespace test istio-injection=enabled +kubectl annotate namespace test linkerd.io/inject=enabled + +echo '>>> Installing the load tester' +kubectl apply -k ${REPO_ROOT}/kustomize/tester +kubectl -n test rollout status deployment/flagger-loadtester + +echo '>>> Deploy podinfo' +kubectl apply -f ${REPO_ROOT}/test/e2e-workload.yaml From ed7016058379795bca3cd8e8e178b2070cc7bc23 Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Sun, 20 Dec 2020 21:09:47 +0200 Subject: [PATCH 08/19] Add NGINX Ingress e2e tests Signed-off-by: Stefan Prodan --- .github/workflows/e2e.yaml | 1 + test/nginx/install.sh | 33 +++++ test/nginx/run.sh | 11 ++ test/nginx/test-canary.sh | 281 +++++++++++++++++++++++++++++++++++++ test/nginx/test-init.sh | 24 ++++ 5 files changed, 350 insertions(+) create mode 100755 test/nginx/install.sh create mode 100755 test/nginx/run.sh create mode 100755 test/nginx/test-canary.sh create mode 100755 test/nginx/test-init.sh diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index aa4d7715..e6e0d1c8 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -16,6 +16,7 @@ jobs: - istio - linkerd - contour + - nginx steps: - name: Checkout uses: actions/checkout@v2 diff --git a/test/nginx/install.sh b/test/nginx/install.sh new file mode 100755 index 00000000..5c3cbbbd --- /dev/null +++ b/test/nginx/install.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +set -o errexit + +NGINX_HELM_VERSION=3.15.2 # ingress v0.41.2 +REPO_ROOT=$(git rev-parse --show-toplevel) + +mkdir -p ${REPO_ROOT}/bin + +echo '>>> Installing NGINX Ingress' +kubectl create ns ingress-nginx +helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx +helm upgrade -i ingress-nginx ingress-nginx/ingress-nginx --version=${NGINX_HELM_VERSION} \ +--wait \ +--namespace ingress-nginx \ +--set controller.metrics.enabled=true \ +--set controller.podAnnotations."prometheus\.io/scrape"=true \ +--set controller.podAnnotations."prometheus\.io/port"=10254 \ +--set controller.service.type=NodePort + +kubectl -n ingress-nginx get all + +echo '>>> Installing Flagger' +helm upgrade -i flagger ${REPO_ROOT}/charts/flagger \ +--set crd.create=false \ +--namespace ingress-nginx \ +--set prometheus.install=true \ +--set meshProvider=nginx + +kubectl -n ingress-nginx set image deployment/flagger flagger=test/flagger:latest + +kubectl -n ingress-nginx rollout status deployment/flagger +kubectl -n ingress-nginx rollout status deployment/flagger-prometheus diff --git a/test/nginx/run.sh b/test/nginx/run.sh new file mode 100755 index 00000000..96efa472 --- /dev/null +++ b/test/nginx/run.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -o errexit + +DIR="$(cd "$(dirname "$0")" && pwd)" + +"$DIR"/install.sh + +"$DIR"/test-init.sh +"$DIR"/test-canary.sh + diff --git a/test/nginx/test-canary.sh b/test/nginx/test-canary.sh new file mode 100755 index 00000000..ff185418 --- /dev/null +++ b/test/nginx/test-canary.sh @@ -0,0 +1,281 @@ +#!/usr/bin/env bash + +# This script runs e2e tests for Canary initialization, analysis and promotion +# Prerequisites: Kubernetes Kind, Helm and NGINX ingress controller + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) + +cat <>> Create metric templates' +cat <>> Waiting for primary to be ready' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n ingress-nginx logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary initialization test passed' + +echo '>>> Triggering canary deployment' +kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1 + +echo '>>> Waiting for canary promotion' +retries=50 +count=0 +ok=false +failed=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Failed' && failed=true || failed=false + if ${failed}; then + kubectl -n ingress-nginx logs deployment/flagger + echo "Canary failed!" + exit 1 + fi + kubectl -n test describe deployment/podinfo-primary | grep '3.1.1' && ok=true || ok=false + sleep 10 + kubectl -n ingress-nginx logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n test describe deployment/podinfo + kubectl -n test describe deployment/podinfo-primary + kubectl -n ingress-nginx logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '>>> Waiting for canary finalization' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n ingress-nginx logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary promotion test passed' + +cat <>> Triggering A/B testing' +kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.2 + +echo '>>> Waiting for A/B testing promotion' +retries=6 +count=0 +ok=false +until ${ok}; do + kubectl -n test describe deployment/podinfo-primary | grep '3.1.2' && ok=true || ok=false + sleep 30 + kubectl -n ingress-nginx logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n test describe deployment/podinfo + kubectl -n test describe deployment/podinfo-primary + kubectl -n ingress-nginx logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ A/B testing promotion test passed' + +kubectl -n ingress-nginx logs deployment/flagger + +echo '✔ All tests passed' diff --git a/test/nginx/test-init.sh b/test/nginx/test-init.sh new file mode 100755 index 00000000..14e3bce7 --- /dev/null +++ b/test/nginx/test-init.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +# This script setups the scenarios for tests by creating a +# Kubernetes namespace, installing the load tester +# and a test workload (podinfo) + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) + +echo '>>> Delete test namespace' +kubectl delete namespace test --ignore-not-found=true --wait=true + +echo '>>> Creating test namespace' +kubectl create namespace test +kubectl label namespace test istio-injection=enabled +kubectl annotate namespace test linkerd.io/inject=enabled + +echo '>>> Installing the load tester' +kubectl apply -k ${REPO_ROOT}/kustomize/tester +kubectl -n test rollout status deployment/flagger-loadtester + +echo '>>> Deploy podinfo' +kubectl apply -f ${REPO_ROOT}/test/e2e-workload.yaml From 4fe4053cdd18722f6d3deccbeab06cae771e23ae Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Sun, 20 Dec 2020 21:38:07 +0200 Subject: [PATCH 09/19] Add workloads to e2e tests Signed-off-by: Stefan Prodan --- test/workloads/daemonset.yaml | 63 +++++++++++++++++++++++++++++++ test/workloads/deployment.yaml | 68 ++++++++++++++++++++++++++++++++++ test/workloads/init.sh | 23 ++++++++++++ 3 files changed, 154 insertions(+) create mode 100644 test/workloads/daemonset.yaml create mode 100644 test/workloads/deployment.yaml create mode 100755 test/workloads/init.sh diff --git a/test/workloads/daemonset.yaml b/test/workloads/daemonset.yaml new file mode 100644 index 00000000..a8f6abb8 --- /dev/null +++ b/test/workloads/daemonset.yaml @@ -0,0 +1,63 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: podinfo-ds + namespace: test + labels: + app.kubernetes.io/name: podinfo-ds +spec: + minReadySeconds: 5 + revisionHistoryLimit: 5 + selector: + matchLabels: + app.kubernetes.io/name: podinfo-ds + template: + metadata: + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9797" + labels: + app.kubernetes.io/name: podinfo-ds + spec: + containers: + - name: podinfod + image: stefanprodan/podinfo:3.1.0 + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 9898 + protocol: TCP + - name: http-metrics + containerPort: 9797 + protocol: TCP + - name: grpc + containerPort: 9999 + protocol: TCP + command: + - ./podinfo + - --port=9898 + - --port-metrics=9797 + - --grpc-port=9999 + - --grpc-service-name=podinfo + - --level=info + - --random-delay=false + - --random-error=false + livenessProbe: + httpGet: + port: 9898 + path: /healthz + initialDelaySeconds: 5 + timeoutSeconds: 5 + readinessProbe: + httpGet: + port: 9898 + path: /readyz + initialDelaySeconds: 5 + timeoutSeconds: 5 + resources: + limits: + cpu: 1000m + memory: 128Mi + requests: + cpu: 1m + memory: 16Mi diff --git a/test/workloads/deployment.yaml b/test/workloads/deployment.yaml new file mode 100644 index 00000000..175803d9 --- /dev/null +++ b/test/workloads/deployment.yaml @@ -0,0 +1,68 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: podinfo + namespace: test + labels: + app: podinfo +spec: + minReadySeconds: 5 + revisionHistoryLimit: 5 + progressDeadlineSeconds: 60 + strategy: + rollingUpdate: + maxUnavailable: 0 + type: RollingUpdate + selector: + matchLabels: + app: podinfo + template: + metadata: + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9797" + labels: + app: podinfo + spec: + containers: + - name: podinfod + image: stefanprodan/podinfo:3.1.0 + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 9898 + protocol: TCP + - name: http-metrics + containerPort: 9797 + protocol: TCP + - name: grpc + containerPort: 9999 + protocol: TCP + command: + - ./podinfo + - --port=9898 + - --port-metrics=9797 + - --grpc-port=9999 + - --grpc-service-name=podinfo + - --level=info + - --random-delay=false + - --random-error=false + livenessProbe: + httpGet: + port: 9898 + path: /healthz + initialDelaySeconds: 5 + timeoutSeconds: 5 + readinessProbe: + httpGet: + port: 9898 + path: /readyz + initialDelaySeconds: 5 + timeoutSeconds: 5 + resources: + limits: + cpu: 1000m + memory: 128Mi + requests: + cpu: 1m + memory: 16Mi diff --git a/test/workloads/init.sh b/test/workloads/init.sh new file mode 100755 index 00000000..3637c80e --- /dev/null +++ b/test/workloads/init.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +# This script creates the test app and load tester + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) + +echo '>>> Delete test namespace' +kubectl delete namespace test --ignore-not-found=true --wait=true + +echo '>>> Creating test namespace' +kubectl create namespace test +kubectl label namespace test istio-injection=enabled +kubectl annotate namespace test linkerd.io/inject=enabled + +echo '>>> Installing the load tester' +kubectl apply -k ${REPO_ROOT}/kustomize/tester +kubectl -n test rollout status deployment/flagger-loadtester + +echo '>>> Deploy podinfo' +kubectl apply -f ${REPO_ROOT}/test/workloads/deployment.yaml +kubectl apply -f ${REPO_ROOT}/test/workloads/daemonset.yaml From b25ff35e5bfbeb229f9eb0a1e8046e57c6f00293 Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Sun, 20 Dec 2020 21:39:10 +0200 Subject: [PATCH 10/19] Use test workloads in e2e Signed-off-by: Stefan Prodan --- .github/workflows/e2e.yaml | 6 +++--- test/contour/run.sh | 4 ++-- test/istio/run.sh | 7 ++++--- test/linkerd/run.sh | 5 +++-- test/nginx/run.sh | 4 ++-- test/nginx/test-canary.sh | 2 +- 6 files changed, 15 insertions(+), 13 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index e6e0d1c8..db1b10fd 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -23,9 +23,9 @@ jobs: - name: Setup Kubernetes uses: engineerd/setup-kind@v0.5.0 - name: Build container image - run: docker build -t test/flagger:latest . - - name: Load test image - run: kind load docker-image test/flagger:latest + run: | + docker build -t test/flagger:latest . + kind load docker-image test/flagger:latest - name: Run tests run: | ./test/${{ matrix['provider'] }}/run.sh diff --git a/test/contour/run.sh b/test/contour/run.sh index 96efa472..67153fa9 100755 --- a/test/contour/run.sh +++ b/test/contour/run.sh @@ -2,10 +2,10 @@ set -o errexit +REPO_ROOT=$(git rev-parse --show-toplevel) DIR="$(cd "$(dirname "$0")" && pwd)" "$DIR"/install.sh -"$DIR"/test-init.sh +"$REPO_ROOT"/test/workloads/init.sh "$DIR"/test-canary.sh - diff --git a/test/istio/run.sh b/test/istio/run.sh index 6b20dae3..b2a3301a 100755 --- a/test/istio/run.sh +++ b/test/istio/run.sh @@ -2,15 +2,16 @@ set -o errexit +REPO_ROOT=$(git rev-parse --show-toplevel) DIR="$(cd "$(dirname "$0")" && pwd)" "$DIR"/install.sh -"$DIR"/test-init.sh +"$REPO_ROOT"/test/workloads/init.sh "$DIR"/test-canary.sh -"$DIR"/test-init.sh +"$REPO_ROOT"/test/workloads/init.sh "$DIR"/test-skip-analysis.sh -"$DIR"/test-init.sh +"$REPO_ROOT"/test/workloads/init.sh "$DIR"/test-delegation.sh diff --git a/test/linkerd/run.sh b/test/linkerd/run.sh index e3174d56..d09d6557 100755 --- a/test/linkerd/run.sh +++ b/test/linkerd/run.sh @@ -2,12 +2,13 @@ set -o errexit +REPO_ROOT=$(git rev-parse --show-toplevel) DIR="$(cd "$(dirname "$0")" && pwd)" "$DIR"/install.sh -"$DIR"/test-init.sh +"$REPO_ROOT"/test/workloads/init.sh "$DIR"/test-canary.sh -"$DIR"/test-init.sh +"$REPO_ROOT"/test/workloads/init.sh "$DIR"/test-steps.sh diff --git a/test/nginx/run.sh b/test/nginx/run.sh index 96efa472..67153fa9 100755 --- a/test/nginx/run.sh +++ b/test/nginx/run.sh @@ -2,10 +2,10 @@ set -o errexit +REPO_ROOT=$(git rev-parse --show-toplevel) DIR="$(cd "$(dirname "$0")" && pwd)" "$DIR"/install.sh -"$DIR"/test-init.sh +"$REPO_ROOT"/test/workloads/init.sh "$DIR"/test-canary.sh - diff --git a/test/nginx/test-canary.sh b/test/nginx/test-canary.sh index ff185418..980c5f76 100755 --- a/test/nginx/test-canary.sh +++ b/test/nginx/test-canary.sh @@ -244,7 +244,7 @@ spec: timeout: 5s metadata: type: bash - cmd: "curl -sH 'x-user: insider' -H 'Host: app.example.com' http://ingress-nginx-controller.ingress-nginx | grep '3.1.2'" + cmd: "curl -sH 'x-user: insider' -H 'Host: app.example.com' http://ingress-nginx-controller.ingress-nginx | grep '3.1.1'" - name: load-test type: rollout url: http://flagger-loadtester.test/ From 73b7fc1cfc1d3f0034f229898c75ddeb76af6869 Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Sun, 20 Dec 2020 22:21:02 +0200 Subject: [PATCH 11/19] Add Traefik e2e tests Signed-off-by: Stefan Prodan --- test/nginx/test-canary.sh | 2 +- test/traefik/install.sh | 36 +++++++++ test/traefik/run.sh | 11 +++ test/traefik/test-canary.sh | 146 ++++++++++++++++++++++++++++++++++++ 4 files changed, 194 insertions(+), 1 deletion(-) create mode 100755 test/traefik/install.sh create mode 100755 test/traefik/run.sh create mode 100755 test/traefik/test-canary.sh diff --git a/test/nginx/test-canary.sh b/test/nginx/test-canary.sh index 980c5f76..baaccf28 100755 --- a/test/nginx/test-canary.sh +++ b/test/nginx/test-canary.sh @@ -244,7 +244,7 @@ spec: timeout: 5s metadata: type: bash - cmd: "curl -sH 'x-user: insider' -H 'Host: app.example.com' http://ingress-nginx-controller.ingress-nginx | grep '3.1.1'" + cmd: "curl -sH 'x-user: insider' -H 'Host: app.example.com' http://ingress-nginx-controller.ingress-nginx" - name: load-test type: rollout url: http://flagger-loadtester.test/ diff --git a/test/traefik/install.sh b/test/traefik/install.sh new file mode 100755 index 00000000..e238edd5 --- /dev/null +++ b/test/traefik/install.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +set -o errexit + +TRAEFIK_CHART_VERSION="9.11.0" # traefik 2.3.3 +REPO_ROOT=$(git rev-parse --show-toplevel) + +mkdir -p ${REPO_ROOT}/bin + +echo '>>> Creating traefik namespace' +kubectl create ns traefik + +echo '>>> Installing Traefik' +helm repo add traefik https://helm.traefik.io/traefik +cat <>> Installing Flagger' +helm upgrade -i flagger ${REPO_ROOT}/charts/flagger \ +--set crd.create=false \ +--namespace traefik \ +--set prometheus.install=true \ +--set meshProvider=traefik \ +--set image.repository=test\/flagger \ +--set image.tag=latest \ + +kubectl -n traefik rollout status deployment/flagger +kubectl -n projectcontour rollout status deployment/flagger-prometheus diff --git a/test/traefik/run.sh b/test/traefik/run.sh new file mode 100755 index 00000000..67153fa9 --- /dev/null +++ b/test/traefik/run.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) +DIR="$(cd "$(dirname "$0")" && pwd)" + +"$DIR"/install.sh + +"$REPO_ROOT"/test/workloads/init.sh +"$DIR"/test-canary.sh diff --git a/test/traefik/test-canary.sh b/test/traefik/test-canary.sh new file mode 100755 index 00000000..8847bda9 --- /dev/null +++ b/test/traefik/test-canary.sh @@ -0,0 +1,146 @@ +#!/usr/bin/env bash + +# This script runs e2e tests for Canary initialization, analysis and promotion +# Prerequisites: Kubernetes Kind, Helm and Traefik ingress controller + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) + +cat <>> Waiting for primary to be ready' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n traefik logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary initialization test passed' + +passed=$(kubectl -n test get svc/podinfo -o jsonpath='{.spec.selector.app}' 2>&1 | { grep podinfo-primary || true; }) +if [ -z "$passed" ]; then + echo -e '\u2716 podinfo selector test failed' + exit 1 +fi +passed=$(kubectl -n test get traefikservice/podinfo -o jsonpath='{.metadata.labels}' 2>&1 | { grep test-label || true; }) +if [ -z "$passed" ]; then + echo -e '\u2716 TraefikService does not have required labels' + exit 1 +fi +passed=$(kubectl -n test get traefikservice/podinfo -o jsonpath='{.metadata.annotations}' 2>&1 | { grep test-annotation || true; }) +if [ -z "$passed" ]; then + echo -e '\u2716 TraefikService does not have required annotations' + exit 1 +fi + +echo '✔ Canary service custom metadata test passed' + +echo '>>> Triggering canary deployment' +kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1 + +echo '>>> Waiting for canary promotion' +retries=60 +count=0 +ok=false +until ${ok}; do + kubectl -n test describe deployment/podinfo-primary | grep '3.1.1' && ok=true || ok=false + sleep 10 + kubectl -n traefik logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n test describe deployment/podinfo + kubectl -n test describe deployment/podinfo-primary + kubectl -n test logs deployment/flagger-loadtester + kubectl -n traefik logs deployment/flagger + kubectl -n traefik get all + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary promotion test passed' From 1fb898ac22bf797b8e14f6182365b7b9a69501a8 Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Sun, 20 Dec 2020 22:22:39 +0200 Subject: [PATCH 12/19] Cleanup e2e tests Signed-off-by: Stefan Prodan --- test/contour/test-init.sh | 24 ------------------------ test/istio/test-init.sh | 23 ----------------------- test/linkerd/test-init.sh | 23 ----------------------- test/nginx/test-init.sh | 24 ------------------------ 4 files changed, 94 deletions(-) delete mode 100755 test/contour/test-init.sh delete mode 100755 test/istio/test-init.sh delete mode 100755 test/linkerd/test-init.sh delete mode 100755 test/nginx/test-init.sh diff --git a/test/contour/test-init.sh b/test/contour/test-init.sh deleted file mode 100755 index 14e3bce7..00000000 --- a/test/contour/test-init.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash - -# This script setups the scenarios for tests by creating a -# Kubernetes namespace, installing the load tester -# and a test workload (podinfo) - -set -o errexit - -REPO_ROOT=$(git rev-parse --show-toplevel) - -echo '>>> Delete test namespace' -kubectl delete namespace test --ignore-not-found=true --wait=true - -echo '>>> Creating test namespace' -kubectl create namespace test -kubectl label namespace test istio-injection=enabled -kubectl annotate namespace test linkerd.io/inject=enabled - -echo '>>> Installing the load tester' -kubectl apply -k ${REPO_ROOT}/kustomize/tester -kubectl -n test rollout status deployment/flagger-loadtester - -echo '>>> Deploy podinfo' -kubectl apply -f ${REPO_ROOT}/test/e2e-workload.yaml diff --git a/test/istio/test-init.sh b/test/istio/test-init.sh deleted file mode 100755 index 4f0ab745..00000000 --- a/test/istio/test-init.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash - -# This script setups the scenarios for istio tests by creating a Kubernetes namespace, installing the load tester and a test workload (podinfo) -# Prerequisites: Kubernetes Kind and Istio - -set -o errexit - -REPO_ROOT=$(git rev-parse --show-toplevel) - -echo '>>> Delete test namespace' -kubectl delete namespace test --ignore-not-found=true --wait=true - -echo '>>> Creating test namespace' -kubectl create namespace test -kubectl label namespace test istio-injection=enabled -kubectl annotate namespace test linkerd.io/inject=enabled - -echo '>>> Installing the load tester' -kubectl apply -k ${REPO_ROOT}/kustomize/tester -kubectl -n test rollout status deployment/flagger-loadtester - -echo '>>> Deploy podinfo' -kubectl apply -f ${REPO_ROOT}/test/e2e-workload.yaml diff --git a/test/linkerd/test-init.sh b/test/linkerd/test-init.sh deleted file mode 100755 index 4f0ab745..00000000 --- a/test/linkerd/test-init.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash - -# This script setups the scenarios for istio tests by creating a Kubernetes namespace, installing the load tester and a test workload (podinfo) -# Prerequisites: Kubernetes Kind and Istio - -set -o errexit - -REPO_ROOT=$(git rev-parse --show-toplevel) - -echo '>>> Delete test namespace' -kubectl delete namespace test --ignore-not-found=true --wait=true - -echo '>>> Creating test namespace' -kubectl create namespace test -kubectl label namespace test istio-injection=enabled -kubectl annotate namespace test linkerd.io/inject=enabled - -echo '>>> Installing the load tester' -kubectl apply -k ${REPO_ROOT}/kustomize/tester -kubectl -n test rollout status deployment/flagger-loadtester - -echo '>>> Deploy podinfo' -kubectl apply -f ${REPO_ROOT}/test/e2e-workload.yaml diff --git a/test/nginx/test-init.sh b/test/nginx/test-init.sh deleted file mode 100755 index 14e3bce7..00000000 --- a/test/nginx/test-init.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash - -# This script setups the scenarios for tests by creating a -# Kubernetes namespace, installing the load tester -# and a test workload (podinfo) - -set -o errexit - -REPO_ROOT=$(git rev-parse --show-toplevel) - -echo '>>> Delete test namespace' -kubectl delete namespace test --ignore-not-found=true --wait=true - -echo '>>> Creating test namespace' -kubectl create namespace test -kubectl label namespace test istio-injection=enabled -kubectl annotate namespace test linkerd.io/inject=enabled - -echo '>>> Installing the load tester' -kubectl apply -k ${REPO_ROOT}/kustomize/tester -kubectl -n test rollout status deployment/flagger-loadtester - -echo '>>> Deploy podinfo' -kubectl apply -f ${REPO_ROOT}/test/e2e-workload.yaml From 5afc800b11835a4f6a3bac77900f767b61140fb9 Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Sun, 20 Dec 2020 22:35:17 +0200 Subject: [PATCH 13/19] Cleanup Istio e2e tests Signed-off-by: Stefan Prodan --- test/istio/test-canary.sh | 33 --------------------------------- 1 file changed, 33 deletions(-) diff --git a/test/istio/test-canary.sh b/test/istio/test-canary.sh index 2954626e..ecd51201 100755 --- a/test/istio/test-canary.sh +++ b/test/istio/test-canary.sh @@ -85,34 +85,6 @@ spec: logCmdOutput: "true" EOF -cat <>> Waiting for primary to be ready' retries=50 count=0 @@ -146,11 +118,6 @@ if [ -z "$passed" ]; then echo -e '\u2716 podinfo selector test failed' exit 1 fi -passed=$(kubectl -n test get svc/podinfo-service-canary -o jsonpath='{.spec.selector.app}' 2>&1 | { grep podinfo || true; }) -if [ -z "$passed" ]; then - echo -e '\u2716 podinfo-service selector test failed' - exit 1 -fi echo '✔ Canary service custom metadata test passed' From 2ea13cec8876904921955e61c3fa2bc38e3e15b6 Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Sun, 20 Dec 2020 22:35:28 +0200 Subject: [PATCH 14/19] Add Gloo e2e tests Signed-off-by: Stefan Prodan --- .github/workflows/e2e.yaml | 2 + test/gloo/install.sh | 31 ++++++++++ test/gloo/run.sh | 11 ++++ test/gloo/test-canary.sh | 113 +++++++++++++++++++++++++++++++++++++ test/istio/test-canary.sh | 1 - test/traefik/install.sh | 3 +- 6 files changed, 159 insertions(+), 2 deletions(-) create mode 100755 test/gloo/install.sh create mode 100755 test/gloo/run.sh create mode 100755 test/gloo/test-canary.sh diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index db1b10fd..851a57d7 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -17,6 +17,8 @@ jobs: - linkerd - contour - nginx + - traefik + - gloo steps: - name: Checkout uses: actions/checkout@v2 diff --git a/test/gloo/install.sh b/test/gloo/install.sh new file mode 100755 index 00000000..b68a6079 --- /dev/null +++ b/test/gloo/install.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +set -o errexit + +GLOO_VER="1.3.28" +REPO_ROOT=$(git rev-parse --show-toplevel) + +mkdir -p ${REPO_ROOT}/bin + +echo '>>> Installing Gloo' +kubectl create ns gloo-system +helm repo add gloo https://storage.googleapis.com/solo-public-helm +helm upgrade -i gloo gloo/gloo --version ${GLOO_VER} \ +--namespace gloo-system \ +--set discovery.enabled=true + +kubectl -n gloo-system rollout status deployment/gloo +kubectl -n gloo-system rollout status deployment/gateway +kubectl -n gloo-system get all + +echo '>>> Installing Flagger' +helm upgrade -i flagger ${REPO_ROOT}/charts/flagger \ +--set crd.create=false \ +--namespace gloo-system \ +--set prometheus.install=true \ +--set meshProvider=gloo + +kubectl -n gloo-system set image deployment/flagger flagger=test/flagger:latest + +kubectl -n gloo-system rollout status deployment/flagger +kubectl -n gloo-system rollout status deployment/flagger-prometheus diff --git a/test/gloo/run.sh b/test/gloo/run.sh new file mode 100755 index 00000000..67153fa9 --- /dev/null +++ b/test/gloo/run.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) +DIR="$(cd "$(dirname "$0")" && pwd)" + +"$DIR"/install.sh + +"$REPO_ROOT"/test/workloads/init.sh +"$DIR"/test-canary.sh diff --git a/test/gloo/test-canary.sh b/test/gloo/test-canary.sh new file mode 100755 index 00000000..72ea019a --- /dev/null +++ b/test/gloo/test-canary.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash + +# This script runs e2e tests for Canary initialization, analysis and promotion + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) + +cat <>> Waiting for primary to be ready' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n gloo-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary initialization test passed' + +echo '>>> Triggering canary deployment' +kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1 + +echo '>>> Waiting for canary promotion' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test describe deployment/podinfo-primary | grep '3.1.1' && ok=true || ok=false + sleep 10 + kubectl -n gloo-system logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n test describe deployment/podinfo + kubectl -n test describe deployment/podinfo-primary + kubectl -n test logs deployment/flagger-loadtester + kubectl -n gloo-system logs deployment/flagger + kubectl -n gloo-system get all + kubectl -n gloo-system get virtualservice podinfo -oyaml + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary promotion test passed' diff --git a/test/istio/test-canary.sh b/test/istio/test-canary.sh index ecd51201..3c84296b 100755 --- a/test/istio/test-canary.sh +++ b/test/istio/test-canary.sh @@ -91,7 +91,6 @@ count=0 ok=false until ${ok}; do kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false - kubectl -n test get canary/podinfo-service | grep 'Initialized' && ok=true || ok=false sleep 5 count=$(($count + 1)) if [[ ${count} -eq ${retries} ]]; then diff --git a/test/traefik/install.sh b/test/traefik/install.sh index e238edd5..f3d1f677 100755 --- a/test/traefik/install.sh +++ b/test/traefik/install.sh @@ -12,7 +12,7 @@ kubectl create ns traefik echo '>>> Installing Traefik' helm repo add traefik https://helm.traefik.io/traefik -cat <>> Installing Flagger' From ccd64a3df94b8ec93012595a62adb52ef3eaa43e Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Mon, 21 Dec 2020 00:38:45 +0200 Subject: [PATCH 15/19] Add Kubernetes B/G e2e tests Signed-off-by: Stefan Prodan --- .github/workflows/e2e.yaml | 1 + test/kubernetes/install.sh | 15 +++ test/kubernetes/run.sh | 12 ++ test/kubernetes/test-daemonset.sh | 189 +++++++++++++++++++++++++++++ test/kubernetes/test-deployment.sh | 100 +++++++++++++++ test/traefik/install.sh | 2 +- 6 files changed, 318 insertions(+), 1 deletion(-) create mode 100755 test/kubernetes/install.sh create mode 100755 test/kubernetes/run.sh create mode 100755 test/kubernetes/test-daemonset.sh create mode 100755 test/kubernetes/test-deployment.sh diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 851a57d7..a2545c17 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -19,6 +19,7 @@ jobs: - nginx - traefik - gloo + - kubernetes steps: - name: Checkout uses: actions/checkout@v2 diff --git a/test/kubernetes/install.sh b/test/kubernetes/install.sh new file mode 100755 index 00000000..01862271 --- /dev/null +++ b/test/kubernetes/install.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) + +mkdir -p ${REPO_ROOT}/bin + +echo '>>> Installing Flagger' +kubectl apply -k ${REPO_ROOT}/kustomize/kubernetes + +kubectl -n flagger-system set image deployment/flagger flagger=test/flagger:latest + +kubectl -n flagger-system rollout status deployment/flagger +kubectl -n flagger-system rollout status deployment/flagger-prometheus \ No newline at end of file diff --git a/test/kubernetes/run.sh b/test/kubernetes/run.sh new file mode 100755 index 00000000..03fad0fc --- /dev/null +++ b/test/kubernetes/run.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) +DIR="$(cd "$(dirname "$0")" && pwd)" + +"$DIR"/install.sh + +"$REPO_ROOT"/test/workloads/init.sh +"$DIR"/test-deployment.sh +"$DIR"/test-daemonset.sh diff --git a/test/kubernetes/test-daemonset.sh b/test/kubernetes/test-daemonset.sh new file mode 100755 index 00000000..cd94f38a --- /dev/null +++ b/test/kubernetes/test-daemonset.sh @@ -0,0 +1,189 @@ +#!/usr/bin/env bash + +# This script runs e2e tests for Blue/Green initialization, analysis and promotion +# Prerequisites: Kubernetes Kind, Kustomize + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) + +cat <>> Waiting for primary to be ready' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo-ds | grep 'Initialized' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n flagger-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary initialization test passed' + +echo '>>> Triggering canary daemonset' +kubectl -n test set image daemonset/podinfo-ds podinfod=stefanprodan/podinfo:3.1.1 + +echo '>>> Waiting for canary promotion' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test describe daemonset/podinfo-ds-primary | grep '3.1.1' && ok=true || ok=false + sleep 10 + kubectl -n flagger-system logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n test describe daemonset/podinfo-ds + kubectl -n test describe daemonset/podinfo-primary-ds + kubectl -n flagger-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary promotion test passed' + + +cat <>> Waiting for finalizers to be present' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl get canary podinfo-ds -n test -o jsonpath='{.metadata.finalizers}' | grep "finalizer.flagger.app" && ok=true || ok=false + sleep 10 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n test describe canary/podinfo + echo "No more retries left" + exit 1 + fi +done + +kubectl delete canary podinfo -n test + +echo '>>> Waiting for primary to revert' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl get daemonset podinfo-ds -n test -o jsonpath='{.status.numberReady}' | grep 1 && ok=true || ok=false + sleep 10 + kubectl -n flagger-system logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n test describe-ds canary/podinfo + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary finalize passed' + +kubectl -n flagger-system logs deployment/flagger diff --git a/test/kubernetes/test-deployment.sh b/test/kubernetes/test-deployment.sh new file mode 100755 index 00000000..42cb7a2d --- /dev/null +++ b/test/kubernetes/test-deployment.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash + +# This script runs e2e tests for Blue/Green initialization, analysis and promotion +# Prerequisites: Kubernetes Kind, Kustomize + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) + +cat <>> Waiting for primary to be ready' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n flagger-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary initialization test passed' + +echo '>>> Triggering canary deployment' +kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1 + +echo '>>> Waiting for canary promotion' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test describe deployment/podinfo-primary | grep '3.1.1' && ok=true || ok=false + sleep 10 + kubectl -n flagger-system logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n test describe deployment/podinfo + kubectl -n test describe deployment/podinfo-primary + kubectl -n flagger-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary promotion test passed' + +kubectl -n flagger-system logs deployment/flagger diff --git a/test/traefik/install.sh b/test/traefik/install.sh index f3d1f677..62ce237c 100755 --- a/test/traefik/install.sh +++ b/test/traefik/install.sh @@ -34,4 +34,4 @@ helm upgrade -i flagger ${REPO_ROOT}/charts/flagger \ --set image.tag=latest \ kubectl -n traefik rollout status deployment/flagger -kubectl -n projectcontour rollout status deployment/flagger-prometheus +kubectl -n traefik rollout status deployment/flagger-prometheus From c8a472c01bd41debb49176e7dee6642381a39eea Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Mon, 21 Dec 2020 10:05:04 +0200 Subject: [PATCH 16/19] Add Skipper e2e tests Signed-off-by: Stefan Prodan --- .github/workflows/e2e.yaml | 1 + test/skipper/install.sh | 24 ++++++++ test/skipper/run.sh | 11 ++++ test/skipper/test-canary.sh | 120 ++++++++++++++++++++++++++++++++++++ 4 files changed, 156 insertions(+) create mode 100755 test/skipper/install.sh create mode 100755 test/skipper/run.sh create mode 100755 test/skipper/test-canary.sh diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index a2545c17..52009600 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -19,6 +19,7 @@ jobs: - nginx - traefik - gloo + - skipper - kubernetes steps: - name: Checkout diff --git a/test/skipper/install.sh b/test/skipper/install.sh new file mode 100755 index 00000000..bc5820f8 --- /dev/null +++ b/test/skipper/install.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -o errexit + +KUSTOMIZE_VERSION=3.8.2 +REPO_ROOT=$(git rev-parse --show-toplevel) + +mkdir -p ${REPO_ROOT}/bin + +echo '>>> Installing Kustomize' +cd ${REPO_ROOT}/bin && kustomize_url=https://github.com/kubernetes-sigs/kustomize/releases/download && \ +curl -sL ${kustomize_url}/kustomize%2Fv${KUSTOMIZE_VERSION}/kustomize_v${KUSTOMIZE_VERSION}_linux_amd64.tar.gz | \ +tar xz + +echo '>>> Installing Skipper' +${REPO_ROOT}/bin/kustomize build ${REPO_ROOT}/test/skipper | kubectl apply -f - + +kubectl -n kube-system rollout status deployment/skipper-ingress + +echo '>>> Installing Flagger' +kubectl -n flagger-system set image deployment/flagger flagger=test/flagger:latest + +kubectl -n flagger-system rollout status deployment/flagger +kubectl -n flagger-system rollout status deployment/flagger-prometheus \ No newline at end of file diff --git a/test/skipper/run.sh b/test/skipper/run.sh new file mode 100755 index 00000000..67153fa9 --- /dev/null +++ b/test/skipper/run.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) +DIR="$(cd "$(dirname "$0")" && pwd)" + +"$DIR"/install.sh + +"$REPO_ROOT"/test/workloads/init.sh +"$DIR"/test-canary.sh diff --git a/test/skipper/test-canary.sh b/test/skipper/test-canary.sh new file mode 100755 index 00000000..a72fb60a --- /dev/null +++ b/test/skipper/test-canary.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash + +# This script runs e2e tests for Skipper canary initialization, analysis and promotion + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) + +echo '>>> Creating ingress' +cat <>> Creating canary' +cat <>> Waiting for primary to be ready' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n flagger-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary initialization test passed' + +echo '>>> Triggering canary deployment' +kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1 + +echo '>>> Waiting for canary promotion' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test describe deployment/podinfo-primary | grep '3.1.1' && ok=true || ok=false + sleep 10 + kubectl -n flagger-system logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n test describe deployment/podinfo + kubectl -n test describe deployment/podinfo-primary + kubectl -n flagger-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary promotion test passed' From fda53fbf80a933828d6cfba6ee7211f4fe3cdd6d Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Mon, 21 Dec 2020 11:52:13 +0200 Subject: [PATCH 17/19] Remove CircleCI testing framework Signed-off-by: Stefan Prodan --- .github/workflows/release.yml | 14 +- test/README.md | 91 +--- test/container-build.sh | 11 - test/container-push.sh | 23 - test/e2e-contour-tests.sh | 327 ------------- test/e2e-contour.sh | 28 -- test/e2e-daemonset.yaml | 63 --- test/e2e-gloo-tests.sh | 173 ------- test/e2e-gloo.sh | 32 -- test/e2e-ingress.yaml | 17 - test/e2e-istio-dependencies.sh | 19 - test/e2e-istio-tests-delegate.sh | 149 ------ test/e2e-istio-tests-skip-analysis.sh | 138 ------ test/e2e-istio-tests.sh | 608 ------------------------ test/e2e-istio-values.yaml | 60 --- test/e2e-istio.sh | 29 -- test/e2e-kind.sh | 40 -- test/e2e-kubernetes-cleanup.sh | 6 - test/e2e-kubernetes-tests-daemonset.sh | 199 -------- test/e2e-kubernetes-tests-deployment.sh | 111 ----- test/e2e-kubernetes-tests-svc.sh | 144 ------ test/e2e-kubernetes.sh | 16 - test/e2e-linkerd-steps-tests.sh | 218 --------- test/e2e-linkerd-tests.sh | 259 ---------- test/e2e-linkerd.sh | 25 - test/e2e-nginx-cleanup.sh | 12 - test/e2e-nginx-custom-annotations.sh | 38 -- test/e2e-nginx-tests.sh | 313 ------------ test/e2e-nginx.sh | 34 -- test/e2e-skipper-canary.yaml | 66 --- test/e2e-skipper-cleanup.sh | 14 - test/e2e-skipper-tests.sh | 83 ---- test/e2e-skipper.sh | 22 - test/e2e-traefik-tests.sh | 156 ------ test/e2e-traefik.sh | 37 -- test/e2e-workload-ingress.yaml | 17 - test/e2e-workload-v2.yaml | 68 --- test/e2e-workload.yaml | 137 ------ test/goreleaser.sh | 28 -- test/local/e2e-skipper.sh | 18 - 40 files changed, 16 insertions(+), 3827 deletions(-) delete mode 100755 test/container-build.sh delete mode 100755 test/container-push.sh delete mode 100755 test/e2e-contour-tests.sh delete mode 100755 test/e2e-contour.sh delete mode 100644 test/e2e-daemonset.yaml delete mode 100755 test/e2e-gloo-tests.sh delete mode 100755 test/e2e-gloo.sh delete mode 100644 test/e2e-ingress.yaml delete mode 100755 test/e2e-istio-dependencies.sh delete mode 100755 test/e2e-istio-tests-delegate.sh delete mode 100755 test/e2e-istio-tests-skip-analysis.sh delete mode 100755 test/e2e-istio-tests.sh delete mode 100644 test/e2e-istio-values.yaml delete mode 100755 test/e2e-istio.sh delete mode 100755 test/e2e-kind.sh delete mode 100755 test/e2e-kubernetes-cleanup.sh delete mode 100755 test/e2e-kubernetes-tests-daemonset.sh delete mode 100755 test/e2e-kubernetes-tests-deployment.sh delete mode 100755 test/e2e-kubernetes-tests-svc.sh delete mode 100755 test/e2e-kubernetes.sh delete mode 100644 test/e2e-linkerd-steps-tests.sh delete mode 100755 test/e2e-linkerd-tests.sh delete mode 100755 test/e2e-linkerd.sh delete mode 100755 test/e2e-nginx-cleanup.sh delete mode 100755 test/e2e-nginx-custom-annotations.sh delete mode 100755 test/e2e-nginx-tests.sh delete mode 100755 test/e2e-nginx.sh delete mode 100644 test/e2e-skipper-canary.yaml delete mode 100755 test/e2e-skipper-cleanup.sh delete mode 100755 test/e2e-skipper-tests.sh delete mode 100755 test/e2e-skipper.sh delete mode 100755 test/e2e-traefik-tests.sh delete mode 100755 test/e2e-traefik.sh delete mode 100644 test/e2e-workload-ingress.yaml delete mode 100644 test/e2e-workload-v2.yaml delete mode 100644 test/e2e-workload.yaml delete mode 100755 test/goreleaser.sh delete mode 100755 test/local/e2e-skipper.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 668ecc6c..d5b8f443 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -33,11 +33,6 @@ jobs: registry: ghcr.io username: fluxcdbot password: ${{ secrets.GHCR_TOKEN }} - - name: Login to Docker Hub - uses: docker/login-action@v1 - with: - username: fluxcdbot - password: ${{ secrets.DOCKER_FLUXCD_PASSWORD }} - name: Publish image uses: docker/build-push-action@v2 with: @@ -45,7 +40,7 @@ jobs: builder: ${{ steps.buildx.outputs.name }} context: . file: ./Dockerfile - platforms: linux/amd64,linux/arm/v7,linux/arm64 + platforms: linux/amd64,linux/arm64,linux/arm/v7 tags: | ghcr.io/fluxcd/flagger:${{ steps.prep.outputs.VERSION }} labels: | @@ -58,8 +53,11 @@ jobs: org.opencontainers.image.created=${{ steps.prep.outputs.BUILD_DATE }} - name: Check images run: | - docker buildx imagetools inspect docker.io/fluxcd/flagger:${{ steps.prep.outputs.VERSION }} docker buildx imagetools inspect ghcr.io/fluxcd/flagger:${{ steps.prep.outputs.VERSION }} + - name: Publish Helm charts + uses: stefanprodan/helm-gh-pages@master + with: + token: ${{ secrets.GITHUB_TOKEN }} - name: Create release uses: actions/create-release@latest env: @@ -70,4 +68,4 @@ jobs: draft: false prerelease: false body: | - [CHANGELOG](https://github.com/fluxcd/flagger/blob/main/CHANGELOG.md) + [CHANGELOG](https://github.com/fluxcd/flagger/blob/main/CHANGELOG.md#${{ steps.prep.outputs.VERSION }}) diff --git a/test/README.md b/test/README.md index e0f7167e..80d880ca 100644 --- a/test/README.md +++ b/test/README.md @@ -1,85 +1,14 @@ # Flagger end-to-end testing -The e2e testing infrastructure is powered by CircleCI and [Kubernetes Kind](https://github.com/kubernetes-sigs/kind). +The e2e testing infrastructure is powered by GitHub Actions and [Kubernetes Kind](https://github.com/kubernetes-sigs/kind). -### CircleCI e2e Istio workflow +### e2e workflow -* install latest stable kubectl [e2e-kind.sh](e2e-kind.sh) -* install Kubernetes Kind [e2e-kind.sh](e2e-kind.sh) -* create local Kubernetes cluster with kind [e2e-kind.sh](e2e-kind.sh) -* install latest stable Helm CLI [e2e-istio.sh](e2e-istio.sh) -* deploy Tiller on the local cluster [e2e-istio.sh](e2e-istio.sh) -* install Istio CRDs with Helm [e2e-istio.sh](e2e-istio.sh) -* install Istio control plane and Prometheus with Helm [e2e-istio.sh](e2e-istio.sh) -* load Flagger image onto the local cluster [e2e-istio.sh](e2e-istio.sh) -* deploy Flagger in the istio-system namespace [e2e-istio.sh](e2e-istio.sh) -* create a test namespace with Istio injection enabled [e2e-tests.sh](e2e-tests.sh) -* deploy the load tester in the test namespace [e2e-tests.sh](e2e-tests.sh) -* deploy a demo workload (podinfo) in the test namespace [e2e-tests.sh](e2e-tests.sh) -* test the canary initialization [e2e-tests.sh](e2e-tests.sh) -* test the canary analysis and promotion using weighted traffic and the load testing webhook [e2e-tests.sh](e2e-tests.sh) -* test the A/B testing analysis and promotion using cookies filters and pre/post rollout webhooks [e2e-tests.sh](e2e-tests.sh) - -### CircleCI e2e Linkerd workflow - -* install latest stable kubectl [e2e-kind.sh](e2e-kind.sh) -* install Kubernetes Kind [e2e-kind.sh](e2e-kind.sh) -* create local Kubernetes cluster with kind [e2e-kind.sh](e2e-kind.sh) -* install Linkerd [e2e-linkerd.sh](e2e-linkerd.sh) -* load Flagger image onto the local cluster [e2e-linkerd.sh](e2e-linkerd.sh) -* deploy Flagger in the linkerd namespace with Kustomize [e2e-linkerd.sh](e2e-linkerd.sh) -* create a test namespace with Linkerd injection enabled [e2e-linkerd-tests.sh](e2e-linkerd-tests.sh) -* deploy the load tester in the test namespace [e2e-linkerd-tests.sh](e2e-linkerd-tests.sh) -* deploy a demo workload (podinfo) in the test namespace [e2e-linkerd-tests.sh](e2e-linkerd-tests.sh) -* test the canary initialization with port discovery enabled and service target port [e2e-linkerd-tests.sh](e2e-linkerd-tests.sh) -* test the canary analysis and promotion using gRPC acceptance tests and HTTP load tests [e2e-linkerd-tests.sh](e2e-linkerd-tests.sh) -* test the canary rollback on HTTP 500 errors [e2e-linkerd-tests.sh](e2e-linkerd-tests.sh) - -### CircleCI e2e NGINX ingress workflow - -* install latest stable kubectl [e2e-kind.sh](e2e-kind.sh) -* install Kubernetes Kind [e2e-kind.sh](e2e-kind.sh) -* create local Kubernetes cluster with kind [e2e-kind.sh](e2e-kind.sh) -* install latest stable Helm CLI [e2e-nginx.sh](e2e-nginx.sh) -* deploy Tiller on the local cluster [e2e-nginx.sh](e2e-nginx.sh) -* install NGINX ingress with Helm [e2e-nginx.sh](e2e-nginx.sh) -* load Flagger image onto the local cluster [e2e-nginx.sh](e2e-nginx.sh) -* install Flagger and Prometheus in the ingress-nginx namespace [e2e-nginx.sh](e2e-nginx.sh) -* create a test namespace [e2e-nginx-tests.sh](e2e-nginx-tests.sh) -* deploy the load tester in the test namespace [e2e-nginx-tests.sh](e2e-nginx-tests.sh) -* deploy the demo workload (podinfo) and ingress in the test namespace [e2e-nginx-tests.sh](e2e-nginx-tests.sh) -* test the canary initialization [e2e-nginx-tests.sh](e2e-nginx-tests.sh) -* test the canary analysis and promotion using weighted traffic and the load testing webhook [e2e-nginx-tests.sh](e2e-nginx-tests.sh) -* test the A/B testing analysis and promotion using header filters and pre/post rollout webhooks [e2e-nginx-tests.sh](e2e-nginx-tests.sh) -* cleanup test environment [e2e-nginx-cleanup.sh](e2e-nginx-cleanup.sh) -* install NGINX Ingress and Flagger with custom ingress annotations prefix [e2e-nginx-custom-annotations.sh](e2e-nginx-custom-annotations.sh) -* repeat the canary and A/B testing workflow [e2e-nginx-tests.sh](e2e-nginx-tests.sh) - -### CircleCI e2e Skipper ingress workflow - -* install latest stable kubectl [e2e-kind.sh](e2e-kind.sh) -* install Kubernetes Kind [e2e-kind.sh](e2e-kind.sh) -* create local Kubernetes cluster with kind [e2e-kind.sh](e2e-kind.sh) -* install Skipper ingress with Kustomize [e2e-skipper.sh](e2e-skipper.sh) -* load Flagger image onto the local cluster [e2e-skipper.sh](e2e-skipper.sh) -* install Flagger and Prometheus in the flagger-system namespace [e2e-skipper.sh](e2e-skipper.sh) -* create a test namespace [e2e-skipper-tests.sh](e2e-skipper-tests.sh) -* deploy the load tester in the test namespace [e2e-skipper-tests.sh](e2e-skipper-tests.sh) -* deploy the demo workload (podinfo) and ingress in the test namespace [e2e-skipper-tests.sh](e2e-skipper-tests.sh) -* test the canary initialization [e2e-skipper-tests.sh](e2e-skipper-tests.sh) -* test the canary analysis and promotion using weighted traffic and the load testing webhook [e2e-skipper-tests.sh]e2e-skipper-tests.sh) -* cleanup test environment [e2e-skipper-cleanup.sh](e2e-skipper-cleanup.sh) - -### CircleCI e2e Traefik workflow - -* install latest stable kubectl [e2e-kind.sh](e2e-kind.sh) -* install Kubernetes Kind [e2e-kind.sh](e2e-kind.sh) -* create local Kubernetes cluster with kind [e2e-kind.sh](e2e-kind.sh) -* install Traeik with Helm [e2e-traefik.sh](e2e-traefik.sh) -* load Flagger image onto the local cluster [e2e-traefik.sh](e2e-traefik.sh) -* install Flagger and Prometheus in the traefik namespace [e2e-traefik.sh](e2e-traefik.sh) -* create a test namespace [e2e-traefik-tests.sh](e2e-traefik-tests.sh) -* deploy the load tester in the test namespace [e2e-traefik-tests.sh](e2e-traefik-tests.sh) -* deploy the demo workload (podinfo) and ingress in the test namespace [e2e-traefik-tests.sh](e2e-traefik-tests.sh) -* test the canary initialization [e2e-traefik-tests.sh](e2e-traefik-tests.sh) -* test the canary analysis and promotion using weighted traffic and the load testing webhook [e2e-traefik-tests.sh]e2e-traefik-tests.sh) +* create local Kubernetes cluster with KinD +* build Flagger container and load the image in KinD +* install the service mesh or ingress provider +* deploy Flagger +* create test namespace, workloads and load tester +* test the canary initialization (port discovery and metadata) +* test the canary release (progressive traffic shifting, headers routing, mirroring, analysis, promotion, rollback) +* test webhooks (conformance, load testing, pre/post rollout) diff --git a/test/container-build.sh b/test/container-build.sh deleted file mode 100755 index 205d27e7..00000000 --- a/test/container-build.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit - -REPO_ROOT=$(git rev-parse --show-toplevel) - -mkdir -p ${REPO_ROOT}/bin -cp /tmp/bin/flagger ${REPO_ROOT}/bin && chmod +x ${REPO_ROOT}/bin/flagger -cp /tmp/bin/loadtester ${REPO_ROOT}/bin && chmod +x ${REPO_ROOT}/bin/loadtester - -docker build -t test/flagger:latest . -f ${REPO_ROOT}/Dockerfile diff --git a/test/container-push.sh b/test/container-push.sh deleted file mode 100755 index 66dec947..00000000 --- a/test/container-push.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit - -push () { - echo $DOCKER_PASS | docker login -u=$DOCKER_USER --password-stdin - - if [[ -z "$CIRCLE_TAG" ]]; then - BRANCH_COMMIT=${CIRCLE_BRANCH}-$(echo ${CIRCLE_SHA1} | head -c7); - docker tag test/flagger:latest weaveworks/flagger:${BRANCH_COMMIT}; - docker push weaveworks/flagger:${BRANCH_COMMIT}; - else - VER=${CIRCLE_TAG:1} - docker tag test/flagger:latest weaveworks/flagger:${VER}; - docker push weaveworks/flagger:${VER}; - fi -} - -if [[ -z "$DOCKER_PASS" ]]; then - echo "No Docker Hub credentials, skipping image push"; -else - push -fi diff --git a/test/e2e-contour-tests.sh b/test/e2e-contour-tests.sh deleted file mode 100755 index 59877228..00000000 --- a/test/e2e-contour-tests.sh +++ /dev/null @@ -1,327 +0,0 @@ -#!/usr/bin/env bash - -# This script runs e2e tests for Canary initialization, analysis and promotion -# Prerequisites: Kubernetes Kind and Contour ingress controller - -set -o errexit - -REPO_ROOT=$(git rev-parse --show-toplevel) - -echo '>>> Creating test namespace' -kubectl create namespace test - -echo '>>> Installing load tester' -kubectl apply -k ${REPO_ROOT}/kustomize/tester -kubectl -n test rollout status deployment/flagger-loadtester - -echo '>>> Initialising canary' -kubectl apply -f ${REPO_ROOT}/test/e2e-workload.yaml - -cat <>> Waiting for primary to be ready' -retries=50 -count=0 -ok=false -until ${ok}; do - kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false - sleep 5 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n projectcontour logs deployment/flagger - echo "No more retries left" - exit 1 - fi -done - -kubectl -n test get httpproxy podinfo -oyaml | grep 'projectcontour.io/ingress.class: contour' - -echo '✔ Canary initialization test passed' - -passed=$(kubectl -n test get svc/podinfo -o jsonpath='{.spec.selector.app}' 2>&1 | { grep podinfo-primary || true; }) -if [ -z "$passed" ]; then - echo -e '\u2716 podinfo selector test failed' - exit 1 -fi -passed=$(kubectl -n test get svc/podinfo-service-canary -o jsonpath='{.spec.selector.app}' 2>&1 | { grep podinfo || true; }) -if [ -z "$passed" ]; then - echo -e '\u2716 podinfo-service selector test failed' - exit 1 -fi - -echo '✔ Canary service custom metadata test passed' - -echo '>>> Triggering canary deployment' -kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1 - -echo '>>> Waiting for canary promotion' -retries=50 -count=0 -ok=false -until ${ok}; do - kubectl -n test describe deployment/podinfo-primary | grep '3.1.1' && ok=true || ok=false - sleep 10 - kubectl -n projectcontour logs deployment/flagger --tail 1 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n test describe deployment/podinfo - kubectl -n test describe deployment/podinfo-primary - kubectl -n test logs deployment/flagger-loadtester - kubectl -n projectcontour logs deployment/flagger - kubectl -n projectcontour get all - kubectl -n test get httpproxy podinfo -oyaml - echo "No more retries left" - exit 1 - fi -done - -echo '>>> Waiting for canary finalization' -retries=50 -count=0 -ok=false -until ${ok}; do - kubectl -n test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false - sleep 5 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n projectcontour logs deployment/flagger - echo "No more retries left" - exit 1 - fi -done - -echo '✔ Canary promotion test passed' - - -cat <>> Triggering canary deployment' -kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.2 - -echo '>>> Waiting for canary rollback' -retries=50 -count=0 -ok=false -until ${ok}; do - kubectl -n test get canary/podinfo | grep 'Failed' && ok=true || ok=false - sleep 10 - kubectl -n projectcontour logs deployment/flagger --tail 1 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n projectcontour logs deployment/flagger - echo "No more retries left" - exit 1 - fi -done - -echo '✔ Canary rollback test passed' - -cat <>> Triggering A/B testing' -kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.3 - -echo '>>> Waiting for A/B testing promotion' -retries=50 -count=0 -ok=false -until ${ok}; do - kubectl -n test describe deployment/podinfo-primary | grep '3.1.3' && ok=true || ok=false - sleep 10 - kubectl -n projectcontour logs deployment/flagger --tail 1 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n projectcontour logs deployment/flagger - echo "No more retries left" - exit 1 - fi -done - -echo '✔ A/B testing promotion test passed' - -kubectl -n projectcontour logs deployment/flagger - -echo '✔ All tests passed' \ No newline at end of file diff --git a/test/e2e-contour.sh b/test/e2e-contour.sh deleted file mode 100755 index f32a1634..00000000 --- a/test/e2e-contour.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit - -REPO_ROOT=$(git rev-parse --show-toplevel) - -CONTOUR_VER="v1.7.0" - -echo '>>> Installing Contour' -kubectl apply -f https://projectcontour.io/quickstart/${CONTOUR_VER}/contour.yaml - -kubectl -n projectcontour rollout status deployment/contour -kubectl -n projectcontour get all - -echo '>>> Installing Flagger' -kind load docker-image test/flagger:latest - -echo '>>> Installing Flagger' -helm upgrade -i flagger ${REPO_ROOT}/charts/flagger \ ---namespace projectcontour \ ---set prometheus.install=true \ ---set meshProvider=contour \ ---set ingressClass=contour - -kubectl -n projectcontour set image deployment/flagger flagger=test/flagger:latest - -kubectl -n projectcontour rollout status deployment/flagger -kubectl -n projectcontour rollout status deployment/flagger-prometheus diff --git a/test/e2e-daemonset.yaml b/test/e2e-daemonset.yaml deleted file mode 100644 index 40cc48b6..00000000 --- a/test/e2e-daemonset.yaml +++ /dev/null @@ -1,63 +0,0 @@ -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: podinfo - namespace: test - labels: - app: podinfo -spec: - minReadySeconds: 5 - revisionHistoryLimit: 5 - selector: - matchLabels: - app: podinfo - template: - metadata: - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "9797" - labels: - app: podinfo - spec: - containers: - - name: podinfod - image: stefanprodan/podinfo:3.1.0 - imagePullPolicy: IfNotPresent - ports: - - name: http - containerPort: 9898 - protocol: TCP - - name: http-metrics - containerPort: 9797 - protocol: TCP - - name: grpc - containerPort: 9999 - protocol: TCP - command: - - ./podinfo - - --port=9898 - - --port-metrics=9797 - - --grpc-port=9999 - - --grpc-service-name=podinfo - - --level=info - - --random-delay=false - - --random-error=false - livenessProbe: - httpGet: - port: 9898 - path: /healthz - initialDelaySeconds: 5 - timeoutSeconds: 5 - readinessProbe: - httpGet: - port: 9898 - path: /readyz - initialDelaySeconds: 5 - timeoutSeconds: 5 - resources: - limits: - cpu: 1000m - memory: 128Mi - requests: - cpu: 1m - memory: 16Mi diff --git a/test/e2e-gloo-tests.sh b/test/e2e-gloo-tests.sh deleted file mode 100755 index a52b7221..00000000 --- a/test/e2e-gloo-tests.sh +++ /dev/null @@ -1,173 +0,0 @@ -#!/usr/bin/env bash - -# This script runs e2e tests for Canary initialization, analysis and promotion -# Prerequisites: Kubernetes Kind, Helm and NGINX ingress controller - -set -o errexit - -REPO_ROOT=$(git rev-parse --show-toplevel) - -echo '>>> Creating test namespace' -kubectl create namespace test - -echo '>>> Installing load tester' -kubectl apply -k ${REPO_ROOT}/kustomize/tester -kubectl -n test rollout status deployment/flagger-loadtester - -echo '>>> Initialising canary' -kubectl apply -f ${REPO_ROOT}/test/e2e-workload.yaml - -cat <>> Waiting for primary to be ready' -retries=50 -count=0 -ok=false -until ${ok}; do - kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false - sleep 5 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n gloo-system logs deployment/flagger - echo "No more retries left" - exit 1 - fi -done - -echo '✔ Canary initialization test passed' - -passed=$(kubectl -n test get svc/podinfo -o jsonpath='{.spec.selector.app}' 2>&1 | { grep podinfo-primary || true; }) -if [ -z "$passed" ]; then - echo -e '\u2716 podinfo selector test failed' - exit 1 -fi -passed=$(kubectl -n test get svc/podinfo-service-canary -o jsonpath='{.spec.selector.app}' 2>&1 | { grep podinfo || true; }) -if [ -z "$passed" ]; then - echo -e '\u2716 podinfo-service selector test failed' - exit 1 -fi - -echo '✔ Canary service custom metadata test passed' - -echo '>>> Triggering canary deployment' -kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1 - -echo '>>> Waiting for canary promotion' -retries=50 -count=0 -ok=false -until ${ok}; do - kubectl -n test describe deployment/podinfo-primary | grep '3.1.1' && ok=true || ok=false - sleep 10 - kubectl -n gloo-system logs deployment/flagger --tail 1 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n test describe deployment/podinfo - kubectl -n test describe deployment/podinfo-primary - kubectl -n test logs deployment/flagger-loadtester - kubectl -n gloo-system logs deployment/flagger - kubectl -n gloo-system get all - kubectl -n gloo-system get virtualservice podinfo -oyaml - echo "No more retries left" - exit 1 - fi -done - -echo '✔ Canary promotion test passed' diff --git a/test/e2e-gloo.sh b/test/e2e-gloo.sh deleted file mode 100755 index 0f95ce0a..00000000 --- a/test/e2e-gloo.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit - -GLOO_VER="1.3.28" -REPO_ROOT=$(git rev-parse --show-toplevel) - -echo '>>> Installing Gloo' -helm repo add gloo https://storage.googleapis.com/solo-public-helm -kubectl create ns gloo-system -helm upgrade -i gloo gloo/gloo --version ${GLOO_VER} \ ---namespace gloo-system \ ---set discovery.enabled=true - -kubectl -n gloo-system rollout status deployment/gloo -kubectl -n gloo-system rollout status deployment/gateway -kubectl -n gloo-system get all - -echo '>>> Installing Flagger' -kind load docker-image test/flagger:latest - -echo '>>> Installing Flagger' -helm upgrade -i flagger ${REPO_ROOT}/charts/flagger \ ---set crd.create=false \ ---namespace gloo-system \ ---set prometheus.install=true \ ---set meshProvider=gloo - -kubectl -n gloo-system set image deployment/flagger flagger=test/flagger:latest - -kubectl -n gloo-system rollout status deployment/flagger -kubectl -n gloo-system rollout status deployment/flagger-prometheus \ No newline at end of file diff --git a/test/e2e-ingress.yaml b/test/e2e-ingress.yaml deleted file mode 100644 index 15eefcfa..00000000 --- a/test/e2e-ingress.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: networking.k8s.io/v1beta1 -kind: Ingress -metadata: - name: podinfo - namespace: test - labels: - app: podinfo - annotations: - kubernetes.io/ingress.class: "nginx" -spec: - rules: - - host: app.example.com - http: - paths: - - backend: - serviceName: podinfo - servicePort: 80 diff --git a/test/e2e-istio-dependencies.sh b/test/e2e-istio-dependencies.sh deleted file mode 100755 index 7bfe93cb..00000000 --- a/test/e2e-istio-dependencies.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash - -# This script setups the scenarios for istio tests by creating a Kubernetes namespace, installing the load tester and a test workload (podinfo) -# Prerequisites: Kubernetes Kind and Istio - -set -o errexit - -REPO_ROOT=$(git rev-parse --show-toplevel) - -echo '>>> Creating test namespace' -kubectl create namespace test -kubectl label namespace test istio-injection=enabled - -echo '>>> Installing the load tester' -kubectl apply -k ${REPO_ROOT}/kustomize/tester -kubectl -n test rollout status deployment/flagger-loadtester - -echo '>>> Deploy podinfo' -kubectl apply -f ${REPO_ROOT}/test/e2e-workload.yaml diff --git a/test/e2e-istio-tests-delegate.sh b/test/e2e-istio-tests-delegate.sh deleted file mode 100755 index 511abd0e..00000000 --- a/test/e2e-istio-tests-delegate.sh +++ /dev/null @@ -1,149 +0,0 @@ -#!/usr/bin/env bash - -# This script runs e2e tests for when the canary delegation is enabled -# Prerequisites: Kubernetes Kind and Istio - -set -o errexit - -echo '>>> Set pilot env to enable virtual service delegate' -kubectl -n istio-system set env deploy istiod PILOT_ENABLE_VIRTUAL_SERVICE_DELEGATE=true -kubectl -n istio-system rollout status deploy istiod - -echo '>>> Initialising Gateway' -cat <>> Initialising root virtual service' -cat <>> Initialising canary for delegate' -cat <>> Waiting for primary to be ready' -retries=50 -count=0 -ok=false -until ${ok}; do - kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false - sleep 5 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n istio-system logs deployment/flagger - echo "No more retries left" - exit 1 - fi -done - -echo '✔ Canary initialization test passed' - -echo '>>> Triggering canary deployment' -kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1 - -echo '>>> Waiting for canary promotion' -retries=50 -count=0 -ok=false -until ${ok}; do - kubectl -n test describe deployment/podinfo-primary | grep '3.1.1' && ok=true || ok=false - sleep 10 - kubectl -n istio-system logs deployment/flagger --tail 1 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n test describe deployment/podinfo - kubectl -n test describe deployment/podinfo-primary - kubectl -n istio-system logs deployment/flagger - echo "No more retries left" - exit 1 - fi -done - -echo '>>> Waiting for canary finalization' -retries=50 -count=0 -ok=false -until ${ok}; do - kubectl -n test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false - sleep 5 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n istio-system logs deployment/flagger - echo "No more retries left" - exit 1 - fi -done - -echo '>>> Set pilot env to disable virtual service delegate' -kubectl -n istio-system set env deploy istiod PILOT_ENABLE_VIRTUAL_SERVICE_DELEGATE=false -kubectl -n istio-system rollout status deploy istiod - -echo '✔ Canary promotion test passed' - -if [[ "$1" = "canary" ]]; then - exit 0 -fi diff --git a/test/e2e-istio-tests-skip-analysis.sh b/test/e2e-istio-tests-skip-analysis.sh deleted file mode 100755 index 621fe285..00000000 --- a/test/e2e-istio-tests-skip-analysis.sh +++ /dev/null @@ -1,138 +0,0 @@ -#!/usr/bin/env bash - -# This script runs e2e tests for when the canary analysis is skipped -# Prerequisites: Kubernetes Kind and Istio - -set -o errexit - -echo '>>> Initialising canary' -cat <>> Waiting for primary to be ready' -retries=50 -count=0 -ok=false -until ${ok}; do - kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false - sleep 5 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n istio-system logs deployment/flagger - echo "No more retries left" - exit 1 - fi -done - -echo '✔ Canary initialization test passed' - -echo '>>> Triggering canary deployment' -kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1 - -echo '>>> Waiting for canary promotion' -retries=50 -count=0 -ok=false -until ${ok}; do - kubectl -n test describe deployment/podinfo-primary | grep '3.1.1' && ok=true || ok=false - sleep 10 - kubectl -n istio-system logs deployment/flagger --tail 1 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n test describe deployment/podinfo - kubectl -n test describe deployment/podinfo-primary - kubectl -n istio-system logs deployment/flagger - echo "No more retries left" - exit 1 - fi -done - -echo '>>> Waiting for canary finalization' -retries=50 -count=0 -ok=false -until ${ok}; do - kubectl -n test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false - sleep 5 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n istio-system logs deployment/flagger - echo "No more retries left" - exit 1 - fi -done - -echo '✔ Canary promotion test passed' - -if [[ "$1" = "canary" ]]; then - exit 0 -fi - -echo '>>> Triggering canary deployment with a bad release (non existent docker image)' -kubectl -n test set image deployment/podinfo podinfod=stefanprodan/potato:1.0.0 - -echo '>>> Waiting for canary to fail' -retries=50 -count=0 -ok=false -until ${ok}; do - kubectl get canary/podinfo -n test -o=jsonpath='{.status.phase}' | grep 'Failed' && ok=true || ok=false - sleep 10 - kubectl -n istio-system logs deployment/flagger --tail 1 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n test describe deployment/podinfo - kubectl -n test describe deployment/podinfo-primary - kubectl -n istio-system logs deployment/flagger - echo "No more retries left" - exit 1 - fi -done - -echo '>>> Confirm primary pod is still running and with correct version' -retries=50 -count=0 -ok=false -until ${okImage} && ${okRunning}; do - kubectl get deployment podinfo-primary -n test -o jsonpath='{.spec.replicas}' | grep 1 && okRunning=true || okRunning=false - kubectl -n test describe deployment/podinfo-primary | grep '3.1.3' && okImage=true || okImage=false - sleep 5 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n istio-system logs deployment/flagger - echo "No more retries left" - exit 1 - fi -done - -kubectl -n istio-system logs deployment/flagger - -echo '✔ All tests passed' diff --git a/test/e2e-istio-tests.sh b/test/e2e-istio-tests.sh deleted file mode 100755 index 2954626e..00000000 --- a/test/e2e-istio-tests.sh +++ /dev/null @@ -1,608 +0,0 @@ -#!/usr/bin/env bash - -# This script runs e2e tests for Canary, B/G and A/B initialization, analysis and promotion -# Prerequisites: Kubernetes Kind and Istio - -set -o errexit - -echo '>>> Create latency metric template' -cat <>> Initialising canaries' -cat <>> Waiting for primary to be ready' -retries=50 -count=0 -ok=false -until ${ok}; do - kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false - kubectl -n test get canary/podinfo-service | grep 'Initialized' && ok=true || ok=false - sleep 5 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n istio-system logs deployment/flagger - echo "No more retries left" - exit 1 - fi -done - -echo '✔ Canary initialization test passed' - -passed=$(kubectl -n test get svc/podinfo -oyaml 2>&1 | { grep annotations-test || true; }) -if [ -z "$passed" ]; then - echo -e '\u2716 podinfo annotations test failed' - exit 1 -fi -passed=$(kubectl -n test get svc/podinfo -oyaml 2>&1 | { grep labels-test || true; }) -if [ -z "$passed" ]; then - echo -e '\u2716 podinfo labels test failed' - exit 1 -fi -passed=$(kubectl -n test get svc/podinfo -o jsonpath='{.spec.selector.app}' 2>&1 | { grep podinfo-primary || true; }) -if [ -z "$passed" ]; then - echo -e '\u2716 podinfo selector test failed' - exit 1 -fi -passed=$(kubectl -n test get svc/podinfo-service-canary -o jsonpath='{.spec.selector.app}' 2>&1 | { grep podinfo || true; }) -if [ -z "$passed" ]; then - echo -e '\u2716 podinfo-service selector test failed' - exit 1 -fi - -echo '✔ Canary service custom metadata test passed' - -echo '>>> Triggering canary deployment' -kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1 - -echo '>>> Waiting for canary promotion' -retries=50 -count=0 -ok=false -until ${ok}; do - kubectl -n test describe deployment/podinfo-primary | grep '3.1.1' && ok=true || ok=false - sleep 10 - kubectl -n istio-system logs deployment/flagger --tail 1 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n test describe deployment/podinfo - kubectl -n test describe deployment/podinfo-primary - kubectl -n istio-system logs deployment/flagger - echo "No more retries left" - exit 1 - fi -done - -echo '>>> Waiting for canary finalization' -retries=50 -count=0 -ok=false -until ${ok}; do - kubectl -n test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false - sleep 5 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n istio-system logs deployment/flagger - echo "No more retries left" - exit 1 - fi -done - -echo '✔ Canary promotion test passed' - -if [[ "$1" = "canary" ]]; then - exit 0 -fi - -cat <>> Triggering B/G deployment' -kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.2 - -echo '>>> Waiting for B/G promotion' -retries=50 -count=0 -ok=false -until ${ok}; do - kubectl -n test describe deployment/podinfo-primary | grep '3.1.2' && ok=true || ok=false - sleep 10 - kubectl -n istio-system logs deployment/flagger --tail 1 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n test describe deployment/podinfo - kubectl -n test describe deployment/podinfo-primary - kubectl -n istio-system logs deployment/flagger - echo "No more retries left" - exit 1 - fi -done - -echo '>>> Waiting for B/G finalization' -retries=50 -count=0 -ok=false -until ${ok}; do - kubectl -n test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false - sleep 5 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n istio-system logs deployment/flagger - echo "No more retries left" - exit 1 - fi -done - -echo '✔ B/G promotion test passed' - -cat <>> Triggering A/B testing' -kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.3 - -echo '>>> Waiting for A/B testing promotion' -retries=50 -count=0 -ok=false -until ${ok}; do - kubectl -n test describe deployment/podinfo-primary | grep '3.1.3' && ok=true || ok=false - sleep 10 - kubectl -n istio-system logs deployment/flagger --tail 1 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n test describe deployment/podinfo - kubectl -n test describe deployment/podinfo-primary - kubectl -n istio-system logs deployment/flagger - echo "No more retries left" - exit 1 - fi -done - -echo '✔ A/B testing promotion test passed' - -cat <>> Waiting for finalizers to be present' -retries=50 -count=0 -ok=false -until ${ok}; do - kubectl get canary podinfo -n test -o jsonpath='{.metadata.finalizers}' | grep "finalizer.flagger.app" && ok=true || ok=false - sleep 10 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n test describe canary/podinfo - echo "No more retries left" - exit 1 - fi -done - -kubectl delete canary podinfo -n test - -echo '>>> Waiting for primary to revert' -retries=50 -count=0 -ok=false -until ${ok}; do - kubectl get deployment podinfo -n test -o jsonpath='{.spec.replicas}' | grep 1 && ok=true || ok=false - sleep 10 - kubectl -n istio-system logs deployment/flagger --tail 10 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n test describe canary/podinfo - echo "No more retries left" - exit 1 - fi -done -echo '✔ Delete testing passed' - - - -cat <>> Waiting for canary to initialize' -retries=50 -count=0 -ok=false -until ${ok}; do - kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false - sleep 5 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n istio-system logs deployment/flagger - echo "No more retries left" - exit 1 - fi -done - -kubectl delete canary podinfo -n test - -echo '>>> Waiting for revert' -retries=50 -count=0 -ok=false -until ${ok}; do - kubectl get svc/podinfo vs/podinfo -n test -o jsonpath="{range .items[*]}{.metadata.name}{'\n'}{end}" | wc -l | grep 2 && ok=true || ok=false - sleep 10 - kubectl -n istio-system logs deployment/flagger --tail 10 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n test describe canary/podinfo - kubectl -n test describe svc/podinfo - kubectl -n test describe vs/podinfo - echo "No more retries left" - exit 1 - fi -done -echo '✔ Revert testing passed' - - -kubectl -n istio-system logs deployment/flagger - -echo '✔ All tests passed' diff --git a/test/e2e-istio-values.yaml b/test/e2e-istio-values.yaml deleted file mode 100644 index b234dedf..00000000 --- a/test/e2e-istio-values.yaml +++ /dev/null @@ -1,60 +0,0 @@ -# -# Minimal Istio Configuration required by Flagger -# - -# pilot configuration -pilot: - enabled: true - sidecar: true - resources: - requests: - cpu: 10m - memory: 128Mi - -gateways: - enabled: false - istio-ingressgateway: - autoscaleMax: 1 - -# sidecar-injector webhook configuration -sidecarInjectorWebhook: - enabled: true - -# galley configuration -galley: - enabled: false - -# mixer configuration -mixer: - policy: - enabled: false - telemetry: - enabled: true - replicaCount: 1 - autoscaleEnabled: false - resources: - requests: - cpu: 10m - memory: 128Mi - -# addon prometheus configuration -prometheus: - enabled: true - scrapeInterval: 5s - -# addon jaeger tracing configuration -tracing: - enabled: false - -# Common settings. -global: - proxy: - # Resources for the sidecar. - resources: - requests: - cpu: 10m - memory: 64Mi - limits: - cpu: 1000m - memory: 256Mi - useMCP: false diff --git a/test/e2e-istio.sh b/test/e2e-istio.sh deleted file mode 100755 index 0058f6db..00000000 --- a/test/e2e-istio.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit - -ISTIO_VER="1.8.0" -REPO_ROOT=$(git rev-parse --show-toplevel) - -echo ">>> Downloading Istio ${ISTIO_VER}" -cd ${REPO_ROOT}/bin && \ -curl -L https://istio.io/downloadIstio | ISTIO_VERSION=${ISTIO_VER} sh - - -echo ">>> Installing Istio ${ISTIO_VER}" -${REPO_ROOT}/bin/istio-${ISTIO_VER}/bin/istioctl manifest install --set profile=default --skip-confirmation \ - --set values.pilot.resources.requests.cpu=100m \ - --set values.pilot.resources.requests.memory=100Mi - -kubectl apply -f https://raw.githubusercontent.com/istio/istio/release-1.8/samples/addons/prometheus.yaml -kubectl -n istio-system rollout status deployment/prometheus - -kubectl -n istio-system get all - -echo '>>> Load Flagger image in Kind' -kind load docker-image test/flagger:latest - -echo '>>> Installing Flagger' -kubectl apply -k ${REPO_ROOT}/kustomize/istio - -kubectl -n istio-system set image deployment/flagger flagger=test/flagger:latest -kubectl -n istio-system rollout status deployment/flagger diff --git a/test/e2e-kind.sh b/test/e2e-kind.sh deleted file mode 100755 index bb2ba52d..00000000 --- a/test/e2e-kind.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit - -REPO_ROOT=$(git rev-parse --show-toplevel) -HELM_VERSION=v3.3.0 -KUSTOMIZE_VERSION=3.8.1 -KIND_VERSION=v0.8.1 -KUBE_VERSION=v1.16.9 - -if [[ "$1" ]]; then - KUBE_VERSION=$1 -fi - -echo ">>> Installing kubectl" -curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && \ -chmod +x kubectl && \ -sudo mv kubectl /usr/local/bin/ - -echo ">>> Installing kind" -curl -sSLo kind "https://github.com/kubernetes-sigs/kind/releases/download/$KIND_VERSION/kind-linux-amd64" -chmod +x kind -sudo mv kind /usr/local/bin/kind - -echo ">>> Creating kind cluster" -kind create cluster --wait 5m --image kindest/node:${KUBE_VERSION} - -kubectl get pods --all-namespaces - -cd /tmp - -echo ">>> Installing Helm" -curl -sSL https://get.helm.sh/helm-${HELM_VERSION}-linux-amd64.tar.gz | tar xz && sudo mv linux-amd64/helm /usr/local/bin/ && rm -rf linux-amd64 -helm repo add stable https://kubernetes-charts.storage.googleapis.com/ - -echo ">>> Installing Kustomize" -kustomize_url=https://github.com/kubernetes-sigs/kustomize/releases/download && \ -curl -sL ${kustomize_url}/kustomize%2Fv${KUSTOMIZE_VERSION}/kustomize_v${KUSTOMIZE_VERSION}_linux_amd64.tar.gz | tar xz -chmod +x kustomize -sudo mv kustomize /usr/local/bin/kustomize diff --git a/test/e2e-kubernetes-cleanup.sh b/test/e2e-kubernetes-cleanup.sh deleted file mode 100755 index b289ac72..00000000 --- a/test/e2e-kubernetes-cleanup.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit - -echo '>>> Delete test namespace' -kubectl delete namespace test --ignore-not-found=true --wait=true diff --git a/test/e2e-kubernetes-tests-daemonset.sh b/test/e2e-kubernetes-tests-daemonset.sh deleted file mode 100755 index e5ab93c3..00000000 --- a/test/e2e-kubernetes-tests-daemonset.sh +++ /dev/null @@ -1,199 +0,0 @@ -#!/usr/bin/env bash - -# This script runs e2e tests for Blue/Green initialization, analysis and promotion -# Prerequisites: Kubernetes Kind, Kustomize - -set -o errexit - -REPO_ROOT=$(git rev-parse --show-toplevel) - -echo '>>> Creating test namespace' -kubectl create namespace test - -echo '>>> Installing the load tester' -kubectl apply -k ${REPO_ROOT}/kustomize/tester -kubectl -n test rollout status deployment/flagger-loadtester - -echo '>>> Initialising canary' -kubectl apply -f ${REPO_ROOT}/test/e2e-daemonset.yaml - -cat <>> Waiting for primary to be ready' -retries=50 -count=0 -ok=false -until ${ok}; do - kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false - sleep 5 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n flagger-system logs deployment/flagger - echo "No more retries left" - exit 1 - fi -done - -echo '✔ Canary initialization test passed' - -echo '>>> Triggering canary daemonset' -kubectl -n test set image daemonset/podinfo podinfod=stefanprodan/podinfo:3.1.1 - -echo '>>> Waiting for canary promotion' -retries=50 -count=0 -ok=false -until ${ok}; do - kubectl -n test describe daemonset/podinfo-primary | grep '3.1.1' && ok=true || ok=false - sleep 10 - kubectl -n flagger-system logs deployment/flagger --tail 1 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n test describe daemonset/podinfo - kubectl -n test describe daemonset/podinfo-primary - kubectl -n flagger-system logs deployment/flagger - echo "No more retries left" - exit 1 - fi -done - -echo '✔ Canary promotion test passed' - - -cat <>> Waiting for finalizers to be present' -retries=50 -count=0 -ok=false -until ${ok}; do - kubectl get canary podinfo -n test -o jsonpath='{.metadata.finalizers}' | grep "finalizer.flagger.app" && ok=true || ok=false - sleep 10 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n test describe canary/podinfo - echo "No more retries left" - exit 1 - fi -done - -kubectl delete canary podinfo -n test - -echo '>>> Waiting for primary to revert' -retries=50 -count=0 -ok=false -until ${ok}; do - kubectl get daemonset podinfo -n test -o jsonpath='{.status.numberReady}' | grep 1 && ok=true || ok=false - sleep 10 - kubectl -n flagger-system logs deployment/flagger --tail 1 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n test describe canary/podinfo - echo "No more retries left" - exit 1 - fi -done - -echo '✔ Canary finalize passed' - -kubectl -n flagger-system logs deployment/flagger diff --git a/test/e2e-kubernetes-tests-deployment.sh b/test/e2e-kubernetes-tests-deployment.sh deleted file mode 100755 index 2cd3bed7..00000000 --- a/test/e2e-kubernetes-tests-deployment.sh +++ /dev/null @@ -1,111 +0,0 @@ -#!/usr/bin/env bash - -# This script runs e2e tests for Blue/Green initialization, analysis and promotion -# Prerequisites: Kubernetes Kind, Kustomize - -set -o errexit - -REPO_ROOT=$(git rev-parse --show-toplevel) - -echo '>>> Creating test namespace' -kubectl create namespace test - -echo '>>> Installing the load tester' -kubectl apply -k ${REPO_ROOT}/kustomize/tester -kubectl -n test rollout status deployment/flagger-loadtester - -echo '>>> Initialising canary' -kubectl apply -f ${REPO_ROOT}/test/e2e-workload.yaml - -cat <>> Waiting for primary to be ready' -retries=50 -count=0 -ok=false -until ${ok}; do - kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false - sleep 5 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n flagger-system logs deployment/flagger - echo "No more retries left" - exit 1 - fi -done - -echo '✔ Canary initialization test passed' - -echo '>>> Triggering canary deployment' -kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1 - -echo '>>> Waiting for canary promotion' -retries=50 -count=0 -ok=false -until ${ok}; do - kubectl -n test describe deployment/podinfo-primary | grep '3.1.1' && ok=true || ok=false - sleep 10 - kubectl -n flagger-system logs deployment/flagger --tail 1 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n test describe deployment/podinfo - kubectl -n test describe deployment/podinfo-primary - kubectl -n flagger-system logs deployment/flagger - echo "No more retries left" - exit 1 - fi -done - -echo '✔ Canary promotion test passed' - -kubectl -n flagger-system logs deployment/flagger diff --git a/test/e2e-kubernetes-tests-svc.sh b/test/e2e-kubernetes-tests-svc.sh deleted file mode 100755 index 9cc2eb39..00000000 --- a/test/e2e-kubernetes-tests-svc.sh +++ /dev/null @@ -1,144 +0,0 @@ -#!/usr/bin/env bash - -# This script runs e2e tests for Blue/Green initialization, analysis and promotion -# Prerequisites: Kubernetes Kind, Kustomize - -set -o errexit - -REPO_ROOT=$(git rev-parse --show-toplevel) - -echo '>>> Creating test namespace' -kubectl create namespace test - -echo '>>> Installing the load tester' -kubectl apply -k ${REPO_ROOT}/kustomize/tester -kubectl -n test rollout status deployment/flagger-loadtester - -echo '>>> Initialising canary' -kubectl apply -f ${REPO_ROOT}/test/e2e-workload.yaml - -kubectl apply -n test -f - <>> Waiting for primary to be ready' -kubectl -n test rollout status deploy/podinfo - -retries=50 -count=0 -ok=false -until ${ok}; do - kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false - sleep 5 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n flagger-system logs deployment/flagger - echo "No more retries left" - exit 1 - fi -done - -echo '✔ Canary initialization test passed' - -echo '>>> Initializing secondary' -kubectl apply -f ${REPO_ROOT}/test/e2e-workload-v2.yaml - -echo '>>> Waiting for secondary to be ready' -kubectl -n test rollout status deploy/podinfo-v2 - -echo '>>> Triggering canary deployment' -kubectl apply -n test -f - <>> Waiting for canary promotion' -retries=50 -count=0 -ok=false -until ${ok}; do - kubectl -n test describe service/podinfo-primary | grep 'podinfo-v2' && ok=true || ok=false - sleep 10 - kubectl -n flagger-system logs deployment/flagger --tail 1 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n test describe deployment/podinfo - kubectl -n test describe deployment/podinfo-primary - kubectl -n flagger-system logs deployment/flagger - echo "No more retries left" - exit 1 - fi -done - -echo '✔ Canary promotion test passed' - -kubectl -n flagger-system logs deployment/flagger diff --git a/test/e2e-kubernetes.sh b/test/e2e-kubernetes.sh deleted file mode 100755 index 12e72733..00000000 --- a/test/e2e-kubernetes.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit - -REPO_ROOT=$(git rev-parse --show-toplevel) - -echo '>>> Loading Flagger image' -kind load docker-image test/flagger:latest - -echo '>>> Installing Flagger' -kubectl apply -k ${REPO_ROOT}/kustomize/kubernetes - -kubectl -n flagger-system set image deployment/flagger flagger=test/flagger:latest - -kubectl -n flagger-system rollout status deployment/flagger -kubectl -n flagger-system rollout status deployment/flagger-prometheus \ No newline at end of file diff --git a/test/e2e-linkerd-steps-tests.sh b/test/e2e-linkerd-steps-tests.sh deleted file mode 100644 index b8f6c53a..00000000 --- a/test/e2e-linkerd-steps-tests.sh +++ /dev/null @@ -1,218 +0,0 @@ -#!/usr/bin/env bash - -# This script runs Linkerd e2e tests for Canary initialization, analysis and promotion - -set -o errexit - -REPO_ROOT=$(git rev-parse --show-toplevel) - -echo '>>> Creating test namespace' -kubectl create namespace test -kubectl annotate namespace test linkerd.io/inject=enabled - -echo '>>> Installing the load tester' -kubectl apply -k ${REPO_ROOT}/kustomize/tester -kubectl -n test rollout status deployment/flagger-loadtester - -echo '>>> Initialising canary' -kubectl apply -f ${REPO_ROOT}/test/e2e-workload.yaml - -cat <>> Waiting for primary to be ready' -retries=50 -count=0 -ok=false -until ${ok}; do - kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false - sleep 5 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n linkerd logs deployment/flagger - echo "No more retries left" - exit 1 - fi -done - -echo '✔ Canary initialization test passed' - -echo '>>> Triggering canary deployment' -kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1 - -echo '>>> Waiting for canary promotion' -retries=50 -count=0 -ok=false -until ${ok}; do - kubectl -n test describe deployment/podinfo-primary | grep '3.1.1' && ok=true || ok=false - sleep 10 - kubectl -n linkerd logs deployment/flagger --tail 1 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n linkerd logs deployment/flagger - echo "No more retries left" - exit 1 - fi -done - -echo '>>> Waiting for canary finalization' -retries=50 -count=0 -ok=false -until ${ok}; do - kubectl -n test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false - sleep 5 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n linkerd logs deployment/flagger - echo "No more retries left" - exit 1 - fi -done - -echo '✔ Canary promotion test passed' - -cat <>> Triggering canary deployment' -kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.2 - -echo '>>> Waiting for canary rollback' -retries=50 -count=0 -ok=false -until ${ok}; do - kubectl -n test get canary/podinfo | grep 'Failed' && ok=true || ok=false - sleep 10 - kubectl -n linkerd logs deployment/flagger --tail 1 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n linkerd logs deployment/flagger - echo "No more retries left" - exit 1 - fi -done - -echo '✔ Canary rollback test passed' \ No newline at end of file diff --git a/test/e2e-linkerd-tests.sh b/test/e2e-linkerd-tests.sh deleted file mode 100755 index d8d1d556..00000000 --- a/test/e2e-linkerd-tests.sh +++ /dev/null @@ -1,259 +0,0 @@ -#!/usr/bin/env bash - -# This script runs Linkerd e2e tests for Canary initialization, analysis and promotion - -set -o errexit - -REPO_ROOT=$(git rev-parse --show-toplevel) - -echo '>>> Creating test namespace' -kubectl create namespace test -kubectl annotate namespace test linkerd.io/inject=enabled - -echo '>>> Installing the load tester' -kubectl apply -k ${REPO_ROOT}/kustomize/tester -kubectl -n test rollout status deployment/flagger-loadtester - -echo '>>> Initialising canary' -kubectl apply -f ${REPO_ROOT}/test/e2e-workload.yaml - -cat <>> Waiting for primary to be ready' -retries=50 -count=0 -ok=false -until ${ok}; do - kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false - sleep 5 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n linkerd logs deployment/flagger - echo "No more retries left" - exit 1 - fi -done - -echo '✔ Canary initialization test passed' - -passed=$(kubectl -n test get svc/podinfo -o jsonpath='{.spec.selector.app}' 2>&1 | { grep podinfo-primary || true; }) -if [ -z "$passed" ]; then - echo -e '\u2716 podinfo selector test failed' - exit 1 -fi -passed=$(kubectl -n test get svc/podinfo-service-canary -o jsonpath='{.spec.selector.app}' 2>&1 | { grep podinfo || true; }) -if [ -z "$passed" ]; then - echo -e '\u2716 podinfo-service selector test failed' - exit 1 -fi - -echo '✔ Canary service custom metadata test passed' - -echo '>>> Triggering canary deployment' -kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1 - -echo '>>> Waiting for canary promotion' -retries=50 -count=0 -ok=false -until ${ok}; do - kubectl -n test describe deployment/podinfo-primary | grep '3.1.1' && ok=true || ok=false - sleep 10 - kubectl -n linkerd logs deployment/flagger --tail 1 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n linkerd logs deployment/flagger - echo "No more retries left" - exit 1 - fi -done - -echo '>>> Waiting for canary finalization' -retries=50 -count=0 -ok=false -until ${ok}; do - kubectl -n test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false - sleep 5 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n linkerd logs deployment/flagger - echo "No more retries left" - exit 1 - fi -done - -echo '✔ Canary promotion test passed' - -cat <>> Triggering canary deployment' -kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.2 - -echo '>>> Waiting for canary rollback' -retries=50 -count=0 -ok=false -until ${ok}; do - kubectl -n test get canary/podinfo | grep 'Failed' && ok=true || ok=false - sleep 10 - kubectl -n linkerd logs deployment/flagger --tail 1 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n linkerd logs deployment/flagger - echo "No more retries left" - exit 1 - fi -done - -echo '✔ Canary rollback test passed' \ No newline at end of file diff --git a/test/e2e-linkerd.sh b/test/e2e-linkerd.sh deleted file mode 100755 index 8d8b143e..00000000 --- a/test/e2e-linkerd.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit - -LINKERD_VER="stable-2.8.1" -REPO_ROOT=$(git rev-parse --show-toplevel) - -curl -SsL https://github.com/linkerd/linkerd2/releases/download/${LINKERD_VER}/linkerd2-cli-${LINKERD_VER}-linux > ${REPO_ROOT}/bin/linkerd -chmod +x ${REPO_ROOT}/bin/linkerd - -echo ">>> Installing Linkerd ${LINKERD_VER}" -${REPO_ROOT}/bin/linkerd install | kubectl apply -f - -${REPO_ROOT}/bin/linkerd check - -kubectl -n linkerd rollout status deployment/linkerd-controller -kubectl -n linkerd rollout status deployment/linkerd-proxy-injector - -echo '>>> Load Flagger image in Kind' -kind load docker-image test/flagger:latest - -echo '>>> Installing Flagger' -kubectl apply -k ${REPO_ROOT}/kustomize/linkerd - -kubectl -n linkerd set image deployment/flagger flagger=test/flagger:latest -kubectl -n linkerd rollout status deployment/flagger diff --git a/test/e2e-nginx-cleanup.sh b/test/e2e-nginx-cleanup.sh deleted file mode 100755 index c27e482a..00000000 --- a/test/e2e-nginx-cleanup.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit - -echo '>>> Deleting NGINX Ingress' -helm delete nginx-ingress -n ingress-nginx - -echo '>>> Deleting Flagger' -helm delete flagger -n ingress-nginx - -echo '>>> Cleanup test namespace' -kubectl delete namespace test --ignore-not-found=true \ No newline at end of file diff --git a/test/e2e-nginx-custom-annotations.sh b/test/e2e-nginx-custom-annotations.sh deleted file mode 100755 index 3193a013..00000000 --- a/test/e2e-nginx-custom-annotations.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit - -REPO_ROOT=$(git rev-parse --show-toplevel) -NGINX_HELM_VERSION=1.34.2 # ingress v0.30.0 - -echo '>>> Installing NGINX Ingress' -helm upgrade -i nginx-ingress stable/nginx-ingress --version=${NGINX_HELM_VERSION} \ ---wait \ ---namespace ingress-nginx \ ---set controller.metrics.enabled=true \ ---set controller.podAnnotations."prometheus\.io/scrape"=true \ ---set controller.podAnnotations."prometheus\.io/port"=10254 \ ---set controller.service.type=NodePort - -kubectl -n ingress-nginx patch deployment/nginx-ingress-controller \ ---type='json' -p='[{"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--annotations-prefix=custom.ingress.kubernetes.io"}]' - -kubectl -n ingress-nginx rollout status deployment/nginx-ingress-controller -kubectl -n ingress-nginx get all - -echo '>>> Loading Flagger image' -kind load docker-image test/flagger:latest - -echo '>>> Installing Flagger' -helm upgrade -i flagger ${REPO_ROOT}/charts/flagger \ ---namespace ingress-nginx \ ---set prometheus.install=true \ ---set ingressAnnotationsPrefix="custom.ingress.kubernetes.io" \ ---set meshProvider=nginx \ ---set crd.create=false - -kubectl -n ingress-nginx set image deployment/flagger flagger=test/flagger:latest - -kubectl -n ingress-nginx rollout status deployment/flagger -kubectl -n ingress-nginx rollout status deployment/flagger-prometheus - diff --git a/test/e2e-nginx-tests.sh b/test/e2e-nginx-tests.sh deleted file mode 100755 index da979601..00000000 --- a/test/e2e-nginx-tests.sh +++ /dev/null @@ -1,313 +0,0 @@ -#!/usr/bin/env bash - -# This script runs e2e tests for Canary initialization, analysis and promotion -# Prerequisites: Kubernetes Kind, Helm and NGINX ingress controller - -set -o errexit - -REPO_ROOT=$(git rev-parse --show-toplevel) - -echo '>>> Creating test namespace' -kubectl create namespace test - -echo '>>> Installing load tester' -kubectl apply -k ${REPO_ROOT}/kustomize/tester -kubectl -n test rollout status deployment/flagger-loadtester - -echo '>>> Initialising canary' -kubectl apply -f ${REPO_ROOT}/test/e2e-workload.yaml -kubectl apply -f ${REPO_ROOT}/test/e2e-ingress.yaml - -echo '>>> Create metric templates' -cat <>> Waiting for primary to be ready' -retries=50 -count=0 -ok=false -until ${ok}; do - kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false - sleep 5 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n ingress-nginx logs deployment/flagger - echo "No more retries left" - exit 1 - fi -done - -echo '✔ Canary initialization test passed' - -passed=$(kubectl -n test get svc/podinfo -o jsonpath='{.spec.selector.app}' | { grep podinfo-primary || true; }) -if [ -z "$passed" ]; then - echo -e '\u2716 podinfo selector test failed' - exit 1 -fi -passed=$(kubectl -n test get svc/podinfo-service-canary -o jsonpath='{.spec.selector.app}' | { grep podinfo || true; }) -if [ -z "$passed" ]; then - echo -e '\u2716 podinfo-service selector test failed' - exit 1 -fi - -echo '✔ Canary service custom metadata test passed' - -echo '>>> Triggering canary deployment' -kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1 - -echo '>>> Waiting for canary promotion' -retries=50 -count=0 -ok=false -failed=false -until ${ok}; do - kubectl -n test get canary/podinfo | grep 'Failed' && failed=true || failed=false - if ${failed}; then - kubectl -n ingress-nginx logs deployment/flagger - echo "Canary failed!" - exit 1 - fi - kubectl -n test describe deployment/podinfo-primary | grep '3.1.1' && ok=true || ok=false - sleep 10 - kubectl -n ingress-nginx logs deployment/flagger --tail 1 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n test describe deployment/podinfo - kubectl -n test describe deployment/podinfo-primary - kubectl -n ingress-nginx logs deployment/flagger - echo "No more retries left" - exit 1 - fi -done - -echo '>>> Waiting for canary finalization' -retries=50 -count=0 -ok=false -until ${ok}; do - kubectl -n test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false - sleep 5 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n ingress-nginx logs deployment/flagger - echo "No more retries left" - exit 1 - fi -done - -echo '✔ Canary promotion test passed' - -cat <>> Triggering A/B testing' -kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.2 - -echo '>>> Waiting for A/B testing promotion' -retries=6 -count=0 -ok=false -until ${ok}; do - kubectl -n test describe deployment/podinfo-primary | grep '3.1.2' && ok=true || ok=false - sleep 30 - kubectl -n ingress-nginx logs deployment/flagger --tail 1 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n test describe deployment/podinfo - kubectl -n test describe deployment/podinfo-primary - kubectl -n ingress-nginx logs deployment/flagger - echo "No more retries left" - exit 1 - fi -done - -echo '✔ A/B testing promotion test passed' - -kubectl -n ingress-nginx logs deployment/flagger - -echo '✔ All tests passed' diff --git a/test/e2e-nginx.sh b/test/e2e-nginx.sh deleted file mode 100755 index e6ba35cf..00000000 --- a/test/e2e-nginx.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit - -REPO_ROOT=$(git rev-parse --show-toplevel) -NGINX_HELM_VERSION=1.39.1 # ingress v0.32.0 - -echo '>>> Installing NGINX Ingress' -kubectl create ns ingress-nginx -helm upgrade -i nginx-ingress stable/nginx-ingress --version=${NGINX_HELM_VERSION} \ ---wait \ ---namespace ingress-nginx \ ---set controller.metrics.enabled=true \ ---set controller.podAnnotations."prometheus\.io/scrape"=true \ ---set controller.podAnnotations."prometheus\.io/port"=10254 \ ---set controller.service.type=NodePort - -kubectl -n ingress-nginx rollout status deployment/nginx-ingress-controller -kubectl -n ingress-nginx get all - -echo '>>> Loading Flagger image' -kind load docker-image test/flagger:latest - -echo '>>> Installing Flagger' -helm upgrade -i flagger ${REPO_ROOT}/charts/flagger \ ---set crd.create=false \ ---namespace ingress-nginx \ ---set prometheus.install=true \ ---set meshProvider=nginx - -kubectl -n ingress-nginx set image deployment/flagger flagger=test/flagger:latest - -kubectl -n ingress-nginx rollout status deployment/flagger -kubectl -n ingress-nginx rollout status deployment/flagger-prometheus diff --git a/test/e2e-skipper-canary.yaml b/test/e2e-skipper-canary.yaml deleted file mode 100644 index 7e0965e4..00000000 --- a/test/e2e-skipper-canary.yaml +++ /dev/null @@ -1,66 +0,0 @@ -apiVersion: flagger.app/v1beta1 -kind: Canary -metadata: - name: podinfo - namespace: test -spec: - provider: skipper - progressDeadlineSeconds: 120 - revertOnDeletion: true - targetRef: - apiVersion: apps/v1 - kind: Deployment - name: podinfo - ingressRef: - apiVersion: networking.k8s.io/v1beta1 - kind: Ingress - name: podinfo-ingress - service: - # service name (defaults to targetRef.name) - name: podinfo-service - # ClusterIP port number - port: 80 - # container port name or number (optional) - targetPort: http - # port name can be http or grpc (default http) - # portName: http - # add all the other container ports - # to the ClusterIP services (default false) - # portDiscovery: false - analysis: - interval: 15s - threshold: 5 - maxWeight: 100 - stepWeight: 10 - metrics: - - name: request-success-rate - interval: 15s - # minimum req success rate (non 5xx responses) - # percentage (0-100) - thresholdRange: - min: 99 - - name: request-duration - interval: 15s - # maximum req duration P99 - # milliseconds - thresholdRange: - max: 500 - webhooks: - - name: gate - type: confirm-rollout - url: http://flagger-loadtester.test/gate/approve - - name: acceptance-test - type: pre-rollout - url: http://flagger-loadtester.test/ - timeout: 10s - metadata: - type: bash - cmd: "curl -sd 'test' http://podinfo-service-canary/token | grep token" - - name: "load test" - type: rollout - url: http://flagger-loadtester.test/ - timeout: 5s - metadata: - type: cmd - cmd: "hey -z 10m -q 10 -c 2 -host app.example.com http://skipper-ingress.kube-system" - logCmdOutput: "true" diff --git a/test/e2e-skipper-cleanup.sh b/test/e2e-skipper-cleanup.sh deleted file mode 100755 index b789007e..00000000 --- a/test/e2e-skipper-cleanup.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash - -REPO_ROOT=$(git rev-parse --show-toplevel) - -echo '>>> Deleting Skipper Ingress' -kustomize build ${REPO_ROOT}/test/skipper | kubectl delete --force --wait=false -f - - -echo '>>> Deleting Flagger' -kubectl delete namespace flagger-system --ignore-not-found=true --force --wait=false - -echo '>>> Cleanup test namespace' -kubectl delete namespace test --ignore-not-found=true --force --wait=false - -exit 0 diff --git a/test/e2e-skipper-tests.sh b/test/e2e-skipper-tests.sh deleted file mode 100755 index 2af94271..00000000 --- a/test/e2e-skipper-tests.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env bash - -# This script runs e2e tests for Canary initialization, analysis and promotion -# Prerequisites: Kubernetes Kind and Skipper ingress controller - -set -o errexit - -REPO_ROOT=$(git rev-parse --show-toplevel) - -echo '>>> Creating test namespace' -kubectl create namespace test || true - -echo '>>> Initialising workload' -kubectl apply -f ${REPO_ROOT}/test/e2e-workload.yaml -kubectl apply -f ${REPO_ROOT}/test/e2e-workload-ingress.yaml - -echo '>>> Installing load tester' -kubectl apply -k ${REPO_ROOT}/kustomize/tester -kubectl -n test rollout status deployment/flagger-loadtester - -echo '>>> Create canary CRD' -kubectl apply -f ${REPO_ROOT}/test/e2e-skipper-canary.yaml -echo '>>> Waiting for primary to be ready' -retries=50 -count=0 -ok=false -until ${ok}; do - kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false - sleep 5 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n flagger-system logs deployment/flagger - echo "No more retries left" - exit 1 - fi -done - -echo '✔ Canary initialization test passed' - -echo '>>> Triggering canary deployment' -kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1 - -echo '>>> Waiting for canary promotion' -retries=50 -count=0 -ok=false -failed=false -until ${ok}; do - kubectl -n test get canary/podinfo | grep 'Failed' && failed=true || failed=false - if ${failed}; then - kubectl -n flagger-system logs deployment/flagger - echo "Canary failed!" - exit 1 - fi - kubectl -n test describe deployment/podinfo-primary | grep '3.1.1' && ok=true || ok=false - sleep 10 - kubectl -n flagger-system logs deployment/flagger --tail 1 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n test describe deployment/podinfo - kubectl -n test describe deployment/podinfo-primary - kubectl -n flagger-system logs deployment/flagger - echo "No more retries left" - exit 1 - fi -done - -echo '>>> Waiting for canary finalization' -retries=50 -count=0 -ok=false -until ${ok}; do - kubectl -n test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false - sleep 5 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n flagger-system logs deployment/flagger - echo "No more retries left" - exit 1 - fi -done - -echo '✔ Canary promotion test passed' diff --git a/test/e2e-skipper.sh b/test/e2e-skipper.sh deleted file mode 100755 index f5dbe2a2..00000000 --- a/test/e2e-skipper.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit - -REPO_ROOT=$(git rev-parse --show-toplevel) - -echo '>>> Loading Flagger image' -kind load docker-image test/flagger:latest - -echo '>>> Installing Skipper Ingress, Flagger and Prometheus' -# use kustomize to avoid compatibility issues: -# https://github.com/kubernetes-sigs/kustomize/issues/2390 -# Skipper will throw an Prometheus warning which can be ignored: -# https://github.com/weaveworks/flagger/issues/664 -kustomize build ${REPO_ROOT}/test/skipper | kubectl apply -f - - -kubectl rollout status deployment/skipper-ingress -n kube-system -kubectl rollout status deployment/flagger-prometheus -n flagger-system - -kubectl -n flagger-system set image deployment/flagger flagger=test/flagger:latest - -kubectl -n flagger-system rollout status deployment/flagger diff --git a/test/e2e-traefik-tests.sh b/test/e2e-traefik-tests.sh deleted file mode 100755 index f927aca7..00000000 --- a/test/e2e-traefik-tests.sh +++ /dev/null @@ -1,156 +0,0 @@ -#!/usr/bin/env bash - -# This script runs e2e tests for Canary initialization, analysis and promotion -# Prerequisites: Kubernetes Kind, Helm and Traefik ingress controller - -set -o errexit - -REPO_ROOT=$(git rev-parse --show-toplevel) - -echo '>>> Creating test namespace' -kubectl create namespace test - -echo '>>> Installing load tester' -kubectl apply -k ${REPO_ROOT}/kustomize/tester -kubectl -n test rollout status deployment/flagger-loadtester - -echo '>>> Initialising canary' -kubectl apply -f ${REPO_ROOT}/test/e2e-workload.yaml - -cat <>> Waiting for primary to be ready' -retries=50 -count=0 -ok=false -until ${ok}; do - kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false - sleep 5 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n traefik logs deployment/flagger - echo "No more retries left" - exit 1 - fi -done - -echo '✔ Canary initialization test passed' - -passed=$(kubectl -n test get svc/podinfo -o jsonpath='{.spec.selector.app}' 2>&1 | { grep podinfo-primary || true; }) -if [ -z "$passed" ]; then - echo -e '\u2716 podinfo selector test failed' - exit 1 -fi -passed=$(kubectl -n test get traefikservice/podinfo -o jsonpath='{.metadata.labels}' 2>&1 | { grep test-label || true; }) -if [ -z "$passed" ]; then - echo -e '\u2716 TraefikService does not have required labels' - exit 1 -fi -passed=$(kubectl -n test get traefikservice/podinfo -o jsonpath='{.metadata.annotations}' 2>&1 | { grep test-annotation || true; }) -if [ -z "$passed" ]; then - echo -e '\u2716 TraefikService does not have required annotations' - exit 1 -fi - -echo '✔ Canary service custom metadata test passed' - -echo '>>> Triggering canary deployment' -kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1 - -echo '>>> Waiting for canary promotion' -retries=60 -count=0 -ok=false -until ${ok}; do - kubectl -n test describe deployment/podinfo-primary | grep '3.1.1' && ok=true || ok=false - sleep 10 - kubectl -n traefik logs deployment/flagger --tail 1 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n test describe deployment/podinfo - kubectl -n test describe deployment/podinfo-primary - kubectl -n test logs deployment/flagger-loadtester - kubectl -n traefik logs deployment/flagger - kubectl -n traefik get all - echo "No more retries left" - exit 1 - fi -done - -echo '✔ Canary promotion test passed' diff --git a/test/e2e-traefik.sh b/test/e2e-traefik.sh deleted file mode 100755 index 5405465c..00000000 --- a/test/e2e-traefik.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit - -REPO_ROOT=$(git rev-parse --show-toplevel) -TRAEFIK_CHART_VERSION="9.11.0" # traefik 2.3.3 - -echo '>>> Creating traefik namespace' -kubectl create ns traefik - -echo '>>> Installing Traefik' -helm repo add traefik https://helm.traefik.io/traefik -cat <>> Loading Flagger image' -kind load docker-image test/flagger:latest - -echo '>>> Installing Flagger' -helm upgrade -i flagger ${REPO_ROOT}/charts/flagger \ ---set crd.create=false \ ---namespace traefik \ ---set prometheus.install=true \ ---set meshProvider=traefik \ ---set image.repository=test\/flagger \ ---set image.tag=latest \ - -kubectl -n traefik rollout status deployment/flagger diff --git a/test/e2e-workload-ingress.yaml b/test/e2e-workload-ingress.yaml deleted file mode 100644 index b7a2c619..00000000 --- a/test/e2e-workload-ingress.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: networking.k8s.io/v1beta1 -kind: Ingress -metadata: - name: podinfo-ingress - namespace: test - labels: - app: podinfo - annotations: - kubernetes.io/ingress.class: skipper -spec: - rules: - - host: app.example.com - http: - paths: - - backend: - serviceName: podinfo-service - servicePort: 80 diff --git a/test/e2e-workload-v2.yaml b/test/e2e-workload-v2.yaml deleted file mode 100644 index eebdc5d3..00000000 --- a/test/e2e-workload-v2.yaml +++ /dev/null @@ -1,68 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: podinfo-v2 - namespace: test - labels: - app: podinfo-v2 -spec: - minReadySeconds: 5 - revisionHistoryLimit: 5 - progressDeadlineSeconds: 60 - strategy: - rollingUpdate: - maxUnavailable: 0 - type: RollingUpdate - selector: - matchLabels: - app: podinfo-v2 - template: - metadata: - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "9797" - labels: - app: podinfo-v2 - spec: - containers: - - name: podinfod - image: stefanprodan/podinfo:3.1.1 - imagePullPolicy: IfNotPresent - ports: - - name: http - containerPort: 9898 - protocol: TCP - - name: http-metrics - containerPort: 9797 - protocol: TCP - - name: grpc - containerPort: 9999 - protocol: TCP - command: - - ./podinfo - - --port=9898 - - --port-metrics=9797 - - --grpc-port=9999 - - --grpc-service-name=podinfo - - --level=info - - --random-delay=false - - --random-error=false - livenessProbe: - httpGet: - port: 9898 - path: /healthz - initialDelaySeconds: 5 - timeoutSeconds: 5 - readinessProbe: - httpGet: - port: 9898 - path: /readyz - initialDelaySeconds: 5 - timeoutSeconds: 5 - resources: - limits: - cpu: 1000m - memory: 128Mi - requests: - cpu: 1m - memory: 16Mi diff --git a/test/e2e-workload.yaml b/test/e2e-workload.yaml deleted file mode 100644 index 8f0660c0..00000000 --- a/test/e2e-workload.yaml +++ /dev/null @@ -1,137 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: podinfo - namespace: test - labels: - app: podinfo -spec: - minReadySeconds: 5 - revisionHistoryLimit: 5 - progressDeadlineSeconds: 60 - strategy: - rollingUpdate: - maxUnavailable: 0 - type: RollingUpdate - selector: - matchLabels: - app: podinfo - template: - metadata: - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "9797" - labels: - app: podinfo - spec: - containers: - - name: podinfod - image: stefanprodan/podinfo:3.1.0 - imagePullPolicy: IfNotPresent - ports: - - name: http - containerPort: 9898 - protocol: TCP - - name: http-metrics - containerPort: 9797 - protocol: TCP - - name: grpc - containerPort: 9999 - protocol: TCP - command: - - ./podinfo - - --port=9898 - - --port-metrics=9797 - - --grpc-port=9999 - - --grpc-service-name=podinfo - - --level=info - - --random-delay=false - - --random-error=false - livenessProbe: - httpGet: - port: 9898 - path: /healthz - initialDelaySeconds: 5 - timeoutSeconds: 5 - readinessProbe: - httpGet: - port: 9898 - path: /readyz - initialDelaySeconds: 5 - timeoutSeconds: 5 - resources: - limits: - cpu: 1000m - memory: 128Mi - requests: - cpu: 1m - memory: 16Mi ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: podinfo-service - namespace: test - labels: - app: podinfo -spec: - minReadySeconds: 5 - revisionHistoryLimit: 5 - progressDeadlineSeconds: 60 - strategy: - rollingUpdate: - maxUnavailable: 0 - type: RollingUpdate - selector: - matchLabels: - app: podinfo - template: - metadata: - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "9797" - labels: - app: podinfo - spec: - containers: - - name: podinfod - image: stefanprodan/podinfo:3.1.0 - imagePullPolicy: IfNotPresent - ports: - - name: http - containerPort: 9898 - protocol: TCP - - name: http-metrics - containerPort: 9797 - protocol: TCP - - name: grpc - containerPort: 9999 - protocol: TCP - command: - - ./podinfo - - --port=9898 - - --port-metrics=9797 - - --grpc-port=9999 - - --grpc-service-name=podinfo - - --level=info - - --random-delay=false - - --random-error=false - livenessProbe: - httpGet: - port: 9898 - path: /healthz - initialDelaySeconds: 5 - timeoutSeconds: 5 - readinessProbe: - httpGet: - port: 9898 - path: /readyz - initialDelaySeconds: 5 - timeoutSeconds: 5 - resources: - limits: - cpu: 1000m - memory: 128Mi - requests: - cpu: 1m - memory: 16Mi \ No newline at end of file diff --git a/test/goreleaser.sh b/test/goreleaser.sh deleted file mode 100755 index c41289ef..00000000 --- a/test/goreleaser.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/sh -set -e - -TAR_FILE="/tmp/goreleaser.tar.gz" -RELEASES_URL="https://github.com/goreleaser/goreleaser/releases" -test -z "$TMPDIR" && TMPDIR="$(mktemp -d)" - -last_version() { - curl -sL -o /dev/null -w %{url_effective} "$RELEASES_URL/latest" | - rev | - cut -f1 -d'/'| - rev -} - -download() { - test -z "$VERSION" && VERSION="$(last_version)" - test -z "$VERSION" && { - echo "Unable to get goreleaser version." >&2 - exit 1 - } - rm -f "$TAR_FILE" - curl -s -L -o "$TAR_FILE" \ - "$RELEASES_URL/download/$VERSION/goreleaser_$(uname -s)_$(uname -m).tar.gz" -} - -download -tar -xf "$TAR_FILE" -C "$TMPDIR" -"${TMPDIR}/goreleaser" --release-notes=/tmp/release.txt diff --git a/test/local/e2e-skipper.sh b/test/local/e2e-skipper.sh deleted file mode 100755 index 0e98d6d0..00000000 --- a/test/local/e2e-skipper.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash - -# This script is intended for local workstation development convenience. -# It will run the e2e tests for Skipper and leave a working setup to play with - -REPO_ROOT=$(git rev-parse --show-toplevel) -cd $REPO_ROOT - -make test -make build -docker tag weaveworks/flagger:latest test/flagger:latest -make loadtester-build -(kind get clusters && kubectl delete ns/test --force) || kind create cluster --wait 5m --image kindest/node:v1.16.9 -./test/e2e-skipper.sh -# port forward prometheus UI to localhost:9090 -kubectl port-forward $(kubectl get pods -l=app=flagger-prometheus -o name -n flagger-system | head -n 1) 9090:9090 -n flagger-system & - -./test/e2e-skipper-tests.sh From 38777801de4135b65aed9fe7292fe87acb96ec5b Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Mon, 21 Dec 2020 17:04:31 +0200 Subject: [PATCH 18/19] Upload coverage to Codecov Signed-off-by: Stefan Prodan --- .github/workflows/build.yaml | 8 +++++++- .github/workflows/release.yml | 1 + .gitignore | 1 + 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index c124734b..8eed6035 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -28,8 +28,10 @@ jobs: run: | go mod download go install golang.org/x/tools/cmd/goimports + - name: Run linters + run: make test-fmt test-codegen - name: Run tests - run: make test + run: go test -race -coverprofile=coverage.txt -covermode=atomic $(go list ./pkg/...) - name: Check if working tree is dirty run: | if [[ $(git diff --stat) != '' ]]; then @@ -37,5 +39,9 @@ jobs: echo 'run make test and commit changes' exit 1 fi + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v1 + with: + file: ./coverage.txt - name: Build container image run: docker build -t test/flagger:latest . diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d5b8f443..b7c9b459 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -58,6 +58,7 @@ jobs: uses: stefanprodan/helm-gh-pages@master with: token: ${{ secrets.GITHUB_TOKEN }} + charts_url: https://flagger.app - name: Create release uses: actions/create-release@latest env: diff --git a/.gitignore b/.gitignore index 92c6a2e3..901c9a04 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,4 @@ artifacts/gcloud/ Makefile.dev vendor +coverage.txt From 333780e78b6b7bd7d92aab946ebca67ed86997a8 Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Mon, 21 Dec 2020 17:26:05 +0200 Subject: [PATCH 19/19] Disable CircleCI main build Signed-off-by: Stefan Prodan --- .circleci/config.yml | 291 +------------------------------------------ 1 file changed, 6 insertions(+), 285 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e328734b..983d7718 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,298 +1,19 @@ version: 2.1 jobs: - - build-binary: - docker: - - image: circleci/golang:1.14 - working_directory: ~/build - steps: - - checkout - - restore_cache: - keys: - - go-mod-v3-{{ checksum "go.sum" }} - - run: - name: Run go mod download - command: go mod download - - run: - name: Check code formatting - command: go install golang.org/x/tools/cmd/goimports && make test-fmt - - run: - name: Build Flagger - command: | - CGO_ENABLED=0 GOOS=linux go build \ - -ldflags "-s -w -X github.com/weaveworks/flagger/pkg/version.REVISION=${CIRCLE_SHA1}" \ - -a -installsuffix cgo -o bin/flagger ./cmd/flagger/*.go - - run: - name: Build Flagger load tester - command: | - CGO_ENABLED=0 GOOS=linux go build \ - -a -installsuffix cgo -o bin/loadtester ./cmd/loadtester/*.go - - run: - name: Run unit tests - command: | - go test -race -coverprofile=coverage.txt -covermode=atomic $(go list ./pkg/...) - bash <(curl -s https://codecov.io/bash) - - run: - name: Verify code gen - command: make test-codegen - - save_cache: - key: go-mod-v3-{{ checksum "go.sum" }} - paths: - - "/go/pkg/mod/" - - persist_to_workspace: - root: bin - paths: - - flagger - - loadtester - - push-container: - docker: - - image: circleci/golang:1.14 - steps: - - checkout - - setup_remote_docker: - docker_layer_caching: true - - attach_workspace: - at: /tmp/bin - - run: test/container-build.sh - - run: test/container-push.sh - - push-binary: - docker: - - image: circleci/golang:1.14 - working_directory: ~/build - steps: - - checkout - - setup_remote_docker: - docker_layer_caching: true - - restore_cache: - keys: - - go-mod-v3-{{ checksum "go.sum" }} - - run: make release-notes - - run: github-release-notes -org weaveworks -repo flagger -since-latest-release -include-author > /tmp/release.txt - - run: test/goreleaser.sh - - e2e-kubernetes-testing: + build: machine: true steps: - - checkout - - attach_workspace: - at: /tmp/bin - - run: test/container-build.sh - - run: test/e2e-kind.sh v1.18.2 - - run: test/e2e-kubernetes.sh - - run: test/e2e-kubernetes-tests-deployment.sh - - run: test/e2e-kubernetes-cleanup.sh - - run: test/e2e-kubernetes-tests-daemonset.sh - - e2e-istio-testing: - machine: true - steps: - - checkout - - attach_workspace: - at: /tmp/bin - - run: test/container-build.sh - - run: test/e2e-kind.sh v1.18.2 - - run: test/e2e-istio.sh - - run: test/e2e-istio-dependencies.sh - - run: test/e2e-istio-tests.sh - - run: test/e2e-istio-tests-skip-analysis.sh - - run: test/e2e-kubernetes-cleanup.sh - - run: test/e2e-istio-dependencies.sh - - run: test/e2e-istio-tests-delegate.sh - - e2e-gloo-testing: - machine: true - steps: - - checkout - - attach_workspace: - at: /tmp/bin - - run: test/container-build.sh - - run: test/e2e-kind.sh - - run: test/e2e-gloo.sh - - run: test/e2e-gloo-tests.sh - - e2e-nginx-testing: - machine: true - steps: - - checkout - - attach_workspace: - at: /tmp/bin - - run: test/container-build.sh - - run: test/e2e-kind.sh - - run: test/e2e-nginx.sh - - run: test/e2e-nginx-tests.sh - - run: test/e2e-nginx-cleanup.sh - - run: test/e2e-nginx-custom-annotations.sh - - run: test/e2e-nginx-tests.sh - - e2e-linkerd-testing: - machine: true - steps: - - checkout - - attach_workspace: - at: /tmp/bin - - run: test/container-build.sh - - run: test/e2e-kind.sh - - run: test/e2e-linkerd.sh - - run: test/e2e-linkerd-tests.sh - - e2e-contour-testing: - machine: true - steps: - - checkout - - attach_workspace: - at: /tmp/bin - - run: test/container-build.sh - - run: test/e2e-kind.sh - - run: test/e2e-contour.sh - - run: test/e2e-contour-tests.sh - - e2e-skipper-testing: - machine: true - steps: - - checkout - - attach_workspace: - at: /tmp/bin - - run: test/container-build.sh - - run: test/e2e-kind.sh - - run: test/e2e-skipper.sh - - run: test/e2e-skipper-tests.sh - - run: test/e2e-skipper-cleanup.sh - - e2e-traefik-testing: - machine: true - steps: - - checkout - - attach_workspace: - at: /tmp/bin - - run: test/container-build.sh - - run: test/e2e-kind.sh - - run: test/e2e-traefik.sh - - run: test/e2e-traefik-tests.sh - - run: test/e2e-skipper-cleanup.sh - - push-helm-charts: - docker: - - image: circleci/golang:1.14 - steps: - - checkout - - run: - name: Install kubectl - command: sudo curl -L https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl -o /usr/local/bin/kubectl && sudo chmod +x /usr/local/bin/kubectl - - run: - name: Install helm - command: sudo curl -L https://storage.googleapis.com/kubernetes-helm/helm-v2.14.2-linux-amd64.tar.gz | tar xz && sudo mv linux-amd64/helm /bin/helm && sudo rm -rf linux-amd64 - - run: - name: Initialize helm - command: helm init --client-only --kubeconfig=$HOME/.kube/kubeconfig - - run: - name: Lint charts - command: | - helm lint ./charts/* - - run: - name: Package charts - command: | - mkdir $HOME/charts - helm package ./charts/* --destination $HOME/charts - - run: - name: Publish charts - command: | - if echo "${CIRCLE_TAG}" | grep v; then - REPOSITORY="https://weaveworksbot:${GITHUB_TOKEN}@github.com/weaveworks/flagger.git" - git config user.email weaveworksbot@users.noreply.github.com - git config user.name weaveworksbot - git remote set-url origin ${REPOSITORY} - git checkout gh-pages - mv -f $HOME/charts/*.tgz . - helm repo index . --url https://flagger.app - git add . - git commit -m "Publish Helm charts v${CIRCLE_TAG}" - git push origin gh-pages - else - echo "Not a release! Skip charts publish" - fi + - run: "echo skip" workflows: version: 2 - build-test-push: + ignore: jobs: - - build-binary: + - build: filters: branches: ignore: - gh-pages - - /^user-.*/ - - github-actions - - main - - e2e-kubernetes-testing: - requires: - - build-binary - - e2e-istio-testing: - requires: - - build-binary - - e2e-gloo-testing: - requires: - - build-binary - - e2e-nginx-testing: - requires: - - build-binary - - e2e-linkerd-testing: - requires: - - build-binary - - e2e-contour-testing: - requires: - - build-binary - - e2e-skipper-testing: - requires: - - build-binary - - e2e-traefik-testing: - requires: - - build-binary - - push-container: - requires: - - build-binary - - e2e-kubernetes-testing - - e2e-istio-testing - - e2e-gloo-testing - - e2e-nginx-testing - - e2e-linkerd-testing - - e2e-skipper-testing - - e2e-traefik-testing - filters: - branches: - only: - master - - release: - jobs: - - build-binary: - filters: - branches: - ignore: /.*/ - tags: - ignore: /^chart.*/ - - push-container: - requires: - - build-binary - filters: - branches: - ignore: /.*/ - tags: - ignore: /^chart.*/ - - push-binary: - requires: - - push-container - filters: - branches: - ignore: /.*/ - tags: - ignore: /^chart.*/ - - push-helm-charts: - requires: - - push-container - filters: - branches: - ignore: /.*/ - tags: - ignore: /^chart.*/ + - main + - github-actions