From f58de3801c9053ec665a7f7a7f876b46e8fd0af5 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Sun, 24 Feb 2019 01:00:03 +0200 Subject: [PATCH 01/15] Add Istio install values for e2e testing --- e2e/istio-values.yaml | 62 +++++++++++++++++++++++++++++++++++++++ e2e/workload.yaml | 67 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 129 insertions(+) create mode 100644 e2e/istio-values.yaml create mode 100644 e2e/workload.yaml diff --git a/e2e/istio-values.yaml b/e2e/istio-values.yaml new file mode 100644 index 00000000..131065ba --- /dev/null +++ b/e2e/istio-values.yaml @@ -0,0 +1,62 @@ +# +# Minimal Istio Configuration required by Flagger +# + +# pilot configuration +pilot: + enabled: true + sidecar: true + +gateways: + enabled: false + istio-ingressgateway: + autoscaleMax: 1 + +# citadel configuration +security: + enabled: true + +# sidecar-injector webhook configuration +sidecarInjectorWebhook: + enabled: true + +# galley configuration +galley: + enabled: false + +# mixer configuration +mixer: + policy: + enabled: false + replicaCount: 1 + autoscaleEnabled: false + telemetry: + enabled: true + replicaCount: 1 + autoscaleEnabled: false + resources: + requests: + cpu: 100m + memory: 128Mi + +# addon prometheus configuration +prometheus: + enabled: true + scrapeInterval: 5s + +# addon jaeger tracing configuration +tracing: + enabled: false + +# Common settings. +global: + proxy: + # Resources for the sidecar. + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 2000m + memory: 128Mi + useMCP: false diff --git a/e2e/workload.yaml b/e2e/workload.yaml new file mode 100644 index 00000000..57ed8a41 --- /dev/null +++ b/e2e/workload.yaml @@ -0,0 +1,67 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: podinfo + namespace: test + labels: + app: podinfo +spec: + minReadySeconds: 5 + revisionHistoryLimit: 5 + progressDeadlineSeconds: 60 + strategy: + rollingUpdate: + maxUnavailable: 0 + type: RollingUpdate + selector: + matchLabels: + app: podinfo + template: + metadata: + annotations: + prometheus.io/scrape: "true" + labels: + app: podinfo + spec: + containers: + - name: podinfod + image: quay.io/stefanprodan/podinfo:1.4.0 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9898 + name: http + protocol: TCP + command: + - ./podinfo + - --port=9898 + - --level=info + - --random-delay=false + - --random-error=false + env: + - name: PODINFO_UI_COLOR + value: blue + livenessProbe: + exec: + command: + - podcli + - check + - http + - localhost:9898/healthz + initialDelaySeconds: 5 + timeoutSeconds: 5 + readinessProbe: + exec: + command: + - podcli + - check + - http + - localhost:9898/readyz + initialDelaySeconds: 5 + timeoutSeconds: 5 + resources: + limits: + cpu: 2000m + memory: 512Mi + requests: + cpu: 100m + memory: 64Mi From 30cbf2a7414f59327c740a4adabca19340edb282 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Sun, 24 Feb 2019 01:02:15 +0200 Subject: [PATCH 02/15] Add e2e tests - create Kubernetes cluster with Kind - install Istio and Prometheus - install Flagger - test canary init and promotion --- e2e/e2e.sh | 119 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100755 e2e/e2e.sh diff --git a/e2e/e2e.sh b/e2e/e2e.sh new file mode 100755 index 00000000..4744719f --- /dev/null +++ b/e2e/e2e.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bash + +set -o errexit + +ISTIO_VER="1.1.0-rc.0" +REPO_ROOT=$(git rev-parse --show-toplevel) + +echo ">>> Starting e2e testing using Istio ${ISTIO_VER}" +kind create cluster --wait 5m + +export KUBECONFIG="$(kind get kubeconfig-path)" +kubectl version + +echo '>>> Installing Tiller' +kubectl --namespace kube-system create sa tiller +kubectl create clusterrolebinding tiller-cluster-rule --clusterrole=cluster-admin --serviceaccount=kube-system:tiller +helm init --service-account tiller --upgrade --wait + +helm repo add istio.io https://storage.googleapis.com/istio-release/releases/${ISTIO_VER}/charts + +echo '>>> Installing Istio CRDs' +helm upgrade -i istio-init istio.io/istio-init --wait --namespace istio-system + +echo '>>> Waiting for Istio CRDs to be ready' +kubectl -n istio-system wait --for=condition=complete job/istio-init-crd-10 +kubectl -n istio-system wait --for=condition=complete job/istio-init-crd-11 + +echo 'Installing Istio control plane' +helm upgrade -i istio istio.io/istio --wait --namespace istio-system -f ./istio-values.yaml + +export KUBECONFIG="$(kind get kubeconfig-path)" + +echo '>>> Installing Flagger' +#cd ${REPO_ROOT} && docker build -t stefanprodan/flagger:latest . -f Dockerfile +#kind load docker-image stefanprodan/flagger:latest +kubectl apply -f ${REPO_ROOT}/artifacts/flagger/ +#kubectl -n istio-system set image deployment/flagger flagger=stefanprodan/flagger:latest +kubectl -n istio-system rollout status deployment/flagger + +echo '>>> Creating test namespace' +kubectl create namespace test +kubectl label namespace test istio-injection=enabled + +echo '>>> Installing the load tester' +kubectl -n test apply -f ${REPO_ROOT}/artifacts/loadtester/ +kubectl -n test rollout status deployment/flagger-loadtester + +echo '>>> Initialising canary' +kubectl apply -f ./workload.yaml + +cat <>> Waiting for primary to be ready' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false + sleep 2 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + echo "No more retries left" + exit 1 + fi +done + +echo '>>> Canary initialization test passed ✔︎' + +echo '>>> Triggering canary deployment' +kubectl -n test set image deployment/podinfo podinfod=quay.io/stefanprodan/podinfo:1.4.2 + +echo '>>> Waiting for canary promotion' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test describe deployment/podinfo-primary | grep '1.4.2' && ok=true || ok=false + sleep 2 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + echo "No more retries left" + exit 1 + fi +done + +echo '>>> Canary promotion test passed ✔︎' From e0e673f5654838df9dc16a60513a4c86868749e0 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Sun, 24 Feb 2019 01:03:39 +0200 Subject: [PATCH 03/15] Install e2e deps and run tests --- .travis.yml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index a7ef8da3..aae581fa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,12 +12,18 @@ addons: packages: - docker-ce +before_script: + - go get -u sigs.k8s.io/kind + - curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get | bash + - curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && chmod +x kubectl && sudo mv kubectl /usr/local/bin/ + script: -- set -e -- make test-fmt -- make test-codegen -- go test -race -coverprofile=coverage.txt -covermode=atomic ./pkg/controller/ -- make build + - set -e + - e2e/e2e.sh + - make test-fmt + - make test-codegen + - go test -race -coverprofile=coverage.txt -covermode=atomic ./pkg/controller/ + - make build after_success: - if [ -z "$DOCKER_USER" ]; then From e2347c84e318300a421617b0ef87f91063e8a32e Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Sun, 24 Feb 2019 01:11:04 +0200 Subject: [PATCH 04/15] Use absolute paths in e2e tests --- e2e/e2e.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e/e2e.sh b/e2e/e2e.sh index 4744719f..7b4a0a9b 100755 --- a/e2e/e2e.sh +++ b/e2e/e2e.sh @@ -26,7 +26,7 @@ kubectl -n istio-system wait --for=condition=complete job/istio-init-crd-10 kubectl -n istio-system wait --for=condition=complete job/istio-init-crd-11 echo 'Installing Istio control plane' -helm upgrade -i istio istio.io/istio --wait --namespace istio-system -f ./istio-values.yaml +helm upgrade -i istio istio.io/istio --wait --namespace istio-system -f ${REPO_ROOT}/e2e/istio-values.yaml export KUBECONFIG="$(kind get kubeconfig-path)" @@ -46,7 +46,7 @@ kubectl -n test apply -f ${REPO_ROOT}/artifacts/loadtester/ kubectl -n test rollout status deployment/flagger-loadtester echo '>>> Initialising canary' -kubectl apply -f ./workload.yaml +kubectl apply -f ${REPO_ROOT}/e2e/workload.yaml cat < Date: Sun, 24 Feb 2019 01:58:03 +0200 Subject: [PATCH 05/15] Add e2e prerequisites --- test/Dockerfile | 4 ++++ .../e2e-istio-values.yaml | 0 e2e/e2e.sh => test/e2e-kind.sh | 4 ++-- test/e2e-prerequisites.sh | 21 +++++++++++++++++++ e2e/workload.yaml => test/e2e-workload.yaml | 0 5 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 test/Dockerfile rename e2e/istio-values.yaml => test/e2e-istio-values.yaml (100%) rename e2e/e2e.sh => test/e2e-kind.sh (99%) create mode 100755 test/e2e-prerequisites.sh rename e2e/workload.yaml => test/e2e-workload.yaml (100%) diff --git a/test/Dockerfile b/test/Dockerfile new file mode 100644 index 00000000..d73a6358 --- /dev/null +++ b/test/Dockerfile @@ -0,0 +1,4 @@ +FROM golang + +RUN go get -u sigs.k8s.io/kind + diff --git a/e2e/istio-values.yaml b/test/e2e-istio-values.yaml similarity index 100% rename from e2e/istio-values.yaml rename to test/e2e-istio-values.yaml diff --git a/e2e/e2e.sh b/test/e2e-kind.sh similarity index 99% rename from e2e/e2e.sh rename to test/e2e-kind.sh index 7b4a0a9b..cd5269b1 100755 --- a/e2e/e2e.sh +++ b/test/e2e-kind.sh @@ -89,7 +89,7 @@ count=0 ok=false until ${ok}; do kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false - sleep 2 + sleep 5 count=$(($count + 1)) if [[ ${count} -eq ${retries} ]]; then echo "No more retries left" @@ -108,7 +108,7 @@ count=0 ok=false until ${ok}; do kubectl -n test describe deployment/podinfo-primary | grep '1.4.2' && ok=true || ok=false - sleep 2 + sleep 5 count=$(($count + 1)) if [[ ${count} -eq ${retries} ]]; then echo "No more retries left" diff --git a/test/e2e-prerequisites.sh b/test/e2e-prerequisites.sh new file mode 100755 index 00000000..cec9972d --- /dev/null +++ b/test/e2e-prerequisites.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +REPO_ROOT=$(git rev-parse --show-toplevel) + +echo ">>> Building sigs.k8s.io/kind" +docker build -t kind:src . -f ${REPO_ROOT}/test/Dockerfile +docker create -ti --name dummy kind:src sh +docker cp dummy:/go/bin/kind ./kind +docker rm -f dummy + +echo ">>> Installing kind" +chmod +x kind && \ +sudo mv kind /usr/local/bin/ + +echo ">>> Installing helm" +curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get | bash + +echo ">>> Installing kubectl" +curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && \ +chmod +x kubectl && \ +sudo mv kubectl /usr/local/bin/ diff --git a/e2e/workload.yaml b/test/e2e-workload.yaml similarity index 100% rename from e2e/workload.yaml rename to test/e2e-workload.yaml From c669dc0c4ba378a5d3e4a7bdfb23e504659087a9 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Sun, 24 Feb 2019 01:58:18 +0200 Subject: [PATCH 06/15] Run e2e tests with CircleCI --- .circleci/config.yml | 14 ++++++++++++++ .travis.yml | 9 ++++----- 2 files changed, 18 insertions(+), 5 deletions(-) create mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000..459c16ea --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,14 @@ +version: 2.1 +jobs: + e2e-testing: + machine: true + steps: + - checkout + - run: test/e2e-prerequisites.sh + - run: test/e2e-kind.sh + +workflows: + version: 2 + build-and-test: + jobs: + - e2e-testing diff --git a/.travis.yml b/.travis.yml index aae581fa..a5c4b59a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,14 +12,13 @@ addons: packages: - docker-ce -before_script: - - go get -u sigs.k8s.io/kind - - curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get | bash - - curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && chmod +x kubectl && sudo mv kubectl /usr/local/bin/ +#before_script: +# - go get -u sigs.k8s.io/kind +# - curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get | bash +# - curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && chmod +x kubectl && sudo mv kubectl /usr/local/bin/ script: - set -e - - e2e/e2e.sh - make test-fmt - make test-codegen - go test -race -coverprofile=coverage.txt -covermode=atomic ./pkg/controller/ From 8a8f68af5ddbab150ce77f249ed05aaa6fb348d8 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Sun, 24 Feb 2019 02:02:37 +0200 Subject: [PATCH 07/15] Test CircleCI --- test/e2e-prerequisites.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e-prerequisites.sh b/test/e2e-prerequisites.sh index cec9972d..b856e08e 100755 --- a/test/e2e-prerequisites.sh +++ b/test/e2e-prerequisites.sh @@ -2,7 +2,7 @@ REPO_ROOT=$(git rev-parse --show-toplevel) -echo ">>> Building sigs.k8s.io/kind" +echo ">>> Building sigs.k8s.io/kind with docker" docker build -t kind:src . -f ${REPO_ROOT}/test/Dockerfile docker create -ti --name dummy kind:src sh docker cp dummy:/go/bin/kind ./kind From ac4972dd8d408859f1c626939e48f358af2d7d5e Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Sun, 24 Feb 2019 02:09:45 +0200 Subject: [PATCH 08/15] Fix e2e paths --- test/e2e-kind.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/e2e-kind.sh b/test/e2e-kind.sh index cd5269b1..083dcdeb 100755 --- a/test/e2e-kind.sh +++ b/test/e2e-kind.sh @@ -26,15 +26,15 @@ kubectl -n istio-system wait --for=condition=complete job/istio-init-crd-10 kubectl -n istio-system wait --for=condition=complete job/istio-init-crd-11 echo 'Installing Istio control plane' -helm upgrade -i istio istio.io/istio --wait --namespace istio-system -f ${REPO_ROOT}/e2e/istio-values.yaml +helm upgrade -i istio istio.io/istio --wait --namespace istio-system -f ${REPO_ROOT}/test/e2e-istio-values.yaml export KUBECONFIG="$(kind get kubeconfig-path)" echo '>>> Installing Flagger' -#cd ${REPO_ROOT} && docker build -t stefanprodan/flagger:latest . -f Dockerfile -#kind load docker-image stefanprodan/flagger:latest +cd ${REPO_ROOT} && docker build -t stefanprodan/flagger:latest . -f Dockerfile +kind load docker-image test/flagger:latest kubectl apply -f ${REPO_ROOT}/artifacts/flagger/ -#kubectl -n istio-system set image deployment/flagger flagger=stefanprodan/flagger:latest +kubectl -n istio-system set image deployment/flagger flagger=test/flagger:latest kubectl -n istio-system rollout status deployment/flagger echo '>>> Creating test namespace' @@ -46,7 +46,7 @@ kubectl -n test apply -f ${REPO_ROOT}/artifacts/loadtester/ kubectl -n test rollout status deployment/flagger-loadtester echo '>>> Initialising canary' -kubectl apply -f ${REPO_ROOT}/e2e/workload.yaml +kubectl apply -f ${REPO_ROOT}/test/e2e-workload.yaml cat < Date: Sun, 24 Feb 2019 02:11:32 +0200 Subject: [PATCH 09/15] Use local docker image for e2e testing --- test/e2e-kind.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e-kind.sh b/test/e2e-kind.sh index 083dcdeb..9abfe995 100755 --- a/test/e2e-kind.sh +++ b/test/e2e-kind.sh @@ -31,7 +31,7 @@ helm upgrade -i istio istio.io/istio --wait --namespace istio-system -f ${REPO_R export KUBECONFIG="$(kind get kubeconfig-path)" echo '>>> Installing Flagger' -cd ${REPO_ROOT} && docker build -t stefanprodan/flagger:latest . -f Dockerfile +cd ${REPO_ROOT} && docker build -t test/flagger:latest . -f Dockerfile kind load docker-image test/flagger:latest kubectl apply -f ${REPO_ROOT}/artifacts/flagger/ kubectl -n istio-system set image deployment/flagger flagger=test/flagger:latest From 87b9fa8ca782115d45fa522aa3f6f48bb0014c0d Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Sun, 24 Feb 2019 02:24:23 +0200 Subject: [PATCH 10/15] Move cluster init to prerequisites --- artifacts/flagger/deployment.yaml | 2 +- test/{Dockerfile => Dockerfile.kind} | 2 +- test/e2e-kind.sh | 7 ------ test/e2e-prerequisites.sh | 32 +++++++++++++++++----------- 4 files changed, 21 insertions(+), 22 deletions(-) rename test/{Dockerfile => Dockerfile.kind} (66%) diff --git a/artifacts/flagger/deployment.yaml b/artifacts/flagger/deployment.yaml index 782fba45..fcd0e813 100644 --- a/artifacts/flagger/deployment.yaml +++ b/artifacts/flagger/deployment.yaml @@ -23,7 +23,7 @@ spec: containers: - name: flagger image: quay.io/stefanprodan/flagger:0.5.1 - imagePullPolicy: Always + imagePullPolicy: IfNotPresent ports: - name: http containerPort: 8080 diff --git a/test/Dockerfile b/test/Dockerfile.kind similarity index 66% rename from test/Dockerfile rename to test/Dockerfile.kind index d73a6358..28f5e083 100644 --- a/test/Dockerfile +++ b/test/Dockerfile.kind @@ -1,4 +1,4 @@ -FROM golang +FROM golang:1.11 RUN go get -u sigs.k8s.io/kind diff --git a/test/e2e-kind.sh b/test/e2e-kind.sh index 9abfe995..eac7ff83 100755 --- a/test/e2e-kind.sh +++ b/test/e2e-kind.sh @@ -6,16 +6,9 @@ ISTIO_VER="1.1.0-rc.0" REPO_ROOT=$(git rev-parse --show-toplevel) echo ">>> Starting e2e testing using Istio ${ISTIO_VER}" -kind create cluster --wait 5m - export KUBECONFIG="$(kind get kubeconfig-path)" kubectl version -echo '>>> Installing Tiller' -kubectl --namespace kube-system create sa tiller -kubectl create clusterrolebinding tiller-cluster-rule --clusterrole=cluster-admin --serviceaccount=kube-system:tiller -helm init --service-account tiller --upgrade --wait - helm repo add istio.io https://storage.googleapis.com/istio-release/releases/${ISTIO_VER}/charts echo '>>> Installing Istio CRDs' diff --git a/test/e2e-prerequisites.sh b/test/e2e-prerequisites.sh index b856e08e..fa30860d 100755 --- a/test/e2e-prerequisites.sh +++ b/test/e2e-prerequisites.sh @@ -2,20 +2,26 @@ REPO_ROOT=$(git rev-parse --show-toplevel) -echo ">>> Building sigs.k8s.io/kind with docker" -docker build -t kind:src . -f ${REPO_ROOT}/test/Dockerfile -docker create -ti --name dummy kind:src sh -docker cp dummy:/go/bin/kind ./kind -docker rm -f dummy - -echo ">>> Installing kind" -chmod +x kind && \ -sudo mv kind /usr/local/bin/ - -echo ">>> Installing helm" -curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get | bash - echo ">>> Installing kubectl" curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && \ chmod +x kubectl && \ sudo mv kubectl /usr/local/bin/ + +echo ">>> Building sigs.k8s.io/kind" +docker build -t kind:src . -f ${REPO_ROOT}/test/Dockerfile.kind +docker create -ti --name dummy kind:src sh +docker cp dummy:/go/bin/kind ./kind +docker rm -f dummy + +echo ">>> Installing kind" +chmod +x kind +sudo mv kind /usr/local/bin/ +kind create cluster --wait 5m + +echo ">>> Installing helm" +curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get | bash + +echo '>>> Installing tiller' +kubectl --namespace kube-system create sa tiller +kubectl create clusterrolebinding tiller-cluster-rule --clusterrole=cluster-admin --serviceaccount=kube-system:tiller +helm init --service-account tiller --upgrade --wait From 0856e13ee60ed614119f829dd359c9fef4baa944 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Sun, 24 Feb 2019 02:35:36 +0200 Subject: [PATCH 11/15] Use kind kubeconfig --- .circleci/config.yml | 2 +- test/e2e-prerequisites.sh | 4 ++++ test/{e2e-kind.sh => e2e-tests.sh} | 8 ++++---- 3 files changed, 9 insertions(+), 5 deletions(-) rename test/{e2e-kind.sh => e2e-tests.sh} (97%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 459c16ea..bd90165d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,7 +5,7 @@ jobs: steps: - checkout - run: test/e2e-prerequisites.sh - - run: test/e2e-kind.sh + - run: test/e2e-tests.sh workflows: version: 2 diff --git a/test/e2e-prerequisites.sh b/test/e2e-prerequisites.sh index fa30860d..6e7996f9 100755 --- a/test/e2e-prerequisites.sh +++ b/test/e2e-prerequisites.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +set -o errexit + REPO_ROOT=$(git rev-parse --show-toplevel) echo ">>> Installing kubectl" @@ -18,6 +20,8 @@ chmod +x kind sudo mv kind /usr/local/bin/ kind create cluster --wait 5m +export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" + echo ">>> Installing helm" curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get | bash diff --git a/test/e2e-kind.sh b/test/e2e-tests.sh similarity index 97% rename from test/e2e-kind.sh rename to test/e2e-tests.sh index eac7ff83..2881b7fc 100755 --- a/test/e2e-kind.sh +++ b/test/e2e-tests.sh @@ -4,11 +4,9 @@ set -o errexit ISTIO_VER="1.1.0-rc.0" REPO_ROOT=$(git rev-parse --show-toplevel) +export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" echo ">>> Starting e2e testing using Istio ${ISTIO_VER}" -export KUBECONFIG="$(kind get kubeconfig-path)" -kubectl version - helm repo add istio.io https://storage.googleapis.com/istio-release/releases/${ISTIO_VER}/charts echo '>>> Installing Istio CRDs' @@ -23,8 +21,10 @@ helm upgrade -i istio istio.io/istio --wait --namespace istio-system -f ${REPO_R export KUBECONFIG="$(kind get kubeconfig-path)" -echo '>>> Installing Flagger' +echo '>>> Building Flagger' cd ${REPO_ROOT} && docker build -t test/flagger:latest . -f Dockerfile + +echo '>>> Installing Flagger' kind load docker-image test/flagger:latest kubectl apply -f ${REPO_ROOT}/artifacts/flagger/ kubectl -n istio-system set image deployment/flagger flagger=test/flagger:latest From f451b4e36c00194feccdac5abff372fb1415ec33 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Sun, 24 Feb 2019 02:52:25 +0200 Subject: [PATCH 12/15] Split e2e prerequisites --- .circleci/config.yml | 3 ++- test/e2e-istio.sh | 29 ++++++++++++++++++++++ test/{e2e-prerequisites.sh => e2e-kind.sh} | 8 +----- test/e2e-tests.sh | 26 ++++++------------- 4 files changed, 40 insertions(+), 26 deletions(-) create mode 100755 test/e2e-istio.sh rename test/{e2e-prerequisites.sh => e2e-kind.sh} (66%) diff --git a/.circleci/config.yml b/.circleci/config.yml index bd90165d..eb4639d0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4,7 +4,8 @@ jobs: machine: true steps: - checkout - - run: test/e2e-prerequisites.sh + - run: test/e2e-kind.sh + - run: test/e2e-istio.sh - run: test/e2e-tests.sh workflows: diff --git a/test/e2e-istio.sh b/test/e2e-istio.sh new file mode 100755 index 00000000..98753e34 --- /dev/null +++ b/test/e2e-istio.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +set -o errexit + +ISTIO_VER="1.1.0-rc.0" +REPO_ROOT=$(git rev-parse --show-toplevel) + +export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" + +echo ">>> Installing Helm" +curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get | bash + +echo '>>> Installing Tiller' +kubectl --namespace kube-system create sa tiller +kubectl create clusterrolebinding tiller-cluster-rule --clusterrole=cluster-admin --serviceaccount=kube-system:tiller +helm init --service-account tiller --upgrade --wait + +echo ">>> Installing Istio ${ISTIO_VER}" +helm repo add istio.io https://storage.googleapis.com/istio-release/releases/${ISTIO_VER}/charts + +echo '>>> Installing Istio CRDs' +helm upgrade -i istio-init istio.io/istio-init --wait --namespace istio-system + +echo '>>> Waiting for Istio CRDs to be ready' +kubectl -n istio-system wait --for=condition=complete job/istio-init-crd-10 +kubectl -n istio-system wait --for=condition=complete job/istio-init-crd-11 + +echo 'Installing Istio control plane' +helm upgrade -i istio istio.io/istio --wait --namespace istio-system -f ${REPO_ROOT}/test/e2e-istio-values.yaml \ No newline at end of file diff --git a/test/e2e-prerequisites.sh b/test/e2e-kind.sh similarity index 66% rename from test/e2e-prerequisites.sh rename to test/e2e-kind.sh index 6e7996f9..2312aebe 100755 --- a/test/e2e-prerequisites.sh +++ b/test/e2e-kind.sh @@ -21,11 +21,5 @@ sudo mv kind /usr/local/bin/ kind create cluster --wait 5m export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" +kubectl get pods --all-namespaces -echo ">>> Installing helm" -curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get | bash - -echo '>>> Installing tiller' -kubectl --namespace kube-system create sa tiller -kubectl create clusterrolebinding tiller-cluster-rule --clusterrole=cluster-admin --serviceaccount=kube-system:tiller -helm init --service-account tiller --upgrade --wait diff --git a/test/e2e-tests.sh b/test/e2e-tests.sh index 2881b7fc..975dd636 100755 --- a/test/e2e-tests.sh +++ b/test/e2e-tests.sh @@ -2,25 +2,9 @@ set -o errexit -ISTIO_VER="1.1.0-rc.0" REPO_ROOT=$(git rev-parse --show-toplevel) export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" -echo ">>> Starting e2e testing using Istio ${ISTIO_VER}" -helm repo add istio.io https://storage.googleapis.com/istio-release/releases/${ISTIO_VER}/charts - -echo '>>> Installing Istio CRDs' -helm upgrade -i istio-init istio.io/istio-init --wait --namespace istio-system - -echo '>>> Waiting for Istio CRDs to be ready' -kubectl -n istio-system wait --for=condition=complete job/istio-init-crd-10 -kubectl -n istio-system wait --for=condition=complete job/istio-init-crd-11 - -echo 'Installing Istio control plane' -helm upgrade -i istio istio.io/istio --wait --namespace istio-system -f ${REPO_ROOT}/test/e2e-istio-values.yaml - -export KUBECONFIG="$(kind get kubeconfig-path)" - echo '>>> Building Flagger' cd ${REPO_ROOT} && docker build -t test/flagger:latest . -f Dockerfile @@ -56,8 +40,8 @@ spec: service: port: 9898 canaryAnalysis: - interval: 10s - threshold: 10 + interval: 15s + threshold: 15 maxWeight: 50 stepWeight: 10 metrics: @@ -85,6 +69,7 @@ until ${ok}; do sleep 5 count=$(($count + 1)) if [[ ${count} -eq ${retries} ]]; then + kubectl -n istio-system logs deployment/flagger echo "No more retries left" exit 1 fi @@ -104,9 +89,14 @@ until ${ok}; do sleep 5 count=$(($count + 1)) if [[ ${count} -eq ${retries} ]]; then + kubectl -n istio-system logs deployment/flagger echo "No more retries left" exit 1 fi done echo '>>> Canary promotion test passed ✔︎' + +kubectl -n istio-system logs deployment/flagger + +echo '>>> All tests passed ✔︎' \ No newline at end of file From ef423b20786a8a19d45fa5487bf312b1a05c06a8 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Sun, 24 Feb 2019 03:10:50 +0200 Subject: [PATCH 13/15] Move Flagger e2e build to a dedicated job --- .circleci/config.yml | 1 + test/e2e-build.sh | 15 +++++++++++++++ test/e2e-istio.sh | 3 +-- test/e2e-tests.sh | 17 +++++------------ test/e2e-workload.yaml | 8 ++++---- 5 files changed, 26 insertions(+), 18 deletions(-) create mode 100755 test/e2e-build.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index eb4639d0..c2304538 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,6 +6,7 @@ jobs: - checkout - run: test/e2e-kind.sh - run: test/e2e-istio.sh + - run: test/e2e-build.sh - run: test/e2e-tests.sh workflows: diff --git a/test/e2e-build.sh b/test/e2e-build.sh new file mode 100755 index 00000000..43dff8d0 --- /dev/null +++ b/test/e2e-build.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) +export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" + +echo '>>> Building Flagger' +cd ${REPO_ROOT} && docker build -t test/flagger:latest . -f Dockerfile + +echo '>>> Installing Flagger' +kind load docker-image test/flagger:latest +kubectl apply -f ${REPO_ROOT}/artifacts/flagger/ +kubectl -n istio-system set image deployment/flagger flagger=test/flagger:latest +kubectl -n istio-system rollout status deployment/flagger diff --git a/test/e2e-istio.sh b/test/e2e-istio.sh index 98753e34..71e2791d 100755 --- a/test/e2e-istio.sh +++ b/test/e2e-istio.sh @@ -4,7 +4,6 @@ set -o errexit ISTIO_VER="1.1.0-rc.0" REPO_ROOT=$(git rev-parse --show-toplevel) - export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" echo ">>> Installing Helm" @@ -25,5 +24,5 @@ echo '>>> Waiting for Istio CRDs to be ready' kubectl -n istio-system wait --for=condition=complete job/istio-init-crd-10 kubectl -n istio-system wait --for=condition=complete job/istio-init-crd-11 -echo 'Installing Istio control plane' +echo '>>> Installing Istio control plane' helm upgrade -i istio istio.io/istio --wait --namespace istio-system -f ${REPO_ROOT}/test/e2e-istio-values.yaml \ No newline at end of file diff --git a/test/e2e-tests.sh b/test/e2e-tests.sh index 975dd636..02589fbd 100755 --- a/test/e2e-tests.sh +++ b/test/e2e-tests.sh @@ -5,15 +5,6 @@ set -o errexit REPO_ROOT=$(git rev-parse --show-toplevel) export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" -echo '>>> Building Flagger' -cd ${REPO_ROOT} && docker build -t test/flagger:latest . -f Dockerfile - -echo '>>> Installing Flagger' -kind load docker-image test/flagger:latest -kubectl apply -f ${REPO_ROOT}/artifacts/flagger/ -kubectl -n istio-system set image deployment/flagger flagger=test/flagger:latest -kubectl -n istio-system rollout status deployment/flagger - echo '>>> Creating test namespace' kubectl create namespace test kubectl label namespace test istio-injection=enabled @@ -56,7 +47,7 @@ spec: url: http://flagger-loadtester.test/ timeout: 5s metadata: - cmd: "hey -z 1m -q 10 -c 2 http://podinfo.test:9898/" + cmd: "hey -z 10m -q 10 -c 2 http://podinfo.test:9898/" EOF @@ -78,17 +69,19 @@ done echo '>>> Canary initialization test passed ✔︎' echo '>>> Triggering canary deployment' -kubectl -n test set image deployment/podinfo podinfod=quay.io/stefanprodan/podinfo:1.4.2 +kubectl -n test set image deployment/podinfo podinfod=quay.io/stefanprodan/podinfo:1.4.1 echo '>>> Waiting for canary promotion' retries=50 count=0 ok=false until ${ok}; do - kubectl -n test describe deployment/podinfo-primary | grep '1.4.2' && ok=true || ok=false + kubectl -n test describe deployment/podinfo-primary | grep '1.4.1' && ok=true || ok=false sleep 5 count=$(($count + 1)) if [[ ${count} -eq ${retries} ]]; then + kubectl -n test describe deployment/podinfo + kubectl -n test describe deployment/podinfo-canary kubectl -n istio-system logs deployment/flagger echo "No more retries left" exit 1 diff --git a/test/e2e-workload.yaml b/test/e2e-workload.yaml index 57ed8a41..83b199d7 100644 --- a/test/e2e-workload.yaml +++ b/test/e2e-workload.yaml @@ -60,8 +60,8 @@ spec: timeoutSeconds: 5 resources: limits: - cpu: 2000m - memory: 512Mi + cpu: 1000m + memory: 128Mi requests: - cpu: 100m - memory: 64Mi + cpu: 1m + memory: 16Mi From ba65975fb540d5040cb6309e709762a09d0184d4 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Sun, 24 Feb 2019 11:41:22 +0200 Subject: [PATCH 14/15] Add e2e testing docs --- test/README.md | 24 ++++++++++++++++++++++++ test/e2e-tests.sh | 12 ++++++++---- 2 files changed, 32 insertions(+), 4 deletions(-) create mode 100644 test/README.md diff --git a/test/README.md b/test/README.md new file mode 100644 index 00000000..b338d303 --- /dev/null +++ b/test/README.md @@ -0,0 +1,24 @@ +# Flagger end-to-end testing + +The e2e testing infrastructure is powered by CircleCI and [Kubernetes Kind](https://github.com/kubernetes-sigs/kind). + +CircleCI e2e workflow: + +* install latest stable kubectl [e2e-kind.sh](e2e-kind.sh) +* build Kubernetes Kind from master [e2e-kind.sh](e2e-kind.sh) +* create local Kubernetes cluster with kind [e2e-kind.sh](e2e-kind.sh) +* install latest stable Helm CLI [e2e-istio.sh](e2e-istio.sh) +* deploy Tiller on the local cluster [e2e-istio.sh](e2e-istio.sh) +* install Istio CRDs with Helm [e2e-istio.sh](e2e-istio.sh) +* install Istio control plane and Prometheus with Helm [e2e-istio.sh](e2e-istio.sh) +* build Flagger container image [e2e-build.sh](e2e-build.sh) +* load Flagger image onto the local cluster [e2e-build.sh](e2e-build.sh) +* deploy Flagger in the istio-system namespace [e2e-build.sh](e2e-build.sh) +* create a test namespace with Istio injection enabled [e2e-tests.sh](e2e-tests.sh) +* deploy the load tester in the test namespace [e2e-tests.sh](e2e-tests.sh) +* deploy a demo workload (podinfo) in the test namespace [e2e-tests.sh](e2e-tests.sh) +* test the canary initialization [e2e-tests.sh](e2e-tests.sh) +* test the canary analysis and promotion [e2e-tests.sh](e2e-tests.sh) + + + diff --git a/test/e2e-tests.sh b/test/e2e-tests.sh index 02589fbd..78edba4b 100755 --- a/test/e2e-tests.sh +++ b/test/e2e-tests.sh @@ -1,5 +1,8 @@ #!/usr/bin/env bash +# This script runs e2e tests for Canary initialization, analysis and promotion +# Prerequisites: Kubernetes Kind, Helm and Istio + set -o errexit REPO_ROOT=$(git rev-parse --show-toplevel) @@ -66,7 +69,7 @@ until ${ok}; do fi done -echo '>>> Canary initialization test passed ✔︎' +echo '✔ Canary initialization test passed' echo '>>> Triggering canary deployment' kubectl -n test set image deployment/podinfo podinfod=quay.io/stefanprodan/podinfo:1.4.1 @@ -78,18 +81,19 @@ ok=false until ${ok}; do kubectl -n test describe deployment/podinfo-primary | grep '1.4.1' && ok=true || ok=false sleep 5 + kubectl -n istio-system logs deployment/flagger --tail 1 count=$(($count + 1)) if [[ ${count} -eq ${retries} ]]; then kubectl -n test describe deployment/podinfo - kubectl -n test describe deployment/podinfo-canary + kubectl -n test describe deployment/podinfo-primary kubectl -n istio-system logs deployment/flagger echo "No more retries left" exit 1 fi done -echo '>>> Canary promotion test passed ✔︎' +echo '✔ Canary promotion test passed' kubectl -n istio-system logs deployment/flagger -echo '>>> All tests passed ✔︎' \ No newline at end of file +echo '✔ All tests passed' \ No newline at end of file From 9d92de234c1dcbff938f9896c5810d59572489fb Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Sun, 24 Feb 2019 11:55:37 +0200 Subject: [PATCH 15/15] Increase promotion e2e wait time to 10s --- docs/gitbook/usage/progressive-delivery.md | 2 +- test/e2e-tests.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/gitbook/usage/progressive-delivery.md b/docs/gitbook/usage/progressive-delivery.md index 8a9dcc56..e1ebf033 100644 --- a/docs/gitbook/usage/progressive-delivery.md +++ b/docs/gitbook/usage/progressive-delivery.md @@ -113,7 +113,7 @@ Trigger a canary deployment by updating the container image: ```bash kubectl -n test set image deployment/podinfo \ -podinfod=quay.io/stefanprodan/podinfo:1.4.0 +podinfod=quay.io/stefanprodan/podinfo:1.4.1 ``` Flagger detects that the deployment revision changed and starts a new rollout: diff --git a/test/e2e-tests.sh b/test/e2e-tests.sh index 78edba4b..958e1a2a 100755 --- a/test/e2e-tests.sh +++ b/test/e2e-tests.sh @@ -80,7 +80,7 @@ count=0 ok=false until ${ok}; do kubectl -n test describe deployment/podinfo-primary | grep '1.4.1' && ok=true || ok=false - sleep 5 + sleep 10 kubectl -n istio-system logs deployment/flagger --tail 1 count=$(($count + 1)) if [[ ${count} -eq ${retries} ]]; then