From 344bd45a0e08bcd864da405e8c994de24941282b Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Fri, 10 May 2019 10:24:35 +0300 Subject: [PATCH 1/7] Add nginx e2e tests --- .circleci/config.yml | 20 ++++- test/e2e-ingress.yaml | 17 ++++ test/e2e-nginx-build.sh | 24 ++++++ test/e2e-nginx-tests.sh | 185 ++++++++++++++++++++++++++++++++++++++++ test/e2e-nginx.sh | 29 +++++++ 5 files changed, 273 insertions(+), 2 deletions(-) create mode 100644 test/e2e-ingress.yaml create mode 100755 test/e2e-nginx-build.sh create mode 100755 test/e2e-nginx-tests.sh create mode 100755 test/e2e-nginx.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 7c254c55..f9164539 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,6 +1,6 @@ version: 2.1 jobs: - e2e-testing: + e2e-istio-testing: machine: true steps: - checkout @@ -18,11 +18,20 @@ jobs: - run: test/e2e-build.sh supergloo:test.supergloo-system - run: test/e2e-tests.sh canary + e2e-nginx-testing: + machine: true + steps: + - checkout + - run: test/e2e-kind.sh + - run: test/e2e-nginx.sh + - run: test/e2e-nginx-build.sh + - run: test/e2e-nginx-tests.sh + workflows: version: 2 build-and-test: jobs: - - e2e-testing: + - e2e-istio-testing: filters: branches: ignore: @@ -36,3 +45,10 @@ workflows: - /gh-pages.*/ - /docs-.*/ - /release-.*/ + - e2e-nginx-testing: + filters: + branches: + ignore: + - /gh-pages.*/ + - /docs-.*/ + - /release-.*/ \ No newline at end of file diff --git a/test/e2e-ingress.yaml b/test/e2e-ingress.yaml new file mode 100644 index 00000000..c5a6fa62 --- /dev/null +++ b/test/e2e-ingress.yaml @@ -0,0 +1,17 @@ +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: podinfo + namespace: test + labels: + app: podinfo + annotations: + kubernetes.io/ingress.class: "nginx" +spec: + rules: + - host: app.example.com + http: + paths: + - backend: + serviceName: podinfo + servicePort: 9898 diff --git a/test/e2e-nginx-build.sh b/test/e2e-nginx-build.sh new file mode 100755 index 00000000..6ae5449a --- /dev/null +++ b/test/e2e-nginx-build.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) +export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" + +echo '>>> Building Flagger' +cd ${REPO_ROOT} && docker build -t test/flagger:latest . -f Dockerfile + +echo '>>> Installing Flagger' +kind load docker-image test/flagger:latest + +echo '>>> Installing Flagger' +helm upgrade -i flagger ${REPO_ROOT}/charts/flagger \ +--wait \ +--namespace ingress-nginx \ +--set prometheus.install=true \ +--set meshProvider=nginx + +kubectl -n ingress-nginx set image deployment/flagger flagger=test/flagger:latest + +kubectl -n ingress-nginx rollout status deployment/flagger +kubectl -n ingress-nginx rollout status deployment/flagger-prometheus diff --git a/test/e2e-nginx-tests.sh b/test/e2e-nginx-tests.sh new file mode 100755 index 00000000..a4064e79 --- /dev/null +++ b/test/e2e-nginx-tests.sh @@ -0,0 +1,185 @@ +#!/usr/bin/env bash + +# This script runs e2e tests for Canary initialization, analysis and promotion +# Prerequisites: Kubernetes Kind, Helm and Istio + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) +export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" + +echo '>>> Creating test namespace' +kubectl create namespace test + +echo '>>> Installing load tester' +kubectl -n test apply -f ${REPO_ROOT}/artifacts/loadtester/ +kubectl -n test rollout status deployment/flagger-loadtester + +echo '>>> Initialising canary' +kubectl apply -f ${REPO_ROOT}/test/e2e-workload.yaml + +cat <>> Waiting for primary to be ready' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n ingress-nginx logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary initialization test passed' + +echo '>>> Triggering canary deployment' +kubectl -n test set image deployment/podinfo podinfod=quay.io/stefanprodan/podinfo:1.4.1 + +echo '>>> Waiting for canary promotion' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test describe deployment/podinfo-primary | grep '1.4.1' && ok=true || ok=false + sleep 10 + kubectl -n ingress-nginx logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n test describe deployment/podinfo + kubectl -n test describe deployment/podinfo-primary + kubectl -n ingress-nginx logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary promotion test passed' + +if [ "$1" = "canary" ]; then + exit 0 +fi + +cat <>> Triggering A/B testing' +kubectl -n test set image deployment/podinfo podinfod=quay.io/stefanprodan/podinfo:1.4.2 + +echo '>>> Waiting for A/B testing promotion' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test describe deployment/podinfo-primary | grep '1.4.2' && ok=true || ok=false + sleep 10 + kubectl -n ingress-nginx logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n test describe deployment/podinfo + kubectl -n test describe deployment/podinfo-primary + kubectl -n ingress-nginx logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ A/B testing promotion test passed' + +kubectl -n ingress-nginx logs deployment/flagger + +echo '✔ All tests passed' \ No newline at end of file diff --git a/test/e2e-nginx.sh b/test/e2e-nginx.sh new file mode 100755 index 00000000..3fa97bb7 --- /dev/null +++ b/test/e2e-nginx.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) +export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" + +echo ">>> Installing Helm" +curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get | bash + +echo '>>> Installing Tiller' +kubectl --namespace kube-system create sa tiller +kubectl create clusterrolebinding tiller-cluster-rule --clusterrole=cluster-admin --serviceaccount=kube-system:tiller +helm init --service-account tiller --upgrade --wait + +echo '>>> Installing NGINX Ingress' +helm upgrade -i nginx-ingress stable/nginx-ingress \ +--wait \ +--namespace ingress-nginx \ +--set controller.stats.enabled=true \ +--set controller.metrics.enabled=true \ +--set controller.podAnnotations."prometheus\.io/scrape"=true \ +--set controller.podAnnotations."prometheus\.io/port"=10254 \ +--set controller.service.type=NodePort + +kubectl -n ingress-nginx rollout status deployment/nginx-ingress-controller +kubectl -n ingress-nginx get all + + From bc84e1c154792423557a0bb21925b80250ad233e Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Fri, 10 May 2019 10:24:47 +0300 Subject: [PATCH 2/7] Fix typos --- artifacts/nginx/ingress.yaml | 2 +- docs/gitbook/usage/nginx-progressive-delivery.md | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/artifacts/nginx/ingress.yaml b/artifacts/nginx/ingress.yaml index 5cb6b826..c5a6fa62 100644 --- a/artifacts/nginx/ingress.yaml +++ b/artifacts/nginx/ingress.yaml @@ -9,7 +9,7 @@ metadata: kubernetes.io/ingress.class: "nginx" spec: rules: - - host: app.exmaple.com + - host: app.example.com http: paths: - backend: diff --git a/docs/gitbook/usage/nginx-progressive-delivery.md b/docs/gitbook/usage/nginx-progressive-delivery.md index 95260a48..4d660bc4 100644 --- a/docs/gitbook/usage/nginx-progressive-delivery.md +++ b/docs/gitbook/usage/nginx-progressive-delivery.md @@ -67,7 +67,7 @@ helm upgrade -i flagger-loadtester flagger/loadtester \ --namespace=test ``` -Create an ingress definition (replace `app.exmaple.com` with your own domain): +Create an ingress definition (replace `app.example.com` with your own domain): ```yaml apiVersion: extensions/v1beta1 @@ -81,7 +81,7 @@ metadata: kubernetes.io/ingress.class: "nginx" spec: rules: - - host: app.exmaple.com + - host: app.example.com http: paths: - backend: @@ -95,7 +95,7 @@ Save the above resource as podinfo-ingress.yaml and then apply it: kubectl apply -f ./podinfo-ingress.yaml ``` -Create a canary custom resource (replace `app.exmaple.com` with your own domain): +Create a canary custom resource (replace `app.example.com` with your own domain): ```yaml apiVersion: flagger.app/v1alpha3 @@ -249,7 +249,7 @@ podinfod=quay.io/stefanprodan/podinfo:1.4.2 Generate HTTP 500 errors: ```bash -watch curl http://app.exmaple.com/status/500 +watch curl http://app.example.com/status/500 ``` When the number of failed checks reaches the canary analysis threshold, the traffic is routed back to the primary, @@ -381,7 +381,7 @@ Edit the canary analysis, remove the max/step weight and add the match condition ``` The above configuration will run an analysis for ten minutes targeting users that have a `canary` cookie set to `always` or -those that call the service using the `X-Canary: always` header. +those that call the service using the `X-Canary: insider` header. Trigger a canary deployment by updating the container image: From cbe72f0aa2ce6f5af9350dea385d42796684678a Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Fri, 10 May 2019 10:29:09 +0300 Subject: [PATCH 3/7] Add ingress target to nginx e2e tests --- test/e2e-nginx-tests.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/e2e-nginx-tests.sh b/test/e2e-nginx-tests.sh index a4064e79..c26f3a4b 100755 --- a/test/e2e-nginx-tests.sh +++ b/test/e2e-nginx-tests.sh @@ -29,6 +29,10 @@ spec: apiVersion: apps/v1 kind: Deployment name: podinfo + ingressRef: + apiVersion: extensions/v1beta1 + kind: Ingress + name: podinfo progressDeadlineSeconds: 60 service: port: 9898 @@ -120,6 +124,10 @@ spec: apiVersion: apps/v1 kind: Deployment name: podinfo + ingressRef: + apiVersion: extensions/v1beta1 + kind: Ingress + name: podinfo progressDeadlineSeconds: 60 service: port: 9898 From e308678ed5bb8336c498f0bae6ad77cb25a7059e Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Fri, 10 May 2019 10:40:38 +0300 Subject: [PATCH 4/7] Deploy ingress for nginx e2e tests --- test/e2e-nginx-tests.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/test/e2e-nginx-tests.sh b/test/e2e-nginx-tests.sh index c26f3a4b..a53ee5ba 100755 --- a/test/e2e-nginx-tests.sh +++ b/test/e2e-nginx-tests.sh @@ -17,6 +17,7 @@ kubectl -n test rollout status deployment/flagger-loadtester echo '>>> Initialising canary' kubectl apply -f ${REPO_ROOT}/test/e2e-workload.yaml +kubectl apply -f ${REPO_ROOT}/test/e2e-ingress.yaml cat < Date: Fri, 10 May 2019 10:50:24 +0300 Subject: [PATCH 5/7] Document the nginx e2e tests --- test/Dockerfile.kind | 4 ---- test/README.md | 18 +++++++++++++++++- test/e2e-nginx-tests.sh | 2 +- 3 files changed, 18 insertions(+), 6 deletions(-) delete mode 100644 test/Dockerfile.kind diff --git a/test/Dockerfile.kind b/test/Dockerfile.kind deleted file mode 100644 index 28f5e083..00000000 --- a/test/Dockerfile.kind +++ /dev/null @@ -1,4 +0,0 @@ -FROM golang:1.11 - -RUN go get -u sigs.k8s.io/kind - diff --git a/test/README.md b/test/README.md index 3f8d586b..43c04b0a 100644 --- a/test/README.md +++ b/test/README.md @@ -2,7 +2,7 @@ The e2e testing infrastructure is powered by CircleCI and [Kubernetes Kind](https://github.com/kubernetes-sigs/kind). -CircleCI e2e workflow: +### CircleCI e2e Istio workflow * install latest stable kubectl [e2e-kind.sh](e2e-kind.sh) * install Kubernetes Kind [e2e-kind.sh](e2e-kind.sh) @@ -21,4 +21,20 @@ CircleCI e2e workflow: * test the canary analysis and promotion using weighted traffic and the load testing webhook [e2e-tests.sh](e2e-tests.sh) * test the A/B testing analysis and promotion using cookies filters and pre/post rollout webhooks [e2e-tests.sh](e2e-tests.sh) +### CircleCI e2e NGINX ingress workflow +* install latest stable kubectl [e2e-kind.sh](e2e-kind.sh) +* install Kubernetes Kind [e2e-kind.sh](e2e-kind.sh) +* create local Kubernetes cluster with kind [e2e-kind.sh](e2e-kind.sh) +* install latest stable Helm CLI [e2e-nginx.sh](e2e-istio.sh) +* deploy Tiller on the local cluster [e2e-nginx.sh](e2e-istio.sh) +* install NGINX ingress with Helm [e2e-nginx.sh](e2e-istio.sh) +* build Flagger container image [e2e-nginx-build.sh](e2e-build.sh) +* load Flagger image onto the local cluster [e2e-nginx-build.sh](e2e-build.sh) +* install Flagger and Prometheus in the ingress-nginx namespace [e2e-nginx-build.sh](e2e-build.sh) +* create a test namespace [e2e-nginx-tests.sh](e2e-tests.sh) +* deploy the load tester in the test namespace [e2e-nginx-tests.sh](e2e-tests.sh) +* deploy the demo workload (podinfo) and ingress in the test namespace [e2e-nginx-tests.sh](e2e-tests.sh) +* test the canary initialization [e2e-nginx-tests.sh](e2e-tests.sh) +* test the canary analysis and promotion using weighted traffic and the load testing webhook [e2e-nginx-tests.sh](e2e-tests.sh) +* test the A/B testing analysis and promotion using header filters and pre/post rollout webhooks [e2e-nginx-tests.sh](e2e-tests.sh) diff --git a/test/e2e-nginx-tests.sh b/test/e2e-nginx-tests.sh index a53ee5ba..dac550f7 100755 --- a/test/e2e-nginx-tests.sh +++ b/test/e2e-nginx-tests.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # This script runs e2e tests for Canary initialization, analysis and promotion -# Prerequisites: Kubernetes Kind, Helm and Istio +# Prerequisites: Kubernetes Kind, Helm and NGINX ingress controller set -o errexit From eadce34d6f75edfe8c82e4627171417982997546 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Fri, 10 May 2019 11:39:52 +0300 Subject: [PATCH 6/7] Add ingress router unit tests --- pkg/router/ingress_test.go | 90 ++++++++++++++++++++++++++++++++++++++ pkg/router/router_test.go | 79 ++++++++++++++++++++++++++++++++- 2 files changed, 167 insertions(+), 2 deletions(-) create mode 100644 pkg/router/ingress_test.go diff --git a/pkg/router/ingress_test.go b/pkg/router/ingress_test.go new file mode 100644 index 00000000..1fda7cd5 --- /dev/null +++ b/pkg/router/ingress_test.go @@ -0,0 +1,90 @@ +package router + +import ( + "fmt" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "testing" +) + +func TestIngressRouter_Reconcile(t *testing.T) { + mocks := setupfakeClients() + router := &IngressRouter{ + logger: mocks.logger, + kubeClient: mocks.kubeClient, + } + + err := router.Reconcile(mocks.ingressCanary) + if err != nil { + t.Fatal(err.Error()) + } + + canaryAn := "nginx.ingress.kubernetes.io/canary" + canaryWeightAn := "nginx.ingress.kubernetes.io/canary-weight" + + canaryName := fmt.Sprintf("%s-canary", mocks.ingressCanary.Spec.IngressRef.Name) + inCanary, err := router.kubeClient.ExtensionsV1beta1().Ingresses("default").Get(canaryName, metav1.GetOptions{}) + if err != nil { + t.Fatal(err.Error()) + } + + if _, ok := inCanary.Annotations[canaryAn]; !ok { + t.Errorf("Canary annotation missing") + } + + // test initialisation + if inCanary.Annotations[canaryAn] != "false" { + t.Errorf("Got canary annotation %v wanted false", inCanary.Annotations[canaryAn]) + } + + if inCanary.Annotations[canaryWeightAn] != "0" { + t.Errorf("Got canary weight annotation %v wanted 0", inCanary.Annotations[canaryWeightAn]) + } +} + +func TestIngressRouter_GetSetRoutes(t *testing.T) { + mocks := setupfakeClients() + router := &IngressRouter{ + logger: mocks.logger, + kubeClient: mocks.kubeClient, + } + + err := router.Reconcile(mocks.ingressCanary) + if err != nil { + t.Fatal(err.Error()) + } + + p, c, err := router.GetRoutes(mocks.ingressCanary) + if err != nil { + t.Fatal(err.Error()) + } + + p = 50 + c = 50 + + err = router.SetRoutes(mocks.ingressCanary, p, c) + if err != nil { + t.Fatal(err.Error()) + } + + canaryAn := "nginx.ingress.kubernetes.io/canary" + canaryWeightAn := "nginx.ingress.kubernetes.io/canary-weight" + + canaryName := fmt.Sprintf("%s-canary", mocks.ingressCanary.Spec.IngressRef.Name) + inCanary, err := router.kubeClient.ExtensionsV1beta1().Ingresses("default").Get(canaryName, metav1.GetOptions{}) + if err != nil { + t.Fatal(err.Error()) + } + + if _, ok := inCanary.Annotations[canaryAn]; !ok { + t.Errorf("Canary annotation missing") + } + + // test initialisation + if inCanary.Annotations[canaryAn] != "true" { + t.Errorf("Got canary annotation %v wanted true", inCanary.Annotations[canaryAn]) + } + + if inCanary.Annotations[canaryWeightAn] != "50" { + t.Errorf("Got canary weight annotation %v wanted 50", inCanary.Annotations[canaryWeightAn]) + } +} diff --git a/pkg/router/router_test.go b/pkg/router/router_test.go index 32c5f770..701cae0c 100644 --- a/pkg/router/router_test.go +++ b/pkg/router/router_test.go @@ -11,7 +11,9 @@ import ( appsv1 "k8s.io/api/apps/v1" hpav1 "k8s.io/api/autoscaling/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/api/extensions/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" ) @@ -20,6 +22,7 @@ type fakeClients struct { canary *v1alpha3.Canary abtest *v1alpha3.Canary appmeshCanary *v1alpha3.Canary + ingressCanary *v1alpha3.Canary kubeClient kubernetes.Interface meshClient clientset.Interface flaggerClient clientset.Interface @@ -30,9 +33,10 @@ func setupfakeClients() fakeClients { canary := newMockCanary() abtest := newMockABTest() appmeshCanary := newMockCanaryAppMesh() - flaggerClient := fakeFlagger.NewSimpleClientset(canary, abtest, appmeshCanary) + ingressCanary := newMockCanaryIngress() + flaggerClient := fakeFlagger.NewSimpleClientset(canary, abtest, appmeshCanary, ingressCanary) - kubeClient := fake.NewSimpleClientset(newMockDeployment(), newMockABTestDeployment()) + kubeClient := fake.NewSimpleClientset(newMockDeployment(), newMockABTestDeployment(), newMockIngress()) meshClient := fakeFlagger.NewSimpleClientset() logger, _ := logger.NewLogger("debug") @@ -41,6 +45,7 @@ func setupfakeClients() fakeClients { canary: canary, abtest: abtest, appmeshCanary: appmeshCanary, + ingressCanary: ingressCanary, kubeClient: kubeClient, meshClient: meshClient, flaggerClient: flaggerClient, @@ -266,3 +271,73 @@ func newMockABTestDeployment() *appsv1.Deployment { return d } + +func newMockCanaryIngress() *v1alpha3.Canary { + cd := &v1alpha3.Canary{ + TypeMeta: metav1.TypeMeta{APIVersion: v1alpha3.SchemeGroupVersion.String()}, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "nginx", + }, + Spec: v1alpha3.CanarySpec{ + TargetRef: hpav1.CrossVersionObjectReference{ + Name: "podinfo", + APIVersion: "apps/v1", + Kind: "Deployment", + }, + IngressRef: &hpav1.CrossVersionObjectReference{ + Name: "podinfo", + APIVersion: "extensions/v1beta1", + Kind: "Ingress", + }, + Service: v1alpha3.CanaryService{ + Port: 9898, + }, CanaryAnalysis: v1alpha3.CanaryAnalysis{ + Threshold: 10, + StepWeight: 10, + MaxWeight: 50, + Metrics: []v1alpha3.CanaryMetric{ + { + Name: "request-success-rate", + Threshold: 99, + Interval: "1m", + }, + }, + }, + }, + } + return cd +} + +func newMockIngress() *v1beta1.Ingress { + return &v1beta1.Ingress{ + TypeMeta: metav1.TypeMeta{APIVersion: v1beta1.SchemeGroupVersion.String()}, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "podinfo", + Annotations: map[string]string{ + "kubernetes.io/ingress.class": "nginx", + }, + }, + Spec: v1beta1.IngressSpec{ + Rules: []v1beta1.IngressRule{ + { + Host: "app.example.com", + IngressRuleValue: v1beta1.IngressRuleValue{ + HTTP: &v1beta1.HTTPIngressRuleValue{ + Paths: []v1beta1.HTTPIngressPath{ + { + Path: "/", + Backend: v1beta1.IngressBackend{ + ServiceName: "podinfo", + ServicePort: intstr.FromInt(9898), + }, + }, + }, + }, + }, + }, + }, + }, + } +} From 752eceed4bb8a0d0eee9e211a1dbb27190c671e9 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Fri, 10 May 2019 11:53:12 +0300 Subject: [PATCH 7/7] Add tests for ingress weight changes --- pkg/router/ingress_test.go | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/pkg/router/ingress_test.go b/pkg/router/ingress_test.go index 1fda7cd5..7d2679e6 100644 --- a/pkg/router/ingress_test.go +++ b/pkg/router/ingress_test.go @@ -79,7 +79,7 @@ func TestIngressRouter_GetSetRoutes(t *testing.T) { t.Errorf("Canary annotation missing") } - // test initialisation + // test rollout if inCanary.Annotations[canaryAn] != "true" { t.Errorf("Got canary annotation %v wanted true", inCanary.Annotations[canaryAn]) } @@ -87,4 +87,26 @@ func TestIngressRouter_GetSetRoutes(t *testing.T) { if inCanary.Annotations[canaryWeightAn] != "50" { t.Errorf("Got canary weight annotation %v wanted 50", inCanary.Annotations[canaryWeightAn]) } + + p = 100 + c = 0 + + err = router.SetRoutes(mocks.ingressCanary, p, c) + if err != nil { + t.Fatal(err.Error()) + } + + inCanary, err = router.kubeClient.ExtensionsV1beta1().Ingresses("default").Get(canaryName, metav1.GetOptions{}) + if err != nil { + t.Fatal(err.Error()) + } + + // test promotion + if inCanary.Annotations[canaryAn] != "false" { + t.Errorf("Got canary annotation %v wanted false", inCanary.Annotations[canaryAn]) + } + + if inCanary.Annotations[canaryWeightAn] != "0" { + t.Errorf("Got canary weight annotation %v wanted 0", inCanary.Annotations[canaryWeightAn]) + } }