diff --git a/.circleci/config.yml b/.circleci/config.yml index 7c254c55..f9164539 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,6 +1,6 @@ version: 2.1 jobs: - e2e-testing: + e2e-istio-testing: machine: true steps: - checkout @@ -18,11 +18,20 @@ jobs: - run: test/e2e-build.sh supergloo:test.supergloo-system - run: test/e2e-tests.sh canary + e2e-nginx-testing: + machine: true + steps: + - checkout + - run: test/e2e-kind.sh + - run: test/e2e-nginx.sh + - run: test/e2e-nginx-build.sh + - run: test/e2e-nginx-tests.sh + workflows: version: 2 build-and-test: jobs: - - e2e-testing: + - e2e-istio-testing: filters: branches: ignore: @@ -36,3 +45,10 @@ workflows: - /gh-pages.*/ - /docs-.*/ - /release-.*/ + - e2e-nginx-testing: + filters: + branches: + ignore: + - /gh-pages.*/ + - /docs-.*/ + - /release-.*/ \ No newline at end of file diff --git a/artifacts/nginx/ingress.yaml b/artifacts/nginx/ingress.yaml index 5cb6b826..c5a6fa62 100644 --- a/artifacts/nginx/ingress.yaml +++ b/artifacts/nginx/ingress.yaml @@ -9,7 +9,7 @@ metadata: kubernetes.io/ingress.class: "nginx" spec: rules: - - host: app.exmaple.com + - host: app.example.com http: paths: - backend: diff --git a/docs/gitbook/usage/nginx-progressive-delivery.md b/docs/gitbook/usage/nginx-progressive-delivery.md index 95260a48..4d660bc4 100644 --- a/docs/gitbook/usage/nginx-progressive-delivery.md +++ b/docs/gitbook/usage/nginx-progressive-delivery.md @@ -67,7 +67,7 @@ helm upgrade -i flagger-loadtester flagger/loadtester \ --namespace=test ``` -Create an ingress definition (replace `app.exmaple.com` with your own domain): +Create an ingress definition (replace `app.example.com` with your own domain): ```yaml apiVersion: extensions/v1beta1 @@ -81,7 +81,7 @@ metadata: kubernetes.io/ingress.class: "nginx" spec: rules: - - host: app.exmaple.com + - host: app.example.com http: paths: - backend: @@ -95,7 +95,7 @@ Save the above resource as podinfo-ingress.yaml and then apply it: kubectl apply -f ./podinfo-ingress.yaml ``` -Create a canary custom resource (replace `app.exmaple.com` with your own domain): +Create a canary custom resource (replace `app.example.com` with your own domain): ```yaml apiVersion: flagger.app/v1alpha3 @@ -249,7 +249,7 @@ podinfod=quay.io/stefanprodan/podinfo:1.4.2 Generate HTTP 500 errors: ```bash -watch curl http://app.exmaple.com/status/500 +watch curl http://app.example.com/status/500 ``` When the number of failed checks reaches the canary analysis threshold, the traffic is routed back to the primary, @@ -381,7 +381,7 @@ Edit the canary analysis, remove the max/step weight and add the match condition ``` The above configuration will run an analysis for ten minutes targeting users that have a `canary` cookie set to `always` or -those that call the service using the `X-Canary: always` header. +those that call the service using the `X-Canary: insider` header. Trigger a canary deployment by updating the container image: diff --git a/pkg/router/ingress_test.go b/pkg/router/ingress_test.go new file mode 100644 index 00000000..7d2679e6 --- /dev/null +++ b/pkg/router/ingress_test.go @@ -0,0 +1,112 @@ +package router + +import ( + "fmt" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "testing" +) + +func TestIngressRouter_Reconcile(t *testing.T) { + mocks := setupfakeClients() + router := &IngressRouter{ + logger: mocks.logger, + kubeClient: mocks.kubeClient, + } + + err := router.Reconcile(mocks.ingressCanary) + if err != nil { + t.Fatal(err.Error()) + } + + canaryAn := "nginx.ingress.kubernetes.io/canary" + canaryWeightAn := "nginx.ingress.kubernetes.io/canary-weight" + + canaryName := fmt.Sprintf("%s-canary", mocks.ingressCanary.Spec.IngressRef.Name) + inCanary, err := router.kubeClient.ExtensionsV1beta1().Ingresses("default").Get(canaryName, metav1.GetOptions{}) + if err != nil { + t.Fatal(err.Error()) + } + + if _, ok := inCanary.Annotations[canaryAn]; !ok { + t.Errorf("Canary annotation missing") + } + + // test initialisation + if inCanary.Annotations[canaryAn] != "false" { + t.Errorf("Got canary annotation %v wanted false", inCanary.Annotations[canaryAn]) + } + + if inCanary.Annotations[canaryWeightAn] != "0" { + t.Errorf("Got canary weight annotation %v wanted 0", inCanary.Annotations[canaryWeightAn]) + } +} + +func TestIngressRouter_GetSetRoutes(t *testing.T) { + mocks := setupfakeClients() + router := &IngressRouter{ + logger: mocks.logger, + kubeClient: mocks.kubeClient, + } + + err := router.Reconcile(mocks.ingressCanary) + if err != nil { + t.Fatal(err.Error()) + } + + p, c, err := router.GetRoutes(mocks.ingressCanary) + if err != nil { + t.Fatal(err.Error()) + } + + p = 50 + c = 50 + + err = router.SetRoutes(mocks.ingressCanary, p, c) + if err != nil { + t.Fatal(err.Error()) + } + + canaryAn := "nginx.ingress.kubernetes.io/canary" + canaryWeightAn := "nginx.ingress.kubernetes.io/canary-weight" + + canaryName := fmt.Sprintf("%s-canary", mocks.ingressCanary.Spec.IngressRef.Name) + inCanary, err := router.kubeClient.ExtensionsV1beta1().Ingresses("default").Get(canaryName, metav1.GetOptions{}) + if err != nil { + t.Fatal(err.Error()) + } + + if _, ok := inCanary.Annotations[canaryAn]; !ok { + t.Errorf("Canary annotation missing") + } + + // test rollout + if inCanary.Annotations[canaryAn] != "true" { + t.Errorf("Got canary annotation %v wanted true", inCanary.Annotations[canaryAn]) + } + + if inCanary.Annotations[canaryWeightAn] != "50" { + t.Errorf("Got canary weight annotation %v wanted 50", inCanary.Annotations[canaryWeightAn]) + } + + p = 100 + c = 0 + + err = router.SetRoutes(mocks.ingressCanary, p, c) + if err != nil { + t.Fatal(err.Error()) + } + + inCanary, err = router.kubeClient.ExtensionsV1beta1().Ingresses("default").Get(canaryName, metav1.GetOptions{}) + if err != nil { + t.Fatal(err.Error()) + } + + // test promotion + if inCanary.Annotations[canaryAn] != "false" { + t.Errorf("Got canary annotation %v wanted false", inCanary.Annotations[canaryAn]) + } + + if inCanary.Annotations[canaryWeightAn] != "0" { + t.Errorf("Got canary weight annotation %v wanted 0", inCanary.Annotations[canaryWeightAn]) + } +} diff --git a/pkg/router/router_test.go b/pkg/router/router_test.go index 32c5f770..701cae0c 100644 --- a/pkg/router/router_test.go +++ b/pkg/router/router_test.go @@ -11,7 +11,9 @@ import ( appsv1 "k8s.io/api/apps/v1" hpav1 "k8s.io/api/autoscaling/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/api/extensions/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" ) @@ -20,6 +22,7 @@ type fakeClients struct { canary *v1alpha3.Canary abtest *v1alpha3.Canary appmeshCanary *v1alpha3.Canary + ingressCanary *v1alpha3.Canary kubeClient kubernetes.Interface meshClient clientset.Interface flaggerClient clientset.Interface @@ -30,9 +33,10 @@ func setupfakeClients() fakeClients { canary := newMockCanary() abtest := newMockABTest() appmeshCanary := newMockCanaryAppMesh() - flaggerClient := fakeFlagger.NewSimpleClientset(canary, abtest, appmeshCanary) + ingressCanary := newMockCanaryIngress() + flaggerClient := fakeFlagger.NewSimpleClientset(canary, abtest, appmeshCanary, ingressCanary) - kubeClient := fake.NewSimpleClientset(newMockDeployment(), newMockABTestDeployment()) + kubeClient := fake.NewSimpleClientset(newMockDeployment(), newMockABTestDeployment(), newMockIngress()) meshClient := fakeFlagger.NewSimpleClientset() logger, _ := logger.NewLogger("debug") @@ -41,6 +45,7 @@ func setupfakeClients() fakeClients { canary: canary, abtest: abtest, appmeshCanary: appmeshCanary, + ingressCanary: ingressCanary, kubeClient: kubeClient, meshClient: meshClient, flaggerClient: flaggerClient, @@ -266,3 +271,73 @@ func newMockABTestDeployment() *appsv1.Deployment { return d } + +func newMockCanaryIngress() *v1alpha3.Canary { + cd := &v1alpha3.Canary{ + TypeMeta: metav1.TypeMeta{APIVersion: v1alpha3.SchemeGroupVersion.String()}, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "nginx", + }, + Spec: v1alpha3.CanarySpec{ + TargetRef: hpav1.CrossVersionObjectReference{ + Name: "podinfo", + APIVersion: "apps/v1", + Kind: "Deployment", + }, + IngressRef: &hpav1.CrossVersionObjectReference{ + Name: "podinfo", + APIVersion: "extensions/v1beta1", + Kind: "Ingress", + }, + Service: v1alpha3.CanaryService{ + Port: 9898, + }, CanaryAnalysis: v1alpha3.CanaryAnalysis{ + Threshold: 10, + StepWeight: 10, + MaxWeight: 50, + Metrics: []v1alpha3.CanaryMetric{ + { + Name: "request-success-rate", + Threshold: 99, + Interval: "1m", + }, + }, + }, + }, + } + return cd +} + +func newMockIngress() *v1beta1.Ingress { + return &v1beta1.Ingress{ + TypeMeta: metav1.TypeMeta{APIVersion: v1beta1.SchemeGroupVersion.String()}, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "podinfo", + Annotations: map[string]string{ + "kubernetes.io/ingress.class": "nginx", + }, + }, + Spec: v1beta1.IngressSpec{ + Rules: []v1beta1.IngressRule{ + { + Host: "app.example.com", + IngressRuleValue: v1beta1.IngressRuleValue{ + HTTP: &v1beta1.HTTPIngressRuleValue{ + Paths: []v1beta1.HTTPIngressPath{ + { + Path: "/", + Backend: v1beta1.IngressBackend{ + ServiceName: "podinfo", + ServicePort: intstr.FromInt(9898), + }, + }, + }, + }, + }, + }, + }, + }, + } +} diff --git a/test/Dockerfile.kind b/test/Dockerfile.kind deleted file mode 100644 index 28f5e083..00000000 --- a/test/Dockerfile.kind +++ /dev/null @@ -1,4 +0,0 @@ -FROM golang:1.11 - -RUN go get -u sigs.k8s.io/kind - diff --git a/test/README.md b/test/README.md index 3f8d586b..43c04b0a 100644 --- a/test/README.md +++ b/test/README.md @@ -2,7 +2,7 @@ The e2e testing infrastructure is powered by CircleCI and [Kubernetes Kind](https://github.com/kubernetes-sigs/kind). -CircleCI e2e workflow: +### CircleCI e2e Istio workflow * install latest stable kubectl [e2e-kind.sh](e2e-kind.sh) * install Kubernetes Kind [e2e-kind.sh](e2e-kind.sh) @@ -21,4 +21,20 @@ CircleCI e2e workflow: * test the canary analysis and promotion using weighted traffic and the load testing webhook [e2e-tests.sh](e2e-tests.sh) * test the A/B testing analysis and promotion using cookies filters and pre/post rollout webhooks [e2e-tests.sh](e2e-tests.sh) +### CircleCI e2e NGINX ingress workflow +* install latest stable kubectl [e2e-kind.sh](e2e-kind.sh) +* install Kubernetes Kind [e2e-kind.sh](e2e-kind.sh) +* create local Kubernetes cluster with kind [e2e-kind.sh](e2e-kind.sh) +* install latest stable Helm CLI [e2e-nginx.sh](e2e-istio.sh) +* deploy Tiller on the local cluster [e2e-nginx.sh](e2e-istio.sh) +* install NGINX ingress with Helm [e2e-nginx.sh](e2e-istio.sh) +* build Flagger container image [e2e-nginx-build.sh](e2e-build.sh) +* load Flagger image onto the local cluster [e2e-nginx-build.sh](e2e-build.sh) +* install Flagger and Prometheus in the ingress-nginx namespace [e2e-nginx-build.sh](e2e-build.sh) +* create a test namespace [e2e-nginx-tests.sh](e2e-tests.sh) +* deploy the load tester in the test namespace [e2e-nginx-tests.sh](e2e-tests.sh) +* deploy the demo workload (podinfo) and ingress in the test namespace [e2e-nginx-tests.sh](e2e-tests.sh) +* test the canary initialization [e2e-nginx-tests.sh](e2e-tests.sh) +* test the canary analysis and promotion using weighted traffic and the load testing webhook [e2e-nginx-tests.sh](e2e-tests.sh) +* test the A/B testing analysis and promotion using header filters and pre/post rollout webhooks [e2e-nginx-tests.sh](e2e-tests.sh) diff --git a/test/e2e-ingress.yaml b/test/e2e-ingress.yaml new file mode 100644 index 00000000..c5a6fa62 --- /dev/null +++ b/test/e2e-ingress.yaml @@ -0,0 +1,17 @@ +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: podinfo + namespace: test + labels: + app: podinfo + annotations: + kubernetes.io/ingress.class: "nginx" +spec: + rules: + - host: app.example.com + http: + paths: + - backend: + serviceName: podinfo + servicePort: 9898 diff --git a/test/e2e-nginx-build.sh b/test/e2e-nginx-build.sh new file mode 100755 index 00000000..6ae5449a --- /dev/null +++ b/test/e2e-nginx-build.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) +export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" + +echo '>>> Building Flagger' +cd ${REPO_ROOT} && docker build -t test/flagger:latest . -f Dockerfile + +echo '>>> Installing Flagger' +kind load docker-image test/flagger:latest + +echo '>>> Installing Flagger' +helm upgrade -i flagger ${REPO_ROOT}/charts/flagger \ +--wait \ +--namespace ingress-nginx \ +--set prometheus.install=true \ +--set meshProvider=nginx + +kubectl -n ingress-nginx set image deployment/flagger flagger=test/flagger:latest + +kubectl -n ingress-nginx rollout status deployment/flagger +kubectl -n ingress-nginx rollout status deployment/flagger-prometheus diff --git a/test/e2e-nginx-tests.sh b/test/e2e-nginx-tests.sh new file mode 100755 index 00000000..dac550f7 --- /dev/null +++ b/test/e2e-nginx-tests.sh @@ -0,0 +1,194 @@ +#!/usr/bin/env bash + +# This script runs e2e tests for Canary initialization, analysis and promotion +# Prerequisites: Kubernetes Kind, Helm and NGINX ingress controller + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) +export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" + +echo '>>> Creating test namespace' +kubectl create namespace test + +echo '>>> Installing load tester' +kubectl -n test apply -f ${REPO_ROOT}/artifacts/loadtester/ +kubectl -n test rollout status deployment/flagger-loadtester + +echo '>>> Initialising canary' +kubectl apply -f ${REPO_ROOT}/test/e2e-workload.yaml +kubectl apply -f ${REPO_ROOT}/test/e2e-ingress.yaml + +cat <>> Waiting for primary to be ready' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n ingress-nginx logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary initialization test passed' + +echo '>>> Triggering canary deployment' +kubectl -n test set image deployment/podinfo podinfod=quay.io/stefanprodan/podinfo:1.4.1 + +echo '>>> Waiting for canary promotion' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test describe deployment/podinfo-primary | grep '1.4.1' && ok=true || ok=false + sleep 10 + kubectl -n ingress-nginx logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n test describe deployment/podinfo + kubectl -n test describe deployment/podinfo-primary + kubectl -n ingress-nginx logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary promotion test passed' + +if [ "$1" = "canary" ]; then + exit 0 +fi + +cat <>> Triggering A/B testing' +kubectl -n test set image deployment/podinfo podinfod=quay.io/stefanprodan/podinfo:1.4.2 + +echo '>>> Waiting for A/B testing promotion' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test describe deployment/podinfo-primary | grep '1.4.2' && ok=true || ok=false + sleep 10 + kubectl -n ingress-nginx logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n test describe deployment/podinfo + kubectl -n test describe deployment/podinfo-primary + kubectl -n ingress-nginx logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ A/B testing promotion test passed' + +kubectl -n ingress-nginx logs deployment/flagger + +echo '✔ All tests passed' \ No newline at end of file diff --git a/test/e2e-nginx.sh b/test/e2e-nginx.sh new file mode 100755 index 00000000..3fa97bb7 --- /dev/null +++ b/test/e2e-nginx.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) +export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" + +echo ">>> Installing Helm" +curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get | bash + +echo '>>> Installing Tiller' +kubectl --namespace kube-system create sa tiller +kubectl create clusterrolebinding tiller-cluster-rule --clusterrole=cluster-admin --serviceaccount=kube-system:tiller +helm init --service-account tiller --upgrade --wait + +echo '>>> Installing NGINX Ingress' +helm upgrade -i nginx-ingress stable/nginx-ingress \ +--wait \ +--namespace ingress-nginx \ +--set controller.stats.enabled=true \ +--set controller.metrics.enabled=true \ +--set controller.podAnnotations."prometheus\.io/scrape"=true \ +--set controller.podAnnotations."prometheus\.io/port"=10254 \ +--set controller.service.type=NodePort + +kubectl -n ingress-nginx rollout status deployment/nginx-ingress-controller +kubectl -n ingress-nginx get all + +