diff --git a/.circleci/config.yml b/.circleci/config.yml index 783c558b..dc4aac5c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -132,6 +132,9 @@ jobs: - run: test/e2e-kind.sh - run: test/e2e-nginx.sh - run: test/e2e-nginx-tests.sh + - run: test/e2e-nginx-cleanup.sh + - run: test/e2e-nginx-custom-annotations.sh + - run: test/e2e-nginx-tests.sh e2e-linkerd-testing: machine: true diff --git a/charts/flagger/README.md b/charts/flagger/README.md index e5e9ba7a..9806445c 100644 --- a/charts/flagger/README.md +++ b/charts/flagger/README.md @@ -74,6 +74,7 @@ Parameter | Description | Default `msteams.url` | Microsoft Teams incoming webhook | None `leaderElection.enabled` | leader election must be enabled when running more than one replica | `false` `leaderElection.replicaCount` | number of replicas | `1` +`ingressAnnotationsPrefix` | annotations prefix for ingresses | `custom.ingress.kubernetes.io` `rbac.create` | if `true`, create and use RBAC resources | `true` `rbac.pspEnabled` | If `true`, create and use a restricted pod security policy | `false` `crd.create` | if `true`, create Flagger's CRDs | `true` diff --git a/charts/flagger/templates/deployment.yaml b/charts/flagger/templates/deployment.yaml index 13b87e50..25ab4e4c 100644 --- a/charts/flagger/templates/deployment.yaml +++ b/charts/flagger/templates/deployment.yaml @@ -72,6 +72,9 @@ spec: - -enable-leader-election=true - -leader-election-namespace={{ .Release.Namespace }} {{- end }} + {{- if .Values.ingressAnnotationsPrefix }} + - -ingress-annotations-prefix={{ .Values.ingressAnnotationsPrefix }} + {{- end }} livenessProbe: exec: command: diff --git a/cmd/flagger/main.go b/cmd/flagger/main.go index 9a17aaba..5556a940 100644 --- a/cmd/flagger/main.go +++ b/cmd/flagger/main.go @@ -33,25 +33,26 @@ import ( ) var ( - masterURL string - kubeconfig string - metricsServer string - controlLoopInterval time.Duration - logLevel string - port string - msteamsURL string - slackURL string - slackUser string - slackChannel string - threadiness int - zapReplaceGlobals bool - zapEncoding string - namespace string - meshProvider string - selectorLabels string - enableLeaderElection bool - leaderElectionNamespace string - ver bool + masterURL string + kubeconfig string + metricsServer string + controlLoopInterval time.Duration + logLevel string + port string + msteamsURL string + slackURL string + slackUser string + slackChannel string + threadiness int + zapReplaceGlobals bool + zapEncoding string + namespace string + meshProvider string + selectorLabels string + ingressAnnotationsPrefix string + enableLeaderElection bool + leaderElectionNamespace string + ver bool ) func init() { @@ -71,6 +72,7 @@ func init() { flag.StringVar(&namespace, "namespace", "", "Namespace that flagger would watch canary object.") flag.StringVar(&meshProvider, "mesh-provider", "istio", "Service mesh provider, can be istio, linkerd, appmesh, supergloo, nginx or smi.") flag.StringVar(&selectorLabels, "selector-labels", "app,name,app.kubernetes.io/name", "List of pod labels that Flagger uses to create pod selectors.") + flag.StringVar(&ingressAnnotationsPrefix, "ingress-annotations-prefix", "nginx.ingress.kubernetes.io", "Annotations prefix for ingresses.") flag.BoolVar(&enableLeaderElection, "enable-leader-election", false, "Enable leader election.") flag.StringVar(&leaderElectionNamespace, "leader-election-namespace", "kube-system", "Namespace used to create the leader election config map.") flag.BoolVar(&ver, "version", false, "Print version") @@ -175,7 +177,7 @@ func main() { // start HTTP server go server.ListenAndServe(port, 3*time.Second, logger, stopCh) - routerFactory := router.NewFactory(cfg, kubeClient, flaggerClient, logger, meshClient) + routerFactory := router.NewFactory(cfg, kubeClient, flaggerClient, ingressAnnotationsPrefix, logger, meshClient) c := controller.NewController( kubeClient, diff --git a/go.mod b/go.mod index 59b27495..e5cddb7d 100644 --- a/go.mod +++ b/go.mod @@ -56,6 +56,7 @@ require ( ) replace ( + git.apache.org/thrift.git => github.com/apache/thrift v0.12.0 github.com/google/uuid => github.com/google/uuid v1.0.0 golang.org/x/crypto => golang.org/x/crypto v0.0.0-20181025213731-e84da0312774 golang.org/x/net => golang.org/x/net v0.0.0-20190206173232-65e2d4e15006 diff --git a/pkg/controller/controller_test.go b/pkg/controller/controller_test.go index 61af3475..18f190ad 100644 --- a/pkg/controller/controller_test.go +++ b/pkg/controller/controller_test.go @@ -82,7 +82,7 @@ func SetupMocks(abtest bool) Mocks { flaggerInformer := flaggerInformerFactory.Flagger().V1alpha3().Canaries() // init router - rf := router.NewFactory(nil, kubeClient, flaggerClient, logger, flaggerClient) + rf := router.NewFactory(nil, kubeClient, flaggerClient, "annotationsPrefix", logger, flaggerClient) // init observer observerFactory, _ := metrics.NewFactory("fake", "istio", 5*time.Second) diff --git a/pkg/router/factory.go b/pkg/router/factory.go index b99f440f..e3ce020c 100644 --- a/pkg/router/factory.go +++ b/pkg/router/factory.go @@ -11,23 +11,26 @@ import ( ) type Factory struct { - kubeConfig *restclient.Config - kubeClient kubernetes.Interface - meshClient clientset.Interface - flaggerClient clientset.Interface - logger *zap.SugaredLogger + kubeConfig *restclient.Config + kubeClient kubernetes.Interface + meshClient clientset.Interface + flaggerClient clientset.Interface + ingressAnnotationsPrefix string + logger *zap.SugaredLogger } func NewFactory(kubeConfig *restclient.Config, kubeClient kubernetes.Interface, flaggerClient clientset.Interface, + ingressAnnotationsPrefix string, logger *zap.SugaredLogger, meshClient clientset.Interface) *Factory { return &Factory{ - kubeConfig: kubeConfig, - meshClient: meshClient, - kubeClient: kubeClient, - flaggerClient: flaggerClient, - logger: logger, + kubeConfig: kubeConfig, + meshClient: meshClient, + kubeClient: kubeClient, + flaggerClient: flaggerClient, + ingressAnnotationsPrefix: ingressAnnotationsPrefix, + logger: logger, } } @@ -51,8 +54,9 @@ func (factory *Factory) MeshRouter(provider string) Interface { return &NopRouter{} case provider == "nginx": return &IngressRouter{ - logger: factory.logger, - kubeClient: factory.kubeClient, + logger: factory.logger, + kubeClient: factory.kubeClient, + annotationsPrefix: factory.ingressAnnotationsPrefix, } case provider == "appmesh": return &AppMeshRouter{ diff --git a/pkg/router/ingress.go b/pkg/router/ingress.go index ffc663b1..eff3ed71 100644 --- a/pkg/router/ingress.go +++ b/pkg/router/ingress.go @@ -2,6 +2,9 @@ package router import ( "fmt" + "strconv" + "strings" + "github.com/google/go-cmp/cmp" flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3" "go.uber.org/zap" @@ -10,13 +13,12 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/kubernetes" - "strconv" - "strings" ) type IngressRouter struct { - kubeClient kubernetes.Interface - logger *zap.SugaredLogger + kubeClient kubernetes.Interface + annotationsPrefix string + logger *zap.SugaredLogger } func (i *IngressRouter) Reconcile(canary *flaggerv1.Canary) error { @@ -115,7 +117,7 @@ func (i *IngressRouter) GetRoutes(canary *flaggerv1.Canary) ( // A/B testing if len(canary.Spec.CanaryAnalysis.Match) > 0 { for k := range canaryIngress.Annotations { - if k == "nginx.ingress.kubernetes.io/canary-by-cookie" || k == "nginx.ingress.kubernetes.io/canary-by-header" { + if k == i.GetAnnotationWithPrefix("canary-by-cookie") || k == i.GetAnnotationWithPrefix("canary-by-header") { return 0, 100, nil } } @@ -123,7 +125,7 @@ func (i *IngressRouter) GetRoutes(canary *flaggerv1.Canary) ( // Canary for k, v := range canaryIngress.Annotations { - if k == "nginx.ingress.kubernetes.io/canary-weight" { + if k == i.GetAnnotationWithPrefix("canary-weight") { val, err := strconv.Atoi(v) if err != nil { return 0, 0, err @@ -170,12 +172,12 @@ func (i *IngressRouter) SetRoutes( iClone.Annotations = i.makeHeaderAnnotations(iClone.Annotations, header, headerValue, cookie) } else { // canary - iClone.Annotations["nginx.ingress.kubernetes.io/canary-weight"] = fmt.Sprintf("%v", canaryWeight) + iClone.Annotations[i.GetAnnotationWithPrefix("canary-weight")] = fmt.Sprintf("%v", canaryWeight) } // toggle canary if canaryWeight > 0 { - iClone.Annotations["nginx.ingress.kubernetes.io/canary"] = "true" + iClone.Annotations[i.GetAnnotationWithPrefix("canary")] = "true" } else { iClone.Annotations = i.makeAnnotations(iClone.Annotations) } @@ -191,14 +193,14 @@ func (i *IngressRouter) SetRoutes( func (i *IngressRouter) makeAnnotations(annotations map[string]string) map[string]string { res := make(map[string]string) for k, v := range annotations { - if !strings.Contains(k, "nginx.ingress.kubernetes.io/canary") && + if !strings.Contains(k, i.GetAnnotationWithPrefix("canary")) && !strings.Contains(k, "kubectl.kubernetes.io/last-applied-configuration") { res[k] = v } } - res["nginx.ingress.kubernetes.io/canary"] = "false" - res["nginx.ingress.kubernetes.io/canary-weight"] = "0" + res[i.GetAnnotationWithPrefix("canary")] = "false" + res[i.GetAnnotationWithPrefix("canary-weight")] = "0" return res } @@ -207,25 +209,29 @@ func (i *IngressRouter) makeHeaderAnnotations(annotations map[string]string, header string, headerValue string, cookie string) map[string]string { res := make(map[string]string) for k, v := range annotations { - if !strings.Contains(v, "nginx.ingress.kubernetes.io/canary") { + if !strings.Contains(v, i.GetAnnotationWithPrefix("canary")) { res[k] = v } } - res["nginx.ingress.kubernetes.io/canary"] = "true" - res["nginx.ingress.kubernetes.io/canary-weight"] = "0" + res[i.GetAnnotationWithPrefix("canary")] = "true" + res[i.GetAnnotationWithPrefix("canary-weight")] = "0" if cookie != "" { - res["nginx.ingress.kubernetes.io/canary-by-cookie"] = cookie + res[i.GetAnnotationWithPrefix("canary-by-cookie")] = cookie } if header != "" { - res["nginx.ingress.kubernetes.io/canary-by-header"] = header + res[i.GetAnnotationWithPrefix("canary-by-header")] = header } if headerValue != "" { - res["nginx.ingress.kubernetes.io/canary-by-header-value"] = headerValue + res[i.GetAnnotationWithPrefix("canary-by-header-value")] = headerValue } return res } + +func (i *IngressRouter) GetAnnotationWithPrefix(suffix string) string { + return fmt.Sprintf("%v/%v", i.annotationsPrefix, suffix) +} diff --git a/pkg/router/ingress_test.go b/pkg/router/ingress_test.go index 7d2679e6..bbda1771 100644 --- a/pkg/router/ingress_test.go +++ b/pkg/router/ingress_test.go @@ -2,15 +2,17 @@ package router import ( "fmt" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func TestIngressRouter_Reconcile(t *testing.T) { mocks := setupfakeClients() router := &IngressRouter{ - logger: mocks.logger, - kubeClient: mocks.kubeClient, + logger: mocks.logger, + kubeClient: mocks.kubeClient, + annotationsPrefix: "custom.ingress.kubernetes.io", } err := router.Reconcile(mocks.ingressCanary) @@ -18,8 +20,8 @@ func TestIngressRouter_Reconcile(t *testing.T) { t.Fatal(err.Error()) } - canaryAn := "nginx.ingress.kubernetes.io/canary" - canaryWeightAn := "nginx.ingress.kubernetes.io/canary-weight" + canaryAn := "custom.ingress.kubernetes.io/canary" + canaryWeightAn := "custom.ingress.kubernetes.io/canary-weight" canaryName := fmt.Sprintf("%s-canary", mocks.ingressCanary.Spec.IngressRef.Name) inCanary, err := router.kubeClient.ExtensionsV1beta1().Ingresses("default").Get(canaryName, metav1.GetOptions{}) @@ -44,8 +46,9 @@ func TestIngressRouter_Reconcile(t *testing.T) { func TestIngressRouter_GetSetRoutes(t *testing.T) { mocks := setupfakeClients() router := &IngressRouter{ - logger: mocks.logger, - kubeClient: mocks.kubeClient, + logger: mocks.logger, + kubeClient: mocks.kubeClient, + annotationsPrefix: "prefix1.nginx.ingress.kubernetes.io", } err := router.Reconcile(mocks.ingressCanary) @@ -66,8 +69,8 @@ func TestIngressRouter_GetSetRoutes(t *testing.T) { t.Fatal(err.Error()) } - canaryAn := "nginx.ingress.kubernetes.io/canary" - canaryWeightAn := "nginx.ingress.kubernetes.io/canary-weight" + canaryAn := "prefix1.nginx.ingress.kubernetes.io/canary" + canaryWeightAn := "prefix1.nginx.ingress.kubernetes.io/canary-weight" canaryName := fmt.Sprintf("%s-canary", mocks.ingressCanary.Spec.IngressRef.Name) inCanary, err := router.kubeClient.ExtensionsV1beta1().Ingresses("default").Get(canaryName, metav1.GetOptions{}) diff --git a/test/README.md b/test/README.md index 264625b2..977b7286 100644 --- a/test/README.md +++ b/test/README.md @@ -25,14 +25,17 @@ The e2e testing infrastructure is powered by CircleCI and [Kubernetes Kind](http * install latest stable kubectl [e2e-kind.sh](e2e-kind.sh) * install Kubernetes Kind [e2e-kind.sh](e2e-kind.sh) * create local Kubernetes cluster with kind [e2e-kind.sh](e2e-kind.sh) -* install latest stable Helm CLI [e2e-nginx.sh](e2e-istio.sh) -* deploy Tiller on the local cluster [e2e-nginx.sh](e2e-istio.sh) -* install NGINX ingress with Helm [e2e-nginx.sh](e2e-istio.sh) +* install latest stable Helm CLI [e2e-nginx.sh](e2e-nginx.sh) +* deploy Tiller on the local cluster [e2e-nginx.sh](e2e-nginx.sh) +* install NGINX ingress with Helm [e2e-nginx.sh](e2e-nginx.sh) * load Flagger image onto the local cluster [e2e-nginx.sh](e2e-nginx.sh) * install Flagger and Prometheus in the ingress-nginx namespace [e2e-nginx.sh](e2e-nginx.sh) -* create a test namespace [e2e-nginx-tests.sh](e2e-tests.sh) -* deploy the load tester in the test namespace [e2e-nginx-tests.sh](e2e-tests.sh) -* deploy the demo workload (podinfo) and ingress in the test namespace [e2e-nginx-tests.sh](e2e-tests.sh) -* test the canary initialization [e2e-nginx-tests.sh](e2e-tests.sh) -* test the canary analysis and promotion using weighted traffic and the load testing webhook [e2e-nginx-tests.sh](e2e-tests.sh) -* test the A/B testing analysis and promotion using header filters and pre/post rollout webhooks [e2e-nginx-tests.sh](e2e-tests.sh) +* create a test namespace [e2e-nginx-tests.sh](e2e-nginx-tests.sh) +* deploy the load tester in the test namespace [e2e-nginx-tests.sh](e2e-nginx-tests.sh) +* deploy the demo workload (podinfo) and ingress in the test namespace [e2e-nginx-tests.sh](e2e-nginx-tests.sh) +* test the canary initialization [e2e-nginx-tests.sh](e2e-nginx-tests.sh) +* test the canary analysis and promotion using weighted traffic and the load testing webhook [e2e-nginx-tests.sh](e2e-nginx-tests.sh) +* test the A/B testing analysis and promotion using header filters and pre/post rollout webhooks [e2e-nginx-tests.sh](e2e-nginx-tests.sh) +* cleanup test environment [e2e-nginx-cleanup.sh](e2e-nginx-cleanup.sh) +* install NGINX Ingress and Flagger with custom ingress annotations prefix [e2e-nginx-custom-annotations.sh](e2e-nginx-custom-annotations.sh) +* repeat the canary and A/B testing workflow [e2e-nginx-tests.sh](e2e-nginx-tests.sh) \ No newline at end of file diff --git a/test/e2e-nginx-cleanup.sh b/test/e2e-nginx-cleanup.sh new file mode 100755 index 00000000..a88f6a11 --- /dev/null +++ b/test/e2e-nginx-cleanup.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +set -o errexit + +export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" + +echo '>>> Deleting NGINX Ingress' +helm delete --purge nginx-ingress + +echo '>>> Deleting Flagger' +helm delete --purge flagger + +echo '>>> Cleanup test namespace' +kubectl delete namespace test --ignore-not-found=true \ No newline at end of file diff --git a/test/e2e-nginx-custom-annotations.sh b/test/e2e-nginx-custom-annotations.sh new file mode 100755 index 00000000..a1980ac6 --- /dev/null +++ b/test/e2e-nginx-custom-annotations.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) +export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" +NGINX_VERSION=1.12.1 + +echo '>>> Installing NGINX Ingress' +helm upgrade -i nginx-ingress stable/nginx-ingress --version=${NGINX_VERSION} \ +--wait \ +--namespace ingress-nginx \ +--set controller.stats.enabled=true \ +--set controller.metrics.enabled=true \ +--set controller.podAnnotations."prometheus\.io/scrape"=true \ +--set controller.podAnnotations."prometheus\.io/port"=10254 \ +--set controller.service.type=NodePort + +kubectl -n ingress-nginx patch deployment/nginx-ingress-controller \ +--type='json' -p='[{"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--annotations-prefix=custom.ingress.kubernetes.io"}]' + +kubectl -n ingress-nginx rollout status deployment/nginx-ingress-controller +kubectl -n ingress-nginx get all + +echo '>>> Loading Flagger image' +kind load docker-image test/flagger:latest + +echo '>>> Installing Flagger' +helm install ${REPO_ROOT}/charts/flagger \ +--name flagger \ +--namespace ingress-nginx \ +--set prometheus.install=true \ +--set ingressAnnotationsPrefix="custom.ingress.kubernetes.io" \ +--set meshProvider=nginx \ +--set crd.create=false + +kubectl -n ingress-nginx set image deployment/flagger flagger=test/flagger:latest + +kubectl -n ingress-nginx rollout status deployment/flagger +kubectl -n ingress-nginx rollout status deployment/flagger-prometheus + diff --git a/test/e2e-nginx-tests.sh b/test/e2e-nginx-tests.sh index dac550f7..54ab89b7 100755 --- a/test/e2e-nginx-tests.sh +++ b/test/e2e-nginx-tests.sh @@ -43,9 +43,31 @@ spec: maxWeight: 30 stepWeight: 10 metrics: - - name: request-success-rate + - name: "http-request-success-rate" threshold: 99 interval: 1m + query: | + 100 - sum( + rate( + http_request_duration_seconds_count{ + kubernetes_namespace="test", + kubernetes_pod_name=~"podinfo-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)", + path="root", + status!~"5.*" + }[1m] + ) + ) + / + sum( + rate( + http_request_duration_seconds_count{ + kubernetes_namespace="test", + kubernetes_pod_name=~"podinfo-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)", + path="root" + }[1m] + ) + ) + * 100 - name: "latency" threshold: 0.5 interval: 1m @@ -55,7 +77,8 @@ spec: rate( http_request_duration_seconds_bucket{ kubernetes_namespace="test", - kubernetes_pod_name=~"podinfo-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)" + kubernetes_pod_name=~"podinfo-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)", + path="root" }[1m] ) ) by (le) @@ -94,7 +117,14 @@ echo '>>> Waiting for canary promotion' retries=50 count=0 ok=false +failed=false until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Failed' && failed=true || failed=false + if ${failed}; then + kubectl -n ingress-nginx logs deployment/test-flagger + echo "Canary failed!" + exit 1 + fi kubectl -n test describe deployment/podinfo-primary | grep '1.4.1' && ok=true || ok=false sleep 10 kubectl -n ingress-nginx logs deployment/flagger --tail 1 @@ -144,9 +174,31 @@ spec: cookie: exact: "canary" metrics: - - name: request-success-rate + - name: "http-request-success-rate" threshold: 99 interval: 1m + query: | + 100 - sum( + rate( + http_request_duration_seconds_count{ + kubernetes_namespace="test", + kubernetes_pod_name=~"podinfo-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)", + path="root", + status!~"5.*" + }[1m] + ) + ) + / + sum( + rate( + http_request_duration_seconds_count{ + kubernetes_namespace="test", + kubernetes_pod_name=~"podinfo-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)", + path="root" + }[1m] + ) + ) + * 100 webhooks: - name: pre type: pre-rollout