Merge pull request #293 from kislitsyn/nginx-annotations-prefix

Add annotations prefix for ingresses

.circleci/config.yml

@@ -132,6 +132,9 @@ jobs:
       - run: test/e2e-kind.sh
       - run: test/e2e-nginx.sh
       - run: test/e2e-nginx-tests.sh
+      - run: test/e2e-nginx-cleanup.sh
+      - run: test/e2e-nginx-custom-annotations.sh
+      - run: test/e2e-nginx-tests.sh
 
   e2e-linkerd-testing:
     machine: true

charts/flagger/README.md

@@ -74,6 +74,7 @@ Parameter | Description | Default
 `msteams.url` | Microsoft Teams incoming webhook | None
 `leaderElection.enabled` | leader election must be enabled when running more than one replica | `false`
 `leaderElection.replicaCount` | number of replicas | `1`
+`ingressAnnotationsPrefix` | annotations prefix for ingresses | `custom.ingress.kubernetes.io`
 `rbac.create` | if `true`, create and use RBAC resources | `true`
 `rbac.pspEnabled` | If `true`, create and use a restricted pod security policy | `false`
 `crd.create` | if `true`, create Flagger's CRDs | `true`

charts/flagger/templates/deployment.yaml

@@ -72,6 +72,9 @@ spec:
           - -enable-leader-election=true
           - -leader-election-namespace={{ .Release.Namespace }}
           {{- end }}
+          {{- if .Values.ingressAnnotationsPrefix }}
+          - -ingress-annotations-prefix={{ .Values.ingressAnnotationsPrefix }}
+          {{- end }}
           livenessProbe:
             exec:
               command:

cmd/flagger/main.go

@@ -33,25 +33,26 @@ import (
 )
 
 var (
-	masterURL               string
-	kubeconfig              string
-	metricsServer           string
-	controlLoopInterval     time.Duration
-	logLevel                string
-	port                    string
-	msteamsURL              string
-	slackURL                string
-	slackUser               string
-	slackChannel            string
-	threadiness             int
-	zapReplaceGlobals       bool
-	zapEncoding             string
-	namespace               string
-	meshProvider            string
-	selectorLabels          string
-	enableLeaderElection    bool
-	leaderElectionNamespace string
-	ver                     bool
+	masterURL                string
+	kubeconfig               string
+	metricsServer            string
+	controlLoopInterval      time.Duration
+	logLevel                 string
+	port                     string
+	msteamsURL               string
+	slackURL                 string
+	slackUser                string
+	slackChannel             string
+	threadiness              int
+	zapReplaceGlobals        bool
+	zapEncoding              string
+	namespace                string
+	meshProvider             string
+	selectorLabels           string
+	ingressAnnotationsPrefix string
+	enableLeaderElection     bool
+	leaderElectionNamespace  string
+	ver                      bool
 )
 
 func init() {
@@ -71,6 +72,7 @@ func init() {
 	flag.StringVar(&namespace, "namespace", "", "Namespace that flagger would watch canary object.")
 	flag.StringVar(&meshProvider, "mesh-provider", "istio", "Service mesh provider, can be istio, linkerd, appmesh, supergloo, nginx or smi.")
 	flag.StringVar(&selectorLabels, "selector-labels", "app,name,app.kubernetes.io/name", "List of pod labels that Flagger uses to create pod selectors.")
+	flag.StringVar(&ingressAnnotationsPrefix, "ingress-annotations-prefix", "nginx.ingress.kubernetes.io", "Annotations prefix for ingresses.")
 	flag.BoolVar(&enableLeaderElection, "enable-leader-election", false, "Enable leader election.")
 	flag.StringVar(&leaderElectionNamespace, "leader-election-namespace", "kube-system", "Namespace used to create the leader election config map.")
 	flag.BoolVar(&ver, "version", false, "Print version")
@@ -175,7 +177,7 @@ func main() {
 	// start HTTP server
 	go server.ListenAndServe(port, 3*time.Second, logger, stopCh)
 
-	routerFactory := router.NewFactory(cfg, kubeClient, flaggerClient, logger, meshClient)
+	routerFactory := router.NewFactory(cfg, kubeClient, flaggerClient, ingressAnnotationsPrefix, logger, meshClient)
 
 	c := controller.NewController(
 		kubeClient,

go.mod

@@ -56,6 +56,7 @@ require (
 )
 
 replace (
+	git.apache.org/thrift.git => github.com/apache/thrift v0.12.0
 	github.com/google/uuid => github.com/google/uuid v1.0.0
 	golang.org/x/crypto => golang.org/x/crypto v0.0.0-20181025213731-e84da0312774
 	golang.org/x/net => golang.org/x/net v0.0.0-20190206173232-65e2d4e15006

pkg/controller/controller_test.go

@@ -82,7 +82,7 @@ func SetupMocks(abtest bool) Mocks {
 	flaggerInformer := flaggerInformerFactory.Flagger().V1alpha3().Canaries()
 
 	// init router
-	rf := router.NewFactory(nil, kubeClient, flaggerClient, logger, flaggerClient)
+	rf := router.NewFactory(nil, kubeClient, flaggerClient, "annotationsPrefix", logger, flaggerClient)
 
 	// init observer
 	observerFactory, _ := metrics.NewFactory("fake", "istio", 5*time.Second)

pkg/router/factory.go

@@ -11,23 +11,26 @@ import (
 )
 
 type Factory struct {
-	kubeConfig    *restclient.Config
-	kubeClient    kubernetes.Interface
-	meshClient    clientset.Interface
-	flaggerClient clientset.Interface
-	logger        *zap.SugaredLogger
+	kubeConfig               *restclient.Config
+	kubeClient               kubernetes.Interface
+	meshClient               clientset.Interface
+	flaggerClient            clientset.Interface
+	ingressAnnotationsPrefix string
+	logger                   *zap.SugaredLogger
 }
 
 func NewFactory(kubeConfig *restclient.Config, kubeClient kubernetes.Interface,
 	flaggerClient clientset.Interface,
+	ingressAnnotationsPrefix string,
 	logger *zap.SugaredLogger,
 	meshClient clientset.Interface) *Factory {
 	return &Factory{
-		kubeConfig:    kubeConfig,
-		meshClient:    meshClient,
-		kubeClient:    kubeClient,
-		flaggerClient: flaggerClient,
-		logger:        logger,
+		kubeConfig:               kubeConfig,
+		meshClient:               meshClient,
+		kubeClient:               kubeClient,
+		flaggerClient:            flaggerClient,
+		ingressAnnotationsPrefix: ingressAnnotationsPrefix,
+		logger:                   logger,
 	}
 }
 
@@ -51,8 +54,9 @@ func (factory *Factory) MeshRouter(provider string) Interface {
 		return &NopRouter{}
 	case provider == "nginx":
 		return &IngressRouter{
-			logger:     factory.logger,
-			kubeClient: factory.kubeClient,
+			logger:            factory.logger,
+			kubeClient:        factory.kubeClient,
+			annotationsPrefix: factory.ingressAnnotationsPrefix,
 		}
 	case provider == "appmesh":
 		return &AppMeshRouter{

pkg/router/ingress.go

@@ -2,6 +2,9 @@ package router
 
 import (
 	"fmt"
+	"strconv"
+	"strings"
+
 	"github.com/google/go-cmp/cmp"
 	flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3"
 	"go.uber.org/zap"
@@ -10,13 +13,12 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/client-go/kubernetes"
-	"strconv"
-	"strings"
 )
 
 type IngressRouter struct {
-	kubeClient kubernetes.Interface
-	logger     *zap.SugaredLogger
+	kubeClient        kubernetes.Interface
+	annotationsPrefix string
+	logger            *zap.SugaredLogger
 }
 
 func (i *IngressRouter) Reconcile(canary *flaggerv1.Canary) error {
@@ -115,7 +117,7 @@ func (i *IngressRouter) GetRoutes(canary *flaggerv1.Canary) (
 	// A/B testing
 	if len(canary.Spec.CanaryAnalysis.Match) > 0 {
 		for k := range canaryIngress.Annotations {
-			if k == "nginx.ingress.kubernetes.io/canary-by-cookie" || k == "nginx.ingress.kubernetes.io/canary-by-header" {
+			if k == i.GetAnnotationWithPrefix("canary-by-cookie") || k == i.GetAnnotationWithPrefix("canary-by-header") {
 				return 0, 100, nil
 			}
 		}
@@ -123,7 +125,7 @@ func (i *IngressRouter) GetRoutes(canary *flaggerv1.Canary) (
 
 	// Canary
 	for k, v := range canaryIngress.Annotations {
-		if k == "nginx.ingress.kubernetes.io/canary-weight" {
+		if k == i.GetAnnotationWithPrefix("canary-weight") {
 			val, err := strconv.Atoi(v)
 			if err != nil {
 				return 0, 0, err
@@ -170,12 +172,12 @@ func (i *IngressRouter) SetRoutes(
 		iClone.Annotations = i.makeHeaderAnnotations(iClone.Annotations, header, headerValue, cookie)
 	} else {
 		// canary
-		iClone.Annotations["nginx.ingress.kubernetes.io/canary-weight"] = fmt.Sprintf("%v", canaryWeight)
+		iClone.Annotations[i.GetAnnotationWithPrefix("canary-weight")] = fmt.Sprintf("%v", canaryWeight)
 	}
 
 	// toggle canary
 	if canaryWeight > 0 {
-		iClone.Annotations["nginx.ingress.kubernetes.io/canary"] = "true"
+		iClone.Annotations[i.GetAnnotationWithPrefix("canary")] = "true"
 	} else {
 		iClone.Annotations = i.makeAnnotations(iClone.Annotations)
 	}
@@ -191,14 +193,14 @@ func (i *IngressRouter) SetRoutes(
 func (i *IngressRouter) makeAnnotations(annotations map[string]string) map[string]string {
 	res := make(map[string]string)
 	for k, v := range annotations {
-		if !strings.Contains(k, "nginx.ingress.kubernetes.io/canary") &&
+		if !strings.Contains(k, i.GetAnnotationWithPrefix("canary")) &&
 			!strings.Contains(k, "kubectl.kubernetes.io/last-applied-configuration") {
 			res[k] = v
 		}
 	}
 
-	res["nginx.ingress.kubernetes.io/canary"] = "false"
-	res["nginx.ingress.kubernetes.io/canary-weight"] = "0"
+	res[i.GetAnnotationWithPrefix("canary")] = "false"
+	res[i.GetAnnotationWithPrefix("canary-weight")] = "0"
 
 	return res
 }
@@ -207,25 +209,29 @@ func (i *IngressRouter) makeHeaderAnnotations(annotations map[string]string,
 	header string, headerValue string, cookie string) map[string]string {
 	res := make(map[string]string)
 	for k, v := range annotations {
-		if !strings.Contains(v, "nginx.ingress.kubernetes.io/canary") {
+		if !strings.Contains(v, i.GetAnnotationWithPrefix("canary")) {
 			res[k] = v
 		}
 	}
 
-	res["nginx.ingress.kubernetes.io/canary"] = "true"
-	res["nginx.ingress.kubernetes.io/canary-weight"] = "0"
+	res[i.GetAnnotationWithPrefix("canary")] = "true"
+	res[i.GetAnnotationWithPrefix("canary-weight")] = "0"
 
 	if cookie != "" {
-		res["nginx.ingress.kubernetes.io/canary-by-cookie"] = cookie
+		res[i.GetAnnotationWithPrefix("canary-by-cookie")] = cookie
 	}
 
 	if header != "" {
-		res["nginx.ingress.kubernetes.io/canary-by-header"] = header
+		res[i.GetAnnotationWithPrefix("canary-by-header")] = header
 	}
 
 	if headerValue != "" {
-		res["nginx.ingress.kubernetes.io/canary-by-header-value"] = headerValue
+		res[i.GetAnnotationWithPrefix("canary-by-header-value")] = headerValue
 	}
 
 	return res
 }
+
+func (i *IngressRouter) GetAnnotationWithPrefix(suffix string) string {
+	return fmt.Sprintf("%v/%v", i.annotationsPrefix, suffix)
+}

pkg/router/ingress_test.go

@@ -2,15 +2,17 @@ package router
 
 import (
 	"fmt"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"testing"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 func TestIngressRouter_Reconcile(t *testing.T) {
 	mocks := setupfakeClients()
 	router := &IngressRouter{
-		logger:     mocks.logger,
-		kubeClient: mocks.kubeClient,
+		logger:            mocks.logger,
+		kubeClient:        mocks.kubeClient,
+		annotationsPrefix: "custom.ingress.kubernetes.io",
 	}
 
 	err := router.Reconcile(mocks.ingressCanary)
@@ -18,8 +20,8 @@ func TestIngressRouter_Reconcile(t *testing.T) {
 		t.Fatal(err.Error())
 	}
 
-	canaryAn := "nginx.ingress.kubernetes.io/canary"
-	canaryWeightAn := "nginx.ingress.kubernetes.io/canary-weight"
+	canaryAn := "custom.ingress.kubernetes.io/canary"
+	canaryWeightAn := "custom.ingress.kubernetes.io/canary-weight"
 
 	canaryName := fmt.Sprintf("%s-canary", mocks.ingressCanary.Spec.IngressRef.Name)
 	inCanary, err := router.kubeClient.ExtensionsV1beta1().Ingresses("default").Get(canaryName, metav1.GetOptions{})
@@ -44,8 +46,9 @@ func TestIngressRouter_Reconcile(t *testing.T) {
 func TestIngressRouter_GetSetRoutes(t *testing.T) {
 	mocks := setupfakeClients()
 	router := &IngressRouter{
-		logger:     mocks.logger,
-		kubeClient: mocks.kubeClient,
+		logger:            mocks.logger,
+		kubeClient:        mocks.kubeClient,
+		annotationsPrefix: "prefix1.nginx.ingress.kubernetes.io",
 	}
 
 	err := router.Reconcile(mocks.ingressCanary)
@@ -66,8 +69,8 @@ func TestIngressRouter_GetSetRoutes(t *testing.T) {
 		t.Fatal(err.Error())
 	}
 
-	canaryAn := "nginx.ingress.kubernetes.io/canary"
-	canaryWeightAn := "nginx.ingress.kubernetes.io/canary-weight"
+	canaryAn := "prefix1.nginx.ingress.kubernetes.io/canary"
+	canaryWeightAn := "prefix1.nginx.ingress.kubernetes.io/canary-weight"
 
 	canaryName := fmt.Sprintf("%s-canary", mocks.ingressCanary.Spec.IngressRef.Name)
 	inCanary, err := router.kubeClient.ExtensionsV1beta1().Ingresses("default").Get(canaryName, metav1.GetOptions{})

test/README.md

@@ -25,14 +25,17 @@ The e2e testing infrastructure is powered by CircleCI and [Kubernetes Kind](http
 * install latest stable kubectl [e2e-kind.sh](e2e-kind.sh)
 * install Kubernetes Kind [e2e-kind.sh](e2e-kind.sh)
 * create local Kubernetes cluster with kind [e2e-kind.sh](e2e-kind.sh)
-* install latest stable Helm CLI [e2e-nginx.sh](e2e-istio.sh)
-* deploy Tiller on the local cluster [e2e-nginx.sh](e2e-istio.sh)
-* install NGINX ingress with Helm [e2e-nginx.sh](e2e-istio.sh)
+* install latest stable Helm CLI [e2e-nginx.sh](e2e-nginx.sh)
+* deploy Tiller on the local cluster [e2e-nginx.sh](e2e-nginx.sh)
+* install NGINX ingress with Helm [e2e-nginx.sh](e2e-nginx.sh)
 * load Flagger image onto the local cluster [e2e-nginx.sh](e2e-nginx.sh)
 * install Flagger and Prometheus in the ingress-nginx namespace [e2e-nginx.sh](e2e-nginx.sh)
-* create a test namespace [e2e-nginx-tests.sh](e2e-tests.sh)
-* deploy the load tester in the test namespace [e2e-nginx-tests.sh](e2e-tests.sh)
-* deploy the demo workload (podinfo) and ingress in the test namespace [e2e-nginx-tests.sh](e2e-tests.sh)
-* test the canary initialization [e2e-nginx-tests.sh](e2e-tests.sh)
-* test the canary analysis and promotion using weighted traffic and the load testing webhook [e2e-nginx-tests.sh](e2e-tests.sh)
-* test the A/B testing analysis and promotion using header filters and pre/post rollout webhooks [e2e-nginx-tests.sh](e2e-tests.sh)
+* create a test namespace [e2e-nginx-tests.sh](e2e-nginx-tests.sh)
+* deploy the load tester in the test namespace [e2e-nginx-tests.sh](e2e-nginx-tests.sh)
+* deploy the demo workload (podinfo) and ingress in the test namespace [e2e-nginx-tests.sh](e2e-nginx-tests.sh)
+* test the canary initialization [e2e-nginx-tests.sh](e2e-nginx-tests.sh)
+* test the canary analysis and promotion using weighted traffic and the load testing webhook [e2e-nginx-tests.sh](e2e-nginx-tests.sh)
+* test the A/B testing analysis and promotion using header filters and pre/post rollout webhooks [e2e-nginx-tests.sh](e2e-nginx-tests.sh)
+* cleanup test environment [e2e-nginx-cleanup.sh](e2e-nginx-cleanup.sh)
+* install NGINX Ingress and Flagger with custom ingress annotations prefix [e2e-nginx-custom-annotations.sh](e2e-nginx-custom-annotations.sh)
+* repeat the canary and A/B testing workflow [e2e-nginx-tests.sh](e2e-nginx-tests.sh)

test/e2e-nginx-cleanup.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+set -o errexit
+
+export KUBECONFIG="$(kind get kubeconfig-path --name="kind")"
+
+echo '>>> Deleting NGINX Ingress'
+helm delete --purge nginx-ingress
+
+echo '>>> Deleting Flagger'
+helm delete --purge flagger
+
+echo '>>> Cleanup test namespace'
+kubectl delete namespace test --ignore-not-found=true

test/e2e-nginx-custom-annotations.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+
+set -o errexit
+
+REPO_ROOT=$(git rev-parse --show-toplevel)
+export KUBECONFIG="$(kind get kubeconfig-path --name="kind")"
+NGINX_VERSION=1.12.1
+
+echo '>>> Installing NGINX Ingress'
+helm upgrade -i nginx-ingress stable/nginx-ingress --version=${NGINX_VERSION} \
+--wait \
+--namespace ingress-nginx \
+--set controller.stats.enabled=true \
+--set controller.metrics.enabled=true \
+--set controller.podAnnotations."prometheus\.io/scrape"=true \
+--set controller.podAnnotations."prometheus\.io/port"=10254 \
+--set controller.service.type=NodePort
+
+kubectl -n ingress-nginx patch deployment/nginx-ingress-controller \
+--type='json' -p='[{"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--annotations-prefix=custom.ingress.kubernetes.io"}]'
+
+kubectl -n ingress-nginx rollout status deployment/nginx-ingress-controller
+kubectl -n ingress-nginx get all
+
+echo '>>> Loading Flagger image'
+kind load docker-image test/flagger:latest
+
+echo '>>> Installing Flagger'
+helm install ${REPO_ROOT}/charts/flagger \
+--name flagger \
+--namespace ingress-nginx \
+--set prometheus.install=true \
+--set ingressAnnotationsPrefix="custom.ingress.kubernetes.io" \
+--set meshProvider=nginx \
+--set crd.create=false
+
+kubectl -n ingress-nginx set image deployment/flagger flagger=test/flagger:latest
+
+kubectl -n ingress-nginx rollout status deployment/flagger
+kubectl -n ingress-nginx rollout status deployment/flagger-prometheus
+

test/e2e-nginx-tests.sh

@@ -43,9 +43,31 @@ spec:
     maxWeight: 30
     stepWeight: 10
     metrics:
-    - name: request-success-rate
+    - name: "http-request-success-rate"
       threshold: 99
       interval: 1m
+      query: |
+        100 - sum(
+                rate(
+                    http_request_duration_seconds_count{
+                        kubernetes_namespace="test",
+                        kubernetes_pod_name=~"podinfo-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)",
+                        path="root",
+                        status!~"5.*"
+                    }[1m]
+                )
+            )
+            /
+            sum(
+                rate(
+                    http_request_duration_seconds_count{
+                        kubernetes_namespace="test",
+                        kubernetes_pod_name=~"podinfo-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)",
+                        path="root"
+                    }[1m]
+                )
+            )
+            * 100
     - name: "latency"
       threshold: 0.5
       interval: 1m
@@ -55,7 +77,8 @@ spec:
             rate(
               http_request_duration_seconds_bucket{
                 kubernetes_namespace="test",
-                kubernetes_pod_name=~"podinfo-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)"
+                kubernetes_pod_name=~"podinfo-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)",
+                path="root"
               }[1m]
             )
           ) by (le)
@@ -94,7 +117,14 @@ echo '>>> Waiting for canary promotion'
 retries=50
 count=0
 ok=false
+failed=false
 until ${ok}; do
+    kubectl -n test get canary/podinfo | grep 'Failed' && failed=true || failed=false
+    if ${failed}; then
+        kubectl -n ingress-nginx logs deployment/test-flagger
+        echo "Canary failed!"
+        exit 1
+    fi
     kubectl -n test describe deployment/podinfo-primary | grep '1.4.1' && ok=true || ok=false
     sleep 10
     kubectl -n ingress-nginx logs deployment/flagger --tail 1
@@ -144,9 +174,31 @@ spec:
           cookie:
             exact: "canary"
     metrics:
-    - name: request-success-rate
+    - name: "http-request-success-rate"
       threshold: 99
       interval: 1m
+      query: |
+        100 - sum(
+                rate(
+                    http_request_duration_seconds_count{
+                        kubernetes_namespace="test",
+                        kubernetes_pod_name=~"podinfo-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)",
+                        path="root",
+                        status!~"5.*"
+                    }[1m]
+                )
+            )
+            /
+            sum(
+                rate(
+                    http_request_duration_seconds_count{
+                        kubernetes_namespace="test",
+                        kubernetes_pod_name=~"podinfo-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)",
+                        path="root"
+                    }[1m]
+                )
+            )
+            * 100
     webhooks:
       - name: pre
         type: pre-rollout