From 38ef4ef4d889bfe1c5cddb20e3f86fce954e6fad Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Thu, 2 Apr 2020 01:34:29 +0300 Subject: [PATCH 1/4] Implement NGINX ingress header regex match --- pkg/router/ingress.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/pkg/router/ingress.go b/pkg/router/ingress.go index db31a30b..bf77eb2a 100644 --- a/pkg/router/ingress.go +++ b/pkg/router/ingress.go @@ -159,19 +159,23 @@ func (i *IngressRouter) SetRoutes( // A/B testing if len(canary.GetAnalysis().Match) > 0 { - var cookie, header, headerValue string + var cookie, header, headerValue, headerRegex string for _, m := range canary.GetAnalysis().Match { for k, v := range m.Headers { if k == "cookie" { cookie = v.Exact } else { header = k - headerValue = v.Exact + if v.Regex != "" { + headerRegex = v.Regex + } else { + headerValue = v.Exact + } } } } - iClone.Annotations = i.makeHeaderAnnotations(iClone.Annotations, header, headerValue, cookie) + iClone.Annotations = i.makeHeaderAnnotations(iClone.Annotations, header, headerValue, headerRegex, cookie) } else { // canary iClone.Annotations[i.GetAnnotationWithPrefix("canary-weight")] = fmt.Sprintf("%v", canaryWeight) @@ -208,7 +212,7 @@ func (i *IngressRouter) makeAnnotations(annotations map[string]string) map[strin } func (i *IngressRouter) makeHeaderAnnotations(annotations map[string]string, - header string, headerValue string, cookie string) map[string]string { + header string, headerValue string, headerRegex string, cookie string) map[string]string { res := make(map[string]string) for k, v := range annotations { if !strings.Contains(v, i.GetAnnotationWithPrefix("canary")) { @@ -231,6 +235,10 @@ func (i *IngressRouter) makeHeaderAnnotations(annotations map[string]string, res[i.GetAnnotationWithPrefix("canary-by-header-value")] = headerValue } + if headerRegex != "" { + res[i.GetAnnotationWithPrefix("canary-by-header-pattern")] = headerRegex + } + return res } From b8e9f57e1e8bcf80ea0276247f007275f6a15229 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Thu, 2 Apr 2020 01:35:15 +0300 Subject: [PATCH 2/4] Add unit tests for ingress A/B Testing --- pkg/router/ingress_test.go | 90 +++++++++++++++++++++++++++++++++++++- 1 file changed, 89 insertions(+), 1 deletion(-) diff --git a/pkg/router/ingress_test.go b/pkg/router/ingress_test.go index 7aac06b5..dbec77b8 100644 --- a/pkg/router/ingress_test.go +++ b/pkg/router/ingress_test.go @@ -6,8 +6,11 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1beta1" + istiov1alpha1 "github.com/weaveworks/flagger/pkg/apis/istio/common/v1alpha1" + istiov1alpha3 "github.com/weaveworks/flagger/pkg/apis/istio/v1alpha3" ) func TestIngressRouter_Reconcile(t *testing.T) { @@ -79,3 +82,88 @@ func TestIngressRouter_GetSetRoutes(t *testing.T) { assert.Equal(t, "false", inCanary.Annotations[canaryAn]) assert.Equal(t, "0", inCanary.Annotations[canaryWeightAn]) } + +func TestIngressRouter_ABTest(t *testing.T) { + mocks := newFixture(nil) + router := &IngressRouter{ + logger: mocks.logger, + kubeClient: mocks.kubeClient, + annotationsPrefix: "nginx.ingress.kubernetes.io", + } + + tables := []struct { + makeCanary func() *flaggerv1.Canary + annotation string + }{ + // Header exact match + { + makeCanary: func() *flaggerv1.Canary { + mocks.ingressCanary.Spec.Analysis.Iterations = 1 + mocks.ingressCanary.Spec.Analysis.Match = []istiov1alpha3.HTTPMatchRequest{ + { + Headers: map[string]istiov1alpha1.StringMatch{ + "x-user-type": { + Exact: "test", + }, + }, + }, + } + return mocks.ingressCanary + }, + annotation: router.GetAnnotationWithPrefix("canary-by-header-value"), + }, + // Header regex match + { + makeCanary: func() *flaggerv1.Canary { + mocks.ingressCanary.Spec.Analysis.Iterations = 1 + mocks.ingressCanary.Spec.Analysis.Match = []istiov1alpha3.HTTPMatchRequest{ + { + Headers: map[string]istiov1alpha1.StringMatch{ + "x-user-type": { + Regex: "test", + }, + }, + }, + } + return mocks.ingressCanary + }, + annotation: router.GetAnnotationWithPrefix("canary-by-header-pattern"), + }, + // Cookie exact match + { + makeCanary: func() *flaggerv1.Canary { + mocks.ingressCanary.Spec.Analysis.Iterations = 1 + mocks.ingressCanary.Spec.Analysis.Match = []istiov1alpha3.HTTPMatchRequest{ + { + Headers: map[string]istiov1alpha1.StringMatch{ + "cookie": { + Exact: "test", + }, + }, + }, + } + return mocks.ingressCanary + }, + annotation: router.GetAnnotationWithPrefix("canary-by-cookie"), + }, + } + + for _, table := range tables { + err := router.Reconcile(table.makeCanary()) + require.NoError(t, err) + + err = router.SetRoutes(table.makeCanary(), 50, 50, false) + require.NoError(t, err) + + canaryAn := router.GetAnnotationWithPrefix("canary") + + canaryName := fmt.Sprintf("%s-canary", table.makeCanary().Spec.IngressRef.Name) + inCanary, err := router.kubeClient.NetworkingV1beta1().Ingresses("default").Get(canaryName, metav1.GetOptions{}) + require.NoError(t, err) + + // test initialisation + assert.Equal(t, "true", inCanary.Annotations[canaryAn]) + assert.Equal(t, "test", inCanary.Annotations[table.annotation]) + } + +} From 14e9c7f46685ab0932c3342529282992a40654d1 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Thu, 2 Apr 2020 08:59:01 +0300 Subject: [PATCH 3/4] Add e2e tests for ingress A/B Testing --- pkg/router/ingress.go | 9 +- pkg/router/ingress_test.go | 3 - test/e2e-nginx-tests.sh | 200 ++++++++++++++++++++----------------- test/e2e-nginx.sh | 2 +- 4 files changed, 110 insertions(+), 104 deletions(-) diff --git a/pkg/router/ingress.go b/pkg/router/ingress.go index bf77eb2a..928fce7a 100644 --- a/pkg/router/ingress.go +++ b/pkg/router/ingress.go @@ -166,11 +166,8 @@ func (i *IngressRouter) SetRoutes( cookie = v.Exact } else { header = k - if v.Regex != "" { - headerRegex = v.Regex - } else { - headerValue = v.Exact - } + headerRegex = v.Regex + headerValue = v.Exact } } } @@ -206,7 +203,6 @@ func (i *IngressRouter) makeAnnotations(annotations map[string]string) map[strin } res[i.GetAnnotationWithPrefix("canary")] = "false" - res[i.GetAnnotationWithPrefix("canary-weight")] = "0" return res } @@ -221,7 +217,6 @@ func (i *IngressRouter) makeHeaderAnnotations(annotations map[string]string, } res[i.GetAnnotationWithPrefix("canary")] = "true" - res[i.GetAnnotationWithPrefix("canary-weight")] = "0" if cookie != "" { res[i.GetAnnotationWithPrefix("canary-by-cookie")] = cookie diff --git a/pkg/router/ingress_test.go b/pkg/router/ingress_test.go index dbec77b8..b1fe5f38 100644 --- a/pkg/router/ingress_test.go +++ b/pkg/router/ingress_test.go @@ -25,7 +25,6 @@ func TestIngressRouter_Reconcile(t *testing.T) { require.NoError(t, err) canaryAn := "custom.ingress.kubernetes.io/canary" - canaryWeightAn := "custom.ingress.kubernetes.io/canary-weight" canaryName := fmt.Sprintf("%s-canary", mocks.ingressCanary.Spec.IngressRef.Name) inCanary, err := router.kubeClient.NetworkingV1beta1().Ingresses("default").Get(canaryName, metav1.GetOptions{}) @@ -33,7 +32,6 @@ func TestIngressRouter_Reconcile(t *testing.T) { // test initialisation assert.Equal(t, "false", inCanary.Annotations[canaryAn]) - assert.Equal(t, "0", inCanary.Annotations[canaryWeightAn]) } func TestIngressRouter_GetSetRoutes(t *testing.T) { @@ -80,7 +78,6 @@ func TestIngressRouter_GetSetRoutes(t *testing.T) { // test promotion assert.Equal(t, "false", inCanary.Annotations[canaryAn]) - assert.Equal(t, "0", inCanary.Annotations[canaryWeightAn]) } func TestIngressRouter_ABTest(t *testing.T) { diff --git a/test/e2e-nginx-tests.sh b/test/e2e-nginx-tests.sh index 462d2016..fe77002d 100755 --- a/test/e2e-nginx-tests.sh +++ b/test/e2e-nginx-tests.sh @@ -18,6 +18,65 @@ echo '>>> Initialising canary' kubectl apply -f ${REPO_ROOT}/test/e2e-workload.yaml kubectl apply -f ${REPO_ROOT}/test/e2e-ingress.yaml +echo '>>> Create metric templates' +cat <>> Waiting for primary to be ready' @@ -173,73 +204,56 @@ spec: progressDeadlineSeconds: 60 service: port: 80 - targetPort: 9898 + targetPort: http analysis: - interval: 10s + interval: 15s threshold: 5 - iterations: 5 + iterations: 3 match: - - headers: - x-canary: - exact: "insider" - - headers: - cookie: - exact: "canary" + - headers: + x-user: + exact: "insider" metrics: - - name: "http-request-success-rate" - threshold: 99 - interval: 1m - query: | - 100 - sum( - rate( - http_request_duration_seconds_count{ - kubernetes_namespace="test", - kubernetes_pod_name=~"podinfo-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)", - path="root", - status!~"5.*" - }[1m] - ) - ) - / - sum( - rate( - http_request_duration_seconds_count{ - kubernetes_namespace="test", - kubernetes_pod_name=~"podinfo-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)", - path="root" - }[1m] - ) - ) - * 100 + - name: error-rate + templateRef: + name: error-rate + namespace: ingress-nginx + thresholdRange: + max: 1 + interval: 30s + - name: latency + templateRef: + name: latency + namespace: ingress-nginx + thresholdRange: + max: 0.5 + interval: 30s webhooks: - - name: pre - type: pre-rollout + - name: test-header-routing + type: rollout url: http://flagger-loadtester.test/ timeout: 5s metadata: - type: cmd - cmd: "hey -z 10m -q 10 -c 2 -H 'X-Canary: insider' -host app.example.com http://nginx-ingress-controller.ingress-nginx" - logCmdOutput: "true" - - name: post - type: post-rollout + type: bash + cmd: "curl -sH 'x-user: insider' -H 'Host: app.example.com' http://nginx-ingress-controller.ingress-nginx | grep '3.1.2'" + - name: load-test + type: rollout url: http://flagger-loadtester.test/ - timeout: 15s metadata: type: cmd - cmd: "curl -sH 'Host: app.example.com' http://nginx-ingress-controller.ingress-nginx" - logCmdOutput: "true" + cmd: "hey -z 2m -q 10 -c 2 -H 'x-user: insider' -host app.example.com http://nginx-ingress-controller.ingress-nginx" EOF echo '>>> Triggering A/B testing' kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.2 echo '>>> Waiting for A/B testing promotion' -retries=50 +retries=6 count=0 ok=false until ${ok}; do kubectl -n test describe deployment/podinfo-primary | grep '3.1.2' && ok=true || ok=false - sleep 10 + sleep 30 kubectl -n ingress-nginx logs deployment/flagger --tail 1 count=$(($count + 1)) if [[ ${count} -eq ${retries} ]]; then @@ -255,4 +269,4 @@ echo '✔ A/B testing promotion test passed' kubectl -n ingress-nginx logs deployment/flagger -echo '✔ All tests passed' \ No newline at end of file +echo '✔ All tests passed' diff --git a/test/e2e-nginx.sh b/test/e2e-nginx.sh index eb7b8e8d..c0f4ee95 100755 --- a/test/e2e-nginx.sh +++ b/test/e2e-nginx.sh @@ -3,7 +3,7 @@ set -o errexit REPO_ROOT=$(git rev-parse --show-toplevel) -NGINX_HELM_VERSION=1.34.2 # ingress v0.30.0 +NGINX_HELM_VERSION=1.34.3 # ingress v0.30.0 echo '>>> Installing NGINX Ingress' kubectl create ns ingress-nginx From e6901467f249f9c4718c8c65304221e63400fc05 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Thu, 2 Apr 2020 16:43:15 +0300 Subject: [PATCH 4/4] Add Prometheus Operator to docs index --- docs/gitbook/SUMMARY.md | 2 +- docs/gitbook/tutorials/prometheus-operator.md | 135 ++++++++++-------- 2 files changed, 75 insertions(+), 62 deletions(-) diff --git a/docs/gitbook/SUMMARY.md b/docs/gitbook/SUMMARY.md index d85459fc..9f565064 100644 --- a/docs/gitbook/SUMMARY.md +++ b/docs/gitbook/SUMMARY.md @@ -29,7 +29,7 @@ * [Contour Canary Deployments](tutorials/contour-progressive-delivery.md) * [Blue/Green Deployments](tutorials/kubernetes-blue-green.md) * [Crossover Canary Deployments](tutorials/crossover-progressive-delivery.md) -* [SMI Istio Canary Deployments](tutorials/flagger-smi-istio.md) +* [Canary analysis with Prometheus Operator](tutorials/prometheus-operator.md) * [Canaries with Helm charts and GitOps](tutorials/canary-helm-gitops.md) * [Zero downtime deployments](tutorials/zero-downtime-deployments.md) diff --git a/docs/gitbook/tutorials/prometheus-operator.md b/docs/gitbook/tutorials/prometheus-operator.md index 5b8ab25f..ceb4bf88 100644 --- a/docs/gitbook/tutorials/prometheus-operator.md +++ b/docs/gitbook/tutorials/prometheus-operator.md @@ -1,17 +1,14 @@ -# Flagger with Prometheus Operator +# Canary analysis with Prometheus Operator -This guide will show you how to use Flagger and Prometheus Operator. -This guide will handle only Blue/Green Deployment with podinfo application +This guide show you how to use Prometheus Operator for canary analysis. ## Prerequisites -Flagger and Prometheus Operator requires a Kubernetes cluster **v1.11** or newer - -Install Prometheus-Operator with Helm v3: +Install Prometheus Operator with Helm v3: ```bash helm repo add stable https://kubernetes-charts.storage.googleapis.com -helm repo update + kubectl create ns monitoring helm upgrade -i prometheus stable/prometheus-operator \ --namespace monitoring \ @@ -19,60 +16,54 @@ helm upgrade -i prometheus stable/prometheus-operator \ --set fullnameOverride=prometheus ``` -The `prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false` option allows Prometheus-Operator to watch serviceMonitor outside of his namespace. +The `prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false` +option allows Prometheus operator to watch serviceMonitors outside of his namespace. -You can also set `prometheus.service.type=nodePort` if you want to have access the Prometheus UI - -Install Flagger with Helm v3: +Install Flagger by setting the metrics server to Prometheus: ```bash helm repo add flagger https://flagger.app -helm repo update -kubectl create ns flagger + +kubectl create ns flagger-system helm upgrade -i flagger flagger/flagger \ ---namespace flagger \ +--namespace flagger-system \ --set metricsServer=http://prometheus-prometheus.monitoring:9090 \ --set meshProvider=kubernetes ``` -The `meshProvider` option can be changed to your value, if you want to do something else than Blue/Green Deployment - -Install Flagger Loadtester with Helm v3: +Install Flagger's tester: ```bash -helm repo add flagger https://flagger.app -helm repo update -kubectl create ns flagger helm upgrade -i loadtester flagger/loadtester \ ---namespace flagger +--namespace flagger-system ``` -Install podinfo with Helm v3: +Install podinfo demo app: ```bash -helm repo add sp https://stefanprodan.github.io/podinfo -helm repo update +helm repo add podinfo https://stefanprodan.github.io/podinfo + kubectl create ns test -helm upgrade -i podinfo sp/podinfo \ ---namespace test +helm upgrade -i podinfo podinfo/podinfo \ +--namespace test \ +--set service.enabled=false ``` -## Setting ServiceMonitor +## Service monitors -Prometheus Operator is using mostly serviceMonitor instead of annotations. -In order to catch metrics for primary and canary service, you will need to create 2 serviceMonitors : +The demo app is instrumented with Prometheus so you can create service monitors to scrape podinfo's metrics endpoint: ```yaml apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: - name: podinfo + name: podinfo-primary namespace: test spec: endpoints: - path: /metrics port: http - interval: 15s + interval: 5s selector: matchLabels: app: podinfo @@ -88,31 +79,31 @@ spec: endpoints: - path: /metrics port: http - interval: 15s + interval: 5s selector: matchLabels: app: podinfo-canary ``` -We are setting `interval: 15s` to have a more aggressive scraping -If you do not define it, you must to use a longer interval in the Canary object +We are setting `interval: 5s` to have a more aggressive scraping. +If you do not define it, you must to use a longer interval in the Canary object. -## Setting Custom metrics +## Metric templates -Prometheus Operator is relabeling for every serviceMonitor, you can create custom metrics to you own filter. +Create a metric template to measure the HTTP requests error rate: ```yaml apiVersion: flagger.app/v1beta1 kind: MetricTemplate metadata: - name: request-success-rate + name: error-rate namespace: test spec: provider: address: http://prometheus-prometheus.monitoring:9090 type: prometheus query: | - rate( + 100 - rate( http_requests_total{ namespace="{{ namespace }}", job="{{ target }}-canary", @@ -127,9 +118,34 @@ spec: ) * 100 ``` -You can also use `pod="{{ target }}-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)"` instead of `job={{ target }}-canary`, if you want. +Amd a metric template to measure the HTTP requests average duration: -## Creating Canary +```yaml +apiVersion: flagger.app/v1beta1 +kind: MetricTemplate +metadata: + name: latency + namespace: test +spec: + provider: + address: http://prometheus-prometheus.monitoring:9090 + type: prometheus + query: | + histogram_quantile(0.99, + sum( + rate( + http_request_duration_seconds_bucket{ + namespace="{{ namespace }}", + job="{{ target }}-canary" + }[{{ interval }}] + ) + ) by (le) + ) +``` + +## Canary analysis + +Using the metrics template you can configure the canary analysis with HTTP error rate and latency checks: ```yaml apiVersion: flagger.app/v1beta1 @@ -145,40 +161,37 @@ spec: name: podinfo progressDeadlineSeconds: 60 service: - port: 9898 - portDiscovery: true + port: 80 + targetPort: http + name: podinfo analysis: interval: 30s iterations: 10 threshold: 2 metrics: - - name: http-success-rate + - name: error-rate templateRef: - name: request-success-rate - namespace: test + name: error-rate thresholdRange: - min: 99 - interval: 1m + max: 1 + interval: 30s + - name: latency + templateRef: + name: latency + thresholdRange: + max: 0.5 + interval: 30s webhooks: - - name: smoke-test - type: pre-rollout - url: "http://loadtester.flagger/" - timeout: 15s - metadata: - type: bash - cmd: "curl -sd 'anon' http://podinfo-canary.test:9898/token | grep token" - name: load-test type: rollout - url: "http://loadtester.flagger/" + url: "http://loadtester.flagger-system/" timeout: 5s metadata: type: cmd - cmd: "hey -z 1m -q 10 -c 2 http://podinfo-canary.test:9898" + cmd: "hey -z 1m -q 10 -c 2 http://podinfo-canary.test/" ``` -## Test the canary - -Execute `kubectl -n test set image deployment/podinfo podinfo=stefanprodan/podinfo:3.1.0` to see if everything works - - +Based on the above specification, Flagger creates the primary and canary Kubernetes ClusterIP service. +During the canary analysis, Prometheus will scrape the canary service and Flagger will use the HTTP error rate and +latency queries to determine if the release should be promoted or rolled back.