From 2b1aacc8e33ce3c5e9ed123ae2c31ab51352aad8 Mon Sep 17 00:00:00 2001 From: Arnaud Hatzenbuhler Date: Thu, 26 Mar 2020 11:15:32 +0100 Subject: [PATCH 1/4] Add documentation for prometheus-operator --- docs/gitbook/tutorials/prometheus-operator.md | 144 ++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 docs/gitbook/tutorials/prometheus-operator.md diff --git a/docs/gitbook/tutorials/prometheus-operator.md b/docs/gitbook/tutorials/prometheus-operator.md new file mode 100644 index 00000000..8c4e3bff --- /dev/null +++ b/docs/gitbook/tutorials/prometheus-operator.md @@ -0,0 +1,144 @@ +# Flagger with Prometheus Operator + +This guide will show you how to use Flagger and Prometheus Operator +This guide will handle only Blue/Green Deployment + +## Prerequisites + +Flagger requires a Kubernetes cluster **v1.11** or newer and NGINX ingress **0.24** or newer. + +Install Prometheus-Operator with Helm v3: + +```bash +helm repo add stable https://kubernetes-charts.storage.googleapis.com +helm repo update +kubectl create ns prometheus +helm upgrade -i prometheus stable/prometheus-operator \ +--namespace prometheus \ +--set prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValue=false +``` +The `prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValue` allows Prometheus-Operator to watch serviceMonitor outside of his namespace. + +Install Flagger with Helm v3: + +```bash +helm repo add flagger https://flagger.app +helm repo update +kubectl create ns flagger +helm upgrade -i flagger flagger/flagger \ +--namespace flagger +--set prometheus-prometheus-oper-prometheus.prometheus:9090 \ +--set meshProvider=kubernetes +``` + +The `meshProvider` option can be changed to your value + +## Setting ServiceMonitor + +Prometheus Operator is using mostly serviceMonitor instead of annotations. +In order to catch metrics for primary and canary service, you will need to create 2 serviceMonitors : + +```yaml +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: service-monitor-primary + namespace: application +spec: + endpoints: + - path: /metrics + port: http + selector: + matchLabels: + app: application +``` + +```yaml +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: service-monitor-canary + namespace: application +spec: + endpoints: + - path: /metrics + port: http + selector: + matchLabels: + app: application-canary +``` + +## Setting Custom metrics + +Prometheus Operator is relabeling for every serviceMonitor, you can create custom metrics + +```yaml +apiVersion: flagger.app/v1beta1 +kind: MetricTemplate +metadata: + name: your-custom-metrics + namespace: application +spec: + provider: + address: http://prometheus-prometheus-oper-prometheus.prometheus:9090 + type: prometheus + query: | + 100 - sum( + rate( + http_server_requests_seconds_max{ + namespace="{{namespace }}", + job="{{ target }}-canary", + status!~\"5.*\" + }[{{ interval }}] + ) + ) + / + sum( + rate( + http_server_requests_seconds_max{ + namespace="{{ namespace }}", + job="{{ target }}-canary" + }[{{ interval }}] + ) + ) + * 100 +``` + +You can use the job to filter instead of pod. You can tune it to your installation. + +## Creating Canary + +```yaml +apiVersion: flagger.app/v1beta1 +kind: Canary +metadata: + name: application + namespace: security-manager +spec: + analysis: + interval: 30s + iterations: 10 + threshold: 5 + metrics: + - interval: 1m + name: http-requests-seconds + templateRef: + name: your-custom-metrics + namespace: application + thresholdRange: + max: 1 + webhooks: [] + progressDeadlineSeconds: 600 + provider: kubernetes + service: + port: 8080 + portDiscovery: true + targetRef: + apiVersion: v1 + kind: Deployment + name: application +``` + +You can define the webhooks of your liking also + + From 37c7ddc9a4a1631c5890edeb6d0f91717b2476c5 Mon Sep 17 00:00:00 2001 From: Arnaud Hatzenbuhler Date: Thu, 26 Mar 2020 17:03:38 +0100 Subject: [PATCH 2/4] Rewrite documentation to use podinfo as exemple --- docs/gitbook/tutorials/prometheus-operator.md | 144 +++++++++++------- 1 file changed, 86 insertions(+), 58 deletions(-) diff --git a/docs/gitbook/tutorials/prometheus-operator.md b/docs/gitbook/tutorials/prometheus-operator.md index 8c4e3bff..a359ed7c 100644 --- a/docs/gitbook/tutorials/prometheus-operator.md +++ b/docs/gitbook/tutorials/prometheus-operator.md @@ -1,11 +1,11 @@ # Flagger with Prometheus Operator -This guide will show you how to use Flagger and Prometheus Operator -This guide will handle only Blue/Green Deployment +This guide will show you how to use Flagger and Prometheus Operator. +This guide will handle only Blue/Green Deployment with podinfo application ## Prerequisites -Flagger requires a Kubernetes cluster **v1.11** or newer and NGINX ingress **0.24** or newer. +Flagger and Prometheus Operator requires a Kubernetes cluster **v1.11** or newer Install Prometheus-Operator with Helm v3: @@ -15,9 +15,12 @@ helm repo update kubectl create ns prometheus helm upgrade -i prometheus stable/prometheus-operator \ --namespace prometheus \ ---set prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValue=false +--set prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false ``` -The `prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValue` allows Prometheus-Operator to watch serviceMonitor outside of his namespace. + +The `prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false` option allows Prometheus-Operator to watch serviceMonitor outside of his namespace. + +You can also set `prometheus.service.type=nodePort` if you want to have Install Flagger with Helm v3: @@ -26,85 +29,91 @@ helm repo add flagger https://flagger.app helm repo update kubectl create ns flagger helm upgrade -i flagger flagger/flagger \ ---namespace flagger ---set prometheus-prometheus-oper-prometheus.prometheus:9090 \ +--namespace flagger \ +--set metricsServer=http://prometheus-prometheus-oper-prometheus.prometheus:9090 \ --set meshProvider=kubernetes ``` -The `meshProvider` option can be changed to your value +The `meshProvider` option can be changed to your value, if you want to do something else than Blue/Green Deployment + +Install Flagger Loadtester with Helm v3: + +```bash +helm repo add flagger https://flagger.app +helm repo update +kubectl create ns flagger +helm upgrade -i loadtester flagger/loadtester \ +--namespace flagger +``` + +Install podinfo with Helm v3: + +```bash +helm repo add sp https://stefanprodan.github.io/podinfo +helm repo update +kubectl create ns test +helm upgrade -i podinfo sp/podinfo \ +--namespace test +``` ## Setting ServiceMonitor -Prometheus Operator is using mostly serviceMonitor instead of annotations. +Prometheus Operator is using mostly serviceMonitor instead of annotations. In order to catch metrics for primary and canary service, you will need to create 2 serviceMonitors : ```yaml apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: - name: service-monitor-primary - namespace: application + name: podinfo + namespace: test spec: endpoints: - path: /metrics port: http + interval: 15s selector: matchLabels: - app: application + app: podinfo ``` ```yaml apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: - name: service-monitor-canary - namespace: application + name: podinfo-canary + namespace: test spec: endpoints: - path: /metrics port: http + interval: 15s selector: matchLabels: - app: application-canary + app: podinfo-canary ``` +We are setting `interval: 15s` to have a more aggressive scraping +If you do not define it, you must to use a longer interval in the Canary object + ## Setting Custom metrics -Prometheus Operator is relabeling for every serviceMonitor, you can create custom metrics +Prometheus Operator is relabeling for every serviceMonitor, you can create custom metrics to you own filter. ```yaml apiVersion: flagger.app/v1beta1 kind: MetricTemplate metadata: - name: your-custom-metrics - namespace: application + name: request-success-rate + namespace: test spec: provider: address: http://prometheus-prometheus-oper-prometheus.prometheus:9090 type: prometheus - query: | - 100 - sum( - rate( - http_server_requests_seconds_max{ - namespace="{{namespace }}", - job="{{ target }}-canary", - status!~\"5.*\" - }[{{ interval }}] - ) - ) - / - sum( - rate( - http_server_requests_seconds_max{ - namespace="{{ namespace }}", - job="{{ target }}-canary" - }[{{ interval }}] - ) - ) - * 100 + query: rate(http_requests_total{namespace="{{ namespace }}",job="{{ target }}-canary",status!~"5.*"}[{{ interval }}]) / rate(http_requests_total{namespace="{{ namespace }}",job="{{ target }}-canary"}[{{ interval }}]) * 100 ``` -You can use the job to filter instead of pod. You can tune it to your installation. +You can also use `pod="{{ target }}-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)"` instead of `job={{ target }}-canary`, if you want. ## Creating Canary @@ -112,33 +121,52 @@ You can use the job to filter instead of pod. You can tune it to your installati apiVersion: flagger.app/v1beta1 kind: Canary metadata: - name: application - namespace: security-manager + name: podinfo + namespace: test spec: + provider: kubernetes + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: podinfo + progressDeadlineSeconds: 60 + service: + port: 9898 + portDiscovery: true analysis: + # We set a longer interval to let Prometheus fetch metrics interval: 30s iterations: 10 - threshold: 5 + threshold: 2 metrics: - - interval: 1m - name: http-requests-seconds + # For some reason, you need to not use a standard name + - name: my-custom-metrics templateRef: - name: your-custom-metrics - namespace: application + name: request-success-rate + namespace: test thresholdRange: - max: 1 - webhooks: [] - progressDeadlineSeconds: 600 - provider: kubernetes - service: - port: 8080 - portDiscovery: true - targetRef: - apiVersion: v1 - kind: Deployment - name: application + min: 99 + interval: 1m + webhooks: + - name: smoke-test + type: pre-rollout + url: "http://loadtester.flagger/" + timeout: 15s + metadata: + type: bash + cmd: "curl -sd 'anon' http://podinfo-canary.test:9898/token | grep token" + - name: load-test + type: rollout + url: "http://loadtester.flagger/" + timeout: 5s + metadata: + type: cmd + cmd: "hey -z {{ interval }} -q 10 -c 2 http://podinfo-canary.test:9898" ``` -You can define the webhooks of your liking also +## Test the canary + +Execute `kubectl -n test set image deployment/podinfo podinfo=stefanprodan/podinfo:3.1.0` to see if everythinf works + From a548bfd8e6f172241ee4c3bd77f4bd05d0d1edff Mon Sep 17 00:00:00 2001 From: Arnaud Hatzenbuhler Date: Thu, 26 Mar 2020 17:08:10 +0100 Subject: [PATCH 3/4] Fix typo --- docs/gitbook/tutorials/prometheus-operator.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/gitbook/tutorials/prometheus-operator.md b/docs/gitbook/tutorials/prometheus-operator.md index a359ed7c..487e220b 100644 --- a/docs/gitbook/tutorials/prometheus-operator.md +++ b/docs/gitbook/tutorials/prometheus-operator.md @@ -166,7 +166,7 @@ spec: ## Test the canary -Execute `kubectl -n test set image deployment/podinfo podinfo=stefanprodan/podinfo:3.1.0` to see if everythinf works +Execute `kubectl -n test set image deployment/podinfo podinfo=stefanprodan/podinfo:3.1.0` to see if everything works From 42c3080c196a94b1544c7ab942d6831bbb51e626 Mon Sep 17 00:00:00 2001 From: Arnaud Hatzenbuhler Date: Thu, 26 Mar 2020 17:54:04 +0100 Subject: [PATCH 4/4] Commit requested changes and fix some typos --- docs/gitbook/tutorials/prometheus-operator.md | 34 +++++++++++++------ 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/docs/gitbook/tutorials/prometheus-operator.md b/docs/gitbook/tutorials/prometheus-operator.md index 487e220b..5b8ab25f 100644 --- a/docs/gitbook/tutorials/prometheus-operator.md +++ b/docs/gitbook/tutorials/prometheus-operator.md @@ -12,15 +12,16 @@ Install Prometheus-Operator with Helm v3: ```bash helm repo add stable https://kubernetes-charts.storage.googleapis.com helm repo update -kubectl create ns prometheus +kubectl create ns monitoring helm upgrade -i prometheus stable/prometheus-operator \ ---namespace prometheus \ ---set prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false +--namespace monitoring \ +--set prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false \ +--set fullnameOverride=prometheus ``` The `prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false` option allows Prometheus-Operator to watch serviceMonitor outside of his namespace. -You can also set `prometheus.service.type=nodePort` if you want to have +You can also set `prometheus.service.type=nodePort` if you want to have access the Prometheus UI Install Flagger with Helm v3: @@ -30,7 +31,7 @@ helm repo update kubectl create ns flagger helm upgrade -i flagger flagger/flagger \ --namespace flagger \ ---set metricsServer=http://prometheus-prometheus-oper-prometheus.prometheus:9090 \ +--set metricsServer=http://prometheus-prometheus.monitoring:9090 \ --set meshProvider=kubernetes ``` @@ -108,9 +109,22 @@ metadata: namespace: test spec: provider: - address: http://prometheus-prometheus-oper-prometheus.prometheus:9090 + address: http://prometheus-prometheus.monitoring:9090 type: prometheus - query: rate(http_requests_total{namespace="{{ namespace }}",job="{{ target }}-canary",status!~"5.*"}[{{ interval }}]) / rate(http_requests_total{namespace="{{ namespace }}",job="{{ target }}-canary"}[{{ interval }}]) * 100 + query: | + rate( + http_requests_total{ + namespace="{{ namespace }}", + job="{{ target }}-canary", + status!~"5.*" + }[{{ interval }}]) + / + rate( + http_requests_total{ + namespace="{{ namespace }}", + job="{{ target }}-canary" + }[{{ interval }}] + ) * 100 ``` You can also use `pod="{{ target }}-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)"` instead of `job={{ target }}-canary`, if you want. @@ -134,13 +148,11 @@ spec: port: 9898 portDiscovery: true analysis: - # We set a longer interval to let Prometheus fetch metrics interval: 30s iterations: 10 threshold: 2 metrics: - # For some reason, you need to not use a standard name - - name: my-custom-metrics + - name: http-success-rate templateRef: name: request-success-rate namespace: test @@ -161,7 +173,7 @@ spec: timeout: 5s metadata: type: cmd - cmd: "hey -z {{ interval }} -q 10 -c 2 http://podinfo-canary.test:9898" + cmd: "hey -z 1m -q 10 -c 2 http://podinfo-canary.test:9898" ``` ## Test the canary