github/flagger
1
0
Fork 0
mirror of https://github.com/fluxcd/flagger.git synced 2026-04-15 06:57:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix envoy dashboard, scheduler, and envoy metrics provider to correctly pass canary analysis and show graphs

Browse Source
This commit is contained in:
Yusuke Kuoka
2019-12-18 10:55:49 +09:00
parent 806b95c8ce
commit d75ade5e8c
5 changed files with 41 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files
charts/grafana/dashboards/envoy.json
+22 -44
View File
@@ -20,7 +20,7 @@
"links": [],
"panels": [
{
"content": "<div class=\"dashboard-header text-center\">\n<span>RED: $canary.$namespace</span>\n</div>",
"content": "<div class=\"dashboard-header text-center\">\n<span>RED: $target.$namespace</span>\n</div>",
"gridPos": {
"h": 3,
"w": 24,
@@ -95,7 +95,7 @@
"tableColumn": "",
"targets": [
{
"expr": "round(sum(rate(envoy_cluster_upstream_rq{kubernetes_namespace=~\"$namespace\",app=~\"$primary\"}[30s])), 0.001)",
"expr": "round(sum(rate(envoy_cluster_upstream_rq{kubernetes_namespace=~\"$namespace\",envoy_cluster_name=~\"$target-primary\"}[30s])), 0.001)",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "",
@@ -179,7 +179,7 @@
"tableColumn": "",
"targets": [
{
"expr": "sum(irate(envoy_cluster_upstream_rq{kubernetes_namespace=~\"$namespace\",app=~\"$primary\",envoy_response_code!~\"5.*\"}[30s])) / sum(irate(envoy_cluster_upstream_rq{kubernetes_namespace=~\"$namespace\",app=~\"$primary\"}[30s]))",
"expr": "sum(irate(envoy_cluster_upstream_rq{kubernetes_namespace=~\"$namespace\",envoy_cluster_name=~\"$target-primary\",envoy_response_code!~\"5.*\"}[30s])) / sum(irate(envoy_cluster_upstream_rq{kubernetes_namespace=~\"$namespace\",envoy_cluster_name=~\"$target-primary\"}[30s]))",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "",
@@ -261,7 +261,7 @@
"tableColumn": "",
"targets": [
{
"expr": "round(sum(rate(envoy_cluster_upstream_rq{kubernetes_namespace=~\"$namespace\",app=~\"$canary\"}[30s])), 0.001)",
"expr": "round(sum(rate(envoy_cluster_upstream_rq{kubernetes_namespace=~\"$namespace\",envoy_cluster_name=~\"$target-canary\"}[30s])), 0.001)",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "",
@@ -345,7 +345,7 @@
"tableColumn": "",
"targets": [
{
"expr": "sum(irate(envoy_cluster_upstream_rq{kubernetes_namespace=~\"$namespace\",app=~\"$canary\",envoy_response_code!~\"5.*\"}[30s])) / sum(irate(envoy_cluster_upstream_rq{kubernetes_namespace=~\"$namespace\",app=~\"$canary\"}[30s]))",
"expr": "sum(irate(envoy_cluster_upstream_rq{kubernetes_namespace=~\"$namespace\",envoy_cluster_name=~\"$target-canary\",envoy_response_code!~\"5.*\"}[30s])) / sum(irate(envoy_cluster_upstream_rq{kubernetes_namespace=~\"$namespace\",envoy_cluster_name=~\"$target-canary\"}[30s]))",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "",
@@ -407,7 +407,7 @@
"steppedLine": false,
"targets": [
{
"expr": "(sum(rate(envoy_cluster_upstream_cx_rx_bytes_total{kubernetes_namespace=~\"$namespace\",app=~\"$primary\"}[30s])))",
"expr": "(sum(rate(envoy_cluster_upstream_cx_rx_bytes_total{kubernetes_namespace=~\"$namespace\",envoy_cluster_name=~\"$target-primary\"}[30s])))",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
@@ -497,7 +497,7 @@
"steppedLine": false,
"targets": [
{
"expr": "(sum(rate(envoy_cluster_upstream_cx_rx_bytes_total{kubernetes_namespace=~\"$namespace\",app=~\"$canary\"}[30s])))",
"expr": "(sum(rate(envoy_cluster_upstream_cx_rx_bytes_total{kubernetes_namespace=~\"$namespace\",envoy_cluster_name=~\"$target-canary\"}[30s])))",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
@@ -547,7 +547,7 @@
}
},
{
"content": "<div class=\"dashboard-header text-center\">\n<span>USE: $canary.$namespace</span>\n</div>",
"content": "<div class=\"dashboard-header text-center\">\n<span>USE: $target.$namespace</span>\n</div>",
"gridPos": {
"h": 3,
"w": 24,
@@ -602,7 +602,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum(rate(container_cpu_usage_seconds_total{cpu=\"total\",namespace=\"$namespace\",pod_name=~\"$primary.*\", container_name!~\"POD|istio-proxy\"}[1m])) by (pod_name)",
"expr": "sum(rate(container_cpu_usage_seconds_total{cpu=\"total\",namespace=\"$namespace\",pod_name=~\"$target-primary.*\", container_name!~\"POD|istio-proxy\"}[1m])) by (pod_name)",
"format": "time_series",
"hide": false,
"intervalFactor": 1,
@@ -692,7 +692,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum(rate(container_cpu_usage_seconds_total{cpu=\"total\",namespace=\"$namespace\",pod_name=~\"$canary.*\", pod_name!~\"$primary.*\", container_name!~\"POD|istio-proxy\"}[1m])) by (pod_name)",
"expr": "sum(rate(container_cpu_usage_seconds_total{cpu=\"total\",namespace=\"$namespace\",pod_name=~\"$target.*\", pod_name!~\"$target-primary.*\", container_name!~\"POD|istio-proxy\"}[1m])) by (pod_name)",
"format": "time_series",
"hide": false,
"intervalFactor": 1,
@@ -782,7 +782,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\",pod_name=~\"$primary.*\", container_name!~\"POD|istio-proxy\"}) by (pod_name)",
"expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\",pod_name=~\"$target-primary.*\", container_name!~\"POD|istio-proxy\"}) by (pod_name)",
"format": "time_series",
"hide": false,
"interval": "",
@@ -874,7 +874,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\",pod_name=~\"$canary.*\", pod_name!~\"$primary.*\", container_name!~\"POD|istio-proxy\"}) by (pod_name)",
"expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\",pod_name=~\"$target.*\", pod_name!~\"$target-primary.*\", container_name!~\"POD|istio-proxy\"}) by (pod_name)",
"format": "time_series",
"hide": false,
"interval": "",
@@ -975,14 +975,14 @@
"steppedLine": false,
"targets": [
{
"expr": "sum(rate (container_network_receive_bytes_total{namespace=\"$namespace\",pod_name=~\"$primary.*\"}[1m])) ",
"expr": "sum(rate (container_network_receive_bytes_total{namespace=\"$namespace\",pod_name=~\"$target-primary.*\"}[1m])) ",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "received",
"refId": "A"
},
{
"expr": "-sum (rate (container_network_transmit_bytes_total{namespace=\"$namespace\",pod_name=~\"$primary.*\"}[1m]))",
"expr": "-sum (rate (container_network_transmit_bytes_total{namespace=\"$namespace\",pod_name=~\"$target-primary.*\"}[1m]))",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "transmited",
@@ -1081,14 +1081,14 @@
"steppedLine": false,
"targets": [
{
"expr": "sum(rate (container_network_receive_bytes_total{namespace=\"$namespace\",pod_name=~\"$canary.*\",pod_name!~\"$primary.*\"}[1m])) ",
"expr": "sum(rate (container_network_receive_bytes_total{namespace=\"$namespace\",pod_name=~\"$target.*\",pod_name!~\"$target-primary.*\"}[1m])) ",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "received",
"refId": "A"
},
{
"expr": "-sum (rate (container_network_transmit_bytes_total{namespace=\"$namespace\",pod_name=~\"$canary.*\",pod_name!~\"$primary.*\"}[1m]))",
"expr": "-sum (rate (container_network_transmit_bytes_total{namespace=\"$namespace\",pod_name=~\"$target.*\",pod_name!~\"$target-primary.*\"}[1m]))",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "transmited",
@@ -1170,38 +1170,16 @@
"allValue": null,
"current": null,
"datasource": "prometheus",
"definition": "query_result(sum(envoy_cluster_upstream_rq{kubernetes_namespace=\"$namespace\",app=~\".*-primary\"}) by (app))",
"definition": "query_result(sum(envoy_cluster_upstream_rq{kubernetes_namespace=\"$namespace\",envoy_cluster_name=~\".*-primary\"}) by (envoy_cluster_name))",
"hide": 0,
"includeAll": false,
"label": "Primary",
"label": "Target",
"multi": false,
"name": "primary",
"name": "target",
"options": [],
"query": "query_result(sum(envoy_cluster_upstream_rq{kubernetes_namespace=\"$namespace\",app=~\".*-primary\"}) by (app))",
"query": "query_result(sum(envoy_cluster_upstream_rq{kubernetes_namespace=\"$namespace\",envoy_cluster_name=~\".*-primary\"}) by (envoy_cluster_name))",
"refresh": 1,
"regex": "/.*app=\"([^\"]*).*/",
"skipUrlSync": false,
"sort": 1,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"allValue": null,
"current": null,
"datasource": "prometheus",
"definition": "query_result(sum(envoy_cluster_upstream_rq{kubernetes_namespace=\"$namespace\",app!~\".*-primary\"}) by (app))",
"hide": 0,
"includeAll": false,
"label": "Canary",
"multi": false,
"name": "canary",
"options": [],
"query": "query_result(sum(envoy_cluster_upstream_rq{kubernetes_namespace=\"$namespace\",app!~\".*-primary\"}) by (app))",
"refresh": 1,
"regex": "/.*app=\"([^\"]*).*/",
"regex": "/.*envoy_cluster_name=\"(.*)-primary\"/",
"skipUrlSync": false,
"sort": 1,
"tagValuesQuery": "",
@@ -1244,5 +1222,5 @@
"timezone": "",
"title": "Envoy Canary",
"uid": "flagger-envoy",
"version": 4
"version": 6
}
docs/gitbook/usage/envoy-progressive-delivery.md
+1 -1
View File
@@ -243,7 +243,7 @@ kubectl port-forward --namespace test svc/flagger-grafana 3000:80
```
The Envoy dashboard URL is
http://localhost:3000/d/flagger-envoy/envoy-canary?refresh=10s&orgId=1&var-namespace=test&var-primary=podinfo-primary&var-canary=podinfo
http://localhost:3000/d/flagger-envoy/envoy-canary?refresh=10s&orgId=1&var-namespace=test&var-target=podinfo
![Envoy Canary Dashboard](https://raw.githubusercontent.com/weaveworks/flagger/master/docs/screens/flagger-grafana-appmesh.png)
pkg/controller/scheduler.go
+13 -10
View File
@@ -747,7 +747,15 @@ func (c *Controller) analyseCanary(r *flaggerv1.Canary) bool {
}
// override the global provider if one is specified in the canary spec
metricsProvider := c.meshProvider
var metricsProvider string
// set the metrics provider to Envoy Prometheus when Envoy is the mesh provider
// For example, `envoy` metrics provider should be used for `smi:envoy` mesh provider
if strings.Contains(c.meshProvider, "envoy") {
metricsProvider = "envoy"
} else {
metricsProvider = c.meshProvider
}
if r.Spec.Provider != "" {
metricsProvider = r.Spec.Provider
@@ -755,11 +763,6 @@ func (c *Controller) analyseCanary(r *flaggerv1.Canary) bool {
if strings.Contains(c.meshProvider, "linkerd") {
metricsProvider = "linkerd"
}
// set the metrics provider to Envoy Prometheus when Envoy is the default mesh provider
if strings.Contains(c.meshProvider, "envoy") {
metricsProvider = "envoy"
}
}
// set the metrics provider to query Prometheus for the canary Kubernetes service if the canary target is Service
if r.Spec.TargetRef.Kind == "Service" {
@@ -792,8 +795,8 @@ func (c *Controller) analyseCanary(r *flaggerv1.Canary) bool {
val, err := observer.GetRequestSuccessRate(r.Spec.TargetRef.Name, r.Namespace, metric.Interval)
if err != nil {
if strings.Contains(err.Error(), "no values found") {
c.recordEventWarningf(r, "Halt advancement no values found for metric %s probably %s.%s is not receiving traffic",
metric.Name, r.Spec.TargetRef.Name, r.Namespace)
c.recordEventWarningf(r, "Halt advancement no values found for %s metric %s probably %s.%s is not receiving traffic",
metricsProvider, metric.Name, r.Spec.TargetRef.Name, r.Namespace)
} else {
c.recordEventErrorf(r, "Metrics server %s query failed: %v", metricsServer, err)
}
@@ -812,8 +815,8 @@ func (c *Controller) analyseCanary(r *flaggerv1.Canary) bool {
val, err := observer.GetRequestDuration(r.Spec.TargetRef.Name, r.Namespace, metric.Interval)
if err != nil {
if strings.Contains(err.Error(), "no values found") {
c.recordEventWarningf(r, "Halt advancement no values found for metric %s probably %s.%s is not receiving traffic",
metric.Name, r.Spec.TargetRef.Name, r.Namespace)
c.recordEventWarningf(r, "Halt advancement no values found for %s metric %s probably %s.%s is not receiving traffic",
metricsProvider, metric.Name, r.Spec.TargetRef.Name, r.Namespace)
} else {
c.recordEventErrorf(r, "Metrics server %s query failed: %v", metricsServer, err)
}
pkg/metrics/envoy.go
+3 -3
View File
@@ -10,7 +10,7 @@ var envoyQueries = map[string]string{
rate(
envoy_cluster_upstream_rq{
kubernetes_namespace="{{ .Namespace }}",
envoy_cluster_name=~"{{ .Name }}-canary-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)",
envoy_cluster_name=~"{{ .Name }}-canary",
envoy_response_code!~"5.*"
}[{{ .Interval }}]
)
@@ -20,7 +20,7 @@ var envoyQueries = map[string]string{
rate(
envoy_cluster_upstream_rq{
kubernetes_namespace="{{ .Namespace }}",
envoy_cluster_name=~"{{ .Name }}-canary-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)"
envoy_cluster_name=~"{{ .Name }}-canary"
}[{{ .Interval }}]
)
)
@@ -32,7 +32,7 @@ var envoyQueries = map[string]string{
rate(
envoy_cluster_upstream_rq_time_bucket{
kubernetes_namespace="{{ .Namespace }}",
envoy_cluster_name=~"{{ .Name }}-canary-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)"
envoy_cluster_name=~"{{ .Name }}-canary"
}[{{ .Interval }}]
)
) by (le)
pkg/metrics/envoy_test.go
+2 -2
View File
@@ -8,7 +8,7 @@ import (
)
func TestEnvoyObserver_GetRequestSuccessRate(t *testing.T) {
expected := ` sum( rate( envoy_cluster_upstream_rq{ kubernetes_namespace="default", envoy_cluster_name=~"podinfo-canary-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)", envoy_response_code!~"5.*" }[1m] ) ) / sum( rate( envoy_cluster_upstream_rq{ kubernetes_namespace="default", envoy_cluster_name=~"podinfo-canary-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)" }[1m] ) ) * 100`
expected := ` sum( rate( envoy_cluster_upstream_rq{ kubernetes_namespace="default", envoy_cluster_name=~"podinfo-canary", envoy_response_code!~"5.*" }[1m] ) ) / sum( rate( envoy_cluster_upstream_rq{ kubernetes_namespace="default", envoy_cluster_name=~"podinfo-canary" }[1m] ) ) * 100`
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
promql := r.URL.Query()["query"][0]
@@ -41,7 +41,7 @@ func TestEnvoyObserver_GetRequestSuccessRate(t *testing.T) {
}
func TestEnvoyObserver_GetRequestDuration(t *testing.T) {
expected := ` histogram_quantile( 0.99, sum( rate( envoy_cluster_upstream_rq_time_bucket{ kubernetes_namespace="default", envoy_cluster_name=~"podinfo-canary-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)" }[1m] ) ) by (le) )`
expected := ` histogram_quantile( 0.99, sum( rate( envoy_cluster_upstream_rq_time_bucket{ kubernetes_namespace="default", envoy_cluster_name=~"podinfo-canary" }[1m] ) ) by (le) )`
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
promql := r.URL.Query()["query"][0]