diff --git a/charts/grafana/dashboards/envoy.json b/charts/grafana/dashboards/envoy.json index de502323..46b27a66 100644 --- a/charts/grafana/dashboards/envoy.json +++ b/charts/grafana/dashboards/envoy.json @@ -20,7 +20,7 @@ "links": [], "panels": [ { - "content": "
\nRED: $canary.$namespace\n
", + "content": "
\nRED: $target.$namespace\n
", "gridPos": { "h": 3, "w": 24, @@ -95,7 +95,7 @@ "tableColumn": "", "targets": [ { - "expr": "round(sum(rate(envoy_cluster_upstream_rq{kubernetes_namespace=~\"$namespace\",app=~\"$primary\"}[30s])), 0.001)", + "expr": "round(sum(rate(envoy_cluster_upstream_rq{kubernetes_namespace=~\"$namespace\",envoy_cluster_name=~\"$target-primary\"}[30s])), 0.001)", "format": "time_series", "intervalFactor": 1, "legendFormat": "", @@ -179,7 +179,7 @@ "tableColumn": "", "targets": [ { - "expr": "sum(irate(envoy_cluster_upstream_rq{kubernetes_namespace=~\"$namespace\",app=~\"$primary\",envoy_response_code!~\"5.*\"}[30s])) / sum(irate(envoy_cluster_upstream_rq{kubernetes_namespace=~\"$namespace\",app=~\"$primary\"}[30s]))", + "expr": "sum(irate(envoy_cluster_upstream_rq{kubernetes_namespace=~\"$namespace\",envoy_cluster_name=~\"$target-primary\",envoy_response_code!~\"5.*\"}[30s])) / sum(irate(envoy_cluster_upstream_rq{kubernetes_namespace=~\"$namespace\",envoy_cluster_name=~\"$target-primary\"}[30s]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "", @@ -261,7 +261,7 @@ "tableColumn": "", "targets": [ { - "expr": "round(sum(rate(envoy_cluster_upstream_rq{kubernetes_namespace=~\"$namespace\",app=~\"$canary\"}[30s])), 0.001)", + "expr": "round(sum(rate(envoy_cluster_upstream_rq{kubernetes_namespace=~\"$namespace\",envoy_cluster_name=~\"$target-canary\"}[30s])), 0.001)", "format": "time_series", "intervalFactor": 1, "legendFormat": "", @@ -345,7 +345,7 @@ "tableColumn": "", "targets": [ { - "expr": "sum(irate(envoy_cluster_upstream_rq{kubernetes_namespace=~\"$namespace\",app=~\"$canary\",envoy_response_code!~\"5.*\"}[30s])) / sum(irate(envoy_cluster_upstream_rq{kubernetes_namespace=~\"$namespace\",app=~\"$canary\"}[30s]))", + "expr": "sum(irate(envoy_cluster_upstream_rq{kubernetes_namespace=~\"$namespace\",envoy_cluster_name=~\"$target-canary\",envoy_response_code!~\"5.*\"}[30s])) / sum(irate(envoy_cluster_upstream_rq{kubernetes_namespace=~\"$namespace\",envoy_cluster_name=~\"$target-canary\"}[30s]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "", @@ -407,7 +407,7 @@ "steppedLine": false, "targets": [ { - "expr": "(sum(rate(envoy_cluster_upstream_cx_rx_bytes_total{kubernetes_namespace=~\"$namespace\",app=~\"$primary\"}[30s])))", + "expr": "(sum(rate(envoy_cluster_upstream_cx_rx_bytes_total{kubernetes_namespace=~\"$namespace\",envoy_cluster_name=~\"$target-primary\"}[30s])))", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -497,7 +497,7 @@ "steppedLine": false, "targets": [ { - "expr": "(sum(rate(envoy_cluster_upstream_cx_rx_bytes_total{kubernetes_namespace=~\"$namespace\",app=~\"$canary\"}[30s])))", + "expr": "(sum(rate(envoy_cluster_upstream_cx_rx_bytes_total{kubernetes_namespace=~\"$namespace\",envoy_cluster_name=~\"$target-canary\"}[30s])))", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -547,7 +547,7 @@ } }, { - "content": "
\nUSE: $canary.$namespace\n
", + "content": "
\nUSE: $target.$namespace\n
", "gridPos": { "h": 3, "w": 24, @@ -602,7 +602,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(container_cpu_usage_seconds_total{cpu=\"total\",namespace=\"$namespace\",pod_name=~\"$primary.*\", container_name!~\"POD|istio-proxy\"}[1m])) by (pod_name)", + "expr": "sum(rate(container_cpu_usage_seconds_total{cpu=\"total\",namespace=\"$namespace\",pod_name=~\"$target-primary.*\", container_name!~\"POD|istio-proxy\"}[1m])) by (pod_name)", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -692,7 +692,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(container_cpu_usage_seconds_total{cpu=\"total\",namespace=\"$namespace\",pod_name=~\"$canary.*\", pod_name!~\"$primary.*\", container_name!~\"POD|istio-proxy\"}[1m])) by (pod_name)", + "expr": "sum(rate(container_cpu_usage_seconds_total{cpu=\"total\",namespace=\"$namespace\",pod_name=~\"$target.*\", pod_name!~\"$target-primary.*\", container_name!~\"POD|istio-proxy\"}[1m])) by (pod_name)", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -782,7 +782,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\",pod_name=~\"$primary.*\", container_name!~\"POD|istio-proxy\"}) by (pod_name)", + "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\",pod_name=~\"$target-primary.*\", container_name!~\"POD|istio-proxy\"}) by (pod_name)", "format": "time_series", "hide": false, "interval": "", @@ -874,7 +874,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\",pod_name=~\"$canary.*\", pod_name!~\"$primary.*\", container_name!~\"POD|istio-proxy\"}) by (pod_name)", + "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\",pod_name=~\"$target.*\", pod_name!~\"$target-primary.*\", container_name!~\"POD|istio-proxy\"}) by (pod_name)", "format": "time_series", "hide": false, "interval": "", @@ -975,14 +975,14 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate (container_network_receive_bytes_total{namespace=\"$namespace\",pod_name=~\"$primary.*\"}[1m])) ", + "expr": "sum(rate (container_network_receive_bytes_total{namespace=\"$namespace\",pod_name=~\"$target-primary.*\"}[1m])) ", "format": "time_series", "intervalFactor": 1, "legendFormat": "received", "refId": "A" }, { - "expr": "-sum (rate (container_network_transmit_bytes_total{namespace=\"$namespace\",pod_name=~\"$primary.*\"}[1m]))", + "expr": "-sum (rate (container_network_transmit_bytes_total{namespace=\"$namespace\",pod_name=~\"$target-primary.*\"}[1m]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "transmited", @@ -1081,14 +1081,14 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate (container_network_receive_bytes_total{namespace=\"$namespace\",pod_name=~\"$canary.*\",pod_name!~\"$primary.*\"}[1m])) ", + "expr": "sum(rate (container_network_receive_bytes_total{namespace=\"$namespace\",pod_name=~\"$target.*\",pod_name!~\"$target-primary.*\"}[1m])) ", "format": "time_series", "intervalFactor": 1, "legendFormat": "received", "refId": "A" }, { - "expr": "-sum (rate (container_network_transmit_bytes_total{namespace=\"$namespace\",pod_name=~\"$canary.*\",pod_name!~\"$primary.*\"}[1m]))", + "expr": "-sum (rate (container_network_transmit_bytes_total{namespace=\"$namespace\",pod_name=~\"$target.*\",pod_name!~\"$target-primary.*\"}[1m]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "transmited", @@ -1170,38 +1170,16 @@ "allValue": null, "current": null, "datasource": "prometheus", - "definition": "query_result(sum(envoy_cluster_upstream_rq{kubernetes_namespace=\"$namespace\",app=~\".*-primary\"}) by (app))", + "definition": "query_result(sum(envoy_cluster_upstream_rq{kubernetes_namespace=\"$namespace\",envoy_cluster_name=~\".*-primary\"}) by (envoy_cluster_name))", "hide": 0, "includeAll": false, - "label": "Primary", + "label": "Target", "multi": false, - "name": "primary", + "name": "target", "options": [], - "query": "query_result(sum(envoy_cluster_upstream_rq{kubernetes_namespace=\"$namespace\",app=~\".*-primary\"}) by (app))", + "query": "query_result(sum(envoy_cluster_upstream_rq{kubernetes_namespace=\"$namespace\",envoy_cluster_name=~\".*-primary\"}) by (envoy_cluster_name))", "refresh": 1, - "regex": "/.*app=\"([^\"]*).*/", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": null, - "datasource": "prometheus", - "definition": "query_result(sum(envoy_cluster_upstream_rq{kubernetes_namespace=\"$namespace\",app!~\".*-primary\"}) by (app))", - "hide": 0, - "includeAll": false, - "label": "Canary", - "multi": false, - "name": "canary", - "options": [], - "query": "query_result(sum(envoy_cluster_upstream_rq{kubernetes_namespace=\"$namespace\",app!~\".*-primary\"}) by (app))", - "refresh": 1, - "regex": "/.*app=\"([^\"]*).*/", + "regex": "/.*envoy_cluster_name=\"(.*)-primary\"/", "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", @@ -1244,5 +1222,5 @@ "timezone": "", "title": "Envoy Canary", "uid": "flagger-envoy", - "version": 4 + "version": 6 } \ No newline at end of file diff --git a/docs/gitbook/usage/envoy-progressive-delivery.md b/docs/gitbook/usage/envoy-progressive-delivery.md index 2074a7d3..ee7a8f6f 100644 --- a/docs/gitbook/usage/envoy-progressive-delivery.md +++ b/docs/gitbook/usage/envoy-progressive-delivery.md @@ -243,7 +243,7 @@ kubectl port-forward --namespace test svc/flagger-grafana 3000:80 ``` The Envoy dashboard URL is -http://localhost:3000/d/flagger-envoy/envoy-canary?refresh=10s&orgId=1&var-namespace=test&var-primary=podinfo-primary&var-canary=podinfo +http://localhost:3000/d/flagger-envoy/envoy-canary?refresh=10s&orgId=1&var-namespace=test&var-target=podinfo ![Envoy Canary Dashboard](https://raw.githubusercontent.com/weaveworks/flagger/master/docs/screens/flagger-grafana-appmesh.png) diff --git a/pkg/controller/scheduler.go b/pkg/controller/scheduler.go index ec16b602..7768daac 100644 --- a/pkg/controller/scheduler.go +++ b/pkg/controller/scheduler.go @@ -747,7 +747,15 @@ func (c *Controller) analyseCanary(r *flaggerv1.Canary) bool { } // override the global provider if one is specified in the canary spec - metricsProvider := c.meshProvider + var metricsProvider string + // set the metrics provider to Envoy Prometheus when Envoy is the mesh provider + // For example, `envoy` metrics provider should be used for `smi:envoy` mesh provider + if strings.Contains(c.meshProvider, "envoy") { + metricsProvider = "envoy" + } else { + metricsProvider = c.meshProvider + } + if r.Spec.Provider != "" { metricsProvider = r.Spec.Provider @@ -755,11 +763,6 @@ func (c *Controller) analyseCanary(r *flaggerv1.Canary) bool { if strings.Contains(c.meshProvider, "linkerd") { metricsProvider = "linkerd" } - - // set the metrics provider to Envoy Prometheus when Envoy is the default mesh provider - if strings.Contains(c.meshProvider, "envoy") { - metricsProvider = "envoy" - } } // set the metrics provider to query Prometheus for the canary Kubernetes service if the canary target is Service if r.Spec.TargetRef.Kind == "Service" { @@ -792,8 +795,8 @@ func (c *Controller) analyseCanary(r *flaggerv1.Canary) bool { val, err := observer.GetRequestSuccessRate(r.Spec.TargetRef.Name, r.Namespace, metric.Interval) if err != nil { if strings.Contains(err.Error(), "no values found") { - c.recordEventWarningf(r, "Halt advancement no values found for metric %s probably %s.%s is not receiving traffic", - metric.Name, r.Spec.TargetRef.Name, r.Namespace) + c.recordEventWarningf(r, "Halt advancement no values found for %s metric %s probably %s.%s is not receiving traffic", + metricsProvider, metric.Name, r.Spec.TargetRef.Name, r.Namespace) } else { c.recordEventErrorf(r, "Metrics server %s query failed: %v", metricsServer, err) } @@ -812,8 +815,8 @@ func (c *Controller) analyseCanary(r *flaggerv1.Canary) bool { val, err := observer.GetRequestDuration(r.Spec.TargetRef.Name, r.Namespace, metric.Interval) if err != nil { if strings.Contains(err.Error(), "no values found") { - c.recordEventWarningf(r, "Halt advancement no values found for metric %s probably %s.%s is not receiving traffic", - metric.Name, r.Spec.TargetRef.Name, r.Namespace) + c.recordEventWarningf(r, "Halt advancement no values found for %s metric %s probably %s.%s is not receiving traffic", + metricsProvider, metric.Name, r.Spec.TargetRef.Name, r.Namespace) } else { c.recordEventErrorf(r, "Metrics server %s query failed: %v", metricsServer, err) } diff --git a/pkg/metrics/envoy.go b/pkg/metrics/envoy.go index ce1870ce..bf568b7a 100644 --- a/pkg/metrics/envoy.go +++ b/pkg/metrics/envoy.go @@ -10,7 +10,7 @@ var envoyQueries = map[string]string{ rate( envoy_cluster_upstream_rq{ kubernetes_namespace="{{ .Namespace }}", - envoy_cluster_name=~"{{ .Name }}-canary-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)", + envoy_cluster_name=~"{{ .Name }}-canary", envoy_response_code!~"5.*" }[{{ .Interval }}] ) @@ -20,7 +20,7 @@ var envoyQueries = map[string]string{ rate( envoy_cluster_upstream_rq{ kubernetes_namespace="{{ .Namespace }}", - envoy_cluster_name=~"{{ .Name }}-canary-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)" + envoy_cluster_name=~"{{ .Name }}-canary" }[{{ .Interval }}] ) ) @@ -32,7 +32,7 @@ var envoyQueries = map[string]string{ rate( envoy_cluster_upstream_rq_time_bucket{ kubernetes_namespace="{{ .Namespace }}", - envoy_cluster_name=~"{{ .Name }}-canary-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)" + envoy_cluster_name=~"{{ .Name }}-canary" }[{{ .Interval }}] ) ) by (le) diff --git a/pkg/metrics/envoy_test.go b/pkg/metrics/envoy_test.go index 8b013e05..3852fd35 100644 --- a/pkg/metrics/envoy_test.go +++ b/pkg/metrics/envoy_test.go @@ -8,7 +8,7 @@ import ( ) func TestEnvoyObserver_GetRequestSuccessRate(t *testing.T) { - expected := ` sum( rate( envoy_cluster_upstream_rq{ kubernetes_namespace="default", envoy_cluster_name=~"podinfo-canary-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)", envoy_response_code!~"5.*" }[1m] ) ) / sum( rate( envoy_cluster_upstream_rq{ kubernetes_namespace="default", envoy_cluster_name=~"podinfo-canary-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)" }[1m] ) ) * 100` + expected := ` sum( rate( envoy_cluster_upstream_rq{ kubernetes_namespace="default", envoy_cluster_name=~"podinfo-canary", envoy_response_code!~"5.*" }[1m] ) ) / sum( rate( envoy_cluster_upstream_rq{ kubernetes_namespace="default", envoy_cluster_name=~"podinfo-canary" }[1m] ) ) * 100` ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { promql := r.URL.Query()["query"][0] @@ -41,7 +41,7 @@ func TestEnvoyObserver_GetRequestSuccessRate(t *testing.T) { } func TestEnvoyObserver_GetRequestDuration(t *testing.T) { - expected := ` histogram_quantile( 0.99, sum( rate( envoy_cluster_upstream_rq_time_bucket{ kubernetes_namespace="default", envoy_cluster_name=~"podinfo-canary-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)" }[1m] ) ) by (le) )` + expected := ` histogram_quantile( 0.99, sum( rate( envoy_cluster_upstream_rq_time_bucket{ kubernetes_namespace="default", envoy_cluster_name=~"podinfo-canary" }[1m] ) ) by (le) )` ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { promql := r.URL.Query()["query"][0]