From 4b6126dd1abf137c0b1a9d85afe46edad0c174d5 Mon Sep 17 00:00:00 2001
From: stefanprodan <stefan.prodan@gmail.com>
Date: Tue, 19 Mar 2019 15:52:26 +0200
Subject: [PATCH] Add Envoy HTTP success rate metric check

---
 pkg/controller/observer.go  | 36 ++++++++++++++++++++++++++++++++++++
 pkg/controller/scheduler.go | 18 ++++++++++++++++++
 2 files changed, 54 insertions(+)

diff --git a/pkg/controller/observer.go b/pkg/controller/observer.go
index 051321e4..1e846c6f 100644
--- a/pkg/controller/observer.go
+++ b/pkg/controller/observer.go
@@ -106,6 +106,42 @@ func (c *CanaryObserver) GetScalar(query string) (float64, error) {
 	return *value, nil
 }
 
+func (c *CanaryObserver) GetEnvoySuccessRate(name string, namespace string, metric string, interval string) (float64, error) {
+	if c.metricsServer == "fake" {
+		return 100, nil
+	}
+
+	var rate *float64
+	querySt := url.QueryEscape(`sum(rate(` +
+		metric + `{kubernetes_namespace="` +
+		namespace + `",app="` +
+		name + `",envoy_response_code!~"5.*"}[1m])) / sum(rate(` +
+		metric + `{kubernetes_namespace="` +
+		namespace + `",app="` +
+		name + `"}[` +
+		interval + `])) * 100 `)
+	result, err := c.queryMetric(querySt)
+	if err != nil {
+		return 0, err
+	}
+
+	for _, v := range result.Data.Result {
+		metricValue := v.Value[1]
+		switch metricValue.(type) {
+		case string:
+			f, err := strconv.ParseFloat(metricValue.(string), 64)
+			if err != nil {
+				return 0, err
+			}
+			rate = &f
+		}
+	}
+	if rate == nil {
+		return 0, fmt.Errorf("no values found for metric %s", metric)
+	}
+	return *rate, nil
+}
+
 // GetDeploymentCounter returns the requests success rate using istio_requests_total metric
 func (c *CanaryObserver) GetDeploymentCounter(name string, namespace string, metric string, interval string) (float64, error) {
 	if c.metricsServer == "fake" {
diff --git a/pkg/controller/scheduler.go b/pkg/controller/scheduler.go
index 73d9334c..5661d0ce 100644
--- a/pkg/controller/scheduler.go
+++ b/pkg/controller/scheduler.go
@@ -493,6 +493,24 @@ func (c *Controller) analyseCanary(r *flaggerv1.Canary) bool {
 			metric.Interval = r.GetMetricInterval()
 		}
 
+		if metric.Name == "envoy_cluster_upstream_rq" {
+			val, err := c.observer.GetEnvoySuccessRate(r.Spec.TargetRef.Name, r.Namespace, metric.Name, metric.Interval)
+			if err != nil {
+				if strings.Contains(err.Error(), "no values found") {
+					c.recordEventWarningf(r, "Halt advancement no values found for metric %s probably %s.%s is not receiving traffic",
+						metric.Name, r.Spec.TargetRef.Name, r.Namespace)
+				} else {
+					c.recordEventErrorf(r, "Metrics server %s query failed: %v", c.observer.metricsServer, err)
+				}
+				return false
+			}
+			if float64(metric.Threshold) > val {
+				c.recordEventWarningf(r, "Halt %s.%s advancement success rate %.2f%% < %v%%",
+					r.Name, r.Namespace, val, metric.Threshold)
+				return false
+			}
+		}
+
 		if metric.Name == "istio_requests_total" {
 			val, err := c.observer.GetDeploymentCounter(r.Spec.TargetRef.Name, r.Namespace, metric.Name, metric.Interval)
 			if err != nil {