Merge pull request #183 from weaveworks/metrics-fix

Fix Istio latency check

pkg/controller/scheduler.go

@@ -581,12 +581,19 @@ func (c *Controller) analyseCanary(r *flaggerv1.Canary) bool {
 					r.Name, r.Namespace, val, metric.Threshold)
 				return false
 			}
+
+			//c.recordEventInfof(r, "Check %s passed %.2f%% > %v%%", metric.Name, val, metric.Threshold)
 		}
 
 		if metric.Name == "request-duration" {
 			val, err := observer.GetRequestDuration(r.Spec.TargetRef.Name, r.Namespace, metric.Interval)
 			if err != nil {
-				c.recordEventErrorf(r, "Metrics server %s query failed: %v", c.observerFactory.Client.GetMetricsServer(), err)
+				if strings.Contains(err.Error(), "no values found") {
+					c.recordEventWarningf(r, "Halt advancement no values found for metric %s probably %s.%s is not receiving traffic",
+						metric.Name, r.Spec.TargetRef.Name, r.Namespace)
+				} else {
+					c.recordEventErrorf(r, "Metrics server %s query failed: %v", c.observerFactory.Client.GetMetricsServer(), err)
+				}
 				return false
 			}
 			t := time.Duration(metric.Threshold) * time.Millisecond
@@ -595,6 +602,8 @@ func (c *Controller) analyseCanary(r *flaggerv1.Canary) bool {
 					r.Name, r.Namespace, val, t)
 				return false
 			}
+
+			//c.recordEventInfof(r, "Check %s passed %v < %v", metric.Name, val, metric.Threshold)
 		}
 
 		// custom checks

pkg/metrics/istio.go

@@ -71,6 +71,6 @@ func (ob *IstioObserver) GetRequestDuration(name string, namespace string, inter
 		return 0, err
 	}
 
-	ms := time.Duration(int64(value)) * time.Millisecond
+	ms := time.Duration(int64(value*1000)) * time.Millisecond
 	return ms, nil
 }

pkg/metrics/istio_test.go

@@ -49,7 +49,7 @@ func TestIstioObserver_GetRequestDuration(t *testing.T) {
 			t.Errorf("\nGot %s \nWanted %s", promql, expected)
 		}
 
-		json := `{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1,"100"]}]}}`
+		json := `{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1,"0.100"]}]}}`
 		w.Write([]byte(json))
 	}))
 	defer ts.Close()