diff --git a/pkg/controller/observer.go b/pkg/controller/observer.go index 91a73e77..15b48d9f 100644 --- a/pkg/controller/observer.go +++ b/pkg/controller/observer.go @@ -8,6 +8,7 @@ import ( "net/http" "net/url" "strconv" + "strings" "time" ) @@ -73,6 +74,38 @@ func (c *CanaryObserver) queryMetric(query string) (*vectorQueryResponse, error) return &values, nil } +// GetScalar runs the promql query and returns the first value found +func (c *CanaryObserver) GetScalar(query string) (float64, error) { + if c.metricsServer == "fake" { + return 100, nil + } + + query = strings.Replace(query, "\n","",-1) + query = strings.Replace(query, " ","",-1) + + var value *float64 + result, err := c.queryMetric(query) + if err != nil { + return 0, err + } + + for _, v := range result.Data.Result { + metricValue := v.Value[1] + switch metricValue.(type) { + case string: + f, err := strconv.ParseFloat(metricValue.(string), 64) + if err != nil { + return 0, err + } + value = &f + } + } + if value == nil { + return 0, fmt.Errorf("no values found for query %s", query) + } + return *value, nil +} + // GetDeploymentCounter returns the requests success rate using istio_requests_total metric func (c *CanaryObserver) GetDeploymentCounter(name string, namespace string, metric string, interval string) (float64, error) { if c.metricsServer == "fake" { diff --git a/pkg/controller/scheduler.go b/pkg/controller/scheduler.go index d928fed1..ebb92189 100644 --- a/pkg/controller/scheduler.go +++ b/pkg/controller/scheduler.go @@ -436,6 +436,24 @@ func (c *Controller) analyseCanary(r *flaggerv1.Canary) bool { return false } } + + if metric.Query != "" { + val, err := c.observer.GetScalar(metric.Query) + if err != nil { + if strings.Contains(err.Error(), "no values found") { + c.recordEventWarningf(r, "Halt advancement no values found for metric %s probably %s.%s is not receiving traffic", + metric.Name, r.Spec.TargetRef.Name, r.Namespace) + } else { + c.recordEventErrorf(r, "Metrics server %s query failed: %v", c.observer.metricsServer, err) + } + return false + } + if val > float64(metric.Threshold) { + c.recordEventWarningf(r, "Halt %s.%s advancement %s %.2f > %v", + r.Name, r.Namespace, metric.Name, val, metric.Threshold) + return false + } + } } return true