Run the wekbooks before the metrics checks

- log warning when no values are found for Istio metric due to lack of traffic
This commit is contained in:
stefanprodan
2019-01-20 13:54:44 +02:00
parent c46fe55ad0
commit 02236374d8
4 changed files with 25 additions and 20 deletions
+1 -1
View File
@@ -136,7 +136,7 @@ type CanaryWebhook struct {
type CanaryWebhookPayload struct {
Name string `json:"name"`
Namespace string `json:"namespace"`
Metadata *map[string]string `json:"metadata,omitempty"`
Metadata map[string]string `json:"metadata,omitempty"`
}
// GetProgressDeadlineSeconds returns the progress deadline (default 600s)
@@ -234,13 +234,9 @@ func (in *CanaryWebhookPayload) DeepCopyInto(out *CanaryWebhookPayload) {
*out = *in
if in.Metadata != nil {
in, out := &in.Metadata, &out.Metadata
*out = new(map[string]string)
if **in != nil {
in, out := *in, *out
*out = make(map[string]string, len(*in))
for key, val := range *in {
(*out)[key] = val
}
*out = make(map[string]string, len(*in))
for key, val := range *in {
(*out)[key] = val
}
}
return
+17 -11
View File
@@ -2,6 +2,7 @@ package controller
import (
"fmt"
"strings"
"time"
flaggerv1 "github.com/stefanprodan/flagger/pkg/apis/flagger/v1alpha3"
@@ -336,12 +337,27 @@ func (c *Controller) hasCanaryRevisionChanged(cd *flaggerv1.Canary) bool {
}
func (c *Controller) analyseCanary(r *flaggerv1.Canary) bool {
// run external checks
for _, webhook := range r.Spec.CanaryAnalysis.Webhooks {
err := CallWebhook(r.Name, r.Namespace, webhook)
if err != nil {
c.recordEventWarningf(r, "Halt %s.%s advancement external check %s failed %v",
r.Name, r.Namespace, webhook.Name, err)
return false
}
}
// run metrics checks
for _, metric := range r.Spec.CanaryAnalysis.Metrics {
if metric.Name == "istio_requests_total" {
val, err := c.observer.GetDeploymentCounter(r.Spec.TargetRef.Name, r.Namespace, metric.Name, metric.Interval)
if err != nil {
c.recordEventErrorf(r, "Metrics server %s query failed: %v", c.observer.metricsServer, err)
if strings.Contains(err.Error(), "no values found"){
c.recordEventWarningf(r, "Halt advancement no values found for metric %s probably %s.%s is not receiving traffic",
metric.Name, r.Spec.TargetRef.Name, r.Namespace)
}else {
c.recordEventErrorf(r, "Metrics server %s query failed: %v", c.observer.metricsServer, err)
}
return false
}
if float64(metric.Threshold) > val {
@@ -366,15 +382,5 @@ func (c *Controller) analyseCanary(r *flaggerv1.Canary) bool {
}
}
// run external checks
for _, webhook := range r.Spec.CanaryAnalysis.Webhooks {
err := CallWebhook(r.Name, r.Namespace, webhook)
if err != nil {
c.recordEventWarningf(r, "Halt %s.%s advancement external check %s failed %v",
r.Name, r.Namespace, webhook.Name, err)
return false
}
}
return true
}
+4 -1
View File
@@ -19,7 +19,10 @@ func CallWebhook(name string, namepace string, w flaggerv1.CanaryWebhook) error
payload := flaggerv1.CanaryWebhookPayload{
Name: name,
Namespace: namepace,
Metadata: w.Metadata,
}
if w.Metadata != nil {
payload.Metadata = *w.Metadata
}
payloadBin, err := json.Marshal(payload)