From 554753319710ed5f5c2590afa7a733be00ad273e Mon Sep 17 00:00:00 2001 From: Ying Liu Date: Thu, 17 Mar 2022 10:44:15 +0800 Subject: [PATCH 1/3] add canary analysis result as prometheus metrics Signed-off-by: Ying Liu --- pkg/controller/scheduler_metrics.go | 6 +++++- pkg/metrics/recorder.go | 13 +++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/pkg/controller/scheduler_metrics.go b/pkg/controller/scheduler_metrics.go index 7524da8b..f15945e2 100644 --- a/pkg/controller/scheduler_metrics.go +++ b/pkg/controller/scheduler_metrics.go @@ -146,7 +146,7 @@ func (c *Controller) runBuiltinMetricChecks(canary *flaggerv1.Canary) bool { } return false } - + c.recorder.SetAnalysis(canary, metric.Name, val) if metric.ThresholdRange != nil { tr := *metric.ThresholdRange if tr.Min != nil && val < *tr.Min { @@ -177,6 +177,7 @@ func (c *Controller) runBuiltinMetricChecks(canary *flaggerv1.Canary) bool { } return false } + c.recorder.SetAnalysis(canary, metric.Name, val.Seconds()) if metric.ThresholdRange != nil { tr := *metric.ThresholdRange if tr.Min != nil && val < time.Duration(*tr.Min)*time.Millisecond { @@ -209,6 +210,7 @@ func (c *Controller) runBuiltinMetricChecks(canary *flaggerv1.Canary) bool { } return false } + c.recorder.SetAnalysis(canary, metric.Name, val) if metric.ThresholdRange != nil { tr := *metric.ThresholdRange if tr.Min != nil && val < *tr.Min { @@ -283,6 +285,8 @@ func (c *Controller) runMetricChecks(canary *flaggerv1.Canary) bool { return false } + c.recorder.SetAnalysis(canary, metric.Name, val) + if metric.ThresholdRange != nil { tr := *metric.ThresholdRange if tr.Min != nil && val < *tr.Min { diff --git a/pkg/metrics/recorder.go b/pkg/metrics/recorder.go index fea9e330..baca8986 100644 --- a/pkg/metrics/recorder.go +++ b/pkg/metrics/recorder.go @@ -31,6 +31,7 @@ type Recorder struct { total *prometheus.GaugeVec status *prometheus.GaugeVec weight *prometheus.GaugeVec + analysis *prometheus.GaugeVec } // NewRecorder creates a new recorder and registers the Prometheus metrics @@ -67,12 +68,19 @@ func NewRecorder(controller string, register bool) Recorder { Help: "The virtual service destination weight current value", }, []string{"workload", "namespace"}) + analysis := prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Subsystem: controller, + Name: "canary_metric_analysis", + Help: "Last canary metric analysis result per different metric template", + }, []string{"name", "namespace", "metricTemplate"}) + if register { prometheus.MustRegister(info) prometheus.MustRegister(duration) prometheus.MustRegister(total) prometheus.MustRegister(status) prometheus.MustRegister(weight) + prometheus.MustRegister(analysis) } return Recorder{ @@ -81,6 +89,7 @@ func NewRecorder(controller string, register bool) Recorder { total: total, status: status, weight: weight, + analysis: analysis, } } @@ -99,6 +108,10 @@ func (cr *Recorder) SetTotal(namespace string, total int) { cr.total.WithLabelValues(namespace).Set(float64(total)) } +func (cr *Recorder) SetAnalysis(cd *flaggerv1.Canary, metricTemplateName string, val float64) { + cr.analysis.WithLabelValues(cd.Spec.TargetRef.Name, cd.Namespace, metricTemplateName).Set(val) +} + // SetStatus sets the last known canary analysis status func (cr *Recorder) SetStatus(cd *flaggerv1.Canary, phase flaggerv1.CanaryPhase) { var status int From 13483321ac9114ac79066cb8b602512fe328759e Mon Sep 17 00:00:00 2001 From: ying Date: Fri, 25 Mar 2022 22:40:31 +0800 Subject: [PATCH 2/3] Update pkg/metrics/recorder.go Co-authored-by: Stefan Prodan Signed-off-by: Ying Liu --- pkg/metrics/recorder.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/metrics/recorder.go b/pkg/metrics/recorder.go index baca8986..d787f913 100644 --- a/pkg/metrics/recorder.go +++ b/pkg/metrics/recorder.go @@ -71,8 +71,8 @@ func NewRecorder(controller string, register bool) Recorder { analysis := prometheus.NewGaugeVec(prometheus.GaugeOpts{ Subsystem: controller, Name: "canary_metric_analysis", - Help: "Last canary metric analysis result per different metric template", - }, []string{"name", "namespace", "metricTemplate"}) + Help: "Last canary analysis result per metric", + }, []string{"name", "namespace", "metric"}) if register { prometheus.MustRegister(info) From e0186cbe2a05fcbcc89c4b798570c1baf3ba9e04 Mon Sep 17 00:00:00 2001 From: Ying Liu Date: Wed, 6 Apr 2022 09:57:46 +0800 Subject: [PATCH 3/3] update docs for metrics part Signed-off-by: Ying Liu --- docs/gitbook/usage/monitoring.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/gitbook/usage/monitoring.md b/docs/gitbook/usage/monitoring.md index fa860e44..67179480 100644 --- a/docs/gitbook/usage/monitoring.md +++ b/docs/gitbook/usage/monitoring.md @@ -117,4 +117,8 @@ flagger_canary_duration_seconds_bucket{name="podinfo",namespace="test",le="10"} flagger_canary_duration_seconds_bucket{name="podinfo",namespace="test",le="+Inf"} 6 flagger_canary_duration_seconds_sum{name="podinfo",namespace="test"} 17.3561329 flagger_canary_duration_seconds_count{name="podinfo",namespace="test"} 6 + +# Last canary metric analysis result per different metrics +flagger_canary_metric_analysis{metric="podinfo-http-successful-rate",name="podinfo",namespace="test"} 1 +flagger_canary_metric_analysis{metric="podinfo-custom-metric",name="podinfo",namespace="test"} 0.918223108974359 ```