mirror of
https://github.com/fluxcd/flagger.git
synced 2026-04-15 06:57:34 +00:00
Migrate the builtin Prometheus checks to metric templates
This commit is contained in:
stefanprodan
@@ -2,6 +2,7 @@ package controller
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/weaveworks/flagger/pkg/metrics/observers"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -49,7 +50,7 @@ type Controller struct {
|
||||
notifier notifier.Interface
|
||||
canaryFactory *canary.Factory
|
||||
routerFactory *router.Factory
|
||||
observerFactory *metrics.Factory
|
||||
observerFactory *observers.Factory
|
||||
meshProvider string
|
||||
eventWebhook string
|
||||
}
|
||||
@@ -64,7 +65,7 @@ func NewController(
|
||||
notifier notifier.Interface,
|
||||
canaryFactory *canary.Factory,
|
||||
routerFactory *router.Factory,
|
||||
observerFactory *metrics.Factory,
|
||||
observerFactory *observers.Factory,
|
||||
meshProvider string,
|
||||
version string,
|
||||
eventWebhook string,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"github.com/weaveworks/flagger/pkg/metrics/observers"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -73,7 +74,7 @@ func SetupMocks(c *flaggerv1.Canary) Mocks {
|
||||
rf := router.NewFactory(nil, kubeClient, flaggerClient, "annotationsPrefix", logger, flaggerClient)
|
||||
|
||||
// init observer
|
||||
observerFactory, _ := metrics.NewFactory("fake", 5*time.Second)
|
||||
observerFactory, _ := observers.NewFactory("fake")
|
||||
|
||||
// init canary factory
|
||||
configTracker := canary.ConfigTracker{
|
||||
|
||||
+101
-74
@@ -7,9 +7,10 @@ import (
|
||||
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3"
|
||||
flaggerv1alpha1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha1"
|
||||
flaggerv1alpha3 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3"
|
||||
"github.com/weaveworks/flagger/pkg/canary"
|
||||
"github.com/weaveworks/flagger/pkg/metrics"
|
||||
"github.com/weaveworks/flagger/pkg/metrics/observers"
|
||||
"github.com/weaveworks/flagger/pkg/router"
|
||||
)
|
||||
|
||||
@@ -25,7 +26,7 @@ func (c *Controller) scheduleCanaries() {
|
||||
stats := make(map[string]int)
|
||||
|
||||
c.canaries.Range(func(key interface{}, value interface{}) bool {
|
||||
canary := value.(*flaggerv1.Canary)
|
||||
canary := value.(*flaggerv1alpha3.Canary)
|
||||
|
||||
// format: <name>.<namespace>
|
||||
name := key.(string)
|
||||
@@ -196,8 +197,8 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
|
||||
}
|
||||
|
||||
// reset status
|
||||
status := flaggerv1.CanaryStatus{
|
||||
Phase: flaggerv1.CanaryPhaseProgressing,
|
||||
status := flaggerv1alpha3.CanaryStatus{
|
||||
Phase: flaggerv1alpha3.CanaryPhaseProgressing,
|
||||
CanaryWeight: 0,
|
||||
FailedChecks: 0,
|
||||
Iterations: 0,
|
||||
@@ -225,8 +226,8 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
|
||||
}
|
||||
|
||||
// check if we should rollback
|
||||
if cd.Status.Phase == flaggerv1.CanaryPhaseProgressing ||
|
||||
cd.Status.Phase == flaggerv1.CanaryPhaseWaiting {
|
||||
if cd.Status.Phase == flaggerv1alpha3.CanaryPhaseProgressing ||
|
||||
cd.Status.Phase == flaggerv1alpha3.CanaryPhaseWaiting {
|
||||
if ok := c.runRollbackHooks(cd, cd.Status.Phase); ok {
|
||||
c.recordEventWarningf(cd, "Rolling back %s.%s manual webhook invoked", cd.Name, cd.Namespace)
|
||||
c.sendNotification(cd, "Rolling back manual webhook invoked", false, true)
|
||||
@@ -236,7 +237,7 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
|
||||
}
|
||||
|
||||
// route all traffic to primary if analysis has succeeded
|
||||
if cd.Status.Phase == flaggerv1.CanaryPhasePromoting {
|
||||
if cd.Status.Phase == flaggerv1alpha3.CanaryPhasePromoting {
|
||||
if provider != "kubernetes" {
|
||||
c.recordEventInfof(cd, "Routing all traffic to primary")
|
||||
if err := meshRouter.SetRoutes(cd, 100, 0, false); err != nil {
|
||||
@@ -247,7 +248,7 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
|
||||
}
|
||||
|
||||
// update status phase
|
||||
if err := canaryController.SetStatusPhase(cd, flaggerv1.CanaryPhaseFinalising); err != nil {
|
||||
if err := canaryController.SetStatusPhase(cd, flaggerv1alpha3.CanaryPhaseFinalising); err != nil {
|
||||
c.recordEventWarningf(cd, "%v", err)
|
||||
return
|
||||
}
|
||||
@@ -256,19 +257,19 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
|
||||
}
|
||||
|
||||
// scale canary to zero if promotion has finished
|
||||
if cd.Status.Phase == flaggerv1.CanaryPhaseFinalising {
|
||||
if cd.Status.Phase == flaggerv1alpha3.CanaryPhaseFinalising {
|
||||
if err := canaryController.Scale(cd, 0); err != nil {
|
||||
c.recordEventWarningf(cd, "%v", err)
|
||||
return
|
||||
}
|
||||
|
||||
// set status to succeeded
|
||||
if err := canaryController.SetStatusPhase(cd, flaggerv1.CanaryPhaseSucceeded); err != nil {
|
||||
if err := canaryController.SetStatusPhase(cd, flaggerv1alpha3.CanaryPhaseSucceeded); err != nil {
|
||||
c.recordEventWarningf(cd, "%v", err)
|
||||
return
|
||||
}
|
||||
c.recorder.SetStatus(cd, flaggerv1.CanaryPhaseSucceeded)
|
||||
c.runPostRolloutHooks(cd, flaggerv1.CanaryPhaseSucceeded)
|
||||
c.recorder.SetStatus(cd, flaggerv1alpha3.CanaryPhaseSucceeded)
|
||||
c.runPostRolloutHooks(cd, flaggerv1alpha3.CanaryPhaseSucceeded)
|
||||
c.recordEventInfof(cd, "Promotion completed! Scaling down %s.%s", cd.Spec.TargetRef.Name, cd.Namespace)
|
||||
c.sendNotification(cd, "Canary analysis completed successfully, promotion finished.",
|
||||
false, false)
|
||||
@@ -276,7 +277,7 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
|
||||
}
|
||||
|
||||
// check if the number of failed checks reached the threshold
|
||||
if cd.Status.Phase == flaggerv1.CanaryPhaseProgressing &&
|
||||
if cd.Status.Phase == flaggerv1alpha3.CanaryPhaseProgressing &&
|
||||
(!retriable || cd.Status.FailedChecks >= cd.Spec.CanaryAnalysis.Threshold) {
|
||||
if !retriable {
|
||||
c.recordEventWarningf(cd, "Rolling back %s.%s progress deadline exceeded %v",
|
||||
@@ -349,7 +350,7 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
|
||||
|
||||
}
|
||||
|
||||
func (c *Controller) runCanary(canary *flaggerv1.Canary, canaryController canary.Controller, meshRouter router.Interface, provider string, mirrored bool, canaryWeight int, primaryWeight int, maxWeight int) {
|
||||
func (c *Controller) runCanary(canary *flaggerv1alpha3.Canary, canaryController canary.Controller, meshRouter router.Interface, provider string, mirrored bool, canaryWeight int, primaryWeight int, maxWeight int) {
|
||||
primaryName := fmt.Sprintf("%s-primary", canary.Spec.TargetRef.Name)
|
||||
|
||||
// increase traffic weight
|
||||
@@ -412,14 +413,14 @@ func (c *Controller) runCanary(canary *flaggerv1.Canary, canaryController canary
|
||||
}
|
||||
|
||||
// update status phase
|
||||
if err := canaryController.SetStatusPhase(canary, flaggerv1.CanaryPhasePromoting); err != nil {
|
||||
if err := canaryController.SetStatusPhase(canary, flaggerv1alpha3.CanaryPhasePromoting); err != nil {
|
||||
c.recordEventWarningf(canary, "%v", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Controller) runAB(canary *flaggerv1.Canary, canaryController canary.Controller, meshRouter router.Interface, provider string) {
|
||||
func (c *Controller) runAB(canary *flaggerv1alpha3.Canary, canaryController canary.Controller, meshRouter router.Interface, provider string) {
|
||||
primaryName := fmt.Sprintf("%s-primary", canary.Spec.TargetRef.Name)
|
||||
|
||||
// route traffic to canary and increment iterations
|
||||
@@ -454,14 +455,14 @@ func (c *Controller) runAB(canary *flaggerv1.Canary, canaryController canary.Con
|
||||
}
|
||||
|
||||
// update status phase
|
||||
if err := canaryController.SetStatusPhase(canary, flaggerv1.CanaryPhasePromoting); err != nil {
|
||||
if err := canaryController.SetStatusPhase(canary, flaggerv1alpha3.CanaryPhasePromoting); err != nil {
|
||||
c.recordEventWarningf(canary, "%v", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Controller) runBlueGreen(canary *flaggerv1.Canary, canaryController canary.Controller, meshRouter router.Interface, provider string, mirrored bool) {
|
||||
func (c *Controller) runBlueGreen(canary *flaggerv1alpha3.Canary, canaryController canary.Controller, meshRouter router.Interface, provider string, mirrored bool) {
|
||||
primaryName := fmt.Sprintf("%s-primary", canary.Spec.TargetRef.Name)
|
||||
|
||||
// increment iterations
|
||||
@@ -522,7 +523,7 @@ func (c *Controller) runBlueGreen(canary *flaggerv1.Canary, canaryController can
|
||||
}
|
||||
|
||||
// update status phase
|
||||
if err := canaryController.SetStatusPhase(canary, flaggerv1.CanaryPhasePromoting); err != nil {
|
||||
if err := canaryController.SetStatusPhase(canary, flaggerv1alpha3.CanaryPhasePromoting); err != nil {
|
||||
c.recordEventWarningf(canary, "%v", err)
|
||||
return
|
||||
}
|
||||
@@ -530,7 +531,7 @@ func (c *Controller) runBlueGreen(canary *flaggerv1.Canary, canaryController can
|
||||
|
||||
}
|
||||
|
||||
func (c *Controller) shouldSkipAnalysis(canary *flaggerv1.Canary, canaryController canary.Controller, meshRouter router.Interface, primaryWeight int, canaryWeight int) bool {
|
||||
func (c *Controller) shouldSkipAnalysis(canary *flaggerv1alpha3.Canary, canaryController canary.Controller, meshRouter router.Interface, primaryWeight int, canaryWeight int) bool {
|
||||
if !canary.Spec.SkipAnalysis {
|
||||
return false
|
||||
}
|
||||
@@ -559,13 +560,13 @@ func (c *Controller) shouldSkipAnalysis(canary *flaggerv1.Canary, canaryControll
|
||||
}
|
||||
|
||||
// update status phase
|
||||
if err := canaryController.SetStatusPhase(canary, flaggerv1.CanaryPhaseSucceeded); err != nil {
|
||||
if err := canaryController.SetStatusPhase(canary, flaggerv1alpha3.CanaryPhaseSucceeded); err != nil {
|
||||
c.recordEventWarningf(canary, "%v", err)
|
||||
return false
|
||||
}
|
||||
|
||||
// notify
|
||||
c.recorder.SetStatus(canary, flaggerv1.CanaryPhaseSucceeded)
|
||||
c.recorder.SetStatus(canary, flaggerv1alpha3.CanaryPhaseSucceeded)
|
||||
c.recordEventInfof(canary, "Promotion completed! Canary analysis was skipped for %s.%s",
|
||||
canary.Spec.TargetRef.Name, canary.Namespace)
|
||||
c.sendNotification(canary, "Canary analysis was skipped, promotion finished.",
|
||||
@@ -574,13 +575,13 @@ func (c *Controller) shouldSkipAnalysis(canary *flaggerv1.Canary, canaryControll
|
||||
return true
|
||||
}
|
||||
|
||||
func (c *Controller) shouldAdvance(canary *flaggerv1.Canary, canaryController canary.Controller) (bool, error) {
|
||||
func (c *Controller) shouldAdvance(canary *flaggerv1alpha3.Canary, canaryController canary.Controller) (bool, error) {
|
||||
if canary.Status.LastAppliedSpec == "" ||
|
||||
canary.Status.Phase == flaggerv1.CanaryPhaseInitializing ||
|
||||
canary.Status.Phase == flaggerv1.CanaryPhaseProgressing ||
|
||||
canary.Status.Phase == flaggerv1.CanaryPhaseWaiting ||
|
||||
canary.Status.Phase == flaggerv1.CanaryPhasePromoting ||
|
||||
canary.Status.Phase == flaggerv1.CanaryPhaseFinalising {
|
||||
canary.Status.Phase == flaggerv1alpha3.CanaryPhaseInitializing ||
|
||||
canary.Status.Phase == flaggerv1alpha3.CanaryPhaseProgressing ||
|
||||
canary.Status.Phase == flaggerv1alpha3.CanaryPhaseWaiting ||
|
||||
canary.Status.Phase == flaggerv1alpha3.CanaryPhasePromoting ||
|
||||
canary.Status.Phase == flaggerv1alpha3.CanaryPhaseFinalising {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
@@ -601,20 +602,20 @@ func (c *Controller) shouldAdvance(canary *flaggerv1.Canary, canaryController ca
|
||||
|
||||
}
|
||||
|
||||
func (c *Controller) checkCanaryStatus(canary *flaggerv1.Canary, canaryController canary.Controller, shouldAdvance bool) bool {
|
||||
func (c *Controller) checkCanaryStatus(canary *flaggerv1alpha3.Canary, canaryController canary.Controller, shouldAdvance bool) bool {
|
||||
c.recorder.SetStatus(canary, canary.Status.Phase)
|
||||
if canary.Status.Phase == flaggerv1.CanaryPhaseProgressing ||
|
||||
canary.Status.Phase == flaggerv1.CanaryPhasePromoting ||
|
||||
canary.Status.Phase == flaggerv1.CanaryPhaseFinalising {
|
||||
if canary.Status.Phase == flaggerv1alpha3.CanaryPhaseProgressing ||
|
||||
canary.Status.Phase == flaggerv1alpha3.CanaryPhasePromoting ||
|
||||
canary.Status.Phase == flaggerv1alpha3.CanaryPhaseFinalising {
|
||||
return true
|
||||
}
|
||||
|
||||
if canary.Status.Phase == "" || canary.Status.Phase == flaggerv1.CanaryPhaseInitializing {
|
||||
if err := canaryController.SyncStatus(canary, flaggerv1.CanaryStatus{Phase: flaggerv1.CanaryPhaseInitialized}); err != nil {
|
||||
if canary.Status.Phase == "" || canary.Status.Phase == flaggerv1alpha3.CanaryPhaseInitializing {
|
||||
if err := canaryController.SyncStatus(canary, flaggerv1alpha3.CanaryStatus{Phase: flaggerv1alpha3.CanaryPhaseInitialized}); err != nil {
|
||||
c.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)).Errorf("%v", err)
|
||||
return false
|
||||
}
|
||||
c.recorder.SetStatus(canary, flaggerv1.CanaryPhaseInitialized)
|
||||
c.recorder.SetStatus(canary, flaggerv1alpha3.CanaryPhaseInitialized)
|
||||
c.recordEventInfof(canary, "Initialization done! %s.%s", canary.Name, canary.Namespace)
|
||||
c.sendNotification(canary, "New deployment detected, initialization completed.",
|
||||
true, false)
|
||||
@@ -623,7 +624,7 @@ func (c *Controller) checkCanaryStatus(canary *flaggerv1.Canary, canaryControlle
|
||||
|
||||
if shouldAdvance {
|
||||
canaryPhaseProgressing := canary.DeepCopy()
|
||||
canaryPhaseProgressing.Status.Phase = flaggerv1.CanaryPhaseProgressing
|
||||
canaryPhaseProgressing.Status.Phase = flaggerv1alpha3.CanaryPhaseProgressing
|
||||
c.recordEventInfof(canaryPhaseProgressing, "New revision detected! Scaling up %s.%s", canaryPhaseProgressing.Spec.TargetRef.Name, canaryPhaseProgressing.Namespace)
|
||||
c.sendNotification(canaryPhaseProgressing, "New revision detected, starting canary analysis.",
|
||||
true, false)
|
||||
@@ -632,18 +633,18 @@ func (c *Controller) checkCanaryStatus(canary *flaggerv1.Canary, canaryControlle
|
||||
c.recordEventErrorf(canary, "%v", err)
|
||||
return false
|
||||
}
|
||||
if err := canaryController.SyncStatus(canary, flaggerv1.CanaryStatus{Phase: flaggerv1.CanaryPhaseProgressing}); err != nil {
|
||||
if err := canaryController.SyncStatus(canary, flaggerv1alpha3.CanaryStatus{Phase: flaggerv1alpha3.CanaryPhaseProgressing}); err != nil {
|
||||
c.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)).Errorf("%v", err)
|
||||
return false
|
||||
}
|
||||
c.recorder.SetStatus(canary, flaggerv1.CanaryPhaseProgressing)
|
||||
c.recorder.SetStatus(canary, flaggerv1alpha3.CanaryPhaseProgressing)
|
||||
return false
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (c *Controller) hasCanaryRevisionChanged(canary *flaggerv1.Canary, canaryController canary.Controller) bool {
|
||||
if canary.Status.Phase == flaggerv1.CanaryPhaseProgressing {
|
||||
func (c *Controller) hasCanaryRevisionChanged(canary *flaggerv1alpha3.Canary, canaryController canary.Controller) bool {
|
||||
if canary.Status.Phase == flaggerv1alpha3.CanaryPhaseProgressing {
|
||||
if diff, _ := canaryController.HasTargetChanged(canary); diff {
|
||||
return true
|
||||
}
|
||||
@@ -654,13 +655,13 @@ func (c *Controller) hasCanaryRevisionChanged(canary *flaggerv1.Canary, canaryCo
|
||||
return false
|
||||
}
|
||||
|
||||
func (c *Controller) runConfirmRolloutHooks(canary *flaggerv1.Canary, canaryController canary.Controller) bool {
|
||||
func (c *Controller) runConfirmRolloutHooks(canary *flaggerv1alpha3.Canary, canaryController canary.Controller) bool {
|
||||
for _, webhook := range canary.Spec.CanaryAnalysis.Webhooks {
|
||||
if webhook.Type == flaggerv1.ConfirmRolloutHook {
|
||||
err := CallWebhook(canary.Name, canary.Namespace, flaggerv1.CanaryPhaseProgressing, webhook)
|
||||
if webhook.Type == flaggerv1alpha3.ConfirmRolloutHook {
|
||||
err := CallWebhook(canary.Name, canary.Namespace, flaggerv1alpha3.CanaryPhaseProgressing, webhook)
|
||||
if err != nil {
|
||||
if canary.Status.Phase != flaggerv1.CanaryPhaseWaiting {
|
||||
if err := canaryController.SetStatusPhase(canary, flaggerv1.CanaryPhaseWaiting); err != nil {
|
||||
if canary.Status.Phase != flaggerv1alpha3.CanaryPhaseWaiting {
|
||||
if err := canaryController.SetStatusPhase(canary, flaggerv1alpha3.CanaryPhaseWaiting); err != nil {
|
||||
c.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)).Errorf("%v", err)
|
||||
}
|
||||
c.recordEventWarningf(canary, "Halt %s.%s advancement waiting for approval %s",
|
||||
@@ -669,8 +670,8 @@ func (c *Controller) runConfirmRolloutHooks(canary *flaggerv1.Canary, canaryCont
|
||||
}
|
||||
return false
|
||||
} else {
|
||||
if canary.Status.Phase == flaggerv1.CanaryPhaseWaiting {
|
||||
if err := canaryController.SetStatusPhase(canary, flaggerv1.CanaryPhaseProgressing); err != nil {
|
||||
if canary.Status.Phase == flaggerv1alpha3.CanaryPhaseWaiting {
|
||||
if err := canaryController.SetStatusPhase(canary, flaggerv1alpha3.CanaryPhaseProgressing); err != nil {
|
||||
c.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)).Errorf("%v", err)
|
||||
return false
|
||||
}
|
||||
@@ -683,10 +684,10 @@ func (c *Controller) runConfirmRolloutHooks(canary *flaggerv1.Canary, canaryCont
|
||||
return true
|
||||
}
|
||||
|
||||
func (c *Controller) runConfirmPromotionHooks(canary *flaggerv1.Canary) bool {
|
||||
func (c *Controller) runConfirmPromotionHooks(canary *flaggerv1alpha3.Canary) bool {
|
||||
for _, webhook := range canary.Spec.CanaryAnalysis.Webhooks {
|
||||
if webhook.Type == flaggerv1.ConfirmPromotionHook {
|
||||
err := CallWebhook(canary.Name, canary.Namespace, flaggerv1.CanaryPhaseProgressing, webhook)
|
||||
if webhook.Type == flaggerv1alpha3.ConfirmPromotionHook {
|
||||
err := CallWebhook(canary.Name, canary.Namespace, flaggerv1alpha3.CanaryPhaseProgressing, webhook)
|
||||
if err != nil {
|
||||
c.recordEventWarningf(canary, "Halt %s.%s advancement waiting for promotion approval %s",
|
||||
canary.Name, canary.Namespace, webhook.Name)
|
||||
@@ -700,10 +701,10 @@ func (c *Controller) runConfirmPromotionHooks(canary *flaggerv1.Canary) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (c *Controller) runPreRolloutHooks(canary *flaggerv1.Canary) bool {
|
||||
func (c *Controller) runPreRolloutHooks(canary *flaggerv1alpha3.Canary) bool {
|
||||
for _, webhook := range canary.Spec.CanaryAnalysis.Webhooks {
|
||||
if webhook.Type == flaggerv1.PreRolloutHook {
|
||||
err := CallWebhook(canary.Name, canary.Namespace, flaggerv1.CanaryPhaseProgressing, webhook)
|
||||
if webhook.Type == flaggerv1alpha3.PreRolloutHook {
|
||||
err := CallWebhook(canary.Name, canary.Namespace, flaggerv1alpha3.CanaryPhaseProgressing, webhook)
|
||||
if err != nil {
|
||||
c.recordEventWarningf(canary, "Halt %s.%s advancement pre-rollout check %s failed %v",
|
||||
canary.Name, canary.Namespace, webhook.Name, err)
|
||||
@@ -716,9 +717,9 @@ func (c *Controller) runPreRolloutHooks(canary *flaggerv1.Canary) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (c *Controller) runPostRolloutHooks(canary *flaggerv1.Canary, phase flaggerv1.CanaryPhase) bool {
|
||||
func (c *Controller) runPostRolloutHooks(canary *flaggerv1alpha3.Canary, phase flaggerv1alpha3.CanaryPhase) bool {
|
||||
for _, webhook := range canary.Spec.CanaryAnalysis.Webhooks {
|
||||
if webhook.Type == flaggerv1.PostRolloutHook {
|
||||
if webhook.Type == flaggerv1alpha3.PostRolloutHook {
|
||||
err := CallWebhook(canary.Name, canary.Namespace, phase, webhook)
|
||||
if err != nil {
|
||||
c.recordEventWarningf(canary, "Post-rollout hook %s failed %v", webhook.Name, err)
|
||||
@@ -731,9 +732,9 @@ func (c *Controller) runPostRolloutHooks(canary *flaggerv1.Canary, phase flagger
|
||||
return true
|
||||
}
|
||||
|
||||
func (c *Controller) runRollbackHooks(canary *flaggerv1.Canary, phase flaggerv1.CanaryPhase) bool {
|
||||
func (c *Controller) runRollbackHooks(canary *flaggerv1alpha3.Canary, phase flaggerv1alpha3.CanaryPhase) bool {
|
||||
for _, webhook := range canary.Spec.CanaryAnalysis.Webhooks {
|
||||
if webhook.Type == flaggerv1.RollbackHook {
|
||||
if webhook.Type == flaggerv1alpha3.RollbackHook {
|
||||
err := CallWebhook(canary.Name, canary.Namespace, phase, webhook)
|
||||
if err != nil {
|
||||
c.recordEventInfof(canary, "Rollback hook %s not signaling a rollback", webhook.Name)
|
||||
@@ -746,11 +747,11 @@ func (c *Controller) runRollbackHooks(canary *flaggerv1.Canary, phase flaggerv1.
|
||||
return false
|
||||
}
|
||||
|
||||
func (c *Controller) runAnalysis(r *flaggerv1.Canary) bool {
|
||||
func (c *Controller) runAnalysis(r *flaggerv1alpha3.Canary) bool {
|
||||
// run external checks
|
||||
for _, webhook := range r.Spec.CanaryAnalysis.Webhooks {
|
||||
if webhook.Type == "" || webhook.Type == flaggerv1.RolloutHook {
|
||||
err := CallWebhook(r.Name, r.Namespace, flaggerv1.CanaryPhaseProgressing, webhook)
|
||||
if webhook.Type == "" || webhook.Type == flaggerv1alpha3.RolloutHook {
|
||||
err := CallWebhook(r.Name, r.Namespace, flaggerv1alpha3.CanaryPhaseProgressing, webhook)
|
||||
if err != nil {
|
||||
c.recordEventWarningf(r, "Halt %s.%s advancement external check %s failed %v",
|
||||
r.Name, r.Namespace, webhook.Name, err)
|
||||
@@ -759,6 +760,15 @@ func (c *Controller) runAnalysis(r *flaggerv1.Canary) bool {
|
||||
}
|
||||
}
|
||||
|
||||
ok := c.runBuiltinMetricChecks(r)
|
||||
if !ok {
|
||||
return ok
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func (c *Controller) runBuiltinMetricChecks(r *flaggerv1alpha3.Canary) bool {
|
||||
// override the global provider if one is specified in the canary spec
|
||||
var metricsProvider string
|
||||
// set the metrics provider to Crossover Prometheus when Crossover is the mesh provider
|
||||
@@ -786,11 +796,9 @@ func (c *Controller) runAnalysis(r *flaggerv1.Canary) bool {
|
||||
observerFactory := c.observerFactory
|
||||
|
||||
// override the global metrics server if one is specified in the canary spec
|
||||
metricsServer := c.observerFactory.Client.GetMetricsServer()
|
||||
if r.Spec.MetricsServer != "" {
|
||||
metricsServer = r.Spec.MetricsServer
|
||||
var err error
|
||||
observerFactory, err = metrics.NewFactory(metricsServer, 5*time.Second)
|
||||
observerFactory, err = observers.NewFactory(r.Spec.MetricsServer)
|
||||
if err != nil {
|
||||
c.recordEventErrorf(r, "Error building Prometheus client for %s %v", r.Spec.MetricsServer, err)
|
||||
return false
|
||||
@@ -805,13 +813,13 @@ func (c *Controller) runAnalysis(r *flaggerv1.Canary) bool {
|
||||
}
|
||||
|
||||
if metric.Name == "request-success-rate" {
|
||||
val, err := observer.GetRequestSuccessRate(r.Spec.TargetRef.Name, r.Namespace, metric.Interval)
|
||||
val, err := observer.GetRequestSuccessRate(toMetricModel(r, metric.Interval))
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "no values found") {
|
||||
c.recordEventWarningf(r, "Halt advancement no values found for %s metric %s probably %s.%s is not receiving traffic",
|
||||
metricsProvider, metric.Name, r.Spec.TargetRef.Name, r.Namespace)
|
||||
} else {
|
||||
c.recordEventErrorf(r, "Metrics server %s query failed: %v", metricsServer, err)
|
||||
c.recordEventErrorf(r, "Prometheus query failed: %v", err)
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -825,13 +833,13 @@ func (c *Controller) runAnalysis(r *flaggerv1.Canary) bool {
|
||||
}
|
||||
|
||||
if metric.Name == "request-duration" {
|
||||
val, err := observer.GetRequestDuration(r.Spec.TargetRef.Name, r.Namespace, metric.Interval)
|
||||
val, err := observer.GetRequestDuration(toMetricModel(r, metric.Interval))
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "no values found") {
|
||||
c.recordEventWarningf(r, "Halt advancement no values found for %s metric %s probably %s.%s is not receiving traffic",
|
||||
metricsProvider, metric.Name, r.Spec.TargetRef.Name, r.Namespace)
|
||||
} else {
|
||||
c.recordEventErrorf(r, "Metrics server %s query failed: %v", metricsServer, err)
|
||||
c.recordEventErrorf(r, "Prometheus query failed: %v", err)
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -853,7 +861,7 @@ func (c *Controller) runAnalysis(r *flaggerv1.Canary) bool {
|
||||
c.recordEventWarningf(r, "Halt advancement no values found for custom metric: %s",
|
||||
metric.Name)
|
||||
} else {
|
||||
c.recordEventErrorf(r, "Metrics server %s query failed for %s: %v", metricsServer, metric.Name, err)
|
||||
c.recordEventErrorf(r, "Prometheus query failed for %s: %v", metric.Name, err)
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -868,7 +876,26 @@ func (c *Controller) runAnalysis(r *flaggerv1.Canary) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (c *Controller) rollback(canary *flaggerv1.Canary, canaryController canary.Controller, meshRouter router.Interface) {
|
||||
func toMetricModel(r *flaggerv1alpha3.Canary, interval string) flaggerv1alpha1.MetricTemplateModel {
|
||||
service := r.Spec.TargetRef.Name
|
||||
if r.Spec.Service.Name != "" {
|
||||
service = r.Spec.Service.Name
|
||||
}
|
||||
ingress := r.Spec.TargetRef.Name
|
||||
if r.Spec.IngressRef != nil {
|
||||
ingress = r.Spec.IngressRef.Name
|
||||
}
|
||||
return flaggerv1alpha1.MetricTemplateModel{
|
||||
Name: r.Name,
|
||||
Namespace: r.Namespace,
|
||||
Target: r.Spec.TargetRef.Name,
|
||||
Service: service,
|
||||
Ingress: ingress,
|
||||
Interval: interval,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Controller) rollback(canary *flaggerv1alpha3.Canary, canaryController canary.Controller, meshRouter router.Interface) {
|
||||
if canary.Status.FailedChecks >= canary.Spec.CanaryAnalysis.Threshold {
|
||||
c.recordEventWarningf(canary, "Rolling back %s.%s failed checks threshold reached %v",
|
||||
canary.Name, canary.Namespace, canary.Status.FailedChecks)
|
||||
@@ -885,7 +912,7 @@ func (c *Controller) rollback(canary *flaggerv1.Canary, canaryController canary.
|
||||
}
|
||||
|
||||
canaryPhaseFailed := canary.DeepCopy()
|
||||
canaryPhaseFailed.Status.Phase = flaggerv1.CanaryPhaseFailed
|
||||
canaryPhaseFailed.Status.Phase = flaggerv1alpha3.CanaryPhaseFailed
|
||||
c.recordEventWarningf(canaryPhaseFailed, "Canary failed! Scaling down %s.%s",
|
||||
canaryPhaseFailed.Name, canaryPhaseFailed.Namespace)
|
||||
|
||||
@@ -898,11 +925,11 @@ func (c *Controller) rollback(canary *flaggerv1.Canary, canaryController canary.
|
||||
}
|
||||
|
||||
// mark canary as failed
|
||||
if err := canaryController.SyncStatus(canary, flaggerv1.CanaryStatus{Phase: flaggerv1.CanaryPhaseFailed, CanaryWeight: 0}); err != nil {
|
||||
if err := canaryController.SyncStatus(canary, flaggerv1alpha3.CanaryStatus{Phase: flaggerv1alpha3.CanaryPhaseFailed, CanaryWeight: 0}); err != nil {
|
||||
c.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)).Errorf("%v", err)
|
||||
return
|
||||
}
|
||||
|
||||
c.recorder.SetStatus(canary, flaggerv1.CanaryPhaseFailed)
|
||||
c.runPostRolloutHooks(canary, flaggerv1.CanaryPhaseFailed)
|
||||
c.recorder.SetStatus(canary, flaggerv1alpha3.CanaryPhaseFailed)
|
||||
c.runPostRolloutHooks(canary, flaggerv1alpha3.CanaryPhaseFailed)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user