From 29075264526234d057f191bc11c9da51781f75d5 Mon Sep 17 00:00:00 2001 From: Daniel Albuquerque Date: Mon, 14 Sep 2020 19:46:35 +0100 Subject: [PATCH] Do not promote when not ready on skip analysis --- pkg/controller/scheduler.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/pkg/controller/scheduler.go b/pkg/controller/scheduler.go index 11fac989..db9edfa1 100644 --- a/pkg/controller/scheduler.go +++ b/pkg/controller/scheduler.go @@ -236,7 +236,7 @@ func (c *Controller) advanceCanary(name string, namespace string) { } // check if analysis should be skipped - if skip := c.shouldSkipAnalysis(cd, canaryController, meshRouter); skip { + if skip := c.shouldSkipAnalysis(cd, canaryController, meshRouter, err, retriable); skip { return } @@ -616,11 +616,20 @@ func (c *Controller) runAnalysis(canary *flaggerv1.Canary) bool { return true } -func (c *Controller) shouldSkipAnalysis(canary *flaggerv1.Canary, canaryController canary.Controller, meshRouter router.Interface) bool { +func (c *Controller) shouldSkipAnalysis(canary *flaggerv1.Canary, canaryController canary.Controller, meshRouter router.Interface, err error, retriable bool) bool { if !canary.SkipAnalysis() { return false } + // regardless if analysis is being skipped, rollback if canary failed to progress + if !retriable || canary.Status.FailedChecks >= canary.GetAnalysisThreshold() { + c.recordEventWarningf(canary, "Rolling back %s.%s progress deadline exceeded %v", canary.Name, canary.Namespace, err) + c.alert(canary, fmt.Sprintf("Progress deadline exceeded %v", err), false, flaggerv1.SeverityError) + c.rollback(canary, canaryController, meshRouter) + + return true + } + // route all traffic to primary primaryWeight := 100 canaryWeight := 0