From 8282f86d9c0b57574795a81ec830ab4a76daa3dd Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Sun, 22 Sep 2019 13:23:19 +0300 Subject: [PATCH] Implement confirm-promotion hook The confirm promotion hooks are executed right before the promotion step. The canary promotion is paused until the hooks return HTTP 200. While the promotion is paused, Flagger will continue to run the metrics checks and load tests. --- pkg/apis/flagger/v1alpha3/types.go | 2 ++ pkg/controller/scheduler.go | 34 ++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/pkg/apis/flagger/v1alpha3/types.go b/pkg/apis/flagger/v1alpha3/types.go index 9cd7cea4..20a98cd2 100755 --- a/pkg/apis/flagger/v1alpha3/types.go +++ b/pkg/apis/flagger/v1alpha3/types.go @@ -139,6 +139,8 @@ const ( PostRolloutHook HookType = "post-rollout" // ConfirmRolloutHook halt canary analysis until webhook returns HTTP 200 ConfirmRolloutHook HookType = "confirm-rollout" + // ConfirmPromotionHook halt canary promotion until webhook returns HTTP 200 + ConfirmPromotionHook HookType = "confirm-promotion" ) // CanaryWebhook holds the reference to external checks used for canary analysis diff --git a/pkg/controller/scheduler.go b/pkg/controller/scheduler.go index 230eb9d0..2dbeea82 100644 --- a/pkg/controller/scheduler.go +++ b/pkg/controller/scheduler.go @@ -323,6 +323,11 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh return } + // check promotion gate + if promote := c.runConfirmPromotionHooks(cd); !promote { + return + } + // promote canary - max iterations reached if cd.Spec.CanaryAnalysis.Iterations == cd.Status.Iterations { c.recordEventInfof(cd, "Copying %s.%s template spec to %s.%s", @@ -375,6 +380,11 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh return } + // check promotion gate + if promote := c.runConfirmPromotionHooks(cd); !promote { + return + } + // route all traffic to canary - max iterations reached if cd.Spec.CanaryAnalysis.Iterations == cd.Status.Iterations { if provider != "kubernetes" { @@ -445,6 +455,13 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh primaryWeight = 100 } + // check promotion gate + if canaryWeight >= maxWeight { + if promote := c.runConfirmPromotionHooks(cd); !promote { + return + } + } + if err := meshRouter.SetRoutes(cd, primaryWeight, canaryWeight); err != nil { c.recordEventWarningf(cd, "%v", err) return @@ -637,6 +654,23 @@ func (c *Controller) runConfirmRolloutHooks(canary *flaggerv1.Canary) bool { return true } +func (c *Controller) runConfirmPromotionHooks(canary *flaggerv1.Canary) bool { + for _, webhook := range canary.Spec.CanaryAnalysis.Webhooks { + if webhook.Type == flaggerv1.ConfirmPromotionHook { + err := CallWebhook(canary.Name, canary.Namespace, flaggerv1.CanaryPhaseProgressing, webhook) + if err != nil { + c.recordEventWarningf(canary, "Halt %s.%s advancement waiting for promotion approval %s", + canary.Name, canary.Namespace, webhook.Name) + c.sendNotification(canary, "Canary promotion is waiting for approval.", false, false) + return false + } else { + c.recordEventInfof(canary, "Confirm-promotion check %s passed", webhook.Name) + } + } + } + return true +} + func (c *Controller) runPreRolloutHooks(canary *flaggerv1.Canary) bool { for _, webhook := range canary.Spec.CanaryAnalysis.Webhooks { if webhook.Type == flaggerv1.PreRolloutHook {