From 873141b9caeb68de06bfc743bc680c92e8a74525 Mon Sep 17 00:00:00 2001 From: Mayank Shah Date: Wed, 10 Mar 2021 21:19:42 +0530 Subject: [PATCH 1/2] Add webhook `confirm-traffic-increase` for manually approving traffic increase Signed-off-by: Mayank Shah --- artifacts/flagger/crd.yaml | 1 + charts/flagger/crds/crd.yaml | 1 + pkg/apis/flagger/v1beta1/canary.go | 2 ++ pkg/controller/scheduler.go | 6 ++++++ pkg/controller/scheduler_hooks.go | 16 ++++++++++++++++ 5 files changed, 26 insertions(+) diff --git a/artifacts/flagger/crd.yaml b/artifacts/flagger/crd.yaml index 41c7ff87..6d3b918e 100644 --- a/artifacts/flagger/crd.yaml +++ b/artifacts/flagger/crd.yaml @@ -940,6 +940,7 @@ spec: - post-rollout - event - rollback + - confirm-traffic-increase url: description: URL address of this webhook type: string diff --git a/charts/flagger/crds/crd.yaml b/charts/flagger/crds/crd.yaml index 41c7ff87..6d3b918e 100644 --- a/charts/flagger/crds/crd.yaml +++ b/charts/flagger/crds/crd.yaml @@ -940,6 +940,7 @@ spec: - post-rollout - event - rollback + - confirm-traffic-increase url: description: URL address of this webhook type: string diff --git a/pkg/apis/flagger/v1beta1/canary.go b/pkg/apis/flagger/v1beta1/canary.go index 60644e66..a660095b 100644 --- a/pkg/apis/flagger/v1beta1/canary.go +++ b/pkg/apis/flagger/v1beta1/canary.go @@ -314,6 +314,8 @@ const ( EventHook HookType = "event" // RollbackHook rollback canary analysis if webhook returns HTTP 200 RollbackHook HookType = "rollback" + // ConfirmTrafficIncreaseHook increases traffic weight if webhook returns HTTP 200 + ConfirmTrafficIncreaseHook = "confirm-traffic-increase" ) // CanaryWebhook holds the reference to external checks used for canary analysis diff --git a/pkg/controller/scheduler.go b/pkg/controller/scheduler.go index 36b8759a..6537983a 100644 --- a/pkg/controller/scheduler.go +++ b/pkg/controller/scheduler.go @@ -412,6 +412,12 @@ func (c *Controller) advanceCanary(name string, namespace string) { // strategy: Canary progressive traffic increase if c.nextStepWeight(cd, canaryWeight) > 0 { + // run hook only if traffic is not mirrored + if !mirrored { + if promote := c.runConfirmTrafficIncreaseHooks(cd); !promote { + return + } + } c.runCanary(cd, canaryController, meshRouter, mirrored, canaryWeight, primaryWeight, maxWeight) } diff --git a/pkg/controller/scheduler_hooks.go b/pkg/controller/scheduler_hooks.go index d535cdd9..cb551630 100644 --- a/pkg/controller/scheduler_hooks.go +++ b/pkg/controller/scheduler_hooks.go @@ -23,6 +23,22 @@ import ( "github.com/fluxcd/flagger/pkg/canary" ) +func (c *Controller) runConfirmTrafficIncreaseHooks(canary *flaggerv1.Canary) bool { + for _, webhook := range canary.GetAnalysis().Webhooks { + if webhook.Type == flaggerv1.ConfirmTrafficIncreaseHook { + err := CallWebhook(canary.Name, canary.Namespace, flaggerv1.CanaryPhaseProgressing, webhook) + if err != nil { + c.recordEventWarningf(canary, "Halt %s.%s advancement waiting for traffic increase approval %s", + canary.Name, canary.Namespace, webhook.Name) + c.alert(canary, "Canary traffic increase is waiting for approval.", false, flaggerv1.SeverityWarn) + return false + } + c.recordEventInfof(canary, "Confirm-traffic-increase check %s passed", webhook.Name) + } + } + return true +} + func (c *Controller) runConfirmRolloutHooks(canary *flaggerv1.Canary, canaryController canary.Controller) bool { for _, webhook := range canary.GetAnalysis().Webhooks { if webhook.Type == flaggerv1.ConfirmRolloutHook { From 164bbb884d3697c4f90adfb60210a4ec14efaf7d Mon Sep 17 00:00:00 2001 From: Mayank Shah Date: Tue, 16 Mar 2021 20:30:57 +0530 Subject: [PATCH 2/2] Add docs for confirm-traffic-increase webhook Signed-off-by: Mayank Shah --- docs/gitbook/usage/webhooks.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/gitbook/usage/webhooks.md b/docs/gitbook/usage/webhooks.md index 08b3c398..a3eba9f4 100644 --- a/docs/gitbook/usage/webhooks.md +++ b/docs/gitbook/usage/webhooks.md @@ -16,6 +16,9 @@ There are several types of hooks: * **rollout** hooks are executed during the analysis on each iteration before the metric checks. If a rollout hook call fails the canary advancement is paused and eventfully rolled back. +* **confirm-traffic-increase** hooks are executed right before the weight on the canary is increased. The canary + advancement is paused until this hook returns HTTP 200. + * **confirm-promotion** hooks are executed before the promotion step. The canary promotion is paused until the hooks return HTTP 200. While the promotion is paused, Flagger will continue to run the metrics checks and rollout hooks. @@ -51,6 +54,9 @@ Spec: timeout: 15s metadata: cmd: "hey -z 1m -q 5 -c 2 http://podinfo-canary.test:9898/" + - name: "traffic increase gate" + type: confirm-traffic-increase + url: http://flagger-loadtester.test/gate/approve - name: "promotion gate" type: confirm-promotion url: http://flagger-loadtester.test/gate/approve @@ -347,7 +353,8 @@ the web-hook will try to call Concord before timing out (Default is 30s). ## Manual Gating For manual approval of a canary deployment you can use the `confirm-rollout` and `confirm-promotion` webhooks. -The confirmation rollout hooks are executed before the pre-rollout hooks. +The confirmation rollout hooks are executed before the pre-rollout hooks. For manually approving traffic weight increase, +you can use the `confirm-traffic-increase` webhook. Flagger will halt the canary traffic shifting and analysis until the confirm webhook returns HTTP status 200. For manual rollback of a canary deployment you can use the `rollback` webhook.