github/flagger
1
0
Fork 0
mirror of https://github.com/fluxcd/flagger.git synced 2026-04-15 06:57:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #307 from weaveworks/confirm-promotion

Browse Source 
Implement confirm-promotion hook
This commit is contained in:
Stefan Prodan
2019-09-23 12:32:52 +03:00
committed by GitHub
parent 2b6966d8e3 96ccfa54fb
commit 9845578cdd
8 changed files with 68 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Makefile
+2 -9
View File
@@ -7,15 +7,8 @@ LT_VERSION?=$(shell grep 'VERSION' cmd/loadtester/main.go | awk '{ print $$4 }'
TS=$(shell date +%Y-%m-%d_%H-%M-%S)
run:
GO111MODULE=on go run cmd/flagger/* -kubeconfig=$$HOME/.kube/config -log-level=info -mesh-provider=istio -namespace=test \
-metrics-server=https://prometheus.istio.weavedx.com \
-enable-leader-election=true
run2:
GO111MODULE=on go run cmd/flagger/* -kubeconfig=$$HOME/.kube/config -log-level=info -mesh-provider=istio -namespace=test \
-metrics-server=https://prometheus.istio.weavedx.com \
-enable-leader-election=true \
-port=9092
GO111MODULE=on go run cmd/flagger/* -kubeconfig=$$HOME/.kube/config -log-level=info -mesh-provider=istio -namespace=test-istio \
-metrics-server=https://prometheus.istio.flagger.dev
run-appmesh:
GO111MODULE=on go run cmd/flagger/* -kubeconfig=$$HOME/.kube/config -log-level=info -mesh-provider=appmesh \
artifacts/flagger/crd.yaml
+2 -1
View File
@@ -215,7 +215,7 @@ spec:
properties:
items:
type: object
required: ["name", "url", "timeout"]
required: ["name", "url"]
properties:
name:
description: Name of the webhook
@@ -228,6 +228,7 @@ spec:
- confirm-rollout
- pre-rollout
- rollout
- confirm-promotion
- post-rollout
url:
description: URL address of this webhook
charts/flagger/templates/crd.yaml
+2 -1
View File
@@ -216,7 +216,7 @@ spec:
properties:
items:
type: object
required: ['name', 'url', 'timeout']
required: ["name", "url"]
properties:
name:
description: Name of the webhook
@@ -229,6 +229,7 @@ spec:
- confirm-rollout
- pre-rollout
- rollout
- confirm-promotion
- post-rollout
url:
description: URL address of this webhook
docs/gitbook/how-it-works.md
+21 -2
View File
@@ -663,6 +663,9 @@ The canary advancement is paused if a pre-rollout hook fails and if the number o
threshold the canary will be rollback.
* Rollout hooks are executed during the analysis on each iteration before the metric checks.
If a rollout hook call fails the canary advancement is paused and eventfully rolled back.
* Confirm-promotion hooks are executed before the promotion step.
The canary promotion is paused until the hooks return HTTP 200.
While the promotion is paused, Flagger will continue to run the metrics checks and rollout hooks.
* Post-rollout hooks are executed after the canary has been promoted or rolled back.
If a post rollout hook fails the error is logged.
@@ -687,6 +690,9 @@ Spec:
timeout: 15s
metadata:
cmd: "hey -z 1m -q 5 -c 2 http://podinfo-canary.test:9898/"
- name: "promotion gate"
type: confirm-promotion
url: http://flagger-loadtester.test/gate/approve
- name: "notify"
type: post-rollout
url: http://telegram.bot:8080/
@@ -914,8 +920,8 @@ Note that you should create a ConfigMap with your Bats tests and mount it inside
### Manual Gating
For manual approval of a canary deployment you can use the `confirm-rollout` webhook.
The confirmation hooks are executed before the pre-rollout hooks.
For manual approval of a canary deployment you can use the `confirm-rollout` and `confirm-promotion` webhooks.
The confirmation rollout hooks are executed before the pre-rollout hooks.
Flagger will halt the canary traffic shifting and analysis until the confirm webhook returns HTTP status 200.
Manual gating with Flagger's tester:
@@ -974,3 +980,16 @@ kubectl get canary/podinfo
NAME STATUS WEIGHT
podinfo Waiting 0
```
The `confirm-promotion` hook type can be used to manually approve the canary promotion.
While the promotion is paused, Flagger will continue to run the metrics checks and load tests.
```yaml
canaryAnalysis:
webhooks:
- name: "promotion gate"
type: confirm-promotion
url: http://flagger-loadtester.test/gate/halt
```
If you have notifications enabled, Flagger will post a message to Slack or MS Teams if a canary promotion is waiting for approval.
kustomize/base/flagger/crd.yaml
+2 -1
View File
@@ -215,7 +215,7 @@ spec:
properties:
items:
type: object
required: ["name", "url", "timeout"]
required: ["name", "url"]
properties:
name:
description: Name of the webhook
@@ -228,6 +228,7 @@ spec:
- confirm-rollout
- pre-rollout
- rollout
- confirm-promotion
- post-rollout
url:
description: URL address of this webhook
pkg/apis/flagger/v1alpha3/types.go
+2
View File
@@ -139,6 +139,8 @@ const (
PostRolloutHook HookType = "post-rollout"
// ConfirmRolloutHook halt canary analysis until webhook returns HTTP 200
ConfirmRolloutHook HookType = "confirm-rollout"
// ConfirmPromotionHook halt canary promotion until webhook returns HTTP 200
ConfirmPromotionHook HookType = "confirm-promotion"
)
// CanaryWebhook holds the reference to external checks used for canary analysis
pkg/controller/scheduler.go
+34
View File
@@ -323,6 +323,11 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
return
}
// check promotion gate
if promote := c.runConfirmPromotionHooks(cd); !promote {
return
}
// promote canary - max iterations reached
if cd.Spec.CanaryAnalysis.Iterations == cd.Status.Iterations {
c.recordEventInfof(cd, "Copying %s.%s template spec to %s.%s",
@@ -375,6 +380,11 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
return
}
// check promotion gate
if promote := c.runConfirmPromotionHooks(cd); !promote {
return
}
// route all traffic to canary - max iterations reached
if cd.Spec.CanaryAnalysis.Iterations == cd.Status.Iterations {
if provider != "kubernetes" {
@@ -445,6 +455,13 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
primaryWeight = 100
}
// check promotion gate
if canaryWeight >= maxWeight {
if promote := c.runConfirmPromotionHooks(cd); !promote {
return
}
}
if err := meshRouter.SetRoutes(cd, primaryWeight, canaryWeight); err != nil {
c.recordEventWarningf(cd, "%v", err)
return
@@ -637,6 +654,23 @@ func (c *Controller) runConfirmRolloutHooks(canary *flaggerv1.Canary) bool {
return true
}
func (c *Controller) runConfirmPromotionHooks(canary *flaggerv1.Canary) bool {
for _, webhook := range canary.Spec.CanaryAnalysis.Webhooks {
if webhook.Type == flaggerv1.ConfirmPromotionHook {
err := CallWebhook(canary.Name, canary.Namespace, flaggerv1.CanaryPhaseProgressing, webhook)
if err != nil {
c.recordEventWarningf(canary, "Halt %s.%s advancement waiting for promotion approval %s",
canary.Name, canary.Namespace, webhook.Name)
c.sendNotification(canary, "Canary promotion is waiting for approval.", false, false)
return false
} else {
c.recordEventInfof(canary, "Confirm-promotion check %s passed", webhook.Name)
}
}
}
return true
}
func (c *Controller) runPreRolloutHooks(canary *flaggerv1.Canary) bool {
for _, webhook := range canary.Spec.CanaryAnalysis.Webhooks {
if webhook.Type == flaggerv1.PreRolloutHook {
test/e2e-tests.sh
+3
View File
@@ -257,6 +257,9 @@ spec:
type: cmd
cmd: "hey -z 10m -q 10 -c 2 -H 'Cookie: type=insider' http://podinfo-canary.test:9898/"
logCmdOutput: "true"
- name: promote-gate
type: confirm-promotion
url: http://flagger-loadtester.test/gate/approve
- name: post
type: post-rollout
url: http://flagger-loadtester.test/