From dfdcfed26eb3df6c1330faad1514d2006596f54e Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 24 Jul 2019 12:00:04 +0300 Subject: [PATCH 01/15] Add `Waiting` canary status phase means the canary rollout is paused (waiting for confirmation to proceed) --- artifacts/flagger/crd.yaml | 1 + charts/flagger/templates/crd.yaml | 1 + kustomize/base/flagger/crd.yaml | 1 + pkg/apis/flagger/v1alpha3/status.go | 2 ++ pkg/canary/status.go | 3 +++ 5 files changed, 8 insertions(+) diff --git a/artifacts/flagger/crd.yaml b/artifacts/flagger/crd.yaml index 1b235577..ed54fbfe 100644 --- a/artifacts/flagger/crd.yaml +++ b/artifacts/flagger/crd.yaml @@ -174,6 +174,7 @@ spec: - "" - Initializing - Initialized + - Waiting - Progressing - Succeeded - Failed diff --git a/charts/flagger/templates/crd.yaml b/charts/flagger/templates/crd.yaml index fcf326f1..e7cd7641 100644 --- a/charts/flagger/templates/crd.yaml +++ b/charts/flagger/templates/crd.yaml @@ -175,6 +175,7 @@ spec: - "" - Initializing - Initialized + - Waiting - Progressing - Succeeded - Failed diff --git a/kustomize/base/flagger/crd.yaml b/kustomize/base/flagger/crd.yaml index 1b235577..ed54fbfe 100644 --- a/kustomize/base/flagger/crd.yaml +++ b/kustomize/base/flagger/crd.yaml @@ -174,6 +174,7 @@ spec: - "" - Initializing - Initialized + - Waiting - Progressing - Succeeded - Failed diff --git a/pkg/apis/flagger/v1alpha3/status.go b/pkg/apis/flagger/v1alpha3/status.go index 5c43b0fd..b0ca1fd9 100644 --- a/pkg/apis/flagger/v1alpha3/status.go +++ b/pkg/apis/flagger/v1alpha3/status.go @@ -43,6 +43,8 @@ const ( // CanaryPhaseInitialized means the primary deployment, hpa and ClusterIP services // have been created along with the service mesh or ingress objects CanaryPhaseInitialized CanaryPhase = "Initialized" + // CanaryPhaseWaiting means the canary rollout is paused (waiting for confirmation to proceed) + CanaryPhaseWaiting CanaryPhase = "Waiting" // CanaryPhaseProgressing means the canary analysis is underway CanaryPhaseProgressing CanaryPhase = "Progressing" // CanaryPhaseSucceeded means the canary analysis has been successful diff --git a/pkg/canary/status.go b/pkg/canary/status.go index 2001470c..cfbacd9c 100644 --- a/pkg/canary/status.go +++ b/pkg/canary/status.go @@ -205,6 +205,9 @@ func (c *Deployer) MakeStatusConditions(canaryStatus flaggerv1.CanaryStatus, case flaggerv1.CanaryPhaseInitialized: status = corev1.ConditionTrue message = "Deployment initialization completed." + case flaggerv1.CanaryPhaseWaiting: + status = corev1.ConditionUnknown + message = "Waiting for approval." case flaggerv1.CanaryPhaseProgressing: status = corev1.ConditionUnknown message = "New revision detected, starting canary analysis." From 919dafa5671f65b251828f49dd096747a1aae5ff Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 24 Jul 2019 12:02:44 +0300 Subject: [PATCH 02/15] Add gate halt and approve endpoints --- artifacts/loadtester/deployment.yaml | 2 +- charts/loadtester/Chart.yaml | 4 ++-- charts/loadtester/values.yaml | 2 +- cmd/loadtester/main.go | 2 +- kustomize/tester/deployment.yaml | 2 +- pkg/loadtester/server.go | 8 ++++++++ 6 files changed, 14 insertions(+), 6 deletions(-) diff --git a/artifacts/loadtester/deployment.yaml b/artifacts/loadtester/deployment.yaml index c5f1004d..33615515 100644 --- a/artifacts/loadtester/deployment.yaml +++ b/artifacts/loadtester/deployment.yaml @@ -17,7 +17,7 @@ spec: spec: containers: - name: loadtester - image: weaveworks/flagger-loadtester:0.5.0 + image: weaveworks/flagger-loadtester:0.6.0 imagePullPolicy: IfNotPresent ports: - name: http diff --git a/charts/loadtester/Chart.yaml b/charts/loadtester/Chart.yaml index 53f5c232..3e651a04 100644 --- a/charts/loadtester/Chart.yaml +++ b/charts/loadtester/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 name: loadtester -version: 0.5.0 -appVersion: 0.5.0 +version: 0.6.0 +appVersion: 0.6.0 kubeVersion: ">=1.11.0-0" engine: gotpl description: Flagger's load testing services based on rakyll/hey and bojand/ghz that generates traffic during canary analysis when configured as a webhook. diff --git a/charts/loadtester/values.yaml b/charts/loadtester/values.yaml index 486ab92b..3a03a5cc 100644 --- a/charts/loadtester/values.yaml +++ b/charts/loadtester/values.yaml @@ -2,7 +2,7 @@ replicaCount: 1 image: repository: weaveworks/flagger-loadtester - tag: 0.4.0 + tag: 0.6.0 pullPolicy: IfNotPresent logLevel: info diff --git a/cmd/loadtester/main.go b/cmd/loadtester/main.go index a042f3b7..4e29ffa4 100644 --- a/cmd/loadtester/main.go +++ b/cmd/loadtester/main.go @@ -10,7 +10,7 @@ import ( "time" ) -var VERSION = "0.5.0" +var VERSION = "0.6.0" var ( logLevel string port string diff --git a/kustomize/tester/deployment.yaml b/kustomize/tester/deployment.yaml index c5f1004d..33615515 100644 --- a/kustomize/tester/deployment.yaml +++ b/kustomize/tester/deployment.yaml @@ -17,7 +17,7 @@ spec: spec: containers: - name: loadtester - image: weaveworks/flagger-loadtester:0.5.0 + image: weaveworks/flagger-loadtester:0.6.0 imagePullPolicy: IfNotPresent ports: - name: http diff --git a/pkg/loadtester/server.go b/pkg/loadtester/server.go index c8ce4c1d..ca907756 100644 --- a/pkg/loadtester/server.go +++ b/pkg/loadtester/server.go @@ -21,6 +21,14 @@ func ListenAndServe(port string, timeout time.Duration, logger *zap.SugaredLogge w.WriteHeader(http.StatusOK) w.Write([]byte("OK")) }) + mux.HandleFunc("/gate/approve", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte("OK")) + }) + mux.HandleFunc("/gate/halt", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusForbidden) + w.Write([]byte("Forbidden")) + }) mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { body, err := ioutil.ReadAll(r.Body) if err != nil { From c46c7b9e2145f76e9576d54cbe512cedad04a9a6 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 24 Jul 2019 12:04:05 +0300 Subject: [PATCH 03/15] Add canary status conditions to docs --- docs/gitbook/how-it-works.md | 47 ++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/docs/gitbook/how-it-works.md b/docs/gitbook/how-it-works.md index 439821c7..72bfb0b1 100644 --- a/docs/gitbook/how-it-works.md +++ b/docs/gitbook/how-it-works.md @@ -105,6 +105,53 @@ convention you can specify your label with the `-selector-labels` flag. The target deployment should expose a TCP port that will be used by Flagger to create the ClusterIP Service and the Istio Virtual Service. The container port from the target deployment should match the `service.port` value. +### Canary status + +Get the current status of canary deployments cluster wide: + +```bash +kubectl get canaries --all-namespaces + +NAMESPACE NAME STATUS WEIGHT LASTTRANSITIONTIME +test podinfo Progressing 15 2019-06-30T14:05:07Z +prod frontend Succeeded 0 2019-06-30T16:15:07Z +prod backend Failed 0 2019-06-30T17:05:07Z +``` + +The status condition reflects the last know state of the canary analysis: + +```bash +kubectl -n test get canary/podinfo -oyaml | awk '/status/,0' +``` + +A successful rollout status: + +```yaml +status: + canaryWeight: 0 + failedChecks: 0 + iterations: 0 + lastAppliedSpec: "14788816656920327485" + lastPromotedSpec: "14788816656920327485" + conditions: + - lastTransitionTime: "2019-07-10T08:23:18Z" + lastUpdateTime: "2019-07-10T08:23:18Z" + message: Canary analysis completed successfully, promotion finished. + reason: Succeeded + status: "True" + type: Promoted +``` + +The `Promoted` status condition can have one of the following reasons: Initialized, Waiting, Progressing, Succeeded or Failed. +A failed canary will have the promoted status set to `false`, +the reason to `failed` and the last applied spec will be different to the last promoted one. + +Wait for a successful rollout: + +```bash +kubectl wait canary/podinfo --for=condition=promoted +``` + ### Istio routing Flagger creates an Istio Virtual Service and Destination Rules based on the Canary service spec. From 04cbacb6e038e726ee47767e34906891c5fbd584 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 24 Jul 2019 12:09:39 +0300 Subject: [PATCH 04/15] Implement confirm rollout gate and hook The confirm-rollout hooks are executed before the pre-rollout hooks. Flagger will halt the canary rollout until the confirm webhook returns HTTP status 200. --- pkg/apis/flagger/v1alpha3/types.go | 2 ++ pkg/controller/scheduler.go | 38 +++++++++++++++++++++++++++++- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/pkg/apis/flagger/v1alpha3/types.go b/pkg/apis/flagger/v1alpha3/types.go index 1f995e02..9cd7cea4 100755 --- a/pkg/apis/flagger/v1alpha3/types.go +++ b/pkg/apis/flagger/v1alpha3/types.go @@ -137,6 +137,8 @@ const ( PreRolloutHook HookType = "pre-rollout" // PreRolloutHook execute webhook after the canary analysis PostRolloutHook HookType = "post-rollout" + // ConfirmRolloutHook halt canary analysis until webhook returns HTTP 200 + ConfirmRolloutHook HookType = "confirm-rollout" ) // CanaryWebhook holds the reference to external checks used for canary analysis diff --git a/pkg/controller/scheduler.go b/pkg/controller/scheduler.go index de02d079..f08ef22e 100644 --- a/pkg/controller/scheduler.go +++ b/pkg/controller/scheduler.go @@ -134,6 +134,12 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh return } + isApproved := c.runConfirmRolloutHooks(cd) + + if !isApproved { + return + } + // set max weight default value to 100% maxWeight := 100 if cd.Spec.CanaryAnalysis.MaxWeight > 0 { @@ -454,7 +460,10 @@ func (c *Controller) shouldSkipAnalysis(cd *flaggerv1.Canary, meshRouter router. } func (c *Controller) shouldAdvance(cd *flaggerv1.Canary) (bool, error) { - if cd.Status.LastAppliedSpec == "" || cd.Status.Phase == flaggerv1.CanaryPhaseInitializing || cd.Status.Phase == flaggerv1.CanaryPhaseProgressing { + if cd.Status.LastAppliedSpec == "" || + cd.Status.Phase == flaggerv1.CanaryPhaseInitializing || + cd.Status.Phase == flaggerv1.CanaryPhaseProgressing || + cd.Status.Phase == flaggerv1.CanaryPhaseWaiting { return true, nil } @@ -523,6 +532,33 @@ func (c *Controller) hasCanaryRevisionChanged(cd *flaggerv1.Canary) bool { return false } +func (c *Controller) runConfirmRolloutHooks(canary *flaggerv1.Canary) bool { + for _, webhook := range canary.Spec.CanaryAnalysis.Webhooks { + if webhook.Type == flaggerv1.ConfirmRolloutHook { + err := CallWebhook(canary.Name, canary.Namespace, flaggerv1.CanaryPhaseProgressing, webhook) + if err != nil { + if canary.Status.Phase != flaggerv1.CanaryPhaseWaiting { + c.recordEventWarningf(canary, "Halt %s.%s advancement waiting for approval %s", + canary.Name, canary.Namespace, webhook.Name) + if err := c.deployer.SetStatusPhase(canary, flaggerv1.CanaryPhaseWaiting); err != nil { + c.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)).Errorf("%v", err) + } + } + return false + } else { + if canary.Status.Phase == flaggerv1.CanaryPhaseWaiting { + if err := c.deployer.SetStatusPhase(canary, flaggerv1.CanaryPhaseProgressing); err != nil { + c.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)).Errorf("%v", err) + return false + } + c.recordEventInfof(canary, "Confirm-rollout check %s passed", webhook.Name) + } + } + } + } + return true +} + func (c *Controller) runPreRolloutHooks(canary *flaggerv1.Canary) bool { for _, webhook := range canary.Spec.CanaryAnalysis.Webhooks { if webhook.Type == flaggerv1.PreRolloutHook { From 75d49304f3d58a0289d6d0c9577868b6a6dc0369 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 24 Jul 2019 12:17:11 +0300 Subject: [PATCH 05/15] Add confirm-rollout hook to docs --- docs/gitbook/how-it-works.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/gitbook/how-it-works.md b/docs/gitbook/how-it-works.md index 72bfb0b1..afd2ffab 100644 --- a/docs/gitbook/how-it-works.md +++ b/docs/gitbook/how-it-works.md @@ -655,6 +655,8 @@ The canary analysis can be extended with webhooks. Flagger will call each webhoo determine from the response status code (HTTP 2xx) if the canary is failing or not. There are three types of hooks: +* Confirm-rollout hooks are executed before scaling up the canary deployment and ca be used for manual approval. +The rollout is paused until the hook returns a successful HTTP status code. * Pre-rollout hooks are executed before routing traffic to canary. The canary advancement is paused if a pre-rollout hook fails and if the number of failures reach the threshold the canary will be rollback. @@ -668,6 +670,9 @@ Spec: ```yaml canaryAnalysis: webhooks: + - name: "start gate" + type: confirm-rollout + url: http://flagger-loadtester.test/gate/approve - name: "smoke test" type: pre-rollout url: http://flagger-helmtester.kube-system/ From 28e7e89047218d703d706130bc6f445f6baa40f8 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 24 Jul 2019 16:09:13 +0300 Subject: [PATCH 06/15] Pause or resume analysis on confirmation gate toggle --- pkg/canary/status.go | 2 +- pkg/controller/scheduler.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/canary/status.go b/pkg/canary/status.go index cfbacd9c..a744d437 100644 --- a/pkg/canary/status.go +++ b/pkg/canary/status.go @@ -156,7 +156,7 @@ func (c *Deployer) SetStatusPhase(cd *flaggerv1.Canary, phase flaggerv1.CanaryPh cdCopy.Status.Phase = phase cdCopy.Status.LastTransitionTime = metav1.Now() - if phase != flaggerv1.CanaryPhaseProgressing { + if phase != flaggerv1.CanaryPhaseProgressing && phase != flaggerv1.CanaryPhaseWaiting { cdCopy.Status.CanaryWeight = 0 cdCopy.Status.Iterations = 0 } diff --git a/pkg/controller/scheduler.go b/pkg/controller/scheduler.go index f08ef22e..bca27600 100644 --- a/pkg/controller/scheduler.go +++ b/pkg/controller/scheduler.go @@ -134,9 +134,8 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh return } - isApproved := c.runConfirmRolloutHooks(cd) - - if !isApproved { + // check gates + if isApproved := c.runConfirmRolloutHooks(cd); !isApproved { return } @@ -552,6 +551,7 @@ func (c *Controller) runConfirmRolloutHooks(canary *flaggerv1.Canary) bool { return false } c.recordEventInfof(canary, "Confirm-rollout check %s passed", webhook.Name) + return false } } } From f204fe53f464b612c1cbe14ce1759265e527c209 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 24 Jul 2019 16:14:22 +0300 Subject: [PATCH 07/15] Implement canary gating API with in-memory storage POST /gate/[check|open|close] --- cmd/loadtester/main.go | 4 +- pkg/loadtester/gate.go | 31 +++++++++++++++ pkg/loadtester/server.go | 82 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 115 insertions(+), 2 deletions(-) create mode 100644 pkg/loadtester/gate.go diff --git a/cmd/loadtester/main.go b/cmd/loadtester/main.go index 4e29ffa4..74c4d09b 100644 --- a/cmd/loadtester/main.go +++ b/cmd/loadtester/main.go @@ -47,5 +47,7 @@ func main() { go taskRunner.Start(100*time.Millisecond, stopCh) logger.Infof("Starting load tester v%s API on port %s", VERSION, port) - loadtester.ListenAndServe(port, time.Minute, logger, taskRunner, stopCh) + + gateStorage := loadtester.NewGateStorage("in-memory") + loadtester.ListenAndServe(port, time.Minute, logger, taskRunner, gateStorage, stopCh) } diff --git a/pkg/loadtester/gate.go b/pkg/loadtester/gate.go new file mode 100644 index 00000000..fdacdd7e --- /dev/null +++ b/pkg/loadtester/gate.go @@ -0,0 +1,31 @@ +package loadtester + +import "sync" + +type GateStorage struct { + backend string + data *sync.Map +} + +func NewGateStorage(backend string) *GateStorage { + return &GateStorage{ + backend: backend, + data: new(sync.Map), + } +} + +func (gs *GateStorage) open(key string) { + gs.data.Store(key, true) +} + +func (gs *GateStorage) close(key string) { + gs.data.Store(key, false) +} + +func (gs *GateStorage) isOpen(key string) (locked bool) { + val, ok := gs.data.LoadOrStore(key, false) + if ok { + return val.(bool) + } + return +} diff --git a/pkg/loadtester/server.go b/pkg/loadtester/server.go index ca907756..3ddc7903 100644 --- a/pkg/loadtester/server.go +++ b/pkg/loadtester/server.go @@ -14,7 +14,7 @@ import ( ) // ListenAndServe starts a web server and waits for SIGTERM -func ListenAndServe(port string, timeout time.Duration, logger *zap.SugaredLogger, taskRunner *TaskRunner, stopCh <-chan struct{}) { +func ListenAndServe(port string, timeout time.Duration, logger *zap.SugaredLogger, taskRunner *TaskRunner, gate *GateStorage, stopCh <-chan struct{}) { mux := http.DefaultServeMux mux.Handle("/metrics", promhttp.Handler()) mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { @@ -29,6 +29,86 @@ func ListenAndServe(port string, timeout time.Duration, logger *zap.SugaredLogge w.WriteHeader(http.StatusForbidden) w.Write([]byte("Forbidden")) }) + mux.HandleFunc("/gate/check", func(w http.ResponseWriter, r *http.Request) { + body, err := ioutil.ReadAll(r.Body) + if err != nil { + logger.Error("reading the request body failed", zap.Error(err)) + w.WriteHeader(http.StatusBadRequest) + return + } + defer r.Body.Close() + + canary := &flaggerv1.CanaryWebhookPayload{} + err = json.Unmarshal(body, canary) + if err != nil { + logger.Error("decoding the request body failed", zap.Error(err)) + w.WriteHeader(http.StatusBadRequest) + return + } + + canaryName := fmt.Sprintf("%s.%s", canary.Name, canary.Namespace) + approved := gate.isOpen(canaryName) + if approved { + w.WriteHeader(http.StatusOK) + w.Write([]byte("Approved")) + } else { + w.WriteHeader(http.StatusForbidden) + w.Write([]byte("Forbidden")) + } + + logger.Infof("%s gate check: approved %v", canaryName, approved) + }) + + mux.HandleFunc("/gate/open", func(w http.ResponseWriter, r *http.Request) { + body, err := ioutil.ReadAll(r.Body) + if err != nil { + logger.Error("reading the request body failed", zap.Error(err)) + w.WriteHeader(http.StatusBadRequest) + return + } + defer r.Body.Close() + + canary := &flaggerv1.CanaryWebhookPayload{} + err = json.Unmarshal(body, canary) + if err != nil { + logger.Error("decoding the request body failed", zap.Error(err)) + w.WriteHeader(http.StatusBadRequest) + return + } + + canaryName := fmt.Sprintf("%s.%s", canary.Name, canary.Namespace) + gate.open(canaryName) + + w.WriteHeader(http.StatusAccepted) + + logger.Infof("%s gate opened", canaryName) + }) + + mux.HandleFunc("/gate/close", func(w http.ResponseWriter, r *http.Request) { + body, err := ioutil.ReadAll(r.Body) + if err != nil { + logger.Error("reading the request body failed", zap.Error(err)) + w.WriteHeader(http.StatusBadRequest) + return + } + defer r.Body.Close() + + canary := &flaggerv1.CanaryWebhookPayload{} + err = json.Unmarshal(body, canary) + if err != nil { + logger.Error("decoding the request body failed", zap.Error(err)) + w.WriteHeader(http.StatusBadRequest) + return + } + + canaryName := fmt.Sprintf("%s.%s", canary.Name, canary.Namespace) + gate.close(canaryName) + + w.WriteHeader(http.StatusAccepted) + + logger.Infof("%s gate closed", canaryName) + }) + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { body, err := ioutil.ReadAll(r.Body) if err != nil { From be22ff9951df8940fa23b934fd1b905a9d02d497 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 24 Jul 2019 16:27:30 +0300 Subject: [PATCH 08/15] Bump load tester version --- artifacts/loadtester/deployment.yaml | 2 +- charts/loadtester/Chart.yaml | 2 +- charts/loadtester/values.yaml | 2 +- cmd/loadtester/main.go | 2 +- kustomize/tester/deployment.yaml | 2 +- kustomize/tester/kustomization.yaml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/artifacts/loadtester/deployment.yaml b/artifacts/loadtester/deployment.yaml index 33615515..8e7a117f 100644 --- a/artifacts/loadtester/deployment.yaml +++ b/artifacts/loadtester/deployment.yaml @@ -17,7 +17,7 @@ spec: spec: containers: - name: loadtester - image: weaveworks/flagger-loadtester:0.6.0 + image: weaveworks/flagger-loadtester:0.6.1 imagePullPolicy: IfNotPresent ports: - name: http diff --git a/charts/loadtester/Chart.yaml b/charts/loadtester/Chart.yaml index 3e651a04..745d4b13 100644 --- a/charts/loadtester/Chart.yaml +++ b/charts/loadtester/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 name: loadtester version: 0.6.0 -appVersion: 0.6.0 +appVersion: 0.6.1 kubeVersion: ">=1.11.0-0" engine: gotpl description: Flagger's load testing services based on rakyll/hey and bojand/ghz that generates traffic during canary analysis when configured as a webhook. diff --git a/charts/loadtester/values.yaml b/charts/loadtester/values.yaml index 3a03a5cc..36ff9bf1 100644 --- a/charts/loadtester/values.yaml +++ b/charts/loadtester/values.yaml @@ -2,7 +2,7 @@ replicaCount: 1 image: repository: weaveworks/flagger-loadtester - tag: 0.6.0 + tag: 0.6.1 pullPolicy: IfNotPresent logLevel: info diff --git a/cmd/loadtester/main.go b/cmd/loadtester/main.go index 74c4d09b..7f790767 100644 --- a/cmd/loadtester/main.go +++ b/cmd/loadtester/main.go @@ -10,7 +10,7 @@ import ( "time" ) -var VERSION = "0.6.0" +var VERSION = "0.6.1" var ( logLevel string port string diff --git a/kustomize/tester/deployment.yaml b/kustomize/tester/deployment.yaml index 33615515..8e7a117f 100644 --- a/kustomize/tester/deployment.yaml +++ b/kustomize/tester/deployment.yaml @@ -17,7 +17,7 @@ spec: spec: containers: - name: loadtester - image: weaveworks/flagger-loadtester:0.6.0 + image: weaveworks/flagger-loadtester:0.6.1 imagePullPolicy: IfNotPresent ports: - name: http diff --git a/kustomize/tester/kustomization.yaml b/kustomize/tester/kustomization.yaml index 1c928795..2e66439d 100644 --- a/kustomize/tester/kustomization.yaml +++ b/kustomize/tester/kustomization.yaml @@ -4,4 +4,4 @@ resources: - deployment.yaml images: - name: weaveworks/flagger-loadtester - newTag: 0.5.0 + newTag: 0.6.1 From bc4bdcdc1c5f7c232e32002cf059349399af8f98 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 24 Jul 2019 19:21:41 +0300 Subject: [PATCH 09/15] Upgrade Gloo e2e to v0.17.6 --- test/e2e-gloo.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e-gloo.sh b/test/e2e-gloo.sh index c3d17227..05d5d043 100755 --- a/test/e2e-gloo.sh +++ b/test/e2e-gloo.sh @@ -2,7 +2,7 @@ set -o errexit -GLOO_VER="0.14.2" +GLOO_VER="0.17.6" REPO_ROOT=$(git rev-parse --show-toplevel) export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" From c4d28e14fc55e98b9d0addce8bc6ebfbbd911ec3 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 24 Jul 2019 19:35:02 +0300 Subject: [PATCH 10/15] Upgrade Gloo e2e to v0.17.5 --- test/e2e-gloo.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e-gloo.sh b/test/e2e-gloo.sh index 05d5d043..d821ef93 100755 --- a/test/e2e-gloo.sh +++ b/test/e2e-gloo.sh @@ -2,7 +2,7 @@ set -o errexit -GLOO_VER="0.17.6" +GLOO_VER="0.17.5" REPO_ROOT=$(git rev-parse --show-toplevel) export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" From aa6a180bcc4b403f4ab86b6a54e1e74604a76db8 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 24 Jul 2019 19:44:06 +0300 Subject: [PATCH 11/15] Remove Gloo NodePort from e2e tests --- test/e2e-gloo.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/e2e-gloo.sh b/test/e2e-gloo.sh index d821ef93..3f86b2bf 100755 --- a/test/e2e-gloo.sh +++ b/test/e2e-gloo.sh @@ -9,9 +9,7 @@ export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" echo '>>> Installing Gloo' helm repo add gloo https://storage.googleapis.com/solo-public-helm helm upgrade -i gloo gloo/gloo --version ${GLOO_VER} \ ---wait \ ---namespace gloo-system \ ---set gatewayProxies.gateway-proxy.service.type=NodePort +--namespace gloo-system kubectl -n gloo-system rollout status deployment/gloo kubectl -n gloo-system rollout status deployment/gateway-proxy From 046245a8b5bb8df4d638996f531a6f38b26fa17b Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 24 Jul 2019 19:54:33 +0300 Subject: [PATCH 12/15] Use Gloo 0.17.6 in e2e tests --- test/e2e-gloo-tests.sh | 2 +- test/e2e-gloo.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e-gloo-tests.sh b/test/e2e-gloo-tests.sh index 6f948367..891d9c53 100755 --- a/test/e2e-gloo-tests.sh +++ b/test/e2e-gloo-tests.sh @@ -55,7 +55,7 @@ spec: canaryAnalysis: interval: 15s threshold: 15 - maxWeight: 30 + maxWeight: 50 stepWeight: 10 metrics: - name: request-success-rate diff --git a/test/e2e-gloo.sh b/test/e2e-gloo.sh index 3f86b2bf..19fcf9fc 100755 --- a/test/e2e-gloo.sh +++ b/test/e2e-gloo.sh @@ -2,7 +2,7 @@ set -o errexit -GLOO_VER="0.17.5" +GLOO_VER="0.17.6" REPO_ROOT=$(git rev-parse --show-toplevel) export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" From e07a82d02476af4580b48564e30535ef9d5981de Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Thu, 25 Jul 2019 13:29:33 +0300 Subject: [PATCH 13/15] Add manual gating to docs --- docs/gitbook/how-it-works.md | 52 ++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/docs/gitbook/how-it-works.md b/docs/gitbook/how-it-works.md index afd2ffab..bec6f796 100644 --- a/docs/gitbook/how-it-works.md +++ b/docs/gitbook/how-it-works.md @@ -884,3 +884,55 @@ As an alternative to Helm you can use the [Bash Automated Testing System](https: ``` Note that you should create a ConfigMap with your Bats tests and mount it inside the tester container. + +### Manual Gating + +For manual approval of a canary deployment you can use the `confirm-rollout` webhook. +The confirmation hooks are executed before the pre-rollout hooks. +Flagger will halt the canary traffic shifting and analysis until the confirm webhook returns HTTP status 200. + +Manual gating with Flagger's tester: + +```yaml + canaryAnalysis: + webhooks: + - name: "gate" + type: confirm-rollout + url: http://flagger-loadtester.test/gate/halt +``` + +The `/gate/halt` returns HTTP 403 thus blocking the rollout. + +Change the URL to `/gate/approve` to start the canary analysis: + +```yaml + canaryAnalysis: + webhooks: + - name: "start gate" + type: confirm-rollout + url: http://flagger-loadtester.test/gate/approve +``` + +Manual gating can be driven with Flagger's tester API. Set the confirmation URL to `/gate/check`: + +```yaml + canaryAnalysis: + webhooks: + - name: "ask confirmation" + type: confirm-rollout + url: http://flagger-loadtester.test/gate/check +``` + +By default the gate is closed, you can start or resume the canary rollout with: + +```bash +kubectl -n test exec -it flagger-loadtester-xxxx-xxxx sh + +curl -d '{"name": "podinfo","namespace":"test"}' http://localhost:8080/gate/open +``` + +You can pause the rollout at any time with: + +```bash +curl -d '{"name": "podinfo","namespace":"test"}' http://localhost:8080/gate/close +``` From 163f5292b0a44b860f888b1a075e776fb00fa404 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Thu, 25 Jul 2019 19:10:46 +0300 Subject: [PATCH 14/15] Push a notification when a canary is waiting for approval --- docs/gitbook/how-it-works.md | 3 ++- pkg/controller/scheduler.go | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/gitbook/how-it-works.md b/docs/gitbook/how-it-works.md index bec6f796..483b6226 100644 --- a/docs/gitbook/how-it-works.md +++ b/docs/gitbook/how-it-works.md @@ -901,7 +901,8 @@ Manual gating with Flagger's tester: url: http://flagger-loadtester.test/gate/halt ``` -The `/gate/halt` returns HTTP 403 thus blocking the rollout. +The `/gate/halt` returns HTTP 403 thus blocking the rollout. +If you have notifications enabled, Flagger will post a message to Slack or MS Teams if a canary rollout is waiting for approval. Change the URL to `/gate/approve` to start the canary analysis: diff --git a/pkg/controller/scheduler.go b/pkg/controller/scheduler.go index bca27600..cf9a595b 100644 --- a/pkg/controller/scheduler.go +++ b/pkg/controller/scheduler.go @@ -537,11 +537,12 @@ func (c *Controller) runConfirmRolloutHooks(canary *flaggerv1.Canary) bool { err := CallWebhook(canary.Name, canary.Namespace, flaggerv1.CanaryPhaseProgressing, webhook) if err != nil { if canary.Status.Phase != flaggerv1.CanaryPhaseWaiting { - c.recordEventWarningf(canary, "Halt %s.%s advancement waiting for approval %s", - canary.Name, canary.Namespace, webhook.Name) if err := c.deployer.SetStatusPhase(canary, flaggerv1.CanaryPhaseWaiting); err != nil { c.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)).Errorf("%v", err) } + c.recordEventWarningf(canary, "Halt %s.%s advancement waiting for approval %s", + canary.Name, canary.Namespace, webhook.Name) + c.sendNotification(canary, "Canary is waiting for approval.", false, false) } return false } else { From e3b03debdee109158e945b20b351de75174723bb Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Fri, 26 Jul 2019 01:19:15 +0300 Subject: [PATCH 15/15] Use podinfo v1.7 --- artifacts/ab-testing/deployment.yaml | 2 +- artifacts/appmesh/deployment.yaml | 2 +- artifacts/canaries/deployment.yaml | 2 +- artifacts/gloo/deployment.yaml | 2 +- artifacts/nginx/deployment.yaml | 2 +- artifacts/workloads/canary-deployment.yaml | 2 +- artifacts/workloads/primary-deployment.yaml | 2 +- charts/podinfo/Chart.yaml | 4 ++-- charts/podinfo/values.yaml | 2 +- docs/gitbook/how-it-works.md | 10 ++++++++++ docs/gitbook/tutorials/canary-helm-gitops.md | 2 +- docs/gitbook/tutorials/flagger-smi-istio.md | 2 +- docs/gitbook/usage/ab-testing.md | 2 +- docs/gitbook/usage/appmesh-progressive-delivery.md | 2 +- docs/gitbook/usage/blue-green.md | 8 ++++---- docs/gitbook/usage/gloo-progressive-delivery.md | 2 +- docs/gitbook/usage/linkerd-progressive-delivery.md | 2 +- docs/gitbook/usage/nginx-progressive-delivery.md | 2 +- docs/gitbook/usage/progressive-delivery.md | 2 +- 19 files changed, 32 insertions(+), 22 deletions(-) diff --git a/artifacts/ab-testing/deployment.yaml b/artifacts/ab-testing/deployment.yaml index 373eb3bf..66c0174e 100644 --- a/artifacts/ab-testing/deployment.yaml +++ b/artifacts/ab-testing/deployment.yaml @@ -25,7 +25,7 @@ spec: spec: containers: - name: podinfod - image: quay.io/stefanprodan/podinfo:1.4.0 + image: quay.io/stefanprodan/podinfo:1.7.0 imagePullPolicy: IfNotPresent ports: - containerPort: 9898 diff --git a/artifacts/appmesh/deployment.yaml b/artifacts/appmesh/deployment.yaml index beeae326..6166cfaf 100644 --- a/artifacts/appmesh/deployment.yaml +++ b/artifacts/appmesh/deployment.yaml @@ -25,7 +25,7 @@ spec: spec: containers: - name: podinfod - image: quay.io/stefanprodan/podinfo:1.4.0 + image: quay.io/stefanprodan/podinfo:1.7.0 imagePullPolicy: IfNotPresent ports: - containerPort: 9898 diff --git a/artifacts/canaries/deployment.yaml b/artifacts/canaries/deployment.yaml index 57ed8a41..dacb34be 100644 --- a/artifacts/canaries/deployment.yaml +++ b/artifacts/canaries/deployment.yaml @@ -25,7 +25,7 @@ spec: spec: containers: - name: podinfod - image: quay.io/stefanprodan/podinfo:1.4.0 + image: quay.io/stefanprodan/podinfo:1.7.0 imagePullPolicy: IfNotPresent ports: - containerPort: 9898 diff --git a/artifacts/gloo/deployment.yaml b/artifacts/gloo/deployment.yaml index 57ed8a41..dacb34be 100644 --- a/artifacts/gloo/deployment.yaml +++ b/artifacts/gloo/deployment.yaml @@ -25,7 +25,7 @@ spec: spec: containers: - name: podinfod - image: quay.io/stefanprodan/podinfo:1.4.0 + image: quay.io/stefanprodan/podinfo:1.7.0 imagePullPolicy: IfNotPresent ports: - containerPort: 9898 diff --git a/artifacts/nginx/deployment.yaml b/artifacts/nginx/deployment.yaml index 814dd9c2..4baf9424 100644 --- a/artifacts/nginx/deployment.yaml +++ b/artifacts/nginx/deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: podinfod - image: quay.io/stefanprodan/podinfo:1.4.0 + image: quay.io/stefanprodan/podinfo:1.7.0 imagePullPolicy: IfNotPresent ports: - containerPort: 9898 diff --git a/artifacts/workloads/canary-deployment.yaml b/artifacts/workloads/canary-deployment.yaml index 814dd9c2..4baf9424 100644 --- a/artifacts/workloads/canary-deployment.yaml +++ b/artifacts/workloads/canary-deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: podinfod - image: quay.io/stefanprodan/podinfo:1.4.0 + image: quay.io/stefanprodan/podinfo:1.7.0 imagePullPolicy: IfNotPresent ports: - containerPort: 9898 diff --git a/artifacts/workloads/primary-deployment.yaml b/artifacts/workloads/primary-deployment.yaml index cc17b82f..6bc393a7 100644 --- a/artifacts/workloads/primary-deployment.yaml +++ b/artifacts/workloads/primary-deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: podinfod - image: quay.io/stefanprodan/podinfo:1.4.1 + image: quay.io/stefanprodan/podinfo:1.7.1 imagePullPolicy: IfNotPresent ports: - containerPort: 9898 diff --git a/charts/podinfo/Chart.yaml b/charts/podinfo/Chart.yaml index 075936f3..0c8d63db 100644 --- a/charts/podinfo/Chart.yaml +++ b/charts/podinfo/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 -version: 2.2.0 -appVersion: 1.4.0 +version: 2.3.0 +appVersion: 1.7.0 name: podinfo engine: gotpl description: Flagger canary deployment demo chart diff --git a/charts/podinfo/values.yaml b/charts/podinfo/values.yaml index deff5c3d..95eb9f3e 100644 --- a/charts/podinfo/values.yaml +++ b/charts/podinfo/values.yaml @@ -1,7 +1,7 @@ # Default values for podinfo. image: repository: quay.io/stefanprodan/podinfo - tag: 1.4.0 + tag: 1.7.0 pullPolicy: IfNotPresent service: diff --git a/docs/gitbook/how-it-works.md b/docs/gitbook/how-it-works.md index 483b6226..829a2c41 100644 --- a/docs/gitbook/how-it-works.md +++ b/docs/gitbook/how-it-works.md @@ -902,6 +902,7 @@ Manual gating with Flagger's tester: ``` The `/gate/halt` returns HTTP 403 thus blocking the rollout. + If you have notifications enabled, Flagger will post a message to Slack or MS Teams if a canary rollout is waiting for approval. Change the URL to `/gate/approve` to start the canary analysis: @@ -937,3 +938,12 @@ You can pause the rollout at any time with: ```bash curl -d '{"name": "podinfo","namespace":"test"}' http://localhost:8080/gate/close ``` + +If a canary analysis is paused the status will change to waiting: + +```bash +kubectl get canary/podinfo + +NAME STATUS WEIGHT +podinfo Waiting 0 +``` diff --git a/docs/gitbook/tutorials/canary-helm-gitops.md b/docs/gitbook/tutorials/canary-helm-gitops.md index ca7f7f93..9633c8bf 100644 --- a/docs/gitbook/tutorials/canary-helm-gitops.md +++ b/docs/gitbook/tutorials/canary-helm-gitops.md @@ -344,7 +344,7 @@ launch the `frontend` and `backend` apps. A CI/CD pipeline for the `frontend` release could look like this: * cut a release from the master branch of the podinfo code repo with the git tag `1.4.1` -* CI builds the image and pushes the `podinfo:1.4.1` image to the container registry +* CI builds the image and pushes the `podinfo:1.7.1` image to the container registry * Flux scans the registry and updates the Helm release `image.tag` to `1.4.1` * Flux commits and push the change to the cluster repo * Flux applies the updated Helm release on the cluster diff --git a/docs/gitbook/tutorials/flagger-smi-istio.md b/docs/gitbook/tutorials/flagger-smi-istio.md index 6b622e58..3fae5dda 100644 --- a/docs/gitbook/tutorials/flagger-smi-istio.md +++ b/docs/gitbook/tutorials/flagger-smi-istio.md @@ -236,7 +236,7 @@ Trigger a canary deployment by updating the container image: ```bash kubectl -n test set image deployment/podinfo \ -podinfod=quay.io/stefanprodan/podinfo:1.4.1 +podinfod=quay.io/stefanprodan/podinfo:1.7.1 ``` Flagger detects that the deployment revision changed and starts a new rollout: diff --git a/docs/gitbook/usage/ab-testing.md b/docs/gitbook/usage/ab-testing.md index aa5a9372..1a99f94b 100644 --- a/docs/gitbook/usage/ab-testing.md +++ b/docs/gitbook/usage/ab-testing.md @@ -131,7 +131,7 @@ Trigger a canary deployment by updating the container image: ```bash kubectl -n test set image deployment/abtest \ -podinfod=quay.io/stefanprodan/podinfo:1.4.1 +podinfod=quay.io/stefanprodan/podinfo:1.7.1 ``` Flagger detects that the deployment revision changed and starts a new rollout: diff --git a/docs/gitbook/usage/appmesh-progressive-delivery.md b/docs/gitbook/usage/appmesh-progressive-delivery.md index 02fe3c29..8a069f3b 100644 --- a/docs/gitbook/usage/appmesh-progressive-delivery.md +++ b/docs/gitbook/usage/appmesh-progressive-delivery.md @@ -176,7 +176,7 @@ Trigger a canary deployment by updating the container image: ```bash kubectl -n test set image deployment/podinfo \ -podinfod=quay.io/stefanprodan/podinfo:1.4.1 +podinfod=quay.io/stefanprodan/podinfo:1.7.1 ``` Flagger detects that the deployment revision changed and starts a new rollout: diff --git a/docs/gitbook/usage/blue-green.md b/docs/gitbook/usage/blue-green.md index 61ab6152..1f8b3ea1 100644 --- a/docs/gitbook/usage/blue-green.md +++ b/docs/gitbook/usage/blue-green.md @@ -59,8 +59,8 @@ Create a deployment and a horizontal pod autoscaler: ```bash export REPO=https://raw.githubusercontent.com/weaveworks/flagger/master -kubectl apply -f ${REPO}/artifacts/canary/deployment.yaml -kubectl apply -f ${REPO}/artifacts/canary/hpa.yaml +kubectl apply -f ${REPO}/artifacts/canaries/deployment.yaml +kubectl apply -f ${REPO}/artifacts/canaries/hpa.yaml ``` Deploy the load testing service to generate traffic during the analysis: @@ -172,7 +172,7 @@ Trigger a deployment by updating the container image: ```bash kubectl -n test set image deployment/podinfo \ -podinfod=quay.io/stefanprodan/podinfo:1.4.1 +podinfod=quay.io/stefanprodan/podinfo:1.7.1 ``` Flagger detects that the deployment revision changed and starts a new rollout: @@ -297,7 +297,7 @@ Trigger a deployment by updating the container image: ```bash kubectl -n test set image deployment/podinfo \ -podinfod=quay.io/stefanprodan/podinfo:1.4.3 +podinfod=quay.io/stefanprodan/podinfo:1.7.3 ``` Generate 404s: diff --git a/docs/gitbook/usage/gloo-progressive-delivery.md b/docs/gitbook/usage/gloo-progressive-delivery.md index bff29cee..81359b2f 100644 --- a/docs/gitbook/usage/gloo-progressive-delivery.md +++ b/docs/gitbook/usage/gloo-progressive-delivery.md @@ -197,7 +197,7 @@ Trigger a canary deployment by updating the container image: ```bash kubectl -n test set image deployment/podinfo \ -podinfod=quay.io/stefanprodan/podinfo:1.4.1 +podinfod=quay.io/stefanprodan/podinfo:1.7.1 ``` Flagger detects that the deployment revision changed and starts a new rollout: diff --git a/docs/gitbook/usage/linkerd-progressive-delivery.md b/docs/gitbook/usage/linkerd-progressive-delivery.md index 40da3627..3b9f64c9 100644 --- a/docs/gitbook/usage/linkerd-progressive-delivery.md +++ b/docs/gitbook/usage/linkerd-progressive-delivery.md @@ -165,7 +165,7 @@ Trigger a canary deployment by updating the container image: ```bash kubectl -n test set image deployment/podinfo \ -podinfod=quay.io/stefanprodan/podinfo:1.4.1 +podinfod=quay.io/stefanprodan/podinfo:1.7.1 ``` Flagger detects that the deployment revision changed and starts a new rollout: diff --git a/docs/gitbook/usage/nginx-progressive-delivery.md b/docs/gitbook/usage/nginx-progressive-delivery.md index 4d660bc4..6261c351 100644 --- a/docs/gitbook/usage/nginx-progressive-delivery.md +++ b/docs/gitbook/usage/nginx-progressive-delivery.md @@ -189,7 +189,7 @@ Trigger a canary deployment by updating the container image: ```bash kubectl -n test set image deployment/podinfo \ -podinfod=quay.io/stefanprodan/podinfo:1.4.1 +podinfod=quay.io/stefanprodan/podinfo:1.7.1 ``` Flagger detects that the deployment revision changed and starts a new rollout: diff --git a/docs/gitbook/usage/progressive-delivery.md b/docs/gitbook/usage/progressive-delivery.md index db2d925f..939e0940 100644 --- a/docs/gitbook/usage/progressive-delivery.md +++ b/docs/gitbook/usage/progressive-delivery.md @@ -119,7 +119,7 @@ Trigger a canary deployment by updating the container image: ```bash kubectl -n test set image deployment/podinfo \ -podinfod=quay.io/stefanprodan/podinfo:1.4.1 +podinfod=quay.io/stefanprodan/podinfo:1.7.1 ``` Flagger detects that the deployment revision changed and starts a new rollout: