From 9b79af9fcdb86d27f23bd51f694b6da38d3f156e Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 13 Feb 2019 15:27:45 +0200 Subject: [PATCH 1/6] Add skipAnalysis field to Canary CRD --- artifacts/flagger/crd.yaml | 2 ++ charts/flagger/templates/crd.yaml | 2 ++ pkg/apis/flagger/v1alpha3/types.go | 4 ++++ 3 files changed, 8 insertions(+) diff --git a/artifacts/flagger/crd.yaml b/artifacts/flagger/crd.yaml index e674913c..a38c927d 100644 --- a/artifacts/flagger/crd.yaml +++ b/artifacts/flagger/crd.yaml @@ -73,6 +73,8 @@ spec: properties: port: type: number + skipAnalysis: + type: boolean canaryAnalysis: properties: interval: diff --git a/charts/flagger/templates/crd.yaml b/charts/flagger/templates/crd.yaml index 22e80bc9..4d9527f7 100644 --- a/charts/flagger/templates/crd.yaml +++ b/charts/flagger/templates/crd.yaml @@ -74,6 +74,8 @@ spec: properties: port: type: number + skipAnalysis: + type: boolean canaryAnalysis: properties: interval: diff --git a/pkg/apis/flagger/v1alpha3/types.go b/pkg/apis/flagger/v1alpha3/types.go index 81cf3221..8c0bdd51 100755 --- a/pkg/apis/flagger/v1alpha3/types.go +++ b/pkg/apis/flagger/v1alpha3/types.go @@ -58,6 +58,10 @@ type CanarySpec struct { // the maximum time in seconds for a canary deployment to make progress // before it is considered to be failed. Defaults to ten minutes. ProgressDeadlineSeconds *int32 `json:"progressDeadlineSeconds,omitempty"` + + // promote the canary without analysing it + // +optional + SkipAnalysis bool `json:"skipAnalysis,omitempty"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object From 29cdd4328865e1a596a737167e1767091298c706 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 13 Feb 2019 15:30:29 +0200 Subject: [PATCH 2/6] Implement skip analysis When skip analysis is enabled, Flagger checks if the canary deployment is healthy and promotes it without analysing it. If an analysis is underway, Flagger cancels it and runs the promotion. --- pkg/controller/scheduler.go | 50 +++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/pkg/controller/scheduler.go b/pkg/controller/scheduler.go index 757af39e..d928fed1 100644 --- a/pkg/controller/scheduler.go +++ b/pkg/controller/scheduler.go @@ -5,6 +5,7 @@ import ( "strings" "time" + istiov1alpha3 "github.com/knative/pkg/apis/istio/v1alpha3" flaggerv1 "github.com/stefanprodan/flagger/pkg/apis/flagger/v1alpha3" "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -170,6 +171,11 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh } } + // check if analysis should be skipped + if skip := c.shouldSkipAnalysis(cd, primaryRoute, canaryRoute); skip { + return + } + // check if the number of failed checks reached the threshold if cd.Status.Phase == flaggerv1.CanaryProgressing && (!retriable || cd.Status.FailedChecks >= cd.Spec.CanaryAnalysis.Threshold) { @@ -294,6 +300,50 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh } } +func (c *Controller) shouldSkipAnalysis(cd *flaggerv1.Canary, primary istiov1alpha3.DestinationWeight, canary istiov1alpha3.DestinationWeight) bool { + if !cd.Spec.SkipAnalysis { + return false + } + + // route all traffic to primary + primary.Weight = 100 + canary.Weight = 0 + if err := c.router.SetRoutes(cd, primary, canary); err != nil { + c.recordEventWarningf(cd, "%v", err) + return false + } + c.recorder.SetWeight(cd, primary.Weight, canary.Weight) + + // copy spec and configs from canary to primary + c.recordEventInfof(cd, "Copying %s.%s template spec to %s-primary.%s", + cd.Spec.TargetRef.Name, cd.Namespace, cd.Spec.TargetRef.Name, cd.Namespace) + if err := c.deployer.Promote(cd); err != nil { + c.recordEventWarningf(cd, "%v", err) + return false + } + + // shutdown canary + if err := c.deployer.Scale(cd, 0); err != nil { + c.recordEventWarningf(cd, "%v", err) + return false + } + + // update status phase + if err := c.deployer.SetStatusPhase(cd, flaggerv1.CanarySucceeded); err != nil { + c.recordEventWarningf(cd, "%v", err) + return false + } + + // notify + c.recorder.SetStatus(cd) + c.recordEventInfof(cd, "Promotion completed! Canary analysis was skipped for %s.%s", + cd.Spec.TargetRef.Name, cd.Namespace) + c.sendNotification(cd, "Canary analysis was skipped, promotion finished.", + false, false) + + return true +} + func (c *Controller) checkCanaryStatus(cd *flaggerv1.Canary, shouldAdvance bool) bool { c.recorder.SetStatus(cd) if cd.Status.Phase == flaggerv1.CanaryProgressing { From 631c55fa6e31fdf7b0de5e67c5fe29f7a25b41f7 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 13 Feb 2019 15:31:01 +0200 Subject: [PATCH 3/6] Document how to skip the canary analysis --- README.md | 3 +++ artifacts/canaries/canary.yaml | 3 +++ docs/gitbook/how-it-works.md | 8 ++++++++ 3 files changed, 14 insertions(+) diff --git a/README.md b/README.md index 5751d5c8..85b4d275 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,9 @@ spec: # Istio virtual service host names (optional) hosts: - podinfo.example.com + # for emergency cases when you want to ship changes + # in production without analysing the canary + skipAnalysis: false canaryAnalysis: # schedule interval (default 60s) interval: 1m diff --git a/artifacts/canaries/canary.yaml b/artifacts/canaries/canary.yaml index 70c86c07..cb1cdfbf 100644 --- a/artifacts/canaries/canary.yaml +++ b/artifacts/canaries/canary.yaml @@ -26,6 +26,9 @@ spec: # Istio virtual service host names (optional) hosts: - app.istio.weavedx.com + # for emergency cases when you want to ship changes + # in production without analysing the canary + skipAnalysis: false canaryAnalysis: # schedule interval (default 60s) interval: 10s diff --git a/docs/gitbook/how-it-works.md b/docs/gitbook/how-it-works.md index 97faec2b..edf35f08 100644 --- a/docs/gitbook/how-it-works.md +++ b/docs/gitbook/how-it-works.md @@ -39,6 +39,9 @@ spec: # Istio virtual service host names (optional) hosts: - podinfo.example.com + # for emergency cases when you want to ship changes + # in production without analysing the canary + skipAnalysis: false canaryAnalysis: # schedule interval (default 60s) interval: 1m @@ -167,6 +170,11 @@ And the time it takes for a canary to be rollback when the metrics or webhook ch interval * threshold ``` +In emergency cases, you may want to skip the analysis phase and ship changes directly to production. +At any time you can set the `spec.skipAnalysis: true`. +When skip analysis is enabled, Flagger checks if the canary deployment is healthy and +promotes it without analysing it. If an analysis is underway, Flagger cancels it and runs the promotion. + ### HTTP Metrics The canary analysis is using the following Prometheus queries: From df1d9e30119d6dd5fe4037af1a88435808da586f Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 13 Feb 2019 15:56:40 +0200 Subject: [PATCH 4/6] Add skip analysis test --- pkg/controller/scheduler_test.go | 41 ++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/pkg/controller/scheduler_test.go b/pkg/controller/scheduler_test.go index 900ddebf..0ab633d0 100644 --- a/pkg/controller/scheduler_test.go +++ b/pkg/controller/scheduler_test.go @@ -64,6 +64,47 @@ func TestScheduler_Rollback(t *testing.T) { } } +func TestScheduler_SkipAnalysis(t *testing.T) { + mocks := SetupMocks() + // init + mocks.ctrl.advanceCanary("podinfo", "default", false) + + // enable skip + cd, err := mocks.flaggerClient.FlaggerV1alpha3().Canaries("default").Get("podinfo", metav1.GetOptions{}) + if err != nil { + t.Fatal(err.Error()) + } + cd.Spec.SkipAnalysis = true + _, err = mocks.flaggerClient.FlaggerV1alpha3().Canaries("default").Update(cd) + if err != nil { + t.Fatal(err.Error()) + } + + // update + dep2 := newTestDeploymentV2() + _, err = mocks.kubeClient.AppsV1().Deployments("default").Update(dep2) + if err != nil { + t.Fatal(err.Error()) + } + + // detect changes + mocks.ctrl.advanceCanary("podinfo", "default", true) + // advance + mocks.ctrl.advanceCanary("podinfo", "default", true) + + c, err := mocks.flaggerClient.FlaggerV1alpha3().Canaries("default").Get("podinfo", metav1.GetOptions{}) + if err != nil { + t.Fatal(err.Error()) + } + if !c.Spec.SkipAnalysis { + t.Errorf("Got skip analysis %v wanted %v", c.Spec.SkipAnalysis, true) + } + + if c.Status.Phase != v1alpha3.CanarySucceeded { + t.Errorf("Got canary state %v wanted %v", c.Status.Phase, v1alpha3.CanarySucceeded) + } +} + func TestScheduler_NewRevisionReset(t *testing.T) { mocks := SetupMocks() // init From d3953004f60e7f3ee857e8fec685b3981968f826 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 13 Feb 2019 16:39:48 +0200 Subject: [PATCH 5/6] Add docs links and trim down the readme --- README.md | 375 +++++------------------------------------------------- 1 file changed, 33 insertions(+), 342 deletions(-) diff --git a/README.md b/README.md index 85b4d275..74b7ad1e 100644 --- a/README.md +++ b/README.md @@ -8,9 +8,36 @@ Flagger is a Kubernetes operator that automates the promotion of canary deployments using Istio routing for traffic shifting and Prometheus metrics for canary analysis. -The canary analysis can be extended with webhooks for running integration tests, +The canary analysis can be extended with webhooks for running acceptance tests, load tests or any other custom validation. +Flagger implements a control loop that gradually shifts traffic to the canary while measuring key performance +indicators like HTTP requests success rate, requests average duration and pods health. +Based on analysis of the KPIs a canary is promoted or aborted, and the analysis result is published to Slack. + +![flagger-overview](https://raw.githubusercontent.com/stefanprodan/flagger/master/docs/diagrams/flagger-canary-overview.png) + +### Documentation + +Flagger documentation can be found at [docs.flagger.app](https://docs.flagger.app) + +* Install + * [Flagger install on Kubernetes](https://docs.flagger.app/install/flagger-install-on-kubernetes) + * [Flagger install on GKE](https://docs.flagger.app/install/flagger-install-on-google-cloud) +* How it works + * [Canary custom resource](https://docs.flagger.app/how-it-works#canary-custom-resource) + * [Canary deployment stages](https://docs.flagger.app/how-it-works#canary-deployment) + * [Canary analysis](https://docs.flagger.app/how-it-works#canary-analysis) + * [HTTP metrics](https://docs.flagger.app/how-it-works#http-metrics) + * [Webhooks](https://docs.flagger.app/how-it-works#webhooks) + * [Load testing](https://docs.flagger.app/how-it-works#load-testing) +* Usage + * [Canary promotions and rollbacks](https://docs.flagger.app/usage/progressive-delivery) + * [Monitoring](https://docs.flagger.app/usage/monitoring) + * [Alerting](https://docs.flagger.app/usage/alerting) +* Tutorials + * [Canary deployments with Helm charts and Weave Flux](https://docs.flagger.app/tutorials/canary-helm-gitops) + ### Install Before installing Flagger make sure you have Istio setup up with Prometheus enabled. @@ -30,50 +57,15 @@ helm upgrade -i flagger flagger/flagger \ Flagger is compatible with Kubernetes >1.11.0 and Istio >1.0.0. -### Usage +### Canary CRD -Flagger takes a Kubernetes deployment and creates a series of objects -(Kubernetes [deployments](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/), -ClusterIP [services](https://kubernetes.io/docs/concepts/services-networking/service/) and -Istio [virtual services](https://istio.io/docs/reference/config/istio.networking.v1alpha3/#VirtualService)) -to drive the canary analysis and promotion. +Flagger takes a Kubernetes deployment and optionally a horizontal pod autoscaler (HPA), +then creates a series of objects (Kubernetes deployments, ClusterIP services and Istio virtual services). +These objects expose the application on the mesh and drive the canary analysis and promotion. Flagger keeps track of ConfigMaps and Secrets referenced by a Kubernetes Deployment and triggers a canary analysis if any of those objects change. When promoting a workload in production, both code (container images) and configuration (config maps and secrets) are being synchronised. -![flagger-overview](https://raw.githubusercontent.com/stefanprodan/flagger/master/docs/diagrams/flagger-canary-overview.png) - -Gated canary promotion stages: - -* scan for canary deployments -* check Istio virtual service routes are mapped to primary and canary ClusterIP services -* check primary and canary deployments status - * halt advancement if a rolling update is underway - * halt advancement if pods are unhealthy -* increase canary traffic weight percentage from 0% to 5% (step weight) -* call webhooks and check results -* check canary HTTP request success rate and latency - * halt advancement if any metric is under the specified threshold - * increment the failed checks counter -* check if the number of failed checks reached the threshold - * route all traffic to primary - * scale to zero the canary deployment and mark it as failed - * wait for the canary deployment to be updated and start over -* increase canary traffic weight by 5% (step weight) till it reaches 50% (max weight) - * halt advancement while canary request success rate is under the threshold - * halt advancement while canary request duration P99 is over the threshold - * halt advancement if the primary or canary deployment becomes unhealthy - * halt advancement while canary deployment is being scaled up/down by HPA -* promote canary to primary - * copy ConfigMaps and Secrets from canary to primary - * copy canary deployment spec template over primary -* wait for primary rolling update to finish - * halt advancement if pods are unhealthy -* route all traffic to primary -* scale to zero the canary deployment -* mark rollout as finished -* wait for the canary deployment to be updated and start over - For a deployment named _podinfo_, a canary promotion can be defined using Flagger's custom resource: ```yaml @@ -140,308 +132,7 @@ spec: cmd: "hey -z 1m -q 10 -c 2 http://podinfo.test:9898/" ``` -The canary analysis is using the following promql queries: - -_HTTP requests success rate percentage_ - -```sql -sum( - rate( - istio_requests_total{ - reporter="destination", - destination_workload_namespace=~"$namespace", - destination_workload=~"$workload", - response_code!~"5.*" - }[$interval] - ) -) -/ -sum( - rate( - istio_requests_total{ - reporter="destination", - destination_workload_namespace=~"$namespace", - destination_workload=~"$workload" - }[$interval] - ) -) -``` - -_HTTP requests milliseconds duration P99_ - -```sql -histogram_quantile(0.99, - sum( - irate( - istio_request_duration_seconds_bucket{ - reporter="destination", - destination_workload=~"$workload", - destination_workload_namespace=~"$namespace" - }[$interval] - ) - ) by (le) -) -``` - -The canary analysis can be extended with webhooks. -Flagger will call the webhooks (HTTP POST) and determine from the response status code (HTTP 2xx) if the canary is failing or not. - -Webhook payload: - -```json -{ - "name": "podinfo", - "namespace": "test", - "metadata": { - "test": "all", - "token": "16688eb5e9f289f1991c" - } -} -``` - -### Automated canary analysis, promotions and rollbacks - -Create a test namespace with Istio sidecar injection enabled: - -```bash -export REPO=https://raw.githubusercontent.com/stefanprodan/flagger/master - -kubectl apply -f ${REPO}/artifacts/namespaces/test.yaml -``` - -Create a deployment and a horizontal pod autoscaler: - -```bash -kubectl apply -f ${REPO}/artifacts/canaries/deployment.yaml -kubectl apply -f ${REPO}/artifacts/canaries/hpa.yaml -``` - -Deploy the load testing service to generate traffic during the canary analysis: - -```bash -kubectl -n test apply -f ${REPO}/artifacts/loadtester/deployment.yaml -kubectl -n test apply -f ${REPO}/artifacts/loadtester/service.yaml -``` - -Create a canary promotion custom resource (replace the Istio gateway and the internet domain with your own): - -```bash -kubectl apply -f ${REPO}/artifacts/canaries/canary.yaml -``` - -After a couple of seconds Flagger will create the canary objects: - -```bash -# applied -deployment.apps/podinfo -horizontalpodautoscaler.autoscaling/podinfo -canary.flagger.app/podinfo -# generated -deployment.apps/podinfo-primary -horizontalpodautoscaler.autoscaling/podinfo-primary -service/podinfo -service/podinfo-canary -service/podinfo-primary -virtualservice.networking.istio.io/podinfo -``` - -![flagger-canary-steps](https://raw.githubusercontent.com/stefanprodan/flagger/master/docs/diagrams/flagger-canary-steps.png) - -Trigger a canary deployment by updating the container image: - -```bash -kubectl -n test set image deployment/podinfo \ -podinfod=quay.io/stefanprodan/podinfo:1.4.0 -``` - -**Note** that Flagger tracks changes in the deployment `PodSpec` but also in `ConfigMaps` and `Secrets` -that are referenced in the pod's volumes and containers environment variables. - -Flagger detects that the deployment revision changed and starts a new canary analysis: - -``` -kubectl -n test describe canary/podinfo - -Status: - Canary Weight: 0 - Failed Checks: 0 - Last Transition Time: 2019-01-16T13:47:16Z - Phase: Succeeded -Events: - Type Reason Age From Message - ---- ------ ---- ---- ------- - Normal Synced 3m flagger New revision detected podinfo.test - Normal Synced 3m flagger Scaling up podinfo.test - Warning Synced 3m flagger Waiting for podinfo.test rollout to finish: 0 of 1 updated replicas are available - Normal Synced 3m flagger Advance podinfo.test canary weight 5 - Normal Synced 3m flagger Advance podinfo.test canary weight 10 - Normal Synced 3m flagger Advance podinfo.test canary weight 15 - Normal Synced 2m flagger Advance podinfo.test canary weight 20 - Normal Synced 2m flagger Advance podinfo.test canary weight 25 - Normal Synced 1m flagger Advance podinfo.test canary weight 30 - Normal Synced 1m flagger Advance podinfo.test canary weight 35 - Normal Synced 55s flagger Advance podinfo.test canary weight 40 - Normal Synced 45s flagger Advance podinfo.test canary weight 45 - Normal Synced 35s flagger Advance podinfo.test canary weight 50 - Normal Synced 25s flagger Copying podinfo.test template spec to podinfo-primary.test - Warning Synced 15s flagger Waiting for podinfo-primary.test rollout to finish: 1 of 2 updated replicas are available - Normal Synced 5s flagger Promotion completed! Scaling down podinfo.test -``` - -You can monitor all canaries with: - -```bash -watch kubectl get canaries --all-namespaces - -NAMESPACE NAME STATUS WEIGHT LASTTRANSITIONTIME -test podinfo Progressing 5 2019-01-16T14:05:07Z -``` - -During the canary analysis you can generate HTTP 500 errors and high latency to test if Flagger pauses the rollout. - -Create a tester pod and exec into it: - -```bash -kubectl -n test run tester --image=quay.io/stefanprodan/podinfo:1.2.1 -- ./podinfo --port=9898 -kubectl -n test exec -it tester-xx-xx sh -``` - -Generate HTTP 500 errors: - -```bash -watch curl http://podinfo-canary:9898/status/500 -``` - -Generate latency: - -```bash -watch curl http://podinfo-canary:9898/delay/1 -``` - -When the number of failed checks reaches the canary analysis threshold, the traffic is routed back to the primary, -the canary is scaled to zero and the rollout is marked as failed. - -``` -kubectl -n test describe canary/podinfo - -Status: - Canary Weight: 0 - Failed Checks: 10 - Last Transition Time: 2019-01-16T13:47:16Z - Phase: Failed -Events: - Type Reason Age From Message - ---- ------ ---- ---- ------- - Normal Synced 3m flagger Starting canary deployment for podinfo.test - Normal Synced 3m flagger Advance podinfo.test canary weight 5 - Normal Synced 3m flagger Advance podinfo.test canary weight 10 - Normal Synced 3m flagger Advance podinfo.test canary weight 15 - Normal Synced 3m flagger Halt podinfo.test advancement success rate 69.17% < 99% - Normal Synced 2m flagger Halt podinfo.test advancement success rate 61.39% < 99% - Normal Synced 2m flagger Halt podinfo.test advancement success rate 55.06% < 99% - Normal Synced 2m flagger Halt podinfo.test advancement success rate 47.00% < 99% - Normal Synced 2m flagger (combined from similar events): Halt podinfo.test advancement success rate 38.08% < 99% - Warning Synced 1m flagger Rolling back podinfo.test failed checks threshold reached 10 - Warning Synced 1m flagger Canary failed! Scaling down podinfo.test -``` - -**Note** that if you apply new changes to the deployment during the canary analysis, Flagger will restart the analysis. - -### Monitoring - -Flagger comes with a Grafana dashboard made for canary analysis. - -Install Grafana with Helm: - -```bash -helm upgrade -i flagger-grafana flagger/grafana \ ---namespace=istio-system \ ---set url=http://prometheus.istio-system:9090 -``` - -The dashboard shows the RED and USE metrics for the primary and canary workloads: - -![flagger-grafana](https://raw.githubusercontent.com/stefanprodan/flagger/master/docs/screens/grafana-canary-analysis.png) - -The canary errors and latency spikes have been recorded as Kubernetes events and logged by Flagger in json format: - -``` -kubectl -n istio-system logs deployment/flagger --tail=100 | jq .msg - -Starting canary deployment for podinfo.test -Advance podinfo.test canary weight 5 -Advance podinfo.test canary weight 10 -Advance podinfo.test canary weight 15 -Advance podinfo.test canary weight 20 -Advance podinfo.test canary weight 25 -Advance podinfo.test canary weight 30 -Advance podinfo.test canary weight 35 -Halt podinfo.test advancement success rate 98.69% < 99% -Advance podinfo.test canary weight 40 -Halt podinfo.test advancement request duration 1.515s > 500ms -Advance podinfo.test canary weight 45 -Advance podinfo.test canary weight 50 -Copying podinfo.test template spec to podinfo-primary.test -Halt podinfo-primary.test advancement waiting for rollout to finish: 1 old replicas are pending termination -Scaling down podinfo.test -Promotion completed! podinfo.test -``` - -Flagger exposes Prometheus metrics that can be used to determine the canary analysis status and the destination weight values: - -```bash -# Canaries total gauge -flagger_canary_total{namespace="test"} 1 - -# Canary promotion last known status gauge -# 0 - running, 1 - successful, 2 - failed -flagger_canary_status{name="podinfo" namespace="test"} 1 - -# Canary traffic weight gauge -flagger_canary_weight{workload="podinfo-primary" namespace="test"} 95 -flagger_canary_weight{workload="podinfo" namespace="test"} 5 - -# Seconds spent performing canary analysis histogram -flagger_canary_duration_seconds_bucket{name="podinfo",namespace="test",le="10"} 6 -flagger_canary_duration_seconds_bucket{name="podinfo",namespace="test",le="+Inf"} 6 -flagger_canary_duration_seconds_sum{name="podinfo",namespace="test"} 17.3561329 -flagger_canary_duration_seconds_count{name="podinfo",namespace="test"} 6 -``` - -### Alerting - -Flagger can be configured to send Slack notifications: - -```bash -helm upgrade -i flagger flagger/flagger \ ---namespace=istio-system \ ---set slack.url=https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK \ ---set slack.channel=general \ ---set slack.user=flagger -``` - -Once configured with a Slack incoming webhook, Flagger will post messages when a canary deployment has been initialized, -when a new revision has been detected and if the canary analysis failed or succeeded. - -![flagger-slack](https://raw.githubusercontent.com/stefanprodan/flagger/master/docs/screens/slack-canary-notifications.png) - -A canary deployment will be rolled back if the progress deadline exceeded or if the analysis -reached the maximum number of failed checks: - -![flagger-slack-errors](https://raw.githubusercontent.com/stefanprodan/flagger/master/docs/screens/slack-canary-failed.png) - -Besides Slack, you can use Alertmanager to trigger alerts when a canary deployment failed: - -```yaml - - alert: canary_rollback - expr: flagger_canary_status > 1 - for: 1m - labels: - severity: warning - annotations: - summary: "Canary failed" - description: "Workload {{ $labels.name }} namespace {{ $labels.namespace }}" -``` +For more details on how the canary analysis and promotion works please [read the docs](https://docs.flagger.app/how-it-works). ### Roadmap From e565789ae824b30c1c63cb3562e5e2327411f099 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 13 Feb 2019 18:18:37 +0200 Subject: [PATCH 6/6] Add link to Helm GitOps repo --- docs/gitbook/SUMMARY.md | 2 +- docs/gitbook/tutorials/canary-helm-gitops.md | 48 ++++++++++++++++++-- 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/docs/gitbook/SUMMARY.md b/docs/gitbook/SUMMARY.md index 62935d6a..0cd9b38b 100644 --- a/docs/gitbook/SUMMARY.md +++ b/docs/gitbook/SUMMARY.md @@ -16,4 +16,4 @@ ## Tutorials -* [Canary Deployments with Helm charts](tutorials/canary-helm-gitops.md) +* [Canaries with Helm charts and GitOps](tutorials/canary-helm-gitops.md) diff --git a/docs/gitbook/tutorials/canary-helm-gitops.md b/docs/gitbook/tutorials/canary-helm-gitops.md index bb19464b..aef6a23f 100644 --- a/docs/gitbook/tutorials/canary-helm-gitops.md +++ b/docs/gitbook/tutorials/canary-helm-gitops.md @@ -1,4 +1,4 @@ -# Canary Deployments with Helm charts +# Canary Deployments with Helm Charts and GitOps This guide shows you how to package a web app into a Helm chart, trigger canary deployments on Helm upgrade and automate the chart release process with Weave Flux. @@ -230,10 +230,20 @@ If you've enabled the Slack notifications, you'll receive an alert with the reas ### GitOps automation -Instead of using Helm CLI from a CI tool to perform the install and upgrade, you could use a Git based approach. +Instead of using Helm CLI from a CI tool to perform the install and upgrade, +you could use a Git based approach. GitOps is a way to do Continuous Delivery, +it works by using Git as a source of truth for declarative infrastructure and workloads. +In the [GitOps model](https://www.weave.works/technologies/gitops/), +any change to production must be committed in source control +prior to being applied on the cluster. This way rollback and audit logs are provided by Git. ![Helm GitOps Canary Deployment](https://raw.githubusercontent.com/stefanprodan/flagger/master/docs/diagrams/flagger-flux-gitops.png) +In order to apply the GitOps pipeline model to Flagger canary deployments you'll need +a Git repository with your workloads definitions in YAML format, +a container registry where your CI system pushes immutable images and +an operator that synchronizes the Git repo with the cluster state. + Create a git repository with the following content: ``` @@ -287,7 +297,34 @@ With the `flux.weave.works` annotations I instruct Flux to automate this release When an image tag in the sem ver range of `1.4.0 - 1.4.99` is pushed to Quay, Flux will upgrade the Helm release and from there Flagger will pick up the change and start a canary deployment. -A CI/CD pipeline for the frontend release could look like this: +Install [Weave Flux](https://github.com/weaveworks/flux) and its Helm Operator by specifying your Git repo URL: + +```bash +helm repo add weaveworks https://weaveworks.github.io/flux + +helm install --name flux \ +--set helmOperator.create=true \ +--set git.url=git@github.com:/ \ +--namespace flux \ +weaveworks/flux +``` + +At startup Flux generates a SSH key and logs the public key. Find the SSH public key with: + +```bash +kubectl -n flux logs deployment/flux | grep identity.pub | cut -d '"' -f2 +``` + +In order to sync your cluster state with Git you need to copy the public key and create a +deploy key with write access on your GitHub repository. + +Open GitHub, navigate to your fork, go to _Setting > Deploy keys_ click on _Add deploy key_, +check _Allow write access_, paste the Flux public key and click _Add key_. + +After a couple of seconds Flux will apply the Kubernetes resources from Git and Flagger will +launch the `frontend` and `backend` apps. + +A CI/CD pipeline for the `frontend` release could look like this: * cut a release from the master branch of the podinfo code repo with the git tag `1.4.1` * CI builds the image and pushes the `podinfo:1.4.1` image to the container registry @@ -302,7 +339,7 @@ A CI/CD pipeline for the frontend release could look like this: If the canary fails, fix the bug, do another patch release eg `1.4.2` and the whole process will run again. -There are a couple of reasons why a canary deployment fails: +A canary deployment can fail due to any of the following reasons: * the container image can't be downloaded * the deployment replica set is stuck for more then ten minutes (eg. due to a container crash loop) @@ -312,4 +349,5 @@ There are a couple of reasons why a canary deployment fails: * the Istio telemetry service is unable to collect traffic metrics * the metrics server (Prometheus) can't be reached - +If you want to find out more about managing Helm releases with Flux here is an in-depth guide +[github.com/stefanprodan/gitops-helm](https://github.com/stefanprodan/gitops-helm).