diff --git a/CHANGELOG.md b/CHANGELOG.md index fbec31a4..f63b63f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,9 +2,9 @@ All notable changes to this project are documented in this file. -## vNext (TBA) +## 0.16.0 (2019-06-23) -Adds support for running blue/green deployments without a service mesh or ingress controller +Adds support for running [Blue/Green deployments](https://docs.flagger.app/usage/blue-green) without a service mesh or ingress controller #### Features @@ -16,6 +16,7 @@ Adds support for running blue/green deployments without a service mesh or ingres - Add [FAQ page](https://docs.flagger.app/faq) to docs website - Switch to go modules in CI [#218](https://github.com/weaveworks/flagger/pull/218) +- Update e2e testing to Kubernetes Kind 0.3.0 and Istio 1.2.0 #### Fixes diff --git a/README.md b/README.md index 7dc11978..330f18e8 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,7 @@ Flagger documentation can be found at [docs.flagger.app](https://docs.flagger.ap * [App Mesh canary deployments](https://docs.flagger.app/usage/appmesh-progressive-delivery) * [NGINX ingress controller canary deployments](https://docs.flagger.app/usage/nginx-progressive-delivery) * [Gloo ingress controller canary deployments](https://docs.flagger.app/usage/gloo-progressive-delivery) + * [Blue/Green deployments](https://docs.flagger.app/usage/blue-green) * [Monitoring](https://docs.flagger.app/usage/monitoring) * [Alerting](https://docs.flagger.app/usage/alerting) * Tutorials @@ -65,6 +66,10 @@ metadata: name: podinfo namespace: test spec: + # service mesh provider (optional) + # can be: kubernetes, istio, appmesh, smi, nginx, gloo, supergloo + # use the kubernetes provider for Blue/Green style deployments + provider: istio # deployment reference targetRef: apiVersion: apps/v1 diff --git a/artifacts/canaries/canary.yaml b/artifacts/canaries/canary.yaml index ed0c8b01..e113a862 100644 --- a/artifacts/canaries/canary.yaml +++ b/artifacts/canaries/canary.yaml @@ -4,6 +4,10 @@ metadata: name: podinfo namespace: test spec: + # service mesh provider (default istio) + # can be: kubernetes, istio, appmesh, smi, nginx, gloo, supergloo + # use the kubernetes provider for Blue/Green style deployments + provider: istio # deployment reference targetRef: apiVersion: apps/v1 diff --git a/artifacts/flagger/deployment.yaml b/artifacts/flagger/deployment.yaml index b030920d..cf6183ed 100644 --- a/artifacts/flagger/deployment.yaml +++ b/artifacts/flagger/deployment.yaml @@ -22,7 +22,7 @@ spec: serviceAccountName: flagger containers: - name: flagger - image: weaveworks/flagger:0.15.0 + image: weaveworks/flagger:0.16.0 imagePullPolicy: IfNotPresent ports: - name: http diff --git a/charts/flagger/Chart.yaml b/charts/flagger/Chart.yaml index 890068ba..aecba758 100644 --- a/charts/flagger/Chart.yaml +++ b/charts/flagger/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 name: flagger -version: 0.15.0 -appVersion: 0.15.0 +version: 0.16.0 +appVersion: 0.16.0 kubeVersion: ">=1.11.0-0" engine: gotpl description: Flagger is a Kubernetes operator that automates the promotion of canary deployments using Istio, App Mesh or NGINX routing for traffic shifting and Prometheus metrics for canary analysis. diff --git a/charts/flagger/values.yaml b/charts/flagger/values.yaml index 9e34eaa5..8a55da93 100644 --- a/charts/flagger/values.yaml +++ b/charts/flagger/values.yaml @@ -2,7 +2,7 @@ image: repository: weaveworks/flagger - tag: 0.15.0 + tag: 0.16.0 pullPolicy: IfNotPresent metricsServer: "http://prometheus:9090" diff --git a/docs/diagrams/flagger-bluegreen-steps.png b/docs/diagrams/flagger-bluegreen-steps.png new file mode 100644 index 00000000..d1d2c938 Binary files /dev/null and b/docs/diagrams/flagger-bluegreen-steps.png differ diff --git a/docs/gitbook/SUMMARY.md b/docs/gitbook/SUMMARY.md index 4a674a2d..f3ec961f 100644 --- a/docs/gitbook/SUMMARY.md +++ b/docs/gitbook/SUMMARY.md @@ -18,6 +18,7 @@ * [App Mesh Canary Deployments](usage/appmesh-progressive-delivery.md) * [NGINX Canary Deployments](usage/nginx-progressive-delivery.md) * [Gloo Canary Deployments](usage/gloo-progressive-delivery.md) +* [Blue/Green Deployments](usage/blue-green.md) * [Monitoring](usage/monitoring.md) * [Alerting](usage/alerting.md) diff --git a/docs/gitbook/faq.md b/docs/gitbook/faq.md index 354da08b..edfc5332 100644 --- a/docs/gitbook/faq.md +++ b/docs/gitbook/faq.md @@ -1,8 +1,19 @@ # Frequently asked questions -### A/B Testing +### Deployment Strategies -When should I use A/B testing instead of progressive traffic shifting? +**Which deployment strategies are supported by Flagger?** + +Flagger can run automated application analysis, promotion and rollback for the following deployment strategies: +* Canary (progressive traffic shifting) +* A/B Testing (HTTP headers and cookies traffic routing) +* Blue/Green (traffic switch) + +For canary deployments you'll need a Layer 7 traffic management solution like a service mesh (Istio, App Mesh) or an ingress controller (NGINX, Gloo). +For A/B testing you'll need a Layer 7 traffic management solution that's capable of routing requests based on HTTP headers and cookies (Istio, NGINX). +For Blue/Green deployments no service mesh or ingress controller is required. + +**When should I use A/B testing instead of progressive traffic shifting?** For frontend applications that require session affinity you should use HTTP headers or cookies match conditions to ensure a set of users will stay on the same version for the whole duration of the canary analysis. @@ -53,9 +64,45 @@ curl -H 'X-Canary: insider' http://app.example.com curl -b 'canary=always' http://app.example.com ``` +**Can I use Flagger to manage applications that live outside of a service mesh?** + +For applications that are not deployed on a service mesh, Flagger can orchestrate Blue/Green style deployments +with Kubernetes L4 networking. + +Blue/Green example: + +```yaml +apiVersion: flagger.app/v1alpha3 +kind: Canary +spec: + provider: kubernetes + canaryAnalysis: + interval: 30s + threshold: 2 + iterations: 10 + metrics: + - name: request-success-rate + threshold: 99 + interval: 1m + - name: request-duration + threshold: 500 + interval: 30s + webhooks: + - name: load-test + url: http://flagger-loadtester.test/ + timeout: 5s + metadata: + type: cmd + cmd: "hey -z 1m -q 10 -c 2 http://podinfo-canary.test:9898/" +``` + +The above configuration will run an analysis for five minutes. +Flagger starts the load test for the canary service (green version) and checks the Prometheus metrics every 30 seconds. +If the analysis result is positive, Flagger will promote the canary (green version) to primary (blue version). + ### Kubernetes services -How is an application exposed inside the cluster? +**How is an application exposed inside the cluster?** Assuming the app name is podinfo you can define a canary like: @@ -139,7 +186,7 @@ canary analysis and can be used for conformance testing or load testing. ### Multiple ports -My application listens on multiple ports, how can I expose them inside the cluster? +**My application listens on multiple ports, how can I expose them inside the cluster?** If port discovery is enabled, Flagger scans the deployment spec and extracts the containers ports excluding the port specified in the canary service and Envoy sidecar ports. @@ -188,7 +235,7 @@ will point to the port specified in `spec.service.port`. ### Label selectors -What labels selectors are supported by Flagger? +**What labels selectors are supported by Flagger?** The target deployment must have a single label selector in the format `app: `: @@ -210,7 +257,7 @@ spec: Besides `app` Flagger supports `name` and `app.kubernetes.io/name` selectors. If you use a different convention you can specify your label with the `-selector-labels` flag. -Is pod affinity and anti affinity supported? +**Is pod affinity and anti affinity supported?** For pod affinity to work you need to use a different label than the `app`, `name` or `app.kubernetes.io/name`. @@ -245,7 +292,7 @@ spec: ### Istio Ingress Gateway -How can I expose multiple canaries on the same external domain? +**How can I expose multiple canaries on the same external domain?** Assuming you have two apps, one that servers the main website and one that serves the REST API. For each app you can define a canary object as: @@ -294,7 +341,7 @@ Note that host merging only works if the canaries are bounded to a ingress gatew ### Istio Mutual TLS -How can I enable mTLS for a canary? +**How can I enable mTLS for a canary?** When deploying Istio with global mTLS enabled, you have to set the TLS mode to `ISTIO_MUTUAL`: @@ -320,7 +367,7 @@ spec: mode: DISABLE ``` -If Flagger is outside of the mesh, how can it start the load test? +**If Flagger is outside of the mesh, how can it start the load test?** In order for Flagger to be able to call the load tester service from outside the mesh, you need to disable mTLS on port 80: diff --git a/docs/gitbook/how-it-works.md b/docs/gitbook/how-it-works.md index 430b812e..51f9a4f1 100644 --- a/docs/gitbook/how-it-works.md +++ b/docs/gitbook/how-it-works.md @@ -17,6 +17,10 @@ metadata: name: podinfo namespace: test spec: + # service mesh provider (optional) + # can be: kubernetes, istio, appmesh, smi, nginx, gloo, supergloo + # use the kubernetes provider for Blue/Green style deployments + provider: istio # deployment reference targetRef: apiVersion: apps/v1 diff --git a/docs/gitbook/usage/blue-green.md b/docs/gitbook/usage/blue-green.md new file mode 100644 index 00000000..9eb12e11 --- /dev/null +++ b/docs/gitbook/usage/blue-green.md @@ -0,0 +1,355 @@ +# Blue/Green Deployments + +This guide shows you how to automate Blue/Green deployments with Flagger and Kubernetes. + +For applications that are not deployed on a service mesh, Flagger can orchestrate Blue/Green style deployments +with Kubernetes L4 networking. + +![Flagger Blue/Green Stages](https://raw.githubusercontent.com/weaveworks/flagger/master/docs/diagrams/flagger-bluegreen-steps.png) + +### Prerequisites + +Flagger requires a Kubernetes cluster **v1.11** or newer. + +Install Flagger and the Prometheus add-on: + +```bash +helm repo add flagger https://flagger.app + +helm upgrade -i flagger flagger/flagger \ +--namespace flagger \ +--set prometheus.install=true \ +--set meshProvider=kubernetes +``` + +If you already have a Prometheus instance running in your cluster, +you can point Flagger to the ClusterIP service with: + +```bash +helm upgrade -i flagger flagger/flagger \ +--namespace flagger \ +--set metricsServer=http://prometheus.monitoring:9090 +``` + +Optionally you can enable Slack notifications: + +```bash +helm upgrade -i flagger flagger/flagger \ +--reuse-values \ +--namespace flagger \ +--set slack.url=https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK \ +--set slack.channel=general \ +--set slack.user=flagger +``` + +### Bootstrap + +Flagger takes a Kubernetes deployment and optionally a horizontal pod autoscaler (HPA), +then creates a series of objects (Kubernetes deployment and ClusterIP services). +These objects expose the application inside the cluster and drive the canary analysis and Blue/Green promotion. + +Create a test namespace: + +```bash +kubectl create ns test +``` + +Create a deployment and a horizontal pod autoscaler: + +```bash +export REPO=https://raw.githubusercontent.com/weaveworks/flagger/master + +kubectl apply -f ${REPO}/artifacts/canary/deployment.yaml +kubectl apply -f ${REPO}/artifacts/canary/hpa.yaml +``` + +Deploy the load testing service to generate traffic during the analysis: + +```bash +kubectl -n test apply -f ${REPO}/artifacts/loadtester/deployment.yaml +kubectl -n test apply -f ${REPO}/artifacts/loadtester/service.yaml +``` + +Create a canary custom resource: + +```yaml +apiVersion: flagger.app/v1alpha3 +kind: Canary +metadata: + name: podinfo + namespace: test +spec: + # service mesh provider can be: kubernetes, istio, appmesh, nginx, gloo + # use the kubernetes provider for Blue/Green style deployments + provider: kubernetes + # deployment reference + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: podinfo + # the maximum time in seconds for the canary deployment + # to make progress before rollback (default 600s) + progressDeadlineSeconds: 60 + # HPA reference (optional) + autoscalerRef: + apiVersion: autoscaling/v2beta1 + kind: HorizontalPodAutoscaler + name: podinfo + service: + # container port + port: 9898 + portDiscovery: true + canaryAnalysis: + # schedule interval (default 60s) + interval: 30s + # max number of failed checks before rollback + threshold: 2 + # number of checks to run before rollback + iterations: 10 + # Prometheus checks based on + # http_request_duration_seconds histogram + metrics: + - name: request-success-rate + # minimum req success rate (non 5xx responses) + # percentage (0-100) + threshold: 99 + interval: 1m + - name: request-duration + # maximum req duration P99 + # milliseconds + threshold: 500 + interval: 30s + # acceptance/load testing hooks + webhooks: + - name: smoke-test + type: pre-rollout + url: http://flagger-loadtester.test/ + timeout: 15s + metadata: + type: bash + cmd: "curl -sd 'anon' http://podinfo-canary.test:9898/token | grep token" + - name: load-test + url: http://flagger-loadtester.test/ + timeout: 5s + metadata: + type: cmd + cmd: "hey -z 1m -q 10 -c 2 http://podinfo-canary.test:9898/" +``` + +The above configuration will run an analysis for five minutes. + +Save the above resource as podinfo-canary.yaml and then apply it: + +```bash +kubectl apply -f ./podinfo-canary.yaml +``` + +After a couple of seconds Flagger will create the canary objects: + +```bash +# applied +deployment.apps/podinfo +horizontalpodautoscaler.autoscaling/podinfo +canary.flagger.app/podinfo + +# generated +deployment.apps/podinfo-primary +horizontalpodautoscaler.autoscaling/podinfo-primary +service/podinfo +service/podinfo-canary +service/podinfo-primary +``` + +Blue/Green scenario: +* on bootstrap, Flagger will create three ClusterIP services (`app-primary`,` app-canary`, `app`) and a shadow deployment named `app-primary` that represents the blue version +* when a new version is detected, Flagger would scale up the green version and run the conformance tests (the tests should target the `app-canary` ClusterIP service to reach the green version) +* if the conformance tests are passing, Flagger would start the load tests and validate them with custom Prometheus queries +* if the load test analysis is successful, Flagger will promote the new version to `app-primary` and scale down the green version + +### Automated Blue/Green promotion + +Trigger a deployment by updating the container image: + +```bash +kubectl -n test set image deployment/podinfo \ +podinfod=quay.io/stefanprodan/podinfo:1.4.1 +``` + +Flagger detects that the deployment revision changed and starts a new rollout: + +```text +kubectl -n test describe canary/podinfo + +Events: + +New revision detected podinfo.test +Waiting for podinfo.test rollout to finish: 0 of 1 updated replicas are available +Advance podinfo.test canary iteration 1/10 +Advance podinfo.test canary iteration 2/10 +Advance podinfo.test canary iteration 3/10 +Advance podinfo.test canary iteration 4/10 +Advance podinfo.test canary iteration 5/10 +Advance podinfo.test canary iteration 6/10 +Advance podinfo.test canary iteration 7/10 +Advance podinfo.test canary iteration 8/10 +Advance podinfo.test canary iteration 9/10 +Advance podinfo.test canary iteration 10/10 +Copying podinfo.test template spec to podinfo-primary.test +Waiting for podinfo-primary.test rollout to finish: 1 of 2 updated replicas are available +Promotion completed! Scaling down podinfo.test +``` + +**Note** that if you apply new changes to the deployment during the canary analysis, Flagger will restart the analysis. + +You can monitor all canaries with: + +```bash +watch kubectl get canaries --all-namespaces + +NAMESPACE NAME STATUS WEIGHT LASTTRANSITIONTIME +test podinfo Progressing 100 2019-06-16T14:05:07Z +prod frontend Succeeded 0 2019-06-15T16:15:07Z +prod backend Failed 0 2019-06-14T17:05:07Z +``` + +### Automated rollback + +During the analysis you can generate HTTP 500 errors and high latency to test Flagger's rollback. + +Exec into the load tester pod with: + +```bash +kubectl -n test exec -it flagger-loadtester-xx-xx sh +``` + +Generate HTTP 500 errors: + +```bash +watch curl http://podinfo-canary.test:9898/status/500 +``` + +Generate latency: + +```bash +watch curl http://podinfo-canary.test:9898/delay/1 +``` + +When the number of failed checks reaches the analysis threshold, +the green version is scaled to zero and the rollout is marked as failed. + +```text +kubectl -n test describe canary/podinfo + +Status: + Failed Checks: 2 + Phase: Failed +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal Synced 3m flagger New revision detected podinfo.test + Normal Synced 3m flagger Advance podinfo.test canary iteration 1/10 + Normal Synced 3m flagger Advance podinfo.test canary iteration 2/10 + Normal Synced 3m flagger Advance podinfo.test canary iteration 3/10 + Normal Synced 3m flagger Halt podinfo.test advancement success rate 69.17% < 99% + Normal Synced 2m flagger Halt podinfo.test advancement success rate 61.39% < 99% + Warning Synced 2m flagger Rolling back podinfo.test failed checks threshold reached 2 + Warning Synced 1m flagger Canary failed! Scaling down podinfo.test +``` + +### Custom metrics + +The analysis can be extended with Prometheus queries. The demo app is instrumented with Prometheus +so you can create a custom check that will use the HTTP request duration histogram to validate the canary (green version). + +Edit the canary analysis and add the following metric: + +```yaml + canaryAnalysis: + metrics: + - name: "404s percentage" + threshold: 5 + query: | + 100 - sum( + rate( + http_request_duration_seconds_count{ + kubernetes_namespace="test", + kubernetes_pod_name=~"podinfo-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)" + status!="404" + }[1m] + ) + ) + / + sum( + rate( + http_request_duration_seconds_count{ + kubernetes_namespace="test", + kubernetes_pod_name=~"podinfo-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)" + }[1m] + ) + ) * 100 +``` + +The above configuration validates the canary (green version) by checking if the HTTP 404 req/sec percentage is below 5 +percent of the total traffic. If the 404s rate reaches the 5% threshold, then the rollout is rolled back. + +Trigger a deployment by updating the container image: + +```bash +kubectl -n test set image deployment/podinfo \ +podinfod=quay.io/stefanprodan/podinfo:1.4.3 +``` + +Generate 404s: + +```bash +watch curl http://podinfo-canary.test:9898/status/400 +``` + +Watch Flagger logs: + +``` +kubectl -n flagger logs deployment/flagger -f | jq .msg + +New revision detected podinfo.test +Scaling up podinfo.test +Advance podinfo.test canary iteration 1/10 +Halt podinfo.test advancement 404s percentage 6.20 > 5 +Halt podinfo.test advancement 404s percentage 6.45 > 5 +Rolling back podinfo.test failed checks threshold reached 2 +Canary failed! Scaling down podinfo.test +``` + +If you have Slack configured, Flagger will send a notification with the reason why the canary failed. + +### Conformance Testing with Helm + +Flagger comes with a testing service that can run Helm tests when configured as a pre-rollout webhook. + +Deploy the Helm test runner in the `kube-system` namespace using the `tiller` service account: + +```bash +helm repo add flagger https://flagger.app + +helm upgrade -i flagger-helmtester flagger/loadtester \ +--namespace=kube-system \ +--set serviceAccountName=tiller +``` + +When deployed the Helm tester API will be available at `http://flagger-helmtester.kube-system/`. + +Add a helm test pre-rollout hook to your chart: + +```yaml + canaryAnalysis: + webhooks: + - name: "conformance testing" + type: pre-rollout + url: http://flagger-helmtester.kube-system/ + timeout: 3m + metadata: + type: "helm" + cmd: "test {{ .Release.Name }} --cleanup" +``` + +When the canary analysis starts, Flagger will call the pre-rollout webhooks. +If the helm test fails, Flagger will retry until the analysis threshold is reached and the canary is rolled back. diff --git a/pkg/version/version.go b/pkg/version/version.go index 13ecc04a..c3168c4a 100644 --- a/pkg/version/version.go +++ b/pkg/version/version.go @@ -1,4 +1,4 @@ package version -var VERSION = "0.15.0" +var VERSION = "0.16.0" var REVISION = "unknown"