diff --git a/artifacts/cluster/namespaces/test.yaml b/artifacts/cluster/namespaces/test.yaml new file mode 100644 index 00000000..6126d753 --- /dev/null +++ b/artifacts/cluster/namespaces/test.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: test + labels: + istio-injection: enabled diff --git a/artifacts/cluster/releases/test/backend.yaml b/artifacts/cluster/releases/test/backend.yaml new file mode 100644 index 00000000..d29e4903 --- /dev/null +++ b/artifacts/cluster/releases/test/backend.yaml @@ -0,0 +1,26 @@ +apiVersion: flux.weave.works/v1beta1 +kind: HelmRelease +metadata: + name: backend + namespace: test + annotations: + flux.weave.works/automated: "true" + flux.weave.works/tag.chart-image: regexp:^1.4.* +spec: + releaseName: backend + chart: + repository: https://flagger.app/ + name: podinfo + version: 2.0.0 + values: + image: + repository: quay.io/stefanprodan/podinfo + tag: 1.4.0 + httpServer: + timeout: 30s + canary: + enabled: true + istioIngress: + enabled: false + loadtest: + enabled: true diff --git a/artifacts/cluster/releases/test/frontend.yaml b/artifacts/cluster/releases/test/frontend.yaml new file mode 100644 index 00000000..f2fa539c --- /dev/null +++ b/artifacts/cluster/releases/test/frontend.yaml @@ -0,0 +1,27 @@ +apiVersion: flux.weave.works/v1beta1 +kind: HelmRelease +metadata: + name: frontend + namespace: test + annotations: + flux.weave.works/automated: "true" + flux.weave.works/tag.chart-image: semver:~1.4 +spec: + releaseName: frontend + chart: + repository: https://flagger.app/ + name: podinfo + version: 2.0.0 + values: + image: + repository: quay.io/stefanprodan/podinfo + tag: 1.4.0 + backend: http://backend-podinfo:9898/echo + canary: + enabled: true + istioIngress: + enabled: true + gateway: public-gateway.istio-system.svc.cluster.local + host: frontend.istio.example.com + loadtest: + enabled: true diff --git a/artifacts/cluster/releases/test/loadtester.yaml b/artifacts/cluster/releases/test/loadtester.yaml new file mode 100644 index 00000000..d098b9f2 --- /dev/null +++ b/artifacts/cluster/releases/test/loadtester.yaml @@ -0,0 +1,18 @@ +apiVersion: flux.weave.works/v1beta1 +kind: HelmRelease +metadata: + name: loadtester + namespace: test + annotations: + flux.weave.works/automated: "true" + flux.weave.works/tag.chart-image: glob:0.* +spec: + releaseName: flagger-loadtester + chart: + repository: https://flagger.app/ + name: loadtester + version: 0.1.0 + values: + image: + repository: quay.io/stefanprodan/flagger-loadtester + tag: 0.1.0 diff --git a/charts/podinfo/templates/configmap.yaml b/charts/podinfo/templates/configmap.yaml new file mode 100644 index 00000000..14b6a765 --- /dev/null +++ b/charts/podinfo/templates/configmap.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "podinfo.fullname" . }} + labels: + app: {{ template "podinfo.name" . }} + chart: {{ template "podinfo.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +data: + config.yaml: |- + # http settings + http-client-timeout: 1m + http-server-timeout: {{ .Values.httpServer.timeout }} + http-server-shutdown-timeout: 5s diff --git a/charts/podinfo/templates/deployment.yaml b/charts/podinfo/templates/deployment.yaml index 20a6751e..5eec44ad 100644 --- a/charts/podinfo/templates/deployment.yaml +++ b/charts/podinfo/templates/deployment.yaml @@ -33,6 +33,7 @@ spec: - --level={{ .Values.logLevel }} - --random-delay={{ .Values.faults.delay }} - --random-error={{ .Values.faults.error }} + - --config-path=/podinfo/config env: {{- if .Values.message }} - name: PODINFO_UI_MESSAGE @@ -67,6 +68,9 @@ spec: volumeMounts: - name: data mountPath: /data + - name: config + mountPath: /podinfo/config + readOnly: true resources: {{ toYaml .Values.resources | indent 12 }} {{- with .Values.nodeSelector }} @@ -84,3 +88,6 @@ spec: volumes: - name: data emptyDir: {} + - name: config + configMap: + name: {{ template "podinfo.fullname" . }} diff --git a/charts/podinfo/templates/hpa.yaml b/charts/podinfo/templates/hpa.yaml index dcb36d3a..9905cda6 100644 --- a/charts/podinfo/templates/hpa.yaml +++ b/charts/podinfo/templates/hpa.yaml @@ -3,6 +3,11 @@ apiVersion: autoscaling/v2beta1 kind: HorizontalPodAutoscaler metadata: name: {{ template "podinfo.fullname" . }} + labels: + app: {{ template "podinfo.name" . }} + chart: {{ template "podinfo.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} spec: scaleTargetRef: apiVersion: apps/v1beta2 diff --git a/charts/podinfo/values.yaml b/charts/podinfo/values.yaml index 3768d618..2aaa0f3b 100644 --- a/charts/podinfo/values.yaml +++ b/charts/podinfo/values.yaml @@ -11,12 +11,12 @@ service: hpa: enabled: true minReplicas: 2 - maxReplicas: 4 + maxReplicas: 2 cpu: 80 memory: 512Mi canary: - enabled: false + enabled: true istioIngress: enabled: false # Istio ingress gateway name @@ -33,7 +33,7 @@ canary: maxWeight: 50 # canary increment step # percentage (0-100) - stepWeight: 2 + stepWeight: 5 thresholds: # minimum req success rate (non 5xx responses) # percentage (0-100) @@ -42,7 +42,7 @@ canary: # milliseconds latency: 500 loadtest: - enabled: true + enabled: false # load tester address url: http://flagger-loadtester.test/ @@ -67,4 +67,7 @@ message: #UI greetings faults: delay: false - error: false \ No newline at end of file + error: false + +httpServer: + timeout: 30s diff --git a/docs/diagrams/flagger-flux-gitops.png b/docs/diagrams/flagger-flux-gitops.png new file mode 100644 index 00000000..1c9a8d0d Binary files /dev/null and b/docs/diagrams/flagger-flux-gitops.png differ diff --git a/docs/gitbook/README.md b/docs/gitbook/README.md index 92dba1e3..8b35a1af 100644 --- a/docs/gitbook/README.md +++ b/docs/gitbook/README.md @@ -6,16 +6,16 @@ description: Flagger is an Istio progressive delivery Kubernetes operator [Flagger](https://github.com/stefanprodan/flagger) is a **Kubernetes** operator that automates the promotion of canary deployments using **Istio** routing for traffic shifting and **Prometheus** metrics for canary analysis. -The canary analysis can be extended with webhooks for running integration tests, -load tests or any other custom validation. +The canary analysis can be extended with webhooks for running +system integration/acceptance tests, load tests, or any other custom validation. Flagger implements a control loop that gradually shifts traffic to the canary while measuring key performance indicators like HTTP requests success rate, requests average duration and pods health. -Based on the **KPIs** analysis a canary is promoted or aborted and the analysis result is published to **Slack**. +Based on analysis of the **KPIs** a canary is promoted or aborted, and the analysis result is published to **Slack**. ![Flagger overview diagram](https://raw.githubusercontent.com/stefanprodan/flagger/master/docs/diagrams/flagger-canary-overview.png) -Flagger can be configured with Kubernetes custom resources \(canaries.flagger.app kind\) and is compatible with +Flagger can be configured with Kubernetes custom resources and is compatible with any CI/CD solutions made for Kubernetes. Since Flagger is declarative and reacts to Kubernetes events, it can be used in **GitOps** pipelines together with Weave Flux or JenkinsX. diff --git a/docs/gitbook/SUMMARY.md b/docs/gitbook/SUMMARY.md index ee7dce00..b16be0c5 100644 --- a/docs/gitbook/SUMMARY.md +++ b/docs/gitbook/SUMMARY.md @@ -14,5 +14,6 @@ * [Monitoring](usage/monitoring.md) * [Alerting](usage/alerting.md) +# Tutorials - +* [Canary Deployments with Helm charts](tutorials/canary-helm-gitops.md) diff --git a/docs/gitbook/tutorials/canary-helm-gitops.md b/docs/gitbook/tutorials/canary-helm-gitops.md new file mode 100644 index 00000000..bb19464b --- /dev/null +++ b/docs/gitbook/tutorials/canary-helm-gitops.md @@ -0,0 +1,315 @@ +# Canary Deployments with Helm charts + +This guide shows you how to package a web app into a Helm chart, trigger canary deployments on Helm upgrade +and automate the chart release process with Weave Flux. + +### Packaging + +You'll be using the [podinfo](https://github.com/stefanprodan/k8s-podinfo) chart. +This chart packages a web app made with Go, it's configuration, a horizontal pod autoscaler (HPA) +and the canary configuration file. + +``` +├── Chart.yaml +├── README.md +├── templates +│   ├── NOTES.txt +│   ├── _helpers.tpl +│   ├── canary.yaml +│   ├── configmap.yaml +│   ├── deployment.yaml +│   └── hpa.yaml +└── values.yaml +``` + +You can find the chart source [here](https://github.com/stefanprodan/flagger/tree/master/charts/podinfo). + +### Install + +Create a test namespace with Istio sidecar injection enabled: + +```bash +export REPO=https://raw.githubusercontent.com/stefanprodan/flagger/master + +kubectl apply -f ${REPO}/artifacts/namespaces/test.yaml +``` + +Add Flagger Helm repository: + +```bash +helm repo add flagger https://flagger.app +``` + +Install podinfo with the release name `frontend` (replace `example.com` with your own domain): + +```bash +helm upgrade -i frontend flagger/podinfo \ +--namespace test \ +--set nameOverride=frontend \ +--set backend=http://backend.test:9898/echo \ +--set canary.enabled=true \ +--set canary.istioIngress.enabled=true \ +--set canary.istioIngress.gateway=public-gateway.istio-system.svc.cluster.local \ +--set canary.istioIngress.host=frontend.istio.example.com +``` + +Flagger takes a Kubernetes deployment and a horizontal pod autoscaler (HPA), +then creates a series of objects (Kubernetes deployments, ClusterIP services and Istio virtual services). +These objects expose the application on the mesh and drive the canary analysis and promotion. + +```bash +# generated by Helm +configmap/frontend +deployment.apps/frontend +horizontalpodautoscaler.autoscaling/frontend +canary.flagger.app/frontend + +# generated by Flagger +configmap/frontend-primary +deployment.apps/frontend-primary +horizontalpodautoscaler.autoscaling/frontend-primary +service/frontend +service/frontend-canary +service/frontend-primary +virtualservice.networking.istio.io/frontend +``` + +When the `frontend-primary` deployment comes online, +Flagger will route all traffic to the primary pods and scale to zero the `frontend` deployment. + +Open your browser and navigate to the frontend URL: + +![Podinfo Frontend](https://raw.githubusercontent.com/stefanprodan/flagger/master/docs/screens/demo-frontend.png) + +Now let's install the `backend` release without exposing it outside the mesh: + +```bash +helm upgrade -i backend flagger/podinfo \ +--namespace test \ +--set nameOverride=backend \ +--set canary.enabled=true \ +--set canary.istioIngress.enabled=false +``` + +Check if Flagger has successfully deployed the canaries: + +``` +kubectl -n test get canaries + +NAME STATUS WEIGHT LASTTRANSITIONTIME +backend Initialized 0 2019-02-12T18:53:18Z +frontend Initialized 0 2019-02-12T17:50:50Z +``` + +Click on the ping button in the `frontend` UI to trigger a HTTP POST request +that will reach the `backend` app: + +![Jaeger Tracing](https://raw.githubusercontent.com/stefanprodan/flagger/master/docs/screens/demo-frontend-jaeger.png) + +We'll use the `/echo` endpoint (same as the one the ping button calls) +to generate load on both apps during a canary deployment. + +### Upgrade + +First let's install a load testing service that will generate traffic during analysis: + +```bash +helm upgrade -i flagger-loadtester flagger/loadtester \ +--namepace=test +``` + +Enable the load tester and deploy a new `frontend` version: + +```bash +helm upgrade -i frontend flagger/podinfo/ \ +--namespace test \ +--reuse-values \ +--set canary.loadtest.enabled=true \ +--set image.tag=1.4.1 +``` + +Flagger detects that the deployment revision changed and starts the canary analysis along with the load test: + +``` +kubectl -n istio-system logs deployment/flagger -f | jq .msg + +New revision detected! Scaling up frontend.test +Halt advancement frontend.test waiting for rollout to finish: 0 of 2 updated replicas are available +Starting canary analysis for frontend.test +Advance frontend.test canary weight 5 +Advance frontend.test canary weight 10 +Advance frontend.test canary weight 15 +Advance frontend.test canary weight 20 +Advance frontend.test canary weight 25 +Advance frontend.test canary weight 30 +Advance frontend.test canary weight 35 +Advance frontend.test canary weight 40 +Advance frontend.test canary weight 45 +Advance frontend.test canary weight 50 +Copying frontend.test template spec to frontend-primary.test +Halt advancement frontend-primary.test waiting for rollout to finish: 1 old replicas are pending termination +Promotion completed! Scaling down frontend.test +``` + +You can monitor the canary deployment with Grafana. Open the Flagger dashboard, +select `test` from the namespace dropdown, `frontend-primary` from the primary dropdown and `frontend` from the +canary dropdown. + +![Flagger Grafana Dashboard](https://raw.githubusercontent.com/stefanprodan/flagger/master/docs/screens/demo-frontend-dashboard.png) + +Now trigger a canary deployment for the `backend` app, but this time you'll change a value in the configmap: + +```bash +helm upgrade -i backend flagger/podinfo/ \ +--namespace test \ +--reuse-values \ +--set canary.loadtest.enabled=true \ +--set httpServer.timeout=25s +``` + +Generate HTTP 500 errors: + +```bash +kubectl -n test exec -it flagger-loadtester-xxx-yyy sh + +watch curl http://backend-canary:9898/status/500 +``` + +Generate latency: + +```bash +kubectl -n test exec -it flagger-loadtester-xxx-yyy sh + +watch curl http://backend-canary:9898/delay/1 +``` + +Flagger detects the config map change and starts a canary analysis. Flagger will pause the advancement +when the HTTP success rate drops under 99% or when the average request duration in the last minute is over 500ms: + +``` +kubectl -n test describe canary backend + +Events: + +ConfigMap backend has changed +New revision detected! Scaling up backend.test +Starting canary analysis for backend.test +Advance backend.test canary weight 5 +Advance backend.test canary weight 10 +Advance backend.test canary weight 15 +Advance backend.test canary weight 20 +Advance backend.test canary weight 25 +Advance backend.test canary weight 30 +Advance backend.test canary weight 35 +Halt backend.test advancement success rate 62.50% < 99% +Halt backend.test advancement success rate 88.24% < 99% +Advance backend.test canary weight 40 +Advance backend.test canary weight 45 +Halt backend.test advancement request duration 2.415s > 500ms +Halt backend.test advancement request duration 2.42s > 500ms +Advance backend.test canary weight 50 +ConfigMap backend-primary synced +Copying backend.test template spec to backend-primary.test +Promotion completed! Scaling down backend.test +``` + +![Flagger Grafana Dashboard](https://raw.githubusercontent.com/stefanprodan/flagger/master/docs/screens/demo-backend-dashboard.png) + +If the number of failed checks reaches the canary analysis threshold, the traffic is routed back to the primary, +the canary is scaled to zero and the rollout is marked as failed. + +```bash +kubectl -n test get canary + +NAME STATUS WEIGHT LASTTRANSITIONTIME +backend Succeeded 0 2019-02-12T19:33:11Z +frontend Failed 0 2019-02-12T19:47:20Z +``` + +If you've enabled the Slack notifications, you'll receive an alert with the reason why the `backend` promotion failed. + +### GitOps automation + +Instead of using Helm CLI from a CI tool to perform the install and upgrade, you could use a Git based approach. + +![Helm GitOps Canary Deployment](https://raw.githubusercontent.com/stefanprodan/flagger/master/docs/diagrams/flagger-flux-gitops.png) + +Create a git repository with the following content: + +``` +├── namespaces +│   └── test.yaml +└── releases + └── test + ├── backend.yaml + ├── frontend.yaml + └── loadtester.yaml +``` + +You can find the git source [here](https://github.com/stefanprodan/flagger/tree/master/artifacts/cluster). + +Define the `frontend` release using Flux `HelmRelease` custom resource: + +```yaml +apiVersion: flux.weave.works/v1beta1 +kind: HelmRelease +metadata: + name: frontend + namespace: test + annotations: + flux.weave.works/automated: "true" + flux.weave.works/tag.chart-image: semver:~1.4 +spec: + releaseName: frontend + chart: + repository: https://stefanprodan.github.io/flagger/ + name: podinfo + version: 2.0.0 + values: + image: + repository: quay.io/stefanprodan/podinfo + tag: 1.4.0 + backend: http://backend-podinfo:9898/echo + canary: + enabled: true + istioIngress: + enabled: true + gateway: public-gateway.istio-system.svc.cluster.local + host: frontend.istio.example.com + loadtest: + enabled: true +``` + +In the `chart` section I've defined the release source by specifying the Helm repository (hosted on GitHub Pages), chart name and version. +In the `values` section I've overwritten the defaults set in values.yaml. + +With the `flux.weave.works` annotations I instruct Flux to automate this release. +When an image tag in the sem ver range of `1.4.0 - 1.4.99` is pushed to Quay, +Flux will upgrade the Helm release and from there Flagger will pick up the change and start a canary deployment. + +A CI/CD pipeline for the frontend release could look like this: + +* cut a release from the master branch of the podinfo code repo with the git tag `1.4.1` +* CI builds the image and pushes the `podinfo:1.4.1` image to the container registry +* Flux scans the registry and updates the Helm release `image.tag` to `1.4.1` +* Flux commits and push the change to the cluster repo +* Flux applies the updated Helm release on the cluster +* Flux Helm Operator picks up the change and calls Tiller to upgrade the release +* Flagger detects a revision change and scales up the `frontend` deployment +* Flagger starts the load test and runs the canary analysis +* Based on the analysis result the canary deployment is promoted to production or rolled back +* Flagger sends a Slack notification with the canary result + +If the canary fails, fix the bug, do another patch release eg `1.4.2` and the whole process will run again. + +There are a couple of reasons why a canary deployment fails: + +* the container image can't be downloaded +* the deployment replica set is stuck for more then ten minutes (eg. due to a container crash loop) +* the webooks (acceptance tests, load tests, etc) are returning a non 2xx response +* the HTTP success rate (non 5xx responses) metric drops under the threshold +* the HTTP average duration metric goes over the threshold +* the Istio telemetry service is unable to collect traffic metrics +* the metrics server (Prometheus) can't be reached + + diff --git a/docs/screens/demo-backend-dashboard.png b/docs/screens/demo-backend-dashboard.png new file mode 100644 index 00000000..d8303417 Binary files /dev/null and b/docs/screens/demo-backend-dashboard.png differ diff --git a/docs/screens/demo-frontend-dashboard.png b/docs/screens/demo-frontend-dashboard.png new file mode 100644 index 00000000..febf78cd Binary files /dev/null and b/docs/screens/demo-frontend-dashboard.png differ diff --git a/docs/screens/demo-frontend-jaeger.png b/docs/screens/demo-frontend-jaeger.png new file mode 100644 index 00000000..28da7ff0 Binary files /dev/null and b/docs/screens/demo-frontend-jaeger.png differ diff --git a/docs/screens/demo-frontend.png b/docs/screens/demo-frontend.png new file mode 100644 index 00000000..1dbf922c Binary files /dev/null and b/docs/screens/demo-frontend.png differ