Merge pull request #44 from stefanprodan/helm-docs

Add Helm and Weave Flux GitOps article
2026-04-15 06:57:34 +00:00 · 2019-02-13 11:51:38 +01:00
parent 8a03ae153d bb0175aebf
commit 5dfb5808c4
16 changed files with 433 additions and 10 deletions
@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: test
+  labels:
+    istio-injection: enabled
@@ -0,0 +1,26 @@
+apiVersion: flux.weave.works/v1beta1
+kind: HelmRelease
+metadata:
+  name: backend
+  namespace: test
+  annotations:
+    flux.weave.works/automated: "true"
+    flux.weave.works/tag.chart-image: regexp:^1.4.*
+spec:
+  releaseName: backend
+  chart:
+    repository: https://flagger.app/
+    name: podinfo
+    version: 2.0.0
+  values:
+    image:
+      repository: quay.io/stefanprodan/podinfo
+      tag: 1.4.0
+      httpServer:
+        timeout: 30s
+      canary:
+        enabled: true
+        istioIngress:
+          enabled: false
+        loadtest:
+          enabled: true
@@ -0,0 +1,27 @@
+apiVersion: flux.weave.works/v1beta1
+kind: HelmRelease
+metadata:
+  name: frontend
+  namespace: test
+  annotations:
+    flux.weave.works/automated: "true"
+    flux.weave.works/tag.chart-image: semver:~1.4
+spec:
+  releaseName: frontend
+  chart:
+    repository: https://flagger.app/
+    name: podinfo
+    version: 2.0.0
+  values:
+    image:
+      repository: quay.io/stefanprodan/podinfo
+      tag: 1.4.0
+      backend: http://backend-podinfo:9898/echo
+      canary:
+        enabled: true
+        istioIngress:
+          enabled: true
+          gateway: public-gateway.istio-system.svc.cluster.local
+          host: frontend.istio.example.com
+        loadtest:
+          enabled: true
@@ -0,0 +1,18 @@
+apiVersion: flux.weave.works/v1beta1
+kind: HelmRelease
+metadata:
+  name: loadtester
+  namespace: test
+  annotations:
+    flux.weave.works/automated: "true"
+    flux.weave.works/tag.chart-image: glob:0.*
+spec:
+  releaseName: flagger-loadtester
+  chart:
+    repository: https://flagger.app/
+    name: loadtester
+    version: 0.1.0
+  values:
+    image:
+      repository: quay.io/stefanprodan/flagger-loadtester
+      tag: 0.1.0
@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "podinfo.fullname" . }}
+  labels:
+    app: {{ template "podinfo.name" . }}
+    chart: {{ template "podinfo.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+data:
+  config.yaml: |-
+    # http settings
+    http-client-timeout: 1m
+    http-server-timeout: {{ .Values.httpServer.timeout }}
+    http-server-shutdown-timeout: 5s
@@ -33,6 +33,7 @@ spec:
            - --level={{ .Values.logLevel }}
            - --random-delay={{ .Values.faults.delay }}
            - --random-error={{ .Values.faults.error }}
+            - --config-path=/podinfo/config
          env:
          {{- if .Values.message }}
          - name: PODINFO_UI_MESSAGE
@@ -67,6 +68,9 @@ spec:
          volumeMounts:
          - name: data
            mountPath: /data
+          - name: config
+            mountPath: /podinfo/config
+            readOnly: true
          resources:
 {{ toYaml .Values.resources | indent 12 }}
    {{- with .Values.nodeSelector }}
@@ -84,3 +88,6 @@ spec:
      volumes:
      - name: data
        emptyDir: {}
+      - name: config
+        configMap:
+          name: {{ template "podinfo.fullname" . }}
@@ -3,6 +3,11 @@ apiVersion: autoscaling/v2beta1
 kind: HorizontalPodAutoscaler
 metadata:
  name: {{ template "podinfo.fullname" . }}
+  labels:
+    app: {{ template "podinfo.name" . }}
+    chart: {{ template "podinfo.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
 spec:
  scaleTargetRef:
    apiVersion: apps/v1beta2
@@ -11,12 +11,12 @@ service:
 hpa:
  enabled: true
  minReplicas: 2
-  maxReplicas: 4
+  maxReplicas: 2
  cpu: 80
  memory: 512Mi

 canary:
-  enabled: false
+  enabled: true
  istioIngress:
    enabled: false
    # Istio ingress gateway name
@@ -33,7 +33,7 @@ canary:
    maxWeight: 50
    # canary increment step
    # percentage (0-100)
-    stepWeight: 2
+    stepWeight: 5
  thresholds:
    # minimum req success rate (non 5xx responses)
    # percentage (0-100)
@@ -42,7 +42,7 @@ canary:
    # milliseconds
    latency: 500
  loadtest:
-    enabled: true
+    enabled: false
    # load tester address
    url: http://flagger-loadtester.test/

@@ -67,4 +67,7 @@ message: #UI greetings

 faults:
  delay: false
-  error: false
+  error: false
+
+httpServer:
+  timeout: 30s
@@ -6,16 +6,16 @@ description: Flagger is an Istio progressive delivery Kubernetes operator

 [Flagger](https://github.com/stefanprodan/flagger) is a **Kubernetes** operator that automates the promotion of canary 
 deployments using **Istio** routing for traffic shifting and **Prometheus** metrics for canary analysis.
-The canary analysis can be extended with webhooks for running integration tests, 
-load tests or any other custom validation.
+The canary analysis can be extended with webhooks for running 
+system integration/acceptance tests, load tests, or any other custom validation.

 Flagger implements a control loop that gradually shifts traffic to the canary while measuring key performance 
 indicators like HTTP requests success rate, requests average duration and pods health. 
-Based on the **KPIs** analysis a canary is promoted or aborted and the analysis result is published to **Slack**.
+Based on analysis of the **KPIs** a canary is promoted or aborted, and the analysis result is published to **Slack**.

 ![Flagger overview diagram](https://raw.githubusercontent.com/stefanprodan/flagger/master/docs/diagrams/flagger-canary-overview.png)

-Flagger can be configured with Kubernetes custom resources \(canaries.flagger.app kind\) and is compatible with 
+Flagger can be configured with Kubernetes custom resources and is compatible with 
 any CI/CD solutions made for Kubernetes. Since Flagger is declarative and reacts to Kubernetes events, 
 it can be used in **GitOps** pipelines together with Weave Flux or JenkinsX.

@@ -14,5 +14,6 @@
 * [Monitoring](usage/monitoring.md)
 * [Alerting](usage/alerting.md)

+# Tutorials

-
+* [Canary Deployments with Helm charts](tutorials/canary-helm-gitops.md)
@@ -0,0 +1,315 @@
+# Canary Deployments with Helm charts
+
+This guide shows you how to package a web app into a Helm chart, trigger canary deployments on Helm upgrade
+and automate the chart release process with Weave Flux.
+
+### Packaging
+
+You'll be using the [podinfo](https://github.com/stefanprodan/k8s-podinfo) chart. 
+This chart packages a web app made with Go, it's configuration, a horizontal pod autoscaler (HPA) 
+and the canary configuration file.
+
+```
+├── Chart.yaml
+├── README.md
+├── templates
+│   ├── NOTES.txt
+│   ├── _helpers.tpl
+│   ├── canary.yaml
+│   ├── configmap.yaml
+│   ├── deployment.yaml
+│   └── hpa.yaml
+└── values.yaml
+```
+
+You can find the chart source [here](https://github.com/stefanprodan/flagger/tree/master/charts/podinfo).
+
+### Install 
+
+Create a test namespace with Istio sidecar injection enabled:
+
+```bash
+export REPO=https://raw.githubusercontent.com/stefanprodan/flagger/master
+
+kubectl apply -f ${REPO}/artifacts/namespaces/test.yaml
+```
+
+Add Flagger Helm repository:
+
+```bash
+helm repo add flagger https://flagger.app
+```
+
+Install podinfo with the release name `frontend` (replace `example.com` with your own domain):
+
+```bash
+helm upgrade -i frontend flagger/podinfo \
+--namespace test \
+--set nameOverride=frontend \
+--set backend=http://backend.test:9898/echo \
+--set canary.enabled=true \
+--set canary.istioIngress.enabled=true \
+--set canary.istioIngress.gateway=public-gateway.istio-system.svc.cluster.local \
+--set canary.istioIngress.host=frontend.istio.example.com
+```
+
+Flagger takes a Kubernetes deployment and a horizontal pod autoscaler (HPA), 
+then creates a series of objects (Kubernetes deployments, ClusterIP services and Istio virtual services). 
+These objects expose the application on the mesh and drive the canary analysis and promotion.
+
+```bash
+# generated by Helm 
+configmap/frontend
+deployment.apps/frontend
+horizontalpodautoscaler.autoscaling/frontend
+canary.flagger.app/frontend
+
+# generated by Flagger
+configmap/frontend-primary
+deployment.apps/frontend-primary
+horizontalpodautoscaler.autoscaling/frontend-primary
+service/frontend
+service/frontend-canary
+service/frontend-primary
+virtualservice.networking.istio.io/frontend
+```
+
+When the `frontend-primary` deployment comes online, 
+Flagger will route all traffic to the primary pods and scale to zero the `frontend` deployment.
+
+Open your browser and navigate to the frontend URL:
+
+![Podinfo Frontend](https://raw.githubusercontent.com/stefanprodan/flagger/master/docs/screens/demo-frontend.png)
+
+Now let's install the `backend` release without exposing it outside the mesh:
+
+```bash
+helm upgrade -i backend flagger/podinfo \
+--namespace test \
+--set nameOverride=backend \
+--set canary.enabled=true \
+--set canary.istioIngress.enabled=false
+```
+
+Check if Flagger has successfully deployed the canaries: 
+
+```
+kubectl -n test get canaries
+
+NAME       STATUS        WEIGHT   LASTTRANSITIONTIME
+backend    Initialized   0        2019-02-12T18:53:18Z
+frontend   Initialized   0        2019-02-12T17:50:50Z
+```
+
+Click on the ping button in the `frontend` UI to trigger a HTTP POST request 
+that will reach the `backend` app:
+
+![Jaeger Tracing](https://raw.githubusercontent.com/stefanprodan/flagger/master/docs/screens/demo-frontend-jaeger.png)
+
+We'll use the `/echo` endpoint (same as the one the ping button calls) 
+to generate load on both apps during a canary deployment.
+
+### Upgrade
+
+First let's install a load testing service that will generate traffic during analysis:
+
+```bash
+helm upgrade -i flagger-loadtester flagger/loadtester \
+--namepace=test
+```
+
+Enable the load tester and deploy a new `frontend` version:
+
+```bash
+helm upgrade -i frontend flagger/podinfo/ \
+--namespace test \
+--reuse-values \
+--set canary.loadtest.enabled=true \
+--set image.tag=1.4.1
+```
+
+Flagger detects that the deployment revision changed and starts the canary analysis along with the load test:
+
+```
+kubectl -n istio-system logs deployment/flagger -f | jq .msg
+
+New revision detected! Scaling up frontend.test
+Halt advancement frontend.test waiting for rollout to finish: 0 of 2 updated replicas are available
+Starting canary analysis for frontend.test
+Advance frontend.test canary weight 5
+Advance frontend.test canary weight 10
+Advance frontend.test canary weight 15
+Advance frontend.test canary weight 20
+Advance frontend.test canary weight 25
+Advance frontend.test canary weight 30
+Advance frontend.test canary weight 35
+Advance frontend.test canary weight 40
+Advance frontend.test canary weight 45
+Advance frontend.test canary weight 50
+Copying frontend.test template spec to frontend-primary.test
+Halt advancement frontend-primary.test waiting for rollout to finish: 1 old replicas are pending termination
+Promotion completed! Scaling down frontend.test
+```
+
+You can monitor the canary deployment with Grafana. Open the Flagger dashboard, 
+select `test` from the namespace dropdown, `frontend-primary` from the primary dropdown and `frontend` from the
+canary dropdown.
+
+![Flagger Grafana Dashboard](https://raw.githubusercontent.com/stefanprodan/flagger/master/docs/screens/demo-frontend-dashboard.png)
+
+Now trigger a canary deployment for the `backend` app, but this time you'll change a value in the configmap:
+
+```bash
+helm upgrade -i backend flagger/podinfo/ \
+--namespace test \
+--reuse-values \
+--set canary.loadtest.enabled=true \
+--set httpServer.timeout=25s
+```
+
+Generate HTTP 500 errors:
+
+```bash
+kubectl -n test exec -it flagger-loadtester-xxx-yyy sh
+
+watch curl http://backend-canary:9898/status/500 
+```
+
+Generate latency:
+
+```bash
+kubectl -n test exec -it flagger-loadtester-xxx-yyy sh
+
+watch curl http://backend-canary:9898/delay/1 
+```
+
+Flagger detects the config map change and starts a canary analysis. Flagger will pause the advancement 
+when the HTTP success rate drops under 99% or when the average request duration in the last minute is over 500ms:
+
+```
+kubectl -n test describe canary backend
+
+Events:
+
+ConfigMap backend has changed
+New revision detected! Scaling up backend.test
+Starting canary analysis for backend.test
+Advance backend.test canary weight 5
+Advance backend.test canary weight 10
+Advance backend.test canary weight 15
+Advance backend.test canary weight 20
+Advance backend.test canary weight 25
+Advance backend.test canary weight 30
+Advance backend.test canary weight 35
+Halt backend.test advancement success rate 62.50% < 99%
+Halt backend.test advancement success rate 88.24% < 99%
+Advance backend.test canary weight 40
+Advance backend.test canary weight 45
+Halt backend.test advancement request duration 2.415s > 500ms
+Halt backend.test advancement request duration 2.42s > 500ms
+Advance backend.test canary weight 50
+ConfigMap backend-primary synced
+Copying backend.test template spec to backend-primary.test
+Promotion completed! Scaling down backend.test
+```
+
+![Flagger Grafana Dashboard](https://raw.githubusercontent.com/stefanprodan/flagger/master/docs/screens/demo-backend-dashboard.png)
+
+If the number of failed checks reaches the canary analysis threshold, the traffic is routed back to the primary, 
+the canary is scaled to zero and the rollout is marked as failed.
+
+```bash
+kubectl -n test get canary
+
+NAME       STATUS        WEIGHT   LASTTRANSITIONTIME
+backend    Succeeded     0        2019-02-12T19:33:11Z
+frontend   Failed        0        2019-02-12T19:47:20Z
+```
+
+If you've enabled the Slack notifications, you'll receive an alert with the reason why the `backend` promotion failed.
+
+### GitOps automation
+
+Instead of using Helm CLI from a CI tool to perform the install and upgrade, you could use a Git based approach. 
+
+![Helm GitOps Canary Deployment](https://raw.githubusercontent.com/stefanprodan/flagger/master/docs/diagrams/flagger-flux-gitops.png)
+
+Create a git repository with the following content:
+
+```
+├── namespaces
+│   └── test.yaml
+└── releases
+    └── test
+        ├── backend.yaml
+        ├── frontend.yaml
+        └── loadtester.yaml
+``` 
+
+You can find the git source [here](https://github.com/stefanprodan/flagger/tree/master/artifacts/cluster).
+
+Define the `frontend` release using Flux `HelmRelease` custom resource:
+
+```yaml
+apiVersion: flux.weave.works/v1beta1
+kind: HelmRelease
+metadata:
+  name: frontend
+  namespace: test
+  annotations:
+    flux.weave.works/automated: "true"
+    flux.weave.works/tag.chart-image: semver:~1.4
+spec:
+  releaseName: frontend
+  chart:
+    repository: https://stefanprodan.github.io/flagger/
+    name: podinfo
+    version: 2.0.0
+  values:
+    image:
+      repository: quay.io/stefanprodan/podinfo
+      tag: 1.4.0
+      backend: http://backend-podinfo:9898/echo
+      canary:
+        enabled: true
+        istioIngress:
+          enabled: true
+          gateway: public-gateway.istio-system.svc.cluster.local
+          host: frontend.istio.example.com
+        loadtest:
+          enabled: true
+```
+
+In the `chart` section I've defined the release source by specifying the Helm repository (hosted on GitHub Pages), chart name and version.
+In the `values` section I've overwritten the defaults set in values.yaml.
+
+With the `flux.weave.works` annotations I instruct Flux to automate this release. 
+When an image tag in the sem ver range of `1.4.0 - 1.4.99` is pushed to Quay, 
+Flux will upgrade the Helm release and from there Flagger will pick up the change and start a canary deployment.
+
+A CI/CD pipeline for the frontend release could look like this:
+
+* cut a release from the master branch of the podinfo code repo with the git tag `1.4.1`
+* CI builds the image and pushes the `podinfo:1.4.1` image to the container registry
+* Flux scans the registry and updates the Helm release `image.tag` to `1.4.1`
+* Flux commits and push the change to the cluster repo
+* Flux applies the updated Helm release on the cluster
+* Flux Helm Operator picks up the change and calls Tiller to upgrade the release
+* Flagger detects a revision change and scales up the `frontend` deployment
+* Flagger starts the load test and runs the canary analysis 
+* Based on the analysis result the canary deployment is promoted to production or rolled back
+* Flagger sends a Slack notification with the canary result
+
+If the canary fails, fix the bug, do another patch release eg `1.4.2` and the whole process will run again.
+
+There are a couple of reasons why a canary deployment fails:
+
+* the container image can't be downloaded 
+* the deployment replica set is stuck for more then ten minutes (eg. due to a container crash loop)
+* the webooks (acceptance tests, load tests, etc) are returning a non 2xx response
+* the HTTP success rate (non 5xx responses) metric drops under the threshold
+* the HTTP average duration metric goes over the threshold
+* the Istio telemetry service is unable to collect traffic metrics
+* the metrics server (Prometheus) can't be reached
+
+