mirror of
https://github.com/fluxcd/flagger.git
synced 2026-04-15 06:57:34 +00:00
Publish website
This commit is contained in:
weaveworksbot
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -0,0 +1,192 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en-US">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<title>SMI Istio Canary Deployments | Flagger</title>
|
||||
<meta name="description" content="Progressive Delivery operator for Kubernetes (Canary, A/B Testing and Blue/Green deployments)">
|
||||
<link rel="icon" href="/favicon.png">
|
||||
<link rel="stylesheet" href="/website.css">
|
||||
<meta name="keywords" content="gitops kubernetes flagger istio linkerd appmesh contour gloo nginx">
|
||||
<meta name="twitter:card" content="summary_large_image">
|
||||
<meta name="twitter:title" content="Flagger">
|
||||
<meta name="twitter:description" content="Progressive delivery Kubernetes operator (Canary, A/B Testing and Blue/Green deployments)">
|
||||
<meta name="twitter:image:src" content="https://flagger.app/flagger-overview.png">
|
||||
|
||||
<link rel="preload" href="/assets/css/0.styles.a0ada997.css" as="style"><link rel="preload" href="/assets/js/app.b4b27991.js" as="script"><link rel="preload" href="/assets/js/2.ba5bf77c.js" as="script"><link rel="preload" href="/assets/js/21.df66561d.js" as="script"><link rel="prefetch" href="/assets/js/10.bf6f38d4.js"><link rel="prefetch" href="/assets/js/11.3af38cfe.js"><link rel="prefetch" href="/assets/js/12.eca4b89c.js"><link rel="prefetch" href="/assets/js/13.9104349a.js"><link rel="prefetch" href="/assets/js/14.a4f3d810.js"><link rel="prefetch" href="/assets/js/15.aada2caa.js"><link rel="prefetch" href="/assets/js/16.3012a10b.js"><link rel="prefetch" href="/assets/js/17.a98b6aa9.js"><link rel="prefetch" href="/assets/js/18.e5188926.js"><link rel="prefetch" href="/assets/js/19.bb9498f7.js"><link rel="prefetch" href="/assets/js/20.dce5a04e.js"><link rel="prefetch" href="/assets/js/22.1c5d14b9.js"><link rel="prefetch" href="/assets/js/23.b3dcc355.js"><link rel="prefetch" href="/assets/js/24.4f54d289.js"><link rel="prefetch" href="/assets/js/25.914acd1d.js"><link rel="prefetch" href="/assets/js/26.b2d4f402.js"><link rel="prefetch" href="/assets/js/27.ea9f1f9c.js"><link rel="prefetch" href="/assets/js/28.deae273d.js"><link rel="prefetch" href="/assets/js/29.17c547e6.js"><link rel="prefetch" href="/assets/js/3.8b254d5d.js"><link rel="prefetch" href="/assets/js/30.38bd809e.js"><link rel="prefetch" href="/assets/js/31.fbfb336c.js"><link rel="prefetch" href="/assets/js/32.591c3f2b.js"><link rel="prefetch" href="/assets/js/33.c456709f.js"><link rel="prefetch" href="/assets/js/34.5a071e36.js"><link rel="prefetch" href="/assets/js/35.fc27c09e.js"><link rel="prefetch" href="/assets/js/36.144098aa.js"><link rel="prefetch" href="/assets/js/4.26a3069c.js"><link rel="prefetch" href="/assets/js/5.f92a6aca.js"><link rel="prefetch" href="/assets/js/6.ee5ffca3.js"><link rel="prefetch" href="/assets/js/7.6bf27a87.js"><link rel="prefetch" href="/assets/js/8.8b72a8d3.js"><link rel="prefetch" href="/assets/js/9.7919fc1d.js">
|
||||
<link rel="stylesheet" href="/assets/css/0.styles.a0ada997.css">
|
||||
</head>
|
||||
<body>
|
||||
<div id="app" data-server-rendered="true"><div class="theme-container"><header class="navbar"><div class="sidebar-button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" role="img" viewBox="0 0 448 512" class="icon"><path fill="currentColor" d="M436 124H12c-6.627 0-12-5.373-12-12V80c0-6.627 5.373-12 12-12h424c6.627 0 12 5.373 12 12v32c0 6.627-5.373 12-12 12zm0 160H12c-6.627 0-12-5.373-12-12v-32c0-6.627 5.373-12 12-12h424c6.627 0 12 5.373 12 12v32c0 6.627-5.373 12-12 12zm0 160H12c-6.627 0-12-5.373-12-12v-32c0-6.627 5.373-12 12-12h424c6.627 0 12 5.373 12 12v32c0 6.627-5.373 12-12 12z"></path></svg></div> <a href="/" class="home-link router-link-active"><!----> <span class="site-name">Flagger</span></a> <div class="links"><div class="search-box"><input aria-label="Search" autocomplete="off" spellcheck="false" value=""> <!----></div> <nav class="nav-links can-hide"><div class="nav-item"><a href="https://docs.flagger.app" target="_blank" rel="noopener noreferrer" class="nav-link external">
|
||||
Docs
|
||||
<svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg></a></div><div class="nav-item"><a href="https://github.com/weaveworks/flagger/blob/master/CHANGELOG.md" target="_blank" rel="noopener noreferrer" class="nav-link external">
|
||||
Changelog
|
||||
<svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg></a></div> <a href="https://github.com/weaveworks/flagger" target="_blank" rel="noopener noreferrer" class="repo-link">
|
||||
GitHub
|
||||
<svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg></a></nav></div></header> <div class="sidebar-mask"></div> <aside class="sidebar"><nav class="nav-links"><div class="nav-item"><a href="https://docs.flagger.app" target="_blank" rel="noopener noreferrer" class="nav-link external">
|
||||
Docs
|
||||
<svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg></a></div><div class="nav-item"><a href="https://github.com/weaveworks/flagger/blob/master/CHANGELOG.md" target="_blank" rel="noopener noreferrer" class="nav-link external">
|
||||
Changelog
|
||||
<svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg></a></div> <a href="https://github.com/weaveworks/flagger" target="_blank" rel="noopener noreferrer" class="repo-link">
|
||||
GitHub
|
||||
<svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg></a></nav> <ul class="sidebar-links"><li><a href="/" class="sidebar-link">Home</a></li><li><section class="sidebar-group depth-0"><a href="/intro/" class="sidebar-heading clickable"><span>Introduction</span> <!----></a> <ul class="sidebar-links sidebar-group-items"><li><a href="/intro/" class="sidebar-link">Get Started</a></li><li><a href="/intro/faq.html" class="sidebar-link">FAQ</a></li></ul></section></li><li><section class="sidebar-group depth-0"><a href="/install/flagger-install-on-kubernetes" class="sidebar-heading clickable"><span>Install</span> <!----></a> <ul class="sidebar-links sidebar-group-items"><li><a href="/install/flagger-install-on-kubernetes.html" class="sidebar-link">On Kubernetes</a></li><li><a href="/install/flagger-install-on-google-cloud.html" class="sidebar-link">On GKE Istio</a></li><li><a href="/install/flagger-install-on-eks-appmesh.html" class="sidebar-link">On EKS App Mesh</a></li></ul></section></li><li><section class="sidebar-group depth-0"><a href="/usage/how-it-works" class="sidebar-heading clickable"><span>Usage</span> <!----></a> <ul class="sidebar-links sidebar-group-items"><li><a href="/usage/how-it-works.html" class="sidebar-link">How it works</a></li><li><a href="/usage/deployment-strategies.html" class="sidebar-link">Deployment Strategies</a></li><li><a href="/usage/metrics.html" class="sidebar-link">Metrics Analysis</a></li><li><a href="/usage/webhooks.html" class="sidebar-link">Webhooks</a></li><li><a href="/usage/alerting.html" class="sidebar-link">Alerting</a></li><li><a href="/usage/monitoring.html" class="sidebar-link">Monitoring</a></li></ul></section></li><li><section class="sidebar-group depth-0"><a href="/tutorials/istio-progressive-delivery" class="sidebar-heading clickable"><span>Tutorials</span> <!----></a> <ul class="sidebar-links sidebar-group-items"><li><a href="/tutorials/istio-progressive-delivery.html" class="sidebar-link">Istio Canaries</a></li><li><a href="/tutorials/istio-ab-testing.html" class="sidebar-link">Istio A/B Testing</a></li><li><a href="/tutorials/linkerd-progressive-delivery.html" class="sidebar-link">Linkerd Canaries</a></li><li><a href="/tutorials/appmesh-progressive-delivery.html" class="sidebar-link">App Mesh Canaries</a></li><li><a href="/tutorials/nginx-progressive-delivery.html" class="sidebar-link">NGINX Ingress Canaries</a></li><li><a href="/tutorials/gloo-progressive-delivery.html" class="sidebar-link">Gloo Canaries</a></li><li><a href="/tutorials/contour-progressive-delivery.html" class="sidebar-link">Contour Canaries</a></li><li><a href="/tutorials/kubernetes-blue-green.html" class="sidebar-link">Kubernetes Blue/Green</a></li><li><a href="/tutorials/canary-helm-gitops.html" class="sidebar-link">Canaries with Helm charts and GitOps</a></li></ul></section></li><li><section class="sidebar-group depth-0"><a href="/dev/dev-guide" class="sidebar-heading clickable"><span>Dev</span> <!----></a> <ul class="sidebar-links sidebar-group-items"><li><a href="/dev/dev-guide.html" class="sidebar-link">Development Guide</a></li><li><a href="/dev/release-guide.html" class="sidebar-link">Release Guide</a></li><li><a href="/dev/upgrade-guide.html" class="sidebar-link">Upgrade Guide</a></li></ul></section></li></ul> </aside> <main class="page"> <div class="theme-default-content content__default"><h1 id="smi-istio-canary-deployments"><a href="#smi-istio-canary-deployments" aria-hidden="true" class="header-anchor">#</a> SMI Istio Canary Deployments</h1> <p>This guide shows you how to use the SMI Istio adapter and Flagger to automate canary deployments.</p> <h2 id="prerequisites"><a href="#prerequisites" aria-hidden="true" class="header-anchor">#</a> Prerequisites</h2> <ul><li>Kubernetes > 1.13</li> <li>Istio > 1.0</li></ul> <h2 id="install-istio-smi-adapter"><a href="#install-istio-smi-adapter" aria-hidden="true" class="header-anchor">#</a> Install Istio SMI adapter</h2> <p>Install the SMI adapter:</p> <div class="language-bash extra-class"><pre class="language-bash"><code>kubectl apply -f https://raw.githubusercontent.com/deislabs/smi-adapter-istio/master/deploy/crds/crds.yaml
|
||||
kubectl apply -f https://raw.githubusercontent.com/deislabs/smi-adapter-istio/master/deploy/operator-and-rbac.yaml
|
||||
</code></pre></div><p>Create a generic Istio gateway to expose services outside the mesh on HTTP:</p> <div class="language-yaml extra-class"><pre class="language-yaml"><code><span class="token key atrule">apiVersion</span><span class="token punctuation">:</span> networking.istio.io/v1alpha3
|
||||
<span class="token key atrule">kind</span><span class="token punctuation">:</span> Gateway
|
||||
<span class="token key atrule">metadata</span><span class="token punctuation">:</span>
|
||||
<span class="token key atrule">name</span><span class="token punctuation">:</span> public<span class="token punctuation">-</span>gateway
|
||||
<span class="token key atrule">namespace</span><span class="token punctuation">:</span> istio<span class="token punctuation">-</span>system
|
||||
<span class="token key atrule">spec</span><span class="token punctuation">:</span>
|
||||
<span class="token key atrule">selector</span><span class="token punctuation">:</span>
|
||||
<span class="token key atrule">istio</span><span class="token punctuation">:</span> ingressgateway
|
||||
<span class="token key atrule">servers</span><span class="token punctuation">:</span>
|
||||
<span class="token punctuation">-</span> <span class="token key atrule">port</span><span class="token punctuation">:</span>
|
||||
<span class="token key atrule">number</span><span class="token punctuation">:</span> <span class="token number">80</span>
|
||||
<span class="token key atrule">name</span><span class="token punctuation">:</span> http
|
||||
<span class="token key atrule">protocol</span><span class="token punctuation">:</span> HTTP
|
||||
<span class="token key atrule">hosts</span><span class="token punctuation">:</span>
|
||||
<span class="token punctuation">-</span> <span class="token string">"*"</span>
|
||||
</code></pre></div><p>Save the above resource as public-gateway.yaml and then apply it:</p> <div class="language-bash extra-class"><pre class="language-bash"><code>kubectl apply -f ./public-gateway.yaml
|
||||
</code></pre></div><p>Find the Gateway load balancer IP and add a DNS record for it:</p> <div class="language-bash extra-class"><pre class="language-bash"><code>kubectl -n istio-system get svc/istio-ingressgateway -ojson <span class="token operator">|</span> jq -r .status.loadBalancer.ingress<span class="token punctuation">[</span><span class="token number">0</span><span class="token punctuation">]</span>.ip
|
||||
</code></pre></div><h2 id="install-flagger-and-grafana"><a href="#install-flagger-and-grafana" aria-hidden="true" class="header-anchor">#</a> Install Flagger and Grafana</h2> <p>Add Flagger Helm repository:</p> <div class="language-bash extra-class"><pre class="language-bash"><code>helm repo <span class="token function">add</span> flagger https://flagger.app
|
||||
</code></pre></div><p>Deploy Flagger in the <em><strong>istio-system</strong></em> namespace:</p> <div class="language-bash extra-class"><pre class="language-bash"><code>helm upgrade -i flagger flagger/flagger <span class="token punctuation">\</span>
|
||||
--namespace<span class="token operator">=</span>istio-system <span class="token punctuation">\</span>
|
||||
--set <span class="token assign-left variable">meshProvider</span><span class="token operator">=</span>smi:istio
|
||||
</code></pre></div><p>Flagger comes with a Grafana dashboard made for monitoring the canary deployments.</p> <p>Deploy Grafana in the <em><strong>istio-system</strong></em> namespace:</p> <div class="language-bash extra-class"><pre class="language-bash"><code>helm upgrade -i flagger-grafana flagger/grafana <span class="token punctuation">\</span>
|
||||
--namespace<span class="token operator">=</span>istio-system <span class="token punctuation">\</span>
|
||||
--set <span class="token assign-left variable">url</span><span class="token operator">=</span>http://prometheus.istio-system:9090
|
||||
</code></pre></div><p>You can access Grafana using port forwarding:</p> <div class="language-bash extra-class"><pre class="language-bash"><code>kubectl -n istio-system port-forward svc/flagger-grafana <span class="token number">3000</span>:80
|
||||
</code></pre></div><h2 id="workloads-bootstrap"><a href="#workloads-bootstrap" aria-hidden="true" class="header-anchor">#</a> Workloads bootstrap</h2> <p>Create a test namespace with Istio sidecar injection enabled:</p> <p>Create a test namespace and enable Linkerd proxy injection:</p> <div class="language-bash extra-class"><pre class="language-bash"><code>kubectl create ns <span class="token builtin class-name">test</span>
|
||||
kubectl label namespace <span class="token builtin class-name">test</span> istio-injection<span class="token operator">=</span>enabled
|
||||
</code></pre></div><p>Create a deployment and a horizontal pod autoscaler:</p> <div class="language-bash extra-class"><pre class="language-bash"><code>kubectl apply -k github.com/weaveworks/flagger//kustomize/podinfo
|
||||
</code></pre></div><p>Deploy the load testing service to generate traffic during the canary analysis:</p> <div class="language-bash extra-class"><pre class="language-bash"><code>kubectl apply -k github.com/weaveworks/flagger//kustomize/tester
|
||||
</code></pre></div><p>Create a canary custom resource (replace example.com with your own domain):</p> <div class="language-yaml extra-class"><pre class="language-yaml"><code><span class="token key atrule">apiVersion</span><span class="token punctuation">:</span> flagger.app/v1beta1
|
||||
<span class="token key atrule">kind</span><span class="token punctuation">:</span> Canary
|
||||
<span class="token key atrule">metadata</span><span class="token punctuation">:</span>
|
||||
<span class="token key atrule">name</span><span class="token punctuation">:</span> podinfo
|
||||
<span class="token key atrule">namespace</span><span class="token punctuation">:</span> test
|
||||
<span class="token key atrule">spec</span><span class="token punctuation">:</span>
|
||||
<span class="token comment"># deployment reference</span>
|
||||
<span class="token key atrule">targetRef</span><span class="token punctuation">:</span>
|
||||
<span class="token key atrule">apiVersion</span><span class="token punctuation">:</span> apps/v1
|
||||
<span class="token key atrule">kind</span><span class="token punctuation">:</span> Deployment
|
||||
<span class="token key atrule">name</span><span class="token punctuation">:</span> podinfo
|
||||
<span class="token comment"># the maximum time in seconds for the canary deployment</span>
|
||||
<span class="token comment"># to make progress before it is rollback (default 600s)</span>
|
||||
<span class="token key atrule">progressDeadlineSeconds</span><span class="token punctuation">:</span> <span class="token number">60</span>
|
||||
<span class="token comment"># HPA reference (optional)</span>
|
||||
<span class="token key atrule">autoscalerRef</span><span class="token punctuation">:</span>
|
||||
<span class="token key atrule">apiVersion</span><span class="token punctuation">:</span> autoscaling/v2beta1
|
||||
<span class="token key atrule">kind</span><span class="token punctuation">:</span> HorizontalPodAutoscaler
|
||||
<span class="token key atrule">name</span><span class="token punctuation">:</span> podinfo
|
||||
<span class="token key atrule">service</span><span class="token punctuation">:</span>
|
||||
<span class="token comment"># container port</span>
|
||||
<span class="token key atrule">port</span><span class="token punctuation">:</span> <span class="token number">9898</span>
|
||||
<span class="token comment"># Istio gateways (optional)</span>
|
||||
<span class="token key atrule">gateways</span><span class="token punctuation">:</span>
|
||||
<span class="token punctuation">-</span> public<span class="token punctuation">-</span>gateway.istio<span class="token punctuation">-</span>system.svc.cluster.local
|
||||
<span class="token comment"># Istio virtual service host names (optional)</span>
|
||||
<span class="token key atrule">hosts</span><span class="token punctuation">:</span>
|
||||
<span class="token punctuation">-</span> app.example.com
|
||||
<span class="token key atrule">analysis</span><span class="token punctuation">:</span>
|
||||
<span class="token comment"># schedule interval (default 60s)</span>
|
||||
<span class="token key atrule">interval</span><span class="token punctuation">:</span> 10s
|
||||
<span class="token comment"># max number of failed metric checks before rollback</span>
|
||||
<span class="token key atrule">threshold</span><span class="token punctuation">:</span> <span class="token number">5</span>
|
||||
<span class="token comment"># max traffic percentage routed to canary</span>
|
||||
<span class="token comment"># percentage (0-100)</span>
|
||||
<span class="token key atrule">maxWeight</span><span class="token punctuation">:</span> <span class="token number">50</span>
|
||||
<span class="token comment"># canary increment step</span>
|
||||
<span class="token comment"># percentage (0-100)</span>
|
||||
<span class="token key atrule">stepWeight</span><span class="token punctuation">:</span> <span class="token number">10</span>
|
||||
<span class="token key atrule">metrics</span><span class="token punctuation">:</span>
|
||||
<span class="token punctuation">-</span> <span class="token key atrule">name</span><span class="token punctuation">:</span> request<span class="token punctuation">-</span>success<span class="token punctuation">-</span>rate
|
||||
<span class="token comment"># minimum req success rate (non 5xx responses)</span>
|
||||
<span class="token comment"># percentage (0-100)</span>
|
||||
<span class="token key atrule">thresholdRange</span><span class="token punctuation">:</span>
|
||||
<span class="token key atrule">min</span><span class="token punctuation">:</span> <span class="token number">99</span>
|
||||
<span class="token key atrule">interval</span><span class="token punctuation">:</span> 1m
|
||||
<span class="token punctuation">-</span> <span class="token key atrule">name</span><span class="token punctuation">:</span> request<span class="token punctuation">-</span>duration
|
||||
<span class="token comment"># maximum req duration P99</span>
|
||||
<span class="token comment"># milliseconds</span>
|
||||
<span class="token key atrule">thresholdRange</span><span class="token punctuation">:</span>
|
||||
<span class="token key atrule">max</span><span class="token punctuation">:</span> <span class="token number">500</span>
|
||||
<span class="token key atrule">interval</span><span class="token punctuation">:</span> 30s
|
||||
<span class="token comment"># generate traffic during analysis</span>
|
||||
<span class="token key atrule">webhooks</span><span class="token punctuation">:</span>
|
||||
<span class="token punctuation">-</span> <span class="token key atrule">name</span><span class="token punctuation">:</span> load<span class="token punctuation">-</span>test
|
||||
<span class="token key atrule">url</span><span class="token punctuation">:</span> http<span class="token punctuation">:</span>//flagger<span class="token punctuation">-</span>loadtester.test/
|
||||
<span class="token key atrule">timeout</span><span class="token punctuation">:</span> 5s
|
||||
<span class="token key atrule">metadata</span><span class="token punctuation">:</span>
|
||||
<span class="token key atrule">cmd</span><span class="token punctuation">:</span> <span class="token string">"hey -z 1m -q 10 -c 2 http://podinfo.test:9898/"</span>
|
||||
</code></pre></div><p>Save the above resource as podinfo-canary.yaml and then apply it:</p> <div class="language-bash extra-class"><pre class="language-bash"><code>kubectl apply -f ./podinfo-canary.yaml
|
||||
</code></pre></div><p>After a couple of seconds Flagger will create the canary objects:</p> <div class="language-bash extra-class"><pre class="language-bash"><code><span class="token comment"># applied </span>
|
||||
deployment.apps/podinfo
|
||||
horizontalpodautoscaler.autoscaling/podinfo
|
||||
canary.flagger.app/podinfo
|
||||
|
||||
<span class="token comment"># generated </span>
|
||||
deployment.apps/podinfo-primary
|
||||
horizontalpodautoscaler.autoscaling/podinfo-primary
|
||||
service/podinfo
|
||||
service/podinfo-canary
|
||||
service/podinfo-primary
|
||||
trafficsplits.split.smi-spec.io/podinfo
|
||||
</code></pre></div><h2 id="automated-canary-promotion"><a href="#automated-canary-promotion" aria-hidden="true" class="header-anchor">#</a> Automated canary promotion</h2> <p>Flagger implements a control loop that gradually shifts traffic to the canary while measuring key performance indicators like HTTP requests success rate, requests average duration and pod health. Based on analysis of the KPIs a canary is promoted or aborted, and the analysis result is published to Slack.</p> <p><img src="https://raw.githubusercontent.com/weaveworks/flagger/master/docs/diagrams/flagger-canary-steps.png" alt="Flagger Canary Stages"></p> <p>Trigger a canary deployment by updating the container image:</p> <div class="language-bash extra-class"><pre class="language-bash"><code>kubectl -n <span class="token builtin class-name">test</span> <span class="token builtin class-name">set</span> image deployment/podinfo <span class="token punctuation">\</span>
|
||||
<span class="token assign-left variable">podinfod</span><span class="token operator">=</span>quay.io/stefanprodan/podinfo:3.1.1
|
||||
</code></pre></div><p>Flagger detects that the deployment revision changed and starts a new rollout:</p> <div class="language-text extra-class"><pre class="language-text"><code>kubectl -n istio-system logs deployment/flagger -f | jq .msg
|
||||
|
||||
|
||||
New revision detected podinfo.test
|
||||
Scaling up podinfo.test
|
||||
Waiting for podinfo.test rollout to finish: 0 of 1 updated replicas are available
|
||||
Advance podinfo.test canary weight 5
|
||||
Advance podinfo.test canary weight 10
|
||||
Advance podinfo.test canary weight 15
|
||||
Advance podinfo.test canary weight 20
|
||||
Advance podinfo.test canary weight 25
|
||||
Advance podinfo.test canary weight 30
|
||||
Advance podinfo.test canary weight 35
|
||||
Advance podinfo.test canary weight 40
|
||||
Advance podinfo.test canary weight 45
|
||||
Advance podinfo.test canary weight 50
|
||||
Copying podinfo.test template spec to podinfo-primary.test
|
||||
Waiting for podinfo-primary.test rollout to finish: 1 of 2 updated replicas are available
|
||||
Promotion completed! Scaling down podinfo.test
|
||||
</code></pre></div><p><strong>Note</strong> that if you apply new changes to the deployment during the canary analysis, Flagger will restart the analysis.</p> <p>During the analysis the canary’s progress can be monitored with Grafana. The Istio dashboard URL is <a href="http://localhost:3000/d/flagger-istio/istio-canary?refresh=10s&orgId=1&var-namespace=test&var-primary=podinfo-primary&var-canary=podinfo" target="_blank" rel="noopener noreferrer">http://localhost:3000/d/flagger-istio/istio-canary?refresh=10s&orgId=1&var-namespace=test&var-primary=podinfo-primary&var-canary=podinfo<svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg></a></p> <p>You can monitor all canaries with:</p> <div class="language-bash extra-class"><pre class="language-bash"><code><span class="token function">watch</span> kubectl get canaries --all-namespaces
|
||||
|
||||
NAMESPACE NAME STATUS WEIGHT LASTTRANSITIONTIME
|
||||
<span class="token builtin class-name">test</span> podinfo Progressing <span class="token number">15</span> <span class="token number">2019</span>-05-16T14:05:07Z
|
||||
prod frontend Succeeded <span class="token number">0</span> <span class="token number">2019</span>-05-15T16:15:07Z
|
||||
prod backend Failed <span class="token number">0</span> <span class="token number">2019</span>-05-14T17:05:07Z
|
||||
</code></pre></div><h2 id="automated-rollback"><a href="#automated-rollback" aria-hidden="true" class="header-anchor">#</a> Automated rollback</h2> <p>During the canary analysis you can generate HTTP 500 errors and high latency to test if Flagger pauses the rollout.</p> <p>Create a tester pod and exec into it:</p> <div class="language-bash extra-class"><pre class="language-bash"><code>kubectl -n <span class="token builtin class-name">test</span> run tester <span class="token punctuation">\</span>
|
||||
--image<span class="token operator">=</span>quay.io/stefanprodan/podinfo:3.1.2 <span class="token punctuation">\</span>
|
||||
-- ./podinfo --port<span class="token operator">=</span><span class="token number">9898</span>
|
||||
|
||||
kubectl -n <span class="token builtin class-name">test</span> <span class="token builtin class-name">exec</span> -it tester-xx-xx <span class="token function">sh</span>
|
||||
</code></pre></div><p>Generate HTTP 500 errors:</p> <div class="language-bash extra-class"><pre class="language-bash"><code><span class="token function">watch</span> <span class="token function">curl</span> http://podinfo-canary:9898/status/500
|
||||
</code></pre></div><p>Generate latency:</p> <div class="language-bash extra-class"><pre class="language-bash"><code><span class="token function">watch</span> <span class="token function">curl</span> http://podinfo-canary:9898/delay/1
|
||||
</code></pre></div><p>When the number of failed checks reaches the canary analysis threshold, the traffic is routed back to the primary, the canary is scaled to zero and the rollout is marked as failed.</p> <div class="language-text extra-class"><pre class="language-text"><code>kubectl -n test describe canary/podinfo
|
||||
|
||||
Status:
|
||||
Canary Weight: 0
|
||||
Failed Checks: 10
|
||||
Phase: Failed
|
||||
Events:
|
||||
Type Reason Age From Message
|
||||
---- ------ ---- ---- -------
|
||||
Normal Synced 3m flagger Starting canary deployment for podinfo.test
|
||||
Normal Synced 3m flagger Advance podinfo.test canary weight 5
|
||||
Normal Synced 3m flagger Advance podinfo.test canary weight 10
|
||||
Normal Synced 3m flagger Advance podinfo.test canary weight 15
|
||||
Normal Synced 3m flagger Halt podinfo.test advancement success rate 69.17% < 99%
|
||||
Normal Synced 2m flagger Halt podinfo.test advancement success rate 61.39% < 99%
|
||||
Normal Synced 2m flagger Halt podinfo.test advancement success rate 55.06% < 99%
|
||||
Normal Synced 2m flagger Halt podinfo.test advancement success rate 47.00% < 99%
|
||||
Normal Synced 2m flagger (combined from similar events): Halt podinfo.test advancement success rate 38.08% < 99%
|
||||
Warning Synced 1m flagger Rolling back podinfo.test failed checks threshold reached 10
|
||||
Warning Synced 1m flagger Canary failed! Scaling down podinfo.test
|
||||
</code></pre></div></div> <footer class="page-edit"><!----> <!----></footer> <!----> </main></div><div class="global-ui"></div></div>
|
||||
<script src="/assets/js/app.b4b27991.js" defer></script><script src="/assets/js/2.ba5bf77c.js" defer></script><script src="/assets/js/21.df66561d.js" defer></script>
|
||||
</body>
|
||||
</html>
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,35 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en-US">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<title>Flagger</title>
|
||||
<meta name="description" content="Progressive Delivery operator for Kubernetes (Canary, A/B Testing and Blue/Green deployments)">
|
||||
<link rel="icon" href="/favicon.png">
|
||||
<link rel="stylesheet" href="/website.css">
|
||||
<meta name="keywords" content="gitops kubernetes flagger istio linkerd appmesh contour gloo nginx">
|
||||
<meta name="twitter:card" content="summary_large_image">
|
||||
<meta name="twitter:title" content="Flagger">
|
||||
<meta name="twitter:description" content="Progressive delivery Kubernetes operator (Canary, A/B Testing and Blue/Green deployments)">
|
||||
<meta name="twitter:image:src" content="https://flagger.app/flagger-overview.png">
|
||||
|
||||
<link rel="preload" href="/assets/css/0.styles.a0ada997.css" as="style"><link rel="preload" href="/assets/js/app.b4b27991.js" as="script"><link rel="preload" href="/assets/js/2.ba5bf77c.js" as="script"><link rel="preload" href="/assets/js/16.3012a10b.js" as="script"><link rel="prefetch" href="/assets/js/10.bf6f38d4.js"><link rel="prefetch" href="/assets/js/11.3af38cfe.js"><link rel="prefetch" href="/assets/js/12.eca4b89c.js"><link rel="prefetch" href="/assets/js/13.9104349a.js"><link rel="prefetch" href="/assets/js/14.a4f3d810.js"><link rel="prefetch" href="/assets/js/15.aada2caa.js"><link rel="prefetch" href="/assets/js/17.a98b6aa9.js"><link rel="prefetch" href="/assets/js/18.e5188926.js"><link rel="prefetch" href="/assets/js/19.bb9498f7.js"><link rel="prefetch" href="/assets/js/20.dce5a04e.js"><link rel="prefetch" href="/assets/js/21.df66561d.js"><link rel="prefetch" href="/assets/js/22.1c5d14b9.js"><link rel="prefetch" href="/assets/js/23.b3dcc355.js"><link rel="prefetch" href="/assets/js/24.4f54d289.js"><link rel="prefetch" href="/assets/js/25.914acd1d.js"><link rel="prefetch" href="/assets/js/26.b2d4f402.js"><link rel="prefetch" href="/assets/js/27.ea9f1f9c.js"><link rel="prefetch" href="/assets/js/28.deae273d.js"><link rel="prefetch" href="/assets/js/29.17c547e6.js"><link rel="prefetch" href="/assets/js/3.8b254d5d.js"><link rel="prefetch" href="/assets/js/30.38bd809e.js"><link rel="prefetch" href="/assets/js/31.fbfb336c.js"><link rel="prefetch" href="/assets/js/32.591c3f2b.js"><link rel="prefetch" href="/assets/js/33.c456709f.js"><link rel="prefetch" href="/assets/js/34.5a071e36.js"><link rel="prefetch" href="/assets/js/35.fc27c09e.js"><link rel="prefetch" href="/assets/js/36.144098aa.js"><link rel="prefetch" href="/assets/js/4.26a3069c.js"><link rel="prefetch" href="/assets/js/5.f92a6aca.js"><link rel="prefetch" href="/assets/js/6.ee5ffca3.js"><link rel="prefetch" href="/assets/js/7.6bf27a87.js"><link rel="prefetch" href="/assets/js/8.8b72a8d3.js"><link rel="prefetch" href="/assets/js/9.7919fc1d.js">
|
||||
<link rel="stylesheet" href="/assets/css/0.styles.a0ada997.css">
|
||||
</head>
|
||||
<body>
|
||||
<div id="app" data-server-rendered="true"><div class="theme-container"><header class="navbar"><div class="sidebar-button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" role="img" viewBox="0 0 448 512" class="icon"><path fill="currentColor" d="M436 124H12c-6.627 0-12-5.373-12-12V80c0-6.627 5.373-12 12-12h424c6.627 0 12 5.373 12 12v32c0 6.627-5.373 12-12 12zm0 160H12c-6.627 0-12-5.373-12-12v-32c0-6.627 5.373-12 12-12h424c6.627 0 12 5.373 12 12v32c0 6.627-5.373 12-12 12zm0 160H12c-6.627 0-12-5.373-12-12v-32c0-6.627 5.373-12 12-12h424c6.627 0 12 5.373 12 12v32c0 6.627-5.373 12-12 12z"></path></svg></div> <a href="/" class="home-link router-link-active"><!----> <span class="site-name">Flagger</span></a> <div class="links"><div class="search-box"><input aria-label="Search" autocomplete="off" spellcheck="false" value=""> <!----></div> <nav class="nav-links can-hide"><div class="nav-item"><a href="https://docs.flagger.app" target="_blank" rel="noopener noreferrer" class="nav-link external">
|
||||
Docs
|
||||
<svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg></a></div><div class="nav-item"><a href="https://github.com/weaveworks/flagger/blob/master/CHANGELOG.md" target="_blank" rel="noopener noreferrer" class="nav-link external">
|
||||
Changelog
|
||||
<svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg></a></div> <a href="https://github.com/weaveworks/flagger" target="_blank" rel="noopener noreferrer" class="repo-link">
|
||||
GitHub
|
||||
<svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg></a></nav></div></header> <div class="sidebar-mask"></div> <aside class="sidebar"><nav class="nav-links"><div class="nav-item"><a href="https://docs.flagger.app" target="_blank" rel="noopener noreferrer" class="nav-link external">
|
||||
Docs
|
||||
<svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg></a></div><div class="nav-item"><a href="https://github.com/weaveworks/flagger/blob/master/CHANGELOG.md" target="_blank" rel="noopener noreferrer" class="nav-link external">
|
||||
Changelog
|
||||
<svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg></a></div> <a href="https://github.com/weaveworks/flagger" target="_blank" rel="noopener noreferrer" class="repo-link">
|
||||
GitHub
|
||||
<svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg></a></nav> <ul class="sidebar-links"><li><a href="/" class="sidebar-link">Home</a></li><li><section class="sidebar-group depth-0"><a href="/intro/" class="sidebar-heading clickable"><span>Introduction</span> <!----></a> <ul class="sidebar-links sidebar-group-items"><li><a href="/intro/" class="sidebar-link">Get Started</a></li><li><a href="/intro/faq.html" class="sidebar-link">FAQ</a></li></ul></section></li><li><section class="sidebar-group depth-0"><a href="/install/flagger-install-on-kubernetes" class="sidebar-heading clickable"><span>Install</span> <!----></a> <ul class="sidebar-links sidebar-group-items"><li><a href="/install/flagger-install-on-kubernetes.html" class="sidebar-link">On Kubernetes</a></li><li><a href="/install/flagger-install-on-google-cloud.html" class="sidebar-link">On GKE Istio</a></li><li><a href="/install/flagger-install-on-eks-appmesh.html" class="sidebar-link">On EKS App Mesh</a></li></ul></section></li><li><section class="sidebar-group depth-0"><a href="/usage/how-it-works" class="sidebar-heading clickable"><span>Usage</span> <!----></a> <ul class="sidebar-links sidebar-group-items"><li><a href="/usage/how-it-works.html" class="sidebar-link">How it works</a></li><li><a href="/usage/deployment-strategies.html" class="sidebar-link">Deployment Strategies</a></li><li><a href="/usage/metrics.html" class="sidebar-link">Metrics Analysis</a></li><li><a href="/usage/webhooks.html" class="sidebar-link">Webhooks</a></li><li><a href="/usage/alerting.html" class="sidebar-link">Alerting</a></li><li><a href="/usage/monitoring.html" class="sidebar-link">Monitoring</a></li></ul></section></li><li><section class="sidebar-group depth-0"><a href="/tutorials/istio-progressive-delivery" class="sidebar-heading clickable"><span>Tutorials</span> <!----></a> <ul class="sidebar-links sidebar-group-items"><li><a href="/tutorials/istio-progressive-delivery.html" class="sidebar-link">Istio Canaries</a></li><li><a href="/tutorials/istio-ab-testing.html" class="sidebar-link">Istio A/B Testing</a></li><li><a href="/tutorials/linkerd-progressive-delivery.html" class="sidebar-link">Linkerd Canaries</a></li><li><a href="/tutorials/appmesh-progressive-delivery.html" class="sidebar-link">App Mesh Canaries</a></li><li><a href="/tutorials/nginx-progressive-delivery.html" class="sidebar-link">NGINX Ingress Canaries</a></li><li><a href="/tutorials/gloo-progressive-delivery.html" class="sidebar-link">Gloo Canaries</a></li><li><a href="/tutorials/contour-progressive-delivery.html" class="sidebar-link">Contour Canaries</a></li><li><a href="/tutorials/kubernetes-blue-green.html" class="sidebar-link">Kubernetes Blue/Green</a></li><li><a href="/tutorials/canary-helm-gitops.html" class="sidebar-link">Canaries with Helm charts and GitOps</a></li></ul></section></li><li><section class="sidebar-group depth-0"><a href="/dev/dev-guide" class="sidebar-heading clickable"><span>Dev</span> <!----></a> <ul class="sidebar-links sidebar-group-items"><li><a href="/dev/dev-guide.html" class="sidebar-link">Development Guide</a></li><li><a href="/dev/release-guide.html" class="sidebar-link">Release Guide</a></li><li><a href="/dev/upgrade-guide.html" class="sidebar-link">Upgrade Guide</a></li></ul></section></li></ul> </aside> <main class="page"> <div class="theme-default-content content__default"></div> <footer class="page-edit"><!----> <!----></footer> <!----> </main></div><div class="global-ui"></div></div>
|
||||
<script src="/assets/js/app.b4b27991.js" defer></script><script src="/assets/js/2.ba5bf77c.js" defer></script><script src="/assets/js/16.3012a10b.js" defer></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,203 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en-US">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<title>Istio A/B Testing | Flagger</title>
|
||||
<meta name="description" content="Progressive Delivery operator for Kubernetes (Canary, A/B Testing and Blue/Green deployments)">
|
||||
<link rel="icon" href="/favicon.png">
|
||||
<link rel="stylesheet" href="/website.css">
|
||||
<meta name="keywords" content="gitops kubernetes flagger istio linkerd appmesh contour gloo nginx">
|
||||
<meta name="twitter:card" content="summary_large_image">
|
||||
<meta name="twitter:title" content="Flagger">
|
||||
<meta name="twitter:description" content="Progressive delivery Kubernetes operator (Canary, A/B Testing and Blue/Green deployments)">
|
||||
<meta name="twitter:image:src" content="https://flagger.app/flagger-overview.png">
|
||||
|
||||
<link rel="preload" href="/assets/css/0.styles.a0ada997.css" as="style"><link rel="preload" href="/assets/js/app.b4b27991.js" as="script"><link rel="preload" href="/assets/js/2.ba5bf77c.js" as="script"><link rel="preload" href="/assets/js/23.b3dcc355.js" as="script"><link rel="prefetch" href="/assets/js/10.bf6f38d4.js"><link rel="prefetch" href="/assets/js/11.3af38cfe.js"><link rel="prefetch" href="/assets/js/12.eca4b89c.js"><link rel="prefetch" href="/assets/js/13.9104349a.js"><link rel="prefetch" href="/assets/js/14.a4f3d810.js"><link rel="prefetch" href="/assets/js/15.aada2caa.js"><link rel="prefetch" href="/assets/js/16.3012a10b.js"><link rel="prefetch" href="/assets/js/17.a98b6aa9.js"><link rel="prefetch" href="/assets/js/18.e5188926.js"><link rel="prefetch" href="/assets/js/19.bb9498f7.js"><link rel="prefetch" href="/assets/js/20.dce5a04e.js"><link rel="prefetch" href="/assets/js/21.df66561d.js"><link rel="prefetch" href="/assets/js/22.1c5d14b9.js"><link rel="prefetch" href="/assets/js/24.4f54d289.js"><link rel="prefetch" href="/assets/js/25.914acd1d.js"><link rel="prefetch" href="/assets/js/26.b2d4f402.js"><link rel="prefetch" href="/assets/js/27.ea9f1f9c.js"><link rel="prefetch" href="/assets/js/28.deae273d.js"><link rel="prefetch" href="/assets/js/29.17c547e6.js"><link rel="prefetch" href="/assets/js/3.8b254d5d.js"><link rel="prefetch" href="/assets/js/30.38bd809e.js"><link rel="prefetch" href="/assets/js/31.fbfb336c.js"><link rel="prefetch" href="/assets/js/32.591c3f2b.js"><link rel="prefetch" href="/assets/js/33.c456709f.js"><link rel="prefetch" href="/assets/js/34.5a071e36.js"><link rel="prefetch" href="/assets/js/35.fc27c09e.js"><link rel="prefetch" href="/assets/js/36.144098aa.js"><link rel="prefetch" href="/assets/js/4.26a3069c.js"><link rel="prefetch" href="/assets/js/5.f92a6aca.js"><link rel="prefetch" href="/assets/js/6.ee5ffca3.js"><link rel="prefetch" href="/assets/js/7.6bf27a87.js"><link rel="prefetch" href="/assets/js/8.8b72a8d3.js"><link rel="prefetch" href="/assets/js/9.7919fc1d.js">
|
||||
<link rel="stylesheet" href="/assets/css/0.styles.a0ada997.css">
|
||||
</head>
|
||||
<body>
|
||||
<div id="app" data-server-rendered="true"><div class="theme-container"><header class="navbar"><div class="sidebar-button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" role="img" viewBox="0 0 448 512" class="icon"><path fill="currentColor" d="M436 124H12c-6.627 0-12-5.373-12-12V80c0-6.627 5.373-12 12-12h424c6.627 0 12 5.373 12 12v32c0 6.627-5.373 12-12 12zm0 160H12c-6.627 0-12-5.373-12-12v-32c0-6.627 5.373-12 12-12h424c6.627 0 12 5.373 12 12v32c0 6.627-5.373 12-12 12zm0 160H12c-6.627 0-12-5.373-12-12v-32c0-6.627 5.373-12 12-12h424c6.627 0 12 5.373 12 12v32c0 6.627-5.373 12-12 12z"></path></svg></div> <a href="/" class="home-link router-link-active"><!----> <span class="site-name">Flagger</span></a> <div class="links"><div class="search-box"><input aria-label="Search" autocomplete="off" spellcheck="false" value=""> <!----></div> <nav class="nav-links can-hide"><div class="nav-item"><a href="https://docs.flagger.app" target="_blank" rel="noopener noreferrer" class="nav-link external">
|
||||
Docs
|
||||
<svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg></a></div><div class="nav-item"><a href="https://github.com/weaveworks/flagger/blob/master/CHANGELOG.md" target="_blank" rel="noopener noreferrer" class="nav-link external">
|
||||
Changelog
|
||||
<svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg></a></div> <a href="https://github.com/weaveworks/flagger" target="_blank" rel="noopener noreferrer" class="repo-link">
|
||||
GitHub
|
||||
<svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg></a></nav></div></header> <div class="sidebar-mask"></div> <aside class="sidebar"><nav class="nav-links"><div class="nav-item"><a href="https://docs.flagger.app" target="_blank" rel="noopener noreferrer" class="nav-link external">
|
||||
Docs
|
||||
<svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg></a></div><div class="nav-item"><a href="https://github.com/weaveworks/flagger/blob/master/CHANGELOG.md" target="_blank" rel="noopener noreferrer" class="nav-link external">
|
||||
Changelog
|
||||
<svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg></a></div> <a href="https://github.com/weaveworks/flagger" target="_blank" rel="noopener noreferrer" class="repo-link">
|
||||
GitHub
|
||||
<svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg></a></nav> <ul class="sidebar-links"><li><a href="/" class="sidebar-link">Home</a></li><li><section class="sidebar-group depth-0"><a href="/intro/" class="sidebar-heading clickable"><span>Introduction</span> <!----></a> <ul class="sidebar-links sidebar-group-items"><li><a href="/intro/" class="sidebar-link">Get Started</a></li><li><a href="/intro/faq.html" class="sidebar-link">FAQ</a></li></ul></section></li><li><section class="sidebar-group depth-0"><a href="/install/flagger-install-on-kubernetes" class="sidebar-heading clickable"><span>Install</span> <!----></a> <ul class="sidebar-links sidebar-group-items"><li><a href="/install/flagger-install-on-kubernetes.html" class="sidebar-link">On Kubernetes</a></li><li><a href="/install/flagger-install-on-google-cloud.html" class="sidebar-link">On GKE Istio</a></li><li><a href="/install/flagger-install-on-eks-appmesh.html" class="sidebar-link">On EKS App Mesh</a></li></ul></section></li><li><section class="sidebar-group depth-0"><a href="/usage/how-it-works" class="sidebar-heading clickable"><span>Usage</span> <!----></a> <ul class="sidebar-links sidebar-group-items"><li><a href="/usage/how-it-works.html" class="sidebar-link">How it works</a></li><li><a href="/usage/deployment-strategies.html" class="sidebar-link">Deployment Strategies</a></li><li><a href="/usage/metrics.html" class="sidebar-link">Metrics Analysis</a></li><li><a href="/usage/webhooks.html" class="sidebar-link">Webhooks</a></li><li><a href="/usage/alerting.html" class="sidebar-link">Alerting</a></li><li><a href="/usage/monitoring.html" class="sidebar-link">Monitoring</a></li></ul></section></li><li><section class="sidebar-group depth-0"><a href="/tutorials/istio-progressive-delivery" class="sidebar-heading clickable open"><span>Tutorials</span> <!----></a> <ul class="sidebar-links sidebar-group-items"><li><a href="/tutorials/istio-progressive-delivery.html" class="sidebar-link">Istio Canaries</a></li><li><a href="/tutorials/istio-ab-testing.html" class="active sidebar-link">Istio A/B Testing</a><ul class="sidebar-sub-headers"><li class="sidebar-sub-header"><a href="/tutorials/istio-ab-testing.html#prerequisites" class="sidebar-link">Prerequisites</a></li><li class="sidebar-sub-header"><a href="/tutorials/istio-ab-testing.html#bootstrap" class="sidebar-link">Bootstrap</a></li><li class="sidebar-sub-header"><a href="/tutorials/istio-ab-testing.html#automated-canary-promotion" class="sidebar-link">Automated canary promotion</a></li><li class="sidebar-sub-header"><a href="/tutorials/istio-ab-testing.html#automated-rollback" class="sidebar-link">Automated rollback</a></li></ul></li><li><a href="/tutorials/linkerd-progressive-delivery.html" class="sidebar-link">Linkerd Canaries</a></li><li><a href="/tutorials/appmesh-progressive-delivery.html" class="sidebar-link">App Mesh Canaries</a></li><li><a href="/tutorials/nginx-progressive-delivery.html" class="sidebar-link">NGINX Ingress Canaries</a></li><li><a href="/tutorials/gloo-progressive-delivery.html" class="sidebar-link">Gloo Canaries</a></li><li><a href="/tutorials/contour-progressive-delivery.html" class="sidebar-link">Contour Canaries</a></li><li><a href="/tutorials/kubernetes-blue-green.html" class="sidebar-link">Kubernetes Blue/Green</a></li><li><a href="/tutorials/canary-helm-gitops.html" class="sidebar-link">Canaries with Helm charts and GitOps</a></li></ul></section></li><li><section class="sidebar-group depth-0"><a href="/dev/dev-guide" class="sidebar-heading clickable"><span>Dev</span> <!----></a> <ul class="sidebar-links sidebar-group-items"><li><a href="/dev/dev-guide.html" class="sidebar-link">Development Guide</a></li><li><a href="/dev/release-guide.html" class="sidebar-link">Release Guide</a></li><li><a href="/dev/upgrade-guide.html" class="sidebar-link">Upgrade Guide</a></li></ul></section></li></ul> </aside> <main class="page"> <div class="theme-default-content content__default"><h1 id="istio-a-b-testing"><a href="#istio-a-b-testing" aria-hidden="true" class="header-anchor">#</a> Istio A/B Testing</h1> <p>This guide shows you how to automate A/B testing with Istio and Flagger.</p> <p>Besides weighted routing, Flagger can be configured to route traffic to the canary based on HTTP match conditions.
|
||||
In an A/B testing scenario, you'll be using HTTP headers or cookies to target a certain segment of your users.
|
||||
This is particularly useful for frontend applications that require session affinity.</p> <p><img src="https://raw.githubusercontent.com/weaveworks/flagger/master/docs/diagrams/flagger-abtest-steps.png" alt="Flagger A/B Testing Stages"></p> <h2 id="prerequisites"><a href="#prerequisites" aria-hidden="true" class="header-anchor">#</a> Prerequisites</h2> <p>Flagger requires a Kubernetes cluster <strong>v1.11</strong> or newer and Istio <strong>v1.0</strong> or newer.</p> <p>Install Istio with telemetry support and Prometheus:</p> <div class="language-bash extra-class"><pre class="language-bash"><code>istioctl manifest apply --set <span class="token assign-left variable">profile</span><span class="token operator">=</span>default
|
||||
</code></pre></div><p>Install Flagger using Kustomize (kubectl 1.14) in the <code>istio-system</code> namespace:</p> <div class="language-bash extra-class"><pre class="language-bash"><code>kubectl apply -k github.com/weaveworks/flagger//kustomize/istio
|
||||
</code></pre></div><p>Create an ingress gateway to expose the demo app outside of the mesh:</p> <div class="language-yaml extra-class"><pre class="language-yaml"><code><span class="token key atrule">apiVersion</span><span class="token punctuation">:</span> networking.istio.io/v1alpha3
|
||||
<span class="token key atrule">kind</span><span class="token punctuation">:</span> Gateway
|
||||
<span class="token key atrule">metadata</span><span class="token punctuation">:</span>
|
||||
<span class="token key atrule">name</span><span class="token punctuation">:</span> public<span class="token punctuation">-</span>gateway
|
||||
<span class="token key atrule">namespace</span><span class="token punctuation">:</span> istio<span class="token punctuation">-</span>system
|
||||
<span class="token key atrule">spec</span><span class="token punctuation">:</span>
|
||||
<span class="token key atrule">selector</span><span class="token punctuation">:</span>
|
||||
<span class="token key atrule">istio</span><span class="token punctuation">:</span> ingressgateway
|
||||
<span class="token key atrule">servers</span><span class="token punctuation">:</span>
|
||||
<span class="token punctuation">-</span> <span class="token key atrule">port</span><span class="token punctuation">:</span>
|
||||
<span class="token key atrule">number</span><span class="token punctuation">:</span> <span class="token number">80</span>
|
||||
<span class="token key atrule">name</span><span class="token punctuation">:</span> http
|
||||
<span class="token key atrule">protocol</span><span class="token punctuation">:</span> HTTP
|
||||
<span class="token key atrule">hosts</span><span class="token punctuation">:</span>
|
||||
<span class="token punctuation">-</span> <span class="token string">"*"</span>
|
||||
</code></pre></div><h2 id="bootstrap"><a href="#bootstrap" aria-hidden="true" class="header-anchor">#</a> Bootstrap</h2> <p>Create a test namespace with Istio sidecar injection enabled:</p> <div class="language-bash extra-class"><pre class="language-bash"><code>kubectl create ns <span class="token builtin class-name">test</span>
|
||||
kubectl label namespace <span class="token builtin class-name">test</span> istio-injection<span class="token operator">=</span>enabled
|
||||
</code></pre></div><p>Create a deployment and a horizontal pod autoscaler:</p> <div class="language-bash extra-class"><pre class="language-bash"><code>kubectl apply -k github.com/weaveworks/flagger//kustomize/podinfo
|
||||
</code></pre></div><p>Deploy the load testing service to generate traffic during the canary analysis:</p> <div class="language-bash extra-class"><pre class="language-bash"><code>kubectl apply -k github.com/weaveworks/flagger//kustomize/tester
|
||||
</code></pre></div><p>Create a canary custom resource (replace example.com with your own domain):</p> <div class="language-yaml extra-class"><pre class="language-yaml"><code><span class="token key atrule">apiVersion</span><span class="token punctuation">:</span> flagger.app/v1beta1
|
||||
<span class="token key atrule">kind</span><span class="token punctuation">:</span> Canary
|
||||
<span class="token key atrule">metadata</span><span class="token punctuation">:</span>
|
||||
<span class="token key atrule">name</span><span class="token punctuation">:</span> podinfo
|
||||
<span class="token key atrule">namespace</span><span class="token punctuation">:</span> test
|
||||
<span class="token key atrule">spec</span><span class="token punctuation">:</span>
|
||||
<span class="token comment"># deployment reference</span>
|
||||
<span class="token key atrule">targetRef</span><span class="token punctuation">:</span>
|
||||
<span class="token key atrule">apiVersion</span><span class="token punctuation">:</span> apps/v1
|
||||
<span class="token key atrule">kind</span><span class="token punctuation">:</span> Deployment
|
||||
<span class="token key atrule">name</span><span class="token punctuation">:</span> podinfo
|
||||
<span class="token comment"># the maximum time in seconds for the canary deployment</span>
|
||||
<span class="token comment"># to make progress before it is rollback (default 600s)</span>
|
||||
<span class="token key atrule">progressDeadlineSeconds</span><span class="token punctuation">:</span> <span class="token number">60</span>
|
||||
<span class="token comment"># HPA reference (optional)</span>
|
||||
<span class="token key atrule">autoscalerRef</span><span class="token punctuation">:</span>
|
||||
<span class="token key atrule">apiVersion</span><span class="token punctuation">:</span> autoscaling/v2beta1
|
||||
<span class="token key atrule">kind</span><span class="token punctuation">:</span> HorizontalPodAutoscaler
|
||||
<span class="token key atrule">name</span><span class="token punctuation">:</span> podinfo
|
||||
<span class="token key atrule">service</span><span class="token punctuation">:</span>
|
||||
<span class="token comment"># container port</span>
|
||||
<span class="token key atrule">port</span><span class="token punctuation">:</span> <span class="token number">9898</span>
|
||||
<span class="token comment"># Istio gateways (optional)</span>
|
||||
<span class="token key atrule">gateways</span><span class="token punctuation">:</span>
|
||||
<span class="token punctuation">-</span> public<span class="token punctuation">-</span>gateway.istio<span class="token punctuation">-</span>system.svc.cluster.local
|
||||
<span class="token comment"># Istio virtual service host names (optional)</span>
|
||||
<span class="token key atrule">hosts</span><span class="token punctuation">:</span>
|
||||
<span class="token punctuation">-</span> app.example.com
|
||||
<span class="token comment"># Istio traffic policy (optional)</span>
|
||||
<span class="token key atrule">trafficPolicy</span><span class="token punctuation">:</span>
|
||||
<span class="token key atrule">tls</span><span class="token punctuation">:</span>
|
||||
<span class="token comment"># use ISTIO_MUTUAL when mTLS is enabled</span>
|
||||
<span class="token key atrule">mode</span><span class="token punctuation">:</span> DISABLE
|
||||
<span class="token key atrule">analysis</span><span class="token punctuation">:</span>
|
||||
<span class="token comment"># schedule interval (default 60s)</span>
|
||||
<span class="token key atrule">interval</span><span class="token punctuation">:</span> 1m
|
||||
<span class="token comment"># total number of iterations</span>
|
||||
<span class="token key atrule">iterations</span><span class="token punctuation">:</span> <span class="token number">10</span>
|
||||
<span class="token comment"># max number of failed iterations before rollback</span>
|
||||
<span class="token key atrule">threshold</span><span class="token punctuation">:</span> <span class="token number">2</span>
|
||||
<span class="token comment"># canary match condition</span>
|
||||
<span class="token key atrule">match</span><span class="token punctuation">:</span>
|
||||
<span class="token punctuation">-</span> <span class="token key atrule">headers</span><span class="token punctuation">:</span>
|
||||
<span class="token key atrule">user-agent</span><span class="token punctuation">:</span>
|
||||
<span class="token key atrule">regex</span><span class="token punctuation">:</span> <span class="token string">"^(?!.*Chrome).*Safari.*"</span>
|
||||
<span class="token punctuation">-</span> <span class="token key atrule">headers</span><span class="token punctuation">:</span>
|
||||
<span class="token key atrule">cookie</span><span class="token punctuation">:</span>
|
||||
<span class="token key atrule">regex</span><span class="token punctuation">:</span> <span class="token string">"^(.*?;)?(type=insider)(;.*)?$"</span>
|
||||
<span class="token key atrule">metrics</span><span class="token punctuation">:</span>
|
||||
<span class="token punctuation">-</span> <span class="token key atrule">name</span><span class="token punctuation">:</span> request<span class="token punctuation">-</span>success<span class="token punctuation">-</span>rate
|
||||
<span class="token comment"># minimum req success rate (non 5xx responses)</span>
|
||||
<span class="token comment"># percentage (0-100)</span>
|
||||
<span class="token key atrule">thresholdRange</span><span class="token punctuation">:</span>
|
||||
<span class="token key atrule">min</span><span class="token punctuation">:</span> <span class="token number">99</span>
|
||||
<span class="token key atrule">interval</span><span class="token punctuation">:</span> 1m
|
||||
<span class="token punctuation">-</span> <span class="token key atrule">name</span><span class="token punctuation">:</span> request<span class="token punctuation">-</span>duration
|
||||
<span class="token comment"># maximum req duration P99</span>
|
||||
<span class="token comment"># milliseconds</span>
|
||||
<span class="token key atrule">thresholdRange</span><span class="token punctuation">:</span>
|
||||
<span class="token key atrule">max</span><span class="token punctuation">:</span> <span class="token number">500</span>
|
||||
<span class="token key atrule">interval</span><span class="token punctuation">:</span> 30s
|
||||
<span class="token comment"># generate traffic during analysis</span>
|
||||
<span class="token key atrule">webhooks</span><span class="token punctuation">:</span>
|
||||
<span class="token punctuation">-</span> <span class="token key atrule">name</span><span class="token punctuation">:</span> load<span class="token punctuation">-</span>test
|
||||
<span class="token key atrule">url</span><span class="token punctuation">:</span> http<span class="token punctuation">:</span>//flagger<span class="token punctuation">-</span>loadtester.test/
|
||||
<span class="token key atrule">timeout</span><span class="token punctuation">:</span> 5s
|
||||
<span class="token key atrule">metadata</span><span class="token punctuation">:</span>
|
||||
<span class="token key atrule">cmd</span><span class="token punctuation">:</span> <span class="token string">"hey -z 1m -q 10 -c 2 -H 'Cookie: type=insider' http://podinfo.test:9898/"</span>
|
||||
</code></pre></div><p>The above configuration will run an analysis for ten minutes targeting Safari users and those that have an insider cookie.</p> <p>Save the above resource as podinfo-abtest.yaml and then apply it:</p> <div class="language-bash extra-class"><pre class="language-bash"><code>kubectl apply -f ./podinfo-abtest.yaml
|
||||
</code></pre></div><p>After a couple of seconds Flagger will create the canary objects:</p> <div class="language-bash extra-class"><pre class="language-bash"><code><span class="token comment"># applied </span>
|
||||
deployment.apps/podinfo
|
||||
horizontalpodautoscaler.autoscaling/podinfo
|
||||
canary.flagger.app/podinfo
|
||||
|
||||
<span class="token comment"># generated </span>
|
||||
deployment.apps/podinfo-primary
|
||||
horizontalpodautoscaler.autoscaling/podinfo-primary
|
||||
service/podinfo
|
||||
service/podinfo-canary
|
||||
service/podinfo-primary
|
||||
destinationrule.networking.istio.io/podinfo-canary
|
||||
destinationrule.networking.istio.io/podinfo-primary
|
||||
virtualservice.networking.istio.io/podinfo
|
||||
</code></pre></div><h2 id="automated-canary-promotion"><a href="#automated-canary-promotion" aria-hidden="true" class="header-anchor">#</a> Automated canary promotion</h2> <p>Trigger a canary deployment by updating the container image:</p> <div class="language-bash extra-class"><pre class="language-bash"><code>kubectl -n <span class="token builtin class-name">test</span> <span class="token builtin class-name">set</span> image deployment/podinfo <span class="token punctuation">\</span>
|
||||
<span class="token assign-left variable">podinfod</span><span class="token operator">=</span>stefanprodan/podinfo:3.1.1
|
||||
</code></pre></div><p>Flagger detects that the deployment revision changed and starts a new rollout:</p> <div class="language-text extra-class"><pre class="language-text"><code>kubectl -n test describe canary/abtest
|
||||
|
||||
Status:
|
||||
Failed Checks: 0
|
||||
Phase: Succeeded
|
||||
Events:
|
||||
Type Reason Age From Message
|
||||
---- ------ ---- ---- -------
|
||||
Normal Synced 3m flagger New revision detected podinfo.test
|
||||
Normal Synced 3m flagger Scaling up podinfo.test
|
||||
Warning Synced 3m flagger Waiting for podinfo.test rollout to finish: 0 of 1 updated replicas are available
|
||||
Normal Synced 3m flagger Advance podinfo.test canary iteration 1/10
|
||||
Normal Synced 3m flagger Advance podinfo.test canary iteration 2/10
|
||||
Normal Synced 3m flagger Advance podinfo.test canary iteration 3/10
|
||||
Normal Synced 2m flagger Advance podinfo.test canary iteration 4/10
|
||||
Normal Synced 2m flagger Advance podinfo.test canary iteration 5/10
|
||||
Normal Synced 1m flagger Advance podinfo.test canary iteration 6/10
|
||||
Normal Synced 1m flagger Advance podinfo.test canary iteration 7/10
|
||||
Normal Synced 55s flagger Advance podinfo.test canary iteration 8/10
|
||||
Normal Synced 45s flagger Advance podinfo.test canary iteration 9/10
|
||||
Normal Synced 35s flagger Advance podinfo.test canary iteration 10/10
|
||||
Normal Synced 25s flagger Copying podinfo.test template spec to podinfo-primary.test
|
||||
Warning Synced 15s flagger Waiting for podinfo-primary.test rollout to finish: 1 of 2 updated replicas are available
|
||||
Normal Synced 5s flagger Promotion completed! Scaling down podinfo.test
|
||||
</code></pre></div><p><strong>Note</strong> that if you apply new changes to the deployment during the canary analysis, Flagger will restart the analysis.</p> <p>You can monitor all canaries with:</p> <div class="language-bash extra-class"><pre class="language-bash"><code><span class="token function">watch</span> kubectl get canaries --all-namespaces
|
||||
|
||||
NAMESPACE NAME STATUS WEIGHT LASTTRANSITIONTIME
|
||||
<span class="token builtin class-name">test</span> podinfo Progressing <span class="token number">100</span> <span class="token number">2019</span>-03-16T14:05:07Z
|
||||
prod frontend Succeeded <span class="token number">0</span> <span class="token number">2019</span>-03-15T16:15:07Z
|
||||
prod backend Failed <span class="token number">0</span> <span class="token number">2019</span>-03-14T17:05:07Z
|
||||
</code></pre></div><h2 id="automated-rollback"><a href="#automated-rollback" aria-hidden="true" class="header-anchor">#</a> Automated rollback</h2> <p>During the canary analysis you can generate HTTP 500 errors and high latency to test Flagger's rollback.</p> <p>Generate HTTP 500 errors:</p> <div class="language-bash extra-class"><pre class="language-bash"><code><span class="token function">watch</span> <span class="token function">curl</span> -b <span class="token string">'type=insider'</span> http://app.example.com/status/500
|
||||
</code></pre></div><p>Generate latency:</p> <div class="language-bash extra-class"><pre class="language-bash"><code><span class="token function">watch</span> <span class="token function">curl</span> -b <span class="token string">'type=insider'</span> http://app.example.com/delay/1
|
||||
</code></pre></div><p>When the number of failed checks reaches the canary analysis threshold, the traffic is routed back to the primary,
|
||||
the canary is scaled to zero and the rollout is marked as failed.</p> <div class="language-text extra-class"><pre class="language-text"><code>kubectl -n test describe canary/podinfo
|
||||
|
||||
Status:
|
||||
Failed Checks: 2
|
||||
Phase: Failed
|
||||
Events:
|
||||
Type Reason Age From Message
|
||||
---- ------ ---- ---- -------
|
||||
Normal Synced 3m flagger Starting canary deployment for podinfo.test
|
||||
Normal Synced 3m flagger Advance podinfo.test canary iteration 1/10
|
||||
Normal Synced 3m flagger Advance podinfo.test canary iteration 2/10
|
||||
Normal Synced 3m flagger Advance podinfo.test canary iteration 3/10
|
||||
Normal Synced 3m flagger Halt podinfo.test advancement success rate 69.17% < 99%
|
||||
Normal Synced 2m flagger Halt podinfo.test advancement success rate 61.39% < 99%
|
||||
Warning Synced 2m flagger Rolling back podinfo.test failed checks threshold reached 2
|
||||
Warning Synced 1m flagger Canary failed! Scaling down podinfo.test
|
||||
</code></pre></div><p>The above procedure can be extended with <a href="/usage/metrics.html">custom metrics</a> checks,
|
||||
<a href="/usage/webhooks.html">webhooks</a>,
|
||||
<a href="/usage/webhooks.html#manual-gating">manual promotion</a> approval and
|
||||
<a href="/usage/alerting.html">Slack or MS Teams</a> notifications.</p></div> <footer class="page-edit"><!----> <!----></footer> <div class="page-nav"><p class="inner"><span class="prev">
|
||||
←
|
||||
<a href="/tutorials/istio-progressive-delivery.html" class="prev">
|
||||
Istio Canaries
|
||||
</a></span> <span class="next"><a href="/tutorials/linkerd-progressive-delivery.html">
|
||||
Linkerd Canaries
|
||||
</a>
|
||||
→
|
||||
</span></p></div> </main></div><div class="global-ui"></div></div>
|
||||
<script src="/assets/js/app.b4b27991.js" defer></script><script src="/assets/js/2.ba5bf77c.js" defer></script><script src="/assets/js/23.b3dcc355.js" defer></script>
|
||||
</body>
|
||||
</html>
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -0,0 +1,140 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en-US">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<title>Zero downtime deployments | Flagger</title>
|
||||
<meta name="description" content="Progressive Delivery operator for Kubernetes (Canary, A/B Testing and Blue/Green deployments)">
|
||||
<link rel="icon" href="/favicon.png">
|
||||
<link rel="stylesheet" href="/website.css">
|
||||
<meta name="keywords" content="gitops kubernetes flagger istio linkerd appmesh contour gloo nginx">
|
||||
<meta name="twitter:card" content="summary_large_image">
|
||||
<meta name="twitter:title" content="Flagger">
|
||||
<meta name="twitter:description" content="Progressive delivery Kubernetes operator (Canary, A/B Testing and Blue/Green deployments)">
|
||||
<meta name="twitter:image:src" content="https://flagger.app/flagger-overview.png">
|
||||
|
||||
<link rel="preload" href="/assets/css/0.styles.a0ada997.css" as="style"><link rel="preload" href="/assets/js/app.b4b27991.js" as="script"><link rel="preload" href="/assets/js/2.ba5bf77c.js" as="script"><link rel="preload" href="/assets/js/28.deae273d.js" as="script"><link rel="prefetch" href="/assets/js/10.bf6f38d4.js"><link rel="prefetch" href="/assets/js/11.3af38cfe.js"><link rel="prefetch" href="/assets/js/12.eca4b89c.js"><link rel="prefetch" href="/assets/js/13.9104349a.js"><link rel="prefetch" href="/assets/js/14.a4f3d810.js"><link rel="prefetch" href="/assets/js/15.aada2caa.js"><link rel="prefetch" href="/assets/js/16.3012a10b.js"><link rel="prefetch" href="/assets/js/17.a98b6aa9.js"><link rel="prefetch" href="/assets/js/18.e5188926.js"><link rel="prefetch" href="/assets/js/19.bb9498f7.js"><link rel="prefetch" href="/assets/js/20.dce5a04e.js"><link rel="prefetch" href="/assets/js/21.df66561d.js"><link rel="prefetch" href="/assets/js/22.1c5d14b9.js"><link rel="prefetch" href="/assets/js/23.b3dcc355.js"><link rel="prefetch" href="/assets/js/24.4f54d289.js"><link rel="prefetch" href="/assets/js/25.914acd1d.js"><link rel="prefetch" href="/assets/js/26.b2d4f402.js"><link rel="prefetch" href="/assets/js/27.ea9f1f9c.js"><link rel="prefetch" href="/assets/js/29.17c547e6.js"><link rel="prefetch" href="/assets/js/3.8b254d5d.js"><link rel="prefetch" href="/assets/js/30.38bd809e.js"><link rel="prefetch" href="/assets/js/31.fbfb336c.js"><link rel="prefetch" href="/assets/js/32.591c3f2b.js"><link rel="prefetch" href="/assets/js/33.c456709f.js"><link rel="prefetch" href="/assets/js/34.5a071e36.js"><link rel="prefetch" href="/assets/js/35.fc27c09e.js"><link rel="prefetch" href="/assets/js/36.144098aa.js"><link rel="prefetch" href="/assets/js/4.26a3069c.js"><link rel="prefetch" href="/assets/js/5.f92a6aca.js"><link rel="prefetch" href="/assets/js/6.ee5ffca3.js"><link rel="prefetch" href="/assets/js/7.6bf27a87.js"><link rel="prefetch" href="/assets/js/8.8b72a8d3.js"><link rel="prefetch" href="/assets/js/9.7919fc1d.js">
|
||||
<link rel="stylesheet" href="/assets/css/0.styles.a0ada997.css">
|
||||
</head>
|
||||
<body>
|
||||
<div id="app" data-server-rendered="true"><div class="theme-container"><header class="navbar"><div class="sidebar-button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" role="img" viewBox="0 0 448 512" class="icon"><path fill="currentColor" d="M436 124H12c-6.627 0-12-5.373-12-12V80c0-6.627 5.373-12 12-12h424c6.627 0 12 5.373 12 12v32c0 6.627-5.373 12-12 12zm0 160H12c-6.627 0-12-5.373-12-12v-32c0-6.627 5.373-12 12-12h424c6.627 0 12 5.373 12 12v32c0 6.627-5.373 12-12 12zm0 160H12c-6.627 0-12-5.373-12-12v-32c0-6.627 5.373-12 12-12h424c6.627 0 12 5.373 12 12v32c0 6.627-5.373 12-12 12z"></path></svg></div> <a href="/" class="home-link router-link-active"><!----> <span class="site-name">Flagger</span></a> <div class="links"><div class="search-box"><input aria-label="Search" autocomplete="off" spellcheck="false" value=""> <!----></div> <nav class="nav-links can-hide"><div class="nav-item"><a href="https://docs.flagger.app" target="_blank" rel="noopener noreferrer" class="nav-link external">
|
||||
Docs
|
||||
<svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg></a></div><div class="nav-item"><a href="https://github.com/weaveworks/flagger/blob/master/CHANGELOG.md" target="_blank" rel="noopener noreferrer" class="nav-link external">
|
||||
Changelog
|
||||
<svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg></a></div> <a href="https://github.com/weaveworks/flagger" target="_blank" rel="noopener noreferrer" class="repo-link">
|
||||
GitHub
|
||||
<svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg></a></nav></div></header> <div class="sidebar-mask"></div> <aside class="sidebar"><nav class="nav-links"><div class="nav-item"><a href="https://docs.flagger.app" target="_blank" rel="noopener noreferrer" class="nav-link external">
|
||||
Docs
|
||||
<svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg></a></div><div class="nav-item"><a href="https://github.com/weaveworks/flagger/blob/master/CHANGELOG.md" target="_blank" rel="noopener noreferrer" class="nav-link external">
|
||||
Changelog
|
||||
<svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg></a></div> <a href="https://github.com/weaveworks/flagger" target="_blank" rel="noopener noreferrer" class="repo-link">
|
||||
GitHub
|
||||
<svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg></a></nav> <ul class="sidebar-links"><li><a href="/" class="sidebar-link">Home</a></li><li><section class="sidebar-group depth-0"><a href="/intro/" class="sidebar-heading clickable"><span>Introduction</span> <!----></a> <ul class="sidebar-links sidebar-group-items"><li><a href="/intro/" class="sidebar-link">Get Started</a></li><li><a href="/intro/faq.html" class="sidebar-link">FAQ</a></li></ul></section></li><li><section class="sidebar-group depth-0"><a href="/install/flagger-install-on-kubernetes" class="sidebar-heading clickable"><span>Install</span> <!----></a> <ul class="sidebar-links sidebar-group-items"><li><a href="/install/flagger-install-on-kubernetes.html" class="sidebar-link">On Kubernetes</a></li><li><a href="/install/flagger-install-on-google-cloud.html" class="sidebar-link">On GKE Istio</a></li><li><a href="/install/flagger-install-on-eks-appmesh.html" class="sidebar-link">On EKS App Mesh</a></li></ul></section></li><li><section class="sidebar-group depth-0"><a href="/usage/how-it-works" class="sidebar-heading clickable"><span>Usage</span> <!----></a> <ul class="sidebar-links sidebar-group-items"><li><a href="/usage/how-it-works.html" class="sidebar-link">How it works</a></li><li><a href="/usage/deployment-strategies.html" class="sidebar-link">Deployment Strategies</a></li><li><a href="/usage/metrics.html" class="sidebar-link">Metrics Analysis</a></li><li><a href="/usage/webhooks.html" class="sidebar-link">Webhooks</a></li><li><a href="/usage/alerting.html" class="sidebar-link">Alerting</a></li><li><a href="/usage/monitoring.html" class="sidebar-link">Monitoring</a></li></ul></section></li><li><section class="sidebar-group depth-0"><a href="/tutorials/istio-progressive-delivery" class="sidebar-heading clickable"><span>Tutorials</span> <!----></a> <ul class="sidebar-links sidebar-group-items"><li><a href="/tutorials/istio-progressive-delivery.html" class="sidebar-link">Istio Canaries</a></li><li><a href="/tutorials/istio-ab-testing.html" class="sidebar-link">Istio A/B Testing</a></li><li><a href="/tutorials/linkerd-progressive-delivery.html" class="sidebar-link">Linkerd Canaries</a></li><li><a href="/tutorials/appmesh-progressive-delivery.html" class="sidebar-link">App Mesh Canaries</a></li><li><a href="/tutorials/nginx-progressive-delivery.html" class="sidebar-link">NGINX Ingress Canaries</a></li><li><a href="/tutorials/gloo-progressive-delivery.html" class="sidebar-link">Gloo Canaries</a></li><li><a href="/tutorials/contour-progressive-delivery.html" class="sidebar-link">Contour Canaries</a></li><li><a href="/tutorials/kubernetes-blue-green.html" class="sidebar-link">Kubernetes Blue/Green</a></li><li><a href="/tutorials/canary-helm-gitops.html" class="sidebar-link">Canaries with Helm charts and GitOps</a></li></ul></section></li><li><section class="sidebar-group depth-0"><a href="/dev/dev-guide" class="sidebar-heading clickable"><span>Dev</span> <!----></a> <ul class="sidebar-links sidebar-group-items"><li><a href="/dev/dev-guide.html" class="sidebar-link">Development Guide</a></li><li><a href="/dev/release-guide.html" class="sidebar-link">Release Guide</a></li><li><a href="/dev/upgrade-guide.html" class="sidebar-link">Upgrade Guide</a></li></ul></section></li></ul> </aside> <main class="page"> <div class="theme-default-content content__default"><h1 id="zero-downtime-deployments"><a href="#zero-downtime-deployments" aria-hidden="true" class="header-anchor">#</a> Zero downtime deployments</h1> <p>This is a list of things you should consider when dealing with a high traffic production environment if you want to minimise the impact of rolling updates and downscaling.</p> <h2 id="deployment-strategy"><a href="#deployment-strategy" aria-hidden="true" class="header-anchor">#</a> Deployment strategy</h2> <p>Limit the number of unavailable pods during a rolling update:</p> <div class="language-yaml extra-class"><pre class="language-yaml"><code><span class="token key atrule">apiVersion</span><span class="token punctuation">:</span> apps/v1
|
||||
<span class="token key atrule">kind</span><span class="token punctuation">:</span> Deployment
|
||||
<span class="token key atrule">spec</span><span class="token punctuation">:</span>
|
||||
<span class="token key atrule">progressDeadlineSeconds</span><span class="token punctuation">:</span> <span class="token number">120</span>
|
||||
<span class="token key atrule">strategy</span><span class="token punctuation">:</span>
|
||||
<span class="token key atrule">type</span><span class="token punctuation">:</span> RollingUpdate
|
||||
<span class="token key atrule">rollingUpdate</span><span class="token punctuation">:</span>
|
||||
<span class="token key atrule">maxUnavailable</span><span class="token punctuation">:</span> <span class="token number">0</span>
|
||||
</code></pre></div><p>The default progress deadline for a deployment is ten minutes. You should consider adjusting this value to make the deployment process fail faster.</p> <h2 id="liveness-health-check"><a href="#liveness-health-check" aria-hidden="true" class="header-anchor">#</a> Liveness health check</h2> <p>You application should expose a HTTP endpoint that Kubernetes can call to determine if your app transitioned to a broken state from which it can't recover and needs to be restarted.</p> <div class="language-yaml extra-class"><pre class="language-yaml"><code><span class="token key atrule">livenessProbe</span><span class="token punctuation">:</span>
|
||||
<span class="token key atrule">exec</span><span class="token punctuation">:</span>
|
||||
<span class="token key atrule">command</span><span class="token punctuation">:</span>
|
||||
<span class="token punctuation">-</span> wget
|
||||
<span class="token punctuation">-</span> <span class="token punctuation">-</span><span class="token punctuation">-</span>quiet
|
||||
<span class="token punctuation">-</span> <span class="token punctuation">-</span><span class="token punctuation">-</span>tries=1
|
||||
<span class="token punctuation">-</span> <span class="token punctuation">-</span><span class="token punctuation">-</span>timeout=4
|
||||
<span class="token punctuation">-</span> <span class="token punctuation">-</span><span class="token punctuation">-</span>spider
|
||||
<span class="token punctuation">-</span> http<span class="token punctuation">:</span>//localhost<span class="token punctuation">:</span>8080/healthz
|
||||
<span class="token key atrule">timeoutSeconds</span><span class="token punctuation">:</span> <span class="token number">5</span>
|
||||
<span class="token key atrule">initialDelaySeconds</span><span class="token punctuation">:</span> <span class="token number">5</span>
|
||||
</code></pre></div><p>If you've enabled mTLS, you'll have to use <code>exec</code> for liveness and readiness checks since kubelet is not part of the service mesh and doesn't have access to the TLS cert.</p> <h2 id="readiness-health-check"><a href="#readiness-health-check" aria-hidden="true" class="header-anchor">#</a> Readiness health check</h2> <p>You application should expose a HTTP endpoint that Kubernetes can call to determine if your app is ready to receive traffic.</p> <div class="language-yaml extra-class"><pre class="language-yaml"><code><span class="token key atrule">readinessProbe</span><span class="token punctuation">:</span>
|
||||
<span class="token key atrule">exec</span><span class="token punctuation">:</span>
|
||||
<span class="token key atrule">command</span><span class="token punctuation">:</span>
|
||||
<span class="token punctuation">-</span> wget
|
||||
<span class="token punctuation">-</span> <span class="token punctuation">-</span><span class="token punctuation">-</span>quiet
|
||||
<span class="token punctuation">-</span> <span class="token punctuation">-</span><span class="token punctuation">-</span>tries=1
|
||||
<span class="token punctuation">-</span> <span class="token punctuation">-</span><span class="token punctuation">-</span>timeout=4
|
||||
<span class="token punctuation">-</span> <span class="token punctuation">-</span><span class="token punctuation">-</span>spider
|
||||
<span class="token punctuation">-</span> http<span class="token punctuation">:</span>//localhost<span class="token punctuation">:</span>8080/readyz
|
||||
<span class="token key atrule">timeoutSeconds</span><span class="token punctuation">:</span> <span class="token number">5</span>
|
||||
<span class="token key atrule">initialDelaySeconds</span><span class="token punctuation">:</span> <span class="token number">5</span>
|
||||
<span class="token key atrule">periodSeconds</span><span class="token punctuation">:</span> <span class="token number">5</span>
|
||||
</code></pre></div><p>If your app depends on external services, you should check if those services are available before allowing Kubernetes to route traffic to an app instance. Keep in mind that the Envoy sidecar can have a slower startup than your app. This means that on application start you should retry for at least a couple of seconds any external connection.</p> <h2 id="graceful-shutdown"><a href="#graceful-shutdown" aria-hidden="true" class="header-anchor">#</a> Graceful shutdown</h2> <p>Before a pod gets terminated, Kubernetes sends a <code>SIGTERM</code> signal to every container and waits for period of time (30s by default) for all containers to exit gracefully. If your app doesn't handle the <code>SIGTERM</code> signal or if it doesn't exit within the grace period, Kubernetes will kill the container and any inflight requests that your app is processing will fail.</p> <div class="language-yaml extra-class"><pre class="language-yaml"><code><span class="token key atrule">apiVersion</span><span class="token punctuation">:</span> apps/v1
|
||||
<span class="token key atrule">kind</span><span class="token punctuation">:</span> Deployment
|
||||
<span class="token key atrule">spec</span><span class="token punctuation">:</span>
|
||||
<span class="token key atrule">template</span><span class="token punctuation">:</span>
|
||||
<span class="token key atrule">spec</span><span class="token punctuation">:</span>
|
||||
<span class="token key atrule">terminationGracePeriodSeconds</span><span class="token punctuation">:</span> <span class="token number">60</span>
|
||||
<span class="token key atrule">containers</span><span class="token punctuation">:</span>
|
||||
<span class="token punctuation">-</span> <span class="token key atrule">name</span><span class="token punctuation">:</span> app
|
||||
<span class="token key atrule">lifecycle</span><span class="token punctuation">:</span>
|
||||
<span class="token key atrule">preStop</span><span class="token punctuation">:</span>
|
||||
<span class="token key atrule">exec</span><span class="token punctuation">:</span>
|
||||
<span class="token key atrule">command</span><span class="token punctuation">:</span>
|
||||
<span class="token punctuation">-</span> sleep
|
||||
<span class="token punctuation">-</span> <span class="token string">"10"</span>
|
||||
</code></pre></div><p>Your app container should have a <code>preStop</code> hook that delays the container shutdown. This will allow the service mesh to drain the traffic and remove this pod from all other Envoy sidecars before your app becomes unavailable.</p> <h2 id="delay-envoy-shutdown"><a href="#delay-envoy-shutdown" aria-hidden="true" class="header-anchor">#</a> Delay Envoy shutdown</h2> <p>Even if your app reacts to <code>SIGTERM</code> and tries to complete the inflight requests before shutdown, that doesn't mean that the response will make it back to the caller. If the Envoy sidecar shuts down before your app, then the caller will receive a 503 error.</p> <p>To mitigate this issue you can add a <code>preStop</code> hook to the Istio proxy and wait for the main app to exist before Envoy exists.</p> <div class="language-bash extra-class"><pre class="language-bash"><code><span class="token shebang important">#!/bin/bash</span>
|
||||
<span class="token builtin class-name">set</span> -e
|
||||
<span class="token keyword">if</span> <span class="token operator">!</span> pidof envoy <span class="token operator">&></span>/dev/null<span class="token punctuation">;</span> <span class="token keyword">then</span>
|
||||
<span class="token builtin class-name">exit</span> <span class="token number">0</span>
|
||||
<span class="token keyword">fi</span>
|
||||
|
||||
<span class="token keyword">if</span> <span class="token operator">!</span> pidof pilot-agent <span class="token operator">&></span>/dev/null<span class="token punctuation">;</span> <span class="token keyword">then</span>
|
||||
<span class="token builtin class-name">exit</span> <span class="token number">0</span>
|
||||
<span class="token keyword">fi</span>
|
||||
|
||||
<span class="token keyword">while</span> <span class="token punctuation">[</span> <span class="token variable"><span class="token variable">$(</span><span class="token function">netstat</span> -plunt <span class="token operator">|</span> <span class="token function">grep</span> tcp <span class="token operator">|</span> <span class="token function">grep</span> -v envoy <span class="token operator">|</span> <span class="token function">wc</span> -l <span class="token operator">|</span> <span class="token function">xargs</span><span class="token variable">)</span></span> -ne <span class="token number">0</span> <span class="token punctuation">]</span><span class="token punctuation">;</span> <span class="token keyword">do</span>
|
||||
<span class="token function">sleep</span> <span class="token number">1</span><span class="token punctuation">;</span>
|
||||
<span class="token keyword">done</span>
|
||||
|
||||
<span class="token builtin class-name">exit</span> <span class="token number">0</span>
|
||||
</code></pre></div><p>You'll have to build your own Envoy docker image with the above script and modify the Istio injection webhook with the <code>preStop</code> directive.</p> <p>Thanks to Stono for his excellent <a href="https://github.com/istio/istio/issues/12183" target="_blank" rel="noopener noreferrer">tips<svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg></a> on minimising 503s.</p> <h2 id="resource-requests-and-limits"><a href="#resource-requests-and-limits" aria-hidden="true" class="header-anchor">#</a> Resource requests and limits</h2> <p>Setting CPU and memory requests/limits for all workloads is a mandatory step if you're running a production system. Without limits your nodes could run out of memory or become unresponsive due to CPU exhausting. Without CPU and memory requests, the Kubernetes scheduler will not be able to make decisions about which nodes to place pods on.</p> <div class="language-yaml extra-class"><pre class="language-yaml"><code><span class="token key atrule">apiVersion</span><span class="token punctuation">:</span> apps/v1
|
||||
<span class="token key atrule">kind</span><span class="token punctuation">:</span> Deployment
|
||||
<span class="token key atrule">spec</span><span class="token punctuation">:</span>
|
||||
<span class="token key atrule">template</span><span class="token punctuation">:</span>
|
||||
<span class="token key atrule">spec</span><span class="token punctuation">:</span>
|
||||
<span class="token key atrule">containers</span><span class="token punctuation">:</span>
|
||||
<span class="token punctuation">-</span> <span class="token key atrule">name</span><span class="token punctuation">:</span> app
|
||||
<span class="token key atrule">resources</span><span class="token punctuation">:</span>
|
||||
<span class="token key atrule">limits</span><span class="token punctuation">:</span>
|
||||
<span class="token key atrule">cpu</span><span class="token punctuation">:</span> 1000m
|
||||
<span class="token key atrule">memory</span><span class="token punctuation">:</span> 1Gi
|
||||
<span class="token key atrule">requests</span><span class="token punctuation">:</span>
|
||||
<span class="token key atrule">cpu</span><span class="token punctuation">:</span> 100m
|
||||
<span class="token key atrule">memory</span><span class="token punctuation">:</span> 128Mi
|
||||
</code></pre></div><p>Note that without resource requests the horizontal pod autoscaler can't determine when to scale your app.</p> <h2 id="autoscaling"><a href="#autoscaling" aria-hidden="true" class="header-anchor">#</a> Autoscaling</h2> <p>A production environment should be able to handle traffic bursts without impacting the quality of service. This can be achieved with Kubernetes autoscaling capabilities. Autoscaling in Kubernetes has two dimensions: the Cluster Autoscaler that deals with node scaling operations and the Horizontal Pod Autoscaler that automatically scales the number of pods in a deployment.</p> <div class="language-yaml extra-class"><pre class="language-yaml"><code><span class="token key atrule">apiVersion</span><span class="token punctuation">:</span> autoscaling/v2beta1
|
||||
<span class="token key atrule">kind</span><span class="token punctuation">:</span> HorizontalPodAutoscaler
|
||||
<span class="token key atrule">spec</span><span class="token punctuation">:</span>
|
||||
<span class="token key atrule">scaleTargetRef</span><span class="token punctuation">:</span>
|
||||
<span class="token key atrule">apiVersion</span><span class="token punctuation">:</span> apps/v1
|
||||
<span class="token key atrule">kind</span><span class="token punctuation">:</span> Deployment
|
||||
<span class="token key atrule">name</span><span class="token punctuation">:</span> app
|
||||
<span class="token key atrule">minReplicas</span><span class="token punctuation">:</span> <span class="token number">2</span>
|
||||
<span class="token key atrule">maxReplicas</span><span class="token punctuation">:</span> <span class="token number">4</span>
|
||||
<span class="token key atrule">metrics</span><span class="token punctuation">:</span>
|
||||
<span class="token punctuation">-</span> <span class="token key atrule">type</span><span class="token punctuation">:</span> Resource
|
||||
<span class="token key atrule">resource</span><span class="token punctuation">:</span>
|
||||
<span class="token key atrule">name</span><span class="token punctuation">:</span> cpu
|
||||
<span class="token key atrule">targetAverageValue</span><span class="token punctuation">:</span> 900m
|
||||
<span class="token punctuation">-</span> <span class="token key atrule">type</span><span class="token punctuation">:</span> Resource
|
||||
<span class="token key atrule">resource</span><span class="token punctuation">:</span>
|
||||
<span class="token key atrule">name</span><span class="token punctuation">:</span> memory
|
||||
<span class="token key atrule">targetAverageValue</span><span class="token punctuation">:</span> 768Mi
|
||||
</code></pre></div><p>The above HPA ensures your app will be scaled up before the pods reach the CPU or memory limits.</p> <h2 id="ingress-retries"><a href="#ingress-retries" aria-hidden="true" class="header-anchor">#</a> Ingress retries</h2> <p>To minimise the impact of downscaling operations you can make use of Envoy retry capabilities.</p> <div class="language-yaml extra-class"><pre class="language-yaml"><code><span class="token key atrule">apiVersion</span><span class="token punctuation">:</span> flagger.app/v1beta1
|
||||
<span class="token key atrule">kind</span><span class="token punctuation">:</span> Canary
|
||||
<span class="token key atrule">spec</span><span class="token punctuation">:</span>
|
||||
<span class="token key atrule">service</span><span class="token punctuation">:</span>
|
||||
<span class="token key atrule">port</span><span class="token punctuation">:</span> <span class="token number">9898</span>
|
||||
<span class="token key atrule">gateways</span><span class="token punctuation">:</span>
|
||||
<span class="token punctuation">-</span> public<span class="token punctuation">-</span>gateway.istio<span class="token punctuation">-</span>system.svc.cluster.local
|
||||
<span class="token key atrule">hosts</span><span class="token punctuation">:</span>
|
||||
<span class="token punctuation">-</span> app.example.com
|
||||
<span class="token key atrule">retries</span><span class="token punctuation">:</span>
|
||||
<span class="token key atrule">attempts</span><span class="token punctuation">:</span> <span class="token number">10</span>
|
||||
<span class="token key atrule">perTryTimeout</span><span class="token punctuation">:</span> 5s
|
||||
<span class="token key atrule">retryOn</span><span class="token punctuation">:</span> <span class="token string">"gateway-error,connect-failure,refused-stream"</span>
|
||||
</code></pre></div><p>When the HPA scales down your app, your users could run into 503 errors. The above configuration will make Envoy retry the HTTP requests that failed due to gateway errors.</p></div> <footer class="page-edit"><!----> <!----></footer> <!----> </main></div><div class="global-ui"></div></div>
|
||||
<script src="/assets/js/app.b4b27991.js" defer></script><script src="/assets/js/2.ba5bf77c.js" defer></script><script src="/assets/js/28.deae273d.js" defer></script>
|
||||
</body>
|
||||
</html>
|
||||
Reference in New Issue
Block a user