mirror of
https://github.com/fluxcd/flagger.git
synced 2026-04-15 06:57:34 +00:00
112 lines
22 KiB
HTML
112 lines
22 KiB
HTML
<!DOCTYPE html>
|
|
<html lang="en-US">
|
|
<head>
|
|
<meta charset="utf-8">
|
|
<meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<title>Monitoring | Flagger</title>
|
|
<meta name="description" content="Progressive Delivery operator for Kubernetes (Canary, A/B Testing and Blue/Green deployments)">
|
|
<link rel="icon" href="/favicon.png">
|
|
<link rel="stylesheet" href="/website.css">
|
|
<meta name="keywords" content="gitops kubernetes flagger istio linkerd appmesh contour gloo nginx">
|
|
<meta name="twitter:card" content="summary_large_image">
|
|
<meta name="twitter:title" content="Flagger">
|
|
<meta name="twitter:description" content="Progressive delivery Kubernetes operator (Canary, A/B Testing and Blue/Green deployments)">
|
|
<meta name="twitter:image:src" content="https://flagger.app/flagger-overview.png">
|
|
|
|
<link rel="preload" href="/assets/css/0.styles.a0ada997.css" as="style"><link rel="preload" href="/assets/js/app.67d16e10.js" as="script"><link rel="preload" href="/assets/js/2.ba5bf77c.js" as="script"><link rel="preload" href="/assets/js/34.5a071e36.js" as="script"><link rel="prefetch" href="/assets/js/10.d04c7ada.js"><link rel="prefetch" href="/assets/js/11.ab251275.js"><link rel="prefetch" href="/assets/js/12.eca4b89c.js"><link rel="prefetch" href="/assets/js/13.6b55cb59.js"><link rel="prefetch" href="/assets/js/14.affa49a6.js"><link rel="prefetch" href="/assets/js/15.aada2caa.js"><link rel="prefetch" href="/assets/js/16.3012a10b.js"><link rel="prefetch" href="/assets/js/17.a98b6aa9.js"><link rel="prefetch" href="/assets/js/18.e5188926.js"><link rel="prefetch" href="/assets/js/19.bb9498f7.js"><link rel="prefetch" href="/assets/js/20.2b1bebf5.js"><link rel="prefetch" href="/assets/js/21.e1068843.js"><link rel="prefetch" href="/assets/js/22.1c5d14b9.js"><link rel="prefetch" href="/assets/js/23.ec1dec63.js"><link rel="prefetch" href="/assets/js/24.721d8c4e.js"><link rel="prefetch" href="/assets/js/25.914acd1d.js"><link rel="prefetch" href="/assets/js/26.b2d4f402.js"><link rel="prefetch" href="/assets/js/27.ea9f1f9c.js"><link rel="prefetch" href="/assets/js/28.deae273d.js"><link rel="prefetch" href="/assets/js/29.17c547e6.js"><link rel="prefetch" href="/assets/js/3.8b254d5d.js"><link rel="prefetch" href="/assets/js/30.38bd809e.js"><link rel="prefetch" href="/assets/js/31.fbfb336c.js"><link rel="prefetch" href="/assets/js/32.591c3f2b.js"><link rel="prefetch" href="/assets/js/33.c456709f.js"><link rel="prefetch" href="/assets/js/35.fc27c09e.js"><link rel="prefetch" href="/assets/js/36.144098aa.js"><link rel="prefetch" href="/assets/js/4.26a3069c.js"><link rel="prefetch" href="/assets/js/5.f0e04ff6.js"><link rel="prefetch" href="/assets/js/6.ee5ffca3.js"><link rel="prefetch" href="/assets/js/7.6bf27a87.js"><link rel="prefetch" href="/assets/js/8.8b72a8d3.js"><link rel="prefetch" href="/assets/js/9.7919fc1d.js">
|
|
<link rel="stylesheet" href="/assets/css/0.styles.a0ada997.css">
|
|
</head>
|
|
<body>
|
|
<div id="app" data-server-rendered="true"><div class="theme-container"><header class="navbar"><div class="sidebar-button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" role="img" viewBox="0 0 448 512" class="icon"><path fill="currentColor" d="M436 124H12c-6.627 0-12-5.373-12-12V80c0-6.627 5.373-12 12-12h424c6.627 0 12 5.373 12 12v32c0 6.627-5.373 12-12 12zm0 160H12c-6.627 0-12-5.373-12-12v-32c0-6.627 5.373-12 12-12h424c6.627 0 12 5.373 12 12v32c0 6.627-5.373 12-12 12zm0 160H12c-6.627 0-12-5.373-12-12v-32c0-6.627 5.373-12 12-12h424c6.627 0 12 5.373 12 12v32c0 6.627-5.373 12-12 12z"></path></svg></div> <a href="/" class="home-link router-link-active"><!----> <span class="site-name">Flagger</span></a> <div class="links"><div class="search-box"><input aria-label="Search" autocomplete="off" spellcheck="false" value=""> <!----></div> <nav class="nav-links can-hide"><div class="nav-item"><a href="https://docs.flagger.app" target="_blank" rel="noopener noreferrer" class="nav-link external">
|
|
Docs
|
|
<svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg></a></div><div class="nav-item"><a href="https://github.com/weaveworks/flagger/blob/master/CHANGELOG.md" target="_blank" rel="noopener noreferrer" class="nav-link external">
|
|
Changelog
|
|
<svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg></a></div> <a href="https://github.com/weaveworks/flagger" target="_blank" rel="noopener noreferrer" class="repo-link">
|
|
GitHub
|
|
<svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg></a></nav></div></header> <div class="sidebar-mask"></div> <aside class="sidebar"><nav class="nav-links"><div class="nav-item"><a href="https://docs.flagger.app" target="_blank" rel="noopener noreferrer" class="nav-link external">
|
|
Docs
|
|
<svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg></a></div><div class="nav-item"><a href="https://github.com/weaveworks/flagger/blob/master/CHANGELOG.md" target="_blank" rel="noopener noreferrer" class="nav-link external">
|
|
Changelog
|
|
<svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg></a></div> <a href="https://github.com/weaveworks/flagger" target="_blank" rel="noopener noreferrer" class="repo-link">
|
|
GitHub
|
|
<svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg></a></nav> <ul class="sidebar-links"><li><a href="/" class="sidebar-link">Home</a></li><li><section class="sidebar-group depth-0"><a href="/intro/" class="sidebar-heading clickable"><span>Introduction</span> <!----></a> <ul class="sidebar-links sidebar-group-items"><li><a href="/intro/" class="sidebar-link">Get Started</a></li><li><a href="/intro/faq.html" class="sidebar-link">FAQ</a></li></ul></section></li><li><section class="sidebar-group depth-0"><a href="/install/flagger-install-on-kubernetes" class="sidebar-heading clickable"><span>Install</span> <!----></a> <ul class="sidebar-links sidebar-group-items"><li><a href="/install/flagger-install-on-kubernetes.html" class="sidebar-link">On Kubernetes</a></li><li><a href="/install/flagger-install-on-google-cloud.html" class="sidebar-link">On GKE Istio</a></li><li><a href="/install/flagger-install-on-eks-appmesh.html" class="sidebar-link">On EKS App Mesh</a></li></ul></section></li><li><section class="sidebar-group depth-0"><a href="/usage/how-it-works" class="sidebar-heading clickable open"><span>Usage</span> <!----></a> <ul class="sidebar-links sidebar-group-items"><li><a href="/usage/how-it-works.html" class="sidebar-link">How it works</a></li><li><a href="/usage/deployment-strategies.html" class="sidebar-link">Deployment Strategies</a></li><li><a href="/usage/metrics.html" class="sidebar-link">Metrics Analysis</a></li><li><a href="/usage/webhooks.html" class="sidebar-link">Webhooks</a></li><li><a href="/usage/alerting.html" class="sidebar-link">Alerting</a></li><li><a href="/usage/monitoring.html" class="active sidebar-link">Monitoring</a><ul class="sidebar-sub-headers"><li class="sidebar-sub-header"><a href="/usage/monitoring.html#grafana" class="sidebar-link">Grafana</a></li><li class="sidebar-sub-header"><a href="/usage/monitoring.html#logging" class="sidebar-link">Logging</a></li><li class="sidebar-sub-header"><a href="/usage/monitoring.html#event-webhook" class="sidebar-link">Event Webhook</a></li><li class="sidebar-sub-header"><a href="/usage/monitoring.html#metrics" class="sidebar-link">Metrics</a></li></ul></li></ul></section></li><li><section class="sidebar-group depth-0"><a href="/tutorials/istio-progressive-delivery" class="sidebar-heading clickable"><span>Tutorials</span> <!----></a> <ul class="sidebar-links sidebar-group-items"><li><a href="/tutorials/istio-progressive-delivery.html" class="sidebar-link">Istio Canaries</a></li><li><a href="/tutorials/istio-ab-testing.html" class="sidebar-link">Istio A/B Testing</a></li><li><a href="/tutorials/linkerd-progressive-delivery.html" class="sidebar-link">Linkerd Canaries</a></li><li><a href="/tutorials/appmesh-progressive-delivery.html" class="sidebar-link">App Mesh Canaries</a></li><li><a href="/tutorials/nginx-progressive-delivery.html" class="sidebar-link">NGINX Ingress Canaries</a></li><li><a href="/tutorials/gloo-progressive-delivery.html" class="sidebar-link">Gloo Canaries</a></li><li><a href="/tutorials/contour-progressive-delivery.html" class="sidebar-link">Contour Canaries</a></li><li><a href="/tutorials/kubernetes-blue-green.html" class="sidebar-link">Kubernetes Blue/Green</a></li><li><a href="/tutorials/canary-helm-gitops.html" class="sidebar-link">Canaries with Helm charts and GitOps</a></li></ul></section></li><li><section class="sidebar-group depth-0"><a href="/dev/dev-guide" class="sidebar-heading clickable"><span>Dev</span> <!----></a> <ul class="sidebar-links sidebar-group-items"><li><a href="/dev/dev-guide.html" class="sidebar-link">Development Guide</a></li><li><a href="/dev/release-guide.html" class="sidebar-link">Release Guide</a></li><li><a href="/dev/upgrade-guide.html" class="sidebar-link">Upgrade Guide</a></li></ul></section></li></ul> </aside> <main class="page"> <div class="theme-default-content content__default"><h1 id="monitoring"><a href="#monitoring" aria-hidden="true" class="header-anchor">#</a> Monitoring</h1> <h2 id="grafana"><a href="#grafana" aria-hidden="true" class="header-anchor">#</a> Grafana</h2> <p>Flagger comes with a Grafana dashboard made for canary analysis. Install Grafana with Helm:</p> <div class="language-bash extra-class"><pre class="language-bash"><code>helm upgrade -i flagger-grafana flagger/grafana <span class="token punctuation">\</span>
|
|
--namespace<span class="token operator">=</span>istio-system <span class="token punctuation">\</span> <span class="token comment"># or appmesh-system</span>
|
|
--set <span class="token assign-left variable">url</span><span class="token operator">=</span>http://prometheus:9090
|
|
</code></pre></div><p>The dashboard shows the RED and USE metrics for the primary and canary workloads:</p> <p><img src="https://raw.githubusercontent.com/weaveworks/flagger/master/docs/screens/grafana-canary-analysis.png" alt="Canary Dashboard"></p> <h2 id="logging"><a href="#logging" aria-hidden="true" class="header-anchor">#</a> Logging</h2> <p>The canary errors and latency spikes have been recorded as Kubernetes events and logged by Flagger in json format:</p> <div class="language-text extra-class"><pre class="language-text"><code>kubectl -n istio-system logs deployment/flagger --tail=100 | jq .msg
|
|
|
|
Starting canary deployment for podinfo.test
|
|
Advance podinfo.test canary weight 5
|
|
Advance podinfo.test canary weight 10
|
|
Advance podinfo.test canary weight 15
|
|
Advance podinfo.test canary weight 20
|
|
Advance podinfo.test canary weight 25
|
|
Advance podinfo.test canary weight 30
|
|
Advance podinfo.test canary weight 35
|
|
Halt podinfo.test advancement success rate 98.69% < 99%
|
|
Advance podinfo.test canary weight 40
|
|
Halt podinfo.test advancement request duration 1.515s > 500ms
|
|
Advance podinfo.test canary weight 45
|
|
Advance podinfo.test canary weight 50
|
|
Copying podinfo.test template spec to podinfo-primary.test
|
|
Halt podinfo-primary.test advancement waiting for rollout to finish: 1 old replicas are pending termination
|
|
Scaling down podinfo.test
|
|
Promotion completed! podinfo.test
|
|
</code></pre></div><h2 id="event-webhook"><a href="#event-webhook" aria-hidden="true" class="header-anchor">#</a> Event Webhook</h2> <p>Flagger can be configured to send event payloads to a specified webhook:</p> <div class="language-bash extra-class"><pre class="language-bash"><code>helm upgrade -i flagger flagger/flagger <span class="token punctuation">\</span>
|
|
--set <span class="token assign-left variable">eventWebhook</span><span class="token operator">=</span>https://example.com/flagger-canary-event-webhook
|
|
</code></pre></div><p>The environment variable <em>EVENT_WEBHOOK_URL</em> can be used for activating the event-webhook, too. This is handy for using a secret to store a sensible value that could contain api keys for example.</p> <p>When configured, every action that Flagger takes during a canary deployment will be sent as JSON via an HTTP POST request. The JSON payload has the following schema:</p> <div class="language-javascript extra-class"><pre class="language-javascript"><code><span class="token punctuation">{</span>
|
|
<span class="token string">"name"</span><span class="token punctuation">:</span> <span class="token string">"string (canary name)"</span><span class="token punctuation">,</span>
|
|
<span class="token string">"namespace"</span><span class="token punctuation">:</span> <span class="token string">"string (canary namespace)"</span><span class="token punctuation">,</span>
|
|
<span class="token string">"phase"</span><span class="token punctuation">:</span> <span class="token string">"string (canary phase)"</span><span class="token punctuation">,</span>
|
|
<span class="token string">"metadata"</span><span class="token punctuation">:</span> <span class="token punctuation">{</span>
|
|
<span class="token string">"eventMessage"</span><span class="token punctuation">:</span> <span class="token string">"string (canary event message)"</span><span class="token punctuation">,</span>
|
|
<span class="token string">"eventType"</span><span class="token punctuation">:</span> <span class="token string">"string (canary event type)"</span><span class="token punctuation">,</span>
|
|
<span class="token string">"timestamp"</span><span class="token punctuation">:</span> <span class="token string">"string (unix timestamp ms)"</span>
|
|
<span class="token punctuation">}</span>
|
|
<span class="token punctuation">}</span>
|
|
</code></pre></div><p>Example:</p> <div class="language-javascript extra-class"><pre class="language-javascript"><code><span class="token punctuation">{</span>
|
|
<span class="token string">"name"</span><span class="token punctuation">:</span> <span class="token string">"podinfo"</span><span class="token punctuation">,</span>
|
|
<span class="token string">"namespace"</span><span class="token punctuation">:</span> <span class="token string">"default"</span><span class="token punctuation">,</span>
|
|
<span class="token string">"phase"</span><span class="token punctuation">:</span> <span class="token string">"Progressing"</span><span class="token punctuation">,</span>
|
|
<span class="token string">"metadata"</span><span class="token punctuation">:</span> <span class="token punctuation">{</span>
|
|
<span class="token string">"eventMessage"</span><span class="token punctuation">:</span> <span class="token string">"New revision detected! Scaling up podinfo.default"</span><span class="token punctuation">,</span>
|
|
<span class="token string">"eventType"</span><span class="token punctuation">:</span> <span class="token string">"Normal"</span><span class="token punctuation">,</span>
|
|
<span class="token string">"timestamp"</span><span class="token punctuation">:</span> <span class="token string">"1578607635167"</span>
|
|
<span class="token punctuation">}</span>
|
|
<span class="token punctuation">}</span>
|
|
</code></pre></div><p>The event webhook can be overwritten at canary level with:</p> <div class="language-yaml extra-class"><pre class="language-yaml"><code> <span class="token key atrule">analysis</span><span class="token punctuation">:</span>
|
|
<span class="token key atrule">webhooks</span><span class="token punctuation">:</span>
|
|
<span class="token punctuation">-</span> <span class="token key atrule">name</span><span class="token punctuation">:</span> <span class="token string">"send to Slack"</span>
|
|
<span class="token key atrule">type</span><span class="token punctuation">:</span> event
|
|
<span class="token key atrule">url</span><span class="token punctuation">:</span> http<span class="token punctuation">:</span>//event<span class="token punctuation">-</span>recevier.notifications/slack
|
|
</code></pre></div><h2 id="metrics"><a href="#metrics" aria-hidden="true" class="header-anchor">#</a> Metrics</h2> <p>Flagger exposes Prometheus metrics that can be used to determine the canary analysis status and the destination weight values:</p> <div class="language-bash extra-class"><pre class="language-bash"><code><span class="token comment"># Flagger version and mesh provider gauge</span>
|
|
flagger_info<span class="token punctuation">{</span>version<span class="token operator">=</span><span class="token string">"0.10.0"</span>, <span class="token assign-left variable">mesh_provider</span><span class="token operator">=</span><span class="token string">"istio"</span><span class="token punctuation">}</span> <span class="token number">1</span>
|
|
|
|
<span class="token comment"># Canaries total gauge</span>
|
|
flagger_canary_total<span class="token punctuation">{</span>namespace<span class="token operator">=</span><span class="token string">"test"</span><span class="token punctuation">}</span> <span class="token number">1</span>
|
|
|
|
<span class="token comment"># Canary promotion last known status gauge</span>
|
|
<span class="token comment"># 0 - running, 1 - successful, 2 - failed</span>
|
|
flagger_canary_status<span class="token punctuation">{</span>name<span class="token operator">=</span><span class="token string">"podinfo"</span> <span class="token assign-left variable">namespace</span><span class="token operator">=</span><span class="token string">"test"</span><span class="token punctuation">}</span> <span class="token number">1</span>
|
|
|
|
<span class="token comment"># Canary traffic weight gauge</span>
|
|
flagger_canary_weight<span class="token punctuation">{</span>workload<span class="token operator">=</span><span class="token string">"podinfo-primary"</span> <span class="token assign-left variable">namespace</span><span class="token operator">=</span><span class="token string">"test"</span><span class="token punctuation">}</span> <span class="token number">95</span>
|
|
flagger_canary_weight<span class="token punctuation">{</span>workload<span class="token operator">=</span><span class="token string">"podinfo"</span> <span class="token assign-left variable">namespace</span><span class="token operator">=</span><span class="token string">"test"</span><span class="token punctuation">}</span> <span class="token number">5</span>
|
|
|
|
<span class="token comment"># Seconds spent performing canary analysis histogram</span>
|
|
flagger_canary_duration_seconds_bucket<span class="token punctuation">{</span>name<span class="token operator">=</span><span class="token string">"podinfo"</span>,namespace<span class="token operator">=</span><span class="token string">"test"</span>,le<span class="token operator">=</span><span class="token string">"10"</span><span class="token punctuation">}</span> <span class="token number">6</span>
|
|
flagger_canary_duration_seconds_bucket<span class="token punctuation">{</span>name<span class="token operator">=</span><span class="token string">"podinfo"</span>,namespace<span class="token operator">=</span><span class="token string">"test"</span>,le<span class="token operator">=</span><span class="token string">"+Inf"</span><span class="token punctuation">}</span> <span class="token number">6</span>
|
|
flagger_canary_duration_seconds_sum<span class="token punctuation">{</span>name<span class="token operator">=</span><span class="token string">"podinfo"</span>,namespace<span class="token operator">=</span><span class="token string">"test"</span><span class="token punctuation">}</span> <span class="token number">17.3561329</span>
|
|
flagger_canary_duration_seconds_count<span class="token punctuation">{</span>name<span class="token operator">=</span><span class="token string">"podinfo"</span>,namespace<span class="token operator">=</span><span class="token string">"test"</span><span class="token punctuation">}</span> <span class="token number">6</span>
|
|
</code></pre></div></div> <footer class="page-edit"><!----> <!----></footer> <div class="page-nav"><p class="inner"><span class="prev">
|
|
←
|
|
<a href="/usage/alerting.html" class="prev">
|
|
Alerting
|
|
</a></span> <span class="next"><a href="/tutorials/istio-progressive-delivery.html">
|
|
Istio Canaries
|
|
</a>
|
|
→
|
|
</span></p></div> </main></div><div class="global-ui"></div></div>
|
|
<script src="/assets/js/app.67d16e10.js" defer></script><script src="/assets/js/2.ba5bf77c.js" defer></script><script src="/assets/js/34.5a071e36.js" defer></script>
|
|
</body>
|
|
</html>
|