mirror of
https://github.com/fluxcd/flagger.git
synced 2026-04-15 06:57:34 +00:00
Merge pull request #670 from o11n/feature-Skipper
Skipper Ingress Controller support
This commit is contained in:
Stefan Prodan
@@ -143,6 +143,18 @@ jobs:
|
||||
- run: test/e2e-contour.sh
|
||||
- run: test/e2e-contour-tests.sh
|
||||
|
||||
e2e-skipper-testing:
|
||||
machine: true
|
||||
steps:
|
||||
- checkout
|
||||
- attach_workspace:
|
||||
at: /tmp/bin
|
||||
- run: test/container-build.sh
|
||||
- run: test/e2e-kind.sh
|
||||
- run: test/e2e-skipper.sh
|
||||
- run: test/e2e-skipper-tests.sh
|
||||
- run: test/e2e-skipper-cleanup.sh
|
||||
|
||||
push-helm-charts:
|
||||
docker:
|
||||
- image: circleci/golang:1.14
|
||||
@@ -212,6 +224,9 @@ workflows:
|
||||
- e2e-contour-testing:
|
||||
requires:
|
||||
- build-binary
|
||||
- e2e-skipper-testing:
|
||||
requires:
|
||||
- build-binary
|
||||
- push-container:
|
||||
requires:
|
||||
- build-binary
|
||||
@@ -220,6 +235,7 @@ workflows:
|
||||
- e2e-gloo-testing
|
||||
- e2e-nginx-testing
|
||||
- e2e-linkerd-testing
|
||||
- e2e-skipper-testing
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
|
||||
@@ -10,4 +10,5 @@ redirects:
|
||||
usage/contour-progressive-delivery: tutorials/contour-progressive-delivery.md
|
||||
usage/gloo-progressive-delivery: tutorials/gloo-progressive-delivery.md
|
||||
usage/nginx-progressive-delivery: tutorials/nginx-progressive-delivery.md
|
||||
usage/skipper-progressive-delivery: tutorials/skipper-progressive-delivery.md
|
||||
usage/crossover-progressive-delivery: tutorials/crossover-progressive-delivery.md
|
||||
|
||||
@@ -13,7 +13,7 @@ by gradually shifting traffic to the new version while measuring metrics and run
|
||||
|
||||
|
||||
Flagger implements several deployment strategies (Canary releases, A/B testing, Blue/Green mirroring)
|
||||
using a service mesh (App Mesh, Istio, Linkerd) or an ingress controller (Contour, Gloo, NGINX) for traffic routing.
|
||||
using a service mesh (App Mesh, Istio, Linkerd) or an ingress controller (Contour, Gloo, NGINX, Skipper) for traffic routing.
|
||||
For release analysis, Flagger can query Prometheus, Datadog or CloudWatch
|
||||
and for alerting it uses Slack, MS Teams, Discord and Rocket.
|
||||
|
||||
@@ -37,6 +37,7 @@ Flagger documentation can be found at [docs.flagger.app](https://docs.flagger.ap
|
||||
* [Contour](https://docs.flagger.app/tutorials/contour-progressive-delivery)
|
||||
* [Gloo](https://docs.flagger.app/tutorials/gloo-progressive-delivery)
|
||||
* [NGINX Ingress](https://docs.flagger.app/tutorials/nginx-progressive-delivery)
|
||||
* [Skipper](https://docs.flagger.app/tutorials/skipper-progressive-delivery)
|
||||
* [Kubernetes Blue/Green](https://docs.flagger.app/tutorials/kubernetes-blue-green)
|
||||
|
||||
### Who is using Flagger
|
||||
@@ -71,7 +72,7 @@ metadata:
|
||||
namespace: test
|
||||
spec:
|
||||
# service mesh provider (optional)
|
||||
# can be: kubernetes, istio, linkerd, appmesh, nginx, contour, gloo, supergloo
|
||||
# can be: kubernetes, istio, linkerd, appmesh, nginx, skipper, contour, gloo, supergloo
|
||||
provider: istio
|
||||
# deployment reference
|
||||
targetRef:
|
||||
@@ -180,17 +181,17 @@ For more details on how the canary analysis and promotion works please [read the
|
||||
|
||||
### Features
|
||||
|
||||
| Feature | Istio | Linkerd | App Mesh | NGINX | Gloo | Contour | CNI |
|
||||
| -------------------------------------------- | ------------------ | ------------------ |------------------ |------------------ |------------------ |------------------ |------------------ |
|
||||
| Canary deployments (weighted traffic) | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_minus_sign: |
|
||||
| A/B testing (headers and cookies routing) | :heavy_check_mark: | :heavy_minus_sign: | :heavy_check_mark: | :heavy_check_mark: | :heavy_minus_sign: | :heavy_check_mark: | :heavy_minus_sign: |
|
||||
| Blue/Green deployments (traffic switch) | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: |
|
||||
| Webhooks (acceptance/load testing) | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: |
|
||||
| Manual gating (approve/pause/resume) | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: |
|
||||
| Request success rate check (L7 metric) | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_minus_sign: |
|
||||
| Request duration check (L7 metric) | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_minus_sign: |
|
||||
| Custom metric checks | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: |
|
||||
| Traffic policy, CORS, retries and timeouts | :heavy_check_mark: | :heavy_minus_sign: | :heavy_minus_sign: | :heavy_minus_sign: | :heavy_minus_sign: | :heavy_check_mark: | :heavy_minus_sign: |
|
||||
| Feature | Istio | Linkerd | App Mesh | NGINX | Skipper | Gloo | Contour | CNI |
|
||||
| ------------------------------------------ | ------------------ | ------------------ | ------------------ | ------------------ | ------------------ | ------------------ | ------------------ | ------------------ |
|
||||
| Canary deployments (weighted traffic) | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_minus_sign: |
|
||||
| A/B testing (headers and cookies routing) | :heavy_check_mark: | :heavy_minus_sign: | :heavy_check_mark: | :heavy_check_mark: | :heavy_minus_sign: | :heavy_minus_sign: | :heavy_check_mark: | :heavy_minus_sign: |
|
||||
| Blue/Green deployments (traffic switch) | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_minus_sign: | :heavy_check_mark: | :heavy_check_mark: |
|
||||
| Webhooks (acceptance/load testing) | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: |
|
||||
| Manual gating (approve/pause/resume) | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: |
|
||||
| Request success rate check (L7 metric) | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_minus_sign: |
|
||||
| Request duration check (L7 metric) | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_minus_sign: |
|
||||
| Custom metric checks | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: |
|
||||
| Traffic policy, CORS, retries and timeouts | :heavy_check_mark: | :heavy_minus_sign: | :heavy_minus_sign: | :heavy_minus_sign: | :heavy_check_mark: | :heavy_minus_sign: | :heavy_check_mark: | :heavy_minus_sign: |
|
||||
|
||||
### Roadmap
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@ Flagger can run automated application analysis, testing, promotion and rollback
|
||||
* A/B Testing (HTTP headers and cookies traffic routing)
|
||||
* Blue/Green (traffic switching and mirroring)
|
||||
|
||||
Flagger works with service mesh solutions (Istio, Linkerd, AWS App Mesh) and with Kubernetes ingress controllers (NGINX, Gloo, Contour).
|
||||
Flagger works with service mesh solutions (Istio, Linkerd, AWS App Mesh) and with Kubernetes ingress controllers (NGINX, Skipper, Gloo, Contour).
|
||||
Flagger can be configured to send alerts to various chat platforms such as Slack, Microsoft Teams, Discord and Rocket.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
+1
-1
@@ -78,7 +78,7 @@ func init() {
|
||||
flag.BoolVar(&zapReplaceGlobals, "zap-replace-globals", false, "Whether to change the logging level of the global zap logger.")
|
||||
flag.StringVar(&zapEncoding, "zap-encoding", "json", "Zap logger encoding.")
|
||||
flag.StringVar(&namespace, "namespace", "", "Namespace that flagger would watch canary object.")
|
||||
flag.StringVar(&meshProvider, "mesh-provider", "istio", "Service mesh provider, can be istio, linkerd, appmesh, contour, gloo or nginx.")
|
||||
flag.StringVar(&meshProvider, "mesh-provider", "istio", "Service mesh provider, can be istio, linkerd, appmesh, contour, gloo, nginx or skipper.")
|
||||
flag.StringVar(&selectorLabels, "selector-labels", "app,name,app.kubernetes.io/name", "List of pod labels that Flagger uses to create pod selectors.")
|
||||
flag.StringVar(&ingressAnnotationsPrefix, "ingress-annotations-prefix", "nginx.ingress.kubernetes.io", "Annotations prefix for NGINX ingresses.")
|
||||
flag.StringVar(&ingressClass, "ingress-class", "", "Ingress class used for annotating HTTPProxy objects.")
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 47 KiB |
@@ -5,7 +5,7 @@ description: Flagger is a progressive delivery Kubernetes operator
|
||||
# Introduction
|
||||
|
||||
[Flagger](https://github.com/weaveworks/flagger) is a **Kubernetes** operator that automates the promotion of
|
||||
canary deployments using **Istio**, **Linkerd**, **App Mesh**, **NGINX**, **Contour** or **Gloo** routing for
|
||||
canary deployments using **Istio**, **Linkerd**, **App Mesh**, **NGINX**, **Skipper**, **Contour** or **Gloo** routing for
|
||||
traffic shifting and **Prometheus** metrics for canary analysis. The canary analysis can be extended with webhooks for
|
||||
running system integration/acceptance tests, load tests, or any other custom validation.
|
||||
|
||||
@@ -39,6 +39,7 @@ After install Flagger, you can follow one of the tutorials:
|
||||
* [Contour](tutorials/contour-progressive-delivery.md)
|
||||
* [Gloo](tutorials/gloo-progressive-delivery.md)
|
||||
* [NGINX Ingress](tutorials/nginx-progressive-delivery.md)
|
||||
* [Skipper Ingress](tutorials/skipper-progressive-delivery.md)
|
||||
|
||||
**Hands-on GitOps workshops**
|
||||
|
||||
|
||||
@@ -25,6 +25,7 @@
|
||||
* [Linkerd Canary Deployments](tutorials/linkerd-progressive-delivery.md)
|
||||
* [App Mesh Canary Deployments](tutorials/appmesh-progressive-delivery.md)
|
||||
* [NGINX Canary Deployments](tutorials/nginx-progressive-delivery.md)
|
||||
* [Skipper Canary Deployments](tutorials/skipper-progressive-delivery.md)
|
||||
* [Gloo Canary Deployments](tutorials/gloo-progressive-delivery.md)
|
||||
* [Contour Canary Deployments](tutorials/contour-progressive-delivery.md)
|
||||
* [Blue/Green Deployments](tutorials/kubernetes-blue-green.md)
|
||||
|
||||
@@ -77,6 +77,7 @@ For ingress controllers, the install instructions are:
|
||||
* [Contour](https://docs.flagger.app/tutorials/contour-progressive-delivery)
|
||||
* [Gloo](https://docs.flagger.app/tutorials/gloo-progressive-delivery)
|
||||
* [NGINX](https://docs.flagger.app/tutorials/nginx-progressive-delivery)
|
||||
* [Skipper](https://docs.flagger.app/tutorials/skipper-progressive-delivery)
|
||||
|
||||
Enable **Slack** notifications:
|
||||
|
||||
@@ -198,7 +199,7 @@ kustomize build https://github.com/weaveworks/flagger/kustomize/linkerd?ref=v1.0
|
||||
|
||||
**Generic installer**
|
||||
|
||||
Install Flagger and Prometheus for Contour, Gloo or NGINX ingress:
|
||||
Install Flagger and Prometheus for Contour, Gloo, NGINX or Skipper ingress:
|
||||
|
||||
```bash
|
||||
kustomize build https://github.com/weaveworks/flagger/kustomize/kubernetes | kubectl apply -f -
|
||||
@@ -219,7 +220,7 @@ metadata:
|
||||
name: app
|
||||
namespace: test
|
||||
spec:
|
||||
# can be: kubernetes, istio, linkerd, appmesh, nginx, gloo
|
||||
# can be: kubernetes, istio, linkerd, appmesh, nginx, skipper, gloo
|
||||
# use the kubernetes provider for Blue/Green style deployments
|
||||
provider: nginx
|
||||
```
|
||||
|
||||
@@ -0,0 +1,383 @@
|
||||
# Skipper Canary Deployments
|
||||
|
||||
This guide shows you how to use the [Skipper ingress controller](https://opensource.zalando.com/skipper/kubernetes/ingress-controller/) and Flagger to automate canary deployments.
|
||||
|
||||
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Flagger requires a Kubernetes cluster **v1.14** or newer and Skipper ingress **0.11.40** or newer.
|
||||
|
||||
Install Skipper ingress-controller using [upstream definition](https://opensource.zalando.com/skipper/kubernetes/ingress-controller/#install-skipper-as-ingress-controller).
|
||||
|
||||
Certain arguments are relevant:
|
||||
|
||||
```yaml
|
||||
- -enable-connection-metrics
|
||||
- -histogram-metric-buckets=.01,1,10,100
|
||||
- -kubernetes
|
||||
- -kubernetes-in-cluster
|
||||
- -kubernetes-path-mode=path-prefix
|
||||
- -metrics-exp-decay-sample
|
||||
- -metrics-flavour=prometheus
|
||||
- -route-backend-metrics
|
||||
- -route-backend-error-counters
|
||||
- -route-response-metrics
|
||||
- -serve-host-metrics
|
||||
- -serve-route-metrics
|
||||
- -whitelisted-healthcheck-cidr=0.0.0.0/0 # permit Kind source health checks
|
||||
```
|
||||
|
||||
Install Flagger using kustomize:
|
||||
|
||||
```bash
|
||||
kustomize build https://github.com/weaveworks/flagger/kustomize/kubernetes | kubectl apply -f -
|
||||
```
|
||||
|
||||
## Bootstrap
|
||||
|
||||
Flagger takes a Kubernetes deployment and optionally a horizontal pod autoscaler (HPA),
|
||||
then creates a series of objects (Kubernetes deployments, ClusterIP services and canary ingress).
|
||||
These objects expose the application outside the cluster and drive the canary analysis and promotion.
|
||||
|
||||
Create a test namespace:
|
||||
|
||||
```bash
|
||||
kubectl create ns test
|
||||
```
|
||||
|
||||
Create a deployment and a horizontal pod autoscaler:
|
||||
|
||||
```bash
|
||||
kubectl apply -k github.com/weaveworks/flagger//kustomize/podinfo
|
||||
```
|
||||
|
||||
Deploy the load testing service to generate traffic during the canary analysis:
|
||||
|
||||
```bash
|
||||
helm upgrade -i flagger-loadtester flagger/loadtester \
|
||||
--namespace=test
|
||||
```
|
||||
|
||||
Create an ingress definition \(replace `app.example.com` with your own domain\):
|
||||
|
||||
```yaml
|
||||
apiVersion: networking.k8s.io/v1beta1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: podinfo
|
||||
namespace: test
|
||||
labels:
|
||||
app: podinfo
|
||||
annotations:
|
||||
kubernetes.io/ingress.class: "skipper"
|
||||
spec:
|
||||
rules:
|
||||
- host: app.example.com
|
||||
http:
|
||||
paths:
|
||||
- backend:
|
||||
serviceName: podinfo
|
||||
servicePort: 80
|
||||
```
|
||||
|
||||
Save the above resource as podinfo-ingress.yaml and then apply it:
|
||||
|
||||
```bash
|
||||
kubectl apply -f ./podinfo-ingress.yaml
|
||||
```
|
||||
|
||||
Create a canary custom resource \(replace `app.example.com` with your own domain\):
|
||||
|
||||
```yaml
|
||||
apiVersion: flagger.app/v1beta1
|
||||
kind: Canary
|
||||
metadata:
|
||||
name: podinfo
|
||||
namespace: test
|
||||
spec:
|
||||
provider: skipper
|
||||
# deployment reference
|
||||
targetRef:
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
name: podinfo
|
||||
# ingress reference
|
||||
ingressRef:
|
||||
apiVersion: networking.k8s.io/v1beta1
|
||||
kind: Ingress
|
||||
name: podinfo
|
||||
# HPA reference (optional)
|
||||
autoscalerRef:
|
||||
apiVersion: autoscaling/v2beta1
|
||||
kind: HorizontalPodAutoscaler
|
||||
name: podinfo
|
||||
# the maximum time in seconds for the canary deployment
|
||||
# to make progress before it is rollback (default 600s)
|
||||
progressDeadlineSeconds: 60
|
||||
service:
|
||||
# ClusterIP port number
|
||||
port: 80
|
||||
# container port number or name
|
||||
targetPort: 9898
|
||||
analysis:
|
||||
# schedule interval (default 60s)
|
||||
interval: 10s
|
||||
# max number of failed metric checks before rollback
|
||||
threshold: 10
|
||||
# max traffic percentage routed to canary
|
||||
# percentage (0-100)
|
||||
maxWeight: 50
|
||||
# canary increment step
|
||||
# percentage (0-100)
|
||||
stepWeight: 5
|
||||
# Skipper Prometheus checks
|
||||
metrics:
|
||||
- name: request-success-rate
|
||||
interval: 1m
|
||||
# minimum req success rate (non 5xx responses)
|
||||
# percentage (0-100)
|
||||
thresholdRange:
|
||||
min: 99
|
||||
- name: request-duration
|
||||
interval: 1m
|
||||
# maximum req duration P99
|
||||
# milliseconds
|
||||
thresholdRange:
|
||||
max: 500
|
||||
webhooks:
|
||||
- name: gate
|
||||
type: confirm-rollout
|
||||
url: http://flagger-loadtester.test/gate/approve
|
||||
- name: acceptance-test
|
||||
type: pre-rollout
|
||||
url: http://flagger-loadtester.test/
|
||||
timeout: 10s
|
||||
metadata:
|
||||
type: bash
|
||||
cmd: "curl -sd 'test' http://podinfo-canary/token | grep token"
|
||||
- name: load-test
|
||||
type: rollout
|
||||
url: http://flagger-loadtester.test/
|
||||
timeout: 5s
|
||||
metadata:
|
||||
type: cmd
|
||||
cmd: "hey -z 10m -q 10 -c 2 -host app.example.com http://skipper-ingress.kube-system"
|
||||
logCmdOutput: "true"
|
||||
```
|
||||
|
||||
Save the above resource as podinfo-canary.yaml and then apply it:
|
||||
|
||||
```bash
|
||||
kubectl apply -f ./podinfo-canary.yaml
|
||||
```
|
||||
|
||||
After a couple of seconds Flagger will create the canary objects:
|
||||
|
||||
```bash
|
||||
# applied
|
||||
deployment.apps/podinfo
|
||||
horizontalpodautoscaler.autoscaling/podinfo
|
||||
ingress.networking.k8s.io/podinfo-ingress
|
||||
canary.flagger.app/podinfo
|
||||
|
||||
# generated
|
||||
deployment.apps/podinfo-primary
|
||||
horizontalpodautoscaler.autoscaling/podinfo-primary
|
||||
service/podinfo
|
||||
service/podinfo-canary
|
||||
service/podinfo-primary
|
||||
ingress.networking.k8s.io/podinfo-canary
|
||||
```
|
||||
|
||||
## Automated canary promotion
|
||||
|
||||
Flagger implements a control loop that gradually shifts traffic to the canary while measuring
|
||||
key performance indicators like HTTP requests success rate, requests average duration and pod health.
|
||||
Based on analysis of the KPIs a canary is promoted or aborted, and the analysis result is published to Slack or MS Teams.
|
||||
|
||||
|
||||
|
||||
Trigger a canary deployment by updating the container image:
|
||||
|
||||
```bash
|
||||
kubectl -n test set image deployment/podinfo \
|
||||
podinfod=stefanprodan/podinfo:4.0.6
|
||||
```
|
||||
|
||||
Flagger detects that the deployment revision changed and starts a new rollout:
|
||||
|
||||
```text
|
||||
kubectl -n test describe canary/podinfo
|
||||
|
||||
Status:
|
||||
Canary Weight: 0
|
||||
Failed Checks: 0
|
||||
Phase: Succeeded
|
||||
Events:
|
||||
New revision detected! Scaling up podinfo.test
|
||||
Waiting for podinfo.test rollout to finish: 0 of 1 updated replicas are available
|
||||
Pre-rollout check acceptance-test passed
|
||||
Advance podinfo.test canary weight 5
|
||||
Advance podinfo.test canary weight 10
|
||||
Advance podinfo.test canary weight 15
|
||||
Advance podinfo.test canary weight 20
|
||||
Advance podinfo.test canary weight 25
|
||||
Advance podinfo.test canary weight 30
|
||||
Advance podinfo.test canary weight 35
|
||||
Advance podinfo.test canary weight 40
|
||||
Advance podinfo.test canary weight 45
|
||||
Advance podinfo.test canary weight 50
|
||||
Copying podinfo.test template spec to podinfo-primary.test
|
||||
Waiting for podinfo-primary.test rollout to finish: 1 of 2 updated replicas are available
|
||||
Routing all traffic to primary
|
||||
Promotion completed! Scaling down podinfo.test
|
||||
```
|
||||
|
||||
**Note** that if you apply new changes to the deployment during the canary analysis, Flagger will restart the analysis.
|
||||
|
||||
You can monitor all canaries with:
|
||||
|
||||
```bash
|
||||
watch kubectl get canaries --all-namespaces
|
||||
|
||||
NAMESPACE NAME STATUS WEIGHT LASTTRANSITIONTIME
|
||||
test podinfo-2 Progressing 30 2020-08-14T12:32:12Z
|
||||
test podinfo Succeeded 0 2020-08-14T11:23:88Z
|
||||
```
|
||||
|
||||
## Automated rollback
|
||||
|
||||
During the canary analysis you can generate HTTP 500 errors to test if Flagger pauses and rolls back the faulted version.
|
||||
|
||||
Trigger another canary deployment:
|
||||
|
||||
```bash
|
||||
kubectl -n test set image deployment/podinfo \
|
||||
podinfod=stefanprodan/podinfo:4.0.6
|
||||
```
|
||||
|
||||
Exec into the load tester pod with:
|
||||
|
||||
```bash
|
||||
kubectl -n test exec -it deploy/flagger-loadtester bash
|
||||
```
|
||||
|
||||
Generate HTTP 500 errors:
|
||||
|
||||
```bash
|
||||
hey -z 1m -c 5 -q 5 http://app.example.com/status/500
|
||||
```
|
||||
|
||||
Generate latency:
|
||||
|
||||
```bash
|
||||
watch -n 1 curl http://app.example.com/delay/1
|
||||
```
|
||||
|
||||
When the number of failed checks reaches the canary analysis threshold, the traffic is routed back to the primary,
|
||||
the canary is scaled to zero and the rollout is marked as failed.
|
||||
|
||||
```text
|
||||
kubectl -n flagger-system logs deploy/flagger -f | jq .msg
|
||||
|
||||
"New revision detected! Scaling up podinfo.test"
|
||||
"canary deployment podinfo.test not ready: waiting for rollout to finish: 0 of 1 updated replicas are available"
|
||||
"Starting canary analysis for podinfo.test"
|
||||
"Pre-rollout check acceptance-test passed"
|
||||
"primaryWeight: 95, canaryWeight: 5"
|
||||
"Advance podinfo.test canary weight 5"
|
||||
"primaryWeight: 90, canaryWeight: 10"
|
||||
"Advance podinfo.test canary weight 10"
|
||||
"primaryWeight: 85, canaryWeight: 15"
|
||||
"Advance podinfo.test canary weight 15"
|
||||
"primaryWeight: 80, canaryWeight: 20"
|
||||
"Advance podinfo.test canary weight 20"
|
||||
"Halt podinfo.test advancement success rate 53.42% < 99%"
|
||||
"Halt podinfo.test advancement success rate 53.19% < 99%"
|
||||
"Halt podinfo.test advancement success rate 48.05% < 99%"
|
||||
"Rolling back podinfo.test failed checks threshold reached 3"
|
||||
"primaryWeight: 100, canaryWeight: 0"
|
||||
"Canary failed! Scaling down podinfo.test"
|
||||
```
|
||||
|
||||
## Custom metrics
|
||||
|
||||
The canary analysis can be extended with Prometheus queries.
|
||||
|
||||
The demo app is instrumented with Prometheus so you can create a custom check that will use the
|
||||
HTTP request duration histogram to validate the canary.
|
||||
|
||||
Create a metric template and apply it on the cluster:
|
||||
|
||||
```yaml
|
||||
apiVersion: flagger.app/v1beta1
|
||||
kind: MetricTemplate
|
||||
metadata:
|
||||
name: latency
|
||||
namespace: test
|
||||
spec:
|
||||
provider:
|
||||
type: prometheus
|
||||
address: http://flagger-prometheus.flagger-system:9090
|
||||
query: |
|
||||
histogram_quantile(0.99,
|
||||
sum(
|
||||
rate(
|
||||
skipper_serve_route_duration_seconds_bucket{
|
||||
route=~"{{ printf "kube(ew)?_%s__%s_canary__.*__%s_canary(_[0-9]+)?" namespace ingress service }}",
|
||||
le="+Inf"
|
||||
}[1m]
|
||||
)
|
||||
) by (le)
|
||||
)
|
||||
```
|
||||
|
||||
Edit the canary analysis and add the latency check:
|
||||
|
||||
```yaml
|
||||
analysis:
|
||||
metrics:
|
||||
- name: "latency"
|
||||
templateRef:
|
||||
name: latency
|
||||
thresholdRange:
|
||||
max: 0.5
|
||||
interval: 1m
|
||||
```
|
||||
|
||||
The threshold is set to 500ms so if the average request duration in the last minute goes over half a second
|
||||
then the analysis will fail and the canary will not be promoted.
|
||||
|
||||
Trigger a canary deployment by updating the container image:
|
||||
|
||||
```bash
|
||||
kubectl -n test set image deployment/podinfo \
|
||||
podinfod=stefanprodan/podinfo:4.0.6
|
||||
```
|
||||
|
||||
Generate high response latency:
|
||||
|
||||
```bash
|
||||
watch curl http://app.exmaple.com/delay/2
|
||||
```
|
||||
|
||||
Watch Flagger logs:
|
||||
|
||||
```text
|
||||
kubectl -n flagger-system logs deployment/flagger -f | jq .msg
|
||||
|
||||
Starting canary deployment for podinfo.test
|
||||
Advance podinfo.test canary weight 5
|
||||
Advance podinfo.test canary weight 10
|
||||
Advance podinfo.test canary weight 15
|
||||
Halt podinfo.test advancement latency 1.20 > 0.5
|
||||
Halt podinfo.test advancement latency 1.45 > 0.5
|
||||
Halt podinfo.test advancement latency 1.60 > 0.5
|
||||
Halt podinfo.test advancement latency 1.69 > 0.5
|
||||
Halt podinfo.test advancement latency 1.70 > 0.5
|
||||
Rolling back podinfo.test failed checks threshold reached 5
|
||||
Canary failed! Scaling down podinfo.test
|
||||
```
|
||||
|
||||
If you have alerting configured, Flagger will send a notification with the reason why the canary failed.
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
Flagger can run automated application analysis, promotion and rollback for the following deployment strategies:
|
||||
* **Canary Release** (progressive traffic shifting)
|
||||
* Istio, Linkerd, App Mesh, NGINX, Contour, Gloo
|
||||
* Istio, Linkerd, App Mesh, NGINX, Skipper, Contour, Gloo
|
||||
* **A/B Testing** (HTTP headers and cookies traffic routing)
|
||||
* Istio, App Mesh, NGINX, Contour
|
||||
* **Blue/Green** (traffic switching)
|
||||
|
||||
@@ -10,6 +10,8 @@ require (
|
||||
github.com/prometheus/client_golang v1.5.1
|
||||
github.com/stretchr/testify v1.5.1
|
||||
go.uber.org/zap v1.14.1
|
||||
golang.org/x/net v0.0.0-20200301022130-244492dfa37a // indirect
|
||||
golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d // indirect
|
||||
gopkg.in/h2non/gock.v1 v1.0.15
|
||||
k8s.io/api v0.18.8
|
||||
k8s.io/apimachinery v0.18.8
|
||||
|
||||
@@ -188,6 +188,7 @@ github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLk
|
||||
github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
|
||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
|
||||
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
|
||||
github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
|
||||
github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
|
||||
@@ -220,6 +221,7 @@ golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnf
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975 h1:/Tl7pH94bvbAAHBdZJT947M/+gp0+CqQXDtMRC0fseo=
|
||||
golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
@@ -229,6 +231,8 @@ golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTk
|
||||
golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs=
|
||||
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
|
||||
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee h1:WG0RUwxtNT4qqaXX3DPA8zHFNm/D9xaBpxzHt1WcA/E=
|
||||
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
|
||||
golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
@@ -245,6 +249,8 @@ golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 h1:rjwSpXsdiK0dV8/Naq3kAw9ym
|
||||
golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
|
||||
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0=
|
||||
golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
@@ -293,7 +299,10 @@ golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtn
|
||||
golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5 h1:hKsoRgsbwY1NafxrwTs+k64bikrLBkAgPir1TNCj3Zs=
|
||||
golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d h1:7M9AXzLrJWWGdDYtBblPHBTnHtaN6KKQ98OYb35mLlY=
|
||||
golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
|
||||
|
||||
+2
-2
@@ -67,12 +67,12 @@ metadata:
|
||||
name: app
|
||||
namespace: test
|
||||
spec:
|
||||
# can be: kubernetes, istio, linkerd, appmesh, nginx, gloo
|
||||
# can be: kubernetes, istio, linkerd, appmesh, nginx, skipper, gloo
|
||||
# use the kubernetes provider for Blue/Green style deployments
|
||||
provider: nginx
|
||||
```
|
||||
|
||||
You'll need Prometheus when using Flagger with AWS App Mesh, Gloo or NGINX ingress controller.
|
||||
You'll need Prometheus when using Flagger with AWS App Mesh, Gloo, NGINX or Skipper ingress controller.
|
||||
The Prometheus instance has a two hours data retention and is configured to scrape all pods in your cluster that
|
||||
have the `prometheus.io/scrape: "true"` annotation.
|
||||
|
||||
|
||||
@@ -9,4 +9,5 @@ const (
|
||||
GlooProvider string = "gloo"
|
||||
NGINXProvider string = "nginx"
|
||||
KubernetesProvider string = "kubernetes"
|
||||
SkipperProvider string = "skipper"
|
||||
)
|
||||
|
||||
@@ -56,6 +56,10 @@ func (factory Factory) Observer(provider string) Interface {
|
||||
return &HttpObserver{
|
||||
client: factory.Client,
|
||||
}
|
||||
case provider == flaggerv1.SkipperProvider:
|
||||
return &SkipperObserver{
|
||||
client: factory.Client,
|
||||
}
|
||||
default:
|
||||
return &IstioObserver{
|
||||
client: factory.Client,
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
package observers
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"time"
|
||||
|
||||
flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1beta1"
|
||||
"github.com/weaveworks/flagger/pkg/logger"
|
||||
|
||||
"github.com/weaveworks/flagger/pkg/metrics/providers"
|
||||
)
|
||||
|
||||
const routePattern = `{{- $route := printf "kube(ew)?_%s__%s_canary__.*__%s_canary(_[0-9]+)?" namespace ingress service }}`
|
||||
|
||||
var skipperQueries = map[string]string{
|
||||
"request-success-rate": routePattern + `
|
||||
sum(rate(skipper_response_duration_seconds_bucket{route=~"{{ $route }}",code!~"5..",le="+Inf"}[{{ interval }}])) /
|
||||
sum(rate(skipper_response_duration_seconds_bucket{route=~"{{ $route }}",le="+Inf"}[{{ interval }}])) * 100`,
|
||||
"request-duration": routePattern + `
|
||||
sum(rate(skipper_serve_route_duration_seconds_sum{route=~"{{ $route }}"}[{{ interval }}])) /
|
||||
sum(rate(skipper_serve_route_duration_seconds_count{route=~"{{ $route }}"}[{{ interval }}])) * 1000`,
|
||||
}
|
||||
|
||||
// SkipperObserver Implementation for Skipper (https://github.com/zalando/skipper)
|
||||
type SkipperObserver struct {
|
||||
client providers.Interface
|
||||
}
|
||||
|
||||
// GetRequestSuccessRate return value for Skipper Request Success Rate
|
||||
func (ob *SkipperObserver) GetRequestSuccessRate(model flaggerv1.MetricTemplateModel) (float64, error) {
|
||||
|
||||
model = encodeModelForSkipper(model)
|
||||
|
||||
query, err := RenderQuery(skipperQueries["request-success-rate"], model)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("rendering query failed: %w", err)
|
||||
}
|
||||
logger, _ := logger.NewLoggerWithEncoding("debug", "json")
|
||||
logger.Debugf("GetRequestSuccessRate: %s", query)
|
||||
|
||||
value, err := ob.client.RunQuery(query)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("running query failed: %w", err)
|
||||
}
|
||||
|
||||
return value, nil
|
||||
}
|
||||
|
||||
// GetRequestDuration return value for Skipper Request Duration
|
||||
func (ob *SkipperObserver) GetRequestDuration(model flaggerv1.MetricTemplateModel) (time.Duration, error) {
|
||||
|
||||
model = encodeModelForSkipper(model)
|
||||
|
||||
query, err := RenderQuery(skipperQueries["request-duration"], model)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("rendering query failed: %w", err)
|
||||
}
|
||||
logger, _ := logger.NewLoggerWithEncoding("debug", "json")
|
||||
logger.Debugf("GetRequestDuration: %s", query)
|
||||
|
||||
value, err := ob.client.RunQuery(query)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("running query failed: %w", err)
|
||||
}
|
||||
|
||||
ms := time.Duration(int64(value)) * time.Millisecond
|
||||
return ms, nil
|
||||
}
|
||||
|
||||
// encodeModelForSkipper replaces non word character in model with underscore to match route names
|
||||
// https://github.com/zalando/skipper/blob/dd70bd65e7f99cfb5dd6b6f71885d9fe3b2707f6/dataclients/kubernetes/ingress.go#L101
|
||||
func encodeModelForSkipper(model flaggerv1.MetricTemplateModel) flaggerv1.MetricTemplateModel {
|
||||
nonWord := regexp.MustCompile(`\W`)
|
||||
model.Ingress = nonWord.ReplaceAllString(model.Ingress, "_")
|
||||
model.Name = nonWord.ReplaceAllString(model.Name, "_")
|
||||
model.Namespace = nonWord.ReplaceAllString(model.Namespace, "_")
|
||||
model.Service = nonWord.ReplaceAllString(model.Service, "_")
|
||||
model.Target = nonWord.ReplaceAllString(model.Target, "_")
|
||||
|
||||
return model
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
package observers
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1beta1"
|
||||
"github.com/weaveworks/flagger/pkg/metrics/providers"
|
||||
)
|
||||
|
||||
func TestSkipperObserver_GetRequestSuccessRate(t *testing.T) {
|
||||
t.Run("ok", func(t *testing.T) {
|
||||
expected := ` sum(rate(skipper_response_duration_seconds_bucket{route=~"kube(ew)?_skipper__skipper_ingress_canary__.*__backend_canary(_[0-9]+)?",code!~"5..",le="+Inf"}[1m])) / sum(rate(skipper_response_duration_seconds_bucket{route=~"kube(ew)?_skipper__skipper_ingress_canary__.*__backend_canary(_[0-9]+)?",le="+Inf"}[1m])) * 100`
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
promql := r.URL.Query()["query"][0]
|
||||
assert.Equal(t, expected, promql)
|
||||
|
||||
json := `{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1,"100"]}]}}`
|
||||
w.Write([]byte(json))
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
client, err := providers.NewPrometheusProvider(flaggerv1.MetricTemplateProvider{
|
||||
Type: "prometheus",
|
||||
Address: ts.URL,
|
||||
SecretRef: nil,
|
||||
}, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
observer := &SkipperObserver{client: client}
|
||||
val, err := observer.GetRequestSuccessRate(flaggerv1.MetricTemplateModel{
|
||||
Namespace: "skipper",
|
||||
Interval: "1m",
|
||||
Service: "backend",
|
||||
Ingress: "skipper-ingress",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, float64(100), val)
|
||||
})
|
||||
|
||||
t.Run("no values", func(t *testing.T) {
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
json := `{"status":"success","data":{"resultType":"vector","result":[]}}`
|
||||
w.Write([]byte(json))
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
client, err := providers.NewPrometheusProvider(flaggerv1.MetricTemplateProvider{
|
||||
Type: "prometheus",
|
||||
Address: ts.URL,
|
||||
SecretRef: nil,
|
||||
}, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
observer := &SkipperObserver{client: client}
|
||||
_, err = observer.GetRequestSuccessRate(flaggerv1.MetricTemplateModel{})
|
||||
require.True(t, errors.Is(err, providers.ErrNoValuesFound))
|
||||
})
|
||||
}
|
||||
|
||||
func TestSkipperObserver_GetRequestDuration(t *testing.T) {
|
||||
expected := ` sum(rate(skipper_serve_route_duration_seconds_sum{route=~"kube(ew)?_skipper__skipper_ingress_canary__.*__backend_canary(_[0-9]+)?"}[1m])) / sum(rate(skipper_serve_route_duration_seconds_count{route=~"kube(ew)?_skipper__skipper_ingress_canary__.*__backend_canary(_[0-9]+)?"}[1m])) * 1000`
|
||||
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
promql := r.URL.Query()["query"][0]
|
||||
assert.Equal(t, expected, promql)
|
||||
|
||||
json := `{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1,"100"]}]}}`
|
||||
w.Write([]byte(json))
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
client, err := providers.NewPrometheusProvider(flaggerv1.MetricTemplateProvider{
|
||||
Type: "prometheus",
|
||||
Address: ts.URL,
|
||||
SecretRef: nil,
|
||||
}, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
observer := &SkipperObserver{client: client}
|
||||
val, err := observer.GetRequestDuration(flaggerv1.MetricTemplateModel{
|
||||
Namespace: "skipper",
|
||||
Interval: "1m",
|
||||
Service: "backend",
|
||||
Ingress: "skipper-ingress",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, 100*time.Millisecond, val)
|
||||
}
|
||||
@@ -122,6 +122,11 @@ func (factory *Factory) MeshRouter(provider string, labelSelector string) Interf
|
||||
kubeClient: factory.kubeClient,
|
||||
annotationsPrefix: factory.ingressAnnotationsPrefix,
|
||||
}
|
||||
case provider == flaggerv1.SkipperProvider:
|
||||
return &SkipperRouter{
|
||||
logger: factory.logger,
|
||||
kubeClient: factory.kubeClient,
|
||||
}
|
||||
case provider == flaggerv1.KubernetesProvider:
|
||||
return &NopRouter{}
|
||||
default:
|
||||
|
||||
@@ -125,9 +125,6 @@ func (c *KubernetesDefaultRouter) reconcileService(canary *flaggerv1.Canary, nam
|
||||
metadata.Annotations = make(map[string]string)
|
||||
}
|
||||
|
||||
c.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)).
|
||||
Debugw(fmt.Sprintf("Creating Service %s", name), "metadata", metadata, "service_configuration", canary.Spec.Service)
|
||||
|
||||
// create service if it doesn't exists
|
||||
svc, err := c.kubeClient.CoreV1().Services(canary.Namespace).Get(context.TODO(), name, metav1.GetOptions{})
|
||||
if errors.IsNotFound(err) {
|
||||
|
||||
@@ -0,0 +1,235 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"go.uber.org/zap"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
|
||||
flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1beta1"
|
||||
)
|
||||
|
||||
/*
|
||||
Skipper Principles:
|
||||
* if only one backend has a weight, only one backend will get 100% traffic
|
||||
* if two of three or more backends have a weight, only those two should get traffic.
|
||||
* if two backends don't have any weight, it's undefined and right now they get equal amount of traffic.
|
||||
* weights can be int or float, but always treated as a ratio.
|
||||
|
||||
Implementation:
|
||||
* apex Ingress is immutable
|
||||
* new canary Ingress contains two paths for primary and canary service
|
||||
* canary Ingress manages weights on primary & canary service, hence no traffic to apex service
|
||||
|
||||
*/
|
||||
|
||||
const (
|
||||
skipperpredicateAnnotationKey = "zalando.org/skipper-predicate"
|
||||
skipperBackendWeightsAnnotationKey = "zalando.org/backend-weights"
|
||||
canaryPatternf = "%s-canary"
|
||||
canaryRouteWeight = "Weight(100)"
|
||||
canaryRouteDisable = "False()"
|
||||
)
|
||||
|
||||
type SkipperRouter struct {
|
||||
kubeClient kubernetes.Interface
|
||||
logger *zap.SugaredLogger
|
||||
}
|
||||
|
||||
// Reconcile creates or updates the ingresses
|
||||
func (skp *SkipperRouter) Reconcile(canary *flaggerv1.Canary) error {
|
||||
if canary.Spec.IngressRef == nil || canary.Spec.IngressRef.Name == "" {
|
||||
return fmt.Errorf("ingress selector is empty")
|
||||
}
|
||||
|
||||
apexSvcName, primarySvcName, canarySvcName := canary.GetServiceNames()
|
||||
apexIngressName, canaryIngressName := skp.getIngressNames(canary.Spec.IngressRef.Name)
|
||||
|
||||
// retrieving apex ingress
|
||||
apexIngress, err := skp.kubeClient.NetworkingV1beta1().Ingresses(canary.Namespace).Get(
|
||||
context.TODO(), apexIngressName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("apexIngress %s.%s get query error: %w", apexIngressName, canary.Namespace, err)
|
||||
}
|
||||
|
||||
// building the canary ingress from apex
|
||||
iClone := apexIngress.DeepCopy()
|
||||
for x := range iClone.Spec.Rules {
|
||||
rule := &iClone.Spec.Rules[x] // ref not value
|
||||
for y := range rule.HTTP.Paths {
|
||||
path := &rule.HTTP.Paths[y] // ref not value
|
||||
if path.Backend.ServiceName == apexSvcName {
|
||||
// flipping to primary service
|
||||
path.Backend.ServiceName = primarySvcName
|
||||
// adding second canary service
|
||||
canaryBackend := path.DeepCopy()
|
||||
canaryBackend.Backend.ServiceName = canarySvcName
|
||||
rule.HTTP.Paths = append(rule.HTTP.Paths, *canaryBackend)
|
||||
}
|
||||
}
|
||||
}
|
||||
if apexIngress.DeepCopy() == iClone {
|
||||
return fmt.Errorf("backend %s not found in ingress %s", apexSvcName, apexIngressName)
|
||||
}
|
||||
|
||||
iClone.Annotations = skp.makeAnnotations(iClone.Annotations, map[string]int{primarySvcName: 100, canarySvcName: 0})
|
||||
iClone.Name = canaryIngressName
|
||||
iClone.Namespace = canary.Namespace
|
||||
iClone.OwnerReferences = []metav1.OwnerReference{
|
||||
*metav1.NewControllerRef(canary, schema.GroupVersionKind{
|
||||
Group: flaggerv1.SchemeGroupVersion.Group,
|
||||
Version: flaggerv1.SchemeGroupVersion.Version,
|
||||
Kind: flaggerv1.CanaryKind,
|
||||
}),
|
||||
}
|
||||
|
||||
// search for existence
|
||||
canaryIngress, err := skp.kubeClient.NetworkingV1beta1().Ingresses(canary.Namespace).Get(
|
||||
context.TODO(), canaryIngressName, metav1.GetOptions{})
|
||||
|
||||
// new ingress
|
||||
if errors.IsNotFound(err) {
|
||||
// Let K8s set this. Otherwise K8s API complains with "resourceVersion should not be set on objects to be created"
|
||||
iClone.ObjectMeta.ResourceVersion = ""
|
||||
_, err := skp.kubeClient.NetworkingV1beta1().Ingresses(canary.Namespace).Create(context.TODO(), iClone, metav1.CreateOptions{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("ingress %s.%s create error: %w", iClone.Name, iClone.Namespace, err)
|
||||
}
|
||||
skp.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)).
|
||||
Infof("Ingress %s.%s created", iClone.GetName(), canary.Namespace)
|
||||
return nil
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("ingress %s.%s query error: %w", canaryIngressName, canary.Namespace, err)
|
||||
}
|
||||
|
||||
// existant, updating
|
||||
if cmp.Diff(iClone.Spec, canaryIngress.Spec) != "" {
|
||||
ingressClone := canaryIngress.DeepCopy()
|
||||
ingressClone.Spec = iClone.Spec
|
||||
ingressClone.Annotations = iClone.Annotations
|
||||
|
||||
_, err := skp.kubeClient.NetworkingV1beta1().Ingresses(canary.Namespace).Update(context.TODO(), ingressClone, metav1.UpdateOptions{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("ingress %s.%s update error: %w", canaryIngressName, ingressClone.Namespace, err)
|
||||
}
|
||||
skp.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)).
|
||||
Infof("Ingress %s updated", canaryIngressName)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (skp *SkipperRouter) GetRoutes(canary *flaggerv1.Canary) (primaryWeight, canaryWeight int, mirrored bool, err error) {
|
||||
_, primarySvcName, canarySvcName := canary.GetServiceNames()
|
||||
|
||||
_, canaryIngressName := skp.getIngressNames(canary.Spec.IngressRef.Name)
|
||||
canaryIngress, err := skp.kubeClient.NetworkingV1beta1().Ingresses(canary.Namespace).Get(context.TODO(), canaryIngressName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
err = fmt.Errorf("ingress %s.%s get query error: %w", canaryIngressName, canary.Namespace, err)
|
||||
return
|
||||
}
|
||||
|
||||
weights, err := skp.backendWeights(canaryIngress.Annotations)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("ingress %s.%s get backendWeights error: %w", canaryIngressName, canary.Namespace, err)
|
||||
return
|
||||
}
|
||||
var ok bool
|
||||
primaryWeight, ok = weights[primarySvcName]
|
||||
if !ok {
|
||||
err = fmt.Errorf("ingress %s.%s could not get weights[primarySvcName]", canaryIngressName, canary.Namespace)
|
||||
return
|
||||
}
|
||||
canaryWeight, ok = weights[canarySvcName]
|
||||
if !ok {
|
||||
err = fmt.Errorf("ingress %s.%s could not get weights[canarySvcName]", canaryIngressName, canary.Namespace)
|
||||
return
|
||||
}
|
||||
mirrored = false
|
||||
skp.logger.With("GetRoutes", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)).
|
||||
Debugf("GetRoutes primaryWeight: %d, canaryWeight: %d", primaryWeight, canaryWeight)
|
||||
return
|
||||
}
|
||||
|
||||
func (skp *SkipperRouter) SetRoutes(canary *flaggerv1.Canary, primaryWeight, canaryWeight int, _ bool) (err error) {
|
||||
_, primarySvcName, canarySvcName := canary.GetServiceNames()
|
||||
_, canaryIngressName := skp.getIngressNames(canary.Spec.IngressRef.Name)
|
||||
canaryIngress, err := skp.kubeClient.NetworkingV1beta1().Ingresses(canary.Namespace).Get(context.TODO(), canaryIngressName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("ingress %s.%s get query error: %w", canaryIngressName, canary.Namespace, err)
|
||||
}
|
||||
|
||||
iClone := canaryIngress.DeepCopy()
|
||||
|
||||
// TODO: A/B testing
|
||||
|
||||
// Canary
|
||||
iClone.Annotations = skp.makeAnnotations(iClone.Annotations, map[string]int{
|
||||
primarySvcName: primaryWeight,
|
||||
canarySvcName: canaryWeight,
|
||||
})
|
||||
|
||||
// Disable the canary-ingress route after the canary process
|
||||
if canaryWeight == 0 {
|
||||
iClone.Annotations[skipperpredicateAnnotationKey] = canaryRouteDisable
|
||||
}
|
||||
|
||||
_, err = skp.kubeClient.NetworkingV1beta1().Ingresses(canary.Namespace).Update(
|
||||
context.TODO(), iClone, metav1.UpdateOptions{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("ingress %s.%s update error %w", iClone.Name, iClone.Namespace, err)
|
||||
}
|
||||
skp.logger.With("SetRoutes", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)).
|
||||
Debugf("primaryWeight: %d, canaryWeight: %d", primaryWeight, canaryWeight)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func (skp *SkipperRouter) Finalize(canary *flaggerv1.Canary) error {
|
||||
gracePeriodSeconds := int64(2)
|
||||
_, canaryIngressName := skp.getIngressNames(canary.Spec.IngressRef.Name)
|
||||
skp.logger.With("deleteCanaryIngress", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)).
|
||||
Debugf("Deleting Canary Ingress: %s", canaryIngressName)
|
||||
|
||||
err := skp.kubeClient.NetworkingV1beta1().Ingresses(canary.Namespace).Delete(
|
||||
context.TODO(), canaryIngressName, metav1.DeleteOptions{GracePeriodSeconds: &gracePeriodSeconds})
|
||||
if err != nil {
|
||||
return fmt.Errorf("ingress %s.%s unable to remove canary ingress: %w", canaryIngressName, canary.Namespace, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (skp *SkipperRouter) makeAnnotations(annotations map[string]string, backendWeights map[string]int) map[string]string {
|
||||
b, err := json.Marshal(backendWeights)
|
||||
if err != nil {
|
||||
skp.logger.Errorf("Skipper:makeAnnotations: unable to marshal backendWeights %w", err)
|
||||
return annotations
|
||||
}
|
||||
annotations[skipperBackendWeightsAnnotationKey] = string(b)
|
||||
// adding more weight to canary route solves traffic bypassing through apexIngress
|
||||
annotations[skipperpredicateAnnotationKey] = canaryRouteWeight
|
||||
|
||||
return annotations
|
||||
}
|
||||
|
||||
// parse backend-weights annotation if it exists
|
||||
func (skp *SkipperRouter) backendWeights(annotation map[string]string) (backendWeights map[string]int, err error) {
|
||||
backends, ok := annotation[skipperBackendWeightsAnnotationKey]
|
||||
if ok {
|
||||
err = json.Unmarshal([]byte(backends), &backendWeights)
|
||||
} else {
|
||||
err = errors.NewNotFound(schema.GroupResource{Group: "Skipper Canary Ingress", Resource: "Annotation"},
|
||||
skipperBackendWeightsAnnotationKey)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// getIngressNames returns the primary and canary Kubernetes Ingress names
|
||||
func (skp *SkipperRouter) getIngressNames(name string) (apexName, canaryName string) {
|
||||
return name, fmt.Sprintf(canaryPatternf, name)
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
func TestSkipperRouter_Reconcile(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
mocks := newFixture(nil)
|
||||
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
mocks func() fixture
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
"creating new canary ingress w/ default settings",
|
||||
func() fixture { return mocks },
|
||||
false,
|
||||
}, {
|
||||
"updating existing canary ingress",
|
||||
func() fixture {
|
||||
ti := newTestIngress()
|
||||
ti.Annotations["something"] = "changed"
|
||||
_, err := mocks.kubeClient.NetworkingV1beta1().Ingresses("default").Update(
|
||||
context.TODO(), ti, metav1.UpdateOptions{})
|
||||
assert.NoError(err)
|
||||
return mocks
|
||||
},
|
||||
false,
|
||||
},
|
||||
} {
|
||||
tt := tt
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
mocks := tt.mocks()
|
||||
router := &SkipperRouter{
|
||||
kubeClient: mocks.kubeClient,
|
||||
logger: mocks.logger,
|
||||
}
|
||||
assert.NoError(router.Reconcile(mocks.ingressCanary))
|
||||
canaryName := fmt.Sprintf("%s-canary", mocks.ingressCanary.Spec.IngressRef.Name)
|
||||
inCanary, err := router.kubeClient.NetworkingV1beta1().Ingresses("default").Get(
|
||||
context.TODO(), canaryName, metav1.GetOptions{})
|
||||
assert.NoError(err)
|
||||
// test initialisation
|
||||
assert.JSONEq(`{ "podinfo-primary": 100, "podinfo-canary": 0 }`, inCanary.Annotations["zalando.org/backend-weights"])
|
||||
assert.Equal("podinfo-primary", inCanary.Spec.Rules[0].HTTP.Paths[0].Backend.ServiceName, "backend flipped over")
|
||||
assert.Equal("podinfo-canary", inCanary.Spec.Rules[0].HTTP.Paths[1].Backend.ServiceName, "backend flipped over")
|
||||
assert.Len(inCanary.Spec.Rules[0].HTTP.Paths, 2)
|
||||
inApex, err := router.kubeClient.NetworkingV1beta1().Ingresses("default").Get(
|
||||
context.TODO(), mocks.ingressCanary.Spec.IngressRef.Name, metav1.GetOptions{})
|
||||
assert.NoError(err)
|
||||
assert.Equal(inCanary.Spec.Rules[0].HTTP.Paths[0].Backend.ServicePort,
|
||||
inApex.Spec.Rules[0].HTTP.Paths[0].Backend.ServicePort, "canary backend not cloned")
|
||||
assert.Equal(inCanary.Spec.Rules[0].HTTP.Paths[0].Backend.ServicePort,
|
||||
inCanary.Spec.Rules[0].HTTP.Paths[1].Backend.ServicePort, "canary backend not cloned")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSkipperRouter_GetSetRoutes(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
mocks := newFixture(nil)
|
||||
|
||||
router := &SkipperRouter{logger: mocks.logger, kubeClient: mocks.kubeClient}
|
||||
assert.NoError(router.Reconcile(mocks.ingressCanary))
|
||||
|
||||
p, c, m, err := router.GetRoutes(mocks.ingressCanary)
|
||||
assert.NoError(err)
|
||||
assert.Equal(100, p)
|
||||
assert.Equal(0, c)
|
||||
assert.Equal(false, m)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
primary, canary int
|
||||
}{
|
||||
{name: "0%", primary: 100, canary: 0},
|
||||
{name: "10%", primary: 90, canary: 10},
|
||||
{name: "20%", primary: 80, canary: 20},
|
||||
{name: "30%", primary: 70, canary: 30},
|
||||
{name: "85%", primary: 15, canary: 85},
|
||||
{name: "100%", primary: 0, canary: 100},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
tt := tt
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
assert.NoError(router.SetRoutes(mocks.ingressCanary, tt.primary, tt.canary, false))
|
||||
inCanary, err := router.kubeClient.NetworkingV1beta1().Ingresses("default").Get(
|
||||
context.TODO(), fmt.Sprintf("%s-canary", mocks.ingressCanary.Spec.IngressRef.Name), metav1.GetOptions{})
|
||||
assert.NoError(err)
|
||||
assert.JSONEq(fmt.Sprintf(`{"podinfo-primary": %d,"podinfo-canary": %d}`, tt.primary, tt.canary),
|
||||
inCanary.Annotations["zalando.org/backend-weights"])
|
||||
p, c, m, err = router.GetRoutes(mocks.ingressCanary)
|
||||
assert.NoError(err)
|
||||
assert.Equal(tt.primary, p)
|
||||
assert.Equal(tt.canary, c)
|
||||
assert.Equal(false, m)
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
+16
-1
@@ -53,4 +53,19 @@ The e2e testing infrastructure is powered by CircleCI and [Kubernetes Kind](http
|
||||
* test the A/B testing analysis and promotion using header filters and pre/post rollout webhooks [e2e-nginx-tests.sh](e2e-nginx-tests.sh)
|
||||
* cleanup test environment [e2e-nginx-cleanup.sh](e2e-nginx-cleanup.sh)
|
||||
* install NGINX Ingress and Flagger with custom ingress annotations prefix [e2e-nginx-custom-annotations.sh](e2e-nginx-custom-annotations.sh)
|
||||
* repeat the canary and A/B testing workflow [e2e-nginx-tests.sh](e2e-nginx-tests.sh)
|
||||
* repeat the canary and A/B testing workflow [e2e-nginx-tests.sh](e2e-nginx-tests.sh)
|
||||
|
||||
### CircleCI e2e Skipper ingress workflow
|
||||
|
||||
* install latest stable kubectl [e2e-kind.sh](e2e-kind.sh)
|
||||
* install Kubernetes Kind [e2e-kind.sh](e2e-kind.sh)
|
||||
* create local Kubernetes cluster with kind [e2e-kind.sh](e2e-kind.sh)
|
||||
* install Skipper ingress with Kustomize [e2e-skipper.sh](e2e-skipper.sh)
|
||||
* load Flagger image onto the local cluster [e2e-skipper.sh](e2e-skipper.sh)
|
||||
* install Flagger and Prometheus in the flagger-system namespace [e2e-skipper.sh](e2e-skipper.sh)
|
||||
* create a test namespace [e2e-skipper-tests.sh](e2e-skipper-tests.sh)
|
||||
* deploy the load tester in the test namespace [e2e-skipper-tests.sh](e2e-skipper-tests.sh)
|
||||
* deploy the demo workload (podinfo) and ingress in the test namespace [e2e-skipper-tests.sh](e2e-skipper-tests.sh)
|
||||
* test the canary initialization [e2e-skipper-tests.sh](e2e-skipper-tests.sh)
|
||||
* test the canary analysis and promotion using weighted traffic and the load testing webhook [e2e-skipper-tests.sh]e2e-skipper-tests.sh)
|
||||
* cleanup test environment [e2e-skipper-cleanup.sh](e2e-skipper-cleanup.sh)
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
apiVersion: flagger.app/v1beta1
|
||||
kind: Canary
|
||||
metadata:
|
||||
name: podinfo
|
||||
namespace: test
|
||||
spec:
|
||||
provider: skipper
|
||||
progressDeadlineSeconds: 120
|
||||
revertOnDeletion: true
|
||||
targetRef:
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
name: podinfo
|
||||
ingressRef:
|
||||
apiVersion: networking.k8s.io/v1beta1
|
||||
kind: Ingress
|
||||
name: podinfo-ingress
|
||||
service:
|
||||
# service name (defaults to targetRef.name)
|
||||
name: podinfo-service
|
||||
# ClusterIP port number
|
||||
port: 80
|
||||
# container port name or number (optional)
|
||||
targetPort: http
|
||||
# port name can be http or grpc (default http)
|
||||
# portName: http
|
||||
# add all the other container ports
|
||||
# to the ClusterIP services (default false)
|
||||
# portDiscovery: false
|
||||
analysis:
|
||||
interval: 15s
|
||||
threshold: 5
|
||||
maxWeight: 100
|
||||
stepWeight: 10
|
||||
metrics:
|
||||
- name: request-success-rate
|
||||
interval: 15s
|
||||
# minimum req success rate (non 5xx responses)
|
||||
# percentage (0-100)
|
||||
thresholdRange:
|
||||
min: 99
|
||||
- name: request-duration
|
||||
interval: 15s
|
||||
# maximum req duration P99
|
||||
# milliseconds
|
||||
thresholdRange:
|
||||
max: 500
|
||||
webhooks:
|
||||
- name: gate
|
||||
type: confirm-rollout
|
||||
url: http://flagger-loadtester.test/gate/approve
|
||||
- name: acceptance-test
|
||||
type: pre-rollout
|
||||
url: http://flagger-loadtester.test/
|
||||
timeout: 10s
|
||||
metadata:
|
||||
type: bash
|
||||
cmd: "curl -sd 'test' http://podinfo-service-canary/token | grep token"
|
||||
- name: "load test"
|
||||
type: rollout
|
||||
url: http://flagger-loadtester.test/
|
||||
timeout: 5s
|
||||
metadata:
|
||||
type: cmd
|
||||
cmd: "hey -z 10m -q 10 -c 2 -host app.example.com http://skipper-ingress.kube-system"
|
||||
logCmdOutput: "true"
|
||||
Executable
+14
@@ -0,0 +1,14 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
REPO_ROOT=$(git rev-parse --show-toplevel)
|
||||
|
||||
echo '>>> Deleting Skipper Ingress'
|
||||
kustomize build ${REPO_ROOT}/test/skipper | kubectl delete --force --wait=false -f -
|
||||
|
||||
echo '>>> Deleting Flagger'
|
||||
kubectl delete namespace flagger-system --ignore-not-found=true --force --wait=false
|
||||
|
||||
echo '>>> Cleanup test namespace'
|
||||
kubectl delete namespace test --ignore-not-found=true --force --wait=false
|
||||
|
||||
exit 0
|
||||
Executable
+83
@@ -0,0 +1,83 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# This script runs e2e tests for Canary initialization, analysis and promotion
|
||||
# Prerequisites: Kubernetes Kind and Skipper ingress controller
|
||||
|
||||
set -o errexit
|
||||
|
||||
REPO_ROOT=$(git rev-parse --show-toplevel)
|
||||
|
||||
echo '>>> Creating test namespace'
|
||||
kubectl create namespace test || true
|
||||
|
||||
echo '>>> Initialising workload'
|
||||
kubectl apply -f ${REPO_ROOT}/test/e2e-workload.yaml
|
||||
kubectl apply -f ${REPO_ROOT}/test/e2e-workload-ingress.yaml
|
||||
|
||||
echo '>>> Installing load tester'
|
||||
kubectl apply -k ${REPO_ROOT}/kustomize/tester
|
||||
kubectl -n test rollout status deployment/flagger-loadtester
|
||||
|
||||
echo '>>> Create canary CRD'
|
||||
kubectl apply -f ${REPO_ROOT}/test/e2e-skipper-canary.yaml
|
||||
echo '>>> Waiting for primary to be ready'
|
||||
retries=50
|
||||
count=0
|
||||
ok=false
|
||||
until ${ok}; do
|
||||
kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false
|
||||
sleep 5
|
||||
count=$(($count + 1))
|
||||
if [[ ${count} -eq ${retries} ]]; then
|
||||
kubectl -n flagger-system logs deployment/flagger
|
||||
echo "No more retries left"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
echo '✔ Canary initialization test passed'
|
||||
|
||||
echo '>>> Triggering canary deployment'
|
||||
kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1
|
||||
|
||||
echo '>>> Waiting for canary promotion'
|
||||
retries=50
|
||||
count=0
|
||||
ok=false
|
||||
failed=false
|
||||
until ${ok}; do
|
||||
kubectl -n test get canary/podinfo | grep 'Failed' && failed=true || failed=false
|
||||
if ${failed}; then
|
||||
kubectl -n flagger-system logs deployment/flagger
|
||||
echo "Canary failed!"
|
||||
exit 1
|
||||
fi
|
||||
kubectl -n test describe deployment/podinfo-primary | grep '3.1.1' && ok=true || ok=false
|
||||
sleep 10
|
||||
kubectl -n flagger-system logs deployment/flagger --tail 1
|
||||
count=$(($count + 1))
|
||||
if [[ ${count} -eq ${retries} ]]; then
|
||||
kubectl -n test describe deployment/podinfo
|
||||
kubectl -n test describe deployment/podinfo-primary
|
||||
kubectl -n flagger-system logs deployment/flagger
|
||||
echo "No more retries left"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
echo '>>> Waiting for canary finalization'
|
||||
retries=50
|
||||
count=0
|
||||
ok=false
|
||||
until ${ok}; do
|
||||
kubectl -n test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false
|
||||
sleep 5
|
||||
count=$(($count + 1))
|
||||
if [[ ${count} -eq ${retries} ]]; then
|
||||
kubectl -n flagger-system logs deployment/flagger
|
||||
echo "No more retries left"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
echo '✔ Canary promotion test passed'
|
||||
Executable
+22
@@ -0,0 +1,22 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -o errexit
|
||||
|
||||
REPO_ROOT=$(git rev-parse --show-toplevel)
|
||||
|
||||
echo '>>> Loading Flagger image'
|
||||
kind load docker-image test/flagger:latest
|
||||
|
||||
echo '>>> Installing Skipper Ingress, Flagger and Prometheus'
|
||||
# use kustomize to avoid compatibility issues:
|
||||
# https://github.com/kubernetes-sigs/kustomize/issues/2390
|
||||
# Skipper will throw an Prometheus warning which can be ignored:
|
||||
# https://github.com/weaveworks/flagger/issues/664
|
||||
kustomize build ${REPO_ROOT}/test/skipper | kubectl apply -f -
|
||||
|
||||
kubectl rollout status deployment/skipper-ingress -n kube-system
|
||||
kubectl rollout status deployment/flagger-prometheus -n flagger-system
|
||||
|
||||
kubectl -n flagger-system set image deployment/flagger flagger=test/flagger:latest
|
||||
|
||||
kubectl -n flagger-system rollout status deployment/flagger
|
||||
@@ -0,0 +1,17 @@
|
||||
apiVersion: networking.k8s.io/v1beta1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: podinfo-ingress
|
||||
namespace: test
|
||||
labels:
|
||||
app: podinfo
|
||||
annotations:
|
||||
kubernetes.io/ingress.class: skipper
|
||||
spec:
|
||||
rules:
|
||||
- host: app.example.com
|
||||
http:
|
||||
paths:
|
||||
- backend:
|
||||
serviceName: podinfo-service
|
||||
servicePort: 80
|
||||
Executable
+18
@@ -0,0 +1,18 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# This script is intended for local workstation development convenience.
|
||||
# It will run the e2e tests for Skipper and leave a working setup to play with
|
||||
|
||||
REPO_ROOT=$(git rev-parse --show-toplevel)
|
||||
cd $REPO_ROOT
|
||||
|
||||
make test
|
||||
make build
|
||||
docker tag weaveworks/flagger:latest test/flagger:latest
|
||||
make loadtester-build
|
||||
(kind get clusters && kubectl delete ns/test --force) || kind create cluster --wait 5m --image kindest/node:v1.16.9
|
||||
./test/e2e-skipper.sh
|
||||
# port forward prometheus UI to localhost:9090
|
||||
kubectl port-forward $(kubectl get pods -l=app=flagger-prometheus -o name -n flagger-system | head -n 1) 9090:9090 -n flagger-system &
|
||||
|
||||
./test/e2e-skipper-tests.sh
|
||||
@@ -0,0 +1,67 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- ../../kustomize/base/prometheus/
|
||||
- ../../kustomize/base/flagger/
|
||||
- https://raw.githubusercontent.com/zalando/skipper/v0.11.140/docs/kubernetes/deploy/deployment/rbac.yaml
|
||||
- https://raw.githubusercontent.com/zalando/skipper/v0.11.140/docs/kubernetes/deploy/deployment/service.yaml
|
||||
- https://raw.githubusercontent.com/zalando/skipper/v0.11.140/docs/kubernetes/deploy/deployment/deployment.yaml
|
||||
- namespace.yaml
|
||||
patchesStrategicMerge:
|
||||
- patch.yaml
|
||||
patches:
|
||||
- target:
|
||||
kind: Deployment
|
||||
name: skipper-ingress
|
||||
patch: |-
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: skipper-ingress
|
||||
spec:
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
prometheus.io/path: /metrics
|
||||
prometheus.io/port: "9911"
|
||||
prometheus.io/scrape: "true"
|
||||
spec:
|
||||
nodeSelector:
|
||||
$patch: delete
|
||||
affinity:
|
||||
$patch: delete
|
||||
containers:
|
||||
- name: skipper-ingress
|
||||
image: registry.opensource.zalan.do/pathfinder/skipper:v0.11.141
|
||||
ports:
|
||||
- name: metrics-port
|
||||
containerPort: 9911
|
||||
resources:
|
||||
$patch: delete
|
||||
readinessProbe:
|
||||
initialDelaySeconds: 5
|
||||
args:
|
||||
- skipper
|
||||
- -default-filters-prepend=enableAccessLog(4,5)
|
||||
- -address=:9999
|
||||
- -disable-metrics-compat
|
||||
- -enable-connection-metrics
|
||||
- -enable-profile
|
||||
- -enable-ratelimits
|
||||
- -experimental-upgrade
|
||||
- -histogram-metric-buckets=.01,1,10,100
|
||||
- -kubernetes-in-cluster
|
||||
- -kubernetes-path-mode=path-prefix
|
||||
- -kubernetes
|
||||
- -lb-healthcheck-interval=3s
|
||||
- -max-audit-body=0
|
||||
- -max-idle-connection-backend=0
|
||||
- -metrics-exp-decay-sample
|
||||
- -metrics-flavour=prometheus
|
||||
- -proxy-preserve-host
|
||||
- -route-backend-metrics
|
||||
- -route-backend-error-counters
|
||||
- -route-response-metrics
|
||||
- -serve-host-metrics
|
||||
- -serve-route-metrics
|
||||
- -whitelisted-healthcheck-cidr=0.0.0.0/0 # kind uses other IP addresse
|
||||
@@ -0,0 +1,4 @@
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: flagger-system
|
||||
@@ -0,0 +1,13 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: flagger
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: flagger
|
||||
args:
|
||||
- -log-level=debug
|
||||
- -mesh-provider=skipper
|
||||
- -metrics-server=http://flagger-prometheus:9090
|
||||
Reference in New Issue
Block a user