From d69e2034793c0f6e58bab424a8b3293446062305 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Mon, 22 Apr 2019 20:08:56 +0300 Subject: [PATCH 01/42] Fix Tiller-less install command --- .../install/flagger-install-on-kubernetes.md | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/docs/gitbook/install/flagger-install-on-kubernetes.md b/docs/gitbook/install/flagger-install-on-kubernetes.md index 73120559..80013f75 100644 --- a/docs/gitbook/install/flagger-install-on-kubernetes.md +++ b/docs/gitbook/install/flagger-install-on-kubernetes.md @@ -52,7 +52,8 @@ If you don't have Tiller you can use the helm template command and apply the gen ```bash # generate -helm template flagger/flagger \ +helm fetch --untar --untardir . flagger/flagger && +helm template flagger \ --name flagger \ --namespace=istio-system \ --set metricsServer=http://prometheus.istio-system:9090 \ @@ -98,12 +99,10 @@ Or use helm template command and apply the generated yaml with kubectl: ```bash # generate -helm template flagger/grafana \ +helm fetch --untar --untardir . flagger/grafana && +helm template grafana \ --name flagger-grafana \ --namespace=istio-system \ ---set url=http://prometheus.istio-system:9090 \ ---set user=admin \ ---set password=change-me \ > $HOME/flagger-grafana.yaml # apply @@ -132,10 +131,14 @@ helm upgrade -i flagger-loadtester flagger/loadtester \ Deploy with kubectl: ```bash -export REPO=https://raw.githubusercontent.com/weaveworks/flagger/master +helm fetch --untar --untardir . flagger/loadtester && +helm template loadtester \ +--name flagger-loadtester \ +--namespace=test +> $HOME/flagger-loadtester.yaml -kubectl -n test apply -f ${REPO}/artifacts/loadtester/deployment.yaml -kubectl -n test apply -f ${REPO}/artifacts/loadtester/service.yaml +# apply +kubectl apply -f $HOME/flagger-loadtester.yaml ``` > **Note** that the load tester should be deployed in a namespace with Istio sidecar injection enabled. From 5ba27c898e4852f71be3e08563dc4660bea7e760 Mon Sep 17 00:00:00 2001 From: Yuval Kohavi Date: Tue, 23 Apr 2019 07:42:52 -0400 Subject: [PATCH 02/42] remove todo --- pkg/router/supergloo_test.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/pkg/router/supergloo_test.go b/pkg/router/supergloo_test.go index 2e31aa40..aa500b63 100644 --- a/pkg/router/supergloo_test.go +++ b/pkg/router/supergloo_test.go @@ -25,10 +25,9 @@ func TestSuperglooRouter_Sync(t *testing.T) { if err := routingRuleClient.Register(); err != nil { t.Fatal(err.Error()) } - // TODO(yuval-k): un hard code this targetMesh := solokitcore.ResourceRef{ Namespace: "supergloo-system", - Name: "yuval", + Name: "mesh", } router := NewSuperglooRouterWithClient(context.TODO(), routingRuleClient, targetMesh, mocks.logger) err = router.Reconcile(mocks.canary) @@ -61,10 +60,9 @@ func TestSuperglooRouter_SetRoutes(t *testing.T) { if err := routingRuleClient.Register(); err != nil { t.Fatal(err.Error()) } - // TODO(yuval-k): un hard code this targetMesh := solokitcore.ResourceRef{ Namespace: "supergloo-system", - Name: "yuval", + Name: "mesh", } router := NewSuperglooRouterWithClient(context.TODO(), routingRuleClient, targetMesh, mocks.logger) @@ -126,10 +124,9 @@ func TestSuperglooRouter_GetRoutes(t *testing.T) { if err := routingRuleClient.Register(); err != nil { t.Fatal(err.Error()) } - // TODO(yuval-k): un hard code this targetMesh := solokitcore.ResourceRef{ Namespace: "supergloo-system", - Name: "yuval", + Name: "mesh", } router := NewSuperglooRouterWithClient(context.TODO(), routingRuleClient, targetMesh, mocks.logger) err = router.Reconcile(mocks.canary) From 3f8f634a1b8454b9cba842e32eea478c4426a423 Mon Sep 17 00:00:00 2001 From: Yuval Kohavi Date: Tue, 23 Apr 2019 18:06:46 -0400 Subject: [PATCH 03/42] add e2e tests --- .circleci/config.yml | 16 ++++++++++++++++ artifacts/flagger/deployment.yaml | 1 + test/e2e-build.sh | 5 +++++ test/e2e-supergloo.sh | 28 ++++++++++++++++++++++++++++ 4 files changed, 50 insertions(+) create mode 100755 test/e2e-supergloo.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 4a44d24e..08edb14c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,6 +9,15 @@ jobs: - run: test/e2e-build.sh - run: test/e2e-tests.sh + e2e-supergloo-testing: + machine: true + steps: + - checkout + - run: test/e2e-kind.sh + - run: test/e2e-supergloo.sh + - run: test/e2e-build.sh supergloo:supergloo-system.test + - run: test/e2e-tests.sh + workflows: version: 2 build-and-test: @@ -20,3 +29,10 @@ workflows: - /gh-pages.*/ - /docs-.*/ - /release-.*/ + - e2e-supergloo-testing: + filters: + branches: + ignore: + - /gh-pages.*/ + - /docs-.*/ + - /release-.*/ diff --git a/artifacts/flagger/deployment.yaml b/artifacts/flagger/deployment.yaml index b0141f1d..80b0e9f7 100644 --- a/artifacts/flagger/deployment.yaml +++ b/artifacts/flagger/deployment.yaml @@ -31,6 +31,7 @@ spec: - ./flagger - -log-level=info - -control-loop-interval=10s + - -mesh-provider=$(MESH_PROVIDER) - -metrics-server=http://prometheus.istio-system.svc.cluster.local:9090 livenessProbe: exec: diff --git a/test/e2e-build.sh b/test/e2e-build.sh index 43dff8d0..d0e4bfa8 100755 --- a/test/e2e-build.sh +++ b/test/e2e-build.sh @@ -11,5 +11,10 @@ cd ${REPO_ROOT} && docker build -t test/flagger:latest . -f Dockerfile echo '>>> Installing Flagger' kind load docker-image test/flagger:latest kubectl apply -f ${REPO_ROOT}/artifacts/flagger/ + +if [ -n "$1" ]; then + kubectl -n istio-system set env deployment/flagger "MESH_PROVIDER=$1" +fi + kubectl -n istio-system set image deployment/flagger flagger=test/flagger:latest kubectl -n istio-system rollout status deployment/flagger diff --git a/test/e2e-supergloo.sh b/test/e2e-supergloo.sh new file mode 100755 index 00000000..3123b45c --- /dev/null +++ b/test/e2e-supergloo.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +set -o errexit + +ISTIO_VER="1.0.6" +REPO_ROOT=$(git rev-parse --show-toplevel) +export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" + +echo ">>> Downloading Supergloo CLI" +curl -SsL https://github.com/solo-io/supergloo/releases/download/v0.3.13/supergloo-cli-linux-amd64 > supergloo-cli +chmod +x supergloo-cli + +echo ">>> Installing Supergloo" +./supergloo-cli init +echo ">>> Installing Istio ${ISTIO_VER}" +kubectl create ns istio-system +./supergloo-cli install istio --name test --namespace supergloo-system --auto-inject true --installation-namespace istio-system --mtls false --prometheus true --version ${ISTIO_VER} + +echo '>>> Waiting for Istio to be ready' +until kubectl -n supergloo-system get mesh test +do + sleep 2 +done + +# add rbac rules +kubectl create clusterrolebinding flagger-supergloo --clusterrole=mesh-discovery --serviceaccount=istio-system:flagger + +kubectl -n istio-system get all From d196bb28568e59a14e4c4b11e7e4e2de41eb4488 Mon Sep 17 00:00:00 2001 From: Yuval Kohavi Date: Wed, 24 Apr 2019 16:00:55 -0400 Subject: [PATCH 04/42] e2e test --- .circleci/config.yml | 2 +- pkg/router/supergloo.go | 60 +++++++++++++++++++++++++++++++++++++++-- test/e2e-supergloo.sh | 8 +++++- test/e2e-tests.sh | 4 +++ 4 files changed, 70 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 08edb14c..4d5a4ada 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,7 +16,7 @@ jobs: - run: test/e2e-kind.sh - run: test/e2e-supergloo.sh - run: test/e2e-build.sh supergloo:supergloo-system.test - - run: test/e2e-tests.sh + - run: test/e2e-tests.sh canary workflows: version: 2 diff --git a/pkg/router/supergloo.go b/pkg/router/supergloo.go index 259a7637..3f812a31 100644 --- a/pkg/router/supergloo.go +++ b/pkg/router/supergloo.go @@ -74,12 +74,22 @@ func (sr *SuperglooRouter) Reconcile(canary *flaggerv1.Canary) error { if err := sr.setRetries(canary); err != nil { return err } + if err := sr.setHeaders(canary); err != nil { + return err + } if err := sr.setCors(canary); err != nil { return err } - return sr.SetRoutes(canary, 100, 0) - + // do we have routes already? + if _, _, err := sr.GetRoutes(canary); err == nil { + // we have routes, no need to do anything else + return nil + } else if solokiterror.IsNotExist(err) { + return sr.SetRoutes(canary, 100, 0) + } else { + return err + } } func (sr *SuperglooRouter) setRetries(canary *flaggerv1.Canary) error { @@ -98,6 +108,52 @@ func (sr *SuperglooRouter) setRetries(canary *flaggerv1.Canary) error { return sr.writeRuleForCanary(canary, rule) } +func (sr *SuperglooRouter) setHeaders(canary *flaggerv1.Canary) error { + if canary.Spec.Service.Headers == nil { + return nil + } + headerManipulation, err := convertHeaders(canary.Spec.Service.Headers) + if err != nil { + return err + } + if headerManipulation == nil { + return nil + } + rule := sr.createRule(canary, "headers", &supergloov1.RoutingRuleSpec{ + RuleType: &supergloov1.RoutingRuleSpec_HeaderManipulation{ + HeaderManipulation: headerManipulation, + }, + }) + + return sr.writeRuleForCanary(canary, rule) +} + +func convertHeaders(headers *istiov1alpha3.Headers) (*supergloov1.HeaderManipulation, error) { + var headersMaipulation *supergloov1.HeaderManipulation + + if headers.Request != nil { + headersMaipulation = &supergloov1.HeaderManipulation{} + + headersMaipulation.RemoveRequestHeaders = headers.Request.Remove + headersMaipulation.AppendRequestHeaders = make(map[string]string) + for k, v := range headers.Request.Add { + headersMaipulation.AppendRequestHeaders[k] = v + } + } + if headers.Response != nil { + if headersMaipulation == nil { + headersMaipulation = &supergloov1.HeaderManipulation{} + } + + headersMaipulation.RemoveResponseHeaders = headers.Response.Remove + headersMaipulation.AppendResponseHeaders = make(map[string]string) + for k, v := range headers.Response.Add { + headersMaipulation.AppendResponseHeaders[k] = v + } + } + + return headersMaipulation, nil +} func convertRetries(retries *istiov1alpha3.HTTPRetry) (*supergloov1.RetryPolicy, error) { perTryTimeout, err := time.ParseDuration(retries.PerTryTimeout) diff --git a/test/e2e-supergloo.sh b/test/e2e-supergloo.sh index 3123b45c..428b9232 100755 --- a/test/e2e-supergloo.sh +++ b/test/e2e-supergloo.sh @@ -14,7 +14,7 @@ echo ">>> Installing Supergloo" ./supergloo-cli init echo ">>> Installing Istio ${ISTIO_VER}" kubectl create ns istio-system -./supergloo-cli install istio --name test --namespace supergloo-system --auto-inject true --installation-namespace istio-system --mtls false --prometheus true --version ${ISTIO_VER} +./supergloo-cli install istio --name test --namespace supergloo-system --auto-inject=true --installation-namespace istio-system --mtls=false --prometheus=true --version ${ISTIO_VER} echo '>>> Waiting for Istio to be ready' until kubectl -n supergloo-system get mesh test @@ -25,4 +25,10 @@ done # add rbac rules kubectl create clusterrolebinding flagger-supergloo --clusterrole=mesh-discovery --serviceaccount=istio-system:flagger +kubectl -n istio-system rollout status deployment/istio-pilot +kubectl -n istio-system rollout status deployment/istio-policy +kubectl -n istio-system rollout status deployment/istio-sidecar-injector +kubectl -n istio-system rollout status deployment/istio-telemetry +kubectl -n istio-system rollout status deployment/prometheus + kubectl -n istio-system get all diff --git a/test/e2e-tests.sh b/test/e2e-tests.sh index 2ef1b7bc..043d2cc9 100755 --- a/test/e2e-tests.sh +++ b/test/e2e-tests.sh @@ -125,6 +125,10 @@ done echo '✔ Canary promotion test passed' +if [ "$1" = "canary" ]; then + exit 0 +fi + cat < Date: Thu, 25 Apr 2019 11:10:23 -0400 Subject: [PATCH 05/42] use name.namespace instead of namespace.name --- .circleci/config.yml | 2 +- pkg/router/supergloo.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4d5a4ada..7c254c55 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,7 +15,7 @@ jobs: - checkout - run: test/e2e-kind.sh - run: test/e2e-supergloo.sh - - run: test/e2e-build.sh supergloo:supergloo-system.test + - run: test/e2e-build.sh supergloo:test.supergloo-system - run: test/e2e-tests.sh canary workflows: diff --git a/pkg/router/supergloo.go b/pkg/router/supergloo.go index 3f812a31..762e845c 100644 --- a/pkg/router/supergloo.go +++ b/pkg/router/supergloo.go @@ -52,14 +52,14 @@ func NewSuperglooRouter(ctx context.Context, provider string, flaggerClient clie // remove the supergloo: prefix provider = strings.TrimPrefix(provider, "supergloo:") - // split namespace.name : + // split name.namespace: parts := strings.Split(provider, ".") if len(parts) != 2 { return nil, fmt.Errorf("invalid format for supergloo provider") } targetMesh := solokitcore.ResourceRef{ - Namespace: parts[0], - Name: parts[1], + Namespace: parts[1], + Name: parts[0], } return NewSuperglooRouterWithClient(ctx, routingRuleClient, targetMesh, logger), nil } From 1a95fc2a9cf0b819d5416ec4fafa6b4fa0c54acc Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Fri, 26 Apr 2019 12:33:09 +0300 Subject: [PATCH 06/42] Add SuperGloo install docs --- README.md | 23 +-- docs/gitbook/SUMMARY.md | 1 + .../install/flagger-install-with-supergloo.md | 135 ++++++++++++++++++ 3 files changed, 148 insertions(+), 11 deletions(-) create mode 100644 docs/gitbook/install/flagger-install-with-supergloo.md diff --git a/README.md b/README.md index c25e5866..fd036842 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ Flagger documentation can be found at [docs.flagger.app](https://docs.flagger.ap * [Flagger install on Kubernetes](https://docs.flagger.app/install/flagger-install-on-kubernetes) * [Flagger install on GKE Istio](https://docs.flagger.app/install/flagger-install-on-google-cloud) * [Flagger install on EKS App Mesh](https://docs.flagger.app/install/flagger-install-on-eks-appmesh) + * [Flagger install with SuperGloo](https://docs.flagger.app/install/flagger-install-with-supergloo) * How it works * [Canary custom resource](https://docs.flagger.app/how-it-works#canary-custom-resource) * [Routing](https://docs.flagger.app/how-it-works#istio-routing) @@ -152,20 +153,20 @@ For more details on how the canary analysis and promotion works please [read the ## Features -| Feature | Istio | App Mesh | -| -------------------------------------------- | ------------------ | ------------------ | -| Canary deployments (weighted traffic) | :heavy_check_mark: | :heavy_check_mark: | -| A/B testing (headers and cookies filters) | :heavy_check_mark: | :heavy_minus_sign: | -| Load testing | :heavy_check_mark: | :heavy_check_mark: | -| Webhooks (custom acceptance tests) | :heavy_check_mark: | :heavy_check_mark: | -| Request success rate check (Envoy metric) | :heavy_check_mark: | :heavy_check_mark: | -| Request duration check (Envoy metric) | :heavy_check_mark: | :heavy_minus_sign: | -| Custom promql checks | :heavy_check_mark: | :heavy_check_mark: | -| Ingress gateway (CORS, retries and timeouts) | :heavy_check_mark: | :heavy_minus_sign: | +| Feature | Istio | App Mesh | SuperGloo | +| -------------------------------------------- | ------------------ | ------------------ |------------------ | +| Canary deployments (weighted traffic) | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | +| A/B testing (headers and cookies filters) | :heavy_check_mark: | :heavy_minus_sign: | :heavy_minus_sign: | +| Load testing | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | +| Webhooks (custom acceptance tests) | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | +| Request success rate check (Envoy metric) | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | +| Request duration check (Envoy metric) | :heavy_check_mark: | :heavy_minus_sign: | :heavy_check_mark: | +| Custom promql checks | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | +| Ingress gateway (CORS, retries and timeouts) | :heavy_check_mark: | :heavy_minus_sign: | :heavy_check_mark: | ## Roadmap -* Integrate with other service mesh technologies like Linkerd v2, Super Gloo or Consul Mesh +* Integrate with other service mesh technologies like Linkerd v2 * Add support for comparing the canary metrics to the primary ones and do the validation based on the derivation between the two ## Contributing diff --git a/docs/gitbook/SUMMARY.md b/docs/gitbook/SUMMARY.md index 4ce8974e..ad8721c1 100644 --- a/docs/gitbook/SUMMARY.md +++ b/docs/gitbook/SUMMARY.md @@ -8,6 +8,7 @@ * [Flagger Install on Kubernetes](install/flagger-install-on-kubernetes.md) * [Flagger Install on GKE Istio](install/flagger-install-on-google-cloud.md) * [Flagger Install on EKS App Mesh](install/flagger-install-on-eks-appmesh.md) +* [Flagger Install with SuperGloo](install/flagger-install-with-supergloo.md) ## Usage diff --git a/docs/gitbook/install/flagger-install-with-supergloo.md b/docs/gitbook/install/flagger-install-with-supergloo.md new file mode 100644 index 00000000..c0c7e1d7 --- /dev/null +++ b/docs/gitbook/install/flagger-install-with-supergloo.md @@ -0,0 +1,135 @@ +# Flagger install on Kubernetes with SuperGloo + +This guide walks you through setting up Flagger on a Kubernetes cluster using [SuperGloo](https://github.com/solo-io/supergloo). + +SuperGloo by [Solo.io](https://solo.io) is an opinionated abstraction layer that will simplify the installation, management, and operation of your service mesh. +It supports running multiple ingress with multiple mesh (Istio, App Mesh, Consul Connect and Linkerd 2) in the same cluster. + +### Prerequisites + +Flagger requires a Kubernetes cluster **v1.11** or newer with the following admission controllers enabled: + +* MutatingAdmissionWebhook +* ValidatingAdmissionWebhook + +### Install Istio with SuperGloo + +Download SuperGloo CLI and add it to your path: + +```bash +curl -sL https://run.solo.io/supergloo/install | sh +export PATH=$HOME/.supergloo/bin:$PATH +``` + +Deploy the SuperGloo controller in the `supergloo-system` namespace: + +```bash +supergloo init +``` + +Create the `istio-system` namespace and install Istio with traffic management, telemetry and Prometheus enabled: + +```bash +ISTIO_VER="1.0.6" + +kubectl create ns istio-system + +supergloo install istio --name istio \ +--namespace=supergloo-system \ +--auto-inject=true \ +--installation-namespace=istio-system \ +--mtls=false \ +--prometheus=true \ +--version ${ISTIO_VER} +``` + +Create a cluster role binding so that Flagger can manipulate SuperGloo custom resources: + +```bash +kubectl create clusterrolebinding flagger-supergloo \ +--clusterrole=mesh-discovery \ +--serviceaccount=istio-system:flagger +``` + +Wait for the Istio control plane to become available: + +```bash +kubectl -n istio-system rollout status deployment/istio-sidecar-injector +kubectl -n istio-system rollout status deployment/prometheus +``` + +### Install Flagger + +Add Flagger Helm repository: + +```bash +helm repo add flagger https://flagger.app +``` + +Deploy Flagger in the _**istio-system**_ namespace and set the service mesh provider to SuperGloo: + +```bash +helm upgrade -i flagger flagger/flagger \ +--namespace=istio-system \ +--set metricsServer=http://prometheus.istio-system:9090 \ +--set meshProvider=supergloo:istio.supergloo-system +``` + +When using SuperGloo the mesh provider format is `supergloo:.`. + +Optionally you can enable **Slack** notifications: + +```bash +helm upgrade -i flagger flagger/flagger \ +--reuse-values \ +--namespace=istio-system \ +--set slack.url=https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK \ +--set slack.channel=general \ +--set slack.user=flagger +``` + +### Install Grafana + +Flagger comes with a Grafana dashboard made for monitoring the canary analysis. + +Deploy Grafana in the _**istio-system**_ namespace: + +```bash +helm upgrade -i flagger-grafana flagger/grafana \ +--namespace=istio-system \ +--set url=http://prometheus.istio-system:9090 +``` + +You can access Grafana using port forwarding: + +```bash +kubectl -n istio-system port-forward svc/flagger-grafana 3000:80 +``` + +### Install Load Tester + +Flagger comes with an optional load testing service that generates traffic +during canary analysis when configured as a webhook. + +Deploy the load test runner with Helm: + +```bash +helm upgrade -i flagger-loadtester flagger/loadtester \ +--namespace=test \ +--set cmd.timeout=1h +``` + +Deploy with kubectl: + +```bash +helm fetch --untar --untardir . flagger/loadtester && +helm template loadtester \ +--name flagger-loadtester \ +--namespace=test +> $HOME/flagger-loadtester.yaml + +# apply +kubectl apply -f $HOME/flagger-loadtester.yaml +``` + +> **Note** that the load tester should be deployed in a namespace with Istio sidecar injection enabled. From 82a1f45cc1e6fbd7d5804f1dbb3135e95aecc784 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Mon, 29 Apr 2019 11:17:19 +0300 Subject: [PATCH 07/42] Fix load tester image repo --- charts/loadtester/Chart.yaml | 2 +- charts/loadtester/values.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/loadtester/Chart.yaml b/charts/loadtester/Chart.yaml index d0127e99..36ea2f42 100644 --- a/charts/loadtester/Chart.yaml +++ b/charts/loadtester/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 name: loadtester -version: 0.3.0 +version: 0.4.0 appVersion: 0.3.0 kubeVersion: ">=1.11.0-0" engine: gotpl diff --git a/charts/loadtester/values.yaml b/charts/loadtester/values.yaml index 3e11175c..9399a8ad 100644 --- a/charts/loadtester/values.yaml +++ b/charts/loadtester/values.yaml @@ -1,7 +1,7 @@ replicaCount: 1 image: - repository: quay.io/weaveworks/flagger-loadtester + repository: weaveworks/flagger-loadtester tag: 0.3.0 pullPolicy: IfNotPresent From 61141c747936fa734806fc474faa5407d5e0da15 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Mon, 29 Apr 2019 16:37:48 +0300 Subject: [PATCH 08/42] Release v0.12.0 --- artifacts/flagger/deployment.yaml | 2 +- charts/flagger/Chart.yaml | 4 ++-- charts/flagger/values.yaml | 2 +- pkg/version/version.go | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/artifacts/flagger/deployment.yaml b/artifacts/flagger/deployment.yaml index 80b0e9f7..4152ea80 100644 --- a/artifacts/flagger/deployment.yaml +++ b/artifacts/flagger/deployment.yaml @@ -22,7 +22,7 @@ spec: serviceAccountName: flagger containers: - name: flagger - image: weaveworks/flagger:0.11.1 + image: weaveworks/flagger:0.12.0 imagePullPolicy: IfNotPresent ports: - name: http diff --git a/charts/flagger/Chart.yaml b/charts/flagger/Chart.yaml index 7c58e2c3..b74b28fc 100644 --- a/charts/flagger/Chart.yaml +++ b/charts/flagger/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 name: flagger -version: 0.11.1 -appVersion: 0.11.1 +version: 0.12.0 +appVersion: 0.12.0 kubeVersion: ">=1.11.0-0" engine: gotpl description: Flagger is a Kubernetes operator that automates the promotion of canary deployments using Istio routing for traffic shifting and Prometheus metrics for canary analysis. diff --git a/charts/flagger/values.yaml b/charts/flagger/values.yaml index d624d440..0f1195ab 100644 --- a/charts/flagger/values.yaml +++ b/charts/flagger/values.yaml @@ -2,7 +2,7 @@ image: repository: weaveworks/flagger - tag: 0.11.1 + tag: 0.12.0 pullPolicy: IfNotPresent metricsServer: "http://prometheus:9090" diff --git a/pkg/version/version.go b/pkg/version/version.go index 2ae05041..4716560a 100644 --- a/pkg/version/version.go +++ b/pkg/version/version.go @@ -1,4 +1,4 @@ package version -var VERSION = "0.11.1" +var VERSION = "0.12.0" var REVISION = "unknown" From 4f01ecde5a9617c2becb957165232e3d3dc094dd Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Mon, 29 Apr 2019 16:41:26 +0300 Subject: [PATCH 09/42] Update changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9dae0295..c3e92630 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ All notable changes to this project are documented in this file. +## 0.12.0 (2019-04-29) + +Adds support for [SuperGloo](https://docs.flagger.app/install/flagger-install-with-supergloo) + +#### Features + +- Supergloo support for canary deployment (weighted traffic) [#151](https://github.com/weaveworks/flagger/pull/151) + ## 0.11.1 (2019-04-18) Move Flagger and the load tester container images to Docker Hub From 82067f13bf13b914250764d5aac49e2b1c0eaf29 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 1 May 2019 13:09:18 +0300 Subject: [PATCH 10/42] Add GitOps diagram --- docs/diagrams/flagger-gitops-istio.png | Bin 0 -> 35796 bytes docs/gitbook/how-it-works.md | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 docs/diagrams/flagger-gitops-istio.png diff --git a/docs/diagrams/flagger-gitops-istio.png b/docs/diagrams/flagger-gitops-istio.png new file mode 100644 index 0000000000000000000000000000000000000000..1974198e80d123a65805b5c143a136e262aa22b8 GIT binary patch literal 35796 zcmb@tWk8hA_Xi4yNC~d8NQ;k1ErNu!^ukioOLuptlz_wn!jjSAOUl8bbAcX!uC zzrX)|d0*XEPn#GeJZh`Qs`xglb2__~c z58OvjPEPLb?%1tYl|CP^IcJXy4{z=4FE1_LNs}};H$ND>t0*u3mYG@jXM=+S=3hJ? z5){-mf5?-$P1XHv`;+5wQM&z1FaOt#I5aeBG)XaGWw*)gnQ)tOY%*VkD5+A+VBv?m zcT%di3jK`6)fHVloi#(|u`W|ALv$=GBoydqw=(mZXlO7Hn$Uej|NmSRG5;C+|GWh3 z1bh=sJ-rYa>p8vg5 z4-Lpr;<)$pR)&ho^V#25BZTzth|%@d7@}e>%JA8JX3wM3s(+rJQlwtLf2dc&5S4O7 zjYFtFN_%^KP8YhlrOny&M*ZgI?2O3QD+lJ(-OMIc!Lj&0*D)q=p*kAL!E|%47S9kB zWbX9mxXrY@9Nz|1vk>te|CNsd`nWz~+WTQ@OdGCwOuS}$%kNo6G(H(5H6GjR@^WZ( zc#zW9t5DouyNb&cw21~)MqQ2^2}J06K@5sgNZrTce|lVG{kt3I{@M|fT2}2UAlH$I zu1$RIJChAdt@k}s^)sr6cxdaQ*qWk@T7*KjjJQXR@9}eN?FQB2qh6{_j1W*MVM77Q~8KjZx%M% z$0S6uR_B$i1~C#q+h&4(`4l<6c)F)KL!;H?Edl_Ui5w{=?LPb()HITDB@U5M9!TYlKfUtW`x> zR&m*k@vrx}MECkQv<1M4?>B2CKzW#bJ0~DrN1o9(Vn1y>Gc&sh7o=gp_|>aufcefpBP>|gfbC?(_57z$2Zs>2jnySgi;&mag=^3~YH>o3O-_Z7X@Bi>4c7<*h zjZAdSy?ce+m{G1*3#oq`b(K^xy63f|>IVPP>wc<_mQ`bFe9zmw4>$C_z0ID_j{?|0 zYS4ua47qD)&L0}j)!T2R>)O2hZSL-*;x3dS$~Ykp8_g;S2(1SyNF|1-mM#+Ni5MnB zel3O1vsfxFZ$a+Pa$q+TPye!Ij$k@%-mcoejD9Ozlb}I)WgT@s_`*d%!kr!ZN9m1V zz>6BxPdey-ImNVtz^owc$W|AzPL-d|hE37;c!^V`9?s5M+-!NBW0cd1m*2G!l|eAj zwiJ!bRgJ2v$haJ%Y&aCtvdk5;)>+bUm`{SJbShUH8%MTn2%@NL?mXV!C-_YkYOok} za9}0vN{%dsX6_QdQc&?+ef@JH@db0Q|GlwbR&DxgS(WHH&p@?T2=pf_44^eIBioq?v5g^6xRFtMP@>+|vfB8UC?bU&%Z8 z9FV(Z=j)gTtl3>YZ20iMaYgI`^_!@TTXLIdG(sKm7|uFvp)D*9(}5N z+|19iMxMntf3gia5#${jxvJ|AWF|rJ#FiZEdDv zv(JTUHa3~C=cw;*0Q!Fe=>E3I9TJ_-qwH*c1`U*imAM6t&JX3G>?~ILebPzYXhfN_ zyxfy%8KMYV!jx2>FOeozUa2V&pV&vyvVFL*zkE<_YO-nH`EExki-#~qrlrH{zVu8H zsek`Gkr%gkZuf13R%Njs;CESAX{U zrr>}%3sz>rI2hZKp-WoJQ{#`=xRsdY6pIIPC0}J(w+uUeDJKkBh3U3{`dV7EB0{&^ zaP03|9j0G()=z|7-PfyA2DUkH0vB{T2d1T!?*h$|MdrJ+!wLW$8RQ<t_pa&^U(%zf`<_K10f+ib&>?k~Jvi0=CW zQc*NwqMG&H?*Of>CAM)RTMdTXTq{FaF%G^Kd*8zC^oJAu!;Hw*vbi=QX6Pz2Wr}%& zkRP-AJfz*Bv%P|+#dPab;!oMUf<5snnoxgcmE`qw7g0Q-j(amil8&=9V(OJ5$`qsK{r82XBHLI zXTpeqb{9*Ou!E4XwHsBI=x>L4Fia*$oJn!?fTq}a>=igvhcL~LQ1y78g2{-0nUb5W zO6J{a)!fGe*(iNB7Ri>*8^x~9(~ZzPl%nA{_|sj>9rkL3%^gqujqRf51D^z-LW%y2 zL9tKRCXcnzXM!Jk`8>JyHB132tSrNvfg_NJSiZTD>*_Q>ZFn@!&Ru`2ZQ@GQ5k7ix zS`6)QCJF?s#h2l}Pn=9H?{w=Puj>0t4%PP9(4iMqIwo+C*0xf-ctNhHsYHgLJ}zxW ztouCKPY@9NN!W?N{x(eMdfO5DVJ^+0lEQY5WN!g8*W;`E=@3Uc%A591m!gm4=VJti z259i~W($vmpwRd+0Jq2UrO_tMH@86oFLB5YFd>=RyP|>i5>|TELZO5yn#o?z=`)|i zd^M<1Mk9G~p=((P>;FTBYfO*>2DQH)vmBbspYB3aHid01iKhH1u)XV`vi?dN^)rch z@q!hL$o;5I)6k6Fl*OS2stJKdKBc2OLC+KJ*}wz_N60KKYC2GajxTw%`ODvO}B>M z5~S+mBMhT5-)=8eO~qx#)IcCD4kf$OZ>Nx4SOzG{y(3vn z50;K?`WXfWQ900bDRf4DR{-IiQ)a+~03A|$pYKpLvB&g>58$mxn70t}=O^It1GaS3 zhn7!-{RYIm*ZhhV{@{c(UVm{dlE7C)7gz)N%-1LkAseRX!`Gat!qIxDGou4eTVVG* zxI#LV{pPXb2Wbo_b$B%4(3{0kKQ0)) zy#2j82z$kmjD4Rij;RG5n#!8pjJ!7J*Cl=2hzA)A0z?=rV}ZR2bCvk+VM;sA*ntfE z4T)!mg1PPe(cp942ZxX0Mn|L^@^zYqAYba|Xe9BODkBF>iL~z@JUY7_c{|EeAkVt@ zsQX>xW=q>2_W~#F+kzSa8OYYa5U&rC^>WM{x-9m}j9lS&lYHyYzTBp5isG;8o7e9_yH{V=V9s>+@*BnT~{)FIgo@4|*vt#(w4TV8_qNAJRAatkWZ9O#7!G6oIlSjM1SVH$eE5!8wjl-(Lh6M%z*wqs)UVZnb+=-a z%nqgTXus&>6it34K|ksA1Bg58GS2ZK|$Ii)Fp0IgDp^AJn=cUmsx-n4&`C zM9=u3LuOQP9M`5onsO6+$RVW7srzh=$VBr?aqnIk;6dB=LQhRIKh}cRD(n$XqF(*g z=<^?-__2fKl&;S|x0Qu}UW{A=)8jwMqzDcHA@{w@{U`G4+ju-DlLJ@LyG+BE3P z=(SpDMU-M$t=Ag}h2?qmW@M8u^Km@m6G6fuPtB3=I**Xwj_s83mJO1k8CFq+yWNO& zTTio|*(DX-!L5wo-Tq92hX@ifN{=vd&WFC4Eb&2_lS5 zNmU(*g$T%e*m1CANJF3b=Pe21JYdle1m*CWmq0rtbHN=aiq=gIoAuW_YWUXKne*nm zaT>mRt$rD~Q|z+=Lms^EgrBU=AdW;VCQ$ANoX6U%>Di~@i2|CW4JqT#5N|b*G}_i> zF#a%A)YOT96aPo?X=cHDhjF0YYiDadFhm18B-4XjQKny6zV)^0r%chUX2M<@0yJ5{ zHx&M^PSPUtA3zA-c`D`yCDGLU{2Io6dra3EK}AGbLx>+_h=94QCuYiW8g#ti8qIFfLIy z!VW9zI7OsON5WQvF$tgZ$0DuEXCQIJ_i#|<`?9ro^dFqMDAiZUwRG%bqKiEkl4}%k zeBK3yw;E{&g?e*Y%5Y)%=D&z6IR<~qG^{9WqWDOhCmq0tVO@^IAl?t6ib2P==l-(% z&JTDD!I<0fdd@UAOcIe1j9W0%zArCE?HUL8 zDYbHx){J^|Rc9n_wSP*kYL3c)VPA@Gs`w;4N=R=?&^ymJY#KEQ<6y)w^#h3ZjboCj zf|5;9K8^Ae>JE$u0Q5^%^q_XAQ3E0>#CX_K4!@z3nNbX(?HB#Dj8vjX7-pSA-*I(p zjJ%X9@GDw)NZ5;aBaW}c47!ThCGL@f2lXT+LBzP}KsQA*pkUpHCb&O^e{b90q)rdd z*ku`~>4jMO>B+EixR4H95edfhck4mFqw`!6kUl|gqgiq!RVe6|84A(Pz%BqwE(TfR zL0iX+JG+1Nk9DcosqXlS#r^T!=@%hS-QmfAXVplAdi%8ly6B8&_nC|3OLCVkt1K7@ zly^#IaXnli&pdx(8Zx5aB0t~{c>2*QmYj}U%q$5ZW19d)Xds^`C_>PDoZ$gKH@@!2 z10_*VT(Q{%T?1 z&<~c&y?)JKLx&;x8GKZywGu(;$KQjHf53K3ncC8%w36M3vRm)zY%P`-bDi+7-r8It zzAA**MNaKTjnurbEd+18HiK6JKLV2%m?Zxt&Qg)m;$Ha_x$dP{~UCVvR%2Q{m!@kFq%0xDQhPIX$b>L7*Mcdi!J zoci|+lHM~YKm28SS*+BxOj^l~&CNiUzTA&2ZVPA0euXCA$0mk<0$R?ihWzOd*Q?iA@%^w*UFACCOZn;j6p#+=PW0weSVg$;5wu3Moo5+Pk zou4m9NTv(%wHZO;?&`7e(_ILeP>}gd!EvTk?&JfmHH*~>T5wn7y#8`0Sm7%;1IuWg z&q(M!tNNVc?ADft#ogeKRh}YPR51{fk$^EL=+k7<9m^Coa`Y1EI*O?L9rA{%(huNi zCvHdj9aaX`VEM|hmQ1xGtO9_pxtI70AF&`&UUJHRBChcXp&m1EFCGC6@I-z(6DNB% z>38mQm>pAU%kY~kF_*y~S&npVb}9x?{_0XeFK_YV$LntJ*7%alPrw--#SZC+%r-W( zP6Ju_6JW}OCZ*xbi|)Nqrz}M2dSI%%>j;*JB1<(gmXniJE8d}5X*QV1Zqm{}{UzY> z1&!W-l_;Y*_F`NEsiO-bMyylTJ*7+r=V4e)5g*l6f93iR0&lxt&vvg*SWB8IoQ6pE z{}=>(ok`I|dUf)K^`R8fb1xqRwxv=QM%KQC)702{Jo_jNYSb_<*HQ?M>^eCNY5~kH z!=gsG#tSpueKOF-Fj9Epw4Sg zb$?J|B>0PhpxDnSKg*UYTW9maOmLq^CFQ(5oSw=uG>eM!GVt>VDdi|mf9YDWxLn}* z8LYp~XZ7n{i@D}ab;9@qMB#%+O}pq2lx$eU5!gNg@Lyq&C0Tf$rM{SvBXwISP2xbwzLqZOCWL{L8)lX@Qe8gwac;- z?JP$u@{gC;n=W*E`8a%NxLX8R6(5OqPA;?YhqyHGUIBZSP?Adm@+Vgxgx+9%_7Qd) zli9|Dj#dIU#RVz$A38UaETPozqU6kwdDO2(m&2IT=lHZ;t7#ltuVv6+REZcTiC_>V z9J3?!NABz1Ig(6NY==sbfC2Ou;&-!vZPrt>ePrfDFQS_`&AjxQ2Jq``qoVbteneUfbXwi4wqU-|k-YZ>s?%Ypq&4b(B@it;Vm8+K;MUL}y4Aw(2fKBFrbS?A5{od9jgTLqDf=O&q?2TpT2@tP$I@j(*{>%frG=&J|$upo6^(H z!WsT_l-B8wY}U^x-gD1Qy*7a=;l?b}a(l7-7q7PoM2}1v*&KgU&)|-rP^Ff z*)o`QkS0I`gb!;T(AX_F<1nmws86gHJq^h?#smJ?pM|m~+LFxWwrHz!p*oUUn z=y;h%Wwh+s`I!=Y_ijpwH+C$(4)ii$G(&4&ZU^8aIArPSyF}`Jbx)DaybCd%g6%nM%Q?}q@s*FP| zUdQ0SphT5uh+o~XKjKyoumN^EZ-K+=c#(7q)TZj`Co zn(plUqU_nlwhprX2VaT0iz%`Xq$=gEjef=I<;3%itpTm!VZDH}OY#d-rT5;;jw)HE z?8g^0W^#Gk_P_6Oms~OF(0a!y?D(F7mxVX+#+)?AOlx2*W#xy%uEts_SGe|OTk({I zNsZ;_wN0fGQAO>^r_Awe9qjcD8psY$ko$z(NfGMRueHOO_rLJw^Z9=sRezu6Srf2U z8$`&QAhh7xMRb?YCX0}X5KlLiEWW!4RqNcOiy-m(rFZi)5n1%o+Bz)wu2L&)#)SU& z`xnx3a>E~STW%!?@^2{rOeMC=_%h9?7S!wB=u3OXr{+SLNO5mWFRvs7w-#j0`1$3s zY2T@Pk)fnI8kZ>+A8`HIKVKp}PGj^S^zD+FS+Oj-s=YC9ae&jL9bir=1$m#DO~-Fq z&<*e+3HQKd1cb&Y0)&r zuROX%1gF5TRgm-0Md7{CTKm0We)YWPN&SqBk4A;X#um=~yIkWMft2S7_U&!kVgf%^ zF(H1D?!8QDlZ}r!Gv39hI*EjD+&8Lj1Weg8U0&Vg{Rqca0Tp6TW<9Sxfp%@luJ3@6 zn{}S-WTTJb_X~dxCfO3fY=^&qQ)ccUVtUNH1C*qmVQ+%P(L|zS;e~z$&D_qtckU{*O zh?q?v@0Pq@W~j9K|Ka$JpbmH<;xbF$SN6151+-&1`SfoadNxSrp9I$<;0!un3Shqs zC(mpI=q*^dbR(>lM`5@CA$>NKA$|8GZ;(O`zbnAlz` zRU%|c-^1Gn)j&RdITPRWNt=k^`*p^npx>mu*u76uOM!YukbYddjf?ol_pk*Gq*A2i z$maEQ_h}km%KkDVcMIun(&lKmSm%d|;e-nhAloQpE{F=t6pAR+Ha{hqfMz;ht!GOZ zXA87@<_vcRMU&3sn_{j7<3x-ZByv0}VhHc{!ALbS&-WCxpl5;rm#Z~=ok|S_+BG-F z{w}U53ND}7lNa#U)NtfH<*JpeyI_g9B^#>tl`r!d8ee4nDnzk*cQL6ts>E2M@Deb3 z?E=}8U%loxcUM-gGR#|dEkEt)X0%Q{J>>+ko^FE;T?|Q z{g4P`%+F%Xp{!Z>!~3eVR;qP~ISvaL?{ABBD?rK9@i$+wqb;NJ)m zahcQ$$7#n{m4TkC=6`;0{PUy2S6yP@z3zX!E&XeY`DAZkCiT%hJ*2%LP&+~oYSMww zN$1nV=goF;CtAS$Cqe!vk#bT@2*_Qru_)B%3#>o-k2}u4n4tJqR5Bd}ZznmAkCqtX zZEy7c<4-;SkMM8T$cY2y-i+BPfY{X6O3?K`q31tI0`!*yaxhhYp;RW{T5uD$7vi8p z%$^)#`Y*~asVd!z%1lciH@#NcpX2)e1K|9hsAAD5kG|&Zd{XbUUEon14_*K9~|LU4zm%#q= zm>{q7dZJ$RqrskM=*|8=-YFpZ_r{FaX5)`$<4lEWe}6ohZ+L2|YYF+4;Pxf5^bq=2 zk(4kJ=@w6#r|CI$DBWty9zb!jBZMU9_>Dn11f$c!2XgX z^PqKB=Son{MljmbP@4Gn#RKjxi)8SD41bXH58dXf*MIZCE#wTmYZ~#^v83EWDC||% zFL#_?5&r0eze8%x?RnR^3m>`BRqd3jJyCDmL1FOP3b$sBT=PXVd&T();5Zxe&SNa= zVF@XVHnUyLxD;H>Si0U+y~WE%2i_Ui6!z)JjUO+$ll+NJ{QmGqgj4~OY?dUuPIQUL z;5W)n&!9^wc$U^d^@)3KGbgS0f5X=7EElvPFIb3Asi?6)tp%uW!XLKU3q9^orfQxdXJV??f2 zW=6Yq;VRTTp~~OI$BlQ7!K37apn7E{?1-I`#L8B6HTvf>uyy#d_d`mB-Ho>CA~{ZL zDxc1NrL)kgbgdGL`E9Dcz4^jm(?y-wRWx*r?)_xu{ptQ z*3E>NFhF;YnH!Y5suFQF;CcVEZwY8yUT)iV%5L-Yh(n3H{xOnR-~0R`&pHSZ%>Aq{ zsjK@Gd2QiZRQqX6lJUq@Vh-+jV;rv&TT^wOIc)?L8 z9qy#IIlZqBoy@K)eOCM#2`EDp3)7fr0$)0Ot$1^Ny*KAme0g;?Pkt=U{d2*_Pt+3X zy;$M793Bz6%rn_V_q9>=qbV6=VX`)n^OFWhMFkJjwCV0*VlB>GtlfoPr1955*TsHc zip7=F1L)S ztNaz7MWI9X%evh4u3%{P&e$v=Mn7kluXmR)C(#uI1KiEg9eBM&s41;p-I_`1wnIel;m2{nz}*p#kb(Bu!`0gZ&Y2Q#yxJ zK&QwnmFB6zD%CQ-e*p;`ilAj^o7uC9tLWq<3DPHbsCeaG!sOsv(FX{+w|p)QQ-(G* zIE4n2FDwkONt1^K1eqY2U=X&G($`6zcFq9H7H|*?oqk~vd#8DPTS8S0Kre4@QsOX7 z$0(Ak0oyy3oMZA^2G zPHtg_~W6i5iG3K{8rFE?W-H?9rFrqUz?h@!DZbP zCGnu|J~wRp0B1Z>J@|}5?POm&1!gIhQv^GiQH|C#-4C~S$qP=-(R0Z^`)y6tnAXH| z!&gHd!u4yySz!MgVR<@TVuSaPMbQTz zj3z@#($p2yK(#C(GAdA-3-72ct0P+Or1#%y)>t+fJnBWg)}WY{);Aq5PQ6|^lhDT&kFPvfLV9I3N@zyZOUu7@%ey!9Ap5=Xk zT>P3nHx9|{t?5fb!PgsPdhp4S(WI$(TEN$`2^9tp`7nqQ!*BTc*ZGA0ECAy9c!9no6LTmtz!|Xm?s3FMQ=VdTSb+#=s65L#@b}fiTUhh-*5hfIHV_I8?pyW8sz7FvWfd zGQdRM61N~CPq<+6ro*t)C~qfd@LrN=GU+QOh+;T(=DPvJ3so8hqKqI&0REXXmn60x zD87tvEM^i+>dfg9c6IjXGq9IIlj#U!8=UaAvwRu0Nhd-Y{w=9oE(QAJBAf^N0Y%t& zb*Gfr+S|`PER+fX65SbYVavXSC~h|nlgU0dS%f;$*i=8aS)wI;Bplixh$dMDIoyR3 za1*(ERlJ7(c4eFj>ct2lVM2nhCY^%m@r_#q)1UUCzYv*W-? zaLAngZ%O^TLTp|gsfvH!BCkUSbEcQTCxY2ZN z!qZ{Itd2fSVHta|x|`!}p^X+l!p?s8xmL2nH@FDGq*K-m1Sn`M&V$A7WM zAyW6sx{&~qcv>c&A6R%P&<(fMe)lv5V196;$&ebZ!N3BC93V@p?=5n&+*NRmA>P@ z-BQ_r?)j2u8Y+QrAt27&PcqJ5ei$?v{C_RI|pu?Y3YM+!^Qr;nthABNlDOz%?UAZZZHM8b3AnZKrVUJ-` z!bW1CC7hw^#-`dHCkY2LI^_ZI%To0HfQ(=P$Y5Ft`_S%#%y(^}r&zB#jiHMxV#`c0 z4&7Gn*fpP^%35Gg0zX9g?o!t3_Tt04i>>bMkdXl}(i$|=EYsV)R?ERIFv=!C)~nm2lDuq z#f^nQRdm45Mho3Jul2B)Bty;YS%qa(Xwer;XRI;2tkc$)Kj$3nWcwm{(wCjS1Y#1a z23S4hAE*FkCIVIX;NUp4Ejjpdh(bUer#@N|9Z-(qGrT|F3oV~EiN@lne_Ic1x&Sre zXH+k2`bw_`YTG!iOZMmo+f(ZG1$kY$sKsuWMN?OH?x%J`>SFIt!1Y!7@|$xavP6FHwvja(4M-iI`Ka zkw6wrDzCDp_C)-_ANUjYO+M@APq!IJn$8-GoxK?hvIYf5szD*!G2jjP<(dgFD=RaD zOW@yNg8a)k@py<5sv^|uRdfQV2pD8$@KR<3XElgECDo+_Yqdgba?koKJ)}RZg~yp* z;wD}RQ`%yU_s-;P;VX=q^72s{rdEE=eRFtx6liQh@IH=yu#OE|uPNwnPzLI0FVCKZ znVZ$sfD^OI0ofSs?+J~$jLtp9nSyTdL}YZl8~AFfm{MAyDoC+gPd8bM&dsno4VwuJ z+Y9kVh{Lpn&&hRrMHrwb99C`I0QtheZCF$^h{b5A&YG3aE~1Z34gL*&IsDNEm!k87 zI#Mdhi1rM`K8lhrZUa@b6G?V~&5LuxVaq3A{ZPzvAfm5zBl6c`osMa@sir^B^LS!> zJHpBBp4Tm_f%=i*13#NjCMtd8TPQKplt67FUusT*gk6=m+u4B2KxnF#X-e~aFQuL% zRD`q5pZc?QIWTjHuka538XvKuiFn+P8We_jmjiP3LQ2-+z+!_A>s+EvQdx?4FU$9K zk#N8{#Kwx2@~5X2ve3ExD^T&YV}h6!A}M(1ZIakR@I8ON3qB(@Q6tm99x@tyPPTh$ z`^m^@HrPpc1@UA*E4&}M9R+3}elpeELY+{ThuiT$9Q#p|R`-=XG?f_IEUDH)@)SWd zJ*`V)_L3tyfjn6YL=l1_`Bo0qyJzYva5Wg6%woE)xYW2xqMT+As|G{QcitO*@|t-kg;wExs+R!!79W znBR3M7$^tYY6#qn)k)IdBf*IN9gmdkizgfM8M#6r?mS5`jgz7nn#D?y2O(Qw14(sX zfoBMc9Wa4Uy}yY-YD#s$j*hgLT+6vzfm=6NKakpXDl)OREhlIh@gz+`7(0zss~nIP z4BtmWa~_+xNqY?W%M`qW`Y^M^T|}jo-#4aOHK*`bsk2)*2;p+q^V^wS7&KcblyFVzIMWn+a8E z5Mr{DG}BE96}r>LfvP3XJy6gBIm5FwY*dlbEIOseK4ryFD&_>m;mTGL$_z< z1n)eG`j=$^t=*29qzxmsNpszIEsrjL%)PqcCdBwy$zh1Dk5qOdkg@!#^ILBR5t;59 z)*$o^6YFIaFWIYauSD9bfrvH|&XOUkDfO&8IcHh+`xCmaXS^G@WJ#QpU}E{JCle9k z9!3F1xi+pTzn2kaWJ!&wuZb>7BeA)Axk)!0u!}OUbOq(Jsy0TCtFikCpq0FaE-=Hb zbVk#ms5>Ic1;STGvwrb2wOM@XGG%O(2%$f`SK&q!-e(1@Y>534zSecar>u~{Y`E&w zoNVyrdX5zFXO-9nCEK%^uM`emS0#e8HnQ#N$VIrf@Wj-o{3b@= zYWU6iwkNW@R$P^>5ORzd} zrU_uNrLkt6`ypiW=F{WQtn8z#w?y_6>Et_ND~L*h8nob?Qqg<9oHwkY(bgfrQr~j9 zJrz5X!-cKAmRWBx>Unx_71wLb$lz{3b7v??$VV!)w^|_RDdIbMQezsFXdn`%t^9G2 z^qu9T4CuZ-pH^ZImGlcWElr%q;BS7(iw!1KcFXbldH^o04^z7=%E_zv)=WV6K%oxM zv_hDwj=zMIA94q#4@^UiLV%rlY*f!Nz{&G1XaK|zbQjMtRftkehuQzY3S+;uc(1X! z8G7z{O%P;C*b=nf-~x@IaS#ZlAEbY7>irJ=nN~5cV^BZiurv|Rq>h-vI}W}9cOg*m zq^-$kQflTEezqc+8;e|WcK)BeCZDy>BA+C&JjsS+S~?OB4vrMx17TF9|Em(4{oU`a z4s_7ea*bR(3&!y6viB=M|71e4>&@rH@>8#&S|#K6_=S7E+UVU)=O&K`i7c zqMXZUNer1TQ0(x#sY2YP`|9y$NzV)0`AY!5ypdi^Q(y3Llw0jcU2a4tPF{UKEX7h6 zY3c5A;Amu` z(W6>rSBbVF_*{nX`DL|5loUb{4_*bIaii} zI$+Ag=0;*~%dCw&>{0+Jfq#bv1EZX#;g1kmx$M+c#W5cj$}UoJ*|hJIm@oGQnB{CX za5cpFG$El(5|jd29ACvBn&{1cD{!9~L+zZvtI%KL1_2Q#aP95jqWA}fpP{_R^zofY zzQl%bDx4y@_%3ti1M5U@@R}3Pn-VGBP|n9Ya`{QO1s7GvQx!n&A(^`y-%mADB#neXmY@>T6Mw0-%}(cbtyIyWvrGC6 zRH${_ZVO;DDxAX`p@0NSBO>HvGCIIyuos{5rT2^Icg1XY5JqC{$KU`U z!pUGxS?m+rPg9~#F>8NoiX5>@wipo)>7pD5k zcgw6&7(*g(=JSGx@S_p$ACu;g+Fe&Y9`3pJbNw`oU!uebiXOi zfyiYjId~p1?+KzZ{xxRoB(b6C)NTyb=>-t-vV0&0+`e#j5*l9s=~PTnhZybvBh+-b zorv}A>Go-yu&gYct!{7K>UQ)D)haTUxf0;EzTxj@$IOwv?>hBovCqy5P2b*sY8qG} z9c}Pod`7#Zd5$*!l(jL7M&G@I;31s&P0VA+7L~{WDZgj`z7m)Vg-W z?&^UaNN*?#-_*0#u4RPjNbuuW-E2;=m^m&b(<{#3IIruv9Wc(OiKTM{ z$l+qnex@%j+(550l;^g!D+PO>{6WY+L{tijY>QFBmMPPy5CN^E7#Y}j7b1xT_)uoy zO6=%#>;{`;Q5Na=l)Yf`P|El$iiC1D<3yMg{XTxSOOg@%B+1SoJ7tUA z)e{Y#)iu|{CoqeWY!h5ZbFl|B;bLt76qgu%X>F1=#@6RY2!j3GF&B(bemo@tA~T|E z)@|)2gf5O%Y|J+{2>;sNvCxC^@9q^rYjAxOlJy&ztcv^OXC8PTp+_27d=BgRRnPgy zfBCT_!}#vg_*zL2A4KG=t26Z*ctEAV={4$Hrmb@(tR%;stOOm=&m^B!A%dK;<|IdE zpp@-CZKUn&jQSHmFJm1x)-Jo5mnuEae7wWF3pgdSnbI^;6x5NWIPN$)rf=3F5Ih;4 zK5bm0$)dbwj9T>aM%d_jewx7VgT zmBb!1HO&o&J2LqAc#DGvZIB&XHBaMqEw_9nNQKif6-!gW#~dF!6t5=Ng|5D=;%~9W^~H1Nut6)2M9Q zvj%yI8!WwUE@OKU-_=^jgO5NzRgV-NNsI9OYLV4T7PU3UgRoUXv578}9~SKGg%qL| z0$o}=ChRcJ4x17(Q18$X!*`2T?A%kt+TRyuwUbO**-Ivh-enxR|7chU07@UFpt6Ef zz`~WNkLPfQ?{KlZ$y#^-TZn6hlqn1CFdDQx7;2Ow`sk$>j|?T-PBj_(ri2YU_C}Xf za+MH>biFcDK#5Wxt4A$^wEPNhnvo~38&4qF{lTdvAtZAVzoZcxllfdkieAF)2WtmU z?EW=}2ofFfgg$dJB=YPH(Whs&NPT;kKCU9s44C7+MY=Z_2kpRG2)kGjwW0GvhIK#Y zE7bHXq-yVw!PFys?P*@IZ_DcamJ&^wiAZETSklSIRg60VE;h!eMEUE9*h!Pl4}M`L zgUBMz@LzH%!?~89fr5)*Oz7@m{aL2KP|Hg-PzLeGMb=jS8JW<~RU>Hkp7=;T{&ikG z?{Qg(qf;B$CKd+~JdmP8pE(}#YwYTZ0KVL>Pc^vM@ClfI2_>d?P>(IJIXk1FcgW-p zvW*od$q+_X;TU+d#-<0qg*S?YLYd|)c<@b8BnhzqGn}R&?RmNHrhE}f%O_zBJ@d(> z4SvSqrowE$OzEg(}Q!s49jFxztz^&a^bJ^oD5N2 z;y2h#u}K(jO)yv9@sKN%b_}03TSIwmMrS88P}IGN)WHN$OvHp{T4yI>ct`%Rmw%dA zC8zUHTaEwwH1JCtc_aK01w0N(d!32B1@F5IQRD}P8nyJ`L~=OIVeg`2PZ{2$Jo{Do zR-YT&`wbIqN%au~=J>k?q2!m!jDRSzgV=AZ?v6jLhMDBsE^#7u8LI54eQy5%5>Z7S zOWREcQ~KL63K_5w>w0}A`Q;!sCA_X&HZBa_q-3`Dy)T4&8hS>)0$9C>5giz`#4-EP+aDr7Y z+eL!WAi~OOZbJ;&sR0)b#rY848ETOD9me%A5x*CxnGkWl5oAh7Z4)Sd7DBG(f)M*J zrW#g$3PT2sf;RG16r`l=!z+#|0L^s${L}M1haF7%H3|p$(2BZX_~IX!m@$+Q$ACK> zm0RA(nA=X5^scwVL(R37XF2_)O+k-J5b!+us{XR3j1ck5eTf1OitEBE3c8_=Skp0R z3V&R#Qm=;TH+r2!rzCx)1u-byh(*H<${dQxto7~CbkEyR;mPUW zSEz3!aZ_dX-;KkNb-deP?*CJu?m?4j!L11_e^_iF3qDtBl#gcDD4 zoR`2ip`F{58aldA*Ro9Dpo#nLtb8eYuU*$=oJMoS@xVr#r?IVQC>{I7V138&`vQ1Y zo`Yuc!!K2SVQK-JU;S4^fKj;?q5nGz5SOu8!cI`4tEDnB;DTO=vUQgBl)GQn-S2cz znI3`1hTjqpT4kZ`R}Z+vW#UHuY=pC5_eXIPf1Zhue$U5F_ zeUMlEB89io(!-GNg|Pclm*l2Gb9h@N*!xKSZEO34Qz;11|28`~99%Y)Y){{O+om3i z_)fg{@U3`zAhuXFH}-)EcKciULY!~7wGO8Z3t#L?KY%^6f`h;c_e-AX z{t9#11KSrR*Vt)`Rz`|uoHT5UpGjS(@BhbP)JP~8 z5Eq#Y-{*-*iArHY&8cy8^@-269+Z3e?N4FGrvt=J(VBbpEWO%N$+m;T_4i+IcECwI zLTt8kBLn7t>VPJ{?r$Jc z3IhhxGLQ~QX#_SH0@7WQ1EjlAKp3MO0@5wrDXrwFAzcDWr${&c27bTq`~Lpi^W1yx z=bZC7=RVu+InO>(Ku{3T2k&4MWADM++~LSzTg8U4s^!Q_c40-<1v5xxLo2fszz_Su zBgo*Lz{j_rerq85%^w~@0TYsG$+m0IFVpytRJ!vH2A$#8;@=f2dR=~5Jevy=)THYX%rTu+{8sIqZo@5;l#sn2lP+@M-G%U9Jfd+)IRh} z5ZIFs{5tmgfSRNZB7KKjx}@-ZL<;6vrf_pm$k1HY+Jey(2ALlGdJDoY3b;!gw!Zw% zrAydTc*?CP0|~-p%25!RUT}TC*3Yu)7VGcf96 zc;`w)$0Xtld-v35VUC)lLtL#`dpzkp`es`N#4UklxfrdbFHUn{PY7@y zpMGVG$Mp^(MqBBKAegf>N=+>$p1*mW9?mf&=1GqjlwIA4{FFZN?fK&6l`$W>rXbVg zlN1Wu@pmMcfJ~RO@oQ6iv2r)^+0@&`rbYT8$4jiv3J{kbBObXI19dSl@l%lqSz=N- z-m4|2AEvZqZ_?Ow{T7Ba+r{s*#zX?wi&p2EAtS*YLeN!6pe~XH#<%>vQcNh@F1qRv z@4h7aUUFk-zDkFgtsoui`nKjmakfoe6)G8sFj#t_^xB(t`7rUwI3*7+l0t5Dn2BMa zog|Hck6rSzO%d^XzWJMXE#%PxZ+0T-ZWSmWEz|AEKwS!~kk^K76;fm0rxLjW!_<0+4ZkvcZ5n>5}vJT}q_W&p$iStaSQ!hCp&wOSp#BTonIj~y|f-Y73Ji$$pD|kq=8Zpa9$+`f8sDO`! zw{&{|dVpkCHv@45)2FBTFN8|PA#S-dp1pK$fUHGF8NyFe$aQaH`(YC&;>TDRw;KGi z7ca)SNE6dc%#&FsXdqWaeUuvM@qbj9t3w7JWc4Lp2Ci0f2*Xlj zsOb}nFdVt73%JYKCcVM;+j(tZ;-k1H<<{2XK(B!2|wd~wJREtl``^-QE>*{-;vOS}RDS#TTcSMiOsu$o+ zXrLlzANJ?^U`frg_-D)T!G}U2q9r2}fkB4YJl1$Whs!?!`{U7?Um5!q?<7Tk-yR!7 zH1N0QtgkD+G6As&CA0Bg7OO8!6l|{^x2KK!{n?x#0iM1i%7Qis8T;O9)|?P>LiVvG z4)ykbkj;V#0R4Y1G&@6Wyw@u32q+4L2_|U?82@}3iuf_bRatZPlIjz4`uleG1US;ht3s((EWtBlnG1#U&qP-!n73yqr8jSA&}zXlr6V z*vHe6;I061Yun3QsBIt*gdHzIALIvRNTd!5W`g%qRyQ}urmDw+5n>OE*m!XgFwcRmqEnTuYSW=Jx4V4PZMrkJzHq#5*@l##XZTR-;qycEQ5KTvsW#(ZNn z*CbJyS~5EcARi>L4KBXaBKbNs3Fy^J)=s${!xFew0B1?tS;b!5+HIt>H@+o6&_4LE zX?3dSd%l~Eu2RTe4-No!JIt* zdnPEXNf<2~%tJN$7!Kms3~ovzuQdh`t@uE;47E8Kl>2OajgYye&HO%jn5DRT+>428 zf{u5TSUV}`K0bR}fS9CA_7aZO^n(+v5X+Nsj4Z}83n zwOcn2w;OzF2{6e9O=pYj9|?u?Vt2AiV-6G9p}&B27JwCQhL?60MP9tf&rKku@=||3 z12AVNxX^I6>wSZtqh%rJEBP){P}S!T=`s|F@fKaniJ2W3;e};1+RB@Gp;8ZmBlihaUy3^rk|E+tL)ZqvlM9;C0xNtQbA9#f9aleZ^YM4S5QuSPTEIbjcXM-=>`uP0owkUIvl{ z)U{8N@+O$jWBaxI5{25fNP=Hpf|r`O^i@gt`d<0p*Lss3xfX$^KSBG(@3~3Blz~$d zZ`~zK*FdyMIHm*NX@zF;QN|irY|h1$#_##Qp4OwCeLY@Rk12?d{$vidtXFyWCEh!2ZA=pVN+z}EQ4pZmnX0g)uUja zdq8QU;v+p%CXD3;Ame--jU|@(J`iq7Ui4;0+bvFmd;9(1_r_v_Sd*s=veMnwb;ol< zG^QGZyBn31_-Om!pdIyB8H|z23-8qdf!a$iWYGcH+v#i$AmVhk;jz%CajunIAPW#& z#BJXxO%>9D9^9@{#4>T#85W{?tV&J`%CEWrxRdrUrwxpc+3h&eM;Yf}B4RW0Ey(NZ z?TNA_jo<|!D}z)SiasMtX$X?5h;<=~yKC_afW~BO%;L$(9OHVp1W7|mTs1HI}7r& zqPy7Iv-7K5MaJZZt|175d$6x0OTV9k6Aly8J+gi%CKXi&hgqemE(r%g0;`zYUid`a zR)7&CYnvOx!68OS?3Fj6TNU>4;Vcx)&sv=G0m>V-KNR&?E;}IpD%50_An1LVGj;$Y zsLB1bu8vJLV)fm=1=q%@V#jxjxFB~%p+Ao~EU*yY)W+ssYqEQ#<$`Q8LCG>d0dv_i zZWXG?S4wCW1I8RC;y!;G!4#?18s-Mn{LB@pw=h%Sq*J`*??wgl_sz&9Qd#F+4iJMc zm+3}j?+5$ruTxa7SSi-;YYgBePANiEib1!#?A7JuAe6cT0AjAJK;L$^t<73_4v(2L z^{5Mw2h)NEWOOSp8Sw>54m0&+2LQj86uW@ub_vFLDLbpfPq@>5ACU zF?X(emGGyo_5(w$`gv*+bi}?44ll2Ed}fcbdGLTn&NM)KKmO-D7MBs*9Xbq1k=90; zWB{<9Lbiogl`2>fD?%Ze0$oRA{-_t)&APz|=LUS?rqM`@5WZ8zdJ=&Os(IQDt zqcj!wbQIg)@mch?P0W||+`XvW1`~g!=tud-Lb%lK;{?KO?>4awdFDcn$0#!jqPJ7l zDR6&mJ%98Qh2vABm=CR`%|~!72#-qi-g#hS)(G|JQc>sy5=@cDkpau* zSWpDZOge2B)jpD4TKo+tVk2JK?dpUCj)LfR-p*!*^LT`IO=Tb+Te-_WgfaB^i3Iq) zP7)zP!7Zi;R$nlN?5pkN9zP(VK`%tyX2$UR$ln+zx_oiiBUx$Pm1&zLkm>QFQ6l2U zH}4UXP!E9OM7O_MU6iRoO@4n24#@!{i$8zmL>$0afN^gU$^=L=RI%8bP8>2$H0OoDuc(QjpefJuJV zZe2Ufl`>TE_GK?2N!>Uy84($+J*Ye~ZJLrY*TYO3;suaIT5H%h8nxF0k6?%2pMW2n zdD3~WwGGrv{fsyi#?9(XbYJikA2MSQb6M-o?esU!0h~Sbw0f5O&CkUM0+E)pse&%i zZPbQIxGf5JCATZbe9_f86QMkd6JXY!`&8uo4atKpVq^m%;@3CAodJatK?Q0 zVJpZYxi2~U>_LG$sMAZl8TTUk&Hc9h=$)}&j!6VXdorg5AHQ7~J%=P|H8Y0dwB&|)S8ojPxep5s zW<&+<6AQI!P)cnHNj{EPK7|i5^9gIIcg;kTs~~IVv>hag0yfe*6Km&OBjrBvJ9bq* zUVaI&^17>_42P1hQUtfVDnQ;VXy-#SL05M}8}#jJ_u2M4{H!k>jnsqKgvx?kwS)#e zxKm>B-usEQio~o>y09x-WN~YBD7FwbMYgv&u*;n1qsq1!(4MOYF+}L#9)~A;Z*vyC zd=Z}9XCGAAKmHhzEbp(nGn^ zSaE{Fn2geS@PMou9mEucNFCv&2V|!nGTIY{2js_5RW#3`D>aVFNcn}P$M=!`Rvu8Y z)rpr8)<3txo=U1z;I+L&kg}N)Ms@Dqqdl{DRUmn~w`pLuDv2$?SadRKvy;Y~QWpASb7keh5R#Q>RoREEy zC@~ykQhI}H16QDtlSLY(jtSIn^ZsD>$WQ$scroi#zb;^yQqk3vNRD2&SdoU{0iE7; zx8Nr*w=r(b{CVQ~ZbzsrF|I)W{-^ML{m8&74o@Tl`eCq?+x}_1dkHrRov(QC4lyaG zPZv8_OqRpPGps!8o-&`Y4kZ&(@O+GircR-MLVZBfCf^n23-UU>PwrAvse+V-1V;Nc z-kfl`KAg*#W(oSX*3MDkhc!Q~gEO?%e@5TfA_xU|SybnPrL({kV@}iC=Q{nHQ zqCLG>81AMlGbKYx6-0!hd9kIjSOURykA6x+u3$U1A4%W6WwGH=spw^d+zt#lx-@oK zmVw+Okblm&`m29jNTf{5_Y4SRg#u363y=gZ^W1Pz=sw+Ff91^oKUG>v55Uql9_)23YvjO_Z z)$XA!erlCY$K`hFzNpF1iN|FM*A;1})QrjN`jZYTvede1*D20i=DNE;Dd$=}E;cbkVvrY7Y@0LYx`d6Sr!LM+_7Y#gma2SHfwNnC4Of!- zUcx4U|02i3d96n&t@EvvMqLSU*d28@^^~$|r`YbtB)Sd9uP=K~*&E;AN1d{Nh<2>~ ztPkkee{52raX(!r^rkA4a(Ak3ht7>`)$~uj=)Sdt?L)|e_f_2=C|iU8mCmo`&+U#)%GDudQ3;{*W{co`ZJ%`aL0yhm znvTL78}`HFYp}aAjoB9;^BzBi(XF@t*8A#}`V>>(*_z7J5%Rad5oF}M!^HgYtAP_; z%D}JHdnvGG%V_-+z~It@X!tvw!8uN;n~KFx93$)piSIt=n_|` z2`K^MX0I^CqArY;LLbMib)HI0JCSY9u44^M&HO$-4Sd%-<(;noc0QQ{aR8dCpSAUc zVOKo5sdhAaDZh9;=G7+Pr)+=UjmX;3#QV{bZPK&cvO&+6gwgv>*f0mRjw&+?`7d7n zjQFDXpUH0K<+{@LY4UkAhbh0&zKpDTv#-6V`z5hyS0VWF2|>l3n=vhyM_xjQdB@?s zZMxP!+NK0{zIdfB-**(WMI|=+?f1FRJEsAAEEV2#+T55+r*q&1`#2$6vv_=|2?b#1!vGk!;R298=lPqx-l#^yty z`7&ey9AXWL>Yjc+IQ>$!kN)AD87-2We$vFaVPZ8Ccd_8t(-Q55Ky*|O@&>ps9=Q#Y}P*SlN%hL zI$v4>O_T2}3U|~ZM9W>5OrwPO%9K4v5jZJu_5H#@H-1SNq=D!~a?&fR<$+n#@uFaJ zx;sRFtIZz=%Uv~dnB6Y>@Njq$F-KPeQu19#>l_CfSQNzSAp!9V-#hU!MdtWu&4;U( zuoV+_HHWcxsg9fxo<;VTl)wc zlQgc>X{9^xQ-Wd?FSwxK8GfnudG|tWgH#kNo`0a6=I-0r&_|%35h)0eyqXlfB0H$? zy9o{#CA+gp$jb9x83LrEEEhxUOX6M-rf*cD#zO{2aY+}Eid%A4@_+BzLjb*?uUZf_ zat45aR?{DFUQY;`lIGR8vMYC!cp$fg$$xnqVf-3|I2O3 z-3V|}4Yo%ny{Sx{4G=qaU z3&=CA4X#CJX@1EmO%IOrLzlp+aO>If%s5|C#DSg%}< zXlFNZpD5Ly71rXzVuvB8bK;fX(m>qfi{$`z=`tLU+jWnMDP2I75z;%oYOO%# zFw2NA#mk*RVcT9-6N`S~9@#o-JU#FmjOdac5<6OSC?6+Kka_{BsmoTpzho(BwkD zDpcCX41;K5AsYZGKO{8C!qwZZ=`lnUh81g_^AUKI!%K5=Gc-z%K9%-^3k9&29N5RV~ekEyE(Z#_!f z?CU07u7xJGOJy)|8=Juj2rqxUZQ!kOCt zLVJB39wM)I2PgGhUHBC^vI|O}am%{L$j;P8l*3_EDRqp5E^=wN-5KM>YlVJB{D{~p z#;gJ5TjtF}IqBLaS5?%Gd6TWoJs%bfKPZMCdODp4h1lD))S(i9 z@`guAl?(h2P)s|vh{23R*5+^S zMI@TY7l$>az)ZG4=&YlxqpZ7D#|c4Fq}7)B+~q@B4RN;d^C1E8y60Wi;tBqYvYeA<_kC6Un_mA0)bj2kshVN zg&6KLVD<#N`RfVhxtQ|x3eeuN=iH3e?@s4tkaT+Tz~Hd$nvmGPoVLo=GROQ4R=@`G zzDFdi0hI490@aD5?a3^A3gPdg51U|||1p}f4l>gSIh@5t{6G(BACk+C@XSY081)Q$>4LrUwRxJmXQD_7-2eJ}brl+~Y%lWPLN z8zFpwe3exonPyOWA!yIKJUt;p7}7ky3JbK%&bp@0Um{@ahf>jG+=+X}<aqw$}hs{koe?f@y~rI)mJRZMN}DUrHKlF!nS9w2^efp3;EgBZ1l zVoHdMeP0;^1g6-DYsQE^QIQZ2vA5|1{-OnXO?Z~Utu@VE?hG?D*b$&dq$A3t11uszJkT)q;A4$VTDk( zxFrRnl|KP4zYPUL9?c5{*I253c+{R!^)+={%cJNrsvcy3c)lhJdk!l`B^V!P@YK0L z4l1pS-kr%eGMkSg99i#VQl;KbtpM>APX(<^+)LtOH$~Vl*>utJ|99rr(CTpptH_3u zW7V$IpOIAZhHgBFRUYNRATg66n56vq;tm52X1ypYTccsPg{WfP#b_ zBn$8T-dT2vD}y z9Mi0B3~@oFSP6R3G2UUbnwqMg!868??Vq#(r*r#Z5FNa>emTNX1A4xO?J5fsnnD=F zGf5VEt!P)(G{PVkzpLWyXo^zx01QcTAqLS8ye8g`Ts??zh(^s-=m?8TYhmHczCo`M z1n`WlzncGV;wn=vy4vjdR~neTGGD7BSNU+^q5k03M_|D|P2~KVwx#cDp|gjS zL+K1N%rjg=NR3eN>zBQA--D0V2YMq~XnP*~#|DE4jzS=hwPdp`ZY0U+(M$;(r{zix zMQ@_|&^?o#&bkCn8$+2o@J+_1^O-J?uecB16?^h^f2}8rNcSHOQQJb|#}L)lNQY)I z=HY1Qj44F6din8Ce$~-A7dt4char=PLlCRXz8I=4msMlWrUd5C%*pxJnyZ+)_ZrY8 zbZkpIz(ka}vLlY{X!V_~ zp1HZXlerahY6C1|+1IN;?n$yk zyF|!21h}M8deFj{;9vbyEFolQ;JSkL=Ej(4Yq8yU`hM!RDI{w;md%U4OI=FD$K;nj zU_^R0td{;kU`=txY!Y{v4rx+)|rV% zUt%NUzNxl{BfWlwK1|}}rQ#ag-%TN!65H5sqCP|?cz$nuMEm4l`;!!-D)o@1DvV>D z$7aL*lSebA^Tp8D{E8BfqJG7WBfwR?8YBAI-6MTVpD8B!qvMApe4Na<|3knz1V<6D zC&X0a{9D8YEUS{Z@wHY?q4suOc~U7*Ru-RM^VFs^yXIrdmmJQQ1C6S`nP|sF6l2QV zF9taO;{h0iYz%QAkIY@zujR@yOtkj+QnkC@_^bc*Ok6zox?!`Aq5b0aZ`aw(&E)1C zu!lWuiWQZ0;Ag9>l9&H4)u@(01u4i(Vc(%imkIu~)3k9#BgOC? z8<7nFL$=B9RLt9D){}gdTD2XNtQ*JYeZ6DCD1+M@Bss2OW%Fh56<2}6F<*k}FSlKd zOOnV&R_0toZIA`Pf{!{|C?}kfE=*To*5R5K=(GEKuQp&k_J_H ztiKtRjlD8Qt3%f9F^{}HaN$?t#}d&6@dWOjK#wW~g>kQ)1=k4oiFBpL%T&8TT>8(R z)`ebQ4|S<4F=_64i_w8W;ztquo1E|JSA{Bl#-1eCb@)*IZ@$ja0?)gE?1W!Cl?@-d z6yy94egzwdB9iP(3O0}el#gYvB@76l`%L=wZ@%qCa3rytZut&{D$!}?Q5In$f(!Cj zbumoc?{S8zvL!@<;D4$9J!mBpxT^#BA8-csU!e&LM)wE0zkb5`H_wAZ9<;s*-i%FlJnwq`eV2Ef&XrFuZvO3rh?V=2|grp5}fP{0v|y{4MP;)D8_Y= z9rAgv*m3`J!2(hX41eA%p>FdZnN=V}*2qBYu+u+#u1PAe-hjXw8Tn}N&VzsC=pl1r z`&nkp!LF6_J{>nGWnm{x#c;CfFShz=QzdFkS%0mvMvxAJltx?O$&QosUSc+)cwd!k zYu#Kgj*Vc|7gYrtj{5H)zh39?)FBFqA%sDo3PLlJ5nrdt<{(ma9sbXU7caBjeH%VuWfbKP<@X|{9P#1clxA!FG?p$`cnb}jKhT9lC za|4JkeXO6Pv2GT+tA%BR)!cN=Bv;-!FFoezK zdj=v-&-whd}N}6*@gd*9(fy1zl5DqP+^WGqUt9wd_toh%OznVpk zH<{%lQ91w&jxXQ;|HuAjkvI^uywX1~5*oEv{Vy2{02DW<66&?{;6MNSu~D0u)!Cq_ zTI=;J4ri|#e8s1z0AR*wPo!!jI1(H(E<9i{w_B z8OP#FCzw;oV##^mSow|e8+24^K`_336%f~L7RUJp`ETvas>P=c+NbXoDk&BN9Z_Mx z+r>Cv(u2`a?SZCcQjC)(d-Mk|fVx4P7Ph90|R@b^ove!xpysd%tN}EmELQZhW zAmz!#Iq#?I} zwqmV9UsX;Q~;Pp-(f0KNUz|r;b9&p z2PK*ZY=M{{v&^VuXbnqwSi}$@M}+}%J+LxH6BSuN zqCWpsISqqTG!NMR(x2`wE<--%;;l_73psm5)s62ak%=v8WFd)v=Z;^`t=ja0-8+wr zsLF1^+AfeAS|_+!dY=hH3&vLMFN|Y43d8WxWe6|13I^&e!B-#+_Z+&PF0579N;+9L zPC1zrhFWwJXBU5@X!bHId4bXB8om<-uEeiu^HB!J2cfucZ}2|Va4*gU#f)-V79ZdY zV2pVELc^fmsCPm-_0yNZAmP6=*RU{X|2+EmiNlV30q9pb z|J70B#$7tj*lJVA*cb$Uf^;k<>8>gU%U(y_mTxktr{x9z*n8_=Qd|Du5< zIP9;$LIlRkf-KH@YSSw_q2)L?eq+Pv-5n@}j1ZxB@*!;l-Po^B#-baPU99CMh8w{4 z&oz7{2{uu(C^4@mS*0yeF5eJ~yf*@fG4%Qz)_;r#uxlu_i}skV-HeD|${yId!Gj`< zqtVv_oI4tEE}fvq#{Nlj(v+(YH)aqxb zuq2<3NTWTfE+Q|wdY>dlNtIXrMj>9b=%;6o_I^(dmWJr2uSE6eG}w5YNL0_Q7xBd2 z3oE*@_~|1pcU1MQr2*LoQzLV1qAMZbF4yeFtr=a7k$3O1IKqlt1LUq@p-|l)Hv%VH35Ca(dQs%SF`j zp}kx^3Y{6CcjD516dKn=8lB@P$^n1dG`>?ikr=t#(6j!ZR?W zxWxf|ZXx$=eXB1QWSz||TDS%Gtl~8k*;hk4tmFI4m^~yn$+{W1B;9=~%)= zrhft43k%Yvv6OEa=~|G$c4MK$+}W)pz4Hfy`OuChGwzdmjJYqY+rV6S!w)r#%!o3}Lheb(?E|ePb9U%x5!L7|k_0^SW)0QG{M#$gi-X6(6y7w}zd< z?ASt+IQ8hh<{*&FQM=d&w4g5wIlOO63H#*!CXl+dt99aBFcKO}* z;5no)KR?uWn1WYg*OH&b*{;(2F!m$Db>j7bf1{R_CFhW%?@f9@eJSpZ64f*e7(13c zt1+ByC>gJtS7!s|hF#UReCUm$Aph#(CfHbrVt?089X(;TCPl5iF0iY#lE8UmXo_+T zcdQ~5ySV0YL7`;LNu3bip*HZG*IjK2kj~1RcD#%8rYQ}a_N+mJysP~m?aYd{aKyCb z5Ba0gLlNH(+l?XiK5%!Wp@w@0b9=<9-*@-ov``BOe&y~JcDEq|$a>}Ckc0#&f> zO-k|Yey^Nw8gns`zOR$Z6?7b%@^imh8R&$i&UBs<*}_|r$bF+!cmec_u1hK6z+;r| z+)iAC0aops8y#@ry>G|O6lIH4*IANZ`)l{|T7;snQ+hJ{uw;JxXPkkf`}HggG(|4V zC0BlFEi=0Db0El90Od`_Hv#U7Sa(7oMva13mvZhKpdq{YCyJ1kMt_!vQRPtEQ^At? zzgz*5%4jJx2#oP7f~r8Jqss;-=MKI$U5#aeV7UJD7@XHJk?WF!YY<5J*hYv%t_nbFI@+s?)lI>W>b?u46(11 zNt=#EsA32RJDja6J5aW9{1rCg8Ku3}nb|NagD7g<37F`Q=2}Ve1Y;0iKQ?88Mw&F1 z$S|d4plH!6-)7a*)E19?kpBx)=K}a%MCgSfk7@6X=p2iz%J#RE8qNJ$$c@)ngW7ec zy${XQU?sU~fGo_~nB+ipS!?fbX~+HLaE*OI1KGX&d#>Q}#$u|ZGisP~_nI=sf77Ui zX2%9q32yd5K@D(HFh0MZ&XRI1 z^vPV_c5^=Tv0W}+(xPtm-*soy0bvM4e^~|vZa9|4{*@1#bBY0*llX~FS0U=pts6qs z_F>i8|TSgQKphjJivK1hTqn`+`359ur>PYy^rv|<^aB^tX zK>%6mZq0*l$w5YYJm8(r*m$SxYw$Gz(H~KF9$Y^*P1d<3Dw0(Q`jDI1U6fjsU-~?& zeFGiCDy2&>JOUyMD%>HETrDeCQ3VL<`l<0&ye439B|Wa;!iF|rbaDJgmxap-iN)+M z50$Y1{v6&&9=-73v!NAbWav^oNS&P@oq%-WB60@y@z zV{_1X^paOnfT?r@%ctz=68EUK#GvD!!<8RVWH9QC+Zgk?wU$ii27y+7SK#yKfqtF2 z#Lqo8{4J$MC$0g;?6a6}3isJ!+}?K0tr#}lVN8xjkmj)fk#K*I16{=@f>lf$esg?- zjj@z-5qzIuY!z`q!2M`{Gh|sVe!F{M-sAN`5Gc89ODM7s6&%o6t$SG{74JvNjG2IW z{4vp3Dg{Y0c<=68G#>7^*ac_+J)L94HQf^ysB36V?KrXm)264HN;F7fw(+hTuc&pF=L;ro%KF!C5fTn z4yfR4>)&7nNqI5oOO~?&X*Vf)^u=lQxBv8OkysBhZ7h@Cle@Ouk3UIBjG;dAyRVA%e4`Mm1M5ES0_7z>ke2;( z$=}dnK6RjYyN}h3eCZ|K6Ox{;61_2jcm^+7$;+hSf3S-Ja$$_VwHf8;s07WdMLvG~YJ7`CS0gNH>*%3u(Kz0h5^q20i-@Z~hspK#h+qb48>5xo(Ss-+rnXyII4aX` zlGnH1S16%n#{*mqpBZy{&CG5Q``{T#rP7$>UG^V*Gw3`%ce6mwf7s`fCc1s1dtcDE zic5lze{nhTPs7Vp=9uoDvgl{NRb!VP?vo<<*30wwKVOhsCAS)5t%~&p zPuILQwJurb32!VIz54AQ<7&QrRQgPL+_$OS?}D7SmvK^s5Tg#~$zH9u2*`;bw~EOj z=D)yV+M)fwlULCm6&s5mVY+&xteI8W2yvn^4+SbnaoAk@yPtQmaNb0yg0p%--1nk> zX?q+^V!9-0VV+q-=7>c^yU20+Yy-XtSdijwEcu;{@FQ*}92SD!UN!{;b}nDMz6jMA z?!0viZ%rO7rIBGI$1kr!!n$n)Miy#zn3NU03l(eY;wH;6m&lANlmjqG?CR58JZaT>4Z&Q^ktn)NZM(!@J^ z+aq}aWZB83mZ5&B><;^$sQ)LZe$`is>w!38L_5~Y8{%&Xe8N%&$u$k{Jk0g-wJBs& zL)n>0)j1b_uqqwjm{c-2`_}7-vFUSfKq`&#v23D*|99&r(V+>hAE-)OIQ6%=oRama znGv4MJgO5Kd4v6Q!OUi`pA{NdV8AV5Mf*#Mp&?N$T);&CKu5f#@&~?b5}C}J7&Q@5 z>6YdRLYwcpj8^T(KEz@6OK~Bm6c}Ha14gFkk2w}9*e^~h_&!2D@a-)+8NP&tBFb*b zZ}yYvg~Y@zmKBMHHKhbGO9jw&;bHWtgl}wp&WmzUBonSz0EG(*vFB8^kFp&)BP-Nz zrK&F3C^77gydnT%bYl6pdfTUnK-?c-U(#)ULtfN>Z;3#?#QCGW<#RFqe9L{d@JVixSUb%++ ztXIw4%-T@Z9)kXcvL$o+2&t}OB8~$GGhQ>PSr%E?Al3$t z88Vcc3xe;g^DgK=X3ek4zMK%EiR(7d?WoZM8!!)YSfx=|evPoqz@T}Ye1biA&eJq| z${M*=VW-_#^)Njgd3S&sda>R#|Bl7w5Cw`84eO_EeCAZpIo>98?L4kQOn~i7jEP5F zP#)=zZTO@k<(_-#a`G<4v4tz=OI{i_PPcSN%(y@uX_j8gdBcPPupc9CH4;@EpO2_E zuXgY99Y6SS4t2Iiul`#0q)ev~UgRI1 z`bqN$Y>0*(TYJig)i-f4o&G39y(U8<+oPGQ;05!IX7&S)k&yYhaHa_pXIXBady@w= z7}j`2y1PTajJ|C-c$=flI8O^3y-2N=H^e(VPJB1}len6oM(_uiQeQb9Y<{76Qh(8{ zb{+K3Uv3{bzu1{>fuD{4d6CnrN5EQL#qhPue)!SgQ(g_QZ4rq^+GoBXj0Krn(NVy_ z&W^?7?tu6}QPFB^=2rdPh9A@H5n_Q*hIBz(Y)s@4cN{obBNn1n>o(|_BH=*K7JOUrq4$a?O zbLg@GTGOcF9D$BPRl;3l^k$E&JH{pVP=^5%^L{w%e*|-fVwPUuCnvvUx${FumChqJt=cPz*m*3Hbbyr$Y7YV)G_+ar->(}-OA?|c#di-KF0pYa@ zzl|>aaDy1p)G_pZq{eTjh0x6o`&Va$g!;R@V$U%G0#EO5SLQMudeOXIix*W3wq+=4lg+zpBUW1=VQYl zP*Ed8-XrY$rp(q@g*iO5L`W`y|ArND3k=98W{EkzP5D~@e`o8dx;lS>4ZVUfNM`?f z3#;t~RnB-N2D!RxjOTZ1^EnDcVO_~$P++L7Gy57Ax?~uzaq1YmI6$qj(YJ2{wGxy+ zthCa1Ik}!a{u1uPiBeG@(7S$W-E zH}`~T=t2XsgKGc~gM+co5iicOS^w{5589nXMA=$TpY|LONBF;sr{cNhNIFD1-7RJ# z%qt)LutIWB#;B4*z(Ua0dhs5{d;XR1j;0cHz5Ag~Dox?Q$WwnPixB7fPjEyXrXqhuBh@tbt7;p>rlq;Yw_2B>ZXPa_) znTGV%_p8nyY-C(NEzw#KBx}*_@ynFkG5*z!xw5Bb<*@Tq1uffl$ln&|r!rHS)Vntf z?>?`7bB-_axz_0`7wy`9>w{#T?cVGhWcM%W`ntL((X6kJ`#iyZojc)N-_DyXkw5zW zE9ccuT@u|OJWF2pb|*09CDez#do!W`y`gD-^tYrDHjHS(utRNih-0DYGJ`mz0n)z*pp zuiw1be>d&!*HzonARej-$=mt<)$!6YkKbDhdRMV*(bjZ+&I5F`*9j%1%w=gxT$=N5 z+}s%TCV1PnS@H#-Ft}fl?mkQSW{c=l{YaPBp|hs&sDi>~%7c7hNzk06yy(>Ndz`ts zA^+B?)@=c%*#IxGkMG`i{9yaH*xFhu`uf^a*WDJIII;lKo@?Z-n~Dp)J?(%wZ`X4B z-3pC9brZ6FuRqIg3UmUOc2CXI*liB!$8U?~H|fnP?_I*Lba>joKR&>aSg_}#dlP`vo73xxZ~OPR^#sN#?oBm(vzfLcgpY*%xd*QNeYeHE&lnxlq>-OEHd+x;>jvcmPYjbCxSe=t| z+6h$V$h_JZp&PMJ;hg#RlCm?BUef1|S^-Zdy*%qUlc1gs&{Do|nT={sWUc1PYOYAU zd*X4BS=F~mX3j-FpUiy(jM#!V%O8cLyWi;Hohz&O?z7I+JKQSAHon{bb6rbF{H}M8 zl`b0r-Dja=Wo@SNwszaILf-dR`;U7sl;8iXjSE=O@K?(1D=lNuU%q6s^1r&3`@QAt zAGI6hUd}jrlo9A~b>3(z>#}cE1-wAJJ|wzXTOSK=zi^Wg=yHXb2hQ<%r?Uek8jiep z!vJ*a21_ez24MM=W83! Date: Thu, 2 May 2019 07:18:00 -0500 Subject: [PATCH 11/42] Update default image repo in flagger chart readme to be weaveworks --- charts/flagger/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/flagger/README.md b/charts/flagger/README.md index 1ecbb95d..5230f5df 100644 --- a/charts/flagger/README.md +++ b/charts/flagger/README.md @@ -45,7 +45,7 @@ The following tables lists the configurable parameters of the Flagger chart and Parameter | Description | Default --- | --- | --- -`image.repository` | image repository | `quay.io/stefanprodan/flagger` +`image.repository` | image repository | `quay.io/weaveworks/flagger` `image.tag` | image tag | `` `image.pullPolicy` | image pull policy | `IfNotPresent` `metricsServer` | Prometheus URL | `http://prometheus.istio-system:9090` From 9e082d9ee36fd54371739e4e549fb217b2dcfbbd Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Thu, 2 May 2019 11:05:43 -0500 Subject: [PATCH 12/42] Update charts/flagger/README.md Co-Authored-By: aackerman --- charts/flagger/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/flagger/README.md b/charts/flagger/README.md index 5230f5df..d47034a6 100644 --- a/charts/flagger/README.md +++ b/charts/flagger/README.md @@ -45,7 +45,7 @@ The following tables lists the configurable parameters of the Flagger chart and Parameter | Description | Default --- | --- | --- -`image.repository` | image repository | `quay.io/weaveworks/flagger` +`image.repository` | image repository | `weaveworks/flagger` `image.tag` | image tag | `` `image.pullPolicy` | image pull policy | `IfNotPresent` `metricsServer` | Prometheus URL | `http://prometheus.istio-system:9090` From c92230c109b9a3b3afb4d0c4451ff2a98f3d3cdd Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Thu, 2 May 2019 19:05:54 +0300 Subject: [PATCH 13/42] Fix duplicate hosts error when using wildcard --- pkg/router/istio.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/router/istio.go b/pkg/router/istio.go index 40370d10..5a1bd227 100644 --- a/pkg/router/istio.go +++ b/pkg/router/istio.go @@ -32,7 +32,7 @@ func (ir *IstioRouter) Reconcile(canary *flaggerv1.Canary) error { hosts := canary.Spec.Service.Hosts var hasServiceHost bool for _, h := range hosts { - if h == targetName { + if h == targetName || h == "*" { hasServiceHost = true break } From e17a7477853fe1cd1381bf660cdb671cba212707 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Fri, 3 May 2019 19:32:29 +0300 Subject: [PATCH 14/42] Change dashboard selector to destination workload --- charts/grafana/dashboards/istio.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/charts/grafana/dashboards/istio.json b/charts/grafana/dashboards/istio.json index 539ef535..84bc0a64 100644 --- a/charts/grafana/dashboards/istio.json +++ b/charts/grafana/dashboards/istio.json @@ -1614,9 +1614,9 @@ "multi": false, "name": "primary", "options": [], - "query": "query_result(sum(istio_requests_total{destination_workload_namespace=~\"$namespace\"}) by (destination_service_name))", + "query": "query_result(sum(istio_requests_total{destination_workload_namespace=~\"$namespace\"}) by (destination_workload))", "refresh": 1, - "regex": "/.*destination_service_name=\"([^\"]*).*/", + "regex": "/.*destination_workload=\"([^\"]*).*/", "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", @@ -1636,9 +1636,9 @@ "multi": false, "name": "canary", "options": [], - "query": "query_result(sum(istio_requests_total{destination_workload_namespace=~\"$namespace\"}) by (destination_service_name))", + "query": "query_result(sum(istio_requests_total{destination_workload_namespace=~\"$namespace\"}) by (destination_workload))", "refresh": 1, - "regex": "/.*destination_service_name=\"([^\"]*).*/", + "regex": "/.*destination_workload=\"([^\"]*).*/", "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", From e4c6903a01f2382bcc6e784ee94ccd4e6f84d1ea Mon Sep 17 00:00:00 2001 From: Scott Cranton Date: Sun, 5 May 2019 15:42:06 -0400 Subject: [PATCH 15/42] Fix and clarify SuperGloo installation docs Added missing `=` for --version, and added brew and helm install options --- .../install/flagger-install-with-supergloo.md | 69 ++++++++++++++++--- 1 file changed, 59 insertions(+), 10 deletions(-) diff --git a/docs/gitbook/install/flagger-install-with-supergloo.md b/docs/gitbook/install/flagger-install-with-supergloo.md index c0c7e1d7..7ec10ed1 100644 --- a/docs/gitbook/install/flagger-install-with-supergloo.md +++ b/docs/gitbook/install/flagger-install-with-supergloo.md @@ -2,37 +2,61 @@ This guide walks you through setting up Flagger on a Kubernetes cluster using [SuperGloo](https://github.com/solo-io/supergloo). -SuperGloo by [Solo.io](https://solo.io) is an opinionated abstraction layer that will simplify the installation, management, and operation of your service mesh. -It supports running multiple ingress with multiple mesh (Istio, App Mesh, Consul Connect and Linkerd 2) in the same cluster. +SuperGloo by [Solo.io](https://solo.io) is an opinionated abstraction layer that simplifies the installation, management, and operation of your service mesh. +It supports running multiple ingresses with multiple meshes (Istio, App Mesh, Consul Connect and Linkerd 2) in the same cluster. ### Prerequisites Flagger requires a Kubernetes cluster **v1.11** or newer with the following admission controllers enabled: * MutatingAdmissionWebhook -* ValidatingAdmissionWebhook +* ValidatingAdmissionWebhook ### Install Istio with SuperGloo -Download SuperGloo CLI and add it to your path: +#### Install SuperGloo command line interface helper + +SuperGloo includes a command line helper (CLI) that makes operation of SuperGloo easier. +The CLI is not required for SuperGloo to function correctly. + +If you use [Homebrew](https://brew.sh) package manager run the following +commands to install the SuperGloo CLI. + +```bash +brew tap solo-io/tap +brew solo-io/tap/supergloo +``` + +Or you can download SuperGloo CLI and add it to your path: ```bash curl -sL https://run.solo.io/supergloo/install | sh export PATH=$HOME/.supergloo/bin:$PATH ``` +#### Install SuperGloo controller + Deploy the SuperGloo controller in the `supergloo-system` namespace: ```bash supergloo init ``` +This is equivalent to installing SuperGloo using its Helm chart + +```bash +helm repo add supergloo http://storage.googleapis.com/supergloo-helm +helm upgrade --install supergloo supergloo/supergloo --namespace supergloo-system +``` + +#### Install Istio using SuperGloo + Create the `istio-system` namespace and install Istio with traffic management, telemetry and Prometheus enabled: ```bash ISTIO_VER="1.0.6" -kubectl create ns istio-system +kubectl create namespace istio-system supergloo install istio --name istio \ --namespace=supergloo-system \ @@ -40,9 +64,34 @@ supergloo install istio --name istio \ --installation-namespace=istio-system \ --mtls=false \ --prometheus=true \ ---version ${ISTIO_VER} +--version=${ISTIO_VER} ``` +This creates a Kubernetes Custom Resource (CRD) like the following. + +```yaml +apiVersion: supergloo.solo.io/v1 +kind: Install +metadata: + name: istio + namespace: supergloo-system +spec: + installationNamespace: istio-system + mesh: + installedMesh: + name: istio + namespace: supergloo-system + istioMesh: + enableAutoInject: true + enableMtls: false + installGrafana: false + installJaeger: false + installPrometheus: true + istioVersion: 1.0.6 +``` + +#### Allow Flagger to manipulate SuperGloo + Create a cluster role binding so that Flagger can manipulate SuperGloo custom resources: ```bash @@ -54,8 +103,8 @@ kubectl create clusterrolebinding flagger-supergloo \ Wait for the Istio control plane to become available: ```bash -kubectl -n istio-system rollout status deployment/istio-sidecar-injector -kubectl -n istio-system rollout status deployment/prometheus +kubectl --namespace istio-system rollout status deployment/istio-sidecar-injector +kubectl --namespace istio-system rollout status deployment/prometheus ``` ### Install Flagger @@ -106,9 +155,9 @@ You can access Grafana using port forwarding: kubectl -n istio-system port-forward svc/flagger-grafana 3000:80 ``` -### Install Load Tester +### Install Load Tester -Flagger comes with an optional load testing service that generates traffic +Flagger comes with an optional load testing service that generates traffic during canary analysis when configured as a webhook. Deploy the load test runner with Helm: From 921ac003837245d35d0bce5c90c1f9927a0c50df Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Mon, 6 May 2019 18:33:00 +0300 Subject: [PATCH 16/42] Add ingress ref to CRD and RBAC --- artifacts/flagger/account.yaml | 6 ++++++ artifacts/flagger/crd.yaml | 12 ++++++++++++ charts/flagger/templates/crd.yaml | 12 ++++++++++++ charts/flagger/templates/rbac.yaml | 6 ++++++ pkg/apis/flagger/v1alpha3/types.go | 4 ++++ pkg/apis/flagger/v1alpha3/zz_generated.deepcopy.go | 5 +++++ 6 files changed, 45 insertions(+) diff --git a/artifacts/flagger/account.yaml b/artifacts/flagger/account.yaml index 9bbd8410..0fac89ad 100644 --- a/artifacts/flagger/account.yaml +++ b/artifacts/flagger/account.yaml @@ -31,6 +31,12 @@ rules: resources: - horizontalpodautoscalers verbs: ["*"] + - apiGroups: + - "extensions" + resources: + - ingresses + - ingresses/status + verbs: ["*"] - apiGroups: - flagger.app resources: diff --git a/artifacts/flagger/crd.yaml b/artifacts/flagger/crd.yaml index 134020e6..61eac408 100644 --- a/artifacts/flagger/crd.yaml +++ b/artifacts/flagger/crd.yaml @@ -69,6 +69,18 @@ spec: type: string name: type: string + ingressRef: + anyOf: + - type: string + - type: object + required: ['apiVersion', 'kind', 'name'] + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string service: type: object required: ['port'] diff --git a/charts/flagger/templates/crd.yaml b/charts/flagger/templates/crd.yaml index aad996b8..f189a31a 100644 --- a/charts/flagger/templates/crd.yaml +++ b/charts/flagger/templates/crd.yaml @@ -70,6 +70,18 @@ spec: type: string name: type: string + ingressRef: + anyOf: + - type: string + - type: object + required: ['apiVersion', 'kind', 'name'] + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string service: type: object required: ['port'] diff --git a/charts/flagger/templates/rbac.yaml b/charts/flagger/templates/rbac.yaml index 95a2f249..e03755c0 100644 --- a/charts/flagger/templates/rbac.yaml +++ b/charts/flagger/templates/rbac.yaml @@ -27,6 +27,12 @@ rules: resources: - horizontalpodautoscalers verbs: ["*"] + - apiGroups: + - "extensions" + resources: + - ingresses + - ingresses/status + verbs: ["*"] - apiGroups: - flagger.app resources: diff --git a/pkg/apis/flagger/v1alpha3/types.go b/pkg/apis/flagger/v1alpha3/types.go index 4b0b9d12..e9eb9a49 100755 --- a/pkg/apis/flagger/v1alpha3/types.go +++ b/pkg/apis/flagger/v1alpha3/types.go @@ -52,6 +52,10 @@ type CanarySpec struct { // +optional AutoscalerRef *hpav1.CrossVersionObjectReference `json:"autoscalerRef,omitempty"` + // reference to NGINX ingress resource + // +optional + IngressRef *hpav1.CrossVersionObjectReference `json:"ingressRef,omitempty"` + // virtual service spec Service CanaryService `json:"service"` diff --git a/pkg/apis/flagger/v1alpha3/zz_generated.deepcopy.go b/pkg/apis/flagger/v1alpha3/zz_generated.deepcopy.go index 4bb9551c..dc710448 100644 --- a/pkg/apis/flagger/v1alpha3/zz_generated.deepcopy.go +++ b/pkg/apis/flagger/v1alpha3/zz_generated.deepcopy.go @@ -205,6 +205,11 @@ func (in *CanarySpec) DeepCopyInto(out *CanarySpec) { *out = new(v1.CrossVersionObjectReference) **out = **in } + if in.IngressRef != nil { + in, out := &in.IngressRef, &out.IngressRef + *out = new(v1.CrossVersionObjectReference) + **out = **in + } in.Service.DeepCopyInto(&out.Service) in.CanaryAnalysis.DeepCopyInto(&out.CanaryAnalysis) if in.ProgressDeadlineSeconds != nil { From 5f544b90d6dd14d4a4c342571b434298829c011f Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Mon, 6 May 2019 18:41:04 +0300 Subject: [PATCH 17/42] Log mesh provider at startup --- cmd/flagger/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/flagger/main.go b/cmd/flagger/main.go index 9297577e..0c09b7dc 100644 --- a/cmd/flagger/main.go +++ b/cmd/flagger/main.go @@ -99,7 +99,7 @@ func main() { canaryInformer := flaggerInformerFactory.Flagger().V1alpha3().Canaries() - logger.Infof("Starting flagger version %s revision %s", version.VERSION, version.REVISION) + logger.Infof("Starting flagger version %s revision %s mesh provider %s", version.VERSION, version.REVISION, meshProvider) ver, err := kubeClient.Discovery().ServerVersion() if err != nil { From 177dc824e3f4229ad9796b224bb042928209313f Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Mon, 6 May 2019 18:42:02 +0300 Subject: [PATCH 18/42] Implement nginx ingress router --- pkg/router/factory.go | 7 +- pkg/router/ingress.go | 165 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 171 insertions(+), 1 deletion(-) create mode 100644 pkg/router/ingress.go diff --git a/pkg/router/factory.go b/pkg/router/factory.go index 9d8f66ec..ab836d31 100644 --- a/pkg/router/factory.go +++ b/pkg/router/factory.go @@ -41,9 +41,14 @@ func (factory *Factory) KubernetesRouter(label string) *KubernetesRouter { } } -// MeshRouter returns a service mesh router (Istio or AppMesh) +// MeshRouter returns a service mesh router func (factory *Factory) MeshRouter(provider string) Interface { switch { + case provider == "nginx": + return &IngressRouter{ + logger: factory.logger, + kubeClient: factory.kubeClient, + } case provider == "appmesh": return &AppMeshRouter{ logger: factory.logger, diff --git a/pkg/router/ingress.go b/pkg/router/ingress.go new file mode 100644 index 00000000..7e9513e0 --- /dev/null +++ b/pkg/router/ingress.go @@ -0,0 +1,165 @@ +package router + +import ( + "fmt" + "github.com/google/go-cmp/cmp" + flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3" + "go.uber.org/zap" + "k8s.io/api/extensions/v1beta1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes" + "strconv" + "strings" +) + +type IngressRouter struct { + kubeClient kubernetes.Interface + logger *zap.SugaredLogger +} + +func (i *IngressRouter) Reconcile(canary *flaggerv1.Canary) error { + if canary.Spec.IngressRef == nil || canary.Spec.IngressRef.Name == "" { + return fmt.Errorf("ingress selector is empty") + } + + targetName := canary.Spec.TargetRef.Name + canaryName := fmt.Sprintf("%s-canary", targetName) + canaryIngressName := fmt.Sprintf("%s-canary", canary.Spec.IngressRef.Name) + + ingress, err := i.kubeClient.ExtensionsV1beta1().Ingresses(canary.Namespace).Get(canary.Spec.IngressRef.Name, metav1.GetOptions{}) + if err != nil { + return err + } + + ingressClone := ingress.DeepCopy() + + // change backend to -canary + backendExists := false + for k, v := range ingressClone.Spec.Rules { + for x, y := range v.HTTP.Paths { + if y.Backend.ServiceName == targetName { + ingressClone.Spec.Rules[k].HTTP.Paths[x].Backend.ServiceName = canaryName + backendExists = true + break + } + } + } + + if !backendExists { + return fmt.Errorf("backend %s not found in ingress %s", targetName, canary.Spec.IngressRef.Name) + } + + canaryIngress, err := i.kubeClient.ExtensionsV1beta1().Ingresses(canary.Namespace).Get(canaryIngressName, metav1.GetOptions{}) + + if errors.IsNotFound(err) { + ing := &v1beta1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: canaryIngressName, + Namespace: canary.Namespace, + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(canary, schema.GroupVersionKind{ + Group: flaggerv1.SchemeGroupVersion.Group, + Version: flaggerv1.SchemeGroupVersion.Version, + Kind: flaggerv1.CanaryKind, + }), + }, + Annotations: i.makeAnnotations(ingressClone.Annotations), + Labels: ingressClone.Labels, + }, + Spec: ingressClone.Spec, + } + + _, err := i.kubeClient.ExtensionsV1beta1().Ingresses(canary.Namespace).Create(ing) + if err != nil { + return err + } + + i.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)). + Infof("Ingress %s.%s created", ing.GetName(), canary.Namespace) + return nil + } + + if err != nil { + return fmt.Errorf("ingress %s query error %v", canaryIngressName, err) + } + + if diff := cmp.Diff(ingressClone.Spec, canaryIngress.Spec); diff != "" { + iClone := canaryIngress.DeepCopy() + iClone.Spec = ingressClone.Spec + + _, err := i.kubeClient.ExtensionsV1beta1().Ingresses(canary.Namespace).Update(iClone) + if err != nil { + return fmt.Errorf("ingress %s update error %v", canaryIngressName, err) + } + + i.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)). + Infof("Ingress %s updated", canaryIngressName) + } + + return nil +} + +func (i *IngressRouter) GetRoutes(canary *flaggerv1.Canary) ( + primaryWeight int, + canaryWeight int, + err error, +) { + canaryIngressName := fmt.Sprintf("%s-canary", canary.Spec.IngressRef.Name) + canaryIngress, err := i.kubeClient.ExtensionsV1beta1().Ingresses(canary.Namespace).Get(canaryIngressName, metav1.GetOptions{}) + if err != nil { + return 0, 0, err + } + + for k, v := range canaryIngress.Annotations { + if k == "nginx.ingress.kubernetes.io/canary-weight" { + val, err := strconv.Atoi(v) + if err != nil { + return 0, 0, err + } + + canaryWeight = val + break + } + } + + primaryWeight = 100 - canaryWeight + return +} + +func (i *IngressRouter) SetRoutes( + canary *flaggerv1.Canary, + primaryWeight int, + canaryWeight int, +) error { + canaryIngressName := fmt.Sprintf("%s-canary", canary.Spec.IngressRef.Name) + canaryIngress, err := i.kubeClient.ExtensionsV1beta1().Ingresses(canary.Namespace).Get(canaryIngressName, metav1.GetOptions{}) + if err != nil { + return err + } + + iClone := canaryIngress.DeepCopy() + iClone.Annotations["nginx.ingress.kubernetes.io/canary-weight"] = fmt.Sprintf("%v", canaryWeight) + + _, err = i.kubeClient.ExtensionsV1beta1().Ingresses(canary.Namespace).Update(iClone) + if err != nil { + return fmt.Errorf("ingress %s update error %v", canaryIngressName, err) + } + + return nil +} + +func (i *IngressRouter) makeAnnotations(annotations map[string]string) map[string]string { + res := make(map[string]string) + for k, v := range annotations { + if !strings.Contains(v, "nginx.ingress.kubernetes.io/canary") { + res[k] = v + } + } + + res["nginx.ingress.kubernetes.io/canary"] = "true" + res["nginx.ingress.kubernetes.io/canary-weight"] = "0" + + return res +} From cf3ba35fb98835fd678e8de84fb48d7da794c861 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Mon, 6 May 2019 18:42:31 +0300 Subject: [PATCH 19/42] Add nginx ingress controller metrics --- Makefile | 6 ++ artifacts/nginx/canary.yaml | 54 ++++++++++++++ artifacts/nginx/deployment.yaml | 69 ++++++++++++++++++ artifacts/nginx/ingress.yaml | 17 +++++ pkg/metrics/nginx.go | 122 ++++++++++++++++++++++++++++++++ pkg/metrics/nginx_test.go | 51 +++++++++++++ 6 files changed, 319 insertions(+) create mode 100644 artifacts/nginx/canary.yaml create mode 100644 artifacts/nginx/deployment.yaml create mode 100644 artifacts/nginx/ingress.yaml create mode 100644 pkg/metrics/nginx.go create mode 100644 pkg/metrics/nginx_test.go diff --git a/Makefile b/Makefile index 3f656da4..262f98b9 100644 --- a/Makefile +++ b/Makefile @@ -18,6 +18,12 @@ run-appmesh: -slack-url=https://hooks.slack.com/services/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 \ -slack-channel="devops-alerts" +run-nginx: + go run cmd/flagger/* -kubeconfig=$$HOME/.kube/config -log-level=info -mesh-provider=nginx -namespace=nginx \ + -metrics-server=http://prometheus-weave.istio.weavedx.com \ + -slack-url=https://hooks.slack.com/services/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 \ + -slack-channel="devops-alerts" + build: docker build -t weaveworks/flagger:$(TAG) . -f Dockerfile diff --git a/artifacts/nginx/canary.yaml b/artifacts/nginx/canary.yaml new file mode 100644 index 00000000..13eb15c9 --- /dev/null +++ b/artifacts/nginx/canary.yaml @@ -0,0 +1,54 @@ +apiVersion: flagger.app/v1alpha3 +kind: Canary +metadata: + name: podinfo + namespace: test +spec: + # deployment reference + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: podinfo + # ingress reference + ingressRef: + apiVersion: extensions/v1beta1 + kind: Ingress + name: podinfo + # the maximum time in seconds for the canary deployment + # to make progress before it is rollback (default 600s) + progressDeadlineSeconds: 60 + service: + # container port + port: 9898 + canaryAnalysis: + # schedule interval (default 60s) + interval: 10s + # max number of failed metric checks before rollback + threshold: 10 + # max traffic percentage routed to canary + # percentage (0-100) + maxWeight: 50 + # canary increment step + # percentage (0-100) + stepWeight: 5 + # NGINX Prometheus checks + metrics: + - name: request-success-rate + # minimum req success rate (non 5xx responses) + # percentage (0-100) + threshold: 99 + interval: 1m + - name: request-duration + # maximum avg req duration + # milliseconds + threshold: 500 + interval: 1m + # external checks (optional) + webhooks: + - name: load-test + url: http://flagger-loadtester.test/ + timeout: 5s + metadata: + type: cmd + cmd: "hey -z 1m -q 10 -c 2 http://app.example.com/" + logCmdOutput: "true" diff --git a/artifacts/nginx/deployment.yaml b/artifacts/nginx/deployment.yaml new file mode 100644 index 00000000..814dd9c2 --- /dev/null +++ b/artifacts/nginx/deployment.yaml @@ -0,0 +1,69 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: podinfo + namespace: test + labels: + app: podinfo +spec: + replicas: 1 + strategy: + rollingUpdate: + maxUnavailable: 0 + type: RollingUpdate + selector: + matchLabels: + app: podinfo + template: + metadata: + annotations: + prometheus.io/scrape: "true" + labels: + app: podinfo + spec: + containers: + - name: podinfod + image: quay.io/stefanprodan/podinfo:1.4.0 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9898 + name: http + protocol: TCP + command: + - ./podinfo + - --port=9898 + - --level=info + - --random-delay=false + - --random-error=false + env: + - name: PODINFO_UI_COLOR + value: green + livenessProbe: + exec: + command: + - podcli + - check + - http + - localhost:9898/healthz + failureThreshold: 3 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 2 + readinessProbe: + exec: + command: + - podcli + - check + - http + - localhost:9898/readyz + failureThreshold: 3 + periodSeconds: 3 + successThreshold: 1 + timeoutSeconds: 2 + resources: + limits: + cpu: 1000m + memory: 256Mi + requests: + cpu: 100m + memory: 16Mi diff --git a/artifacts/nginx/ingress.yaml b/artifacts/nginx/ingress.yaml new file mode 100644 index 00000000..5cb6b826 --- /dev/null +++ b/artifacts/nginx/ingress.yaml @@ -0,0 +1,17 @@ +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: podinfo + namespace: test + labels: + app: podinfo + annotations: + kubernetes.io/ingress.class: "nginx" +spec: + rules: + - host: app.exmaple.com + http: + paths: + - backend: + serviceName: podinfo + servicePort: 9898 diff --git a/pkg/metrics/nginx.go b/pkg/metrics/nginx.go new file mode 100644 index 00000000..7c6eb56e --- /dev/null +++ b/pkg/metrics/nginx.go @@ -0,0 +1,122 @@ +package metrics + +import ( + "fmt" + "net/url" + "strconv" + "time" +) + +const nginxSuccessRateQuery = ` +sum(rate( +nginx_ingress_controller_requests{kubernetes_namespace="{{ .Namespace }}", +ingress="{{ .Name }}", +status!~"5.*"} +[{{ .Interval }}])) +/ +sum(rate( +nginx_ingress_controller_requests{kubernetes_namespace="{{ .Namespace }}", +ingress="{{ .Name }}"} +[{{ .Interval }}])) +* 100 +` + +// GetNginxSuccessRate returns the requests success rate (non 5xx) using nginx_ingress_controller_requests metric +func (c *Observer) GetNginxSuccessRate(name string, namespace string, metric string, interval string) (float64, error) { + if c.metricsServer == "fake" { + return 100, nil + } + + meta := struct { + Name string + Namespace string + Interval string + }{ + name, + namespace, + interval, + } + + query, err := render(meta, nginxSuccessRateQuery) + if err != nil { + return 0, err + } + + var rate *float64 + querySt := url.QueryEscape(query) + result, err := c.queryMetric(querySt) + if err != nil { + return 0, err + } + + for _, v := range result.Data.Result { + metricValue := v.Value[1] + switch metricValue.(type) { + case string: + f, err := strconv.ParseFloat(metricValue.(string), 64) + if err != nil { + return 0, err + } + rate = &f + } + } + if rate == nil { + return 0, fmt.Errorf("no values found for metric %s", metric) + } + return *rate, nil +} + +const nginxRequestDurationQuery = ` +sum(rate( +nginx_ingress_controller_ingress_upstream_latency_seconds_sum{kubernetes_namespace="{{ .Namespace }}", +ingress="{{ .Name }}"}[{{ .Interval }}])) +/ +sum(rate(nginx_ingress_controller_ingress_upstream_latency_seconds_count{kubernetes_namespace="{{ .Namespace }}", +ingress="{{ .Name }}"}[{{ .Interval }}])) * 1000 +` + +// GetNginxRequestDuration returns the avg requests latency using nginx_ingress_controller_ingress_upstream_latency_seconds_sum metric +func (c *Observer) GetNginxRequestDuration(name string, namespace string, metric string, interval string) (time.Duration, error) { + if c.metricsServer == "fake" { + return 1, nil + } + + meta := struct { + Name string + Namespace string + Interval string + }{ + name, + namespace, + interval, + } + + query, err := render(meta, nginxRequestDurationQuery) + if err != nil { + return 0, err + } + + var rate *float64 + querySt := url.QueryEscape(query) + result, err := c.queryMetric(querySt) + if err != nil { + return 0, err + } + + for _, v := range result.Data.Result { + metricValue := v.Value[1] + switch metricValue.(type) { + case string: + f, err := strconv.ParseFloat(metricValue.(string), 64) + if err != nil { + return 0, err + } + rate = &f + } + } + if rate == nil { + return 0, fmt.Errorf("no values found for metric %s", metric) + } + ms := time.Duration(int64(*rate)) * time.Millisecond + return ms, nil +} diff --git a/pkg/metrics/nginx_test.go b/pkg/metrics/nginx_test.go new file mode 100644 index 00000000..e2a93c5f --- /dev/null +++ b/pkg/metrics/nginx_test.go @@ -0,0 +1,51 @@ +package metrics + +import ( + "testing" +) + +func Test_NginxSuccessRateQueryRender(t *testing.T) { + meta := struct { + Name string + Namespace string + Interval string + }{ + "podinfo", + "nginx", + "1m", + } + + query, err := render(meta, nginxSuccessRateQuery) + if err != nil { + t.Fatal(err) + } + + expected := `sum(rate(nginx_ingress_controller_requests{kubernetes_namespace="nginx",ingress="podinfo",status!~"5.*"}[1m])) / sum(rate(nginx_ingress_controller_requests{kubernetes_namespace="nginx",ingress="podinfo"}[1m])) * 100` + + if query != expected { + t.Errorf("\nGot %s \nWanted %s", query, expected) + } +} + +func Test_NginxRequestDurationQueryRender(t *testing.T) { + meta := struct { + Name string + Namespace string + Interval string + }{ + "podinfo", + "nginx", + "1m", + } + + query, err := render(meta, nginxRequestDurationQuery) + if err != nil { + t.Fatal(err) + } + + expected := `sum(rate(nginx_ingress_controller_ingress_upstream_latency_seconds_sum{kubernetes_namespace="nginx",ingress="podinfo"}[1m])) /sum(rate(nginx_ingress_controller_ingress_upstream_latency_seconds_count{kubernetes_namespace="nginx",ingress="podinfo"}[1m])) * 1000` + + if query != expected { + t.Errorf("\nGot %s \nWanted %s", query, expected) + } +} From f7db0210ea0da32b3f3bf35fdf2b2580cf1d06bc Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Mon, 6 May 2019 18:43:02 +0300 Subject: [PATCH 20/42] Add nginx ingress controller checks --- pkg/controller/scheduler.go | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/pkg/controller/scheduler.go b/pkg/controller/scheduler.go index 5c16cb4e..0fc06978 100644 --- a/pkg/controller/scheduler.go +++ b/pkg/controller/scheduler.go @@ -632,6 +632,41 @@ func (c *Controller) analyseCanary(r *flaggerv1.Canary) bool { } } + // NGINX checks + if c.meshProvider == "nginx" { + if metric.Name == "request-success-rate" { + val, err := c.observer.GetNginxSuccessRate(r.Spec.IngressRef.Name, r.Namespace, metric.Name, metric.Interval) + if err != nil { + if strings.Contains(err.Error(), "no values found") { + c.recordEventWarningf(r, "Halt advancement no values found for metric %s probably %s.%s is not receiving traffic", + metric.Name, r.Spec.TargetRef.Name, r.Namespace) + } else { + c.recordEventErrorf(r, "Metrics server %s query failed: %v", c.observer.GetMetricsServer(), err) + } + return false + } + if float64(metric.Threshold) > val { + c.recordEventWarningf(r, "Halt %s.%s advancement success rate %.2f%% < %v%%", + r.Name, r.Namespace, val, metric.Threshold) + return false + } + } + + if metric.Name == "request-duration" { + val, err := c.observer.GetNginxRequestDuration(r.Spec.IngressRef.Name, r.Namespace, metric.Name, metric.Interval) + if err != nil { + c.recordEventErrorf(r, "Metrics server %s query failed: %v", c.observer.GetMetricsServer(), err) + return false + } + t := time.Duration(metric.Threshold) * time.Millisecond + if val > t { + c.recordEventWarningf(r, "Halt %s.%s advancement request duration %v > %v", + r.Name, r.Namespace, val, t) + return false + } + } + } + // custom checks if metric.Query != "" { val, err := c.observer.GetScalar(metric.Query) From 00151e92fe360538dcfa8cbf3ace7be245b0c80d Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Tue, 7 May 2019 10:33:40 +0300 Subject: [PATCH 21/42] Implement A/B testing for nginx ingress --- pkg/router/ingress.go | 65 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 64 insertions(+), 1 deletion(-) diff --git a/pkg/router/ingress.go b/pkg/router/ingress.go index 7e9513e0..8e3f51d5 100644 --- a/pkg/router/ingress.go +++ b/pkg/router/ingress.go @@ -112,6 +112,16 @@ func (i *IngressRouter) GetRoutes(canary *flaggerv1.Canary) ( return 0, 0, err } + // A/B testing + if len(canary.Spec.CanaryAnalysis.Match) > 0 { + for k := range canaryIngress.Annotations { + if k == "nginx.ingress.kubernetes.io/canary-by-cookie" || k == "nginx.ingress.kubernetes.io/canary-by-header" { + return 0, 100, nil + } + } + } + + // Canary for k, v := range canaryIngress.Annotations { if k == "nginx.ingress.kubernetes.io/canary-weight" { val, err := strconv.Atoi(v) @@ -140,7 +150,33 @@ func (i *IngressRouter) SetRoutes( } iClone := canaryIngress.DeepCopy() - iClone.Annotations["nginx.ingress.kubernetes.io/canary-weight"] = fmt.Sprintf("%v", canaryWeight) + + // A/B testing + if len(canary.Spec.CanaryAnalysis.Match) > 0 { + cookie := "" + header := "" + headerValue := "" + for _, m := range canary.Spec.CanaryAnalysis.Match { + for k, v := range m.Headers { + if k == "cookie" { + cookie = v.Exact + } else { + header = k + headerValue = v.Exact + } + } + } + + if canaryWeight > 0 { + iClone.Annotations = i.makeHeaderAnnotations(iClone.Annotations, header, headerValue, cookie) + } else { + iClone.Annotations = i.makeAnnotations(iClone.Annotations) + } + + } else { + // canary + iClone.Annotations["nginx.ingress.kubernetes.io/canary-weight"] = fmt.Sprintf("%v", canaryWeight) + } _, err = i.kubeClient.ExtensionsV1beta1().Ingresses(canary.Namespace).Update(iClone) if err != nil { @@ -163,3 +199,30 @@ func (i *IngressRouter) makeAnnotations(annotations map[string]string) map[strin return res } + +func (i *IngressRouter) makeHeaderAnnotations(annotations map[string]string, + header string, headerValue string, cookie string) map[string]string { + res := make(map[string]string) + for k, v := range annotations { + if !strings.Contains(v, "nginx.ingress.kubernetes.io/canary") { + res[k] = v + } + } + + res["nginx.ingress.kubernetes.io/canary"] = "true" + res["nginx.ingress.kubernetes.io/canary-weight"] = "0" + + if cookie != "" { + res["nginx.ingress.kubernetes.io/canary-by-cookie"] = cookie + } + + if header != "" { + res["nginx.ingress.kubernetes.io/canary-by-header"] = header + } + + if headerValue != "" { + res["nginx.ingress.kubernetes.io/canary-by-header-value"] = headerValue + } + + return res +} From 0d94c01678b58a02ad1403d0dcf5da494554fe46 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Tue, 7 May 2019 11:10:19 +0300 Subject: [PATCH 22/42] Toggle canary annotation based on weight --- pkg/router/ingress.go | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/pkg/router/ingress.go b/pkg/router/ingress.go index 8e3f51d5..ffc663b1 100644 --- a/pkg/router/ingress.go +++ b/pkg/router/ingress.go @@ -167,17 +167,19 @@ func (i *IngressRouter) SetRoutes( } } - if canaryWeight > 0 { - iClone.Annotations = i.makeHeaderAnnotations(iClone.Annotations, header, headerValue, cookie) - } else { - iClone.Annotations = i.makeAnnotations(iClone.Annotations) - } - + iClone.Annotations = i.makeHeaderAnnotations(iClone.Annotations, header, headerValue, cookie) } else { // canary iClone.Annotations["nginx.ingress.kubernetes.io/canary-weight"] = fmt.Sprintf("%v", canaryWeight) } + // toggle canary + if canaryWeight > 0 { + iClone.Annotations["nginx.ingress.kubernetes.io/canary"] = "true" + } else { + iClone.Annotations = i.makeAnnotations(iClone.Annotations) + } + _, err = i.kubeClient.ExtensionsV1beta1().Ingresses(canary.Namespace).Update(iClone) if err != nil { return fmt.Errorf("ingress %s update error %v", canaryIngressName, err) @@ -189,12 +191,13 @@ func (i *IngressRouter) SetRoutes( func (i *IngressRouter) makeAnnotations(annotations map[string]string) map[string]string { res := make(map[string]string) for k, v := range annotations { - if !strings.Contains(v, "nginx.ingress.kubernetes.io/canary") { + if !strings.Contains(k, "nginx.ingress.kubernetes.io/canary") && + !strings.Contains(k, "kubectl.kubernetes.io/last-applied-configuration") { res[k] = v } } - res["nginx.ingress.kubernetes.io/canary"] = "true" + res["nginx.ingress.kubernetes.io/canary"] = "false" res["nginx.ingress.kubernetes.io/canary-weight"] = "0" return res From a233b99f0b53d75bc73f12f36769903500e67ced Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Tue, 7 May 2019 11:12:36 +0300 Subject: [PATCH 23/42] Add HPA to nginx demo --- artifacts/nginx/canary.yaml | 5 +++++ artifacts/nginx/hpa.yaml | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 artifacts/nginx/hpa.yaml diff --git a/artifacts/nginx/canary.yaml b/artifacts/nginx/canary.yaml index 13eb15c9..bca0a709 100644 --- a/artifacts/nginx/canary.yaml +++ b/artifacts/nginx/canary.yaml @@ -14,6 +14,11 @@ spec: apiVersion: extensions/v1beta1 kind: Ingress name: podinfo + # HPA reference (optional) + autoscalerRef: + apiVersion: autoscaling/v2beta1 + kind: HorizontalPodAutoscaler + name: podinfo # the maximum time in seconds for the canary deployment # to make progress before it is rollback (default 600s) progressDeadlineSeconds: 60 diff --git a/artifacts/nginx/hpa.yaml b/artifacts/nginx/hpa.yaml new file mode 100644 index 00000000..fa2b5a6f --- /dev/null +++ b/artifacts/nginx/hpa.yaml @@ -0,0 +1,19 @@ +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: podinfo + namespace: test +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: podinfo + minReplicas: 2 + maxReplicas: 4 + metrics: + - type: Resource + resource: + name: cpu + # scale up if usage is above + # 99% of the requested CPU (100m) + targetAverageUtilization: 99 From 79b337089294a92961bc8446fd185b38c50a32df Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 8 May 2019 15:44:28 +0300 Subject: [PATCH 24/42] Add Prometheus add-on to Flagger chart --- charts/flagger/templates/deployment.yaml | 4 + charts/flagger/templates/prometheus.yaml | 292 +++++++++++++++++++++++ charts/flagger/values.yaml | 6 +- 3 files changed, 301 insertions(+), 1 deletion(-) create mode 100644 charts/flagger/templates/prometheus.yaml diff --git a/charts/flagger/templates/deployment.yaml b/charts/flagger/templates/deployment.yaml index c23ce93b..0df390b7 100644 --- a/charts/flagger/templates/deployment.yaml +++ b/charts/flagger/templates/deployment.yaml @@ -38,7 +38,11 @@ spec: {{- if .Values.meshProvider }} - -mesh-provider={{ .Values.meshProvider }} {{- end }} + {{- if .Values.prometheus.install }} + - -metrics-server=http://{{ template "flagger.fullname" . }}-prometheus:9090 + {{- else }} - -metrics-server={{ .Values.metricsServer }} + {{- end }} {{- if .Values.namespace }} - -namespace={{ .Values.namespace }} {{- end }} diff --git a/charts/flagger/templates/prometheus.yaml b/charts/flagger/templates/prometheus.yaml new file mode 100644 index 00000000..f1fe583b --- /dev/null +++ b/charts/flagger/templates/prometheus.yaml @@ -0,0 +1,292 @@ +{{- if .Values.prometheus.install }} +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: {{ template "flagger.fullname" . }}-prometheus + labels: + helm.sh/chart: {{ template "flagger.chart" . }} + app.kubernetes.io/name: {{ template "flagger.name" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +rules: + - apiGroups: [""] + resources: + - nodes + - services + - endpoints + - pods + - nodes/proxy + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: + - configmaps + verbs: ["get"] + - nonResourceURLs: ["/metrics"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: {{ template "flagger.fullname" . }}-prometheus + labels: + helm.sh/chart: {{ template "flagger.chart" . }} + app.kubernetes.io/name: {{ template "flagger.name" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "flagger.fullname" . }}-prometheus +subjects: + - kind: ServiceAccount + name: {{ template "flagger.serviceAccountName" . }}-prometheus + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "flagger.serviceAccountName" . }}-prometheus + namespace: {{ .Release.Namespace }} + labels: + helm.sh/chart: {{ template "flagger.chart" . }} + app.kubernetes.io/name: {{ template "flagger.name" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "flagger.fullname" . }}-prometheus + namespace: {{ .Release.Namespace }} + labels: + helm.sh/chart: {{ template "flagger.chart" . }} + app.kubernetes.io/name: {{ template "flagger.name" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +data: + prometheus.yml: |- + global: + scrape_interval: 5s + scrape_configs: + + # Scrape config for AppMesh Envoy sidecar + - job_name: 'appmesh-envoy' + metrics_path: /stats/prometheus + kubernetes_sd_configs: + - role: pod + + relabel_configs: + - source_labels: [__meta_kubernetes_pod_container_name] + action: keep + regex: '^envoy$' + - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] + action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: ${1}:9901 + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: kubernetes_pod_name + + # Exclude high cardinality metrics + metric_relabel_configs: + - source_labels: [ cluster_name ] + regex: '(outbound|inbound|prometheus_stats).*' + action: drop + - source_labels: [ tcp_prefix ] + regex: '(outbound|inbound|prometheus_stats).*' + action: drop + - source_labels: [ listener_address ] + regex: '(.+)' + action: drop + - source_labels: [ http_conn_manager_listener_prefix ] + regex: '(.+)' + action: drop + - source_labels: [ http_conn_manager_prefix ] + regex: '(.+)' + action: drop + - source_labels: [ __name__ ] + regex: 'envoy_tls.*' + action: drop + - source_labels: [ __name__ ] + regex: 'envoy_tcp_downstream.*' + action: drop + - source_labels: [ __name__ ] + regex: 'envoy_http_(stats|admin).*' + action: drop + - source_labels: [ __name__ ] + regex: 'envoy_cluster_(lb|retry|bind|internal|max|original).*' + action: drop + + # Scrape config for API servers + - job_name: 'kubernetes-apiservers' + kubernetes_sd_configs: + - role: endpoints + namespaces: + names: + - default + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + relabel_configs: + - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] + action: keep + regex: kubernetes;https + + # Scrape config for nodes + - job_name: 'kubernetes-nodes' + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - target_label: __address__ + replacement: kubernetes.default.svc:443 + - source_labels: [__meta_kubernetes_node_name] + regex: (.+) + target_label: __metrics_path__ + replacement: /api/v1/nodes/${1}/proxy/metrics + + # scrape config for cAdvisor + - job_name: 'kubernetes-cadvisor' + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - target_label: __address__ + replacement: kubernetes.default.svc:443 + - source_labels: [__meta_kubernetes_node_name] + regex: (.+) + target_label: __metrics_path__ + replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor + + # scrape config for pods + - job_name: kubernetes-pods + kubernetes_sd_configs: + - role: pod + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scrape + - source_labels: [ __address__ ] + regex: '.*9901.*' + action: drop + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + source_labels: + - __address__ + - __meta_kubernetes_pod_annotation_prometheus_io_port + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: kubernetes_namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: kubernetes_pod_name +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "flagger.fullname" . }}-prometheus + namespace: {{ .Release.Namespace }} + labels: + helm.sh/chart: {{ template "flagger.chart" . }} + app.kubernetes.io/name: {{ template "flagger.name" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: {{ template "flagger.name" . }}-prometheus + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ template "flagger.name" . }}-prometheus + app.kubernetes.io/instance: {{ .Release.Name }} + annotations: + appmesh.k8s.aws/sidecarInjectorWebhook: disabled + sidecar.istio.io/inject: "false" + spec: + serviceAccountName: {{ template "flagger.serviceAccountName" . }}-prometheus + containers: + - name: prometheus + image: "docker.io/prom/prometheus:v2.7.1" + imagePullPolicy: IfNotPresent + args: + - '--storage.tsdb.retention=6h' + - '--config.file=/etc/prometheus/prometheus.yml' + ports: + - containerPort: 9090 + name: http + livenessProbe: + httpGet: + path: /-/healthy + port: 9090 + readinessProbe: + httpGet: + path: /-/ready + port: 9090 + resources: + requests: + cpu: 10m + memory: 128Mi + volumeMounts: + - name: config-volume + mountPath: /etc/prometheus + - name: data-volume + mountPath: /prometheus/data + + volumes: + - name: config-volume + configMap: + name: {{ template "flagger.fullname" . }}-prometheus + - name: data-volume + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "flagger.fullname" . }}-prometheus + namespace: {{ .Release.Namespace }} + labels: + helm.sh/chart: {{ template "flagger.chart" . }} + app.kubernetes.io/name: {{ template "flagger.name" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +spec: + selector: + app.kubernetes.io/name: {{ template "flagger.name" . }}-prometheus + app.kubernetes.io/instance: {{ .Release.Name }} + ports: + - name: http + protocol: TCP + port: 9090 +{{- end }} diff --git a/charts/flagger/values.yaml b/charts/flagger/values.yaml index 0f1195ab..0d1a07af 100644 --- a/charts/flagger/values.yaml +++ b/charts/flagger/values.yaml @@ -7,7 +7,7 @@ image: metricsServer: "http://prometheus:9090" -# accepted values are istio or appmesh (defaults to istio) +# accepted values are istio, appmesh, nginx or supergloo:mesh.namespace (defaults to istio) meshProvider: "" # single namespace restriction @@ -49,3 +49,7 @@ nodeSelector: {} tolerations: [] affinity: {} + +prometheus: + # to be used with AppMesh or nginx ingress + install: false From 8914f26754c27da2aea9f1d59e0897cc5a896b49 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 8 May 2019 17:03:36 +0300 Subject: [PATCH 25/42] Add ngnix docs --- docs/diagrams/flagger-nginx-overview.png | Bin 0 -> 40939 bytes .../usage/nginx-progressive-delivery.md | 355 ++++++++++++++++++ 2 files changed, 355 insertions(+) create mode 100644 docs/diagrams/flagger-nginx-overview.png create mode 100644 docs/gitbook/usage/nginx-progressive-delivery.md diff --git a/docs/diagrams/flagger-nginx-overview.png b/docs/diagrams/flagger-nginx-overview.png new file mode 100644 index 0000000000000000000000000000000000000000..f8dcaadcd91b0fbecba555a280aa50babb75d693 GIT binary patch literal 40939 zcmc$_WmuG7^fo#m3PTPz@cLDNK$-h}?`473?V%sP*NQl2?`T!@>~Brh zy?MX2^f}Ji*~vC)eW+Yx!_&?nvW+5m)D8q<1Su=X>H17<%n~_NIa1$x6jVpF%cU8{ z7<_&ybCO>)nc?)jv~SF{_50-5RC~jY`1Z6a!5kKdo^gi_1VTbUvUpe^2pj|gaISyE zj9mSAHS%xr{~Y;$7XN=$@V|@yf7<&e{(tk}e^>DTvwKKL0QUch)T79SsvLMgD=b?)B8Ri@^<6^$nH6S|b#}aUSq}D~7519|PLcxUW`jEE zqHinR=0)>l*{Oo{JMy@;4OKZjto-O;Yt$F7ULJXp@l|jNYhTxnqu1xfBjT|Ixvp0wh@Z)5Q*zy0*2jnA-Hk?n zHs98|UQYkjdo4)*@^PL;HhGfH)oNXd9gC%j%ZJKIoHv(suf#|9MBKfL?+-F}s=P9c ze+|J_^Md{O@t%z&4%?mcO1)Cbv#yc*C2eXPcXO@*8Oa*hfAU6L2fwfIDk>(C)El8p zR6b_sb(o=|tD!39ZXS~Kl)-y@$uo3tj%(a?^=5qz^P^21S-nFvuf5Bs2(Fux{eaV~ zt8K0Fisst(VqGBU`8;?`VwOz_+sutsfg=fw_3;4V7TdgEY412b)>4hTPZqUdKy~^DD9W^YW+W>TRj^CPB}n7QyQv zF$p6|lfrX(Fh4oZW9bAwtFyUY1mJuO6U4;|*7=jVJ#gtfKL6a|kpIqmkK~n1uok`z z90~bub3bN3Dne_=Y5v5y)_C#hi-*|v$qq6kxvFLuUt+*uLV^Z=q{+Q3b0`g%dOp8J zoH2i#r?p)_uI{hjvZ*)TOyg*%7nY2gPi*7-XwI=XDrdB8aCY!Xr#Kn%y*7;>ufmyT zckuZ2lf)JoDN7&o(Z*bq&#srI`S_FKy1Jn2g-4Njs1`bK^_zfaZIGCilh}kSdMzF> z%1--a_lNn(oqfcWSyMlr42~-Kz{b;FCh|pt0-j;Ry11LS!-4$N6u(ad^gH)vRR1Ft z$L7@Wfi)!c_}9+^y_9dywGorBC@U(dg-?#h@P&0a!k*P?F7K<*5vN_8o{uYjasVDb z8TO}y*g(*O^3Z{g5c>#i1{Yj)QqI=#KhY3q!IUTdtB(Itg6pELI(cj5eN3viK%+07a^!fLwccSY_L7@SWr~+C z(~Q(zXWnJghybhOpe3KXwGX{+lzl09F4!ilVlvw#Byx&OY&kAfm^+##c7N*}AE6=Jk(qs&@si^ZRpzQMOt@ z;@+E&=M$x=O^1(~PDBqQiywqEoM$%#CVW9 zs`p7~kvzpco&NS%;aVTlzX{g~1hK7v8hY>CVw=;iO~?6oLeX$av9prC{c#oF!icx+ zN7ZbWhS0om$(k82z>(F8IB!P`>5=--yh*8B`kwA7 zU(ip;5T%kSrS^v~>h3geov{M!SUO?uWwB;zX{X+;mcqglECZtch{ z&Fk5=55`3{^2{Y9$6&kLLjvmo%^Sk&u@+d*Lq4Mr9455zKV%%2%}U)B9MJJukyD5^ zF)ej};9AZ;|Lli{wD2hb^wf~bpOvsiNcFt2-3LSpx{3sFs-&3JY0$uHrMVw7oWAnd zpFZF$ADHeJ>FRUcHgtNu8~*zzJpdD(xqHv#;Bn!DlwU_YDMzG762O}r8}}!Urg+LE ziBm`?iEd)=9CYIG$3%2&AF?lt`~nQDHV+5v-ew7@X7i<@*lTaTe=ofJ%jJgYbYzJr z1jlZIz3LT$8h|0cUk11ZmOI^Dw1VG?)s;$1?+*GFPN_FrwxvJ6lF@>u>l_u<=47(O zUkNKa2oW$=IU#Y!9F0I1H%THuF@{QfzO`h6+i_LT3=P7Z#WNc_1Ku^x3!w=I(5_?= zX;lx)*mYSJJI~5 zS)LQw=_7&XTo+2!xQEPOutOGy5 zVNB#>3W)FYT0};L8g``c8-ZgjJ;<3Cwk89f*6ls`&fE|3YaBPjxmcC%vf+S0-%07I zv0#}#bM32nMQu-cKu#j(Q$>>XG^fJA2y?7?z~-s38lWHgi!x|N7fmEoH*+V1|D3;< zB0MturYBmE{_PK*d=wY_NBqwu0iURk?)0hU@y$2H0PWGVu7|4&7pRJqJP%u$z!Qg} zVn37vC0{7S2LttQ0m-EscQ4vMY#&Z~4n-ws=vK~cA~v^#gq}}qkNQV(Ap20aJsw+$ zPkSbm6yvgqITnj9E4eKwzJFXP=;Tf6My!D#&#Hg`wV%eb{5obUQ^+&h!XZY&m38xU zC4@@P^pD5_+2F+|=8xwVvJ4dPBmoe}p9&K!cP3+!N=?)!=x(q*Md95+mj z81jp>aJoyOZC1337K&Xxt`Q#KzViL&87NV>1uvX)lZ$AE`PrTJPd}4h(RWb`hBUZY z7rb%OgRUKqhqShyUC4I?a!*CXe8ctOPRqyajRfJDGi<>IfDEG1<*j%!K-(7IScPnGSCE&3@c*Q>u;CLIWjjQ)|11xuws_*AE|N5n6h-n_%hZW!S z2I~Ut)REUKbJ!Bn&i)+!q)cI}31KslK8Xf{qJz$Y``txSz_?&=TQ(7E0qOTy=B8F& zvgGGQuzTT`w>~dnOZa`#_x)D!jV?zXwjwF$U=3$d$bgQ~_}wGVt__pLWVE6^cJVV= zrLj$m{mZ&anAgDVuR~s;4&_R=y!-P{y-8#6${9pP)wDkv9I83BK$&zFJcK0#Ev!h1 zssleZ-P_Il4pvt1ZPrW*L(vxOOP~35jPBw$MJpj;@dObyxwG25w?A+KLJBoUcwwlS zgY9N0eqLz8I)Ev6ADUE5S;#any3T;RZ9bFL6;??)ytu9NPUG2Vi|^)TYqI#gDpBI% zG^yM%tN6sRo6ui7X(FOKsrjPrS@3GZp80ojkpkE|++3H!AND7}NHecPm& zxgm4xxkU1L)u%~%@->>#I_=#~sz^~9LEX<=)HYGCGRS~^P4;STvsgR@Orzy{Jd6i{ zMJi?djLJP0SXCQs+B7~VTc^8!Geq@MRz~Z-ClL|5i`#!m4sec|+j3`z=ynmvoG)*Fv4Kk$F*+TEYl zWOw%sSa|OBFKJedT(Lo_J5q-@+@F;ejPC>r7Bfl_GZ04?QaI`d`|Gny^1#FExY^E} z*xA8QcL0Y82lR5h(i#Wl`!q>dY4{b5Kcx2Y%RHM!&!FF;{-QX8g@h*+nm+o-iWJ^d zF;VgeoNh~cT5M8mP;2jm05;OyK$+)5*&J?+PhD9PN$mKA#D@CY_F(zD)lp_WC=!C8 z0tJLG?Nd^6xCi?~Fx2xuixxPj!^0(xW~+=ZAB%Gt{5I8LY?p4-qi#PDH%Os@Y#s6N z+-u7!Nb;)DyhtW;d^WjkUgUM0U9ncNzEdu1Q~k$ zw7?UK>?T0M?;O3;{k807S5sEQ(O}X?f0U{-E+Ren5ckJ|*(<{Q$C7MslVB~2 z&5*T);!M`y0^}D$dA<4W1DhcOzI3{TR*=YUvl+jm4M=GwqI&e= z5x(yu9MOC*(=H4T4iszN6|fS?55cTpymi+ykleT($S1+|2W|Fvmh%{k>EUrp`}Gnx z>d<0EzI+G8$$}WOv(r>fPk4o5TH?Y>iC7C`rI)>q)`r6oIi@N$hKf4m&xLp%5Ie~T zzcmqoI}f3JbIXo`^!_k&soSFQra>x$?BvxdKP}pg_Nd;L-E( z4;NHrrL;eALc7V@s!L6t+(N>2voyC>uES!yRpIBAr*5^k3RKzSBqft}bwJ{kpWKiA zDm-^65yRu4uHrE?6B%+>ga5p|jjZY&ub`lC29f>r2Vz6f2 zj(3+fh9g9SC3Z#A2r?3oX)&f>n?P8vBjlJH9J506D${ z-IyGvAG(O<%iFwYLW}*V_qQI!yQ>(7gtE`J^vdnZ$@+fMTR z5g_^B#L)tXr#*~oudH5}F%Y{R60Ge5iTR)9SgTU9{}5zxop=7ZdF1C&^1SFKm7axu zR+ZaSxRLHBA?NkPo^ba=4*i|*GIeW37N1}EWG0_K%>>NaFJ_ok!kB$F3j)brVlu|LT> z$iBhLj%0J#V|zoyO9Pz@|FKvJ(!ewTh7dcu^c^zpW~eiyH!);YjolJ;P+P4RS9afavskb{ZKwBBBUA?oQpRE zYxv1o?0Vxm+MV)K=VdyP7nXMQ*3Ser&bVgGhxA^z<@e4ng2lAh)1)Y`4oTmC_AvUd zr}^$j7yeqZO@CJQ;;3Zb{2WFkqFbo3p)M&dMInBk{v^j{&M~y_MR-B5a?*v*kKzrJ zmKAfkmQ)9`Q(g^p*#CgAn{{?BzRfhv8EAr`;B;gMLtSZUYE-#)Zp}BMN}1h<;$QFW z&Eyru`Ap1cJ%L&Rza29u>fv@hdwu*f2EOnCGJLkEn^y<hs#$cK+&PW zy!&l;Yz?&C|M>UqN6LjZQjJTVL25XBfnlI<`uis*CobK=94~BlwkfBUi+FEHlv6x; zIG@`W8Nr3_JG*yPB~z+XB2(JH5awle42X=3G6!FFP<;Wd^>4d=DJYX!Z;X1yRzWz z3(v+2ns|(bDWeUX#w}Q~&)sJYYciV}r+)hdz4+0mkk%y19xxDy3NQ^TH@(uRl~}N&a<8^gE=wid#Z0T zpam1E5QZo*YEA318^s43Yu+dy-Gv8|M+NJj>B7nnkP49h0#kfo&l372LLqcyEw3SI z^j5MZj?XVd4T0u)9p9{ewfVeBze`v_iR$S|;(oFD`0H(k->1;1CrW<$Etp#!X4n0j zZxUVWWO7y~z0l@xRB1(3uJ1Dy--J=A`wvp;JiK6Y{k}df=ELy~A_jtuqfA(28rlYW zv$pS&MVRdC4!B!31WWo%7hK1MWn`+PFtv6e4;)=*Z#uU2b_XX|FX3nrp?Z2Fov@La zWDl`xsnWOtOMW37Sy$NzJ=R*bb1*&eZrPPGSy-*B%p@4W==Hdqb~t|eJkiGkYJ)(S zWT0k4;EREU%cvrqG>5=Zge!#F!9{~y&Lz`~xopB2>+ z?KDNSsdui=*IUZAb#nQ)v3QGY;~u=Ypl8hx9z8pMzo_u3FBq+6mMoIs%dWG;x!$9- z)P2`32>>d6;a>Fr)M04iS&xnff5r|hS7q?5y9kl>K&+e1JF9zV{Z6>!-mnl1?NEWH{pe=T{(Vnm(I$| z;57vkj@`x2jz!VF@FTkatd^brC{$dzWEaRv83-~mB-J(%2kesli;3J)p5p9CK-pL` zTyl!^RV5q1LOl1RHPB`M0WeLTmH3%)wu$=ri5w3f)8fKSyOwh2fa3#>O>Z?}Y#+L? z*xzrYcau5|DXKMtJgqWNQvW4I_;Vzv=|m(`BSCgF|4SI(W|&!QjJlmR}w&C+KvKVE2#u9Ydx!_Gxl2%9mBDi5mmJj z7so_aekl{}6S)Qk$ajz~E;kh3e*NRNklXH8xAiK5It*go`1o^`c;;DA;JAHmg8VZ$ z-j$aSzMcoSl-1_XAUD48KMne=fVQ1ttbPmH*CF|`Ehlwt5)}G@AjYgS`kub>uS%MG z9QSp!Mw{>qYcQ+}N|>=t%kYDRtyKJt1Ef$aH`5oD+pLE0tdfs30-mwMpFXSUEvN}7 zm7mLxE4cK$jMa_Yxb_kx8~5>iuU|6t>% zX)qk32W+k~_GodznrI(V<`Y$+@jwju;bUajyGU%zA)1%mdXkW#FR6?|pdrs#=692` zx?PK+ZQ7a7b*Q0`m*vlHKfd*>eJ;Y)Qh~&XPE?J-ne20_S3=;yWD=>>=;c^4|dGi_FcIcF0bx5xH3k5LawNm2Z?NwbT=Sv zYAKDRJX4>%ue;~@Pte(pP$}Ce)?Or)yP7kXTP;2=LkSToj3{`$!~9g$`UskcYkRx7d zp{usB-Nmtt-A-}`$=041GAi0Gf1Ja3hnf^vvRuAj$+pIS+V-8m<{I%$eQ?>0492Ro%Nr-wS*x?QRd z4a62z7P5PEqd0fYPyvd6|JsQe^7Mvja$pmrH%KtWThcpjSg}>A74zS>07iLMuZY6u zg%3Bj7}>~)Z(pk=rSW1yl;hO*9pw{Db3^75b=!`0O<_~)E;y(?=!(#+!id-Eb^%9WdS0AU z8;_}y^FG~trsFim&(^d}mZm9ca^=n=C|2U=gOiPT`|7A4Jl~DYR@t$mHAS_r=XwbI zN`&!1cc?KrQ7e=l=9y*$y~M*@)%-brB#vT%m~b#mIxej!B?du&D_E`btUdoqwBpIr zV!m&{^f75J-jW+21x$+qJsUI8#|4FAnh+?h(h1Toa|3kcg7uR?5Qu3pC!n+#F}y!- zs2|TCE=bwhtJJ#&0+G6LlV(3Kn`|j?biEC$-B#Y5le#>2ox>u;^l;QKj5Xgxbswwr zyUS4pB0LYXUxod?{W*8CA7}UMUjXYn@byJc3Z8=jVZg%5(LN_R-NcI`NL|#e; zo<9d?9uYQ1Oc=Ow0d?R2dh&|y7t3@L4P=f28 zJcMq?)QbBG613%>&^?A&M$`K^0rdVvYIeAoF^QlVTb9uM!ae^x%{Q(WT8RmOvfmSl zCHxN}=Q_Q;Pc;JzlP(djzg+6Nu)u(W~D+XDK zi9aYAy@w~T_+{M8<_U^N7s1-!XZZLpu*OWVL}SKZk7U#xiJ(@@YD0bBx}F-+y}pfM zscQ5yG9pV{lN>m`<|F0VKFT588aWL2suJ*`Q(xixo+9kJSC?de6+&zf0?i#4_otc?s~)rOs9=3tt7c_j+k39lx4h0=(|Is?1G2O01#t%2(vnhoyD)7O6Ts< z+%91Oq7Cf6TErlqpwab3H=;f=3$lNR$ozef$*duVh|#f z;q}deI19jC?9MV*=*ilbCqj$Ia=^F}8?qhKWx%`01Z@CFwf16D!0jFfIgZoKgMll* zJE#~;B1oTvw*QpYkXw+u$sq)jmGAU`t6#AS)s8bs)Q;cAY2G0}KHQ_Ni)(E9a`F(~+M_iQCun+2vccAIqa+8K}F81+r%WY8sk zym4j>-!ci{%Z?u(5NzQS(Db6!(cdGyMBuFTA`$?}HLE&KrOL>W3aj<7CII+JJq|vx zR%%@LGA#z~5&q@Wg`kqaLzbIj)44K_VaUYH$Krta(G@%$PY~~udpz+l=y{Le%b_=C z7PrhS;{gkF63wRh+(W;Z#>fTHsWoxU^9^bcCJO{yj(tuN=##Wa>ePl)BJe;u3Jo)- zSB7U2js8lp$m8NXtqAw=@DZal+1TBcJG!@aSZg3b5;N|w({OHo9^v})XV5?-76#EV zca*P%vd31N7(tlT$76!-O0w*8&d_apL>gCco{sqGUEUy=q!DsdC*Vlyf~@dt7`K;~ zc(8n^z8>Ajfa@0T*Y1@m{(V4DWngA}tbmb$s}cvG^?Y8nDa=9{ z;ph;|0t|LK+~NJtgLx>l+{MduO%d+(6c`od6q@1giT~9=Kwp4f_^-NqGQSUKHH++9N@M|(Sq>6OJGE-&!8()R=5^|}jVuvQtF zX3IIb&S5i2<^Bk`dz<6L|9VL;zmatP0Sk~lW2zI2`ZXl^I}#IXYXBF()>x zpck|W+$rOOIj5sC3Lnp^BWP?V+I!Yo-iaTv0g5l(G!u|AS+$7IYKZ_ zf(@qgW@>($yetzb1j`K-oU_)UXr_Sa`nK-+sD|2Osuu+FpS}O}H+?TR`YY!39%e6k)|2hk{pHVnFu%>hR#NQg7PYJD>A0DKSE`?gLMWZ7 z=z?#;hOzt=$SbxR2K!9v2>3PE_ejCuJflbIpEK+53~BdLp^%Qa%}t!LGdR+ zoz_&}FJ&O3U_GRTU{rR+nNDHJn9B8JOb*pjd-p^p_R@h=dR`oL(z4M{qmhnk<`IAa z*WjLTL9<^F!JpYP+f8`~1_q&LzaGT48nER)FP(LNL#X_>`RK=^x{W<>m*bms;e_ec z5}ppuU1XjUVc<+s1zg`J--rbl_TDRzw>*v>_zD};4oBq0GKE%>_S2ZO_o}_hV{CiO z25X<7bi@A-!va_?+o~pStb)>=mT45kyyZAfX0$b6G~nl|-x`^Lo*EIpHH4mJv3I15 zNgfKn+I&zlEWo@e<=)S#5N|i~h0CaVi`OZyeNxHx{`i~FQbA*o*YsNCQ$~b5(WQOV zaq>x)yowF%dlT3WBFjuimHn@v<)M4C8$mNc?aRqDTA?vN23|u-8h_N14NWoSKw}`+ z6;sLusPy&1)yK@H8>J%EW~opRx~+9kqtWvbXSQ6=@Cf055E|;hBSr*S6s#ItHHHu7 zjFd@rgcSJi9x8HM^AzvQfbG(m9-Hg;vH+hCpXptJj)L$(1w44mRGJduM`TFq+20W> z>WE6k{FPL`L?p87r`9m0P-NlZiPO8efscnSzFGQ$_A)+S2_;O?+f zLOk$ay0fz6O2(B5)_%3yWT%e)Gn?DE!lnE>5d~xVCxF=+Vr)uZ|Ek$mXiy?q6t+tx zBpwR6R7W`cPmaf8AyUF%JI7~54v09f8BqZO8pnM5%AW*wuhbFiqc)rgiNd91?J5lV&9On}kKw#Qjill$fa6I|>dla)fX+ zr>UO*#N{O|-@pxQoRkpxs>M4_b??D7LCCZS+s@B*=?Y@o@9m${)`>goUOHNPe%+jB zCQeHAn;}76wR&^`0v(XEr*^5z0$A#pPBWMv8^RW!QY70KB$>iDo+c7xZZg$7KPB=# zY0@f~<-HJXnNC*_B}vx4YgK>3JKTSCcw@uu-0@}F)pjba2v;gqJ{I(#Nj6yGR}UuI zMpQ2%J9uh*0$?ze(8+tcl-ZH}$6Xtoz>>Ze+|g_DLRV4z!1@uJ-U>+Us-yZxAR`Q8 z)d>lMK5<6vy7d;k^mT*xK$F{{Q0=MUXf2+-A!-%e?!HRG5Fx0xYXXESM1wU>BvE>K zVfpi^z>Dn0Tj;BPxTphD@QzdPolA30>UAa7-QWAXZE;RbfCz6CE%nQpOzDX7*hNh$ZS03<8| zA;$Ua6`$= z-|!3i%DM<zfm`^ zMyOp&Z|`gI&U0$U>q_@j4Ndq^fIN$cMYY_Bh2szp^?yu5l`m8^{-%QNdAbM9(+DW@ z0$WOO5AG?ahUYb~#oN5yRu9q@7m5kTOoZPn(f`HPY8E=w;(Gcq4)s|X2uGl+$DhX` z{ZIbbxtHI8Ee8o{Y<~{mVFXx@aO}{0&=~W_%pw!2w;T+sjVVr~|AA7>qHv&&Ps0Vw zw{M-k4fGJ?WrH0lc&gwnnM)wjV#21uhidwB?7*5gn1ehDna=lLAooe3{L{JyCq+x#Q=@7EYbbZz=bctL8%6CcuDbxc(NFCwkNOcT{nm6)f2xW z{;{mo0DiPzt9ssqy9TK_NS%xRGvYU4R>mL9)PL6L4z@EG{euf8_2uF}{M^)My7<%L zFTM3Sfz#x%x2wY^kMS&~P2qnQk$72c+_$xJi^Y6-fuo`3CVMq6HlR!Yox+N58ItF+ zMH139R(-9!v-@kV98gYDVUsfas*>A%$KpifpO6c-ZrYWkkDm6 z&{@Z@aBcLZMi}ypn;X!7dUh_eJ{f9a7^^QIp)u}%P>+mDapl%|vQG@wnmkz_3Yw`* zcw72(-^}pM_CKIVqERh5EQ=Ch$fWm!od?@kYyQy!G%)Mow^UH3p1ck90)3*Vs_^2S z(2*$x#pwQ&dfy#ACVR(%KmTY~k_e$1DTVHtL&2`8BuJ8O)30GdxnJ#0Uo^!)yc<~IuCY}|42__QJrR~*ty*NE!+>3S zOP1;zQA0hhp0`Lm<2t5ZkLMcJw||N@Y+<=wolOiIk?eLh@hFBnu$z+=UsmJeW7`?M z5;)$D?~&Jb)h@v5_SJwVeiDz=c8*T|_Ki|w_)&h>AAb8!$TLN__FW&-hxgt)AbyOd z9=6w>B0tRs{B;csUafx7CjzY!YS&2CX$VG&7|W);F0!>AsVx0$3h+X%ng{Igh6=OI zx?X|;9OvZa4R?4-_3}rtv4}ELhqYAsdRhL@W12H1%AZk;dKtw8Q(i5nhfT zmcF{pGI2i!EgPAqidCu;4f_6>^re%H_hf^w`-+QaRz03mQfUw5+XsZo2;-4sZ@rxmh{95F*&``hhP^2FoC+M15dM2Jdc_n3$K zmrXT7H-09zTygSl4a^hXB11JwZnXY@!vPUZHk2D|vSO7KMl;8Td~|*FL_ykvYyws7 z2t4bbepc)Iord~jeH|@2#3(c|_|r)5dImYy8@K?OV1;<6_Z@%o^{Z?n;Aah-R@|}t zRDIoSV~`%V5VxKN$3RO{%>Ebied+PB7r*%u!D0OkyyQ-7N~Mg3+4SoLn;iJCb<sKh9 zqf;Ee<6>wlj7&wEq@E$8|CU zPKy2VZ3;D{yZ8d*)Cw*wfj{3P7YmwIUe@Z! zm%6BfpO)rZMewx+R>M%AZF{fa3&wLZs*Qu?E!N^S(mylxrZa4e0oegj2m)Xm%r`<9n0LI)Cd2Ct%U8Ax{{r z`Ii7h{D53uM2vP0qj4A(LYWdw49(F>vvd5Kl)-5vHz@>PEq=%( z`rBjRjvwhaUgD5)b>mFZmz6!>Aqn!qpZi1q7ZByIdj98$aBABo-hJ7)F4+(FzZO4d zXQ%sc5pnG>BE`JZp1CC3gyEC-meIQsPLU!gHlbgezF6p~{3}Y&zooyyJ$#lI&8eE( zqfnYsmD%z&0T0hnie~ZFUl+ow=R#a4pk!m`r`XUc#_KiMmVf@YO9!80j@*Wb@k)=!vBQ&&_CkrYN@0L_d_V|TvG`L^9z)>PZ;h+ z-M@RCGmIJcEVZAE=^ses%?OU5e||8Gh$ZgOSJhqc%|; zcqeoIOvcZ)J%^J-;fHX2sj&aOthfB6a+jA;pU9`QO$?fY*2J9q%GjTo*9nja+TMc6 zGvZ+*Wg(1K6#RNozVYj_)x0=zFCm!0Qb6x5>A*Sc&ytej{|s{TW}6gk)``SDs8*YN zMJ%)qhv4A7Vw6P>g-dMK56qy-NO1ZaAUIfXMm$JWp?3V?F9Afd_2B`E*F6^zEc;LT zb}p+522J(AS&Re>oeL$0`T5hhSu$4=KA^l$A@Hf zhfhas`~0Q$u;Gldq0W{gPJ{7JGu2M4V*F-HPn(^ewQ+*F5h>6oih7yO+OP1)B&35G zXmmg^3|V@5&pRlP6|a4L{~DeruQwRf^H%XY05`mC3?`VHD5ceri-Dka=%K*tp4qbD z1v8NBT*2DUTi19#4|?iXwl?Y+oDff(@o|3N`$VT3$Rhpz{Vg>US;CeZE1kq51yXjq zH)O`0Qs6T369#n1LbCW~eR?UPpRM+ixi1^}Nrr~0CytR|??3gL_i{eG8DCLBo{31wM+jnVouz-(;9I^D{B)V-9rx z#e4H;ETrt$I(fvZSF2Temh~Ec@k3b+-kfy$fzeb?Fu`X=V@(9QD*bKlW5TDX_fnAxTSciu=R?p3%G#pD#+v2)P~-_ZzhX} zgSUI)e%BpN?(+~g^s77EDxF$cPsjNxXc=npk^6C_r2Capk%IKgky_ zwpqhe=8}yP*hRCFEg|=$YR{Ii3>yqZvY^xz@0o6IGn2S=wI1~YxjVnm823SdG*{M4 zVeHf+yC;d09K`mS7cw(g`T+oBR)SY{Aa;9v4sW~e+f)}ftqS?)+mP6?_uOkZb0J6 zkkp^Qf+&*s;t>VD!O*rZDW0nMc#}qQ^%$2}tVr@9nE0{a!UIt_X}D6tSRWo%Z&@Zl zI=`+#rg!_MJ%O*6q)aVQ*xxp6 zX6rK`WM1g{nC^~1`Q~qAmKP%8s#sf?KTgExT?iVAasTFDF7IhLE2L#Gz8R*mpX!+? zhgw%&^->aFG?oH73HB<;sqbR`=nvCRe@Ek1KJGRk0i~AJ1X%Ix0VsFm;={Gt9BwQi}d$GrNlM#(ThNrpTkgEK~|N zxVfEdsF55+^okAVH3+yUT>PUGV{#)t)f%cJ$5XqIvB-L;l zOJ4xnRc~Kj4E0+ryaE_D!Ieb(@(I}g1;^MXMp6a)o<@9@J>K7-Q}riD3h-iDUNSg3 z2ab@fsKFy7s!|n8k>)RUf!cy>RxYPJsS~9VLu8haMVI=f2no-P`Zn6V?hF)%>MM&E zG?G_5-F*qi4r1J;E^qCyndLBp9ED(guLD4e$r^`9HFJDMqHBd!ew9LWv?=*Q>ybOi z30K`j3)g}$sm%?wU+$8?eggKR8`nA5V36-z(dZOeCq6=|$)RkFI%pZyWA#?y$wokD zdO*P%?09d1JgEsweh8+j?E}x(LA&4>KH^}c#lB9aw7R2J^%g`U^hND{$;RpX@SK1y z7LlV;^6uEZg6OnorjGz?91S!hAG$Zsh3Q6O8!~wIQ%%e(G}}h{cH^BJ*Gl}y3vFhx zGEl5m$Z}h`k%E@;jBjQJo^M6zZ^^vxv0H_l?31;U3A~hhmeA(3qfM;=ocv#8eR(+4 z-}ktzGq%RQXI_joOSX{xWgC&5s3;6V_7F*8Xb=Wjvqbi6NlHjrhipl9iELT3XaBzN z{ye`we$UhMnCEuyJ?lO9+;i_aF9Yo=*7_X|0M(=2@E=2O+ok#JD*;_ZyShChxMcYU z0#~@qiwM}t7@hQmA;thQ;O7I7(5Gu7hCMiF-Vej;tdgJ6pvs%(L(xSJ-ycPaQTQ&e z_F)zcafZ`$X7hCUIo)B{??_a*K(6~`LXPf}Yw#sw#vhNtzHf|`k6~vD#hv}7>tlT; z{)##E-4;%I|Ngh;PY^2@6?&$0`l-0UH7tH6x$*gDv84=l^>u@gvvsyE-;M`Ph9Ss+ zx$nkt+Ib7feqp&kCY9grR4!OgGhqKsL6@gOczJ}EVzF@gf#Cn3yT^oKZr@uh&9EfWybv1p@6BBg+XPyp&E%DYY0cq!@1a-n2GdHGj61-*~d-@CM&1Aju`+0boWeJcIA`#Z*yR!00^@FO(d9{imF{ zT^B=DfNgP_KZ#$mGl^ex5!)d7;8ON(xYooH+|o{{6N{M41#^!$;QNOD{f}jLsX#qh zv+tD`t*#io{{5f?9+EaUKk%6Y;$yX<%gwDC4y z`qQ}&ey!;)%@QB{bg{S?_U?la%E*)d;H^_WEq9SlyoQWTA$0UI^|qw|r>b{(TwM9D ze8%2hgaa2v!fFul7iIBw2q!>Q4fgKw<1{~Hv%lnZMk6`#*B|l=0o3d)8P&#%Y(#A~ zc@jBDyl_Oa#1nG5UT(iWau`=OK$o9a^{cCTv$SPz#v$;zkL%;Lq$y8DbYTv0v?do9 z`|sjth6RZglyM(3n0CYtgYNksv8y`N0aImz8Sb=|a)s~ID=t#@+P*#{m&-8qU z1H-Z^dSjYL>{@XM$Ne|ug`PRXH|tdmtK}1Oj%3`nSzOt}DSS=n`oj~EXRG&i$)^)+ zZ~@)JN{{N5xD=7rG_w-8^swNf%+G(5a2pp9GD!wd5711bSU_pJA##e8EQX)NV=w2_n*%M!t)GHo% zd_g=w5%9s(_~=_NH+Kr>9_(hi@*my*(^@u4w4U0BG^y626$1-8Z1)}G`57A32cwI> z3))Ztb?s3p*I`V*h5(dW?eTkmkZ|Q>z1*dEDm{SbfBj{Px2%Wvy#-R_wsZwu?N*9$ zL;O~fB1Y%_n%0#=`AXpYZx%W!{e4`Og2>>5wVXGJJo zN>!$GjMZ9Hg#k_jS4i@U($LOO*=KA) z1g?Knw(VlkjAGw?lThAkflz49nOD2gPP0$$>cCsQ(pUE=b?Tfy#zj~tJ-IRcWSdqs zNb@PZ`L2z@xGf=O$I#W7&DZ6e-0l8xUGL1mk5)a+aHXww_npes*D0U~t~nsZkAz9( zNt_nd&u$5-t?$Mpyfj!;!#!@FXzkaxS{SFo1hA!V83{ZyDaT70F0sXmjA1U4oP1+? zf&eO5^7vT$LO_F|2j_C!E0;%dzwg-pl)b!EQ2YAf+ttFjkCa+3`$}yd>Dj8;U6<1( zjVt817$TRdf=Ou!K^zX!5*x9rtHr89bqh`$x*iPfp+z!k@0?dMpT_R_TF>W~$tCr# zWa2ROPRXM^HaHsJ9_LnxFblJmf312B8CawGZjAxn_kk#iHyqy8>_ehnT!|A>r3ZQj z=#rwpfB&SYJo)5H4AS}W`3JTwnQ`wE=(q=|M!RE-5+2B2i+!?g-Xj}^v3FvToe{ztXJXfo%^ldn z^foou1{kc}9Yi0dLK#galWH(JAJ7~*%^r?_O9MzU@c6lR9?BxIdylVSz(~h?ZKr+= zUY2R2z?@%#C%Jqy6Ty6iP}%W(L>c)i!BUpW-uqU9h_0q${VmHsacV-;$m`}`I_cLW z0XCHdW9NB`;h0H&x?|<6Ie3H&(}hrc)UadrPs7Iwtq;Jmq$e7yUAUX`<0+!B8q)bG zoz(y6tFEM7ltjkK8UGY9s}_;^Axhkn{NA&PhA4_O2xWraZX3EL%HnLxW>Q)$nZJI; zugv%ntClKmoVxs~g9Wcq%*>HrCW78Y7_W_-zSI4hQ5UxRTP8YAg- zZ_c66hGK(_yz(N#HX}Xt&+kAt{lBO7;A5#NpZYAfBva)w-0}&cvh&Nv#KQuf9ZwRQ zOe$b$NjG~q*o@z9)9A*s4f_+JR1%<$HFc83QXu!+l5s>+P5-o3$(~~tROdvNDd;>9 z=7w-&cv>>1cFg(!pIm}P+e)$)f1==UulBes35?Mwd^lD+1HAXeTb*-YrtO-)Fy=c$ z{Dn}3-B<8iA8#ab`Hy@nSt41SnA@pf=(55LymAoQyZGj7sbkm_oufiH&W;xDiNwxm z(zD9|IRc6|*lcy^1cI4ti6eI{09Y z)kMzfK?1ib;pOOR<;|?8TYBg-R}gt~slu%t$oOKp`vGq|ay%T=af~U(gP!$N&0|X)8q_O6B7pc!V6& zN*F@h*E43fHox7{s-I@5laJ%Pg*N!pb>-e_XUbR$Gd6viaZ-uaheSbzq4CT7h~g4l zXJ_y1AU4lp`xHvf-`h9$SW13<+vuy6CH+G?n)#*Qh(Cf?9mNJ44!;9f!#TwjH7`x4 zGD;Ch-vbwCR25BDA(#DD3 z6i#Vjj!WtpDL3bvZx`%#k6+`KGZ|K6W(ak*J0JID>XzvUS^PsbMrSp2_EN!=NzF$i zp;yB|>}O>8I(B*_T?B3Oonif15kCCAcMu$ybNS6_#MbYpA(O~SK5SI@_wTRVpa1Lx zgy}~;ysD*u>&F}Hc4f&wzDuk(%mW7KMAWWZ)@cxh_z>(&7dD&t3%z*L{wm6q;IkdS zDZKbIIwLSKnqP}7N>_s*Fs*4c?K8vVRj~1{_FH!G#)wex<%y@Wd=WV;{vg`cEJpq( zq&n%8ewX{kk_LuGGkAg?nu^#4kL!V04P#H0HDs}{<#AwX*3#DV-vsUyDd$w|}0D;1+)Wgu} zBgJHrQB!|-o4Z}ecAJOp@jKO>{r_bCvn@OnsmSE@JJ^j`|4VGO05bSW!pzO|Mw#N5 z*9YN%E&W~U8XI`TWhP%@pg~w{8^dORyJ``CsO%3N^O2+tgF+oP&KJ8Hp!y_&l8Rs5Z9ffi^zw=rCoWAB>cyp3HV0pnK{bQ6_ah4B`2hcR!OJVu^~O|Vd&#s=o@HZe5*ywDWqXlhfC zpd8%p=BV}2nwuL4K*G`#T!C4vW8*QHrimwB(5Z^h6u;Rv*1oM6~BICd(Xt7B%^K!_T)o zA7z!Uy?=Z+nXu5mFr{@J_Frq;p*NW#njl1~`d=x+} zk>C&8nHLJmeV0Pq^B~qNRqDe$5tvu{um~k4*x#mAu2+|^r$LhS)*Cz63O^F%V7D19 zr{vNlVN&)i%|A9zuK<3LBd|&{0aUaSlPdR#AbnT-QvJF0FkOGAYrYPivFgF=Kb2AOz89pSmgBypJ)QzEQLKpUFc()%o=v(9{D z3;V#KIYQ83Ql-)}tv{sN$8G55YkRf?+%0K81|;Sr&CZHc=eiPbd`Fl_CC~uZ{y%=2 zzs^LIqxr>^HB;vBH_ui2c4ZCuj~)inh%)zJyr}GLRb~@2@hum6cXoOf@>)|UJ@Unr zfY-Mf&!~Q7W`KKt;M!J}U|(<$RM?o6(lP#cR1t5vIQeF^Ra}+uDWFXbLhm57i0A<3 z4`%LhLV;^2z#hBjnTj9KX=B(Kv>P#~jB;Kr3uyYjNw0TEsrs`x$~Kn4J6fW(5ufff2~*NAIi! zP(BO`L2Qy(s{<_@FMV_W`{9FA3t4}J$aJ27J3oE}#_{lAS z;~y;yb$I4wDLO3?F98=Jk?JE&2{?WCXYSu9uz2>fPZytP|9T@3S0d9;(l1zyKs9t2 zg&|{{gHjVWTJrDw{?RoY9J`6dnWCGE5WnwrzWfv*EIu6|h7L@*P3!!e{eDh>7|y8a zVEw+0H(+zpo#+AhcN0ql^uD^W%Ec>)S|Ur?OndKR-kuO+{=2oHLhu?^`@iJ(%R&#KXCb$3`&X1c47m!ez8!xh*495B{fK8hn_IC zNv-ZV;##}1V27YbP>>_m0EQq|D)uqJm1zD5i;=F9{G!|*o!CC^=rptM#C_v1GLyF_ zc7`vONqTNF-<;*H)xbTvhlYqoif>w>I^8A$K3&&4B0>-nVs^)b(H_~Fph5u+V>Gj) zaPuAfmrN=apC_{{ap#Q?jpR(WVjk0X488J_)8^FaAMq-$G#}I1dP9{rPNL0EE|}N6 z`=F9$TFV_UtpD>28`C-%%m{B>nC(DD*j;)> zUeBFybNkFCCqA_1$i%yPFC;se4?I*d3hRshj&NdhBojEF_)ZYsE@U=zM-BI`zR}vD z>g6y!NI&)RYUg;$@n?pd;-$_Lyzo?=`A7AG8Y`o2^>3IXhvFB}| zzWLhyI#t`JPG1>v1hH5u#d(KfE_a)Ob@J8vksGiVM9n%8f=nR0I_*!Vk&>$-u=WW3 zWI3Xw#Aq_$9a~Ge{eXZZ+bCj{*-^egnjE+Z}cK}gh`$V8vb}#fn}x2QsMs#qsEf) z#3f;0bXJd#qghnmsPR`^W=;!#tJ*%YU5#4Z>#BtAhm5WYtJY4~uKP+X)Q%?@X z_pUX#;#-t$>k><6ZP^*qc8VY03Ns$v){I6*$LKO(8Ml~HFK3#m;T~MFm=ATYtkENU z^(bC8_Gm^%WQSgMDhqRYEAst5n6sIk@E#5%egRhrA*c5zFa2`2I2gSMG(Nw#{`7Vs z?+L?%3RH7%tdV2?Rx{A|ty`LE^1~w53w7?Qrn}mMLD`7cd7p6#u!xZc(`%&P*0#bJ za&Edz!okRZXOjln{1|Il^Op0%`BeuurSfv8i3cCT5RwXq@$e$Yq|`P=eCIutEcO@Y z6*77X%1BHy!i=M{qiyxj63^;uIZd9y+j&uqqyX6%;o*YM>r{NP`K757D#E?=B=}J} z7e3luNXJ9n*ID(hjd(D~m$V`o*7LE?Ot)q2WoM|tx3MV%6_H~yR1S9YBAT&HQaBDX)V~}3Z2Xhgw=V5B5!U{ zSUaUe)xVf~3Jk_Zga^7`hUGn+?1}@@w^|B)H(Z}G~Txq7yrWXl3quG?la~)f5Uu-Mv5*m%3cgWSU^iQGtBHtGa@q z?}IYQ&H+3!AX@q3A+6laZfyU?XqyEkEnBXCwXVc>_4it3e)51HOvjiPyA<^fA-CTg z2YX_mx~yJaZDT2!$EE{2?aJ_`BvaG=9}KATq+D&40+_;{RWRaAVD0C><+aF%Ic<4A zx=m%dw>bG_*>ZF7`ybAGsnlB)$Ma@p;43n@&}t&f&gT>#@e7TXjCV*|KO!us8D3Rrtq_V}X_0+xD@^g$-cu!KvPq;6%7 z4|I6hO&A~%%>Ci&U)!?Gnq@i0@m^$qwO1+BlqqqEXl&pmqARNIskA(x@qC|wDQs?S z8aw@1Z~E_dYh(Ug+Ja9tjN4_2&t0;wNanM%ConefMtg(tUU#WQ=bMOtH%$Nl!HhS2 zPQP9KA=2i=lY6|73@@5{FYA}kp~4DePxv%&eLJIH}6-b+Pb=3j?3?{lFZqxgKmd|2YI{_Q(PsrWLF1$5hO@6Elff*@Xy09)I7#%A1{N<$tMjIB4K$SkwyAe`nsC@xyc%ODI8&ut!G1~ z0`K}hq3$H+Y*aJL+D){*#Q}W%imgkTINCS9qbt_(LGdzqDqdbxE$e@80hBMGHJqAn z>A+B?=YZaR_Ntq+$8Eb68HU9Kg&_4d@yZW+XeRx47L-x;cw*4Dt^A1|LHsf-FTRFp zn^YAhomX{14_Kk&wvzhJ8`@aq8g3uQUm9 ze+S*|!%3ke#1jg*su@+;AY+)J#>z_vU2=UAa6(9OdBj^TJdpFpA?;D^vUASkl)S_snGXI6 zEsoZgpAf_1Ub2-D}U}Kb&KOf zqE=P@=6E$hpUXMar5^{}Rr~@z$Zg_)w6{LAm2bvnEn|->+{`oJMcZB06G#krk5AAn zK?-(h`kkpT>=wT0eWYr61xiYhffzXXCn<#0=zNNP#-G~#HOx0Kg$!d@I5=c=FAn}h zE9u5Peo}TNhDr*oqWgnJs?bXj=HuNvnPGPj+>G>+LIc$)q?|Gsu%l6dl?c)h5VpZ@ zh<=KpBwa(X<-UH+FlDt>hVFjVO+os16^o)qNOF+3-|$fb_*-zf0a0PO&OWb&5+B@F zx=7r0WnHJ`+#W?1-;%z+t(;ATH*N|w7Nmey@6gi^U~7`$^w zxI?0p9>{AH5A)b0D6-fI<|piKf}0()VZTH(p3GJ<5}o_`js#o9hEcsu{pXP6iqC{G z`)AmiQ&?y7748ixu5YFd#&6~GiK%}n(G$a*(vBp|R`%T0fa)oUvJvH$!B%8p+rQZn zWXe;fM#ET3Jrsrmk+avOJ@)OB@vmYUYPLjYk~^gnl(!X=ccmVx;&vne4{*e9 zY=t6z%Ri+jiRD4|o8 z34136IK)K8Rvv*+#mRC{%k*kUYhB6>ngCWu!!3pj5!sX22e5dnHnggrVfz$5**x+Zd|Pz2G@oKx&Yl)B3fGr<}dKFLvk}p?4}!Pdh@of<2@4Q8nh zeDmrr3QJ}>OHd<>FIK)VX10#;v%1mNO(+jX@fYlU#VV>2iu{$Zdpdu&$nxB8-=R$W z6u~z)ywt@ihY8?7wHqUX@gG^$U1`fkxkh=osJ&i6jCogefe92AfJ~L9@XG>oS;5?j z$?rtb*T0V-6<=;(5vCY^KeeE;?l@Z=gzfcbKr#ia<~pW%FqBkSvYImF%izbwi3tF; z_d%$Izh5`bdgfQ!C0j65qX0yt#gE?_q|+U`z$t0tC^o$87mVL-A%F$HU+MpFFVcqX zRtgeutba_S^l%@Jb`O|9aL-7v;}Q^lskc z{TD5=i1zNPR}bU@{&EBp#>bv^E$PtmJopfZ+Rw_uu#AlTC}9Ve(!8^x@mHAE?_mR5 zn9E<@B_(A`g6l|Aq;2-~`;H7>pmfMoy?|Oili!6MBQqt7us9v$Ul@?-z># z6fQb4A^NDmj)2Z&HRi1p#P`vs_F&^;Ru3o+Og~Y(6Mjzyx*+b|TzO^h>DPs@mH)AM zFBtDFiyhK1;WM+SJd^?^C+O9z6@Y^{HMdBpztl;8$tr6Y9UU77h=YUo0)lm(ar7Rg z;&Ut2tkrSSmk4bX>qVoTPtXufzSM25! zgxyWONHpXkoJ^vq279Po08d9~$_5>-$_nJt0Ie&myrFl`HJ{;$dulP(?u#RJ^~efQ;35mKSOf1wM)D6gpx%$hSD zX1M*KB!*OB^i)dZ+;PL&)DDSdQZzSWSw`mlDo*v=_x=iJm&JL9pO(Zks!(nUlnkl? zj@8OncvDpe(obFeTK?w6cX*9Fmz0NeGUG;fg-R%tgao}+4Z}W|7JCU?PZl~B&T|+> zVR>qM+B*MWHxm$BZ&(fFy|>7&o#o=;xA-(XCagm36A+8qVPg|;3HI~GJ-0MPNui8i zA{6EkeCiZHfnJbn zy3ZT&E#L_whvZ4ZC=$4+OSn-hfd-jpy7gg2+RwKZ3i|z(^$9Pbw6EL&I1PYbLy}XkF3L_I^kkD)=8JW^hlTOV^f5X=^Kd~ED{_Z zN>2~h@jf0*N2pWy{}HCnL7X9$n1J{#1pKCq|DurSPB-PWPR#%c`kxbXpAOb&U*)O8|yFrT*10(Th> z4_M@{e;%wGrSi~w4vV-xi<}^*4fzkzX*$&OK87}00JzuvhRtS!&(~W4xNj;%P$^Pi zHW3g)gS}Ad3RFQO8Ikr2IdkYUg3|b@_)FqBc)znwAzZA3bNctqeZ{5LGydE|0*&** zD>hdVURR>|f$1K0JBN*BiH%t-EHCt6fBq`Y@CrIoVd-pADEYog2vA8tVK$kqB)p%c zc4gJedxQPj-sQA?aR@mWl-mitA{L{!ec;)Nq7Zsrl!nsa1L%Ku-0%FT;Lxi!LQjA3 zz3ruKcv${!OBQVLitSe{4^??s>rzPf8ndrg7C+kCie$ zV8w%;7@pc$A`ehf$j{Ik3jor;keB|Su9Wfsb2f0BQ2LPNlsJ`QO;qlE{0)%)e|TBz z;4U(AL#G>)^WRFHG*(_>Q|jp2OO=pPuNiVmLaX>NU)PZL6|G~qNbL?DP-klW$4llU zL{sfzz8oReOY)(@ilGiJ>dW$#qjbnlx;FWkus52cN|#?C4?V9!Lb6pz)50E1E#7{k)X)!?3U zIyhVkUsips0^mx~j>6ngM6X(OrY1c{w7Jt+`3MB^!`04Kt2W4A1$K)%vC0uKbUI@( z5I`%k$+0A>J!mW!Ma@<8z2~8~j^&b{BN`h1ndoYSHPru^{6%5XJEv;pK?N z;pFD3Nr4?-@aRP5-@e_t@=hSK1OR{&iQXy2YQ$Z7hi@9rWmzAueY!HX32JN@8i0zD zGT8&}*8i@P{o&(%oU(rfhw)40V2D$$^wI&dztu zJ4@Tw;Oe;$`3JJ~9UX-j? zsdz1XGTUyY->=sg;ns8EP$>g3x2H6bfWLoVJe-ok(h@92Va2+#*|eW~YZiySGGTve z8_`}7lyh4Lm(`G_amwsnUHVEDF0wa5Fo^{$-WXiR-I0d>iKWNV+cU}wQKXC~HuOD6|Z$T0>fQKt(7hQe6F;p}8@1WzV`&1nKavZ4hmgve*L zq(pqwJ;X98r` zeyYR^UhxYm)Gz*64|L;TE7@5r8>@qN^WIePf0cp%<;?k>-dO2c7Istnxy;6RvJ#g! zFJMm(LRDO}!Dbh4w|r@u7Gr*ix*y{5N%WQ7kEE54_9;%x?M2}Hp9@fdko?f0_gqpV z@VJ<(pj;{%-J@1GgQU{MeMc0{Vhw*W(i@ZU_!Ar54YSp$dd*!oM5sBMDw#rxoFt?F zos;#rU)$t1%x-N-Q3!hyFB@+ckZj zx!{NGZ6U#A;#?1?{6G1Wm|#Pp(o2wM%yE~gg}MblxW8ELPv|E|~G zJvM!LEB;VnjS@b&CYzp0nciUyNA>$+4C|sVqRW7AW^hb=!Y>aP&;Ul{ zfoAnYY!yR{4csV3&;^Bs(*A|vLTyk=I-xcZ@i7ta)jnzpo_;I^Zn^D}tf!9qP=G8& z<+=#CQu*&oPbafwMmx1uv9Q~A4f$FLU}Lh4LF=KHSm{xT&^Fu4ad=61dmHw_uH1Iz zF5SB{F8gyDZ=N4|sEQ7y(T6};)sSVb^lS5Is|hztB7o*J0O&{KgAt$=D)P9c@x{@jW9Ixt?dn}KgZ-67#&8*A$WW2Yz#lZqa;*qx5wKN zq+FeZC=ds76mwoADvMoPh~sEmHvY*+*A+|`dun_i>w|r2aHIM5!DqAzjvHWd}euM|!ek*q<2Hv=SU(cFB-q`oPY@lnj_OfV%bG*V= z@@LZpJoNtL2X-l`ljNj8dF>pvA;gZ~irljNdyVBI%@U?J0lnR%+CdIof z>@%(|Q7?q;3p#>ed=88gW0k@b5;>)9l3j0^^Lc|;+~Qa?=hWF+;sJ_ng*oEzGYUXs z<}gMUnkYLCmCLC+kbrmRrLa-p@ft4dX~YLO-Tco>!Q(NPR-k3JhJt6W`2jU8p8tNy z@($|Y3bkG>ps*Ku)o#$xUtzx9CzNz~o9E23E1|HT!nNX^P|a2W`nc)s*2yNnHdxmm z)?zLV0l=9;{;idKl7L&(7mU{(CzCTkNl6F^cU$L$7+rMbU$|8)m5T~Gt=29rqNuIv z1-7|MWe1gO&${`lBsXYv5xF~le|LCv=~rhMc~bQw-7}-{^$KiGYeQbc`GL3IO>iu4 zHRjE~(A*jY{$kGNf--EK>=ztA+=x4BSe`;iAo#iIPKRL9nXL7}z@|~VkLvBllWAz& z8jY{i)9O)dzOqpaWyAt=aR>mHS1As~ko@`j4N!JE8#I@Qh^26tI9}#TeSV7*`F{R%G*81hZXr$zjCP$o*IYk{e7Y9KTSQ)51f0iK;^i@2L!o)C;QJy zTu;1Q`$um_(d&*-_FjF>tS)**CK+5SQ< zsO${WuhUDBz`uOEb*7wLsOewFkn^@0!2l&#CzBqn(r9N{|Uw7LJdZFFngr&NTB zx^V;5wH3fps`|CZ#$kT@ z(8)y=n;VI*6Pjg%vnetfhjl$ZKeD0hZwj0jZ#dEpy`?OGB07*(0YI^#i_>*+`28m& z0D$3e$3ru1y9L$XPP@5{L(ug3>Ejx@i0IZ$c&7W&gu@Choso)NwkLmFBRV&)+v#R|_uE0o8(10;G31=)8^} z4ymH&>ax7w*2z{u zwiyy^X(}N1e{f{b&=!dhID`%n0*7xjQ=yGfHyU5Z{=s!&`ML=hITe)K)sen=P$v%@ z@-+9|g1*xk`GedHau*dIwL4+y2wR)qPzW-8`JT}W#zZ`fA8(7 zxOd4w{fy`&E5J%WrgabLG_Q>-;?qYC=+NieLJ#FvlYgo<9I$l4p=TF#UaZtLO7{s( zt{d{wZWpxG-Q2^wbfU9;I?jt_Fq~}lUt{o-OpT$k><8w89{tq$bS3|tF%<$ZiNQ7-KANE3 zpox_m7oW0vO-x@qnwmn1!wV$>{ZwfTQ}-L@eG@6sB4ahQ=jbGh0)Dj?(PO5*>j#Lb ze;&U(6+2-*%tM*`Y&JKDGgneZW^Ib~b<4T@W8Zr}$u8YKJTGxye_J854Q0OK8x9iD zNgfthG^(N)P)-8Wzf(LTiZ7!ZPMF3oA9u1T05|F;G1=5@2 zk_XRqG6GL2oiqu3jPG1czBA&kLrtszybWZBug_Qz+y(_h7;j>i%5FU(+R#K>w~2iF z$ZBHT&`+j@x3$q5u;d`Q*M z87~_kdrKSIuTcB^914f8BcxGPTdUlzQX#ybYYY;bp3O4%&gf9xq?GQ-!Vircd}HgUdf zJcYabCk6V2tV1p_Dhl6ZJU%3dZz-w`;O1F=pi%}ZEVSwVwR>FTU>8(ZETzqkbiz*C zr`wNK4ix%b-G6(qk%hG%{{*J%1~Wm#R?U^ix09DdN+9C(--eSL0Yr;Y_%_5J7S`d< za!%)a0@FwC)6h3wscsa!^7S>5ISa7g@bg7}it{@+p9eCto zJ^{^xv9+t0p1o8-zeJv=#&1}J2=>t;D68AbHeM*Uhb5Bgl+f5jzi4Jk(!0 zB1UzSJEq^c=1CJwixH-<&+NP?@dZ{;_C9HDO!RQ!89Gloiv7p5wmR4FmE&_!!-Xr~ zKV5<9GG{l1FIqinC5^k10-q9FUL-*Pyoc8scCLkEu8o_=Zm4z)Jo=(Eavd_it-H(p zX4D2QzN2NXFJ@g2-FEx=yX*=y9NmmQ(v)yRxL=ILhtB!u$G)Mf`~5U3(;-mV3>_bwX$fV1td4MP*mki zn-Nq8nK4UHp0W6KE-OUf+<~&rTa_ZrIM>lHg{e zf^g9*??+}aodCV09m+`CB(P6e#7PF>h20sMw|;005vu|FG~ss-`m^#jh880B>yMSn zlL7dAwGyJ;0^^f8rP1`W%f!%M+zB=%>Kx#*U|_i^q#~p=@u25WSC?a6qek!qnLN9UiP41U&!GzvYTzHuy4thvvI)fhOkIEO=s)ksSd zevzCqHdzMuTKxp>ta(5o@)v|#1bg%5)ibuLJkzO%&{RwNWp?k3n`Z!evreW@FQCo# z`48kNkI5$$6HoqQ(;Yuz#CmO#DHs|Ybc04x@@%>yVboc_6NM-B(`{o?#RfmjAYvR@OSJ_n|J0!to~BY;3t=Tikh z_anoaUY_t*%4M5oO7N8CB^r8g=JaRC=?cS&7RsFRG}j3~V3e|te|^fp)Z3?mVMh(y zT!+wVem_kcN^%|&^9!N)cGeJL1i()EKP6MeR-8*@`Fb|+ln9!T69o10=0OoI^0RIn zKLSe-j~UO=HI5j3Ah&#W-VO+pg%k5&w}c}7<`lxaPLRvM{#>>yMl zO^@IDi76H@7_x?SzI-oKPKpE8o~#`ow_OyC&0*yBzTx9gidm5tmh_%{R#LzJ?tZfD zl|%Okk$6JzTT-aV!vRnD;?2&G^wpiw>sltPu~1au#bo+Y~!LoeB3ioeHsOp}=uCYP^;9tpcX;yeSyY-gQ zzSysSso*C-=&!CLxo3gST=J9Y+lQ9}4yYST2EnrL{5RTc*1K!qLL@gy#`k4l^`C#X-Iq0KS6gnDereAh>ItI@-#9v~zM+cHYCHEp1^2!SZg^EWR zk}~;n;TV3Be-!Dlb{Bcby#1kjh3vxo!cS|lb#QybU$n=JLiP-{Csl0zEr%GIKDICt z9pEXJjM;1uC~Ib?fe3H@y_QpWRFQMTTqpR_#~f-qWkzU{wSMyKLSJ)JikQd+krEGt zBO^`eNwK*DRii^sO`i&8p@L3(mc_uSe;DZS&`iSkBg@=0DpG1TEs~B6>KDi<9Jyd{ zvDp6mrGVLwp+7Kza?oYQ6f4vyQf%L0v$O75kYQ z0|kp!N(%Lx!er43lu)}Uw#P8Vt{FJzXu`0?Kxgqr~*#?z)Zg|iR#7Vbz~Ab}dx zPy57N>`oP+fU7H`AwoR;U;g?Yxge|oDjJ39eTPc#G_x`O;I=TOAmhE8N%+Tk!U8S` zeG60XONTYQ(6K}n9OL4TT!_6`EX6OqtKekQ^pq5m`$`N$;0+@yEFH!7L3tNpW!{t7 zNj0}9n185sP`*FTO(1ql%Hrz!IMJ6j@p0bG~Xd^G1N2v;3>U(+n<+u z_xTCWm{b^HYOX5{g)vRj!w%0neV=~+vzaGw%ZTGxi=R#-?c&6tHlJOPVwocJ8z{N#z*CqL1SY$`y&-mYoC5Yo3Zux@E@sk@iTeM zl;wO}4Iv%fXVh>ZR6G-Asl1K>X{ka*oy7}0+qcC<&iHdw^QtgNOKr0ekQv8cw+E+f zoFKE;GsCB@KMkJoH|d4u7Jg-DBnR0xvfbyPl1?h-5F`b>wUa2&fpW6LmWca`PG#&5 zpSnOs_ry%?lk8SNz14g;OuG3oKPD)(zv;^ow&Zw`x3D?}9#s$z#kx?VMeSj7$E9 zI<2Kh@jE2~$JdblDEhv#S!)$XpxyZ$d->}hyWfh_#m!yC4-l#L>!MN4YqyZoz1JfC zpfn1*Dpw2!^!ZM;$}uU@c?N0w(9ROyGRdlLI&eOgpoh|s$K_2-UIoELzra;JLbvk8 z+1_%qfy%}F#S0cy&rT&!t}1d{a%P!LV)Wh6L}tas3-1x?;BmTM`{dfyN)yQy{YjN#oDg2g`~&HXw%>b1d<(jXonigsNh1EG?NSX>(O%{KLC z_bPIcQJL*+$1N+BFgKpA=`v{4O?Z*)@l|ld=mmud5Q?2wM$-j;#Qtl?TCB2ukEz2& zVAFjxeIH9A&va=*gz?96i`&DVzCu48oO76>5WqkYXL@cmDFnla7HeuNMr^5f!4L2Bp7TPlvav1qL0J5 zmOp*!zm$4ls~UdFjw3qV>zxmmrdY=YFFCn8?>vXFBV3Y|RZ^K3f0ps0iatKgw z^9#>^_=Yvpk}njCdaVY=PtJ6MeI5_;NdmPT?@LEBn%`kX^zpmDOA)6lSFXw0^nLk&gTQaOTH;}Bz{j#-2Rr3Jxzv%X z!EVql>k1dLcc*s}fCq5qyIKl%8PdnMS*rBswAx_4pZ}n7c)%f3k)WSH-&*QK_{2-^ z_psN2kxiwC0|PEaGCKK@yGb%$|`M10O@@ugN{^Co@cCgG1lmxb}p-7#^yztl>rus3wq-CU$9 z*4OpW)ttJn+SYpdUXN;-jPa+J&bRJ}7XN5jko#jt9aWII+$bg!>UXc&BwD`s?fOfs zUMQic#hWM#OJ!|_iVhx6k{0;tQM?6WhP@afXm@+NWu z00vuwl$BZifeN=&e93==^jJhewXa`9u~qL&jDLG)v``vgIQaXik4j$agQv6d@OHC1 zRUZK$)UzrcY4*@o@E7kgAC#`#KW{BPqTOhu8!o!9kIt#(R{zbr5+J0rmy>ftQJGz8 zTaiBlA+d+6XNo_vr+{9{LA#MuiRL@wC>`o|kBYhOvGpOhsvmv$A)3(Td7C0UtWX(U zai}Axp@UBOLlqQg7rC-Iv)yp5pH5iZ$XViTtUfs6{-YtgCt@=L@B5+Pv1nfgL($yv ztl5G2B39&CYiCT>a;qatt6fLK81$;~1fD|kqd2^BQsf!zll|2pXEbcR$%%g$B?hBi*`%IFRjxQeS^crt#w1bfc=#`ecnc{ zJ=n1{4sV}rZ8#j4e`brt2XX=osnpfC7knl6qSa$xUG|v~Y;_s^an&(49E`VS>L+V3 zxW`HiIHEKfIFBtr%2y2WJ!pczfYEW_*5y=u8xLT@ZKHT*4T9b<{V3HW9i)oc3iGaj zzT@yrDZz8r&7rvjZ~AB;N9+ zrslk!`+e1p)(tV^?_&ZAt4mKQt`^L!591t`)y0gC8XF~oi$==+;1efkRi&$e$~7+edf7|YnaU%Y9-Cf86nv0t=VnT3pFqfumLFJ4#8zmfeee<=8aP(*FLV`u=r z%}+r>?xh55)PLV?<$55GE{FknDw)yK9d`sKY@ii+5b|c(j+@Xf%)CV0P$l;Dv}HKU z$#)K+i;UjYlIC6=H}uI;;=|$5OQ8^NYv*{VI zWHIPQR~7>rYQ8+XopIM~=c(5ZvUl`yjtk>Qi=o^uRpf6oO)N%`@SSaNFZW9ZP z8hd`6PSpU9sMfld;2g)3H5?V!%s80m&|N3nkV&w3`ljRe6ZxPozbQ4ar~P{UNCbJw z&%!@iw5Ls{xW(a|MZaua0J5U!%T&K$gu(M4l12RS(|SG=rJ`jIVl_~=)L}z+pE+r~ zw9`Ts>zmZQN?O&sYtU0#CcakJs~}y9^8V~FzDu@l?jNv@BwjU;?WiS8ap7G{JAIXJ zEcjuX@MwG?+HKg_R{bbt$XYMk{Va zKO6za^E|DTG#T7#yOa6DM}d@fEqSd)M17eu5@|#MfmFJdjEj|RH!oe@w$Y^r0f9hR zo1%CPls?9tyk+fyM2{5PRSo-^^KvZ}Vv8xUelB!VyH~b6g+$vp$ClllLb^zovk%JI z%u0@*Mk~Ibz>K&t84yT!c|>y`OWV9#^;hS$yjF1#~uK*WoX5>9{Kz*UR`dX=*B1_>N19Y{wWt{({FzCj1VW!w+@q(H)$} zR=#Is>+si=Bw|t<>(kW`)sx-ToR*so^lwF$oY%K5wNDlQu#i zT|Pk}UcKk7x9CApbg$M-4AW(WU4~EJXNNcrDpu(vCKy>qT%!wnCPtYI?{bAXYhu4; zP~kQ2QujYPhCY7@!R2b;?u{7a7Vl2{aVGAATV{#~yyV z?6CJj2x-sLs`C&4S-ehSVaJvS&C4FwC2~zV*JF|!VBW-*2~gfkO${xsH!7kz0iU5r zyWvs3q@`%wV~!icf-w@eZ_G`L`wMM$-V>)^w{i$;iE1xnlQVdjtG!1mvr2RDe}Kic z7+K2qguPxpMfJsWpUTUev{jah=kYA`=`%j>3Zh+He9N(FZ5$UErNgN$$j`sko*T9J zP_wZ~`ha3Ru(_fROX&L1(UGq+yP+AV>o_{{RpNvhRBjA7vqEz%_c)78kb+AnGR2*np5_bxMC|Mn~LrO8m(! z#&X8o_=Q>8XvK@l{STwX*U$K8C<(xup9tuL+kf0lZba7*uqbn{xdIJGjfS(Yzr2#I z{zyA4$70nO1unazOK{YE&YRr8O@zSn8tj8*S0Z6aA(k+TT>Hih4+VnLE8OYomUq2A zGs9U!MwnnfFSXB?p&atRWtY9~$j?JQwXO-t5R*k~BOBjj$3Wc-LeCvbtN4S!xx+j- zoj36N0?aAhfaQ75h`>|d^D1p8VTP+cUu7?+Rbb=F9<5MFw}t7<7kwSao}xtxZm=&G z>5;jCf#~9s0EXea95?Wu3oElji~00!zd3>$k9ib(wy4bs@>3f*)t&1p-d~nG^=j$n zBe8-oH*;_zL*w~}jSsY-LQ3mGP&qU}TWglTBf|p~G!*PIePszr#01(_Poj=lao-96R%ZITs?lp%({hRv)f?M@iMI-cXox{Z1_Rp7TXsRc_^AYXj4gIb#-BuET`i4G?3TmMs8h zHBXmCHgw=g;J6Iqs`>ryKyki$yo`e+u`lCkKZ!V=3!~J;zc86J8E9CQU?cu;Fpp z63*7rw{+(kN$7pCRH$OJ9I7}lx}=7wRoy|;d_VkS?h^iF zsdkz4mg&J1eLwmiGiO&I_i8Mfcg=oE*)1gKAyKC{Zry(Wln8B|p5GMD$~P(e;A8_a zW#USv$@fu=j*9ap7W%ZlHWmCkc43)ui1mE7zF*#@?BujkziWFss2S#=%dOdc$f@eR zEmf#su9lc4Hsdl{fKGLiy;bU@46@+^o1DJcrWP`V=XjhYB^K(nTRGqu@b2n!eeM%9 zu$SIKw?9ZKy=ZYJ^Ux*g-yHPGQnNp_zH=rG@epuYJa*sjuPERFFCcwv>F!-}1eD$3 zG18BSJ@_F7xS>lDj1dcJ_Vbq$p9I!lY?*l2@S%kwG9|AY~_XOx};jFnFa!s9Eir1R_!>A)pmT0wu_kx z0u3Z=Vs##TCs}XChp+YcFkZ?OpdNl6@t5o!hZzbyTgtbX+gRg|&kb7s5CP0+Myl(I7`y?B|elyL5j3)WJ&7i|R80Rlz8oS_(S-Cos!>yI5 z$h4Qd@3j|oK>H6Lf9qRvsioECSwK%Ld|FWr7HB&Xv?(T<{Jx}C3VT>vB`lZ}dal5M z{W{?$=^IKng+XT~UMEm~_JTD`N5nn8_~Ym?*mwqDOJWBuMXMc$1z{ThSF}oEtmhTj zD@o^9Gml&`GF+0cfAOqfDx&Ejrg2}|e9TgDtJP|-f(sNF-3S*O6)%z6U|;6J)xxw) zACJHamw|_=ev@t`uX($%n%f6&D)yxhsk=2gER&4*k{(&aNZ}n_R zlR<-%A?&8;rPi3LJR`%lz#K>D_H%8nw&UTQ-+k$Ut1d$Fb{*_M3cz;ZWL99p*Q zq=zJtdC5VJq%HLepNesrP>d#AuWa7SFWx<)rS{U>%j zGIb_*20P$u_ishNRlX8*;oY9?%3DF#>blEWFD!0L$ujo!-jy{qp=Aa0ri(mchro2X zPekKn@K8I)5$V8(^83q-e>&|B?kz-rE+;4hA+9kEV?T1rJ%bT~PmQ7QzH)P%_7Lio5Xf?Gw-uw)8ip^}X zU0*Fj_6$Fkv0?_o3dJNsr+0|JkhWX`@CWOR21w|i%^USKT5`Ep;L*N+iBCDmMpNnG z*{LP9F18fy*=i=AS$n)S1OirX3JR;?2iW`0ixqe{#;*hTg-6|Jj&=ooiNC zeHi<%-xnXwc#?5n>y2Y@H}FspV(%O!^W$yWa%W!Q6aOwF{_s*1Lbz(?w4|K@%`o)^|goH^*-2~%fM|sLBG4d|E3-vvgI!uEV z5=EDkkQ+rQdvSReBgH5iHWRX2zYid9ZP}H9iRw~d5jOm`(zs60lRbwh5DfamB%!N5FRktOn0bc5 zVa7z*#6T{B@_Uz-yj~Qu6UiMR@~6?O@q8uHE&jgetDAFhZncPwuc|uR^%V!IdjSg^ zaXFv^7==?EHHDvwEYhYQhlG8C6+uTH6>-!p)ccLI(6><){{+=U+1)L#u>`_$l*V|k3E@~jSeN)CJAT3mJHTHfNvKOYmHcb+1n*K7C zb(+|GHnFjU{OLX4W)p8QYzSixwZ1T1FJ@siX2BEB+iB2lLex7VD0~VL^bcZP?H2iR zxjOzWMUW;Ahq~NhVl!znfy9M#b3(vs?2LmamCcvAsjB!| z8epg$oRnAJoL7(EWOh$`^pUyo?6Ei;mdTMIb%U^>@>}RJ}l={z)m+sJA>7tfQz&%u#ZJDH4z=HQCHnS!+>uF(0qx{wg zP*!$cJvpzQrTS@R-`Pu?C$+H7Dww}|@$n9)z||0Q??ES-vzx%5W)GIDf5YiO%s%j? zf)Wi#;Oi(R6VZ2{X3aj}jVel^5fmP4kwc}cpdKOQ*FVKK-HdQs&ieF_I^0eKsQ`jc z&y;tjC|upnIpdk_(6ZB>2=v)-rqv zKiZ(No*7~|5oQJlXNEmN33W^Wzz}&H|Na6KNFSvpIR8H+4vgmk3;%;?fwy=d%YP9t zqdJx4KM0tEoyzhbN JQx4ci{SSGb(sKX+ literal 0 HcmV?d00001 diff --git a/docs/gitbook/usage/nginx-progressive-delivery.md b/docs/gitbook/usage/nginx-progressive-delivery.md new file mode 100644 index 00000000..2fa7a549 --- /dev/null +++ b/docs/gitbook/usage/nginx-progressive-delivery.md @@ -0,0 +1,355 @@ +# NGNIX Ingress Controller Canary Deployments + +This guide shows you how to use the NGINX ingress controller and Flagger to automate canary deployments and A/B testing. + +![Flagger NGINX Ingress Controller](https://raw.githubusercontent.com/weaveworks/flagger/master/docs/diagrams/flagger-nginx-overview.png) + +### Prerequisites + +Flagger requires a Kubernetes cluster **v1.11** or newer and NGINX ingress **0.24** or newer. + +Install NGINX with Helm: + +```bash +helm upgrade -i nginx-ingress stable/nginx-ingress \ +--namespace ingress-nginx \ +--set controller.stats.enabled=true \ +--set controller.metrics.enabled=true +``` + +Install Flagger and the Prometheus add-on in the same namespace as NGINX: + +```bash +helm repo add flagger https://flagger.app + +helm upgrade -i flagger flagger/flagger \ +--namespace ingress-nginx \ +--set prometheus.install=true \ +--set meshProvider=nginx +``` + +Optionally you can enable Slack notifications: + +```bash +helm upgrade -i flagger flagger/flagger \ +--reuse-values \ +--namespace ingress-nginx \ +--set slack.url=https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK \ +--set slack.channel=general \ +--set slack.user=flagger +``` + +### Bootstrap + +Flagger takes a Kubernetes deployment and optionally a horizontal pod autoscaler (HPA), +then creates a series of objects (Kubernetes deployments, ClusterIP services and canary ingress). +These objects expose the application outside the cluster and drive the canary analysis and promotion. + +Create a test namespace: + +```bash +kubectl create ns test +``` + +Create a deployment and a horizontal pod autoscaler: + +```bash +kubectl apply -f ${REPO}/artifacts/nginx/deployment.yaml +kubectl apply -f ${REPO}/artifacts/nginx/hpa.yaml +``` + +Deploy the load testing service to generate traffic during the canary analysis: + +```bash +helm upgrade -i flagger-loadtester flagger/loadtester \ +--namespace=test +``` + +Create an ingress definition (replace `app.exmaple.com` with your own domain): + +```yaml +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: podinfo + namespace: test + labels: + app: podinfo + annotations: + kubernetes.io/ingress.class: "nginx" +spec: + rules: + - host: app.exmaple.com + http: + paths: + - backend: + serviceName: podinfo + servicePort: 9898 +``` + +Save the above resource as podinfo-ingress.yaml and then apply it: + +```bash +kubectl apply -f ./podinfo-ingress.yaml +``` + +Create a canary custom resource (replace `app.exmaple.com` with your own domain): + +```yaml +apiVersion: flagger.app/v1alpha3 +kind: Canary +metadata: + name: podinfo + namespace: test +spec: + # deployment reference + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: podinfo + # ingress reference + ingressRef: + apiVersion: extensions/v1beta1 + kind: Ingress + name: podinfo + # HPA reference (optional) + autoscalerRef: + apiVersion: autoscaling/v2beta1 + kind: HorizontalPodAutoscaler + name: podinfo + # the maximum time in seconds for the canary deployment + # to make progress before it is rollback (default 600s) + progressDeadlineSeconds: 60 + service: + # container port + port: 9898 + canaryAnalysis: + # schedule interval (default 60s) + interval: 10s + # max number of failed metric checks before rollback + threshold: 10 + # max traffic percentage routed to canary + # percentage (0-100) + maxWeight: 50 + # canary increment step + # percentage (0-100) + stepWeight: 5 + # NGINX Prometheus checks + metrics: + - name: request-success-rate + # minimum req success rate (non 5xx responses) + # percentage (0-100) + threshold: 99 + interval: 1m + # load testing (optional) + webhooks: + - name: load-test + url: http://flagger-loadtester.test/ + timeout: 5s + metadata: + type: cmd + cmd: "hey -z 1m -q 10 -c 2 http://app.example.com/" +``` + +Save the above resource as podinfo-canary.yaml and then apply it: + +```bash +kubectl apply -f ./podinfo-canary.yaml +``` + +After a couple of seconds Flagger will create the canary objects: + +```bash +# applied +deployment.apps/podinfo +horizontalpodautoscaler.autoscaling/podinfo +ingresses.extensions/podinfo +canary.flagger.app/podinfo + +# generated +deployment.apps/podinfo-primary +horizontalpodautoscaler.autoscaling/podinfo-primary +service/podinfo +service/podinfo-canary +service/podinfo-primary +ingresses.extensions/podinfo-canary +``` + +### Automated canary promotion + +Flagger implements a control loop that gradually shifts traffic to the canary while measuring key performance indicators +like HTTP requests success rate, requests average duration and pod health. +Based on analysis of the KPIs a canary is promoted or aborted, and the analysis result is published to Slack. + +![Flagger Canary Stages](https://raw.githubusercontent.com/weaveworks/flagger/master/docs/diagrams/flagger-canary-steps.png) + +Trigger a canary deployment by updating the container image: + +```bash +kubectl -n test set image deployment/podinfo \ +podinfod=quay.io/stefanprodan/podinfo:1.4.1 +``` + +Flagger detects that the deployment revision changed and starts a new rollout: + +```text +kubectl -n test describe canary/podinfo + +Status: + Canary Weight: 0 + Failed Checks: 0 + Phase: Succeeded +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal Synced 3m flagger New revision detected podinfo.test + Normal Synced 3m flagger Scaling up podinfo.test + Warning Synced 3m flagger Waiting for podinfo.test rollout to finish: 0 of 1 updated replicas are available + Normal Synced 3m flagger Advance podinfo.test canary weight 5 + Normal Synced 3m flagger Advance podinfo.test canary weight 10 + Normal Synced 3m flagger Advance podinfo.test canary weight 15 + Normal Synced 2m flagger Advance podinfo.test canary weight 20 + Normal Synced 2m flagger Advance podinfo.test canary weight 25 + Normal Synced 1m flagger Advance podinfo.test canary weight 30 + Normal Synced 1m flagger Advance podinfo.test canary weight 35 + Normal Synced 55s flagger Advance podinfo.test canary weight 40 + Normal Synced 45s flagger Advance podinfo.test canary weight 45 + Normal Synced 35s flagger Advance podinfo.test canary weight 50 + Normal Synced 25s flagger Copying podinfo.test template spec to podinfo-primary.test + Warning Synced 15s flagger Waiting for podinfo-primary.test rollout to finish: 1 of 2 updated replicas are available + Normal Synced 5s flagger Promotion completed! Scaling down podinfo.test +``` + +**Note** that if you apply new changes to the deployment during the canary analysis, Flagger will restart the analysis. + +You can monitor all canaries with: + +```bash +watch kubectl get canaries --all-namespaces + +NAMESPACE NAME STATUS WEIGHT LASTTRANSITIONTIME +test podinfo Progressing 15 2019-05-06T14:05:07Z +prod frontend Succeeded 0 2019-05-05T16:15:07Z +prod backend Failed 0 2019-05-04T17:05:07Z +``` + +### Automated rollback + +During the canary analysis you can generate HTTP 500 errors to test if Flagger pauses and rolls back the faulted version. + +Trigger another canary deployment: + +```bash +kubectl -n test set image deployment/podinfo \ +podinfod=quay.io/stefanprodan/podinfo:1.4.2 +``` + +Generate HTTP 500 errors: + +```bash +watch curl http://app.exmaple.com/status/500 +``` + +When the number of failed checks reaches the canary analysis threshold, the traffic is routed back to the primary, +the canary is scaled to zero and the rollout is marked as failed. + +```text +kubectl -n test describe canary/podinfo + +Status: + Canary Weight: 0 + Failed Checks: 10 + Phase: Failed +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal Synced 3m flagger Starting canary deployment for podinfo.test + Normal Synced 3m flagger Advance podinfo.test canary weight 5 + Normal Synced 3m flagger Advance podinfo.test canary weight 10 + Normal Synced 3m flagger Advance podinfo.test canary weight 15 + Normal Synced 3m flagger Halt podinfo.test advancement success rate 69.17% < 99% + Normal Synced 2m flagger Halt podinfo.test advancement success rate 61.39% < 99% + Normal Synced 2m flagger Halt podinfo.test advancement success rate 55.06% < 99% + Normal Synced 2m flagger Halt podinfo.test advancement success rate 47.00% < 99% + Normal Synced 2m flagger (combined from similar events): Halt podinfo.test advancement success rate 38.08% < 99% + Warning Synced 1m flagger Rolling back podinfo.test failed checks threshold reached 10 + Warning Synced 1m flagger Canary failed! Scaling down podinfo.test +``` + +### A/B Testing + +Besides weighted routing, Flagger can be configured to route traffic to the canary based on HTTP match conditions. +In an A/B testing scenario, you'll be using HTTP headers or cookies to target a certain segment of your users. +This is particularly useful for frontend applications that require session affinity. + +![Flagger A/B Testing Stages](https://raw.githubusercontent.com/weaveworks/flagger/master/docs/diagrams/flagger-abtest-steps.png) + +Edit the canary analysis, remove the max/step weight and add the match conditions and iterations: + +```yaml + canaryAnalysis: + interval: 1m + threshold: 10 + iterations: 10 + match: + # curl -H 'X-Canary: insider' http://app.example.com + - headers: + x-canary: + exact: "insider" + # curl -b 'canary=always' http://app.example.com + - headers: + cookie: + exact: "canary" + metrics: + - name: request-success-rate + threshold: 99 + interval: 1m + webhooks: + - name: load-test + url: http://localhost:8888/ + timeout: 5s + metadata: + type: cmd + cmd: "hey -z 1m -q 10 -c 2 -H 'Cookie: canary=always' http://app.example.com/" + logCmdOutput: "true" +``` + +The above configuration will run an analysis for ten minutes targeting users that have a `canary` cookie set to `always` or +those that call the service using the `X-Canary: always` header. + +Trigger a canary deployment by updating the container image: + +```bash +kubectl -n test set image deployment/podinfo \ +podinfod=quay.io/stefanprodan/podinfo:1.5.0 +``` + +Flagger detects that the deployment revision changed and starts the A/B testing: + +```text +kubectl -n test describe canary/podinfo + +Status: + Failed Checks: 0 + Phase: Succeeded +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal Synced 3m flagger New revision detected podinfo.test + Normal Synced 3m flagger Scaling up podinfo.test + Warning Synced 3m flagger Waiting for podinfo.test rollout to finish: 0 of 1 updated replicas are available + Normal Synced 3m flagger Advance podinfo.test canary iteration 1/10 + Normal Synced 3m flagger Advance podinfo.test canary iteration 2/10 + Normal Synced 3m flagger Advance podinfo.test canary iteration 3/10 + Normal Synced 2m flagger Advance podinfo.test canary iteration 4/10 + Normal Synced 2m flagger Advance podinfo.test canary iteration 5/10 + Normal Synced 1m flagger Advance podinfo.test canary iteration 6/10 + Normal Synced 1m flagger Advance podinfo.test canary iteration 7/10 + Normal Synced 55s flagger Advance podinfo.test canary iteration 8/10 + Normal Synced 45s flagger Advance podinfo.test canary iteration 9/10 + Normal Synced 35s flagger Advance podinfo.test canary iteration 10/10 + Normal Synced 25s flagger Copying podinfo.test template spec to podinfo-primary.test + Warning Synced 15s flagger Waiting for podinfo-primary.test rollout to finish: 1 of 2 updated replicas are available + Normal Synced 5s flagger Promotion completed! Scaling down podinfo.test +``` + From 1f9f6fb55a02bb9356eecc531fc0508bcf33adec Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 8 May 2019 18:05:47 +0300 Subject: [PATCH 26/42] Release v0.13.0 --- CHANGELOG.md | 13 +++++++++++++ artifacts/flagger/deployment.yaml | 2 +- charts/flagger/Chart.yaml | 6 +++--- charts/flagger/values.yaml | 2 +- docs/gitbook/SUMMARY.md | 1 + pkg/version/version.go | 2 +- 6 files changed, 20 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c3e92630..3a5e9722 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,19 @@ All notable changes to this project are documented in this file. +## 0.13.0 (2019-04-08) + +Adds support for [NGINX](https://docs.flagger.app/usage/nginx-progressive-delivery) ingress controller + +#### Features + +- Add support for nginx ingress controller (weighted traffic and A/B testing) [#170](https://github.com/weaveworks/flagger/pull/170) +- Add Prometheus add-on to Flagger Helm chart for App Mesh and NGINX [79b3370](https://github.com/weaveworks/flagger/pull/170/commits/79b337089294a92961bc8446fd185b38c50a32df) + +#### Fixes + +- Fix duplicate hosts Istio error when using wildcards [#162](https://github.com/weaveworks/flagger/pull/162) + ## 0.12.0 (2019-04-29) Adds support for [SuperGloo](https://docs.flagger.app/install/flagger-install-with-supergloo) diff --git a/artifacts/flagger/deployment.yaml b/artifacts/flagger/deployment.yaml index 4152ea80..8d15cecd 100644 --- a/artifacts/flagger/deployment.yaml +++ b/artifacts/flagger/deployment.yaml @@ -22,7 +22,7 @@ spec: serviceAccountName: flagger containers: - name: flagger - image: weaveworks/flagger:0.12.0 + image: weaveworks/flagger:0.13.0 imagePullPolicy: IfNotPresent ports: - name: http diff --git a/charts/flagger/Chart.yaml b/charts/flagger/Chart.yaml index b74b28fc..80384c9a 100644 --- a/charts/flagger/Chart.yaml +++ b/charts/flagger/Chart.yaml @@ -1,10 +1,10 @@ apiVersion: v1 name: flagger -version: 0.12.0 -appVersion: 0.12.0 +version: 0.13.0 +appVersion: 0.13.0 kubeVersion: ">=1.11.0-0" engine: gotpl -description: Flagger is a Kubernetes operator that automates the promotion of canary deployments using Istio routing for traffic shifting and Prometheus metrics for canary analysis. +description: Flagger is a Kubernetes operator that automates the promotion of canary deployments using Istio, App Mesh or NGINX routing for traffic shifting and Prometheus metrics for canary analysis. home: https://docs.flagger.app icon: https://raw.githubusercontent.com/weaveworks/flagger/master/docs/logo/flagger-icon.png sources: diff --git a/charts/flagger/values.yaml b/charts/flagger/values.yaml index 0d1a07af..85984032 100644 --- a/charts/flagger/values.yaml +++ b/charts/flagger/values.yaml @@ -2,7 +2,7 @@ image: repository: weaveworks/flagger - tag: 0.12.0 + tag: 0.13.0 pullPolicy: IfNotPresent metricsServer: "http://prometheus:9090" diff --git a/docs/gitbook/SUMMARY.md b/docs/gitbook/SUMMARY.md index ad8721c1..59625ced 100644 --- a/docs/gitbook/SUMMARY.md +++ b/docs/gitbook/SUMMARY.md @@ -15,6 +15,7 @@ * [Istio Canary Deployments](usage/progressive-delivery.md) * [Istio A/B Testing](usage/ab-testing.md) * [App Mesh Canary Deployments](usage/appmesh-progressive-delivery.md) +* [NGINX Canary Deployments](usage/nginx-progressive-delivery.md) * [Monitoring](usage/monitoring.md) * [Alerting](usage/alerting.md) diff --git a/pkg/version/version.go b/pkg/version/version.go index 4716560a..f940057e 100644 --- a/pkg/version/version.go +++ b/pkg/version/version.go @@ -1,4 +1,4 @@ package version -var VERSION = "0.12.0" +var VERSION = "0.13.0" var REVISION = "unknown" From d57fc7d03e0f1f106f12012fdea60ab9ff61aa68 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 8 May 2019 18:05:58 +0300 Subject: [PATCH 27/42] Add v0.13.0 change log --- README.md | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index fd036842..8e2b4ab1 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ [![release](https://img.shields.io/github/release/weaveworks/flagger/all.svg)](https://github.com/weaveworks/flagger/releases) Flagger is a Kubernetes operator that automates the promotion of canary deployments -using Istio or App Mesh routing for traffic shifting and Prometheus metrics for canary analysis. +using Istio, App Mesh or NGINX routing for traffic shifting and Prometheus metrics for canary analysis. The canary analysis can be extended with webhooks for running acceptance tests, load tests or any other custom validation. @@ -39,6 +39,7 @@ Flagger documentation can be found at [docs.flagger.app](https://docs.flagger.ap * [Istio canary deployments](https://docs.flagger.app/usage/progressive-delivery) * [Istio A/B testing](https://docs.flagger.app/usage/ab-testing) * [App Mesh canary deployments](https://docs.flagger.app/usage/appmesh-progressive-delivery) + * [NGINX ingress controller canary deployments](https://docs.flagger.app/usage/nginx-progressive-delivery) * [Monitoring](https://docs.flagger.app/usage/monitoring) * [Alerting](https://docs.flagger.app/usage/alerting) * Tutorials @@ -153,16 +154,16 @@ For more details on how the canary analysis and promotion works please [read the ## Features -| Feature | Istio | App Mesh | SuperGloo | -| -------------------------------------------- | ------------------ | ------------------ |------------------ | -| Canary deployments (weighted traffic) | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | -| A/B testing (headers and cookies filters) | :heavy_check_mark: | :heavy_minus_sign: | :heavy_minus_sign: | -| Load testing | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | -| Webhooks (custom acceptance tests) | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | -| Request success rate check (Envoy metric) | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | -| Request duration check (Envoy metric) | :heavy_check_mark: | :heavy_minus_sign: | :heavy_check_mark: | -| Custom promql checks | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | -| Ingress gateway (CORS, retries and timeouts) | :heavy_check_mark: | :heavy_minus_sign: | :heavy_check_mark: | +| Feature | Istio | App Mesh | SuperGloo | NGINX Ingress | +| -------------------------------------------- | ------------------ | ------------------ |------------------ |------------------ | +| Canary deployments (weighted traffic) | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | +| A/B testing (headers and cookies filters) | :heavy_check_mark: | :heavy_minus_sign: | :heavy_minus_sign: | :heavy_check_mark: | +| Load testing | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | +| Webhooks (custom acceptance tests) | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | +| Request success rate check (Envoy metric) | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | +| Request duration check (Envoy metric) | :heavy_check_mark: | :heavy_minus_sign: | :heavy_check_mark: | :heavy_check_mark: | +| Custom promql checks | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | +| Ingress gateway (CORS, retries and timeouts) | :heavy_check_mark: | :heavy_minus_sign: | :heavy_check_mark: | :heavy_check_mark: | ## Roadmap From 9d12794600acf17e083f0ceca072b0dfa29d721d Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 8 May 2019 18:30:00 +0300 Subject: [PATCH 28/42] Add NGINX to readme --- README.md | 4 ++-- docs/gitbook/README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 8e2b4ab1..60e382c8 100644 --- a/README.md +++ b/README.md @@ -160,8 +160,8 @@ For more details on how the canary analysis and promotion works please [read the | A/B testing (headers and cookies filters) | :heavy_check_mark: | :heavy_minus_sign: | :heavy_minus_sign: | :heavy_check_mark: | | Load testing | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | | Webhooks (custom acceptance tests) | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | -| Request success rate check (Envoy metric) | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | -| Request duration check (Envoy metric) | :heavy_check_mark: | :heavy_minus_sign: | :heavy_check_mark: | :heavy_check_mark: | +| Request success rate check (L7 metric) | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | +| Request duration check (L7 metric) | :heavy_check_mark: | :heavy_minus_sign: | :heavy_check_mark: | :heavy_check_mark: | | Custom promql checks | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | | Ingress gateway (CORS, retries and timeouts) | :heavy_check_mark: | :heavy_minus_sign: | :heavy_check_mark: | :heavy_check_mark: | diff --git a/docs/gitbook/README.md b/docs/gitbook/README.md index 4f6c4ed7..5f027115 100644 --- a/docs/gitbook/README.md +++ b/docs/gitbook/README.md @@ -5,7 +5,7 @@ description: Flagger is a progressive delivery Kubernetes operator # Introduction [Flagger](https://github.com/weaveworks/flagger) is a **Kubernetes** operator that automates the promotion of canary -deployments using **Istio** or **App Mesh** routing for traffic shifting and **Prometheus** metrics for canary analysis. +deployments using **Istio**, **App Mesh** or **NGINX** routing for traffic shifting and **Prometheus** metrics for canary analysis. The canary analysis can be extended with webhooks for running system integration/acceptance tests, load tests, or any other custom validation. From 1335210cf5153d9b3b6730e52efc4f71bb2d8e7f Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 8 May 2019 19:03:53 +0300 Subject: [PATCH 29/42] Add the Prometheus add-on to App Mesh docs --- .../install/flagger-install-on-eks-appmesh.md | 20 ++++--------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/docs/gitbook/install/flagger-install-on-eks-appmesh.md b/docs/gitbook/install/flagger-install-on-eks-appmesh.md index 5e64dba8..761ba436 100644 --- a/docs/gitbook/install/flagger-install-on-eks-appmesh.md +++ b/docs/gitbook/install/flagger-install-on-eks-appmesh.md @@ -125,19 +125,6 @@ Status: Type: MeshActive ``` -### Install Prometheus - -In order to collect the App Mesh metrics that Flagger needs to run the canary analysis, -you'll need to setup a Prometheus instance to scrape the Envoy sidecars. - -Deploy Prometheus in the `appmesh-system` namespace: - -```bash -REPO=https://raw.githubusercontent.com/weaveworks/flagger/master - -kubectl apply -f ${REPO}/artifacts/eks/appmesh-prometheus.yaml -``` - ### Install Flagger and Grafana Add Flagger Helm repository: @@ -146,16 +133,17 @@ Add Flagger Helm repository: helm repo add flagger https://flagger.app ``` -Deploy Flagger in the _**appmesh-system**_ namespace: +Deploy Flagger and Prometheus in the _**appmesh-system**_ namespace: ```bash helm upgrade -i flagger flagger/flagger \ --namespace=appmesh-system \ --set meshProvider=appmesh \ ---set metricsServer=http://prometheus.appmesh-system:9090 +--set prometheus.install=true ``` -You can install Flagger in any namespace as long as it can talk to the Istio Prometheus service on port 9090. +In order to collect the App Mesh metrics that Flagger needs to run the canary analysis, +you'll need to setup a Prometheus instance to scrape the Envoy sidecars. You can enable **Slack** notifications with: From c933476fff0149a036cce0b441a3bec428185b3c Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 8 May 2019 20:26:40 +0300 Subject: [PATCH 30/42] Bump Grafana chart version --- charts/grafana/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/grafana/Chart.yaml b/charts/grafana/Chart.yaml index 3ed9f1c2..7e057602 100644 --- a/charts/grafana/Chart.yaml +++ b/charts/grafana/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 name: grafana -version: 1.1.0 +version: 1.2.0 appVersion: 5.4.3 description: Grafana dashboards for monitoring Flagger canary deployments icon: https://raw.githubusercontent.com/weaveworks/flagger/master/docs/logo/flagger-icon.png From ecaa2030914e9777c2acd5a7b8baf8d8de663490 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Thu, 9 May 2019 13:49:48 +0300 Subject: [PATCH 31/42] Fix custom metric checks - escape the prom query before encoding it --- pkg/metrics/observer.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/metrics/observer.go b/pkg/metrics/observer.go index ddb87119..ef0a5cfd 100644 --- a/pkg/metrics/observer.go +++ b/pkg/metrics/observer.go @@ -99,7 +99,9 @@ func (c *Observer) GetScalar(query string) (float64, error) { query = strings.Replace(query, " ", "", -1) var value *float64 - result, err := c.queryMetric(query) + + querySt := url.QueryEscape(query) + result, err := c.queryMetric(querySt) if err != nil { return 0, err } From 121a65fad0354afe65ef0792327869f2b028c730 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Thu, 9 May 2019 13:50:47 +0300 Subject: [PATCH 32/42] Fix nginx promql namespace selector --- pkg/metrics/nginx.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/metrics/nginx.go b/pkg/metrics/nginx.go index 7c6eb56e..bdd1c8f3 100644 --- a/pkg/metrics/nginx.go +++ b/pkg/metrics/nginx.go @@ -9,13 +9,13 @@ import ( const nginxSuccessRateQuery = ` sum(rate( -nginx_ingress_controller_requests{kubernetes_namespace="{{ .Namespace }}", +nginx_ingress_controller_requests{namespace="{{ .Namespace }}", ingress="{{ .Name }}", status!~"5.*"} [{{ .Interval }}])) / sum(rate( -nginx_ingress_controller_requests{kubernetes_namespace="{{ .Namespace }}", +nginx_ingress_controller_requests{namespace="{{ .Namespace }}", ingress="{{ .Name }}"} [{{ .Interval }}])) * 100 @@ -68,10 +68,10 @@ func (c *Observer) GetNginxSuccessRate(name string, namespace string, metric str const nginxRequestDurationQuery = ` sum(rate( -nginx_ingress_controller_ingress_upstream_latency_seconds_sum{kubernetes_namespace="{{ .Namespace }}", +nginx_ingress_controller_ingress_upstream_latency_seconds_sum{namespace="{{ .Namespace }}", ingress="{{ .Name }}"}[{{ .Interval }}])) / -sum(rate(nginx_ingress_controller_ingress_upstream_latency_seconds_count{kubernetes_namespace="{{ .Namespace }}", +sum(rate(nginx_ingress_controller_ingress_upstream_latency_seconds_count{namespace="{{ .Namespace }}", ingress="{{ .Name }}"}[{{ .Interval }}])) * 1000 ` From 8d0b54e0595e05982c4736e516ed208110b5aed5 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Thu, 9 May 2019 13:51:37 +0300 Subject: [PATCH 33/42] Add custom metrics to nginx docs --- artifacts/nginx/canary.yaml | 17 +++-- .../usage/nginx-progressive-delivery.md | 68 ++++++++++++++++++- 2 files changed, 80 insertions(+), 5 deletions(-) diff --git a/artifacts/nginx/canary.yaml b/artifacts/nginx/canary.yaml index bca0a709..6186889b 100644 --- a/artifacts/nginx/canary.yaml +++ b/artifacts/nginx/canary.yaml @@ -43,11 +43,20 @@ spec: # percentage (0-100) threshold: 99 interval: 1m - - name: request-duration - # maximum avg req duration - # milliseconds - threshold: 500 + - name: "latency" + threshold: 0.5 interval: 1m + query: | + histogram_quantile(0.99, + sum( + rate( + http_request_duration_seconds_bucket{ + kubernetes_namespace="test", + kubernetes_pod_name=~"podinfo-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)" + }[1m] + ) + ) by (le) + ) # external checks (optional) webhooks: - name: load-test diff --git a/docs/gitbook/usage/nginx-progressive-delivery.md b/docs/gitbook/usage/nginx-progressive-delivery.md index 2fa7a549..95260a48 100644 --- a/docs/gitbook/usage/nginx-progressive-delivery.md +++ b/docs/gitbook/usage/nginx-progressive-delivery.md @@ -14,7 +14,9 @@ Install NGINX with Helm: helm upgrade -i nginx-ingress stable/nginx-ingress \ --namespace ingress-nginx \ --set controller.stats.enabled=true \ ---set controller.metrics.enabled=true +--set controller.metrics.enabled=true \ +--set controller.podAnnotations."prometheus\.io/scrape"=true \ +--set controller.podAnnotations."prometheus\.io/port"=10254 ``` Install Flagger and the Prometheus add-on in the same namespace as NGINX: @@ -276,6 +278,70 @@ Events: Warning Synced 1m flagger Canary failed! Scaling down podinfo.test ``` +### Custom metrics + +The canary analysis can be extended with Prometheus queries. + +The demo app is instrumented with Prometheus so you can create a custom check that will use the HTTP request duration +histogram to validate the canary. + +Edit the canary analysis and add the following metric: + +```yaml + canaryAnalysis: + metrics: + - name: "latency" + threshold: 0.5 + interval: 1m + query: | + histogram_quantile(0.99, + sum( + rate( + http_request_duration_seconds_bucket{ + kubernetes_namespace="test", + kubernetes_pod_name=~"podinfo-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)" + }[1m] + ) + ) by (le) + ) +``` + +The threshold is set to 500ms so if the average request duration in the last minute +goes over half a second then the analysis will fail and the canary will not be promoted. + +Trigger a canary deployment by updating the container image: + +```bash +kubectl -n test set image deployment/podinfo \ +podinfod=quay.io/stefanprodan/podinfo:1.4.3 +``` + +Generate high response latency: + +```bash +watch curl http://app.exmaple.com/delay/2 +``` + +Watch Flagger logs: + +``` +kubectl -n nginx-ingress logs deployment/flagger -f | jq .msg + +Starting canary deployment for podinfo.test +Advance podinfo.test canary weight 5 +Advance podinfo.test canary weight 10 +Advance podinfo.test canary weight 15 +Halt podinfo.test advancement latency 1.20 > 0.5 +Halt podinfo.test advancement latency 1.45 > 0.5 +Halt podinfo.test advancement latency 1.60 > 0.5 +Halt podinfo.test advancement latency 1.69 > 0.5 +Halt podinfo.test advancement latency 1.70 > 0.5 +Rolling back podinfo.test failed checks threshold reached 5 +Canary failed! Scaling down podinfo.test +``` + +If you have Slack configured, Flagger will send a notification with the reason why the canary failed. + ### A/B Testing Besides weighted routing, Flagger can be configured to route traffic to the canary based on HTTP match conditions. From 2ff695ecfe471d1932c7b6434ae52b53e0007ff2 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Thu, 9 May 2019 14:00:15 +0300 Subject: [PATCH 34/42] Fix nginx metrics tests --- pkg/metrics/nginx_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/metrics/nginx_test.go b/pkg/metrics/nginx_test.go index e2a93c5f..00e9ef90 100644 --- a/pkg/metrics/nginx_test.go +++ b/pkg/metrics/nginx_test.go @@ -20,7 +20,7 @@ func Test_NginxSuccessRateQueryRender(t *testing.T) { t.Fatal(err) } - expected := `sum(rate(nginx_ingress_controller_requests{kubernetes_namespace="nginx",ingress="podinfo",status!~"5.*"}[1m])) / sum(rate(nginx_ingress_controller_requests{kubernetes_namespace="nginx",ingress="podinfo"}[1m])) * 100` + expected := `sum(rate(nginx_ingress_controller_requests{namespace="nginx",ingress="podinfo",status!~"5.*"}[1m])) / sum(rate(nginx_ingress_controller_requests{namespace="nginx",ingress="podinfo"}[1m])) * 100` if query != expected { t.Errorf("\nGot %s \nWanted %s", query, expected) @@ -43,7 +43,7 @@ func Test_NginxRequestDurationQueryRender(t *testing.T) { t.Fatal(err) } - expected := `sum(rate(nginx_ingress_controller_ingress_upstream_latency_seconds_sum{kubernetes_namespace="nginx",ingress="podinfo"}[1m])) /sum(rate(nginx_ingress_controller_ingress_upstream_latency_seconds_count{kubernetes_namespace="nginx",ingress="podinfo"}[1m])) * 1000` + expected := `sum(rate(nginx_ingress_controller_ingress_upstream_latency_seconds_sum{namespace="nginx",ingress="podinfo"}[1m])) /sum(rate(nginx_ingress_controller_ingress_upstream_latency_seconds_count{namespace="nginx",ingress="podinfo"}[1m])) * 1000` if query != expected { t.Errorf("\nGot %s \nWanted %s", query, expected) From 72014f736f2e9e2157e1d2193157950106245a79 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Thu, 9 May 2019 14:29:42 +0300 Subject: [PATCH 35/42] Release v0.13.1 --- CHANGELOG.md | 9 +++++++++ artifacts/flagger/deployment.yaml | 2 +- charts/flagger/Chart.yaml | 4 ++-- charts/flagger/values.yaml | 2 +- pkg/version/version.go | 2 +- 5 files changed, 14 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a5e9722..db761266 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ All notable changes to this project are documented in this file. +## 0.13.1 (2019-04-09) + +Fixes for custom metrics checks and NGINX Prometheus queries + +#### Fixes + +- Fix promql queries for custom checks and NGINX [#174](https://github.com/weaveworks/flagger/pull/174) + + ## 0.13.0 (2019-04-08) Adds support for [NGINX](https://docs.flagger.app/usage/nginx-progressive-delivery) ingress controller diff --git a/artifacts/flagger/deployment.yaml b/artifacts/flagger/deployment.yaml index 8d15cecd..833be438 100644 --- a/artifacts/flagger/deployment.yaml +++ b/artifacts/flagger/deployment.yaml @@ -22,7 +22,7 @@ spec: serviceAccountName: flagger containers: - name: flagger - image: weaveworks/flagger:0.13.0 + image: weaveworks/flagger:0.13.1 imagePullPolicy: IfNotPresent ports: - name: http diff --git a/charts/flagger/Chart.yaml b/charts/flagger/Chart.yaml index 80384c9a..c7b1ca67 100644 --- a/charts/flagger/Chart.yaml +++ b/charts/flagger/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 name: flagger -version: 0.13.0 -appVersion: 0.13.0 +version: 0.13.1 +appVersion: 0.13.1 kubeVersion: ">=1.11.0-0" engine: gotpl description: Flagger is a Kubernetes operator that automates the promotion of canary deployments using Istio, App Mesh or NGINX routing for traffic shifting and Prometheus metrics for canary analysis. diff --git a/charts/flagger/values.yaml b/charts/flagger/values.yaml index 85984032..6b6dafdd 100644 --- a/charts/flagger/values.yaml +++ b/charts/flagger/values.yaml @@ -2,7 +2,7 @@ image: repository: weaveworks/flagger - tag: 0.13.0 + tag: 0.13.1 pullPolicy: IfNotPresent metricsServer: "http://prometheus:9090" diff --git a/pkg/version/version.go b/pkg/version/version.go index f940057e..d9f5035d 100644 --- a/pkg/version/version.go +++ b/pkg/version/version.go @@ -1,4 +1,4 @@ package version -var VERSION = "0.13.0" +var VERSION = "0.13.1" var REVISION = "unknown" From 344bd45a0e08bcd864da405e8c994de24941282b Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Fri, 10 May 2019 10:24:35 +0300 Subject: [PATCH 36/42] Add nginx e2e tests --- .circleci/config.yml | 20 ++++- test/e2e-ingress.yaml | 17 ++++ test/e2e-nginx-build.sh | 24 ++++++ test/e2e-nginx-tests.sh | 185 ++++++++++++++++++++++++++++++++++++++++ test/e2e-nginx.sh | 29 +++++++ 5 files changed, 273 insertions(+), 2 deletions(-) create mode 100644 test/e2e-ingress.yaml create mode 100755 test/e2e-nginx-build.sh create mode 100755 test/e2e-nginx-tests.sh create mode 100755 test/e2e-nginx.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 7c254c55..f9164539 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,6 +1,6 @@ version: 2.1 jobs: - e2e-testing: + e2e-istio-testing: machine: true steps: - checkout @@ -18,11 +18,20 @@ jobs: - run: test/e2e-build.sh supergloo:test.supergloo-system - run: test/e2e-tests.sh canary + e2e-nginx-testing: + machine: true + steps: + - checkout + - run: test/e2e-kind.sh + - run: test/e2e-nginx.sh + - run: test/e2e-nginx-build.sh + - run: test/e2e-nginx-tests.sh + workflows: version: 2 build-and-test: jobs: - - e2e-testing: + - e2e-istio-testing: filters: branches: ignore: @@ -36,3 +45,10 @@ workflows: - /gh-pages.*/ - /docs-.*/ - /release-.*/ + - e2e-nginx-testing: + filters: + branches: + ignore: + - /gh-pages.*/ + - /docs-.*/ + - /release-.*/ \ No newline at end of file diff --git a/test/e2e-ingress.yaml b/test/e2e-ingress.yaml new file mode 100644 index 00000000..c5a6fa62 --- /dev/null +++ b/test/e2e-ingress.yaml @@ -0,0 +1,17 @@ +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: podinfo + namespace: test + labels: + app: podinfo + annotations: + kubernetes.io/ingress.class: "nginx" +spec: + rules: + - host: app.example.com + http: + paths: + - backend: + serviceName: podinfo + servicePort: 9898 diff --git a/test/e2e-nginx-build.sh b/test/e2e-nginx-build.sh new file mode 100755 index 00000000..6ae5449a --- /dev/null +++ b/test/e2e-nginx-build.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) +export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" + +echo '>>> Building Flagger' +cd ${REPO_ROOT} && docker build -t test/flagger:latest . -f Dockerfile + +echo '>>> Installing Flagger' +kind load docker-image test/flagger:latest + +echo '>>> Installing Flagger' +helm upgrade -i flagger ${REPO_ROOT}/charts/flagger \ +--wait \ +--namespace ingress-nginx \ +--set prometheus.install=true \ +--set meshProvider=nginx + +kubectl -n ingress-nginx set image deployment/flagger flagger=test/flagger:latest + +kubectl -n ingress-nginx rollout status deployment/flagger +kubectl -n ingress-nginx rollout status deployment/flagger-prometheus diff --git a/test/e2e-nginx-tests.sh b/test/e2e-nginx-tests.sh new file mode 100755 index 00000000..a4064e79 --- /dev/null +++ b/test/e2e-nginx-tests.sh @@ -0,0 +1,185 @@ +#!/usr/bin/env bash + +# This script runs e2e tests for Canary initialization, analysis and promotion +# Prerequisites: Kubernetes Kind, Helm and Istio + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) +export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" + +echo '>>> Creating test namespace' +kubectl create namespace test + +echo '>>> Installing load tester' +kubectl -n test apply -f ${REPO_ROOT}/artifacts/loadtester/ +kubectl -n test rollout status deployment/flagger-loadtester + +echo '>>> Initialising canary' +kubectl apply -f ${REPO_ROOT}/test/e2e-workload.yaml + +cat <>> Waiting for primary to be ready' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n ingress-nginx logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary initialization test passed' + +echo '>>> Triggering canary deployment' +kubectl -n test set image deployment/podinfo podinfod=quay.io/stefanprodan/podinfo:1.4.1 + +echo '>>> Waiting for canary promotion' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test describe deployment/podinfo-primary | grep '1.4.1' && ok=true || ok=false + sleep 10 + kubectl -n ingress-nginx logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n test describe deployment/podinfo + kubectl -n test describe deployment/podinfo-primary + kubectl -n ingress-nginx logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary promotion test passed' + +if [ "$1" = "canary" ]; then + exit 0 +fi + +cat <>> Triggering A/B testing' +kubectl -n test set image deployment/podinfo podinfod=quay.io/stefanprodan/podinfo:1.4.2 + +echo '>>> Waiting for A/B testing promotion' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test describe deployment/podinfo-primary | grep '1.4.2' && ok=true || ok=false + sleep 10 + kubectl -n ingress-nginx logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n test describe deployment/podinfo + kubectl -n test describe deployment/podinfo-primary + kubectl -n ingress-nginx logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ A/B testing promotion test passed' + +kubectl -n ingress-nginx logs deployment/flagger + +echo '✔ All tests passed' \ No newline at end of file diff --git a/test/e2e-nginx.sh b/test/e2e-nginx.sh new file mode 100755 index 00000000..3fa97bb7 --- /dev/null +++ b/test/e2e-nginx.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) +export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" + +echo ">>> Installing Helm" +curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get | bash + +echo '>>> Installing Tiller' +kubectl --namespace kube-system create sa tiller +kubectl create clusterrolebinding tiller-cluster-rule --clusterrole=cluster-admin --serviceaccount=kube-system:tiller +helm init --service-account tiller --upgrade --wait + +echo '>>> Installing NGINX Ingress' +helm upgrade -i nginx-ingress stable/nginx-ingress \ +--wait \ +--namespace ingress-nginx \ +--set controller.stats.enabled=true \ +--set controller.metrics.enabled=true \ +--set controller.podAnnotations."prometheus\.io/scrape"=true \ +--set controller.podAnnotations."prometheus\.io/port"=10254 \ +--set controller.service.type=NodePort + +kubectl -n ingress-nginx rollout status deployment/nginx-ingress-controller +kubectl -n ingress-nginx get all + + From bc84e1c154792423557a0bb21925b80250ad233e Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Fri, 10 May 2019 10:24:47 +0300 Subject: [PATCH 37/42] Fix typos --- artifacts/nginx/ingress.yaml | 2 +- docs/gitbook/usage/nginx-progressive-delivery.md | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/artifacts/nginx/ingress.yaml b/artifacts/nginx/ingress.yaml index 5cb6b826..c5a6fa62 100644 --- a/artifacts/nginx/ingress.yaml +++ b/artifacts/nginx/ingress.yaml @@ -9,7 +9,7 @@ metadata: kubernetes.io/ingress.class: "nginx" spec: rules: - - host: app.exmaple.com + - host: app.example.com http: paths: - backend: diff --git a/docs/gitbook/usage/nginx-progressive-delivery.md b/docs/gitbook/usage/nginx-progressive-delivery.md index 95260a48..4d660bc4 100644 --- a/docs/gitbook/usage/nginx-progressive-delivery.md +++ b/docs/gitbook/usage/nginx-progressive-delivery.md @@ -67,7 +67,7 @@ helm upgrade -i flagger-loadtester flagger/loadtester \ --namespace=test ``` -Create an ingress definition (replace `app.exmaple.com` with your own domain): +Create an ingress definition (replace `app.example.com` with your own domain): ```yaml apiVersion: extensions/v1beta1 @@ -81,7 +81,7 @@ metadata: kubernetes.io/ingress.class: "nginx" spec: rules: - - host: app.exmaple.com + - host: app.example.com http: paths: - backend: @@ -95,7 +95,7 @@ Save the above resource as podinfo-ingress.yaml and then apply it: kubectl apply -f ./podinfo-ingress.yaml ``` -Create a canary custom resource (replace `app.exmaple.com` with your own domain): +Create a canary custom resource (replace `app.example.com` with your own domain): ```yaml apiVersion: flagger.app/v1alpha3 @@ -249,7 +249,7 @@ podinfod=quay.io/stefanprodan/podinfo:1.4.2 Generate HTTP 500 errors: ```bash -watch curl http://app.exmaple.com/status/500 +watch curl http://app.example.com/status/500 ``` When the number of failed checks reaches the canary analysis threshold, the traffic is routed back to the primary, @@ -381,7 +381,7 @@ Edit the canary analysis, remove the max/step weight and add the match condition ``` The above configuration will run an analysis for ten minutes targeting users that have a `canary` cookie set to `always` or -those that call the service using the `X-Canary: always` header. +those that call the service using the `X-Canary: insider` header. Trigger a canary deployment by updating the container image: From cbe72f0aa2ce6f5af9350dea385d42796684678a Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Fri, 10 May 2019 10:29:09 +0300 Subject: [PATCH 38/42] Add ingress target to nginx e2e tests --- test/e2e-nginx-tests.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/e2e-nginx-tests.sh b/test/e2e-nginx-tests.sh index a4064e79..c26f3a4b 100755 --- a/test/e2e-nginx-tests.sh +++ b/test/e2e-nginx-tests.sh @@ -29,6 +29,10 @@ spec: apiVersion: apps/v1 kind: Deployment name: podinfo + ingressRef: + apiVersion: extensions/v1beta1 + kind: Ingress + name: podinfo progressDeadlineSeconds: 60 service: port: 9898 @@ -120,6 +124,10 @@ spec: apiVersion: apps/v1 kind: Deployment name: podinfo + ingressRef: + apiVersion: extensions/v1beta1 + kind: Ingress + name: podinfo progressDeadlineSeconds: 60 service: port: 9898 From e308678ed5bb8336c498f0bae6ad77cb25a7059e Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Fri, 10 May 2019 10:40:38 +0300 Subject: [PATCH 39/42] Deploy ingress for nginx e2e tests --- test/e2e-nginx-tests.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/test/e2e-nginx-tests.sh b/test/e2e-nginx-tests.sh index c26f3a4b..a53ee5ba 100755 --- a/test/e2e-nginx-tests.sh +++ b/test/e2e-nginx-tests.sh @@ -17,6 +17,7 @@ kubectl -n test rollout status deployment/flagger-loadtester echo '>>> Initialising canary' kubectl apply -f ${REPO_ROOT}/test/e2e-workload.yaml +kubectl apply -f ${REPO_ROOT}/test/e2e-ingress.yaml cat < Date: Fri, 10 May 2019 10:50:24 +0300 Subject: [PATCH 40/42] Document the nginx e2e tests --- test/Dockerfile.kind | 4 ---- test/README.md | 18 +++++++++++++++++- test/e2e-nginx-tests.sh | 2 +- 3 files changed, 18 insertions(+), 6 deletions(-) delete mode 100644 test/Dockerfile.kind diff --git a/test/Dockerfile.kind b/test/Dockerfile.kind deleted file mode 100644 index 28f5e083..00000000 --- a/test/Dockerfile.kind +++ /dev/null @@ -1,4 +0,0 @@ -FROM golang:1.11 - -RUN go get -u sigs.k8s.io/kind - diff --git a/test/README.md b/test/README.md index 3f8d586b..43c04b0a 100644 --- a/test/README.md +++ b/test/README.md @@ -2,7 +2,7 @@ The e2e testing infrastructure is powered by CircleCI and [Kubernetes Kind](https://github.com/kubernetes-sigs/kind). -CircleCI e2e workflow: +### CircleCI e2e Istio workflow * install latest stable kubectl [e2e-kind.sh](e2e-kind.sh) * install Kubernetes Kind [e2e-kind.sh](e2e-kind.sh) @@ -21,4 +21,20 @@ CircleCI e2e workflow: * test the canary analysis and promotion using weighted traffic and the load testing webhook [e2e-tests.sh](e2e-tests.sh) * test the A/B testing analysis and promotion using cookies filters and pre/post rollout webhooks [e2e-tests.sh](e2e-tests.sh) +### CircleCI e2e NGINX ingress workflow +* install latest stable kubectl [e2e-kind.sh](e2e-kind.sh) +* install Kubernetes Kind [e2e-kind.sh](e2e-kind.sh) +* create local Kubernetes cluster with kind [e2e-kind.sh](e2e-kind.sh) +* install latest stable Helm CLI [e2e-nginx.sh](e2e-istio.sh) +* deploy Tiller on the local cluster [e2e-nginx.sh](e2e-istio.sh) +* install NGINX ingress with Helm [e2e-nginx.sh](e2e-istio.sh) +* build Flagger container image [e2e-nginx-build.sh](e2e-build.sh) +* load Flagger image onto the local cluster [e2e-nginx-build.sh](e2e-build.sh) +* install Flagger and Prometheus in the ingress-nginx namespace [e2e-nginx-build.sh](e2e-build.sh) +* create a test namespace [e2e-nginx-tests.sh](e2e-tests.sh) +* deploy the load tester in the test namespace [e2e-nginx-tests.sh](e2e-tests.sh) +* deploy the demo workload (podinfo) and ingress in the test namespace [e2e-nginx-tests.sh](e2e-tests.sh) +* test the canary initialization [e2e-nginx-tests.sh](e2e-tests.sh) +* test the canary analysis and promotion using weighted traffic and the load testing webhook [e2e-nginx-tests.sh](e2e-tests.sh) +* test the A/B testing analysis and promotion using header filters and pre/post rollout webhooks [e2e-nginx-tests.sh](e2e-tests.sh) diff --git a/test/e2e-nginx-tests.sh b/test/e2e-nginx-tests.sh index a53ee5ba..dac550f7 100755 --- a/test/e2e-nginx-tests.sh +++ b/test/e2e-nginx-tests.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # This script runs e2e tests for Canary initialization, analysis and promotion -# Prerequisites: Kubernetes Kind, Helm and Istio +# Prerequisites: Kubernetes Kind, Helm and NGINX ingress controller set -o errexit From eadce34d6f75edfe8c82e4627171417982997546 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Fri, 10 May 2019 11:39:52 +0300 Subject: [PATCH 41/42] Add ingress router unit tests --- pkg/router/ingress_test.go | 90 ++++++++++++++++++++++++++++++++++++++ pkg/router/router_test.go | 79 ++++++++++++++++++++++++++++++++- 2 files changed, 167 insertions(+), 2 deletions(-) create mode 100644 pkg/router/ingress_test.go diff --git a/pkg/router/ingress_test.go b/pkg/router/ingress_test.go new file mode 100644 index 00000000..1fda7cd5 --- /dev/null +++ b/pkg/router/ingress_test.go @@ -0,0 +1,90 @@ +package router + +import ( + "fmt" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "testing" +) + +func TestIngressRouter_Reconcile(t *testing.T) { + mocks := setupfakeClients() + router := &IngressRouter{ + logger: mocks.logger, + kubeClient: mocks.kubeClient, + } + + err := router.Reconcile(mocks.ingressCanary) + if err != nil { + t.Fatal(err.Error()) + } + + canaryAn := "nginx.ingress.kubernetes.io/canary" + canaryWeightAn := "nginx.ingress.kubernetes.io/canary-weight" + + canaryName := fmt.Sprintf("%s-canary", mocks.ingressCanary.Spec.IngressRef.Name) + inCanary, err := router.kubeClient.ExtensionsV1beta1().Ingresses("default").Get(canaryName, metav1.GetOptions{}) + if err != nil { + t.Fatal(err.Error()) + } + + if _, ok := inCanary.Annotations[canaryAn]; !ok { + t.Errorf("Canary annotation missing") + } + + // test initialisation + if inCanary.Annotations[canaryAn] != "false" { + t.Errorf("Got canary annotation %v wanted false", inCanary.Annotations[canaryAn]) + } + + if inCanary.Annotations[canaryWeightAn] != "0" { + t.Errorf("Got canary weight annotation %v wanted 0", inCanary.Annotations[canaryWeightAn]) + } +} + +func TestIngressRouter_GetSetRoutes(t *testing.T) { + mocks := setupfakeClients() + router := &IngressRouter{ + logger: mocks.logger, + kubeClient: mocks.kubeClient, + } + + err := router.Reconcile(mocks.ingressCanary) + if err != nil { + t.Fatal(err.Error()) + } + + p, c, err := router.GetRoutes(mocks.ingressCanary) + if err != nil { + t.Fatal(err.Error()) + } + + p = 50 + c = 50 + + err = router.SetRoutes(mocks.ingressCanary, p, c) + if err != nil { + t.Fatal(err.Error()) + } + + canaryAn := "nginx.ingress.kubernetes.io/canary" + canaryWeightAn := "nginx.ingress.kubernetes.io/canary-weight" + + canaryName := fmt.Sprintf("%s-canary", mocks.ingressCanary.Spec.IngressRef.Name) + inCanary, err := router.kubeClient.ExtensionsV1beta1().Ingresses("default").Get(canaryName, metav1.GetOptions{}) + if err != nil { + t.Fatal(err.Error()) + } + + if _, ok := inCanary.Annotations[canaryAn]; !ok { + t.Errorf("Canary annotation missing") + } + + // test initialisation + if inCanary.Annotations[canaryAn] != "true" { + t.Errorf("Got canary annotation %v wanted true", inCanary.Annotations[canaryAn]) + } + + if inCanary.Annotations[canaryWeightAn] != "50" { + t.Errorf("Got canary weight annotation %v wanted 50", inCanary.Annotations[canaryWeightAn]) + } +} diff --git a/pkg/router/router_test.go b/pkg/router/router_test.go index 32c5f770..701cae0c 100644 --- a/pkg/router/router_test.go +++ b/pkg/router/router_test.go @@ -11,7 +11,9 @@ import ( appsv1 "k8s.io/api/apps/v1" hpav1 "k8s.io/api/autoscaling/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/api/extensions/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" ) @@ -20,6 +22,7 @@ type fakeClients struct { canary *v1alpha3.Canary abtest *v1alpha3.Canary appmeshCanary *v1alpha3.Canary + ingressCanary *v1alpha3.Canary kubeClient kubernetes.Interface meshClient clientset.Interface flaggerClient clientset.Interface @@ -30,9 +33,10 @@ func setupfakeClients() fakeClients { canary := newMockCanary() abtest := newMockABTest() appmeshCanary := newMockCanaryAppMesh() - flaggerClient := fakeFlagger.NewSimpleClientset(canary, abtest, appmeshCanary) + ingressCanary := newMockCanaryIngress() + flaggerClient := fakeFlagger.NewSimpleClientset(canary, abtest, appmeshCanary, ingressCanary) - kubeClient := fake.NewSimpleClientset(newMockDeployment(), newMockABTestDeployment()) + kubeClient := fake.NewSimpleClientset(newMockDeployment(), newMockABTestDeployment(), newMockIngress()) meshClient := fakeFlagger.NewSimpleClientset() logger, _ := logger.NewLogger("debug") @@ -41,6 +45,7 @@ func setupfakeClients() fakeClients { canary: canary, abtest: abtest, appmeshCanary: appmeshCanary, + ingressCanary: ingressCanary, kubeClient: kubeClient, meshClient: meshClient, flaggerClient: flaggerClient, @@ -266,3 +271,73 @@ func newMockABTestDeployment() *appsv1.Deployment { return d } + +func newMockCanaryIngress() *v1alpha3.Canary { + cd := &v1alpha3.Canary{ + TypeMeta: metav1.TypeMeta{APIVersion: v1alpha3.SchemeGroupVersion.String()}, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "nginx", + }, + Spec: v1alpha3.CanarySpec{ + TargetRef: hpav1.CrossVersionObjectReference{ + Name: "podinfo", + APIVersion: "apps/v1", + Kind: "Deployment", + }, + IngressRef: &hpav1.CrossVersionObjectReference{ + Name: "podinfo", + APIVersion: "extensions/v1beta1", + Kind: "Ingress", + }, + Service: v1alpha3.CanaryService{ + Port: 9898, + }, CanaryAnalysis: v1alpha3.CanaryAnalysis{ + Threshold: 10, + StepWeight: 10, + MaxWeight: 50, + Metrics: []v1alpha3.CanaryMetric{ + { + Name: "request-success-rate", + Threshold: 99, + Interval: "1m", + }, + }, + }, + }, + } + return cd +} + +func newMockIngress() *v1beta1.Ingress { + return &v1beta1.Ingress{ + TypeMeta: metav1.TypeMeta{APIVersion: v1beta1.SchemeGroupVersion.String()}, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "podinfo", + Annotations: map[string]string{ + "kubernetes.io/ingress.class": "nginx", + }, + }, + Spec: v1beta1.IngressSpec{ + Rules: []v1beta1.IngressRule{ + { + Host: "app.example.com", + IngressRuleValue: v1beta1.IngressRuleValue{ + HTTP: &v1beta1.HTTPIngressRuleValue{ + Paths: []v1beta1.HTTPIngressPath{ + { + Path: "/", + Backend: v1beta1.IngressBackend{ + ServiceName: "podinfo", + ServicePort: intstr.FromInt(9898), + }, + }, + }, + }, + }, + }, + }, + }, + } +} From 752eceed4bb8a0d0eee9e211a1dbb27190c671e9 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Fri, 10 May 2019 11:53:12 +0300 Subject: [PATCH 42/42] Add tests for ingress weight changes --- pkg/router/ingress_test.go | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/pkg/router/ingress_test.go b/pkg/router/ingress_test.go index 1fda7cd5..7d2679e6 100644 --- a/pkg/router/ingress_test.go +++ b/pkg/router/ingress_test.go @@ -79,7 +79,7 @@ func TestIngressRouter_GetSetRoutes(t *testing.T) { t.Errorf("Canary annotation missing") } - // test initialisation + // test rollout if inCanary.Annotations[canaryAn] != "true" { t.Errorf("Got canary annotation %v wanted true", inCanary.Annotations[canaryAn]) } @@ -87,4 +87,26 @@ func TestIngressRouter_GetSetRoutes(t *testing.T) { if inCanary.Annotations[canaryWeightAn] != "50" { t.Errorf("Got canary weight annotation %v wanted 50", inCanary.Annotations[canaryWeightAn]) } + + p = 100 + c = 0 + + err = router.SetRoutes(mocks.ingressCanary, p, c) + if err != nil { + t.Fatal(err.Error()) + } + + inCanary, err = router.kubeClient.ExtensionsV1beta1().Ingresses("default").Get(canaryName, metav1.GetOptions{}) + if err != nil { + t.Fatal(err.Error()) + } + + // test promotion + if inCanary.Annotations[canaryAn] != "false" { + t.Errorf("Got canary annotation %v wanted false", inCanary.Annotations[canaryAn]) + } + + if inCanary.Annotations[canaryWeightAn] != "0" { + t.Errorf("Got canary weight annotation %v wanted 0", inCanary.Annotations[canaryWeightAn]) + } }