From 350efb2bfe7f4555f3e0276a63312c9f08447e60 Mon Sep 17 00:00:00 2001 From: Yuval Kohavi Date: Tue, 23 Apr 2019 07:47:50 -0400 Subject: [PATCH 01/10] gloo upstream group support --- pkg/router/factory.go | 6 ++ pkg/router/gloo.go | 181 ++++++++++++++++++++++++++++++++++++++++ pkg/router/gloo_test.go | 135 ++++++++++++++++++++++++++++++ 3 files changed, 322 insertions(+) create mode 100644 pkg/router/gloo.go create mode 100644 pkg/router/gloo_test.go diff --git a/pkg/router/factory.go b/pkg/router/factory.go index 9d8f66ec..5b78243c 100644 --- a/pkg/router/factory.go +++ b/pkg/router/factory.go @@ -57,6 +57,12 @@ func (factory *Factory) MeshRouter(provider string) Interface { panic("failed creating supergloo client") } return supergloo + case strings.HasPrefix(provider, "gloo"): + gloo, err := NewGlooRouter(context.TODO(), provider, factory.flaggerClient, factory.logger, factory.kubeConfig) + if err != nil { + panic("failed creating gloo client") + } + return gloo default: return &IstioRouter{ logger: factory.logger, diff --git a/pkg/router/gloo.go b/pkg/router/gloo.go new file mode 100644 index 00000000..eb1d22ed --- /dev/null +++ b/pkg/router/gloo.go @@ -0,0 +1,181 @@ +package router + +import ( + "context" + "fmt" + "strings" + + solokitclients "github.com/solo-io/solo-kit/pkg/api/v1/clients" + "github.com/solo-io/solo-kit/pkg/api/v1/clients/factory" + "github.com/solo-io/solo-kit/pkg/api/v1/clients/kube" + crdv1 "github.com/solo-io/solo-kit/pkg/api/v1/clients/kube/crd/solo.io/v1" + solokitcore "github.com/solo-io/solo-kit/pkg/api/v1/resources/core" + solokiterror "github.com/solo-io/solo-kit/pkg/errors" + + gloov1 "github.com/solo-io/gloo/projects/gloo/pkg/api/v1" + flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3" + clientset "github.com/weaveworks/flagger/pkg/client/clientset/versioned" + "go.uber.org/zap" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/rest" +) + +// GlooRouter is managing Istio virtual services +type GlooRouter struct { + ugClient gloov1.UpstreamGroupClient + logger *zap.SugaredLogger + upstreamDiscoveryNs string +} + +func NewGlooRouter(ctx context.Context, provider string, flaggerClient clientset.Interface, logger *zap.SugaredLogger, cfg *rest.Config) (*GlooRouter, error) { + // TODO if cfg is nil use memory client instead? + sharedCache := kube.NewKubeCache(ctx) + upstreamGroupClient, err := gloov1.NewUpstreamGroupClient(&factory.KubeResourceClientFactory{ + Crd: gloov1.UpstreamGroupCrd, + Cfg: cfg, + SharedCache: sharedCache, + SkipCrdCreation: true, + }) + if err != nil { + // this should never happen. + return nil, fmt.Errorf("creating UpstreamGroup client %v", err) + } + if err := upstreamGroupClient.Register(); err != nil { + return nil, err + } + upstreamDiscoveryNs := "" + if strings.HasPrefix(provider, "gloo:") { + upstreamDiscoveryNs = strings.TrimPrefix(provider, "gloo:") + } + + return NewGlooRouterWithClient(ctx, upstreamGroupClient, upstreamDiscoveryNs, logger), nil +} + +func NewGlooRouterWithClient(ctx context.Context, routingRuleClient gloov1.UpstreamGroupClient, upstreamDiscoveryNs string, logger *zap.SugaredLogger) *GlooRouter { + + if upstreamDiscoveryNs == "" { + upstreamDiscoveryNs = "gloo-system" + } + return &GlooRouter{ugClient: routingRuleClient, logger: logger, upstreamDiscoveryNs: upstreamDiscoveryNs} +} + +// Reconcile creates or updates the Istio virtual service +func (gr *GlooRouter) Reconcile(canary *flaggerv1.Canary) error { + return gr.SetRoutes(canary, 100, 0) +} + +// GetRoutes returns the destinations weight for primary and canary +func (gr *GlooRouter) GetRoutes(canary *flaggerv1.Canary) ( + primaryWeight int, + canaryWeight int, + err error, +) { + targetName := canary.Spec.TargetRef.Name + var ug *gloov1.UpstreamGroup + ug, err = gr.ugClient.Read(canary.Namespace, targetName, solokitclients.ReadOpts{}) + if err != nil { + return + } + + dests := ug.GetDestinations() + for _, dest := range dests { + if dest.GetDestination().GetUpstream().Name == upstreamName(canary.Namespace, fmt.Sprintf("%s-primary", targetName), canary.Spec.Service.Port) { + primaryWeight = int(dest.Weight) + } + if dest.GetDestination().GetUpstream().Name == upstreamName(canary.Namespace, fmt.Sprintf("%s-canary", targetName), canary.Spec.Service.Port) { + canaryWeight = int(dest.Weight) + } + } + + if primaryWeight == 0 && canaryWeight == 0 { + err = fmt.Errorf("RoutingRule %s.%s does not contain routes for %s-primary and %s-canary", + targetName, canary.Namespace, targetName, targetName) + } + + return +} + +// SetRoutes updates the destinations weight for primary and canary +func (gr *GlooRouter) SetRoutes( + canary *flaggerv1.Canary, + primaryWeight int, + canaryWeight int, +) error { + targetName := canary.Spec.TargetRef.Name + + destinations := []*gloov1.WeightedDestination{} + if primaryWeight != 0 { + destinations = append(destinations, &gloov1.WeightedDestination{ + Destination: &gloov1.Destination{ + Upstream: solokitcore.ResourceRef{ + Name: upstreamName(canary.Namespace, fmt.Sprintf("%s-primary", targetName), canary.Spec.Service.Port), + Namespace: gr.upstreamDiscoveryNs, + }, + }, + Weight: uint32(primaryWeight), + }) + } + + if canaryWeight != 0 { + destinations = append(destinations, &gloov1.WeightedDestination{ + Destination: &gloov1.Destination{ + Upstream: solokitcore.ResourceRef{ + Name: upstreamName(canary.Namespace, fmt.Sprintf("%s-canary", targetName), canary.Spec.Service.Port), + Namespace: gr.upstreamDiscoveryNs, + }, + }, + Weight: uint32(canaryWeight), + }) + } + + if len(destinations) == 0 { + return fmt.Errorf("RoutingRule %s.%s update failed: no valid weights", targetName, canary.Namespace) + } + + upstreamGroup := &gloov1.UpstreamGroup{ + Metadata: solokitcore.Metadata{ + Name: canary.Spec.TargetRef.Name, + Namespace: canary.Namespace, + }, + Destinations: destinations, + } + + return gr.writeUpstreamGroupRuleForCanary(canary, upstreamGroup) +} + +func (gr *GlooRouter) writeUpstreamGroupRuleForCanary(canary *flaggerv1.Canary, ug *gloov1.UpstreamGroup) error { + targetName := canary.Spec.TargetRef.Name + + if oldUg, err := gr.ugClient.Read(ug.Metadata.Namespace, ug.Metadata.Name, solokitclients.ReadOpts{}); err != nil { + // ignore not exist errors.. + if !solokiterror.IsNotExist(err) { + return fmt.Errorf("RoutingRule %s.%s read failed: %v", targetName, canary.Namespace, err) + } + } else { + ug.Metadata.ResourceVersion = oldUg.Metadata.ResourceVersion + // if the old and the new one are equal, no need to do anything. + oldUg.Status = solokitcore.Status{} + if oldUg.Equal(ug) { + return nil + } + } + + kubeWriteOpts := &kube.KubeWriteOpts{ + PreWriteCallback: func(r *crdv1.Resource) { + r.ObjectMeta.OwnerReferences = []metav1.OwnerReference{ + *metav1.NewControllerRef(canary, schema.GroupVersionKind{ + Group: flaggerv1.SchemeGroupVersion.Group, + Version: flaggerv1.SchemeGroupVersion.Version, + Kind: flaggerv1.CanaryKind, + }), + } + }, + } + writeOpts := solokitclients.WriteOpts{OverwriteExisting: true, StorageWriteOpts: kubeWriteOpts} + _, err := gr.ugClient.Write(ug, writeOpts) + if err != nil { + return fmt.Errorf("UpstreamGroup %s.%s update failed: %v", targetName, canary.Namespace, err) + } + return nil +} diff --git a/pkg/router/gloo_test.go b/pkg/router/gloo_test.go new file mode 100644 index 00000000..bcce4d20 --- /dev/null +++ b/pkg/router/gloo_test.go @@ -0,0 +1,135 @@ +package router + +import ( + "context" + "fmt" + "testing" + + gloov1 "github.com/solo-io/gloo/projects/gloo/pkg/api/v1" + solokitclients "github.com/solo-io/solo-kit/pkg/api/v1/clients" + "github.com/solo-io/solo-kit/pkg/api/v1/clients/factory" + solokitmemory "github.com/solo-io/solo-kit/pkg/api/v1/clients/memory" +) + +func TestGlooRouter_Sync(t *testing.T) { + mocks := setupfakeClients() + + upstreamGroupClient, err := gloov1.NewUpstreamGroupClient(&factory.MemoryResourceClientFactory{ + Cache: solokitmemory.NewInMemoryResourceCache(), + }) + if err != nil { + t.Fatal(err.Error()) + } + if err := upstreamGroupClient.Register(); err != nil { + t.Fatal(err.Error()) + } + router := NewGlooRouterWithClient(context.TODO(), upstreamGroupClient, "gloo-system", mocks.logger) + err = router.Reconcile(mocks.canary) + if err != nil { + t.Fatal(err.Error()) + } + + // test insert + ug, err := upstreamGroupClient.Read("default", "podinfo", solokitclients.ReadOpts{}) + if err != nil { + t.Fatal(err.Error()) + } + dests := ug.GetDestinations() + if len(dests) != 1 { + t.Errorf("Got RoutingRule Destinations %v wanted %v", len(dests), 1) + } + +} + +func TestGlooRouter_SetRoutes(t *testing.T) { + + mocks := setupfakeClients() + + upstreamGroupClient, err := gloov1.NewUpstreamGroupClient(&factory.MemoryResourceClientFactory{ + Cache: solokitmemory.NewInMemoryResourceCache(), + }) + if err != nil { + t.Fatal(err.Error()) + } + if err := upstreamGroupClient.Register(); err != nil { + t.Fatal(err.Error()) + } + router := NewGlooRouterWithClient(context.TODO(), upstreamGroupClient, "gloo-system", mocks.logger) + + err = router.Reconcile(mocks.canary) + if err != nil { + t.Fatal(err.Error()) + } + + p, c, err := router.GetRoutes(mocks.canary) + if err != nil { + t.Fatal(err.Error()) + } + + p = 50 + c = 50 + + err = router.SetRoutes(mocks.canary, p, c) + if err != nil { + t.Fatal(err.Error()) + } + + ug, err := upstreamGroupClient.Read("default", "podinfo", solokitclients.ReadOpts{}) + if err != nil { + t.Fatal(err.Error()) + } + + var pRoute *gloov1.WeightedDestination + var cRoute *gloov1.WeightedDestination + targetName := mocks.canary.Spec.TargetRef.Name + + for _, dest := range ug.GetDestinations() { + if dest.GetDestination().GetUpstream().Name == upstreamName(mocks.canary.Namespace, fmt.Sprintf("%s-primary", targetName), mocks.canary.Spec.Service.Port) { + pRoute = dest + } + if dest.GetDestination().GetUpstream().Name == upstreamName(mocks.canary.Namespace, fmt.Sprintf("%s-canary", targetName), mocks.canary.Spec.Service.Port) { + cRoute = dest + } + } + + if pRoute.Weight != uint32(p) { + t.Errorf("Got primary weight %v wanted %v", pRoute.Weight, p) + } + + if cRoute.Weight != uint32(c) { + t.Errorf("Got canary weight %v wanted %v", cRoute.Weight, c) + } + +} + +func TestGlooRouter_GetRoutes(t *testing.T) { + mocks := setupfakeClients() + + upstreamGroupClient, err := gloov1.NewUpstreamGroupClient(&factory.MemoryResourceClientFactory{ + Cache: solokitmemory.NewInMemoryResourceCache(), + }) + if err != nil { + t.Fatal(err.Error()) + } + if err := upstreamGroupClient.Register(); err != nil { + t.Fatal(err.Error()) + } + router := NewGlooRouterWithClient(context.TODO(), upstreamGroupClient, "gloo-system", mocks.logger) + err = router.Reconcile(mocks.canary) + if err != nil { + t.Fatal(err.Error()) + } + + p, c, err := router.GetRoutes(mocks.canary) + if err != nil { + t.Fatal(err.Error()) + } + + if p != 100 { + t.Errorf("Got primary weight %v wanted %v", p, 100) + } + + if c != 0 { + t.Errorf("Got canary weight %v wanted %v", c, 0) + } +} From 87e9dfe3d308bb8566832267d31c03d2e2bcd371 Mon Sep 17 00:00:00 2001 From: Yuval Kohavi Date: Fri, 10 May 2019 19:16:16 -0400 Subject: [PATCH 02/10] e2e test --- .circleci/config.yml | 9 +++++ test/e2e-gloo-build.sh | 27 +++++++++++++++ test/e2e-gloo-tests.sh | 79 ++++++++++++++++++++++++++++++++++++++++++ test/e2e-gloo.sh | 25 +++++++++++++ 4 files changed, 140 insertions(+) create mode 100755 test/e2e-gloo-build.sh create mode 100755 test/e2e-gloo-tests.sh create mode 100755 test/e2e-gloo.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index f9164539..73c5e87f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -18,6 +18,15 @@ jobs: - run: test/e2e-build.sh supergloo:test.supergloo-system - run: test/e2e-tests.sh canary + e2e-gloo-testing: + machine: true + steps: + - checkout + - run: test/e2e-kind.sh + - run: test/e2e-gloo.sh + - run: test/e2e-gloo-build.sh + - run: test/e2e-gloo-tests.sh + e2e-nginx-testing: machine: true steps: diff --git a/test/e2e-gloo-build.sh b/test/e2e-gloo-build.sh new file mode 100755 index 00000000..8799082d --- /dev/null +++ b/test/e2e-gloo-build.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) +export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" + +echo '>>> Building Flagger' +cd ${REPO_ROOT} && docker build -t test/flagger:latest . -f Dockerfile + +echo '>>> Installing Flagger' +kind load docker-image test/flagger:latest + +echo '>>> Installing Flagger' +helm upgrade -i flagger ${REPO_ROOT}/charts/flagger \ +--wait \ +--namespace gloo-system \ +--set prometheus.install=true \ +--set meshProvider=gloo + +# Give flagger permissions for gloo objects +kubectl create clusterrolebinding flagger-gloo --clusterrole=gloo-role-gateway --serviceaccount=gloo-system:flagger + +kubectl -n gloo-system set image deployment/flagger flagger=test/flagger:latest + +kubectl -n gloo-system rollout status deployment/flagger +kubectl -n gloo-system rollout status deployment/flagger-prometheus diff --git a/test/e2e-gloo-tests.sh b/test/e2e-gloo-tests.sh new file mode 100755 index 00000000..33ea0341 --- /dev/null +++ b/test/e2e-gloo-tests.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash + +# This script runs e2e tests for Canary initialization, analysis and promotion +# Prerequisites: Kubernetes Kind, Helm and NGINX ingress controller + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) +export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" + +echo '>>> Creating test namespace' +kubectl create namespace test + +echo ">>> Downloading Gloo CLI" +curl -SsL https://github.com/solo-io/gloo/releases/download/v0.13.25/glooctl-linux-amd64 > glooctl +chmod +x glooctl + +echo '>>> Installing load tester' +kubectl -n test apply -f ${REPO_ROOT}/artifacts/loadtester/ +kubectl -n test rollout status deployment/flagger-loadtester + +echo '>>> Initialising canary' +kubectl apply -f ${REPO_ROOT}/test/e2e-workload.yaml +./glooctl add route --path-prefix / --upstream-group-name podinfo --upstream-group-namespace test + +cat <>> Waiting for primary to be ready' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n gloo-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary initialization test passed' diff --git a/test/e2e-gloo.sh b/test/e2e-gloo.sh new file mode 100755 index 00000000..430cb983 --- /dev/null +++ b/test/e2e-gloo.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) +export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" + +echo ">>> Installing Helm" +# curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get | bash + +echo '>>> Installing Tiller' +kubectl --namespace kube-system create sa tiller +kubectl create clusterrolebinding tiller-cluster-rule --clusterrole=cluster-admin --serviceaccount=kube-system:tiller +helm init --service-account tiller --upgrade --wait +helm repo add gloo https://storage.googleapis.com/solo-public-helm + +echo '>>> Installing Gloo' +helm upgrade -i gloo gloo/gloo --version 0.13.25 \ +--wait \ +--namespace gloo-system \ +--set gatewayProxies.gateway-proxy.service.type=NodePort + +kubectl -n gloo-system rollout status deployment/gloo +kubectl -n gloo-system rollout status deployment/gateway-proxy +kubectl -n gloo-system get all From 9c1bcc08bb7cda1d0e351139556143c81fcc6b39 Mon Sep 17 00:00:00 2001 From: Yuval Kohavi Date: Fri, 10 May 2019 19:21:08 -0400 Subject: [PATCH 03/10] float -> percent --- test/e2e-gloo-tests.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/test/e2e-gloo-tests.sh b/test/e2e-gloo-tests.sh index 33ea0341..ca4f0505 100755 --- a/test/e2e-gloo-tests.sh +++ b/test/e2e-gloo-tests.sh @@ -51,6 +51,7 @@ spec: envoy_response_code!~"5.*"}[1m])) / sum(rate(envoy_cluster_upstream_rq{kubernetes_namespace="gloo-system", gloo="gateway-proxy"}[1m])) + * 100 webhooks: - name: load-test url: http://flagger-loadtester.test/ From a6c0f08fccef72227e5c89a083917f47dc2dcb43 Mon Sep 17 00:00:00 2001 From: Yuval Kohavi Date: Fri, 10 May 2019 19:44:46 -0400 Subject: [PATCH 04/10] add gloo to circle --- .circleci/config.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 73c5e87f..cff705a3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -55,6 +55,13 @@ workflows: - /docs-.*/ - /release-.*/ - e2e-nginx-testing: + filters: + branches: + ignore: + - /gh-pages.*/ + - /docs-.*/ + - /release-.*/ + - e2e-gloo-testing: filters: branches: ignore: From 7aca9468ac8f3e15cda6d810d80aece86fbe0066 Mon Sep 17 00:00:00 2001 From: Yuval Kohavi Date: Fri, 10 May 2019 19:48:22 -0400 Subject: [PATCH 05/10] re-enable helm --- test/e2e-gloo.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e-gloo.sh b/test/e2e-gloo.sh index 430cb983..7788d897 100755 --- a/test/e2e-gloo.sh +++ b/test/e2e-gloo.sh @@ -6,7 +6,7 @@ REPO_ROOT=$(git rev-parse --show-toplevel) export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" echo ">>> Installing Helm" -# curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get | bash +curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get | bash echo '>>> Installing Tiller' kubectl --namespace kube-system create sa tiller From 0fbf4dcdb2ad832b1f24c85dea6207994404b21f Mon Sep 17 00:00:00 2001 From: Yuval Kohavi Date: Fri, 10 May 2019 20:16:21 -0400 Subject: [PATCH 06/10] add canary promotion --- test/e2e-gloo-tests.sh | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/test/e2e-gloo-tests.sh b/test/e2e-gloo-tests.sh index ca4f0505..9984a429 100755 --- a/test/e2e-gloo-tests.sh +++ b/test/e2e-gloo-tests.sh @@ -78,3 +78,26 @@ until ${ok}; do done echo '✔ Canary initialization test passed' + +echo '>>> Triggering canary deployment' +kubectl -n test set image deployment/podinfo podinfod=quay.io/stefanprodan/podinfo:1.4.1 + +echo '>>> Waiting for canary promotion' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test describe deployment/podinfo-primary | grep '1.4.1' && ok=true || ok=false + sleep 10 + kubectl -n gloo-system logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n test describe deployment/podinfo + kubectl -n test describe deployment/podinfo-primary + kubectl -n gloo-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary promotion test passed' From 677b9d91975428260154217c78f84290fd3bb3fc Mon Sep 17 00:00:00 2001 From: Yuval Kohavi Date: Tue, 14 May 2019 17:48:13 -0400 Subject: [PATCH 07/10] gloo metrics --- pkg/metrics/factory.go | 5 +++ pkg/metrics/gloo.go | 72 ++++++++++++++++++++++++++++++++++++++ pkg/metrics/gloo_test.go | 74 ++++++++++++++++++++++++++++++++++++++++ pkg/router/gloo.go | 58 ++++++++++++++++--------------- test/e2e-gloo-tests.sh | 10 ++---- test/e2e-gloo.sh | 2 +- 6 files changed, 185 insertions(+), 36 deletions(-) create mode 100644 pkg/metrics/gloo.go create mode 100644 pkg/metrics/gloo_test.go diff --git a/pkg/metrics/factory.go b/pkg/metrics/factory.go index b351a44d..986ffb42 100644 --- a/pkg/metrics/factory.go +++ b/pkg/metrics/factory.go @@ -1,6 +1,7 @@ package metrics import ( + "strings" "time" ) @@ -31,6 +32,10 @@ func (factory Factory) Observer() Interface { return &NginxObserver{ client: factory.Client, } + case strings.HasPrefix(factory.MeshProvider, "gloo"): + return &GlooObserver{ + client: factory.Client, + } case factory.MeshProvider == "smi:linkerd": return &LinkerdObserver{ client: factory.Client, diff --git a/pkg/metrics/gloo.go b/pkg/metrics/gloo.go new file mode 100644 index 00000000..0acd361e --- /dev/null +++ b/pkg/metrics/gloo.go @@ -0,0 +1,72 @@ +package metrics + +import ( + "time" +) + +//envoy_cluster_name="test-podinfo-primary-9898_gloo-system" + +var glooQueries = map[string]string{ + "request-success-rate": ` + sum( + rate( + envoy_cluster_upstream_rq{ + envoy_cluster_name=~"{{ .Namespace }}-{{ .Name }}-[0-9a-zA-Z-]+_[0-9a-zA-Z-]+", + envoy_response_code!~"5.*" + }[{{ .Interval }}] + ) + ) + / + sum( + rate( + envoy_cluster_upstream_rq{ + envoy_cluster_name=~"{{ .Namespace }}-{{ .Name }}-[0-9a-zA-Z-]+_[0-9a-zA-Z-]+", + }[{{ .Interval }}] + ) + ) + * 100`, + "request-duration": ` + histogram_quantile( + 0.99, + sum( + rate( + envoy_cluster_upstream_rq_time_bucket{ + envoy_cluster_name=~"{{ .Namespace }}-{{ .Name }}-[0-9a-zA-Z-]+_[0-9a-zA-Z-]+", + }[{{ .Interval }}] + ) + ) by (le) + )`, +} + +type GlooObserver struct { + client *PrometheusClient +} + +func (ob *GlooObserver) GetRequestSuccessRate(name string, namespace string, interval string) (float64, error) { + query, err := ob.client.RenderQuery(name, namespace, interval, glooQueries["request-success-rate"]) + if err != nil { + return 0, err + } + + value, err := ob.client.RunQuery(query) + if err != nil { + return 0, err + } + + return value, nil +} + +func (ob *GlooObserver) GetRequestDuration(name string, namespace string, interval string) (time.Duration, error) { + query, err := ob.client.RenderQuery(name, namespace, interval, glooQueries["request-duration"]) + if err != nil { + return 0, err + } + + value, err := ob.client.RunQuery(query) + if err != nil { + return 0, err + } + + ms := time.Duration(int64(value)) * time.Millisecond + return ms, nil +} diff --git a/pkg/metrics/gloo_test.go b/pkg/metrics/gloo_test.go new file mode 100644 index 00000000..7a69f918 --- /dev/null +++ b/pkg/metrics/gloo_test.go @@ -0,0 +1,74 @@ +package metrics + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" +) + +func TestGlooObserver_GetRequestSuccessRate(t *testing.T) { + expected := `sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"default-podinfo-[0-9a-zA-Z-]+_[0-9a-zA-Z-]+",envoy_response_code!~"5.*"}[1m]))/sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"default-podinfo-[0-9a-zA-Z-]+_[0-9a-zA-Z-]+",}[1m]))*100` + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + promql := r.URL.Query()["query"][0] + if promql != expected { + t.Errorf("\nGot %s \nWanted %s", promql, expected) + } + + json := `{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1,"100"]}]}}` + w.Write([]byte(json)) + })) + defer ts.Close() + + client, err := NewPrometheusClient(ts.URL, time.Second) + if err != nil { + t.Fatal(err) + } + + observer := &GlooObserver{ + client: client, + } + + val, err := observer.GetRequestSuccessRate("podinfo", "default", "1m") + if err != nil { + t.Fatal(err.Error()) + } + + if val != 100 { + t.Errorf("Got %v wanted %v", val, 100) + } +} + +func TestGlooObserver_GetRequestDuration(t *testing.T) { + expected := `histogram_quantile(0.99,sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"default-podinfo-[0-9a-zA-Z-]+_[0-9a-zA-Z-]+",}[1m]))by(le))` + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + promql := r.URL.Query()["query"][0] + if promql != expected { + t.Errorf("\nGot %s \nWanted %s", promql, expected) + } + + json := `{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1,"100"]}]}}` + w.Write([]byte(json)) + })) + defer ts.Close() + + client, err := NewPrometheusClient(ts.URL, time.Second) + if err != nil { + t.Fatal(err) + } + + observer := &GlooObserver{ + client: client, + } + + val, err := observer.GetRequestDuration("podinfo", "default", "1m") + if err != nil { + t.Fatal(err.Error()) + } + + if val != 100*time.Millisecond { + t.Errorf("Got %v wanted %v", val, 100*time.Millisecond) + } +} diff --git a/pkg/router/gloo.go b/pkg/router/gloo.go index eb1d22ed..09bcd743 100644 --- a/pkg/router/gloo.go +++ b/pkg/router/gloo.go @@ -62,7 +62,15 @@ func NewGlooRouterWithClient(ctx context.Context, routingRuleClient gloov1.Upstr // Reconcile creates or updates the Istio virtual service func (gr *GlooRouter) Reconcile(canary *flaggerv1.Canary) error { - return gr.SetRoutes(canary, 100, 0) + // do we have routes already? + if _, _, err := gr.GetRoutes(canary); err == nil { + // we have routes, no need to do anything else + return nil + } else if solokiterror.IsNotExist(err) { + return gr.SetRoutes(canary, 100, 0) + } else { + return err + } } // GetRoutes returns the destinations weight for primary and canary @@ -104,35 +112,31 @@ func (gr *GlooRouter) SetRoutes( ) error { targetName := canary.Spec.TargetRef.Name - destinations := []*gloov1.WeightedDestination{} - if primaryWeight != 0 { - destinations = append(destinations, &gloov1.WeightedDestination{ - Destination: &gloov1.Destination{ - Upstream: solokitcore.ResourceRef{ - Name: upstreamName(canary.Namespace, fmt.Sprintf("%s-primary", targetName), canary.Spec.Service.Port), - Namespace: gr.upstreamDiscoveryNs, - }, - }, - Weight: uint32(primaryWeight), - }) - } - - if canaryWeight != 0 { - destinations = append(destinations, &gloov1.WeightedDestination{ - Destination: &gloov1.Destination{ - Upstream: solokitcore.ResourceRef{ - Name: upstreamName(canary.Namespace, fmt.Sprintf("%s-canary", targetName), canary.Spec.Service.Port), - Namespace: gr.upstreamDiscoveryNs, - }, - }, - Weight: uint32(canaryWeight), - }) - } - - if len(destinations) == 0 { + if primaryWeight == 0 && canaryWeight == 0 { return fmt.Errorf("RoutingRule %s.%s update failed: no valid weights", targetName, canary.Namespace) } + destinations := []*gloov1.WeightedDestination{} + destinations = append(destinations, &gloov1.WeightedDestination{ + Destination: &gloov1.Destination{ + Upstream: solokitcore.ResourceRef{ + Name: upstreamName(canary.Namespace, fmt.Sprintf("%s-primary", targetName), canary.Spec.Service.Port), + Namespace: gr.upstreamDiscoveryNs, + }, + }, + Weight: uint32(primaryWeight), + }) + + destinations = append(destinations, &gloov1.WeightedDestination{ + Destination: &gloov1.Destination{ + Upstream: solokitcore.ResourceRef{ + Name: upstreamName(canary.Namespace, fmt.Sprintf("%s-canary", targetName), canary.Spec.Service.Port), + Namespace: gr.upstreamDiscoveryNs, + }, + }, + Weight: uint32(canaryWeight), + }) + upstreamGroup := &gloov1.UpstreamGroup{ Metadata: solokitcore.Metadata{ Name: canary.Spec.TargetRef.Name, diff --git a/test/e2e-gloo-tests.sh b/test/e2e-gloo-tests.sh index 9984a429..ab113633 100755 --- a/test/e2e-gloo-tests.sh +++ b/test/e2e-gloo-tests.sh @@ -12,7 +12,7 @@ echo '>>> Creating test namespace' kubectl create namespace test echo ">>> Downloading Gloo CLI" -curl -SsL https://github.com/solo-io/gloo/releases/download/v0.13.25/glooctl-linux-amd64 > glooctl +curl -SsL https://github.com/solo-io/gloo/releases/download/v0.13.27/glooctl-linux-amd64 > glooctl chmod +x glooctl echo '>>> Installing load tester' @@ -43,15 +43,9 @@ spec: maxWeight: 30 stepWeight: 10 metrics: - - name: envoy-success-rate + - name: request-success-rate threshold: 99 interval: 1m - query: | - sum(rate(envoy_cluster_upstream_rq{kubernetes_namespace="gloo-system", gloo="gateway-proxy", - envoy_response_code!~"5.*"}[1m])) - / - sum(rate(envoy_cluster_upstream_rq{kubernetes_namespace="gloo-system", gloo="gateway-proxy"}[1m])) - * 100 webhooks: - name: load-test url: http://flagger-loadtester.test/ diff --git a/test/e2e-gloo.sh b/test/e2e-gloo.sh index 7788d897..5bdefef7 100755 --- a/test/e2e-gloo.sh +++ b/test/e2e-gloo.sh @@ -15,7 +15,7 @@ helm init --service-account tiller --upgrade --wait helm repo add gloo https://storage.googleapis.com/solo-public-helm echo '>>> Installing Gloo' -helm upgrade -i gloo gloo/gloo --version 0.13.25 \ +helm upgrade -i gloo gloo/gloo --version 0.13.27 \ --wait \ --namespace gloo-system \ --set gatewayProxies.gateway-proxy.service.type=NodePort From 6a66a87a447c95f7acdfdbe3677012834f1c9caa Mon Sep 17 00:00:00 2001 From: Yuval Kohavi Date: Thu, 16 May 2019 07:28:22 -0400 Subject: [PATCH 08/10] PR updates --- pkg/metrics/gloo.go | 6 +++--- pkg/router/gloo.go | 1 + test/e2e-gloo-tests.sh | 2 +- test/e2e-gloo.sh | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/pkg/metrics/gloo.go b/pkg/metrics/gloo.go index 0acd361e..67afd661 100644 --- a/pkg/metrics/gloo.go +++ b/pkg/metrics/gloo.go @@ -11,7 +11,7 @@ var glooQueries = map[string]string{ sum( rate( envoy_cluster_upstream_rq{ - envoy_cluster_name=~"{{ .Namespace }}-{{ .Name }}-[0-9a-zA-Z-]+_[0-9a-zA-Z-]+", + envoy_cluster_name=~"{{ .Namespace }}-{{ .Name }}-canary-[0-9a-zA-Z-]+_[0-9a-zA-Z-]+", envoy_response_code!~"5.*" }[{{ .Interval }}] ) @@ -20,7 +20,7 @@ var glooQueries = map[string]string{ sum( rate( envoy_cluster_upstream_rq{ - envoy_cluster_name=~"{{ .Namespace }}-{{ .Name }}-[0-9a-zA-Z-]+_[0-9a-zA-Z-]+", + envoy_cluster_name=~"{{ .Namespace }}-{{ .Name }}-canary-[0-9a-zA-Z-]+_[0-9a-zA-Z-]+", }[{{ .Interval }}] ) ) @@ -31,7 +31,7 @@ var glooQueries = map[string]string{ sum( rate( envoy_cluster_upstream_rq_time_bucket{ - envoy_cluster_name=~"{{ .Namespace }}-{{ .Name }}-[0-9a-zA-Z-]+_[0-9a-zA-Z-]+", + envoy_cluster_name=~"{{ .Namespace }}-{{ .Name }}-canary-[0-9a-zA-Z-]+_[0-9a-zA-Z-]+", }[{{ .Interval }}] ) ) by (le) diff --git a/pkg/router/gloo.go b/pkg/router/gloo.go index 09bcd743..62303e20 100644 --- a/pkg/router/gloo.go +++ b/pkg/router/gloo.go @@ -181,5 +181,6 @@ func (gr *GlooRouter) writeUpstreamGroupRuleForCanary(canary *flaggerv1.Canary, if err != nil { return fmt.Errorf("UpstreamGroup %s.%s update failed: %v", targetName, canary.Namespace, err) } + gr.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)).Infof("UpstreamGroup %s updated", ug.Metadata.Name) return nil } diff --git a/test/e2e-gloo-tests.sh b/test/e2e-gloo-tests.sh index ab113633..8414427e 100755 --- a/test/e2e-gloo-tests.sh +++ b/test/e2e-gloo-tests.sh @@ -12,7 +12,7 @@ echo '>>> Creating test namespace' kubectl create namespace test echo ">>> Downloading Gloo CLI" -curl -SsL https://github.com/solo-io/gloo/releases/download/v0.13.27/glooctl-linux-amd64 > glooctl +curl -SsL https://github.com/solo-io/gloo/releases/download/v0.13.29/glooctl-linux-amd64 > glooctl chmod +x glooctl echo '>>> Installing load tester' diff --git a/test/e2e-gloo.sh b/test/e2e-gloo.sh index 5bdefef7..b4467b19 100755 --- a/test/e2e-gloo.sh +++ b/test/e2e-gloo.sh @@ -15,7 +15,7 @@ helm init --service-account tiller --upgrade --wait helm repo add gloo https://storage.googleapis.com/solo-public-helm echo '>>> Installing Gloo' -helm upgrade -i gloo gloo/gloo --version 0.13.27 \ +helm upgrade -i gloo gloo/gloo --version 0.13.29 \ --wait \ --namespace gloo-system \ --set gatewayProxies.gateway-proxy.service.type=NodePort From eb0331f2bfda213c6a84a858ef660c2facade0dc Mon Sep 17 00:00:00 2001 From: Yuval Kohavi Date: Thu, 16 May 2019 12:48:03 -0400 Subject: [PATCH 09/10] fix tests --- pkg/metrics/gloo_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/metrics/gloo_test.go b/pkg/metrics/gloo_test.go index 7a69f918..fd6ddeba 100644 --- a/pkg/metrics/gloo_test.go +++ b/pkg/metrics/gloo_test.go @@ -8,7 +8,7 @@ import ( ) func TestGlooObserver_GetRequestSuccessRate(t *testing.T) { - expected := `sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"default-podinfo-[0-9a-zA-Z-]+_[0-9a-zA-Z-]+",envoy_response_code!~"5.*"}[1m]))/sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"default-podinfo-[0-9a-zA-Z-]+_[0-9a-zA-Z-]+",}[1m]))*100` + expected := `sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"default-podinfo-canary-[0-9a-zA-Z-]+_[0-9a-zA-Z-]+",envoy_response_code!~"5.*"}[1m]))/sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"default-podinfo-canary-[0-9a-zA-Z-]+_[0-9a-zA-Z-]+",}[1m]))*100` ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { promql := r.URL.Query()["query"][0] @@ -41,7 +41,7 @@ func TestGlooObserver_GetRequestSuccessRate(t *testing.T) { } func TestGlooObserver_GetRequestDuration(t *testing.T) { - expected := `histogram_quantile(0.99,sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"default-podinfo-[0-9a-zA-Z-]+_[0-9a-zA-Z-]+",}[1m]))by(le))` + expected := `histogram_quantile(0.99,sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"default-podinfo-canary-[0-9a-zA-Z-]+_[0-9a-zA-Z-]+",}[1m]))by(le))` ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { promql := r.URL.Query()["query"][0] From 52d93ddda2a9be2f9207c7f572e904173d3642e5 Mon Sep 17 00:00:00 2001 From: Yuval Kohavi Date: Thu, 16 May 2019 13:08:53 -0400 Subject: [PATCH 10/10] fix router tests --- pkg/router/gloo_test.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/pkg/router/gloo_test.go b/pkg/router/gloo_test.go index bcce4d20..408be347 100644 --- a/pkg/router/gloo_test.go +++ b/pkg/router/gloo_test.go @@ -35,8 +35,15 @@ func TestGlooRouter_Sync(t *testing.T) { t.Fatal(err.Error()) } dests := ug.GetDestinations() - if len(dests) != 1 { - t.Errorf("Got RoutingRule Destinations %v wanted %v", len(dests), 1) + if len(dests) != 2 { + t.Errorf("Got Destinations %v wanted %v", len(dests), 2) + } + + if dests[0].Weight != 100 { + t.Errorf("Primary weight should is %v wanted 100", dests[0].Weight) + } + if dests[1].Weight != 0 { + t.Errorf("Canary weight should is %v wanted 0", dests[0].Weight) } }