diff --git a/.circleci/config.yml b/.circleci/config.yml index 9d79a90b..07411ff4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -27,6 +27,15 @@ jobs: - run: test/e2e-build.sh supergloo:test.supergloo-system - run: test/e2e-tests.sh canary + e2e-gloo-testing: + machine: true + steps: + - checkout + - run: test/e2e-kind.sh + - run: test/e2e-gloo.sh + - run: test/e2e-gloo-build.sh + - run: test/e2e-gloo-tests.sh + e2e-nginx-testing: machine: true steps: @@ -62,6 +71,13 @@ workflows: - /docs-.*/ - /release-.*/ - e2e-nginx-testing: + filters: + branches: + ignore: + - /gh-pages.*/ + - /docs-.*/ + - /release-.*/ + - e2e-gloo-testing: filters: branches: ignore: diff --git a/pkg/metrics/factory.go b/pkg/metrics/factory.go index b351a44d..986ffb42 100644 --- a/pkg/metrics/factory.go +++ b/pkg/metrics/factory.go @@ -1,6 +1,7 @@ package metrics import ( + "strings" "time" ) @@ -31,6 +32,10 @@ func (factory Factory) Observer() Interface { return &NginxObserver{ client: factory.Client, } + case strings.HasPrefix(factory.MeshProvider, "gloo"): + return &GlooObserver{ + client: factory.Client, + } case factory.MeshProvider == "smi:linkerd": return &LinkerdObserver{ client: factory.Client, diff --git a/pkg/metrics/gloo.go b/pkg/metrics/gloo.go new file mode 100644 index 00000000..67afd661 --- /dev/null +++ b/pkg/metrics/gloo.go @@ -0,0 +1,72 @@ +package metrics + +import ( + "time" +) + +//envoy_cluster_name="test-podinfo-primary-9898_gloo-system" + +var glooQueries = map[string]string{ + "request-success-rate": ` + sum( + rate( + envoy_cluster_upstream_rq{ + envoy_cluster_name=~"{{ .Namespace }}-{{ .Name }}-canary-[0-9a-zA-Z-]+_[0-9a-zA-Z-]+", + envoy_response_code!~"5.*" + }[{{ .Interval }}] + ) + ) + / + sum( + rate( + envoy_cluster_upstream_rq{ + envoy_cluster_name=~"{{ .Namespace }}-{{ .Name }}-canary-[0-9a-zA-Z-]+_[0-9a-zA-Z-]+", + }[{{ .Interval }}] + ) + ) + * 100`, + "request-duration": ` + histogram_quantile( + 0.99, + sum( + rate( + envoy_cluster_upstream_rq_time_bucket{ + envoy_cluster_name=~"{{ .Namespace }}-{{ .Name }}-canary-[0-9a-zA-Z-]+_[0-9a-zA-Z-]+", + }[{{ .Interval }}] + ) + ) by (le) + )`, +} + +type GlooObserver struct { + client *PrometheusClient +} + +func (ob *GlooObserver) GetRequestSuccessRate(name string, namespace string, interval string) (float64, error) { + query, err := ob.client.RenderQuery(name, namespace, interval, glooQueries["request-success-rate"]) + if err != nil { + return 0, err + } + + value, err := ob.client.RunQuery(query) + if err != nil { + return 0, err + } + + return value, nil +} + +func (ob *GlooObserver) GetRequestDuration(name string, namespace string, interval string) (time.Duration, error) { + query, err := ob.client.RenderQuery(name, namespace, interval, glooQueries["request-duration"]) + if err != nil { + return 0, err + } + + value, err := ob.client.RunQuery(query) + if err != nil { + return 0, err + } + + ms := time.Duration(int64(value)) * time.Millisecond + return ms, nil +} diff --git a/pkg/metrics/gloo_test.go b/pkg/metrics/gloo_test.go new file mode 100644 index 00000000..fd6ddeba --- /dev/null +++ b/pkg/metrics/gloo_test.go @@ -0,0 +1,74 @@ +package metrics + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" +) + +func TestGlooObserver_GetRequestSuccessRate(t *testing.T) { + expected := `sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"default-podinfo-canary-[0-9a-zA-Z-]+_[0-9a-zA-Z-]+",envoy_response_code!~"5.*"}[1m]))/sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"default-podinfo-canary-[0-9a-zA-Z-]+_[0-9a-zA-Z-]+",}[1m]))*100` + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + promql := r.URL.Query()["query"][0] + if promql != expected { + t.Errorf("\nGot %s \nWanted %s", promql, expected) + } + + json := `{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1,"100"]}]}}` + w.Write([]byte(json)) + })) + defer ts.Close() + + client, err := NewPrometheusClient(ts.URL, time.Second) + if err != nil { + t.Fatal(err) + } + + observer := &GlooObserver{ + client: client, + } + + val, err := observer.GetRequestSuccessRate("podinfo", "default", "1m") + if err != nil { + t.Fatal(err.Error()) + } + + if val != 100 { + t.Errorf("Got %v wanted %v", val, 100) + } +} + +func TestGlooObserver_GetRequestDuration(t *testing.T) { + expected := `histogram_quantile(0.99,sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"default-podinfo-canary-[0-9a-zA-Z-]+_[0-9a-zA-Z-]+",}[1m]))by(le))` + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + promql := r.URL.Query()["query"][0] + if promql != expected { + t.Errorf("\nGot %s \nWanted %s", promql, expected) + } + + json := `{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1,"100"]}]}}` + w.Write([]byte(json)) + })) + defer ts.Close() + + client, err := NewPrometheusClient(ts.URL, time.Second) + if err != nil { + t.Fatal(err) + } + + observer := &GlooObserver{ + client: client, + } + + val, err := observer.GetRequestDuration("podinfo", "default", "1m") + if err != nil { + t.Fatal(err.Error()) + } + + if val != 100*time.Millisecond { + t.Errorf("Got %v wanted %v", val, 100*time.Millisecond) + } +} diff --git a/pkg/router/factory.go b/pkg/router/factory.go index 09ed940b..212b5460 100644 --- a/pkg/router/factory.go +++ b/pkg/router/factory.go @@ -71,6 +71,12 @@ func (factory *Factory) MeshRouter(provider string) Interface { panic("failed creating supergloo client") } return supergloo + case strings.HasPrefix(provider, "gloo"): + gloo, err := NewGlooRouter(context.TODO(), provider, factory.flaggerClient, factory.logger, factory.kubeConfig) + if err != nil { + panic("failed creating gloo client") + } + return gloo default: return &IstioRouter{ logger: factory.logger, diff --git a/pkg/router/gloo.go b/pkg/router/gloo.go new file mode 100644 index 00000000..62303e20 --- /dev/null +++ b/pkg/router/gloo.go @@ -0,0 +1,186 @@ +package router + +import ( + "context" + "fmt" + "strings" + + solokitclients "github.com/solo-io/solo-kit/pkg/api/v1/clients" + "github.com/solo-io/solo-kit/pkg/api/v1/clients/factory" + "github.com/solo-io/solo-kit/pkg/api/v1/clients/kube" + crdv1 "github.com/solo-io/solo-kit/pkg/api/v1/clients/kube/crd/solo.io/v1" + solokitcore "github.com/solo-io/solo-kit/pkg/api/v1/resources/core" + solokiterror "github.com/solo-io/solo-kit/pkg/errors" + + gloov1 "github.com/solo-io/gloo/projects/gloo/pkg/api/v1" + flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3" + clientset "github.com/weaveworks/flagger/pkg/client/clientset/versioned" + "go.uber.org/zap" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/rest" +) + +// GlooRouter is managing Istio virtual services +type GlooRouter struct { + ugClient gloov1.UpstreamGroupClient + logger *zap.SugaredLogger + upstreamDiscoveryNs string +} + +func NewGlooRouter(ctx context.Context, provider string, flaggerClient clientset.Interface, logger *zap.SugaredLogger, cfg *rest.Config) (*GlooRouter, error) { + // TODO if cfg is nil use memory client instead? + sharedCache := kube.NewKubeCache(ctx) + upstreamGroupClient, err := gloov1.NewUpstreamGroupClient(&factory.KubeResourceClientFactory{ + Crd: gloov1.UpstreamGroupCrd, + Cfg: cfg, + SharedCache: sharedCache, + SkipCrdCreation: true, + }) + if err != nil { + // this should never happen. + return nil, fmt.Errorf("creating UpstreamGroup client %v", err) + } + if err := upstreamGroupClient.Register(); err != nil { + return nil, err + } + upstreamDiscoveryNs := "" + if strings.HasPrefix(provider, "gloo:") { + upstreamDiscoveryNs = strings.TrimPrefix(provider, "gloo:") + } + + return NewGlooRouterWithClient(ctx, upstreamGroupClient, upstreamDiscoveryNs, logger), nil +} + +func NewGlooRouterWithClient(ctx context.Context, routingRuleClient gloov1.UpstreamGroupClient, upstreamDiscoveryNs string, logger *zap.SugaredLogger) *GlooRouter { + + if upstreamDiscoveryNs == "" { + upstreamDiscoveryNs = "gloo-system" + } + return &GlooRouter{ugClient: routingRuleClient, logger: logger, upstreamDiscoveryNs: upstreamDiscoveryNs} +} + +// Reconcile creates or updates the Istio virtual service +func (gr *GlooRouter) Reconcile(canary *flaggerv1.Canary) error { + // do we have routes already? + if _, _, err := gr.GetRoutes(canary); err == nil { + // we have routes, no need to do anything else + return nil + } else if solokiterror.IsNotExist(err) { + return gr.SetRoutes(canary, 100, 0) + } else { + return err + } +} + +// GetRoutes returns the destinations weight for primary and canary +func (gr *GlooRouter) GetRoutes(canary *flaggerv1.Canary) ( + primaryWeight int, + canaryWeight int, + err error, +) { + targetName := canary.Spec.TargetRef.Name + var ug *gloov1.UpstreamGroup + ug, err = gr.ugClient.Read(canary.Namespace, targetName, solokitclients.ReadOpts{}) + if err != nil { + return + } + + dests := ug.GetDestinations() + for _, dest := range dests { + if dest.GetDestination().GetUpstream().Name == upstreamName(canary.Namespace, fmt.Sprintf("%s-primary", targetName), canary.Spec.Service.Port) { + primaryWeight = int(dest.Weight) + } + if dest.GetDestination().GetUpstream().Name == upstreamName(canary.Namespace, fmt.Sprintf("%s-canary", targetName), canary.Spec.Service.Port) { + canaryWeight = int(dest.Weight) + } + } + + if primaryWeight == 0 && canaryWeight == 0 { + err = fmt.Errorf("RoutingRule %s.%s does not contain routes for %s-primary and %s-canary", + targetName, canary.Namespace, targetName, targetName) + } + + return +} + +// SetRoutes updates the destinations weight for primary and canary +func (gr *GlooRouter) SetRoutes( + canary *flaggerv1.Canary, + primaryWeight int, + canaryWeight int, +) error { + targetName := canary.Spec.TargetRef.Name + + if primaryWeight == 0 && canaryWeight == 0 { + return fmt.Errorf("RoutingRule %s.%s update failed: no valid weights", targetName, canary.Namespace) + } + + destinations := []*gloov1.WeightedDestination{} + destinations = append(destinations, &gloov1.WeightedDestination{ + Destination: &gloov1.Destination{ + Upstream: solokitcore.ResourceRef{ + Name: upstreamName(canary.Namespace, fmt.Sprintf("%s-primary", targetName), canary.Spec.Service.Port), + Namespace: gr.upstreamDiscoveryNs, + }, + }, + Weight: uint32(primaryWeight), + }) + + destinations = append(destinations, &gloov1.WeightedDestination{ + Destination: &gloov1.Destination{ + Upstream: solokitcore.ResourceRef{ + Name: upstreamName(canary.Namespace, fmt.Sprintf("%s-canary", targetName), canary.Spec.Service.Port), + Namespace: gr.upstreamDiscoveryNs, + }, + }, + Weight: uint32(canaryWeight), + }) + + upstreamGroup := &gloov1.UpstreamGroup{ + Metadata: solokitcore.Metadata{ + Name: canary.Spec.TargetRef.Name, + Namespace: canary.Namespace, + }, + Destinations: destinations, + } + + return gr.writeUpstreamGroupRuleForCanary(canary, upstreamGroup) +} + +func (gr *GlooRouter) writeUpstreamGroupRuleForCanary(canary *flaggerv1.Canary, ug *gloov1.UpstreamGroup) error { + targetName := canary.Spec.TargetRef.Name + + if oldUg, err := gr.ugClient.Read(ug.Metadata.Namespace, ug.Metadata.Name, solokitclients.ReadOpts{}); err != nil { + // ignore not exist errors.. + if !solokiterror.IsNotExist(err) { + return fmt.Errorf("RoutingRule %s.%s read failed: %v", targetName, canary.Namespace, err) + } + } else { + ug.Metadata.ResourceVersion = oldUg.Metadata.ResourceVersion + // if the old and the new one are equal, no need to do anything. + oldUg.Status = solokitcore.Status{} + if oldUg.Equal(ug) { + return nil + } + } + + kubeWriteOpts := &kube.KubeWriteOpts{ + PreWriteCallback: func(r *crdv1.Resource) { + r.ObjectMeta.OwnerReferences = []metav1.OwnerReference{ + *metav1.NewControllerRef(canary, schema.GroupVersionKind{ + Group: flaggerv1.SchemeGroupVersion.Group, + Version: flaggerv1.SchemeGroupVersion.Version, + Kind: flaggerv1.CanaryKind, + }), + } + }, + } + writeOpts := solokitclients.WriteOpts{OverwriteExisting: true, StorageWriteOpts: kubeWriteOpts} + _, err := gr.ugClient.Write(ug, writeOpts) + if err != nil { + return fmt.Errorf("UpstreamGroup %s.%s update failed: %v", targetName, canary.Namespace, err) + } + gr.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)).Infof("UpstreamGroup %s updated", ug.Metadata.Name) + return nil +} diff --git a/pkg/router/gloo_test.go b/pkg/router/gloo_test.go new file mode 100644 index 00000000..408be347 --- /dev/null +++ b/pkg/router/gloo_test.go @@ -0,0 +1,142 @@ +package router + +import ( + "context" + "fmt" + "testing" + + gloov1 "github.com/solo-io/gloo/projects/gloo/pkg/api/v1" + solokitclients "github.com/solo-io/solo-kit/pkg/api/v1/clients" + "github.com/solo-io/solo-kit/pkg/api/v1/clients/factory" + solokitmemory "github.com/solo-io/solo-kit/pkg/api/v1/clients/memory" +) + +func TestGlooRouter_Sync(t *testing.T) { + mocks := setupfakeClients() + + upstreamGroupClient, err := gloov1.NewUpstreamGroupClient(&factory.MemoryResourceClientFactory{ + Cache: solokitmemory.NewInMemoryResourceCache(), + }) + if err != nil { + t.Fatal(err.Error()) + } + if err := upstreamGroupClient.Register(); err != nil { + t.Fatal(err.Error()) + } + router := NewGlooRouterWithClient(context.TODO(), upstreamGroupClient, "gloo-system", mocks.logger) + err = router.Reconcile(mocks.canary) + if err != nil { + t.Fatal(err.Error()) + } + + // test insert + ug, err := upstreamGroupClient.Read("default", "podinfo", solokitclients.ReadOpts{}) + if err != nil { + t.Fatal(err.Error()) + } + dests := ug.GetDestinations() + if len(dests) != 2 { + t.Errorf("Got Destinations %v wanted %v", len(dests), 2) + } + + if dests[0].Weight != 100 { + t.Errorf("Primary weight should is %v wanted 100", dests[0].Weight) + } + if dests[1].Weight != 0 { + t.Errorf("Canary weight should is %v wanted 0", dests[0].Weight) + } + +} + +func TestGlooRouter_SetRoutes(t *testing.T) { + + mocks := setupfakeClients() + + upstreamGroupClient, err := gloov1.NewUpstreamGroupClient(&factory.MemoryResourceClientFactory{ + Cache: solokitmemory.NewInMemoryResourceCache(), + }) + if err != nil { + t.Fatal(err.Error()) + } + if err := upstreamGroupClient.Register(); err != nil { + t.Fatal(err.Error()) + } + router := NewGlooRouterWithClient(context.TODO(), upstreamGroupClient, "gloo-system", mocks.logger) + + err = router.Reconcile(mocks.canary) + if err != nil { + t.Fatal(err.Error()) + } + + p, c, err := router.GetRoutes(mocks.canary) + if err != nil { + t.Fatal(err.Error()) + } + + p = 50 + c = 50 + + err = router.SetRoutes(mocks.canary, p, c) + if err != nil { + t.Fatal(err.Error()) + } + + ug, err := upstreamGroupClient.Read("default", "podinfo", solokitclients.ReadOpts{}) + if err != nil { + t.Fatal(err.Error()) + } + + var pRoute *gloov1.WeightedDestination + var cRoute *gloov1.WeightedDestination + targetName := mocks.canary.Spec.TargetRef.Name + + for _, dest := range ug.GetDestinations() { + if dest.GetDestination().GetUpstream().Name == upstreamName(mocks.canary.Namespace, fmt.Sprintf("%s-primary", targetName), mocks.canary.Spec.Service.Port) { + pRoute = dest + } + if dest.GetDestination().GetUpstream().Name == upstreamName(mocks.canary.Namespace, fmt.Sprintf("%s-canary", targetName), mocks.canary.Spec.Service.Port) { + cRoute = dest + } + } + + if pRoute.Weight != uint32(p) { + t.Errorf("Got primary weight %v wanted %v", pRoute.Weight, p) + } + + if cRoute.Weight != uint32(c) { + t.Errorf("Got canary weight %v wanted %v", cRoute.Weight, c) + } + +} + +func TestGlooRouter_GetRoutes(t *testing.T) { + mocks := setupfakeClients() + + upstreamGroupClient, err := gloov1.NewUpstreamGroupClient(&factory.MemoryResourceClientFactory{ + Cache: solokitmemory.NewInMemoryResourceCache(), + }) + if err != nil { + t.Fatal(err.Error()) + } + if err := upstreamGroupClient.Register(); err != nil { + t.Fatal(err.Error()) + } + router := NewGlooRouterWithClient(context.TODO(), upstreamGroupClient, "gloo-system", mocks.logger) + err = router.Reconcile(mocks.canary) + if err != nil { + t.Fatal(err.Error()) + } + + p, c, err := router.GetRoutes(mocks.canary) + if err != nil { + t.Fatal(err.Error()) + } + + if p != 100 { + t.Errorf("Got primary weight %v wanted %v", p, 100) + } + + if c != 0 { + t.Errorf("Got canary weight %v wanted %v", c, 0) + } +} diff --git a/test/e2e-gloo-build.sh b/test/e2e-gloo-build.sh new file mode 100755 index 00000000..8799082d --- /dev/null +++ b/test/e2e-gloo-build.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) +export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" + +echo '>>> Building Flagger' +cd ${REPO_ROOT} && docker build -t test/flagger:latest . -f Dockerfile + +echo '>>> Installing Flagger' +kind load docker-image test/flagger:latest + +echo '>>> Installing Flagger' +helm upgrade -i flagger ${REPO_ROOT}/charts/flagger \ +--wait \ +--namespace gloo-system \ +--set prometheus.install=true \ +--set meshProvider=gloo + +# Give flagger permissions for gloo objects +kubectl create clusterrolebinding flagger-gloo --clusterrole=gloo-role-gateway --serviceaccount=gloo-system:flagger + +kubectl -n gloo-system set image deployment/flagger flagger=test/flagger:latest + +kubectl -n gloo-system rollout status deployment/flagger +kubectl -n gloo-system rollout status deployment/flagger-prometheus diff --git a/test/e2e-gloo-tests.sh b/test/e2e-gloo-tests.sh new file mode 100755 index 00000000..8414427e --- /dev/null +++ b/test/e2e-gloo-tests.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash + +# This script runs e2e tests for Canary initialization, analysis and promotion +# Prerequisites: Kubernetes Kind, Helm and NGINX ingress controller + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) +export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" + +echo '>>> Creating test namespace' +kubectl create namespace test + +echo ">>> Downloading Gloo CLI" +curl -SsL https://github.com/solo-io/gloo/releases/download/v0.13.29/glooctl-linux-amd64 > glooctl +chmod +x glooctl + +echo '>>> Installing load tester' +kubectl -n test apply -f ${REPO_ROOT}/artifacts/loadtester/ +kubectl -n test rollout status deployment/flagger-loadtester + +echo '>>> Initialising canary' +kubectl apply -f ${REPO_ROOT}/test/e2e-workload.yaml +./glooctl add route --path-prefix / --upstream-group-name podinfo --upstream-group-namespace test + +cat <>> Waiting for primary to be ready' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n gloo-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary initialization test passed' + +echo '>>> Triggering canary deployment' +kubectl -n test set image deployment/podinfo podinfod=quay.io/stefanprodan/podinfo:1.4.1 + +echo '>>> Waiting for canary promotion' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test describe deployment/podinfo-primary | grep '1.4.1' && ok=true || ok=false + sleep 10 + kubectl -n gloo-system logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n test describe deployment/podinfo + kubectl -n test describe deployment/podinfo-primary + kubectl -n gloo-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary promotion test passed' diff --git a/test/e2e-gloo.sh b/test/e2e-gloo.sh new file mode 100755 index 00000000..b4467b19 --- /dev/null +++ b/test/e2e-gloo.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) +export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" + +echo ">>> Installing Helm" +curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get | bash + +echo '>>> Installing Tiller' +kubectl --namespace kube-system create sa tiller +kubectl create clusterrolebinding tiller-cluster-rule --clusterrole=cluster-admin --serviceaccount=kube-system:tiller +helm init --service-account tiller --upgrade --wait +helm repo add gloo https://storage.googleapis.com/solo-public-helm + +echo '>>> Installing Gloo' +helm upgrade -i gloo gloo/gloo --version 0.13.29 \ +--wait \ +--namespace gloo-system \ +--set gatewayProxies.gateway-proxy.service.type=NodePort + +kubectl -n gloo-system rollout status deployment/gloo +kubectl -n gloo-system rollout status deployment/gateway-proxy +kubectl -n gloo-system get all