diff --git a/pkg/metrics/factory.go b/pkg/metrics/factory.go index b351a44d..986ffb42 100644 --- a/pkg/metrics/factory.go +++ b/pkg/metrics/factory.go @@ -1,6 +1,7 @@ package metrics import ( + "strings" "time" ) @@ -31,6 +32,10 @@ func (factory Factory) Observer() Interface { return &NginxObserver{ client: factory.Client, } + case strings.HasPrefix(factory.MeshProvider, "gloo"): + return &GlooObserver{ + client: factory.Client, + } case factory.MeshProvider == "smi:linkerd": return &LinkerdObserver{ client: factory.Client, diff --git a/pkg/metrics/gloo.go b/pkg/metrics/gloo.go new file mode 100644 index 00000000..0acd361e --- /dev/null +++ b/pkg/metrics/gloo.go @@ -0,0 +1,72 @@ +package metrics + +import ( + "time" +) + +//envoy_cluster_name="test-podinfo-primary-9898_gloo-system" + +var glooQueries = map[string]string{ + "request-success-rate": ` + sum( + rate( + envoy_cluster_upstream_rq{ + envoy_cluster_name=~"{{ .Namespace }}-{{ .Name }}-[0-9a-zA-Z-]+_[0-9a-zA-Z-]+", + envoy_response_code!~"5.*" + }[{{ .Interval }}] + ) + ) + / + sum( + rate( + envoy_cluster_upstream_rq{ + envoy_cluster_name=~"{{ .Namespace }}-{{ .Name }}-[0-9a-zA-Z-]+_[0-9a-zA-Z-]+", + }[{{ .Interval }}] + ) + ) + * 100`, + "request-duration": ` + histogram_quantile( + 0.99, + sum( + rate( + envoy_cluster_upstream_rq_time_bucket{ + envoy_cluster_name=~"{{ .Namespace }}-{{ .Name }}-[0-9a-zA-Z-]+_[0-9a-zA-Z-]+", + }[{{ .Interval }}] + ) + ) by (le) + )`, +} + +type GlooObserver struct { + client *PrometheusClient +} + +func (ob *GlooObserver) GetRequestSuccessRate(name string, namespace string, interval string) (float64, error) { + query, err := ob.client.RenderQuery(name, namespace, interval, glooQueries["request-success-rate"]) + if err != nil { + return 0, err + } + + value, err := ob.client.RunQuery(query) + if err != nil { + return 0, err + } + + return value, nil +} + +func (ob *GlooObserver) GetRequestDuration(name string, namespace string, interval string) (time.Duration, error) { + query, err := ob.client.RenderQuery(name, namespace, interval, glooQueries["request-duration"]) + if err != nil { + return 0, err + } + + value, err := ob.client.RunQuery(query) + if err != nil { + return 0, err + } + + ms := time.Duration(int64(value)) * time.Millisecond + return ms, nil +} diff --git a/pkg/metrics/gloo_test.go b/pkg/metrics/gloo_test.go new file mode 100644 index 00000000..7a69f918 --- /dev/null +++ b/pkg/metrics/gloo_test.go @@ -0,0 +1,74 @@ +package metrics + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" +) + +func TestGlooObserver_GetRequestSuccessRate(t *testing.T) { + expected := `sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"default-podinfo-[0-9a-zA-Z-]+_[0-9a-zA-Z-]+",envoy_response_code!~"5.*"}[1m]))/sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"default-podinfo-[0-9a-zA-Z-]+_[0-9a-zA-Z-]+",}[1m]))*100` + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + promql := r.URL.Query()["query"][0] + if promql != expected { + t.Errorf("\nGot %s \nWanted %s", promql, expected) + } + + json := `{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1,"100"]}]}}` + w.Write([]byte(json)) + })) + defer ts.Close() + + client, err := NewPrometheusClient(ts.URL, time.Second) + if err != nil { + t.Fatal(err) + } + + observer := &GlooObserver{ + client: client, + } + + val, err := observer.GetRequestSuccessRate("podinfo", "default", "1m") + if err != nil { + t.Fatal(err.Error()) + } + + if val != 100 { + t.Errorf("Got %v wanted %v", val, 100) + } +} + +func TestGlooObserver_GetRequestDuration(t *testing.T) { + expected := `histogram_quantile(0.99,sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"default-podinfo-[0-9a-zA-Z-]+_[0-9a-zA-Z-]+",}[1m]))by(le))` + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + promql := r.URL.Query()["query"][0] + if promql != expected { + t.Errorf("\nGot %s \nWanted %s", promql, expected) + } + + json := `{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1,"100"]}]}}` + w.Write([]byte(json)) + })) + defer ts.Close() + + client, err := NewPrometheusClient(ts.URL, time.Second) + if err != nil { + t.Fatal(err) + } + + observer := &GlooObserver{ + client: client, + } + + val, err := observer.GetRequestDuration("podinfo", "default", "1m") + if err != nil { + t.Fatal(err.Error()) + } + + if val != 100*time.Millisecond { + t.Errorf("Got %v wanted %v", val, 100*time.Millisecond) + } +} diff --git a/pkg/router/gloo.go b/pkg/router/gloo.go index eb1d22ed..09bcd743 100644 --- a/pkg/router/gloo.go +++ b/pkg/router/gloo.go @@ -62,7 +62,15 @@ func NewGlooRouterWithClient(ctx context.Context, routingRuleClient gloov1.Upstr // Reconcile creates or updates the Istio virtual service func (gr *GlooRouter) Reconcile(canary *flaggerv1.Canary) error { - return gr.SetRoutes(canary, 100, 0) + // do we have routes already? + if _, _, err := gr.GetRoutes(canary); err == nil { + // we have routes, no need to do anything else + return nil + } else if solokiterror.IsNotExist(err) { + return gr.SetRoutes(canary, 100, 0) + } else { + return err + } } // GetRoutes returns the destinations weight for primary and canary @@ -104,35 +112,31 @@ func (gr *GlooRouter) SetRoutes( ) error { targetName := canary.Spec.TargetRef.Name - destinations := []*gloov1.WeightedDestination{} - if primaryWeight != 0 { - destinations = append(destinations, &gloov1.WeightedDestination{ - Destination: &gloov1.Destination{ - Upstream: solokitcore.ResourceRef{ - Name: upstreamName(canary.Namespace, fmt.Sprintf("%s-primary", targetName), canary.Spec.Service.Port), - Namespace: gr.upstreamDiscoveryNs, - }, - }, - Weight: uint32(primaryWeight), - }) - } - - if canaryWeight != 0 { - destinations = append(destinations, &gloov1.WeightedDestination{ - Destination: &gloov1.Destination{ - Upstream: solokitcore.ResourceRef{ - Name: upstreamName(canary.Namespace, fmt.Sprintf("%s-canary", targetName), canary.Spec.Service.Port), - Namespace: gr.upstreamDiscoveryNs, - }, - }, - Weight: uint32(canaryWeight), - }) - } - - if len(destinations) == 0 { + if primaryWeight == 0 && canaryWeight == 0 { return fmt.Errorf("RoutingRule %s.%s update failed: no valid weights", targetName, canary.Namespace) } + destinations := []*gloov1.WeightedDestination{} + destinations = append(destinations, &gloov1.WeightedDestination{ + Destination: &gloov1.Destination{ + Upstream: solokitcore.ResourceRef{ + Name: upstreamName(canary.Namespace, fmt.Sprintf("%s-primary", targetName), canary.Spec.Service.Port), + Namespace: gr.upstreamDiscoveryNs, + }, + }, + Weight: uint32(primaryWeight), + }) + + destinations = append(destinations, &gloov1.WeightedDestination{ + Destination: &gloov1.Destination{ + Upstream: solokitcore.ResourceRef{ + Name: upstreamName(canary.Namespace, fmt.Sprintf("%s-canary", targetName), canary.Spec.Service.Port), + Namespace: gr.upstreamDiscoveryNs, + }, + }, + Weight: uint32(canaryWeight), + }) + upstreamGroup := &gloov1.UpstreamGroup{ Metadata: solokitcore.Metadata{ Name: canary.Spec.TargetRef.Name, diff --git a/test/e2e-gloo-tests.sh b/test/e2e-gloo-tests.sh index 9984a429..ab113633 100755 --- a/test/e2e-gloo-tests.sh +++ b/test/e2e-gloo-tests.sh @@ -12,7 +12,7 @@ echo '>>> Creating test namespace' kubectl create namespace test echo ">>> Downloading Gloo CLI" -curl -SsL https://github.com/solo-io/gloo/releases/download/v0.13.25/glooctl-linux-amd64 > glooctl +curl -SsL https://github.com/solo-io/gloo/releases/download/v0.13.27/glooctl-linux-amd64 > glooctl chmod +x glooctl echo '>>> Installing load tester' @@ -43,15 +43,9 @@ spec: maxWeight: 30 stepWeight: 10 metrics: - - name: envoy-success-rate + - name: request-success-rate threshold: 99 interval: 1m - query: | - sum(rate(envoy_cluster_upstream_rq{kubernetes_namespace="gloo-system", gloo="gateway-proxy", - envoy_response_code!~"5.*"}[1m])) - / - sum(rate(envoy_cluster_upstream_rq{kubernetes_namespace="gloo-system", gloo="gateway-proxy"}[1m])) - * 100 webhooks: - name: load-test url: http://flagger-loadtester.test/ diff --git a/test/e2e-gloo.sh b/test/e2e-gloo.sh index 7788d897..5bdefef7 100755 --- a/test/e2e-gloo.sh +++ b/test/e2e-gloo.sh @@ -15,7 +15,7 @@ helm init --service-account tiller --upgrade --wait helm repo add gloo https://storage.googleapis.com/solo-public-helm echo '>>> Installing Gloo' -helm upgrade -i gloo gloo/gloo --version 0.13.25 \ +helm upgrade -i gloo gloo/gloo --version 0.13.27 \ --wait \ --namespace gloo-system \ --set gatewayProxies.gateway-proxy.service.type=NodePort