github/flagger
1
0
Fork 0
mirror of https://github.com/fluxcd/flagger.git synced 2026-04-15 06:57:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

gloo metrics

Browse Source
This commit is contained in:
Yuval Kohavi
2019-05-14 17:48:13 -04:00
parent 786c5aa93a
commit 677b9d9197
6 changed files with 185 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/metrics/factory.go
+5
View File
@@ -1,6 +1,7 @@
package metrics
import (
"strings"
"time"
)
@@ -31,6 +32,10 @@ func (factory Factory) Observer() Interface {
return &NginxObserver{
client: factory.Client,
}
case strings.HasPrefix(factory.MeshProvider, "gloo"):
return &GlooObserver{
client: factory.Client,
}
case factory.MeshProvider == "smi:linkerd":
return &LinkerdObserver{
client: factory.Client,
pkg/metrics/gloo.go
+72
View File
@@ -0,0 +1,72 @@
package metrics
import (
"time"
)
//envoy_cluster_name="test-podinfo-primary-9898_gloo-system"
var glooQueries = map[string]string{
"request-success-rate": `
sum(
rate(
envoy_cluster_upstream_rq{
envoy_cluster_name=~"{{ .Namespace }}-{{ .Name }}-[0-9a-zA-Z-]+_[0-9a-zA-Z-]+",
envoy_response_code!~"5.*"
}[{{ .Interval }}]
)
)
/
sum(
rate(
envoy_cluster_upstream_rq{
envoy_cluster_name=~"{{ .Namespace }}-{{ .Name }}-[0-9a-zA-Z-]+_[0-9a-zA-Z-]+",
}[{{ .Interval }}]
)
)
* 100`,
"request-duration": `
histogram_quantile(
0.99,
sum(
rate(
envoy_cluster_upstream_rq_time_bucket{
envoy_cluster_name=~"{{ .Namespace }}-{{ .Name }}-[0-9a-zA-Z-]+_[0-9a-zA-Z-]+",
}[{{ .Interval }}]
)
) by (le)
)`,
}
type GlooObserver struct {
client *PrometheusClient
}
func (ob *GlooObserver) GetRequestSuccessRate(name string, namespace string, interval string) (float64, error) {
query, err := ob.client.RenderQuery(name, namespace, interval, glooQueries["request-success-rate"])
if err != nil {
return 0, err
}
value, err := ob.client.RunQuery(query)
if err != nil {
return 0, err
}
return value, nil
}
func (ob *GlooObserver) GetRequestDuration(name string, namespace string, interval string) (time.Duration, error) {
query, err := ob.client.RenderQuery(name, namespace, interval, glooQueries["request-duration"])
if err != nil {
return 0, err
}
value, err := ob.client.RunQuery(query)
if err != nil {
return 0, err
}
ms := time.Duration(int64(value)) * time.Millisecond
return ms, nil
}
pkg/metrics/gloo_test.go
+74
View File
@@ -0,0 +1,74 @@
package metrics
import (
"net/http"
"net/http/httptest"
"testing"
"time"
)
func TestGlooObserver_GetRequestSuccessRate(t *testing.T) {
expected := `sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"default-podinfo-[0-9a-zA-Z-]+_[0-9a-zA-Z-]+",envoy_response_code!~"5.*"}[1m]))/sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"default-podinfo-[0-9a-zA-Z-]+_[0-9a-zA-Z-]+",}[1m]))*100`
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
promql := r.URL.Query()["query"][0]
if promql != expected {
t.Errorf("\nGot %s \nWanted %s", promql, expected)
}
json := `{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1,"100"]}]}}`
w.Write([]byte(json))
}))
defer ts.Close()
client, err := NewPrometheusClient(ts.URL, time.Second)
if err != nil {
t.Fatal(err)
}
observer := &GlooObserver{
client: client,
}
val, err := observer.GetRequestSuccessRate("podinfo", "default", "1m")
if err != nil {
t.Fatal(err.Error())
}
if val != 100 {
t.Errorf("Got %v wanted %v", val, 100)
}
}
func TestGlooObserver_GetRequestDuration(t *testing.T) {
expected := `histogram_quantile(0.99,sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"default-podinfo-[0-9a-zA-Z-]+_[0-9a-zA-Z-]+",}[1m]))by(le))`
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
promql := r.URL.Query()["query"][0]
if promql != expected {
t.Errorf("\nGot %s \nWanted %s", promql, expected)
}
json := `{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1,"100"]}]}}`
w.Write([]byte(json))
}))
defer ts.Close()
client, err := NewPrometheusClient(ts.URL, time.Second)
if err != nil {
t.Fatal(err)
}
observer := &GlooObserver{
client: client,
}
val, err := observer.GetRequestDuration("podinfo", "default", "1m")
if err != nil {
t.Fatal(err.Error())
}
if val != 100*time.Millisecond {
t.Errorf("Got %v wanted %v", val, 100*time.Millisecond)
}
}
pkg/router/gloo.go
+31 -27
View File
@@ -62,7 +62,15 @@ func NewGlooRouterWithClient(ctx context.Context, routingRuleClient gloov1.Upstr
// Reconcile creates or updates the Istio virtual service
func (gr *GlooRouter) Reconcile(canary *flaggerv1.Canary) error {
return gr.SetRoutes(canary, 100, 0)
// do we have routes already?
if _, _, err := gr.GetRoutes(canary); err == nil {
// we have routes, no need to do anything else
return nil
} else if solokiterror.IsNotExist(err) {
return gr.SetRoutes(canary, 100, 0)
} else {
return err
}
}
// GetRoutes returns the destinations weight for primary and canary
@@ -104,35 +112,31 @@ func (gr *GlooRouter) SetRoutes(
) error {
targetName := canary.Spec.TargetRef.Name
destinations := []*gloov1.WeightedDestination{}
if primaryWeight != 0 {
destinations = append(destinations, &gloov1.WeightedDestination{
Destination: &gloov1.Destination{
Upstream: solokitcore.ResourceRef{
Name: upstreamName(canary.Namespace, fmt.Sprintf("%s-primary", targetName), canary.Spec.Service.Port),
Namespace: gr.upstreamDiscoveryNs,
},
},
Weight: uint32(primaryWeight),
})
}
if canaryWeight != 0 {
destinations = append(destinations, &gloov1.WeightedDestination{
Destination: &gloov1.Destination{
Upstream: solokitcore.ResourceRef{
Name: upstreamName(canary.Namespace, fmt.Sprintf("%s-canary", targetName), canary.Spec.Service.Port),
Namespace: gr.upstreamDiscoveryNs,
},
},
Weight: uint32(canaryWeight),
})
}
if len(destinations) == 0 {
if primaryWeight == 0 && canaryWeight == 0 {
return fmt.Errorf("RoutingRule %s.%s update failed: no valid weights", targetName, canary.Namespace)
}
destinations := []*gloov1.WeightedDestination{}
destinations = append(destinations, &gloov1.WeightedDestination{
Destination: &gloov1.Destination{
Upstream: solokitcore.ResourceRef{
Name: upstreamName(canary.Namespace, fmt.Sprintf("%s-primary", targetName), canary.Spec.Service.Port),
Namespace: gr.upstreamDiscoveryNs,
},
},
Weight: uint32(primaryWeight),
})
destinations = append(destinations, &gloov1.WeightedDestination{
Destination: &gloov1.Destination{
Upstream: solokitcore.ResourceRef{
Name: upstreamName(canary.Namespace, fmt.Sprintf("%s-canary", targetName), canary.Spec.Service.Port),
Namespace: gr.upstreamDiscoveryNs,
},
},
Weight: uint32(canaryWeight),
})
upstreamGroup := &gloov1.UpstreamGroup{
Metadata: solokitcore.Metadata{
Name: canary.Spec.TargetRef.Name,
test/e2e-gloo-tests.sh
+2 -8
View File
@@ -12,7 +12,7 @@ echo '>>> Creating test namespace'
kubectl create namespace test
echo ">>> Downloading Gloo CLI"
curl -SsL https://github.com/solo-io/gloo/releases/download/v0.13.25/glooctl-linux-amd64 > glooctl
curl -SsL https://github.com/solo-io/gloo/releases/download/v0.13.27/glooctl-linux-amd64 > glooctl
chmod +x glooctl
echo '>>> Installing load tester'
@@ -43,15 +43,9 @@ spec:
maxWeight: 30
stepWeight: 10
metrics:
- name: envoy-success-rate
- name: request-success-rate
threshold: 99
interval: 1m
query: |
sum(rate(envoy_cluster_upstream_rq{kubernetes_namespace="gloo-system", gloo="gateway-proxy",
envoy_response_code!~"5.*"}[1m]))
/
sum(rate(envoy_cluster_upstream_rq{kubernetes_namespace="gloo-system", gloo="gateway-proxy"}[1m]))
* 100
webhooks:
- name: load-test
url: http://flagger-loadtester.test/
test/e2e-gloo.sh
+1 -1
View File
@@ -15,7 +15,7 @@ helm init --service-account tiller --upgrade --wait
helm repo add gloo https://storage.googleapis.com/solo-public-helm
echo '>>> Installing Gloo'
helm upgrade -i gloo gloo/gloo --version 0.13.25 \
helm upgrade -i gloo gloo/gloo --version 0.13.27 \
--wait \
--namespace gloo-system \
--set gatewayProxies.gateway-proxy.service.type=NodePort