From 39e55daa040b9aa5991a84e16df8e2536a8e5baa Mon Sep 17 00:00:00 2001 From: Samuel Lang Date: Fri, 14 Aug 2020 12:54:11 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=93=88=20Skipper=20Metrics=20Observer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Te be able to distinct Skipper routes we need to combine the Canary data to generate the Skipper metric label. "request-success-rate" and "request-duration" queries are implemented and tested that provide those obersvations from Skipper metrics * Takes into account how Skipper renders the paths accordingly and reformats the quieries. --- pkg/apis/flagger/v1beta1/provider.go | 1 + pkg/metrics/observers/factory.go | 4 + pkg/metrics/observers/skipper.go | 82 ++++++++++++++++++++ pkg/metrics/observers/skipper_test.go | 106 ++++++++++++++++++++++++++ 4 files changed, 193 insertions(+) create mode 100644 pkg/metrics/observers/skipper.go create mode 100644 pkg/metrics/observers/skipper_test.go diff --git a/pkg/apis/flagger/v1beta1/provider.go b/pkg/apis/flagger/v1beta1/provider.go index 9dbcf017..a27bbc88 100644 --- a/pkg/apis/flagger/v1beta1/provider.go +++ b/pkg/apis/flagger/v1beta1/provider.go @@ -9,4 +9,5 @@ const ( GlooProvider string = "gloo" NGINXProvider string = "nginx" KubernetesProvider string = "kubernetes" + SkipperProvider string = "skipper" ) diff --git a/pkg/metrics/observers/factory.go b/pkg/metrics/observers/factory.go index 35555c7c..ac20884d 100644 --- a/pkg/metrics/observers/factory.go +++ b/pkg/metrics/observers/factory.go @@ -56,6 +56,10 @@ func (factory Factory) Observer(provider string) Interface { return &HttpObserver{ client: factory.Client, } + case provider == flaggerv1.SkipperProvider: + return &SkipperObserver{ + client: factory.Client, + } default: return &IstioObserver{ client: factory.Client, diff --git a/pkg/metrics/observers/skipper.go b/pkg/metrics/observers/skipper.go new file mode 100644 index 00000000..9f4effea --- /dev/null +++ b/pkg/metrics/observers/skipper.go @@ -0,0 +1,82 @@ +package observers + +import ( + "fmt" + "regexp" + "time" + + flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1beta1" + "github.com/weaveworks/flagger/pkg/logger" + + "github.com/weaveworks/flagger/pkg/metrics/providers" +) + +const routePattern = `{{- $route := printf "kube(ew)?_%s__%s_canary__.*__%s_canary(_[0-9]+)?" namespace ingress service }}` + +var skipperQueries = map[string]string{ + "request-success-rate": routePattern + ` + sum(rate(skipper_response_duration_seconds_bucket{route=~"{{ $route }}",code!~"5..",le="+Inf"}[{{ interval }}])) / + sum(rate(skipper_response_duration_seconds_bucket{route=~"{{ $route }}",le="+Inf"}[{{ interval }}])) * 100`, + "request-duration": routePattern + ` + sum(rate(skipper_serve_route_duration_seconds_sum{route=~"{{ $route }}"}[{{ interval }}])) / + sum(rate(skipper_serve_route_duration_seconds_count{route=~"{{ $route }}"}[{{ interval }}])) * 1000`, +} + +// SkipperObserver Implementation for Skipper (https://github.com/zalando/skipper) +type SkipperObserver struct { + client providers.Interface +} + +// GetRequestSuccessRate return value for Skipper Request Success Rate +func (ob *SkipperObserver) GetRequestSuccessRate(model flaggerv1.MetricTemplateModel) (float64, error) { + + model = encodeModelForSkipper(model) + + query, err := RenderQuery(skipperQueries["request-success-rate"], model) + if err != nil { + return 0, fmt.Errorf("rendering query failed: %w", err) + } + logger, _ := logger.NewLoggerWithEncoding("debug", "json") + logger.Debugf("GetRequestSuccessRate: %s", query) + + value, err := ob.client.RunQuery(query) + if err != nil { + return 0, fmt.Errorf("running query failed: %w", err) + } + + return value, nil +} + +// GetRequestDuration return value for Skipper Request Duration +func (ob *SkipperObserver) GetRequestDuration(model flaggerv1.MetricTemplateModel) (time.Duration, error) { + + model = encodeModelForSkipper(model) + + query, err := RenderQuery(skipperQueries["request-duration"], model) + if err != nil { + return 0, fmt.Errorf("rendering query failed: %w", err) + } + logger, _ := logger.NewLoggerWithEncoding("debug", "json") + logger.Debugf("GetRequestDuration: %s", query) + + value, err := ob.client.RunQuery(query) + if err != nil { + return 0, fmt.Errorf("running query failed: %w", err) + } + + ms := time.Duration(int64(value)) * time.Millisecond + return ms, nil +} + +// encodeModelForSkipper replaces non word character in model with underscore to match route names +// https://github.com/zalando/skipper/blob/dd70bd65e7f99cfb5dd6b6f71885d9fe3b2707f6/dataclients/kubernetes/ingress.go#L101 +func encodeModelForSkipper(model flaggerv1.MetricTemplateModel) flaggerv1.MetricTemplateModel { + nonWord := regexp.MustCompile(`\W`) + model.Ingress = nonWord.ReplaceAllString(model.Ingress, "_") + model.Name = nonWord.ReplaceAllString(model.Name, "_") + model.Namespace = nonWord.ReplaceAllString(model.Namespace, "_") + model.Service = nonWord.ReplaceAllString(model.Service, "_") + model.Target = nonWord.ReplaceAllString(model.Target, "_") + + return model +} diff --git a/pkg/metrics/observers/skipper_test.go b/pkg/metrics/observers/skipper_test.go new file mode 100644 index 00000000..b75a6e8c --- /dev/null +++ b/pkg/metrics/observers/skipper_test.go @@ -0,0 +1,106 @@ +package observers + +import ( + "errors" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1beta1" + "github.com/weaveworks/flagger/pkg/metrics/providers" +) + +func TestSkipperObserver_GetRequestSuccessRate(t *testing.T) { + t.Run("ok", func(t *testing.T) { + expected := ` sum(rate(skipper_response_duration_seconds_bucket{route=~"kube(ew)?_skipper__skipper_ingress_canary__.*__backend_canary(_[0-9]+)?",code!~"5..",le="+Inf"}[1m])) / sum(rate(skipper_response_duration_seconds_bucket{route=~"kube(ew)?_skipper__skipper_ingress_canary__.*__backend_canary(_[0-9]+)?",le="+Inf"}[1m])) * 100` + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + promql := r.URL.Query()["query"][0] + assert.Equal(t, expected, promql) + + json := `{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1,"100"]}]}}` + w.Write([]byte(json)) + })) + defer ts.Close() + + client, err := providers.NewPrometheusProvider(flaggerv1.MetricTemplateProvider{ + Type: "prometheus", + Address: ts.URL, + SecretRef: nil, + }, nil) + require.NoError(t, err) + + observer := &SkipperObserver{ + client: client, + } + + val, err := observer.GetRequestSuccessRate(flaggerv1.MetricTemplateModel{ + Namespace: "skipper", + Interval: "1m", + Service: "backend", + Ingress: "skipper-ingress", + }) + require.NoError(t, err) + + assert.Equal(t, float64(100), val) + }) + + t.Run("no values", func(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + json := `{"status":"success","data":{"resultType":"vector","result":[]}}` + w.Write([]byte(json)) + })) + defer ts.Close() + + client, err := providers.NewPrometheusProvider(flaggerv1.MetricTemplateProvider{ + Type: "prometheus", + Address: ts.URL, + SecretRef: nil, + }, nil) + require.NoError(t, err) + + observer := &SkipperObserver{ + client: client, + } + + _, err = observer.GetRequestSuccessRate(flaggerv1.MetricTemplateModel{}) + require.True(t, errors.Is(err, providers.ErrNoValuesFound)) + }) +} + +func TestSkipperObserver_GetRequestDuration(t *testing.T) { + expected := ` sum(rate(skipper_serve_route_duration_seconds_sum{route=~"kube(ew)?_skipper__skipper_ingress_canary__.*__backend_canary(_[0-9]+)?"}[1m])) / sum(rate(skipper_serve_route_duration_seconds_count{route=~"kube(ew)?_skipper__skipper_ingress_canary__.*__backend_canary(_[0-9]+)?"}[1m])) * 1000` + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + promql := r.URL.Query()["query"][0] + assert.Equal(t, expected, promql) + + json := `{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1,"100"]}]}}` + w.Write([]byte(json)) + })) + defer ts.Close() + + client, err := providers.NewPrometheusProvider(flaggerv1.MetricTemplateProvider{ + Type: "prometheus", + Address: ts.URL, + SecretRef: nil, + }, nil) + require.NoError(t, err) + + observer := &SkipperObserver{ + client: client, + } + + val, err := observer.GetRequestDuration(flaggerv1.MetricTemplateModel{ + Namespace: "skipper", + Interval: "1m", + Service: "backend", + Ingress: "skipper-ingress", + }) + require.NoError(t, err) + + assert.Equal(t, 100*time.Millisecond, val) +}