diff --git a/README.md b/README.md index 0c7dde7e..7f6be2a4 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ Flagger documentation can be found at [docs.flagger.app](https://docs.flagger.ap * [Istio canary deployments](https://docs.flagger.app/usage/progressive-delivery) * [Linkerd canary deployments](https://docs.flagger.app/usage/linkerd-progressive-delivery) * [App Mesh canary deployments](https://docs.flagger.app/usage/appmesh-progressive-delivery) - * [Envoy canary deployments](https://docs.flagger.app/usage/envoy-progressive-delivery) + * [Crossover canary deployments](https://docs.flagger.app/usage/crossover-progressive-delivery) * [NGINX ingress controller canary deployments](https://docs.flagger.app/usage/nginx-progressive-delivery) * [Gloo ingress controller canary deployments](https://docs.flagger.app/usage/gloo-progressive-delivery) * [Blue/Green deployments](https://docs.flagger.app/usage/blue-green) diff --git a/docs/gitbook/usage/envoy-progressive-delivery.md b/docs/gitbook/usage/crossover-progressive-delivery.md similarity index 93% rename from docs/gitbook/usage/envoy-progressive-delivery.md rename to docs/gitbook/usage/crossover-progressive-delivery.md index ee7a8f6f..1559f0fc 100644 --- a/docs/gitbook/usage/envoy-progressive-delivery.md +++ b/docs/gitbook/usage/crossover-progressive-delivery.md @@ -1,6 +1,8 @@ -# Envoy Canary Deployments +# Envoy/Crossover Canary Deployments -This guide shows you how to use Envoy and Flagger to automate canary deployments. +This guide shows you how to use Envoy, [Crossover](https://github.com/mumoshu/crossover) and Flagger to automate canary deployments. + +Crossover is a minimal Envoy xDS implementation supports [Service Mesh Interface](https://smi-spec.io/). ### Prerequisites @@ -12,7 +14,7 @@ Create a test namespace: kubectl create ns test ``` -Install Envoy along with the sidecar with Helm: +Install Envoy along with the Crossover sidecar with Helm: ```bash helm repo add crossover https://mumoshu.github.io/crossover @@ -23,7 +25,7 @@ helm upgrade --install envoy crossover/envoy \ smi: apiVersions: trafficSplits: v1alpha1 -services: +upstreams: podinfo: smi: enabled: true @@ -46,7 +48,7 @@ helm repo add flagger https://flagger.app helm upgrade -i flagger flagger/flagger \ --namespace test \ --set prometheus.install=true \ ---set meshProvider=smi:envoy +--set meshProvider=smi:crossover ``` Optionally you can enable Slack notifications: @@ -90,7 +92,7 @@ metadata: namespace: test spec: # specify mesh provider if it isn't the default one - # provider: "smi:envoy" + # provider: "smi:crossover" # deployment reference targetRef: apiVersion: apps/v1 @@ -146,7 +148,7 @@ spec: url: http://flagger-loadtester.test/ timeout: 5s metadata: - cmd: "hey -z 1m -q 10 -c 2 http://envoy.test:10000/" + cmd: "hey -z 1m -q 10 -c 2 -H 'Host: podinfo.test' http://envoy.test:10000/" ``` Save the above resource as podinfo-canary.yaml and then apply it: @@ -282,13 +284,13 @@ kubectl -n test exec -it deploy/flagger-loadtester bash Generate HTTP 500 errors: ```bash -hey -z 1m -c 5 -q 5 http://envoy.test:10000/status/500 +hey -z 1m -c 5 -q 5 -H 'Host: podinfo.test' http://envoy.test:10000/status/500 ``` Generate latency: ```bash -watch -n 1 curl http://envoy.test:10000/delay/1 +watch -n 1 curl -H 'Host: podinfo.test' http://envoy.test:10000/delay/1 ``` When the number of failed checks reaches the canary analysis threshold, the traffic is routed back to the primary, diff --git a/pkg/controller/scheduler.go b/pkg/controller/scheduler.go index 7768daac..30783265 100644 --- a/pkg/controller/scheduler.go +++ b/pkg/controller/scheduler.go @@ -748,10 +748,10 @@ func (c *Controller) analyseCanary(r *flaggerv1.Canary) bool { // override the global provider if one is specified in the canary spec var metricsProvider string - // set the metrics provider to Envoy Prometheus when Envoy is the mesh provider - // For example, `envoy` metrics provider should be used for `smi:envoy` mesh provider - if strings.Contains(c.meshProvider, "envoy") { - metricsProvider = "envoy" + // set the metrics provider to Crossover Prometheus when Crossover is the mesh provider + // For example, `crossover` metrics provider should be used for `smi:crossover` mesh provider + if strings.Contains(c.meshProvider, "crossover") { + metricsProvider = "crossover" } else { metricsProvider = c.meshProvider } diff --git a/pkg/metrics/appmesh.go b/pkg/metrics/appmesh.go new file mode 100644 index 00000000..f922c55a --- /dev/null +++ b/pkg/metrics/appmesh.go @@ -0,0 +1,73 @@ +package metrics + +import ( + "time" +) + +var appMeshQueries = map[string]string{ + "request-success-rate": ` + sum( + rate( + envoy_cluster_upstream_rq{ + kubernetes_namespace="{{ .Namespace }}", + kubernetes_pod_name=~"{{ .Name }}-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)", + envoy_response_code!~"5.*" + }[{{ .Interval }}] + ) + ) + / + sum( + rate( + envoy_cluster_upstream_rq{ + kubernetes_namespace="{{ .Namespace }}", + kubernetes_pod_name=~"{{ .Name }}-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)" + }[{{ .Interval }}] + ) + ) + * 100`, + "request-duration": ` + histogram_quantile( + 0.99, + sum( + rate( + envoy_cluster_upstream_rq_time_bucket{ + kubernetes_namespace="{{ .Namespace }}", + kubernetes_pod_name=~"{{ .Name }}-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)" + }[{{ .Interval }}] + ) + ) by (le) + )`, +} + +type AppMeshObserver struct { + client *PrometheusClient +} + +func (ob *AppMeshObserver) GetRequestSuccessRate(name string, namespace string, interval string) (float64, error) { + query, err := ob.client.RenderQuery(name, namespace, interval, appMeshQueries["request-success-rate"]) + if err != nil { + return 0, err + } + + value, err := ob.client.RunQuery(query) + if err != nil { + return 0, err + } + + return value, nil +} + +func (ob *AppMeshObserver) GetRequestDuration(name string, namespace string, interval string) (time.Duration, error) { + query, err := ob.client.RenderQuery(name, namespace, interval, appMeshQueries["request-duration"]) + if err != nil { + return 0, err + } + + value, err := ob.client.RunQuery(query) + if err != nil { + return 0, err + } + + ms := time.Duration(int64(value)) * time.Millisecond + return ms, nil +} diff --git a/pkg/metrics/appmesh_test.go b/pkg/metrics/appmesh_test.go new file mode 100644 index 00000000..471be5c4 --- /dev/null +++ b/pkg/metrics/appmesh_test.go @@ -0,0 +1,74 @@ +package metrics + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" +) + +func TestAppMeshObserver_GetRequestSuccessRate(t *testing.T) { + expected := ` sum( rate( envoy_cluster_upstream_rq{ kubernetes_namespace="default", kubernetes_pod_name=~"podinfo-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)", envoy_response_code!~"5.*" }[1m] ) ) / sum( rate( envoy_cluster_upstream_rq{ kubernetes_namespace="default", kubernetes_pod_name=~"podinfo-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)" }[1m] ) ) * 100` + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + promql := r.URL.Query()["query"][0] + if promql != expected { + t.Errorf("\nGot %s \nWanted %s", promql, expected) + } + + json := `{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1,"100"]}]}}` + w.Write([]byte(json)) + })) + defer ts.Close() + + client, err := NewPrometheusClient(ts.URL, time.Second) + if err != nil { + t.Fatal(err) + } + + observer := &AppMeshObserver{ + client: client, + } + + val, err := observer.GetRequestSuccessRate("podinfo", "default", "1m") + if err != nil { + t.Fatal(err.Error()) + } + + if val != 100 { + t.Errorf("Got %v wanted %v", val, 100) + } +} + +func TestAppMeshObserver_GetRequestDuration(t *testing.T) { + expected := ` histogram_quantile( 0.99, sum( rate( envoy_cluster_upstream_rq_time_bucket{ kubernetes_namespace="default", kubernetes_pod_name=~"podinfo-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)" }[1m] ) ) by (le) )` + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + promql := r.URL.Query()["query"][0] + if promql != expected { + t.Errorf("\nGot %s \nWanted %s", promql, expected) + } + + json := `{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1,"100"]}]}}` + w.Write([]byte(json)) + })) + defer ts.Close() + + client, err := NewPrometheusClient(ts.URL, time.Second) + if err != nil { + t.Fatal(err) + } + + observer := &AppMeshObserver{ + client: client, + } + + val, err := observer.GetRequestDuration("podinfo", "default", "1m") + if err != nil { + t.Fatal(err.Error()) + } + + if val != 100*time.Millisecond { + t.Errorf("Got %v wanted %v", val, 100*time.Millisecond) + } +} diff --git a/pkg/metrics/envoy.go b/pkg/metrics/crossover.go similarity index 74% rename from pkg/metrics/envoy.go rename to pkg/metrics/crossover.go index bf568b7a..54a5e290 100644 --- a/pkg/metrics/envoy.go +++ b/pkg/metrics/crossover.go @@ -4,7 +4,7 @@ import ( "time" ) -var envoyQueries = map[string]string{ +var crossoverQueries = map[string]string{ "request-success-rate": ` sum( rate( @@ -39,12 +39,12 @@ var envoyQueries = map[string]string{ )`, } -type EnvoyObserver struct { +type CrossoverObserver struct { client *PrometheusClient } -func (ob *EnvoyObserver) GetRequestSuccessRate(name string, namespace string, interval string) (float64, error) { - query, err := ob.client.RenderQuery(name, namespace, interval, envoyQueries["request-success-rate"]) +func (ob *CrossoverObserver) GetRequestSuccessRate(name string, namespace string, interval string) (float64, error) { + query, err := ob.client.RenderQuery(name, namespace, interval, crossoverQueries["request-success-rate"]) if err != nil { return 0, err } @@ -57,8 +57,8 @@ func (ob *EnvoyObserver) GetRequestSuccessRate(name string, namespace string, in return value, nil } -func (ob *EnvoyObserver) GetRequestDuration(name string, namespace string, interval string) (time.Duration, error) { - query, err := ob.client.RenderQuery(name, namespace, interval, envoyQueries["request-duration"]) +func (ob *CrossoverObserver) GetRequestDuration(name string, namespace string, interval string) (time.Duration, error) { + query, err := ob.client.RenderQuery(name, namespace, interval, crossoverQueries["request-duration"]) if err != nil { return 0, err } diff --git a/pkg/metrics/envoy_service.go b/pkg/metrics/crossover_service.go similarity index 72% rename from pkg/metrics/envoy_service.go rename to pkg/metrics/crossover_service.go index 5e9d26c3..bde9f0a1 100644 --- a/pkg/metrics/envoy_service.go +++ b/pkg/metrics/crossover_service.go @@ -4,7 +4,7 @@ import ( "time" ) -var envoyServiceQueries = map[string]string{ +var crossoverServiceQueries = map[string]string{ "request-success-rate": ` sum( rate( @@ -39,12 +39,12 @@ var envoyServiceQueries = map[string]string{ )`, } -type EnvoyServiceObserver struct { +type CrossoverServiceObserver struct { client *PrometheusClient } -func (ob *EnvoyServiceObserver) GetRequestSuccessRate(name string, namespace string, interval string) (float64, error) { - query, err := ob.client.RenderQuery(name, namespace, interval, envoyServiceQueries["request-success-rate"]) +func (ob *CrossoverServiceObserver) GetRequestSuccessRate(name string, namespace string, interval string) (float64, error) { + query, err := ob.client.RenderQuery(name, namespace, interval, crossoverServiceQueries["request-success-rate"]) if err != nil { return 0, err } @@ -57,8 +57,8 @@ func (ob *EnvoyServiceObserver) GetRequestSuccessRate(name string, namespace str return value, nil } -func (ob *EnvoyServiceObserver) GetRequestDuration(name string, namespace string, interval string) (time.Duration, error) { - query, err := ob.client.RenderQuery(name, namespace, interval, envoyServiceQueries["request-duration"]) +func (ob *CrossoverServiceObserver) GetRequestDuration(name string, namespace string, interval string) (time.Duration, error) { + query, err := ob.client.RenderQuery(name, namespace, interval, crossoverServiceQueries["request-duration"]) if err != nil { return 0, err } diff --git a/pkg/metrics/envoy_service_test.go b/pkg/metrics/crossover_service_test.go similarity index 89% rename from pkg/metrics/envoy_service_test.go rename to pkg/metrics/crossover_service_test.go index fca854bd..8d65bbab 100644 --- a/pkg/metrics/envoy_service_test.go +++ b/pkg/metrics/crossover_service_test.go @@ -7,7 +7,7 @@ import ( "time" ) -func TestEnvoyServiceObserver_GetRequestSuccessRate(t *testing.T) { +func TestCrossoverServiceObserver_GetRequestSuccessRate(t *testing.T) { expected := ` sum( rate( envoy_cluster_upstream_rq{ kubernetes_namespace="default", envoy_cluster_name="podinfo-canary", envoy_response_code!~"5.*" }[1m] ) ) / sum( rate( envoy_cluster_upstream_rq{ kubernetes_namespace="default", envoy_cluster_name="podinfo-canary" }[1m] ) ) * 100` ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -26,7 +26,7 @@ func TestEnvoyServiceObserver_GetRequestSuccessRate(t *testing.T) { t.Fatal(err) } - observer := &EnvoyServiceObserver{ + observer := &CrossoverServiceObserver{ client: client, } @@ -40,7 +40,7 @@ func TestEnvoyServiceObserver_GetRequestSuccessRate(t *testing.T) { } } -func TestEnvoyServiceObserver_GetRequestDuration(t *testing.T) { +func TestCrossoverServiceObserver_GetRequestDuration(t *testing.T) { expected := ` histogram_quantile( 0.99, sum( rate( envoy_cluster_upstream_rq_time_bucket{ kubernetes_namespace="default", envoy_cluster_name="podinfo-canary" }[1m] ) ) by (le) )` ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -59,7 +59,7 @@ func TestEnvoyServiceObserver_GetRequestDuration(t *testing.T) { t.Fatal(err) } - observer := &EnvoyServiceObserver{ + observer := &CrossoverServiceObserver{ client: client, } diff --git a/pkg/metrics/envoy_test.go b/pkg/metrics/crossover_test.go similarity index 90% rename from pkg/metrics/envoy_test.go rename to pkg/metrics/crossover_test.go index 3852fd35..dd788a6f 100644 --- a/pkg/metrics/envoy_test.go +++ b/pkg/metrics/crossover_test.go @@ -7,7 +7,7 @@ import ( "time" ) -func TestEnvoyObserver_GetRequestSuccessRate(t *testing.T) { +func TestCrossoverObserver_GetRequestSuccessRate(t *testing.T) { expected := ` sum( rate( envoy_cluster_upstream_rq{ kubernetes_namespace="default", envoy_cluster_name=~"podinfo-canary", envoy_response_code!~"5.*" }[1m] ) ) / sum( rate( envoy_cluster_upstream_rq{ kubernetes_namespace="default", envoy_cluster_name=~"podinfo-canary" }[1m] ) ) * 100` ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -26,7 +26,7 @@ func TestEnvoyObserver_GetRequestSuccessRate(t *testing.T) { t.Fatal(err) } - observer := &EnvoyObserver{ + observer := &CrossoverObserver{ client: client, } @@ -40,7 +40,7 @@ func TestEnvoyObserver_GetRequestSuccessRate(t *testing.T) { } } -func TestEnvoyObserver_GetRequestDuration(t *testing.T) { +func TestCrossoverObserver_GetRequestDuration(t *testing.T) { expected := ` histogram_quantile( 0.99, sum( rate( envoy_cluster_upstream_rq_time_bucket{ kubernetes_namespace="default", envoy_cluster_name=~"podinfo-canary" }[1m] ) ) by (le) )` ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -59,7 +59,7 @@ func TestEnvoyObserver_GetRequestDuration(t *testing.T) { t.Fatal(err) } - observer := &EnvoyObserver{ + observer := &CrossoverObserver{ client: client, } diff --git a/pkg/metrics/factory.go b/pkg/metrics/factory.go index ce6d85d0..c34b0d45 100644 --- a/pkg/metrics/factory.go +++ b/pkg/metrics/factory.go @@ -30,8 +30,12 @@ func (factory Factory) Observer(provider string) Interface { return &HttpObserver{ client: factory.Client, } - case provider == "appmesh", provider == "envoy": - return &EnvoyObserver{ + case provider == "appmesh": + return &AppMeshObserver{ + client: factory.Client, + } + case provider == "crossover": + return &CrossoverObserver{ client: factory.Client, } case provider == "nginx": @@ -43,7 +47,7 @@ func (factory Factory) Observer(provider string) Interface { client: factory.Client, } case provider == "appmesh:service", provider == "envoy:service": - return &EnvoyServiceObserver{ + return &CrossoverServiceObserver{ client: factory.Client, } case provider == "linkerd":