Add HTTP metrics when no mesh provider is specified

Implement request-success-rate and request-duration checks using http_request_duration_seconds histogram
This commit is contained in:
stefanprodan
2019-06-19 13:15:17 +03:00
parent 7e72d23b60
commit 647d02890f
3 changed files with 149 additions and 0 deletions
+4
View File
@@ -24,6 +24,10 @@ func NewFactory(metricsServer string, meshProvider string, timeout time.Duration
func (factory Factory) Observer() Interface {
switch {
case factory.MeshProvider == "none":
return &HttpObserver{
client: factory.Client,
}
case factory.MeshProvider == "appmesh":
return &EnvoyObserver{
client: factory.Client,
+71
View File
@@ -0,0 +1,71 @@
package metrics
import "time"
var httpQueries = map[string]string{
"request-success-rate": `
sum(
rate(
http_request_duration_seconds_count{
kubernetes_namespace="{{ .Namespace }}",
kubernetes_pod_name=~"{{ .Name }}-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)",
status!~"5.*"
}[{{ .Interval }}]
)
)
/
sum(
rate(
http_request_duration_seconds_count{
kubernetes_namespace="{{ .Namespace }}",
kubernetes_pod_name=~"{{ .Name }}-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)"
}[{{ .Interval }}]
)
)
* 100`,
"request-duration": `
histogram_quantile(
0.99,
sum(
rate(
http_request_duration_seconds_bucket{
kubernetes_namespace="{{ .Namespace }}",
kubernetes_pod_name=~"{{ .Name }}-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)"
}[{{ .Interval }}]
)
) by (le)
)`,
}
type HttpObserver struct {
client *PrometheusClient
}
func (ob *HttpObserver) GetRequestSuccessRate(name string, namespace string, interval string) (float64, error) {
query, err := ob.client.RenderQuery(name, namespace, interval, httpQueries["request-success-rate"])
if err != nil {
return 0, err
}
value, err := ob.client.RunQuery(query)
if err != nil {
return 0, err
}
return value, nil
}
func (ob *HttpObserver) GetRequestDuration(name string, namespace string, interval string) (time.Duration, error) {
query, err := ob.client.RenderQuery(name, namespace, interval, httpQueries["request-duration"])
if err != nil {
return 0, err
}
value, err := ob.client.RunQuery(query)
if err != nil {
return 0, err
}
ms := time.Duration(int64(value*1000)) * time.Millisecond
return ms, nil
}
+74
View File
@@ -0,0 +1,74 @@
package metrics
import (
"net/http"
"net/http/httptest"
"testing"
"time"
)
func TestHttpObserver_GetRequestSuccessRate(t *testing.T) {
expected := `sum(rate(http_request_duration_seconds_count{kubernetes_namespace="default",kubernetes_pod_name=~"podinfo-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)",status!~"5.*"}[1m]))/sum(rate(http_request_duration_seconds_count{kubernetes_namespace="default",kubernetes_pod_name=~"podinfo-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)"}[1m]))*100`
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
promql := r.URL.Query()["query"][0]
if promql != expected {
t.Errorf("\nGot %s \nWanted %s", promql, expected)
}
json := `{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1,"100"]}]}}`
w.Write([]byte(json))
}))
defer ts.Close()
client, err := NewPrometheusClient(ts.URL, time.Second)
if err != nil {
t.Fatal(err)
}
observer := &HttpObserver{
client: client,
}
val, err := observer.GetRequestSuccessRate("podinfo", "default", "1m")
if err != nil {
t.Fatal(err.Error())
}
if val != 100 {
t.Errorf("Got %v wanted %v", val, 100)
}
}
func TestHttpObserver_GetRequestDuration(t *testing.T) {
expected := `histogram_quantile(0.99,sum(rate(http_request_duration_seconds_bucket{kubernetes_namespace="default",kubernetes_pod_name=~"podinfo-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)"}[1m]))by(le))`
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
promql := r.URL.Query()["query"][0]
if promql != expected {
t.Errorf("\nGot %s \nWanted %s", promql, expected)
}
json := `{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1,"0.100"]}]}}`
w.Write([]byte(json))
}))
defer ts.Close()
client, err := NewPrometheusClient(ts.URL, time.Second)
if err != nil {
t.Fatal(err)
}
observer := &HttpObserver{
client: client,
}
val, err := observer.GetRequestDuration("podinfo", "default", "1m")
if err != nil {
t.Fatal(err.Error())
}
if val != 100*time.Millisecond {
t.Errorf("Got %v wanted %v", val, 100*time.Millisecond)
}
}