From cf3ba35fb98835fd678e8de84fb48d7da794c861 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Mon, 6 May 2019 18:42:31 +0300 Subject: [PATCH] Add nginx ingress controller metrics --- Makefile | 6 ++ artifacts/nginx/canary.yaml | 54 ++++++++++++++ artifacts/nginx/deployment.yaml | 69 ++++++++++++++++++ artifacts/nginx/ingress.yaml | 17 +++++ pkg/metrics/nginx.go | 122 ++++++++++++++++++++++++++++++++ pkg/metrics/nginx_test.go | 51 +++++++++++++ 6 files changed, 319 insertions(+) create mode 100644 artifacts/nginx/canary.yaml create mode 100644 artifacts/nginx/deployment.yaml create mode 100644 artifacts/nginx/ingress.yaml create mode 100644 pkg/metrics/nginx.go create mode 100644 pkg/metrics/nginx_test.go diff --git a/Makefile b/Makefile index 3f656da4..262f98b9 100644 --- a/Makefile +++ b/Makefile @@ -18,6 +18,12 @@ run-appmesh: -slack-url=https://hooks.slack.com/services/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 \ -slack-channel="devops-alerts" +run-nginx: + go run cmd/flagger/* -kubeconfig=$$HOME/.kube/config -log-level=info -mesh-provider=nginx -namespace=nginx \ + -metrics-server=http://prometheus-weave.istio.weavedx.com \ + -slack-url=https://hooks.slack.com/services/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 \ + -slack-channel="devops-alerts" + build: docker build -t weaveworks/flagger:$(TAG) . -f Dockerfile diff --git a/artifacts/nginx/canary.yaml b/artifacts/nginx/canary.yaml new file mode 100644 index 00000000..13eb15c9 --- /dev/null +++ b/artifacts/nginx/canary.yaml @@ -0,0 +1,54 @@ +apiVersion: flagger.app/v1alpha3 +kind: Canary +metadata: + name: podinfo + namespace: test +spec: + # deployment reference + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: podinfo + # ingress reference + ingressRef: + apiVersion: extensions/v1beta1 + kind: Ingress + name: podinfo + # the maximum time in seconds for the canary deployment + # to make progress before it is rollback (default 600s) + progressDeadlineSeconds: 60 + service: + # container port + port: 9898 + canaryAnalysis: + # schedule interval (default 60s) + interval: 10s + # max number of failed metric checks before rollback + threshold: 10 + # max traffic percentage routed to canary + # percentage (0-100) + maxWeight: 50 + # canary increment step + # percentage (0-100) + stepWeight: 5 + # NGINX Prometheus checks + metrics: + - name: request-success-rate + # minimum req success rate (non 5xx responses) + # percentage (0-100) + threshold: 99 + interval: 1m + - name: request-duration + # maximum avg req duration + # milliseconds + threshold: 500 + interval: 1m + # external checks (optional) + webhooks: + - name: load-test + url: http://flagger-loadtester.test/ + timeout: 5s + metadata: + type: cmd + cmd: "hey -z 1m -q 10 -c 2 http://app.example.com/" + logCmdOutput: "true" diff --git a/artifacts/nginx/deployment.yaml b/artifacts/nginx/deployment.yaml new file mode 100644 index 00000000..814dd9c2 --- /dev/null +++ b/artifacts/nginx/deployment.yaml @@ -0,0 +1,69 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: podinfo + namespace: test + labels: + app: podinfo +spec: + replicas: 1 + strategy: + rollingUpdate: + maxUnavailable: 0 + type: RollingUpdate + selector: + matchLabels: + app: podinfo + template: + metadata: + annotations: + prometheus.io/scrape: "true" + labels: + app: podinfo + spec: + containers: + - name: podinfod + image: quay.io/stefanprodan/podinfo:1.4.0 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9898 + name: http + protocol: TCP + command: + - ./podinfo + - --port=9898 + - --level=info + - --random-delay=false + - --random-error=false + env: + - name: PODINFO_UI_COLOR + value: green + livenessProbe: + exec: + command: + - podcli + - check + - http + - localhost:9898/healthz + failureThreshold: 3 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 2 + readinessProbe: + exec: + command: + - podcli + - check + - http + - localhost:9898/readyz + failureThreshold: 3 + periodSeconds: 3 + successThreshold: 1 + timeoutSeconds: 2 + resources: + limits: + cpu: 1000m + memory: 256Mi + requests: + cpu: 100m + memory: 16Mi diff --git a/artifacts/nginx/ingress.yaml b/artifacts/nginx/ingress.yaml new file mode 100644 index 00000000..5cb6b826 --- /dev/null +++ b/artifacts/nginx/ingress.yaml @@ -0,0 +1,17 @@ +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: podinfo + namespace: test + labels: + app: podinfo + annotations: + kubernetes.io/ingress.class: "nginx" +spec: + rules: + - host: app.exmaple.com + http: + paths: + - backend: + serviceName: podinfo + servicePort: 9898 diff --git a/pkg/metrics/nginx.go b/pkg/metrics/nginx.go new file mode 100644 index 00000000..7c6eb56e --- /dev/null +++ b/pkg/metrics/nginx.go @@ -0,0 +1,122 @@ +package metrics + +import ( + "fmt" + "net/url" + "strconv" + "time" +) + +const nginxSuccessRateQuery = ` +sum(rate( +nginx_ingress_controller_requests{kubernetes_namespace="{{ .Namespace }}", +ingress="{{ .Name }}", +status!~"5.*"} +[{{ .Interval }}])) +/ +sum(rate( +nginx_ingress_controller_requests{kubernetes_namespace="{{ .Namespace }}", +ingress="{{ .Name }}"} +[{{ .Interval }}])) +* 100 +` + +// GetNginxSuccessRate returns the requests success rate (non 5xx) using nginx_ingress_controller_requests metric +func (c *Observer) GetNginxSuccessRate(name string, namespace string, metric string, interval string) (float64, error) { + if c.metricsServer == "fake" { + return 100, nil + } + + meta := struct { + Name string + Namespace string + Interval string + }{ + name, + namespace, + interval, + } + + query, err := render(meta, nginxSuccessRateQuery) + if err != nil { + return 0, err + } + + var rate *float64 + querySt := url.QueryEscape(query) + result, err := c.queryMetric(querySt) + if err != nil { + return 0, err + } + + for _, v := range result.Data.Result { + metricValue := v.Value[1] + switch metricValue.(type) { + case string: + f, err := strconv.ParseFloat(metricValue.(string), 64) + if err != nil { + return 0, err + } + rate = &f + } + } + if rate == nil { + return 0, fmt.Errorf("no values found for metric %s", metric) + } + return *rate, nil +} + +const nginxRequestDurationQuery = ` +sum(rate( +nginx_ingress_controller_ingress_upstream_latency_seconds_sum{kubernetes_namespace="{{ .Namespace }}", +ingress="{{ .Name }}"}[{{ .Interval }}])) +/ +sum(rate(nginx_ingress_controller_ingress_upstream_latency_seconds_count{kubernetes_namespace="{{ .Namespace }}", +ingress="{{ .Name }}"}[{{ .Interval }}])) * 1000 +` + +// GetNginxRequestDuration returns the avg requests latency using nginx_ingress_controller_ingress_upstream_latency_seconds_sum metric +func (c *Observer) GetNginxRequestDuration(name string, namespace string, metric string, interval string) (time.Duration, error) { + if c.metricsServer == "fake" { + return 1, nil + } + + meta := struct { + Name string + Namespace string + Interval string + }{ + name, + namespace, + interval, + } + + query, err := render(meta, nginxRequestDurationQuery) + if err != nil { + return 0, err + } + + var rate *float64 + querySt := url.QueryEscape(query) + result, err := c.queryMetric(querySt) + if err != nil { + return 0, err + } + + for _, v := range result.Data.Result { + metricValue := v.Value[1] + switch metricValue.(type) { + case string: + f, err := strconv.ParseFloat(metricValue.(string), 64) + if err != nil { + return 0, err + } + rate = &f + } + } + if rate == nil { + return 0, fmt.Errorf("no values found for metric %s", metric) + } + ms := time.Duration(int64(*rate)) * time.Millisecond + return ms, nil +} diff --git a/pkg/metrics/nginx_test.go b/pkg/metrics/nginx_test.go new file mode 100644 index 00000000..e2a93c5f --- /dev/null +++ b/pkg/metrics/nginx_test.go @@ -0,0 +1,51 @@ +package metrics + +import ( + "testing" +) + +func Test_NginxSuccessRateQueryRender(t *testing.T) { + meta := struct { + Name string + Namespace string + Interval string + }{ + "podinfo", + "nginx", + "1m", + } + + query, err := render(meta, nginxSuccessRateQuery) + if err != nil { + t.Fatal(err) + } + + expected := `sum(rate(nginx_ingress_controller_requests{kubernetes_namespace="nginx",ingress="podinfo",status!~"5.*"}[1m])) / sum(rate(nginx_ingress_controller_requests{kubernetes_namespace="nginx",ingress="podinfo"}[1m])) * 100` + + if query != expected { + t.Errorf("\nGot %s \nWanted %s", query, expected) + } +} + +func Test_NginxRequestDurationQueryRender(t *testing.T) { + meta := struct { + Name string + Namespace string + Interval string + }{ + "podinfo", + "nginx", + "1m", + } + + query, err := render(meta, nginxRequestDurationQuery) + if err != nil { + t.Fatal(err) + } + + expected := `sum(rate(nginx_ingress_controller_ingress_upstream_latency_seconds_sum{kubernetes_namespace="nginx",ingress="podinfo"}[1m])) /sum(rate(nginx_ingress_controller_ingress_upstream_latency_seconds_count{kubernetes_namespace="nginx",ingress="podinfo"}[1m])) * 1000` + + if query != expected { + t.Errorf("\nGot %s \nWanted %s", query, expected) + } +}