From f26c875c4c1ac68b9e6e3250bc57b6f2a45ea50c Mon Sep 17 00:00:00 2001 From: Roland Schilter Date: Wed, 16 Aug 2017 17:08:51 +0100 Subject: [PATCH] Filter by namespace in metric queries (#2819) And get rid of B -> MB conversion Fixes #2817 --- render/detailed/links.go | 28 ++++++++++++------- render/detailed/links_test.go | 52 ++++++++++++++++++++++++++++++----- 2 files changed, 63 insertions(+), 17 deletions(-) diff --git a/render/detailed/links.go b/render/detailed/links.go index 000f588b9..9dd175d5f 100644 --- a/render/detailed/links.go +++ b/render/detailed/links.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/weaveworks/scope/probe/docker" + "github.com/weaveworks/scope/probe/kubernetes" "github.com/weaveworks/scope/report" "github.com/ugorji/go/codec" @@ -46,22 +47,22 @@ var ( // Queries on pod names of the format `name--` // See also: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#pod-template-hash-label - podIDHashQueries = formatMetricQueries(`pod_name=~"^{{label}}-[^-]+-[^-]+$"`, []string{docker.MemoryUsage, docker.CPUTotalUsage}) + podIDHashQueries = formatMetricQueries(`pod_name=~"^{{label}}-[^-]+-[^-]+$",namespace="{{namespace}}"`, []string{docker.MemoryUsage, docker.CPUTotalUsage}) // Prometheus queries for topologies topologyQueries = map[string]map[string]string{ // Containers - report.Container: formatMetricQueries(`container_name="{{label}}"`, []string{docker.MemoryUsage, docker.CPUTotalUsage}), + report.Container: formatMetricQueries(`name="{{containerName}}"`, []string{docker.MemoryUsage, docker.CPUTotalUsage}), report.ContainerImage: formatMetricQueries(`image="{{label}}"`, []string{docker.MemoryUsage, docker.CPUTotalUsage}), // Kubernetes topologies report.Pod: formatMetricQueries( - `pod_name="{{label}}"`, + `pod_name="{{label}}",namespace="{{namespace}}"`, []string{docker.MemoryUsage, docker.CPUTotalUsage}, ), - report.DaemonSet: formatMetricQueries(`pod_name=~"^{{label}}-[^-]+$"`, []string{docker.MemoryUsage, docker.CPUTotalUsage}), + report.DaemonSet: formatMetricQueries(`pod_name=~"^{{label}}-[^-]+$",namespace="{{namespace}}"`, []string{docker.MemoryUsage, docker.CPUTotalUsage}), report.Deployment: podIDHashQueries, report.StatefulSet: podIDHashQueries, report.CronJob: podIDHashQueries, @@ -70,8 +71,8 @@ var ( // NB: Pods need to be labeled and selected by their respective Service name, meaning: // - The Service's `spec.selector` needs to select on `name` // - The Service's `metadata.name` needs to be the same value as `spec.selector.name` - docker.CPUTotalUsage: `namespace_label_name:container_cpu_usage_seconds_total:sum_rate{label_name="{{label}}"}/1024/1024`, - docker.MemoryUsage: `namespace_label_name:container_memory_usage_bytes:sum{label_name="{{label}}"}`, + docker.CPUTotalUsage: `namespace_label_name:container_cpu_usage_seconds_total:sum_rate{label_name="{{label}}",namespace="{{namespace}}"}`, + docker.MemoryUsage: `namespace_label_name:container_memory_usage_bytes:sum{label_name="{{label}}",namespace="{{namespace}}"}`, }, } ) @@ -82,7 +83,7 @@ func formatMetricQueries(filter string, ids []string) map[string]string { // All `container_*`metrics are provided by cAdvisor in Kubelets switch id { case docker.MemoryUsage: - queries[id] = fmt.Sprintf("sum(container_memory_usage_bytes{%s})/1024/1024", filter) + queries[id] = fmt.Sprintf("sum(container_memory_usage_bytes{%s})", filter) case docker.CPUTotalUsage: queries[id] = fmt.Sprintf( "sum(rate(container_cpu_usage_seconds_total{%s}[1m]))/count(container_cpu_usage_seconds_total{%s})*100", @@ -90,9 +91,9 @@ func formatMetricQueries(filter string, ids []string) map[string]string { filter, ) case idReceiveBytes: - queries[id] = fmt.Sprintf(`sum(rate(container_network_receive_bytes_total{%s}[5m]))`, filter) + queries[id] = fmt.Sprintf("sum(rate(container_network_receive_bytes_total{%s}[5m]))", filter) case idTransmitBytes: - queries[id] = fmt.Sprintf(`sum(rate(container_network_transmit_bytes_total{%s}[5m]))`, filter) + queries[id] = fmt.Sprintf("sum(rate(container_network_transmit_bytes_total{%s}[5m]))", filter) } } @@ -160,7 +161,14 @@ func metricQuery(summary NodeSummary, n report.Node, metricID string) string { return "" } - return strings.Replace(queries[metricID], "{{label}}", summary.Label, -1) + namespace, _ := n.Latest.Lookup(kubernetes.Namespace) + name, _ := n.Latest.Lookup(docker.ContainerName) + r := strings.NewReplacer( + "{{label}}", summary.Label, + "{{namespace}}", namespace, + "{{containerName}}", name, + ) + return r.Replace(queries[metricID]) } // metricURL builds the URL by embedding it into the configured `metricsGraphURL`. diff --git a/render/detailed/links_test.go b/render/detailed/links_test.go index 91b3f3291..cce6fc01c 100644 --- a/render/detailed/links_test.go +++ b/render/detailed/links_test.go @@ -1,10 +1,12 @@ package detailed_test import ( + "net/url" "strings" "testing" "github.com/weaveworks/scope/probe/docker" + "github.com/weaveworks/scope/probe/kubernetes" "github.com/weaveworks/scope/render/detailed" "github.com/weaveworks/scope/report" @@ -17,8 +19,13 @@ const ( var ( sampleUnknownNode = report.MakeNode("???").WithTopology("foo") - samplePodNode = report.MakeNode("noo").WithTopology(report.Pod) - sampleMetrics = []report.MetricRow{ + samplePodNode = report.MakeNode("noo"). + WithTopology(report.Pod). + WithLatests(map[string]string{kubernetes.Namespace: "noospace"}) + sampleContainerNode = report.MakeNode("coo"). + WithTopology(report.Container). + WithLatests(map[string]string{docker.ContainerName: "cooname"}) + sampleMetrics = []report.MetricRow{ {ID: docker.MemoryUsage}, {ID: docker.CPUTotalUsage}, } @@ -40,14 +47,39 @@ func TestRenderMetricURLs_UnknownTopology(t *testing.T) { assert.Empty(t, result.Metrics[1].URL) } -func TestRenderMetricURLs(t *testing.T) { +func TestRenderMetricURLs_Pod(t *testing.T) { s := detailed.NodeSummary{Label: "foo", Metrics: sampleMetrics} result := detailed.RenderMetricURLs(s, samplePodNode, sampleMetricsGraphURL) assert.Equal(t, 0, strings.Index(result.Metrics[0].URL, sampleMetricsGraphURL)) - assert.Contains(t, result.Metrics[0].URL, "container_memory_usage_bytes%7Bpod_name%3D%5C%22foo%5C%22%7D") + // Double quotes are escaped since these are json marshaled strings + contains := []string{"container_memory_usage_bytes", `pod_name=\"foo\"`, `namespace=\"noospace\"`} + for _, contain := range contains { + assert.Contains(t, result.Metrics[0].URL, url.QueryEscape(contain)) + } + assert.Equal(t, 0, strings.Index(result.Metrics[1].URL, sampleMetricsGraphURL)) - assert.Contains(t, result.Metrics[1].URL, "container_cpu_usage_seconds_total%7Bpod_name%3D%5C%22foo%5C%22%7D") + contains = []string{"container_cpu_usage_seconds", `pod_name=\"foo\"`, `namespace=\"noospace\"`} + for _, contain := range contains { + assert.Contains(t, result.Metrics[1].URL, url.QueryEscape(contain)) + } +} + +func TestRenderMetricURLs_Container(t *testing.T) { + s := detailed.NodeSummary{Label: "foo", Metrics: sampleMetrics} + result := detailed.RenderMetricURLs(s, sampleContainerNode, sampleMetricsGraphURL) + + assert.Equal(t, 0, strings.Index(result.Metrics[0].URL, sampleMetricsGraphURL)) + contains := []string{"container_memory_usage_bytes", `name=\"cooname\"`} + for _, contain := range contains { + assert.Contains(t, result.Metrics[0].URL, url.QueryEscape(contain)) + } + + assert.Equal(t, 0, strings.Index(result.Metrics[1].URL, sampleMetricsGraphURL)) + contains = []string{"container_cpu_usage_seconds", `name=\"cooname\"`} + for _, contain := range contains { + assert.Contains(t, result.Metrics[1].URL, url.QueryEscape(contain)) + } } func TestRenderMetricURLs_EmptyMetrics(t *testing.T) { @@ -86,7 +118,13 @@ func TestRenderMetricURLs_QueryReplacement(t *testing.T) { result := detailed.RenderMetricURLs(s, samplePodNode, "http://example.test/?q=:query") assert.Contains(t, result.Metrics[0].URL, "http://example.test/?q=") - assert.Contains(t, result.Metrics[0].URL, "container_memory_usage_bytes%7Bpod_name%3D%22foo%22%7D") + contains := []string{"container_memory_usage_bytes", `pod_name="foo"`, `namespace="noospace"`} + for _, contain := range contains { + assert.Contains(t, result.Metrics[0].URL, url.QueryEscape(contain)) + } assert.Contains(t, result.Metrics[1].URL, "http://example.test/?q=") - assert.Contains(t, result.Metrics[1].URL, "container_cpu_usage_seconds_total%7Bpod_name%3D%22foo%22%7D") + contains = []string{"container_cpu_usage_seconds", `pod_name="foo"`, `namespace="noospace"`} + for _, contain := range contains { + assert.Contains(t, result.Metrics[1].URL, url.QueryEscape(contain)) + } }