Merge pull request #2956 from weaveworks/sizing-merge-slice

optimisation: allocate less memory in LatestMap merging

app/benchmark_internal_test.go

@@ -4,6 +4,8 @@ import (
 	"flag"
 	"net/http"
 	"net/url"
+	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/weaveworks/scope/render"
@@ -12,9 +14,59 @@ import (
 )
 
 var (
-	benchReportFile = flag.String("bench-report-file", "", "report file to use for benchmarking (relative to this package)")
+	benchReportPath = flag.String("bench-report-path", "", "report file, or dir with files, to use for benchmarking (relative to this package)")
 )
 
+func readReportFiles(path string) ([]report.Report, error) {
+	reports := []report.Report{}
+	if err := filepath.Walk(path,
+		func(p string, info os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+			if info.IsDir() {
+				return nil
+			}
+			rpt, err := report.MakeFromFile(p)
+			if err != nil {
+				return err
+			}
+			reports = append(reports, rpt)
+			return nil
+		}); err != nil {
+		return nil, err
+	}
+	return reports, nil
+}
+
+func BenchmarkReportUnmarshal(b *testing.B) {
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		b.StopTimer()
+		b.StartTimer()
+		if _, err := readReportFiles(*benchReportPath); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkReportMerge(b *testing.B) {
+	reports, err := readReportFiles(*benchReportPath)
+	if err != nil {
+		b.Fatal(err)
+	}
+	merger := NewSmartMerger()
+
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		b.StopTimer()
+		b.StartTimer()
+		merger.Merge(reports)
+	}
+}
+
 func BenchmarkTopologyList(b *testing.B) {
 	benchmarkRender(b, func(report report.Report) {
 		request := &http.Request{
@@ -26,11 +78,12 @@ func BenchmarkTopologyList(b *testing.B) {
 
 func benchmarkRender(b *testing.B, f func(report.Report)) {
 	r := fixture.Report
-	if *benchReportFile != "" {
-		var err error
-		if r, err = report.MakeFromFile(*benchReportFile); err != nil {
+	if *benchReportPath != "" {
+		reports, err := readReportFiles(*benchReportPath)
+		if err != nil {
 			b.Fatal(err)
 		}
+		r = NewSmartMerger().Merge(reports)
 	}
 
 	b.ReportAllocs()
@@ -48,10 +101,22 @@ func BenchmarkTopologyHosts(b *testing.B) {
 	benchmarkOneTopology(b, "hosts")
 }
 
+func BenchmarkTopologyControllers(b *testing.B) {
+	benchmarkOneTopology(b, "kube-controllers")
+}
+
+func BenchmarkTopologyPods(b *testing.B) {
+	benchmarkOneTopology(b, "pods")
+}
+
 func BenchmarkTopologyContainers(b *testing.B) {
 	benchmarkOneTopology(b, "containers")
 }
 
+func BenchmarkTopologyProcesses(b *testing.B) {
+	benchmarkOneTopology(b, "processes")
+}
+
 func benchmarkOneTopology(b *testing.B, topologyID string) {
 	benchmarkRender(b, func(report report.Report) {
 		renderer, filter, err := topologyRegistry.RendererForTopology(topologyID, url.Values{}, report)

extras/generate_latest_map

@@ -83,7 +83,11 @@ function generate_latest_map() {
         case n.entries == nil:
             return m
         }
-        out := make([]${entry_type}, 0, len(m.entries)+len(n.entries))
+        l := len(m.entries)
+        if len(n.entries) > l {
+            l = len(n.entries)
+        }
+        out := make([]${entry_type}, 0, l)
 
         i, j := 0, 0
         for i < len(m.entries) {

report/benchmark_internal_test.go

@@ -1,40 +0,0 @@
-package report
-
-import (
-	"flag"
-	"os"
-	"path/filepath"
-	"testing"
-)
-
-var (
-	benchReportPath = flag.String("bench-report-path", "", "report file, or dir with files, to use for benchmarking (relative to this package)")
-)
-
-func BenchmarkReportUnmarshal(b *testing.B) {
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		b.StopTimer()
-		b.StartTimer()
-		if err := readReportFiles(*benchReportPath); err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
-func readReportFiles(path string) error {
-	return filepath.Walk(path,
-		func(p string, info os.FileInfo, err error) error {
-			if err != nil {
-				return err
-			}
-			if info.IsDir() {
-				return nil
-			}
-			if _, err := MakeFromFile(p); err != nil {
-				return err
-			}
-			return nil
-		})
-}

report/latest_map_generated.go

@@ -51,7 +51,11 @@ func (m StringLatestMap) Merge(n StringLatestMap) StringLatestMap {
 	case n.entries == nil:
 		return m
 	}
-	out := make([]stringLatestEntry, 0, len(m.entries)+len(n.entries))
+	l := len(m.entries)
+	if len(n.entries) > l {
+		l = len(n.entries)
+	}
+	out := make([]stringLatestEntry, 0, l)
 
 	i, j := 0, 0
 	for i < len(m.entries) {
@@ -268,7 +272,11 @@ func (m NodeControlDataLatestMap) Merge(n NodeControlDataLatestMap) NodeControlD
 	case n.entries == nil:
 		return m
 	}
-	out := make([]nodeControlDataLatestEntry, 0, len(m.entries)+len(n.entries))
+	l := len(m.entries)
+	if len(n.entries) > l {
+		l = len(n.entries)
+	}
+	out := make([]nodeControlDataLatestEntry, 0, l)
 
 	i, j := 0, 0
 	for i < len(m.entries) {