diff --git a/app/api_topologies.go b/app/api_topologies.go index 15b094044..1d8594be7 100644 --- a/app/api_topologies.go +++ b/app/api_topologies.go @@ -189,23 +189,25 @@ func (r *registry) walk(f func(APITopologyDesc)) { // makeTopologyList returns a handler that yields an APITopologyList. func (r *registry) makeTopologyList(rep Reporter) func(w http.ResponseWriter, r *http.Request) { return func(w http.ResponseWriter, req *http.Request) { - var ( - rpt = rep.Report() - topologies = []APITopologyDesc{} - ) - r.walk(func(desc APITopologyDesc) { - renderer := renderedForRequest(req, desc) - desc.Stats = decorateWithStats(rpt, renderer) - for i := range desc.SubTopologies { - renderer := renderedForRequest(req, desc.SubTopologies[i]) - desc.SubTopologies[i].Stats = decorateWithStats(rpt, renderer) - } - topologies = append(topologies, desc) - }) + topologies := r.renderTopologies(rep.Report(), req) respondWith(w, http.StatusOK, topologies) } } +func (r *registry) renderTopologies(rpt report.Report, req *http.Request) []APITopologyDesc { + topologies := []APITopologyDesc{} + r.walk(func(desc APITopologyDesc) { + renderer := renderedForRequest(req, desc) + desc.Stats = decorateWithStats(rpt, renderer) + for i := range desc.SubTopologies { + renderer := renderedForRequest(req, desc.SubTopologies[i]) + desc.SubTopologies[i].Stats = decorateWithStats(rpt, renderer) + } + topologies = append(topologies, desc) + }) + return topologies +} + func decorateWithStats(rpt report.Report, renderer render.Renderer) topologyStats { var ( nodes int diff --git a/app/benchmark_internal_test.go b/app/benchmark_internal_test.go new file mode 100644 index 000000000..ccc5ff4a8 --- /dev/null +++ b/app/benchmark_internal_test.go @@ -0,0 +1,55 @@ +package app + +import ( + "encoding/json" + "flag" + "io/ioutil" + "net/http" + "net/url" + "testing" + + "github.com/weaveworks/scope/render" + "github.com/weaveworks/scope/report" + "github.com/weaveworks/scope/test/fixture" +) + +// StaticReport is used as a fixture in tests. It emulates an xfer.Collector. +type StaticReporter struct{ r report.Report } + +func (s StaticReporter) Report() report.Report { return s.r } +func (s StaticReporter) WaitOn(chan struct{}) {} +func (s StaticReporter) UnWait(chan struct{}) {} + +var ( + benchReportFile = flag.String("bench-report-file", "", "json report file to use for benchmarking (relative to this package)") +) + +func loadReport() (report.Report, error) { + if *benchReportFile == "" { + return fixture.Report, nil + } + + b, err := ioutil.ReadFile(*benchReportFile) + if err != nil { + return fixture.Report, err + } + rpt := report.MakeReport() + err = json.Unmarshal(b, &rpt) + return rpt, err +} + +func BenchmarkTopologyList(b *testing.B) { + report, err := loadReport() + if err != nil { + b.Fatal(err) + } + b.ReportAllocs() + b.ResetTimer() + request := &http.Request{ + Form: url.Values{}, + } + for i := 0; i < b.N; i++ { + render.ResetCache() + topologyRegistry.renderTopologies(report, request) + } +} diff --git a/app/collector.go b/app/collector.go index aebcb08e0..fff99788c 100644 --- a/app/collector.go +++ b/app/collector.go @@ -1,9 +1,12 @@ package app import ( + "fmt" "sync" "time" + "github.com/spaolacci/murmur3" + "github.com/weaveworks/scope/report" ) @@ -33,6 +36,7 @@ type collector struct { mtx sync.Mutex reports []timestampReport window time.Duration + cached *report.Report waitableCondition } @@ -83,6 +87,7 @@ func (c *collector) Add(rpt report.Report) { defer c.mtx.Unlock() c.reports = append(c.reports, timestampReport{now(), rpt}) c.reports = clean(c.reports, c.window) + c.cached = nil if rpt.Shortcut { c.Broadcast() } @@ -94,12 +99,24 @@ func (c *collector) Report() report.Report { c.mtx.Lock() defer c.mtx.Unlock() + // If the oldest report is still within range, + // and there is a cached report, return that. + if c.cached != nil && len(c.reports) > 0 { + oldest := now().Add(-c.window) + if c.reports[0].timestamp.Before(oldest) { + return *c.cached + } + } c.reports = clean(c.reports, c.window) rpt := report.MakeReport() + id := murmur3.New64() for _, tr := range c.reports { rpt = rpt.Merge(tr.report) + id.Write([]byte(tr.report.ID)) } + rpt.ID = fmt.Sprintf("%x", id.Sum64()) + c.cached = &rpt return rpt } diff --git a/app/collector_test.go b/app/collector_test.go index 9e77fcb9e..672e4f433 100644 --- a/app/collector_test.go +++ b/app/collector_test.go @@ -1,13 +1,13 @@ package app_test import ( - "reflect" "testing" "time" "github.com/weaveworks/scope/app" "github.com/weaveworks/scope/report" "github.com/weaveworks/scope/test" + "github.com/weaveworks/scope/test/reflect" ) func TestCollector(t *testing.T) { diff --git a/probe/probe_internal_test.go b/probe/probe_internal_test.go index f27e7e92c..51a6fff63 100644 --- a/probe/probe_internal_test.go +++ b/probe/probe_internal_test.go @@ -4,12 +4,12 @@ import ( "compress/gzip" "encoding/gob" "io" - "reflect" "testing" "time" "github.com/weaveworks/scope/report" "github.com/weaveworks/scope/test" + "github.com/weaveworks/scope/test/reflect" ) func TestApply(t *testing.T) { diff --git a/render/benchmark_test.go b/render/benchmark_test.go index baad3f498..3c53148d5 100644 --- a/render/benchmark_test.go +++ b/render/benchmark_test.go @@ -61,6 +61,7 @@ func benchmarkRender(b *testing.B, r render.Renderer) { b.ResetTimer() for i := 0; i < b.N; i++ { + render.ResetCache() benchmarkRenderResult = r.Render(report) if len(benchmarkRenderResult) == 0 { b.Errorf("Rendered topology contained no nodes") @@ -78,6 +79,7 @@ func benchmarkStats(b *testing.B, r render.Renderer) { for i := 0; i < b.N; i++ { // No way to tell if this was successful :( + render.ResetCache() benchmarkStatsResult = r.Stats(report) } } @@ -87,11 +89,11 @@ func loadReport() (report.Report, error) { return fixture.Report, nil } - var rpt report.Report b, err := ioutil.ReadFile(*benchReportFile) if err != nil { return rpt, err } + rpt := report.MakeReport() err = json.Unmarshal(b, &rpt) return rpt, err } diff --git a/render/filters.go b/render/filters.go index 205306764..9feda297d 100644 --- a/render/filters.go +++ b/render/filters.go @@ -60,17 +60,22 @@ type Filter struct { FilterFunc func(RenderableNode) bool } +// MakeFilter makes a new Filter. +func MakeFilter(f func(RenderableNode) bool, r Renderer) Renderer { + return &Filter{r, f} +} + // Render implements Renderer -func (f Filter) Render(rpt report.Report) RenderableNodes { +func (f *Filter) Render(rpt report.Report) RenderableNodes { nodes, _ := f.render(rpt) return nodes } -func (f Filter) render(rpt report.Report) (RenderableNodes, int) { +func (f *Filter) render(rpt report.Report) (RenderableNodes, int) { output := RenderableNodes{} inDegrees := map[string]int{} filtered := 0 - for id, node := range f.Renderer.Render(rpt) { + for id, node := range memoisedRender(f.Renderer, rpt) { if f.FilterFunc(node) { output[id] = node inDegrees[id] = 0 @@ -122,24 +127,24 @@ const IsConnected = "is_connected" // FilterPseudo produces a renderer that removes pseudo nodes from the given // renderer func FilterPseudo(r Renderer) Renderer { - return Filter{ - Renderer: r, - FilterFunc: func(node RenderableNode) bool { + return MakeFilter( + func(node RenderableNode) bool { return !node.Pseudo }, - } + r, + ) } // FilterUnconnected produces a renderer that filters unconnected nodes // from the given renderer func FilterUnconnected(r Renderer) Renderer { - return Filter{ - Renderer: ColorConnected(r), - FilterFunc: func(node RenderableNode) bool { + return MakeFilter( + func(node RenderableNode) bool { _, ok := node.Latest.Lookup(IsConnected) return ok }, - } + ColorConnected(r), + ) } // FilterNoop does nothing. @@ -149,20 +154,19 @@ func FilterNoop(in Renderer) Renderer { // FilterStopped filters out stopped containers. func FilterStopped(r Renderer) Renderer { - return Filter{ - Renderer: r, - FilterFunc: func(node RenderableNode) bool { + return MakeFilter( + func(node RenderableNode) bool { containerState, ok := node.Latest.Lookup(docker.ContainerState) return !ok || containerState != docker.StateStopped }, - } + r, + ) } // FilterSystem is a Renderer which filters out system nodes. func FilterSystem(r Renderer) Renderer { - return Filter{ - Renderer: r, - FilterFunc: func(node RenderableNode) bool { + return MakeFilter( + func(node RenderableNode) bool { containerName, _ := node.Latest.Lookup(docker.ContainerName) if _, ok := systemContainerNames[containerName]; ok { return false @@ -186,7 +190,8 @@ func FilterSystem(r Renderer) Renderer { } return true }, - } + r, + ) } var systemContainerNames = map[string]struct{}{ diff --git a/render/render.go b/render/render.go index 5cb96ee19..f2e3ec6da 100644 --- a/render/render.go +++ b/render/render.go @@ -1,9 +1,38 @@ package render import ( + "fmt" + "reflect" + + "github.com/bluele/gcache" + "github.com/weaveworks/scope/report" ) +var renderCache = gcache.New(100).LRU().Build() + +func memoisedRender(r Renderer, rpt report.Report) RenderableNodes { + key := "" + v := reflect.ValueOf(r) + switch v.Kind() { + case reflect.Ptr, reflect.Func: + key = fmt.Sprintf("%s-%x", rpt.ID, v.Pointer()) + default: + return r.Render(rpt) + } + if result, err := renderCache.Get(key); err == nil { + return result.(RenderableNodes) + } + output := r.Render(rpt) + renderCache.Set(key, output) + return output +} + +// ResetCache blows away the rendered node cache. +func ResetCache() { + renderCache.Purge() +} + // Renderer is something that can render a report to a set of RenderableNodes. type Renderer interface { Render(report.Report) RenderableNodes @@ -27,22 +56,23 @@ type Reduce []Renderer // MakeReduce is the only sane way to produce a Reduce Renderer. func MakeReduce(renderers ...Renderer) Renderer { - return Reduce(renderers) + r := Reduce(renderers) + return &r } // Render produces a set of RenderableNodes given a Report. -func (r Reduce) Render(rpt report.Report) RenderableNodes { +func (r *Reduce) Render(rpt report.Report) RenderableNodes { result := RenderableNodes{} - for _, renderer := range r { - result = result.Merge(renderer.Render(rpt)) + for _, renderer := range *r { + result = result.Merge(memoisedRender(renderer, rpt)) } return result } // Stats implements Renderer -func (r Reduce) Stats(rpt report.Report) Stats { +func (r *Reduce) Stats(rpt report.Report) Stats { var result Stats - for _, renderer := range r { + for _, renderer := range *r { result = result.merge(renderer.Stats(rpt)) } return result @@ -55,24 +85,29 @@ type Map struct { Renderer } +// MakeMap makes a new Map +func MakeMap(f MapFunc, r Renderer) Renderer { + return &Map{f, r} +} + // Render transforms a set of RenderableNodes produces by another Renderer. // using a map function -func (m Map) Render(rpt report.Report) RenderableNodes { +func (m *Map) Render(rpt report.Report) RenderableNodes { output, _ := m.render(rpt) return output } // Stats implements Renderer -func (m Map) Stats(rpt report.Report) Stats { +func (m *Map) Stats(rpt report.Report) Stats { // There doesn't seem to be an instance where we want stats to recurse // through Maps - for instance we don't want to see the number of filtered // processes in the container renderer. return Stats{} } -func (m Map) render(rpt report.Report) (RenderableNodes, map[string]report.IDList) { +func (m *Map) render(rpt report.Report) (RenderableNodes, map[string]report.IDList) { var ( - input = m.Renderer.Render(rpt) + input = memoisedRender(m.Renderer, rpt) output = RenderableNodes{} mapped = map[string]report.IDList{} // input node ID -> output node IDs adjacencies = map[string]report.IDList{} // output node ID -> input node Adjacencies diff --git a/render/topologies.go b/render/topologies.go index 4c88c241b..ab68bc2e3 100644 --- a/render/topologies.go +++ b/render/topologies.go @@ -9,22 +9,22 @@ import ( ) // EndpointRenderer is a Renderer which produces a renderable endpoint graph. -var EndpointRenderer = Map{ - MapFunc: MapEndpointIdentity, - Renderer: SelectEndpoint, -} +var EndpointRenderer = MakeMap( + MapEndpointIdentity, + SelectEndpoint, +) // ProcessRenderer is a Renderer which produces a renderable process // graph by merging the endpoint graph and the process topology. var ProcessRenderer = MakeReduce( - Map{ - MapFunc: MapEndpoint2Process, - Renderer: EndpointRenderer, - }, - Map{ - MapFunc: MapProcessIdentity, - Renderer: SelectProcess, - }, + MakeMap( + MapEndpoint2Process, + EndpointRenderer, + ), + MakeMap( + MapProcessIdentity, + SelectProcess, + ), ) // processWithContainerNameRenderer is a Renderer which produces a process @@ -35,10 +35,10 @@ type processWithContainerNameRenderer struct { func (r processWithContainerNameRenderer) Render(rpt report.Report) RenderableNodes { processes := r.Renderer.Render(rpt) - containers := Map{ - MapFunc: MapContainerIdentity, - Renderer: SelectContainer, - }.Render(rpt) + containers := MakeMap( + MapContainerIdentity, + SelectContainer, + ).Render(rpt) for id, p := range processes { pid, ok := p.Node.Latest.Lookup(process.PID) @@ -66,13 +66,13 @@ var ProcessWithContainerNameRenderer = processWithContainerNameRenderer{ProcessR // ProcessNameRenderer is a Renderer which produces a renderable process // name graph by munging the progess graph. -var ProcessNameRenderer = Map{ - MapFunc: MapCountProcessName, - Renderer: Map{ - MapFunc: MapProcess2Name, - Renderer: ProcessRenderer, - }, -} +var ProcessNameRenderer = MakeMap( + MapCountProcessName, + MakeMap( + MapProcess2Name, + ProcessRenderer, + ), +) // ContainerRenderer is a Renderer which produces a renderable container // graph by merging the process graph and the container topology. @@ -80,40 +80,40 @@ var ProcessNameRenderer = Map{ // but we need to be careful to ensure we only include each edge once, by only // including the ProcessRenderer once. var ContainerRenderer = MakeReduce( - Filter{ - FilterFunc: func(n RenderableNode) bool { + MakeFilter( + func(n RenderableNode) bool { _, inContainer := n.Node.Latest.Lookup(docker.ContainerID) _, isConnected := n.Node.Latest.Lookup(IsConnected) return inContainer || isConnected }, - Renderer: Map{ - MapFunc: MapProcess2Container, - Renderer: ColorConnected(ProcessRenderer), - }, - }, + MakeMap( + MapProcess2Container, + ColorConnected(ProcessRenderer), + ), + ), // This mapper brings in short lived connections by joining with container IPs. // We need to be careful to ensure we only include each edge once. Edges brought in // by the above renders will have a pid, so its enough to filter out any nodes with // pids. - FilterUnconnected(Map{ - MapFunc: MapIP2Container, - Renderer: MakeReduce( - Map{ - MapFunc: MapContainer2IP, - Renderer: SelectContainer, - }, - Map{ - MapFunc: MapEndpoint2IP, - Renderer: SelectEndpoint, - }, + FilterUnconnected(MakeMap( + MapIP2Container, + MakeReduce( + MakeMap( + MapContainer2IP, + SelectContainer, + ), + MakeMap( + MapEndpoint2IP, + SelectEndpoint, + ), ), - }), + )), - Map{ - MapFunc: MapContainerIdentity, - Renderer: SelectContainer, - }, + MakeMap( + MapContainerIdentity, + SelectContainer, + ), ) type containerWithImageNameRenderer struct { @@ -125,10 +125,10 @@ type containerWithImageNameRenderer struct { // container metadata. func (r containerWithImageNameRenderer) Render(rpt report.Report) RenderableNodes { containers := r.Renderer.Render(rpt) - images := Map{ - MapFunc: MapContainerImageIdentity, - Renderer: SelectContainerImage, - }.Render(rpt) + images := MakeMap( + MapContainerImageIdentity, + SelectContainerImage, + ).Render(rpt) for id, c := range containers { imageID, ok := c.Node.Latest.Lookup(docker.ImageID) @@ -153,99 +153,99 @@ var ContainerWithImageNameRenderer = containerWithImageNameRenderer{ContainerRen // ContainerImageRenderer is a Renderer which produces a renderable container // image graph by merging the container graph and the container image topology. -var ContainerImageRenderer = Map{ - MapFunc: MapCountContainers, - Renderer: Map{ - MapFunc: MapContainerImage2Name, - Renderer: MakeReduce( - Map{ - MapFunc: MapContainer2ContainerImage, - Renderer: ContainerRenderer, - }, - Map{ - MapFunc: MapContainerImageIdentity, - Renderer: SelectContainerImage, - }, +var ContainerImageRenderer = MakeMap( + MapCountContainers, + MakeMap( + MapContainerImage2Name, + MakeReduce( + MakeMap( + MapContainer2ContainerImage, + ContainerRenderer, + ), + MakeMap( + MapContainerImageIdentity, + SelectContainerImage, + ), ), - }, -} + ), +) // ContainerHostnameRenderer is a Renderer which produces a renderable container // by hostname graph.. -var ContainerHostnameRenderer = Map{ - MapFunc: MapCountContainers, - Renderer: Map{ - MapFunc: MapContainer2Hostname, - Renderer: ContainerRenderer, - }, -} +var ContainerHostnameRenderer = MakeMap( + MapCountContainers, + MakeMap( + MapContainer2Hostname, + ContainerRenderer, + ), +) // AddressRenderer is a Renderer which produces a renderable address // graph from the address topology. -var AddressRenderer = Map{ - MapFunc: MapAddressIdentity, - Renderer: SelectAddress, -} +var AddressRenderer = MakeMap( + MapAddressIdentity, + SelectAddress, +) // HostRenderer is a Renderer which produces a renderable host // graph from the host topology and address graph. var HostRenderer = MakeReduce( - Map{ - MapFunc: MapX2Host, - Renderer: Map{ - MapFunc: MapContainerImageIdentity, - Renderer: SelectContainerImage, - }, - }, - Map{ - MapFunc: MapX2Host, - Renderer: FilterPseudo(ContainerRenderer), - }, - Map{ - MapFunc: MapX2Host, - Renderer: Map{ - MapFunc: MapPodIdentity, - Renderer: SelectPod, - }, - }, - Map{ - MapFunc: MapX2Host, - Renderer: AddressRenderer, - }, - Map{ - MapFunc: MapHostIdentity, - Renderer: SelectHost, - }, + MakeMap( + MapX2Host, + MakeMap( + MapContainerImageIdentity, + SelectContainerImage, + ), + ), + MakeMap( + MapX2Host, + FilterPseudo(ContainerRenderer), + ), + MakeMap( + MapX2Host, + MakeMap( + MapPodIdentity, + SelectPod, + ), + ), + MakeMap( + MapX2Host, + AddressRenderer, + ), + MakeMap( + MapHostIdentity, + SelectHost, + ), ) // PodRenderer is a Renderer which produces a renderable kubernetes // graph by merging the container graph and the pods topology. -var PodRenderer = Map{ - MapFunc: MapCountContainers, - Renderer: MakeReduce( - Map{ - MapFunc: MapContainer2Pod, - Renderer: ContainerRenderer, - }, - Map{ - MapFunc: MapPodIdentity, - Renderer: SelectPod, - }, +var PodRenderer = MakeMap( + MapCountContainers, + MakeReduce( + MakeMap( + MapContainer2Pod, + ContainerRenderer, + ), + MakeMap( + MapPodIdentity, + SelectPod, + ), ), -} +) // PodServiceRenderer is a Renderer which produces a renderable kubernetes services // graph by merging the pods graph and the services topology. -var PodServiceRenderer = Map{ - MapFunc: MapCountPods, - Renderer: MakeReduce( - Map{ - MapFunc: MapPod2Service, - Renderer: PodRenderer, - }, - Map{ - MapFunc: MapServiceIdentity, - Renderer: SelectService, - }, +var PodServiceRenderer = MakeMap( + MapCountPods, + MakeReduce( + MakeMap( + MapPod2Service, + PodRenderer, + ), + MakeMap( + MapServiceIdentity, + SelectService, + ), ), -} +) diff --git a/report/report.go b/report/report.go index a63639d20..e33c8e85b 100644 --- a/report/report.go +++ b/report/report.go @@ -2,6 +2,7 @@ package report import ( "fmt" + "math/rand" "strings" "time" ) @@ -80,6 +81,12 @@ type Report struct { // Shortcut reports should be propogated to the UI as quickly as possible, // bypassing the usual spy interval, publish interval and app ws interval. Shortcut bool + + // ID a random identifier for this report, used when caching + // rendered views of the report. Reports with the same id + // must be equal, but we don't require that equal reports have + // the same id. + ID string `deepequal:"skip"` } // MakeReport makes a clean report, ready to Merge() other reports into. @@ -96,6 +103,7 @@ func MakeReport() Report { Overlay: MakeTopology(), Sampling: Sampling{}, Window: 0, + ID: fmt.Sprintf("%d", rand.Int63()), } } @@ -113,6 +121,7 @@ func (r Report) Copy() Report { Overlay: r.Overlay.Copy(), Sampling: r.Sampling, Window: r.Window, + ID: fmt.Sprintf("%d", rand.Int63()), } } diff --git a/test/fixture/report_fixture.go b/test/fixture/report_fixture.go index 4b5f72231..6462827ab 100644 --- a/test/fixture/report_fixture.go +++ b/test/fixture/report_fixture.go @@ -121,6 +121,7 @@ var ( ServerHostLoad15Metric = report.MakeMetric().Add(Now, 0.16).WithFirst(Now.Add(-16 * time.Second)) Report = report.Report{ + ID: "test-report", Endpoint: report.Topology{ Nodes: report.Nodes{ // Node is arbitrary. We're free to put only precisely what we diff --git a/test/reflect/deepequal.go b/test/reflect/deepequal.go index 12d34fefa..70857196e 100644 --- a/test/reflect/deepequal.go +++ b/test/reflect/deepequal.go @@ -105,6 +105,9 @@ func deepValueEqual(v1, v2 reflect.Value, visited map[visit]bool, depth int) boo return deepValueEqual(v1.Elem(), v2.Elem(), visited, depth+1) case reflect.Struct: for i, n := 0, v1.NumField(); i < n; i++ { + if v1.Type().Field(i).Tag.Get("deepequal") == "skip" { + continue + } if !deepValueEqual(v1.Field(i), v2.Field(i), visited, depth+1) { return false }