diff --git a/render/benchmark_test.go b/render/benchmark_test.go index baad3f498..eb3516e44 100644 --- a/render/benchmark_test.go +++ b/render/benchmark_test.go @@ -87,11 +87,11 @@ func loadReport() (report.Report, error) { return fixture.Report, nil } - var rpt report.Report b, err := ioutil.ReadFile(*benchReportFile) if err != nil { return rpt, err } + rpt := report.MakeReport() err = json.Unmarshal(b, &rpt) return rpt, err } diff --git a/render/filters.go b/render/filters.go index 205306764..9feda297d 100644 --- a/render/filters.go +++ b/render/filters.go @@ -60,17 +60,22 @@ type Filter struct { FilterFunc func(RenderableNode) bool } +// MakeFilter makes a new Filter. +func MakeFilter(f func(RenderableNode) bool, r Renderer) Renderer { + return &Filter{r, f} +} + // Render implements Renderer -func (f Filter) Render(rpt report.Report) RenderableNodes { +func (f *Filter) Render(rpt report.Report) RenderableNodes { nodes, _ := f.render(rpt) return nodes } -func (f Filter) render(rpt report.Report) (RenderableNodes, int) { +func (f *Filter) render(rpt report.Report) (RenderableNodes, int) { output := RenderableNodes{} inDegrees := map[string]int{} filtered := 0 - for id, node := range f.Renderer.Render(rpt) { + for id, node := range memoisedRender(f.Renderer, rpt) { if f.FilterFunc(node) { output[id] = node inDegrees[id] = 0 @@ -122,24 +127,24 @@ const IsConnected = "is_connected" // FilterPseudo produces a renderer that removes pseudo nodes from the given // renderer func FilterPseudo(r Renderer) Renderer { - return Filter{ - Renderer: r, - FilterFunc: func(node RenderableNode) bool { + return MakeFilter( + func(node RenderableNode) bool { return !node.Pseudo }, - } + r, + ) } // FilterUnconnected produces a renderer that filters unconnected nodes // from the given renderer func FilterUnconnected(r Renderer) Renderer { - return Filter{ - Renderer: ColorConnected(r), - FilterFunc: func(node RenderableNode) bool { + return MakeFilter( + func(node RenderableNode) bool { _, ok := node.Latest.Lookup(IsConnected) return ok }, - } + ColorConnected(r), + ) } // FilterNoop does nothing. @@ -149,20 +154,19 @@ func FilterNoop(in Renderer) Renderer { // FilterStopped filters out stopped containers. func FilterStopped(r Renderer) Renderer { - return Filter{ - Renderer: r, - FilterFunc: func(node RenderableNode) bool { + return MakeFilter( + func(node RenderableNode) bool { containerState, ok := node.Latest.Lookup(docker.ContainerState) return !ok || containerState != docker.StateStopped }, - } + r, + ) } // FilterSystem is a Renderer which filters out system nodes. func FilterSystem(r Renderer) Renderer { - return Filter{ - Renderer: r, - FilterFunc: func(node RenderableNode) bool { + return MakeFilter( + func(node RenderableNode) bool { containerName, _ := node.Latest.Lookup(docker.ContainerName) if _, ok := systemContainerNames[containerName]; ok { return false @@ -186,7 +190,8 @@ func FilterSystem(r Renderer) Renderer { } return true }, - } + r, + ) } var systemContainerNames = map[string]struct{}{ diff --git a/render/render.go b/render/render.go index 5cb96ee19..a8d9a6f6e 100644 --- a/render/render.go +++ b/render/render.go @@ -1,9 +1,35 @@ package render import ( + "fmt" + "reflect" + + "github.com/bluele/gcache" + "github.com/weaveworks/scope/report" ) +var renderCache = gcache.New(100).LRU().Build() + +func memoisedRender(r Renderer, rpt report.Report) RenderableNodes { + key := "" + v := reflect.ValueOf(r) + switch v.Kind() { + case reflect.Ptr, reflect.Func: + key = fmt.Sprintf("%s-%x", rpt.ID, v.Pointer()) + default: + return r.Render(rpt) + } + if result, err := renderCache.Get(key); err == nil { + //fmt.Println("Hit", key) + return result.(RenderableNodes) + } + //fmt.Println("Miss", key) + output := r.Render(rpt) + renderCache.Set(key, output) + return output +} + // Renderer is something that can render a report to a set of RenderableNodes. type Renderer interface { Render(report.Report) RenderableNodes @@ -27,22 +53,23 @@ type Reduce []Renderer // MakeReduce is the only sane way to produce a Reduce Renderer. func MakeReduce(renderers ...Renderer) Renderer { - return Reduce(renderers) + r := Reduce(renderers) + return &r } // Render produces a set of RenderableNodes given a Report. -func (r Reduce) Render(rpt report.Report) RenderableNodes { +func (r *Reduce) Render(rpt report.Report) RenderableNodes { result := RenderableNodes{} - for _, renderer := range r { - result = result.Merge(renderer.Render(rpt)) + for _, renderer := range *r { + result = result.Merge(memoisedRender(renderer, rpt)) } return result } // Stats implements Renderer -func (r Reduce) Stats(rpt report.Report) Stats { +func (r *Reduce) Stats(rpt report.Report) Stats { var result Stats - for _, renderer := range r { + for _, renderer := range *r { result = result.merge(renderer.Stats(rpt)) } return result @@ -55,24 +82,29 @@ type Map struct { Renderer } +// MakeMap makes a new Map +func MakeMap(f MapFunc, r Renderer) Renderer { + return &Map{f, r} +} + // Render transforms a set of RenderableNodes produces by another Renderer. // using a map function -func (m Map) Render(rpt report.Report) RenderableNodes { +func (m *Map) Render(rpt report.Report) RenderableNodes { output, _ := m.render(rpt) return output } // Stats implements Renderer -func (m Map) Stats(rpt report.Report) Stats { +func (m *Map) Stats(rpt report.Report) Stats { // There doesn't seem to be an instance where we want stats to recurse // through Maps - for instance we don't want to see the number of filtered // processes in the container renderer. return Stats{} } -func (m Map) render(rpt report.Report) (RenderableNodes, map[string]report.IDList) { +func (m *Map) render(rpt report.Report) (RenderableNodes, map[string]report.IDList) { var ( - input = m.Renderer.Render(rpt) + input = memoisedRender(m.Renderer, rpt) output = RenderableNodes{} mapped = map[string]report.IDList{} // input node ID -> output node IDs adjacencies = map[string]report.IDList{} // output node ID -> input node Adjacencies diff --git a/render/topologies.go b/render/topologies.go index 92c87029d..fd0673698 100644 --- a/render/topologies.go +++ b/render/topologies.go @@ -9,22 +9,22 @@ import ( ) // EndpointRenderer is a Renderer which produces a renderable endpoint graph. -var EndpointRenderer = Map{ - MapFunc: MapEndpointIdentity, - Renderer: SelectEndpoint, -} +var EndpointRenderer = MakeMap( + MapEndpointIdentity, + SelectEndpoint, +) // ProcessRenderer is a Renderer which produces a renderable process // graph by merging the endpoint graph and the process topology. var ProcessRenderer = MakeReduce( - Map{ - MapFunc: MapEndpoint2Process, - Renderer: EndpointRenderer, - }, - Map{ - MapFunc: MapProcessIdentity, - Renderer: SelectProcess, - }, + MakeMap( + MapEndpoint2Process, + EndpointRenderer, + ), + MakeMap( + MapProcessIdentity, + SelectProcess, + ), ) // processWithContainerNameRenderer is a Renderer which produces a process @@ -35,10 +35,10 @@ type processWithContainerNameRenderer struct { func (r processWithContainerNameRenderer) Render(rpt report.Report) RenderableNodes { processes := r.Renderer.Render(rpt) - containers := Map{ - MapFunc: MapContainerIdentity, - Renderer: SelectContainer, - }.Render(rpt) + containers := MakeMap( + MapContainerIdentity, + SelectContainer, + ).Render(rpt) for id, p := range processes { pid, ok := p.Node.Latest.Lookup(process.PID) @@ -66,13 +66,13 @@ var ProcessWithContainerNameRenderer = processWithContainerNameRenderer{ProcessR // ProcessNameRenderer is a Renderer which produces a renderable process // name graph by munging the progess graph. -var ProcessNameRenderer = Map{ - MapFunc: MapCountProcessName, - Renderer: Map{ - MapFunc: MapProcess2Name, - Renderer: ProcessRenderer, - }, -} +var ProcessNameRenderer = MakeMap( + MapCountProcessName, + MakeMap( + MapProcess2Name, + ProcessRenderer, + ), +) // ContainerRenderer is a Renderer which produces a renderable container // graph by merging the process graph and the container topology. @@ -80,40 +80,40 @@ var ProcessNameRenderer = Map{ // but we need to be careful to ensure we only include each edge once, by only // including the ProcessRenderer once. var ContainerRenderer = MakeReduce( - Filter{ - FilterFunc: func(n RenderableNode) bool { + MakeFilter( + func(n RenderableNode) bool { _, inContainer := n.Node.Latest.Lookup(docker.ContainerID) _, isConnected := n.Node.Latest.Lookup(IsConnected) return inContainer || isConnected }, - Renderer: Map{ - MapFunc: MapProcess2Container, - Renderer: ColorConnected(ProcessRenderer), - }, - }, + MakeMap( + MapProcess2Container, + ColorConnected(ProcessRenderer), + ), + ), // This mapper brings in short lived connections by joining with container IPs. // We need to be careful to ensure we only include each edge once. Edges brought in // by the above renders will have a pid, so its enough to filter out any nodes with // pids. - FilterUnconnected(Map{ - MapFunc: MapIP2Container, - Renderer: MakeReduce( - Map{ - MapFunc: MapContainer2IP, - Renderer: SelectContainer, - }, - Map{ - MapFunc: MapEndpoint2IP, - Renderer: SelectEndpoint, - }, + FilterUnconnected(MakeMap( + MapIP2Container, + MakeReduce( + MakeMap( + MapContainer2IP, + SelectContainer, + ), + MakeMap( + MapEndpoint2IP, + SelectEndpoint, + ), ), - }), + )), - Map{ - MapFunc: MapContainerIdentity, - Renderer: SelectContainer, - }, + MakeMap( + MapContainerIdentity, + SelectContainer, + ), ) type containerWithImageNameRenderer struct { @@ -125,10 +125,10 @@ type containerWithImageNameRenderer struct { // container metadata. func (r containerWithImageNameRenderer) Render(rpt report.Report) RenderableNodes { containers := r.Renderer.Render(rpt) - images := Map{ - MapFunc: MapContainerImageIdentity, - Renderer: SelectContainerImage, - }.Render(rpt) + images := MakeMap( + MapContainerImageIdentity, + SelectContainerImage, + ).Render(rpt) for id, c := range containers { imageID, ok := c.Node.Latest.Lookup(docker.ImageID) @@ -153,99 +153,99 @@ var ContainerWithImageNameRenderer = containerWithImageNameRenderer{ContainerRen // ContainerImageRenderer is a Renderer which produces a renderable container // image graph by merging the container graph and the container image topology. -var ContainerImageRenderer = Map{ - MapFunc: MapCountContainers, - Renderer: Map{ - MapFunc: MapContainerImage2Name, - Renderer: MakeReduce( - Map{ - MapFunc: MapContainer2ContainerImage, - Renderer: ContainerRenderer, - }, - Map{ - MapFunc: MapContainerImageIdentity, - Renderer: SelectContainerImage, - }, +var ContainerImageRenderer = MakeMap( + MapCountContainers, + MakeMap( + MapContainerImage2Name, + MakeReduce( + MakeMap( + MapContainer2ContainerImage, + ContainerRenderer, + ), + MakeMap( + MapContainerImageIdentity, + SelectContainerImage, + ), ), - }, -} + ), +) // ContainerHostnameRenderer is a Renderer which produces a renderable container // by hostname graph.. -var ContainerHostnameRenderer = Map{ - MapFunc: MapCountContainers, - Renderer: Map{ - MapFunc: MapContainer2Hostname, - Renderer: ContainerRenderer, - }, -} +var ContainerHostnameRenderer = MakeMap( + MapCountContainers, + MakeMap( + MapContainer2Hostname, + ContainerRenderer, + ), +) // AddressRenderer is a Renderer which produces a renderable address // graph from the address topology. -var AddressRenderer = Map{ - MapFunc: MapAddressIdentity, - Renderer: SelectAddress, -} +var AddressRenderer = MakeMap( + MapAddressIdentity, + SelectAddress, +) // HostRenderer is a Renderer which produces a renderable host // graph from the host topology and address graph. var HostRenderer = MakeReduce( - Map{ - MapFunc: MapX2Host, - Renderer: Map{ - MapFunc: MapContainerImageIdentity, - Renderer: SelectContainerImage, - }, - }, - Map{ - MapFunc: MapX2Host, - Renderer: FilterPseudo(ContainerRenderer), - }, - Map{ - MapFunc: MapX2Host, - Renderer: Map{ - MapFunc: MapPodIdentity, - Renderer: SelectPod, - }, - }, - Map{ - MapFunc: MapX2Host, - Renderer: AddressRenderer, - }, - Map{ - MapFunc: MapHostIdentity, - Renderer: SelectHost, - }, + MakeMap( + MapX2Host, + MakeMap( + MapContainerImageIdentity, + SelectContainerImage, + ), + ), + MakeMap( + MapX2Host, + FilterPseudo(ContainerRenderer), + ), + MakeMap( + MapX2Host, + MakeMap( + MapPodIdentity, + SelectPod, + ), + ), + MakeMap( + MapX2Host, + AddressRenderer, + ), + MakeMap( + MapHostIdentity, + SelectHost, + ), ) // PodRenderer is a Renderer which produces a renderable kubernetes // graph by merging the container graph and the pods topology. -var PodRenderer = Map{ - MapFunc: MapCountContainers, - Renderer: MakeReduce( - Map{ - MapFunc: MapContainer2Pod, - Renderer: ContainerRenderer, - }, - Map{ - MapFunc: MapPodIdentity, - Renderer: SelectPod, - }, +var PodRenderer = MakeMap( + MapCountContainers, + MakeReduce( + MakeMap( + MapContainer2Pod, + ContainerRenderer, + ), + MakeMap( + MapPodIdentity, + SelectPod, + ), ), -} +) // PodServiceRenderer is a Renderer which produces a renderable kubernetes services // graph by merging the pods graph and the services topology. -var PodServiceRenderer = Map{ - MapFunc: MapCountPods, - Renderer: MakeReduce( - Map{ - MapFunc: MapPod2Service, - Renderer: PodRenderer, - }, - Map{ - MapFunc: MapServiceIdentity, - Renderer: SelectService, - }, +var PodServiceRenderer = MakeMap( + MapCountPods, + MakeReduce( + MakeMap( + MapPod2Service, + PodRenderer, + ), + MakeMap( + MapServiceIdentity, + SelectService, + ), ), -} +) diff --git a/report/report.go b/report/report.go index a63639d20..aacaddbbf 100644 --- a/report/report.go +++ b/report/report.go @@ -2,6 +2,7 @@ package report import ( "fmt" + "math/rand" "strings" "time" ) @@ -80,6 +81,10 @@ type Report struct { // Shortcut reports should be propogated to the UI as quickly as possible, // bypassing the usual spy interval, publish interval and app ws interval. Shortcut bool + + // ID a random identifier for this report, used when caching + // rendered views of the report + ID string } // MakeReport makes a clean report, ready to Merge() other reports into. @@ -96,6 +101,7 @@ func MakeReport() Report { Overlay: MakeTopology(), Sampling: Sampling{}, Window: 0, + ID: fmt.Sprintf("%d", rand.Int63()), } } @@ -113,6 +119,7 @@ func (r Report) Copy() Report { Overlay: r.Overlay.Copy(), Sampling: r.Sampling, Window: r.Window, + ID: fmt.Sprintf("%d", rand.Int63()), } } diff --git a/test/fixture/report_fixture.go b/test/fixture/report_fixture.go index 4b5f72231..6462827ab 100644 --- a/test/fixture/report_fixture.go +++ b/test/fixture/report_fixture.go @@ -121,6 +121,7 @@ var ( ServerHostLoad15Metric = report.MakeMetric().Add(Now, 0.16).WithFirst(Now.Add(-16 * time.Second)) Report = report.Report{ + ID: "test-report", Endpoint: report.Topology{ Nodes: report.Nodes{ // Node is arbitrary. We're free to put only precisely what we