From afc27887779f45144546ba1256505142fc748470 Mon Sep 17 00:00:00 2001 From: Tom Wilkie Date: Tue, 1 Sep 2015 15:07:05 +0000 Subject: [PATCH 1/2] Move EdgeMetadata into the NodeMetadata struct. --- experimental/demoprobe/main.go | 17 +- experimental/genreport/generate.go | 17 +- probe/docker/reporter_test.go | 2 - probe/endpoint/reporter.go | 38 ++--- probe/overlay/weave_test.go | 1 - probe/process/reporter_test.go | 1 - probe/sniff/sniffer.go | 36 ++-- probe/sniff/sniffer_internal_test.go | 7 +- probe/sniff/sniffer_test.go | 18 +- render/render.go | 64 +++---- render/render_test.go | 25 +-- report/id.go | 14 -- report/id_test.go | 48 ------ report/topology.go | 242 +++++++++++++-------------- test/report_fixture.go | 69 ++++---- 15 files changed, 251 insertions(+), 348 deletions(-) diff --git a/experimental/demoprobe/main.go b/experimental/demoprobe/main.go index a3694433f..2e277edf5 100644 --- a/experimental/demoprobe/main.go +++ b/experimental/demoprobe/main.go @@ -88,23 +88,16 @@ func demoReport(nodeCount int) report.Report { process.PID: "4000", "name": c.srcProc, "domain": "node-" + src, - }).WithAdjacent(dstPortID)) + }).WithEdgeMetadata(dstPortID, report.EdgeMetadata{ + MaxConnCountTCP: newu64(uint64(rand.Intn(100) + 10)), + })) r.Endpoint = r.Endpoint.WithNode(dstPortID, report.MakeNodeMetadata().WithMetadata(map[string]string{ process.PID: "4000", "name": c.dstProc, "domain": "node-" + dst, - }).WithAdjacent(srcPortID)) - - var ( - edgeKeyEgress = report.MakeEdgeID(srcPortID, dstPortID) - edgeKeyIngress = report.MakeEdgeID(dstPortID, srcPortID) - ) - r.Endpoint.EdgeMetadatas[edgeKeyEgress] = report.EdgeMetadata{ + }).WithEdgeMetadata(srcPortID, report.EdgeMetadata{ MaxConnCountTCP: newu64(uint64(rand.Intn(100) + 10)), - } - r.Endpoint.EdgeMetadatas[edgeKeyIngress] = report.EdgeMetadata{ - MaxConnCountTCP: newu64(uint64(rand.Intn(100) + 10)), - } + })) // Address topology r.Address = r.Address.WithNode(srcAddressID, report.MakeNodeMetadata().WithMetadata(map[string]string{ diff --git a/experimental/genreport/generate.go b/experimental/genreport/generate.go index f6a4f106e..0af8a084e 100644 --- a/experimental/genreport/generate.go +++ b/experimental/genreport/generate.go @@ -68,23 +68,16 @@ func DemoReport(nodeCount int) report.Report { "pid": "4000", "name": c.srcProc, "domain": "node-" + src, - }).WithAdjacent(dstPortID)) + }).WithEdgeMetadata(dstPortID, report.EdgeMetadata{ + MaxConnCountTCP: newu64(uint64(rand.Intn(100) + 10)), + })) r.Endpoint = r.Endpoint.WithNode(dstPortID, report.MakeNodeMetadata().WithMetadata(map[string]string{ "pid": "4000", "name": c.dstProc, "domain": "node-" + dst, - }).WithAdjacent(srcPortID)) - - var ( - edgeKeyEgress = report.MakeEdgeID(srcPortID, dstPortID) - edgeKeyIngress = report.MakeEdgeID(dstPortID, srcPortID) - ) - r.Endpoint.EdgeMetadatas[edgeKeyEgress] = report.EdgeMetadata{ + }).WithEdgeMetadata(srcPortID, report.EdgeMetadata{ MaxConnCountTCP: newu64(uint64(rand.Intn(100) + 10)), - } - r.Endpoint.EdgeMetadatas[edgeKeyIngress] = report.EdgeMetadata{ - MaxConnCountTCP: newu64(uint64(rand.Intn(100) + 10)), - } + })) // Address topology r.Address = r.Address.WithNode(srcAddressID, report.MakeNodeMetadata().WithMetadata(map[string]string{ diff --git a/probe/docker/reporter_test.go b/probe/docker/reporter_test.go index 8f3b05fdf..b1fed39f8 100644 --- a/probe/docker/reporter_test.go +++ b/probe/docker/reporter_test.go @@ -50,7 +50,6 @@ var ( func TestReporter(t *testing.T) { want := report.MakeReport() want.Container = report.Topology{ - EdgeMetadatas: report.EdgeMetadatas{}, NodeMetadatas: report.NodeMetadatas{ report.MakeContainerNodeID("", "ping"): report.MakeNodeMetadataWith(map[string]string{ docker.ContainerID: "ping", @@ -60,7 +59,6 @@ func TestReporter(t *testing.T) { }, } want.ContainerImage = report.Topology{ - EdgeMetadatas: report.EdgeMetadatas{}, NodeMetadatas: report.NodeMetadatas{ report.MakeContainerNodeID("", "baz"): report.MakeNodeMetadataWith(map[string]string{ docker.ImageID: "baz", diff --git a/probe/endpoint/reporter.go b/probe/endpoint/reporter.go index cc99e1ce1..187f3879e 100644 --- a/probe/endpoint/reporter.go +++ b/probe/endpoint/reporter.go @@ -135,9 +135,7 @@ func (r *Reporter) addConnection(rpt *report.Report, localAddr, remoteAddr strin var ( localAddressNodeID = report.MakeAddressNodeID(r.hostID, localAddr) remoteAddressNodeID = report.MakeAddressNodeID(r.hostID, remoteAddr) - edgeID = "" - - localNode = report.MakeNodeMetadataWith(map[string]string{ + localNode = report.MakeNodeMetadataWith(map[string]string{ "name": r.hostName, Addr: localAddr, report.HostNodeID: hostNodeID, @@ -148,16 +146,18 @@ func (r *Reporter) addConnection(rpt *report.Report, localAddr, remoteAddr strin ) if localIsClient { - localNode.Adjacency = localNode.Adjacency.Add(remoteAddressNodeID) - edgeID = report.MakeEdgeID(localAddressNodeID, remoteAddressNodeID) + // New nodes are merged into the report so we don't need to do any counting here; the merge does it for us. + localNode = localNode.WithEdgeMetadata(remoteAddressNodeID, report.EdgeMetadata{ + MaxConnCountTCP: newu64(1), + }) } else { - remoteNode.Adjacency = localNode.Adjacency.Add(localAddressNodeID) - edgeID = report.MakeEdgeID(remoteAddressNodeID, localAddressNodeID) + remoteNode = localNode.WithEdgeMetadata(localAddressNodeID, report.EdgeMetadata{ + MaxConnCountTCP: newu64(1), + }) } rpt.Address = rpt.Address.WithNode(localAddressNodeID, localNode) rpt.Address = rpt.Address.WithNode(remoteAddressNodeID, remoteNode) - countTCPConnection(rpt.Address.EdgeMetadatas, edgeID) } // Update endpoint topology @@ -165,7 +165,6 @@ func (r *Reporter) addConnection(rpt *report.Report, localAddr, remoteAddr strin var ( localEndpointNodeID = report.MakeEndpointNodeID(r.hostID, localAddr, strconv.Itoa(int(localPort))) remoteEndpointNodeID = report.MakeEndpointNodeID(r.hostID, remoteAddr, strconv.Itoa(int(remotePort))) - edgeID = "" localNode = report.MakeNodeMetadataWith(map[string]string{ Addr: localAddr, @@ -179,11 +178,14 @@ func (r *Reporter) addConnection(rpt *report.Report, localAddr, remoteAddr strin ) if localIsClient { - localNode.Adjacency = localNode.Adjacency.Add(remoteEndpointNodeID) - edgeID = report.MakeEdgeID(localEndpointNodeID, remoteEndpointNodeID) + // New nodes are merged into the report so we don't need to do any counting here; the merge does it for us. + localNode = localNode.WithEdgeMetadata(remoteEndpointNodeID, report.EdgeMetadata{ + MaxConnCountTCP: newu64(1), + }) } else { - remoteNode.Adjacency = remoteNode.Adjacency.Add(localEndpointNodeID) - edgeID = report.MakeEdgeID(remoteEndpointNodeID, localEndpointNodeID) + remoteNode = remoteNode.WithEdgeMetadata(localEndpointNodeID, report.EdgeMetadata{ + MaxConnCountTCP: newu64(1), + }) } if proc != nil && proc.PID > 0 { @@ -192,15 +194,9 @@ func (r *Reporter) addConnection(rpt *report.Report, localAddr, remoteAddr strin rpt.Endpoint = rpt.Endpoint.WithNode(localEndpointNodeID, localNode) rpt.Endpoint = rpt.Endpoint.WithNode(remoteEndpointNodeID, remoteNode) - countTCPConnection(rpt.Endpoint.EdgeMetadatas, edgeID) } } -func countTCPConnection(mds report.EdgeMetadatas, key string) { - md := mds[key] - if md.MaxConnCountTCP == nil { - md.MaxConnCountTCP = new(uint64) - } - *md.MaxConnCountTCP++ - mds[key] = md +func newu64(i uint64) *uint64 { + return &i } diff --git a/probe/overlay/weave_test.go b/probe/overlay/weave_test.go index b9e33f074..47b0dd150 100644 --- a/probe/overlay/weave_test.go +++ b/probe/overlay/weave_test.go @@ -36,7 +36,6 @@ func TestWeaveTaggerOverlayTopology(t *testing.T) { t.Fatal(err) } if want, have := (report.Topology{ - EdgeMetadatas: report.EdgeMetadatas{}, NodeMetadatas: report.NodeMetadatas{ report.MakeOverlayNodeID(mockWeavePeerName): report.MakeNodeMetadataWith(map[string]string{ overlay.WeavePeerName: mockWeavePeerName, diff --git a/probe/process/reporter_test.go b/probe/process/reporter_test.go index fa4d3f30e..069565f75 100644 --- a/probe/process/reporter_test.go +++ b/probe/process/reporter_test.go @@ -34,7 +34,6 @@ func TestReporter(t *testing.T) { reporter := process.NewReporter(walker, "") want := report.MakeReport() want.Process = report.Topology{ - EdgeMetadatas: report.EdgeMetadatas{}, NodeMetadatas: report.NodeMetadatas{ report.MakeProcessNodeID("", "1"): report.MakeNodeMetadataWith(map[string]string{ process.PID: "1", diff --git a/probe/sniff/sniffer.go b/probe/sniff/sniffer.go index 498dd69e4..b105a9327 100644 --- a/probe/sniff/sniffer.go +++ b/probe/sniff/sniffer.go @@ -118,18 +118,20 @@ func interpolateCounts(r report.Report) { } factor := 1.0 / rate for _, topology := range r.Topologies() { - for _, emd := range topology.EdgeMetadatas { - if emd.EgressPacketCount != nil { - *emd.EgressPacketCount = uint64(float64(*emd.EgressPacketCount) * factor) - } - if emd.IngressPacketCount != nil { - *emd.IngressPacketCount = uint64(float64(*emd.IngressPacketCount) * factor) - } - if emd.EgressByteCount != nil { - *emd.EgressByteCount = uint64(float64(*emd.EgressByteCount) * factor) - } - if emd.IngressByteCount != nil { - *emd.IngressByteCount = uint64(float64(*emd.IngressByteCount) * factor) + for _, nmd := range topology.NodeMetadatas { + for _, emd := range nmd.EdgeMetadatas { + if emd.EgressPacketCount != nil { + *emd.EgressPacketCount = uint64(float64(*emd.EgressPacketCount) * factor) + } + if emd.IngressPacketCount != nil { + *emd.IngressPacketCount = uint64(float64(*emd.IngressPacketCount) * factor) + } + if emd.EgressByteCount != nil { + *emd.EgressByteCount = uint64(float64(*emd.EgressByteCount) * factor) + } + if emd.IngressByteCount != nil { + *emd.IngressByteCount = uint64(float64(*emd.IngressByteCount) * factor) + } } } } @@ -261,12 +263,11 @@ func (s *Sniffer) Merge(p Packet, rpt *report.Report) { var ( srcNodeID = report.MakeAddressNodeID(s.hostID, localIP) dstNodeID = report.MakeAddressNodeID(s.hostID, remoteIP) - edgeID = report.MakeEdgeID(srcNodeID, dstNodeID) ) rpt.Address = addAdjacency(rpt.Address, srcNodeID, dstNodeID) - emd := rpt.Address.EdgeMetadatas[edgeID] + emd := rpt.Address.NodeMetadatas[srcNodeID].EdgeMetadatas[dstNodeID] if egress { if emd.EgressPacketCount == nil { emd.EgressPacketCount = new(uint64) @@ -286,7 +287,7 @@ func (s *Sniffer) Merge(p Packet, rpt *report.Report) { } *emd.IngressByteCount += uint64(p.Network) } - rpt.Address.EdgeMetadatas[edgeID] = emd + rpt.Address.NodeMetadatas[srcNodeID].EdgeMetadatas[dstNodeID] = emd } // If we have ports, we can add to the endpoint topology, too. @@ -294,12 +295,11 @@ func (s *Sniffer) Merge(p Packet, rpt *report.Report) { var ( srcNodeID = report.MakeEndpointNodeID(s.hostID, localIP, localPort) dstNodeID = report.MakeEndpointNodeID(s.hostID, remoteIP, remotePort) - edgeID = report.MakeEdgeID(srcNodeID, dstNodeID) ) rpt.Endpoint = addAdjacency(rpt.Endpoint, srcNodeID, dstNodeID) - emd := rpt.Endpoint.EdgeMetadatas[edgeID] + emd := rpt.Endpoint.NodeMetadatas[srcNodeID].EdgeMetadatas[dstNodeID] if egress { if emd.EgressPacketCount == nil { emd.EgressPacketCount = new(uint64) @@ -319,6 +319,6 @@ func (s *Sniffer) Merge(p Packet, rpt *report.Report) { } *emd.IngressByteCount += uint64(p.Transport) } - rpt.Endpoint.EdgeMetadatas[edgeID] = emd + rpt.Endpoint.NodeMetadatas[srcNodeID].EdgeMetadatas[dstNodeID] = emd } } diff --git a/probe/sniff/sniffer_internal_test.go b/probe/sniff/sniffer_internal_test.go index 72f78bf44..1313f798b 100644 --- a/probe/sniff/sniffer_internal_test.go +++ b/probe/sniff/sniffer_internal_test.go @@ -11,7 +11,6 @@ func TestInterpolateCounts(t *testing.T) { hostID = "macbook-air" srcNodeID = report.MakeEndpointNodeID(hostID, "1.2.3.4", "5678") dstNodeID = report.MakeEndpointNodeID(hostID, "5.6.7.8", "9012") - edgeID = report.MakeEdgeID(srcNodeID, dstNodeID) samplingCount = uint64(200) samplingTotal = uint64(2345) packetCount = uint64(123) @@ -21,12 +20,12 @@ func TestInterpolateCounts(t *testing.T) { r := report.MakeReport() r.Sampling.Count = samplingCount r.Sampling.Total = samplingTotal - r.Endpoint.EdgeMetadatas[edgeID] = report.EdgeMetadata{ + r.Endpoint.NodeMetadatas[srcNodeID] = report.MakeNodeMetadata().WithEdgeMetadata(dstNodeID, report.EdgeMetadata{ EgressPacketCount: newu64(packetCount), IngressPacketCount: newu64(packetCount), EgressByteCount: newu64(byteCount), IngressByteCount: newu64(byteCount), - } + }) interpolateCounts(r) @@ -34,7 +33,7 @@ func TestInterpolateCounts(t *testing.T) { rate = float64(samplingCount) / float64(samplingTotal) factor = 1.0 / rate apply = func(v uint64) uint64 { return uint64(factor * float64(v)) } - emd = r.Endpoint.EdgeMetadatas[edgeID] + emd = r.Endpoint.NodeMetadatas[srcNodeID].EdgeMetadatas[dstNodeID] ) if want, have := apply(packetCount), (*emd.EgressPacketCount); want != have { t.Errorf("want %d packets, have %d", want, have) diff --git a/probe/sniff/sniffer_test.go b/probe/sniff/sniffer_test.go index 1989b506b..e9ed3f7c2 100644 --- a/probe/sniff/sniffer_test.go +++ b/probe/sniff/sniffer_test.go @@ -65,14 +65,11 @@ func TestMerge(t *testing.T) { dstEndpointNodeID = report.MakeEndpointNodeID(hostID, p.DstIP, p.DstPort) ) if want, have := (report.Topology{ - EdgeMetadatas: report.EdgeMetadatas{ - report.MakeEdgeID(srcEndpointNodeID, dstEndpointNodeID): report.EdgeMetadata{ + NodeMetadatas: report.NodeMetadatas{ + srcEndpointNodeID: report.MakeNodeMetadata().WithEdgeMetadata(dstEndpointNodeID, report.EdgeMetadata{ EgressPacketCount: newu64(1), EgressByteCount: newu64(256), - }, - }, - NodeMetadatas: report.NodeMetadatas{ - srcEndpointNodeID: report.MakeNodeMetadata().WithAdjacent(dstEndpointNodeID), + }), dstEndpointNodeID: report.MakeNodeMetadata(), }, }), rpt.Endpoint; !reflect.DeepEqual(want, have) { @@ -84,14 +81,11 @@ func TestMerge(t *testing.T) { dstAddressNodeID = report.MakeAddressNodeID(hostID, p.DstIP) ) if want, have := (report.Topology{ - EdgeMetadatas: report.EdgeMetadatas{ - report.MakeEdgeID(srcAddressNodeID, dstAddressNodeID): report.EdgeMetadata{ + NodeMetadatas: report.NodeMetadatas{ + srcAddressNodeID: report.MakeNodeMetadata().WithEdgeMetadata(dstAddressNodeID, report.EdgeMetadata{ EgressPacketCount: newu64(1), EgressByteCount: newu64(512), - }, - }, - NodeMetadatas: report.NodeMetadatas{ - srcAddressNodeID: report.MakeNodeMetadata().WithAdjacent(dstAddressNodeID), + }), dstAddressNodeID: report.MakeNodeMetadata(), }, }), rpt.Address; !reflect.DeepEqual(want, have) { diff --git a/render/render.go b/render/render.go index 7371b16aa..cb3775703 100644 --- a/render/render.go +++ b/render/render.go @@ -1,8 +1,6 @@ package render import ( - "log" - "github.com/weaveworks/scope/report" ) @@ -171,6 +169,27 @@ func (m LeafMap) Render(rpt report.Report) RenderableNodes { } } + // We propagate edge metadata to nodes on both ends of the edges. + // TODO we should 'reverse' one end of the edge meta data - ingress -> egress etc. + for srcNodeID, nmd := range t.NodeMetadatas { + for _, srcRenderableID := range source2mapped[srcNodeID] { + srcRenderableNode := nodes[srcRenderableID] + + for dstNodeID, emd := range nmd.EdgeMetadatas { + for _, dstRenderableID := range source2mapped[dstNodeID] { + dstRenderableNode := nodes[dstRenderableID] + + srcRenderableNode.EdgeMetadata = srcRenderableNode.EdgeMetadata.Merge(emd) + dstRenderableNode.EdgeMetadata = dstRenderableNode.EdgeMetadata.Merge(emd) + + nodes[dstRenderableID] = dstRenderableNode + } + } + + nodes[srcRenderableID] = srcRenderableNode + } + } + // Walk the graph and make connections. for srcNodeID, dstNodeIDs := range adjacencies { for _, srcRenderableID := range source2mapped[srcNodeID] { @@ -178,17 +197,7 @@ func (m LeafMap) Render(rpt report.Report) RenderableNodes { for _, dstNodeID := range dstNodeIDs { for _, dstRenderableID := range source2mapped[dstNodeID] { - dstRenderableNode := nodes[dstRenderableID] - srcRenderableNode.Adjacency = srcRenderableNode.Adjacency.Add(dstRenderableID) - - // We propagate edge metadata to nodes on both ends of the edges. - // TODO we should 'reverse' one end of the edge meta data - ingress -> egress etc. - if md, ok := t.EdgeMetadatas[report.MakeEdgeID(srcNodeID, dstNodeID)]; ok { - srcRenderableNode.EdgeMetadata = srcRenderableNode.EdgeMetadata.Merge(md) - dstRenderableNode.EdgeMetadata = dstRenderableNode.EdgeMetadata.Merge(md) - nodes[dstRenderableID] = dstRenderableNode - } } } @@ -217,23 +226,20 @@ func (m LeafMap) EdgeMetadata(rpt report.Report, srcRenderableID, dstRenderableI localNetworks = LocalNetworks(rpt) metadata = report.EdgeMetadata{} ) - for edgeID, edgeMeta := range t.EdgeMetadatas { - src, dst, ok := report.ParseEdgeID(edgeID) - if !ok { - log.Printf("bad edge ID %q", edgeID) - continue - } - srcs, dsts := report.MakeIDList(src), report.MakeIDList(dst) - if src != report.TheInternet { - mapped := m.Mapper(t.NodeMetadatas[src], localNetworks) - srcs = ids(mapped) - } - if dst != report.TheInternet { - mapped := m.Mapper(t.NodeMetadatas[dst], localNetworks) - dsts = ids(mapped) - } - if srcs.Contains(srcRenderableID) && dsts.Contains(dstRenderableID) { - metadata = metadata.Flatten(edgeMeta) + for src, nmd := range t.NodeMetadatas { + for dst, edgeMeta := range nmd.EdgeMetadatas { + srcs, dsts := report.MakeIDList(src), report.MakeIDList(dst) + if src != report.TheInternet { + mapped := m.Mapper(t.NodeMetadatas[src], localNetworks) + srcs = ids(mapped) + } + if dst != report.TheInternet { + mapped := m.Mapper(t.NodeMetadatas[dst], localNetworks) + dsts = ids(mapped) + } + if srcs.Contains(srcRenderableID) && dsts.Contains(dstRenderableID) { + metadata = metadata.Flatten(edgeMeta) + } } } return metadata diff --git a/render/render_test.go b/render/render_test.go index d881f5761..dbb2bee01 100644 --- a/render/render_test.go +++ b/render/render_test.go @@ -113,18 +113,19 @@ func TestMapEdge(t *testing.T) { selector := func(_ report.Report) report.Topology { return report.Topology{ NodeMetadatas: report.NodeMetadatas{ - "foo": { - Metadata: map[string]string{"id": "foo"}, - Adjacency: report.MakeIDList("bar"), - }, - "bar": { - Metadata: map[string]string{"id": "bar"}, - Adjacency: report.MakeIDList("foo"), - }, - }, - EdgeMetadatas: report.EdgeMetadatas{ - "foo|bar": report.EdgeMetadata{EgressPacketCount: newu64(1), EgressByteCount: newu64(2)}, - "bar|foo": report.EdgeMetadata{EgressPacketCount: newu64(3), EgressByteCount: newu64(4)}, + "foo": report.MakeNodeMetadata().WithMetadata(map[string]string{ + "id": "foo", + }).WithEdgeMetadata("bar", report.EdgeMetadata{ + EgressPacketCount: newu64(1), + EgressByteCount: newu64(2), + }), + + "bar": report.MakeNodeMetadata().WithMetadata(map[string]string{ + "id": "bar", + }).WithEdgeMetadata("foo", report.EdgeMetadata{ + EgressPacketCount: newu64(3), + EgressByteCount: newu64(4), + }), }, } } diff --git a/report/id.go b/report/id.go index f7adad8cc..2d3ebc0c0 100644 --- a/report/id.go +++ b/report/id.go @@ -21,20 +21,6 @@ const ( EdgeDelim = "|" ) -// MakeEdgeID produces an edge ID from composite parts. -func MakeEdgeID(srcNodeID, dstNodeID string) string { - return srcNodeID + EdgeDelim + dstNodeID -} - -// ParseEdgeID splits an edge ID to its composite parts. -func ParseEdgeID(edgeID string) (srcNodeID, dstNodeID string, ok bool) { - fields := strings.SplitN(edgeID, EdgeDelim, 2) - if len(fields) != 2 { - return "", "", false - } - return fields[0], fields[1], true -} - // MakeEndpointNodeID produces an endpoint node ID from its composite parts. func MakeEndpointNodeID(hostID, address, port string) string { return MakeAddressNodeID(hostID, address) + ScopeDelim + port diff --git a/report/id_test.go b/report/id_test.go index 6dd1a84e8..31e8d71a2 100644 --- a/report/id_test.go +++ b/report/id_test.go @@ -65,51 +65,3 @@ func TestEndpointNodeID(t *testing.T) { } } } - -func TestEdgeID(t *testing.T) { - for _, bad := range []string{ - client54001EndpointNodeID, - client54002EndpointNodeID, - unknown1EndpointNodeID, - unknown2EndpointNodeID, - unknown3EndpointNodeID, - clientAddressNodeID, - serverAddressNodeID, - unknownAddressNodeID, - clientHostNodeID, - serverHostNodeID, - ">1.2.3.4", - ">", - ";", - "", - } { - if srcNodeID, dstNodeID, ok := report.ParseEdgeID(bad); ok { - t.Errorf("%q: expected failure, but got (%q, %q)", bad, srcNodeID, dstNodeID) - } - } - - for input, want := range map[string]struct{ srcNodeID, dstNodeID string }{ - report.MakeEdgeID("a", report.MakeEndpointNodeID("a", "b", "c")): {"a", report.MakeEndpointNodeID("a", "b", "c")}, - report.MakeEdgeID("a", report.MakeAddressNodeID("a", "b")): {"a", report.MakeAddressNodeID("a", "b")}, - report.MakeEdgeID("a", report.MakeProcessNodeID("a", "b")): {"a", report.MakeProcessNodeID("a", "b")}, - report.MakeEdgeID("a", report.MakeHostNodeID("a")): {"a", report.MakeHostNodeID("a")}, - "host.com|1.2.3.4": {"host.com", "1.2.3.4"}, - "a|b;c": {"a", "b;c"}, - "a|b": {"a", "b"}, - "a|": {"a", ""}, - "|b": {"", "b"}, - "|": {"", ""}, - } { - srcNodeID, dstNodeID, ok := report.ParseEdgeID(input) - if !ok { - t.Errorf("%q: not OK", input) - continue - } - if want, have := want.srcNodeID, srcNodeID; want != have { - t.Errorf("%q: want %q, have %q", input, want, have) - } - if want, have := want.dstNodeID, dstNodeID; want != have { - t.Errorf("%q: want %q, have %q", input, want, have) - } - } -} diff --git a/report/topology.go b/report/topology.go index 2522a261d..52a0c048b 100644 --- a/report/topology.go +++ b/report/topology.go @@ -10,14 +10,12 @@ import ( // and NodeMetadatas respectively. Edges are directional, and embedded in the // NodeMetadata. type Topology struct { - EdgeMetadatas NodeMetadatas } // MakeTopology gives you a Topology. func MakeTopology() Topology { return Topology{ - EdgeMetadatas: map[string]EdgeMetadata{}, NodeMetadatas: map[string]NodeMetadata{}, } } @@ -36,7 +34,6 @@ func (t Topology) WithNode(nodeID string, nmd NodeMetadata) Topology { // Copy returns a value copy of the Topology. func (t Topology) Copy() Topology { return Topology{ - EdgeMetadatas: t.EdgeMetadatas.Copy(), NodeMetadatas: t.NodeMetadatas.Copy(), } } @@ -45,34 +42,10 @@ func (t Topology) Copy() Topology { // The original is not modified. func (t Topology) Merge(other Topology) Topology { return Topology{ - EdgeMetadatas: t.EdgeMetadatas.Merge(other.EdgeMetadatas), NodeMetadatas: t.NodeMetadatas.Merge(other.NodeMetadatas), } } -// EdgeMetadatas collect metadata about each edge in a topology. Keys are a -// concatenation of node IDs. -type EdgeMetadatas map[string]EdgeMetadata - -// Copy returns a value copy of the EdgeMetadatas. -func (e EdgeMetadatas) Copy() EdgeMetadatas { - cp := make(EdgeMetadatas, len(e)) - for k, v := range e { - cp[k] = v.Copy() - } - return cp -} - -// Merge merges the other object into this one, and returns the result object. -// The original is not modified. -func (e EdgeMetadatas) Merge(other EdgeMetadatas) EdgeMetadatas { - cp := e.Copy() - for k, v := range other { - cp[k] = cp[k].Merge(v) - } - return cp -} - // NodeMetadatas collect metadata about each node in a topology. Keys are node // IDs. type NodeMetadatas map[string]NodeMetadata @@ -98,6 +71,119 @@ func (n NodeMetadatas) Merge(other NodeMetadatas) NodeMetadatas { return cp } +// NodeMetadata describes a superset of the metadata that probes can collect +// about a given node in a given topology. +type NodeMetadata struct { + Metadata map[string]string + Counters map[string]int + Adjacency IDList + EdgeMetadatas +} + +// MakeNodeMetadata creates a new NodeMetadata with no initial metadata. +func MakeNodeMetadata() NodeMetadata { + return NodeMetadata{ + Metadata: map[string]string{}, + Counters: map[string]int{}, + Adjacency: MakeIDList(), + EdgeMetadatas: EdgeMetadatas{}, + } +} + +// MakeNodeMetadataWith creates a new NodeMetadata with the supplied map. +func MakeNodeMetadataWith(m map[string]string) NodeMetadata { + return MakeNodeMetadata().WithMetadata(m) +} + +// WithMetadata returns a fresh copy of n, with Metadata set to m +func (n NodeMetadata) WithMetadata(m map[string]string) NodeMetadata { + result := n.Copy() + result.Metadata = m + return result +} + +// WithCounters returns a fresh copy of n, with Counters set to c +func (n NodeMetadata) WithCounters(c map[string]int) NodeMetadata { + result := n.Copy() + result.Counters = c + return result +} + +// WithAdjacency returns a fresh copy of n, with Adjacency set to a +func (n NodeMetadata) WithAdjacency(a IDList) NodeMetadata { + result := n.Copy() + result.Adjacency = a + return result +} + +// WithAdjacent returns a fresh copy of n, with 'a' added to Adjacency +func (n NodeMetadata) WithAdjacent(a string) NodeMetadata { + result := n.Copy() + result.Adjacency = result.Adjacency.Add(a) + return result +} + +// WithEdgeMetadata returns a fresh copy of n, with 'dst' added to Adjacency and md added to EdgeMetadata +func (n NodeMetadata) WithEdgeMetadata(dst string, md EdgeMetadata) NodeMetadata { + result := n.Copy() + result.Adjacency = result.Adjacency.Add(dst) + result.EdgeMetadatas[dst] = md + return result +} + +// Copy returns a value copy of the NodeMetadata. +func (n NodeMetadata) Copy() NodeMetadata { + cp := MakeNodeMetadata() + for k, v := range n.Metadata { + cp.Metadata[k] = v + } + for k, v := range n.Counters { + cp.Counters[k] = v + } + cp.Adjacency = n.Adjacency.Copy() + cp.EdgeMetadatas = n.EdgeMetadatas.Copy() + return cp +} + +// Merge merges two node metadata maps together. In case of conflict, the +// other (right-hand) side wins. Always reassign the result of merge to the +// destination. Merge does not modify the receiver. +func (n NodeMetadata) Merge(other NodeMetadata) NodeMetadata { + cp := n.Copy() + for k, v := range other.Metadata { + cp.Metadata[k] = v // other takes precedence + } + for k, v := range other.Counters { + cp.Counters[k] = n.Counters[k] + v + } + cp.Adjacency = cp.Adjacency.Merge(other.Adjacency) + cp.EdgeMetadatas = cp.EdgeMetadatas.Merge(n.EdgeMetadatas) + return cp +} + +// EdgeMetadatas collect metadata about each edge in a topology. Keys are +// the remote node IDs, as in Adjacency. +type EdgeMetadatas map[string]EdgeMetadata + +// Copy returns a value copy of the EdgeMetadatas. +func (e EdgeMetadatas) Copy() EdgeMetadatas { + cp := make(EdgeMetadatas, len(e)) + for k, v := range e { + cp[k] = v.Copy() + } + return cp +} + +// Merge merges the other object into this one, and returns the result object. +// The original is not modified. +func (e EdgeMetadatas) Merge(other EdgeMetadatas) EdgeMetadatas { + cp := e.Copy() + for k, v := range other { + cp[k] = cp[k].Merge(v) + } + return cp +} + // EdgeMetadata describes a superset of the metadata that probes can possibly // collect about a directed edge between two nodes in any topology. type EdgeMetadata struct { @@ -155,102 +241,9 @@ func (e EdgeMetadata) Flatten(other EdgeMetadata) EdgeMetadata { return cp } -// NodeMetadata describes a superset of the metadata that probes can collect -// about a given node in a given topology. -type NodeMetadata struct { - Metadata map[string]string - Counters map[string]int - Adjacency IDList -} - -// MakeNodeMetadata creates a new NodeMetadata with no initial metadata. -func MakeNodeMetadata() NodeMetadata { - return MakeNodeMetadataWith(map[string]string{}) -} - -// MakeNodeMetadataWith creates a new NodeMetadata with the supplied map. -func MakeNodeMetadataWith(m map[string]string) NodeMetadata { - return NodeMetadata{ - Metadata: m, - Counters: map[string]int{}, - Adjacency: MakeIDList(), - } -} - -// WithMetadata returns a fresh copy of n, with Metadata set to m -func (n NodeMetadata) WithMetadata(m map[string]string) NodeMetadata { - result := n.Copy() - result.Metadata = m - return result -} - -// WithCounters returns a fresh copy of n, with Counters set to c -func (n NodeMetadata) WithCounters(c map[string]int) NodeMetadata { - result := n.Copy() - result.Counters = c - return result -} - -// WithAdjacency returns a fresh copy of n, with Adjacency set to a -func (n NodeMetadata) WithAdjacency(a IDList) NodeMetadata { - result := n.Copy() - result.Adjacency = a - return result -} - -// WithAdjacent returns a fresh copy of n, with 'a' added to Adjacency -func (n NodeMetadata) WithAdjacent(a string) NodeMetadata { - result := n.Copy() - result.Adjacency = result.Adjacency.Add(a) - return result -} - -// Copy returns a value copy of the NodeMetadata. -func (n NodeMetadata) Copy() NodeMetadata { - cp := MakeNodeMetadata() - for k, v := range n.Metadata { - cp.Metadata[k] = v - } - for k, v := range n.Counters { - cp.Counters[k] = v - } - cp.Adjacency = n.Adjacency.Copy() - return cp -} - -// Merge merges two node metadata maps together. In case of conflict, the -// other (right-hand) side wins. Always reassign the result of merge to the -// destination. Merge does not modify the receiver. -func (n NodeMetadata) Merge(other NodeMetadata) NodeMetadata { - cp := n.Copy() - for k, v := range other.Metadata { - cp.Metadata[k] = v // other takes precedence - } - for k, v := range other.Counters { - cp.Counters[k] = n.Counters[k] + v - } - cp.Adjacency = cp.Adjacency.Merge(other.Adjacency) - return cp -} - // Validate checks the topology for various inconsistencies. func (t Topology) Validate() error { - // Check all edge metadata keys must have the appropriate entries in - // adjacencies & node metadata. - var errs []string - for edgeID := range t.EdgeMetadatas { - srcNodeID, dstNodeID, ok := ParseEdgeID(edgeID) - if !ok { - errs = append(errs, fmt.Sprintf("invalid edge ID %q", edgeID)) - continue - } - // For each edge, ensure they are connected in the right direction - if src, ok := t.NodeMetadatas[srcNodeID]; !ok { - errs = append(errs, fmt.Sprintf("node %s metadatas missing for edge %q", srcNodeID, edgeID)) - } else if !src.Adjacency.Contains(dstNodeID) { - errs = append(errs, fmt.Sprintf("adjacency destination missing for destination node ID %q (from edge %q)", srcNodeID, edgeID)) - } - } + errs := []string{} // Check all node metadatas are valid, and the keys are parseable, i.e. // contain a scope. @@ -268,6 +261,13 @@ func (t Topology) Validate() error { errs = append(errs, fmt.Sprintf("node metadata missing from adjacency %q -> %q", nodeID, dstNodeID)) } } + + // Check all the edge metadatas have entries in adjacencies + for dstNodeID := range nmd.EdgeMetadatas { + if _, ok := t.NodeMetadatas[dstNodeID]; !ok { + errs = append(errs, fmt.Sprintf("node %s metadatas missing for edge %q", dstNodeID, nodeID)) + } + } } if len(errs) > 0 { diff --git a/test/report_fixture.go b/test/report_fixture.go index 4c6a0e18c..346d86226 100644 --- a/test/report_fixture.go +++ b/test/report_fixture.go @@ -95,14 +95,20 @@ var ( endpoint.Port: ClientPort54001, process.PID: Client1PID, report.HostNodeID: ClientHostNodeID, - }).WithAdjacency(report.MakeIDList(Server80NodeID)), + }).WithEdgeMetadata(Server80NodeID, report.EdgeMetadata{ + EgressPacketCount: newu64(10), + EgressByteCount: newu64(100), + }), Client54002NodeID: report.MakeNodeMetadata().WithMetadata(map[string]string{ endpoint.Addr: ClientIP, endpoint.Port: ClientPort54002, process.PID: Client2PID, report.HostNodeID: ClientHostNodeID, - }).WithAdjacency(report.MakeIDList(Server80NodeID)), + }).WithEdgeMetadata(Server80NodeID, report.EdgeMetadata{ + EgressPacketCount: newu64(20), + EgressByteCount: newu64(200), + }), Server80NodeID: report.MakeNodeMetadata().WithMetadata(map[string]string{ endpoint.Addr: ServerIP, @@ -116,60 +122,46 @@ var ( endpoint.Port: NonContainerClientPort, process.PID: NonContainerPID, report.HostNodeID: ServerHostNodeID, - }).WithAdjacency(report.MakeIDList(GoogleEndpointNodeID)), + }).WithAdjacent(GoogleEndpointNodeID), // Probe pseudo nodes UnknownClient1NodeID: report.MakeNodeMetadata().WithMetadata(map[string]string{ endpoint.Addr: UnknownClient1IP, endpoint.Port: UnknownClient1Port, - }).WithAdjacency(report.MakeIDList(Server80NodeID)), + }).WithEdgeMetadata(Server80NodeID, report.EdgeMetadata{ + EgressPacketCount: newu64(30), + EgressByteCount: newu64(300), + }), UnknownClient2NodeID: report.MakeNodeMetadata().WithMetadata(map[string]string{ endpoint.Addr: UnknownClient2IP, endpoint.Port: UnknownClient2Port, - }).WithAdjacency(report.MakeIDList(Server80NodeID)), + }).WithEdgeMetadata(Server80NodeID, report.EdgeMetadata{ + EgressPacketCount: newu64(40), + EgressByteCount: newu64(400), + }), UnknownClient3NodeID: report.MakeNodeMetadata().WithMetadata(map[string]string{ endpoint.Addr: UnknownClient3IP, endpoint.Port: UnknownClient3Port, - }).WithAdjacency(report.MakeIDList(Server80NodeID)), + }).WithEdgeMetadata(Server80NodeID, report.EdgeMetadata{ + EgressPacketCount: newu64(50), + EgressByteCount: newu64(500), + }), RandomClientNodeID: report.MakeNodeMetadata().WithMetadata(map[string]string{ endpoint.Addr: RandomClientIP, endpoint.Port: RandomClientPort, - }).WithAdjacency(report.MakeIDList(Server80NodeID)), + }).WithEdgeMetadata(Server80NodeID, report.EdgeMetadata{ + EgressPacketCount: newu64(60), + EgressByteCount: newu64(600), + }), GoogleEndpointNodeID: report.MakeNodeMetadata().WithMetadata(map[string]string{ endpoint.Addr: GoogleIP, endpoint.Port: GooglePort, }), }, - EdgeMetadatas: report.EdgeMetadatas{ - report.MakeEdgeID(Client54001NodeID, Server80NodeID): report.EdgeMetadata{ - EgressPacketCount: newu64(10), - EgressByteCount: newu64(100), - }, - report.MakeEdgeID(Client54002NodeID, Server80NodeID): report.EdgeMetadata{ - EgressPacketCount: newu64(20), - EgressByteCount: newu64(200), - }, - report.MakeEdgeID(UnknownClient1NodeID, Server80NodeID): report.EdgeMetadata{ - EgressPacketCount: newu64(30), - EgressByteCount: newu64(300), - }, - report.MakeEdgeID(UnknownClient2NodeID, Server80NodeID): report.EdgeMetadata{ - EgressPacketCount: newu64(40), - EgressByteCount: newu64(400), - }, - report.MakeEdgeID(UnknownClient3NodeID, Server80NodeID): report.EdgeMetadata{ - EgressPacketCount: newu64(50), - EgressByteCount: newu64(500), - }, - report.MakeEdgeID(RandomClientNodeID, Server80NodeID): report.EdgeMetadata{ - EgressPacketCount: newu64(60), - EgressByteCount: newu64(600), - }, - }, }, Process: report.Topology{ NodeMetadatas: report.NodeMetadatas{ @@ -197,7 +189,6 @@ var ( report.HostNodeID: ServerHostNodeID, }), }, - EdgeMetadatas: report.EdgeMetadatas{}, }, Container: report.Topology{ NodeMetadatas: report.NodeMetadatas{ @@ -238,7 +229,9 @@ var ( ClientAddressNodeID: report.MakeNodeMetadata().WithMetadata(map[string]string{ endpoint.Addr: ClientIP, report.HostNodeID: ClientHostNodeID, - }).WithAdjacency(report.MakeIDList(ServerAddressNodeID)), + }).WithEdgeMetadata(ServerAddressNodeID, report.EdgeMetadata{ + MaxConnCountTCP: newu64(3), + }), ServerAddressNodeID: report.MakeNodeMetadata().WithMetadata(map[string]string{ endpoint.Addr: ServerIP, @@ -261,11 +254,6 @@ var ( endpoint.Addr: RandomClientIP, }).WithAdjacency(report.MakeIDList(ServerAddressNodeID)), }, - EdgeMetadatas: report.EdgeMetadatas{ - report.MakeEdgeID(ClientAddressNodeID, ServerAddressNodeID): report.EdgeMetadata{ - MaxConnCountTCP: newu64(3), - }, - }, }, Host: report.Topology{ NodeMetadatas: report.NodeMetadatas{ @@ -284,7 +272,6 @@ var ( report.HostNodeID: ServerHostNodeID, }), }, - EdgeMetadatas: report.EdgeMetadatas{}, }, Sampling: report.Sampling{ Count: 1024, From 144a7986024a43af3148cdd32b39d0a4a5c768f7 Mon Sep 17 00:00:00 2001 From: Tom Wilkie Date: Tue, 1 Sep 2015 17:35:14 +0000 Subject: [PATCH 2/2] Review Feedback --- experimental/demoprobe/main.go | 4 +- experimental/genreport/generate.go | 4 +- probe/endpoint/reporter.go | 8 +-- probe/sniff/sniffer.go | 10 ++-- probe/sniff/sniffer_internal_test.go | 4 +- probe/sniff/sniffer_test.go | 4 +- render/render.go | 4 +- render/render_test.go | 4 +- report/topology.go | 90 +++++++++++++++++++--------- test/report_fixture.go | 14 ++--- 10 files changed, 91 insertions(+), 55 deletions(-) diff --git a/experimental/demoprobe/main.go b/experimental/demoprobe/main.go index 2e277edf5..f084d79c1 100644 --- a/experimental/demoprobe/main.go +++ b/experimental/demoprobe/main.go @@ -88,14 +88,14 @@ func demoReport(nodeCount int) report.Report { process.PID: "4000", "name": c.srcProc, "domain": "node-" + src, - }).WithEdgeMetadata(dstPortID, report.EdgeMetadata{ + }).WithEdge(dstPortID, report.EdgeMetadata{ MaxConnCountTCP: newu64(uint64(rand.Intn(100) + 10)), })) r.Endpoint = r.Endpoint.WithNode(dstPortID, report.MakeNodeMetadata().WithMetadata(map[string]string{ process.PID: "4000", "name": c.dstProc, "domain": "node-" + dst, - }).WithEdgeMetadata(srcPortID, report.EdgeMetadata{ + }).WithEdge(srcPortID, report.EdgeMetadata{ MaxConnCountTCP: newu64(uint64(rand.Intn(100) + 10)), })) diff --git a/experimental/genreport/generate.go b/experimental/genreport/generate.go index 0af8a084e..7df1ffd12 100644 --- a/experimental/genreport/generate.go +++ b/experimental/genreport/generate.go @@ -68,14 +68,14 @@ func DemoReport(nodeCount int) report.Report { "pid": "4000", "name": c.srcProc, "domain": "node-" + src, - }).WithEdgeMetadata(dstPortID, report.EdgeMetadata{ + }).WithEdge(dstPortID, report.EdgeMetadata{ MaxConnCountTCP: newu64(uint64(rand.Intn(100) + 10)), })) r.Endpoint = r.Endpoint.WithNode(dstPortID, report.MakeNodeMetadata().WithMetadata(map[string]string{ "pid": "4000", "name": c.dstProc, "domain": "node-" + dst, - }).WithEdgeMetadata(srcPortID, report.EdgeMetadata{ + }).WithEdge(srcPortID, report.EdgeMetadata{ MaxConnCountTCP: newu64(uint64(rand.Intn(100) + 10)), })) diff --git a/probe/endpoint/reporter.go b/probe/endpoint/reporter.go index 187f3879e..333c76566 100644 --- a/probe/endpoint/reporter.go +++ b/probe/endpoint/reporter.go @@ -147,11 +147,11 @@ func (r *Reporter) addConnection(rpt *report.Report, localAddr, remoteAddr strin if localIsClient { // New nodes are merged into the report so we don't need to do any counting here; the merge does it for us. - localNode = localNode.WithEdgeMetadata(remoteAddressNodeID, report.EdgeMetadata{ + localNode = localNode.WithEdge(remoteAddressNodeID, report.EdgeMetadata{ MaxConnCountTCP: newu64(1), }) } else { - remoteNode = localNode.WithEdgeMetadata(localAddressNodeID, report.EdgeMetadata{ + remoteNode = localNode.WithEdge(localAddressNodeID, report.EdgeMetadata{ MaxConnCountTCP: newu64(1), }) } @@ -179,11 +179,11 @@ func (r *Reporter) addConnection(rpt *report.Report, localAddr, remoteAddr strin if localIsClient { // New nodes are merged into the report so we don't need to do any counting here; the merge does it for us. - localNode = localNode.WithEdgeMetadata(remoteEndpointNodeID, report.EdgeMetadata{ + localNode = localNode.WithEdge(remoteEndpointNodeID, report.EdgeMetadata{ MaxConnCountTCP: newu64(1), }) } else { - remoteNode = remoteNode.WithEdgeMetadata(localEndpointNodeID, report.EdgeMetadata{ + remoteNode = remoteNode.WithEdge(localEndpointNodeID, report.EdgeMetadata{ MaxConnCountTCP: newu64(1), }) } diff --git a/probe/sniff/sniffer.go b/probe/sniff/sniffer.go index b105a9327..628c9c648 100644 --- a/probe/sniff/sniffer.go +++ b/probe/sniff/sniffer.go @@ -119,7 +119,7 @@ func interpolateCounts(r report.Report) { factor := 1.0 / rate for _, topology := range r.Topologies() { for _, nmd := range topology.NodeMetadatas { - for _, emd := range nmd.EdgeMetadatas { + for _, emd := range nmd.Edges { if emd.EgressPacketCount != nil { *emd.EgressPacketCount = uint64(float64(*emd.EgressPacketCount) * factor) } @@ -267,7 +267,7 @@ func (s *Sniffer) Merge(p Packet, rpt *report.Report) { rpt.Address = addAdjacency(rpt.Address, srcNodeID, dstNodeID) - emd := rpt.Address.NodeMetadatas[srcNodeID].EdgeMetadatas[dstNodeID] + emd := rpt.Address.NodeMetadatas[srcNodeID].Edges[dstNodeID] if egress { if emd.EgressPacketCount == nil { emd.EgressPacketCount = new(uint64) @@ -287,7 +287,7 @@ func (s *Sniffer) Merge(p Packet, rpt *report.Report) { } *emd.IngressByteCount += uint64(p.Network) } - rpt.Address.NodeMetadatas[srcNodeID].EdgeMetadatas[dstNodeID] = emd + rpt.Address.NodeMetadatas[srcNodeID].Edges[dstNodeID] = emd } // If we have ports, we can add to the endpoint topology, too. @@ -299,7 +299,7 @@ func (s *Sniffer) Merge(p Packet, rpt *report.Report) { rpt.Endpoint = addAdjacency(rpt.Endpoint, srcNodeID, dstNodeID) - emd := rpt.Endpoint.NodeMetadatas[srcNodeID].EdgeMetadatas[dstNodeID] + emd := rpt.Endpoint.NodeMetadatas[srcNodeID].Edges[dstNodeID] if egress { if emd.EgressPacketCount == nil { emd.EgressPacketCount = new(uint64) @@ -319,6 +319,6 @@ func (s *Sniffer) Merge(p Packet, rpt *report.Report) { } *emd.IngressByteCount += uint64(p.Transport) } - rpt.Endpoint.NodeMetadatas[srcNodeID].EdgeMetadatas[dstNodeID] = emd + rpt.Endpoint.NodeMetadatas[srcNodeID].Edges[dstNodeID] = emd } } diff --git a/probe/sniff/sniffer_internal_test.go b/probe/sniff/sniffer_internal_test.go index 1313f798b..512f5f0fc 100644 --- a/probe/sniff/sniffer_internal_test.go +++ b/probe/sniff/sniffer_internal_test.go @@ -20,7 +20,7 @@ func TestInterpolateCounts(t *testing.T) { r := report.MakeReport() r.Sampling.Count = samplingCount r.Sampling.Total = samplingTotal - r.Endpoint.NodeMetadatas[srcNodeID] = report.MakeNodeMetadata().WithEdgeMetadata(dstNodeID, report.EdgeMetadata{ + r.Endpoint.NodeMetadatas[srcNodeID] = report.MakeNodeMetadata().WithEdge(dstNodeID, report.EdgeMetadata{ EgressPacketCount: newu64(packetCount), IngressPacketCount: newu64(packetCount), EgressByteCount: newu64(byteCount), @@ -33,7 +33,7 @@ func TestInterpolateCounts(t *testing.T) { rate = float64(samplingCount) / float64(samplingTotal) factor = 1.0 / rate apply = func(v uint64) uint64 { return uint64(factor * float64(v)) } - emd = r.Endpoint.NodeMetadatas[srcNodeID].EdgeMetadatas[dstNodeID] + emd = r.Endpoint.NodeMetadatas[srcNodeID].Edges[dstNodeID] ) if want, have := apply(packetCount), (*emd.EgressPacketCount); want != have { t.Errorf("want %d packets, have %d", want, have) diff --git a/probe/sniff/sniffer_test.go b/probe/sniff/sniffer_test.go index e9ed3f7c2..3fb4d6bc2 100644 --- a/probe/sniff/sniffer_test.go +++ b/probe/sniff/sniffer_test.go @@ -66,7 +66,7 @@ func TestMerge(t *testing.T) { ) if want, have := (report.Topology{ NodeMetadatas: report.NodeMetadatas{ - srcEndpointNodeID: report.MakeNodeMetadata().WithEdgeMetadata(dstEndpointNodeID, report.EdgeMetadata{ + srcEndpointNodeID: report.MakeNodeMetadata().WithEdge(dstEndpointNodeID, report.EdgeMetadata{ EgressPacketCount: newu64(1), EgressByteCount: newu64(256), }), @@ -82,7 +82,7 @@ func TestMerge(t *testing.T) { ) if want, have := (report.Topology{ NodeMetadatas: report.NodeMetadatas{ - srcAddressNodeID: report.MakeNodeMetadata().WithEdgeMetadata(dstAddressNodeID, report.EdgeMetadata{ + srcAddressNodeID: report.MakeNodeMetadata().WithEdge(dstAddressNodeID, report.EdgeMetadata{ EgressPacketCount: newu64(1), EgressByteCount: newu64(512), }), diff --git a/render/render.go b/render/render.go index cb3775703..66cee9f11 100644 --- a/render/render.go +++ b/render/render.go @@ -175,7 +175,7 @@ func (m LeafMap) Render(rpt report.Report) RenderableNodes { for _, srcRenderableID := range source2mapped[srcNodeID] { srcRenderableNode := nodes[srcRenderableID] - for dstNodeID, emd := range nmd.EdgeMetadatas { + for dstNodeID, emd := range nmd.Edges { for _, dstRenderableID := range source2mapped[dstNodeID] { dstRenderableNode := nodes[dstRenderableID] @@ -227,7 +227,7 @@ func (m LeafMap) EdgeMetadata(rpt report.Report, srcRenderableID, dstRenderableI metadata = report.EdgeMetadata{} ) for src, nmd := range t.NodeMetadatas { - for dst, edgeMeta := range nmd.EdgeMetadatas { + for dst, edgeMeta := range nmd.Edges { srcs, dsts := report.MakeIDList(src), report.MakeIDList(dst) if src != report.TheInternet { mapped := m.Mapper(t.NodeMetadatas[src], localNetworks) diff --git a/render/render_test.go b/render/render_test.go index dbb2bee01..22e84d013 100644 --- a/render/render_test.go +++ b/render/render_test.go @@ -115,14 +115,14 @@ func TestMapEdge(t *testing.T) { NodeMetadatas: report.NodeMetadatas{ "foo": report.MakeNodeMetadata().WithMetadata(map[string]string{ "id": "foo", - }).WithEdgeMetadata("bar", report.EdgeMetadata{ + }).WithEdge("bar", report.EdgeMetadata{ EgressPacketCount: newu64(1), EgressByteCount: newu64(2), }), "bar": report.MakeNodeMetadata().WithMetadata(map[string]string{ "id": "bar", - }).WithEdgeMetadata("foo", report.EdgeMetadata{ + }).WithEdge("foo", report.EdgeMetadata{ EgressPacketCount: newu64(3), EgressByteCount: newu64(4), }), diff --git a/report/topology.go b/report/topology.go index 52a0c048b..c30fd49d0 100644 --- a/report/topology.go +++ b/report/topology.go @@ -74,19 +74,19 @@ func (n NodeMetadatas) Merge(other NodeMetadatas) NodeMetadatas { // NodeMetadata describes a superset of the metadata that probes can collect // about a given node in a given topology. type NodeMetadata struct { - Metadata map[string]string - Counters map[string]int + Metadata + Counters Adjacency IDList - EdgeMetadatas + Edges EdgeMetadatas } // MakeNodeMetadata creates a new NodeMetadata with no initial metadata. func MakeNodeMetadata() NodeMetadata { return NodeMetadata{ - Metadata: map[string]string{}, - Counters: map[string]int{}, - Adjacency: MakeIDList(), - EdgeMetadatas: EdgeMetadatas{}, + Metadata: Metadata{}, + Counters: Counters{}, + Adjacency: MakeIDList(), + Edges: EdgeMetadatas{}, } } @@ -123,42 +123,78 @@ func (n NodeMetadata) WithAdjacent(a string) NodeMetadata { return result } -// WithEdgeMetadata returns a fresh copy of n, with 'dst' added to Adjacency and md added to EdgeMetadata -func (n NodeMetadata) WithEdgeMetadata(dst string, md EdgeMetadata) NodeMetadata { +// WithEdge returns a fresh copy of n, with 'dst' added to Adjacency and md added to EdgeMetadata +func (n NodeMetadata) WithEdge(dst string, md EdgeMetadata) NodeMetadata { result := n.Copy() result.Adjacency = result.Adjacency.Add(dst) - result.EdgeMetadatas[dst] = md + result.Edges[dst] = md return result } // Copy returns a value copy of the NodeMetadata. func (n NodeMetadata) Copy() NodeMetadata { cp := MakeNodeMetadata() - for k, v := range n.Metadata { - cp.Metadata[k] = v - } - for k, v := range n.Counters { - cp.Counters[k] = v - } + cp.Metadata = n.Metadata.Copy() + cp.Counters = n.Counters.Copy() cp.Adjacency = n.Adjacency.Copy() - cp.EdgeMetadatas = n.EdgeMetadatas.Copy() + cp.Edges = n.Edges.Copy() return cp } +// Merge mergses the individual components of a node and returns a +// fresh node. +func (n NodeMetadata) Merge(other NodeMetadata) NodeMetadata { + cp := n.Copy() + cp.Metadata = cp.Metadata.Merge(other.Metadata) + cp.Counters = cp.Counters.Merge(other.Counters) + cp.Adjacency = cp.Adjacency.Merge(other.Adjacency) + cp.Edges = cp.Edges.Merge(n.Edges) + return cp +} + +// Metadata is a string->string map +type Metadata map[string]string + // Merge merges two node metadata maps together. In case of conflict, the // other (right-hand) side wins. Always reassign the result of merge to the // destination. Merge does not modify the receiver. -func (n NodeMetadata) Merge(other NodeMetadata) NodeMetadata { - cp := n.Copy() - for k, v := range other.Metadata { - cp.Metadata[k] = v // other takes precedence +func (m Metadata) Merge(other Metadata) Metadata { + result := m.Copy() + for k, v := range other { + result[k] = v // other takes precedence } - for k, v := range other.Counters { - cp.Counters[k] = n.Counters[k] + v + return result +} + +// Copy creates a deep copy of the Metadata +func (m Metadata) Copy() Metadata { + result := Metadata{} + for k, v := range m { + result[k] = v } - cp.Adjacency = cp.Adjacency.Merge(other.Adjacency) - cp.EdgeMetadatas = cp.EdgeMetadatas.Merge(n.EdgeMetadatas) - return cp + return result +} + +// Counters is a string->int map +type Counters map[string]int + +// Merge merges two sets of counters into a fresh set of counters, +// summing values where appropriate +func (c Counters) Merge(other Counters) Counters { + result := c.Copy() + for k, v := range other { + result[k] = result[k] + v + } + return result +} + +// Copy creates a deep copy of the Counters +func (c Counters) Copy() Counters { + result := Counters{} + for k, v := range c { + result[k] = v + } + return result } // EdgeMetadatas collect metadata about each edge in a topology. Keys are @@ -263,7 +299,7 @@ func (t Topology) Validate() error { } // Check all the edge metadatas have entries in adjacencies - for dstNodeID := range nmd.EdgeMetadatas { + for dstNodeID := range nmd.Edges { if _, ok := t.NodeMetadatas[dstNodeID]; !ok { errs = append(errs, fmt.Sprintf("node %s metadatas missing for edge %q", dstNodeID, nodeID)) } diff --git a/test/report_fixture.go b/test/report_fixture.go index 346d86226..78a52c762 100644 --- a/test/report_fixture.go +++ b/test/report_fixture.go @@ -95,7 +95,7 @@ var ( endpoint.Port: ClientPort54001, process.PID: Client1PID, report.HostNodeID: ClientHostNodeID, - }).WithEdgeMetadata(Server80NodeID, report.EdgeMetadata{ + }).WithEdge(Server80NodeID, report.EdgeMetadata{ EgressPacketCount: newu64(10), EgressByteCount: newu64(100), }), @@ -105,7 +105,7 @@ var ( endpoint.Port: ClientPort54002, process.PID: Client2PID, report.HostNodeID: ClientHostNodeID, - }).WithEdgeMetadata(Server80NodeID, report.EdgeMetadata{ + }).WithEdge(Server80NodeID, report.EdgeMetadata{ EgressPacketCount: newu64(20), EgressByteCount: newu64(200), }), @@ -128,7 +128,7 @@ var ( UnknownClient1NodeID: report.MakeNodeMetadata().WithMetadata(map[string]string{ endpoint.Addr: UnknownClient1IP, endpoint.Port: UnknownClient1Port, - }).WithEdgeMetadata(Server80NodeID, report.EdgeMetadata{ + }).WithEdge(Server80NodeID, report.EdgeMetadata{ EgressPacketCount: newu64(30), EgressByteCount: newu64(300), }), @@ -136,7 +136,7 @@ var ( UnknownClient2NodeID: report.MakeNodeMetadata().WithMetadata(map[string]string{ endpoint.Addr: UnknownClient2IP, endpoint.Port: UnknownClient2Port, - }).WithEdgeMetadata(Server80NodeID, report.EdgeMetadata{ + }).WithEdge(Server80NodeID, report.EdgeMetadata{ EgressPacketCount: newu64(40), EgressByteCount: newu64(400), }), @@ -144,7 +144,7 @@ var ( UnknownClient3NodeID: report.MakeNodeMetadata().WithMetadata(map[string]string{ endpoint.Addr: UnknownClient3IP, endpoint.Port: UnknownClient3Port, - }).WithEdgeMetadata(Server80NodeID, report.EdgeMetadata{ + }).WithEdge(Server80NodeID, report.EdgeMetadata{ EgressPacketCount: newu64(50), EgressByteCount: newu64(500), }), @@ -152,7 +152,7 @@ var ( RandomClientNodeID: report.MakeNodeMetadata().WithMetadata(map[string]string{ endpoint.Addr: RandomClientIP, endpoint.Port: RandomClientPort, - }).WithEdgeMetadata(Server80NodeID, report.EdgeMetadata{ + }).WithEdge(Server80NodeID, report.EdgeMetadata{ EgressPacketCount: newu64(60), EgressByteCount: newu64(600), }), @@ -229,7 +229,7 @@ var ( ClientAddressNodeID: report.MakeNodeMetadata().WithMetadata(map[string]string{ endpoint.Addr: ClientIP, report.HostNodeID: ClientHostNodeID, - }).WithEdgeMetadata(ServerAddressNodeID, report.EdgeMetadata{ + }).WithEdge(ServerAddressNodeID, report.EdgeMetadata{ MaxConnCountTCP: newu64(3), }),