github/weave-scope
1
0
Fork 0
mirror of https://github.com/weaveworks/scope.git synced 2026-03-03 18:20:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #192 from weaveworks/host-topology

Browse Source 
Host topology
This commit is contained in:
Peter Bourgon
2015-06-09 16:53:54 +02:00
parent 1e0b83eb6d da90f4489a
commit d5dd377097
26 changed files with 405 additions and 691 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
app/api_topology.go
View File

@@ -66,7 +66,7 @@ func handleNode(rep Reporter, t topologyView, w http.ResponseWriter, r *http.Req
http.NotFound(w, r)
return
}
originHostFunc := func(id string) (OriginHost, bool) { return getOriginHost(rpt.HostMetadatas, id) }
originHostFunc := func(id string) (OriginHost, bool) { return getOriginHost(rpt.Host, id) }
originNodeFunc := func(id string) (OriginNode, bool) { return getOriginNode(t.selector(rpt), id) }
respondWith(w, http.StatusOK, APINode{Node: makeDetailed(node, originHostFunc, originNodeFunc)})
}

13
app/detail_pane.go
View File

@@ -56,9 +56,8 @@ outer:
Numeric: false,
Rows: []report.Row{
{"Hostname", host.Hostname, ""},
{"Load", fmt.Sprintf("%.2f %.2f %.2f", host.LoadOne, host.LoadFive, host.LoadFifteen), ""},
{"Load", host.Load, ""},
{"OS", host.OS, ""},
//{"Addresses", strings.Join(host.Addresses, ", "), ""},
{"ID", id, ""},
},
})
@@ -75,12 +74,10 @@ outer:
func unknownOriginHost(id string) OriginHost {
return OriginHost{
Hostname: fmt.Sprintf("[%s]", id),
OS: "unknown",
Addresses: []string{},
LoadOne: 0.0,
LoadFive: 0.0,
LoadFifteen: 0.0,
Hostname: fmt.Sprintf("[%s]", id),
OS: "unknown",
Networks: []string{},
Load: "",
}
}

30
app/mock_reporter_test.go
View File

@@ -115,21 +115,23 @@ func (s StaticReport) Report() report.Report {
},
},
HostMetadatas: report.HostMetadatas{
"hostA": report.HostMetadata{
Hostname: "node-a.local",
LocalNets: []*net.IPNet{localNet},
OS: "Linux",
LoadOne: 3.1415,
LoadFive: 2.7182,
LoadFifteen: 1.6180,
},
"hostB": report.HostMetadata{
Hostname: "node-b.local",
LocalNets: []*net.IPNet{localNet},
OS: "Linux",
Host: report.Topology{
Adjacency: report.Adjacency{},
EdgeMetadatas: report.EdgeMetadatas{},
NodeMetadatas: report.NodeMetadatas{
report.MakeHostNodeID("hostA"): report.NodeMetadata{
"host_name": "node-a.local",
"os": "Linux",
"local_networks": localNet.String(),
"load": "3.14 2.71 1.61",
},
report.MakeHostNodeID("hostB"): report.NodeMetadata{
"host_name": "node-b.local",
"os": "Linux",
"local_networks": localNet.String(),
},
},
},
}
return testReport.SquashRemote()
return testReport.Squash()
}

32
app/origin_host.go
View File

@@ -2,6 +2,7 @@ package main
import (
"net/http"
"strings"
"github.com/gorilla/mux"
@@ -12,32 +13,23 @@ import (
// some data in the system. The struct is returned by the /api/origin/{id}
// handler.
type OriginHost struct {
Hostname string `json:"hostname"`
OS string `json:"os"`
Addresses []string `json:"addresses"`
LoadOne float64 `json:"load_one"`
LoadFive float64 `json:"load_five"`
LoadFifteen float64 `json:"load_fifteen"`
Hostname string `json:"hostname"`
OS string `json:"os"`
Networks []string `json:"networks"`
Load string `json:"load"`
}
func getOriginHost(mds report.HostMetadatas, nodeID string) (OriginHost, bool) {
host, ok := mds[nodeID]
func getOriginHost(t report.Topology, nodeID string) (OriginHost, bool) {
host, ok := t.NodeMetadatas[nodeID]
if !ok {
return OriginHost{}, false
}
var addrs []string
for _, l := range host.LocalNets {
addrs = append(addrs, l.String())
}
return OriginHost{
Hostname: host.Hostname,
OS: host.OS,
Addresses: addrs,
LoadOne: host.LoadOne,
LoadFive: host.LoadFive,
LoadFifteen: host.LoadFifteen,
Hostname: host["host_name"],
OS: host["os"],
Networks: strings.Split(host["local_networks"], " "),
Load: host["load"],
}, true
}
@@ -48,7 +40,7 @@ func makeOriginHostHandler(rep Reporter) http.HandlerFunc {
vars = mux.Vars(r)
nodeID = vars["id"]
)
origin, ok := getOriginHost(rep.Report().HostMetadatas, nodeID)
origin, ok := getOriginHost(rep.Report().Host, nodeID)
if !ok {
http.NotFound(w, r)
return

10
app/origin_host_test.go
View File

@@ -15,7 +15,7 @@ func TestAPIOriginHost(t *testing.T) {
{
// Origin
body := getRawJSON(t, ts, "/api/origin/host/hostA")
body := getRawJSON(t, ts, "/api/origin/host/hostA;<host>") // TODO MakeHostNodeID
var o OriginHost
if err := json.Unmarshal(body, &o); err != nil {
t.Fatalf("JSON parse error: %s", err)
@@ -23,13 +23,7 @@ func TestAPIOriginHost(t *testing.T) {
if want, have := "Linux", o.OS; want != have {
t.Errorf("Origin error. Want %v, have %v", want, have)
}
if want, have := 3.1415, o.LoadOne; want != have {
t.Errorf("Origin error. Want %v, have %v", want, have)
}
if want, have := 2.7182, o.LoadFive; want != have {
t.Errorf("Origin error. Want %v, have %v", want, have)
}
if want, have := 1.6180, o.LoadFifteen; want != have {
if want, have := "3.14 2.71 1.61", o.Load; want != have {
t.Errorf("Origin error. Want %v, have %v", want, have)
}
}

2
app/report_lifo.go
View File

@@ -41,7 +41,7 @@ func NewReportLIFO(r reporter, maxAge time.Duration) *ReportLIFO {
select {
case report := <-r.Reports():
// Incoming report from the collecter.
report = report.SquashRemote() // TODO?: make this a CLI argument.
report = report.Squash() // TODO?: make this a CLI argument.
tr := timedReport{
Timestamp: time.Now(),
Report: report,

8
app/scope_test.go
View File

@@ -71,17 +71,19 @@ func checkRequest(t *testing.T, ts *httptest.Server, method, path string, body [
func getRawJSON(t *testing.T, ts *httptest.Server, path string) []byte {
res, body := checkGet(t, ts, path)
_, file, line, _ := runtime.Caller(1)
file = filepath.Base(file)
if res.StatusCode != 200 {
t.Fatalf("Expected status %d, got %d. Path: %s", 200, res.StatusCode, path)
t.Fatalf("%s:%d: Expected status %d, got %d. Path: %s", file, line, 200, res.StatusCode, path)
}
foundCtype := res.Header.Get("content-type")
if foundCtype != "application/json" {
t.Errorf("Wrong Content-type for JSON: %s", foundCtype)
t.Errorf("%s:%d: Wrong Content-type for JSON: %s", file, line, foundCtype)
}
if len(body) == 0 {
t.Errorf("No response body")
t.Errorf("%s:%d: No response body", file, line)
}
// fmt.Printf("Body: %s", body)

2
experimental/bridge/main.go
View File

@@ -127,7 +127,7 @@ func discover(c collector, p publisher, fixed []string) {
var (
now = time.Now()
localNets = r.LocalNets()
localNets = r.LocalNetworks()
)
for _, adjacent := range r.Address.Adjacency {

10
experimental/demoprobe/generate.go
View File

@@ -112,11 +112,11 @@ func DemoReport(nodeCount int) report.Report {
r.Address.Adjacency[nodeDstAddressID] = r.Address.Adjacency[nodeDstAddressID].Add(srcAddressID)
// Host data
r.HostMetadatas["hostX"] = report.HostMetadata{
Timestamp: time.Now().UTC(),
Hostname: "host-x",
LocalNets: []*net.IPNet{localNet},
OS: "linux",
r.Host.NodeMetadatas["hostX"] = report.NodeMetadata{
"ts": time.Now().UTC().Format(time.RFC3339Nano),
"host_name": "host-x",
"local_networks": localNet.String(),
"os": "linux",
}
}

10
experimental/genreport/generate.go
View File

@@ -112,11 +112,11 @@ func DemoReport(nodeCount int) report.Report {
r.Address.Adjacency[nodeDstAddressID] = r.Address.Adjacency[nodeDstAddressID].Add(srcAddressID)
// Host data
r.HostMetadatas["hostX"] = report.HostMetadata{
Timestamp: time.Now().UTC(),
Hostname: "host-x",
LocalNets: []*net.IPNet{localNet},
OS: "linux",
r.Host.NodeMetadatas["hostX"] = report.NodeMetadata{
"ts": time.Now().UTC().Format(time.RFC3339Nano),
"host_name": "host-x",
"local_networks": localNet.String(),
"os": "linux",
}
}

2
experimental/graphviz/report_lifo.go
View File

@@ -41,7 +41,7 @@ func NewReportLIFO(r reporter, maxAge time.Duration) *ReportLIFO {
for {
select {
case report := <-r.Reports():
report = report.SquashRemote()
report = report.Squash()
tr := timedReport{
Timestamp: time.Now(),
Report: report,

58
probe/main.go
View File

@@ -9,6 +9,7 @@ import (
"os/signal"
"runtime"
"strconv"
"strings"
"syscall"
"time"
@@ -79,8 +80,8 @@ func main() {
defer close(quit)
go func() {
var (
hostname = hostname()
nodeID = hostname // TODO: we should sanitize the hostname
hostName = hostname()
hostID = hostName // TODO: we should sanitize the hostname
pubTick = time.Tick(*publishInterval)
spyTick = time.Tick(*spyInterval)
r = report.MakeReport()
@@ -90,12 +91,12 @@ func main() {
select {
case <-pubTick:
publishTicks.WithLabelValues().Add(1)
r.HostMetadatas[nodeID] = hostMetadata(hostname)
r.Host = hostTopology(hostID, hostName)
publisher.Publish(r)
r = report.MakeReport()
case <-spyTick:
r.Merge(spy(hostname, hostname, *spyProcs))
r.Merge(spy(hostID, hostName, *spyProcs))
r = tag.Apply(r, taggers)
// log.Printf("merged report:\n%#v\n", r)
@@ -108,34 +109,31 @@ func main() {
log.Printf("%s", <-interrupt())
}
// hostTopology produces a host topology for this host. No need to do this
// more than once per published report.
func hostTopology(hostID, hostName string) report.Topology {
var localCIDRs []string
if localNets, err := net.InterfaceAddrs(); err == nil {
// Not all networks are IP networks.
for _, localNet := range localNets {
if ipNet, ok := localNet.(*net.IPNet); ok {
localCIDRs = append(localCIDRs, ipNet.String())
}
}
}
t := report.NewTopology()
t.NodeMetadatas[report.MakeHostNodeID(hostID)] = report.NodeMetadata{
"ts": time.Now().UTC().Format(time.RFC3339Nano),
"host_name": hostName,
"local_networks": strings.Join(localCIDRs, " "),
"os": runtime.GOOS,
"load": getLoad(),
}
return t
}
func interrupt() chan os.Signal {
c := make(chan os.Signal)
signal.Notify(c, syscall.SIGINT, syscall.SIGTERM)
return c
}
// hostMetadata produces an instantaneous HostMetadata for this host. No need
// to do this more than once per published report.
func hostMetadata(hostname string) report.HostMetadata {
loadOne, loadFive, loadFifteen := getLoads()
host := report.HostMetadata{
Timestamp: time.Now().UTC(),
Hostname: hostname,
OS: runtime.GOOS,
LoadOne: loadOne,
LoadFive: loadFive,
LoadFifteen: loadFifteen,
}
if localNets, err := net.InterfaceAddrs(); err == nil {
// Not all networks are IP networks.
for _, localNet := range localNets {
if net, ok := localNet.(*net.IPNet); ok {
host.LocalNets = append(host.LocalNets, net)
}
}
}
return host
}

32
probe/system_darwin.go
View File

@@ -1,33 +1,21 @@
package main
import (
"fmt"
"os/exec"
"strconv"
"strings"
"regexp"
)
func getLoads() (float64, float64, float64) {
var loadRe = regexp.MustCompile(`load average\: ([0-9\.]+), ([0-9\.]+), ([0-9\.]+)`)
func getLoad() string {
out, err := exec.Command("w").CombinedOutput()
if err != nil {
return -1, -1, -1
return "unknown"
}
noCommas := strings.NewReplacer(",", "")
firstLine := strings.Split(string(out), "\n")[0]
toks := strings.Fields(firstLine)
if len(toks) < 5 {
return -1, -1, -1
matches := loadRe.FindAllStringSubmatch(string(out), -1)
if matches == nil || len(matches) < 1 || len(matches[0]) < 4 {
return "unknown"
}
one, err := strconv.ParseFloat(noCommas.Replace(toks[len(toks)-3]), 64)
if err != nil {
return -1, -1, -1
}
five, err := strconv.ParseFloat(noCommas.Replace(toks[len(toks)-2]), 64)
if err != nil {
return -1, -1, -1
}
fifteen, err := strconv.ParseFloat(noCommas.Replace(toks[len(toks)-1]), 64)
if err != nil {
return -1, -1, -1
}
return one, five, fifteen
return fmt.Sprintf("%s %s %s", matches[0][1], matches[0][2], matches[0][3])
}

15
probe/system_linux.go
View File

@@ -1,31 +1,32 @@
package main
import (
"fmt"
"io/ioutil"
"strconv"
"strings"
)
func getLoads() (float64, float64, float64) {
func getLoad() string {
buf, err := ioutil.ReadFile("/proc/loadavg")
if err != nil {
return -1, -1, -1
return "unknown"
}
toks := strings.Fields(string(buf))
if len(toks) < 3 {
return -1, -1, -1
return "unknown"
}
one, err := strconv.ParseFloat(toks[0], 64)
if err != nil {
return -1, -1, -1
return "unknown"
}
five, err := strconv.ParseFloat(toks[1], 64)
if err != nil {
return -1, -1, -1
return "unknown"
}
fifteen, err := strconv.ParseFloat(toks[2], 64)
if err != nil {
return -1, -1, -1
return "unknown"
}
return one, five, fifteen
return fmt.Sprintf("%.2f %.2f %.2f", one, five, fifteen)
}

21
report/diff_test.go Normal file
View File

@@ -0,0 +1,21 @@
package report_test
import (
"github.com/davecgh/go-spew/spew"
"github.com/pmezard/go-difflib/difflib"
)
func init() {
spew.Config.SortKeys = true // :\
}
func diff(want, have interface{}) string {
text, _ := difflib.GetUnifiedDiffString(difflib.UnifiedDiff{
A: difflib.SplitLines(spew.Sdump(want)),
B: difflib.SplitLines(spew.Sdump(have)),
FromFile: "want",
ToFile: "have",
Context: 5,
})
return "\n" + text
}

29
report/fixture_test.go
View File

@@ -1,29 +0,0 @@
package report_test
import (
"github.com/weaveworks/scope/report"
)
var (
clientHostID = "client.host.com"
clientHostName = clientHostID
clientHostNodeID = report.MakeHostNodeID(clientHostID)
clientAddress = "10.10.10.20"
serverHostID = "server.host.com"
serverHostName = serverHostID
serverHostNodeID = report.MakeHostNodeID(serverHostID)
serverAddress = "10.10.10.1"
unknownHostID = "" // by definition, we don't know it
unknownAddress = "172.16.93.112" // will be a pseudonode, no corresponding host
client54001EndpointNodeID = report.MakeEndpointNodeID(clientHostID, clientAddress, "54001") // i.e. curl
client54002EndpointNodeID = report.MakeEndpointNodeID(clientHostID, clientAddress, "54002") // also curl
server80EndpointNodeID = report.MakeEndpointNodeID(serverHostID, serverAddress, "80") // i.e. apache
unknown1EndpointNodeID = report.MakeEndpointNodeID(unknownHostID, unknownAddress, "10001")
unknown2EndpointNodeID = report.MakeEndpointNodeID(unknownHostID, unknownAddress, "10002")
unknown3EndpointNodeID = report.MakeEndpointNodeID(unknownHostID, unknownAddress, "10003")
clientAddressNodeID = report.MakeAddressNodeID(clientHostID, clientAddress)
serverAddressNodeID = report.MakeAddressNodeID(serverHostID, serverAddress)
unknownAddressNodeID = report.MakeAddressNodeID(unknownHostID, unknownAddress)
)

16
report/merge.go
View File

@@ -8,7 +8,7 @@ package report
func (r *Report) Merge(other Report) {
r.Endpoint.Merge(other.Endpoint)
r.Address.Merge(other.Address)
r.HostMetadatas.Merge(other.HostMetadatas)
r.Host.Merge(other.Host)
}
// Merge merges another Topology into the receiver.
@@ -45,20 +45,6 @@ func (e *EdgeMetadatas) Merge(other EdgeMetadatas) {
}
}
// Merge merges another HostMetadata into the receiver.
// It'll takes the lastest version if there are conflicts.
func (e *HostMetadatas) Merge(other HostMetadatas) {
for hostID, meta := range other {
if existing, ok := (*e)[hostID]; ok {
// Conflict. Take the newest.
if existing.Timestamp.After(meta.Timestamp) {
continue
}
}
(*e)[hostID] = meta
}
}
// Merge merges another EdgeMetadata into the receiver. The two edge metadatas
// should represent the same edge on different times.
func (m *EdgeMetadata) Merge(other EdgeMetadata) {

102
report/merge_test.go
View File

@@ -3,7 +3,6 @@ package report
import (
"reflect"
"testing"
"time"
)
func TestMergeAdjacency(t *testing.T) {
@@ -215,107 +214,6 @@ func TestMergeEdgeMetadatas(t *testing.T) {
}
}
func TestMergeHostMetadatas(t *testing.T) {
now := time.Now()
for name, c := range map[string]struct {
a, b, want HostMetadatas
}{
"Empty a": {
a: HostMetadatas{},
b: HostMetadatas{
"hostA": HostMetadata{
Timestamp: now,
Hostname: "host-a",
OS: "linux",
},
},
want: HostMetadatas{
"hostA": HostMetadata{
Timestamp: now,
Hostname: "host-a",
OS: "linux",
},
},
},
"Empty b": {
a: HostMetadatas{
"hostA": HostMetadata{
Timestamp: now,
Hostname: "host-a",
OS: "linux",
},
},
b: HostMetadatas{},
want: HostMetadatas{
"hostA": HostMetadata{
Timestamp: now,
Hostname: "host-a",
OS: "linux",
},
},
},
"Host merge": {
a: HostMetadatas{
"hostA": HostMetadata{
Timestamp: now,
Hostname: "host-a",
OS: "linux",
},
},
b: HostMetadatas{
"hostB": HostMetadata{
Timestamp: now,
Hostname: "host-b",
OS: "freedos",
},
},
want: HostMetadatas{
"hostB": HostMetadata{
Timestamp: now,
Hostname: "host-b",
OS: "freedos",
},
"hostA": HostMetadata{
Timestamp: now,
Hostname: "host-a",
OS: "linux",
},
},
},
"Host conflict": {
a: HostMetadatas{
"hostA": HostMetadata{
Timestamp: now,
Hostname: "host-a",
OS: "linux1",
},
},
b: HostMetadatas{
"hostA": HostMetadata{
Timestamp: now.Add(-10 * time.Second),
Hostname: "host-a",
OS: "linux0",
},
},
want: HostMetadatas{
"hostA": HostMetadata{
Timestamp: now,
Hostname: "host-a",
OS: "linux1",
},
},
},
} {
have := c.a
have.Merge(c.b)
if !reflect.DeepEqual(c.want, have) {
t.Errorf("%s: want\n\t%#v, have\n\t%#v", name, c.want, have)
}
}
}
func TestMergeNodeMetadatas(t *testing.T) {
for name, c := range map[string]struct {
a, b, want NodeMetadatas

110
report/report.go
View File

@@ -1,9 +1,8 @@
package report
import (
"encoding/json"
"net"
"time"
"strings"
)
// Report is the core data type. It's produced by probes, and consumed and
@@ -20,20 +19,10 @@ type Report struct {
// endpoints (e.g. ICMP). Edges are present.
Address Topology
HostMetadatas
}
// HostMetadatas contains metadata about the host(s) represented in the Report.
type HostMetadatas map[string]HostMetadata
// HostMetadata describes metadata that probes can collect about the host that
// they run on. It has a timestamp when the measurement was made.
type HostMetadata struct {
Timestamp time.Time
Hostname string
LocalNets []*net.IPNet
OS string
LoadOne, LoadFive, LoadFifteen float64
// Host nodes are physical hosts that run probes. Metadata includes things
// like operating system, load, etc. The information is scraped by the
// probes with each published report. Edges are not present.
Host Topology
}
// RenderableNode is the data type that's yielded to the JavaScript layer as
@@ -79,69 +68,46 @@ type Row struct {
// MakeReport makes a clean report, ready to Merge() other reports into.
func MakeReport() Report {
return Report{
Endpoint: NewTopology(),
Address: NewTopology(),
HostMetadatas: map[string]HostMetadata{},
Endpoint: NewTopology(),
Address: NewTopology(),
Host: NewTopology(),
}
}
// SquashRemote folds all remote nodes into a special supernode. It uses the
// LocalNets of the hosts in HostMetadata to determine which addresses are
// local.
func (r Report) SquashRemote() Report {
localNets := r.HostMetadatas.LocalNets()
return Report{
Endpoint: Squash(r.Endpoint, EndpointIDAddresser, localNets),
Address: Squash(r.Address, AddressIDAddresser, localNets),
HostMetadatas: r.HostMetadatas,
}
// Squash squashes all non-local nodes in the report to a super-node called
// the Internet.
func (r Report) Squash() Report {
localNetworks := r.LocalNetworks()
r.Endpoint = r.Endpoint.Squash(EndpointIDAddresser, localNetworks)
r.Address = r.Address.Squash(AddressIDAddresser, localNetworks)
r.Host = r.Host.Squash(PanicIDAddresser, localNetworks)
return r
}
// LocalNets gives the union of all local network IPNets for all hosts
// represented in the HostMetadatas.
func (m HostMetadatas) LocalNets() []*net.IPNet {
var nets []*net.IPNet
for _, node := range m {
OUTER:
for _, local := range node.LocalNets {
for _, existing := range nets {
if existing == local {
continue OUTER
// LocalNetworks returns a superset of the networks (think: CIDRs) that are
// "local" from the perspective of each host represented in the report. It's
// used to determine which nodes in the report are "remote", i.e. outside of
// our infrastructure.
func (r Report) LocalNetworks() []*net.IPNet {
var ipNets []*net.IPNet
for _, md := range r.Host.NodeMetadatas {
val, ok := md["local_networks"]
if !ok {
continue
}
outer:
for _, s := range strings.Fields(val) {
_, ipNet, err := net.ParseCIDR(s)
if err != nil {
continue
}
for _, existing := range ipNets {
if ipNet.String() == existing.String() {
continue outer
}
}
nets = append(nets, local)
ipNets = append(ipNets, ipNet)
}
}
return nets
}
// UnmarshalJSON is a custom JSON deserializer for HostMetadata to deal with
// the Localnets.
func (m *HostMetadata) UnmarshalJSON(data []byte) error {
type netmask struct {
IP net.IP
Mask []byte
}
tmpHMD := struct {
Timestamp time.Time
Hostname string
LocalNets []*netmask
OS string
LoadOne, LoadFive, LoadFifteen float64
}{}
err := json.Unmarshal(data, &tmpHMD)
if err != nil {
return err
}
m.Timestamp = tmpHMD.Timestamp
m.Hostname = tmpHMD.Hostname
m.OS = tmpHMD.OS
m.LoadOne = tmpHMD.LoadOne
m.LoadFive = tmpHMD.LoadFive
m.LoadFifteen = tmpHMD.LoadFifteen
for _, ln := range tmpHMD.LocalNets {
m.LocalNets = append(m.LocalNets, &net.IPNet{IP: ln.IP, Mask: ln.Mask})
}
return nil
return ipNets
}

166
report/report_fixture_test.go Normal file
View File

@@ -0,0 +1,166 @@
package report_test
import (
"github.com/weaveworks/scope/report"
)
var reportFixture = report.Report{
Endpoint: report.Topology{
Adjacency: report.Adjacency{
report.MakeAdjacencyID(clientHostID, client54001EndpointNodeID): report.MakeIDList(server80EndpointNodeID),
report.MakeAdjacencyID(clientHostID, client54002EndpointNodeID): report.MakeIDList(server80EndpointNodeID),
report.MakeAdjacencyID(serverHostID, server80EndpointNodeID): report.MakeIDList(client54001EndpointNodeID, client54002EndpointNodeID, unknown1EndpointNodeID, unknown2EndpointNodeID, unknown3EndpointNodeID),
},
NodeMetadatas: report.NodeMetadatas{
client54001EndpointNodeID: report.NodeMetadata{
"process_node_id": report.MakeProcessNodeID(clientHostID, "4242"),
"address_node_id": report.MakeAddressNodeID(clientHostID, clientAddress),
},
client54002EndpointNodeID: report.NodeMetadata{
//"process_node_id": report.MakeProcessNodeID(clientHostID, "4242"), // leave it out, to test a branch in Render
"address_node_id": report.MakeAddressNodeID(clientHostID, clientAddress),
},
server80EndpointNodeID: report.NodeMetadata{
"process_node_id": report.MakeProcessNodeID(serverHostID, "215"),
"address_node_id": report.MakeAddressNodeID(serverHostID, serverAddress),
},
"process-not-available": report.NodeMetadata{}, // for TestProcess{PID,Name,Container[Name]}
"process-badly-linked": report.NodeMetadata{"process_node_id": "none"}, // for TestProcess{PID,Name,Container[Name]}
"process-no-container": report.NodeMetadata{"process_node_id": "no-container"}, // for TestProcessContainer[Name]
"address-not-available": report.NodeMetadata{}, // for TestAddressHostname
"address-badly-linked": report.NodeMetadata{"address_node_id": "none"}, // for TestAddressHostname
},
EdgeMetadatas: report.EdgeMetadatas{
report.MakeEdgeID(client54001EndpointNodeID, server80EndpointNodeID): report.EdgeMetadata{
WithBytes: true,
BytesEgress: 10, // src -> dst
BytesIngress: 100, // src <- dst
},
report.MakeEdgeID(client54002EndpointNodeID, server80EndpointNodeID): report.EdgeMetadata{
WithBytes: true,
BytesEgress: 20,
BytesIngress: 200,
},
report.MakeEdgeID(server80EndpointNodeID, client54001EndpointNodeID): report.EdgeMetadata{
WithBytes: true,
BytesEgress: 100,
BytesIngress: 10,
},
report.MakeEdgeID(server80EndpointNodeID, client54002EndpointNodeID): report.EdgeMetadata{
WithBytes: true,
BytesEgress: 200,
BytesIngress: 20,
},
report.MakeEdgeID(server80EndpointNodeID, unknown1EndpointNodeID): report.EdgeMetadata{
WithBytes: true,
BytesEgress: 400,
BytesIngress: 40,
},
report.MakeEdgeID(server80EndpointNodeID, unknown2EndpointNodeID): report.EdgeMetadata{
WithBytes: true,
BytesEgress: 800,
BytesIngress: 80,
},
report.MakeEdgeID(server80EndpointNodeID, unknown3EndpointNodeID): report.EdgeMetadata{
WithBytes: true,
BytesEgress: 1600,
BytesIngress: 160,
},
},
},
Address: report.Topology{
Adjacency: report.Adjacency{
report.MakeAdjacencyID(clientHostID, clientAddressNodeID): report.MakeIDList(serverAddressNodeID),
report.MakeAdjacencyID(serverHostID, serverAddressNodeID): report.MakeIDList(clientAddressNodeID, unknownAddressNodeID),
},
NodeMetadatas: report.NodeMetadatas{
clientAddressNodeID: report.NodeMetadata{
"host_name": "client.host.com",
},
serverAddressNodeID: report.NodeMetadata{},
"no-host-name": report.NodeMetadata{},
},
EdgeMetadatas: report.EdgeMetadatas{
report.MakeEdgeID(clientAddressNodeID, serverAddressNodeID): report.EdgeMetadata{
WithBytes: true,
BytesEgress: 10 + 20 + 1,
BytesIngress: 100 + 200 + 2,
},
report.MakeEdgeID(serverAddressNodeID, clientAddressNodeID): report.EdgeMetadata{
WithBytes: true,
BytesEgress: 100 + 200 + 3,
BytesIngress: 10 + 20 + 4,
},
report.MakeEdgeID(serverAddressNodeID, unknownAddressNodeID): report.EdgeMetadata{
WithBytes: true,
BytesEgress: 400 + 800 + 1600 + 5,
BytesIngress: 40 + 80 + 160 + 6,
},
},
},
//Process: report.Topology{
// Adjacency: report.Adjacency{},
// NodeMetadatas: report.NodeMetadatas{
// report.MakeProcessNodeID(clientHostID, "4242"): report.NodeMetadata{
// "host_name": "client.host.com",
// "pid": "4242",
// "process_name": "curl",
// "docker_container_id": "a1b2c3d4e5",
// "docker_container_name": "fixture-container",
// "docker_image_id": "0000000000",
// "docker_image_name": "fixture/container:latest",
// },
// report.MakeProcessNodeID(serverHostID, "215"): report.NodeMetadata{
// "pid": "215",
// "process_name": "apache",
// },
//
// "no-container": report.NodeMetadata{},
// },
// EdgeMetadatas: report.EdgeMetadatas{},
//},
Host: report.Topology{
Adjacency: report.Adjacency{},
NodeMetadatas: report.NodeMetadatas{
report.MakeHostNodeID(clientHostID): report.NodeMetadata{
"host_name": clientHostName,
"local_networks": "10.10.10.0/24",
"os": "OS/2",
"load": "0.11 0.22 0.33",
},
report.MakeHostNodeID(serverHostID): report.NodeMetadata{
"host_name": serverHostName,
"local_networks": "10.10.10.0/24",
"os": "Linux",
"load": "0.01 0.01 0.01",
},
},
EdgeMetadatas: report.EdgeMetadatas{},
},
}
var (
clientHostID = "client.host.com"
clientHostName = clientHostID
clientHostNodeID = report.MakeHostNodeID(clientHostID)
clientAddress = "10.10.10.20"
serverHostID = "server.host.com"
serverHostName = serverHostID
serverHostNodeID = report.MakeHostNodeID(serverHostID)
serverAddress = "10.10.10.1"
unknownHostID = "" // by definition, we don't know it
unknownAddress = "172.16.93.112" // will be a pseudonode, no corresponding host
client54001EndpointNodeID = report.MakeEndpointNodeID(clientHostID, clientAddress, "54001") // i.e. curl
client54002EndpointNodeID = report.MakeEndpointNodeID(clientHostID, clientAddress, "54002") // also curl
server80EndpointNodeID = report.MakeEndpointNodeID(serverHostID, serverAddress, "80") // i.e. apache
unknown1EndpointNodeID = report.MakeEndpointNodeID(unknownHostID, unknownAddress, "10001")
unknown2EndpointNodeID = report.MakeEndpointNodeID(unknownHostID, unknownAddress, "10002")
unknown3EndpointNodeID = report.MakeEndpointNodeID(unknownHostID, unknownAddress, "10003")
clientAddressNodeID = report.MakeAddressNodeID(clientHostID, clientAddress)
serverAddressNodeID = report.MakeAddressNodeID(serverHostID, serverAddress)
unknownAddressNodeID = report.MakeAddressNodeID(unknownHostID, unknownAddress)
)

76
report/report_test.go
View File

@@ -1,37 +1,55 @@
package report
package report_test
import (
"encoding/json"
"fmt"
"net"
"reflect"
"testing"
"time"
"github.com/weaveworks/scope/report"
)
func TestHostJSON(t *testing.T) {
_, localNet, _ := net.ParseCIDR("192.168.1.2/16")
host := HostMetadata{
Timestamp: time.Now(),
Hostname: "euclid",
LocalNets: []*net.IPNet{localNet},
OS: "linux",
func TestReportLocalNetworks(t *testing.T) {
r := report.MakeReport()
r.Merge(report.Report{Host: report.Topology{NodeMetadatas: report.NodeMetadatas{
"nonets": {},
"foo": {"local_networks": "10.0.0.1/8 192.168.1.1/24 10.0.0.1/8 badnet/33"},
}}})
if want, have := []*net.IPNet{
mustParseCIDR("10.0.0.1/8"),
mustParseCIDR("192.168.1.1/24"),
}, r.LocalNetworks(); !reflect.DeepEqual(want, have) {
t.Errorf("want %+v, have %+v", want, have)
}
e, err := json.Marshal(host)
if err != nil {
t.Fatalf("Marshal error: %v", err)
}
var hostAgain HostMetadata
err = json.Unmarshal(e, &hostAgain)
if err != nil {
t.Fatalf("Unarshal error: %v", err)
}
// need to compare pointers. No fun.
want := fmt.Sprintf("%+v", host)
got := fmt.Sprintf("%+v", hostAgain)
if want != got {
t.Errorf("Host not the same. Want \n%+v, got \n%+v", want, got)
}
}
func TestReportSquash(t *testing.T) {
{
want := report.Adjacency{
report.MakeAdjacencyID(clientHostID, client54001EndpointNodeID): report.MakeIDList(server80EndpointNodeID),
report.MakeAdjacencyID(clientHostID, client54002EndpointNodeID): report.MakeIDList(server80EndpointNodeID),
report.MakeAdjacencyID(serverHostID, server80EndpointNodeID): report.MakeIDList(client54001EndpointNodeID, client54002EndpointNodeID, report.TheInternet),
}
have := reportFixture.Squash().Endpoint.Adjacency
if !reflect.DeepEqual(want, have) {
t.Error(diff(want, have))
}
}
{
want := report.Adjacency{
report.MakeAdjacencyID(clientHostID, clientAddressNodeID): report.MakeIDList(serverAddressNodeID),
report.MakeAdjacencyID(serverHostID, serverAddressNodeID): report.MakeIDList(clientAddressNodeID, report.TheInternet),
}
have := reportFixture.Squash().Address.Adjacency
if !reflect.DeepEqual(want, have) {
t.Error(diff(want, have))
}
}
}
func mustParseCIDR(s string) *net.IPNet {
_, ipNet, err := net.ParseCIDR(s)
if err != nil {
panic(err)
}
return ipNet
}

57
report/squash.go
View File

@@ -1,57 +0,0 @@
package report
import (
"log"
"net"
)
// Squash takes a Topology, and folds all remote nodes into a supernode.
func Squash(t Topology, f IDAddresser, localNets []*net.IPNet) Topology {
newTopo := NewTopology()
isRemote := func(ip net.IP) bool { return !netsContain(localNets, ip) }
// If any node ID on the right-hand (destination) side of an adjacency
// list is remote, rename it to TheInternet. (We'll never have remote
// nodes on the left-hand (source) side of an adjacency list, by
// definition.)
for nodeID, adjacent := range t.Adjacency {
var newAdjacency IDList
for _, adjacentID := range adjacent {
if isRemote(f(adjacentID)) {
adjacentID = TheInternet
}
newAdjacency = newAdjacency.Add(adjacentID)
}
newTopo.Adjacency[nodeID] = newAdjacency
}
// Edge metadata keys are "<src node ID>|<dst node ID>". If the dst node
// ID is remote, rename it to TheInternet.
for key, metadata := range t.EdgeMetadatas {
srcNodeID, dstNodeID, ok := ParseEdgeID(key)
if !ok {
log.Printf("bad edge ID %q", key)
continue
}
if ip := f(dstNodeID); ip != nil && isRemote(ip) {
key = MakeEdgeID(srcNodeID, TheInternet)
}
// Could be we're merging two keys into one now.
summedMetadata := newTopo.EdgeMetadatas[key]
summedMetadata.Flatten(metadata)
newTopo.EdgeMetadatas[key] = summedMetadata
}
newTopo.NodeMetadatas = t.NodeMetadatas
return newTopo
}
func netsContain(nets []*net.IPNet, ip net.IP) bool {
for _, net := range nets {
if net.Contains(ip) {
return true
}
}
return false
}

256
report/squash_test.go
View File

@@ -1,256 +0,0 @@
package report
import (
"net"
"reflect"
"testing"
)
var (
_, netdot1, _ = net.ParseCIDR("192.168.1.0/24")
_, netdot2, _ = net.ParseCIDR("192.168.2.0/24")
)
func reportToSquash() Report {
return Report{
Endpoint: Topology{
Adjacency: Adjacency{
"hostA|;192.168.1.1;12345": []string{";192.168.1.2;80"},
"hostA|;192.168.1.1;8888": []string{";1.2.3.4;22", ";1.2.3.4;23"},
"hostB|;192.168.1.2;80": []string{";192.168.1.1;12345"},
"hostZ|;192.168.2.2;80": []string{";192.168.1.1;12345"},
},
EdgeMetadatas: EdgeMetadatas{
";192.168.1.1;12345|;192.168.1.2;80": EdgeMetadata{
WithBytes: true,
BytesEgress: 12,
BytesIngress: 0,
},
";192.168.1.1;8888|;1.2.3.4;22": EdgeMetadata{
WithBytes: true,
BytesEgress: 200,
BytesIngress: 0,
},
";192.168.1.1;8888|;1.2.3.4;23": EdgeMetadata{
WithBytes: true,
BytesEgress: 200,
BytesIngress: 0,
},
";192.168.1.2;80|;192.168.1.1;12345": EdgeMetadata{
WithBytes: true,
BytesEgress: 0,
BytesIngress: 12,
},
";192.168.2.2;80|;192.168.1.1;12345": EdgeMetadata{
WithBytes: true,
BytesEgress: 0,
BytesIngress: 12,
},
},
NodeMetadatas: NodeMetadatas{
";192.168.1.1;12345": NodeMetadata{
"pid": "23128",
"name": "curl",
"domain": "node-a.local",
},
";192.168.1.1;8888": NodeMetadata{
"pid": "55100",
"name": "ssh",
"domain": "node-a.local",
},
";192.168.1.2;80": NodeMetadata{
"pid": "215",
"name": "apache",
"domain": "node-b.local",
},
";192.168.2.2;80": NodeMetadata{
"pid": "213",
"name": "apache",
"domain": "node-z.local",
},
},
},
Address: Topology{
Adjacency: Adjacency{
"hostA|;192.168.1.1": []string{";192.168.1.2", ";1.2.3.4"},
"hostB|;192.168.1.2": []string{";192.168.1.1"},
"hostZ|;192.168.2.2": []string{";192.168.1.1"},
},
EdgeMetadatas: EdgeMetadatas{
";192.168.1.1|;192.168.1.2": EdgeMetadata{
WithBytes: true,
BytesEgress: 12,
BytesIngress: 0,
},
";192.168.1.1|;1.2.3.4": EdgeMetadata{
WithBytes: true,
BytesEgress: 200,
BytesIngress: 0,
},
";192.168.1.2|;192.168.1.1": EdgeMetadata{
WithBytes: true,
BytesEgress: 0,
BytesIngress: 12,
},
";192.168.2.2|;192.168.1.1": EdgeMetadata{
WithBytes: true,
BytesEgress: 0,
BytesIngress: 12,
},
},
NodeMetadatas: NodeMetadatas{
";192.168.1.1": NodeMetadata{
"name": "host-a",
},
";192.168.1.2": NodeMetadata{
"name": "host-b",
},
";192.168.2.2": NodeMetadata{
"name": "host-z",
},
},
},
HostMetadatas: HostMetadatas{
"hostA": HostMetadata{
Hostname: "node-a.local",
OS: "Linux",
LocalNets: []*net.IPNet{netdot1},
},
"hostB": HostMetadata{
Hostname: "node-b.local",
OS: "Linux",
LocalNets: []*net.IPNet{netdot1},
},
"hostZ": HostMetadata{
Hostname: "node-z.local",
OS: "Linux",
LocalNets: []*net.IPNet{netdot2},
},
},
}
}
func TestSquashTopology(t *testing.T) {
// Tests just a topology
want := Topology{
Adjacency: Adjacency{
"hostA|;192.168.1.1;12345": []string{";192.168.1.2;80"},
"hostA|;192.168.1.1;8888": []string{"theinternet"},
"hostB|;192.168.1.2;80": []string{";192.168.1.1;12345"},
"hostZ|;192.168.2.2;80": []string{";192.168.1.1;12345"},
},
EdgeMetadatas: EdgeMetadatas{
";192.168.1.1;12345|;192.168.1.2;80": EdgeMetadata{
WithBytes: true,
BytesEgress: 12,
BytesIngress: 0,
},
";192.168.1.1;8888|theinternet": EdgeMetadata{
WithBytes: true,
BytesEgress: 2 * 200,
BytesIngress: 2 * 0,
},
";192.168.1.2;80|;192.168.1.1;12345": EdgeMetadata{
WithBytes: true,
BytesEgress: 0,
BytesIngress: 12,
},
";192.168.2.2;80|;192.168.1.1;12345": EdgeMetadata{
WithBytes: true,
BytesEgress: 0,
BytesIngress: 12,
},
},
NodeMetadatas: reportToSquash().Endpoint.NodeMetadatas,
}
have := Squash(reportToSquash().Endpoint, EndpointIDAddresser, reportToSquash().HostMetadatas.LocalNets())
if !reflect.DeepEqual(want, have) {
t.Errorf("want\n\t%#v, have\n\t%#v", want, have)
}
}
func TestSquashReport(t *testing.T) {
// Tests a full report squash.
want := Report{
Endpoint: Topology{
Adjacency: Adjacency{
"hostA|;192.168.1.1;12345": []string{";192.168.1.2;80"},
"hostA|;192.168.1.1;8888": []string{"theinternet"},
"hostB|;192.168.1.2;80": []string{";192.168.1.1;12345"},
"hostZ|;192.168.2.2;80": []string{";192.168.1.1;12345"},
},
EdgeMetadatas: EdgeMetadatas{
";192.168.1.1;12345|;192.168.1.2;80": EdgeMetadata{
WithBytes: true,
BytesEgress: 12,
BytesIngress: 0,
},
";192.168.1.1;8888|theinternet": EdgeMetadata{
WithBytes: true,
BytesEgress: 2 * 200,
BytesIngress: 2 * 0,
},
";192.168.1.2;80|;192.168.1.1;12345": EdgeMetadata{
WithBytes: true,
BytesEgress: 0,
BytesIngress: 12,
},
";192.168.2.2;80|;192.168.1.1;12345": EdgeMetadata{
WithBytes: true,
BytesEgress: 0,
BytesIngress: 12,
},
},
NodeMetadatas: reportToSquash().Endpoint.NodeMetadatas,
},
Address: Topology{
Adjacency: Adjacency{
"hostA|;192.168.1.1": []string{";192.168.1.2", "theinternet"},
"hostB|;192.168.1.2": []string{";192.168.1.1"},
"hostZ|;192.168.2.2": []string{";192.168.1.1"},
},
EdgeMetadatas: EdgeMetadatas{
";192.168.1.1|;192.168.1.2": EdgeMetadata{
WithBytes: true,
BytesEgress: 12,
BytesIngress: 0,
},
";192.168.1.1|theinternet": EdgeMetadata{
WithBytes: true,
BytesEgress: 200,
BytesIngress: 0,
},
";192.168.1.2|;192.168.1.1": EdgeMetadata{
WithBytes: true,
BytesEgress: 0,
BytesIngress: 12,
},
";192.168.2.2|;192.168.1.1": EdgeMetadata{
WithBytes: true,
BytesEgress: 0,
BytesIngress: 12,
},
},
NodeMetadatas: NodeMetadatas{
";192.168.1.1": NodeMetadata{
"name": "host-a",
},
";192.168.1.2": NodeMetadata{
"name": "host-b",
},
";192.168.2.2": NodeMetadata{
"name": "host-z",
},
},
},
HostMetadatas: reportToSquash().HostMetadatas,
}
have := reportToSquash().SquashRemote()
if !reflect.DeepEqual(want, have) {
t.Errorf("want\n\t%#v, have\n\t%#v", want, have)
}
}

27
report/topology.go
View File

@@ -2,6 +2,7 @@ package report
import (
"log"
"net"
"reflect"
)
@@ -182,6 +183,32 @@ func (t Topology) EdgeMetadata(mapFunc MapFunc, srcRenderableID, dstRenderableID
return metadata
}
// Squash squashes all non-local nodes in the topology to a super-node called
// the Internet.
func (t Topology) Squash(f IDAddresser, localNets []*net.IPNet) Topology {
isRemote := func(ip net.IP) bool { return !netsContain(localNets, ip) }
for srcID, dstIDs := range t.Adjacency {
newDstIDs := make(IDList, 0, len(dstIDs))
for _, dstID := range dstIDs {
if ip := f(dstID); ip != nil && isRemote(ip) {
dstID = TheInternet
}
newDstIDs = newDstIDs.Add(dstID)
}
t.Adjacency[srcID] = newDstIDs
}
return t
}
func netsContain(nets []*net.IPNet, ip net.IP) bool {
for _, net := range nets {
if net.Contains(ip) {
return true
}
}
return false
}
// Diff is returned by TopoDiff. It represents the changes between two
// RenderableNode maps.
type Diff struct {

2
report/topology_test.go
View File

@@ -406,5 +406,5 @@ func diff(want, have interface{}) string {
ToFile: "have",
Context: 3,
})
return text
return "\n" + text
}

8
xfer/merge_test.go
View File

@@ -39,12 +39,12 @@ func TestMerge(t *testing.T) {
{
r := report.MakeReport()
r.HostMetadatas["p1"] = report.HostMetadata{Hostname: "test1"}
r.Host.NodeMetadatas["p1"] = report.NodeMetadata{"host_name": "test1"}
p1.Publish(r)
}
{
r := report.MakeReport()
r.HostMetadatas["p2"] = report.HostMetadata{Hostname: "test2"}
r.Host.NodeMetadatas["p2"] = report.NodeMetadata{"host_name": "test2"}
p2.Publish(r)
}
@@ -52,10 +52,10 @@ func TestMerge(t *testing.T) {
go func() {
defer close(success)
for r := range c.Reports() {
if r.HostMetadatas["p1"].Hostname != "test1" {
if r.Host.NodeMetadatas["p1"]["host_name"] != "test1" {
continue
}
if r.HostMetadatas["p2"].Hostname != "test2" {
if r.Host.NodeMetadatas["p2"]["host_name"] != "test2" {
continue
}
return