Add metrics for docker container cpu and memory, and host load.

probe/docker/container.go

@@ -91,9 +91,10 @@ type Container interface {
 
 type container struct {
 	sync.RWMutex
-	container   *docker.Container
-	statsConn   ClientConn
-	latestStats *docker.Stats
+	container    *docker.Container
+	statsConn    ClientConn
+	latestStats  *docker.Stats
+	pendingStats []*docker.Stats
 }
 
 // NewContainer creates a new Container
@@ -190,6 +191,7 @@ func (c *container) StartGatheringStats() error {
 
 			c.Lock()
 			c.latestStats = stats
+			c.pendingStats = append(c.pendingStats, stats)
 			c.Unlock()
 
 			stats = &docker.Stats{}
@@ -238,6 +240,52 @@ func (c *container) ports(localAddrs []net.IP) report.StringSet {
 	return report.MakeStringSet(ports...)
 }
 
+func (c *container) memoryUsageMetric() report.Metric {
+	result := report.MakeMetric()
+	for _, s := range c.pendingStats {
+		result = result.Add(s.Read, float64(s.MemoryStats.Usage))
+	}
+	return result
+}
+
+func (c *container) cpuPercentMetric() report.Metric {
+	result := report.MakeMetric()
+	if len(c.pendingStats) < 2 {
+		return result
+	}
+
+	previous := c.pendingStats[0]
+	for _, s := range c.pendingStats[1:] {
+		// Copies from docker/api/client/stats.go#L205
+		cpuDelta := float64(s.CPUStats.CPUUsage.TotalUsage - previous.CPUStats.CPUUsage.TotalUsage)
+		systemDelta := float64(s.CPUStats.SystemCPUUsage - previous.CPUStats.SystemCPUUsage)
+		cpuPercent := 0.0
+		if systemDelta > 0.0 && cpuDelta > 0.0 {
+			cpuPercent = (cpuDelta / systemDelta) * float64(len(s.CPUStats.CPUUsage.PercpuUsage)) * 100.0
+		}
+		result = result.Add(s.Read, cpuPercent)
+		available := float64(len(s.CPUStats.CPUUsage.PercpuUsage)) * 100.0
+		if available >= result.Max {
+			result.Max = available
+		}
+		previous = s
+	}
+	return result
+}
+
+func (c *container) metrics() report.Metrics {
+	result := report.Metrics{
+		MemoryUsage:   c.memoryUsageMetric(),
+		CPUTotalUsage: c.cpuPercentMetric(),
+	}
+
+	// Keep the latest report to help with relative metric reporting.
+	if len(c.pendingStats) > 0 {
+		c.pendingStats = c.pendingStats[len(c.pendingStats)-1:]
+	}
+	return result
+}
+
 func (c *container) GetNode(hostID string, localAddrs []net.IP) report.Node {
 	c.RLock()
 	defer c.RUnlock()
@@ -269,7 +317,9 @@ func (c *container) GetNode(hostID string, localAddrs []net.IP) report.Node {
 		ContainerPorts:         c.ports(localAddrs),
 		ContainerIPs:           report.MakeStringSet(ips...),
 		ContainerIPsWithScopes: report.MakeStringSet(ipsWithScopes...),
-	}).WithLatest(ContainerState, mtime.Now(), state)
+	}).WithLatest(
+		ContainerState, mtime.Now(), state,
+	).WithMetrics(c.metrics())
 
 	if c.container.State.Paused {
 		result = result.WithControls(UnpauseContainer)
@@ -305,8 +355,7 @@ func (c *container) GetNode(hostID string, localAddrs []net.IP) report.Node {
 		CPUTotalUsage:        strconv.FormatUint(c.latestStats.CPUStats.CPUUsage.TotalUsage, 10),
 		CPUUsageInKernelmode: strconv.FormatUint(c.latestStats.CPUStats.CPUUsage.UsageInKernelmode, 10),
 		CPUSystemCPUUsage:    strconv.FormatUint(c.latestStats.CPUStats.SystemCPUUsage, 10),
-	})
-
+	}).WithMetrics(c.metrics())
 	return result
 }
 

probe/docker/container_test.go

@@ -58,17 +58,18 @@ func TestContainer(t *testing.T) {
 	}
 	defer c.StopGatheringStats()
 
+	now := time.Unix(12345, 67890).UTC()
+	mtime.NowForce(now)
+	defer mtime.NowReset()
+
 	// Send some stats to the docker container
 	stats := &client.Stats{}
+	stats.Read = now
 	stats.MemoryStats.Usage = 12345
 	if err = json.NewEncoder(writer).Encode(&stats); err != nil {
 		t.Error(err)
 	}
 
-	now := time.Now()
-	mtime.NowForce(now)
-	defer mtime.NowReset()
-
 	// Now see if we go them
 	want := report.MakeNode().WithMetadata(map[string]string{
 		"docker_container_command": " ",
@@ -85,8 +86,12 @@ func TestContainer(t *testing.T) {
 		"docker_container_ips_with_scopes": report.MakeStringSet("scope;1.2.3.4"),
 	}).WithControls(
 		docker.RestartContainer, docker.StopContainer, docker.PauseContainer,
-	).WithLatest("docker_container_state", now, "running")
-
+	).WithLatest(
+		"docker_container_state", now, "running",
+	).WithMetrics(report.Metrics{
+		"cpu_total_usage": report.MakeMetric(),
+		"memory_usage":    report.MakeMetric().Add(now, 12345),
+	})
 	test.Poll(t, 100*time.Millisecond, want, func() interface{} {
 		node := c.GetNode("scope", []net.IP{})
 		for k, v := range node.Metadata {

probe/host/reporter.go

@@ -13,9 +13,11 @@ const (
 	HostName      = "host_name"
 	LocalNetworks = "local_networks"
 	OS            = "os"
-	Load          = "load"
 	KernelVersion = "kernel_version"
 	Uptime        = "uptime"
+	Load1         = "load1"
+	Load5         = "load5"
+	Load15        = "load15"
 )
 
 // Exposed for testing.
@@ -71,12 +73,11 @@ func (r *Reporter) Report() (report.Report, error) {
 		Timestamp:     Now(),
 		HostName:      r.hostName,
 		OS:            runtime.GOOS,
-		Load:          GetLoad(),
 		KernelVersion: kernel,
 		Uptime:        uptime.String(),
 	}).WithSets(report.Sets{
 		LocalNetworks: report.MakeStringSet(localCIDRs...),
-	}))
+	}).WithMetrics(GetLoad()))
 
 	return rep, nil
 }

probe/host/reporter_test.go

@@ -14,13 +14,18 @@ import (
 
 func TestReporter(t *testing.T) {
 	var (
-		release     = "release"
-		version     = "version"
-		network     = "192.168.0.0/16"
-		hostID      = "hostid"
-		now         = "now"
-		hostname    = "hostname"
-		load        = "0.59 0.36 0.29"
+		release   = "release"
+		version   = "version"
+		network   = "192.168.0.0/16"
+		hostID    = "hostid"
+		now       = "now"
+		hostname  = "hostname"
+		timestamp = time.Now()
+		load      = report.Metrics{
+			host.Load1:  report.MakeMetric().Add(timestamp, 1.0),
+			host.Load5:  report.MakeMetric().Add(timestamp, 5.0),
+			host.Load15: report.MakeMetric().Add(timestamp, 15.0),
+		}
 		uptime      = "278h55m43s"
 		kernel      = "release version"
 		_, ipnet, _ = net.ParseCIDR(network)
@@ -40,7 +45,7 @@ func TestReporter(t *testing.T) {
 		host.Now = oldNow
 	}()
 	host.GetKernelVersion = func() (string, error) { return release + " " + version, nil }
-	host.GetLoad = func() string { return load }
+	host.GetLoad = func() report.Metrics { return load }
 	host.GetUptime = func() (time.Duration, error) { return time.ParseDuration(uptime) }
 	host.Now = func() string { return now }
 
@@ -49,12 +54,11 @@ func TestReporter(t *testing.T) {
 		host.Timestamp:     now,
 		host.HostName:      hostname,
 		host.OS:            runtime.GOOS,
-		host.Load:          load,
 		host.Uptime:        uptime,
 		host.KernelVersion: kernel,
 	}).WithSets(report.Sets{
 		host.LocalNetworks: report.MakeStringSet(network),
-	}))
+	}).WithMetrics(load))
 	have, _ := host.NewReporter(hostID, hostname, localNets).Report()
 	if !reflect.DeepEqual(want, have) {
 		t.Errorf("%s", test.Diff(want, have))

probe/host/system_darwin.go

@@ -6,6 +6,8 @@ import (
 	"regexp"
 	"strconv"
 	"time"
+
+	"github.com/weaveworks/scope/report"
 )
 
 var (
@@ -27,17 +29,35 @@ var GetKernelVersion = func() (string, error) {
 	return fmt.Sprintf("Darwin %s", matches[0][1]), nil
 }
 
-// GetLoad returns the current load averages in standard form.
-var GetLoad = func() string {
+// GetLoad returns the current load averages as metrics.
+var GetLoad = func() report.Metrics {
 	out, err := exec.Command("w").CombinedOutput()
 	if err != nil {
-		return "unknown"
+		return nil
 	}
+	now := time.Now()
 	matches := loadRe.FindAllStringSubmatch(string(out), -1)
 	if matches == nil || len(matches) < 1 || len(matches[0]) < 4 {
-		return "unknown"
+		return nil
+	}
+
+	one, err := strconv.ParseFloat(matches[0][1], 64)
+	if err != nil {
+		return nil
+	}
+	five, err := strconv.ParseFloat(matches[0][2], 64)
+	if err != nil {
+		return nil
+	}
+	fifteen, err := strconv.ParseFloat(matches[0][3], 64)
+	if err != nil {
+		return nil
+	}
+	return report.Metrics{
+		Load1:  report.MakeMetric().Add(now, one),
+		Load5:  report.MakeMetric().Add(now, five),
+		Load15: report.MakeMetric().Add(now, fifteen),
 	}
-	return fmt.Sprintf("%s %s %s", matches[0][1], matches[0][2], matches[0][3])
 }
 
 // GetUptime returns the uptime of the host.

probe/host/system_linux.go

@@ -7,6 +7,8 @@ import (
 	"strings"
 	"syscall"
 	"time"
+
+	"github.com/weaveworks/scope/report"
 )
 
 // Uname is swappable for mocking in tests.
@@ -21,29 +23,34 @@ var GetKernelVersion = func() (string, error) {
 	return fmt.Sprintf("%s %s", charsToString(utsname.Release), charsToString(utsname.Version)), nil
 }
 
-// GetLoad returns the current load averages in standard form.
-var GetLoad = func() string {
+// GetLoad returns the current load averages as metrics.
+var GetLoad = func() report.Metrics {
 	buf, err := ioutil.ReadFile("/proc/loadavg")
 	if err != nil {
-		return "unknown"
+		return nil
 	}
+	now := time.Now()
 	toks := strings.Fields(string(buf))
 	if len(toks) < 3 {
-		return "unknown"
+		return nil
 	}
 	one, err := strconv.ParseFloat(toks[0], 64)
 	if err != nil {
-		return "unknown"
+		return nil
 	}
 	five, err := strconv.ParseFloat(toks[1], 64)
 	if err != nil {
-		return "unknown"
+		return nil
 	}
 	fifteen, err := strconv.ParseFloat(toks[2], 64)
 	if err != nil {
-		return "unknown"
+		return nil
+	}
+	return report.Metrics{
+		Load1:  report.MakeMetric().Add(now, one),
+		Load5:  report.MakeMetric().Add(now, five),
+		Load15: report.MakeMetric().Add(now, fifteen),
 	}
-	return fmt.Sprintf("%.2f %.2f %.2f", one, five, fifteen)
 }
 
 // GetUptime returns the uptime of the host.

probe/host/system_test.go

@@ -20,10 +20,14 @@ func TestGetKernelVersion(t *testing.T) {
 
 func TestGetLoad(t *testing.T) {
 	have := host.GetLoad()
-	if strings.Contains(have, "unknown") {
-		t.Fatal(have)
+	if len(have) != 3 {
+		t.Fatalf("Expected 3 metrics, but got: %v", have)
+	}
+	for key, metric := range have {
+		if metric.Len() != 1 {
+			t.Errorf("Expected metric %v to have 1 sample, but had: %d", key, metric.Len())
+		}
 	}
-	t.Log(have)
 }
 
 func TestGetUptime(t *testing.T) {