diff --git a/probe/docker/container.go b/probe/docker/container.go index 82c9bcdc9..56bdfc01b 100644 --- a/probe/docker/container.go +++ b/probe/docker/container.go @@ -91,9 +91,10 @@ type Container interface { type container struct { sync.RWMutex - container *docker.Container - statsConn ClientConn - latestStats *docker.Stats + container *docker.Container + statsConn ClientConn + latestStats *docker.Stats + pendingStats []*docker.Stats } // NewContainer creates a new Container @@ -190,6 +191,7 @@ func (c *container) StartGatheringStats() error { c.Lock() c.latestStats = stats + c.pendingStats = append(c.pendingStats, stats) c.Unlock() stats = &docker.Stats{} @@ -238,6 +240,52 @@ func (c *container) ports(localAddrs []net.IP) report.StringSet { return report.MakeStringSet(ports...) } +func (c *container) memoryUsageMetric() report.Metric { + result := report.MakeMetric() + for _, s := range c.pendingStats { + result = result.Add(s.Read, float64(s.MemoryStats.Usage)) + } + return result +} + +func (c *container) cpuPercentMetric() report.Metric { + result := report.MakeMetric() + if len(c.pendingStats) < 2 { + return result + } + + previous := c.pendingStats[0] + for _, s := range c.pendingStats[1:] { + // Copies from docker/api/client/stats.go#L205 + cpuDelta := float64(s.CPUStats.CPUUsage.TotalUsage - previous.CPUStats.CPUUsage.TotalUsage) + systemDelta := float64(s.CPUStats.SystemCPUUsage - previous.CPUStats.SystemCPUUsage) + cpuPercent := 0.0 + if systemDelta > 0.0 && cpuDelta > 0.0 { + cpuPercent = (cpuDelta / systemDelta) * float64(len(s.CPUStats.CPUUsage.PercpuUsage)) * 100.0 + } + result = result.Add(s.Read, cpuPercent) + available := float64(len(s.CPUStats.CPUUsage.PercpuUsage)) * 100.0 + if available >= result.Max { + result.Max = available + } + previous = s + } + return result +} + +func (c *container) metrics() report.Metrics { + result := report.Metrics{ + MemoryUsage: c.memoryUsageMetric(), + CPUTotalUsage: c.cpuPercentMetric(), + } + + // Keep the latest report to help with relative metric reporting. + if len(c.pendingStats) > 0 { + c.pendingStats = c.pendingStats[len(c.pendingStats)-1:] + } + return result +} + func (c *container) GetNode(hostID string, localAddrs []net.IP) report.Node { c.RLock() defer c.RUnlock() @@ -269,7 +317,9 @@ func (c *container) GetNode(hostID string, localAddrs []net.IP) report.Node { ContainerPorts: c.ports(localAddrs), ContainerIPs: report.MakeStringSet(ips...), ContainerIPsWithScopes: report.MakeStringSet(ipsWithScopes...), - }).WithLatest(ContainerState, mtime.Now(), state) + }).WithLatest( + ContainerState, mtime.Now(), state, + ).WithMetrics(c.metrics()) if c.container.State.Paused { result = result.WithControls(UnpauseContainer) @@ -305,8 +355,7 @@ func (c *container) GetNode(hostID string, localAddrs []net.IP) report.Node { CPUTotalUsage: strconv.FormatUint(c.latestStats.CPUStats.CPUUsage.TotalUsage, 10), CPUUsageInKernelmode: strconv.FormatUint(c.latestStats.CPUStats.CPUUsage.UsageInKernelmode, 10), CPUSystemCPUUsage: strconv.FormatUint(c.latestStats.CPUStats.SystemCPUUsage, 10), - }) - + }).WithMetrics(c.metrics()) return result } diff --git a/probe/docker/container_test.go b/probe/docker/container_test.go index a7c60cdd0..7add75481 100644 --- a/probe/docker/container_test.go +++ b/probe/docker/container_test.go @@ -58,17 +58,18 @@ func TestContainer(t *testing.T) { } defer c.StopGatheringStats() + now := time.Unix(12345, 67890).UTC() + mtime.NowForce(now) + defer mtime.NowReset() + // Send some stats to the docker container stats := &client.Stats{} + stats.Read = now stats.MemoryStats.Usage = 12345 if err = json.NewEncoder(writer).Encode(&stats); err != nil { t.Error(err) } - now := time.Now() - mtime.NowForce(now) - defer mtime.NowReset() - // Now see if we go them want := report.MakeNode().WithMetadata(map[string]string{ "docker_container_command": " ", @@ -85,8 +86,12 @@ func TestContainer(t *testing.T) { "docker_container_ips_with_scopes": report.MakeStringSet("scope;1.2.3.4"), }).WithControls( docker.RestartContainer, docker.StopContainer, docker.PauseContainer, - ).WithLatest("docker_container_state", now, "running") - + ).WithLatest( + "docker_container_state", now, "running", + ).WithMetrics(report.Metrics{ + "cpu_total_usage": report.MakeMetric(), + "memory_usage": report.MakeMetric().Add(now, 12345), + }) test.Poll(t, 100*time.Millisecond, want, func() interface{} { node := c.GetNode("scope", []net.IP{}) for k, v := range node.Metadata { diff --git a/probe/host/reporter.go b/probe/host/reporter.go index ab4ca9249..9ab8dfc1b 100644 --- a/probe/host/reporter.go +++ b/probe/host/reporter.go @@ -13,9 +13,11 @@ const ( HostName = "host_name" LocalNetworks = "local_networks" OS = "os" - Load = "load" KernelVersion = "kernel_version" Uptime = "uptime" + Load1 = "load1" + Load5 = "load5" + Load15 = "load15" ) // Exposed for testing. @@ -71,12 +73,11 @@ func (r *Reporter) Report() (report.Report, error) { Timestamp: Now(), HostName: r.hostName, OS: runtime.GOOS, - Load: GetLoad(), KernelVersion: kernel, Uptime: uptime.String(), }).WithSets(report.Sets{ LocalNetworks: report.MakeStringSet(localCIDRs...), - })) + }).WithMetrics(GetLoad())) return rep, nil } diff --git a/probe/host/reporter_test.go b/probe/host/reporter_test.go index 48529294b..49ff3ca9a 100644 --- a/probe/host/reporter_test.go +++ b/probe/host/reporter_test.go @@ -14,13 +14,18 @@ import ( func TestReporter(t *testing.T) { var ( - release = "release" - version = "version" - network = "192.168.0.0/16" - hostID = "hostid" - now = "now" - hostname = "hostname" - load = "0.59 0.36 0.29" + release = "release" + version = "version" + network = "192.168.0.0/16" + hostID = "hostid" + now = "now" + hostname = "hostname" + timestamp = time.Now() + load = report.Metrics{ + host.Load1: report.MakeMetric().Add(timestamp, 1.0), + host.Load5: report.MakeMetric().Add(timestamp, 5.0), + host.Load15: report.MakeMetric().Add(timestamp, 15.0), + } uptime = "278h55m43s" kernel = "release version" _, ipnet, _ = net.ParseCIDR(network) @@ -40,7 +45,7 @@ func TestReporter(t *testing.T) { host.Now = oldNow }() host.GetKernelVersion = func() (string, error) { return release + " " + version, nil } - host.GetLoad = func() string { return load } + host.GetLoad = func() report.Metrics { return load } host.GetUptime = func() (time.Duration, error) { return time.ParseDuration(uptime) } host.Now = func() string { return now } @@ -49,12 +54,11 @@ func TestReporter(t *testing.T) { host.Timestamp: now, host.HostName: hostname, host.OS: runtime.GOOS, - host.Load: load, host.Uptime: uptime, host.KernelVersion: kernel, }).WithSets(report.Sets{ host.LocalNetworks: report.MakeStringSet(network), - })) + }).WithMetrics(load)) have, _ := host.NewReporter(hostID, hostname, localNets).Report() if !reflect.DeepEqual(want, have) { t.Errorf("%s", test.Diff(want, have)) diff --git a/probe/host/system_darwin.go b/probe/host/system_darwin.go index f985a59ff..453446ab2 100644 --- a/probe/host/system_darwin.go +++ b/probe/host/system_darwin.go @@ -6,6 +6,8 @@ import ( "regexp" "strconv" "time" + + "github.com/weaveworks/scope/report" ) var ( @@ -27,17 +29,35 @@ var GetKernelVersion = func() (string, error) { return fmt.Sprintf("Darwin %s", matches[0][1]), nil } -// GetLoad returns the current load averages in standard form. -var GetLoad = func() string { +// GetLoad returns the current load averages as metrics. +var GetLoad = func() report.Metrics { out, err := exec.Command("w").CombinedOutput() if err != nil { - return "unknown" + return nil } + now := time.Now() matches := loadRe.FindAllStringSubmatch(string(out), -1) if matches == nil || len(matches) < 1 || len(matches[0]) < 4 { - return "unknown" + return nil + } + + one, err := strconv.ParseFloat(matches[0][1], 64) + if err != nil { + return nil + } + five, err := strconv.ParseFloat(matches[0][2], 64) + if err != nil { + return nil + } + fifteen, err := strconv.ParseFloat(matches[0][3], 64) + if err != nil { + return nil + } + return report.Metrics{ + Load1: report.MakeMetric().Add(now, one), + Load5: report.MakeMetric().Add(now, five), + Load15: report.MakeMetric().Add(now, fifteen), } - return fmt.Sprintf("%s %s %s", matches[0][1], matches[0][2], matches[0][3]) } // GetUptime returns the uptime of the host. diff --git a/probe/host/system_linux.go b/probe/host/system_linux.go index 232769eab..d713e3b91 100644 --- a/probe/host/system_linux.go +++ b/probe/host/system_linux.go @@ -7,6 +7,8 @@ import ( "strings" "syscall" "time" + + "github.com/weaveworks/scope/report" ) // Uname is swappable for mocking in tests. @@ -21,29 +23,34 @@ var GetKernelVersion = func() (string, error) { return fmt.Sprintf("%s %s", charsToString(utsname.Release), charsToString(utsname.Version)), nil } -// GetLoad returns the current load averages in standard form. -var GetLoad = func() string { +// GetLoad returns the current load averages as metrics. +var GetLoad = func() report.Metrics { buf, err := ioutil.ReadFile("/proc/loadavg") if err != nil { - return "unknown" + return nil } + now := time.Now() toks := strings.Fields(string(buf)) if len(toks) < 3 { - return "unknown" + return nil } one, err := strconv.ParseFloat(toks[0], 64) if err != nil { - return "unknown" + return nil } five, err := strconv.ParseFloat(toks[1], 64) if err != nil { - return "unknown" + return nil } fifteen, err := strconv.ParseFloat(toks[2], 64) if err != nil { - return "unknown" + return nil + } + return report.Metrics{ + Load1: report.MakeMetric().Add(now, one), + Load5: report.MakeMetric().Add(now, five), + Load15: report.MakeMetric().Add(now, fifteen), } - return fmt.Sprintf("%.2f %.2f %.2f", one, five, fifteen) } // GetUptime returns the uptime of the host. diff --git a/probe/host/system_test.go b/probe/host/system_test.go index 361490b88..c868550e2 100644 --- a/probe/host/system_test.go +++ b/probe/host/system_test.go @@ -20,10 +20,14 @@ func TestGetKernelVersion(t *testing.T) { func TestGetLoad(t *testing.T) { have := host.GetLoad() - if strings.Contains(have, "unknown") { - t.Fatal(have) + if len(have) != 3 { + t.Fatalf("Expected 3 metrics, but got: %v", have) + } + for key, metric := range have { + if metric.Len() != 1 { + t.Errorf("Expected metric %v to have 1 sample, but had: %d", key, metric.Len()) + } } - t.Log(have) } func TestGetUptime(t *testing.T) {