Scope slow: improve error messages for debugging (#1534)

* alpine: dl-4.alpinelinux.org is dead, use another server

* increase buffer for docker stats

Attempt to avoid the following message:
docker container: dropping stats.

* probe: better timeout error messages

The logs contains the following messages:

Process reporter took longer than 1s
K8s reporter took longer than 1s
Docker reporter took longer than 1s
Endpoint reporter took longer than 1s

This patch prints how long it takes.

docker/Dockerfile

@@ -2,7 +2,7 @@ FROM alpine:3.3
 MAINTAINER Weaveworks Inc <help@weave.works>
 LABEL works.weave.role=system
 WORKDIR /home/weave
-RUN echo "http://dl-4.alpinelinux.org/alpine/edge/testing" >>/etc/apk/repositories && \
+RUN echo "http://dl-3.alpinelinux.org/alpine/edge/testing" >>/etc/apk/repositories && \
 	apk add --update bash runit conntrack-tools iproute2 util-linux curl && \
 	rm -rf /var/cache/apk/*
 ADD ./docker.tgz /

probe/docker/container.go

@@ -116,7 +116,7 @@ type container struct {
 	container    *docker.Container
 	statsConn    ClientConn
 	latestStats  docker.Stats
-	pendingStats [20]docker.Stats
+	pendingStats [60]docker.Stats
 	numPending   int
 	hostID       string
 	baseNode     report.Node

probe/probe.go

@@ -152,7 +152,9 @@ func (p *Probe) report() report.Report {
 			t := time.Now()
 			timer := time.AfterFunc(p.spyInterval, func() { log.Warningf("%v reporter took longer than %v", rep.Name(), p.spyInterval) })
 			newReport, err := rep.Report()
-			timer.Stop()
+			if !timer.Stop() {
+				log.Warningf("%v reporter took %v (longer than %v)", rep.Name(), time.Now().Sub(t), p.spyInterval)
+			}
 			metrics.MeasureSince([]string{rep.Name(), "reporter"}, t)
 			if err != nil {
 				log.Errorf("error generating report: %v", err)
@@ -175,7 +177,9 @@ func (p *Probe) tag(r report.Report) report.Report {
 		t := time.Now()
 		timer := time.AfterFunc(p.spyInterval, func() { log.Warningf("%v tagger took longer than %v", tagger.Name(), p.spyInterval) })
 		r, err = tagger.Tag(r)
-		timer.Stop()
+		if !timer.Stop() {
+			log.Warningf("%v tagger took %v (longer than %v)", tagger.Name(), time.Now().Sub(t), p.spyInterval)
+		}
 		metrics.MeasureSince([]string{tagger.Name(), "tagger"}, t)
 		if err != nil {
 			log.Errorf("error applying tagger: %v", err)

vendor/runsvinit/example/Dockerfile

@@ -1,5 +1,5 @@
 FROM alpine:latest
-RUN echo "http://dl-4.alpinelinux.org/alpine/edge/testing" >>/etc/apk/repositories && \
+RUN echo "http://dl-3.alpinelinux.org/alpine/edge/testing" >>/etc/apk/repositories && \
     apk add --update runit && \
     rm -rf /var/cache/apk/*
 

vendor/runsvinit/zombietest/Dockerfile

@@ -1,5 +1,5 @@
 FROM alpine:latest
-RUN echo "http://dl-4.alpinelinux.org/alpine/edge/testing" >>/etc/apk/repositories && \
+RUN echo "http://dl-3.alpinelinux.org/alpine/edge/testing" >>/etc/apk/repositories && \
     apk add --update runit && \
     rm -rf /var/cache/apk/*