From 42ad3aa12b2f637eeafd27e035c7c2535ab9950d Mon Sep 17 00:00:00 2001 From: Alban Crequy Date: Sun, 22 May 2016 19:21:55 +0200 Subject: [PATCH] Scope slow: improve error messages for debugging (#1534) * alpine: dl-4.alpinelinux.org is dead, use another server * increase buffer for docker stats Attempt to avoid the following message: docker container: dropping stats. * probe: better timeout error messages The logs contains the following messages: Process reporter took longer than 1s K8s reporter took longer than 1s Docker reporter took longer than 1s Endpoint reporter took longer than 1s This patch prints how long it takes. --- docker/Dockerfile | 2 +- probe/docker/container.go | 2 +- probe/probe.go | 8 ++++++-- vendor/runsvinit/example/Dockerfile | 2 +- vendor/runsvinit/zombietest/Dockerfile | 2 +- 5 files changed, 10 insertions(+), 6 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index aa79c9f9b..2c925cf7f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -2,7 +2,7 @@ FROM alpine:3.3 MAINTAINER Weaveworks Inc LABEL works.weave.role=system WORKDIR /home/weave -RUN echo "http://dl-4.alpinelinux.org/alpine/edge/testing" >>/etc/apk/repositories && \ +RUN echo "http://dl-3.alpinelinux.org/alpine/edge/testing" >>/etc/apk/repositories && \ apk add --update bash runit conntrack-tools iproute2 util-linux curl && \ rm -rf /var/cache/apk/* ADD ./docker.tgz / diff --git a/probe/docker/container.go b/probe/docker/container.go index 1936a96ea..24727b46c 100644 --- a/probe/docker/container.go +++ b/probe/docker/container.go @@ -116,7 +116,7 @@ type container struct { container *docker.Container statsConn ClientConn latestStats docker.Stats - pendingStats [20]docker.Stats + pendingStats [60]docker.Stats numPending int hostID string baseNode report.Node diff --git a/probe/probe.go b/probe/probe.go index 0c5697be2..61124a933 100644 --- a/probe/probe.go +++ b/probe/probe.go @@ -152,7 +152,9 @@ func (p *Probe) report() report.Report { t := time.Now() timer := time.AfterFunc(p.spyInterval, func() { log.Warningf("%v reporter took longer than %v", rep.Name(), p.spyInterval) }) newReport, err := rep.Report() - timer.Stop() + if !timer.Stop() { + log.Warningf("%v reporter took %v (longer than %v)", rep.Name(), time.Now().Sub(t), p.spyInterval) + } metrics.MeasureSince([]string{rep.Name(), "reporter"}, t) if err != nil { log.Errorf("error generating report: %v", err) @@ -175,7 +177,9 @@ func (p *Probe) tag(r report.Report) report.Report { t := time.Now() timer := time.AfterFunc(p.spyInterval, func() { log.Warningf("%v tagger took longer than %v", tagger.Name(), p.spyInterval) }) r, err = tagger.Tag(r) - timer.Stop() + if !timer.Stop() { + log.Warningf("%v tagger took %v (longer than %v)", tagger.Name(), time.Now().Sub(t), p.spyInterval) + } metrics.MeasureSince([]string{tagger.Name(), "tagger"}, t) if err != nil { log.Errorf("error applying tagger: %v", err) diff --git a/vendor/runsvinit/example/Dockerfile b/vendor/runsvinit/example/Dockerfile index 44a63e472..036092b20 100644 --- a/vendor/runsvinit/example/Dockerfile +++ b/vendor/runsvinit/example/Dockerfile @@ -1,5 +1,5 @@ FROM alpine:latest -RUN echo "http://dl-4.alpinelinux.org/alpine/edge/testing" >>/etc/apk/repositories && \ +RUN echo "http://dl-3.alpinelinux.org/alpine/edge/testing" >>/etc/apk/repositories && \ apk add --update runit && \ rm -rf /var/cache/apk/* diff --git a/vendor/runsvinit/zombietest/Dockerfile b/vendor/runsvinit/zombietest/Dockerfile index 3012c1a05..9cc5160d9 100644 --- a/vendor/runsvinit/zombietest/Dockerfile +++ b/vendor/runsvinit/zombietest/Dockerfile @@ -1,5 +1,5 @@ FROM alpine:latest -RUN echo "http://dl-4.alpinelinux.org/alpine/edge/testing" >>/etc/apk/repositories && \ +RUN echo "http://dl-3.alpinelinux.org/alpine/edge/testing" >>/etc/apk/repositories && \ apk add --update runit && \ rm -rf /var/cache/apk/*