Merge remote-tracking branch 'origin/0.9'

CHANGELOG.md

@@ -1,3 +1,47 @@
+## Release 0.9.0
+
+New features:
+- Add basic Kubernetes views for pods and services
+  [#441](https://github.com/weaveworks/scope/pull/441)
+- Support for Weave 1.2
+  [#574](https://github.com/weaveworks/scope/pull/574)
+- Add containers-by-hostname view
+  [#545](https://github.com/weaveworks/scope/pull/545)
+- Build using Go 1.5, with vendored dependancies
+  [#584](https://github.com/weaveworks/scope/pull/584)
+- Make `scope launch` work from remote hosts, with an appropriately defined DOCKER_HOST
+  [#524](https://github.com/weaveworks/scope/pull/524)
+- Increase DNS poll frequency such that Scope clusters up more quickly
+  [#524](https://github.com/weaveworks/scope/pull/524)
+- Add `scope command` for printing the Docker commands used to run Scope
+  [#553](https://github.com/weaveworks/scope/pull/553)
+- Include some basic documentation on how to run Scope
+  [#572](https://github.com/weaveworks/scope/pull/572)
+- Warn if the users tries to run Scope on Docker versions <1.5.0
+  [#557](https://github.com/weaveworks/scope/pull/557)
+- Add support for loading the Scope UI from https endpoints
+  [#572](https://github.com/weaveworks/scope/pull/572)
+- Add support from probe sending reports to https endpoints
+  [#575](https://github.com/weaveworks/scope/pull/575)
+
+Bug fixes:
+- Correctly track short-lived connections from the internet
+  [#493](https://github.com/weaveworks/scope/pull/493)
+- Fix a corner case where short-lived connections between containers are incorrectly attributed
+  [#577](https://github.com/weaveworks/scope/pull/577)
+- Ensure service credentials are sent when doing initial probe<->app handshake
+  [#564](https://github.com/weaveworks/scope/pull/564)
+- Sort reverse-DNS-resolved names to mitigate some UI fluttering
+  [#562](https://github.com/weaveworks/scope/pull/562)
+- Don't leak goroutines in the probe
+  [#531](https://github.com/weaveworks/scope/issue/531)
+- Rerun background conntrack processes if they fail
+  [#581](https://github.com/weaveworks/scope/issue/581)
+- Build and test using Go 1.5 and vendor all dependancies
+  [#584](https://github.com/weaveworks/scope/pull/584)
+- Fix "close on nil channel" error on shutdown
+  [#599](https://github.com/weaveworks/scope/issues/599)
+
 ## Release 0.8.0
 
 New features:

Makefile

@@ -43,7 +43,7 @@ $(PROBE_EXE): probe/*.go probe/docker/*.go probe/kubernetes/*.go probe/endpoint/
 
 ifeq ($(BUILD_IN_CONTAINER),true)
 $(APP_EXE) $(PROBE_EXE) $(RUNSVINIT): $(SCOPE_BACKEND_BUILD_UPTODATE)
-	$(SUDO) docker run -ti $(RM) -v $(shell pwd):/go/src/github.com/weaveworks/scope -e GOARCH -e GOOS \
+	$(SUDO) docker run $(RM) -v $(shell pwd):/go/src/github.com/weaveworks/scope -e GOARCH -e GOOS \
 		$(SCOPE_BACKEND_BUILD_IMAGE) $@
 else
 $(APP_EXE) $(PROBE_EXE):
@@ -67,22 +67,22 @@ static: client/build/app.js
 ifeq ($(BUILD_IN_CONTAINER),true)
 client/build/app.js: $(shell find client/app/scripts -type f)
 	mkdir -p client/build
-	$(SUDO) docker run -ti $(RM) -v $(shell pwd)/client/app:/home/weave/app \
+	$(SUDO) docker run $(RM) -v $(shell pwd)/client/app:/home/weave/app \
 		-v $(shell pwd)/client/build:/home/weave/build \
 		$(SCOPE_UI_BUILD_IMAGE) npm run build
 
 client-test: $(shell find client/app/scripts -type f)
-	$(SUDO) docker run -ti $(RM) -v $(shell pwd)/client/app:/home/weave/app \
+	$(SUDO) docker run $(RM) -v $(shell pwd)/client/app:/home/weave/app \
 		-v $(shell pwd)/client/test:/home/weave/test \
 		$(SCOPE_UI_BUILD_IMAGE) npm test
 
 client-lint:
-	$(SUDO) docker run -ti $(RM) -v $(shell pwd)/client/app:/home/weave/app \
+	$(SUDO) docker run $(RM) -v $(shell pwd)/client/app:/home/weave/app \
 		-v $(shell pwd)/client/test:/home/weave/test \
 		$(SCOPE_UI_BUILD_IMAGE) npm run lint
 
 client-start:
-	$(SUDO) docker run -ti $(RM) --net=host -v $(shell pwd)/client/app:/home/weave/app \
+	$(SUDO) docker run $(RM) --net=host -v $(shell pwd)/client/app:/home/weave/app \
 		-v $(shell pwd)/client/build:/home/weave/build \
 		$(SCOPE_UI_BUILD_IMAGE) npm start
 endif
@@ -105,7 +105,7 @@ clean:
 
 ifeq ($(BUILD_IN_CONTAINER),true)
 tests:
-	$(SUDO) docker run -ti $(RM) -v $(shell pwd):/go/src/github.com/weaveworks/scope \
+	$(SUDO) docker run $(RM) -v $(shell pwd):/go/src/github.com/weaveworks/scope \
 		-e GOARCH -e GOOS -e CIRCLECI --entrypoint=/bin/sh $(SCOPE_BACKEND_BUILD_IMAGE) -c \
 		"cd /go/src/github.com/weaveworks/scope && ./tools/test -no-go-get"
 else

app/api_topologies.go

@@ -120,7 +120,7 @@ type APITopologyDesc struct {
 
 	URL           string            `json:"url"`
 	SubTopologies []APITopologyDesc `json:"sub_topologies,omitempty"`
-	Stats         *topologyStats    `json:"stats,omitempty"`
+	Stats         topologyStats     `json:"stats,omitempty"`
 }
 
 type byName []APITopologyDesc
@@ -193,11 +193,11 @@ func (r *registry) makeTopologyList(rep xfer.Reporter) func(w http.ResponseWrite
 			topologies = []APITopologyDesc{}
 		)
 		r.walk(func(desc APITopologyDesc) {
-			decorateTopologyForRequest(req, &desc)
-			decorateWithStats(&desc, rpt)
+			renderer := renderedForRequest(req, desc)
+			desc.Stats = decorateWithStats(rpt, renderer)
 			for i := range desc.SubTopologies {
-				decorateTopologyForRequest(req, &desc.SubTopologies[i])
-				decorateWithStats(&desc.SubTopologies[i], rpt)
+				renderer := renderedForRequest(req, desc.SubTopologies[i])
+				desc.Stats = decorateWithStats(rpt, renderer)
 			}
 			topologies = append(topologies, desc)
 		})
@@ -205,21 +205,21 @@ func (r *registry) makeTopologyList(rep xfer.Reporter) func(w http.ResponseWrite
 	}
 }
 
-func decorateWithStats(desc *APITopologyDesc, rpt report.Report) {
+func decorateWithStats(rpt report.Report, renderer render.Renderer) topologyStats {
 	var (
 		nodes     int
 		realNodes int
 		edges     int
 	)
-	for _, n := range desc.renderer.Render(rpt) {
+	for _, n := range renderer.Render(rpt) {
 		nodes++
 		if !n.Pseudo {
 			realNodes++
 		}
 		edges += len(n.Adjacency)
 	}
-	renderStats := desc.renderer.Stats(rpt)
-	desc.Stats = &topologyStats{
+	renderStats := renderer.Stats(rpt)
+	return topologyStats{
 		NodeCount:          nodes,
 		NonpseudoNodeCount: realNodes,
 		EdgeCount:          edges,
@@ -233,25 +233,27 @@ func (r *registry) enableKubernetesTopologies() {
 	r.add(kubernetesTopologies...)
 }
 
-func decorateTopologyForRequest(r *http.Request, topology *APITopologyDesc) {
+func renderedForRequest(r *http.Request, topology APITopologyDesc) render.Renderer {
+	renderer := topology.renderer
 	for param, opts := range topology.Options {
 		value := r.FormValue(param)
 		for _, opt := range opts {
 			if (value == "" && opt.Default) || (opt.Value != "" && opt.Value == value) {
-				topology.renderer = opt.decorator(topology.renderer)
+				renderer = opt.decorator(renderer)
 			}
 		}
 	}
+	return renderer
 }
 
-func (r *registry) captureTopology(rep xfer.Reporter, f func(xfer.Reporter, APITopologyDesc, http.ResponseWriter, *http.Request)) http.HandlerFunc {
+func (r *registry) captureRenderer(rep xfer.Reporter, f func(xfer.Reporter, render.Renderer, http.ResponseWriter, *http.Request)) http.HandlerFunc {
 	return func(w http.ResponseWriter, req *http.Request) {
 		topology, ok := r.get(mux.Vars(req)["topology"])
 		if !ok {
 			http.NotFound(w, req)
 			return
 		}
-		decorateTopologyForRequest(req, &topology)
-		f(rep, topology, w, req)
+		renderer := renderedForRequest(req, topology)
+		f(rep, renderer, w, req)
 	}
 }

app/api_topology.go

@@ -33,14 +33,14 @@ type APIEdge struct {
 }
 
 // Full topology.
-func handleTopology(rep xfer.Reporter, t APITopologyDesc, w http.ResponseWriter, r *http.Request) {
+func handleTopology(rep xfer.Reporter, renderer render.Renderer, w http.ResponseWriter, r *http.Request) {
 	respondWith(w, http.StatusOK, APITopology{
-		Nodes: t.renderer.Render(rep.Report()).Prune(),
+		Nodes: renderer.Render(rep.Report()).Prune(),
 	})
 }
 
 // Websocket for the full topology. This route overlaps with the next.
-func handleWs(rep xfer.Reporter, t APITopologyDesc, w http.ResponseWriter, r *http.Request) {
+func handleWs(rep xfer.Reporter, renderer render.Renderer, w http.ResponseWriter, r *http.Request) {
 	if err := r.ParseForm(); err != nil {
 		respondWith(w, http.StatusInternalServerError, err.Error())
 		return
@@ -53,16 +53,16 @@ func handleWs(rep xfer.Reporter, t APITopologyDesc, w http.ResponseWriter, r *ht
 			return
 		}
 	}
-	handleWebsocket(w, r, rep, t, loop)
+	handleWebsocket(w, r, rep, renderer, loop)
 }
 
 // Individual nodes.
-func handleNode(rep xfer.Reporter, t APITopologyDesc, w http.ResponseWriter, r *http.Request) {
+func handleNode(rep xfer.Reporter, renderer render.Renderer, w http.ResponseWriter, r *http.Request) {
 	var (
 		vars     = mux.Vars(r)
 		nodeID   = vars["id"]
 		rpt      = rep.Report()
-		node, ok = t.renderer.Render(rep.Report())[nodeID]
+		node, ok = renderer.Render(rep.Report())[nodeID]
 	)
 	if !ok {
 		http.NotFound(w, r)
@@ -72,13 +72,13 @@ func handleNode(rep xfer.Reporter, t APITopologyDesc, w http.ResponseWriter, r *
 }
 
 // Individual edges.
-func handleEdge(rep xfer.Reporter, t APITopologyDesc, w http.ResponseWriter, r *http.Request) {
+func handleEdge(rep xfer.Reporter, renderer render.Renderer, w http.ResponseWriter, r *http.Request) {
 	var (
 		vars     = mux.Vars(r)
 		localID  = vars["local"]
 		remoteID = vars["remote"]
 		rpt      = rep.Report()
-		metadata = t.renderer.EdgeMetadata(rpt, localID, remoteID)
+		metadata = renderer.EdgeMetadata(rpt, localID, remoteID)
 	)
 
 	respondWith(w, http.StatusOK, APIEdge{Metadata: metadata})
@@ -92,7 +92,7 @@ func handleWebsocket(
 	w http.ResponseWriter,
 	r *http.Request,
 	rep xfer.Reporter,
-	t APITopologyDesc,
+	renderer render.Renderer,
 	loop time.Duration,
 ) {
 	conn, err := upgrader.Upgrade(w, r, nil)
@@ -117,7 +117,7 @@ func handleWebsocket(
 		tick         = time.Tick(loop)
 	)
 	for {
-		newTopo := t.renderer.Render(rep.Report()).Prune()
+		newTopo := renderer.Render(rep.Report()).Prune()
 		diff := render.TopoDiff(previousTopo, newTopo)
 		previousTopo = newTopo
 

app/router.go

@@ -65,13 +65,13 @@ func Router(c collector) *mux.Router {
 	get.HandleFunc("/api", gzipHandler(apiHandler))
 	get.HandleFunc("/api/topology", gzipHandler(topologyRegistry.makeTopologyList(c)))
 	get.HandleFunc("/api/topology/{topology}",
-		gzipHandler(topologyRegistry.captureTopology(c, handleTopology)))
+		gzipHandler(topologyRegistry.captureRenderer(c, handleTopology)))
 	get.HandleFunc("/api/topology/{topology}/ws",
-		topologyRegistry.captureTopology(c, handleWs)) // NB not gzip!
+		topologyRegistry.captureRenderer(c, handleWs)) // NB not gzip!
 	get.MatcherFunc(URLMatcher("/api/topology/{topology}/{id}")).HandlerFunc(
-		gzipHandler(topologyRegistry.captureTopology(c, handleNode)))
+		gzipHandler(topologyRegistry.captureRenderer(c, handleNode)))
 	get.MatcherFunc(URLMatcher("/api/topology/{topology}/{local}/{remote}")).HandlerFunc(
-		gzipHandler(topologyRegistry.captureTopology(c, handleEdge)))
+		gzipHandler(topologyRegistry.captureRenderer(c, handleEdge)))
 	get.MatcherFunc(URLMatcher("/api/origin/host/{id}")).HandlerFunc(
 		gzipHandler(makeOriginHostHandler(c)))
 	get.HandleFunc("/api/report", gzipHandler(makeRawReportHandler(c)))

probe/endpoint/conntrack.go

@@ -94,6 +94,7 @@ func newConntrackFlowWalker(useConntrack bool, args ...string) flowWalker {
 	result := &conntrackWalker{
 		activeFlows: map[int64]flow{},
 		args:        args,
+		quit:        make(chan struct{}),
 	}
 	go result.loop()
 	return result

probe/endpoint/conntrack_internal_test.go

@@ -92,6 +92,7 @@ func TestConntracker(t *testing.T) {
 	}
 
 	flowWalker := newConntrackFlowWalker(true)
+	defer flowWalker.stop()
 
 	// First write out some empty xml for the existing connections
 	ecbw := bufio.NewWriter(existingConnectionsWriter)

xfer/http_publisher.go

@@ -119,7 +119,12 @@ func (p HTTPPublisher) Publish(r io.Reader) error {
 }
 
 // Stop implements Publisher
-func (p HTTPPublisher) Stop() {}
+func (p *HTTPPublisher) Stop() {
+	// We replace the HTTPPublishers pretty regularly, so we need to ensure the
+	// underlying connections get closed, or we end up with lots of idle
+	// goroutines on the server (see #604)
+	p.client.Transport.(*http.Transport).CloseIdleConnections()
+}
 
 // AuthorizationHeader returns a value suitable for an HTTP Authorization
 // header, based on the passed token string.