Merge pull request #2094 from weaveworks/maybe-old-master

Fix force-push to master

extras/dialer/.gitignore

@@ -0,0 +1 @@
+bin/

extras/dialer/Dockerfile

@@ -0,0 +1,2 @@
+FROM golang:1.7
+ADD ./bin/dialer /go/bin

extras/dialer/README.md

@@ -0,0 +1,46 @@
+## Overview
+
+The dialer scripts can be used to test Scope with a high number of
+containers and connections.
+
+The dialer server is a TCP server that holds incoming connections
+forever.
+
+The dialer client is a TCP client that opens n connection to a server
+and holds them forever.
+
+The `listener` script starts a dialer server and prints its address for
+usage with the `dialer` script. The `dialer` scripts starts up to n
+(default 50) client containers, each opening a random (1-20) number of
+connections.
+
+`time-scope-probe` then can be used to measure the scheduled time
+(utime + stime) of the scope-probe process on the host. The results
+can be used to compare performance under different scenarios/branches.
+
+## Usage example
+
+```
+# Start a listener
+./tools/dialer/listener
+Listening on :8082
+IP addr + port: 172.17.0.2:8082
+
+
+# Start the dialer script with a maximum of 10 dialer containers
+# (default 50)
+./tools/dialer/dialer 172.17.0.2:8082 10
+
+
+# Start time-scope-probe to measure the scheduled time of scope-probe
+# every 3 seconds (default 10 seconds) for 3 times (default 60 times)
+sudo ./tools/dialer/time-scope-probe 3 3
+...
+```
+
+## Build dialer container
+
+```
+go build -o bin/dialer
+docker build -t dialer .
+```

extras/dialer/dialer

@@ -0,0 +1,26 @@
+#!/bin/bash
+set -eu
+if [ $# -lt 1 ]; then
+  echo "Usage: $0 <ip:port> [<max_dialer>]" >&2
+  exit 1
+fi
+
+readonly addr=$1
+readonly max_dialer=${2:-50}
+
+dialer=()
+trap 'echo -n "stopping ... "; for c in "${dialer[@]}"; do docker rm -f "$c" >/dev/null; done; echo "done"' EXIT
+
+while true; do
+  rand=$(( ( RANDOM % max_dialer )  + 1 ))
+  dialer+=("$(docker run -d dialer /go/bin/dialer connect "$addr" "$rand")")
+
+  if [ ${#dialer[@]} -gt "$max_dialer" ]; then
+    container=${dialer[$rand]}
+    docker rm -f "$container" >/dev/null &
+    unset dialer[$rand]
+    dialer=("${dialer[@]}")
+  fi
+
+  sleep $(( rand % 3 ))
+done

extras/dialer/dialer.go

@@ -0,0 +1,76 @@
+package main
+
+import (
+	"fmt"
+	"net"
+	"os"
+	"strconv"
+)
+
+func connect(url string, numConn int) {
+	fmt.Printf("Establishing %d TCP connections to %s\n", numConn, url)
+	for x := 0; x < numConn; x++ {
+		_, err := net.Dial("tcp", url)
+
+		if err != nil {
+			fmt.Printf("Error: %v\n", err)
+			os.Exit(1)
+		}
+	}
+
+	// wait forever
+	select {}
+}
+
+func listen(url string) {
+	l, err := net.Listen("tcp", url)
+	if err != nil {
+		fmt.Println("Error listening:", err.Error())
+		os.Exit(1)
+	}
+	defer l.Close()
+	fmt.Println("Listening on " + url)
+	for {
+		_, err := l.Accept()
+		if err != nil {
+			fmt.Println("Error accepting: ", err.Error())
+			os.Exit(1)
+		}
+	}
+
+}
+
+func main() {
+	if len(os.Args) < 2 {
+		fmt.Println("Not enough arguments")
+		os.Exit(1)
+	}
+
+	verb := os.Args[1]
+
+	if verb == "connect" {
+		if len(os.Args) != 4 {
+			fmt.Println("Not enough arguments")
+			os.Exit(1)
+		}
+
+		url := os.Args[2]
+		numConn, err := strconv.Atoi(os.Args[3])
+		if err != nil {
+			fmt.Printf("Error with second argument\n")
+			os.Exit(1)
+		}
+
+		connect(url, numConn)
+	}
+	if verb == "listen" {
+		if len(os.Args) != 3 {
+			fmt.Println("Not enough arguments")
+			os.Exit(1)
+		}
+
+		port := os.Args[2]
+		listen(":" + port)
+	}
+
+}

extras/dialer/listener

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -eu
+readonly port=${1:-8082}
+(
+sleep 5 # wait for dialerserver to start
+echo "IP addr + port: $(docker inspect -f '{{ .NetworkSettings.IPAddress }}' dialerserver):$port"
+) &
+docker run --rm --name dialerserver dialer /go/bin/dialer listen "$port"

extras/dialer/time-scope-probe

@@ -0,0 +1,33 @@
+#!/bin/bash
+set -eu
+if [ $EUID -ne 0 ]; then
+  echo "You must be root!" >&2
+  exit 1
+fi
+
+readonly interval_num=${1:-60}
+readonly interval_sleep=${2:-10}
+
+TIME_U1=0
+TIME_K1=0
+TIME_T1=0
+TIME_U2=0
+TIME_K2=0
+TIME_T2=0
+
+if [ "$(pidof scope-probe)" == "" ]; then
+  echo "No scope-probe process running - aborting" >&2
+  exit 1
+fi
+
+for ((i=0; i<"$interval_num"; i++)); do
+  sleep "$interval_sleep"
+  TIME_U1=$TIME_U2
+  TIME_K1=$TIME_K2
+  TIME_T1=$TIME_T2
+
+  TIME_U2=$(gawk '{print $14"*10"}'<"/proc/$(pidof scope-probe)/stat" | bc)
+  TIME_K2=$(gawk '{print $15"*10"}'<"/proc/$(pidof scope-probe)/stat" | bc)
+  TIME_T2=$(( TIME_U2 + TIME_K2 ))
+  echo "utime $(( TIME_U2 - TIME_U1)) + stime $(( TIME_K2 - TIME_K1 )) = $(( TIME_T2 - TIME_T1 ))"
+done

probe/awsecs/client.go

@@ -73,7 +73,7 @@ func (c ecsClient) getServices() map[string]*ecs.Service {
 				}
 
 				for _, failure := range resp.Failures {
-					log.Warnf("Failed to describe ECS service %s, ECS service report may be incomplete: %s", failure.Arn, failure.Reason)
+					log.Warnf("Failed to describe ECS service %s, ECS service report may be incomplete: %s", *failure.Arn, *failure.Reason)
 				}
 
 				lock.Lock()
@@ -110,7 +110,7 @@ func (c ecsClient) getTasks(taskArns []string) (map[string]*ecs.Task, error) {
 	}
 
 	for _, failure := range resp.Failures {
-		log.Warnf("Failed to describe ECS task %s, ECS service report may be incomplete: %s", failure.Arn, failure.Reason)
+		log.Warnf("Failed to describe ECS task %s, ECS service report may be incomplete: %s", *failure.Arn, *failure.Reason)
 	}
 
 	results := make(map[string]*ecs.Task, len(resp.Tasks))

prog/app.go

@@ -270,7 +270,10 @@ func appMain(flags appFlags) {
 
 	handler := router(collector, controlRouter, pipeRouter, flags.externalUI)
 	if flags.logHTTP {
-		handler = middleware.LogFailed.Wrap(handler)
+		handler = middleware.Log{
+			LogRequestHeaders: flags.logHTTPHeaders,
+			LogSuccess:        false,
+		}.Wrap(handler)
 	}
 
 	server := &graceful.Server{

prog/main.go

@@ -112,12 +112,13 @@ type probeFlags struct {
 }
 
 type appFlags struct {
-	window      time.Duration
-	listen      string
-	stopTimeout time.Duration
-	logLevel    string
-	logPrefix   string
-	logHTTP     bool
+	window         time.Duration
+	listen         string
+	stopTimeout    time.Duration
+	logLevel       string
+	logPrefix      string
+	logHTTP        bool
+	logHTTPHeaders bool
 
 	weaveEnabled   bool
 	weaveAddr      string
@@ -298,6 +299,7 @@ func main() {
 	flag.StringVar(&flags.app.logLevel, "app.log.level", "info", "logging threshold level: debug|info|warn|error|fatal|panic")
 	flag.StringVar(&flags.app.logPrefix, "app.log.prefix", "<app>", "prefix for each log line")
 	flag.BoolVar(&flags.app.logHTTP, "app.log.http", false, "Log individual HTTP requests")
+	flag.BoolVar(&flags.app.logHTTPHeaders, "app.log.httpHeaders", false, "Log HTTP headers. Needs app.log.http to be enabled.")
 
 	flag.StringVar(&flags.app.weaveAddr, "app.weave.addr", app.DefaultWeaveURL, "Address on which to contact WeaveDNS")
 	flag.StringVar(&flags.app.weaveHostname, "app.weave.hostname", app.DefaultHostname, "Hostname to advertise in WeaveDNS")

vendor/github.com/weaveworks/common/logging/event.go

@@ -1,135 +0,0 @@
-package logging
-
-import (
-	"fmt"
-	"net"
-	"net/http"
-	"strconv"
-
-	log "github.com/Sirupsen/logrus"
-	"github.com/fluent/fluent-logger-golang/fluent"
-)
-
-const maxBufferedEvents = 100
-
-// Event is a user event to be sent to out analytics system
-type Event struct {
-	ID             string `msg:"event"`
-	SessionID      string `msg:"session_id"`
-	Product        string `msg:"product"`
-	Version        string `msg:"version"`
-	UserAgent      string `msg:"user_agent"`
-	ClientID       string `msg:"client_id"`
-	OrganizationID string `msg:"org_id"`
-	UserID         string `msg:"user_id"`
-	Values         string `msg:"values"`
-}
-
-// EventLogger logs events to the analytics system
-type EventLogger struct {
-	stop   chan struct{}
-	events chan Event
-	logger *fluent.Fluent
-}
-
-// NewEventLogger creates a new EventLogger.
-func NewEventLogger(fluentHostPort string) (*EventLogger, error) {
-	host, port, err := net.SplitHostPort(fluentHostPort)
-	if err != nil {
-		return nil, err
-	}
-	intPort, err := strconv.Atoi(port)
-	if err != nil {
-		return nil, err
-	}
-	logger, err := fluent.New(fluent.Config{
-		FluentPort:   intPort,
-		FluentHost:   host,
-		AsyncConnect: true,
-		MaxRetry:     -1,
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	el := &EventLogger{
-		stop:   make(chan struct{}),
-		events: make(chan Event, maxBufferedEvents),
-		logger: logger,
-	}
-	go el.logLoop()
-	return el, nil
-}
-
-func (el *EventLogger) post(e Event) {
-	if err := el.logger.Post("events", e); err != nil {
-		log.Warnf("EventLogger: failed to log event: %v", e)
-	}
-}
-
-func (el *EventLogger) logLoop() {
-	for done := false; !done; {
-		select {
-		case event := <-el.events:
-			el.post(event)
-		case <-el.stop:
-			done = true
-		}
-	}
-
-	// flush remaining events
-	for done := false; !done; {
-		select {
-		case event := <-el.events:
-			el.post(event)
-		default:
-			done = true
-		}
-	}
-
-	el.logger.Close()
-}
-
-// Close closes and deallocates the event logger
-func (el *EventLogger) Close() error {
-	close(el.stop)
-	return nil
-}
-
-// LogEvent logs an event to the analytics system
-func (el *EventLogger) LogEvent(e Event) error {
-	select {
-	case <-el.stop:
-		return fmt.Errorf("Stopping, discarding event: %v", e)
-	default:
-	}
-
-	select {
-	case el.events <- e: // Put event in the channel unless it is full
-		return nil
-	default:
-		// full
-	}
-	return fmt.Errorf("Reached event buffer limit (%d), discarding event: %v", maxBufferedEvents, e)
-}
-
-// HTTPEventExtractor extracts an event from an http requests indicating whether it should be loggged
-type HTTPEventExtractor func(*http.Request) (Event, bool)
-
-// HTTPEventLogger logs an events extracted from an http request
-type HTTPEventLogger struct {
-	Extractor HTTPEventExtractor
-	Logger    *EventLogger
-}
-
-// Wrap implements middleware.Wrap()
-func (el HTTPEventLogger) Wrap(next http.Handler) http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if event, shouldLog := el.Extractor(r); shouldLog {
-			if err := el.Logger.LogEvent(event); err != nil {
-				log.Warnf("HTTPEventLogger: failed to log event: %v", err)
-			}
-		}
-		next.ServeHTTP(w, r)
-	})
-}

vendor/github.com/weaveworks/common/middleware/logging.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"net"
 	"net/http"
+	"net/http/httputil"
 	"time"
 
 	log "github.com/Sirupsen/logrus"
@@ -12,7 +13,8 @@ import (
 
 // Log middleware logs http requests
 type Log struct {
-	LogSuccess bool // LogSuccess true -> log successful queries; false -> only log failed queries
+	LogSuccess        bool // LogSuccess true -> log successful queries; false -> only log failed queries
+	LogRequestHeaders bool // LogRequestHeaders true -> dump http headers at debug log level
 }
 
 // Wrap implements Middleware
@@ -20,6 +22,15 @@ func (l Log) Wrap(next http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		begin := time.Now()
 		uri := r.RequestURI // capture the URI before running next, as it may get rewritten
+		if l.LogRequestHeaders {
+			// Log headers before running 'next' in case other interceptors change the data.
+			headers, err := httputil.DumpRequest(r, false)
+			if err != nil {
+				log.Warnf("Could not dump request headers: %v", err)
+				return
+			}
+			log.Debugf("Is websocket request: %v\n%s", IsWSHandshakeRequest(r), string(headers))
+		}
 		i := &interceptor{ResponseWriter: w, statusCode: http.StatusOK}
 		next.ServeHTTP(i, r)
 		if l.LogSuccess || !(100 <= i.statusCode && i.statusCode < 400) {
@@ -31,7 +42,8 @@ func (l Log) Wrap(next http.Handler) http.Handler {
 // Logging middleware logs each HTTP request method, path, response code and
 // duration for all HTTP requests.
 var Logging = Log{
-	LogSuccess: true,
+	LogSuccess:        true,
+	LogRequestHeaders: false,
 }
 
 // LogFailed middleware logs each HTTP request method, path, response code and

vendor/github.com/weaveworks/common/middleware/redirect.go

@@ -0,0 +1,48 @@
+package middleware
+
+import (
+	"net/http"
+	"net/url"
+)
+
+// Redirect middleware, will redirect requests to hosts which match any of the
+// Matches to RedirectScheme://RedirectHost
+type Redirect struct {
+	Matches []Match
+
+	RedirectHost   string
+	RedirectScheme string
+}
+
+// Match specifies a match for a redirect.  Host and/or Scheme can be empty
+// signify match-all.
+type Match struct {
+	Host, Scheme string
+}
+
+func (m Match) match(u *url.URL) bool {
+	if m.Host != "" && m.Host != u.Host {
+		return false
+	}
+
+	if m.Scheme != "" && m.Scheme != u.Scheme {
+		return false
+	}
+
+	return true
+}
+
+// Wrap implements Middleware
+func (m Redirect) Wrap(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		for _, match := range m.Matches {
+			if match.match(r.URL) {
+				r.URL.Host = m.RedirectHost
+				r.URL.Scheme = m.RedirectScheme
+				http.Redirect(w, r, r.URL.String(), http.StatusMovedPermanently)
+				return
+			}
+		}
+		next.ServeHTTP(w, r)
+	})
+}

vendor/manifest

@@ -1361,7 +1361,7 @@
 			"importpath": "github.com/weaveworks/common",
 			"repository": "https://github.com/weaveworks/common",
 			"vcs": "git",
-			"revision": "cc20acf03ebf74be0facaae4259dff6cde01ce77",
+			"revision": "139d0313ac15170e9de8187b26e7df03b4cb910e",
 			"branch": "master",
 			"notests": true
 		},