From 8b52b10e38553a35d8134b428d6118a1ac011804 Mon Sep 17 00:00:00 2001 From: Tom Wilkie Date: Mon, 26 Oct 2015 15:13:15 +0000 Subject: [PATCH] Squashed 'tools/' content from commit 58c9a39 git-subtree-dir: tools git-subtree-split: 58c9a390c54860547254fa3a66e0acd8916f889c --- .gitignore | 4 + README.md | 36 +++++++ circle.yml | 23 ++++ cover/Makefile | 11 ++ cover/cover.go | 97 +++++++++++++++++ lint | 157 +++++++++++++++++++++++++++ rebuild-image | 63 +++++++++++ runner/Makefile | 11 ++ runner/runner.go | 275 +++++++++++++++++++++++++++++++++++++++++++++++ socks/Dockerfile | 7 ++ socks/Makefile | 29 +++++ socks/README.md | 53 +++++++++ socks/connect.sh | 23 ++++ socks/main.go | 87 +++++++++++++++ test | 66 ++++++++++++ 15 files changed, 942 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 circle.yml create mode 100644 cover/Makefile create mode 100644 cover/cover.go create mode 100755 lint create mode 100755 rebuild-image create mode 100644 runner/Makefile create mode 100644 runner/runner.go create mode 100644 socks/Dockerfile create mode 100644 socks/Makefile create mode 100644 socks/README.md create mode 100755 socks/connect.sh create mode 100644 socks/main.go create mode 100755 test diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..f5c83426c --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +cover/cover +socks/proxy +socks/image.tar +runner/runner diff --git a/README.md b/README.md new file mode 100644 index 000000000..85fdfeb39 --- /dev/null +++ b/README.md @@ -0,0 +1,36 @@ +# Weaveworks Build Tools + +Included in this repo are tools shared by weave.git and scope.git. They include + +- ```cover```: a tool which merges overlapping coverage reports generated by go + test +- ```lint```: a script to lint Go project; runs various tools like golint, go + vet, errcheck etc +- ```rebuild-image```: a script to rebuild docker images when their input files + change; useful when you using docker images to build your software, but you + don't want to build the image every time. +- ```socks```: a simple, dockerised SOCKS proxy for getting your laptop onto + the Weave network +- ```test```: a script to run all go unit tests in subdirectories, gather the + coverage results, and merge them into a single report. +- ```runner```: a tool for running tests in parallel; given each test is + suffixed with the number of hosts it requires, and the hosts available are + contained in the environment variable HOSTS, the tool will run tests in + parallel, on different hosts. + +## Using build-tools.git + +To allow you to tie your code to a specific version of build-tools.git, such +that future changes don't break you, we recommendation that you [`git subtree`]() +this repository into your own repository: + +[`git subtree`]: http://blogs.atlassian.com/2013/05/alternatives-to-git-submodule-git-subtree/ + +``` +git subtree add --prefix tools https://github.com/weaveworks/build-tools.git master --squash +```` + +To update the code in build-tools.git, the process is therefore: +- PR into build-tools.git, go through normal review process etc. +- Do `git subtree pull --prefix tools https://github.com/weaveworks/build-tools.git master --squash` + in your repo, and PR that. diff --git a/circle.yml b/circle.yml new file mode 100644 index 000000000..0c1ebddb1 --- /dev/null +++ b/circle.yml @@ -0,0 +1,23 @@ +machine: + services: + - docker + environment: + GOPATH: /home/ubuntu + SRCDIR: /home/ubuntu/src/github.com/weaveworks/tools + PATH: $PATH:$HOME/bin + +dependencies: + post: + - go clean -i net + - go install -tags netgo std + - mkdir -p $(dirname $SRCDIR) + - cp -r $(pwd)/ $SRCDIR + - go get github.com/golang/lint/golint github.com/fzipp/gocyclo github.com/kisielk/errcheck + +test: + override: + - cd $SRCDIR; ./lint . + - cd $SRCDIR/cover; make + - cd $SRCDIR/socks; make + - cd $SRCDIR/runner; make + diff --git a/cover/Makefile b/cover/Makefile new file mode 100644 index 000000000..1453e63e3 --- /dev/null +++ b/cover/Makefile @@ -0,0 +1,11 @@ +.PHONY: all clean + +all: cover + +cover: *.go + go get -tags netgo ./$(@D) + go build -ldflags "-extldflags \"-static\" -linkmode=external" -tags netgo -o $@ ./$(@D) + +clean: + rm -rf cover + go clean ./... diff --git a/cover/cover.go b/cover/cover.go new file mode 100644 index 000000000..4c5fcfd7d --- /dev/null +++ b/cover/cover.go @@ -0,0 +1,97 @@ +package main + +import ( + "fmt" + "os" + "sort" + + "golang.org/x/tools/cover" +) + +func merge(p1, p2 *cover.Profile) *cover.Profile { + output := cover.Profile{ + FileName: p1.FileName, + Mode: p1.Mode, + } + + i, j := 0, 0 + for i < len(p1.Blocks) && j < len(p2.Blocks) { + bi, bj := p1.Blocks[i], p2.Blocks[j] + if bi.StartLine == bj.StartLine && bi.StartCol == bj.StartCol { + + if bi.EndLine != bj.EndLine || + bi.EndCol != bj.EndCol || + bi.NumStmt != bj.NumStmt { + panic("Not run on same source!") + } + + output.Blocks = append(output.Blocks, cover.ProfileBlock{ + StartLine: bi.StartLine, + StartCol: bi.StartCol, + EndLine: bi.EndLine, + EndCol: bi.EndCol, + NumStmt: bi.NumStmt, + Count: bi.Count + bj.Count, + }) + i++ + j++ + } else if bi.StartLine < bj.StartLine || bi.StartLine == bj.StartLine && bi.StartCol < bj.StartCol { + output.Blocks = append(output.Blocks, bi) + i++ + } else { + output.Blocks = append(output.Blocks, bj) + j++ + } + } + + for ; i < len(p1.Blocks); i++ { + output.Blocks = append(output.Blocks, p1.Blocks[i]) + } + + for ; j < len(p2.Blocks); j++ { + output.Blocks = append(output.Blocks, p2.Blocks[j]) + } + + return &output +} + +func print(profiles []*cover.Profile) { + fmt.Println("mode: atomic") + for _, profile := range profiles { + for _, block := range profile.Blocks { + fmt.Printf("%s:%d.%d,%d.%d %d %d\n", profile.FileName, block.StartLine, block.StartCol, + block.EndLine, block.EndCol, block.NumStmt, block.Count) + } + } +} + +// Copied from https://github.com/golang/tools/blob/master/cover/profile.go +type byFileName []*cover.Profile + +func (p byFileName) Len() int { return len(p) } +func (p byFileName) Less(i, j int) bool { return p[i].FileName < p[j].FileName } +func (p byFileName) Swap(i, j int) { p[i], p[j] = p[j], p[i] } + +func main() { + outputProfiles := map[string]*cover.Profile{} + for _, input := range os.Args[1:] { + inputProfiles, err := cover.ParseProfiles(input) + if err != nil { + panic(fmt.Sprintf("Error parsing %s: %v", input, err)) + } + for _, ip := range inputProfiles { + op := outputProfiles[ip.FileName] + if op == nil { + outputProfiles[ip.FileName] = ip + } else { + outputProfiles[ip.FileName] = merge(op, ip) + } + } + } + profiles := make([]*cover.Profile, 0, len(outputProfiles)) + for _, profile := range outputProfiles { + profiles = append(profiles, profile) + } + sort.Sort(byFileName(profiles)) + print(profiles) +} diff --git a/lint b/lint new file mode 100755 index 000000000..818c05a25 --- /dev/null +++ b/lint @@ -0,0 +1,157 @@ +#!/bin/bash +# This scipt lints go files for common errors. +# +# Its runs gofmt and go vet, and optionally golint and +# gocyclo, if they are installed. +# +# With no arguments, it lints the current files staged +# for git commit. Or you can pass it explicit filenames +# (or directories) and it will lint them. +# +# To use this script automatically, run: +# ln -s ../../bin/lint .git/hooks/pre-commit + +set -e + +IGNORE_LINT_COMMENT= +IGNORE_TEST_PACKAGES= +while true; do + case "$1" in + -nocomment) + IGNORE_LINT_COMMENT=1 + shift 1 + ;; + -notestpackage) + IGNORE_TEST_PACKAGES=1 + shift 1 + ;; + *) + break + esac +done + + +function spell_check { + filename="$1" + local lint_result=0 + + if grep -iH --color=always psueod "${filename}"; then + echo "${filename}: spelling mistake" + lint_result=1 + fi + + return $lint_result +} + +function test_mismatch { + filename="$1" + package=$(grep '^package ' $filename | awk '{print $2}') + local lint_result=0 + + if [[ $package == "main" ]]; then + continue # in package main, all bets are off + fi + + if [[ $filename == *"_internal_test.go" ]]; then + if [[ $package == *"_test" ]]; then + lint_result=1 + echo "${filename}: should not be part of a _test package" + fi + else + if [[ ! $package == *"_test" ]]; then + lint_result=1 + echo "${filename}: should be part of a _test package" + fi + fi + + return $lint_result +} + +function lint_go { + filename="$1" + local lint_result=0 + + if [ -n "$(gofmt -s -l "${filename}")" ]; then + lint_result=1 + echo "${filename}: run gofmt -s -w ${filename}!" + fi + + go tool vet "${filename}" || lint_result=$? + + # golint is completely optional. If you don't like it + # don't have it installed. + if type golint >/dev/null 2>&1; then + # golint doesn't set an exit code it seems + if [ -z "$IGNORE_LINT_COMMENT" ]; then + lintoutput=$(golint "${filename}") + else + lintoutput=$(golint "${filename}" | grep -vE 'comment|dot imports') + fi + if [ -n "$lintoutput" ]; then + lint_result=1 + echo "$lintoutput" + fi + fi + + # gocyclo is completely optional. If you don't like it + # don't have it installed. Also never blocks a commit, + # it just warns. + if type gocyclo >/dev/null 2>&1; then + gocyclo -over 25 "${filename}" | while read line; do + echo "${filename}": higher than 25 cyclomatic complexity - "${line}" + done + fi + + return $lint_result +} + +function lint { + filename="$1" + ext="${filename##*\.}" + local lint_result=0 + + # Don't lint deleted files + if [ ! -f "$filename" ]; then + return + fi + + # Don't lint this script or static.go + case "$(basename "${filename}")" in + lint) return;; + static.go) return;; + coverage.html) return;; + esac + + case "$ext" in + go) lint_go "${filename}" || lint_result=1 + ;; + esac + + if [ -z "$IGNORE_TEST_PACKAGES" ]; then + if [[ "$filename" == *"_test.go" ]]; then + test_mismatch "${filename}" || lint_result=1 + fi + fi + + spell_check "${filename}" || lint_result=1 + + return $lint_result +} + +function lint_files { + local lint_result=0 + while read filename; do + lint "${filename}" || lint_result=1 + done + exit $lint_result +} + +function list_files { + if [ $# -gt 0 ]; then + find "$@" -type f | grep -vE '^\./(\.git|vendor)/' + else + git diff --cached --name-only + fi +} + +list_files "$@" | lint_files diff --git a/rebuild-image b/rebuild-image new file mode 100755 index 000000000..0e582b0ba --- /dev/null +++ b/rebuild-image @@ -0,0 +1,63 @@ +#!/bin/bash +# Rebuild a cached docker image if the input files have changed. +# Usage: ./rebuild-image + +set -eux + +IMAGENAME=$1 +SAVEDNAME=$(echo $IMAGENAME | sed "s/[\/\-]/\./g") +IMAGEDIR=$2 +shift 2 + +INPUTFILES=$@ +CACHEDIR=$HOME/docker/ + +# Rebuild the image +rebuild() { + mkdir -p $CACHEDIR + rm $CACHEDIR/$SAVEDNAME* || true + docker build -t $IMAGENAME $IMAGEDIR + docker save $IMAGENAME:latest > $CACHEDIR/$SAVEDNAME-$CIRCLE_SHA1 +} + +# Get the revision the cached image was build at +cached_image_rev() { + find $CACHEDIR -name "$SAVEDNAME-*" -type f | sed 's/[^\-]*\-//' +} + +# Have there been any revision beween $1 and $2 +has_changes() { + local rev1=$1 + local rev2=$2 + local changes=$(git log --oneline $rev1..$rev2 -- $INPUTFILES | wc -l) + [ "$changes" -gt 0 ] +} + +commit_timestamp() { + local rev=$1 + git show -s --format=%ct $rev +} + +cached_revision=$(cached_image_rev) +if [ -z "$cached_revision" ]; then + echo ">>> No cached image found; rebuilding" + rebuild + exit 0 +fi + +echo ">>> Found cached image rev $cached_revision" +if has_changes $cached_revision $CIRCLE_SHA1 ; then + echo ">>> Found changes, rebuilding" + rebuild + exit 0 +fi + +if [ "$(commit_timestamp $cached_revision)" -lt "$(( $(date +%s) - 86400 ))" ]; then + echo ">>> Image is more the 24hrs old; rebuilding" + rebuild + exit 0 +fi + +# we didn't rebuild; import cached version +echo ">>> No changes found, importing cached image" +docker load -i $CACHEDIR/$SAVEDNAME-$cached_revision diff --git a/runner/Makefile b/runner/Makefile new file mode 100644 index 000000000..f19bcc7d8 --- /dev/null +++ b/runner/Makefile @@ -0,0 +1,11 @@ +.PHONY: all clean + +all: runner + +runner: *.go + go get -tags netgo ./$(@D) + go build -ldflags "-extldflags \"-static\" -linkmode=external" -tags netgo -o $@ ./$(@D) + +clean: + rm -rf runner + go clean ./... diff --git a/runner/runner.go b/runner/runner.go new file mode 100644 index 000000000..bfac9c58b --- /dev/null +++ b/runner/runner.go @@ -0,0 +1,275 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "net/url" + "os" + "os/exec" + "sort" + "strconv" + "strings" + "sync" + "time" + + "github.com/docker/docker/pkg/mflag" + "github.com/mgutz/ansi" +) + +const ( + defaultSchedulerHost = "positive-cocoa-90213.appspot.com" + jsonContentType = "application/json" +) + +var ( + start = ansi.ColorCode("black+ub") + fail = ansi.ColorCode("red+b") + succ = ansi.ColorCode("green+b") + reset = ansi.ColorCode("reset") + + schedulerHost = defaultSchedulerHost + useScheduler = false + runParallel = false + verbose = false + + consoleLock = sync.Mutex{} +) + +type test struct { + name string + hosts int +} + +type schedule struct { + Tests []string `json:"tests"` +} + +type result struct { + test + errored bool + hosts []string +} + +type tests []test + +func (ts tests) Len() int { return len(ts) } +func (ts tests) Swap(i, j int) { ts[i], ts[j] = ts[j], ts[i] } +func (ts tests) Less(i, j int) bool { + if ts[i].hosts != ts[j].hosts { + return ts[i].hosts < ts[j].hosts + } + return ts[i].name < ts[j].name +} + +func (ts *tests) pick(availible int) (test, bool) { + // pick the first test that fits in the availible hosts + for i, test := range *ts { + if test.hosts <= availible { + *ts = append((*ts)[:i], (*ts)[i+1:]...) + return test, true + } + } + + return test{}, false +} + +func (t test) run(hosts []string) bool { + consoleLock.Lock() + fmt.Printf("%s>>> Running %s on %s%s\n", start, t.name, hosts, reset) + consoleLock.Unlock() + + var out bytes.Buffer + + cmd := exec.Command(t.name) + cmd.Env = os.Environ() + cmd.Stdout = &out + cmd.Stderr = &out + + // replace HOSTS in env + for i, env := range cmd.Env { + if strings.HasPrefix(env, "HOSTS") { + cmd.Env[i] = fmt.Sprintf("HOSTS=%s", strings.Join(hosts, " ")) + break + } + } + + start := time.Now() + err := cmd.Run() + duration := float64(time.Now().Sub(start)) / float64(time.Second) + + consoleLock.Lock() + if err != nil { + fmt.Printf("%s>>> Test %s finished after %0.1f secs with error: %v%s\n", fail, t.name, duration, err, reset) + } else { + fmt.Printf("%s>>> Test %s finished with success after %0.1f secs%s\n", succ, t.name, duration, reset) + } + if err != nil || verbose { + fmt.Print(out.String()) + fmt.Println() + } + consoleLock.Unlock() + + if err != nil && useScheduler { + updateScheduler(t.name, duration) + } + + return err != nil +} + +func updateScheduler(test string, duration float64) { + req := &http.Request{ + Method: "POST", + Host: schedulerHost, + URL: &url.URL{ + Opaque: fmt.Sprintf("/record/%s/%0.2f", url.QueryEscape(test), duration), + Scheme: "http", + Host: schedulerHost, + }, + Close: true, + } + if resp, err := http.DefaultClient.Do(req); err != nil { + fmt.Printf("Error updating scheduler: %v\n", err) + } else { + resp.Body.Close() + } +} + +func getSchedule(tests []string) ([]string, error) { + var ( + testRun = "integration-" + os.Getenv("CIRCLE_BUILD_NUM") + shardCount = os.Getenv("CIRCLE_NODE_TOTAL") + shardID = os.Getenv("CIRCLE_NODE_INDEX") + requestBody = &bytes.Buffer{} + ) + if err := json.NewEncoder(requestBody).Encode(schedule{tests}); err != nil { + return []string{}, err + } + url := fmt.Sprintf("http://%s/schedule/%s/%s/%s", schedulerHost, testRun, shardCount, shardID) + resp, err := http.Post(url, jsonContentType, requestBody) + if err != nil { + return []string{}, err + } + var sched schedule + if err := json.NewDecoder(resp.Body).Decode(&sched); err != nil { + return []string{}, err + } + return sched.Tests, nil +} + +func getTests(testNames []string) (tests, error) { + var err error + if useScheduler { + testNames, err = getSchedule(testNames) + if err != nil { + return tests{}, err + } + } + tests := tests{} + for _, name := range testNames { + parts := strings.Split(strings.TrimSuffix(name, "_test.sh"), "_") + numHosts, err := strconv.Atoi(parts[len(parts)-1]) + if err != nil { + numHosts = 1 + } + tests = append(tests, test{name, numHosts}) + fmt.Printf("Test %s needs %d hosts\n", name, numHosts) + } + return tests, nil +} + +func summary(tests, failed tests) { + if len(failed) > 0 { + fmt.Printf("%s>>> Ran %d tests, %d failed%s\n", fail, len(tests), len(failed), reset) + for _, test := range failed { + fmt.Printf("%s>>> Fail %s%s\n", fail, test.name, reset) + } + } else { + fmt.Printf("%s>>> Ran %d tests, all succeeded%s\n", succ, len(tests), reset) + } +} + +func parallel(ts tests, hosts []string) bool { + testsCopy := ts + sort.Sort(sort.Reverse(ts)) + resultsChan := make(chan result) + outstanding := 0 + failed := tests{} + for len(ts) > 0 || outstanding > 0 { + // While we have some free hosts, try and schedule + // a test on them + for len(hosts) > 0 { + test, ok := ts.pick(len(hosts)) + if !ok { + break + } + testHosts := hosts[:test.hosts] + hosts = hosts[test.hosts:] + + go func() { + errored := test.run(testHosts) + resultsChan <- result{test, errored, testHosts} + }() + outstanding++ + } + + // Otherwise, wait for the test to finish and return + // the hosts to the pool + result := <-resultsChan + hosts = append(hosts, result.hosts...) + outstanding-- + if result.errored { + failed = append(failed, result.test) + } + } + summary(testsCopy, failed) + return len(failed) > 0 +} + +func sequential(ts tests, hosts []string) bool { + failed := tests{} + for _, test := range ts { + if test.run(hosts) { + failed = append(failed, test) + } + } + summary(ts, failed) + return len(failed) > 0 +} + +func main() { + mflag.BoolVar(&useScheduler, []string{"scheduler"}, false, "Use scheduler to distribute tests across shards") + mflag.BoolVar(&runParallel, []string{"parallel"}, false, "Run tests in parallel on hosts where possible") + mflag.BoolVar(&verbose, []string{"v"}, false, "Print output from all tests (Also enabled via DEBUG=1)") + mflag.StringVar(&schedulerHost, []string{"scheduler-host"}, defaultSchedulerHost, "Hostname of scheduler.") + mflag.Parse() + + if len(os.Getenv("DEBUG")) > 0 { + verbose = true + } + + tests, err := getTests(mflag.Args()) + if err != nil { + fmt.Printf("Error parsing tests: %v\n", err) + os.Exit(1) + } + + hosts := strings.Fields(os.Getenv("HOSTS")) + maxHosts := len(hosts) + if maxHosts == 0 { + fmt.Print("No HOSTS specified.\n") + os.Exit(1) + } + + var errored bool + if runParallel { + errored = parallel(tests, hosts) + } else { + errored = sequential(tests, hosts) + } + + if errored { + os.Exit(1) + } +} diff --git a/socks/Dockerfile b/socks/Dockerfile new file mode 100644 index 000000000..867cd6bc5 --- /dev/null +++ b/socks/Dockerfile @@ -0,0 +1,7 @@ +FROM gliderlabs/alpine +MAINTAINER Weaveworks Inc +WORKDIR / +COPY proxy / +EXPOSE 8000 +EXPOSE 8080 +ENTRYPOINT ["/proxy"] diff --git a/socks/Makefile b/socks/Makefile new file mode 100644 index 000000000..2daeda643 --- /dev/null +++ b/socks/Makefile @@ -0,0 +1,29 @@ +.PHONY: all clean + +IMAGE_TAR=image.tar +IMAGE_NAME=weaveworks/socksproxy +PROXY_EXE=proxy +NETGO_CHECK=@strings $@ | grep cgo_stub\\\.go >/dev/null || { \ + rm $@; \ + echo "\nYour go standard library was built without the 'netgo' build tag."; \ + echo "To fix that, run"; \ + echo " sudo go clean -i net"; \ + echo " sudo go install -tags netgo std"; \ + false; \ +} + +all: $(IMAGE_TAR) + +$(IMAGE_TAR): Dockerfile $(PROXY_EXE) + docker build -t $(IMAGE_NAME) . + docker save $(IMAGE_NAME):latest > $@ + +$(PROXY_EXE): *.go + go get -tags netgo ./$(@D) + go build -ldflags "-extldflags \"-static\" -linkmode=external" -tags netgo -o $@ ./$(@D) + $(NETGO_CHECK) + +clean: + -docker rmi $(IMAGE_NAME) + rm -rf $(PROXY_EXE) $(IMAGE_TAR) + go clean ./... diff --git a/socks/README.md b/socks/README.md new file mode 100644 index 000000000..596c2b32e --- /dev/null +++ b/socks/README.md @@ -0,0 +1,53 @@ +# SOCKS Proxy + +The challenge: you’ve built and deployed your microservices based +application on a Weave network, running on a set of VMs on EC2. Many +of the services’ public API are reachable from the internet via an +Nginx-based reverse proxy, but some of the services also expose +private monitoring and manage endpoints via embedded HTTP servers. +How do I securely get access to these from my laptop, without exposing +them to the world? + +One method we’ve started using at Weaveworks is a 90’s technology - a +SOCKS proxy combined with a PAC script. It’s relatively +straight-forward: one ssh’s into any of the VMs participating in the +Weave network, starts the SOCKS proxy in a container on Weave the +network, and SSH port forwards a few local port to the proxy. All +that’s left is for the user to configure his browser to use the proxy, +and voila, you can now access your Docker containers, via the Weave +network (and with all the magic of weavedns), from your laptop’s +browser! + +It is perhaps worth noting there is nothing Weave-specific about this +approach - this should work with any SDN or private network. + +A quick example: + +``` +vm1$ weave launch +vm1$ eval $(weave env) +vm1$ docker run -d --name nginx nginx +``` + +And on your laptop + +``` +laptop$ git clone https://github.com/weaveworks/tools +laptop$ cd tools/socks +laptop$ ./connect.sh vm1 +Starting proxy container... +Please configure your browser for proxy +http://localhost:8080/proxy.pac +``` + +To configure your Mac to use the proxy: + +1. Open System Preferences +2. Select Network +3. Click the 'Advanced' button +4. Select the Proxies tab +5. Click the 'Automatic Proxy Configuration' check box +6. Enter 'http://localhost:8080/proxy.pac' in the URL box +7. Remove `*.local` from the 'Bypass proxy settings for these Hosts & Domains' + +Now point your browser at http://nginx.weave.local/ diff --git a/socks/connect.sh b/socks/connect.sh new file mode 100755 index 000000000..0d5ef84da --- /dev/null +++ b/socks/connect.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -eu + +if [ $# -ne 1 ]; then + echo "Usage: $0 " + exit 1 +fi + +HOST=$1 + +echo "Starting proxy container..." +PROXY_CONTAINER=$(ssh $HOST weave run -d weaveworks/socksproxy) + +function finish { + echo "Removing proxy container.." + ssh $HOST docker rm -f $PROXY_CONTAINER +} +trap finish EXIT + +PROXY_IP=$(ssh $HOST -- "docker inspect --format='{{.NetworkSettings.IPAddress}}' $PROXY_CONTAINER") +echo 'Please configure your browser for proxy http://localhost:8080/proxy.pac' +ssh -L8000:$PROXY_IP:8000 -L8080:$PROXY_IP:8080 $HOST docker attach $PROXY_CONTAINER diff --git a/socks/main.go b/socks/main.go new file mode 100644 index 000000000..171b5fc13 --- /dev/null +++ b/socks/main.go @@ -0,0 +1,87 @@ +package main + +import ( + "fmt" + "net" + "net/http" + "os" + "strings" + "text/template" + + socks5 "github.com/armon/go-socks5" + "github.com/docker/docker/pkg/mflag" + "github.com/weaveworks/weave/common/mflagext" +) + +const ( + pacfile = ` +function FindProxyForURL(url, host) { + if(shExpMatch(host, "*.weave.local")) { + return "SOCKS5 localhost:8000"; + } + {{range $key, $value := .}} + if (host == "{{$key}}") { + return "SOCKS5 localhost:8000"; + } + {{end}} + return "DIRECT"; +} +` +) + +func main() { + var as []string + mflagext.ListVar(&as, []string{"a", "-alias"}, []string{}, "Specify hostname aliases in the form alias:hostname. Can be repeated.") + mflag.Parse() + + var aliases = map[string]string{} + for _, a := range as { + parts := strings.SplitN(a, ":", 2) + if len(parts) != 2 { + fmt.Printf("'%s' is not a valid alias.\n", a) + mflag.Usage() + os.Exit(1) + } + aliases[parts[0]] = parts[1] + } + + go socksProxy(aliases) + + t := template.Must(template.New("pacfile").Parse(pacfile)) + http.HandleFunc("/proxy.pac", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/x-ns-proxy-autoconfig") + t.Execute(w, aliases) + }) + + if err := http.ListenAndServe(":8080", nil); err != nil { + panic(err) + } +} + +type aliasingResolver struct { + aliases map[string]string + socks5.NameResolver +} + +func (r aliasingResolver) Resolve(name string) (net.IP, error) { + if alias, ok := r.aliases[name]; ok { + return r.NameResolver.Resolve(alias) + } + return r.NameResolver.Resolve(name) +} + +func socksProxy(aliases map[string]string) { + conf := &socks5.Config{ + Resolver: aliasingResolver{ + aliases: aliases, + NameResolver: socks5.DNSResolver{}, + }, + } + server, err := socks5.New(conf) + if err != nil { + panic(err) + } + if err := server.ListenAndServe("tcp", ":8000"); err != nil { + panic(err) + } +} diff --git a/test b/test new file mode 100755 index 000000000..017fdc80f --- /dev/null +++ b/test @@ -0,0 +1,66 @@ +#!/bin/bash + +set -e + +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +GO_TEST_ARGS="-tags netgo -cpu 4 -timeout 8m" + +if [ -n "$SLOW" -o "$1" = "-slow" -o -n "$CIRCLECI" ]; then + SLOW=true +fi + +if [ -n "$SLOW" ]; then + GO_TEST_ARGS="$GO_TEST_ARGS -race -covermode=atomic" + + if [ -n "$COVERDIR" ] ; then + coverdir="$COVERDIR" + else + coverdir=$(mktemp -d coverage.XXXXXXXXXX) + fi + + mkdir -p $coverdir +fi + +fail=0 + +TESTDIRS=$(find . -type f -name '*_test.go' | xargs -n1 dirname | grep -vE '^\./(\.git|vendor|prog|experimental)/' | sort -u) + +# If running on circle, use the scheduler to work out what tests to run on what shard +if [ -n "$CIRCLECI" -a -z "$NO_SCHEDULER" -a -x "$DIR/sched" ]; then + TESTDIRS=$(echo $TESTDIRS | "$DIR/sched" sched units-$CIRCLE_BUILD_NUM $CIRCLE_NODE_TOTAL $CIRCLE_NODE_INDEX) + echo $TESTDIRS +fi + +PACKAGE_BASE=$(go list -e ./) + +for dir in $TESTDIRS; do + go get -t -tags netgo $dir + + GO_TEST_ARGS_RUN="$GO_TEST_ARGS" + if [ -n "$SLOW" ]; then + COVERPKGS=$( (go list $dir; go list -f '{{join .Deps "\n"}}' $dir | grep "^$PACKAGE_BASE/") | paste -s -d,) + output=$(mktemp $coverdir/unit.XXXXXXXXXX) + GO_TEST_ARGS_RUN="$GO_TEST_ARGS -coverprofile=$output -coverpkg=$COVERPKGS" + fi + + START=$(date +%s) + if ! go test $GO_TEST_ARGS_RUN $dir ; then + fail=1 + fi + RUNTIME=$(( $(date +%s) - $START )) + + # Report test runtime when running on circle, to help scheduler + if [ -n "$CIRCLECI" -a -z "$NO_SCHEDULER" -a -x "$DIR/sched" ]; then + "$DIR/sched" time $dir $RUNTIME + fi +done + +if [ -n "$SLOW" -a -z "$COVERDIR" ] ; then + go get github.com/weaveworks/tools/cover + cover $coverdir/* >profile.cov + rm -rf $coverdir + go tool cover -html=profile.cov -o=coverage.html + go tool cover -func=profile.cov | tail -n1 +fi + +exit $fail