From f53a68b1b7ac6a726b43d210d600dab0c4c7e76e Mon Sep 17 00:00:00 2001 From: Tom Wilkie Date: Fri, 23 Oct 2015 09:29:33 +0000 Subject: [PATCH 1/3] Copy test runner from weave repo, add some more details to README. --- .gitignore | 1 + README.md | 42 ++++++-- circle.yml | 1 + runner/Makefile | 11 ++ runner/runner.go | 273 +++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 319 insertions(+), 9 deletions(-) create mode 100644 runner/Makefile create mode 100644 runner/runner.go diff --git a/.gitignore b/.gitignore index 78fe8261e..f5c83426c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ cover/cover socks/proxy socks/image.tar +runner/runner diff --git a/README.md b/README.md index ad801aa00..154b8e75b 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,36 @@ -# Weaveworks Tools +# Weaveworks Build Tools Included in this repo are tools shared by weave.git and scope.git. They include -- ```cover```: a tool which merges overlapping coverage reports generated by go test -- ```lint```: a script to lint Go project; runs various tools like golint, go vet, errcheck etc -- ```rebuild-image```: a script to rebuild docker images when their input files change; useful - when you using docker images to build your software, but you don't want to build the - image every time. -- ```socks```: a simple, dockerised SOCKS proxy for getting your laptop onto the Weave network -- ```test```: a script to run all go unit tests in subdirectories, gather the coverage results, - and merge them into a single report. +- ```cover```: a tool which merges overlapping coverage reports generated by go + test +- ```lint```: a script to lint Go project; runs various tools like golint, go + vet, errcheck etc +- ```rebuild-image```: a script to rebuild docker images when their input files + change; useful when you using docker images to build your software, but you + don't want to build the image every time. +- ```socks```: a simple, dockerised SOCKS proxy for getting your laptop onto + the Weave network +- ```test```: a script to run all go unit tests in subdirectories, gather the + coverage results, and merge them into a single report. +- ```runner```: a tools for running tests in parallel; given each test is + suffixes with the number of hosts it requires, and the host availible are + contained in the environment variable HOSTS, the tool will run tests in + parallel, on different hosts. + +## Using build-tools.git + +To allow you to tie your code to a specific version of build-tools.git, such +that future changes don't break you, we recommendation that you [`git subtree`]() +this repository into your own repository: + +[`git subtree`]: http://blogs.atlassian.com/2013/05/alternatives-to-git-submodule-git-subtree/ + +``` +git subtree add --prefix tools https://github.com/weaveworks/build-tools.git master --squash +```` + +To update the code in build-tools.git, the process is therefore: +- PR into build-tools.git, go through normal review process etc. +- Do `git subtree pull --prefix tools https://github.com/weaveworks/build-tools.git master --squash` + in your repo, and PR that. diff --git a/circle.yml b/circle.yml index b79465799..0c1ebddb1 100644 --- a/circle.yml +++ b/circle.yml @@ -19,4 +19,5 @@ test: - cd $SRCDIR; ./lint . - cd $SRCDIR/cover; make - cd $SRCDIR/socks; make + - cd $SRCDIR/runner; make diff --git a/runner/Makefile b/runner/Makefile new file mode 100644 index 000000000..f19bcc7d8 --- /dev/null +++ b/runner/Makefile @@ -0,0 +1,11 @@ +.PHONY: all clean + +all: runner + +runner: *.go + go get -tags netgo ./$(@D) + go build -ldflags "-extldflags \"-static\" -linkmode=external" -tags netgo -o $@ ./$(@D) + +clean: + rm -rf runner + go clean ./... diff --git a/runner/runner.go b/runner/runner.go new file mode 100644 index 000000000..89b8c6ad1 --- /dev/null +++ b/runner/runner.go @@ -0,0 +1,273 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "net/url" + "os" + "os/exec" + "sort" + "strconv" + "strings" + "sync" + "time" + + "github.com/docker/docker/pkg/mflag" + "github.com/mgutz/ansi" +) + +const ( + schedulerHost = "positive-cocoa-90213.appspot.com" + jsonContentType = "application/json" +) + +var ( + start = ansi.ColorCode("black+ub") + fail = ansi.ColorCode("red+b") + succ = ansi.ColorCode("green+b") + reset = ansi.ColorCode("reset") + + useScheduler = false + runParallel = false + verbose = false + + consoleLock = sync.Mutex{} +) + +type test struct { + name string + hosts int +} + +type schedule struct { + Tests []string `json:"tests"` +} + +type result struct { + test + errored bool + hosts []string +} + +type tests []test + +func (ts tests) Len() int { return len(ts) } +func (ts tests) Swap(i, j int) { ts[i], ts[j] = ts[j], ts[i] } +func (ts tests) Less(i, j int) bool { + if ts[i].hosts != ts[j].hosts { + return ts[i].hosts < ts[j].hosts + } + return ts[i].name < ts[j].name +} + +func (ts *tests) pick(availible int) (test, bool) { + // pick the first test that fits in the availible hosts + for i, test := range *ts { + if test.hosts <= availible { + *ts = append((*ts)[:i], (*ts)[i+1:]...) + return test, true + } + } + + return test{}, false +} + +func (t test) run(hosts []string) bool { + consoleLock.Lock() + fmt.Printf("%s>>> Running %s on %s%s\n", start, t.name, hosts, reset) + consoleLock.Unlock() + + var out bytes.Buffer + + cmd := exec.Command(t.name) + cmd.Env = os.Environ() + cmd.Stdout = &out + cmd.Stderr = &out + + // replace HOSTS in env + for i, env := range cmd.Env { + if strings.HasPrefix(env, "HOSTS") { + cmd.Env[i] = fmt.Sprintf("HOSTS=%s", strings.Join(hosts, " ")) + break + } + } + + start := time.Now() + err := cmd.Run() + duration := float64(time.Now().Sub(start)) / float64(time.Second) + + consoleLock.Lock() + if err != nil { + fmt.Printf("%s>>> Test %s finished after %0.1f secs with error: %v%s\n", fail, t.name, duration, err, reset) + } else { + fmt.Printf("%s>>> Test %s finished with success after %0.1f secs%s\n", succ, t.name, duration, reset) + } + if err != nil || verbose { + fmt.Print(out.String()) + fmt.Println() + } + consoleLock.Unlock() + + if err != nil && useScheduler { + updateScheduler(t.name, duration) + } + + return err != nil +} + +func updateScheduler(test string, duration float64) { + req := &http.Request{ + Method: "POST", + Host: schedulerHost, + URL: &url.URL{ + Opaque: fmt.Sprintf("/record/%s/%0.2f", url.QueryEscape(test), duration), + Scheme: "http", + Host: schedulerHost, + }, + Close: true, + } + if resp, err := http.DefaultClient.Do(req); err != nil { + fmt.Printf("Error updating scheduler: %v\n", err) + } else { + resp.Body.Close() + } +} + +func getSchedule(tests []string) ([]string, error) { + var ( + testRun = "integration-" + os.Getenv("CIRCLE_BUILD_NUM") + shardCount = os.Getenv("CIRCLE_NODE_TOTAL") + shardID = os.Getenv("CIRCLE_NODE_INDEX") + requestBody = &bytes.Buffer{} + ) + if err := json.NewEncoder(requestBody).Encode(schedule{tests}); err != nil { + return []string{}, err + } + url := fmt.Sprintf("http://%s/schedule/%s/%s/%s", schedulerHost, testRun, shardCount, shardID) + resp, err := http.Post(url, jsonContentType, requestBody) + if err != nil { + return []string{}, err + } + var sched schedule + if err := json.NewDecoder(resp.Body).Decode(&sched); err != nil { + return []string{}, err + } + return sched.Tests, nil +} + +func getTests(testNames []string) (tests, error) { + var err error + if useScheduler { + testNames, err = getSchedule(testNames) + if err != nil { + return tests{}, err + } + } + tests := tests{} + for _, name := range testNames { + parts := strings.Split(strings.TrimSuffix(name, "_test.sh"), "_") + numHosts, err := strconv.Atoi(parts[len(parts)-1]) + if err != nil { + numHosts = 1 + } + tests = append(tests, test{name, numHosts}) + fmt.Printf("Test %s needs %d hosts\n", name, numHosts) + } + return tests, nil +} + +func summary(tests, failed tests) { + if len(failed) > 0 { + fmt.Printf("%s>>> Ran %d tests, %d failed%s\n", fail, len(tests), len(failed), reset) + for _, test := range failed { + fmt.Printf("%s>>> Fail %s%s\n", fail, test.name, reset) + } + } else { + fmt.Printf("%s>>> Ran %d tests, all succeeded%s\n", succ, len(tests), reset) + } +} + +func parallel(ts tests, hosts []string) bool { + testsCopy := ts + sort.Sort(sort.Reverse(ts)) + resultsChan := make(chan result) + outstanding := 0 + failed := tests{} + for len(ts) > 0 || outstanding > 0 { + // While we have some free hosts, try and schedule + // a test on them + for len(hosts) > 0 { + test, ok := ts.pick(len(hosts)) + if !ok { + break + } + testHosts := hosts[:test.hosts] + hosts = hosts[test.hosts:] + + go func() { + errored := test.run(testHosts) + resultsChan <- result{test, errored, testHosts} + }() + outstanding++ + } + + // Otherwise, wait for the test to finish and return + // the hosts to the pool + result := <-resultsChan + hosts = append(hosts, result.hosts...) + outstanding-- + if result.errored { + failed = append(failed, result.test) + } + } + summary(testsCopy, failed) + return len(failed) > 0 +} + +func sequential(ts tests, hosts []string) bool { + failed := tests{} + for _, test := range ts { + if test.run(hosts) { + failed = append(failed, test) + } + } + summary(ts, failed) + return len(failed) > 0 +} + +func main() { + mflag.BoolVar(&useScheduler, []string{"scheduler"}, false, "Use scheduler to distribute tests across shards") + mflag.BoolVar(&runParallel, []string{"parallel"}, false, "Run tests in parallel on hosts where possible") + mflag.BoolVar(&verbose, []string{"v"}, false, "Print output from all tests (Also enabled via DEBUG=1)") + mflag.Parse() + + if len(os.Getenv("DEBUG")) > 0 { + verbose = true + } + + tests, err := getTests(mflag.Args()) + if err != nil { + fmt.Printf("Error parsing tests: %v\n", err) + os.Exit(1) + } + + hosts := strings.Fields(os.Getenv("HOSTS")) + maxHosts := len(hosts) + if maxHosts == 0 { + fmt.Print("No HOSTS specified.\n") + os.Exit(1) + } + + var errored bool + if runParallel { + errored = parallel(tests, hosts) + } else { + errored = sequential(tests, hosts) + } + + if errored { + os.Exit(1) + } +} From 23e4459d567cca2fbb13afbb3aa64636b4c1c102 Mon Sep 17 00:00:00 2001 From: Paul Bellamy Date: Fri, 23 Oct 2015 10:49:39 +0100 Subject: [PATCH 2/3] fixing up typos --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 154b8e75b..85fdfeb39 100644 --- a/README.md +++ b/README.md @@ -13,8 +13,8 @@ Included in this repo are tools shared by weave.git and scope.git. They include the Weave network - ```test```: a script to run all go unit tests in subdirectories, gather the coverage results, and merge them into a single report. -- ```runner```: a tools for running tests in parallel; given each test is - suffixes with the number of hosts it requires, and the host availible are +- ```runner```: a tool for running tests in parallel; given each test is + suffixed with the number of hosts it requires, and the hosts available are contained in the environment variable HOSTS, the tool will run tests in parallel, on different hosts. From beb4494499e4fa4c53c006ae299c65b59d81ab82 Mon Sep 17 00:00:00 2001 From: Tom Wilkie Date: Fri, 23 Oct 2015 10:00:45 +0000 Subject: [PATCH 3/3] Review feedback --- runner/runner.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/runner/runner.go b/runner/runner.go index 89b8c6ad1..bfac9c58b 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -19,8 +19,8 @@ import ( ) const ( - schedulerHost = "positive-cocoa-90213.appspot.com" - jsonContentType = "application/json" + defaultSchedulerHost = "positive-cocoa-90213.appspot.com" + jsonContentType = "application/json" ) var ( @@ -29,9 +29,10 @@ var ( succ = ansi.ColorCode("green+b") reset = ansi.ColorCode("reset") - useScheduler = false - runParallel = false - verbose = false + schedulerHost = defaultSchedulerHost + useScheduler = false + runParallel = false + verbose = false consoleLock = sync.Mutex{} ) @@ -241,6 +242,7 @@ func main() { mflag.BoolVar(&useScheduler, []string{"scheduler"}, false, "Use scheduler to distribute tests across shards") mflag.BoolVar(&runParallel, []string{"parallel"}, false, "Run tests in parallel on hosts where possible") mflag.BoolVar(&verbose, []string{"v"}, false, "Print output from all tests (Also enabled via DEBUG=1)") + mflag.StringVar(&schedulerHost, []string{"scheduler-host"}, defaultSchedulerHost, "Hostname of scheduler.") mflag.Parse() if len(os.Getenv("DEBUG")) > 0 {