diff --git a/.gitignore b/.gitignore index 78fe8261e..f5c83426c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ cover/cover socks/proxy socks/image.tar +runner/runner diff --git a/README.md b/README.md index ad801aa00..85fdfeb39 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,36 @@ -# Weaveworks Tools +# Weaveworks Build Tools Included in this repo are tools shared by weave.git and scope.git. They include -- ```cover```: a tool which merges overlapping coverage reports generated by go test -- ```lint```: a script to lint Go project; runs various tools like golint, go vet, errcheck etc -- ```rebuild-image```: a script to rebuild docker images when their input files change; useful - when you using docker images to build your software, but you don't want to build the - image every time. -- ```socks```: a simple, dockerised SOCKS proxy for getting your laptop onto the Weave network -- ```test```: a script to run all go unit tests in subdirectories, gather the coverage results, - and merge them into a single report. +- ```cover```: a tool which merges overlapping coverage reports generated by go + test +- ```lint```: a script to lint Go project; runs various tools like golint, go + vet, errcheck etc +- ```rebuild-image```: a script to rebuild docker images when their input files + change; useful when you using docker images to build your software, but you + don't want to build the image every time. +- ```socks```: a simple, dockerised SOCKS proxy for getting your laptop onto + the Weave network +- ```test```: a script to run all go unit tests in subdirectories, gather the + coverage results, and merge them into a single report. +- ```runner```: a tool for running tests in parallel; given each test is + suffixed with the number of hosts it requires, and the hosts available are + contained in the environment variable HOSTS, the tool will run tests in + parallel, on different hosts. + +## Using build-tools.git + +To allow you to tie your code to a specific version of build-tools.git, such +that future changes don't break you, we recommendation that you [`git subtree`]() +this repository into your own repository: + +[`git subtree`]: http://blogs.atlassian.com/2013/05/alternatives-to-git-submodule-git-subtree/ + +``` +git subtree add --prefix tools https://github.com/weaveworks/build-tools.git master --squash +```` + +To update the code in build-tools.git, the process is therefore: +- PR into build-tools.git, go through normal review process etc. +- Do `git subtree pull --prefix tools https://github.com/weaveworks/build-tools.git master --squash` + in your repo, and PR that. diff --git a/circle.yml b/circle.yml index b79465799..0c1ebddb1 100644 --- a/circle.yml +++ b/circle.yml @@ -19,4 +19,5 @@ test: - cd $SRCDIR; ./lint . - cd $SRCDIR/cover; make - cd $SRCDIR/socks; make + - cd $SRCDIR/runner; make diff --git a/runner/Makefile b/runner/Makefile new file mode 100644 index 000000000..f19bcc7d8 --- /dev/null +++ b/runner/Makefile @@ -0,0 +1,11 @@ +.PHONY: all clean + +all: runner + +runner: *.go + go get -tags netgo ./$(@D) + go build -ldflags "-extldflags \"-static\" -linkmode=external" -tags netgo -o $@ ./$(@D) + +clean: + rm -rf runner + go clean ./... diff --git a/runner/runner.go b/runner/runner.go new file mode 100644 index 000000000..bfac9c58b --- /dev/null +++ b/runner/runner.go @@ -0,0 +1,275 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "net/url" + "os" + "os/exec" + "sort" + "strconv" + "strings" + "sync" + "time" + + "github.com/docker/docker/pkg/mflag" + "github.com/mgutz/ansi" +) + +const ( + defaultSchedulerHost = "positive-cocoa-90213.appspot.com" + jsonContentType = "application/json" +) + +var ( + start = ansi.ColorCode("black+ub") + fail = ansi.ColorCode("red+b") + succ = ansi.ColorCode("green+b") + reset = ansi.ColorCode("reset") + + schedulerHost = defaultSchedulerHost + useScheduler = false + runParallel = false + verbose = false + + consoleLock = sync.Mutex{} +) + +type test struct { + name string + hosts int +} + +type schedule struct { + Tests []string `json:"tests"` +} + +type result struct { + test + errored bool + hosts []string +} + +type tests []test + +func (ts tests) Len() int { return len(ts) } +func (ts tests) Swap(i, j int) { ts[i], ts[j] = ts[j], ts[i] } +func (ts tests) Less(i, j int) bool { + if ts[i].hosts != ts[j].hosts { + return ts[i].hosts < ts[j].hosts + } + return ts[i].name < ts[j].name +} + +func (ts *tests) pick(availible int) (test, bool) { + // pick the first test that fits in the availible hosts + for i, test := range *ts { + if test.hosts <= availible { + *ts = append((*ts)[:i], (*ts)[i+1:]...) + return test, true + } + } + + return test{}, false +} + +func (t test) run(hosts []string) bool { + consoleLock.Lock() + fmt.Printf("%s>>> Running %s on %s%s\n", start, t.name, hosts, reset) + consoleLock.Unlock() + + var out bytes.Buffer + + cmd := exec.Command(t.name) + cmd.Env = os.Environ() + cmd.Stdout = &out + cmd.Stderr = &out + + // replace HOSTS in env + for i, env := range cmd.Env { + if strings.HasPrefix(env, "HOSTS") { + cmd.Env[i] = fmt.Sprintf("HOSTS=%s", strings.Join(hosts, " ")) + break + } + } + + start := time.Now() + err := cmd.Run() + duration := float64(time.Now().Sub(start)) / float64(time.Second) + + consoleLock.Lock() + if err != nil { + fmt.Printf("%s>>> Test %s finished after %0.1f secs with error: %v%s\n", fail, t.name, duration, err, reset) + } else { + fmt.Printf("%s>>> Test %s finished with success after %0.1f secs%s\n", succ, t.name, duration, reset) + } + if err != nil || verbose { + fmt.Print(out.String()) + fmt.Println() + } + consoleLock.Unlock() + + if err != nil && useScheduler { + updateScheduler(t.name, duration) + } + + return err != nil +} + +func updateScheduler(test string, duration float64) { + req := &http.Request{ + Method: "POST", + Host: schedulerHost, + URL: &url.URL{ + Opaque: fmt.Sprintf("/record/%s/%0.2f", url.QueryEscape(test), duration), + Scheme: "http", + Host: schedulerHost, + }, + Close: true, + } + if resp, err := http.DefaultClient.Do(req); err != nil { + fmt.Printf("Error updating scheduler: %v\n", err) + } else { + resp.Body.Close() + } +} + +func getSchedule(tests []string) ([]string, error) { + var ( + testRun = "integration-" + os.Getenv("CIRCLE_BUILD_NUM") + shardCount = os.Getenv("CIRCLE_NODE_TOTAL") + shardID = os.Getenv("CIRCLE_NODE_INDEX") + requestBody = &bytes.Buffer{} + ) + if err := json.NewEncoder(requestBody).Encode(schedule{tests}); err != nil { + return []string{}, err + } + url := fmt.Sprintf("http://%s/schedule/%s/%s/%s", schedulerHost, testRun, shardCount, shardID) + resp, err := http.Post(url, jsonContentType, requestBody) + if err != nil { + return []string{}, err + } + var sched schedule + if err := json.NewDecoder(resp.Body).Decode(&sched); err != nil { + return []string{}, err + } + return sched.Tests, nil +} + +func getTests(testNames []string) (tests, error) { + var err error + if useScheduler { + testNames, err = getSchedule(testNames) + if err != nil { + return tests{}, err + } + } + tests := tests{} + for _, name := range testNames { + parts := strings.Split(strings.TrimSuffix(name, "_test.sh"), "_") + numHosts, err := strconv.Atoi(parts[len(parts)-1]) + if err != nil { + numHosts = 1 + } + tests = append(tests, test{name, numHosts}) + fmt.Printf("Test %s needs %d hosts\n", name, numHosts) + } + return tests, nil +} + +func summary(tests, failed tests) { + if len(failed) > 0 { + fmt.Printf("%s>>> Ran %d tests, %d failed%s\n", fail, len(tests), len(failed), reset) + for _, test := range failed { + fmt.Printf("%s>>> Fail %s%s\n", fail, test.name, reset) + } + } else { + fmt.Printf("%s>>> Ran %d tests, all succeeded%s\n", succ, len(tests), reset) + } +} + +func parallel(ts tests, hosts []string) bool { + testsCopy := ts + sort.Sort(sort.Reverse(ts)) + resultsChan := make(chan result) + outstanding := 0 + failed := tests{} + for len(ts) > 0 || outstanding > 0 { + // While we have some free hosts, try and schedule + // a test on them + for len(hosts) > 0 { + test, ok := ts.pick(len(hosts)) + if !ok { + break + } + testHosts := hosts[:test.hosts] + hosts = hosts[test.hosts:] + + go func() { + errored := test.run(testHosts) + resultsChan <- result{test, errored, testHosts} + }() + outstanding++ + } + + // Otherwise, wait for the test to finish and return + // the hosts to the pool + result := <-resultsChan + hosts = append(hosts, result.hosts...) + outstanding-- + if result.errored { + failed = append(failed, result.test) + } + } + summary(testsCopy, failed) + return len(failed) > 0 +} + +func sequential(ts tests, hosts []string) bool { + failed := tests{} + for _, test := range ts { + if test.run(hosts) { + failed = append(failed, test) + } + } + summary(ts, failed) + return len(failed) > 0 +} + +func main() { + mflag.BoolVar(&useScheduler, []string{"scheduler"}, false, "Use scheduler to distribute tests across shards") + mflag.BoolVar(&runParallel, []string{"parallel"}, false, "Run tests in parallel on hosts where possible") + mflag.BoolVar(&verbose, []string{"v"}, false, "Print output from all tests (Also enabled via DEBUG=1)") + mflag.StringVar(&schedulerHost, []string{"scheduler-host"}, defaultSchedulerHost, "Hostname of scheduler.") + mflag.Parse() + + if len(os.Getenv("DEBUG")) > 0 { + verbose = true + } + + tests, err := getTests(mflag.Args()) + if err != nil { + fmt.Printf("Error parsing tests: %v\n", err) + os.Exit(1) + } + + hosts := strings.Fields(os.Getenv("HOSTS")) + maxHosts := len(hosts) + if maxHosts == 0 { + fmt.Print("No HOSTS specified.\n") + os.Exit(1) + } + + var errored bool + if runParallel { + errored = parallel(tests, hosts) + } else { + errored = sequential(tests, hosts) + } + + if errored { + os.Exit(1) + } +}