Speed up tests on circle.

- Ensure we use up to date build container when running tests. - Parallelise host setup for integration tests. - Pass some more circle env vars through to ./tools/tests so the scheduler works. - Gather coverage from multiple shards. - Create files as the right owner inside the container. - Parallelise the circle.yaml - Parallelise the intergration tests using existing machinery - Build experimental in parallel. - Use custom scheduler prefix for integration tests, so schedules don't clash with weave.
2026-07-29 02:01:43 +00:00 · 2015-11-03 14:46:07 +00:00
parent ad09d1effc
commit 1ad7b4fce5
9 changed files with 111 additions and 42 deletions
--- a/5
+++ b/5
@@ -104,9 +104,10 @@ clean:
 		$(APP_EXE) $(PROBE_EXE) $(RUNSVINIT) client/build/app.js docker/weave

 ifeq ($(BUILD_IN_CONTAINER),true)
-tests:
+tests: $(SCOPE_BACKEND_BUILD_UPTODATE)
 	$(SUDO) docker run $(RM) -v $(shell pwd):/go/src/github.com/weaveworks/scope \
-		-e GOARCH -e GOOS -e CIRCLECI $(SCOPE_BACKEND_BUILD_IMAGE) tests
+		-e GOARCH -e GOOS -e CIRCLECI -e CIRCLE_BUILD_NUM -e CIRCLE_NODE_TOTAL -e CIRCLE_NODE_INDEX -e COVERDIR\
+		$(SCOPE_BACKEND_BUILD_IMAGE) tests
 else
 tests:
 	./tools/test -no-go-get
--- a/backend/Dockerfile
+++ b/backend/Dockerfile
@@ -1,5 +1,6 @@
 FROM golang:1.5.1
 ENV GO15VENDOREXPERIMENT 1
-RUN apt-get update && apt-get install -y libpcap-dev
+RUN apt-get update && apt-get install -y libpcap-dev python-pip
+RUN pip install requests
 COPY build.sh /
 ENTRYPOINT ["/build.sh"]
--- a/backend/build.sh
+++ b/backend/build.sh
@@ -1,10 +1,20 @@
 #!/bin/sh

-set -eux
+set -eu
+
+SCOPE_SRC=$GOPATH/src/github.com/weaveworks/scope

 # Mount the scope repo:
 #  -v $(pwd):/go/src/github.com/weaveworks/scope

-cd $GOPATH/src/github.com/weaveworks/scope
-make BUILD_IN_CONTAINER=false $@
+# If we run make directly, any files created on the bind mount
+# will have awkward ownership.  So we switch to a user with the
+# same user and group IDs as source directory.  We have to set a
+# few things up so that sudo works without complaining later on.
+uid=$(stat --format="%u" $SCOPE_SRC)
+gid=$(stat --format="%g" $SCOPE_SRC)
+echo "weave:x:$uid:$gid::$SCOPE_SRC:/bin/sh" >>/etc/passwd
+echo "weave:*:::::::" >>/etc/shadow
+echo "weave	ALL=(ALL)	NOPASSWD: ALL" >>/etc/sudoers

+su weave -c "PATH=$PATH make -C $SCOPE_SRC BUILD_IN_CONTAINER=false $*"
--- a/circle.yml
+++ b/circle.yml
@@ -37,23 +37,39 @@ dependencies:

 test:
  override:
-    - cd $SRCDIR; ./tools/lint .
-    - cd $SRCDIR; make RM= tests
-    - cd $SRCDIR; make RM= client-test
-    - cd $SRCDIR; make RM= static
-    - cd $SRCDIR; rm -f app/scope-app probe/scope-probe; GOARCH=arm make RM= app/scope-app probe/scope-probe
-    - cd $SRCDIR; rm -f app/scope-app probe/scope-probe; GOOS=darwin make RM= app/scope-app probe/scope-probe
-    - cd $SRCDIR; rm -f app/scope-app probe/scope-probe; make RM=
-    - cd $SRCDIR/experimental; make RM=
-    - test -z "$SECRET_PASSWORD" || (cd $SRCDIR/integration; ./gce.sh setup)
-    - test -z "$SECRET_PASSWORD" || (cd $SRCDIR/integration; eval $(./gce.sh hosts); ./setup.sh)
+    - cd $SRCDIR; ./tools/lint .:
+        parallel: true
+    - cd $SRCDIR; COVERDIR=./coverage make RM= tests:
+        parallel: true
+    - cd $SRCDIR; make RM= client-test:
+        parallel: true
+    - cd $SRCDIR; make RM= static:
+        parallel: true
+    - cd $SRCDIR; rm -f app/scope-app probe/scope-probe; if [ "$CIRCLE_NODE_INDEX" = "0" ]; then GOARCH=arm make RM= app/scope-app probe/scope-probe; else GOOS=darwin make RM= app/scope-app probe/scope-probe; fi:
+        parallel: true
+    - cd $SRCDIR; rm -f app/scope-app probe/scope-probe; make RM=:
+        parallel: true
+    - cd $SRCDIR/experimental; ./build_on_circle.sh:
+        parallel: true
+    - test -z "$SECRET_PASSWORD" || (cd $SRCDIR/integration; ./gce.sh setup):
+        parallel: true
+    - test -z "$SECRET_PASSWORD" || (cd $SRCDIR/integration; eval $(./gce.sh hosts); ./setup.sh):
+        parallel: true
    - test -z "$SECRET_PASSWORD" || (cd $SRCDIR/integration; eval $(./gce.sh hosts); ./run_all.sh):
        timeout: 300
+        parallel: true
  post:
-    - test -z "$SECRET_PASSWORD" || (cd $SRCDIR/integration; ./gce.sh destroy)
-    - goveralls -repotoken $COVERALLS_REPO_TOKEN -coverprofile=$SRCDIR/profile.cov -service=circleci || true
-    - cd $SRCDIR; cp coverage.html $CIRCLE_ARTIFACTS
-    - cd $SRCDIR; cp scope.tar $CIRCLE_ARTIFACTS
+    - test -z "$SECRET_PASSWORD" || (cd $SRCDIR/integration; ./gce.sh destroy):
+        parallel: true
+
+teardown:
+  pre:
+    - test "$CIRCLE_NODE_INDEX" != "0" || (cd $SRCDIR; ./tools/cover/gather_coverage.sh ./coverage $SRCDIR/coverage):
+        parallel: true
+    - test "$CIRCLE_NODE_INDEX" != "0" || (goveralls -repotoken $COVERALLS_REPO_TOKEN -coverprofile=$SRCDIR/profile.cov -service=circleci || true):
+        parallel: true
+    - test "$CIRCLE_NODE_INDEX" != "0" || (cd $SRCDIR; cp coverage.* scope.tar $CIRCLE_ARTIFACTS):
+        parallel: true

 deployment:
  hub:
--- a/experimental/Makefile
+++ b/experimental/Makefile
@@ -1,7 +1,7 @@
 .PHONY: all test clean

 DIRS=$(shell find . -maxdepth 2 -name *.go -printf "%h\n" | sort -u)
-TARGETS=$(join $(patsubst %,%/,$(DIRS)),$(DIRS))
+TARGETS=$(join $(patsubst %,%/,$(DIRS)),$(patsubst ./%,%,$(DIRS)))
 BUILD_IN_CONTAINER=true
 RM=--rm

--- a/experimental/build_on_circle.sh
+++ b/experimental/build_on_circle.sh
@@ -0,0 +1,5 @@
+#!/bin/sh
+
+set -e
+
+./in_parallel.sh "make RM=" $(find . -maxdepth 2 -name *.go -printf "%h\n" | sort -u | sed -n 's/\.\/\(.*\)/\1\/\1/p')
--- a/experimental/in_parallel.sh
+++ b/experimental/in_parallel.sh
@@ -0,0 +1,31 @@
+#!/bin/sh
+
+set -e
+
+if [ $# -lt 2 ]; then
+    echo "Usage: $0 <cmd> [args...]"
+    echo "  Will run cmd arg1, cmd arg2 etc on different circle shared,"
+    echo "  based on what the scheduler says."
+    exit 2
+fi
+
+if [ -z "$CIRCLECI" ]; then
+    echo "I'm afraid this only works when run on CircleCI"
+    exit 1
+fi
+
+COMMAND=$1
+shift 1
+
+INPUTS="$*"
+INPUTS=$(echo $INPUTS | "../tools/sched" sched parallel-$CIRCLE_BUILD_NUM $CIRCLE_NODE_TOTAL $CIRCLE_NODE_INDEX)
+
+echo Doing $INPUTS
+
+for INPUT in $INPUTS; do
+    START=$(date +%s)
+    $COMMAND $INPUT
+    RUNTIME=$(( $(date +%s) - $START ))
+
+    "../tools/sched" time $INPUT $RUNTIME
+done
--- a/integration/run_all.sh
+++ b/integration/run_all.sh
@@ -4,4 +4,5 @@ set -e

 . ./config.sh

-NO_SCHEDULER=1 "$WEAVE_ROOT/test/run_all.sh" "$@"
+SCHEDULER_PREFIX=scope-integration
+. $WEAVE_ROOT/test/run_all.sh "$@"
--- a/integration/setup.sh
+++ b/integration/setup.sh
@@ -1,32 +1,36 @@
 #!/bin/bash

-set -e
+set -e # NB don't set -u, as weave's config.sh doesn't like that.

 . ./config.sh

 echo Copying scope images and scripts to hosts
 for HOST in $HOSTS; do
-	SIZE=$(stat --printf="%s" ../scope.tar)
-	cat ../scope.tar | pv -N "scope.tar" -s $SIZE | $SSH -C $HOST sudo docker load
-	upload_executable $HOST ../scope
-	upload_executable $HOST ../scope /usr/local/scope/bin/scope
+    SIZE=$(stat --printf="%s" ../scope.tar)
+    cat ../scope.tar | pv -N "scope.tar" -s $SIZE | $SSH -C $HOST sudo docker load
+    upload_executable $HOST ../scope
+    upload_executable $HOST ../scope /usr/local/scope/bin/scope
 done

-echo Installing weave
-# Download the latest released weave script locally,
-# for use by weave_on
-curl -sL git.io/weave -o ./weave
-chmod a+x ./weave
+setup_host() {
+    local HOST=$1
+    echo Installing weave on $HOST
+    # Download the latest released weave script locally,
+    # for use by weave_on
+    curl -sL git.io/weave -o ./weave
+    chmod a+x ./weave
+    run_on $HOST "sudo curl -sL git.io/weave -o /usr/local/bin/weave"
+    run_on $HOST "sudo chmod a+x /usr/local/bin/weave"
+    weave_on $HOST setup
+
+    echo Prefetching Images on $HOST
+    docker_on $HOST pull peterbourgon/tns-db
+    docker_on $HOST pull alpine
+    docker_on $HOST pull nginx
+}
+
 for HOST in $HOSTS; do
-	run_on $HOST "sudo curl -sL git.io/weave -o /usr/local/bin/weave"
-	run_on $HOST "sudo chmod a+x /usr/local/bin/weave"
-	weave_on $HOST setup
-done
-
-echo Prefetching Images
-for HOST in $HOSTS; do
-	docker_on $HOST pull peterbourgon/tns-db
-	docker_on $HOST pull alpine
-	docker_on $HOST pull nginx
+    setup_host $HOST &
 done

+wait