diff --git a/Makefile b/Makefile index a08fb5ec1..fff6d3386 100644 --- a/Makefile +++ b/Makefile @@ -104,9 +104,10 @@ clean: $(APP_EXE) $(PROBE_EXE) $(RUNSVINIT) client/build/app.js docker/weave ifeq ($(BUILD_IN_CONTAINER),true) -tests: +tests: $(SCOPE_BACKEND_BUILD_UPTODATE) $(SUDO) docker run $(RM) -v $(shell pwd):/go/src/github.com/weaveworks/scope \ - -e GOARCH -e GOOS -e CIRCLECI $(SCOPE_BACKEND_BUILD_IMAGE) tests + -e GOARCH -e GOOS -e CIRCLECI -e CIRCLE_BUILD_NUM -e CIRCLE_NODE_TOTAL -e CIRCLE_NODE_INDEX -e COVERDIR\ + $(SCOPE_BACKEND_BUILD_IMAGE) tests else tests: ./tools/test -no-go-get diff --git a/backend/Dockerfile b/backend/Dockerfile index 0bdea51e7..4c02665e3 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,5 +1,6 @@ FROM golang:1.5.1 ENV GO15VENDOREXPERIMENT 1 -RUN apt-get update && apt-get install -y libpcap-dev +RUN apt-get update && apt-get install -y libpcap-dev python-pip +RUN pip install requests COPY build.sh / ENTRYPOINT ["/build.sh"] diff --git a/backend/build.sh b/backend/build.sh index e1402a18a..e45a694c8 100755 --- a/backend/build.sh +++ b/backend/build.sh @@ -1,10 +1,20 @@ #!/bin/sh -set -eux +set -eu + +SCOPE_SRC=$GOPATH/src/github.com/weaveworks/scope # Mount the scope repo: # -v $(pwd):/go/src/github.com/weaveworks/scope -cd $GOPATH/src/github.com/weaveworks/scope -make BUILD_IN_CONTAINER=false $@ +# If we run make directly, any files created on the bind mount +# will have awkward ownership. So we switch to a user with the +# same user and group IDs as source directory. We have to set a +# few things up so that sudo works without complaining later on. +uid=$(stat --format="%u" $SCOPE_SRC) +gid=$(stat --format="%g" $SCOPE_SRC) +echo "weave:x:$uid:$gid::$SCOPE_SRC:/bin/sh" >>/etc/passwd +echo "weave:*:::::::" >>/etc/shadow +echo "weave ALL=(ALL) NOPASSWD: ALL" >>/etc/sudoers +su weave -c "PATH=$PATH make -C $SCOPE_SRC BUILD_IN_CONTAINER=false $*" diff --git a/circle.yml b/circle.yml index 6d546305a..10b74c6db 100644 --- a/circle.yml +++ b/circle.yml @@ -37,23 +37,39 @@ dependencies: test: override: - - cd $SRCDIR; ./tools/lint . - - cd $SRCDIR; make RM= tests - - cd $SRCDIR; make RM= client-test - - cd $SRCDIR; make RM= static - - cd $SRCDIR; rm -f app/scope-app probe/scope-probe; GOARCH=arm make RM= app/scope-app probe/scope-probe - - cd $SRCDIR; rm -f app/scope-app probe/scope-probe; GOOS=darwin make RM= app/scope-app probe/scope-probe - - cd $SRCDIR; rm -f app/scope-app probe/scope-probe; make RM= - - cd $SRCDIR/experimental; make RM= - - test -z "$SECRET_PASSWORD" || (cd $SRCDIR/integration; ./gce.sh setup) - - test -z "$SECRET_PASSWORD" || (cd $SRCDIR/integration; eval $(./gce.sh hosts); ./setup.sh) + - cd $SRCDIR; ./tools/lint .: + parallel: true + - cd $SRCDIR; COVERDIR=./coverage make RM= tests: + parallel: true + - cd $SRCDIR; make RM= client-test: + parallel: true + - cd $SRCDIR; make RM= static: + parallel: true + - cd $SRCDIR; rm -f app/scope-app probe/scope-probe; if [ "$CIRCLE_NODE_INDEX" = "0" ]; then GOARCH=arm make RM= app/scope-app probe/scope-probe; else GOOS=darwin make RM= app/scope-app probe/scope-probe; fi: + parallel: true + - cd $SRCDIR; rm -f app/scope-app probe/scope-probe; make RM=: + parallel: true + - cd $SRCDIR/experimental; ./build_on_circle.sh: + parallel: true + - test -z "$SECRET_PASSWORD" || (cd $SRCDIR/integration; ./gce.sh setup): + parallel: true + - test -z "$SECRET_PASSWORD" || (cd $SRCDIR/integration; eval $(./gce.sh hosts); ./setup.sh): + parallel: true - test -z "$SECRET_PASSWORD" || (cd $SRCDIR/integration; eval $(./gce.sh hosts); ./run_all.sh): timeout: 300 + parallel: true post: - - test -z "$SECRET_PASSWORD" || (cd $SRCDIR/integration; ./gce.sh destroy) - - goveralls -repotoken $COVERALLS_REPO_TOKEN -coverprofile=$SRCDIR/profile.cov -service=circleci || true - - cd $SRCDIR; cp coverage.html $CIRCLE_ARTIFACTS - - cd $SRCDIR; cp scope.tar $CIRCLE_ARTIFACTS + - test -z "$SECRET_PASSWORD" || (cd $SRCDIR/integration; ./gce.sh destroy): + parallel: true + +teardown: + pre: + - test "$CIRCLE_NODE_INDEX" != "0" || (cd $SRCDIR; ./tools/cover/gather_coverage.sh ./coverage $SRCDIR/coverage): + parallel: true + - test "$CIRCLE_NODE_INDEX" != "0" || (goveralls -repotoken $COVERALLS_REPO_TOKEN -coverprofile=$SRCDIR/profile.cov -service=circleci || true): + parallel: true + - test "$CIRCLE_NODE_INDEX" != "0" || (cd $SRCDIR; cp coverage.* scope.tar $CIRCLE_ARTIFACTS): + parallel: true deployment: hub: diff --git a/experimental/Makefile b/experimental/Makefile index f1ff6977a..4e6993352 100644 --- a/experimental/Makefile +++ b/experimental/Makefile @@ -1,7 +1,7 @@ .PHONY: all test clean DIRS=$(shell find . -maxdepth 2 -name *.go -printf "%h\n" | sort -u) -TARGETS=$(join $(patsubst %,%/,$(DIRS)),$(DIRS)) +TARGETS=$(join $(patsubst %,%/,$(DIRS)),$(patsubst ./%,%,$(DIRS))) BUILD_IN_CONTAINER=true RM=--rm diff --git a/experimental/build_on_circle.sh b/experimental/build_on_circle.sh new file mode 100755 index 000000000..d372e51ee --- /dev/null +++ b/experimental/build_on_circle.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +set -e + +./in_parallel.sh "make RM=" $(find . -maxdepth 2 -name *.go -printf "%h\n" | sort -u | sed -n 's/\.\/\(.*\)/\1\/\1/p') diff --git a/experimental/in_parallel.sh b/experimental/in_parallel.sh new file mode 100755 index 000000000..eaccc2f32 --- /dev/null +++ b/experimental/in_parallel.sh @@ -0,0 +1,31 @@ +#!/bin/sh + +set -e + +if [ $# -lt 2 ]; then + echo "Usage: $0 [args...]" + echo " Will run cmd arg1, cmd arg2 etc on different circle shared," + echo " based on what the scheduler says." + exit 2 +fi + +if [ -z "$CIRCLECI" ]; then + echo "I'm afraid this only works when run on CircleCI" + exit 1 +fi + +COMMAND=$1 +shift 1 + +INPUTS="$*" +INPUTS=$(echo $INPUTS | "../tools/sched" sched parallel-$CIRCLE_BUILD_NUM $CIRCLE_NODE_TOTAL $CIRCLE_NODE_INDEX) + +echo Doing $INPUTS + +for INPUT in $INPUTS; do + START=$(date +%s) + $COMMAND $INPUT + RUNTIME=$(( $(date +%s) - $START )) + + "../tools/sched" time $INPUT $RUNTIME +done diff --git a/integration/run_all.sh b/integration/run_all.sh index 91fc91b11..b6cbd3cf0 100755 --- a/integration/run_all.sh +++ b/integration/run_all.sh @@ -4,4 +4,5 @@ set -e . ./config.sh -NO_SCHEDULER=1 "$WEAVE_ROOT/test/run_all.sh" "$@" +SCHEDULER_PREFIX=scope-integration +. $WEAVE_ROOT/test/run_all.sh "$@" diff --git a/integration/setup.sh b/integration/setup.sh index c0a52a536..fd7166702 100755 --- a/integration/setup.sh +++ b/integration/setup.sh @@ -1,32 +1,36 @@ #!/bin/bash -set -e +set -e # NB don't set -u, as weave's config.sh doesn't like that. . ./config.sh echo Copying scope images and scripts to hosts for HOST in $HOSTS; do - SIZE=$(stat --printf="%s" ../scope.tar) - cat ../scope.tar | pv -N "scope.tar" -s $SIZE | $SSH -C $HOST sudo docker load - upload_executable $HOST ../scope - upload_executable $HOST ../scope /usr/local/scope/bin/scope + SIZE=$(stat --printf="%s" ../scope.tar) + cat ../scope.tar | pv -N "scope.tar" -s $SIZE | $SSH -C $HOST sudo docker load + upload_executable $HOST ../scope + upload_executable $HOST ../scope /usr/local/scope/bin/scope done -echo Installing weave -# Download the latest released weave script locally, -# for use by weave_on -curl -sL git.io/weave -o ./weave -chmod a+x ./weave +setup_host() { + local HOST=$1 + echo Installing weave on $HOST + # Download the latest released weave script locally, + # for use by weave_on + curl -sL git.io/weave -o ./weave + chmod a+x ./weave + run_on $HOST "sudo curl -sL git.io/weave -o /usr/local/bin/weave" + run_on $HOST "sudo chmod a+x /usr/local/bin/weave" + weave_on $HOST setup + + echo Prefetching Images on $HOST + docker_on $HOST pull peterbourgon/tns-db + docker_on $HOST pull alpine + docker_on $HOST pull nginx +} + for HOST in $HOSTS; do - run_on $HOST "sudo curl -sL git.io/weave -o /usr/local/bin/weave" - run_on $HOST "sudo chmod a+x /usr/local/bin/weave" - weave_on $HOST setup -done - -echo Prefetching Images -for HOST in $HOSTS; do - docker_on $HOST pull peterbourgon/tns-db - docker_on $HOST pull alpine - docker_on $HOST pull nginx + setup_host $HOST & done +wait