From a0d60e4de9013b35542a8aee145ef2e95f3d9eb2 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Wed, 2 Oct 2019 14:34:20 +0000 Subject: [PATCH] Squashed 'tools/' changes from ec369f58d..f041a74ff f041a74ff Undo some quoting that broke the test script (#160) b1c21a068 Merge pull request #158 from weaveworks/go-1-13 d5c7dd0cd Run shell-lint during CI, and fix warnings 6db1abd14 Update to Go 1.13.1 d6cc704a2 Fix comment 7139116ae Revert "Push comments to the left so they don't appear in scripts" e47e58f7b Push comments to the left so they don't appear in scripts 3945fcec8 Remove nonexistent env var GIT_TAG cd6299284 Merge pull request #156 from weaveworks/drop-quay af0eb5119 Merge pull request #157 from weaveworks/fix-image-tag-prefix-length 0b9aee4f2 Fix image-tag object name prefix length to 8 chars. 813c28fe7 Move from CircleCI 1.0 to 2.0 425cf4ef1 Move from quay.io to Dockerhub 87ccf4fd1 Merge pull request #155 from weaveworks/go-1-12 c31bc2865 Update lint script to work with Go 1.12 ed8e380d7 Update to Go 1.12.1 git-subtree-dir: tools git-subtree-split: f041a74ffbf273b627d6c960f17357108d0dbd1c --- .circleci/config.yml | 35 ++++++++++++++++++++++++++++++++ build/Makefile | 2 +- build/golang/Dockerfile | 2 +- circle.yml | 44 ----------------------------------------- cover/Makefile | 2 +- image-tag | 5 ++++- lint | 41 ++++++++++++++++++++++++-------------- runner/Makefile | 2 +- socks/Makefile | 2 +- test | 7 ++----- 10 files changed, 72 insertions(+), 70 deletions(-) create mode 100644 .circleci/config.yml delete mode 100644 circle.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 000000000..2e1a47005 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,35 @@ +version: 2 +jobs: + build: + # Use 'machine' type so we can run the lint step with directory mounted + machine: + docker_layer_caching: true + working_directory: /home/circleci/src/github.com/weaveworks/build-tools + environment: + GOPATH: /home/circleci/ + steps: + - checkout + - run: cd build; make + - run: docker run --rm -v "$PWD:$PWD" -w "$PWD" --entrypoint sh weaveworks/build-golang -c ./lint . + - run: docker run --rm -v "$PWD:$PWD" -w "$PWD" --entrypoint sh weaveworks/build-golang -c ./shell-lint . + - run: docker run --rm -v "$PWD/cover:/go/src/cover" -w "/go/src/cover" --entrypoint sh weaveworks/build-golang -c make + - run: docker run --rm -v "$PWD/socks:/go/src/socks" -w "/go/src/socks" --entrypoint sh weaveworks/build-golang -c "make proxy" + - run: docker run --rm -v "$PWD/runner:/go/src/runner" -w "/go/src/runner" --entrypoint sh weaveworks/build-golang -c make + + - deploy: + command: | + if [ "${CIRCLE_BRANCH}" == "master" ]; then + cd build + docker login -u $DOCKER_REGISTRY_USER -p $DOCKER_REGISTRY_PASSWORD + for image in $(make images); do + # Push all tags - latest and git-tag + docker push "${image}" + + # Tag the built images with something derived from the base images in + # their respective Dockerfiles. So "FROM golang:1.8.0-stretch" as a + # base image would lead to a tag of "1.8.0-stretch" + IMG_TAG=$(make "imagetag-${image#weaveworks/build-}") + docker tag "${image}:latest" "${image}:${IMG_TAG}" + docker push "${image}:${IMG_TAG}" + done + fi diff --git a/build/Makefile b/build/Makefile index a84198c38..5c5db8b50 100644 --- a/build/Makefile +++ b/build/Makefile @@ -3,7 +3,7 @@ # Boiler plate for bulding Docker containers. # All this must go at top of file I'm afraid. -IMAGE_PREFIX := quay.io/weaveworks/build- +IMAGE_PREFIX := weaveworks/build- IMAGE_TAG := $(shell ../image-tag) GIT_REVISION := $(shell git rev-parse HEAD) UPTODATE := .uptodate diff --git a/build/golang/Dockerfile b/build/golang/Dockerfile index 17c19ed3f..df97b2c83 100644 --- a/build/golang/Dockerfile +++ b/build/golang/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.11.1-stretch +FROM golang:1.13.1-stretch RUN apt-get update && \ apt-get install -y \ curl \ diff --git a/circle.yml b/circle.yml deleted file mode 100644 index 68976ff09..000000000 --- a/circle.yml +++ /dev/null @@ -1,44 +0,0 @@ -machine: - services: - - docker - environment: - GOPATH: /home/ubuntu - SRCDIR: /home/ubuntu/src/github.com/weaveworks/tools - PATH: $PATH:$HOME/bin - -dependencies: - post: - - sudo chmod a+wr --recursive /usr/local/go/pkg - - go clean -i net - - go install -tags netgo std - - mkdir -p $(dirname $SRCDIR) - - cp -r $(pwd)/ $SRCDIR - -test: - override: - - | - cd $SRCDIR/build && \ - make && \ - docker run --rm -v "$SRCDIR:$SRCDIR" -w "$SRCDIR" --entrypoint sh quay.io/weaveworks/build-golang -c ./lint . - - cd $SRCDIR/cover; make - - cd $SRCDIR/socks; make - - cd $SRCDIR/runner; make - -deployment: - snapshot: - branch: master - commands: - - docker login -e "$DOCKER_REGISTRY_EMAIL" -u "$DOCKER_REGISTRY_USER" -p "$DOCKER_REGISTRY_PASS" "$DOCKER_REGISTRY_URL" - - | - cd $SRCDIR/build; - for image in $(make images); do - # Tag the built images with the revision of this repo. - docker push "${image}:${GIT_TAG}" - - # Tag the built images with something derived from the base images in - # their respective Dockerfiles. So "FROM golang:1.8.0-stretch" as a - # base image would lead to a tag of "1.8.0-stretch" - IMG_TAG=$(make "imagetag-${image#quay.io/weaveworks/build-}") - docker tag "${image}:latest" "${image}:${IMG_TAG}" - docker push "${image}:${IMG_TAG}" - done diff --git a/cover/Makefile b/cover/Makefile index 1453e63e3..f015a48fa 100644 --- a/cover/Makefile +++ b/cover/Makefile @@ -4,7 +4,7 @@ all: cover cover: *.go go get -tags netgo ./$(@D) - go build -ldflags "-extldflags \"-static\" -linkmode=external" -tags netgo -o $@ ./$(@D) + go build -ldflags "-extldflags \"-static\"" -tags netgo -o $@ ./$(@D) clean: rm -rf cover diff --git a/image-tag b/image-tag index 31f023dac..6a11cb249 100755 --- a/image-tag +++ b/image-tag @@ -6,4 +6,7 @@ set -o pipefail WORKING_SUFFIX=$(if git status --porcelain | grep -qE '^(?:[^?][^ ]|[^ ][^?])\s'; then echo "-WIP"; else echo ""; fi) BRANCH_PREFIX=$(git rev-parse --abbrev-ref HEAD) -echo "${BRANCH_PREFIX//\//-}-$(git rev-parse --short HEAD)$WORKING_SUFFIX" + +# Fix the object name prefix length to 8 characters to have it consistent across the system. +# See https://git-scm.com/docs/git-rev-parse#Documentation/git-rev-parse.txt---shortlength +echo "${BRANCH_PREFIX//\//-}-$(git rev-parse --short=8 HEAD)$WORKING_SUFFIX" diff --git a/lint b/lint index 9db7970d1..72621acbb 100755 --- a/lint +++ b/lint @@ -22,7 +22,6 @@ LINT_IGNORE_FILE=${LINT_IGNORE_FILE:-".lintignore"} IGNORE_LINT_COMMENT= IGNORE_SPELLINGS= -PARALLEL= while true; do case "$1" in -nocomment) @@ -37,10 +36,6 @@ while true; do IGNORE_SPELLINGS="$2,$IGNORE_SPELLINGS" shift 2 ;; - -p) - PARALLEL=1 - shift 1 - ;; *) break ;; @@ -65,6 +60,7 @@ spell_check() { } lint_go() { + # This function is called on a whole directory containing Go files local filename="$1" local lint_result=0 @@ -73,7 +69,7 @@ lint_go() { echo "${filename}: run gofmt -s -w ${filename}" fi - go tool vet "${filename}" || lint_result=$? + go vet "${filename}" || lint_result=$? # golint is completely optional. If you don't like it # don't have it installed. @@ -185,7 +181,7 @@ lint() { case "$mimetype.$ext" in text/x-shellscript.*) lint_sh "${filename}" || lint_result=1 ;; - *.go) lint_go "${filename}" || lint_result=1 ;; + *.go) ;; # done at directory level *.tf) lint_tf "${filename}" || lint_result=1 ;; *.md) lint_md "${filename}" || lint_result=1 ;; *.py) lint_py "${filename}" || lint_result=1 ;; @@ -208,7 +204,7 @@ lint_files() { while read -r filename; do lint "${filename}" || lint_result=1 done - exit $lint_result + return $lint_result } matches_any() { @@ -239,18 +235,33 @@ filter_out() { fi } -list_files() { +lint_directory() { + local dirname="$1" + local lint_result=0 + # This test is just checking if there are any Go files in the directory + if compgen -G "$dirname/*.go" >/dev/null; then + lint_go "${dirname}" || lint_result=1 + fi + find . -maxdepth 1 "$dirname" | filter_out "$LINT_IGNORE_FILE" | lint_files + return $lint_result +} + +lint_directories() { + local lint_result=0 + while read -r dirname; do + lint_directory "${dirname}" || lint_result=1 + done + exit $lint_result +} + +list_directories() { if [ $# -gt 0 ]; then - find "$@" \( -name vendor -o -name .git \) -prune -o -type f - else - git ls-files --exclude-standard | grep -vE '(^|/)vendor/' + find "$@" \( -name vendor -o -name .git -o -name .cache -o -name .pkg \) -prune -o -type d fi } if [ $# = 1 ] && [ -f "$1" ]; then lint "$1" -elif [ -n "$PARALLEL" ]; then - list_files "$@" | filter_out "$LINT_IGNORE_FILE" | xargs -n1 -P16 "$0" else - list_files "$@" | filter_out "$LINT_IGNORE_FILE" | lint_files + list_directories "$@" | lint_directories fi diff --git a/runner/Makefile b/runner/Makefile index f19bcc7d8..1b7f30f45 100644 --- a/runner/Makefile +++ b/runner/Makefile @@ -4,7 +4,7 @@ all: runner runner: *.go go get -tags netgo ./$(@D) - go build -ldflags "-extldflags \"-static\" -linkmode=external" -tags netgo -o $@ ./$(@D) + go build -ldflags "-extldflags \"-static\"" -tags netgo -o $@ ./$(@D) clean: rm -rf runner diff --git a/socks/Makefile b/socks/Makefile index b33586492..4f04ed2a9 100644 --- a/socks/Makefile +++ b/socks/Makefile @@ -21,7 +21,7 @@ $(IMAGE_TAR): Dockerfile $(PROXY_EXE) $(PROXY_EXE): *.go go get -tags netgo ./$(@D) - go build -ldflags "-extldflags \"-static\" -linkmode=external" -tags netgo -o $@ ./$(@D) + go build -ldflags "-extldflags \"-static\"" -tags netgo -o $@ ./$(@D) $(NETGO_CHECK) clean: diff --git a/test b/test index c284e4947..745bd30ab 100755 --- a/test +++ b/test @@ -60,7 +60,7 @@ while [ $# -gt 0 ]; do esac done -GO_TEST_ARGS=(-tags "${TAGS[@]}" -cpu 4 -timeout $TIMEOUT $VERBOSE) +GO_TEST_ARGS=(-tags "${TAGS[@]}" -cpu 4 -timeout "$TIMEOUT" $VERBOSE) if [ -n "$SLOW" ] || [ -n "$CIRCLECI" ]; then SLOW=true @@ -100,9 +100,6 @@ fi PACKAGE_BASE=$(go list -e ./) -# Speed up the tests by compiling and installing their dependencies first. -go test -i "${GO_TEST_ARGS[@]}" "${TESTDIRS[@]}" - run_test() { local dir=$1 if [ -z "$NO_GO_GET" ]; then @@ -118,7 +115,7 @@ run_test() { ) | paste -s -d, -) local output output=$(mktemp "$coverdir/unit.XXXXXXXXXX") - local GO_TEST_ARGS_RUN=("${GO_TEST_ARGS[@]}" -coverprofile=$output -coverpkg=$COVERPKGS) + local GO_TEST_ARGS_RUN=("${GO_TEST_ARGS[@]}" "-coverprofile=$output" "-coverpkg=$COVERPKGS") fi local START