From d9ce1d58e9cbcd67823ee0abe8dd346a9aa6d120 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iago=20L=C3=B3pez=20Galeiras?= Date: Mon, 9 Jan 2017 14:40:34 +0100 Subject: [PATCH] Squashed 'tools/' changes from b990f488..4b7d5c61 4b7d5c61 Merge pull request #59 from weaveworks/57-fix-lint-properly b7f0e692 Merge pull request #58 from weaveworks/fix-lint 794702c7 Pin version of shfmt ab1b11de Fix lint 81d80f35 Merge pull request #55 from weaveworks/lint-tf 05ad5f27 Review feedback 4c0d0469 Use hclfmt to lint terraform. fd875e27 Fix test wrt shellcheck 54ec2d92 Don't capitalise error messages 19d3b6e2 Merge pull request #49 from weaveworks/pin-shfmt fea98f66 Go get from the vendor dir 1d867b06 Try and vendor a specific version of shfmt 76619c2d Merge pull request #48 from weaveworks/revert-41-user-tokens 4f96c519 Revert "Add experimental support for user tokens" d00033fd Merge pull request #41 from weaveworks/user-tokens 245ed267 Merge pull request #47 from weaveworks/46-shfmt c1d7815a Fix shfmt error cb397466 Don't overright lint_result with 0 when shellcheck succeeds 8ab80e87 Merge pull request #45 from weaveworks/lint 83d5bd1f getting integration/config and test shellcheck-compliant cff9ec36 Fix some shellcheck errors 7a843d6d run shellcheck as part of lint if it is installed 31552a0e removing spurious space from test 6ca7c5f0 Merge pull request #44 from weaveworks/shfmt 952356d8 Allow lint to lint itself b7ac59c3 Run shfmt on all shell files in this repo 5570b0e9 Add shfmt formatting of shell files in lint 0a675941 fix circle build by splatting gopath permissions 354e0838 Fixing lint 586060b2 Add experimental support for user tokens git-subtree-dir: tools git-subtree-split: 4b7d5c617e662acb8b1bee4203d7671fb0aa1cba --- circle.yml | 9 +- cmd/wcloud/client.go | 14 +- image-tag | 5 +- integration/assert.sh | 61 +++++---- integration/config.sh | 52 ++++---- integration/gce.sh | 220 ++++++++++++++++--------------- integration/run_all.sh | 8 +- lint | 290 +++++++++++++++++++++++------------------ publish-site | 12 +- rebuild-image | 52 ++++---- socks/connect.sh | 12 +- test | 27 ++-- 12 files changed, 413 insertions(+), 349 deletions(-) mode change 100644 => 100755 integration/assert.sh diff --git a/circle.yml b/circle.yml index 74cf3d40a..84d02dc1a 100644 --- a/circle.yml +++ b/circle.yml @@ -8,11 +8,18 @@ machine: dependencies: post: + - sudo chmod a+wr --recursive /usr/local/go/pkg - go clean -i net - go install -tags netgo std - mkdir -p $(dirname $SRCDIR) - cp -r $(pwd)/ $SRCDIR - - go get github.com/golang/lint/golint github.com/fzipp/gocyclo github.com/kisielk/errcheck + - | + cd $SRCDIR; + go get \ + github.com/fzipp/gocyclo \ + github.com/golang/lint/golint \ + github.com/kisielk/errcheck \ + gopkg.in/mvdan/sh.v1/cmd/shfmt test: override: diff --git a/cmd/wcloud/client.go b/cmd/wcloud/client.go index 57836cd9a..02cbbaa74 100644 --- a/cmd/wcloud/client.go +++ b/cmd/wcloud/client.go @@ -47,7 +47,7 @@ func (c Client) Deploy(deployment Deployment) error { return err } if res.StatusCode != 204 { - return fmt.Errorf("Error making request: %s", res.Status) + return fmt.Errorf("error making request: %s", res.Status) } return nil } @@ -63,7 +63,7 @@ func (c Client) GetDeployments(from, through int64) ([]Deployment, error) { return nil, err } if res.StatusCode != 200 { - return nil, fmt.Errorf("Error making request: %s", res.Status) + return nil, fmt.Errorf("error making request: %s", res.Status) } var response struct { Deployments []Deployment `json:"deployments"` @@ -85,7 +85,7 @@ func (c Client) GetEvents(from, through int64) ([]byte, error) { return nil, err } if res.StatusCode != 200 { - return nil, fmt.Errorf("Error making request: %s", res.Status) + return nil, fmt.Errorf("error making request: %s", res.Status) } return ioutil.ReadAll(res.Body) } @@ -101,10 +101,10 @@ func (c Client) GetConfig() (*Config, error) { return nil, err } if res.StatusCode == 404 { - return nil, fmt.Errorf("No configuration uploaded yet.") + return nil, fmt.Errorf("no configuration uploaded yet") } if res.StatusCode != 200 { - return nil, fmt.Errorf("Error making request: %s", res.Status) + return nil, fmt.Errorf("error making request: %s", res.Status) } var config Config if err := json.NewDecoder(res.Body).Decode(&config); err != nil { @@ -128,7 +128,7 @@ func (c Client) SetConfig(config *Config) error { return err } if res.StatusCode != 204 { - return fmt.Errorf("Error making request: %s", res.Status) + return fmt.Errorf("error making request: %s", res.Status) } return nil } @@ -144,7 +144,7 @@ func (c Client) GetLogs(deployID string) ([]byte, error) { return nil, err } if res.StatusCode != 200 { - return nil, fmt.Errorf("Error making request: %s", res.Status) + return nil, fmt.Errorf("error making request: %s", res.Status) } return ioutil.ReadAll(res.Body) } diff --git a/image-tag b/image-tag index edee35930..d1fd2f728 100755 --- a/image-tag +++ b/image-tag @@ -4,9 +4,6 @@ set -o errexit set -o nounset set -o pipefail -WORKING_SUFFIX=$(if ! git diff --exit-code --quiet HEAD >&2; \ - then echo "-WIP"; \ - else echo ""; \ - fi) +WORKING_SUFFIX=$(if ! git diff --exit-code --quiet HEAD >&2; then echo "-WIP"; else echo ""; fi) BRANCH_PREFIX=$(git rev-parse --abbrev-ref HEAD) echo "${BRANCH_PREFIX//\//-}-$(git rev-parse --short HEAD)$WORKING_SUFFIX" diff --git a/integration/assert.sh b/integration/assert.sh old mode 100644 new mode 100755 index da689b3eb..200b393bc --- a/integration/assert.sh +++ b/integration/assert.sh @@ -25,14 +25,15 @@ export CONTINUE=${CONTINUE:-} args="$(getopt -n "$0" -l \ verbose,help,stop,discover,invariant,continue vhxdic "$@")" \ -|| exit -1 + || exit -1 for arg in $args; do case "$arg" in -h) echo "$0 [-vxidc]" \ "[--verbose] [--stop] [--invariant] [--discover] [--continue]" - echo "$(sed 's/./ /g' <<< "$0") [-h] [--help]" - exit 0;; + echo "$(sed 's/./ /g' <<<"$0") [-h] [--help]" + exit 0 + ;; --help) cat < [stdin] - (( tests_ran++ )) || : + ((tests_ran++)) || : [[ -z "$DISCOVERONLY" ]] || return expected=$(echo -ne "${2:-}") - result="$(eval 2>/dev/null "$1" <<< "${3:-}")" || true + result="$(eval "$1" 2>/dev/null <<<"${3:-}")" || true if [[ "$result" == "$expected" ]]; then [[ -z "$DEBUG" ]] || echo -n . return fi - result="$(sed -e :a -e '$!N;s/\n/\\n/;ta' <<< "$result")" + result="$(sed -e :a -e '$!N;s/\n/\\n/;ta' <<<"$result")" [[ -z "$result" ]] && result="nothing" || result="\"$result\"" [[ -z "$2" ]] && expected="nothing" || expected="\"$2\"" _assert_fail "expected $expected${_indent}got $result" "$1" "$3" @@ -116,10 +124,10 @@ assert() { assert_raises() { # assert_raises [stdin] - (( tests_ran++ )) || : + ((tests_ran++)) || : [[ -z "$DISCOVERONLY" ]] || return status=0 - (eval "$1" <<< "${3:-}") > /dev/null 2>&1 || status=$? + (eval "$1" <<<"${3:-}") >/dev/null 2>&1 || status=$? expected=${2:-0} if [[ "$status" -eq "$expected" ]]; then [[ -z "$DEBUG" ]] || echo -n . @@ -138,12 +146,12 @@ _assert_fail() { exit 1 fi tests_errors[$tests_failed]="$report" - (( tests_failed++ )) || : + ((tests_failed++)) || : } skip_if() { # skip_if - (eval "$@") > /dev/null 2>&1 && status=0 || status=$? + (eval "$@") >/dev/null 2>&1 && status=0 || status=$? [[ "$status" -eq 0 ]] || return skip } @@ -175,9 +183,8 @@ _skip() { fi } - _assert_reset -: ${tests_suite_status:=0} # remember if any of the tests failed so far +: ${tests_suite_status:=0} # remember if any of the tests failed so far _assert_cleanup() { local status=$? # modify exit code if it's not already non-zero diff --git a/integration/config.sh b/integration/config.sh index 3d7a8dafe..9bf47ea44 100644 --- a/integration/config.sh +++ b/integration/config.sh @@ -1,3 +1,4 @@ +#!/bin/bash # NB only to be sourced set -e @@ -16,49 +17,55 @@ N_MACHINES=${N_MACHINES:-3} IP_PREFIX=${IP_PREFIX:-192.168.48} IP_SUFFIX_BASE=${IP_SUFFIX_BASE:-10} -if [ -z "$HOSTS" ] ; then - for i in $(seq 1 $N_MACHINES); do - IP="${IP_PREFIX}.$((${IP_SUFFIX_BASE}+$i))" +if [ -z "$HOSTS" ]; then + for i in $(seq 1 "$N_MACHINES"); do + IP="${IP_PREFIX}.$((IP_SUFFIX_BASE + i))" HOSTS="$HOSTS $IP" done fi # these are used by the tests -HOST1=$(echo $HOSTS | cut -f 1 -d ' ') -HOST2=$(echo $HOSTS | cut -f 2 -d ' ') -HOST3=$(echo $HOSTS | cut -f 3 -d ' ') +# shellcheck disable=SC2034 +HOST1=$(echo "$HOSTS" | cut -f 1 -d ' ') +# shellcheck disable=SC2034 +HOST2=$(echo "$HOSTS" | cut -f 2 -d ' ') +# shellcheck disable=SC2034 +HOST3=$(echo "$HOSTS" | cut -f 3 -d ' ') +# shellcheck disable=SC1090 . "$DIR/assert.sh" SSH_DIR=${SSH_DIR:-$DIR} -SSH=${SSH:-ssh -l vagrant -i "$SSH_DIR/insecure_private_key" -o "UserKnownHostsFile=$SSH_DIR/.ssh_known_hosts" -o CheckHostIP=no -o StrictHostKeyChecking=no} +SSH=${SSH:-ssh -l vagrant -i \"$SSH_DIR/insecure_private_key\" -o \"UserKnownHostsFile=$SSH_DIR/.ssh_known_hosts\" -o CheckHostIP=no -o StrictHostKeyChecking=no} SMALL_IMAGE="alpine" +# shellcheck disable=SC2034 TEST_IMAGES="$SMALL_IMAGE" +# shellcheck disable=SC2034 PING="ping -nq -W 1 -c 1" DOCKER_PORT=2375 remote() { rem=$1 shift 1 - "$@" > >(while read line; do echo -e $'\e[0;34m'"$rem>"$'\e[0m'" $line"; done) + "$@" > >(while read -r line; do echo -e $'\e[0;34m'"$rem>"$'\e[0m'" $line"; done) } colourise() { - [ -t 0 ] && echo -ne $'\e['$1'm' || true + ([ -t 0 ] && echo -ne $'\e['"$1"'m') || true shift # It's important that we don't do this in a subshell, as some # commands we execute need to modify global state "$@" - [ -t 0 ] && echo -ne $'\e[0m' || true + ([ -t 0 ] && echo -ne $'\e[0m') || true } whitely() { colourise '1;37' "$@" } -greyly () { +greyly() { colourise '0;37' "$@" } @@ -73,21 +80,21 @@ greenly() { run_on() { host=$1 shift 1 - [ -z "$DEBUG" ] || greyly echo "Running on $host: $@" >&2 - remote $host $SSH $host "$@" + [ -z "$DEBUG" ] || greyly echo "Running on $host:" "$@" >&2 + remote "$host" "$SSH" "$host" "$@" } docker_on() { host=$1 shift 1 - [ -z "$DEBUG" ] || greyly echo "Docker on $host:$DOCKER_PORT: $@" >&2 - docker -H tcp://$host:$DOCKER_PORT "$@" + [ -z "$DEBUG" ] || greyly echo "Docker on $host:$DOCKER_PORT:" "$@" >&2 + docker -H "tcp://$host:$DOCKER_PORT" "$@" } weave_on() { host=$1 shift 1 - [ -z "$DEBUG" ] || greyly echo "Weave on $host:$DOCKER_PORT: $@" >&2 + [ -z "$DEBUG" ] || greyly echo "Weave on $host:$DOCKER_PORT:" "$@" >&2 DOCKER_HOST=tcp://$host:$DOCKER_PORT $WEAVE "$@" } @@ -95,24 +102,24 @@ exec_on() { host=$1 container=$2 shift 2 - docker -H tcp://$host:$DOCKER_PORT exec $container "$@" + docker -H "tcp://$host:$DOCKER_PORT" exec "$container" "$@" } rm_containers() { host=$1 shift - [ $# -eq 0 ] || docker_on $host rm -f "$@" >/dev/null + [ $# -eq 0 ] || docker_on "$host" rm -f "$@" >/dev/null } start_suite() { for host in $HOSTS; do [ -z "$DEBUG" ] || echo "Cleaning up on $host: removing all containers and resetting weave" - PLUGIN_ID=$(docker_on $host ps -aq --filter=name=weaveplugin) + PLUGIN_ID=$(docker_on "$host" ps -aq --filter=name=weaveplugin) PLUGIN_FILTER="cat" [ -n "$PLUGIN_ID" ] && PLUGIN_FILTER="grep -v $PLUGIN_ID" - rm_containers $host $(docker_on $host ps -aq 2>/dev/null | $PLUGIN_FILTER) - run_on $host "docker network ls | grep -q ' weave ' && docker network rm weave" || true - weave_on $host reset 2>/dev/null + rm_containers "$host" "$(docker_on "$host" ps -aq 2>/dev/null | "$PLUGIN_FILTER")" + run_on "$host" "docker network ls | grep -q ' weave ' && docker network rm weave" || true + weave_on "$host" reset 2>/dev/null done whitely echo "$@" } @@ -122,4 +129,3 @@ end_suite() { } WEAVE=$DIR/../weave - diff --git a/integration/gce.sh b/integration/gce.sh index dd0e92770..5129916ed 100755 --- a/integration/gce.sh +++ b/integration/gce.sh @@ -16,68 +16,73 @@ set -e : "${NUM_HOSTS:=}" if [ -z "${PROJECT}" ] || [ -z "${NUM_HOSTS}" ] || [ -z "${TEMPLATE_NAME}" ]; then - echo "Must specify PROJECT, NUM_HOSTS and TEMPLATE_NAME" - exit 1 + echo "Must specify PROJECT, NUM_HOSTS and TEMPLATE_NAME" + exit 1 fi SUFFIX="" if [ -n "$CIRCLECI" ]; then - SUFFIX="-${CIRCLE_BUILD_NUM}-$CIRCLE_NODE_INDEX" + SUFFIX="-${CIRCLE_BUILD_NUM}-$CIRCLE_NODE_INDEX" fi # Setup authentication gcloud auth activate-service-account --key-file "$KEY_FILE" 1>/dev/null gcloud config set project "$PROJECT" -function vm_names { - local names= - for i in $(seq 1 "$NUM_HOSTS"); do - names=( "host$i$SUFFIX" "${names[@]}" ) - done - echo "${names[@]}" +function vm_names() { + local names= + for i in $(seq 1 "$NUM_HOSTS"); do + names=("host$i$SUFFIX" "${names[@]}") + done + echo "${names[@]}" } # Delete all vms in this account -function destroy { - local names - names="$(vm_names)" - if [ "$(gcloud compute instances list --zone "$ZONE" -q "$names" | wc -l)" -le 1 ] ; then - return 0 - fi - for i in {0..10}; do - # gcloud instances delete can sometimes hang. - case $(set +e; timeout 60s /bin/bash -c "gcloud compute instances delete --zone $ZONE -q $names >/dev/null 2>&1"; echo $?) in - 0) - return 0 - ;; - 124) - # 124 means it timed out - break - ;; - *) - return 1 - esac - done +function destroy() { + local names + names="$(vm_names)" + if [ "$(gcloud compute instances list --zone "$ZONE" -q "$names" | wc -l)" -le 1 ]; then + return 0 + fi + for i in {0..10}; do + # gcloud instances delete can sometimes hang. + case $( + set +e + timeout 60s /bin/bash -c "gcloud compute instances delete --zone $ZONE -q $names >/dev/null 2>&1" + echo $? + ) in + 0) + return 0 + ;; + 124) + # 124 means it timed out + break + ;; + *) + return 1 + ;; + esac + done } -function internal_ip { - jq -r ".[] | select(.name == \"$2\") | .networkInterfaces[0].networkIP" "$1" +function internal_ip() { + jq -r ".[] | select(.name == \"$2\") | .networkInterfaces[0].networkIP" "$1" } -function external_ip { - jq -r ".[] | select(.name == \"$2\") | .networkInterfaces[0].accessConfigs[0].natIP" "$1" +function external_ip() { + jq -r ".[] | select(.name == \"$2\") | .networkInterfaces[0].accessConfigs[0].natIP" "$1" } -function try_connect { - for i in {0..10}; do - ssh -t "$1" true && return - sleep 2 - done +function try_connect() { + for i in {0..10}; do + ssh -t "$1" true && return + sleep 2 + done } -function install_docker_on { - name=$1 - ssh -t "$name" sudo bash -x -s <> /etc/default/docker; service docker restart EOF - # It seems we need a short delay for docker to start up, so I put this in - # a separate ssh connection. This installs nsenter. - ssh -t "$name" sudo docker run --rm -v /usr/local/bin:/target jpetazzo/nsenter + # It seems we need a short delay for docker to start up, so I put this in + # a separate ssh connection. This installs nsenter. + ssh -t "$name" sudo docker run --rm -v /usr/local/bin:/target jpetazzo/nsenter } -function copy_hosts { - hostname=$1 - hosts=$2 - ssh -t "$hostname" "sudo -- sh -c \"cat >>/etc/hosts\"" < "$hosts" +function copy_hosts() { + hostname=$1 + hosts=$2 + ssh -t "$hostname" "sudo -- sh -c \"cat >>/etc/hosts\"" <"$hosts" } # Create new set of VMs -function setup { - destroy +function setup() { + destroy - names=( $(vm_names) ) - gcloud compute instances create "${names[@]}" --image "$TEMPLATE_NAME" --zone "$ZONE" - gcloud compute config-ssh --ssh-key-file "$SSH_KEY_FILE" - sed -i '/UserKnownHostsFile=\/dev\/null/d' ~/.ssh/config + names=($(vm_names)) + gcloud compute instances create "${names[@]}" --image "$TEMPLATE_NAME" --zone "$ZONE" + gcloud compute config-ssh --ssh-key-file "$SSH_KEY_FILE" + sed -i '/UserKnownHostsFile=\/dev\/null/d' ~/.ssh/config - # build an /etc/hosts file for these vms - hosts=$(mktemp hosts.XXXXXXXXXX) - json=$(mktemp json.XXXXXXXXXX) - gcloud compute instances list --format=json > "$json" - for name in "${names[@]}"; do - echo "$(internal_ip "$json" "$name") $name.$ZONE.$PROJECT" >> "$hosts" - done + # build an /etc/hosts file for these vms + hosts=$(mktemp hosts.XXXXXXXXXX) + json=$(mktemp json.XXXXXXXXXX) + gcloud compute instances list --format=json >"$json" + for name in "${names[@]}"; do + echo "$(internal_ip "$json" "$name") $name.$ZONE.$PROJECT" >>"$hosts" + done - for name in "${names[@]}"; do - hostname="$name.$ZONE.$PROJECT" + for name in "${names[@]}"; do + hostname="$name.$ZONE.$PROJECT" - # Add the remote ip to the local /etc/hosts - sudo sed -i "/$hostname/d" /etc/hosts - sudo sh -c "echo \"$(external_ip "$json" "$name") $hostname\" >>/etc/hosts" - try_connect "$hostname" + # Add the remote ip to the local /etc/hosts + sudo sed -i "/$hostname/d" /etc/hosts + sudo sh -c "echo \"$(external_ip "$json" "$name") $hostname\" >>/etc/hosts" + try_connect "$hostname" - copy_hosts "$hostname" "$hosts" & - done + copy_hosts "$hostname" "$hosts" & + done - wait + wait - rm "$hosts" "$json" + rm "$hosts" "$json" } -function make_template { - gcloud compute instances create "$TEMPLATE_NAME" --image "$IMAGE" --zone "$ZONE" - gcloud compute config-ssh --ssh-key-file "$SSH_KEY_FILE" - name="$TEMPLATE_NAME.$ZONE.$PROJECT" - try_connect "$name" - install_docker_on "$name" - gcloud -q compute instances delete "$TEMPLATE_NAME" --keep-disks boot --zone "$ZONE" - gcloud compute images create "$TEMPLATE_NAME" --source-disk "$TEMPLATE_NAME" --source-disk-zone "$ZONE" +function make_template() { + gcloud compute instances create "$TEMPLATE_NAME" --image "$IMAGE" --zone "$ZONE" + gcloud compute config-ssh --ssh-key-file "$SSH_KEY_FILE" + name="$TEMPLATE_NAME.$ZONE.$PROJECT" + try_connect "$name" + install_docker_on "$name" + gcloud -q compute instances delete "$TEMPLATE_NAME" --keep-disks boot --zone "$ZONE" + gcloud compute images create "$TEMPLATE_NAME" --source-disk "$TEMPLATE_NAME" --source-disk-zone "$ZONE" } -function hosts { - hosts= - args= - json=$(mktemp json.XXXXXXXXXX) - gcloud compute instances list --format=json > "$json" - for name in $(vm_names); do - hostname="$name.$ZONE.$PROJECT" - hosts=( $hostname "${hosts[@]}" ) - args=( "--add-host=$hostname:$(internal_ip "$json" "$name")" "${args[@]}" ) - done - echo export SSH=\"ssh -l vagrant\" - echo "export HOSTS=\"${hosts[*]}\"" - echo "export ADD_HOST_ARGS=\"${args[*]}\"" - rm "$json" +function hosts() { + hosts= + args= + json=$(mktemp json.XXXXXXXXXX) + gcloud compute instances list --format=json >"$json" + for name in $(vm_names); do + hostname="$name.$ZONE.$PROJECT" + hosts=($hostname "${hosts[@]}") + args=("--add-host=$hostname:$(internal_ip "$json" "$name")" "${args[@]}") + done + echo export SSH=\"ssh -l vagrant\" + echo "export HOSTS=\"${hosts[*]}\"" + echo "export ADD_HOST_ARGS=\"${args[*]}\"" + rm "$json" } case "$1" in -setup) - setup - ;; + setup) + setup + ;; -hosts) - hosts - ;; + hosts) + hosts + ;; -destroy) - destroy - ;; + destroy) + destroy + ;; -make_template) - # see if template exists - if ! gcloud compute images list | grep "$PROJECT" | grep "$TEMPLATE_NAME"; then - make_template - fi + make_template) + # see if template exists + if ! gcloud compute images list | grep "$PROJECT" | grep "$TEMPLATE_NAME"; then + make_template + fi + ;; esac diff --git a/integration/run_all.sh b/integration/run_all.sh index cb6d93d2c..837e0d497 100755 --- a/integration/run_all.sh +++ b/integration/run_all.sh @@ -14,17 +14,17 @@ fi whitely echo ...ok # shellcheck disable=SC2068 -TESTS=( ${@:-$(find . -name '*_test.sh')} ) -RUNNER_ARGS=( ) +TESTS=(${@:-$(find . -name '*_test.sh')}) +RUNNER_ARGS=() # If running on circle, use the scheduler to work out what tests to run if [ -n "$CIRCLECI" ] && [ -z "$NO_SCHEDULER" ]; then - RUNNER_ARGS=( "${RUNNER_ARGS[@]}" -scheduler ) + RUNNER_ARGS=("${RUNNER_ARGS[@]}" -scheduler) fi # If running on circle or PARALLEL is not empty, run tests in parallel if [ -n "$CIRCLECI" ] || [ -n "$PARALLEL" ]; then - RUNNER_ARGS=( "${RUNNER_ARGS[@]}" -parallel ) + RUNNER_ARGS=("${RUNNER_ARGS[@]}" -parallel) fi make -C "${DIR}/../runner" diff --git a/lint b/lint index 30e8f904e..566e329b6 100755 --- a/lint +++ b/lint @@ -1,9 +1,13 @@ #!/bin/bash -# This scipt lints go files for common errors. +# This scipt lints files for common errors. # -# It runs gofmt and go vet, and optionally golint and +# For go files, it runs gofmt and go vet, and optionally golint and # gocyclo, if they are installed. # +# For shell files, it runs shfmt. If you don't have that installed, you can get +# it with: +# go get -u github.com/mvdan/sh/cmd/shfmt +# # With no arguments, it lints the current files staged # for git commit. Or you can pass it explicit filenames # (or directories) and it will lint them. @@ -17,159 +21,193 @@ IGNORE_LINT_COMMENT= IGNORE_TEST_PACKAGES= IGNORE_SPELLINGS= while true; do - case "$1" in - -nocomment) - IGNORE_LINT_COMMENT=1 - shift 1 - ;; - -notestpackage) - IGNORE_TEST_PACKAGES=1 - shift 1 - ;; + case "$1" in + -nocomment) + IGNORE_LINT_COMMENT=1 + shift 1 + ;; + -notestpackage) + IGNORE_TEST_PACKAGES=1 + shift 1 + ;; -ignorespelling) - IGNORE_SPELLINGS="$2,$IGNORE_SPELLINGS" - shift 2 - ;; - *) - break - esac + IGNORE_SPELLINGS="$2,$IGNORE_SPELLINGS" + shift 2 + ;; + *) + break + ;; + esac done +spell_check() { + local filename="$1" + local lint_result=0 -function spell_check { - filename="$1" - local lint_result=0 + # we don't want to spell check tar balls, binaries, Makefile and json files + if file "$filename" | grep executable >/dev/null 2>&1; then + return $lint_result + fi + if [[ $filename == *".tar" || $filename == *".gz" || $filename == *".json" || $(basename "$filename") == "Makefile" ]]; then + return $lint_result + fi - # we don't want to spell check tar balls, binaries, Makefile and json files - if file "$filename" | grep executable >/dev/null 2>&1; then - return $lint_result - fi - if [[ $filename == *".tar" || $filename == *".gz" || $filename == *".json" || $(basename "$filename") == "Makefile" ]]; then - return $lint_result - fi + # misspell is completely optional. If you don't like it + # don't have it installed. + if ! type misspell >/dev/null 2>&1; then + return $lint_result + fi - # misspell is completely optional. If you don't like it - # don't have it installed. - if ! type misspell >/dev/null 2>&1; then - return $lint_result - fi + if ! misspell -error -i "$IGNORE_SPELLINGS" "${filename}"; then + lint_result=1 + fi - if ! misspell -error -i "$IGNORE_SPELLINGS" "${filename}"; then - lint_result=1 - fi - - return $lint_result + return $lint_result } -function test_mismatch { - filename="$1" - package=$(grep '^package ' "$filename" | awk '{print $2}') - local lint_result=0 +test_mismatch() { + local filename="$1" + local package=$(grep '^package ' "$filename" | awk '{print $2}') + local lint_result=0 - if [[ $package == "main" ]]; then - return # in package main, all bets are off - fi + if [[ $package == "main" ]]; then + return # in package main, all bets are off + fi - if [[ $filename == *"_internal_test.go" ]]; then - if [[ $package == *"_test" ]]; then - lint_result=1 - echo "${filename}: should not be part of a _test package" - fi - else - if [[ ! $package == *"_test" ]]; then - lint_result=1 - echo "${filename}: should be part of a _test package" - fi - fi + if [[ $filename == *"_internal_test.go" ]]; then + if [[ $package == *"_test" ]]; then + lint_result=1 + echo "${filename}: should not be part of a _test package" + fi + else + if [[ ! $package == *"_test" ]]; then + lint_result=1 + echo "${filename}: should be part of a _test package" + fi + fi - return $lint_result + return $lint_result } -function lint_go { - filename="$1" - local lint_result=0 +lint_go() { + local filename="$1" + local lint_result=0 - if [ -n "$(gofmt -s -l "${filename}")" ]; then - lint_result=1 - echo "${filename}: run gofmt -s -w ${filename}!" - fi + if [ -n "$(gofmt -s -l "${filename}")" ]; then + lint_result=1 + echo "${filename}: run gofmt -s -w ${filename}" + fi - go tool vet "${filename}" || lint_result=$? + go tool vet "${filename}" || lint_result=$? - # golint is completely optional. If you don't like it - # don't have it installed. - if type golint >/dev/null 2>&1; then - # golint doesn't set an exit code it seems - if [ -z "$IGNORE_LINT_COMMENT" ]; then - lintoutput=$(golint "${filename}") - else - lintoutput=$(golint "${filename}" | grep -vE 'comment|dot imports|ALL_CAPS') - fi - if [ -n "$lintoutput" ]; then - lint_result=1 - echo "$lintoutput" - fi - fi + # golint is completely optional. If you don't like it + # don't have it installed. + if type golint >/dev/null 2>&1; then + # golint doesn't set an exit code it seems + if [ -z "$IGNORE_LINT_COMMENT" ]; then + lintoutput=$(golint "${filename}") + else + lintoutput=$(golint "${filename}" | grep -vE 'comment|dot imports|ALL_CAPS') + fi + if [ -n "$lintoutput" ]; then + lint_result=1 + echo "$lintoutput" + fi + fi - # gocyclo is completely optional. If you don't like it - # don't have it installed. Also never blocks a commit, - # it just warns. - if type gocyclo >/dev/null 2>&1; then - gocyclo -over 25 "${filename}" | while read -r line; do - echo "${filename}": higher than 25 cyclomatic complexity - "${line}" - done - fi + # gocyclo is completely optional. If you don't like it + # don't have it installed. Also never blocks a commit, + # it just warns. + if type gocyclo >/dev/null 2>&1; then + gocyclo -over 25 "${filename}" | while read -r line; do + echo "${filename}": higher than 25 cyclomatic complexity - "${line}" + done + fi - return $lint_result + return $lint_result } -function lint { - filename="$1" - ext="${filename##*\.}" - local lint_result=0 +lint_sh() { + local filename="$1" + local lint_result=0 - # Don't lint deleted files - if [ ! -f "$filename" ]; then - return - fi + if ! diff <(shfmt -i 4 "${filename}") "${filename}" >/dev/null; then + lint_result=1 + echo "${filename}: run shfmt -i 4 -w ${filename}" + fi - # Don't lint this script or static.go - case "$(basename "${filename}")" in - lint) return;; - static.go) return;; - coverage.html) return;; - esac + # the shellcheck is completely optional. If you don't like it + # don't have it installed. + if type shellcheck >/dev/null 2>&1; then + shellcheck "${filename}" || lint_result=1 + fi - case "$ext" in - go) lint_go "${filename}" || lint_result=1 - ;; - esac - - if [ -z "$IGNORE_TEST_PACKAGES" ]; then - if [[ "$filename" == *"_test.go" ]]; then - test_mismatch "${filename}" || lint_result=1 - fi - fi - - spell_check "${filename}" || lint_result=1 - - return $lint_result + return $lint_result } -function lint_files { - local lint_result=0 - while read -r filename; do - lint "${filename}" || lint_result=1 - done - exit $lint_result +lint_tf() { + local filename="$1" + local lint_result=0 + + if ! diff <(hclfmt "${filename}") "${filename}" >/dev/null; then + lint_result=1 + echo "${filename}: run hclfmt -w ${filename}" + fi + + return $lint_result } -function list_files { - if [ $# -gt 0 ]; then +lint() { + filename="$1" + ext="${filename##*\.}" + local lint_result=0 + + # Don't lint deleted files + if [ ! -f "$filename" ]; then + return + fi + + # Don't lint static.go + case "$(basename "${filename}")" in + static.go) return ;; + coverage.html) return ;; + esac + + if [[ "$(file --mime-type "${filename}" | awk '{print $2}')" == "text/x-shellscript" ]]; then + ext="sh" + fi + + case "$ext" in + go) lint_go "${filename}" || lint_result=1 ;; + sh) lint_sh "${filename}" || lint_result=1 ;; + tf) lint_tf "${filename}" || lint_result=1 ;; + esac + + if [ -z "$IGNORE_TEST_PACKAGES" ]; then + if [[ "$filename" == *"_test.go" ]]; then + test_mismatch "${filename}" || lint_result=1 + fi + fi + + spell_check "${filename}" || lint_result=1 + + return $lint_result +} + +lint_files() { + local lint_result=0 + while read -r filename; do + lint "${filename}" || lint_result=1 + done + exit $lint_result +} + +list_files() { + if [ $# -gt 0 ]; then git ls-files --exclude-standard | grep -vE '(^|/)vendor/' - else - git diff --cached --name-only - fi + else + git diff --cached --name-only + fi } list_files "$@" | lint_files diff --git a/publish-site b/publish-site index 900eb7cb6..107604045 100755 --- a/publish-site +++ b/publish-site @@ -10,15 +10,15 @@ fatal() { exit 1 } -if [ ! -d .git ] ; then +if [ ! -d .git ]; then fatal "Current directory is not a git clone" fi if [ -z "${PRODUCT}" ]; then - fatal "Must specify PRODUCT" + fatal "Must specify PRODUCT" fi -if ! BRANCH=$(git symbolic-ref --short HEAD) || [ -z "$BRANCH" ] ; then +if ! BRANCH=$(git symbolic-ref --short HEAD) || [ -z "$BRANCH" ]; then fatal "Could not determine branch" fi @@ -28,9 +28,9 @@ case "$BRANCH" in TAGS="$VERSION" ;; *) - if echo "$BRANCH" | grep -qE '^[0-9]+\.[0-9]+' ; then + if echo "$BRANCH" | grep -qE '^[0-9]+\.[0-9]+'; then DESCRIBE=$(git describe --match 'v*') - if ! VERSION=$(echo "$DESCRIBE" | grep -oP '(?<=^v)[0-9]+\.[0-9]+\.[0-9]+') ; then + if ! VERSION=$(echo "$DESCRIBE" | grep -oP '(?<=^v)[0-9]+\.[0-9]+\.[0-9]+'); then fatal "Could not infer latest $BRANCH version from $DESCRIBE" fi TAGS="$VERSION latest" @@ -41,7 +41,7 @@ case "$BRANCH" in ;; esac -for TAG in $TAGS ; do +for TAG in $TAGS; do echo ">>> Publishing $PRODUCT $VERSION to $1/docs/$PRODUCT/$TAG" wordepress \ --url "$1" --user "$2" --password "$3" \ diff --git a/rebuild-image b/rebuild-image index bc1360bee..1f0bb1091 100755 --- a/rebuild-image +++ b/rebuild-image @@ -10,7 +10,7 @@ SAVEDNAME=$(echo "$IMAGENAME" | sed "s/[\/\-]/\./g") IMAGEDIR=$2 shift 2 -INPUTFILES=( "$@" ) +INPUTFILES=("$@") CACHEDIR=$HOME/docker/ # Rebuild the image @@ -18,60 +18,60 @@ rebuild() { mkdir -p "$CACHEDIR" rm "$CACHEDIR/$SAVEDNAME"* || true docker build -t "$IMAGENAME" "$IMAGEDIR" - docker save "$IMAGENAME:latest" | gzip - > "$CACHEDIR/$SAVEDNAME-$CIRCLE_SHA1.gz" + docker save "$IMAGENAME:latest" | gzip - >"$CACHEDIR/$SAVEDNAME-$CIRCLE_SHA1.gz" } # Get the revision the cached image was build at cached_image_rev() { - find "$CACHEDIR" -name "$SAVEDNAME-*" -type f | sed -n 's/^[^\-]*\-\([a-z0-9]*\).gz$/\1/p' + find "$CACHEDIR" -name "$SAVEDNAME-*" -type f | sed -n 's/^[^\-]*\-\([a-z0-9]*\).gz$/\1/p' } # Have there been any revision between $1 and $2 has_changes() { - local rev1=$1 - local rev2=$2 - local changes - changes=$(git diff --oneline "$rev1..$rev2" -- "${INPUTFILES[@]}" | wc -l) - [ "$changes" -gt 0 ] + local rev1=$1 + local rev2=$2 + local changes + changes=$(git diff --oneline "$rev1..$rev2" -- "${INPUTFILES[@]}" | wc -l) + [ "$changes" -gt 0 ] } commit_timestamp() { - local rev=$1 - git show -s --format=%ct "$rev" + local rev=$1 + git show -s --format=%ct "$rev" } # Is the SHA1 actually present in the repo? # It could be it isn't, e.g. after a force push is_valid_commit() { - local rev=$1 - git rev-parse --quiet --verify "$rev^{commit}" > /dev/null + local rev=$1 + git rev-parse --quiet --verify "$rev^{commit}" >/dev/null } cached_revision=$(cached_image_rev) if [ -z "$cached_revision" ]; then - echo ">>> No cached image found; rebuilding" - rebuild - exit 0 + echo ">>> No cached image found; rebuilding" + rebuild + exit 0 fi if ! is_valid_commit "$cached_revision"; then - echo ">>> Git commit of cached image not found in repo; rebuilding" - rebuild - exit 0 + echo ">>> Git commit of cached image not found in repo; rebuilding" + rebuild + exit 0 fi echo ">>> Found cached image rev $cached_revision" -if has_changes "$cached_revision" "$CIRCLE_SHA1" ; then - echo ">>> Found changes, rebuilding" - rebuild - exit 0 +if has_changes "$cached_revision" "$CIRCLE_SHA1"; then + echo ">>> Found changes, rebuilding" + rebuild + exit 0 fi -IMAGE_TIMEOUT="$(( 3 * 24 * 60 * 60 ))" +IMAGE_TIMEOUT="$((3 * 24 * 60 * 60))" if [ "$(commit_timestamp "$cached_revision")" -lt "${IMAGE_TIMEOUT}" ]; then - echo ">>> Image is more the 24hrs old; rebuilding" - rebuild - exit 0 + echo ">>> Image is more the 24hrs old; rebuilding" + rebuild + exit 0 fi # we didn't rebuild; import cached version diff --git a/socks/connect.sh b/socks/connect.sh index 25dcf8221..7e7975a38 100755 --- a/socks/connect.sh +++ b/socks/connect.sh @@ -3,8 +3,8 @@ set -eu if [ $# -ne 1 ]; then - echo "Usage: $0 " - exit 1 + echo "Usage: $0 " + exit 1 fi HOST=$1 @@ -12,10 +12,10 @@ HOST=$1 echo "Starting proxy container..." PROXY_CONTAINER=$(ssh "$HOST" weave run -d weaveworks/socksproxy) -function finish { - echo "Removing proxy container.." - # shellcheck disable=SC2029 - ssh "$HOST" docker rm -f "$PROXY_CONTAINER" +function finish() { + echo "Removing proxy container.." + # shellcheck disable=SC2029 + ssh "$HOST" docker rm -f "$PROXY_CONTAINER" } trap finish EXIT diff --git a/test b/test index 4ee4ca1a4..6a970884e 100755 --- a/test +++ b/test @@ -3,7 +3,7 @@ set -e DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -GO_TEST_ARGS=( -tags netgo -cpu 4 -timeout 8m ) +GO_TEST_ARGS=(-tags netgo -cpu 4 -timeout 8m) SLOW= NO_GO_GET= @@ -33,10 +33,10 @@ if [ -n "$SLOW" ] || [ -n "$CIRCLECI" ]; then fi if [ -n "$SLOW" ]; then - GO_TEST_ARGS=( "${GO_TEST_ARGS[@]}" -race -covermode=atomic ) + GO_TEST_ARGS=("${GO_TEST_ARGS[@]}" -race -covermode=atomic) # shellcheck disable=SC2153 - if [ -n "$COVERDIR" ] ; then + if [ -n "$COVERDIR" ]; then coverdir="$COVERDIR" else coverdir=$(mktemp -d coverage.XXXXXXXXXX) @@ -49,18 +49,18 @@ fail=0 if [ -z "$TESTDIRS" ]; then # NB: Relies on paths being prefixed with './'. - TESTDIRS=( $(git ls-files -- '*_test.go' | grep -vE '^(vendor|prog|experimental)/' | xargs -n1 dirname | sort -u | sed -e 's|^|./|') ) + TESTDIRS=($(git ls-files -- '*_test.go' | grep -vE '^(vendor|prog|experimental)/' | xargs -n1 dirname | sort -u | sed -e 's|^|./|')) else # TESTDIRS on the right side is not really an array variable, it # is just a string with spaces, but it is written like that to # shut up the shellcheck tool. - TESTDIRS=( $(for d in ${TESTDIRS[*]}; do echo "$d"; done) ) + TESTDIRS=($(for d in ${TESTDIRS[*]}; do echo "$d"; done)) fi # If running on circle, use the scheduler to work out what tests to run on what shard if [ -n "$CIRCLECI" ] && [ -z "$NO_SCHEDULER" ] && [ -x "$DIR/sched" ]; then PREFIX=$(go list -e ./ | sed -e 's/\//-/g') - TESTDIRS=( $(echo "${TESTDIRS[@]}" | "$DIR/sched" sched "$PREFIX-$CIRCLE_BUILD_NUM" "$CIRCLE_NODE_TOTAL" "$CIRCLE_NODE_INDEX") ) + TESTDIRS=($(echo "${TESTDIRS[@]}" | "$DIR/sched" sched "$PREFIX-$CIRCLE_BUILD_NUM" "$CIRCLE_NODE_TOTAL" "$CIRCLE_NODE_INDEX")) echo "${TESTDIRS[@]}" fi @@ -74,26 +74,29 @@ for dir in "${TESTDIRS[@]}"; do go get -t -tags netgo "$dir" fi - GO_TEST_ARGS_RUN=( "${GO_TEST_ARGS[@]}" ) + GO_TEST_ARGS_RUN=("${GO_TEST_ARGS[@]}") if [ -n "$SLOW" ]; then - COVERPKGS=$( (go list "$dir"; go list -f '{{join .Deps "\n"}}' "$dir" | grep -v "vendor" | grep "^$PACKAGE_BASE/") | paste -s -d, -) + COVERPKGS=$( ( + go list "$dir" + go list -f '{{join .Deps "\n"}}' "$dir" | grep -v "vendor" | grep "^$PACKAGE_BASE/" + ) | paste -s -d, -) output=$(mktemp "$coverdir/unit.XXXXXXXXXX") - GO_TEST_ARGS_RUN=( "${GO_TEST_ARGS[@]}" -coverprofile=$output -coverpkg=$COVERPKGS ) + GO_TEST_ARGS_RUN=("${GO_TEST_ARGS[@]}" -coverprofile=$output -coverpkg=$COVERPKGS) fi START=$(date +%s) if ! go test "${GO_TEST_ARGS_RUN[@]}" "$dir"; then fail=1 fi - RUNTIME=$(( $(date +%s) - START )) + RUNTIME=$(($(date +%s) - START)) # Report test runtime when running on circle, to help scheduler if [ -n "$CIRCLECI" ] && [ -z "$NO_SCHEDULER" ] && [ -x "$DIR/sched" ]; then - "$DIR/sched" time "$dir" $RUNTIME + "$DIR/sched" time "$dir" "$RUNTIME" fi done -if [ -n "$SLOW" ] && [ -z "$COVERDIR" ] ; then +if [ -n "$SLOW" ] && [ -z "$COVERDIR" ]; then go get github.com/weaveworks/tools/cover cover "$coverdir"/* >profile.cov rm -rf "$coverdir"