Squashed 'tools/' changes from b990f48..8ab80e8

8ab80e8 Merge pull request #45 from weaveworks/lint
83d5bd1 getting integration/config and test shellcheck-compliant
cff9ec3 Fix some shellcheck errors
7a843d6 run shellcheck as part of lint if it is installed
31552a0 removing spurious space from test
6ca7c5f Merge pull request #44 from weaveworks/shfmt
952356d Allow lint to lint itself
b7ac59c Run shfmt on all shell files in this repo
5570b0e Add shfmt formatting of shell files in lint
0a67594 fix circle build by splatting gopath permissions

git-subtree-dir: tools
git-subtree-split: 8ab80e878b0e414fb0a4c0b7c97d665b32dfc463
This commit is contained in:
Paul Bellamy
2016-10-26 15:32:28 +01:00
parent 7eb07eebff
commit c03f7d50ff
11 changed files with 391 additions and 341 deletions

View File

@@ -25,14 +25,15 @@ export CONTINUE=${CONTINUE:-}
args="$(getopt -n "$0" -l \
verbose,help,stop,discover,invariant,continue vhxdic "$@")" \
|| exit -1
|| exit -1
for arg in $args; do
case "$arg" in
-h)
echo "$0 [-vxidc]" \
"[--verbose] [--stop] [--invariant] [--discover] [--continue]"
echo "$(sed 's/./ /g' <<< "$0") [-h] [--help]"
exit 0;;
echo "$(sed 's/./ /g' <<<"$0") [-h] [--help]"
exit 0
;;
--help)
cat <<EOF
Usage: $0 [options]
@@ -47,17 +48,23 @@ Options:
-h show brief usage information and exit
--help show this help message and exit
EOF
exit 0;;
-v|--verbose)
DEBUG=1;;
-x|--stop)
STOP=1;;
-i|--invariant)
INVARIANT=1;;
-d|--discover)
DISCOVERONLY=1;;
-c|--continue)
CONTINUE=1;;
exit 0
;;
-v | --verbose)
DEBUG=1
;;
-x | --stop)
STOP=1
;;
-i | --invariant)
INVARIANT=1
;;
-d | --discover)
DISCOVERONLY=1
;;
-c | --continue)
CONTINUE=1
;;
esac
done
@@ -74,9 +81,9 @@ assert_end() {
# assert_end [suite ..]
tests_endtime="$(date +%s%N)"
# required visible decimal place for seconds (leading zeros if needed)
tests_time="$( \
printf "%010d" "$(( ${tests_endtime/%N/000000000}
- ${tests_starttime/%N/000000000} ))")" # in ns
tests_time="$(
printf "%010d" "$((${tests_endtime/%N/000000000} - ${tests_starttime/%N/000000000}))"
)" # in ns
tests="$tests_ran ${*:+$* }tests"
[[ -n "$DISCOVERONLY" ]] && echo "collected $tests." && _assert_reset && return
[[ -n "$DEBUG" ]] && echo
@@ -100,15 +107,15 @@ assert_end() {
assert() {
# assert <command> <expected stdout> [stdin]
(( tests_ran++ )) || :
((tests_ran++)) || :
[[ -z "$DISCOVERONLY" ]] || return
expected=$(echo -ne "${2:-}")
result="$(eval 2>/dev/null "$1" <<< "${3:-}")" || true
result="$(eval "$1" 2>/dev/null <<<"${3:-}")" || true
if [[ "$result" == "$expected" ]]; then
[[ -z "$DEBUG" ]] || echo -n .
return
fi
result="$(sed -e :a -e '$!N;s/\n/\\n/;ta' <<< "$result")"
result="$(sed -e :a -e '$!N;s/\n/\\n/;ta' <<<"$result")"
[[ -z "$result" ]] && result="nothing" || result="\"$result\""
[[ -z "$2" ]] && expected="nothing" || expected="\"$2\""
_assert_fail "expected $expected${_indent}got $result" "$1" "$3"
@@ -116,10 +123,10 @@ assert() {
assert_raises() {
# assert_raises <command> <expected code> [stdin]
(( tests_ran++ )) || :
((tests_ran++)) || :
[[ -z "$DISCOVERONLY" ]] || return
status=0
(eval "$1" <<< "${3:-}") > /dev/null 2>&1 || status=$?
(eval "$1" <<<"${3:-}") >/dev/null 2>&1 || status=$?
expected=${2:-0}
if [[ "$status" -eq "$expected" ]]; then
[[ -z "$DEBUG" ]] || echo -n .
@@ -138,12 +145,12 @@ _assert_fail() {
exit 1
fi
tests_errors[$tests_failed]="$report"
(( tests_failed++ )) || :
((tests_failed++)) || :
}
skip_if() {
# skip_if <command ..>
(eval "$@") > /dev/null 2>&1 && status=0 || status=$?
(eval "$@") >/dev/null 2>&1 && status=0 || status=$?
[[ "$status" -eq 0 ]] || return
skip
}
@@ -175,9 +182,8 @@ _skip() {
fi
}
_assert_reset
: ${tests_suite_status:=0} # remember if any of the tests failed so far
: ${tests_suite_status:=0} # remember if any of the tests failed so far
_assert_cleanup() {
local status=$?
# modify exit code if it's not already non-zero

View File

@@ -1,3 +1,4 @@
#!/bin/bash
# NB only to be sourced
set -e
@@ -16,49 +17,55 @@ N_MACHINES=${N_MACHINES:-3}
IP_PREFIX=${IP_PREFIX:-192.168.48}
IP_SUFFIX_BASE=${IP_SUFFIX_BASE:-10}
if [ -z "$HOSTS" ] ; then
for i in $(seq 1 $N_MACHINES); do
IP="${IP_PREFIX}.$((${IP_SUFFIX_BASE}+$i))"
if [ -z "$HOSTS" ]; then
for i in $(seq 1 "$N_MACHINES"); do
IP="${IP_PREFIX}.$((IP_SUFFIX_BASE + i))"
HOSTS="$HOSTS $IP"
done
fi
# these are used by the tests
HOST1=$(echo $HOSTS | cut -f 1 -d ' ')
HOST2=$(echo $HOSTS | cut -f 2 -d ' ')
HOST3=$(echo $HOSTS | cut -f 3 -d ' ')
# shellcheck disable=SC2034
HOST1=$(echo "$HOSTS" | cut -f 1 -d ' ')
# shellcheck disable=SC2034
HOST2=$(echo "$HOSTS" | cut -f 2 -d ' ')
# shellcheck disable=SC2034
HOST3=$(echo "$HOSTS" | cut -f 3 -d ' ')
# shellcheck disable=SC1090
. "$DIR/assert.sh"
SSH_DIR=${SSH_DIR:-$DIR}
SSH=${SSH:-ssh -l vagrant -i "$SSH_DIR/insecure_private_key" -o "UserKnownHostsFile=$SSH_DIR/.ssh_known_hosts" -o CheckHostIP=no -o StrictHostKeyChecking=no}
SSH=${SSH:-ssh -l vagrant -i \"$SSH_DIR/insecure_private_key\" -o \"UserKnownHostsFile=$SSH_DIR/.ssh_known_hosts\" -o CheckHostIP=no -o StrictHostKeyChecking=no}
SMALL_IMAGE="alpine"
# shellcheck disable=SC2034
TEST_IMAGES="$SMALL_IMAGE"
# shellcheck disable=SC2034
PING="ping -nq -W 1 -c 1"
DOCKER_PORT=2375
remote() {
rem=$1
shift 1
"$@" > >(while read line; do echo -e $'\e[0;34m'"$rem>"$'\e[0m'" $line"; done)
"$@" > >(while read -r line; do echo -e $'\e[0;34m'"$rem>"$'\e[0m'" $line"; done)
}
colourise() {
[ -t 0 ] && echo -ne $'\e['$1'm' || true
([ -t 0 ] && echo -ne $'\e['"$1"'m') || true
shift
# It's important that we don't do this in a subshell, as some
# commands we execute need to modify global state
"$@"
[ -t 0 ] && echo -ne $'\e[0m' || true
([ -t 0 ] && echo -ne $'\e[0m') || true
}
whitely() {
colourise '1;37' "$@"
}
greyly () {
greyly() {
colourise '0;37' "$@"
}
@@ -73,21 +80,21 @@ greenly() {
run_on() {
host=$1
shift 1
[ -z "$DEBUG" ] || greyly echo "Running on $host: $@" >&2
remote $host $SSH $host "$@"
[ -z "$DEBUG" ] || greyly echo "Running on $host:" "$@" >&2
remote "$host" "$SSH" "$host" "$@"
}
docker_on() {
host=$1
shift 1
[ -z "$DEBUG" ] || greyly echo "Docker on $host:$DOCKER_PORT: $@" >&2
docker -H tcp://$host:$DOCKER_PORT "$@"
[ -z "$DEBUG" ] || greyly echo "Docker on $host:$DOCKER_PORT:" "$@" >&2
docker -H "tcp://$host:$DOCKER_PORT" "$@"
}
weave_on() {
host=$1
shift 1
[ -z "$DEBUG" ] || greyly echo "Weave on $host:$DOCKER_PORT: $@" >&2
[ -z "$DEBUG" ] || greyly echo "Weave on $host:$DOCKER_PORT:" "$@" >&2
DOCKER_HOST=tcp://$host:$DOCKER_PORT $WEAVE "$@"
}
@@ -95,24 +102,24 @@ exec_on() {
host=$1
container=$2
shift 2
docker -H tcp://$host:$DOCKER_PORT exec $container "$@"
docker -H "tcp://$host:$DOCKER_PORT" exec "$container" "$@"
}
rm_containers() {
host=$1
shift
[ $# -eq 0 ] || docker_on $host rm -f "$@" >/dev/null
[ $# -eq 0 ] || docker_on "$host" rm -f "$@" >/dev/null
}
start_suite() {
for host in $HOSTS; do
[ -z "$DEBUG" ] || echo "Cleaning up on $host: removing all containers and resetting weave"
PLUGIN_ID=$(docker_on $host ps -aq --filter=name=weaveplugin)
PLUGIN_ID=$(docker_on "$host" ps -aq --filter=name=weaveplugin)
PLUGIN_FILTER="cat"
[ -n "$PLUGIN_ID" ] && PLUGIN_FILTER="grep -v $PLUGIN_ID"
rm_containers $host $(docker_on $host ps -aq 2>/dev/null | $PLUGIN_FILTER)
run_on $host "docker network ls | grep -q ' weave ' && docker network rm weave" || true
weave_on $host reset 2>/dev/null
rm_containers "$host" "$(docker_on "$host" ps -aq 2>/dev/null | "$PLUGIN_FILTER")"
run_on "$host" "docker network ls | grep -q ' weave ' && docker network rm weave" || true
weave_on "$host" reset 2>/dev/null
done
whitely echo "$@"
}
@@ -122,4 +129,3 @@ end_suite() {
}
WEAVE=$DIR/../weave

View File

@@ -16,68 +16,73 @@ set -e
: "${NUM_HOSTS:=}"
if [ -z "${PROJECT}" ] || [ -z "${NUM_HOSTS}" ] || [ -z "${TEMPLATE_NAME}" ]; then
echo "Must specify PROJECT, NUM_HOSTS and TEMPLATE_NAME"
exit 1
echo "Must specify PROJECT, NUM_HOSTS and TEMPLATE_NAME"
exit 1
fi
SUFFIX=""
if [ -n "$CIRCLECI" ]; then
SUFFIX="-${CIRCLE_BUILD_NUM}-$CIRCLE_NODE_INDEX"
SUFFIX="-${CIRCLE_BUILD_NUM}-$CIRCLE_NODE_INDEX"
fi
# Setup authentication
gcloud auth activate-service-account --key-file "$KEY_FILE" 1>/dev/null
gcloud config set project "$PROJECT"
function vm_names {
local names=
for i in $(seq 1 "$NUM_HOSTS"); do
names=( "host$i$SUFFIX" "${names[@]}" )
done
echo "${names[@]}"
function vm_names() {
local names=
for i in $(seq 1 "$NUM_HOSTS"); do
names=("host$i$SUFFIX" "${names[@]}")
done
echo "${names[@]}"
}
# Delete all vms in this account
function destroy {
local names
names="$(vm_names)"
if [ "$(gcloud compute instances list --zone "$ZONE" -q "$names" | wc -l)" -le 1 ] ; then
return 0
fi
for i in {0..10}; do
# gcloud instances delete can sometimes hang.
case $(set +e; timeout 60s /bin/bash -c "gcloud compute instances delete --zone $ZONE -q $names >/dev/null 2>&1"; echo $?) in
0)
return 0
;;
124)
# 124 means it timed out
break
;;
*)
return 1
esac
done
function destroy() {
local names
names="$(vm_names)"
if [ "$(gcloud compute instances list --zone "$ZONE" -q "$names" | wc -l)" -le 1 ]; then
return 0
fi
for i in {0..10}; do
# gcloud instances delete can sometimes hang.
case $(
set +e
timeout 60s /bin/bash -c "gcloud compute instances delete --zone $ZONE -q $names >/dev/null 2>&1"
echo $?
) in
0)
return 0
;;
124)
# 124 means it timed out
break
;;
*)
return 1
;;
esac
done
}
function internal_ip {
jq -r ".[] | select(.name == \"$2\") | .networkInterfaces[0].networkIP" "$1"
function internal_ip() {
jq -r ".[] | select(.name == \"$2\") | .networkInterfaces[0].networkIP" "$1"
}
function external_ip {
jq -r ".[] | select(.name == \"$2\") | .networkInterfaces[0].accessConfigs[0].natIP" "$1"
function external_ip() {
jq -r ".[] | select(.name == \"$2\") | .networkInterfaces[0].accessConfigs[0].natIP" "$1"
}
function try_connect {
for i in {0..10}; do
ssh -t "$1" true && return
sleep 2
done
function try_connect() {
for i in {0..10}; do
ssh -t "$1" true && return
sleep 2
done
}
function install_docker_on {
name=$1
ssh -t "$name" sudo bash -x -s <<EOF
function install_docker_on() {
name=$1
ssh -t "$name" sudo bash -x -s <<EOF
curl -sSL https://get.docker.com/gpg | sudo apt-key add -
curl -sSL https://get.docker.com/ | sh
apt-get update -qq;
@@ -86,92 +91,93 @@ usermod -a -G docker vagrant;
echo 'DOCKER_OPTS="-H unix:///var/run/docker.sock -H unix:///var/run/alt-docker.sock -H tcp://0.0.0.0:2375 -s overlay"' >> /etc/default/docker;
service docker restart
EOF
# It seems we need a short delay for docker to start up, so I put this in
# a separate ssh connection. This installs nsenter.
ssh -t "$name" sudo docker run --rm -v /usr/local/bin:/target jpetazzo/nsenter
# It seems we need a short delay for docker to start up, so I put this in
# a separate ssh connection. This installs nsenter.
ssh -t "$name" sudo docker run --rm -v /usr/local/bin:/target jpetazzo/nsenter
}
function copy_hosts {
hostname=$1
hosts=$2
ssh -t "$hostname" "sudo -- sh -c \"cat >>/etc/hosts\"" < "$hosts"
function copy_hosts() {
hostname=$1
hosts=$2
ssh -t "$hostname" "sudo -- sh -c \"cat >>/etc/hosts\"" <"$hosts"
}
# Create new set of VMs
function setup {
destroy
function setup() {
destroy
names=( $(vm_names) )
gcloud compute instances create "${names[@]}" --image "$TEMPLATE_NAME" --zone "$ZONE"
gcloud compute config-ssh --ssh-key-file "$SSH_KEY_FILE"
sed -i '/UserKnownHostsFile=\/dev\/null/d' ~/.ssh/config
names=($(vm_names))
gcloud compute instances create "${names[@]}" --image "$TEMPLATE_NAME" --zone "$ZONE"
gcloud compute config-ssh --ssh-key-file "$SSH_KEY_FILE"
sed -i '/UserKnownHostsFile=\/dev\/null/d' ~/.ssh/config
# build an /etc/hosts file for these vms
hosts=$(mktemp hosts.XXXXXXXXXX)
json=$(mktemp json.XXXXXXXXXX)
gcloud compute instances list --format=json > "$json"
for name in "${names[@]}"; do
echo "$(internal_ip "$json" "$name") $name.$ZONE.$PROJECT" >> "$hosts"
done
# build an /etc/hosts file for these vms
hosts=$(mktemp hosts.XXXXXXXXXX)
json=$(mktemp json.XXXXXXXXXX)
gcloud compute instances list --format=json >"$json"
for name in "${names[@]}"; do
echo "$(internal_ip "$json" "$name") $name.$ZONE.$PROJECT" >>"$hosts"
done
for name in "${names[@]}"; do
hostname="$name.$ZONE.$PROJECT"
for name in "${names[@]}"; do
hostname="$name.$ZONE.$PROJECT"
# Add the remote ip to the local /etc/hosts
sudo sed -i "/$hostname/d" /etc/hosts
sudo sh -c "echo \"$(external_ip "$json" "$name") $hostname\" >>/etc/hosts"
try_connect "$hostname"
# Add the remote ip to the local /etc/hosts
sudo sed -i "/$hostname/d" /etc/hosts
sudo sh -c "echo \"$(external_ip "$json" "$name") $hostname\" >>/etc/hosts"
try_connect "$hostname"
copy_hosts "$hostname" "$hosts" &
done
copy_hosts "$hostname" "$hosts" &
done
wait
wait
rm "$hosts" "$json"
rm "$hosts" "$json"
}
function make_template {
gcloud compute instances create "$TEMPLATE_NAME" --image "$IMAGE" --zone "$ZONE"
gcloud compute config-ssh --ssh-key-file "$SSH_KEY_FILE"
name="$TEMPLATE_NAME.$ZONE.$PROJECT"
try_connect "$name"
install_docker_on "$name"
gcloud -q compute instances delete "$TEMPLATE_NAME" --keep-disks boot --zone "$ZONE"
gcloud compute images create "$TEMPLATE_NAME" --source-disk "$TEMPLATE_NAME" --source-disk-zone "$ZONE"
function make_template() {
gcloud compute instances create "$TEMPLATE_NAME" --image "$IMAGE" --zone "$ZONE"
gcloud compute config-ssh --ssh-key-file "$SSH_KEY_FILE"
name="$TEMPLATE_NAME.$ZONE.$PROJECT"
try_connect "$name"
install_docker_on "$name"
gcloud -q compute instances delete "$TEMPLATE_NAME" --keep-disks boot --zone "$ZONE"
gcloud compute images create "$TEMPLATE_NAME" --source-disk "$TEMPLATE_NAME" --source-disk-zone "$ZONE"
}
function hosts {
hosts=
args=
json=$(mktemp json.XXXXXXXXXX)
gcloud compute instances list --format=json > "$json"
for name in $(vm_names); do
hostname="$name.$ZONE.$PROJECT"
hosts=( $hostname "${hosts[@]}" )
args=( "--add-host=$hostname:$(internal_ip "$json" "$name")" "${args[@]}" )
done
echo export SSH=\"ssh -l vagrant\"
echo "export HOSTS=\"${hosts[*]}\""
echo "export ADD_HOST_ARGS=\"${args[*]}\""
rm "$json"
function hosts() {
hosts=
args=
json=$(mktemp json.XXXXXXXXXX)
gcloud compute instances list --format=json >"$json"
for name in $(vm_names); do
hostname="$name.$ZONE.$PROJECT"
hosts=($hostname "${hosts[@]}")
args=("--add-host=$hostname:$(internal_ip "$json" "$name")" "${args[@]}")
done
echo export SSH=\"ssh -l vagrant\"
echo "export HOSTS=\"${hosts[*]}\""
echo "export ADD_HOST_ARGS=\"${args[*]}\""
rm "$json"
}
case "$1" in
setup)
setup
;;
setup)
setup
;;
hosts)
hosts
;;
hosts)
hosts
;;
destroy)
destroy
;;
destroy)
destroy
;;
make_template)
# see if template exists
if ! gcloud compute images list | grep "$PROJECT" | grep "$TEMPLATE_NAME"; then
make_template
fi
make_template)
# see if template exists
if ! gcloud compute images list | grep "$PROJECT" | grep "$TEMPLATE_NAME"; then
make_template
fi
;;
esac

View File

@@ -14,17 +14,17 @@ fi
whitely echo ...ok
# shellcheck disable=SC2068
TESTS=( ${@:-$(find . -name '*_test.sh')} )
RUNNER_ARGS=( )
TESTS=(${@:-$(find . -name '*_test.sh')})
RUNNER_ARGS=()
# If running on circle, use the scheduler to work out what tests to run
if [ -n "$CIRCLECI" ] && [ -z "$NO_SCHEDULER" ]; then
RUNNER_ARGS=( "${RUNNER_ARGS[@]}" -scheduler )
RUNNER_ARGS=("${RUNNER_ARGS[@]}" -scheduler)
fi
# If running on circle or PARALLEL is not empty, run tests in parallel
if [ -n "$CIRCLECI" ] || [ -n "$PARALLEL" ]; then
RUNNER_ARGS=( "${RUNNER_ARGS[@]}" -parallel )
RUNNER_ARGS=("${RUNNER_ARGS[@]}" -parallel)
fi
make -C "${DIR}/../runner"