Merge commit 'd9ce1d58e9cbcd67823ee0abe8dd346a9aa6d120' into iaguis/fix-lint

This commit is contained in:
Iago López Galeiras
2017-01-09 14:40:34 +01:00
11 changed files with 406 additions and 342 deletions

61
tools/integration/assert.sh Normal file → Executable file
View File

@@ -25,14 +25,15 @@ export CONTINUE=${CONTINUE:-}
args="$(getopt -n "$0" -l \
verbose,help,stop,discover,invariant,continue vhxdic "$@")" \
|| exit -1
|| exit -1
for arg in $args; do
case "$arg" in
-h)
echo "$0 [-vxidc]" \
"[--verbose] [--stop] [--invariant] [--discover] [--continue]"
echo "$(sed 's/./ /g' <<< "$0") [-h] [--help]"
exit 0;;
echo "$(sed 's/./ /g' <<<"$0") [-h] [--help]"
exit 0
;;
--help)
cat <<EOF
Usage: $0 [options]
@@ -47,17 +48,23 @@ Options:
-h show brief usage information and exit
--help show this help message and exit
EOF
exit 0;;
-v|--verbose)
DEBUG=1;;
-x|--stop)
STOP=1;;
-i|--invariant)
INVARIANT=1;;
-d|--discover)
DISCOVERONLY=1;;
-c|--continue)
CONTINUE=1;;
exit 0
;;
-v | --verbose)
DEBUG=1
;;
-x | --stop)
STOP=1
;;
-i | --invariant)
INVARIANT=1
;;
-d | --discover)
DISCOVERONLY=1
;;
-c | --continue)
CONTINUE=1
;;
esac
done
@@ -74,9 +81,9 @@ assert_end() {
# assert_end [suite ..]
tests_endtime="$(date +%s%N)"
# required visible decimal place for seconds (leading zeros if needed)
tests_time="$( \
printf "%010d" "$(( ${tests_endtime/%N/000000000}
- ${tests_starttime/%N/000000000} ))")" # in ns
tests_time="$(
printf "%010d" "$((${tests_endtime/%N/000000000} - ${tests_starttime/%N/000000000}))"
)" # in ns
tests="$tests_ran ${*:+$* }tests"
[[ -n "$DISCOVERONLY" ]] && echo "collected $tests." && _assert_reset && return
[[ -n "$DEBUG" ]] && echo
@@ -84,7 +91,8 @@ assert_end() {
# ${tests_time:0:${#tests_time}-9} - seconds
# ${tests_time:${#tests_time}-9:3} - milliseconds
if [[ -z "$INVARIANT" ]]; then
report_time=" in ${tests_time:0:${#tests_time}-9}.${tests_time:${#tests_time}-9:3}s"
idx=$((${#tests_time} - 9))
report_time=" in ${tests_time:0:${idx}}.${tests_time:${idx}:3}s"
else
report_time=
fi
@@ -100,15 +108,15 @@ assert_end() {
assert() {
# assert <command> <expected stdout> [stdin]
(( tests_ran++ )) || :
((tests_ran++)) || :
[[ -z "$DISCOVERONLY" ]] || return
expected=$(echo -ne "${2:-}")
result="$(eval 2>/dev/null "$1" <<< "${3:-}")" || true
result="$(eval "$1" 2>/dev/null <<<"${3:-}")" || true
if [[ "$result" == "$expected" ]]; then
[[ -z "$DEBUG" ]] || echo -n .
return
fi
result="$(sed -e :a -e '$!N;s/\n/\\n/;ta' <<< "$result")"
result="$(sed -e :a -e '$!N;s/\n/\\n/;ta' <<<"$result")"
[[ -z "$result" ]] && result="nothing" || result="\"$result\""
[[ -z "$2" ]] && expected="nothing" || expected="\"$2\""
_assert_fail "expected $expected${_indent}got $result" "$1" "$3"
@@ -116,10 +124,10 @@ assert() {
assert_raises() {
# assert_raises <command> <expected code> [stdin]
(( tests_ran++ )) || :
((tests_ran++)) || :
[[ -z "$DISCOVERONLY" ]] || return
status=0
(eval "$1" <<< "${3:-}") > /dev/null 2>&1 || status=$?
(eval "$1" <<<"${3:-}") >/dev/null 2>&1 || status=$?
expected=${2:-0}
if [[ "$status" -eq "$expected" ]]; then
[[ -z "$DEBUG" ]] || echo -n .
@@ -138,12 +146,12 @@ _assert_fail() {
exit 1
fi
tests_errors[$tests_failed]="$report"
(( tests_failed++ )) || :
((tests_failed++)) || :
}
skip_if() {
# skip_if <command ..>
(eval "$@") > /dev/null 2>&1 && status=0 || status=$?
(eval "$@") >/dev/null 2>&1 && status=0 || status=$?
[[ "$status" -eq 0 ]] || return
skip
}
@@ -175,9 +183,8 @@ _skip() {
fi
}
_assert_reset
: ${tests_suite_status:=0} # remember if any of the tests failed so far
: ${tests_suite_status:=0} # remember if any of the tests failed so far
_assert_cleanup() {
local status=$?
# modify exit code if it's not already non-zero

View File

@@ -1,3 +1,4 @@
#!/bin/bash
# NB only to be sourced
set -e
@@ -16,49 +17,55 @@ N_MACHINES=${N_MACHINES:-3}
IP_PREFIX=${IP_PREFIX:-192.168.48}
IP_SUFFIX_BASE=${IP_SUFFIX_BASE:-10}
if [ -z "$HOSTS" ] ; then
for i in $(seq 1 $N_MACHINES); do
IP="${IP_PREFIX}.$((${IP_SUFFIX_BASE}+$i))"
if [ -z "$HOSTS" ]; then
for i in $(seq 1 "$N_MACHINES"); do
IP="${IP_PREFIX}.$((IP_SUFFIX_BASE + i))"
HOSTS="$HOSTS $IP"
done
fi
# these are used by the tests
HOST1=$(echo $HOSTS | cut -f 1 -d ' ')
HOST2=$(echo $HOSTS | cut -f 2 -d ' ')
HOST3=$(echo $HOSTS | cut -f 3 -d ' ')
# shellcheck disable=SC2034
HOST1=$(echo "$HOSTS" | cut -f 1 -d ' ')
# shellcheck disable=SC2034
HOST2=$(echo "$HOSTS" | cut -f 2 -d ' ')
# shellcheck disable=SC2034
HOST3=$(echo "$HOSTS" | cut -f 3 -d ' ')
# shellcheck disable=SC1090
. "$DIR/assert.sh"
SSH_DIR=${SSH_DIR:-$DIR}
SSH=${SSH:-ssh -l vagrant -i "$SSH_DIR/insecure_private_key" -o "UserKnownHostsFile=$SSH_DIR/.ssh_known_hosts" -o CheckHostIP=no -o StrictHostKeyChecking=no}
SSH=${SSH:-ssh -l vagrant -i \"$SSH_DIR/insecure_private_key\" -o \"UserKnownHostsFile=$SSH_DIR/.ssh_known_hosts\" -o CheckHostIP=no -o StrictHostKeyChecking=no}
SMALL_IMAGE="alpine"
# shellcheck disable=SC2034
TEST_IMAGES="$SMALL_IMAGE"
# shellcheck disable=SC2034
PING="ping -nq -W 1 -c 1"
DOCKER_PORT=2375
remote() {
rem=$1
shift 1
"$@" > >(while read line; do echo -e $'\e[0;34m'"$rem>"$'\e[0m'" $line"; done)
"$@" > >(while read -r line; do echo -e $'\e[0;34m'"$rem>"$'\e[0m'" $line"; done)
}
colourise() {
[ -t 0 ] && echo -ne $'\e['$1'm' || true
([ -t 0 ] && echo -ne $'\e['"$1"'m') || true
shift
# It's important that we don't do this in a subshell, as some
# commands we execute need to modify global state
"$@"
[ -t 0 ] && echo -ne $'\e[0m' || true
([ -t 0 ] && echo -ne $'\e[0m') || true
}
whitely() {
colourise '1;37' "$@"
}
greyly () {
greyly() {
colourise '0;37' "$@"
}
@@ -73,21 +80,21 @@ greenly() {
run_on() {
host=$1
shift 1
[ -z "$DEBUG" ] || greyly echo "Running on $host: $@" >&2
remote $host $SSH $host "$@"
[ -z "$DEBUG" ] || greyly echo "Running on $host:" "$@" >&2
remote "$host" "$SSH" "$host" "$@"
}
docker_on() {
host=$1
shift 1
[ -z "$DEBUG" ] || greyly echo "Docker on $host:$DOCKER_PORT: $@" >&2
docker -H tcp://$host:$DOCKER_PORT "$@"
[ -z "$DEBUG" ] || greyly echo "Docker on $host:$DOCKER_PORT:" "$@" >&2
docker -H "tcp://$host:$DOCKER_PORT" "$@"
}
weave_on() {
host=$1
shift 1
[ -z "$DEBUG" ] || greyly echo "Weave on $host:$DOCKER_PORT: $@" >&2
[ -z "$DEBUG" ] || greyly echo "Weave on $host:$DOCKER_PORT:" "$@" >&2
DOCKER_HOST=tcp://$host:$DOCKER_PORT $WEAVE "$@"
}
@@ -95,24 +102,24 @@ exec_on() {
host=$1
container=$2
shift 2
docker -H tcp://$host:$DOCKER_PORT exec $container "$@"
docker -H "tcp://$host:$DOCKER_PORT" exec "$container" "$@"
}
rm_containers() {
host=$1
shift
[ $# -eq 0 ] || docker_on $host rm -f "$@" >/dev/null
[ $# -eq 0 ] || docker_on "$host" rm -f "$@" >/dev/null
}
start_suite() {
for host in $HOSTS; do
[ -z "$DEBUG" ] || echo "Cleaning up on $host: removing all containers and resetting weave"
PLUGIN_ID=$(docker_on $host ps -aq --filter=name=weaveplugin)
PLUGIN_ID=$(docker_on "$host" ps -aq --filter=name=weaveplugin)
PLUGIN_FILTER="cat"
[ -n "$PLUGIN_ID" ] && PLUGIN_FILTER="grep -v $PLUGIN_ID"
rm_containers $host $(docker_on $host ps -aq 2>/dev/null | $PLUGIN_FILTER)
run_on $host "docker network ls | grep -q ' weave ' && docker network rm weave" || true
weave_on $host reset 2>/dev/null
rm_containers "$host" "$(docker_on "$host" ps -aq 2>/dev/null | "$PLUGIN_FILTER")"
run_on "$host" "docker network ls | grep -q ' weave ' && docker network rm weave" || true
weave_on "$host" reset 2>/dev/null
done
whitely echo "$@"
}
@@ -122,4 +129,3 @@ end_suite() {
}
WEAVE=$DIR/../weave

View File

@@ -16,68 +16,73 @@ set -e
: "${NUM_HOSTS:=}"
if [ -z "${PROJECT}" ] || [ -z "${NUM_HOSTS}" ] || [ -z "${TEMPLATE_NAME}" ]; then
echo "Must specify PROJECT, NUM_HOSTS and TEMPLATE_NAME"
exit 1
echo "Must specify PROJECT, NUM_HOSTS and TEMPLATE_NAME"
exit 1
fi
SUFFIX=""
if [ -n "$CIRCLECI" ]; then
SUFFIX="-${CIRCLE_BUILD_NUM}-$CIRCLE_NODE_INDEX"
SUFFIX="-${CIRCLE_BUILD_NUM}-$CIRCLE_NODE_INDEX"
fi
# Setup authentication
gcloud auth activate-service-account --key-file "$KEY_FILE" 1>/dev/null
gcloud config set project "$PROJECT"
function vm_names {
local names=
for i in $(seq 1 "$NUM_HOSTS"); do
names=( "host$i$SUFFIX" "${names[@]}" )
done
echo "${names[@]}"
function vm_names() {
local names=
for i in $(seq 1 "$NUM_HOSTS"); do
names=("host$i$SUFFIX" "${names[@]}")
done
echo "${names[@]}"
}
# Delete all vms in this account
function destroy {
local names
names="$(vm_names)"
if [ "$(gcloud compute instances list --zone "$ZONE" -q "$names" | wc -l)" -le 1 ] ; then
return 0
fi
for i in {0..10}; do
# gcloud instances delete can sometimes hang.
case $(set +e; timeout 60s /bin/bash -c "gcloud compute instances delete --zone $ZONE -q $names >/dev/null 2>&1"; echo $?) in
0)
return 0
;;
124)
# 124 means it timed out
break
;;
*)
return 1
esac
done
function destroy() {
local names
names="$(vm_names)"
if [ "$(gcloud compute instances list --zone "$ZONE" -q "$names" | wc -l)" -le 1 ]; then
return 0
fi
for i in {0..10}; do
# gcloud instances delete can sometimes hang.
case $(
set +e
timeout 60s /bin/bash -c "gcloud compute instances delete --zone $ZONE -q $names >/dev/null 2>&1"
echo $?
) in
0)
return 0
;;
124)
# 124 means it timed out
break
;;
*)
return 1
;;
esac
done
}
function internal_ip {
jq -r ".[] | select(.name == \"$2\") | .networkInterfaces[0].networkIP" "$1"
function internal_ip() {
jq -r ".[] | select(.name == \"$2\") | .networkInterfaces[0].networkIP" "$1"
}
function external_ip {
jq -r ".[] | select(.name == \"$2\") | .networkInterfaces[0].accessConfigs[0].natIP" "$1"
function external_ip() {
jq -r ".[] | select(.name == \"$2\") | .networkInterfaces[0].accessConfigs[0].natIP" "$1"
}
function try_connect {
for i in {0..10}; do
ssh -t "$1" true && return
sleep 2
done
function try_connect() {
for i in {0..10}; do
ssh -t "$1" true && return
sleep 2
done
}
function install_docker_on {
name=$1
ssh -t "$name" sudo bash -x -s <<EOF
function install_docker_on() {
name=$1
ssh -t "$name" sudo bash -x -s <<EOF
curl -sSL https://get.docker.com/gpg | sudo apt-key add -
curl -sSL https://get.docker.com/ | sh
apt-get update -qq;
@@ -86,92 +91,93 @@ usermod -a -G docker vagrant;
echo 'DOCKER_OPTS="-H unix:///var/run/docker.sock -H unix:///var/run/alt-docker.sock -H tcp://0.0.0.0:2375 -s overlay"' >> /etc/default/docker;
service docker restart
EOF
# It seems we need a short delay for docker to start up, so I put this in
# a separate ssh connection. This installs nsenter.
ssh -t "$name" sudo docker run --rm -v /usr/local/bin:/target jpetazzo/nsenter
# It seems we need a short delay for docker to start up, so I put this in
# a separate ssh connection. This installs nsenter.
ssh -t "$name" sudo docker run --rm -v /usr/local/bin:/target jpetazzo/nsenter
}
function copy_hosts {
hostname=$1
hosts=$2
ssh -t "$hostname" "sudo -- sh -c \"cat >>/etc/hosts\"" < "$hosts"
function copy_hosts() {
hostname=$1
hosts=$2
ssh -t "$hostname" "sudo -- sh -c \"cat >>/etc/hosts\"" <"$hosts"
}
# Create new set of VMs
function setup {
destroy
function setup() {
destroy
names=( $(vm_names) )
gcloud compute instances create "${names[@]}" --image "$TEMPLATE_NAME" --zone "$ZONE"
gcloud compute config-ssh --ssh-key-file "$SSH_KEY_FILE"
sed -i '/UserKnownHostsFile=\/dev\/null/d' ~/.ssh/config
names=($(vm_names))
gcloud compute instances create "${names[@]}" --image "$TEMPLATE_NAME" --zone "$ZONE"
gcloud compute config-ssh --ssh-key-file "$SSH_KEY_FILE"
sed -i '/UserKnownHostsFile=\/dev\/null/d' ~/.ssh/config
# build an /etc/hosts file for these vms
hosts=$(mktemp hosts.XXXXXXXXXX)
json=$(mktemp json.XXXXXXXXXX)
gcloud compute instances list --format=json > "$json"
for name in "${names[@]}"; do
echo "$(internal_ip "$json" "$name") $name.$ZONE.$PROJECT" >> "$hosts"
done
# build an /etc/hosts file for these vms
hosts=$(mktemp hosts.XXXXXXXXXX)
json=$(mktemp json.XXXXXXXXXX)
gcloud compute instances list --format=json >"$json"
for name in "${names[@]}"; do
echo "$(internal_ip "$json" "$name") $name.$ZONE.$PROJECT" >>"$hosts"
done
for name in "${names[@]}"; do
hostname="$name.$ZONE.$PROJECT"
for name in "${names[@]}"; do
hostname="$name.$ZONE.$PROJECT"
# Add the remote ip to the local /etc/hosts
sudo sed -i "/$hostname/d" /etc/hosts
sudo sh -c "echo \"$(external_ip "$json" "$name") $hostname\" >>/etc/hosts"
try_connect "$hostname"
# Add the remote ip to the local /etc/hosts
sudo sed -i "/$hostname/d" /etc/hosts
sudo sh -c "echo \"$(external_ip "$json" "$name") $hostname\" >>/etc/hosts"
try_connect "$hostname"
copy_hosts "$hostname" "$hosts" &
done
copy_hosts "$hostname" "$hosts" &
done
wait
wait
rm "$hosts" "$json"
rm "$hosts" "$json"
}
function make_template {
gcloud compute instances create "$TEMPLATE_NAME" --image "$IMAGE" --zone "$ZONE"
gcloud compute config-ssh --ssh-key-file "$SSH_KEY_FILE"
name="$TEMPLATE_NAME.$ZONE.$PROJECT"
try_connect "$name"
install_docker_on "$name"
gcloud -q compute instances delete "$TEMPLATE_NAME" --keep-disks boot --zone "$ZONE"
gcloud compute images create "$TEMPLATE_NAME" --source-disk "$TEMPLATE_NAME" --source-disk-zone "$ZONE"
function make_template() {
gcloud compute instances create "$TEMPLATE_NAME" --image "$IMAGE" --zone "$ZONE"
gcloud compute config-ssh --ssh-key-file "$SSH_KEY_FILE"
name="$TEMPLATE_NAME.$ZONE.$PROJECT"
try_connect "$name"
install_docker_on "$name"
gcloud -q compute instances delete "$TEMPLATE_NAME" --keep-disks boot --zone "$ZONE"
gcloud compute images create "$TEMPLATE_NAME" --source-disk "$TEMPLATE_NAME" --source-disk-zone "$ZONE"
}
function hosts {
hosts=
args=
json=$(mktemp json.XXXXXXXXXX)
gcloud compute instances list --format=json > "$json"
for name in $(vm_names); do
hostname="$name.$ZONE.$PROJECT"
hosts=( $hostname "${hosts[@]}" )
args=( "--add-host=$hostname:$(internal_ip "$json" "$name")" "${args[@]}" )
done
echo export SSH=\"ssh -l vagrant\"
echo "export HOSTS=\"${hosts[*]}\""
echo "export ADD_HOST_ARGS=\"${args[*]}\""
rm "$json"
function hosts() {
hosts=
args=
json=$(mktemp json.XXXXXXXXXX)
gcloud compute instances list --format=json >"$json"
for name in $(vm_names); do
hostname="$name.$ZONE.$PROJECT"
hosts=($hostname "${hosts[@]}")
args=("--add-host=$hostname:$(internal_ip "$json" "$name")" "${args[@]}")
done
echo export SSH=\"ssh -l vagrant\"
echo "export HOSTS=\"${hosts[*]}\""
echo "export ADD_HOST_ARGS=\"${args[*]}\""
rm "$json"
}
case "$1" in
setup)
setup
;;
setup)
setup
;;
hosts)
hosts
;;
hosts)
hosts
;;
destroy)
destroy
;;
destroy)
destroy
;;
make_template)
# see if template exists
if ! gcloud compute images list | grep "$PROJECT" | grep "$TEMPLATE_NAME"; then
make_template
fi
make_template)
# see if template exists
if ! gcloud compute images list | grep "$PROJECT" | grep "$TEMPLATE_NAME"; then
make_template
fi
;;
esac

View File

@@ -14,17 +14,17 @@ fi
whitely echo ...ok
# shellcheck disable=SC2068
TESTS=( ${@:-$(find . -name '*_test.sh')} )
RUNNER_ARGS=( )
TESTS=(${@:-$(find . -name '*_test.sh')})
RUNNER_ARGS=()
# If running on circle, use the scheduler to work out what tests to run
if [ -n "$CIRCLECI" ] && [ -z "$NO_SCHEDULER" ]; then
RUNNER_ARGS=( "${RUNNER_ARGS[@]}" -scheduler )
RUNNER_ARGS=("${RUNNER_ARGS[@]}" -scheduler)
fi
# If running on circle or PARALLEL is not empty, run tests in parallel
if [ -n "$CIRCLECI" ] || [ -n "$PARALLEL" ]; then
RUNNER_ARGS=( "${RUNNER_ARGS[@]}" -parallel )
RUNNER_ARGS=("${RUNNER_ARGS[@]}" -parallel)
fi
make -C "${DIR}/../runner"