Prepare deployment scripts for Ubuntu 18.04

This adds a few features:
- ./workshopctl kubereset TAG (closes #306)
- remove python-setuptools (prepare for #353)
- ./workshopctl weavetest TAG (help detecting weave issues
  like we had at OSCON, July 2018)
- remove a bit of dead code

prepare-vms/lib/commands.sh

@@ -53,11 +53,18 @@ _cmd_deploy() {
 
     echo deploying > tags/$TAG/status
     sep "Deploying tag $TAG"
+
+    # Wait for cloudinit to be done
+    pssh "
+    while [ ! -f /var/lib/cloud/instance/boot-finished ]; do
+        sleep 1
+    done"
+
+    # Copy settings and install Python YAML parser
     pssh -I tee /tmp/settings.yaml <tags/$TAG/settings.yaml
     pssh "
     sudo apt-get update &&
-    sudo apt-get install -y python-setuptools &&
-    sudo easy_install pyyaml"
+    sudo apt-get install -y python-yaml"
 
     # Copy postprep.py to the remote machines, and execute it, feeding it the list of IP addresses
     pssh -I tee /tmp/postprep.py <lib/postprep.py
@@ -178,6 +185,14 @@ EOF"
     sep "Done"
 }
 
+_cmd kubereset "Wipe out Kubernetes configuration on all nodes"
+_cmd_kubereset() {
+    TAG=$1
+    need_tag
+
+    pssh "sudo kubeadm reset --force"
+}
+
 _cmd kubetest "Check that all nodes are reporting as Ready"
 _cmd_kubetest() {
     TAG=$1
@@ -261,6 +276,15 @@ _cmd_opensg() {
     infra_opensg
 }
 
+_cmd pssh "Run an arbitrary command on all nodes"
+_cmd_pssh() {
+    TAG=$1
+    need_tag
+    shift
+
+    pssh "$@"
+}
+
 _cmd pull_images "Pre-pull a bunch of Docker images"
 _cmd_pull_images() {
     TAG=$1
@@ -376,6 +400,22 @@ _cmd_test() {
     test_tag
 }
 
+# Sometimes, weave fails to come up on some nodes.
+# Symptom: the pods on a node are unreachable (they don't even ping).
+# Remedy: wipe out Weave state and delete weave pod on that node.
+# Specifically, identify the weave pod that is defective, then:
+# kubectl -n kube-system exec weave-net-XXXXX -c weave rm /weavedb/weave-netdata.db
+# kubectl -n kube-system delete pod weave-net-XXXXX
+_cmd weavetest "Check that weave seems properly setup"
+_cmd_weavetest() {
+    TAG=$1
+    need_tag
+    pssh "
+    kubectl -n kube-system get pods -o name | grep weave | cut -d/ -f2 |
+    xargs -I POD kubectl -n kube-system exec POD -c weave -- \
+    sh -c \"./weave --local status | grep Connections | grep -q ' 1 failed' || ! echo POD \""
+}
+
 greet() {
     IAMUSER=$(aws iam get-user --query 'User.UserName')
     info "Hello! You seem to be UNIX user $USER, and IAM user $IAMUSER."
@@ -501,12 +541,3 @@ make_tag() {
     fi
     date +%Y-%m-%d-%H-%M-$USER
 }
-
-describe_tag() {
-    FIXME
-    # Display instance details and reachability/status information
-    TAG=$1
-    need_tag $TAG
-    aws_display_instances_by_tag $TAG
-    aws_display_instance_statuses_by_tag $TAG
-}

prepare-vms/lib/infra/aws.sh

@@ -94,14 +94,11 @@ wait_until_tag_is_running() {
     done_count=0
     while [[ $done_count -lt $COUNT ]]; do
         let "i += 1"
-        info "$(printf "%d instances online" $done_count)"
+        info "$(printf "%d/%d instances online" $done_count $COUNT)"
         done_count=$(aws ec2 describe-instances \
-            --filters "Name=instance-state-name,Values=running" \
-            "Name=tag:Name,Values=$TAG" \
-            --query "Reservations[*].Instances[*].State.Name" \
-            | tr "\t" "\n" \
-            | wc -l)
-
+            --filters "Name=tag:Name,Values=$TAG" \
+            "Name=instance-state-name,Values=running" \
+            --query "length(Reservations[].Instances[])")
         if [[ $i -gt $max_retry ]]; then
             die "Timed out while waiting for instance creation (after $max_retry retries)"
         fi
@@ -165,7 +162,6 @@ aws_get_instance_ids_by_filter() {
 
 aws_get_instance_ids_by_client_token() {
     TOKEN=$1
-    need_tag $TOKEN
     aws_get_instance_ids_by_filter Name=client-token,Values=$TOKEN
 }
 

prepare-vms/lib/postprep.py

@@ -83,7 +83,7 @@ system("sudo sed -i 's/PasswordAuthentication no/PasswordAuthentication yes/' /e
 
 system("sudo service ssh restart")
 system("sudo apt-get -q update")
-system("sudo apt-get -qy install git jq python-pip")
+system("sudo apt-get -qy install git jq")
 
 #######################
 ### DOCKER INSTALLS ###
@@ -98,7 +98,6 @@ system("sudo apt-get -q update")
 system("sudo apt-get -qy install docker-ce")
 
 ### Install docker-compose
-#system("sudo pip install -U docker-compose=={}".format(COMPOSE_VERSION))
 system("sudo curl -sSL -o /usr/local/bin/docker-compose https://github.com/docker/compose/releases/download/{}/docker-compose-{}-{}".format(COMPOSE_VERSION, platform.system(), platform.machine()))
 system("sudo chmod +x /usr/local/bin/docker-compose")
 system("docker-compose version")