Big refactor of deployment script

Add support for OVHcloud, Hetzner; refactor Scaleway support

prepare-vms/infra/example.openstack-cli

@@ -0,0 +1,24 @@
+INFRACLASS=openstack-cli
+
+# Copy that file to e.g. openstack or ovh, then customize it.
+# Some Openstack providers (like OVHcloud) will let you download
+# a file containing credentials. That's what you need to use.
+# The file below contains some example values.
+export OS_AUTH_URL=https://auth.cloud.ovh.net/v3/
+export OS_IDENTITY_API_VERSION=3
+export OS_USER_DOMAIN_NAME=${OS_USER_DOMAIN_NAME:-"Default"}
+export OS_PROJECT_DOMAIN_NAME=${OS_PROJECT_DOMAIN_NAME:-"Default"}
+export OS_TENANT_ID=abcd1234
+export OS_TENANT_NAME="0123456"
+export OS_USERNAME="user-xyz123"
+export OS_PASSWORD=AbCd1234
+export OS_REGION_NAME="GRA7"
+
+# And then some values to indicate server type, image, etc.
+# You can see available flavors with `openstack flavor list`
+export OS_FLAVOR=s1-4
+# You can see available images with `openstack image list`
+export OS_IMAGE=896c5f54-51dc-44f0-8c22-ce99ba7164df
+# You can create a key with `openstack keypair create --public-key ~/.ssh/id_rsa.pub containertraining`
+export OS_KEY=containertraining
+

prepare-vms/infra/example.openstack-tf

@@ -1,4 +1,5 @@
-INFRACLASS=openstack
+INFRACLASS=openstack-tf
+
 # If you are using OpenStack, copy this file (e.g. to "openstack" or "enix")
 # and customize the variables below.
 export TF_VAR_user="jpetazzo"
@@ -6,4 +7,4 @@ export TF_VAR_tenant="training"
 export TF_VAR_domain="Default"
 export TF_VAR_password="..."
 export TF_VAR_auth_url="https://api.r1.nxs.enix.io/v3"
-export TF_VAR_flavor="GP1.S"
+export TF_VAR_flavor="GP1.S"

prepare-vms/infra/hetzner

@@ -0,0 +1,5 @@
+INFRACLASS=hetzner
+if ! [ -f ~/.config/hcloud/cli.toml ]; then
+  warn "~/.config/hcloud/cli.toml not found."
+  warn "Make sure that the Hetzner CLI (hcloud) is installed and configured."
+fi

prepare-vms/infra/scaleway

@@ -1,5 +1 @@
 INFRACLASS=scaleway
-if ! [ -f ~/.config/scw/config.yaml ]; then
-  warn "~/.config/scw/config.yaml not found."
-  warn "Make sure that the scaleway CLI is installed and configured."
-fi

prepare-vms/lib/commands.sh

@@ -73,6 +73,19 @@ _cmd_deploy() {
     apt-get update && apt-get install sudo -y"
     fi
 
+    # FIXME
+    # Special case for hetzner since it doesn't have an ubuntu user
+    #if [ "$INFRACLASS" = "hetzner" ]; then
+    #    pssh -l root "
+    #[ -d /home/ubuntu ] ||
+    #    useradd ubuntu -m -s /bin/bash
+    #echo 'ubuntu ALL=(ALL:ALL) NOPASSWD:ALL' > /etc/sudoers.d/ubuntu
+    #[ -d /home/ubuntu/.ssh ] ||
+    #    install --owner=ubuntu --mode=700 --directory /home/ubuntu/.ssh
+    #[ -f /home/ubuntu/.ssh/authorized_keys ] ||
+    #    install --owner=ubuntu --mode=600 /root/.ssh/authorized_keys --target-directory /home/ubuntu/.ssh"
+    #fi
+
     # Copy settings and install Python YAML parser
     pssh -I tee /tmp/settings.yaml <tags/$TAG/settings.yaml
     pssh "
@@ -187,7 +200,7 @@ _cmd_kube() {
     pssh --timeout 200 "
     if i_am_first_node && [ ! -f /etc/kubernetes/admin.conf ]; then
         kubeadm token generate > /tmp/token &&
-	sudo kubeadm init $EXTRA_KUBEADM --token \$(cat /tmp/token) --apiserver-cert-extra-sans \$(cat /tmp/ipv4)
+	sudo kubeadm init $EXTRA_KUBEADM --token \$(cat /tmp/token) --apiserver-cert-extra-sans \$(cat /tmp/ipv4) --ignore-preflight-errors=NumCPU
     fi"
 
     # Put kubeconfig in ubuntu's and docker's accounts
@@ -224,13 +237,13 @@ _cmd_kube() {
     # Install kubectx and kubens
     pssh "
     [ -d kubectx ] || git clone https://github.com/ahmetb/kubectx &&
-    sudo ln -sf /home/ubuntu/kubectx/kubectx /usr/local/bin/kctx &&
-    sudo ln -sf /home/ubuntu/kubectx/kubens /usr/local/bin/kns &&
-    sudo cp /home/ubuntu/kubectx/completion/*.bash /etc/bash_completion.d &&
+    sudo ln -sf \$HOME/kubectx/kubectx /usr/local/bin/kctx &&
+    sudo ln -sf \$HOME/kubectx/kubens /usr/local/bin/kns &&
+    sudo cp \$HOME/kubectx/completion/*.bash /etc/bash_completion.d &&
     [ -d kube-ps1 ] || git clone https://github.com/jonmosco/kube-ps1 &&
     sudo -u docker sed -i s/docker-prompt/kube_ps1/ /home/docker/.bashrc &&
     sudo -u docker tee -a /home/docker/.bashrc <<EOF
-. /home/ubuntu/kube-ps1/kube-ps1.sh
+. \$HOME/kube-ps1/kube-ps1.sh
 KUBE_PS1_PREFIX=""
 KUBE_PS1_SUFFIX=""
 KUBE_PS1_SYMBOL_ENABLE="false"
@@ -303,25 +316,12 @@ _cmd_kubetest() {
     set -e
     if i_am_first_node; then
       which kubectl
-      for NODE in \$(awk /[0-9]\$/\ {print\ \\\$2} /etc/hosts); do
+      for NODE in \$(grep [0-9]\$ /etc/hosts | grep -v ^127 | awk {print\ \\\$2}); do
         echo \$NODE ; kubectl get nodes | grep -w \$NODE | grep -w Ready
       done
     fi"
 }
 
-_cmd ids "(FIXME) List the instance IDs belonging to a given tag or token"
-_cmd_ids() {
-    TAG=$1
-    need_tag $TAG
-
-    info "Looking up by tag:"
-    aws_get_instance_ids_by_tag $TAG
-
-    # Just in case we managed to create instances but weren't able to tag them
-    info "Looking up by token:"
-    aws_get_instance_ids_by_client_token $TAG
-}
-
 _cmd ips "Show the IP addresses for a given tag"
 _cmd_ips() {
     TAG=$1
@@ -435,16 +435,6 @@ _cmd_opensg() {
     infra_opensg
 }
 
-_cmd portworx "Prepare the nodes for Portworx deployment"
-_cmd_portworx() {
-    TAG=$1
-    need_tag
-
-    pssh "
-    sudo truncate --size 10G /portworx.blk &&
-    sudo losetup /dev/loop4 /portworx.blk"
-}
-
 _cmd disableaddrchecks "Disable source/destination IP address checks"
 _cmd_disableaddrchecks() {
     TAG=$1
@@ -489,18 +479,6 @@ _cmd_quotas() {
     infra_quotas
 }
 
-_cmd retag "(FIXME) Apply a new tag to a group of VMs"
-_cmd_retag() {
-    OLDTAG=$1
-    NEWTAG=$2
-    TAG=$OLDTAG
-    need_tag
-    if [[ -z "$NEWTAG" ]]; then
-        die "You must specify a new tag to apply."
-    fi
-    aws_tag_instances $OLDTAG $NEWTAG
-}
-
 _cmd ssh "Open an SSH session to the first node of a tag"
 _cmd_ssh() {
     TAG=$1
@@ -687,11 +665,12 @@ _cmd_webssh() {
     sudo apt-get update &&
     sudo apt-get install python-tornado python-paramiko -y"
     pssh "
-    [ -d webssh ] || git clone https://github.com/jpetazzo/webssh"
+    cd /opt
+    [ -d webssh ] || sudo git clone https://github.com/jpetazzo/webssh"
     pssh "
     for KEYFILE in /etc/ssh/*.pub; do
       read a b c < \$KEYFILE; echo localhost \$a \$b
-    done > webssh/known_hosts"
+    done | sudo tee /opt/webssh/known_hosts"
     pssh "cat >webssh.service <<EOF
 [Unit]
 Description=webssh
@@ -700,7 +679,7 @@ Description=webssh
 WantedBy=multi-user.target
 
 [Service]
-WorkingDirectory=/home/ubuntu/webssh
+WorkingDirectory=/opt/webssh
 ExecStart=/usr/bin/env python run.py --fbidhttp=false --port=1080 --policy=reject
 User=nobody
 Group=nogroup
@@ -723,11 +702,6 @@ _cmd_www() {
     python3 -m http.server
 }
 
-greet() {
-    IAMUSER=$(aws iam get-user --query 'User.UserName')
-    info "Hello! You seem to be UNIX user $USER, and IAM user $IAMUSER."
-}
-
 pull_tag() {
     # Pre-pull a bunch of images
     pssh --timeout 900 'for I in \
@@ -817,27 +791,3 @@ make_key_name() {
     SHORT_FINGERPRINT=$(ssh-add -l | grep RSA | head -n1 | cut -d " " -f 2 | tr -d : | cut -c 1-8)
     echo "${SHORT_FINGERPRINT}-${USER}"
 }
-
-sync_keys() {
-    # make sure ssh-add -l contains "RSA"
-    ssh-add -l | grep -q RSA \
-        || die "The output of \`ssh-add -l\` doesn't contain 'RSA'. Start the agent, add your keys?"
-
-    AWS_KEY_NAME=$(make_key_name)
-    info "Syncing keys... "
-    if ! aws ec2 describe-key-pairs --key-name "$AWS_KEY_NAME" &>/dev/null; then
-        aws ec2 import-key-pair --key-name $AWS_KEY_NAME \
-            --public-key-material "$(ssh-add -L \
-                | grep -i RSA \
-                | head -n1 \
-                | cut -d " " -f 1-2)" &>/dev/null
-
-        if ! aws ec2 describe-key-pairs --key-name "$AWS_KEY_NAME" &>/dev/null; then
-            die "Somehow, importing the key didn't work. Make sure that 'ssh-add -l | grep RSA | head -n1' returns an RSA key?"
-        else
-            info "Imported new key $AWS_KEY_NAME."
-        fi
-    else
-        info "Using existing key $AWS_KEY_NAME."
-    fi
-}

prepare-vms/lib/infra/aws.sh

@@ -1,9 +1,13 @@
+if ! command -v aws >/dev/null; then
+    warn "AWS CLI (aws) not found."
+fi
+
 infra_list() {
     aws_display_tags
 }
 
 infra_quotas() {
-    greet
+    aws_greet
 
     max_instances=$(aws ec2 describe-account-attributes \
         --attribute-names max-instances \
@@ -21,10 +25,10 @@ infra_start() {
     COUNT=$1
 
     # Print our AWS username, to ease the pain of credential-juggling
-    greet
+    aws_greet
 
     # Upload our SSH keys to AWS if needed, to be added to each VM's authorized_keys
-    key_name=$(sync_keys)
+    key_name=$(aws_sync_keys)
 
     AMI=$(aws_get_ami)    # Retrieve the AWS image ID
     if [ -z "$AMI" ]; then
@@ -61,7 +65,7 @@ infra_start() {
     aws_tag_instances $TAG $TAG
 
     # Wait until EC2 API tells us that the instances are running
-    wait_until_tag_is_running $TAG $COUNT
+    aws_wait_until_tag_is_running $TAG $COUNT
 
     aws_get_instance_ips_by_tag $TAG > tags/$TAG/ips.txt
 }
@@ -98,7 +102,7 @@ infra_disableaddrchecks() {
     done
 }
 
-wait_until_tag_is_running() {
+aws_wait_until_tag_is_running() {
     max_retry=100
     i=0
     done_count=0
@@ -214,3 +218,32 @@ aws_get_ami() {
     ##VERSION##
     find_ubuntu_ami -r $AWS_DEFAULT_REGION -a amd64 -v 18.04 -t hvm:ebs -N -q
 }
+
+aws_greet() {
+    IAMUSER=$(aws iam get-user --query 'User.UserName')
+    info "Hello! You seem to be UNIX user $USER, and IAM user $IAMUSER."
+}
+
+aws_sync_keys() {
+    # make sure ssh-add -l contains "RSA"
+    ssh-add -l | grep -q RSA \
+        || die "The output of \`ssh-add -l\` doesn't contain 'RSA'. Start the agent, add your keys?"
+
+    AWS_KEY_NAME=$(make_key_name)
+    info "Syncing keys... "
+    if ! aws ec2 describe-key-pairs --key-name "$AWS_KEY_NAME" &>/dev/null; then
+        aws ec2 import-key-pair --key-name $AWS_KEY_NAME \
+            --public-key-material "$(ssh-add -L \
+                | grep -i RSA \
+                | head -n1 \
+                | cut -d " " -f 1-2)" &>/dev/null
+
+        if ! aws ec2 describe-key-pairs --key-name "$AWS_KEY_NAME" &>/dev/null; then
+            die "Somehow, importing the key didn't work. Make sure that 'ssh-add -l | grep RSA | head -n1' returns an RSA key?"
+        else
+            info "Imported new key $AWS_KEY_NAME."
+        fi
+    else
+        info "Using existing key $AWS_KEY_NAME."
+    fi
+}

prepare-vms/lib/infra/hetzner.sh

@@ -0,0 +1,50 @@
+if ! command -v hcloud >/dev/null; then
+  warn "Hetzner CLI (hcloud) not found."
+fi
+if ! [ -f ~/.config/hcloud/cli.toml ]; then
+  warn "~/.config/hcloud/cli.toml not found."
+fi
+
+infra_start() {
+    COUNT=$1
+
+    HETZNER_INSTANCE_TYPE=${HETZNER_INSTANCE_TYPE-cx21}
+    HETZNER_DATACENTER=${HETZNER_DATACENTER-nbg1-dc3}
+    HETZNER_IMAGE=${HETZNER_IMAGE-168855}
+
+    for I in $(seq 1 $COUNT); do
+        NAME=$(printf "%s-%03d" $TAG $I)
+        sep "Starting instance $I/$COUNT"
+        info "    Datacenter: $HETZNER_DATACENTER"
+        info "          Name: $NAME"
+        info " Instance type: $HETZNER_INSTANCE_TYPE"
+        hcloud server create \
+            --type=${HETZNER_INSTANCE_TYPE} \
+            --datacenter=${HETZNER_DATACENTER} \
+            --image=${HETZNER_IMAGE} \
+            --name=$NAME \
+            --label=tag=$TAG \
+            --ssh-key ~/.ssh/id_rsa.pub
+    done
+
+    hetzner_get_ips_by_tag $TAG > tags/$TAG/ips.txt
+}
+
+infra_stop() {
+    for ID in $(hetzner_get_ids_by_tag $TAG); do
+        info "Scheduling deletion of instance $ID..."
+        hcloud server delete $ID &
+    done
+    info "Waiting for deletion to complete..."
+    wait
+}
+
+hetzner_get_ids_by_tag() {
+    TAG=$1
+    hcloud server list --selector=tag=$TAG -o json | jq -r .[].name
+}
+
+hetzner_get_ips_by_tag() {
+    TAG=$1
+    hcloud server list --selector=tag=$TAG -o json | jq -r .[].public_net.ipv4.ip
+}

prepare-vms/lib/infra/openstack-cli.sh

@@ -0,0 +1,48 @@
+infra_start() {
+    COUNT=$1
+
+    sep "Starting $COUNT instances"
+    info " Region: $OS_REGION_NAME"
+    info "   User: $OS_USERNAME"
+    info " Flavor: $OS_FLAVOR"
+    info "  Image: $OS_IMAGE"
+    openstack server create \
+    	--flavor $OS_FLAVOR \
+    	--image $OS_IMAGE \
+    	--key-name $OS_KEY \
+    	--min $COUNT --max $COUNT \
+    	--property workshopctl=$TAG \
+    	$TAG
+
+    sep "Waiting for IP addresses to be available"
+    GOT=0
+    while [ "$GOT" != "$COUNT" ]; do
+    	echo "Got $GOT/$COUNT IP addresses."
+    	oscli_get_ips_by_tag $TAG > tags/$TAG/ips.txt
+    	GOT="$(wc -l < tags/$TAG/ips.txt)"
+    done
+
+}
+
+infra_stop() {
+	info "Counting instances..."
+	oscli_get_instances_json $TAG | 
+		jq -r .[].Name | 
+		wc -l
+	info "Deleting instances..."
+	oscli_get_instances_json $TAG |
+		jq -r .[].Name |
+		xargs -P10 -n1 openstack server delete
+	info "Done."
+}
+
+oscli_get_instances_json() {
+	TAG=$1
+    openstack server list -f json --name "${TAG}-[0-9]*"
+}
+
+oscli_get_ips_by_tag() {
+    TAG=$1
+    oscli_get_instances_json $TAG |
+    	jq -r .[].Networks | cut -d= -f2 | cut -d, -f1 | grep . || true
+}

prepare-vms/lib/infra/scaleway.sh

@@ -1,15 +1,13 @@
-infra_list() {
-    die "unimplemented"
-}
-
-infra_quotas() {
-    die "unimplemented"
-}
+if ! command -v scw >/dev/null; then
+  warn "Scaleway CLI (scw) not found."
+fi
+if ! [ -f ~/.config/scw/config.yaml ]; then
+  warn "~/.config/scw/config.yaml not found."
+fi
 
 infra_start() {
     COUNT=$1
 
-    AWS_KEY_NAME=$(make_key_name)
     SCW_INSTANCE_TYPE=${SCW_INSTANCE_TYPE-DEV1-M}
     SCW_ZONE=${SCW_ZONE-fr-par-1}
 
@@ -29,12 +27,12 @@ infra_start() {
 }
 
 infra_stop() {
-    for ID in $(scw_get_ids_by_tag $TAG); do
-        info "Scheduling deletion of instance $ID..."
-        scw instance server delete force-shutdown=true server-id=$ID &
-    done
-    info "Waiting for deletion to complete..."
-    wait
+    info "Counting instances..."
+    scw_get_ids_by_tag $TAG | wc -l
+    info "Deleting instances..."
+    scw_get_ids_by_tag $TAG | 
+        xargs -n1 -P10 -I@@ \
+        scw instance server delete force-shutdown=true server-id=@@
 }
 
 scw_get_ids_by_tag() {
@@ -46,11 +44,3 @@ scw_get_ips_by_tag() {
     TAG=$1
     scw instance server list name=$TAG -o json | jq -r .[].public_ip.address
 }
-
-infra_opensg() {
-    die "unimplemented"
-}
-
-infra_disableaddrchecks() {
-    die "unimplemented"
-}

prepare-vms/lib/infra/unimplemented.sh

@@ -0,0 +1,23 @@
+infra_disableaddrchecks() {
+    die "unimplemented"
+}
+
+infra_list() {
+    die "unimplemented"
+}
+
+infra_opensg() {
+    die "unimplemented"
+}
+
+infra_quotas() {
+    die "unimplemented"
+}
+
+infra_start() {
+    die "unimplemented"
+}
+
+infra_stop() {
+    die "unimplemented"
+}

prepare-vms/lib/postprep.py

@@ -37,7 +37,7 @@ def system(cmd):
         td = str(t2-t1)[:5]
         f.write(bold("[{}] in {}s\n".format(retcode, td)))
         STEP += 1
-    with open("/home/ubuntu/.bash_history", "a") as f:
+    with open(os.environ["HOME"] + "/.bash_history", "a") as f:
         f.write("{}\n".format(cmd))
     if retcode != 0:
         msg = "The following command failed with exit code {}:\n".format(retcode)

prepare-vms/lib/pssh.sh

@@ -18,7 +18,13 @@ pssh() {
     echo "[parallel-ssh] $@"
     export PSSH=$(which pssh || which parallel-ssh)
 
-    $PSSH -h $HOSTFILE -l ubuntu \
+    if [ "$INFRACLASS" = hetzner ]; then
+        LOGIN=root
+    else
+        LOGIN=ubuntu
+    fi
+
+    $PSSH -h $HOSTFILE -l $LOGIN \
         --par 100 \
         -O LogLevel=ERROR \
         -O UserKnownHostsFile=/dev/null \

prepare-vms/terraform/machines.tf

@@ -1,7 +1,7 @@
 resource "openstack_compute_instance_v2" "machine" {
   count           = "${var.count}" 
   name            = "${format("%s-%04d", "${var.prefix}", count.index+1)}"
-  image_name      = "Ubuntu 16.04.5 (Xenial Xerus)"
+  image_name      = "Ubuntu 18.04.4 20200324"
   flavor_name     = "${var.flavor}"
   security_groups = ["${openstack_networking_secgroup_v2.full_access.name}"]
   key_pair        = "${openstack_compute_keypair_v2.ssh_deploy_key.name}"

prepare-vms/workshopctl

@@ -15,7 +15,6 @@ for lib in lib/*.sh; do
 done
 
 DEPENDENCIES="
-    aws
     ssh
     curl
     jq