diff --git a/prepare-vms/lib/cli.sh b/prepare-vms/lib/cli.sh index 0290e2ea..22575c0b 100644 --- a/prepare-vms/lib/cli.sh +++ b/prepare-vms/lib/cli.sh @@ -88,3 +88,8 @@ need_settings() { die "Settings file $1 doesn't exist." fi } + +need_login_password() { + USER_LOGIN=$(yq -r .user_login < tags/$TAG/settings.yaml) + USER_PASSWORD=$(yq -r .user_password < tags/$TAG/settings.yaml) +} \ No newline at end of file diff --git a/prepare-vms/lib/clusterize.py b/prepare-vms/lib/clusterize.py new file mode 100644 index 00000000..869b59e9 --- /dev/null +++ b/prepare-vms/lib/clusterize.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python +import os +import sys +import time +import yaml + +################################# + +config = yaml.load(open("/tmp/settings.yaml")) +CLUSTER_SIZE = config["clustersize"] +CLUSTER_PREFIX = config["clusterprefix"] + +################################# + +# This script will be run as ubuntu user, which has root privileges. + +STEP = 0 + +def bold(msg): + return "{} {} {}".format("$(tput smso)", msg, "$(tput rmso)") + +def system(cmd): + global STEP + with open("/tmp/pp.status", "a") as f: + t1 = time.time() + f.write(bold("--- RUNNING [step {}] ---> {}...".format(STEP, cmd))) + retcode = os.system(cmd) + t2 = time.time() + td = str(t2-t1)[:5] + f.write(bold("[{}] in {}s\n".format(retcode, td))) + STEP += 1 + with open(os.environ["HOME"] + "/.bash_history", "a") as f: + f.write("{}\n".format(cmd)) + if retcode != 0: + msg = "The following command failed with exit code {}:\n".format(retcode) + msg+= cmd + raise(Exception(msg)) + +# Get our public IP address +# ipv4_retrieval_endpoint = "http://169.254.169.254/latest/meta-data/public-ipv4" +ipv4_retrieval_endpoint = "http://myip.enix.org/REMOTE_ADDR" +system("curl --silent {} > /tmp/ipv4".format(ipv4_retrieval_endpoint)) +ipv4 = open("/tmp/ipv4").read() +system("echo HOSTIP={} | sudo tee -a /etc/environment".format(ipv4)) + +### BEGIN CLUSTERING ### + +addresses = list(l.strip() for l in sys.stdin) + +assert ipv4 in addresses + +def makenames(addrs): + return [ "%s%s"%(CLUSTER_PREFIX, i+1) for i in range(len(addrs)) ] + +while addresses: + cluster = addresses[:CLUSTER_SIZE] + addresses = addresses[CLUSTER_SIZE:] + if ipv4 not in cluster: + continue + names = makenames(cluster) + for ipaddr, name in zip(cluster, names): + system("grep ^{} /etc/hosts || echo {} {} | sudo tee -a /etc/hosts" + .format(ipaddr, ipaddr, name)) + print(cluster) + + mynode = cluster.index(ipv4) + 1 + system("echo {}{} | sudo tee /etc/hostname".format(CLUSTER_PREFIX, mynode)) + system("sudo hostname {}{}".format(CLUSTER_PREFIX, mynode)) + + # Record the IPV4 and name of the first node + system("echo {} | sudo tee /etc/ipv4_of_first_node".format(cluster[0])) + system("echo {} | sudo tee /etc/name_of_first_node".format(names[0])) + + # Create a convenience file to easily check if we're the first node + if ipv4 == cluster[0]: + system("sudo ln -sf /bin/true /usr/local/bin/i_am_first_node") + else: + system("sudo ln -sf /bin/false /usr/local/bin/i_am_first_node") diff --git a/prepare-vms/lib/commands.sh b/prepare-vms/lib/commands.sh index 3a84943a..8caae092 100644 --- a/prepare-vms/lib/commands.sh +++ b/prepare-vms/lib/commands.sh @@ -57,30 +57,105 @@ _cmd_clean() { done } -_cmd deploy "Install Docker on a bunch of running VMs" -_cmd_deploy() { +_cmd createuser "Create the user that students will use" +_cmd_createuser() { + TAG=$1 + need_tag + need_login_password + + pssh " + set -e + # Create the user if it doesn't exist yet. + id $USER_LOGIN || sudo useradd -d /home/$USER_LOGIN -g users -m -s /bin/bash $USER_LOGIN + # Add them to the docker group, if there is one. + grep ^docker: /etc/group && sudo usermod -aG docker $USER_LOGIN + # Set their password. + echo $USER_LOGIN:$USER_PASSWORD | sudo chpasswd + # Add them to sudoers and allow passwordless authentication. + echo '$USER_LOGIN ALL=(ALL) NOPASSWD:ALL' | sudo tee /etc/sudoers.d/$USER_LOGIN + " + + pssh " + set -e + sudo sed -i 's/PasswordAuthentication no/PasswordAuthentication yes/' /etc/ssh/sshd_config + sudo service ssh restart + " + + pssh " + set -e + cd /home/$USER_LOGIN + sudo -u $USER_LOGIN mkdir -p .ssh + if i_am_first_node; then + # Generate a key pair with an empty passphrase. + if ! sudo -u $USER_LOGIN [ -f .ssh/id_rsa ]; then + sudo -u $USER_LOGIN ssh-keygen -t rsa -f .ssh/id_rsa -P '' + sudo -u $USER_LOGIN cp .ssh/id_rsa.pub .ssh/authorized_keys + fi + fi + " + + pssh " + set -e + cd /home/$USER_LOGIN + if ! i_am_first_node; then + # Copy keys from the first node. + ssh $SSHOPTS \$(cat /etc/name_of_first_node) sudo -u $USER_LOGIN tar -C /home/$USER_LOGIN -cvf- .ssh | + sudo -u $USER_LOGIN tar -xf- + fi + " + + # FIXME do this only once. + pssh -I "sudo -u $USER_LOGIN tee -a /home/$USER_LOGIN/.bashrc" <<"SQRL" + +# Fancy prompt courtesy of @soulshake. +export PS1='\e[1m\e[31m[$HOSTIP] \e[32m($(docker-prompt)) \e[34m\u@\h\e[35m \w\e[0m\n$ ' + +# Bigger history, in a different file, and saved before executing each command. +export HISTSIZE=9999 +export HISTFILESIZE=9999 +shopt -s histappend +trap 'history -a' DEBUG +export HISTFILE=~/.history +SQRL + + pssh -I "sudo -u $USER_LOGIN tee /home/$USER_LOGIN/.vimrc" </dev/stderr echo -n "." - sleep 2 - done - >/dev/stderr echo "" - - echo deploying > tags/$TAG/status - sep "Deploying tag $TAG" - - # If this VM image is using cloud-init, - # wait for cloud-init to be done - pssh " - if [ -d /var/lib/cloud ]; then - while [ ! -f /var/lib/cloud/instance/boot-finished ]; do - sleep 1 - done - fi" + echo clusterizing > tags/$TAG/status + sep "Clusterizing tag $TAG" # Special case for scaleway since it doesn't come with sudo if [ "$INFRACLASS" = "scaleway" ]; then @@ -116,40 +191,20 @@ _cmd_deploy() { #fi" # Copy postprep.py to the remote machines, and execute it, feeding it the list of IP addresses - pssh -I tee /tmp/postprep.py >/tmp/pp.out 2>>/tmp/pp.err" >/tmp/pp.out 2>>/tmp/pp.err" tags/$TAG/status - info "You may want to run one of the following commands:" - info "$0 kube $TAG" - info "$0 pull_images $TAG" - info "$0 cards $TAG" + sep "Clusterized tag $TAG" + echo clusterized > tags/$TAG/status } _cmd disabledocker "Stop Docker Engine and don't restart it automatically" @@ -158,10 +213,51 @@ _cmd_disabledocker() { need_tag pssh " - sudo systemctl disable docker.service - sudo systemctl disable docker.socket - sudo systemctl stop docker - sudo killall containerd + sudo systemctl disable docker.socket --now + sudo systemctl disable docker.service --now + sudo systemctl disable containerd.service --now + " +} + +_cmd docker "Install and start Docker" +_cmd_docker() { + TAG=$1 + need_tag + + pssh " + set -e + # On EC2, the ephemeral disk might be mounted on /mnt. + # If /mnt is a mountpoint, place Docker workspace on it. + if mountpoint -q /mnt; then + sudo mkdir -p /mnt/docker + sudo ln -sfn /mnt/docker /var/lib/docker + fi + + # This will install the latest Docker. + sudo apt-get -qy install apt-transport-https ca-certificates curl software-properties-common + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + sudo add-apt-repository 'deb https://download.docker.com/linux/ubuntu bionic stable' + sudo apt-get -q update + sudo apt-get -qy install docker-ce + " + + pssh " + set -e + ### Install docker-compose. + ##VERSION## https://docs.docker.com/compose/release-notes/ + COMPOSE_VERSION=1.29.2 + sudo curl -fsSL -o /usr/local/bin/docker-compose \ + https://github.com/docker/compose/releases/download/\$COMPOSE_VERSION/docker-compose-\$(uname -s)-\$(uname -m) + sudo chmod +x /usr/local/bin/docker-compose + docker-compose version + + ### Install docker-machine. + ##VERSION## + MACHINE_VERSION=v0.16.2 + sudo curl -fsSL -o /usr/local/bin/docker-machine \ + https://github.com/docker/machine/releases/download/\$MACHINE_VERSION/docker-machine-\$(uname -s)-\$(uname -m) + sudo chmod +x /usr/local/bin/docker-machine + docker-machine version " } @@ -200,6 +296,7 @@ _cmd kube "Setup kubernetes clusters with kubeadm (must be run AFTER deploy)" _cmd_kube() { TAG=$1 need_tag + need_login_password # Optional version, e.g. 1.13.5 KUBEVERSION=$2 @@ -255,14 +352,14 @@ EOF sudo kubeadm init --config=/tmp/kubeadm-config.yaml --ignore-preflight-errors=NumCPU fi" - # Put kubeconfig in ubuntu's and docker's accounts + # Put kubeconfig in ubuntu's and $USER_LOGIN's accounts pssh " if i_am_first_node; then - sudo mkdir -p \$HOME/.kube /home/docker/.kube && + sudo mkdir -p \$HOME/.kube /home/$USER_LOGIN/.kube && sudo cp /etc/kubernetes/admin.conf \$HOME/.kube/config && - sudo cp /etc/kubernetes/admin.conf /home/docker/.kube/config && + sudo cp /etc/kubernetes/admin.conf /home/$USER_LOGIN/.kube/config && sudo chown -R \$(id -u) \$HOME/.kube && - sudo chown -R docker /home/docker/.kube + sudo chown -R $USER_LOGIN /home/$USER_LOGIN/.kube fi" # Install weave as the pod network @@ -291,23 +388,35 @@ _cmd kubetools "Install a bunch of CLI tools for Kubernetes" _cmd_kubetools() { TAG=$1 need_tag + need_login_password # Install kubectx and kubens pssh " - [ -d kubectx ] || git clone https://github.com/ahmetb/kubectx && - sudo ln -sf \$HOME/kubectx/kubectx /usr/local/bin/kctx && - sudo ln -sf \$HOME/kubectx/kubens /usr/local/bin/kns && - sudo cp \$HOME/kubectx/completion/*.bash /etc/bash_completion.d && - [ -d kube-ps1 ] || git clone https://github.com/jonmosco/kube-ps1 && - sudo -u docker sed -i s/docker-prompt/kube_ps1/ /home/docker/.bashrc && - sudo -u docker tee -a /home/docker/.bashrc <&1 >/dev/null; do + >/dev/stderr echo -n "." + sleep 2 + done + >/dev/stderr echo "" + + # If this VM image is using cloud-init, + # wait for cloud-init to be done + info "Waiting for cloud-init to be done on $TAG instances..." + pssh " + if [ -d /var/lib/cloud ]; then + while [ ! -f /var/lib/cloud/instance/boot-finished ]; do + sleep 1 + done + fi" +} + # Sometimes, weave fails to come up on some nodes. # Symptom: the pods on a node are unreachable (they don't even ping). # Remedy: wipe out Weave state and delete weave pod on that node. @@ -872,16 +1023,12 @@ pull_tag() { google/cadvisor \ dockersamples/visualizer \ nathanleclaire/redisonrails; do - sudo -u docker docker pull $I + sudo docker pull $I done' info "Finished pulling images for $TAG." } -tag_is_reachable() { - pssh -t 5 true 2>&1 >/dev/null -} - test_tag() { ips_file=tags/$TAG/ips.txt info "Picking a random IP address in $ips_file to run tests." diff --git a/prepare-vms/lib/postprep.py b/prepare-vms/lib/postprep.py deleted file mode 100755 index 61c3da33..00000000 --- a/prepare-vms/lib/postprep.py +++ /dev/null @@ -1,188 +0,0 @@ -#!/usr/bin/env python -import os -import platform -import sys -import time -import urllib -import yaml - -################################# - -config = yaml.load(open("/tmp/settings.yaml")) -COMPOSE_VERSION = config["compose_version"] -MACHINE_VERSION = config["machine_version"] -CLUSTER_SIZE = config["clustersize"] -CLUSTER_PREFIX = config["clusterprefix"] -ENGINE_VERSION = config["engine_version"] -DOCKER_USER_PASSWORD = config["docker_user_password"] - -################################# - -# This script will be run as ubuntu user, which has root privileges. -# docker commands will require sudo because the ubuntu user has no access to the docker socket. - -STEP = 0 -START = time.time() - -def bold(msg): - return "{} {} {}".format("$(tput smso)", msg, "$(tput rmso)") - -def system(cmd): - global STEP - with open("/tmp/pp.status", "a") as f: - t1 = time.time() - f.write(bold("--- RUNNING [step {}] ---> {}...".format(STEP, cmd))) - retcode = os.system(cmd) - t2 = time.time() - td = str(t2-t1)[:5] - f.write(bold("[{}] in {}s\n".format(retcode, td))) - STEP += 1 - with open(os.environ["HOME"] + "/.bash_history", "a") as f: - f.write("{}\n".format(cmd)) - if retcode != 0: - msg = "The following command failed with exit code {}:\n".format(retcode) - msg+= cmd - raise(Exception(msg)) - - -# On EC2, the ephemeral disk might be mounted on /mnt. -# If /mnt is a mountpoint, place Docker workspace on it. -system("if mountpoint -q /mnt; then sudo mkdir -p /mnt/docker && sudo ln -sfn /mnt/docker /var/lib/docker; fi") - -# Put our public IP in /tmp/ipv4 -# ipv4_retrieval_endpoint = "http://169.254.169.254/latest/meta-data/public-ipv4" -ipv4_retrieval_endpoint = "http://myip.enix.org/REMOTE_ADDR" -system("curl --silent {} > /tmp/ipv4".format(ipv4_retrieval_endpoint)) - -ipv4 = open("/tmp/ipv4").read() - -# Add a "docker" user with password coming from the settings -system("id docker || sudo useradd -d /home/docker -m -s /bin/bash docker") -system("echo docker:{} | sudo chpasswd".format(DOCKER_USER_PASSWORD)) - -# Fancy prompt courtesy of @soulshake. -system("""sudo -u docker tee -a /home/docker/.bashrc <{{ clusternumber + loop.index }} {% endif %} login: - docker + {{ user_login }} password: - {{ docker_user_password }} + {{ user_password }}

diff --git a/prepare-vms/workshopctl b/prepare-vms/workshopctl index 072fc31c..99b26fae 100755 --- a/prepare-vms/workshopctl +++ b/prepare-vms/workshopctl @@ -15,13 +15,14 @@ for lib in lib/*.sh; do done DEPENDENCIES=" - ssh curl fping jq - pssh - wkhtmltopdf man + pssh + ssh + wkhtmltopdf + yq " # Check for missing dependencies, and issue a warning if necessary.