From c9820376ffb34f33ea9811da3186df6e5118bae9 Mon Sep 17 00:00:00 2001 From: Jerome Petazzoni Date: Tue, 19 Oct 2021 18:45:41 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=8F=AD=EF=B8=8F=20Refactor=20deployment?= =?UTF-8?q?=20code;=20allow=20changing=20user=20login?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It is now possible to set the user login (instead of having it hardcoded to "docker"). Also, various actions have been broken out in separate functions to facilitate future maintenance. --- prepare-vms/lib/cli.sh | 5 + prepare-vms/lib/clusterize.py | 78 +++++ prepare-vms/lib/commands.sh | 317 +++++++++++++++------ prepare-vms/lib/postprep.py | 188 ------------ prepare-vms/settings/admin-dmuc.yaml | 26 +- prepare-vms/settings/admin-kubenet.yaml | 26 +- prepare-vms/settings/admin-kuberouter.yaml | 26 +- prepare-vms/settings/admin-test.yaml | 26 +- prepare-vms/settings/docker.yaml | 19 +- prepare-vms/settings/example.yaml | 12 +- prepare-vms/settings/kubernetes.yaml | 21 +- prepare-vms/settings/swarm.yaml | 12 +- prepare-vms/setup-admin-clusters.sh | 43 +-- prepare-vms/templates/cards.html | 4 +- prepare-vms/workshopctl | 7 +- 15 files changed, 415 insertions(+), 395 deletions(-) create mode 100644 prepare-vms/lib/clusterize.py delete mode 100755 prepare-vms/lib/postprep.py diff --git a/prepare-vms/lib/cli.sh b/prepare-vms/lib/cli.sh index 0290e2ea..22575c0b 100644 --- a/prepare-vms/lib/cli.sh +++ b/prepare-vms/lib/cli.sh @@ -88,3 +88,8 @@ need_settings() { die "Settings file $1 doesn't exist." fi } + +need_login_password() { + USER_LOGIN=$(yq -r .user_login < tags/$TAG/settings.yaml) + USER_PASSWORD=$(yq -r .user_password < tags/$TAG/settings.yaml) +} \ No newline at end of file diff --git a/prepare-vms/lib/clusterize.py b/prepare-vms/lib/clusterize.py new file mode 100644 index 00000000..869b59e9 --- /dev/null +++ b/prepare-vms/lib/clusterize.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python +import os +import sys +import time +import yaml + +################################# + +config = yaml.load(open("/tmp/settings.yaml")) +CLUSTER_SIZE = config["clustersize"] +CLUSTER_PREFIX = config["clusterprefix"] + +################################# + +# This script will be run as ubuntu user, which has root privileges. + +STEP = 0 + +def bold(msg): + return "{} {} {}".format("$(tput smso)", msg, "$(tput rmso)") + +def system(cmd): + global STEP + with open("/tmp/pp.status", "a") as f: + t1 = time.time() + f.write(bold("--- RUNNING [step {}] ---> {}...".format(STEP, cmd))) + retcode = os.system(cmd) + t2 = time.time() + td = str(t2-t1)[:5] + f.write(bold("[{}] in {}s\n".format(retcode, td))) + STEP += 1 + with open(os.environ["HOME"] + "/.bash_history", "a") as f: + f.write("{}\n".format(cmd)) + if retcode != 0: + msg = "The following command failed with exit code {}:\n".format(retcode) + msg+= cmd + raise(Exception(msg)) + +# Get our public IP address +# ipv4_retrieval_endpoint = "http://169.254.169.254/latest/meta-data/public-ipv4" +ipv4_retrieval_endpoint = "http://myip.enix.org/REMOTE_ADDR" +system("curl --silent {} > /tmp/ipv4".format(ipv4_retrieval_endpoint)) +ipv4 = open("/tmp/ipv4").read() +system("echo HOSTIP={} | sudo tee -a /etc/environment".format(ipv4)) + +### BEGIN CLUSTERING ### + +addresses = list(l.strip() for l in sys.stdin) + +assert ipv4 in addresses + +def makenames(addrs): + return [ "%s%s"%(CLUSTER_PREFIX, i+1) for i in range(len(addrs)) ] + +while addresses: + cluster = addresses[:CLUSTER_SIZE] + addresses = addresses[CLUSTER_SIZE:] + if ipv4 not in cluster: + continue + names = makenames(cluster) + for ipaddr, name in zip(cluster, names): + system("grep ^{} /etc/hosts || echo {} {} | sudo tee -a /etc/hosts" + .format(ipaddr, ipaddr, name)) + print(cluster) + + mynode = cluster.index(ipv4) + 1 + system("echo {}{} | sudo tee /etc/hostname".format(CLUSTER_PREFIX, mynode)) + system("sudo hostname {}{}".format(CLUSTER_PREFIX, mynode)) + + # Record the IPV4 and name of the first node + system("echo {} | sudo tee /etc/ipv4_of_first_node".format(cluster[0])) + system("echo {} | sudo tee /etc/name_of_first_node".format(names[0])) + + # Create a convenience file to easily check if we're the first node + if ipv4 == cluster[0]: + system("sudo ln -sf /bin/true /usr/local/bin/i_am_first_node") + else: + system("sudo ln -sf /bin/false /usr/local/bin/i_am_first_node") diff --git a/prepare-vms/lib/commands.sh b/prepare-vms/lib/commands.sh index 3a84943a..8caae092 100644 --- a/prepare-vms/lib/commands.sh +++ b/prepare-vms/lib/commands.sh @@ -57,30 +57,105 @@ _cmd_clean() { done } -_cmd deploy "Install Docker on a bunch of running VMs" -_cmd_deploy() { +_cmd createuser "Create the user that students will use" +_cmd_createuser() { + TAG=$1 + need_tag + need_login_password + + pssh " + set -e + # Create the user if it doesn't exist yet. + id $USER_LOGIN || sudo useradd -d /home/$USER_LOGIN -g users -m -s /bin/bash $USER_LOGIN + # Add them to the docker group, if there is one. + grep ^docker: /etc/group && sudo usermod -aG docker $USER_LOGIN + # Set their password. + echo $USER_LOGIN:$USER_PASSWORD | sudo chpasswd + # Add them to sudoers and allow passwordless authentication. + echo '$USER_LOGIN ALL=(ALL) NOPASSWD:ALL' | sudo tee /etc/sudoers.d/$USER_LOGIN + " + + pssh " + set -e + sudo sed -i 's/PasswordAuthentication no/PasswordAuthentication yes/' /etc/ssh/sshd_config + sudo service ssh restart + " + + pssh " + set -e + cd /home/$USER_LOGIN + sudo -u $USER_LOGIN mkdir -p .ssh + if i_am_first_node; then + # Generate a key pair with an empty passphrase. + if ! sudo -u $USER_LOGIN [ -f .ssh/id_rsa ]; then + sudo -u $USER_LOGIN ssh-keygen -t rsa -f .ssh/id_rsa -P '' + sudo -u $USER_LOGIN cp .ssh/id_rsa.pub .ssh/authorized_keys + fi + fi + " + + pssh " + set -e + cd /home/$USER_LOGIN + if ! i_am_first_node; then + # Copy keys from the first node. + ssh $SSHOPTS \$(cat /etc/name_of_first_node) sudo -u $USER_LOGIN tar -C /home/$USER_LOGIN -cvf- .ssh | + sudo -u $USER_LOGIN tar -xf- + fi + " + + # FIXME do this only once. + pssh -I "sudo -u $USER_LOGIN tee -a /home/$USER_LOGIN/.bashrc" <<"SQRL" + +# Fancy prompt courtesy of @soulshake. +export PS1='\e[1m\e[31m[$HOSTIP] \e[32m($(docker-prompt)) \e[34m\u@\h\e[35m \w\e[0m\n$ ' + +# Bigger history, in a different file, and saved before executing each command. +export HISTSIZE=9999 +export HISTFILESIZE=9999 +shopt -s histappend +trap 'history -a' DEBUG +export HISTFILE=~/.history +SQRL + + pssh -I "sudo -u $USER_LOGIN tee /home/$USER_LOGIN/.vimrc" </dev/stderr echo -n "." - sleep 2 - done - >/dev/stderr echo "" - - echo deploying > tags/$TAG/status - sep "Deploying tag $TAG" - - # If this VM image is using cloud-init, - # wait for cloud-init to be done - pssh " - if [ -d /var/lib/cloud ]; then - while [ ! -f /var/lib/cloud/instance/boot-finished ]; do - sleep 1 - done - fi" + echo clusterizing > tags/$TAG/status + sep "Clusterizing tag $TAG" # Special case for scaleway since it doesn't come with sudo if [ "$INFRACLASS" = "scaleway" ]; then @@ -116,40 +191,20 @@ _cmd_deploy() { #fi" # Copy postprep.py to the remote machines, and execute it, feeding it the list of IP addresses - pssh -I tee /tmp/postprep.py >/tmp/pp.out 2>>/tmp/pp.err" >/tmp/pp.out 2>>/tmp/pp.err" tags/$TAG/status - info "You may want to run one of the following commands:" - info "$0 kube $TAG" - info "$0 pull_images $TAG" - info "$0 cards $TAG" + sep "Clusterized tag $TAG" + echo clusterized > tags/$TAG/status } _cmd disabledocker "Stop Docker Engine and don't restart it automatically" @@ -158,10 +213,51 @@ _cmd_disabledocker() { need_tag pssh " - sudo systemctl disable docker.service - sudo systemctl disable docker.socket - sudo systemctl stop docker - sudo killall containerd + sudo systemctl disable docker.socket --now + sudo systemctl disable docker.service --now + sudo systemctl disable containerd.service --now + " +} + +_cmd docker "Install and start Docker" +_cmd_docker() { + TAG=$1 + need_tag + + pssh " + set -e + # On EC2, the ephemeral disk might be mounted on /mnt. + # If /mnt is a mountpoint, place Docker workspace on it. + if mountpoint -q /mnt; then + sudo mkdir -p /mnt/docker + sudo ln -sfn /mnt/docker /var/lib/docker + fi + + # This will install the latest Docker. + sudo apt-get -qy install apt-transport-https ca-certificates curl software-properties-common + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + sudo add-apt-repository 'deb https://download.docker.com/linux/ubuntu bionic stable' + sudo apt-get -q update + sudo apt-get -qy install docker-ce + " + + pssh " + set -e + ### Install docker-compose. + ##VERSION## https://docs.docker.com/compose/release-notes/ + COMPOSE_VERSION=1.29.2 + sudo curl -fsSL -o /usr/local/bin/docker-compose \ + https://github.com/docker/compose/releases/download/\$COMPOSE_VERSION/docker-compose-\$(uname -s)-\$(uname -m) + sudo chmod +x /usr/local/bin/docker-compose + docker-compose version + + ### Install docker-machine. + ##VERSION## + MACHINE_VERSION=v0.16.2 + sudo curl -fsSL -o /usr/local/bin/docker-machine \ + https://github.com/docker/machine/releases/download/\$MACHINE_VERSION/docker-machine-\$(uname -s)-\$(uname -m) + sudo chmod +x /usr/local/bin/docker-machine + docker-machine version " } @@ -200,6 +296,7 @@ _cmd kube "Setup kubernetes clusters with kubeadm (must be run AFTER deploy)" _cmd_kube() { TAG=$1 need_tag + need_login_password # Optional version, e.g. 1.13.5 KUBEVERSION=$2 @@ -255,14 +352,14 @@ EOF sudo kubeadm init --config=/tmp/kubeadm-config.yaml --ignore-preflight-errors=NumCPU fi" - # Put kubeconfig in ubuntu's and docker's accounts + # Put kubeconfig in ubuntu's and $USER_LOGIN's accounts pssh " if i_am_first_node; then - sudo mkdir -p \$HOME/.kube /home/docker/.kube && + sudo mkdir -p \$HOME/.kube /home/$USER_LOGIN/.kube && sudo cp /etc/kubernetes/admin.conf \$HOME/.kube/config && - sudo cp /etc/kubernetes/admin.conf /home/docker/.kube/config && + sudo cp /etc/kubernetes/admin.conf /home/$USER_LOGIN/.kube/config && sudo chown -R \$(id -u) \$HOME/.kube && - sudo chown -R docker /home/docker/.kube + sudo chown -R $USER_LOGIN /home/$USER_LOGIN/.kube fi" # Install weave as the pod network @@ -291,23 +388,35 @@ _cmd kubetools "Install a bunch of CLI tools for Kubernetes" _cmd_kubetools() { TAG=$1 need_tag + need_login_password # Install kubectx and kubens pssh " - [ -d kubectx ] || git clone https://github.com/ahmetb/kubectx && - sudo ln -sf \$HOME/kubectx/kubectx /usr/local/bin/kctx && - sudo ln -sf \$HOME/kubectx/kubens /usr/local/bin/kns && - sudo cp \$HOME/kubectx/completion/*.bash /etc/bash_completion.d && - [ -d kube-ps1 ] || git clone https://github.com/jonmosco/kube-ps1 && - sudo -u docker sed -i s/docker-prompt/kube_ps1/ /home/docker/.bashrc && - sudo -u docker tee -a /home/docker/.bashrc <&1 >/dev/null; do + >/dev/stderr echo -n "." + sleep 2 + done + >/dev/stderr echo "" + + # If this VM image is using cloud-init, + # wait for cloud-init to be done + info "Waiting for cloud-init to be done on $TAG instances..." + pssh " + if [ -d /var/lib/cloud ]; then + while [ ! -f /var/lib/cloud/instance/boot-finished ]; do + sleep 1 + done + fi" +} + # Sometimes, weave fails to come up on some nodes. # Symptom: the pods on a node are unreachable (they don't even ping). # Remedy: wipe out Weave state and delete weave pod on that node. @@ -872,16 +1023,12 @@ pull_tag() { google/cadvisor \ dockersamples/visualizer \ nathanleclaire/redisonrails; do - sudo -u docker docker pull $I + sudo docker pull $I done' info "Finished pulling images for $TAG." } -tag_is_reachable() { - pssh -t 5 true 2>&1 >/dev/null -} - test_tag() { ips_file=tags/$TAG/ips.txt info "Picking a random IP address in $ips_file to run tests." diff --git a/prepare-vms/lib/postprep.py b/prepare-vms/lib/postprep.py deleted file mode 100755 index 61c3da33..00000000 --- a/prepare-vms/lib/postprep.py +++ /dev/null @@ -1,188 +0,0 @@ -#!/usr/bin/env python -import os -import platform -import sys -import time -import urllib -import yaml - -################################# - -config = yaml.load(open("/tmp/settings.yaml")) -COMPOSE_VERSION = config["compose_version"] -MACHINE_VERSION = config["machine_version"] -CLUSTER_SIZE = config["clustersize"] -CLUSTER_PREFIX = config["clusterprefix"] -ENGINE_VERSION = config["engine_version"] -DOCKER_USER_PASSWORD = config["docker_user_password"] - -################################# - -# This script will be run as ubuntu user, which has root privileges. -# docker commands will require sudo because the ubuntu user has no access to the docker socket. - -STEP = 0 -START = time.time() - -def bold(msg): - return "{} {} {}".format("$(tput smso)", msg, "$(tput rmso)") - -def system(cmd): - global STEP - with open("/tmp/pp.status", "a") as f: - t1 = time.time() - f.write(bold("--- RUNNING [step {}] ---> {}...".format(STEP, cmd))) - retcode = os.system(cmd) - t2 = time.time() - td = str(t2-t1)[:5] - f.write(bold("[{}] in {}s\n".format(retcode, td))) - STEP += 1 - with open(os.environ["HOME"] + "/.bash_history", "a") as f: - f.write("{}\n".format(cmd)) - if retcode != 0: - msg = "The following command failed with exit code {}:\n".format(retcode) - msg+= cmd - raise(Exception(msg)) - - -# On EC2, the ephemeral disk might be mounted on /mnt. -# If /mnt is a mountpoint, place Docker workspace on it. -system("if mountpoint -q /mnt; then sudo mkdir -p /mnt/docker && sudo ln -sfn /mnt/docker /var/lib/docker; fi") - -# Put our public IP in /tmp/ipv4 -# ipv4_retrieval_endpoint = "http://169.254.169.254/latest/meta-data/public-ipv4" -ipv4_retrieval_endpoint = "http://myip.enix.org/REMOTE_ADDR" -system("curl --silent {} > /tmp/ipv4".format(ipv4_retrieval_endpoint)) - -ipv4 = open("/tmp/ipv4").read() - -# Add a "docker" user with password coming from the settings -system("id docker || sudo useradd -d /home/docker -m -s /bin/bash docker") -system("echo docker:{} | sudo chpasswd".format(DOCKER_USER_PASSWORD)) - -# Fancy prompt courtesy of @soulshake. -system("""sudo -u docker tee -a /home/docker/.bashrc <{{ clusternumber + loop.index }} {% endif %} login: - docker + {{ user_login }} password: - {{ docker_user_password }} + {{ user_password }}

diff --git a/prepare-vms/workshopctl b/prepare-vms/workshopctl index 072fc31c..99b26fae 100755 --- a/prepare-vms/workshopctl +++ b/prepare-vms/workshopctl @@ -15,13 +15,14 @@ for lib in lib/*.sh; do done DEPENDENCIES=" - ssh curl fping jq - pssh - wkhtmltopdf man + pssh + ssh + wkhtmltopdf + yq " # Check for missing dependencies, and issue a warning if necessary.