🏭️ Big refactoring to move to Ubuntu 22.04

Instead of Ubuntu 18.04, we should use 22.04 (especially as
18.04 will be EOL soon). This moves a few providers to 22.04
(and more will follow).

We now ship a small containerd configuration file (instead
of defaulting to an empty configuration like we did before)
since it looks like recent versions of containerd cause
infinite crashloops if the cgroups driver isn't set properly.

Also, Linode is now provisioned using Terraform (instead of
the old-style system relying on linode-cli) which should make
instance provisioning faster (thanks to Terraform parallelism).

The "wait" command now tries to log in with both "ubuntu" and
"root", and if it fails with "ubuntu" but succeeds with "root",
it will create the "ubuntu" user and give it full sudo rights.

Finally, a "standardize" action has been created to gather all
the commands that deal with non-standard Ubuntu images.

Note that for completeness, we should check that all providers
work correctly; currently only Linode has been validated.

prepare-vms/lib/commands.sh

@@ -67,6 +67,8 @@ _cmd_createuser() {
     set -e
     # Create the user if it doesn't exist yet.
     id $USER_LOGIN || sudo useradd -d /home/$USER_LOGIN -g users -m -s /bin/bash $USER_LOGIN
+    # Make sure there are at least exec permission on their home.
+    sudo chmod a+X /home/$USER_LOGIN
     # Add them to the docker group, if there is one.
     grep ^docker: /etc/group && sudo usermod -aG docker $USER_LOGIN
     # Set their password.
@@ -154,8 +156,8 @@ SQRL
     echo user_ok > tags/$TAG/status
 }
 
-_cmd clusterize "Group VMs in clusters"
-_cmd_clusterize() {
+_cmd standardize "Deal with non-standard Ubuntu cloud images"
+_cmd_standardize() {
     TAG=$1
     need_tag
 
@@ -165,24 +167,11 @@ _cmd_clusterize() {
     # Special case for scaleway since it doesn't come with sudo
     if [ "$INFRACLASS" = "scaleway" ]; then
         pssh -l root "
-    grep DEBIAN_FRONTEND /etc/environment || echo DEBIAN_FRONTEND=noninteractive >> /etc/environment
-    grep cloud-init /etc/sudoers && rm /etc/sudoers
-    apt-get update && apt-get install sudo -y"
+        grep DEBIAN_FRONTEND /etc/environment || echo DEBIAN_FRONTEND=noninteractive >> /etc/environment
+        grep cloud-init /etc/sudoers && rm /etc/sudoers
+        apt-get update && apt-get install sudo -y"
     fi
 
-    # FIXME
-    # Special case for hetzner since it doesn't have an ubuntu user
-    #if [ "$INFRACLASS" = "hetzner" ]; then
-    #    pssh -l root "
-    #[ -d /home/ubuntu ] ||
-    #    useradd ubuntu -m -s /bin/bash
-    #echo 'ubuntu ALL=(ALL:ALL) NOPASSWD:ALL' > /etc/sudoers.d/ubuntu
-    #[ -d /home/ubuntu/.ssh ] ||
-    #    install --owner=ubuntu --mode=700 --directory /home/ubuntu/.ssh
-    #[ -f /home/ubuntu/.ssh/authorized_keys ] ||
-    #    install --owner=ubuntu --mode=600 /root/.ssh/authorized_keys --target-directory /home/ubuntu/.ssh"
-    #fi
-
     # Special case for oracle since their iptables blocks everything but SSH
     pssh "
     if [ -f /etc/iptables/rules.v4 ]; then
@@ -203,18 +192,18 @@ _cmd_clusterize() {
         sudo snap remove oracle-cloud-agent
         sudo dpkg --remove --force-remove-reinstreq unified-monitoring-agent
     fi"
+}
+
+_cmd clusterize "Group VMs in clusters"
+_cmd_clusterize() {
+    TAG=$1
+    need_tag
 
     # Copy settings and install Python YAML parser
     pssh -I tee /tmp/settings.yaml <tags/$TAG/settings.yaml
     pssh "
     sudo apt-get update &&
-    sudo apt-get install -y python-yaml"
-
-    # If there is no "python" binary, symlink to python3
-    pssh "
-    if ! which python; then
-        sudo ln -s $(which python3) /usr/local/bin/python
-    fi"
+    sudo apt-get install -y python3-yaml python-is-python3"
 
     # Copy postprep.py to the remote machines, and execute it, feeding it the list of IP addresses
     pssh -I tee /tmp/clusterize.py <lib/clusterize.py
@@ -261,7 +250,7 @@ _cmd_docker() {
     # This will install the latest Docker.
     sudo apt-get -qy install apt-transport-https ca-certificates curl software-properties-common
     curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
-    sudo add-apt-repository 'deb https://download.docker.com/linux/ubuntu bionic stable'
+    sudo add-apt-repository 'deb https://download.docker.com/linux/ubuntu jammy stable'
     sudo apt-get -q update
     sudo apt-get -qy install docker-ce
 
@@ -372,10 +361,12 @@ EOF"
         sudo swapoff -a"
     fi
 
-    # Re-enable CRI interface in containerd
-    pssh "
-    echo '# Use default parameters for containerd.' | sudo tee /etc/containerd/config.toml
-    sudo systemctl restart containerd"
+    # Install a valid configuration for containerd
+    # (first, the CRI interface needs to be re-enabled;
+    # also, the correct systemd cgroup driver must be selected,
+    # otherwise containerd just restarts containers for no good reason)
+    pssh -I "sudo tee /etc/containerd/config.toml" < lib/containerd-config.toml
+    pssh "sudo systemctl restart containerd"
 
     # Initialize kube control plane
     pssh --timeout 200 "
@@ -387,8 +378,6 @@ apiVersion: kubeadm.k8s.io/v1beta2
 bootstrapTokens:
 - token: \$(cat /tmp/token)
 nodeRegistration:
-  # Comment out the next line to switch back to Docker.
-  criSocket: /run/containerd/containerd.sock
   ignorePreflightErrors:
   - NumCPU
 ---
@@ -400,16 +389,12 @@ discovery:
     token: \$(cat /tmp/token)
     unsafeSkipCAVerification: true
 nodeRegistration:
-  # Comment out the next line to switch back to Docker.
-  criSocket: /run/containerd/containerd.sock
   ignorePreflightErrors:
   - NumCPU
 ---
 kind: KubeletConfiguration
 apiVersion: kubelet.config.k8s.io/v1beta1
-# The following line is necessary when using Docker.
-# It doesn't seem necessary when using containerd.
-#cgroupDriver: cgroupfs
+failSwapOn: false
 ---
 kind: ClusterConfiguration
 apiVersion: kubeadm.k8s.io/v1beta2
@@ -433,8 +418,6 @@ EOF
     # Install weave as the pod network
     pssh "
     if i_am_first_node; then
-        #kubever=\$(kubectl version | base64 | tr -d '\n') &&
-        #kubectl apply -f https://cloud.weave.works/k8s/net?k8s-version=\$kubever
         kubectl apply -f https://github.com/weaveworks/weave/releases/download/v2.8.1/weave-daemonset-k8s-1.11.yaml
     fi"
 
@@ -804,8 +787,6 @@ _cmd_tools() {
     sudo apt-get -qy install apache2-utils emacs-nox git httping htop jid joe jq mosh python-setuptools tree unzip
     # This is for VMs with broken PRNG (symptom: running docker-compose randomly hangs)
     sudo apt-get -qy install haveged
-    # I don't remember why we need to remove this
-    sudo apt-get remove -y --purge dnsmasq-base
     "
 }
 
@@ -1060,15 +1041,22 @@ _cmd_passwords() {
     info "Done."
 }
 
-_cmd wait "Wait until VMs are ready (reachable and cloud init is done)"
+_cmd wait "Wait until VMs are ready (reachable, cloud init is done, ubuntu user is up)"
 _cmd_wait() {
     TAG=$1
     need_tag
 
     # Wait until all hosts are reachable.
     info "Trying to reach $TAG instances..."
-    while ! pssh -t 5 true 2>&1 >/dev/null; do
-        >/dev/stderr echo -n "."
+    while >/dev/stderr echo -n "."; do
+        pssh -t 5 true 2>&1 >/dev/null && {
+            SSH_USER=ubuntu
+            break
+        }
+        pssh -l root -t 5 true 2>&1 >/dev/null && {
+            SSH_USER=root
+            break
+        }
         sleep 2
     done
     >/dev/stderr echo ""
@@ -1076,12 +1064,26 @@ _cmd_wait() {
     # If this VM image is using cloud-init,
     # wait for cloud-init to be done
     info "Waiting for cloud-init to be done on $TAG instances..."
-    pssh "
+    pssh -l $SSH_USER "
     if [ -d /var/lib/cloud ]; then
-        while [ ! -f /var/lib/cloud/instance/boot-finished ]; do
-            sleep 1
-        done
+        cloud-init status --wait
     fi"
+
+    if [ "$SSH_USER" = "root" ]; then
+        pssh -l root "
+        getent passwd ubuntu || {
+            useradd ubuntu -m -s /bin/bash
+            echo 'ubuntu ALL=(ALL:ALL) NOPASSWD:ALL' > /etc/sudoers.d/ubuntu
+        }
+        [ -d /home/ubuntu/.ssh ] ||
+            install --owner=ubuntu --mode=700 --directory /home/ubuntu/.ssh
+        [ -f /home/ubuntu/.ssh/authorized_keys ] ||
+            install --owner=ubuntu --mode=600 /root/.ssh/authorized_keys --target-directory /home/ubuntu/.ssh
+        "
+    fi
+
+    # Now make sure that we have an ubuntu user
+    pssh true
 }
 
 # Sometimes, weave fails to come up on some nodes.
@@ -1106,7 +1108,6 @@ _cmd_webssh() {
     need_tag
     pssh "
     sudo apt-get update &&
-    sudo apt-get install python-tornado python-paramiko -y ||
     sudo apt-get install python3-tornado python3-paramiko -y"
     pssh "
     cd /opt

prepare-vms/lib/containerd-config.toml

@@ -0,0 +1,7 @@
+version = 2
+[plugins."io.containerd.grpc.v1.cri".containerd]
+default_runtime_name = "runc"
+[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+runtime_type = "io.containerd.runc.v2"
+[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+SystemdCgroup = true

prepare-vms/lib/infra/linode.sh

@@ -1,70 +0,0 @@
-if ! command -v linode-cli >/dev/null; then
-  warning "Linode CLI (linode-cli) not found."
-fi
-if ! [ -f ~/.config/linode-cli ]; then
-  warning "~/.config/linode-cli not found."
-fi
-
-# To view available regions: "linode-cli regions list"
-LINODE_REGION=${LINODE_REGION-us-west}
-
-# To view available types: "linode-cli linodes types"
-LINODE_TYPE=${LINODE_TYPE-g6-standard-2}
-
-infra_list() {
-    linode-cli linodes list --json |
-        jq -r '.[] | [.id, .label, .status, .type] | @tsv'
-}
-
-infra_start() {
-    COUNT=$1
-
-    for I in $(seq 1 $COUNT); do
-        NAME=$(printf "%s-%03d" $TAG $I)
-        sep "Starting instance $I/$COUNT"
-        info "          Zone: $LINODE_REGION"
-        info "          Name: $NAME"
-        info " Instance type: $LINODE_TYPE"
-        ROOT_PASS="$(base64 /dev/urandom | cut -c1-20 | head -n 1)"
-        MAX_TRY=5
-        TRY=1
-        WAIT=1
-        while ! linode-cli linodes create \
-            --type=${LINODE_TYPE} --region=${LINODE_REGION} \
-            --image=linode/ubuntu18.04 \
-            --authorized_keys="${LINODE_SSHKEY}" \
-            --root_pass="${ROOT_PASS}" \
-            --tags=${TAG} --label=${NAME}; do
-              warning "Failed to create VM (attempt $TRY/$MAX_TRY)."
-              if [ $TRY -ge $MAX_TRY ]; then
-                  die "Giving up."
-              fi
-              info "Waiting $WAIT seconds and retrying."
-              sleep $WAIT
-              TRY=$(($TRY+1))
-              WAIT=$(($WAIT*2))
-            done
-    done
-    sep
-
-    linode_get_ips_by_tag $TAG > tags/$TAG/ips.txt
-}
-
-infra_stop() {
-    info "Counting instances..."
-    linode_get_ids_by_tag $TAG | wc -l
-    info "Deleting instances..."
-    linode_get_ids_by_tag $TAG | 
-        xargs -n1 -P10 \
-        linode-cli linodes delete
-}
-
-linode_get_ids_by_tag() {
-    TAG=$1
-    linode-cli linodes list --tags $TAG --json | jq -r ".[].id"
-}
-
-linode_get_ips_by_tag() {
-    TAG=$1
-    linode-cli linodes list --tags $TAG --json | jq -r ".[].ipv4[0]"
-}

prepare-vms/settings/admin-dmuc.yaml

@@ -18,6 +18,7 @@ image:
 
 steps:
   - wait
+  - standardize
   - clusterize
   - tools
   - docker

prepare-vms/settings/admin-kubenet.yaml

@@ -20,6 +20,7 @@ image:
 steps:
   - disableaddrchecks
   - wait
+  - standardize
   - clusterize
   - tools
   - docker

prepare-vms/settings/admin-kuberouter.yaml

@@ -20,6 +20,7 @@ image:
 steps:
   - disableaddrchecks
   - wait
+  - standardize
   - clusterize
   - tools
   - docker

prepare-vms/settings/admin-oldversion.yaml

@@ -22,6 +22,7 @@ image:
 
 steps:
   - wait
+  - standardize
   - clusterize
   - tools
   - docker

prepare-vms/settings/admin-test.yaml

@@ -18,6 +18,7 @@ image:
 
 steps:
   - wait
+  - standardize
   - clusterize
   - tools
   - docker

prepare-vms/settings/csv.yaml

@@ -1,8 +0,0 @@
-# Number of VMs per cluster
-clustersize: 5
-
-# The hostname of each node will be clusterprefix + a number
-clusterprefix: node
-
-# Jinja2 template to use to generate ready-to-cut cards
-cards_template: clusters.csv

prepare-vms/settings/docker.yaml

@@ -18,6 +18,7 @@ user_password: training
 
 steps:
   - wait
+  - standardize
   - clusterize
   - tools
   - docker

prepare-vms/settings/example.yaml

@@ -1,17 +0,0 @@
-# customize your cluster size, your cards template, and the versions
-
-# Number of VMs per cluster
-clustersize: 5
-
-# The hostname of each node will be clusterprefix + a number
-clusterprefix: node
-
-# Jinja2 template to use to generate ready-to-cut cards
-cards_template: cards.html
-
-# Use "Letter" in the US, and "A4" everywhere else
-paper_size: Letter
-
-# Login and password that students will use
-user_login: docker
-user_password: training

prepare-vms/settings/kubernetes.yaml

@@ -16,6 +16,7 @@ user_password: training
 
 steps:
   - wait
+  - standardize
   - clusterize
   - tools
   - docker

prepare-vms/settings/portal.yaml

@@ -18,6 +18,7 @@ user_password: CHANGEME
 
 steps:
   - wait
+  - standardize
   - clusterize
   - tools
   - docker

prepare-vms/settings/swarm.yaml

@@ -1,17 +0,0 @@
-# This file is passed by trainer-cli to scripts/ips-txt-to-html.py
-
-# Number of VMs per cluster
-clustersize: 3
-
-# The hostname of each node will be clusterprefix + a number
-clusterprefix: node
-
-# Jinja2 template to use to generate ready-to-cut cards
-cards_template: cards.html
-
-# Use "Letter" in the US, and "A4" everywhere else
-paper_size: Letter
-
-# Login and password that students will use
-user_login: docker
-user_password: training

prepare-vms/terraform/azure/main.tf

@@ -1,5 +1,5 @@
 resource "azurerm_resource_group" "_" {
-  name          = var.prefix
+  name     = var.prefix
   location = var.location
 }
 
@@ -8,7 +8,7 @@ resource "azurerm_public_ip" "_" {
   name                = format("%s-%04d", var.prefix, count.index + 1)
   location            = azurerm_resource_group._.location
   resource_group_name = azurerm_resource_group._.name
-  allocation_method = "Dynamic"
+  allocation_method   = "Dynamic"
 }
 
 resource "azurerm_network_interface" "_" {
@@ -21,7 +21,7 @@ resource "azurerm_network_interface" "_" {
     name                          = "internal"
     subnet_id                     = azurerm_subnet._.id
     private_ip_address_allocation = "Dynamic"
-    public_ip_address_id = azurerm_public_ip._[count.index].id
+    public_ip_address_id          = azurerm_public_ip._[count.index].id
   }
 }
 
@@ -49,7 +49,7 @@ resource "azurerm_linux_virtual_machine" "_" {
   source_image_reference {
     publisher = "Canonical"
     offer     = "UbuntuServer"
-    sku       = "18.04-LTS" # FIXME
+    sku       = "22.04-LTS"
     version   = "latest"
   }
 }
@@ -63,7 +63,7 @@ data "azurerm_public_ip" "_" {
   count               = var.how_many_nodes
   name                = format("%s-%04d", var.prefix, count.index + 1)
   resource_group_name = azurerm_resource_group._.name
-  depends_on = [azurerm_linux_virtual_machine._]
+  depends_on          = [azurerm_linux_virtual_machine._]
 }
 
 output "ip_addresses" {

prepare-vms/terraform/linode/main.tf

@@ -0,0 +1,12 @@
+resource "linode_instance" "_" {
+  count           = var.how_many_nodes
+  label           = format("%s-%04d", var.prefix, count.index + 1)
+  region          = var.location
+  type            = var.size
+  authorized_keys = local.authorized_keys
+  image           = "linode/ubuntu22.04"
+}
+
+output "ip_addresses" {
+  value = join("", formatlist("%s\n", linode_instance._.*.ip_address))
+}

prepare-vms/terraform/linode/provider.tf

@@ -0,0 +1,12 @@
+terraform {
+  required_version = ">= 1"
+  required_providers {
+    linode = {
+      source = "linode/linode"
+    }
+  }
+}
+
+provider "linode" {
+  token = regex("\ntoken *= *([0-9a-f]+)\n", file("~/.config/linode-cli"))[0]
+}

prepare-vms/terraform/linode/variables.tf

@@ -0,0 +1,32 @@
+variable "prefix" {
+  type    = string
+  default = "provisioned-with-terraform"
+}
+
+variable "how_many_nodes" {
+  type    = number
+  default = 2
+}
+
+locals {
+  authorized_keys = split("\n", trimspace(file("~/.ssh/id_rsa.pub")))
+}
+
+/*
+Available sizes:
+"g6-standard-1" # CPU=1 RAM=2
+"g6-standard-2" # CPU=2 RAM=4
+"g6-standard-4" # CPU=4 RAM=8
+"g6-standard-6" # CPU=6 RAM=16
+"g6-standard-8" # CPU=8 RAM=32
+*/
+
+variable "size" {
+  type    = string
+  default = "g6-standard-2"
+}
+
+variable "location" {
+  type    = string
+  default = "eu-west"
+}

prepare-vms/terraform/oci/main.tf

@@ -16,7 +16,7 @@ data "oci_core_images" "_" {
   compartment_id           = local.compartment_id
   shape                    = var.shape
   operating_system         = "Canonical Ubuntu"
-  operating_system_version = "20.04"
+  operating_system_version = "22.04"
   #operating_system         = "Oracle Linux"
   #operating_system_version = "7.9"
 }

prepare-vms/terraform/openstack/machines.tf

@@ -1,6 +1,6 @@
 resource "openstack_compute_instance_v2" "machine" {
   count           = var.how_many_nodes
-  name            = format("%s-%04d", var.prefix, count.index+1)
+  name            = format("%s-%04d", var.prefix, count.index + 1)
   image_name      = var.image
   flavor_name     = var.flavor
   security_groups = [openstack_networking_secgroup_v2.full_access.name]
@@ -8,27 +8,23 @@ resource "openstack_compute_instance_v2" "machine" {
 
   network {
     name        = openstack_networking_network_v2.internal.name
-    fixed_ip_v4 = cidrhost(openstack_networking_subnet_v2.internal.cidr, count.index+10)
+    fixed_ip_v4 = cidrhost(openstack_networking_subnet_v2.internal.cidr, count.index + 10)
   }
 }
 
 resource "openstack_compute_floatingip_v2" "machine" {
-  count           = var.how_many_nodes
+  count = var.how_many_nodes
   # This is something provided to us by Enix when our tenant was provisioned.
   pool = "Public Floating"
 }
 
 resource "openstack_compute_floatingip_associate_v2" "machine" {
-  count           = var.how_many_nodes
+  count       = var.how_many_nodes
   floating_ip = openstack_compute_floatingip_v2.machine.*.address[count.index]
   instance_id = openstack_compute_instance_v2.machine.*.id[count.index]
-  fixed_ip    = cidrhost(openstack_networking_subnet_v2.internal.cidr, count.index+10)
+  fixed_ip    = cidrhost(openstack_networking_subnet_v2.internal.cidr, count.index + 10)
 }
 
 output "ip_addresses" {
   value = join("", formatlist("%s\n", openstack_compute_floatingip_v2.machine.*.address))
 }
-
-variable "flavor" {}
-
-variable "image" {}

prepare-vms/terraform/openstack/network.tf

@@ -1,5 +1,5 @@
 resource "openstack_networking_network_v2" "internal" {
-  name           = var.prefix
+  name = var.prefix
 }
 
 resource "openstack_networking_subnet_v2" "internal" {

prepare-vms/terraform/openstack/provider.tf

@@ -1,5 +1,5 @@
 terraform {
-required_version = ">= 1"
+  required_version = ">= 1"
   required_providers {
     openstack = {
       source  = "terraform-provider-openstack/openstack"

prepare-vms/terraform/openstack/variables.tf

@@ -0,0 +1,16 @@
+variable "prefix" {
+  type = string
+}
+
+variable "how_many_nodes" {
+  type = number
+}
+
+variable "flavor" {
+  type = string
+}
+
+variable "image" {
+  type    = string
+  default = "Ubuntu 22.04"
+}

prepare-vms/terraform/openstack/vars.tf

@@ -1,7 +0,0 @@
-variable "prefix" {
-  type = string
-}
-
-variable "how_many_nodes" {
-  type = number
-}