From 70e60d7f4e1a2f7b976a202f960a3b8012e70a7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Petazzoni?= Date: Thu, 23 Feb 2023 16:27:32 +0100 Subject: [PATCH] =?UTF-8?q?=F0=9F=8F=AD=EF=B8=8F=20Big=20refactoring=20to?= =?UTF-8?q?=20move=20to=20Ubuntu=2022.04?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of Ubuntu 18.04, we should use 22.04 (especially as 18.04 will be EOL soon). This moves a few providers to 22.04 (and more will follow). We now ship a small containerd configuration file (instead of defaulting to an empty configuration like we did before) since it looks like recent versions of containerd cause infinite crashloops if the cgroups driver isn't set properly. Also, Linode is now provisioned using Terraform (instead of the old-style system relying on linode-cli) which should make instance provisioning faster (thanks to Terraform parallelism). The "wait" command now tries to log in with both "ubuntu" and "root", and if it fails with "ubuntu" but succeeds with "root", it will create the "ubuntu" user and give it full sudo rights. Finally, a "standardize" action has been created to gather all the commands that deal with non-standard Ubuntu images. Note that for completeness, we should check that all providers work correctly; currently only Linode has been validated. --- prepare-vms/lib/commands.sh | 99 ++++++++++---------- prepare-vms/lib/containerd-config.toml | 7 ++ prepare-vms/lib/infra/linode.sh | 70 -------------- prepare-vms/settings/admin-dmuc.yaml | 1 + prepare-vms/settings/admin-kubenet.yaml | 1 + prepare-vms/settings/admin-kuberouter.yaml | 1 + prepare-vms/settings/admin-oldversion.yaml | 1 + prepare-vms/settings/admin-test.yaml | 1 + prepare-vms/settings/csv.yaml | 8 -- prepare-vms/settings/docker.yaml | 1 + prepare-vms/settings/example.yaml | 17 ---- prepare-vms/settings/kubernetes.yaml | 1 + prepare-vms/settings/portal.yaml | 1 + prepare-vms/settings/swarm.yaml | 17 ---- prepare-vms/terraform/azure/main.tf | 10 +- prepare-vms/terraform/linode/main.tf | 12 +++ prepare-vms/terraform/linode/provider.tf | 12 +++ prepare-vms/terraform/linode/variables.tf | 32 +++++++ prepare-vms/terraform/oci/main.tf | 2 +- prepare-vms/terraform/openstack/machines.tf | 14 +-- prepare-vms/terraform/openstack/network.tf | 2 +- prepare-vms/terraform/openstack/provider.tf | 2 +- prepare-vms/terraform/openstack/variables.tf | 16 ++++ prepare-vms/terraform/openstack/vars.tf | 7 -- 24 files changed, 150 insertions(+), 185 deletions(-) create mode 100644 prepare-vms/lib/containerd-config.toml delete mode 100644 prepare-vms/lib/infra/linode.sh delete mode 100644 prepare-vms/settings/csv.yaml delete mode 100644 prepare-vms/settings/example.yaml delete mode 100644 prepare-vms/settings/swarm.yaml create mode 100644 prepare-vms/terraform/linode/main.tf create mode 100644 prepare-vms/terraform/linode/provider.tf create mode 100644 prepare-vms/terraform/linode/variables.tf create mode 100644 prepare-vms/terraform/openstack/variables.tf delete mode 100644 prepare-vms/terraform/openstack/vars.tf diff --git a/prepare-vms/lib/commands.sh b/prepare-vms/lib/commands.sh index 5b89953a..df6fab5a 100644 --- a/prepare-vms/lib/commands.sh +++ b/prepare-vms/lib/commands.sh @@ -67,6 +67,8 @@ _cmd_createuser() { set -e # Create the user if it doesn't exist yet. id $USER_LOGIN || sudo useradd -d /home/$USER_LOGIN -g users -m -s /bin/bash $USER_LOGIN + # Make sure there are at least exec permission on their home. + sudo chmod a+X /home/$USER_LOGIN # Add them to the docker group, if there is one. grep ^docker: /etc/group && sudo usermod -aG docker $USER_LOGIN # Set their password. @@ -154,8 +156,8 @@ SQRL echo user_ok > tags/$TAG/status } -_cmd clusterize "Group VMs in clusters" -_cmd_clusterize() { +_cmd standardize "Deal with non-standard Ubuntu cloud images" +_cmd_standardize() { TAG=$1 need_tag @@ -165,24 +167,11 @@ _cmd_clusterize() { # Special case for scaleway since it doesn't come with sudo if [ "$INFRACLASS" = "scaleway" ]; then pssh -l root " - grep DEBIAN_FRONTEND /etc/environment || echo DEBIAN_FRONTEND=noninteractive >> /etc/environment - grep cloud-init /etc/sudoers && rm /etc/sudoers - apt-get update && apt-get install sudo -y" + grep DEBIAN_FRONTEND /etc/environment || echo DEBIAN_FRONTEND=noninteractive >> /etc/environment + grep cloud-init /etc/sudoers && rm /etc/sudoers + apt-get update && apt-get install sudo -y" fi - # FIXME - # Special case for hetzner since it doesn't have an ubuntu user - #if [ "$INFRACLASS" = "hetzner" ]; then - # pssh -l root " - #[ -d /home/ubuntu ] || - # useradd ubuntu -m -s /bin/bash - #echo 'ubuntu ALL=(ALL:ALL) NOPASSWD:ALL' > /etc/sudoers.d/ubuntu - #[ -d /home/ubuntu/.ssh ] || - # install --owner=ubuntu --mode=700 --directory /home/ubuntu/.ssh - #[ -f /home/ubuntu/.ssh/authorized_keys ] || - # install --owner=ubuntu --mode=600 /root/.ssh/authorized_keys --target-directory /home/ubuntu/.ssh" - #fi - # Special case for oracle since their iptables blocks everything but SSH pssh " if [ -f /etc/iptables/rules.v4 ]; then @@ -203,18 +192,18 @@ _cmd_clusterize() { sudo snap remove oracle-cloud-agent sudo dpkg --remove --force-remove-reinstreq unified-monitoring-agent fi" +} + +_cmd clusterize "Group VMs in clusters" +_cmd_clusterize() { + TAG=$1 + need_tag # Copy settings and install Python YAML parser pssh -I tee /tmp/settings.yaml &1 >/dev/null; do - >/dev/stderr echo -n "." + while >/dev/stderr echo -n "."; do + pssh -t 5 true 2>&1 >/dev/null && { + SSH_USER=ubuntu + break + } + pssh -l root -t 5 true 2>&1 >/dev/null && { + SSH_USER=root + break + } sleep 2 done >/dev/stderr echo "" @@ -1076,12 +1064,26 @@ _cmd_wait() { # If this VM image is using cloud-init, # wait for cloud-init to be done info "Waiting for cloud-init to be done on $TAG instances..." - pssh " + pssh -l $SSH_USER " if [ -d /var/lib/cloud ]; then - while [ ! -f /var/lib/cloud/instance/boot-finished ]; do - sleep 1 - done + cloud-init status --wait fi" + + if [ "$SSH_USER" = "root" ]; then + pssh -l root " + getent passwd ubuntu || { + useradd ubuntu -m -s /bin/bash + echo 'ubuntu ALL=(ALL:ALL) NOPASSWD:ALL' > /etc/sudoers.d/ubuntu + } + [ -d /home/ubuntu/.ssh ] || + install --owner=ubuntu --mode=700 --directory /home/ubuntu/.ssh + [ -f /home/ubuntu/.ssh/authorized_keys ] || + install --owner=ubuntu --mode=600 /root/.ssh/authorized_keys --target-directory /home/ubuntu/.ssh + " + fi + + # Now make sure that we have an ubuntu user + pssh true } # Sometimes, weave fails to come up on some nodes. @@ -1106,7 +1108,6 @@ _cmd_webssh() { need_tag pssh " sudo apt-get update && - sudo apt-get install python-tornado python-paramiko -y || sudo apt-get install python3-tornado python3-paramiko -y" pssh " cd /opt diff --git a/prepare-vms/lib/containerd-config.toml b/prepare-vms/lib/containerd-config.toml new file mode 100644 index 00000000..580564a4 --- /dev/null +++ b/prepare-vms/lib/containerd-config.toml @@ -0,0 +1,7 @@ +version = 2 +[plugins."io.containerd.grpc.v1.cri".containerd] +default_runtime_name = "runc" +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] +runtime_type = "io.containerd.runc.v2" +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] +SystemdCgroup = true diff --git a/prepare-vms/lib/infra/linode.sh b/prepare-vms/lib/infra/linode.sh deleted file mode 100644 index c541963c..00000000 --- a/prepare-vms/lib/infra/linode.sh +++ /dev/null @@ -1,70 +0,0 @@ -if ! command -v linode-cli >/dev/null; then - warning "Linode CLI (linode-cli) not found." -fi -if ! [ -f ~/.config/linode-cli ]; then - warning "~/.config/linode-cli not found." -fi - -# To view available regions: "linode-cli regions list" -LINODE_REGION=${LINODE_REGION-us-west} - -# To view available types: "linode-cli linodes types" -LINODE_TYPE=${LINODE_TYPE-g6-standard-2} - -infra_list() { - linode-cli linodes list --json | - jq -r '.[] | [.id, .label, .status, .type] | @tsv' -} - -infra_start() { - COUNT=$1 - - for I in $(seq 1 $COUNT); do - NAME=$(printf "%s-%03d" $TAG $I) - sep "Starting instance $I/$COUNT" - info " Zone: $LINODE_REGION" - info " Name: $NAME" - info " Instance type: $LINODE_TYPE" - ROOT_PASS="$(base64 /dev/urandom | cut -c1-20 | head -n 1)" - MAX_TRY=5 - TRY=1 - WAIT=1 - while ! linode-cli linodes create \ - --type=${LINODE_TYPE} --region=${LINODE_REGION} \ - --image=linode/ubuntu18.04 \ - --authorized_keys="${LINODE_SSHKEY}" \ - --root_pass="${ROOT_PASS}" \ - --tags=${TAG} --label=${NAME}; do - warning "Failed to create VM (attempt $TRY/$MAX_TRY)." - if [ $TRY -ge $MAX_TRY ]; then - die "Giving up." - fi - info "Waiting $WAIT seconds and retrying." - sleep $WAIT - TRY=$(($TRY+1)) - WAIT=$(($WAIT*2)) - done - done - sep - - linode_get_ips_by_tag $TAG > tags/$TAG/ips.txt -} - -infra_stop() { - info "Counting instances..." - linode_get_ids_by_tag $TAG | wc -l - info "Deleting instances..." - linode_get_ids_by_tag $TAG | - xargs -n1 -P10 \ - linode-cli linodes delete -} - -linode_get_ids_by_tag() { - TAG=$1 - linode-cli linodes list --tags $TAG --json | jq -r ".[].id" -} - -linode_get_ips_by_tag() { - TAG=$1 - linode-cli linodes list --tags $TAG --json | jq -r ".[].ipv4[0]" -} diff --git a/prepare-vms/settings/admin-dmuc.yaml b/prepare-vms/settings/admin-dmuc.yaml index cdbdcb94..1ed9abe7 100644 --- a/prepare-vms/settings/admin-dmuc.yaml +++ b/prepare-vms/settings/admin-dmuc.yaml @@ -18,6 +18,7 @@ image: steps: - wait + - standardize - clusterize - tools - docker diff --git a/prepare-vms/settings/admin-kubenet.yaml b/prepare-vms/settings/admin-kubenet.yaml index 51a7f7e4..43403798 100644 --- a/prepare-vms/settings/admin-kubenet.yaml +++ b/prepare-vms/settings/admin-kubenet.yaml @@ -20,6 +20,7 @@ image: steps: - disableaddrchecks - wait + - standardize - clusterize - tools - docker diff --git a/prepare-vms/settings/admin-kuberouter.yaml b/prepare-vms/settings/admin-kuberouter.yaml index 3bde36bf..ab95e489 100644 --- a/prepare-vms/settings/admin-kuberouter.yaml +++ b/prepare-vms/settings/admin-kuberouter.yaml @@ -20,6 +20,7 @@ image: steps: - disableaddrchecks - wait + - standardize - clusterize - tools - docker diff --git a/prepare-vms/settings/admin-oldversion.yaml b/prepare-vms/settings/admin-oldversion.yaml index 41f7bca0..5b2b9a64 100644 --- a/prepare-vms/settings/admin-oldversion.yaml +++ b/prepare-vms/settings/admin-oldversion.yaml @@ -22,6 +22,7 @@ image: steps: - wait + - standardize - clusterize - tools - docker diff --git a/prepare-vms/settings/admin-test.yaml b/prepare-vms/settings/admin-test.yaml index 7c1bfe63..ef5882a2 100644 --- a/prepare-vms/settings/admin-test.yaml +++ b/prepare-vms/settings/admin-test.yaml @@ -18,6 +18,7 @@ image: steps: - wait + - standardize - clusterize - tools - docker diff --git a/prepare-vms/settings/csv.yaml b/prepare-vms/settings/csv.yaml deleted file mode 100644 index c36bcb50..00000000 --- a/prepare-vms/settings/csv.yaml +++ /dev/null @@ -1,8 +0,0 @@ -# Number of VMs per cluster -clustersize: 5 - -# The hostname of each node will be clusterprefix + a number -clusterprefix: node - -# Jinja2 template to use to generate ready-to-cut cards -cards_template: clusters.csv diff --git a/prepare-vms/settings/docker.yaml b/prepare-vms/settings/docker.yaml index da0b1dab..fa124ce7 100644 --- a/prepare-vms/settings/docker.yaml +++ b/prepare-vms/settings/docker.yaml @@ -18,6 +18,7 @@ user_password: training steps: - wait + - standardize - clusterize - tools - docker diff --git a/prepare-vms/settings/example.yaml b/prepare-vms/settings/example.yaml deleted file mode 100644 index 9e05dbbd..00000000 --- a/prepare-vms/settings/example.yaml +++ /dev/null @@ -1,17 +0,0 @@ -# customize your cluster size, your cards template, and the versions - -# Number of VMs per cluster -clustersize: 5 - -# The hostname of each node will be clusterprefix + a number -clusterprefix: node - -# Jinja2 template to use to generate ready-to-cut cards -cards_template: cards.html - -# Use "Letter" in the US, and "A4" everywhere else -paper_size: Letter - -# Login and password that students will use -user_login: docker -user_password: training diff --git a/prepare-vms/settings/kubernetes.yaml b/prepare-vms/settings/kubernetes.yaml index ccc0589d..6d3a9b8a 100644 --- a/prepare-vms/settings/kubernetes.yaml +++ b/prepare-vms/settings/kubernetes.yaml @@ -16,6 +16,7 @@ user_password: training steps: - wait + - standardize - clusterize - tools - docker diff --git a/prepare-vms/settings/portal.yaml b/prepare-vms/settings/portal.yaml index 4c8fcbd9..00b68fb8 100644 --- a/prepare-vms/settings/portal.yaml +++ b/prepare-vms/settings/portal.yaml @@ -18,6 +18,7 @@ user_password: CHANGEME steps: - wait + - standardize - clusterize - tools - docker diff --git a/prepare-vms/settings/swarm.yaml b/prepare-vms/settings/swarm.yaml deleted file mode 100644 index 073c90dd..00000000 --- a/prepare-vms/settings/swarm.yaml +++ /dev/null @@ -1,17 +0,0 @@ -# This file is passed by trainer-cli to scripts/ips-txt-to-html.py - -# Number of VMs per cluster -clustersize: 3 - -# The hostname of each node will be clusterprefix + a number -clusterprefix: node - -# Jinja2 template to use to generate ready-to-cut cards -cards_template: cards.html - -# Use "Letter" in the US, and "A4" everywhere else -paper_size: Letter - -# Login and password that students will use -user_login: docker -user_password: training diff --git a/prepare-vms/terraform/azure/main.tf b/prepare-vms/terraform/azure/main.tf index 59dedd20..651c34c2 100644 --- a/prepare-vms/terraform/azure/main.tf +++ b/prepare-vms/terraform/azure/main.tf @@ -1,5 +1,5 @@ resource "azurerm_resource_group" "_" { - name = var.prefix + name = var.prefix location = var.location } @@ -8,7 +8,7 @@ resource "azurerm_public_ip" "_" { name = format("%s-%04d", var.prefix, count.index + 1) location = azurerm_resource_group._.location resource_group_name = azurerm_resource_group._.name - allocation_method = "Dynamic" + allocation_method = "Dynamic" } resource "azurerm_network_interface" "_" { @@ -21,7 +21,7 @@ resource "azurerm_network_interface" "_" { name = "internal" subnet_id = azurerm_subnet._.id private_ip_address_allocation = "Dynamic" - public_ip_address_id = azurerm_public_ip._[count.index].id + public_ip_address_id = azurerm_public_ip._[count.index].id } } @@ -49,7 +49,7 @@ resource "azurerm_linux_virtual_machine" "_" { source_image_reference { publisher = "Canonical" offer = "UbuntuServer" - sku = "18.04-LTS" # FIXME + sku = "22.04-LTS" version = "latest" } } @@ -63,7 +63,7 @@ data "azurerm_public_ip" "_" { count = var.how_many_nodes name = format("%s-%04d", var.prefix, count.index + 1) resource_group_name = azurerm_resource_group._.name - depends_on = [azurerm_linux_virtual_machine._] + depends_on = [azurerm_linux_virtual_machine._] } output "ip_addresses" { diff --git a/prepare-vms/terraform/linode/main.tf b/prepare-vms/terraform/linode/main.tf new file mode 100644 index 00000000..604da553 --- /dev/null +++ b/prepare-vms/terraform/linode/main.tf @@ -0,0 +1,12 @@ +resource "linode_instance" "_" { + count = var.how_many_nodes + label = format("%s-%04d", var.prefix, count.index + 1) + region = var.location + type = var.size + authorized_keys = local.authorized_keys + image = "linode/ubuntu22.04" +} + +output "ip_addresses" { + value = join("", formatlist("%s\n", linode_instance._.*.ip_address)) +} diff --git a/prepare-vms/terraform/linode/provider.tf b/prepare-vms/terraform/linode/provider.tf new file mode 100644 index 00000000..708803e7 --- /dev/null +++ b/prepare-vms/terraform/linode/provider.tf @@ -0,0 +1,12 @@ +terraform { + required_version = ">= 1" + required_providers { + linode = { + source = "linode/linode" + } + } +} + +provider "linode" { + token = regex("\ntoken *= *([0-9a-f]+)\n", file("~/.config/linode-cli"))[0] +} \ No newline at end of file diff --git a/prepare-vms/terraform/linode/variables.tf b/prepare-vms/terraform/linode/variables.tf new file mode 100644 index 00000000..9e369e24 --- /dev/null +++ b/prepare-vms/terraform/linode/variables.tf @@ -0,0 +1,32 @@ +variable "prefix" { + type = string + default = "provisioned-with-terraform" +} + +variable "how_many_nodes" { + type = number + default = 2 +} + +locals { + authorized_keys = split("\n", trimspace(file("~/.ssh/id_rsa.pub"))) +} + +/* +Available sizes: +"g6-standard-1" # CPU=1 RAM=2 +"g6-standard-2" # CPU=2 RAM=4 +"g6-standard-4" # CPU=4 RAM=8 +"g6-standard-6" # CPU=6 RAM=16 +"g6-standard-8" # CPU=8 RAM=32 +*/ + +variable "size" { + type = string + default = "g6-standard-2" +} + +variable "location" { + type = string + default = "eu-west" +} diff --git a/prepare-vms/terraform/oci/main.tf b/prepare-vms/terraform/oci/main.tf index ff50d53d..db39d349 100644 --- a/prepare-vms/terraform/oci/main.tf +++ b/prepare-vms/terraform/oci/main.tf @@ -16,7 +16,7 @@ data "oci_core_images" "_" { compartment_id = local.compartment_id shape = var.shape operating_system = "Canonical Ubuntu" - operating_system_version = "20.04" + operating_system_version = "22.04" #operating_system = "Oracle Linux" #operating_system_version = "7.9" } diff --git a/prepare-vms/terraform/openstack/machines.tf b/prepare-vms/terraform/openstack/machines.tf index 32acd10e..b0d42eaa 100644 --- a/prepare-vms/terraform/openstack/machines.tf +++ b/prepare-vms/terraform/openstack/machines.tf @@ -1,6 +1,6 @@ resource "openstack_compute_instance_v2" "machine" { count = var.how_many_nodes - name = format("%s-%04d", var.prefix, count.index+1) + name = format("%s-%04d", var.prefix, count.index + 1) image_name = var.image flavor_name = var.flavor security_groups = [openstack_networking_secgroup_v2.full_access.name] @@ -8,27 +8,23 @@ resource "openstack_compute_instance_v2" "machine" { network { name = openstack_networking_network_v2.internal.name - fixed_ip_v4 = cidrhost(openstack_networking_subnet_v2.internal.cidr, count.index+10) + fixed_ip_v4 = cidrhost(openstack_networking_subnet_v2.internal.cidr, count.index + 10) } } resource "openstack_compute_floatingip_v2" "machine" { - count = var.how_many_nodes + count = var.how_many_nodes # This is something provided to us by Enix when our tenant was provisioned. pool = "Public Floating" } resource "openstack_compute_floatingip_associate_v2" "machine" { - count = var.how_many_nodes + count = var.how_many_nodes floating_ip = openstack_compute_floatingip_v2.machine.*.address[count.index] instance_id = openstack_compute_instance_v2.machine.*.id[count.index] - fixed_ip = cidrhost(openstack_networking_subnet_v2.internal.cidr, count.index+10) + fixed_ip = cidrhost(openstack_networking_subnet_v2.internal.cidr, count.index + 10) } output "ip_addresses" { value = join("", formatlist("%s\n", openstack_compute_floatingip_v2.machine.*.address)) } - -variable "flavor" {} - -variable "image" {} diff --git a/prepare-vms/terraform/openstack/network.tf b/prepare-vms/terraform/openstack/network.tf index 84d3bbf0..0b6f8c2c 100644 --- a/prepare-vms/terraform/openstack/network.tf +++ b/prepare-vms/terraform/openstack/network.tf @@ -1,5 +1,5 @@ resource "openstack_networking_network_v2" "internal" { - name = var.prefix + name = var.prefix } resource "openstack_networking_subnet_v2" "internal" { diff --git a/prepare-vms/terraform/openstack/provider.tf b/prepare-vms/terraform/openstack/provider.tf index 625e404a..1202ba4e 100644 --- a/prepare-vms/terraform/openstack/provider.tf +++ b/prepare-vms/terraform/openstack/provider.tf @@ -1,5 +1,5 @@ terraform { -required_version = ">= 1" + required_version = ">= 1" required_providers { openstack = { source = "terraform-provider-openstack/openstack" diff --git a/prepare-vms/terraform/openstack/variables.tf b/prepare-vms/terraform/openstack/variables.tf new file mode 100644 index 00000000..b4f3b8d2 --- /dev/null +++ b/prepare-vms/terraform/openstack/variables.tf @@ -0,0 +1,16 @@ +variable "prefix" { + type = string +} + +variable "how_many_nodes" { + type = number +} + +variable "flavor" { + type = string +} + +variable "image" { + type = string + default = "Ubuntu 22.04" +} diff --git a/prepare-vms/terraform/openstack/vars.tf b/prepare-vms/terraform/openstack/vars.tf deleted file mode 100644 index 9d5329f3..00000000 --- a/prepare-vms/terraform/openstack/vars.tf +++ /dev/null @@ -1,7 +0,0 @@ -variable "prefix" { - type = string -} - -variable "how_many_nodes" { - type = number -}