🏭️ Small refactorings to prepare Terraform migration

- add support for Digital Ocean (through Terraform)
- add support for per-cluster SSH key (hackish for now)
- pre-load Kubernetes APT GPG key (because of GCS outage)

prepare-vms/lib/commands.sh

@@ -98,6 +98,14 @@ _cmd_createuser() {
     fi
     "
 
+    # FIXME this is a gross hack to add the deployment key to our SSH agent,
+    # so that it can be used to bounce from host to host (which is necessary
+    # in the next deployment step). In the long run, we probably want to
+    # generate these keys locally and push them to the machines instead
+    # (once we move everything to Terraform).
+    if [ -f "tags/$TAG/id_rsa" ]; then
+        ssh-add tags/$TAG/id_rsa
+    fi
     pssh "
     set -e
     cd /home/$USER_LOGIN
@@ -107,6 +115,9 @@ _cmd_createuser() {
       sudo -u $USER_LOGIN tar -xf-
     fi
     "
+    if [ -f "tags/$TAG/id_rsa" ]; then
+        ssh-add -d tags/$TAG/id_rsa
+    fi
 
     # FIXME do this only once.
     pssh -I "sudo -u $USER_LOGIN tee -a /home/$USER_LOGIN/.bashrc" <<"SQRL"
@@ -164,6 +175,13 @@ _cmd_standardize() {
     # Disable unattended upgrades so that they don't mess up with the subsequent steps
     pssh sudo rm -f /etc/apt/apt.conf.d/50unattended-upgrades
 
+    # Digital Ocean's cloud init disables password authentication; re-enable it.
+    pssh "
+    if [ -f /etc/ssh/sshd_config.d/50-cloud-init.conf ]; then
+        sudo rm /etc/ssh/sshd_config.d/50-cloud-init.conf
+        sudo systemctl restart ssh.service
+    fi"
+
     # Special case for scaleway since it doesn't come with sudo
     if [ "$INFRACLASS" = "scaleway" ]; then
         pssh -l root "
@@ -339,10 +357,14 @@ Pin-Priority: 1000
 EOF"
     fi
 
+    # As of February 27th, 2023, packages.cloud.google.com seems broken
+    # (serves HTTP 500 errors for the GPG key), so let's pre-load that key.
+    pssh -I "sudo apt-key add -" < lib/kubernetes-apt-key.gpg
+
     # Install packages
     pssh --timeout 200 "
-    curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg |
-    sudo apt-key add - &&
+    #curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg |
+    #sudo apt-key add - &&
     echo deb http://apt.kubernetes.io/ kubernetes-xenial main |
     sudo tee /etc/apt/sources.list.d/kubernetes.list"
     pssh --timeout 200 "
@@ -421,6 +443,14 @@ EOF
         kubectl apply -f https://github.com/weaveworks/weave/releases/download/v2.8.1/weave-daemonset-k8s-1.11.yaml
     fi"
 
+    # FIXME this is a gross hack to add the deployment key to our SSH agent,
+    # so that it can be used to bounce from host to host (which is necessary
+    # in the next deployment step). In the long run, we probably want to
+    # generate these keys locally and push them to the machines instead
+    # (once we move everything to Terraform).
+    if [ -f "tags/$TAG/id_rsa" ]; then
+        ssh-add tags/$TAG/id_rsa
+    fi
     # Join the other nodes to the cluster
     pssh --timeout 200 "
     if ! i_am_first_node && [ ! -f /etc/kubernetes/kubelet.conf ]; then
@@ -428,6 +458,9 @@ EOF
         ssh $SSHOPTS \$FIRSTNODE cat /tmp/kubeadm-config.yaml > /tmp/kubeadm-config.yaml &&
         sudo kubeadm join --config /tmp/kubeadm-config.yaml
     fi"
+    if [ -f "tags/$TAG/id_rsa" ]; then
+        ssh-add -d tags/$TAG/id_rsa
+    fi
 
     # Install metrics server
     pssh "

prepare-vms/lib/pssh.sh

@@ -24,7 +24,13 @@ pssh() {
         *)       LOGIN=ubuntu ;;
     esac
 
-    $PSSH -h $HOSTFILE -l $LOGIN \
+    if [ -f "tags/$TAG/id_rsa" ]; then
+        KEYFLAG="-O IdentityFile=tags/$TAG/id_rsa"
+    else
+        KEYFLAG=""
+    fi
+
+    $PSSH $KEYFLAG -h $HOSTFILE -l $LOGIN \
         --par 100 \
         --timeout 300 \
         -O LogLevel=ERROR \

prepare-vms/terraform/digitalocean/main.tf

@@ -0,0 +1,34 @@
+resource "digitalocean_droplet" "_" {
+  count    = var.how_many_nodes
+  name     = format("%s-%04d", var.prefix, count.index + 1)
+  region   = var.location
+  size     = var.size
+  ssh_keys = [digitalocean_ssh_key._.id]
+  image    = "ubuntu-22-04-x64"
+}
+
+resource "digitalocean_ssh_key" "_" {
+  name       = var.prefix
+  public_key = tls_private_key.ssh.public_key_openssh
+}
+
+output "ip_addresses" {
+  value = join("", formatlist("%s\n", digitalocean_droplet._.*.ipv4_address))
+}
+
+resource "tls_private_key" "ssh" {
+  algorithm = "RSA"
+  rsa_bits  = "4096"
+}
+
+resource "local_file" "ssh_private_key" {
+  content         = tls_private_key.ssh.private_key_pem
+  filename        = "id_rsa"
+  file_permission = "0600"
+}
+
+resource "local_file" "ssh_public_key" {
+  content         = tls_private_key.ssh.public_key_openssh
+  filename        = "id_rsa.pub"
+  file_permission = "0600"
+}

prepare-vms/terraform/digitalocean/provider.tf

@@ -0,0 +1,12 @@
+terraform {
+  required_version = ">= 1"
+  required_providers {
+    digitalocean = {
+      source = "digitalocean/digitalocean"
+    }
+  }
+}
+
+provider "digitalocean" {
+  token = yamldecode(file("~/.config/doctl/config.yaml"))["access-token"]
+}

prepare-vms/terraform/digitalocean/variables.tf

@@ -0,0 +1,32 @@
+variable "prefix" {
+  type    = string
+  default = "provisioned-with-terraform"
+}
+
+variable "how_many_nodes" {
+  type    = number
+  default = 2
+}
+
+locals {
+  authorized_keys = split("\n", trimspace(file("~/.ssh/id_rsa.pub")))
+}
+
+/*
+Available sizes:
+s-1vcpu-2gb
+s-2vcpu-2gb
+s-2vcpu-4gb
+s-4vcpu-8gb
+*/
+
+variable "size" {
+  type    = string
+  default = "s-2vcpu-4gb"
+}
+
+/* doctl compute region list */
+variable "location" {
+  type    = string
+  default = "lon1"
+}