From 14679999bedefc4e305d914c0b3427c6ad57473b Mon Sep 17 00:00:00 2001 From: Jerome Petazzoni Date: Wed, 9 Sep 2020 19:37:15 +0200 Subject: [PATCH] Big refactor of deployment script Add support for OVHcloud, Hetzner; refactor Scaleway support --- prepare-vms/infra/example.openstack-cli | 24 +++++ ...example.openstack => example.openstack-tf} | 5 +- prepare-vms/infra/hetzner | 5 + prepare-vms/infra/scaleway | 4 - prepare-vms/lib/commands.sh | 96 +++++-------------- prepare-vms/lib/infra/aws.sh | 43 ++++++++- prepare-vms/lib/infra/hetzner.sh | 50 ++++++++++ prepare-vms/lib/infra/openstack-cli.sh | 48 ++++++++++ .../infra/{openstack.sh => openstack-tf.sh} | 0 prepare-vms/lib/infra/scaleway.sh | 34 +++---- prepare-vms/lib/infra/unimplemented.sh | 23 +++++ prepare-vms/lib/postprep.py | 2 +- prepare-vms/lib/pssh.sh | 8 +- prepare-vms/terraform/machines.tf | 2 +- prepare-vms/workshopctl | 1 - 15 files changed, 235 insertions(+), 110 deletions(-) create mode 100644 prepare-vms/infra/example.openstack-cli rename prepare-vms/infra/{example.openstack => example.openstack-tf} (84%) create mode 100644 prepare-vms/infra/hetzner create mode 100644 prepare-vms/lib/infra/hetzner.sh create mode 100644 prepare-vms/lib/infra/openstack-cli.sh rename prepare-vms/lib/infra/{openstack.sh => openstack-tf.sh} (100%) create mode 100644 prepare-vms/lib/infra/unimplemented.sh diff --git a/prepare-vms/infra/example.openstack-cli b/prepare-vms/infra/example.openstack-cli new file mode 100644 index 00000000..d20f79d6 --- /dev/null +++ b/prepare-vms/infra/example.openstack-cli @@ -0,0 +1,24 @@ +INFRACLASS=openstack-cli + +# Copy that file to e.g. openstack or ovh, then customize it. +# Some Openstack providers (like OVHcloud) will let you download +# a file containing credentials. That's what you need to use. +# The file below contains some example values. +export OS_AUTH_URL=https://auth.cloud.ovh.net/v3/ +export OS_IDENTITY_API_VERSION=3 +export OS_USER_DOMAIN_NAME=${OS_USER_DOMAIN_NAME:-"Default"} +export OS_PROJECT_DOMAIN_NAME=${OS_PROJECT_DOMAIN_NAME:-"Default"} +export OS_TENANT_ID=abcd1234 +export OS_TENANT_NAME="0123456" +export OS_USERNAME="user-xyz123" +export OS_PASSWORD=AbCd1234 +export OS_REGION_NAME="GRA7" + +# And then some values to indicate server type, image, etc. +# You can see available flavors with `openstack flavor list` +export OS_FLAVOR=s1-4 +# You can see available images with `openstack image list` +export OS_IMAGE=896c5f54-51dc-44f0-8c22-ce99ba7164df +# You can create a key with `openstack keypair create --public-key ~/.ssh/id_rsa.pub containertraining` +export OS_KEY=containertraining + diff --git a/prepare-vms/infra/example.openstack b/prepare-vms/infra/example.openstack-tf similarity index 84% rename from prepare-vms/infra/example.openstack rename to prepare-vms/infra/example.openstack-tf index a99c43af..bad23dfa 100644 --- a/prepare-vms/infra/example.openstack +++ b/prepare-vms/infra/example.openstack-tf @@ -1,4 +1,5 @@ -INFRACLASS=openstack +INFRACLASS=openstack-tf + # If you are using OpenStack, copy this file (e.g. to "openstack" or "enix") # and customize the variables below. export TF_VAR_user="jpetazzo" @@ -6,4 +7,4 @@ export TF_VAR_tenant="training" export TF_VAR_domain="Default" export TF_VAR_password="..." export TF_VAR_auth_url="https://api.r1.nxs.enix.io/v3" -export TF_VAR_flavor="GP1.S" \ No newline at end of file +export TF_VAR_flavor="GP1.S" diff --git a/prepare-vms/infra/hetzner b/prepare-vms/infra/hetzner new file mode 100644 index 00000000..77547446 --- /dev/null +++ b/prepare-vms/infra/hetzner @@ -0,0 +1,5 @@ +INFRACLASS=hetzner +if ! [ -f ~/.config/hcloud/cli.toml ]; then + warn "~/.config/hcloud/cli.toml not found." + warn "Make sure that the Hetzner CLI (hcloud) is installed and configured." +fi diff --git a/prepare-vms/infra/scaleway b/prepare-vms/infra/scaleway index 7615e024..53f232e7 100644 --- a/prepare-vms/infra/scaleway +++ b/prepare-vms/infra/scaleway @@ -1,5 +1 @@ INFRACLASS=scaleway -if ! [ -f ~/.config/scw/config.yaml ]; then - warn "~/.config/scw/config.yaml not found." - warn "Make sure that the scaleway CLI is installed and configured." -fi diff --git a/prepare-vms/lib/commands.sh b/prepare-vms/lib/commands.sh index 1a190b7e..9f90d656 100644 --- a/prepare-vms/lib/commands.sh +++ b/prepare-vms/lib/commands.sh @@ -73,6 +73,19 @@ _cmd_deploy() { apt-get update && apt-get install sudo -y" fi + # FIXME + # Special case for hetzner since it doesn't have an ubuntu user + #if [ "$INFRACLASS" = "hetzner" ]; then + # pssh -l root " + #[ -d /home/ubuntu ] || + # useradd ubuntu -m -s /bin/bash + #echo 'ubuntu ALL=(ALL:ALL) NOPASSWD:ALL' > /etc/sudoers.d/ubuntu + #[ -d /home/ubuntu/.ssh ] || + # install --owner=ubuntu --mode=700 --directory /home/ubuntu/.ssh + #[ -f /home/ubuntu/.ssh/authorized_keys ] || + # install --owner=ubuntu --mode=600 /root/.ssh/authorized_keys --target-directory /home/ubuntu/.ssh" + #fi + # Copy settings and install Python YAML parser pssh -I tee /tmp/settings.yaml /tmp/token && - sudo kubeadm init $EXTRA_KUBEADM --token \$(cat /tmp/token) --apiserver-cert-extra-sans \$(cat /tmp/ipv4) + sudo kubeadm init $EXTRA_KUBEADM --token \$(cat /tmp/token) --apiserver-cert-extra-sans \$(cat /tmp/ipv4) --ignore-preflight-errors=NumCPU fi" # Put kubeconfig in ubuntu's and docker's accounts @@ -224,13 +237,13 @@ _cmd_kube() { # Install kubectx and kubens pssh " [ -d kubectx ] || git clone https://github.com/ahmetb/kubectx && - sudo ln -sf /home/ubuntu/kubectx/kubectx /usr/local/bin/kctx && - sudo ln -sf /home/ubuntu/kubectx/kubens /usr/local/bin/kns && - sudo cp /home/ubuntu/kubectx/completion/*.bash /etc/bash_completion.d && + sudo ln -sf \$HOME/kubectx/kubectx /usr/local/bin/kctx && + sudo ln -sf \$HOME/kubectx/kubens /usr/local/bin/kns && + sudo cp \$HOME/kubectx/completion/*.bash /etc/bash_completion.d && [ -d kube-ps1 ] || git clone https://github.com/jonmosco/kube-ps1 && sudo -u docker sed -i s/docker-prompt/kube_ps1/ /home/docker/.bashrc && sudo -u docker tee -a /home/docker/.bashrc < webssh/known_hosts" + done | sudo tee /opt/webssh/known_hosts" pssh "cat >webssh.service </dev/null; then - aws ec2 import-key-pair --key-name $AWS_KEY_NAME \ - --public-key-material "$(ssh-add -L \ - | grep -i RSA \ - | head -n1 \ - | cut -d " " -f 1-2)" &>/dev/null - - if ! aws ec2 describe-key-pairs --key-name "$AWS_KEY_NAME" &>/dev/null; then - die "Somehow, importing the key didn't work. Make sure that 'ssh-add -l | grep RSA | head -n1' returns an RSA key?" - else - info "Imported new key $AWS_KEY_NAME." - fi - else - info "Using existing key $AWS_KEY_NAME." - fi -} diff --git a/prepare-vms/lib/infra/aws.sh b/prepare-vms/lib/infra/aws.sh index f7721003..a01459d1 100644 --- a/prepare-vms/lib/infra/aws.sh +++ b/prepare-vms/lib/infra/aws.sh @@ -1,9 +1,13 @@ +if ! command -v aws >/dev/null; then + warn "AWS CLI (aws) not found." +fi + infra_list() { aws_display_tags } infra_quotas() { - greet + aws_greet max_instances=$(aws ec2 describe-account-attributes \ --attribute-names max-instances \ @@ -21,10 +25,10 @@ infra_start() { COUNT=$1 # Print our AWS username, to ease the pain of credential-juggling - greet + aws_greet # Upload our SSH keys to AWS if needed, to be added to each VM's authorized_keys - key_name=$(sync_keys) + key_name=$(aws_sync_keys) AMI=$(aws_get_ami) # Retrieve the AWS image ID if [ -z "$AMI" ]; then @@ -61,7 +65,7 @@ infra_start() { aws_tag_instances $TAG $TAG # Wait until EC2 API tells us that the instances are running - wait_until_tag_is_running $TAG $COUNT + aws_wait_until_tag_is_running $TAG $COUNT aws_get_instance_ips_by_tag $TAG > tags/$TAG/ips.txt } @@ -98,7 +102,7 @@ infra_disableaddrchecks() { done } -wait_until_tag_is_running() { +aws_wait_until_tag_is_running() { max_retry=100 i=0 done_count=0 @@ -214,3 +218,32 @@ aws_get_ami() { ##VERSION## find_ubuntu_ami -r $AWS_DEFAULT_REGION -a amd64 -v 18.04 -t hvm:ebs -N -q } + +aws_greet() { + IAMUSER=$(aws iam get-user --query 'User.UserName') + info "Hello! You seem to be UNIX user $USER, and IAM user $IAMUSER." +} + +aws_sync_keys() { + # make sure ssh-add -l contains "RSA" + ssh-add -l | grep -q RSA \ + || die "The output of \`ssh-add -l\` doesn't contain 'RSA'. Start the agent, add your keys?" + + AWS_KEY_NAME=$(make_key_name) + info "Syncing keys... " + if ! aws ec2 describe-key-pairs --key-name "$AWS_KEY_NAME" &>/dev/null; then + aws ec2 import-key-pair --key-name $AWS_KEY_NAME \ + --public-key-material "$(ssh-add -L \ + | grep -i RSA \ + | head -n1 \ + | cut -d " " -f 1-2)" &>/dev/null + + if ! aws ec2 describe-key-pairs --key-name "$AWS_KEY_NAME" &>/dev/null; then + die "Somehow, importing the key didn't work. Make sure that 'ssh-add -l | grep RSA | head -n1' returns an RSA key?" + else + info "Imported new key $AWS_KEY_NAME." + fi + else + info "Using existing key $AWS_KEY_NAME." + fi +} diff --git a/prepare-vms/lib/infra/hetzner.sh b/prepare-vms/lib/infra/hetzner.sh new file mode 100644 index 00000000..7d61ff5a --- /dev/null +++ b/prepare-vms/lib/infra/hetzner.sh @@ -0,0 +1,50 @@ +if ! command -v hcloud >/dev/null; then + warn "Hetzner CLI (hcloud) not found." +fi +if ! [ -f ~/.config/hcloud/cli.toml ]; then + warn "~/.config/hcloud/cli.toml not found." +fi + +infra_start() { + COUNT=$1 + + HETZNER_INSTANCE_TYPE=${HETZNER_INSTANCE_TYPE-cx21} + HETZNER_DATACENTER=${HETZNER_DATACENTER-nbg1-dc3} + HETZNER_IMAGE=${HETZNER_IMAGE-168855} + + for I in $(seq 1 $COUNT); do + NAME=$(printf "%s-%03d" $TAG $I) + sep "Starting instance $I/$COUNT" + info " Datacenter: $HETZNER_DATACENTER" + info " Name: $NAME" + info " Instance type: $HETZNER_INSTANCE_TYPE" + hcloud server create \ + --type=${HETZNER_INSTANCE_TYPE} \ + --datacenter=${HETZNER_DATACENTER} \ + --image=${HETZNER_IMAGE} \ + --name=$NAME \ + --label=tag=$TAG \ + --ssh-key ~/.ssh/id_rsa.pub + done + + hetzner_get_ips_by_tag $TAG > tags/$TAG/ips.txt +} + +infra_stop() { + for ID in $(hetzner_get_ids_by_tag $TAG); do + info "Scheduling deletion of instance $ID..." + hcloud server delete $ID & + done + info "Waiting for deletion to complete..." + wait +} + +hetzner_get_ids_by_tag() { + TAG=$1 + hcloud server list --selector=tag=$TAG -o json | jq -r .[].name +} + +hetzner_get_ips_by_tag() { + TAG=$1 + hcloud server list --selector=tag=$TAG -o json | jq -r .[].public_net.ipv4.ip +} diff --git a/prepare-vms/lib/infra/openstack-cli.sh b/prepare-vms/lib/infra/openstack-cli.sh new file mode 100644 index 00000000..aa4fe7cd --- /dev/null +++ b/prepare-vms/lib/infra/openstack-cli.sh @@ -0,0 +1,48 @@ +infra_start() { + COUNT=$1 + + sep "Starting $COUNT instances" + info " Region: $OS_REGION_NAME" + info " User: $OS_USERNAME" + info " Flavor: $OS_FLAVOR" + info " Image: $OS_IMAGE" + openstack server create \ + --flavor $OS_FLAVOR \ + --image $OS_IMAGE \ + --key-name $OS_KEY \ + --min $COUNT --max $COUNT \ + --property workshopctl=$TAG \ + $TAG + + sep "Waiting for IP addresses to be available" + GOT=0 + while [ "$GOT" != "$COUNT" ]; do + echo "Got $GOT/$COUNT IP addresses." + oscli_get_ips_by_tag $TAG > tags/$TAG/ips.txt + GOT="$(wc -l < tags/$TAG/ips.txt)" + done + +} + +infra_stop() { + info "Counting instances..." + oscli_get_instances_json $TAG | + jq -r .[].Name | + wc -l + info "Deleting instances..." + oscli_get_instances_json $TAG | + jq -r .[].Name | + xargs -P10 -n1 openstack server delete + info "Done." +} + +oscli_get_instances_json() { + TAG=$1 + openstack server list -f json --name "${TAG}-[0-9]*" +} + +oscli_get_ips_by_tag() { + TAG=$1 + oscli_get_instances_json $TAG | + jq -r .[].Networks | cut -d= -f2 | cut -d, -f1 | grep . || true +} diff --git a/prepare-vms/lib/infra/openstack.sh b/prepare-vms/lib/infra/openstack-tf.sh similarity index 100% rename from prepare-vms/lib/infra/openstack.sh rename to prepare-vms/lib/infra/openstack-tf.sh diff --git a/prepare-vms/lib/infra/scaleway.sh b/prepare-vms/lib/infra/scaleway.sh index 9bd07ede..70b7f127 100644 --- a/prepare-vms/lib/infra/scaleway.sh +++ b/prepare-vms/lib/infra/scaleway.sh @@ -1,15 +1,13 @@ -infra_list() { - die "unimplemented" -} - -infra_quotas() { - die "unimplemented" -} +if ! command -v scw >/dev/null; then + warn "Scaleway CLI (scw) not found." +fi +if ! [ -f ~/.config/scw/config.yaml ]; then + warn "~/.config/scw/config.yaml not found." +fi infra_start() { COUNT=$1 - AWS_KEY_NAME=$(make_key_name) SCW_INSTANCE_TYPE=${SCW_INSTANCE_TYPE-DEV1-M} SCW_ZONE=${SCW_ZONE-fr-par-1} @@ -29,12 +27,12 @@ infra_start() { } infra_stop() { - for ID in $(scw_get_ids_by_tag $TAG); do - info "Scheduling deletion of instance $ID..." - scw instance server delete force-shutdown=true server-id=$ID & - done - info "Waiting for deletion to complete..." - wait + info "Counting instances..." + scw_get_ids_by_tag $TAG | wc -l + info "Deleting instances..." + scw_get_ids_by_tag $TAG | + xargs -n1 -P10 -I@@ \ + scw instance server delete force-shutdown=true server-id=@@ } scw_get_ids_by_tag() { @@ -46,11 +44,3 @@ scw_get_ips_by_tag() { TAG=$1 scw instance server list name=$TAG -o json | jq -r .[].public_ip.address } - -infra_opensg() { - die "unimplemented" -} - -infra_disableaddrchecks() { - die "unimplemented" -} diff --git a/prepare-vms/lib/infra/unimplemented.sh b/prepare-vms/lib/infra/unimplemented.sh new file mode 100644 index 00000000..c32e2356 --- /dev/null +++ b/prepare-vms/lib/infra/unimplemented.sh @@ -0,0 +1,23 @@ +infra_disableaddrchecks() { + die "unimplemented" +} + +infra_list() { + die "unimplemented" +} + +infra_opensg() { + die "unimplemented" +} + +infra_quotas() { + die "unimplemented" +} + +infra_start() { + die "unimplemented" +} + +infra_stop() { + die "unimplemented" +} diff --git a/prepare-vms/lib/postprep.py b/prepare-vms/lib/postprep.py index e60079bf..e14bd9c4 100755 --- a/prepare-vms/lib/postprep.py +++ b/prepare-vms/lib/postprep.py @@ -37,7 +37,7 @@ def system(cmd): td = str(t2-t1)[:5] f.write(bold("[{}] in {}s\n".format(retcode, td))) STEP += 1 - with open("/home/ubuntu/.bash_history", "a") as f: + with open(os.environ["HOME"] + "/.bash_history", "a") as f: f.write("{}\n".format(cmd)) if retcode != 0: msg = "The following command failed with exit code {}:\n".format(retcode) diff --git a/prepare-vms/lib/pssh.sh b/prepare-vms/lib/pssh.sh index abb94539..ca3bc639 100644 --- a/prepare-vms/lib/pssh.sh +++ b/prepare-vms/lib/pssh.sh @@ -18,7 +18,13 @@ pssh() { echo "[parallel-ssh] $@" export PSSH=$(which pssh || which parallel-ssh) - $PSSH -h $HOSTFILE -l ubuntu \ + if [ "$INFRACLASS" = hetzner ]; then + LOGIN=root + else + LOGIN=ubuntu + fi + + $PSSH -h $HOSTFILE -l $LOGIN \ --par 100 \ -O LogLevel=ERROR \ -O UserKnownHostsFile=/dev/null \ diff --git a/prepare-vms/terraform/machines.tf b/prepare-vms/terraform/machines.tf index 41ff96e7..78b9da6a 100644 --- a/prepare-vms/terraform/machines.tf +++ b/prepare-vms/terraform/machines.tf @@ -1,7 +1,7 @@ resource "openstack_compute_instance_v2" "machine" { count = "${var.count}" name = "${format("%s-%04d", "${var.prefix}", count.index+1)}" - image_name = "Ubuntu 16.04.5 (Xenial Xerus)" + image_name = "Ubuntu 18.04.4 20200324" flavor_name = "${var.flavor}" security_groups = ["${openstack_networking_secgroup_v2.full_access.name}"] key_pair = "${openstack_compute_keypair_v2.ssh_deploy_key.name}" diff --git a/prepare-vms/workshopctl b/prepare-vms/workshopctl index 870d3e06..9ebd6ceb 100755 --- a/prepare-vms/workshopctl +++ b/prepare-vms/workshopctl @@ -15,7 +15,6 @@ for lib in lib/*.sh; do done DEPENDENCIES=" - aws ssh curl jq