From 9e5c7c8216fb875605765daf855ae0e0911a6e2c Mon Sep 17 00:00:00 2001 From: AJ Bowen Date: Fri, 25 Mar 2016 13:59:53 +0100 Subject: [PATCH] Complete rewrite of deployment process The former VM deployment process relied on extra scripts located in a (private) repo. The new process is standalone. --- prepare-vms/Dockerfile | 52 ++ prepare-vms/README.md | 125 +++++ prepare-vms/docker-compose.yml | 38 ++ prepare-vms/ips-txt-to-html.py | 82 --- prepare-vms/{ => media}/docker-nb.svg | 0 prepare-vms/{ => media}/swarm.png | Bin prepare-vms/postprep.rc | 51 -- prepare-vms/rc | 23 - prepare-vms/scripts/aws.sh | 107 ++++ prepare-vms/scripts/cli.sh | 40 ++ prepare-vms/scripts/colors.sh | 15 + prepare-vms/scripts/find-ubuntu-ami.sh | 132 +++++ prepare-vms/scripts/ips-txt-to-html.py | 119 +++++ prepare-vms/scripts/postprep.rc | 157 ++++++ prepare-vms/scripts/rc | 22 + prepare-vms/scripts/trainer-cli | 489 ++++++++++++++++++ prepare-vms/settings.yaml | 1 + .../settings/fundamentals-settings.yaml | 30 ++ .../settings/orchestration-settings.yaml | 30 ++ prepare-vms/trainer | 137 +++++ 20 files changed, 1494 insertions(+), 156 deletions(-) create mode 100644 prepare-vms/Dockerfile create mode 100644 prepare-vms/README.md create mode 100644 prepare-vms/docker-compose.yml delete mode 100755 prepare-vms/ips-txt-to-html.py rename prepare-vms/{ => media}/docker-nb.svg (100%) rename prepare-vms/{ => media}/swarm.png (100%) delete mode 100755 prepare-vms/postprep.rc delete mode 100644 prepare-vms/rc create mode 100755 prepare-vms/scripts/aws.sh create mode 100755 prepare-vms/scripts/cli.sh create mode 100644 prepare-vms/scripts/colors.sh create mode 100755 prepare-vms/scripts/find-ubuntu-ami.sh create mode 100755 prepare-vms/scripts/ips-txt-to-html.py create mode 100755 prepare-vms/scripts/postprep.rc create mode 100755 prepare-vms/scripts/rc create mode 100755 prepare-vms/scripts/trainer-cli create mode 120000 prepare-vms/settings.yaml create mode 100644 prepare-vms/settings/fundamentals-settings.yaml create mode 100644 prepare-vms/settings/orchestration-settings.yaml create mode 100755 prepare-vms/trainer diff --git a/prepare-vms/Dockerfile b/prepare-vms/Dockerfile new file mode 100644 index 00000000..4b650eb0 --- /dev/null +++ b/prepare-vms/Dockerfile @@ -0,0 +1,52 @@ +FROM debian:jessie +MAINTAINER AJ Bowen + +RUN apt-get update +RUN apt-get install -y ca-certificates +RUN apt-get install -y groff +RUN apt-get install -y less +RUN apt-get install -y python python-pip +RUN apt-get install -y python-docutils +RUN apt-get install -y sudo +RUN apt-get install -y \ + bsdmainutils \ + curl \ + jq \ + less \ + man \ + pssh \ + ssh + +RUN pip install awscli +RUN pip install \ + pdfkit \ + PyYAML \ + termcolor + +RUN apt-get install -y wkhtmltopdf + +ENV HOME /home/user + +RUN useradd --create-home --home-dir $HOME user \ + && mkdir -p $HOME/.config/gandi \ + && chown -R user:user $HOME + +RUN echo "user ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers + +# Replace 1000 with your user / group id +#RUN export uid=1000 gid=1000 && \ +# mkdir -p /home/user && \ +# mkdir -p /etc/sudoers.d && \ +# echo "user:x:${uid}:${gid}:user,,,:/home/user:/bin/bash" >> /etc/passwd && \ +# echo "user:x:${uid}:" >> /etc/group && \ +# echo "user ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/user && \ +# chmod 0440 /etc/sudoers.d/user && \ +# chown ${uid}:${gid} -R /home/user + +WORKDIR $HOME +RUN echo "alias ll='ls -lahF'" >> /home/user/.bashrc +RUN echo "export PATH=$PATH:/home/user/bin" >> /home/user/.bashrc +RUN mkdir -p /home/user/bin +RUN ln -s /home/user/prepare-vms/scripts/trainer-cli /home/user/bin/trainer-cli +USER user + diff --git a/prepare-vms/README.md b/prepare-vms/README.md new file mode 100644 index 00000000..554c7d96 --- /dev/null +++ b/prepare-vms/README.md @@ -0,0 +1,125 @@ +# Trainer tools to prepare VMs for Docker workshops + +There are several options for using these tools: + +### Clone the repo + + $ git clone https://github.com/soulshake/prepare-vms.git + $ cd prepare-vms + $ docker-compose build + $ mkdir $HOME/bin && ln -s `pwd`/trainer $HOME/bin/trainer + +### Via the image + + $ docker pull soulshake/prepare-vms + +### Submodule + +This repo can be added as a submodule in the repo of the Docker workshop: + + $ git submodule add https://github.com/soulshake/prepare-vms.git + +## Setup + +### Export needed envvars + +Required environment variables: + +* `AWS_ACCESS_KEY_ID` +* `AWS_SECRET_ACCESS_KEY` +* `AWS_DEFAULT_REGION` + + + +### Update settings.yaml + +If you have more than one workshop: + + $ cp settings.yaml settings/YOUR_WORKSHOP_NAME-settings.yaml + $ ln -s settings/YOUR_WORKSHOP_NAME-settings.yaml `pwd`/settings.yaml + +Update the `settings.yaml` as needed. This is the file that will be used to generate cards. + +## Usage + +### Summary + +Summary of steps to launch a batch of instances for a workshop: + +* Export the environment variables needed by the AWS CLI (see **Requirements** below) +* `trainer start NUMBER_OF_VMS` to create AWS instances +* `trainer deploy TAG` to run `scripts/postprep.rc` via parallel-ssh +* `trainer pull-images TAG` to pre-pull a bunch of Docker images +* `trainer test TAG` +* `trainer cards TAG` to generate a PDF and an HTML file you can print + +The `trainer` script can be executed directly. + +It will check for the necessary environment variables. Then, if all its dependencies are installed +locally, it will execute `trainer-cli`. If not, it will look for a local Docker image +tagged `soulshake/trainer-tools`. If found, it will run in a container. If not found, +the user will be prompted to either install the missing dependencies or download +the Docker image. + +## Detailed usage + +### Start some VMs + + $ trainer start 10 + +A few things will happen: + +#### Sync of SSH keys + +When the `start` command is run, your local RSA SSH public key will be added to your AWS EC2 keychain. + +To see which local key will be uploaded, run `ssh-add -l | grep RSA`. + +#### Instance + tag creation + +10 VMs will be started, with an automatically generated tag (timestamp + your username). + +Your SSH key will be added to the `authorized_keys` of the ubuntu user. + +#### Creation of ./$TAG/ directory and contents + +Following the creation of the VMs, a text file will be created containing a list of their IPs. + +This ips.txt file will be created in the $TAG/ directory and a symlink will be placed in the working directory of the script. + +If you create new VMs, the symlinked file will be overwritten. + +## Deployment + +Instances can be deployed manually using the `deploy` command: + + $ trainer deploy TAG + +The `postprep.rc` file will be copied via parallel-ssh to all of the VMs and executed. + +### Pre-pull images + + $ trainer pull-images TAG + +### Generate cards + + $ trainer cards TAG + +### List tags + + $ trainer list + +### List VMs + + $ trainer list TAG + +This will print a human-friendly list containing some information about each instance. + +### Stop and destroy VMs + + $ trainer stop TAG + +## ToDo + + * Don't write to bash history in system() in postprep + * compose, etc version inconsistent (int vs str) diff --git a/prepare-vms/docker-compose.yml b/prepare-vms/docker-compose.yml new file mode 100644 index 00000000..3b590d6b --- /dev/null +++ b/prepare-vms/docker-compose.yml @@ -0,0 +1,38 @@ +prepare-vms: + build: . + container_name: prepare-vms + working_dir: /home/user/prepare-vms + volumes: + - $HOME/.aws/:$HOME.aws/ + - $HOME/.ssh/:/home/user/.ssh/ + - /etc/localtime:/etc/localtime:ro + #- /home:/home + - /tmp/.X11-unix:/tmp/.X11-unix + - $SSH_AUTH_DIRNAME:$SSH_AUTH_DIRNAME + - $SCRIPT_DIR/:/home/user/prepare-vms/ + #- $SCRIPT_DIR/:$HOME/prepare-vms/ + #- $HOME/trainer-tools/ + #- $(dirname $SSH_AUTH_SOCK):$(dirname $SSH_AUTH_SOCK) + #- /etc/passwd:/etc/passwd:ro + #- /etc/group:/etc/group:ro + #- /run/user:/run/user + environment: + HOME: /home/user + #SCRIPT_DIR: /home/aj/git/prepare-vms + SCRIPT_DIR: /home/user/prepare-vms + HOME: /home/user + SSH_AUTH_SOCK: ${SSH_AUTH_SOCK} + SSH_AGENT_PID: ${SSH_AGENT_PID} + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} + DISPLAY: ${DISPLAY} + USER: ${USER} + AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION} + AWS_DEFAULT_OUTPUT: + AWS_INSTANCE_TYPE: ${AWS_INSTANCE_TYPE} + AWS_VPC_ID: ${AWS_VPC_ID} + entrypoint: /home/user/prepare-vms/scripts/trainer-cli + #entrypoint: trainer + + #AWS_DEFAULT_PROFILE: ${AWS_DEFAULT_PROFILE} + #command: /home/user/prepare-vms/trainer-cli diff --git a/prepare-vms/ips-txt-to-html.py b/prepare-vms/ips-txt-to-html.py deleted file mode 100755 index 4479d684..00000000 --- a/prepare-vms/ips-txt-to-html.py +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env python - -SETTINGS_BASIC = dict( - clustersize=1, - pagesize=15, - blurb="

Here is the connection information to your very own " - "VM for this intro to Docker workshop. You can connect " - "to the VM using your SSH client.

\n" - "

Your VM is reachable on the following address:

\n", - prettify=lambda x: x, - footer="

You can find the last version of the slides on " - "http://view.dckr.info/.

", - ) - -SETTINGS_ADVANCED = dict( - clustersize=5, - pagesize=12, - blurb="

Here is the connection information to your very own " - "cluster for this orchestration workshop. You can connect " - "to each VM with any SSH client.

\n" - "

Your machines are:

") - html.write("

login=docker password=training

\n") - html.write(footer) - html.write("") -html.close() - diff --git a/prepare-vms/docker-nb.svg b/prepare-vms/media/docker-nb.svg similarity index 100% rename from prepare-vms/docker-nb.svg rename to prepare-vms/media/docker-nb.svg diff --git a/prepare-vms/swarm.png b/prepare-vms/media/swarm.png similarity index 100% rename from prepare-vms/swarm.png rename to prepare-vms/media/swarm.png diff --git a/prepare-vms/postprep.rc b/prepare-vms/postprep.rc deleted file mode 100755 index a47eb114..00000000 --- a/prepare-vms/postprep.rc +++ /dev/null @@ -1,51 +0,0 @@ -pssh -I tee /tmp/postprep.py <>/tmp/pp.out 2>>/tmp/pp.err" < ips.txt -pssh "[ -f .ssh/id_rsa ] || scp -o StrictHostKeyChecking=no node1:.ssh/id_rsa* .ssh" -pssh "grep docker@ .ssh/authorized_keys || cat .ssh/id_rsa.pub >> .ssh/authorized_keys" diff --git a/prepare-vms/rc b/prepare-vms/rc deleted file mode 100644 index 5415c1a8..00000000 --- a/prepare-vms/rc +++ /dev/null @@ -1,23 +0,0 @@ -pssh () { - HOSTFILES="hosts ips.txt /tmp/hosts" - for HOSTFILE in $HOSTFILES; do - [ -f $HOSTFILE ] && break - done - [ -f $HOSTFILE ] || { - echo "No hostfile found (tried $HOSTFILES)" - return - } - parallel-ssh -h $HOSTFILE -l docker \ - -O UserKnownHostsFile=/dev/null -O StrictHostKeyChecking=no \ - -O ForwardAgent=yes \ - "$@" -} - -pcopykey () { - ssh-add -L | pssh --askpass --send-input \ - "mkdir -p .ssh; tee .ssh/authorized_keys" - ssh-add -L | pssh --send-input \ - "sudo mkdir -p /root/.ssh; sudo tee /root/.ssh/authorized_keys" -} - - diff --git a/prepare-vms/scripts/aws.sh b/prepare-vms/scripts/aws.sh new file mode 100755 index 00000000..90d018a3 --- /dev/null +++ b/prepare-vms/scripts/aws.sh @@ -0,0 +1,107 @@ +#!/bin/bash + +source scripts/cli.sh + +aws_display_tags(){ + # Print all "Name" tags in our region with their instance count + echo "[#] [Status] [Tag]" | awk '{ printf " %7s %8s %10s \n", $1, $2, $3}' + aws ec2 describe-instances --filter "Name=tag:Name,Values=[*]" \ + --query "Reservations[*].Instances[*].[{Tags:Tags[0].Value,State:State.Name}]" \ + | awk '{ printf " %-13s %-10s %-1s\n", $1, $2, $3}' \ + | uniq -c +} + +aws_display_tokens(){ + # Print all tokens in our region with their instance count + echo "[#] [Token] [Tag]" | awk '{ printf " %7s %12s %30s\n", $1, $2, $3}' + # --query 'Volumes[*].{ID:VolumeId,AZ:AvailabilityZone,Size:Size}' + aws ec2 describe-instances --output text \ + --query 'Reservations[*].Instances[*].{ClientToken:ClientToken,Tags:Tags[0].Value}' \ + | awk '{ printf " %7s %12s %50s\n", $1, $2, $3}' \ + | sort \ + | uniq -c +} + +aws_get_tokens() { + aws ec2 describe-instances --output text \ + --query 'Reservations[*].Instances[*].[ClientToken]' \ + | sort -u +} + +aws_display_instance_statuses_by_tag() { + TAG=$1 + need_tag $TAG + + IDS=$(aws ec2 describe-instances \ + --filters "Name=tag:Name,Values=$TAG" \ + --query "Reservations[*].Instances[*].InstanceId" | tr '\t' ' ' ) + + aws ec2 describe-instance-status \ + --instance-ids $IDS \ + --query "InstanceStatuses[*].{ID:InstanceId,InstanceState:InstanceState.Name,InstanceStatus:InstanceStatus.Status,SystemStatus:SystemStatus.Status,Reachability:InstanceStatus.Status}" \ + --output table +} + +aws_display_instances_by_tag() { + TAG=$1 + need_tag $TAG + result=$(aws ec2 describe-instances --output table \ + --filter "Name=tag:Name,Values=$TAG" \ + --query "Reservations[*].Instances[*].[ \ + InstanceId, \ + State.Name, \ + Tags[0].Value, \ + PublicIpAddress, \ + InstanceType \ + ]" + ) + if [[ -z $result ]]; then + echo "No instances found with tag $TAG in region $AWS_DEFAULT_REGION." + else + echo "ID State Tags IP Type" \ + | awk '{ printf "%9s %12s %15s %20s %14s \n", $1, $2, $3, $4, $5}' # column -t -c 70} + echo "$result" + fi +} + +aws_get_instance_ids_by_client_token() { + TOKEN=$1 + need_tag $TOKEN + aws ec2 describe-instances --filters "Name=client-token,Values=$TOKEN" \ + | grep ^INSTANCE \ + | awk '{print $8}' +} + +aws_get_instance_ids_by_tag() { + TAG=$1 + need_tag $TAG + aws ec2 describe-instances --filters "Name=tag:Name,Values=$TAG" \ + | grep ^INSTANCE \ + | awk '{print $8}' +} + +aws_get_instance_ips_by_tag() { + TAG=$1 + need_tag $TAG + aws ec2 describe-instances --filter "Name=tag:Name,Values=$TAG" \ + --output text \ + --query "Reservations[*].Instances[*].PublicIpAddress" \ + | tr "\t" "\n" \ + | sort -n -t . -k 1,1 -k 2,2 -k 3,3 -k 4,4 # sort IPs +} + +aws_kill_instances_by_tag() { + TAG=$1 + need_tag $TAG + IDS=$(aws_get_instance_ids_by_tag $TAG) + if [ -z "$IDS" ]; then + die "Invalid tag." + else + echo "Deleting instances with tag $TAG" + fi + + echo "$IDS" \ + | xargs -r aws ec2 terminate-instances --instance-ids \ + | grep ^TERMINATINGINSTANCES +} + diff --git a/prepare-vms/scripts/cli.sh b/prepare-vms/scripts/cli.sh new file mode 100755 index 00000000..e8135dda --- /dev/null +++ b/prepare-vms/scripts/cli.sh @@ -0,0 +1,40 @@ +die () { + if [ -z "$1" ]; then + exit 1 + fi + echo -n $(tput setaf 1) + echo -e "$1" + echo -n $(tput sgr0) + exit 1 +} + +need_tag(){ + TAG=$1 + if [ -z "$TAG" ]; then + echo "Please specify a tag. Here's the list: " + aws_display_tags + die + fi +} + +need_token(){ + TOKEN=$1 + if [ -z "$TOKEN" ]; then + echo "Please specify a token. Here's the list: " + aws_display_tokens + die + fi +} + +need_ips_file() { + IPS_FILE=$1 + if [ -z "$IPS_FILE" ]; then + echo "IPS_FILE not set." + die + fi + + if [ ! -s "$IPS_FILE" ]; then + echo "IPS_FILE $IPS_FILE not found. Please run: trainer ips " + die + fi +} diff --git a/prepare-vms/scripts/colors.sh b/prepare-vms/scripts/colors.sh new file mode 100644 index 00000000..a10ac9d2 --- /dev/null +++ b/prepare-vms/scripts/colors.sh @@ -0,0 +1,15 @@ +bold() { + msg=$1 + echo "$(tput bold)$1$(tput sgr0)" +} + +green() { + msg=$1 + echo "$(tput setaf 2)$1$(tput sgr0)" +} + +yellow(){ + msg=$1 + echo "$(tput setaf 3)$1$(tput sgr0)" +} + diff --git a/prepare-vms/scripts/find-ubuntu-ami.sh b/prepare-vms/scripts/find-ubuntu-ami.sh new file mode 100755 index 00000000..774a7fb1 --- /dev/null +++ b/prepare-vms/scripts/find-ubuntu-ami.sh @@ -0,0 +1,132 @@ +#!/bin/bash +# borrowed from https://gist.github.com/kirikaza/6627072 + +usage() { + cat >&2 <<__ +usage: find-ubuntu-ami.sh [ ... ] [ ] +where: + is pair of key and substring to search + -r + -n + -v + -a + -t + -d + -i + -k + is on of: + -R by region + -N by name + -V by version + -A by arch + -T by type + -D by date + -I by image + -K by kernel + + protip for Docker orchestration workshop admin: + ./find-ubuntu-ami.sh -t hvm:ebs -r \$AWS_REGION -v 15.10 -N +__ + exit 1 +} + +args=`getopt hr:n:v:a:t:d:i:k:RNVATDIK $*` +if [ $? != 0 ] ; then + echo >&2 + usage +fi + +region= +name= +version= +arch= +type= +date= +image= +kernel= + +sort=date + +set -- $args +for a ; do + case "$a" in + -h) usage ;; + + -r) region=$2 ; shift ;; + -n) name=$2 ; shift ;; + -v) version=$2 ; shift ;; + -a) arch=$2 ; shift ;; + -t) type=$2 ; shift ;; + -d) date=$2 ; shift ;; + -i) image=$2 ; shift ;; + -k) kernel=$2 ; shift ;; + + -R) sort=region ;; + -N) sort=name ;; + -V) sort=version ;; + -A) sort=arch ;; + -T) sort=type ;; + -D) sort=date ;; + -I) sort=image ;; + -K) sort=kernel ;; + + --) shift ; break ;; + *) continue ;; + esac + shift +done + +[ $# = 0 ] || usage + +fix_json() { + tr -d \\n | sed 's/,]}/]}/' +} + +jq_query() { cat <<__ + .aaData | map ( + { + region: .[0], + name: .[1], + version: .[2], + arch: .[3], + type: .[4], + date: .[5], + image: .[6], + kernel: .[7] + } | select ( + (.region | contains("$region")) and + (.name | contains("$name")) and + (.version | contains("$version")) and + (.arch | contains("$arch")) and + (.type | contains("$type")) and + (.date | contains("$date")) and + (.image | contains("$image")) and + (.kernel | contains("$kernel")) + ) + ) | sort_by(.$sort) | .[] | + "\(.region)|\(.name)|\(.version)|\(.arch)|\(.type)|\(.date)|\(.image)|\(.kernel)" +__ +} + +trim_quotes() { + sed 's/^"//;s/"$//' +} + +escape_spaces() { + sed 's/ /\\\ /g' +} + +url=http://cloud-images.ubuntu.com/locator/ec2/releasesTable + +{ + echo REGION NAME VERSION ARCH TYPE DATE IMAGE KERNEL + curl -s $url | fix_json | jq "`jq_query`" | trim_quotes | escape_spaces | tr \| ' ' +} | + while read region name version arch type date image kernel ; do + image=${image%<*} + image=${image#*>} + echo "$region|$name|$version|$arch|$type|$date|$image|$kernel" + done | column -t -s \| + + + diff --git a/prepare-vms/scripts/ips-txt-to-html.py b/prepare-vms/scripts/ips-txt-to-html.py new file mode 100755 index 00000000..5be916fd --- /dev/null +++ b/prepare-vms/scripts/ips-txt-to-html.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python +import os +import sys +import yaml +import pdfkit + +def prettify(l): + l = [ip.strip() for ip in l] + ret = [ "node{}: {}".format(i+1, s) for (i, s) in zip(range(len(l)), l) ] + return ret + + +# Read settings from settings.yaml +#with open(sys.argv[1]) as f: +#with open("settings.yaml") as f: +with open("settings/fundamentals-settings.yaml") as f: + data = f.read() + +SETTINGS = yaml.load(data) +SETTINGS['footer'] = SETTINGS['footer'].format(url=SETTINGS['url']) +globals().update(SETTINGS) + +############################################################################### + +ips = list(open("ips.txt")) + +print("Current settings (as defined in settings.yaml):") +print(" Number of IPs: {}".format(len(ips))) +print(" VMs per cluster: {}".format(clustersize)) +print("Background image: {}".format(background_image)) +print("---------------------------------------------") + +assert len(ips)%clustersize == 0 + +if clustersize == 1: + blurb = blurb.format( + cluster_or_machine="machine", + this_or_each="this", + machine_is_or_machines_are="machine is", + workshop_name=workshop_short_name, + ) +else: + blurb = blurb.format( + cluster_or_machine="cluster", + this_or_each="each", + machine_is_or_machines_are="machines are", + workshop_name=workshop_short_name, + ) + +clusters = [] + +while ips: + cluster = ips[:clustersize] + ips = ips[clustersize:] + clusters.append(cluster) + +html = open("ips.html", "w") +html.write("") +for i, cluster in enumerate(clusters): + if i>0 and i%pagesize==0: + html.write('\n') + + html.write("
") + html.write(blurb) + for s in prettify(cluster): + html.write("
  • %s
    • \n"%s) + html.write("

    ") + html.write("

    login: {}
    password: {}

    \n".format(instance_login, instance_password)) + html.write(footer) + html.write("
    ") +html.close() + +""" + html.write("
    ") + html.write("

    {}

    ".format(blurb)) + for s in prettify(cluster): + html.write("
  • {}
    • ".format(s)) + html.write("

    ") + html.write("
    ") + html.write("

    login: {}    password: {}

    \n".format(instance_login, instance_password)) + html.write("
    ") + html.write(footer) + html.write("
    ") +html.close() +""" + +with open('ips.html') as f: + pdfkit.from_file(f, 'ips.pdf') diff --git a/prepare-vms/scripts/postprep.rc b/prepare-vms/scripts/postprep.rc new file mode 100755 index 00000000..35510b2b --- /dev/null +++ b/prepare-vms/scripts/postprep.rc @@ -0,0 +1,157 @@ +pssh -I tee /tmp/postprep.py < {}...".format(STEP, cmd))) + retcode = os.system(cmd) + if retcode: + retcode = bold(retcode) + t2 = time.time() + td = str(t2-t1)[:5] + f.write("[{}] in {}s\n".format(retcode, td)) + STEP += 1 + with open("/home/ubuntu/.bash_history", "a") as f: + f.write("{}\n".format(cmd)) + +# On EC2, the ephemeral disk might be mounted on /mnt. +# If /mnt is a mountpoint, place Docker workspace on it. +system("if mountpoint -q /mnt; then sudo mkdir /mnt/docker && sudo ln -s /mnt/docker /var/lib/docker; fi") + +# Put our public IP in /tmp/ipv4 +# ipv4_retrieval_endpoint = "http://169.254.169.254/latest/meta-data/public-ipv4" +ipv4_retrieval_endpoint = "http://myip.enix.org/REMOTE_ADDR" +system("curl --silent {} > /tmp/ipv4".format(ipv4_retrieval_endpoint)) + +ipv4 = open("/tmp/ipv4").read() + +# Add a "docker" user with password "training" +system("sudo useradd -d /home/docker -m -s /bin/bash docker") +system("echo docker:training | sudo chpasswd") + +# Fancy prompt courtesy of @soulshake. +system("echo \"export PS1='\e[1m\[\033[0;32m\][\h] \e[1m\[\033[0;35m\]\u@{}\[\033[0;33m\] \w\[\033[00m\]\n$ '\" | sudo -u docker tee -a /home/docker/.bashrc".format(ipv4)) + +# add docker user to sudoers and allow password authentication +system("sudo bash -c \"echo 'docker ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/docker\"") +system("sudo sed -i 's/PasswordAuthentication no/PasswordAuthentication yes/' /etc/ssh/sshd_config") + +system("sudo service ssh restart") +system("sudo apt-get -q update") +system("sudo apt-get -qy install git jq python-pip") + +# increase the size of the conntrack table so we don't blow it up when going crazy with http load testing +system("echo 1000000 | sudo tee /proc/sys/net/nf_conntrack_max") + +####################### +### DOCKER INSTALLS ### +####################### + +# This will install the latest Docker. +system("curl --silent https://get.docker.com/ | grep -v '( set -x; sleep 20 )' | sudo sh") + +# Make sure that the daemon listens on 55555 (for orchestration workshop). +# To test, run: export DOCKER_HOST=tcp://localhost:55555 ; docker ps +# or, run "curl localhost:55555" (it should return 404 not found). If it tells you connection refused, that's a bad sign +system("sudo sed -i 's,-H fd://$,-H fd:// -H tcp://0.0.0.0:55555,' /lib/systemd/system/docker.service") +system("sudo systemctl daemon-reload") + +# There seems to be a bug in the systemd scripts; so work around it. +# See https://github.com/docker/docker/issues/18444 +# If docker is already running, need to do a restart +system("curl --silent localhost:55555 || sudo systemctl restart docker ") # does this work? if not, next line should cover it +system("sudo systemctl start docker || true") + +### Install docker-compose +system("sudo pip install -U docker-compose=={}".format(COMPOSE_VERSION)) + +### Install docker-machine +system("sudo curl -sSL -o /usr/local/bin/docker-machine https://github.com/docker/machine/releases/download/v{}/docker-machine-$(uname -s)-$(uname -m)".format(MACHINE_VERSION)) +system("sudo chmod +x /usr/local/bin/docker-machine*") + +# Wait for Docker to be up. If we don't do this, Docker will not be responsive during the next step. +system("while ! sudo -u docker docker version ; do sleep 2; done") + +system("sudo apt-get remove -y --purge dnsmasq-base") +system("sudo apt-get -qy install python-setuptools pssh apache2-utils httping htop unzip") + +### Install Swarm +system("docker pull swarm:{}".format(SWARM_VERSION)) +system("docker tag -f swarm:{} swarm".format(SWARM_VERSION)) + + +### BEGIN CLUSTERING ### + +addresses = list(l.strip() for l in sys.stdin) + +assert ipv4 in addresses + +def makenames(addrs): + return [ "node%s"%(i+1) for i in range(len(addrs)) ] + +while addresses: + cluster = addresses[:CLUSTER_SIZE] + addresses = addresses[CLUSTER_SIZE:] + if ipv4 not in cluster: + continue + names = makenames(cluster) + for ipaddr, name in zip(cluster, names): + system("grep ^{} /etc/hosts || echo {} {} | sudo tee -a /etc/hosts" + .format(ipaddr, ipaddr, name)) + print(cluster) + + mynode = cluster.index(ipv4) + 1 + system("echo 'node{}' | sudo -u docker tee /tmp/node".format(mynode)) + system("sudo -u docker mkdir -p /home/docker/.ssh") + system("sudo -u docker touch /home/docker/.ssh/authorized_keys") + + if ipv4 == cluster[0]: + # If I'm node1 and don't have a private key, generate one (with empty passphrase) + system("sudo -u docker [ -f /home/docker/.ssh/id_rsa ] || sudo -u docker ssh-keygen -t rsa -f /home/docker/.ssh/id_rsa -P ''") + +FINISH = time.time() +duration = "Initial deployment took {}s".format(str(FINISH - START)[:5]) +system("echo {}".format(duration)) + +EOF + +IPS_FILE=ips.txt +if [ ! -s $IPS_FILE ]; then + echo "ips.txt not found." + exit 1 +fi + +pssh --timeout 900 --send-input "python /tmp/postprep.py >>/tmp/pp.out 2>>/tmp/pp.err" < $IPS_FILE + +# If /home/docker/.ssh/id_rsa doesn't exist, copy it from node1 +pssh "sudo -u docker [ -f /home/docker/.ssh/id_rsa ] || ssh -o StrictHostKeyChecking=no node1 sudo -u docker tar -C /home/docker -cvf- .ssh | sudo -u docker tar -C /home/docker -xf-" + +# if 'docker@' doesn't appear in /home/docker/.ssh/authorized_keys, copy it there +pssh "grep docker@ /home/docker/.ssh/authorized_keys \ + || cat /home/docker/.ssh/id_rsa.pub \ + | sudo -u docker tee -a /home/docker/.ssh/authorized_keys" + diff --git a/prepare-vms/scripts/rc b/prepare-vms/scripts/rc new file mode 100755 index 00000000..dd1966bc --- /dev/null +++ b/prepare-vms/scripts/rc @@ -0,0 +1,22 @@ +# This file can be sourced in order to directly run commands on +# a batch of VMs whose IPs are located in ips.txt of the directory in which +# the command is run. + +pssh () { + HOSTFILE="ips.txt" + + [ -f $HOSTFILE ] || { + echo "No hostfile found at $HOSTFILE" + return + } + + echo "[parallel-ssh] $@" + + parallel-ssh -h $HOSTFILE -l ubuntu \ + --par 100 \ + -O LogLevel=ERROR \ + -O UserKnownHostsFile=/dev/null \ + -O StrictHostKeyChecking=no \ + -O ForwardAgent=yes \ + "$@" +} diff --git a/prepare-vms/scripts/trainer-cli b/prepare-vms/scripts/trainer-cli new file mode 100755 index 00000000..2244894f --- /dev/null +++ b/prepare-vms/scripts/trainer-cli @@ -0,0 +1,489 @@ +#!/bin/bash +# Don't execute this script directly. Use ../trainer instead. + +set -e # if we encounter an error, abort + +greet() { + hello=$(aws iam get-user --query 'User.UserName') + echo "Greetings, $hello!" +} + +deploy_hq(){ + TAG=$1 + need_tag $TAG + REMOTE_USER=ubuntu + REMOTE_HOST=$(aws_get_instance_ips_by_tag $TAG) + echo "Trying to reach $TAG instances..." + while ! tag_is_reachable $TAG; do + echo -n "." + sleep 2 + done + env | grep -i aws > envvars.sh + scp \ + -o "UserKnownHostsFile /dev/null" \ + -o "StrictHostKeyChecking=no" \ + scripts/remote-execution.sh \ + envvars.sh \ + $REMOTE_USER@$REMOTE_HOST:/tmp/ + + ssh -A $REMOTE_USER@$REMOTE_HOST "bash /tmp/remote-execution.sh >>/tmp/pre.out 2>>/tmp/pre.err" + ssh -A $REMOTE_USER@$REMOTE_HOST +} + +deploy_tag(){ + TAG=$1 + need_tag $TAG + link_tag $TAG + + + count=$(cat ips.txt | wc -l) + + # wait until all hosts are reachable before trying to deploy + echo "Trying to reach $TAG instances..." + while ! tag_is_reachable $TAG; do + echo -n "." + sleep 2 + done + + echo "[[ Deploying tag $TAG ]]" + source scripts/postprep.rc + echo "Finished deploying $TAG." + echo "You may want to run one of the following commands:" + echo "trainer pull-images $TAG" + echo "trainer cards $TAG" +} + +link_tag() { + TAG=$1 + need_tag $TAG + IPS_FILE=tags/$TAG/ips.txt + need_ips_file $IPS_FILE + ln -sf $IPS_FILE ips.txt +} + +pull_tag(){ + TAG=$1 + need_tag $TAG + link_tag $TAG + cards_file=ips.html + if [ ! -s $IPS_FILE ]; then + echo "Nonexistent or empty IPs file $IPS_FILE" + fi + + # Pre-pull a bunch of images + pssh --timeout 900 'for I in \ + debian:latest \ + ubuntu:latest \ + fedora:latest \ + centos:latest \ + postgres \ + redis \ + training/namer \ + nathanleclaire/redisonrails; do + sudo -u docker docker pull $I + done' + + echo "Finished pulling images for $TAG" + + echo "You may now want to run:" + echo "trainer cards $TAG" +} + +wait_until_tag_is_running() { + # Wait up to 10 seconds for instances to be created + timeout=10 + TAG=$1 + COUNT=$2 + i=0 + done_count=0 + while [[ $done_count -lt $COUNT ]]; do \ + let "i += 1" + echo "Waiting: $done_count/$COUNT instances online" + done_count=$(aws ec2 describe-instances \ + --filters "Name=instance-state-name,Values=running" \ + "Name=tag:Name,Values=$TAG" \ + --query "Reservations[*].Instances[*].State.Name" \ + | tr "\t" "\n" \ + | wc -l) + + if [[ $i -gt $timeout ]]; then \ + die " Timed out while waiting for instance creation ($timeout secs)"; \ + fi + sleep 1 + done +} + +tag_is_reachable() { + TAG=$1 + need_tag $TAG + link_tag $TAG + if ! pssh -t 5 true 2>&1 >/dev/null; then + return 1 + fi + return 0 +} + +test_tag(){ + ips_file=tags/$TAG/ips.txt + echo "Using random IP in $ips_file to run tests on $TAG" + ip=$(shuf -n 1 $ips_file) + test_vm $ip + echo "Tests complete. You may want to run one of the following commands:" + echo "trainer cards $TAG" +} + +test_vm() { + ip=$1 + echo "[[ Testing instance with IP $(tput bold)$ip $(tput sgr0) ]]" + user=ubuntu + + for cmd in "hostname" \ + "whoami" \ + "hostname -i" \ + "cat /tmp/node" \ + "cat /tmp/ipv4" \ + "cat /etc/hosts" \ + "hostnamectl status" \ + "docker version | grep Version -B1" \ + "docker-compose version" \ + "docker-machine version" \ + "docker images" \ + "docker ps" \ + "which fig" \ + "curl --silent localhost:55555" \ + "sudo ls -la /mnt/ | grep docker" \ + "env" \ + "ls -la /home/docker/.ssh"; do + echo "=== $cmd ===" + echo "$cmd" | + ssh -A -q \ + -o "UserKnownHostsFile /dev/null" \ + -o "StrictHostKeyChecking=no" \ + $user@$ip sudo -u docker -i + echo + done +} + +make_key_name(){ + SHORT_FINGERPRINT=$(ssh-add -l | grep RSA | head -n1 | cut -d " " -f 2 | tr -d : | cut -c 1-8) + echo "${SHORT_FINGERPRINT}-${USER}" +} + +sync_keys() { + # make sure ssh-add -l contains "RSA" + ssh-add -l | grep -q RSA || + die "The output of \`ssh-add -l\` doesn't contain 'RSA'. Start the agent, add your keys?" + + AWS_KEY_NAME=$(make_key_name) + echo -n "Syncing keys... " + if ! aws ec2 describe-key-pairs --key-name "$AWS_KEY_NAME" &> /dev/null; then + aws ec2 import-key-pair --key-name $AWS_KEY_NAME \ + --public-key-material "$(ssh-add -L \ + | grep -i RSA \ + | head -n1 \ + | cut -d " " -f 1-2)" &> /dev/null + + if ! aws ec2 describe-key-pairs --key-name "$AWS_KEY_NAME" &> /dev/null; then + die "Somehow, importing the key didn't work. Make sure that 'ssh-add -l | grep RSA | head -n1' returns an RSA key?" + else + echo "Imported new key $AWS_KEY_NAME." + fi + else + echo "Using existing key $AWS_KEY_NAME." + fi +} + +suggest_amis() { + scripts/find-ubuntu-ami.sh -r $AWS_DEFAULT_REGION -a amd64 -v 15.10 -t hvm:ebs -N +} + +get_token() { + if [ -z $USER ]; then + export USER=anonymous + fi + date +%Y-%m-%d-%H-%M-$USER +} + +get_ami() { + # using find-ubuntu-ami script in `trainer-tools/scripts`: + #AMI=$(./scripts/find-ubuntu-ami.sh -r $AWS_DEFAULT_REGION -a amd64 -v 15.10 -t hvm:ebs -N | grep -v ^REGION | head -1 | awk '{print $7}') + #AMI=$(suggest_amis | grep -v ^REGION | head -1 | awk '{print $7}') + case $AWS_DEFAULT_REGION in + eu-central-1) + AMI=ami-74a4bc18 + ;; + eu-west-1) + AMI=ami-cda312be + ;; + us-west-2) + AMI=ami-495bbd29 + ;; + us-east-1) + AMI=ami-1711387d + ;; + esac + echo $AMI +} + + +make_cards(){ + # Generate cards for a given tag + TAG=$1 + IPS=$(aws_get_instance_ips_by_tag $TAG) + echo "$IPS" > tags/$TAG/ips.txt + + # Remove symlinks to old cards if they exist + for f in ips.html ips.pdf; do + if [ -e $f ]; then rm $f; fi + done + + # This will generate two files in the base dir: ips.pdf and ips.html + python scripts/ips-txt-to-html.py settings.yaml + + for f in ips.html ips.pdf; do + # Remove old versions of cards if they exist + if [ -e tags/$TAG/$f ]; then rm tags/$TAG/$f; fi + + # Move the generated file and replace it with a symlink + mv -f $f tags/$TAG/$f && ln -s tags/$TAG/$f $f + done + + echo "Cards created. You may want to run:" + echo "chromium ips.html" + echo "chromium ips.pdf" +} + +describe_tag() { + # Display instance details and reachability/status information + TAG=$1 + need_tag $TAG + echo "============= Tag: $TAG =============" + aws_display_instances_by_tag $TAG + aws_display_instance_statuses_by_tag $TAG +} + +run_cli() { + case "$1" in + ami) + # A wrapper for scripts/find-ubuntu-ami.sh + shift + scripts/find-ubuntu-ami.sh -r $AWS_DEFAULT_REGION $* + echo + echo "Protip:" + echo "trainer ami -a amd64 -v 15.10 -t hvm:ebs -N | grep -v ^REGION | cut -d\" \" -f15" + echo + echo "Suggestions:" + suggest_amis + ;; + cards) + TAG=$2 + need_tag $TAG + make_cards $TAG + ;; + deploy) + TAG=$2 + need_tag $TAG + if [[ $TAG == *"-hq"* ]]; then + echo "Deploying HQ" + deploy_hq $TAG + else + workshop=$2 + if [ -z $workshop ]; then + echo "Please specify a workshop name (fundamentals or orchestration)" + exit 1 + fi + echo "Deploying with $workshop settings" + ln -sf settings/$workshop-settings.yaml settings.yaml + deploy_tag $TAG + + fi + ;; + ids) + TAG=$2 + need_tag $TAG + IDS=$(aws_get_instance_ids_by_tag $TAG) + echo "$IDS" + + # Just in case we managed to create instances but weren't able to tag them + echo "Lookup by client token $TAG:" + IDS=$(aws_get_instance_ids_by_client_token $TAG) + echo "$IDS" + ;; + ips) + TAG=$2 + need_tag $TAG + mkdir -p tags/$TAG + aws_get_instance_ips_by_tag $TAG | tee tags/$TAG/ips.txt + link_tag $TAG + ;; + list) + # list existing instances in a given batch + # to list batches, see "tags" command + echo "Using region $AWS_DEFAULT_REGION." + TAG=$2 + need_tag $TAG + describe_tag $TAG + tag_is_reachable $TAG + echo "You may be interested in running one of the following commands:" + echo "trainer ips $TAG" + echo "trainer deploy $TAG" + ;; + opensg) + aws ec2 authorize-security-group-ingress \ + --group-name default \ + --protocol icmp \ + --port -1 \ + --cidr 0.0.0.0/0 + + aws ec2 authorize-security-group-ingress \ + --group-name default \ + --protocol udp \ + --port 0-65535 \ + --cidr 0.0.0.0/0 + + aws ec2 authorize-security-group-ingress \ + --group-name default \ + --protocol tcp \ + --port 0-65535 \ + --cidr 0.0.0.0/0 + ;; + pull-images) + TAG=$2 + need_tag $TAG + pull_tag $TAG + ;; + shell) + # Get a shell in the container + export PS1="trainer@$AWS_DEFAULT_REGION# " + exec $SHELL + ;; + start) + # Create $2 instances + COUNT=$2 + + if [ -z "$COUNT" ]; then + die "Indicate number of instances to start." + fi + + greet # Print our AWS username, to ease the pain of credential-juggling + key_name=$(sync_keys) # Upload our SSH keys to AWS if needed, to be added to each VM's authorized_keys + AMI=$(get_ami) # Retrieve the AWS image ID + TOKEN=$(get_token) # generate a timestamp token for this batch of VMs + if [ ! -z $3 ]; then + # If an extra arg is present, append it to the tag + TOKEN=$TOKEN-$3 + fi + + echo "-----------------------------------" + echo "Starting $COUNT instances:" + echo " Region: $AWS_DEFAULT_REGION" + echo " Token/tag: $TOKEN" + echo " AMI: $AMI" + + AWS_KEY_NAME=$(make_key_name) + result=$(aws ec2 run-instances \ + --key-name $AWS_KEY_NAME \ + --count $2 \ + --instance-type c3.large \ + --client-token $TOKEN \ + --image-id $AMI) + reservation_id=$(echo "$result" | head -1 | awk '{print $2}' ) + echo " Key name: $AWS_KEY_NAME" + echo "Reservation ID: $reservation_id" + echo "-----------------------------------" + + # if instance creation succeeded, we should have some IDs + IDS=$(aws_get_instance_ids_by_client_token $TOKEN) + if [ -z "$IDS" ]; then + die "Instance creation failed." + fi + + # Tag these new instances with a tag that is the same as the token + aws ec2 create-tags --tag Key=Name,Value=$TOKEN --resources $IDS >> /dev/null + TAG=$TOKEN + unset TOKEN + + wait_until_tag_is_running $TAG $COUNT + + echo "[-------------------------------------------------------------------------------------]" + echo " Successfully created $2 instances with tag: $TAG" + echo "[-------------------------------------------------------------------------------------]" + + mkdir -p tags/$TAG + IPS=$(aws_get_instance_ips_by_tag $TAG) + echo "$IPS" > tags/$TAG/ips.txt + link_tag $TAG + echo "To deploy or kill these instances, run one of the following:" + echo "trainer deploy $TAG" + echo "trainer list $TAG" + ;; + status) + greet && echo + + max_instances=$(aws ec2 describe-account-attributes \ + --attribute-names max-instances \ + --query 'AccountAttributes[*][AttributeValues]') + echo "Max instances: $max_instances" && echo + + # Print list of AWS EC2 regions, highlighting ours ($AWS_DEFAULT_REGION) in the list + # If our $AWS_DEFAULT_REGION is not valid, the error message will be pretty descriptive: + # Could not connect to the endpoint URL: "https://ec2.foo.amazonaws.com/" + echo "Region:" # $AWS_DEFAULT_REGION." + aws ec2 describe-regions | awk '{print $3}' | grep --color=auto $AWS_DEFAULT_REGION -C50 + + ;; + stop) + TAG=$2 + need_tag $TAG + aws_kill_instances_by_tag $TAG + ;; + tag) + # add a tag to a batch of VMs + TAG=$2 + NEW_TAG_KEY=$3 + NEW_TAG_VALUE=$4 + need_tag $TAG + need_tag $NEW_TAG_KEY + need_tag $NEW_TAG_VALUE + ;; + test) + TAG=$2 + need_tag $TAG + test_tag $TAG + ;; + *) + echo && echo "trainer COMMAND [n-instances|tag] + + Core commands: + start n Start n instances + list [TAG] If a tag is provided, list its VMs. Otherwise, list tags. + deploy TAG Deploy all instances with a given tag + pull-images TAG Pre-pull docker images. Run only after deploying. + stop TAG Stop and delete instances tagged TAG + + Extras: + ips TAG List all IPs of instances with a given tag (updates ips.txt) + ids TAG/TOKEN List all instance IDs with a given tag + shell Get a shell in the trainer container + status TAG Print information about this tag and its VMs + tags List all tags (per-region) + + Beta: + ami Look up Amazon Machine Images + cards Generate cards (see settings.txt) + opensg Modify AWS security groups + " + ;; + esac +} + +( + cd $SCRIPT_DIR + source scripts/cli.sh + source scripts/aws.sh + source scripts/rc + source scripts/colors.sh + mkdir -p tags + # TODO: unset empty envvars + run_cli "$@" +) diff --git a/prepare-vms/settings.yaml b/prepare-vms/settings.yaml new file mode 120000 index 00000000..2c4b7d7d --- /dev/null +++ b/prepare-vms/settings.yaml @@ -0,0 +1 @@ +./settings/fundamentals-settings.yaml \ No newline at end of file diff --git a/prepare-vms/settings/fundamentals-settings.yaml b/prepare-vms/settings/fundamentals-settings.yaml new file mode 100644 index 00000000..5e228ef5 --- /dev/null +++ b/prepare-vms/settings/fundamentals-settings.yaml @@ -0,0 +1,30 @@ +# This file is passed by trainer-cli to scripts/ips-txt-to-html.py + +workshop_name: Docker fundamentals +workshop_short_name: Docker # appears on VM connection cards +repo: https://github.com/docker/docker-fundamentals + +instance_login: docker +instance_password: training + +clustersize: 1 # Number of VMs per cluster +pagesize: 15 # Number of cards to print per page + +#background_image: https://myapps.developer.ubuntu.com/site_media/appmedia/2014/12/swarm.png +background_image: http://www.yellosoft.us/public/images/docker.png +#background_image: ../media/swarm.png + +# To be printed on the cards: +blurb: > + Here is the connection information to your very own + {cluster_or_machine} for this {workshop_name} workshop. You can connect + to {this_or_each} VM with any SSH client. + + Your {machine_is_or_machines_are}: + +# {url} will be replaced by the script +footer: > +

    For slides, chat and other useful links, see:

    +
    {url}
    + +url: http://container.training/ diff --git a/prepare-vms/settings/orchestration-settings.yaml b/prepare-vms/settings/orchestration-settings.yaml new file mode 100644 index 00000000..b549bb71 --- /dev/null +++ b/prepare-vms/settings/orchestration-settings.yaml @@ -0,0 +1,30 @@ +# This file is passed by trainer-cli to scripts/ips-txt-to-html.py + +workshop_name: Advanced Docker Orchestration +workshop_short_name: orchestration +repo: https://github.com/jpetazzo/orchestration-workshop + +instance_login: docker +instance_password: training + +clustersize: 5 # Number of VMs per cluster +pagesize: 15 # Number of cards to print per page + +#background_image: https://myapps.developer.ubuntu.com/site_media/appmedia/2014/12/swarm.png +background_image: http://www.yellosoft.us/public/images/docker.png +#background_image: ../media/swarm.png + +# To be printed on the cards: +blurb: > + Here is the connection information to your very own + {cluster_or_machine} for this {workshop_name} workshop. You can connect + to {this_or_each} VM with any SSH client. + + Your {machine_is_or_machines_are}: + +# {url} will be replaced by the script +footer: > +

    For slides, chat and other useful links, see:

    +
    {url}
    + +url: http://container.training/ diff --git a/prepare-vms/trainer b/prepare-vms/trainer new file mode 100755 index 00000000..db165422 --- /dev/null +++ b/prepare-vms/trainer @@ -0,0 +1,137 @@ +#!/bin/bash + +TRAINER_IMAGE="soulshake/prepare-vms" + +PIP_DEPENDENCIES=" + awscli +" + +# dpkg -l/pip/type-friendly +DEPENDENCIES=" + aws + awscli + ssh + curl + jq + bsdmainutils + pssh + pip + python-pip + wkhtmltopdf + JP_PLACEHOLDER + " + +OPTIONAL_DEPENDENCIES=" + man + " + +ENVVARS=" + AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY + AWS_DEFAULT_REGION + AWS_DEFAULT_PROFILE + SSH_AGENT_PID + SSH_AUTH_SOCK + SSH_AUTH_DIRNAME + " + +envvars_ok() { + # This script expects plain text output from the AWS CLI in some places + export AWS_DEFAULT_OUTPUT=text + + for envvar in $ENVVARS; do + if [ -z "${!envvar}" ]; then + echo "Please export $envvar environment variable. Unsetting." + unset $envvar + #return 1 + fi + done + return 0 +} + +package_is_installed() { + # Try "type", "dpkg -l", and "pip freeze" to see if a package is installed + dependency=$1 + type $dependency >/dev/null 2>&1 \ + || dpkg -l $dependency >/dev/null 2>&1 \ + || pip freeze | grep $dependency >/dev/null 2>&1 || return 1 +} + +show_missing_dependencies() { + for dependency in $DEPENDENCIES; do + package_is_installed $dependency \ + && echo " $dependency: [OK]" \ + || { echo >&2 "$(tput setaf 1) $dependency: [NOK] $(tput sgr0)" ;} + done + + echo "Recommended:" + for dependency in $OPTIONAL_DEPENDENCIES; do + package_is_installed $dependency \ + && echo " $dependency: [OK]" \ + || { echo >&2 "$(tput setaf 1) $dependency: [NOK] $(tput sgr0)" ;} + done +} + +run_natively() { + scripts/trainer-cli "$@" +} + +run_containerized() { + if [ -z $SSH_AUTH_SOCK ]; then + echo -n "SSH_AUTH_SOCK envvar not set, so its parent directory can't be " + echo "mounted as a volume in a container." + echo "Try running the command below and trying again:" + echo "eval \$(ssh-agent) && ssh-agent && ssh-add" + exit 1 + fi + + #/tmp/ssh-uN1oMimerRo0/agent.1411 + ( + cd $SCRIPT_DIR + export SSH_AUTH_DIRNAME=$(dirname $SSH_AUTH_SOCK) + docker-compose -f docker-compose.yml run prepare-vms "$@" + + ) +} + +dependencies_fulfilled() { + for dependency in $DEPENDENCIES ; do + package_is_installed $dependency || return 1 + done + + echo "All dependencies fulfilled. Running locally." +} + +trainer_image_exists_locally() { + ret=$(docker inspect $TRAINER_IMAGE >/dev/null 2>&1) + return $? +} + +# Get the script's real directory, whether we're being called directly or via a symlink +if [ -L $0 ]; then + export SCRIPT_DIR=$(dirname $(readlink $0)) +else + export SCRIPT_DIR=$(dirname $(readlink -f $0)) +fi + +export SSH_AUTH_DIRNAME=$(dirname $SSH_AUTH_SOCK) + +if ! envvars_ok; then + exit 1 +fi + +if dependencies_fulfilled; then + ( + cd $SCRIPT_DIR + run_natively "$@" + ) +elif trainer_image_exists_locally; then + run_containerized "$@" +else + echo "Some dependencies are missing, and docker image $TRAINER_IMAGE doesn't exist locally." + echo "Please do one of the following: " + echo "- run \`docker build -t soulshake/prepare-vms .\`" + echo "- run \`docker pull soulshake/prepare-vms\`" + echo "- install all dependencies: " + show_missing_dependencies +fi