mirror of
https://github.com/jpetazzo/container.training.git
synced 2026-08-18 03:26:23 +00:00
Complete rewrite of deployment process
The former VM deployment process relied on extra scripts located in a (private) repo. The new process is standalone.
This commit is contained in:
AJ Bowen
committed by
Jérôme Petazzoni
Jérôme Petazzoni
parent
1be92ea3fa
commit
9e5c7c8216
@@ -0,0 +1,52 @@
|
||||
FROM debian:jessie
|
||||
MAINTAINER AJ Bowen <aj@soulshake.net>
|
||||
|
||||
RUN apt-get update
|
||||
RUN apt-get install -y ca-certificates
|
||||
RUN apt-get install -y groff
|
||||
RUN apt-get install -y less
|
||||
RUN apt-get install -y python python-pip
|
||||
RUN apt-get install -y python-docutils
|
||||
RUN apt-get install -y sudo
|
||||
RUN apt-get install -y \
|
||||
bsdmainutils \
|
||||
curl \
|
||||
jq \
|
||||
less \
|
||||
man \
|
||||
pssh \
|
||||
ssh
|
||||
|
||||
RUN pip install awscli
|
||||
RUN pip install \
|
||||
pdfkit \
|
||||
PyYAML \
|
||||
termcolor
|
||||
|
||||
RUN apt-get install -y wkhtmltopdf
|
||||
|
||||
ENV HOME /home/user
|
||||
|
||||
RUN useradd --create-home --home-dir $HOME user \
|
||||
&& mkdir -p $HOME/.config/gandi \
|
||||
&& chown -R user:user $HOME
|
||||
|
||||
RUN echo "user ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
|
||||
|
||||
# Replace 1000 with your user / group id
|
||||
#RUN export uid=1000 gid=1000 && \
|
||||
# mkdir -p /home/user && \
|
||||
# mkdir -p /etc/sudoers.d && \
|
||||
# echo "user:x:${uid}:${gid}:user,,,:/home/user:/bin/bash" >> /etc/passwd && \
|
||||
# echo "user:x:${uid}:" >> /etc/group && \
|
||||
# echo "user ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/user && \
|
||||
# chmod 0440 /etc/sudoers.d/user && \
|
||||
# chown ${uid}:${gid} -R /home/user
|
||||
|
||||
WORKDIR $HOME
|
||||
RUN echo "alias ll='ls -lahF'" >> /home/user/.bashrc
|
||||
RUN echo "export PATH=$PATH:/home/user/bin" >> /home/user/.bashrc
|
||||
RUN mkdir -p /home/user/bin
|
||||
RUN ln -s /home/user/prepare-vms/scripts/trainer-cli /home/user/bin/trainer-cli
|
||||
USER user
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
# Trainer tools to prepare VMs for Docker workshops
|
||||
|
||||
There are several options for using these tools:
|
||||
|
||||
### Clone the repo
|
||||
|
||||
$ git clone https://github.com/soulshake/prepare-vms.git
|
||||
$ cd prepare-vms
|
||||
$ docker-compose build
|
||||
$ mkdir $HOME/bin && ln -s `pwd`/trainer $HOME/bin/trainer
|
||||
|
||||
### Via the image
|
||||
|
||||
$ docker pull soulshake/prepare-vms
|
||||
|
||||
### Submodule
|
||||
|
||||
This repo can be added as a submodule in the repo of the Docker workshop:
|
||||
|
||||
$ git submodule add https://github.com/soulshake/prepare-vms.git
|
||||
|
||||
## Setup
|
||||
|
||||
### Export needed envvars
|
||||
|
||||
Required environment variables:
|
||||
|
||||
* `AWS_ACCESS_KEY_ID`
|
||||
* `AWS_SECRET_ACCESS_KEY`
|
||||
* `AWS_DEFAULT_REGION`
|
||||
|
||||
|
||||
|
||||
### Update settings.yaml
|
||||
|
||||
If you have more than one workshop:
|
||||
|
||||
$ cp settings.yaml settings/YOUR_WORKSHOP_NAME-settings.yaml
|
||||
$ ln -s settings/YOUR_WORKSHOP_NAME-settings.yaml `pwd`/settings.yaml
|
||||
|
||||
Update the `settings.yaml` as needed. This is the file that will be used to generate cards.
|
||||
|
||||
## Usage
|
||||
|
||||
### Summary
|
||||
|
||||
Summary of steps to launch a batch of instances for a workshop:
|
||||
|
||||
* Export the environment variables needed by the AWS CLI (see **Requirements** below)
|
||||
* `trainer start NUMBER_OF_VMS` to create AWS instances
|
||||
* `trainer deploy TAG` to run `scripts/postprep.rc` via parallel-ssh
|
||||
* `trainer pull-images TAG` to pre-pull a bunch of Docker images
|
||||
* `trainer test TAG`
|
||||
* `trainer cards TAG` to generate a PDF and an HTML file you can print
|
||||
|
||||
The `trainer` script can be executed directly.
|
||||
|
||||
It will check for the necessary environment variables. Then, if all its dependencies are installed
|
||||
locally, it will execute `trainer-cli`. If not, it will look for a local Docker image
|
||||
tagged `soulshake/trainer-tools`. If found, it will run in a container. If not found,
|
||||
the user will be prompted to either install the missing dependencies or download
|
||||
the Docker image.
|
||||
|
||||
## Detailed usage
|
||||
|
||||
### Start some VMs
|
||||
|
||||
$ trainer start 10
|
||||
|
||||
A few things will happen:
|
||||
|
||||
#### Sync of SSH keys
|
||||
|
||||
When the `start` command is run, your local RSA SSH public key will be added to your AWS EC2 keychain.
|
||||
|
||||
To see which local key will be uploaded, run `ssh-add -l | grep RSA`.
|
||||
|
||||
#### Instance + tag creation
|
||||
|
||||
10 VMs will be started, with an automatically generated tag (timestamp + your username).
|
||||
|
||||
Your SSH key will be added to the `authorized_keys` of the ubuntu user.
|
||||
|
||||
#### Creation of ./$TAG/ directory and contents
|
||||
|
||||
Following the creation of the VMs, a text file will be created containing a list of their IPs.
|
||||
|
||||
This ips.txt file will be created in the $TAG/ directory and a symlink will be placed in the working directory of the script.
|
||||
|
||||
If you create new VMs, the symlinked file will be overwritten.
|
||||
|
||||
## Deployment
|
||||
|
||||
Instances can be deployed manually using the `deploy` command:
|
||||
|
||||
$ trainer deploy TAG
|
||||
|
||||
The `postprep.rc` file will be copied via parallel-ssh to all of the VMs and executed.
|
||||
|
||||
### Pre-pull images
|
||||
|
||||
$ trainer pull-images TAG
|
||||
|
||||
### Generate cards
|
||||
|
||||
$ trainer cards TAG
|
||||
|
||||
### List tags
|
||||
|
||||
$ trainer list
|
||||
|
||||
### List VMs
|
||||
|
||||
$ trainer list TAG
|
||||
|
||||
This will print a human-friendly list containing some information about each instance.
|
||||
|
||||
### Stop and destroy VMs
|
||||
|
||||
$ trainer stop TAG
|
||||
|
||||
## ToDo
|
||||
|
||||
* Don't write to bash history in system() in postprep
|
||||
* compose, etc version inconsistent (int vs str)
|
||||
@@ -0,0 +1,38 @@
|
||||
prepare-vms:
|
||||
build: .
|
||||
container_name: prepare-vms
|
||||
working_dir: /home/user/prepare-vms
|
||||
volumes:
|
||||
- $HOME/.aws/:$HOME.aws/
|
||||
- $HOME/.ssh/:/home/user/.ssh/
|
||||
- /etc/localtime:/etc/localtime:ro
|
||||
#- /home:/home
|
||||
- /tmp/.X11-unix:/tmp/.X11-unix
|
||||
- $SSH_AUTH_DIRNAME:$SSH_AUTH_DIRNAME
|
||||
- $SCRIPT_DIR/:/home/user/prepare-vms/
|
||||
#- $SCRIPT_DIR/:$HOME/prepare-vms/
|
||||
#- $HOME/trainer-tools/
|
||||
#- $(dirname $SSH_AUTH_SOCK):$(dirname $SSH_AUTH_SOCK)
|
||||
#- /etc/passwd:/etc/passwd:ro
|
||||
#- /etc/group:/etc/group:ro
|
||||
#- /run/user:/run/user
|
||||
environment:
|
||||
HOME: /home/user
|
||||
#SCRIPT_DIR: /home/aj/git/prepare-vms
|
||||
SCRIPT_DIR: /home/user/prepare-vms
|
||||
HOME: /home/user
|
||||
SSH_AUTH_SOCK: ${SSH_AUTH_SOCK}
|
||||
SSH_AGENT_PID: ${SSH_AGENT_PID}
|
||||
AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
|
||||
AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
|
||||
DISPLAY: ${DISPLAY}
|
||||
USER: ${USER}
|
||||
AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION}
|
||||
AWS_DEFAULT_OUTPUT:
|
||||
AWS_INSTANCE_TYPE: ${AWS_INSTANCE_TYPE}
|
||||
AWS_VPC_ID: ${AWS_VPC_ID}
|
||||
entrypoint: /home/user/prepare-vms/scripts/trainer-cli
|
||||
#entrypoint: trainer
|
||||
|
||||
#AWS_DEFAULT_PROFILE: ${AWS_DEFAULT_PROFILE}
|
||||
#command: /home/user/prepare-vms/trainer-cli
|
||||
@@ -1,82 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
SETTINGS_BASIC = dict(
|
||||
clustersize=1,
|
||||
pagesize=15,
|
||||
blurb="<p>Here is the connection information to your very own "
|
||||
"VM for this intro to Docker workshop. You can connect "
|
||||
"to the VM using your SSH client.</p>\n"
|
||||
"<p>Your VM is reachable on the following address:</p>\n",
|
||||
prettify=lambda x: x,
|
||||
footer="<p>You can find the last version of the slides on "
|
||||
"http://view.dckr.info/.</p>",
|
||||
)
|
||||
|
||||
SETTINGS_ADVANCED = dict(
|
||||
clustersize=5,
|
||||
pagesize=12,
|
||||
blurb="<p>Here is the connection information to your very own "
|
||||
"cluster for this orchestration workshop. You can connect "
|
||||
"to each VM with any SSH client.</p>\n"
|
||||
"<p>Your machines are:<ul>\n",
|
||||
prettify=lambda l: [ "node%d: %s"%(i+1, s)
|
||||
for (i, s) in zip(range(len(l)), l) ],
|
||||
footer="<p>You can find the last version of the slides on -> "
|
||||
"http://container.training/</p>"
|
||||
)
|
||||
|
||||
SETTINGS = SETTINGS_ADVANCED
|
||||
|
||||
globals().update(SETTINGS)
|
||||
|
||||
###############################################################################
|
||||
|
||||
ips = list(open("ips.txt"))
|
||||
|
||||
assert len(ips)%clustersize == 0
|
||||
|
||||
clusters = []
|
||||
|
||||
while ips:
|
||||
cluster = ips[:clustersize]
|
||||
ips = ips[clustersize:]
|
||||
clusters.append(cluster)
|
||||
|
||||
html = open("ips.html", "w")
|
||||
html.write("<html><head><style>")
|
||||
html.write("""
|
||||
div {
|
||||
float:left;
|
||||
border: 1px solid black;
|
||||
width: 28%;
|
||||
padding: 4% 2.5% 2.5% 2.5%;
|
||||
font-size: x-small;
|
||||
background-image: url("docker-nb.svg");
|
||||
background-size: 15%;
|
||||
background-position-x: 50%;
|
||||
background-repeat: no-repeat;
|
||||
}
|
||||
p {
|
||||
margin: 0.5em 0 0.5em 0;
|
||||
}
|
||||
.pagebreak {
|
||||
page-break-before: always;
|
||||
clear: both;
|
||||
display: block;
|
||||
height: 8px;
|
||||
}
|
||||
""")
|
||||
html.write("</style></head><body>")
|
||||
for i, cluster in enumerate(clusters):
|
||||
if i>0 and i%pagesize==0:
|
||||
html.write('<span class="pagebreak"></span>\n')
|
||||
html.write("<div>")
|
||||
html.write(blurb)
|
||||
for s in prettify(cluster):
|
||||
html.write("<li>%s</li>\n"%s)
|
||||
html.write("</ul></p>")
|
||||
html.write("<p>login=docker password=training</p>\n")
|
||||
html.write(footer)
|
||||
html.write("</div>")
|
||||
html.close()
|
||||
|
||||
|
Before Width: | Height: | Size: 6.5 KiB After Width: | Height: | Size: 6.5 KiB |
|
Before Width: | Height: | Size: 65 KiB After Width: | Height: | Size: 65 KiB |
@@ -1,51 +0,0 @@
|
||||
pssh -I tee /tmp/postprep.py <<EOF
|
||||
#!/usr/bin/env python
|
||||
COMPOSE_VERSION = "1.6.2"
|
||||
MACHINE_VERSION = "0.6.0"
|
||||
SWARM_VERSION = "1.1.3"
|
||||
|
||||
import os
|
||||
import sys
|
||||
import urllib
|
||||
|
||||
clustersize = 5
|
||||
|
||||
myaddr = urllib.urlopen("http://myip.enix.org/REMOTE_ADDR").read()
|
||||
|
||||
addresses = list(l.strip() for l in sys.stdin)
|
||||
|
||||
def makenames(addrs):
|
||||
return [ "node%s"%(i+1) for i in range(len(addrs)) ]
|
||||
|
||||
while addresses:
|
||||
cluster = addresses[:clustersize]
|
||||
addresses = addresses[clustersize:]
|
||||
if myaddr not in cluster:
|
||||
continue
|
||||
names = makenames(cluster)
|
||||
for ipaddr, name in zip(cluster, names):
|
||||
os.system("grep ^%s /etc/hosts || echo %s %s | sudo tee -a /etc/hosts"
|
||||
%(ipaddr, ipaddr, name))
|
||||
if myaddr == cluster[0]:
|
||||
os.system("[ -f .ssh/id_rsa ] || ssh-keygen -t rsa -f .ssh/id_rsa -P ''")
|
||||
|
||||
os.system("sudo apt-get remove -y --purge dnsmasq-base")
|
||||
os.system("sudo apt-get -qy install python-setuptools pssh apache2-utils httping htop unzip")
|
||||
os.system("sudo easy_install pip")
|
||||
os.system("sudo pip uninstall -y docker-compose")
|
||||
#os.system("sudo pip install docker-compose=={}".format(COMPOSE_VERSION))
|
||||
os.system("sudo curl -sSL -o /usr/local/bin/docker-compose https://github.com/docker/compose/releases/download/{}/docker-compose-$(uname -s)-$(uname -m)".format(COMPOSE_VERSION))
|
||||
os.system("sudo chmod +x /usr/local/bin/docker-compose")
|
||||
os.system("docker pull swarm:{}".format(SWARM_VERSION))
|
||||
os.system("docker tag -f swarm:{} swarm".format(SWARM_VERSION))
|
||||
#os.system("sudo curl -sSL https://github.com/docker/machine/releases/download/v{}/docker-machine_linux-amd64.zip -o /tmp/docker-machine.zip".format(MACHINE_VERSION))
|
||||
#os.system("cd /usr/local/bin ; sudo unzip /tmp/docker-machine.zip")
|
||||
os.system("sudo curl -sSL -o /usr/local/bin/docker-machine https://github.com/docker/machine/releases/download/v{}/docker-machine-$(uname -s)-$(uname -m)".format(MACHINE_VERSION))
|
||||
os.system("sudo chmod +x /usr/local/bin/docker-machine*")
|
||||
os.system("echo 1000000 | sudo tee /proc/sys/net/nf_conntrack_max")
|
||||
#os.system("""sudo sed -i 's,^DOCKER_OPTS=.*,DOCKER_OPTS="-H unix:///var/run/docker.sock -H tcp://0.0.0.0:55555",' /etc/default/docker""")
|
||||
#os.system("sudo service docker restart")
|
||||
EOF
|
||||
pssh -t 300 -I "python /tmp/postprep.py >>/tmp/pp.out 2>>/tmp/pp.err" < ips.txt
|
||||
pssh "[ -f .ssh/id_rsa ] || scp -o StrictHostKeyChecking=no node1:.ssh/id_rsa* .ssh"
|
||||
pssh "grep docker@ .ssh/authorized_keys || cat .ssh/id_rsa.pub >> .ssh/authorized_keys"
|
||||
@@ -1,23 +0,0 @@
|
||||
pssh () {
|
||||
HOSTFILES="hosts ips.txt /tmp/hosts"
|
||||
for HOSTFILE in $HOSTFILES; do
|
||||
[ -f $HOSTFILE ] && break
|
||||
done
|
||||
[ -f $HOSTFILE ] || {
|
||||
echo "No hostfile found (tried $HOSTFILES)"
|
||||
return
|
||||
}
|
||||
parallel-ssh -h $HOSTFILE -l docker \
|
||||
-O UserKnownHostsFile=/dev/null -O StrictHostKeyChecking=no \
|
||||
-O ForwardAgent=yes \
|
||||
"$@"
|
||||
}
|
||||
|
||||
pcopykey () {
|
||||
ssh-add -L | pssh --askpass --send-input \
|
||||
"mkdir -p .ssh; tee .ssh/authorized_keys"
|
||||
ssh-add -L | pssh --send-input \
|
||||
"sudo mkdir -p /root/.ssh; sudo tee /root/.ssh/authorized_keys"
|
||||
}
|
||||
|
||||
|
||||
Executable
+107
@@ -0,0 +1,107 @@
|
||||
#!/bin/bash
|
||||
|
||||
source scripts/cli.sh
|
||||
|
||||
aws_display_tags(){
|
||||
# Print all "Name" tags in our region with their instance count
|
||||
echo "[#] [Status] [Tag]" | awk '{ printf " %7s %8s %10s \n", $1, $2, $3}'
|
||||
aws ec2 describe-instances --filter "Name=tag:Name,Values=[*]" \
|
||||
--query "Reservations[*].Instances[*].[{Tags:Tags[0].Value,State:State.Name}]" \
|
||||
| awk '{ printf " %-13s %-10s %-1s\n", $1, $2, $3}' \
|
||||
| uniq -c
|
||||
}
|
||||
|
||||
aws_display_tokens(){
|
||||
# Print all tokens in our region with their instance count
|
||||
echo "[#] [Token] [Tag]" | awk '{ printf " %7s %12s %30s\n", $1, $2, $3}'
|
||||
# --query 'Volumes[*].{ID:VolumeId,AZ:AvailabilityZone,Size:Size}'
|
||||
aws ec2 describe-instances --output text \
|
||||
--query 'Reservations[*].Instances[*].{ClientToken:ClientToken,Tags:Tags[0].Value}' \
|
||||
| awk '{ printf " %7s %12s %50s\n", $1, $2, $3}' \
|
||||
| sort \
|
||||
| uniq -c
|
||||
}
|
||||
|
||||
aws_get_tokens() {
|
||||
aws ec2 describe-instances --output text \
|
||||
--query 'Reservations[*].Instances[*].[ClientToken]' \
|
||||
| sort -u
|
||||
}
|
||||
|
||||
aws_display_instance_statuses_by_tag() {
|
||||
TAG=$1
|
||||
need_tag $TAG
|
||||
|
||||
IDS=$(aws ec2 describe-instances \
|
||||
--filters "Name=tag:Name,Values=$TAG" \
|
||||
--query "Reservations[*].Instances[*].InstanceId" | tr '\t' ' ' )
|
||||
|
||||
aws ec2 describe-instance-status \
|
||||
--instance-ids $IDS \
|
||||
--query "InstanceStatuses[*].{ID:InstanceId,InstanceState:InstanceState.Name,InstanceStatus:InstanceStatus.Status,SystemStatus:SystemStatus.Status,Reachability:InstanceStatus.Status}" \
|
||||
--output table
|
||||
}
|
||||
|
||||
aws_display_instances_by_tag() {
|
||||
TAG=$1
|
||||
need_tag $TAG
|
||||
result=$(aws ec2 describe-instances --output table \
|
||||
--filter "Name=tag:Name,Values=$TAG" \
|
||||
--query "Reservations[*].Instances[*].[ \
|
||||
InstanceId, \
|
||||
State.Name, \
|
||||
Tags[0].Value, \
|
||||
PublicIpAddress, \
|
||||
InstanceType \
|
||||
]"
|
||||
)
|
||||
if [[ -z $result ]]; then
|
||||
echo "No instances found with tag $TAG in region $AWS_DEFAULT_REGION."
|
||||
else
|
||||
echo "ID State Tags IP Type" \
|
||||
| awk '{ printf "%9s %12s %15s %20s %14s \n", $1, $2, $3, $4, $5}' # column -t -c 70}
|
||||
echo "$result"
|
||||
fi
|
||||
}
|
||||
|
||||
aws_get_instance_ids_by_client_token() {
|
||||
TOKEN=$1
|
||||
need_tag $TOKEN
|
||||
aws ec2 describe-instances --filters "Name=client-token,Values=$TOKEN" \
|
||||
| grep ^INSTANCE \
|
||||
| awk '{print $8}'
|
||||
}
|
||||
|
||||
aws_get_instance_ids_by_tag() {
|
||||
TAG=$1
|
||||
need_tag $TAG
|
||||
aws ec2 describe-instances --filters "Name=tag:Name,Values=$TAG" \
|
||||
| grep ^INSTANCE \
|
||||
| awk '{print $8}'
|
||||
}
|
||||
|
||||
aws_get_instance_ips_by_tag() {
|
||||
TAG=$1
|
||||
need_tag $TAG
|
||||
aws ec2 describe-instances --filter "Name=tag:Name,Values=$TAG" \
|
||||
--output text \
|
||||
--query "Reservations[*].Instances[*].PublicIpAddress" \
|
||||
| tr "\t" "\n" \
|
||||
| sort -n -t . -k 1,1 -k 2,2 -k 3,3 -k 4,4 # sort IPs
|
||||
}
|
||||
|
||||
aws_kill_instances_by_tag() {
|
||||
TAG=$1
|
||||
need_tag $TAG
|
||||
IDS=$(aws_get_instance_ids_by_tag $TAG)
|
||||
if [ -z "$IDS" ]; then
|
||||
die "Invalid tag."
|
||||
else
|
||||
echo "Deleting instances with tag $TAG"
|
||||
fi
|
||||
|
||||
echo "$IDS" \
|
||||
| xargs -r aws ec2 terminate-instances --instance-ids \
|
||||
| grep ^TERMINATINGINSTANCES
|
||||
}
|
||||
|
||||
Executable
+40
@@ -0,0 +1,40 @@
|
||||
die () {
|
||||
if [ -z "$1" ]; then
|
||||
exit 1
|
||||
fi
|
||||
echo -n $(tput setaf 1)
|
||||
echo -e "$1"
|
||||
echo -n $(tput sgr0)
|
||||
exit 1
|
||||
}
|
||||
|
||||
need_tag(){
|
||||
TAG=$1
|
||||
if [ -z "$TAG" ]; then
|
||||
echo "Please specify a tag. Here's the list: "
|
||||
aws_display_tags
|
||||
die
|
||||
fi
|
||||
}
|
||||
|
||||
need_token(){
|
||||
TOKEN=$1
|
||||
if [ -z "$TOKEN" ]; then
|
||||
echo "Please specify a token. Here's the list: "
|
||||
aws_display_tokens
|
||||
die
|
||||
fi
|
||||
}
|
||||
|
||||
need_ips_file() {
|
||||
IPS_FILE=$1
|
||||
if [ -z "$IPS_FILE" ]; then
|
||||
echo "IPS_FILE not set."
|
||||
die
|
||||
fi
|
||||
|
||||
if [ ! -s "$IPS_FILE" ]; then
|
||||
echo "IPS_FILE $IPS_FILE not found. Please run: trainer ips <TAG>"
|
||||
die
|
||||
fi
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
bold() {
|
||||
msg=$1
|
||||
echo "$(tput bold)$1$(tput sgr0)"
|
||||
}
|
||||
|
||||
green() {
|
||||
msg=$1
|
||||
echo "$(tput setaf 2)$1$(tput sgr0)"
|
||||
}
|
||||
|
||||
yellow(){
|
||||
msg=$1
|
||||
echo "$(tput setaf 3)$1$(tput sgr0)"
|
||||
}
|
||||
|
||||
Executable
+132
@@ -0,0 +1,132 @@
|
||||
#!/bin/bash
|
||||
# borrowed from https://gist.github.com/kirikaza/6627072
|
||||
|
||||
usage() {
|
||||
cat >&2 <<__
|
||||
usage: find-ubuntu-ami.sh [ <filter>... ] [ <sorting> ]
|
||||
where:
|
||||
<filter> is pair of key and substring to search
|
||||
-r <region>
|
||||
-n <name>
|
||||
-v <version>
|
||||
-a <arch>
|
||||
-t <type>
|
||||
-d <date>
|
||||
-i <image>
|
||||
-k <kernel>
|
||||
<sorting> is on of:
|
||||
-R by region
|
||||
-N by name
|
||||
-V by version
|
||||
-A by arch
|
||||
-T by type
|
||||
-D by date
|
||||
-I by image
|
||||
-K by kernel
|
||||
|
||||
protip for Docker orchestration workshop admin:
|
||||
./find-ubuntu-ami.sh -t hvm:ebs -r \$AWS_REGION -v 15.10 -N
|
||||
__
|
||||
exit 1
|
||||
}
|
||||
|
||||
args=`getopt hr:n:v:a:t:d:i:k:RNVATDIK $*`
|
||||
if [ $? != 0 ] ; then
|
||||
echo >&2
|
||||
usage
|
||||
fi
|
||||
|
||||
region=
|
||||
name=
|
||||
version=
|
||||
arch=
|
||||
type=
|
||||
date=
|
||||
image=
|
||||
kernel=
|
||||
|
||||
sort=date
|
||||
|
||||
set -- $args
|
||||
for a ; do
|
||||
case "$a" in
|
||||
-h) usage ;;
|
||||
|
||||
-r) region=$2 ; shift ;;
|
||||
-n) name=$2 ; shift ;;
|
||||
-v) version=$2 ; shift ;;
|
||||
-a) arch=$2 ; shift ;;
|
||||
-t) type=$2 ; shift ;;
|
||||
-d) date=$2 ; shift ;;
|
||||
-i) image=$2 ; shift ;;
|
||||
-k) kernel=$2 ; shift ;;
|
||||
|
||||
-R) sort=region ;;
|
||||
-N) sort=name ;;
|
||||
-V) sort=version ;;
|
||||
-A) sort=arch ;;
|
||||
-T) sort=type ;;
|
||||
-D) sort=date ;;
|
||||
-I) sort=image ;;
|
||||
-K) sort=kernel ;;
|
||||
|
||||
--) shift ; break ;;
|
||||
*) continue ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
[ $# = 0 ] || usage
|
||||
|
||||
fix_json() {
|
||||
tr -d \\n | sed 's/,]}/]}/'
|
||||
}
|
||||
|
||||
jq_query() { cat <<__
|
||||
.aaData | map (
|
||||
{
|
||||
region: .[0],
|
||||
name: .[1],
|
||||
version: .[2],
|
||||
arch: .[3],
|
||||
type: .[4],
|
||||
date: .[5],
|
||||
image: .[6],
|
||||
kernel: .[7]
|
||||
} | select (
|
||||
(.region | contains("$region")) and
|
||||
(.name | contains("$name")) and
|
||||
(.version | contains("$version")) and
|
||||
(.arch | contains("$arch")) and
|
||||
(.type | contains("$type")) and
|
||||
(.date | contains("$date")) and
|
||||
(.image | contains("$image</a>")) and
|
||||
(.kernel | contains("$kernel"))
|
||||
)
|
||||
) | sort_by(.$sort) | .[] |
|
||||
"\(.region)|\(.name)|\(.version)|\(.arch)|\(.type)|\(.date)|\(.image)|\(.kernel)"
|
||||
__
|
||||
}
|
||||
|
||||
trim_quotes() {
|
||||
sed 's/^"//;s/"$//'
|
||||
}
|
||||
|
||||
escape_spaces() {
|
||||
sed 's/ /\\\ /g'
|
||||
}
|
||||
|
||||
url=http://cloud-images.ubuntu.com/locator/ec2/releasesTable
|
||||
|
||||
{
|
||||
echo REGION NAME VERSION ARCH TYPE DATE IMAGE KERNEL
|
||||
curl -s $url | fix_json | jq "`jq_query`" | trim_quotes | escape_spaces | tr \| ' '
|
||||
} |
|
||||
while read region name version arch type date image kernel ; do
|
||||
image=${image%<*}
|
||||
image=${image#*>}
|
||||
echo "$region|$name|$version|$arch|$type|$date|$image|$kernel"
|
||||
done | column -t -s \|
|
||||
|
||||
|
||||
|
||||
Executable
+119
@@ -0,0 +1,119 @@
|
||||
#!/usr/bin/env python
|
||||
import os
|
||||
import sys
|
||||
import yaml
|
||||
import pdfkit
|
||||
|
||||
def prettify(l):
|
||||
l = [ip.strip() for ip in l]
|
||||
ret = [ "node{}: <code>{}</code>".format(i+1, s) for (i, s) in zip(range(len(l)), l) ]
|
||||
return ret
|
||||
|
||||
|
||||
# Read settings from settings.yaml
|
||||
#with open(sys.argv[1]) as f:
|
||||
#with open("settings.yaml") as f:
|
||||
with open("settings/fundamentals-settings.yaml") as f:
|
||||
data = f.read()
|
||||
|
||||
SETTINGS = yaml.load(data)
|
||||
SETTINGS['footer'] = SETTINGS['footer'].format(url=SETTINGS['url'])
|
||||
globals().update(SETTINGS)
|
||||
|
||||
###############################################################################
|
||||
|
||||
ips = list(open("ips.txt"))
|
||||
|
||||
print("Current settings (as defined in settings.yaml):")
|
||||
print(" Number of IPs: {}".format(len(ips)))
|
||||
print(" VMs per cluster: {}".format(clustersize))
|
||||
print("Background image: {}".format(background_image))
|
||||
print("---------------------------------------------")
|
||||
|
||||
assert len(ips)%clustersize == 0
|
||||
|
||||
if clustersize == 1:
|
||||
blurb = blurb.format(
|
||||
cluster_or_machine="machine",
|
||||
this_or_each="this",
|
||||
machine_is_or_machines_are="machine is",
|
||||
workshop_name=workshop_short_name,
|
||||
)
|
||||
else:
|
||||
blurb = blurb.format(
|
||||
cluster_or_machine="cluster",
|
||||
this_or_each="each",
|
||||
machine_is_or_machines_are="machines are",
|
||||
workshop_name=workshop_short_name,
|
||||
)
|
||||
|
||||
clusters = []
|
||||
|
||||
while ips:
|
||||
cluster = ips[:clustersize]
|
||||
ips = ips[clustersize:]
|
||||
clusters.append(cluster)
|
||||
|
||||
html = open("ips.html", "w")
|
||||
html.write("<html><head><style>")
|
||||
head = """
|
||||
div {{
|
||||
float:left;
|
||||
border: 1px dotted black;
|
||||
width: 27%;
|
||||
padding: 6% 3.5% 2.5% 2.5%;
|
||||
font-size: x-medium;
|
||||
background-image: url("{background_image}");
|
||||
background-size: 13%;
|
||||
background-position-x: 50%;
|
||||
background-position-y: 5%;
|
||||
background-repeat: no-repeat;
|
||||
}}
|
||||
|
||||
p {{
|
||||
margin: 0.5em 0 0.5em 0;
|
||||
}}
|
||||
|
||||
.pagebreak {{
|
||||
page-break-before: always;
|
||||
clear: both;
|
||||
display: block;
|
||||
height: 8px;
|
||||
}}
|
||||
"""
|
||||
|
||||
|
||||
head = head.format(background_image=SETTINGS['background_image'])
|
||||
html.write(head)
|
||||
|
||||
html.write("</style></head><body>")
|
||||
for i, cluster in enumerate(clusters):
|
||||
if i>0 and i%pagesize==0:
|
||||
html.write('<span class="pagebreak"></span>\n')
|
||||
|
||||
html.write("<div>")
|
||||
html.write(blurb)
|
||||
for s in prettify(cluster):
|
||||
html.write("<li>%s</li>\n"%s)
|
||||
html.write("</ul></p>")
|
||||
html.write("<p>login: <b><code>{}</code></b> <br>password: <b><code>{}</code></b></p>\n".format(instance_login, instance_password))
|
||||
html.write(footer)
|
||||
html.write("</div>")
|
||||
html.close()
|
||||
|
||||
"""
|
||||
html.write("<div>")
|
||||
html.write("<p>{}</p>".format(blurb))
|
||||
for s in prettify(cluster):
|
||||
html.write("<li>{}</li>".format(s))
|
||||
html.write("</ul></p>")
|
||||
html.write("<center>")
|
||||
html.write("<p>login: <b><code>{}</code></b>    password: <b><code>{}</code></b></p>\n".format(instance_login, instance_password))
|
||||
html.write("</center>")
|
||||
html.write(footer)
|
||||
html.write("</div>")
|
||||
html.close()
|
||||
"""
|
||||
|
||||
with open('ips.html') as f:
|
||||
pdfkit.from_file(f, 'ips.pdf')
|
||||
Executable
+157
@@ -0,0 +1,157 @@
|
||||
pssh -I tee /tmp/postprep.py <<EOF
|
||||
#!/usr/bin/env python
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import urllib
|
||||
|
||||
#################################
|
||||
|
||||
COMPOSE_VERSION = "1.6.2"
|
||||
MACHINE_VERSION = os.environ.get("MACHINE_VERSION") or "0.6.0"
|
||||
SWARM_VERSION = os.environ.get("SWARM_VERSION") or "1.1.3"
|
||||
CLUSTER_SIZE = os.environ.get("CLUSTER_SIZE") or 1
|
||||
|
||||
#################################
|
||||
|
||||
# This script will be run as ubuntu user, which has root privileges.
|
||||
# docker commands will require sudo because the ubuntu user has no access to the docker socket.
|
||||
|
||||
STEP = 0
|
||||
START = time.time()
|
||||
|
||||
def bold(msg):
|
||||
return "{} {} {}".format("$(tput smso)", msg, "$(tput rmso)")
|
||||
|
||||
def system(cmd):
|
||||
global STEP
|
||||
with open("/tmp/pp.status", "a") as f:
|
||||
t1 = time.time()
|
||||
f.write(bold("--- RUNNING [step {}] ---> {}...".format(STEP, cmd)))
|
||||
retcode = os.system(cmd)
|
||||
if retcode:
|
||||
retcode = bold(retcode)
|
||||
t2 = time.time()
|
||||
td = str(t2-t1)[:5]
|
||||
f.write("[{}] in {}s\n".format(retcode, td))
|
||||
STEP += 1
|
||||
with open("/home/ubuntu/.bash_history", "a") as f:
|
||||
f.write("{}\n".format(cmd))
|
||||
|
||||
# On EC2, the ephemeral disk might be mounted on /mnt.
|
||||
# If /mnt is a mountpoint, place Docker workspace on it.
|
||||
system("if mountpoint -q /mnt; then sudo mkdir /mnt/docker && sudo ln -s /mnt/docker /var/lib/docker; fi")
|
||||
|
||||
# Put our public IP in /tmp/ipv4
|
||||
# ipv4_retrieval_endpoint = "http://169.254.169.254/latest/meta-data/public-ipv4"
|
||||
ipv4_retrieval_endpoint = "http://myip.enix.org/REMOTE_ADDR"
|
||||
system("curl --silent {} > /tmp/ipv4".format(ipv4_retrieval_endpoint))
|
||||
|
||||
ipv4 = open("/tmp/ipv4").read()
|
||||
|
||||
# Add a "docker" user with password "training"
|
||||
system("sudo useradd -d /home/docker -m -s /bin/bash docker")
|
||||
system("echo docker:training | sudo chpasswd")
|
||||
|
||||
# Fancy prompt courtesy of @soulshake.
|
||||
system("echo \"export PS1='\e[1m\[\033[0;32m\][\h] \e[1m\[\033[0;35m\]\u@{}\[\033[0;33m\] \w\[\033[00m\]\n$ '\" | sudo -u docker tee -a /home/docker/.bashrc".format(ipv4))
|
||||
|
||||
# add docker user to sudoers and allow password authentication
|
||||
system("sudo bash -c \"echo 'docker ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/docker\"")
|
||||
system("sudo sed -i 's/PasswordAuthentication no/PasswordAuthentication yes/' /etc/ssh/sshd_config")
|
||||
|
||||
system("sudo service ssh restart")
|
||||
system("sudo apt-get -q update")
|
||||
system("sudo apt-get -qy install git jq python-pip")
|
||||
|
||||
# increase the size of the conntrack table so we don't blow it up when going crazy with http load testing
|
||||
system("echo 1000000 | sudo tee /proc/sys/net/nf_conntrack_max")
|
||||
|
||||
#######################
|
||||
### DOCKER INSTALLS ###
|
||||
#######################
|
||||
|
||||
# This will install the latest Docker.
|
||||
system("curl --silent https://get.docker.com/ | grep -v '( set -x; sleep 20 )' | sudo sh")
|
||||
|
||||
# Make sure that the daemon listens on 55555 (for orchestration workshop).
|
||||
# To test, run: export DOCKER_HOST=tcp://localhost:55555 ; docker ps
|
||||
# or, run "curl localhost:55555" (it should return 404 not found). If it tells you connection refused, that's a bad sign
|
||||
system("sudo sed -i 's,-H fd://$,-H fd:// -H tcp://0.0.0.0:55555,' /lib/systemd/system/docker.service")
|
||||
system("sudo systemctl daemon-reload")
|
||||
|
||||
# There seems to be a bug in the systemd scripts; so work around it.
|
||||
# See https://github.com/docker/docker/issues/18444
|
||||
# If docker is already running, need to do a restart
|
||||
system("curl --silent localhost:55555 || sudo systemctl restart docker ") # does this work? if not, next line should cover it
|
||||
system("sudo systemctl start docker || true")
|
||||
|
||||
### Install docker-compose
|
||||
system("sudo pip install -U docker-compose=={}".format(COMPOSE_VERSION))
|
||||
|
||||
### Install docker-machine
|
||||
system("sudo curl -sSL -o /usr/local/bin/docker-machine https://github.com/docker/machine/releases/download/v{}/docker-machine-$(uname -s)-$(uname -m)".format(MACHINE_VERSION))
|
||||
system("sudo chmod +x /usr/local/bin/docker-machine*")
|
||||
|
||||
# Wait for Docker to be up. If we don't do this, Docker will not be responsive during the next step.
|
||||
system("while ! sudo -u docker docker version ; do sleep 2; done")
|
||||
|
||||
system("sudo apt-get remove -y --purge dnsmasq-base")
|
||||
system("sudo apt-get -qy install python-setuptools pssh apache2-utils httping htop unzip")
|
||||
|
||||
### Install Swarm
|
||||
system("docker pull swarm:{}".format(SWARM_VERSION))
|
||||
system("docker tag -f swarm:{} swarm".format(SWARM_VERSION))
|
||||
|
||||
|
||||
### BEGIN CLUSTERING ###
|
||||
|
||||
addresses = list(l.strip() for l in sys.stdin)
|
||||
|
||||
assert ipv4 in addresses
|
||||
|
||||
def makenames(addrs):
|
||||
return [ "node%s"%(i+1) for i in range(len(addrs)) ]
|
||||
|
||||
while addresses:
|
||||
cluster = addresses[:CLUSTER_SIZE]
|
||||
addresses = addresses[CLUSTER_SIZE:]
|
||||
if ipv4 not in cluster:
|
||||
continue
|
||||
names = makenames(cluster)
|
||||
for ipaddr, name in zip(cluster, names):
|
||||
system("grep ^{} /etc/hosts || echo {} {} | sudo tee -a /etc/hosts"
|
||||
.format(ipaddr, ipaddr, name))
|
||||
print(cluster)
|
||||
|
||||
mynode = cluster.index(ipv4) + 1
|
||||
system("echo 'node{}' | sudo -u docker tee /tmp/node".format(mynode))
|
||||
system("sudo -u docker mkdir -p /home/docker/.ssh")
|
||||
system("sudo -u docker touch /home/docker/.ssh/authorized_keys")
|
||||
|
||||
if ipv4 == cluster[0]:
|
||||
# If I'm node1 and don't have a private key, generate one (with empty passphrase)
|
||||
system("sudo -u docker [ -f /home/docker/.ssh/id_rsa ] || sudo -u docker ssh-keygen -t rsa -f /home/docker/.ssh/id_rsa -P ''")
|
||||
|
||||
FINISH = time.time()
|
||||
duration = "Initial deployment took {}s".format(str(FINISH - START)[:5])
|
||||
system("echo {}".format(duration))
|
||||
|
||||
EOF
|
||||
|
||||
IPS_FILE=ips.txt
|
||||
if [ ! -s $IPS_FILE ]; then
|
||||
echo "ips.txt not found."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
pssh --timeout 900 --send-input "python /tmp/postprep.py >>/tmp/pp.out 2>>/tmp/pp.err" < $IPS_FILE
|
||||
|
||||
# If /home/docker/.ssh/id_rsa doesn't exist, copy it from node1
|
||||
pssh "sudo -u docker [ -f /home/docker/.ssh/id_rsa ] || ssh -o StrictHostKeyChecking=no node1 sudo -u docker tar -C /home/docker -cvf- .ssh | sudo -u docker tar -C /home/docker -xf-"
|
||||
|
||||
# if 'docker@' doesn't appear in /home/docker/.ssh/authorized_keys, copy it there
|
||||
pssh "grep docker@ /home/docker/.ssh/authorized_keys \
|
||||
|| cat /home/docker/.ssh/id_rsa.pub \
|
||||
| sudo -u docker tee -a /home/docker/.ssh/authorized_keys"
|
||||
|
||||
Executable
+22
@@ -0,0 +1,22 @@
|
||||
# This file can be sourced in order to directly run commands on
|
||||
# a batch of VMs whose IPs are located in ips.txt of the directory in which
|
||||
# the command is run.
|
||||
|
||||
pssh () {
|
||||
HOSTFILE="ips.txt"
|
||||
|
||||
[ -f $HOSTFILE ] || {
|
||||
echo "No hostfile found at $HOSTFILE"
|
||||
return
|
||||
}
|
||||
|
||||
echo "[parallel-ssh] $@"
|
||||
|
||||
parallel-ssh -h $HOSTFILE -l ubuntu \
|
||||
--par 100 \
|
||||
-O LogLevel=ERROR \
|
||||
-O UserKnownHostsFile=/dev/null \
|
||||
-O StrictHostKeyChecking=no \
|
||||
-O ForwardAgent=yes \
|
||||
"$@"
|
||||
}
|
||||
Executable
+489
@@ -0,0 +1,489 @@
|
||||
#!/bin/bash
|
||||
# Don't execute this script directly. Use ../trainer instead.
|
||||
|
||||
set -e # if we encounter an error, abort
|
||||
|
||||
greet() {
|
||||
hello=$(aws iam get-user --query 'User.UserName')
|
||||
echo "Greetings, $hello!"
|
||||
}
|
||||
|
||||
deploy_hq(){
|
||||
TAG=$1
|
||||
need_tag $TAG
|
||||
REMOTE_USER=ubuntu
|
||||
REMOTE_HOST=$(aws_get_instance_ips_by_tag $TAG)
|
||||
echo "Trying to reach $TAG instances..."
|
||||
while ! tag_is_reachable $TAG; do
|
||||
echo -n "."
|
||||
sleep 2
|
||||
done
|
||||
env | grep -i aws > envvars.sh
|
||||
scp \
|
||||
-o "UserKnownHostsFile /dev/null" \
|
||||
-o "StrictHostKeyChecking=no" \
|
||||
scripts/remote-execution.sh \
|
||||
envvars.sh \
|
||||
$REMOTE_USER@$REMOTE_HOST:/tmp/
|
||||
|
||||
ssh -A $REMOTE_USER@$REMOTE_HOST "bash /tmp/remote-execution.sh >>/tmp/pre.out 2>>/tmp/pre.err"
|
||||
ssh -A $REMOTE_USER@$REMOTE_HOST
|
||||
}
|
||||
|
||||
deploy_tag(){
|
||||
TAG=$1
|
||||
need_tag $TAG
|
||||
link_tag $TAG
|
||||
|
||||
|
||||
count=$(cat ips.txt | wc -l)
|
||||
|
||||
# wait until all hosts are reachable before trying to deploy
|
||||
echo "Trying to reach $TAG instances..."
|
||||
while ! tag_is_reachable $TAG; do
|
||||
echo -n "."
|
||||
sleep 2
|
||||
done
|
||||
|
||||
echo "[[ Deploying tag $TAG ]]"
|
||||
source scripts/postprep.rc
|
||||
echo "Finished deploying $TAG."
|
||||
echo "You may want to run one of the following commands:"
|
||||
echo "trainer pull-images $TAG"
|
||||
echo "trainer cards $TAG"
|
||||
}
|
||||
|
||||
link_tag() {
|
||||
TAG=$1
|
||||
need_tag $TAG
|
||||
IPS_FILE=tags/$TAG/ips.txt
|
||||
need_ips_file $IPS_FILE
|
||||
ln -sf $IPS_FILE ips.txt
|
||||
}
|
||||
|
||||
pull_tag(){
|
||||
TAG=$1
|
||||
need_tag $TAG
|
||||
link_tag $TAG
|
||||
cards_file=ips.html
|
||||
if [ ! -s $IPS_FILE ]; then
|
||||
echo "Nonexistent or empty IPs file $IPS_FILE"
|
||||
fi
|
||||
|
||||
# Pre-pull a bunch of images
|
||||
pssh --timeout 900 'for I in \
|
||||
debian:latest \
|
||||
ubuntu:latest \
|
||||
fedora:latest \
|
||||
centos:latest \
|
||||
postgres \
|
||||
redis \
|
||||
training/namer \
|
||||
nathanleclaire/redisonrails; do
|
||||
sudo -u docker docker pull $I
|
||||
done'
|
||||
|
||||
echo "Finished pulling images for $TAG"
|
||||
|
||||
echo "You may now want to run:"
|
||||
echo "trainer cards $TAG"
|
||||
}
|
||||
|
||||
wait_until_tag_is_running() {
|
||||
# Wait up to 10 seconds for instances to be created
|
||||
timeout=10
|
||||
TAG=$1
|
||||
COUNT=$2
|
||||
i=0
|
||||
done_count=0
|
||||
while [[ $done_count -lt $COUNT ]]; do \
|
||||
let "i += 1"
|
||||
echo "Waiting: $done_count/$COUNT instances online"
|
||||
done_count=$(aws ec2 describe-instances \
|
||||
--filters "Name=instance-state-name,Values=running" \
|
||||
"Name=tag:Name,Values=$TAG" \
|
||||
--query "Reservations[*].Instances[*].State.Name" \
|
||||
| tr "\t" "\n" \
|
||||
| wc -l)
|
||||
|
||||
if [[ $i -gt $timeout ]]; then \
|
||||
die " Timed out while waiting for instance creation ($timeout secs)"; \
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
}
|
||||
|
||||
tag_is_reachable() {
|
||||
TAG=$1
|
||||
need_tag $TAG
|
||||
link_tag $TAG
|
||||
if ! pssh -t 5 true 2>&1 >/dev/null; then
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
test_tag(){
|
||||
ips_file=tags/$TAG/ips.txt
|
||||
echo "Using random IP in $ips_file to run tests on $TAG"
|
||||
ip=$(shuf -n 1 $ips_file)
|
||||
test_vm $ip
|
||||
echo "Tests complete. You may want to run one of the following commands:"
|
||||
echo "trainer cards $TAG"
|
||||
}
|
||||
|
||||
test_vm() {
|
||||
ip=$1
|
||||
echo "[[ Testing instance with IP $(tput bold)$ip $(tput sgr0) ]]"
|
||||
user=ubuntu
|
||||
|
||||
for cmd in "hostname" \
|
||||
"whoami" \
|
||||
"hostname -i" \
|
||||
"cat /tmp/node" \
|
||||
"cat /tmp/ipv4" \
|
||||
"cat /etc/hosts" \
|
||||
"hostnamectl status" \
|
||||
"docker version | grep Version -B1" \
|
||||
"docker-compose version" \
|
||||
"docker-machine version" \
|
||||
"docker images" \
|
||||
"docker ps" \
|
||||
"which fig" \
|
||||
"curl --silent localhost:55555" \
|
||||
"sudo ls -la /mnt/ | grep docker" \
|
||||
"env" \
|
||||
"ls -la /home/docker/.ssh"; do
|
||||
echo "=== $cmd ==="
|
||||
echo "$cmd" |
|
||||
ssh -A -q \
|
||||
-o "UserKnownHostsFile /dev/null" \
|
||||
-o "StrictHostKeyChecking=no" \
|
||||
$user@$ip sudo -u docker -i
|
||||
echo
|
||||
done
|
||||
}
|
||||
|
||||
make_key_name(){
|
||||
SHORT_FINGERPRINT=$(ssh-add -l | grep RSA | head -n1 | cut -d " " -f 2 | tr -d : | cut -c 1-8)
|
||||
echo "${SHORT_FINGERPRINT}-${USER}"
|
||||
}
|
||||
|
||||
sync_keys() {
|
||||
# make sure ssh-add -l contains "RSA"
|
||||
ssh-add -l | grep -q RSA ||
|
||||
die "The output of \`ssh-add -l\` doesn't contain 'RSA'. Start the agent, add your keys?"
|
||||
|
||||
AWS_KEY_NAME=$(make_key_name)
|
||||
echo -n "Syncing keys... "
|
||||
if ! aws ec2 describe-key-pairs --key-name "$AWS_KEY_NAME" &> /dev/null; then
|
||||
aws ec2 import-key-pair --key-name $AWS_KEY_NAME \
|
||||
--public-key-material "$(ssh-add -L \
|
||||
| grep -i RSA \
|
||||
| head -n1 \
|
||||
| cut -d " " -f 1-2)" &> /dev/null
|
||||
|
||||
if ! aws ec2 describe-key-pairs --key-name "$AWS_KEY_NAME" &> /dev/null; then
|
||||
die "Somehow, importing the key didn't work. Make sure that 'ssh-add -l | grep RSA | head -n1' returns an RSA key?"
|
||||
else
|
||||
echo "Imported new key $AWS_KEY_NAME."
|
||||
fi
|
||||
else
|
||||
echo "Using existing key $AWS_KEY_NAME."
|
||||
fi
|
||||
}
|
||||
|
||||
suggest_amis() {
|
||||
scripts/find-ubuntu-ami.sh -r $AWS_DEFAULT_REGION -a amd64 -v 15.10 -t hvm:ebs -N
|
||||
}
|
||||
|
||||
get_token() {
|
||||
if [ -z $USER ]; then
|
||||
export USER=anonymous
|
||||
fi
|
||||
date +%Y-%m-%d-%H-%M-$USER
|
||||
}
|
||||
|
||||
get_ami() {
|
||||
# using find-ubuntu-ami script in `trainer-tools/scripts`:
|
||||
#AMI=$(./scripts/find-ubuntu-ami.sh -r $AWS_DEFAULT_REGION -a amd64 -v 15.10 -t hvm:ebs -N | grep -v ^REGION | head -1 | awk '{print $7}')
|
||||
#AMI=$(suggest_amis | grep -v ^REGION | head -1 | awk '{print $7}')
|
||||
case $AWS_DEFAULT_REGION in
|
||||
eu-central-1)
|
||||
AMI=ami-74a4bc18
|
||||
;;
|
||||
eu-west-1)
|
||||
AMI=ami-cda312be
|
||||
;;
|
||||
us-west-2)
|
||||
AMI=ami-495bbd29
|
||||
;;
|
||||
us-east-1)
|
||||
AMI=ami-1711387d
|
||||
;;
|
||||
esac
|
||||
echo $AMI
|
||||
}
|
||||
|
||||
|
||||
make_cards(){
|
||||
# Generate cards for a given tag
|
||||
TAG=$1
|
||||
IPS=$(aws_get_instance_ips_by_tag $TAG)
|
||||
echo "$IPS" > tags/$TAG/ips.txt
|
||||
|
||||
# Remove symlinks to old cards if they exist
|
||||
for f in ips.html ips.pdf; do
|
||||
if [ -e $f ]; then rm $f; fi
|
||||
done
|
||||
|
||||
# This will generate two files in the base dir: ips.pdf and ips.html
|
||||
python scripts/ips-txt-to-html.py settings.yaml
|
||||
|
||||
for f in ips.html ips.pdf; do
|
||||
# Remove old versions of cards if they exist
|
||||
if [ -e tags/$TAG/$f ]; then rm tags/$TAG/$f; fi
|
||||
|
||||
# Move the generated file and replace it with a symlink
|
||||
mv -f $f tags/$TAG/$f && ln -s tags/$TAG/$f $f
|
||||
done
|
||||
|
||||
echo "Cards created. You may want to run:"
|
||||
echo "chromium ips.html"
|
||||
echo "chromium ips.pdf"
|
||||
}
|
||||
|
||||
describe_tag() {
|
||||
# Display instance details and reachability/status information
|
||||
TAG=$1
|
||||
need_tag $TAG
|
||||
echo "============= Tag: $TAG ============="
|
||||
aws_display_instances_by_tag $TAG
|
||||
aws_display_instance_statuses_by_tag $TAG
|
||||
}
|
||||
|
||||
run_cli() {
|
||||
case "$1" in
|
||||
ami)
|
||||
# A wrapper for scripts/find-ubuntu-ami.sh
|
||||
shift
|
||||
scripts/find-ubuntu-ami.sh -r $AWS_DEFAULT_REGION $*
|
||||
echo
|
||||
echo "Protip:"
|
||||
echo "trainer ami -a amd64 -v 15.10 -t hvm:ebs -N | grep -v ^REGION | cut -d\" \" -f15"
|
||||
echo
|
||||
echo "Suggestions:"
|
||||
suggest_amis
|
||||
;;
|
||||
cards)
|
||||
TAG=$2
|
||||
need_tag $TAG
|
||||
make_cards $TAG
|
||||
;;
|
||||
deploy)
|
||||
TAG=$2
|
||||
need_tag $TAG
|
||||
if [[ $TAG == *"-hq"* ]]; then
|
||||
echo "Deploying HQ"
|
||||
deploy_hq $TAG
|
||||
else
|
||||
workshop=$2
|
||||
if [ -z $workshop ]; then
|
||||
echo "Please specify a workshop name (fundamentals or orchestration)"
|
||||
exit 1
|
||||
fi
|
||||
echo "Deploying with $workshop settings"
|
||||
ln -sf settings/$workshop-settings.yaml settings.yaml
|
||||
deploy_tag $TAG
|
||||
|
||||
fi
|
||||
;;
|
||||
ids)
|
||||
TAG=$2
|
||||
need_tag $TAG
|
||||
IDS=$(aws_get_instance_ids_by_tag $TAG)
|
||||
echo "$IDS"
|
||||
|
||||
# Just in case we managed to create instances but weren't able to tag them
|
||||
echo "Lookup by client token $TAG:"
|
||||
IDS=$(aws_get_instance_ids_by_client_token $TAG)
|
||||
echo "$IDS"
|
||||
;;
|
||||
ips)
|
||||
TAG=$2
|
||||
need_tag $TAG
|
||||
mkdir -p tags/$TAG
|
||||
aws_get_instance_ips_by_tag $TAG | tee tags/$TAG/ips.txt
|
||||
link_tag $TAG
|
||||
;;
|
||||
list)
|
||||
# list existing instances in a given batch
|
||||
# to list batches, see "tags" command
|
||||
echo "Using region $AWS_DEFAULT_REGION."
|
||||
TAG=$2
|
||||
need_tag $TAG
|
||||
describe_tag $TAG
|
||||
tag_is_reachable $TAG
|
||||
echo "You may be interested in running one of the following commands:"
|
||||
echo "trainer ips $TAG"
|
||||
echo "trainer deploy $TAG"
|
||||
;;
|
||||
opensg)
|
||||
aws ec2 authorize-security-group-ingress \
|
||||
--group-name default \
|
||||
--protocol icmp \
|
||||
--port -1 \
|
||||
--cidr 0.0.0.0/0
|
||||
|
||||
aws ec2 authorize-security-group-ingress \
|
||||
--group-name default \
|
||||
--protocol udp \
|
||||
--port 0-65535 \
|
||||
--cidr 0.0.0.0/0
|
||||
|
||||
aws ec2 authorize-security-group-ingress \
|
||||
--group-name default \
|
||||
--protocol tcp \
|
||||
--port 0-65535 \
|
||||
--cidr 0.0.0.0/0
|
||||
;;
|
||||
pull-images)
|
||||
TAG=$2
|
||||
need_tag $TAG
|
||||
pull_tag $TAG
|
||||
;;
|
||||
shell)
|
||||
# Get a shell in the container
|
||||
export PS1="trainer@$AWS_DEFAULT_REGION# "
|
||||
exec $SHELL
|
||||
;;
|
||||
start)
|
||||
# Create $2 instances
|
||||
COUNT=$2
|
||||
|
||||
if [ -z "$COUNT" ]; then
|
||||
die "Indicate number of instances to start."
|
||||
fi
|
||||
|
||||
greet # Print our AWS username, to ease the pain of credential-juggling
|
||||
key_name=$(sync_keys) # Upload our SSH keys to AWS if needed, to be added to each VM's authorized_keys
|
||||
AMI=$(get_ami) # Retrieve the AWS image ID
|
||||
TOKEN=$(get_token) # generate a timestamp token for this batch of VMs
|
||||
if [ ! -z $3 ]; then
|
||||
# If an extra arg is present, append it to the tag
|
||||
TOKEN=$TOKEN-$3
|
||||
fi
|
||||
|
||||
echo "-----------------------------------"
|
||||
echo "Starting $COUNT instances:"
|
||||
echo " Region: $AWS_DEFAULT_REGION"
|
||||
echo " Token/tag: $TOKEN"
|
||||
echo " AMI: $AMI"
|
||||
|
||||
AWS_KEY_NAME=$(make_key_name)
|
||||
result=$(aws ec2 run-instances \
|
||||
--key-name $AWS_KEY_NAME \
|
||||
--count $2 \
|
||||
--instance-type c3.large \
|
||||
--client-token $TOKEN \
|
||||
--image-id $AMI)
|
||||
reservation_id=$(echo "$result" | head -1 | awk '{print $2}' )
|
||||
echo " Key name: $AWS_KEY_NAME"
|
||||
echo "Reservation ID: $reservation_id"
|
||||
echo "-----------------------------------"
|
||||
|
||||
# if instance creation succeeded, we should have some IDs
|
||||
IDS=$(aws_get_instance_ids_by_client_token $TOKEN)
|
||||
if [ -z "$IDS" ]; then
|
||||
die "Instance creation failed."
|
||||
fi
|
||||
|
||||
# Tag these new instances with a tag that is the same as the token
|
||||
aws ec2 create-tags --tag Key=Name,Value=$TOKEN --resources $IDS >> /dev/null
|
||||
TAG=$TOKEN
|
||||
unset TOKEN
|
||||
|
||||
wait_until_tag_is_running $TAG $COUNT
|
||||
|
||||
echo "[-------------------------------------------------------------------------------------]"
|
||||
echo " Successfully created $2 instances with tag: $TAG"
|
||||
echo "[-------------------------------------------------------------------------------------]"
|
||||
|
||||
mkdir -p tags/$TAG
|
||||
IPS=$(aws_get_instance_ips_by_tag $TAG)
|
||||
echo "$IPS" > tags/$TAG/ips.txt
|
||||
link_tag $TAG
|
||||
echo "To deploy or kill these instances, run one of the following:"
|
||||
echo "trainer deploy $TAG"
|
||||
echo "trainer list $TAG"
|
||||
;;
|
||||
status)
|
||||
greet && echo
|
||||
|
||||
max_instances=$(aws ec2 describe-account-attributes \
|
||||
--attribute-names max-instances \
|
||||
--query 'AccountAttributes[*][AttributeValues]')
|
||||
echo "Max instances: $max_instances" && echo
|
||||
|
||||
# Print list of AWS EC2 regions, highlighting ours ($AWS_DEFAULT_REGION) in the list
|
||||
# If our $AWS_DEFAULT_REGION is not valid, the error message will be pretty descriptive:
|
||||
# Could not connect to the endpoint URL: "https://ec2.foo.amazonaws.com/"
|
||||
echo "Region:" # $AWS_DEFAULT_REGION."
|
||||
aws ec2 describe-regions | awk '{print $3}' | grep --color=auto $AWS_DEFAULT_REGION -C50
|
||||
|
||||
;;
|
||||
stop)
|
||||
TAG=$2
|
||||
need_tag $TAG
|
||||
aws_kill_instances_by_tag $TAG
|
||||
;;
|
||||
tag)
|
||||
# add a tag to a batch of VMs
|
||||
TAG=$2
|
||||
NEW_TAG_KEY=$3
|
||||
NEW_TAG_VALUE=$4
|
||||
need_tag $TAG
|
||||
need_tag $NEW_TAG_KEY
|
||||
need_tag $NEW_TAG_VALUE
|
||||
;;
|
||||
test)
|
||||
TAG=$2
|
||||
need_tag $TAG
|
||||
test_tag $TAG
|
||||
;;
|
||||
*)
|
||||
echo && echo "trainer COMMAND [n-instances|tag]
|
||||
|
||||
Core commands:
|
||||
start n Start n instances
|
||||
list [TAG] If a tag is provided, list its VMs. Otherwise, list tags.
|
||||
deploy TAG Deploy all instances with a given tag
|
||||
pull-images TAG Pre-pull docker images. Run only after deploying.
|
||||
stop TAG Stop and delete instances tagged TAG
|
||||
|
||||
Extras:
|
||||
ips TAG List all IPs of instances with a given tag (updates ips.txt)
|
||||
ids TAG/TOKEN List all instance IDs with a given tag
|
||||
shell Get a shell in the trainer container
|
||||
status TAG Print information about this tag and its VMs
|
||||
tags List all tags (per-region)
|
||||
|
||||
Beta:
|
||||
ami Look up Amazon Machine Images
|
||||
cards Generate cards (see settings.txt)
|
||||
opensg Modify AWS security groups
|
||||
"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
(
|
||||
cd $SCRIPT_DIR
|
||||
source scripts/cli.sh
|
||||
source scripts/aws.sh
|
||||
source scripts/rc
|
||||
source scripts/colors.sh
|
||||
mkdir -p tags
|
||||
# TODO: unset empty envvars
|
||||
run_cli "$@"
|
||||
)
|
||||
Symlink
+1
@@ -0,0 +1 @@
|
||||
./settings/fundamentals-settings.yaml
|
||||
@@ -0,0 +1,30 @@
|
||||
# This file is passed by trainer-cli to scripts/ips-txt-to-html.py
|
||||
|
||||
workshop_name: Docker fundamentals
|
||||
workshop_short_name: Docker # appears on VM connection cards
|
||||
repo: https://github.com/docker/docker-fundamentals
|
||||
|
||||
instance_login: docker
|
||||
instance_password: training
|
||||
|
||||
clustersize: 1 # Number of VMs per cluster
|
||||
pagesize: 15 # Number of cards to print per page
|
||||
|
||||
#background_image: https://myapps.developer.ubuntu.com/site_media/appmedia/2014/12/swarm.png
|
||||
background_image: http://www.yellosoft.us/public/images/docker.png
|
||||
#background_image: ../media/swarm.png
|
||||
|
||||
# To be printed on the cards:
|
||||
blurb: >
|
||||
Here is the connection information to your very own
|
||||
{cluster_or_machine} for this {workshop_name} workshop. You can connect
|
||||
to {this_or_each} VM with any SSH client.
|
||||
|
||||
Your {machine_is_or_machines_are}:
|
||||
|
||||
# {url} will be replaced by the script
|
||||
footer: >
|
||||
<p>For slides, chat and other useful links, see: </p>
|
||||
<center>{url}</center>
|
||||
|
||||
url: http://container.training/
|
||||
@@ -0,0 +1,30 @@
|
||||
# This file is passed by trainer-cli to scripts/ips-txt-to-html.py
|
||||
|
||||
workshop_name: Advanced Docker Orchestration
|
||||
workshop_short_name: orchestration
|
||||
repo: https://github.com/jpetazzo/orchestration-workshop
|
||||
|
||||
instance_login: docker
|
||||
instance_password: training
|
||||
|
||||
clustersize: 5 # Number of VMs per cluster
|
||||
pagesize: 15 # Number of cards to print per page
|
||||
|
||||
#background_image: https://myapps.developer.ubuntu.com/site_media/appmedia/2014/12/swarm.png
|
||||
background_image: http://www.yellosoft.us/public/images/docker.png
|
||||
#background_image: ../media/swarm.png
|
||||
|
||||
# To be printed on the cards:
|
||||
blurb: >
|
||||
Here is the connection information to your very own
|
||||
{cluster_or_machine} for this {workshop_name} workshop. You can connect
|
||||
to {this_or_each} VM with any SSH client.
|
||||
|
||||
Your {machine_is_or_machines_are}:
|
||||
|
||||
# {url} will be replaced by the script
|
||||
footer: >
|
||||
<p>For slides, chat and other useful links, see: </p>
|
||||
<center>{url}</center>
|
||||
|
||||
url: http://container.training/
|
||||
Executable
+137
@@ -0,0 +1,137 @@
|
||||
#!/bin/bash
|
||||
|
||||
TRAINER_IMAGE="soulshake/prepare-vms"
|
||||
|
||||
PIP_DEPENDENCIES="
|
||||
awscli
|
||||
"
|
||||
|
||||
# dpkg -l/pip/type-friendly
|
||||
DEPENDENCIES="
|
||||
aws
|
||||
awscli
|
||||
ssh
|
||||
curl
|
||||
jq
|
||||
bsdmainutils
|
||||
pssh
|
||||
pip
|
||||
python-pip
|
||||
wkhtmltopdf
|
||||
JP_PLACEHOLDER
|
||||
"
|
||||
|
||||
OPTIONAL_DEPENDENCIES="
|
||||
man
|
||||
"
|
||||
|
||||
ENVVARS="
|
||||
AWS_ACCESS_KEY_ID
|
||||
AWS_SECRET_ACCESS_KEY
|
||||
AWS_DEFAULT_REGION
|
||||
AWS_DEFAULT_PROFILE
|
||||
SSH_AGENT_PID
|
||||
SSH_AUTH_SOCK
|
||||
SSH_AUTH_DIRNAME
|
||||
"
|
||||
|
||||
envvars_ok() {
|
||||
# This script expects plain text output from the AWS CLI in some places
|
||||
export AWS_DEFAULT_OUTPUT=text
|
||||
|
||||
for envvar in $ENVVARS; do
|
||||
if [ -z "${!envvar}" ]; then
|
||||
echo "Please export $envvar environment variable. Unsetting."
|
||||
unset $envvar
|
||||
#return 1
|
||||
fi
|
||||
done
|
||||
return 0
|
||||
}
|
||||
|
||||
package_is_installed() {
|
||||
# Try "type", "dpkg -l", and "pip freeze" to see if a package is installed
|
||||
dependency=$1
|
||||
type $dependency >/dev/null 2>&1 \
|
||||
|| dpkg -l $dependency >/dev/null 2>&1 \
|
||||
|| pip freeze | grep $dependency >/dev/null 2>&1 || return 1
|
||||
}
|
||||
|
||||
show_missing_dependencies() {
|
||||
for dependency in $DEPENDENCIES; do
|
||||
package_is_installed $dependency \
|
||||
&& echo " $dependency: [OK]" \
|
||||
|| { echo >&2 "$(tput setaf 1) $dependency: [NOK] $(tput sgr0)" ;}
|
||||
done
|
||||
|
||||
echo "Recommended:"
|
||||
for dependency in $OPTIONAL_DEPENDENCIES; do
|
||||
package_is_installed $dependency \
|
||||
&& echo " $dependency: [OK]" \
|
||||
|| { echo >&2 "$(tput setaf 1) $dependency: [NOK] $(tput sgr0)" ;}
|
||||
done
|
||||
}
|
||||
|
||||
run_natively() {
|
||||
scripts/trainer-cli "$@"
|
||||
}
|
||||
|
||||
run_containerized() {
|
||||
if [ -z $SSH_AUTH_SOCK ]; then
|
||||
echo -n "SSH_AUTH_SOCK envvar not set, so its parent directory can't be "
|
||||
echo "mounted as a volume in a container."
|
||||
echo "Try running the command below and trying again:"
|
||||
echo "eval \$(ssh-agent) && ssh-agent && ssh-add"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
#/tmp/ssh-uN1oMimerRo0/agent.1411
|
||||
(
|
||||
cd $SCRIPT_DIR
|
||||
export SSH_AUTH_DIRNAME=$(dirname $SSH_AUTH_SOCK)
|
||||
docker-compose -f docker-compose.yml run prepare-vms "$@"
|
||||
|
||||
)
|
||||
}
|
||||
|
||||
dependencies_fulfilled() {
|
||||
for dependency in $DEPENDENCIES ; do
|
||||
package_is_installed $dependency || return 1
|
||||
done
|
||||
|
||||
echo "All dependencies fulfilled. Running locally."
|
||||
}
|
||||
|
||||
trainer_image_exists_locally() {
|
||||
ret=$(docker inspect $TRAINER_IMAGE >/dev/null 2>&1)
|
||||
return $?
|
||||
}
|
||||
|
||||
# Get the script's real directory, whether we're being called directly or via a symlink
|
||||
if [ -L $0 ]; then
|
||||
export SCRIPT_DIR=$(dirname $(readlink $0))
|
||||
else
|
||||
export SCRIPT_DIR=$(dirname $(readlink -f $0))
|
||||
fi
|
||||
|
||||
export SSH_AUTH_DIRNAME=$(dirname $SSH_AUTH_SOCK)
|
||||
|
||||
if ! envvars_ok; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if dependencies_fulfilled; then
|
||||
(
|
||||
cd $SCRIPT_DIR
|
||||
run_natively "$@"
|
||||
)
|
||||
elif trainer_image_exists_locally; then
|
||||
run_containerized "$@"
|
||||
else
|
||||
echo "Some dependencies are missing, and docker image $TRAINER_IMAGE doesn't exist locally."
|
||||
echo "Please do one of the following: "
|
||||
echo "- run \`docker build -t soulshake/prepare-vms .\`"
|
||||
echo "- run \`docker pull soulshake/prepare-vms\`"
|
||||
echo "- install all dependencies: "
|
||||
show_missing_dependencies
|
||||
fi
|
||||
Reference in New Issue
Block a user