From 5c600a05d040561908715d57c6d1586f84b86b1c Mon Sep 17 00:00:00 2001 From: AJ Bowen Date: Sat, 28 May 2016 18:37:29 -0700 Subject: [PATCH 1/3] Replace 'user' with 'root' in images. Squash layers in Dockerfile. Update README. Clean up docker-compose.yaml. --- prepare-vms/Dockerfile | 50 ++++++------------- prepare-vms/README.md | 87 ++++++++++++++++----------------- prepare-vms/docker-compose.yml | 33 ++++--------- prepare-vms/scripts/trainer-cli | 26 +++++----- prepare-vms/trainer | 7 ++- 5 files changed, 83 insertions(+), 120 deletions(-) diff --git a/prepare-vms/Dockerfile b/prepare-vms/Dockerfile index 4b650eb0..949cc0e5 100644 --- a/prepare-vms/Dockerfile +++ b/prepare-vms/Dockerfile @@ -1,52 +1,32 @@ FROM debian:jessie MAINTAINER AJ Bowen -RUN apt-get update -RUN apt-get install -y ca-certificates -RUN apt-get install -y groff -RUN apt-get install -y less -RUN apt-get install -y python python-pip -RUN apt-get install -y python-docutils -RUN apt-get install -y sudo -RUN apt-get install -y \ +RUN apt-get update && apt-get install -y \ + wkhtmltopdf \ bsdmainutils \ + ca-certificates \ curl \ + groff \ jq \ less \ man \ pssh \ - ssh + python \ + python-pip \ + python-docutils \ + ssh \ + --no-install-recommends \ + && rm -rf /var/lib/apt/lists/* -RUN pip install awscli RUN pip install \ + awscli \ pdfkit \ PyYAML \ termcolor -RUN apt-get install -y wkhtmltopdf - -ENV HOME /home/user - -RUN useradd --create-home --home-dir $HOME user \ - && mkdir -p $HOME/.config/gandi \ - && chown -R user:user $HOME - -RUN echo "user ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers - -# Replace 1000 with your user / group id -#RUN export uid=1000 gid=1000 && \ -# mkdir -p /home/user && \ -# mkdir -p /etc/sudoers.d && \ -# echo "user:x:${uid}:${gid}:user,,,:/home/user:/bin/bash" >> /etc/passwd && \ -# echo "user:x:${uid}:" >> /etc/group && \ -# echo "user ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/user && \ -# chmod 0440 /etc/sudoers.d/user && \ -# chown ${uid}:${gid} -R /home/user - WORKDIR $HOME -RUN echo "alias ll='ls -lahF'" >> /home/user/.bashrc -RUN echo "export PATH=$PATH:/home/user/bin" >> /home/user/.bashrc -RUN mkdir -p /home/user/bin -RUN ln -s /home/user/prepare-vms/scripts/trainer-cli /home/user/bin/trainer-cli -USER user +RUN echo "alias ll='ls -lahF'" >> /root/.bashrc +RUN echo "export PATH=$PATH:/root/bin" >> /root/.bashrc +RUN mkdir -p /root/bin +RUN ln -s /root/prepare-vms/scripts/trainer-cli /root/bin/trainer-cli diff --git a/prepare-vms/README.md b/prepare-vms/README.md index 554c7d96..1b1bc2e8 100644 --- a/prepare-vms/README.md +++ b/prepare-vms/README.md @@ -1,27 +1,18 @@ # Trainer tools to prepare VMs for Docker workshops -There are several options for using these tools: +## 1. Prerequisites -### Clone the repo +* [Docker](https://docs.docker.com/engine/installation/) +* [Docker Compose](https://docs.docker.com/compose/install/) - $ git clone https://github.com/soulshake/prepare-vms.git - $ cd prepare-vms +## 2. Clone the repo + + $ git clone https://github.com/jpetazzo/orchestration-workshop.git + $ cd orchestration-workshop/prepare-vms $ docker-compose build - $ mkdir $HOME/bin && ln -s `pwd`/trainer $HOME/bin/trainer + $ ./trainer # See "Summary of commands" section below -### Via the image - - $ docker pull soulshake/prepare-vms - -### Submodule - -This repo can be added as a submodule in the repo of the Docker workshop: - - $ git submodule add https://github.com/soulshake/prepare-vms.git - -## Setup - -### Export needed envvars +## 3. Preparing the environment Required environment variables: @@ -29,46 +20,54 @@ Required environment variables: * `AWS_SECRET_ACCESS_KEY` * `AWS_DEFAULT_REGION` - - -### Update settings.yaml +### 4. Update settings.yaml If you have more than one workshop: - $ cp settings.yaml settings/YOUR_WORKSHOP_NAME-settings.yaml - $ ln -s settings/YOUR_WORKSHOP_NAME-settings.yaml `pwd`/settings.yaml + $ cp settings/settings.yaml settings/YOUR_WORKSHOP_NAME-settings.yaml + $ ln -s settings/YOUR_WORKSHOP_NAME-settings.yaml settings.yaml -Update the `settings.yaml` as needed. This is the file that will be used to generate cards. +Update the symlinked `settings.yaml` as needed. This is the file that will be used to generate cards. ## Usage -### Summary - -Summary of steps to launch a batch of instances for a workshop: - -* Export the environment variables needed by the AWS CLI (see **Requirements** below) -* `trainer start NUMBER_OF_VMS` to create AWS instances -* `trainer deploy TAG` to run `scripts/postprep.rc` via parallel-ssh -* `trainer pull-images TAG` to pre-pull a bunch of Docker images -* `trainer test TAG` -* `trainer cards TAG` to generate a PDF and an HTML file you can print +### Summary of commands The `trainer` script can be executed directly. +Summary of steps to launch a batch of instances for a workshop: + +* Export the environment variables needed by the AWS CLI (see **2. Preparing the environment** above) +* `./trainer start N` (where `N` is the number of AWS instances to create) +* `./trainer list` to view the list of tags +* `./trainer list TAG` to view the instances with a given `TAG` +* `./trainer deploy TAG` to run `scripts/postprep.rc` via parallel-ssh +* `./trainer pull-images TAG` to pre-pull a bunch of Docker images to the instances +* `./trainer test TAG` +* `./trainer cards TAG` to generate a PDF and an HTML file you can print and cut to hand out cards with connection information to attendees + +`./trainer` will run locally if all its dependencies are fulfilled; otherwise it will run in a Docker container. + It will check for the necessary environment variables. Then, if all its dependencies are installed locally, it will execute `trainer-cli`. If not, it will look for a local Docker image -tagged `soulshake/trainer-tools`. If found, it will run in a container. If not found, -the user will be prompted to either install the missing dependencies or download -the Docker image. +tagged `preparevms_prepare-vms` (created automatically when you run `docker-compose build`). +If found, it will run in a container. If not found, the user will be prompted to +either install the missing dependencies or run `docker-compose build`. ## Detailed usage ### Start some VMs - $ trainer start 10 + $ ./trainer start 10 A few things will happen: +* Your local SSH key will be synced +* AWS instances will be created and tagged +* A directory will be created + +Details below. + #### Sync of SSH keys When the `start` command is run, your local RSA SSH public key will be added to your AWS EC2 keychain. @@ -93,31 +92,31 @@ If you create new VMs, the symlinked file will be overwritten. Instances can be deployed manually using the `deploy` command: - $ trainer deploy TAG + $ ./trainer deploy TAG The `postprep.rc` file will be copied via parallel-ssh to all of the VMs and executed. ### Pre-pull images - $ trainer pull-images TAG + $ ./trainer pull-images TAG ### Generate cards - $ trainer cards TAG + $ ./trainer cards TAG ### List tags - $ trainer list + $ ./trainer list ### List VMs - $ trainer list TAG + $ ./trainer list TAG This will print a human-friendly list containing some information about each instance. ### Stop and destroy VMs - $ trainer stop TAG + $ ./trainer stop TAG ## ToDo diff --git a/prepare-vms/docker-compose.yml b/prepare-vms/docker-compose.yml index 3b590d6b..d91ea3bd 100644 --- a/prepare-vms/docker-compose.yml +++ b/prepare-vms/docker-compose.yml @@ -1,38 +1,25 @@ prepare-vms: build: . container_name: prepare-vms - working_dir: /home/user/prepare-vms + working_dir: /root/prepare-vms volumes: - - $HOME/.aws/:$HOME.aws/ - - $HOME/.ssh/:/home/user/.ssh/ + - $HOME/.aws/:/root/.aws/ - /etc/localtime:/etc/localtime:ro - #- /home:/home - /tmp/.X11-unix:/tmp/.X11-unix - $SSH_AUTH_DIRNAME:$SSH_AUTH_DIRNAME - - $SCRIPT_DIR/:/home/user/prepare-vms/ - #- $SCRIPT_DIR/:$HOME/prepare-vms/ - #- $HOME/trainer-tools/ - #- $(dirname $SSH_AUTH_SOCK):$(dirname $SSH_AUTH_SOCK) - #- /etc/passwd:/etc/passwd:ro - #- /etc/group:/etc/group:ro - #- /run/user:/run/user + #- $SCRIPT_DIR/:/root/prepare-vms/ + - $PWD/:/root/prepare-vms/ environment: - HOME: /home/user - #SCRIPT_DIR: /home/aj/git/prepare-vms - SCRIPT_DIR: /home/user/prepare-vms - HOME: /home/user + SCRIPT_DIR: /root/prepare-vms + DISPLAY: ${DISPLAY} SSH_AUTH_SOCK: ${SSH_AUTH_SOCK} SSH_AGENT_PID: ${SSH_AGENT_PID} AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} - DISPLAY: ${DISPLAY} - USER: ${USER} AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION} - AWS_DEFAULT_OUTPUT: + AWS_DEFAULT_OUTPUT: json AWS_INSTANCE_TYPE: ${AWS_INSTANCE_TYPE} AWS_VPC_ID: ${AWS_VPC_ID} - entrypoint: /home/user/prepare-vms/scripts/trainer-cli - #entrypoint: trainer - - #AWS_DEFAULT_PROFILE: ${AWS_DEFAULT_PROFILE} - #command: /home/user/prepare-vms/trainer-cli + USER: ${USER} + entrypoint: /root/prepare-vms/scripts/trainer-cli + #entrypoint: bash diff --git a/prepare-vms/scripts/trainer-cli b/prepare-vms/scripts/trainer-cli index e06c204d..f9c953e3 100755 --- a/prepare-vms/scripts/trainer-cli +++ b/prepare-vms/scripts/trainer-cli @@ -7,7 +7,7 @@ export AWS_DEFAULT_OUTPUT=text greet() { hello=$(aws iam get-user --query 'User.UserName') - echo "Greetings, $hello!" + echo "Greetings, $hello/${USER}!" } deploy_hq(){ @@ -53,8 +53,8 @@ deploy_tag(){ source scripts/postprep.rc echo "Finished deploying $TAG." echo "You may want to run one of the following commands:" - echo "trainer pull-images $TAG" - echo "trainer cards $TAG" + echo "./trainer pull-images $TAG" + echo "./trainer cards $TAG settings.yaml" } link_tag() { @@ -69,7 +69,6 @@ pull_tag(){ TAG=$1 need_tag $TAG link_tag $TAG - cards_file=ips.html if [ ! -s $IPS_FILE ]; then echo "Nonexistent or empty IPs file $IPS_FILE" fi @@ -90,7 +89,7 @@ pull_tag(){ echo "Finished pulling images for $TAG" echo "You may now want to run:" - echo "trainer cards $TAG" + echo "./trainer cards $TAG settings.yaml" } wait_until_tag_is_running() { @@ -129,7 +128,7 @@ test_tag(){ ip=$(shuf -n 1 $ips_file) test_vm $ip echo "Tests complete. You may want to run one of the following commands:" - echo "trainer cards $TAG" + echo "./trainer cards $TAG settings.yaml" } test_vm() { @@ -149,7 +148,6 @@ test_vm() { "docker-machine version" \ "docker images" \ "docker ps" \ - "which fig" \ "curl --silent localhost:55555" \ "sudo ls -la /mnt/ | grep docker" \ "env" \ @@ -276,7 +274,7 @@ run_cli() { scripts/find-ubuntu-ami.sh -r $AWS_DEFAULT_REGION $* echo echo "Protip:" - echo "trainer ami -a amd64 -v 16.04 -t hvm:ebs -N | grep -v ^REGION | cut -d\" \" -f15" + echo "./trainer ami -a amd64 -v 16.04 -t hvm:ebs -N | grep -v ^REGION | cut -d\" \" -f15" echo echo "Suggestions:" suggest_amis @@ -333,8 +331,8 @@ run_cli() { describe_tag $TAG tag_is_reachable $TAG echo "You may be interested in running one of the following commands:" - echo "trainer ips $TAG" - echo "trainer deploy $TAG " + echo "./trainer ips $TAG" + echo "./trainer deploy $TAG settings.yaml" ;; opensg) aws ec2 authorize-security-group-ingress \ @@ -427,8 +425,8 @@ run_cli() { echo "$IPS" > tags/$TAG/ips.txt link_tag $TAG echo "To deploy or kill these instances, run one of the following:" - echo "trainer deploy $TAG " - echo "trainer list $TAG" + echo "./trainer deploy $TAG settings.yaml" + echo "./trainer list $TAG" ;; status) greet && echo @@ -466,7 +464,7 @@ run_cli() { ;; *) echo " -trainer COMMAND [n-instances|tag] +./trainer COMMAND [n-instances|tag|settings_file] Core commands: start n Start n instances @@ -485,7 +483,7 @@ Extras: Beta: ami Look up Amazon Machine Images - cards Generate cards + cards FILE Generate cards opensg Modify AWS security groups " ;; diff --git a/prepare-vms/trainer b/prepare-vms/trainer index 3d0831c8..cb501223 100755 --- a/prepare-vms/trainer +++ b/prepare-vms/trainer @@ -1,6 +1,6 @@ #!/bin/bash -TRAINER_IMAGE="soulshake/prepare-vms" +TRAINER_IMAGE="preparevms_prepare-vms" DEPENDENCIES=" aws @@ -71,11 +71,10 @@ if check_dependencies; then elif check_image; then check_ssh_auth_sock export SSH_AUTH_DIRNAME=$(dirname $SSH_AUTH_SOCK) - docker-compose -f docker-compose.yml run prepare-vms "$@" + docker-compose run prepare-vms "$@" else echo "Some dependencies are missing, and docker image $TRAINER_IMAGE doesn't exist locally." echo "Please do one of the following: " - echo "- run \`docker build -t soulshake/prepare-vms .\`" - echo "- run \`docker pull soulshake/prepare-vms\`" + echo "- run \`docker-compose build\`" echo "- install missing dependencies" fi From a2b77ff63b44eea160e94a27eb1d28f6eb05c08a Mon Sep 17 00:00:00 2001 From: AJ Bowen Date: Sat, 28 May 2016 18:40:07 -0700 Subject: [PATCH 2/3] remove two more comments from docker-compose.yaml --- prepare-vms/docker-compose.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/prepare-vms/docker-compose.yml b/prepare-vms/docker-compose.yml index d91ea3bd..e439cc2f 100644 --- a/prepare-vms/docker-compose.yml +++ b/prepare-vms/docker-compose.yml @@ -7,7 +7,6 @@ prepare-vms: - /etc/localtime:/etc/localtime:ro - /tmp/.X11-unix:/tmp/.X11-unix - $SSH_AUTH_DIRNAME:$SSH_AUTH_DIRNAME - #- $SCRIPT_DIR/:/root/prepare-vms/ - $PWD/:/root/prepare-vms/ environment: SCRIPT_DIR: /root/prepare-vms @@ -22,4 +21,3 @@ prepare-vms: AWS_VPC_ID: ${AWS_VPC_ID} USER: ${USER} entrypoint: /root/prepare-vms/scripts/trainer-cli - #entrypoint: bash From a747058a72f1cddb9ee203020f8b8e5146c66152 Mon Sep 17 00:00:00 2001 From: AJ Bowen Date: Sat, 28 May 2016 19:46:38 -0700 Subject: [PATCH 3/3] Replace settings.yaml with in the documentation, as per @jpetazzo request; add entrypoint to Dockerfile; remove symlink and path manipulation from Dockerfile. --- prepare-vms/Dockerfile | 4 +--- prepare-vms/README.md | 13 ++++++------- prepare-vms/scripts/trainer-cli | 12 ++++++------ 3 files changed, 13 insertions(+), 16 deletions(-) diff --git a/prepare-vms/Dockerfile b/prepare-vms/Dockerfile index 949cc0e5..21c0475f 100644 --- a/prepare-vms/Dockerfile +++ b/prepare-vms/Dockerfile @@ -26,7 +26,5 @@ RUN pip install \ WORKDIR $HOME RUN echo "alias ll='ls -lahF'" >> /root/.bashrc -RUN echo "export PATH=$PATH:/root/bin" >> /root/.bashrc -RUN mkdir -p /root/bin -RUN ln -s /root/prepare-vms/scripts/trainer-cli /root/bin/trainer-cli +ENTRYPOINT ["/root/prepare-vms/scripts/trainer-cli"] diff --git a/prepare-vms/README.md b/prepare-vms/README.md index 1b1bc2e8..c0bb41c9 100644 --- a/prepare-vms/README.md +++ b/prepare-vms/README.md @@ -24,10 +24,9 @@ Required environment variables: If you have more than one workshop: - $ cp settings/settings.yaml settings/YOUR_WORKSHOP_NAME-settings.yaml - $ ln -s settings/YOUR_WORKSHOP_NAME-settings.yaml settings.yaml + $ cp settings/default.yaml settings/YOUR_WORKSHOP_NAME-settings.yaml -Update the symlinked `settings.yaml` as needed. This is the file that will be used to generate cards. +Then pass `settings/YOUR_WORKSHOP_NAME-settings.yaml` as an argument to `deploy`, `cards`, etc. ## Usage @@ -41,10 +40,10 @@ Summary of steps to launch a batch of instances for a workshop: * `./trainer start N` (where `N` is the number of AWS instances to create) * `./trainer list` to view the list of tags * `./trainer list TAG` to view the instances with a given `TAG` -* `./trainer deploy TAG` to run `scripts/postprep.rc` via parallel-ssh +* `./trainer deploy TAG settings/somefile.yaml` to run `scripts/postprep.rc` via parallel-ssh * `./trainer pull-images TAG` to pre-pull a bunch of Docker images to the instances * `./trainer test TAG` -* `./trainer cards TAG` to generate a PDF and an HTML file you can print and cut to hand out cards with connection information to attendees +* `./trainer cards TAG settings/somefile.yaml` to generate a PDF and an HTML file you can print and cut to hand out cards with connection information to attendees `./trainer` will run locally if all its dependencies are fulfilled; otherwise it will run in a Docker container. @@ -92,7 +91,7 @@ If you create new VMs, the symlinked file will be overwritten. Instances can be deployed manually using the `deploy` command: - $ ./trainer deploy TAG + $ ./trainer deploy TAG settings/somefile.yaml The `postprep.rc` file will be copied via parallel-ssh to all of the VMs and executed. @@ -102,7 +101,7 @@ The `postprep.rc` file will be copied via parallel-ssh to all of the VMs and exe ### Generate cards - $ ./trainer cards TAG + $ ./trainer cards TAG settings/somefile.yaml ### List tags diff --git a/prepare-vms/scripts/trainer-cli b/prepare-vms/scripts/trainer-cli index f9c953e3..1583f49f 100755 --- a/prepare-vms/scripts/trainer-cli +++ b/prepare-vms/scripts/trainer-cli @@ -54,7 +54,7 @@ deploy_tag(){ echo "Finished deploying $TAG." echo "You may want to run one of the following commands:" echo "./trainer pull-images $TAG" - echo "./trainer cards $TAG settings.yaml" + echo "./trainer cards $TAG " } link_tag() { @@ -89,7 +89,7 @@ pull_tag(){ echo "Finished pulling images for $TAG" echo "You may now want to run:" - echo "./trainer cards $TAG settings.yaml" + echo "./trainer cards $TAG " } wait_until_tag_is_running() { @@ -128,7 +128,7 @@ test_tag(){ ip=$(shuf -n 1 $ips_file) test_vm $ip echo "Tests complete. You may want to run one of the following commands:" - echo "./trainer cards $TAG settings.yaml" + echo "./trainer cards $TAG " } test_vm() { @@ -332,7 +332,7 @@ run_cli() { tag_is_reachable $TAG echo "You may be interested in running one of the following commands:" echo "./trainer ips $TAG" - echo "./trainer deploy $TAG settings.yaml" + echo "./trainer deploy $TAG " ;; opensg) aws ec2 authorize-security-group-ingress \ @@ -425,7 +425,7 @@ run_cli() { echo "$IPS" > tags/$TAG/ips.txt link_tag $TAG echo "To deploy or kill these instances, run one of the following:" - echo "./trainer deploy $TAG settings.yaml" + echo "./trainer deploy $TAG " echo "./trainer list $TAG" ;; status) @@ -464,7 +464,7 @@ run_cli() { ;; *) echo " -./trainer COMMAND [n-instances|tag|settings_file] +./trainer [n-instances|tag] [settings/file.yaml] Core commands: start n Start n instances