Merge branch 'the-big-2017-refactor' of github.com:jpetazzo/orchestration-workshop into the-big-2017-refactor

2026-07-18 20:39:17 +00:00 · 2017-10-29 19:52:32 -07:00
parent 4747860226 0508d24046
commit 030d100d70
7 changed files with 184 additions and 158 deletions
--- a/prepare-vms/lib/aws.sh
+++ b/prepare-vms/lib/aws.sh
@@ -1,9 +1,9 @@
-aws_display_tags(){
+aws_display_tags() {
    # Print all "Name" tags in our region with their instance count
    echo "[#] [Status] [Token] [Tag]" \
        | awk '{ printf "%-7s %-12s %-25s %-25s\n", $1, $2, $3, $4}'
    aws ec2 describe-instances \
-            --query "Reservations[*].Instances[*].[State.Name,ClientToken,Tags[0].Value]" \
+        --query "Reservations[*].Instances[*].[State.Name,ClientToken,Tags[0].Value]" \
        | tr -d "\r" \
        | uniq -c \
        | sort -k 3 \
@@ -12,17 +12,17 @@ aws_display_tags(){

 aws_get_tokens() {
    aws ec2 describe-instances --output text \
-            --query 'Reservations[*].Instances[*].[ClientToken]' \
+        --query 'Reservations[*].Instances[*].[ClientToken]' \
        | sort -u
 }

 aws_display_instance_statuses_by_tag() {
    TAG=$1
    need_tag $TAG
-    
+
    IDS=$(aws ec2 describe-instances \
        --filters "Name=tag:Name,Values=$TAG" \
-        --query "Reservations[*].Instances[*].InstanceId" | tr '\t' ' ' )
+        --query "Reservations[*].Instances[*].InstanceId" | tr '\t' ' ')

    aws ec2 describe-instance-status \
        --instance-ids $IDS \
@@ -34,20 +34,20 @@ aws_display_instances_by_tag() {
    TAG=$1
    need_tag $TAG
    result=$(aws ec2 describe-instances --output table \
-                    --filter "Name=tag:Name,Values=$TAG" \
-                    --query "Reservations[*].Instances[*].[ \
+        --filter "Name=tag:Name,Values=$TAG" \
+        --query "Reservations[*].Instances[*].[ \
                        InstanceId, \
                        State.Name, \
                        Tags[0].Value, \
                        PublicIpAddress, \
                        InstanceType \
                        ]"
-            )
-        if [[ -z $result ]]; then
-            die "No instances found with tag $TAG in region $AWS_DEFAULT_REGION."
-        else
-            echo "$result"
-        fi
+    )
+    if [[ -z $result ]]; then
+        die "No instances found with tag $TAG in region $AWS_DEFAULT_REGION."
+    else
+        echo "$result"
+    fi
 }

 aws_get_instance_ids_by_filter() {
@@ -57,7 +57,6 @@ aws_get_instance_ids_by_filter() {
        --output text | tr "\t" "\n" | tr -d "\r"
 }

-
 aws_get_instance_ids_by_client_token() {
    TOKEN=$1
    need_tag $TOKEN
@@ -76,8 +75,8 @@ aws_get_instance_ips_by_tag() {
    aws ec2 describe-instances --filter "Name=tag:Name,Values=$TAG" \
        --output text \
        --query "Reservations[*].Instances[*].PublicIpAddress" \
-            | tr "\t" "\n" \
-            | sort -n -t . -k 1,1 -k 2,2 -k 3,3 -k 4,4  # sort IPs
+        | tr "\t" "\n" \
+        | sort -n -t . -k 1,1 -k 2,2 -k 3,3 -k 4,4 # sort IPs
 }

 aws_kill_instances_by_tag() {
--- a/prepare-vms/lib/cli.sh
+++ b/prepare-vms/lib/cli.sh
@@ -31,18 +31,18 @@ sep() {
    if [ -z "$COLUMNS" ]; then
        COLUMNS=80
    fi
-    SEP=$(yes = | tr -d "\n" | head -c $[$COLUMNS - 1])
+    SEP=$(yes = | tr -d "\n" | head -c $(($COLUMNS - 1)))
    if [ -z "$1" ]; then
        >/dev/stderr echo $SEP
    else
        MSGLEN=$(echo "$1" | wc -c)
-        if [ $[ $MSGLEN +4 ] -gt $COLUMNS ]; then
+        if [ $(($MSGLEN + 4)) -gt $COLUMNS ]; then
            >/dev/stderr echo "$SEP"
            >/dev/stderr echo "$1"
            >/dev/stderr echo "$SEP"
        else
-            LEFTLEN=$[ ($COLUMNS - $MSGLEN - 2) / 2 ]
-            RIGHTLEN=$[ $COLUMNS - $MSGLEN - 2 - $LEFTLEN ]
+            LEFTLEN=$((($COLUMNS - $MSGLEN - 2) / 2))
+            RIGHTLEN=$(($COLUMNS - $MSGLEN - 2 - $LEFTLEN))
            LEFTSEP=$(echo $SEP | head -c $LEFTLEN)
            RIGHTSEP=$(echo $SEP | head -c $RIGHTLEN)
            >/dev/stderr echo "$LEFTSEP $1 $RIGHTSEP"
--- a/prepare-vms/lib/colors.sh
+++ b/prepare-vms/lib/colors.sh
@@ -1,15 +1,15 @@
-bold() { 
+bold() {
    echo "$(tput bold)$1$(tput sgr0)"
-} 
- 
-red() { 
-    echo "$(tput setaf 1)$1$(tput sgr0)"
-} 
+}

-green() { 
+red() {
+    echo "$(tput setaf 1)$1$(tput sgr0)"
+}
+
+green() {
    echo "$(tput setaf 2)$1$(tput sgr0)"
-} 
- 
-yellow(){ 
+}
+
+yellow() {
    echo "$(tput setaf 3)$1$(tput sgr0)"
-} 
+}
--- a/prepare-vms/lib/commands.sh
+++ b/prepare-vms/lib/commands.sh
@@ -1,7 +1,7 @@
 export AWS_DEFAULT_OUTPUT=text

 HELP=""
-_cmd () {
+_cmd() {
    HELP="$(printf "%s\n%-12s %s\n" "$HELP" "$1" "$2")"
 }

@@ -39,7 +39,7 @@ _cmd_cards() {
    need_tag $TAG
    need_settings $SETTINGS

-    aws_get_instance_ips_by_tag $TAG > tags/$TAG/ips.txt
+    aws_get_instance_ips_by_tag $TAG >tags/$TAG/ips.txt

    # Remove symlinks to old cards
    rm -f ips.html ips.pdf
@@ -78,18 +78,18 @@ _cmd_deploy() {
    >/dev/stderr echo ""

    sep "Deploying tag $TAG"
-    pssh -I tee /tmp/settings.yaml < $SETTINGS
+    pssh -I tee /tmp/settings.yaml <$SETTINGS
    pssh "
    sudo apt-get update &&
    sudo apt-get install -y python-setuptools &&
    sudo easy_install pyyaml"

    # Copy postprep.py to the remote machines, and execute it, feeding it the list of IP addresses
-    pssh -I tee /tmp/postprep.py < lib/postprep.py
-    pssh --timeout 900 --send-input "python /tmp/postprep.py >>/tmp/pp.out 2>>/tmp/pp.err" < ips.txt
+    pssh -I tee /tmp/postprep.py <lib/postprep.py
+    pssh --timeout 900 --send-input "python /tmp/postprep.py >>/tmp/pp.out 2>>/tmp/pp.err" <ips.txt

    # Install docker-prompt script
-    pssh -I sudo tee /usr/local/bin/docker-prompt < lib/docker-prompt
+    pssh -I sudo tee /usr/local/bin/docker-prompt <lib/docker-prompt
    pssh sudo chmod +x /usr/local/bin/docker-prompt

    # If /home/docker/.ssh/id_rsa doesn't exist, copy it from node1
@@ -206,7 +206,7 @@ _cmd_ips() {
 }

 _cmd list "List available batches in the current region"
-_cmd_list(){
+_cmd_list() {
    info "Listing batches in region $AWS_DEFAULT_REGION:"
    aws_display_tags
 }
@@ -259,7 +259,7 @@ _cmd_retag() {
    if [[ -z "$NEWTAG" ]]; then
        die "You must specify a new tag to apply."
    fi
-    aws_tag_instances $OLDTAG $NEWTAG 
+    aws_tag_instances $OLDTAG $NEWTAG
 }

 _cmd start "Start a batch of VMs"
@@ -279,8 +279,8 @@ _cmd_start() {
    # Upload our SSH keys to AWS if needed, to be added to each VM's authorized_keys
    key_name=$(sync_keys)

-    AMI=$(_cmd_ami)  # Retrieve the AWS image ID
-    TOKEN=$(get_token)  # generate a timestamp token for this batch of VMs
+    AMI=$(_cmd_ami)    # Retrieve the AWS image ID
+    TOKEN=$(get_token) # generate a timestamp token for this batch of VMs
    AWS_KEY_NAME=$(make_key_name)

    sep "Starting instances"
@@ -295,7 +295,7 @@ _cmd_start() {
        --instance-type t2.medium \
        --client-token $TOKEN \
        --image-id $AMI)
-    reservation_id=$(echo "$result" | head -1 | awk '{print $2}' )
+    reservation_id=$(echo "$result" | head -1 | awk '{print $2}')
    info "Reservation ID: $reservation_id"
    sep

@@ -317,7 +317,7 @@ _cmd_start() {

    mkdir -p tags/$TAG
    IPS=$(aws_get_instance_ips_by_tag $TAG)
-    echo "$IPS" > tags/$TAG/ips.txt
+    echo "$IPS" >tags/$TAG/ips.txt
    link_tag $TAG
    if [ -n "$SETTINGS" ]; then
        _cmd_deploy $TAG $SETTINGS
@@ -325,16 +325,16 @@ _cmd_start() {
        info "To deploy or kill these instances, run one of the following:"
        info "$0 deploy $TAG <settings/somefile.yaml>"
        info "$0 stop $TAG"
-    fi    
+    fi
 }

 _cmd ec2quotas "Check our EC2 quotas (max instances)"
-_cmd_ec2quotas(){
+_cmd_ec2quotas() {
    greet

    max_instances=$(aws ec2 describe-account-attributes \
-                    --attribute-names max-instances \
-                    --query 'AccountAttributes[*][AttributeValues]')
+        --attribute-names max-instances \
+        --query 'AccountAttributes[*][AttributeValues]')
    info "In the current region ($AWS_DEFAULT_REGION) you can deploy up to $max_instances instances."

    # Print list of AWS EC2 regions, highlighting ours ($AWS_DEFAULT_REGION) in the list
@@ -373,7 +373,7 @@ link_tag() {
    ln -sf $IPS_FILE ips.txt
 }

-pull_tag(){
+pull_tag() {
    TAG=$1
    need_tag $TAG
    link_tag $TAG
@@ -405,13 +405,13 @@ wait_until_tag_is_running() {
    COUNT=$2
    i=0
    done_count=0
-    while [[ $done_count -lt $COUNT ]]; do \
+    while [[ $done_count -lt $COUNT ]]; do
        let "i += 1"
        info "$(printf "%d/%d instances online" $done_count $COUNT)"
        done_count=$(aws ec2 describe-instances \
-                --filters "Name=instance-state-name,Values=running" \
-                          "Name=tag:Name,Values=$TAG" \
-                --query "Reservations[*].Instances[*].State.Name" \
+            --filters "Name=instance-state-name,Values=running" \
+            "Name=tag:Name,Values=$TAG" \
+            --query "Reservations[*].Instances[*].State.Name" \
            | tr "\t" "\n" \
            | wc -l)

@@ -432,7 +432,7 @@ tag_is_reachable() {
 test_tag() {
    ips_file=tags/$TAG/ips.txt
    info "Picking a random IP address in $ips_file to run tests."
-    n=$[ 1 + $RANDOM % $(wc -l < $ips_file) ]
+    n=$((1 + $RANDOM % $(wc -l <$ips_file)))
    ip=$(head -n $n $ips_file | tail -n 1)
    test_vm $ip
    info "Tests complete."
@@ -461,8 +461,8 @@ test_vm() {
        "env" \
        "ls -la /home/docker/.ssh"; do
        sep "$cmd"
-        echo "$cmd" | 
-            ssh -A -q \
+        echo "$cmd" \
+            | ssh -A -q \
                -o "UserKnownHostsFile /dev/null" \
                -o "StrictHostKeyChecking=no" \
                $user@$ip sudo -u docker -i \
@@ -480,26 +480,26 @@ test_vm() {
    info "Test VM was $ip."
 }

-make_key_name(){
+make_key_name() {
    SHORT_FINGERPRINT=$(ssh-add -l | grep RSA | head -n1 | cut -d " " -f 2 | tr -d : | cut -c 1-8)
    echo "${SHORT_FINGERPRINT}-${USER}"
 }

 sync_keys() {
    # make sure ssh-add -l contains "RSA"
-    ssh-add -l | grep -q RSA ||
-        die "The output of \`ssh-add -l\` doesn't contain 'RSA'. Start the agent, add your keys?"
+    ssh-add -l | grep -q RSA \
+        || die "The output of \`ssh-add -l\` doesn't contain 'RSA'. Start the agent, add your keys?"

    AWS_KEY_NAME=$(make_key_name)
    info "Syncing keys... "
-    if ! aws ec2 describe-key-pairs --key-name "$AWS_KEY_NAME" &> /dev/null; then
+    if ! aws ec2 describe-key-pairs --key-name "$AWS_KEY_NAME" &>/dev/null; then
        aws ec2 import-key-pair --key-name $AWS_KEY_NAME \
            --public-key-material "$(ssh-add -L \
-                                    | grep -i RSA \
-                                    | head -n1 \
-                                    | cut -d " " -f 1-2)" &> /dev/null
+                | grep -i RSA \
+                | head -n1 \
+                | cut -d " " -f 1-2)" &>/dev/null

-        if ! aws ec2 describe-key-pairs --key-name "$AWS_KEY_NAME" &> /dev/null; then
+        if ! aws ec2 describe-key-pairs --key-name "$AWS_KEY_NAME" &>/dev/null; then
            die "Somehow, importing the key didn't work. Make sure that 'ssh-add -l | grep RSA | head -n1' returns an RSA key?"
        else
            info "Imported new key $AWS_KEY_NAME."
@@ -523,4 +523,3 @@ describe_tag() {
    aws_display_instances_by_tag $TAG
    aws_display_instance_statuses_by_tag $TAG
 }
-
--- a/prepare-vms/lib/find-ubuntu-ami.sh
+++ b/prepare-vms/lib/find-ubuntu-ami.sh
@@ -3,10 +3,10 @@
 # That way, it can be safely invoked as a function from other scripts.

 find_ubuntu_ami() {
-(
+    (

-usage() {
-    cat >&2 <<__
+        usage() {
+            cat >&2 <<__
 usage: find-ubuntu-ami.sh [ <filter>... ] [ <sorting> ] [ <options> ]
 where:
    <filter> is pair of key and substring to search
@@ -33,66 +33,94 @@ where:
    protip for Docker orchestration workshop admin:
        ./find-ubuntu-ami.sh -t hvm:ebs -r \$AWS_REGION -v 15.10 -N
 __
-    exit 1
-}
+            exit 1
+        }

-args=`getopt hr:n:v:a:t:d:i:k:RNVATDIKq $*`
-if [ $? != 0 ] ; then
-    echo >&2
-    usage
-fi
+        args=$(getopt hr:n:v:a:t:d:i:k:RNVATDIKq $*)
+        if [ $? != 0 ]; then
+            echo >&2
+            usage
+        fi

-region=
-name=
-version=
-arch=
-type=
-date=
-image=
-kernel=
+        region=
+        name=
+        version=
+        arch=
+        type=
+        date=
+        image=
+        kernel=

-sort=date
+        sort=date

-quiet=
+        quiet=

-set -- $args
-for a ; do
-    case "$a" in
-        -h) usage ;;
+        set -- $args
+        for a; do
+            case "$a" in
+            -h) usage ;;

-        -r) region=$2 ; shift ;;
-        -n) name=$2 ; shift ;;
-        -v) version=$2 ; shift ;;
-        -a) arch=$2 ; shift ;;
-        -t) type=$2 ; shift ;;
-        -d) date=$2 ; shift ;;
-        -i) image=$2 ; shift ;;
-        -k) kernel=$2 ; shift ;;
-        
-        -R) sort=region ;;
-        -N) sort=name ;;
-        -V) sort=version ;;
-        -A) sort=arch ;;
-        -T) sort=type ;;
-        -D) sort=date ;;
-        -I) sort=image ;;
-        -K) sort=kernel ;;
+            -r)
+                region=$2
+                shift
+                ;;
+            -n)
+                name=$2
+                shift
+                ;;
+            -v)
+                version=$2
+                shift
+                ;;
+            -a)
+                arch=$2
+                shift
+                ;;
+            -t)
+                type=$2
+                shift
+                ;;
+            -d)
+                date=$2
+                shift
+                ;;
+            -i)
+                image=$2
+                shift
+                ;;
+            -k)
+                kernel=$2
+                shift
+                ;;

-        -q) quiet=y ;;
-        
-        --) shift ; break ;;
-        *) continue ;;
-    esac
-    shift
-done
+            -R) sort=region ;;
+            -N) sort=name ;;
+            -V) sort=version ;;
+            -A) sort=arch ;;
+            -T) sort=type ;;
+            -D) sort=date ;;
+            -I) sort=image ;;
+            -K) sort=kernel ;;

-[ $# = 0 ] || usage
+            -q) quiet=y ;;

-fix_json() {
-    tr -d \\n | sed 's/,]}/]}/'
-}
+            --)
+                shift
+                break
+                ;;
+            *) continue ;;
+            esac
+            shift
+        done

-jq_query() { cat <<__
+        [ $# = 0 ] || usage
+
+        fix_json() {
+            tr -d \\n | sed 's/,]}/]}/'
+        }
+
+        jq_query() {
+            cat <<__
    .aaData | map (
        {
            region: .[0],
@@ -116,31 +144,31 @@ jq_query() { cat <<__
    ) | sort_by(.$sort) | .[] |
    "\(.region)|\(.name)|\(.version)|\(.arch)|\(.type)|\(.date)|\(.image)|\(.kernel)"
 __
+        }
+
+        trim_quotes() {
+            sed 's/^"//;s/"$//'
+        }
+
+        escape_spaces() {
+            sed 's/ /\\\ /g'
+        }
+
+        url=http://cloud-images.ubuntu.com/locator/ec2/releasesTable
+
+        {
+            [ "$quiet" ] || echo REGION NAME VERSION ARCH TYPE DATE IMAGE KERNEL
+            curl -s $url | fix_json | jq "$(jq_query)" | trim_quotes | escape_spaces | tr \| ' '
+        } \
+            | while read region name version arch type date image kernel; do
+                image=${image%<*}
+                image=${image#*>}
+                if [ "$quiet" ]; then
+                    echo $image
+                else
+                    echo "$region|$name|$version|$arch|$type|$date|$image|$kernel"
+                fi
+            done | column -t -s \|
+
+    )
 }
-
-trim_quotes() {
-    sed 's/^"//;s/"$//'
-}
-
-escape_spaces() {
-    sed 's/ /\\\ /g'
-}
-
-url=http://cloud-images.ubuntu.com/locator/ec2/releasesTable
-
-{
-    [ "$quiet" ] || echo REGION NAME VERSION ARCH TYPE DATE IMAGE KERNEL
-    curl -s $url | fix_json | jq "`jq_query`" | trim_quotes | escape_spaces | tr \| ' '
-} |
-    while read region name version arch type date image kernel ; do
-        image=${image%<*}
-        image=${image#*>}
-        if [ "$quiet" ]; then
-            echo $image
-        else
-            echo "$region|$name|$version|$arch|$type|$date|$image|$kernel"
-        fi
-    done | column -t -s \|
-
-)
-}
--- a/prepare-vms/lib/pssh.sh
+++ b/prepare-vms/lib/pssh.sh
@@ -1,8 +1,8 @@
-# This file can be sourced in order to directly run commands on 
+# This file can be sourced in order to directly run commands on
 # a batch of VMs whose IPs are located in ips.txt of the directory in which
 # the command is run.

-pssh () {
+pssh() {
    HOSTFILE="ips.txt"

    [ -f $HOSTFILE ] || {
@@ -14,10 +14,10 @@ pssh () {
    export PSSH=$(which pssh || which parallel-ssh)

    $PSSH -h $HOSTFILE -l ubuntu \
-    --par 100 \
-    -O LogLevel=ERROR \
-    -O UserKnownHostsFile=/dev/null \
-    -O StrictHostKeyChecking=no \
-    -O ForwardAgent=yes \
-    "$@"
+        --par 100 \
+        -O LogLevel=ERROR \
+        -O UserKnownHostsFile=/dev/null \
+        -O StrictHostKeyChecking=no \
+        -O ForwardAgent=yes \
+        "$@"
 }
--- a/prepare-vms/workshopctl
+++ b/prepare-vms/workshopctl
@@ -50,8 +50,8 @@ check_dependencies() {
    status=0
    for dependency in $DEPENDENCIES; do
        if ! command -v $dependency >/dev/null; then
-             warning "Dependency $dependency could not be found."
-             status=1
+            warning "Dependency $dependency could not be found."
+            status=1
        fi
    done
    return $status
@@ -61,11 +61,11 @@ check_image() {
    docker inspect $TRAINER_IMAGE >/dev/null 2>&1
 }

-check_envvars ||
-    die "Please set all required environment variables."
+check_envvars \
+    || die "Please set all required environment variables."

-check_dependencies || 
-    warning "At least one dependency is missing. Install it or try the image wrapper."
+check_dependencies \
+    || warning "At least one dependency is missing. Install it or try the image wrapper."

 # Now check which command was invoked and execute it
 if [ "$1" ]; then