From 784b2a3e4ea44201c7341441c6581bc285d4ddb5 Mon Sep 17 00:00:00 2001 From: Jerome Petazzoni Date: Mon, 20 Jan 2020 14:23:20 -0600 Subject: [PATCH] Big update to autopilot Autopilot can now continue when errors happen, and it writes success/failure of each snippet in a log file for later review. Also added e2e.sh to provision a test environment and start the remote tmux instance. --- prepare-vms/e2e.sh | 10 ++ prepare-vms/lib/commands.sh | 28 ++-- slides/autopilot/autotest.py | 241 +++++++++++++++++++++-------------- 3 files changed, 177 insertions(+), 102 deletions(-) create mode 100755 prepare-vms/e2e.sh diff --git a/prepare-vms/e2e.sh b/prepare-vms/e2e.sh new file mode 100755 index 00000000..7c4e3901 --- /dev/null +++ b/prepare-vms/e2e.sh @@ -0,0 +1,10 @@ +#!/bin/sh +set -e +TAG=$(./workshopctl maketag) +./workshopctl start --settings settings/jerome.yaml --infra infra/aws-eu-central-1 --tag $TAG +./workshopctl deploy $TAG +./workshopctl kube $TAG +./workshopctl helmprom $TAG +while ! ./workshopctl kubetest $TAG; do sleep 1; done +./workshopctl tmux $TAG +echo ./workshopctl stop $TAG diff --git a/prepare-vms/lib/commands.sh b/prepare-vms/lib/commands.sh index 71bcf643..b4087de1 100644 --- a/prepare-vms/lib/commands.sh +++ b/prepare-vms/lib/commands.sh @@ -323,6 +323,15 @@ _cmd_listall() { done } +_cmd maketag "Generate a quasi-unique tag for a group of instances" +_cmd_maketag() { + if [ -z $USER ]; then + export USER=anonymous + fi + MS=$(($(date +%N)/1000000)) + date +%Y-%m-%d-%H-%M-$MS-$USER +} + _cmd ping "Ping VMs in a given tag, to check that they have network access" _cmd_ping() { TAG=$1 @@ -465,7 +474,7 @@ _cmd_start() { need_infra $INFRA if [ -z "$TAG" ]; then - TAG=$(make_tag) + TAG=$(_cmd_maketag) fi mkdir -p tags/$TAG ln -s ../../$INFRA tags/$TAG/infra.sh @@ -527,6 +536,16 @@ _cmd_test() { test_tag } +_cmd tmux "Log into the first node and start a tmux server" +_cmd_tmux() { + TAG=$1 + need_tag + IP=$(head -1 tags/$TAG/ips.txt) + info "Opening ssh+tmux with $IP" + rm -f /tmp/tmux-$UID/default + ssh -t -L /tmp/tmux-$UID/default:/tmp/tmux-1001/default docker@$IP tmux new-session -As 0 +} + _cmd helmprom "Install Helm and Prometheus" _cmd_helmprom() { TAG=$1 @@ -721,10 +740,3 @@ sync_keys() { info "Using existing key $AWS_KEY_NAME." fi } - -make_tag() { - if [ -z $USER ]; then - export USER=anonymous - fi - date +%Y-%m-%d-%H-%M-$USER -} diff --git a/slides/autopilot/autotest.py b/slides/autopilot/autotest.py index cdd3e603..4f047a3e 100755 --- a/slides/autopilot/autotest.py +++ b/slides/autopilot/autotest.py @@ -28,8 +28,8 @@ class State(object): def __init__(self): self.clipboard = "" self.interactive = True - self.verify_status = False - self.simulate_type = True + self.verify_status = True + self.simulate_type = False self.switch_desktop = False self.sync_slides = False self.open_links = False @@ -69,6 +69,8 @@ class State(object): state = State() +outfile = open("autopilot.log", "w") + def hrule(): return "="*int(subprocess.check_output(["tput", "cols"])) @@ -240,6 +242,8 @@ tmux rm -f /tmp/tmux-{uid}/default && ssh -t -L /tmp/tmux-{uid}/default:/tmp/tmux-1001/default docker@{ipaddr} tmux new-session -As 0 +(Or use workshopctl tmux) + 3. If you cannot control a remote tmux: tmux new-session ssh docker@{ipaddr} @@ -264,38 +268,11 @@ for slide in re.split("\n---?\n", content): slide_classes = slide_classes[0].split(",") slide_classes = [c.strip() for c in slide_classes] if excluded_classes & set(slide_classes): - logging.info("Skipping excluded slide.") + logging.debug("Skipping excluded slide.") continue slides.append(Slide(slide)) -# Send a single key. -# Useful for special keys, e.g. tmux interprets these strings: -# ^C (and all other sequences starting with a caret) -# Space -# ... and many others (check tmux manpage for details). -def send_key(data): - subprocess.check_call(["tmux", "send-keys", data]) - - -# Send multiple keys. -# If keystroke simulation is off, all keys are sent at once. -# If keystroke simulation is on, keys are sent one by one, with a delay between them. -def send_keys(data): - if not state.simulate_type: - subprocess.check_call(["tmux", "send-keys", data]) - else: - for key in data: - if key == ";": - key = "\\;" - if key == "\n": - if interruptible_sleep(1): return - send_key(key) - if interruptible_sleep(0.15*random.random()): return - if key == "\n": - if interruptible_sleep(1): return - - def capture_pane(): return subprocess.check_output(["tmux", "capture-pane", "-p"]).decode('utf-8') @@ -305,7 +282,7 @@ setup_tmux_and_ssh() try: state.load() - logging.info("Successfully loaded state from file.") + logging.debug("Successfully loaded state from file.") # Let's override the starting state, so that when an error occurs, # we can restart the auto-tester and then single-step or debug. # (Instead of running again through the same issue immediately.) @@ -314,6 +291,7 @@ except Exception as e: logging.exception("Could not load state from file.") logging.warning("Using default values.") + def move_forward(): state.snippet += 1 if state.snippet > len(slides[state.slide].snippets): @@ -337,6 +315,140 @@ def check_bounds(): state.slide = len(slides)-1 +########################################################## +# All functions starting with action_ correspond to the +# code to be executed when seeing ```foo``` blocks in the +# input. ```foo``` would call action_foo(state, snippet). +########################################################## + + +def send_keys(keys): + subprocess.check_call(["tmux", "send-keys", keys]) + +# Send a single key. +# Useful for special keys, e.g. tmux interprets these strings: +# ^C (and all other sequences starting with a caret) +# Space +# ... and many others (check tmux manpage for details). +def action_key(state, snippet): + send_keys(snippet.data) + + +# Send multiple keys. +# If keystroke simulation is off, all keys are sent at once. +# If keystroke simulation is on, keys are sent one by one, with a delay between them. +def action_keys(state, snippet, keys=None): + if keys is None: + keys = snippet.data + if not state.simulate_type: + send_keys(keys) + else: + for key in keys: + if key == ";": + key = "\\;" + if key == "\n": + if interruptible_sleep(1): return + send_keys(key) + if interruptible_sleep(0.15*random.random()): return + if key == "\n": + if interruptible_sleep(1): return + + +def action_hide(state, snippet): + if state.run_hidden: + action_bash(state, snippet) + + +def action_bash(state, snippet): + data = snippet.data + # Make sure that we're ready + wait_for_prompt() + # Strip leading spaces + data = re.sub("\n +", "\n", data) + # Remove backticks (they are used to highlight sections) + data = data.replace('`', '') + # Add "RETURN" at the end of the command :) + data += "\n" + # Send command + action_keys(state, snippet, data) + # Force a short sleep to avoid race condition + time.sleep(0.5) + if snippet.next and snippet.next.method == "wait": + wait_for_string(snippet.next.data) + elif snippet.next and snippet.next.method == "longwait": + wait_for_string(snippet.next.data, 10*TIMEOUT) + else: + wait_for_prompt() + # Verify return code + check_exit_status() + + +def action_copy(state, snippet): + screen = capture_pane() + matches = re.findall(snippet.data, screen, flags=re.DOTALL) + if len(matches) == 0: + raise Exception("Could not find regex {} in output.".format(snippet.data)) + # Arbitrarily get the most recent match + match = matches[-1] + # Remove line breaks (like a screen copy paste would do) + match = match.replace('\n', '') + logging.debug("Copied {} to clipboard.".format(match)) + state.clipboard = match + + +def action_paste(state, snippet): + logging.debug("Pasting {} from clipboard.".format(state.clipboard)) + action_keys(state, snippet, state.clipboard) + + +def action_check(state, snippet): + wait_for_prompt() + check_exit_status() + + +def action_open(state, snippet): + # Cheap way to get node1's IP address + screen = capture_pane() + url = snippet.data.replace("/node1", "/{}".format(IPADDR)) + # This should probably be adapted to run on different OS + if state.open_links: + subprocess.check_output(["xdg-open", url]) + focus_browser() + if state.interactive: + print("Press any key to continue to next step...") + click.getchar() + + +def action_tmux(state, snippet): + subprocess.check_call(["tmux"] + snippet.data.split()) + + +def action_unknown(state, snippet): + logging.warning("Unknown method {}: {!r}".format(snippet.method, snippet.data)) + + +def run_snippet(state, snippet): + logging.info("Running with method {}: {}".format(snippet.method, snippet.data)) + try: + action = globals()["action_"+snippet.method] + except KeyError: + action = action_unknown + try: + action(state, snippet) + result = "OK" + except: + result = "ERR" + logging.exception("While running method {} with {!r}".format(snippet.method, snippet.data)) + # Try to recover + try: + wait_for_prompt() + except: + subprocess.check_call(["tmux", "new-window"]) + wait_for_prompt() + outfile.write("{} SLIDE={} METHOD={} DATA={!r}\n".format(result, state.slide, snippet.method, snippet.data)) + outfile.flush() + + while True: state.save() slide = slides[state.slide] @@ -405,7 +517,10 @@ while True: # continue until next timeout state.interactive = False elif command in ("y", "\r", " "): - if not snippet: + if snippet: + run_snippet(state, snippet) + move_forward() + else: # Advance to next snippet # Advance until a slide that has snippets while not slides[state.slide].snippets: @@ -415,67 +530,5 @@ while True: break # And then advance to the snippet move_forward() - continue - method, data = snippet.method, snippet.data - logging.info("Running with method {}: {}".format(method, data)) - if method == "key": - send_key(data) - elif method == "keys": - send_keys(data) - elif method == "bash" or (method == "hide" and state.run_hidden): - # Make sure that we're ready - wait_for_prompt() - # Strip leading spaces - data = re.sub("\n +", "\n", data) - # Remove backticks (they are used to highlight sections) - data = data.replace('`', '') - # Add "RETURN" at the end of the command :) - data += "\n" - # Send command - send_keys(data) - # Force a short sleep to avoid race condition - time.sleep(0.5) - if snippet.next and snippet.next.method == "wait": - wait_for_string(snippet.next.data) - elif snippet.next and snippet.next.method == "longwait": - wait_for_string(snippet.next.data, 10*TIMEOUT) - else: - wait_for_prompt() - # Verify return code - check_exit_status() - elif method == "copy": - screen = capture_pane() - matches = re.findall(data, screen, flags=re.DOTALL) - if len(matches) == 0: - raise Exception("Could not find regex {} in output.".format(data)) - # Arbitrarily get the most recent match - match = matches[-1] - # Remove line breaks (like a screen copy paste would do) - match = match.replace('\n', '') - logging.info("Copied {} to clipboard.".format(match)) - state.clipboard = match - elif method == "paste": - logging.info("Pasting {} from clipboard.".format(state.clipboard)) - send_keys(state.clipboard) - elif method == "check": - wait_for_prompt() - check_exit_status() - elif method == "open": - # Cheap way to get node1's IP address - screen = capture_pane() - url = data.replace("/node1", "/{}".format(IPADDR)) - # This should probably be adapted to run on different OS - if state.open_links: - subprocess.check_output(["xdg-open", url]) - focus_browser() - if state.interactive: - print("Press any key to continue to next step...") - click.getchar() - elif method == "tmux": - subprocess.check_call(["tmux"] + data.split()) - else: - logging.warning("Unknown method {}: {!r}".format(method, data)) - move_forward() - else: logging.warning("Unknown command {}.".format(command))