diff --git a/dev-tools/logs.sh b/dev-tools/logs.sh new file mode 100644 index 0000000..c74e002 --- /dev/null +++ b/dev-tools/logs.sh @@ -0,0 +1,2 @@ +#! /bin/bash +kubectl logs $(kubectl get pods -n kubeinvaders | grep 'kubeinvaders-' | awk '{ print $1 }') -n kubeinvaders -f diff --git a/html5/index.html b/html5/index.html index 402ccfb..5725118 100644 --- a/html5/index.html +++ b/html5/index.html @@ -239,13 +239,12 @@ experiments:
-

Regular Expression - PODs name

- -
+ +
@@ -295,6 +294,7 @@ experiments: $("#controlAutoPilotButton").text("Stop"); } } + function setLogConsole() { chaos_program_screen.style.display = "none"; programming_mode_buttons.style.display = "none"; diff --git a/html5/kubeinvaders.js b/html5/kubeinvaders.js index 92e1d4d..e95535f 100644 --- a/html5/kubeinvaders.js +++ b/html5/kubeinvaders.js @@ -245,7 +245,7 @@ function setLogRegex() { } };; oReq.setRequestHeader("Content-Type", "application/json"); - oReq.send($('#logConsoleRegex').val()); + oReq.send($('#logTailRegex').val()); } function setChaosContainer() { diff --git a/scripts/chaos-containers.lua b/scripts/chaos-containers.lua index a4081dc..32e221e 100644 --- a/scripts/chaos-containers.lua +++ b/scripts/chaos-containers.lua @@ -37,15 +37,20 @@ elseif ngx.var.request_method == "POST" and action == 'set' then elseif ngx.var.request_method == "POST" and action == "set_log_regex" then local body_data = ngx.req.get_body_data() red:set("log_pod_regex", body_data) - ngx.say("New container definition has been saved in Redis") + os.execute("> /var/www/html/chaoslogs.html") + ngx.say("New container definition has been saved in Redis => " .. body_data) return ngx.exit(ngx.status) elseif ngx.var.request_method == "POST" and action == "enable_log_tail" then local body_data = ngx.req.get_body_data() red:set("logs_enabled", "1") + os.execute("> /var/www/html/chaoslogs.html") + ngx.say("Enable Log Tail") return ngx.exit(ngx.status) elseif ngx.var.request_method == "POST" and action == "disable_log_tail" then red:set("logs_enabled", "0") + os.execute("> /var/www/html/chaoslogs.html") + ngx.say("Disable Log Tail") return ngx.exit(ngx.status) end diff --git a/scripts/logs_loop/start.py b/scripts/logs_loop/start.py index 09b7f75..b89453c 100644 --- a/scripts/logs_loop/start.py +++ b/scripts/logs_loop/start.py @@ -13,6 +13,23 @@ import random import redis import time import re +from hashlib import sha256 +import time + +def line_prepender(filename, line): + log_html_file = pathlib.Path(filename) + if not log_html_file.exists(): + with open(log_html_file, "w") as myfile: + myfile.write('') + + with open(filename, 'r+') as f: + content = f.read() + f.seek(0, 0) + f.write(line.rstrip('\r\n') + '\n' + content) + +# create logger +logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO")) +logging.info('Starting script for KubeInvaders taking logs from pods...') file = pathlib.Path('/tmp/redis.sock') @@ -22,11 +39,22 @@ else: r = redis.Redis("127.0.0.1", charset="utf-8", decode_responses=True) if os.environ.get("DEV"): + logging.info("Setting env var for dev...") r.set("log_pod_regex", ".*") - -# create logger -logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO")) -logging.info('Starting script for KubeInvaders programming mode') + r.set("logs_enabled", 1) + logging.info(r.get("log_pod_regex")) + logging.info(r.get("logs_enabled")) + +if r.exists("log_pod_regex"): + logging.info("The Redis key log_pod_regex exists...") +else: + logging.info("The Redis key log_pod_regex does NOT exists...") + r.set("log_pod_regex", ".*") + +if r.exists('logs_enabled'): + logging.info("The Redis key logs_enabled exists...") +else: + logging.info("The Redis key logs_enabled does NOT exists...") configuration = client.Configuration() token = os.environ["TOKEN"] @@ -47,10 +75,17 @@ namespace = "kubeinvaders" # r.delete(key) while True: + logging.info("Loop iteration...") + file = pathlib.Path('/var/www/html/chaoslogs.html') + if not file.exists(): + for key in r.scan_iter("log:*"): + r.delete(key) + webtail_pods = [] final_pod_list = [] if r.exists("log_pod_regex") and r.exists('logs_enabled'): - if r.get("logs_enabled") == 1: + logging.info("Found Redis keys for log tail...") + if r.get("logs_enabled") == "1": logging.info("Found regex log_pod_regex in Redis. Logs from all pods should be collected") log_pod_regex = r.get("log_pod_regex") try: @@ -68,16 +103,57 @@ while True: except ApiException as e: logging.info(e) + webtail_switch = False + final_pod_list = webtail_pods + api_response.items + if len(webtail_pods) > 0: + webtail_switch = True for pod in final_pod_list: - if ((pod.metadata.name in webtail_pods) or (pod.metadata.labels.get('approle') != None and pod.metadata.labels['approle'] == 'chaosnode' and pod.status.phase != "Pending")): + if webtail_switch or (pod.metadata.labels.get('approle') != None and pod.metadata.labels['approle'] == 'chaosnode' and pod.status.phase != "Pending"): try: + latest_log_tail = r.get(f"log_time:{pod.metadata.name}") logging.info(f"Reading logs of {pod.metadata.name} on {pod.metadata.namespace}") - api_response = api_instance.read_namespaced_pod_log(name=pod.metadata.name, namespace=pod.metadata.namespace) - logrow = f"
" - r.set(f"log:{pod.metadata.name}", logrow) - r.expire(f"log:{pod.metadata.name}", 30) + + if r.exists(f"log_time:{pod.metadata.name}"): + latest_log_tail_time = r.get(f"log_time:{pod.metadata.name}") + else: + latest_log_tail_time = time.time() + logging.info(f"Latest latest_log_tail for {pod.metadata.name} is {latest_log_tail_time}. Current Unix Time is {time.time()}") + + since = int(time.time() - float(latest_log_tail_time)) + + logging.info(f"Diff from time.time() and latest_log_tail_time for {pod.metadata.name} is {since}") + + if since == 0: + since = 1 + + api_response = api_instance.read_namespaced_pod_log(name=pod.metadata.name, namespace=pod.metadata.namespace, tail_lines=1, since_seconds=since) + + if api_response == "": + continue + logging.info(f"API Response: {api_response}") + + logrow = f"
[namespace:{pod.metadata.namespace}][pod:{pod.metadata.name}]
>>>{api_response}
" + + store = False + sha256log = sha256(logrow.encode('utf-8')).hexdigest() + + if r.exists(f"log:{pod.metadata.name}:{sha256log}"): + current_row = r.get(f"log:{pod.metadata.name}:{sha256log}") + if current_row != logrow: + store = True + + if not r.exists(f"log:{pod.metadata.name}:{sha256log}") or store: + file = pathlib.Path('/var/www/html') + if file.exists(): + log_html_file = pathlib.Path('/var/www/html/chaoslogs.html') + line_prepender(log_html_file, logrow) + + r.set(f"log:{pod.metadata.name}:{sha256log}", logrow) + r.set(f"log_time:{pod.metadata.name}", time.time()) + r.expire(f"log:{pod.metadata.name}:{sha256log}", 30) + logging.info(f"Phase of {pod.metadata.name} is {pod.status.phase}") if pod.status.phase == "Succeeded" and pod.metadata.labels['approle'] == 'chaosnode': try: @@ -87,13 +163,4 @@ while True: logging.info(e) except ApiException as e: logging.info(e) - file = pathlib.Path('/var/www/html') - if file.exists(): - log_html_file = pathlib.Path('/var/www/html/chaoslogs.html') - if log_html_file.exists(): - with open("/var/www/html/chaoslogs.html", "w") as myfile: - myfile.write('') - for key in r.scan_iter("log:*"): - with open("/var/www/html/chaoslogs.html", "a") as myfile: - myfile.write(str(r.get(key))) - time.sleep(1) + time.sleep(0.5) diff --git a/scripts/logs_loop/start.sh b/scripts/logs_loop/start.sh index f1fee92..0cbfb56 100755 --- a/scripts/logs_loop/start.sh +++ b/scripts/logs_loop/start.sh @@ -7,5 +7,5 @@ else # Source the service account token from the container directly. export TOKEN="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" fi - +export PYTHONWARNINGS="ignore:Unverified HTTPS request" python3 /opt/logs_loop/start.py https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT_HTTPS}