From 317a6cd708b788edb22bf958c15cab9097d776be Mon Sep 17 00:00:00 2001
From: Eugenio Marzo <eugenio.marzo@sourcesense.com>
Date: Thu, 5 Jan 2023 13:25:39 +0100
Subject: [PATCH] added cache for webtail logs

---
 html5/index.html             |  2 +-
 html5/kubeinvaders.js        |  7 +++-
 nginx/KubeInvaders.conf      |  9 +++++
 scripts/chaos-containers.lua |  6 ++-
 scripts/logs_loop/start.py   | 76 ++++++++++++++++++++++++------------
 5 files changed, 72 insertions(+), 28 deletions(-)

diff --git a/html5/index.html b/html5/index.html
index 0f9ed93..7a34717 100644
--- a/html5/index.html
+++ b/html5/index.html
@@ -426,7 +426,7 @@ experiments:
     
       <div id="logTailRegexInput" style="display: block;">
         <div class="row" style="margin-top: 2%; padding-left: 2%; padding-right: 2%;">
-          <input type="text" style="font-family: Courier New, Courier, monospace;" id="logTailRegex" value='{"pod":".*", "namespace":"namespace1", "labels":".*", "annotations":".*", "containers": ".*"}'/>
+          <input type="text" style="font-family: Courier New, Courier, monospace;" id="logTailRegex" value='{"pod":".*", "namespace":"namespace1", "labels":".*", "annotations":".*", "containers":".*"}'/>
         </div>
         <div class="row" style="margin-top: 1%;">
           <div class="col text-center">
diff --git a/html5/kubeinvaders.js b/html5/kubeinvaders.js
index 8c03bfc..bbe501f 100644
--- a/html5/kubeinvaders.js
+++ b/html5/kubeinvaders.js
@@ -71,6 +71,8 @@ var help = false;
 var chaos_nodes = true;
 var chaos_pods = true;
 var log_tail_alert = '<div id="alert_placeholder3" style="margin-top: 2%; margin-bottom: 1%; background-color: #161616; color: #ffffff" class="alert" role="alert">';
+var log_tail_alert_no_pixel = '<div id="alert_placeholder3" style="margin-top: 2%; margin-bottom: 1%; background-color: #161616; color: #ffffff; font-family: Courier, monospace;" class="alert" role="alert">';
+
 var alert_div = '<div id="alert_placeholder" style="margin-top: 2%; margin-bottom: 1%; background-color: #161616; color: #ffffff" class="alert" role="alert">';
 var alert_div_webtail = '<div id="alert_placeholder3" style="margin-top: 2%; background-color: #161616; color: #ffffff" class="alert" role="alert">';
 
@@ -304,8 +306,9 @@ function keepAliveJobsLogs() {
     var oReq = new XMLHttpRequest();
     oReq.onreadystatechange = function () {
         if (this.readyState === XMLHttpRequest.DONE && this.status === 200) {
-            // TO DO..
-            //$('#alert_placeholder').replaceWith("Sent keepalive for logs current session...");
+            if (!this.responseText.toLowerCase().match(/.*null.*/)) {
+                $('#alert_placeholder3').replaceWith(log_tail_alert_no_pixel + this.responseText.replace("nil", "") + '</div>');
+            }
         }
     };;
     oReq.open("GET", "https://" + clu_endpoint + "/chaos/logs/keepalive?logid=" + random_code);
diff --git a/nginx/KubeInvaders.conf b/nginx/KubeInvaders.conf
index 1c403f1..fba70f0 100644
--- a/nginx/KubeInvaders.conf
+++ b/nginx/KubeInvaders.conf
@@ -153,10 +153,19 @@ server {
       local data = ngx.req.get_body_data()
       local okredis, errredis = red:connect("unix:/tmp/redis.sock")
       logid = args["logid"]
+      
       red:set("do_not_clean_log:" .. logid, "1")
       red:expire("do_not_clean_log:" .. logid, "10")
+      
       red:set("logs_enabled:" .. logid, "1")
       red:expire("logs_enabled:" .. logid, "10")
+
+      if red:exists("log_status:".. logid) then
+        ngx.say(red:get("log_status:".. logid))
+      else
+        ngx.say('Waiting for log collector status...')
+      end
+
     }
   }
 
diff --git a/scripts/chaos-containers.lua b/scripts/chaos-containers.lua
index 08f84f2..c4979be 100644
--- a/scripts/chaos-containers.lua
+++ b/scripts/chaos-containers.lua
@@ -39,7 +39,11 @@ elseif ngx.var.request_method == "POST" and action == "set_log_regex" then
   red:set("log_pod_regex:" .. arg['id'], body_data)
   red:set("programming_mode", "0")
   ngx.say("Regex has been set => " .. body_data)
-  ngx.log(ngx.ERR, "Set Regex for web log tail. log id " .. arg['id'])
+  ngx.log(ngx.ERR, "Set Regex for web log tail. Log id " .. arg['id'])
+  --local redis_del_cmd = "(redis-cli KEYS 'regex_cmp:'" .. arg['id'] .. "| xargs redis-cli DEL) || echo"
+  --local handle = io.popen(redis_del_cmd)
+  --local result = handle:read("*a")
+  --local rc = handle:close()
   return ngx.exit(ngx.status)
 
 elseif ngx.var.request_method == "POST" and action == "enable_log_tail" then
diff --git a/scripts/logs_loop/start.py b/scripts/logs_loop/start.py
index 215c4a9..e43b559 100644
--- a/scripts/logs_loop/start.py
+++ b/scripts/logs_loop/start.py
@@ -118,10 +118,13 @@ while True:
             logid = key.split(":")[1]
 
             if r.exists(f"log_pod_regex:{logid}"):
+                current_regex = r.get(f"log_pod_regex:{logid}")
+                r.set(f"log_status:{logid}", f"Regex for this logging session is {current_regex}")
                 logging.info(f"[logid:{logid}] The Redis key log_pod_regex exists...")
             else:
                 logging.info(f"[logid:{logid}] The Redis key log_pod_regex does NOT exists...")
                 r.set(f"log_pod_regex:{logid}", '{"pod":".*", "namespace":".*", "labels":".*", "annotations":".*", "containers": ".*"}')
+                r.set(f"log_status:{logid}", "Regex for this logging session not found using default {\"pod\":\".*\", \"namespace\":\".*\", \"labels\":\".*\", \"annotations\":\".*\", \"containers\":\".*\"}")
 
             if r.exists(f"logs_enabled:{logid}"):
                 logging.info(f"[logid:{logid}] The Redis key logs_enabled exists...")
@@ -163,9 +166,13 @@ while True:
                         api_response = api_instance.list_pod_for_all_namespaces()
                     except ApiException as e:
                         logging.info(e)
-                    logging.info(f"[logid:{logid}] Going to search pod compliant with the regex on {len(api_response.items)} pods")
+                    
+                    pods_found_info = f"[logid:{logid}] Looking for pods compliant with the current regex. Scanning {len(api_response.items)} pods"
+                    logging.info(pods_found_info)
+                    r.set(f"log_status:{logid}", pods_found_info)
 
                     json_re = json.loads(log_pod_regex)
+                    regexsha = sha256(log_pod_regex.encode('utf-8')).hexdigest()
                     pod_re = json_re["pod"]
                     namespace_re = json_re["namespace"]
                     annotations_re = json_re["annotations"]
@@ -178,30 +185,51 @@ while True:
                     logging.info(f"[logid:{logid}] Regex for labels is |{labels_re}|")
                     logging.info(f"[logid:{logid}] Regex for annotation is |{annotations_re}|")
 
-                    for pod in api_response.items:
-                        logging.info(f"[logid:{logid}] Regex comparison |{pod_re}| |{pod.metadata.name}|")
-                        logging.info(f"[logid:{logid}] Regex comparison |{namespace_re}| |{pod.metadata.namespace}|")
-                        logging.info(f"[logid:{logid}] Regex comparison |{labels_re}| |{str(pod.metadata.labels)}|")
-                        logging.info(f"[logid:{logid}] Regex comparison |{annotations_re}| |{str(pod.metadata.annotations)}|")
+                    for pod in api_response.items:   
+                        if r.exists(f"regex_cmp:{logid}:{pod.metadata.namespace}:{pod.metadata.name}"):
+                            cached_regex_match = r.get(f"regex_cmp:{logid}:{pod.metadata.namespace}:{pod.metadata.name}")
+                            if cached_regex_match == "maching":
+                                webtail_pods.append(pod)
+                                regex_match_info = f"[logid:{logid}] Taking logs of {pod.metadata.name}. Redis has cached that {log_pod_regex} is good for {pod.metadata.name}"
+                                logging.info(regex_match_info)
+                                r.set(f"log_status:{logid}", regex_match_info)
+                            else:
+                                regex_match_info = f"[logid:{logid}] SKIPPING logs of {pod.metadata.name}. Redis has cached that {log_pod_regex} is not good for {pod.metadata.name}"
+                                logging.info(regex_match_info)
+                                r.set(f"log_status:{logid}", regex_match_info)
+                        else:
+                            logging.info(f"[logid:{logid}] Regex comparison |{pod_re}| |{pod.metadata.name}|")
+                            logging.info(f"[logid:{logid}] Regex comparison |{namespace_re}| |{pod.metadata.namespace}|")
+                            logging.info(f"[logid:{logid}] Regex comparison |{labels_re}| |{str(pod.metadata.labels)}|")
+                            logging.info(f"[logid:{logid}] Regex comparison |{annotations_re}| |{str(pod.metadata.annotations)}|")
 
-                        if re.search(f"{pod_re}", pod.metadata.name):
-                            logging.info(f"[logid:{logid}] Regex comparison |{pod_re}| |{pod.metadata.name}| RESULT: OK")
-                            if re.search(f"{namespace_re}", pod.metadata.namespace):
-                                logging.info(f"[logid:{logid}] Regex comparison |{namespace_re}| |{pod.metadata.namespace}| RESULT: OK")
-                                if re.search(f"{labels_re}", str(pod.metadata.labels)):
-                                    logging.info(f"[logid:{logid}] Regex comparison |{labels_re}| |{str(pod.metadata.labels)}| RESULT: OK")
-                                    if re.search(f"{annotations_re}", str(pod.metadata.annotations)):
-                                        logging.info(f"[logid:{logid}] Regex comparison |{annotations_re}| |{str(pod.metadata.annotations)}| RESULT: OK")
-                                        webtail_pods.append(pod)
-                                        logging.info(f"[logid:{logid}] Taking log of {pod.metadata.name} because it is compliant with the regex {log_pod_regex}")
-                        #             else:
-                        #                 logging.info(f"[logid:{logid}] Regex comparison |{annotations_re}| |{str(pod.metadata.annotations)}| RESULT FAILED!")
-                        #         else:
-                        #             logging.info(f"[logid:{logid}] Regex comparison |{labels_re}| |{str(pod.metadata.labels)}| RESULT: FAILED!")
-                        #     else:
-                        #         logging.info(f"[logid:{logid}] Regex comparison |{namespace_re}| |{pod.metadata.namespace}| RESULT: FAILED!")
-                        # else:
-                        #     logging.info(f"[logid:{logid}] Regex comparison |{pod_re}| |{pod.metadata.name}| RESULT FAILED!")
+                            if re.search(f"{pod_re}", pod.metadata.name):
+                                logging.info(f"[logid:{logid}] Regex comparison |{pod_re}| |{pod.metadata.name}| RESULT: OK")
+                                regex_key_name = "regex_cmp:{regexsha}:{regexsha}:{logid}:{pod.metadata.namespace}:{pod.metadata.name}"
+
+                                if re.search(f"{namespace_re}", pod.metadata.namespace):
+                                    logging.info(f"[logid:{logid}] Regex comparison |{namespace_re}| |{pod.metadata.namespace}| RESULT: OK")
+                                    if re.search(f"{labels_re}", str(pod.metadata.labels)):
+                                        logging.info(f"[logid:{logid}] Regex comparison |{labels_re}| |{str(pod.metadata.labels)}| RESULT: OK")
+                                        if re.search(f"{annotations_re}", str(pod.metadata.annotations)):
+                                            logging.info(f"[logid:{logid}] Regex comparison |{annotations_re}| |{str(pod.metadata.annotations)}| RESULT: OK")
+                                            webtail_pods.append(pod)
+                                            regex_match_info = f"[logid:{logid}] Taking logs from {pod.metadata.name}. It is compliant with the Regex {log_pod_regex}"
+                                            r.set(regex_key_name, "maching")
+                                            logging.info(regex_match_info)
+                                            r.set(f"log_status:{logid}", regex_match_info)
+                                        else:
+                                            logging.info(f"[logid:{logid}] Regex comparison |{annotations_re}| |{str(pod.metadata.annotations)}| RESULT FAILED!")
+                                            r.set(regex_key_name, "not_maching")
+                                    else:
+                                        logging.info(f"[logid:{logid}] Regex comparison |{labels_re}| |{str(pod.metadata.labels)}| RESULT: FAILED!")
+                                        r.set(regex_key_name, "not_maching")
+                                else:
+                                    logging.info(f"[logid:{logid}] Regex comparison |{namespace_re}| |{pod.metadata.namespace}| RESULT: FAILED!")
+                                    r.set(regex_key_name, "not_maching")
+                            else:
+                                logging.info(f"[logid:{logid}] Regex comparison |{pod_re}| |{pod.metadata.name}| RESULT FAILED!")
+                                r.set(regex_key_name, "not_maching")
 
             try:
                 api_response = api_instance.list_namespaced_pod(namespace="kubeinvaders")