From cd390ebecbca91866f0d9d86f114e43d5ac88f2e Mon Sep 17 00:00:00 2001 From: Eugenio Marzo Date: Tue, 24 Jan 2023 20:51:12 +0100 Subject: [PATCH] fix logs --- scripts/logs_loop/start.py | 45 +++++++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/scripts/logs_loop/start.py b/scripts/logs_loop/start.py index 12a9604..7843d5c 100644 --- a/scripts/logs_loop/start.py +++ b/scripts/logs_loop/start.py @@ -19,7 +19,6 @@ import time import urllib3 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - def create_pod_list(logid, api_response_items, current_regex): json_re = json.loads(current_regex) regexsha = sha256(current_regex.encode('utf-8')).hexdigest() @@ -214,7 +213,23 @@ while True: r.set(f"pods_match_regex:{logid}", webtail_pods_len) logging.debug(f"[logid:{logid}][k-inv][logs-loop] Current Regex: {current_regex}") + pods_pending = 0 + pods_running = 0 + pods_succeeded = 0 + for pod in webtail_pods: + if pod.status.phase == "Pending": + pods_pending = pods_pending + 1 + if pod.status.phase == "Running": + pods_running = pods_running + 1 + if pod.status.phase == "Succeeded": + pods_succeeded = pods_succeeded + 1 + + r.set(f"logs:chaoslogs-{logid}", f"
[k-inv] pods on Pending phase: {pods_pending}
[k-inv] pods on Succeeded phase: {pods_succeeded}
[k-inv] pods on Running phase: {pods_running}
") + + for pod in webtail_pods: + if pod.status.phase == "Unknown" and pod.status.phase == "Pending": + continue logging.debug(f"[logid:{logid}][k-inv][logs-loop] Taking logs from {pod.metadata.name}") container_list = [] for container in pod.spec.containers: @@ -224,7 +239,7 @@ while True: for container in container_list: logging.debug(f"[logid:{logid}][k-inv][logs-loop] Listing containers of {pod.metadata.name}. Computing {container} phase: {pod.status.phase}") - if pod.status.phase != "Unknown": + if pod.status.phase != "Unknown" and pod.status.phase != "Pending": logging.debug(f"[logid:{logid}][k-inv][logs-loop] Container {container} on pod {pod.metadata.name} has accepted phase for taking logs") try: if r.exists(f"log_time:{logid}:{pod.metadata.name}:{container}"): @@ -238,25 +253,35 @@ while True: if since == 0: since = 1 - logging.debug(f"[logid:{logid}][k-inv][logs-loop] Calling K8s API for reading logs of {pod.metadata.name} container {container} in namespace {pod.metadata.namespace} since {since} seconds") + logging.debug(f"[logid:{logid}][k-inv][logs-loop] Calling K8s API for reading logs of {pod.metadata.name} container {container} in namespace {pod.metadata.namespace} since {since} seconds - phase {pod.status.phase}") - api_response = api_instance.read_namespaced_pod_log(name=pod.metadata.name, namespace=pod.metadata.namespace, since_seconds=since, container=container) + api_response = api_instance.read_namespaced_pod_log(name=pod.metadata.name, namespace=pod.metadata.namespace, since_seconds=since, container=container, tail_lines=since) - logging.debug(f"[logid:{logid}][k-inv][logs-loop] Computing K8s API response for reading logs of {pod.metadata.name} in namespace {pod.metadata.namespace}") + logging.debug(f"[logid:{logid}][k-inv][logs-loop] Computing K8s API response for reading logs of {pod.metadata.name} in namespace {pod.metadata.namespace} - phase {pod.status.phase}") logging.debug(f"[logid:{logid}][k-inv][logs-loop] {type(api_response)} {api_response}") r.set(f"log_time:{logid}:{pod.metadata.name}:{container}", time.time()) k = 5 - - while api_response == "" and k < 120: - logging.debug(f"[logid:{logid}][k-inv][logs-loop][logs collector attempt {k}] Calling K8s API for reading logs of {pod.metadata.name} container {container} in namespace {pod.metadata.namespace} since {since} seconds") - api_response = api_instance.read_namespaced_pod_log(name=pod.metadata.name, namespace=pod.metadata.namespace, since_seconds=since, container=container) + + regex_return = re.search(r'[\w]+', api_response) + logging.debug(f"[logid:{logid}][k-inv][logs-loop] Regex on api_response: {regex_return}") + + while not re.search(r'[\w]+', api_response) and k < 60: + logging.debug(f"[logid:{logid}][k-inv][logs-loop][logs collector attempt {k}] Calling K8s API for reading logs of {pod.metadata.name} container {container} in namespace {pod.metadata.namespace} since {since} seconds - phase {pod.status.phase}") + api_response = api_instance.read_namespaced_pod_log(name=pod.metadata.name, namespace=pod.metadata.namespace, since_seconds=since, container=container, tail_lines=since) + logging.debug(f"[logid:{logid}][k-inv][logs-loop] API Response: {api_response}") + # regex_return = re.search(r'[\w]+', api_response) + # logging.debug(f"[logid:{logid}][k-inv][logs-loop] Regex on api_response: {regex_return}") + since = since + 1 k = k + 1 time.sleep(0.5) - if api_response == "": + if not re.search(r'[\w]+', api_response): + # regex_return = re.search(r'[\w]+', api_response) + # logging.debug(f"[logid:{logid}][k-inv][logs-loop] Regex on api_response: {regex_return}") + logging.debug(f"[logid:{logid}][k-inv][logs-loop] API Response for reading logs of {pod.metadata.name} in namespace {pod.metadata.namespace} is still empty") continue compute_line(api_response, container)