diff --git a/html/index.html b/html/index.html index d707976..d38b53c 100644 --- a/html/index.html +++ b/html/index.html @@ -6,11 +6,10 @@ - + - @@ -379,6 +378,162 @@ + + + + + + +
@@ -392,10 +547,10 @@ - + - + '); @@ -320,6 +320,10 @@ function saveChaosReport() { } function updateElapsedTimeArray(projectName) { + if (!projectName || projectName.trim() === "") { + return; + } + $("#chaosReportSessionTimeDiv").html(diffBetweenTwoDates(chaos_report_start_date, new Date()) + " seconds"); // console.log("[SAVE-CHAOS-REPORT-CONF] Diff Between Dates: " + toString(diffBetweenTwoDates(chaos_report_start_date, new Date()))); // console.log("[SAVE-CHAOS-REPORT-CONF] Updating elapsed time array for project: " + projectName); @@ -330,10 +334,12 @@ function updateElapsedTimeArray(projectName) { oReq.onreadystatechange = function () { if (this.readyState === XMLHttpRequest.DONE && this.status === 200) { - // console.log("[SAVE-CHAOS-REPORT-CONF] Elapsed time array received from Redis: " + parseFloat(this.responseText)); - chaos_report_http_elapsed_time_array.push(parseFloat(this.responseText)); - while (chaos_report_http_elapsed_time_array.length > 40) { - chaos_report_http_elapsed_time_array.shift(); + var elapsed = parseFloat(this.responseText); + if (!isNaN(elapsed) && isFinite(elapsed)) { + chaos_report_http_elapsed_time_array.push(elapsed); + while (chaos_report_http_elapsed_time_array.length > 40) { + chaos_report_http_elapsed_time_array.shift(); + } } } };; @@ -345,6 +351,10 @@ function updateElapsedTimeArray(projectName) { } function updateStatusCodePieChart(projectName) { + if (!projectName || projectName.trim() === "") { + return; + } + // console.log("[SAVE-CHAOS-REPORT-CONF] Updating Status Code Pie Chart for project: " + projectName); var oReq = new XMLHttpRequest(); @@ -354,9 +364,18 @@ function updateStatusCodePieChart(projectName) { oReq.onreadystatechange = function () { if (this.readyState === XMLHttpRequest.DONE && this.status === 200) { var status_code = this.responseText.trim(); - // console.log("[SAVE-CHAOS-REPORT-CONF] Status Code Pie Chart received from Redis: |" + status_code + "|"); + if (status_code === "" || status_code === "Key not found") { + return; + } + if (!(status_code in chart_status_code_dict)) { + if (/^\d{3}$/.test(status_code)) { + status_code = "Other"; + } else { + status_code = "Connection Error"; + } + } - chart_status_code_dict[status_code] = chart_status_code_dict[status_code] + 1 + chart_status_code_dict[status_code] = (chart_status_code_dict[status_code] || 0) + 1 myHTTPStatusCodeChart.setOption({ series: [ diff --git a/html/js/kubeinvaders.js b/html/js/kubeinvaders.js index b68810f..2d90eb4 100644 --- a/html/js/kubeinvaders.js +++ b/html/js/kubeinvaders.js @@ -841,7 +841,13 @@ window.setInterval(function draw() { ctx.fillStyle = 'white'; ctx.font = "18px 'Ubuntu Mono'"; - ctx.fillText('Cluster: ' + endpoint, 10, startYforHelp); + if (localStorage.getItem('k8s_api_endpoint') != "") { + ctx.fillText('API Endpoint: ' + localStorage.getItem('k8s_api_endpoint'), 10, startYforHelp); + } + else if (endpoint != "") { + ctx.fillText('Cluster: ' + endpoint, 10, startYforHelp); + + } ctx.fillText('Current Namespace: ' + namespace, 10, startYforHelp + 20); ctx.fillText('Alien Shuffle: ' + shuffle, 10, startYforHelp + 40); ctx.fillText('Auto Namespaces Switch: ' + namespacesJumpStatus, 10, startYforHelp + 60); @@ -1018,7 +1024,10 @@ document.addEventListener("keyup", keyUpHandler, false); setSystemSettings(); waitForReachableK8sUrl(function () { - getEndpoint(); + if (endpoint != null && endpoint != "") { + console.log("[K-INV] Connected to Kubernetes API at " + endpoint); + getEndpoint(); + } getNamespaces(); getSavedPresets(); }); diff --git a/nginx/KubeInvaders.conf b/nginx/KubeInvaders.conf index 5157763..78b0986 100644 --- a/nginx/KubeInvaders.conf +++ b/nginx/KubeInvaders.conf @@ -217,10 +217,21 @@ server { ngx.header['Access-Control-Allow-Methods'] = 'GET, POST, OPTIONS' ngx.header['Access-Control-Allow-Headers'] = 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range' ngx.header['Access-Control-Expose-Headers'] = 'Content-Length,Content-Range'; - if os.getenv("APPLICATION_URL") == nil then - ngx.say("Error! APPLICATION_URL is nil!") + local endpoint = os.getenv("APPLICATION_URL") + + if not endpoint or endpoint == "" then + if os.getenv("KUBERNETES_SERVICE_HOST") and os.getenv("KUBERNETES_SERVICE_PORT_HTTPS") then + endpoint = "https://" .. os.getenv("KUBERNETES_SERVICE_HOST") .. ":" .. os.getenv("KUBERNETES_SERVICE_PORT_HTTPS") + else + endpoint = os.getenv("ENDPOINT") + end + end + + if not endpoint or endpoint == "" then + ngx.status = 500 + ngx.say("Error! No cluster endpoint configured (APPLICATION_URL, KUBERNETES_SERVICE_HOST, or ENDPOINT).") else - ngx.say(os.getenv("APPLICATION_URL")) + ngx.say(endpoint) end } } @@ -651,7 +662,7 @@ server { else red:set("chaos_report_project_list", args["project"]) end - red:expire("chaos_report_project_list", 5) + red:expire("chaos_report_project_list", 120) } } @@ -689,6 +700,30 @@ server { local okredis, errredis = red:connect("unix:/tmp/redis.sock") red:set("chaos_report_project_" .. args["project"], data) + local project = args["project"] or "" + if project ~= "" then + local check_url_counter_key = project .. "_check_url_counter" + local check_url_status_code_key = project .. "_check_url_status_code" + local check_url_elapsed_time_key = project .. "_check_url_elapsed_time" + local check_url_start_time_key = project .. "_check_url_start_time" + + if red:exists(check_url_counter_key) == 0 then + red:set(check_url_counter_key, 0) + end + + if red:exists(check_url_status_code_key) == 0 then + red:set(check_url_status_code_key, "Other") + end + + if red:exists(check_url_elapsed_time_key) == 0 then + red:set(check_url_elapsed_time_key, 0) + end + + if red:exists(check_url_start_time_key) == 0 then + red:set(check_url_start_time_key, os.date("%Y-%m-%d %H:%M:%S")) + end + end + if check_if_redis_key_exists(red, "chaos_report_project_list") then if not check_if_string_contain_project(red:get("chaos_report_project_list"), args["project"]) then red:set("chaos_report_project_list", add_project(red:get("chaos_report_project_list"), args["project"])) @@ -697,7 +732,7 @@ server { red:set("chaos_report_project_list", args["project"]) end - red:expire("chaos_report_project_list", 5) + red:expire("chaos_report_project_list", 120) -- ngx.log(ngx.INFO, "[CHAOS-REPORT-SAVE] " .. data) } diff --git a/scripts/metrics_loop/start.py b/scripts/metrics_loop/start.py index 37191af..923ec16 100644 --- a/scripts/metrics_loop/start.py +++ b/scripts/metrics_loop/start.py @@ -22,11 +22,22 @@ def join_string_to_array(project_list_string, separator): def do_http_request(url, method, headers, data): try: - response = requests.request(method, url, headers=headers, data=data, verify=False, allow_redirects=True, timeout=10) - elaped_time = response.elapsed.total_seconds() + # Do not inherit HTTP(S)_PROXY from the container environment; + # those proxies often cannot resolve in-cluster or local ingress hosts. + with requests.Session() as session: + session.trust_env = False + response = session.request( + method, + url, + headers=headers, + data=data, + verify=False, + allow_redirects=True, + timeout=10, + ) return response except requests.exceptions.RequestException as e: - logging.error(f"Error while sending HTTP request: {e}") + logging.error(f"Error while sending HTTP request to {url} with method {method}: {e}") return "Connection Error" def check_if_json_is_valid(json_data): @@ -75,7 +86,12 @@ def create_job(job_name, pod_template): r = redis.Redis(unix_socket_path='/tmp/redis.sock') -logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO")) +logging.basicConfig( + level=os.environ.get("LOGLEVEL", "INFO"), + stream=sys.stdout, + format="%(asctime)s %(levelname)s %(message)s", + force=True, +) logging.getLogger('kubernetes').setLevel(logging.ERROR) logging.debug('Starting script for KubeInvaders metrics loop') @@ -97,19 +113,45 @@ batch_api = client.BatchV1Api() while True: #logging.info(f"[k-inv][metrics_loop] Metrics loop is active - {r.exists('chaos_report_project_list')}") if r.exists('chaos_report_project_list'): - #logging.info("[k-inv][metrics_loop] Found chaos_report_project_list Redis Key") + logging.info("[k-inv][metrics_loop] Found chaos_report_project_list Redis Key") for project in join_string_to_array(r.get('chaos_report_project_list').decode(), ','): - #logging.info(f"[k-inv][metrics_loop] Computing Chaos Report Project: {project}") + logging.info(f"[k-inv][metrics_loop] Computing Chaos Report Project: {project}") chaos_program_key = f"chaos_report_project_{project}" if r.exists(chaos_program_key): - #logging.info(f"[k-inv][metrics_loop][chaos_report] Found chaos_report_project_{project} key in Redis. Starting {project} ") + logging.info(f"[k-inv][metrics_loop][chaos_report] Found chaos_report_project_{project} key in Redis. Starting {project} ") if check_if_json_is_valid(r.get(chaos_program_key)): chaos_report_program = json.loads(r.get(chaos_program_key)) now = datetime.datetime.now() - #logging.info(f"[k-inv][metrics_loop][chaos_report] chaos_report_program is valid JSON: {chaos_report_program}") - response = do_http_request(chaos_report_program['chaosReportCheckSiteURL'], chaos_report_program['chaosReportCheckSiteURLMethod'], json.loads(chaos_report_program['chaosReportCheckSiteURLHeaders']), chaos_report_program['chaosReportCheckSiteURLPayload']) + logging.info(f"[k-inv][metrics_loop][chaos_report] chaos_report_program is valid JSON: {chaos_report_program}") + url = str(chaos_report_program.get('chaosReportCheckSiteURL', '')).strip() + method = str(chaos_report_program.get('chaosReportCheckSiteURLMethod', 'GET')).strip().upper() + payload = chaos_report_program.get('chaosReportCheckSiteURLPayload', '') + + logging.info( + f"[k-inv][metrics_loop][chaos_report] project={project} parsed url='{url}' method='{method}'" + ) + + headers = {"Content-Type": "application/json; charset=utf-8"} + raw_headers = chaos_report_program.get('chaosReportCheckSiteURLHeaders', '{}') + try: + parsed_headers = json.loads(raw_headers) if isinstance(raw_headers, str) else raw_headers + if isinstance(parsed_headers, dict): + headers = parsed_headers + except Exception as e: + logging.warning(f"Invalid chaos report headers for project {project}: {e}") + + if method not in ["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"]: + method = "GET" + + if not url: + logging.warning( + f"[k-inv][metrics_loop][chaos_report] Empty chaosReportCheckSiteURL for project={project}. Raw program: {chaos_report_program}" + ) + response = "Connection Error" + else: + response = do_http_request(url, method, headers, payload) check_url_counter_key = f"{chaos_report_program['chaosReportProject']}_check_url_counter" check_url_status_code_key = f"{chaos_report_program['chaosReportProject']}_check_url_status_code" check_url_elapsed_time_key = f"{chaos_report_program['chaosReportProject']}_check_url_elapsed_time" @@ -124,11 +166,11 @@ while True: r.set(check_url_start_time, now.strftime("%Y-%m-%d %H:%M:%S")) if response == "Connection Error": - #logging.info(f"[k-inv][metrics_loop][chaos_report] Connection Error while checking {chaos_report_program['chaosReportCheckSiteURL']}") + logging.info(f"[k-inv][metrics_loop][chaos_report] Connection Error while checking {chaos_report_program['chaosReportCheckSiteURL']}") r.set(check_url_status_code_key, "Connection Error") r.set(check_url_elapsed_time_key, 0) else: - #logging.info(f"[k-inv][metrics_loop][chaos_report] Status code {response.status_code} while checking {chaos_report_program['chaosReportCheckSiteURL']}") + logging.info(f"[k-inv][metrics_loop][chaos_report] Status code {response.status_code} while checking {chaos_report_program['chaosReportCheckSiteURL']}") r.set(check_url_status_code_key, response.status_code) r.set(check_url_elapsed_time_key, float(response.elapsed.total_seconds())) try: diff --git a/scripts/metrics_loop/start.sh b/scripts/metrics_loop/start.sh index 40b0b2f..27477a4 100755 --- a/scripts/metrics_loop/start.sh +++ b/scripts/metrics_loop/start.sh @@ -8,11 +8,13 @@ else export TOKEN="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" fi +export PYTHONUNBUFFERED=1 + if python3 -c "import urllib.request; urllib.request.urlopen('https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT_HTTPS}', timeout=5)" 2>/dev/null; then - python3 /opt/metrics_loop/start.py https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT_HTTPS} & + python3 -u /opt/metrics_loop/start.py https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT_HTTPS} & while true do - pgrep -a -f -c "^python3.*metrics_loop.*$" > /dev/null || ( python3 /opt/metrics_loop/start.py https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT_HTTPS} & ) + pgrep -a -f -c "^python3.*metrics_loop.*$" > /dev/null || ( python3 -u /opt/metrics_loop/start.py https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT_HTTPS} & ) sleep 2 done fi