From 2fba2bcb8a1a7a2fc87cde3d6c3f18fdc5456ac2 Mon Sep 17 00:00:00 2001 From: luckysideburn Date: Mon, 12 Aug 2024 12:49:57 +0000 Subject: [PATCH] fixed wrong Prometheus metrics --- html5/kubeinvaders.js | 20 +++++++++++++++++++- html5/programming_mode.js | 8 +++----- nginx/KubeInvaders.conf | 12 ++++++++++++ scripts/metrics_loop/start.py | 6 +++++- scripts/programming_mode/start.py | 2 +- 5 files changed, 40 insertions(+), 8 deletions(-) diff --git a/html5/kubeinvaders.js b/html5/kubeinvaders.js index e630e06..ba88cae 100644 --- a/html5/kubeinvaders.js +++ b/html5/kubeinvaders.js @@ -98,7 +98,7 @@ var latest_preset_name = ""; var latest_preset_lang = ""; var codename = getCodeName(); const codename_regex = /chaos-codename:\ [a-zA-Z_]*/g; -const chaos_job_regex = /chaos_jobs_status.*/g; +const chaos_job_regex = /chaos_jobs_pod_phase.*/g; var codename_configured = false; var chaos_jobs_status = new Map(); var current_color_mode = "light"; @@ -203,6 +203,23 @@ function getMetrics() { oReq.send(); } +function getChaosJobsPodsPhase() { + var oReq = new XMLHttpRequest(); + oReq.onload = function () { + var lines = this.responseText.split('\n'); + for (var i = 0;i < lines.length;i++){ + metric = lines[i].split(' '); + + if (metric[0].match(chaos_job_regex)) { + metrics_split = metric[0].split(":"); + chaos_jobs_status.set(metrics_split[1] + ":" + metrics_split[2] + ":" + metrics_split[3], metric[1]); + } + } + };; + oReq.open("GET", k8s_url + "/chaos_jobs_pod_phase"); + oReq.send(); +} + function scroll_backwards() { if (chaos_logs_pos > 0){ chaos_logs_pos = chaos_logs_pos -1; @@ -859,6 +876,7 @@ window.setInterval(function backgroundTasks() { if (game_mode_switch || programming_mode_switch || log_tail_switch) { getMetrics(); + getChaosJobsPodsPhase(); updateMainMetricsChart(); } diff --git a/html5/programming_mode.js b/html5/programming_mode.js index 6f4d279..877285a 100644 --- a/html5/programming_mode.js +++ b/html5/programming_mode.js @@ -129,7 +129,7 @@ function savePreset(action) { oReq.onreadystatechange = function () { if (this.readyState === XMLHttpRequest.DONE && this.status === 200 && (action == "apply" || action == "save-chaos-program")) { if (latest_preset_lang == "k-inv") { - console.log("[SAVE-PRESET-CHAOSPROGRAM] Payload: " + $('#currentLoadTest').val()); + // console.log("[SAVE-PRESET-CHAOSPROGRAM] Payload: " + $('#currentLoadTest').val()); if ($('#currentLoadTest').val() != "") { presetBody = $('#currentLoadTest').val(); } @@ -210,7 +210,6 @@ function drawChaosProgramFlow() { else { times = flow["experiments"][i]["loop"] + " times" } - console.log(flow_html); if (current_color_mode == "light") { flow_html = flow_html + '
'; } @@ -221,7 +220,6 @@ function drawChaosProgramFlow() { flow_html = flow_html + ''; - //console.log("Search " + search_job); for (let [key, value] of chaos_jobs_status) { if (key.search(search_job) != -1 ) { if (current_color_mode == "light") { @@ -250,7 +248,7 @@ function getSavedPresets() { oReq.onreadystatechange = function () { if (this.readyState === XMLHttpRequest.DONE && this.status === 200) { if ((this.responseText.trim() != "nil") && (this.responseText.trim() != "")) { - console.log("[GET-PRESETS] Response from backend: <" + this.responseText.trim() + ">"); + // console.log("[GET-PRESETS] Response from backend: <" + this.responseText.trim() + ">"); var savedPresets = this.responseText.split(","); for (i = 0; i < savedPresets.length; i++) { var currentPresetName = savedPresets[i].split("_")[1]; @@ -260,7 +258,7 @@ function getSavedPresets() { // console.log("[GET-PRESETS] Change border color of buttonId: " + buttonId); // console.log(document.getElementById(buttonId)); if (document.getElementById(buttonId) == null){ - console.log("[GET-PRESETS] Appending button to loadButtonGroup. id: " + buttonId + " presetname: " + currentPresetName.trim()); + // console.log("[GET-PRESETS] Appending button to loadButtonGroup. id: " + buttonId + " presetname: " + currentPresetName.trim()); latest_preset_lang = "k-inv"; createChaosProgramButton(currentPresetName.trim(), latest_preset_lang); } else { diff --git a/nginx/KubeInvaders.conf b/nginx/KubeInvaders.conf index 4d4b22c..b68f7f6 100644 --- a/nginx/KubeInvaders.conf +++ b/nginx/KubeInvaders.conf @@ -194,6 +194,18 @@ server { } } + location /chaos_jobs_pod_phase { + default_type text/html; + content_by_lua_block { + local redis = require "resty.redis" + local red = redis:new() + local okredis, errredis = red:connect("unix:/tmp/redis.sock") + for i, res in ipairs(red:keys("chaos_jobs_pod_phase*")) do + ngx.say(res .. " " .. red:get(res)) + end + } + } + location /kube/chaos/containers { lua_need_request_body 'on'; access_by_lua_file "/usr/local/openresty/nginx/conf/kubeinvaders/chaos-containers.lua"; diff --git a/scripts/metrics_loop/start.py b/scripts/metrics_loop/start.py index ec78d0b..26dac34 100644 --- a/scripts/metrics_loop/start.py +++ b/scripts/metrics_loop/start.py @@ -163,5 +163,9 @@ while True: codename = pod.metadata.labels.get('chaos-codename') job_name = pod.metadata.labels.get('job-name') exp_name = pod.metadata.labels.get('experiment-name') - r.set(f"chaos_jobs_status:{codename}:{exp_name}:{job_name}", pod.status.phase) + if pod.status.phase in ["Pending", "Running", "Succeeded"]: + r.set(f"chaos_jobs_status:{codename}:{exp_name}:{job_name}", 1.0) + else: + r.set(f"chaos_jobs_status:{codename}:{exp_name}:{job_name}", -1) + r.set(f"chaos_jobs_pod_phase:{codename}:{exp_name}:{job_name}", pod.status.phase) time.sleep(1) diff --git a/scripts/programming_mode/start.py b/scripts/programming_mode/start.py index 2fc9dd9..fcea750 100644 --- a/scripts/programming_mode/start.py +++ b/scripts/programming_mode/start.py @@ -119,7 +119,7 @@ for exp in parsed_yaml["experiments"]: if 'additional-labels' in job_attrs and 'chaos-codename' in job_attrs['additional-labels']: codename = job_attrs['additional-labels']['chaos-codename'] - r.set(f"chaos_jobs_status:{codename}:{exp['name']}:{exp['job']}", "Created") + r.set(f"chaos_jobs_status:{codename}:{exp['name']}:{exp['job']}", 0.0) if r.exists('chaos_node_jobs_total') == 1: r.incr('chaos_node_jobs_total')