From fe1f19e06374c9848c56490da9cbf794cd7d5819 Mon Sep 17 00:00:00 2001 From: luckysideburn Date: Sat, 2 Mar 2024 02:11:36 +0000 Subject: [PATCH] feature/resilience_namespace_report --- Dockerfile | 1 + html5/chaos_report.js | 107 ++++++---------------------------- nginx/KubeInvaders.conf | 90 ++++++++++++++++++++++------ scripts/metrics_loop/start.py | 28 +++++---- 4 files changed, 105 insertions(+), 121 deletions(-) diff --git a/Dockerfile b/Dockerfile index f4c2667..6d7b9b4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,5 @@ FROM nginx:stable +# FROM localhost/local/base:latest # Update repo and install some utilities and prerequisites RUN apt-get update -y diff --git a/html5/chaos_report.js b/html5/chaos_report.js index e9d78fc..52a5f7a 100644 --- a/html5/chaos_report.js +++ b/html5/chaos_report.js @@ -1,7 +1,7 @@ function diffBetweenTwoDates(date1, date2) { - console.log("[SAVE-CHAOS-REPORT-CONF] Diff between two dates: " + date1 + " and " + date2); + // console.log("[SAVE-CHAOS-REPORT-CONF] Diff between two dates: " + date1 + " and " + date2); var diff = (date2.getTime() - date1.getTime()) / 1000; - console.log("[SAVE-CHAOS-REPORT-CONF] Diff between two dates: " + diff + " seconds") + // console.log("[SAVE-CHAOS-REPORT-CONF] Diff between two dates: " + diff + " seconds") return diff; } @@ -140,12 +140,12 @@ function sendSavedChaosReport() { oReq.onreadystatechange = function () { if (this.readyState === XMLHttpRequest.DONE && this.status === 200) { - console.log("[SAVE-CHAOS-REPORT-CONF] Configuration sent to Nginx") + // console.log("[SAVE-CHAOS-REPORT-CONF] Configuration sent to Nginx") } };; oReq.setRequestHeader("Content-Type", "application/json"); - console.log("[SAVE-CHAOS-REPORT-CONF] Sending configuration to Nginx: " + JSON.stringify(presetBodyDict)); + // console.log("[SAVE-CHAOS-REPORT-CONF] Sending configuration to Nginx: " + JSON.stringify(presetBodyDict)); oReq.send(JSON.stringify(presetBodyDict)); closePrepareChaosReportModal(); @@ -163,11 +163,11 @@ function sendSavedChaosReport() { } function readContentOfUploadedFile() { - console.log("[SAVE-CHAOS-REPORT-CONF] Reading content of uploaded file"); + // console.log("[SAVE-CHAOS-REPORT-CONF] Reading content of uploaded file"); var file = document.getElementById("formFile").files[0]; var reader = new FileReader(); reader.onload = function (e) { - //console.log("[SAVE-CHAOS-REPORT-CONF] Content of uploaded file: " + e.target.result); + // console.log("[SAVE-CHAOS-REPORT-CONF] Content of uploaded file: " + e.target.result); chaos_report_post_data = e.target.result; sendSavedChaosReport() }; @@ -175,28 +175,28 @@ function readContentOfUploadedFile() { } function saveChaosReport() { - console.log("[SAVE-CHAOS-REPORT-CONF] Going to save Chaos Report program"); + // console.log("[SAVE-CHAOS-REPORT-CONF] Going to save Chaos Report program"); if ($("#chaosReportCheckSiteURLMethod").val() == "POST") { - console.log("[SAVE-CHAOS-REPORT-CONF] Check url is POST, reading content of uploaded file and sending saved Chaos Report"); + // console.log("[SAVE-CHAOS-REPORT-CONF] Check url is POST, reading content of uploaded file and sending saved Chaos Report"); readContentOfUploadedFile(); } else { - console.log("[SAVE-CHAOS-REPORT-CONF] Check url is GET, sending saved Chaos Report"); + // console.log("[SAVE-CHAOS-REPORT-CONF] Check url is GET, sending saved Chaos Report"); sendSavedChaosReport(); } } function updateElapsedTimeArray(projectName) { $("#chaosReportSessionTimeDiv").html(diffBetweenTwoDates(chaos_report_start_date, new Date()) + " seconds"); - console.log("[SAVE-CHAOS-REPORT-CONF] Diff Between Dates: " + toString(diffBetweenTwoDates(chaos_report_start_date, new Date()))); - console.log("[SAVE-CHAOS-REPORT-CONF] Updating elapsed time array for project: " + projectName); + // console.log("[SAVE-CHAOS-REPORT-CONF] Diff Between Dates: " + toString(diffBetweenTwoDates(chaos_report_start_date, new Date()))); + // console.log("[SAVE-CHAOS-REPORT-CONF] Updating elapsed time array for project: " + projectName); var oReq = new XMLHttpRequest(); var redis_key = projectName + "_check_url_elapsed_time"; - console.log("[SAVE-CHAOS-REPORT-CONF] Redis key: " + redis_key); + // console.log("[SAVE-CHAOS-REPORT-CONF] Redis key: " + redis_key); oReq.onreadystatechange = function () { if (this.readyState === XMLHttpRequest.DONE && this.status === 200) { - console.log("[SAVE-CHAOS-REPORT-CONF] Elapsed time array received from Redis: " + parseFloat(this.responseText)); + // console.log("[SAVE-CHAOS-REPORT-CONF] Elapsed time array received from Redis: " + parseFloat(this.responseText)); chaos_report_http_elapsed_time_array.push(parseFloat(this.responseText)); while (chaos_report_http_elapsed_time_array.length > 40) { chaos_report_http_elapsed_time_array.shift(); @@ -211,16 +211,16 @@ function updateElapsedTimeArray(projectName) { } function updateStatusCodePieChart(projectName) { - console.log("[SAVE-CHAOS-REPORT-CONF] Updating Status Code Pie Chart for project: " + projectName); + // console.log("[SAVE-CHAOS-REPORT-CONF] Updating Status Code Pie Chart for project: " + projectName); var oReq = new XMLHttpRequest(); var redis_key = projectName + "_check_url_status_code"; - console.log("[SAVE-CHAOS-REPORT-CONF] Redis key: " + redis_key); + // console.log("[SAVE-CHAOS-REPORT-CONF] Redis key: " + redis_key); oReq.onreadystatechange = function () { if (this.readyState === XMLHttpRequest.DONE && this.status === 200) { var status_code = this.responseText.trim(); - console.log("[SAVE-CHAOS-REPORT-CONF] Status Code Pie Chart received from Redis: |" + status_code + "|"); + // console.log("[SAVE-CHAOS-REPORT-CONF] Status Code Pie Chart received from Redis: |" + status_code + "|"); chart_status_code_dict[status_code] = chart_status_code_dict[status_code] + 1 @@ -298,11 +298,11 @@ function updateStatusCodePieChart(projectName) { } function updateChaosReportStartTime(projectName) { - console.log("[SAVE-CHAOS-REPORT-CONF] Updating Start Time for project: " + projectName); + // console.log("[SAVE-CHAOS-REPORT-CONF] Updating Start Time for project: " + projectName); var oReq = new XMLHttpRequest(); var redis_key = projectName + "_check_url_start_time"; - console.log("[SAVE-CHAOS-REPORT-CONF] Redis key: " + redis_key); + // console.log("[SAVE-CHAOS-REPORT-CONF] Redis key: " + redis_key); oReq.onreadystatechange = function () { if (this.readyState === XMLHttpRequest.DONE && this.status === 200) { @@ -317,7 +317,7 @@ function updateChaosReportStartTime(projectName) { function drawCanvasHTTPStatusCodeStats() { - console.log("[SAVE-CHAOS-REPORT-CONF] Updating chart"); + // // console.log("[SAVE-CHAOS-REPORT-CONF] Updating chart"); myHTTPElapsedChart.setOption({ xAxis: {}, @@ -333,14 +333,6 @@ function drawCanvasHTTPStatusCodeStats() { ] }); - // console.log("[SAVE-CHAOS-REPORT-CONF] Updating metrics"); - // console.log("[SAVE-CHAOS-REPORT-CONF] Deleted Pods: " + chart_deleted_pods_total); - // console.log("[SAVE-CHAOS-REPORT-CONF] Chaos Jobs: " + chart_chaos_jobs_total); - // console.log("[SAVE-CHAOS-REPORT-CONF] Current Chaos Pods: " + chart_current_chaos_job_pod); - // console.log("[SAVE-CHAOS-REPORT-CONF] Not Running Pods: " + chart_pods_not_running_on); - // console.log("[SAVE-CHAOS-REPORT-CONF] Current Replicas State Delay: " + chart_fewer_replicas_seconds); - // console.log("[SAVE-CHAOS-REPORT-CONF] Latest Replicas State Delay: " + chart_latest_fewer_replicas_seconds); - myMainChaosMetrics.setOption({ series: [ { @@ -376,67 +368,6 @@ function drawCanvasHTTPStatusCodeStats() { ] }); -// while (chaos_report_http_elapsed_time_array.length > 40) { -// chaos_report_http_elapsed_time_array.shift(); -// } - -// var elapsedTimeArray = chaos_report_http_elapsed_time_array; -// var canvas = document.getElementById('httpStatsCanvas'); -// var context = canvas.getContext('2d'); -// context.clearRect(0, 0, canvas.width, canvas.height); - -// // Imposta la dimensione dei rettangoli e il numero di colonne e righe -// var rectSize = 20; -// var rows = canvas.height / rectSize; -// var columns = canvas.width / rectSize; - -// // Disegna la griglia di rettangoli -// for (var i = 0; i < rows; i++) { -// for (var j = 0; j < columns; j++) { -// // Calcola le coordinate del rettangolo -// var x = j * rectSize; -// var y = i * rectSize; - -// context.strokeStyle = 'black'; -// context.fillStyle = 'white'; - -// // Disegna il rettangolo -// context.fillRect(x, y, rectSize, rectSize); - -// // Aggiungi bordi per una migliore visualizzazione -// //context.strokeRect(x, y, rectSize, rectSize); -// } -// } - -// for (var i = 0; i < elapsedTimeArray.length; i++) { -// var x = i * rectSize; -// var y = 80; - -// var width = rectSize; -// var height = rectSize; - -// if (elapsedTimeArray[i] > 3) { -// context.strokeStyle = 'black'; -// context.fillStyle = 'red'; -// } else if (elapsedTimeArray[i] > 2) { -// context.strokeStyle = 'black'; -// context.fillStyle = 'orange'; -// } else { -// context.strokeStyle = 'black'; -// context.fillStyle = "green"; -// } - -// if (i == elapsedTimeArray.length - 1) { -// var text = String(elapsedTimeArray[i]); -// context.fillStyle = 'black'; -// context.font = '20px Courier New'; -// context.fillText(text, x + width / 2 - context.measureText(text).width / 2, y - (height / 2)); -// } - -// context.fillRect(x, y, width, height); - -// //context.strokeRect(x, y, 20, 20); -// } } var myHTTPElapsedChart = echarts.init(document.getElementById('httpElapsedChart')); diff --git a/nginx/KubeInvaders.conf b/nginx/KubeInvaders.conf index 3d29eda..7546fa2 100644 --- a/nginx/KubeInvaders.conf +++ b/nginx/KubeInvaders.conf @@ -123,7 +123,7 @@ server { red:set("latest_codename", random_word) - ngx.log(ngx.INFO, "[programming_mode_codename] Choosing random word " .. random_word) + -- ngx.log(ngx.INFO, "[programming_mode_codename] Choosing random word " .. random_word) ngx.say(random_word) } @@ -308,7 +308,7 @@ server { ngx.req.read_body() local okredis, errredis = red:connect("unix:/tmp/redis.sock") local key_name = args["lang"] .. "_" .. args["name"] - ngx.log(ngx.INFO, "[PRESETS-RESET] The item " .. key_name .. " should be removed from the key presets_list") + -- ngx.log(ngx.INFO, "[PRESETS-RESET] The item " .. key_name .. " should be removed from the key presets_list") local handle = io.popen("redis-cli --scan --pattern " .. key_name .. " | xargs redis-cli del") local result = handle:read("*a") @@ -323,7 +323,7 @@ server { local new_preset_list = ""; local cnt = 0; for key, value in ipairs(mysplit(res, ",")) do - ngx.log(ngx.INFO, "[PRESETS-RESET] Split the key presets_list. Current preset is: " .. value) + -- ngx.log(ngx.INFO, "[PRESETS-RESET] Split the key presets_list. Current preset is: " .. value) if value ~= key_name then preset = value if cnt > 0 then @@ -334,7 +334,7 @@ server { end cnt = cnt + 1 end - ngx.log(ngx.INFO, "[PRESETS-RESET] New key presets_list is: " .. new_preset_list) + -- ngx.log(ngx.INFO, "[PRESETS-RESET] New key presets_list is: " .. new_preset_list) red:set("presets_list", new_preset_list) end } @@ -358,14 +358,64 @@ server { ngx.say(err) else ngx.say(res) - ngx.log(ngx.INFO, "[PRESETS_LIST] " .. res) + -- ngx.log(ngx.INFO, "[PRESETS_LIST] " .. res) end } } + location /chaos/report/keepalive { + content_by_lua_block { + + local function check_if_string_contain_project(project_list, project) + if string.find(project_list, project) then + return true + end + return false + end + + local function add_project(project_list, new_project) + local new_project_list = project_list .. "," .. new_project + return new_project_list + end + + local function check_if_redis_key_exists(redis, key) + if redis:exists(key) == 1 then + return true + end + return false + end + + local redis = require "resty.redis" + local red = redis:new() + local args = ngx.req.get_uri_args() + ngx.header['Access-Control-Allow-Origin'] = '*' + ngx.header['Access-Control-Allow-Methods'] = 'GET, POST, OPTIONS' + ngx.header['Access-Control-Allow-Headers'] = 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range' + ngx.header['Access-Control-Expose-Headers'] = 'Content-Length,Content-Range'; + ngx.req.read_body() + local data = ngx.req.get_body_data() + local okredis, errredis = red:connect("unix:/tmp/redis.sock") + if check_if_redis_key_exists(red, "chaos_report_project_list") then + if not check_if_string_contain_project(red:get("chaos_report_project_list"), args["project"]) then + red:set("chaos_report_project_list", add_project(red:get("chaos_report_project_list"), args["project"])) + end + else + red:set("chaos_report_project_list", args["project"]) + end + red:expire("chaos_report_project_list", 5) + } + } + location /chaos/report/save { content_by_lua_block { + local function check_if_string_contain_project(project_list, project) + if string.find(project_list, project) then + return true + end + return false + end + local function add_project(project_list, new_project) local new_project_list = project_list .. "," .. new_project return new_project_list @@ -391,12 +441,16 @@ server { red:set("chaos_report_project_" .. args["project"], data) if check_if_redis_key_exists(red, "chaos_report_project_list") then - red:set("chaos_report_project_list", add_project(red:get("chaos_report_project_list"), args["project"])) + if not check_if_string_contain_project(red:get("chaos_report_project_list"), args["project"]) then + red:set("chaos_report_project_list", add_project(red:get("chaos_report_project_list"), args["project"])) + end else red:set("chaos_report_project_list", args["project"]) end + + red:expire("chaos_report_project_list", 5) - ngx.log(ngx.INFO, "[CHAOS-REPORT-SAVE] " .. data) + -- ngx.log(ngx.INFO, "[CHAOS-REPORT-SAVE] " .. data) } } @@ -416,10 +470,10 @@ server { local all_presets_key = "" if data ~= nil then - ngx.log(ngx.INFO, "[SAVE-PRESETS] lang:" .. args["lang"] .. ", name:" .. args["name"] .. ", payload:" .. data .. "\n key_name:" .. key_name .. "\n key_exists: " .. red:exists(key_name)) + -- ngx.log(ngx.INFO, "[SAVE-PRESETS] lang:" .. args["lang"] .. ", name:" .. args["name"] .. ", payload:" .. data .. "\n key_name:" .. key_name .. "\n key_exists: " .. red:exists(key_name)) -- if (red:exists(key_name) == 0) then red:set(key_name, data) - ngx.log(ngx.INFO, "[SAVE-PRESETS] key_name:" .. key_name .. ", data:" .. data) + -- ngx.log(ngx.INFO, "[SAVE-PRESETS] key_name:" .. key_name .. ", data:" .. data) local res, err = red:get(key_name) if res == ngx.null then @@ -481,13 +535,13 @@ server { ngx.header['Access-Control-Expose-Headers'] = 'Content-Length,Content-Range'; ngx.req.read_body() local data = ngx.req.get_body_data() - -- ngx.log(ngx.INFO, "[PROGRAMMING-MODE-DIAGRAM] data sent from web interface => " .. data) + -- -- ngx.log(ngx.INFO, "[PROGRAMMING-MODE-DIAGRAM] data sent from web interface => " .. data) math.randomseed(os.clock()*100000000000) local rand = math.random(999, 9999) local filename = "/tmp/chaosprogram" .. rand - ngx.log(ngx.INFO, "[PROGRAMMING-MODE-DIAGRAM] write temp file " .. filename) + -- ngx.log(ngx.INFO, "[PROGRAMMING-MODE-DIAGRAM] write temp file " .. filename) local yamlfile = io.open(filename, "w") yamlfile:write(data) @@ -497,16 +551,16 @@ server { local result = handle:read("*a") local rc = handle:close() - ngx.log(ngx.INFO, "[programming_mode_diagram] Result of yaml syntax check =>||" .. result .. "||") + -- ngx.log(ngx.INFO, "[programming_mode_diagram] Result of yaml syntax check =>||" .. result .. "||") if data == nil then error = "[PROGRAMMING-MODE-DIAGRAM] No chaos program already loaded." - ngx.log(ngx.INFO, error) + -- ngx.log(ngx.INFO, error) ngx.say(error) elseif not string.match(result, '{.*}') then error = "[PROGRAMMING-MODE-DIAGRAM] Chaos program seems not correct. Please check yaml syntax." - ngx.log(ngx.INFO, error) + -- ngx.log(ngx.INFO, error) ngx.say(error) end @@ -514,22 +568,22 @@ server { if not key_exists(yaml_data, "jobs") then error = "[PROGRAMMING-MODE-DIAGRAM] Chaos program does not contain 'jobs' key. Please fix yaml definition." - ngx.log(ngx.INFO, error) + -- ngx.log(ngx.INFO, error) ngx.say(error) elseif not key_exists(yaml_data, "experiments") then error = "[PROGRAMMING-MODE-DIAGRAM] Chaos program does not contain 'experiments' key. Please fix yaml definition" - ngx.log(ngx.INFO, error) + -- ngx.log(ngx.INFO, error) ngx.say(error) elseif is_key_empty(yaml_data, "jobs") then error = "[PROGRAMMING-MODE-DIAGRAM] Chaos program does not contain valid 'jobs' key. Please fix yaml definition, add jobs" - ngx.log(ngx.INFO, error) + -- ngx.log(ngx.INFO, error) ngx.say(error) elseif is_key_empty(yaml_data, "experiments") then error = "[PROGRAMMING-MODE-DIAGRAM] Chaos program does not contain valid 'experiments' key. Please fix yaml definition, add experiments" - ngx.log(ngx.INFO, error) + -- ngx.log(ngx.INFO, error) ngx.say(error) else diff --git a/scripts/metrics_loop/start.py b/scripts/metrics_loop/start.py index b6727f8..ec78d0b 100644 --- a/scripts/metrics_loop/start.py +++ b/scripts/metrics_loop/start.py @@ -17,7 +17,6 @@ import time import datetime urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - def join_string_to_array(project_list_string, separator): return project_list_string.split(separator) @@ -28,7 +27,7 @@ def do_http_request(url, method, headers, data): return response except requests.exceptions.RequestException as e: logging.error(f"Error while sending HTTP request: {e}") - return None + return "Connection Error" def check_if_json_is_valid(json_data): try: @@ -96,26 +95,21 @@ api_instance = client.CoreV1Api() batch_api = client.BatchV1Api() while True: - - logging.info(f"[k-inv][metrics_loop] Metrics loop is active - {r.exists('chaos_report_project_list')}") - + #logging.info(f"[k-inv][metrics_loop] Metrics loop is active - {r.exists('chaos_report_project_list')}") if r.exists('chaos_report_project_list'): - logging.info("[k-inv][metrics_loop] Found chaos_report_project_list Redis Key") + #logging.info("[k-inv][metrics_loop] Found chaos_report_project_list Redis Key") for project in join_string_to_array(r.get('chaos_report_project_list').decode(), ','): - logging.info(f"[k-inv][metrics_loop] Computing Chaos Report Project: {project}") + #logging.info(f"[k-inv][metrics_loop] Computing Chaos Report Project: {project}") chaos_program_key = f"chaos_report_project_{project}" if r.exists(chaos_program_key): - logging.info(f"[k-inv][metrics_loop][chaos_report] Found chaos_report_project_{project} key in Redis. Starting {project} ") + #logging.info(f"[k-inv][metrics_loop][chaos_report] Found chaos_report_project_{project} key in Redis. Starting {project} ") if check_if_json_is_valid(r.get(chaos_program_key)): chaos_report_program = json.loads(r.get(chaos_program_key)) now = datetime.datetime.now() - logging.info(f"[k-inv][metrics_loop][chaos_report] chaos_report_program is valid JSON: {chaos_report_program}") + #logging.info(f"[k-inv][metrics_loop][chaos_report] chaos_report_program is valid JSON: {chaos_report_program}") response = do_http_request(chaos_report_program['chaosReportCheckSiteURL'], chaos_report_program['chaosReportCheckSiteURLMethod'], json.loads(chaos_report_program['chaosReportCheckSiteURLHeaders']), chaos_report_program['chaosReportCheckSiteURLPayload']) - logging.info(f"[k-inv][metrics_loop][chaos_report] chaos_report_program response: {response.status_code}") - - check_url_counter_key = f"{chaos_report_program['chaosReportProject']}_check_url_counter" check_url_status_code_key = f"{chaos_report_program['chaosReportProject']}_check_url_status_code" check_url_elapsed_time_key = f"{chaos_report_program['chaosReportProject']}_check_url_elapsed_time" @@ -129,9 +123,13 @@ while True: if r.get(check_url_start_time) == None: r.set(check_url_start_time, now.strftime("%Y-%m-%d %H:%M:%S")) - r.set(check_url_status_code_key, response.status_code) - r.set(check_url_elapsed_time_key, float(response.elapsed.total_seconds())) - + if response == "Connection Error": + #logging.info(f"[k-inv][metrics_loop][chaos_report] Connection Error while checking {chaos_report_program['chaosReportCheckSiteURL']}") + r.set(check_url_status_code_key, "Connection Error") + r.set(check_url_elapsed_time_key, 0) + else: + r.set(check_url_status_code_key, response.status_code) + r.set(check_url_elapsed_time_key, float(response.elapsed.total_seconds())) try: label_selector="chaos-controller=kubeinvaders" api_response = api_instance.list_pod_for_all_namespaces(label_selector=label_selector)