github/kubeinvaders
1
0
Fork 0
mirror of https://github.com/lucky-sideburn/kubeinvaders.git synced 2026-08-23 21:46:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

feature/resilience_namespace_report

Browse Source
This commit is contained in:
luckysideburn
2024-03-02 02:11:36 +00:00
parent afcbbc85cd
commit fe1f19e063
4 changed files with 105 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Dockerfile
+1
View File
@@ -1,4 +1,5 @@
FROM nginx:stable
# FROM localhost/local/base:latest
# Update repo and install some utilities and prerequisites
RUN apt-get update -y
html5/chaos_report.js
+19 -88
View File
@@ -1,7 +1,7 @@
function diffBetweenTwoDates(date1, date2) {
console.log("[SAVE-CHAOS-REPORT-CONF] Diff between two dates: " + date1 + " and " + date2);
// console.log("[SAVE-CHAOS-REPORT-CONF] Diff between two dates: " + date1 + " and " + date2);
var diff = (date2.getTime() - date1.getTime()) / 1000;
console.log("[SAVE-CHAOS-REPORT-CONF] Diff between two dates: " + diff + " seconds")
// console.log("[SAVE-CHAOS-REPORT-CONF] Diff between two dates: " + diff + " seconds")
return diff;
}
@@ -140,12 +140,12 @@ function sendSavedChaosReport() {
oReq.onreadystatechange = function () {
if (this.readyState === XMLHttpRequest.DONE && this.status === 200) {
console.log("[SAVE-CHAOS-REPORT-CONF] Configuration sent to Nginx")
// console.log("[SAVE-CHAOS-REPORT-CONF] Configuration sent to Nginx")
}
};;
oReq.setRequestHeader("Content-Type", "application/json");
console.log("[SAVE-CHAOS-REPORT-CONF] Sending configuration to Nginx: " + JSON.stringify(presetBodyDict));
// console.log("[SAVE-CHAOS-REPORT-CONF] Sending configuration to Nginx: " + JSON.stringify(presetBodyDict));
oReq.send(JSON.stringify(presetBodyDict));
closePrepareChaosReportModal();
@@ -163,11 +163,11 @@ function sendSavedChaosReport() {
}
function readContentOfUploadedFile() {
console.log("[SAVE-CHAOS-REPORT-CONF] Reading content of uploaded file");
// console.log("[SAVE-CHAOS-REPORT-CONF] Reading content of uploaded file");
var file = document.getElementById("formFile").files[0];
var reader = new FileReader();
reader.onload = function (e) {
//console.log("[SAVE-CHAOS-REPORT-CONF] Content of uploaded file: " + e.target.result);
// console.log("[SAVE-CHAOS-REPORT-CONF] Content of uploaded file: " + e.target.result);
chaos_report_post_data = e.target.result;
sendSavedChaosReport()
};
@@ -175,28 +175,28 @@ function readContentOfUploadedFile() {
}
function saveChaosReport() {
console.log("[SAVE-CHAOS-REPORT-CONF] Going to save Chaos Report program");
// console.log("[SAVE-CHAOS-REPORT-CONF] Going to save Chaos Report program");
if ($("#chaosReportCheckSiteURLMethod").val() == "POST") {
console.log("[SAVE-CHAOS-REPORT-CONF] Check url is POST, reading content of uploaded file and sending saved Chaos Report");
// console.log("[SAVE-CHAOS-REPORT-CONF] Check url is POST, reading content of uploaded file and sending saved Chaos Report");
readContentOfUploadedFile();
} else {
console.log("[SAVE-CHAOS-REPORT-CONF] Check url is GET, sending saved Chaos Report");
// console.log("[SAVE-CHAOS-REPORT-CONF] Check url is GET, sending saved Chaos Report");
sendSavedChaosReport();
}
}
function updateElapsedTimeArray(projectName) {
$("#chaosReportSessionTimeDiv").html(diffBetweenTwoDates(chaos_report_start_date, new Date()) + " seconds");
console.log("[SAVE-CHAOS-REPORT-CONF] Diff Between Dates: " + toString(diffBetweenTwoDates(chaos_report_start_date, new Date())));
console.log("[SAVE-CHAOS-REPORT-CONF] Updating elapsed time array for project: " + projectName);
// console.log("[SAVE-CHAOS-REPORT-CONF] Diff Between Dates: " + toString(diffBetweenTwoDates(chaos_report_start_date, new Date())));
// console.log("[SAVE-CHAOS-REPORT-CONF] Updating elapsed time array for project: " + projectName);
var oReq = new XMLHttpRequest();
var redis_key = projectName + "_check_url_elapsed_time";
console.log("[SAVE-CHAOS-REPORT-CONF] Redis key: " + redis_key);
// console.log("[SAVE-CHAOS-REPORT-CONF] Redis key: " + redis_key);
oReq.onreadystatechange = function () {
if (this.readyState === XMLHttpRequest.DONE && this.status === 200) {
console.log("[SAVE-CHAOS-REPORT-CONF] Elapsed time array received from Redis: " + parseFloat(this.responseText));
// console.log("[SAVE-CHAOS-REPORT-CONF] Elapsed time array received from Redis: " + parseFloat(this.responseText));
chaos_report_http_elapsed_time_array.push(parseFloat(this.responseText));
while (chaos_report_http_elapsed_time_array.length > 40) {
chaos_report_http_elapsed_time_array.shift();
@@ -211,16 +211,16 @@ function updateElapsedTimeArray(projectName) {
}
function updateStatusCodePieChart(projectName) {
console.log("[SAVE-CHAOS-REPORT-CONF] Updating Status Code Pie Chart for project: " + projectName);
// console.log("[SAVE-CHAOS-REPORT-CONF] Updating Status Code Pie Chart for project: " + projectName);
var oReq = new XMLHttpRequest();
var redis_key = projectName + "_check_url_status_code";
console.log("[SAVE-CHAOS-REPORT-CONF] Redis key: " + redis_key);
// console.log("[SAVE-CHAOS-REPORT-CONF] Redis key: " + redis_key);
oReq.onreadystatechange = function () {
if (this.readyState === XMLHttpRequest.DONE && this.status === 200) {
var status_code = this.responseText.trim();
console.log("[SAVE-CHAOS-REPORT-CONF] Status Code Pie Chart received from Redis: |" + status_code + "|");
// console.log("[SAVE-CHAOS-REPORT-CONF] Status Code Pie Chart received from Redis: |" + status_code + "|");
chart_status_code_dict[status_code] = chart_status_code_dict[status_code] + 1
@@ -298,11 +298,11 @@ function updateStatusCodePieChart(projectName) {
}
function updateChaosReportStartTime(projectName) {
console.log("[SAVE-CHAOS-REPORT-CONF] Updating Start Time for project: " + projectName);
// console.log("[SAVE-CHAOS-REPORT-CONF] Updating Start Time for project: " + projectName);
var oReq = new XMLHttpRequest();
var redis_key = projectName + "_check_url_start_time";
console.log("[SAVE-CHAOS-REPORT-CONF] Redis key: " + redis_key);
// console.log("[SAVE-CHAOS-REPORT-CONF] Redis key: " + redis_key);
oReq.onreadystatechange = function () {
if (this.readyState === XMLHttpRequest.DONE && this.status === 200) {
@@ -317,7 +317,7 @@ function updateChaosReportStartTime(projectName) {
function drawCanvasHTTPStatusCodeStats() {
console.log("[SAVE-CHAOS-REPORT-CONF] Updating chart");
// // console.log("[SAVE-CHAOS-REPORT-CONF] Updating chart");
myHTTPElapsedChart.setOption({
xAxis: {},
@@ -333,14 +333,6 @@ function drawCanvasHTTPStatusCodeStats() {
]
});
// console.log("[SAVE-CHAOS-REPORT-CONF] Updating metrics");
// console.log("[SAVE-CHAOS-REPORT-CONF] Deleted Pods: " + chart_deleted_pods_total);
// console.log("[SAVE-CHAOS-REPORT-CONF] Chaos Jobs: " + chart_chaos_jobs_total);
// console.log("[SAVE-CHAOS-REPORT-CONF] Current Chaos Pods: " + chart_current_chaos_job_pod);
// console.log("[SAVE-CHAOS-REPORT-CONF] Not Running Pods: " + chart_pods_not_running_on);
// console.log("[SAVE-CHAOS-REPORT-CONF] Current Replicas State Delay: " + chart_fewer_replicas_seconds);
// console.log("[SAVE-CHAOS-REPORT-CONF] Latest Replicas State Delay: " + chart_latest_fewer_replicas_seconds);
myMainChaosMetrics.setOption({
series: [
{
@@ -376,67 +368,6 @@ function drawCanvasHTTPStatusCodeStats() {
]
});
// while (chaos_report_http_elapsed_time_array.length > 40) {
// chaos_report_http_elapsed_time_array.shift();
// }
// var elapsedTimeArray = chaos_report_http_elapsed_time_array;
// var canvas = document.getElementById('httpStatsCanvas');
// var context = canvas.getContext('2d');
// context.clearRect(0, 0, canvas.width, canvas.height);
// // Imposta la dimensione dei rettangoli e il numero di colonne e righe
// var rectSize = 20;
// var rows = canvas.height / rectSize;
// var columns = canvas.width / rectSize;
// // Disegna la griglia di rettangoli
// for (var i = 0; i < rows; i++) {
// for (var j = 0; j < columns; j++) {
// // Calcola le coordinate del rettangolo
// var x = j * rectSize;
// var y = i * rectSize;
// context.strokeStyle = 'black';
// context.fillStyle = 'white';
// // Disegna il rettangolo
// context.fillRect(x, y, rectSize, rectSize);
// // Aggiungi bordi per una migliore visualizzazione
// //context.strokeRect(x, y, rectSize, rectSize);
// }
// }
// for (var i = 0; i < elapsedTimeArray.length; i++) {
// var x = i * rectSize;
// var y = 80;
// var width = rectSize;
// var height = rectSize;
// if (elapsedTimeArray[i] > 3) {
// context.strokeStyle = 'black';
// context.fillStyle = 'red';
// } else if (elapsedTimeArray[i] > 2) {
// context.strokeStyle = 'black';
// context.fillStyle = 'orange';
// } else {
// context.strokeStyle = 'black';
// context.fillStyle = "green";
// }
// if (i == elapsedTimeArray.length - 1) {
// var text = String(elapsedTimeArray[i]);
// context.fillStyle = 'black';
// context.font = '20px Courier New';
// context.fillText(text, x + width / 2 - context.measureText(text).width / 2, y - (height / 2));
// }
// context.fillRect(x, y, width, height);
// //context.strokeRect(x, y, 20, 20);
// }
}
var myHTTPElapsedChart = echarts.init(document.getElementById('httpElapsedChart'));
nginx/KubeInvaders.conf
+72 -18
View File
@@ -123,7 +123,7 @@ server {
red:set("latest_codename", random_word)
ngx.log(ngx.INFO, "[programming_mode_codename] Choosing random word " .. random_word)
-- ngx.log(ngx.INFO, "[programming_mode_codename] Choosing random word " .. random_word)
ngx.say(random_word)
}
@@ -308,7 +308,7 @@ server {
ngx.req.read_body()
local okredis, errredis = red:connect("unix:/tmp/redis.sock")
local key_name = args["lang"] .. "_" .. args["name"]
ngx.log(ngx.INFO, "[PRESETS-RESET] The item " .. key_name .. " should be removed from the key presets_list")
-- ngx.log(ngx.INFO, "[PRESETS-RESET] The item " .. key_name .. " should be removed from the key presets_list")
local handle = io.popen("redis-cli --scan --pattern " .. key_name .. " | xargs redis-cli del")
local result = handle:read("*a")
@@ -323,7 +323,7 @@ server {
local new_preset_list = "";
local cnt = 0;
for key, value in ipairs(mysplit(res, ",")) do
ngx.log(ngx.INFO, "[PRESETS-RESET] Split the key presets_list. Current preset is: " .. value)
-- ngx.log(ngx.INFO, "[PRESETS-RESET] Split the key presets_list. Current preset is: " .. value)
if value ~= key_name then
preset = value
if cnt > 0 then
@@ -334,7 +334,7 @@ server {
end
cnt = cnt + 1
end
ngx.log(ngx.INFO, "[PRESETS-RESET] New key presets_list is: " .. new_preset_list)
-- ngx.log(ngx.INFO, "[PRESETS-RESET] New key presets_list is: " .. new_preset_list)
red:set("presets_list", new_preset_list)
end
}
@@ -358,14 +358,64 @@ server {
ngx.say(err)
else
ngx.say(res)
ngx.log(ngx.INFO, "[PRESETS_LIST] " .. res)
-- ngx.log(ngx.INFO, "[PRESETS_LIST] " .. res)
end
}
}
location /chaos/report/keepalive {
content_by_lua_block {
local function check_if_string_contain_project(project_list, project)
if string.find(project_list, project) then
return true
end
return false
end
local function add_project(project_list, new_project)
local new_project_list = project_list .. "," .. new_project
return new_project_list
end
local function check_if_redis_key_exists(redis, key)
if redis:exists(key) == 1 then
return true
end
return false
end
local redis = require "resty.redis"
local red = redis:new()
local args = ngx.req.get_uri_args()
ngx.header['Access-Control-Allow-Origin'] = '*'
ngx.header['Access-Control-Allow-Methods'] = 'GET, POST, OPTIONS'
ngx.header['Access-Control-Allow-Headers'] = 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range'
ngx.header['Access-Control-Expose-Headers'] = 'Content-Length,Content-Range';
ngx.req.read_body()
local data = ngx.req.get_body_data()
local okredis, errredis = red:connect("unix:/tmp/redis.sock")
if check_if_redis_key_exists(red, "chaos_report_project_list") then
if not check_if_string_contain_project(red:get("chaos_report_project_list"), args["project"]) then
red:set("chaos_report_project_list", add_project(red:get("chaos_report_project_list"), args["project"]))
end
else
red:set("chaos_report_project_list", args["project"])
end
red:expire("chaos_report_project_list", 5)
}
}
location /chaos/report/save {
content_by_lua_block {
local function check_if_string_contain_project(project_list, project)
if string.find(project_list, project) then
return true
end
return false
end
local function add_project(project_list, new_project)
local new_project_list = project_list .. "," .. new_project
return new_project_list
@@ -391,12 +441,16 @@ server {
red:set("chaos_report_project_" .. args["project"], data)
if check_if_redis_key_exists(red, "chaos_report_project_list") then
red:set("chaos_report_project_list", add_project(red:get("chaos_report_project_list"), args["project"]))
if not check_if_string_contain_project(red:get("chaos_report_project_list"), args["project"]) then
red:set("chaos_report_project_list", add_project(red:get("chaos_report_project_list"), args["project"]))
end
else
red:set("chaos_report_project_list", args["project"])
end
red:expire("chaos_report_project_list", 5)
ngx.log(ngx.INFO, "[CHAOS-REPORT-SAVE] " .. data)
-- ngx.log(ngx.INFO, "[CHAOS-REPORT-SAVE] " .. data)
}
}
@@ -416,10 +470,10 @@ server {
local all_presets_key = ""
if data ~= nil then
ngx.log(ngx.INFO, "[SAVE-PRESETS] lang:" .. args["lang"] .. ", name:" .. args["name"] .. ", payload:" .. data .. "\n key_name:" .. key_name .. "\n key_exists: " .. red:exists(key_name))
-- ngx.log(ngx.INFO, "[SAVE-PRESETS] lang:" .. args["lang"] .. ", name:" .. args["name"] .. ", payload:" .. data .. "\n key_name:" .. key_name .. "\n key_exists: " .. red:exists(key_name))
-- if (red:exists(key_name) == 0) then
red:set(key_name, data)
ngx.log(ngx.INFO, "[SAVE-PRESETS] key_name:" .. key_name .. ", data:" .. data)
-- ngx.log(ngx.INFO, "[SAVE-PRESETS] key_name:" .. key_name .. ", data:" .. data)
local res, err = red:get(key_name)
if res == ngx.null then
@@ -481,13 +535,13 @@ server {
ngx.header['Access-Control-Expose-Headers'] = 'Content-Length,Content-Range';
ngx.req.read_body()
local data = ngx.req.get_body_data()
-- ngx.log(ngx.INFO, "[PROGRAMMING-MODE-DIAGRAM] data sent from web interface => " .. data)
-- -- ngx.log(ngx.INFO, "[PROGRAMMING-MODE-DIAGRAM] data sent from web interface => " .. data)
math.randomseed(os.clock()*100000000000)
local rand = math.random(999, 9999)
local filename = "/tmp/chaosprogram" .. rand
ngx.log(ngx.INFO, "[PROGRAMMING-MODE-DIAGRAM] write temp file " .. filename)
-- ngx.log(ngx.INFO, "[PROGRAMMING-MODE-DIAGRAM] write temp file " .. filename)
local yamlfile = io.open(filename, "w")
yamlfile:write(data)
@@ -497,16 +551,16 @@ server {
local result = handle:read("*a")
local rc = handle:close()
ngx.log(ngx.INFO, "[programming_mode_diagram] Result of yaml syntax check =>||" .. result .. "||")
-- ngx.log(ngx.INFO, "[programming_mode_diagram] Result of yaml syntax check =>||" .. result .. "||")
if data == nil then
error = "[PROGRAMMING-MODE-DIAGRAM] No chaos program already loaded."
ngx.log(ngx.INFO, error)
-- ngx.log(ngx.INFO, error)
ngx.say(error)
elseif not string.match(result, '{.*}') then
error = "[PROGRAMMING-MODE-DIAGRAM] Chaos program seems not correct. Please check yaml syntax."
ngx.log(ngx.INFO, error)
-- ngx.log(ngx.INFO, error)
ngx.say(error)
end
@@ -514,22 +568,22 @@ server {
if not key_exists(yaml_data, "jobs") then
error = "[PROGRAMMING-MODE-DIAGRAM] Chaos program does not contain 'jobs' key. Please fix yaml definition."
ngx.log(ngx.INFO, error)
-- ngx.log(ngx.INFO, error)
ngx.say(error)
elseif not key_exists(yaml_data, "experiments") then
error = "[PROGRAMMING-MODE-DIAGRAM] Chaos program does not contain 'experiments' key. Please fix yaml definition"
ngx.log(ngx.INFO, error)
-- ngx.log(ngx.INFO, error)
ngx.say(error)
elseif is_key_empty(yaml_data, "jobs") then
error = "[PROGRAMMING-MODE-DIAGRAM] Chaos program does not contain valid 'jobs' key. Please fix yaml definition, add jobs"
ngx.log(ngx.INFO, error)
-- ngx.log(ngx.INFO, error)
ngx.say(error)
elseif is_key_empty(yaml_data, "experiments") then
error = "[PROGRAMMING-MODE-DIAGRAM] Chaos program does not contain valid 'experiments' key. Please fix yaml definition, add experiments"
ngx.log(ngx.INFO, error)
-- ngx.log(ngx.INFO, error)
ngx.say(error)
else
scripts/metrics_loop/start.py
+13 -15
View File
@@ -17,7 +17,6 @@ import time
import datetime
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
def join_string_to_array(project_list_string, separator):
return project_list_string.split(separator)
@@ -28,7 +27,7 @@ def do_http_request(url, method, headers, data):
return response
except requests.exceptions.RequestException as e:
logging.error(f"Error while sending HTTP request: {e}")
return None
return "Connection Error"
def check_if_json_is_valid(json_data):
try:
@@ -96,26 +95,21 @@ api_instance = client.CoreV1Api()
batch_api = client.BatchV1Api()
while True:
logging.info(f"[k-inv][metrics_loop] Metrics loop is active - {r.exists('chaos_report_project_list')}")
#logging.info(f"[k-inv][metrics_loop] Metrics loop is active - {r.exists('chaos_report_project_list')}")
if r.exists('chaos_report_project_list'):
logging.info("[k-inv][metrics_loop] Found chaos_report_project_list Redis Key")
#logging.info("[k-inv][metrics_loop] Found chaos_report_project_list Redis Key")
for project in join_string_to_array(r.get('chaos_report_project_list').decode(), ','):
logging.info(f"[k-inv][metrics_loop] Computing Chaos Report Project: {project}")
#logging.info(f"[k-inv][metrics_loop] Computing Chaos Report Project: {project}")
chaos_program_key = f"chaos_report_project_{project}"
if r.exists(chaos_program_key):
logging.info(f"[k-inv][metrics_loop][chaos_report] Found chaos_report_project_{project} key in Redis. Starting {project} ")
#logging.info(f"[k-inv][metrics_loop][chaos_report] Found chaos_report_project_{project} key in Redis. Starting {project} ")
if check_if_json_is_valid(r.get(chaos_program_key)):
chaos_report_program = json.loads(r.get(chaos_program_key))
now = datetime.datetime.now()
logging.info(f"[k-inv][metrics_loop][chaos_report] chaos_report_program is valid JSON: {chaos_report_program}")
#logging.info(f"[k-inv][metrics_loop][chaos_report] chaos_report_program is valid JSON: {chaos_report_program}")
response = do_http_request(chaos_report_program['chaosReportCheckSiteURL'], chaos_report_program['chaosReportCheckSiteURLMethod'], json.loads(chaos_report_program['chaosReportCheckSiteURLHeaders']), chaos_report_program['chaosReportCheckSiteURLPayload'])
logging.info(f"[k-inv][metrics_loop][chaos_report] chaos_report_program response: {response.status_code}")
check_url_counter_key = f"{chaos_report_program['chaosReportProject']}_check_url_counter"
check_url_status_code_key = f"{chaos_report_program['chaosReportProject']}_check_url_status_code"
check_url_elapsed_time_key = f"{chaos_report_program['chaosReportProject']}_check_url_elapsed_time"
@@ -129,9 +123,13 @@ while True:
if r.get(check_url_start_time) == None:
r.set(check_url_start_time, now.strftime("%Y-%m-%d %H:%M:%S"))
r.set(check_url_status_code_key, response.status_code)
r.set(check_url_elapsed_time_key, float(response.elapsed.total_seconds()))
if response == "Connection Error":
#logging.info(f"[k-inv][metrics_loop][chaos_report] Connection Error while checking {chaos_report_program['chaosReportCheckSiteURL']}")
r.set(check_url_status_code_key, "Connection Error")
r.set(check_url_elapsed_time_key, 0)
else:
r.set(check_url_status_code_key, response.status_code)
r.set(check_url_elapsed_time_key, float(response.elapsed.total_seconds()))
try:
label_selector="chaos-controller=kubeinvaders"
api_response = api_instance.list_pod_for_all_namespaces(label_selector=label_selector)