mirror of
https://github.com/lucky-sideburn/kubeinvaders.git
synced 2026-02-14 09:39:56 +00:00
fix logs and chaos programming console
This commit is contained in:
Eugenio Marzo
@@ -106,7 +106,7 @@
|
||||
<div class="modal-footer">
|
||||
<button type="button" class="btn btn-light" onclick="savePreset('apply')">Save and apply</button>
|
||||
<button type="button" class="btn btn-light" onclick="savePreset('save')">Save</button>
|
||||
<button type="button" class="btn btn-danger" onclick="resetPreset('reset')">Reset to default</button>
|
||||
<button type="button" id="resetToDefaultButton" class="btn btn-light-saved" onclick="resetPreset('reset')">Reset to default</button>
|
||||
<button type="button" class="btn btn-dark" data-dismiss="modal" onclick="closeSetLoadTestModal()">Close</button>
|
||||
</div>
|
||||
</form>
|
||||
@@ -151,7 +151,7 @@
|
||||
</div>
|
||||
<div class="col text-center">
|
||||
<div id="programming-mode-buttons" style="display: none;">
|
||||
<button type="button" id="runProgrammingModeButton" class="btn btn-dark" onclick="runChaosProgram()">RUN</button>
|
||||
<!-- <button type="button" id="runProgrammingModeButton" class="btn btn-dark" onclick="runChaosProgram()">RUN</button> -->
|
||||
</div>
|
||||
</div>
|
||||
<div class="col text-center">
|
||||
@@ -218,7 +218,7 @@
|
||||
</div>
|
||||
<div class="">
|
||||
<div class="row" style="margin-top: 2%;">
|
||||
<font size="2">Load Testing Presets</font>
|
||||
<font size="2">Load Testing Presets & Chaos Programs</font>
|
||||
</div>
|
||||
<div class="row">
|
||||
<div class="btn-group" id="loadButtonGroup" role="group" aria-label="Basic example" style="overflow-x: scroll; width: 100%; padding-bottom: 2%; padding-top: 2%; scrollbar-color: dark;">
|
||||
@@ -240,6 +240,10 @@
|
||||
<button type="button" id="loadSSH" class="btn btn-light btn-sm" onclick="loadPreset('SSH', 'python')">SSH</button>
|
||||
<button type="button" id="loadVault" class="btn btn-light btn-sm" onclick="loadPreset('vault', 'python')">Vault</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="row">
|
||||
<div class="btn-group" id="loadChaosProgramButtonGroup" role="group" aria-label="Basic example" style="overflow-x: scroll; width: 100%; padding-bottom: 2.0%; padding-top: 0.5%; scrollbar-color: dark;">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -248,42 +252,50 @@
|
||||
<!-- START CHAOS PROGRAMMING MODE SCREEN -->
|
||||
<div id="chaos-program-screen" style="display: none;">
|
||||
<div class="row" style="margin-top: 1%;">
|
||||
<div id="alert_placeholder4" style="margin-top: 1%;"></div>
|
||||
<div id="alert_placeholder_programming_mode" style="margin-top: 1%;"></div>
|
||||
<div id="alert_placeholder_programming_mode" style="margin-top: 1%; margin-bottom: 1%;"></div>
|
||||
<div class="col">
|
||||
<div class="row" width="10px">
|
||||
<div class="col text-center">
|
||||
<button type="button" id="saveChaosProgramButton" class="btn btn-light btn-sm" style="width: 50%;" onclick="savePreset('save-chaos-program')">SAVE</button>
|
||||
</div>
|
||||
<div class="col text-center">
|
||||
<button type="button" id="runChaosProgramButton" class="btn btn-dark" style="width: 50%;" onclick="runChaosProgram()">RUN</button>
|
||||
</div>
|
||||
</div>
|
||||
<form>
|
||||
<div class="form-group">
|
||||
<label for="chaosProgramTextArea"></label>
|
||||
<textarea class="form-control chaos-prog-area" id="chaosProgramTextArea" rows="50" style="min-width: 100%; font-family: Courier, monospace;">
|
||||
jobs:
|
||||
cpu-attack-job:
|
||||
additional-labels:
|
||||
chaos-controller: kubeinvaders
|
||||
chaos-type: stress-ng
|
||||
chaos-codename: CODENAME_PLACEHOLDER
|
||||
image: docker.io/luckysideburn/kubeinvaders-stress-ng:latest
|
||||
command: "stress-ng"
|
||||
args:
|
||||
- --help
|
||||
chaos-codename: CODENAME_PLACEHOLDER
|
||||
jobs:
|
||||
cpu-attack-job:
|
||||
additional-labels:
|
||||
chaos-controller: kubeinvaders
|
||||
chaos-type: stress-ng
|
||||
chaos-codename: CODENAME_PLACEHOLDER
|
||||
image: docker.io/luckysideburn/kubeinvaders-stress-ng:latest
|
||||
command: "stress-ng"
|
||||
args:
|
||||
- --help
|
||||
|
||||
mem-attack-job:
|
||||
additional-labels:
|
||||
chaos-controller: kubeinvaders
|
||||
chaos-type: stress-ng
|
||||
chaos-codename: CODENAME_PLACEHOLDER
|
||||
image: docker.io/luckysideburn/kubeinvaders-stress-ng:latest
|
||||
command: "stress-ng"
|
||||
args:
|
||||
- --help
|
||||
mem-attack-job:
|
||||
additional-labels:
|
||||
chaos-controller: kubeinvaders
|
||||
chaos-type: stress-ng
|
||||
chaos-codename: CODENAME_PLACEHOLDER
|
||||
image: docker.io/luckysideburn/kubeinvaders-stress-ng:latest
|
||||
command: "stress-ng"
|
||||
args:
|
||||
- --help
|
||||
|
||||
experiments:
|
||||
- name: cpu-attack-exp
|
||||
job: cpu-attack-job
|
||||
loop: 5
|
||||
experiments:
|
||||
- name: cpu-attack-exp
|
||||
job: cpu-attack-job
|
||||
loop: 5
|
||||
|
||||
- name: mem-attack-exp
|
||||
job: mem-attack-job
|
||||
loop: 5
|
||||
- name: mem-attack-exp
|
||||
job: mem-attack-job
|
||||
loop: 5
|
||||
</textarea>
|
||||
</div>
|
||||
</form>
|
||||
|
||||
@@ -100,6 +100,10 @@ const chaos_job_regex = /chaos_jobs_status.*/g;
|
||||
var codename_configured = false;
|
||||
var chaos_jobs_status = new Map();
|
||||
|
||||
function rand_id() {
|
||||
return getRandomInt(9999);
|
||||
}
|
||||
|
||||
function isJsonString(str) {
|
||||
try {
|
||||
JSON.parse(str);
|
||||
@@ -124,6 +128,24 @@ function getCodeName() {
|
||||
oReq.send();
|
||||
}
|
||||
|
||||
function createChaosProgramButton(name, lang) {
|
||||
let btn = document.createElement("button");
|
||||
let capitalizedName = name.charAt(0).toUpperCase() + name.slice(1);
|
||||
console.log("[CREATE-CHAOS-PROGRAM] Creating button for " + name);
|
||||
btn.innerHTML = capitalizedName;
|
||||
btn.type = "button";
|
||||
btn.name = "load" + capitalizedName;
|
||||
btn.id = "load" + capitalizedName;
|
||||
if (document.getElementById("load" + capitalizedName)) {
|
||||
return;
|
||||
}
|
||||
btn.style = "padding: 0% 2%;"
|
||||
btn.classList = "btn btn-light btn-sm";
|
||||
btn.addEventListener("click", function(){ loadPreset(name, lang); });
|
||||
document.getElementById("loadButtonGroup").appendChild(btn);
|
||||
document.getElementById("loadButtonGroup").scrollLeft = document.getElementById("loadButtonGroup").scrollWidth;
|
||||
}
|
||||
|
||||
function getSavedPresets() {
|
||||
var oReq = new XMLHttpRequest();
|
||||
oReq.onreadystatechange = function () {
|
||||
@@ -136,9 +158,16 @@ function getSavedPresets() {
|
||||
currentPresetName = currentPresetName.charAt(0).toUpperCase() + currentPresetName.slice(1);
|
||||
//console.log("[GET-PRESETS] computing preset: " + currentPresetName);
|
||||
var buttonId = "load" + currentPresetName.trim();
|
||||
console.log("[GET-PRESETS] Change border color of buttonId: " + buttonId);
|
||||
document.getElementById(buttonId).classList.remove('btn-light');
|
||||
document.getElementById(buttonId).classList.add('btn-light-saved');
|
||||
// console.log("[GET-PRESETS] Change border color of buttonId: " + buttonId);
|
||||
// console.log(document.getElementById(buttonId));
|
||||
if (document.getElementById(buttonId) == null){
|
||||
console.log("[GET-PRESETS] Appending button to loadButtonGroup. id: " + buttonId + " presetname: " + currentPresetName.trim());
|
||||
latest_preset_lang = "k-inv";
|
||||
createChaosProgramButton(currentPresetName.trim(), latest_preset_lang);
|
||||
} else {
|
||||
// document.getElementById(buttonId).classList.remove('btn-light');
|
||||
// document.getElementById(buttonId).classList.add('btn-light-saved');
|
||||
}
|
||||
}
|
||||
} else {
|
||||
console.log("[GET-PRESETS] There is no saved presets in Redis");
|
||||
@@ -153,7 +182,6 @@ function loadSavedPreset(tool, lang, defaultpreset) {
|
||||
var oReq = new XMLHttpRequest();
|
||||
oReq.onreadystatechange = function () {
|
||||
if (this.readyState === XMLHttpRequest.DONE && this.status === 200) {
|
||||
//console.log("response of loadSavedPreset: ||" + this.responseText + "||");
|
||||
if (this.responseText.trim() != "nil") {
|
||||
$("#currentLoadTest").val(this.responseText.trim());
|
||||
} else {
|
||||
@@ -174,8 +202,8 @@ function resetPreset() {
|
||||
if (this.readyState === XMLHttpRequest.DONE && this.status === 200) {
|
||||
let capitalizedPreset = latest_preset_name.charAt(0).toUpperCase() + latest_preset_name.slice(1);
|
||||
let buttonId = "load" + capitalizedPreset;
|
||||
document.getElementById(buttonId).classList.remove('btn-light-saved');
|
||||
document.getElementById(buttonId).classList.add('btn-light');
|
||||
// document.getElementById(buttonId).classList.remove('btn-light-saved');
|
||||
// document.getElementById(buttonId).classList.add('btn-light');
|
||||
closeSetLoadTestModal();
|
||||
getSavedPresets();
|
||||
console.log("[RESET-PRESETS] " + latest_preset_name + " restored with default preset");
|
||||
@@ -189,10 +217,32 @@ function resetPreset() {
|
||||
}
|
||||
|
||||
function savePreset(action) {
|
||||
console.log("[SAVE-PRESET-CHAOSPROGRAM] Saving item...");
|
||||
var presetName = "";
|
||||
presetBody = $("#currentLoadTest").val();
|
||||
presetLang = latest_preset_lang;
|
||||
presetName = latest_preset_name;
|
||||
console.log("[SAVE-PRESET-CHAOSPROGRAM] Saving " + presetBody);
|
||||
|
||||
if (action == "save-chaos-program") {
|
||||
presetLang = "k-inv";
|
||||
presetName = codename + "-" + rand_id();
|
||||
latest_preset_lang = "k-inv";
|
||||
console.log("[SAVE-PRESET-CHAOSPROGRAM] lang: " + presetLang + " name:" + presetName);
|
||||
presetBody = $('#chaosProgramTextArea').val();
|
||||
document.getElementById("resetToDefaultButton").style.display = "none";
|
||||
}
|
||||
else if (latest_preset_lang == "k-inv") {
|
||||
presetLang = "k-inv";
|
||||
presetName = codename;
|
||||
latest_preset_lang = "k-inv";
|
||||
console.log("[SAVE-PRESET-CHAOSPROGRAM] lang: " + presetLang + " name:" + codename);
|
||||
presetBody = $('#currentLoadTest').val();
|
||||
document.getElementById("resetToDefaultButton").style.display = "none";
|
||||
}
|
||||
else {
|
||||
presetLang = latest_preset_lang;
|
||||
presetName = latest_preset_name;
|
||||
document.getElementById("resetToDefaultButton").style.display = "block";
|
||||
}
|
||||
|
||||
//console.log("Saving preset. name:" + presetName + ", lang:" + presetName + ", body: " + presetBody);
|
||||
var oReq = new XMLHttpRequest();
|
||||
@@ -200,26 +250,38 @@ function savePreset(action) {
|
||||
oReq.open("POST", k8s_url + "/chaos/loadpreset/save?name=" + presetName + "&lang=" + presetLang, true);
|
||||
|
||||
oReq.onreadystatechange = function () {
|
||||
if (this.readyState === XMLHttpRequest.DONE && this.status === 200 && action == "apply") {
|
||||
// console.log(this.responseText);
|
||||
// $('#alert_placeholder').replaceWith(this.responseText);
|
||||
presetBody = $('#chaosProgramTextArea').val(`chaos-codename: ${codename}\njobs:
|
||||
if (this.readyState === XMLHttpRequest.DONE && this.status === 200 && (action == "apply" || action == "save-chaos-program")) {
|
||||
if (latest_preset_lang == "k-inv") {
|
||||
console.log("[SAVE-PRESET-CHAOSPROGRAM] Payload: " + $('#currentLoadTest').val());
|
||||
if ($('#currentLoadTest').val() != "") {
|
||||
presetBody = $('#currentLoadTest').val();
|
||||
}
|
||||
|
||||
//$('#chaosProgramTextArea').val(presetBody);
|
||||
|
||||
document.getElementById("chaosProgramTextArea").value = presetBody;
|
||||
}
|
||||
else {
|
||||
presetBody = $('#chaosProgramTextArea').val(`chaos-codename: ${codename}
|
||||
jobs:
|
||||
${presetName}-job:
|
||||
additional-labels:
|
||||
chaos-controller: kubeinvaders
|
||||
chaos-lang: ${presetLang}
|
||||
chaos-type: loadtest
|
||||
chaos-codename: ${codename}
|
||||
image: docker.io/luckysideburn/chaos-exec:v1.0.0
|
||||
chaos-controller: kubeinvaders
|
||||
chaos-lang: ${presetLang}
|
||||
chaos-type: loadtest
|
||||
chaos-codename: ${codename}
|
||||
image: docker.io/luckysideburn/chaos-exec:v1.0.4
|
||||
command: bash
|
||||
args:
|
||||
- start.sh
|
||||
- ${presetLang}
|
||||
- http://kubeinvaders:8080/${presetLang}_${presetName}
|
||||
- code=${btoa(presetBody).trim()}
|
||||
|
||||
experiments:
|
||||
- name: ${presetName}-exp
|
||||
job: ${presetName}-job
|
||||
loop: 5`);
|
||||
}
|
||||
}
|
||||
};;
|
||||
|
||||
@@ -227,10 +289,16 @@ experiments:
|
||||
oReq.send(presetBody);
|
||||
closeSetLoadTestModal();
|
||||
|
||||
let presetNameCapitalized = presetName.charAt(0).toUpperCase() + presetName.slice(1);
|
||||
var buttonId = "load" + presetNameCapitalized.trim();
|
||||
document.getElementById(buttonId).classList.remove('btn-light');
|
||||
document.getElementById(buttonId).classList.add('btn-light-saved');
|
||||
if (action != "save-chaos-program") {
|
||||
let presetNameCapitalized = presetName.charAt(0).toUpperCase() + presetName.slice(1);
|
||||
var buttonId = "load" + presetNameCapitalized.trim();
|
||||
// document.getElementById(buttonId).classList.remove('btn-light');
|
||||
// document.getElementById(buttonId).classList.add('btn-light-saved');
|
||||
}
|
||||
else {
|
||||
console.log("[SAVE-PRESET-CHAOSPROGRAM] Creating new button for lang: " + presetLang + " name:" + presetName);
|
||||
createChaosProgramButton(presetName, 'k-inv');
|
||||
}
|
||||
|
||||
getSavedPresets();
|
||||
|
||||
@@ -1009,4 +1077,4 @@ document.getElementById("metricsPresetsRow").style.opacity = 0;
|
||||
document.getElementById("gameContainer").style.visibility = "visible";
|
||||
document.getElementById("metricsPresetsRow").style.visibility = "visible";
|
||||
document.getElementById("gameContainer").style.opacity = 1;
|
||||
document.getElementById("metricsPresetsRow").style.opacity = 1;
|
||||
document.getElementById("metricsPresetsRow").style.opacity = 1;
|
||||
|
||||
@@ -25,12 +25,23 @@ loadPresetsCodeJson = `{
|
||||
|
||||
function loadPreset(tool, lang) {
|
||||
let decodedStringAtoB = "";
|
||||
console.log("[GET-PRESETS] Loaded preset for " + tool);
|
||||
loadPresetsCodeParsed = JSON.parse(loadPresetsCodeJson);
|
||||
decodedStringAtoB = atob(loadPresetsCodeParsed[tool]);
|
||||
console.log("[GET-PRESETS] Loaded preset for " + tool + " with lang " + lang);
|
||||
|
||||
latest_preset_name = tool;
|
||||
latest_preset_lang = lang;
|
||||
loadSavedPreset(tool, lang, decodedStringAtoB);
|
||||
console.log("[GET-PRESETS] |" + lang + "|");
|
||||
|
||||
if (lang == "k-inv") {
|
||||
loadSavedPreset(tool, lang, $('#chaosProgramTextArea').text());
|
||||
document.getElementById("resetToDefaultButton").style.display = "none";
|
||||
} else {
|
||||
console.log("[GET-PRESETS] foo Loaded preset for " + tool + " with lang " + lang);
|
||||
console.log("[GET-PRESET] loadPresetsCodeJson " +loadPresetsCodeJson);
|
||||
loadPresetsCodeParsed = JSON.parse(loadPresetsCodeJson);
|
||||
decodedStringAtoB = atob(loadPresetsCodeParsed[tool]);
|
||||
loadSavedPreset(tool, lang, decodedStringAtoB);
|
||||
document.getElementById("resetToDefaultButton").style.display = "block";
|
||||
}
|
||||
$("#presetLang").val(lang);
|
||||
$("#presetName").val(tool);
|
||||
$('#setLoadTestModal').modal('show');
|
||||
@@ -189,8 +200,4 @@ loadPresetsCodeJson = `{
|
||||
function closeKubeLinterModal() {
|
||||
$('#kubeLinterModal').modal('hide');
|
||||
modal_opened = false;
|
||||
}
|
||||
|
||||
// $('textarea').on('input', function() {
|
||||
// $('#alert_placeholder2').text('');
|
||||
// });
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
#!/bin/bash
|
||||
|
||||
docker build . -t docker.io/luckysideburn/chaos-exec:latest
|
||||
docker tag docker.io/luckysideburn/chaos-exec:latest docker.io/luckysideburn/chaos-exec:v1.0.0
|
||||
docker tag docker.io/luckysideburn/chaos-exec:latest docker.io/luckysideburn/chaos-exec:v1.0.4
|
||||
docker push docker.io/luckysideburn/chaos-exec:latest
|
||||
docker push docker.io/luckysideburn/chaos-exec:v1.0.0
|
||||
docker push docker.io/luckysideburn/chaos-exec:v1.0.4
|
||||
|
||||
@@ -1,10 +1,8 @@
|
||||
#!/bin/bash
|
||||
|
||||
echo "Starting chaos-exec general purpose image for KubeInvaders"
|
||||
|
||||
echo "Downloading chaos script from KubeInvaders"
|
||||
curl -o /tmp/run.chaos $2
|
||||
echo $2 | sed 's/code=//g' | base64 -d > /tmp/run.chaos
|
||||
echo "Executing chaos script"
|
||||
|
||||
cat /tmp/run.chaos
|
||||
($1 /tmp/run.chaos) || echo "Chaos job completed with error. Please check code of this experiment"
|
||||
|
||||
|
||||
@@ -334,9 +334,10 @@ server {
|
||||
local all_presets_key = ""
|
||||
|
||||
if data ~= nil then
|
||||
ngx.log(ngx.INFO, "[SAVE-PRESETS] lang:" .. args["lang"] .. ", name:" .. args["name"] .. ", payload:" .. data)
|
||||
if (red:exists(key_name) == 0) then
|
||||
ngx.log(ngx.INFO, "[SAVE-PRESETS] lang:" .. args["lang"] .. ", name:" .. args["name"] .. ", payload:" .. data .. "\n key_name:" .. key_name .. "\n key_exists: " .. red:exists(key_name))
|
||||
-- if (red:exists(key_name) == 0) then
|
||||
red:set(key_name, data)
|
||||
ngx.log(ngx.INFO, "[SAVE-PRESETS] key_name:" .. key_name .. ", data:" .. data)
|
||||
local res, err = red:get(key_name)
|
||||
|
||||
if res == ngx.null then
|
||||
@@ -359,7 +360,7 @@ server {
|
||||
end
|
||||
|
||||
end
|
||||
end
|
||||
-- end
|
||||
else
|
||||
ngx.say("Error in payload sent by web interface.")
|
||||
end
|
||||
@@ -398,7 +399,7 @@ server {
|
||||
ngx.header['Access-Control-Expose-Headers'] = 'Content-Length,Content-Range';
|
||||
ngx.req.read_body()
|
||||
local data = ngx.req.get_body_data()
|
||||
ngx.log(ngx.INFO, "[PROGRAMMING-MODE-DIAGRAM] data sent from web interface => " .. data)
|
||||
-- ngx.log(ngx.INFO, "[PROGRAMMING-MODE-DIAGRAM] data sent from web interface => " .. data)
|
||||
|
||||
math.randomseed(os.clock()*100000000000)
|
||||
local rand = math.random(999, 9999)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
worker_processes 2;
|
||||
error_log /dev/stdout warn;
|
||||
error_log /dev/stdout info;
|
||||
pid /var/run/nginx.pid;
|
||||
env REDIS_HOST;
|
||||
env TOKEN;
|
||||
|
||||
@@ -37,40 +37,40 @@ def create_pod_list(logid, api_response_items, current_regex):
|
||||
webtail_pods.append(pod)
|
||||
regex_match_info = f"[logid:{logid}][k-inv][logs-loop] Taking logs of {pod.metadata.name}. Redis has cached that {current_regex} is good for {pod.metadata.name}"
|
||||
r.set(f"log_status:{logid}", regex_match_info)
|
||||
logging.info(f"[k-inv][regexmatch][logid:{logid}][{cached_regex_match}] IS CHACHED IN REDIS")
|
||||
#logging.debug(f"[k-inv][regexmatch][logid:{logid}][{cached_regex_match}] IS CHACHED IN REDIS")
|
||||
|
||||
else:
|
||||
regex_match_info = f"[logs-loop][logid:{logid}] Skipping logs of {pod.metadata.name}. Redis has cached that {current_regex} is not good for {pod.metadata.name}"
|
||||
logging.debug(regex_match_info)
|
||||
logging.info(f"[k-inv][regexmatch][logid:{logid}][{cached_regex_match}] IS CHACHED IN REDIS")
|
||||
#logging.debug(regex_match_info)
|
||||
#logging.debug(f"[k-inv][regexmatch][logid:{logid}][{cached_regex_match}] IS CHACHED IN REDIS")
|
||||
|
||||
else:
|
||||
if re.search(f"{pod_re}", pod.metadata.name) or re.search(r"{pod_re}", pod.metadata.name):
|
||||
logging.info(f"[logid:{logid}][k-in][regexmatch] |{pod_re}| |{pod.metadata.name}| MATCHED")
|
||||
#logging.debug(f"[logid:{logid}][k-in][regexmatch] |{pod_re}| |{pod.metadata.name}| MATCHED")
|
||||
regex_key_name = f"regex_cmp:{regexsha}:{logid}:{pod.metadata.namespace}:{pod.metadata.name}"
|
||||
|
||||
if re.search(f"{namespace_re}", pod.metadata.namespace) or re.search(r"{namespace_re}", pod.metadata.namespace):
|
||||
logging.info(f"[logid:{logid}][k-inv][regexmatch] |{namespace_re}| |{pod.metadata.namespace}| MATCHED")
|
||||
#logging.debug(f"[logid:{logid}][k-inv][regexmatch] |{namespace_re}| |{pod.metadata.namespace}| MATCHED")
|
||||
if re.search(f"{labels_re}", str(pod.metadata.labels)) or re.search(r"{labels_re}", str(pod.metadata.labels)):
|
||||
logging.info(f"[logid:{logid}][k-inv][regexmatch] |{labels_re}| |{str(pod.metadata.labels)}| MATCHED")
|
||||
#logging.debug(f"[logid:{logid}][k-inv][regexmatch] |{labels_re}| |{str(pod.metadata.labels)}| MATCHED")
|
||||
if re.search(f"{annotations_re}", str(pod.metadata.annotations)) or re.search(r"{annotations_re}", str(pod.metadata.annotations)):
|
||||
logging.info(f"[logid:{logid}][k-inv][regexmatch] |{annotations_re}| |{str(pod.metadata.annotations)}| MATCHED")
|
||||
#logging.debug(f"[logid:{logid}][k-inv][regexmatch] |{annotations_re}| |{str(pod.metadata.annotations)}| MATCHED")
|
||||
webtail_pods.append(pod)
|
||||
regex_match_info = f"[logid:{logid}] Taking logs from {pod.metadata.name}. It is compliant with the Regex {current_regex}"
|
||||
r.set(regex_key_name, "maching")
|
||||
logging.info(regex_match_info)
|
||||
logging.debug(regex_match_info)
|
||||
r.set(f"log_status:{logid}", regex_match_info)
|
||||
else:
|
||||
logging.info(f"[logid:{logid}][k-inv][regexmatch] |{annotations_re}| |{str(pod.metadata.annotations)}| FAILED")
|
||||
logging.debug(f"[logid:{logid}][k-inv][regexmatch] |{annotations_re}| |{str(pod.metadata.annotations)}| FAILED")
|
||||
r.set(regex_key_name, "not_maching")
|
||||
else:
|
||||
logging.info(f"[logid:{logid}][k-inv][regexmatch] |{labels_re}| |{str(pod.metadata.labels)}| FAILED")
|
||||
logging.debug(f"[logid:{logid}][k-inv][regexmatch] |{labels_re}| |{str(pod.metadata.labels)}| FAILED")
|
||||
r.set(regex_key_name, "not_maching")
|
||||
else:
|
||||
logging.info(f"[logid:{logid}][k-inv][regexmatch] |{namespace_re}| |{pod.metadata.namespace}| FAILED")
|
||||
logging.debug(f"[logid:{logid}][k-inv][regexmatch] |{namespace_re}| |{pod.metadata.namespace}| FAILED")
|
||||
r.set(regex_key_name, "not_maching")
|
||||
else:
|
||||
logging.info(f"[logid:{logid}][k-inv][regexmatch] |{pod_re}| |{pod.metadata.name}| FAILED")
|
||||
logging.debug(f"[logid:{logid}][k-inv][regexmatch] |{pod_re}| |{pod.metadata.name}| FAILED")
|
||||
r.set(regex_key_name, "not_maching")
|
||||
return webtail_pods
|
||||
|
||||
@@ -130,7 +130,7 @@ def compute_line(api_response_line, container):
|
||||
sha256log = sha256(logrow.encode('utf-8')).hexdigest()
|
||||
|
||||
if not r.exists(f"log:{logid}:{pod.metadata.name}:{container}:{sha256log}"):
|
||||
logging.info(f"[logid:{logid}][k-inv][logs-loop] The key log:{logid}:{pod.metadata.name}:{container}:{sha256log} does not exists. Preparing to store log content")
|
||||
logging.debug(f"[logid:{logid}][k-inv][logs-loop] The key log:{logid}:{pod.metadata.name}:{container}:{sha256log} does not exists. Preparing to store log content")
|
||||
old_rows = r.get(f"logs:chaoslogs-{logid}")
|
||||
logrow = f"{logrow}\n{old_rows}"
|
||||
r.set(f"logs:chaoslogs-{logid}", logrow)
|
||||
@@ -139,9 +139,10 @@ def compute_line(api_response_line, container):
|
||||
r.set(f"log_time:{logid}:{pod.metadata.name}:{container}", time.time())
|
||||
r.expire(f"log:{logid}:{pod.metadata.name}:{container}:{sha256log}", 30)
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO"))
|
||||
logging.getLogger('kubernetes').setLevel(logging.ERROR)
|
||||
|
||||
logging.info('Starting script for KubeInvaders taking logs from pods...')
|
||||
logging.debug('Starting script for KubeInvaders taking logs from pods...')
|
||||
|
||||
file = pathlib.Path('/tmp/redis.sock')
|
||||
|
||||
@@ -151,13 +152,13 @@ else:
|
||||
r = redis.Redis("127.0.0.1", charset="utf-8", decode_responses=True)
|
||||
|
||||
if os.environ.get("DEV"):
|
||||
logging.info("Setting env var for dev...")
|
||||
logging.debug("Setting env var for dev...")
|
||||
r.set("log_pod_regex", '{"pod":".*", "namespace":"namespace1", "labels":".*", "annotations":".*", "containers": ".*"}')
|
||||
r.set("logs_enabled:aaaa", 1)
|
||||
r.expire("logs_enabled:aaaa", 10)
|
||||
r.set("programming_mode", 0)
|
||||
logging.info(r.get("log_pod_regex:aaaa"))
|
||||
logging.info(r.get("logs_enabled:aaaa"))
|
||||
logging.debug(r.get("log_pod_regex:aaaa"))
|
||||
logging.debug(r.get("logs_enabled:aaaa"))
|
||||
|
||||
configuration = client.Configuration()
|
||||
token = os.environ["TOKEN"]
|
||||
@@ -178,7 +179,7 @@ while True:
|
||||
for key in r.scan_iter("logs_enabled:*"):
|
||||
if r.get(key) == "1":
|
||||
logid = key.split(":")[1]
|
||||
logging.info(f"Found key {key} and it is enabled.")
|
||||
logging.debug(f"Found key {key} and it is enabled.")
|
||||
r.set(f"log_status:{logid}", f"[k-inv][logs-loop] Found key {key}. Starting collecting logs...")
|
||||
webtail_pods = []
|
||||
current_regex = get_regex(logid)
|
||||
@@ -188,14 +189,14 @@ while True:
|
||||
else:
|
||||
r.set(f"log_status:{logid}", f"[k-inv][logs-loop] {key} is using this regex: {current_regex}")
|
||||
|
||||
logging.info(f"[logid:{logid}] Checking do_not_clean_log Redis key")
|
||||
logging.debug(f"[logid:{logid}] Checking do_not_clean_log Redis key")
|
||||
|
||||
log_cleaner(logid)
|
||||
|
||||
try:
|
||||
api_response = api_instance.list_pod_for_all_namespaces()
|
||||
except ApiException as e:
|
||||
logging.info(e)
|
||||
logging.debug(e)
|
||||
|
||||
pods_found_info = f"[logid:{logid}][k-inv][logs-loop] Looking for pods compliant with the current regex. Scanning {len(api_response.items)} pods"
|
||||
r.set(f"log_status:{logid}", pods_found_info)
|
||||
@@ -211,45 +212,60 @@ while True:
|
||||
|
||||
r.set(f"logs:webtail_pods_len:{logid}", webtail_pods_len)
|
||||
r.set(f"pods_match_regex:{logid}", webtail_pods_len)
|
||||
logging.debug(f"[logid:{logid}][k-inv][logs-loop] Current Regex: {current_regex}")
|
||||
|
||||
for pod in webtail_pods:
|
||||
logging.debug(f"[logid:{logid}][k-inv][logs-loop] Taking logs from {pod.metadata.name}")
|
||||
container_list = []
|
||||
for container in pod.spec.containers:
|
||||
if "containers_re" in locals() or "containers_re" in globals():
|
||||
if re.search(f"{containers_re}", container.name):
|
||||
container_list.append(container.name)
|
||||
# if "containers_re" in locals() or "containers_re" in globals():
|
||||
# if re.search(f"{containers_re}", container.name):
|
||||
container_list.append(container.name)
|
||||
|
||||
for container in container_list:
|
||||
if pod.status.phase != "Pending":
|
||||
logging.debug(f"[logid:{logid}][k-inv][logs-loop] Listing containers of {pod.metadata.name}. Computing {container} phase: {pod.status.phase}")
|
||||
if pod.status.phase != "Unknown":
|
||||
logging.debug(f"[logid:{logid}][k-inv][logs-loop] Container {container} on pod {pod.metadata.name} has accepted phase for taking logs")
|
||||
try:
|
||||
if r.exists(f"log_time:{logid}:{pod.metadata.name}:{container}"):
|
||||
latest_log_tail_time = r.get(f"log_time:{logid}:{pod.metadata.name}:{container}")
|
||||
else:
|
||||
latest_log_tail_time = time.time()
|
||||
|
||||
|
||||
since = int(time.time() - float(latest_log_tail_time)) + 2
|
||||
logging.debug(f"[logid:{logid}][k-inv][logs-loop] Time types: {type(latest_log_tail_time)} {type(time.time())} {type(since)} since={since}")
|
||||
|
||||
if since == 0:
|
||||
since = 1
|
||||
since = 2
|
||||
|
||||
logging.info(f"[logid:{logid}][k-inv][logs-loop] Calling K8s API for reading logs of {pod.metadata.name} in namespace {pod.metadata.namespace}")
|
||||
logging.debug(f"[logid:{logid}][k-inv][logs-loop] Calling K8s API for reading logs of {pod.metadata.name} container {container} in namespace {pod.metadata.namespace} since {since} seconds")
|
||||
|
||||
api_response = api_instance.read_namespaced_pod_log(name=pod.metadata.name, namespace=pod.metadata.namespace, since_seconds=since, container=container)
|
||||
#api_response = api_instance.read_namespaced_pod_log(name=pod.metadata.name, namespace=pod.metadata.namespace, container=container)
|
||||
|
||||
logging.info(f"[logid:{logid}][k-inv][logs-loop] Computing K8s API response for reading logs of {pod.metadata.name} in namespace {pod.metadata.namespace}")
|
||||
logging.debug(f"[logid:{logid}][k-inv][logs-loop] Computing K8s API response for reading logs of {pod.metadata.name} in namespace {pod.metadata.namespace}")
|
||||
logging.debug(f"[logid:{logid}][k-inv][logs-loop] {api_response}")
|
||||
|
||||
r.set(f"log_time:{logid}:{pod.metadata.name}:{container}", time.time())
|
||||
|
||||
if api_response == "":
|
||||
continue
|
||||
|
||||
compute_line(api_response, container)
|
||||
|
||||
logs = ""
|
||||
|
||||
if type(api_response) is list:
|
||||
for api_response_line in api_response:
|
||||
compute_line(api_response_line, container)
|
||||
#compute_line(api_response_line, container)
|
||||
logs = f"{logs}</br>{api_response_line}"
|
||||
else:
|
||||
for api_response_line in api_response.splitlines():
|
||||
compute_line(api_response_line, container)
|
||||
#compute_line(api_response_line, container)
|
||||
logs = f"{logs}</br>{api_response_line}"
|
||||
|
||||
compute_line(logs, container)
|
||||
|
||||
except ApiException as e:
|
||||
logging.info(e)
|
||||
time.sleep(0.5)
|
||||
logging.debug(f"[k-inv][logs-loop] EXCEPTION {e}")
|
||||
time.sleep(1)
|
||||
|
||||
@@ -24,7 +24,7 @@ def create_container(image, name, command, args):
|
||||
command=command,
|
||||
)
|
||||
|
||||
logging.info(
|
||||
logging.debug(
|
||||
f"Created container with name: {container.name}, "
|
||||
f"image: {container.image} and args: {container.args}"
|
||||
)
|
||||
@@ -54,9 +54,10 @@ def create_job(job_name, pod_template):
|
||||
|
||||
r = redis.Redis(unix_socket_path='/tmp/redis.sock')
|
||||
|
||||
# create logger
|
||||
logging.basicConfig(level=os.environ.get("LOGLEVEL", "DEBUG"))
|
||||
logging.info('Starting script for KubeInvaders programming mode')
|
||||
logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO"))
|
||||
logging.getLogger('kubernetes').setLevel(logging.ERROR)
|
||||
|
||||
logging.debug('Starting script for KubeInvaders programming mode')
|
||||
|
||||
configuration = client.Configuration()
|
||||
token = os.environ["TOKEN"]
|
||||
@@ -77,31 +78,31 @@ while True:
|
||||
label_selector="chaos-controller=kubeinvaders"
|
||||
api_response = api_instance.list_pod_for_all_namespaces(label_selector=label_selector)
|
||||
except ApiException as e:
|
||||
logging.info(e)
|
||||
logging.debug(e)
|
||||
|
||||
r.set("current_chaos_job_pod", 0)
|
||||
|
||||
for pod in api_response.items:
|
||||
if pod.status.phase == "Pending" or pod.status.phase == "Running":
|
||||
logging.info(f"[k-inv][metrics_loop] Found pod {pod.metadata.name}. It is in {pod.status.phase} phase. Incrementing current_chaos_job_pod Redis key")
|
||||
logging.debug(f"[k-inv][metrics_loop] Found pod {pod.metadata.name}. It is in {pod.status.phase} phase. Incrementing current_chaos_job_pod Redis key")
|
||||
r.incr('current_chaos_job_pod')
|
||||
|
||||
if pod.status.phase != "Pending" and pod.status.phase != "Running" and not r.exists(f"pod:time:{pod.metadata.namespace}:{pod.metadata.name}"):
|
||||
logging.info(f"[k-inv][metrics_loop] Found pod {pod.metadata.name}. It is in {pod.status.phase} phase. Tracking time in pod:time:{pod.metadata.namespace}:{pod.metadata.name} Redis key")
|
||||
logging.debug(f"[k-inv][metrics_loop] Found pod {pod.metadata.name}. It is in {pod.status.phase} phase. Tracking time in pod:time:{pod.metadata.namespace}:{pod.metadata.name} Redis key")
|
||||
r.set(f"pod:time:{pod.metadata.namespace}:{pod.metadata.name}", int(time.time()))
|
||||
|
||||
elif pod.status.phase != "Pending" and pod.status.phase != "Running" and r.exists(f"pod:time:{pod.metadata.namespace}:{pod.metadata.name}"):
|
||||
logging.info(f"[k-inv][metrics_loop] Found pod {pod.metadata.name}. It is in {pod.status.phase} phase. Comparing time in pod:time:{pod.metadata.namespace}:{pod.metadata.name} Redis key with now")
|
||||
logging.debug(f"[k-inv][metrics_loop] Found pod {pod.metadata.name}. It is in {pod.status.phase} phase. Comparing time in pod:time:{pod.metadata.namespace}:{pod.metadata.name} Redis key with now")
|
||||
now = int(time.time())
|
||||
pod_time = int(r.get(f"pod:time:{pod.metadata.namespace}:{pod.metadata.name}"))
|
||||
logging.info(f"[k-inv][metrics_loop] For {pod.metadata.name} comparing now:{now} with pod_time:{pod_time}")
|
||||
if (now - pod_time > 30):
|
||||
logging.debug(f"[k-inv][metrics_loop] For {pod.metadata.name} comparing now:{now} with pod_time:{pod_time}")
|
||||
if (now - pod_time > 240):
|
||||
try:
|
||||
api_instance.delete_namespaced_pod(pod.metadata.name, namespace = pod.metadata.namespace)
|
||||
logging.info(f"[k-inv][metrics_loop] Deleting pod {pod.metadata.name}")
|
||||
logging.debug(f"[k-inv][metrics_loop] Deleting pod {pod.metadata.name}")
|
||||
r.delete(f"pod:time:{pod.metadata.namespace}:{pod.metadata.name}")
|
||||
except ApiException as e:
|
||||
logging.info(e)
|
||||
logging.debug(e)
|
||||
if pod.metadata.labels.get('chaos-codename') != None:
|
||||
codename = pod.metadata.labels.get('chaos-codename')
|
||||
job_name = pod.metadata.labels.get('job-name')
|
||||
|
||||
Reference in New Issue
Block a user