some fixes

This commit is contained in:
Eugenio Marzo
2026-03-14 18:28:10 +01:00
parent 6cb6ce29d2
commit c036e6a30f
6 changed files with 307 additions and 45 deletions

View File

@@ -6,11 +6,10 @@
<link rel="icon" type="image/x-icon" href="./images/favicon.ico">
<meta name="viewport" content="width=device-width, initial-scale=1">
<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/js/bootstrap.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/js/bootstrap.bundle.min.js"></script>
<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/css/bootstrap.min.css" rel="stylesheet">
<script src="https://cdn.jsdelivr.net/npm/codemirror@5.65.18/lib/codemirror.min.js"></script>
<link href="https://cdn.jsdelivr.net/npm/codemirror@5.65.18/lib/codemirror.min.css" rel="stylesheet">
<script src="https://cdn.jsdelivr.net/npm/@popperjs/core@2.11.8/dist/umd/popper.min.js "></script>
<script src="https://cdn.jsdelivr.net/npm/jquery@3.7.1/dist/jquery.min.js "></script>
<script src="https://cdn.jsdelivr.net/npm/echarts@5.5.1/dist/echarts.min.js"></script>
@@ -379,6 +378,162 @@
</div>
</div>
<!-- Modal: Kubernetes Namespaces Help -->
<div class="modal fade" id="k8sNamespacesHelpModal" tabindex="-1" aria-labelledby="k8sNamespacesHelpModalLabel" aria-hidden="true">
<div class="modal-dialog modal-lg modal-dialog-scrollable">
<div class="modal-content">
<div class="modal-header">
<h5 class="modal-title" id="k8sNamespacesHelpModalLabel">Kubernetes Namespaces</h5>
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
</div>
<div class="modal-body">
<p style="font-size: small; font-weight: normal;">Provide a <strong>comma-separated</strong> list of namespaces that KubeInvaders should target. Pods and resources across all listed namespaces will be included in the chaos session.</p>
<p style="font-size: small; font-weight: normal;"><strong>Example:</strong></p>
<div style="position: relative;">
<button class="btn btn-sm btn-outline-secondary" style="position: absolute; top: 6px; right: 6px; font-size: 11px;" onclick="copyToClipboard('k8sNamespacesExample')">Copy</button>
<pre id="k8sNamespacesExample" style="background: #1e1e1e; color: #d4d4d4; padding: 14px; border-radius: 6px; font-size: 11px;">default,kube-system,my-app</pre>
</div>
<p style="font-size: small; font-weight: normal; margin-top: 12px;">To list available namespaces in your cluster, run:</p>
<div style="position: relative;">
<button class="btn btn-sm btn-outline-secondary" style="position: absolute; top: 6px; right: 6px; font-size: 11px;" onclick="copyToClipboard('k8sGetNamespacesCmd')">Copy</button>
<pre id="k8sGetNamespacesCmd" style="background: #1e1e1e; color: #d4d4d4; padding: 14px; border-radius: 6px; font-size: 11px;">kubectl get namespaces</pre>
</div>
</div>
<div class="modal-footer">
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Close</button>
</div>
</div>
</div>
</div>
<!-- Modal: Kubernetes Token Help -->
<div class="modal fade" id="k8sTokenHelpModal" tabindex="-1" aria-labelledby="k8sTokenHelpModalLabel" aria-hidden="true">
<div class="modal-dialog modal-lg modal-dialog-scrollable">
<div class="modal-content">
<div class="modal-header">
<h5 class="modal-title" id="k8sTokenHelpModalLabel">How to create a Kubernetes Service Account Token</h5>
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
</div>
<div class="modal-body">
<p style="font-size: small; font-weight: normal;">Run the following command to create the required service account, RBAC roles, and a long-lived token secret:</p>
<div style="position: relative;">
<button class="btn btn-sm btn-outline-secondary" style="position: absolute; top: 6px; right: 6px; font-size: 11px;" onclick="copyK8sTokenScript()">Copy</button>
<pre id="k8sTokenScript" style="background: #1e1e1e; color: #d4d4d4; padding: 14px; border-radius: 6px; font-size: 11px; overflow-x: auto; white-space: pre;">cat &lt;&lt; 'EOF' | kubectl apply -f -
apiVersion: v1
kind: Namespace
metadata:
name: kubeinvaders
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: kinv-cr
rules:
- apiGroups:
- ""
resources:
- pods
- pods/log
verbs:
- delete
- apiGroups:
- batch
- extensions
resources:
- jobs
verbs:
- get
- list
- watch
- create
- update
- patch
- delete
- apiGroups:
- "*"
resources:
- "*"
verbs:
- get
- watch
- list
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: kinv-sa
namespace: kubeinvaders
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: kinv-crb
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: kinv-cr
subjects:
- kind: ServiceAccount
name: kinv-sa
namespace: kubeinvaders
---
apiVersion: v1
kind: Secret
type: kubernetes.io/service-account-token
metadata:
name: kinv-sa-token
namespace: kubeinvaders
annotations:
kubernetes.io/service-account.name: kinv-sa
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
namespace: default
name: kubevirt-vm-restart-role
rules:
- apiGroups: ["subresources.kubevirt.io"]
resources: ["virtualmachines/restart"]
verbs: ["update"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: kubevirt-vm-restart-binding
namespace: default
subjects:
- kind: ServiceAccount
name: kubeinvaders
namespace: kubeinvaders
roleRef:
kind: ClusterRole
name: kubevirt-vm-restart-role
apiGroup: rbac.authorization.k8s.io
EOF</pre>
</div>
<p style="font-size: small; font-weight: normal; margin-top: 12px;">After applying, retrieve the token with:</p>
<div style="position: relative;">
<button class="btn btn-sm btn-outline-secondary" style="position: absolute; top: 6px; right: 6px; font-size: 11px;" onclick="copyToClipboard('k8sGetTokenCmd')">Copy</button>
<pre id="k8sGetTokenCmd" style="background: #1e1e1e; color: #d4d4d4; padding: 14px; border-radius: 6px; font-size: 11px;">kubectl get secret kinv-sa-token -n kubeinvaders -o jsonpath='{.data.token}' | base64 --decode</pre>
</div>
</div>
<div class="modal-footer">
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Close</button>
</div>
</div>
</div>
</div>
<script>
function copyToClipboard(elementId) {
const text = document.getElementById(elementId).innerText;
navigator.clipboard.writeText(text);
}
function copyK8sTokenScript() {
const raw = document.getElementById('k8sTokenScript').innerText;
navigator.clipboard.writeText(raw);
}
</script>
<div class="container" id="gameContainer">
<!-- START FIRST ROW -->
<div class="row custom-btn-group" style="margin-top: 2%;">
@@ -392,10 +547,10 @@
<label for="k8s_api_endpoint">Kubernetes API Endpoint</label>
<input type="text" id="k8s_api_endpoint" name="k8s_api_endpoint" placeholder="https://kubernetes.default.svc">
<label for="k8s_token">Kubernetes Token</label>
<label for="k8s_token">Kubernetes Token &nbsp;<button type="button" class="btn btn-sm btn-outline-secondary" data-bs-toggle="modal" data-bs-target="#k8sTokenHelpModal" title="How to create a service account token" style="padding: 1px 6px; font-size: 11px; line-height: 1.4;">&#9432;</button></label>
<input type="password" id="k8s_token" name="k8s_token" placeholder="Enter your token">
<label for="k8s_namespaces">Kubernetes Namespaces</label>
<label for="k8s_namespaces">Kubernetes Namespaces &nbsp;<button type="button" class="btn btn-sm btn-outline-secondary" data-bs-toggle="modal" data-bs-target="#k8sNamespacesHelpModal" title="About Kubernetes Namespaces" style="padding: 1px 6px; font-size: 11px; line-height: 1.4;">&#9432;</button></label>
<input
type="text"
id="k8s_namespaces"

View File

@@ -247,21 +247,6 @@ function sendSavedChaosReport() {
chaos_report_start_date = new Date();
}
//startGameMode()
chaos_report_switch = true;
document.getElementById("httpStatsCanvasDiv").style.display = "block";
document.getElementById("chartDiv").style.display = "block";
drawCanvasHTTPStatusCodeStats();
chaosReportprojectName = presetBodyDict["chaosReportProject"];
$("#chaosReportAuthorDiv").html(presetBodyDict["chaosReportAuthor"]);
$("#chaosReportProjectDiv").html(presetBodyDict["chaosReportProject"]);
$("#chaosReportDateDiv").html(chaos_report_start_date.toLocaleString());
$("#chaosReportSessionTimeDiv").html(diffBetweenTwoDates(chaos_report_start_date, new Date()) + " seconds");
$("#chaosReportCheckSiteURLDiv").html(presetBodyDict["chaosReportCheckSiteURL"]);
if (headerAreLikePythonRequestHeaders(presetBodyDict["chaosReportCheckSiteURLHeaders"]) == false) {
alert("Invalid headers. Insert them like this: \"Content-Type\": \"application/json; charset=utf-8\";\"Authorization\"");
return;
@@ -277,6 +262,20 @@ function sendSavedChaosReport() {
return;
}
// Enable charts only after all validations pass.
chaos_report_switch = true;
chaosReportprojectName = presetBodyDict["chaosReportProject"];
document.getElementById("httpStatsCanvasDiv").style.display = "block";
document.getElementById("chartDiv").style.display = "block";
$("#chaosReportAuthorDiv").html(presetBodyDict["chaosReportAuthor"]);
$("#chaosReportProjectDiv").html(presetBodyDict["chaosReportProject"]);
$("#chaosReportDateDiv").html(chaos_report_start_date.toLocaleString());
$("#chaosReportSessionTimeDiv").html(diffBetweenTwoDates(chaos_report_start_date, new Date()) + " seconds");
$("#chaosReportCheckSiteURLDiv").html(presetBodyDict["chaosReportCheckSiteURL"]);
drawCanvasHTTPStatusCodeStats();
var oReq = new XMLHttpRequest();
oReq.open("POST", k8s_url + "/chaos/report/save?project=" + $("#chaosReportProject").val(), true);
@@ -291,6 +290,7 @@ function sendSavedChaosReport() {
oReq.send(JSON.stringify(presetBodyDict));
closePrepareChaosReportModal();
resizeCharts();
setTimeout(resizeCharts, 100);
document.getElementById("myCanvas").scrollIntoView(true);
document.getElementById("flagChaosReport").checked = true;
$('#alert_placeholder').replaceWith(alert_div + 'RETURN TO TOP, PRESS START TO BEGIN AUTOMATIC SESSION </div>');
@@ -320,6 +320,10 @@ function saveChaosReport() {
}
function updateElapsedTimeArray(projectName) {
if (!projectName || projectName.trim() === "") {
return;
}
$("#chaosReportSessionTimeDiv").html(diffBetweenTwoDates(chaos_report_start_date, new Date()) + " seconds");
// console.log("[SAVE-CHAOS-REPORT-CONF] Diff Between Dates: " + toString(diffBetweenTwoDates(chaos_report_start_date, new Date())));
// console.log("[SAVE-CHAOS-REPORT-CONF] Updating elapsed time array for project: " + projectName);
@@ -330,10 +334,12 @@ function updateElapsedTimeArray(projectName) {
oReq.onreadystatechange = function () {
if (this.readyState === XMLHttpRequest.DONE && this.status === 200) {
// console.log("[SAVE-CHAOS-REPORT-CONF] Elapsed time array received from Redis: " + parseFloat(this.responseText));
chaos_report_http_elapsed_time_array.push(parseFloat(this.responseText));
while (chaos_report_http_elapsed_time_array.length > 40) {
chaos_report_http_elapsed_time_array.shift();
var elapsed = parseFloat(this.responseText);
if (!isNaN(elapsed) && isFinite(elapsed)) {
chaos_report_http_elapsed_time_array.push(elapsed);
while (chaos_report_http_elapsed_time_array.length > 40) {
chaos_report_http_elapsed_time_array.shift();
}
}
}
};;
@@ -345,6 +351,10 @@ function updateElapsedTimeArray(projectName) {
}
function updateStatusCodePieChart(projectName) {
if (!projectName || projectName.trim() === "") {
return;
}
// console.log("[SAVE-CHAOS-REPORT-CONF] Updating Status Code Pie Chart for project: " + projectName);
var oReq = new XMLHttpRequest();
@@ -354,9 +364,18 @@ function updateStatusCodePieChart(projectName) {
oReq.onreadystatechange = function () {
if (this.readyState === XMLHttpRequest.DONE && this.status === 200) {
var status_code = this.responseText.trim();
// console.log("[SAVE-CHAOS-REPORT-CONF] Status Code Pie Chart received from Redis: |" + status_code + "|");
if (status_code === "" || status_code === "Key not found") {
return;
}
if (!(status_code in chart_status_code_dict)) {
if (/^\d{3}$/.test(status_code)) {
status_code = "Other";
} else {
status_code = "Connection Error";
}
}
chart_status_code_dict[status_code] = chart_status_code_dict[status_code] + 1
chart_status_code_dict[status_code] = (chart_status_code_dict[status_code] || 0) + 1
myHTTPStatusCodeChart.setOption({
series: [

View File

@@ -841,7 +841,13 @@ window.setInterval(function draw() {
ctx.fillStyle = 'white';
ctx.font = "18px 'Ubuntu Mono'";
ctx.fillText('Cluster: ' + endpoint, 10, startYforHelp);
if (localStorage.getItem('k8s_api_endpoint') != "") {
ctx.fillText('API Endpoint: ' + localStorage.getItem('k8s_api_endpoint'), 10, startYforHelp);
}
else if (endpoint != "") {
ctx.fillText('Cluster: ' + endpoint, 10, startYforHelp);
}
ctx.fillText('Current Namespace: ' + namespace, 10, startYforHelp + 20);
ctx.fillText('Alien Shuffle: ' + shuffle, 10, startYforHelp + 40);
ctx.fillText('Auto Namespaces Switch: ' + namespacesJumpStatus, 10, startYforHelp + 60);
@@ -1018,7 +1024,10 @@ document.addEventListener("keyup", keyUpHandler, false);
setSystemSettings();
waitForReachableK8sUrl(function () {
getEndpoint();
if (endpoint != null && endpoint != "") {
console.log("[K-INV] Connected to Kubernetes API at " + endpoint);
getEndpoint();
}
getNamespaces();
getSavedPresets();
});

View File

@@ -217,10 +217,21 @@ server {
ngx.header['Access-Control-Allow-Methods'] = 'GET, POST, OPTIONS'
ngx.header['Access-Control-Allow-Headers'] = 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range'
ngx.header['Access-Control-Expose-Headers'] = 'Content-Length,Content-Range';
if os.getenv("APPLICATION_URL") == nil then
ngx.say("Error! APPLICATION_URL is nil!")
local endpoint = os.getenv("APPLICATION_URL")
if not endpoint or endpoint == "" then
if os.getenv("KUBERNETES_SERVICE_HOST") and os.getenv("KUBERNETES_SERVICE_PORT_HTTPS") then
endpoint = "https://" .. os.getenv("KUBERNETES_SERVICE_HOST") .. ":" .. os.getenv("KUBERNETES_SERVICE_PORT_HTTPS")
else
endpoint = os.getenv("ENDPOINT")
end
end
if not endpoint or endpoint == "" then
ngx.status = 500
ngx.say("Error! No cluster endpoint configured (APPLICATION_URL, KUBERNETES_SERVICE_HOST, or ENDPOINT).")
else
ngx.say(os.getenv("APPLICATION_URL"))
ngx.say(endpoint)
end
}
}
@@ -651,7 +662,7 @@ server {
else
red:set("chaos_report_project_list", args["project"])
end
red:expire("chaos_report_project_list", 5)
red:expire("chaos_report_project_list", 120)
}
}
@@ -689,6 +700,30 @@ server {
local okredis, errredis = red:connect("unix:/tmp/redis.sock")
red:set("chaos_report_project_" .. args["project"], data)
local project = args["project"] or ""
if project ~= "" then
local check_url_counter_key = project .. "_check_url_counter"
local check_url_status_code_key = project .. "_check_url_status_code"
local check_url_elapsed_time_key = project .. "_check_url_elapsed_time"
local check_url_start_time_key = project .. "_check_url_start_time"
if red:exists(check_url_counter_key) == 0 then
red:set(check_url_counter_key, 0)
end
if red:exists(check_url_status_code_key) == 0 then
red:set(check_url_status_code_key, "Other")
end
if red:exists(check_url_elapsed_time_key) == 0 then
red:set(check_url_elapsed_time_key, 0)
end
if red:exists(check_url_start_time_key) == 0 then
red:set(check_url_start_time_key, os.date("%Y-%m-%d %H:%M:%S"))
end
end
if check_if_redis_key_exists(red, "chaos_report_project_list") then
if not check_if_string_contain_project(red:get("chaos_report_project_list"), args["project"]) then
red:set("chaos_report_project_list", add_project(red:get("chaos_report_project_list"), args["project"]))
@@ -697,7 +732,7 @@ server {
red:set("chaos_report_project_list", args["project"])
end
red:expire("chaos_report_project_list", 5)
red:expire("chaos_report_project_list", 120)
-- ngx.log(ngx.INFO, "[CHAOS-REPORT-SAVE] " .. data)
}

View File

@@ -22,11 +22,22 @@ def join_string_to_array(project_list_string, separator):
def do_http_request(url, method, headers, data):
try:
response = requests.request(method, url, headers=headers, data=data, verify=False, allow_redirects=True, timeout=10)
elaped_time = response.elapsed.total_seconds()
# Do not inherit HTTP(S)_PROXY from the container environment;
# those proxies often cannot resolve in-cluster or local ingress hosts.
with requests.Session() as session:
session.trust_env = False
response = session.request(
method,
url,
headers=headers,
data=data,
verify=False,
allow_redirects=True,
timeout=10,
)
return response
except requests.exceptions.RequestException as e:
logging.error(f"Error while sending HTTP request: {e}")
logging.error(f"Error while sending HTTP request to {url} with method {method}: {e}")
return "Connection Error"
def check_if_json_is_valid(json_data):
@@ -75,7 +86,12 @@ def create_job(job_name, pod_template):
r = redis.Redis(unix_socket_path='/tmp/redis.sock')
logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO"))
logging.basicConfig(
level=os.environ.get("LOGLEVEL", "INFO"),
stream=sys.stdout,
format="%(asctime)s %(levelname)s %(message)s",
force=True,
)
logging.getLogger('kubernetes').setLevel(logging.ERROR)
logging.debug('Starting script for KubeInvaders metrics loop')
@@ -97,19 +113,45 @@ batch_api = client.BatchV1Api()
while True:
#logging.info(f"[k-inv][metrics_loop] Metrics loop is active - {r.exists('chaos_report_project_list')}")
if r.exists('chaos_report_project_list'):
#logging.info("[k-inv][metrics_loop] Found chaos_report_project_list Redis Key")
logging.info("[k-inv][metrics_loop] Found chaos_report_project_list Redis Key")
for project in join_string_to_array(r.get('chaos_report_project_list').decode(), ','):
#logging.info(f"[k-inv][metrics_loop] Computing Chaos Report Project: {project}")
logging.info(f"[k-inv][metrics_loop] Computing Chaos Report Project: {project}")
chaos_program_key = f"chaos_report_project_{project}"
if r.exists(chaos_program_key):
#logging.info(f"[k-inv][metrics_loop][chaos_report] Found chaos_report_project_{project} key in Redis. Starting {project} ")
logging.info(f"[k-inv][metrics_loop][chaos_report] Found chaos_report_project_{project} key in Redis. Starting {project} ")
if check_if_json_is_valid(r.get(chaos_program_key)):
chaos_report_program = json.loads(r.get(chaos_program_key))
now = datetime.datetime.now()
#logging.info(f"[k-inv][metrics_loop][chaos_report] chaos_report_program is valid JSON: {chaos_report_program}")
response = do_http_request(chaos_report_program['chaosReportCheckSiteURL'], chaos_report_program['chaosReportCheckSiteURLMethod'], json.loads(chaos_report_program['chaosReportCheckSiteURLHeaders']), chaos_report_program['chaosReportCheckSiteURLPayload'])
logging.info(f"[k-inv][metrics_loop][chaos_report] chaos_report_program is valid JSON: {chaos_report_program}")
url = str(chaos_report_program.get('chaosReportCheckSiteURL', '')).strip()
method = str(chaos_report_program.get('chaosReportCheckSiteURLMethod', 'GET')).strip().upper()
payload = chaos_report_program.get('chaosReportCheckSiteURLPayload', '')
logging.info(
f"[k-inv][metrics_loop][chaos_report] project={project} parsed url='{url}' method='{method}'"
)
headers = {"Content-Type": "application/json; charset=utf-8"}
raw_headers = chaos_report_program.get('chaosReportCheckSiteURLHeaders', '{}')
try:
parsed_headers = json.loads(raw_headers) if isinstance(raw_headers, str) else raw_headers
if isinstance(parsed_headers, dict):
headers = parsed_headers
except Exception as e:
logging.warning(f"Invalid chaos report headers for project {project}: {e}")
if method not in ["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"]:
method = "GET"
if not url:
logging.warning(
f"[k-inv][metrics_loop][chaos_report] Empty chaosReportCheckSiteURL for project={project}. Raw program: {chaos_report_program}"
)
response = "Connection Error"
else:
response = do_http_request(url, method, headers, payload)
check_url_counter_key = f"{chaos_report_program['chaosReportProject']}_check_url_counter"
check_url_status_code_key = f"{chaos_report_program['chaosReportProject']}_check_url_status_code"
check_url_elapsed_time_key = f"{chaos_report_program['chaosReportProject']}_check_url_elapsed_time"
@@ -124,11 +166,11 @@ while True:
r.set(check_url_start_time, now.strftime("%Y-%m-%d %H:%M:%S"))
if response == "Connection Error":
#logging.info(f"[k-inv][metrics_loop][chaos_report] Connection Error while checking {chaos_report_program['chaosReportCheckSiteURL']}")
logging.info(f"[k-inv][metrics_loop][chaos_report] Connection Error while checking {chaos_report_program['chaosReportCheckSiteURL']}")
r.set(check_url_status_code_key, "Connection Error")
r.set(check_url_elapsed_time_key, 0)
else:
#logging.info(f"[k-inv][metrics_loop][chaos_report] Status code {response.status_code} while checking {chaos_report_program['chaosReportCheckSiteURL']}")
logging.info(f"[k-inv][metrics_loop][chaos_report] Status code {response.status_code} while checking {chaos_report_program['chaosReportCheckSiteURL']}")
r.set(check_url_status_code_key, response.status_code)
r.set(check_url_elapsed_time_key, float(response.elapsed.total_seconds()))
try:

View File

@@ -8,11 +8,13 @@ else
export TOKEN="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
fi
export PYTHONUNBUFFERED=1
if python3 -c "import urllib.request; urllib.request.urlopen('https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT_HTTPS}', timeout=5)" 2>/dev/null; then
python3 /opt/metrics_loop/start.py https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT_HTTPS} &
python3 -u /opt/metrics_loop/start.py https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT_HTTPS} &
while true
do
pgrep -a -f -c "^python3.*metrics_loop.*$" > /dev/null || ( python3 /opt/metrics_loop/start.py https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT_HTTPS} & )
pgrep -a -f -c "^python3.*metrics_loop.*$" > /dev/null || ( python3 -u /opt/metrics_loop/start.py https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT_HTTPS} & )
sleep 2
done
fi