mirror of
https://github.com/lucky-sideburn/kubeinvaders.git
synced 2026-08-23 13:36:29 +00:00
Merge branch 'feature/programmming_mode' of ssh://github.com/lucky-sideburn/KubeInvaders
This commit is contained in:
Eugenio Marzo
+5
-1
@@ -41,7 +41,6 @@ RUN sed -i.bak 's/listen\(.*\)80;/listen 8081;/' /etc/nginx/conf.d/default.conf
|
||||
RUN mkdir /usr/local/openresty/nginx/conf/kubeinvaders
|
||||
|
||||
COPY nginx/nginx.conf /etc/nginx/nginx.conf
|
||||
|
||||
COPY scripts/metrics.lua /usr/local/openresty/nginx/conf/kubeinvaders/metrics.lua
|
||||
COPY scripts/pod.lua /usr/local/openresty/nginx/conf/kubeinvaders/pod.lua
|
||||
COPY scripts/node.lua /usr/local/openresty/nginx/conf/kubeinvaders/node.lua
|
||||
@@ -51,7 +50,12 @@ COPY scripts/chaos-containers.lua /usr/local/openresty/nginx/conf/kubeinvaders/c
|
||||
COPY scripts/programming_mode.lua /usr/local/openresty/nginx/conf/kubeinvaders/programming_mode.lua
|
||||
COPY scripts/config_kubeinv.lua /usr/local/openresty/lualib/config_kubeinv.lua
|
||||
COPY scripts/programming_mode /opt/programming_mode/
|
||||
COPY scripts/metrics_loop /opt/metrics_loop/
|
||||
COPY scripts/logs_loop /opt/logs_loop/
|
||||
|
||||
RUN pip3 install -r /opt/programming_mode/requirements.txt
|
||||
RUN pip3 install -r /opt/programming_mode/requirements.txt
|
||||
|
||||
COPY nginx/KubeInvaders.conf /etc/nginx/conf.d/KubeInvaders.conf
|
||||
RUN chmod g+rwx /var/cache/nginx /var/run /var/log/nginx /var/www/html /etc/nginx/conf.d
|
||||
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
#! /bin/bash
|
||||
kubectl exec -it $(kubectl get pods -n kubeinvaders | grep 'kubeinvaders-' | awk '{ print $1 }') -n kubeinvaders bash
|
||||
|
||||
@@ -10,4 +10,10 @@ fi
|
||||
# TODO: use a sidecar
|
||||
redis-server /etc/redis/redis.conf &
|
||||
|
||||
# TODO: use a sidecar
|
||||
bash /opt/metrics_loop/start.sh &
|
||||
bash /opt/logs_loop/start.sh &
|
||||
|
||||
echo '<p>waiting for logs...</p>' > /var/www/html/chaoslogs.html
|
||||
|
||||
nginx -c /etc/nginx/nginx.conf -g 'daemon off;'
|
||||
|
||||
@@ -12,12 +12,12 @@ metadata:
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["pods", "pods/log"]
|
||||
verbs: ["get", "watch", "list", "delete"]
|
||||
verbs: ["delete"]
|
||||
- apiGroups: ["batch", "extensions"]
|
||||
resources: ["jobs"]
|
||||
verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
|
||||
- apiGroups: [""]
|
||||
resources: ["nodes"]
|
||||
- apiGroups: ["*"]
|
||||
resources: ["*"]
|
||||
verbs: ["get", "watch", "list"]
|
||||
{{- end -}}
|
||||
{{- if or .Values.clusterRole.create .Values.clusterRole.name }}
|
||||
|
||||
+3
-2
@@ -44,7 +44,7 @@
|
||||
<div class="modal-dialog" role="document">
|
||||
<div class="modal-content">
|
||||
<div class="modal-body">
|
||||
<pre id="currentChaosContainrYaml"></pre>
|
||||
<pre id="currentChaosContainerYaml"></pre>
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
<button type="button" class="btn btn-light" data-dismiss="modal" onclick="closeCurrentChaosJobModal()">Close</button>
|
||||
@@ -169,7 +169,7 @@
|
||||
<div style="margin-top: 1%;">
|
||||
<span style="color: #000000;">Deleted Pod Total: <span id="deleted_pods_total" style="color: #8f00bb">0</span> - </span>
|
||||
<span style="color: #000000">Chaos Jobs Total: <span id="chaos_jobs_total" style="color: #8f00bb">0</span> - </span>
|
||||
<span style="color: #000000">Current Chaos Jobs: <span id="chaos_jobs_current" style="color: #8f00bb">0</span> - </span>
|
||||
<span style="color: #000000">Current Chaos Jobs Pods: <span id="current_chaos_job_pod" style="color: #8f00bb">0</span> - </span>
|
||||
<span style="color: #000000">Not Running Pods: <span id="pods_not_running_on" style="color: #8f00bb">0</span> - </span>
|
||||
<span style="color: #000000">Current Replicas State Delay: </span><span id="fewer_replicas_seconds" style="color: #8f00bb">0</span><font color="#4f4f4f"> sec</font> - </span>
|
||||
<span style="color: #000000">Latest Replicas State Delay: <span id="latest_fewer_replicas_seconds" style="color: #8f00bb">0</span><font color="#4f4f4f"> sec</font> </span>
|
||||
@@ -207,6 +207,7 @@ experiments:
|
||||
</textarea>
|
||||
</div>
|
||||
</form>
|
||||
<div id="chaosJobLogsDiv" style="margin-top: 2%; overflow-y: scroll; height:700px;"></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
+17
-3
@@ -132,8 +132,8 @@ function getMetrics() {
|
||||
else if (metric[0] == "pods_not_running_on_selected_ns") {
|
||||
$('#pods_not_running_on').text(metric[1]);
|
||||
}
|
||||
else if (metric[0] == "chaos_jobs_current") {
|
||||
$('#chaos_jobs_current').text(metric[1]);
|
||||
else if (metric[0] == "current_chaos_job_pod") {
|
||||
$('#current_chaos_job_pod').text(metric[1]);
|
||||
}
|
||||
}
|
||||
};;
|
||||
@@ -141,6 +141,16 @@ function getMetrics() {
|
||||
oReq.send();
|
||||
}
|
||||
|
||||
function getChaosJobsLogs() {
|
||||
var oReq = new XMLHttpRequest();
|
||||
oReq.onload = function () {
|
||||
document.getElementById("chaosJobLogsDiv").innerHTML = "";
|
||||
document.getElementById("chaosJobLogsDiv").innerHTML = this.responseText;
|
||||
};;
|
||||
oReq.open("GET", "https://" + clu_endpoint + "/chaoslogs.html");
|
||||
oReq.send();
|
||||
}
|
||||
|
||||
function runKubeLinter() {
|
||||
$('#kubeLinterModal').modal('show');
|
||||
modal_opened = true;
|
||||
@@ -178,7 +188,7 @@ function getCurrentChaosContainer() {
|
||||
oReq.onload = function () {
|
||||
//console.log(this.responseText);
|
||||
job_parsed = JSON.stringify(JSON.parse(this.responseText), null, 4);
|
||||
$('#currentChaosContainrYaml').text(job_parsed);
|
||||
$('#currentChaosContainerYaml').text(job_parsed);
|
||||
$('#currentChaosContainerJsonTextArea').val(job_parsed);
|
||||
};;
|
||||
oReq.open("GET", "https://" + clu_endpoint + "/kube/chaos/containers?action=container_definition");
|
||||
@@ -711,6 +721,10 @@ window.setInterval(function metrics() {
|
||||
if (game_mode_switch || programming_mode_switch) {
|
||||
getMetrics()
|
||||
}
|
||||
|
||||
if (programming_mode_switch) {
|
||||
getChaosJobsLogs()
|
||||
}
|
||||
}, 2000)
|
||||
|
||||
getEndpoint();
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
redis
|
||||
pyyaml
|
||||
requests
|
||||
kubernetes
|
||||
@@ -0,0 +1,76 @@
|
||||
from asyncio.log import logger
|
||||
import yaml
|
||||
import logging
|
||||
import os
|
||||
import pathlib
|
||||
import sys
|
||||
from kubernetes import client, config
|
||||
from kubernetes.client.rest import ApiException
|
||||
import requests
|
||||
from string import Template
|
||||
import string
|
||||
import random
|
||||
import redis
|
||||
import time
|
||||
|
||||
file = pathlib.Path('/tmp/redis.sock')
|
||||
if file.exists():
|
||||
r = redis.Redis(unix_socket_path='/tmp/redis.sock', charset="utf-8", decode_responses=True)
|
||||
else:
|
||||
r = redis.Redis("127.0.0.1", charset="utf-8", decode_responses=True)
|
||||
|
||||
# create logger
|
||||
logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO"))
|
||||
logging.info('Starting script for KubeInvaders programming mode')
|
||||
|
||||
configuration = client.Configuration()
|
||||
token = os.environ["TOKEN"]
|
||||
configuration.api_key = {"authorization": f"Bearer {token}"}
|
||||
configuration.host = sys.argv[1]
|
||||
|
||||
configuration.insecure_skip_tls_verify = True
|
||||
configuration.verify_ssl = False
|
||||
|
||||
client.Configuration.set_default(configuration)
|
||||
client.Configuration.set_default(configuration)
|
||||
|
||||
api_instance = client.CoreV1Api()
|
||||
batch_api = client.BatchV1Api()
|
||||
namespace = "kubeinvaders"
|
||||
|
||||
#for key in r.scan_iter("log:*"):
|
||||
# r.delete(key)
|
||||
|
||||
while True:
|
||||
try:
|
||||
api_response = api_instance.list_namespaced_pod(namespace="kubeinvaders")
|
||||
except ApiException as e:
|
||||
logging.info(e)
|
||||
|
||||
for pod in api_response.items:
|
||||
if pod.metadata.labels.get('approle') != None and pod.metadata.labels['approle'] == 'chaosnode' and pod.status.phase != "Pending":
|
||||
try:
|
||||
logging.info(f"Reading logs of {pod.metadata.name}")
|
||||
api_response = api_instance.read_namespaced_pod_log(name=pod.metadata.name, namespace='kubeinvaders')
|
||||
logrow = f"<div class='row'><div style='margin-top: 2%; background-color:#400075; color: #e9f0ef; text-align: center;' class='alert' role='alert'>Pod Name: {pod.metadata.name} (Log TTL: 30sec)</div><div style='margin-top: 1.5px; background-color:#000000; color: #e9f0ef; font-size: 12px; font-family: Courier New, Courier, monospace;' class='alert' role='alert'>{api_response}</div></div>"
|
||||
r.set(f"log:{pod.metadata.name}", logrow)
|
||||
r.expire(f"log:{pod.metadata.name}", 30)
|
||||
logging.info(f"Phase of {pod.metadata.name} is {pod.status.phase}")
|
||||
if pod.status.phase == "Succeeded":
|
||||
try:
|
||||
api_response = api_instance.delete_namespaced_pod(pod.metadata.name, namespace='kubeinvaders')
|
||||
logging.info(f"Deleted pod {pod.metadata.name}")
|
||||
except ApiException as e:
|
||||
logging.info(e)
|
||||
except ApiException as e:
|
||||
logging.info(e)
|
||||
file = pathlib.Path('/var/www/html')
|
||||
if file.exists():
|
||||
log_html_file = pathlib.Path('/var/www/html/chaoslogs.html')
|
||||
if log_html_file.exists():
|
||||
with open("/var/www/html/chaoslogs.html", "w") as myfile:
|
||||
myfile.write('')
|
||||
for key in r.scan_iter("log:*"):
|
||||
with open("/var/www/html/chaoslogs.html", "a") as myfile:
|
||||
myfile.write(str(r.get(key)))
|
||||
time.sleep(1)
|
||||
Executable
+11
@@ -0,0 +1,11 @@
|
||||
#!/bin/sh
|
||||
|
||||
if [ ! -z "$K8S_TOKEN" ];then
|
||||
echo 'Found K8S_TOKEN... using K8S_TOKEN instead of TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)'
|
||||
export TOKEN=$K8S_TOKEN
|
||||
else
|
||||
# Source the service account token from the container directly.
|
||||
export TOKEN="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
|
||||
fi
|
||||
|
||||
python3 /opt/logs_loop/start.py https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT_HTTPS}
|
||||
@@ -0,0 +1,6 @@
|
||||
#! /bin/bash
|
||||
secret=$(kubectl get secret -n kubeinvaders | grep 'service-account-token' | grep kubeinvaders | awk '{ print $1}')
|
||||
token=$(kubectl describe secret $secret -n kubeinvaders | grep 'token:' | awk '{ print $2}')
|
||||
ip=$(ip address show eth0 | grep inet | grep -v inet6 | awk '{ print $2 }' | awk -F/ '{ print $1 }')
|
||||
export TOKEN=$token
|
||||
python3 start.py https://$ip:6443
|
||||
@@ -0,0 +1,53 @@
|
||||
jobs:
|
||||
# cpu-attack:
|
||||
# image: docker.io/luckysideburn/kubeinvaders-stress-ng:latest
|
||||
# command: "stress-ng"
|
||||
# args:
|
||||
# - --cpu
|
||||
# - 4
|
||||
# - --io
|
||||
# - 2
|
||||
# - --vm
|
||||
# - 1
|
||||
# - --vm-bytes
|
||||
# - 1G
|
||||
# - --timeout
|
||||
# - 10s
|
||||
# - --metrics-brief
|
||||
|
||||
# mem-attack:
|
||||
# image: docker.io/luckysideburn/kubeinvaders-stress-ng:latest
|
||||
# command: "stress-ng"
|
||||
# args:
|
||||
# - --vm
|
||||
# - 2
|
||||
# - --vm-bytes
|
||||
# - 1G
|
||||
|
||||
cpu-attack:
|
||||
image: docker.io/luckysideburn/kubeinvaders-stress-ng:latest
|
||||
command: "stress-ng"
|
||||
args:
|
||||
- --help
|
||||
|
||||
mem-attack:
|
||||
image: docker.io/luckysideburn/kubeinvaders-stress-ng:latest
|
||||
command: "stress-ng"
|
||||
args:
|
||||
- --help
|
||||
|
||||
experiments:
|
||||
- name: cpu-attack-exp
|
||||
job: cpu-attack
|
||||
loop: 5
|
||||
after:
|
||||
check_url: https://google.it
|
||||
check_payload: ''
|
||||
check_tls_ignore: true
|
||||
- name: mem-attack-exp
|
||||
job: mem-attack
|
||||
loop: 5
|
||||
after:
|
||||
check_url: https://google.it
|
||||
check_payload: ''
|
||||
check_tls_ignore: true
|
||||
@@ -0,0 +1,4 @@
|
||||
redis
|
||||
pyyaml
|
||||
requests
|
||||
kubernetes
|
||||
@@ -0,0 +1,87 @@
|
||||
from asyncio.log import logger
|
||||
import yaml
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from kubernetes import client, config
|
||||
from kubernetes.client.rest import ApiException
|
||||
import requests
|
||||
from string import Template
|
||||
import string
|
||||
import random
|
||||
import redis
|
||||
import time
|
||||
|
||||
def create_container(image, name, command, args):
|
||||
container = client.V1Container(
|
||||
image=image,
|
||||
name=name,
|
||||
image_pull_policy='IfNotPresent',
|
||||
args=args,
|
||||
command=command,
|
||||
)
|
||||
|
||||
logging.info(
|
||||
f"Created container with name: {container.name}, "
|
||||
f"image: {container.image} and args: {container.args}"
|
||||
)
|
||||
|
||||
return container
|
||||
|
||||
def create_pod_template(pod_name, container):
|
||||
pod_template = client.V1PodTemplateSpec(
|
||||
spec=client.V1PodSpec(restart_policy="Never", containers=[container]),
|
||||
metadata=client.V1ObjectMeta(name=pod_name, labels={"pod_name": pod_name, "approle": "chaosnode"}),
|
||||
)
|
||||
|
||||
return pod_template
|
||||
|
||||
def create_job(job_name, pod_template):
|
||||
metadata = client.V1ObjectMeta(name=job_name, labels={"job_name": job_name, "approle": "chaosnode"})
|
||||
|
||||
job = client.V1Job(
|
||||
api_version="batch/v1",
|
||||
kind="Job",
|
||||
metadata=metadata,
|
||||
spec=client.V1JobSpec(backoff_limit=0, template=pod_template),
|
||||
)
|
||||
#logger.info(job)
|
||||
return job
|
||||
|
||||
|
||||
r = redis.Redis(unix_socket_path='/tmp/redis.sock')
|
||||
|
||||
# create logger
|
||||
logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO"))
|
||||
logging.info('Starting script for KubeInvaders programming mode')
|
||||
|
||||
configuration = client.Configuration()
|
||||
token = os.environ["TOKEN"]
|
||||
configuration.api_key = {"authorization": f"Bearer {token}"}
|
||||
configuration.host = sys.argv[1]
|
||||
|
||||
configuration.insecure_skip_tls_verify = True
|
||||
configuration.verify_ssl = False
|
||||
|
||||
client.Configuration.set_default(configuration)
|
||||
client.Configuration.set_default(configuration)
|
||||
|
||||
api_instance = client.CoreV1Api()
|
||||
batch_api = client.BatchV1Api()
|
||||
namespace = "kubeinvaders"
|
||||
|
||||
while True:
|
||||
try:
|
||||
api_response = api_instance.list_namespaced_pod(namespace="kubeinvaders")
|
||||
#logging.info(api_response)
|
||||
except ApiException as e:
|
||||
logging.info(e)
|
||||
|
||||
r.set("current_chaos_job_pod", 0)
|
||||
|
||||
for pod in api_response.items:
|
||||
if pod.metadata.labels.get('approle') != None and pod.metadata.labels['approle'] == 'chaosnode':
|
||||
logging.info(pod.status.phase)
|
||||
if pod.status.phase == "Pending" or pod.status.phase == "Running":
|
||||
r.incr('current_chaos_job_pod')
|
||||
time.sleep(1)
|
||||
Executable
+11
@@ -0,0 +1,11 @@
|
||||
#!/bin/sh
|
||||
|
||||
if [ ! -z "$K8S_TOKEN" ];then
|
||||
echo 'Found K8S_TOKEN... using K8S_TOKEN instead of TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)'
|
||||
export TOKEN=$K8S_TOKEN
|
||||
else
|
||||
# Source the service account token from the container directly.
|
||||
export TOKEN="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
|
||||
fi
|
||||
|
||||
python3 /opt/metrics_loop/start.py https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT_HTTPS}
|
||||
@@ -0,0 +1,6 @@
|
||||
#! /bin/bash
|
||||
secret=$(kubectl get secret -n kubeinvaders | grep 'service-account-token' | grep kubeinvaders | awk '{ print $1}')
|
||||
token=$(kubectl describe secret $secret -n kubeinvaders | grep 'token:' | awk '{ print $2}')
|
||||
ip=$(ip address show eth0 | grep inet | grep -v inet6 | awk '{ print $2 }' | awk -F/ '{ print $1 }')
|
||||
export TOKEN=$token
|
||||
python3 start.py https://$ip:6443
|
||||
@@ -30,7 +30,7 @@ def create_container(image, name, command, args):
|
||||
def create_pod_template(pod_name, container):
|
||||
pod_template = client.V1PodTemplateSpec(
|
||||
spec=client.V1PodSpec(restart_policy="Never", containers=[container]),
|
||||
metadata=client.V1ObjectMeta(name=pod_name, labels={"pod_name": pod_name, "approle": "chaosnode"}),
|
||||
metadata=client.V1ObjectMeta(name=pod_name, labels={"app": "kubeinvaders", "pod_name": pod_name, "approle": "chaosnode"}),
|
||||
)
|
||||
|
||||
return pod_template
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
#! /bin/bash
|
||||
|
||||
secret=$(kubectl get secret -n kubeinvaders | grep 'service-account-token' | awk '{ print $1}')
|
||||
token=$(kubectl describe secret $secret -n kubeinvaders | grep 'token:' | awk '{ print $2}')
|
||||
ip=$(ip address show eth0 | grep inet | grep -v inet6 | awk '{ print $2 }' | awk -F/ '{ print $1 }')
|
||||
export TOKEN=$token
|
||||
python3 start.py http://$ip:6443
|
||||
Reference in New Issue
Block a user