waiting for logs...
' > /var/www/html/chaoslogs.html
+
nginx -c /etc/nginx/nginx.conf -g 'daemon off;'
diff --git a/helm-charts/kubeinvaders/templates/rbac-cluster.yaml b/helm-charts/kubeinvaders/templates/rbac-cluster.yaml
index bb4aba9..926bcf8 100644
--- a/helm-charts/kubeinvaders/templates/rbac-cluster.yaml
+++ b/helm-charts/kubeinvaders/templates/rbac-cluster.yaml
@@ -12,12 +12,12 @@ metadata:
rules:
- apiGroups: [""]
resources: ["pods", "pods/log"]
- verbs: ["get", "watch", "list", "delete"]
+ verbs: ["delete"]
- apiGroups: ["batch", "extensions"]
resources: ["jobs"]
verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
-- apiGroups: [""]
- resources: ["nodes"]
+- apiGroups: ["*"]
+ resources: ["*"]
verbs: ["get", "watch", "list"]
{{- end -}}
{{- if or .Values.clusterRole.create .Values.clusterRole.name }}
diff --git a/html5/index.html b/html5/index.html
index a744662..7b0727d 100644
--- a/html5/index.html
+++ b/html5/index.html
@@ -44,7 +44,7 @@
diff --git a/html5/kubeinvaders.js b/html5/kubeinvaders.js
index 400c34a..594ff31 100644
--- a/html5/kubeinvaders.js
+++ b/html5/kubeinvaders.js
@@ -132,8 +132,8 @@ function getMetrics() {
else if (metric[0] == "pods_not_running_on_selected_ns") {
$('#pods_not_running_on').text(metric[1]);
}
- else if (metric[0] == "chaos_jobs_current") {
- $('#chaos_jobs_current').text(metric[1]);
+ else if (metric[0] == "current_chaos_job_pod") {
+ $('#current_chaos_job_pod').text(metric[1]);
}
}
};;
@@ -141,6 +141,16 @@ function getMetrics() {
oReq.send();
}
+function getChaosJobsLogs() {
+ var oReq = new XMLHttpRequest();
+ oReq.onload = function () {
+ document.getElementById("chaosJobLogsDiv").innerHTML = "";
+ document.getElementById("chaosJobLogsDiv").innerHTML = this.responseText;
+ };;
+ oReq.open("GET", "https://" + clu_endpoint + "/chaoslogs.html");
+ oReq.send();
+}
+
function runKubeLinter() {
$('#kubeLinterModal').modal('show');
modal_opened = true;
@@ -178,7 +188,7 @@ function getCurrentChaosContainer() {
oReq.onload = function () {
//console.log(this.responseText);
job_parsed = JSON.stringify(JSON.parse(this.responseText), null, 4);
- $('#currentChaosContainrYaml').text(job_parsed);
+ $('#currentChaosContainerYaml').text(job_parsed);
$('#currentChaosContainerJsonTextArea').val(job_parsed);
};;
oReq.open("GET", "https://" + clu_endpoint + "/kube/chaos/containers?action=container_definition");
@@ -711,6 +721,10 @@ window.setInterval(function metrics() {
if (game_mode_switch || programming_mode_switch) {
getMetrics()
}
+
+ if (programming_mode_switch) {
+ getChaosJobsLogs()
+ }
}, 2000)
getEndpoint();
diff --git a/scripts/logs_loop/requirements.txt b/scripts/logs_loop/requirements.txt
new file mode 100644
index 0000000..6abc4e4
--- /dev/null
+++ b/scripts/logs_loop/requirements.txt
@@ -0,0 +1,4 @@
+redis
+pyyaml
+requests
+kubernetes
diff --git a/scripts/logs_loop/start.py b/scripts/logs_loop/start.py
new file mode 100644
index 0000000..06b5be7
--- /dev/null
+++ b/scripts/logs_loop/start.py
@@ -0,0 +1,76 @@
+from asyncio.log import logger
+import yaml
+import logging
+import os
+import pathlib
+import sys
+from kubernetes import client, config
+from kubernetes.client.rest import ApiException
+import requests
+from string import Template
+import string
+import random
+import redis
+import time
+
+file = pathlib.Path('/tmp/redis.sock')
+if file.exists():
+ r = redis.Redis(unix_socket_path='/tmp/redis.sock', charset="utf-8", decode_responses=True)
+else:
+ r = redis.Redis("127.0.0.1", charset="utf-8", decode_responses=True)
+
+# create logger
+logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO"))
+logging.info('Starting script for KubeInvaders programming mode')
+
+configuration = client.Configuration()
+token = os.environ["TOKEN"]
+configuration.api_key = {"authorization": f"Bearer {token}"}
+configuration.host = sys.argv[1]
+
+configuration.insecure_skip_tls_verify = True
+configuration.verify_ssl = False
+
+client.Configuration.set_default(configuration)
+client.Configuration.set_default(configuration)
+
+api_instance = client.CoreV1Api()
+batch_api = client.BatchV1Api()
+namespace = "kubeinvaders"
+
+#for key in r.scan_iter("log:*"):
+# r.delete(key)
+
+while True:
+ try:
+ api_response = api_instance.list_namespaced_pod(namespace="kubeinvaders")
+ except ApiException as e:
+ logging.info(e)
+
+ for pod in api_response.items:
+ if pod.metadata.labels.get('approle') != None and pod.metadata.labels['approle'] == 'chaosnode' and pod.status.phase != "Pending":
+ try:
+ logging.info(f"Reading logs of {pod.metadata.name}")
+ api_response = api_instance.read_namespaced_pod_log(name=pod.metadata.name, namespace='kubeinvaders')
+ logrow = f"
Pod Name: {pod.metadata.name} (Log TTL: 30sec)
{api_response}
"
+ r.set(f"log:{pod.metadata.name}", logrow)
+ r.expire(f"log:{pod.metadata.name}", 30)
+ logging.info(f"Phase of {pod.metadata.name} is {pod.status.phase}")
+ if pod.status.phase == "Succeeded":
+ try:
+ api_response = api_instance.delete_namespaced_pod(pod.metadata.name, namespace='kubeinvaders')
+ logging.info(f"Deleted pod {pod.metadata.name}")
+ except ApiException as e:
+ logging.info(e)
+ except ApiException as e:
+ logging.info(e)
+ file = pathlib.Path('/var/www/html')
+ if file.exists():
+ log_html_file = pathlib.Path('/var/www/html/chaoslogs.html')
+ if log_html_file.exists():
+ with open("/var/www/html/chaoslogs.html", "w") as myfile:
+ myfile.write('')
+ for key in r.scan_iter("log:*"):
+ with open("/var/www/html/chaoslogs.html", "a") as myfile:
+ myfile.write(str(r.get(key)))
+ time.sleep(1)
diff --git a/scripts/logs_loop/start.sh b/scripts/logs_loop/start.sh
new file mode 100755
index 0000000..f1fee92
--- /dev/null
+++ b/scripts/logs_loop/start.sh
@@ -0,0 +1,11 @@
+#!/bin/sh
+
+if [ ! -z "$K8S_TOKEN" ];then
+ echo 'Found K8S_TOKEN... using K8S_TOKEN instead of TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)'
+ export TOKEN=$K8S_TOKEN
+else
+ # Source the service account token from the container directly.
+ export TOKEN="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
+fi
+
+python3 /opt/logs_loop/start.py https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT_HTTPS}
diff --git a/scripts/logs_loop/start_dev.sh b/scripts/logs_loop/start_dev.sh
new file mode 100644
index 0000000..27c823a
--- /dev/null
+++ b/scripts/logs_loop/start_dev.sh
@@ -0,0 +1,6 @@
+#! /bin/bash
+secret=$(kubectl get secret -n kubeinvaders | grep 'service-account-token' | grep kubeinvaders | awk '{ print $1}')
+token=$(kubectl describe secret $secret -n kubeinvaders | grep 'token:' | awk '{ print $2}')
+ip=$(ip address show eth0 | grep inet | grep -v inet6 | awk '{ print $2 }' | awk -F/ '{ print $1 }')
+export TOKEN=$token
+python3 start.py https://$ip:6443
diff --git a/scripts/metrics_loop/experiments.yaml b/scripts/metrics_loop/experiments.yaml
new file mode 100644
index 0000000..599e76d
--- /dev/null
+++ b/scripts/metrics_loop/experiments.yaml
@@ -0,0 +1,53 @@
+jobs:
+ # cpu-attack:
+ # image: docker.io/luckysideburn/kubeinvaders-stress-ng:latest
+ # command: "stress-ng"
+ # args:
+ # - --cpu
+ # - 4
+ # - --io
+ # - 2
+ # - --vm
+ # - 1
+ # - --vm-bytes
+ # - 1G
+ # - --timeout
+ # - 10s
+ # - --metrics-brief
+
+ # mem-attack:
+ # image: docker.io/luckysideburn/kubeinvaders-stress-ng:latest
+ # command: "stress-ng"
+ # args:
+ # - --vm
+ # - 2
+ # - --vm-bytes
+ # - 1G
+
+ cpu-attack:
+ image: docker.io/luckysideburn/kubeinvaders-stress-ng:latest
+ command: "stress-ng"
+ args:
+ - --help
+
+ mem-attack:
+ image: docker.io/luckysideburn/kubeinvaders-stress-ng:latest
+ command: "stress-ng"
+ args:
+ - --help
+
+experiments:
+ - name: cpu-attack-exp
+ job: cpu-attack
+ loop: 5
+ after:
+ check_url: https://google.it
+ check_payload: ''
+ check_tls_ignore: true
+ - name: mem-attack-exp
+ job: mem-attack
+ loop: 5
+ after:
+ check_url: https://google.it
+ check_payload: ''
+ check_tls_ignore: true
\ No newline at end of file
diff --git a/scripts/metrics_loop/requirements.txt b/scripts/metrics_loop/requirements.txt
new file mode 100644
index 0000000..6abc4e4
--- /dev/null
+++ b/scripts/metrics_loop/requirements.txt
@@ -0,0 +1,4 @@
+redis
+pyyaml
+requests
+kubernetes
diff --git a/scripts/metrics_loop/start.py b/scripts/metrics_loop/start.py
new file mode 100644
index 0000000..bc9e639
--- /dev/null
+++ b/scripts/metrics_loop/start.py
@@ -0,0 +1,87 @@
+from asyncio.log import logger
+import yaml
+import logging
+import os
+import sys
+from kubernetes import client, config
+from kubernetes.client.rest import ApiException
+import requests
+from string import Template
+import string
+import random
+import redis
+import time
+
+def create_container(image, name, command, args):
+ container = client.V1Container(
+ image=image,
+ name=name,
+ image_pull_policy='IfNotPresent',
+ args=args,
+ command=command,
+ )
+
+ logging.info(
+ f"Created container with name: {container.name}, "
+ f"image: {container.image} and args: {container.args}"
+ )
+
+ return container
+
+def create_pod_template(pod_name, container):
+ pod_template = client.V1PodTemplateSpec(
+ spec=client.V1PodSpec(restart_policy="Never", containers=[container]),
+ metadata=client.V1ObjectMeta(name=pod_name, labels={"pod_name": pod_name, "approle": "chaosnode"}),
+ )
+
+ return pod_template
+
+def create_job(job_name, pod_template):
+ metadata = client.V1ObjectMeta(name=job_name, labels={"job_name": job_name, "approle": "chaosnode"})
+
+ job = client.V1Job(
+ api_version="batch/v1",
+ kind="Job",
+ metadata=metadata,
+ spec=client.V1JobSpec(backoff_limit=0, template=pod_template),
+ )
+ #logger.info(job)
+ return job
+
+
+r = redis.Redis(unix_socket_path='/tmp/redis.sock')
+
+# create logger
+logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO"))
+logging.info('Starting script for KubeInvaders programming mode')
+
+configuration = client.Configuration()
+token = os.environ["TOKEN"]
+configuration.api_key = {"authorization": f"Bearer {token}"}
+configuration.host = sys.argv[1]
+
+configuration.insecure_skip_tls_verify = True
+configuration.verify_ssl = False
+
+client.Configuration.set_default(configuration)
+client.Configuration.set_default(configuration)
+
+api_instance = client.CoreV1Api()
+batch_api = client.BatchV1Api()
+namespace = "kubeinvaders"
+
+while True:
+ try:
+ api_response = api_instance.list_namespaced_pod(namespace="kubeinvaders")
+ #logging.info(api_response)
+ except ApiException as e:
+ logging.info(e)
+
+ r.set("current_chaos_job_pod", 0)
+
+ for pod in api_response.items:
+ if pod.metadata.labels.get('approle') != None and pod.metadata.labels['approle'] == 'chaosnode':
+ logging.info(pod.status.phase)
+ if pod.status.phase == "Pending" or pod.status.phase == "Running":
+ r.incr('current_chaos_job_pod')
+ time.sleep(1)
diff --git a/scripts/metrics_loop/start.sh b/scripts/metrics_loop/start.sh
new file mode 100755
index 0000000..00177e7
--- /dev/null
+++ b/scripts/metrics_loop/start.sh
@@ -0,0 +1,11 @@
+#!/bin/sh
+
+if [ ! -z "$K8S_TOKEN" ];then
+ echo 'Found K8S_TOKEN... using K8S_TOKEN instead of TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)'
+ export TOKEN=$K8S_TOKEN
+else
+ # Source the service account token from the container directly.
+ export TOKEN="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
+fi
+
+python3 /opt/metrics_loop/start.py https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT_HTTPS}
diff --git a/scripts/metrics_loop/start_dev.sh b/scripts/metrics_loop/start_dev.sh
new file mode 100644
index 0000000..27c823a
--- /dev/null
+++ b/scripts/metrics_loop/start_dev.sh
@@ -0,0 +1,6 @@
+#! /bin/bash
+secret=$(kubectl get secret -n kubeinvaders | grep 'service-account-token' | grep kubeinvaders | awk '{ print $1}')
+token=$(kubectl describe secret $secret -n kubeinvaders | grep 'token:' | awk '{ print $2}')
+ip=$(ip address show eth0 | grep inet | grep -v inet6 | awk '{ print $2 }' | awk -F/ '{ print $1 }')
+export TOKEN=$token
+python3 start.py https://$ip:6443
diff --git a/scripts/programming_mode/start.py b/scripts/programming_mode/start.py
index bf128d4..a5e296b 100644
--- a/scripts/programming_mode/start.py
+++ b/scripts/programming_mode/start.py
@@ -30,7 +30,7 @@ def create_container(image, name, command, args):
def create_pod_template(pod_name, container):
pod_template = client.V1PodTemplateSpec(
spec=client.V1PodSpec(restart_policy="Never", containers=[container]),
- metadata=client.V1ObjectMeta(name=pod_name, labels={"pod_name": pod_name, "approle": "chaosnode"}),
+ metadata=client.V1ObjectMeta(name=pod_name, labels={"app": "kubeinvaders", "pod_name": pod_name, "approle": "chaosnode"}),
)
return pod_template
diff --git a/scripts/programming_mode/start_dev.sh b/scripts/programming_mode/start_dev.sh
new file mode 100644
index 0000000..8073f82
--- /dev/null
+++ b/scripts/programming_mode/start_dev.sh
@@ -0,0 +1,7 @@
+#! /bin/bash
+
+secret=$(kubectl get secret -n kubeinvaders | grep 'service-account-token' | awk '{ print $1}')
+token=$(kubectl describe secret $secret -n kubeinvaders | grep 'token:' | awk '{ print $2}')
+ip=$(ip address show eth0 | grep inet | grep -v inet6 | awk '{ print $2 }' | awk -F/ '{ print $1 }')
+export TOKEN=$token
+python3 start.py http://$ip:6443