From b4828e00ad1255a63702ef386f12e9676d38a081 Mon Sep 17 00:00:00 2001 From: Luckysideburn Date: Sat, 22 Oct 2022 10:11:04 +0000 Subject: [PATCH 1/2] added new metrics for curron chaos job pod --- Dockerfile | 5 +- entrypoint.sh | 3 + .../kubeinvaders/templates/rbac-cluster.yaml | 6 +- html5/index.html | 4 +- html5/kubeinvaders.js | 6 +- scripts/metrics_loop/experiments.yaml | 53 +++++++++++ scripts/metrics_loop/requirements.txt | 4 + scripts/metrics_loop/start.py | 87 +++++++++++++++++++ scripts/metrics_loop/start.sh | 11 +++ scripts/metrics_loop/start_dev.sh | 6 ++ scripts/programming_mode/start.py | 2 +- scripts/programming_mode/start_dev.sh | 7 ++ 12 files changed, 184 insertions(+), 10 deletions(-) create mode 100644 scripts/metrics_loop/experiments.yaml create mode 100644 scripts/metrics_loop/requirements.txt create mode 100644 scripts/metrics_loop/start.py create mode 100755 scripts/metrics_loop/start.sh create mode 100644 scripts/metrics_loop/start_dev.sh create mode 100644 scripts/programming_mode/start_dev.sh diff --git a/Dockerfile b/Dockerfile index 1146e01..dcb5b23 100644 --- a/Dockerfile +++ b/Dockerfile @@ -41,7 +41,6 @@ RUN sed -i.bak 's/listen\(.*\)80;/listen 8081;/' /etc/nginx/conf.d/default.conf RUN mkdir /usr/local/openresty/nginx/conf/kubeinvaders COPY nginx/nginx.conf /etc/nginx/nginx.conf - COPY scripts/metrics.lua /usr/local/openresty/nginx/conf/kubeinvaders/metrics.lua COPY scripts/pod.lua /usr/local/openresty/nginx/conf/kubeinvaders/pod.lua COPY scripts/node.lua /usr/local/openresty/nginx/conf/kubeinvaders/node.lua @@ -51,7 +50,11 @@ COPY scripts/chaos-containers.lua /usr/local/openresty/nginx/conf/kubeinvaders/c COPY scripts/programming_mode.lua /usr/local/openresty/nginx/conf/kubeinvaders/programming_mode.lua COPY scripts/config_kubeinv.lua /usr/local/openresty/lualib/config_kubeinv.lua COPY scripts/programming_mode /opt/programming_mode/ +COPY scripts/metrics_loop /opt/metrics_loop/ + RUN pip3 install -r /opt/programming_mode/requirements.txt +RUN pip3 install -r /opt/programming_mode/requirements.txt + COPY nginx/KubeInvaders.conf /etc/nginx/conf.d/KubeInvaders.conf RUN chmod g+rwx /var/cache/nginx /var/run /var/log/nginx /var/www/html /etc/nginx/conf.d diff --git a/entrypoint.sh b/entrypoint.sh index b723569..02938f6 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -10,4 +10,7 @@ fi # TODO: use a sidecar redis-server /etc/redis/redis.conf & +# TODO: use a sidecar +bash /opt/metrics_loop/start.sh & + nginx -c /etc/nginx/nginx.conf -g 'daemon off;' diff --git a/helm-charts/kubeinvaders/templates/rbac-cluster.yaml b/helm-charts/kubeinvaders/templates/rbac-cluster.yaml index bb4aba9..926bcf8 100644 --- a/helm-charts/kubeinvaders/templates/rbac-cluster.yaml +++ b/helm-charts/kubeinvaders/templates/rbac-cluster.yaml @@ -12,12 +12,12 @@ metadata: rules: - apiGroups: [""] resources: ["pods", "pods/log"] - verbs: ["get", "watch", "list", "delete"] + verbs: ["delete"] - apiGroups: ["batch", "extensions"] resources: ["jobs"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] -- apiGroups: [""] - resources: ["nodes"] +- apiGroups: ["*"] + resources: ["*"] verbs: ["get", "watch", "list"] {{- end -}} {{- if or .Values.clusterRole.create .Values.clusterRole.name }} diff --git a/html5/index.html b/html5/index.html index a744662..f4da27a 100644 --- a/html5/index.html +++ b/html5/index.html @@ -44,7 +44,7 @@
- 
 
+            
 
           

           

             
@@ -169,7 +169,7 @@
           

             Deleted Pod Total:  0 - 
             Chaos Jobs Total:  0 - 
-            Current Chaos Jobs:  0 - 	    
+            Current Chaos Jobs Pods:  0 - 	    
             Not Running Pods:  0 - 
             Current Replicas State Delay:  0 sec - 
             Latest Replicas State Delay:  0 sec 
diff --git a/html5/kubeinvaders.js b/html5/kubeinvaders.js
index 400c34a..66e1987 100644
--- a/html5/kubeinvaders.js
+++ b/html5/kubeinvaders.js
@@ -132,8 +132,8 @@ function getMetrics() {
             else if (metric[0] == "pods_not_running_on_selected_ns") {
                 $('#pods_not_running_on').text(metric[1]);            
             }
-	    else if (metric[0] == "chaos_jobs_current") {
-	        $('#chaos_jobs_current').text(metric[1]);
+	    else if (metric[0] == "current_chaos_job_pod") {
+	        $('#current_chaos_job_pod').text(metric[1]);
 	    }
         }
     };;
@@ -178,7 +178,7 @@ function getCurrentChaosContainer() {
     oReq.onload = function () {
         //console.log(this.responseText);
         job_parsed = JSON.stringify(JSON.parse(this.responseText), null, 4);
-        $('#currentChaosContainrYaml').text(job_parsed);
+        $('#currentChaosContainerYaml').text(job_parsed);
         $('#currentChaosContainerJsonTextArea').val(job_parsed);
     };;
     oReq.open("GET", "https://" + clu_endpoint + "/kube/chaos/containers?action=container_definition");
diff --git a/scripts/metrics_loop/experiments.yaml b/scripts/metrics_loop/experiments.yaml
new file mode 100644
index 0000000..599e76d
--- /dev/null
+++ b/scripts/metrics_loop/experiments.yaml
@@ -0,0 +1,53 @@
+jobs:
+  # cpu-attack:
+  #   image: docker.io/luckysideburn/kubeinvaders-stress-ng:latest
+  #   command: "stress-ng"
+  #   args:
+  #     - --cpu
+  #     - 4
+  #     - --io
+  #     - 2
+  #     - --vm
+  #     - 1
+  #     - --vm-bytes
+  #     - 1G
+  #     - --timeout
+  #     - 10s
+  #     - --metrics-brief
+
+  # mem-attack:
+  #   image: docker.io/luckysideburn/kubeinvaders-stress-ng:latest
+  #   command: "stress-ng"
+  #   args:
+  #     - --vm
+  #     - 2
+  #     - --vm-bytes
+  #     - 1G
+
+  cpu-attack:
+    image: docker.io/luckysideburn/kubeinvaders-stress-ng:latest
+    command: "stress-ng"
+    args:
+      - --help
+
+  mem-attack:
+    image: docker.io/luckysideburn/kubeinvaders-stress-ng:latest
+    command: "stress-ng"
+    args:
+      - --help
+
+experiments:
+  - name: cpu-attack-exp
+    job: cpu-attack
+    loop: 5
+    after:
+      check_url: https://google.it
+      check_payload: ''
+      check_tls_ignore: true
+  - name: mem-attack-exp
+    job: mem-attack
+    loop: 5
+    after:
+      check_url: https://google.it
+      check_payload: ''
+      check_tls_ignore: true
\ No newline at end of file
diff --git a/scripts/metrics_loop/requirements.txt b/scripts/metrics_loop/requirements.txt
new file mode 100644
index 0000000..6abc4e4
--- /dev/null
+++ b/scripts/metrics_loop/requirements.txt
@@ -0,0 +1,4 @@
+redis
+pyyaml
+requests
+kubernetes
diff --git a/scripts/metrics_loop/start.py b/scripts/metrics_loop/start.py
new file mode 100644
index 0000000..bc9e639
--- /dev/null
+++ b/scripts/metrics_loop/start.py
@@ -0,0 +1,87 @@
+from asyncio.log import logger
+import yaml
+import logging
+import os
+import sys
+from kubernetes import client, config
+from kubernetes.client.rest import ApiException
+import requests
+from string import Template
+import string
+import random
+import redis
+import time
+
+def create_container(image, name, command, args):
+    container = client.V1Container(
+        image=image,
+        name=name,
+        image_pull_policy='IfNotPresent',
+        args=args,
+        command=command,
+    )
+
+    logging.info(
+        f"Created container with name: {container.name}, "
+        f"image: {container.image} and args: {container.args}"
+    )
+
+    return container
+
+def create_pod_template(pod_name, container):
+    pod_template = client.V1PodTemplateSpec(
+        spec=client.V1PodSpec(restart_policy="Never", containers=[container]),
+        metadata=client.V1ObjectMeta(name=pod_name, labels={"pod_name": pod_name, "approle": "chaosnode"}),
+    )
+
+    return pod_template
+
+def create_job(job_name, pod_template):
+    metadata = client.V1ObjectMeta(name=job_name, labels={"job_name": job_name, "approle": "chaosnode"})
+
+    job = client.V1Job(
+        api_version="batch/v1",
+        kind="Job",
+        metadata=metadata,
+        spec=client.V1JobSpec(backoff_limit=0, template=pod_template),
+    )
+    #logger.info(job)
+    return job
+
+
+r = redis.Redis(unix_socket_path='/tmp/redis.sock')
+
+# create logger
+logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO"))
+logging.info('Starting script for KubeInvaders programming mode')
+
+configuration = client.Configuration()
+token = os.environ["TOKEN"]
+configuration.api_key = {"authorization": f"Bearer {token}"}
+configuration.host = sys.argv[1]
+
+configuration.insecure_skip_tls_verify = True
+configuration.verify_ssl = False
+
+client.Configuration.set_default(configuration)
+client.Configuration.set_default(configuration)
+
+api_instance = client.CoreV1Api()
+batch_api = client.BatchV1Api()
+namespace = "kubeinvaders"
+
+while True:
+    try:
+        api_response = api_instance.list_namespaced_pod(namespace="kubeinvaders")
+        #logging.info(api_response)
+    except ApiException as e:
+        logging.info(e)
+
+    r.set("current_chaos_job_pod", 0)
+
+    for pod in api_response.items:
+        if pod.metadata.labels.get('approle') != None and pod.metadata.labels['approle'] == 'chaosnode':
+            logging.info(pod.status.phase)
+            if pod.status.phase == "Pending" or pod.status.phase == "Running":
+                r.incr('current_chaos_job_pod')
+    time.sleep(1)
diff --git a/scripts/metrics_loop/start.sh b/scripts/metrics_loop/start.sh
new file mode 100755
index 0000000..3c806aa
--- /dev/null
+++ b/scripts/metrics_loop/start.sh
@@ -0,0 +1,11 @@
+#!/bin/sh
+
+if [ ! -z "$K8S_TOKEN" ];then
+  echo 'Found K8S_TOKEN... using K8S_TOKEN instead of TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)'
+  export TOKEN=$K8S_TOKEN
+else
+  # Source the service account token from the container directly.
+  export TOKEN="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
+fi
+
+python3 /opt/metrics_loop/start.py https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT_HTTPS} &
diff --git a/scripts/metrics_loop/start_dev.sh b/scripts/metrics_loop/start_dev.sh
new file mode 100644
index 0000000..27c823a
--- /dev/null
+++ b/scripts/metrics_loop/start_dev.sh
@@ -0,0 +1,6 @@
+#! /bin/bash
+secret=$(kubectl get secret -n kubeinvaders | grep 'service-account-token' | grep kubeinvaders | awk '{ print $1}')
+token=$(kubectl describe secret $secret -n kubeinvaders | grep 'token:' | awk '{ print $2}')
+ip=$(ip address show eth0 | grep inet | grep -v inet6 | awk '{ print $2 }' | awk -F/ '{ print $1 }')
+export TOKEN=$token
+python3 start.py https://$ip:6443
diff --git a/scripts/programming_mode/start.py b/scripts/programming_mode/start.py
index bf128d4..a5e296b 100644
--- a/scripts/programming_mode/start.py
+++ b/scripts/programming_mode/start.py
@@ -30,7 +30,7 @@ def create_container(image, name, command, args):
 def create_pod_template(pod_name, container):
     pod_template = client.V1PodTemplateSpec(
         spec=client.V1PodSpec(restart_policy="Never", containers=[container]),
-        metadata=client.V1ObjectMeta(name=pod_name, labels={"pod_name": pod_name, "approle": "chaosnode"}),
+        metadata=client.V1ObjectMeta(name=pod_name, labels={"app": "kubeinvaders", "pod_name": pod_name, "approle": "chaosnode"}),
     )
 
     return pod_template
diff --git a/scripts/programming_mode/start_dev.sh b/scripts/programming_mode/start_dev.sh
new file mode 100644
index 0000000..8073f82
--- /dev/null
+++ b/scripts/programming_mode/start_dev.sh
@@ -0,0 +1,7 @@
+#! /bin/bash
+
+secret=$(kubectl get secret -n kubeinvaders | grep 'service-account-token' | awk '{ print $1}')
+token=$(kubectl describe secret $secret -n kubeinvaders | grep 'token:' | awk '{ print $2}')
+ip=$(ip address show eth0 | grep inet | grep -v inet6 | awk '{ print $2 }' | awk -F/ '{ print $1 }')
+export TOKEN=$token
+python3 start.py http://$ip:6443

From 75af8a8a146cd08d9970c9a1a224a23f622066f3 Mon Sep 17 00:00:00 2001
From: Luckysideburn 
Date: Sat, 22 Oct 2022 17:42:06 +0000
Subject: [PATCH 2/2] added log console

---
 Dockerfile                         |  1 +
 dev-tools/shell.sh                 |  3 ++
 entrypoint.sh                      |  3 ++
 html5/index.html                   |  1 +
 html5/kubeinvaders.js              | 14 ++++++
 scripts/logs_loop/requirements.txt |  4 ++
 scripts/logs_loop/start.py         | 76 ++++++++++++++++++++++++++++++
 scripts/logs_loop/start.sh         | 11 +++++
 scripts/logs_loop/start_dev.sh     |  6 +++
 scripts/metrics_loop/start.sh      |  2 +-
 10 files changed, 120 insertions(+), 1 deletion(-)
 create mode 100644 dev-tools/shell.sh
 create mode 100644 scripts/logs_loop/requirements.txt
 create mode 100644 scripts/logs_loop/start.py
 create mode 100755 scripts/logs_loop/start.sh
 create mode 100644 scripts/logs_loop/start_dev.sh

diff --git a/Dockerfile b/Dockerfile
index dcb5b23..23da403 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -51,6 +51,7 @@ COPY scripts/programming_mode.lua /usr/local/openresty/nginx/conf/kubeinvaders/p
 COPY scripts/config_kubeinv.lua /usr/local/openresty/lualib/config_kubeinv.lua
 COPY scripts/programming_mode /opt/programming_mode/
 COPY scripts/metrics_loop /opt/metrics_loop/
+COPY scripts/logs_loop /opt/logs_loop/
 
 RUN pip3 install -r /opt/programming_mode/requirements.txt
 RUN pip3 install -r /opt/programming_mode/requirements.txt
diff --git a/dev-tools/shell.sh b/dev-tools/shell.sh
new file mode 100644
index 0000000..63333fb
--- /dev/null
+++ b/dev-tools/shell.sh
@@ -0,0 +1,3 @@
+#! /bin/bash
+kubectl exec -it $(kubectl get pods -n kubeinvaders | grep 'kubeinvaders-' | awk '{ print $1 }') -n kubeinvaders bash
+
diff --git a/entrypoint.sh b/entrypoint.sh
index 02938f6..384f64c 100644
--- a/entrypoint.sh
+++ b/entrypoint.sh
@@ -12,5 +12,8 @@ redis-server /etc/redis/redis.conf &
 
 # TODO: use a sidecar
 bash /opt/metrics_loop/start.sh &
+bash /opt/logs_loop/start.sh &
+
+echo '
waiting for logs...
' > /var/www/html/chaoslogs.html
 
 nginx -c /etc/nginx/nginx.conf -g 'daemon off;'
diff --git a/html5/index.html b/html5/index.html
index f4da27a..7b0727d 100644
--- a/html5/index.html
+++ b/html5/index.html
@@ -207,6 +207,7 @@ experiments:
                 
               

             
+	    

           

         

 
diff --git a/html5/kubeinvaders.js b/html5/kubeinvaders.js
index 66e1987..594ff31 100644
--- a/html5/kubeinvaders.js
+++ b/html5/kubeinvaders.js
@@ -141,6 +141,16 @@ function getMetrics() {
     oReq.send();
 }
 
+function getChaosJobsLogs() {
+    var oReq = new XMLHttpRequest();
+    oReq.onload = function () {
+	document.getElementById("chaosJobLogsDiv").innerHTML = "";    
+	document.getElementById("chaosJobLogsDiv").innerHTML = this.responseText;
+    };;	    
+    oReq.open("GET", "https://" + clu_endpoint + "/chaoslogs.html");
+    oReq.send();
+}
+
 function runKubeLinter() {
     $('#kubeLinterModal').modal('show');
     modal_opened = true;
@@ -711,6 +721,10 @@ window.setInterval(function metrics() {
     if (game_mode_switch || programming_mode_switch) {
         getMetrics()
     }
+    
+    if (programming_mode_switch) {
+	getChaosJobsLogs()
+    }
 }, 2000)
 
 getEndpoint();
diff --git a/scripts/logs_loop/requirements.txt b/scripts/logs_loop/requirements.txt
new file mode 100644
index 0000000..6abc4e4
--- /dev/null
+++ b/scripts/logs_loop/requirements.txt
@@ -0,0 +1,4 @@
+redis
+pyyaml
+requests
+kubernetes
diff --git a/scripts/logs_loop/start.py b/scripts/logs_loop/start.py
new file mode 100644
index 0000000..06b5be7
--- /dev/null
+++ b/scripts/logs_loop/start.py
@@ -0,0 +1,76 @@
+from asyncio.log import logger
+import yaml
+import logging
+import os
+import pathlib
+import sys
+from kubernetes import client, config
+from kubernetes.client.rest import ApiException
+import requests
+from string import Template
+import string
+import random
+import redis
+import time
+
+file = pathlib.Path('/tmp/redis.sock')
+if file.exists():
+    r = redis.Redis(unix_socket_path='/tmp/redis.sock', charset="utf-8", decode_responses=True)
+else:
+    r = redis.Redis("127.0.0.1", charset="utf-8", decode_responses=True)
+
+# create logger
+logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO"))
+logging.info('Starting script for KubeInvaders programming mode')
+
+configuration = client.Configuration()
+token = os.environ["TOKEN"]
+configuration.api_key = {"authorization": f"Bearer {token}"}
+configuration.host = sys.argv[1]
+
+configuration.insecure_skip_tls_verify = True
+configuration.verify_ssl = False
+
+client.Configuration.set_default(configuration)
+client.Configuration.set_default(configuration)
+
+api_instance = client.CoreV1Api()
+batch_api = client.BatchV1Api()
+namespace = "kubeinvaders"
+
+#for key in r.scan_iter("log:*"):
+#    r.delete(key)
+
+while True:
+    try:
+        api_response = api_instance.list_namespaced_pod(namespace="kubeinvaders")
+    except ApiException as e:
+        logging.info(e)
+
+    for pod in api_response.items:
+        if pod.metadata.labels.get('approle') != None and pod.metadata.labels['approle'] == 'chaosnode' and pod.status.phase != "Pending":
+            try:
+                logging.info(f"Reading logs of {pod.metadata.name}")
+                api_response = api_instance.read_namespaced_pod_log(name=pod.metadata.name, namespace='kubeinvaders')
+                logrow = f"
"
+                r.set(f"log:{pod.metadata.name}", logrow)
+                r.expire(f"log:{pod.metadata.name}", 30)
+                logging.info(f"Phase of {pod.metadata.name} is {pod.status.phase}")   
+                if pod.status.phase == "Succeeded":
+                    try:
+                        api_response = api_instance.delete_namespaced_pod(pod.metadata.name, namespace='kubeinvaders')
+                        logging.info(f"Deleted pod {pod.metadata.name}")
+                    except ApiException as e:
+                        logging.info(e)
+            except ApiException as e:
+                logging.info(e)
+    file = pathlib.Path('/var/www/html')
+    if file.exists():
+        log_html_file = pathlib.Path('/var/www/html/chaoslogs.html')
+        if log_html_file.exists():
+            with open("/var/www/html/chaoslogs.html", "w") as myfile:
+                myfile.write('')
+        for key in r.scan_iter("log:*"):
+            with open("/var/www/html/chaoslogs.html", "a") as myfile:
+                myfile.write(str(r.get(key)))
+    time.sleep(1)
diff --git a/scripts/logs_loop/start.sh b/scripts/logs_loop/start.sh
new file mode 100755
index 0000000..f1fee92
--- /dev/null
+++ b/scripts/logs_loop/start.sh
@@ -0,0 +1,11 @@
+#!/bin/sh
+
+if [ ! -z "$K8S_TOKEN" ];then
+  echo 'Found K8S_TOKEN... using K8S_TOKEN instead of TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)'
+  export TOKEN=$K8S_TOKEN
+else
+  # Source the service account token from the container directly.
+  export TOKEN="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
+fi
+
+python3 /opt/logs_loop/start.py https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT_HTTPS}
diff --git a/scripts/logs_loop/start_dev.sh b/scripts/logs_loop/start_dev.sh
new file mode 100644
index 0000000..27c823a
--- /dev/null
+++ b/scripts/logs_loop/start_dev.sh
@@ -0,0 +1,6 @@
+#! /bin/bash
+secret=$(kubectl get secret -n kubeinvaders | grep 'service-account-token' | grep kubeinvaders | awk '{ print $1}')
+token=$(kubectl describe secret $secret -n kubeinvaders | grep 'token:' | awk '{ print $2}')
+ip=$(ip address show eth0 | grep inet | grep -v inet6 | awk '{ print $2 }' | awk -F/ '{ print $1 }')
+export TOKEN=$token
+python3 start.py https://$ip:6443
diff --git a/scripts/metrics_loop/start.sh b/scripts/metrics_loop/start.sh
index 3c806aa..00177e7 100755
--- a/scripts/metrics_loop/start.sh
+++ b/scripts/metrics_loop/start.sh
@@ -8,4 +8,4 @@ else
   export TOKEN="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
 fi
 
-python3 /opt/metrics_loop/start.py https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT_HTTPS} &
+python3 /opt/metrics_loop/start.py https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT_HTTPS}