added chaos engineering for nodes

Dockerfile

@@ -8,7 +8,7 @@ RUN mv ./kubectl /usr/local/bin/kubectl
 
 # Install Openresty
 RUN apt-get update
-RUN apt-get -y install --no-install-recommends wget gnupg ca-certificates jq
+RUN apt-get -y install --no-install-recommends wget gnupg ca-certificates jq openssl
 RUN wget -O - https://openresty.org/package/pubkey.gpg | apt-key add -
 RUN codename=`grep -Po 'VERSION="[0-9]+ \(\K[^)]+' /etc/os-release` && echo "deb http://openresty.org/package/debian $codename openresty" | tee /etc/apt/sources.list.d/openresty.list
 RUN apt-get update
@@ -23,6 +23,10 @@ RUN chmod 775 /usr/local/bin/kube-linter
 COPY kube-linter/kube-linter-parser.sh /opt/kube-linter-parser.sh
 RUN chmod +x /opt/kube-linter-parser.sh
 
+# Install chaos-node
+COPY chaos-node/chaos-node.sh /opt/
+RUN chmod +x /opt/chaos-node.sh
+
 # Install KubeInvaders
 COPY ./js-web/KubeInvaders /var/www/html
 

assets/k8s_node.atlas

@@ -29,6 +29,21 @@ animations {
   flip_horizontal: 0
   flip_vertical: 0
 }
+animations {
+  id: "chaos"
+  images {
+    image: "/assets/k8s_node_black.png"
+    sprite_trim_mode: SPRITE_TRIM_MODE_OFF
+  }
+  images {
+    image: "/assets/k8s_node_red.png"
+    sprite_trim_mode: SPRITE_TRIM_MODE_OFF
+  }
+  playback: PLAYBACK_LOOP_FORWARD
+  fps: 2
+  flip_horizontal: 0
+  flip_vertical: 0
+}
 margin: 0
 extrude_borders: 2
 inner_padding: 0

assets/untitled.atlas

@@ -76,6 +76,29 @@ animations {
   flip_horizontal: 0
   flip_vertical: 0
 }
+animations {
+  id: "stress-ng"
+  images {
+    image: "/assets/sprite_invader_yellow.png"
+    sprite_trim_mode: SPRITE_TRIM_MODE_OFF
+  }
+  images {
+    image: "/assets/sprite_invader.png"
+    sprite_trim_mode: SPRITE_TRIM_MODE_OFF
+  }
+  images {
+    image: "/assets/sprite_invader_yellow.png"
+    sprite_trim_mode: SPRITE_TRIM_MODE_OFF
+  }
+  images {
+    image: "/assets/sprite_invader.png"
+    sprite_trim_mode: SPRITE_TRIM_MODE_OFF
+  }
+  playback: PLAYBACK_LOOP_PINGPONG
+  fps: 2
+  flip_horizontal: 0
+  flip_vertical: 0
+}
 margin: 0
 extrude_borders: 0
 inner_padding: 0

chaos-node/Dockerfile

@@ -0,0 +1,3 @@
+FROM alpine
+RUN apk update
+RUN apk add stress-ng

chaos-node/chaos-node.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+K8S_URL=$1
+TOKEN=$2
+NODE_NAME=$3
+NAMESPACE=$4
+RAND=$(openssl rand -hex 4)
+YAML_FILE=/tmp/kubeinvaders-chaos-node-$RAND.yaml
+
+# Very temporary work-around
+KUBE=$(cat /etc/nginx/conf.d/KubeInvaders.conf | grep proxy_pass | head -n1 | awk '{ print $2 }' | sed 's/;//g')
+
+echo "$TOKEN $KUBE $RAND $NODE_NAME" > /tmp/foo
+
+cat <<EOF >$YAML_FILE
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: kubeinvaders-chaos-random_suffix
+  labels:
+    app: kubeinvaders
+    approle: chaosnode
+spec:
+  template:
+    spec:
+      containers:
+      - name: kubeinvaders-chaos-node
+        image: docker.io/luckysideburn/kubeinvaders-stress-ng:latest
+        command: ["stress-ng", "--cpu", "4",  "--io",  "2",  "--vm", "1", "--vm-bytes", "1G", "--timeout", "10s", "--metrics-brief"]
+      restartPolicy: Never
+      nodeSelector:
+        kubernetes.io/hostname: node_name_placeholder
+  backoffLimit: 1
+EOF
+
+sed -i  "s/node_name_placeholder/$NODE_NAME/g" $YAML_FILE
+sed -i  "s/random_suffix/$RAND/g" $YAML_FILE
+
+kubectl create -f $YAML_FILE --token=${TOKEN} --server=${KUBE}  -n ${NAMESPACE} --insecure-skip-tls-verify=true >> /tmp/kubectl.log 2>&1
+
+kubectl delete job -l app=kubeinvaders -l approle=chaosnode -o=jsonpath='{.items[?(@.status.succeeded==1)].metadata.name}' --token=${TOKEN} --server=${KUBE} -n $NAMESPACE >> /tmp/kubectl.log 2>&1
+kubectl delete pod --token=${TOKEN} --server=${KUBE} -l app=kubeinvaders -l approle=chaosnode --field-selector=status.phase==Succeeded -n $NAMESPACE >> /tmp/kubectl.log 2>&1
+
+rm -f $YAML_FILE

main/bullet.go

@@ -43,6 +43,7 @@ embedded_components {
   "restitution: 0.5\n"
   "group: \"bullet\"\n"
   "mask: \"pod\"\n"
+  "mask: \"k8s_node\"\n"
   "embedded_collision_shape {\n"
   "  shapes {\n"
   "    shape_type: TYPE_SPHERE\n"

main/bullet.script

@@ -1,4 +1,5 @@
 local COLLISION_RESPONSE = hash("collision_response")
+local k8s_node = require "main.mymodules.node"
 
 function refresh_pods()
 	http.request(endpoint .. "/api/v1/namespaces/" .. namespace .. "/pods", "GET", pod_api.http_pod_result,headers)
@@ -16,7 +17,19 @@ end
 function on_message(self, message_id, message, sender)
 	print "[on_message] bullet object received a message"
 	if message_id == COLLISION_RESPONSE then
-		print "[on_message] collision detected"		
+		print "[on_message] collision detected"	
+
+		-- check if it is a k8s_node
+		print("check if the bullet collides with a k8s_node")
+		for i,value in ipairs(kubernetes_nodes) do
+			if message["other_id"] == value["id"] then
+				print("Spaceship collides with " .. value["name"])
+				bullet_collision_action = "chaos_node"
+				print("Set bullet_collision_action to chaos_node")
+				node_name = value["name"]
+			end
+		end
+				
 		if bullet_collision_action == "delete_pod" then
 			msg.post(message["other_id"], "play_animation", {id = hash("delete")})
 			for i,value in ipairs(current_pods) do
@@ -28,6 +41,7 @@ function on_message(self, message_id, message, sender)
 					go.delete(value["id"])
 				end
 			end
+
 		elseif bullet_collision_action == "kubelinter" then
 			for i,value in ipairs(current_pods) do
 				print("[on_message] pod of collision is " .. value["color"] .. " " .. "id is " .. message["other_id"] )
@@ -38,7 +52,23 @@ function on_message(self, message_id, message, sender)
 			print("start kubelinter analysis of " .. message["other_id"] .. " pod = " .. pod_name)
 			msg.post(message["other_id"], "play_animation", {id = hash("view")})
 			kubelinter_pod = pod_name
+
+		elseif bullet_collision_action == "chaos_node" then
+			msg.post(message["other_id"], "play_animation", {id = hash("chaos")})
+			print("launch deploy of chaos_node to " .. node_name)
+			k8s_node.deploy_chaos_node(node_name)
+			-- msg.post(message["other_id"], "play_animation", {id = hash("led")})	
+			bullet_collision_action = "delete_pod"
+			for k,v in ipairs(kubernetes_nodes) do
+				if v["id"] == message["other_id"] then
+					new_value = v
+					new_value["fired"] = true
+					new_value["fired_time"] = os.clock()
+					kubernetes_nodes[k] = new_value
+				end
+			end
 		end
+
 		go.delete()
 	end
 	print "[on_message] end of on_message function"

main/gui.gui

@@ -559,6 +559,7 @@ nodes {
   type: TYPE_TEXT
   blend_mode: BLEND_MODE_ALPHA
   text: "Special Keys:\n"
+  "w: show kubernetes nodes\n"
   "k: do kube-linter analysis\n"
   "i: Show pod\'s name. Move the ship towards an alien\n"
   "n: Jump between namespaces. Read REAME.md for configuring TARGET_NAMESPACE\n"

main/kubeinvaders.gui_script

@@ -33,16 +33,16 @@ function on_message(self, message_id, message, sender)
 		node_name_pos = vmath.vector3(spaceship_x, spaceship_y - 50, 0)
 		gui.set_text(gui.get_node("node_name"), message.node_name)	
 		gui.set_position(gui.get_node("node_name"), node_name_pos)
-		k8s_nodes_show = true
+		k8s_node_name_show = true
 
-	elseif  message_id == hash("reset_node_name") and k8s_nodes_show then
+	elseif  message_id == hash("reset_node_name") and k8s_node_name_show then
 		node_name_pos = gui.get_position(gui.get_node("node_name"))
 		print("Reset node name text box")
 		if node_name_pos.x > 0 then
 			gui.set_text(gui.get_node("node_name"), '')	
 			node_name_pos = vmath.vector3(-501, -501, 0)
 			print("Reset node_name text node")
-			k8s_nodes_show = false
+			k8s_node_name_show = false
 		end
 		
 	elseif  message_id == hash("pod_log") then

main/mymodules/node.lua

@@ -4,6 +4,29 @@ function M.node_metrics(pod_name)
 	http.request(endpoint .. "/apis/metrics.k8s.io/v1beta1/nodes", "GET", M.http_node_metrics_result, headers)
 end
 
+function M.deploy_chaos_node(node_name)
+	print("Call OpenResty endpoint for deploy chaos-node")
+	http.request(chaos_node_url .. "?namespace=" .. namespace .. "&node_name=".. node_name .. "&token=" .. token .. "&k8s_url=" .. endpoint, "GET", M.deploy_chaos_node_result,headers)	
+	print("waiting for a response...")
+end
+
+function M.deploy_chaos_node_result(self, _, response)
+	print("Deploy Chaos Node output")
+	print(response.status)
+	print(response.response)
+	print(response.headers)
+	--msg.post("ui#gui","open_kubelinter_box",{ position = vmath.vector3(850, 550, 0) })
+	--kubelinter_read = true
+	--kubelinter_index = 0
+
+	--for i,value in ipairs(kubelinter_table) do
+	--	kubelintermessage = value
+	--	kubelinter_index = kubelinter_index + 1
+	--	msg.post("ui#gui", "set_kubelinter_text",{ kubelintermessage = value })
+	--	break
+	--end
+end
+
 function M.http_node_metrics_result(self, _, response)
 	print(response.response)
 end
@@ -29,10 +52,10 @@ function M.http_get_nodes_result(self, _, response)
 				print("Found k8s node: " .. v2["name"])
 				pos = vmath.vector3(pos_x, pos_y, 0) 
 				new_node = factory.create("/k8s_node#k8s_node_factory", pos)
-				table.insert(kubernetes_nodes,{ name = v2["name"], id = new_node})
+				table.insert(kubernetes_nodes,{ name = v2["name"], id = new_node })
 			
 				if ((nodes_cnt % 15 ) == 0) then
-					pos_y = pos_y + 100
+					pos_y = pos_y + 200
 					pos_x = 80
 				else
 					pos_x = pos_x + 80

main/mymodules/pod_api.lua

@@ -1,11 +1,11 @@
 local M = {}
 
 function M.update_pod()
-  http.request(endpoint .. "/api/v1/namespaces/" .. namespace .. "/pods", "GET", M.http_update_pod_result,headers)
+  http.request(endpoint .. "/api/v1/namespaces/" .. namespace .. "/pods", "GET", M.http_update_pod_result, headers, '', { timeout=2 })
 end
 
 function M.set_pods()
-  http.request(endpoint .. "/api/v1/namespaces/".. namespace .. "/pods", "GET", M.http_pod_result,headers)
+  http.request(endpoint .. "/api/v1/namespaces/".. namespace .. "/pods", "GET", M.http_pod_result, headers,'', { timeout=2 })
 end
 
 function M.http_update_pod_result(self, _, response)
@@ -120,9 +120,10 @@ function M.http_pod_result(self, _, response)
 
       pos.y = y_pos			
       if phase == "Running" and this_pod['metadata']['deletionTimestamp'] == nil then
-        local pod = factory.create("/pod#podfactory", pos)				
+        local pod = factory.create("/pod#podfactory", pos)
         table.insert(current_pods, { id = pod , color = "white", pod_name = this_pod['metadata']['name'] })
         check_current_pods = true
+  --    elseif phase ~= "Succeeded" then
       else
         local pod = factory.create("/pod_not_running#podfactory", pos)
         table.insert(current_pods, { id = pod , color = "red", pod_name = this_pod['metadata']['name'] })

main/spaceship.script

@@ -80,12 +80,20 @@ kubelinter_read = false
 -- Keetp track of the line showed of the kubelinter analysis content
 kubelinter_index = 0
 
--- by default the k8s nodes are not shown. Set this var to true
-k8s_nodes_show = false
+-- by default the k8s nodes name are not shown. Use this var for keep track of requests for show node name
+k8s_node_name_show = false
 
 -- global table that contains k8s nodes
 kubernetes_nodes = {}
 
+-- Endpoint for deploy chaos-node
+chaos_node_url = ""
+
+-- keep track of request for showing k8s nodes
+kubernetes_nodes_request_show = false
+
+-- keep time of the latest request of showing nodes
+latest_kubernetes_nodes_request_show = 0
 
 local COLLISION_RESPONSE = hash("collision_response")
 local timer = require "main.mymodules.timer"
@@ -145,7 +153,10 @@ function start_chaos()
 		namespace = namespace_list[0]
 
 		info = "Endpoint: " .. conf["endpoint"] .. "\n\n" .. "Namespace: " .. namespace
+
 		kubelinter_url = endpoint .. "/kube-linter"
+		chaos_node_url = endpoint .. "/chaos-node"
+ 
 		msg.post("ui#gui", "info",{ infomessage =  info })
 
 		msg.post(".", "acquire_input_focus")	
@@ -155,10 +166,28 @@ function start_chaos()
 			["Content-Type"] = "application/json",
 			["Authorization"] = "Bearer " .. token
 		}
+
 		timer.repeat_seconds(pod_update_time, function()
 			pod_api.update_pod()
 			pod_mod.swap_pod(kubernetes_pods)		
 		end)
+
+		timer.repeat_seconds(2, function()
+			current_time_a = os.clock()
+			for k,v in ipairs(kubernetes_nodes) do
+				if  v["fired"] ~= nil and v["fired"] then
+					print(v["name"] .. " is in chaos state...")
+					if ((current_time_a - v["fired_time"]) > 0.8) then
+						print(v["name"] .. " is in chaos state since too time, it is time to switch to led state")
+						msg.post(v["id"], "play_animation", {id = hash("led")})	
+						new_value = v
+						new_value["fired"] = false
+						new_value["fired_time"] = 0
+						kubernetes_nodes[k] = new_value
+					end
+				end
+			end
+		end)
 		
 		timer.repeat_seconds(1, function()			
 			msg.post("ui#gui", "pod_name", { pod_name = false, position = go.get_position()})
@@ -249,9 +278,30 @@ function on_input(self, action_id, action)
 		msg.post("ui#gui", "mode",{ mode = "Press \"a\" to use automatic pilot"})
 
 	elseif action_id == hash("worker") then
-		print("Show kubernetes nodes...")
-		k8s_node.get_nodes()
 
+		current_time = os.clock()
+		
+		worker_diff_time = current_time - latest_kubernetes_nodes_request_show
+
+		if worker_diff_time > 0.05 then
+			kubernetes_nodes_size = table.getn(kubernetes_nodes)
+			if kubernetes_nodes_size == 0  and not kubernetes_nodes_request_show then
+				print("Show kubernetes nodes...")
+				k8s_node.get_nodes()
+				kubernetes_nodes_request_show = true
+			else
+				print("Kubernetes nodes already present...")
+				for k,v in ipairs(kubernetes_nodes) do
+					go.delete(v["id"])
+				end
+				kubernetes_nodes_request_show = false
+				kubernetes_nodes = {}
+			end
+			latest_kubernetes_nodes_request_show = os.clock()
+		end
+		
+
+	
 	elseif action_id == hash("namespace") then
 		print("[change_namespace] change namespaces requested by user")
 		msg.post("ui#gui", "error",{ errormessage = "changing namespace!" })
@@ -350,8 +400,7 @@ end
 
 function on_message(self, message_id, message, sender)
 	current_time = os.clock()
-	
-	if message_id == COLLISION_RESPONSE and not k8s_nodes_show then
+	if message_id == COLLISION_RESPONSE then
 		for i,value in ipairs(kubernetes_nodes) do
 			if message["other_id"] == value["id"] then
 				print("Spaceship collides with " .. value["name"])
@@ -359,10 +408,9 @@ function on_message(self, message_id, message, sender)
 				msg.post("ui#gui","node_name", { node_name = value["name"], spaceship_pos = spaceship_pos })				
 			end
 		end
-		return
 	end
 
-	if message_id == COLLISION_RESPONSE and started_chaos then
+	if message_id == COLLISION_RESPONSE and started_chaos and last_pod_log then
 
 		msg.post("ui#gui","pod_log")
 		

nginx/KubeInvaders.templ

@@ -14,6 +14,18 @@ server {
       ngx.say(result)
     }
   }
+
+  location /kube/chaos-node {
+    default_type text/html;
+    content_by_lua_block {
+      local arg = ngx.req.get_uri_args()
+      ngx.req.read_body()
+      local request_body = ngx.req.get_body_data()
+      local handle = io.popen("/opt/chaos-node.sh " .. arg['k8s_url'] .. " " .. arg['token'] .. " " .. arg['node_name'] .. " " ..  arg['namespace'])
+      local result = handle:read("*a")
+      ngx.say(result)
+    }
+  }
   
   location / {
     try_files $uri $uri/ =404;

nginx/KubeInvaders_dev.templ

@@ -14,6 +14,18 @@ server {
       ngx.say(result)
     }
   }
+
+  location /kube/chaos-node {
+    default_type text/html;
+    content_by_lua_block {
+      local arg = ngx.req.get_uri_args()
+      ngx.req.read_body()
+      local request_body = ngx.req.get_body_data()
+      local handle = io.popen("/opt/chaos-node.sh " .. arg['k8s_url'] .. " " .. arg['token'] .. " " .. arg['node_name'] .. " " ..  arg['namespace'])
+      local result = handle:read("*a")
+      ngx.say(result)
+    }
+  }
   
   location / {
     try_files $uri $uri/ =404;