mirror of
https://github.com/krkn-chaos/krkn.git
synced 2026-02-14 18:10:00 +00:00
* Added dataclasses to store info retrieved from k8 client calls * Replaced few invoke commands in common_litmus * Minor Documentation Changes * Removed unused import and redundant variable Signed-off-by: Shreyas Anantha Ramaprasad <ars.shreyas@gmail.com>
This commit is contained in:
Shreyas Anantha Ramaprasad
committed by
GitHub
GitHub
parent
bbb66aa322
commit
b01d9895fb
@@ -1,8 +1,10 @@
|
||||
from kubernetes import client, config
|
||||
from kubernetes import client, config, utils
|
||||
from kubernetes.dynamic.client import DynamicClient
|
||||
from kubernetes.stream import stream
|
||||
from kubernetes.client.rest import ApiException
|
||||
from dataclasses import dataclass
|
||||
from typing import List
|
||||
import logging
|
||||
import kraken.invoke.command as runcommand
|
||||
import sys
|
||||
import re
|
||||
import time
|
||||
@@ -14,10 +16,17 @@ kraken_node_name = ""
|
||||
def initialize_clients(kubeconfig_path):
|
||||
global cli
|
||||
global batch_cli
|
||||
global api_client
|
||||
global dyn_client
|
||||
global custom_object_client
|
||||
try:
|
||||
config.load_kube_config(kubeconfig_path)
|
||||
cli = client.CoreV1Api()
|
||||
batch_cli = client.BatchV1Api()
|
||||
api_client = client.ApiClient()
|
||||
custom_object_client = client.CustomObjectsApi()
|
||||
k8s_client = config.new_client_from_config()
|
||||
dyn_client = DynamicClient(k8s_client)
|
||||
except ApiException as e:
|
||||
logging.error("Failed to initialize kubernetes client: %s\n" % e)
|
||||
sys.exit(1)
|
||||
@@ -31,8 +40,7 @@ def get_host() -> str:
|
||||
def get_clusterversion_string() -> str:
|
||||
"""Returns clusterversion status text on OpenShift, empty string on other distributions"""
|
||||
try:
|
||||
custom_objects_api = client.CustomObjectsApi()
|
||||
cvs = custom_objects_api.list_cluster_custom_object(
|
||||
cvs = custom_object_client.list_cluster_custom_object(
|
||||
"config.openshift.io",
|
||||
"v1",
|
||||
"clusterversions",
|
||||
@@ -400,6 +408,161 @@ def monitor_component(iteration, component_namespace):
|
||||
return watch_component_status, failed_component_pods
|
||||
|
||||
|
||||
def apply_yaml(path, namespace='default'):
|
||||
"""
|
||||
Apply yaml config to create Kubernetes resources
|
||||
|
||||
Args:
|
||||
path (string)
|
||||
- Path to the YAML file
|
||||
namespace (string)
|
||||
- Namespace to create the resource
|
||||
|
||||
Returns:
|
||||
The object created
|
||||
"""
|
||||
|
||||
return utils.create_from_yaml(api_client, yaml_file=path, namespace=namespace)
|
||||
|
||||
|
||||
# Data class to hold information regarding containers in a pod
|
||||
@dataclass(frozen=True, order=False)
|
||||
class Container:
|
||||
image: str
|
||||
name: str
|
||||
ready: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True, order=False)
|
||||
class Pod:
|
||||
"""Data class to hold information regarding a pod"""
|
||||
name: str
|
||||
podIP: str
|
||||
namespace: str
|
||||
containers: List[Container]
|
||||
nodeName: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, order=False)
|
||||
class LitmusChaosObject:
|
||||
"""Data class to hold information regarding a custom object of litmus project"""
|
||||
kind: str
|
||||
group: str
|
||||
namespace: str
|
||||
name: str
|
||||
plural: str
|
||||
version: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, order=False)
|
||||
class ChaosEngine(LitmusChaosObject):
|
||||
"""Data class to hold information regarding a ChaosEngine object"""
|
||||
engineStatus: str
|
||||
expStatus: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, order=False)
|
||||
class ChaosResult(LitmusChaosObject):
|
||||
"""Data class to hold information regarding a ChaosResult object"""
|
||||
verdict: str
|
||||
failStep: str
|
||||
|
||||
|
||||
def get_pod_info(pod_name: str, namespace: str = 'default') -> Pod:
|
||||
"""
|
||||
Function to retrieve information about a specific pod
|
||||
in a given namespace. The kubectl command is given by:
|
||||
kubectl get pods <pod_name> -n <namespace>
|
||||
|
||||
Args:
|
||||
pod_name (string)
|
||||
- Name of the pod
|
||||
|
||||
namespace (string)
|
||||
- Namespace to look for the pod
|
||||
|
||||
Returns:
|
||||
Data class object of type Pod with the output of the above kubectl command
|
||||
in the given format
|
||||
"""
|
||||
|
||||
response = cli.read_namespaced_pod(name=pod_name, namespace=namespace, pretty='true')
|
||||
container_list = []
|
||||
|
||||
for container in response.status.container_statuses:
|
||||
container_list.append(Container(name=container.name, image=container.image, ready=container.ready))
|
||||
|
||||
pod_info = Pod(name=response.metadata.name, podIP=response.status.pod_ip, namespace=response.metadata.namespace,
|
||||
containers=container_list, nodeName=response.spec.node_name)
|
||||
return pod_info
|
||||
|
||||
|
||||
def get_litmus_chaos_object(kind: str, name: str, namespace: str) -> LitmusChaosObject:
|
||||
"""
|
||||
Function that returns an object of a custom resource typer the litmus project. Currently, only
|
||||
ChaosEngine and ChaosResult is supported.
|
||||
|
||||
Args:
|
||||
kind (string)
|
||||
- The custom resource type
|
||||
|
||||
namespace (string)
|
||||
- Namespace where the custom object is present
|
||||
|
||||
Returns:
|
||||
Data class object of a subclass of LitmusChaosObject
|
||||
"""
|
||||
|
||||
group = 'litmuschaos.io'
|
||||
version = 'v1alpha1'
|
||||
|
||||
if kind.lower() == 'chaosengine':
|
||||
plural = 'chaosengines'
|
||||
response = custom_object_client.get_namespaced_custom_object(group=group, plural=plural,
|
||||
version=version, namespace=namespace, name=name)
|
||||
try:
|
||||
engine_status = response['status']['engineStatus']
|
||||
exp_status = response['status']['experiments'][0]['status']
|
||||
except:
|
||||
engine_status = 'Not Initialized'
|
||||
exp_status = 'Not Initialized'
|
||||
custom_object = ChaosEngine(kind='ChaosEngine', group=group, namespace=namespace, name=name,
|
||||
plural=plural, version=version, engineStatus=engine_status,
|
||||
expStatus=exp_status)
|
||||
elif kind.lower() == 'chaosresult':
|
||||
plural = 'chaosresults'
|
||||
response = custom_object_client.get_namespaced_custom_object(group=group, plural=plural,
|
||||
version=version, namespace=namespace, name=name)
|
||||
try:
|
||||
verdict = response['status']['experimentStatus']['verdict']
|
||||
fail_step = response['status']['experimentStatus']['failStep']
|
||||
except:
|
||||
verdict = 'N/A'
|
||||
fail_step = 'N/A'
|
||||
custom_object = ChaosResult(kind='ChaosResult', group=group, namespace=namespace, name=name,
|
||||
plural=plural, version=version, verdict=verdict, failStep=fail_step)
|
||||
else:
|
||||
logging.error("Invalid litmus chaos custom resource name")
|
||||
custom_object = None
|
||||
return custom_object
|
||||
|
||||
|
||||
def check_if_namespace_exists(name: str):
|
||||
"""
|
||||
Function that checks if a namespace exists by parsing through the list of projects
|
||||
Args:
|
||||
name (string)
|
||||
- Namespace name
|
||||
|
||||
Returns:
|
||||
Boolean value indicating whethere the namespace exists or not
|
||||
"""
|
||||
|
||||
v1_projects = dyn_client.resources.get(api_version='project.openshift.io/v1', kind='Project')
|
||||
project_list = v1_projects.get()
|
||||
return True if name in str(project_list) else False
|
||||
|
||||
|
||||
# Find the node kraken is deployed on
|
||||
# Set global kraken node to not delete
|
||||
def find_kraken_node():
|
||||
@@ -415,14 +578,7 @@ def find_kraken_node():
|
||||
if kraken_pod_name:
|
||||
# get kraken-deployment pod, find node name
|
||||
try:
|
||||
node_name = runcommand.invoke(
|
||||
"kubectl get pods/"
|
||||
+ str(kraken_pod_name)
|
||||
+ ' -o jsonpath="{.spec.nodeName}"'
|
||||
+ " -n"
|
||||
+ str(kraken_project)
|
||||
)
|
||||
|
||||
node_name = get_pod_info(kraken_pod_name, kraken_project).nodeName
|
||||
global kraken_node_name
|
||||
kraken_node_name = node_name
|
||||
except Exception as e:
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import kraken.invoke.command as runcommand
|
||||
import kraken.kubernetes.client as kubecli
|
||||
import logging
|
||||
import time
|
||||
import sys
|
||||
@@ -86,18 +87,17 @@ def deploy_all_experiments(version_string, namespace):
|
||||
|
||||
|
||||
def wait_for_initialized(engine_name, experiment_name, namespace):
|
||||
chaos_engine = runcommand.invoke(
|
||||
"kubectl get chaosengines/%s -n %s -o jsonpath='{.status.engineStatus}'" % (engine_name, namespace)
|
||||
)
|
||||
|
||||
chaos_engine = kubecli.get_litmus_chaos_object(kind='chaosengine', name=engine_name,
|
||||
namespace=namespace).engineStatus
|
||||
engine_status = chaos_engine.strip()
|
||||
max_tries = 30
|
||||
engine_counter = 0
|
||||
while engine_status.lower() != "initialized":
|
||||
time.sleep(10)
|
||||
logging.info("Waiting for " + experiment_name + " to be initialized")
|
||||
chaos_engine = runcommand.invoke(
|
||||
"kubectl get chaosengines/%s -n %s -o jsonpath='{.status.engineStatus}'" % (engine_name, namespace)
|
||||
)
|
||||
chaos_engine = kubecli.get_litmus_chaos_object(kind='chaosengine', name=engine_name,
|
||||
namespace=namespace).engineStatus
|
||||
engine_status = chaos_engine.strip()
|
||||
if engine_counter >= max_tries:
|
||||
logging.error("Chaos engine " + experiment_name + " took longer than 5 minutes to be initialized")
|
||||
@@ -117,18 +117,16 @@ def wait_for_status(engine_name, expected_status, experiment_name, namespace):
|
||||
if not response:
|
||||
logging.info("Chaos engine never initialized, exiting")
|
||||
return False
|
||||
chaos_engine = runcommand.invoke(
|
||||
"kubectl get chaosengines/%s -n %s -o jsonpath='{.status.experiments[0].status}'" % (engine_name, namespace)
|
||||
)
|
||||
chaos_engine = kubecli.get_litmus_chaos_object(kind='chaosengine', name=engine_name,
|
||||
namespace=namespace).expStatus
|
||||
engine_status = chaos_engine.strip()
|
||||
max_tries = 30
|
||||
engine_counter = 0
|
||||
while engine_status.lower() != expected_status:
|
||||
time.sleep(10)
|
||||
logging.info("Waiting for " + experiment_name + " to be " + expected_status)
|
||||
chaos_engine = runcommand.invoke(
|
||||
"kubectl get chaosengines/%s -n %s -o jsonpath='{.status.experiments[0].status}'" % (engine_name, namespace)
|
||||
)
|
||||
chaos_engine = kubecli.get_litmus_chaos_object(kind='chaosengine', name=engine_name,
|
||||
namespace=namespace).expStatus
|
||||
engine_status = chaos_engine.strip()
|
||||
if engine_counter >= max_tries:
|
||||
logging.error("Chaos engine " + experiment_name + " took longer than 5 minutes to be " + expected_status)
|
||||
@@ -151,20 +149,14 @@ def check_experiment(engine_name, experiment_name, namespace):
|
||||
else:
|
||||
sys.exit(1)
|
||||
|
||||
chaos_result = runcommand.invoke(
|
||||
"kubectl get chaosresult %s"
|
||||
"-%s -n %s -o "
|
||||
"jsonpath='{.status.experimentStatus.verdict}'" % (engine_name, experiment_name, namespace)
|
||||
)
|
||||
chaos_result = kubecli.get_litmus_chaos_object(kind='chaosresult', name=engine_name+'-'+experiment_name,
|
||||
namespace=namespace).verdict
|
||||
if chaos_result == "Pass":
|
||||
logging.info("Engine " + str(engine_name) + " finished with status " + str(chaos_result))
|
||||
return True
|
||||
else:
|
||||
chaos_result = runcommand.invoke(
|
||||
"kubectl get chaosresult %s"
|
||||
"-%s -n %s -o jsonpath="
|
||||
"'{.status.experimentStatus.failStep}'" % (engine_name, experiment_name, namespace)
|
||||
)
|
||||
chaos_result = kubecli.get_litmus_chaos_object(kind='chaosresult', name=engine_name+'-'+experiment_name,
|
||||
namespace=namespace).failStep
|
||||
logging.info("Chaos scenario:" + engine_name + " failed with error: " + str(chaos_result))
|
||||
logging.info(
|
||||
"See 'kubectl get chaosresult %s"
|
||||
@@ -176,8 +168,7 @@ def check_experiment(engine_name, experiment_name, namespace):
|
||||
# Delete all chaos engines in a given namespace
|
||||
def delete_chaos_experiments(namespace):
|
||||
|
||||
namespace_exists = runcommand.invoke("oc get project -o name | grep -c " + namespace + " | xargs")
|
||||
if namespace_exists.strip() != "0":
|
||||
if kubecli.check_if_namespace_exists(namespace):
|
||||
chaos_exp_exists = runcommand.invoke_no_exit("kubectl get chaosexperiment")
|
||||
if "returned non-zero exit status 1" not in chaos_exp_exists:
|
||||
logging.info("Deleting all litmus experiments")
|
||||
@@ -187,8 +178,7 @@ def delete_chaos_experiments(namespace):
|
||||
# Delete all chaos engines in a given namespace
|
||||
def delete_chaos(namespace):
|
||||
|
||||
namespace_exists = runcommand.invoke("oc get project -o name | grep -c " + namespace + " | xargs")
|
||||
if namespace_exists.strip() != "0":
|
||||
if kubecli.check_if_namespace_exists(namespace):
|
||||
logging.info("Deleting all litmus run objects")
|
||||
chaos_engine_exists = runcommand.invoke_no_exit("kubectl get chaosengine")
|
||||
if "returned non-zero exit status 1" not in chaos_engine_exists:
|
||||
@@ -201,8 +191,8 @@ def delete_chaos(namespace):
|
||||
|
||||
|
||||
def uninstall_litmus(version, litmus_namespace):
|
||||
namespace_exists = runcommand.invoke("oc get project -o name | grep -c " + litmus_namespace + " | xargs")
|
||||
if namespace_exists.strip() != "0":
|
||||
|
||||
if kubecli.check_if_namespace_exists(litmus_namespace):
|
||||
logging.info("Uninstalling Litmus operator")
|
||||
runcommand.invoke_no_exit(
|
||||
"kubectl delete -n %s -f "
|
||||
|
||||
@@ -119,14 +119,13 @@ def container_killing_in_pod(cont_scenario):
|
||||
container_pod_list = []
|
||||
for pod in pods:
|
||||
if type(pod) == list:
|
||||
container_names = runcommand.invoke(
|
||||
'kubectl get pods %s -n %s -o jsonpath="{.spec.containers[*].name}"' % (pod[0], pod[1])
|
||||
).split(" ")
|
||||
pod_output = kubecli.get_pod_info(pod[0], pod[1])
|
||||
container_names = [container.name for container in pod_output.containers]
|
||||
|
||||
container_pod_list.append([pod[0], pod[1], container_names])
|
||||
else:
|
||||
container_names = runcommand.invoke(
|
||||
'oc get pods %s -n %s -o jsonpath="{.spec.containers[*].name}"' % (pod, namespace)
|
||||
).split(" ")
|
||||
pod_output = kubecli.get_pod_info(pod, namespace)
|
||||
container_names = [container.name for container in pod_output.containers]
|
||||
container_pod_list.append([pod, namespace, container_names])
|
||||
|
||||
killed_count = 0
|
||||
@@ -176,13 +175,11 @@ def check_failed_containers(killed_container_list, wait_time):
|
||||
while timer <= wait_time:
|
||||
for killed_container in killed_container_list:
|
||||
# pod namespace contain name
|
||||
pod_output = runcommand.invoke(
|
||||
"kubectl get pods %s -n %s -o yaml" % (killed_container[0], killed_container[1])
|
||||
)
|
||||
pod_output_yaml = yaml.full_load(pod_output)
|
||||
for statuses in pod_output_yaml["status"]["containerStatuses"]:
|
||||
if statuses["name"] == killed_container[2]:
|
||||
if str(statuses["ready"]).lower() == "true":
|
||||
pod_output = kubecli.get_pod_info(killed_container[0], killed_container[1])
|
||||
|
||||
for container in pod_output.containers:
|
||||
if container.name == killed_container[2]:
|
||||
if container.ready:
|
||||
container_ready.append(killed_container)
|
||||
if len(container_ready) != 0:
|
||||
for item in container_ready:
|
||||
|
||||
Reference in New Issue
Block a user