github/krkn
1
0
Fork 0
mirror of https://github.com/krkn-chaos/krkn.git synced 2026-04-15 06:57:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

Hog scenario porting from arcaflow to native (#748)

Browse Source 
* added new native hog scenario

* removed arcaflow dependency + legacy hog scenarios

* config update

* changed hog configuration structure + added average samples

* fix on cpu count

* removes tripledes warning

* changed selector format

* changed selector syntax

* number of nodes option

* documentation

* functional tests

* exception handling on hog deployment thread

Signed-off-by: Paige Patton <prubenda@redhat.com>
This commit is contained in:
Tullio Sebastiani
2025-01-31 13:45:59 -05:00
committed by Naga Ravi Chaitanya Elluri
parent c7e068a562
commit b024cfde19
14 changed files with 294 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files
krkn/scenario_plugins/abstract_scenario_plugin.py
+10 -7
View File
@@ -56,6 +56,7 @@ class AbstractScenarioPlugin(ABC):
scenario_telemetries: list[ScenarioTelemetry] = []
failed_scenarios = []
wait_duration = krkn_config["tunings"]["wait_duration"]
events_backup = krkn_config["telemetry"]["events_backup"]
for scenario_config in scenarios_list:
if isinstance(scenario_config, list):
logging.error(
@@ -99,13 +100,15 @@ class AbstractScenarioPlugin(ABC):
int(scenario_telemetry.start_timestamp),
int(scenario_telemetry.end_timestamp),
)
utils.populate_cluster_events(
scenario_telemetry,
parsed_scenario_config,
telemetry.get_lib_kubernetes(),
int(scenario_telemetry.start_timestamp),
int(scenario_telemetry.end_timestamp),
)
if events_backup:
utils.populate_cluster_events(
scenario_telemetry,
parsed_scenario_config,
telemetry.get_lib_kubernetes(),
int(scenario_telemetry.start_timestamp),
int(scenario_telemetry.end_timestamp),
)
if scenario_telemetry.exit_status != 0:
failed_scenarios.append(scenario_config)
krkn/scenario_plugins/node_actions/abstract_node_scenarios.py
+14 -5
View File
@@ -4,14 +4,16 @@ import time
import krkn.invoke.command as runcommand
import krkn.scenario_plugins.node_actions.common_node_functions as nodeaction
from krkn_lib.k8s import KrknKubernetes
from krkn_lib.models.k8s import AffectedNode, AffectedNodeStatus
# krkn_lib
class abstract_node_scenarios:
kubecli: KrknKubernetes
affected_nodes_status: AffectedNodeStatus
def __init__(self, kubecli: KrknKubernetes):
def __init__(self, kubecli: KrknKubernetes, affected_nodes_status: AffectedNodeStatus):
self.kubecli = kubecli
self.affected_nodes_status = affected_nodes_status
# Node scenario to start the node
def node_start_scenario(self, instance_kill_count, node, timeout):
@@ -28,6 +30,7 @@ class abstract_node_scenarios:
logging.info("Waiting for %s seconds before starting the node" % (duration))
time.sleep(duration)
self.node_start_scenario(instance_kill_count, node, timeout)
self.affected_nodes_status.merge_affected_nodes()
logging.info("node_stop_start_scenario has been successfully injected!")
def helper_node_stop_start_scenario(self, instance_kill_count, node, timeout):
@@ -61,13 +64,15 @@ class abstract_node_scenarios:
# Node scenario to stop the kubelet
def stop_kubelet_scenario(self, instance_kill_count, node, timeout):
for _ in range(instance_kill_count):
affected_node = AffectedNode(node)
try:
logging.info("Starting stop_kubelet_scenario injection")
logging.info("Stopping the kubelet of the node %s" % (node))
runcommand.run(
"oc debug node/" + node + " -- chroot /host systemctl stop kubelet"
)
nodeaction.wait_for_unknown_status(node, timeout, self.kubecli)
nodeaction.wait_for_unknown_status(node, timeout, self.kubecli, affected_node)
logging.info("The kubelet of the node %s has been stopped" % (node))
logging.info("stop_kubelet_scenario has been successfuly injected!")
except Exception as e:
@@ -77,17 +82,20 @@ class abstract_node_scenarios:
)
logging.error("stop_kubelet_scenario injection failed!")
raise e
self.add_affected_node(affected_node)
# Node scenario to stop and start the kubelet
def stop_start_kubelet_scenario(self, instance_kill_count, node, timeout):
logging.info("Starting stop_start_kubelet_scenario injection")
self.stop_kubelet_scenario(instance_kill_count, node, timeout)
self.node_reboot_scenario(instance_kill_count, node, timeout)
self.affected_nodes_status.merge_affected_nodes()
logging.info("stop_start_kubelet_scenario has been successfully injected!")
# Node scenario to restart the kubelet
def restart_kubelet_scenario(self, instance_kill_count, node, timeout):
for _ in range(instance_kill_count):
affected_node = AffectedNode(node)
try:
logging.info("Starting restart_kubelet_scenario injection")
logging.info("Restarting the kubelet of the node %s" % (node))
@@ -96,8 +104,8 @@ class abstract_node_scenarios:
+ node
+ " -- chroot /host systemctl restart kubelet &"
)
nodeaction.wait_for_not_ready_status(node, timeout, self.kubecli)
nodeaction.wait_for_ready_status(node, timeout, self.kubecli)
nodeaction.wait_for_not_ready_status(node, timeout, self.kubecli, affected_node)
nodeaction.wait_for_ready_status(node, timeout, self.kubecli,affected_node)
logging.info("The kubelet of the node %s has been restarted" % (node))
logging.info("restart_kubelet_scenario has been successfuly injected!")
except Exception as e:
@@ -107,6 +115,7 @@ class abstract_node_scenarios:
)
logging.error("restart_kubelet_scenario injection failed!")
raise e
self.add_affected_node(affected_node)
# Node scenario to crash the node
def node_crash_scenario(self, instance_kill_count, node, timeout):
krkn/scenario_plugins/node_actions/alibaba_node_scenarios.py
+35 -13
View File
@@ -18,7 +18,7 @@ from krkn.scenario_plugins.node_actions.abstract_node_scenarios import (
abstract_node_scenarios,
)
from krkn_lib.k8s import KrknKubernetes
from krkn_lib.models.k8s import AffectedNode, AffectedNodeStatus
class Alibaba:
def __init__(self):
@@ -161,8 +161,9 @@ class Alibaba:
return None
# Wait until the node instance is running
def wait_until_running(self, instance_id, timeout):
def wait_until_running(self, instance_id, timeout, affected_node):
time_counter = 0
start_time = time.time()
status = self.get_vm_status(instance_id)
while status != "Running":
status = self.get_vm_status(instance_id)
@@ -174,11 +175,15 @@ class Alibaba:
if time_counter >= timeout:
logging.info("ECS %s is still not ready in allotted time" % instance_id)
return False
end_time = time.time()
if affected_node:
affected_node.set_affected_node_status("running", end_time - start_time)
return True
# Wait until the node instance is stopped
def wait_until_stopped(self, instance_id, timeout):
def wait_until_stopped(self, instance_id, timeout, affected_node):
time_counter = 0
start_time = time.time()
status = self.get_vm_status(instance_id)
while status != "Stopped":
status = self.get_vm_status(instance_id)
@@ -192,10 +197,14 @@ class Alibaba:
"Vm %s is still not stopped in allotted time" % instance_id
)
return False
end_time = time.time()
if affected_node:
affected_node.set_affected_node_status("running", end_time - start_time)
return True
# Wait until the node instance is terminated
def wait_until_released(self, instance_id, timeout):
def wait_until_released(self, instance_id, timeout, affected_node):
start_time = time.time()
statuses = self.get_vm_status(instance_id)
time_counter = 0
while statuses and statuses != "Released":
@@ -210,17 +219,23 @@ class Alibaba:
return False
logging.info("ECS %s is released" % instance_id)
end_time = time.time()
if affected_node:
affected_node.set_affected_node_status("terminated", end_time - start_time)
return True
# krkn_lib
class alibaba_node_scenarios(abstract_node_scenarios):
def __init__(self, kubecli: KrknKubernetes):
def __init__(self, kubecli: KrknKubernetes, affected_nodes_status: AffectedNodeStatus):
super().__init__(kubecli, affected_nodes_status)
self.alibaba = Alibaba()
# Node scenario to start the node
def node_start_scenario(self, instance_kill_count, node, timeout):
for _ in range(instance_kill_count):
affected_node = AffectedNode(node)
try:
logging.info("Starting node_start_scenario injection")
vm_id = self.alibaba.get_instance_id(node)
@@ -228,8 +243,8 @@ class alibaba_node_scenarios(abstract_node_scenarios):
"Starting the node %s with instance ID: %s " % (node, vm_id)
)
self.alibaba.start_instances(vm_id)
self.alibaba.wait_until_running(vm_id, timeout)
nodeaction.wait_for_ready_status(node, timeout, self.kubecli)
self.alibaba.wait_until_running(vm_id, timeout, affected_node)
nodeaction.wait_for_ready_status(node, timeout, self.kubecli, affected_node)
logging.info("Node with instance ID: %s is in running state" % node)
logging.info("node_start_scenario has been successfully injected!")
except Exception as e:
@@ -239,10 +254,12 @@ class alibaba_node_scenarios(abstract_node_scenarios):
)
logging.error("node_start_scenario injection failed!")
raise e
self.affected_nodes_status.affected_nodes.append(affected_node)
# Node scenario to stop the node
def node_stop_scenario(self, instance_kill_count, node, timeout):
for _ in range(instance_kill_count):
affected_node = AffectedNode(node)
try:
logging.info("Starting node_stop_scenario injection")
vm_id = self.alibaba.get_instance_id(node)
@@ -250,9 +267,9 @@ class alibaba_node_scenarios(abstract_node_scenarios):
"Stopping the node %s with instance ID: %s " % (node, vm_id)
)
self.alibaba.stop_instances(vm_id)
self.alibaba.wait_until_stopped(vm_id, timeout)
self.alibaba.wait_until_stopped(vm_id, timeout, affected_node)
logging.info("Node with instance ID: %s is in stopped state" % vm_id)
nodeaction.wait_for_unknown_status(node, timeout, self.kubecli)
nodeaction.wait_for_unknown_status(node, timeout, self.kubecli, affected_node)
except Exception as e:
logging.error(
"Failed to stop node instance. Encountered following exception: %s. "
@@ -260,23 +277,25 @@ class alibaba_node_scenarios(abstract_node_scenarios):
)
logging.error("node_stop_scenario injection failed!")
raise e
self.affected_nodes_status.affected_nodes.append(affected_node)
# Might need to stop and then release the instance
# Node scenario to terminate the node
def node_termination_scenario(self, instance_kill_count, node, timeout):
for _ in range(instance_kill_count):
affected_node = AffectedNode(node)
try:
logging.info(
"Starting node_termination_scenario injection by first stopping instance"
)
vm_id = self.alibaba.get_instance_id(node)
self.alibaba.stop_instances(vm_id)
self.alibaba.wait_until_stopped(vm_id, timeout)
self.alibaba.wait_until_stopped(vm_id, timeout, affected_node)
logging.info(
"Releasing the node %s with instance ID: %s " % (node, vm_id)
)
self.alibaba.release_instance(vm_id)
self.alibaba.wait_until_released(vm_id, timeout)
self.alibaba.wait_until_released(vm_id, timeout, affected_node)
logging.info("Node with instance ID: %s has been released" % node)
logging.info(
"node_termination_scenario has been successfully injected!"
@@ -288,17 +307,19 @@ class alibaba_node_scenarios(abstract_node_scenarios):
)
logging.error("node_termination_scenario injection failed!")
raise e
self.affected_nodes_status.affected_nodes.append(affected_node)
# Node scenario to reboot the node
def node_reboot_scenario(self, instance_kill_count, node, timeout):
for _ in range(instance_kill_count):
affected_node = AffectedNode(node)
try:
logging.info("Starting node_reboot_scenario injection")
instance_id = self.alibaba.get_instance_id(node)
logging.info("Rebooting the node with instance ID: %s " % (instance_id))
self.alibaba.reboot_instances(instance_id)
nodeaction.wait_for_unknown_status(node, timeout, self.kubecli)
nodeaction.wait_for_ready_status(node, timeout, self.kubecli)
nodeaction.wait_for_unknown_status(node, timeout, self.kubecli, affected_node)
nodeaction.wait_for_ready_status(node, timeout, self.kubecli, affected_node)
logging.info(
"Node with instance ID: %s has been rebooted" % (instance_id)
)
@@ -310,3 +331,4 @@ class alibaba_node_scenarios(abstract_node_scenarios):
)
logging.error("node_reboot_scenario injection failed!")
raise e
self.affected_nodes_status.affected_nodes.append(affected_node)
krkn/scenario_plugins/node_actions/aws_node_scenarios.py
+33 -13
View File
@@ -7,7 +7,7 @@ from krkn.scenario_plugins.node_actions.abstract_node_scenarios import (
abstract_node_scenarios,
)
from krkn_lib.k8s import KrknKubernetes
from krkn_lib.models.k8s import AffectedNode, AffectedNodeStatus
class AWS:
def __init__(self):
@@ -77,9 +77,13 @@ class AWS:
# until a successful state is reached. An error is returned after 40 failed checks
# Setting timeout for consistency with other cloud functions
# Wait until the node instance is running
def wait_until_running(self, instance_id, timeout=600):
def wait_until_running(self, instance_id, timeout=600, affected_node=None):
try:
start_time = time.time()
self.boto_instance.wait_until_running(InstanceIds=[instance_id])
end_time = time.time()
if affected_node:
affected_node.set_affected_node_status("running", end_time - start_time)
return True
except Exception as e:
logging.error(
@@ -89,9 +93,13 @@ class AWS:
return False
# Wait until the node instance is stopped
def wait_until_stopped(self, instance_id, timeout=600):
def wait_until_stopped(self, instance_id, timeout=600, affected_node= None):
try:
start_time = time.time()
self.boto_instance.wait_until_stopped(InstanceIds=[instance_id])
end_time = time.time()
if affected_node:
affected_node.set_affected_node_status("stopped", end_time - start_time)
return True
except Exception as e:
logging.error(
@@ -101,9 +109,13 @@ class AWS:
return False
# Wait until the node instance is terminated
def wait_until_terminated(self, instance_id, timeout=600):
def wait_until_terminated(self, instance_id, timeout=600, affected_node= None):
try:
start_time = time.time()
self.boto_instance.wait_until_terminated(InstanceIds=[instance_id])
end_time = time.time()
if affected_node:
affected_node.set_affected_node_status("terminated", end_time - start_time)
return True
except Exception as e:
logging.error(
@@ -249,13 +261,14 @@ class AWS:
# krkn_lib
class aws_node_scenarios(abstract_node_scenarios):
def __init__(self, kubecli: KrknKubernetes):
super().__init__(kubecli)
def __init__(self, kubecli: KrknKubernetes, affected_nodes_status: AffectedNodeStatus):
super().__init__(kubecli, affected_nodes_status)
self.aws = AWS()
# Node scenario to start the node
def node_start_scenario(self, instance_kill_count, node, timeout):
for _ in range(instance_kill_count):
affected_node = AffectedNode(node)
try:
logging.info("Starting node_start_scenario injection")
instance_id = self.aws.get_instance_id(node)
@@ -263,8 +276,8 @@ class aws_node_scenarios(abstract_node_scenarios):
"Starting the node %s with instance ID: %s " % (node, instance_id)
)
self.aws.start_instances(instance_id)
self.aws.wait_until_running(instance_id)
nodeaction.wait_for_ready_status(node, timeout, self.kubecli)
self.aws.wait_until_running(instance_id, affected_node=affected_node)
nodeaction.wait_for_ready_status(node, timeout, self.kubecli, affected_node)
logging.info(
"Node with instance ID: %s is in running state" % (instance_id)
)
@@ -277,10 +290,12 @@ class aws_node_scenarios(abstract_node_scenarios):
logging.error("node_start_scenario injection failed!")
raise RuntimeError()
self.affected_nodes_status.affected_nodes.append(affected_node)
# Node scenario to stop the node
def node_stop_scenario(self, instance_kill_count, node, timeout):
for _ in range(instance_kill_count):
affected_node = AffectedNode(node)
try:
logging.info("Starting node_stop_scenario injection")
instance_id = self.aws.get_instance_id(node)
@@ -288,11 +303,11 @@ class aws_node_scenarios(abstract_node_scenarios):
"Stopping the node %s with instance ID: %s " % (node, instance_id)
)
self.aws.stop_instances(instance_id)
self.aws.wait_until_stopped(instance_id)
self.aws.wait_until_stopped(instance_id, affected_node=affected_node)
logging.info(
"Node with instance ID: %s is in stopped state" % (instance_id)
)
nodeaction.wait_for_unknown_status(node, timeout, self.kubecli)
nodeaction.wait_for_unknown_status(node, timeout, self.kubecli, affected_node=affected_node)
except Exception as e:
logging.error(
"Failed to stop node instance. Encountered following exception: %s. "
@@ -301,10 +316,12 @@ class aws_node_scenarios(abstract_node_scenarios):
logging.error("node_stop_scenario injection failed!")
raise RuntimeError()
self.affected_nodes_status.affected_nodes.append(affected_node)
# Node scenario to terminate the node
def node_termination_scenario(self, instance_kill_count, node, timeout):
for _ in range(instance_kill_count):
affected_node = AffectedNode(node)
try:
logging.info("Starting node_termination_scenario injection")
instance_id = self.aws.get_instance_id(node)
@@ -313,7 +330,7 @@ class aws_node_scenarios(abstract_node_scenarios):
% (node, instance_id)
)
self.aws.terminate_instances(instance_id)
self.aws.wait_until_terminated(instance_id)
self.aws.wait_until_terminated(instance_id, affected_node=affected_node)
for _ in range(timeout):
if node not in self.kubecli.list_nodes():
break
@@ -332,10 +349,12 @@ class aws_node_scenarios(abstract_node_scenarios):
logging.error("node_termination_scenario injection failed!")
raise RuntimeError()
self.affected_nodes_status.affected_nodes.append(affected_node)
# Node scenario to reboot the node
def node_reboot_scenario(self, instance_kill_count, node, timeout):
for _ in range(instance_kill_count):
affected_node = AffectedNode(node)
try:
logging.info("Starting node_reboot_scenario injection" + str(node))
instance_id = self.aws.get_instance_id(node)
@@ -343,8 +362,8 @@ class aws_node_scenarios(abstract_node_scenarios):
"Rebooting the node %s with instance ID: %s " % (node, instance_id)
)
self.aws.reboot_instances(instance_id)
nodeaction.wait_for_unknown_status(node, timeout, self.kubecli)
nodeaction.wait_for_ready_status(node, timeout, self.kubecli)
nodeaction.wait_for_unknown_status(node, timeout, self.kubecli, affected_node)
nodeaction.wait_for_ready_status(node, timeout, self.kubecli, affected_node)
logging.info(
"Node with instance ID: %s has been rebooted" % (instance_id)
)
@@ -357,6 +376,7 @@ class aws_node_scenarios(abstract_node_scenarios):
logging.error("node_reboot_scenario injection failed!")
raise RuntimeError()
self.affected_nodes_status.affected_nodes.append(affected_node)
# Get volume attachment info
def get_disk_attachment_info(self, instance_kill_count, node):
krkn/scenario_plugins/node_actions/az_node_scenarios.py
+43 -14
View File
@@ -8,7 +8,7 @@ from krkn.scenario_plugins.node_actions.abstract_node_scenarios import (
from azure.mgmt.compute import ComputeManagementClient
from azure.identity import DefaultAzureCredential
from krkn_lib.k8s import KrknKubernetes
from krkn_lib.models.k8s import AffectedNode, AffectedNodeStatus
class Azure:
def __init__(self):
@@ -18,8 +18,11 @@ class Azure:
logging.info("credential " + str(credentials))
# az_account = runcommand.invoke("az account list -o yaml")
# az_account_yaml = yaml.safe_load(az_account, Loader=yaml.FullLoader)
logger = logging.getLogger("azure")
logger.setLevel(logging.WARNING)
subscription_id = os.getenv("AZURE_SUBSCRIPTION_ID")
self.compute_client = ComputeManagementClient(credentials, subscription_id)
self.compute_client = ComputeManagementClient(credentials, subscription_id,logging=logger)
# Get the instance ID of the node
def get_instance_id(self, node_name):
@@ -90,8 +93,9 @@ class Azure:
return status
# Wait until the node instance is running
def wait_until_running(self, resource_group, vm_name, timeout):
def wait_until_running(self, resource_group, vm_name, timeout, affected_node):
time_counter = 0
start_time = time.time()
status = self.get_vm_status(resource_group, vm_name)
while status and status.code != "PowerState/running":
status = self.get_vm_status(resource_group, vm_name)
@@ -101,11 +105,15 @@ class Azure:
if time_counter >= timeout:
logging.info("Vm %s is still not ready in allotted time" % vm_name)
return False
end_time = time.time()
if affected_node:
affected_node.set_affected_node_status("running", end_time - start_time)
return True
# Wait until the node instance is stopped
def wait_until_stopped(self, resource_group, vm_name, timeout):
def wait_until_stopped(self, resource_group, vm_name, timeout, affected_node):
time_counter = 0
start_time = time.time()
status = self.get_vm_status(resource_group, vm_name)
while status and status.code != "PowerState/stopped":
status = self.get_vm_status(resource_group, vm_name)
@@ -115,10 +123,14 @@ class Azure:
if time_counter >= timeout:
logging.info("Vm %s is still not stopped in allotted time" % vm_name)
return False
end_time = time.time()
if affected_node:
affected_node.set_affected_node_status("stopped", end_time - start_time)
return True
# Wait until the node instance is terminated
def wait_until_terminated(self, resource_group, vm_name, timeout):
def wait_until_terminated(self, resource_group, vm_name, timeout, affected_node):
start_time = time.time()
statuses = self.compute_client.virtual_machines.instance_view(
resource_group, vm_name
).statuses[0]
@@ -137,29 +149,35 @@ class Azure:
return False
except Exception:
logging.info("Vm %s is terminated" % vm_name)
end_time = time.time()
if affected_node:
affected_node.set_affected_node_status("terminated", end_time - start_time)
return True
# krkn_lib
class azure_node_scenarios(abstract_node_scenarios):
def __init__(self, kubecli: KrknKubernetes):
super().__init__(kubecli)
def __init__(self, kubecli: KrknKubernetes, affected_nodes_status: AffectedNodeStatus):
super().__init__(kubecli, affected_nodes_status)
logging.info("init in azure")
self.azure = Azure()
# Node scenario to start the node
def node_start_scenario(self, instance_kill_count, node, timeout):
for _ in range(instance_kill_count):
affected_node = AffectedNode(node)
try:
logging.info("Starting node_start_scenario injection")
vm_name, resource_group = self.azure.get_instance_id(node)
logging.info(
"Starting the node %s with instance ID: %s "
% (vm_name, resource_group)
)
self.azure.start_instances(resource_group, vm_name)
self.azure.wait_until_running(resource_group, vm_name, timeout)
nodeaction.wait_for_ready_status(vm_name, timeout, self.kubecli)
self.azure.wait_until_running(resource_group, vm_name, timeout, affected_node=affected_node)
nodeaction.wait_for_ready_status(vm_name, timeout, self.kubecli, affected_node)
logging.info("Node with instance ID: %s is in running state" % node)
logging.info("node_start_scenario has been successfully injected!")
except Exception as e:
@@ -170,10 +188,12 @@ class azure_node_scenarios(abstract_node_scenarios):
logging.error("node_start_scenario injection failed!")
raise RuntimeError()
self.affected_nodes_status.affected_nodes.append(affected_node)
# Node scenario to stop the node
def node_stop_scenario(self, instance_kill_count, node, timeout):
for _ in range(instance_kill_count):
affected_node = AffectedNode(node)
try:
logging.info("Starting node_stop_scenario injection")
vm_name, resource_group = self.azure.get_instance_id(node)
@@ -182,9 +202,9 @@ class azure_node_scenarios(abstract_node_scenarios):
% (vm_name, resource_group)
)
self.azure.stop_instances(resource_group, vm_name)
self.azure.wait_until_stopped(resource_group, vm_name, timeout)
self.azure.wait_until_stopped(resource_group, vm_name, timeout, affected_node=affected_node)
logging.info("Node with instance ID: %s is in stopped state" % vm_name)
nodeaction.wait_for_unknown_status(vm_name, timeout, self.kubecli)
nodeaction.wait_for_unknown_status(vm_name, timeout, self.kubecli, affected_node)
except Exception as e:
logging.error(
"Failed to stop node instance. Encountered following exception: %s. "
@@ -193,19 +213,22 @@ class azure_node_scenarios(abstract_node_scenarios):
logging.error("node_stop_scenario injection failed!")
raise RuntimeError()
self.affected_nodes_status.affected_nodes.append(affected_node)
# Node scenario to terminate the node
def node_termination_scenario(self, instance_kill_count, node, timeout):
for _ in range(instance_kill_count):
affected_node = AffectedNode(node)
try:
logging.info("Starting node_termination_scenario injection")
affected_node = AffectedNode(node)
vm_name, resource_group = self.azure.get_instance_id(node)
logging.info(
"Terminating the node %s with instance ID: %s "
% (vm_name, resource_group)
)
self.azure.terminate_instances(resource_group, vm_name)
self.azure.wait_until_terminated(resource_group, vm_name, timeout)
self.azure.wait_until_terminated(resource_group, vm_name, timeout, affected_node)
for _ in range(timeout):
if vm_name not in self.kubecli.list_nodes():
break
@@ -224,10 +247,13 @@ class azure_node_scenarios(abstract_node_scenarios):
logging.error("node_termination_scenario injection failed!")
raise RuntimeError()
self.affected_nodes_status.affected_nodes.append(affected_node)
# Node scenario to reboot the node
def node_reboot_scenario(self, instance_kill_count, node, timeout):
for _ in range(instance_kill_count):
affected_node = AffectedNode(node)
try:
logging.info("Starting node_reboot_scenario injection")
vm_name, resource_group = self.azure.get_instance_id(node)
@@ -235,9 +261,11 @@ class azure_node_scenarios(abstract_node_scenarios):
"Rebooting the node %s with instance ID: %s "
% (vm_name, resource_group)
)
self.azure.reboot_instances(resource_group, vm_name)
nodeaction.wait_for_unknown_status(vm_name, timeout, self.kubecli)
nodeaction.wait_for_ready_status(vm_name, timeout, self.kubecli)
nodeaction.wait_for_ready_status(vm_name, timeout, self.kubecli, affected_node)
logging.info("Node with instance ID: %s has been rebooted" % (vm_name))
logging.info("node_reboot_scenario has been successfully injected!")
except Exception as e:
@@ -248,3 +276,4 @@ class azure_node_scenarios(abstract_node_scenarios):
logging.error("node_reboot_scenario injection failed!")
raise RuntimeError()
self.affected_nodes_status.affected_nodes.append(affected_node)
krkn/scenario_plugins/node_actions/bm_node_scenarios.py
+11 -7
View File
@@ -9,7 +9,7 @@ import pyipmi.interfaces
import time
import traceback
from krkn_lib.k8s import KrknKubernetes
from krkn_lib.models.k8s import AffectedNode, AffectedNodeStatus
class BM:
def __init__(self, bm_info, user, passwd):
@@ -127,8 +127,8 @@ class BM:
# krkn_lib
class bm_node_scenarios(abstract_node_scenarios):
def __init__(self, bm_info, user, passwd, kubecli: KrknKubernetes):
super().__init__(kubecli)
def __init__(self, bm_info, user, passwd, kubecli: KrknKubernetes,affected_nodes_status: AffectedNodeStatus):
super().__init__(kubecli, affected_nodes_status)
self.bm = BM(bm_info, user, passwd)
# Node scenario to start the node
@@ -159,6 +159,7 @@ class bm_node_scenarios(abstract_node_scenarios):
# Node scenario to stop the node
def node_stop_scenario(self, instance_kill_count, node, timeout):
for _ in range(instance_kill_count):
affected_node = AffectedNode(node)
try:
logging.info("Starting node_stop_scenario injection")
bmc_addr = self.bm.get_bmc_addr(node)
@@ -166,11 +167,11 @@ class bm_node_scenarios(abstract_node_scenarios):
"Stopping the node %s with bmc address: %s " % (node, bmc_addr)
)
self.bm.stop_instances(bmc_addr, node)
self.bm.wait_until_stopped(bmc_addr, node)
self.bm.wait_until_stopped(bmc_addr, node, affected_node)
logging.info(
"Node with bmc address: %s is in stopped state" % (bmc_addr)
)
nodeaction.wait_for_unknown_status(node, timeout, self.kubecli)
nodeaction.wait_for_unknown_status(node, timeout, self.kubecli, affected_node)
except Exception as e:
logging.error(
"Failed to stop node instance. Encountered following exception: %s. "
@@ -179,6 +180,7 @@ class bm_node_scenarios(abstract_node_scenarios):
)
logging.error("node_stop_scenario injection failed!")
raise e
self.affected_nodes_status.affected_nodes.append(affected_node)
# Node scenario to terminate the node
def node_termination_scenario(self, instance_kill_count, node, timeout):
@@ -187,6 +189,7 @@ class bm_node_scenarios(abstract_node_scenarios):
# Node scenario to reboot the node
def node_reboot_scenario(self, instance_kill_count, node, timeout):
for _ in range(instance_kill_count):
affected_node = AffectedNode(node)
try:
logging.info("Starting node_reboot_scenario injection")
bmc_addr = self.bm.get_bmc_addr(node)
@@ -195,8 +198,8 @@ class bm_node_scenarios(abstract_node_scenarios):
"Rebooting the node %s with bmc address: %s " % (node, bmc_addr)
)
self.bm.reboot_instances(bmc_addr, node)
nodeaction.wait_for_unknown_status(node, timeout, self.kubecli)
nodeaction.wait_for_ready_status(node, timeout, self.kubecli)
nodeaction.wait_for_unknown_status(node, timeout, self.kubecli, affected_node)
nodeaction.wait_for_ready_status(node, timeout, self.kubecli, affected_node)
logging.info("Node with bmc address: %s has been rebooted" % (bmc_addr))
logging.info("node_reboot_scenario has been successfuly injected!")
except Exception as e:
@@ -208,3 +211,4 @@ class bm_node_scenarios(abstract_node_scenarios):
traceback.print_exc()
logging.error("node_reboot_scenario injection failed!")
raise e
self.affected_nodes_status.affected_nodes.append(affected_node)
krkn/scenario_plugins/node_actions/common_node_functions.py
+15 -9
View File
@@ -1,9 +1,13 @@
import datetime
import time
import random
import logging
import paramiko
from krkn_lib.models.k8s import AffectedNode
import krkn.invoke.command as runcommand
from krkn_lib.k8s import KrknKubernetes
from krkn_lib.models.k8s import AffectedNode, AffectedNodeStatus
from krkn_lib.models.k8s import AffectedNode
node_general = False
@@ -40,23 +44,25 @@ def get_node(label_selector, instance_kill_count, kubecli: KrknKubernetes):
nodes.remove(node_to_add)
return nodes_to_return
# krkn_lib
# Wait until the node status becomes Ready
def wait_for_ready_status(node, timeout, kubecli: KrknKubernetes):
kubecli.watch_node_status(node, "True", timeout)
def wait_for_ready_status(node, timeout, kubecli: KrknKubernetes, affected_node: AffectedNode = None):
affected_node = kubecli.watch_node_status(node, "True", timeout, affected_node)
return affected_node
# krkn_lib
# Wait until the node status becomes Not Ready
def wait_for_not_ready_status(node, timeout, kubecli: KrknKubernetes):
kubecli.watch_node_status(node, "False", timeout)
def wait_for_not_ready_status(node, timeout, kubecli: KrknKubernetes, affected_node: AffectedNode = None):
affected_node = kubecli.watch_node_status(node, "False", timeout, affected_node)
return affected_node
# krkn_lib
# Wait until the node status becomes Unknown
def wait_for_unknown_status(node, timeout, kubecli: KrknKubernetes):
kubecli.watch_node_status(node, "Unknown", timeout)
def wait_for_unknown_status(node, timeout, kubecli: KrknKubernetes, affected_node: AffectedNode = None):
affected_node = kubecli.watch_node_status(node, "Unknown", timeout, affected_node)
return affected_node
# Get the ip of the cluster node
krkn/scenario_plugins/node_actions/docker_node_scenarios.py
+13 -7
View File
@@ -5,7 +5,7 @@ from krkn.scenario_plugins.node_actions.abstract_node_scenarios import (
import logging
import docker
from krkn_lib.k8s import KrknKubernetes
from krkn_lib.models.k8s import AffectedNode, AffectedNodeStatus
class Docker:
def __init__(self):
@@ -38,13 +38,14 @@ class Docker:
class docker_node_scenarios(abstract_node_scenarios):
def __init__(self, kubecli: KrknKubernetes):
super().__init__(kubecli)
def __init__(self, kubecli: KrknKubernetes, affected_nodes_status: AffectedNodeStatus):
super().__init__(kubecli, affected_nodes_status)
self.docker = Docker()
# Node scenario to start the node
def node_start_scenario(self, instance_kill_count, node, timeout):
for _ in range(instance_kill_count):
affected_node = AffectedNode(node)
try:
logging.info("Starting node_start_scenario injection")
container_id = self.docker.get_container_id(node)
@@ -52,7 +53,7 @@ class docker_node_scenarios(abstract_node_scenarios):
"Starting the node %s with container ID: %s " % (node, container_id)
)
self.docker.start_instances(node)
nodeaction.wait_for_ready_status(node, timeout, self.kubecli)
nodeaction.wait_for_ready_status(node, timeout, self.kubecli, affected_node)
logging.info(
"Node with container ID: %s is in running state" % (container_id)
)
@@ -64,10 +65,12 @@ class docker_node_scenarios(abstract_node_scenarios):
)
logging.error("node_start_scenario injection failed!")
raise e
self.affected_nodes_status.affected_nodes.append(affected_node)
# Node scenario to stop the node
def node_stop_scenario(self, instance_kill_count, node, timeout):
for _ in range(instance_kill_count):
affected_node = AffectedNode(node)
try:
logging.info("Starting node_stop_scenario injection")
container_id = self.docker.get_container_id(node)
@@ -78,7 +81,7 @@ class docker_node_scenarios(abstract_node_scenarios):
logging.info(
"Node with container ID: %s is in stopped state" % (container_id)
)
nodeaction.wait_for_unknown_status(node, timeout, self.kubecli)
nodeaction.wait_for_unknown_status(node, timeout, self.kubecli, affected_node)
except Exception as e:
logging.error(
"Failed to stop node instance. Encountered following exception: %s. "
@@ -86,6 +89,7 @@ class docker_node_scenarios(abstract_node_scenarios):
)
logging.error("node_stop_scenario injection failed!")
raise e
self.affected_nodes_status.affected_nodes.append(affected_node)
# Node scenario to terminate the node
def node_termination_scenario(self, instance_kill_count, node, timeout):
@@ -113,6 +117,7 @@ class docker_node_scenarios(abstract_node_scenarios):
# Node scenario to reboot the node
def node_reboot_scenario(self, instance_kill_count, node, timeout):
for _ in range(instance_kill_count):
affected_node = AffectedNode(node)
try:
logging.info("Starting node_reboot_scenario injection")
container_id = self.docker.get_container_id(node)
@@ -121,8 +126,8 @@ class docker_node_scenarios(abstract_node_scenarios):
% (node, container_id)
)
self.docker.reboot_instances(node)
nodeaction.wait_for_unknown_status(node, timeout, self.kubecli)
nodeaction.wait_for_ready_status(node, timeout, self.kubecli)
nodeaction.wait_for_unknown_status(node, timeout, self.kubecli, affected_node)
nodeaction.wait_for_ready_status(node, timeout, self.kubecli, affected_node)
logging.info(
"Node with container ID: %s has been rebooted" % (container_id)
)
@@ -134,3 +139,4 @@ class docker_node_scenarios(abstract_node_scenarios):
)
logging.error("node_reboot_scenario injection failed!")
raise e
self.affected_nodes_status.affected_nodes.append(affected_node)
krkn/scenario_plugins/node_actions/gcp_node_scenarios.py
+44 -17
View File
@@ -7,7 +7,7 @@ from krkn.scenario_plugins.node_actions.abstract_node_scenarios import (
)
from google.cloud import compute_v1
from krkn_lib.k8s import KrknKubernetes
from krkn_lib.models.k8s import AffectedNode, AffectedNodeStatus
class GCP:
def __init__(self):
@@ -173,10 +173,10 @@ class GCP:
"Failed to get status of instance %s. Encountered following "
"exception: %s." % (instance_id, e)
)
raise RuntimeError()
if instance_status == expected_status:
logging.info('status matches, end' + str(expected_status) + str(instance_status))
return True
time.sleep(sleeper)
i += sleeper
@@ -191,28 +191,45 @@ class GCP:
return self.get_instance_status(instance_id, "SUSPENDED", timeout)
# Wait until the node instance is running
def wait_until_running(self, instance_id, timeout):
return self.get_instance_status(instance_id, "RUNNING", timeout)
def wait_until_running(self, instance_id, timeout, affected_node):
start_time = time.time()
instance_status = self.get_instance_status(instance_id, "RUNNING", timeout)
end_time = time.time()
if affected_node:
affected_node.set_affected_node_status("running", end_time - start_time)
return instance_status
# Wait until the node instance is stopped
def wait_until_stopped(self, instance_id, timeout):
def wait_until_stopped(self, instance_id, timeout, affected_node):
# In GCP, the next state after STOPPING is TERMINATED
return self.get_instance_status(instance_id, "TERMINATED", timeout)
start_time = time.time()
instance_status = self.get_instance_status(instance_id, "TERMINATED", timeout)
end_time = time.time()
if affected_node:
affected_node.set_affected_node_status("stopped", end_time - start_time)
return instance_status
# Wait until the node instance is terminated
def wait_until_terminated(self, instance_id, timeout):
return self.get_instance_status(instance_id, "TERMINATED", timeout)
def wait_until_terminated(self, instance_id, timeout, affected_node):
start_time = time.time()
instance_status = self.get_instance_status(instance_id, "TERMINATED", timeout)
end_time = time.time()
if affected_node:
affected_node.set_affected_node_status("terminated", end_time - start_time)
return instance_status
# krkn_lib
class gcp_node_scenarios(abstract_node_scenarios):
def __init__(self, kubecli: KrknKubernetes):
super().__init__(kubecli)
def __init__(self, kubecli: KrknKubernetes, affected_nodes_status: AffectedNodeStatus):
super().__init__(kubecli, affected_nodes_status)
self.gcp = GCP()
print("selfkeys" + str(vars(self)))
# Node scenario to start the node
def node_start_scenario(self, instance_kill_count, node, timeout):
for _ in range(instance_kill_count):
affected_node = AffectedNode(node)
try:
logging.info("Starting node_start_scenario injection")
instance = self.gcp.get_node_instance(node)
@@ -221,8 +238,8 @@ class gcp_node_scenarios(abstract_node_scenarios):
"Starting the node %s with instance ID: %s " % (node, instance_id)
)
self.gcp.start_instances(instance_id)
self.gcp.wait_until_running(instance_id, timeout)
nodeaction.wait_for_ready_status(node, timeout, self.kubecli)
self.gcp.wait_until_running(instance_id, timeout, affected_node)
nodeaction.wait_for_ready_status(node, timeout, self.kubecli, affected_node)
logging.info(
"Node with instance ID: %s is in running state" % instance_id
)
@@ -235,10 +252,13 @@ class gcp_node_scenarios(abstract_node_scenarios):
logging.error("node_start_scenario injection failed!")
raise RuntimeError()
logging.info("started affected node" + str(affected_node.to_json()))
self.affected_nodes_status.affected_nodes.append(affected_node)
# Node scenario to stop the node
def node_stop_scenario(self, instance_kill_count, node, timeout):
for _ in range(instance_kill_count):
affected_node = AffectedNode(node)
try:
logging.info("Starting node_stop_scenario injection")
instance = self.gcp.get_node_instance(node)
@@ -247,11 +267,11 @@ class gcp_node_scenarios(abstract_node_scenarios):
"Stopping the node %s with instance ID: %s " % (node, instance_id)
)
self.gcp.stop_instances(instance_id)
self.gcp.wait_until_stopped(instance_id, timeout)
self.gcp.wait_until_stopped(instance_id, timeout, affected_node=affected_node)
logging.info(
"Node with instance ID: %s is in stopped state" % instance_id
)
nodeaction.wait_for_unknown_status(node, timeout, self.kubecli)
nodeaction.wait_for_unknown_status(node, timeout, self.kubecli, affected_node)
except Exception as e:
logging.error(
"Failed to stop node instance. Encountered following exception: %s. "
@@ -260,10 +280,13 @@ class gcp_node_scenarios(abstract_node_scenarios):
logging.error("node_stop_scenario injection failed!")
raise RuntimeError()
logging.info("stopedd affected node" + str(affected_node.to_json()))
self.affected_nodes_status.affected_nodes.append(affected_node)
# Node scenario to terminate the node
def node_termination_scenario(self, instance_kill_count, node, timeout):
for _ in range(instance_kill_count):
affected_node = AffectedNode(node)
try:
logging.info("Starting node_termination_scenario injection")
instance = self.gcp.get_node_instance(node)
@@ -273,7 +296,7 @@ class gcp_node_scenarios(abstract_node_scenarios):
% (node, instance_id)
)
self.gcp.terminate_instances(instance_id)
self.gcp.wait_until_terminated(instance_id, timeout)
self.gcp.wait_until_terminated(instance_id, timeout, affected_node=affected_node)
for _ in range(timeout):
if node not in self.kubecli.list_nodes():
break
@@ -292,10 +315,12 @@ class gcp_node_scenarios(abstract_node_scenarios):
logging.error("node_termination_scenario injection failed!")
raise RuntimeError()
self.affected_nodes_status.affected_nodes.append(affected_node)
# Node scenario to reboot the node
def node_reboot_scenario(self, instance_kill_count, node, timeout):
for _ in range(instance_kill_count):
affected_node = AffectedNode(node)
try:
logging.info("Starting node_reboot_scenario injection")
instance = self.gcp.get_node_instance(node)
@@ -304,8 +329,9 @@ class gcp_node_scenarios(abstract_node_scenarios):
"Rebooting the node %s with instance ID: %s " % (node, instance_id)
)
self.gcp.reboot_instances(instance_id)
self.gcp.wait_until_running(instance_id, timeout)
nodeaction.wait_for_ready_status(node, timeout, self.kubecli)
nodeaction.wait_for_unknown_status(node, timeout, self.kubecli, affected_node)
self.gcp.wait_until_running(instance_id, timeout, affected_node)
nodeaction.wait_for_ready_status(node, timeout, self.kubecli, affected_node)
logging.info(
"Node with instance ID: %s has been rebooted" % instance_id
)
@@ -318,3 +344,4 @@ class gcp_node_scenarios(abstract_node_scenarios):
logging.error("node_reboot_scenario injection failed!")
raise RuntimeError()
self.affected_nodes_status.affected_nodes.append(affected_node)
krkn/scenario_plugins/node_actions/general_cloud_node_scenarios.py
+3 -3
View File
@@ -3,7 +3,7 @@ from krkn.scenario_plugins.node_actions.abstract_node_scenarios import (
abstract_node_scenarios,
)
from krkn_lib.k8s import KrknKubernetes
from krkn_lib.models.k8s import AffectedNodeStatus
class GENERAL:
def __init__(self):
@@ -12,8 +12,8 @@ class GENERAL:
# krkn_lib
class general_node_scenarios(abstract_node_scenarios):
def __init__(self, kubecli: KrknKubernetes):
super().__init__(kubecli)
def __init__(self, kubecli: KrknKubernetes, affected_nodes_status: AffectedNodeStatus):
super().__init__(kubecli, affected_nodes_status)
self.general = GENERAL()
# Node scenario to start the node
krkn/scenario_plugins/node_actions/node_actions_scenario_plugin.py
+17 -12
View File
@@ -6,6 +6,7 @@ from itertools import repeat
import yaml
from krkn_lib.k8s import KrknKubernetes
from krkn_lib.models.telemetry import ScenarioTelemetry
from krkn_lib.models.k8s import AffectedNodeStatus
from krkn_lib.telemetry.ocp import KrknTelemetryOpenshift
from krkn_lib.utils import get_yaml_item_value, log_exception
@@ -49,6 +50,7 @@ class NodeActionsScenarioPlugin(AbstractScenarioPlugin):
node_scenario,
node_scenario_object,
lib_telemetry.get_lib_kubernetes(),
scenario_telemetry,
)
end_time = int(time.time())
cerberus.get_status(krkn_config, start_time, end_time)
@@ -59,28 +61,29 @@ class NodeActionsScenarioPlugin(AbstractScenarioPlugin):
return 0
def get_node_scenario_object(self, node_scenario, kubecli: KrknKubernetes):
affected_nodes_status = AffectedNodeStatus()
if (
"cloud_type" not in node_scenario.keys()
or node_scenario["cloud_type"] == "generic"
):
global node_general
node_general = True
return general_node_scenarios(kubecli)
return general_node_scenarios(kubecli, affected_nodes_status)
if node_scenario["cloud_type"].lower() == "aws":
return aws_node_scenarios(kubecli)
return aws_node_scenarios(kubecli, affected_nodes_status)
elif node_scenario["cloud_type"].lower() == "gcp":
return gcp_node_scenarios(kubecli)
return gcp_node_scenarios(kubecli, affected_nodes_status)
elif node_scenario["cloud_type"].lower() == "openstack":
from krkn.scenario_plugins.node_actions.openstack_node_scenarios import (
openstack_node_scenarios,
)
return openstack_node_scenarios(kubecli)
return openstack_node_scenarios(kubecli, affected_nodes_status)
elif (
node_scenario["cloud_type"].lower() == "azure"
or node_scenario["cloud_type"] == "az"
):
return azure_node_scenarios(kubecli)
return azure_node_scenarios(kubecli, affected_nodes_status)
elif (
node_scenario["cloud_type"].lower() == "alibaba"
or node_scenario["cloud_type"] == "alicloud"
@@ -89,7 +92,7 @@ class NodeActionsScenarioPlugin(AbstractScenarioPlugin):
alibaba_node_scenarios,
)
return alibaba_node_scenarios(kubecli)
return alibaba_node_scenarios(kubecli, affected_nodes_status)
elif node_scenario["cloud_type"].lower() == "bm":
from krkn.scenario_plugins.node_actions.bm_node_scenarios import (
bm_node_scenarios,
@@ -100,9 +103,10 @@ class NodeActionsScenarioPlugin(AbstractScenarioPlugin):
node_scenario.get("bmc_user", None),
node_scenario.get("bmc_password", None),
kubecli,
affected_nodes_status
)
elif node_scenario["cloud_type"].lower() == "docker":
return docker_node_scenarios(kubecli)
return docker_node_scenarios(kubecli, affected_nodes_status)
else:
logging.error(
"Cloud type "
@@ -120,7 +124,7 @@ class NodeActionsScenarioPlugin(AbstractScenarioPlugin):
)
def inject_node_scenario(
self, action, node_scenario, node_scenario_object, kubecli: KrknKubernetes
self, action, node_scenario, node_scenario_object, kubecli: KrknKubernetes, scenario_telemetry: ScenarioTelemetry
):
# Get the node scenario configurations for setting nodes
@@ -138,17 +142,18 @@ class NodeActionsScenarioPlugin(AbstractScenarioPlugin):
nodes = common_node_functions.get_node(
label_selector, instance_kill_count, kubecli
)
# GCP api doesn't support multiprocessing calls, will only actually run 1
if parallel_nodes and node_scenario['cloud_type'].lower() != "gcp":
if parallel_nodes:
self.multiprocess_nodes(nodes, node_scenario_object, action, node_scenario)
else:
for single_node in nodes:
self.run_node(single_node, node_scenario_object, action, node_scenario)
affected_nodes_status = node_scenario_object.affected_nodes_status
scenario_telemetry.affected_nodes.extend(affected_nodes_status.affected_nodes)
def multiprocess_nodes(self, nodes, node_scenario_object, action, node_scenario):
try:
logging.info("parallely call to nodes")
# pool object with number of element
pool = ThreadPool(processes=len(nodes))
@@ -160,7 +165,6 @@ class NodeActionsScenarioPlugin(AbstractScenarioPlugin):
def run_node(self, single_node, node_scenario_object, action, node_scenario):
logging.info("action" + str(action))
# Get the scenario specifics for running action nodes
run_kill_count = get_yaml_item_value(node_scenario, "runs", 1)
if action in ("node_stop_start_scenario", "node_disk_detach_attach_scenario"):
@@ -248,5 +252,6 @@ class NodeActionsScenarioPlugin(AbstractScenarioPlugin):
% action
)
def get_scenario_types(self) -> list[str]:
return ["node_scenarios"]
krkn/scenario_plugins/node_actions/openstack_node_scenarios.py
+36 -15
View File
@@ -7,7 +7,7 @@ from krkn.scenario_plugins.node_actions.abstract_node_scenarios import (
abstract_node_scenarios,
)
from krkn_lib.k8s import KrknKubernetes
from krkn_lib.models.k8s import AffectedNode, AffectedNodeStatus
class OPENSTACKCLOUD:
def __init__(self):
@@ -56,12 +56,22 @@ class OPENSTACKCLOUD:
raise RuntimeError()
# Wait until the node instance is running
def wait_until_running(self, node, timeout):
return self.get_instance_status(node, "ACTIVE", timeout)
def wait_until_running(self, node, timeout, affected_node):
start_time = time.time()
instance_status= self.get_instance_status(node, "ACTIVE", timeout)
end_time = time.time()
if affected_node:
affected_node.set_affected_node_status("running", end_time - start_time)
return instance_status
# Wait until the node instance is stopped
def wait_until_stopped(self, node, timeout):
return self.get_instance_status(node, "SHUTOFF", timeout)
def wait_until_stopped(self, node, timeout, affected_node):
start_time = time.time()
instance_status = self.get_instance_status(node, "SHUTOFF", timeout)
end_time = time.time()
if affected_node:
affected_node.set_affected_node_status("stopped", end_time - start_time)
return instance_status
# Get instance status
def get_instance_status(self, node, expected_status, timeout):
@@ -107,19 +117,21 @@ class OPENSTACKCLOUD:
# krkn_lib
class openstack_node_scenarios(abstract_node_scenarios):
def __init__(self, kubecli: KrknKubernetes):
def __init__(self, kubecli: KrknKubernetes, affected_nodes_status: AffectedNodeStatus ):
super().__init__(kubecli, affected_nodes_status)
self.openstackcloud = OPENSTACKCLOUD()
# Node scenario to start the node
def node_start_scenario(self, instance_kill_count, node, timeout):
for _ in range(instance_kill_count):
affected_node = AffectedNode(node)
try:
logging.info("Starting node_start_scenario injection")
logging.info("Starting the node %s" % (node))
openstack_node_name = self.openstackcloud.get_instance_id(node)
self.openstackcloud.start_instances(openstack_node_name)
self.openstackcloud.wait_until_running(openstack_node_name, timeout)
nodeaction.wait_for_ready_status(node, timeout, self.kubecli)
self.openstackcloud.wait_until_running(openstack_node_name, timeout, affected_node)
nodeaction.wait_for_ready_status(node, timeout, self.kubecli, affected_node)
logging.info("Node with instance ID: %s is in running state" % (node))
logging.info("node_start_scenario has been successfully injected!")
except Exception as e:
@@ -130,18 +142,20 @@ class openstack_node_scenarios(abstract_node_scenarios):
logging.error("node_start_scenario injection failed!")
raise RuntimeError()
self.affected_nodes_status.affected_nodes.append(affected_node)
# Node scenario to stop the node
def node_stop_scenario(self, instance_kill_count, node, timeout):
for _ in range(instance_kill_count):
affected_node = AffectedNode(node)
try:
logging.info("Starting node_stop_scenario injection")
logging.info("Stopping the node %s " % (node))
openstack_node_name = self.openstackcloud.get_instance_id(node)
self.openstackcloud.stop_instances(openstack_node_name)
self.openstackcloud.wait_until_stopped(openstack_node_name, timeout)
self.openstackcloud.wait_until_stopped(openstack_node_name, timeout, affected_node)
logging.info("Node with instance name: %s is in stopped state" % (node))
nodeaction.wait_for_ready_status(node, timeout, self.kubecli)
nodeaction.wait_for_not_ready_status(node, timeout, self.kubecli, affected_node)
except Exception as e:
logging.error(
"Failed to stop node instance. Encountered following exception: %s. "
@@ -150,17 +164,19 @@ class openstack_node_scenarios(abstract_node_scenarios):
logging.error("node_stop_scenario injection failed!")
raise RuntimeError()
self.affected_nodes_status.affected_nodes.append(affected_node)
# Node scenario to reboot the node
def node_reboot_scenario(self, instance_kill_count, node, timeout):
for _ in range(instance_kill_count):
affected_node = AffectedNode(node)
try:
logging.info("Starting node_reboot_scenario injection")
logging.info("Rebooting the node %s" % (node))
openstack_node_name = self.openstackcloud.get_instance_id(node)
self.openstackcloud.reboot_instances(openstack_node_name)
nodeaction.wait_for_unknown_status(node, timeout, self.kubecli)
nodeaction.wait_for_ready_status(node, timeout, self.kubecli)
nodeaction.wait_for_unknown_status(node, timeout, self.kubecli, affected_node)
nodeaction.wait_for_ready_status(node, timeout, self.kubecli, affected_node)
logging.info("Node with instance name: %s has been rebooted" % (node))
logging.info("node_reboot_scenario has been successfuly injected!")
except Exception as e:
@@ -171,10 +187,12 @@ class openstack_node_scenarios(abstract_node_scenarios):
logging.error("node_reboot_scenario injection failed!")
raise RuntimeError()
self.affected_nodes_status.affected_nodes.append(affected_node)
# Node scenario to start the node
def helper_node_start_scenario(self, instance_kill_count, node_ip, timeout):
for _ in range(instance_kill_count):
affected_node = AffectedNode(node_ip)
try:
logging.info("Starting helper_node_start_scenario injection")
openstack_node_name = self.openstackcloud.get_openstack_nodename(
@@ -182,7 +200,7 @@ class openstack_node_scenarios(abstract_node_scenarios):
)
logging.info("Starting the helper node %s" % (openstack_node_name))
self.openstackcloud.start_instances(openstack_node_name)
self.openstackcloud.wait_until_running(openstack_node_name, timeout)
self.openstackcloud.wait_until_running(openstack_node_name, timeout, affected_node)
logging.info("Helper node with IP: %s is in running state" % (node_ip))
logging.info("node_start_scenario has been successfully injected!")
except Exception as e:
@@ -193,10 +211,12 @@ class openstack_node_scenarios(abstract_node_scenarios):
logging.error("helper_node_start_scenario injection failed!")
raise RuntimeError()
self.affected_nodes_status.affected_nodes.append(affected_node)
# Node scenario to stop the node
def helper_node_stop_scenario(self, instance_kill_count, node_ip, timeout):
for _ in range(instance_kill_count):
affected_node = AffectedNode(node_ip)
try:
logging.info("Starting helper_node_stop_scenario injection")
openstack_node_name = self.openstackcloud.get_openstack_nodename(
@@ -204,7 +224,7 @@ class openstack_node_scenarios(abstract_node_scenarios):
)
logging.info("Stopping the helper node %s " % (openstack_node_name))
self.openstackcloud.stop_instances(openstack_node_name)
self.openstackcloud.wait_until_stopped(openstack_node_name, timeout)
self.openstackcloud.wait_until_stopped(openstack_node_name, timeout, affected_node)
logging.info("Helper node with IP: %s is in stopped state" % (node_ip))
except Exception as e:
logging.error(
@@ -214,6 +234,7 @@ class openstack_node_scenarios(abstract_node_scenarios):
logging.error("helper_node_stop_scenario injection failed!")
raise RuntimeError()
self.affected_nodes_status.affected_nodes.append(affected_node)
def helper_node_service_status(self, node_ip, service, ssh_private_key, timeout):
try:
krkn/scenario_plugins/shut_down/shut_down_scenario_plugin.py
+19 -6
View File
@@ -15,6 +15,7 @@ from krkn.scenario_plugins.node_actions.gcp_node_scenarios import GCP
from krkn.scenario_plugins.node_actions.openstack_node_scenarios import OPENSTACKCLOUD
from krkn.scenario_plugins.native.node_scenarios.ibmcloud_plugin import IbmCloud
from krkn_lib.models.k8s import AffectedNodeStatus, AffectedNode
class ShutDownScenarioPlugin(AbstractScenarioPlugin):
def run(
@@ -32,9 +33,12 @@ class ShutDownScenarioPlugin(AbstractScenarioPlugin):
"cluster_shut_down_scenario"
]
start_time = int(time.time())
affected_nodes_status = AffectedNodeStatus()
self.cluster_shut_down(
shut_down_config_scenario, lib_telemetry.get_lib_kubernetes()
shut_down_config_scenario, lib_telemetry.get_lib_kubernetes(), affected_nodes_status
)
scenario_telemetry.affected_nodes = affected_nodes_status
end_time = int(time.time())
cerberus.publish_kraken_status(krkn_config, [], start_time, end_time)
return 0
@@ -72,7 +76,7 @@ class ShutDownScenarioPlugin(AbstractScenarioPlugin):
# Inject the cluster shut down scenario
# krkn_lib
def cluster_shut_down(self, shut_down_config, kubecli: KrknKubernetes):
def cluster_shut_down(self, shut_down_config, kubecli: KrknKubernetes, affected_nodes_status: AffectedNodeStatus):
runs = shut_down_config["runs"]
shut_down_duration = shut_down_config["shut_down_duration"]
cloud_type = shut_down_config["cloud_type"]
@@ -101,6 +105,7 @@ class ShutDownScenarioPlugin(AbstractScenarioPlugin):
node_id = []
for node in nodes:
instance_id = cloud_object.get_instance_id(node)
affected_nodes_status.affected_nodes.append(AffectedNode(node))
node_id.append(instance_id)
logging.info("node id list " + str(node_id))
for _ in range(runs):
@@ -108,14 +113,18 @@ class ShutDownScenarioPlugin(AbstractScenarioPlugin):
stopping_nodes = set(node_id)
self.multiprocess_nodes(cloud_object.stop_instances, node_id, processes)
stopped_nodes = stopping_nodes.copy()
start_time = time.time()
while len(stopping_nodes) > 0:
for node in stopping_nodes:
affected_node = affected_nodes_status.get_affected_node_index(node)
# need to add in time that is passing while waiting for other nodes to be stopped
affected_node.set_cloud_stopping_time(time.time() - start_time)
if type(node) is tuple:
node_status = cloud_object.wait_until_stopped(
node[1], node[0], timeout
node[1], node[0], timeout, affected_node
)
else:
node_status = cloud_object.wait_until_stopped(node, timeout)
node_status = cloud_object.wait_until_stopped(node, timeout, affected_node)
# Only want to remove node from stopping list
# when fully stopped/no error
@@ -132,16 +141,20 @@ class ShutDownScenarioPlugin(AbstractScenarioPlugin):
logging.info("Restarting the nodes")
restarted_nodes = set(node_id)
self.multiprocess_nodes(cloud_object.start_instances, node_id, processes)
start_time = time.time()
logging.info("Wait for each node to be running again")
not_running_nodes = restarted_nodes.copy()
while len(not_running_nodes) > 0:
for node in not_running_nodes:
affected_node = affected_nodes_status.get_affected_node_index(node)
# need to add in time that is passing while waiting for other nodes to be running
affected_node.set_cloud_running_time(time.time() - start_time)
if type(node) is tuple:
node_status = cloud_object.wait_until_running(
node[1], node[0], timeout
node[1], node[0], timeout, affected_node
)
else:
node_status = cloud_object.wait_until_running(node, timeout)
node_status = cloud_object.wait_until_running(node, timeout, affected_node)
if node_status:
restarted_nodes.remove(node)
not_running_nodes = restarted_nodes.copy()
requirements.txt
+1 -1
View File
@@ -15,7 +15,7 @@ google-cloud-compute==1.22.0
ibm_cloud_sdk_core==3.18.0
ibm_vpc==0.20.0
jinja2==3.1.5
krkn-lib@git+https://github.com/krkn-chaos/krkn-lib.git@arca_to_native
krkn-lib==4.0.7
lxml==5.1.0
kubernetes==28.1.0
numpy==1.26.4