From 149eb8fcd3607ed462d4fc49acd0442ab07aa8e4 Mon Sep 17 00:00:00 2001 From: Paige Patton Date: Thu, 5 Jun 2025 16:08:21 -0400 Subject: [PATCH] adding kube_check as option into node scenarios Signed-off-by: Paige Patton --- .../node_actions/abstract_node_scenarios.py | 4 ++- .../node_actions/alibaba_node_scenarios.py | 16 +++++++----- .../node_actions/aws_node_scenarios.py | 16 +++++++----- .../node_actions/az_node_scenarios.py | 15 ++++++----- .../node_actions/bm_node_scenarios.py | 16 +++++++----- .../node_actions/docker_node_scenarios.py | 16 +++++++----- .../node_actions/gcp_node_scenarios.py | 17 ++++++++----- .../general_cloud_node_scenarios.py | 5 ++-- .../node_actions/ibmcloud_node_scenarios.py | 25 +++++++++++-------- .../node_actions_scenario_plugin.py | 22 +++++++++------- .../node_actions/openstack_node_scenarios.py | 16 +++++++----- .../node_actions/vmware_node_scenarios.py | 23 +++++++++-------- scenarios/openshift/aws_node_scenarios.yml | 1 + scenarios/openshift/azure_node_scenarios.yml | 2 ++ scenarios/openshift/gcp_node_scenarios.yml | 2 ++ 15 files changed, 121 insertions(+), 75 deletions(-) diff --git a/krkn/scenario_plugins/node_actions/abstract_node_scenarios.py b/krkn/scenario_plugins/node_actions/abstract_node_scenarios.py index 2c8b1a4d..0ef71cb0 100644 --- a/krkn/scenario_plugins/node_actions/abstract_node_scenarios.py +++ b/krkn/scenario_plugins/node_actions/abstract_node_scenarios.py @@ -10,10 +10,12 @@ from krkn_lib.models.k8s import AffectedNode, AffectedNodeStatus class abstract_node_scenarios: kubecli: KrknKubernetes affected_nodes_status: AffectedNodeStatus + node_action_kube_check: bool - def __init__(self, kubecli: KrknKubernetes, affected_nodes_status: AffectedNodeStatus): + def __init__(self, kubecli: KrknKubernetes, node_action_kube_check: bool, affected_nodes_status: AffectedNodeStatus): self.kubecli = kubecli self.affected_nodes_status = affected_nodes_status + self.node_action_kube_check = node_action_kube_check # Node scenario to start the node def node_start_scenario(self, instance_kill_count, node, timeout): diff --git a/krkn/scenario_plugins/node_actions/alibaba_node_scenarios.py b/krkn/scenario_plugins/node_actions/alibaba_node_scenarios.py index 4d401316..569c1bb4 100644 --- a/krkn/scenario_plugins/node_actions/alibaba_node_scenarios.py +++ b/krkn/scenario_plugins/node_actions/alibaba_node_scenarios.py @@ -227,9 +227,10 @@ class Alibaba: # krkn_lib class alibaba_node_scenarios(abstract_node_scenarios): - def __init__(self, kubecli: KrknKubernetes, affected_nodes_status: AffectedNodeStatus): - super().__init__(kubecli, affected_nodes_status) + def __init__(self, kubecli: KrknKubernetes, node_action_kube_check: bool, affected_nodes_status: AffectedNodeStatus): + super().__init__(kubecli, node_action_kube_check, affected_nodes_status) self.alibaba = Alibaba() + self.node_action_kube_check = node_action_kube_check # Node scenario to start the node @@ -245,7 +246,8 @@ class alibaba_node_scenarios(abstract_node_scenarios): ) self.alibaba.start_instances(vm_id) self.alibaba.wait_until_running(vm_id, timeout, affected_node) - nodeaction.wait_for_ready_status(node, timeout, self.kubecli, affected_node) + if self.node_action_kube_check: + nodeaction.wait_for_ready_status(node, timeout, self.kubecli, affected_node) logging.info("Node with instance ID: %s is in running state" % node) logging.info("node_start_scenario has been successfully injected!") except Exception as e: @@ -271,7 +273,8 @@ class alibaba_node_scenarios(abstract_node_scenarios): self.alibaba.stop_instances(vm_id) self.alibaba.wait_until_stopped(vm_id, timeout, affected_node) logging.info("Node with instance ID: %s is in stopped state" % vm_id) - nodeaction.wait_for_unknown_status(node, timeout, self.kubecli, affected_node) + if self.node_action_kube_check: + nodeaction.wait_for_unknown_status(node, timeout, self.kubecli, affected_node) except Exception as e: logging.error( "Failed to stop node instance. Encountered following exception: %s. " @@ -322,8 +325,9 @@ class alibaba_node_scenarios(abstract_node_scenarios): affected_node.node_id = instance_id logging.info("Rebooting the node with instance ID: %s " % (instance_id)) self.alibaba.reboot_instances(instance_id) - nodeaction.wait_for_unknown_status(node, timeout, self.kubecli, affected_node) - nodeaction.wait_for_ready_status(node, timeout, self.kubecli, affected_node) + if self.node_action_kube_check: + nodeaction.wait_for_unknown_status(node, timeout, self.kubecli, affected_node) + nodeaction.wait_for_ready_status(node, timeout, self.kubecli, affected_node) logging.info( "Node with instance ID: %s has been rebooted" % (instance_id) ) diff --git a/krkn/scenario_plugins/node_actions/aws_node_scenarios.py b/krkn/scenario_plugins/node_actions/aws_node_scenarios.py index 1ac7c55e..e41dcaf2 100644 --- a/krkn/scenario_plugins/node_actions/aws_node_scenarios.py +++ b/krkn/scenario_plugins/node_actions/aws_node_scenarios.py @@ -261,9 +261,10 @@ class AWS: # krkn_lib class aws_node_scenarios(abstract_node_scenarios): - def __init__(self, kubecli: KrknKubernetes, affected_nodes_status: AffectedNodeStatus): - super().__init__(kubecli, affected_nodes_status) + def __init__(self, kubecli: KrknKubernetes, node_action_kube_check: bool, affected_nodes_status: AffectedNodeStatus): + super().__init__(kubecli, node_action_kube_check, affected_nodes_status) self.aws = AWS() + self.node_action_kube_check = node_action_kube_check # Node scenario to start the node def node_start_scenario(self, instance_kill_count, node, timeout): @@ -278,7 +279,8 @@ class aws_node_scenarios(abstract_node_scenarios): ) self.aws.start_instances(instance_id) self.aws.wait_until_running(instance_id, affected_node=affected_node) - nodeaction.wait_for_ready_status(node, timeout, self.kubecli, affected_node) + if self.node_action_kube_check: + nodeaction.wait_for_ready_status(node, timeout, self.kubecli, affected_node) logging.info( "Node with instance ID: %s is in running state" % (instance_id) ) @@ -309,7 +311,8 @@ class aws_node_scenarios(abstract_node_scenarios): logging.info( "Node with instance ID: %s is in stopped state" % (instance_id) ) - nodeaction.wait_for_unknown_status(node, timeout, self.kubecli, affected_node=affected_node) + if self.node_action_kube_check: + nodeaction.wait_for_unknown_status(node, timeout, self.kubecli, affected_node=affected_node) except Exception as e: logging.error( "Failed to stop node instance. Encountered following exception: %s. " @@ -366,8 +369,9 @@ class aws_node_scenarios(abstract_node_scenarios): "Rebooting the node %s with instance ID: %s " % (node, instance_id) ) self.aws.reboot_instances(instance_id) - nodeaction.wait_for_unknown_status(node, timeout, self.kubecli, affected_node) - nodeaction.wait_for_ready_status(node, timeout, self.kubecli, affected_node) + if self.node_action_kube_check: + nodeaction.wait_for_unknown_status(node, timeout, self.kubecli, affected_node) + nodeaction.wait_for_ready_status(node, timeout, self.kubecli, affected_node) logging.info( "Node with instance ID: %s has been rebooted" % (instance_id) ) diff --git a/krkn/scenario_plugins/node_actions/az_node_scenarios.py b/krkn/scenario_plugins/node_actions/az_node_scenarios.py index 92f7b846..0928341a 100644 --- a/krkn/scenario_plugins/node_actions/az_node_scenarios.py +++ b/krkn/scenario_plugins/node_actions/az_node_scenarios.py @@ -210,10 +210,11 @@ class Azure: # krkn_lib class azure_node_scenarios(abstract_node_scenarios): - def __init__(self, kubecli: KrknKubernetes, affected_nodes_status: AffectedNodeStatus): - super().__init__(kubecli, affected_nodes_status) + def __init__(self, kubecli: KrknKubernetes, node_action_kube_check: bool, affected_nodes_status: AffectedNodeStatus): + super().__init__(kubecli, node_action_kube_check, affected_nodes_status) logging.info("init in azure") self.azure = Azure() + self.node_action_kube_check = node_action_kube_check # Node scenario to start the node @@ -230,7 +231,8 @@ class azure_node_scenarios(abstract_node_scenarios): ) self.azure.start_instances(resource_group, vm_name) self.azure.wait_until_running(resource_group, vm_name, timeout, affected_node=affected_node) - nodeaction.wait_for_ready_status(vm_name, timeout, self.kubecli, affected_node) + if self.node_action_kube_check: + nodeaction.wait_for_ready_status(vm_name, timeout, self.kubecli, affected_node) logging.info("Node with instance ID: %s is in running state" % node) logging.info("node_start_scenario has been successfully injected!") except Exception as e: @@ -258,7 +260,8 @@ class azure_node_scenarios(abstract_node_scenarios): self.azure.stop_instances(resource_group, vm_name) self.azure.wait_until_stopped(resource_group, vm_name, timeout, affected_node=affected_node) logging.info("Node with instance ID: %s is in stopped state" % vm_name) - nodeaction.wait_for_unknown_status(vm_name, timeout, self.kubecli, affected_node) + if self.node_action_kube_check: + nodeaction.wait_for_unknown_status(vm_name, timeout, self.kubecli, affected_node) except Exception as e: logging.error( "Failed to stop node instance. Encountered following exception: %s. " @@ -318,8 +321,8 @@ class azure_node_scenarios(abstract_node_scenarios): ) self.azure.reboot_instances(resource_group, vm_name) - - nodeaction.wait_for_ready_status(vm_name, timeout, self.kubecli, affected_node) + if self.node_action_kube_check: + nodeaction.wait_for_ready_status(vm_name, timeout, self.kubecli, affected_node) logging.info("Node with instance ID: %s has been rebooted" % (vm_name)) logging.info("node_reboot_scenario has been successfully injected!") diff --git a/krkn/scenario_plugins/node_actions/bm_node_scenarios.py b/krkn/scenario_plugins/node_actions/bm_node_scenarios.py index fa8ec24b..b868cf6f 100644 --- a/krkn/scenario_plugins/node_actions/bm_node_scenarios.py +++ b/krkn/scenario_plugins/node_actions/bm_node_scenarios.py @@ -135,9 +135,10 @@ class BM: # krkn_lib class bm_node_scenarios(abstract_node_scenarios): - def __init__(self, bm_info, user, passwd, kubecli: KrknKubernetes,affected_nodes_status: AffectedNodeStatus): - super().__init__(kubecli, affected_nodes_status) + def __init__(self, bm_info, user, passwd, kubecli: KrknKubernetes,node_action_kube_check: bool, affected_nodes_status: AffectedNodeStatus): + super().__init__(kubecli, node_action_kube_check, affected_nodes_status) self.bm = BM(bm_info, user, passwd) + self.node_action_kube_check = node_action_kube_check # Node scenario to start the node def node_start_scenario(self, instance_kill_count, node, timeout): @@ -152,7 +153,8 @@ class bm_node_scenarios(abstract_node_scenarios): ) self.bm.start_instances(bmc_addr, node) self.bm.wait_until_running(bmc_addr, node, affected_node) - nodeaction.wait_for_ready_status(node, timeout, self.kubecli, affected_node) + if self.node_action_kube_check: + nodeaction.wait_for_ready_status(node, timeout, self.kubecli, affected_node) logging.info( "Node with bmc address: %s is in running state" % (bmc_addr) ) @@ -183,7 +185,8 @@ class bm_node_scenarios(abstract_node_scenarios): logging.info( "Node with bmc address: %s is in stopped state" % (bmc_addr) ) - nodeaction.wait_for_unknown_status(node, timeout, self.kubecli, affected_node) + if self.node_action_kube_check: + nodeaction.wait_for_unknown_status(node, timeout, self.kubecli, affected_node) except Exception as e: logging.error( "Failed to stop node instance. Encountered following exception: %s. " @@ -210,8 +213,9 @@ class bm_node_scenarios(abstract_node_scenarios): "Rebooting the node %s with bmc address: %s " % (node, bmc_addr) ) self.bm.reboot_instances(bmc_addr, node) - nodeaction.wait_for_unknown_status(node, timeout, self.kubecli, affected_node) - nodeaction.wait_for_ready_status(node, timeout, self.kubecli, affected_node) + if self.node_action_kube_check: + nodeaction.wait_for_unknown_status(node, timeout, self.kubecli, affected_node) + nodeaction.wait_for_ready_status(node, timeout, self.kubecli, affected_node) logging.info("Node with bmc address: %s has been rebooted" % (bmc_addr)) logging.info("node_reboot_scenario has been successfuly injected!") except Exception as e: diff --git a/krkn/scenario_plugins/node_actions/docker_node_scenarios.py b/krkn/scenario_plugins/node_actions/docker_node_scenarios.py index 1730656d..dbc5349e 100644 --- a/krkn/scenario_plugins/node_actions/docker_node_scenarios.py +++ b/krkn/scenario_plugins/node_actions/docker_node_scenarios.py @@ -38,9 +38,10 @@ class Docker: class docker_node_scenarios(abstract_node_scenarios): - def __init__(self, kubecli: KrknKubernetes, affected_nodes_status: AffectedNodeStatus): - super().__init__(kubecli, affected_nodes_status) + def __init__(self, kubecli: KrknKubernetes, node_action_kube_check: bool, affected_nodes_status: AffectedNodeStatus): + super().__init__(kubecli, node_action_kube_check, affected_nodes_status) self.docker = Docker() + self.node_action_kube_check = node_action_kube_check # Node scenario to start the node def node_start_scenario(self, instance_kill_count, node, timeout): @@ -54,7 +55,8 @@ class docker_node_scenarios(abstract_node_scenarios): "Starting the node %s with container ID: %s " % (node, container_id) ) self.docker.start_instances(node) - nodeaction.wait_for_ready_status(node, timeout, self.kubecli, affected_node) + if self.node_action_kube_check: + nodeaction.wait_for_ready_status(node, timeout, self.kubecli, affected_node) logging.info( "Node with container ID: %s is in running state" % (container_id) ) @@ -83,7 +85,8 @@ class docker_node_scenarios(abstract_node_scenarios): logging.info( "Node with container ID: %s is in stopped state" % (container_id) ) - nodeaction.wait_for_unknown_status(node, timeout, self.kubecli, affected_node) + if self.node_action_kube_check: + nodeaction.wait_for_unknown_status(node, timeout, self.kubecli, affected_node) except Exception as e: logging.error( "Failed to stop node instance. Encountered following exception: %s. " @@ -128,8 +131,9 @@ class docker_node_scenarios(abstract_node_scenarios): % (node, container_id) ) self.docker.reboot_instances(node) - nodeaction.wait_for_unknown_status(node, timeout, self.kubecli, affected_node) - nodeaction.wait_for_ready_status(node, timeout, self.kubecli, affected_node) + if self.node_action_kube_check: + nodeaction.wait_for_unknown_status(node, timeout, self.kubecli, affected_node) + nodeaction.wait_for_ready_status(node, timeout, self.kubecli, affected_node) logging.info( "Node with container ID: %s has been rebooted" % (container_id) ) diff --git a/krkn/scenario_plugins/node_actions/gcp_node_scenarios.py b/krkn/scenario_plugins/node_actions/gcp_node_scenarios.py index ca8ee5b8..d9867896 100644 --- a/krkn/scenario_plugins/node_actions/gcp_node_scenarios.py +++ b/krkn/scenario_plugins/node_actions/gcp_node_scenarios.py @@ -221,9 +221,10 @@ class GCP: # krkn_lib class gcp_node_scenarios(abstract_node_scenarios): - def __init__(self, kubecli: KrknKubernetes, affected_nodes_status: AffectedNodeStatus): - super().__init__(kubecli, affected_nodes_status) + def __init__(self, kubecli: KrknKubernetes, node_action_kube_check: bool, affected_nodes_status: AffectedNodeStatus): + super().__init__(kubecli, node_action_kube_check, affected_nodes_status) self.gcp = GCP() + self.node_action_kube_check = node_action_kube_check # Node scenario to start the node def node_start_scenario(self, instance_kill_count, node, timeout): @@ -239,7 +240,8 @@ class gcp_node_scenarios(abstract_node_scenarios): ) self.gcp.start_instances(instance_id) self.gcp.wait_until_running(instance_id, timeout, affected_node) - nodeaction.wait_for_ready_status(node, timeout, self.kubecli, affected_node) + if self.node_action_kube_check: + nodeaction.wait_for_ready_status(node, timeout, self.kubecli, affected_node) logging.info( "Node with instance ID: %s is in running state" % instance_id ) @@ -271,7 +273,8 @@ class gcp_node_scenarios(abstract_node_scenarios): logging.info( "Node with instance ID: %s is in stopped state" % instance_id ) - nodeaction.wait_for_unknown_status(node, timeout, self.kubecli, affected_node) + if self.node_action_kube_check: + nodeaction.wait_for_unknown_status(node, timeout, self.kubecli, affected_node) except Exception as e: logging.error( "Failed to stop node instance. Encountered following exception: %s. " @@ -330,9 +333,11 @@ class gcp_node_scenarios(abstract_node_scenarios): "Rebooting the node %s with instance ID: %s " % (node, instance_id) ) self.gcp.reboot_instances(instance_id) - nodeaction.wait_for_unknown_status(node, timeout, self.kubecli, affected_node) + if self.node_action_kube_check: + nodeaction.wait_for_unknown_status(node, timeout, self.kubecli, affected_node) self.gcp.wait_until_running(instance_id, timeout, affected_node) - nodeaction.wait_for_ready_status(node, timeout, self.kubecli, affected_node) + if self.node_action_kube_check: + nodeaction.wait_for_ready_status(node, timeout, self.kubecli, affected_node) logging.info( "Node with instance ID: %s has been rebooted" % instance_id ) diff --git a/krkn/scenario_plugins/node_actions/general_cloud_node_scenarios.py b/krkn/scenario_plugins/node_actions/general_cloud_node_scenarios.py index 151de42d..ae16af56 100644 --- a/krkn/scenario_plugins/node_actions/general_cloud_node_scenarios.py +++ b/krkn/scenario_plugins/node_actions/general_cloud_node_scenarios.py @@ -12,9 +12,10 @@ class GENERAL: # krkn_lib class general_node_scenarios(abstract_node_scenarios): - def __init__(self, kubecli: KrknKubernetes, affected_nodes_status: AffectedNodeStatus): - super().__init__(kubecli, affected_nodes_status) + def __init__(self, kubecli: KrknKubernetes, node_action_kube_check: bool, affected_nodes_status: AffectedNodeStatus): + super().__init__(kubecli, node_action_kube_check, affected_nodes_status) self.general = GENERAL() + self.node_action_kube_check = node_action_kube_check # Node scenario to start the node def node_start_scenario(self, instance_kill_count, node, timeout): diff --git a/krkn/scenario_plugins/node_actions/ibmcloud_node_scenarios.py b/krkn/scenario_plugins/node_actions/ibmcloud_node_scenarios.py index 8c57034f..b1546360 100644 --- a/krkn/scenario_plugins/node_actions/ibmcloud_node_scenarios.py +++ b/krkn/scenario_plugins/node_actions/ibmcloud_node_scenarios.py @@ -260,9 +260,10 @@ class IbmCloud: @dataclass class ibm_node_scenarios(abstract_node_scenarios): - def __init__(self, kubecli: KrknKubernetes, affected_nodes_status: AffectedNodeStatus): - super().__init__(kubecli, affected_nodes_status) + def __init__(self, kubecli: KrknKubernetes, node_action_kube_check: bool, affected_nodes_status: AffectedNodeStatus): + super().__init__(kubecli, node_action_kube_check, affected_nodes_status) self.ibmcloud = IbmCloud() + self.node_action_kube_check = node_action_kube_check def node_start_scenario(self, instance_kill_count, node, timeout): try: @@ -276,9 +277,10 @@ class ibm_node_scenarios(abstract_node_scenarios): vm_started = self.ibmcloud.start_instances(instance_id) if vm_started: self.ibmcloud.wait_until_running(instance_id, timeout, affected_node) - nodeaction.wait_for_ready_status( - node, timeout, self.kubecli, affected_node - ) + if self.node_action_kube_check: + nodeaction.wait_for_ready_status( + node, timeout, self.kubecli, affected_node + ) logging.info( "Node with instance ID: %s is in running state" % node ) @@ -326,12 +328,13 @@ class ibm_node_scenarios(abstract_node_scenarios): logging.info("Rebooting the node %s " % (node)) self.ibmcloud.reboot_instances(instance_id) self.ibmcloud.wait_until_rebooted(instance_id, timeout) - nodeaction.wait_for_unknown_status( - node, timeout, affected_node - ) - nodeaction.wait_for_ready_status( - node, timeout, affected_node - ) + if self.node_action_kube_check: + nodeaction.wait_for_unknown_status( + node, timeout, affected_node + ) + nodeaction.wait_for_ready_status( + node, timeout, affected_node + ) logging.info( "Node with instance ID: %s has rebooted successfully" % node ) diff --git a/krkn/scenario_plugins/node_actions/node_actions_scenario_plugin.py b/krkn/scenario_plugins/node_actions/node_actions_scenario_plugin.py index f419d85c..d71190ea 100644 --- a/krkn/scenario_plugins/node_actions/node_actions_scenario_plugin.py +++ b/krkn/scenario_plugins/node_actions/node_actions_scenario_plugin.py @@ -62,28 +62,30 @@ class NodeActionsScenarioPlugin(AbstractScenarioPlugin): def get_node_scenario_object(self, node_scenario, kubecli: KrknKubernetes): affected_nodes_status = AffectedNodeStatus() + + node_action_kube_check = get_yaml_item_value(node_scenario,"kube_check",True) if ( "cloud_type" not in node_scenario.keys() or node_scenario["cloud_type"] == "generic" ): global node_general node_general = True - return general_node_scenarios(kubecli, affected_nodes_status) + return general_node_scenarios(kubecli, node_action_kube_check, affected_nodes_status) if node_scenario["cloud_type"].lower() == "aws": - return aws_node_scenarios(kubecli, affected_nodes_status) + return aws_node_scenarios(kubecli, node_action_kube_check, affected_nodes_status) elif node_scenario["cloud_type"].lower() == "gcp": - return gcp_node_scenarios(kubecli, affected_nodes_status) + return gcp_node_scenarios(kubecli, node_action_kube_check, affected_nodes_status) elif node_scenario["cloud_type"].lower() == "openstack": from krkn.scenario_plugins.node_actions.openstack_node_scenarios import ( openstack_node_scenarios, ) - return openstack_node_scenarios(kubecli, affected_nodes_status) + return openstack_node_scenarios(kubecli, node_action_kube_check, affected_nodes_status) elif ( node_scenario["cloud_type"].lower() == "azure" or node_scenario["cloud_type"].lower() == "az" ): - return azure_node_scenarios(kubecli, affected_nodes_status) + return azure_node_scenarios(kubecli, node_action_kube_check, affected_nodes_status) elif ( node_scenario["cloud_type"].lower() == "alibaba" or node_scenario["cloud_type"].lower() == "alicloud" @@ -92,7 +94,7 @@ class NodeActionsScenarioPlugin(AbstractScenarioPlugin): alibaba_node_scenarios, ) - return alibaba_node_scenarios(kubecli, affected_nodes_status) + return alibaba_node_scenarios(kubecli, node_action_kube_check, affected_nodes_status) elif node_scenario["cloud_type"].lower() == "bm": from krkn.scenario_plugins.node_actions.bm_node_scenarios import ( bm_node_scenarios, @@ -103,20 +105,22 @@ class NodeActionsScenarioPlugin(AbstractScenarioPlugin): node_scenario.get("bmc_user", None), node_scenario.get("bmc_password", None), kubecli, + node_action_kube_check, affected_nodes_status ) elif node_scenario["cloud_type"].lower() == "docker": - return docker_node_scenarios(kubecli) + return docker_node_scenarios(kubecli,node_action_kube_check, + affected_nodes_status) elif ( node_scenario["cloud_type"].lower() == "vsphere" or node_scenario["cloud_type"].lower() == "vmware" ): - return vmware_node_scenarios(kubecli, affected_nodes_status) + return vmware_node_scenarios(kubecli, node_action_kube_check,affected_nodes_status) elif ( node_scenario["cloud_type"].lower() == "ibm" or node_scenario["cloud_type"].lower() == "ibmcloud" ): - return ibm_node_scenarios(kubecli, affected_nodes_status) + return ibm_node_scenarios(kubecli, node_action_kube_check, affected_nodes_status) else: logging.error( "Cloud type " diff --git a/krkn/scenario_plugins/node_actions/openstack_node_scenarios.py b/krkn/scenario_plugins/node_actions/openstack_node_scenarios.py index ee8f8314..169ffe7a 100644 --- a/krkn/scenario_plugins/node_actions/openstack_node_scenarios.py +++ b/krkn/scenario_plugins/node_actions/openstack_node_scenarios.py @@ -116,9 +116,10 @@ class OPENSTACKCLOUD: # krkn_lib class openstack_node_scenarios(abstract_node_scenarios): - def __init__(self, kubecli: KrknKubernetes, affected_nodes_status: AffectedNodeStatus ): - super().__init__(kubecli, affected_nodes_status) + def __init__(self, kubecli: KrknKubernetes,node_action_kube_check: bool, affected_nodes_status: AffectedNodeStatus ): + super().__init__(kubecli, node_action_kube_check, affected_nodes_status) self.openstackcloud = OPENSTACKCLOUD() + self.node_action_kube_check = node_action_kube_check # Node scenario to start the node def node_start_scenario(self, instance_kill_count, node, timeout): @@ -131,7 +132,8 @@ class openstack_node_scenarios(abstract_node_scenarios): openstack_node_name = self.openstackcloud.get_instance_id(openstack_node_ip) self.openstackcloud.start_instances(openstack_node_name) self.openstackcloud.wait_until_running(openstack_node_name, timeout, affected_node) - nodeaction.wait_for_ready_status(node, timeout, self.kubecli, affected_node) + if self.node_action_kube_check: + nodeaction.wait_for_ready_status(node, timeout, self.kubecli, affected_node) logging.info("Node with instance ID: %s is in running state" % (node)) logging.info("node_start_scenario has been successfully injected!") except Exception as e: @@ -156,7 +158,8 @@ class openstack_node_scenarios(abstract_node_scenarios): self.openstackcloud.stop_instances(openstack_node_name) self.openstackcloud.wait_until_stopped(openstack_node_name, timeout, affected_node) logging.info("Node with instance name: %s is in stopped state" % (node)) - nodeaction.wait_for_not_ready_status(node, timeout, self.kubecli, affected_node) + if self.node_action_kube_check: + nodeaction.wait_for_not_ready_status(node, timeout, self.kubecli, affected_node) except Exception as e: logging.error( "Failed to stop node instance. Encountered following exception: %s. " @@ -177,8 +180,9 @@ class openstack_node_scenarios(abstract_node_scenarios): openstack_node_ip = self.kubecli.get_node_ip(node) openstack_node_name = self.openstackcloud.get_instance_id(openstack_node_ip) self.openstackcloud.reboot_instances(openstack_node_name) - nodeaction.wait_for_unknown_status(node, timeout, self.kubecli, affected_node) - nodeaction.wait_for_ready_status(node, timeout, self.kubecli, affected_node) + if self.node_action_kube_check: + nodeaction.wait_for_unknown_status(node, timeout, self.kubecli, affected_node) + nodeaction.wait_for_ready_status(node, timeout, self.kubecli, affected_node) logging.info("Node with instance name: %s has been rebooted" % (node)) logging.info("node_reboot_scenario has been successfuly injected!") except Exception as e: diff --git a/krkn/scenario_plugins/node_actions/vmware_node_scenarios.py b/krkn/scenario_plugins/node_actions/vmware_node_scenarios.py index ab9ed4ad..219c5063 100644 --- a/krkn/scenario_plugins/node_actions/vmware_node_scenarios.py +++ b/krkn/scenario_plugins/node_actions/vmware_node_scenarios.py @@ -384,9 +384,10 @@ class vSphere: @dataclass class vmware_node_scenarios(abstract_node_scenarios): - def __init__(self, kubecli: KrknKubernetes, affected_nodes_status: AffectedNodeStatus): - super().__init__(kubecli, affected_nodes_status) + def __init__(self, kubecli: KrknKubernetes, node_action_kube_check: bool, affected_nodes_status: AffectedNodeStatus): + super().__init__(kubecli, node_action_kube_check, affected_nodes_status) self.vsphere = vSphere() + self.node_action_kube_check = node_action_kube_check def node_start_scenario(self, instance_kill_count, node, timeout): try: @@ -397,7 +398,8 @@ class vmware_node_scenarios(abstract_node_scenarios): vm_started = self.vsphere.start_instances(node) if vm_started: self.vsphere.wait_until_running(node, timeout, affected_node) - nodeaction.wait_for_ready_status(node, timeout, self.kubecli, affected_node) + if self.node_action_kube_check: + nodeaction.wait_for_ready_status(node, timeout, self.kubecli, affected_node) logging.info(f"Node with instance ID: {node} is in running state") logging.info("node_start_scenario has been successfully injected!") self.affected_nodes_status.affected_nodes.append(affected_node) @@ -416,9 +418,10 @@ class vmware_node_scenarios(abstract_node_scenarios): vm_stopped = self.vsphere.stop_instances(node) if vm_stopped: self.vsphere.wait_until_stopped(node, timeout, affected_node) - nodeaction.wait_for_ready_status( - node, timeout, self.kubecli, affected_node - ) + if self.node_action_kube_check: + nodeaction.wait_for_ready_status( + node, timeout, self.kubecli, affected_node + ) logging.info(f"Node with instance ID: {node} is in stopped state") logging.info("node_stop_scenario has been successfully injected!") self.affected_nodes_status.affected_nodes.append(affected_node) @@ -436,10 +439,10 @@ class vmware_node_scenarios(abstract_node_scenarios): logging.info("Starting node_reboot_scenario injection") logging.info(f"Rebooting the node {node} ") self.vsphere.reboot_instances(node) - - nodeaction.wait_for_unknown_status( - node, timeout, self.kubecli, affected_node - ) + if self.node_action_kube_check: + nodeaction.wait_for_unknown_status( + node, timeout, self.kubecli, affected_node + ) logging.info( f"Node with instance ID: {node} has rebooted " "successfully" diff --git a/scenarios/openshift/aws_node_scenarios.yml b/scenarios/openshift/aws_node_scenarios.yml index 76953786..cf8da497 100644 --- a/scenarios/openshift/aws_node_scenarios.yml +++ b/scenarios/openshift/aws_node_scenarios.yml @@ -9,6 +9,7 @@ node_scenarios: duration: 20 # duration to stop the node before running the start action cloud_type: aws # cloud type on which Kubernetes/OpenShift runs parallel: true # Run action on label or node name in parallel or sequential, defaults to sequential + kube_check: true # Run the kubernetes api calls to see if the node gets to a certain state during the node scenario - actions: - node_reboot_scenario node_name: diff --git a/scenarios/openshift/azure_node_scenarios.yml b/scenarios/openshift/azure_node_scenarios.yml index 7a24927d..710be7d3 100644 --- a/scenarios/openshift/azure_node_scenarios.yml +++ b/scenarios/openshift/azure_node_scenarios.yml @@ -6,6 +6,8 @@ node_scenarios: instance_count: 1 timeout: 120 cloud_type: azure + parallel: true # Run action on label or node name in parallel or sequential, defaults to sequential + kube_check: true # Run the kubernetes api calls to see if the node gets to a certain state during the node scenario - actions: - node_stop_start_scenario node_name: diff --git a/scenarios/openshift/gcp_node_scenarios.yml b/scenarios/openshift/gcp_node_scenarios.yml index 8850847d..403caf40 100644 --- a/scenarios/openshift/gcp_node_scenarios.yml +++ b/scenarios/openshift/gcp_node_scenarios.yml @@ -6,6 +6,8 @@ node_scenarios: instance_count: 1 timeout: 120 cloud_type: gcp + parallel: true # Run action on label or node name in parallel or sequential, defaults to sequential + kube_check: true # Run the kubernetes api calls to see if the node gets to a certain state during the node scenario - actions: - node_stop_start_scenario node_name: