From adb465cab02f7b1ecc289e99f3c8d4ed773ca08a Mon Sep 17 00:00:00 2001 From: Naga Ravi Chaitanya Elluri Date: Wed, 18 Aug 2021 11:05:46 -0400 Subject: [PATCH] Add support for multi-zone disruption This will enable users to disrupt multiple zones in the cluster simultaneously to be able to understand the behaviour of various components. --- docs/zone_outage.md | 5 ++- kraken/node_actions/aws_node_scenarios.py | 6 +-- kraken/zone_outage/actions.py | 47 +++++++++++++++-------- scenarios/zone_outage.yaml | 2 +- 4 files changed, 37 insertions(+), 23 deletions(-) diff --git a/docs/zone_outage.md b/docs/zone_outage.md index 4bea5feb..7ea35428 100644 --- a/docs/zone_outage.md +++ b/docs/zone_outage.md @@ -1,5 +1,5 @@ ### Zone outage scenario -Scenario to create outage in a targeted zone in the public cloud to understand the impact on both Kubernetes/OpenShift platform as well as applications running on the worker nodes in that zone. It tweaks the network acl of the zone to simulate the failure and that in turn will stop both ingress and egress traffic from all the nodes in a particualar zone for the specified duration and reverts it back to the previous state. Zone outage can be injected by placing the zone_outage config file under zone_outages option in the [kraken config](https://github.com/cloud-bulldozer/kraken/blob/master/config/config.yaml). Refer to [zone_outage_scenario](https://github.com/openshift-scale/kraken/blob/master/scenarios/zone_outage.yaml) config file for the parameters that need to be defined. +Scenario to create outage in a targeted zone in the public cloud to understand the impact on both Kubernetes/OpenShift control plane as well as applications running on the worker nodes in that zone. It tweaks the network acl of the zone to simulate the failure and that in turn will stop both ingress and egress traffic from all the nodes in a particualar zone for the specified duration and reverts it back to the previous state. Zone outage can be injected by placing the zone_outage config file under zone_outages option in the [kraken config](https://github.com/cloud-bulldozer/kraken/blob/master/config/config.yaml). Refer to [zone_outage_scenario](https://github.com/openshift-scale/kraken/blob/master/scenarios/zone_outage.yaml) config file for the parameters that need to be defined. Refer to [cloud setup](cloud_setup.md) to configure your cli properly for the cloud provider of the cluster you want to shut down @@ -12,10 +12,11 @@ zone_outage: # Scenario to create an out cloud_type: aws # cloud type on which Kubernetes/OpenShift runs. aws is only platform supported currently for this scenario. duration: 600 # duration in seconds after which the zone will be back online vpc_id: # cluster virtual private network to target - subnet_id: # subnet-id to deny both ingress and egress traffic + subnet_id: [subnet1, subnet2] # List of subnet-id's to deny both ingress and egress traffic ``` **NOTE**: vpc_id and subnet_id can be obtained from the cloud web console by selecting one of the instances in the targeted zone ( us-west-2a for example ). +**NOTE**: Multiple zones will experience downtime in case of targeting multiple subnets which might have an impact on the cluster health especially if the zones have control plane components deployed. ##### Debugging steps in case of failures In case of failures during the steps which revert back the network acl to allow traffic and bring back the cluster nodes in the zone, the nodes in the particular zone will be in `NotReady` condition. Here is how to fix it: diff --git a/kraken/node_actions/aws_node_scenarios.py b/kraken/node_actions/aws_node_scenarios.py index 46225ace..e96e12ab 100644 --- a/kraken/node_actions/aws_node_scenarios.py +++ b/kraken/node_actions/aws_node_scenarios.py @@ -100,7 +100,7 @@ class AWS: except Exception as e: logging.error( "Failed to create the default network_acl: %s" - "Making sure you have aws cli configured on the host and set for the region of your vpc/subnet" % (e) + "Make sure you have aws cli configured on the host and set for the region of your vpc/subnet" % (e) ) sys.exit(1) return acl_id @@ -129,7 +129,7 @@ class AWS: except Exception as e: logging.error( "Failed to describe network acl: %s." - "Making sure you have aws cli configured on the host and set for the region of your vpc/subnet" % (e) + "Make sure you have aws cli configured on the host and set for the region of your vpc/subnet" % (e) ) sys.exit(1) associations = response["NetworkAcls"][0]["Associations"] @@ -145,7 +145,7 @@ class AWS: except Exception as e: logging.error( "Failed to delete network_acl %s: %s" - "Making sure you have aws cli configured on the host and set for the region of your vpc/subnet" + "Make sure you have aws cli configured on the host and set for the region of your vpc/subnet" % (acl_id, e) ) sys.exit(1) diff --git a/kraken/zone_outage/actions.py b/kraken/zone_outage/actions.py index 700e280f..3d2414f8 100644 --- a/kraken/zone_outage/actions.py +++ b/kraken/zone_outage/actions.py @@ -15,10 +15,11 @@ def run(scenarios_list, config, wait_duration): zone_outage_config_yaml = yaml.full_load(f) scenario_config = zone_outage_config_yaml["zone_outage"] vpc_id = scenario_config["vpc_id"] - subnet_id = scenario_config["subnet_id"] + subnet_ids = scenario_config["subnet_id"] duration = scenario_config["duration"] cloud_type = scenario_config["cloud_type"] - network_association_ids = [] + ids = {} + acl_ids_created = [] if cloud_type.lower() == "aws": cloud_object = AWS() @@ -28,29 +29,41 @@ def run(scenarios_list, config, wait_duration): start_time = int(time.time()) - associations, original_acl_id = cloud_object.describe_network_acls(vpc_id, subnet_id) - for entry in associations: - if entry["SubnetId"] == subnet_id: - network_association_ids.append(entry["NetworkAclAssociationId"]) - logging.info( - "Network association ids associated with the subnet %s: %s" % (subnet_id, network_association_ids) - ) - acl_id = cloud_object.create_default_network_acl(vpc_id) - new_association_id = cloud_object.replace_network_acl_association(network_association_ids[0], acl_id) + for subnet_id in subnet_ids: + logging.info("Targeting subnet_id") + network_association_ids = [] + associations, original_acl_id = cloud_object.describe_network_acls(vpc_id, subnet_id) + for entry in associations: + if entry["SubnetId"] == subnet_id: + network_association_ids.append(entry["NetworkAclAssociationId"]) + logging.info( + "Network association ids associated with the subnet %s: %s" + % (subnet_id, network_association_ids) + ) + acl_id = cloud_object.create_default_network_acl(vpc_id) + new_association_id = cloud_object.replace_network_acl_association( + network_association_ids[0], acl_id + ) + + # capture the orginal_acl_id, created_acl_id and new association_id to use during the recovery + ids[new_association_id] = original_acl_id + acl_ids_created.append(acl_id) # wait for the specified duration - logging.info("Waiting for the specified duration: %s" % (duration)) + logging.info("Waiting for the specified duration in the config: %s" % (duration)) time.sleep(duration) # replace the applied acl with the previous acl in use - logging.info("Replacing the applied acl with the original acl: %s" % (original_acl_id)) - cloud_object.replace_network_acl_association(new_association_id, original_acl_id) + for new_association_id, original_acl_id in ids.items(): + cloud_object.replace_network_acl_association(new_association_id, original_acl_id) + logging.info("Wating for 60 seconds to make sure the changes are in place") + time.sleep(60) # delete the network acl created for the run - logging.info("Deleting the network acl created for the run: %s" % (acl_id)) - cloud_object.delete_network_acl(acl_id) + for acl_id in acl_ids_created: + cloud_object.delete_network_acl(acl_id) - logging.info("Waiting for the specified duration: %s" % (wait_duration)) + logging.info("End of scenario. Waiting for the specified duration: %s" % (wait_duration)) time.sleep(wait_duration) end_time = int(time.time()) diff --git a/scenarios/zone_outage.yaml b/scenarios/zone_outage.yaml index 8ca7ffbf..a54c000b 100644 --- a/scenarios/zone_outage.yaml +++ b/scenarios/zone_outage.yaml @@ -2,4 +2,4 @@ zone_outage: # Scenario to create an out cloud_type: aws # cloud type on which Kubernetes/OpenShift runs. aws is only platform supported currently for this scenario. duration: 600 # duration in seconds after which the zone will be back online vpc_id: # cluster virtual private network to target - subnet_id: # subnet-id to deny both ingress and egress traffic + subnet_id: [subnet1, subnet2] # List of subnet-id's to deny both ingress and egress traffic