From 8e0f4e63afae3e0eea40a4a4e693183659b23c04 Mon Sep 17 00:00:00 2001 From: prubenda Date: Mon, 28 Jun 2021 21:36:58 -0400 Subject: [PATCH] Adding container for pod_exec in time scenarios --- docs/time_scenarios.md | 3 ++ kraken/kubernetes/client.py | 9 ++++ kraken/time_actions/common_time_functions.py | 53 +++++++++++++++++--- scenarios/time_scenarios_example.yml | 1 + 4 files changed, 58 insertions(+), 8 deletions(-) diff --git a/docs/time_scenarios.md b/docs/time_scenarios.md index a6d80a8d..15ea0408 100644 --- a/docs/time_scenarios.md +++ b/docs/time_scenarios.md @@ -12,6 +12,8 @@ Configuration Options: **label_selector:** label on the nodes or pods you want to skew +**container_name:** container name in pod you want to reset time on, if left blank it will randomly select one + **object_name:** list of the names of pods or nodes you want to skew Refer to [time_scenarios_example](https://github.com/openshift-scale/kraken/blob/master/scenarios/time_scenarios_example.yml) config file. @@ -24,6 +26,7 @@ time_scenarios: - apiserver-868595fcbb-6qnsc - apiserver-868595fcbb-mb9j5 namespace: openshift-apiserver + container_name: openshift-apiserver - action: skew_date object_type: node label_selector: node-role.kubernetes.io/worker diff --git a/kraken/kubernetes/client.py b/kraken/kubernetes/client.py index 7cc38849..ad279206 100644 --- a/kraken/kubernetes/client.py +++ b/kraken/kubernetes/client.py @@ -164,6 +164,15 @@ def exec_cmd_in_pod(command, pod_name, namespace, container=None): return ret +def get_containers_in_pod(pod_name, namespace): + pod_info = cli.read_namespaced_pod(pod_name, namespace) + container_names = [] + + for cont in pod_info.spec.containers: + container_names.append(cont.name) + return container_names + + # Obtain node status def get_node_status(node): try: diff --git a/kraken/time_actions/common_time_functions.py b/kraken/time_actions/common_time_functions.py index e27cbb90..3abfd351 100644 --- a/kraken/time_actions/common_time_functions.py +++ b/kraken/time_actions/common_time_functions.py @@ -7,12 +7,13 @@ import re import sys import kraken.cerberus.setup as cerberus import yaml +import random -def pod_exec(pod_name, command, namespace): +def pod_exec(pod_name, command, namespace, container_name): i = 0 for i in range(5): - response = kubecli.exec_cmd_in_pod(command, pod_name, namespace) + response = kubecli.exec_cmd_in_pod(command, pod_name, namespace, container_name) if not response: time.sleep(2) continue @@ -29,6 +30,19 @@ def node_debug(node_name, command): return response +def get_container_name(pod_name, namespace, container_name=""): + + container_names = kubecli.get_containers_in_pod(pod_name, namespace) + if container_name != "": + if container_name in container_names: + return container_name + else: + logging.error("Container name %s not an existing container in pod %s" % (container_name, pod_name)) + else: + container_name = container_names[random.randint(0, len(container_names) - 1)] + return container_name + + def skew_time(scenario): skew_command = "date --set " if scenario["action"] == "skew_date": @@ -50,6 +64,7 @@ def skew_time(scenario): return "node", node_names elif "pod" in scenario["object_type"]: + container_name = scenario.get("container_name", "") pod_names = [] if "object_name" in scenario.keys() and scenario["object_name"]: for name in scenario["object_name"]: @@ -79,23 +94,45 @@ def skew_time(scenario): if len(pod_names) == 0: logging.info("Cannot find pods matching the namespace/label_selector, please check") sys.exit(1) + pod_counter = 0 for pod in pod_names: if len(pod) > 1: - pod_exec(pod[0], skew_command, pod[1]) + selected_container_name = get_container_name(pod[0], pod[1], container_name) + pod_exec_response = pod_exec(pod[0], skew_command, pod[1], selected_container_name) + if pod_exec_response is False: + logging.error( + "Couldn't reset time on container %s in pod %s in namespace %s" + % (selected_container_name, pod[0], pod[1]) + ) + sys.exit(1) + pod_names[pod_counter].append(selected_container_name) else: - pod_exec(pod, skew_command, scenario["namespace"]) + selected_container_name = get_container_name(pod, scenario["namespace"], container_name) + pod_exec_response = pod_exec(pod, skew_command, scenario["namespace"], selected_container_name) + if pod_exec_response is False: + logging.error( + "Couldn't reset time on container %s in pod %s in namespace %s" + % (selected_container_name, pod, scenario["namespace"]) + ) + sys.exit(1) + pod_names[pod_counter].append(selected_container_name) logging.info("Reset date/time on pod " + str(pod[0])) + pod_counter += 1 return "pod", pod_names # From kubectl/oc command get time output def parse_string_date(obj_datetime): try: + logging.info("obj_date time " + str(obj_datetime)) date_line = re.search( - r"[a-zA-Z0-9_() .]*\w{3}\s{1,}\w{3}\s{1,}\d{2}\s{1,}\d{2}:\d{2}:\d{2}\s{1,}\w{3} " r"\d{4}\W*", obj_datetime + r"[\s\S\n]*\w{3}\s{1,}\w{3}\s{1,}\d{2}\s{1,}\d{2}:\d{2}:\d{2}\s{1,}\w{3} " r"\d{4}\W*", obj_datetime ) - return date_line.group().strip() + search_response = date_line.group().strip() + logging.info("search_res" + str(search_response)) + return search_response except Exception: + logging.info("exception") return "" @@ -135,13 +172,13 @@ def check_date_time(object_type, names): for pod_name in names: first_date_time = datetime.datetime.utcnow() counter = 0 - pod_datetime_string = pod_exec(pod_name[0], skew_command, pod_name[1]) + pod_datetime_string = pod_exec(pod_name[0], skew_command, pod_name[1], pod_name[2]) pod_datetime = string_to_date(pod_datetime_string) while not first_date_time < pod_datetime < datetime.datetime.utcnow(): time.sleep(10) logging.info("Date/time on pod %s still not reset, waiting 10 seconds and retrying" % pod_name[0]) first_date_time = datetime.datetime.utcnow() - pod_datetime = pod_exec(pod_name[0], skew_command, pod_name[1]) + pod_datetime = pod_exec(pod_name[0], skew_command, pod_name[1], pod_name[2]) pod_datetime = string_to_date(pod_datetime) counter += 1 if counter > max_retries: diff --git a/scenarios/time_scenarios_example.yml b/scenarios/time_scenarios_example.yml index b4aed4c2..e5129bc8 100644 --- a/scenarios/time_scenarios_example.yml +++ b/scenarios/time_scenarios_example.yml @@ -2,6 +2,7 @@ time_scenarios: - action: skew_time object_type: pod namespace: openshift-etcd + container_name: etcd label_selector: app=etcd - action: skew_date object_type: node