From ccd902565e45b5aede1b8063a3e015d36b108562 Mon Sep 17 00:00:00 2001 From: Janos Bonic <86970079+janosdebugs@users.noreply.github.com> Date: Wed, 20 Jul 2022 18:43:32 +0200 Subject: [PATCH] Fixes #265: Replace Powerfulseal and introduce Wolkenwalze SDK for plugin system --- .github/workflows/build.yml | 15 +- CI/scenarios/hello_pod_killing.yml | 37 +-- config/config.yaml | 13 +- config/config_kubernetes.yaml | 4 +- config/config_performance.yaml | 13 +- docs/getting_started.md | 48 +--- docs/pod_scenarios.md | 44 ++- kraken/__init__.py | 0 kraken/plugins/__init__.py | 169 +++++++++++ kraken/plugins/__main__.py | 4 + kraken/plugins/pod_plugin.py | 269 ++++++++++++++++++ kraken/plugins/run_python_plugin.py | 51 ++++ kraken/plugins/test_pod_plugin.py | 175 ++++++++++++ kraken/plugins/test_run_python_plugin.py | 28 ++ kraken/pod_scenarios/setup.py | 31 +- kraken/post_actions/actions.py | 16 +- requirements.txt | 4 +- run_kraken.py | 10 +- scenarios/kube/pod.yml | 6 + scenarios/kube/scheduler.yml | 42 +-- scenarios/openshift/customapp_pod.yaml | 42 +-- scenarios/openshift/etcd.yml | 42 +-- scenarios/openshift/openshift-apiserver.yml | 45 +-- .../openshift/openshift-kube-apiserver.yml | 4 + .../openshift/post_action_prometheus.yml | 31 +- scenarios/openshift/prometheus.yml | 46 +-- .../openshift/regex_openshift_pod_kill.yml | 26 +- scenarios/plugin.schema.README.md | 5 + scenarios/plugin.schema.json | 157 ++++++++++ 29 files changed, 1043 insertions(+), 334 deletions(-) create mode 100644 kraken/__init__.py create mode 100644 kraken/plugins/__init__.py create mode 100644 kraken/plugins/__main__.py create mode 100755 kraken/plugins/pod_plugin.py create mode 100644 kraken/plugins/run_python_plugin.py create mode 100644 kraken/plugins/test_pod_plugin.py create mode 100644 kraken/plugins/test_run_python_plugin.py create mode 100644 scenarios/kube/pod.yml create mode 100644 scenarios/plugin.schema.README.md create mode 100644 scenarios/plugin.schema.json diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ff468c68..30a24309 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -11,16 +11,23 @@ jobs: steps: - name: Check out code uses: actions/checkout@v3 - - name: Build the Docker images - run: docker build --no-cache -t quay.io/chaos-kubox/krkn containers/ - name: Create multi-node KinD cluster - uses: chaos-kubox/actions/kind@main + uses: redhat-chaos/actions/kind@main + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: '3.9' + architecture: 'x64' - name: Install environment run: | sudo apt-get install build-essential python3-dev pip install -r requirements.txt + - name: Run unit tests + run: python -m unittest discover - name: Run CI run: ./CI/run.sh + - name: Build the Docker images + run: docker build --no-cache -t quay.io/chaos-kubox/krkn containers/ - name: Login in quay if: github.ref == 'refs/heads/main' && github.event_name == 'push' run: docker login quay.io -u ${QUAY_USER} -p ${QUAY_TOKEN} @@ -32,7 +39,7 @@ jobs: run: docker push quay.io/chaos-kubox/krkn - name: Rebuild krkn-hub if: github.ref == 'refs/heads/main' && github.event_name == 'push' - uses: chaos-kubox/actions/krkn-hub@main + uses: redhat-chaos/actions/krkn-hub@main with: QUAY_USER: ${{ secrets.QUAY_USER_1 }} QUAY_TOKEN: ${{ secrets.QUAY_TOKEN_1 }} diff --git a/CI/scenarios/hello_pod_killing.yml b/CI/scenarios/hello_pod_killing.yml index 7e355503..791f8ba8 100755 --- a/CI/scenarios/hello_pod_killing.yml +++ b/CI/scenarios/hello_pod_killing.yml @@ -1,31 +1,6 @@ -config: - runStrategy: - runs: 1 - maxSecondsBetweenRuns: 30 - minSecondsBetweenRuns: 1 -scenarios: - - name: "delete hello pods" - steps: - - podAction: - matches: - - labels: - namespace: "default" - selector: "hello-openshift" - filters: - - randomSample: - size: 1 - actions: - - kill: - probability: 1 - force: true - - podAction: - matches: - - labels: - namespace: "default" - selector: "hello-openshift" - retries: - retriesTimeout: - timeout: 180 - actions: - - checkPodCount: - count: 1 +# yaml-language-server: $schema=../../scenarios/plugin.schema.json +- id: kill-pods + config: + label_selector: name=hello-openshift + namespace_pattern: ^default$ + kill: 1 diff --git a/config/config.yaml b/config/config.yaml index 65974cee..feb1f8cd 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -11,15 +11,14 @@ kraken: chaos_scenarios: # List of policies/chaos scenarios to load - container_scenarios: # List of chaos pod scenarios to load - - scenarios/openshift/container_etcd.yml - - pod_scenarios: - - - scenarios/openshift/etcd.yml - - - scenarios/openshift/regex_openshift_pod_kill.yml - - scenarios/openshift/post_action_regex.py + - plugin_scenarios: + - scenarios/openshift/etcd.yml + - scenarios/openshift/regex_openshift_pod_kill.yml - node_scenarios: # List of chaos node scenarios to load - scenarios/openshift/node_scenarios_example.yml - - pod_scenarios: - - - scenarios/openshift/openshift-apiserver.yml - - - scenarios/openshift/openshift-kube-apiserver.yml + - plugin_scenarios: + - scenarios/openshift/openshift-apiserver.yml + - scenarios/openshift/openshift-kube-apiserver.yml - time_scenarios: # List of chaos time scenarios to load - scenarios/openshift/time_scenarios_example.yml - litmus_scenarios: # List of litmus scenarios to load diff --git a/config/config_kubernetes.yaml b/config/config_kubernetes.yaml index 8a227369..34c871e0 100644 --- a/config/config_kubernetes.yaml +++ b/config/config_kubernetes.yaml @@ -11,8 +11,8 @@ kraken: chaos_scenarios: # List of policies/chaos scenarios to load - container_scenarios: # List of chaos pod scenarios to load - - scenarios/kube/container_dns.yml - - pod_scenarios: - - - scenarios/kube/scheduler.yml + - plugin_scenarios: + - scenarios/kube/scheduler.yml cerberus: cerberus_enabled: False # Enable it when cerberus is previously installed diff --git a/config/config_performance.yaml b/config/config_performance.yaml index 2c6ed4df..05c2f3f6 100644 --- a/config/config_performance.yaml +++ b/config/config_performance.yaml @@ -9,15 +9,14 @@ kraken: litmus_uninstall: False # If you want to uninstall litmus if failure litmus_uninstall_before_run: True # If you want to uninstall litmus before a new run starts chaos_scenarios: # List of policies/chaos scenarios to load - - pod_scenarios: # List of chaos pod scenarios to load - - - scenarios/openshift/etcd.yml - - - scenarios/openshift/regex_openshift_pod_kill.yml - - scenarios/openshift/post_action_regex.py + - plugin_scenarios: # List of chaos pod scenarios to load + - scenarios/openshift/etcd.yml + - scenarios/openshift/regex_openshift_pod_kill.yml - node_scenarios: # List of chaos node scenarios to load - scenarios/openshift/node_scenarios_example.yml - - pod_scenarios: - - - scenarios/openshift/openshift-apiserver.yml - - - scenarios/openshift/openshift-kube-apiserver.yml + - plugin_scenarios: + - scenarios/openshift/openshift-apiserver.yml + - scenarios/openshift/openshift-kube-apiserver.yml - time_scenarios: # List of chaos time scenarios to load - scenarios/openshift/time_scenarios_example.yml - litmus_scenarios: # List of litmus scenarios to load diff --git a/docs/getting_started.md b/docs/getting_started.md index 82c49482..6d3f5fef 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -8,45 +8,19 @@ You can either copy an existing yaml file and make it your own, or fill in one o #### Pod Scenario Yaml Template For example, for adding a pod level scenario for a new application, refer to the sample scenario below to know what fields are necessary and what to add in each location: ``` -config: - runStrategy: - runs: - #This will choose a random number to wait between min and max - maxSecondsBetweenRuns: 30 - minSecondsBetweenRuns: 1 -scenarios: - - name: "delete pods example" - steps: - - podAction: - matches: - - labels: - namespace: "" - selector: "" # This can be left blank. - filters: - - randomSample: - size: - actions: - - kill: - probability: 1 - force: true - - podAction: - matches: - - labels: - namespace: "" - selector: "" # This can be left blank. - retries: - retriesTimeout: - # Amount of time to wait with retrying, before failing if pod count does not match expected - # timeout: 180. - - actions: - - checkPodCount: - count: $ + label_selector: + kill: +- id: wait-for-pods + config: + namespace_pattern: ^$ + label_selector: + count: ``` -More information on specific items that you can add to the pod killing scenarios can be found in the [powerfulseal policies](https://powerfulseal.github.io/powerfulseal/policies) documentation - - #### Node Scenario Yaml Template ``` diff --git a/docs/pod_scenarios.md b/docs/pod_scenarios.md index 0433be3a..ddf6e6d1 100644 --- a/docs/pod_scenarios.md +++ b/docs/pod_scenarios.md @@ -1,14 +1,40 @@ ### Pod Scenarios -Kraken consumes [Powerfulseal](https://github.com/powerfulseal/powerfulseal) under the hood to run the pod scenarios. -These scenarios are in a simple yaml format that you can manipulate to run your specific tests or use the pre-existing scenarios to see how it works. + +Krkn recently replaced PowerfulSeal with its own internal pod scenarios using a plugin system. You can run pod scenarios by adding the following config to Krkn: + +```yaml +kraken: + chaos_scenarios: + - plugin_scenarios: + - path/to/scenario.yaml +``` + +You can then create the scenario file with the following contents: + +```yaml +# yaml-language-server: $schema=../plugin.schema.json +- id: kill-pods + config: + namespace_pattern: ^kube-system$ + label_selector: k8s-app=kube-scheduler +- id: wait-for-pods + config: + namespace_pattern: ^kube-system$ + label_selector: k8s-app=kube-scheduler + count: 3 +``` + +Please adjust the schema reference to point to the [schema file](../scenarios/plugin.schema.json). This file will give you code completion and documentation for the available options in your IDE. #### Pod Chaos Scenarios + The following are the components of Kubernetes/OpenShift for which a basic chaos scenario config exists today. -Component | Description | Working ------------------------- |----------------------------------------------------------------------------------------------| ------------------------- | -[Etcd](https://github.com/chaos-kubox/krkn/blob/main/scenarios/etcd.yml) | Kills a single/multiple etcd replicas for the specified number of times in a loop. | :heavy_check_mark: | -[Kube ApiServer](https://github.com/chaos-kubox/krkn/blob/main/scenarios/openshift-kube-apiserver.yml) | Kills a single/multiple kube-apiserver replicas for the specified number of times in a loop. | :heavy_check_mark: | -[ApiServer](https://github.com/chaos-kubox/krkn/blob/main/scenarios/openshift-apiserver.yml) | Kills a single/multiple apiserver replicas for the specified number of times in a loop. | :heavy_check_mark: | -[Prometheus](https://github.com/chaos-kubox/krkn/blob/main/scenarios/prometheus.yml) | Kills a single/multiple prometheus replicas for the specified number of times in a loop. | :heavy_check_mark: | -[OpenShift System Pods](https://github.com/chaos-kubox/krkn/blob/main/scenarios/regex_openshift_pod_kill.yml) | Kills random pods running in the OpenShift system namespaces. | :heavy_check_mark: | +| Component | Description | Working | +| ------------------------ |-------------| -------- | +| [Basic pod scenario](../scenarios/kube/pod.yml) | Kill a pod. | :heavy_check_mark: | +| [Etcd](../scenarios/openshift/etcd.yml) | Kills a single/multiple etcd replicas. | :heavy_check_mark: | +| [Kube ApiServer](../scenarios/openshift/openshift-kube-apiserver.yml)| Kills a single/multiple kube-apiserver replicas. | :heavy_check_mark: | +| [ApiServer](../scenarios/openshift/openshift-apiserver.yml) | Kills a single/multiple apiserver replicas. | :heavy_check_mark: | +| [Prometheus](../scenarios/openshift/prometheus.yml) | Kills a single/multiple prometheus replicas. | :heavy_check_mark: | +| [OpenShift System Pods](../scenarios/openshift/regex_openshift_pod_kill.yml) | Kills random pods running in the OpenShift system namespaces. | :heavy_check_mark: | diff --git a/kraken/__init__.py b/kraken/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/kraken/plugins/__init__.py b/kraken/plugins/__init__.py new file mode 100644 index 00000000..2c07a4de --- /dev/null +++ b/kraken/plugins/__init__.py @@ -0,0 +1,169 @@ +import dataclasses +import json +import logging +from os.path import abspath +from typing import List, Dict + +from arcaflow_plugin_sdk import schema, serialization, jsonschema + +from kraken.plugins.pod_plugin import kill_pods, wait_for_pods +from kraken.plugins.run_python_plugin import run_python_file + + +@dataclasses.dataclass +class PluginStep: + schema: schema.StepSchema + error_output_ids: List[str] + + def render_output(self, output_id: str, output_data) -> str: + return json.dumps({ + "output_id": output_id, + "output_data": self.schema.outputs[output_id].serialize(output_data), + }, indent='\t') + + +class Plugins: + """ + Plugins is a class that can run plugins sequentially. The output is rendered to the standard output and the process + is aborted if a step fails. + """ + steps_by_id: Dict[str, PluginStep] + + def __init__(self, steps: List[PluginStep]): + self.steps_by_id = dict() + for step in steps: + if step.schema.id in self.steps_by_id: + raise Exception( + "Duplicate step ID: {}".format(step.schema.id) + ) + self.steps_by_id[step.schema.id] = step + + def run(self, file: str, kubeconfig_path: str): + """ + Run executes a series of steps + """ + data = serialization.load_from_file(abspath(file)) + if not isinstance(data, list): + raise Exception( + "Invalid scenario configuration file: {} expected list, found {}".format(file, type(data).__name__) + ) + i = 0 + for entry in data: + if not isinstance(entry, dict): + raise Exception( + "Invalid scenario configuration file: {} expected a list of dict's, found {} on step {}".format( + file, + type(entry).__name__, + i + ) + ) + if "id" not in entry: + raise Exception( + "Invalid scenario configuration file: {} missing 'id' field on step {}".format( + file, + i, + ) + ) + if "config" not in entry: + raise Exception( + "Invalid scenario configuration file: {} missing 'config' field on step {}".format( + file, + i, + ) + ) + + if entry["id"] not in self.steps_by_id: + raise Exception( + "Invalid step {} in {} ID: {} expected one of: {}".format( + i, + file, + entry["id"], + ', '.join(self.steps_by_id.keys()) + ) + ) + step = self.steps_by_id[entry["id"]] + unserialized_input = step.schema.input.unserialize(entry["config"]) + if "kubeconfig_path" in step.schema.input.properties: + unserialized_input.kubeconfig_path = kubeconfig_path + output_id, output_data = step.schema(unserialized_input) + logging.info(step.render_output(output_id, output_data) + "\n") + if output_id in step.error_output_ids: + raise Exception( + "Step {} in {} ({}) failed".format(i, file, step.schema.id) + ) + i = i + 1 + + def json_schema(self): + """ + This function generates a JSON schema document and renders it from the steps passed. + """ + result = { + "$id": "https://github.com/chaos-kubox/krkn/", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Kraken Arcaflow scenarios", + "description": "Serial execution of Arcaflow Python plugins. See https://github.com/arcaflow for details.", + "type": "array", + "minContains": 1, + "items": { + "oneOf": [ + + ] + } + } + for step_id in self.steps_by_id.keys(): + step = self.steps_by_id[step_id] + step_input = jsonschema.step_input(step.schema) + del step_input["$id"] + del step_input["$schema"] + del step_input["title"] + del step_input["description"] + result["items"]["oneOf"].append({ + "type": "object", + "properties": { + "id": { + "type": "string", + "const": step_id, + }, + "config": step_input, + }, + "required": [ + "id", + "config", + ] + }) + return json.dumps(result, indent="\t") + + +PLUGINS = Plugins( + [ + PluginStep( + kill_pods, + [ + "error", + ] + ), + PluginStep( + wait_for_pods, + [ + "error" + ] + ), + PluginStep( + run_python_file, + [ + "error" + ] + ) + ] +) + + +def run(scenarios: List[str], kubeconfig_path: str, failed_post_scenarios: List[str]) -> List[str]: + for scenario in scenarios: + try: + PLUGINS.run(scenario, kubeconfig_path) + except Exception as e: + failed_post_scenarios.append(scenario) + logging.error("Error while running {}: {}".format(scenario, e)) + return failed_post_scenarios + return failed_post_scenarios diff --git a/kraken/plugins/__main__.py b/kraken/plugins/__main__.py new file mode 100644 index 00000000..6cbd0454 --- /dev/null +++ b/kraken/plugins/__main__.py @@ -0,0 +1,4 @@ +from kraken.plugins import PLUGINS + +if __name__ == "__main__": + print(PLUGINS.json_schema()) \ No newline at end of file diff --git a/kraken/plugins/pod_plugin.py b/kraken/plugins/pod_plugin.py new file mode 100755 index 00000000..4e3b12c9 --- /dev/null +++ b/kraken/plugins/pod_plugin.py @@ -0,0 +1,269 @@ +#!/usr/bin/env python +import re +import sys +import time +import typing +from dataclasses import dataclass, field +import random +from datetime import datetime +from traceback import format_exc + +from kubernetes import config, client +from kubernetes.client import V1PodList, V1Pod, ApiException, V1DeleteOptions +from arcaflow_plugin_sdk import validation, plugin, schema + + +def setup_kubernetes(kubeconfig_path): + if kubeconfig_path is None: + kubeconfig_path = config.KUBE_CONFIG_DEFAULT_LOCATION + kubeconfig = config.kube_config.KubeConfigMerger(kubeconfig_path) + + if kubeconfig.config is None: + raise Exception( + 'Invalid kube-config file: %s. ' + 'No configuration found.' % kubeconfig_path + ) + loader = config.kube_config.KubeConfigLoader( + config_dict=kubeconfig.config, + ) + client_config = client.Configuration() + loader.load_and_set(client_config) + return client.ApiClient(configuration=client_config) + + +def _find_pods(core_v1, label_selector, name_pattern, namespace_pattern): + pods: typing.List[V1Pod] = [] + _continue = None + finished = False + while not finished: + pod_response: V1PodList = core_v1.list_pod_for_all_namespaces( + watch=False, + label_selector=label_selector + ) + for pod in pod_response.items: + pod: V1Pod + if (name_pattern is None or name_pattern.match(pod.metadata.name)) and \ + namespace_pattern.match(pod.metadata.namespace): + pods.append(pod) + _continue = pod_response.metadata._continue + if _continue is None: + finished = True + return pods + + +@dataclass +class Pod: + namespace: str + name: str + + +@dataclass +class PodKillSuccessOutput: + pods: typing.Dict[int, Pod] = field(metadata={ + "name": "Pods removed", + "description": "Map between timestamps and the pods removed. The timestamp is provided in nanoseconds." + }) + + +@dataclass +class PodWaitSuccessOutput: + pods: typing.List[Pod] = field(metadata={ + "name": "Pods", + "description": "List of pods that have been found to run." + }) + + +@dataclass +class PodErrorOutput: + error: str + + +@dataclass +class KillPodConfig: + """ + This is a configuration structure specific to pod kill scenario. It describes which pod from which + namespace(s) to select for killing and how many pods to kill. + """ + + namespace_pattern: re.Pattern = field(metadata={ + "name": "Namespace pattern", + "description": "Regular expression for target pod namespaces." + }) + + name_pattern: typing.Annotated[ + typing.Optional[re.Pattern], + validation.required_if_not("label_selector") + ] = field(default=None, metadata={ + "name": "Name pattern", + "description": "Regular expression for target pods. Required if label_selector is not set." + }) + + kill: typing.Annotated[int, validation.min(1)] = field( + default=1, + metadata={"name": "Number of pods to kill", "description": "How many pods should we attempt to kill?"} + ) + + label_selector: typing.Annotated[ + typing.Optional[str], + validation.min(1), + validation.required_if_not("name_pattern") + ] = field(default=None, metadata={ + "name": "Label selector", + "description": "Kubernetes label selector for the target pods. Required if name_pattern is not set.\n" + "See https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ for details." + }) + + kubeconfig_path: typing.Optional[str] = field(default=None, metadata={ + "name": "Kubeconfig path", + "description": "Path to your Kubeconfig file. Defaults to ~/.kube/config.\n" + "See https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/ for " + "details." + }) + + timeout: int = field(default=180, metadata={ + "name": "Timeout", + "description": "Timeout to wait for the target pod(s) to be removed in seconds." + }) + + backoff: int = field(default=1, metadata={ + "name": "Backoff", + "description": "How many seconds to wait between checks for the target pod status." + }) + + +@plugin.step( + "kill-pods", + "Kill pods", + "Kill pods as specified by parameters", + {"success": PodKillSuccessOutput, "error": PodErrorOutput} +) +def kill_pods(cfg: KillPodConfig) -> typing.Tuple[str, typing.Union[PodKillSuccessOutput, PodErrorOutput]]: + try: + with setup_kubernetes(None) as cli: + core_v1 = client.CoreV1Api(cli) + + # region Select target pods + pods = _find_pods(core_v1, cfg.label_selector, cfg.name_pattern, cfg.namespace_pattern) + if len(pods) < cfg.kill: + return "error", PodErrorOutput( + "Not enough pods match the criteria, expected {} but found only {} pods".format(cfg.kill, len(pods)) + ) + random.shuffle(pods) + # endregion + + # region Remove pods + killed_pods: typing.Dict[int, Pod] = {} + watch_pods: typing.List[Pod] = [] + for i in range(cfg.kill): + pod = pods[i] + core_v1.delete_namespaced_pod(pod.metadata.name, pod.metadata.namespace, body=V1DeleteOptions( + grace_period_seconds=0, + )) + p = Pod( + pod.metadata.namespace, + pod.metadata.name + ) + killed_pods[int(time.time_ns())] = p + watch_pods.append(p) + # endregion + + # region Wait for pods to be removed + start_time = time.time() + while len(watch_pods) > 0: + time.sleep(cfg.backoff) + new_watch_pods: typing.List[Pod] = [] + for p in watch_pods: + try: + read_pod = core_v1.read_namespaced_pod(p.name, p.namespace) + new_watch_pods.append(p) + except ApiException as e: + if e.status != 404: + raise + watch_pods = new_watch_pods + current_time = time.time() + if current_time - start_time > cfg.timeout: + return "error", PodErrorOutput("Timeout while waiting for pods to be removed.") + return "success", PodKillSuccessOutput(killed_pods) + # endregion + except Exception: + return "error", PodErrorOutput( + format_exc() + ) + + +@dataclass +class WaitForPodsConfig: + """ + WaitForPodsConfig is a configuration structure for wait-for-pod steps. + """ + + namespace_pattern: re.Pattern + + name_pattern: typing.Annotated[ + typing.Optional[re.Pattern], + validation.required_if_not("label_selector") + ] = None + + label_selector: typing.Annotated[ + typing.Optional[str], + validation.min(1), + validation.required_if_not("name_pattern") + ] = None + + count: typing.Annotated[int, validation.min(1)] = field( + default=1, + metadata={"name": "Pod count", "description": "Wait for at least this many pods to exist"} + ) + + timeout: typing.Annotated[int, validation.min(1)] = field( + default=180, + metadata={"name": "Timeout", "description": "How many seconds to wait for?"} + ) + + backoff: int = field(default=1, metadata={ + "name": "Backoff", + "description": "How many seconds to wait between checks for the target pod status." + }) + + kubeconfig_path: typing.Optional[str] = None + + +@plugin.step( + "wait-for-pods", + "Wait for pods", + "Wait for the specified number of pods to be present", + {"success": PodWaitSuccessOutput, "error": PodErrorOutput} +) +def wait_for_pods(cfg: WaitForPodsConfig) -> typing.Tuple[str, typing.Union[PodWaitSuccessOutput, PodErrorOutput]]: + try: + with setup_kubernetes(None) as cli: + core_v1 = client.CoreV1Api(cli) + + timeout = False + start_time = datetime.now() + while not timeout: + pods = _find_pods(core_v1, cfg.label_selector, cfg.name_pattern, cfg.namespace_pattern) + if len(pods) >= cfg.count: + return "success", \ + PodWaitSuccessOutput(list(map(lambda p: Pod(p.metadata.namespace, p.metadata.name), pods))) + + time.sleep(cfg.backoff) + + now_time = datetime.now() + + time_diff = now_time - start_time + if time_diff.seconds > cfg.timeout: + return "error", PodErrorOutput( + "timeout while waiting for pods to come up" + ) + except Exception: + return "error", PodErrorOutput( + format_exc() + ) + + +if __name__ == "__main__": + sys.exit(plugin.run(plugin.build_schema( + kill_pods, + wait_for_pods, + ))) diff --git a/kraken/plugins/run_python_plugin.py b/kraken/plugins/run_python_plugin.py new file mode 100644 index 00000000..32ec88ed --- /dev/null +++ b/kraken/plugins/run_python_plugin.py @@ -0,0 +1,51 @@ +import dataclasses +import io +import subprocess +import sys +import typing + +from arcaflow_plugin_sdk import plugin + + +@dataclasses.dataclass +class RunPythonFileInput: + filename: str + + +@dataclasses.dataclass +class RunPythonFileOutput: + stdout: str + stderr: str + + +@dataclasses.dataclass +class RunPythonFileError: + exit_code: int + stdout: str + stderr: str + + +@plugin.step( + id="run_python", + name="Run a Python script", + description="Run a specified Python script", + outputs={"success": RunPythonFileOutput, "error": RunPythonFileError} +) +def run_python_file(params: RunPythonFileInput) -> typing.Tuple[ + str, + typing.Union[RunPythonFileOutput, RunPythonFileError] +]: + run_results = subprocess.run( + [sys.executable, params.filename], + capture_output=True + ) + if run_results.returncode == 0: + return "success", RunPythonFileOutput( + str(run_results.stdout, 'utf-8'), + str(run_results.stderr, 'utf-8') + ) + return "error", RunPythonFileError( + run_results.returncode, + str(run_results.stdout, 'utf-8'), + str(run_results.stderr, 'utf-8') + ) diff --git a/kraken/plugins/test_pod_plugin.py b/kraken/plugins/test_pod_plugin.py new file mode 100644 index 00000000..f4688884 --- /dev/null +++ b/kraken/plugins/test_pod_plugin.py @@ -0,0 +1,175 @@ +import random +import re +import string +import threading +import unittest + +from arcaflow_plugin_sdk import plugin +from kubernetes.client import V1Pod, V1ObjectMeta, V1PodSpec, V1Container, ApiException + +from kraken.plugins import pod_plugin +from kraken.plugins.pod_plugin import setup_kubernetes, KillPodConfig, PodKillSuccessOutput +from kubernetes import client + + +class KillPodTest(unittest.TestCase): + def test_serialization(self): + plugin.test_object_serialization( + pod_plugin.KillPodConfig( + namespace_pattern=re.compile(".*"), + name_pattern=re.compile(".*") + ), + self.fail, + ) + plugin.test_object_serialization( + pod_plugin.PodKillSuccessOutput( + pods={} + ), + self.fail, + ) + plugin.test_object_serialization( + pod_plugin.PodErrorOutput( + error="Hello world!" + ), + self.fail, + ) + + def test_not_enough_pods(self): + name = ''.join(random.choices(string.ascii_lowercase, k=8)) + output_id, output_data = pod_plugin.kill_pods(KillPodConfig( + namespace_pattern=re.compile("^default$"), + name_pattern=re.compile("^unit-test-" + re.escape(name) + "$"), + )) + if output_id != "error": + self.fail("Not enough pods did not result in an error.") + print(output_data.error) + + def test_kill_pod(self): + with setup_kubernetes(None) as cli: + core_v1 = client.CoreV1Api(cli) + pod = core_v1.create_namespaced_pod("default", V1Pod( + metadata=V1ObjectMeta( + generate_name="test-", + ), + spec=V1PodSpec( + containers=[ + V1Container( + name="test", + image="alpine", + tty=True, + ) + ] + ), + )) + + def remove_test_pod(): + try: + core_v1.delete_namespaced_pod(pod.metadata.name, pod.metadata.namespace) + except ApiException as e: + if e.status != 404: + raise + + self.addCleanup(remove_test_pod) + + output_id, output_data = pod_plugin.kill_pods(KillPodConfig( + namespace_pattern=re.compile("^default$"), + name_pattern=re.compile("^" + re.escape(pod.metadata.name) + "$"), + )) + + if output_id == "error": + self.fail(output_data.error) + self.assertIsInstance(output_data, PodKillSuccessOutput) + out: PodKillSuccessOutput = output_data + self.assertEqual(1, len(out.pods)) + pod_list = list(out.pods.values()) + self.assertEqual(pod.metadata.name, pod_list[0].name) + + try: + core_v1.read_namespaced_pod(pod_list[0].name, pod_list[0].namespace) + self.fail("Killed pod is still present.") + except ApiException as e: + if e.status != 404: + self.fail("Incorrect API exception encountered: {}".format(e)) + + +class WaitForPodTest(unittest.TestCase): + def test_serialization(self): + plugin.test_object_serialization( + pod_plugin.WaitForPodsConfig( + namespace_pattern=re.compile(".*"), + name_pattern=re.compile(".*") + ), + self.fail, + ) + plugin.test_object_serialization( + pod_plugin.WaitForPodsConfig( + namespace_pattern=re.compile(".*"), + label_selector="app=nginx" + ), + self.fail, + ) + plugin.test_object_serialization( + pod_plugin.PodWaitSuccessOutput( + pods=[] + ), + self.fail, + ) + plugin.test_object_serialization( + pod_plugin.PodErrorOutput( + error="Hello world!" + ), + self.fail, + ) + + def test_timeout(self): + name = "watch-test-" + ''.join(random.choices(string.ascii_lowercase, k=8)) + output_id, output_data = pod_plugin.wait_for_pods(pod_plugin.WaitForPodsConfig( + namespace_pattern=re.compile("^default$"), + name_pattern=re.compile("^" + re.escape(name) + "$"), + timeout=1 + )) + self.assertEqual("error", output_id) + + def test_watch(self): + with setup_kubernetes(None) as cli: + core_v1 = client.CoreV1Api(cli) + name = "watch-test-" + ''.join(random.choices(string.ascii_lowercase, k=8)) + + def create_test_pod(): + core_v1.create_namespaced_pod("default", V1Pod( + metadata=V1ObjectMeta( + name=name, + ), + spec=V1PodSpec( + containers=[ + V1Container( + name="test", + image="alpine", + tty=True, + ) + ] + ), + )) + + def remove_test_pod(): + try: + core_v1.delete_namespaced_pod(name, "default") + except ApiException as e: + if e.status != 404: + raise + + self.addCleanup(remove_test_pod) + + t = threading.Timer(10, create_test_pod) + t.start() + + output_id, output_data = pod_plugin.wait_for_pods(pod_plugin.WaitForPodsConfig( + namespace_pattern=re.compile("^default$"), + name_pattern=re.compile("^" + re.escape(name) + "$"), + timeout=60 + )) + self.assertEqual("success", output_id) + + +if __name__ == '__main__': + unittest.main() diff --git a/kraken/plugins/test_run_python_plugin.py b/kraken/plugins/test_run_python_plugin.py new file mode 100644 index 00000000..273ac74b --- /dev/null +++ b/kraken/plugins/test_run_python_plugin.py @@ -0,0 +1,28 @@ +import tempfile +import unittest + +from kraken.plugins import run_python_file +from kraken.plugins.run_python_plugin import RunPythonFileInput + + +class RunPythonPluginTest(unittest.TestCase): + def test_success_execution(self): + tmp_file = tempfile.NamedTemporaryFile() + tmp_file.write(bytes("print('Hello world!')", 'utf-8')) + tmp_file.flush() + output_id, output_data = run_python_file(RunPythonFileInput(tmp_file.name)) + self.assertEqual("success", output_id) + self.assertEqual("Hello world!\n", output_data.stdout) + + def test_error_execution(self): + tmp_file = tempfile.NamedTemporaryFile() + tmp_file.write(bytes("import sys\nprint('Hello world!')\nsys.exit(42)\n", 'utf-8')) + tmp_file.flush() + output_id, output_data = run_python_file(RunPythonFileInput(tmp_file.name)) + self.assertEqual("error", output_id) + self.assertEqual(42, output_data.exit_code) + self.assertEqual("Hello world!\n", output_data.stdout) + + +if __name__ == '__main__': + unittest.main() diff --git a/kraken/pod_scenarios/setup.py b/kraken/pod_scenarios/setup.py index 997a0fe9..d3e9631c 100644 --- a/kraken/pod_scenarios/setup.py +++ b/kraken/pod_scenarios/setup.py @@ -1,5 +1,8 @@ import logging -import kraken.invoke.command as runcommand + +from arcaflow_plugin_sdk import serialization +from kraken.plugins import pod_plugin + import kraken.cerberus.setup as cerberus import kraken.post_actions.actions as post_actions import kraken.kubernetes.client as kubecli @@ -20,20 +23,30 @@ def run(kubeconfig_path, scenarios_list, config, failed_post_scenarios, wait_dur try: # capture start time start_time = int(time.time()) - scenario_logs = runcommand.invoke( - "powerfulseal autonomous --use-pod-delete-instead-" - "of-ssh-kill --policy-file %s --kubeconfig %s " - "--no-cloud --inventory-kubernetes --headless" % (pod_scenario[0], kubeconfig_path) - ) + + input = serialization.load_from_file(pod_scenario) + + s = pod_plugin.get_schema() + input_data: pod_plugin.KillPodConfig = s.unserialize_input("pod", input) + + if kubeconfig_path is not None: + input_data.kubeconfig_path = kubeconfig_path + + output_id, output_data = s.call_step("pod", input_data) + + if output_id == "error": + data: pod_plugin.PodErrorOutput = output_data + logging.error("Failed to run pod scenario: {}".format(data.error)) + else: + data: pod_plugin.PodSuccessOutput = output_data + for pod in data.pods: + print("Deleted pod {} in namespace {}\n".format(pod.pod_name, pod.pod_namespace)) except Exception as e: logging.error( "Failed to run scenario: %s. Encountered the following " "exception: %s" % (pod_scenario[0], e) ) sys.exit(1) - # Display pod scenario logs/actions - print(scenario_logs) - logging.info("Scenario: %s has been successfully injected!" % (pod_scenario[0])) logging.info("Waiting for the specified duration: %s" % (wait_duration)) time.sleep(wait_duration) diff --git a/kraken/post_actions/actions.py b/kraken/post_actions/actions.py index ea525624..e7cb3a5a 100644 --- a/kraken/post_actions/actions.py +++ b/kraken/post_actions/actions.py @@ -5,21 +5,7 @@ import kraken.invoke.command as runcommand def run(kubeconfig_path, scenario, pre_action_output=""): if scenario.endswith(".yaml") or scenario.endswith(".yml"): - action_output = runcommand.invoke( - "powerfulseal autonomous " - "--use-pod-delete-instead-of-ssh-kill" - " --policy-file %s --kubeconfig %s --no-cloud" - " --inventory-kubernetes --headless" % (scenario, kubeconfig_path) - ) - # read output to make sure no error - if "ERROR" in action_output: - action_output.split("ERROR")[1].split("\n")[0] - if not pre_action_output: - logging.info("Powerful seal pre action check failed for " + str(scenario)) - return False - else: - logging.info(scenario + " post action checks passed") - + logging.error("Powerfulseal support has recently been removed. Please switch to using plugins instead.") elif scenario.endswith(".py"): action_output = runcommand.invoke("python3 " + scenario).strip() if pre_action_output: diff --git a/requirements.txt b/requirements.txt index 2058d2ad..0f28cbd4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,14 +1,13 @@ datetime pyfiglet PyYAML>=5.1 -git+https://github.com/powerfulseal/powerfulseal.git@3.3.0 requests boto3 google-api-python-client azure-mgmt-compute azure-keyvault azure-identity -kubernetes==18.20.0 +kubernetes oauth2client>=4.1.3 python-openstackclient gitpython @@ -23,3 +22,4 @@ itsdangerous==2.0.1 werkzeug==2.0.3 aliyun-python-sdk-core-v3 aliyun-python-sdk-ecs +arcaflow-plugin-sdk==0.3.0 diff --git a/run_kraken.py b/run_kraken.py index d22ed26f..8e0e9b8c 100644 --- a/run_kraken.py +++ b/run_kraken.py @@ -22,6 +22,7 @@ import kraken.application_outage.actions as application_outage import kraken.pvc.pvc_scenario as pvc_scenario import kraken.network_chaos.actions as network_chaos import server as server +from kraken import plugins def publish_kraken_status(status): @@ -158,10 +159,11 @@ def main(cfg): if scenarios_list: # Inject pod chaos scenarios specified in the config if scenario_type == "pod_scenarios": - logging.info("Running pod scenarios") - failed_post_scenarios = pod_scenarios.run( - kubeconfig_path, scenarios_list, config, failed_post_scenarios, wait_duration - ) + logging.error("Pod scenarios have been removed, please use plugin_scenarios with the " + "kill-pods configuration instead.") + sys.exit(1) + elif scenario_type == "plugin_scenarios": + failed_post_scenarios = plugins.run(scenarios_list, kubeconfig_path, failed_post_scenarios) elif scenario_type == "container_scenarios": logging.info("Running container scenarios") failed_post_scenarios = pod_scenarios.container_run( diff --git a/scenarios/kube/pod.yml b/scenarios/kube/pod.yml new file mode 100644 index 00000000..b9f30a0e --- /dev/null +++ b/scenarios/kube/pod.yml @@ -0,0 +1,6 @@ +# yaml-language-server: $schema=../plugin.schema.json +- id: kill-pods + config: + name_pattern: ^nginx-.*$ + namespace_pattern: ^default$ + kill: 1 diff --git a/scenarios/kube/scheduler.yml b/scenarios/kube/scheduler.yml index b3ee8b3e..e2168dcb 100755 --- a/scenarios/kube/scheduler.yml +++ b/scenarios/kube/scheduler.yml @@ -1,32 +1,10 @@ -config: - runStrategy: - runs: 1 - maxSecondsBetweenRuns: 30 - minSecondsBetweenRuns: 1 -scenarios: - - name: "delete scheduler pods" - steps: - - podAction: - matches: - - labels: - namespace: "kube-system" - selector: "k8s-app=kube-scheduler" - filters: - - randomSample: - size: 1 - actions: - - kill: - probability: 1 - force: true - - podAction: - matches: - - labels: - namespace: "kube-system" - selector: "k8s-app=kube-scheduler" - retries: - retriesTimeout: - timeout: 180 - - actions: - - checkPodCount: - count: 3 +# yaml-language-server: $schema=../plugin.schema.json +- id: kill-pods + config: + namespace_pattern: ^kube-system$ + label_selector: k8s-app=kube-scheduler +- id: wait-for-pods + config: + namespace_pattern: ^kube-system$ + label_selector: k8s-app=kube-scheduler + count: 3 diff --git a/scenarios/openshift/customapp_pod.yaml b/scenarios/openshift/customapp_pod.yaml index e3f1bcbd..cd836521 100644 --- a/scenarios/openshift/customapp_pod.yaml +++ b/scenarios/openshift/customapp_pod.yaml @@ -1,32 +1,10 @@ -config: - runStrategy: - runs: 1 - maxSecondsBetweenRuns: 30 - minSecondsBetweenRuns: 1 -scenarios: - - name: "delete acme-air pods" - steps: - - podAction: - matches: - - labels: - namespace: "acme-air" - selector: "" - filters: - - randomSample: - size: 1 - actions: - - kill: - probability: 1 - force: true - - podAction: - matches: - - labels: - namespace: "acme-air" - selector: "" - retries: - retriesTimeout: - timeout: 180 - - actions: - - checkPodCount: - count: 8 +# yaml-language-server: $schema=../plugin.schema.json +- id: kill-pods + config: + namespace_pattern: ^acme-air$ + name_pattern: .* +- id: wait-for-pods + config: + namespace_pattern: ^acme-air$ + name_pattern: .* + count: 8 \ No newline at end of file diff --git a/scenarios/openshift/etcd.yml b/scenarios/openshift/etcd.yml index d1386290..650bfb5d 100755 --- a/scenarios/openshift/etcd.yml +++ b/scenarios/openshift/etcd.yml @@ -1,32 +1,10 @@ -config: - runStrategy: - runs: 1 - maxSecondsBetweenRuns: 30 - minSecondsBetweenRuns: 1 -scenarios: - - name: "delete etcd pods" - steps: - - podAction: - matches: - - labels: - namespace: "openshift-etcd" - selector: "k8s-app=etcd" - filters: - - randomSample: - size: 1 - actions: - - kill: - probability: 1 - force: true - - podAction: - matches: - - labels: - namespace: "openshift-etcd" - selector: "k8s-app=etcd" - retries: - retriesTimeout: - timeout: 180 - - actions: - - checkPodCount: - count: 3 +# yaml-language-server: $schema=../plugin.schema.json +- id: kill-pods + config: + namespace_pattern: ^openshift-etcd$ + label_selector: k8s-app=etcd +- id: wait-for-pods + config: + namespace_pattern: ^openshift-etcd$ + label_selector: k8s-app=etcd + count: 3 diff --git a/scenarios/openshift/openshift-apiserver.yml b/scenarios/openshift/openshift-apiserver.yml index 5018c514..eedfeca0 100755 --- a/scenarios/openshift/openshift-apiserver.yml +++ b/scenarios/openshift/openshift-apiserver.yml @@ -1,35 +1,10 @@ -config: - runStrategy: - runs: 1 - maxSecondsBetweenRuns: 30 - minSecondsBetweenRuns: 1 -scenarios: - - name: "delete openshift-apiserver pods" - steps: - - podAction: - matches: - - labels: - namespace: "openshift-apiserver" - selector: "app=openshift-apiserver-a" - - filters: - - randomSample: - size: 1 - - # The actions will be executed in the order specified - actions: - - kill: - probability: 1 - force: true - - podAction: - matches: - - labels: - namespace: "openshift-apiserver" - selector: "app=openshift-apiserver-a" - retries: - retriesTimeout: - timeout: 180 - - actions: - - checkPodCount: - count: 3 +# yaml-language-server: $schema=../plugin.schema.json +- id: kill-pods + config: + namespace_pattern: ^openshift-apiserver$ + label_selector: app=openshift-apiserver-a +- id: wait-for-pods + config: + namespace_pattern: ^openshift-apiserver$ + label_selector: app=openshift-apiserver-a + count: 3 diff --git a/scenarios/openshift/openshift-kube-apiserver.yml b/scenarios/openshift/openshift-kube-apiserver.yml index ef118b85..26663aa2 100755 --- a/scenarios/openshift/openshift-kube-apiserver.yml +++ b/scenarios/openshift/openshift-kube-apiserver.yml @@ -1,3 +1,7 @@ +# yaml-language-server: $schema=../pod.schema.json +namespace_pattern: openshift-kube-apiserver +kill: 1 + config: runStrategy: runs: 1 diff --git a/scenarios/openshift/post_action_prometheus.yml b/scenarios/openshift/post_action_prometheus.yml index 76405fb0..33bd4545 100644 --- a/scenarios/openshift/post_action_prometheus.yml +++ b/scenarios/openshift/post_action_prometheus.yml @@ -1,21 +1,10 @@ -config: - runStrategy: - runs: 1 - maxSecondsBetweenRuns: 10 - minSecondsBetweenRuns: 1 -scenarios: - - name: "check 2 pods are in namespace with selector: prometheus" - steps: - - podAction: - matches: - - labels: - namespace: "openshift-monitoring" - selector: "app=prometheus" - filters: - - property: - name: "state" - value: "Running" - # The actions will be executed in the order specified - actions: - - checkPodCount: - count: 2 +# yaml-language-server: $schema=../plugin.schema.json +- id: kill-pods + config: + namespace_pattern: ^openshift-monitoring$ + label_selector: app=prometheus +- id: wait-for-pods + config: + namespace_pattern: ^openshift-monitoring$ + label_selector: app=prometheus + count: 2 \ No newline at end of file diff --git a/scenarios/openshift/prometheus.yml b/scenarios/openshift/prometheus.yml index d1401275..0dfbddc7 100644 --- a/scenarios/openshift/prometheus.yml +++ b/scenarios/openshift/prometheus.yml @@ -1,35 +1,11 @@ -config: - runStrategy: - runs: 1 - maxSecondsBetweenRuns: 30 - minSecondsBetweenRuns: 1 -scenarios: - - name: "delete prometheus pods" - steps: - - podAction: - matches: - - labels: - namespace: "openshift-monitoring" - selector: "app=prometheus" - - filters: - - randomSample: - size: 1 - - # The actions will be executed in the order specified - actions: - - kill: - probability: 1 - force: true - - podAction: - matches: - - labels: - namespace: "openshift-monitoring" - selector: "app=prometheus" - retries: - retriesTimeout: - timeout: 180 - - actions: - - checkPodCount: - count: 2 +# yaml-language-server: $schema=../plugin.schema.json +- id: kill-pods + config: + namespace_pattern: ^openshift-monitoring$ + label_selector: app=prometheus +- id: wait-for-pods + config: + namespace_pattern: ^openshift-monitoring$ + label_selector: app=prometheus + count: 2 + timeout: 180 \ No newline at end of file diff --git a/scenarios/openshift/regex_openshift_pod_kill.yml b/scenarios/openshift/regex_openshift_pod_kill.yml index 9bcce66b..c0f676ca 100755 --- a/scenarios/openshift/regex_openshift_pod_kill.yml +++ b/scenarios/openshift/regex_openshift_pod_kill.yml @@ -1,20 +1,6 @@ -config: - runStrategy: - runs: 1 - maxSecondsBetweenRuns: 30 - minSecondsBetweenRuns: 1 -scenarios: - - name: kill up to 3 pods in any openshift namespace - steps: - - podAction: - matches: - - namespace: "openshift-.*" - filters: - - property: - name: "state" - value: "Running" - - randomSample: - size: 3 - actions: - - kill: - probability: .7 +# yaml-language-server: $schema=../plugin.schema.json +- id: kill-pods + config: + namespace_pattern: ^openshift-.*$ + name_pattern: .* + kill: 3 diff --git a/scenarios/plugin.schema.README.md b/scenarios/plugin.schema.README.md new file mode 100644 index 00000000..436d3fb9 --- /dev/null +++ b/scenarios/plugin.schema.README.md @@ -0,0 +1,5 @@ +This file is generated by running the "plugins" module in the kraken project: + +``` +python -m kraken.plugins >scenarios/plugin.schema.json +``` \ No newline at end of file diff --git a/scenarios/plugin.schema.json b/scenarios/plugin.schema.json new file mode 100644 index 00000000..c1c6d827 --- /dev/null +++ b/scenarios/plugin.schema.json @@ -0,0 +1,157 @@ +{ + "$id": "https://github.com/chaos-kubox/krkn/", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Kraken Arcaflow scenarios", + "description": "Serial execution of Arcaflow Python plugins. See https://github.com/arcaflow for details.", + "type": "array", + "minContains": 1, + "items": { + "oneOf": [ + { + "type": "object", + "properties": { + "id": { + "type": "string", + "const": "kill-pods" + }, + "config": { + "type": "object", + "properties": { + "namespace_pattern": { + "type": "string", + "format": "regex", + "title": "Namespace pattern", + "description": "Regular expression for target pod namespaces." + }, + "name_pattern": { + "type": "string", + "format": "regex", + "title": "Name pattern", + "description": "Regular expression for target pods. Required if label_selector is not set." + }, + "kill": { + "type": "integer", + "minimum": 1, + "title": "Number of pods to kill", + "description": "How many pods should we attempt to kill?" + }, + "label_selector": { + "type": "string", + "minLength": 1, + "title": "Label selector", + "description": "Kubernetes label selector for the target pods. Required if name_pattern is not set.\nSee https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ for details." + }, + "kubeconfig_path": { + "type": "string", + "title": "Kubeconfig path", + "description": "Path to your Kubeconfig file. Defaults to ~/.kube/config.\nSee https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/ for details." + }, + "timeout": { + "type": "integer", + "title": "Timeout", + "description": "Timeout to wait for the target pod(s) to be removed in seconds." + }, + "backoff": { + "type": "integer", + "title": "Backoff", + "description": "How many seconds to wait between checks for the target pod status." + } + }, + "additionalProperties": false, + "required": [ + "namespace_pattern" + ] + } + }, + "required": [ + "id", + "config" + ] + }, + { + "type": "object", + "properties": { + "id": { + "type": "string", + "const": "wait-for-pods" + }, + "config": { + "type": "object", + "properties": { + "namespace_pattern": { + "type": "string", + "format": "regex", + "title": "namespace_pattern" + }, + "name_pattern": { + "type": "string", + "format": "regex", + "title": "name_pattern" + }, + "label_selector": { + "type": "string", + "minLength": 1, + "title": "label_selector" + }, + "count": { + "type": "integer", + "minimum": 1, + "title": "Pod count", + "description": "Wait for at least this many pods to exist" + }, + "timeout": { + "type": "integer", + "minimum": 1, + "title": "Timeout", + "description": "How many seconds to wait for?" + }, + "backoff": { + "type": "integer", + "title": "Backoff", + "description": "How many seconds to wait between checks for the target pod status." + }, + "kubeconfig_path": { + "type": "string", + "title": "kubeconfig_path" + } + }, + "additionalProperties": false, + "required": [ + "namespace_pattern" + ] + } + }, + "required": [ + "id", + "config" + ] + }, + { + "type": "object", + "properties": { + "id": { + "type": "string", + "const": "run_python" + }, + "config": { + "type": "object", + "properties": { + "filename": { + "type": "string", + "title": "filename" + } + }, + "additionalProperties": false, + "required": [ + "filename" + ] + } + }, + "required": [ + "id", + "config" + ] + } + ] + } +}