mirror of
https://github.com/krkn-chaos/krkn.git
synced 2026-08-25 09:27:36 +00:00
Fixes #265: Replace Powerfulseal and introduce Wolkenwalze SDK for plugin system
This commit is contained in:
Janos Bonic
@@ -11,16 +11,23 @@ jobs:
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v3
|
||||
- name: Build the Docker images
|
||||
run: docker build --no-cache -t quay.io/chaos-kubox/krkn containers/
|
||||
- name: Create multi-node KinD cluster
|
||||
uses: chaos-kubox/actions/kind@main
|
||||
uses: redhat-chaos/actions/kind@main
|
||||
- name: Install Python
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: '3.9'
|
||||
architecture: 'x64'
|
||||
- name: Install environment
|
||||
run: |
|
||||
sudo apt-get install build-essential python3-dev
|
||||
pip install -r requirements.txt
|
||||
- name: Run unit tests
|
||||
run: python -m unittest discover
|
||||
- name: Run CI
|
||||
run: ./CI/run.sh
|
||||
- name: Build the Docker images
|
||||
run: docker build --no-cache -t quay.io/chaos-kubox/krkn containers/
|
||||
- name: Login in quay
|
||||
if: github.ref == 'refs/heads/main' && github.event_name == 'push'
|
||||
run: docker login quay.io -u ${QUAY_USER} -p ${QUAY_TOKEN}
|
||||
@@ -32,7 +39,7 @@ jobs:
|
||||
run: docker push quay.io/chaos-kubox/krkn
|
||||
- name: Rebuild krkn-hub
|
||||
if: github.ref == 'refs/heads/main' && github.event_name == 'push'
|
||||
uses: chaos-kubox/actions/krkn-hub@main
|
||||
uses: redhat-chaos/actions/krkn-hub@main
|
||||
with:
|
||||
QUAY_USER: ${{ secrets.QUAY_USER_1 }}
|
||||
QUAY_TOKEN: ${{ secrets.QUAY_TOKEN_1 }}
|
||||
|
||||
@@ -1,31 +1,6 @@
|
||||
config:
|
||||
runStrategy:
|
||||
runs: 1
|
||||
maxSecondsBetweenRuns: 30
|
||||
minSecondsBetweenRuns: 1
|
||||
scenarios:
|
||||
- name: "delete hello pods"
|
||||
steps:
|
||||
- podAction:
|
||||
matches:
|
||||
- labels:
|
||||
namespace: "default"
|
||||
selector: "hello-openshift"
|
||||
filters:
|
||||
- randomSample:
|
||||
size: 1
|
||||
actions:
|
||||
- kill:
|
||||
probability: 1
|
||||
force: true
|
||||
- podAction:
|
||||
matches:
|
||||
- labels:
|
||||
namespace: "default"
|
||||
selector: "hello-openshift"
|
||||
retries:
|
||||
retriesTimeout:
|
||||
timeout: 180
|
||||
actions:
|
||||
- checkPodCount:
|
||||
count: 1
|
||||
# yaml-language-server: $schema=../../scenarios/plugin.schema.json
|
||||
- id: kill-pods
|
||||
config:
|
||||
label_selector: name=hello-openshift
|
||||
namespace_pattern: ^default$
|
||||
kill: 1
|
||||
|
||||
+6
-7
@@ -11,15 +11,14 @@ kraken:
|
||||
chaos_scenarios: # List of policies/chaos scenarios to load
|
||||
- container_scenarios: # List of chaos pod scenarios to load
|
||||
- - scenarios/openshift/container_etcd.yml
|
||||
- pod_scenarios:
|
||||
- - scenarios/openshift/etcd.yml
|
||||
- - scenarios/openshift/regex_openshift_pod_kill.yml
|
||||
- scenarios/openshift/post_action_regex.py
|
||||
- plugin_scenarios:
|
||||
- scenarios/openshift/etcd.yml
|
||||
- scenarios/openshift/regex_openshift_pod_kill.yml
|
||||
- node_scenarios: # List of chaos node scenarios to load
|
||||
- scenarios/openshift/node_scenarios_example.yml
|
||||
- pod_scenarios:
|
||||
- - scenarios/openshift/openshift-apiserver.yml
|
||||
- - scenarios/openshift/openshift-kube-apiserver.yml
|
||||
- plugin_scenarios:
|
||||
- scenarios/openshift/openshift-apiserver.yml
|
||||
- scenarios/openshift/openshift-kube-apiserver.yml
|
||||
- time_scenarios: # List of chaos time scenarios to load
|
||||
- scenarios/openshift/time_scenarios_example.yml
|
||||
- litmus_scenarios: # List of litmus scenarios to load
|
||||
|
||||
@@ -11,8 +11,8 @@ kraken:
|
||||
chaos_scenarios: # List of policies/chaos scenarios to load
|
||||
- container_scenarios: # List of chaos pod scenarios to load
|
||||
- - scenarios/kube/container_dns.yml
|
||||
- pod_scenarios:
|
||||
- - scenarios/kube/scheduler.yml
|
||||
- plugin_scenarios:
|
||||
- scenarios/kube/scheduler.yml
|
||||
|
||||
cerberus:
|
||||
cerberus_enabled: False # Enable it when cerberus is previously installed
|
||||
|
||||
@@ -9,15 +9,14 @@ kraken:
|
||||
litmus_uninstall: False # If you want to uninstall litmus if failure
|
||||
litmus_uninstall_before_run: True # If you want to uninstall litmus before a new run starts
|
||||
chaos_scenarios: # List of policies/chaos scenarios to load
|
||||
- pod_scenarios: # List of chaos pod scenarios to load
|
||||
- - scenarios/openshift/etcd.yml
|
||||
- - scenarios/openshift/regex_openshift_pod_kill.yml
|
||||
- scenarios/openshift/post_action_regex.py
|
||||
- plugin_scenarios: # List of chaos pod scenarios to load
|
||||
- scenarios/openshift/etcd.yml
|
||||
- scenarios/openshift/regex_openshift_pod_kill.yml
|
||||
- node_scenarios: # List of chaos node scenarios to load
|
||||
- scenarios/openshift/node_scenarios_example.yml
|
||||
- pod_scenarios:
|
||||
- - scenarios/openshift/openshift-apiserver.yml
|
||||
- - scenarios/openshift/openshift-kube-apiserver.yml
|
||||
- plugin_scenarios:
|
||||
- scenarios/openshift/openshift-apiserver.yml
|
||||
- scenarios/openshift/openshift-kube-apiserver.yml
|
||||
- time_scenarios: # List of chaos time scenarios to load
|
||||
- scenarios/openshift/time_scenarios_example.yml
|
||||
- litmus_scenarios: # List of litmus scenarios to load
|
||||
|
||||
+11
-37
@@ -8,45 +8,19 @@ You can either copy an existing yaml file and make it your own, or fill in one o
|
||||
#### Pod Scenario Yaml Template
|
||||
For example, for adding a pod level scenario for a new application, refer to the sample scenario below to know what fields are necessary and what to add in each location:
|
||||
```
|
||||
config:
|
||||
runStrategy:
|
||||
runs: <number of times to execute the scenario>
|
||||
#This will choose a random number to wait between min and max
|
||||
maxSecondsBetweenRuns: 30
|
||||
minSecondsBetweenRuns: 1
|
||||
scenarios:
|
||||
- name: "delete pods example"
|
||||
steps:
|
||||
- podAction:
|
||||
matches:
|
||||
- labels:
|
||||
namespace: "<namespace>"
|
||||
selector: "<pod label>" # This can be left blank.
|
||||
filters:
|
||||
- randomSample:
|
||||
size: <number of pods to kill>
|
||||
actions:
|
||||
- kill:
|
||||
probability: 1
|
||||
force: true
|
||||
- podAction:
|
||||
matches:
|
||||
- labels:
|
||||
namespace: "<namespace>"
|
||||
selector: "<pod label>" # This can be left blank.
|
||||
retries:
|
||||
retriesTimeout:
|
||||
# Amount of time to wait with retrying, before failing if pod count does not match expected
|
||||
# timeout: 180.
|
||||
|
||||
actions:
|
||||
- checkPodCount:
|
||||
count: <expected number of pods that match namespace and label"
|
||||
# yaml-language-server: $schema=../plugin.schema.json
|
||||
- id: kill-pods
|
||||
config:
|
||||
namespace_pattern: ^<namespace>$
|
||||
label_selector: <pod label>
|
||||
kill: <number of pods to kill>
|
||||
- id: wait-for-pods
|
||||
config:
|
||||
namespace_pattern: ^<namespace>$
|
||||
label_selector: <pod label>
|
||||
count: <expected number of pods that match namespace and label>
|
||||
```
|
||||
|
||||
More information on specific items that you can add to the pod killing scenarios can be found in the [powerfulseal policies](https://powerfulseal.github.io/powerfulseal/policies) documentation
|
||||
|
||||
|
||||
#### Node Scenario Yaml Template
|
||||
|
||||
```
|
||||
|
||||
+35
-9
@@ -1,14 +1,40 @@
|
||||
### Pod Scenarios
|
||||
Kraken consumes [Powerfulseal](https://github.com/powerfulseal/powerfulseal) under the hood to run the pod scenarios.
|
||||
These scenarios are in a simple yaml format that you can manipulate to run your specific tests or use the pre-existing scenarios to see how it works.
|
||||
|
||||
Krkn recently replaced PowerfulSeal with its own internal pod scenarios using a plugin system. You can run pod scenarios by adding the following config to Krkn:
|
||||
|
||||
```yaml
|
||||
kraken:
|
||||
chaos_scenarios:
|
||||
- plugin_scenarios:
|
||||
- path/to/scenario.yaml
|
||||
```
|
||||
|
||||
You can then create the scenario file with the following contents:
|
||||
|
||||
```yaml
|
||||
# yaml-language-server: $schema=../plugin.schema.json
|
||||
- id: kill-pods
|
||||
config:
|
||||
namespace_pattern: ^kube-system$
|
||||
label_selector: k8s-app=kube-scheduler
|
||||
- id: wait-for-pods
|
||||
config:
|
||||
namespace_pattern: ^kube-system$
|
||||
label_selector: k8s-app=kube-scheduler
|
||||
count: 3
|
||||
```
|
||||
|
||||
Please adjust the schema reference to point to the [schema file](../scenarios/plugin.schema.json). This file will give you code completion and documentation for the available options in your IDE.
|
||||
|
||||
#### Pod Chaos Scenarios
|
||||
|
||||
The following are the components of Kubernetes/OpenShift for which a basic chaos scenario config exists today.
|
||||
|
||||
Component | Description | Working
|
||||
------------------------ |----------------------------------------------------------------------------------------------| ------------------------- |
|
||||
[Etcd](https://github.com/chaos-kubox/krkn/blob/main/scenarios/etcd.yml) | Kills a single/multiple etcd replicas for the specified number of times in a loop. | :heavy_check_mark: |
|
||||
[Kube ApiServer](https://github.com/chaos-kubox/krkn/blob/main/scenarios/openshift-kube-apiserver.yml) | Kills a single/multiple kube-apiserver replicas for the specified number of times in a loop. | :heavy_check_mark: |
|
||||
[ApiServer](https://github.com/chaos-kubox/krkn/blob/main/scenarios/openshift-apiserver.yml) | Kills a single/multiple apiserver replicas for the specified number of times in a loop. | :heavy_check_mark: |
|
||||
[Prometheus](https://github.com/chaos-kubox/krkn/blob/main/scenarios/prometheus.yml) | Kills a single/multiple prometheus replicas for the specified number of times in a loop. | :heavy_check_mark: |
|
||||
[OpenShift System Pods](https://github.com/chaos-kubox/krkn/blob/main/scenarios/regex_openshift_pod_kill.yml) | Kills random pods running in the OpenShift system namespaces. | :heavy_check_mark: |
|
||||
| Component | Description | Working |
|
||||
| ------------------------ |-------------| -------- |
|
||||
| [Basic pod scenario](../scenarios/kube/pod.yml) | Kill a pod. | :heavy_check_mark: |
|
||||
| [Etcd](../scenarios/openshift/etcd.yml) | Kills a single/multiple etcd replicas. | :heavy_check_mark: |
|
||||
| [Kube ApiServer](../scenarios/openshift/openshift-kube-apiserver.yml)| Kills a single/multiple kube-apiserver replicas. | :heavy_check_mark: |
|
||||
| [ApiServer](../scenarios/openshift/openshift-apiserver.yml) | Kills a single/multiple apiserver replicas. | :heavy_check_mark: |
|
||||
| [Prometheus](../scenarios/openshift/prometheus.yml) | Kills a single/multiple prometheus replicas. | :heavy_check_mark: |
|
||||
| [OpenShift System Pods](../scenarios/openshift/regex_openshift_pod_kill.yml) | Kills random pods running in the OpenShift system namespaces. | :heavy_check_mark: |
|
||||
|
||||
@@ -0,0 +1,169 @@
|
||||
import dataclasses
|
||||
import json
|
||||
import logging
|
||||
from os.path import abspath
|
||||
from typing import List, Dict
|
||||
|
||||
from arcaflow_plugin_sdk import schema, serialization, jsonschema
|
||||
|
||||
from kraken.plugins.pod_plugin import kill_pods, wait_for_pods
|
||||
from kraken.plugins.run_python_plugin import run_python_file
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class PluginStep:
|
||||
schema: schema.StepSchema
|
||||
error_output_ids: List[str]
|
||||
|
||||
def render_output(self, output_id: str, output_data) -> str:
|
||||
return json.dumps({
|
||||
"output_id": output_id,
|
||||
"output_data": self.schema.outputs[output_id].serialize(output_data),
|
||||
}, indent='\t')
|
||||
|
||||
|
||||
class Plugins:
|
||||
"""
|
||||
Plugins is a class that can run plugins sequentially. The output is rendered to the standard output and the process
|
||||
is aborted if a step fails.
|
||||
"""
|
||||
steps_by_id: Dict[str, PluginStep]
|
||||
|
||||
def __init__(self, steps: List[PluginStep]):
|
||||
self.steps_by_id = dict()
|
||||
for step in steps:
|
||||
if step.schema.id in self.steps_by_id:
|
||||
raise Exception(
|
||||
"Duplicate step ID: {}".format(step.schema.id)
|
||||
)
|
||||
self.steps_by_id[step.schema.id] = step
|
||||
|
||||
def run(self, file: str, kubeconfig_path: str):
|
||||
"""
|
||||
Run executes a series of steps
|
||||
"""
|
||||
data = serialization.load_from_file(abspath(file))
|
||||
if not isinstance(data, list):
|
||||
raise Exception(
|
||||
"Invalid scenario configuration file: {} expected list, found {}".format(file, type(data).__name__)
|
||||
)
|
||||
i = 0
|
||||
for entry in data:
|
||||
if not isinstance(entry, dict):
|
||||
raise Exception(
|
||||
"Invalid scenario configuration file: {} expected a list of dict's, found {} on step {}".format(
|
||||
file,
|
||||
type(entry).__name__,
|
||||
i
|
||||
)
|
||||
)
|
||||
if "id" not in entry:
|
||||
raise Exception(
|
||||
"Invalid scenario configuration file: {} missing 'id' field on step {}".format(
|
||||
file,
|
||||
i,
|
||||
)
|
||||
)
|
||||
if "config" not in entry:
|
||||
raise Exception(
|
||||
"Invalid scenario configuration file: {} missing 'config' field on step {}".format(
|
||||
file,
|
||||
i,
|
||||
)
|
||||
)
|
||||
|
||||
if entry["id"] not in self.steps_by_id:
|
||||
raise Exception(
|
||||
"Invalid step {} in {} ID: {} expected one of: {}".format(
|
||||
i,
|
||||
file,
|
||||
entry["id"],
|
||||
', '.join(self.steps_by_id.keys())
|
||||
)
|
||||
)
|
||||
step = self.steps_by_id[entry["id"]]
|
||||
unserialized_input = step.schema.input.unserialize(entry["config"])
|
||||
if "kubeconfig_path" in step.schema.input.properties:
|
||||
unserialized_input.kubeconfig_path = kubeconfig_path
|
||||
output_id, output_data = step.schema(unserialized_input)
|
||||
logging.info(step.render_output(output_id, output_data) + "\n")
|
||||
if output_id in step.error_output_ids:
|
||||
raise Exception(
|
||||
"Step {} in {} ({}) failed".format(i, file, step.schema.id)
|
||||
)
|
||||
i = i + 1
|
||||
|
||||
def json_schema(self):
|
||||
"""
|
||||
This function generates a JSON schema document and renders it from the steps passed.
|
||||
"""
|
||||
result = {
|
||||
"$id": "https://github.com/chaos-kubox/krkn/",
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"title": "Kraken Arcaflow scenarios",
|
||||
"description": "Serial execution of Arcaflow Python plugins. See https://github.com/arcaflow for details.",
|
||||
"type": "array",
|
||||
"minContains": 1,
|
||||
"items": {
|
||||
"oneOf": [
|
||||
|
||||
]
|
||||
}
|
||||
}
|
||||
for step_id in self.steps_by_id.keys():
|
||||
step = self.steps_by_id[step_id]
|
||||
step_input = jsonschema.step_input(step.schema)
|
||||
del step_input["$id"]
|
||||
del step_input["$schema"]
|
||||
del step_input["title"]
|
||||
del step_input["description"]
|
||||
result["items"]["oneOf"].append({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string",
|
||||
"const": step_id,
|
||||
},
|
||||
"config": step_input,
|
||||
},
|
||||
"required": [
|
||||
"id",
|
||||
"config",
|
||||
]
|
||||
})
|
||||
return json.dumps(result, indent="\t")
|
||||
|
||||
|
||||
PLUGINS = Plugins(
|
||||
[
|
||||
PluginStep(
|
||||
kill_pods,
|
||||
[
|
||||
"error",
|
||||
]
|
||||
),
|
||||
PluginStep(
|
||||
wait_for_pods,
|
||||
[
|
||||
"error"
|
||||
]
|
||||
),
|
||||
PluginStep(
|
||||
run_python_file,
|
||||
[
|
||||
"error"
|
||||
]
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def run(scenarios: List[str], kubeconfig_path: str, failed_post_scenarios: List[str]) -> List[str]:
|
||||
for scenario in scenarios:
|
||||
try:
|
||||
PLUGINS.run(scenario, kubeconfig_path)
|
||||
except Exception as e:
|
||||
failed_post_scenarios.append(scenario)
|
||||
logging.error("Error while running {}: {}".format(scenario, e))
|
||||
return failed_post_scenarios
|
||||
return failed_post_scenarios
|
||||
@@ -0,0 +1,4 @@
|
||||
from kraken.plugins import PLUGINS
|
||||
|
||||
if __name__ == "__main__":
|
||||
print(PLUGINS.json_schema())
|
||||
Executable
+269
@@ -0,0 +1,269 @@
|
||||
#!/usr/bin/env python
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import typing
|
||||
from dataclasses import dataclass, field
|
||||
import random
|
||||
from datetime import datetime
|
||||
from traceback import format_exc
|
||||
|
||||
from kubernetes import config, client
|
||||
from kubernetes.client import V1PodList, V1Pod, ApiException, V1DeleteOptions
|
||||
from arcaflow_plugin_sdk import validation, plugin, schema
|
||||
|
||||
|
||||
def setup_kubernetes(kubeconfig_path):
|
||||
if kubeconfig_path is None:
|
||||
kubeconfig_path = config.KUBE_CONFIG_DEFAULT_LOCATION
|
||||
kubeconfig = config.kube_config.KubeConfigMerger(kubeconfig_path)
|
||||
|
||||
if kubeconfig.config is None:
|
||||
raise Exception(
|
||||
'Invalid kube-config file: %s. '
|
||||
'No configuration found.' % kubeconfig_path
|
||||
)
|
||||
loader = config.kube_config.KubeConfigLoader(
|
||||
config_dict=kubeconfig.config,
|
||||
)
|
||||
client_config = client.Configuration()
|
||||
loader.load_and_set(client_config)
|
||||
return client.ApiClient(configuration=client_config)
|
||||
|
||||
|
||||
def _find_pods(core_v1, label_selector, name_pattern, namespace_pattern):
|
||||
pods: typing.List[V1Pod] = []
|
||||
_continue = None
|
||||
finished = False
|
||||
while not finished:
|
||||
pod_response: V1PodList = core_v1.list_pod_for_all_namespaces(
|
||||
watch=False,
|
||||
label_selector=label_selector
|
||||
)
|
||||
for pod in pod_response.items:
|
||||
pod: V1Pod
|
||||
if (name_pattern is None or name_pattern.match(pod.metadata.name)) and \
|
||||
namespace_pattern.match(pod.metadata.namespace):
|
||||
pods.append(pod)
|
||||
_continue = pod_response.metadata._continue
|
||||
if _continue is None:
|
||||
finished = True
|
||||
return pods
|
||||
|
||||
|
||||
@dataclass
|
||||
class Pod:
|
||||
namespace: str
|
||||
name: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class PodKillSuccessOutput:
|
||||
pods: typing.Dict[int, Pod] = field(metadata={
|
||||
"name": "Pods removed",
|
||||
"description": "Map between timestamps and the pods removed. The timestamp is provided in nanoseconds."
|
||||
})
|
||||
|
||||
|
||||
@dataclass
|
||||
class PodWaitSuccessOutput:
|
||||
pods: typing.List[Pod] = field(metadata={
|
||||
"name": "Pods",
|
||||
"description": "List of pods that have been found to run."
|
||||
})
|
||||
|
||||
|
||||
@dataclass
|
||||
class PodErrorOutput:
|
||||
error: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class KillPodConfig:
|
||||
"""
|
||||
This is a configuration structure specific to pod kill scenario. It describes which pod from which
|
||||
namespace(s) to select for killing and how many pods to kill.
|
||||
"""
|
||||
|
||||
namespace_pattern: re.Pattern = field(metadata={
|
||||
"name": "Namespace pattern",
|
||||
"description": "Regular expression for target pod namespaces."
|
||||
})
|
||||
|
||||
name_pattern: typing.Annotated[
|
||||
typing.Optional[re.Pattern],
|
||||
validation.required_if_not("label_selector")
|
||||
] = field(default=None, metadata={
|
||||
"name": "Name pattern",
|
||||
"description": "Regular expression for target pods. Required if label_selector is not set."
|
||||
})
|
||||
|
||||
kill: typing.Annotated[int, validation.min(1)] = field(
|
||||
default=1,
|
||||
metadata={"name": "Number of pods to kill", "description": "How many pods should we attempt to kill?"}
|
||||
)
|
||||
|
||||
label_selector: typing.Annotated[
|
||||
typing.Optional[str],
|
||||
validation.min(1),
|
||||
validation.required_if_not("name_pattern")
|
||||
] = field(default=None, metadata={
|
||||
"name": "Label selector",
|
||||
"description": "Kubernetes label selector for the target pods. Required if name_pattern is not set.\n"
|
||||
"See https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ for details."
|
||||
})
|
||||
|
||||
kubeconfig_path: typing.Optional[str] = field(default=None, metadata={
|
||||
"name": "Kubeconfig path",
|
||||
"description": "Path to your Kubeconfig file. Defaults to ~/.kube/config.\n"
|
||||
"See https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/ for "
|
||||
"details."
|
||||
})
|
||||
|
||||
timeout: int = field(default=180, metadata={
|
||||
"name": "Timeout",
|
||||
"description": "Timeout to wait for the target pod(s) to be removed in seconds."
|
||||
})
|
||||
|
||||
backoff: int = field(default=1, metadata={
|
||||
"name": "Backoff",
|
||||
"description": "How many seconds to wait between checks for the target pod status."
|
||||
})
|
||||
|
||||
|
||||
@plugin.step(
|
||||
"kill-pods",
|
||||
"Kill pods",
|
||||
"Kill pods as specified by parameters",
|
||||
{"success": PodKillSuccessOutput, "error": PodErrorOutput}
|
||||
)
|
||||
def kill_pods(cfg: KillPodConfig) -> typing.Tuple[str, typing.Union[PodKillSuccessOutput, PodErrorOutput]]:
|
||||
try:
|
||||
with setup_kubernetes(None) as cli:
|
||||
core_v1 = client.CoreV1Api(cli)
|
||||
|
||||
# region Select target pods
|
||||
pods = _find_pods(core_v1, cfg.label_selector, cfg.name_pattern, cfg.namespace_pattern)
|
||||
if len(pods) < cfg.kill:
|
||||
return "error", PodErrorOutput(
|
||||
"Not enough pods match the criteria, expected {} but found only {} pods".format(cfg.kill, len(pods))
|
||||
)
|
||||
random.shuffle(pods)
|
||||
# endregion
|
||||
|
||||
# region Remove pods
|
||||
killed_pods: typing.Dict[int, Pod] = {}
|
||||
watch_pods: typing.List[Pod] = []
|
||||
for i in range(cfg.kill):
|
||||
pod = pods[i]
|
||||
core_v1.delete_namespaced_pod(pod.metadata.name, pod.metadata.namespace, body=V1DeleteOptions(
|
||||
grace_period_seconds=0,
|
||||
))
|
||||
p = Pod(
|
||||
pod.metadata.namespace,
|
||||
pod.metadata.name
|
||||
)
|
||||
killed_pods[int(time.time_ns())] = p
|
||||
watch_pods.append(p)
|
||||
# endregion
|
||||
|
||||
# region Wait for pods to be removed
|
||||
start_time = time.time()
|
||||
while len(watch_pods) > 0:
|
||||
time.sleep(cfg.backoff)
|
||||
new_watch_pods: typing.List[Pod] = []
|
||||
for p in watch_pods:
|
||||
try:
|
||||
read_pod = core_v1.read_namespaced_pod(p.name, p.namespace)
|
||||
new_watch_pods.append(p)
|
||||
except ApiException as e:
|
||||
if e.status != 404:
|
||||
raise
|
||||
watch_pods = new_watch_pods
|
||||
current_time = time.time()
|
||||
if current_time - start_time > cfg.timeout:
|
||||
return "error", PodErrorOutput("Timeout while waiting for pods to be removed.")
|
||||
return "success", PodKillSuccessOutput(killed_pods)
|
||||
# endregion
|
||||
except Exception:
|
||||
return "error", PodErrorOutput(
|
||||
format_exc()
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class WaitForPodsConfig:
|
||||
"""
|
||||
WaitForPodsConfig is a configuration structure for wait-for-pod steps.
|
||||
"""
|
||||
|
||||
namespace_pattern: re.Pattern
|
||||
|
||||
name_pattern: typing.Annotated[
|
||||
typing.Optional[re.Pattern],
|
||||
validation.required_if_not("label_selector")
|
||||
] = None
|
||||
|
||||
label_selector: typing.Annotated[
|
||||
typing.Optional[str],
|
||||
validation.min(1),
|
||||
validation.required_if_not("name_pattern")
|
||||
] = None
|
||||
|
||||
count: typing.Annotated[int, validation.min(1)] = field(
|
||||
default=1,
|
||||
metadata={"name": "Pod count", "description": "Wait for at least this many pods to exist"}
|
||||
)
|
||||
|
||||
timeout: typing.Annotated[int, validation.min(1)] = field(
|
||||
default=180,
|
||||
metadata={"name": "Timeout", "description": "How many seconds to wait for?"}
|
||||
)
|
||||
|
||||
backoff: int = field(default=1, metadata={
|
||||
"name": "Backoff",
|
||||
"description": "How many seconds to wait between checks for the target pod status."
|
||||
})
|
||||
|
||||
kubeconfig_path: typing.Optional[str] = None
|
||||
|
||||
|
||||
@plugin.step(
|
||||
"wait-for-pods",
|
||||
"Wait for pods",
|
||||
"Wait for the specified number of pods to be present",
|
||||
{"success": PodWaitSuccessOutput, "error": PodErrorOutput}
|
||||
)
|
||||
def wait_for_pods(cfg: WaitForPodsConfig) -> typing.Tuple[str, typing.Union[PodWaitSuccessOutput, PodErrorOutput]]:
|
||||
try:
|
||||
with setup_kubernetes(None) as cli:
|
||||
core_v1 = client.CoreV1Api(cli)
|
||||
|
||||
timeout = False
|
||||
start_time = datetime.now()
|
||||
while not timeout:
|
||||
pods = _find_pods(core_v1, cfg.label_selector, cfg.name_pattern, cfg.namespace_pattern)
|
||||
if len(pods) >= cfg.count:
|
||||
return "success", \
|
||||
PodWaitSuccessOutput(list(map(lambda p: Pod(p.metadata.namespace, p.metadata.name), pods)))
|
||||
|
||||
time.sleep(cfg.backoff)
|
||||
|
||||
now_time = datetime.now()
|
||||
|
||||
time_diff = now_time - start_time
|
||||
if time_diff.seconds > cfg.timeout:
|
||||
return "error", PodErrorOutput(
|
||||
"timeout while waiting for pods to come up"
|
||||
)
|
||||
except Exception:
|
||||
return "error", PodErrorOutput(
|
||||
format_exc()
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(plugin.run(plugin.build_schema(
|
||||
kill_pods,
|
||||
wait_for_pods,
|
||||
)))
|
||||
@@ -0,0 +1,51 @@
|
||||
import dataclasses
|
||||
import io
|
||||
import subprocess
|
||||
import sys
|
||||
import typing
|
||||
|
||||
from arcaflow_plugin_sdk import plugin
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class RunPythonFileInput:
|
||||
filename: str
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class RunPythonFileOutput:
|
||||
stdout: str
|
||||
stderr: str
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class RunPythonFileError:
|
||||
exit_code: int
|
||||
stdout: str
|
||||
stderr: str
|
||||
|
||||
|
||||
@plugin.step(
|
||||
id="run_python",
|
||||
name="Run a Python script",
|
||||
description="Run a specified Python script",
|
||||
outputs={"success": RunPythonFileOutput, "error": RunPythonFileError}
|
||||
)
|
||||
def run_python_file(params: RunPythonFileInput) -> typing.Tuple[
|
||||
str,
|
||||
typing.Union[RunPythonFileOutput, RunPythonFileError]
|
||||
]:
|
||||
run_results = subprocess.run(
|
||||
[sys.executable, params.filename],
|
||||
capture_output=True
|
||||
)
|
||||
if run_results.returncode == 0:
|
||||
return "success", RunPythonFileOutput(
|
||||
str(run_results.stdout, 'utf-8'),
|
||||
str(run_results.stderr, 'utf-8')
|
||||
)
|
||||
return "error", RunPythonFileError(
|
||||
run_results.returncode,
|
||||
str(run_results.stdout, 'utf-8'),
|
||||
str(run_results.stderr, 'utf-8')
|
||||
)
|
||||
@@ -0,0 +1,175 @@
|
||||
import random
|
||||
import re
|
||||
import string
|
||||
import threading
|
||||
import unittest
|
||||
|
||||
from arcaflow_plugin_sdk import plugin
|
||||
from kubernetes.client import V1Pod, V1ObjectMeta, V1PodSpec, V1Container, ApiException
|
||||
|
||||
from kraken.plugins import pod_plugin
|
||||
from kraken.plugins.pod_plugin import setup_kubernetes, KillPodConfig, PodKillSuccessOutput
|
||||
from kubernetes import client
|
||||
|
||||
|
||||
class KillPodTest(unittest.TestCase):
|
||||
def test_serialization(self):
|
||||
plugin.test_object_serialization(
|
||||
pod_plugin.KillPodConfig(
|
||||
namespace_pattern=re.compile(".*"),
|
||||
name_pattern=re.compile(".*")
|
||||
),
|
||||
self.fail,
|
||||
)
|
||||
plugin.test_object_serialization(
|
||||
pod_plugin.PodKillSuccessOutput(
|
||||
pods={}
|
||||
),
|
||||
self.fail,
|
||||
)
|
||||
plugin.test_object_serialization(
|
||||
pod_plugin.PodErrorOutput(
|
||||
error="Hello world!"
|
||||
),
|
||||
self.fail,
|
||||
)
|
||||
|
||||
def test_not_enough_pods(self):
|
||||
name = ''.join(random.choices(string.ascii_lowercase, k=8))
|
||||
output_id, output_data = pod_plugin.kill_pods(KillPodConfig(
|
||||
namespace_pattern=re.compile("^default$"),
|
||||
name_pattern=re.compile("^unit-test-" + re.escape(name) + "$"),
|
||||
))
|
||||
if output_id != "error":
|
||||
self.fail("Not enough pods did not result in an error.")
|
||||
print(output_data.error)
|
||||
|
||||
def test_kill_pod(self):
|
||||
with setup_kubernetes(None) as cli:
|
||||
core_v1 = client.CoreV1Api(cli)
|
||||
pod = core_v1.create_namespaced_pod("default", V1Pod(
|
||||
metadata=V1ObjectMeta(
|
||||
generate_name="test-",
|
||||
),
|
||||
spec=V1PodSpec(
|
||||
containers=[
|
||||
V1Container(
|
||||
name="test",
|
||||
image="alpine",
|
||||
tty=True,
|
||||
)
|
||||
]
|
||||
),
|
||||
))
|
||||
|
||||
def remove_test_pod():
|
||||
try:
|
||||
core_v1.delete_namespaced_pod(pod.metadata.name, pod.metadata.namespace)
|
||||
except ApiException as e:
|
||||
if e.status != 404:
|
||||
raise
|
||||
|
||||
self.addCleanup(remove_test_pod)
|
||||
|
||||
output_id, output_data = pod_plugin.kill_pods(KillPodConfig(
|
||||
namespace_pattern=re.compile("^default$"),
|
||||
name_pattern=re.compile("^" + re.escape(pod.metadata.name) + "$"),
|
||||
))
|
||||
|
||||
if output_id == "error":
|
||||
self.fail(output_data.error)
|
||||
self.assertIsInstance(output_data, PodKillSuccessOutput)
|
||||
out: PodKillSuccessOutput = output_data
|
||||
self.assertEqual(1, len(out.pods))
|
||||
pod_list = list(out.pods.values())
|
||||
self.assertEqual(pod.metadata.name, pod_list[0].name)
|
||||
|
||||
try:
|
||||
core_v1.read_namespaced_pod(pod_list[0].name, pod_list[0].namespace)
|
||||
self.fail("Killed pod is still present.")
|
||||
except ApiException as e:
|
||||
if e.status != 404:
|
||||
self.fail("Incorrect API exception encountered: {}".format(e))
|
||||
|
||||
|
||||
class WaitForPodTest(unittest.TestCase):
|
||||
def test_serialization(self):
|
||||
plugin.test_object_serialization(
|
||||
pod_plugin.WaitForPodsConfig(
|
||||
namespace_pattern=re.compile(".*"),
|
||||
name_pattern=re.compile(".*")
|
||||
),
|
||||
self.fail,
|
||||
)
|
||||
plugin.test_object_serialization(
|
||||
pod_plugin.WaitForPodsConfig(
|
||||
namespace_pattern=re.compile(".*"),
|
||||
label_selector="app=nginx"
|
||||
),
|
||||
self.fail,
|
||||
)
|
||||
plugin.test_object_serialization(
|
||||
pod_plugin.PodWaitSuccessOutput(
|
||||
pods=[]
|
||||
),
|
||||
self.fail,
|
||||
)
|
||||
plugin.test_object_serialization(
|
||||
pod_plugin.PodErrorOutput(
|
||||
error="Hello world!"
|
||||
),
|
||||
self.fail,
|
||||
)
|
||||
|
||||
def test_timeout(self):
|
||||
name = "watch-test-" + ''.join(random.choices(string.ascii_lowercase, k=8))
|
||||
output_id, output_data = pod_plugin.wait_for_pods(pod_plugin.WaitForPodsConfig(
|
||||
namespace_pattern=re.compile("^default$"),
|
||||
name_pattern=re.compile("^" + re.escape(name) + "$"),
|
||||
timeout=1
|
||||
))
|
||||
self.assertEqual("error", output_id)
|
||||
|
||||
def test_watch(self):
|
||||
with setup_kubernetes(None) as cli:
|
||||
core_v1 = client.CoreV1Api(cli)
|
||||
name = "watch-test-" + ''.join(random.choices(string.ascii_lowercase, k=8))
|
||||
|
||||
def create_test_pod():
|
||||
core_v1.create_namespaced_pod("default", V1Pod(
|
||||
metadata=V1ObjectMeta(
|
||||
name=name,
|
||||
),
|
||||
spec=V1PodSpec(
|
||||
containers=[
|
||||
V1Container(
|
||||
name="test",
|
||||
image="alpine",
|
||||
tty=True,
|
||||
)
|
||||
]
|
||||
),
|
||||
))
|
||||
|
||||
def remove_test_pod():
|
||||
try:
|
||||
core_v1.delete_namespaced_pod(name, "default")
|
||||
except ApiException as e:
|
||||
if e.status != 404:
|
||||
raise
|
||||
|
||||
self.addCleanup(remove_test_pod)
|
||||
|
||||
t = threading.Timer(10, create_test_pod)
|
||||
t.start()
|
||||
|
||||
output_id, output_data = pod_plugin.wait_for_pods(pod_plugin.WaitForPodsConfig(
|
||||
namespace_pattern=re.compile("^default$"),
|
||||
name_pattern=re.compile("^" + re.escape(name) + "$"),
|
||||
timeout=60
|
||||
))
|
||||
self.assertEqual("success", output_id)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,28 @@
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from kraken.plugins import run_python_file
|
||||
from kraken.plugins.run_python_plugin import RunPythonFileInput
|
||||
|
||||
|
||||
class RunPythonPluginTest(unittest.TestCase):
|
||||
def test_success_execution(self):
|
||||
tmp_file = tempfile.NamedTemporaryFile()
|
||||
tmp_file.write(bytes("print('Hello world!')", 'utf-8'))
|
||||
tmp_file.flush()
|
||||
output_id, output_data = run_python_file(RunPythonFileInput(tmp_file.name))
|
||||
self.assertEqual("success", output_id)
|
||||
self.assertEqual("Hello world!\n", output_data.stdout)
|
||||
|
||||
def test_error_execution(self):
|
||||
tmp_file = tempfile.NamedTemporaryFile()
|
||||
tmp_file.write(bytes("import sys\nprint('Hello world!')\nsys.exit(42)\n", 'utf-8'))
|
||||
tmp_file.flush()
|
||||
output_id, output_data = run_python_file(RunPythonFileInput(tmp_file.name))
|
||||
self.assertEqual("error", output_id)
|
||||
self.assertEqual(42, output_data.exit_code)
|
||||
self.assertEqual("Hello world!\n", output_data.stdout)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -1,5 +1,8 @@
|
||||
import logging
|
||||
import kraken.invoke.command as runcommand
|
||||
|
||||
from arcaflow_plugin_sdk import serialization
|
||||
from kraken.plugins import pod_plugin
|
||||
|
||||
import kraken.cerberus.setup as cerberus
|
||||
import kraken.post_actions.actions as post_actions
|
||||
import kraken.kubernetes.client as kubecli
|
||||
@@ -20,20 +23,30 @@ def run(kubeconfig_path, scenarios_list, config, failed_post_scenarios, wait_dur
|
||||
try:
|
||||
# capture start time
|
||||
start_time = int(time.time())
|
||||
scenario_logs = runcommand.invoke(
|
||||
"powerfulseal autonomous --use-pod-delete-instead-"
|
||||
"of-ssh-kill --policy-file %s --kubeconfig %s "
|
||||
"--no-cloud --inventory-kubernetes --headless" % (pod_scenario[0], kubeconfig_path)
|
||||
)
|
||||
|
||||
input = serialization.load_from_file(pod_scenario)
|
||||
|
||||
s = pod_plugin.get_schema()
|
||||
input_data: pod_plugin.KillPodConfig = s.unserialize_input("pod", input)
|
||||
|
||||
if kubeconfig_path is not None:
|
||||
input_data.kubeconfig_path = kubeconfig_path
|
||||
|
||||
output_id, output_data = s.call_step("pod", input_data)
|
||||
|
||||
if output_id == "error":
|
||||
data: pod_plugin.PodErrorOutput = output_data
|
||||
logging.error("Failed to run pod scenario: {}".format(data.error))
|
||||
else:
|
||||
data: pod_plugin.PodSuccessOutput = output_data
|
||||
for pod in data.pods:
|
||||
print("Deleted pod {} in namespace {}\n".format(pod.pod_name, pod.pod_namespace))
|
||||
except Exception as e:
|
||||
logging.error(
|
||||
"Failed to run scenario: %s. Encountered the following " "exception: %s" % (pod_scenario[0], e)
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
# Display pod scenario logs/actions
|
||||
print(scenario_logs)
|
||||
|
||||
logging.info("Scenario: %s has been successfully injected!" % (pod_scenario[0]))
|
||||
logging.info("Waiting for the specified duration: %s" % (wait_duration))
|
||||
time.sleep(wait_duration)
|
||||
|
||||
@@ -5,21 +5,7 @@ import kraken.invoke.command as runcommand
|
||||
def run(kubeconfig_path, scenario, pre_action_output=""):
|
||||
|
||||
if scenario.endswith(".yaml") or scenario.endswith(".yml"):
|
||||
action_output = runcommand.invoke(
|
||||
"powerfulseal autonomous "
|
||||
"--use-pod-delete-instead-of-ssh-kill"
|
||||
" --policy-file %s --kubeconfig %s --no-cloud"
|
||||
" --inventory-kubernetes --headless" % (scenario, kubeconfig_path)
|
||||
)
|
||||
# read output to make sure no error
|
||||
if "ERROR" in action_output:
|
||||
action_output.split("ERROR")[1].split("\n")[0]
|
||||
if not pre_action_output:
|
||||
logging.info("Powerful seal pre action check failed for " + str(scenario))
|
||||
return False
|
||||
else:
|
||||
logging.info(scenario + " post action checks passed")
|
||||
|
||||
logging.error("Powerfulseal support has recently been removed. Please switch to using plugins instead.")
|
||||
elif scenario.endswith(".py"):
|
||||
action_output = runcommand.invoke("python3 " + scenario).strip()
|
||||
if pre_action_output:
|
||||
|
||||
+2
-2
@@ -1,14 +1,13 @@
|
||||
datetime
|
||||
pyfiglet
|
||||
PyYAML>=5.1
|
||||
git+https://github.com/powerfulseal/powerfulseal.git@3.3.0
|
||||
requests
|
||||
boto3
|
||||
google-api-python-client
|
||||
azure-mgmt-compute
|
||||
azure-keyvault
|
||||
azure-identity
|
||||
kubernetes==18.20.0
|
||||
kubernetes
|
||||
oauth2client>=4.1.3
|
||||
python-openstackclient
|
||||
gitpython
|
||||
@@ -23,3 +22,4 @@ itsdangerous==2.0.1
|
||||
werkzeug==2.0.3
|
||||
aliyun-python-sdk-core-v3
|
||||
aliyun-python-sdk-ecs
|
||||
arcaflow-plugin-sdk==0.3.0
|
||||
|
||||
+6
-4
@@ -22,6 +22,7 @@ import kraken.application_outage.actions as application_outage
|
||||
import kraken.pvc.pvc_scenario as pvc_scenario
|
||||
import kraken.network_chaos.actions as network_chaos
|
||||
import server as server
|
||||
from kraken import plugins
|
||||
|
||||
|
||||
def publish_kraken_status(status):
|
||||
@@ -158,10 +159,11 @@ def main(cfg):
|
||||
if scenarios_list:
|
||||
# Inject pod chaos scenarios specified in the config
|
||||
if scenario_type == "pod_scenarios":
|
||||
logging.info("Running pod scenarios")
|
||||
failed_post_scenarios = pod_scenarios.run(
|
||||
kubeconfig_path, scenarios_list, config, failed_post_scenarios, wait_duration
|
||||
)
|
||||
logging.error("Pod scenarios have been removed, please use plugin_scenarios with the "
|
||||
"kill-pods configuration instead.")
|
||||
sys.exit(1)
|
||||
elif scenario_type == "plugin_scenarios":
|
||||
failed_post_scenarios = plugins.run(scenarios_list, kubeconfig_path, failed_post_scenarios)
|
||||
elif scenario_type == "container_scenarios":
|
||||
logging.info("Running container scenarios")
|
||||
failed_post_scenarios = pod_scenarios.container_run(
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
# yaml-language-server: $schema=../plugin.schema.json
|
||||
- id: kill-pods
|
||||
config:
|
||||
name_pattern: ^nginx-.*$
|
||||
namespace_pattern: ^default$
|
||||
kill: 1
|
||||
@@ -1,32 +1,10 @@
|
||||
config:
|
||||
runStrategy:
|
||||
runs: 1
|
||||
maxSecondsBetweenRuns: 30
|
||||
minSecondsBetweenRuns: 1
|
||||
scenarios:
|
||||
- name: "delete scheduler pods"
|
||||
steps:
|
||||
- podAction:
|
||||
matches:
|
||||
- labels:
|
||||
namespace: "kube-system"
|
||||
selector: "k8s-app=kube-scheduler"
|
||||
filters:
|
||||
- randomSample:
|
||||
size: 1
|
||||
actions:
|
||||
- kill:
|
||||
probability: 1
|
||||
force: true
|
||||
- podAction:
|
||||
matches:
|
||||
- labels:
|
||||
namespace: "kube-system"
|
||||
selector: "k8s-app=kube-scheduler"
|
||||
retries:
|
||||
retriesTimeout:
|
||||
timeout: 180
|
||||
|
||||
actions:
|
||||
- checkPodCount:
|
||||
count: 3
|
||||
# yaml-language-server: $schema=../plugin.schema.json
|
||||
- id: kill-pods
|
||||
config:
|
||||
namespace_pattern: ^kube-system$
|
||||
label_selector: k8s-app=kube-scheduler
|
||||
- id: wait-for-pods
|
||||
config:
|
||||
namespace_pattern: ^kube-system$
|
||||
label_selector: k8s-app=kube-scheduler
|
||||
count: 3
|
||||
|
||||
@@ -1,32 +1,10 @@
|
||||
config:
|
||||
runStrategy:
|
||||
runs: 1
|
||||
maxSecondsBetweenRuns: 30
|
||||
minSecondsBetweenRuns: 1
|
||||
scenarios:
|
||||
- name: "delete acme-air pods"
|
||||
steps:
|
||||
- podAction:
|
||||
matches:
|
||||
- labels:
|
||||
namespace: "acme-air"
|
||||
selector: ""
|
||||
filters:
|
||||
- randomSample:
|
||||
size: 1
|
||||
actions:
|
||||
- kill:
|
||||
probability: 1
|
||||
force: true
|
||||
- podAction:
|
||||
matches:
|
||||
- labels:
|
||||
namespace: "acme-air"
|
||||
selector: ""
|
||||
retries:
|
||||
retriesTimeout:
|
||||
timeout: 180
|
||||
|
||||
actions:
|
||||
- checkPodCount:
|
||||
count: 8
|
||||
# yaml-language-server: $schema=../plugin.schema.json
|
||||
- id: kill-pods
|
||||
config:
|
||||
namespace_pattern: ^acme-air$
|
||||
name_pattern: .*
|
||||
- id: wait-for-pods
|
||||
config:
|
||||
namespace_pattern: ^acme-air$
|
||||
name_pattern: .*
|
||||
count: 8
|
||||
@@ -1,32 +1,10 @@
|
||||
config:
|
||||
runStrategy:
|
||||
runs: 1
|
||||
maxSecondsBetweenRuns: 30
|
||||
minSecondsBetweenRuns: 1
|
||||
scenarios:
|
||||
- name: "delete etcd pods"
|
||||
steps:
|
||||
- podAction:
|
||||
matches:
|
||||
- labels:
|
||||
namespace: "openshift-etcd"
|
||||
selector: "k8s-app=etcd"
|
||||
filters:
|
||||
- randomSample:
|
||||
size: 1
|
||||
actions:
|
||||
- kill:
|
||||
probability: 1
|
||||
force: true
|
||||
- podAction:
|
||||
matches:
|
||||
- labels:
|
||||
namespace: "openshift-etcd"
|
||||
selector: "k8s-app=etcd"
|
||||
retries:
|
||||
retriesTimeout:
|
||||
timeout: 180
|
||||
|
||||
actions:
|
||||
- checkPodCount:
|
||||
count: 3
|
||||
# yaml-language-server: $schema=../plugin.schema.json
|
||||
- id: kill-pods
|
||||
config:
|
||||
namespace_pattern: ^openshift-etcd$
|
||||
label_selector: k8s-app=etcd
|
||||
- id: wait-for-pods
|
||||
config:
|
||||
namespace_pattern: ^openshift-etcd$
|
||||
label_selector: k8s-app=etcd
|
||||
count: 3
|
||||
|
||||
@@ -1,35 +1,10 @@
|
||||
config:
|
||||
runStrategy:
|
||||
runs: 1
|
||||
maxSecondsBetweenRuns: 30
|
||||
minSecondsBetweenRuns: 1
|
||||
scenarios:
|
||||
- name: "delete openshift-apiserver pods"
|
||||
steps:
|
||||
- podAction:
|
||||
matches:
|
||||
- labels:
|
||||
namespace: "openshift-apiserver"
|
||||
selector: "app=openshift-apiserver-a"
|
||||
|
||||
filters:
|
||||
- randomSample:
|
||||
size: 1
|
||||
|
||||
# The actions will be executed in the order specified
|
||||
actions:
|
||||
- kill:
|
||||
probability: 1
|
||||
force: true
|
||||
- podAction:
|
||||
matches:
|
||||
- labels:
|
||||
namespace: "openshift-apiserver"
|
||||
selector: "app=openshift-apiserver-a"
|
||||
retries:
|
||||
retriesTimeout:
|
||||
timeout: 180
|
||||
|
||||
actions:
|
||||
- checkPodCount:
|
||||
count: 3
|
||||
# yaml-language-server: $schema=../plugin.schema.json
|
||||
- id: kill-pods
|
||||
config:
|
||||
namespace_pattern: ^openshift-apiserver$
|
||||
label_selector: app=openshift-apiserver-a
|
||||
- id: wait-for-pods
|
||||
config:
|
||||
namespace_pattern: ^openshift-apiserver$
|
||||
label_selector: app=openshift-apiserver-a
|
||||
count: 3
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
# yaml-language-server: $schema=../pod.schema.json
|
||||
namespace_pattern: openshift-kube-apiserver
|
||||
kill: 1
|
||||
|
||||
config:
|
||||
runStrategy:
|
||||
runs: 1
|
||||
|
||||
@@ -1,21 +1,10 @@
|
||||
config:
|
||||
runStrategy:
|
||||
runs: 1
|
||||
maxSecondsBetweenRuns: 10
|
||||
minSecondsBetweenRuns: 1
|
||||
scenarios:
|
||||
- name: "check 2 pods are in namespace with selector: prometheus"
|
||||
steps:
|
||||
- podAction:
|
||||
matches:
|
||||
- labels:
|
||||
namespace: "openshift-monitoring"
|
||||
selector: "app=prometheus"
|
||||
filters:
|
||||
- property:
|
||||
name: "state"
|
||||
value: "Running"
|
||||
# The actions will be executed in the order specified
|
||||
actions:
|
||||
- checkPodCount:
|
||||
count: 2
|
||||
# yaml-language-server: $schema=../plugin.schema.json
|
||||
- id: kill-pods
|
||||
config:
|
||||
namespace_pattern: ^openshift-monitoring$
|
||||
label_selector: app=prometheus
|
||||
- id: wait-for-pods
|
||||
config:
|
||||
namespace_pattern: ^openshift-monitoring$
|
||||
label_selector: app=prometheus
|
||||
count: 2
|
||||
@@ -1,35 +1,11 @@
|
||||
config:
|
||||
runStrategy:
|
||||
runs: 1
|
||||
maxSecondsBetweenRuns: 30
|
||||
minSecondsBetweenRuns: 1
|
||||
scenarios:
|
||||
- name: "delete prometheus pods"
|
||||
steps:
|
||||
- podAction:
|
||||
matches:
|
||||
- labels:
|
||||
namespace: "openshift-monitoring"
|
||||
selector: "app=prometheus"
|
||||
|
||||
filters:
|
||||
- randomSample:
|
||||
size: 1
|
||||
|
||||
# The actions will be executed in the order specified
|
||||
actions:
|
||||
- kill:
|
||||
probability: 1
|
||||
force: true
|
||||
- podAction:
|
||||
matches:
|
||||
- labels:
|
||||
namespace: "openshift-monitoring"
|
||||
selector: "app=prometheus"
|
||||
retries:
|
||||
retriesTimeout:
|
||||
timeout: 180
|
||||
|
||||
actions:
|
||||
- checkPodCount:
|
||||
count: 2
|
||||
# yaml-language-server: $schema=../plugin.schema.json
|
||||
- id: kill-pods
|
||||
config:
|
||||
namespace_pattern: ^openshift-monitoring$
|
||||
label_selector: app=prometheus
|
||||
- id: wait-for-pods
|
||||
config:
|
||||
namespace_pattern: ^openshift-monitoring$
|
||||
label_selector: app=prometheus
|
||||
count: 2
|
||||
timeout: 180
|
||||
@@ -1,20 +1,6 @@
|
||||
config:
|
||||
runStrategy:
|
||||
runs: 1
|
||||
maxSecondsBetweenRuns: 30
|
||||
minSecondsBetweenRuns: 1
|
||||
scenarios:
|
||||
- name: kill up to 3 pods in any openshift namespace
|
||||
steps:
|
||||
- podAction:
|
||||
matches:
|
||||
- namespace: "openshift-.*"
|
||||
filters:
|
||||
- property:
|
||||
name: "state"
|
||||
value: "Running"
|
||||
- randomSample:
|
||||
size: 3
|
||||
actions:
|
||||
- kill:
|
||||
probability: .7
|
||||
# yaml-language-server: $schema=../plugin.schema.json
|
||||
- id: kill-pods
|
||||
config:
|
||||
namespace_pattern: ^openshift-.*$
|
||||
name_pattern: .*
|
||||
kill: 3
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
This file is generated by running the "plugins" module in the kraken project:
|
||||
|
||||
```
|
||||
python -m kraken.plugins >scenarios/plugin.schema.json
|
||||
```
|
||||
@@ -0,0 +1,157 @@
|
||||
{
|
||||
"$id": "https://github.com/chaos-kubox/krkn/",
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"title": "Kraken Arcaflow scenarios",
|
||||
"description": "Serial execution of Arcaflow Python plugins. See https://github.com/arcaflow for details.",
|
||||
"type": "array",
|
||||
"minContains": 1,
|
||||
"items": {
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string",
|
||||
"const": "kill-pods"
|
||||
},
|
||||
"config": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"namespace_pattern": {
|
||||
"type": "string",
|
||||
"format": "regex",
|
||||
"title": "Namespace pattern",
|
||||
"description": "Regular expression for target pod namespaces."
|
||||
},
|
||||
"name_pattern": {
|
||||
"type": "string",
|
||||
"format": "regex",
|
||||
"title": "Name pattern",
|
||||
"description": "Regular expression for target pods. Required if label_selector is not set."
|
||||
},
|
||||
"kill": {
|
||||
"type": "integer",
|
||||
"minimum": 1,
|
||||
"title": "Number of pods to kill",
|
||||
"description": "How many pods should we attempt to kill?"
|
||||
},
|
||||
"label_selector": {
|
||||
"type": "string",
|
||||
"minLength": 1,
|
||||
"title": "Label selector",
|
||||
"description": "Kubernetes label selector for the target pods. Required if name_pattern is not set.\nSee https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ for details."
|
||||
},
|
||||
"kubeconfig_path": {
|
||||
"type": "string",
|
||||
"title": "Kubeconfig path",
|
||||
"description": "Path to your Kubeconfig file. Defaults to ~/.kube/config.\nSee https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/ for details."
|
||||
},
|
||||
"timeout": {
|
||||
"type": "integer",
|
||||
"title": "Timeout",
|
||||
"description": "Timeout to wait for the target pod(s) to be removed in seconds."
|
||||
},
|
||||
"backoff": {
|
||||
"type": "integer",
|
||||
"title": "Backoff",
|
||||
"description": "How many seconds to wait between checks for the target pod status."
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"namespace_pattern"
|
||||
]
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"id",
|
||||
"config"
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string",
|
||||
"const": "wait-for-pods"
|
||||
},
|
||||
"config": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"namespace_pattern": {
|
||||
"type": "string",
|
||||
"format": "regex",
|
||||
"title": "namespace_pattern"
|
||||
},
|
||||
"name_pattern": {
|
||||
"type": "string",
|
||||
"format": "regex",
|
||||
"title": "name_pattern"
|
||||
},
|
||||
"label_selector": {
|
||||
"type": "string",
|
||||
"minLength": 1,
|
||||
"title": "label_selector"
|
||||
},
|
||||
"count": {
|
||||
"type": "integer",
|
||||
"minimum": 1,
|
||||
"title": "Pod count",
|
||||
"description": "Wait for at least this many pods to exist"
|
||||
},
|
||||
"timeout": {
|
||||
"type": "integer",
|
||||
"minimum": 1,
|
||||
"title": "Timeout",
|
||||
"description": "How many seconds to wait for?"
|
||||
},
|
||||
"backoff": {
|
||||
"type": "integer",
|
||||
"title": "Backoff",
|
||||
"description": "How many seconds to wait between checks for the target pod status."
|
||||
},
|
||||
"kubeconfig_path": {
|
||||
"type": "string",
|
||||
"title": "kubeconfig_path"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"namespace_pattern"
|
||||
]
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"id",
|
||||
"config"
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string",
|
||||
"const": "run_python"
|
||||
},
|
||||
"config": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"filename": {
|
||||
"type": "string",
|
||||
"title": "filename"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"filename"
|
||||
]
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"id",
|
||||
"config"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user