diff --git a/.github/workflows/tests_v2.yml b/.github/workflows/tests_v2.yml index 21a1fef1..3c8c5f10 100644 --- a/.github/workflows/tests_v2.yml +++ b/.github/workflows/tests_v2.yml @@ -12,6 +12,12 @@ jobs: - name: Check out code uses: actions/checkout@v3 + - name: Free up disk space + run: | + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android "$AGENT_TOOLSDIRECTORY" + sudo docker image prune -af + df -h + - name: Create KinD cluster uses: redhat-chaos/actions/kind@main diff --git a/CI/tests_v2/conftest.py b/CI/tests_v2/conftest.py index c2ec3034..352d79ff 100644 --- a/CI/tests_v2/conftest.py +++ b/CI/tests_v2/conftest.py @@ -174,5 +174,5 @@ from lib.k8s import ( # noqa: E402, F401 k8s_networking, kubectl, ) -from lib.namespace import _cleanup_stale_namespaces, test_namespace # noqa: E402, F401 +from lib.namespace import _cleanup_stale_namespaces, make_namespace, test_namespace # noqa: E402, F401 from lib.preflight import _preflight_checks # noqa: E402, F401 diff --git a/CI/tests_v2/lib/deploy.py b/CI/tests_v2/lib/deploy.py index 1d6bf796..2d6adbb4 100644 --- a/CI/tests_v2/lib/deploy.py +++ b/CI/tests_v2/lib/deploy.py @@ -9,6 +9,7 @@ from pathlib import Path import pytest import yaml from kubernetes import utils as k8s_utils +from kubernetes.client.rest import ApiException from lib.base import READINESS_TIMEOUT from lib.utils import patch_namespace_in_docs @@ -49,6 +50,104 @@ def wait_for_deployment_replicas(k8s_apps, namespace: str, name: str, timeout: i ) +def deployment_exists(k8s_apps, namespace: str, name: str) -> bool: + """Return True if the named deployment exists in the namespace, False on 404.""" + try: + k8s_apps.read_namespaced_deployment(name=name, namespace=namespace) + return True + except ApiException as e: + if e.status == 404: + return False + raise + + +def wait_for_no_deployment(k8s_apps, namespace: str, name: str, timeout: int = 45) -> None: + """Poll until the named deployment is gone from the namespace; raise if it persists.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if not deployment_exists(k8s_apps, namespace, name): + return + time.sleep(1) + raise AssertionError( + f"Deployment {name} still present in namespace={namespace} after {timeout}s" + ) + + +def service_exists(k8s_core, namespace: str, name: str) -> bool: + """Return True if the named service exists in the namespace, False on 404.""" + try: + k8s_core.read_namespaced_service(name=name, namespace=namespace) + return True + except ApiException as e: + if e.status == 404: + return False + raise + + +def wait_for_no_service(k8s_core, namespace: str, name: str, timeout: int = 45) -> None: + """Poll until the named service is gone from the namespace; raise if it persists.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if not service_exists(k8s_core, namespace, name): + return + time.sleep(1) + raise AssertionError( + f"Service {name} still present in namespace={namespace} after {timeout}s" + ) + + +def wait_for_present_deployment_count( + k8s_apps, namespaces, name: str, expected: int, timeout: int = 45 +) -> list: + """ + Poll until exactly `expected` of the given namespaces still contain the named deployment. + Returns the namespaces where it is still present. Useful for delete_count assertions. + """ + deadline = time.monotonic() + timeout + # Snapshot once so a one-shot iterable (e.g. generator) is not exhausted by the + # first poll, which would make every subsequent iteration see an empty list. + ns_list = list(namespaces) + present = list(ns_list) + while time.monotonic() < deadline: + present = [n for n in ns_list if deployment_exists(k8s_apps, n, name)] + if len(present) == expected: + return present + time.sleep(1) + raise AssertionError( + f"Expected {expected} namespace(s) with {name} present, " + f"found {len(present)}: {present}" + ) + + +def deploy_manifest_to_namespace( + k8s_client, + k8s_apps, + manifest, + namespace: str, + deployment_name: str, + *, + repo_root=None, + timeout: int = READINESS_TIMEOUT, +) -> None: + """ + Apply a manifest file into `namespace` (overriding each doc's namespace) and wait for the + named deployment to become ready. `manifest` may be absolute or relative to repo_root. + Reusable by any scenario that needs to seed a namespace with a target workload. + """ + path = Path(manifest) + if not path.is_absolute() and repo_root is not None: + path = Path(repo_root) / path + docs = patch_namespace_in_docs(list(yaml.safe_load_all(path.read_text())), namespace) + try: + k8s_utils.create_from_yaml(k8s_client, yaml_objects=docs, namespace=namespace) + except k8s_utils.FailToCreateError as e: + msgs = [str(exc) for exc in e.api_exceptions] + raise RuntimeError( + f"Failed to create resources in namespace={namespace}: {'; '.join(msgs)}" + ) from e + wait_for_deployment_replicas(k8s_apps, namespace, deployment_name, timeout=timeout) + + @pytest.fixture def wait_for_pods_running(k8s_core): """ diff --git a/CI/tests_v2/lib/namespace.py b/CI/tests_v2/lib/namespace.py index e78466ec..847f7d6e 100644 --- a/CI/tests_v2/lib/namespace.py +++ b/CI/tests_v2/lib/namespace.py @@ -16,6 +16,17 @@ logger = logging.getLogger(__name__) STALE_NS_AGE_MINUTES = 30 +# Privileged pod-security labels applied to ephemeral test namespaces so the same +# workloads are admitted on both Kubernetes and OpenShift. Shared by the test_namespace +# fixture and the make_namespace factory. +POD_SECURITY_PRIVILEGED_LABELS = { + "pod-security.kubernetes.io/audit": "privileged", + "pod-security.kubernetes.io/enforce": "privileged", + "pod-security.kubernetes.io/enforce-version": "v1.24", + "pod-security.kubernetes.io/warn": "privileged", + "security.openshift.io/scc.podSecurityLabelSync": "false", +} + def _namespace_age_minutes(metadata) -> float: """Return age of namespace in minutes from its creation_timestamp.""" @@ -47,6 +58,40 @@ def _wait_for_namespace_gone(k8s_core, name: str, timeout: int = 60): raise TimeoutError(f"Namespace {name} did not disappear within {timeout}s") +def create_labeled_namespace(k8s_core, name: str, extra_labels: dict = None) -> str: + """Create a namespace with privileged pod-security labels plus any extra_labels. + + Reusable across scenarios that need ad-hoc namespaces (e.g. multi-namespace + selection or label-selector targeting). Returns the namespace name. + """ + labels = dict(POD_SECURITY_PRIVILEGED_LABELS) + if extra_labels: + labels.update(extra_labels) + body = client.V1Namespace(metadata=client.V1ObjectMeta(name=name, labels=labels)) + k8s_core.create_namespace(body=body) + logger.info("Created test namespace: %s", name) + return name + + +def delete_namespace_quietly(k8s_core, name: str) -> None: + """Background-delete a namespace, logging (never raising) on failure. Safe in finalizers.""" + try: + k8s_core.delete_namespace( + name=name, + body=client.V1DeleteOptions(propagation_policy="Background"), + ) + except Exception as e: # noqa: BLE001 - cleanup must never raise + logger.warning("Failed to delete namespace %s: %s", name, e) + + +def _keep_namespace_on_fail(request) -> bool: + """True when --keep-ns-on-fail is set and the test's call phase failed.""" + keep_on_fail = request.config.getoption("--keep-ns-on-fail", False) + rep_call = getattr(request.node, "rep_call", None) + failed = rep_call is not None and rep_call.failed + return bool(keep_on_fail and failed) + + @pytest.fixture(scope="function") def test_namespace(request, k8s_core): """ @@ -57,13 +102,7 @@ def test_namespace(request, k8s_core): ns = client.V1Namespace( metadata=client.V1ObjectMeta( name=name, - labels={ - "pod-security.kubernetes.io/audit": "privileged", - "pod-security.kubernetes.io/enforce": "privileged", - "pod-security.kubernetes.io/enforce-version": "v1.24", - "pod-security.kubernetes.io/warn": "privileged", - "security.openshift.io/scc.podSecurityLabelSync": "false", - }, + labels=dict(POD_SECURITY_PRIVILEGED_LABELS), ) ) k8s_core.create_namespace(body=ns) @@ -71,21 +110,37 @@ def test_namespace(request, k8s_core): yield name - keep_on_fail = request.config.getoption("--keep-ns-on-fail", False) - rep_call = getattr(request.node, "rep_call", None) - failed = rep_call is not None and rep_call.failed - if keep_on_fail and failed: + if _keep_namespace_on_fail(request): logger.info("[keep-ns-on-fail] Keeping namespace %s for debugging", name) return - try: - k8s_core.delete_namespace( - name=name, - body=client.V1DeleteOptions(propagation_policy="Background"), - ) - logger.debug("Scheduled background deletion for namespace: %s", name) - except Exception as e: - logger.warning("Failed to delete namespace %s: %s", name, e) + delete_namespace_quietly(k8s_core, name) + + +@pytest.fixture(scope="function") +def make_namespace(request, k8s_core): + """ + Factory fixture to create ad-hoc privileged test namespaces during a test. + + Returns a callable make(name, extra_labels=None) -> name. Each created namespace + is registered for teardown deletion, honouring --keep-ns-on-fail the same way the + test_namespace fixture does. Useful for scenarios that need several namespaces + (multi-namespace selection) or a uniquely labelled namespace (label-selector targeting). + """ + + def _make(name: str, extra_labels: dict = None) -> str: + create_labeled_namespace(k8s_core, name, extra_labels=extra_labels) + + def _finalize(ns_name=name): + if _keep_namespace_on_fail(request): + logger.info("[keep-ns-on-fail] Keeping namespace %s for debugging", ns_name) + return + delete_namespace_quietly(k8s_core, ns_name) + + request.addfinalizer(_finalize) + return name + + return _make @pytest.fixture(scope="session", autouse=True) diff --git a/CI/tests_v2/lib/utils.py b/CI/tests_v2/lib/utils.py index 313e7c14..5c3d701c 100644 --- a/CI/tests_v2/lib/utils.py +++ b/CI/tests_v2/lib/utils.py @@ -24,6 +24,7 @@ SCENARIO_EXECUTION_MARKERS = { "pod_disruption": r"Deleting pod |waiting up to .* seconds for pod recovery", "application_outage": r"Creating the network policy|Deleting the network policy", "storage_throttle": r"Setting io\.max|Verified blkio settings|Privileged pod deployed", + "namespace_deletion": r"Delete objects in selected namespace|Deleted all objects in namespace", } # nodeid -> {"scenario", "pattern", "verified"}; consumed by conftest to build the diff --git a/CI/tests_v2/pytest.ini b/CI/tests_v2/pytest.ini index eb0bc0df..8dede3fe 100644 --- a/CI/tests_v2/pytest.ini +++ b/CI/tests_v2/pytest.ini @@ -13,6 +13,7 @@ markers = cpu_hog: marks a test as a CPU hog scenario test memory_hog: marks a test as a memory hog scenario test node_scenarios: marks a test as a node chaos scenario test (node reboot/stop-start) + namespace_deletion: marks a test as a namespace deletion (service_disruption) scenario test no_workload: skip workload deployment for this test (e.g. negative tests) order: set test order (pytest-order) junit_family = xunit2 diff --git a/CI/tests_v2/scenarios/namespace_deletion/resource.yaml b/CI/tests_v2/scenarios/namespace_deletion/resource.yaml new file mode 100644 index 00000000..d5a04aa0 --- /dev/null +++ b/CI/tests_v2/scenarios/namespace_deletion/resource.yaml @@ -0,0 +1,33 @@ +# Target workload for namespace_deletion (service_disruption) scenario tests. +# A Deployment plus a Service so the scenario has multiple object kinds to delete. +# Namespace is patched at deploy time by the test framework. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: namespace-deletion-target +spec: + replicas: 1 + selector: + matchLabels: + app: namespace-deletion-target + template: + metadata: + labels: + app: namespace-deletion-target + spec: + containers: + - name: app + image: nginx:alpine + ports: + - containerPort: 80 +--- +apiVersion: v1 +kind: Service +metadata: + name: namespace-deletion-target +spec: + selector: + app: namespace-deletion-target + ports: + - port: 80 + targetPort: 80 diff --git a/CI/tests_v2/scenarios/namespace_deletion/scenario_base.yaml b/CI/tests_v2/scenarios/namespace_deletion/scenario_base.yaml new file mode 100644 index 00000000..56f91f90 --- /dev/null +++ b/CI/tests_v2/scenarios/namespace_deletion/scenario_base.yaml @@ -0,0 +1,12 @@ +# Base service_disruption (namespace deletion) scenario. +# Tests load this and patch scenarios[0].namespace with ^$ (NAMESPACE_IS_REGEX), +# and override delete_count / runs / sleep / wait_time / label_selector as needed. +# The service_disruption plugin deletes all objects (deployments, daemonsets, +# statefulsets, replicasets, services) inside each matched namespace. +scenarios: +- namespace: "^krkn-test-namespace-deletion-placeholder$" + label_selector: "" + delete_count: 1 + runs: 1 + sleep: 1 + wait_time: 30 diff --git a/CI/tests_v2/scenarios/namespace_deletion/test_namespace_deletion.py b/CI/tests_v2/scenarios/namespace_deletion/test_namespace_deletion.py new file mode 100644 index 00000000..ed265019 --- /dev/null +++ b/CI/tests_v2/scenarios/namespace_deletion/test_namespace_deletion.py @@ -0,0 +1,243 @@ +""" +Functional test for the namespace_deletion (service_disruption) scenario. +Migrated from CI/tests/test_namespace.sh. + +The service_disruption plugin selects namespaces by regex (scenarios[].namespace) +or by namespace label_selector, then deletes all objects (deployments, daemonsets, +statefulsets, replicasets, services) inside each selected namespace. delete_count +controls how many matched namespaces are disrupted per run; runs repeats the loop. + +Safety: every namespace targeted here is the per-test ephemeral namespace or one the +test creates with a unique krkn-test- prefix, so the regex/label can never match +a namespace the test did not create. +""" + +import logging +import os +import uuid + +import pytest + +from lib.base import BaseScenarioTest +from lib.deploy import ( + deploy_manifest_to_namespace, + deployment_exists, + wait_for_no_deployment, + wait_for_no_service, + wait_for_present_deployment_count, +) +from lib.utils import ( + assert_kraken_failure, + assert_kraken_success, + assert_scenario_executed, +) + +logger = logging.getLogger(__name__) + +_TARGET_NAME = "namespace-deletion-target" + + +@pytest.mark.functional +@pytest.mark.namespace_deletion +class TestNamespaceDeletion(BaseScenarioTest): + """namespace_deletion scenario: delete all objects in selected namespaces.""" + + WORKLOAD_MANIFEST = "CI/tests_v2/scenarios/namespace_deletion/resource.yaml" + WORKLOAD_IS_PATH = True + LABEL_SELECTOR = "app=namespace-deletion-target" + SCENARIO_NAME = "namespace_deletion" + SCENARIO_TYPE = "service_disruption_scenarios" + NAMESPACE_KEY_PATH = ["scenarios", 0, "namespace"] + NAMESPACE_IS_REGEX = True + OVERRIDES_KEY_PATH = ["scenarios", 0] + + # --- happy-path tests ---------------------------------------------------- + + @pytest.mark.order(1) + def test_single_namespace_object_deletion(self): + """Regex matches exactly one namespace; all its objects are deleted, Krkn exits 0.""" + ns = self.ns + assert deployment_exists(self.k8s_apps, ns, _TARGET_NAME), f"Workload not deployed in {ns}" + result = self.run_scenario( + self.tmp_path, ns, + overrides={"delete_count": 1, "runs": 1, "sleep": 1}, + config_filename="ns_del_single.yaml", + ) + assert_kraken_success(result, context=f"namespace={ns}", tmp_path=self.tmp_path) + assert_scenario_executed( + result, self.SCENARIO_NAME, context=f"namespace={ns}", tmp_path=self.tmp_path + ) + if os.environ.get("KRKN_TEST_DRY_RUN", "0") != "1": + wait_for_no_deployment(self.k8s_apps, ns, _TARGET_NAME) + wait_for_no_service(self.k8s_core, ns, _TARGET_NAME) + + @pytest.mark.no_workload + def test_multiple_namespace_delete_count(self, make_namespace): + """Regex matches 3 namespaces with delete_count=2; exactly 2 are disrupted, 1 untouched.""" + prefix = f"krkn-test-{uuid.uuid4().hex[:8]}-multi" + namespaces = [] + for i in range(3): + name = make_namespace(f"{prefix}-{i}") + deploy_manifest_to_namespace( + self.k8s_client, self.k8s_apps, self.WORKLOAD_MANIFEST, name, + _TARGET_NAME, repo_root=self.repo_root, + ) + namespaces.append(name) + result = self.run_scenario( + self.tmp_path, f"{prefix}-.*", + overrides={"delete_count": 2, "runs": 1, "sleep": 1}, + config_filename="ns_del_multi.yaml", + ) + assert_kraken_success(result, context=f"prefix={prefix}", tmp_path=self.tmp_path) + assert_scenario_executed( + result, self.SCENARIO_NAME, context=f"prefix={prefix}", tmp_path=self.tmp_path + ) + # Exactly one of the three namespaces should still hold its deployment. + if os.environ.get("KRKN_TEST_DRY_RUN", "0") != "1": + wait_for_present_deployment_count(self.k8s_apps, namespaces, _TARGET_NAME, expected=1) + + def test_multiple_runs_repeat_disruption_loop(self): + """runs=2 executes the disruption loop twice. + + The plugin logs "Delete objects in selected namespace" once per run, so the + count proves the outer runs loop iterated twice. Actual object removal is + asserted in test_single_namespace_object_deletion; here the workload is deleted + in run 1 and never recreated (Krkn does not redeploy between runs), so run 2 + re-selects the now-empty namespace and deletes nothing. This test therefore + verifies that the loop repeats, not that deletion recurs against live objects. + """ + ns = self.ns + result = self.run_scenario( + self.tmp_path, ns, + overrides={"runs": 2, "delete_count": 1, "sleep": 1}, + config_filename="ns_del_runs.yaml", + ) + assert_kraken_success(result, context=f"namespace={ns}", tmp_path=self.tmp_path) + assert_scenario_executed( + result, self.SCENARIO_NAME, context=f"namespace={ns}", tmp_path=self.tmp_path + ) + if os.environ.get("KRKN_TEST_DRY_RUN", "0") != "1": + combined = f"{result.stdout or ''}\n{result.stderr or ''}" + count = combined.count("Delete objects in selected namespace") + assert count >= 2, ( + f"Expected the disruption loop to iterate >=2 times for runs=2, " + f"saw {count} (namespace={ns})" + ) + + def test_wait_time_accepted(self): + """A configured wait_time is accepted and the scenario completes successfully.""" + ns = self.ns + # Use a non-default wait_time (base scenario_base.yaml defaults to 30) so the test + # actually exercises override patching rather than passing on the base value. + result = self.run_scenario( + self.tmp_path, ns, + overrides={"wait_time": 5, "delete_count": 1, "runs": 1, "sleep": 1}, + config_filename="ns_del_wait.yaml", + ) + assert_kraken_success(result, context=f"namespace={ns}", tmp_path=self.tmp_path) + assert_scenario_executed( + result, self.SCENARIO_NAME, context=f"namespace={ns}", tmp_path=self.tmp_path + ) + + @pytest.mark.no_workload + def test_label_selector_targeting(self, make_namespace): + """With namespace empty and a label_selector set, targeting works by namespace label.""" + label_key = "krkn-test-ns" + label_value = uuid.uuid4().hex[:8] + name = make_namespace( + f"krkn-test-{label_value}-label", extra_labels={label_key: label_value} + ) + deploy_manifest_to_namespace( + self.k8s_client, self.k8s_apps, self.WORKLOAD_MANIFEST, name, + _TARGET_NAME, repo_root=self.repo_root, + ) + # Build the scenario directly: label-selector mode requires the namespace field to be a + # literal empty string (""), but NAMESPACE_IS_REGEX=True makes load_and_patch_scenario wrap + # any namespace as ^...$, so an empty namespace through run_scenario would become "^$". + # We patch with a real name (harmless) and then blank the namespace to "" explicitly. + scenario = self.load_and_patch_scenario( + self.repo_root, name, + label_selector=f"{label_key}={label_value}", delete_count=1, runs=1, sleep=1, + ) + scenario["scenarios"][0]["namespace"] = "" + scenario_path = self.write_scenario(self.tmp_path, scenario, suffix="_label") + config_path = self.build_config( + self.SCENARIO_TYPE, str(scenario_path), filename="ns_del_label.yaml" + ) + # This test bypasses run_scenario (to blank the namespace field), so the dry-run + # short-circuit in BaseScenarioTest.run_scenario does not apply here. Honor it + # explicitly: skip the real Kraken invocation and the cluster post-check. + if os.environ.get("KRKN_TEST_DRY_RUN", "0") == "1": + logger.info( + "[dry-run] Would run Kraken with config=%s (label-selector mode)", config_path + ) + return + result = self.run_kraken(config_path) + assert_kraken_success(result, context=f"namespace={name}", tmp_path=self.tmp_path) + assert_scenario_executed( + result, self.SCENARIO_NAME, context=f"namespace={name}", tmp_path=self.tmp_path + ) + wait_for_no_deployment(self.k8s_apps, name, _TARGET_NAME) + wait_for_no_service(self.k8s_core, name, _TARGET_NAME) + + # --- negative / failure-mode tests --------------------------------------- + + @pytest.mark.no_workload + def test_no_match_namespace_fails(self): + """A regex matching zero namespaces makes Krkn exit non-zero with a clear error.""" + # UUID-based name so the regex is guaranteed not to match any pre-existing + # namespace (e.g. leftover from a previous run); this namespace is never created. + ns = f"krkn-test-nonexistent-{uuid.uuid4().hex}" + result = self.run_scenario( + self.tmp_path, ns, + overrides={"delete_count": 1, "runs": 1}, + config_filename="ns_del_nomatch.yaml", + ) + if os.environ.get("KRKN_TEST_DRY_RUN", "0") == "1": + return # Krkn is skipped under dry-run; the failure path can't be exercised. + assert_kraken_failure(result, context=f"namespace={ns}", tmp_path=self.tmp_path) + combined = f"{result.stdout or ''}\n{result.stderr or ''}".lower() + # A regex matching zero namespaces makes krkn_lib's check_namespaces raise before the + # plugin's delete loop, surfacing "there exists no namespaces matching: {...}". The plugin's + # own "not enough namespaces matching" message (service_disruption_scenario_plugin.py:82-90) + # only fires when >=1 namespace matches but fewer than delete_count -- that path is covered by + # test_delete_count_exceeds_available_fails. + assert "no namespaces matching" in combined, ( + "Expected a 'no namespaces matching' error in Krkn output" + ) + + @pytest.mark.no_workload + def test_namespace_and_label_mutual_exclusion_fails(self): + """Setting both namespace and label_selector makes Krkn exit 1 with a mutual-exclusion error.""" + ns = "krkn-test-mutual-excl" + label_selector = "app=foo" + result = self.run_scenario( + self.tmp_path, ns, + overrides={"label_selector": label_selector, "delete_count": 1, "runs": 1}, + config_filename="ns_del_mutual.yaml", + ) + if os.environ.get("KRKN_TEST_DRY_RUN", "0") == "1": + return # Krkn is skipped under dry-run; the failure path can't be exercised. + assert_kraken_failure( + result, context=f"namespace={ns}, label_selector={label_selector}", tmp_path=self.tmp_path + ) + combined = f"{result.stdout or ''}\n{result.stderr or ''}".lower() + assert "you can only have namespace or label set" in combined, ( + "Expected the mutual-exclusion error in Krkn output" + ) + + def test_delete_count_exceeds_available_fails(self): + """delete_count greater than the number of matched namespaces fails with 'not enough namespaces'.""" + ns = self.ns + result = self.run_scenario( + self.tmp_path, ns, + overrides={"delete_count": 5, "runs": 1, "sleep": 1}, + config_filename="ns_del_exceeds.yaml", + ) + if os.environ.get("KRKN_TEST_DRY_RUN", "0") == "1": + return # Krkn is skipped under dry-run; the failure path can't be exercised. + assert_kraken_failure(result, context=f"namespace={ns}", tmp_path=self.tmp_path) + combined = f"{result.stdout or ''}\n{result.stderr or ''}".lower() + assert "not enough namespaces" in combined, ( + f"Expected 'not enough namespaces' error in Krkn output (namespace={ns})" + )