fixes krknctl describe bug

This is needed to avoid issues due to comparing two different data types:
TypeError: Invalid comparison between dtype=float64 and str. This commit also
avoids setting defaults for the thresholds to make it mandatory for the users
to define them as it plays a key role in determining the outliers.

Signed-off-by: Naga Ravi Chaitanya Elluri <nelluri@redhat.com>

.github/workflows/build.yml

@@ -1,51 +0,0 @@
-name: Build Krkn
-on:
-  pull_request:
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out code
-        uses: actions/checkout@v3
-      - name: Create multi-node KinD cluster
-        uses: redhat-chaos/actions/kind@main
-      - name: Install Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.9'
-          architecture: 'x64'
-      - name: Install environment
-        run: |
-          sudo apt-get install build-essential python3-dev
-          pip install --upgrade pip
-          pip install -r requirements.txt
-      - name: Run unit tests
-        run: python -m coverage run -a -m unittest discover -s tests -v
-      - name: Run CI
-        run: |
-          ./CI/run.sh
-          cat ./CI/results.markdown >> $GITHUB_STEP_SUMMARY
-          echo >> $GITHUB_STEP_SUMMARY
-      - name: Upload CI logs
-        uses: actions/upload-artifact@v3
-        with:
-          name: ci-logs
-          path: CI/out
-          if-no-files-found: error
-      - name: Collect coverage report
-        run: |
-          python -m coverage html
-      - name: Publish coverage report to job summary
-        run: |
-          pip install html2text
-          html2text --ignore-images --ignore-links -b 0 htmlcov/index.html >> $GITHUB_STEP_SUMMARY
-      - name: Upload coverage data
-        uses: actions/upload-artifact@v3
-        with:
-          name: coverage
-          path: htmlcov
-          if-no-files-found: error
-      - name: Check CI results
-        run: grep Fail CI/results.markdown && false || true
-      

.github/workflows/docker-image.yml

@@ -1,8 +1,7 @@
 name: Docker Image CI
 on:
   push:
-    branches:
-      - main
+    tags: ['v[0-9].[0-9]+.[0-9]+']
   pull_request:
 
 jobs:
@@ -12,19 +11,45 @@ jobs:
     - name: Check out code
       uses: actions/checkout@v3
     - name: Build the Docker images
-      run: docker build --no-cache -t quay.io/redhat-chaos/krkn containers/
+      if: startsWith(github.ref, 'refs/tags')
+      run:  |
+        ./containers/compile_dockerfile.sh
+        docker build --no-cache -t quay.io/krkn-chaos/krkn containers/ --build-arg TAG=${GITHUB_REF#refs/tags/}
+        docker tag quay.io/krkn-chaos/krkn quay.io/redhat-chaos/krkn
+        docker tag quay.io/krkn-chaos/krkn quay.io/krkn-chaos/krkn:${GITHUB_REF#refs/tags/}
+        docker tag quay.io/krkn-chaos/krkn quay.io/redhat-chaos/krkn:${GITHUB_REF#refs/tags/}
+
+    - name: Test Build the Docker images
+      if: ${{ github.event_name == 'pull_request' }}
+      run: |
+        ./containers/compile_dockerfile.sh
+        docker build --no-cache -t quay.io/krkn-chaos/krkn containers/ --build-arg PR_NUMBER=${{ github.event.pull_request.number }}
     - name: Login in quay
-      if: github.ref == 'refs/heads/main' && github.event_name == 'push'
+      if: startsWith(github.ref, 'refs/tags')
+      run: docker login quay.io -u ${QUAY_USER} -p ${QUAY_TOKEN}
+      env:
+        QUAY_USER: ${{ secrets.QUAY_USERNAME }}
+        QUAY_TOKEN: ${{ secrets.QUAY_PASSWORD }}
+    - name: Push the KrknChaos Docker images
+      if: startsWith(github.ref, 'refs/tags')
+      run: |
+        docker push quay.io/krkn-chaos/krkn
+        docker push quay.io/krkn-chaos/krkn:${GITHUB_REF#refs/tags/}
+    - name: Login in to redhat-chaos quay
+      if: startsWith(github.ref, 'refs/tags/v')
       run: docker login quay.io -u ${QUAY_USER} -p ${QUAY_TOKEN}
       env:
         QUAY_USER: ${{ secrets.QUAY_USER_1 }}
         QUAY_TOKEN: ${{ secrets.QUAY_TOKEN_1 }}
-    - name: Push the Docker images
-      if: github.ref == 'refs/heads/main' && github.event_name == 'push'
-      run: docker push quay.io/redhat-chaos/krkn
+    - name: Push the RedHat Chaos Docker images
+      if: startsWith(github.ref, 'refs/tags')
+      run: | 
+        docker push quay.io/redhat-chaos/krkn
+        docker push quay.io/redhat-chaos/krkn:${GITHUB_REF#refs/tags/}
     - name: Rebuild krkn-hub
-      if: github.ref == 'refs/heads/main' && github.event_name == 'push'
+      if: startsWith(github.ref, 'refs/tags')
       uses: redhat-chaos/actions/krkn-hub@main
       with:
-        QUAY_USER: ${{ secrets.QUAY_USER_1 }}
-        QUAY_TOKEN: ${{ secrets.QUAY_TOKEN_1 }}
+        QUAY_USER: ${{ secrets.QUAY_USERNAME }}
+        QUAY_TOKEN: ${{ secrets.QUAY_PASSWORD }}
+        AUTOPUSH: ${{ secrets.AUTOPUSH }}

.github/workflows/functional_tests.yaml

@@ -1,111 +0,0 @@
-on: issue_comment
-
-jobs:
-  check_user:
-    # This job only runs for pull request comments
-    name: Check User Authorization
-    env:
-      USERS: ${{vars.USERS}}
-    if: contains(github.event.comment.body, '/funtest') && contains(github.event.comment.html_url, '/pull/')
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check User
-        run: |
-          for name in `echo $USERS`
-          do 
-            name="${name//$'\r'/}"
-            name="${name//$'\n'/}"
-            if [ $name == "${{github.event.sender.login}}" ]
-            then
-              echo "user ${{github.event.sender.login}} authorized, action started..."
-              exit 0
-            fi
-          done
-          echo "user ${{github.event.sender.login}} is not allowed to run functional tests Action"
-          exit 1
-  pr_commented:
-    # This job only runs for pull request comments containing /functional
-    name: Functional Tests
-    if: contains(github.event.comment.body, '/funtest') && contains(github.event.comment.html_url, '/pull/')
-    runs-on: ubuntu-latest
-    needs:
-      - check_user
-    steps:
-      - name: Check out Kraken
-        uses: actions/checkout@v3
-      - name: Checkout Pull Request
-        run: gh pr checkout ${{ github.event.issue.number }}
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - name: Install OC CLI
-        uses: redhat-actions/oc-installer@v1
-        with:
-          oc_version: latest
-      - name: Install python 3.9
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.9' 
-      - name: Setup kraken dependencies
-        run:  pip install -r requirements.txt
-      - name: Create Workdir & export the path
-        run: |
-         mkdir workdir
-         echo "WORKDIR_PATH=`pwd`/workdir" >> $GITHUB_ENV
-      - name: Teardown CRC (Post Action)
-        uses: webiny/action-post-run@3.0.0
-        id: post-run-command
-        with:
-          # currently using image coming from tsebastiani quay.io repo
-          # waiting that a fix is merged in the upstream one
-          # post action run cannot (apparently) be properly indented 
-          run: docker run -v "${{ env.WORKDIR_PATH }}:/workdir" -e WORKING_MODE=T -e AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} -e AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} -e AWS_DEFAULT_REGION=us-west-2 -e TEARDOWN_RUN_ID=crc quay.io/tsebastiani/crc-cloud
-      - name: Run CRC
-          # currently using image coming from tsebastiani quay.io repo
-          # waiting that a fix is merged in the upstream one
-        run: |
-            docker run -v "${{ env.WORKDIR_PATH }}:/workdir" \
-            -e WORKING_MODE=C \
-            -e PULL_SECRET="${{ secrets.PULL_SECRET }}" \
-            -e AWS_ACCESS_KEY_ID="${{ secrets.AWS_ACCESS_KEY_ID }}" \
-            -e AWS_SECRET_ACCESS_KEY="${{ secrets.AWS_SECRET_ACCESS_KEY }}" \
-            -e AWS_DEFAULT_REGION=us-west-2 \
-            -e CREATE_RUN_ID=crc \
-            -e PASS_KUBEADMIN="${{ secrets.KUBEADMIN_PWD }}" \
-            -e PASS_REDHAT="${{ secrets.REDHAT_PWD }}" \
-            -e PASS_DEVELOPER="${{ secrets.DEVELOPER_PWD }}" \
-            quay.io/tsebastiani/crc-cloud
-      - name: OpenShift login and example deployment, GitHub Action env init 
-        env:
-          NAMESPACE: test-namespace
-          DEPLOYMENT_NAME: test-nginx
-          KUBEADMIN_PWD: '${{ secrets.KUBEADMIN_PWD }}'
-        run: ./CI/CRC/init_github_action.sh
-      - name: Setup test suite
-        run: |
-          yq -i '.kraken.port="8081"' CI/config/common_test_config.yaml
-          yq -i '.kraken.signal_address="0.0.0.0"' CI/config/common_test_config.yaml
-          
-          echo "test_app_outages_gh" > ./CI/tests/my_tests
-          echo "test_container"      >> ./CI/tests/my_tests
-          echo "test_namespace"      >> ./CI/tests/my_tests
-          echo "test_net_chaos"      >> ./CI/tests/my_tests
-          echo "test_time"           >> ./CI/tests/my_tests
-
-      - name: Print affected config files
-        run: |
-          echo -e "## CI/config/common_test_config.yaml\n\n"
-          cat CI/config/common_test_config.yaml
-
-      - name: Running test suite
-        run: |
-          ./CI/run.sh
-      - name: Print test output
-        run:  cat CI/out/*
-      - name: Create coverage report
-        run: |
-          echo "# Test results" > $GITHUB_STEP_SUMMARY
-          cat CI/results.markdown >> $GITHUB_STEP_SUMMARY
-          echo "# Test coverage" >> $GITHUB_STEP_SUMMARY
-          python -m coverage report --format=markdown >> $GITHUB_STEP_SUMMARY 
-
-

.github/workflows/tests.yml

@@ -0,0 +1,198 @@
+name: Functional & Unit Tests
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+jobs:
+  tests:
+    # Common steps
+    name: Functional & Unit Tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v3
+      - name: Create multi-node KinD cluster
+        uses: redhat-chaos/actions/kind@main
+      - name: Install Helm & add repos
+        run: |
+          curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
+          helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
+          helm repo add stable https://charts.helm.sh/stable
+          helm repo update
+      - name: Deploy prometheus & Port Forwarding
+        run: |
+          kubectl create namespace prometheus-k8s
+          helm install \
+          --wait --timeout 360s \
+          kind-prometheus \
+          prometheus-community/kube-prometheus-stack \
+          --namespace prometheus-k8s \
+          --set prometheus.service.nodePort=30000 \
+          --set prometheus.service.type=NodePort \
+          --set grafana.service.nodePort=31000 \
+          --set grafana.service.type=NodePort \
+          --set alertmanager.service.nodePort=32000 \
+          --set alertmanager.service.type=NodePort \
+          --set prometheus-node-exporter.service.nodePort=32001 \
+          --set prometheus-node-exporter.service.type=NodePort
+
+          SELECTOR=`kubectl -n prometheus-k8s get service kind-prometheus-kube-prome-prometheus -o wide --no-headers=true | awk '{ print $7 }'`
+          POD_NAME=`kubectl -n prometheus-k8s get pods --selector="$SELECTOR" --no-headers=true | awk '{ print $1 }'`
+          kubectl -n prometheus-k8s port-forward $POD_NAME 9090:9090 &
+          sleep 5
+      - name: Install Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+          architecture: 'x64'
+      - name: Install environment
+        run: |
+          sudo apt-get install build-essential python3-dev
+          pip install --upgrade pip
+          pip install -r requirements.txt
+
+      - name: Deploy test workloads
+        run: |
+          kubectl apply -f CI/templates/outage_pod.yaml
+          kubectl wait --for=condition=ready pod -l scenario=outage --timeout=300s
+          kubectl apply -f CI/templates/container_scenario_pod.yaml
+          kubectl wait --for=condition=ready pod -l scenario=container --timeout=300s
+          kubectl create namespace namespace-scenario
+          kubectl apply -f CI/templates/time_pod.yaml
+          kubectl wait --for=condition=ready pod -l scenario=time-skew --timeout=300s
+          kubectl apply -f CI/templates/service_hijacking.yaml
+          kubectl wait --for=condition=ready pod -l "app.kubernetes.io/name=proxy" --timeout=300s
+      - name: Get Kind nodes
+        run: |
+          kubectl get nodes --show-labels=true
+      # Pull request only steps
+      - name: Run unit tests
+        if: github.event_name == 'pull_request'
+        run: python -m coverage run -a -m unittest discover -s tests -v
+
+      - name: Setup Pull Request Functional Tests
+        if: |
+          github.event_name == 'pull_request'
+        run: |
+            yq -i '.kraken.port="8081"' CI/config/common_test_config.yaml
+            yq -i '.kraken.signal_address="0.0.0.0"' CI/config/common_test_config.yaml
+            yq -i '.kraken.performance_monitoring="localhost:9090"' CI/config/common_test_config.yaml
+            echo "test_service_hijacking" > ./CI/tests/functional_tests
+            echo "test_app_outages" >> ./CI/tests/functional_tests
+            echo "test_container"      >> ./CI/tests/functional_tests
+            echo "test_namespace"      >> ./CI/tests/functional_tests
+            echo "test_net_chaos"      >> ./CI/tests/functional_tests
+            echo "test_time"           >> ./CI/tests/functional_tests
+            echo "test_arca_cpu_hog" >> ./CI/tests/functional_tests
+            echo "test_arca_memory_hog" >> ./CI/tests/functional_tests
+            echo "test_arca_io_hog" >> ./CI/tests/functional_tests
+
+
+      # Push on main only steps + all other functional to collect coverage
+      # for the badge
+      - name: Configure AWS Credentials
+        if: github.ref == 'refs/heads/main' && github.event_name == 'push'
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region : ${{ secrets.AWS_REGION }}
+      - name: Setup Post Merge Request Functional Tests
+        if: github.ref == 'refs/heads/main' && github.event_name == 'push'
+        run: |
+          yq -i '.kraken.port="8081"' CI/config/common_test_config.yaml
+          yq -i '.kraken.signal_address="0.0.0.0"' CI/config/common_test_config.yaml
+          yq -i '.kraken.performance_monitoring="localhost:9090"' CI/config/common_test_config.yaml
+          yq -i '.telemetry.username="${{secrets.TELEMETRY_USERNAME}}"' CI/config/common_test_config.yaml
+          yq -i '.telemetry.password="${{secrets.TELEMETRY_PASSWORD}}"' CI/config/common_test_config.yaml
+          echo "test_telemetry" > ./CI/tests/functional_tests
+          echo "test_service_hijacking" >> ./CI/tests/functional_tests
+          echo "test_app_outages" >> ./CI/tests/functional_tests
+          echo "test_container"      >> ./CI/tests/functional_tests
+          echo "test_namespace"      >> ./CI/tests/functional_tests
+          echo "test_net_chaos"      >> ./CI/tests/functional_tests
+          echo "test_time"           >> ./CI/tests/functional_tests
+          echo "test_arca_cpu_hog" >> ./CI/tests/functional_tests
+          echo "test_arca_memory_hog" >> ./CI/tests/functional_tests
+          echo "test_arca_io_hog" >> ./CI/tests/functional_tests
+
+      # Final common steps
+      - name: Run Functional tests
+        env:
+          AWS_BUCKET: ${{ secrets.AWS_BUCKET }}
+        run: |
+          ./CI/run.sh
+          cat ./CI/results.markdown >> $GITHUB_STEP_SUMMARY
+          echo >> $GITHUB_STEP_SUMMARY
+      - name: Upload CI logs
+        uses: actions/upload-artifact@v4
+        with:
+          name: ci-logs
+          path: CI/out
+          if-no-files-found: error
+      - name: Collect coverage report
+        run: |
+          python -m coverage html
+          python -m coverage json
+      - name: Publish coverage report to job summary
+        run: |
+          pip install html2text
+          html2text --ignore-images --ignore-links -b 0 htmlcov/index.html >> $GITHUB_STEP_SUMMARY
+      - name: Upload coverage data
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage
+          path: htmlcov
+          if-no-files-found: error
+      - name: Upload json coverage
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage.json
+          path: coverage.json
+          if-no-files-found: error
+      - name: Check CI results
+        run: grep Fail CI/results.markdown && false || true
+  badge:
+    permissions:
+      contents: write
+    name: Generate Coverage Badge
+    runs-on: ubuntu-latest
+    needs:
+      - tests
+    if: github.ref == 'refs/heads/main' && github.event_name == 'push'
+    steps:
+        - name: Check out doc repo
+          uses: actions/checkout@master
+          with:
+            repository: krkn-chaos/krkn-lib-docs
+            path: krkn-lib-docs
+            ssh-key: ${{ secrets.KRKN_LIB_DOCS_PRIV_KEY }}
+        - name: Download json coverage
+          uses: actions/download-artifact@v4
+          with:
+            name: coverage.json
+        - name: Set up Python
+          uses: actions/setup-python@v4
+          with:
+            python-version: 3.9
+        - name: Copy badge on GitHub Page Repo
+          env:
+            COLOR: yellow
+          run: |
+            # generate coverage badge on previously calculated total coverage
+            # and copy in the docs page
+            export TOTAL=$(python -c "import json;print(json.load(open('coverage.json'))['totals']['percent_covered_display'])")
+            [[ $TOTAL > 40 ]] && COLOR=green
+            echo "TOTAL: $TOTAL"
+            echo "COLOR: $COLOR"
+            curl "https://img.shields.io/badge/coverage-$TOTAL%25-$COLOR" > ./krkn-lib-docs/coverage_badge_krkn.svg
+        - name: Push updated Coverage Badge
+          run: |
+            cd krkn-lib-docs
+            git add .
+            git config user.name "krkn-chaos"
+            git config user.email "<>"
+            git commit -m "[KRKN] Coverage Badge ${GITHUB_REF##*/}" || echo "no changes to commit"
+            git push
+      

.gitignore

@@ -16,6 +16,7 @@ __pycache__/*
 *.out
 kube-burner*
 kube_burner*
+recommender_*.json
 
 # Project files
 .ropeproject
@@ -61,7 +62,7 @@ inspect.local.*
 !CI/config/common_test_config.yaml
 CI/out/*
 CI/ci_results
-CI/scenarios/*node.yaml
+CI/legacy/*node.yaml
 CI/results.markdown
 
 #env

CI/CRC/deployment.yaml

@@ -1,44 +0,0 @@
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: $NAMESPACE
-
----
-
-apiVersion: v1
-kind: Service
-metadata:
-  name: $DEPLOYMENT_NAME-service
-  namespace: $NAMESPACE
-spec:
-  selector:
-    app: $DEPLOYMENT_NAME
-  ports:
-    - name: http
-      port: 80
-      targetPort: 8080
-  type: ClusterIP
-
---- 
-
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  namespace: $NAMESPACE
-  name: $DEPLOYMENT_NAME-deployment
-spec:
-  replicas: 3
-  selector:
-    matchLabels:
-      app: $DEPLOYMENT_NAME
-  template:
-    metadata:
-      labels:
-        app: $DEPLOYMENT_NAME
-    spec:
-      containers:
-        - name: $DEPLOYMENT_NAME
-          image: nginxinc/nginx-unprivileged:stable-alpine
-          ports:
-            - name: http
-              containerPort: 8080

CI/CRC/init_github_action.sh

@@ -1,72 +0,0 @@
-#!/bin/bash
-SCRIPT_PATH=./CI/CRC
-DEPLOYMENT_PATH=$SCRIPT_PATH/deployment.yaml
-CLUSTER_INFO=cluster_infos.json
-
-[[ -z $WORKDIR_PATH ]] && echo "[ERROR] please set \$WORKDIR_PATH environment variable" && exit 1
-CLUSTER_INFO_PATH=$WORKDIR_PATH/crc/$CLUSTER_INFO
-
-[[ ! -f $DEPLOYMENT_PATH ]] && echo "[ERROR] please run $0 from GitHub action root directory" && exit 1
-[[ -z $KUBEADMIN_PWD ]] && echo "[ERROR] kubeadmin password not set, please check the repository secrets" && exit 1
-[[ -z $DEPLOYMENT_NAME ]] && echo "[ERROR] please set \$DEPLOYMENT_NAME environment variable" && exit 1
-[[ -z $NAMESPACE ]] && echo "[ERROR] please set \$NAMESPACE environment variable" && exit 1
-[[ ! -f $CLUSTER_INFO_PATH ]] && echo  "[ERROR] cluster_info.json not found in $CLUSTER_INFO_PATH" && exit 1 
-
-OPENSSL=`which openssl 2>/dev/null`
-[[ $? != 0 ]] && echo "[ERROR]: openssl missing, please install it and try again" && exit 1
-OC=`which oc 2>/dev/null`
-[[ $? != 0 ]] && echo "[ERROR]: oc missing, please install it and try again" && exit 1
-SED=`which sed 2>/dev/null`
-[[ $? != 0 ]] && echo "[ERROR]: sed missing, please install it and try again" && exit 1
-JQ=`which jq 2>/dev/null`
-[[ $? != 0 ]] && echo "[ERROR]: jq missing, please install it and try again" && exit 1
-ENVSUBST=`which envsubst 2>/dev/null`
-[[ $? != 0 ]] && echo "[ERROR]: envsubst missing, please install it and try again" && exit 1
-
-API_ADDRESS="$($JQ -r '.api.address' $CLUSTER_INFO_PATH)"
-API_PORT="$($JQ -r '.api.port' $CLUSTER_INFO_PATH)"
-BASE_HOST=`$JQ -r '.api.address'  $CLUSTER_INFO_PATH | sed -r 's#https:\/\/api\.(.+\.nip\.io)#\1#'`
-FQN=$DEPLOYMENT_NAME.apps.$BASE_HOST
-
-echo "[INF] logging on $API_ADDRESS:$API_PORT"
-COUNTER=1
-until `$OC login  --insecure-skip-tls-verify -u kubeadmin -p $KUBEADMIN_PWD $API_ADDRESS:$API_PORT > /dev/null 2>&1`
-do 
-    echo "[INF] login attempt $COUNTER"
-    [[ $COUNTER == 20 ]] && echo "[ERR] maximum login attempts exceeded, failing" && exit 1
-    ((COUNTER++))
-    sleep 10
-done
-
-echo "[INF] deploying example deployment: $DEPLOYMENT_NAME in namespace: $NAMESPACE"
-$ENVSUBST < $DEPLOYMENT_PATH | $OC apply -f - > /dev/null 2>&1
-
-echo "[INF] creating SSL self-signed certificates for route https://$FQN"
-$OPENSSL genrsa -out servercakey.pem > /dev/null 2>&1
-$OPENSSL req -new -x509 -key servercakey.pem -out serverca.crt -subj "/CN=$FQN/O=Red Hat Inc./C=US" > /dev/null 2>&1
-$OPENSSL genrsa -out server.key > /dev/null 2>&1
-$OPENSSL req -new -key server.key -out server_reqout.txt -subj "/CN=$FQN/O=Red Hat Inc./C=US" > /dev/null 2>&1
-$OPENSSL x509 -req -in server_reqout.txt -days 3650 -sha256 -CAcreateserial -CA serverca.crt -CAkey servercakey.pem -out server.crt > /dev/null 2>&1
-echo "[INF] creating deployment: $DEPLOYMENT_NAME public route: https://$FQN"
-$OC create route --namespace $NAMESPACE  edge --service=$DEPLOYMENT_NAME-service --cert=server.crt --key=server.key --ca-cert=serverca.crt --hostname="$FQN" > /dev/null 2>&1
-
-
-echo "[INF] setting github action environment variables"
-
-NODE_NAME="`$OC get nodes  -o json | $JQ -r '.items[0].metadata.name'`"
-COVERAGE_FILE="`pwd`/coverage.md"
-echo "DEPLOYMENT_NAME=$DEPLOYMENT_NAME" >> $GITHUB_ENV
-echo "DEPLOYMENT_FQN=$FQN" >> $GITHUB_ENV
-echo "API_ADDRESS=$API_ADDRESS" >> $GITHUB_ENV
-echo "API_PORT=$API_PORT" >> $GITHUB_ENV
-echo "NODE_NAME=$NODE_NAME" >> $GITHUB_ENV
-echo "NAMESPACE=$NAMESPACE" >> $GITHUB_ENV
-echo "COVERAGE_FILE=$COVERAGE_FILE" >> $GITHUB_ENV
-
-echo "[INF] deployment fully qualified name will be available in \${{ env.DEPLOYMENT_NAME }} with value $DEPLOYMENT_NAME"
-echo "[INF] deployment name will be available in \${{ env.DEPLOYMENT_FQN }} with value $FQN"
-echo "[INF] OCP API address will be available in \${{ env.API_ADDRESS }} with value $API_ADDRESS"
-echo "[INF] OCP API port will be available in \${{ env.API_PORT }} with value $API_PORT"
-echo "[INF] OCP node name will be available in \${{ env.NODE_NAME }} with value $NODE_NAME"
-echo "[INF] coverage file will ve available in \${{ env.COVERAGE_FILE }} with value $COVERAGE_FILE"
-

CI/README.md

@@ -1,7 +1,7 @@
 ## CI Tests
 
 ### First steps
-Edit [my_tests](tests/my_tests) with tests you want to run
+Edit [functional_tests](tests/functional_tests) with tests you want to run
 
 ### How to run
 ```./CI/run.sh```
@@ -11,7 +11,7 @@ This will run kraken using python, make sure python3 is set up and configured pr
 
 ### Adding a test case
 
-1. Add in simple scenario yaml file to execute under [../CI/scenarios/](scenarios)
+1. Add in simple scenario yaml file to execute under [../CI/scenarios/](legacy)
 
 2. Copy [test_application_outages.sh](tests/test_app_outages.sh) for example on how to get started
 
@@ -27,7 +27,7 @@ This will run kraken using python, make sure python3 is set up and configured pr
 
     e. 15: Make sure name of config in line 14 matches what you pass on this line
 
-4. Add test name to [my_tests](../CI/tests/my_tests) file
+4. Add test name to [functional_tests](../CI/tests/functional_tests) file
 
     a. This will be the name of the file without ".sh"
 

CI/config/common_test_config.yaml

@@ -1,5 +1,5 @@
 kraken:
-    distribution: openshift                                # Distribution can be kubernetes or openshift.
+    distribution: kubernetes                                # Distribution can be kubernetes or openshift.
     kubeconfig_path: ~/.kube/config                        # Path to kubeconfig.
     exit_on_failure: False                                 # Exit when a post action scenario fails.
     litmus_version: v1.13.6                                # Litmus version to install.
@@ -15,15 +15,13 @@ cerberus:
 performance_monitoring:
     deploy_dashboards: False                               # Install a mutable grafana and load the performance dashboards. Enable this only when running on OpenShift.
     repo: "https://github.com/cloud-bulldozer/performance-dashboards.git"
-    kube_burner_binary_url: "https://github.com/cloud-bulldozer/kube-burner/releases/download/v0.9.1/kube-burner-0.9.1-Linux-x86_64.tar.gz"
     capture_metrics: False
-    config_path: config/kube_burner.yaml                   # Define the Elasticsearch url and index name in this config.
     metrics_profile_path: config/metrics-aggregated.yaml
     prometheus_url:                                        # The prometheus url/route is automatically obtained in case of OpenShift, please set it when the distribution is Kubernetes.
     prometheus_bearer_token:                               # The bearer token is automatically obtained in case of OpenShift, please set it when the distribution is Kubernetes. This is needed to authenticate with prometheus.
     uuid:                                                  # uuid for the run is generated by default if not set.
     enable_alerts: False                                   # Runs the queries specified in the alert profile and displays the info or exits 1 when severity=error.
-    alert_profile: config/alerts                           # Path to alert profile with the prometheus queries.
+    alert_profile: config/alerts.yaml                           # Path to alert profile with the prometheus queries.
 
 tunings:
     wait_duration: 6                                       # Duration to wait between each chaos scenario.
@@ -31,9 +29,12 @@ tunings:
     daemon_mode: False                                     # Iterations are set to infinity which means that the kraken will cause chaos forever.
 telemetry:
     enabled: False                                           # enable/disables the telemetry collection feature
-    api_url: https://ulnmf9xv7j.execute-api.us-west-2.amazonaws.com/production #telemetry service endpoint
-    username: username                                      # telemetry service username
-    password: password                                      # telemetry service password
+    api_url: https://yvnn4rfoi7.execute-api.us-west-2.amazonaws.com/test #telemetry service endpoint
+    username: $TELEMETRY_USERNAME                                      # telemetry service username
+    password: $TELEMETRY_PASSWORD                                      # telemetry service password
+    prometheus_namespace: 'prometheus-k8s'                                # prometheus namespace
+    prometheus_pod_name: 'prometheus-kind-prometheus-kube-prome-prometheus-0'                                 # prometheus pod_name
+    prometheus_container_name: 'prometheus'
     prometheus_backup: True                                 # enables/disables prometheus data collection
     full_prometheus_backup: False                           # if is set to False only the /prometheus/wal folder will be downloaded.
     backup_threads: 5                                       # number of telemetry download/upload threads
@@ -41,3 +42,23 @@ telemetry:
     max_retries: 0                                          # maximum number of upload retries (if 0 will retry forever)
     run_tag: ''                                             # if set, this will be appended to the run folder in the bucket (useful to group the runs)
     archive_size: 10000                                     # the size of the prometheus data archive size in KB. The lower the size of archive is
+    logs_backup: True
+    logs_filter_patterns:
+        - "(\\w{3}\\s\\d{1,2}\\s\\d{2}:\\d{2}:\\d{2}\\.\\d+).+"         # Sep 9 11:20:36.123425532
+        - "kinit (\\d+/\\d+/\\d+\\s\\d{2}:\\d{2}:\\d{2})\\s+"          # kinit 2023/09/15 11:20:36 log
+        - "(\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d+Z).+"      # 2023-09-15T11:20:36.123425532Z log
+    oc_cli_path: /usr/bin/oc                                # optional, if not specified will be search in $PATH
+    events_backup: True                                     # enables/disables cluster events collection
+    telemetry_group: "funtests"
+elastic:
+    enable_elastic: False
+    collect_metrics: False
+    collect_alerts: False
+    verify_certs: False
+    elastic_url: "https://192.168.39.196"                                         # To track results in elasticsearch, give url to server here; will post telemetry details when url and index not blank
+    elastic_port: 32766
+    username: "elastic"
+    password: "test"
+    metrics_index: "krkn-metrics"
+    alerts_index: "krkn-alerts"
+    telemetry_index: "krkn-telemetry"

CI/run.sh

@@ -1,15 +1,14 @@
 #!/bin/bash
-set -x
 MAX_RETRIES=60
 
-OC=`which oc 2>/dev/null`
-[[ $? != 0 ]] && echo "[ERROR]: oc missing, please install it and try again" && exit 1
+KUBECTL=`which kubectl 2>/dev/null`
+[[ $? != 0 ]] && echo "[ERROR]: kubectl missing, please install it and try again" && exit 1
 
 wait_cluster_become_ready() {
   COUNT=1
-  until `$OC get namespace > /dev/null 2>&1`
+  until `$KUBECTL get namespace > /dev/null 2>&1`
   do
-    echo "[INF] waiting OpenShift to become ready, after $COUNT check"
+    echo "[INF] waiting Kubernetes to become ready, after $COUNT check"
     sleep 3
     [[ $COUNT == $MAX_RETRIES ]] && echo "[ERR] max retries exceeded, failing" && exit 1
     ((COUNT++))
@@ -18,9 +17,9 @@ wait_cluster_become_ready() {
 
 
 
-ci_tests_loc="CI/tests/my_tests"
+ci_tests_loc="CI/tests/functional_tests"
 
-echo "running test suit consisting of ${ci_tests}"
+echo -e "********* Running Functional Tests Suite *********\n\n"
 
 rm -rf CI/out
 
@@ -37,9 +36,32 @@ echo 'Test                   | Result | Duration' >> $results
 echo '-----------------------|--------|---------' >> $results
 
 # Run each test
-for test_name in `cat CI/tests/my_tests`
+failed_tests=()
+for test_name in `cat CI/tests/functional_tests`
 do
-  wait_cluster_become_ready
-  ./CI/run_test.sh $test_name $results
+  #wait_cluster_become_ready
+  return_value=`./CI/run_test.sh $test_name $results`
+  if [[ $return_value == 1 ]]
+  then
+    echo "Failed"
+    failed_tests+=("$test_name")
+  fi
   wait_cluster_become_ready
 done
+
+
+if (( ${#failed_tests[@]}>0 ))
+then
+  echo -e "\n\n======================================================================"
+  echo -e "\n     FUNCTIONAL TESTS FAILED  ${failed_tests[*]} ABORTING"
+  echo -e "\n======================================================================\n\n"
+
+  for test in "${failed_tests[@]}"
+  do
+    echo -e "\n********** $test KRKN RUN OUTPUT **********\n"
+    cat "CI/out/$test.out"
+    echo -e "\n********************************************\n\n\n\n"
+  done
+
+  exit 1
+fi

CI/run_test.sh

@@ -1,5 +1,4 @@
 #!/bin/bash
-set -x
 readonly SECONDS_PER_HOUR=3600
 readonly SECONDS_PER_MINUTE=60
 function get_time_format() {
@@ -14,9 +13,7 @@ ci_test=`echo $1`
 
 results_file=$2
 
-echo -e "\n======================================================================"
-echo -e "     CI test for ${ci_test}                    "
-echo -e "======================================================================\n"
+echo -e "test: ${ci_test}" >&2
 
 ci_results="CI/out/$ci_test.out"
 # Test ci
@@ -28,13 +25,16 @@ then
   # if the test passes update the results and complete
   duration=$SECONDS
   duration=$(get_time_format $duration)
-  echo "$ci_test: Successful"
+  echo -e  "> $ci_test: Successful\n" >&2
   echo "$ci_test | Pass | $duration" >> $results_file
   count=$retries
+  # return value for run.sh
+  echo 0
 else
   duration=$SECONDS
   duration=$(get_time_format $duration)
-  echo "$ci_test: Failed"
+  echo -e "> $ci_test: Failed\n" >&2
   echo "$ci_test | Fail | $duration" >> $results_file
-  echo "Logs for "$ci_test
+  # return value for run.sh
+  echo 1
 fi

CI/scenarios/app_outage.yaml

@@ -1,5 +0,0 @@
-application_outage:                                  # Scenario to create an outage of an application by blocking traffic
-  duration: 10                                      # Duration in seconds after which the routes will be accessible
-  namespace: openshift-monitoring            # Namespace to target - all application routes will go inaccessible if pod selector is empty
-  pod_selector: {}                            # Pods to target
-  block: [Ingress, Egress]                           # It can be Ingress or Egress or Ingress, Egress

CI/scenarios/container_scenario.yml

@@ -1,8 +0,0 @@
-scenarios:
-- name: "kill machine config container"
-  namespace: "openshift-machine-config-operator"
-  label_selector: "k8s-app=machine-config-server"
-  container_name: "hello-openshift"
-  action: "kill 1"
-  count: 1
-  retry_wait: 60

CI/scenarios/network_chaos.yaml

@@ -1,6 +0,0 @@
-network_chaos:                   # Scenario to create an outage by simulating random variations in the network.
-  duration: 10                   # seconds
-  instance_count: 1
-  execution: serial
-  egress:
-    bandwidth: 100mbit

CI/scenarios/network_diagnostics_namespace.yaml

@@ -1,7 +0,0 @@
-scenarios:
-  - action: delete
-    namespace: "^$openshift-network-diagnostics$"
-    label_selector:
-    runs: 1
-    sleep: 15
-    wait_time: 30

CI/scenarios/node_cpu_hog_engine.yaml

@@ -1,34 +0,0 @@
-apiVersion: litmuschaos.io/v1alpha1
-kind: ChaosEngine
-metadata:
-  name: nginx-chaos
-  namespace: litmus
-spec:
-  # It can be true/false
-  annotationCheck: 'false'
-  # It can be active/stop
-  engineState: 'active'
-  chaosServiceAccount: litmus-sa
-  monitoring: false
-  # It can be delete/retain
-  jobCleanUpPolicy: 'delete'
-  experiments:
-    - name: node-cpu-hog
-      spec:
-        components:
-          env:
-            # set chaos duration (in sec) as desired
-            - name: TOTAL_CHAOS_DURATION
-              value: '10'
-
-            # Number of cores of node CPU to be consumed
-            - name: NODE_CPU_CORE
-              value: '1'
-
-            # percentage of total nodes to target
-            - name: NODES_AFFECTED_PERC
-              value: '30'
-
-            # ENTER THE COMMA SEPARATED TARGET NODES NAME
-            - name: TARGET_NODES
-              value: $WORKER_NODE

CI/scenarios/node_cpu_hog_engine_node.yaml

@@ -1,34 +0,0 @@
-apiVersion: litmuschaos.io/v1alpha1
-kind: ChaosEngine
-metadata:
-  name: nginx-chaos
-  namespace: litmus
-spec:
-  # It can be true/false
-  annotationCheck: 'false'
-  # It can be active/stop
-  engineState: 'active'
-  chaosServiceAccount: litmus-sa
-  monitoring: false
-  # It can be delete/retain
-  jobCleanUpPolicy: 'delete'
-  experiments:
-    - name: node-cpu-hog
-      spec:
-        components:
-          env:
-            # set chaos duration (in sec) as desired
-            - name: TOTAL_CHAOS_DURATION
-              value: '10'
-
-            # Number of cores of node CPU to be consumed
-            - name: NODE_CPU_CORE
-              value: '1'
-
-            # percentage of total nodes to target
-            - name: NODES_AFFECTED_PERC
-              value: '30'
-
-            # ENTER THE COMMA SEPARATED TARGET NODES NAME
-            - name: TARGET_NODES
-              value:

CI/scenarios/node_io_engine.yaml

@@ -1,35 +0,0 @@
-apiVersion: litmuschaos.io/v1alpha1
-kind: ChaosEngine
-metadata:
-  name: nginx-chaos
-  namespace: litmus
-spec:
-  # It can be delete/retain
-  jobCleanUpPolicy: 'retain'
-  # It can be active/stop
-  engineState: 'active'
-  chaosServiceAccount: litmus-sa
-  experiments:
-    - name: node-io-stress
-      spec:
-        components:
-          env:
-            # set chaos duration (in sec) as desired
-            - name: TOTAL_CHAOS_DURATION
-              value: '10'
-
-            ## specify the size as percentage of free space on the file system
-            - name: FILESYSTEM_UTILIZATION_PERCENTAGE
-              value: '100'
-
-            ## Number of core of CPU
-            - name: CPU
-              value: '1'
-
-            ## Total number of workers default value is 4
-            - name: NUMBER_OF_WORKERS
-              value: '3'
-
-             ## enter the comma separated target nodes name
-            - name: TARGET_NODES
-              value: $WORKER_NODE

CI/scenarios/node_io_engine_node.yaml

@@ -1,35 +0,0 @@
-apiVersion: litmuschaos.io/v1alpha1
-kind: ChaosEngine
-metadata:
-  name: nginx-chaos
-  namespace: litmus
-spec:
-  # It can be delete/retain
-  jobCleanUpPolicy: 'retain'
-  # It can be active/stop
-  engineState: 'active'
-  chaosServiceAccount: litmus-sa
-  experiments:
-    - name: node-io-stress
-      spec:
-        components:
-          env:
-            # set chaos duration (in sec) as desired
-            - name: TOTAL_CHAOS_DURATION
-              value: '10'
-
-            ## specify the size as percentage of free space on the file system
-            - name: FILESYSTEM_UTILIZATION_PERCENTAGE
-              value: '100'
-
-            ## Number of core of CPU
-            - name: CPU
-              value: '1'
-
-            ## Total number of workers default value is 4
-            - name: NUMBER_OF_WORKERS
-              value: '3'
-
-             ## enter the comma separated target nodes name
-            - name: TARGET_NODES
-              value:

CI/scenarios/node_mem_engine.yaml

@@ -1,28 +0,0 @@
-apiVersion: litmuschaos.io/v1alpha1
-kind: ChaosEngine
-metadata:
-  name: nginx-chaos
-  namespace: litmus
-spec:
-  # It can be delete/retain
-  jobCleanUpPolicy: 'retain'
-  # It can be active/stop
-  engineState: 'active'
-  chaosServiceAccount: litmus-sa
-  experiments:
-    - name: node-memory-hog
-      spec:
-        components:
-          env:
-            # set chaos duration (in sec) as desired
-            - name: TOTAL_CHAOS_DURATION
-              value: '10'
-
-            ## Specify the size as percent of total node capacity Ex: '30'
-            ## Note: For consuming memory in mebibytes change the variable to MEMORY_CONSUMPTION_MEBIBYTES
-            - name: MEMORY_CONSUMPTION_PERCENTAGE
-              value: '30'
-
-            # ENTER THE COMMA SEPARATED TARGET NODES NAME
-            - name: TARGET_NODES
-              value: $WORKER_NODE

CI/scenarios/node_mem_engine_node.yaml

@@ -1,28 +0,0 @@
-apiVersion: litmuschaos.io/v1alpha1
-kind: ChaosEngine
-metadata:
-  name: nginx-chaos
-  namespace: litmus
-spec:
-  # It can be delete/retain
-  jobCleanUpPolicy: 'retain'
-  # It can be active/stop
-  engineState: 'active'
-  chaosServiceAccount: litmus-sa
-  experiments:
-    - name: node-memory-hog
-      spec:
-        components:
-          env:
-            # set chaos duration (in sec) as desired
-            - name: TOTAL_CHAOS_DURATION
-              value: '10'
-
-            ## Specify the size as percent of total node capacity Ex: '30'
-            ## Note: For consuming memory in mebibytes change the variable to MEMORY_CONSUMPTION_MEBIBYTES
-            - name: MEMORY_CONSUMPTION_PERCENTAGE
-              value: '30'
-
-            # ENTER THE COMMA SEPARATED TARGET NODES NAME
-            - name: TARGET_NODES
-              value:

CI/scenarios/time_scenarios.yml

@@ -1,5 +0,0 @@
-time_scenarios:
-- action: skew_time
-  object_type: pod
-  label_selector: k8s-app=etcd
-  container_name: ""

CI/templates/container_scenario_pod.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: container
+  labels:
+    scenario: container
+spec:
+  hostNetwork: true
+  containers:
+  - name: fedtools
+    image: docker.io/fedora/tools
+    command:
+    - /bin/sh
+    - -c
+    - |
+      sleep infinity

CI/templates/outage_pod.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: outage
+  labels:
+    scenario: outage
+spec:
+  hostNetwork: true
+  containers:
+  - name: fedtools
+    image: docker.io/fedora/tools
+    command:
+    - /bin/sh
+    - -c
+    - |
+      sleep infinity

CI/templates/service_hijacking.yaml

@@ -0,0 +1,29 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: nginx
+  labels:
+    app.kubernetes.io/name: proxy
+spec:
+  containers:
+  - name: nginx
+    image: nginx:stable
+    ports:
+      - containerPort: 80
+        name: http-web-svc
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: nginx-service
+spec:
+  selector:
+    app.kubernetes.io/name: proxy
+  type: NodePort
+  ports:
+  - name: name-of-service-port
+    protocol: TCP
+    port: 80
+    targetPort: http-web-svc
+    nodePort: 30036

CI/templates/time_pod.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: time-skew
+  labels:
+    scenario: time-skew
+spec:
+  hostNetwork: true
+  containers:
+  - name: fedtools
+    image: docker.io/fedora/tools
+    command:
+    - /bin/sh
+    - -c
+    - |
+      sleep infinity

CI/tests/common.sh

@@ -1,18 +1,26 @@
 ERRORED=false
 
 function finish {
-    if [ $? -eq 1 ] && [ $ERRORED != "true" ]
+    if [ $? != 0 ] && [ $ERRORED != "true" ]
     then
         error
     fi
 }
 
 function error {
-    echo "Error caught."
-    ERRORED=true
+    exit_code=$?
+    if [ $exit_code == 1 ]
+    then
+      echo "Error caught."
+      ERRORED=true
+    elif [ $exit_code == 2 ]
+    then
+      echo "Run with exit code 2 detected, it is expected, wrapping the exit code with 0 to avoid pipeline failure"
+      exit 0
+    fi
 }
 
 function get_node {
-  worker_node=$(oc get nodes --no-headers | grep worker | head -n 1)
+  worker_node=$(kubectl get nodes --no-headers | grep worker | head -n 1)
   export WORKER_NODE=$worker_node
 }

CI/tests/functional_tests

@@ -0,0 +1 @@
+

CI/tests/my_tests

@@ -1 +0,0 @@
-test_net_chaos

CI/tests/test_app_outages.sh

@@ -7,9 +7,11 @@ trap finish EXIT
 
 
 function functional_test_app_outage {
-
-  export scenario_type="application_outages"
-  export scenario_file="CI/scenarios/app_outage.yaml"
+  yq -i '.application_outage.duration=10' scenarios/openshift/app_outage.yaml
+  yq -i '.application_outage.pod_selector={"scenario":"outage"}' scenarios/openshift/app_outage.yaml
+  yq -i '.application_outage.namespace="default"' scenarios/openshift/app_outage.yaml
+  export scenario_type="application_outages_scenarios"
+  export scenario_file="scenarios/openshift/app_outage.yaml"
   export post_config=""
   envsubst < CI/config/common_test_config.yaml > CI/config/app_outage.yaml
   python3 -m coverage run -a run_kraken.py -c CI/config/app_outage.yaml

CI/tests/test_app_outages_gh.sh

@@ -1,21 +0,0 @@
-set -xeEo pipefail
-
-source CI/tests/common.sh
-
-trap error ERR
-trap finish EXIT
-
-
-function functional_test_app_outage {
-  [ -z $DEPLOYMENT_NAME ] && echo "[ERR] DEPLOYMENT_NAME variable not set, failing." && exit 1
-  yq -i '.application_outage.pod_selector={"app":"'$DEPLOYMENT_NAME'"}' CI/scenarios/app_outage.yaml
-  yq -i '.application_outage.namespace="'$NAMESPACE'"' CI/scenarios/app_outage.yaml
-  export scenario_type="application_outages"
-  export scenario_file="CI/scenarios/app_outage.yaml"
-  export post_config=""
-  envsubst < CI/config/common_test_config.yaml > CI/config/app_outage.yaml
-  python3 -m coverage run -a run_kraken.py -c CI/config/app_outage.yaml
-  echo "App outage scenario test: Success"
-}
-
-functional_test_app_outage

CI/tests/test_arca_cpu_hog.sh

@@ -0,0 +1,19 @@
+set -xeEo pipefail
+
+source CI/tests/common.sh
+
+trap error ERR
+trap finish EXIT
+
+
+function functional_test_arca_cpu_hog {
+  yq -i '.input_list[0].node_selector={"kubernetes.io/hostname":"kind-worker2"}' scenarios/kube/cpu-hog/input.yaml
+  export scenario_type="hog_scenarios"
+  export scenario_file="scenarios/kube/cpu-hog/input.yaml"
+  export post_config=""
+  envsubst < CI/config/common_test_config.yaml > CI/config/arca_cpu_hog.yaml
+  python3 -m coverage run -a run_kraken.py -c CI/config/arca_cpu_hog.yaml
+  echo "Arcaflow CPU Hog: Success"
+}
+
+functional_test_arca_cpu_hog

CI/tests/test_arca_io_hog.sh

@@ -0,0 +1,19 @@
+set -xeEo pipefail
+
+source CI/tests/common.sh
+
+trap error ERR
+trap finish EXIT
+
+
+function functional_test_arca_io_hog {
+  yq -i '.input_list[0].node_selector={"kubernetes.io/hostname":"kind-worker2"}' scenarios/kube/io-hog/input.yaml
+  export scenario_type="hog_scenarios"
+  export scenario_file="scenarios/kube/io-hog/input.yaml"
+  export post_config=""
+  envsubst < CI/config/common_test_config.yaml > CI/config/arca_io_hog.yaml
+  python3 -m coverage run -a run_kraken.py -c CI/config/arca_io_hog.yaml
+  echo "Arcaflow IO Hog: Success"
+}
+
+functional_test_arca_io_hog

CI/tests/test_arca_memory_hog.sh

@@ -0,0 +1,19 @@
+set -xeEo pipefail
+
+source CI/tests/common.sh
+
+trap error ERR
+trap finish EXIT
+
+
+function functional_test_arca_memory_hog {
+  yq -i '.input_list[0].node_selector={"kubernetes.io/hostname":"kind-worker2"}' scenarios/kube/memory-hog/input.yaml
+  export scenario_type="hog_scenarios"
+  export scenario_file="scenarios/kube/memory-hog/input.yaml"
+  export post_config=""
+  envsubst < CI/config/common_test_config.yaml > CI/config/arca_memory_hog.yaml
+  python3 -m coverage run -a run_kraken.py -c CI/config/arca_memory_hog.yaml
+  echo "Arcaflow Memory Hog: Success"
+}
+
+functional_test_arca_memory_hog

CI/tests/test_container.sh

@@ -8,9 +8,11 @@ trap finish EXIT
 pod_file="CI/scenarios/hello_pod.yaml"
 
 function functional_test_container_crash {
-
+  yq -i '.scenarios[0].namespace="default"' scenarios/openshift/container_etcd.yml
+  yq -i '.scenarios[0].label_selector="scenario=container"' scenarios/openshift/container_etcd.yml
+  yq -i '.scenarios[0].container_name="fedtools"' scenarios/openshift/container_etcd.yml
   export scenario_type="container_scenarios"
-  export scenario_file="- CI/scenarios/container_scenario.yml"
+  export scenario_file="scenarios/openshift/container_etcd.yml"
   export post_config=""
   envsubst < CI/config/common_test_config.yaml > CI/config/container_config.yaml
 

CI/tests/test_cpu_hog.sh

@@ -1,20 +0,0 @@
-set -xeEo pipefail
-
-source CI/tests/common.sh
-
-trap error ERR
-trap finish EXIT
-
-
-function functional_test_litmus_cpu {
-
-  export scenario_type="litmus_scenarios"
-  export scenario_file="- scenarios/templates/litmus-rbac.yaml"
-  export post_config="- CI/scenarios/node_cpu_hog_engine_node.yaml"
-  envsubst < CI/config/common_test_config.yaml > CI/config/litmus_config.yaml
-  envsubst < CI/scenarios/node_cpu_hog_engine.yaml > CI/scenarios/node_cpu_hog_engine_node.yaml
-  python3 -m coverage run -a run_kraken.py -c CI/config/litmus_config.yaml
-  echo "Litmus scenario test: Success"
-}
-
-functional_test_litmus_cpu

CI/tests/test_cpu_hog_gh.sh

@@ -1,20 +0,0 @@
-set -xeEo pipefail
-
-source CI/tests/common.sh
-
-trap error ERR
-trap finish EXIT
-
-
-function functional_test_litmus_cpu {
-  [ -z $NODE_NAME ] && echo "[ERR] NODE_NAME variable not set, failing." && exit 1
-  yq -i ' .spec.experiments = [{"name": "node-cpu-hog", "spec":{"components":{"env":[{"name":"TOTAL_CHAOS_DURATION","value":"10"},{"name":"NODE_CPU_CORE","value":"1"},{"name":"NODES_AFFECTED_PERC","value":"30"},{"name":"TARGET_NODES","value":"'$NODE_NAME'"}]}}}]' CI/scenarios/node_cpu_hog_engine_node.yaml
-
-  cp CI/config/common_test_config.yaml CI/config/litmus_config.yaml
-  yq '.kraken.chaos_scenarios = [{"litmus_scenarios":[["scenarios/openshift/templates/litmus-rbac.yaml","CI/scenarios/node_cpu_hog_engine_node.yaml"]]}]' -i CI/config/litmus_config.yaml
-  
-  python3 -m coverage run -a run_kraken.py -c CI/config/litmus_config.yaml
-  echo "Litmus scenario test: Success"
-}
-
-functional_test_litmus_cpu

CI/tests/test_io_hog.sh

@@ -1,20 +0,0 @@
-set -xeEo pipefail
-
-source CI/tests/common.sh
-
-trap error ERR
-trap finish EXIT
-
-
-function functional_test_litmus_io {
-
-  export scenario_type="litmus_scenarios"
-  export scenario_file="- scenarios/templates/litmus-rbac.yaml"
-  export post_config="- CI/scenarios/node_io_engine_node.yaml"
-  envsubst < CI/config/common_test_config.yaml > CI/config/litmus_config.yaml
-  envsubst < CI/scenarios/node_io_engine.yaml > CI/scenarios/node_io_engine_node.yaml
-  python3 -m coverage run -a run_kraken.py -c CI/config/litmus_config.yaml
-  echo "Litmus scenario test: Success"
-}
-
-functional_test_litmus_io

CI/tests/test_io_hog_gh.sh

@@ -1,19 +0,0 @@
-set -xeEo pipefail
-
-source CI/tests/common.sh
-
-trap error ERR
-trap finish EXIT
-
-
-function functional_test_litmus_io {
-  [ -z $NODE_NAME ] && echo "[ERR] NODE_NAME variable not set, failing." && exit 1
-  yq -i ' .spec.experiments = [{"name": "node-io-stress", "spec":{"components":{"env":[{"name":"TOTAL_CHAOS_DURATION","value":"10"},{"name":"FILESYSTEM_UTILIZATION_PERCENTAGE","value":"100"},{"name":"CPU","value":"1"},{"name":"NUMBER_OF_WORKERS","value":"3"},{"name":"TARGET_NODES","value":"'$NODE_NAME'"}]}}}]' CI/scenarios/node_io_engine_node.yaml
-  cp CI/config/common_test_config.yaml CI/config/litmus_config.yaml
-  yq '.kraken.chaos_scenarios = [{"litmus_scenarios":[["scenarios/openshift/templates/litmus-rbac.yaml","CI/scenarios/node_io_engine_node.yaml"]]}]' -i CI/config/litmus_config.yaml
-  
-  python3 -m coverage run -a run_kraken.py -c CI/config/litmus_config.yaml
-  echo "Litmus scenario test: Success"
-}
-
-functional_test_litmus_io

CI/tests/test_mem_hog.sh

@@ -1,20 +0,0 @@
-set -xeEo pipefail
-
-source CI/tests/common.sh
-
-trap error ERR
-trap finish EXIT
-
-
-function functional_test_litmus_mem {
-
-  export scenario_type="litmus_scenarios"
-  export scenario_file="- scenarios/templates/litmus-rbac.yaml"
-  export post_config="- CI/scenarios/node_mem_engine_node.yaml"
-  envsubst < CI/config/common_test_config.yaml > CI/config/litmus_config.yaml
-  envsubst < CI/scenarios/node_mem_engine.yaml > CI/scenarios/node_mem_engine_node.yaml
-  python3 -m coverage run -a run_kraken.py -c CI/config/litmus_config.yaml
-  echo "Litmus scenario $1 test: Success"
-}
-
-functional_test_litmus_mem "- CI/scenarios/node_mem_engine.yaml"

CI/tests/test_mem_hog_gh.sh

@@ -1,19 +0,0 @@
-set -xeEo pipefail
-
-source CI/tests/common.sh
-
-trap error ERR
-trap finish EXIT
-
-
-function functional_test_litmus_mem {
-  [ -z $NODE_NAME ] && echo "[ERR] NODE_NAME variable not set, failing." && exit 1
-  yq -i ' .spec.experiments = [{"name": "node-io-stress", "spec":{"components":{"env":[{"name":"TOTAL_CHAOS_DURATION","value":"10"},{"name":"CPU","value":"1"},{"name":"TARGET_NODES","value":"'$NODE_NAME'"}]}}}]' CI/scenarios/node_mem_engine_node.yaml
-  cp CI/config/common_test_config.yaml CI/config/litmus_config.yaml
-  yq '.kraken.chaos_scenarios = [{"litmus_scenarios":[["scenarios/openshift/templates/litmus-rbac.yaml","CI/scenarios/node_mem_engine_node.yaml"]]}]' -i CI/config/litmus_config.yaml
-
-  python3 -m coverage run -a run_kraken.py -c CI/config/litmus_config.yaml
-  echo "Litmus scenario test: Success"
-}
-
-functional_test_litmus_mem

CI/tests/test_namespace.sh

@@ -6,13 +6,14 @@ trap error ERR
 trap finish EXIT
 
 function funtional_test_namespace_deletion {
-  export scenario_type="namespace_scenarios"
-  export scenario_file="-  CI/scenarios/network_diagnostics_namespace.yaml"
+  export scenario_type="service_disruption_scenarios"
+  export scenario_file="scenarios/openshift/ingress_namespace.yaml"
   export post_config=""
-  yq '.scenarios.[0].namespace="^openshift-network-diagnostics$"' -i CI/scenarios/network_diagnostics_namespace.yaml
+  yq '.scenarios[0].namespace="^namespace-scenario$"' -i scenarios/openshift/ingress_namespace.yaml
+  yq '.scenarios[0].wait_time=30' -i scenarios/openshift/ingress_namespace.yaml
+  yq '.scenarios[0].action="delete"' -i scenarios/openshift/ingress_namespace.yaml
   envsubst < CI/config/common_test_config.yaml > CI/config/namespace_config.yaml
   python3 -m coverage run -a run_kraken.py -c CI/config/namespace_config.yaml
-  echo $?
   echo "Namespace scenario test: Success"
 }
 

CI/tests/test_net_chaos.sh

@@ -7,9 +7,16 @@ trap finish EXIT
 
 
 function functional_test_network_chaos {
+  yq -i '.network_chaos.duration=10' scenarios/openshift/network_chaos.yaml
+  yq -i '.network_chaos.node_name="kind-worker2"' scenarios/openshift/network_chaos.yaml
+  yq -i '.network_chaos.egress.bandwidth="100mbit"' scenarios/openshift/network_chaos.yaml
+  yq -i 'del(.network_chaos.interfaces)' scenarios/openshift/network_chaos.yaml
+  yq -i 'del(.network_chaos.label_selector)' scenarios/openshift/network_chaos.yaml
+  yq -i 'del(.network_chaos.egress.latency)' scenarios/openshift/network_chaos.yaml
+  yq -i 'del(.network_chaos.egress.loss)' scenarios/openshift/network_chaos.yaml
 
-  export scenario_type="network_chaos"
-  export scenario_file="CI/scenarios/network_chaos.yaml"
+  export scenario_type="network_chaos_scenarios"
+  export scenario_file="scenarios/openshift/network_chaos.yaml"
   export post_config=""
   envsubst < CI/config/common_test_config.yaml > CI/config/network_chaos.yaml
   python3 -m coverage run -a run_kraken.py -c CI/config/network_chaos.yaml

CI/tests/test_service_hijacking.sh

@@ -0,0 +1,114 @@
+set -xeEo pipefail
+
+source CI/tests/common.sh
+
+trap error ERR
+trap finish EXIT
+# port mapping has been configured in kind-config.yml
+SERVICE_URL=http://localhost:8888
+PAYLOAD_GET_1="{ \
+  \"status\":\"internal server error\" \
+}"
+STATUS_CODE_GET_1=500
+
+PAYLOAD_PATCH_1="resource patched"
+STATUS_CODE_PATCH_1=201
+
+PAYLOAD_POST_1="{ \
+  \"status\": \"unauthorized\" \
+}"
+STATUS_CODE_POST_1=401
+
+PAYLOAD_GET_2="{ \
+  \"status\":\"resource created\" \
+}"
+STATUS_CODE_GET_2=201
+
+PAYLOAD_PATCH_2="bad request"
+STATUS_CODE_PATCH_2=400
+
+PAYLOAD_POST_2="not found"
+STATUS_CODE_POST_2=404
+
+JSON_MIME="application/json"
+TEXT_MIME="text/plain; charset=utf-8"
+
+function functional_test_service_hijacking {
+
+  export scenario_type="service_hijacking_scenarios"
+  export scenario_file="scenarios/kube/service_hijacking.yaml"
+  export post_config=""
+  envsubst < CI/config/common_test_config.yaml > CI/config/service_hijacking.yaml
+  python3 -m coverage run -a run_kraken.py -c CI/config/service_hijacking.yaml  > /dev/null 2>&1 &
+  PID=$!
+  #Waiting the hijacking to have effect
+  COUNTER=0
+  while [ `curl -X GET -s -o /dev/null -I -w "%{http_code}" $SERVICE_URL/list/index.php` == 404 ]
+   do
+    echo "waiting scenario to kick in."
+    sleep 1
+    COUNTER=$((COUNTER+1))
+    [ $COUNTER -eq "100" ] && echo "maximum number of retry reached, test failed" && exit 1
+  done
+
+  #Checking Step 1 GET on /list/index.php
+  OUT_GET="`curl -X GET -s $SERVICE_URL/list/index.php`"
+  OUT_CONTENT=`curl -X GET -s -o /dev/null -I -w "%{content_type}" $SERVICE_URL/list/index.php`
+  OUT_STATUS_CODE=`curl -X GET -s -o /dev/null -I -w "%{http_code}" $SERVICE_URL/list/index.php`
+  [ "${PAYLOAD_GET_1//[$'\t\r\n ']}" == "${OUT_GET//[$'\t\r\n ']}" ] && echo "Step 1 GET Payload OK" || (echo "Payload did not match. Test failed." && exit 1)
+  [ "$OUT_STATUS_CODE" == "$STATUS_CODE_GET_1" ] && echo "Step 1 GET Status Code OK" || (echo " Step 1 GET status code did not match. Test failed." && exit 1)
+  [ "$OUT_CONTENT" == "$JSON_MIME" ] && echo "Step 1 GET MIME OK" || (echo " Step 1 GET MIME did not match. Test failed." && exit 1)
+
+  #Checking Step 1 POST on /list/index.php
+  OUT_POST="`curl -s -X POST $SERVICE_URL/list/index.php`"
+  OUT_STATUS_CODE=`curl -X POST -s -o /dev/null -I -w "%{http_code}" $SERVICE_URL/list/index.php`
+  OUT_CONTENT=`curl -X POST -s -o /dev/null -I -w "%{content_type}" $SERVICE_URL/list/index.php`
+  [ "${PAYLOAD_POST_1//[$'\t\r\n ']}" == "${OUT_POST//[$'\t\r\n ']}" ] && echo "Step 1 POST Payload OK" || (echo "Payload did not match. Test failed." && exit 1)
+  [ "$OUT_STATUS_CODE" == "$STATUS_CODE_POST_1" ] && echo "Step 1 POST Status Code OK" || (echo "Step 1 POST status code did not match. Test failed." && exit 1)
+  [ "$OUT_CONTENT" == "$JSON_MIME" ] && echo "Step 1 POST MIME OK" || (echo " Step 1 POST MIME did not match. Test failed." && exit 1)
+
+  #Checking Step 1 PATCH on /patch
+  OUT_PATCH="`curl -s -X PATCH $SERVICE_URL/patch`"
+  OUT_STATUS_CODE=`curl -X PATCH -s -o /dev/null -I -w "%{http_code}" $SERVICE_URL/patch`
+  OUT_CONTENT=`curl -X PATCH -s -o /dev/null -I -w "%{content_type}" $SERVICE_URL/patch`
+  [ "${PAYLOAD_PATCH_1//[$'\t\r\n ']}" == "${OUT_PATCH//[$'\t\r\n ']}" ] && echo "Step 1 PATCH Payload OK" || (echo "Payload did not match. Test failed." && exit 1)
+  [ "$OUT_STATUS_CODE" == "$STATUS_CODE_PATCH_1" ] && echo "Step 1 PATCH Status Code OK" || (echo "Step 1 PATCH status code did not match. Test failed." && exit 1)
+  [ "$OUT_CONTENT" == "$TEXT_MIME" ] && echo "Step 1 PATCH MIME OK" || (echo " Step 1 PATCH MIME did not match. Test failed." && exit 1)
+  # wait for the next step
+  sleep 16
+
+  #Checking Step 2 GET on /list/index.php
+  OUT_GET="`curl -X GET -s $SERVICE_URL/list/index.php`"
+  OUT_CONTENT=`curl -X GET -s -o /dev/null -I -w "%{content_type}" $SERVICE_URL/list/index.php`
+  OUT_STATUS_CODE=`curl -X GET -s -o /dev/null -I -w "%{http_code}" $SERVICE_URL/list/index.php`
+  [ "${PAYLOAD_GET_2//[$'\t\r\n ']}" == "${OUT_GET//[$'\t\r\n ']}" ] && echo "Step 2 GET Payload OK" || (echo "Step 2 GET Payload did not match. Test failed." && exit 1)
+  [ "$OUT_STATUS_CODE" == "$STATUS_CODE_GET_2" ] && echo "Step 2 GET Status Code OK" || (echo "Step 2 GET status code did not match. Test failed." && exit 1)
+  [ "$OUT_CONTENT" == "$JSON_MIME" ] && echo "Step 2 GET MIME OK" || (echo " Step 2 GET MIME did not match. Test failed." && exit 1)
+
+  #Checking Step 2 POST on /list/index.php
+  OUT_POST="`curl -s -X POST $SERVICE_URL/list/index.php`"
+  OUT_CONTENT=`curl -X POST -s -o /dev/null -I -w "%{content_type}" $SERVICE_URL/list/index.php`
+  OUT_STATUS_CODE=`curl -X POST -s -o /dev/null -I -w "%{http_code}" $SERVICE_URL/list/index.php`
+  [ "${PAYLOAD_POST_2//[$'\t\r\n ']}" == "${OUT_POST//[$'\t\r\n ']}" ] && echo "Step 2 POST Payload OK" || (echo "Step 2 POST Payload did not match. Test failed." && exit 1)
+  [ "$OUT_STATUS_CODE" == "$STATUS_CODE_POST_2" ] && echo "Step 2 POST Status Code OK" || (echo "Step 2 POST status code did not match. Test failed." && exit 1)
+  [ "$OUT_CONTENT" == "$TEXT_MIME" ] && echo "Step 2 POST MIME OK" || (echo " Step 2 POST MIME did not match. Test failed." && exit 1)
+
+  #Checking Step 2 PATCH on /patch
+  OUT_PATCH="`curl -s -X PATCH $SERVICE_URL/patch`"
+  OUT_CONTENT=`curl -X PATCH -s -o /dev/null -I -w "%{content_type}" $SERVICE_URL/patch`
+  OUT_STATUS_CODE=`curl -X PATCH -s -o /dev/null -I -w "%{http_code}" $SERVICE_URL/patch`
+  [ "${PAYLOAD_PATCH_2//[$'\t\r\n ']}" == "${OUT_PATCH//[$'\t\r\n ']}" ] && echo "Step 2 PATCH Payload OK" || (echo "Step 2 PATCH Payload did not match. Test failed." && exit 1)
+  [ "$OUT_STATUS_CODE" == "$STATUS_CODE_PATCH_2" ] && echo "Step 2 PATCH Status Code OK" || (echo "Step 2 PATCH status code did not match. Test failed." && exit 1)
+  [ "$OUT_CONTENT" == "$TEXT_MIME" ] && echo "Step 2 PATCH MIME OK" || (echo " Step 2 PATCH MIME did not match. Test failed." && exit 1)
+  wait $PID
+
+  # now checking  if service has been restore correctly and nginx responds correctly
+  curl -s  $SERVICE_URL | grep nginx! && echo "BODY: Service restored!" || (echo "BODY: failed to restore service" && exit 1)
+  OUT_STATUS_CODE=`curl -X GET -s -o /dev/null -I -w "%{http_code}" $SERVICE_URL`
+  [ "$OUT_STATUS_CODE" == "200" ] && echo "STATUS_CODE: Service restored!" || (echo "STATUS_CODE: failed to restore service" && exit 1)
+
+  echo "Service Hijacking Chaos test: Success"
+}
+
+
+functional_test_service_hijacking

CI/tests/test_telemetry.sh

@@ -0,0 +1,36 @@
+set -xeEo pipefail
+
+source CI/tests/common.sh
+
+trap error ERR
+trap finish EXIT
+
+
+function functional_test_telemetry {
+  AWS_CLI=`which aws`
+  [ -z "$AWS_CLI" ]&& echo "AWS cli not found in path" && exit 1
+  [ -z "$AWS_BUCKET" ] && echo "AWS bucket not set in environment" && exit 1
+
+  export RUN_TAG="funtest-telemetry"
+  yq -i '.telemetry.enabled=True' CI/config/common_test_config.yaml
+  yq -i '.telemetry.full_prometheus_backup=True' CI/config/common_test_config.yaml
+  yq -i '.performance_monitoring.check_critical_alerts=True' CI/config/common_test_config.yaml
+  yq -i '.performance_monitoring.prometheus_url="http://localhost:9090"' CI/config/common_test_config.yaml
+  yq -i '.telemetry.run_tag=env(RUN_TAG)' CI/config/common_test_config.yaml
+
+  export scenario_type="hog_scenarios"
+  export scenario_file="scenarios/kube/cpu-hog/input.yaml"
+  export post_config=""
+  envsubst < CI/config/common_test_config.yaml > CI/config/telemetry.yaml
+  retval=$(python3 -m coverage run -a run_kraken.py -c CI/config/telemetry.yaml)
+  RUN_FOLDER=`cat CI/out/test_telemetry.out | grep amazonaws.com | sed -rn "s#.*https:\/\/.*\/files/(.*)#\1#p"`
+  $AWS_CLI s3 ls "s3://$AWS_BUCKET/$RUN_FOLDER/" | awk '{ print $4 }' > s3_remote_files
+  echo "checking if telemetry files are uploaded on s3"
+  cat s3_remote_files | grep critical-alerts-00.log || ( echo "FAILED: critical-alerts-00.log not uploaded"  && exit 1 )
+  cat s3_remote_files | grep prometheus-00.tar || ( echo "FAILED: prometheus backup not uploaded"  && exit 1 )
+  cat s3_remote_files | grep telemetry.json || ( echo "FAILED: telemetry.json not uploaded"  && exit 1 )
+  echo "all files uploaded!"
+  echo "Telemetry Collection: Success"
+}
+
+functional_test_telemetry

CI/tests/test_time.sh

@@ -7,8 +7,12 @@ trap finish EXIT
 
 
 function functional_test_time_scenario {
+  yq -i '.time_scenarios[0].label_selector="scenario=time-skew"' scenarios/openshift/time_scenarios_example.yml
+  yq -i '.time_scenarios[0].container_name=""' scenarios/openshift/time_scenarios_example.yml
+  yq -i '.time_scenarios[0].namespace="default"' scenarios/openshift/time_scenarios_example.yml
+  yq -i '.time_scenarios[1].label_selector="kubernetes.io/hostname=kind-worker2"' scenarios/openshift/time_scenarios_example.yml
   export scenario_type="time_scenarios"
-  export scenario_file="CI/scenarios/time_scenarios.yml"
+  export scenario_file="scenarios/openshift/time_scenarios_example.yml"
   export post_config=""
   envsubst < CI/config/common_test_config.yaml > CI/config/time_config.yaml
 

CODE_OF_CONDUCT.md

@@ -1,21 +1,50 @@
-# Contributor Covenant Code of Conduct
+## CNCF Community Code of Conduct v1.3
 
-## Our Pledge
+Other languages available:
+- [Arabic/العربية](code-of-conduct-languages/ar.md)
+- [Bulgarian/Български](code-of-conduct-languages/bg.md)
+- [Chinese/中文](code-of-conduct-languages/zh.md)
+- [Czech/Česky](code-of-conduct-languages/cs.md)
+- [Farsi/فارسی](code-of-conduct-languages/fa.md)
+- [French/Français](code-of-conduct-languages/fr.md)
+- [German/Deutsch](code-of-conduct-languages/de.md)
+- [Hindi/हिन्दी](code-of-conduct-languages/hi.md)
+- [Indonesian/Bahasa Indonesia](code-of-conduct-languages/id.md)
+- [Italian/Italiano](code-of-conduct-languages/it.md)
+- [Japanese/日本語](code-of-conduct-languages/jp.md)
+- [Korean/한국어](code-of-conduct-languages/ko.md)
+- [Polish/Polski](code-of-conduct-languages/pl.md)
+- [Portuguese/Português](code-of-conduct-languages/pt.md)
+- [Russian/Русский](code-of-conduct-languages/ru.md)
+- [Spanish/Español](code-of-conduct-languages/es.md)
+- [Turkish/Türkçe](code-of-conduct-languages/tr.md)
+- [Ukrainian/Українська](code-of-conduct-languages/uk.md)
+- [Vietnamese/Tiếng Việt](code-of-conduct-languages/vi.md)
 
-We as members, contributors, and leaders pledge to make participation in our
-community a harassment-free experience for everyone, regardless of age, body
-size, visible or invisible disability, ethnicity, sex characteristics, gender
-identity and expression, level of experience, education, socio-economic status,
-nationality, personal appearance, race, religion, or sexual identity
-and orientation.
+### Community Code of Conduct
 
-We pledge to act and interact in ways that contribute to an open, welcoming,
-diverse, inclusive, and healthy community.
+As contributors, maintainers, and participants in the CNCF community, and in the interest of fostering
+an open and welcoming community, we pledge to respect all people who participate or contribute
+through reporting issues, posting feature requests, updating documentation,
+submitting pull requests or patches, attending conferences or events, or engaging in other community or project activities.
+
+We are committed to making participation in the CNCF community a harassment-free experience for everyone, regardless of age, body size, caste, disability, ethnicity, level of experience, family status, gender, gender identity and expression, marital status, military or veteran status, nationality, personal appearance, race, religion, sexual orientation, socioeconomic status, tribe, or any other dimension of diversity.
+
+## Scope
+
+This code of conduct applies:
+* within project and community spaces,
+* in other spaces when an individual CNCF community participant's words or actions are directed at or are about a CNCF project, the CNCF community, or another CNCF community participant.
+
+### CNCF Events
+
+CNCF events that are produced by the Linux Foundation with professional events staff are governed by the Linux Foundation [Events Code of Conduct](https://events.linuxfoundation.org/code-of-conduct/) available on the event page. This is designed to be used in conjunction with the CNCF Code of Conduct.
 
 ## Our Standards
 
-Examples of behavior that contributes to a positive environment for our
-community include:
+The CNCF Community is open, inclusive and respectful. Every member of our community has the right to have their identity respected.
+
+Examples of behavior that contributes to a positive environment include but are not limited to:
 
 * Demonstrating empathy and kindness toward other people
 * Being respectful of differing opinions, viewpoints, and experiences
@@ -24,104 +53,52 @@ community include:
   and learning from the experience
 * Focusing on what is best not just for us as individuals, but for the
   overall community
+* Using welcoming and inclusive language
 
-Examples of unacceptable behavior include:
 
-* The use of sexualized language or imagery, and sexual attention or
-  advances of any kind
+Examples of unacceptable behavior include but are not limited to:
+
+* The use of sexualized language or imagery
 * Trolling, insulting or derogatory comments, and personal or political attacks
-* Public or private harassment
+* Public or private harassment in any form
 * Publishing others' private information, such as a physical or email
   address, without their explicit permission
+* Violence, threatening violence, or encouraging others to engage in violent behavior
+* Stalking or following someone without their consent
+* Unwelcome physical contact
+* Unwelcome sexual or romantic attention or advances
 * Other conduct which could reasonably be considered inappropriate in a
   professional setting
 
-## Enforcement Responsibilities
+The following behaviors are also prohibited:
+* Providing knowingly false or misleading information in connection with a Code of Conduct investigation or otherwise intentionally tampering with an investigation.
+* Retaliating against a person because they reported an incident or provided information about an incident as a witness.
 
-Community leaders are responsible for clarifying and enforcing our standards of
-acceptable behavior and will take appropriate and fair corrective action in
-response to any behavior that they deem inappropriate, threatening, offensive,
-or harmful.
+Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct.
+By adopting this Code of Conduct, project maintainers commit themselves to fairly and consistently applying these principles to every aspect
+of managing a CNCF project.
+Project maintainers who do not follow or enforce the Code of Conduct may be temporarily or permanently removed from the project team.
 
-Community leaders have the right and responsibility to remove, edit, or reject
-comments, commits, code, wiki edits, issues, and other contributions that are
-not aligned to this Code of Conduct, and will communicate reasons for moderation
-decisions when appropriate.
+## Reporting
 
-## Scope
+For incidents occurring in the Kubernetes community, contact the [Kubernetes Code of Conduct Committee](https://git.k8s.io/community/committee-code-of-conduct) via <conduct@kubernetes.io>. You can expect a response within three business days.
 
-This Code of Conduct applies within all community spaces, and also applies when
-an individual is officially representing the community in public spaces.
-Examples of representing our community include using an official e-mail address,
-posting via an official social media account, or acting as an appointed
-representative at an online or offline event.
+For other projects, or for incidents that are project-agnostic or impact multiple CNCF projects, please contact the [CNCF Code of Conduct Committee](https://www.cncf.io/conduct/committee/) via <conduct@cncf.io>.  Alternatively, you can contact any of the individual members of the [CNCF Code of Conduct Committee](https://www.cncf.io/conduct/committee/) to submit your report. For more detailed instructions on how to submit a report, including how to submit a report anonymously, please see our [Incident Resolution Procedures](https://github.com/cncf/foundation/blob/main/code-of-conduct/coc-incident-resolution-procedures.md). You can expect a response within three business days.
+
+For incidents occurring at CNCF event that is produced by the Linux Foundation, please contact <eventconduct@cncf.io>.
 
 ## Enforcement
 
-Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported to the community leaders responsible for enforcement.
-All complaints will be reviewed and investigated promptly and fairly.
+Upon review and investigation of a reported incident, the CoC response team that has jurisdiction will determine what action is appropriate based on this Code of Conduct and its related documentation.
 
-All community leaders are obligated to respect the privacy and security of the
-reporter of any incident.
+For information about which Code of Conduct incidents are handled by project leadership, which incidents are handled by the CNCF Code of Conduct Committee, and which incidents are handled by the Linux Foundation (including its events team), see our [Jurisdiction Policy](https://github.com/cncf/foundation/blob/main/code-of-conduct/coc-committee-jurisdiction-policy.md).
 
-## Enforcement Guidelines
+## Amendments
 
-Community leaders will follow these Community Impact Guidelines in determining
-the consequences for any action they deem in violation of this Code of Conduct:
+Consistent with the CNCF Charter, any substantive changes to this Code of Conduct must be approved by the Technical Oversight Committee.
 
-### 1. Correction
+## Acknowledgements
 
-**Community Impact**: Use of inappropriate language or other behavior deemed
-unprofessional or unwelcome in the community.
-
-**Consequence**: A private, written warning from community leaders, providing
-clarity around the nature of the violation and an explanation of why the
-behavior was inappropriate. A public apology may be requested.
-
-### 2. Warning
-
-**Community Impact**: A violation through a single incident or series
-of actions.
-
-**Consequence**: A warning with consequences for continued behavior. No
-interaction with the people involved, including unsolicited interaction with
-those enforcing the Code of Conduct, for a specified period of time. This
-includes avoiding interactions in community spaces as well as external channels
-like social media. Violating these terms may lead to a temporary or
-permanent ban.
-
-### 3. Temporary Ban
-
-**Community Impact**: A serious violation of community standards, including
-sustained inappropriate behavior.
-
-**Consequence**: A temporary ban from any sort of interaction or public
-communication with the community for a specified period of time. No public or
-private interaction with the people involved, including unsolicited interaction
-with those enforcing the Code of Conduct, is allowed during this period.
-Violating these terms may lead to a permanent ban.
-
-### 4. Permanent Ban
-
-**Community Impact**: Demonstrating a pattern of violation of community
-standards, including sustained inappropriate behavior,  harassment of an
-individual, or aggression toward or disparagement of classes of individuals.
-
-**Consequence**: A permanent ban from any sort of public interaction within
-the community.
-
-## Attribution
-
-This Code of Conduct is adapted from the [Contributor Covenant][homepage],
-version 2.0, available at
-https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
-
-Community Impact Guidelines were inspired by [Mozilla's code of conduct
-enforcement ladder](https://github.com/mozilla/diversity).
-
-[homepage]: https://www.contributor-covenant.org
-
-For answers to common questions about this code of conduct, see the FAQ at
-https://www.contributor-covenant.org/faq. Translations are available at
-https://www.contributor-covenant.org/translations.
+This Code of Conduct is adapted from the Contributor Covenant
+(http://contributor-covenant.org), version 2.0 available at
+http://contributor-covenant.org/version/2/0/code_of_conduct/

README.md

@@ -1,11 +1,12 @@
 # Krkn aka Kraken
-[![Docker Repository on Quay](https://quay.io/repository/redhat-chaos/krkn/status "Docker Repository on Quay")](https://quay.io/repository/redhat-chaos/krkn?tab=tags&tag=latest)
-![Workflow-Status](https://github.com/redhat-chaos/krkn/actions/workflows/docker-image.yml/badge.svg)
+![Workflow-Status](https://github.com/krkn-chaos/krkn/actions/workflows/docker-image.yml/badge.svg)
+![coverage](https://krkn-chaos.github.io/krkn-lib-docs/coverage_badge_krkn.svg)
+![action](https://github.com/krkn-chaos/krkn/actions/workflows/tests.yml/badge.svg)
 
 ![Krkn logo](media/logo.png)
 
-Chaos and resiliency testing tool for Kubernetes and OpenShift.
-Kraken injects deliberate failures into Kubernetes/OpenShift clusters to check if it is resilient to turbulent conditions.
+Chaos and resiliency testing tool for Kubernetes.
+Kraken injects deliberate failures into Kubernetes clusters to check if it is resilient to turbulent conditions.
 
 
 ### Workflow
@@ -18,67 +19,59 @@ Kraken injects deliberate failures into Kubernetes/OpenShift clusters to check i
 ### Chaos Testing Guide
 [Guide](docs/index.md) encapsulates:
 - Test methodology that needs to be embraced.
-- Best practices that an OpenShift cluster, platform and applications running on top of it should take into account for best user experience, performance, resilience and reliability.
+- Best practices that an Kubernetes cluster, platform and applications running on top of it should take into account for best user experience, performance, resilience and reliability.
 - Tooling.
 - Scenarios supported.
 - Test environment recommendations as to how and where to run chaos tests.
 - Chaos testing in practice.
 
-The guide is hosted at https://redhat-chaos.github.io/krkn.
+The guide is hosted at https://krkn-chaos.github.io/krkn.
 
 
 ### How to Get Started
 Instructions on how to setup, configure and run Kraken can be found at [Installation](docs/installation.md).
 
+You may consider utilizing the chaos recommendation tool prior to initiating the chaos runs to profile the application service(s) under test. This tool discovers a list of Krkn scenarios with a high probability of causing failures or disruptions to your application service(s). The tool can be accessed at [Chaos-Recommender](utils/chaos_recommender/README.md).
+
 See the [getting started doc](docs/getting_started.md) on support on how to get started with your own custom scenario or editing current scenarios for your specific usage.
 
 After installation, refer back to the below sections for supported scenarios and how to tweak the kraken config to load them on your cluster.
 
 
 #### Running Kraken with minimal configuration tweaks
-For cases where you want to run Kraken with minimal configuration changes, refer to [Kraken-hub](https://github.com/redhat-chaos/krkn-hub). One use case is CI integration where you do not want to carry around different configuration files for the scenarios.
-
-### Setting up infrastructure dependencies
-Kraken indexes the metrics specified in the profile into Elasticsearch in addition to leveraging Cerberus for understanding the health of the Kubernetes/OpenShift cluster under test. More information on the features is documented below. The infrastructure pieces can be easily installed and uninstalled by running:
-
-```
-$ cd kraken
-$ podman-compose up or $ docker-compose up      # Spins up the containers specified in the docker-compose.yml file present in the run directory.
-$ podman-compose down or $ docker-compose down  # Delete the containers installed.
-```
-This will manage the Cerberus and Elasticsearch containers on the host on which you are running Kraken.
-
-**NOTE**: Make sure you have enough resources (memory and disk) on the machine on top of which the containers are running as Elasticsearch is resource intensive. Cerberus monitors the system components by default, the [config](config/cerberus.yaml) can be tweaked to add applications namespaces, routes and other components to monitor as well. The command will keep running until killed since detached mode is not supported as of now.
+For cases where you want to run Kraken with minimal configuration changes, refer to [krkn-hub](https://github.com/krkn-chaos/krkn-hub). One use case is CI integration where you do not want to carry around different configuration files for the scenarios.
 
 
 ### Config
 Instructions on how to setup the config and the options supported can be found at [Config](docs/config.md).
 
 
-### Kubernetes/OpenShift chaos scenarios supported
+### Kubernetes chaos scenarios supported
 
-Scenario type               | Kubernetes    | OpenShift          
---------------------------- | ------------- |--------------------|  
-[Pod Scenarios](docs/pod_scenarios.md) | :heavy_check_mark: | :heavy_check_mark: |
-[Pod Network Scenarios](docs/pod_network_scenarios.md) | :x: | :heavy_check_mark: |
-[Container Scenarios](docs/container_scenarios.md) | :heavy_check_mark: | :heavy_check_mark: |
-[Node Scenarios](docs/node_scenarios.md) | :heavy_check_mark: | :heavy_check_mark: |
-[Time Scenarios](docs/time_scenarios.md) | :x: | :heavy_check_mark: |
-[Hog Scenarios: CPU, Memory](docs/arcaflow_scenarios.md) | :heavy_check_mark: | :heavy_check_mark: |
-[Cluster Shut Down Scenarios](docs/cluster_shut_down_scenarios.md) | :heavy_check_mark: | :heavy_check_mark: |
-[Service Disruption Scenarios](docs/service_disruption_scenarios.md.md) | :heavy_check_mark: | :heavy_check_mark: |
-[Zone Outage Scenarios](docs/zone_outage.md) | :heavy_check_mark: | :heavy_check_mark: |
-[Application_outages](docs/application_outages.md) | :heavy_check_mark: | :heavy_check_mark: |
-[PVC scenario](docs/pvc_scenario.md) | :heavy_check_mark: | :heavy_check_mark: |
-[Network_Chaos](docs/network_chaos.md) | :heavy_check_mark: | :heavy_check_mark: |
-[ManagedCluster Scenarios](docs/managedcluster_scenarios.md) | :heavy_check_mark: | :question:         |
+Scenario type               | Kubernetes    
+--------------------------- | ------------- | 
+[Pod Scenarios](docs/pod_scenarios.md) | :heavy_check_mark: |
+[Pod Network Scenarios](docs/pod_network_scenarios.md) | :x: |
+[Container Scenarios](docs/container_scenarios.md) | :heavy_check_mark: |
+[Node Scenarios](docs/node_scenarios.md) | :heavy_check_mark: |
+[Time Scenarios](docs/time_scenarios.md) | :heavy_check_mark: |
+[Hog Scenarios: CPU, Memory](docs/arcaflow_scenarios.md) | :heavy_check_mark: |
+[Cluster Shut Down Scenarios](docs/cluster_shut_down_scenarios.md) | :heavy_check_mark: |
+[Service Disruption Scenarios](docs/service_disruption_scenarios.md.md) | :heavy_check_mark: |
+[Zone Outage Scenarios](docs/zone_outage.md) | :heavy_check_mark: |
+[Application_outages](docs/application_outages.md) | :heavy_check_mark: |
+[PVC scenario](docs/pvc_scenario.md) | :heavy_check_mark: |
+[Network_Chaos](docs/network_chaos.md) | :heavy_check_mark: |
+[ManagedCluster Scenarios](docs/managedcluster_scenarios.md) | :heavy_check_mark: |
+[Service Hijacking Scenarios](docs/service_hijacking_scenarios.md) | :heavy_check_mark: |
+[SYN Flood Scenarios](docs/syn_flood_scenarios.md) | :heavy_check_mark: |
 
 
 ### Kraken scenario pass/fail criteria and report
-It is important to make sure to check if the targeted component recovered from the chaos injection and also if the Kubernetes/OpenShift cluster is healthy as failures in one component can have an adverse impact on other components. Kraken does this by:
+It is important to make sure to check if the targeted component recovered from the chaos injection and also if the Kubernetes cluster is healthy as failures in one component can have an adverse impact on other components. Kraken does this by:
 - Having built in checks for pod and node based scenarios to ensure the expected number of replicas and nodes are up. It also supports running custom scripts with the checks.
-- Leveraging [Cerberus](https://github.com/openshift-scale/cerberus) to monitor the cluster under test and consuming the aggregated go/no-go signal to determine pass/fail post chaos. It is highly recommended to turn on the Cerberus health check feature available in Kraken. Instructions on installing and setting up Cerberus can be found [here](https://github.com/openshift-scale/cerberus#installation) or can be installed from Kraken using the [instructions](https://github.com/redhat-chaos/krkn#setting-up-infrastructure-dependencies). Once Cerberus is up and running, set cerberus_enabled to True and cerberus_url to the url where Cerberus publishes go/no-go signal in the Kraken config file. Cerberus can monitor [application routes](https://github.com/redhat-chaos/cerberus/blob/main/docs/config.md#watch-routes) during the chaos and fails the run if it encounters downtime as it is a potential downtime in a customers, or users environment as well. It is especially important during the control plane chaos scenarios including the API server, Etcd, Ingress etc. It can be enabled by setting `check_applicaton_routes: True` in the [Kraken config](https://github.com/redhat-chaos/krkn/blob/main/config/config.yaml) provided application routes are being monitored in the [cerberus config](https://github.com/redhat-chaos/krkn/blob/main/config/cerberus.yaml).
-- Leveraging [kube-burner](docs/alerts.md) alerting feature to fail the runs in case of critical alerts.
+- Leveraging [Cerberus](https://github.com/krkn-chaos/cerberus) to monitor the cluster under test and consuming the aggregated go/no-go signal to determine pass/fail post chaos. It is highly recommended to turn on the Cerberus health check feature available in Kraken. Instructions on installing and setting up Cerberus can be found [here](https://github.com/openshift-scale/cerberus#installation) or can be installed from Kraken using the [instructions](https://github.com/krkn-chaos/krkn#setting-up-infrastructure-dependencies). Once Cerberus is up and running, set cerberus_enabled to True and cerberus_url to the url where Cerberus publishes go/no-go signal in the Kraken config file. Cerberus can monitor [application routes](https://github.com/redhat-chaos/cerberus/blob/main/docs/config.md#watch-routes) during the chaos and fails the run if it encounters downtime as it is a potential downtime in a customers, or users environment as well. It is especially important during the control plane chaos scenarios including the API server, Etcd, Ingress etc. It can be enabled by setting `check_applicaton_routes: True` in the [Kraken config](https://github.com/redhat-chaos/krkn/blob/main/config/config.yaml) provided application routes are being monitored in the [cerberus config](https://github.com/redhat-chaos/krkn/blob/main/config/cerberus.yaml).
+- Leveraging built-in alert collection feature to fail the runs in case of critical alerts.
 
 ### Signaling
 In CI runs or any external job it is useful to stop Kraken once a certain test or state gets reached. We created a way to signal to kraken to pause the chaos or stop it completely using a signal posted to a port of your choice.
@@ -92,10 +85,6 @@ More detailed information on enabling and leveraging this feature can be found [
 Monitoring the Kubernetes/OpenShift cluster to observe the impact of Kraken chaos scenarios on various components is key to find out the bottlenecks as it is important to make sure the cluster is healthy in terms if both recovery as well as performance during/after the failure has been injected. Instructions on enabling it can be found [here](docs/performance_dashboards.md).
 
 
-### Scraping and storing metrics long term
-Kraken supports capturing metrics for the duration of the scenarios defined in the config and indexes then into Elasticsearch to be able to store and evaluate the state of the runs long term. The indexed metrics can be visualized with the help of Grafana. It uses [Kube-burner](https://github.com/cloud-bulldozer/kube-burner) under the hood. The metrics to capture need to be defined in a metrics profile which Kraken consumes to query prometheus ( installed by default in OpenShift ) with the start and end timestamp of the run. Information on enabling and leveraging this feature can be found [here](docs/metrics.md).
-
-
 ### SLOs validation during and post chaos
 - In addition to checking the recovery and health of the cluster and components under test, Kraken takes in a profile with the Prometheus expressions to validate and alerts, exits with a non-zero return code depending on the severity set. This feature can be used to determine pass/fail or alert on abnormalities observed in the cluster based on the metrics. 
 - Kraken also provides ability to check if any critical alerts are firing in the cluster post chaos and pass/fail's. 
@@ -105,14 +94,17 @@ Information on enabling and leveraging this feature can be found [here](docs/SLO
 
 ### OCM / ACM integration
 
-Kraken supports injecting faults into [Open Cluster Management (OCM)](https://open-cluster-management.io/) and [Red Hat Advanced Cluster Management for Kubernetes (ACM)](https://www.redhat.com/en/technologies/management/advanced-cluster-management) managed clusters through [ManagedCluster Scenarios](docs/managedcluster_scenarios.md).
+Kraken supports injecting faults into [Open Cluster Management (OCM)](https://open-cluster-management.io/) and [Red Hat Advanced Cluster Management for Kubernetes (ACM)](https://www.krkn.com/en/technologies/management/advanced-cluster-management) managed clusters through [ManagedCluster Scenarios](docs/managedcluster_scenarios.md).
+
 
 ### Blogs and other useful resources
 - Blog post on introduction to Kraken: https://www.openshift.com/blog/introduction-to-kraken-a-chaos-tool-for-openshift/kubernetes
 - Discussion and demo on how Kraken can be leveraged to ensure OpenShift is reliable, performant and scalable: https://www.youtube.com/watch?v=s1PvupI5sD0&ab_channel=OpenShift
 - Blog post emphasizing the importance of making Chaos part of Performance and Scale runs to mimic the production environments: https://www.openshift.com/blog/making-chaos-part-of-kubernetes/openshift-performance-and-scalability-tests
 - Blog post on findings from Chaos test runs: https://cloud.redhat.com/blog/openshift/kubernetes-chaos-stories
-
+- Discussion with CNCF TAG App Delivery on Krkn workflow, features and addition to CNCF sandbox: [Github](https://github.com/cncf/sandbox/issues/44), [Tracker](https://github.com/cncf/tag-app-delivery/issues/465), [recording](https://www.youtube.com/watch?v=nXQkBFK_MWc&t=722s)
+- Blog post on supercharging chaos testing using AI integration in Krkn: https://www.redhat.com/en/blog/supercharging-chaos-testing-using-ai
+- Blog post announcing Krkn joining CNCF Sandbox: https://www.redhat.com/en/blog/krknchaos-joining-cncf-sandbox
 
 ### Roadmap
 Enhancements being planned can be found in the [roadmap](ROADMAP.md).
@@ -127,7 +119,14 @@ If adding a new scenario or tweaking the main config, be sure to add in updates
 Please read [this file]((CI/README.md#adding-a-test-case)) for more information on updates.
 
 
+### Scenario Plugin Development
+
+If you're gearing up to develop new scenarios, take a moment to review our 
+[Scenario Plugin API Documentation](docs/scenario_plugin_api.md). 
+It’s the perfect starting point to tap into your chaotic creativity!
+
 ### Community
-Key Members(slack_usernames/full name): paigerube14/Paige Rubendall, mffiedler/Mike Fiedler, ravielluri/Naga Ravi Chaitanya Elluri.
-* [**#krkn on Kubernetes Slack**](https://kubernetes.slack.com)
-* [**#forum-chaos on CoreOS Slack internal to Red Hat**](https://coreos.slack.com)
+Key Members(slack_usernames/full name): paigerube14/Paige Rubendall, mffiedler/Mike Fiedler, tsebasti/Tullio Sebastiani, yogi/Yogananth Subramanian, sahil/Sahil Shah, pradeep/Pradeep Surisetty and ravielluri/Naga Ravi Chaitanya Elluri.
+* [**#krkn on Kubernetes Slack**](https://kubernetes.slack.com/messages/C05SFMHRWK1)
+
+The Linux Foundation® (TLF) has registered trademarks and uses trademarks. For a list of TLF trademarks, see [Trademark Usage](https://www.linuxfoundation.org/legal/trademark-usage).

ROADMAP.md

@@ -2,12 +2,15 @@
 
 Following are a list of enhancements that we are planning to work on adding support in Krkn. Of course any help/contributions are greatly appreciated.
 
-- [ ] [Ability to run multiple chaos scenarios in parallel under load to mimic real world outages](https://github.com/redhat-chaos/krkn/issues/424)
-- [x] [Centralized storage for chaos experiments artifacts](https://github.com/redhat-chaos/krkn/issues/423)
-- [ ] [Support for causing DNS outages](https://github.com/redhat-chaos/krkn/issues/394)
-- [ ] [Support for pod level network traffic shaping](https://github.com/redhat-chaos/krkn/issues/393)
-- [ ] [Ability to visualize the metrics that are being captured by Kraken and stored in Elasticsearch](https://github.com/redhat-chaos/krkn/issues/124)
-- [ ] Support for running all the scenarios of Kraken on Kubernetes distribution - see https://github.com/redhat-chaos/krkn/issues/185, https://github.com/redhat-chaos/krkn/issues/186
-- [ ] Continue to improve [Chaos Testing Guide](https://redhat-chaos.github.io/krkn) in terms of adding best practices, test environment recommendations and scenarios to make sure the OpenShift platform, as well the applications running on top it, are resilient and performant under chaotic conditions.
-- [ ] [Switch documentation references to Kubernetes](https://github.com/redhat-chaos/krkn/issues/495)
-- [ ] [OCP and Kubernetes functionalities segregation](https://github.com/redhat-chaos/krkn/issues/497)
+- [ ] [Ability to run multiple chaos scenarios in parallel under load to mimic real world outages](https://github.com/krkn-chaos/krkn/issues/424)
+- [x] [Centralized storage for chaos experiments artifacts](https://github.com/krkn-chaos/krkn/issues/423)
+- [ ] [Support for causing DNS outages](https://github.com/krkn-chaos/krkn/issues/394)
+- [x] [Chaos recommender](https://github.com/krkn-chaos/krkn/tree/main/utils/chaos-recommender) to suggest scenarios having probability of impacting the service under test using profiling results 
+- [] Chaos AI integration to improve test coverage while reducing fault space to save costs and execution time
+- [x] [Support for pod level network traffic shaping](https://github.com/krkn-chaos/krkn/issues/393)
+- [ ] [Ability to visualize the metrics that are being captured by Kraken and stored in Elasticsearch](https://github.com/krkn-chaos/krkn/issues/124)
+- [x] Support for running all the scenarios of Kraken on Kubernetes distribution - see https://github.com/krkn-chaos/krkn/issues/185, https://github.com/redhat-chaos/krkn/issues/186
+- [x] Continue to improve [Chaos Testing Guide](https://krkn-chaos.github.io/krkn) in terms of adding best practices, test environment recommendations and scenarios to make sure the OpenShift platform, as well the applications running on top it, are resilient and performant under chaotic conditions.
+- [x] [Switch documentation references to Kubernetes](https://github.com/krkn-chaos/krkn/issues/495)
+- [x] [OCP and Kubernetes functionalities segregation](https://github.com/krkn-chaos/krkn/issues/497)
+- [x] [Krknctl - client for running Krkn scenarios with ease](https://github.com/krkn-chaos/krknctl)

config/alerts.yaml

@@ -0,0 +1,129 @@
+# etcd
+
+- expr: avg_over_time(histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket[2m]))[10m:]) > 0.01
+  description: 10 minutes avg. 99th etcd fsync latency on {{$labels.pod}} higher than 10ms. {{$value}}s
+  severity: warning
+
+- expr: avg_over_time(histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket[2m]))[10m:]) > 1
+  description: 10 minutes avg. 99th etcd fsync latency on {{$labels.pod}} higher than 1s. {{$value}}s
+  severity: error
+
+- expr: avg_over_time(histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket[2m]))[10m:]) > 0.03
+  description: 10 minutes avg. 99th etcd commit latency on {{$labels.pod}} higher than 30ms. {{$value}}s
+  severity: warning
+
+- expr: rate(etcd_server_leader_changes_seen_total[2m]) > 0
+  description: etcd leader changes observed
+  severity: warning
+
+- expr: (last_over_time(etcd_mvcc_db_total_size_in_bytes[5m]) / last_over_time(etcd_server_quota_backend_bytes[5m]))*100 > 95
+  description: etcd cluster database is running full. 
+  severity: critical
+  
+- expr: (last_over_time(etcd_mvcc_db_total_size_in_use_in_bytes[5m]) / last_over_time(etcd_mvcc_db_total_size_in_bytes[5m])) < 0.5
+  description: etcd database size in use is less than 50% of the actual allocated storage.
+  severity: warning  
+
+- expr: rate(etcd_server_proposals_failed_total{job=~".*etcd.*"}[15m]) > 5
+  description: etcd cluster has high number of proposal failures.
+  severity: warning 
+
+- expr: histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~".*etcd.*"}[5m])) > 0.15
+  description: etcd cluster member communication is slow.
+  severity: warning 
+
+- expr: histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job=~".*etcd.*", grpc_method!="Defragment", grpc_type="unary"}[5m])) without(grpc_type)) > 0.15
+  description: etcd grpc requests are slow.
+  severity: critical
+      
+- expr: 100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) without (grpc_type, grpc_code) / sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) without (grpc_type, grpc_code) > 5
+  description: etcd cluster has high number of failed grpc requests.
+  severity: critical      
+      
+- expr: etcd_server_has_leader{job=~".*etcd.*"} == 0
+  description: etcd cluster has no leader.
+  severity: warning    
+    
+- expr: sum(up{job=~".*etcd.*"} == bool 1) without (instance) < ((count(up{job=~".*etcd.*"}) without (instance) + 1) / 2)
+  description: etcd cluster has insufficient number of members.
+  severity: warning      
+
+- expr: max without (endpoint) ( sum without (instance) (up{job=~".*etcd.*"} == bool 0) or count without (To) ( sum without (instance) (rate(etcd_network_peer_sent_failures_total{job=~".*etcd.*"}[120s])) > 0.01 )) > 0
+  description: etcd cluster members are down.
+  severity: warning  
+
+# API server
+- expr: avg_over_time(histogram_quantile(0.99, sum(irate(apiserver_request_duration_seconds_bucket{apiserver="kube-apiserver", verb=~"POST|PUT|DELETE|PATCH", subresource!~"log|exec|portforward|attach|proxy"}[2m])) by (le, resource, verb))[10m:]) > 1
+  description: 10 minutes avg. 99th mutating API call latency for {{$labels.verb}}/{{$labels.resource}} higher than 1 second. {{$value}}s
+  severity: error
+
+- expr: avg_over_time(histogram_quantile(0.99, sum(irate(apiserver_request_duration_seconds_bucket{apiserver="kube-apiserver", verb=~"LIST|GET", subresource!~"log|exec|portforward|attach|proxy", scope="resource"}[2m])) by (le, resource, verb, scope))[5m:]) > 1
+  description: 5 minutes avg. 99th read-only API call latency for {{$labels.verb}}/{{$labels.resource}} in scope {{$labels.scope}} higher than 1 second. {{$value}}s
+  severity: error
+
+- expr: avg_over_time(histogram_quantile(0.99, sum(irate(apiserver_request_duration_seconds_bucket{apiserver="kube-apiserver", verb=~"LIST|GET", subresource!~"log|exec|portforward|attach|proxy", scope="namespace"}[2m])) by (le, resource, verb, scope))[5m:]) > 5
+  description: 5 minutes avg. 99th read-only API call latency for {{$labels.verb}}/{{$labels.resource}} in scope {{$labels.scope}} higher than 5 seconds. {{$value}}s
+  severity: error
+
+- expr: avg_over_time(histogram_quantile(0.99, sum(irate(apiserver_request_duration_seconds_bucket{apiserver="kube-apiserver", verb=~"LIST|GET", subresource!~"log|exec|portforward|attach|proxy", scope="cluster"}[2m])) by (le, resource, verb, scope))[5m:]) > 30
+  description: 5 minutes avg. 99th read-only API call latency for {{$labels.verb}}/{{$labels.resource}} in scope {{$labels.scope}} higher than 30 seconds. {{$value}}s
+  severity: error
+
+# Control plane pods
+
+- expr: up{job=~"crio|kubelet"} == 0
+  description: "{{$labels.node}}/{{$labels.job}} down"
+  severity: warning
+
+- expr: up{job="ovnkube-node"} == 0
+  description: "{{$labels.instance}}/{{$labels.pod}} {{$labels.job}} down"
+  severity: warning
+
+# Service sync latency
+- expr: histogram_quantile(0.99, sum(rate(kubeproxy_network_programming_duration_seconds_bucket[2m])) by (le)) > 10
+  description: 99th Kubeproxy network programming latency higher than 10 seconds. {{$value}}s 
+  severity: warning
+
+# Prometheus alerts
+- expr: ALERTS{severity="critical", alertstate="firing"} > 0
+  description: Critical prometheus alert. {{$labels.alertname}}
+  severity: warning
+
+# etcd CPU and usage increase
+- expr: sum(rate(container_cpu_usage_seconds_total{image!='', namespace='openshift-etcd', container='etcd'}[1m])) * 100 / sum(machine_cpu_cores)  > 5
+  description: Etcd CPU usage increased significantly by {{$value}}%
+  severity: critical
+
+# etcd memory usage increase
+- expr: sum(deriv(container_memory_usage_bytes{image!='', namespace='openshift-etcd', container='etcd'}[5m])) * 100 / sum(node_memory_MemTotal_bytes) > 5
+  description: Etcd memory usage increased significantly by {{$value}}%
+  severity: critical
+
+# Openshift API server CPU and memory usage increase
+- expr: sum(rate(container_cpu_usage_seconds_total{image!='', namespace='openshift-apiserver', container='openshift-apiserver'}[1m])) * 100 / sum(machine_cpu_cores) > 5
+  description: openshift apiserver cpu usage increased significantly by {{$value}}%
+  severity: critical
+
+- expr: (sum(deriv(container_memory_usage_bytes{namespace='openshift-apiserver', container='openshift-apiserver'}[5m]))) * 100 / sum(node_memory_MemTotal_bytes) > 5
+  description: openshift apiserver memory usage increased significantly by {{$value}}%
+  severity: critical
+
+# Openshift kube API server CPU and memory usage increase
+- expr: sum(rate(container_cpu_usage_seconds_total{image!='', namespace='openshift-kube-apiserver', container='kube-apiserver'}[1m])) * 100 / sum(machine_cpu_cores) > 5
+  description: openshift apiserver cpu usage increased significantly by {{$value}}%
+  severity: critical
+
+- expr: (sum(deriv(container_memory_usage_bytes{namespace='openshift-kube-apiserver', container='kube-apiserver'}[5m]))) * 100 / sum(node_memory_MemTotal_bytes) > 5
+  description: openshift apiserver memory usage increased significantly by {{$value}}%
+  severity: critical
+
+# Master node CPU usage increase
+- expr: (sum((sum(deriv(pod:container_cpu_usage:sum{container="",pod!=""}[5m])) BY (namespace, pod) * on(pod, namespace) group_left(node) (node_namespace_pod:kube_pod_info:)  )  *  on(node) group_left(role) (max by (node) (kube_node_role{role="master"})))) * 100 / sum(machine_cpu_cores) > 5
+  description: master nodes cpu usage increased significantly by {{$value}}%
+  severity: critical
+
+# Master nodes memory usage increase
+- expr: (sum((sum(deriv(container_memory_usage_bytes{container="",pod!=""}[5m])) BY (namespace, pod) * on(pod, namespace) group_left(node) (node_namespace_pod:kube_pod_info:)  )  *  on(node) group_left(role) (max by (node) (kube_node_role{role="master"})))) * 100 / sum(node_memory_MemTotal_bytes) > 5
+  description: master nodes memory usage increased significantly by {{$value}}%
+  severity: critical
+

config/alerts_openshift.yaml

@@ -99,3 +99,41 @@
 - expr: ALERTS{severity="critical", alertstate="firing"} > 0
   description: Critical prometheus alert. {{$labels.alertname}}
   severity: warning
+
+# etcd CPU and usage increase
+- expr: sum(rate(container_cpu_usage_seconds_total{image!='', namespace='openshift-etcd', container='etcd'}[1m])) * 100 / sum(machine_cpu_cores)  > 5
+  description: Etcd CPU usage increased significantly by {{$value}}%
+  severity: critical
+
+# etcd memory usage increase
+- expr: sum(deriv(container_memory_usage_bytes{image!='', namespace='openshift-etcd', container='etcd'}[5m])) * 100 / sum(node_memory_MemTotal_bytes) > 5
+  description: Etcd memory usage increased significantly by {{$value}}%
+  severity: critical
+
+# Openshift API server CPU and memory usage increase
+- expr: sum(rate(container_cpu_usage_seconds_total{image!='', namespace='openshift-apiserver', container='openshift-apiserver'}[1m])) * 100 / sum(machine_cpu_cores) > 5
+  description: openshift apiserver cpu usage increased significantly by {{$value}}%
+  severity: critical
+
+- expr: (sum(deriv(container_memory_usage_bytes{namespace='openshift-apiserver', container='openshift-apiserver'}[5m]))) * 100 / sum(node_memory_MemTotal_bytes) > 5
+  description: openshift apiserver memory usage increased significantly by {{$value}}%
+  severity: critical
+
+# Openshift kube API server CPU and memory usage increase
+- expr: sum(rate(container_cpu_usage_seconds_total{image!='', namespace='openshift-kube-apiserver', container='kube-apiserver'}[1m])) * 100 / sum(machine_cpu_cores) > 5
+  description: openshift apiserver cpu usage increased significantly by {{$value}}%
+  severity: critical
+
+- expr: (sum(deriv(container_memory_usage_bytes{namespace='openshift-kube-apiserver', container='kube-apiserver'}[5m]))) * 100 / sum(node_memory_MemTotal_bytes) > 5
+  description: openshift apiserver memory usage increased significantly by {{$value}}%
+  severity: critical
+
+# Master node CPU usage increase
+- expr: (sum((sum(deriv(pod:container_cpu_usage:sum{container="",pod!=""}[5m])) BY (namespace, pod) * on(pod, namespace) group_left(node) (node_namespace_pod:kube_pod_info:)  )  *  on(node) group_left(role) (max by (node) (kube_node_role{role="master"})))) * 100 / sum(machine_cpu_cores) > 5
+  description: master nodes cpu usage increased significantly by {{$value}}%
+  severity: critical
+
+# Master nodes memory usage increase
+- expr: (sum((sum(deriv(container_memory_usage_bytes{container="",pod!=""}[5m])) BY (namespace, pod) * on(pod, namespace) group_left(node) (node_namespace_pod:kube_pod_info:)  )  *  on(node) group_left(role) (max by (node) (kube_node_role{role="master"})))) * 100 / sum(node_memory_MemTotal_bytes) > 5
+  description: master nodes memory usage increased significantly by {{$value}}%
+  severity: critical

config/config.yaml

@@ -1,6 +1,6 @@
 kraken:
-    distribution: openshift                                # Distribution can be kubernetes or openshift
-    kubeconfig_path: ~/.kube/config                        # Path to kubeconfig
+    distribution: kubernetes                                # Distribution can be kubernetes or openshift
+    kubeconfig_path: ~/.kube/config                     # Path to kubeconfig
     exit_on_failure: False                                 # Exit when a post action scenario fails
     publish_kraken_status: True                            # Can be accessed at http://0.0.0.0:8081
     signal_state: RUN                                      # Will wait for the RUN signal when set to PAUSE before running the scenarios, refer docs/signal.md for more details
@@ -8,47 +8,47 @@ kraken:
     port: 8081                                             # Signal port
     chaos_scenarios:
         # List of policies/chaos scenarios to load
-        - arcaflow_scenarios:
-            - scenarios/arcaflow/cpu-hog/input.yaml
-            - scenarios/arcaflow/memory-hog/input.yaml
-            - scenarios/arcaflow/io-hog/input.yaml
-        - application_outages:
+        - hog_scenarios:
+            - scenarios/kube/cpu-hog/input.yaml
+            - scenarios/kube/memory-hog/input.yaml
+            - scenarios/kube/io-hog/input.yaml
+            - scenarios/kube/io-hog/input.yaml
+        - application_outages_scenarios:
             - scenarios/openshift/app_outage.yaml
         - container_scenarios:                             # List of chaos pod scenarios to load
-            - -    scenarios/openshift/container_etcd.yml
-        - plugin_scenarios:
+            - scenarios/openshift/container_etcd.yml
+        - pod_network_scenarios:
+              - scenarios/openshift/network_chaos_ingress.yml
+              - scenarios/openshift/pod_network_outage.yml
+        - pod_disruption_scenarios:
             - scenarios/openshift/etcd.yml
             - scenarios/openshift/regex_openshift_pod_kill.yml
-            - scenarios/openshift/vmware_node_scenarios.yml
-            - scenarios/openshift/network_chaos_ingress.yml
             - scenarios/openshift/prom_kill.yml
-        - node_scenarios:                                  # List of chaos node scenarios to load
-            -   scenarios/openshift/node_scenarios_example.yml
-        - plugin_scenarios:
             - scenarios/openshift/openshift-apiserver.yml
             - scenarios/openshift/openshift-kube-apiserver.yml
+        - vmware_node_scenarios:
+            - scenarios/openshift/vmware_node_scenarios.yml
+        - ibmcloud_node_scenarios:
+            - scenarios/openshift/ibmcloud_node_scenarios.yml
+        - node_scenarios:                                  # List of chaos node scenarios to load
+            - scenarios/openshift/aws_node_scenarios.yml
         - time_scenarios:                                  # List of chaos time scenarios to load
             - scenarios/openshift/time_scenarios_example.yml
-        - litmus_scenarios:                                # List of litmus scenarios to load
-            - - scenarios/openshift/templates/litmus-rbac.yaml
-              - scenarios/openshift/node_cpu_hog_engine.yaml
-            - - scenarios/openshift/templates/litmus-rbac.yaml
-              - scenarios/openshift/node_mem_engine.yaml
-            - - scenarios/openshift/templates/litmus-rbac.yaml
-              - scenarios/openshift/node_io_engine.yaml
         - cluster_shut_down_scenarios:
-            - - scenarios/openshift/cluster_shut_down_scenario.yml
-              - scenarios/openshift/post_action_shut_down.py
+            - scenarios/openshift/cluster_shut_down_scenario.yml
         - service_disruption_scenarios:
-             - - scenarios/openshift/regex_namespace.yaml
-             - - scenarios/openshift/ingress_namespace.yaml
-               - scenarios/openshift/post_action_namespace.py
-        - zone_outages:
+             - scenarios/openshift/regex_namespace.yaml
+             - scenarios/openshift/ingress_namespace.yaml
+        - zone_outages_scenarios:
             - scenarios/openshift/zone_outage.yaml
         - pvc_scenarios:
             - scenarios/openshift/pvc_scenario.yaml
-        - network_chaos:
+        - network_chaos_scenarios:
             - scenarios/openshift/network_chaos.yaml
+        - service_hijacking_scenarios:
+              - scenarios/kube/service_hijacking.yaml
+        - syn_flood_scenarios:
+              - scenarios/kube/syn_flood.yaml
 
 cerberus:
     cerberus_enabled: False                                # Enable it when cerberus is previously installed
@@ -58,16 +58,27 @@ cerberus:
 performance_monitoring:
     deploy_dashboards: False                              # Install a mutable grafana and load the performance dashboards. Enable this only when running on OpenShift
     repo: "https://github.com/cloud-bulldozer/performance-dashboards.git"
-    kube_burner_binary_url: "https://github.com/cloud-bulldozer/kube-burner/releases/download/v1.7.0/kube-burner-1.7.0-Linux-x86_64.tar.gz"
-    capture_metrics: False
-    config_path: config/kube_burner.yaml                  # Define the Elasticsearch url and index name in this config
-    metrics_profile_path: config/metrics-aggregated.yaml
-    prometheus_url:                                       # The prometheus url/route is automatically obtained in case of OpenShift, please set it when the distribution is Kubernetes.
+    prometheus_url: ''                                      # The prometheus url/route is automatically obtained in case of OpenShift, please set it when the distribution is Kubernetes.
     prometheus_bearer_token:                              # The bearer token is automatically obtained in case of OpenShift, please set it when the distribution is Kubernetes. This is needed to authenticate with prometheus.
     uuid:                                                 # uuid for the run is generated by default if not set
     enable_alerts: False                                  # Runs the queries specified in the alert profile and displays the info or exits 1 when severity=error
-    alert_profile: config/alerts                          # Path or URL to alert profile with the prometheus queries
+    enable_metrics: False
+    alert_profile: config/alerts.yaml                          # Path or URL to alert profile with the prometheus queries
+    metrics_profile: config/metrics.yaml
     check_critical_alerts: False                          # When enabled will check prometheus for critical alerts firing post chaos
+elastic:
+    enable_elastic: False
+    collect_metrics: False
+    collect_alerts: False
+    verify_certs: False
+    elastic_url: ""                                         # To track results in elasticsearch, give url to server here; will post telemetry details when url and index not blank
+    elastic_port: 32766
+    username: "elastic"
+    password: "test"
+    metrics_index: "krkn-metrics"
+    alerts_index: "krkn-alerts"
+    telemetry_index: "krkn-telemetry"
+
 tunings:
     wait_duration: 60                                      # Duration to wait between each chaos scenario
     iterations: 1                                          # Number of times to execute the scenarios
@@ -76,14 +87,19 @@ telemetry:
     enabled: False                                           # enable/disables the telemetry collection feature
     api_url: https://ulnmf9xv7j.execute-api.us-west-2.amazonaws.com/production #telemetry service endpoint
     username: username                                      # telemetry service username
-    password: password                                      # telemetry service password
+    password: password                                    # telemetry service password
     prometheus_backup: True                                 # enables/disables prometheus data collection
+    prometheus_namespace: ""                                # namespace where prometheus is deployed (if distribution is kubernetes)
+    prometheus_container_name: ""                           # name of the prometheus container name (if distribution is kubernetes)
+    prometheus_pod_name: ""                                 # name of the prometheus pod (if distribution is kubernetes)
     full_prometheus_backup: False                           # if is set to False only the /prometheus/wal folder will be downloaded.
     backup_threads: 5                                       # number of telemetry download/upload threads
     archive_path: /tmp                                      # local path where the archive files will be temporarly stored
     max_retries: 0                                          # maximum number of upload retries (if 0 will retry forever)
     run_tag: ''                                             # if set, this will be appended to the run folder in the bucket (useful to group the runs)
-    archive_size: 10000                                     # the size of the prometheus data archive size in KB. The lower the size of archive is
+    archive_size: 500000
+    telemetry_group: ''                                     # if set will archive the telemetry in the S3 bucket on a folder named after the value, otherwise will use "default"
+    # the size of the prometheus data archive size in KB. The lower the size of archive is
                                                             # the higher the number of archive files will be produced and uploaded (and processed by backup_threads
                                                             # simultaneously).
                                                             # For unstable/slow connection is better to keep this value low
@@ -95,4 +111,9 @@ telemetry:
      - "kinit (\\d+/\\d+/\\d+\\s\\d{2}:\\d{2}:\\d{2})\\s+"          # kinit 2023/09/15 11:20:36 log
      - "(\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d+Z).+"      # 2023-09-15T11:20:36.123425532Z log
     oc_cli_path: /usr/bin/oc                                # optional, if not specified will be search in $PATH
+    events_backup: True                                     # enables/disables cluster events collection
+
+
+
+
 

config/config_kind.yaml

@@ -6,10 +6,6 @@ kraken:
     publish_kraken_status: True                            # Can be accessed at http://0.0.0.0:8081
     signal_state: RUN                                      # Will wait for the RUN signal when set to PAUSE before running the scenarios, refer docs/signal.md for more details
     signal_address: 0.0.0.0                                # Signal listening address
-    litmus_install: True                                   # Installs specified version, set to False if it's already setup
-    litmus_version: v1.13.6                                # Litmus version to install
-    litmus_uninstall: False                                # If you want to uninstall litmus if failure
-    litmus_uninstall_before_run: True                      # If you want to uninstall litmus before a new run starts
     chaos_scenarios:                                       # List of policies/chaos scenarios to load
         - plugin_scenarios:
             - scenarios/kind/scheduler.yml
@@ -24,15 +20,11 @@ cerberus:
 performance_monitoring:
     deploy_dashboards: False                              # Install a mutable grafana and load the performance dashboards. Enable this only when running on OpenShift
     repo: "https://github.com/cloud-bulldozer/performance-dashboards.git"
-    kube_burner_binary_url: "https://github.com/cloud-bulldozer/kube-burner/releases/download/v0.9.1/kube-burner-0.9.1-Linux-x86_64.tar.gz"
-    capture_metrics: False
-    config_path: config/kube_burner.yaml                  # Define the Elasticsearch url and index name in this config
-    metrics_profile_path: config/metrics-aggregated.yaml
     prometheus_url:                                       # The prometheus url/route is automatically obtained in case of OpenShift, please set it when the distribution is Kubernetes.
     prometheus_bearer_token:                              # The bearer token is automatically obtained in case of OpenShift, please set it when the distribution is Kubernetes. This is needed to authenticate with prometheus.
     uuid:                                                 # uuid for the run is generated by default if not set
     enable_alerts: False                                  # Runs the queries specified in the alert profile and displays the info or exits 1 when severity=error
-    alert_profile: config/alerts                          # Path to alert profile with the prometheus queries
+    alert_profile: config/alerts.yaml                          # Path to alert profile with the prometheus queries
 
 tunings:
     wait_duration: 60                                      # Duration to wait between each chaos scenario

config/config_kubernetes.yaml

@@ -5,13 +5,9 @@ kraken:
     port: 8081
     publish_kraken_status: True                            # Can be accessed at http://0.0.0.0:8081
     signal_state: RUN                                      # Will wait for the RUN signal when set to PAUSE before running the scenarios, refer docs/signal.md for more details
-    litmus_install: True                                   # Installs specified version, set to False if it's already setup
-    litmus_version: v1.13.6                                # Litmus version to install
-    litmus_uninstall: False                                # If you want to uninstall litmus if failure
-    litmus_uninstall_before_run: True                      # If you want to uninstall litmus before a new run starts
     chaos_scenarios:                                       # List of policies/chaos scenarios to load
         -   container_scenarios:                                 # List of chaos pod scenarios to load
-            - -    scenarios/kube/container_dns.yml
+            - scenarios/kube/container_dns.yml
         -   plugin_scenarios:
             - scenarios/kube/scheduler.yml
 
@@ -23,15 +19,11 @@ cerberus:
 performance_monitoring:
     deploy_dashboards: False                              # Install a mutable grafana and load the performance dashboards. Enable this only when running on OpenShift
     repo: "https://github.com/cloud-bulldozer/performance-dashboards.git"
-    kube_burner_binary_url: "https://github.com/cloud-bulldozer/kube-burner/releases/download/v0.9.1/kube-burner-0.9.1-Linux-x86_64.tar.gz"
-    capture_metrics: False
-    config_path: config/kube_burner.yaml                  # Define the Elasticsearch url and index name in this config
-    metrics_profile_path: config/metrics-aggregated.yaml
     prometheus_url:                                       # The prometheus url/route is automatically obtained in case of OpenShift, please set it when the distribution is Kubernetes.
     prometheus_bearer_token:                              # The bearer token is automatically obtained in case of OpenShift, please set it when the distribution is Kubernetes. This is needed to authenticate with prometheus.
     uuid:                                                 # uuid for the run is generated by default if not set
     enable_alerts: False                                  # Runs the queries specified in the alert profile and displays the info or exits 1 when severity=error
-    alert_profile: config/alerts                          # Path to alert profile with the prometheus queries
+    alert_profile: config/alerts.yaml                         # Path to alert profile with the prometheus queries
     check_critical_alerts: False                          # When enabled will check prometheus for critical alerts firing post chaos after soak time for the cluster to settle down
 tunings:
     wait_duration: 60                                      # Duration to wait between each chaos scenario

config/config_performance.yaml

@@ -6,27 +6,20 @@ kraken:
     signal_state: RUN                                      # Will wait for the RUN signal when set to PAUSE before running the scenarios, refer docs/signal.md for more details
     signal_address: 0.0.0.0                                # Signal listening address
     port: 8081                                             # Signal port
-    litmus_version: v1.13.6                                # Litmus version to install
-    litmus_uninstall: False                                # If you want to uninstall litmus if failure
-    litmus_uninstall_before_run: True                      # If you want to uninstall litmus before a new run starts
     chaos_scenarios:                                       # List of policies/chaos scenarios to load
         -   plugin_scenarios:                                 # List of chaos pod scenarios to load
             - scenarios/openshift/etcd.yml
             - scenarios/openshift/regex_openshift_pod_kill.yml
             - scenarios/openshift/prom_kill.yml
         -   node_scenarios:                                # List of chaos node scenarios to load
-            -   scenarios/openshift/node_scenarios_example.yml
+            -  scenarios/openshift/node_scenarios_example.yml
         -   plugin_scenarios:
             - scenarios/openshift/openshift-apiserver.yml
             - scenarios/openshift/openshift-kube-apiserver.yml
         -   time_scenarios:                                # List of chaos time scenarios to load
             - scenarios/openshift/time_scenarios_example.yml
-        -   litmus_scenarios:                              # List of litmus scenarios to load
-            - - https://hub.litmuschaos.io/api/chaos/1.10.0?file=charts/generic/node-cpu-hog/rbac.yaml
-              - scenarios/openshift/node_cpu_hog_engine.yaml
         -   cluster_shut_down_scenarios:
-            - - scenarios/openshift/cluster_shut_down_scenario.yml
-              - scenarios/openshift/post_action_shut_down.py
+            - scenarios/openshift/cluster_shut_down_scenario.yml
         -   service_disruption_scenarios:
             - scenarios/openshift/regex_namespace.yaml
             - scenarios/openshift/ingress_namespace.yaml
@@ -47,17 +40,44 @@ cerberus:
 performance_monitoring:
     deploy_dashboards: True                               # Install a mutable grafana and load the performance dashboards. Enable this only when running on OpenShift
     repo: "https://github.com/cloud-bulldozer/performance-dashboards.git"
-    kube_burner_binary_url: "https://github.com/cloud-bulldozer/kube-burner/releases/download/v0.9.1/kube-burner-0.9.1-Linux-x86_64.tar.gz"
     capture_metrics: True
-    config_path: config/kube_burner.yaml                  # Define the Elasticsearch url and index name in this config
     metrics_profile_path: config/metrics-aggregated.yaml
     prometheus_url:                                       # The prometheus url/route is automatically obtained in case of OpenShift, please set it when the distribution is Kubernetes.
     prometheus_bearer_token:                              # The bearer token is automatically obtained in case of OpenShift, please set it when the distribution is Kubernetes. This is needed to authenticate with prometheus.
     uuid:                                                 # uuid for the run is generated by default if not set
     enable_alerts: True                                   # Runs the queries specified in the alert profile and displays the info or exits 1 when severity=error
-    alert_profile: config/alerts                          # Path to alert profile with the prometheus queries
+    alert_profile: config/alerts.yaml                          # Path to alert profile with the prometheus queries
 
 tunings:
     wait_duration: 60                                      # Duration to wait between each chaos scenario
     iterations: 1                                          # Number of times to execute the scenarios
     daemon_mode: False                                     # Iterations are set to infinity which means that the kraken will cause chaos forever
+
+telemetry:
+    enabled: False                                           # enable/disables the telemetry collection feature
+    api_url: https://ulnmf9xv7j.execute-api.us-west-2.amazonaws.com/production #telemetry service endpoint
+    username: username                                      # telemetry service username
+    password: password                                      # telemetry service password
+    prometheus_backup: True                                 # enables/disables prometheus data collection
+    full_prometheus_backup: False                           # if is set to False only the /prometheus/wal folder will be downloaded.
+    backup_threads: 5                                       # number of telemetry download/upload threads
+    archive_path: /tmp                                      # local path where the archive files will be temporarly stored
+    max_retries: 0                                          # maximum number of upload retries (if 0 will retry forever)
+    run_tag: ''                                             # if set, this will be appended to the run folder in the bucket (useful to group the runs)
+    archive_size: 500000                                     # the size of the prometheus data archive size in KB. The lower the size of archive is
+                                                            # the higher the number of archive files will be produced and uploaded (and processed by backup_threads
+                                                            # simultaneously).
+                                                            # For unstable/slow connection is better to keep this value low
+                                                            # increasing the number of backup_threads, in this way, on upload failure, the retry will happen only on the
+                                                            # failed chunk without affecting the whole upload.
+    logs_backup: True
+    logs_filter_patterns:
+     - "(\\w{3}\\s\\d{1,2}\\s\\d{2}:\\d{2}:\\d{2}\\.\\d+).+"         # Sep 9 11:20:36.123425532
+     - "kinit (\\d+/\\d+/\\d+\\s\\d{2}:\\d{2}:\\d{2})\\s+"          # kinit 2023/09/15 11:20:36 log
+     - "(\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d+Z).+"      # 2023-09-15T11:20:36.123425532Z log
+    oc_cli_path: /usr/bin/oc                                # optional, if not specified will be search in $PATH
+elastic: 
+    elastic_url: ""                                         # To track results in elasticsearch, give url to server here; will post telemetry details when url and index not blank
+    elastic_index: ""                                       # Elastic search index pattern to post results to
+
+

config/kube_burner.yaml

@@ -1,15 +0,0 @@
----
-
-global:
-  writeToFile: true
-  metricsDirectory: collected-metrics
-  measurements:
-    - name: podLatency
-      esIndex: kraken
-
-  indexerConfig:
-    enabled: true
-    esServers: [http://0.0.0.0:9200]  # Please change this to the respective Elasticsearch in use if you haven't run the podman-compose command to setup the infrastructure containers
-    insecureSkipVerify: true
-    defaultIndex: kraken
-    type: elastic

config/recommender_config.yaml

@@ -0,0 +1,35 @@
+application: openshift-etcd
+namespaces: openshift-etcd
+labels: app=openshift-etcd
+kubeconfig: ~/.kube/config.yaml
+prometheus_endpoint: <Prometheus_Endpoint>
+auth_token: <Auth_Token>
+scrape_duration: 10m
+chaos_library: "kraken"
+log_level: INFO
+json_output_file: False
+json_output_folder_path:
+
+# for output purpose only do not change if not needed
+chaos_tests:
+  GENERIC:
+    - pod_failure
+    - container_failure
+    - node_failure
+    - zone_outage
+    - time_skew
+    - namespace_failure
+    - power_outage
+  CPU:
+    - node_cpu_hog
+  NETWORK:
+    - application_outage
+    - node_network_chaos
+    - pod_network_chaos
+  MEM:
+    - node_memory_hog
+    - pvc_disk_fill
+
+threshold: .7
+cpu_threshold: .5
+mem_threshold: .5

containers/Dockerfile

@@ -1,30 +0,0 @@
-# Dockerfile for kraken
-
-FROM mcr.microsoft.com/azure-cli:latest as azure-cli
-
-FROM registry.access.redhat.com/ubi8/ubi:latest
-
-LABEL org.opencontainers.image.authors="Red Hat OpenShift Chaos Engineering"
-
-ENV KUBECONFIG /root/.kube/config
-
-# Copy azure client binary from azure-cli image
-COPY --from=azure-cli /usr/local/bin/az /usr/bin/az
-
-# Install dependencies
-RUN yum install -y git python39 python3-pip jq gettext wget && \
-    python3.9 -m pip install -U pip && \
-    git clone https://github.com/redhat-chaos/krkn.git --branch v1.4.7 /root/kraken && \
-    mkdir -p /root/.kube && cd /root/kraken && \
-    pip3.9 install -r requirements.txt && \
-    pip3.9 install virtualenv && \
-    wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq && chmod +x /usr/bin/yq
-
-# Get Kubernetes and OpenShift clients from stable releases
-WORKDIR /tmp
-RUN wget https://mirror.openshift.com/pub/openshift-v4/clients/ocp/stable/openshift-client-linux.tar.gz && tar -xvf openshift-client-linux.tar.gz && cp oc /usr/local/bin/oc && cp kubectl /usr/local/bin/kubectl
-
-WORKDIR /root/kraken
-
-ENTRYPOINT ["python3.9", "run_kraken.py"]
-CMD ["--config=config/config.yaml"]

containers/Dockerfile-ppc64le

@@ -1,29 +0,0 @@
-# Dockerfile for kraken
-
-FROM ppc64le/centos:8
-
-FROM mcr.microsoft.com/azure-cli:latest as azure-cli
-
-LABEL org.opencontainers.image.authors="Red Hat OpenShift Chaos Engineering"
-
-ENV KUBECONFIG /root/.kube/config
-
-# Copy azure client binary from azure-cli image
-COPY --from=azure-cli /usr/local/bin/az /usr/bin/az
-
-# Install dependencies
-RUN yum install -y git python39 python3-pip jq gettext wget && \
-    python3.9 -m pip install -U pip && \
-    git clone https://github.com/redhat-chaos/krkn.git --branch v1.4.7 /root/kraken && \
-    mkdir -p /root/.kube && cd /root/kraken && \
-    pip3.9 install -r requirements.txt && \
-    pip3.9 install virtualenv && \
-    wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq && chmod +x /usr/bin/yq
-
-# Get Kubernetes and OpenShift clients from stable releases
-WORKDIR /tmp
-RUN wget https://mirror.openshift.com/pub/openshift-v4/clients/ocp/stable/openshift-client-linux.tar.gz && tar -xvf openshift-client-linux.tar.gz && cp oc /usr/local/bin/oc && cp kubectl /usr/local/bin/kubectl
-
-WORKDIR /root/kraken
-
-ENTRYPOINT python3.9 run_kraken.py --config=config/config.yaml

containers/Dockerfile.template

@@ -0,0 +1,60 @@
+# oc build
+FROM golang:1.22.5 AS oc-build
+RUN apt-get update && apt-get install -y --no-install-recommends libkrb5-dev
+WORKDIR /tmp
+RUN git clone --branch release-4.18 https://github.com/openshift/oc.git
+WORKDIR /tmp/oc
+RUN go mod edit -go 1.22.5 &&\
+    go get github.com/moby/buildkit@v0.12.5 &&\
+    go get github.com/containerd/containerd@v1.7.11&&\
+    go get github.com/docker/docker@v25.0.6&&\
+    go get github.com/opencontainers/runc@v1.1.14&&\
+    go mod tidy && go mod vendor
+RUN make GO_REQUIRED_MIN_VERSION:= oc
+
+FROM fedora:40
+ARG PR_NUMBER
+ARG TAG
+RUN groupadd -g 1001 krkn && useradd -m -u 1001 -g krkn krkn
+RUN dnf update -y
+
+ENV KUBECONFIG /home/krkn/.kube/config
+
+# install kubectl
+RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" &&\
+    cp kubectl /usr/local/bin/kubectl && chmod +x /usr/local/bin/kubectl &&\
+    cp kubectl /usr/bin/kubectl && chmod +x /usr/bin/kubectl
+
+# This overwrites any existing configuration in /etc/yum.repos.d/kubernetes.repo
+RUN dnf update && dnf install -y --setopt=install_weak_deps=False \
+    git python39 jq yq gettext wget which &&\
+    dnf clean all
+
+# copy oc client binary from oc-build image
+COPY --from=oc-build /tmp/oc/oc /usr/bin/oc
+
+# krkn build
+RUN git clone https://github.com/krkn-chaos/krkn.git /home/krkn/kraken && \
+    mkdir -p /home/krkn/.kube
+
+WORKDIR /home/krkn/kraken
+
+# default behaviour will be to build main
+# if it is a PR trigger the PR itself will be checked out
+RUN if [ -n "$PR_NUMBER" ]; then git fetch origin pull/${PR_NUMBER}/head:pr-${PR_NUMBER} && git checkout pr-${PR_NUMBER};fi
+# if it is a TAG trigger checkout the tag
+RUN if [ -n "$TAG" ]; then git checkout "$TAG";fi
+
+RUN python3.9 -m ensurepip
+RUN pip3.9 install -r requirements.txt
+RUN pip3.9 install jsonschema
+
+LABEL krknctl.title.global="Krkn Base Image"
+LABEL krknctl.description.global="This is the krkn base image."
+LABEL krknctl.input_fields.global='$KRKNCTL_INPUT'
+
+
+RUN chown -R krkn:krkn /home/krkn && chmod 755 /home/krkn
+USER krkn
+ENTRYPOINT ["python3.9", "run_kraken.py"]
+CMD ["--config=config/config.yaml"]

containers/README.md

@@ -1,53 +1,14 @@
 
 ### Kraken image
 
-  
-
 Container image gets automatically built by quay.io at [Kraken image](https://quay.io/redhat-chaos/krkn).
 
-  
 
 ### Run containerized version
 
 Refer [instructions](https://github.com/redhat-chaos/krkn/blob/main/docs/installation.md#run-containerized-version) for information on how to run the containerized version of kraken.
 
-  
-  
 
 ### Run Custom Kraken Image
 
 Refer to [instructions](https://github.com/redhat-chaos/krkn/blob/main/containers/build_own_image-README.md) for information on how to run a custom containerized version of kraken using podman.
-
-  
-  
-
-### Kraken as a KubeApp
-
-#### GENERAL NOTES:
-
-- It is not generally recommended to run Kraken internal to the cluster as the pod which is running Kraken might get disrupted, the suggested use case to run kraken from inside k8s/OpenShift is to target **another** cluster (eg. to bypass network restrictions or to leverage cluster's computational resources)
-
-- your kubeconfig might contain several cluster contexts and credentials so be sure, before creating the ConfigMap, to keep **only** the credentials related to the destination cluster. Please refer to the [Kubernetes documentation](https://kubernetes.io/docs/tasks/access-application-cluster/configure-access-multiple-clusters/) for more details
-- to add privileges to the service account you must be logged in the cluster with an highly privileged account (ideally kubeadmin)
-
-  
-
-To run containerized Kraken as a Kubernetes/OpenShift Deployment, follow these steps:
-
-1. Configure the [config.yaml](https://github.com/redhat-chaos/krkn/blob/main/config/config.yaml) file according to your requirements.
-
-**NOTE**: both the scenarios ConfigMaps are needed regardless you're running kraken in Kubernetes or OpenShift
-
-2. Create a namespace under which you want to run the kraken pod using `kubectl create ns <namespace>`.
-3. Switch to `<namespace>` namespace:
-- In Kubernetes, use `kubectl config set-context --current --namespace=<namespace>`
-- In OpenShift, use `oc project <namespace>`
-  
-4. Create a ConfigMap named kube-config using `kubectl create configmap kube-config --from-file=<path_to_kubeconfig>`  *(eg. ~/.kube/config)*
-5. Create a ConfigMap named kraken-config using `kubectl create configmap kraken-config --from-file=<path_to_kraken>/config`
-6. Create a ConfigMap named scenarios-config using `kubectl create configmap scenarios-config --from-file=<path_to_kraken>/scenarios`
-7. Create a ConfigMap named scenarios-openshift-config using `kubectl create configmap scenarios-openshift-config --from-file=<path_to_kraken>/scenarios/openshift`
-8. Create a ConfigMap named scenarios-kube-config using `kubectl create configmap scenarios-kube-config --from-file=<path_to_kraken>/scenarios/kube` 
-9. Create a service account to run the kraken pod `kubectl create serviceaccount useroot`.
-10.  In Openshift, add privileges to service account and execute `oc adm policy add-scc-to-user privileged -z useroot`.
-11.  Create a Job using `kubectl apply -f <path_to_kraken>/containers/kraken.yml` and monitor the status using `oc get jobs` and `oc get pods`.

containers/build_own_image-README.md

@@ -1,13 +1,13 @@
 # Building your own Kraken image
 
-1. Git clone the Kraken repository using `git clone https://github.com/openshift-scale/kraken.git`.
+1. Git clone the Kraken repository using `git clone https://github.com/redhat-chaos/krkn.git`.
 2. Modify the python code and yaml files to address your needs.
 3. Execute `podman build -t <new_image_name>:latest .` in the containers directory within kraken to build an image from a Dockerfile.
 4. Execute `podman run --detach --name <container_name> <new_image_name>:latest` to start a container based on your new image.
 
 # Building the Kraken image on IBM Power (ppc64le)
 
-1. Git clone the Kraken repository using `git clone https://github.com/cloud-bulldozer/kraken.git` on an IBM Power Systems server.
+1. Git clone the Kraken repository using `git clone https://github.com/redhat-chaos/krkn.git` on an IBM Power Systems server.
 2. Modify the python code and yaml files to address your needs.
 3. Execute `podman build -t <new_image_name>:latest -f Dockerfile-ppc64le` in the containers directory within kraken to build an image from the Dockerfile for Power.
 4. Execute `podman run --detach --name <container_name> <new_image_name>:latest` to start a container based on your new image.

containers/compile_dockerfile.sh

@@ -0,0 +1,5 @@
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+cd "$SCRIPT_DIR"
+export KRKNCTL_INPUT=$(cat krknctl-input.json|tr -d "\n")
+
+envsubst '${KRKNCTL_INPUT}' < Dockerfile.template > Dockerfile

containers/kraken.yml

@@ -1,49 +0,0 @@
----
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: kraken
-spec:
-  parallelism: 1
-  completions: 1
-  template:
-    metadata:
-      labels:
-        tool: Kraken
-    spec:
-      serviceAccountName: useroot
-      containers:
-        - name: kraken
-          securityContext:
-            privileged: true
-          image: quay.io/redhat-chaos/krkn
-          command: ["/bin/sh", "-c"]
-          args: ["python3.9 run_kraken.py -c config/config.yaml"]
-          volumeMounts:
-            - mountPath: "/root/.kube"
-              name: config
-            - mountPath: "/root/kraken/config"
-              name: kraken-config
-            - mountPath: "/root/kraken/scenarios"
-              name: scenarios-config
-            - mountPath: "/root/kraken/scenarios/openshift"
-              name: scenarios-openshift-config
-            - mountPath: "/root/kraken/scenarios/kube"
-              name: scenarios-kube-config
-      restartPolicy: Never
-      volumes:
-        - name: config
-          configMap:
-            name: kube-config
-        - name: kraken-config
-          configMap:
-            name: kraken-config
-        - name: scenarios-config
-          configMap:
-            name: scenarios-config
-        - name: scenarios-openshift-config
-          configMap:
-            name: scenarios-openshift-config
-        - name: scenarios-kube-config
-          configMap:
-            name: scenarios-kube-config

containers/krknctl-input.json

@@ -0,0 +1,396 @@
+[
+  {
+    "name": "cerberus-enabled",
+    "short_description": "Enable Cerberus",
+    "description": "Enables Cerberus Support",
+    "variable": "CERBERUS_ENABLED",
+    "type": "enum",
+    "default": "False",
+    "allowed_values": "True,False",
+    "separator": ",",
+    "required": "false"
+  },
+  {
+    "name": "cerberus-url",
+    "short_description": "Cerberus URL",
+    "description": "Cerberus http url",
+    "variable": "CERBERUS_URL",
+    "type": "string",
+    "default": "http://0.0.0.0:8080",
+    "validator": "^(http|https):\/\/.*",
+    "required": "false"
+  },
+  {
+    "name": "distribution",
+    "short_description": "Orchestrator distribution",
+    "description": "Selects the orchestrator distribution",
+    "variable": "DISTRIBUTION",
+    "type": "enum",
+    "default": "openshift",
+    "allowed_values": "openshift,kubernetes",
+    "separator": ",",
+    "required": "false"
+  },
+  {
+    "name": "krkn-kubeconfig",
+    "short_description": "Krkn kubeconfig path",
+    "description": "Sets the path where krkn will search for kubeconfig (in container)",
+    "variable": "KRKN_KUBE_CONFIG",
+    "type": "string",
+    "default": "/home/krkn/.kube/config",
+    "required": "false"
+  },
+  {
+    "name": "wait-duration",
+    "short_description": "Post chaos wait duration",
+    "description": "waits for a certain amount of time after the scenario",
+    "variable": "WAIT_DURATION",
+    "type": "number",
+    "default": "1"
+  },
+  {
+    "name": "iterations",
+    "short_description": "Chaos scenario iterations",
+    "description": "number of times the same chaos scenario will be executed",
+    "variable": "ITERATIONS",
+    "type": "number",
+    "default": "1"
+  },
+  {
+    "name": "daemon-mode",
+    "short_description": "Sets krkn daemon mode",
+    "description": "if set the scenario will execute forever",
+    "variable": "DAEMON_MODE",
+    "type": "enum",
+    "allowed_values": "True,False",
+    "separator": ",",
+    "default": "False",
+    "required": "false"
+  },
+  {
+    "name": "uuid",
+    "short_description": "Sets krkn run uuid",
+    "description": "sets krkn run uuid instead of generating it",
+    "variable": "UUID",
+    "type": "string",
+    "default": "",
+    "required": "false"
+  },
+  {
+    "name": "capture-metrics",
+    "short_description": "Enables metrics capture",
+    "description": "Enables metrics capture",
+    "variable": "CAPTURE_METRICS",
+    "type": "enum",
+    "allowed_values": "True,False",
+    "separator": ",",
+    "default": "False",
+    "required": "false"
+  },
+  {
+    "name": "enable-alerts",
+    "short_description": "Enables cluster alerts check",
+    "description": "Enables cluster alerts check",
+    "variable": "ENABLE_ALERTS",
+    "type": "enum",
+    "allowed_values": "True,False",
+    "separator": ",",
+    "default": "False",
+    "required": "false"
+  },
+  {
+    "name": "alerts-path",
+    "short_description": "Cluster alerts path file (in container)",
+    "description": "Enables cluster alerts check",
+    "variable": "ALERTS_PATH",
+    "type": "string",
+    "default": "config/alerts.yaml",
+    "required": "false"
+  },
+  {
+    "name": "enable-es",
+    "short_description": "Enables elastic search data collection",
+    "description": "Enables elastic search data collection",
+    "variable": "ENABLE_ES",
+    "type": "enum",
+    "allowed_values": "True,False",
+    "separator": ",",
+    "default": "False",
+    "required": "false"
+  },
+  {
+    "name": "es-server",
+    "short_description": "Elasticsearch instance URL",
+    "description": "Elasticsearch instance URL",
+    "variable": "ES_SERVER",
+    "type": "string",
+    "default": "http://0.0.0.0",
+    "validator": "^(http|https):\/\/.*",
+    "required": "false"
+  },
+  {
+    "name": "es-port",
+    "short_description": "Elasticsearch instance port",
+    "description": "Elasticsearch instance port",
+    "variable": "ES_PORT",
+    "type": "number",
+    "default": "443",
+    "required": "false"
+  },
+  {
+    "name": "es-username",
+    "short_description": "Elasticsearch instance username",
+    "description": "Elasticsearch instance username",
+    "variable": "ES_USERNAME",
+    "type": "string",
+    "default": "elastic",
+    "required": "false"
+  },
+  {
+    "name": "es-password",
+    "short_description": "Elasticsearch instance password",
+    "description": "Elasticsearch instance password",
+    "variable": "ES_PASSWORD",
+    "type": "string",
+    "default": "",
+    "required": "false"
+  },
+  {
+    "name": "es-verify-certs",
+    "short_description": "Enables elasticsearch TLS certificate verification",
+    "description": "Enables elasticsearch TLS certificate verification",
+    "variable": "ES_VERIFY_CERTS",
+    "type": "enum",
+    "allowed_values": "True,False",
+    "separator": ",",
+    "default": "False",
+    "required": "false"
+  },
+  {
+    "name": "es-collect-metrics",
+    "short_description": "Enables metrics collection on elastic search",
+    "description": "Enables metrics collection on elastic search",
+    "variable": "ES_COLLECT_METRICS",
+    "type": "enum",
+    "allowed_values": "True,False",
+    "separator": ",",
+    "default": "False",
+    "required": "false"
+  },
+  {
+    "name": "es-collect-alerts",
+    "short_description": "Enables alerts collection on elastic search",
+    "description": "Enables alerts collection on elastic search",
+    "variable": "ES_COLLECT_ALERTS",
+    "type": "enum",
+    "allowed_values": "True,False",
+    "separator": ",",
+    "default": "False",
+    "required": "false"
+  },
+  {
+    "name": "es-metrics-index",
+    "short_description": "Elasticsearch metrics index",
+    "description": "Index name for metrics in Elasticsearch",
+    "variable": "ES_METRICS_INDEX",
+    "type": "string",
+    "default": "krkn-metrics",
+    "required": "false"
+  },
+  {
+    "name": "es-alerts-index",
+    "short_description": "Elasticsearch alerts index",
+    "description": "Index name for alerts in Elasticsearch",
+    "variable": "ES_ALERTS_INDEX",
+    "type": "string",
+    "default": "krkn-alerts",
+    "required": "false"
+  },
+  {
+    "name": "es-telemetry-index",
+    "short_description": "Elasticsearch telemetry index",
+    "description": "Index name for telemetry in Elasticsearch",
+    "variable": "ES_TELEMETRY_INDEX",
+    "type": "string",
+    "default": "krkn-telemetry",
+    "required": "false"
+  },
+  {
+    "name": "check-critical-alerts",
+    "short_description": "Check critical alerts",
+    "description": "Enables checking for critical alerts",
+    "variable": "CHECK_CRITICAL_ALERTS",
+    "type": "enum",
+    "allowed_values": "True,False",
+    "separator": ",",
+    "default": "False",
+    "required": "false"
+  },
+  {
+    "name": "telemetry-enabled",
+    "short_description": "Enable telemetry",
+    "description": "Enables telemetry support",
+    "variable": "TELEMETRY_ENABLED",
+    "type": "enum",
+    "allowed_values": "True,False",
+    "separator": ",",
+    "default": "False",
+    "required": "false"
+  },
+  {
+    "name": "telemetry-api-url",
+    "short_description": "Telemetry API URL",
+    "description": "API endpoint for telemetry data",
+    "variable": "TELEMETRY_API_URL",
+    "type": "string",
+    "default": "https://ulnmf9xv7j.execute-api.us-west-2.amazonaws.com/production",
+    "validator": "^(http|https):\/\/.*",
+    "required": "false"
+  },
+  {
+    "name": "telemetry-username",
+    "short_description": "Telemetry username",
+    "description": "Username for telemetry authentication",
+    "variable": "TELEMETRY_USERNAME",
+    "type": "string",
+    "default": "redhat-chaos",
+    "required": "false"
+  },
+  {
+    "name": "telemetry-password",
+    "short_description": "Telemetry password",
+    "description": "Password for telemetry authentication",
+    "variable": "TELEMETRY_PASSWORD",
+    "type": "string",
+    "default": "",
+    "required": "false"
+  },
+  {
+    "name": "telemetry-prometheus-backup",
+    "short_description": "Prometheus backup for telemetry",
+    "description": "Enables Prometheus backup for telemetry",
+    "variable": "TELEMETRY_PROMETHEUS_BACKUP",
+    "type": "enum",
+    "allowed_values": "True,False",
+    "separator": ",",
+    "default": "True",
+    "required": "false"
+  },
+  {
+    "name": "telemetry-full-prometheus-backup",
+    "short_description": "Full Prometheus backup",
+    "description": "Enables full Prometheus backup for telemetry",
+    "variable": "TELEMETRY_FULL_PROMETHEUS_BACKUP",
+    "type": "enum",
+    "allowed_values": "True,False",
+    "separator": ",",
+    "default": "False",
+    "required": "false"
+  },
+  {
+    "name": "telemetry-backup-threads",
+    "short_description": "Telemetry backup threads",
+    "description": "Number of threads for telemetry backup",
+    "variable": "TELEMETRY_BACKUP_THREADS",
+    "type": "number",
+    "default": "5",
+    "required": "false"
+  },
+  {
+    "name": "telemetry-archive-path",
+    "short_description": "Telemetry archive path",
+    "description": "Path to save telemetry archive",
+    "variable": "TELEMETRY_ARCHIVE_PATH",
+    "type": "string",
+    "default": "/tmp",
+    "required": "false"
+  },
+  {
+    "name": "telemetry-max-retries",
+    "short_description": "Telemetry max retries",
+    "description": "Maximum retries for telemetry operations",
+    "variable": "TELEMETRY_MAX_RETRIES",
+    "type": "number",
+    "default": "0",
+    "required": "false"
+  },
+  {
+    "name": "telemetry-run-tag",
+    "short_description": "Telemetry run tag",
+    "description": "Tag for telemetry run",
+    "variable": "TELEMETRY_RUN_TAG",
+    "type": "string",
+    "default": "chaos",
+    "required": "false"
+  },
+  {
+    "name": "telemetry-group",
+    "short_description": "Telemetry group",
+    "description": "Group name for telemetry data",
+    "variable": "TELEMETRY_GROUP",
+    "type": "string",
+    "default": "default",
+    "required": "false"
+  },
+  {
+    "name": "telemetry-archive-size",
+    "short_description": "Telemetry archive size",
+    "description": "Maximum size for telemetry archives",
+    "variable": "TELEMETRY_ARCHIVE_SIZE",
+    "type": "number",
+    "default": "1000",
+    "required": "false"
+  },
+  {
+    "name": "telemetry-logs-backup",
+    "short_description": "Telemetry logs backup",
+    "description": "Enables logs backup for telemetry",
+    "variable": "TELEMETRY_LOGS_BACKUP",
+    "type": "enum",
+    "allowed_values": "True,False",
+    "separator": ",",
+    "default": "False",
+    "required": "false"
+  },
+  {
+    "name": "telemetry-filter-pattern",
+    "short_description": "Telemetry filter pattern",
+    "description": "Filter pattern for telemetry logs",
+    "variable": "TELEMETRY_FILTER_PATTERN",
+    "type": "string",
+    "default": "[\"(\\\\w{3}\\\\s\\\\d{1,2}\\\\s\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d+).+\",\"kinit (\\\\d+/\\\\d+/\\\\d+\\\\s\\\\d{2}:\\\\d{2}:\\\\d{2})\\\\s+\",\"(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d+Z).+\"]",
+    "required": "false"
+  },
+  {
+    "name": "telemetry-cli-path",
+    "short_description": "Telemetry CLI path (oc)",
+    "description": "Path to telemetry CLI tool (oc)",
+    "variable": "TELEMETRY_CLI_PATH",
+    "type": "string",
+    "default": "",
+    "required": "false"
+  },
+  {
+    "name": "telemetry-events-backup",
+    "short_description": "Telemetry events backup",
+    "description": "Enables events backup for telemetry",
+    "variable": "TELEMETRY_EVENTS_BACKUP",
+    "type": "enum",
+    "allowed_values": "True,False",
+    "separator": ",",
+    "default": "True",
+    "required": "false"
+  },
+  {
+    "name": "krkn-debug",
+    "short_description": "Krkn debug mode",
+    "description": "Enables debug mode for Krkn",
+    "variable": "KRKN_DEBUG",
+    "type": "enum",
+    "allowed_values": "True,False",
+    "separator": ",",
+    "default": "False",
+    "required": "false"
+  }
+]
+

docker-compose.yml

@@ -1,31 +0,0 @@
-version: "3"
-services:
-  elastic:
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.13.2
-    deploy:
-      replicas: 1
-      restart_policy:
-        condition: on-failure
-    network_mode: host
-    environment:
-      discovery.type: single-node
-  kibana:
-    image: docker.elastic.co/kibana/kibana:7.13.2
-    deploy:
-      replicas: 1
-      restart_policy:
-        condition: on-failure
-    network_mode: host
-    environment:
-      ELASTICSEARCH_HOSTS: "http://0.0.0.0:9200"
-  cerberus:
-    image: quay.io/openshift-scale/cerberus:latest
-    privileged: true
-    deploy:
-      replicas: 1
-      restart_policy:
-        condition: on-failure
-    network_mode: host
-    volumes:
-       - ./config/cerberus.yaml:/root/cerberus/config/config.yaml:Z  # Modify the config in case of the need to monitor additional components
-       - ${HOME}/.kube/config:/root/.kube/config:Z

docs/SLOs_validation.md

@@ -11,19 +11,18 @@ performance_monitoring:
 ```
 
 ### Validation and alerting based on the queries defined by the user during chaos
-Takes PromQL queries as input and modifies the return code of the run to determine pass/fail. It's especially useful in case of automated runs in CI where user won't be able to monitor the system. It uses [Kube-burner](https://kube-burner.readthedocs.io/en/latest/) under the hood. This feature can be enabled in the [config](https://github.com/redhat-chaos/krkn/blob/main/config/config.yaml) by setting the following:
+Takes PromQL queries as input and modifies the return code of the run to determine pass/fail. It's especially useful in case of automated runs in CI where user won't be able to monitor the system. This feature can be enabled in the [config](https://github.com/redhat-chaos/krkn/blob/main/config/config.yaml) by setting the following:
 
 ```
 performance_monitoring:
-    kube_burner_binary_url: "https://github.com/cloud-bulldozer/kube-burner/releases/download/v0.9.1/kube-burner-0.9.1-Linux-x86_64.tar.gz"
     prometheus_url:                                       # The prometheus url/route is automatically obtained in case of OpenShift, please set it when the distribution is Kubernetes.
     prometheus_bearer_token:                              # The bearer token is automatically obtained in case of OpenShift, please set it when the distribution is Kubernetes. This is needed to authenticate with prometheus.
     enable_alerts: True                                   # Runs the queries specified in the alert profile and displays the info or exits 1 when severity=error.
-    alert_profile: config/alerts                          # Path to alert profile with the prometheus queries.
+    alert_profile: config/alerts.yaml                          # Path to alert profile with the prometheus queries.
 ```
 
 #### Alert profile
-A couple of [alert profiles](https://github.com/redhat-chaos/krkn/tree/main/config) [alerts](https://github.com/redhat-chaos/krkn/blob/main/config/alerts) are shipped by default and can be tweaked to add more queries to alert on. User can provide a URL or path to the file in the [config](https://github.com/redhat-chaos/krkn/blob/main/config/config.yaml). The following are a few alerts examples:
+A couple of [alert profiles](https://github.com/redhat-chaos/krkn/tree/main/config) [alerts](https://github.com/redhat-chaos/krkn/blob/main/config/alerts.yaml) are shipped by default and can be tweaked to add more queries to alert on. User can provide a URL or path to the file in the [config](https://github.com/redhat-chaos/krkn/blob/main/config/config.yaml). The following are a few alerts examples:
 
 ```
 - expr: avg_over_time(histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket[2m]))[5m:]) > 0.01
@@ -39,11 +38,11 @@ A couple of [alert profiles](https://github.com/redhat-chaos/krkn/tree/main/conf
   severity: critical
 ```
 
-Kube-burner supports setting the severity for the alerts with each one having different effects:
+Krkn supports setting the severity for the alerts with each one having different effects:
 
 ```
 info: Prints an info message with the alarm description to stdout. By default all expressions have this severity.
 warning: Prints a warning message with the alarm description to stdout.
-error: Prints a error message with the alarm description to stdout and makes kube-burner rc = 1
+error: Prints a error message with the alarm description to stdout and sets Krkn rc = 1
 critical: Prints a fatal message with the alarm description to stdout and exits execution inmediatly with rc != 0
 ```

docs/cloud_setup.md

@@ -13,13 +13,26 @@ Supported Cloud Providers:
 **NOTE**: For clusters with AWS make sure [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html) is installed and properly [configured](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-quickstart.html) using an AWS account
 
 ## GCP
-**NOTE**: For clusters with GCP make sure [GCP CLI](https://cloud.google.com/sdk/docs/install#linux) is installed.
 
-A google service account is required to give proper authentication to GCP for node actions. See [here](https://cloud.google.com/docs/authentication/getting-started) for how to create a service account.
+In order to set up Application Default Credentials (ADC) for use by Cloud Client Libraries, you can provide either service account credentials or the credentials associated with your user acccount:
 
-**NOTE**: A user with 'resourcemanager.projects.setIamPolicy' permission is required to grant project-level permissions to the service account.
+- Using service account credentials:
 
-After creating the service account you will need to enable the account using the following: ```export GOOGLE_APPLICATION_CREDENTIALS="<serviceaccount.json>"```
+  A google service account is required to give proper authentication to GCP for node actions. See [here](https://cloud.google.com/docs/authentication/getting-started) for how to create a service account.
+
+  **NOTE**: A user with 'resourcemanager.projects.setIamPolicy' permission is required to grant project-level permissions to the service account.
+
+  After creating the service account you will need to enable the account using the following: ```export GOOGLE_APPLICATION_CREDENTIALS="<serviceaccount.json>"```
+
+- Using the credentials associated with your user acccount:
+
+  1. Make sure that the [GCP CLI](https://cloud.google.com/sdk/docs/install#linux) is installed and [initialized](https://cloud.google.com/sdk/docs/initializing) by running:
+
+     ```gcloud init```
+
+  2. Create local authentication credentials for your user account:
+
+     ```gcloud auth application-default login```
 
 ## Openstack
 
@@ -27,14 +40,13 @@ After creating the service account you will need to enable the account using the
 
 ## Azure
 
-**NOTE**: For Azure node killing scenarios, make sure [Azure CLI](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli?view=azure-cli-latest) is installed.
-
-You will also need to create a service principal and give it the correct access, see [here](https://docs.openshift.com/container-platform/4.5/installing/installing_azure/installing-azure-account.html) for creating the service principal and setting the proper permissions.
+**NOTE**: You will need to create a service principal and give it the correct access, see [here](https://docs.openshift.com/container-platform/4.5/installing/installing_azure/installing-azure-account.html) for creating the service principal and setting the proper permissions.
 
 To properly run the service principal requires “Azure Active Directory Graph/Application.ReadWrite.OwnedBy” api permission granted and “User Access Administrator”.
 
 Before running you will need to set the following:
-1. Login using ```az login```
+
+1. ```export AZURE_SUBSCRIPTION_ID=<subscription_id>```
 
 2. ```export AZURE_TENANT_ID=<tenant_id>```
 
@@ -68,9 +80,10 @@ Set the following environment variables
 
 These are the credentials that you would normally use to access the vSphere client.
 
-
 ## IBMCloud
-If no api key is set up with proper VPC resource permissions, use the following to create: 
+
+If no API key is set up with proper VPC resource permissions, use the following to create it:
+
 * Access group
 * Service id with the following access
   * With policy **VPC Infrastructure Services**

docs/cluster_shut_down_scenarios.md

@@ -1,5 +1,5 @@
-#### Kubernetes/OpenShift cluster shut down scenario
-Scenario to shut down all the nodes including the masters and restart them after specified duration. Cluster shut down scenario can be injected by placing the shut_down config file under cluster_shut_down_scenario option in the kraken config. Refer to [cluster_shut_down_scenario](https://github.com/redhat-chaos/krkn/blob/main/scenarios/cluster_shut_down_scenario.yml) config file.
+#### Kubernetes cluster shut down scenario
+Scenario to shut down all the nodes including the masters and restart them after specified duration. Cluster shut down scenario can be injected by placing the shut_down config file under cluster_shut_down_scenario option in the kraken config. Refer to [cluster_shut_down_scenario](https://github.com/krkn-chaos/krkn/blob/main/scenarios/cluster_shut_down_scenario.yml) config file.
 
 Refer to [cloud setup](cloud_setup.md) to configure your cli properly for the cloud provider of the cluster you want to shut down.
 
@@ -8,6 +8,7 @@ Current accepted cloud types:
 * [GCP](cloud_setup.md#gcp)
 * [AWS](cloud_setup.md#aws)
 * [Openstack](cloud_setup.md#openstack)
+* [IBMCloud](cloud_setup.md#ibmcloud)
 
 
 ```

docs/container_scenarios.md

@@ -4,18 +4,18 @@ This can be based on the pods namespace or labels. If you know the exact object
 These scenarios are in a simple yaml format that you can manipulate to run your specific tests or use the pre-existing scenarios to see how it works.
 
 ####  Example Config
-The following are the components of Kubernetes/OpenShift for which a basic chaos scenario config exists today.
+The following are the components of Kubernetes for which a basic chaos scenario config exists today.
 
 ```
 scenarios:
-- name: "<Name of scenario>"
+- name: "<name of scenario>"
   namespace: "<specific namespace>" # can specify "*" if you want to find in all namespaces
   label_selector: "<label of pod(s)>"
   container_name: "<specific container name>"  # This is optional, can take out and will kill all containers in all pods found under namespace and label
   pod_names:  # This is optional, can take out and will select all pods with given namespace and label
   - <pod_name>
   count: <number of containers to disrupt, default=1>
-  action: <Action to run. For example kill 1 ( hang up ) or kill 9. Default is set to kill 1>
+  action: <kill signal to run. For example 1 ( hang up ) or 9. Default is set to 1>
   expected_recovery_time: <number of seconds to wait for container to be running again> (defaults to 120seconds)
 ```
 
@@ -25,7 +25,7 @@ In all scenarios we do a post chaos check to wait and verify the specific compon
 Here there are two options:
 1. Pass a custom script in the main config scenario list that will run before the chaos and verify the output matches post chaos scenario.
 
-See [scenarios/post_action_etcd_container.py](https://github.com/redhat-chaos/krkn/blob/main/scenarios/post_action_etcd_container.py) for an example.
+See [scenarios/post_action_etcd_container.py](https://github.com/krkn-chaos/krkn/blob/main/scenarios/post_action_etcd_container.py) for an example.
 ```
 -   container_scenarios:                                 # List of chaos pod scenarios to load.
             - -    scenarios/container_etcd.yml

docs/contribute.md

@@ -62,7 +62,7 @@ If changes go into the main repository while you're working on your code it is b
 
 If not already configured, set the upstream url for kraken.
 ```
- git remote add upstream https://github.com/cloud-bulldozer/kraken.git
+ git remote add upstream https://github.com/krkn-chaos/krkn.git
 ```
 
 Rebase to upstream master branch.

docs/getting_started.md

@@ -14,11 +14,7 @@ For example, for adding a pod level scenario for a new application, refer to the
     namespace_pattern: ^<namespace>$
     label_selector: <pod label>
     kill: <number of pods to kill>
-- id: wait-for-pods
-  config:
-    namespace_pattern: ^<namespace>$
-    label_selector: <pod label>
-    count: <expected number of pods that match namespace and label>
+    krkn_pod_recovery_time: <expected time for the pod to become ready>
 ```
 
 #### Node Scenario Yaml Template

docs/index.md

@@ -11,7 +11,7 @@
 * [Scenarios](#scenarios)
 * [Test Environment Recommendations - how and where to run chaos tests](#test-environment-recommendations---how-and-where-to-run-chaos-tests)
 * [Chaos testing in Practice](#chaos-testing-in-practice)
-  * [OpenShift oraganization](#openshift-organization)
+  * [OpenShift organization](#openshift-organization)
   * [startx-lab](#startx-lab)
 
 
@@ -48,7 +48,7 @@ Failures in production are costly. To help mitigate risk to service health, cons
 
 
 ### Best Practices
-Now that we understand the test methodology, let us take a look at the best practices for an OpenShift cluster.  On that platform there are user applications and cluster workloads that need to be designed for stability and to provide the best user experience possible:
+Now that we understand the test methodology, let us take a look at the best practices for an Kubernetes cluster.  On that platform there are user applications and cluster workloads that need to be designed for stability and to provide the best user experience possible:
 
 - Alerts with appropriate severity should get fired.
   - Alerts are key to identify when a component starts degrading, and can help focus the investigation effort on affected system components.
@@ -77,11 +77,11 @@ We want to look at this in terms of CPU, Memory, Disk, Throughput, Network etc.
   - The controller watching the component should recognize a failure as soon as possible. The component needs to have minimal initialization time to avoid extended downtime or overloading the replicas if it is a highly available configuration. The cause of failure can be because of issues with the infrastructure on top of which it is running, application failures, or because of service failures that it depends on.
 
 - High Availability deployment strategy.
-  - There should be multiple replicas ( both OpenShift and application control planes ) running preferably in different availability zones to survive outages while still serving the user/system requests. Avoid single points of failure.
+  - There should be multiple replicas ( both Kubernetes and application control planes ) running preferably in different availability zones to survive outages while still serving the user/system requests. Avoid single points of failure.
 - Backed by persistent storage
   - It is important to have the system/application backed by persistent storage. This is especially important in cases where the application is a database or a stateful application given that a node, pod, or container failure will wipe off the data.
 
-- There should be fallback routes to the backend in case of using CDN, for example, Akamai in case of console.redhat.com - a managed service deployed on top of OpenShift dedicated:
+- There should be fallback routes to the backend in case of using CDN, for example, Akamai in case of console.redhat.com - a managed service deployed on top of Kubernetes dedicated:
   - Content delivery networks (CDNs) are commonly used to host resources such as images, JavaScript files, and CSS. The average web page is nearly 2 MB in size, and offloading heavy resources to third-parties is extremely effective for reducing backend server traffic and latency. However, this makes each CDN an additional point of failure for every site that relies on it. If the CDN fails, its customers could also fail.
   - To test how the application reacts to failures, drop all network traffic between the system and CDN. The application should still serve the content to the user irrespective of the failure.
 
@@ -92,10 +92,10 @@ We want to look at this in terms of CPU, Memory, Disk, Throughput, Network etc.
 
 
 ### Tooling
-Now that we looked at the best practices, In this section, we will go through how [Kraken](https://github.com/redhat-chaos/krkn) - a chaos testing framework can help test the resilience of OpenShift and make sure the applications and services are following the best practices.
+Now that we looked at the best practices, In this section, we will go through how [Kraken](https://github.com/redhat-chaos/krkn) - a chaos testing framework can help test the resilience of Kubernetes and make sure the applications and services are following the best practices.
 
 #### Workflow
-Let us start by understanding the workflow of kraken: the user will start by running kraken by pointing to a specific OpenShift cluster using kubeconfig to be able to talk to the platform on top of which the OpenShift cluster is hosted. This can be done by either the oc/kubectl API or the cloud API. Based on the configuration of kraken, it will inject specific chaos scenarios as shown below, talk to [Cerberus](https://github.com/redhat-chaos/cerberus) to get the go/no-go signal representing the overall health of the cluster ( optional - can be turned off ), scrapes metrics from in-cluster prometheus given a metrics profile with the promql queries and stores them long term in Elasticsearch configured  ( optional - can be turned off ), evaluates the promql expressions specified in the alerts profile ( optional - can be turned off ) and aggregated everything to set the pass/fail i.e. exits 0 or 1. More about the metrics collection, cerberus and metrics evaluation can be found in the next section.
+Let us start by understanding the workflow of kraken: the user will start by running kraken by pointing to a specific Kubernetes cluster using kubeconfig to be able to talk to the platform on top of which the Kubernetes cluster is hosted. This can be done by either the oc/kubectl API or the cloud API. Based on the configuration of kraken, it will inject specific chaos scenarios as shown below, talk to [Cerberus](https://github.com/redhat-chaos/cerberus) to get the go/no-go signal representing the overall health of the cluster ( optional - can be turned off ), scrapes metrics from in-cluster prometheus given a metrics profile with the promql queries and stores them long term in Elasticsearch configured  ( optional - can be turned off ), evaluates the promql expressions specified in the alerts profile ( optional - can be turned off ) and aggregated everything to set the pass/fail i.e. exits 0 or 1. More about the metrics collection, cerberus and metrics evaluation can be found in the next section.
 
 ![Kraken workflow](../media/kraken-workflow.png)
 
@@ -112,15 +112,15 @@ If the monitoring tool, cerberus is enabled it will consume the signal and conti
 
 ### Scenarios
 
-Let us take a look at how to run the chaos scenarios on your OpenShift clusters using Kraken-hub - a lightweight wrapper around Kraken to ease the runs by providing the ability to run them by just running container images using podman with parameters set as environment variables. This eliminates the need to carry around and edit configuration files and makes it easy for any CI framework integration. Here are the scenarios supported:
+Let us take a look at how to run the chaos scenarios on your Kubernetes clusters using Kraken-hub - a lightweight wrapper around Kraken to ease the runs by providing the ability to run them by just running container images using podman with parameters set as environment variables. This eliminates the need to carry around and edit configuration files and makes it easy for any CI framework integration. Here are the scenarios supported:
 
 - Pod Scenarios ([Documentation](https://github.com/redhat-chaos/krkn-hub/blob/main/docs/pod-scenarios.md))
-  - Disrupts OpenShift/Kubernetes and applications deployed as pods:
+  - Disrupts Kubernetes/Kubernetes and applications deployed as pods:
     - Helps understand the availability of the application, the initialization timing and recovery status.
   - [Demo](https://asciinema.org/a/452351?speed=3&theme=solarized-dark)
 
 - Container Scenarios ([Documentation](https://github.com/redhat-chaos/krkn-hub/blob/main/docs/container-scenarios.md))
-  - Disrupts OpenShift/Kubernetes and applications deployed as containers running as part of a pod(s) using a specified kill signal to mimic failures:
+  - Disrupts Kubernetes/Kubernetes and applications deployed as containers running as part of a pod(s) using a specified kill signal to mimic failures:
     - Helps understand the impact and recovery timing when the program/process running in the containers are disrupted - hangs, paused, killed etc., using various kill signals, i.e. SIGHUP, SIGTERM, SIGKILL etc.
   - [Demo](https://asciinema.org/a/BXqs9JSGDSEKcydTIJ5LpPZBM?speed=3&theme=solarized-dark)
 
@@ -134,8 +134,8 @@ Let us take a look at how to run the chaos scenarios on your OpenShift clusters
   - [Demo](https://asciinema.org/a/ANZY7HhPdWTNaWt4xMFanF6Q5)
 
 - Zone Outages ([Documentation](https://github.com/redhat-chaos/krkn-hub/blob/main/docs/zone-outages.md))
-  - Creates outage of availability zone(s) in a targeted region in the public cloud where the OpenShift cluster is running by tweaking the network acl of the zone to simulate the failure, and that in turn will stop both ingress and egress traffic from all nodes in a particular zone for the specified duration and reverts it back to the previous state.
-    - Helps understand the impact on both Kubernetes/OpenShift control plane as well as applications and services running on the worker nodes in that zone.
+  - Creates outage of availability zone(s) in a targeted region in the public cloud where the Kubernetes cluster is running by tweaking the network acl of the zone to simulate the failure, and that in turn will stop both ingress and egress traffic from all nodes in a particular zone for the specified duration and reverts it back to the previous state.
+    - Helps understand the impact on both Kubernetes/Kubernetes control plane as well as applications and services running on the worker nodes in that zone.
     - Currently, only set up for AWS cloud platform: 1 VPC and multiples subnets within the VPC can be specified.
     - [Demo](https://asciinema.org/a/452672?speed=3&theme=solarized-dark)
 
@@ -200,7 +200,7 @@ Let us take a look at few recommendations on how and where to run the chaos test
 - Enable Observability:
   - Chaos Engineering Without Observability ... Is Just Chaos.
   - Make sure to have logging and monitoring installed on the cluster to help with understanding the behaviour as to why it is happening. In case of running the tests in the CI where it is not humanly possible to monitor the cluster all the time, it is recommended to leverage Cerberus to capture the state during the runs and metrics collection in Kraken to store metrics long term even after the cluster is gone.
-  - Kraken ships with dashboards that will help understand API, Etcd and OpenShift cluster level stats and performance metrics.
+  - Kraken ships with dashboards that will help understand API, Etcd and Kubernetes cluster level stats and performance metrics.
   - Pay attention to Prometheus alerts. Check if they are firing as expected.
 
 - Run multiple chaos tests at once to mimic the production outages:

docs/installation.md

@@ -3,13 +3,13 @@
 The following ways are supported to run Kraken:
 
 - Standalone python program through Git.
-- Containerized version using either Podman or Docker as the runtime.
-- Kubernetes or OpenShift deployment.
+- Containerized version using either Podman or Docker as the runtime via [Krkn-hub](https://github.com/krkn-chaos/krkn-hub)
+- Kubernetes or OpenShift deployment ( unsupported )
 
 **NOTE**: It is recommended to run Kraken external to the cluster ( Standalone or Containerized ) hitting the Kubernetes/OpenShift API as running it internal to the cluster might be disruptive to itself and also might not report back the results if the chaos leads to cluster's API server instability.
 
 **NOTE**: To run Kraken on Power (ppc64le) architecture, build and run a containerized version by following the
- instructions given [here](https://github.com/redhat-chaos/krkn/blob/main/containers/build_own_image-README.md).
+ instructions given [here](https://github.com/krkn-chaos/krkn/blob/main/containers/build_own_image-README.md).
 
 **NOTE**: Helper functions for interactions in Krkn are part of [krkn-lib](https://github.com/redhat-chaos/krkn-lib). 
 Please feel free to reuse and expand them as you see fit when adding a new scenario or expanding 
@@ -19,10 +19,10 @@ the capabilities of the current supported scenarios.
 ### Git
 
 #### Clone the repository
-Pick the latest stable release to install [here](https://github.com/redhat-chaos/krkn/releases).
+Pick the latest stable release to install [here](https://github.com/krkn-chaos/krkn/releases).
 ```
-$ git clone https://github.com/redhat-chaos/krkn.git --branch <release version>
-$ cd kraken
+$ git clone https://github.com/krkn-chaos/krkn.git --branch <release version>
+$ cd krkn
 ```
 
 #### Install the dependencies
@@ -40,29 +40,6 @@ $ python3.9 run_kraken.py --config <config_file_location>
 ```
 
 ### Run containerized version
-Assuming that the latest docker ( 17.05 or greater with multi-build support ) is installed on the host, run:
-```
-$ docker pull quay.io/redhat-chaos/krkn:latest
-$ docker run --name=kraken --net=host -v <path_to_kubeconfig>:/root/.kube/config:Z -v <path_to_kraken_config>:/root/kraken/config/config.yaml:Z -d quay.io/redhat-chaos/krkn:latest
-$ docker run --name=kraken --net=host -v <path_to_kubeconfig>:/root/.kube/config:Z -v <path_to_kraken_config>:/root/kraken/config/config.yaml:Z -v <path_to_scenarios_directory>:/root/kraken/scenarios:Z -d quay.io/redhat-chaos/krkn:latest #custom or tweaked scenario configs
-$ docker logs -f kraken
-```
+[Krkn-hub](https://github.com/krkn-chaos/krkn-hub) is a wrapper that allows running Krkn chaos scenarios via podman or docker runtime with scenario parameters/configuration defined as environment variables.
 
-Similarly, podman can be used to achieve the same:
-```
-$ podman pull quay.io/redhat-chaos/krkn
-$ podman run --name=kraken --net=host -v <path_to_kubeconfig>:/root/.kube/config:Z -v <path_to_kraken_config>:/root/kraken/config/config.yaml:Z -d quay.io/redhat-chaos/krkn:latest
-$ podman run --name=kraken --net=host -v <path_to_kubeconfig>:/root/.kube/config:Z -v <path_to_kraken_config>:/root/kraken/config/config.yaml:Z -v <path_to_scenarios_directory>:/root/kraken/scenarios:Z -d quay.io/redhat-chaos/krkn:latest #custom or tweaked scenario configs
-$ podman logs -f kraken
-```
-
-If you want to build your own kraken image see [here](https://github.com/redhat-chaos/krkn/blob/main/containers/build_own_image-README.md)
-
-
-### Run Kraken as a Kubernetes deployment
-Refer [Instructions](https://github.com/redhat-chaos/krkn/blob/main/containers/README.md) on how to deploy and run Kraken as a Kubernetes/OpenShift deployment.
-
-
-Refer to the [chaos-kraken chart manpage](https://artifacthub.io/packages/helm/startx/chaos-kraken)
-and especially the [kraken configuration values](https://artifacthub.io/packages/helm/startx/chaos-kraken#chaos-kraken-values-dictionary) 
-for details on how to configure this chart.
+Refer [instructions](https://github.com/krkn-chaos/krkn-hub#supported-chaos-scenarios) to get started.

docs/metrics.md

@@ -1,51 +0,0 @@
-## Scraping and storing metrics for the run
-
-There are cases where the state of the cluster and metrics on the cluster during the chaos test run need to be stored long term to review after the cluster is terminated, for example CI and automation test runs. To help with this, Kraken supports capturing metrics for the duration of the scenarios defined in the config and indexes them into Elasticsearch. The indexed metrics can be visualized with the help of Grafana.
-
-It uses [Kube-burner](https://github.com/cloud-bulldozer/kube-burner) under the hood. The metrics to capture need to be defined in a metrics profile which Kraken consumes to query prometheus ( installed by default in OpenShift ) with the start and end timestamp of the run. Each run has a unique identifier ( uuid ) and all the metrics/documents in Elasticsearch will be associated with it. The uuid is generated automatically if not set in the config. This feature can be enabled in the [config](https://github.com/redhat-chaos/krkn/blob/main/config/config.yaml) by setting the following:
-
-```
-performance_monitoring:
-    kube_burner_binary_url: "https://github.com/cloud-bulldozer/kube-burner/releases/download/v0.9.1/kube-burner-0.9.1-Linux-x86_64.tar.gz"
-    capture_metrics: True
-    config_path: config/kube_burner.yaml                  # Define the Elasticsearch url and index name in this config.
-    metrics_profile_path: config/metrics-aggregated.yaml
-    prometheus_url:                                       # The prometheus url/route is automatically obtained in case of OpenShift, please set it when the distribution is Kubernetes.
-    prometheus_bearer_token:                              # The bearer token is automatically obtained in case of OpenShift, please set it when the distribution is Kubernetes. This is needed to authenticate with prometheus.
-    uuid:                                                 # uuid for the run is generated by default if not set.
-```
-
-### Metrics profile
-A couple of [metric profiles](https://github.com/redhat-chaos/krkn/tree/main/config), [metrics.yaml](https://github.com/redhat-chaos/krkn/blob/main/config/metrics.yaml), and [metrics-aggregated.yaml](https://github.com/redhat-chaos/krkn/blob/main/config/metrics-aggregated.yaml) are shipped by default and can be tweaked to add more metrics to capture during the run. The following are the API server metrics for example:
-
-```
-metrics:
-# API server
-  - query: histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{apiserver="kube-apiserver", verb!~"WATCH", subresource!="log"}[2m])) by (verb,resource,subresource,instance,le)) > 0
-    metricName: API99thLatency
-
-  - query: sum(irate(apiserver_request_total{apiserver="kube-apiserver",verb!="WATCH",subresource!="log"}[2m])) by (verb,instance,resource,code) > 0
-    metricName: APIRequestRate
-
-  - query: sum(apiserver_current_inflight_requests{}) by (request_kind) > 0
-    metricName: APIInflightRequests
-```
-
-### Indexing
-Define the Elasticsearch and index to store the metrics/documents in the kube_burner config:
-
-```
-global:
-  writeToFile: true
-  metricsDirectory: collected-metrics
-  measurements:
-    - name: podLatency
-      esIndex: kube-burner
-
-  indexerConfig:
-    enabled: true
-    esServers: [https://elastic.example.com:9200]
-    insecureSkipVerify: true
-    defaultIndex: kraken
-    type: elastic
-```

docs/network_chaos.md

@@ -12,13 +12,13 @@ network_chaos:                                    # Scenario to create an outage
   - "ens5"                                        # Interface name would be the Kernel host network interface name.
   execution: serial|parallel                      # Execute each of the egress options as a single scenario(parallel) or as separate scenario(serial).
   egress:
-    latency: 50ms
-    loss: 0.02                                    # percentage
-    bandwidth: 100mbit
+    latency: 500ms
+    loss: 50%                                    # percentage
+    bandwidth: 10mbit
 ```
 
 ##### Sample scenario config for ingress traffic shaping (using a plugin)
-'''
+```
 - id: network_chaos
   config:
     node_interface_name:                            # Dictionary with key as node name(s) and value as a list of its interfaces to test
@@ -30,12 +30,12 @@ network_chaos:                                    # Scenario to create an outage
     kubeconfig_path: ~/.kube/config                 # Path to kubernetes config file. If not specified, it defaults to ~/.kube/config
     execution_type: parallel                        # Execute each of the ingress options as a single scenario(parallel) or as separate scenario(serial).
     network_params:
-        latency: 50ms
-        loss: '0.02'
-        bandwidth: 100mbit
+        latency: 500ms
+        loss: '50%'
+        bandwidth: 10mbit
     wait_duration: 120
     test_duration: 60
-  '''
+```
 
   Note: For ingress traffic shaping, ensure that your node doesn't have any [IFB](https://wiki.linuxfoundation.org/networking/ifb) interfaces already present. The scenario relies on creating IFBs to do the shaping, and they are deleted at the end of the scenario.
 

docs/node_scenarios.md

@@ -4,29 +4,36 @@ The following node chaos scenarios are supported:
 
 1. **node_start_scenario**: Scenario to stop the node instance.
 2. **node_stop_scenario**: Scenario to stop the node instance.
-3. **node_stop_start_scenario**: Scenario to stop and then start the node instance. Not supported on VMware.
+3. **node_stop_start_scenario**: Scenario to stop the node instance for specified duration and then start the node instance. Not supported on VMware.
 4. **node_termination_scenario**: Scenario to terminate the node instance.
 5. **node_reboot_scenario**: Scenario to reboot the node instance.
 6. **stop_kubelet_scenario**: Scenario to stop the kubelet of the node instance.
 7. **stop_start_kubelet_scenario**: Scenario to stop and start the kubelet of the node instance.
-8. **node_crash_scenario**: Scenario to crash the node instance.
-9. **stop_start_helper_node_scenario**: Scenario to stop and start the helper node and check service status.
+8. **restart_kubelet_scenario**: Scenario to restart the kubelet of the node instance.
+9. **node_crash_scenario**: Scenario to crash the node instance.
+10. **stop_start_helper_node_scenario**: Scenario to stop and start the helper node and check service status.
+11. **node_disk_detach_attach_scenario**: Scenario to detach node disk for specified duration.
 
 
 **NOTE**: If the node does not recover from the node_crash_scenario injection, reboot the node to get it back to Ready state.
 
 **NOTE**: node_start_scenario, node_stop_scenario, node_stop_start_scenario, node_termination_scenario
-, node_reboot_scenario and stop_start_kubelet_scenario are supported only on AWS, Azure, OpenStack, BareMetal, GCP
-, VMware and Alibaba as of now.
+, node_reboot_scenario and stop_start_kubelet_scenario are supported on AWS, Azure, OpenStack, BareMetal, GCP
+, VMware and Alibaba.
 
-**NOTE**: Node scenarios are supported only when running the standalone version of Kraken until https://github.com/redhat-chaos/krkn/issues/106 gets fixed.
+**NOTE**: node_disk_detach_attach_scenario is supported only on AWS and cannot detach root disk.
 
 
 #### AWS
 
-How to set up AWS cli to run node scenarios is defined [here](cloud_setup.md#aws).
+Cloud setup instructions can be found [here](cloud_setup.md#aws). Sample scenario config can be found [here](https://github.com/krkn-chaos/krkn/blob/main/scenarios/openshift/aws_node_scenarios.yml).
+
+
 
 #### Baremetal
+
+Sample scenario config can be found [here](https://github.com/krkn-chaos/krkn/blob/main/scenarios/openshift/baremetal_node_scenarios.yml).
+
 **NOTE**: Baremetal requires setting the IPMI user and password to power on, off, and reboot nodes, using the config options `bm_user` and `bm_password`. It can either be set in the root of the entry in the scenarios config, or it can be set per machine.
 
 If no per-machine addresses are specified, kraken attempts to use the BMC value in the BareMetalHost object. To list them, you can do 'oc get bmh -o wide --all-namespaces'. If the BMC values are blank, you must specify them per-machine using the config option 'bmc_addr' as specified below.
@@ -38,6 +45,8 @@ See the example node scenario or the example below.
 
 **NOTE**: Baremetal machines are fragile. Some node actions can occasionally corrupt the filesystem if it does not shut down properly, and sometimes the kubelet does not start properly.
 
+
+
 #### Docker
 
 The Docker provider can be used to run node scenarios against kind clusters.
@@ -46,8 +55,13 @@ The Docker provider can be used to run node scenarios against kind clusters.
 
 kind was primarily designed for testing Kubernetes itself, but may be used for local development or CI.
 
+
+
 #### GCP
-How to set up GCP cli to run node scenarios is defined [here](cloud_setup.md#gcp).
+Cloud setup instructions can be found [here](cloud_setup.md#gcp). Sample scenario config can be found [here](https://github.com/krkn-chaos/krkn/blob/main/scenarios/openshift/gcp_node_scenarios.yml).
+
+NOTE: The parallel option is not available for GCP, the api doesn't perform processes at the same time
+
 
 #### Openstack
 
@@ -60,9 +74,11 @@ The supported node level chaos scenarios on an OPENSTACK cloud are `node_stop_st
 To execute the scenario, ensure the value for `ssh_private_key` in the node scenarios config file is set with the correct private key file path for ssh connection to the helper node. Ensure passwordless ssh is configured on the host running Kraken and the helper node to avoid connection errors.
 
 
+
 #### Azure
 
-How to set up Azure cli to run node scenarios is defined [here](cloud_setup.md#azure).
+Cloud setup instructions can be found [here](cloud_setup.md#azure). Sample scenario config can be found [here](https://github.com/krkn-chaos/krkn/blob/main/scenarios/openshift/azure_node_scenarios.yml).
+
 
 
 #### Alibaba
@@ -73,43 +89,28 @@ How to set up Alibaba cli to run node scenarios is defined [here](cloud_setup.md
 . Releasing a node is 2 steps, stopping the node and then releasing it.
 
 
+
 #### VMware
 How to set up VMware vSphere to run node scenarios is defined [here](cloud_setup.md#vmware)
 
 This cloud type uses a different configuration style, see actions below and [example config file](../scenarios/openshift/vmware_node_scenarios.yml)
 
-*vmware-node-terminate, vmware-node-reboot, vmware-node-stop, vmware-node-start*
+- vmware-node-terminate
+- vmware-node-reboot
+- vmware-node-stop
+- vmware-node-start
+
+
 
 #### IBMCloud
 How to set up IBMCloud to run node scenarios is defined [here](cloud_setup.md#ibmcloud)
 
 This cloud type uses a different configuration style, see actions below and [example config file](../scenarios/openshift/ibmcloud_node_scenarios.yml)
 
-*ibmcloud-node-terminate, ibmcloud-node-reboot, ibmcloud-node-stop, ibmcloud-node-start
-*
-
-
-#### IBMCloud and Vmware example 
-
-
-```
-- id: ibmcloud-node-stop
-  config:
-    name: "<node_name>"        
-    label_selector: "node-role.kubernetes.io/worker"    # When node_name is not specified, a node with matching label_selector is selected for node chaos scenario injection 
-    runs: 1                             # Number of times to inject each scenario under actions (will perform on same node each time)                                                           
-    instance_count: 1                   # Number of nodes to perform action/select that match the label selector                                             
-    timeout: 30                        # Duration to wait for completion of node scenario injection
-    skip_openshift_checks: False       # Set to True if you don't want to wait for the status of the nodes to change on OpenShift before passing the scenario 
-- id: ibmcloud-node-start
-  config:
-    name: "<node_name>" #Same name as before       
-    label_selector: "node-role.kubernetes.io/worker"    # When node_name is not specified, a node with matching label_selector is selected for node chaos scenario injection 
-    runs: 1                             # Number of times to inject each scenario under actions (will perform on same node each time)                                                           
-    instance_count: 1                   # Number of nodes to perform action/select that match the label selector                                             
-    timeout: 30                        # Duration to wait for completion of node scenario injection
-    skip_openshift_checks: False       # Set to True if you don't want to wait for the status of the nodes to change on OpenShift before passing the scenario 
-    ```
+- ibmcloud-node-terminate
+- ibmcloud-node-reboot
+- ibmcloud-node-stop
+- ibmcloud-node-start
 
 
 
@@ -118,60 +119,3 @@ This cloud type uses a different configuration style, see actions below and [exa
 **NOTE**: The `node_crash_scenario` and `stop_kubelet_scenario` scenario is supported independent of the cloud platform.
 
 Use 'generic' or do not add the 'cloud_type' key to your scenario if your cluster is not set up using one of the current supported cloud types.
-
-Node scenarios can be injected by placing the node scenarios config files under node_scenarios option in the kraken config. Refer to [node_scenarios_example](https://github.com/redhat-chaos/krkn/blob/main/scenarios/node_scenarios_example.yml) config file.
-
-
-```
-node_scenarios:
-  - actions:                                                        # Node chaos scenarios to be injected.
-    - node_stop_start_scenario
-    - stop_start_kubelet_scenario
-    - node_crash_scenario
-    node_name:                                                      # Node on which scenario has to be injected.
-    label_selector: node-role.kubernetes.io/worker                  # When node_name is not specified, a node with matching label_selector is selected for node chaos scenario injection.
-    instance_count: 1                                               # Number of nodes to perform action/select that match the label selector.
-    runs: 1                                                         # Number of times to inject each scenario under actions (will perform on same node each time).
-    timeout: 120                                                    # Duration to wait for completion of node scenario injection.
-    cloud_type: aws                                                 # Cloud type on which Kubernetes/OpenShift runs.
-  - actions:
-    - node_reboot_scenario
-    node_name:
-    label_selector: node-role.kubernetes.io/infra
-    instance_count: 1
-    timeout: 120
-    cloud_type: azure
-  - actions:
-    - node_crash_scenario
-    node_name:
-    label_selector: node-role.kubernetes.io/infra
-    instance_count: 1
-    timeout: 120
-  - actions:
-    - stop_start_helper_node_scenario                               # Node chaos scenario for helper node.
-    instance_count: 1
-    timeout: 120
-    helper_node_ip:                                                 # ip address of the helper node.
-    service:                                                        # Check status of the services on the helper node.
-      - haproxy
-      - dhcpd
-      - named
-    ssh_private_key: /root/.ssh/id_rsa                              # ssh key to access the helper node.
-    cloud_type: openstack
-  - actions:
-    - node_stop_start_scenario
-    node_name:
-    label_selector: node-role.kubernetes.io/worker
-    instance_count: 1
-    timeout: 120
-    cloud_type: bm
-    bmc_user: defaultuser                                           # For baremetal (bm) cloud type. The default IPMI username. Optional if specified for all machines.
-    bmc_password: defaultpass                                       # For baremetal (bm) cloud type. The default IPMI password. Optional if specified for all machines.
-    bmc_info:                                                       # This section is here to specify baremetal per-machine info, so it is optional if there is no per-machine info.
-      node-1:                                                       # The node name for the baremetal machine
-        bmc_addr: mgmt-machine1.example.com                         # Optional. For baremetal nodes with the IPMI BMC address missing from 'oc get bmh'.
-      node-2:
-        bmc_addr: mgmt-machine2.example.com
-        bmc_user: user                                              # The baremetal IPMI user. Overrides the default IPMI user specified above. Optional if the default is set.
-        bmc_password: pass                                          # The baremetal IPMI password. Overrides the default IPMI user specified above. Optional if the default is set.
-```

docs/pod_network_scenarios.md

@@ -27,6 +27,15 @@ Scenario to introduce network latency, packet loss, and bandwidth restriction in
     network_params:
         latency: 500ms             # Add 500ms latency to egress traffic from the pod.
 ```
+##### Sample scenario config for ingress traffic shaping (using plugin)
+```
+- id: pod_ingress_shaping
+  config:
+    namespace: openshift-console   # Required - Namespace of the pod to which filter need to be applied.
+    label_selector: 'component=ui' # Applies traffic shaping to access openshift console.
+    network_params:
+        latency: 500ms             # Add 500ms latency to egress traffic from the pod.
+```
 
 ##### Steps
  - Pick the pods to introduce the network anomaly either from label_selector or pod_name.

docs/pod_scenarios.md

@@ -17,11 +17,8 @@ You can then create the scenario file with the following contents:
   config:
     namespace_pattern: ^kube-system$
     label_selector: k8s-app=kube-scheduler
-- id: wait-for-pods
-  config:
-    namespace_pattern: ^kube-system$
-    label_selector: k8s-app=kube-scheduler
-    count: 3
+    krkn_pod_recovery_time: 120
+    
 ```
 
 Please adjust the schema reference to point to the [schema file](../scenarios/plugin.schema.json). This file will give you code completion and documentation for the available options in your IDE.

docs/scenario_plugin_api.md

@@ -0,0 +1,136 @@
+# Scenario Plugin API:
+
+This API enables seamless integration of Scenario Plugins for Krkn. Plugins are automatically 
+detected and loaded by the plugin loader, provided they extend the `AbstractPluginScenario` 
+abstract class, implement the required methods, and adhere to the specified [naming conventions](#naming-conventions).
+
+## Plugin folder:
+
+The plugin loader automatically loads plugins found in the `krkn/scenario_plugins` directory, 
+relative to the Krkn root folder. Each plugin must reside in its own directory and can consist 
+of one or more Python files. The entry point for each plugin is a Python class that extends the 
+[AbstractPluginScenario](../krkn/scenario_plugins/abstract_scenario_plugin.py) abstract class and implements its required methods.
+
+## `AbstractPluginScenario` abstract class:
+
+This [abstract class](../krkn/scenario_plugins/abstract_scenario_plugin.py) defines the contract between the plugin and krkn.
+It consists of two methods:
+- `run(...)`
+- `get_scenario_type()`
+
+Most IDEs can automatically suggest and implement the abstract methods defined in `AbstractPluginScenario`:
+![pycharm](scenario_plugin_pycharm.gif)
+_(IntelliJ PyCharm)_
+
+### `run(...)`
+
+```python
+    def run(
+        self,
+        run_uuid: str,
+        scenario: str,
+        krkn_config: dict[str, any],
+        lib_telemetry: KrknTelemetryOpenshift,
+        scenario_telemetry: ScenarioTelemetry,
+    ) -> int:
+
+```
+
+This method represents the entry point of the plugin and the first method 
+that will be executed.
+#### Parameters:
+
+- `run_uuid`:
+  - the uuid of the chaos run generated by krkn for every single run.
+- `scenario`:
+  - the config file of the scenario that is currently executed
+- `krkn_config`:
+  - the full dictionary representation of the `config.yaml`
+- `lib_telemetry`
+  - it is a composite object of all the [krkn-lib](https://krkn-chaos.github.io/krkn-lib-docs/modules.html) objects and methods needed by a krkn plugin to run.
+- `scenario_telemetry`
+  - the `ScenarioTelemetry` object of the scenario that is currently executed
+ 
+### Return value:
+Returns 0 if the scenario succeeds and 1 if it fails.
+> [!WARNING]
+> All the exception must be handled __inside__ the run method and not propagated.
+
+### `get_scenario_types()`:
+
+```python    def get_scenario_types(self) -> list[str]:```
+
+Indicates the scenario types specified in the `config.yaml`. For the plugin to be properly
+loaded, recognized and executed, it must be implemented and must return one or more
+strings matching `scenario_type` strings set in the config.
+> [!WARNING]
+> Multiple strings can map to a *single*  `ScenarioPlugin` but the same string cannot map
+> to different plugins, an exception will be thrown for scenario_type redefinition.
+
+> [!Note]  
+> The `scenario_type` strings must be unique across all plugins; otherwise, an exception will be thrown.
+
+## Naming conventions:
+A key requirement for developing a plugin that will be properly loaded 
+by the plugin loader is following the established naming conventions. 
+These conventions are enforced to maintain a uniform and readable codebase, 
+making it easier to onboard new developers from the community.
+
+### plugin folder:
+- the plugin folder must be placed in the `krkn/scenario_plugin` folder starting from the krkn root folder
+- the plugin folder __cannot__ contain the words
+  - `plugin`
+  - `scenario`
+### plugin file name and class name:
+- the plugin file containing the main plugin class must be named in _snake case_ and must have the suffix `_scenario_plugin`: 
+  - `example_scenario_plugin.py`
+- the main plugin class must named in _capital camel case_ and must have the suffix `ScenarioPlugin` : 
+  - `ExampleScenarioPlugin`
+- the file name must match the class name in the respective syntax:
+  - `example_scenario_plugin.py` -> `ExampleScenarioPlugin`
+
+### scenario type:
+- the scenario type __must__ be unique between all the scenarios.
+
+### logging:
+If your new scenario does not adhere to the naming conventions, an error log will be generated in the Krkn standard output,
+providing details about the issue:
+
+```commandline
+2024-10-03 18:06:31,136 [INFO] 📣 `ScenarioPluginFactory`: types from config.yaml mapped to respective classes for execution:
+2024-10-03 18:06:31,136 [INFO]   ✅ type: application_outages_scenarios ➡️ `ApplicationOutageScenarioPlugin` 
+2024-10-03 18:06:31,136 [INFO]   ✅ types: [hog_scenarios, arcaflow_scenario] ➡️ `ArcaflowScenarioPlugin` 
+2024-10-03 18:06:31,136 [INFO]   ✅ type: container_scenarios ➡️ `ContainerScenarioPlugin` 
+2024-10-03 18:06:31,136 [INFO]   ✅ type: managedcluster_scenarios ➡️ `ManagedClusterScenarioPlugin` 
+2024-10-03 18:06:31,137 [INFO]   ✅ types: [pod_disruption_scenarios, pod_network_scenario, vmware_node_scenarios, ibmcloud_node_scenarios] ➡️ `NativeScenarioPlugin` 
+2024-10-03 18:06:31,137 [INFO]   ✅ type: network_chaos_scenarios ➡️ `NetworkChaosScenarioPlugin` 
+2024-10-03 18:06:31,137 [INFO]   ✅ type: node_scenarios ➡️ `NodeActionsScenarioPlugin` 
+2024-10-03 18:06:31,137 [INFO]   ✅ type: pvc_scenarios ➡️ `PvcScenarioPlugin` 
+2024-10-03 18:06:31,137 [INFO]   ✅ type: service_disruption_scenarios ➡️ `ServiceDisruptionScenarioPlugin` 
+2024-10-03 18:06:31,137 [INFO]   ✅ type: service_hijacking_scenarios ➡️ `ServiceHijackingScenarioPlugin` 
+2024-10-03 18:06:31,137 [INFO]   ✅ type: cluster_shut_down_scenarios ➡️ `ShutDownScenarioPlugin` 
+2024-10-03 18:06:31,137 [INFO]   ✅ type: syn_flood_scenarios ➡️ `SynFloodScenarioPlugin` 
+2024-10-03 18:06:31,137 [INFO]   ✅ type: time_scenarios ➡️ `TimeActionsScenarioPlugin` 
+2024-10-03 18:06:31,137 [INFO]   ✅ type: zone_outages_scenarios ➡️ `ZoneOutageScenarioPlugin`
+
+2024-09-18 14:48:41,735 [INFO] Failed to load Scenario Plugins:
+
+2024-09-18 14:48:41,735 [ERROR] ⛔ Class: ExamplePluginScenario Module: krkn.scenario_plugins.example.example_scenario_plugin
+2024-09-18 14:48:41,735 [ERROR] ⚠️ scenario plugin class name must start with a capital letter, end with `ScenarioPlugin`, and cannot be just `ScenarioPlugin`.
+```
+
+>[!NOTE]
+>If you're trying to understand how the scenario types in the config.yaml are mapped to 
+> their corresponding plugins, this log will guide you! 
+> Each scenario plugin class mentioned can be found in the `krkn/scenario_plugin` folder
+> simply convert the camel case notation and remove the ScenarioPlugin suffix from the class name
+> e.g `ShutDownScenarioPlugin` class can be found in the `krkn/scenario_plugin/shut_down` folder.
+
+## ExampleScenarioPlugin
+The [ExampleScenarioPlugin](../krkn/tests/test_classes/example_scenario_plugin.py) class included in the tests folder can be used as a scaffolding for new plugins and it is considered
+part of the documentation.
+
+For any questions or further guidance, feel free to reach out to us on the 
+[Kubernetes workspace](https://kubernetes.slack.com/) in the `#krkn` channel. 
+We’re happy to assist. Now, __release the Krkn!__
+

docs/service_disruption_scenarios.md

@@ -16,7 +16,7 @@ Set to '^.*$' and label_selector to "" to randomly select any namespace in your
 
 **sleep:** Number of seconds to wait between each iteration/count of killing namespaces. Defaults to 10 seconds if not set
 
-Refer to [namespace_scenarios_example](https://github.com/redhat-chaos/krkn/blob/main/scenarios/regex_namespace.yaml) config file.
+Refer to [namespace_scenarios_example](https://github.com/krkn-chaos/krkn/blob/main/scenarios/regex_namespace.yaml) config file.
 
 ```
 scenarios:

docs/service_hijacking_scenarios.md

@@ -0,0 +1,80 @@
+### Service Hijacking Scenarios
+
+Service Hijacking Scenarios aim to simulate fake HTTP responses from a workload targeted by a 
+`Service` already deployed in the cluster. 
+This scenario is executed by deploying a custom-made web service and modifying the target `Service`
+selector to direct traffic to this web service for a specified duration.
+
+The web service's source code is available [here](https://github.com/krkn-chaos/krkn-service-hijacking). 
+It employs a time-based test plan from the scenario configuration file, which specifies the behavior of resources during the chaos scenario as follows:
+
+```yaml
+service_target_port: http-web-svc # The port of the service to be hijacked (can be named or numeric, based on the workload and service configuration).
+service_name: nginx-service # The name of the service that will be hijacked.
+service_namespace: default # The namespace where the target service is located.
+image: quay.io/krkn-chaos/krkn-service-hijacking:v0.1.3 # Image of the krkn web service to be deployed to receive traffic.
+chaos_duration: 30 # Total duration of the chaos scenario in seconds.
+plan:
+  - resource: "/list/index.php" # Specifies the resource or path to respond to in the scenario. For paths, both the path and query parameters are captured but ignored. For resources, only query parameters are captured.
+
+    steps:                      # A time-based plan consisting of steps can be defined for each resource.
+      GET:                      # One or more HTTP methods can be specified for each step. Note: Non-standard methods are supported for fully custom web services (e.g., using NONEXISTENT instead of POST).
+
+        - duration: 15          # Duration in seconds for this step before moving to the next one, if defined. Otherwise, this step will continue until the chaos scenario ends.
+
+          status: 500           # HTTP status code to be returned in this step.
+          mime_type: "application/json" # MIME type of the response for this step.
+          payload: |            # The response payload for this step.
+            {
+              "status":"internal server error"
+            }
+        - duration: 15
+          status: 201
+          mime_type: "application/json"
+          payload: |
+            {
+              "status":"resource created"
+            }
+      POST:
+        - duration: 15
+          status: 401
+          mime_type: "application/json"
+          payload: |
+            {
+               "status": "unauthorized"
+            }
+        - duration: 15
+          status: 404
+          mime_type: "text/plain"
+          payload: "not found"
+
+
+```
+The scenario will focus on the `service_name` within the `service_namespace`, 
+substituting the selector with a randomly generated one, which is added as a label in the mock service manifest.
+This allows multiple scenarios to be executed in the same namespace, each targeting different services without 
+causing conflicts.
+
+The newly deployed mock web service will expose a `service_target_port`, 
+which can be either a named or numeric port based on the service configuration. 
+This ensures that the Service correctly routes HTTP traffic to the mock web service during the chaos run.
+
+Each step will last for `duration` seconds from the deployment of the mock web service in the cluster. 
+For each HTTP resource, defined as a top-level YAML property of the plan 
+(it could be a specific resource, e.g., /list/index.php, or a path-based resource typical in MVC frameworks), 
+one or more HTTP request methods can be specified. Both standard and custom request methods are supported.
+
+During this time frame, the web service will respond with:
+
+- `status`: The [HTTP status code](https://datatracker.ietf.org/doc/html/rfc7231#section-6) (can be standard or custom).
+- `mime_type`: The [MIME type](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types) (can be standard or custom).
+- `payload`: The response body to be returned to the client.
+
+At the end of the step `duration`, the web service will proceed to the next step (if available) until 
+the global `chaos_duration` concludes. At this point, the original service will be restored, 
+and the custom web service and its resources will be undeployed.
+
+__NOTE__: Some clients (e.g., cURL, jQuery) may optimize queries using lightweight methods (like HEAD or OPTIONS) 
+to probe API behavior. If these methods are not defined in the test plan, the web service may respond with 
+a `405` or `404` status code. If you encounter unexpected behavior, consider this use case.
+

docs/syn_flood_scenarios.md

@@ -0,0 +1,33 @@
+### SYN Flood Scenarios
+
+This scenario generates a substantial amount of TCP traffic directed at one or more Kubernetes services within 
+the cluster to test the server's resiliency under extreme traffic conditions. 
+It can also target hosts outside the cluster by specifying a reachable IP address or hostname. 
+This scenario leverages the distributed nature of Kubernetes clusters to instantiate multiple instances 
+of the same pod against a single host, significantly increasing the effectiveness of the attack. 
+The configuration also allows for the specification of multiple node selectors, enabling Kubernetes to schedule 
+the attacker pods on a user-defined subset of nodes to make the test more realistic.
+
+ ```yaml
+packet-size: 120 # hping3 packet size
+window-size: 64 # hping 3 TCP window size
+duration: 10 # chaos scenario duration
+namespace: default # namespace where the target service(s) are deployed
+target-service: target-svc # target service name (if set target-service-label must be empty)
+target-port: 80 # target service TCP port
+target-service-label : "" # target service label, can be used to target multiple target at the same time
+                          # if they have the same label set (if set target-service must be empty)
+number-of-pods: 2 # number of attacker pod instantiated per each target
+image: quay.io/krkn-chaos/krkn-syn-flood # syn flood attacker container image
+attacker-nodes: # this will set the node affinity to schedule the attacker node. Per each node label selector
+                # can be specified multiple values in this way the kube scheduler will schedule the attacker pods
+                # in the best way possible based on the provided labels. Multiple labels can be specified
+  kubernetes.io/hostname:
+    - host_1
+    - host_2
+  kubernetes.io/os:
+    - linux
+
+ ```
+
+The attacker container source code is available [here](https://github.com/krkn-chaos/krkn-syn-flood).

docs/time_scenarios.md

@@ -16,7 +16,7 @@ Configuration Options:
 
 **object_name:** List of the names of pods or nodes you want to skew.
 
-Refer to [time_scenarios_example](https://github.com/redhat-chaos/krkn/blob/main/scenarios/time_scenarios_example.yml) config file.
+Refer to [time_scenarios_example](https://github.com/krkn-chaos/krkn/blob/main/scenarios/time_scenarios_example.yml) config file.
 
 ```
 time_scenarios:

docs/zone_outage.md

@@ -13,10 +13,12 @@ zone_outage:                                         # Scenario to create an out
   duration: 600                                      # Duration in seconds after which the zone will be back online.
   vpc_id:                                            # Cluster virtual private network to target.
   subnet_id: [subnet1, subnet2]                      # List of subnet-id's to deny both ingress and egress traffic.
+  default_acl_id: acl-xxxxxxxx                       # (Optional) ID of an existing network ACL to use instead of creating a new one. If provided, this ACL will not be deleted after the scenario.
 ```
 
 **NOTE**: vpc_id and subnet_id can be obtained from the cloud web console by selecting one of the instances in the targeted zone ( us-west-2a for example ).
 **NOTE**: Multiple zones will experience downtime in case of targeting multiple subnets which might have an impact on the cluster health especially if the zones have control plane components deployed.
+**NOTE**: default_acl_id can be obtained from the AWS VPC Console by selecting "Network ACLs" from the left sidebar ( the ID will be in the format 'acl-xxxxxxxx' ). Make sure the selected ACL has the desired ingress/egress rules for your outage scenario ( i.e., deny all ).
 
 ##### Debugging steps in case of failures
 In case of failures during the steps which revert back the network acl to allow traffic and bring back the cluster nodes in the zone, the nodes in the particular zone will be in `NotReady` condition. Here is how to fix it:

kind-config.yml

@@ -2,6 +2,9 @@ kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
 nodes:
   - role: control-plane
+    extraPortMappings:
+      - containerPort: 30036
+        hostPort: 8888
   - role: control-plane
   - role: control-plane
   - role: worker

kraken/application_outage/actions.py

@@ -1,76 +0,0 @@
-import yaml
-import logging
-import time
-import kraken.cerberus.setup as cerberus
-from jinja2 import Template
-import kraken.invoke.command as runcommand
-from krkn_lib.telemetry import KrknTelemetry
-from krkn_lib.models.telemetry import ScenarioTelemetry
-
-# Reads the scenario config, applies and deletes a network policy to
-# block the traffic for the specified duration
-def run(scenarios_list, config, wait_duration, telemetry: KrknTelemetry) -> (list[str], list[ScenarioTelemetry]):
-    failed_post_scenarios = ""
-    scenario_telemetries: list[ScenarioTelemetry] = []
-    failed_scenarios = []
-    for app_outage_config in scenarios_list:
-        scenario_telemetry = ScenarioTelemetry()
-        scenario_telemetry.scenario = app_outage_config
-        scenario_telemetry.startTimeStamp = time.time()
-        telemetry.set_parameters_base64(scenario_telemetry, app_outage_config)
-        if len(app_outage_config) > 1:
-            try:
-                with open(app_outage_config, "r") as f:
-                    app_outage_config_yaml = yaml.full_load(f)
-                    scenario_config = app_outage_config_yaml["application_outage"]
-                    pod_selector = scenario_config.get("pod_selector", "{}")
-                    traffic_type = scenario_config.get("block", "[Ingress, Egress]")
-                    namespace = scenario_config.get("namespace", "")
-                    duration = scenario_config.get("duration", 60)
-
-                    start_time = int(time.time())
-
-                    network_policy_template = """---
-apiVersion: networking.k8s.io/v1
-kind: NetworkPolicy
-metadata:
-  name: kraken-deny
-spec:
-  podSelector:
-    matchLabels: {{ pod_selector }}
-  policyTypes: {{ traffic_type }}
-                    """
-                    t = Template(network_policy_template)
-                    rendered_spec = t.render(pod_selector=pod_selector, traffic_type=traffic_type)
-                    # Write the rendered template to a file
-                    with open("kraken_network_policy.yaml", "w") as f:
-                        f.write(rendered_spec)
-                    # Block the traffic by creating network policy
-                    logging.info("Creating the network policy")
-                    runcommand.invoke(
-                        "kubectl create -f %s -n %s --validate=false" % ("kraken_network_policy.yaml", namespace)
-                    )
-
-                    # wait for the specified duration
-                    logging.info("Waiting for the specified duration in the config: %s" % (duration))
-                    time.sleep(duration)
-
-                    # unblock the traffic by deleting the network policy
-                    logging.info("Deleting the network policy")
-                    runcommand.invoke("kubectl delete -f %s -n %s" % ("kraken_network_policy.yaml", namespace))
-
-                    logging.info("End of scenario. Waiting for the specified duration: %s" % (wait_duration))
-                    time.sleep(wait_duration)
-
-                    end_time = int(time.time())
-                    cerberus.publish_kraken_status(config, failed_post_scenarios, start_time, end_time)
-            except Exception as e :
-                scenario_telemetry.exitStatus = 1
-                failed_scenarios.append(app_outage_config)
-                telemetry.log_exception(app_outage_config)
-            else:
-                scenario_telemetry.exitStatus = 0
-            scenario_telemetry.endTimeStamp = time.time()
-            scenario_telemetries.append(scenario_telemetry)
-    return failed_scenarios, scenario_telemetries
-

kraken/arcaflow_plugin/__init__.py

@@ -1,2 +0,0 @@
-from .arcaflow_plugin import *
-from .context_auth import ContextAuth

kraken/arcaflow_plugin/arcaflow_plugin.py

@@ -1,178 +0,0 @@
-import time
-import arcaflow
-import os
-import yaml
-import logging
-from pathlib import Path
-from typing import List
-from .context_auth import ContextAuth
-from krkn_lib.telemetry import KrknTelemetry
-from krkn_lib.models.telemetry import ScenarioTelemetry
-
-
-def run(scenarios_list: List[str], kubeconfig_path: str, telemetry: KrknTelemetry) -> (list[str], list[ScenarioTelemetry]):
-    scenario_telemetries: list[ScenarioTelemetry] = []
-    failed_post_scenarios = []
-    for scenario in scenarios_list:
-        scenario_telemetry = ScenarioTelemetry()
-        scenario_telemetry.scenario = scenario
-        scenario_telemetry.startTimeStamp = time.time()
-        telemetry.set_parameters_base64(scenario_telemetry,scenario)
-        engine_args = build_args(scenario)
-        status_code = run_workflow(engine_args, kubeconfig_path)
-        scenario_telemetry.endTimeStamp = time.time()
-        scenario_telemetry.exitStatus = status_code
-        scenario_telemetries.append(scenario_telemetry)
-        if status_code != 0:
-            failed_post_scenarios.append(scenario)
-    return failed_post_scenarios, scenario_telemetries
-
-
-def run_workflow(engine_args: arcaflow.EngineArgs, kubeconfig_path: str) -> int:
-    set_arca_kubeconfig(engine_args, kubeconfig_path)
-    exit_status = arcaflow.run(engine_args)
-    return exit_status
-
-
-def build_args(input_file: str) -> arcaflow.EngineArgs:
-    """sets the kubeconfig parsed by setArcaKubeConfig as an input to the arcaflow workflow"""
-    context = Path(input_file).parent
-    workflow = "{}/workflow.yaml".format(context)
-    config = "{}/config.yaml".format(context)
-    if not os.path.exists(context):
-        raise Exception(
-            "context folder for arcaflow workflow not found: {}".format(
-                context)
-        )
-    if not os.path.exists(input_file):
-        raise Exception(
-            "input file for arcaflow workflow not found: {}".format(input_file))
-    if not os.path.exists(workflow):
-        raise Exception(
-            "workflow file for arcaflow workflow not found: {}".format(
-                workflow)
-        )
-    if not os.path.exists(config):
-        raise Exception(
-            "configuration file for arcaflow workflow not found: {}".format(
-                config)
-        )
-
-    engine_args = arcaflow.EngineArgs()
-    engine_args.context = context
-    engine_args.config = config
-    engine_args.input = input_file
-    return engine_args
-
-
-def set_arca_kubeconfig(engine_args: arcaflow.EngineArgs, kubeconfig_path: str):
-
-    context_auth = ContextAuth()
-    if not os.path.exists(kubeconfig_path):
-        raise Exception("kubeconfig not found in {}".format(kubeconfig_path))
-
-    with open(kubeconfig_path, "r") as stream:
-        try:
-            kubeconfig = yaml.safe_load(stream)
-            context_auth.fetch_auth_data(kubeconfig)
-        except Exception as e:
-            logging.error("impossible to read kubeconfig file in: {}".format(
-                    kubeconfig_path))
-            raise e
-
-    kubeconfig_str = set_kubeconfig_auth(kubeconfig, context_auth)
-
-    with open(engine_args.input, "r") as stream:
-        input_file = yaml.safe_load(stream)
-        if "input_list" in input_file and isinstance(input_file["input_list"],list):
-            for index, _ in enumerate(input_file["input_list"]):
-                if isinstance(input_file["input_list"][index], dict):
-                    input_file["input_list"][index]["kubeconfig"] = kubeconfig_str
-        else:
-            input_file["kubeconfig"] = kubeconfig_str
-        stream.close()
-    with open(engine_args.input, "w") as stream:
-        yaml.safe_dump(input_file, stream)
-
-    with open(engine_args.config, "r") as stream:
-        config_file = yaml.safe_load(stream)
-    if config_file["deployer"]["type"] == "kubernetes":
-        kube_connection = set_kubernetes_deployer_auth(config_file["deployer"]["connection"], context_auth)
-        config_file["deployer"]["connection"]=kube_connection
-        with open(engine_args.config, "w") as stream:
-            yaml.safe_dump(config_file, stream,explicit_start=True, width=4096)
-
-
-def set_kubernetes_deployer_auth(deployer: any, context_auth: ContextAuth) -> any:
-    if context_auth.clusterHost is not None :
-        deployer["host"] = context_auth.clusterHost
-    if context_auth.clientCertificateData is not None :
-        deployer["cert"] = context_auth.clientCertificateData
-    if context_auth.clientKeyData is not None:
-        deployer["key"] = context_auth.clientKeyData
-    if context_auth.clusterCertificateData is not None:
-        deployer["cacert"] = context_auth.clusterCertificateData
-    if context_auth.username is not None:
-        deployer["username"] = context_auth.username
-    if context_auth.password is not None:
-        deployer["password"] = context_auth.password
-    if context_auth.bearerToken is not None:
-        deployer["bearerToken"] = context_auth.bearerToken
-    return deployer
-
-
-def set_kubeconfig_auth(kubeconfig: any, context_auth: ContextAuth) -> str:
-    """
-    Builds an arcaflow-compatible kubeconfig representation and returns it as a string.
-    In order to run arcaflow plugins in kubernetes/openshift the kubeconfig must contain client certificate/key
-    and server certificate base64 encoded within the kubeconfig file itself in *-data fields. That is not always the
-    case, infact kubeconfig may contain filesystem paths to those files, this function builds an arcaflow-compatible
-    kubeconfig file and returns it as a string that can be safely included in input.yaml 
-    """
-
-    if "current-context" not in kubeconfig.keys():
-        raise Exception(
-            "invalid kubeconfig file, impossible to determine current-context"
-        )
-    user_id = None
-    cluster_id = None
-    user_name = None
-    cluster_name = None
-    current_context = kubeconfig["current-context"]
-    for context in kubeconfig["contexts"]:
-        if context["name"] == current_context:
-            user_name = context["context"]["user"]
-            cluster_name = context["context"]["cluster"]
-    if user_name is None:
-        raise Exception(
-            "user not set for context {} in kubeconfig file".format(current_context)
-        )
-    if cluster_name is None:
-        raise Exception(
-            "cluster not set for context {} in kubeconfig file".format(current_context)
-        )
-
-    for index, user in enumerate(kubeconfig["users"]):
-        if user["name"] == user_name:
-            user_id = index
-    for index, cluster in enumerate(kubeconfig["clusters"]):
-        if cluster["name"] == cluster_name:
-            cluster_id = index
-
-    if cluster_id is None:
-        raise Exception(
-            "no cluster {} found in kubeconfig users".format(cluster_name)
-        )
-    if "client-certificate" in kubeconfig["users"][user_id]["user"]:
-        kubeconfig["users"][user_id]["user"]["client-certificate-data"] = context_auth.clientCertificateDataBase64
-        del kubeconfig["users"][user_id]["user"]["client-certificate"]
-
-    if "client-key" in kubeconfig["users"][user_id]["user"]:
-        kubeconfig["users"][user_id]["user"]["client-key-data"] = context_auth.clientKeyDataBase64
-        del kubeconfig["users"][user_id]["user"]["client-key"]
-
-    if "certificate-authority" in kubeconfig["clusters"][cluster_id]["cluster"]:
-        kubeconfig["clusters"][cluster_id]["cluster"]["certificate-authority-data"] = context_auth.clusterCertificateDataBase64
-        del kubeconfig["clusters"][cluster_id]["cluster"]["certificate-authority"]
-    kubeconfig_str = yaml.dump(kubeconfig)
-    return kubeconfig_str

kraken/kube_burner/client.py

@@ -1,116 +0,0 @@
-import subprocess
-import logging
-import urllib.request
-import shutil
-import sys
-import requests
-import tempfile
-import kraken.prometheus.client as prometheus
-from urllib.parse import urlparse
-
-
-def setup(url):
-    """
-    Downloads and unpacks kube-burner binary
-    """
-
-    filename = "kube_burner.tar"
-    try:
-        logging.info("Fetching kube-burner binary")
-        urllib.request.urlretrieve(url, filename)
-    except Exception as e:
-        logging.error("Failed to download kube-burner binary located at %s" % url, e)
-        sys.exit(1)
-    try:
-        logging.info("Unpacking kube-burner tar ball")
-        shutil.unpack_archive(filename)
-    except Exception as e:
-        logging.error("Failed to unpack the kube-burner binary tarball: %s" % e)
-        sys.exit(1)
-
-
-def scrape_metrics(
-    distribution, uuid, prometheus_url, prometheus_bearer_token, start_time, end_time, config_path, metrics_profile
-):
-    """
-    Scrapes metrics defined in the profile from Prometheus and indexes them into Elasticsearch
-    """
-
-    if not prometheus_url:
-        if distribution == "openshift":
-            logging.info("Looks like prometheus_url is not defined, trying to use the default instance on the cluster")
-            prometheus_url, prometheus_bearer_token = prometheus.instance(
-                distribution, prometheus_url, prometheus_bearer_token
-            )
-        else:
-            logging.error("Looks like prometheus url is not defined, exiting")
-            sys.exit(1)
-    command = (
-        "./kube-burner index --uuid "
-        + str(uuid)
-        + " -u "
-        + str(prometheus_url)
-        + " -t "
-        + str(prometheus_bearer_token)
-        + " -m "
-        + str(metrics_profile)
-        + " --start "
-        + str(start_time)
-        + " --end "
-        + str(end_time)
-        + " -c "
-        + str(config_path)
-    )
-    try:
-        logging.info("Running kube-burner to capture the metrics: %s" % command)
-        logging.info("UUID for the run: %s" % uuid)
-        subprocess.run(command, shell=True, universal_newlines=True)
-    except Exception as e:
-        logging.error("Failed to run kube-burner, error: %s" % (e))
-        sys.exit(1)
-
-
-def alerts(distribution, prometheus_url, prometheus_bearer_token, start_time, end_time, alert_profile):
-    """
-    Scrapes metrics defined in the profile from Prometheus and alerts based on the severity defined
-    """
-
-    is_url = urlparse(alert_profile)
-    if is_url.scheme and is_url.netloc:
-        response = requests.get(alert_profile)
-        temp_alerts = tempfile.NamedTemporaryFile()
-        temp_alerts.write(response.content)
-        temp_alerts.flush()
-        alert_profile = temp_alerts.name
-
-    if not prometheus_url:
-        if distribution == "openshift":
-            logging.info("Looks like prometheus_url is not defined, trying to use the default instance on the cluster")
-            prometheus_url, prometheus_bearer_token = prometheus.instance(
-                distribution, prometheus_url, prometheus_bearer_token
-            )
-        else:
-            logging.error("Looks like prometheus url is not defined, exiting")
-            sys.exit(1)
-    command = (
-        "./kube-burner check-alerts "
-        + " -u "
-        + str(prometheus_url)
-        + " -t "
-        + str(prometheus_bearer_token)
-        + " -a "
-        + str(alert_profile)
-        + " --start "
-        + str(start_time)
-        + " --end "
-        + str(end_time)
-    )
-    try:
-        logging.info("Running kube-burner to capture the metrics: %s" % command)
-        output = subprocess.run(command, shell=True, universal_newlines=True)
-        if output.returncode != 0:
-            logging.error("command exited with a non-zero rc, please check the logs for errors or critical alerts")
-            sys.exit(output.returncode)
-    except Exception as e:
-        logging.error("Failed to run kube-burner, error: %s" % (e))
-        sys.exit(1)

kraken/kubernetes_legacy/client.py

@@ -1,892 +0,0 @@
-import logging
-import re
-import sys
-import time
-
-from kubernetes import client, config, utils, watch
-from kubernetes.client.rest import ApiException
-from kubernetes.dynamic.client import DynamicClient
-from kubernetes.stream import stream
-
-from ..kubernetes.resources import (PVC, ChaosEngine, ChaosResult, Container,
-                                    LitmusChaosObject, Pod, Volume,
-                                    VolumeMount)
-
-kraken_node_name = ""
-
-
-# Load kubeconfig and initialize kubernetes python client
-def initialize_clients(kubeconfig_path):
-    global cli
-    global batch_cli
-    global watch_resource
-    global api_client
-    global dyn_client
-    global custom_object_client
-    try:
-        if kubeconfig_path:
-            config.load_kube_config(kubeconfig_path)
-        else:
-            config.load_incluster_config()
-        api_client = client.ApiClient()
-        cli = client.CoreV1Api(api_client)
-        batch_cli = client.BatchV1Api(api_client)
-        custom_object_client = client.CustomObjectsApi(api_client)
-        dyn_client = DynamicClient(api_client)
-        watch_resource = watch.Watch()
-    except ApiException as e:
-        logging.error("Failed to initialize kubernetes client: %s\n" % e)
-        sys.exit(1)
-
-
-def get_host() -> str:
-    """Returns the Kubernetes server URL"""
-    return client.configuration.Configuration.get_default_copy().host
-
-
-def get_clusterversion_string() -> str:
-    """
-    Returns clusterversion status text on OpenShift, empty string
-    on other distributions
-    """
-    try:
-        cvs = custom_object_client.list_cluster_custom_object(
-            "config.openshift.io",
-            "v1",
-            "clusterversions",
-        )
-        for cv in cvs["items"]:
-            for condition in cv["status"]["conditions"]:
-                if condition["type"] == "Progressing":
-                    return condition["message"]
-        return ""
-    except client.exceptions.ApiException as e:
-        if e.status == 404:
-            return ""
-        else:
-            raise
-
-
-# List all namespaces
-def list_namespaces(label_selector=None):
-    namespaces = []
-    try:
-        if label_selector:
-            ret = cli.list_namespace(
-                pretty=True,
-                label_selector=label_selector
-                )
-        else:
-            ret = cli.list_namespace(pretty=True)
-    except ApiException as e:
-        logging.error(
-            "Exception when calling CoreV1Api->list_namespaced_pod: %s\n" % e
-        )
-        raise e
-    for namespace in ret.items:
-        namespaces.append(namespace.metadata.name)
-    return namespaces
-
-
-def get_namespace_status(namespace_name):
-    """Get status of a given namespace"""
-    ret = ""
-    try:
-        ret = cli.read_namespace_status(namespace_name)
-    except ApiException as e:
-        logging.error(
-            "Exception when calling CoreV1Api->read_namespace_status: %s\n" % e
-        )
-    return ret.status.phase
-
-
-def delete_namespace(namespace):
-    """Deletes a given namespace using kubernetes python client"""
-    try:
-        api_response = cli.delete_namespace(namespace)
-        logging.debug(
-            "Namespace deleted. status='%s'" % str(api_response.status)
-        )
-        return api_response
-    except Exception as e:
-        logging.error(
-            "Exception when calling \
-                       CoreV1Api->delete_namespace: %s\n"
-            % e
-        )
-
-
-def check_namespaces(namespaces, label_selectors=None):
-    """Check if all the watch_namespaces are valid"""
-    try:
-        valid_namespaces = list_namespaces(label_selectors)
-        regex_namespaces = set(namespaces) - set(valid_namespaces)
-        final_namespaces = set(namespaces) - set(regex_namespaces)
-        valid_regex = set()
-        if regex_namespaces:
-            for namespace in valid_namespaces:
-                for regex_namespace in regex_namespaces:
-                    if re.search(regex_namespace, namespace):
-                        final_namespaces.add(namespace)
-                        valid_regex.add(regex_namespace)
-                        break
-        invalid_namespaces = regex_namespaces - valid_regex
-        if invalid_namespaces:
-            raise Exception(
-                "There exists no namespaces matching: %s" %
-                (invalid_namespaces)
-            )
-        return list(final_namespaces)
-    except Exception as e:
-        logging.info("%s" % (e))
-        sys.exit(1)
-
-
-# List nodes in the cluster
-def list_nodes(label_selector=None):
-    nodes = []
-    try:
-        if label_selector:
-            ret = cli.list_node(pretty=True, label_selector=label_selector)
-        else:
-            ret = cli.list_node(pretty=True)
-    except ApiException as e:
-        logging.error("Exception when calling CoreV1Api->list_node: %s\n" % e)
-        raise e
-    for node in ret.items:
-        nodes.append(node.metadata.name)
-    return nodes
-
-
-# List nodes in the cluster that can be killed
-def list_killable_nodes(label_selector=None):
-    nodes = []
-    try:
-        if label_selector:
-            ret = cli.list_node(pretty=True, label_selector=label_selector)
-        else:
-            ret = cli.list_node(pretty=True)
-    except ApiException as e:
-        logging.error("Exception when calling CoreV1Api->list_node: %s\n" % e)
-        raise e
-    for node in ret.items:
-        if kraken_node_name != node.metadata.name:
-            for cond in node.status.conditions:
-                if str(cond.type) == "Ready" and str(cond.status) == "True":
-                    nodes.append(node.metadata.name)
-    return nodes
-
-
-# List managedclusters attached to the hub that can be killed
-def list_killable_managedclusters(label_selector=None):
-    managedclusters = []
-    try:
-        ret = custom_object_client.list_cluster_custom_object(
-            group="cluster.open-cluster-management.io",
-            version="v1",
-            plural="managedclusters",
-            label_selector=label_selector
-        )
-    except ApiException as e:
-        logging.error("Exception when calling CustomObjectsApi->list_cluster_custom_object: %s\n" % e)
-        raise e
-    for managedcluster in ret['items']:
-        conditions = managedcluster['status']['conditions']
-        available = list(filter(lambda condition: condition['reason'] == 'ManagedClusterAvailable', conditions))
-        if available and available[0]['status'] == 'True':
-            managedclusters.append(managedcluster['metadata']['name'])
-    return managedclusters
-
-# List pods in the given namespace
-def list_pods(namespace, label_selector=None):
-    pods = []
-    try:
-        if label_selector:
-            ret = cli.list_namespaced_pod(
-                namespace,
-                pretty=True,
-                label_selector=label_selector
-            )
-        else:
-            ret = cli.list_namespaced_pod(namespace, pretty=True)
-    except ApiException as e:
-        logging.error(
-            "Exception when calling \
-                       CoreV1Api->list_namespaced_pod: %s\n"
-            % e
-        )
-        raise e
-    for pod in ret.items:
-        pods.append(pod.metadata.name)
-    return pods
-
-
-def get_all_pods(label_selector=None):
-    pods = []
-    if label_selector:
-        ret = cli.list_pod_for_all_namespaces(
-            pretty=True,
-            label_selector=label_selector
-        )
-    else:
-        ret = cli.list_pod_for_all_namespaces(pretty=True)
-    for pod in ret.items:
-        pods.append([pod.metadata.name, pod.metadata.namespace])
-    return pods
-
-
-# Execute command in pod
-def exec_cmd_in_pod(
-    command,
-    pod_name,
-    namespace,
-    container=None,
-    base_command="bash"
-):
-
-    exec_command = [base_command, "-c", command]
-    try:
-        if container:
-            ret = stream(
-                cli.connect_get_namespaced_pod_exec,
-                pod_name,
-                namespace,
-                container=container,
-                command=exec_command,
-                stderr=True,
-                stdin=False,
-                stdout=True,
-                tty=False,
-            )
-        else:
-            ret = stream(
-                cli.connect_get_namespaced_pod_exec,
-                pod_name,
-                namespace,
-                command=exec_command,
-                stderr=True,
-                stdin=False,
-                stdout=True,
-                tty=False,
-            )
-    except Exception:
-        return False
-    return ret
-
-
-def delete_pod(name, namespace):
-    try:
-        cli.delete_namespaced_pod(name=name, namespace=namespace)
-        while cli.read_namespaced_pod(name=name, namespace=namespace):
-            time.sleep(1)
-    except ApiException as e:
-        if e.status == 404:
-            logging.info("Pod already deleted")
-        else:
-            logging.error("Failed to delete pod %s" % e)
-            raise e
-
-
-def create_pod(body, namespace, timeout=120):
-    try:
-        pod_stat = None
-        pod_stat = cli.create_namespaced_pod(body=body, namespace=namespace)
-        end_time = time.time() + timeout
-        while True:
-            pod_stat = cli.read_namespaced_pod(
-                name=body["metadata"]["name"],
-                namespace=namespace
-            )
-            if pod_stat.status.phase == "Running":
-                break
-            if time.time() > end_time:
-                raise Exception("Starting pod failed")
-            time.sleep(1)
-    except Exception as e:
-        logging.error("Pod creation failed %s" % e)
-        if pod_stat:
-            logging.error(pod_stat.status.container_statuses)
-        delete_pod(body["metadata"]["name"], namespace)
-        sys.exit(1)
-
-
-def read_pod(name, namespace="default"):
-    return cli.read_namespaced_pod(name=name, namespace=namespace)
-
-
-def get_pod_log(name, namespace="default"):
-    return cli.read_namespaced_pod_log(
-        name=name,
-        namespace=namespace,
-        _return_http_data_only=True,
-        _preload_content=False
-    )
-
-
-def get_containers_in_pod(pod_name, namespace):
-    pod_info = cli.read_namespaced_pod(pod_name, namespace)
-    container_names = []
-
-    for cont in pod_info.spec.containers:
-        container_names.append(cont.name)
-    return container_names
-
-
-def delete_job(name, namespace="default"):
-    try:
-        api_response = batch_cli.delete_namespaced_job(
-            name=name,
-            namespace=namespace,
-            body=client.V1DeleteOptions(
-                propagation_policy="Foreground",
-                grace_period_seconds=0
-            ),
-        )
-        logging.debug("Job deleted. status='%s'" % str(api_response.status))
-        return api_response
-    except ApiException as api:
-        logging.warn(
-            "Exception when calling \
-                       BatchV1Api->create_namespaced_job: %s"
-            % api
-        )
-        logging.warn("Job already deleted\n")
-    except Exception as e:
-        logging.error(
-            "Exception when calling \
-                       BatchV1Api->delete_namespaced_job: %s\n"
-            % e
-        )
-        sys.exit(1)
-
-
-def create_job(body, namespace="default"):
-    try:
-        api_response = batch_cli.create_namespaced_job(
-            body=body,
-            namespace=namespace
-        )
-        return api_response
-    except ApiException as api:
-        logging.warn(
-            "Exception when calling \
-                       BatchV1Api->create_job: %s"
-            % api
-        )
-        if api.status == 409:
-            logging.warn("Job already present")
-    except Exception as e:
-        logging.error(
-            "Exception when calling \
-                       BatchV1Api->create_namespaced_job: %s"
-            % e
-        )
-        raise
-
-
-def create_manifestwork(body, namespace):
-    try:
-        api_response = custom_object_client.create_namespaced_custom_object(
-            group="work.open-cluster-management.io", 
-            version="v1",
-            plural="manifestworks",
-            body=body,
-            namespace=namespace
-        )
-        return api_response
-    except ApiException as e:
-        print("Exception when calling CustomObjectsApi->create_namespaced_custom_object: %s\n" % e)
-
-
-def delete_manifestwork(namespace):
-    try:
-        api_response = custom_object_client.delete_namespaced_custom_object(
-            group="work.open-cluster-management.io", 
-            version="v1",
-            plural="manifestworks",
-            name="managedcluster-scenarios-template",
-            namespace=namespace
-        )
-        return api_response
-    except ApiException as e:
-        print("Exception when calling CustomObjectsApi->delete_namespaced_custom_object: %s\n" % e)
-
-def get_job_status(name, namespace="default"):
-    try:
-        return batch_cli.read_namespaced_job_status(
-            name=name,
-            namespace=namespace
-        )
-    except Exception as e:
-        logging.error(
-            "Exception when calling \
-                       BatchV1Api->read_namespaced_job_status: %s"
-            % e
-        )
-        raise
-
-
-# Monitor the status of the cluster nodes and set the status to true or false
-def monitor_nodes():
-    nodes = list_nodes()
-    notready_nodes = []
-    node_kerneldeadlock_status = "False"
-    for node in nodes:
-        try:
-            node_info = cli.read_node_status(node, pretty=True)
-        except ApiException as e:
-            logging.error(
-                "Exception when calling \
-                           CoreV1Api->read_node_status: %s\n"
-                % e
-            )
-            raise e
-        for condition in node_info.status.conditions:
-            if condition.type == "KernelDeadlock":
-                node_kerneldeadlock_status = condition.status
-            elif condition.type == "Ready":
-                node_ready_status = condition.status
-            else:
-                continue
-        if node_kerneldeadlock_status != "False" or node_ready_status != "True":  # noqa  # noqa
-            notready_nodes.append(node)
-    if len(notready_nodes) != 0:
-        status = False
-    else:
-        status = True
-    return status, notready_nodes
-
-
-# Monitor the status of the pods in the specified namespace
-# and set the status to true or false
-def monitor_namespace(namespace):
-    pods = list_pods(namespace)
-    notready_pods = []
-    for pod in pods:
-        try:
-            pod_info = cli.read_namespaced_pod_status(
-                pod,
-                namespace,
-                pretty=True
-            )
-        except ApiException as e:
-            logging.error(
-                "Exception when calling \
-                           CoreV1Api->read_namespaced_pod_status: %s\n"
-                % e
-            )
-            raise e
-        pod_status = pod_info.status.phase
-        if (
-            pod_status != "Running" and
-            pod_status != "Completed" and
-            pod_status != "Succeeded"
-        ):
-            notready_pods.append(pod)
-    if len(notready_pods) != 0:
-        status = False
-    else:
-        status = True
-    return status, notready_pods
-
-
-# Monitor component namespace
-def monitor_component(iteration, component_namespace):
-    watch_component_status, failed_component_pods = \
-        monitor_namespace(component_namespace)
-    logging.info(
-        "Iteration %s: %s: %s" % (
-            iteration,
-            component_namespace,
-            watch_component_status
-        )
-    )
-    return watch_component_status, failed_component_pods
-
-
-def apply_yaml(path, namespace='default'):
-    """
-    Apply yaml config to create Kubernetes resources
-
-    Args:
-        path (string)
-            - Path to the YAML file
-        namespace (string)
-            - Namespace to create the resource
-
-    Returns:
-        The object created
-    """
-
-    return utils.create_from_yaml(
-        api_client,
-        yaml_file=path,
-        namespace=namespace
-    )
-
-
-def get_pod_info(name: str, namespace: str = 'default') -> Pod:
-    """
-    Function to retrieve information about a specific pod
-    in a given namespace. The kubectl command is given by:
-        kubectl get pods <name> -n <namespace>
-
-    Args:
-        name (string)
-            - Name of the pod
-
-        namespace (string)
-            - Namespace to look for the pod
-
-    Returns:
-        - Data class object of type Pod with the output of the above
-          kubectl command in the given format if the pod exists
-        - Returns None if the pod doesn't exist
-    """
-    pod_exists = check_if_pod_exists(name=name, namespace=namespace)
-    if pod_exists:
-        response = cli.read_namespaced_pod(
-            name=name,
-            namespace=namespace,
-            pretty='true'
-        )
-        container_list = []
-
-        # Create a list of containers present in the pod
-        for container in response.spec.containers:
-            volume_mount_list = []
-            for volume_mount in container.volume_mounts:
-                volume_mount_list.append(
-                    VolumeMount(
-                        name=volume_mount.name,
-                        mountPath=volume_mount.mount_path
-                    )
-                )
-            container_list.append(
-                Container(
-                    name=container.name,
-                    image=container.image,
-                    volumeMounts=volume_mount_list
-                )
-            )
-
-        for i, container in enumerate(response.status.container_statuses):
-            container_list[i].ready = container.ready
-
-        # Create a list of volumes associated with the pod
-        volume_list = []
-        for volume in response.spec.volumes:
-            volume_name = volume.name
-            pvc_name = (
-                volume.persistent_volume_claim.claim_name
-                if volume.persistent_volume_claim is not None
-                else None
-            )
-            volume_list.append(Volume(name=volume_name, pvcName=pvc_name))
-
-        # Create the Pod data class object
-        pod_info = Pod(
-            name=response.metadata.name,
-            podIP=response.status.pod_ip,
-            namespace=response.metadata.namespace,
-            containers=container_list,
-            nodeName=response.spec.node_name,
-            volumes=volume_list
-        )
-        return pod_info
-    else:
-        logging.error(
-                "Pod '%s' doesn't exist in namespace '%s'" % (
-                    str(name),
-                    str(namespace)
-                )
-        )
-        return None
-
-
-def get_litmus_chaos_object(
-        kind: str,
-        name: str,
-        namespace: str
-) -> LitmusChaosObject:
-    """
-    Function that returns an object of a custom resource type of
-    the litmus project. Currently, only ChaosEngine and ChaosResult
-    objects are supported.
-
-    Args:
-        kind (string)
-            - The custom resource type
-
-        namespace (string)
-            - Namespace where the custom object is present
-
-    Returns:
-        Data class object of a subclass of LitmusChaosObject
-    """
-
-    group = 'litmuschaos.io'
-    version = 'v1alpha1'
-
-    if kind.lower() == 'chaosengine':
-        plural = 'chaosengines'
-        response = custom_object_client.get_namespaced_custom_object(
-            group=group,
-            plural=plural,
-            version=version,
-            namespace=namespace,
-            name=name
-        )
-        try:
-            engine_status = response['status']['engineStatus']
-            exp_status = response['status']['experiments'][0]['status']
-        except Exception:
-            engine_status = 'Not Initialized'
-            exp_status = 'Not Initialized'
-        custom_object = ChaosEngine(
-            kind='ChaosEngine',
-            group=group,
-            namespace=namespace,
-            name=name,
-            plural=plural,
-            version=version,
-            engineStatus=engine_status,
-            expStatus=exp_status
-        )
-    elif kind.lower() == 'chaosresult':
-        plural = 'chaosresults'
-        response = custom_object_client.get_namespaced_custom_object(
-            group=group,
-            plural=plural,
-            version=version,
-            namespace=namespace,
-            name=name
-        )
-        try:
-            verdict = response['status']['experimentStatus']['verdict']
-            fail_step = response['status']['experimentStatus']['failStep']
-        except Exception:
-            verdict = 'N/A'
-            fail_step = 'N/A'
-        custom_object = ChaosResult(
-            kind='ChaosResult',
-            group=group,
-            namespace=namespace,
-            name=name,
-            plural=plural,
-            version=version,
-            verdict=verdict,
-            failStep=fail_step
-        )
-    else:
-        logging.error("Invalid litmus chaos custom resource name")
-        custom_object = None
-    return custom_object
-
-
-def check_if_namespace_exists(name: str) -> bool:
-    """
-    Function that checks if a namespace exists by parsing through
-    the list of projects.
-    Args:
-        name (string)
-            - Namespace name
-
-    Returns:
-        Boolean value indicating whether the namespace exists or not
-    """
-
-    v1_projects = dyn_client.resources.get(
-        api_version='project.openshift.io/v1',
-        kind='Project'
-    )
-    project_list = v1_projects.get()
-    return True if name in str(project_list) else False
-
-
-def check_if_pod_exists(name: str, namespace: str) -> bool:
-    """
-    Function that checks if a pod exists in the given namespace
-    Args:
-        name (string)
-            - Pod name
-
-        namespace (string)
-            - Namespace name
-
-    Returns:
-        Boolean value indicating whether the pod exists or not
-    """
-
-    namespace_exists = check_if_namespace_exists(namespace)
-    if namespace_exists:
-        pod_list = list_pods(namespace=namespace)
-        if name in pod_list:
-            return True
-    else:
-        logging.error("Namespace '%s' doesn't exist" % str(namespace))
-    return False
-
-
-def check_if_pvc_exists(name: str, namespace: str) -> bool:
-    """
-    Function that checks if a namespace exists by parsing through
-    the list of projects.
-    Args:
-        name (string)
-            - PVC name
-
-        namespace (string)
-            - Namespace name
-
-    Returns:
-        Boolean value indicating whether the Persistent Volume Claim
-        exists or not.
-    """
-    namespace_exists = check_if_namespace_exists(namespace)
-    if namespace_exists:
-        response = cli.list_namespaced_persistent_volume_claim(
-            namespace=namespace
-        )
-        pvc_list = [pvc.metadata.name for pvc in response.items]
-        if name in pvc_list:
-            return True
-    else:
-        logging.error("Namespace '%s' doesn't exist" % str(namespace))
-    return False
-
-
-def get_pvc_info(name: str, namespace: str) -> PVC:
-    """
-    Function to retrieve information about a Persistent Volume Claim in a
-    given namespace
-
-    Args:
-        name (string)
-            - Name of the persistent volume claim
-
-        namespace (string)
-            - Namespace where the persistent volume claim is present
-
-    Returns:
-        - A PVC data class containing the name, capacity, volume name,
-          namespace and associated pod names of the PVC if the PVC exists
-        - Returns None if the PVC doesn't exist
-    """
-
-    pvc_exists = check_if_pvc_exists(name=name, namespace=namespace)
-    if pvc_exists:
-        pvc_info_response = cli.read_namespaced_persistent_volume_claim(
-            name=name,
-            namespace=namespace,
-            pretty=True
-        )
-        pod_list_response = cli.list_namespaced_pod(namespace=namespace)
-
-        capacity = pvc_info_response.status.capacity['storage']
-        volume_name = pvc_info_response.spec.volume_name
-
-        # Loop through all pods in the namespace to find associated PVCs
-        pvc_pod_list = []
-        for pod in pod_list_response.items:
-            for volume in pod.spec.volumes:
-                if (
-                    volume.persistent_volume_claim is not None
-                    and volume.persistent_volume_claim.claim_name == name
-                ):
-                    pvc_pod_list.append(pod.metadata.name)
-
-        pvc_info = PVC(
-            name=name,
-            capacity=capacity,
-            volumeName=volume_name,
-            podNames=pvc_pod_list,
-            namespace=namespace
-        )
-        return pvc_info
-    else:
-        logging.error(
-            "PVC '%s' doesn't exist in namespace '%s'" % (
-                str(name),
-                str(namespace)
-            )
-        )
-        return None
-
-
-# Find the node kraken is deployed on
-# Set global kraken node to not delete
-def find_kraken_node():
-    pods = get_all_pods()
-    kraken_pod_name = None
-    for pod in pods:
-        if "kraken-deployment" in pod[0]:
-            kraken_pod_name = pod[0]
-            kraken_project = pod[1]
-            break
-    # have to switch to proper project
-
-    if kraken_pod_name:
-        # get kraken-deployment pod, find node name
-        try:
-            node_name = get_pod_info(kraken_pod_name, kraken_project).nodeName
-            global kraken_node_name
-            kraken_node_name = node_name
-        except Exception as e:
-            logging.info("%s" % (e))
-            sys.exit(1)
-
-
-# Watch for a specific node status
-def watch_node_status(node, status, timeout, resource_version):
-    count = timeout
-    for event in watch_resource.stream(
-        cli.list_node,
-        field_selector=f"metadata.name={node}",
-        timeout_seconds=timeout,
-        resource_version=f"{resource_version}"
-    ):
-        conditions = [
-            status
-            for status in event["object"].status.conditions
-            if status.type == "Ready"
-        ]
-        if conditions[0].status == status:
-            watch_resource.stop()
-            break
-        else:
-            count -= 1
-            logging.info(
-                "Status of node " + node + ": " + str(conditions[0].status)
-            )
-        if not count:
-            watch_resource.stop()
-
-
-# Watch for a specific managedcluster status
-# TODO: Implement this with a watcher instead of polling
-def watch_managedcluster_status(managedcluster, status, timeout):
-    elapsed_time = 0
-    while True:
-        conditions = custom_object_client.get_cluster_custom_object_status(
-            "cluster.open-cluster-management.io", "v1", "managedclusters", managedcluster
-        )['status']['conditions']
-        available = list(filter(lambda condition: condition['reason'] == 'ManagedClusterAvailable', conditions))
-        if status == "True":
-            if available and available[0]['status'] == "True":
-                logging.info("Status of managedcluster " + managedcluster + ": Available")
-                return True
-        else:
-            if not available:
-                logging.info("Status of managedcluster " + managedcluster + ": Unavailable")
-                return True
-        time.sleep(2)
-        elapsed_time += 2
-        if elapsed_time >= timeout:
-            logging.info("Timeout waiting for managedcluster " + managedcluster + " to become: " + status)
-            return False
-
-
-# Get the resource version for the specified node
-def get_node_resource_version(node):
-    return cli.read_node(name=node).metadata.resource_version