updated krkn-lib to 2.1.0 (#588)

Signed-off-by: Tullio Sebastiani <tsebasti@redhat.com>

.github/workflows/build.yml

@@ -1,51 +0,0 @@
-name: Build Krkn
-on:
-  pull_request:
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out code
-        uses: actions/checkout@v3
-      - name: Create multi-node KinD cluster
-        uses: redhat-chaos/actions/kind@main
-      - name: Install Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.9'
-          architecture: 'x64'
-      - name: Install environment
-        run: |
-          sudo apt-get install build-essential python3-dev
-          pip install --upgrade pip
-          pip install -r requirements.txt
-      - name: Run unit tests
-        run: python -m coverage run -a -m unittest discover -s tests -v
-      - name: Run CI
-        run: |
-          ./CI/run.sh
-          cat ./CI/results.markdown >> $GITHUB_STEP_SUMMARY
-          echo >> $GITHUB_STEP_SUMMARY
-      - name: Upload CI logs
-        uses: actions/upload-artifact@v3
-        with:
-          name: ci-logs
-          path: CI/out
-          if-no-files-found: error
-      - name: Collect coverage report
-        run: |
-          python -m coverage html
-      - name: Publish coverage report to job summary
-        run: |
-          pip install html2text
-          html2text --ignore-images --ignore-links -b 0 htmlcov/index.html >> $GITHUB_STEP_SUMMARY
-      - name: Upload coverage data
-        uses: actions/upload-artifact@v3
-        with:
-          name: coverage
-          path: htmlcov
-          if-no-files-found: error
-      - name: Check CI results
-        run: grep Fail CI/results.markdown && false || true
-      

.github/workflows/docker-image.yml

@@ -12,19 +12,30 @@ jobs:
     - name: Check out code
       uses: actions/checkout@v3
     - name: Build the Docker images
-      run: docker build --no-cache -t quay.io/redhat-chaos/krkn containers/
+      run:  |
+        docker build --no-cache -t quay.io/krkn-chaos/krkn containers/
+        docker tag quay.io/krkn-chaos/krkn quay.io/redhat-chaos/krkn
     - name: Login in quay
+      if: github.ref == 'refs/heads/main' && github.event_name == 'push'
+      run: docker login quay.io -u ${QUAY_USER} -p ${QUAY_TOKEN}
+      env:
+        QUAY_USER: ${{ secrets.QUAY_USERNAME }}
+        QUAY_TOKEN: ${{ secrets.QUAY_PASSWORD }}
+    - name: Push the KrknChaos Docker images
+      if: github.ref == 'refs/heads/main' && github.event_name == 'push'
+      run: docker push quay.io/krkn-chaos/krkn
+    - name: Login in to redhat-chaos quay
       if: github.ref == 'refs/heads/main' && github.event_name == 'push'
       run: docker login quay.io -u ${QUAY_USER} -p ${QUAY_TOKEN}
       env:
         QUAY_USER: ${{ secrets.QUAY_USER_1 }}
         QUAY_TOKEN: ${{ secrets.QUAY_TOKEN_1 }}
-    - name: Push the Docker images
+    - name: Push the RedHat Chaos Docker images
       if: github.ref == 'refs/heads/main' && github.event_name == 'push'
       run: docker push quay.io/redhat-chaos/krkn
     - name: Rebuild krkn-hub
       if: github.ref == 'refs/heads/main' && github.event_name == 'push'
       uses: redhat-chaos/actions/krkn-hub@main
       with:
-        QUAY_USER: ${{ secrets.QUAY_USER_1 }}
-        QUAY_TOKEN: ${{ secrets.QUAY_TOKEN_1 }}
+        QUAY_USER: ${{ secrets.QUAY_USERNAME }}
+        QUAY_TOKEN: ${{ secrets.QUAY_PASSWORD }}

.github/workflows/functional_tests.yaml

@@ -1,129 +0,0 @@
-on: issue_comment
-
-jobs:
-  check_user:
-    # This job only runs for pull request comments
-    name: Check User Authorization
-    env:
-      USERS: ${{vars.USERS}}
-    if: contains(github.event.comment.body, '/funtest') && contains(github.event.comment.html_url, '/pull/')
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check User
-        run: |
-          for name in `echo $USERS`
-          do 
-            name="${name//$'\r'/}"
-            name="${name//$'\n'/}"
-            if [ $name == "${{github.event.sender.login}}" ]
-            then
-              echo "user ${{github.event.sender.login}} authorized, action started..."
-              exit 0
-            fi
-          done
-          echo "user ${{github.event.sender.login}} is not allowed to run functional tests Action"
-          exit 1
-  pr_commented:
-    # This job only runs for pull request comments containing /functional
-    name: Functional Tests
-    if: contains(github.event.comment.body, '/funtest') && contains(github.event.comment.html_url, '/pull/')
-    runs-on: ubuntu-latest
-    needs:
-      - check_user
-    steps:
-      - name: Check out Kraken
-        uses: actions/checkout@v3
-      - name: Checkout Pull Request
-        run: gh pr checkout ${{ github.event.issue.number }}
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - name: Install OC CLI
-        uses: redhat-actions/oc-installer@v1
-        with:
-          oc_version: latest
-      - name: Install python 3.9
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.9'
-      - name: Setup kraken dependencies
-        run:  pip install -r requirements.txt
-      - name: Create Workdir & export the path
-        run: |
-         mkdir workdir
-         echo "WORKDIR_PATH=`pwd`/workdir" >> $GITHUB_ENV
-      - name: Generate run id
-        run: |
-          echo "RUN_ID=`date +%s`" > $GITHUB_ENV
-          echo "Run Id: ${RUN_ID}"
-      - name: Write Pull Secret
-        env:
-          PULLSECRET_BASE64: ${{ secrets.PS_64 }}
-        run: |
-          echo "$PULLSECRET_BASE64" | base64 --decode > pullsecret.txt
-      - name: Write Boot Private Key
-        env:
-          BOOT_KEY: ${{ secrets.CRC_KEY_FILE }}
-        run: |
-          echo -n "$BOOT_KEY" > key.txt
-      - name: Teardown CRC (Post Action)
-        uses: webiny/action-post-run@3.0.0
-        id: post-run-command
-        with:
-          run: podman run --rm -v "${{ github.workspace }}:/workspace:z" -e AWS_ACCESS_KEY_ID="${{ secrets.AWS_ACCESS_KEY_ID }}" -e AWS_SECRET_ACCESS_KEY="${{ secrets.AWS_SECRET_ACCESS_KEY }}" -e AWS_DEFAULT_REGION=us-west-2 quay.io/crcont/crc-cloud:v0.0.2 destroy --project-name "chaos-funtest-${{ env.RUN_ID }}" --backed-url "s3://krkn-crc-state/${{ env.RUN_ID }}" --provider "aws"
-      - name: Create cluster
-        run: |
-          podman run  --name crc-cloud-create --rm \
-          -v ${PWD}:/workspace:z \
-          -e AWS_ACCESS_KEY_ID="${{ secrets.AWS_ACCESS_KEY_ID }}" \
-          -e AWS_SECRET_ACCESS_KEY="${{ secrets.AWS_SECRET_ACCESS_KEY }}" \
-          -e AWS_DEFAULT_REGION="us-west-2" \
-          quay.io/crcont/crc-cloud:v0.0.2 \
-          create aws \
-          --project-name "chaos-funtest-${RUN_ID}" \
-          --backed-url "s3://krkn-crc-state/${RUN_ID}" \
-          --output "/workspace" \
-          --aws-ami-id "ami-00f5eaf98cf42ef9f" \
-          --pullsecret-filepath /workspace/pullsecret.txt \
-          --key-filepath /workspace/key.txt
-
-      - name: Setup kubeconfig
-        continue-on-error: true
-        run: |
-          ssh  -o StrictHostKeyChecking=no -i id_rsa core@$(cat host) "cat /opt/kubeconfig" > kubeconfig
-          sed -i "s/https:\/\/api.crc.testing:6443/https:\/\/`cat host`.nip.io:6443/g" kubeconfig
-          echo "KUBECONFIG=${PWD}/kubeconfig" > $GITHUB_ENV
-
-      - name: Example deployment, GitHub Action env init
-        env:
-          NAMESPACE: test-namespace
-          DEPLOYMENT_NAME: test-nginx
-        run: ./CI/CRC/init_github_action.sh
-      - name: Setup test suite
-        run: |
-          yq -i '.kraken.port="8081"' CI/config/common_test_config.yaml
-          yq -i '.kraken.signal_address="0.0.0.0"' CI/config/common_test_config.yaml
-          yq -i '.kraken.kubeconfig_path="'${KUBECONFIG}'"' CI/config/common_test_config.yaml
-          echo "test_app_outages_gh" > ./CI/tests/my_tests
-          echo "test_container"      >> ./CI/tests/my_tests
-          echo "test_namespace"      >> ./CI/tests/my_tests
-          echo "test_net_chaos"      >> ./CI/tests/my_tests
-          echo "test_time"           >> ./CI/tests/my_tests
-
-      - name: Print affected config files
-        run: |
-          echo -e "## CI/config/common_test_config.yaml\n\n"
-          cat CI/config/common_test_config.yaml
-
-      - name: Running test suite
-        run: |
-          ./CI/run.sh
-      - name: Print test output
-        run:  cat CI/out/*
-      - name: Create coverage report
-        run: |
-          echo "# Test results" > $GITHUB_STEP_SUMMARY
-          cat CI/results.markdown >> $GITHUB_STEP_SUMMARY
-          echo "# Test coverage" >> $GITHUB_STEP_SUMMARY
-          python -m coverage report --format=markdown >> $GITHUB_STEP_SUMMARY
-
-

.github/workflows/tests.yml

@@ -0,0 +1,134 @@
+name: Functional & Unit Tests
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+jobs:
+  tests:
+    # Common steps
+    name: Functional & Unit Tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v3
+      - name: Create multi-node KinD cluster
+        uses: redhat-chaos/actions/kind@main
+      - name: Install Helm & add repos
+        run: |
+          curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
+          helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
+          helm repo add stable https://charts.helm.sh/stable
+          helm repo update
+      - name: Deploy prometheus & Port Forwarding
+        run: |
+          kubectl create namespace prometheus-k8s
+          helm install \
+          --wait --timeout 360s \
+          kind-prometheus \
+          prometheus-community/kube-prometheus-stack \
+          --namespace prometheus-k8s \
+          --set prometheus.service.nodePort=30000 \
+          --set prometheus.service.type=NodePort \
+          --set grafana.service.nodePort=31000 \
+          --set grafana.service.type=NodePort \
+          --set alertmanager.service.nodePort=32000 \
+          --set alertmanager.service.type=NodePort \
+          --set prometheus-node-exporter.service.nodePort=32001 \
+          --set prometheus-node-exporter.service.type=NodePort
+
+          SELECTOR=`kubectl -n prometheus-k8s get service kind-prometheus-kube-prome-prometheus -o wide --no-headers=true | awk '{ print $7 }'`
+          POD_NAME=`kubectl -n prometheus-k8s get pods --selector="$SELECTOR" --no-headers=true | awk '{ print $1 }'`
+          kubectl -n prometheus-k8s port-forward $POD_NAME 9090:9090 &
+          sleep 5
+      - name: Install Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+          architecture: 'x64'
+      - name: Install environment
+        run: |
+          sudo apt-get install build-essential python3-dev
+          pip install --upgrade pip
+          pip install -r requirements.txt
+
+      - name: Deploy test workloads
+        run: |
+          kubectl apply -f CI/templates/outage_pod.yaml
+          kubectl wait --for=condition=ready pod -l scenario=outage --timeout=300s
+          kubectl apply -f CI/templates/container_scenario_pod.yaml
+          kubectl wait --for=condition=ready pod -l scenario=container --timeout=300s
+          kubectl create namespace namespace-scenario
+          kubectl apply -f CI/templates/time_pod.yaml
+          kubectl wait --for=condition=ready pod -l scenario=time-skew --timeout=300s
+      - name: Get Kind nodes
+        run: |
+          kubectl get nodes --show-labels=true
+      # Pull request only steps
+      - name: Run unit tests
+        if: github.event_name == 'pull_request'
+        run: python -m coverage run -a -m unittest discover -s tests -v
+
+      - name: Setup Pull Request Functional Tests
+        if: github.event_name == 'pull_request'
+        run: |
+            yq -i '.kraken.port="8081"' CI/config/common_test_config.yaml
+            yq -i '.kraken.signal_address="0.0.0.0"' CI/config/common_test_config.yaml
+            yq -i '.kraken.performance_monitoring="localhost:9090"' CI/config/common_test_config.yaml
+            echo "test_app_outages" > ./CI/tests/functional_tests
+            echo "test_container"      >> ./CI/tests/functional_tests
+            echo "test_namespace"      >> ./CI/tests/functional_tests
+            echo "test_net_chaos"      >> ./CI/tests/functional_tests
+            echo "test_time"           >> ./CI/tests/functional_tests
+            echo "test_arca_cpu_hog" >> ./CI/tests/functional_tests
+            echo "test_arca_memory_hog" >> ./CI/tests/functional_tests
+            echo "test_arca_io_hog" >> ./CI/tests/functional_tests
+
+      # Push on main only steps
+      - name: Configure AWS Credentials
+        if: github.ref == 'refs/heads/main' && github.event_name == 'push'
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region : ${{ secrets.AWS_REGION }}
+      - name: Setup Post Merge Request Functional Tests
+        if: github.ref == 'refs/heads/main' && github.event_name == 'push'
+        run: |
+          yq -i '.kraken.port="8081"' CI/config/common_test_config.yaml
+          yq -i '.kraken.signal_address="0.0.0.0"' CI/config/common_test_config.yaml
+          yq -i '.kraken.performance_monitoring="localhost:9090"' CI/config/common_test_config.yaml
+          yq -i '.telemetry.username="${{secrets.TELEMETRY_USERNAME}}"' CI/config/common_test_config.yaml
+          yq -i '.telemetry.password="${{secrets.TELEMETRY_PASSWORD}}"' CI/config/common_test_config.yaml
+          echo "test_telemetry" > ./CI/tests/functional_tests
+
+      # Final common steps
+      - name: Run Functional tests
+        env:
+          AWS_BUCKET: ${{ secrets.AWS_BUCKET }}
+        run: |
+          ./CI/run.sh
+          cat ./CI/results.markdown >> $GITHUB_STEP_SUMMARY
+          echo >> $GITHUB_STEP_SUMMARY
+      - name: Upload CI logs
+        uses: actions/upload-artifact@v3
+        with:
+          name: ci-logs
+          path: CI/out
+          if-no-files-found: error
+      - name: Collect coverage report
+        run: |
+          python -m coverage html
+      - name: Publish coverage report to job summary
+        run: |
+          pip install html2text
+          html2text --ignore-images --ignore-links -b 0 htmlcov/index.html >> $GITHUB_STEP_SUMMARY
+      - name: Upload coverage data
+        uses: actions/upload-artifact@v3
+        with:
+          name: coverage
+          path: htmlcov
+          if-no-files-found: error
+      - name: Check CI results
+        run: grep Fail CI/results.markdown && false || true
+      

.gitignore

@@ -16,6 +16,7 @@ __pycache__/*
 *.out
 kube-burner*
 kube_burner*
+recommender_*.json
 
 # Project files
 .ropeproject
@@ -61,7 +62,7 @@ inspect.local.*
 !CI/config/common_test_config.yaml
 CI/out/*
 CI/ci_results
-CI/scenarios/*node.yaml
+CI/legacy/*node.yaml
 CI/results.markdown
 
 #env

CI/CRC/deployment.yaml

@@ -1,44 +0,0 @@
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: $NAMESPACE
-
----
-
-apiVersion: v1
-kind: Service
-metadata:
-  name: $DEPLOYMENT_NAME-service
-  namespace: $NAMESPACE
-spec:
-  selector:
-    app: $DEPLOYMENT_NAME
-  ports:
-    - name: http
-      port: 80
-      targetPort: 8080
-  type: ClusterIP
-
---- 
-
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  namespace: $NAMESPACE
-  name: $DEPLOYMENT_NAME-deployment
-spec:
-  replicas: 3
-  selector:
-    matchLabels:
-      app: $DEPLOYMENT_NAME
-  template:
-    metadata:
-      labels:
-        app: $DEPLOYMENT_NAME
-    spec:
-      containers:
-        - name: $DEPLOYMENT_NAME
-          image: nginxinc/nginx-unprivileged:stable-alpine
-          ports:
-            - name: http
-              containerPort: 8080

CI/CRC/init_github_action.sh

@@ -1,58 +0,0 @@
-#!/bin/bash
-SCRIPT_PATH=./CI/CRC
-DEPLOYMENT_PATH=$SCRIPT_PATH/deployment.yaml
-
-[[ ! -f $DEPLOYMENT_PATH ]] && echo "[ERROR] please run $0 from GitHub action root directory" && exit 1
-[[ -z $DEPLOYMENT_NAME ]] && echo "[ERROR] please set \$DEPLOYMENT_NAME environment variable" && exit 1
-[[ -z $NAMESPACE ]] && echo "[ERROR] please set \$NAMESPACE environment variable" && exit 1
-
-
-OPENSSL=`which openssl 2>/dev/null`
-[[ $? != 0 ]] && echo "[ERROR]: openssl missing, please install it and try again" && exit 1
-OC=`which oc 2>/dev/null`
-[[ $? != 0 ]] && echo "[ERROR]: oc missing, please install it and try again" && exit 1
-SED=`which sed 2>/dev/null`
-[[ $? != 0 ]] && echo "[ERROR]: sed missing, please install it and try again" && exit 1
-JQ=`which jq 2>/dev/null`
-[[ $? != 0 ]] && echo "[ERROR]: jq missing, please install it and try again" && exit 1
-ENVSUBST=`which envsubst 2>/dev/null`
-[[ $? != 0 ]] && echo "[ERROR]: envsubst missing, please install it and try again" && exit 1
-
-
-API_PORT="6443"
-API_ADDRESS="https://api.`cat host`.nip.io:${API_PORT}"
-FQN=$DEPLOYMENT_NAME.apps.$API_ADDRESS
-
-
-echo "[INF] deploying example deployment: $DEPLOYMENT_NAME in namespace: $NAMESPACE"
-$ENVSUBST < $DEPLOYMENT_PATH | $OC apply -f - > /dev/null 2>&1
-
-echo "[INF] creating SSL self-signed certificates for route https://$FQN"
-$OPENSSL genrsa -out servercakey.pem > /dev/null 2>&1
-$OPENSSL req -new -x509 -key servercakey.pem -out serverca.crt -subj "/CN=$FQN/O=Red Hat Inc./C=US" > /dev/null 2>&1
-$OPENSSL genrsa -out server.key > /dev/null 2>&1
-$OPENSSL req -new -key server.key -out server_reqout.txt -subj "/CN=$FQN/O=Red Hat Inc./C=US" > /dev/null 2>&1
-$OPENSSL x509 -req -in server_reqout.txt -days 3650 -sha256 -CAcreateserial -CA serverca.crt -CAkey servercakey.pem -out server.crt > /dev/null 2>&1
-echo "[INF] creating deployment: $DEPLOYMENT_NAME public route: https://$FQN"
-$OC create route --namespace $NAMESPACE  edge --service=$DEPLOYMENT_NAME-service --cert=server.crt --key=server.key --ca-cert=serverca.crt --hostname="$FQN" > /dev/null 2>&1
-
-
-echo "[INF] setting github action environment variables"
-
-NODE_NAME="`$OC get nodes  -o json | $JQ -r '.items[0].metadata.name'`"
-COVERAGE_FILE="`pwd`/coverage.md"
-echo "DEPLOYMENT_NAME=$DEPLOYMENT_NAME" >> $GITHUB_ENV
-echo "DEPLOYMENT_FQN=$FQN" >> $GITHUB_ENV
-echo "API_ADDRESS=$API_ADDRESS" >> $GITHUB_ENV
-echo "API_PORT=$API_PORT" >> $GITHUB_ENV
-echo "NODE_NAME=$NODE_NAME" >> $GITHUB_ENV
-echo "NAMESPACE=$NAMESPACE" >> $GITHUB_ENV
-echo "COVERAGE_FILE=$COVERAGE_FILE" >> $GITHUB_ENV
-
-echo "[INF] deployment fully qualified name will be available in \${{ env.DEPLOYMENT_NAME }} with value $DEPLOYMENT_NAME"
-echo "[INF] deployment name will be available in \${{ env.DEPLOYMENT_FQN }} with value $FQN"
-echo "[INF] OCP API address will be available in \${{ env.API_ADDRESS }} with value $API_ADDRESS"
-echo "[INF] OCP API port will be available in \${{ env.API_PORT }} with value $API_PORT"
-echo "[INF] OCP node name will be available in \${{ env.NODE_NAME }} with value $NODE_NAME"
-echo "[INF] coverage file will ve available in \${{ env.COVERAGE_FILE }} with value $COVERAGE_FILE"
-

CI/README.md

@@ -1,7 +1,7 @@
 ## CI Tests
 
 ### First steps
-Edit [my_tests](tests/my_tests) with tests you want to run
+Edit [functional_tests](tests/functional_tests) with tests you want to run
 
 ### How to run
 ```./CI/run.sh```
@@ -11,7 +11,7 @@ This will run kraken using python, make sure python3 is set up and configured pr
 
 ### Adding a test case
 
-1. Add in simple scenario yaml file to execute under [../CI/scenarios/](scenarios)
+1. Add in simple scenario yaml file to execute under [../CI/scenarios/](legacy)
 
 2. Copy [test_application_outages.sh](tests/test_app_outages.sh) for example on how to get started
 
@@ -27,7 +27,7 @@ This will run kraken using python, make sure python3 is set up and configured pr
 
     e. 15: Make sure name of config in line 14 matches what you pass on this line
 
-4. Add test name to [my_tests](../CI/tests/my_tests) file
+4. Add test name to [functional_tests](../CI/tests/functional_tests) file
 
     a. This will be the name of the file without ".sh"
 

CI/config/common_test_config.yaml

@@ -1,5 +1,5 @@
 kraken:
-    distribution: openshift                                # Distribution can be kubernetes or openshift.
+    distribution: kubernetes                                # Distribution can be kubernetes or openshift.
     kubeconfig_path: ~/.kube/config                        # Path to kubeconfig.
     exit_on_failure: False                                 # Exit when a post action scenario fails.
     litmus_version: v1.13.6                                # Litmus version to install.
@@ -29,9 +29,12 @@ tunings:
     daemon_mode: False                                     # Iterations are set to infinity which means that the kraken will cause chaos forever.
 telemetry:
     enabled: False                                           # enable/disables the telemetry collection feature
-    api_url: https://ulnmf9xv7j.execute-api.us-west-2.amazonaws.com/production #telemetry service endpoint
-    username: username                                      # telemetry service username
-    password: password                                      # telemetry service password
+    api_url: https://yvnn4rfoi7.execute-api.us-west-2.amazonaws.com/test #telemetry service endpoint
+    username: $TELEMETRY_USERNAME                                      # telemetry service username
+    password: $TELEMETRY_PASSWORD                                      # telemetry service password
+    prometheus_namespace: 'prometheus-k8s'                                # prometheus namespace
+    prometheus_pod_name: 'prometheus-kind-prometheus-kube-prome-prometheus-0'                                 # prometheus pod_name
+    prometheus_container_name: 'prometheus'
     prometheus_backup: True                                 # enables/disables prometheus data collection
     full_prometheus_backup: False                           # if is set to False only the /prometheus/wal folder will be downloaded.
     backup_threads: 5                                       # number of telemetry download/upload threads
@@ -39,3 +42,11 @@ telemetry:
     max_retries: 0                                          # maximum number of upload retries (if 0 will retry forever)
     run_tag: ''                                             # if set, this will be appended to the run folder in the bucket (useful to group the runs)
     archive_size: 10000                                     # the size of the prometheus data archive size in KB. The lower the size of archive is
+    logs_backup: True
+    logs_filter_patterns:
+        - "(\\w{3}\\s\\d{1,2}\\s\\d{2}:\\d{2}:\\d{2}\\.\\d+).+"         # Sep 9 11:20:36.123425532
+        - "kinit (\\d+/\\d+/\\d+\\s\\d{2}:\\d{2}:\\d{2})\\s+"          # kinit 2023/09/15 11:20:36 log
+        - "(\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d+Z).+"      # 2023-09-15T11:20:36.123425532Z log
+    oc_cli_path: /usr/bin/oc                                # optional, if not specified will be search in $PATH
+    events_backup: True                                     # enables/disables cluster events collection
+    telemetry_group: "funtests"

CI/run.sh

@@ -1,15 +1,14 @@
 #!/bin/bash
-set -x
 MAX_RETRIES=60
 
-OC=`which oc 2>/dev/null`
-[[ $? != 0 ]] && echo "[ERROR]: oc missing, please install it and try again" && exit 1
+KUBECTL=`which kubectl 2>/dev/null`
+[[ $? != 0 ]] && echo "[ERROR]: kubectl missing, please install it and try again" && exit 1
 
 wait_cluster_become_ready() {
   COUNT=1
-  until `$OC get namespace > /dev/null 2>&1`
+  until `$KUBECTL get namespace > /dev/null 2>&1`
   do
-    echo "[INF] waiting OpenShift to become ready, after $COUNT check"
+    echo "[INF] waiting Kubernetes to become ready, after $COUNT check"
     sleep 3
     [[ $COUNT == $MAX_RETRIES ]] && echo "[ERR] max retries exceeded, failing" && exit 1
     ((COUNT++))
@@ -18,9 +17,9 @@ wait_cluster_become_ready() {
 
 
 
-ci_tests_loc="CI/tests/my_tests"
+ci_tests_loc="CI/tests/functional_tests"
 
-echo "running test suit consisting of ${ci_tests}"
+echo -e "********* Running Functional Tests Suite *********\n\n"
 
 rm -rf CI/out
 
@@ -37,9 +36,32 @@ echo 'Test                   | Result | Duration' >> $results
 echo '-----------------------|--------|---------' >> $results
 
 # Run each test
-for test_name in `cat CI/tests/my_tests`
+failed_tests=()
+for test_name in `cat CI/tests/functional_tests`
 do
-  wait_cluster_become_ready
-  ./CI/run_test.sh $test_name $results
+  #wait_cluster_become_ready
+  return_value=`./CI/run_test.sh $test_name $results`
+  if [[ $return_value == 1 ]]
+  then
+    echo "Failed"
+    failed_tests+=("$test_name")
+  fi
   wait_cluster_become_ready
 done
+
+
+if (( ${#failed_tests[@]}>0 ))
+then
+  echo -e "\n\n======================================================================"
+  echo -e "\n     FUNCTIONAL TESTS FAILED  ${failed_tests[*]} ABORTING"
+  echo -e "\n======================================================================\n\n"
+
+  for test in "${failed_tests[@]}"
+  do
+    echo -e "\n********** $test KRKN RUN OUTPUT **********\n"
+    cat "CI/out/$test.out"
+    echo -e "\n********************************************\n\n\n\n"
+  done
+
+  exit 1
+fi

CI/run_test.sh

@@ -1,5 +1,4 @@
 #!/bin/bash
-set -x
 readonly SECONDS_PER_HOUR=3600
 readonly SECONDS_PER_MINUTE=60
 function get_time_format() {
@@ -14,9 +13,7 @@ ci_test=`echo $1`
 
 results_file=$2
 
-echo -e "\n======================================================================"
-echo -e "     CI test for ${ci_test}                    "
-echo -e "======================================================================\n"
+echo -e "test: ${ci_test}" >&2
 
 ci_results="CI/out/$ci_test.out"
 # Test ci
@@ -28,13 +25,16 @@ then
   # if the test passes update the results and complete
   duration=$SECONDS
   duration=$(get_time_format $duration)
-  echo "$ci_test: Successful"
+  echo -e  "> $ci_test: Successful\n" >&2
   echo "$ci_test | Pass | $duration" >> $results_file
   count=$retries
+  # return value for run.sh
+  echo 0
 else
   duration=$SECONDS
   duration=$(get_time_format $duration)
-  echo "$ci_test: Failed"
+  echo -e "> $ci_test: Failed\n" >&2
   echo "$ci_test | Fail | $duration" >> $results_file
-  echo "Logs for "$ci_test
+  # return value for run.sh
+  echo 1
 fi

CI/scenarios/app_outage.yaml

@@ -1,5 +0,0 @@
-application_outage:                                  # Scenario to create an outage of an application by blocking traffic
-  duration: 10                                      # Duration in seconds after which the routes will be accessible
-  namespace: openshift-monitoring            # Namespace to target - all application routes will go inaccessible if pod selector is empty
-  pod_selector: {}                            # Pods to target
-  block: [Ingress, Egress]                           # It can be Ingress or Egress or Ingress, Egress

CI/scenarios/container_scenario.yml

@@ -1,8 +0,0 @@
-scenarios:
-- name: "kill machine config container"
-  namespace: "openshift-machine-config-operator"
-  label_selector: "k8s-app=machine-config-server"
-  container_name: "hello-openshift"
-  action: "kill 1"
-  count: 1
-  retry_wait: 60

CI/scenarios/network_chaos.yaml

@@ -1,6 +0,0 @@
-network_chaos:                   # Scenario to create an outage by simulating random variations in the network.
-  duration: 10                   # seconds
-  instance_count: 1
-  execution: serial
-  egress:
-    bandwidth: 100mbit

CI/scenarios/network_diagnostics_namespace.yaml

@@ -1,7 +0,0 @@
-scenarios:
-  - action: delete
-    namespace: "^$openshift-network-diagnostics$"
-    label_selector:
-    runs: 1
-    sleep: 15
-    wait_time: 30

CI/scenarios/time_scenarios.yml

@@ -1,5 +0,0 @@
-time_scenarios:
-- action: skew_time
-  object_type: pod
-  label_selector: k8s-app=etcd
-  container_name: ""

CI/templates/container_scenario_pod.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: container
+  labels:
+    scenario: container
+spec:
+  hostNetwork: true
+  containers:
+  - name: fedtools
+    image: docker.io/fedora/tools
+    command:
+    - /bin/sh
+    - -c
+    - |
+      sleep infinity

CI/templates/outage_pod.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: outage
+  labels:
+    scenario: outage
+spec:
+  hostNetwork: true
+  containers:
+  - name: fedtools
+    image: docker.io/fedora/tools
+    command:
+    - /bin/sh
+    - -c
+    - |
+      sleep infinity

CI/templates/time_pod.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: time-skew
+  labels:
+    scenario: time-skew
+spec:
+  hostNetwork: true
+  containers:
+  - name: fedtools
+    image: docker.io/fedora/tools
+    command:
+    - /bin/sh
+    - -c
+    - |
+      sleep infinity

CI/tests/common.sh

@@ -13,6 +13,6 @@ function error {
 }
 
 function get_node {
-  worker_node=$(oc get nodes --no-headers | grep worker | head -n 1)
+  worker_node=$(kubectl get nodes --no-headers | grep worker | head -n 1)
   export WORKER_NODE=$worker_node
 }

CI/tests/functional_tests

@@ -0,0 +1 @@
+

CI/tests/my_tests

@@ -1 +0,0 @@
-test_net_chaos

CI/tests/test_app_outages.sh

@@ -7,9 +7,11 @@ trap finish EXIT
 
 
 function functional_test_app_outage {
-
+  yq -i '.application_outage.duration=10' scenarios/openshift/app_outage.yaml
+  yq -i '.application_outage.pod_selector={"scenario":"outage"}' scenarios/openshift/app_outage.yaml
+  yq -i '.application_outage.namespace="default"' scenarios/openshift/app_outage.yaml
   export scenario_type="application_outages"
-  export scenario_file="CI/scenarios/app_outage.yaml"
+  export scenario_file="scenarios/openshift/app_outage.yaml"
   export post_config=""
   envsubst < CI/config/common_test_config.yaml > CI/config/app_outage.yaml
   python3 -m coverage run -a run_kraken.py -c CI/config/app_outage.yaml

CI/tests/test_app_outages_gh.sh

@@ -1,21 +0,0 @@
-set -xeEo pipefail
-
-source CI/tests/common.sh
-
-trap error ERR
-trap finish EXIT
-
-
-function functional_test_app_outage {
-  [ -z $DEPLOYMENT_NAME ] && echo "[ERR] DEPLOYMENT_NAME variable not set, failing." && exit 1
-  yq -i '.application_outage.pod_selector={"app":"'$DEPLOYMENT_NAME'"}' CI/scenarios/app_outage.yaml
-  yq -i '.application_outage.namespace="'$NAMESPACE'"' CI/scenarios/app_outage.yaml
-  export scenario_type="application_outages"
-  export scenario_file="CI/scenarios/app_outage.yaml"
-  export post_config=""
-  envsubst < CI/config/common_test_config.yaml > CI/config/app_outage.yaml
-  python3 -m coverage run -a run_kraken.py -c CI/config/app_outage.yaml
-  echo "App outage scenario test: Success"
-}
-
-functional_test_app_outage

CI/tests/test_arca_cpu_hog.sh

@@ -0,0 +1,19 @@
+set -xeEo pipefail
+
+source CI/tests/common.sh
+
+trap error ERR
+trap finish EXIT
+
+
+function functional_test_arca_cpu_hog {
+  yq -i '.input_list[0].node_selector={"kubernetes.io/hostname":"kind-worker2"}' scenarios/arcaflow/cpu-hog/input.yaml
+  export scenario_type="arcaflow_scenarios"
+  export scenario_file="scenarios/arcaflow/cpu-hog/input.yaml"
+  export post_config=""
+  envsubst < CI/config/common_test_config.yaml > CI/config/arca_cpu_hog.yaml
+  python3 -m coverage run -a run_kraken.py -c CI/config/arca_cpu_hog.yaml
+  echo "Arcaflow CPU Hog: Success"
+}
+
+functional_test_arca_cpu_hog

CI/tests/test_arca_io_hog.sh

@@ -0,0 +1,19 @@
+set -xeEo pipefail
+
+source CI/tests/common.sh
+
+trap error ERR
+trap finish EXIT
+
+
+function functional_test_arca_io_hog {
+  yq -i '.input_list[0].node_selector={"kubernetes.io/hostname":"kind-worker2"}' scenarios/arcaflow/io-hog/input.yaml
+  export scenario_type="arcaflow_scenarios"
+  export scenario_file="scenarios/arcaflow/io-hog/input.yaml"
+  export post_config=""
+  envsubst < CI/config/common_test_config.yaml > CI/config/arca_io_hog.yaml
+  python3 -m coverage run -a run_kraken.py -c CI/config/arca_io_hog.yaml
+  echo "Arcaflow IO Hog: Success"
+}
+
+functional_test_arca_io_hog

CI/tests/test_arca_memory_hog.sh

@@ -0,0 +1,19 @@
+set -xeEo pipefail
+
+source CI/tests/common.sh
+
+trap error ERR
+trap finish EXIT
+
+
+function functional_test_arca_memory_hog {
+  yq -i '.input_list[0].node_selector={"kubernetes.io/hostname":"kind-worker2"}' scenarios/arcaflow/memory-hog/input.yaml
+  export scenario_type="arcaflow_scenarios"
+  export scenario_file="scenarios/arcaflow/memory-hog/input.yaml"
+  export post_config=""
+  envsubst < CI/config/common_test_config.yaml > CI/config/arca_memory_hog.yaml
+  python3 -m coverage run -a run_kraken.py -c CI/config/arca_memory_hog.yaml
+  echo "Arcaflow Memory Hog: Success"
+}
+
+functional_test_arca_memory_hog

CI/tests/test_container.sh

@@ -8,9 +8,11 @@ trap finish EXIT
 pod_file="CI/scenarios/hello_pod.yaml"
 
 function functional_test_container_crash {
-
+  yq -i '.scenarios[0].namespace="default"' scenarios/openshift/app_outage.yaml
+  yq -i '.scenarios[0].label_selector="scenario=container"' scenarios/openshift/app_outage.yaml
+  yq -i '.scenarios[0].container_name="fedtools"' scenarios/openshift/app_outage.yaml
   export scenario_type="container_scenarios"
-  export scenario_file="- CI/scenarios/container_scenario.yml"
+  export scenario_file="- scenarios/openshift/app_outage.yaml"
   export post_config=""
   envsubst < CI/config/common_test_config.yaml > CI/config/container_config.yaml
 

CI/tests/test_namespace.sh

@@ -7,12 +7,13 @@ trap finish EXIT
 
 function funtional_test_namespace_deletion {
   export scenario_type="namespace_scenarios"
-  export scenario_file="-  CI/scenarios/network_diagnostics_namespace.yaml"
+  export scenario_file="-  scenarios/openshift/ingress_namespace.yaml"
   export post_config=""
-  yq '.scenarios.[0].namespace="^openshift-network-diagnostics$"' -i CI/scenarios/network_diagnostics_namespace.yaml
+  yq '.scenarios[0].namespace="^namespace-scenario$"' -i scenarios/openshift/ingress_namespace.yaml
+  yq '.scenarios[0].wait_time=30' -i scenarios/openshift/ingress_namespace.yaml
+  yq '.scenarios[0].action="delete"' -i scenarios/openshift/ingress_namespace.yaml
   envsubst < CI/config/common_test_config.yaml > CI/config/namespace_config.yaml
   python3 -m coverage run -a run_kraken.py -c CI/config/namespace_config.yaml
-  echo $?
   echo "Namespace scenario test: Success"
 }
 

CI/tests/test_net_chaos.sh

@@ -7,9 +7,16 @@ trap finish EXIT
 
 
 function functional_test_network_chaos {
+  yq -i '.network_chaos.duration=10' scenarios/openshift/network_chaos.yaml
+  yq -i '.network_chaos.node_name="kind-worker2"' scenarios/openshift/network_chaos.yaml
+  yq -i '.network_chaos.egress.bandwidth="100mbit"' scenarios/openshift/network_chaos.yaml
+  yq -i 'del(.network_chaos.interfaces)' scenarios/openshift/network_chaos.yaml
+  yq -i 'del(.network_chaos.label_selector)' scenarios/openshift/network_chaos.yaml
+  yq -i 'del(.network_chaos.egress.latency)' scenarios/openshift/network_chaos.yaml
+  yq -i 'del(.network_chaos.egress.loss)' scenarios/openshift/network_chaos.yaml
 
   export scenario_type="network_chaos"
-  export scenario_file="CI/scenarios/network_chaos.yaml"
+  export scenario_file="scenarios/openshift/network_chaos.yaml"
   export post_config=""
   envsubst < CI/config/common_test_config.yaml > CI/config/network_chaos.yaml
   python3 -m coverage run -a run_kraken.py -c CI/config/network_chaos.yaml

CI/tests/test_telemetry.sh

@@ -0,0 +1,37 @@
+set -xeEo pipefail
+
+source CI/tests/common.sh
+
+trap error ERR
+trap finish EXIT
+
+
+function functional_test_telemetry {
+  AWS_CLI=`which aws`
+  [ -z "$AWS_CLI" ]&& echo "AWS cli not found in path" && exit 1
+  [ -z "$AWS_BUCKET" ] && echo "AWS bucket not set in environment" && exit 1
+
+  export RUN_TAG="funtest-telemetry"
+  yq -i '.telemetry.enabled=True' CI/config/common_test_config.yaml
+  yq -i '.telemetry.full_prometheus_backup=True' CI/config/common_test_config.yaml
+  yq -i '.performance_monitoring.check_critical_alerts=True' CI/config/common_test_config.yaml
+  yq -i '.performance_monitoring.prometheus_url="http://localhost:9090"' CI/config/common_test_config.yaml
+  yq -i '.telemetry.run_tag=env(RUN_TAG)' CI/config/common_test_config.yaml
+
+  export scenario_type="arcaflow_scenarios"
+  export scenario_file="scenarios/arcaflow/cpu-hog/input.yaml"
+  export post_config=""
+  envsubst < CI/config/common_test_config.yaml > CI/config/telemetry.yaml
+  python3 -m coverage run -a run_kraken.py -c CI/config/telemetry.yaml
+  RUN_FOLDER=`cat CI/out/test_telemetry.out | grep amazonaws.com | sed -rn "s#.*https:\/\/.*\/files/(.*)#\1#p"`
+  $AWS_CLI s3 ls "s3://$AWS_BUCKET/$RUN_FOLDER/" | awk '{ print $4 }' > s3_remote_files
+  echo "checking if telemetry files are uploaded on s3"
+  cat s3_remote_files | grep events-00.json || ( echo "FAILED: events-00.json not uploaded" && exit 1 )
+  cat s3_remote_files | grep critical-alerts-00.json || ( echo "FAILED: critical-alerts-00.json not uploaded" && exit 1 )
+  cat s3_remote_files | grep prometheus-00.tar || ( echo "FAILED: prometheus backup not uploaded" && exit 1 )
+  cat s3_remote_files | grep telemetry.json || ( echo "FAILED: telemetry.json not uploaded" && exit 1 )
+  echo "all files uploaded!"
+  echo "Telemetry Collection: Success"
+}
+
+functional_test_telemetry

CI/tests/test_time.sh

@@ -7,8 +7,12 @@ trap finish EXIT
 
 
 function functional_test_time_scenario {
+  yq -i '.time_scenarios[0].label_selector="scenario=time-skew"' scenarios/openshift/time_scenarios_example.yml
+  yq -i '.time_scenarios[0].container_name=""' scenarios/openshift/time_scenarios_example.yml
+  yq -i '.time_scenarios[0].namespace="default"' scenarios/openshift/time_scenarios_example.yml
+  yq -i '.time_scenarios[1].label_selector="kubernetes.io/hostname=kind-worker2"' scenarios/openshift/time_scenarios_example.yml
   export scenario_type="time_scenarios"
-  export scenario_file="CI/scenarios/time_scenarios.yml"
+  export scenario_file="scenarios/openshift/time_scenarios_example.yml"
   export post_config=""
   envsubst < CI/config/common_test_config.yaml > CI/config/time_config.yaml
 

README.md

@@ -1,11 +1,11 @@
-# KrknChaos aka Kraken
-[![Docker Repository on Quay](https://quay.io/repository/redhat-chaos/krkn/status "Docker Repository on Quay")](https://quay.io/repository/redhat-chaos/krkn?tab=tags&tag=latest)
-![Workflow-Status](https://github.com/redhat-chaos/krkn/actions/workflows/docker-image.yml/badge.svg)
+# Krkn aka Kraken
+[![Docker Repository on Quay](https://quay.io/repository/krkn-chaos/krkn/status "Docker Repository on Quay")](https://quay.io/repository/krkn-chaos/krkn?tab=tags&tag=latest)
+![Workflow-Status](https://github.com/krkn-chaos/krkn/actions/workflows/docker-image.yml/badge.svg)
 
 ![Krkn logo](media/logo.png)
 
-Chaos and resiliency testing tool for Kubernetes and OpenShift.
-Kraken injects deliberate failures into Kubernetes/OpenShift clusters to check if it is resilient to turbulent conditions.
+Chaos and resiliency testing tool for Kubernetes.
+Kraken injects deliberate failures into Kubernetes clusters to check if it is resilient to turbulent conditions.
 
 
 ### Workflow
@@ -18,13 +18,13 @@ Kraken injects deliberate failures into Kubernetes/OpenShift clusters to check i
 ### Chaos Testing Guide
 [Guide](docs/index.md) encapsulates:
 - Test methodology that needs to be embraced.
-- Best practices that an OpenShift cluster, platform and applications running on top of it should take into account for best user experience, performance, resilience and reliability.
+- Best practices that an Kubernetes cluster, platform and applications running on top of it should take into account for best user experience, performance, resilience and reliability.
 - Tooling.
 - Scenarios supported.
 - Test environment recommendations as to how and where to run chaos tests.
 - Chaos testing in practice.
 
-The guide is hosted at https://redhat-chaos.github.io/krknChoas.
+The guide is hosted at https://krkn-chaos.github.io/krkn.
 
 
 ### How to Get Started
@@ -38,10 +38,10 @@ After installation, refer back to the below sections for supported scenarios and
 
 
 #### Running Kraken with minimal configuration tweaks
-For cases where you want to run Kraken with minimal configuration changes, refer to [Kraken-hub](https://github.com/redhat-chaos/krknChaos-hub). One use case is CI integration where you do not want to carry around different configuration files for the scenarios.
+For cases where you want to run Kraken with minimal configuration changes, refer to [krkn-hub](https://github.com/krkn-chaos/krkn-hub). One use case is CI integration where you do not want to carry around different configuration files for the scenarios.
 
 ### Setting up infrastructure dependencies
-Kraken indexes the metrics specified in the profile into Elasticsearch in addition to leveraging Cerberus for understanding the health of the Kubernetes/OpenShift cluster under test. More information on the features is documented below. The infrastructure pieces can be easily installed and uninstalled by running:
+Kraken indexes the metrics specified in the profile into Elasticsearch in addition to leveraging Cerberus for understanding the health of the Kubernetes cluster under test. More information on the features is documented below. The infrastructure pieces can be easily installed and uninstalled by running:
 
 ```
 $ cd kraken
@@ -57,29 +57,29 @@ This will manage the Cerberus and Elasticsearch containers on the host on which
 Instructions on how to setup the config and the options supported can be found at [Config](docs/config.md).
 
 
-### Kubernetes/OpenShift chaos scenarios supported
+### Kubernetes chaos scenarios supported
 
-Scenario type               | Kubernetes    | OpenShift          
---------------------------- | ------------- |--------------------|  
-[Pod Scenarios](docs/pod_scenarios.md) | :heavy_check_mark: | :heavy_check_mark: |
-[Pod Network Scenarios](docs/pod_network_scenarios.md) | :x: | :heavy_check_mark: |
-[Container Scenarios](docs/container_scenarios.md) | :heavy_check_mark: | :heavy_check_mark: |
-[Node Scenarios](docs/node_scenarios.md) | :heavy_check_mark: | :heavy_check_mark: |
-[Time Scenarios](docs/time_scenarios.md) | :x: | :heavy_check_mark: |
-[Hog Scenarios: CPU, Memory](docs/arcaflow_scenarios.md) | :heavy_check_mark: | :heavy_check_mark: |
-[Cluster Shut Down Scenarios](docs/cluster_shut_down_scenarios.md) | :heavy_check_mark: | :heavy_check_mark: |
-[Service Disruption Scenarios](docs/service_disruption_scenarios.md.md) | :heavy_check_mark: | :heavy_check_mark: |
-[Zone Outage Scenarios](docs/zone_outage.md) | :heavy_check_mark: | :heavy_check_mark: |
-[Application_outages](docs/application_outages.md) | :heavy_check_mark: | :heavy_check_mark: |
-[PVC scenario](docs/pvc_scenario.md) | :heavy_check_mark: | :heavy_check_mark: |
-[Network_Chaos](docs/network_chaos.md) | :heavy_check_mark: | :heavy_check_mark: |
-[ManagedCluster Scenarios](docs/managedcluster_scenarios.md) | :heavy_check_mark: | :question:         |
+Scenario type               | Kubernetes    
+--------------------------- | ------------- | 
+[Pod Scenarios](docs/pod_scenarios.md) | :heavy_check_mark: |
+[Pod Network Scenarios](docs/pod_network_scenarios.md) | :x: |
+[Container Scenarios](docs/container_scenarios.md) | :heavy_check_mark: |
+[Node Scenarios](docs/node_scenarios.md) | :heavy_check_mark: |
+[Time Scenarios](docs/time_scenarios.md) | :heavy_check_mark: |
+[Hog Scenarios: CPU, Memory](docs/arcaflow_scenarios.md) | :heavy_check_mark: |
+[Cluster Shut Down Scenarios](docs/cluster_shut_down_scenarios.md) | :heavy_check_mark: |
+[Service Disruption Scenarios](docs/service_disruption_scenarios.md.md) | :heavy_check_mark: |
+[Zone Outage Scenarios](docs/zone_outage.md) | :heavy_check_mark: |
+[Application_outages](docs/application_outages.md) | :heavy_check_mark: |
+[PVC scenario](docs/pvc_scenario.md) | :heavy_check_mark: |
+[Network_Chaos](docs/network_chaos.md) | :heavy_check_mark: |
+[ManagedCluster Scenarios](docs/managedcluster_scenarios.md) | :heavy_check_mark: |
 
 
 ### Kraken scenario pass/fail criteria and report
-It is important to make sure to check if the targeted component recovered from the chaos injection and also if the Kubernetes/OpenShift cluster is healthy as failures in one component can have an adverse impact on other components. Kraken does this by:
+It is important to make sure to check if the targeted component recovered from the chaos injection and also if the Kubernetes cluster is healthy as failures in one component can have an adverse impact on other components. Kraken does this by:
 - Having built in checks for pod and node based scenarios to ensure the expected number of replicas and nodes are up. It also supports running custom scripts with the checks.
-- Leveraging [Cerberus](https://github.com/openshift-scale/cerberus) to monitor the cluster under test and consuming the aggregated go/no-go signal to determine pass/fail post chaos. It is highly recommended to turn on the Cerberus health check feature available in Kraken. Instructions on installing and setting up Cerberus can be found [here](https://github.com/openshift-scale/cerberus#installation) or can be installed from Kraken using the [instructions](https://github.com/redhat-chaos/krkn#setting-up-infrastructure-dependencies). Once Cerberus is up and running, set cerberus_enabled to True and cerberus_url to the url where Cerberus publishes go/no-go signal in the Kraken config file. Cerberus can monitor [application routes](https://github.com/redhat-chaos/cerberus/blob/main/docs/config.md#watch-routes) during the chaos and fails the run if it encounters downtime as it is a potential downtime in a customers, or users environment as well. It is especially important during the control plane chaos scenarios including the API server, Etcd, Ingress etc. It can be enabled by setting `check_applicaton_routes: True` in the [Kraken config](https://github.com/redhat-chaos/krkn/blob/main/config/config.yaml) provided application routes are being monitored in the [cerberus config](https://github.com/redhat-chaos/krkn/blob/main/config/cerberus.yaml).
+- Leveraging [Cerberus](https://github.com/krkn-chaos/cerberus) to monitor the cluster under test and consuming the aggregated go/no-go signal to determine pass/fail post chaos. It is highly recommended to turn on the Cerberus health check feature available in Kraken. Instructions on installing and setting up Cerberus can be found [here](https://github.com/openshift-scale/cerberus#installation) or can be installed from Kraken using the [instructions](https://github.com/krkn-chaos/krkn#setting-up-infrastructure-dependencies). Once Cerberus is up and running, set cerberus_enabled to True and cerberus_url to the url where Cerberus publishes go/no-go signal in the Kraken config file. Cerberus can monitor [application routes](https://github.com/redhat-chaos/cerberus/blob/main/docs/config.md#watch-routes) during the chaos and fails the run if it encounters downtime as it is a potential downtime in a customers, or users environment as well. It is especially important during the control plane chaos scenarios including the API server, Etcd, Ingress etc. It can be enabled by setting `check_applicaton_routes: True` in the [Kraken config](https://github.com/redhat-chaos/krkn/blob/main/config/config.yaml) provided application routes are being monitored in the [cerberus config](https://github.com/redhat-chaos/krkn/blob/main/config/cerberus.yaml).
 - Leveraging built-in alert collection feature to fail the runs in case of critical alerts.
 
 ### Signaling
@@ -94,10 +94,6 @@ More detailed information on enabling and leveraging this feature can be found [
 Monitoring the Kubernetes/OpenShift cluster to observe the impact of Kraken chaos scenarios on various components is key to find out the bottlenecks as it is important to make sure the cluster is healthy in terms if both recovery as well as performance during/after the failure has been injected. Instructions on enabling it can be found [here](docs/performance_dashboards.md).
 
 
-### Scraping and storing metrics long term
-Kraken supports capturing metrics for the duration of the scenarios defined in the config and indexes then into Elasticsearch to be able to store and evaluate the state of the runs long term. The indexed metrics can be visualized with the help of Grafana. It uses [Kube-burner](https://github.com/kube-burner/kube-burner) under the hood. The metrics to capture need to be defined in a metrics profile which Kraken consumes to query prometheus ( installed by default in OpenShift ) with the start and end timestamp of the run. Information on enabling and leveraging this feature can be found [here](docs/metrics.md).
-
-
 ### SLOs validation during and post chaos
 - In addition to checking the recovery and health of the cluster and components under test, Kraken takes in a profile with the Prometheus expressions to validate and alerts, exits with a non-zero return code depending on the severity set. This feature can be used to determine pass/fail or alert on abnormalities observed in the cluster based on the metrics. 
 - Kraken also provides ability to check if any critical alerts are firing in the cluster post chaos and pass/fail's. 
@@ -107,7 +103,7 @@ Information on enabling and leveraging this feature can be found [here](docs/SLO
 
 ### OCM / ACM integration
 
-Kraken supports injecting faults into [Open Cluster Management (OCM)](https://open-cluster-management.io/) and [Red Hat Advanced Cluster Management for Kubernetes (ACM)](https://www.redhat.com/en/technologies/management/advanced-cluster-management) managed clusters through [ManagedCluster Scenarios](docs/managedcluster_scenarios.md).
+Kraken supports injecting faults into [Open Cluster Management (OCM)](https://open-cluster-management.io/) and [Red Hat Advanced Cluster Management for Kubernetes (ACM)](https://www.krkn.com/en/technologies/management/advanced-cluster-management) managed clusters through [ManagedCluster Scenarios](docs/managedcluster_scenarios.md).
 
 
 ### Blogs and other useful resources
@@ -116,7 +112,8 @@ Kraken supports injecting faults into [Open Cluster Management (OCM)](https://op
 - Blog post emphasizing the importance of making Chaos part of Performance and Scale runs to mimic the production environments: https://www.openshift.com/blog/making-chaos-part-of-kubernetes/openshift-performance-and-scalability-tests
 - Blog post on findings from Chaos test runs: https://cloud.redhat.com/blog/openshift/kubernetes-chaos-stories
 - Discussion with CNCF TAG App Delivery on Krkn workflow, features and addition to CNCF sandbox: [Github](https://github.com/cncf/sandbox/issues/44), [Tracker](https://github.com/cncf/tag-app-delivery/issues/465), [recording](https://www.youtube.com/watch?v=nXQkBFK_MWc&t=722s)
-
+- Blog post on supercharging chaos testing using AI integration in Krkn: https://www.redhat.com/en/blog/supercharging-chaos-testing-using-ai
+- Blog post announcing Krkn joining CNCF Sandbox: https://www.redhat.com/en/blog/krknchaos-joining-cncf-sandbox
 
 ### Roadmap
 Enhancements being planned can be found in the [roadmap](ROADMAP.md).
@@ -132,6 +129,7 @@ Please read [this file]((CI/README.md#adding-a-test-case)) for more information
 
 
 ### Community
-Key Members(slack_usernames/full name): paigerube14/Paige Rubendall, mffiedler/Mike Fiedler, ravielluri/Naga Ravi Chaitanya Elluri.
-* [**#krkn on Kubernetes Slack**](https://kubernetes.slack.com)
-* [**#forum-chaos on CoreOS Slack internal to Red Hat**](https://coreos.slack.com)
+Key Members(slack_usernames/full name): paigerube14/Paige Rubendall, mffiedler/Mike Fiedler, tsebasti/Tullio Sebastiani, yogi/Yogananth Subramanian, sahil/Sahil Shah, pradeep/Pradeep Surisetty and ravielluri/Naga Ravi Chaitanya Elluri.
+* [**#krkn on Kubernetes Slack**](https://kubernetes.slack.com/messages/C05SFMHRWK1)
+
+The Linux Foundation® (TLF) has registered trademarks and uses trademarks. For a list of TLF trademarks, see [Trademark Usage](https://www.linuxfoundation.org/legal/trademark-usage).

ROADMAP.md

@@ -2,14 +2,14 @@
 
 Following are a list of enhancements that we are planning to work on adding support in Krkn. Of course any help/contributions are greatly appreciated.
 
-- [ ] [Ability to run multiple chaos scenarios in parallel under load to mimic real world outages](https://github.com/redhat-chaos/krkn/issues/424)
-- [x] [Centralized storage for chaos experiments artifacts](https://github.com/redhat-chaos/krkn/issues/423)
-- [ ] [Support for causing DNS outages](https://github.com/redhat-chaos/krkn/issues/394)
-- [x] [Chaos recommender](https://github.com/redhat-chaos/krkn/tree/main/utils/chaos-recommender) to suggest scenarios having probability of impacting the service under test using profiling results 
+- [ ] [Ability to run multiple chaos scenarios in parallel under load to mimic real world outages](https://github.com/krkn-chaos/krkn/issues/424)
+- [x] [Centralized storage for chaos experiments artifacts](https://github.com/krkn-chaos/krkn/issues/423)
+- [ ] [Support for causing DNS outages](https://github.com/krkn-chaos/krkn/issues/394)
+- [x] [Chaos recommender](https://github.com/krkn-chaos/krkn/tree/main/utils/chaos-recommender) to suggest scenarios having probability of impacting the service under test using profiling results 
 - [ ] Chaos AI integration to improve and automate test coverage
-- [x] [Support for pod level network traffic shaping](https://github.com/redhat-chaos/krkn/issues/393)
-- [ ] [Ability to visualize the metrics that are being captured by Kraken and stored in Elasticsearch](https://github.com/redhat-chaos/krkn/issues/124)
-- [ ] Support for running all the scenarios of Kraken on Kubernetes distribution - see https://github.com/redhat-chaos/krkn/issues/185, https://github.com/redhat-chaos/krkn/issues/186
-- [ ] Continue to improve [Chaos Testing Guide](https://redhat-chaos.github.io/krkn) in terms of adding best practices, test environment recommendations and scenarios to make sure the OpenShift platform, as well the applications running on top it, are resilient and performant under chaotic conditions.
-- [ ] [Switch documentation references to Kubernetes](https://github.com/redhat-chaos/krkn/issues/495)
-- [ ] [OCP and Kubernetes functionalities segregation](https://github.com/redhat-chaos/krkn/issues/497)
+- [x] [Support for pod level network traffic shaping](https://github.com/krkn-chaos/krkn/issues/393)
+- [ ] [Ability to visualize the metrics that are being captured by Kraken and stored in Elasticsearch](https://github.com/krkn-chaos/krkn/issues/124)
+- [ ] Support for running all the scenarios of Kraken on Kubernetes distribution - see https://github.com/krkn-chaos/krkn/issues/185, https://github.com/redhat-chaos/krkn/issues/186
+- [ ] Continue to improve [Chaos Testing Guide](https://krkn-chaos.github.io/krkn) in terms of adding best practices, test environment recommendations and scenarios to make sure the OpenShift platform, as well the applications running on top it, are resilient and performant under chaotic conditions.
+- [ ] [Switch documentation references to Kubernetes](https://github.com/krkn-chaos/krkn/issues/495)
+- [ ] [OCP and Kubernetes functionalities segregation](https://github.com/krkn-chaos/krkn/issues/497)

config/alerts.yaml

@@ -8,7 +8,7 @@
   description: 10 minutes avg. 99th etcd fsync latency on {{$labels.pod}} higher than 1s. {{$value}}s
   severity: error
 
-- expr: avg_over_time(histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket[2m]))[10m:]) > 0.007
+- expr: avg_over_time(histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket[2m]))[10m:]) > 0.03
   description: 10 minutes avg. 99th etcd commit latency on {{$labels.pod}} higher than 30ms. {{$value}}s
   severity: warning
 

config/config.yaml

@@ -1,5 +1,5 @@
 kraken:
-    distribution: openshift                                # Distribution can be kubernetes or openshift
+    distribution: kubernetes                                # Distribution can be kubernetes or openshift
     kubeconfig_path: ~/.kube/config                        # Path to kubeconfig
     exit_on_failure: False                                 # Exit when a post action scenario fails
     publish_kraken_status: True                            # Can be accessed at http://0.0.0.0:8081
@@ -51,9 +51,7 @@ cerberus:
 performance_monitoring:
     deploy_dashboards: False                              # Install a mutable grafana and load the performance dashboards. Enable this only when running on OpenShift
     repo: "https://github.com/cloud-bulldozer/performance-dashboards.git"
-    capture_metrics: False
-    metrics_profile_path: config/metrics-aggregated.yaml
-    prometheus_url:                                       # The prometheus url/route is automatically obtained in case of OpenShift, please set it when the distribution is Kubernetes.
+    prometheus_url:                                      # The prometheus url/route is automatically obtained in case of OpenShift, please set it when the distribution is Kubernetes.
     prometheus_bearer_token:                              # The bearer token is automatically obtained in case of OpenShift, please set it when the distribution is Kubernetes. This is needed to authenticate with prometheus.
     uuid:                                                 # uuid for the run is generated by default if not set
     enable_alerts: False                                  # Runs the queries specified in the alert profile and displays the info or exits 1 when severity=error
@@ -67,14 +65,19 @@ telemetry:
     enabled: False                                           # enable/disables the telemetry collection feature
     api_url: https://ulnmf9xv7j.execute-api.us-west-2.amazonaws.com/production #telemetry service endpoint
     username: username                                      # telemetry service username
-    password: password                                      # telemetry service password
+    password: password                                    # telemetry service password
     prometheus_backup: True                                 # enables/disables prometheus data collection
+    prometheus_namespace: ""                                # namespace where prometheus is deployed (if distribution is kubernetes)
+    prometheus_container_name: ""                           # name of the prometheus container name (if distribution is kubernetes)
+    prometheus_pod_name: ""                                 # name of the prometheus pod (if distribution is kubernetes)
     full_prometheus_backup: False                           # if is set to False only the /prometheus/wal folder will be downloaded.
     backup_threads: 5                                       # number of telemetry download/upload threads
     archive_path: /tmp                                      # local path where the archive files will be temporarly stored
     max_retries: 0                                          # maximum number of upload retries (if 0 will retry forever)
     run_tag: ''                                             # if set, this will be appended to the run folder in the bucket (useful to group the runs)
-    archive_size: 500000                                     # the size of the prometheus data archive size in KB. The lower the size of archive is
+    archive_size: 500000
+    telemetry_group: ''                                     # if set will archive the telemetry in the S3 bucket on a folder named after the value, otherwise will use "default"
+    # the size of the prometheus data archive size in KB. The lower the size of archive is
                                                             # the higher the number of archive files will be produced and uploaded (and processed by backup_threads
                                                             # simultaneously).
                                                             # For unstable/slow connection is better to keep this value low
@@ -87,6 +90,9 @@ telemetry:
      - "(\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d+Z).+"      # 2023-09-15T11:20:36.123425532Z log
     oc_cli_path: /usr/bin/oc                                # optional, if not specified will be search in $PATH
     events_backup: True                                     # enables/disables cluster events collection
+elastic: 
+    elastic_url: ""                                         # To track results in elasticsearch, give url to server here; will post telemetry details when url and index not blank
+    elastic_index: ""                                       # Elastic search index pattern to post results to
 
 
 

config/config_kind.yaml

@@ -20,8 +20,6 @@ cerberus:
 performance_monitoring:
     deploy_dashboards: False                              # Install a mutable grafana and load the performance dashboards. Enable this only when running on OpenShift
     repo: "https://github.com/cloud-bulldozer/performance-dashboards.git"
-    capture_metrics: False
-    metrics_profile_path: config/metrics-aggregated.yaml
     prometheus_url:                                       # The prometheus url/route is automatically obtained in case of OpenShift, please set it when the distribution is Kubernetes.
     prometheus_bearer_token:                              # The bearer token is automatically obtained in case of OpenShift, please set it when the distribution is Kubernetes. This is needed to authenticate with prometheus.
     uuid:                                                 # uuid for the run is generated by default if not set

config/config_kubernetes.yaml

@@ -19,8 +19,6 @@ cerberus:
 performance_monitoring:
     deploy_dashboards: False                              # Install a mutable grafana and load the performance dashboards. Enable this only when running on OpenShift
     repo: "https://github.com/cloud-bulldozer/performance-dashboards.git"
-    capture_metrics: False
-    metrics_profile_path: config/metrics-aggregated.yaml
     prometheus_url:                                       # The prometheus url/route is automatically obtained in case of OpenShift, please set it when the distribution is Kubernetes.
     prometheus_bearer_token:                              # The bearer token is automatically obtained in case of OpenShift, please set it when the distribution is Kubernetes. This is needed to authenticate with prometheus.
     uuid:                                                 # uuid for the run is generated by default if not set

config/config_performance.yaml

@@ -77,3 +77,8 @@ telemetry:
      - "kinit (\\d+/\\d+/\\d+\\s\\d{2}:\\d{2}:\\d{2})\\s+"          # kinit 2023/09/15 11:20:36 log
      - "(\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d+Z).+"      # 2023-09-15T11:20:36.123425532Z log
     oc_cli_path: /usr/bin/oc                                # optional, if not specified will be search in $PATH
+elastic: 
+    elastic_url: ""                                         # To track results in elasticsearch, give url to server here; will post telemetry details when url and index not blank
+    elastic_index: ""                                       # Elastic search index pattern to post results to
+
+

config/recommender_config.yaml

@@ -7,6 +7,8 @@ auth_token: <Auth_Token>
 scrape_duration: 10m
 chaos_library: "kraken"
 log_level: INFO
+json_output_file: False
+json_output_folder_path:
 
 # for output purpose only do not change if not needed
 chaos_tests:
@@ -26,4 +28,8 @@ chaos_tests:
     - pod_network_chaos
   MEM:
     - node_memory_hog
-    - pvc_disk_fill
+    - pvc_disk_fill
+
+threshold: .7
+cpu_threshold: .5
+mem_threshold: .5

containers/Dockerfile

@@ -4,9 +4,8 @@ FROM mcr.microsoft.com/azure-cli:latest as azure-cli
 
 FROM registry.access.redhat.com/ubi8/ubi:latest
 
-LABEL org.opencontainers.image.authors="Red Hat OpenShift Chaos Engineering"
-
 ENV KUBECONFIG /root/.kube/config
+ENV PATH=$PATH:/usr/local/bin
 
 # Copy azure client binary from azure-cli image
 COPY --from=azure-cli /usr/local/bin/az /usr/bin/az
@@ -14,7 +13,7 @@ COPY --from=azure-cli /usr/local/bin/az /usr/bin/az
 # Install dependencies
 RUN yum install -y git python39 python3-pip jq gettext wget && \
     python3.9 -m pip install -U pip && \
-    git clone https://github.com/redhat-chaos/krkn.git --branch v1.5.3 /root/kraken && \
+    git clone https://github.com/krkn-chaos/krkn.git --branch v1.5.9 /root/kraken && \
     mkdir -p /root/.kube && cd /root/kraken && \
     pip3.9 install -r requirements.txt && \
     pip3.9 install virtualenv && \

containers/Dockerfile-ppc64le

@@ -7,6 +7,7 @@ FROM mcr.microsoft.com/azure-cli:latest as azure-cli
 LABEL org.opencontainers.image.authors="Red Hat OpenShift Chaos Engineering"
 
 ENV KUBECONFIG /root/.kube/config
+ENV PATH=$PATH:/usr/local/bin
 
 # Copy azure client binary from azure-cli image
 COPY --from=azure-cli /usr/local/bin/az /usr/bin/az
@@ -14,7 +15,7 @@ COPY --from=azure-cli /usr/local/bin/az /usr/bin/az
 # Install dependencies
 RUN yum install -y git python39 python3-pip jq gettext wget && \
     python3.9 -m pip install -U pip && \
-    git clone https://github.com/redhat-chaos/krkn.git --branch v1.5.3 /root/kraken && \
+    git clone https://github.com/redhat-chaos/krkn.git --branch v1.5.9 /root/kraken && \
     mkdir -p /root/.kube && cd /root/kraken && \
     pip3.9 install -r requirements.txt && \
     pip3.9 install virtualenv && \

docs/cluster_shut_down_scenarios.md

@@ -1,5 +1,5 @@
-#### Kubernetes/OpenShift cluster shut down scenario
-Scenario to shut down all the nodes including the masters and restart them after specified duration. Cluster shut down scenario can be injected by placing the shut_down config file under cluster_shut_down_scenario option in the kraken config. Refer to [cluster_shut_down_scenario](https://github.com/redhat-chaos/krkn/blob/main/scenarios/cluster_shut_down_scenario.yml) config file.
+#### Kubernetes cluster shut down scenario
+Scenario to shut down all the nodes including the masters and restart them after specified duration. Cluster shut down scenario can be injected by placing the shut_down config file under cluster_shut_down_scenario option in the kraken config. Refer to [cluster_shut_down_scenario](https://github.com/krkn-chaos/krkn/blob/main/scenarios/cluster_shut_down_scenario.yml) config file.
 
 Refer to [cloud setup](cloud_setup.md) to configure your cli properly for the cloud provider of the cluster you want to shut down.
 

docs/container_scenarios.md

@@ -4,7 +4,7 @@ This can be based on the pods namespace or labels. If you know the exact object
 These scenarios are in a simple yaml format that you can manipulate to run your specific tests or use the pre-existing scenarios to see how it works.
 
 ####  Example Config
-The following are the components of Kubernetes/OpenShift for which a basic chaos scenario config exists today.
+The following are the components of Kubernetes for which a basic chaos scenario config exists today.
 
 ```
 scenarios:
@@ -25,7 +25,7 @@ In all scenarios we do a post chaos check to wait and verify the specific compon
 Here there are two options:
 1. Pass a custom script in the main config scenario list that will run before the chaos and verify the output matches post chaos scenario.
 
-See [scenarios/post_action_etcd_container.py](https://github.com/redhat-chaos/krkn/blob/main/scenarios/post_action_etcd_container.py) for an example.
+See [scenarios/post_action_etcd_container.py](https://github.com/krkn-chaos/krkn/blob/main/scenarios/post_action_etcd_container.py) for an example.
 ```
 -   container_scenarios:                                 # List of chaos pod scenarios to load.
             - -    scenarios/container_etcd.yml

docs/contribute.md

@@ -62,7 +62,7 @@ If changes go into the main repository while you're working on your code it is b
 
 If not already configured, set the upstream url for kraken.
 ```
- git remote add upstream https://github.com/redhat-chaos/krkn.git
+ git remote add upstream https://github.com/krkn-chaos/krkn.git
 ```
 
 Rebase to upstream master branch.

docs/installation.md

@@ -3,13 +3,13 @@
 The following ways are supported to run Kraken:
 
 - Standalone python program through Git.
-- Containerized version using either Podman or Docker as the runtime via [Krkn-hub](https://github.com/redhat-chaos/krkn-hub)
+- Containerized version using either Podman or Docker as the runtime via [Krkn-hub](https://github.com/krkn-chaos/krkn-hub)
 - Kubernetes or OpenShift deployment ( unsupported )
 
 **NOTE**: It is recommended to run Kraken external to the cluster ( Standalone or Containerized ) hitting the Kubernetes/OpenShift API as running it internal to the cluster might be disruptive to itself and also might not report back the results if the chaos leads to cluster's API server instability.
 
 **NOTE**: To run Kraken on Power (ppc64le) architecture, build and run a containerized version by following the
- instructions given [here](https://github.com/redhat-chaos/krkn/blob/main/containers/build_own_image-README.md).
+ instructions given [here](https://github.com/krkn-chaos/krkn/blob/main/containers/build_own_image-README.md).
 
 **NOTE**: Helper functions for interactions in Krkn are part of [krkn-lib](https://github.com/redhat-chaos/krkn-lib). 
 Please feel free to reuse and expand them as you see fit when adding a new scenario or expanding 
@@ -19,9 +19,9 @@ the capabilities of the current supported scenarios.
 ### Git
 
 #### Clone the repository
-Pick the latest stable release to install [here](https://github.com/redhat-chaos/krkn/releases).
+Pick the latest stable release to install [here](https://github.com/krkn-chaos/krkn/releases).
 ```
-$ git clone https://github.com/redhat-chaos/krkn.git --branch <release version>
+$ git clone https://github.com/krkn-chaos/krkn.git --branch <release version>
 $ cd kraken
 ```
 
@@ -40,13 +40,13 @@ $ python3.9 run_kraken.py --config <config_file_location>
 ```
 
 ### Run containerized version
-[Krkn-hub](https://github.com/redhat-chaos/krkn-hub) is a wrapper that allows running Krkn chaos scenarios via podman or docker runtime with scenario parameters/configuration defined as environment variables.
+[Krkn-hub](https://github.com/krkn-chaos/krkn-hub) is a wrapper that allows running Krkn chaos scenarios via podman or docker runtime with scenario parameters/configuration defined as environment variables.
 
-Refer [instructions](https://github.com/redhat-chaos/krkn-hub#supported-chaos-scenarios) to get started.
+Refer [instructions](https://github.com/krkn-chaos/krkn-hub#supported-chaos-scenarios) to get started.
 
 
 ### Run Kraken as a Kubernetes deployment ( unsupported option - standalone or containerized deployers are recommended )
-Refer [Instructions](https://github.com/redhat-chaos/krkn/blob/main/containers/README.md) on how to deploy and run Kraken as a Kubernetes/OpenShift deployment.
+Refer [Instructions](https://github.com/krkn-chaos/krkn/blob/main/containers/README.md) on how to deploy and run Kraken as a Kubernetes/OpenShift deployment.
 
 
 Refer to the [chaos-kraken chart manpage](https://artifacthub.io/packages/helm/startx/chaos-kraken)

docs/metrics.md

@@ -1,31 +0,0 @@
-## Scraping and storing metrics for the run
-
-There are cases where the state of the cluster and metrics on the cluster during the chaos test run need to be stored long term to review after the cluster is terminated, for example CI and automation test runs. To help with this, Kraken supports capturing metrics for the duration of the scenarios defined in the config.
-
-The metrics to capture need to be defined in a metrics profile which Kraken consumes to query prometheus with the start and end timestamp of the run. Each run has a unique identifier ( uuid ). The uuid is generated automatically if not set in the config. This feature can be enabled in the [config](https://github.com/redhat-chaos/krkn/blob/main/config/config.yaml) by setting the following:
-
-```
-performance_monitoring:
-    capture_metrics: True
-    metrics_profile_path: config/metrics-aggregated.yaml
-    prometheus_url:                                       # The prometheus url/route is automatically obtained in case of OpenShift, please set it when the distribution is Kubernetes.
-    prometheus_bearer_token:                              # The bearer token is automatically obtained in case of OpenShift, please set it when the distribution is Kubernetes. This is needed to authenticate with prometheus.
-    uuid:                                                 # uuid for the run is generated by default if not set.
-```
-
-### Metrics profile
-A couple of [metric profiles](https://github.com/redhat-chaos/krkn/tree/main/config), [metrics.yaml](https://github.com/redhat-chaos/krkn/blob/main/config/metrics.yaml), and [metrics-aggregated.yaml](https://github.com/redhat-chaos/krkn/blob/main/config/metrics-aggregated.yaml) are shipped by default and can be tweaked to add more metrics to capture during the run. The following are the API server metrics for example:
-
-```
-metrics:
-# API server
-  - query: histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{apiserver="kube-apiserver", verb!~"WATCH", subresource!="log"}[2m])) by (verb,resource,subresource,instance,le)) > 0
-    metricName: API99thLatency
-
-  - query: sum(irate(apiserver_request_total{apiserver="kube-apiserver",verb!="WATCH",subresource!="log"}[2m])) by (verb,instance,resource,code) > 0
-    metricName: APIRequestRate
-
-  - query: sum(apiserver_current_inflight_requests{}) by (request_kind) > 0
-    metricName: APIInflightRequests
-```
-

docs/service_disruption_scenarios.md

@@ -16,7 +16,7 @@ Set to '^.*$' and label_selector to "" to randomly select any namespace in your
 
 **sleep:** Number of seconds to wait between each iteration/count of killing namespaces. Defaults to 10 seconds if not set
 
-Refer to [namespace_scenarios_example](https://github.com/redhat-chaos/krkn/blob/main/scenarios/regex_namespace.yaml) config file.
+Refer to [namespace_scenarios_example](https://github.com/krkn-chaos/krkn/blob/main/scenarios/regex_namespace.yaml) config file.
 
 ```
 scenarios:

docs/time_scenarios.md

@@ -16,7 +16,7 @@ Configuration Options:
 
 **object_name:** List of the names of pods or nodes you want to skew.
 
-Refer to [time_scenarios_example](https://github.com/redhat-chaos/krkn/blob/main/scenarios/time_scenarios_example.yml) config file.
+Refer to [time_scenarios_example](https://github.com/krkn-chaos/krkn/blob/main/scenarios/time_scenarios_example.yml) config file.
 
 ```
 time_scenarios:

kraken/chaos_recommender/analysis.py

@@ -4,13 +4,10 @@ import pandas as pd
 import kraken.chaos_recommender.kraken_tests as kraken_tests
 import time
 
-threshold = .7  # Adjust the threshold as needed
-heatmap_cpu_threshold = .5
-heatmap_mem_threshold = .5
-
 KRAKEN_TESTS_PATH = "./kraken_chaos_tests.txt"
 
-#Placeholder, this should be done with topology
+
+# Placeholder, this should be done with topology
 def return_critical_services():
     return ["web", "cart"]
 
@@ -19,6 +16,7 @@ def load_telemetry_data(file_path):
     data = pd.read_csv(file_path, delimiter=r"\s+")
     return data
 
+
 def calculate_zscores(data):
     zscores = pd.DataFrame()
     zscores["Service"] = data["service"]
@@ -27,7 +25,8 @@ def calculate_zscores(data):
     zscores["Network"] = (data["NETWORK"] - data["NETWORK"].mean()) / data["NETWORK"].std()
     return zscores
 
-def identify_outliers(data):
+
+def identify_outliers(data, threshold):
     outliers_cpu = data[data["CPU"] > threshold]["Service"].tolist()
     outliers_memory = data[data["Memory"] > threshold]["Service"].tolist()
     outliers_network = data[data["Network"] > threshold]["Service"].tolist()
@@ -47,44 +46,64 @@ def get_services_above_heatmap_threshold(dataframe, cpu_threshold, mem_threshold
     return cpu_services, mem_services
 
 
-def analysis(file_path, chaos_tests_config):
+def analysis(file_path, chaos_tests_config, threshold, heatmap_cpu_threshold, heatmap_mem_threshold):
     # Load the telemetry data from file
+    logging.info("Fetching the Telemetry data")
     data = load_telemetry_data(file_path)
 
     # Calculate Z-scores for CPU, Memory, and Network columns
     zscores = calculate_zscores(data)
 
     # Identify outliers
-    outliers_cpu, outliers_memory, outliers_network = identify_outliers(zscores)
+    logging.info("Identifying outliers")
+    outliers_cpu, outliers_memory, outliers_network = identify_outliers(zscores, threshold)
     cpu_services, mem_services = get_services_above_heatmap_threshold(data, heatmap_cpu_threshold, heatmap_mem_threshold)
 
-    # Display the identified outliers
-    logging.info("======================== Profiling ==================================")
-    logging.info(f"CPU outliers: {outliers_cpu}")
-    logging.info(f"Memory outliers: {outliers_memory}")
-    logging.info(f"Network outliers: {outliers_network}")
-    logging.info("===================== HeatMap Analysis ==============================")
+    analysis_data = analysis_json(outliers_cpu, outliers_memory,
+                                  outliers_network, cpu_services,
+                                  mem_services, chaos_tests_config)
+
+    if not cpu_services:
+        logging.info("There are no services that are using significant CPU compared to their assigned limits (infinite in case no limits are set).")
+    if not mem_services:
+        logging.info("There are no services that are using significant MEMORY compared to their assigned limits (infinite in case no limits are set).")
+    time.sleep(2)
+
+    logging.info("Please check data in utilisation.txt for further analysis")
+
+    return analysis_data
+
+
+def analysis_json(outliers_cpu, outliers_memory, outliers_network,
+                  cpu_services, mem_services, chaos_tests_config):
+
+    profiling = {
+        "cpu_outliers": outliers_cpu,
+        "memory_outliers": outliers_memory,
+        "network_outliers": outliers_network
+    }
+
+    heatmap = {
+        "services_with_cpu_heatmap_above_threshold": cpu_services,
+        "services_with_mem_heatmap_above_threshold": mem_services
+    }
+
+    recommendations = {}
 
     if cpu_services:
-        logging.info("Services with CPU_HEATMAP above threshold:", cpu_services)
-    else:
-        logging.info("There are no services that are using siginificant CPU compared to their assigned limits (infinite in case no limits are set).")
+        cpu_recommend = {"services": cpu_services,
+                         "tests": chaos_tests_config['CPU']}
+        recommendations["cpu_services_recommendations"] = cpu_recommend
+
     if mem_services:
-        logging.info("Services with MEM_HEATMAP above threshold:", mem_services)
-    else:
-        logging.info("There are no services that are using siginificant MEMORY compared to their assigned limits (infinite in case no limits are set).")
-    time.sleep(2)
-    logging.info("======================= Recommendations =============================")
-    if cpu_services:
-        logging.info(f"Recommended tests for {str(cpu_services)}  :\n {chaos_tests_config['CPU']}")
-        logging.info("\n")
-    if mem_services:
-        logging.info(f"Recommended tests for {str(mem_services)}  :\n {chaos_tests_config['MEM']}")
-        logging.info("\n")
+        mem_recommend = {"services": mem_services,
+                         "tests": chaos_tests_config['MEM']}
+        recommendations["mem_services_recommendations"] = mem_recommend
 
     if outliers_network:
-        logging.info(f"Recommended tests for  str(outliers_network)  :\n {chaos_tests_config['NETWORK']}")
-        logging.info("\n")
+        outliers_network_recommend = {"outliers_networks": outliers_network,
+                                      "tests": chaos_tests_config['NETWORK']}
+        recommendations["outliers_network_recommendations"] = (
+            outliers_network_recommend)
 
-    logging.info("\n")
-    logging.info("Please check data in utilisation.txt for further analysis")
+    return [profiling, heatmap, recommendations]

kraken/chaos_recommender/prometheus.py

@@ -1,6 +1,5 @@
 import logging
 
-import pandas
 from prometheus_api_client import PrometheusConnect
 import pandas as pd
 import urllib3
@@ -8,6 +7,7 @@ import urllib3
 
 saved_metrics_path = "./utilisation.txt"
 
+
 def convert_data_to_dataframe(data, label):
     df = pd.DataFrame()
     df['service'] = [item['metric']['pod'] for item in data]
@@ -25,6 +25,7 @@ def convert_data(data, service):
         result[pod_name] = value
     return result.get(service, '100000000000') # for those pods whose limits are not defined they can take as much resources, there assigning a very high value
 
+
 def save_utilization_to_file(cpu_data, cpu_limits_result, mem_data, mem_limits_result, network_data, filename):
     df_cpu = convert_data_to_dataframe(cpu_data, "CPU")
     merged_df = pd.DataFrame(columns=['service','CPU','CPU_LIMITS','MEM','MEM_LIMITS','NETWORK'])
@@ -39,8 +40,6 @@ def save_utilization_to_file(cpu_data, cpu_limits_result, mem_data, mem_limits_r
                     "NETWORK" : convert_data(network_data, s)}, index=[0])
         merged_df = pd.concat([merged_df, new_row_df], ignore_index=True)
 
-
-
     # Convert columns to string
     merged_df['CPU'] = merged_df['CPU'].astype(str)
     merged_df['MEM'] = merged_df['MEM'].astype(str)
@@ -57,40 +56,39 @@ def save_utilization_to_file(cpu_data, cpu_limits_result, mem_data, mem_limits_r
 
     merged_df.to_csv(filename, sep='\t', index=False)
 
+
 def fetch_utilization_from_prometheus(prometheus_endpoint, auth_token, namespace, scrape_duration):
     urllib3.disable_warnings()
     prometheus = PrometheusConnect(url=prometheus_endpoint, headers={'Authorization':'Bearer {}'.format(auth_token)}, disable_ssl=True)
 
     # Fetch CPU utilization
+    logging.info("Fetching utilization")
     cpu_query = 'sum (rate (container_cpu_usage_seconds_total{image!="", namespace="%s"}[%s])) by (pod) *1000' % (namespace,scrape_duration)
-    logging.info(cpu_query)
     cpu_result = prometheus.custom_query(cpu_query)
-    cpu_data = cpu_result
-
 
     cpu_limits_query = '(sum by (pod) (kube_pod_container_resource_limits{resource="cpu", namespace="%s"}))*1000' %(namespace)
-    logging.info(cpu_limits_query)
     cpu_limits_result = prometheus.custom_query(cpu_limits_query)
 
-
     mem_query = 'sum by (pod) (avg_over_time(container_memory_usage_bytes{image!="", namespace="%s"}[%s]))' % (namespace, scrape_duration)
-    logging.info(mem_query)
     mem_result = prometheus.custom_query(mem_query)
-    mem_data = mem_result
 
     mem_limits_query = 'sum by (pod) (kube_pod_container_resource_limits{resource="memory", namespace="%s"})  ' %(namespace)
-    logging.info(mem_limits_query)
     mem_limits_result = prometheus.custom_query(mem_limits_query)
 
-
     network_query = 'sum by (pod) ((avg_over_time(container_network_transmit_bytes_total{namespace="%s"}[%s])) + \
     (avg_over_time(container_network_receive_bytes_total{namespace="%s"}[%s])))' % (namespace, scrape_duration, namespace, scrape_duration)
     network_result = prometheus.custom_query(network_query)
-    logging.info(network_query)
-    network_data = network_result
-
-
-    save_utilization_to_file(cpu_data, cpu_limits_result, mem_data, mem_limits_result, network_data, saved_metrics_path)
-    return saved_metrics_path
+
+    save_utilization_to_file(cpu_result, cpu_limits_result, mem_result, mem_limits_result, network_result, saved_metrics_path)
+    queries = json_queries(cpu_query, cpu_limits_query, mem_query, mem_limits_query)
+    return saved_metrics_path, queries
 
 
+def json_queries(cpu_query, cpu_limits_query, mem_query, mem_limits_query):
+    queries = {
+        "cpu_query": cpu_query,
+        "cpu_limit_query": cpu_limits_query,
+        "memory_query": mem_query,
+        "memory_limit_query": mem_limits_query
+    }
+    return queries

kraken/prometheus/client.py

@@ -1,10 +1,13 @@
 import datetime
 import os.path
+from typing import Optional
+
 import urllib3
 import logging
 import sys
 
 import yaml
+from krkn_lib.models.krkn import ChaosRunAlertSummary, ChaosRunAlert
 from krkn_lib.prometheus.krkn_prometheus import KrknPrometheus
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 def alerts(prom_cli: KrknPrometheus, start_time, end_time, alert_profile):
@@ -27,4 +30,59 @@ def alerts(prom_cli: KrknPrometheus, start_time, end_time, alert_profile):
 
             prom_cli.process_alert(alert,
                                    datetime.datetime.fromtimestamp(start_time),
-                                   datetime.datetime.fromtimestamp(end_time))
+                                   datetime.datetime.fromtimestamp(end_time))
+
+
+def critical_alerts(prom_cli: KrknPrometheus,
+                    summary: ChaosRunAlertSummary,
+                    run_id,
+                    scenario,
+                    start_time,
+                    end_time):
+    summary.scenario = scenario
+    summary.run_id = run_id
+    query = r"""ALERTS{severity="critical"}"""
+    logging.info("Checking for critical alerts firing post chaos")
+
+    during_critical_alerts = prom_cli.process_prom_query_in_range(
+        query,
+        start_time=datetime.datetime.fromtimestamp(start_time),
+        end_time=end_time
+
+    )
+
+    for alert in during_critical_alerts:
+        if "metric" in alert:
+            alertname = alert["metric"]["alertname"] if "alertname" in alert["metric"] else "none"
+            alertstate = alert["metric"]["alertstate"] if "alertstate" in alert["metric"] else "none"
+            namespace = alert["metric"]["namespace"] if "namespace" in alert["metric"] else "none"
+            severity = alert["metric"]["severity"] if "severity" in alert["metric"] else "none"
+            alert = ChaosRunAlert(alertname, alertstate, namespace, severity)
+            summary.chaos_alerts.append(alert)
+
+
+    post_critical_alerts = prom_cli.process_query(
+        query
+    )
+
+    for alert in post_critical_alerts:
+        if "metric" in alert:
+            alertname = alert["metric"]["alertname"] if "alertname" in alert["metric"] else "none"
+            alertstate = alert["metric"]["alertstate"] if "alertstate" in alert["metric"] else "none"
+            namespace = alert["metric"]["namespace"] if "namespace" in alert["metric"] else "none"
+            severity = alert["metric"]["severity"] if "severity" in alert["metric"] else "none"
+            alert = ChaosRunAlert(alertname, alertstate, namespace, severity)
+            summary.post_chaos_alerts.append(alert)
+
+    during_critical_alerts_count = len(during_critical_alerts)
+    post_critical_alerts_count = len(post_critical_alerts)
+    firing_alerts = False
+
+    if during_critical_alerts_count > 0:
+        firing_alerts = True
+
+    if post_critical_alerts_count > 0:
+        firing_alerts = True
+
+    if not firing_alerts:
+        logging.info("No critical alerts are firing!!")

kraken/time_actions/common_time_functions.py

@@ -2,14 +2,18 @@ import datetime
 import time
 import logging
 import re
+
 import yaml
 import random
+
+from krkn_lib import utils
+from kubernetes.client import ApiException
+
 from ..cerberus import setup as cerberus
-from ..invoke import command as runcommand
 from krkn_lib.k8s import KrknKubernetes
 from krkn_lib.telemetry.k8s import KrknTelemetryKubernetes
 from krkn_lib.models.telemetry import ScenarioTelemetry
-from krkn_lib.utils.functions import get_yaml_item_value, log_exception
+from krkn_lib.utils.functions import get_yaml_item_value, log_exception, get_random_string
 
 
 # krkn_lib
@@ -35,13 +39,6 @@ def pod_exec(pod_name, command, namespace, container_name, kubecli:KrknKubernete
     return response
 
 
-def node_debug(node_name, command):
-    response = runcommand.invoke(
-        "oc debug node/" + node_name + " -- chroot /host " + command
-    )
-    return response
-
-
 # krkn_lib
 def get_container_name(pod_name, namespace, kubecli:KrknKubernetes, container_name=""):
 
@@ -65,15 +62,46 @@ def get_container_name(pod_name, namespace, kubecli:KrknKubernetes, container_na
         return container_name
 
 
+
+def skew_node(node_name: str, action: str, kubecli: KrknKubernetes):
+    pod_namespace = "default"
+    status_pod_name = f"time-skew-pod-{get_random_string(5)}"
+    skew_pod_name = f"time-skew-pod-{get_random_string(5)}"
+    ntp_enabled = True
+    logging.info(f'Creating pod to skew {"time" if action == "skew_time" else "date"} on node {node_name}')
+    status_command = ["timedatectl"]
+    param = "2001-01-01"
+    skew_command = ["timedatectl", "set-time"]
+    if action == "skew_time":
+        skew_command.append("01:01:01")
+    else:
+        skew_command.append("2001-01-01")
+
+    try:
+        status_response = kubecli.exec_command_on_node(node_name, status_command, status_pod_name, pod_namespace)
+        if "Network time on: no" in status_response:
+            ntp_enabled = False
+
+            logging.warning(f'ntp unactive on node {node_name} skewing {"time" if action == "skew_time" else "date"} to {param}')
+            pod_exec(skew_pod_name, skew_command, pod_namespace, None, kubecli)
+        else:
+            logging.info(f'ntp active in cluster node, {"time" if action == "skew_time" else "date"} skewing will have no effect, skipping')
+    except ApiException:
+        pass
+    except Exception as e:
+        logging.error(f"failed to execute skew command in pod: {e}")
+    finally:
+        kubecli.delete_pod(status_pod_name, pod_namespace)
+        if not ntp_enabled :
+            kubecli.delete_pod(skew_pod_name, pod_namespace)
+
+
+
 # krkn_lib
 def skew_time(scenario, kubecli:KrknKubernetes):
-    skew_command = "date --date "
-    if scenario["action"] == "skew_date":
-        skewed_date = "00-01-01"
-        skew_command += skewed_date
-    elif scenario["action"] == "skew_time":
-        skewed_time = "01:01:01"
-        skew_command += skewed_time
+    if scenario["action"] not in ["skew_date","skew_time"]:
+        raise RuntimeError(f'{scenario["action"]} is not a valid time skew action')
+
     if "node" in scenario["object_type"]:
         node_names = []
         if "object_name" in scenario.keys() and scenario["object_name"]:
@@ -83,13 +111,19 @@ def skew_time(scenario, kubecli:KrknKubernetes):
             scenario["label_selector"]
         ):
             node_names = kubecli.list_nodes(scenario["label_selector"])
-
         for node in node_names:
-            node_debug(node, skew_command)
+            skew_node(node, scenario["action"], kubecli)
             logging.info("Reset date/time on node " + str(node))
         return "node", node_names
 
     elif "pod" in scenario["object_type"]:
+        skew_command = "date --date "
+        if scenario["action"] == "skew_date":
+            skewed_date = "00-01-01"
+            skew_command += skewed_date
+        elif scenario["action"] == "skew_time":
+            skewed_time = "01:01:01"
+            skew_command += skewed_time
         container_name = get_yaml_item_value(scenario, "container_name", "")
         pod_names = []
         if "object_name" in scenario.keys() and scenario["object_name"]:
@@ -241,7 +275,8 @@ def check_date_time(object_type, names, kubecli:KrknKubernetes):
     if object_type == "node":
         for node_name in names:
             first_date_time = datetime.datetime.utcnow()
-            node_datetime_string = node_debug(node_name, skew_command)
+            check_pod_name = f"time-skew-pod-{get_random_string(5)}"
+            node_datetime_string = kubecli.exec_command_on_node(node_name, [skew_command], check_pod_name)
             node_datetime = string_to_date(node_datetime_string)
             counter = 0
             while not (
@@ -252,7 +287,8 @@ def check_date_time(object_type, names, kubecli:KrknKubernetes):
                     "Date/time on node %s still not reset, "
                     "waiting 10 seconds and retrying" % node_name
                 )
-                node_datetime_string = node_debug(node_name, skew_command)
+
+                node_datetime_string = kubecli.exec_cmd_in_pod([skew_command], check_pod_name, "default")
                 node_datetime = string_to_date(node_datetime_string)
                 counter += 1
                 if counter > max_retries:
@@ -266,6 +302,8 @@ def check_date_time(object_type, names, kubecli:KrknKubernetes):
                 logging.info(
                     "Date in node " + str(node_name) + " reset properly"
                 )
+            kubecli.delete_pod(check_pod_name)
+
     elif object_type == "pod":
         for pod_name in names:
             first_date_time = datetime.datetime.utcnow()

requirements.txt

@@ -1,40 +1,40 @@
-PyYAML>=5.1
 aliyun-python-sdk-core==2.13.36
 aliyun-python-sdk-ecs==4.24.25
 arcaflow==0.9.0
 arcaflow-plugin-sdk==0.10.0
-azure-identity
-azure-keyvault
-azure-mgmt-compute
 boto3==1.28.61
-coverage
-datetime
-docker
-docker-compose
-git+https://github.com/redhat-chaos/arcaflow-plugin-kill-pod.git
-git+https://github.com/vmware/vsphere-automation-sdk-python.git@v8.0.0.0
-gitpython
-google-api-python-client
-ibm_cloud_sdk_core
-ibm_vpc
+azure-identity==1.15.0
+azure-keyvault==4.2.0
+azure-mgmt-compute==30.5.0
 itsdangerous==2.0.1
-jinja2==3.0.3
-krkn-lib >= 1.4.5
-kubernetes
-lxml >= 4.3.0
-oauth2client>=4.1.3
-openshift-client
-paramiko
-podman-compose
-pyVmomi >= 6.7
-pyfiglet
-pytest
-python-ipmi
-python-openstackclient
-requests
-service_identity
+coverage==7.4.1
+datetime==5.4
+docker==7.0.0
+gitpython==3.1.41
+google-api-python-client==2.116.0
+ibm_cloud_sdk_core==3.18.0
+ibm_vpc==0.20.0
+jinja2==3.1.3
+krkn-lib==2.1.0
+lxml==5.1.0
+kubernetes==26.1.0
+oauth2client==4.1.3
+pandas==2.2.0
+openshift-client==1.0.21
+paramiko==3.4.0
+podman-compose==1.0.6
+pyVmomi==8.0.2.0.1
+pyfiglet==1.0.2
+pytest==8.0.0
+python-ipmi==0.5.4
+python-openstackclient==6.5.0
+requests==2.31.0
+service_identity==24.1.0
+PyYAML==6.0
 setuptools==65.5.1
 werkzeug==3.0.1
-wheel
+wheel==0.42.0
 zope.interface==5.4.0
-pandas<2.0.0
+
+git+https://github.com/krkn-chaos/arcaflow-plugin-kill-pod.git
+git+https://github.com/vmware/vsphere-automation-sdk-python.git@v8.0.0.0

run_kraken.py

@@ -9,6 +9,8 @@ import optparse
 import pyfiglet
 import uuid
 import time
+
+from krkn_lib.models.krkn import ChaosRunOutput, ChaosRunAlertSummary
 from krkn_lib.prometheus.krkn_prometheus import KrknPrometheus
 import kraken.time_actions.common_time_functions as time_actions
 import kraken.performance_dashboards.setup as performance_dashboards
@@ -27,6 +29,7 @@ import server as server
 from kraken import plugins
 from krkn_lib.k8s import KrknKubernetes
 from krkn_lib.ocp import KrknOpenshift
+from krkn_lib.telemetry.elastic import KrknElastic
 from krkn_lib.telemetry.k8s import KrknTelemetryKubernetes
 from krkn_lib.telemetry.ocp import KrknTelemetryOpenshift
 from krkn_lib.models.telemetry import ChaosRunTelemetry
@@ -34,7 +37,7 @@ from krkn_lib.utils import SafeLogger
 from krkn_lib.utils.functions import get_yaml_item_value
 
 
-
+report_file = ""
 
 # Main function
 def main(cfg):
@@ -94,6 +97,9 @@ def main(cfg):
             config["performance_monitoring"], "check_critical_alerts", False
         )
         telemetry_api_url = config["telemetry"].get("api_url")
+        elastic_config = get_yaml_item_value(config,"elastic",{})
+        elastic_url = get_yaml_item_value(elastic_config,"elastic_url","")
+        elastic_index = get_yaml_item_value(elastic_config,"elastic_index","")
         
         # Initialize clients
         if (not os.path.isfile(kubeconfig_path) and
@@ -129,8 +135,6 @@ def main(cfg):
         except:
             kubecli.initialize_clients(None)
 
-
-
         # find node kraken might be running on
         kubecli.find_kraken_node()
 
@@ -156,12 +160,22 @@ def main(cfg):
         # Cluster info
         logging.info("Fetching cluster info")
         cv = ""
-        if config["kraken"]["distribution"] == "openshift":
+        if distribution == "openshift":
             cv = ocpcli.get_clusterversion_string()
             if prometheus_url is None:
-                connection_data = ocpcli.get_prometheus_api_connection_data()
-                prometheus_url = connection_data.endpoint
-                prometheus_bearer_token = connection_data.token
+                try:
+                    connection_data = ocpcli.get_prometheus_api_connection_data()
+                    if connection_data:
+                        prometheus_url = connection_data.endpoint
+                        prometheus_bearer_token = connection_data.token
+                    else: 
+                        # If can't make a connection, set alerts to false
+                        enable_alerts = False
+                        critical_alerts = False
+                except Exception:
+                    logging.error("invalid distribution selected, running openshift scenarios against kubernetes cluster."
+                                  "Please set 'kubernetes' in config.yaml krkn.platform and try again")
+                    sys.exit(1)
         if cv != "":
             logging.info(cv)
         else:
@@ -170,7 +184,10 @@ def main(cfg):
         # KrknTelemetry init
         telemetry_k8s = KrknTelemetryKubernetes(safe_logger, kubecli)
         telemetry_ocp = KrknTelemetryOpenshift(safe_logger, ocpcli)
-        prometheus = KrknPrometheus(prometheus_url, prometheus_bearer_token)
+        telemetry_elastic = KrknElastic(safe_logger,elastic_url)
+        summary = ChaosRunAlertSummary()
+        if enable_alerts or check_critical_alerts:
+            prometheus = KrknPrometheus(prometheus_url, prometheus_bearer_token)
 
         logging.info("Server URL: %s" % kubecli.get_host())
 
@@ -200,6 +217,8 @@ def main(cfg):
 
         # Capture the start time
         start_time = int(time.time())
+        post_critical_alerts = 0
+        chaos_output = ChaosRunOutput()
         chaos_telemetry = ChaosRunTelemetry()
         chaos_telemetry.run_uuid = run_uuid
         # Loop to run the chaos starts here
@@ -280,16 +299,9 @@ def main(cfg):
                         # in the config
                         # krkn_lib
                         elif scenario_type == "time_scenarios":
-                            if distribution == "openshift":
                                 logging.info("Running time skew scenarios")
                                 failed_post_scenarios, scenario_telemetries = time_actions.run(scenarios_list, config, wait_duration, kubecli, telemetry_k8s)
                                 chaos_telemetry.scenarios.extend(scenario_telemetries)
-                            else:
-                                logging.error(
-                                    "Litmus scenarios are currently "
-                                    "supported only on openshift"
-                                )
-                                sys.exit(1)
                         # Inject cluster shutdown scenarios
                         # krkn_lib
                         elif scenario_type == "cluster_shut_down_scenarios":
@@ -337,19 +349,21 @@ def main(cfg):
                             failed_post_scenarios, scenario_telemetries = network_chaos.run(scenarios_list, config, wait_duration, kubecli, telemetry_k8s)
 
                         # Check for critical alerts when enabled
+                        post_critical_alerts = 0
                         if check_critical_alerts:
-                            logging.info("Checking for critical alerts firing post choas")
+                            prometheus_plugin.critical_alerts(prometheus,
+                                                              summary,
+                                                              run_uuid,
+                                                              scenario_type,
+                                                              start_time,
+                                                              datetime.datetime.now())
+
+                            chaos_output.critical_alerts = summary
+                            post_critical_alerts = len(summary.post_chaos_alerts)
+                            if post_critical_alerts > 0:
+                                logging.error("Post chaos critical alerts firing please check, exiting")
+                                break
 
-                            ##PROM
-                            query = r"""ALERTS{severity="critical"}"""
-                            critical_alerts = prometheus.process_prom_query_in_range(query, datetime.datetime.fromtimestamp(start_time))
-                            critical_alerts_count = len(critical_alerts)
-                            if critical_alerts_count > 0:
-                                logging.error("Critical alerts are firing: %s", critical_alerts)
-                                logging.error("Please check, exiting")
-                                sys.exit(1)
-                            else:
-                                logging.info("No critical alerts are firing!!")
 
             iteration += 1
             logging.info("")
@@ -363,27 +377,52 @@ def main(cfg):
         # if platform is openshift will be collected
         # Cloud platform and network plugins metadata
         # through OCP specific APIs
-        if config["kraken"]["distribution"] == "openshift":
+        if distribution == "openshift":
             telemetry_ocp.collect_cluster_metadata(chaos_telemetry)
         else:
             telemetry_k8s.collect_cluster_metadata(chaos_telemetry)
 
         decoded_chaos_run_telemetry = ChaosRunTelemetry(json.loads(chaos_telemetry.to_json()))
-        logging.info(f"Telemetry data:\n{decoded_chaos_run_telemetry.to_json()}")
-
+        chaos_output.telemetry = decoded_chaos_run_telemetry
+        logging.info(f"Chaos data:\n{chaos_output.to_json()}")
+        telemetry_elastic.upload_data_to_elasticsearch(decoded_chaos_run_telemetry.to_json(), elastic_index)
         if config["telemetry"]["enabled"]:
-            logging.info(f"telemetry data will be stored on s3 bucket folder: {telemetry_api_url}/download/{telemetry_request_id}")
+            logging.info(f'telemetry data will be stored on s3 bucket folder: {telemetry_api_url}/files/'
+                         f'{(config["telemetry"]["telemetry_group"] if config["telemetry"]["telemetry_group"] else "default")}/'
+                         f'{telemetry_request_id}')
             logging.info(f"telemetry upload log: {safe_logger.log_file_name}")
             try:
                 telemetry_k8s.send_telemetry(config["telemetry"], telemetry_request_id, chaos_telemetry)
                 telemetry_k8s.put_cluster_events(telemetry_request_id, config["telemetry"], start_time, end_time)
+                telemetry_k8s.put_critical_alerts(telemetry_request_id, config["telemetry"], summary)
                 # prometheus data collection is available only on Openshift
-                if config["telemetry"]["prometheus_backup"] and config["kraken"]["distribution"] == "openshift":
-                    safe_logger.info("archives download started:")
-                    prometheus_archive_files = telemetry_ocp.get_ocp_prometheus_data(config["telemetry"], telemetry_request_id)
-                    safe_logger.info("archives upload started:")
-                    telemetry_k8s.put_prometheus_data(config["telemetry"], prometheus_archive_files, telemetry_request_id)
-                if config["telemetry"]["logs_backup"]:
+                if config["telemetry"]["prometheus_backup"]:
+                    prometheus_archive_files = ''
+                    if distribution == "openshift" :
+                        prometheus_archive_files = telemetry_ocp.get_ocp_prometheus_data(config["telemetry"], telemetry_request_id)
+                    else:
+                        if (config["telemetry"]["prometheus_namespace"] and
+                                config["telemetry"]["prometheus_pod_name"] and
+                                config["telemetry"]["prometheus_container_name"]):
+                            try:
+                                prometheus_archive_files = telemetry_k8s.get_prometheus_pod_data(
+                                    config["telemetry"],
+                                    telemetry_request_id,
+                                    config["telemetry"]["prometheus_pod_name"],
+                                    config["telemetry"]["prometheus_container_name"],
+                                    config["telemetry"]["prometheus_namespace"]
+                                )
+                            except Exception as e:
+                                logging.error(f"failed to get prometheus backup with exception {str(e)}")
+                        else:
+                            logging.warning("impossible to backup prometheus,"
+                                            "check if config contains telemetry.prometheus_namespace, "
+                                            "telemetry.prometheus_pod_name and "
+                                            "telemetry.prometheus_container_name")
+                    if prometheus_archive_files:
+                        safe_logger.info("starting prometheus archive upload:")
+                        telemetry_k8s.put_prometheus_data(config["telemetry"], prometheus_archive_files, telemetry_request_id)
+                if config["telemetry"]["logs_backup"] and distribution == "openshift":
                     telemetry_ocp.put_ocp_logs(telemetry_request_id, config["telemetry"], start_time, end_time)
             except Exception as e:
                 logging.error(f"failed to send telemetry data: {str(e)}")
@@ -405,16 +444,19 @@ def main(cfg):
                 logging.error("Alert profile is not defined")
                 sys.exit(1)
 
+        if post_critical_alerts > 0:
+            logging.error("Critical alerts are firing, please check; exiting")
+            sys.exit(1)
+
         if failed_post_scenarios:
             logging.error(
                 "Post scenarios are still failing at the end of all iterations"
             )
             sys.exit(1)
 
-        run_dir = os.getcwd() + "/kraken.report"
         logging.info(
             "Successfully finished running Kraken. UUID for the run: "
-            "%s. Report generated at %s. Exiting" % (run_uuid, run_dir)
+            "%s. Report generated at %s. Exiting" % (run_uuid, report_file)
         )
     else:
         logging.error("Cannot find a config at %s, please check" % (cfg))
@@ -431,12 +473,21 @@ if __name__ == "__main__":
         help="config location",
         default="config/config.yaml",
     )
+    parser.add_option(
+        "-o",
+        "--output",
+        dest="output",
+        help="output report location",
+        default="kraken.report",
+    )
+    
     (options, args) = parser.parse_args()
+    report_file = options.output
     logging.basicConfig(
         level=logging.INFO,
         format="%(asctime)s [%(levelname)s] %(message)s",
         handlers=[
-            logging.FileHandler("kraken.report", mode="w"),
+            logging.FileHandler(report_file, mode="w"),
             logging.StreamHandler(),
         ],
     )

scenarios/arcaflow/cpu-hog/input.yaml

@@ -1,8 +1,13 @@
 input_list:
-- cpu_count: 1
-  cpu_load_percentage: 80
-  cpu_method: all
-  duration: 1s
-  kubeconfig: ''
-  namespace: default
-  node_selector: {}
+  - cpu_count: 1
+    cpu_load_percentage: 80
+    cpu_method: all
+    duration: 1s
+    kubeconfig: ''
+    namespace: default
+    # set the node selector as a key-value pair eg.
+    # node_selector:
+    #  kubernetes.io/hostname: kind-worker2
+    node_selector: {}
+
+

scenarios/arcaflow/io-hog/input.yaml

@@ -5,6 +5,9 @@ input_list:
   io_write_bytes: 10m
   kubeconfig: ''
   namespace: default
+  # set the node selector as a key-value pair eg.
+  # node_selector:
+  #  kubernetes.io/hostname: kind-worker2
   node_selector: {}
   target_pod_folder: /hog-data
   target_pod_volume:

scenarios/arcaflow/io-hog/sub-workflow.yaml

@@ -2,6 +2,22 @@ version: v0.2.0
 input:
   root: RootObject
   objects:
+    hostPath:
+      id: HostPathVolumeSource
+      properties:
+        path:
+          type:
+            type_id: string
+    Volume:
+      id: Volume
+      properties:
+        name:
+          type:
+            type_id: string
+        hostPath:
+          type:
+            id: hostPath
+            type_id: ref
     RootObject:
       id: input_item
       properties:
@@ -73,25 +89,8 @@ input:
             description: the volume that will be attached to the pod. In order to stress
                          the node storage only hosPath mode is currently supported
           type:
-            type_id: object
-            id: k8s_volume
-            properties:
-              name:
-                display:
-                  description: name of the volume (must match the name in pod definition)
-                type:
-                  type_id: string
-                required: true
-              hostPath:
-                  display:
-                    description: hostPath options expressed as string map (key-value)
-                  type:
-                    type_id: map
-                    values:
-                      type_id: string
-                    keys:
-                      type_id: string
-                  required: true
+            type_id: ref
+            id: Volume
           required: true
 
 steps:

scenarios/arcaflow/io-hog/workflow.yaml

@@ -2,6 +2,22 @@ version: v0.2.0
 input:
   root: RootObject
   objects:
+    hostPath:
+      id: HostPathVolumeSource
+      properties:
+        path:
+          type:
+            type_id: string
+    Volume:
+      id: Volume
+      properties:
+        name:
+          type:
+            type_id: string
+        hostPath:
+          type:
+            id: hostPath
+            type_id: ref
     RootObject:
       id: RootObject
       properties:
@@ -80,25 +96,8 @@ input:
                     description: the volume that will be attached to the pod. In order to stress
                       the node storage only hosPath mode is currently supported
                   type:
-                    type_id: object
-                    id: k8s_volume
-                    properties:
-                      name:
-                        display:
-                          description: name of the volume (must match the name in pod definition)
-                        type:
-                          type_id: string
-                        required: true
-                      hostPath:
-                        display:
-                          description: hostPath options expressed as string map (key-value)
-                        type:
-                          type_id: map
-                          values:
-                            type_id: string
-                          keys:
-                            type_id: string
-                        required: true
+                    type_id: ref
+                    id: Volume
                   required: true
 steps:
   workload_loop:

scenarios/arcaflow/memory-hog/input.yaml

@@ -2,10 +2,10 @@ input_list:
 - duration: 30s
   vm_bytes: 10%
   vm_workers: 2
-  node_selector: { }
-  # node selector example
+  # set the node selector as a key-value pair eg.
   # node_selector:
-  #   kubernetes.io/hostname: master
+  #  kubernetes.io/hostname: kind-worker2
+  node_selector: { }
   kubeconfig: ""
   namespace: default
 

scenarios/kube/container_dns.yml

@@ -3,6 +3,6 @@ scenarios:
   namespace: "kube-system"
   label_selector: "k8s-app=kube-dns"
   container_name: ""
-  action: "kill 1"
+  action: 1
   count: 1
   retry_wait: 60

scenarios/openshift/network_chaos.yaml

@@ -1,12 +1,11 @@
-network_chaos:                       # Scenario to create an outage by simulating random variations in the network.
-  duration: 300                      # seconds
-  node_name:                         # node on which scenario has to be injected;
-  label_selector: <label_selector>   # when node_name is not specified, a node with matching label_selector is selected for running the scenario.
+network_chaos: # Scenario to create an outage by simulating random variations in the network.
+  duration: 300 # seconds
+  node_name: # node on which scenario has to be injected;
+  label_selector: <label_selector> # when node_name is not specified, a node with matching label_selector is selected for running the scenario.
   instance_count: 1
-  interfaces:                        # Interface name would be the Kernel host network interface name.
-  - "<interface_name>"
+  interfaces: # Interface name would be the Kernel host network interface name.
+    - "<interface_name>"
   execution: serial
   egress:
-    latency: 50ms                    # 50ms
-    loss: 0.02                       # percentage
-    bandwidth: 100mbit
+    latency: 50ms # 50ms
+    loss: 0.02 # percentage

utils/chaos_recommender/README.md

@@ -17,9 +17,11 @@ This tool profiles an application and gathers telemetry data such as CPU, Memory
     ```
     $ python3.9 -m venv chaos
     $ source chaos/bin/activate
-    $ git clone https://github.com/redhat-chaos/krkn.git 
+    $ git clone https://github.com/krkn-chaos/krkn.git 
     $ cd krkn
     $ pip3 install -r requirements.txt
+    Edit configuration file:
+    $ vi config/recommender_config.yaml 
     $ python3.9 utils/chaos_recommender/chaos_recommender.py
     ```
 
@@ -37,11 +39,16 @@ You can customize the default values by editing the `krkn/config/recommender_con
   - `auth_token`: Auth token to connect to prometheus endpoint (must).
   - `scrape_duration`: For how long data should be fetched, e.g., '1m' (must).
   - `chaos_library`: "kraken" (currently it only supports kraken).
+  - `json_output_file`: True or False (by default False).
+  - `json_output_folder_path`: Specify folder path where output should be saved. If empty the default path is used.
   - `chaos_tests`: (for output purpose only do not change if not needed)
     - `GENERAL`: list of general purpose tests available in Krkn
     - `MEM`: list of memory related tests available in Krkn
     - `NETWORK`: list of network related tests available in Krkn
     - `CPU`: list of memory related tests available in Krkn
+  - `threshold`: Specify the threshold to use for comparison and identifying outliers
+  - `cpu_threshold`: Specify the cpu threshold to compare with the cpu limits set on the pods and identify outliers
+  - `mem_threshold`: Specify the memory threshold to compare with the memory limits set on the pods and identify outliers
 
 *TIP:* to collect prometheus endpoint and token from your OpenShift cluster you can run the following commands:
         ```
@@ -74,6 +81,8 @@ You can also provide the input values through command-line arguments launching t
                         Chaos library
   -L LOG_LEVEL, --log-level LOG_LEVEL
                         log level (DEBUG, INFO, WARNING, ERROR, CRITICAL
+  -J [FOLDER_PATH], --json-output-file [FOLDER_PATH]
+                        Create output file, the path to the folder can be specified, if not specified the default folder is used.
   -M MEM [MEM ...], --MEM MEM [MEM ...]
                         Memory related chaos tests (space separated list)
   -C CPU [CPU ...], --CPU CPU [CPU ...]
@@ -82,14 +91,19 @@ You can also provide the input values through command-line arguments launching t
                         Network related chaos tests (space separated list)
   -G GENERIC [GENERIC ...], --GENERIC GENERIC [GENERIC ...]
                         Memory related chaos tests (space separated list)
-
+  --threshold THRESHOLD
+                        Threshold
+  --cpu_threshold CPU_THRESHOLD
+                        CPU threshold to compare with the cpu limits
+  --mem_threshold MEM_THRESHOLD
+                        Memory threshold to compare with the memory limits
 ```
 
 If you provide the input values through command-line arguments, the corresponding config file inputs would be ignored.
 
 ## Podman & Docker image
 
-To run the recommender image please visit the [krkn-hub](https://github.com/redhat-chaos/krkn-hub for further infos.
+To run the recommender image please visit the [krkn-hub](https://github.com/krkn-chaos/krkn-hub for further infos.
 
 ## How it works
 
@@ -97,7 +111,7 @@ After obtaining telemetry data, sourced either locally or from Prometheus, the t
 
 ## Customizing Thresholds and Options
 
-You can customize the thresholds and options used for data analysis by modifying the `krkn/kraken/chaos_recommender/analysis.py` file. For example, you can adjust the threshold for identifying outliers by changing the value of the `threshold` variable in the `identify_outliers` function.
+You can customize the thresholds and options used for data analysis and identifying the outliers by setting the threshold, cpu_threshold and mem_threshold parameters in the config.
 
 ## Additional Files
 

utils/chaos_recommender/chaos_recommender.py

@@ -1,7 +1,9 @@
 import argparse
+import json
 import logging
 import os.path
 import sys
+import time
 import yaml
 # kraken module import for running the recommender
 # both from the root directory and the recommender
@@ -9,12 +11,13 @@ import yaml
 sys.path.insert(0, './')
 sys.path.insert(0, '../../')
 
+from krkn_lib.utils import get_yaml_item_value
+
 import kraken.chaos_recommender.analysis as analysis
 import kraken.chaos_recommender.prometheus as prometheus
 from kubernetes import config as kube_config
 
 
-
 def parse_arguments(parser):
 
     # command line options
@@ -27,6 +30,9 @@ def parse_arguments(parser):
     parser.add_argument("-s", "--scrape-duration", action="store", default="10m", help="Prometheus scrape duration")
     parser.add_argument("-L", "--log-level", action="store", default="INFO", help="log level (DEBUG, INFO, WARNING, ERROR, CRITICAL")
 
+    parser.add_argument("-J", "--json-output-file", default=False, nargs="?", action="store",
+                        help="Create output file, the path to the folder can be specified, if not specified the default folder is used")
+
     parser.add_argument("-M", "--MEM", nargs='+', action="store", default=[],
                         help="Memory related chaos tests (space separated list)")
     parser.add_argument("-C", "--CPU", nargs='+', action="store", default=[],
@@ -35,10 +41,13 @@ def parse_arguments(parser):
                         help="Network related chaos tests (space separated list)")
     parser.add_argument("-G", "--GENERIC", nargs='+', action="store", default=[],
                         help="Memory related chaos tests (space separated list)")
-
+    parser.add_argument("--threshold", action="store", default="", help="Threshold")
+    parser.add_argument("--cpu-threshold", action="store", default="", help="CPU threshold")
+    parser.add_argument("--mem-threshold", action="store", default="", help="Memory threshold")
 
     return parser.parse_args()
 
+
 def read_configuration(config_file_path):
     if not os.path.exists(config_file_path):
         logging.error(f"Config file not found: {config_file_path}")
@@ -48,15 +57,25 @@ def read_configuration(config_file_path):
         config = yaml.safe_load(config_file)
 
     log_level = config.get("log level", "INFO")
-    namespace = config.get("namespace", "")
-    kubeconfig = config.get("kubeconfig", kube_config.KUBE_CONFIG_DEFAULT_LOCATION)
+    namespace = config.get("namespace")
+    kubeconfig = get_yaml_item_value(config, "kubeconfig", kube_config.KUBE_CONFIG_DEFAULT_LOCATION)
 
-    prometheus_endpoint = config.get("prometheus_endpoint", "")
-    auth_token = config.get("auth_token", "")
-    scrape_duration = config.get("scrape_duration", "10m")
-    chaos_tests = config.get("chaos_tests" , {})
+    prometheus_endpoint = config.get("prometheus_endpoint")
+    auth_token = config.get("auth_token")
+    scrape_duration = get_yaml_item_value(config, "scrape_duration", "10m")
+    threshold = get_yaml_item_value(config, "threshold", ".7")
+    heatmap_cpu_threshold = get_yaml_item_value(config, "cpu_threshold", ".5")
+    heatmap_mem_threshold = get_yaml_item_value(config, "mem_threshold", ".3")
+    output_file = config.get("json_output_file", False)
+    if output_file is True:
+        output_path = config.get("json_output_folder_path")
+    else:
+        output_path = False
+    chaos_tests = config.get("chaos_tests", {})
     return (namespace, kubeconfig, prometheus_endpoint, auth_token, scrape_duration,
-            chaos_tests, log_level)
+            chaos_tests, log_level, threshold, heatmap_cpu_threshold,
+            heatmap_mem_threshold, output_path)
+
 
 def prompt_input(prompt, default_value):
     user_input = input(f"{prompt} [{default_value}]: ")
@@ -64,6 +83,44 @@ def prompt_input(prompt, default_value):
         return user_input
     return default_value
 
+
+def make_json_output(inputs, queries, analysis_data, output_path):
+    time_str = time.strftime("%Y-%m-%d_%H-%M-%S", time.localtime())
+
+    data = {
+        "inputs": inputs,
+        "queries": queries,
+        "profiling": analysis_data[0],
+        "heatmap_analysis": analysis_data[1],
+        "recommendations": analysis_data[2]
+    }
+
+    logging.info(f"Summary\n{json.dumps(data, indent=4)}")
+
+    if output_path is not False:
+        file = f"recommender_{inputs['namespace']}_{time_str}.json"
+        path = f"{os.path.expanduser(output_path)}/{file}"
+
+        with open(path, "w") as json_output:
+            logging.info(f"Saving output file in {output_path} folder...")
+            json_output.write(json.dumps(data, indent=4))
+            logging.info(f"Recommendation output saved in {file}.")
+
+
+def json_inputs(namespace, kubeconfig, prometheus_endpoint, scrape_duration, chaos_tests, threshold, heatmap_cpu_threshold, heatmap_mem_threshold):
+    inputs = {
+        "namespace": namespace,
+        "kubeconfig": kubeconfig,
+        "prometheus_endpoint": prometheus_endpoint,
+        "scrape_duration": scrape_duration,
+        "chaos_tests": chaos_tests,
+        "threshold": threshold,
+        "heatmap_cpu_threshold": heatmap_cpu_threshold,
+        "heatmap_mem_threshold": heatmap_mem_threshold
+    }
+    return inputs
+
+
 def main():
     parser = argparse.ArgumentParser(description="Krkn Chaos Recommender Command-Line tool")
     args = parse_arguments(parser)
@@ -81,7 +138,11 @@ def main():
          auth_token,
          scrape_duration,
          chaos_tests,
-         log_level
+         log_level,
+         threshold,
+         heatmap_cpu_threshold,
+         heatmap_mem_threshold,
+         output_path
          ) = read_configuration(args.config_file)
 
     if args.options:
@@ -91,27 +152,35 @@ def main():
         scrape_duration = args.scrape_duration
         log_level = args.log_level
         prometheus_endpoint = args.prometheus_endpoint
+        output_path = args.json_output_file
         chaos_tests = {"MEM": args.MEM, "GENERIC": args.GENERIC, "CPU": args.CPU, "NETWORK": args.NETWORK}
+        threshold = args.threshold
+        heatmap_mem_threshold = args.mem_threshold
+        heatmap_cpu_threshold = args.cpu_threshold
 
-    if log_level not in ["DEBUG","INFO", "WARNING", "ERROR","CRITICAL"]:
+    if log_level not in ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]:
         logging.error(f"{log_level} not a valid log level")
         sys.exit(1)
 
     logging.basicConfig(level=log_level)
 
-    logging.info("============================INPUTS===================================")
-    logging.info(f"Namespace: {namespace}")
-    logging.info(f"Kubeconfig: {kubeconfig}")
-    logging.info(f"Prometheus endpoint: {prometheus_endpoint}")
-    logging.info(f"Scrape duration: {scrape_duration}")
-    for test in chaos_tests.keys():
-        logging.info(f"Chaos tests {test}: {chaos_tests[test]}")
-    logging.info("=====================================================================")
+    if output_path is not False:
+        if output_path is None:
+            output_path = "./recommender_output"
+            logging.info(f"Path for output file not specified. "
+                         f"Using default folder {output_path}")
+        if not os.path.exists(os.path.expanduser(output_path)):
+            logging.error(f"Folder {output_path} for output not found.")
+            sys.exit(1)
+    logging.info("Loading inputs...")
+    inputs = json_inputs(namespace, kubeconfig, prometheus_endpoint, scrape_duration, chaos_tests, threshold, heatmap_cpu_threshold, heatmap_mem_threshold)
     logging.info("Starting Analysis ...")
-    logging.info("Fetching the Telemetry data")
 
-    file_path = prometheus.fetch_utilization_from_prometheus(prometheus_endpoint, auth_token, namespace, scrape_duration)
-    analysis(file_path, chaos_tests)
+    file_path, queries = prometheus.fetch_utilization_from_prometheus(prometheus_endpoint, auth_token, namespace, scrape_duration)
+    analysis_data = analysis(file_path, chaos_tests, threshold, heatmap_cpu_threshold, heatmap_mem_threshold)
+
+    make_json_output(inputs, queries, analysis_data, output_path)
+
 
 if __name__ == "__main__":
     main()