Allow critical alerts check when enable_alerts is disabled

This covers use case where user wants to just check for critical alerts
post chaos without having to enable the alerts evaluation feature which
evaluates prom queries specified in an alerts file.

Signed-off-by: Naga Ravi Chaitanya Elluri <nelluri@redhat.com>

.github/workflows/docker-image.yml

@@ -37,5 +37,5 @@ jobs:
       if: github.ref == 'refs/heads/main' && github.event_name == 'push'
       uses: redhat-chaos/actions/krkn-hub@main
       with:
-        QUAY_USER: ${{ secrets.QUAY_USER_1 }}
-        QUAY_TOKEN: ${{ secrets.QUAY_TOKEN_1 }}
+        QUAY_USER: ${{ secrets.QUAY_USERNAME }}
+        QUAY_TOKEN: ${{ secrets.QUAY_PASSWORD }}

.github/workflows/tests.yml

@@ -1,8 +1,12 @@
 name: Functional & Unit Tests
 on:
   pull_request:
+  push:
+    branches:
+      - main
 jobs:
   tests:
+    # Common steps
     name: Functional & Unit Tests
     runs-on: ubuntu-latest
     steps:
@@ -47,8 +51,7 @@ jobs:
           sudo apt-get install build-essential python3-dev
           pip install --upgrade pip
           pip install -r requirements.txt
-#      - name: Run unit tests
-#        run: python -m coverage run -a -m unittest discover -s tests -v
+
       - name: Deploy test workloads
         run: |
           kubectl apply -f CI/templates/outage_pod.yaml
@@ -61,10 +64,14 @@ jobs:
       - name: Get Kind nodes
         run: |
           kubectl get nodes --show-labels=true
+      # Pull request only steps
+      - name: Run unit tests
+        if: github.event_name == 'pull_request'
+        run: python -m coverage run -a -m unittest discover -s tests -v
 
-      - name: Setup Functional Tests
+      - name: Setup Pull Request Functional Tests
+        if: github.event_name == 'pull_request'
         run: |
-            yq -i '.kraken.distribution="kubernetes"' CI/config/common_test_config.yaml
             yq -i '.kraken.port="8081"' CI/config/common_test_config.yaml
             yq -i '.kraken.signal_address="0.0.0.0"' CI/config/common_test_config.yaml
             yq -i '.kraken.performance_monitoring="localhost:9090"' CI/config/common_test_config.yaml
@@ -76,13 +83,33 @@ jobs:
             echo "test_arca_cpu_hog" >> ./CI/tests/functional_tests
             echo "test_arca_memory_hog" >> ./CI/tests/functional_tests
             echo "test_arca_io_hog" >> ./CI/tests/functional_tests
+
+      # Push on main only steps
+      - name: Configure AWS Credentials
+        if: github.ref == 'refs/heads/main' && github.event_name == 'push'
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region : ${{ secrets.AWS_REGION }}
+      - name: Setup Post Merge Request Functional Tests
+        if: github.ref == 'refs/heads/main' && github.event_name == 'push'
+        run: |
+          yq -i '.kraken.port="8081"' CI/config/common_test_config.yaml
+          yq -i '.kraken.signal_address="0.0.0.0"' CI/config/common_test_config.yaml
+          yq -i '.kraken.performance_monitoring="localhost:9090"' CI/config/common_test_config.yaml
+          yq -i '.telemetry.username="${{secrets.TELEMETRY_USERNAME}}"' CI/config/common_test_config.yaml
+          yq -i '.telemetry.password="${{secrets.TELEMETRY_PASSWORD}}"' CI/config/common_test_config.yaml
+          echo "test_telemetry" > ./CI/tests/functional_tests
+
+      # Final common steps
       - name: Run Functional tests
+        env:
+          AWS_BUCKET: ${{ secrets.AWS_BUCKET }}
         run: |
           ./CI/run.sh
           cat ./CI/results.markdown >> $GITHUB_STEP_SUMMARY
           echo >> $GITHUB_STEP_SUMMARY
-      - name: Run Unit tests
-        run: python -m coverage run -a -m unittest discover -s tests -v
       - name: Upload CI logs
         uses: actions/upload-artifact@v3
         with:

CI/config/common_test_config.yaml

@@ -1,5 +1,5 @@
 kraken:
-    distribution: openshift                                # Distribution can be kubernetes or openshift.
+    distribution: kubernetes                                # Distribution can be kubernetes or openshift.
     kubeconfig_path: ~/.kube/config                        # Path to kubeconfig.
     exit_on_failure: False                                 # Exit when a post action scenario fails.
     litmus_version: v1.13.6                                # Litmus version to install.
@@ -29,9 +29,12 @@ tunings:
     daemon_mode: False                                     # Iterations are set to infinity which means that the kraken will cause chaos forever.
 telemetry:
     enabled: False                                           # enable/disables the telemetry collection feature
-    api_url: https://ulnmf9xv7j.execute-api.us-west-2.amazonaws.com/production #telemetry service endpoint
-    username: username                                      # telemetry service username
-    password: password                                      # telemetry service password
+    api_url: https://yvnn4rfoi7.execute-api.us-west-2.amazonaws.com/test #telemetry service endpoint
+    username: $TELEMETRY_USERNAME                                      # telemetry service username
+    password: $TELEMETRY_PASSWORD                                      # telemetry service password
+    prometheus_namespace: 'prometheus-k8s'                                # prometheus namespace
+    prometheus_pod_name: 'prometheus-kind-prometheus-kube-prome-prometheus-0'                                 # prometheus pod_name
+    prometheus_container_name: 'prometheus'
     prometheus_backup: True                                 # enables/disables prometheus data collection
     full_prometheus_backup: False                           # if is set to False only the /prometheus/wal folder will be downloaded.
     backup_threads: 5                                       # number of telemetry download/upload threads
@@ -39,3 +42,10 @@ telemetry:
     max_retries: 0                                          # maximum number of upload retries (if 0 will retry forever)
     run_tag: ''                                             # if set, this will be appended to the run folder in the bucket (useful to group the runs)
     archive_size: 10000                                     # the size of the prometheus data archive size in KB. The lower the size of archive is
+    logs_backup: True
+    logs_filter_patterns:
+        - "(\\w{3}\\s\\d{1,2}\\s\\d{2}:\\d{2}:\\d{2}\\.\\d+).+"         # Sep 9 11:20:36.123425532
+        - "kinit (\\d+/\\d+/\\d+\\s\\d{2}:\\d{2}:\\d{2})\\s+"          # kinit 2023/09/15 11:20:36 log
+        - "(\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d+Z).+"      # 2023-09-15T11:20:36.123425532Z log
+    oc_cli_path: /usr/bin/oc                                # optional, if not specified will be search in $PATH
+    events_backup: True                                     # enables/disables cluster events collection

CI/run.sh

@@ -39,7 +39,7 @@ echo '-----------------------|--------|---------' >> $results
 failed_tests=()
 for test_name in `cat CI/tests/functional_tests`
 do
-  wait_cluster_become_ready
+  #wait_cluster_become_ready
   return_value=`./CI/run_test.sh $test_name $results`
   if [[ $return_value == 1 ]]
   then
@@ -49,6 +49,7 @@ do
   wait_cluster_become_ready
 done
 
+
 if (( ${#failed_tests[@]}>0 ))
 then
   echo -e "\n\n======================================================================"

CI/tests/test_telemetry.sh

@@ -0,0 +1,33 @@
+set -xeEo pipefail
+
+source CI/tests/common.sh
+
+trap error ERR
+trap finish EXIT
+
+
+function functional_test_telemetry {
+  AWS_CLI=`which aws`
+  [ -z "$AWS_CLI" ]&& echo "AWS cli not found in path" && exit 1
+  [ -z "$AWS_BUCKET" ] && echo "AWS bucket not set in environment" && exit 1
+
+  export RUN_TAG="funtest-telemetry"
+  yq -i '.telemetry.enabled=True' CI/config/common_test_config.yaml
+  yq -i '.telemetry.full_prometheus_backup=True' CI/config/common_test_config.yaml
+  yq -i '.telemetry.run_tag=env(RUN_TAG)' CI/config/common_test_config.yaml
+  export scenario_type="arcaflow_scenarios"
+  export scenario_file="scenarios/arcaflow/cpu-hog/input.yaml"
+  export post_config=""
+  envsubst < CI/config/common_test_config.yaml > CI/config/telemetry.yaml
+  python3 -m coverage run -a run_kraken.py -c CI/config/telemetry.yaml
+  RUN_FOLDER=`cat CI/out/test_telemetry.out | grep amazonaws.com | sed -rn "s#.*https:\/\/.*\/download/(.*)#\1#p"`
+  $AWS_CLI s3 ls "s3://$AWS_BUCKET/$RUN_FOLDER/" | awk '{ print $4 }' > s3_remote_files
+  echo "checking if telemetry files are uploaded on s3"
+  cat s3_remote_files | grep events-00.json || ( echo "FAILED: events-00.json not uploaded" && exit 1 )
+  cat s3_remote_files | grep prometheus-00.tar || ( echo "FAILED: prometheus backup not uploaded" && exit 1 )
+  cat s3_remote_files | grep telemetry.json || ( echo "FAILED: telemetry.json not uploaded" && exit 1 )
+  echo "all files uploaded!"
+  echo "Telemetry Collection: Success"
+}
+
+functional_test_telemetry

config/alerts.yaml

@@ -8,7 +8,7 @@
   description: 10 minutes avg. 99th etcd fsync latency on {{$labels.pod}} higher than 1s. {{$value}}s
   severity: error
 
-- expr: avg_over_time(histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket[2m]))[10m:]) > 0.007
+- expr: avg_over_time(histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket[2m]))[10m:]) > 0.03
   description: 10 minutes avg. 99th etcd commit latency on {{$labels.pod}} higher than 30ms. {{$value}}s
   severity: warning
 

config/config.yaml

@@ -85,6 +85,9 @@ telemetry:
      - "(\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d+Z).+"      # 2023-09-15T11:20:36.123425532Z log
     oc_cli_path: /usr/bin/oc                                # optional, if not specified will be search in $PATH
     events_backup: True                                     # enables/disables cluster events collection
+elastic: 
+    elastic_url: ""                                         # To track results in elasticsearch, give url to server here; will post telemetry details when url and index not blank
+    elastic_index: ""                                       # Elastic search index pattern to post results to
 
 
 

config/config_performance.yaml

@@ -77,3 +77,8 @@ telemetry:
      - "kinit (\\d+/\\d+/\\d+\\s\\d{2}:\\d{2}:\\d{2})\\s+"          # kinit 2023/09/15 11:20:36 log
      - "(\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d+Z).+"      # 2023-09-15T11:20:36.123425532Z log
     oc_cli_path: /usr/bin/oc                                # optional, if not specified will be search in $PATH
+elastic: 
+    elastic_url: ""                                         # To track results in elasticsearch, give url to server here; will post telemetry details when url and index not blank
+    elastic_index: ""                                       # Elastic search index pattern to post results to
+
+

containers/Dockerfile

@@ -12,7 +12,7 @@ COPY --from=azure-cli /usr/local/bin/az /usr/bin/az
 # Install dependencies
 RUN yum install -y git python39 python3-pip jq gettext wget && \
     python3.9 -m pip install -U pip && \
-    git clone https://github.com/krkn-chaos/krkn.git --branch v1.5.5 /root/kraken && \
+    git clone https://github.com/krkn-chaos/krkn.git --branch v1.5.7 /root/kraken && \
     mkdir -p /root/.kube && cd /root/kraken && \
     pip3.9 install -r requirements.txt && \
     pip3.9 install virtualenv && \

containers/Dockerfile-ppc64le

@@ -14,7 +14,7 @@ COPY --from=azure-cli /usr/local/bin/az /usr/bin/az
 # Install dependencies
 RUN yum install -y git python39 python3-pip jq gettext wget && \
     python3.9 -m pip install -U pip && \
-    git clone https://github.com/redhat-chaos/krkn.git --branch v1.5.5 /root/kraken && \
+    git clone https://github.com/redhat-chaos/krkn.git --branch v1.5.7 /root/kraken && \
     mkdir -p /root/.kube && cd /root/kraken && \
     pip3.9 install -r requirements.txt && \
     pip3.9 install virtualenv && \

requirements.txt

@@ -1,40 +1,40 @@
-PyYAML>=5.1
 aliyun-python-sdk-core==2.13.36
 aliyun-python-sdk-ecs==4.24.25
 arcaflow==0.9.0
 arcaflow-plugin-sdk==0.10.0
-azure-identity
-azure-keyvault
-azure-mgmt-compute
 boto3==1.28.61
-coverage
-datetime
-docker
-docker-compose
-git+https://github.com/krkn-chaos/arcaflow-plugin-kill-pod.git
-git+https://github.com/vmware/vsphere-automation-sdk-python.git@v8.0.0.0
-gitpython
-google-api-python-client
-ibm_cloud_sdk_core
-ibm_vpc
+azure-identity==1.15.0
+azure-keyvault==4.2.0
+azure-mgmt-compute==30.5.0
 itsdangerous==2.0.1
+coverage==7.4.1
+datetime==5.4
+docker==7.0.0
+gitpython==3.1.41
+google-api-python-client==2.116.0
+ibm_cloud_sdk_core==3.18.0
+ibm_vpc==0.20.0
 jinja2==3.1.3
-krkn-lib >= 1.4.6
-kubernetes
-lxml >= 4.3.0
-oauth2client>=4.1.3
-openshift-client
-paramiko
-podman-compose
-pyVmomi >= 6.7
-pyfiglet
-pytest
-python-ipmi
-python-openstackclient
-requests
-service_identity
+krkn-lib==1.4.12
+lxml==5.1.0
+kubernetes==26.1.0
+oauth2client==4.1.3
+pandas==2.2.0
+openshift-client==1.0.21
+paramiko==3.4.0
+podman-compose==1.0.6
+pyVmomi==8.0.2.0.1
+pyfiglet==1.0.2
+pytest==8.0.0
+python-ipmi==0.5.4
+python-openstackclient==6.5.0
+requests==2.31.0
+service_identity==24.1.0
+PyYAML==6.0
 setuptools==65.5.1
 werkzeug==3.0.1
-wheel>=0.38.0
+wheel==0.42.0
 zope.interface==5.4.0
-pandas>=2.2.0
+
+git+https://github.com/krkn-chaos/arcaflow-plugin-kill-pod.git
+git+https://github.com/vmware/vsphere-automation-sdk-python.git@v8.0.0.0

run_kraken.py

@@ -27,6 +27,7 @@ import server as server
 from kraken import plugins
 from krkn_lib.k8s import KrknKubernetes
 from krkn_lib.ocp import KrknOpenshift
+from krkn_lib.telemetry.elastic import KrknElastic
 from krkn_lib.telemetry.k8s import KrknTelemetryKubernetes
 from krkn_lib.telemetry.ocp import KrknTelemetryOpenshift
 from krkn_lib.models.telemetry import ChaosRunTelemetry
@@ -94,6 +95,9 @@ def main(cfg):
             config["performance_monitoring"], "check_critical_alerts", False
         )
         telemetry_api_url = config["telemetry"].get("api_url")
+        elastic_config = get_yaml_item_value(config,"elastic",{})
+        elastic_url = get_yaml_item_value(elastic_config,"elastic_url","")
+        elastic_index = get_yaml_item_value(elastic_config,"elastic_index","")
         
         # Initialize clients
         if (not os.path.isfile(kubeconfig_path) and
@@ -129,8 +133,6 @@ def main(cfg):
         except:
             kubecli.initialize_clients(None)
 
-
-
         # find node kraken might be running on
         kubecli.find_kraken_node()
 
@@ -156,12 +158,22 @@ def main(cfg):
         # Cluster info
         logging.info("Fetching cluster info")
         cv = ""
-        if config["kraken"]["distribution"] == "openshift":
+        if distribution == "openshift":
             cv = ocpcli.get_clusterversion_string()
             if prometheus_url is None:
-                connection_data = ocpcli.get_prometheus_api_connection_data()
-                prometheus_url = connection_data.endpoint
-                prometheus_bearer_token = connection_data.token
+                try:
+                    connection_data = ocpcli.get_prometheus_api_connection_data()
+                    if connection_data:
+                        prometheus_url = connection_data.endpoint
+                        prometheus_bearer_token = connection_data.token
+                    else: 
+                        # If can't make a connection, set alerts to false
+                        enable_alerts = False
+                        critical_alerts = False
+                except Exception:
+                    logging.error("invalid distribution selected, running openshift scenarios against kubernetes cluster."
+                                  "Please set 'kubernetes' in config.yaml krkn.platform and try again")
+                    sys.exit(1)
         if cv != "":
             logging.info(cv)
         else:
@@ -170,9 +182,9 @@ def main(cfg):
         # KrknTelemetry init
         telemetry_k8s = KrknTelemetryKubernetes(safe_logger, kubecli)
         telemetry_ocp = KrknTelemetryOpenshift(safe_logger, ocpcli)
+        telemetry_elastic = KrknElastic(safe_logger,elastic_url)
 
-
-        if enable_alerts:
+        if enable_alerts or check_critical_alerts:
             prometheus = KrknPrometheus(prometheus_url, prometheus_bearer_token)
 
         logging.info("Server URL: %s" % kubecli.get_host())
@@ -203,6 +215,7 @@ def main(cfg):
 
         # Capture the start time
         start_time = int(time.time())
+        critical_alerts_count = 0
 
         chaos_telemetry = ChaosRunTelemetry()
         chaos_telemetry.run_uuid = run_uuid
@@ -334,23 +347,20 @@ def main(cfg):
                             failed_post_scenarios, scenario_telemetries = network_chaos.run(scenarios_list, config, wait_duration, kubecli, telemetry_k8s)
 
                         # Check for critical alerts when enabled
-                        if enable_alerts and check_critical_alerts :
+                        if check_critical_alerts:
                             logging.info("Checking for critical alerts firing post choas")
 
                             ##PROM
                             query = r"""ALERTS{severity="critical"}"""
                             end_time = datetime.datetime.now()
-                            critical_alerts = prometheus.process_prom_query_in_range(
-                                query,
-                                start_time = datetime.datetime.fromtimestamp(start_time),
-                                end_time = end_time
-
+                            critical_alerts = prometheus.process_query(
+                                query
                             )
                             critical_alerts_count = len(critical_alerts)
                             if critical_alerts_count > 0:
                                 logging.error("Critical alerts are firing: %s", critical_alerts)
                                 logging.error("Please check, exiting")
-                                sys.exit(1)
+                                break
                             else:
                                 logging.info("No critical alerts are firing!!")
 
@@ -366,14 +376,14 @@ def main(cfg):
         # if platform is openshift will be collected
         # Cloud platform and network plugins metadata
         # through OCP specific APIs
-        if config["kraken"]["distribution"] == "openshift":
+        if distribution == "openshift":
             telemetry_ocp.collect_cluster_metadata(chaos_telemetry)
         else:
             telemetry_k8s.collect_cluster_metadata(chaos_telemetry)
 
         decoded_chaos_run_telemetry = ChaosRunTelemetry(json.loads(chaos_telemetry.to_json()))
         logging.info(f"Telemetry data:\n{decoded_chaos_run_telemetry.to_json()}")
-
+        telemetry_elastic.upload_data_to_elasticsearch(decoded_chaos_run_telemetry.to_json(), elastic_index)
         if config["telemetry"]["enabled"]:
             logging.info(f"telemetry data will be stored on s3 bucket folder: {telemetry_api_url}/download/{telemetry_request_id}")
             logging.info(f"telemetry upload log: {safe_logger.log_file_name}")
@@ -381,12 +391,33 @@ def main(cfg):
                 telemetry_k8s.send_telemetry(config["telemetry"], telemetry_request_id, chaos_telemetry)
                 telemetry_k8s.put_cluster_events(telemetry_request_id, config["telemetry"], start_time, end_time)
                 # prometheus data collection is available only on Openshift
-                if config["telemetry"]["prometheus_backup"] and config["kraken"]["distribution"] == "openshift":
-                    safe_logger.info("archives download started:")
-                    prometheus_archive_files = telemetry_ocp.get_ocp_prometheus_data(config["telemetry"], telemetry_request_id)
-                    safe_logger.info("archives upload started:")
-                    telemetry_k8s.put_prometheus_data(config["telemetry"], prometheus_archive_files, telemetry_request_id)
-                if config["telemetry"]["logs_backup"]:
+                if config["telemetry"]["prometheus_backup"]:
+                    prometheus_archive_files = ''
+                    if distribution == "openshift" :
+                        prometheus_archive_files = telemetry_ocp.get_ocp_prometheus_data(config["telemetry"], telemetry_request_id)
+                    else:
+                        if (config["telemetry"]["prometheus_namespace"] and
+                                config["telemetry"]["prometheus_pod_name"] and
+                                config["telemetry"]["prometheus_container_name"]):
+                            try:
+                                prometheus_archive_files = telemetry_k8s.get_prometheus_pod_data(
+                                    config["telemetry"],
+                                    telemetry_request_id,
+                                    config["telemetry"]["prometheus_pod_name"],
+                                    config["telemetry"]["prometheus_container_name"],
+                                    config["telemetry"]["prometheus_namespace"]
+                                )
+                            except Exception as e:
+                                logging.error(f"failed to get prometheus backup with exception {str(e)}")
+                        else:
+                            logging.warning("impossible to backup prometheus,"
+                                            "check if config contains telemetry.prometheus_namespace, "
+                                            "telemetry.prometheus_pod_name and "
+                                            "telemetry.prometheus_container_name")
+                    if prometheus_archive_files:
+                        safe_logger.info("starting prometheus archive upload:")
+                        telemetry_k8s.put_prometheus_data(config["telemetry"], prometheus_archive_files, telemetry_request_id)
+                if config["telemetry"]["logs_backup"] and distribution == "openshift":
                     telemetry_ocp.put_ocp_logs(telemetry_request_id, config["telemetry"], start_time, end_time)
             except Exception as e:
                 logging.error(f"failed to send telemetry data: {str(e)}")
@@ -408,6 +439,10 @@ def main(cfg):
                 logging.error("Alert profile is not defined")
                 sys.exit(1)
 
+        if critical_alerts_count > 0:
+            logging.error("Critical alerts are firing, please check; exiting")
+            sys.exit(1)
+
         if failed_post_scenarios:
             logging.error(
                 "Post scenarios are still failing at the end of all iterations"

scenarios/arcaflow/cpu-hog/input.yaml

@@ -5,5 +5,9 @@ input_list:
     duration: 1s
     kubeconfig: ''
     namespace: default
-    node_selector:
-      kubernetes.io/hostname: kind-worker2
+    # set the node selector as a key-value pair eg.
+    # node_selector:
+    #  kubernetes.io/hostname: kind-worker2
+    node_selector: {}
+
+

scenarios/arcaflow/io-hog/input.yaml

@@ -5,6 +5,9 @@ input_list:
   io_write_bytes: 10m
   kubeconfig: ''
   namespace: default
+  # set the node selector as a key-value pair eg.
+  # node_selector:
+  #  kubernetes.io/hostname: kind-worker2
   node_selector: {}
   target_pod_folder: /hog-data
   target_pod_volume:

scenarios/arcaflow/memory-hog/input.yaml

@@ -2,10 +2,10 @@ input_list:
 - duration: 30s
   vm_bytes: 10%
   vm_workers: 2
-  node_selector: { }
-  # node selector example
+  # set the node selector as a key-value pair eg.
   # node_selector:
-  #   kubernetes.io/hostname: master
+  #  kubernetes.io/hostname: kind-worker2
+  node_selector: { }
   kubeconfig: ""
   namespace: default