Github Action + functional tests (no *hog tests) (#382)

* Github Action + functional tests (no *hog tests) * changed the trigger keyword to /test * removed deprecated kill_pod scenario + added namespace to app_outage (new kill_pod) * #365: renamed ingress_namespace scenario to network_diagnostrcs * requested team filter added --------- Co-authored-by: Tullio Sebastiani <tullio.sebastiani@x3solutions.it>
2026-08-25 09:27:36 +00:00 · 2023-02-16 09:42:33 +01:00
parent 93686ca736
commit 8b18fa8a35
14 changed files with 312 additions and 65 deletions
@@ -0,0 +1,89 @@
+on: issue_comment
+
+jobs:
+  pr_commented:
+    # This job only runs for pull request comments containing /functional
+    name: Functional Tests
+    if: contains(github.event.comment.body, '/test') && contains(github.event.comment.html_url, '/pull/') && github.event.requested_team.name == 'redhat-chaos'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out Kraken
+        uses: actions/checkout@v3
+      - name: Checkout Pull Request
+        run: hub pr checkout ${{ github.event.issue.number }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Install OC CLI
+        uses: redhat-actions/oc-installer@v1
+        with:
+          oc_version: latest
+      - name: Install python 3.9
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.9' 
+      - name: Setup kraken dependencies
+        run:  pip install -r requirements.txt
+      - name: Create Workdir & export the path
+        run: |
+         mkdir workdir
+         echo "WORKDIR_PATH=`pwd`/workdir" >> $GITHUB_ENV
+      - name: Teardown CRC (Post Action)
+        uses: webiny/action-post-run@3.0.0
+        id: post-run-command
+        with:
+          # currently using image coming from tsebastiani quay.io repo
+          # waiting that a fix is merged in the upstream one
+          # post action run cannot (apparently) be properly indented 
+          run: docker run -v "${{ env.WORKDIR_PATH }}:/workdir" -e WORKING_MODE=T -e AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} -e AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} -e AWS_DEFAULT_REGION=us-west-2 -e TEARDOWN_RUN_ID=crc quay.io/tsebastiani/crc-cloud
+      - name: Run CRC
+          # currently using image coming from tsebastiani quay.io repo
+          # waiting that a fix is merged in the upstream one
+        run: |
+            docker run -v "${{ env.WORKDIR_PATH }}:/workdir" \
+            -e WORKING_MODE=C \
+            -e PULL_SECRET="${{ secrets.PULL_SECRET }}" \
+            -e AWS_ACCESS_KEY_ID="${{ secrets.AWS_ACCESS_KEY_ID }}" \
+            -e AWS_SECRET_ACCESS_KEY="${{ secrets.AWS_SECRET_ACCESS_KEY }}" \
+            -e AWS_DEFAULT_REGION=us-west-2 \
+            -e CREATE_RUN_ID=crc \
+            -e PASS_KUBEADMIN="${{ secrets.KUBEADMIN_PWD }}" \
+            -e PASS_REDHAT="${{ secrets.REDHAT_PWD }}" \
+            -e PASS_DEVELOPER="${{ secrets.DEVELOPER_PWD }}" \
+            quay.io/tsebastiani/crc-cloud
+      - name: OpenShift login and example deployment, GitHub Action env init 
+        env:
+          NAMESPACE: test-namespace
+          DEPLOYMENT_NAME: test-nginx
+          KUBEADMIN_PWD: '${{ secrets.KUBEADMIN_PWD }}'
+        run: ./CI/CRC/init_github_action.sh
+      - name: Setup test suite
+        run: |
+          yq -i '.kraken.port="8081"' CI/config/common_test_config.yaml
+          yq -i '.kraken.signal_address="0.0.0.0"' CI/config/common_test_config.yaml
+          
+          echo "test_app_outages_gh" > ./CI/tests/my_tests
+          echo "test_container"      >> ./CI/tests/my_tests
+          echo "test_namespace"      >> ./CI/tests/my_tests
+          echo "test_net_chaos"      >> ./CI/tests/my_tests
+          echo "test_time"           >> ./CI/tests/my_tests
+
+      - name: Print affected config files
+        run: |
+          echo -e "## CI/config/common_test_config.yaml\n\n"
+          cat CI/config/common_test_config.yaml
+          echo -e "## CI/scenarios/ingress_namespace.yaml\n\n"
+          cat CI/scenarios/ingress_namespace.yaml
+
+      - name: Running test suite
+        run: |
+          ./CI/run.sh
+      - name: Print test output
+        run:  cat CI/out/*
+      - name: Create coverage report
+        run: |
+          echo "# Test results" > $GITHUB_STEP_SUMMARY
+          cat CI/results.markdown >> $GITHUB_STEP_SUMMARY
+          echo "# Test coverage" >> $GITHUB_STEP_SUMMARY
+          python -m coverage report --format=markdown >> $GITHUB_STEP_SUMMARY 
+
+
@@ -0,0 +1,44 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: $NAMESPACE
+
+---
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: $DEPLOYMENT_NAME-service
+  namespace: $NAMESPACE
+spec:
+  selector:
+    app: $DEPLOYMENT_NAME
+  ports:
+    - name: http
+      port: 80
+      targetPort: 8080
+  type: ClusterIP
+
+--- 
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  namespace: $NAMESPACE
+  name: $DEPLOYMENT_NAME-deployment
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: $DEPLOYMENT_NAME
+  template:
+    metadata:
+      labels:
+        app: $DEPLOYMENT_NAME
+    spec:
+      containers:
+        - name: $DEPLOYMENT_NAME
+          image: nginxinc/nginx-unprivileged:stable-alpine
+          ports:
+            - name: http
+              containerPort: 8080
@@ -0,0 +1,72 @@
+#!/bin/bash
+SCRIPT_PATH=./CI/CRC
+DEPLOYMENT_PATH=$SCRIPT_PATH/deployment.yaml
+CLUSTER_INFO=cluster_infos.json
+
+[[ -z $WORKDIR_PATH ]] && echo "[ERROR] please set \$WORKDIR_PATH environment variable" && exit 1
+CLUSTER_INFO_PATH=$WORKDIR_PATH/crc/$CLUSTER_INFO
+
+[[ ! -f $DEPLOYMENT_PATH ]] && echo "[ERROR] please run $0 from GitHub action root directory" && exit 1
+[[ -z $KUBEADMIN_PWD ]] && echo "[ERROR] kubeadmin password not set, please check the repository secrets" && exit 1
+[[ -z $DEPLOYMENT_NAME ]] && echo "[ERROR] please set \$DEPLOYMENT_NAME environment variable" && exit 1
+[[ -z $NAMESPACE ]] && echo "[ERROR] please set \$NAMESPACE environment variable" && exit 1
+[[ ! -f $CLUSTER_INFO_PATH ]] && echo  "[ERROR] cluster_info.json not found in $CLUSTER_INFO_PATH" && exit 1 
+
+OPENSSL=`which openssl 2>/dev/null`
+[[ $? != 0 ]] && echo "[ERROR]: openssl missing, please install it and try again" && exit 1
+OC=`which oc 2>/dev/null`
+[[ $? != 0 ]] && echo "[ERROR]: oc missing, please install it and try again" && exit 1
+SED=`which sed 2>/dev/null`
+[[ $? != 0 ]] && echo "[ERROR]: sed missing, please install it and try again" && exit 1
+JQ=`which jq 2>/dev/null`
+[[ $? != 0 ]] && echo "[ERROR]: jq missing, please install it and try again" && exit 1
+ENVSUBST=`which envsubst 2>/dev/null`
+[[ $? != 0 ]] && echo "[ERROR]: envsubst missing, please install it and try again" && exit 1
+
+API_ADDRESS="$($JQ -r '.api.address' $CLUSTER_INFO_PATH)"
+API_PORT="$($JQ -r '.api.port' $CLUSTER_INFO_PATH)"
+BASE_HOST=`$JQ -r '.api.address'  $CLUSTER_INFO_PATH | sed -r 's#https:\/\/api\.(.+\.nip\.io)#\1#'`
+FQN=$DEPLOYMENT_NAME.apps.$BASE_HOST
+
+echo "[INF] logging on $API_ADDRESS:$API_PORT"
+COUNTER=1
+until `$OC login  --insecure-skip-tls-verify -u kubeadmin -p $KUBEADMIN_PWD $API_ADDRESS:$API_PORT > /dev/null 2>&1`
+do 
+    echo "[INF] login attempt $COUNTER"
+    [[ $COUNTER == 20 ]] && echo "[ERR] maximum login attempts exceeded, failing" && exit 1
+    ((COUNTER++))
+    sleep 10
+done
+
+echo "[INF] deploying example deployment: $DEPLOYMENT_NAME in namespace: $NAMESPACE"
+$ENVSUBST < $DEPLOYMENT_PATH | $OC apply -f - > /dev/null 2>&1
+
+echo "[INF] creating SSL self-signed certificates for route https://$FQN"
+$OPENSSL genrsa -out servercakey.pem > /dev/null 2>&1
+$OPENSSL req -new -x509 -key servercakey.pem -out serverca.crt -subj "/CN=$FQN/O=Red Hat Inc./C=US" > /dev/null 2>&1
+$OPENSSL genrsa -out server.key > /dev/null 2>&1
+$OPENSSL req -new -key server.key -out server_reqout.txt -subj "/CN=$FQN/O=Red Hat Inc./C=US" > /dev/null 2>&1
+$OPENSSL x509 -req -in server_reqout.txt -days 3650 -sha256 -CAcreateserial -CA serverca.crt -CAkey servercakey.pem -out server.crt > /dev/null 2>&1
+echo "[INF] creating deployment: $DEPLOYMENT_NAME public route: https://$FQN"
+$OC create route --namespace $NAMESPACE  edge --service=$DEPLOYMENT_NAME-service --cert=server.crt --key=server.key --ca-cert=serverca.crt --hostname="$FQN" > /dev/null 2>&1
+
+
+echo "[INF] setting github action environment variables"
+
+NODE_NAME="`$OC get nodes  -o json | $JQ -r '.items[0].metadata.name'`"
+COVERAGE_FILE="`pwd`/coverage.md"
+echo "DEPLOYMENT_NAME=$DEPLOYMENT_NAME" >> $GITHUB_ENV
+echo "DEPLOYMENT_FQN=$FQN" >> $GITHUB_ENV
+echo "API_ADDRESS=$API_ADDRESS" >> $GITHUB_ENV
+echo "API_PORT=$API_PORT" >> $GITHUB_ENV
+echo "NODE_NAME=$NODE_NAME" >> $GITHUB_ENV
+echo "NAMESPACE=$NAMESPACE" >> $GITHUB_ENV
+echo "COVERAGE_FILE=$COVERAGE_FILE" >> $GITHUB_ENV
+
+echo "[INF] deployment fully qualified name will be available in \${{ env.DEPLOYMENT_NAME }} with value $DEPLOYMENT_NAME"
+echo "[INF] deployment name will be available in \${{ env.DEPLOYMENT_FQN }} with value $FQN"
+echo "[INF] OCP API address will be available in \${{ env.API_ADDRESS }} with value $API_ADDRESS"
+echo "[INF] OCP API port will be available in \${{ env.API_PORT }} with value $API_PORT"
+echo "[INF] OCP node name will be available in \${{ env.NODE_NAME }} with value $NODE_NAME"
+echo "[INF] coverage file will ve available in \${{ env.COVERAGE_FILE }} with value $COVERAGE_FILE"
+
@@ -1,5 +1,22 @@
 #!/bin/bash
 set -x
+MAX_RETRIES=60
+
+OC=`which oc 2>/dev/null`
+[[ $? != 0 ]] && echo "[ERROR]: oc missing, please install it and try again" && exit 1
+
+wait_cluster_become_ready() {
+  COUNT=1
+  until `$OC get namespace > /dev/null 2>&1`
+  do
+    echo "[INF] waiting OpenShift to become ready, after $COUNT check"
+    sleep 3
+    [[ $COUNT == $MAX_RETRIES ]] && echo "[ERR] max retries exceeded, failing" && exit 1
+    ((COUNT++))
+  done 
+}
+
+

 ci_tests_loc="CI/tests/my_tests"

@@ -22,5 +39,7 @@ echo '-----------------------|--------|---------' >> $results
 # Run each test
 for test_name in `cat CI/tests/my_tests`
 do
+  wait_cluster_become_ready
  ./CI/run_test.sh $test_name $results
+  wait_cluster_become_ready
 done
@@ -1,31 +0,0 @@
---
-kind: Pod
-apiVersion: v1
-metadata:
-  name: hello-pod
-  creationTimestamp:
-  labels:
-    name: hello-openshift
-spec:
-  containers:
-  - name: hello-openshift
-    image: openshift/hello-openshift
-    ports:
-    - containerPort: 5050
-      protocol: TCP
-    resources: {}
-    volumeMounts:
-    - name: tmp
-      mountPath: "/tmp"
-    terminationMessagePath: "/dev/termination-log"
-    imagePullPolicy: IfNotPresent
-    securityContext:
-      capabilities: {}
-      privileged: false
-  volumes:
-  - name: tmp
-    emptyDir: {}
-  restartPolicy: Always
-  dnsPolicy: ClusterFirst
-  serviceAccount: ''
-status: {}
@@ -1,6 +0,0 @@
-# yaml-language-server: $schema=../../scenarios/plugin.schema.json
- id: kill-pods
-  config:
-    label_selector: name=hello-openshift
-    namespace_pattern: ^default$
-    kill: 1
@@ -1,7 +0,0 @@
-scenarios:
- action: delete
-  namespace: "^.*ingress.*$"
-  label_selector:
-  runs: 1
-  sleep: 15
-  wait_time: 30
@@ -0,0 +1,7 @@
+scenarios:
+  - action: delete
+    namespace: "^$openshift-network-diagnostics$"
+    label_selector:
+    runs: 1
+    sleep: 15
+    wait_time: 30
@@ -0,0 +1,21 @@
+set -xeEo pipefail
+
+source CI/tests/common.sh
+
+trap error ERR
+trap finish EXIT
+
+
+function functional_test_app_outage {
+  [ -z $DEPLOYMENT_NAME ] && echo "[ERR] DEPLOYMENT_NAME variable not set, failing." && exit 1
+  yq -i '.application_outage.pod_selector={"app":"'$DEPLOYMENT_NAME'"}' CI/scenarios/app_outage.yaml
+  yq -i '.application_outage.namespace="'$NAMESPACE'"' CI/scenarios/app_outage.yaml
+  export scenario_type="application_outages"
+  export scenario_file="CI/scenarios/app_outage.yaml"
+  export post_config=""
+  envsubst < CI/config/common_test_config.yaml > CI/config/app_outage.yaml
+  python3 -m coverage run -a run_kraken.py -c CI/config/app_outage.yaml
+  echo "App outage scenario test: Success"
+}
+
+functional_test_app_outage
@@ -0,0 +1,20 @@
+set -xeEo pipefail
+
+source CI/tests/common.sh
+
+trap error ERR
+trap finish EXIT
+
+
+function functional_test_litmus_cpu {
+  [ -z $NODE_NAME ] && echo "[ERR] NODE_NAME variable not set, failing." && exit 1
+  yq -i ' .spec.experiments = [{"name": "node-cpu-hog", "spec":{"components":{"env":[{"name":"TOTAL_CHAOS_DURATION","value":"10"},{"name":"NODE_CPU_CORE","value":"1"},{"name":"NODES_AFFECTED_PERC","value":"30"},{"name":"TARGET_NODES","value":"'$NODE_NAME'"}]}}}]' CI/scenarios/node_cpu_hog_engine_node.yaml
+
+  cp CI/config/common_test_config.yaml CI/config/litmus_config.yaml
+  yq '.kraken.chaos_scenarios = [{"litmus_scenarios":[["scenarios/openshift/templates/litmus-rbac.yaml","CI/scenarios/node_cpu_hog_engine_node.yaml"]]}]' -i CI/config/litmus_config.yaml
+  
+  python3 -m coverage run -a run_kraken.py -c CI/config/litmus_config.yaml
+  echo "Litmus scenario test: Success"
+}
+
+functional_test_litmus_cpu
@@ -0,0 +1,19 @@
+set -xeEo pipefail
+
+source CI/tests/common.sh
+
+trap error ERR
+trap finish EXIT
+
+
+function functional_test_litmus_io {
+  [ -z $NODE_NAME ] && echo "[ERR] NODE_NAME variable not set, failing." && exit 1
+  yq -i ' .spec.experiments = [{"name": "node-io-stress", "spec":{"components":{"env":[{"name":"TOTAL_CHAOS_DURATION","value":"10"},{"name":"FILESYSTEM_UTILIZATION_PERCENTAGE","value":"100"},{"name":"CPU","value":"1"},{"name":"NUMBER_OF_WORKERS","value":"3"},{"name":"TARGET_NODES","value":"'$NODE_NAME'"}]}}}]' CI/scenarios/node_io_engine_node.yaml
+  cp CI/config/common_test_config.yaml CI/config/litmus_config.yaml
+  yq '.kraken.chaos_scenarios = [{"litmus_scenarios":[["scenarios/openshift/templates/litmus-rbac.yaml","CI/scenarios/node_io_engine_node.yaml"]]}]' -i CI/config/litmus_config.yaml
+  
+  python3 -m coverage run -a run_kraken.py -c CI/config/litmus_config.yaml
+  echo "Litmus scenario test: Success"
+}
+
+functional_test_litmus_io
@@ -0,0 +1,19 @@
+set -xeEo pipefail
+
+source CI/tests/common.sh
+
+trap error ERR
+trap finish EXIT
+
+
+function functional_test_litmus_mem {
+  [ -z $NODE_NAME ] && echo "[ERR] NODE_NAME variable not set, failing." && exit 1
+  yq -i ' .spec.experiments = [{"name": "node-io-stress", "spec":{"components":{"env":[{"name":"TOTAL_CHAOS_DURATION","value":"10"},{"name":"CPU","value":"1"},{"name":"TARGET_NODES","value":"'$NODE_NAME'"}]}}}]' CI/scenarios/node_mem_engine_node.yaml
+  cp CI/config/common_test_config.yaml CI/config/litmus_config.yaml
+  yq '.kraken.chaos_scenarios = [{"litmus_scenarios":[["scenarios/openshift/templates/litmus-rbac.yaml","CI/scenarios/node_mem_engine_node.yaml"]]}]' -i CI/config/litmus_config.yaml
+
+  python3 -m coverage run -a run_kraken.py -c CI/config/litmus_config.yaml
+  echo "Litmus scenario test: Success"
+}
+
+functional_test_litmus_mem
@@ -7,10 +7,10 @@ trap finish EXIT

 function funtional_test_namespace_deletion {
  export scenario_type="namespace_scenarios"
-  export scenario_file="-  CI/scenarios/ingress_namespace.yaml"
+  export scenario_file="-  CI/scenarios/network_diagnostics_namespace.yaml"
  export post_config=""
+  yq '.scenarios.[0].namespace="^openshift-network-diagnostics$"' -i CI/scenarios/network_diagnostics_namespace.yaml
  envsubst < CI/config/common_test_config.yaml > CI/config/namespace_config.yaml
-
  python3 -m coverage run -a run_kraken.py -c CI/config/namespace_config.yaml
  echo $?
  echo "Namespace scenario test: Success"
@@ -1,19 +0,0 @@
-set -xeEo pipefail
-
-source CI/tests/common.sh
-
-trap error ERR
-trap finish EXIT
-
-function funtional_test_pod_deletion {
-  export scenario_type="pod_scenarios"
-  export scenario_file="-  CI/scenarios/hello_pod_killing.yml"
-  export post_config=""
-  envsubst < CI/config/common_test_config.yaml > CI/config/pod_config.yaml
-
-  python3 -m coverage run -a run_kraken.py -c CI/config/pod_config.yaml
-  echo $?
-  echo "Pod scenario test: Success"
-}
-
-funtional_test_pod_deletion