diff --git a/.github/workflows/functional_tests.yaml b/.github/workflows/functional_tests.yaml new file mode 100644 index 00000000..4e18cd03 --- /dev/null +++ b/.github/workflows/functional_tests.yaml @@ -0,0 +1,89 @@ +on: issue_comment + +jobs: + pr_commented: + # This job only runs for pull request comments containing /functional + name: Functional Tests + if: contains(github.event.comment.body, '/test') && contains(github.event.comment.html_url, '/pull/') && github.event.requested_team.name == 'redhat-chaos' + runs-on: ubuntu-latest + steps: + - name: Check out Kraken + uses: actions/checkout@v3 + - name: Checkout Pull Request + run: hub pr checkout ${{ github.event.issue.number }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Install OC CLI + uses: redhat-actions/oc-installer@v1 + with: + oc_version: latest + - name: Install python 3.9 + uses: actions/setup-python@v4 + with: + python-version: '3.9' + - name: Setup kraken dependencies + run: pip install -r requirements.txt + - name: Create Workdir & export the path + run: | + mkdir workdir + echo "WORKDIR_PATH=`pwd`/workdir" >> $GITHUB_ENV + - name: Teardown CRC (Post Action) + uses: webiny/action-post-run@3.0.0 + id: post-run-command + with: + # currently using image coming from tsebastiani quay.io repo + # waiting that a fix is merged in the upstream one + # post action run cannot (apparently) be properly indented + run: docker run -v "${{ env.WORKDIR_PATH }}:/workdir" -e WORKING_MODE=T -e AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} -e AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} -e AWS_DEFAULT_REGION=us-west-2 -e TEARDOWN_RUN_ID=crc quay.io/tsebastiani/crc-cloud + - name: Run CRC + # currently using image coming from tsebastiani quay.io repo + # waiting that a fix is merged in the upstream one + run: | + docker run -v "${{ env.WORKDIR_PATH }}:/workdir" \ + -e WORKING_MODE=C \ + -e PULL_SECRET="${{ secrets.PULL_SECRET }}" \ + -e AWS_ACCESS_KEY_ID="${{ secrets.AWS_ACCESS_KEY_ID }}" \ + -e AWS_SECRET_ACCESS_KEY="${{ secrets.AWS_SECRET_ACCESS_KEY }}" \ + -e AWS_DEFAULT_REGION=us-west-2 \ + -e CREATE_RUN_ID=crc \ + -e PASS_KUBEADMIN="${{ secrets.KUBEADMIN_PWD }}" \ + -e PASS_REDHAT="${{ secrets.REDHAT_PWD }}" \ + -e PASS_DEVELOPER="${{ secrets.DEVELOPER_PWD }}" \ + quay.io/tsebastiani/crc-cloud + - name: OpenShift login and example deployment, GitHub Action env init + env: + NAMESPACE: test-namespace + DEPLOYMENT_NAME: test-nginx + KUBEADMIN_PWD: '${{ secrets.KUBEADMIN_PWD }}' + run: ./CI/CRC/init_github_action.sh + - name: Setup test suite + run: | + yq -i '.kraken.port="8081"' CI/config/common_test_config.yaml + yq -i '.kraken.signal_address="0.0.0.0"' CI/config/common_test_config.yaml + + echo "test_app_outages_gh" > ./CI/tests/my_tests + echo "test_container" >> ./CI/tests/my_tests + echo "test_namespace" >> ./CI/tests/my_tests + echo "test_net_chaos" >> ./CI/tests/my_tests + echo "test_time" >> ./CI/tests/my_tests + + - name: Print affected config files + run: | + echo -e "## CI/config/common_test_config.yaml\n\n" + cat CI/config/common_test_config.yaml + echo -e "## CI/scenarios/ingress_namespace.yaml\n\n" + cat CI/scenarios/ingress_namespace.yaml + + - name: Running test suite + run: | + ./CI/run.sh + - name: Print test output + run: cat CI/out/* + - name: Create coverage report + run: | + echo "# Test results" > $GITHUB_STEP_SUMMARY + cat CI/results.markdown >> $GITHUB_STEP_SUMMARY + echo "# Test coverage" >> $GITHUB_STEP_SUMMARY + python -m coverage report --format=markdown >> $GITHUB_STEP_SUMMARY + + diff --git a/CI/CRC/deployment.yaml b/CI/CRC/deployment.yaml new file mode 100644 index 00000000..89eb1d6d --- /dev/null +++ b/CI/CRC/deployment.yaml @@ -0,0 +1,44 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: $NAMESPACE + +--- + +apiVersion: v1 +kind: Service +metadata: + name: $DEPLOYMENT_NAME-service + namespace: $NAMESPACE +spec: + selector: + app: $DEPLOYMENT_NAME + ports: + - name: http + port: 80 + targetPort: 8080 + type: ClusterIP + +--- + +apiVersion: apps/v1 +kind: Deployment +metadata: + namespace: $NAMESPACE + name: $DEPLOYMENT_NAME-deployment +spec: + replicas: 3 + selector: + matchLabels: + app: $DEPLOYMENT_NAME + template: + metadata: + labels: + app: $DEPLOYMENT_NAME + spec: + containers: + - name: $DEPLOYMENT_NAME + image: nginxinc/nginx-unprivileged:stable-alpine + ports: + - name: http + containerPort: 8080 diff --git a/CI/CRC/init_github_action.sh b/CI/CRC/init_github_action.sh new file mode 100755 index 00000000..3e008e41 --- /dev/null +++ b/CI/CRC/init_github_action.sh @@ -0,0 +1,72 @@ +#!/bin/bash +SCRIPT_PATH=./CI/CRC +DEPLOYMENT_PATH=$SCRIPT_PATH/deployment.yaml +CLUSTER_INFO=cluster_infos.json + +[[ -z $WORKDIR_PATH ]] && echo "[ERROR] please set \$WORKDIR_PATH environment variable" && exit 1 +CLUSTER_INFO_PATH=$WORKDIR_PATH/crc/$CLUSTER_INFO + +[[ ! -f $DEPLOYMENT_PATH ]] && echo "[ERROR] please run $0 from GitHub action root directory" && exit 1 +[[ -z $KUBEADMIN_PWD ]] && echo "[ERROR] kubeadmin password not set, please check the repository secrets" && exit 1 +[[ -z $DEPLOYMENT_NAME ]] && echo "[ERROR] please set \$DEPLOYMENT_NAME environment variable" && exit 1 +[[ -z $NAMESPACE ]] && echo "[ERROR] please set \$NAMESPACE environment variable" && exit 1 +[[ ! -f $CLUSTER_INFO_PATH ]] && echo "[ERROR] cluster_info.json not found in $CLUSTER_INFO_PATH" && exit 1 + +OPENSSL=`which openssl 2>/dev/null` +[[ $? != 0 ]] && echo "[ERROR]: openssl missing, please install it and try again" && exit 1 +OC=`which oc 2>/dev/null` +[[ $? != 0 ]] && echo "[ERROR]: oc missing, please install it and try again" && exit 1 +SED=`which sed 2>/dev/null` +[[ $? != 0 ]] && echo "[ERROR]: sed missing, please install it and try again" && exit 1 +JQ=`which jq 2>/dev/null` +[[ $? != 0 ]] && echo "[ERROR]: jq missing, please install it and try again" && exit 1 +ENVSUBST=`which envsubst 2>/dev/null` +[[ $? != 0 ]] && echo "[ERROR]: envsubst missing, please install it and try again" && exit 1 + +API_ADDRESS="$($JQ -r '.api.address' $CLUSTER_INFO_PATH)" +API_PORT="$($JQ -r '.api.port' $CLUSTER_INFO_PATH)" +BASE_HOST=`$JQ -r '.api.address' $CLUSTER_INFO_PATH | sed -r 's#https:\/\/api\.(.+\.nip\.io)#\1#'` +FQN=$DEPLOYMENT_NAME.apps.$BASE_HOST + +echo "[INF] logging on $API_ADDRESS:$API_PORT" +COUNTER=1 +until `$OC login --insecure-skip-tls-verify -u kubeadmin -p $KUBEADMIN_PWD $API_ADDRESS:$API_PORT > /dev/null 2>&1` +do + echo "[INF] login attempt $COUNTER" + [[ $COUNTER == 20 ]] && echo "[ERR] maximum login attempts exceeded, failing" && exit 1 + ((COUNTER++)) + sleep 10 +done + +echo "[INF] deploying example deployment: $DEPLOYMENT_NAME in namespace: $NAMESPACE" +$ENVSUBST < $DEPLOYMENT_PATH | $OC apply -f - > /dev/null 2>&1 + +echo "[INF] creating SSL self-signed certificates for route https://$FQN" +$OPENSSL genrsa -out servercakey.pem > /dev/null 2>&1 +$OPENSSL req -new -x509 -key servercakey.pem -out serverca.crt -subj "/CN=$FQN/O=Red Hat Inc./C=US" > /dev/null 2>&1 +$OPENSSL genrsa -out server.key > /dev/null 2>&1 +$OPENSSL req -new -key server.key -out server_reqout.txt -subj "/CN=$FQN/O=Red Hat Inc./C=US" > /dev/null 2>&1 +$OPENSSL x509 -req -in server_reqout.txt -days 3650 -sha256 -CAcreateserial -CA serverca.crt -CAkey servercakey.pem -out server.crt > /dev/null 2>&1 +echo "[INF] creating deployment: $DEPLOYMENT_NAME public route: https://$FQN" +$OC create route --namespace $NAMESPACE edge --service=$DEPLOYMENT_NAME-service --cert=server.crt --key=server.key --ca-cert=serverca.crt --hostname="$FQN" > /dev/null 2>&1 + + +echo "[INF] setting github action environment variables" + +NODE_NAME="`$OC get nodes -o json | $JQ -r '.items[0].metadata.name'`" +COVERAGE_FILE="`pwd`/coverage.md" +echo "DEPLOYMENT_NAME=$DEPLOYMENT_NAME" >> $GITHUB_ENV +echo "DEPLOYMENT_FQN=$FQN" >> $GITHUB_ENV +echo "API_ADDRESS=$API_ADDRESS" >> $GITHUB_ENV +echo "API_PORT=$API_PORT" >> $GITHUB_ENV +echo "NODE_NAME=$NODE_NAME" >> $GITHUB_ENV +echo "NAMESPACE=$NAMESPACE" >> $GITHUB_ENV +echo "COVERAGE_FILE=$COVERAGE_FILE" >> $GITHUB_ENV + +echo "[INF] deployment fully qualified name will be available in \${{ env.DEPLOYMENT_NAME }} with value $DEPLOYMENT_NAME" +echo "[INF] deployment name will be available in \${{ env.DEPLOYMENT_FQN }} with value $FQN" +echo "[INF] OCP API address will be available in \${{ env.API_ADDRESS }} with value $API_ADDRESS" +echo "[INF] OCP API port will be available in \${{ env.API_PORT }} with value $API_PORT" +echo "[INF] OCP node name will be available in \${{ env.NODE_NAME }} with value $NODE_NAME" +echo "[INF] coverage file will ve available in \${{ env.COVERAGE_FILE }} with value $COVERAGE_FILE" + diff --git a/CI/run.sh b/CI/run.sh index 31ed2f7c..9c00832b 100755 --- a/CI/run.sh +++ b/CI/run.sh @@ -1,5 +1,22 @@ #!/bin/bash set -x +MAX_RETRIES=60 + +OC=`which oc 2>/dev/null` +[[ $? != 0 ]] && echo "[ERROR]: oc missing, please install it and try again" && exit 1 + +wait_cluster_become_ready() { + COUNT=1 + until `$OC get namespace > /dev/null 2>&1` + do + echo "[INF] waiting OpenShift to become ready, after $COUNT check" + sleep 3 + [[ $COUNT == $MAX_RETRIES ]] && echo "[ERR] max retries exceeded, failing" && exit 1 + ((COUNT++)) + done +} + + ci_tests_loc="CI/tests/my_tests" @@ -22,5 +39,7 @@ echo '-----------------------|--------|---------' >> $results # Run each test for test_name in `cat CI/tests/my_tests` do + wait_cluster_become_ready ./CI/run_test.sh $test_name $results + wait_cluster_become_ready done diff --git a/CI/scenarios/hello_pod.yaml b/CI/scenarios/hello_pod.yaml deleted file mode 100644 index 9d029b9f..00000000 --- a/CI/scenarios/hello_pod.yaml +++ /dev/null @@ -1,31 +0,0 @@ ---- -kind: Pod -apiVersion: v1 -metadata: - name: hello-pod - creationTimestamp: - labels: - name: hello-openshift -spec: - containers: - - name: hello-openshift - image: openshift/hello-openshift - ports: - - containerPort: 5050 - protocol: TCP - resources: {} - volumeMounts: - - name: tmp - mountPath: "/tmp" - terminationMessagePath: "/dev/termination-log" - imagePullPolicy: IfNotPresent - securityContext: - capabilities: {} - privileged: false - volumes: - - name: tmp - emptyDir: {} - restartPolicy: Always - dnsPolicy: ClusterFirst - serviceAccount: '' -status: {} diff --git a/CI/scenarios/hello_pod_killing.yml b/CI/scenarios/hello_pod_killing.yml deleted file mode 100755 index 791f8ba8..00000000 --- a/CI/scenarios/hello_pod_killing.yml +++ /dev/null @@ -1,6 +0,0 @@ -# yaml-language-server: $schema=../../scenarios/plugin.schema.json -- id: kill-pods - config: - label_selector: name=hello-openshift - namespace_pattern: ^default$ - kill: 1 diff --git a/CI/scenarios/ingress_namespace.yaml b/CI/scenarios/ingress_namespace.yaml deleted file mode 100644 index 5b1bd112..00000000 --- a/CI/scenarios/ingress_namespace.yaml +++ /dev/null @@ -1,7 +0,0 @@ -scenarios: -- action: delete - namespace: "^.*ingress.*$" - label_selector: - runs: 1 - sleep: 15 - wait_time: 30 diff --git a/CI/scenarios/network_diagnostics_namespace.yaml b/CI/scenarios/network_diagnostics_namespace.yaml new file mode 100644 index 00000000..b8705e62 --- /dev/null +++ b/CI/scenarios/network_diagnostics_namespace.yaml @@ -0,0 +1,7 @@ +scenarios: + - action: delete + namespace: "^$openshift-network-diagnostics$" + label_selector: + runs: 1 + sleep: 15 + wait_time: 30 diff --git a/CI/tests/test_app_outages_gh.sh b/CI/tests/test_app_outages_gh.sh new file mode 100755 index 00000000..dce95e7d --- /dev/null +++ b/CI/tests/test_app_outages_gh.sh @@ -0,0 +1,21 @@ +set -xeEo pipefail + +source CI/tests/common.sh + +trap error ERR +trap finish EXIT + + +function functional_test_app_outage { + [ -z $DEPLOYMENT_NAME ] && echo "[ERR] DEPLOYMENT_NAME variable not set, failing." && exit 1 + yq -i '.application_outage.pod_selector={"app":"'$DEPLOYMENT_NAME'"}' CI/scenarios/app_outage.yaml + yq -i '.application_outage.namespace="'$NAMESPACE'"' CI/scenarios/app_outage.yaml + export scenario_type="application_outages" + export scenario_file="CI/scenarios/app_outage.yaml" + export post_config="" + envsubst < CI/config/common_test_config.yaml > CI/config/app_outage.yaml + python3 -m coverage run -a run_kraken.py -c CI/config/app_outage.yaml + echo "App outage scenario test: Success" +} + +functional_test_app_outage diff --git a/CI/tests/test_cpu_hog_gh.sh b/CI/tests/test_cpu_hog_gh.sh new file mode 100755 index 00000000..53019485 --- /dev/null +++ b/CI/tests/test_cpu_hog_gh.sh @@ -0,0 +1,20 @@ +set -xeEo pipefail + +source CI/tests/common.sh + +trap error ERR +trap finish EXIT + + +function functional_test_litmus_cpu { + [ -z $NODE_NAME ] && echo "[ERR] NODE_NAME variable not set, failing." && exit 1 + yq -i ' .spec.experiments = [{"name": "node-cpu-hog", "spec":{"components":{"env":[{"name":"TOTAL_CHAOS_DURATION","value":"10"},{"name":"NODE_CPU_CORE","value":"1"},{"name":"NODES_AFFECTED_PERC","value":"30"},{"name":"TARGET_NODES","value":"'$NODE_NAME'"}]}}}]' CI/scenarios/node_cpu_hog_engine_node.yaml + + cp CI/config/common_test_config.yaml CI/config/litmus_config.yaml + yq '.kraken.chaos_scenarios = [{"litmus_scenarios":[["scenarios/openshift/templates/litmus-rbac.yaml","CI/scenarios/node_cpu_hog_engine_node.yaml"]]}]' -i CI/config/litmus_config.yaml + + python3 -m coverage run -a run_kraken.py -c CI/config/litmus_config.yaml + echo "Litmus scenario test: Success" +} + +functional_test_litmus_cpu diff --git a/CI/tests/test_io_hog_gh.sh b/CI/tests/test_io_hog_gh.sh new file mode 100755 index 00000000..8fd2b0e1 --- /dev/null +++ b/CI/tests/test_io_hog_gh.sh @@ -0,0 +1,19 @@ +set -xeEo pipefail + +source CI/tests/common.sh + +trap error ERR +trap finish EXIT + + +function functional_test_litmus_io { + [ -z $NODE_NAME ] && echo "[ERR] NODE_NAME variable not set, failing." && exit 1 + yq -i ' .spec.experiments = [{"name": "node-io-stress", "spec":{"components":{"env":[{"name":"TOTAL_CHAOS_DURATION","value":"10"},{"name":"FILESYSTEM_UTILIZATION_PERCENTAGE","value":"100"},{"name":"CPU","value":"1"},{"name":"NUMBER_OF_WORKERS","value":"3"},{"name":"TARGET_NODES","value":"'$NODE_NAME'"}]}}}]' CI/scenarios/node_io_engine_node.yaml + cp CI/config/common_test_config.yaml CI/config/litmus_config.yaml + yq '.kraken.chaos_scenarios = [{"litmus_scenarios":[["scenarios/openshift/templates/litmus-rbac.yaml","CI/scenarios/node_io_engine_node.yaml"]]}]' -i CI/config/litmus_config.yaml + + python3 -m coverage run -a run_kraken.py -c CI/config/litmus_config.yaml + echo "Litmus scenario test: Success" +} + +functional_test_litmus_io diff --git a/CI/tests/test_mem_hog_gh.sh b/CI/tests/test_mem_hog_gh.sh new file mode 100755 index 00000000..c4693139 --- /dev/null +++ b/CI/tests/test_mem_hog_gh.sh @@ -0,0 +1,19 @@ +set -xeEo pipefail + +source CI/tests/common.sh + +trap error ERR +trap finish EXIT + + +function functional_test_litmus_mem { + [ -z $NODE_NAME ] && echo "[ERR] NODE_NAME variable not set, failing." && exit 1 + yq -i ' .spec.experiments = [{"name": "node-io-stress", "spec":{"components":{"env":[{"name":"TOTAL_CHAOS_DURATION","value":"10"},{"name":"CPU","value":"1"},{"name":"TARGET_NODES","value":"'$NODE_NAME'"}]}}}]' CI/scenarios/node_mem_engine_node.yaml + cp CI/config/common_test_config.yaml CI/config/litmus_config.yaml + yq '.kraken.chaos_scenarios = [{"litmus_scenarios":[["scenarios/openshift/templates/litmus-rbac.yaml","CI/scenarios/node_mem_engine_node.yaml"]]}]' -i CI/config/litmus_config.yaml + + python3 -m coverage run -a run_kraken.py -c CI/config/litmus_config.yaml + echo "Litmus scenario test: Success" +} + +functional_test_litmus_mem diff --git a/CI/tests/test_namespace.sh b/CI/tests/test_namespace.sh index c0a711ff..469d0aac 100755 --- a/CI/tests/test_namespace.sh +++ b/CI/tests/test_namespace.sh @@ -7,10 +7,10 @@ trap finish EXIT function funtional_test_namespace_deletion { export scenario_type="namespace_scenarios" - export scenario_file="- CI/scenarios/ingress_namespace.yaml" + export scenario_file="- CI/scenarios/network_diagnostics_namespace.yaml" export post_config="" + yq '.scenarios.[0].namespace="^openshift-network-diagnostics$"' -i CI/scenarios/network_diagnostics_namespace.yaml envsubst < CI/config/common_test_config.yaml > CI/config/namespace_config.yaml - python3 -m coverage run -a run_kraken.py -c CI/config/namespace_config.yaml echo $? echo "Namespace scenario test: Success" diff --git a/CI/tests/test_pods.sh b/CI/tests/test_pods.sh deleted file mode 100755 index 45087c66..00000000 --- a/CI/tests/test_pods.sh +++ /dev/null @@ -1,19 +0,0 @@ -set -xeEo pipefail - -source CI/tests/common.sh - -trap error ERR -trap finish EXIT - -function funtional_test_pod_deletion { - export scenario_type="pod_scenarios" - export scenario_file="- CI/scenarios/hello_pod_killing.yml" - export post_config="" - envsubst < CI/config/common_test_config.yaml > CI/config/pod_config.yaml - - python3 -m coverage run -a run_kraken.py -c CI/config/pod_config.yaml - echo $? - echo "Pod scenario test: Success" -} - -funtional_test_pod_deletion