From a02ae67559b438ca03617a52718e781185db1661 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Evrard Date: Wed, 2 Oct 2024 23:26:33 +0200 Subject: [PATCH] Accelerate CI jobs Without this, some CI jobs are flaky or slow due to the following issues: - Triggering a reboot cause an unrecoverable boot loop. This fixes it by restarting the containers that are incorrectly exited. - API server is down while operations happen. This fixes it by ensuring at least one API server is up. In this case, we don't add a reboot marker on the unique api server. - The amount of nodes in a test environment is larger than necessary. This fixes it by ensuring two nodes are required to reboot. This is enough for concurrency, and for the e2e testing. - The wait time between operations is high, and can cause a heartbeat to be missed in the check script. This fixes it by checking more often, at the expense of more logging. This is compensated by increasing the amount of tries. Signed-off-by: Jean-Philippe Evrard --- .github/kind-cluster-1.28.yaml | 4 ---- .github/kind-cluster-1.29.yaml | 4 ---- .github/kind-cluster-1.30.yaml | 4 ---- .github/workflows/on-pr.yaml | 16 ++++++++-------- tests/kind/create-reboot-sentinels.sh | 5 +++-- tests/kind/follow-coordinated-reboot.sh | 17 +++++++++++++---- 6 files changed, 24 insertions(+), 26 deletions(-) diff --git a/.github/kind-cluster-1.28.yaml b/.github/kind-cluster-1.28.yaml index 11a9b99..9eda6de 100644 --- a/.github/kind-cluster-1.28.yaml +++ b/.github/kind-cluster-1.28.yaml @@ -1,10 +1,6 @@ kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 nodes: -- role: control-plane - image: "kindest/node:v1.28.9" -- role: control-plane - image: "kindest/node:v1.28.9" - role: control-plane image: "kindest/node:v1.28.9" - role: worker diff --git a/.github/kind-cluster-1.29.yaml b/.github/kind-cluster-1.29.yaml index 492374a..69eb6b7 100644 --- a/.github/kind-cluster-1.29.yaml +++ b/.github/kind-cluster-1.29.yaml @@ -1,10 +1,6 @@ kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 nodes: -- role: control-plane - image: "kindest/node:v1.29.4" -- role: control-plane - image: "kindest/node:v1.29.4" - role: control-plane image: "kindest/node:v1.29.4" - role: worker diff --git a/.github/kind-cluster-1.30.yaml b/.github/kind-cluster-1.30.yaml index d926e2a..eaf9d9c 100644 --- a/.github/kind-cluster-1.30.yaml +++ b/.github/kind-cluster-1.30.yaml @@ -1,10 +1,6 @@ kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 nodes: -- role: control-plane - image: "kindest/node:v1.30.2" -- role: control-plane - image: "kindest/node:v1.30.2" - role: control-plane image: "kindest/node:v1.30.2" - role: worker diff --git a/.github/workflows/on-pr.yaml b/.github/workflows/on-pr.yaml index a56a436..7b97533 100644 --- a/.github/workflows/on-pr.yaml +++ b/.github/workflows/on-pr.yaml @@ -145,7 +145,7 @@ jobs: EOF # Default name for helm/kind-action kind clusters is "chart-testing" - - name: Create kind cluster with 5 nodes + - name: Create kind cluster with 3 nodes uses: helm/kind-action@v1.10.0 with: config: .github/kind-cluster-${{ matrix.kubernetes }}.yaml @@ -169,7 +169,7 @@ jobs: max_attempts: 10 retry_wait_seconds: 60 # DESIRED CURRENT READY UP-TO-DATE AVAILABLE should all be = to cluster_size - command: "kubectl get ds -n kube-system kured | grep -E 'kured.*5.*5.*5.*5.*5'" + command: "kubectl get ds -n kube-system kured | grep -E 'kured.*3.*3.*3.*3.*3'" - name: Create reboot sentinel files run: | @@ -230,7 +230,7 @@ jobs: EOF # Default name for helm/kind-action kind clusters is "chart-testing" - - name: Create kind cluster with 5 nodes + - name: Create kind cluster with 3 nodes uses: helm/kind-action@v1.10.0 with: config: .github/kind-cluster-${{ matrix.kubernetes }}.yaml @@ -241,7 +241,7 @@ jobs: - name: Do not wait for an hour before detecting the rebootSentinel run: | - sed -i 's/#\(.*\)--period=1h/\1--period=30s/g' kured-ds-signal.yaml + sed -i 's/#\(.*\)--period=1h/\1--period=15s/g' kured-ds-signal.yaml - name: Install kured with kubectl run: | @@ -254,7 +254,7 @@ jobs: max_attempts: 10 retry_wait_seconds: 60 # DESIRED CURRENT READY UP-TO-DATE AVAILABLE should all be = to cluster_size - command: "kubectl get ds -n kube-system kured | grep -E 'kured.*5.*5.*5.*5.*5'" + command: "kubectl get ds -n kube-system kured | grep -E 'kured.*3.*3.*3.*3.*3'" - name: Create reboot sentinel files run: | @@ -316,7 +316,7 @@ jobs: EOF # Default name for helm/kind-action kind clusters is "chart-testing" - - name: Create kind cluster with 5 nodes + - name: Create kind cluster with 3 nodes uses: helm/kind-action@v1.10.0 with: config: .github/kind-cluster-${{ matrix.kubernetes }}.yaml @@ -327,7 +327,7 @@ jobs: - name: Do not wait for an hour before detecting the rebootSentinel run: | - sed -i 's/#\(.*\)--period=1h/\1--period=30s/g' kured-ds.yaml + sed -i 's/#\(.*\)--period=1h/\1--period=15s/g' kured-ds.yaml sed -i 's/#\(.*\)--concurrency=1/\1--concurrency=2/g' kured-ds.yaml - name: Install kured with kubectl @@ -341,7 +341,7 @@ jobs: max_attempts: 10 retry_wait_seconds: 60 # DESIRED CURRENT READY UP-TO-DATE AVAILABLE should all be = to cluster_size - command: "kubectl get ds -n kube-system kured | grep -E 'kured.*5.*5.*5.*5.*5'" + command: "kubectl get ds -n kube-system kured | grep -E 'kured.*3.*3.*3.*3.*3'" - name: Create reboot sentinel files run: | diff --git a/tests/kind/create-reboot-sentinels.sh b/tests/kind/create-reboot-sentinels.sh index e95dc3b..51bd127 100755 --- a/tests/kind/create-reboot-sentinels.sh +++ b/tests/kind/create-reboot-sentinels.sh @@ -4,9 +4,10 @@ KUBECTL_CMD="${KUBECTL_CMD:-kubectl}" SENTINEL_FILE="${SENTINEL_FILE:-/var/run/reboot-required}" -echo "Creating reboot sentinel on all nodes" +echo "Creating reboot sentinel on worker nodes" -for nodename in $("$KUBECTL_CMD" get nodes -o name); do +# To speed up the system, let's not kill the control plane. +for nodename in $("$KUBECTL_CMD" get nodes -o name | grep -v control-plane); do docker exec "${nodename/node\//}" hostname docker exec "${nodename/node\//}" touch "${SENTINEL_FILE}" done diff --git a/tests/kind/follow-coordinated-reboot.sh b/tests/kind/follow-coordinated-reboot.sh index e5885fd..4559ec1 100755 --- a/tests/kind/follow-coordinated-reboot.sh +++ b/tests/kind/follow-coordinated-reboot.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -NODECOUNT=${NODECOUNT:-5} +NODECOUNT=${NODECOUNT:-2} KUBECTL_CMD="${KUBECTL_CMD:-kubectl}" DEBUG="${DEBUG:-false}" CONTAINER_NAME_FORMAT=${CONTAINER_NAME_FORMAT:-"chart-testing-*"} @@ -35,10 +35,12 @@ trap gather_logs_and_cleanup EXIT declare -A was_unschedulable declare -A has_recovered -max_attempts="60" -sleep_time=60 +max_attempts="200" +sleep_time=5 attempt_num=1 +# Get docker info of each of those kind containers. If one has crashed, restart it. + set +o errexit echo "There are $NODECOUNT nodes in the cluster" until [ ${#was_unschedulable[@]} == "$NODECOUNT" ] && [ ${#has_recovered[@]} == "$NODECOUNT" ] @@ -52,13 +54,14 @@ do # cat "$tmp_dir"/node_output #fi - "$KUBECTL_CMD" get nodes -o custom-columns=NAME:.metadata.name,SCHEDULABLE:.spec.unschedulable --no-headers > "$tmp_dir"/node_output + "$KUBECTL_CMD" get nodes -o custom-columns=NAME:.metadata.name,SCHEDULABLE:.spec.unschedulable --no-headers | grep -v control-plane > "$tmp_dir"/node_output if [[ "$DEBUG" == "true" ]]; then # This is useful to see if a node gets stuck after drain, and doesn't # come back up. echo "Result of command $KUBECTL_CMD get nodes ... showing unschedulable nodes:" cat "$tmp_dir"/node_output fi + while read -r node; do unschedulable=$(echo "$node" | grep true | cut -f 1 -d ' ') if [ -n "$unschedulable" ] && [ -z ${was_unschedulable["$unschedulable"]+x} ] ; then @@ -70,6 +73,12 @@ do echo "$schedulable has recovered!" has_recovered["$schedulable"]=1 fi + + # If the container has crashed, restart it. + node_name=$(echo "$node" | cut -f 1 -d ' ') + stopped_container_id=$(docker container ls --filter=name="$node_name" --filter=status=exited -q) + if [ -n "$stopped_container_id" ]; then echo "Node $stopped_container_id needs restart"; docker start "$stopped_container_id"; echo "Container started."; fi + done < "$tmp_dir"/node_output if [[ "${#has_recovered[@]}" == "$NODECOUNT" ]]; then