github/kured
1
0
Fork 0
mirror of https://github.com/kubereboot/kured.git synced 2026-08-23 21:36:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

Accelerate CI jobs

Browse Source 
Without this, some CI jobs are flaky or slow due to the following
issues:
- Triggering a reboot cause an unrecoverable boot loop.
  This fixes it by restarting the containers that are incorrectly
  exited.
- API server is down while operations happen.
  This fixes it by ensuring at least one API server is up. In this
  case, we don't add a reboot marker on the unique api server.
- The amount of nodes in a test environment is larger than
  necessary.
  This fixes it by ensuring two nodes are required to reboot.
  This is enough for concurrency, and for the e2e testing.
- The wait time between operations is high, and can cause
  a heartbeat to be missed in the check script.
  This fixes it by checking more often, at the expense of
  more logging. This is compensated by increasing the amount
  of tries.

Signed-off-by: Jean-Philippe Evrard <open-source@a.spamming.party>
This commit is contained in:
Jean-Philippe Evrard
2024-10-02 23:30:41 +02:00
parent 5536bf7e30
commit a02ae67559
6 changed files with 24 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/kind-cluster-1.28.yaml
-4
View File
@@ -1,10 +1,6 @@
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
nodes:
- role: control-plane
image: "kindest/node:v1.28.9"
- role: control-plane
image: "kindest/node:v1.28.9"
- role: control-plane
image: "kindest/node:v1.28.9"
- role: worker
.github/kind-cluster-1.29.yaml
-4
View File
@@ -1,10 +1,6 @@
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
nodes:
- role: control-plane
image: "kindest/node:v1.29.4"
- role: control-plane
image: "kindest/node:v1.29.4"
- role: control-plane
image: "kindest/node:v1.29.4"
- role: worker
.github/kind-cluster-1.30.yaml
-4
View File
@@ -1,10 +1,6 @@
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
nodes:
- role: control-plane
image: "kindest/node:v1.30.2"
- role: control-plane
image: "kindest/node:v1.30.2"
- role: control-plane
image: "kindest/node:v1.30.2"
- role: worker
.github/workflows/on-pr.yaml
+8 -8
View File
@@ -145,7 +145,7 @@ jobs:
EOF
# Default name for helm/kind-action kind clusters is "chart-testing"
- name: Create kind cluster with 5 nodes
- name: Create kind cluster with 3 nodes
uses: helm/kind-action@v1.10.0
with:
config: .github/kind-cluster-${{ matrix.kubernetes }}.yaml
@@ -169,7 +169,7 @@ jobs:
max_attempts: 10
retry_wait_seconds: 60
# DESIRED CURRENT READY UP-TO-DATE AVAILABLE should all be = to cluster_size
command: "kubectl get ds -n kube-system kured | grep -E 'kured.*5.*5.*5.*5.*5'"
command: "kubectl get ds -n kube-system kured | grep -E 'kured.*3.*3.*3.*3.*3'"
- name: Create reboot sentinel files
run: |
@@ -230,7 +230,7 @@ jobs:
EOF
# Default name for helm/kind-action kind clusters is "chart-testing"
- name: Create kind cluster with 5 nodes
- name: Create kind cluster with 3 nodes
uses: helm/kind-action@v1.10.0
with:
config: .github/kind-cluster-${{ matrix.kubernetes }}.yaml
@@ -241,7 +241,7 @@ jobs:
- name: Do not wait for an hour before detecting the rebootSentinel
run: |
sed -i 's/#\(.*\)--period=1h/\1--period=30s/g' kured-ds-signal.yaml
sed -i 's/#\(.*\)--period=1h/\1--period=15s/g' kured-ds-signal.yaml
- name: Install kured with kubectl
run: |
@@ -254,7 +254,7 @@ jobs:
max_attempts: 10
retry_wait_seconds: 60
# DESIRED CURRENT READY UP-TO-DATE AVAILABLE should all be = to cluster_size
command: "kubectl get ds -n kube-system kured | grep -E 'kured.*5.*5.*5.*5.*5'"
command: "kubectl get ds -n kube-system kured | grep -E 'kured.*3.*3.*3.*3.*3'"
- name: Create reboot sentinel files
run: |
@@ -316,7 +316,7 @@ jobs:
EOF
# Default name for helm/kind-action kind clusters is "chart-testing"
- name: Create kind cluster with 5 nodes
- name: Create kind cluster with 3 nodes
uses: helm/kind-action@v1.10.0
with:
config: .github/kind-cluster-${{ matrix.kubernetes }}.yaml
@@ -327,7 +327,7 @@ jobs:
- name: Do not wait for an hour before detecting the rebootSentinel
run: |
sed -i 's/#\(.*\)--period=1h/\1--period=30s/g' kured-ds.yaml
sed -i 's/#\(.*\)--period=1h/\1--period=15s/g' kured-ds.yaml
sed -i 's/#\(.*\)--concurrency=1/\1--concurrency=2/g' kured-ds.yaml
- name: Install kured with kubectl
@@ -341,7 +341,7 @@ jobs:
max_attempts: 10
retry_wait_seconds: 60
# DESIRED CURRENT READY UP-TO-DATE AVAILABLE should all be = to cluster_size
command: "kubectl get ds -n kube-system kured | grep -E 'kured.*5.*5.*5.*5.*5'"
command: "kubectl get ds -n kube-system kured | grep -E 'kured.*3.*3.*3.*3.*3'"
- name: Create reboot sentinel files
run: |
tests/kind/create-reboot-sentinels.sh
+3 -2
View File
@@ -4,9 +4,10 @@
KUBECTL_CMD="${KUBECTL_CMD:-kubectl}"
SENTINEL_FILE="${SENTINEL_FILE:-/var/run/reboot-required}"
echo "Creating reboot sentinel on all nodes"
echo "Creating reboot sentinel on worker nodes"
for nodename in $("$KUBECTL_CMD" get nodes -o name); do
# To speed up the system, let's not kill the control plane.
for nodename in $("$KUBECTL_CMD" get nodes -o name | grep -v control-plane); do
docker exec "${nodename/node\//}" hostname
docker exec "${nodename/node\//}" touch "${SENTINEL_FILE}"
done
tests/kind/follow-coordinated-reboot.sh
+13 -4
View File
@@ -1,6 +1,6 @@
#!/usr/bin/env bash
NODECOUNT=${NODECOUNT:-5}
NODECOUNT=${NODECOUNT:-2}
KUBECTL_CMD="${KUBECTL_CMD:-kubectl}"
DEBUG="${DEBUG:-false}"
CONTAINER_NAME_FORMAT=${CONTAINER_NAME_FORMAT:-"chart-testing-*"}
@@ -35,10 +35,12 @@ trap gather_logs_and_cleanup EXIT
declare -A was_unschedulable
declare -A has_recovered
max_attempts="60"
sleep_time=60
max_attempts="200"
sleep_time=5
attempt_num=1
# Get docker info of each of those kind containers. If one has crashed, restart it.
set +o errexit
echo "There are $NODECOUNT nodes in the cluster"
until [ ${#was_unschedulable[@]} == "$NODECOUNT" ] && [ ${#has_recovered[@]} == "$NODECOUNT" ]
@@ -52,13 +54,14 @@ do
# cat "$tmp_dir"/node_output
#fi
"$KUBECTL_CMD" get nodes -o custom-columns=NAME:.metadata.name,SCHEDULABLE:.spec.unschedulable --no-headers > "$tmp_dir"/node_output
"$KUBECTL_CMD" get nodes -o custom-columns=NAME:.metadata.name,SCHEDULABLE:.spec.unschedulable --no-headers | grep -v control-plane > "$tmp_dir"/node_output
if [[ "$DEBUG" == "true" ]]; then
# This is useful to see if a node gets stuck after drain, and doesn't
# come back up.
echo "Result of command $KUBECTL_CMD get nodes ... showing unschedulable nodes:"
cat "$tmp_dir"/node_output
fi
while read -r node; do
unschedulable=$(echo "$node" | grep true | cut -f 1 -d ' ')
if [ -n "$unschedulable" ] && [ -z ${was_unschedulable["$unschedulable"]+x} ] ; then
@@ -70,6 +73,12 @@ do
echo "$schedulable has recovered!"
has_recovered["$schedulable"]=1
fi
# If the container has crashed, restart it.
node_name=$(echo "$node" | cut -f 1 -d ' ')
stopped_container_id=$(docker container ls --filter=name="$node_name" --filter=status=exited -q)
if [ -n "$stopped_container_id" ]; then echo "Node $stopped_container_id needs restart"; docker start "$stopped_container_id"; echo "Container started."; fi
done < "$tmp_dir"/node_output
if [[ "${#has_recovered[@]}" == "$NODECOUNT" ]]; then