k3k/pkg/controller/cluster/server/template.go

package server

var StartupCommand string = `
info()
{
    echo "[INFO] [$(date +"%c")]" "$@"
}

fatal()
{
    echo "[FATAL] [$(date +"%c")] " "$@" >&2
    exit 1
}

# safe mode function to reset node IP after pod restarts
safe_mode() {
	CURRENT_IP=""
    if [ -f /var/lib/rancher/k3s/k3k-node-ip ]; then
        CURRENT_IP=$(cat /var/lib/rancher/k3s/k3k-node-ip)
    fi

    if [ -z "$CURRENT_IP" ] || [ "$CURRENT_IP" = "$POD_IP" ] || [ "{{.K3K_MODE}}" != "virtual" ]; then
        return
    fi

	# skipping if the node is starting for the first time
	if [ -d "{{.ETCD_DIR}}" ]; then

		info "Starting K3s in Safe Mode (Network Policy Disabled) to patch Node IP from ${CURRENT_IP} to ${POD_IP}"
		/bin/k3s server --disable-network-policy --config $1 $EXTRA_ARGS > /dev/null 2>&1 &
		PID=$!

		# Start the loop to wait for the nodeIP to change
		info "Waiting for Node IP to update to ${POD_IP}."
		count=0
		until kubectl get nodes -o wide 2>/dev/null | grep -q "${POD_IP}"; do
			if ! kill -0 $PID 2>/dev/null; then
				fatal "safe Mode K3s process died unexpectedly!"
			fi
			sleep 2
			count=$((count+1))

			if [ $count -gt 60 ]; then
				fatal "timed out waiting for node to change IP from $CURRENT_IP to $POD_IP"
			fi
		done
		
		info "Node IP is set to ${POD_IP} successfully. Stopping Safe Mode process..."
		kill $PID
		wait $PID 2>/dev/null || true
	fi
}

start_single_node() {
	info "Starting single node setup..."

	# checking for existing data in single server if found we must perform reset
	if [ -d "{{.ETCD_DIR}}" ]; then
		info "Existing data found in single node setup. Performing cluster-reset to ensure quorum..."

		if ! /bin/k3s server --cluster-reset --config {{.INIT_CONFIG}} $EXTRA_ARGS > /dev/null 2>&1; then
			fatal "cluster reset failed!"
		fi
		info "Cluster reset complete. Removing Reset flag file."
		rm -f /var/lib/rancher/k3s/server/db/reset-flag
	fi

	# entering safe mode to ensure correct NodeIP
	safe_mode {{.INIT_CONFIG}}

	info "Adding pod IP file."
	echo $POD_IP > /var/lib/rancher/k3s/k3k-node-ip

	/bin/k3s server --config {{.INIT_CONFIG}} $EXTRA_ARGS 2>&1 | tee /var/log/k3s.log
}

start_ha_node() {
	info "Starting pod $POD_NAME in HA node setup"

	if [ ${POD_NAME: -1} == 0 ] && [ ! -d "{{.ETCD_DIR}}" ]; then
		info "Adding pod IP file."
		echo $POD_IP > /var/lib/rancher/k3s/k3k-node-ip

		/bin/k3s server --config {{.INIT_CONFIG}} $EXTRA_ARGS 2>&1 | tee /var/log/k3s.log
	else
		safe_mode {{.SERVER_CONFIG}}

		info "Adding pod IP file."
		echo $POD_IP > /var/lib/rancher/k3s/k3k-node-ip

		/bin/k3s server --config {{.SERVER_CONFIG}} $EXTRA_ARGS 2>&1 | tee /var/log/k3s.info 
	fi
}

# Configuring cgroups for k3s process in virtual mode
configure_cgroups() {
	runtime_class="{{.RUNTIME_CLASS}}"
	if [ "${runtime_class#kata}" != "$runtime_class" ]; then

		CGROUP_PATH=$(cat /proc/self/cgroup | cut -d: -f3)
		CGROUP_DIR="/sys/fs/cgroup${CGROUP_PATH}"

		# Move shell to init subcgroup to keep main cgroup clean for k3s children
		INIT_DIR="${CGROUP_DIR}init"
		mkdir -p "$INIT_DIR" 2>/dev/null

		PID=$(cut -d' ' -f4 /proc/self/stat)

		echo "$PID" > "$INIT_DIR/cgroup.procs"

		for controller in $(cat "$CGROUP_DIR/cgroup.controllers"); do
			echo "+$controller" > "$CGROUP_DIR/cgroup.subtree_control" 2>/dev/null || true
		done

		return
	fi

	# only configure the cgroups if the runtime used is the default and the mode is virtual
	# shared and hcp run agentless (no kubelet) and don't need cgroup overrides.
	if [ -n "$runtime_class" ] || [ "{{.K3K_MODE}}" != "virtual" ]; then	
		return
	fi

	root_cgroup_raw=$(cat /proc/self/cgroup)
	root_cgroup_stripped="${root_cgroup_raw#0::}"
	root_cgroup_parent=$(dirname "$root_cgroup_stripped")

	info "Current CGROUPS for $POD_NAME: ${root_cgroup_raw}"

	# overriding kubelet cgroup and the cgroup root for pods, this will prevent k3s
	# automatic placement see: https://github.com/k3s-io/k3s/blob/main/pkg/cgroups/cgroups_linux.go#L114-L127
	EXTRA_ARGS="$EXTRA_ARGS --kubelet-arg=kubelet-cgroups=$root_cgroup_parent/k3s --kubelet-arg=cgroup-root=$root_cgroup_parent"
}

EXTRA_ARGS="{{.EXTRA_ARGS}}"
configure_cgroups

case "{{.CLUSTER_MODE}}" in
    "ha")
        start_ha_node
        ;;
    "single"|*)
        start_single_node
        ;;
esac`