Initial support for kata runtimes (#814)

* update shell script to exec and remove pipe, add conditional kmsg mount for kata

update logging config

Apply suggestions from code review

Co-authored-by: Enrico Candino <enrico.candino@gmail.com>

pr comments, remove isKata, fix agent unit tests, update config

fix validation

fix template

revert log changes

update init script

remove emptydir mounts from kata

remove emptydir mounts from kata

fix tests

lint

* add quickstart docs

* centralise kata spotspec modifications

* update doc

* update docs

pkg/controller/cluster/agent/virtual.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"strings"
 
 	"gopkg.in/yaml.v3"
 	"k8s.io/utils/ptr"
@@ -324,5 +325,11 @@ func (v *VirtualAgent) podSpec(ctx context.Context, image, name string) corev1.P
 
 	podSpec.HostUsers = hostUsers
 
+	if podSpec.RuntimeClassName != nil && strings.HasPrefix(*podSpec.RuntimeClassName, "kata") {
+		mounts.AddKmsgMount(&podSpec)
+
+		mounts.FilterEmptyDirVolumes(&podSpec)
+	}
+
 	return podSpec
 }

pkg/controller/cluster/agent/virtual_test.go

@@ -125,6 +125,63 @@ func baseVirtualAgentPodSpec(v VirtualAgent) corev1.PodSpec {
 	}
 }
 
+func kataVirtualAgentPodSpec(v VirtualAgent) corev1.PodSpec {
+	return corev1.PodSpec{
+		Affinity:         nil,
+		NodeSelector:     v.cluster.Spec.NodeSelector,
+		RuntimeClassName: ptr.To("kata"),
+		Volumes: []corev1.Volume{
+			{
+				Name: "config",
+				VolumeSource: corev1.VolumeSource{
+					Secret: &corev1.SecretVolumeSource{
+						SecretName: configSecretName(v.cluster.Name),
+						Items: []corev1.KeyToPath{
+							{
+								Key:  "config.yaml",
+								Path: "config.yaml",
+							},
+						},
+					},
+				},
+			},
+			{
+				Name: "dev-kmsg",
+				VolumeSource: corev1.VolumeSource{
+					HostPath: &corev1.HostPathVolumeSource{
+						Path: "/dev/kmsg",
+					},
+				},
+			},
+		},
+		Containers: []corev1.Container{
+			{
+				Name:            "k3k-agent",
+				Image:           v.Image,
+				ImagePullPolicy: corev1.PullPolicy(v.ImagePullPolicy),
+				SecurityContext: &corev1.SecurityContext{
+					Privileged: ptr.To(true),
+				},
+				Args: []string{"agent", "--config", "/opt/rancher/k3s/config.yaml"},
+				Command: []string{
+					"/bin/k3s",
+				},
+				VolumeMounts: []corev1.VolumeMount{
+					{
+						Name:      "config",
+						MountPath: "/opt/rancher/k3s/",
+						ReadOnly:  false,
+					},
+					{
+						Name:      "dev-kmsg",
+						MountPath: "/dev/kmsg",
+					},
+				},
+			},
+		},
+	}
+}
+
 func Test_virtualAgentData(t *testing.T) {
 	type args struct {
 		serviceIP string
@@ -454,8 +511,7 @@ func Test_virtualAgentPodSpec(t *testing.T) {
 				Image: "rancher/k3k:latest",
 			},
 			expectedPodSpec: func(sa VirtualAgent) corev1.PodSpec {
-				spec := baseVirtualAgentPodSpec(sa)
-				spec.RuntimeClassName = ptr.To("kata")
+				spec := kataVirtualAgentPodSpec(sa)
 
 				return spec
 			},

pkg/controller/cluster/mounts/mounts.go

@@ -58,3 +58,48 @@ func buildSecretMountVolume(secretMount v1beta1.SecretMount) (corev1.Volume, cor
 
 	return vol, volMount
 }
+
+func FilterEmptyDirVolumes(podSpec *corev1.PodSpec) {
+	// Remove all EmptyDir volumes and their corresponding mounts.
+	emptyDirNames := make(map[string]bool)
+
+	var filteredVolumes []corev1.Volume
+
+	for _, vol := range podSpec.Volumes {
+		if vol.EmptyDir != nil {
+			emptyDirNames[vol.Name] = true
+		} else {
+			filteredVolumes = append(filteredVolumes, vol)
+		}
+	}
+
+	podSpec.Volumes = filteredVolumes
+
+	for i := range podSpec.Containers {
+		var filteredMounts []corev1.VolumeMount
+
+		for _, mount := range podSpec.Containers[i].VolumeMounts {
+			if !emptyDirNames[mount.Name] {
+				filteredMounts = append(filteredMounts, mount)
+			}
+		}
+
+		podSpec.Containers[i].VolumeMounts = filteredMounts
+	}
+}
+
+func AddKmsgMount(podSpec *corev1.PodSpec) {
+	podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{
+		Name: "dev-kmsg",
+		VolumeSource: corev1.VolumeSource{
+			HostPath: &corev1.HostPathVolumeSource{
+				Path: "/dev/kmsg",
+			},
+		},
+	})
+
+	podSpec.Containers[0].VolumeMounts = append(podSpec.Containers[0].VolumeMounts, corev1.VolumeMount{
+		Name:      "dev-kmsg",
+		MountPath: "/dev/kmsg",
+	})
+}

pkg/controller/cluster/server/server.go

@@ -304,6 +304,12 @@ func (s *Server) podSpec(ctx context.Context, image, name string, persistent boo
 		podSpec.ImagePullSecrets = append(podSpec.ImagePullSecrets, corev1.LocalObjectReference{Name: imagePullSecret})
 	}
 
+	if podSpec.RuntimeClassName != nil && strings.HasPrefix(*podSpec.RuntimeClassName, "kata") {
+		mounts.AddKmsgMount(&podSpec)
+
+		mounts.FilterEmptyDirVolumes(&podSpec)
+	}
+
 	return podSpec
 }
 

pkg/controller/cluster/server/template.go

@@ -94,8 +94,29 @@ start_ha_node() {
 
 # Configuring cgroups for k3s process in virtual mode
 configure_cgroups() {
+	runtime_class="{{.RUNTIME_CLASS}}"
+	if [ "${runtime_class#kata}" != "$runtime_class" ]; then
+
+		CGROUP_PATH=$(cat /proc/self/cgroup | cut -d: -f3)
+		CGROUP_DIR="/sys/fs/cgroup${CGROUP_PATH}"
+
+		# Move shell to init subcgroup to keep main cgroup clean for k3s children
+		INIT_DIR="${CGROUP_DIR}init"
+		mkdir -p "$INIT_DIR" 2>/dev/null
+
+		PID=$(cut -d' ' -f4 /proc/self/stat)
+
+		echo "$PID" > "$INIT_DIR/cgroup.procs"
+
+		for controller in $(cat "$CGROUP_DIR/cgroup.controllers"); do
+			echo "+$controller" > "$CGROUP_DIR/cgroup.subtree_control" 2>/dev/null || true
+		done
+
+		return
+	fi
+
 	# only configure the cgroups if the runtime used is the default and the mode is virtual
-	if [ -n "{{.RUNTIME_CLASS}}" ] || [ "{{.K3K_MODE}}" != "virtual" ]; then
+	if [ -n "$runtime_class" ] || [ "{{.K3K_MODE}}" != "virtual" ]; then
 		return
 	fi