mirror of
https://github.com/rancher/k3k.git
synced 2026-08-19 12:26:17 +00:00
Initial support for kata runtimes (#814)
* update shell script to exec and remove pipe, add conditional kmsg mount for kata update logging config Apply suggestions from code review Co-authored-by: Enrico Candino <enrico.candino@gmail.com> pr comments, remove isKata, fix agent unit tests, update config fix validation fix template revert log changes update init script remove emptydir mounts from kata remove emptydir mounts from kata fix tests lint * add quickstart docs * centralise kata spotspec modifications * update doc * update docs
This commit is contained in:
bcraft
Enrico Candino
@@ -0,0 +1,230 @@
|
||||
# Using Kata Containers
|
||||
|
||||
> **Experimental:** Kata Containers support is in early development. Expect rough edges.
|
||||
|
||||
Kata Containers runs each pod inside a lightweight QEMU VM, providing stronger isolation than runc. When used with k3k, this means each virtual cluster's server and agent pods run as hardware-virtualized guests on the host node.
|
||||
|
||||
This guide covers the full setup: installing Kata, configuring the QEMU runtime and devmapper snapshotter, preparing a custom K3s image, and deploying a cluster.
|
||||
|
||||
## Requirements
|
||||
|
||||
- KVM available on the host node (`/dev/kvm` accessible)
|
||||
- `vhost_net` and `vhost_vsock` kernel modules available
|
||||
- `dmsetup` and `losetup` available (device-mapper utilities)
|
||||
- K3s or RKE2 installed on the host
|
||||
|
||||
## 1. Install Kata Containers
|
||||
|
||||
Download and extract the Kata static binary bundle, then symlink the containerd shim:
|
||||
|
||||
```bash
|
||||
curl -sSL https://github.com/kata-containers/kata-containers/releases/download/3.31.0/kata-static-3.31.0-amd64.tar.zst \
|
||||
| sudo tar --zstd -xvf - -C /
|
||||
|
||||
ln -s /opt/kata/runtime-rs/bin/containerd-shim-kata-v2 /usr/local/bin/containerd-shim-kata-v2
|
||||
```
|
||||
|
||||
## 2. Configure the QEMU Runtime
|
||||
|
||||
The default Kata QEMU config requires a few changes. The key ones switch the rootfs and block device transports from `virtio-pmem`/`virtio-scsi` to `virtio-blk-pci`, which is necessary because the devmapper snapshotter provides block devices rather than overlay filesystems. `shared_fs` is also disabled since `virtio-fs` is not needed when using block-backed storage.
|
||||
|
||||
```bash
|
||||
sed -i \
|
||||
-e 's/vm_rootfs_driver = "virtio-pmem"/vm_rootfs_driver = "virtio-blk-pci"/' \
|
||||
-e 's/shared_fs = "virtio-fs"/shared_fs = "none"/' \
|
||||
-e 's/block_device_driver = "virtio-scsi"/block_device_driver = "virtio-blk-pci"/' \
|
||||
-e 's/disable_image_nvdimm = false/disable_image_nvdimm = true/' \
|
||||
/opt/kata/share/defaults/kata-containers/runtime-rs/configuration-qemu-runtime-rs.toml
|
||||
```
|
||||
|
||||
## 3. Set Up the devmapper Snapshotter
|
||||
|
||||
The default snapshotter will use virtio-fs as the container filesystem, the default overlayfs snapshotter does not work with this type, so devmapper is required in order to provide a block device.
|
||||
|
||||
The script below creates a thin-pool backed by loopback devices. This is adequate for development and testing. The [device-mapper thin provisioning documentation](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/thin-provisioning.html) covers the full lifecycle, which is more involved than the example here. `erofs` may be a suitable replacement in future to eliminate the need to manage this pool entirely.
|
||||
|
||||
```bash
|
||||
#!/bin/bash
|
||||
|
||||
DIR=/opt/devmapper
|
||||
CONTAINERD_SOCK="/run/k3s/containerd/containerd.sock"
|
||||
|
||||
set -euxo pipefail
|
||||
|
||||
name="k3s"
|
||||
|
||||
if [ "$EUID" -ne 0 ]; then
|
||||
echo "Please run as root."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
create_loopback_device() {
|
||||
local path=$1
|
||||
local size=$2
|
||||
|
||||
if [[ ! -f "$path" ]]; then
|
||||
touch "$path"
|
||||
truncate -s "$size" "$path"
|
||||
fi
|
||||
|
||||
local dev=$(losetup --output NAME --noheadings --associated "$path")
|
||||
if [[ -z "$dev" ]]; then
|
||||
dev=$(losetup --find --show $path)
|
||||
fi
|
||||
echo $dev
|
||||
}
|
||||
|
||||
pool_create() {
|
||||
mkdir -p $DIR
|
||||
|
||||
local datadev=$(create_loopback_device "$DIR/data" '10G')
|
||||
local metadev=$(create_loopback_device "$DIR/metadata" '1G')
|
||||
|
||||
if dmsetup info "$name" &>/dev/null; then
|
||||
echo "Thin pool '$name' already exists, ensuring it is active."
|
||||
dmsetup resume "$name" 2>/dev/null || true
|
||||
return
|
||||
fi
|
||||
|
||||
local sectorsize=512
|
||||
local datasize="$(blockdev --getsize64 -q ${datadev})"
|
||||
local length_sectors=$(bc <<< "${datasize}/${sectorsize}")
|
||||
local thinp_table="0 ${length_sectors} thin-pool ${metadev} ${datadev} 128 32768 1 skip_block_zeroing"
|
||||
dmsetup create "$name" --table "${thinp_table}"
|
||||
}
|
||||
|
||||
pool_create
|
||||
```
|
||||
|
||||
The pool does not survive a reboot. Consider wrapping `pool_create` in a systemd service that runs before `k3s.service`.
|
||||
|
||||
## 4. Configure K3s containerd
|
||||
|
||||
Place the following template at `/var/lib/rancher/k3s/agent/etc/containerd/config-v3.toml.d/custom.toml`. It registers the `kata-qemu` runtime handler and configures the devmapper snapshotter.
|
||||
|
||||
```toml
|
||||
version = 3
|
||||
|
||||
[plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.kata-qemu]
|
||||
runtime_type = "io.containerd.kata.v2"
|
||||
snapshotter = "devmapper"
|
||||
privileged_without_host_devices = true
|
||||
pod_annotations = ["io.katacontainers.*"]
|
||||
|
||||
[plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.kata-qemu.options]
|
||||
ConfigPath = "/opt/kata/share/defaults/kata-containers/runtime-rs/configuration-qemu-runtime-rs.toml"
|
||||
|
||||
[plugins.'io.containerd.snapshotter.v1.devmapper']
|
||||
pool_name = "k3s"
|
||||
root_path = "/opt/devmapper"
|
||||
base_image_size = "4GB"
|
||||
discard_blocks = true
|
||||
```
|
||||
## 5. Fixing emptyDir mounts
|
||||
|
||||
The standard K3s image declares `emptyDir` volumes in its image manifest. Kata detects these as tmpfs mounts and fails to start the container. The k3k controller automatically strips `emptyDir` volumes from the pod spec at runtime when the cluster's `runtimeClassName` starts with `kata`, but the declarations baked into the image manifest will still be in place by default.
|
||||
|
||||
We can avoid emptyDir mounts from image manifests by either of the following options:
|
||||
|
||||
### 5.1 Ignore the image manifest volume declaration (recommended)
|
||||
|
||||
Configuring containerd to ignore the image manifest volumes, this is a simpler option but is a global setting and will apply to all runtimes on a particular node.
|
||||
|
||||
Place the below at `/var/lib/rancher/k3s/agent/etc/containerd/config-v3.toml.d/volumes.toml`
|
||||
```toml
|
||||
version = 3
|
||||
|
||||
[plugins.'io.containerd.cri.v1.runtime']
|
||||
ignore_image_defined_volumes = true
|
||||
```
|
||||
|
||||
### 5.2 Build a custom K3S image
|
||||
|
||||
Build a custom image that omits those declarations:
|
||||
|
||||
```dockerfile
|
||||
FROM rancher/k3s:v1.35.3-k3s1 AS rancher
|
||||
|
||||
FROM scratch
|
||||
|
||||
COPY --from=rancher / /
|
||||
|
||||
ENV PATH=/var/lib/rancher/k3s/data/cni:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/bin/aux
|
||||
ENV CRI_CONFIG_FILE=/var/lib/rancher/k3s/agent/etc/crictl.yaml
|
||||
```
|
||||
|
||||
Push this image to a registry accessible from the host cluster and configure the controller to use it for Kata-based clusters.
|
||||
|
||||
An alternative would be an OCI hook to strip the mounts at runtime, but that falls outside the scope of using the Kata project's supplied build artifacts.
|
||||
|
||||
|
||||
## 6. Load Kernel Modules
|
||||
|
||||
`vhost_net` and `vhost_vsock` are required for VM networking and host-to-guest communication over vsock:
|
||||
|
||||
```bash
|
||||
modprobe vhost_net
|
||||
modprobe vhost_vsock
|
||||
```
|
||||
|
||||
To persist these across reboots, add them to `/etc/modules-load.d/kata.conf`.
|
||||
|
||||
## 7. Create the RuntimeClass
|
||||
|
||||
The `RuntimeClass` name **must start with `kata-`**. This prefix is how the k3k controller identifies Kata-based clusters and applies the necessary pod spec adjustments: stripping `emptyDir` volumes, injecting a `/dev/kmsg` host path mount, and configuring cgroup handling in the K3s startup script.
|
||||
|
||||
```yaml
|
||||
apiVersion: node.k8s.io/v1
|
||||
kind: RuntimeClass
|
||||
handler: kata-qemu
|
||||
metadata:
|
||||
name: kata-qemu
|
||||
```
|
||||
|
||||
## 8. Create a Cluster
|
||||
|
||||
Kata-based clusters require `mode: virtual` and `persistence.type: ephemeral`:
|
||||
|
||||
- **Virtual mode** is required because Kata needs a full K3s agent with its own container runtime running on the host. Shared mode uses a virtual kubelet that bypasses node-level runtimes entirely.
|
||||
- **Ephemeral persistence** is required because dynamic persistence creates a PVC that conflicts with how Kata manages block device storage inside the guest VM.
|
||||
|
||||
The example below uses an external PostgreSQL datastore via `secretMounts` to provide persistence.
|
||||
|
||||
```yaml
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: test-k3k
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: datastore-config
|
||||
namespace: test-k3k
|
||||
type: Opaque
|
||||
stringData:
|
||||
config.yaml: |
|
||||
datastore-endpoint: postgres://username:password@host:5432/dbname
|
||||
cluster-init: false
|
||||
server: ""
|
||||
---
|
||||
apiVersion: k3k.io/v1beta1
|
||||
kind: Cluster
|
||||
metadata:
|
||||
name: test
|
||||
namespace: test-k3k
|
||||
spec:
|
||||
mode: virtual
|
||||
runtimeClassName: kata-qemu
|
||||
persistence:
|
||||
type: ephemeral
|
||||
servers: 3
|
||||
secretMounts:
|
||||
- name: externaldb-init-config
|
||||
secretName: datastore-config
|
||||
mountPath: /opt/rancher/k3s/init/config.yaml.d/
|
||||
role: server
|
||||
- name: externaldb-server-config
|
||||
secretName: datastore-config
|
||||
mountPath: /opt/rancher/k3s/server/config.yaml.d/
|
||||
```
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
"k8s.io/utils/ptr"
|
||||
@@ -324,5 +325,11 @@ func (v *VirtualAgent) podSpec(ctx context.Context, image, name string) corev1.P
|
||||
|
||||
podSpec.HostUsers = hostUsers
|
||||
|
||||
if podSpec.RuntimeClassName != nil && strings.HasPrefix(*podSpec.RuntimeClassName, "kata") {
|
||||
mounts.AddKmsgMount(&podSpec)
|
||||
|
||||
mounts.FilterEmptyDirVolumes(&podSpec)
|
||||
}
|
||||
|
||||
return podSpec
|
||||
}
|
||||
|
||||
@@ -125,6 +125,63 @@ func baseVirtualAgentPodSpec(v VirtualAgent) corev1.PodSpec {
|
||||
}
|
||||
}
|
||||
|
||||
func kataVirtualAgentPodSpec(v VirtualAgent) corev1.PodSpec {
|
||||
return corev1.PodSpec{
|
||||
Affinity: nil,
|
||||
NodeSelector: v.cluster.Spec.NodeSelector,
|
||||
RuntimeClassName: ptr.To("kata"),
|
||||
Volumes: []corev1.Volume{
|
||||
{
|
||||
Name: "config",
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
Secret: &corev1.SecretVolumeSource{
|
||||
SecretName: configSecretName(v.cluster.Name),
|
||||
Items: []corev1.KeyToPath{
|
||||
{
|
||||
Key: "config.yaml",
|
||||
Path: "config.yaml",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "dev-kmsg",
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
HostPath: &corev1.HostPathVolumeSource{
|
||||
Path: "/dev/kmsg",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
Containers: []corev1.Container{
|
||||
{
|
||||
Name: "k3k-agent",
|
||||
Image: v.Image,
|
||||
ImagePullPolicy: corev1.PullPolicy(v.ImagePullPolicy),
|
||||
SecurityContext: &corev1.SecurityContext{
|
||||
Privileged: ptr.To(true),
|
||||
},
|
||||
Args: []string{"agent", "--config", "/opt/rancher/k3s/config.yaml"},
|
||||
Command: []string{
|
||||
"/bin/k3s",
|
||||
},
|
||||
VolumeMounts: []corev1.VolumeMount{
|
||||
{
|
||||
Name: "config",
|
||||
MountPath: "/opt/rancher/k3s/",
|
||||
ReadOnly: false,
|
||||
},
|
||||
{
|
||||
Name: "dev-kmsg",
|
||||
MountPath: "/dev/kmsg",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func Test_virtualAgentData(t *testing.T) {
|
||||
type args struct {
|
||||
serviceIP string
|
||||
@@ -454,8 +511,7 @@ func Test_virtualAgentPodSpec(t *testing.T) {
|
||||
Image: "rancher/k3k:latest",
|
||||
},
|
||||
expectedPodSpec: func(sa VirtualAgent) corev1.PodSpec {
|
||||
spec := baseVirtualAgentPodSpec(sa)
|
||||
spec.RuntimeClassName = ptr.To("kata")
|
||||
spec := kataVirtualAgentPodSpec(sa)
|
||||
|
||||
return spec
|
||||
},
|
||||
|
||||
@@ -58,3 +58,48 @@ func buildSecretMountVolume(secretMount v1beta1.SecretMount) (corev1.Volume, cor
|
||||
|
||||
return vol, volMount
|
||||
}
|
||||
|
||||
func FilterEmptyDirVolumes(podSpec *corev1.PodSpec) {
|
||||
// Remove all EmptyDir volumes and their corresponding mounts.
|
||||
emptyDirNames := make(map[string]bool)
|
||||
|
||||
var filteredVolumes []corev1.Volume
|
||||
|
||||
for _, vol := range podSpec.Volumes {
|
||||
if vol.EmptyDir != nil {
|
||||
emptyDirNames[vol.Name] = true
|
||||
} else {
|
||||
filteredVolumes = append(filteredVolumes, vol)
|
||||
}
|
||||
}
|
||||
|
||||
podSpec.Volumes = filteredVolumes
|
||||
|
||||
for i := range podSpec.Containers {
|
||||
var filteredMounts []corev1.VolumeMount
|
||||
|
||||
for _, mount := range podSpec.Containers[i].VolumeMounts {
|
||||
if !emptyDirNames[mount.Name] {
|
||||
filteredMounts = append(filteredMounts, mount)
|
||||
}
|
||||
}
|
||||
|
||||
podSpec.Containers[i].VolumeMounts = filteredMounts
|
||||
}
|
||||
}
|
||||
|
||||
func AddKmsgMount(podSpec *corev1.PodSpec) {
|
||||
podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{
|
||||
Name: "dev-kmsg",
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
HostPath: &corev1.HostPathVolumeSource{
|
||||
Path: "/dev/kmsg",
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
podSpec.Containers[0].VolumeMounts = append(podSpec.Containers[0].VolumeMounts, corev1.VolumeMount{
|
||||
Name: "dev-kmsg",
|
||||
MountPath: "/dev/kmsg",
|
||||
})
|
||||
}
|
||||
|
||||
@@ -304,6 +304,12 @@ func (s *Server) podSpec(ctx context.Context, image, name string, persistent boo
|
||||
podSpec.ImagePullSecrets = append(podSpec.ImagePullSecrets, corev1.LocalObjectReference{Name: imagePullSecret})
|
||||
}
|
||||
|
||||
if podSpec.RuntimeClassName != nil && strings.HasPrefix(*podSpec.RuntimeClassName, "kata") {
|
||||
mounts.AddKmsgMount(&podSpec)
|
||||
|
||||
mounts.FilterEmptyDirVolumes(&podSpec)
|
||||
}
|
||||
|
||||
return podSpec
|
||||
}
|
||||
|
||||
|
||||
@@ -94,8 +94,29 @@ start_ha_node() {
|
||||
|
||||
# Configuring cgroups for k3s process in virtual mode
|
||||
configure_cgroups() {
|
||||
runtime_class="{{.RUNTIME_CLASS}}"
|
||||
if [ "${runtime_class#kata}" != "$runtime_class" ]; then
|
||||
|
||||
CGROUP_PATH=$(cat /proc/self/cgroup | cut -d: -f3)
|
||||
CGROUP_DIR="/sys/fs/cgroup${CGROUP_PATH}"
|
||||
|
||||
# Move shell to init subcgroup to keep main cgroup clean for k3s children
|
||||
INIT_DIR="${CGROUP_DIR}init"
|
||||
mkdir -p "$INIT_DIR" 2>/dev/null
|
||||
|
||||
PID=$(cut -d' ' -f4 /proc/self/stat)
|
||||
|
||||
echo "$PID" > "$INIT_DIR/cgroup.procs"
|
||||
|
||||
for controller in $(cat "$CGROUP_DIR/cgroup.controllers"); do
|
||||
echo "+$controller" > "$CGROUP_DIR/cgroup.subtree_control" 2>/dev/null || true
|
||||
done
|
||||
|
||||
return
|
||||
fi
|
||||
|
||||
# only configure the cgroups if the runtime used is the default and the mode is virtual
|
||||
if [ -n "{{.RUNTIME_CLASS}}" ] || [ "{{.K3K_MODE}}" != "virtual" ]; then
|
||||
if [ -n "$runtime_class" ] || [ "{{.K3K_MODE}}" != "virtual" ]; then
|
||||
return
|
||||
fi
|
||||
|
||||
|
||||
Reference in New Issue
Block a user