From e948104f6b414153b7c677c47a1123f2a4ced852 Mon Sep 17 00:00:00 2001 From: Hussein Galal Date: Sat, 18 Apr 2026 00:55:20 +0200 Subject: [PATCH] [release/v1.0] Add WorkerLimit to shared cluster and add unit tests (#804) * Add WorkerLimit to shared cluster and add unit tests (#798) * Fix WorkerLimit to shared agents * Add unit tests for pod spec for both virtual and shared modes * Fix image registry for virtual mode --------- Signed-off-by: galal-hussein * lint fixes Signed-off-by: galal-hussein * lint fixes Signed-off-by: galal-hussein * Fix unit tests Signed-off-by: galal-hussein * Fix unit tests Signed-off-by: galal-hussein --------- Signed-off-by: galal-hussein --- pkg/controller/cluster/agent/shared.go | 10 +- pkg/controller/cluster/agent/shared_test.go | 312 +++++++++++++++++- pkg/controller/cluster/agent/virtual.go | 4 + pkg/controller/cluster/agent/virtual_test.go | 317 +++++++++++++++++++ 4 files changed, 636 insertions(+), 7 deletions(-) diff --git a/pkg/controller/cluster/agent/shared.go b/pkg/controller/cluster/agent/shared.go index ecd96fe..ee96277 100644 --- a/pkg/controller/cluster/agent/shared.go +++ b/pkg/controller/cluster/agent/shared.go @@ -213,9 +213,6 @@ func (s *SharedAgent) podSpec() v1.PodSpec { Name: s.Name(), Image: image, ImagePullPolicy: v1.PullPolicy(s.imagePullPolicy), - Resources: v1.ResourceRequirements{ - Limits: v1.ResourceList{}, - }, Env: append([]v1.EnvVar{ { Name: "AGENT_HOSTNAME", @@ -267,6 +264,13 @@ func (s *SharedAgent) podSpec() v1.PodSpec { podSpec.ImagePullSecrets = append(podSpec.ImagePullSecrets, v1.LocalObjectReference{Name: imagePullSecret}) } + // specify resource limits if specified for the agents. + if s.cluster.Spec.WorkerLimit != nil { + podSpec.Containers[0].Resources = v1.ResourceRequirements{ + Limits: s.cluster.Spec.WorkerLimit, + } + } + return podSpec } diff --git a/pkg/controller/cluster/agent/shared_test.go b/pkg/controller/cluster/agent/shared_test.go index b4d09d6..df661ae 100644 --- a/pkg/controller/cluster/agent/shared_test.go +++ b/pkg/controller/cluster/agent/shared_test.go @@ -5,12 +5,316 @@ import ( "github.com/stretchr/testify/assert" "gopkg.in/yaml.v2" + "k8s.io/apimachinery/pkg/api/resource" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" ) +func baseSharedAgentPodSpec(sharedAgent SharedAgent) corev1.PodSpec { + return corev1.PodSpec{ + HostNetwork: false, + DNSPolicy: corev1.DNSClusterFirst, + ServiceAccountName: sharedAgent.Name(), + NodeSelector: sharedAgent.cluster.Spec.NodeSelector, + Volumes: []corev1.Volume{ + { + Name: "config", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: configSecretName(sharedAgent.cluster.Name), + Items: []corev1.KeyToPath{ + { + Key: "config.yaml", + Path: "config.yaml", + }, + }, + }, + }, + }, + { + Name: "webhook-certs", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: WebhookSecretName(sharedAgent.cluster.Name), + Items: []corev1.KeyToPath{ + { + Key: "tls.crt", + Path: "tls.crt", + }, + { + Key: "tls.key", + Path: "tls.key", + }, + { + Key: "ca.crt", + Path: "ca.crt", + }, + }, + }, + }, + }, + }, + Containers: []corev1.Container{ + { + Name: sharedAgent.Name(), + Image: sharedAgent.image, + ImagePullPolicy: corev1.PullPolicy(sharedAgent.imagePullPolicy), + Env: []corev1.EnvVar{ + { + Name: "AGENT_HOSTNAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "spec.nodeName", + }, + }, + }, + { + Name: "POD_IP", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "status.podIP", + }, + }, + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "config", + MountPath: "/opt/rancher/k3k/", + ReadOnly: false, + }, + { + Name: "webhook-certs", + MountPath: "/opt/rancher/k3k-webhook", + ReadOnly: false, + }, + }, + Ports: []corev1.ContainerPort{ + { + Name: "kubelet-port", + Protocol: corev1.ProtocolTCP, + ContainerPort: int32(sharedAgent.kubeletPort), + }, + { + Name: "webhook-port", + Protocol: corev1.ProtocolTCP, + ContainerPort: int32(sharedAgent.webhookPort), + }, + }, + }, + }, + } +} + +func Test_sharedAgentPodSpec(t *testing.T) { + tests := []struct { + name string + sharedAgent SharedAgent + expectedPodSpec func(SharedAgent) corev1.PodSpec + }{ + { + name: "default shared cluster", + sharedAgent: SharedAgent{ + Config: &Config{ + cluster: &v1beta1.Cluster{ + TypeMeta: metav1.TypeMeta{ + Kind: "Cluster", + APIVersion: "k3k.io/v1beta", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "sc-default", + Namespace: "shared-test", + }, + Spec: v1beta1.ClusterSpec{}, + }, + }, + image: "rancher/k3k-kubelet:latest", + kubeletPort: 10250, + }, + expectedPodSpec: func(sa SharedAgent) corev1.PodSpec { + return baseSharedAgentPodSpec(sa) + }, + }, + { + name: "mirror host nodes enables host networking", + sharedAgent: SharedAgent{ + Config: &Config{ + cluster: &v1beta1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sc-mirror", + Namespace: "shared-test", + }, + Spec: v1beta1.ClusterSpec{ + MirrorHostNodes: true, + }, + }, + }, + image: "rancher/k3k-kubelet:latest", + kubeletPort: 10250, + }, + expectedPodSpec: func(sa SharedAgent) corev1.PodSpec { + spec := baseSharedAgentPodSpec(sa) + spec.HostNetwork = true + spec.DNSPolicy = corev1.DNSClusterFirstWithHostNet + + return spec + }, + }, + { + name: "image registry is prepended and image pull policy is applied", + sharedAgent: SharedAgent{ + Config: &Config{ + cluster: &v1beta1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sc-image", + Namespace: "shared-test", + }, + }, + }, + image: "rancher/k3k-kubelet:v1.2.3", + imageRegistry: "registry.example.com", + imagePullPolicy: "Always", + kubeletPort: 10250, + }, + expectedPodSpec: func(sa SharedAgent) corev1.PodSpec { + spec := baseSharedAgentPodSpec(sa) + spec.Containers[0].Image = "registry.example.com/rancher/k3k-kubelet:v1.2.3" + + return spec + }, + }, + { + name: "node selector from spec is set on pod", + sharedAgent: SharedAgent{ + Config: &Config{ + cluster: &v1beta1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sc-nodeselector", + Namespace: "shared-test", + }, + Spec: v1beta1.ClusterSpec{ + NodeSelector: map[string]string{ + "disktype": "ssd", + "topology.k8s.io/zone": "us-east-1a", + }, + }, + }, + }, + image: "rancher/k3k-kubelet:latest", + kubeletPort: 10250, + }, + expectedPodSpec: func(sa SharedAgent) corev1.PodSpec { + spec := baseSharedAgentPodSpec(sa) + spec.NodeSelector = map[string]string{ + "disktype": "ssd", + "topology.k8s.io/zone": "us-east-1a", + } + + return spec + }, + }, + { + name: "agent envs from spec are appended to default env", + sharedAgent: SharedAgent{ + Config: &Config{ + cluster: &v1beta1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sc-agentenvs", + Namespace: "shared-test", + }, + Spec: v1beta1.ClusterSpec{ + AgentEnvs: []corev1.EnvVar{ + {Name: "CUSTOM_VAR", Value: "custom-value"}, + {Name: "ANOTHER_VAR", Value: "another-value"}, + }, + }, + }, + }, + image: "rancher/k3k-kubelet:latest", + kubeletPort: 10250, + }, + expectedPodSpec: func(sa SharedAgent) corev1.PodSpec { + spec := baseSharedAgentPodSpec(sa) + spec.Containers[0].Env = append(spec.Containers[0].Env, + corev1.EnvVar{Name: "CUSTOM_VAR", Value: "custom-value"}, + corev1.EnvVar{Name: "ANOTHER_VAR", Value: "another-value"}, + ) + + return spec + }, + }, + { + name: "image pull secrets are set on pod", + sharedAgent: SharedAgent{ + Config: &Config{ + cluster: &v1beta1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sc-pullsecrets", + Namespace: "shared-test", + }, + }, + }, + image: "rancher/k3k-kubelet:latest", + kubeletPort: 10250, + imagePullSecrets: []string{"secret-1", "secret-2"}, + }, + expectedPodSpec: func(sa SharedAgent) corev1.PodSpec { + spec := baseSharedAgentPodSpec(sa) + spec.ImagePullSecrets = []corev1.LocalObjectReference{ + {Name: "secret-1"}, + {Name: "secret-2"}, + } + + return spec + }, + }, + { + name: "worker limit sets container resource limits", + sharedAgent: SharedAgent{ + Config: &Config{ + cluster: &v1beta1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sc-workerlimit", + Namespace: "shared-test", + }, + Spec: v1beta1.ClusterSpec{ + WorkerLimit: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("256Mi"), + }, + }, + }, + }, + image: "rancher/k3k-kubelet:latest", + kubeletPort: 10250, + }, + expectedPodSpec: func(sa SharedAgent) corev1.PodSpec { + spec := baseSharedAgentPodSpec(sa) + spec.Containers[0].Resources = corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("256Mi"), + }, + } + + return spec + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + podSpec := tt.sharedAgent.podSpec() + assert.Equal(t, tt.expectedPodSpec(tt.sharedAgent), podSpec) + }) + } +} + func Test_sharedAgentData(t *testing.T) { type args struct { cluster *v1beta1.Cluster @@ -30,7 +334,7 @@ func Test_sharedAgentData(t *testing.T) { name: "simple config", args: args{ cluster: &v1beta1.Cluster{ - ObjectMeta: v1.ObjectMeta{ + ObjectMeta: metav1.ObjectMeta{ Name: "mycluster", Namespace: "ns-1", }, @@ -60,7 +364,7 @@ func Test_sharedAgentData(t *testing.T) { name: "version in status", args: args{ cluster: &v1beta1.Cluster{ - ObjectMeta: v1.ObjectMeta{ + ObjectMeta: metav1.ObjectMeta{ Name: "mycluster", Namespace: "ns-1", }, @@ -93,7 +397,7 @@ func Test_sharedAgentData(t *testing.T) { name: "missing version in spec", args: args{ cluster: &v1beta1.Cluster{ - ObjectMeta: v1.ObjectMeta{ + ObjectMeta: metav1.ObjectMeta{ Name: "mycluster", Namespace: "ns-1", }, diff --git a/pkg/controller/cluster/agent/virtual.go b/pkg/controller/cluster/agent/virtual.go index 15a7304..5d0237a 100644 --- a/pkg/controller/cluster/agent/virtual.go +++ b/pkg/controller/cluster/agent/virtual.go @@ -137,6 +137,10 @@ func (v *VirtualAgent) deployment(ctx context.Context) error { func (v *VirtualAgent) podSpec(image, name string) v1.PodSpec { var limit v1.ResourceList + if v.ImageRegistry != "" { + image = v.ImageRegistry + "/" + image + } + args := v.cluster.Spec.AgentArgs args = append([]string{"agent", "--config", "/opt/rancher/k3s/config.yaml"}, args...) diff --git a/pkg/controller/cluster/agent/virtual_test.go b/pkg/controller/cluster/agent/virtual_test.go index 7052e7c..9d3679c 100644 --- a/pkg/controller/cluster/agent/virtual_test.go +++ b/pkg/controller/cluster/agent/virtual_test.go @@ -5,8 +5,125 @@ import ( "github.com/stretchr/testify/assert" "gopkg.in/yaml.v2" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/utils/ptr" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" ) +func baseVirtualAgentPodSpec(v VirtualAgent) corev1.PodSpec { + return corev1.PodSpec{ + Affinity: nil, + NodeSelector: v.cluster.Spec.NodeSelector, + Volumes: []corev1.Volume{ + { + Name: "config", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: configSecretName(v.cluster.Name), + Items: []corev1.KeyToPath{ + { + Key: "config.yaml", + Path: "config.yaml", + }, + }, + }, + }, + }, + { + Name: "run", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + { + Name: "varrun", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + { + Name: "varlibcni", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + { + Name: "varlog", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + { + Name: "varlibkubelet", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + { + Name: "varlibrancherk3s", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + }, + Containers: []corev1.Container{ + { + Name: "k3k-agent", + Image: v.Image, + ImagePullPolicy: corev1.PullPolicy(v.ImagePullPolicy), + SecurityContext: &corev1.SecurityContext{ + Privileged: ptr.To(true), + }, + Args: []string{"agent", "--config", "/opt/rancher/k3s/config.yaml"}, + Command: []string{ + "/bin/k3s", + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "config", + MountPath: "/opt/rancher/k3s/", + ReadOnly: false, + }, + { + Name: "run", + MountPath: "/run", + ReadOnly: false, + }, + { + Name: "varrun", + MountPath: "/var/run", + ReadOnly: false, + }, + { + Name: "varlibcni", + MountPath: "/var/lib/cni", + ReadOnly: false, + }, + { + Name: "varlibkubelet", + MountPath: "/var/lib/kubelet", + ReadOnly: false, + }, + { + Name: "varlibrancherk3s", + MountPath: "/var/lib/rancher/k3s", + ReadOnly: false, + }, + { + Name: "varlog", + MountPath: "/var/log", + ReadOnly: false, + }, + }, + }, + }, + } +} + func Test_virtualAgentData(t *testing.T) { type args struct { serviceIP string @@ -44,3 +161,203 @@ func Test_virtualAgentData(t *testing.T) { }) } } + +func Test_virtualAgentPodSpec(t *testing.T) { + tests := []struct { + name string + virtualAgent VirtualAgent + expectedPodSpec func(VirtualAgent) corev1.PodSpec + }{ + { + name: "default virtual mode cluster", + virtualAgent: VirtualAgent{ + Config: &Config{ + cluster: &v1beta1.Cluster{ + TypeMeta: metav1.TypeMeta{ + Kind: "Cluster", + APIVersion: "k3k.io/v1beta", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "vc-default", + Namespace: "virtual-test", + }, + Spec: v1beta1.ClusterSpec{}, + }, + }, + Image: "rancher/k3k:latest", + }, + expectedPodSpec: func(sa VirtualAgent) corev1.PodSpec { + return baseVirtualAgentPodSpec(sa) + }, + }, + { + name: "image registry is prepended and image pull policy is applied", + virtualAgent: VirtualAgent{ + Config: &Config{ + cluster: &v1beta1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vc-image", + Namespace: "virtual-test", + }, + }, + }, + Image: "rancher/k3k:v1.2.3", + ImageRegistry: "registry.example.com", + ImagePullPolicy: "Always", + }, + expectedPodSpec: func(sa VirtualAgent) corev1.PodSpec { + spec := baseVirtualAgentPodSpec(sa) + spec.Containers[0].Image = "registry.example.com/rancher/k3k:v1.2.3" + + return spec + }, + }, + { + name: "node selector from spec is set on pod", + virtualAgent: VirtualAgent{ + Config: &Config{ + cluster: &v1beta1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vc-nodeselector", + Namespace: "virtual-test", + }, + Spec: v1beta1.ClusterSpec{ + NodeSelector: map[string]string{ + "disktype": "ssd", + "topology.k8s.io/zone": "us-east-1a", + }, + }, + }, + }, + Image: "rancher/k3k:latest", + }, + expectedPodSpec: func(sa VirtualAgent) corev1.PodSpec { + spec := baseVirtualAgentPodSpec(sa) + spec.NodeSelector = map[string]string{ + "disktype": "ssd", + "topology.k8s.io/zone": "us-east-1a", + } + + return spec + }, + }, + { + name: "agent args from spec are appended to default args", + virtualAgent: VirtualAgent{ + Config: &Config{ + cluster: &v1beta1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vc-agentenvs", + Namespace: "virtual-test", + }, + Spec: v1beta1.ClusterSpec{ + AgentArgs: []string{ + "fake-arg-1=true", + "fake-arg-2=true", + }, + }, + }, + }, + Image: "rancher/k3k:latest", + }, + expectedPodSpec: func(va VirtualAgent) corev1.PodSpec { + spec := baseVirtualAgentPodSpec(va) + spec.Containers[0].Args = append(spec.Containers[0].Args, "fake-arg-1=true", "fake-arg-2=true") + + return spec + }, + }, + { + name: "agent envs from spec are appended to default env", + virtualAgent: VirtualAgent{ + Config: &Config{ + cluster: &v1beta1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vc-agentenvs", + Namespace: "virtual-test", + }, + Spec: v1beta1.ClusterSpec{ + AgentEnvs: []corev1.EnvVar{ + {Name: "CUSTOM_VAR", Value: "custom-value"}, + {Name: "ANOTHER_VAR", Value: "another-value"}, + }, + }, + }, + }, + Image: "rancher/k3k:latest", + }, + expectedPodSpec: func(sa VirtualAgent) corev1.PodSpec { + spec := baseVirtualAgentPodSpec(sa) + spec.Containers[0].Env = []corev1.EnvVar{ + {Name: "CUSTOM_VAR", Value: "custom-value"}, + {Name: "ANOTHER_VAR", Value: "another-value"}, + } + + return spec + }, + }, + { + name: "image pull secrets are set on pod", + virtualAgent: VirtualAgent{ + Config: &Config{ + cluster: &v1beta1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vc-pullsecrets", + Namespace: "virtual-test", + }, + }, + }, + Image: "rancher/k3k:latest", + + imagePullSecrets: []string{"secret-1", "secret-2"}, + }, + expectedPodSpec: func(sa VirtualAgent) corev1.PodSpec { + spec := baseVirtualAgentPodSpec(sa) + spec.ImagePullSecrets = []corev1.LocalObjectReference{ + {Name: "secret-1"}, + {Name: "secret-2"}, + } + + return spec + }, + }, + { + name: "worker limit sets container resource limits", + virtualAgent: VirtualAgent{ + Config: &Config{ + cluster: &v1beta1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vc-workerlimit", + Namespace: "virtual-test", + }, + Spec: v1beta1.ClusterSpec{ + WorkerLimit: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("256Mi"), + }, + }, + }, + }, + Image: "rancher/k3k:latest", + }, + expectedPodSpec: func(sa VirtualAgent) corev1.PodSpec { + spec := baseVirtualAgentPodSpec(sa) + spec.Containers[0].Resources = corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("256Mi"), + }, + } + + return spec + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + podSpec := tt.virtualAgent.podSpec(tt.virtualAgent.Image, "k3k-agent") + assert.Equal(t, tt.expectedPodSpec(tt.virtualAgent), podSpec) + }) + } +}