diff --git a/charts/k3k/templates/crds/k3k.io_clusters.yaml b/charts/k3k/templates/crds/k3k.io_clusters.yaml index ce0c966b..236c172b 100644 --- a/charts/k3k/templates/crds/k3k.io_clusters.yaml +++ b/charts/k3k/templates/crds/k3k.io_clusters.yaml @@ -638,6 +638,65 @@ spec: x-kubernetes-int-or-string: true description: ServerLimit specifies resource limits for server nodes. type: object + serverResources: + description: ServerResources specifies resources limits and requests for server nodes. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object servers: default: 1 description: |- @@ -807,6 +866,65 @@ spec: x-kubernetes-int-or-string: true description: WorkerLimit specifies resource limits for agent nodes. type: object + workerResources: + description: WorkerResources specifies resources limits and requests for worker nodes. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object type: object status: default: {} diff --git a/docs/crds/crds.adoc b/docs/crds/crds.adoc index c60d56d3..10c3c680 100644 --- a/docs/crds/crds.adoc +++ b/docs/crds/crds.adoc @@ -201,6 +201,8 @@ Example: ["--node-name=my-agent-node"] + | | | *`addons`* __xref:{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-addon[$$Addon$$] array__ | Addons specifies secrets containing raw YAML to deploy on cluster startup. + | | | *`serverLimit`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#resourcelist-v1-core[$$ResourceList$$]__ | ServerLimit specifies resource limits for server nodes. + | | | *`workerLimit`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#resourcelist-v1-core[$$ResourceList$$]__ | WorkerLimit specifies resource limits for agent nodes. + | | +| *`serverResources`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#resourcerequirements-v1-core[$$ResourceRequirements$$]__ | ServerResources specifies resources limits and requests for server nodes. + | | +| *`workerResources`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#resourcerequirements-v1-core[$$ResourceRequirements$$]__ | WorkerResources specifies resources limits and requests for worker nodes. + | | | *`mirrorHostNodes`* __boolean__ | MirrorHostNodes controls whether node objects from the host cluster + are mirrored into the virtual cluster. + | | | *`customCAs`* __xref:{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-customcas[$$CustomCAs$$]__ | CustomCAs specifies the cert/key pairs for custom CA certificates. + | | diff --git a/docs/crds/crds.md b/docs/crds/crds.md index a44a1ade..d2c33583 100644 --- a/docs/crds/crds.md +++ b/docs/crds/crds.md @@ -150,6 +150,8 @@ _Appears in:_ | `addons` _[Addon](#addon) array_ | Addons specifies secrets containing raw YAML to deploy on cluster startup. | | | | `serverLimit` _[ResourceList](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#resourcelist-v1-core)_ | ServerLimit specifies resource limits for server nodes. | | | | `workerLimit` _[ResourceList](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#resourcelist-v1-core)_ | WorkerLimit specifies resource limits for agent nodes. | | | +| `serverResources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#resourcerequirements-v1-core)_ | ServerResources specifies resources limits and requests for server nodes. | | | +| `workerResources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#resourcerequirements-v1-core)_ | WorkerResources specifies resources limits and requests for worker nodes. | | | | `mirrorHostNodes` _boolean_ | MirrorHostNodes controls whether node objects from the host cluster
are mirrored into the virtual cluster. | | | | `customCAs` _[CustomCAs](#customcas)_ | CustomCAs specifies the cert/key pairs for custom CA certificates. | | | | `sync` _[SyncConfig](#syncconfig)_ | Sync specifies the resources types that will be synced from virtual cluster to host cluster. | \{ \} | | diff --git a/pkg/apis/k3k.io/v1beta1/types.go b/pkg/apis/k3k.io/v1beta1/types.go index e2e019ec..42709375 100644 --- a/pkg/apis/k3k.io/v1beta1/types.go +++ b/pkg/apis/k3k.io/v1beta1/types.go @@ -169,6 +169,16 @@ type ClusterSpec struct { // +optional WorkerLimit corev1.ResourceList `json:"workerLimit,omitempty"` + // ServerResources specifies resources limits and requests for server nodes. + // + // +optional + ServerResources *corev1.ResourceRequirements `json:"serverResources,omitempty"` + + // WorkerResources specifies resources limits and requests for worker nodes. + // + // +optional + WorkerResources *corev1.ResourceRequirements `json:"workerResources,omitempty"` + // MirrorHostNodes controls whether node objects from the host cluster // are mirrored into the virtual cluster. // diff --git a/pkg/apis/k3k.io/v1beta1/zz_generated.deepcopy.go b/pkg/apis/k3k.io/v1beta1/zz_generated.deepcopy.go index b4b2d14a..7392d17f 100644 --- a/pkg/apis/k3k.io/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/k3k.io/v1beta1/zz_generated.deepcopy.go @@ -190,6 +190,16 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) { (*out)[key] = val.DeepCopy() } } + if in.ServerResources != nil { + in, out := &in.ServerResources, &out.ServerResources + *out = new(v1.ResourceRequirements) + (*in).DeepCopyInto(*out) + } + if in.WorkerResources != nil { + in, out := &in.WorkerResources, &out.WorkerResources + *out = new(v1.ResourceRequirements) + (*in).DeepCopyInto(*out) + } if in.CustomCAs != nil { in, out := &in.CustomCAs, &out.CustomCAs *out = new(CustomCAs) diff --git a/pkg/controller/cluster/agent/shared.go b/pkg/controller/cluster/agent/shared.go index ee962773..8e15818a 100644 --- a/pkg/controller/cluster/agent/shared.go +++ b/pkg/controller/cluster/agent/shared.go @@ -271,6 +271,13 @@ func (s *SharedAgent) podSpec() v1.PodSpec { } } + // specifying WorkerResources will take precedence over WorkerLimits + if s.cluster.Spec.WorkerResources != nil { + // removing container previous limit + podSpec.Containers[0].Resources = v1.ResourceRequirements{} + podSpec.Resources = s.cluster.Spec.WorkerResources + } + return podSpec } diff --git a/pkg/controller/cluster/agent/shared_test.go b/pkg/controller/cluster/agent/shared_test.go index df661aeb..9dab89b6 100644 --- a/pkg/controller/cluster/agent/shared_test.go +++ b/pkg/controller/cluster/agent/shared_test.go @@ -302,6 +302,95 @@ func Test_sharedAgentPodSpec(t *testing.T) { }, } + return spec + }, + }, + { + name: "worker resources sets pods resource limits/requests", + sharedAgent: SharedAgent{ + Config: &Config{ + cluster: &v1beta1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sc-workerResources", + Namespace: "shared-test", + }, + Spec: v1beta1.ClusterSpec{ + WorkerResources: &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("200m"), + corev1.ResourceMemory: resource.MustParse("256Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + }, + }, + }, + }, + image: "rancher/k3k-kubelet:latest", + kubeletPort: 10250, + }, + expectedPodSpec: func(sa SharedAgent) corev1.PodSpec { + spec := baseSharedAgentPodSpec(sa) + spec.Resources = &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("200m"), + corev1.ResourceMemory: resource.MustParse("256Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + } + + return spec + }, + }, + { + name: "worker resources takes precedence over WorkerLimit", + sharedAgent: SharedAgent{ + Config: &Config{ + cluster: &v1beta1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sc-workerResources", + Namespace: "shared-test", + }, + Spec: v1beta1.ClusterSpec{ + WorkerLimit: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("128Mi"), + }, + WorkerResources: &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("200m"), + corev1.ResourceMemory: resource.MustParse("256Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + }, + }, + }, + }, + image: "rancher/k3k-kubelet:latest", + kubeletPort: 10250, + }, + expectedPodSpec: func(sa SharedAgent) corev1.PodSpec { + spec := baseSharedAgentPodSpec(sa) + spec.Resources = &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("200m"), + corev1.ResourceMemory: resource.MustParse("256Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + } + spec.Containers[0].Resources = corev1.ResourceRequirements{} + return spec }, }, diff --git a/pkg/controller/cluster/agent/virtual.go b/pkg/controller/cluster/agent/virtual.go index 5d0237af..c21a0b3b 100644 --- a/pkg/controller/cluster/agent/virtual.go +++ b/pkg/controller/cluster/agent/virtual.go @@ -262,6 +262,13 @@ func (v *VirtualAgent) podSpec(image, name string) v1.PodSpec { } } + // specifying WorkerResources will take precedence over WorkerLimits + if v.cluster.Spec.WorkerResources != nil { + // removing container previous limit + podSpec.Containers[0].Resources = v1.ResourceRequirements{} + podSpec.Resources = v.cluster.Spec.WorkerResources + } + for _, imagePullSecret := range v.imagePullSecrets { podSpec.ImagePullSecrets = append(podSpec.ImagePullSecrets, v1.LocalObjectReference{Name: imagePullSecret}) } diff --git a/pkg/controller/cluster/agent/virtual_test.go b/pkg/controller/cluster/agent/virtual_test.go index 9d3679c4..de13cd43 100644 --- a/pkg/controller/cluster/agent/virtual_test.go +++ b/pkg/controller/cluster/agent/virtual_test.go @@ -349,6 +349,93 @@ func Test_virtualAgentPodSpec(t *testing.T) { }, } + return spec + }, + }, + { + name: "worker resources sets pods resource limits/requests", + virtualAgent: VirtualAgent{ + Config: &Config{ + cluster: &v1beta1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sc-workerResources", + Namespace: "shared-test", + }, + Spec: v1beta1.ClusterSpec{ + WorkerResources: &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("200m"), + corev1.ResourceMemory: resource.MustParse("256Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + }, + }, + }, + }, + Image: "rancher/k3k:latest", + }, + expectedPodSpec: func(va VirtualAgent) corev1.PodSpec { + spec := baseVirtualAgentPodSpec(va) + spec.Resources = &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("200m"), + corev1.ResourceMemory: resource.MustParse("256Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + } + + return spec + }, + }, + { + name: "worker resources takes precedence over WorkerLimit", + virtualAgent: VirtualAgent{ + Config: &Config{ + cluster: &v1beta1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sc-workerResources", + Namespace: "shared-test", + }, + Spec: v1beta1.ClusterSpec{ + WorkerLimit: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("128Mi"), + }, + WorkerResources: &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("200m"), + corev1.ResourceMemory: resource.MustParse("256Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + }, + }, + }, + }, + Image: "rancher/k3k:latest", + }, + expectedPodSpec: func(va VirtualAgent) corev1.PodSpec { + spec := baseVirtualAgentPodSpec(va) + spec.Resources = &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("200m"), + corev1.ResourceMemory: resource.MustParse("256Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + } + spec.Containers[0].Resources = corev1.ResourceRequirements{} + return spec }, }, diff --git a/pkg/controller/cluster/server/server.go b/pkg/controller/cluster/server/server.go index 3d47707e..04432b03 100644 --- a/pkg/controller/cluster/server/server.go +++ b/pkg/controller/cluster/server/server.go @@ -242,6 +242,13 @@ func (s *Server) podSpec(image, name string, persistent bool, startupCmd string) } } + // specifying ServerResources will take precedence over ServerLimits + if s.cluster.Spec.ServerResources != nil { + // removing container previous limit + podSpec.Containers[0].Resources = v1.ResourceRequirements{} + podSpec.Resources = s.cluster.Spec.ServerResources + } + podSpec.Containers[0].Env = append(podSpec.Containers[0].Env, s.cluster.Spec.ServerEnvs...) // add image pull secrets